xref: /openbmc/linux/fs/btrfs/inode.c (revision e2f1cf25)
1 /*
2  * Copyright (C) 2007 Oracle.  All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public
6  * License v2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11  * General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public
14  * License along with this program; if not, write to the
15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16  * Boston, MA 021110-1307, USA.
17  */
18 
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/buffer_head.h>
22 #include <linux/file.h>
23 #include <linux/fs.h>
24 #include <linux/pagemap.h>
25 #include <linux/highmem.h>
26 #include <linux/time.h>
27 #include <linux/init.h>
28 #include <linux/string.h>
29 #include <linux/backing-dev.h>
30 #include <linux/mpage.h>
31 #include <linux/swap.h>
32 #include <linux/writeback.h>
33 #include <linux/statfs.h>
34 #include <linux/compat.h>
35 #include <linux/bit_spinlock.h>
36 #include <linux/xattr.h>
37 #include <linux/posix_acl.h>
38 #include <linux/falloc.h>
39 #include <linux/slab.h>
40 #include <linux/ratelimit.h>
41 #include <linux/mount.h>
42 #include <linux/btrfs.h>
43 #include <linux/blkdev.h>
44 #include <linux/posix_acl_xattr.h>
45 #include <linux/uio.h>
46 #include "ctree.h"
47 #include "disk-io.h"
48 #include "transaction.h"
49 #include "btrfs_inode.h"
50 #include "print-tree.h"
51 #include "ordered-data.h"
52 #include "xattr.h"
53 #include "tree-log.h"
54 #include "volumes.h"
55 #include "compression.h"
56 #include "locking.h"
57 #include "free-space-cache.h"
58 #include "inode-map.h"
59 #include "backref.h"
60 #include "hash.h"
61 #include "props.h"
62 #include "qgroup.h"
63 
64 struct btrfs_iget_args {
65 	struct btrfs_key *location;
66 	struct btrfs_root *root;
67 };
68 
69 static const struct inode_operations btrfs_dir_inode_operations;
70 static const struct inode_operations btrfs_symlink_inode_operations;
71 static const struct inode_operations btrfs_dir_ro_inode_operations;
72 static const struct inode_operations btrfs_special_inode_operations;
73 static const struct inode_operations btrfs_file_inode_operations;
74 static const struct address_space_operations btrfs_aops;
75 static const struct address_space_operations btrfs_symlink_aops;
76 static const struct file_operations btrfs_dir_file_operations;
77 static struct extent_io_ops btrfs_extent_io_ops;
78 
79 static struct kmem_cache *btrfs_inode_cachep;
80 static struct kmem_cache *btrfs_delalloc_work_cachep;
81 struct kmem_cache *btrfs_trans_handle_cachep;
82 struct kmem_cache *btrfs_transaction_cachep;
83 struct kmem_cache *btrfs_path_cachep;
84 struct kmem_cache *btrfs_free_space_cachep;
85 
86 #define S_SHIFT 12
87 static unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
88 	[S_IFREG >> S_SHIFT]	= BTRFS_FT_REG_FILE,
89 	[S_IFDIR >> S_SHIFT]	= BTRFS_FT_DIR,
90 	[S_IFCHR >> S_SHIFT]	= BTRFS_FT_CHRDEV,
91 	[S_IFBLK >> S_SHIFT]	= BTRFS_FT_BLKDEV,
92 	[S_IFIFO >> S_SHIFT]	= BTRFS_FT_FIFO,
93 	[S_IFSOCK >> S_SHIFT]	= BTRFS_FT_SOCK,
94 	[S_IFLNK >> S_SHIFT]	= BTRFS_FT_SYMLINK,
95 };
96 
97 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
98 static int btrfs_truncate(struct inode *inode);
99 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
100 static noinline int cow_file_range(struct inode *inode,
101 				   struct page *locked_page,
102 				   u64 start, u64 end, int *page_started,
103 				   unsigned long *nr_written, int unlock);
104 static struct extent_map *create_pinned_em(struct inode *inode, u64 start,
105 					   u64 len, u64 orig_start,
106 					   u64 block_start, u64 block_len,
107 					   u64 orig_block_len, u64 ram_bytes,
108 					   int type);
109 
110 static int btrfs_dirty_inode(struct inode *inode);
111 
112 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
113 void btrfs_test_inode_set_ops(struct inode *inode)
114 {
115 	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
116 }
117 #endif
118 
119 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
120 				     struct inode *inode,  struct inode *dir,
121 				     const struct qstr *qstr)
122 {
123 	int err;
124 
125 	err = btrfs_init_acl(trans, inode, dir);
126 	if (!err)
127 		err = btrfs_xattr_security_init(trans, inode, dir, qstr);
128 	return err;
129 }
130 
131 /*
132  * this does all the hard work for inserting an inline extent into
133  * the btree.  The caller should have done a btrfs_drop_extents so that
134  * no overlapping inline items exist in the btree
135  */
136 static int insert_inline_extent(struct btrfs_trans_handle *trans,
137 				struct btrfs_path *path, int extent_inserted,
138 				struct btrfs_root *root, struct inode *inode,
139 				u64 start, size_t size, size_t compressed_size,
140 				int compress_type,
141 				struct page **compressed_pages)
142 {
143 	struct extent_buffer *leaf;
144 	struct page *page = NULL;
145 	char *kaddr;
146 	unsigned long ptr;
147 	struct btrfs_file_extent_item *ei;
148 	int err = 0;
149 	int ret;
150 	size_t cur_size = size;
151 	unsigned long offset;
152 
153 	if (compressed_size && compressed_pages)
154 		cur_size = compressed_size;
155 
156 	inode_add_bytes(inode, size);
157 
158 	if (!extent_inserted) {
159 		struct btrfs_key key;
160 		size_t datasize;
161 
162 		key.objectid = btrfs_ino(inode);
163 		key.offset = start;
164 		key.type = BTRFS_EXTENT_DATA_KEY;
165 
166 		datasize = btrfs_file_extent_calc_inline_size(cur_size);
167 		path->leave_spinning = 1;
168 		ret = btrfs_insert_empty_item(trans, root, path, &key,
169 					      datasize);
170 		if (ret) {
171 			err = ret;
172 			goto fail;
173 		}
174 	}
175 	leaf = path->nodes[0];
176 	ei = btrfs_item_ptr(leaf, path->slots[0],
177 			    struct btrfs_file_extent_item);
178 	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
179 	btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
180 	btrfs_set_file_extent_encryption(leaf, ei, 0);
181 	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
182 	btrfs_set_file_extent_ram_bytes(leaf, ei, size);
183 	ptr = btrfs_file_extent_inline_start(ei);
184 
185 	if (compress_type != BTRFS_COMPRESS_NONE) {
186 		struct page *cpage;
187 		int i = 0;
188 		while (compressed_size > 0) {
189 			cpage = compressed_pages[i];
190 			cur_size = min_t(unsigned long, compressed_size,
191 				       PAGE_CACHE_SIZE);
192 
193 			kaddr = kmap_atomic(cpage);
194 			write_extent_buffer(leaf, kaddr, ptr, cur_size);
195 			kunmap_atomic(kaddr);
196 
197 			i++;
198 			ptr += cur_size;
199 			compressed_size -= cur_size;
200 		}
201 		btrfs_set_file_extent_compression(leaf, ei,
202 						  compress_type);
203 	} else {
204 		page = find_get_page(inode->i_mapping,
205 				     start >> PAGE_CACHE_SHIFT);
206 		btrfs_set_file_extent_compression(leaf, ei, 0);
207 		kaddr = kmap_atomic(page);
208 		offset = start & (PAGE_CACHE_SIZE - 1);
209 		write_extent_buffer(leaf, kaddr + offset, ptr, size);
210 		kunmap_atomic(kaddr);
211 		page_cache_release(page);
212 	}
213 	btrfs_mark_buffer_dirty(leaf);
214 	btrfs_release_path(path);
215 
216 	/*
217 	 * we're an inline extent, so nobody can
218 	 * extend the file past i_size without locking
219 	 * a page we already have locked.
220 	 *
221 	 * We must do any isize and inode updates
222 	 * before we unlock the pages.  Otherwise we
223 	 * could end up racing with unlink.
224 	 */
225 	BTRFS_I(inode)->disk_i_size = inode->i_size;
226 	ret = btrfs_update_inode(trans, root, inode);
227 
228 	return ret;
229 fail:
230 	return err;
231 }
232 
233 
234 /*
235  * conditionally insert an inline extent into the file.  This
236  * does the checks required to make sure the data is small enough
237  * to fit as an inline extent.
238  */
239 static noinline int cow_file_range_inline(struct btrfs_root *root,
240 					  struct inode *inode, u64 start,
241 					  u64 end, size_t compressed_size,
242 					  int compress_type,
243 					  struct page **compressed_pages)
244 {
245 	struct btrfs_trans_handle *trans;
246 	u64 isize = i_size_read(inode);
247 	u64 actual_end = min(end + 1, isize);
248 	u64 inline_len = actual_end - start;
249 	u64 aligned_end = ALIGN(end, root->sectorsize);
250 	u64 data_len = inline_len;
251 	int ret;
252 	struct btrfs_path *path;
253 	int extent_inserted = 0;
254 	u32 extent_item_size;
255 
256 	if (compressed_size)
257 		data_len = compressed_size;
258 
259 	if (start > 0 ||
260 	    actual_end > PAGE_CACHE_SIZE ||
261 	    data_len > BTRFS_MAX_INLINE_DATA_SIZE(root) ||
262 	    (!compressed_size &&
263 	    (actual_end & (root->sectorsize - 1)) == 0) ||
264 	    end + 1 < isize ||
265 	    data_len > root->fs_info->max_inline) {
266 		return 1;
267 	}
268 
269 	path = btrfs_alloc_path();
270 	if (!path)
271 		return -ENOMEM;
272 
273 	trans = btrfs_join_transaction(root);
274 	if (IS_ERR(trans)) {
275 		btrfs_free_path(path);
276 		return PTR_ERR(trans);
277 	}
278 	trans->block_rsv = &root->fs_info->delalloc_block_rsv;
279 
280 	if (compressed_size && compressed_pages)
281 		extent_item_size = btrfs_file_extent_calc_inline_size(
282 		   compressed_size);
283 	else
284 		extent_item_size = btrfs_file_extent_calc_inline_size(
285 		    inline_len);
286 
287 	ret = __btrfs_drop_extents(trans, root, inode, path,
288 				   start, aligned_end, NULL,
289 				   1, 1, extent_item_size, &extent_inserted);
290 	if (ret) {
291 		btrfs_abort_transaction(trans, root, ret);
292 		goto out;
293 	}
294 
295 	if (isize > actual_end)
296 		inline_len = min_t(u64, isize, actual_end);
297 	ret = insert_inline_extent(trans, path, extent_inserted,
298 				   root, inode, start,
299 				   inline_len, compressed_size,
300 				   compress_type, compressed_pages);
301 	if (ret && ret != -ENOSPC) {
302 		btrfs_abort_transaction(trans, root, ret);
303 		goto out;
304 	} else if (ret == -ENOSPC) {
305 		ret = 1;
306 		goto out;
307 	}
308 
309 	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
310 	btrfs_delalloc_release_metadata(inode, end + 1 - start);
311 	btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0);
312 out:
313 	btrfs_free_path(path);
314 	btrfs_end_transaction(trans, root);
315 	return ret;
316 }
317 
318 struct async_extent {
319 	u64 start;
320 	u64 ram_size;
321 	u64 compressed_size;
322 	struct page **pages;
323 	unsigned long nr_pages;
324 	int compress_type;
325 	struct list_head list;
326 };
327 
328 struct async_cow {
329 	struct inode *inode;
330 	struct btrfs_root *root;
331 	struct page *locked_page;
332 	u64 start;
333 	u64 end;
334 	struct list_head extents;
335 	struct btrfs_work work;
336 };
337 
338 static noinline int add_async_extent(struct async_cow *cow,
339 				     u64 start, u64 ram_size,
340 				     u64 compressed_size,
341 				     struct page **pages,
342 				     unsigned long nr_pages,
343 				     int compress_type)
344 {
345 	struct async_extent *async_extent;
346 
347 	async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
348 	BUG_ON(!async_extent); /* -ENOMEM */
349 	async_extent->start = start;
350 	async_extent->ram_size = ram_size;
351 	async_extent->compressed_size = compressed_size;
352 	async_extent->pages = pages;
353 	async_extent->nr_pages = nr_pages;
354 	async_extent->compress_type = compress_type;
355 	list_add_tail(&async_extent->list, &cow->extents);
356 	return 0;
357 }
358 
359 static inline int inode_need_compress(struct inode *inode)
360 {
361 	struct btrfs_root *root = BTRFS_I(inode)->root;
362 
363 	/* force compress */
364 	if (btrfs_test_opt(root, FORCE_COMPRESS))
365 		return 1;
366 	/* bad compression ratios */
367 	if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
368 		return 0;
369 	if (btrfs_test_opt(root, COMPRESS) ||
370 	    BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
371 	    BTRFS_I(inode)->force_compress)
372 		return 1;
373 	return 0;
374 }
375 
376 /*
377  * we create compressed extents in two phases.  The first
378  * phase compresses a range of pages that have already been
379  * locked (both pages and state bits are locked).
380  *
381  * This is done inside an ordered work queue, and the compression
382  * is spread across many cpus.  The actual IO submission is step
383  * two, and the ordered work queue takes care of making sure that
384  * happens in the same order things were put onto the queue by
385  * writepages and friends.
386  *
387  * If this code finds it can't get good compression, it puts an
388  * entry onto the work queue to write the uncompressed bytes.  This
389  * makes sure that both compressed inodes and uncompressed inodes
390  * are written in the same order that the flusher thread sent them
391  * down.
392  */
393 static noinline void compress_file_range(struct inode *inode,
394 					struct page *locked_page,
395 					u64 start, u64 end,
396 					struct async_cow *async_cow,
397 					int *num_added)
398 {
399 	struct btrfs_root *root = BTRFS_I(inode)->root;
400 	u64 num_bytes;
401 	u64 blocksize = root->sectorsize;
402 	u64 actual_end;
403 	u64 isize = i_size_read(inode);
404 	int ret = 0;
405 	struct page **pages = NULL;
406 	unsigned long nr_pages;
407 	unsigned long nr_pages_ret = 0;
408 	unsigned long total_compressed = 0;
409 	unsigned long total_in = 0;
410 	unsigned long max_compressed = 128 * 1024;
411 	unsigned long max_uncompressed = 128 * 1024;
412 	int i;
413 	int will_compress;
414 	int compress_type = root->fs_info->compress_type;
415 	int redirty = 0;
416 
417 	/* if this is a small write inside eof, kick off a defrag */
418 	if ((end - start + 1) < 16 * 1024 &&
419 	    (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
420 		btrfs_add_inode_defrag(NULL, inode);
421 
422 	actual_end = min_t(u64, isize, end + 1);
423 again:
424 	will_compress = 0;
425 	nr_pages = (end >> PAGE_CACHE_SHIFT) - (start >> PAGE_CACHE_SHIFT) + 1;
426 	nr_pages = min(nr_pages, (128 * 1024UL) / PAGE_CACHE_SIZE);
427 
428 	/*
429 	 * we don't want to send crud past the end of i_size through
430 	 * compression, that's just a waste of CPU time.  So, if the
431 	 * end of the file is before the start of our current
432 	 * requested range of bytes, we bail out to the uncompressed
433 	 * cleanup code that can deal with all of this.
434 	 *
435 	 * It isn't really the fastest way to fix things, but this is a
436 	 * very uncommon corner.
437 	 */
438 	if (actual_end <= start)
439 		goto cleanup_and_bail_uncompressed;
440 
441 	total_compressed = actual_end - start;
442 
443 	/*
444 	 * skip compression for a small file range(<=blocksize) that
445 	 * isn't an inline extent, since it dosen't save disk space at all.
446 	 */
447 	if (total_compressed <= blocksize &&
448 	   (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
449 		goto cleanup_and_bail_uncompressed;
450 
451 	/* we want to make sure that amount of ram required to uncompress
452 	 * an extent is reasonable, so we limit the total size in ram
453 	 * of a compressed extent to 128k.  This is a crucial number
454 	 * because it also controls how easily we can spread reads across
455 	 * cpus for decompression.
456 	 *
457 	 * We also want to make sure the amount of IO required to do
458 	 * a random read is reasonably small, so we limit the size of
459 	 * a compressed extent to 128k.
460 	 */
461 	total_compressed = min(total_compressed, max_uncompressed);
462 	num_bytes = ALIGN(end - start + 1, blocksize);
463 	num_bytes = max(blocksize,  num_bytes);
464 	total_in = 0;
465 	ret = 0;
466 
467 	/*
468 	 * we do compression for mount -o compress and when the
469 	 * inode has not been flagged as nocompress.  This flag can
470 	 * change at any time if we discover bad compression ratios.
471 	 */
472 	if (inode_need_compress(inode)) {
473 		WARN_ON(pages);
474 		pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
475 		if (!pages) {
476 			/* just bail out to the uncompressed code */
477 			goto cont;
478 		}
479 
480 		if (BTRFS_I(inode)->force_compress)
481 			compress_type = BTRFS_I(inode)->force_compress;
482 
483 		/*
484 		 * we need to call clear_page_dirty_for_io on each
485 		 * page in the range.  Otherwise applications with the file
486 		 * mmap'd can wander in and change the page contents while
487 		 * we are compressing them.
488 		 *
489 		 * If the compression fails for any reason, we set the pages
490 		 * dirty again later on.
491 		 */
492 		extent_range_clear_dirty_for_io(inode, start, end);
493 		redirty = 1;
494 		ret = btrfs_compress_pages(compress_type,
495 					   inode->i_mapping, start,
496 					   total_compressed, pages,
497 					   nr_pages, &nr_pages_ret,
498 					   &total_in,
499 					   &total_compressed,
500 					   max_compressed);
501 
502 		if (!ret) {
503 			unsigned long offset = total_compressed &
504 				(PAGE_CACHE_SIZE - 1);
505 			struct page *page = pages[nr_pages_ret - 1];
506 			char *kaddr;
507 
508 			/* zero the tail end of the last page, we might be
509 			 * sending it down to disk
510 			 */
511 			if (offset) {
512 				kaddr = kmap_atomic(page);
513 				memset(kaddr + offset, 0,
514 				       PAGE_CACHE_SIZE - offset);
515 				kunmap_atomic(kaddr);
516 			}
517 			will_compress = 1;
518 		}
519 	}
520 cont:
521 	if (start == 0) {
522 		/* lets try to make an inline extent */
523 		if (ret || total_in < (actual_end - start)) {
524 			/* we didn't compress the entire range, try
525 			 * to make an uncompressed inline extent.
526 			 */
527 			ret = cow_file_range_inline(root, inode, start, end,
528 						    0, 0, NULL);
529 		} else {
530 			/* try making a compressed inline extent */
531 			ret = cow_file_range_inline(root, inode, start, end,
532 						    total_compressed,
533 						    compress_type, pages);
534 		}
535 		if (ret <= 0) {
536 			unsigned long clear_flags = EXTENT_DELALLOC |
537 				EXTENT_DEFRAG;
538 			unsigned long page_error_op;
539 
540 			clear_flags |= (ret < 0) ? EXTENT_DO_ACCOUNTING : 0;
541 			page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
542 
543 			/*
544 			 * inline extent creation worked or returned error,
545 			 * we don't need to create any more async work items.
546 			 * Unlock and free up our temp pages.
547 			 */
548 			extent_clear_unlock_delalloc(inode, start, end, NULL,
549 						     clear_flags, PAGE_UNLOCK |
550 						     PAGE_CLEAR_DIRTY |
551 						     PAGE_SET_WRITEBACK |
552 						     page_error_op |
553 						     PAGE_END_WRITEBACK);
554 			goto free_pages_out;
555 		}
556 	}
557 
558 	if (will_compress) {
559 		/*
560 		 * we aren't doing an inline extent round the compressed size
561 		 * up to a block size boundary so the allocator does sane
562 		 * things
563 		 */
564 		total_compressed = ALIGN(total_compressed, blocksize);
565 
566 		/*
567 		 * one last check to make sure the compression is really a
568 		 * win, compare the page count read with the blocks on disk
569 		 */
570 		total_in = ALIGN(total_in, PAGE_CACHE_SIZE);
571 		if (total_compressed >= total_in) {
572 			will_compress = 0;
573 		} else {
574 			num_bytes = total_in;
575 		}
576 	}
577 	if (!will_compress && pages) {
578 		/*
579 		 * the compression code ran but failed to make things smaller,
580 		 * free any pages it allocated and our page pointer array
581 		 */
582 		for (i = 0; i < nr_pages_ret; i++) {
583 			WARN_ON(pages[i]->mapping);
584 			page_cache_release(pages[i]);
585 		}
586 		kfree(pages);
587 		pages = NULL;
588 		total_compressed = 0;
589 		nr_pages_ret = 0;
590 
591 		/* flag the file so we don't compress in the future */
592 		if (!btrfs_test_opt(root, FORCE_COMPRESS) &&
593 		    !(BTRFS_I(inode)->force_compress)) {
594 			BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
595 		}
596 	}
597 	if (will_compress) {
598 		*num_added += 1;
599 
600 		/* the async work queues will take care of doing actual
601 		 * allocation on disk for these compressed pages,
602 		 * and will submit them to the elevator.
603 		 */
604 		add_async_extent(async_cow, start, num_bytes,
605 				 total_compressed, pages, nr_pages_ret,
606 				 compress_type);
607 
608 		if (start + num_bytes < end) {
609 			start += num_bytes;
610 			pages = NULL;
611 			cond_resched();
612 			goto again;
613 		}
614 	} else {
615 cleanup_and_bail_uncompressed:
616 		/*
617 		 * No compression, but we still need to write the pages in
618 		 * the file we've been given so far.  redirty the locked
619 		 * page if it corresponds to our extent and set things up
620 		 * for the async work queue to run cow_file_range to do
621 		 * the normal delalloc dance
622 		 */
623 		if (page_offset(locked_page) >= start &&
624 		    page_offset(locked_page) <= end) {
625 			__set_page_dirty_nobuffers(locked_page);
626 			/* unlocked later on in the async handlers */
627 		}
628 		if (redirty)
629 			extent_range_redirty_for_io(inode, start, end);
630 		add_async_extent(async_cow, start, end - start + 1,
631 				 0, NULL, 0, BTRFS_COMPRESS_NONE);
632 		*num_added += 1;
633 	}
634 
635 	return;
636 
637 free_pages_out:
638 	for (i = 0; i < nr_pages_ret; i++) {
639 		WARN_ON(pages[i]->mapping);
640 		page_cache_release(pages[i]);
641 	}
642 	kfree(pages);
643 }
644 
645 static void free_async_extent_pages(struct async_extent *async_extent)
646 {
647 	int i;
648 
649 	if (!async_extent->pages)
650 		return;
651 
652 	for (i = 0; i < async_extent->nr_pages; i++) {
653 		WARN_ON(async_extent->pages[i]->mapping);
654 		page_cache_release(async_extent->pages[i]);
655 	}
656 	kfree(async_extent->pages);
657 	async_extent->nr_pages = 0;
658 	async_extent->pages = NULL;
659 }
660 
661 /*
662  * phase two of compressed writeback.  This is the ordered portion
663  * of the code, which only gets called in the order the work was
664  * queued.  We walk all the async extents created by compress_file_range
665  * and send them down to the disk.
666  */
667 static noinline void submit_compressed_extents(struct inode *inode,
668 					      struct async_cow *async_cow)
669 {
670 	struct async_extent *async_extent;
671 	u64 alloc_hint = 0;
672 	struct btrfs_key ins;
673 	struct extent_map *em;
674 	struct btrfs_root *root = BTRFS_I(inode)->root;
675 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
676 	struct extent_io_tree *io_tree;
677 	int ret = 0;
678 
679 again:
680 	while (!list_empty(&async_cow->extents)) {
681 		async_extent = list_entry(async_cow->extents.next,
682 					  struct async_extent, list);
683 		list_del(&async_extent->list);
684 
685 		io_tree = &BTRFS_I(inode)->io_tree;
686 
687 retry:
688 		/* did the compression code fall back to uncompressed IO? */
689 		if (!async_extent->pages) {
690 			int page_started = 0;
691 			unsigned long nr_written = 0;
692 
693 			lock_extent(io_tree, async_extent->start,
694 					 async_extent->start +
695 					 async_extent->ram_size - 1);
696 
697 			/* allocate blocks */
698 			ret = cow_file_range(inode, async_cow->locked_page,
699 					     async_extent->start,
700 					     async_extent->start +
701 					     async_extent->ram_size - 1,
702 					     &page_started, &nr_written, 0);
703 
704 			/* JDM XXX */
705 
706 			/*
707 			 * if page_started, cow_file_range inserted an
708 			 * inline extent and took care of all the unlocking
709 			 * and IO for us.  Otherwise, we need to submit
710 			 * all those pages down to the drive.
711 			 */
712 			if (!page_started && !ret)
713 				extent_write_locked_range(io_tree,
714 						  inode, async_extent->start,
715 						  async_extent->start +
716 						  async_extent->ram_size - 1,
717 						  btrfs_get_extent,
718 						  WB_SYNC_ALL);
719 			else if (ret)
720 				unlock_page(async_cow->locked_page);
721 			kfree(async_extent);
722 			cond_resched();
723 			continue;
724 		}
725 
726 		lock_extent(io_tree, async_extent->start,
727 			    async_extent->start + async_extent->ram_size - 1);
728 
729 		ret = btrfs_reserve_extent(root,
730 					   async_extent->compressed_size,
731 					   async_extent->compressed_size,
732 					   0, alloc_hint, &ins, 1, 1);
733 		if (ret) {
734 			free_async_extent_pages(async_extent);
735 
736 			if (ret == -ENOSPC) {
737 				unlock_extent(io_tree, async_extent->start,
738 					      async_extent->start +
739 					      async_extent->ram_size - 1);
740 
741 				/*
742 				 * we need to redirty the pages if we decide to
743 				 * fallback to uncompressed IO, otherwise we
744 				 * will not submit these pages down to lower
745 				 * layers.
746 				 */
747 				extent_range_redirty_for_io(inode,
748 						async_extent->start,
749 						async_extent->start +
750 						async_extent->ram_size - 1);
751 
752 				goto retry;
753 			}
754 			goto out_free;
755 		}
756 		/*
757 		 * here we're doing allocation and writeback of the
758 		 * compressed pages
759 		 */
760 		btrfs_drop_extent_cache(inode, async_extent->start,
761 					async_extent->start +
762 					async_extent->ram_size - 1, 0);
763 
764 		em = alloc_extent_map();
765 		if (!em) {
766 			ret = -ENOMEM;
767 			goto out_free_reserve;
768 		}
769 		em->start = async_extent->start;
770 		em->len = async_extent->ram_size;
771 		em->orig_start = em->start;
772 		em->mod_start = em->start;
773 		em->mod_len = em->len;
774 
775 		em->block_start = ins.objectid;
776 		em->block_len = ins.offset;
777 		em->orig_block_len = ins.offset;
778 		em->ram_bytes = async_extent->ram_size;
779 		em->bdev = root->fs_info->fs_devices->latest_bdev;
780 		em->compress_type = async_extent->compress_type;
781 		set_bit(EXTENT_FLAG_PINNED, &em->flags);
782 		set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
783 		em->generation = -1;
784 
785 		while (1) {
786 			write_lock(&em_tree->lock);
787 			ret = add_extent_mapping(em_tree, em, 1);
788 			write_unlock(&em_tree->lock);
789 			if (ret != -EEXIST) {
790 				free_extent_map(em);
791 				break;
792 			}
793 			btrfs_drop_extent_cache(inode, async_extent->start,
794 						async_extent->start +
795 						async_extent->ram_size - 1, 0);
796 		}
797 
798 		if (ret)
799 			goto out_free_reserve;
800 
801 		ret = btrfs_add_ordered_extent_compress(inode,
802 						async_extent->start,
803 						ins.objectid,
804 						async_extent->ram_size,
805 						ins.offset,
806 						BTRFS_ORDERED_COMPRESSED,
807 						async_extent->compress_type);
808 		if (ret) {
809 			btrfs_drop_extent_cache(inode, async_extent->start,
810 						async_extent->start +
811 						async_extent->ram_size - 1, 0);
812 			goto out_free_reserve;
813 		}
814 
815 		/*
816 		 * clear dirty, set writeback and unlock the pages.
817 		 */
818 		extent_clear_unlock_delalloc(inode, async_extent->start,
819 				async_extent->start +
820 				async_extent->ram_size - 1,
821 				NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
822 				PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
823 				PAGE_SET_WRITEBACK);
824 		ret = btrfs_submit_compressed_write(inode,
825 				    async_extent->start,
826 				    async_extent->ram_size,
827 				    ins.objectid,
828 				    ins.offset, async_extent->pages,
829 				    async_extent->nr_pages);
830 		if (ret) {
831 			struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
832 			struct page *p = async_extent->pages[0];
833 			const u64 start = async_extent->start;
834 			const u64 end = start + async_extent->ram_size - 1;
835 
836 			p->mapping = inode->i_mapping;
837 			tree->ops->writepage_end_io_hook(p, start, end,
838 							 NULL, 0);
839 			p->mapping = NULL;
840 			extent_clear_unlock_delalloc(inode, start, end, NULL, 0,
841 						     PAGE_END_WRITEBACK |
842 						     PAGE_SET_ERROR);
843 			free_async_extent_pages(async_extent);
844 		}
845 		alloc_hint = ins.objectid + ins.offset;
846 		kfree(async_extent);
847 		cond_resched();
848 	}
849 	return;
850 out_free_reserve:
851 	btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1);
852 out_free:
853 	extent_clear_unlock_delalloc(inode, async_extent->start,
854 				     async_extent->start +
855 				     async_extent->ram_size - 1,
856 				     NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
857 				     EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
858 				     PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
859 				     PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
860 				     PAGE_SET_ERROR);
861 	free_async_extent_pages(async_extent);
862 	kfree(async_extent);
863 	goto again;
864 }
865 
866 static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
867 				      u64 num_bytes)
868 {
869 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
870 	struct extent_map *em;
871 	u64 alloc_hint = 0;
872 
873 	read_lock(&em_tree->lock);
874 	em = search_extent_mapping(em_tree, start, num_bytes);
875 	if (em) {
876 		/*
877 		 * if block start isn't an actual block number then find the
878 		 * first block in this inode and use that as a hint.  If that
879 		 * block is also bogus then just don't worry about it.
880 		 */
881 		if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
882 			free_extent_map(em);
883 			em = search_extent_mapping(em_tree, 0, 0);
884 			if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
885 				alloc_hint = em->block_start;
886 			if (em)
887 				free_extent_map(em);
888 		} else {
889 			alloc_hint = em->block_start;
890 			free_extent_map(em);
891 		}
892 	}
893 	read_unlock(&em_tree->lock);
894 
895 	return alloc_hint;
896 }
897 
898 /*
899  * when extent_io.c finds a delayed allocation range in the file,
900  * the call backs end up in this code.  The basic idea is to
901  * allocate extents on disk for the range, and create ordered data structs
902  * in ram to track those extents.
903  *
904  * locked_page is the page that writepage had locked already.  We use
905  * it to make sure we don't do extra locks or unlocks.
906  *
907  * *page_started is set to one if we unlock locked_page and do everything
908  * required to start IO on it.  It may be clean and already done with
909  * IO when we return.
910  */
911 static noinline int cow_file_range(struct inode *inode,
912 				   struct page *locked_page,
913 				   u64 start, u64 end, int *page_started,
914 				   unsigned long *nr_written,
915 				   int unlock)
916 {
917 	struct btrfs_root *root = BTRFS_I(inode)->root;
918 	u64 alloc_hint = 0;
919 	u64 num_bytes;
920 	unsigned long ram_size;
921 	u64 disk_num_bytes;
922 	u64 cur_alloc_size;
923 	u64 blocksize = root->sectorsize;
924 	struct btrfs_key ins;
925 	struct extent_map *em;
926 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
927 	int ret = 0;
928 
929 	if (btrfs_is_free_space_inode(inode)) {
930 		WARN_ON_ONCE(1);
931 		ret = -EINVAL;
932 		goto out_unlock;
933 	}
934 
935 	num_bytes = ALIGN(end - start + 1, blocksize);
936 	num_bytes = max(blocksize,  num_bytes);
937 	disk_num_bytes = num_bytes;
938 
939 	/* if this is a small write inside eof, kick off defrag */
940 	if (num_bytes < 64 * 1024 &&
941 	    (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
942 		btrfs_add_inode_defrag(NULL, inode);
943 
944 	if (start == 0) {
945 		/* lets try to make an inline extent */
946 		ret = cow_file_range_inline(root, inode, start, end, 0, 0,
947 					    NULL);
948 		if (ret == 0) {
949 			extent_clear_unlock_delalloc(inode, start, end, NULL,
950 				     EXTENT_LOCKED | EXTENT_DELALLOC |
951 				     EXTENT_DEFRAG, PAGE_UNLOCK |
952 				     PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
953 				     PAGE_END_WRITEBACK);
954 
955 			*nr_written = *nr_written +
956 			     (end - start + PAGE_CACHE_SIZE) / PAGE_CACHE_SIZE;
957 			*page_started = 1;
958 			goto out;
959 		} else if (ret < 0) {
960 			goto out_unlock;
961 		}
962 	}
963 
964 	BUG_ON(disk_num_bytes >
965 	       btrfs_super_total_bytes(root->fs_info->super_copy));
966 
967 	alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
968 	btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0);
969 
970 	while (disk_num_bytes > 0) {
971 		unsigned long op;
972 
973 		cur_alloc_size = disk_num_bytes;
974 		ret = btrfs_reserve_extent(root, cur_alloc_size,
975 					   root->sectorsize, 0, alloc_hint,
976 					   &ins, 1, 1);
977 		if (ret < 0)
978 			goto out_unlock;
979 
980 		em = alloc_extent_map();
981 		if (!em) {
982 			ret = -ENOMEM;
983 			goto out_reserve;
984 		}
985 		em->start = start;
986 		em->orig_start = em->start;
987 		ram_size = ins.offset;
988 		em->len = ins.offset;
989 		em->mod_start = em->start;
990 		em->mod_len = em->len;
991 
992 		em->block_start = ins.objectid;
993 		em->block_len = ins.offset;
994 		em->orig_block_len = ins.offset;
995 		em->ram_bytes = ram_size;
996 		em->bdev = root->fs_info->fs_devices->latest_bdev;
997 		set_bit(EXTENT_FLAG_PINNED, &em->flags);
998 		em->generation = -1;
999 
1000 		while (1) {
1001 			write_lock(&em_tree->lock);
1002 			ret = add_extent_mapping(em_tree, em, 1);
1003 			write_unlock(&em_tree->lock);
1004 			if (ret != -EEXIST) {
1005 				free_extent_map(em);
1006 				break;
1007 			}
1008 			btrfs_drop_extent_cache(inode, start,
1009 						start + ram_size - 1, 0);
1010 		}
1011 		if (ret)
1012 			goto out_reserve;
1013 
1014 		cur_alloc_size = ins.offset;
1015 		ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
1016 					       ram_size, cur_alloc_size, 0);
1017 		if (ret)
1018 			goto out_drop_extent_cache;
1019 
1020 		if (root->root_key.objectid ==
1021 		    BTRFS_DATA_RELOC_TREE_OBJECTID) {
1022 			ret = btrfs_reloc_clone_csums(inode, start,
1023 						      cur_alloc_size);
1024 			if (ret)
1025 				goto out_drop_extent_cache;
1026 		}
1027 
1028 		if (disk_num_bytes < cur_alloc_size)
1029 			break;
1030 
1031 		/* we're not doing compressed IO, don't unlock the first
1032 		 * page (which the caller expects to stay locked), don't
1033 		 * clear any dirty bits and don't set any writeback bits
1034 		 *
1035 		 * Do set the Private2 bit so we know this page was properly
1036 		 * setup for writepage
1037 		 */
1038 		op = unlock ? PAGE_UNLOCK : 0;
1039 		op |= PAGE_SET_PRIVATE2;
1040 
1041 		extent_clear_unlock_delalloc(inode, start,
1042 					     start + ram_size - 1, locked_page,
1043 					     EXTENT_LOCKED | EXTENT_DELALLOC,
1044 					     op);
1045 		disk_num_bytes -= cur_alloc_size;
1046 		num_bytes -= cur_alloc_size;
1047 		alloc_hint = ins.objectid + ins.offset;
1048 		start += cur_alloc_size;
1049 	}
1050 out:
1051 	return ret;
1052 
1053 out_drop_extent_cache:
1054 	btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0);
1055 out_reserve:
1056 	btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1);
1057 out_unlock:
1058 	extent_clear_unlock_delalloc(inode, start, end, locked_page,
1059 				     EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
1060 				     EXTENT_DELALLOC | EXTENT_DEFRAG,
1061 				     PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
1062 				     PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK);
1063 	goto out;
1064 }
1065 
1066 /*
1067  * work queue call back to started compression on a file and pages
1068  */
1069 static noinline void async_cow_start(struct btrfs_work *work)
1070 {
1071 	struct async_cow *async_cow;
1072 	int num_added = 0;
1073 	async_cow = container_of(work, struct async_cow, work);
1074 
1075 	compress_file_range(async_cow->inode, async_cow->locked_page,
1076 			    async_cow->start, async_cow->end, async_cow,
1077 			    &num_added);
1078 	if (num_added == 0) {
1079 		btrfs_add_delayed_iput(async_cow->inode);
1080 		async_cow->inode = NULL;
1081 	}
1082 }
1083 
1084 /*
1085  * work queue call back to submit previously compressed pages
1086  */
1087 static noinline void async_cow_submit(struct btrfs_work *work)
1088 {
1089 	struct async_cow *async_cow;
1090 	struct btrfs_root *root;
1091 	unsigned long nr_pages;
1092 
1093 	async_cow = container_of(work, struct async_cow, work);
1094 
1095 	root = async_cow->root;
1096 	nr_pages = (async_cow->end - async_cow->start + PAGE_CACHE_SIZE) >>
1097 		PAGE_CACHE_SHIFT;
1098 
1099 	if (atomic_sub_return(nr_pages, &root->fs_info->async_delalloc_pages) <
1100 	    5 * 1024 * 1024 &&
1101 	    waitqueue_active(&root->fs_info->async_submit_wait))
1102 		wake_up(&root->fs_info->async_submit_wait);
1103 
1104 	if (async_cow->inode)
1105 		submit_compressed_extents(async_cow->inode, async_cow);
1106 }
1107 
1108 static noinline void async_cow_free(struct btrfs_work *work)
1109 {
1110 	struct async_cow *async_cow;
1111 	async_cow = container_of(work, struct async_cow, work);
1112 	if (async_cow->inode)
1113 		btrfs_add_delayed_iput(async_cow->inode);
1114 	kfree(async_cow);
1115 }
1116 
1117 static int cow_file_range_async(struct inode *inode, struct page *locked_page,
1118 				u64 start, u64 end, int *page_started,
1119 				unsigned long *nr_written)
1120 {
1121 	struct async_cow *async_cow;
1122 	struct btrfs_root *root = BTRFS_I(inode)->root;
1123 	unsigned long nr_pages;
1124 	u64 cur_end;
1125 	int limit = 10 * 1024 * 1024;
1126 
1127 	clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
1128 			 1, 0, NULL, GFP_NOFS);
1129 	while (start < end) {
1130 		async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
1131 		BUG_ON(!async_cow); /* -ENOMEM */
1132 		async_cow->inode = igrab(inode);
1133 		async_cow->root = root;
1134 		async_cow->locked_page = locked_page;
1135 		async_cow->start = start;
1136 
1137 		if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
1138 		    !btrfs_test_opt(root, FORCE_COMPRESS))
1139 			cur_end = end;
1140 		else
1141 			cur_end = min(end, start + 512 * 1024 - 1);
1142 
1143 		async_cow->end = cur_end;
1144 		INIT_LIST_HEAD(&async_cow->extents);
1145 
1146 		btrfs_init_work(&async_cow->work,
1147 				btrfs_delalloc_helper,
1148 				async_cow_start, async_cow_submit,
1149 				async_cow_free);
1150 
1151 		nr_pages = (cur_end - start + PAGE_CACHE_SIZE) >>
1152 			PAGE_CACHE_SHIFT;
1153 		atomic_add(nr_pages, &root->fs_info->async_delalloc_pages);
1154 
1155 		btrfs_queue_work(root->fs_info->delalloc_workers,
1156 				 &async_cow->work);
1157 
1158 		if (atomic_read(&root->fs_info->async_delalloc_pages) > limit) {
1159 			wait_event(root->fs_info->async_submit_wait,
1160 			   (atomic_read(&root->fs_info->async_delalloc_pages) <
1161 			    limit));
1162 		}
1163 
1164 		while (atomic_read(&root->fs_info->async_submit_draining) &&
1165 		      atomic_read(&root->fs_info->async_delalloc_pages)) {
1166 			wait_event(root->fs_info->async_submit_wait,
1167 			  (atomic_read(&root->fs_info->async_delalloc_pages) ==
1168 			   0));
1169 		}
1170 
1171 		*nr_written += nr_pages;
1172 		start = cur_end + 1;
1173 	}
1174 	*page_started = 1;
1175 	return 0;
1176 }
1177 
1178 static noinline int csum_exist_in_range(struct btrfs_root *root,
1179 					u64 bytenr, u64 num_bytes)
1180 {
1181 	int ret;
1182 	struct btrfs_ordered_sum *sums;
1183 	LIST_HEAD(list);
1184 
1185 	ret = btrfs_lookup_csums_range(root->fs_info->csum_root, bytenr,
1186 				       bytenr + num_bytes - 1, &list, 0);
1187 	if (ret == 0 && list_empty(&list))
1188 		return 0;
1189 
1190 	while (!list_empty(&list)) {
1191 		sums = list_entry(list.next, struct btrfs_ordered_sum, list);
1192 		list_del(&sums->list);
1193 		kfree(sums);
1194 	}
1195 	return 1;
1196 }
1197 
1198 /*
1199  * when nowcow writeback call back.  This checks for snapshots or COW copies
1200  * of the extents that exist in the file, and COWs the file as required.
1201  *
1202  * If no cow copies or snapshots exist, we write directly to the existing
1203  * blocks on disk
1204  */
1205 static noinline int run_delalloc_nocow(struct inode *inode,
1206 				       struct page *locked_page,
1207 			      u64 start, u64 end, int *page_started, int force,
1208 			      unsigned long *nr_written)
1209 {
1210 	struct btrfs_root *root = BTRFS_I(inode)->root;
1211 	struct btrfs_trans_handle *trans;
1212 	struct extent_buffer *leaf;
1213 	struct btrfs_path *path;
1214 	struct btrfs_file_extent_item *fi;
1215 	struct btrfs_key found_key;
1216 	u64 cow_start;
1217 	u64 cur_offset;
1218 	u64 extent_end;
1219 	u64 extent_offset;
1220 	u64 disk_bytenr;
1221 	u64 num_bytes;
1222 	u64 disk_num_bytes;
1223 	u64 ram_bytes;
1224 	int extent_type;
1225 	int ret, err;
1226 	int type;
1227 	int nocow;
1228 	int check_prev = 1;
1229 	bool nolock;
1230 	u64 ino = btrfs_ino(inode);
1231 
1232 	path = btrfs_alloc_path();
1233 	if (!path) {
1234 		extent_clear_unlock_delalloc(inode, start, end, locked_page,
1235 					     EXTENT_LOCKED | EXTENT_DELALLOC |
1236 					     EXTENT_DO_ACCOUNTING |
1237 					     EXTENT_DEFRAG, PAGE_UNLOCK |
1238 					     PAGE_CLEAR_DIRTY |
1239 					     PAGE_SET_WRITEBACK |
1240 					     PAGE_END_WRITEBACK);
1241 		return -ENOMEM;
1242 	}
1243 
1244 	nolock = btrfs_is_free_space_inode(inode);
1245 
1246 	if (nolock)
1247 		trans = btrfs_join_transaction_nolock(root);
1248 	else
1249 		trans = btrfs_join_transaction(root);
1250 
1251 	if (IS_ERR(trans)) {
1252 		extent_clear_unlock_delalloc(inode, start, end, locked_page,
1253 					     EXTENT_LOCKED | EXTENT_DELALLOC |
1254 					     EXTENT_DO_ACCOUNTING |
1255 					     EXTENT_DEFRAG, PAGE_UNLOCK |
1256 					     PAGE_CLEAR_DIRTY |
1257 					     PAGE_SET_WRITEBACK |
1258 					     PAGE_END_WRITEBACK);
1259 		btrfs_free_path(path);
1260 		return PTR_ERR(trans);
1261 	}
1262 
1263 	trans->block_rsv = &root->fs_info->delalloc_block_rsv;
1264 
1265 	cow_start = (u64)-1;
1266 	cur_offset = start;
1267 	while (1) {
1268 		ret = btrfs_lookup_file_extent(trans, root, path, ino,
1269 					       cur_offset, 0);
1270 		if (ret < 0)
1271 			goto error;
1272 		if (ret > 0 && path->slots[0] > 0 && check_prev) {
1273 			leaf = path->nodes[0];
1274 			btrfs_item_key_to_cpu(leaf, &found_key,
1275 					      path->slots[0] - 1);
1276 			if (found_key.objectid == ino &&
1277 			    found_key.type == BTRFS_EXTENT_DATA_KEY)
1278 				path->slots[0]--;
1279 		}
1280 		check_prev = 0;
1281 next_slot:
1282 		leaf = path->nodes[0];
1283 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1284 			ret = btrfs_next_leaf(root, path);
1285 			if (ret < 0)
1286 				goto error;
1287 			if (ret > 0)
1288 				break;
1289 			leaf = path->nodes[0];
1290 		}
1291 
1292 		nocow = 0;
1293 		disk_bytenr = 0;
1294 		num_bytes = 0;
1295 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1296 
1297 		if (found_key.objectid > ino ||
1298 		    found_key.type > BTRFS_EXTENT_DATA_KEY ||
1299 		    found_key.offset > end)
1300 			break;
1301 
1302 		if (found_key.offset > cur_offset) {
1303 			extent_end = found_key.offset;
1304 			extent_type = 0;
1305 			goto out_check;
1306 		}
1307 
1308 		fi = btrfs_item_ptr(leaf, path->slots[0],
1309 				    struct btrfs_file_extent_item);
1310 		extent_type = btrfs_file_extent_type(leaf, fi);
1311 
1312 		ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1313 		if (extent_type == BTRFS_FILE_EXTENT_REG ||
1314 		    extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1315 			disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1316 			extent_offset = btrfs_file_extent_offset(leaf, fi);
1317 			extent_end = found_key.offset +
1318 				btrfs_file_extent_num_bytes(leaf, fi);
1319 			disk_num_bytes =
1320 				btrfs_file_extent_disk_num_bytes(leaf, fi);
1321 			if (extent_end <= start) {
1322 				path->slots[0]++;
1323 				goto next_slot;
1324 			}
1325 			if (disk_bytenr == 0)
1326 				goto out_check;
1327 			if (btrfs_file_extent_compression(leaf, fi) ||
1328 			    btrfs_file_extent_encryption(leaf, fi) ||
1329 			    btrfs_file_extent_other_encoding(leaf, fi))
1330 				goto out_check;
1331 			if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
1332 				goto out_check;
1333 			if (btrfs_extent_readonly(root, disk_bytenr))
1334 				goto out_check;
1335 			if (btrfs_cross_ref_exist(trans, root, ino,
1336 						  found_key.offset -
1337 						  extent_offset, disk_bytenr))
1338 				goto out_check;
1339 			disk_bytenr += extent_offset;
1340 			disk_bytenr += cur_offset - found_key.offset;
1341 			num_bytes = min(end + 1, extent_end) - cur_offset;
1342 			/*
1343 			 * if there are pending snapshots for this root,
1344 			 * we fall into common COW way.
1345 			 */
1346 			if (!nolock) {
1347 				err = btrfs_start_write_no_snapshoting(root);
1348 				if (!err)
1349 					goto out_check;
1350 			}
1351 			/*
1352 			 * force cow if csum exists in the range.
1353 			 * this ensure that csum for a given extent are
1354 			 * either valid or do not exist.
1355 			 */
1356 			if (csum_exist_in_range(root, disk_bytenr, num_bytes))
1357 				goto out_check;
1358 			nocow = 1;
1359 		} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1360 			extent_end = found_key.offset +
1361 				btrfs_file_extent_inline_len(leaf,
1362 						     path->slots[0], fi);
1363 			extent_end = ALIGN(extent_end, root->sectorsize);
1364 		} else {
1365 			BUG_ON(1);
1366 		}
1367 out_check:
1368 		if (extent_end <= start) {
1369 			path->slots[0]++;
1370 			if (!nolock && nocow)
1371 				btrfs_end_write_no_snapshoting(root);
1372 			goto next_slot;
1373 		}
1374 		if (!nocow) {
1375 			if (cow_start == (u64)-1)
1376 				cow_start = cur_offset;
1377 			cur_offset = extent_end;
1378 			if (cur_offset > end)
1379 				break;
1380 			path->slots[0]++;
1381 			goto next_slot;
1382 		}
1383 
1384 		btrfs_release_path(path);
1385 		if (cow_start != (u64)-1) {
1386 			ret = cow_file_range(inode, locked_page,
1387 					     cow_start, found_key.offset - 1,
1388 					     page_started, nr_written, 1);
1389 			if (ret) {
1390 				if (!nolock && nocow)
1391 					btrfs_end_write_no_snapshoting(root);
1392 				goto error;
1393 			}
1394 			cow_start = (u64)-1;
1395 		}
1396 
1397 		if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1398 			struct extent_map *em;
1399 			struct extent_map_tree *em_tree;
1400 			em_tree = &BTRFS_I(inode)->extent_tree;
1401 			em = alloc_extent_map();
1402 			BUG_ON(!em); /* -ENOMEM */
1403 			em->start = cur_offset;
1404 			em->orig_start = found_key.offset - extent_offset;
1405 			em->len = num_bytes;
1406 			em->block_len = num_bytes;
1407 			em->block_start = disk_bytenr;
1408 			em->orig_block_len = disk_num_bytes;
1409 			em->ram_bytes = ram_bytes;
1410 			em->bdev = root->fs_info->fs_devices->latest_bdev;
1411 			em->mod_start = em->start;
1412 			em->mod_len = em->len;
1413 			set_bit(EXTENT_FLAG_PINNED, &em->flags);
1414 			set_bit(EXTENT_FLAG_FILLING, &em->flags);
1415 			em->generation = -1;
1416 			while (1) {
1417 				write_lock(&em_tree->lock);
1418 				ret = add_extent_mapping(em_tree, em, 1);
1419 				write_unlock(&em_tree->lock);
1420 				if (ret != -EEXIST) {
1421 					free_extent_map(em);
1422 					break;
1423 				}
1424 				btrfs_drop_extent_cache(inode, em->start,
1425 						em->start + em->len - 1, 0);
1426 			}
1427 			type = BTRFS_ORDERED_PREALLOC;
1428 		} else {
1429 			type = BTRFS_ORDERED_NOCOW;
1430 		}
1431 
1432 		ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
1433 					       num_bytes, num_bytes, type);
1434 		BUG_ON(ret); /* -ENOMEM */
1435 
1436 		if (root->root_key.objectid ==
1437 		    BTRFS_DATA_RELOC_TREE_OBJECTID) {
1438 			ret = btrfs_reloc_clone_csums(inode, cur_offset,
1439 						      num_bytes);
1440 			if (ret) {
1441 				if (!nolock && nocow)
1442 					btrfs_end_write_no_snapshoting(root);
1443 				goto error;
1444 			}
1445 		}
1446 
1447 		extent_clear_unlock_delalloc(inode, cur_offset,
1448 					     cur_offset + num_bytes - 1,
1449 					     locked_page, EXTENT_LOCKED |
1450 					     EXTENT_DELALLOC, PAGE_UNLOCK |
1451 					     PAGE_SET_PRIVATE2);
1452 		if (!nolock && nocow)
1453 			btrfs_end_write_no_snapshoting(root);
1454 		cur_offset = extent_end;
1455 		if (cur_offset > end)
1456 			break;
1457 	}
1458 	btrfs_release_path(path);
1459 
1460 	if (cur_offset <= end && cow_start == (u64)-1) {
1461 		cow_start = cur_offset;
1462 		cur_offset = end;
1463 	}
1464 
1465 	if (cow_start != (u64)-1) {
1466 		ret = cow_file_range(inode, locked_page, cow_start, end,
1467 				     page_started, nr_written, 1);
1468 		if (ret)
1469 			goto error;
1470 	}
1471 
1472 error:
1473 	err = btrfs_end_transaction(trans, root);
1474 	if (!ret)
1475 		ret = err;
1476 
1477 	if (ret && cur_offset < end)
1478 		extent_clear_unlock_delalloc(inode, cur_offset, end,
1479 					     locked_page, EXTENT_LOCKED |
1480 					     EXTENT_DELALLOC | EXTENT_DEFRAG |
1481 					     EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
1482 					     PAGE_CLEAR_DIRTY |
1483 					     PAGE_SET_WRITEBACK |
1484 					     PAGE_END_WRITEBACK);
1485 	btrfs_free_path(path);
1486 	return ret;
1487 }
1488 
1489 static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
1490 {
1491 
1492 	if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
1493 	    !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
1494 		return 0;
1495 
1496 	/*
1497 	 * @defrag_bytes is a hint value, no spinlock held here,
1498 	 * if is not zero, it means the file is defragging.
1499 	 * Force cow if given extent needs to be defragged.
1500 	 */
1501 	if (BTRFS_I(inode)->defrag_bytes &&
1502 	    test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
1503 			   EXTENT_DEFRAG, 0, NULL))
1504 		return 1;
1505 
1506 	return 0;
1507 }
1508 
1509 /*
1510  * extent_io.c call back to do delayed allocation processing
1511  */
1512 static int run_delalloc_range(struct inode *inode, struct page *locked_page,
1513 			      u64 start, u64 end, int *page_started,
1514 			      unsigned long *nr_written)
1515 {
1516 	int ret;
1517 	int force_cow = need_force_cow(inode, start, end);
1518 
1519 	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
1520 		ret = run_delalloc_nocow(inode, locked_page, start, end,
1521 					 page_started, 1, nr_written);
1522 	} else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
1523 		ret = run_delalloc_nocow(inode, locked_page, start, end,
1524 					 page_started, 0, nr_written);
1525 	} else if (!inode_need_compress(inode)) {
1526 		ret = cow_file_range(inode, locked_page, start, end,
1527 				      page_started, nr_written, 1);
1528 	} else {
1529 		set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
1530 			&BTRFS_I(inode)->runtime_flags);
1531 		ret = cow_file_range_async(inode, locked_page, start, end,
1532 					   page_started, nr_written);
1533 	}
1534 	return ret;
1535 }
1536 
1537 static void btrfs_split_extent_hook(struct inode *inode,
1538 				    struct extent_state *orig, u64 split)
1539 {
1540 	u64 size;
1541 
1542 	/* not delalloc, ignore it */
1543 	if (!(orig->state & EXTENT_DELALLOC))
1544 		return;
1545 
1546 	size = orig->end - orig->start + 1;
1547 	if (size > BTRFS_MAX_EXTENT_SIZE) {
1548 		u64 num_extents;
1549 		u64 new_size;
1550 
1551 		/*
1552 		 * See the explanation in btrfs_merge_extent_hook, the same
1553 		 * applies here, just in reverse.
1554 		 */
1555 		new_size = orig->end - split + 1;
1556 		num_extents = div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
1557 					BTRFS_MAX_EXTENT_SIZE);
1558 		new_size = split - orig->start;
1559 		num_extents += div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
1560 					BTRFS_MAX_EXTENT_SIZE);
1561 		if (div64_u64(size + BTRFS_MAX_EXTENT_SIZE - 1,
1562 			      BTRFS_MAX_EXTENT_SIZE) >= num_extents)
1563 			return;
1564 	}
1565 
1566 	spin_lock(&BTRFS_I(inode)->lock);
1567 	BTRFS_I(inode)->outstanding_extents++;
1568 	spin_unlock(&BTRFS_I(inode)->lock);
1569 }
1570 
1571 /*
1572  * extent_io.c merge_extent_hook, used to track merged delayed allocation
1573  * extents so we can keep track of new extents that are just merged onto old
1574  * extents, such as when we are doing sequential writes, so we can properly
1575  * account for the metadata space we'll need.
1576  */
1577 static void btrfs_merge_extent_hook(struct inode *inode,
1578 				    struct extent_state *new,
1579 				    struct extent_state *other)
1580 {
1581 	u64 new_size, old_size;
1582 	u64 num_extents;
1583 
1584 	/* not delalloc, ignore it */
1585 	if (!(other->state & EXTENT_DELALLOC))
1586 		return;
1587 
1588 	if (new->start > other->start)
1589 		new_size = new->end - other->start + 1;
1590 	else
1591 		new_size = other->end - new->start + 1;
1592 
1593 	/* we're not bigger than the max, unreserve the space and go */
1594 	if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
1595 		spin_lock(&BTRFS_I(inode)->lock);
1596 		BTRFS_I(inode)->outstanding_extents--;
1597 		spin_unlock(&BTRFS_I(inode)->lock);
1598 		return;
1599 	}
1600 
1601 	/*
1602 	 * We have to add up either side to figure out how many extents were
1603 	 * accounted for before we merged into one big extent.  If the number of
1604 	 * extents we accounted for is <= the amount we need for the new range
1605 	 * then we can return, otherwise drop.  Think of it like this
1606 	 *
1607 	 * [ 4k][MAX_SIZE]
1608 	 *
1609 	 * So we've grown the extent by a MAX_SIZE extent, this would mean we
1610 	 * need 2 outstanding extents, on one side we have 1 and the other side
1611 	 * we have 1 so they are == and we can return.  But in this case
1612 	 *
1613 	 * [MAX_SIZE+4k][MAX_SIZE+4k]
1614 	 *
1615 	 * Each range on their own accounts for 2 extents, but merged together
1616 	 * they are only 3 extents worth of accounting, so we need to drop in
1617 	 * this case.
1618 	 */
1619 	old_size = other->end - other->start + 1;
1620 	num_extents = div64_u64(old_size + BTRFS_MAX_EXTENT_SIZE - 1,
1621 				BTRFS_MAX_EXTENT_SIZE);
1622 	old_size = new->end - new->start + 1;
1623 	num_extents += div64_u64(old_size + BTRFS_MAX_EXTENT_SIZE - 1,
1624 				 BTRFS_MAX_EXTENT_SIZE);
1625 
1626 	if (div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
1627 		      BTRFS_MAX_EXTENT_SIZE) >= num_extents)
1628 		return;
1629 
1630 	spin_lock(&BTRFS_I(inode)->lock);
1631 	BTRFS_I(inode)->outstanding_extents--;
1632 	spin_unlock(&BTRFS_I(inode)->lock);
1633 }
1634 
1635 static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
1636 				      struct inode *inode)
1637 {
1638 	spin_lock(&root->delalloc_lock);
1639 	if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1640 		list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
1641 			      &root->delalloc_inodes);
1642 		set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1643 			&BTRFS_I(inode)->runtime_flags);
1644 		root->nr_delalloc_inodes++;
1645 		if (root->nr_delalloc_inodes == 1) {
1646 			spin_lock(&root->fs_info->delalloc_root_lock);
1647 			BUG_ON(!list_empty(&root->delalloc_root));
1648 			list_add_tail(&root->delalloc_root,
1649 				      &root->fs_info->delalloc_roots);
1650 			spin_unlock(&root->fs_info->delalloc_root_lock);
1651 		}
1652 	}
1653 	spin_unlock(&root->delalloc_lock);
1654 }
1655 
1656 static void btrfs_del_delalloc_inode(struct btrfs_root *root,
1657 				     struct inode *inode)
1658 {
1659 	spin_lock(&root->delalloc_lock);
1660 	if (!list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1661 		list_del_init(&BTRFS_I(inode)->delalloc_inodes);
1662 		clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1663 			  &BTRFS_I(inode)->runtime_flags);
1664 		root->nr_delalloc_inodes--;
1665 		if (!root->nr_delalloc_inodes) {
1666 			spin_lock(&root->fs_info->delalloc_root_lock);
1667 			BUG_ON(list_empty(&root->delalloc_root));
1668 			list_del_init(&root->delalloc_root);
1669 			spin_unlock(&root->fs_info->delalloc_root_lock);
1670 		}
1671 	}
1672 	spin_unlock(&root->delalloc_lock);
1673 }
1674 
1675 /*
1676  * extent_io.c set_bit_hook, used to track delayed allocation
1677  * bytes in this file, and to maintain the list of inodes that
1678  * have pending delalloc work to be done.
1679  */
1680 static void btrfs_set_bit_hook(struct inode *inode,
1681 			       struct extent_state *state, unsigned *bits)
1682 {
1683 
1684 	if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
1685 		WARN_ON(1);
1686 	/*
1687 	 * set_bit and clear bit hooks normally require _irqsave/restore
1688 	 * but in this case, we are only testing for the DELALLOC
1689 	 * bit, which is only set or cleared with irqs on
1690 	 */
1691 	if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1692 		struct btrfs_root *root = BTRFS_I(inode)->root;
1693 		u64 len = state->end + 1 - state->start;
1694 		bool do_list = !btrfs_is_free_space_inode(inode);
1695 
1696 		if (*bits & EXTENT_FIRST_DELALLOC) {
1697 			*bits &= ~EXTENT_FIRST_DELALLOC;
1698 		} else {
1699 			spin_lock(&BTRFS_I(inode)->lock);
1700 			BTRFS_I(inode)->outstanding_extents++;
1701 			spin_unlock(&BTRFS_I(inode)->lock);
1702 		}
1703 
1704 		/* For sanity tests */
1705 		if (btrfs_test_is_dummy_root(root))
1706 			return;
1707 
1708 		__percpu_counter_add(&root->fs_info->delalloc_bytes, len,
1709 				     root->fs_info->delalloc_batch);
1710 		spin_lock(&BTRFS_I(inode)->lock);
1711 		BTRFS_I(inode)->delalloc_bytes += len;
1712 		if (*bits & EXTENT_DEFRAG)
1713 			BTRFS_I(inode)->defrag_bytes += len;
1714 		if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1715 					 &BTRFS_I(inode)->runtime_flags))
1716 			btrfs_add_delalloc_inodes(root, inode);
1717 		spin_unlock(&BTRFS_I(inode)->lock);
1718 	}
1719 }
1720 
1721 /*
1722  * extent_io.c clear_bit_hook, see set_bit_hook for why
1723  */
1724 static void btrfs_clear_bit_hook(struct inode *inode,
1725 				 struct extent_state *state,
1726 				 unsigned *bits)
1727 {
1728 	u64 len = state->end + 1 - state->start;
1729 	u64 num_extents = div64_u64(len + BTRFS_MAX_EXTENT_SIZE -1,
1730 				    BTRFS_MAX_EXTENT_SIZE);
1731 
1732 	spin_lock(&BTRFS_I(inode)->lock);
1733 	if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG))
1734 		BTRFS_I(inode)->defrag_bytes -= len;
1735 	spin_unlock(&BTRFS_I(inode)->lock);
1736 
1737 	/*
1738 	 * set_bit and clear bit hooks normally require _irqsave/restore
1739 	 * but in this case, we are only testing for the DELALLOC
1740 	 * bit, which is only set or cleared with irqs on
1741 	 */
1742 	if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1743 		struct btrfs_root *root = BTRFS_I(inode)->root;
1744 		bool do_list = !btrfs_is_free_space_inode(inode);
1745 
1746 		if (*bits & EXTENT_FIRST_DELALLOC) {
1747 			*bits &= ~EXTENT_FIRST_DELALLOC;
1748 		} else if (!(*bits & EXTENT_DO_ACCOUNTING)) {
1749 			spin_lock(&BTRFS_I(inode)->lock);
1750 			BTRFS_I(inode)->outstanding_extents -= num_extents;
1751 			spin_unlock(&BTRFS_I(inode)->lock);
1752 		}
1753 
1754 		/*
1755 		 * We don't reserve metadata space for space cache inodes so we
1756 		 * don't need to call dellalloc_release_metadata if there is an
1757 		 * error.
1758 		 */
1759 		if (*bits & EXTENT_DO_ACCOUNTING &&
1760 		    root != root->fs_info->tree_root)
1761 			btrfs_delalloc_release_metadata(inode, len);
1762 
1763 		/* For sanity tests. */
1764 		if (btrfs_test_is_dummy_root(root))
1765 			return;
1766 
1767 		if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
1768 		    && do_list && !(state->state & EXTENT_NORESERVE))
1769 			btrfs_free_reserved_data_space(inode, len);
1770 
1771 		__percpu_counter_add(&root->fs_info->delalloc_bytes, -len,
1772 				     root->fs_info->delalloc_batch);
1773 		spin_lock(&BTRFS_I(inode)->lock);
1774 		BTRFS_I(inode)->delalloc_bytes -= len;
1775 		if (do_list && BTRFS_I(inode)->delalloc_bytes == 0 &&
1776 		    test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1777 			     &BTRFS_I(inode)->runtime_flags))
1778 			btrfs_del_delalloc_inode(root, inode);
1779 		spin_unlock(&BTRFS_I(inode)->lock);
1780 	}
1781 }
1782 
1783 /*
1784  * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
1785  * we don't create bios that span stripes or chunks
1786  */
1787 int btrfs_merge_bio_hook(int rw, struct page *page, unsigned long offset,
1788 			 size_t size, struct bio *bio,
1789 			 unsigned long bio_flags)
1790 {
1791 	struct btrfs_root *root = BTRFS_I(page->mapping->host)->root;
1792 	u64 logical = (u64)bio->bi_iter.bi_sector << 9;
1793 	u64 length = 0;
1794 	u64 map_length;
1795 	int ret;
1796 
1797 	if (bio_flags & EXTENT_BIO_COMPRESSED)
1798 		return 0;
1799 
1800 	length = bio->bi_iter.bi_size;
1801 	map_length = length;
1802 	ret = btrfs_map_block(root->fs_info, rw, logical,
1803 			      &map_length, NULL, 0);
1804 	/* Will always return 0 with map_multi == NULL */
1805 	BUG_ON(ret < 0);
1806 	if (map_length < length + size)
1807 		return 1;
1808 	return 0;
1809 }
1810 
1811 /*
1812  * in order to insert checksums into the metadata in large chunks,
1813  * we wait until bio submission time.   All the pages in the bio are
1814  * checksummed and sums are attached onto the ordered extent record.
1815  *
1816  * At IO completion time the cums attached on the ordered extent record
1817  * are inserted into the btree
1818  */
1819 static int __btrfs_submit_bio_start(struct inode *inode, int rw,
1820 				    struct bio *bio, int mirror_num,
1821 				    unsigned long bio_flags,
1822 				    u64 bio_offset)
1823 {
1824 	struct btrfs_root *root = BTRFS_I(inode)->root;
1825 	int ret = 0;
1826 
1827 	ret = btrfs_csum_one_bio(root, inode, bio, 0, 0);
1828 	BUG_ON(ret); /* -ENOMEM */
1829 	return 0;
1830 }
1831 
1832 /*
1833  * in order to insert checksums into the metadata in large chunks,
1834  * we wait until bio submission time.   All the pages in the bio are
1835  * checksummed and sums are attached onto the ordered extent record.
1836  *
1837  * At IO completion time the cums attached on the ordered extent record
1838  * are inserted into the btree
1839  */
1840 static int __btrfs_submit_bio_done(struct inode *inode, int rw, struct bio *bio,
1841 			  int mirror_num, unsigned long bio_flags,
1842 			  u64 bio_offset)
1843 {
1844 	struct btrfs_root *root = BTRFS_I(inode)->root;
1845 	int ret;
1846 
1847 	ret = btrfs_map_bio(root, rw, bio, mirror_num, 1);
1848 	if (ret)
1849 		bio_endio(bio, ret);
1850 	return ret;
1851 }
1852 
1853 /*
1854  * extent_io.c submission hook. This does the right thing for csum calculation
1855  * on write, or reading the csums from the tree before a read
1856  */
1857 static int btrfs_submit_bio_hook(struct inode *inode, int rw, struct bio *bio,
1858 			  int mirror_num, unsigned long bio_flags,
1859 			  u64 bio_offset)
1860 {
1861 	struct btrfs_root *root = BTRFS_I(inode)->root;
1862 	int ret = 0;
1863 	int skip_sum;
1864 	int metadata = 0;
1865 	int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
1866 
1867 	skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
1868 
1869 	if (btrfs_is_free_space_inode(inode))
1870 		metadata = 2;
1871 
1872 	if (!(rw & REQ_WRITE)) {
1873 		ret = btrfs_bio_wq_end_io(root->fs_info, bio, metadata);
1874 		if (ret)
1875 			goto out;
1876 
1877 		if (bio_flags & EXTENT_BIO_COMPRESSED) {
1878 			ret = btrfs_submit_compressed_read(inode, bio,
1879 							   mirror_num,
1880 							   bio_flags);
1881 			goto out;
1882 		} else if (!skip_sum) {
1883 			ret = btrfs_lookup_bio_sums(root, inode, bio, NULL);
1884 			if (ret)
1885 				goto out;
1886 		}
1887 		goto mapit;
1888 	} else if (async && !skip_sum) {
1889 		/* csum items have already been cloned */
1890 		if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
1891 			goto mapit;
1892 		/* we're doing a write, do the async checksumming */
1893 		ret = btrfs_wq_submit_bio(BTRFS_I(inode)->root->fs_info,
1894 				   inode, rw, bio, mirror_num,
1895 				   bio_flags, bio_offset,
1896 				   __btrfs_submit_bio_start,
1897 				   __btrfs_submit_bio_done);
1898 		goto out;
1899 	} else if (!skip_sum) {
1900 		ret = btrfs_csum_one_bio(root, inode, bio, 0, 0);
1901 		if (ret)
1902 			goto out;
1903 	}
1904 
1905 mapit:
1906 	ret = btrfs_map_bio(root, rw, bio, mirror_num, 0);
1907 
1908 out:
1909 	if (ret < 0)
1910 		bio_endio(bio, ret);
1911 	return ret;
1912 }
1913 
1914 /*
1915  * given a list of ordered sums record them in the inode.  This happens
1916  * at IO completion time based on sums calculated at bio submission time.
1917  */
1918 static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
1919 			     struct inode *inode, u64 file_offset,
1920 			     struct list_head *list)
1921 {
1922 	struct btrfs_ordered_sum *sum;
1923 
1924 	list_for_each_entry(sum, list, list) {
1925 		trans->adding_csums = 1;
1926 		btrfs_csum_file_blocks(trans,
1927 		       BTRFS_I(inode)->root->fs_info->csum_root, sum);
1928 		trans->adding_csums = 0;
1929 	}
1930 	return 0;
1931 }
1932 
1933 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
1934 			      struct extent_state **cached_state)
1935 {
1936 	WARN_ON((end & (PAGE_CACHE_SIZE - 1)) == 0);
1937 	return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
1938 				   cached_state, GFP_NOFS);
1939 }
1940 
1941 /* see btrfs_writepage_start_hook for details on why this is required */
1942 struct btrfs_writepage_fixup {
1943 	struct page *page;
1944 	struct btrfs_work work;
1945 };
1946 
1947 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
1948 {
1949 	struct btrfs_writepage_fixup *fixup;
1950 	struct btrfs_ordered_extent *ordered;
1951 	struct extent_state *cached_state = NULL;
1952 	struct page *page;
1953 	struct inode *inode;
1954 	u64 page_start;
1955 	u64 page_end;
1956 	int ret;
1957 
1958 	fixup = container_of(work, struct btrfs_writepage_fixup, work);
1959 	page = fixup->page;
1960 again:
1961 	lock_page(page);
1962 	if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
1963 		ClearPageChecked(page);
1964 		goto out_page;
1965 	}
1966 
1967 	inode = page->mapping->host;
1968 	page_start = page_offset(page);
1969 	page_end = page_offset(page) + PAGE_CACHE_SIZE - 1;
1970 
1971 	lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end, 0,
1972 			 &cached_state);
1973 
1974 	/* already ordered? We're done */
1975 	if (PagePrivate2(page))
1976 		goto out;
1977 
1978 	ordered = btrfs_lookup_ordered_extent(inode, page_start);
1979 	if (ordered) {
1980 		unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
1981 				     page_end, &cached_state, GFP_NOFS);
1982 		unlock_page(page);
1983 		btrfs_start_ordered_extent(inode, ordered, 1);
1984 		btrfs_put_ordered_extent(ordered);
1985 		goto again;
1986 	}
1987 
1988 	ret = btrfs_delalloc_reserve_space(inode, PAGE_CACHE_SIZE);
1989 	if (ret) {
1990 		mapping_set_error(page->mapping, ret);
1991 		end_extent_writepage(page, ret, page_start, page_end);
1992 		ClearPageChecked(page);
1993 		goto out;
1994 	 }
1995 
1996 	btrfs_set_extent_delalloc(inode, page_start, page_end, &cached_state);
1997 	ClearPageChecked(page);
1998 	set_page_dirty(page);
1999 out:
2000 	unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
2001 			     &cached_state, GFP_NOFS);
2002 out_page:
2003 	unlock_page(page);
2004 	page_cache_release(page);
2005 	kfree(fixup);
2006 }
2007 
2008 /*
2009  * There are a few paths in the higher layers of the kernel that directly
2010  * set the page dirty bit without asking the filesystem if it is a
2011  * good idea.  This causes problems because we want to make sure COW
2012  * properly happens and the data=ordered rules are followed.
2013  *
2014  * In our case any range that doesn't have the ORDERED bit set
2015  * hasn't been properly setup for IO.  We kick off an async process
2016  * to fix it up.  The async helper will wait for ordered extents, set
2017  * the delalloc bit and make it safe to write the page.
2018  */
2019 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
2020 {
2021 	struct inode *inode = page->mapping->host;
2022 	struct btrfs_writepage_fixup *fixup;
2023 	struct btrfs_root *root = BTRFS_I(inode)->root;
2024 
2025 	/* this page is properly in the ordered list */
2026 	if (TestClearPagePrivate2(page))
2027 		return 0;
2028 
2029 	if (PageChecked(page))
2030 		return -EAGAIN;
2031 
2032 	fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2033 	if (!fixup)
2034 		return -EAGAIN;
2035 
2036 	SetPageChecked(page);
2037 	page_cache_get(page);
2038 	btrfs_init_work(&fixup->work, btrfs_fixup_helper,
2039 			btrfs_writepage_fixup_worker, NULL, NULL);
2040 	fixup->page = page;
2041 	btrfs_queue_work(root->fs_info->fixup_workers, &fixup->work);
2042 	return -EBUSY;
2043 }
2044 
2045 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2046 				       struct inode *inode, u64 file_pos,
2047 				       u64 disk_bytenr, u64 disk_num_bytes,
2048 				       u64 num_bytes, u64 ram_bytes,
2049 				       u8 compression, u8 encryption,
2050 				       u16 other_encoding, int extent_type)
2051 {
2052 	struct btrfs_root *root = BTRFS_I(inode)->root;
2053 	struct btrfs_file_extent_item *fi;
2054 	struct btrfs_path *path;
2055 	struct extent_buffer *leaf;
2056 	struct btrfs_key ins;
2057 	int extent_inserted = 0;
2058 	int ret;
2059 
2060 	path = btrfs_alloc_path();
2061 	if (!path)
2062 		return -ENOMEM;
2063 
2064 	/*
2065 	 * we may be replacing one extent in the tree with another.
2066 	 * The new extent is pinned in the extent map, and we don't want
2067 	 * to drop it from the cache until it is completely in the btree.
2068 	 *
2069 	 * So, tell btrfs_drop_extents to leave this extent in the cache.
2070 	 * the caller is expected to unpin it and allow it to be merged
2071 	 * with the others.
2072 	 */
2073 	ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
2074 				   file_pos + num_bytes, NULL, 0,
2075 				   1, sizeof(*fi), &extent_inserted);
2076 	if (ret)
2077 		goto out;
2078 
2079 	if (!extent_inserted) {
2080 		ins.objectid = btrfs_ino(inode);
2081 		ins.offset = file_pos;
2082 		ins.type = BTRFS_EXTENT_DATA_KEY;
2083 
2084 		path->leave_spinning = 1;
2085 		ret = btrfs_insert_empty_item(trans, root, path, &ins,
2086 					      sizeof(*fi));
2087 		if (ret)
2088 			goto out;
2089 	}
2090 	leaf = path->nodes[0];
2091 	fi = btrfs_item_ptr(leaf, path->slots[0],
2092 			    struct btrfs_file_extent_item);
2093 	btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2094 	btrfs_set_file_extent_type(leaf, fi, extent_type);
2095 	btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
2096 	btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
2097 	btrfs_set_file_extent_offset(leaf, fi, 0);
2098 	btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2099 	btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
2100 	btrfs_set_file_extent_compression(leaf, fi, compression);
2101 	btrfs_set_file_extent_encryption(leaf, fi, encryption);
2102 	btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
2103 
2104 	btrfs_mark_buffer_dirty(leaf);
2105 	btrfs_release_path(path);
2106 
2107 	inode_add_bytes(inode, num_bytes);
2108 
2109 	ins.objectid = disk_bytenr;
2110 	ins.offset = disk_num_bytes;
2111 	ins.type = BTRFS_EXTENT_ITEM_KEY;
2112 	ret = btrfs_alloc_reserved_file_extent(trans, root,
2113 					root->root_key.objectid,
2114 					btrfs_ino(inode), file_pos, &ins);
2115 out:
2116 	btrfs_free_path(path);
2117 
2118 	return ret;
2119 }
2120 
2121 /* snapshot-aware defrag */
2122 struct sa_defrag_extent_backref {
2123 	struct rb_node node;
2124 	struct old_sa_defrag_extent *old;
2125 	u64 root_id;
2126 	u64 inum;
2127 	u64 file_pos;
2128 	u64 extent_offset;
2129 	u64 num_bytes;
2130 	u64 generation;
2131 };
2132 
2133 struct old_sa_defrag_extent {
2134 	struct list_head list;
2135 	struct new_sa_defrag_extent *new;
2136 
2137 	u64 extent_offset;
2138 	u64 bytenr;
2139 	u64 offset;
2140 	u64 len;
2141 	int count;
2142 };
2143 
2144 struct new_sa_defrag_extent {
2145 	struct rb_root root;
2146 	struct list_head head;
2147 	struct btrfs_path *path;
2148 	struct inode *inode;
2149 	u64 file_pos;
2150 	u64 len;
2151 	u64 bytenr;
2152 	u64 disk_len;
2153 	u8 compress_type;
2154 };
2155 
2156 static int backref_comp(struct sa_defrag_extent_backref *b1,
2157 			struct sa_defrag_extent_backref *b2)
2158 {
2159 	if (b1->root_id < b2->root_id)
2160 		return -1;
2161 	else if (b1->root_id > b2->root_id)
2162 		return 1;
2163 
2164 	if (b1->inum < b2->inum)
2165 		return -1;
2166 	else if (b1->inum > b2->inum)
2167 		return 1;
2168 
2169 	if (b1->file_pos < b2->file_pos)
2170 		return -1;
2171 	else if (b1->file_pos > b2->file_pos)
2172 		return 1;
2173 
2174 	/*
2175 	 * [------------------------------] ===> (a range of space)
2176 	 *     |<--->|   |<---->| =============> (fs/file tree A)
2177 	 * |<---------------------------->| ===> (fs/file tree B)
2178 	 *
2179 	 * A range of space can refer to two file extents in one tree while
2180 	 * refer to only one file extent in another tree.
2181 	 *
2182 	 * So we may process a disk offset more than one time(two extents in A)
2183 	 * and locate at the same extent(one extent in B), then insert two same
2184 	 * backrefs(both refer to the extent in B).
2185 	 */
2186 	return 0;
2187 }
2188 
2189 static void backref_insert(struct rb_root *root,
2190 			   struct sa_defrag_extent_backref *backref)
2191 {
2192 	struct rb_node **p = &root->rb_node;
2193 	struct rb_node *parent = NULL;
2194 	struct sa_defrag_extent_backref *entry;
2195 	int ret;
2196 
2197 	while (*p) {
2198 		parent = *p;
2199 		entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
2200 
2201 		ret = backref_comp(backref, entry);
2202 		if (ret < 0)
2203 			p = &(*p)->rb_left;
2204 		else
2205 			p = &(*p)->rb_right;
2206 	}
2207 
2208 	rb_link_node(&backref->node, parent, p);
2209 	rb_insert_color(&backref->node, root);
2210 }
2211 
2212 /*
2213  * Note the backref might has changed, and in this case we just return 0.
2214  */
2215 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
2216 				       void *ctx)
2217 {
2218 	struct btrfs_file_extent_item *extent;
2219 	struct btrfs_fs_info *fs_info;
2220 	struct old_sa_defrag_extent *old = ctx;
2221 	struct new_sa_defrag_extent *new = old->new;
2222 	struct btrfs_path *path = new->path;
2223 	struct btrfs_key key;
2224 	struct btrfs_root *root;
2225 	struct sa_defrag_extent_backref *backref;
2226 	struct extent_buffer *leaf;
2227 	struct inode *inode = new->inode;
2228 	int slot;
2229 	int ret;
2230 	u64 extent_offset;
2231 	u64 num_bytes;
2232 
2233 	if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
2234 	    inum == btrfs_ino(inode))
2235 		return 0;
2236 
2237 	key.objectid = root_id;
2238 	key.type = BTRFS_ROOT_ITEM_KEY;
2239 	key.offset = (u64)-1;
2240 
2241 	fs_info = BTRFS_I(inode)->root->fs_info;
2242 	root = btrfs_read_fs_root_no_name(fs_info, &key);
2243 	if (IS_ERR(root)) {
2244 		if (PTR_ERR(root) == -ENOENT)
2245 			return 0;
2246 		WARN_ON(1);
2247 		pr_debug("inum=%llu, offset=%llu, root_id=%llu\n",
2248 			 inum, offset, root_id);
2249 		return PTR_ERR(root);
2250 	}
2251 
2252 	key.objectid = inum;
2253 	key.type = BTRFS_EXTENT_DATA_KEY;
2254 	if (offset > (u64)-1 << 32)
2255 		key.offset = 0;
2256 	else
2257 		key.offset = offset;
2258 
2259 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2260 	if (WARN_ON(ret < 0))
2261 		return ret;
2262 	ret = 0;
2263 
2264 	while (1) {
2265 		cond_resched();
2266 
2267 		leaf = path->nodes[0];
2268 		slot = path->slots[0];
2269 
2270 		if (slot >= btrfs_header_nritems(leaf)) {
2271 			ret = btrfs_next_leaf(root, path);
2272 			if (ret < 0) {
2273 				goto out;
2274 			} else if (ret > 0) {
2275 				ret = 0;
2276 				goto out;
2277 			}
2278 			continue;
2279 		}
2280 
2281 		path->slots[0]++;
2282 
2283 		btrfs_item_key_to_cpu(leaf, &key, slot);
2284 
2285 		if (key.objectid > inum)
2286 			goto out;
2287 
2288 		if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
2289 			continue;
2290 
2291 		extent = btrfs_item_ptr(leaf, slot,
2292 					struct btrfs_file_extent_item);
2293 
2294 		if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
2295 			continue;
2296 
2297 		/*
2298 		 * 'offset' refers to the exact key.offset,
2299 		 * NOT the 'offset' field in btrfs_extent_data_ref, ie.
2300 		 * (key.offset - extent_offset).
2301 		 */
2302 		if (key.offset != offset)
2303 			continue;
2304 
2305 		extent_offset = btrfs_file_extent_offset(leaf, extent);
2306 		num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2307 
2308 		if (extent_offset >= old->extent_offset + old->offset +
2309 		    old->len || extent_offset + num_bytes <=
2310 		    old->extent_offset + old->offset)
2311 			continue;
2312 		break;
2313 	}
2314 
2315 	backref = kmalloc(sizeof(*backref), GFP_NOFS);
2316 	if (!backref) {
2317 		ret = -ENOENT;
2318 		goto out;
2319 	}
2320 
2321 	backref->root_id = root_id;
2322 	backref->inum = inum;
2323 	backref->file_pos = offset;
2324 	backref->num_bytes = num_bytes;
2325 	backref->extent_offset = extent_offset;
2326 	backref->generation = btrfs_file_extent_generation(leaf, extent);
2327 	backref->old = old;
2328 	backref_insert(&new->root, backref);
2329 	old->count++;
2330 out:
2331 	btrfs_release_path(path);
2332 	WARN_ON(ret);
2333 	return ret;
2334 }
2335 
2336 static noinline bool record_extent_backrefs(struct btrfs_path *path,
2337 				   struct new_sa_defrag_extent *new)
2338 {
2339 	struct btrfs_fs_info *fs_info = BTRFS_I(new->inode)->root->fs_info;
2340 	struct old_sa_defrag_extent *old, *tmp;
2341 	int ret;
2342 
2343 	new->path = path;
2344 
2345 	list_for_each_entry_safe(old, tmp, &new->head, list) {
2346 		ret = iterate_inodes_from_logical(old->bytenr +
2347 						  old->extent_offset, fs_info,
2348 						  path, record_one_backref,
2349 						  old);
2350 		if (ret < 0 && ret != -ENOENT)
2351 			return false;
2352 
2353 		/* no backref to be processed for this extent */
2354 		if (!old->count) {
2355 			list_del(&old->list);
2356 			kfree(old);
2357 		}
2358 	}
2359 
2360 	if (list_empty(&new->head))
2361 		return false;
2362 
2363 	return true;
2364 }
2365 
2366 static int relink_is_mergable(struct extent_buffer *leaf,
2367 			      struct btrfs_file_extent_item *fi,
2368 			      struct new_sa_defrag_extent *new)
2369 {
2370 	if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
2371 		return 0;
2372 
2373 	if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2374 		return 0;
2375 
2376 	if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
2377 		return 0;
2378 
2379 	if (btrfs_file_extent_encryption(leaf, fi) ||
2380 	    btrfs_file_extent_other_encoding(leaf, fi))
2381 		return 0;
2382 
2383 	return 1;
2384 }
2385 
2386 /*
2387  * Note the backref might has changed, and in this case we just return 0.
2388  */
2389 static noinline int relink_extent_backref(struct btrfs_path *path,
2390 				 struct sa_defrag_extent_backref *prev,
2391 				 struct sa_defrag_extent_backref *backref)
2392 {
2393 	struct btrfs_file_extent_item *extent;
2394 	struct btrfs_file_extent_item *item;
2395 	struct btrfs_ordered_extent *ordered;
2396 	struct btrfs_trans_handle *trans;
2397 	struct btrfs_fs_info *fs_info;
2398 	struct btrfs_root *root;
2399 	struct btrfs_key key;
2400 	struct extent_buffer *leaf;
2401 	struct old_sa_defrag_extent *old = backref->old;
2402 	struct new_sa_defrag_extent *new = old->new;
2403 	struct inode *src_inode = new->inode;
2404 	struct inode *inode;
2405 	struct extent_state *cached = NULL;
2406 	int ret = 0;
2407 	u64 start;
2408 	u64 len;
2409 	u64 lock_start;
2410 	u64 lock_end;
2411 	bool merge = false;
2412 	int index;
2413 
2414 	if (prev && prev->root_id == backref->root_id &&
2415 	    prev->inum == backref->inum &&
2416 	    prev->file_pos + prev->num_bytes == backref->file_pos)
2417 		merge = true;
2418 
2419 	/* step 1: get root */
2420 	key.objectid = backref->root_id;
2421 	key.type = BTRFS_ROOT_ITEM_KEY;
2422 	key.offset = (u64)-1;
2423 
2424 	fs_info = BTRFS_I(src_inode)->root->fs_info;
2425 	index = srcu_read_lock(&fs_info->subvol_srcu);
2426 
2427 	root = btrfs_read_fs_root_no_name(fs_info, &key);
2428 	if (IS_ERR(root)) {
2429 		srcu_read_unlock(&fs_info->subvol_srcu, index);
2430 		if (PTR_ERR(root) == -ENOENT)
2431 			return 0;
2432 		return PTR_ERR(root);
2433 	}
2434 
2435 	if (btrfs_root_readonly(root)) {
2436 		srcu_read_unlock(&fs_info->subvol_srcu, index);
2437 		return 0;
2438 	}
2439 
2440 	/* step 2: get inode */
2441 	key.objectid = backref->inum;
2442 	key.type = BTRFS_INODE_ITEM_KEY;
2443 	key.offset = 0;
2444 
2445 	inode = btrfs_iget(fs_info->sb, &key, root, NULL);
2446 	if (IS_ERR(inode)) {
2447 		srcu_read_unlock(&fs_info->subvol_srcu, index);
2448 		return 0;
2449 	}
2450 
2451 	srcu_read_unlock(&fs_info->subvol_srcu, index);
2452 
2453 	/* step 3: relink backref */
2454 	lock_start = backref->file_pos;
2455 	lock_end = backref->file_pos + backref->num_bytes - 1;
2456 	lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2457 			 0, &cached);
2458 
2459 	ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
2460 	if (ordered) {
2461 		btrfs_put_ordered_extent(ordered);
2462 		goto out_unlock;
2463 	}
2464 
2465 	trans = btrfs_join_transaction(root);
2466 	if (IS_ERR(trans)) {
2467 		ret = PTR_ERR(trans);
2468 		goto out_unlock;
2469 	}
2470 
2471 	key.objectid = backref->inum;
2472 	key.type = BTRFS_EXTENT_DATA_KEY;
2473 	key.offset = backref->file_pos;
2474 
2475 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2476 	if (ret < 0) {
2477 		goto out_free_path;
2478 	} else if (ret > 0) {
2479 		ret = 0;
2480 		goto out_free_path;
2481 	}
2482 
2483 	extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
2484 				struct btrfs_file_extent_item);
2485 
2486 	if (btrfs_file_extent_generation(path->nodes[0], extent) !=
2487 	    backref->generation)
2488 		goto out_free_path;
2489 
2490 	btrfs_release_path(path);
2491 
2492 	start = backref->file_pos;
2493 	if (backref->extent_offset < old->extent_offset + old->offset)
2494 		start += old->extent_offset + old->offset -
2495 			 backref->extent_offset;
2496 
2497 	len = min(backref->extent_offset + backref->num_bytes,
2498 		  old->extent_offset + old->offset + old->len);
2499 	len -= max(backref->extent_offset, old->extent_offset + old->offset);
2500 
2501 	ret = btrfs_drop_extents(trans, root, inode, start,
2502 				 start + len, 1);
2503 	if (ret)
2504 		goto out_free_path;
2505 again:
2506 	key.objectid = btrfs_ino(inode);
2507 	key.type = BTRFS_EXTENT_DATA_KEY;
2508 	key.offset = start;
2509 
2510 	path->leave_spinning = 1;
2511 	if (merge) {
2512 		struct btrfs_file_extent_item *fi;
2513 		u64 extent_len;
2514 		struct btrfs_key found_key;
2515 
2516 		ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2517 		if (ret < 0)
2518 			goto out_free_path;
2519 
2520 		path->slots[0]--;
2521 		leaf = path->nodes[0];
2522 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2523 
2524 		fi = btrfs_item_ptr(leaf, path->slots[0],
2525 				    struct btrfs_file_extent_item);
2526 		extent_len = btrfs_file_extent_num_bytes(leaf, fi);
2527 
2528 		if (extent_len + found_key.offset == start &&
2529 		    relink_is_mergable(leaf, fi, new)) {
2530 			btrfs_set_file_extent_num_bytes(leaf, fi,
2531 							extent_len + len);
2532 			btrfs_mark_buffer_dirty(leaf);
2533 			inode_add_bytes(inode, len);
2534 
2535 			ret = 1;
2536 			goto out_free_path;
2537 		} else {
2538 			merge = false;
2539 			btrfs_release_path(path);
2540 			goto again;
2541 		}
2542 	}
2543 
2544 	ret = btrfs_insert_empty_item(trans, root, path, &key,
2545 					sizeof(*extent));
2546 	if (ret) {
2547 		btrfs_abort_transaction(trans, root, ret);
2548 		goto out_free_path;
2549 	}
2550 
2551 	leaf = path->nodes[0];
2552 	item = btrfs_item_ptr(leaf, path->slots[0],
2553 				struct btrfs_file_extent_item);
2554 	btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
2555 	btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
2556 	btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
2557 	btrfs_set_file_extent_num_bytes(leaf, item, len);
2558 	btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
2559 	btrfs_set_file_extent_generation(leaf, item, trans->transid);
2560 	btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
2561 	btrfs_set_file_extent_compression(leaf, item, new->compress_type);
2562 	btrfs_set_file_extent_encryption(leaf, item, 0);
2563 	btrfs_set_file_extent_other_encoding(leaf, item, 0);
2564 
2565 	btrfs_mark_buffer_dirty(leaf);
2566 	inode_add_bytes(inode, len);
2567 	btrfs_release_path(path);
2568 
2569 	ret = btrfs_inc_extent_ref(trans, root, new->bytenr,
2570 			new->disk_len, 0,
2571 			backref->root_id, backref->inum,
2572 			new->file_pos, 0);	/* start - extent_offset */
2573 	if (ret) {
2574 		btrfs_abort_transaction(trans, root, ret);
2575 		goto out_free_path;
2576 	}
2577 
2578 	ret = 1;
2579 out_free_path:
2580 	btrfs_release_path(path);
2581 	path->leave_spinning = 0;
2582 	btrfs_end_transaction(trans, root);
2583 out_unlock:
2584 	unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2585 			     &cached, GFP_NOFS);
2586 	iput(inode);
2587 	return ret;
2588 }
2589 
2590 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
2591 {
2592 	struct old_sa_defrag_extent *old, *tmp;
2593 
2594 	if (!new)
2595 		return;
2596 
2597 	list_for_each_entry_safe(old, tmp, &new->head, list) {
2598 		list_del(&old->list);
2599 		kfree(old);
2600 	}
2601 	kfree(new);
2602 }
2603 
2604 static void relink_file_extents(struct new_sa_defrag_extent *new)
2605 {
2606 	struct btrfs_path *path;
2607 	struct sa_defrag_extent_backref *backref;
2608 	struct sa_defrag_extent_backref *prev = NULL;
2609 	struct inode *inode;
2610 	struct btrfs_root *root;
2611 	struct rb_node *node;
2612 	int ret;
2613 
2614 	inode = new->inode;
2615 	root = BTRFS_I(inode)->root;
2616 
2617 	path = btrfs_alloc_path();
2618 	if (!path)
2619 		return;
2620 
2621 	if (!record_extent_backrefs(path, new)) {
2622 		btrfs_free_path(path);
2623 		goto out;
2624 	}
2625 	btrfs_release_path(path);
2626 
2627 	while (1) {
2628 		node = rb_first(&new->root);
2629 		if (!node)
2630 			break;
2631 		rb_erase(node, &new->root);
2632 
2633 		backref = rb_entry(node, struct sa_defrag_extent_backref, node);
2634 
2635 		ret = relink_extent_backref(path, prev, backref);
2636 		WARN_ON(ret < 0);
2637 
2638 		kfree(prev);
2639 
2640 		if (ret == 1)
2641 			prev = backref;
2642 		else
2643 			prev = NULL;
2644 		cond_resched();
2645 	}
2646 	kfree(prev);
2647 
2648 	btrfs_free_path(path);
2649 out:
2650 	free_sa_defrag_extent(new);
2651 
2652 	atomic_dec(&root->fs_info->defrag_running);
2653 	wake_up(&root->fs_info->transaction_wait);
2654 }
2655 
2656 static struct new_sa_defrag_extent *
2657 record_old_file_extents(struct inode *inode,
2658 			struct btrfs_ordered_extent *ordered)
2659 {
2660 	struct btrfs_root *root = BTRFS_I(inode)->root;
2661 	struct btrfs_path *path;
2662 	struct btrfs_key key;
2663 	struct old_sa_defrag_extent *old;
2664 	struct new_sa_defrag_extent *new;
2665 	int ret;
2666 
2667 	new = kmalloc(sizeof(*new), GFP_NOFS);
2668 	if (!new)
2669 		return NULL;
2670 
2671 	new->inode = inode;
2672 	new->file_pos = ordered->file_offset;
2673 	new->len = ordered->len;
2674 	new->bytenr = ordered->start;
2675 	new->disk_len = ordered->disk_len;
2676 	new->compress_type = ordered->compress_type;
2677 	new->root = RB_ROOT;
2678 	INIT_LIST_HEAD(&new->head);
2679 
2680 	path = btrfs_alloc_path();
2681 	if (!path)
2682 		goto out_kfree;
2683 
2684 	key.objectid = btrfs_ino(inode);
2685 	key.type = BTRFS_EXTENT_DATA_KEY;
2686 	key.offset = new->file_pos;
2687 
2688 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2689 	if (ret < 0)
2690 		goto out_free_path;
2691 	if (ret > 0 && path->slots[0] > 0)
2692 		path->slots[0]--;
2693 
2694 	/* find out all the old extents for the file range */
2695 	while (1) {
2696 		struct btrfs_file_extent_item *extent;
2697 		struct extent_buffer *l;
2698 		int slot;
2699 		u64 num_bytes;
2700 		u64 offset;
2701 		u64 end;
2702 		u64 disk_bytenr;
2703 		u64 extent_offset;
2704 
2705 		l = path->nodes[0];
2706 		slot = path->slots[0];
2707 
2708 		if (slot >= btrfs_header_nritems(l)) {
2709 			ret = btrfs_next_leaf(root, path);
2710 			if (ret < 0)
2711 				goto out_free_path;
2712 			else if (ret > 0)
2713 				break;
2714 			continue;
2715 		}
2716 
2717 		btrfs_item_key_to_cpu(l, &key, slot);
2718 
2719 		if (key.objectid != btrfs_ino(inode))
2720 			break;
2721 		if (key.type != BTRFS_EXTENT_DATA_KEY)
2722 			break;
2723 		if (key.offset >= new->file_pos + new->len)
2724 			break;
2725 
2726 		extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
2727 
2728 		num_bytes = btrfs_file_extent_num_bytes(l, extent);
2729 		if (key.offset + num_bytes < new->file_pos)
2730 			goto next;
2731 
2732 		disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
2733 		if (!disk_bytenr)
2734 			goto next;
2735 
2736 		extent_offset = btrfs_file_extent_offset(l, extent);
2737 
2738 		old = kmalloc(sizeof(*old), GFP_NOFS);
2739 		if (!old)
2740 			goto out_free_path;
2741 
2742 		offset = max(new->file_pos, key.offset);
2743 		end = min(new->file_pos + new->len, key.offset + num_bytes);
2744 
2745 		old->bytenr = disk_bytenr;
2746 		old->extent_offset = extent_offset;
2747 		old->offset = offset - key.offset;
2748 		old->len = end - offset;
2749 		old->new = new;
2750 		old->count = 0;
2751 		list_add_tail(&old->list, &new->head);
2752 next:
2753 		path->slots[0]++;
2754 		cond_resched();
2755 	}
2756 
2757 	btrfs_free_path(path);
2758 	atomic_inc(&root->fs_info->defrag_running);
2759 
2760 	return new;
2761 
2762 out_free_path:
2763 	btrfs_free_path(path);
2764 out_kfree:
2765 	free_sa_defrag_extent(new);
2766 	return NULL;
2767 }
2768 
2769 static void btrfs_release_delalloc_bytes(struct btrfs_root *root,
2770 					 u64 start, u64 len)
2771 {
2772 	struct btrfs_block_group_cache *cache;
2773 
2774 	cache = btrfs_lookup_block_group(root->fs_info, start);
2775 	ASSERT(cache);
2776 
2777 	spin_lock(&cache->lock);
2778 	cache->delalloc_bytes -= len;
2779 	spin_unlock(&cache->lock);
2780 
2781 	btrfs_put_block_group(cache);
2782 }
2783 
2784 /* as ordered data IO finishes, this gets called so we can finish
2785  * an ordered extent if the range of bytes in the file it covers are
2786  * fully written.
2787  */
2788 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
2789 {
2790 	struct inode *inode = ordered_extent->inode;
2791 	struct btrfs_root *root = BTRFS_I(inode)->root;
2792 	struct btrfs_trans_handle *trans = NULL;
2793 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2794 	struct extent_state *cached_state = NULL;
2795 	struct new_sa_defrag_extent *new = NULL;
2796 	int compress_type = 0;
2797 	int ret = 0;
2798 	u64 logical_len = ordered_extent->len;
2799 	bool nolock;
2800 	bool truncated = false;
2801 
2802 	nolock = btrfs_is_free_space_inode(inode);
2803 
2804 	if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
2805 		ret = -EIO;
2806 		goto out;
2807 	}
2808 
2809 	btrfs_free_io_failure_record(inode, ordered_extent->file_offset,
2810 				     ordered_extent->file_offset +
2811 				     ordered_extent->len - 1);
2812 
2813 	if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
2814 		truncated = true;
2815 		logical_len = ordered_extent->truncated_len;
2816 		/* Truncated the entire extent, don't bother adding */
2817 		if (!logical_len)
2818 			goto out;
2819 	}
2820 
2821 	if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
2822 		BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
2823 		btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2824 		if (nolock)
2825 			trans = btrfs_join_transaction_nolock(root);
2826 		else
2827 			trans = btrfs_join_transaction(root);
2828 		if (IS_ERR(trans)) {
2829 			ret = PTR_ERR(trans);
2830 			trans = NULL;
2831 			goto out;
2832 		}
2833 		trans->block_rsv = &root->fs_info->delalloc_block_rsv;
2834 		ret = btrfs_update_inode_fallback(trans, root, inode);
2835 		if (ret) /* -ENOMEM or corruption */
2836 			btrfs_abort_transaction(trans, root, ret);
2837 		goto out;
2838 	}
2839 
2840 	lock_extent_bits(io_tree, ordered_extent->file_offset,
2841 			 ordered_extent->file_offset + ordered_extent->len - 1,
2842 			 0, &cached_state);
2843 
2844 	ret = test_range_bit(io_tree, ordered_extent->file_offset,
2845 			ordered_extent->file_offset + ordered_extent->len - 1,
2846 			EXTENT_DEFRAG, 1, cached_state);
2847 	if (ret) {
2848 		u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
2849 		if (0 && last_snapshot >= BTRFS_I(inode)->generation)
2850 			/* the inode is shared */
2851 			new = record_old_file_extents(inode, ordered_extent);
2852 
2853 		clear_extent_bit(io_tree, ordered_extent->file_offset,
2854 			ordered_extent->file_offset + ordered_extent->len - 1,
2855 			EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS);
2856 	}
2857 
2858 	if (nolock)
2859 		trans = btrfs_join_transaction_nolock(root);
2860 	else
2861 		trans = btrfs_join_transaction(root);
2862 	if (IS_ERR(trans)) {
2863 		ret = PTR_ERR(trans);
2864 		trans = NULL;
2865 		goto out_unlock;
2866 	}
2867 
2868 	trans->block_rsv = &root->fs_info->delalloc_block_rsv;
2869 
2870 	if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
2871 		compress_type = ordered_extent->compress_type;
2872 	if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
2873 		BUG_ON(compress_type);
2874 		ret = btrfs_mark_extent_written(trans, inode,
2875 						ordered_extent->file_offset,
2876 						ordered_extent->file_offset +
2877 						logical_len);
2878 	} else {
2879 		BUG_ON(root == root->fs_info->tree_root);
2880 		ret = insert_reserved_file_extent(trans, inode,
2881 						ordered_extent->file_offset,
2882 						ordered_extent->start,
2883 						ordered_extent->disk_len,
2884 						logical_len, logical_len,
2885 						compress_type, 0, 0,
2886 						BTRFS_FILE_EXTENT_REG);
2887 		if (!ret)
2888 			btrfs_release_delalloc_bytes(root,
2889 						     ordered_extent->start,
2890 						     ordered_extent->disk_len);
2891 	}
2892 	unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
2893 			   ordered_extent->file_offset, ordered_extent->len,
2894 			   trans->transid);
2895 	if (ret < 0) {
2896 		btrfs_abort_transaction(trans, root, ret);
2897 		goto out_unlock;
2898 	}
2899 
2900 	add_pending_csums(trans, inode, ordered_extent->file_offset,
2901 			  &ordered_extent->list);
2902 
2903 	btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2904 	ret = btrfs_update_inode_fallback(trans, root, inode);
2905 	if (ret) { /* -ENOMEM or corruption */
2906 		btrfs_abort_transaction(trans, root, ret);
2907 		goto out_unlock;
2908 	}
2909 	ret = 0;
2910 out_unlock:
2911 	unlock_extent_cached(io_tree, ordered_extent->file_offset,
2912 			     ordered_extent->file_offset +
2913 			     ordered_extent->len - 1, &cached_state, GFP_NOFS);
2914 out:
2915 	if (root != root->fs_info->tree_root)
2916 		btrfs_delalloc_release_metadata(inode, ordered_extent->len);
2917 	if (trans)
2918 		btrfs_end_transaction(trans, root);
2919 
2920 	if (ret || truncated) {
2921 		u64 start, end;
2922 
2923 		if (truncated)
2924 			start = ordered_extent->file_offset + logical_len;
2925 		else
2926 			start = ordered_extent->file_offset;
2927 		end = ordered_extent->file_offset + ordered_extent->len - 1;
2928 		clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS);
2929 
2930 		/* Drop the cache for the part of the extent we didn't write. */
2931 		btrfs_drop_extent_cache(inode, start, end, 0);
2932 
2933 		/*
2934 		 * If the ordered extent had an IOERR or something else went
2935 		 * wrong we need to return the space for this ordered extent
2936 		 * back to the allocator.  We only free the extent in the
2937 		 * truncated case if we didn't write out the extent at all.
2938 		 */
2939 		if ((ret || !logical_len) &&
2940 		    !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
2941 		    !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
2942 			btrfs_free_reserved_extent(root, ordered_extent->start,
2943 						   ordered_extent->disk_len, 1);
2944 	}
2945 
2946 
2947 	/*
2948 	 * This needs to be done to make sure anybody waiting knows we are done
2949 	 * updating everything for this ordered extent.
2950 	 */
2951 	btrfs_remove_ordered_extent(inode, ordered_extent);
2952 
2953 	/* for snapshot-aware defrag */
2954 	if (new) {
2955 		if (ret) {
2956 			free_sa_defrag_extent(new);
2957 			atomic_dec(&root->fs_info->defrag_running);
2958 		} else {
2959 			relink_file_extents(new);
2960 		}
2961 	}
2962 
2963 	/* once for us */
2964 	btrfs_put_ordered_extent(ordered_extent);
2965 	/* once for the tree */
2966 	btrfs_put_ordered_extent(ordered_extent);
2967 
2968 	return ret;
2969 }
2970 
2971 static void finish_ordered_fn(struct btrfs_work *work)
2972 {
2973 	struct btrfs_ordered_extent *ordered_extent;
2974 	ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
2975 	btrfs_finish_ordered_io(ordered_extent);
2976 }
2977 
2978 static int btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
2979 				struct extent_state *state, int uptodate)
2980 {
2981 	struct inode *inode = page->mapping->host;
2982 	struct btrfs_root *root = BTRFS_I(inode)->root;
2983 	struct btrfs_ordered_extent *ordered_extent = NULL;
2984 	struct btrfs_workqueue *wq;
2985 	btrfs_work_func_t func;
2986 
2987 	trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
2988 
2989 	ClearPagePrivate2(page);
2990 	if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
2991 					    end - start + 1, uptodate))
2992 		return 0;
2993 
2994 	if (btrfs_is_free_space_inode(inode)) {
2995 		wq = root->fs_info->endio_freespace_worker;
2996 		func = btrfs_freespace_write_helper;
2997 	} else {
2998 		wq = root->fs_info->endio_write_workers;
2999 		func = btrfs_endio_write_helper;
3000 	}
3001 
3002 	btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
3003 			NULL);
3004 	btrfs_queue_work(wq, &ordered_extent->work);
3005 
3006 	return 0;
3007 }
3008 
3009 static int __readpage_endio_check(struct inode *inode,
3010 				  struct btrfs_io_bio *io_bio,
3011 				  int icsum, struct page *page,
3012 				  int pgoff, u64 start, size_t len)
3013 {
3014 	char *kaddr;
3015 	u32 csum_expected;
3016 	u32 csum = ~(u32)0;
3017 	static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
3018 				      DEFAULT_RATELIMIT_BURST);
3019 
3020 	csum_expected = *(((u32 *)io_bio->csum) + icsum);
3021 
3022 	kaddr = kmap_atomic(page);
3023 	csum = btrfs_csum_data(kaddr + pgoff, csum,  len);
3024 	btrfs_csum_final(csum, (char *)&csum);
3025 	if (csum != csum_expected)
3026 		goto zeroit;
3027 
3028 	kunmap_atomic(kaddr);
3029 	return 0;
3030 zeroit:
3031 	if (__ratelimit(&_rs))
3032 		btrfs_warn(BTRFS_I(inode)->root->fs_info,
3033 			   "csum failed ino %llu off %llu csum %u expected csum %u",
3034 			   btrfs_ino(inode), start, csum, csum_expected);
3035 	memset(kaddr + pgoff, 1, len);
3036 	flush_dcache_page(page);
3037 	kunmap_atomic(kaddr);
3038 	if (csum_expected == 0)
3039 		return 0;
3040 	return -EIO;
3041 }
3042 
3043 /*
3044  * when reads are done, we need to check csums to verify the data is correct
3045  * if there's a match, we allow the bio to finish.  If not, the code in
3046  * extent_io.c will try to find good copies for us.
3047  */
3048 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
3049 				      u64 phy_offset, struct page *page,
3050 				      u64 start, u64 end, int mirror)
3051 {
3052 	size_t offset = start - page_offset(page);
3053 	struct inode *inode = page->mapping->host;
3054 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3055 	struct btrfs_root *root = BTRFS_I(inode)->root;
3056 
3057 	if (PageChecked(page)) {
3058 		ClearPageChecked(page);
3059 		return 0;
3060 	}
3061 
3062 	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
3063 		return 0;
3064 
3065 	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
3066 	    test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
3067 		clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM,
3068 				  GFP_NOFS);
3069 		return 0;
3070 	}
3071 
3072 	phy_offset >>= inode->i_sb->s_blocksize_bits;
3073 	return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
3074 				      start, (size_t)(end - start + 1));
3075 }
3076 
3077 struct delayed_iput {
3078 	struct list_head list;
3079 	struct inode *inode;
3080 };
3081 
3082 /* JDM: If this is fs-wide, why can't we add a pointer to
3083  * btrfs_inode instead and avoid the allocation? */
3084 void btrfs_add_delayed_iput(struct inode *inode)
3085 {
3086 	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
3087 	struct delayed_iput *delayed;
3088 
3089 	if (atomic_add_unless(&inode->i_count, -1, 1))
3090 		return;
3091 
3092 	delayed = kmalloc(sizeof(*delayed), GFP_NOFS | __GFP_NOFAIL);
3093 	delayed->inode = inode;
3094 
3095 	spin_lock(&fs_info->delayed_iput_lock);
3096 	list_add_tail(&delayed->list, &fs_info->delayed_iputs);
3097 	spin_unlock(&fs_info->delayed_iput_lock);
3098 }
3099 
3100 void btrfs_run_delayed_iputs(struct btrfs_root *root)
3101 {
3102 	LIST_HEAD(list);
3103 	struct btrfs_fs_info *fs_info = root->fs_info;
3104 	struct delayed_iput *delayed;
3105 	int empty;
3106 
3107 	spin_lock(&fs_info->delayed_iput_lock);
3108 	empty = list_empty(&fs_info->delayed_iputs);
3109 	spin_unlock(&fs_info->delayed_iput_lock);
3110 	if (empty)
3111 		return;
3112 
3113 	down_read(&fs_info->delayed_iput_sem);
3114 
3115 	spin_lock(&fs_info->delayed_iput_lock);
3116 	list_splice_init(&fs_info->delayed_iputs, &list);
3117 	spin_unlock(&fs_info->delayed_iput_lock);
3118 
3119 	while (!list_empty(&list)) {
3120 		delayed = list_entry(list.next, struct delayed_iput, list);
3121 		list_del(&delayed->list);
3122 		iput(delayed->inode);
3123 		kfree(delayed);
3124 	}
3125 
3126 	up_read(&root->fs_info->delayed_iput_sem);
3127 }
3128 
3129 /*
3130  * This is called in transaction commit time. If there are no orphan
3131  * files in the subvolume, it removes orphan item and frees block_rsv
3132  * structure.
3133  */
3134 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
3135 			      struct btrfs_root *root)
3136 {
3137 	struct btrfs_block_rsv *block_rsv;
3138 	int ret;
3139 
3140 	if (atomic_read(&root->orphan_inodes) ||
3141 	    root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
3142 		return;
3143 
3144 	spin_lock(&root->orphan_lock);
3145 	if (atomic_read(&root->orphan_inodes)) {
3146 		spin_unlock(&root->orphan_lock);
3147 		return;
3148 	}
3149 
3150 	if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
3151 		spin_unlock(&root->orphan_lock);
3152 		return;
3153 	}
3154 
3155 	block_rsv = root->orphan_block_rsv;
3156 	root->orphan_block_rsv = NULL;
3157 	spin_unlock(&root->orphan_lock);
3158 
3159 	if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) &&
3160 	    btrfs_root_refs(&root->root_item) > 0) {
3161 		ret = btrfs_del_orphan_item(trans, root->fs_info->tree_root,
3162 					    root->root_key.objectid);
3163 		if (ret)
3164 			btrfs_abort_transaction(trans, root, ret);
3165 		else
3166 			clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
3167 				  &root->state);
3168 	}
3169 
3170 	if (block_rsv) {
3171 		WARN_ON(block_rsv->size > 0);
3172 		btrfs_free_block_rsv(root, block_rsv);
3173 	}
3174 }
3175 
3176 /*
3177  * This creates an orphan entry for the given inode in case something goes
3178  * wrong in the middle of an unlink/truncate.
3179  *
3180  * NOTE: caller of this function should reserve 5 units of metadata for
3181  *	 this function.
3182  */
3183 int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct inode *inode)
3184 {
3185 	struct btrfs_root *root = BTRFS_I(inode)->root;
3186 	struct btrfs_block_rsv *block_rsv = NULL;
3187 	int reserve = 0;
3188 	int insert = 0;
3189 	int ret;
3190 
3191 	if (!root->orphan_block_rsv) {
3192 		block_rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP);
3193 		if (!block_rsv)
3194 			return -ENOMEM;
3195 	}
3196 
3197 	spin_lock(&root->orphan_lock);
3198 	if (!root->orphan_block_rsv) {
3199 		root->orphan_block_rsv = block_rsv;
3200 	} else if (block_rsv) {
3201 		btrfs_free_block_rsv(root, block_rsv);
3202 		block_rsv = NULL;
3203 	}
3204 
3205 	if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3206 			      &BTRFS_I(inode)->runtime_flags)) {
3207 #if 0
3208 		/*
3209 		 * For proper ENOSPC handling, we should do orphan
3210 		 * cleanup when mounting. But this introduces backward
3211 		 * compatibility issue.
3212 		 */
3213 		if (!xchg(&root->orphan_item_inserted, 1))
3214 			insert = 2;
3215 		else
3216 			insert = 1;
3217 #endif
3218 		insert = 1;
3219 		atomic_inc(&root->orphan_inodes);
3220 	}
3221 
3222 	if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3223 			      &BTRFS_I(inode)->runtime_flags))
3224 		reserve = 1;
3225 	spin_unlock(&root->orphan_lock);
3226 
3227 	/* grab metadata reservation from transaction handle */
3228 	if (reserve) {
3229 		ret = btrfs_orphan_reserve_metadata(trans, inode);
3230 		BUG_ON(ret); /* -ENOSPC in reservation; Logic error? JDM */
3231 	}
3232 
3233 	/* insert an orphan item to track this unlinked/truncated file */
3234 	if (insert >= 1) {
3235 		ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
3236 		if (ret) {
3237 			atomic_dec(&root->orphan_inodes);
3238 			if (reserve) {
3239 				clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3240 					  &BTRFS_I(inode)->runtime_flags);
3241 				btrfs_orphan_release_metadata(inode);
3242 			}
3243 			if (ret != -EEXIST) {
3244 				clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3245 					  &BTRFS_I(inode)->runtime_flags);
3246 				btrfs_abort_transaction(trans, root, ret);
3247 				return ret;
3248 			}
3249 		}
3250 		ret = 0;
3251 	}
3252 
3253 	/* insert an orphan item to track subvolume contains orphan files */
3254 	if (insert >= 2) {
3255 		ret = btrfs_insert_orphan_item(trans, root->fs_info->tree_root,
3256 					       root->root_key.objectid);
3257 		if (ret && ret != -EEXIST) {
3258 			btrfs_abort_transaction(trans, root, ret);
3259 			return ret;
3260 		}
3261 	}
3262 	return 0;
3263 }
3264 
3265 /*
3266  * We have done the truncate/delete so we can go ahead and remove the orphan
3267  * item for this particular inode.
3268  */
3269 static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3270 			    struct inode *inode)
3271 {
3272 	struct btrfs_root *root = BTRFS_I(inode)->root;
3273 	int delete_item = 0;
3274 	int release_rsv = 0;
3275 	int ret = 0;
3276 
3277 	spin_lock(&root->orphan_lock);
3278 	if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3279 			       &BTRFS_I(inode)->runtime_flags))
3280 		delete_item = 1;
3281 
3282 	if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3283 			       &BTRFS_I(inode)->runtime_flags))
3284 		release_rsv = 1;
3285 	spin_unlock(&root->orphan_lock);
3286 
3287 	if (delete_item) {
3288 		atomic_dec(&root->orphan_inodes);
3289 		if (trans)
3290 			ret = btrfs_del_orphan_item(trans, root,
3291 						    btrfs_ino(inode));
3292 	}
3293 
3294 	if (release_rsv)
3295 		btrfs_orphan_release_metadata(inode);
3296 
3297 	return ret;
3298 }
3299 
3300 /*
3301  * this cleans up any orphans that may be left on the list from the last use
3302  * of this root.
3303  */
3304 int btrfs_orphan_cleanup(struct btrfs_root *root)
3305 {
3306 	struct btrfs_path *path;
3307 	struct extent_buffer *leaf;
3308 	struct btrfs_key key, found_key;
3309 	struct btrfs_trans_handle *trans;
3310 	struct inode *inode;
3311 	u64 last_objectid = 0;
3312 	int ret = 0, nr_unlink = 0, nr_truncate = 0;
3313 
3314 	if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3315 		return 0;
3316 
3317 	path = btrfs_alloc_path();
3318 	if (!path) {
3319 		ret = -ENOMEM;
3320 		goto out;
3321 	}
3322 	path->reada = -1;
3323 
3324 	key.objectid = BTRFS_ORPHAN_OBJECTID;
3325 	key.type = BTRFS_ORPHAN_ITEM_KEY;
3326 	key.offset = (u64)-1;
3327 
3328 	while (1) {
3329 		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3330 		if (ret < 0)
3331 			goto out;
3332 
3333 		/*
3334 		 * if ret == 0 means we found what we were searching for, which
3335 		 * is weird, but possible, so only screw with path if we didn't
3336 		 * find the key and see if we have stuff that matches
3337 		 */
3338 		if (ret > 0) {
3339 			ret = 0;
3340 			if (path->slots[0] == 0)
3341 				break;
3342 			path->slots[0]--;
3343 		}
3344 
3345 		/* pull out the item */
3346 		leaf = path->nodes[0];
3347 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3348 
3349 		/* make sure the item matches what we want */
3350 		if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3351 			break;
3352 		if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3353 			break;
3354 
3355 		/* release the path since we're done with it */
3356 		btrfs_release_path(path);
3357 
3358 		/*
3359 		 * this is where we are basically btrfs_lookup, without the
3360 		 * crossing root thing.  we store the inode number in the
3361 		 * offset of the orphan item.
3362 		 */
3363 
3364 		if (found_key.offset == last_objectid) {
3365 			btrfs_err(root->fs_info,
3366 				"Error removing orphan entry, stopping orphan cleanup");
3367 			ret = -EINVAL;
3368 			goto out;
3369 		}
3370 
3371 		last_objectid = found_key.offset;
3372 
3373 		found_key.objectid = found_key.offset;
3374 		found_key.type = BTRFS_INODE_ITEM_KEY;
3375 		found_key.offset = 0;
3376 		inode = btrfs_iget(root->fs_info->sb, &found_key, root, NULL);
3377 		ret = PTR_ERR_OR_ZERO(inode);
3378 		if (ret && ret != -ESTALE)
3379 			goto out;
3380 
3381 		if (ret == -ESTALE && root == root->fs_info->tree_root) {
3382 			struct btrfs_root *dead_root;
3383 			struct btrfs_fs_info *fs_info = root->fs_info;
3384 			int is_dead_root = 0;
3385 
3386 			/*
3387 			 * this is an orphan in the tree root. Currently these
3388 			 * could come from 2 sources:
3389 			 *  a) a snapshot deletion in progress
3390 			 *  b) a free space cache inode
3391 			 * We need to distinguish those two, as the snapshot
3392 			 * orphan must not get deleted.
3393 			 * find_dead_roots already ran before us, so if this
3394 			 * is a snapshot deletion, we should find the root
3395 			 * in the dead_roots list
3396 			 */
3397 			spin_lock(&fs_info->trans_lock);
3398 			list_for_each_entry(dead_root, &fs_info->dead_roots,
3399 					    root_list) {
3400 				if (dead_root->root_key.objectid ==
3401 				    found_key.objectid) {
3402 					is_dead_root = 1;
3403 					break;
3404 				}
3405 			}
3406 			spin_unlock(&fs_info->trans_lock);
3407 			if (is_dead_root) {
3408 				/* prevent this orphan from being found again */
3409 				key.offset = found_key.objectid - 1;
3410 				continue;
3411 			}
3412 		}
3413 		/*
3414 		 * Inode is already gone but the orphan item is still there,
3415 		 * kill the orphan item.
3416 		 */
3417 		if (ret == -ESTALE) {
3418 			trans = btrfs_start_transaction(root, 1);
3419 			if (IS_ERR(trans)) {
3420 				ret = PTR_ERR(trans);
3421 				goto out;
3422 			}
3423 			btrfs_debug(root->fs_info, "auto deleting %Lu",
3424 				found_key.objectid);
3425 			ret = btrfs_del_orphan_item(trans, root,
3426 						    found_key.objectid);
3427 			btrfs_end_transaction(trans, root);
3428 			if (ret)
3429 				goto out;
3430 			continue;
3431 		}
3432 
3433 		/*
3434 		 * add this inode to the orphan list so btrfs_orphan_del does
3435 		 * the proper thing when we hit it
3436 		 */
3437 		set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3438 			&BTRFS_I(inode)->runtime_flags);
3439 		atomic_inc(&root->orphan_inodes);
3440 
3441 		/* if we have links, this was a truncate, lets do that */
3442 		if (inode->i_nlink) {
3443 			if (WARN_ON(!S_ISREG(inode->i_mode))) {
3444 				iput(inode);
3445 				continue;
3446 			}
3447 			nr_truncate++;
3448 
3449 			/* 1 for the orphan item deletion. */
3450 			trans = btrfs_start_transaction(root, 1);
3451 			if (IS_ERR(trans)) {
3452 				iput(inode);
3453 				ret = PTR_ERR(trans);
3454 				goto out;
3455 			}
3456 			ret = btrfs_orphan_add(trans, inode);
3457 			btrfs_end_transaction(trans, root);
3458 			if (ret) {
3459 				iput(inode);
3460 				goto out;
3461 			}
3462 
3463 			ret = btrfs_truncate(inode);
3464 			if (ret)
3465 				btrfs_orphan_del(NULL, inode);
3466 		} else {
3467 			nr_unlink++;
3468 		}
3469 
3470 		/* this will do delete_inode and everything for us */
3471 		iput(inode);
3472 		if (ret)
3473 			goto out;
3474 	}
3475 	/* release the path since we're done with it */
3476 	btrfs_release_path(path);
3477 
3478 	root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
3479 
3480 	if (root->orphan_block_rsv)
3481 		btrfs_block_rsv_release(root, root->orphan_block_rsv,
3482 					(u64)-1);
3483 
3484 	if (root->orphan_block_rsv ||
3485 	    test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3486 		trans = btrfs_join_transaction(root);
3487 		if (!IS_ERR(trans))
3488 			btrfs_end_transaction(trans, root);
3489 	}
3490 
3491 	if (nr_unlink)
3492 		btrfs_debug(root->fs_info, "unlinked %d orphans", nr_unlink);
3493 	if (nr_truncate)
3494 		btrfs_debug(root->fs_info, "truncated %d orphans", nr_truncate);
3495 
3496 out:
3497 	if (ret)
3498 		btrfs_err(root->fs_info,
3499 			"could not do orphan cleanup %d", ret);
3500 	btrfs_free_path(path);
3501 	return ret;
3502 }
3503 
3504 /*
3505  * very simple check to peek ahead in the leaf looking for xattrs.  If we
3506  * don't find any xattrs, we know there can't be any acls.
3507  *
3508  * slot is the slot the inode is in, objectid is the objectid of the inode
3509  */
3510 static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3511 					  int slot, u64 objectid,
3512 					  int *first_xattr_slot)
3513 {
3514 	u32 nritems = btrfs_header_nritems(leaf);
3515 	struct btrfs_key found_key;
3516 	static u64 xattr_access = 0;
3517 	static u64 xattr_default = 0;
3518 	int scanned = 0;
3519 
3520 	if (!xattr_access) {
3521 		xattr_access = btrfs_name_hash(POSIX_ACL_XATTR_ACCESS,
3522 					strlen(POSIX_ACL_XATTR_ACCESS));
3523 		xattr_default = btrfs_name_hash(POSIX_ACL_XATTR_DEFAULT,
3524 					strlen(POSIX_ACL_XATTR_DEFAULT));
3525 	}
3526 
3527 	slot++;
3528 	*first_xattr_slot = -1;
3529 	while (slot < nritems) {
3530 		btrfs_item_key_to_cpu(leaf, &found_key, slot);
3531 
3532 		/* we found a different objectid, there must not be acls */
3533 		if (found_key.objectid != objectid)
3534 			return 0;
3535 
3536 		/* we found an xattr, assume we've got an acl */
3537 		if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3538 			if (*first_xattr_slot == -1)
3539 				*first_xattr_slot = slot;
3540 			if (found_key.offset == xattr_access ||
3541 			    found_key.offset == xattr_default)
3542 				return 1;
3543 		}
3544 
3545 		/*
3546 		 * we found a key greater than an xattr key, there can't
3547 		 * be any acls later on
3548 		 */
3549 		if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3550 			return 0;
3551 
3552 		slot++;
3553 		scanned++;
3554 
3555 		/*
3556 		 * it goes inode, inode backrefs, xattrs, extents,
3557 		 * so if there are a ton of hard links to an inode there can
3558 		 * be a lot of backrefs.  Don't waste time searching too hard,
3559 		 * this is just an optimization
3560 		 */
3561 		if (scanned >= 8)
3562 			break;
3563 	}
3564 	/* we hit the end of the leaf before we found an xattr or
3565 	 * something larger than an xattr.  We have to assume the inode
3566 	 * has acls
3567 	 */
3568 	if (*first_xattr_slot == -1)
3569 		*first_xattr_slot = slot;
3570 	return 1;
3571 }
3572 
3573 /*
3574  * read an inode from the btree into the in-memory inode
3575  */
3576 static void btrfs_read_locked_inode(struct inode *inode)
3577 {
3578 	struct btrfs_path *path;
3579 	struct extent_buffer *leaf;
3580 	struct btrfs_inode_item *inode_item;
3581 	struct btrfs_root *root = BTRFS_I(inode)->root;
3582 	struct btrfs_key location;
3583 	unsigned long ptr;
3584 	int maybe_acls;
3585 	u32 rdev;
3586 	int ret;
3587 	bool filled = false;
3588 	int first_xattr_slot;
3589 
3590 	ret = btrfs_fill_inode(inode, &rdev);
3591 	if (!ret)
3592 		filled = true;
3593 
3594 	path = btrfs_alloc_path();
3595 	if (!path)
3596 		goto make_bad;
3597 
3598 	memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
3599 
3600 	ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3601 	if (ret)
3602 		goto make_bad;
3603 
3604 	leaf = path->nodes[0];
3605 
3606 	if (filled)
3607 		goto cache_index;
3608 
3609 	inode_item = btrfs_item_ptr(leaf, path->slots[0],
3610 				    struct btrfs_inode_item);
3611 	inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3612 	set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3613 	i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
3614 	i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3615 	btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item));
3616 
3617 	inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
3618 	inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
3619 
3620 	inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
3621 	inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
3622 
3623 	inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
3624 	inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
3625 
3626 	BTRFS_I(inode)->i_otime.tv_sec =
3627 		btrfs_timespec_sec(leaf, &inode_item->otime);
3628 	BTRFS_I(inode)->i_otime.tv_nsec =
3629 		btrfs_timespec_nsec(leaf, &inode_item->otime);
3630 
3631 	inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3632 	BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
3633 	BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
3634 
3635 	inode->i_version = btrfs_inode_sequence(leaf, inode_item);
3636 	inode->i_generation = BTRFS_I(inode)->generation;
3637 	inode->i_rdev = 0;
3638 	rdev = btrfs_inode_rdev(leaf, inode_item);
3639 
3640 	BTRFS_I(inode)->index_cnt = (u64)-1;
3641 	BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
3642 
3643 cache_index:
3644 	/*
3645 	 * If we were modified in the current generation and evicted from memory
3646 	 * and then re-read we need to do a full sync since we don't have any
3647 	 * idea about which extents were modified before we were evicted from
3648 	 * cache.
3649 	 *
3650 	 * This is required for both inode re-read from disk and delayed inode
3651 	 * in delayed_nodes_tree.
3652 	 */
3653 	if (BTRFS_I(inode)->last_trans == root->fs_info->generation)
3654 		set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
3655 			&BTRFS_I(inode)->runtime_flags);
3656 
3657 	path->slots[0]++;
3658 	if (inode->i_nlink != 1 ||
3659 	    path->slots[0] >= btrfs_header_nritems(leaf))
3660 		goto cache_acl;
3661 
3662 	btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
3663 	if (location.objectid != btrfs_ino(inode))
3664 		goto cache_acl;
3665 
3666 	ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
3667 	if (location.type == BTRFS_INODE_REF_KEY) {
3668 		struct btrfs_inode_ref *ref;
3669 
3670 		ref = (struct btrfs_inode_ref *)ptr;
3671 		BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
3672 	} else if (location.type == BTRFS_INODE_EXTREF_KEY) {
3673 		struct btrfs_inode_extref *extref;
3674 
3675 		extref = (struct btrfs_inode_extref *)ptr;
3676 		BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
3677 								     extref);
3678 	}
3679 cache_acl:
3680 	/*
3681 	 * try to precache a NULL acl entry for files that don't have
3682 	 * any xattrs or acls
3683 	 */
3684 	maybe_acls = acls_after_inode_item(leaf, path->slots[0],
3685 					   btrfs_ino(inode), &first_xattr_slot);
3686 	if (first_xattr_slot != -1) {
3687 		path->slots[0] = first_xattr_slot;
3688 		ret = btrfs_load_inode_props(inode, path);
3689 		if (ret)
3690 			btrfs_err(root->fs_info,
3691 				  "error loading props for ino %llu (root %llu): %d",
3692 				  btrfs_ino(inode),
3693 				  root->root_key.objectid, ret);
3694 	}
3695 	btrfs_free_path(path);
3696 
3697 	if (!maybe_acls)
3698 		cache_no_acl(inode);
3699 
3700 	switch (inode->i_mode & S_IFMT) {
3701 	case S_IFREG:
3702 		inode->i_mapping->a_ops = &btrfs_aops;
3703 		BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
3704 		inode->i_fop = &btrfs_file_operations;
3705 		inode->i_op = &btrfs_file_inode_operations;
3706 		break;
3707 	case S_IFDIR:
3708 		inode->i_fop = &btrfs_dir_file_operations;
3709 		if (root == root->fs_info->tree_root)
3710 			inode->i_op = &btrfs_dir_ro_inode_operations;
3711 		else
3712 			inode->i_op = &btrfs_dir_inode_operations;
3713 		break;
3714 	case S_IFLNK:
3715 		inode->i_op = &btrfs_symlink_inode_operations;
3716 		inode->i_mapping->a_ops = &btrfs_symlink_aops;
3717 		break;
3718 	default:
3719 		inode->i_op = &btrfs_special_inode_operations;
3720 		init_special_inode(inode, inode->i_mode, rdev);
3721 		break;
3722 	}
3723 
3724 	btrfs_update_iflags(inode);
3725 	return;
3726 
3727 make_bad:
3728 	btrfs_free_path(path);
3729 	make_bad_inode(inode);
3730 }
3731 
3732 /*
3733  * given a leaf and an inode, copy the inode fields into the leaf
3734  */
3735 static void fill_inode_item(struct btrfs_trans_handle *trans,
3736 			    struct extent_buffer *leaf,
3737 			    struct btrfs_inode_item *item,
3738 			    struct inode *inode)
3739 {
3740 	struct btrfs_map_token token;
3741 
3742 	btrfs_init_map_token(&token);
3743 
3744 	btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
3745 	btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
3746 	btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
3747 				   &token);
3748 	btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
3749 	btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3750 
3751 	btrfs_set_token_timespec_sec(leaf, &item->atime,
3752 				     inode->i_atime.tv_sec, &token);
3753 	btrfs_set_token_timespec_nsec(leaf, &item->atime,
3754 				      inode->i_atime.tv_nsec, &token);
3755 
3756 	btrfs_set_token_timespec_sec(leaf, &item->mtime,
3757 				     inode->i_mtime.tv_sec, &token);
3758 	btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3759 				      inode->i_mtime.tv_nsec, &token);
3760 
3761 	btrfs_set_token_timespec_sec(leaf, &item->ctime,
3762 				     inode->i_ctime.tv_sec, &token);
3763 	btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3764 				      inode->i_ctime.tv_nsec, &token);
3765 
3766 	btrfs_set_token_timespec_sec(leaf, &item->otime,
3767 				     BTRFS_I(inode)->i_otime.tv_sec, &token);
3768 	btrfs_set_token_timespec_nsec(leaf, &item->otime,
3769 				      BTRFS_I(inode)->i_otime.tv_nsec, &token);
3770 
3771 	btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
3772 				     &token);
3773 	btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
3774 					 &token);
3775 	btrfs_set_token_inode_sequence(leaf, item, inode->i_version, &token);
3776 	btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
3777 	btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
3778 	btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
3779 	btrfs_set_token_inode_block_group(leaf, item, 0, &token);
3780 }
3781 
3782 /*
3783  * copy everything in the in-memory inode into the btree.
3784  */
3785 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
3786 				struct btrfs_root *root, struct inode *inode)
3787 {
3788 	struct btrfs_inode_item *inode_item;
3789 	struct btrfs_path *path;
3790 	struct extent_buffer *leaf;
3791 	int ret;
3792 
3793 	path = btrfs_alloc_path();
3794 	if (!path)
3795 		return -ENOMEM;
3796 
3797 	path->leave_spinning = 1;
3798 	ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
3799 				 1);
3800 	if (ret) {
3801 		if (ret > 0)
3802 			ret = -ENOENT;
3803 		goto failed;
3804 	}
3805 
3806 	leaf = path->nodes[0];
3807 	inode_item = btrfs_item_ptr(leaf, path->slots[0],
3808 				    struct btrfs_inode_item);
3809 
3810 	fill_inode_item(trans, leaf, inode_item, inode);
3811 	btrfs_mark_buffer_dirty(leaf);
3812 	btrfs_set_inode_last_trans(trans, inode);
3813 	ret = 0;
3814 failed:
3815 	btrfs_free_path(path);
3816 	return ret;
3817 }
3818 
3819 /*
3820  * copy everything in the in-memory inode into the btree.
3821  */
3822 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
3823 				struct btrfs_root *root, struct inode *inode)
3824 {
3825 	int ret;
3826 
3827 	/*
3828 	 * If the inode is a free space inode, we can deadlock during commit
3829 	 * if we put it into the delayed code.
3830 	 *
3831 	 * The data relocation inode should also be directly updated
3832 	 * without delay
3833 	 */
3834 	if (!btrfs_is_free_space_inode(inode)
3835 	    && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
3836 	    && !root->fs_info->log_root_recovering) {
3837 		btrfs_update_root_times(trans, root);
3838 
3839 		ret = btrfs_delayed_update_inode(trans, root, inode);
3840 		if (!ret)
3841 			btrfs_set_inode_last_trans(trans, inode);
3842 		return ret;
3843 	}
3844 
3845 	return btrfs_update_inode_item(trans, root, inode);
3846 }
3847 
3848 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
3849 					 struct btrfs_root *root,
3850 					 struct inode *inode)
3851 {
3852 	int ret;
3853 
3854 	ret = btrfs_update_inode(trans, root, inode);
3855 	if (ret == -ENOSPC)
3856 		return btrfs_update_inode_item(trans, root, inode);
3857 	return ret;
3858 }
3859 
3860 /*
3861  * unlink helper that gets used here in inode.c and in the tree logging
3862  * recovery code.  It remove a link in a directory with a given name, and
3863  * also drops the back refs in the inode to the directory
3864  */
3865 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
3866 				struct btrfs_root *root,
3867 				struct inode *dir, struct inode *inode,
3868 				const char *name, int name_len)
3869 {
3870 	struct btrfs_path *path;
3871 	int ret = 0;
3872 	struct extent_buffer *leaf;
3873 	struct btrfs_dir_item *di;
3874 	struct btrfs_key key;
3875 	u64 index;
3876 	u64 ino = btrfs_ino(inode);
3877 	u64 dir_ino = btrfs_ino(dir);
3878 
3879 	path = btrfs_alloc_path();
3880 	if (!path) {
3881 		ret = -ENOMEM;
3882 		goto out;
3883 	}
3884 
3885 	path->leave_spinning = 1;
3886 	di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
3887 				    name, name_len, -1);
3888 	if (IS_ERR(di)) {
3889 		ret = PTR_ERR(di);
3890 		goto err;
3891 	}
3892 	if (!di) {
3893 		ret = -ENOENT;
3894 		goto err;
3895 	}
3896 	leaf = path->nodes[0];
3897 	btrfs_dir_item_key_to_cpu(leaf, di, &key);
3898 	ret = btrfs_delete_one_dir_name(trans, root, path, di);
3899 	if (ret)
3900 		goto err;
3901 	btrfs_release_path(path);
3902 
3903 	/*
3904 	 * If we don't have dir index, we have to get it by looking up
3905 	 * the inode ref, since we get the inode ref, remove it directly,
3906 	 * it is unnecessary to do delayed deletion.
3907 	 *
3908 	 * But if we have dir index, needn't search inode ref to get it.
3909 	 * Since the inode ref is close to the inode item, it is better
3910 	 * that we delay to delete it, and just do this deletion when
3911 	 * we update the inode item.
3912 	 */
3913 	if (BTRFS_I(inode)->dir_index) {
3914 		ret = btrfs_delayed_delete_inode_ref(inode);
3915 		if (!ret) {
3916 			index = BTRFS_I(inode)->dir_index;
3917 			goto skip_backref;
3918 		}
3919 	}
3920 
3921 	ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
3922 				  dir_ino, &index);
3923 	if (ret) {
3924 		btrfs_info(root->fs_info,
3925 			"failed to delete reference to %.*s, inode %llu parent %llu",
3926 			name_len, name, ino, dir_ino);
3927 		btrfs_abort_transaction(trans, root, ret);
3928 		goto err;
3929 	}
3930 skip_backref:
3931 	ret = btrfs_delete_delayed_dir_index(trans, root, dir, index);
3932 	if (ret) {
3933 		btrfs_abort_transaction(trans, root, ret);
3934 		goto err;
3935 	}
3936 
3937 	ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len,
3938 					 inode, dir_ino);
3939 	if (ret != 0 && ret != -ENOENT) {
3940 		btrfs_abort_transaction(trans, root, ret);
3941 		goto err;
3942 	}
3943 
3944 	ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len,
3945 					   dir, index);
3946 	if (ret == -ENOENT)
3947 		ret = 0;
3948 	else if (ret)
3949 		btrfs_abort_transaction(trans, root, ret);
3950 err:
3951 	btrfs_free_path(path);
3952 	if (ret)
3953 		goto out;
3954 
3955 	btrfs_i_size_write(dir, dir->i_size - name_len * 2);
3956 	inode_inc_iversion(inode);
3957 	inode_inc_iversion(dir);
3958 	inode->i_ctime = dir->i_mtime = dir->i_ctime = CURRENT_TIME;
3959 	ret = btrfs_update_inode(trans, root, dir);
3960 out:
3961 	return ret;
3962 }
3963 
3964 int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
3965 		       struct btrfs_root *root,
3966 		       struct inode *dir, struct inode *inode,
3967 		       const char *name, int name_len)
3968 {
3969 	int ret;
3970 	ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
3971 	if (!ret) {
3972 		drop_nlink(inode);
3973 		ret = btrfs_update_inode(trans, root, inode);
3974 	}
3975 	return ret;
3976 }
3977 
3978 /*
3979  * helper to start transaction for unlink and rmdir.
3980  *
3981  * unlink and rmdir are special in btrfs, they do not always free space, so
3982  * if we cannot make our reservations the normal way try and see if there is
3983  * plenty of slack room in the global reserve to migrate, otherwise we cannot
3984  * allow the unlink to occur.
3985  */
3986 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
3987 {
3988 	struct btrfs_trans_handle *trans;
3989 	struct btrfs_root *root = BTRFS_I(dir)->root;
3990 	int ret;
3991 
3992 	/*
3993 	 * 1 for the possible orphan item
3994 	 * 1 for the dir item
3995 	 * 1 for the dir index
3996 	 * 1 for the inode ref
3997 	 * 1 for the inode
3998 	 */
3999 	trans = btrfs_start_transaction(root, 5);
4000 	if (!IS_ERR(trans) || PTR_ERR(trans) != -ENOSPC)
4001 		return trans;
4002 
4003 	if (PTR_ERR(trans) == -ENOSPC) {
4004 		u64 num_bytes = btrfs_calc_trans_metadata_size(root, 5);
4005 
4006 		trans = btrfs_start_transaction(root, 0);
4007 		if (IS_ERR(trans))
4008 			return trans;
4009 		ret = btrfs_cond_migrate_bytes(root->fs_info,
4010 					       &root->fs_info->trans_block_rsv,
4011 					       num_bytes, 5);
4012 		if (ret) {
4013 			btrfs_end_transaction(trans, root);
4014 			return ERR_PTR(ret);
4015 		}
4016 		trans->block_rsv = &root->fs_info->trans_block_rsv;
4017 		trans->bytes_reserved = num_bytes;
4018 	}
4019 	return trans;
4020 }
4021 
4022 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4023 {
4024 	struct btrfs_root *root = BTRFS_I(dir)->root;
4025 	struct btrfs_trans_handle *trans;
4026 	struct inode *inode = d_inode(dentry);
4027 	int ret;
4028 
4029 	trans = __unlink_start_trans(dir);
4030 	if (IS_ERR(trans))
4031 		return PTR_ERR(trans);
4032 
4033 	btrfs_record_unlink_dir(trans, dir, d_inode(dentry), 0);
4034 
4035 	ret = btrfs_unlink_inode(trans, root, dir, d_inode(dentry),
4036 				 dentry->d_name.name, dentry->d_name.len);
4037 	if (ret)
4038 		goto out;
4039 
4040 	if (inode->i_nlink == 0) {
4041 		ret = btrfs_orphan_add(trans, inode);
4042 		if (ret)
4043 			goto out;
4044 	}
4045 
4046 out:
4047 	btrfs_end_transaction(trans, root);
4048 	btrfs_btree_balance_dirty(root);
4049 	return ret;
4050 }
4051 
4052 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4053 			struct btrfs_root *root,
4054 			struct inode *dir, u64 objectid,
4055 			const char *name, int name_len)
4056 {
4057 	struct btrfs_path *path;
4058 	struct extent_buffer *leaf;
4059 	struct btrfs_dir_item *di;
4060 	struct btrfs_key key;
4061 	u64 index;
4062 	int ret;
4063 	u64 dir_ino = btrfs_ino(dir);
4064 
4065 	path = btrfs_alloc_path();
4066 	if (!path)
4067 		return -ENOMEM;
4068 
4069 	di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4070 				   name, name_len, -1);
4071 	if (IS_ERR_OR_NULL(di)) {
4072 		if (!di)
4073 			ret = -ENOENT;
4074 		else
4075 			ret = PTR_ERR(di);
4076 		goto out;
4077 	}
4078 
4079 	leaf = path->nodes[0];
4080 	btrfs_dir_item_key_to_cpu(leaf, di, &key);
4081 	WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4082 	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4083 	if (ret) {
4084 		btrfs_abort_transaction(trans, root, ret);
4085 		goto out;
4086 	}
4087 	btrfs_release_path(path);
4088 
4089 	ret = btrfs_del_root_ref(trans, root->fs_info->tree_root,
4090 				 objectid, root->root_key.objectid,
4091 				 dir_ino, &index, name, name_len);
4092 	if (ret < 0) {
4093 		if (ret != -ENOENT) {
4094 			btrfs_abort_transaction(trans, root, ret);
4095 			goto out;
4096 		}
4097 		di = btrfs_search_dir_index_item(root, path, dir_ino,
4098 						 name, name_len);
4099 		if (IS_ERR_OR_NULL(di)) {
4100 			if (!di)
4101 				ret = -ENOENT;
4102 			else
4103 				ret = PTR_ERR(di);
4104 			btrfs_abort_transaction(trans, root, ret);
4105 			goto out;
4106 		}
4107 
4108 		leaf = path->nodes[0];
4109 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4110 		btrfs_release_path(path);
4111 		index = key.offset;
4112 	}
4113 	btrfs_release_path(path);
4114 
4115 	ret = btrfs_delete_delayed_dir_index(trans, root, dir, index);
4116 	if (ret) {
4117 		btrfs_abort_transaction(trans, root, ret);
4118 		goto out;
4119 	}
4120 
4121 	btrfs_i_size_write(dir, dir->i_size - name_len * 2);
4122 	inode_inc_iversion(dir);
4123 	dir->i_mtime = dir->i_ctime = CURRENT_TIME;
4124 	ret = btrfs_update_inode_fallback(trans, root, dir);
4125 	if (ret)
4126 		btrfs_abort_transaction(trans, root, ret);
4127 out:
4128 	btrfs_free_path(path);
4129 	return ret;
4130 }
4131 
4132 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
4133 {
4134 	struct inode *inode = d_inode(dentry);
4135 	int err = 0;
4136 	struct btrfs_root *root = BTRFS_I(dir)->root;
4137 	struct btrfs_trans_handle *trans;
4138 
4139 	if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
4140 		return -ENOTEMPTY;
4141 	if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID)
4142 		return -EPERM;
4143 
4144 	trans = __unlink_start_trans(dir);
4145 	if (IS_ERR(trans))
4146 		return PTR_ERR(trans);
4147 
4148 	if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4149 		err = btrfs_unlink_subvol(trans, root, dir,
4150 					  BTRFS_I(inode)->location.objectid,
4151 					  dentry->d_name.name,
4152 					  dentry->d_name.len);
4153 		goto out;
4154 	}
4155 
4156 	err = btrfs_orphan_add(trans, inode);
4157 	if (err)
4158 		goto out;
4159 
4160 	/* now the directory is empty */
4161 	err = btrfs_unlink_inode(trans, root, dir, d_inode(dentry),
4162 				 dentry->d_name.name, dentry->d_name.len);
4163 	if (!err)
4164 		btrfs_i_size_write(inode, 0);
4165 out:
4166 	btrfs_end_transaction(trans, root);
4167 	btrfs_btree_balance_dirty(root);
4168 
4169 	return err;
4170 }
4171 
4172 static int truncate_space_check(struct btrfs_trans_handle *trans,
4173 				struct btrfs_root *root,
4174 				u64 bytes_deleted)
4175 {
4176 	int ret;
4177 
4178 	bytes_deleted = btrfs_csum_bytes_to_leaves(root, bytes_deleted);
4179 	ret = btrfs_block_rsv_add(root, &root->fs_info->trans_block_rsv,
4180 				  bytes_deleted, BTRFS_RESERVE_NO_FLUSH);
4181 	if (!ret)
4182 		trans->bytes_reserved += bytes_deleted;
4183 	return ret;
4184 
4185 }
4186 
4187 /*
4188  * this can truncate away extent items, csum items and directory items.
4189  * It starts at a high offset and removes keys until it can't find
4190  * any higher than new_size
4191  *
4192  * csum items that cross the new i_size are truncated to the new size
4193  * as well.
4194  *
4195  * min_type is the minimum key type to truncate down to.  If set to 0, this
4196  * will kill all the items on this inode, including the INODE_ITEM_KEY.
4197  */
4198 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
4199 			       struct btrfs_root *root,
4200 			       struct inode *inode,
4201 			       u64 new_size, u32 min_type)
4202 {
4203 	struct btrfs_path *path;
4204 	struct extent_buffer *leaf;
4205 	struct btrfs_file_extent_item *fi;
4206 	struct btrfs_key key;
4207 	struct btrfs_key found_key;
4208 	u64 extent_start = 0;
4209 	u64 extent_num_bytes = 0;
4210 	u64 extent_offset = 0;
4211 	u64 item_end = 0;
4212 	u64 last_size = new_size;
4213 	u32 found_type = (u8)-1;
4214 	int found_extent;
4215 	int del_item;
4216 	int pending_del_nr = 0;
4217 	int pending_del_slot = 0;
4218 	int extent_type = -1;
4219 	int ret;
4220 	int err = 0;
4221 	u64 ino = btrfs_ino(inode);
4222 	u64 bytes_deleted = 0;
4223 	bool be_nice = 0;
4224 	bool should_throttle = 0;
4225 	bool should_end = 0;
4226 
4227 	BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
4228 
4229 	/*
4230 	 * for non-free space inodes and ref cows, we want to back off from
4231 	 * time to time
4232 	 */
4233 	if (!btrfs_is_free_space_inode(inode) &&
4234 	    test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4235 		be_nice = 1;
4236 
4237 	path = btrfs_alloc_path();
4238 	if (!path)
4239 		return -ENOMEM;
4240 	path->reada = -1;
4241 
4242 	/*
4243 	 * We want to drop from the next block forward in case this new size is
4244 	 * not block aligned since we will be keeping the last block of the
4245 	 * extent just the way it is.
4246 	 */
4247 	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4248 	    root == root->fs_info->tree_root)
4249 		btrfs_drop_extent_cache(inode, ALIGN(new_size,
4250 					root->sectorsize), (u64)-1, 0);
4251 
4252 	/*
4253 	 * This function is also used to drop the items in the log tree before
4254 	 * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4255 	 * it is used to drop the loged items. So we shouldn't kill the delayed
4256 	 * items.
4257 	 */
4258 	if (min_type == 0 && root == BTRFS_I(inode)->root)
4259 		btrfs_kill_delayed_inode_items(inode);
4260 
4261 	key.objectid = ino;
4262 	key.offset = (u64)-1;
4263 	key.type = (u8)-1;
4264 
4265 search_again:
4266 	/*
4267 	 * with a 16K leaf size and 128MB extents, you can actually queue
4268 	 * up a huge file in a single leaf.  Most of the time that
4269 	 * bytes_deleted is > 0, it will be huge by the time we get here
4270 	 */
4271 	if (be_nice && bytes_deleted > 32 * 1024 * 1024) {
4272 		if (btrfs_should_end_transaction(trans, root)) {
4273 			err = -EAGAIN;
4274 			goto error;
4275 		}
4276 	}
4277 
4278 
4279 	path->leave_spinning = 1;
4280 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
4281 	if (ret < 0) {
4282 		err = ret;
4283 		goto out;
4284 	}
4285 
4286 	if (ret > 0) {
4287 		/* there are no items in the tree for us to truncate, we're
4288 		 * done
4289 		 */
4290 		if (path->slots[0] == 0)
4291 			goto out;
4292 		path->slots[0]--;
4293 	}
4294 
4295 	while (1) {
4296 		fi = NULL;
4297 		leaf = path->nodes[0];
4298 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
4299 		found_type = found_key.type;
4300 
4301 		if (found_key.objectid != ino)
4302 			break;
4303 
4304 		if (found_type < min_type)
4305 			break;
4306 
4307 		item_end = found_key.offset;
4308 		if (found_type == BTRFS_EXTENT_DATA_KEY) {
4309 			fi = btrfs_item_ptr(leaf, path->slots[0],
4310 					    struct btrfs_file_extent_item);
4311 			extent_type = btrfs_file_extent_type(leaf, fi);
4312 			if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4313 				item_end +=
4314 				    btrfs_file_extent_num_bytes(leaf, fi);
4315 			} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4316 				item_end += btrfs_file_extent_inline_len(leaf,
4317 							 path->slots[0], fi);
4318 			}
4319 			item_end--;
4320 		}
4321 		if (found_type > min_type) {
4322 			del_item = 1;
4323 		} else {
4324 			if (item_end < new_size)
4325 				break;
4326 			if (found_key.offset >= new_size)
4327 				del_item = 1;
4328 			else
4329 				del_item = 0;
4330 		}
4331 		found_extent = 0;
4332 		/* FIXME, shrink the extent if the ref count is only 1 */
4333 		if (found_type != BTRFS_EXTENT_DATA_KEY)
4334 			goto delete;
4335 
4336 		if (del_item)
4337 			last_size = found_key.offset;
4338 		else
4339 			last_size = new_size;
4340 
4341 		if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4342 			u64 num_dec;
4343 			extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
4344 			if (!del_item) {
4345 				u64 orig_num_bytes =
4346 					btrfs_file_extent_num_bytes(leaf, fi);
4347 				extent_num_bytes = ALIGN(new_size -
4348 						found_key.offset,
4349 						root->sectorsize);
4350 				btrfs_set_file_extent_num_bytes(leaf, fi,
4351 							 extent_num_bytes);
4352 				num_dec = (orig_num_bytes -
4353 					   extent_num_bytes);
4354 				if (test_bit(BTRFS_ROOT_REF_COWS,
4355 					     &root->state) &&
4356 				    extent_start != 0)
4357 					inode_sub_bytes(inode, num_dec);
4358 				btrfs_mark_buffer_dirty(leaf);
4359 			} else {
4360 				extent_num_bytes =
4361 					btrfs_file_extent_disk_num_bytes(leaf,
4362 									 fi);
4363 				extent_offset = found_key.offset -
4364 					btrfs_file_extent_offset(leaf, fi);
4365 
4366 				/* FIXME blocksize != 4096 */
4367 				num_dec = btrfs_file_extent_num_bytes(leaf, fi);
4368 				if (extent_start != 0) {
4369 					found_extent = 1;
4370 					if (test_bit(BTRFS_ROOT_REF_COWS,
4371 						     &root->state))
4372 						inode_sub_bytes(inode, num_dec);
4373 				}
4374 			}
4375 		} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4376 			/*
4377 			 * we can't truncate inline items that have had
4378 			 * special encodings
4379 			 */
4380 			if (!del_item &&
4381 			    btrfs_file_extent_compression(leaf, fi) == 0 &&
4382 			    btrfs_file_extent_encryption(leaf, fi) == 0 &&
4383 			    btrfs_file_extent_other_encoding(leaf, fi) == 0) {
4384 				u32 size = new_size - found_key.offset;
4385 
4386 				if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4387 					inode_sub_bytes(inode, item_end + 1 -
4388 							new_size);
4389 
4390 				/*
4391 				 * update the ram bytes to properly reflect
4392 				 * the new size of our item
4393 				 */
4394 				btrfs_set_file_extent_ram_bytes(leaf, fi, size);
4395 				size =
4396 				    btrfs_file_extent_calc_inline_size(size);
4397 				btrfs_truncate_item(root, path, size, 1);
4398 			} else if (test_bit(BTRFS_ROOT_REF_COWS,
4399 					    &root->state)) {
4400 				inode_sub_bytes(inode, item_end + 1 -
4401 						found_key.offset);
4402 			}
4403 		}
4404 delete:
4405 		if (del_item) {
4406 			if (!pending_del_nr) {
4407 				/* no pending yet, add ourselves */
4408 				pending_del_slot = path->slots[0];
4409 				pending_del_nr = 1;
4410 			} else if (pending_del_nr &&
4411 				   path->slots[0] + 1 == pending_del_slot) {
4412 				/* hop on the pending chunk */
4413 				pending_del_nr++;
4414 				pending_del_slot = path->slots[0];
4415 			} else {
4416 				BUG();
4417 			}
4418 		} else {
4419 			break;
4420 		}
4421 		should_throttle = 0;
4422 
4423 		if (found_extent &&
4424 		    (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4425 		     root == root->fs_info->tree_root)) {
4426 			btrfs_set_path_blocking(path);
4427 			bytes_deleted += extent_num_bytes;
4428 			ret = btrfs_free_extent(trans, root, extent_start,
4429 						extent_num_bytes, 0,
4430 						btrfs_header_owner(leaf),
4431 						ino, extent_offset, 0);
4432 			BUG_ON(ret);
4433 			if (btrfs_should_throttle_delayed_refs(trans, root))
4434 				btrfs_async_run_delayed_refs(root,
4435 					trans->delayed_ref_updates * 2, 0);
4436 			if (be_nice) {
4437 				if (truncate_space_check(trans, root,
4438 							 extent_num_bytes)) {
4439 					should_end = 1;
4440 				}
4441 				if (btrfs_should_throttle_delayed_refs(trans,
4442 								       root)) {
4443 					should_throttle = 1;
4444 				}
4445 			}
4446 		}
4447 
4448 		if (found_type == BTRFS_INODE_ITEM_KEY)
4449 			break;
4450 
4451 		if (path->slots[0] == 0 ||
4452 		    path->slots[0] != pending_del_slot ||
4453 		    should_throttle || should_end) {
4454 			if (pending_del_nr) {
4455 				ret = btrfs_del_items(trans, root, path,
4456 						pending_del_slot,
4457 						pending_del_nr);
4458 				if (ret) {
4459 					btrfs_abort_transaction(trans,
4460 								root, ret);
4461 					goto error;
4462 				}
4463 				pending_del_nr = 0;
4464 			}
4465 			btrfs_release_path(path);
4466 			if (should_throttle) {
4467 				unsigned long updates = trans->delayed_ref_updates;
4468 				if (updates) {
4469 					trans->delayed_ref_updates = 0;
4470 					ret = btrfs_run_delayed_refs(trans, root, updates * 2);
4471 					if (ret && !err)
4472 						err = ret;
4473 				}
4474 			}
4475 			/*
4476 			 * if we failed to refill our space rsv, bail out
4477 			 * and let the transaction restart
4478 			 */
4479 			if (should_end) {
4480 				err = -EAGAIN;
4481 				goto error;
4482 			}
4483 			goto search_again;
4484 		} else {
4485 			path->slots[0]--;
4486 		}
4487 	}
4488 out:
4489 	if (pending_del_nr) {
4490 		ret = btrfs_del_items(trans, root, path, pending_del_slot,
4491 				      pending_del_nr);
4492 		if (ret)
4493 			btrfs_abort_transaction(trans, root, ret);
4494 	}
4495 error:
4496 	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID)
4497 		btrfs_ordered_update_i_size(inode, last_size, NULL);
4498 
4499 	btrfs_free_path(path);
4500 
4501 	if (be_nice && bytes_deleted > 32 * 1024 * 1024) {
4502 		unsigned long updates = trans->delayed_ref_updates;
4503 		if (updates) {
4504 			trans->delayed_ref_updates = 0;
4505 			ret = btrfs_run_delayed_refs(trans, root, updates * 2);
4506 			if (ret && !err)
4507 				err = ret;
4508 		}
4509 	}
4510 	return err;
4511 }
4512 
4513 /*
4514  * btrfs_truncate_page - read, zero a chunk and write a page
4515  * @inode - inode that we're zeroing
4516  * @from - the offset to start zeroing
4517  * @len - the length to zero, 0 to zero the entire range respective to the
4518  *	offset
4519  * @front - zero up to the offset instead of from the offset on
4520  *
4521  * This will find the page for the "from" offset and cow the page and zero the
4522  * part we want to zero.  This is used with truncate and hole punching.
4523  */
4524 int btrfs_truncate_page(struct inode *inode, loff_t from, loff_t len,
4525 			int front)
4526 {
4527 	struct address_space *mapping = inode->i_mapping;
4528 	struct btrfs_root *root = BTRFS_I(inode)->root;
4529 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4530 	struct btrfs_ordered_extent *ordered;
4531 	struct extent_state *cached_state = NULL;
4532 	char *kaddr;
4533 	u32 blocksize = root->sectorsize;
4534 	pgoff_t index = from >> PAGE_CACHE_SHIFT;
4535 	unsigned offset = from & (PAGE_CACHE_SIZE-1);
4536 	struct page *page;
4537 	gfp_t mask = btrfs_alloc_write_mask(mapping);
4538 	int ret = 0;
4539 	u64 page_start;
4540 	u64 page_end;
4541 
4542 	if ((offset & (blocksize - 1)) == 0 &&
4543 	    (!len || ((len & (blocksize - 1)) == 0)))
4544 		goto out;
4545 	ret = btrfs_delalloc_reserve_space(inode, PAGE_CACHE_SIZE);
4546 	if (ret)
4547 		goto out;
4548 
4549 again:
4550 	page = find_or_create_page(mapping, index, mask);
4551 	if (!page) {
4552 		btrfs_delalloc_release_space(inode, PAGE_CACHE_SIZE);
4553 		ret = -ENOMEM;
4554 		goto out;
4555 	}
4556 
4557 	page_start = page_offset(page);
4558 	page_end = page_start + PAGE_CACHE_SIZE - 1;
4559 
4560 	if (!PageUptodate(page)) {
4561 		ret = btrfs_readpage(NULL, page);
4562 		lock_page(page);
4563 		if (page->mapping != mapping) {
4564 			unlock_page(page);
4565 			page_cache_release(page);
4566 			goto again;
4567 		}
4568 		if (!PageUptodate(page)) {
4569 			ret = -EIO;
4570 			goto out_unlock;
4571 		}
4572 	}
4573 	wait_on_page_writeback(page);
4574 
4575 	lock_extent_bits(io_tree, page_start, page_end, 0, &cached_state);
4576 	set_page_extent_mapped(page);
4577 
4578 	ordered = btrfs_lookup_ordered_extent(inode, page_start);
4579 	if (ordered) {
4580 		unlock_extent_cached(io_tree, page_start, page_end,
4581 				     &cached_state, GFP_NOFS);
4582 		unlock_page(page);
4583 		page_cache_release(page);
4584 		btrfs_start_ordered_extent(inode, ordered, 1);
4585 		btrfs_put_ordered_extent(ordered);
4586 		goto again;
4587 	}
4588 
4589 	clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, page_end,
4590 			  EXTENT_DIRTY | EXTENT_DELALLOC |
4591 			  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
4592 			  0, 0, &cached_state, GFP_NOFS);
4593 
4594 	ret = btrfs_set_extent_delalloc(inode, page_start, page_end,
4595 					&cached_state);
4596 	if (ret) {
4597 		unlock_extent_cached(io_tree, page_start, page_end,
4598 				     &cached_state, GFP_NOFS);
4599 		goto out_unlock;
4600 	}
4601 
4602 	if (offset != PAGE_CACHE_SIZE) {
4603 		if (!len)
4604 			len = PAGE_CACHE_SIZE - offset;
4605 		kaddr = kmap(page);
4606 		if (front)
4607 			memset(kaddr, 0, offset);
4608 		else
4609 			memset(kaddr + offset, 0, len);
4610 		flush_dcache_page(page);
4611 		kunmap(page);
4612 	}
4613 	ClearPageChecked(page);
4614 	set_page_dirty(page);
4615 	unlock_extent_cached(io_tree, page_start, page_end, &cached_state,
4616 			     GFP_NOFS);
4617 
4618 out_unlock:
4619 	if (ret)
4620 		btrfs_delalloc_release_space(inode, PAGE_CACHE_SIZE);
4621 	unlock_page(page);
4622 	page_cache_release(page);
4623 out:
4624 	return ret;
4625 }
4626 
4627 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
4628 			     u64 offset, u64 len)
4629 {
4630 	struct btrfs_trans_handle *trans;
4631 	int ret;
4632 
4633 	/*
4634 	 * Still need to make sure the inode looks like it's been updated so
4635 	 * that any holes get logged if we fsync.
4636 	 */
4637 	if (btrfs_fs_incompat(root->fs_info, NO_HOLES)) {
4638 		BTRFS_I(inode)->last_trans = root->fs_info->generation;
4639 		BTRFS_I(inode)->last_sub_trans = root->log_transid;
4640 		BTRFS_I(inode)->last_log_commit = root->last_log_commit;
4641 		return 0;
4642 	}
4643 
4644 	/*
4645 	 * 1 - for the one we're dropping
4646 	 * 1 - for the one we're adding
4647 	 * 1 - for updating the inode.
4648 	 */
4649 	trans = btrfs_start_transaction(root, 3);
4650 	if (IS_ERR(trans))
4651 		return PTR_ERR(trans);
4652 
4653 	ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
4654 	if (ret) {
4655 		btrfs_abort_transaction(trans, root, ret);
4656 		btrfs_end_transaction(trans, root);
4657 		return ret;
4658 	}
4659 
4660 	ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), offset,
4661 				       0, 0, len, 0, len, 0, 0, 0);
4662 	if (ret)
4663 		btrfs_abort_transaction(trans, root, ret);
4664 	else
4665 		btrfs_update_inode(trans, root, inode);
4666 	btrfs_end_transaction(trans, root);
4667 	return ret;
4668 }
4669 
4670 /*
4671  * This function puts in dummy file extents for the area we're creating a hole
4672  * for.  So if we are truncating this file to a larger size we need to insert
4673  * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4674  * the range between oldsize and size
4675  */
4676 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
4677 {
4678 	struct btrfs_root *root = BTRFS_I(inode)->root;
4679 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4680 	struct extent_map *em = NULL;
4681 	struct extent_state *cached_state = NULL;
4682 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
4683 	u64 hole_start = ALIGN(oldsize, root->sectorsize);
4684 	u64 block_end = ALIGN(size, root->sectorsize);
4685 	u64 last_byte;
4686 	u64 cur_offset;
4687 	u64 hole_size;
4688 	int err = 0;
4689 
4690 	/*
4691 	 * If our size started in the middle of a page we need to zero out the
4692 	 * rest of the page before we expand the i_size, otherwise we could
4693 	 * expose stale data.
4694 	 */
4695 	err = btrfs_truncate_page(inode, oldsize, 0, 0);
4696 	if (err)
4697 		return err;
4698 
4699 	if (size <= hole_start)
4700 		return 0;
4701 
4702 	while (1) {
4703 		struct btrfs_ordered_extent *ordered;
4704 
4705 		lock_extent_bits(io_tree, hole_start, block_end - 1, 0,
4706 				 &cached_state);
4707 		ordered = btrfs_lookup_ordered_range(inode, hole_start,
4708 						     block_end - hole_start);
4709 		if (!ordered)
4710 			break;
4711 		unlock_extent_cached(io_tree, hole_start, block_end - 1,
4712 				     &cached_state, GFP_NOFS);
4713 		btrfs_start_ordered_extent(inode, ordered, 1);
4714 		btrfs_put_ordered_extent(ordered);
4715 	}
4716 
4717 	cur_offset = hole_start;
4718 	while (1) {
4719 		em = btrfs_get_extent(inode, NULL, 0, cur_offset,
4720 				block_end - cur_offset, 0);
4721 		if (IS_ERR(em)) {
4722 			err = PTR_ERR(em);
4723 			em = NULL;
4724 			break;
4725 		}
4726 		last_byte = min(extent_map_end(em), block_end);
4727 		last_byte = ALIGN(last_byte , root->sectorsize);
4728 		if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
4729 			struct extent_map *hole_em;
4730 			hole_size = last_byte - cur_offset;
4731 
4732 			err = maybe_insert_hole(root, inode, cur_offset,
4733 						hole_size);
4734 			if (err)
4735 				break;
4736 			btrfs_drop_extent_cache(inode, cur_offset,
4737 						cur_offset + hole_size - 1, 0);
4738 			hole_em = alloc_extent_map();
4739 			if (!hole_em) {
4740 				set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4741 					&BTRFS_I(inode)->runtime_flags);
4742 				goto next;
4743 			}
4744 			hole_em->start = cur_offset;
4745 			hole_em->len = hole_size;
4746 			hole_em->orig_start = cur_offset;
4747 
4748 			hole_em->block_start = EXTENT_MAP_HOLE;
4749 			hole_em->block_len = 0;
4750 			hole_em->orig_block_len = 0;
4751 			hole_em->ram_bytes = hole_size;
4752 			hole_em->bdev = root->fs_info->fs_devices->latest_bdev;
4753 			hole_em->compress_type = BTRFS_COMPRESS_NONE;
4754 			hole_em->generation = root->fs_info->generation;
4755 
4756 			while (1) {
4757 				write_lock(&em_tree->lock);
4758 				err = add_extent_mapping(em_tree, hole_em, 1);
4759 				write_unlock(&em_tree->lock);
4760 				if (err != -EEXIST)
4761 					break;
4762 				btrfs_drop_extent_cache(inode, cur_offset,
4763 							cur_offset +
4764 							hole_size - 1, 0);
4765 			}
4766 			free_extent_map(hole_em);
4767 		}
4768 next:
4769 		free_extent_map(em);
4770 		em = NULL;
4771 		cur_offset = last_byte;
4772 		if (cur_offset >= block_end)
4773 			break;
4774 	}
4775 	free_extent_map(em);
4776 	unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state,
4777 			     GFP_NOFS);
4778 	return err;
4779 }
4780 
4781 static int wait_snapshoting_atomic_t(atomic_t *a)
4782 {
4783 	schedule();
4784 	return 0;
4785 }
4786 
4787 static void wait_for_snapshot_creation(struct btrfs_root *root)
4788 {
4789 	while (true) {
4790 		int ret;
4791 
4792 		ret = btrfs_start_write_no_snapshoting(root);
4793 		if (ret)
4794 			break;
4795 		wait_on_atomic_t(&root->will_be_snapshoted,
4796 				 wait_snapshoting_atomic_t,
4797 				 TASK_UNINTERRUPTIBLE);
4798 	}
4799 }
4800 
4801 static int btrfs_setsize(struct inode *inode, struct iattr *attr)
4802 {
4803 	struct btrfs_root *root = BTRFS_I(inode)->root;
4804 	struct btrfs_trans_handle *trans;
4805 	loff_t oldsize = i_size_read(inode);
4806 	loff_t newsize = attr->ia_size;
4807 	int mask = attr->ia_valid;
4808 	int ret;
4809 
4810 	/*
4811 	 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
4812 	 * special case where we need to update the times despite not having
4813 	 * these flags set.  For all other operations the VFS set these flags
4814 	 * explicitly if it wants a timestamp update.
4815 	 */
4816 	if (newsize != oldsize) {
4817 		inode_inc_iversion(inode);
4818 		if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
4819 			inode->i_ctime = inode->i_mtime =
4820 				current_fs_time(inode->i_sb);
4821 	}
4822 
4823 	if (newsize > oldsize) {
4824 		truncate_pagecache(inode, newsize);
4825 		/*
4826 		 * Don't do an expanding truncate while snapshoting is ongoing.
4827 		 * This is to ensure the snapshot captures a fully consistent
4828 		 * state of this file - if the snapshot captures this expanding
4829 		 * truncation, it must capture all writes that happened before
4830 		 * this truncation.
4831 		 */
4832 		wait_for_snapshot_creation(root);
4833 		ret = btrfs_cont_expand(inode, oldsize, newsize);
4834 		if (ret) {
4835 			btrfs_end_write_no_snapshoting(root);
4836 			return ret;
4837 		}
4838 
4839 		trans = btrfs_start_transaction(root, 1);
4840 		if (IS_ERR(trans)) {
4841 			btrfs_end_write_no_snapshoting(root);
4842 			return PTR_ERR(trans);
4843 		}
4844 
4845 		i_size_write(inode, newsize);
4846 		btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
4847 		ret = btrfs_update_inode(trans, root, inode);
4848 		btrfs_end_write_no_snapshoting(root);
4849 		btrfs_end_transaction(trans, root);
4850 	} else {
4851 
4852 		/*
4853 		 * We're truncating a file that used to have good data down to
4854 		 * zero. Make sure it gets into the ordered flush list so that
4855 		 * any new writes get down to disk quickly.
4856 		 */
4857 		if (newsize == 0)
4858 			set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
4859 				&BTRFS_I(inode)->runtime_flags);
4860 
4861 		/*
4862 		 * 1 for the orphan item we're going to add
4863 		 * 1 for the orphan item deletion.
4864 		 */
4865 		trans = btrfs_start_transaction(root, 2);
4866 		if (IS_ERR(trans))
4867 			return PTR_ERR(trans);
4868 
4869 		/*
4870 		 * We need to do this in case we fail at _any_ point during the
4871 		 * actual truncate.  Once we do the truncate_setsize we could
4872 		 * invalidate pages which forces any outstanding ordered io to
4873 		 * be instantly completed which will give us extents that need
4874 		 * to be truncated.  If we fail to get an orphan inode down we
4875 		 * could have left over extents that were never meant to live,
4876 		 * so we need to garuntee from this point on that everything
4877 		 * will be consistent.
4878 		 */
4879 		ret = btrfs_orphan_add(trans, inode);
4880 		btrfs_end_transaction(trans, root);
4881 		if (ret)
4882 			return ret;
4883 
4884 		/* we don't support swapfiles, so vmtruncate shouldn't fail */
4885 		truncate_setsize(inode, newsize);
4886 
4887 		/* Disable nonlocked read DIO to avoid the end less truncate */
4888 		btrfs_inode_block_unlocked_dio(inode);
4889 		inode_dio_wait(inode);
4890 		btrfs_inode_resume_unlocked_dio(inode);
4891 
4892 		ret = btrfs_truncate(inode);
4893 		if (ret && inode->i_nlink) {
4894 			int err;
4895 
4896 			/*
4897 			 * failed to truncate, disk_i_size is only adjusted down
4898 			 * as we remove extents, so it should represent the true
4899 			 * size of the inode, so reset the in memory size and
4900 			 * delete our orphan entry.
4901 			 */
4902 			trans = btrfs_join_transaction(root);
4903 			if (IS_ERR(trans)) {
4904 				btrfs_orphan_del(NULL, inode);
4905 				return ret;
4906 			}
4907 			i_size_write(inode, BTRFS_I(inode)->disk_i_size);
4908 			err = btrfs_orphan_del(trans, inode);
4909 			if (err)
4910 				btrfs_abort_transaction(trans, root, err);
4911 			btrfs_end_transaction(trans, root);
4912 		}
4913 	}
4914 
4915 	return ret;
4916 }
4917 
4918 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
4919 {
4920 	struct inode *inode = d_inode(dentry);
4921 	struct btrfs_root *root = BTRFS_I(inode)->root;
4922 	int err;
4923 
4924 	if (btrfs_root_readonly(root))
4925 		return -EROFS;
4926 
4927 	err = inode_change_ok(inode, attr);
4928 	if (err)
4929 		return err;
4930 
4931 	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
4932 		err = btrfs_setsize(inode, attr);
4933 		if (err)
4934 			return err;
4935 	}
4936 
4937 	if (attr->ia_valid) {
4938 		setattr_copy(inode, attr);
4939 		inode_inc_iversion(inode);
4940 		err = btrfs_dirty_inode(inode);
4941 
4942 		if (!err && attr->ia_valid & ATTR_MODE)
4943 			err = posix_acl_chmod(inode, inode->i_mode);
4944 	}
4945 
4946 	return err;
4947 }
4948 
4949 /*
4950  * While truncating the inode pages during eviction, we get the VFS calling
4951  * btrfs_invalidatepage() against each page of the inode. This is slow because
4952  * the calls to btrfs_invalidatepage() result in a huge amount of calls to
4953  * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
4954  * extent_state structures over and over, wasting lots of time.
4955  *
4956  * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
4957  * those expensive operations on a per page basis and do only the ordered io
4958  * finishing, while we release here the extent_map and extent_state structures,
4959  * without the excessive merging and splitting.
4960  */
4961 static void evict_inode_truncate_pages(struct inode *inode)
4962 {
4963 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4964 	struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
4965 	struct rb_node *node;
4966 
4967 	ASSERT(inode->i_state & I_FREEING);
4968 	truncate_inode_pages_final(&inode->i_data);
4969 
4970 	write_lock(&map_tree->lock);
4971 	while (!RB_EMPTY_ROOT(&map_tree->map)) {
4972 		struct extent_map *em;
4973 
4974 		node = rb_first(&map_tree->map);
4975 		em = rb_entry(node, struct extent_map, rb_node);
4976 		clear_bit(EXTENT_FLAG_PINNED, &em->flags);
4977 		clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
4978 		remove_extent_mapping(map_tree, em);
4979 		free_extent_map(em);
4980 		if (need_resched()) {
4981 			write_unlock(&map_tree->lock);
4982 			cond_resched();
4983 			write_lock(&map_tree->lock);
4984 		}
4985 	}
4986 	write_unlock(&map_tree->lock);
4987 
4988 	/*
4989 	 * Keep looping until we have no more ranges in the io tree.
4990 	 * We can have ongoing bios started by readpages (called from readahead)
4991 	 * that have their endio callback (extent_io.c:end_bio_extent_readpage)
4992 	 * still in progress (unlocked the pages in the bio but did not yet
4993 	 * unlocked the ranges in the io tree). Therefore this means some
4994 	 * ranges can still be locked and eviction started because before
4995 	 * submitting those bios, which are executed by a separate task (work
4996 	 * queue kthread), inode references (inode->i_count) were not taken
4997 	 * (which would be dropped in the end io callback of each bio).
4998 	 * Therefore here we effectively end up waiting for those bios and
4999 	 * anyone else holding locked ranges without having bumped the inode's
5000 	 * reference count - if we don't do it, when they access the inode's
5001 	 * io_tree to unlock a range it may be too late, leading to an
5002 	 * use-after-free issue.
5003 	 */
5004 	spin_lock(&io_tree->lock);
5005 	while (!RB_EMPTY_ROOT(&io_tree->state)) {
5006 		struct extent_state *state;
5007 		struct extent_state *cached_state = NULL;
5008 		u64 start;
5009 		u64 end;
5010 
5011 		node = rb_first(&io_tree->state);
5012 		state = rb_entry(node, struct extent_state, rb_node);
5013 		start = state->start;
5014 		end = state->end;
5015 		spin_unlock(&io_tree->lock);
5016 
5017 		lock_extent_bits(io_tree, start, end, 0, &cached_state);
5018 		clear_extent_bit(io_tree, start, end,
5019 				 EXTENT_LOCKED | EXTENT_DIRTY |
5020 				 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
5021 				 EXTENT_DEFRAG, 1, 1,
5022 				 &cached_state, GFP_NOFS);
5023 
5024 		cond_resched();
5025 		spin_lock(&io_tree->lock);
5026 	}
5027 	spin_unlock(&io_tree->lock);
5028 }
5029 
5030 void btrfs_evict_inode(struct inode *inode)
5031 {
5032 	struct btrfs_trans_handle *trans;
5033 	struct btrfs_root *root = BTRFS_I(inode)->root;
5034 	struct btrfs_block_rsv *rsv, *global_rsv;
5035 	int steal_from_global = 0;
5036 	u64 min_size = btrfs_calc_trunc_metadata_size(root, 1);
5037 	int ret;
5038 
5039 	trace_btrfs_inode_evict(inode);
5040 
5041 	evict_inode_truncate_pages(inode);
5042 
5043 	if (inode->i_nlink &&
5044 	    ((btrfs_root_refs(&root->root_item) != 0 &&
5045 	      root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
5046 	     btrfs_is_free_space_inode(inode)))
5047 		goto no_delete;
5048 
5049 	if (is_bad_inode(inode)) {
5050 		btrfs_orphan_del(NULL, inode);
5051 		goto no_delete;
5052 	}
5053 	/* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
5054 	btrfs_wait_ordered_range(inode, 0, (u64)-1);
5055 
5056 	btrfs_free_io_failure_record(inode, 0, (u64)-1);
5057 
5058 	if (root->fs_info->log_root_recovering) {
5059 		BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
5060 				 &BTRFS_I(inode)->runtime_flags));
5061 		goto no_delete;
5062 	}
5063 
5064 	if (inode->i_nlink > 0) {
5065 		BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5066 		       root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
5067 		goto no_delete;
5068 	}
5069 
5070 	ret = btrfs_commit_inode_delayed_inode(inode);
5071 	if (ret) {
5072 		btrfs_orphan_del(NULL, inode);
5073 		goto no_delete;
5074 	}
5075 
5076 	rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP);
5077 	if (!rsv) {
5078 		btrfs_orphan_del(NULL, inode);
5079 		goto no_delete;
5080 	}
5081 	rsv->size = min_size;
5082 	rsv->failfast = 1;
5083 	global_rsv = &root->fs_info->global_block_rsv;
5084 
5085 	btrfs_i_size_write(inode, 0);
5086 
5087 	/*
5088 	 * This is a bit simpler than btrfs_truncate since we've already
5089 	 * reserved our space for our orphan item in the unlink, so we just
5090 	 * need to reserve some slack space in case we add bytes and update
5091 	 * inode item when doing the truncate.
5092 	 */
5093 	while (1) {
5094 		ret = btrfs_block_rsv_refill(root, rsv, min_size,
5095 					     BTRFS_RESERVE_FLUSH_LIMIT);
5096 
5097 		/*
5098 		 * Try and steal from the global reserve since we will
5099 		 * likely not use this space anyway, we want to try as
5100 		 * hard as possible to get this to work.
5101 		 */
5102 		if (ret)
5103 			steal_from_global++;
5104 		else
5105 			steal_from_global = 0;
5106 		ret = 0;
5107 
5108 		/*
5109 		 * steal_from_global == 0: we reserved stuff, hooray!
5110 		 * steal_from_global == 1: we didn't reserve stuff, boo!
5111 		 * steal_from_global == 2: we've committed, still not a lot of
5112 		 * room but maybe we'll have room in the global reserve this
5113 		 * time.
5114 		 * steal_from_global == 3: abandon all hope!
5115 		 */
5116 		if (steal_from_global > 2) {
5117 			btrfs_warn(root->fs_info,
5118 				"Could not get space for a delete, will truncate on mount %d",
5119 				ret);
5120 			btrfs_orphan_del(NULL, inode);
5121 			btrfs_free_block_rsv(root, rsv);
5122 			goto no_delete;
5123 		}
5124 
5125 		trans = btrfs_join_transaction(root);
5126 		if (IS_ERR(trans)) {
5127 			btrfs_orphan_del(NULL, inode);
5128 			btrfs_free_block_rsv(root, rsv);
5129 			goto no_delete;
5130 		}
5131 
5132 		/*
5133 		 * We can't just steal from the global reserve, we need tomake
5134 		 * sure there is room to do it, if not we need to commit and try
5135 		 * again.
5136 		 */
5137 		if (steal_from_global) {
5138 			if (!btrfs_check_space_for_delayed_refs(trans, root))
5139 				ret = btrfs_block_rsv_migrate(global_rsv, rsv,
5140 							      min_size);
5141 			else
5142 				ret = -ENOSPC;
5143 		}
5144 
5145 		/*
5146 		 * Couldn't steal from the global reserve, we have too much
5147 		 * pending stuff built up, commit the transaction and try it
5148 		 * again.
5149 		 */
5150 		if (ret) {
5151 			ret = btrfs_commit_transaction(trans, root);
5152 			if (ret) {
5153 				btrfs_orphan_del(NULL, inode);
5154 				btrfs_free_block_rsv(root, rsv);
5155 				goto no_delete;
5156 			}
5157 			continue;
5158 		} else {
5159 			steal_from_global = 0;
5160 		}
5161 
5162 		trans->block_rsv = rsv;
5163 
5164 		ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
5165 		if (ret != -ENOSPC && ret != -EAGAIN)
5166 			break;
5167 
5168 		trans->block_rsv = &root->fs_info->trans_block_rsv;
5169 		btrfs_end_transaction(trans, root);
5170 		trans = NULL;
5171 		btrfs_btree_balance_dirty(root);
5172 	}
5173 
5174 	btrfs_free_block_rsv(root, rsv);
5175 
5176 	/*
5177 	 * Errors here aren't a big deal, it just means we leave orphan items
5178 	 * in the tree.  They will be cleaned up on the next mount.
5179 	 */
5180 	if (ret == 0) {
5181 		trans->block_rsv = root->orphan_block_rsv;
5182 		btrfs_orphan_del(trans, inode);
5183 	} else {
5184 		btrfs_orphan_del(NULL, inode);
5185 	}
5186 
5187 	trans->block_rsv = &root->fs_info->trans_block_rsv;
5188 	if (!(root == root->fs_info->tree_root ||
5189 	      root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
5190 		btrfs_return_ino(root, btrfs_ino(inode));
5191 
5192 	btrfs_end_transaction(trans, root);
5193 	btrfs_btree_balance_dirty(root);
5194 no_delete:
5195 	btrfs_remove_delayed_node(inode);
5196 	clear_inode(inode);
5197 	return;
5198 }
5199 
5200 /*
5201  * this returns the key found in the dir entry in the location pointer.
5202  * If no dir entries were found, location->objectid is 0.
5203  */
5204 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
5205 			       struct btrfs_key *location)
5206 {
5207 	const char *name = dentry->d_name.name;
5208 	int namelen = dentry->d_name.len;
5209 	struct btrfs_dir_item *di;
5210 	struct btrfs_path *path;
5211 	struct btrfs_root *root = BTRFS_I(dir)->root;
5212 	int ret = 0;
5213 
5214 	path = btrfs_alloc_path();
5215 	if (!path)
5216 		return -ENOMEM;
5217 
5218 	di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir), name,
5219 				    namelen, 0);
5220 	if (IS_ERR(di))
5221 		ret = PTR_ERR(di);
5222 
5223 	if (IS_ERR_OR_NULL(di))
5224 		goto out_err;
5225 
5226 	btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5227 out:
5228 	btrfs_free_path(path);
5229 	return ret;
5230 out_err:
5231 	location->objectid = 0;
5232 	goto out;
5233 }
5234 
5235 /*
5236  * when we hit a tree root in a directory, the btrfs part of the inode
5237  * needs to be changed to reflect the root directory of the tree root.  This
5238  * is kind of like crossing a mount point.
5239  */
5240 static int fixup_tree_root_location(struct btrfs_root *root,
5241 				    struct inode *dir,
5242 				    struct dentry *dentry,
5243 				    struct btrfs_key *location,
5244 				    struct btrfs_root **sub_root)
5245 {
5246 	struct btrfs_path *path;
5247 	struct btrfs_root *new_root;
5248 	struct btrfs_root_ref *ref;
5249 	struct extent_buffer *leaf;
5250 	struct btrfs_key key;
5251 	int ret;
5252 	int err = 0;
5253 
5254 	path = btrfs_alloc_path();
5255 	if (!path) {
5256 		err = -ENOMEM;
5257 		goto out;
5258 	}
5259 
5260 	err = -ENOENT;
5261 	key.objectid = BTRFS_I(dir)->root->root_key.objectid;
5262 	key.type = BTRFS_ROOT_REF_KEY;
5263 	key.offset = location->objectid;
5264 
5265 	ret = btrfs_search_slot(NULL, root->fs_info->tree_root, &key, path,
5266 				0, 0);
5267 	if (ret) {
5268 		if (ret < 0)
5269 			err = ret;
5270 		goto out;
5271 	}
5272 
5273 	leaf = path->nodes[0];
5274 	ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5275 	if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) ||
5276 	    btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
5277 		goto out;
5278 
5279 	ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
5280 				   (unsigned long)(ref + 1),
5281 				   dentry->d_name.len);
5282 	if (ret)
5283 		goto out;
5284 
5285 	btrfs_release_path(path);
5286 
5287 	new_root = btrfs_read_fs_root_no_name(root->fs_info, location);
5288 	if (IS_ERR(new_root)) {
5289 		err = PTR_ERR(new_root);
5290 		goto out;
5291 	}
5292 
5293 	*sub_root = new_root;
5294 	location->objectid = btrfs_root_dirid(&new_root->root_item);
5295 	location->type = BTRFS_INODE_ITEM_KEY;
5296 	location->offset = 0;
5297 	err = 0;
5298 out:
5299 	btrfs_free_path(path);
5300 	return err;
5301 }
5302 
5303 static void inode_tree_add(struct inode *inode)
5304 {
5305 	struct btrfs_root *root = BTRFS_I(inode)->root;
5306 	struct btrfs_inode *entry;
5307 	struct rb_node **p;
5308 	struct rb_node *parent;
5309 	struct rb_node *new = &BTRFS_I(inode)->rb_node;
5310 	u64 ino = btrfs_ino(inode);
5311 
5312 	if (inode_unhashed(inode))
5313 		return;
5314 	parent = NULL;
5315 	spin_lock(&root->inode_lock);
5316 	p = &root->inode_tree.rb_node;
5317 	while (*p) {
5318 		parent = *p;
5319 		entry = rb_entry(parent, struct btrfs_inode, rb_node);
5320 
5321 		if (ino < btrfs_ino(&entry->vfs_inode))
5322 			p = &parent->rb_left;
5323 		else if (ino > btrfs_ino(&entry->vfs_inode))
5324 			p = &parent->rb_right;
5325 		else {
5326 			WARN_ON(!(entry->vfs_inode.i_state &
5327 				  (I_WILL_FREE | I_FREEING)));
5328 			rb_replace_node(parent, new, &root->inode_tree);
5329 			RB_CLEAR_NODE(parent);
5330 			spin_unlock(&root->inode_lock);
5331 			return;
5332 		}
5333 	}
5334 	rb_link_node(new, parent, p);
5335 	rb_insert_color(new, &root->inode_tree);
5336 	spin_unlock(&root->inode_lock);
5337 }
5338 
5339 static void inode_tree_del(struct inode *inode)
5340 {
5341 	struct btrfs_root *root = BTRFS_I(inode)->root;
5342 	int empty = 0;
5343 
5344 	spin_lock(&root->inode_lock);
5345 	if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
5346 		rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
5347 		RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
5348 		empty = RB_EMPTY_ROOT(&root->inode_tree);
5349 	}
5350 	spin_unlock(&root->inode_lock);
5351 
5352 	if (empty && btrfs_root_refs(&root->root_item) == 0) {
5353 		synchronize_srcu(&root->fs_info->subvol_srcu);
5354 		spin_lock(&root->inode_lock);
5355 		empty = RB_EMPTY_ROOT(&root->inode_tree);
5356 		spin_unlock(&root->inode_lock);
5357 		if (empty)
5358 			btrfs_add_dead_root(root);
5359 	}
5360 }
5361 
5362 void btrfs_invalidate_inodes(struct btrfs_root *root)
5363 {
5364 	struct rb_node *node;
5365 	struct rb_node *prev;
5366 	struct btrfs_inode *entry;
5367 	struct inode *inode;
5368 	u64 objectid = 0;
5369 
5370 	if (!test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
5371 		WARN_ON(btrfs_root_refs(&root->root_item) != 0);
5372 
5373 	spin_lock(&root->inode_lock);
5374 again:
5375 	node = root->inode_tree.rb_node;
5376 	prev = NULL;
5377 	while (node) {
5378 		prev = node;
5379 		entry = rb_entry(node, struct btrfs_inode, rb_node);
5380 
5381 		if (objectid < btrfs_ino(&entry->vfs_inode))
5382 			node = node->rb_left;
5383 		else if (objectid > btrfs_ino(&entry->vfs_inode))
5384 			node = node->rb_right;
5385 		else
5386 			break;
5387 	}
5388 	if (!node) {
5389 		while (prev) {
5390 			entry = rb_entry(prev, struct btrfs_inode, rb_node);
5391 			if (objectid <= btrfs_ino(&entry->vfs_inode)) {
5392 				node = prev;
5393 				break;
5394 			}
5395 			prev = rb_next(prev);
5396 		}
5397 	}
5398 	while (node) {
5399 		entry = rb_entry(node, struct btrfs_inode, rb_node);
5400 		objectid = btrfs_ino(&entry->vfs_inode) + 1;
5401 		inode = igrab(&entry->vfs_inode);
5402 		if (inode) {
5403 			spin_unlock(&root->inode_lock);
5404 			if (atomic_read(&inode->i_count) > 1)
5405 				d_prune_aliases(inode);
5406 			/*
5407 			 * btrfs_drop_inode will have it removed from
5408 			 * the inode cache when its usage count
5409 			 * hits zero.
5410 			 */
5411 			iput(inode);
5412 			cond_resched();
5413 			spin_lock(&root->inode_lock);
5414 			goto again;
5415 		}
5416 
5417 		if (cond_resched_lock(&root->inode_lock))
5418 			goto again;
5419 
5420 		node = rb_next(node);
5421 	}
5422 	spin_unlock(&root->inode_lock);
5423 }
5424 
5425 static int btrfs_init_locked_inode(struct inode *inode, void *p)
5426 {
5427 	struct btrfs_iget_args *args = p;
5428 	inode->i_ino = args->location->objectid;
5429 	memcpy(&BTRFS_I(inode)->location, args->location,
5430 	       sizeof(*args->location));
5431 	BTRFS_I(inode)->root = args->root;
5432 	return 0;
5433 }
5434 
5435 static int btrfs_find_actor(struct inode *inode, void *opaque)
5436 {
5437 	struct btrfs_iget_args *args = opaque;
5438 	return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5439 		args->root == BTRFS_I(inode)->root;
5440 }
5441 
5442 static struct inode *btrfs_iget_locked(struct super_block *s,
5443 				       struct btrfs_key *location,
5444 				       struct btrfs_root *root)
5445 {
5446 	struct inode *inode;
5447 	struct btrfs_iget_args args;
5448 	unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5449 
5450 	args.location = location;
5451 	args.root = root;
5452 
5453 	inode = iget5_locked(s, hashval, btrfs_find_actor,
5454 			     btrfs_init_locked_inode,
5455 			     (void *)&args);
5456 	return inode;
5457 }
5458 
5459 /* Get an inode object given its location and corresponding root.
5460  * Returns in *is_new if the inode was read from disk
5461  */
5462 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
5463 			 struct btrfs_root *root, int *new)
5464 {
5465 	struct inode *inode;
5466 
5467 	inode = btrfs_iget_locked(s, location, root);
5468 	if (!inode)
5469 		return ERR_PTR(-ENOMEM);
5470 
5471 	if (inode->i_state & I_NEW) {
5472 		btrfs_read_locked_inode(inode);
5473 		if (!is_bad_inode(inode)) {
5474 			inode_tree_add(inode);
5475 			unlock_new_inode(inode);
5476 			if (new)
5477 				*new = 1;
5478 		} else {
5479 			unlock_new_inode(inode);
5480 			iput(inode);
5481 			inode = ERR_PTR(-ESTALE);
5482 		}
5483 	}
5484 
5485 	return inode;
5486 }
5487 
5488 static struct inode *new_simple_dir(struct super_block *s,
5489 				    struct btrfs_key *key,
5490 				    struct btrfs_root *root)
5491 {
5492 	struct inode *inode = new_inode(s);
5493 
5494 	if (!inode)
5495 		return ERR_PTR(-ENOMEM);
5496 
5497 	BTRFS_I(inode)->root = root;
5498 	memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
5499 	set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
5500 
5501 	inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
5502 	inode->i_op = &btrfs_dir_ro_inode_operations;
5503 	inode->i_fop = &simple_dir_operations;
5504 	inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5505 	inode->i_mtime = CURRENT_TIME;
5506 	inode->i_atime = inode->i_mtime;
5507 	inode->i_ctime = inode->i_mtime;
5508 	BTRFS_I(inode)->i_otime = inode->i_mtime;
5509 
5510 	return inode;
5511 }
5512 
5513 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
5514 {
5515 	struct inode *inode;
5516 	struct btrfs_root *root = BTRFS_I(dir)->root;
5517 	struct btrfs_root *sub_root = root;
5518 	struct btrfs_key location;
5519 	int index;
5520 	int ret = 0;
5521 
5522 	if (dentry->d_name.len > BTRFS_NAME_LEN)
5523 		return ERR_PTR(-ENAMETOOLONG);
5524 
5525 	ret = btrfs_inode_by_name(dir, dentry, &location);
5526 	if (ret < 0)
5527 		return ERR_PTR(ret);
5528 
5529 	if (location.objectid == 0)
5530 		return ERR_PTR(-ENOENT);
5531 
5532 	if (location.type == BTRFS_INODE_ITEM_KEY) {
5533 		inode = btrfs_iget(dir->i_sb, &location, root, NULL);
5534 		return inode;
5535 	}
5536 
5537 	BUG_ON(location.type != BTRFS_ROOT_ITEM_KEY);
5538 
5539 	index = srcu_read_lock(&root->fs_info->subvol_srcu);
5540 	ret = fixup_tree_root_location(root, dir, dentry,
5541 				       &location, &sub_root);
5542 	if (ret < 0) {
5543 		if (ret != -ENOENT)
5544 			inode = ERR_PTR(ret);
5545 		else
5546 			inode = new_simple_dir(dir->i_sb, &location, sub_root);
5547 	} else {
5548 		inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
5549 	}
5550 	srcu_read_unlock(&root->fs_info->subvol_srcu, index);
5551 
5552 	if (!IS_ERR(inode) && root != sub_root) {
5553 		down_read(&root->fs_info->cleanup_work_sem);
5554 		if (!(inode->i_sb->s_flags & MS_RDONLY))
5555 			ret = btrfs_orphan_cleanup(sub_root);
5556 		up_read(&root->fs_info->cleanup_work_sem);
5557 		if (ret) {
5558 			iput(inode);
5559 			inode = ERR_PTR(ret);
5560 		}
5561 	}
5562 
5563 	return inode;
5564 }
5565 
5566 static int btrfs_dentry_delete(const struct dentry *dentry)
5567 {
5568 	struct btrfs_root *root;
5569 	struct inode *inode = d_inode(dentry);
5570 
5571 	if (!inode && !IS_ROOT(dentry))
5572 		inode = d_inode(dentry->d_parent);
5573 
5574 	if (inode) {
5575 		root = BTRFS_I(inode)->root;
5576 		if (btrfs_root_refs(&root->root_item) == 0)
5577 			return 1;
5578 
5579 		if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
5580 			return 1;
5581 	}
5582 	return 0;
5583 }
5584 
5585 static void btrfs_dentry_release(struct dentry *dentry)
5586 {
5587 	kfree(dentry->d_fsdata);
5588 }
5589 
5590 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
5591 				   unsigned int flags)
5592 {
5593 	struct inode *inode;
5594 
5595 	inode = btrfs_lookup_dentry(dir, dentry);
5596 	if (IS_ERR(inode)) {
5597 		if (PTR_ERR(inode) == -ENOENT)
5598 			inode = NULL;
5599 		else
5600 			return ERR_CAST(inode);
5601 	}
5602 
5603 	return d_splice_alias(inode, dentry);
5604 }
5605 
5606 unsigned char btrfs_filetype_table[] = {
5607 	DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
5608 };
5609 
5610 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
5611 {
5612 	struct inode *inode = file_inode(file);
5613 	struct btrfs_root *root = BTRFS_I(inode)->root;
5614 	struct btrfs_item *item;
5615 	struct btrfs_dir_item *di;
5616 	struct btrfs_key key;
5617 	struct btrfs_key found_key;
5618 	struct btrfs_path *path;
5619 	struct list_head ins_list;
5620 	struct list_head del_list;
5621 	int ret;
5622 	struct extent_buffer *leaf;
5623 	int slot;
5624 	unsigned char d_type;
5625 	int over = 0;
5626 	u32 di_cur;
5627 	u32 di_total;
5628 	u32 di_len;
5629 	int key_type = BTRFS_DIR_INDEX_KEY;
5630 	char tmp_name[32];
5631 	char *name_ptr;
5632 	int name_len;
5633 	int is_curr = 0;	/* ctx->pos points to the current index? */
5634 
5635 	/* FIXME, use a real flag for deciding about the key type */
5636 	if (root->fs_info->tree_root == root)
5637 		key_type = BTRFS_DIR_ITEM_KEY;
5638 
5639 	if (!dir_emit_dots(file, ctx))
5640 		return 0;
5641 
5642 	path = btrfs_alloc_path();
5643 	if (!path)
5644 		return -ENOMEM;
5645 
5646 	path->reada = 1;
5647 
5648 	if (key_type == BTRFS_DIR_INDEX_KEY) {
5649 		INIT_LIST_HEAD(&ins_list);
5650 		INIT_LIST_HEAD(&del_list);
5651 		btrfs_get_delayed_items(inode, &ins_list, &del_list);
5652 	}
5653 
5654 	key.type = key_type;
5655 	key.offset = ctx->pos;
5656 	key.objectid = btrfs_ino(inode);
5657 
5658 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5659 	if (ret < 0)
5660 		goto err;
5661 
5662 	while (1) {
5663 		leaf = path->nodes[0];
5664 		slot = path->slots[0];
5665 		if (slot >= btrfs_header_nritems(leaf)) {
5666 			ret = btrfs_next_leaf(root, path);
5667 			if (ret < 0)
5668 				goto err;
5669 			else if (ret > 0)
5670 				break;
5671 			continue;
5672 		}
5673 
5674 		item = btrfs_item_nr(slot);
5675 		btrfs_item_key_to_cpu(leaf, &found_key, slot);
5676 
5677 		if (found_key.objectid != key.objectid)
5678 			break;
5679 		if (found_key.type != key_type)
5680 			break;
5681 		if (found_key.offset < ctx->pos)
5682 			goto next;
5683 		if (key_type == BTRFS_DIR_INDEX_KEY &&
5684 		    btrfs_should_delete_dir_index(&del_list,
5685 						  found_key.offset))
5686 			goto next;
5687 
5688 		ctx->pos = found_key.offset;
5689 		is_curr = 1;
5690 
5691 		di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
5692 		di_cur = 0;
5693 		di_total = btrfs_item_size(leaf, item);
5694 
5695 		while (di_cur < di_total) {
5696 			struct btrfs_key location;
5697 
5698 			if (verify_dir_item(root, leaf, di))
5699 				break;
5700 
5701 			name_len = btrfs_dir_name_len(leaf, di);
5702 			if (name_len <= sizeof(tmp_name)) {
5703 				name_ptr = tmp_name;
5704 			} else {
5705 				name_ptr = kmalloc(name_len, GFP_NOFS);
5706 				if (!name_ptr) {
5707 					ret = -ENOMEM;
5708 					goto err;
5709 				}
5710 			}
5711 			read_extent_buffer(leaf, name_ptr,
5712 					   (unsigned long)(di + 1), name_len);
5713 
5714 			d_type = btrfs_filetype_table[btrfs_dir_type(leaf, di)];
5715 			btrfs_dir_item_key_to_cpu(leaf, di, &location);
5716 
5717 
5718 			/* is this a reference to our own snapshot? If so
5719 			 * skip it.
5720 			 *
5721 			 * In contrast to old kernels, we insert the snapshot's
5722 			 * dir item and dir index after it has been created, so
5723 			 * we won't find a reference to our own snapshot. We
5724 			 * still keep the following code for backward
5725 			 * compatibility.
5726 			 */
5727 			if (location.type == BTRFS_ROOT_ITEM_KEY &&
5728 			    location.objectid == root->root_key.objectid) {
5729 				over = 0;
5730 				goto skip;
5731 			}
5732 			over = !dir_emit(ctx, name_ptr, name_len,
5733 				       location.objectid, d_type);
5734 
5735 skip:
5736 			if (name_ptr != tmp_name)
5737 				kfree(name_ptr);
5738 
5739 			if (over)
5740 				goto nopos;
5741 			di_len = btrfs_dir_name_len(leaf, di) +
5742 				 btrfs_dir_data_len(leaf, di) + sizeof(*di);
5743 			di_cur += di_len;
5744 			di = (struct btrfs_dir_item *)((char *)di + di_len);
5745 		}
5746 next:
5747 		path->slots[0]++;
5748 	}
5749 
5750 	if (key_type == BTRFS_DIR_INDEX_KEY) {
5751 		if (is_curr)
5752 			ctx->pos++;
5753 		ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
5754 		if (ret)
5755 			goto nopos;
5756 	}
5757 
5758 	/* Reached end of directory/root. Bump pos past the last item. */
5759 	ctx->pos++;
5760 
5761 	/*
5762 	 * Stop new entries from being returned after we return the last
5763 	 * entry.
5764 	 *
5765 	 * New directory entries are assigned a strictly increasing
5766 	 * offset.  This means that new entries created during readdir
5767 	 * are *guaranteed* to be seen in the future by that readdir.
5768 	 * This has broken buggy programs which operate on names as
5769 	 * they're returned by readdir.  Until we re-use freed offsets
5770 	 * we have this hack to stop new entries from being returned
5771 	 * under the assumption that they'll never reach this huge
5772 	 * offset.
5773 	 *
5774 	 * This is being careful not to overflow 32bit loff_t unless the
5775 	 * last entry requires it because doing so has broken 32bit apps
5776 	 * in the past.
5777 	 */
5778 	if (key_type == BTRFS_DIR_INDEX_KEY) {
5779 		if (ctx->pos >= INT_MAX)
5780 			ctx->pos = LLONG_MAX;
5781 		else
5782 			ctx->pos = INT_MAX;
5783 	}
5784 nopos:
5785 	ret = 0;
5786 err:
5787 	if (key_type == BTRFS_DIR_INDEX_KEY)
5788 		btrfs_put_delayed_items(&ins_list, &del_list);
5789 	btrfs_free_path(path);
5790 	return ret;
5791 }
5792 
5793 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc)
5794 {
5795 	struct btrfs_root *root = BTRFS_I(inode)->root;
5796 	struct btrfs_trans_handle *trans;
5797 	int ret = 0;
5798 	bool nolock = false;
5799 
5800 	if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
5801 		return 0;
5802 
5803 	if (btrfs_fs_closing(root->fs_info) && btrfs_is_free_space_inode(inode))
5804 		nolock = true;
5805 
5806 	if (wbc->sync_mode == WB_SYNC_ALL) {
5807 		if (nolock)
5808 			trans = btrfs_join_transaction_nolock(root);
5809 		else
5810 			trans = btrfs_join_transaction(root);
5811 		if (IS_ERR(trans))
5812 			return PTR_ERR(trans);
5813 		ret = btrfs_commit_transaction(trans, root);
5814 	}
5815 	return ret;
5816 }
5817 
5818 /*
5819  * This is somewhat expensive, updating the tree every time the
5820  * inode changes.  But, it is most likely to find the inode in cache.
5821  * FIXME, needs more benchmarking...there are no reasons other than performance
5822  * to keep or drop this code.
5823  */
5824 static int btrfs_dirty_inode(struct inode *inode)
5825 {
5826 	struct btrfs_root *root = BTRFS_I(inode)->root;
5827 	struct btrfs_trans_handle *trans;
5828 	int ret;
5829 
5830 	if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
5831 		return 0;
5832 
5833 	trans = btrfs_join_transaction(root);
5834 	if (IS_ERR(trans))
5835 		return PTR_ERR(trans);
5836 
5837 	ret = btrfs_update_inode(trans, root, inode);
5838 	if (ret && ret == -ENOSPC) {
5839 		/* whoops, lets try again with the full transaction */
5840 		btrfs_end_transaction(trans, root);
5841 		trans = btrfs_start_transaction(root, 1);
5842 		if (IS_ERR(trans))
5843 			return PTR_ERR(trans);
5844 
5845 		ret = btrfs_update_inode(trans, root, inode);
5846 	}
5847 	btrfs_end_transaction(trans, root);
5848 	if (BTRFS_I(inode)->delayed_node)
5849 		btrfs_balance_delayed_items(root);
5850 
5851 	return ret;
5852 }
5853 
5854 /*
5855  * This is a copy of file_update_time.  We need this so we can return error on
5856  * ENOSPC for updating the inode in the case of file write and mmap writes.
5857  */
5858 static int btrfs_update_time(struct inode *inode, struct timespec *now,
5859 			     int flags)
5860 {
5861 	struct btrfs_root *root = BTRFS_I(inode)->root;
5862 
5863 	if (btrfs_root_readonly(root))
5864 		return -EROFS;
5865 
5866 	if (flags & S_VERSION)
5867 		inode_inc_iversion(inode);
5868 	if (flags & S_CTIME)
5869 		inode->i_ctime = *now;
5870 	if (flags & S_MTIME)
5871 		inode->i_mtime = *now;
5872 	if (flags & S_ATIME)
5873 		inode->i_atime = *now;
5874 	return btrfs_dirty_inode(inode);
5875 }
5876 
5877 /*
5878  * find the highest existing sequence number in a directory
5879  * and then set the in-memory index_cnt variable to reflect
5880  * free sequence numbers
5881  */
5882 static int btrfs_set_inode_index_count(struct inode *inode)
5883 {
5884 	struct btrfs_root *root = BTRFS_I(inode)->root;
5885 	struct btrfs_key key, found_key;
5886 	struct btrfs_path *path;
5887 	struct extent_buffer *leaf;
5888 	int ret;
5889 
5890 	key.objectid = btrfs_ino(inode);
5891 	key.type = BTRFS_DIR_INDEX_KEY;
5892 	key.offset = (u64)-1;
5893 
5894 	path = btrfs_alloc_path();
5895 	if (!path)
5896 		return -ENOMEM;
5897 
5898 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5899 	if (ret < 0)
5900 		goto out;
5901 	/* FIXME: we should be able to handle this */
5902 	if (ret == 0)
5903 		goto out;
5904 	ret = 0;
5905 
5906 	/*
5907 	 * MAGIC NUMBER EXPLANATION:
5908 	 * since we search a directory based on f_pos we have to start at 2
5909 	 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
5910 	 * else has to start at 2
5911 	 */
5912 	if (path->slots[0] == 0) {
5913 		BTRFS_I(inode)->index_cnt = 2;
5914 		goto out;
5915 	}
5916 
5917 	path->slots[0]--;
5918 
5919 	leaf = path->nodes[0];
5920 	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5921 
5922 	if (found_key.objectid != btrfs_ino(inode) ||
5923 	    found_key.type != BTRFS_DIR_INDEX_KEY) {
5924 		BTRFS_I(inode)->index_cnt = 2;
5925 		goto out;
5926 	}
5927 
5928 	BTRFS_I(inode)->index_cnt = found_key.offset + 1;
5929 out:
5930 	btrfs_free_path(path);
5931 	return ret;
5932 }
5933 
5934 /*
5935  * helper to find a free sequence number in a given directory.  This current
5936  * code is very simple, later versions will do smarter things in the btree
5937  */
5938 int btrfs_set_inode_index(struct inode *dir, u64 *index)
5939 {
5940 	int ret = 0;
5941 
5942 	if (BTRFS_I(dir)->index_cnt == (u64)-1) {
5943 		ret = btrfs_inode_delayed_dir_index_count(dir);
5944 		if (ret) {
5945 			ret = btrfs_set_inode_index_count(dir);
5946 			if (ret)
5947 				return ret;
5948 		}
5949 	}
5950 
5951 	*index = BTRFS_I(dir)->index_cnt;
5952 	BTRFS_I(dir)->index_cnt++;
5953 
5954 	return ret;
5955 }
5956 
5957 static int btrfs_insert_inode_locked(struct inode *inode)
5958 {
5959 	struct btrfs_iget_args args;
5960 	args.location = &BTRFS_I(inode)->location;
5961 	args.root = BTRFS_I(inode)->root;
5962 
5963 	return insert_inode_locked4(inode,
5964 		   btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
5965 		   btrfs_find_actor, &args);
5966 }
5967 
5968 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
5969 				     struct btrfs_root *root,
5970 				     struct inode *dir,
5971 				     const char *name, int name_len,
5972 				     u64 ref_objectid, u64 objectid,
5973 				     umode_t mode, u64 *index)
5974 {
5975 	struct inode *inode;
5976 	struct btrfs_inode_item *inode_item;
5977 	struct btrfs_key *location;
5978 	struct btrfs_path *path;
5979 	struct btrfs_inode_ref *ref;
5980 	struct btrfs_key key[2];
5981 	u32 sizes[2];
5982 	int nitems = name ? 2 : 1;
5983 	unsigned long ptr;
5984 	int ret;
5985 
5986 	path = btrfs_alloc_path();
5987 	if (!path)
5988 		return ERR_PTR(-ENOMEM);
5989 
5990 	inode = new_inode(root->fs_info->sb);
5991 	if (!inode) {
5992 		btrfs_free_path(path);
5993 		return ERR_PTR(-ENOMEM);
5994 	}
5995 
5996 	/*
5997 	 * O_TMPFILE, set link count to 0, so that after this point,
5998 	 * we fill in an inode item with the correct link count.
5999 	 */
6000 	if (!name)
6001 		set_nlink(inode, 0);
6002 
6003 	/*
6004 	 * we have to initialize this early, so we can reclaim the inode
6005 	 * number if we fail afterwards in this function.
6006 	 */
6007 	inode->i_ino = objectid;
6008 
6009 	if (dir && name) {
6010 		trace_btrfs_inode_request(dir);
6011 
6012 		ret = btrfs_set_inode_index(dir, index);
6013 		if (ret) {
6014 			btrfs_free_path(path);
6015 			iput(inode);
6016 			return ERR_PTR(ret);
6017 		}
6018 	} else if (dir) {
6019 		*index = 0;
6020 	}
6021 	/*
6022 	 * index_cnt is ignored for everything but a dir,
6023 	 * btrfs_get_inode_index_count has an explanation for the magic
6024 	 * number
6025 	 */
6026 	BTRFS_I(inode)->index_cnt = 2;
6027 	BTRFS_I(inode)->dir_index = *index;
6028 	BTRFS_I(inode)->root = root;
6029 	BTRFS_I(inode)->generation = trans->transid;
6030 	inode->i_generation = BTRFS_I(inode)->generation;
6031 
6032 	/*
6033 	 * We could have gotten an inode number from somebody who was fsynced
6034 	 * and then removed in this same transaction, so let's just set full
6035 	 * sync since it will be a full sync anyway and this will blow away the
6036 	 * old info in the log.
6037 	 */
6038 	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
6039 
6040 	key[0].objectid = objectid;
6041 	key[0].type = BTRFS_INODE_ITEM_KEY;
6042 	key[0].offset = 0;
6043 
6044 	sizes[0] = sizeof(struct btrfs_inode_item);
6045 
6046 	if (name) {
6047 		/*
6048 		 * Start new inodes with an inode_ref. This is slightly more
6049 		 * efficient for small numbers of hard links since they will
6050 		 * be packed into one item. Extended refs will kick in if we
6051 		 * add more hard links than can fit in the ref item.
6052 		 */
6053 		key[1].objectid = objectid;
6054 		key[1].type = BTRFS_INODE_REF_KEY;
6055 		key[1].offset = ref_objectid;
6056 
6057 		sizes[1] = name_len + sizeof(*ref);
6058 	}
6059 
6060 	location = &BTRFS_I(inode)->location;
6061 	location->objectid = objectid;
6062 	location->offset = 0;
6063 	location->type = BTRFS_INODE_ITEM_KEY;
6064 
6065 	ret = btrfs_insert_inode_locked(inode);
6066 	if (ret < 0)
6067 		goto fail;
6068 
6069 	path->leave_spinning = 1;
6070 	ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
6071 	if (ret != 0)
6072 		goto fail_unlock;
6073 
6074 	inode_init_owner(inode, dir, mode);
6075 	inode_set_bytes(inode, 0);
6076 
6077 	inode->i_mtime = CURRENT_TIME;
6078 	inode->i_atime = inode->i_mtime;
6079 	inode->i_ctime = inode->i_mtime;
6080 	BTRFS_I(inode)->i_otime = inode->i_mtime;
6081 
6082 	inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6083 				  struct btrfs_inode_item);
6084 	memset_extent_buffer(path->nodes[0], 0, (unsigned long)inode_item,
6085 			     sizeof(*inode_item));
6086 	fill_inode_item(trans, path->nodes[0], inode_item, inode);
6087 
6088 	if (name) {
6089 		ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6090 				     struct btrfs_inode_ref);
6091 		btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
6092 		btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
6093 		ptr = (unsigned long)(ref + 1);
6094 		write_extent_buffer(path->nodes[0], name, ptr, name_len);
6095 	}
6096 
6097 	btrfs_mark_buffer_dirty(path->nodes[0]);
6098 	btrfs_free_path(path);
6099 
6100 	btrfs_inherit_iflags(inode, dir);
6101 
6102 	if (S_ISREG(mode)) {
6103 		if (btrfs_test_opt(root, NODATASUM))
6104 			BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6105 		if (btrfs_test_opt(root, NODATACOW))
6106 			BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6107 				BTRFS_INODE_NODATASUM;
6108 	}
6109 
6110 	inode_tree_add(inode);
6111 
6112 	trace_btrfs_inode_new(inode);
6113 	btrfs_set_inode_last_trans(trans, inode);
6114 
6115 	btrfs_update_root_times(trans, root);
6116 
6117 	ret = btrfs_inode_inherit_props(trans, inode, dir);
6118 	if (ret)
6119 		btrfs_err(root->fs_info,
6120 			  "error inheriting props for ino %llu (root %llu): %d",
6121 			  btrfs_ino(inode), root->root_key.objectid, ret);
6122 
6123 	return inode;
6124 
6125 fail_unlock:
6126 	unlock_new_inode(inode);
6127 fail:
6128 	if (dir && name)
6129 		BTRFS_I(dir)->index_cnt--;
6130 	btrfs_free_path(path);
6131 	iput(inode);
6132 	return ERR_PTR(ret);
6133 }
6134 
6135 static inline u8 btrfs_inode_type(struct inode *inode)
6136 {
6137 	return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT];
6138 }
6139 
6140 /*
6141  * utility function to add 'inode' into 'parent_inode' with
6142  * a give name and a given sequence number.
6143  * if 'add_backref' is true, also insert a backref from the
6144  * inode to the parent directory.
6145  */
6146 int btrfs_add_link(struct btrfs_trans_handle *trans,
6147 		   struct inode *parent_inode, struct inode *inode,
6148 		   const char *name, int name_len, int add_backref, u64 index)
6149 {
6150 	int ret = 0;
6151 	struct btrfs_key key;
6152 	struct btrfs_root *root = BTRFS_I(parent_inode)->root;
6153 	u64 ino = btrfs_ino(inode);
6154 	u64 parent_ino = btrfs_ino(parent_inode);
6155 
6156 	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6157 		memcpy(&key, &BTRFS_I(inode)->root->root_key, sizeof(key));
6158 	} else {
6159 		key.objectid = ino;
6160 		key.type = BTRFS_INODE_ITEM_KEY;
6161 		key.offset = 0;
6162 	}
6163 
6164 	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6165 		ret = btrfs_add_root_ref(trans, root->fs_info->tree_root,
6166 					 key.objectid, root->root_key.objectid,
6167 					 parent_ino, index, name, name_len);
6168 	} else if (add_backref) {
6169 		ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
6170 					     parent_ino, index);
6171 	}
6172 
6173 	/* Nothing to clean up yet */
6174 	if (ret)
6175 		return ret;
6176 
6177 	ret = btrfs_insert_dir_item(trans, root, name, name_len,
6178 				    parent_inode, &key,
6179 				    btrfs_inode_type(inode), index);
6180 	if (ret == -EEXIST || ret == -EOVERFLOW)
6181 		goto fail_dir_item;
6182 	else if (ret) {
6183 		btrfs_abort_transaction(trans, root, ret);
6184 		return ret;
6185 	}
6186 
6187 	btrfs_i_size_write(parent_inode, parent_inode->i_size +
6188 			   name_len * 2);
6189 	inode_inc_iversion(parent_inode);
6190 	parent_inode->i_mtime = parent_inode->i_ctime = CURRENT_TIME;
6191 	ret = btrfs_update_inode(trans, root, parent_inode);
6192 	if (ret)
6193 		btrfs_abort_transaction(trans, root, ret);
6194 	return ret;
6195 
6196 fail_dir_item:
6197 	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6198 		u64 local_index;
6199 		int err;
6200 		err = btrfs_del_root_ref(trans, root->fs_info->tree_root,
6201 				 key.objectid, root->root_key.objectid,
6202 				 parent_ino, &local_index, name, name_len);
6203 
6204 	} else if (add_backref) {
6205 		u64 local_index;
6206 		int err;
6207 
6208 		err = btrfs_del_inode_ref(trans, root, name, name_len,
6209 					  ino, parent_ino, &local_index);
6210 	}
6211 	return ret;
6212 }
6213 
6214 static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
6215 			    struct inode *dir, struct dentry *dentry,
6216 			    struct inode *inode, int backref, u64 index)
6217 {
6218 	int err = btrfs_add_link(trans, dir, inode,
6219 				 dentry->d_name.name, dentry->d_name.len,
6220 				 backref, index);
6221 	if (err > 0)
6222 		err = -EEXIST;
6223 	return err;
6224 }
6225 
6226 static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
6227 			umode_t mode, dev_t rdev)
6228 {
6229 	struct btrfs_trans_handle *trans;
6230 	struct btrfs_root *root = BTRFS_I(dir)->root;
6231 	struct inode *inode = NULL;
6232 	int err;
6233 	int drop_inode = 0;
6234 	u64 objectid;
6235 	u64 index = 0;
6236 
6237 	if (!new_valid_dev(rdev))
6238 		return -EINVAL;
6239 
6240 	/*
6241 	 * 2 for inode item and ref
6242 	 * 2 for dir items
6243 	 * 1 for xattr if selinux is on
6244 	 */
6245 	trans = btrfs_start_transaction(root, 5);
6246 	if (IS_ERR(trans))
6247 		return PTR_ERR(trans);
6248 
6249 	err = btrfs_find_free_ino(root, &objectid);
6250 	if (err)
6251 		goto out_unlock;
6252 
6253 	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6254 				dentry->d_name.len, btrfs_ino(dir), objectid,
6255 				mode, &index);
6256 	if (IS_ERR(inode)) {
6257 		err = PTR_ERR(inode);
6258 		goto out_unlock;
6259 	}
6260 
6261 	/*
6262 	* If the active LSM wants to access the inode during
6263 	* d_instantiate it needs these. Smack checks to see
6264 	* if the filesystem supports xattrs by looking at the
6265 	* ops vector.
6266 	*/
6267 	inode->i_op = &btrfs_special_inode_operations;
6268 	init_special_inode(inode, inode->i_mode, rdev);
6269 
6270 	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6271 	if (err)
6272 		goto out_unlock_inode;
6273 
6274 	err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
6275 	if (err) {
6276 		goto out_unlock_inode;
6277 	} else {
6278 		btrfs_update_inode(trans, root, inode);
6279 		unlock_new_inode(inode);
6280 		d_instantiate(dentry, inode);
6281 	}
6282 
6283 out_unlock:
6284 	btrfs_end_transaction(trans, root);
6285 	btrfs_balance_delayed_items(root);
6286 	btrfs_btree_balance_dirty(root);
6287 	if (drop_inode) {
6288 		inode_dec_link_count(inode);
6289 		iput(inode);
6290 	}
6291 	return err;
6292 
6293 out_unlock_inode:
6294 	drop_inode = 1;
6295 	unlock_new_inode(inode);
6296 	goto out_unlock;
6297 
6298 }
6299 
6300 static int btrfs_create(struct inode *dir, struct dentry *dentry,
6301 			umode_t mode, bool excl)
6302 {
6303 	struct btrfs_trans_handle *trans;
6304 	struct btrfs_root *root = BTRFS_I(dir)->root;
6305 	struct inode *inode = NULL;
6306 	int drop_inode_on_err = 0;
6307 	int err;
6308 	u64 objectid;
6309 	u64 index = 0;
6310 
6311 	/*
6312 	 * 2 for inode item and ref
6313 	 * 2 for dir items
6314 	 * 1 for xattr if selinux is on
6315 	 */
6316 	trans = btrfs_start_transaction(root, 5);
6317 	if (IS_ERR(trans))
6318 		return PTR_ERR(trans);
6319 
6320 	err = btrfs_find_free_ino(root, &objectid);
6321 	if (err)
6322 		goto out_unlock;
6323 
6324 	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6325 				dentry->d_name.len, btrfs_ino(dir), objectid,
6326 				mode, &index);
6327 	if (IS_ERR(inode)) {
6328 		err = PTR_ERR(inode);
6329 		goto out_unlock;
6330 	}
6331 	drop_inode_on_err = 1;
6332 	/*
6333 	* If the active LSM wants to access the inode during
6334 	* d_instantiate it needs these. Smack checks to see
6335 	* if the filesystem supports xattrs by looking at the
6336 	* ops vector.
6337 	*/
6338 	inode->i_fop = &btrfs_file_operations;
6339 	inode->i_op = &btrfs_file_inode_operations;
6340 	inode->i_mapping->a_ops = &btrfs_aops;
6341 
6342 	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6343 	if (err)
6344 		goto out_unlock_inode;
6345 
6346 	err = btrfs_update_inode(trans, root, inode);
6347 	if (err)
6348 		goto out_unlock_inode;
6349 
6350 	err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
6351 	if (err)
6352 		goto out_unlock_inode;
6353 
6354 	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
6355 	unlock_new_inode(inode);
6356 	d_instantiate(dentry, inode);
6357 
6358 out_unlock:
6359 	btrfs_end_transaction(trans, root);
6360 	if (err && drop_inode_on_err) {
6361 		inode_dec_link_count(inode);
6362 		iput(inode);
6363 	}
6364 	btrfs_balance_delayed_items(root);
6365 	btrfs_btree_balance_dirty(root);
6366 	return err;
6367 
6368 out_unlock_inode:
6369 	unlock_new_inode(inode);
6370 	goto out_unlock;
6371 
6372 }
6373 
6374 static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6375 		      struct dentry *dentry)
6376 {
6377 	struct btrfs_trans_handle *trans;
6378 	struct btrfs_root *root = BTRFS_I(dir)->root;
6379 	struct inode *inode = d_inode(old_dentry);
6380 	u64 index;
6381 	int err;
6382 	int drop_inode = 0;
6383 
6384 	/* do not allow sys_link's with other subvols of the same device */
6385 	if (root->objectid != BTRFS_I(inode)->root->objectid)
6386 		return -EXDEV;
6387 
6388 	if (inode->i_nlink >= BTRFS_LINK_MAX)
6389 		return -EMLINK;
6390 
6391 	err = btrfs_set_inode_index(dir, &index);
6392 	if (err)
6393 		goto fail;
6394 
6395 	/*
6396 	 * 2 items for inode and inode ref
6397 	 * 2 items for dir items
6398 	 * 1 item for parent inode
6399 	 */
6400 	trans = btrfs_start_transaction(root, 5);
6401 	if (IS_ERR(trans)) {
6402 		err = PTR_ERR(trans);
6403 		goto fail;
6404 	}
6405 
6406 	/* There are several dir indexes for this inode, clear the cache. */
6407 	BTRFS_I(inode)->dir_index = 0ULL;
6408 	inc_nlink(inode);
6409 	inode_inc_iversion(inode);
6410 	inode->i_ctime = CURRENT_TIME;
6411 	ihold(inode);
6412 	set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6413 
6414 	err = btrfs_add_nondir(trans, dir, dentry, inode, 1, index);
6415 
6416 	if (err) {
6417 		drop_inode = 1;
6418 	} else {
6419 		struct dentry *parent = dentry->d_parent;
6420 		err = btrfs_update_inode(trans, root, inode);
6421 		if (err)
6422 			goto fail;
6423 		if (inode->i_nlink == 1) {
6424 			/*
6425 			 * If new hard link count is 1, it's a file created
6426 			 * with open(2) O_TMPFILE flag.
6427 			 */
6428 			err = btrfs_orphan_del(trans, inode);
6429 			if (err)
6430 				goto fail;
6431 		}
6432 		d_instantiate(dentry, inode);
6433 		btrfs_log_new_name(trans, inode, NULL, parent);
6434 	}
6435 
6436 	btrfs_end_transaction(trans, root);
6437 	btrfs_balance_delayed_items(root);
6438 fail:
6439 	if (drop_inode) {
6440 		inode_dec_link_count(inode);
6441 		iput(inode);
6442 	}
6443 	btrfs_btree_balance_dirty(root);
6444 	return err;
6445 }
6446 
6447 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
6448 {
6449 	struct inode *inode = NULL;
6450 	struct btrfs_trans_handle *trans;
6451 	struct btrfs_root *root = BTRFS_I(dir)->root;
6452 	int err = 0;
6453 	int drop_on_err = 0;
6454 	u64 objectid = 0;
6455 	u64 index = 0;
6456 
6457 	/*
6458 	 * 2 items for inode and ref
6459 	 * 2 items for dir items
6460 	 * 1 for xattr if selinux is on
6461 	 */
6462 	trans = btrfs_start_transaction(root, 5);
6463 	if (IS_ERR(trans))
6464 		return PTR_ERR(trans);
6465 
6466 	err = btrfs_find_free_ino(root, &objectid);
6467 	if (err)
6468 		goto out_fail;
6469 
6470 	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6471 				dentry->d_name.len, btrfs_ino(dir), objectid,
6472 				S_IFDIR | mode, &index);
6473 	if (IS_ERR(inode)) {
6474 		err = PTR_ERR(inode);
6475 		goto out_fail;
6476 	}
6477 
6478 	drop_on_err = 1;
6479 	/* these must be set before we unlock the inode */
6480 	inode->i_op = &btrfs_dir_inode_operations;
6481 	inode->i_fop = &btrfs_dir_file_operations;
6482 
6483 	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6484 	if (err)
6485 		goto out_fail_inode;
6486 
6487 	btrfs_i_size_write(inode, 0);
6488 	err = btrfs_update_inode(trans, root, inode);
6489 	if (err)
6490 		goto out_fail_inode;
6491 
6492 	err = btrfs_add_link(trans, dir, inode, dentry->d_name.name,
6493 			     dentry->d_name.len, 0, index);
6494 	if (err)
6495 		goto out_fail_inode;
6496 
6497 	d_instantiate(dentry, inode);
6498 	/*
6499 	 * mkdir is special.  We're unlocking after we call d_instantiate
6500 	 * to avoid a race with nfsd calling d_instantiate.
6501 	 */
6502 	unlock_new_inode(inode);
6503 	drop_on_err = 0;
6504 
6505 out_fail:
6506 	btrfs_end_transaction(trans, root);
6507 	if (drop_on_err) {
6508 		inode_dec_link_count(inode);
6509 		iput(inode);
6510 	}
6511 	btrfs_balance_delayed_items(root);
6512 	btrfs_btree_balance_dirty(root);
6513 	return err;
6514 
6515 out_fail_inode:
6516 	unlock_new_inode(inode);
6517 	goto out_fail;
6518 }
6519 
6520 /* Find next extent map of a given extent map, caller needs to ensure locks */
6521 static struct extent_map *next_extent_map(struct extent_map *em)
6522 {
6523 	struct rb_node *next;
6524 
6525 	next = rb_next(&em->rb_node);
6526 	if (!next)
6527 		return NULL;
6528 	return container_of(next, struct extent_map, rb_node);
6529 }
6530 
6531 static struct extent_map *prev_extent_map(struct extent_map *em)
6532 {
6533 	struct rb_node *prev;
6534 
6535 	prev = rb_prev(&em->rb_node);
6536 	if (!prev)
6537 		return NULL;
6538 	return container_of(prev, struct extent_map, rb_node);
6539 }
6540 
6541 /* helper for btfs_get_extent.  Given an existing extent in the tree,
6542  * the existing extent is the nearest extent to map_start,
6543  * and an extent that you want to insert, deal with overlap and insert
6544  * the best fitted new extent into the tree.
6545  */
6546 static int merge_extent_mapping(struct extent_map_tree *em_tree,
6547 				struct extent_map *existing,
6548 				struct extent_map *em,
6549 				u64 map_start)
6550 {
6551 	struct extent_map *prev;
6552 	struct extent_map *next;
6553 	u64 start;
6554 	u64 end;
6555 	u64 start_diff;
6556 
6557 	BUG_ON(map_start < em->start || map_start >= extent_map_end(em));
6558 
6559 	if (existing->start > map_start) {
6560 		next = existing;
6561 		prev = prev_extent_map(next);
6562 	} else {
6563 		prev = existing;
6564 		next = next_extent_map(prev);
6565 	}
6566 
6567 	start = prev ? extent_map_end(prev) : em->start;
6568 	start = max_t(u64, start, em->start);
6569 	end = next ? next->start : extent_map_end(em);
6570 	end = min_t(u64, end, extent_map_end(em));
6571 	start_diff = start - em->start;
6572 	em->start = start;
6573 	em->len = end - start;
6574 	if (em->block_start < EXTENT_MAP_LAST_BYTE &&
6575 	    !test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
6576 		em->block_start += start_diff;
6577 		em->block_len -= start_diff;
6578 	}
6579 	return add_extent_mapping(em_tree, em, 0);
6580 }
6581 
6582 static noinline int uncompress_inline(struct btrfs_path *path,
6583 				      struct inode *inode, struct page *page,
6584 				      size_t pg_offset, u64 extent_offset,
6585 				      struct btrfs_file_extent_item *item)
6586 {
6587 	int ret;
6588 	struct extent_buffer *leaf = path->nodes[0];
6589 	char *tmp;
6590 	size_t max_size;
6591 	unsigned long inline_size;
6592 	unsigned long ptr;
6593 	int compress_type;
6594 
6595 	WARN_ON(pg_offset != 0);
6596 	compress_type = btrfs_file_extent_compression(leaf, item);
6597 	max_size = btrfs_file_extent_ram_bytes(leaf, item);
6598 	inline_size = btrfs_file_extent_inline_item_len(leaf,
6599 					btrfs_item_nr(path->slots[0]));
6600 	tmp = kmalloc(inline_size, GFP_NOFS);
6601 	if (!tmp)
6602 		return -ENOMEM;
6603 	ptr = btrfs_file_extent_inline_start(item);
6604 
6605 	read_extent_buffer(leaf, tmp, ptr, inline_size);
6606 
6607 	max_size = min_t(unsigned long, PAGE_CACHE_SIZE, max_size);
6608 	ret = btrfs_decompress(compress_type, tmp, page,
6609 			       extent_offset, inline_size, max_size);
6610 	kfree(tmp);
6611 	return ret;
6612 }
6613 
6614 /*
6615  * a bit scary, this does extent mapping from logical file offset to the disk.
6616  * the ugly parts come from merging extents from the disk with the in-ram
6617  * representation.  This gets more complex because of the data=ordered code,
6618  * where the in-ram extents might be locked pending data=ordered completion.
6619  *
6620  * This also copies inline extents directly into the page.
6621  */
6622 
6623 struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
6624 				    size_t pg_offset, u64 start, u64 len,
6625 				    int create)
6626 {
6627 	int ret;
6628 	int err = 0;
6629 	u64 extent_start = 0;
6630 	u64 extent_end = 0;
6631 	u64 objectid = btrfs_ino(inode);
6632 	u32 found_type;
6633 	struct btrfs_path *path = NULL;
6634 	struct btrfs_root *root = BTRFS_I(inode)->root;
6635 	struct btrfs_file_extent_item *item;
6636 	struct extent_buffer *leaf;
6637 	struct btrfs_key found_key;
6638 	struct extent_map *em = NULL;
6639 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
6640 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
6641 	struct btrfs_trans_handle *trans = NULL;
6642 	const bool new_inline = !page || create;
6643 
6644 again:
6645 	read_lock(&em_tree->lock);
6646 	em = lookup_extent_mapping(em_tree, start, len);
6647 	if (em)
6648 		em->bdev = root->fs_info->fs_devices->latest_bdev;
6649 	read_unlock(&em_tree->lock);
6650 
6651 	if (em) {
6652 		if (em->start > start || em->start + em->len <= start)
6653 			free_extent_map(em);
6654 		else if (em->block_start == EXTENT_MAP_INLINE && page)
6655 			free_extent_map(em);
6656 		else
6657 			goto out;
6658 	}
6659 	em = alloc_extent_map();
6660 	if (!em) {
6661 		err = -ENOMEM;
6662 		goto out;
6663 	}
6664 	em->bdev = root->fs_info->fs_devices->latest_bdev;
6665 	em->start = EXTENT_MAP_HOLE;
6666 	em->orig_start = EXTENT_MAP_HOLE;
6667 	em->len = (u64)-1;
6668 	em->block_len = (u64)-1;
6669 
6670 	if (!path) {
6671 		path = btrfs_alloc_path();
6672 		if (!path) {
6673 			err = -ENOMEM;
6674 			goto out;
6675 		}
6676 		/*
6677 		 * Chances are we'll be called again, so go ahead and do
6678 		 * readahead
6679 		 */
6680 		path->reada = 1;
6681 	}
6682 
6683 	ret = btrfs_lookup_file_extent(trans, root, path,
6684 				       objectid, start, trans != NULL);
6685 	if (ret < 0) {
6686 		err = ret;
6687 		goto out;
6688 	}
6689 
6690 	if (ret != 0) {
6691 		if (path->slots[0] == 0)
6692 			goto not_found;
6693 		path->slots[0]--;
6694 	}
6695 
6696 	leaf = path->nodes[0];
6697 	item = btrfs_item_ptr(leaf, path->slots[0],
6698 			      struct btrfs_file_extent_item);
6699 	/* are we inside the extent that was found? */
6700 	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6701 	found_type = found_key.type;
6702 	if (found_key.objectid != objectid ||
6703 	    found_type != BTRFS_EXTENT_DATA_KEY) {
6704 		/*
6705 		 * If we backup past the first extent we want to move forward
6706 		 * and see if there is an extent in front of us, otherwise we'll
6707 		 * say there is a hole for our whole search range which can
6708 		 * cause problems.
6709 		 */
6710 		extent_end = start;
6711 		goto next;
6712 	}
6713 
6714 	found_type = btrfs_file_extent_type(leaf, item);
6715 	extent_start = found_key.offset;
6716 	if (found_type == BTRFS_FILE_EXTENT_REG ||
6717 	    found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6718 		extent_end = extent_start +
6719 		       btrfs_file_extent_num_bytes(leaf, item);
6720 	} else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
6721 		size_t size;
6722 		size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
6723 		extent_end = ALIGN(extent_start + size, root->sectorsize);
6724 	}
6725 next:
6726 	if (start >= extent_end) {
6727 		path->slots[0]++;
6728 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
6729 			ret = btrfs_next_leaf(root, path);
6730 			if (ret < 0) {
6731 				err = ret;
6732 				goto out;
6733 			}
6734 			if (ret > 0)
6735 				goto not_found;
6736 			leaf = path->nodes[0];
6737 		}
6738 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6739 		if (found_key.objectid != objectid ||
6740 		    found_key.type != BTRFS_EXTENT_DATA_KEY)
6741 			goto not_found;
6742 		if (start + len <= found_key.offset)
6743 			goto not_found;
6744 		if (start > found_key.offset)
6745 			goto next;
6746 		em->start = start;
6747 		em->orig_start = start;
6748 		em->len = found_key.offset - start;
6749 		goto not_found_em;
6750 	}
6751 
6752 	btrfs_extent_item_to_extent_map(inode, path, item, new_inline, em);
6753 
6754 	if (found_type == BTRFS_FILE_EXTENT_REG ||
6755 	    found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6756 		goto insert;
6757 	} else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
6758 		unsigned long ptr;
6759 		char *map;
6760 		size_t size;
6761 		size_t extent_offset;
6762 		size_t copy_size;
6763 
6764 		if (new_inline)
6765 			goto out;
6766 
6767 		size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
6768 		extent_offset = page_offset(page) + pg_offset - extent_start;
6769 		copy_size = min_t(u64, PAGE_CACHE_SIZE - pg_offset,
6770 				size - extent_offset);
6771 		em->start = extent_start + extent_offset;
6772 		em->len = ALIGN(copy_size, root->sectorsize);
6773 		em->orig_block_len = em->len;
6774 		em->orig_start = em->start;
6775 		ptr = btrfs_file_extent_inline_start(item) + extent_offset;
6776 		if (create == 0 && !PageUptodate(page)) {
6777 			if (btrfs_file_extent_compression(leaf, item) !=
6778 			    BTRFS_COMPRESS_NONE) {
6779 				ret = uncompress_inline(path, inode, page,
6780 							pg_offset,
6781 							extent_offset, item);
6782 				if (ret) {
6783 					err = ret;
6784 					goto out;
6785 				}
6786 			} else {
6787 				map = kmap(page);
6788 				read_extent_buffer(leaf, map + pg_offset, ptr,
6789 						   copy_size);
6790 				if (pg_offset + copy_size < PAGE_CACHE_SIZE) {
6791 					memset(map + pg_offset + copy_size, 0,
6792 					       PAGE_CACHE_SIZE - pg_offset -
6793 					       copy_size);
6794 				}
6795 				kunmap(page);
6796 			}
6797 			flush_dcache_page(page);
6798 		} else if (create && PageUptodate(page)) {
6799 			BUG();
6800 			if (!trans) {
6801 				kunmap(page);
6802 				free_extent_map(em);
6803 				em = NULL;
6804 
6805 				btrfs_release_path(path);
6806 				trans = btrfs_join_transaction(root);
6807 
6808 				if (IS_ERR(trans))
6809 					return ERR_CAST(trans);
6810 				goto again;
6811 			}
6812 			map = kmap(page);
6813 			write_extent_buffer(leaf, map + pg_offset, ptr,
6814 					    copy_size);
6815 			kunmap(page);
6816 			btrfs_mark_buffer_dirty(leaf);
6817 		}
6818 		set_extent_uptodate(io_tree, em->start,
6819 				    extent_map_end(em) - 1, NULL, GFP_NOFS);
6820 		goto insert;
6821 	}
6822 not_found:
6823 	em->start = start;
6824 	em->orig_start = start;
6825 	em->len = len;
6826 not_found_em:
6827 	em->block_start = EXTENT_MAP_HOLE;
6828 	set_bit(EXTENT_FLAG_VACANCY, &em->flags);
6829 insert:
6830 	btrfs_release_path(path);
6831 	if (em->start > start || extent_map_end(em) <= start) {
6832 		btrfs_err(root->fs_info, "bad extent! em: [%llu %llu] passed [%llu %llu]",
6833 			em->start, em->len, start, len);
6834 		err = -EIO;
6835 		goto out;
6836 	}
6837 
6838 	err = 0;
6839 	write_lock(&em_tree->lock);
6840 	ret = add_extent_mapping(em_tree, em, 0);
6841 	/* it is possible that someone inserted the extent into the tree
6842 	 * while we had the lock dropped.  It is also possible that
6843 	 * an overlapping map exists in the tree
6844 	 */
6845 	if (ret == -EEXIST) {
6846 		struct extent_map *existing;
6847 
6848 		ret = 0;
6849 
6850 		existing = search_extent_mapping(em_tree, start, len);
6851 		/*
6852 		 * existing will always be non-NULL, since there must be
6853 		 * extent causing the -EEXIST.
6854 		 */
6855 		if (start >= extent_map_end(existing) ||
6856 		    start <= existing->start) {
6857 			/*
6858 			 * The existing extent map is the one nearest to
6859 			 * the [start, start + len) range which overlaps
6860 			 */
6861 			err = merge_extent_mapping(em_tree, existing,
6862 						   em, start);
6863 			free_extent_map(existing);
6864 			if (err) {
6865 				free_extent_map(em);
6866 				em = NULL;
6867 			}
6868 		} else {
6869 			free_extent_map(em);
6870 			em = existing;
6871 			err = 0;
6872 		}
6873 	}
6874 	write_unlock(&em_tree->lock);
6875 out:
6876 
6877 	trace_btrfs_get_extent(root, em);
6878 
6879 	if (path)
6880 		btrfs_free_path(path);
6881 	if (trans) {
6882 		ret = btrfs_end_transaction(trans, root);
6883 		if (!err)
6884 			err = ret;
6885 	}
6886 	if (err) {
6887 		free_extent_map(em);
6888 		return ERR_PTR(err);
6889 	}
6890 	BUG_ON(!em); /* Error is always set */
6891 	return em;
6892 }
6893 
6894 struct extent_map *btrfs_get_extent_fiemap(struct inode *inode, struct page *page,
6895 					   size_t pg_offset, u64 start, u64 len,
6896 					   int create)
6897 {
6898 	struct extent_map *em;
6899 	struct extent_map *hole_em = NULL;
6900 	u64 range_start = start;
6901 	u64 end;
6902 	u64 found;
6903 	u64 found_end;
6904 	int err = 0;
6905 
6906 	em = btrfs_get_extent(inode, page, pg_offset, start, len, create);
6907 	if (IS_ERR(em))
6908 		return em;
6909 	if (em) {
6910 		/*
6911 		 * if our em maps to
6912 		 * -  a hole or
6913 		 * -  a pre-alloc extent,
6914 		 * there might actually be delalloc bytes behind it.
6915 		 */
6916 		if (em->block_start != EXTENT_MAP_HOLE &&
6917 		    !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
6918 			return em;
6919 		else
6920 			hole_em = em;
6921 	}
6922 
6923 	/* check to see if we've wrapped (len == -1 or similar) */
6924 	end = start + len;
6925 	if (end < start)
6926 		end = (u64)-1;
6927 	else
6928 		end -= 1;
6929 
6930 	em = NULL;
6931 
6932 	/* ok, we didn't find anything, lets look for delalloc */
6933 	found = count_range_bits(&BTRFS_I(inode)->io_tree, &range_start,
6934 				 end, len, EXTENT_DELALLOC, 1);
6935 	found_end = range_start + found;
6936 	if (found_end < range_start)
6937 		found_end = (u64)-1;
6938 
6939 	/*
6940 	 * we didn't find anything useful, return
6941 	 * the original results from get_extent()
6942 	 */
6943 	if (range_start > end || found_end <= start) {
6944 		em = hole_em;
6945 		hole_em = NULL;
6946 		goto out;
6947 	}
6948 
6949 	/* adjust the range_start to make sure it doesn't
6950 	 * go backwards from the start they passed in
6951 	 */
6952 	range_start = max(start, range_start);
6953 	found = found_end - range_start;
6954 
6955 	if (found > 0) {
6956 		u64 hole_start = start;
6957 		u64 hole_len = len;
6958 
6959 		em = alloc_extent_map();
6960 		if (!em) {
6961 			err = -ENOMEM;
6962 			goto out;
6963 		}
6964 		/*
6965 		 * when btrfs_get_extent can't find anything it
6966 		 * returns one huge hole
6967 		 *
6968 		 * make sure what it found really fits our range, and
6969 		 * adjust to make sure it is based on the start from
6970 		 * the caller
6971 		 */
6972 		if (hole_em) {
6973 			u64 calc_end = extent_map_end(hole_em);
6974 
6975 			if (calc_end <= start || (hole_em->start > end)) {
6976 				free_extent_map(hole_em);
6977 				hole_em = NULL;
6978 			} else {
6979 				hole_start = max(hole_em->start, start);
6980 				hole_len = calc_end - hole_start;
6981 			}
6982 		}
6983 		em->bdev = NULL;
6984 		if (hole_em && range_start > hole_start) {
6985 			/* our hole starts before our delalloc, so we
6986 			 * have to return just the parts of the hole
6987 			 * that go until  the delalloc starts
6988 			 */
6989 			em->len = min(hole_len,
6990 				      range_start - hole_start);
6991 			em->start = hole_start;
6992 			em->orig_start = hole_start;
6993 			/*
6994 			 * don't adjust block start at all,
6995 			 * it is fixed at EXTENT_MAP_HOLE
6996 			 */
6997 			em->block_start = hole_em->block_start;
6998 			em->block_len = hole_len;
6999 			if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
7000 				set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
7001 		} else {
7002 			em->start = range_start;
7003 			em->len = found;
7004 			em->orig_start = range_start;
7005 			em->block_start = EXTENT_MAP_DELALLOC;
7006 			em->block_len = found;
7007 		}
7008 	} else if (hole_em) {
7009 		return hole_em;
7010 	}
7011 out:
7012 
7013 	free_extent_map(hole_em);
7014 	if (err) {
7015 		free_extent_map(em);
7016 		return ERR_PTR(err);
7017 	}
7018 	return em;
7019 }
7020 
7021 static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
7022 						  u64 start, u64 len)
7023 {
7024 	struct btrfs_root *root = BTRFS_I(inode)->root;
7025 	struct extent_map *em;
7026 	struct btrfs_key ins;
7027 	u64 alloc_hint;
7028 	int ret;
7029 
7030 	alloc_hint = get_extent_allocation_hint(inode, start, len);
7031 	ret = btrfs_reserve_extent(root, len, root->sectorsize, 0,
7032 				   alloc_hint, &ins, 1, 1);
7033 	if (ret)
7034 		return ERR_PTR(ret);
7035 
7036 	em = create_pinned_em(inode, start, ins.offset, start, ins.objectid,
7037 			      ins.offset, ins.offset, ins.offset, 0);
7038 	if (IS_ERR(em)) {
7039 		btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1);
7040 		return em;
7041 	}
7042 
7043 	ret = btrfs_add_ordered_extent_dio(inode, start, ins.objectid,
7044 					   ins.offset, ins.offset, 0);
7045 	if (ret) {
7046 		btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1);
7047 		free_extent_map(em);
7048 		return ERR_PTR(ret);
7049 	}
7050 
7051 	return em;
7052 }
7053 
7054 /*
7055  * returns 1 when the nocow is safe, < 1 on error, 0 if the
7056  * block must be cow'd
7057  */
7058 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7059 			      u64 *orig_start, u64 *orig_block_len,
7060 			      u64 *ram_bytes)
7061 {
7062 	struct btrfs_trans_handle *trans;
7063 	struct btrfs_path *path;
7064 	int ret;
7065 	struct extent_buffer *leaf;
7066 	struct btrfs_root *root = BTRFS_I(inode)->root;
7067 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7068 	struct btrfs_file_extent_item *fi;
7069 	struct btrfs_key key;
7070 	u64 disk_bytenr;
7071 	u64 backref_offset;
7072 	u64 extent_end;
7073 	u64 num_bytes;
7074 	int slot;
7075 	int found_type;
7076 	bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
7077 
7078 	path = btrfs_alloc_path();
7079 	if (!path)
7080 		return -ENOMEM;
7081 
7082 	ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode),
7083 				       offset, 0);
7084 	if (ret < 0)
7085 		goto out;
7086 
7087 	slot = path->slots[0];
7088 	if (ret == 1) {
7089 		if (slot == 0) {
7090 			/* can't find the item, must cow */
7091 			ret = 0;
7092 			goto out;
7093 		}
7094 		slot--;
7095 	}
7096 	ret = 0;
7097 	leaf = path->nodes[0];
7098 	btrfs_item_key_to_cpu(leaf, &key, slot);
7099 	if (key.objectid != btrfs_ino(inode) ||
7100 	    key.type != BTRFS_EXTENT_DATA_KEY) {
7101 		/* not our file or wrong item type, must cow */
7102 		goto out;
7103 	}
7104 
7105 	if (key.offset > offset) {
7106 		/* Wrong offset, must cow */
7107 		goto out;
7108 	}
7109 
7110 	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
7111 	found_type = btrfs_file_extent_type(leaf, fi);
7112 	if (found_type != BTRFS_FILE_EXTENT_REG &&
7113 	    found_type != BTRFS_FILE_EXTENT_PREALLOC) {
7114 		/* not a regular extent, must cow */
7115 		goto out;
7116 	}
7117 
7118 	if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
7119 		goto out;
7120 
7121 	extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
7122 	if (extent_end <= offset)
7123 		goto out;
7124 
7125 	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7126 	if (disk_bytenr == 0)
7127 		goto out;
7128 
7129 	if (btrfs_file_extent_compression(leaf, fi) ||
7130 	    btrfs_file_extent_encryption(leaf, fi) ||
7131 	    btrfs_file_extent_other_encoding(leaf, fi))
7132 		goto out;
7133 
7134 	backref_offset = btrfs_file_extent_offset(leaf, fi);
7135 
7136 	if (orig_start) {
7137 		*orig_start = key.offset - backref_offset;
7138 		*orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
7139 		*ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
7140 	}
7141 
7142 	if (btrfs_extent_readonly(root, disk_bytenr))
7143 		goto out;
7144 
7145 	num_bytes = min(offset + *len, extent_end) - offset;
7146 	if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7147 		u64 range_end;
7148 
7149 		range_end = round_up(offset + num_bytes, root->sectorsize) - 1;
7150 		ret = test_range_bit(io_tree, offset, range_end,
7151 				     EXTENT_DELALLOC, 0, NULL);
7152 		if (ret) {
7153 			ret = -EAGAIN;
7154 			goto out;
7155 		}
7156 	}
7157 
7158 	btrfs_release_path(path);
7159 
7160 	/*
7161 	 * look for other files referencing this extent, if we
7162 	 * find any we must cow
7163 	 */
7164 	trans = btrfs_join_transaction(root);
7165 	if (IS_ERR(trans)) {
7166 		ret = 0;
7167 		goto out;
7168 	}
7169 
7170 	ret = btrfs_cross_ref_exist(trans, root, btrfs_ino(inode),
7171 				    key.offset - backref_offset, disk_bytenr);
7172 	btrfs_end_transaction(trans, root);
7173 	if (ret) {
7174 		ret = 0;
7175 		goto out;
7176 	}
7177 
7178 	/*
7179 	 * adjust disk_bytenr and num_bytes to cover just the bytes
7180 	 * in this extent we are about to write.  If there
7181 	 * are any csums in that range we have to cow in order
7182 	 * to keep the csums correct
7183 	 */
7184 	disk_bytenr += backref_offset;
7185 	disk_bytenr += offset - key.offset;
7186 	if (csum_exist_in_range(root, disk_bytenr, num_bytes))
7187 				goto out;
7188 	/*
7189 	 * all of the above have passed, it is safe to overwrite this extent
7190 	 * without cow
7191 	 */
7192 	*len = num_bytes;
7193 	ret = 1;
7194 out:
7195 	btrfs_free_path(path);
7196 	return ret;
7197 }
7198 
7199 bool btrfs_page_exists_in_range(struct inode *inode, loff_t start, loff_t end)
7200 {
7201 	struct radix_tree_root *root = &inode->i_mapping->page_tree;
7202 	int found = false;
7203 	void **pagep = NULL;
7204 	struct page *page = NULL;
7205 	int start_idx;
7206 	int end_idx;
7207 
7208 	start_idx = start >> PAGE_CACHE_SHIFT;
7209 
7210 	/*
7211 	 * end is the last byte in the last page.  end == start is legal
7212 	 */
7213 	end_idx = end >> PAGE_CACHE_SHIFT;
7214 
7215 	rcu_read_lock();
7216 
7217 	/* Most of the code in this while loop is lifted from
7218 	 * find_get_page.  It's been modified to begin searching from a
7219 	 * page and return just the first page found in that range.  If the
7220 	 * found idx is less than or equal to the end idx then we know that
7221 	 * a page exists.  If no pages are found or if those pages are
7222 	 * outside of the range then we're fine (yay!) */
7223 	while (page == NULL &&
7224 	       radix_tree_gang_lookup_slot(root, &pagep, NULL, start_idx, 1)) {
7225 		page = radix_tree_deref_slot(pagep);
7226 		if (unlikely(!page))
7227 			break;
7228 
7229 		if (radix_tree_exception(page)) {
7230 			if (radix_tree_deref_retry(page)) {
7231 				page = NULL;
7232 				continue;
7233 			}
7234 			/*
7235 			 * Otherwise, shmem/tmpfs must be storing a swap entry
7236 			 * here as an exceptional entry: so return it without
7237 			 * attempting to raise page count.
7238 			 */
7239 			page = NULL;
7240 			break; /* TODO: Is this relevant for this use case? */
7241 		}
7242 
7243 		if (!page_cache_get_speculative(page)) {
7244 			page = NULL;
7245 			continue;
7246 		}
7247 
7248 		/*
7249 		 * Has the page moved?
7250 		 * This is part of the lockless pagecache protocol. See
7251 		 * include/linux/pagemap.h for details.
7252 		 */
7253 		if (unlikely(page != *pagep)) {
7254 			page_cache_release(page);
7255 			page = NULL;
7256 		}
7257 	}
7258 
7259 	if (page) {
7260 		if (page->index <= end_idx)
7261 			found = true;
7262 		page_cache_release(page);
7263 	}
7264 
7265 	rcu_read_unlock();
7266 	return found;
7267 }
7268 
7269 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
7270 			      struct extent_state **cached_state, int writing)
7271 {
7272 	struct btrfs_ordered_extent *ordered;
7273 	int ret = 0;
7274 
7275 	while (1) {
7276 		lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7277 				 0, cached_state);
7278 		/*
7279 		 * We're concerned with the entire range that we're going to be
7280 		 * doing DIO to, so we need to make sure theres no ordered
7281 		 * extents in this range.
7282 		 */
7283 		ordered = btrfs_lookup_ordered_range(inode, lockstart,
7284 						     lockend - lockstart + 1);
7285 
7286 		/*
7287 		 * We need to make sure there are no buffered pages in this
7288 		 * range either, we could have raced between the invalidate in
7289 		 * generic_file_direct_write and locking the extent.  The
7290 		 * invalidate needs to happen so that reads after a write do not
7291 		 * get stale data.
7292 		 */
7293 		if (!ordered &&
7294 		    (!writing ||
7295 		     !btrfs_page_exists_in_range(inode, lockstart, lockend)))
7296 			break;
7297 
7298 		unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7299 				     cached_state, GFP_NOFS);
7300 
7301 		if (ordered) {
7302 			btrfs_start_ordered_extent(inode, ordered, 1);
7303 			btrfs_put_ordered_extent(ordered);
7304 		} else {
7305 			/* Screw you mmap */
7306 			ret = btrfs_fdatawrite_range(inode, lockstart, lockend);
7307 			if (ret)
7308 				break;
7309 			ret = filemap_fdatawait_range(inode->i_mapping,
7310 						      lockstart,
7311 						      lockend);
7312 			if (ret)
7313 				break;
7314 
7315 			/*
7316 			 * If we found a page that couldn't be invalidated just
7317 			 * fall back to buffered.
7318 			 */
7319 			ret = invalidate_inode_pages2_range(inode->i_mapping,
7320 					lockstart >> PAGE_CACHE_SHIFT,
7321 					lockend >> PAGE_CACHE_SHIFT);
7322 			if (ret)
7323 				break;
7324 		}
7325 
7326 		cond_resched();
7327 	}
7328 
7329 	return ret;
7330 }
7331 
7332 static struct extent_map *create_pinned_em(struct inode *inode, u64 start,
7333 					   u64 len, u64 orig_start,
7334 					   u64 block_start, u64 block_len,
7335 					   u64 orig_block_len, u64 ram_bytes,
7336 					   int type)
7337 {
7338 	struct extent_map_tree *em_tree;
7339 	struct extent_map *em;
7340 	struct btrfs_root *root = BTRFS_I(inode)->root;
7341 	int ret;
7342 
7343 	em_tree = &BTRFS_I(inode)->extent_tree;
7344 	em = alloc_extent_map();
7345 	if (!em)
7346 		return ERR_PTR(-ENOMEM);
7347 
7348 	em->start = start;
7349 	em->orig_start = orig_start;
7350 	em->mod_start = start;
7351 	em->mod_len = len;
7352 	em->len = len;
7353 	em->block_len = block_len;
7354 	em->block_start = block_start;
7355 	em->bdev = root->fs_info->fs_devices->latest_bdev;
7356 	em->orig_block_len = orig_block_len;
7357 	em->ram_bytes = ram_bytes;
7358 	em->generation = -1;
7359 	set_bit(EXTENT_FLAG_PINNED, &em->flags);
7360 	if (type == BTRFS_ORDERED_PREALLOC)
7361 		set_bit(EXTENT_FLAG_FILLING, &em->flags);
7362 
7363 	do {
7364 		btrfs_drop_extent_cache(inode, em->start,
7365 				em->start + em->len - 1, 0);
7366 		write_lock(&em_tree->lock);
7367 		ret = add_extent_mapping(em_tree, em, 1);
7368 		write_unlock(&em_tree->lock);
7369 	} while (ret == -EEXIST);
7370 
7371 	if (ret) {
7372 		free_extent_map(em);
7373 		return ERR_PTR(ret);
7374 	}
7375 
7376 	return em;
7377 }
7378 
7379 
7380 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
7381 				   struct buffer_head *bh_result, int create)
7382 {
7383 	struct extent_map *em;
7384 	struct btrfs_root *root = BTRFS_I(inode)->root;
7385 	struct extent_state *cached_state = NULL;
7386 	u64 start = iblock << inode->i_blkbits;
7387 	u64 lockstart, lockend;
7388 	u64 len = bh_result->b_size;
7389 	u64 *outstanding_extents = NULL;
7390 	int unlock_bits = EXTENT_LOCKED;
7391 	int ret = 0;
7392 
7393 	if (create)
7394 		unlock_bits |= EXTENT_DIRTY;
7395 	else
7396 		len = min_t(u64, len, root->sectorsize);
7397 
7398 	lockstart = start;
7399 	lockend = start + len - 1;
7400 
7401 	if (current->journal_info) {
7402 		/*
7403 		 * Need to pull our outstanding extents and set journal_info to NULL so
7404 		 * that anything that needs to check if there's a transction doesn't get
7405 		 * confused.
7406 		 */
7407 		outstanding_extents = current->journal_info;
7408 		current->journal_info = NULL;
7409 	}
7410 
7411 	/*
7412 	 * If this errors out it's because we couldn't invalidate pagecache for
7413 	 * this range and we need to fallback to buffered.
7414 	 */
7415 	if (lock_extent_direct(inode, lockstart, lockend, &cached_state, create))
7416 		return -ENOTBLK;
7417 
7418 	em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
7419 	if (IS_ERR(em)) {
7420 		ret = PTR_ERR(em);
7421 		goto unlock_err;
7422 	}
7423 
7424 	/*
7425 	 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7426 	 * io.  INLINE is special, and we could probably kludge it in here, but
7427 	 * it's still buffered so for safety lets just fall back to the generic
7428 	 * buffered path.
7429 	 *
7430 	 * For COMPRESSED we _have_ to read the entire extent in so we can
7431 	 * decompress it, so there will be buffering required no matter what we
7432 	 * do, so go ahead and fallback to buffered.
7433 	 *
7434 	 * We return -ENOTBLK because thats what makes DIO go ahead and go back
7435 	 * to buffered IO.  Don't blame me, this is the price we pay for using
7436 	 * the generic code.
7437 	 */
7438 	if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
7439 	    em->block_start == EXTENT_MAP_INLINE) {
7440 		free_extent_map(em);
7441 		ret = -ENOTBLK;
7442 		goto unlock_err;
7443 	}
7444 
7445 	/* Just a good old fashioned hole, return */
7446 	if (!create && (em->block_start == EXTENT_MAP_HOLE ||
7447 			test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
7448 		free_extent_map(em);
7449 		goto unlock_err;
7450 	}
7451 
7452 	/*
7453 	 * We don't allocate a new extent in the following cases
7454 	 *
7455 	 * 1) The inode is marked as NODATACOW.  In this case we'll just use the
7456 	 * existing extent.
7457 	 * 2) The extent is marked as PREALLOC.  We're good to go here and can
7458 	 * just use the extent.
7459 	 *
7460 	 */
7461 	if (!create) {
7462 		len = min(len, em->len - (start - em->start));
7463 		lockstart = start + len;
7464 		goto unlock;
7465 	}
7466 
7467 	if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
7468 	    ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
7469 	     em->block_start != EXTENT_MAP_HOLE)) {
7470 		int type;
7471 		u64 block_start, orig_start, orig_block_len, ram_bytes;
7472 
7473 		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7474 			type = BTRFS_ORDERED_PREALLOC;
7475 		else
7476 			type = BTRFS_ORDERED_NOCOW;
7477 		len = min(len, em->len - (start - em->start));
7478 		block_start = em->block_start + (start - em->start);
7479 
7480 		if (can_nocow_extent(inode, start, &len, &orig_start,
7481 				     &orig_block_len, &ram_bytes) == 1) {
7482 			if (type == BTRFS_ORDERED_PREALLOC) {
7483 				free_extent_map(em);
7484 				em = create_pinned_em(inode, start, len,
7485 						       orig_start,
7486 						       block_start, len,
7487 						       orig_block_len,
7488 						       ram_bytes, type);
7489 				if (IS_ERR(em)) {
7490 					ret = PTR_ERR(em);
7491 					goto unlock_err;
7492 				}
7493 			}
7494 
7495 			ret = btrfs_add_ordered_extent_dio(inode, start,
7496 					   block_start, len, len, type);
7497 			if (ret) {
7498 				free_extent_map(em);
7499 				goto unlock_err;
7500 			}
7501 			goto unlock;
7502 		}
7503 	}
7504 
7505 	/*
7506 	 * this will cow the extent, reset the len in case we changed
7507 	 * it above
7508 	 */
7509 	len = bh_result->b_size;
7510 	free_extent_map(em);
7511 	em = btrfs_new_extent_direct(inode, start, len);
7512 	if (IS_ERR(em)) {
7513 		ret = PTR_ERR(em);
7514 		goto unlock_err;
7515 	}
7516 	len = min(len, em->len - (start - em->start));
7517 unlock:
7518 	bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7519 		inode->i_blkbits;
7520 	bh_result->b_size = len;
7521 	bh_result->b_bdev = em->bdev;
7522 	set_buffer_mapped(bh_result);
7523 	if (create) {
7524 		if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7525 			set_buffer_new(bh_result);
7526 
7527 		/*
7528 		 * Need to update the i_size under the extent lock so buffered
7529 		 * readers will get the updated i_size when we unlock.
7530 		 */
7531 		if (start + len > i_size_read(inode))
7532 			i_size_write(inode, start + len);
7533 
7534 		/*
7535 		 * If we have an outstanding_extents count still set then we're
7536 		 * within our reservation, otherwise we need to adjust our inode
7537 		 * counter appropriately.
7538 		 */
7539 		if (*outstanding_extents) {
7540 			(*outstanding_extents)--;
7541 		} else {
7542 			spin_lock(&BTRFS_I(inode)->lock);
7543 			BTRFS_I(inode)->outstanding_extents++;
7544 			spin_unlock(&BTRFS_I(inode)->lock);
7545 		}
7546 
7547 		current->journal_info = outstanding_extents;
7548 		btrfs_free_reserved_data_space(inode, len);
7549 		set_bit(BTRFS_INODE_DIO_READY, &BTRFS_I(inode)->runtime_flags);
7550 	}
7551 
7552 	/*
7553 	 * In the case of write we need to clear and unlock the entire range,
7554 	 * in the case of read we need to unlock only the end area that we
7555 	 * aren't using if there is any left over space.
7556 	 */
7557 	if (lockstart < lockend) {
7558 		clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
7559 				 lockend, unlock_bits, 1, 0,
7560 				 &cached_state, GFP_NOFS);
7561 	} else {
7562 		free_extent_state(cached_state);
7563 	}
7564 
7565 	free_extent_map(em);
7566 
7567 	return 0;
7568 
7569 unlock_err:
7570 	clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7571 			 unlock_bits, 1, 0, &cached_state, GFP_NOFS);
7572 	if (outstanding_extents)
7573 		current->journal_info = outstanding_extents;
7574 	return ret;
7575 }
7576 
7577 static inline int submit_dio_repair_bio(struct inode *inode, struct bio *bio,
7578 					int rw, int mirror_num)
7579 {
7580 	struct btrfs_root *root = BTRFS_I(inode)->root;
7581 	int ret;
7582 
7583 	BUG_ON(rw & REQ_WRITE);
7584 
7585 	bio_get(bio);
7586 
7587 	ret = btrfs_bio_wq_end_io(root->fs_info, bio,
7588 				  BTRFS_WQ_ENDIO_DIO_REPAIR);
7589 	if (ret)
7590 		goto err;
7591 
7592 	ret = btrfs_map_bio(root, rw, bio, mirror_num, 0);
7593 err:
7594 	bio_put(bio);
7595 	return ret;
7596 }
7597 
7598 static int btrfs_check_dio_repairable(struct inode *inode,
7599 				      struct bio *failed_bio,
7600 				      struct io_failure_record *failrec,
7601 				      int failed_mirror)
7602 {
7603 	int num_copies;
7604 
7605 	num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
7606 				      failrec->logical, failrec->len);
7607 	if (num_copies == 1) {
7608 		/*
7609 		 * we only have a single copy of the data, so don't bother with
7610 		 * all the retry and error correction code that follows. no
7611 		 * matter what the error is, it is very likely to persist.
7612 		 */
7613 		pr_debug("Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
7614 			 num_copies, failrec->this_mirror, failed_mirror);
7615 		return 0;
7616 	}
7617 
7618 	failrec->failed_mirror = failed_mirror;
7619 	failrec->this_mirror++;
7620 	if (failrec->this_mirror == failed_mirror)
7621 		failrec->this_mirror++;
7622 
7623 	if (failrec->this_mirror > num_copies) {
7624 		pr_debug("Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
7625 			 num_copies, failrec->this_mirror, failed_mirror);
7626 		return 0;
7627 	}
7628 
7629 	return 1;
7630 }
7631 
7632 static int dio_read_error(struct inode *inode, struct bio *failed_bio,
7633 			  struct page *page, u64 start, u64 end,
7634 			  int failed_mirror, bio_end_io_t *repair_endio,
7635 			  void *repair_arg)
7636 {
7637 	struct io_failure_record *failrec;
7638 	struct bio *bio;
7639 	int isector;
7640 	int read_mode;
7641 	int ret;
7642 
7643 	BUG_ON(failed_bio->bi_rw & REQ_WRITE);
7644 
7645 	ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
7646 	if (ret)
7647 		return ret;
7648 
7649 	ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
7650 					 failed_mirror);
7651 	if (!ret) {
7652 		free_io_failure(inode, failrec);
7653 		return -EIO;
7654 	}
7655 
7656 	if (failed_bio->bi_vcnt > 1)
7657 		read_mode = READ_SYNC | REQ_FAILFAST_DEV;
7658 	else
7659 		read_mode = READ_SYNC;
7660 
7661 	isector = start - btrfs_io_bio(failed_bio)->logical;
7662 	isector >>= inode->i_sb->s_blocksize_bits;
7663 	bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
7664 				      0, isector, repair_endio, repair_arg);
7665 	if (!bio) {
7666 		free_io_failure(inode, failrec);
7667 		return -EIO;
7668 	}
7669 
7670 	btrfs_debug(BTRFS_I(inode)->root->fs_info,
7671 		    "Repair DIO Read Error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d\n",
7672 		    read_mode, failrec->this_mirror, failrec->in_validation);
7673 
7674 	ret = submit_dio_repair_bio(inode, bio, read_mode,
7675 				    failrec->this_mirror);
7676 	if (ret) {
7677 		free_io_failure(inode, failrec);
7678 		bio_put(bio);
7679 	}
7680 
7681 	return ret;
7682 }
7683 
7684 struct btrfs_retry_complete {
7685 	struct completion done;
7686 	struct inode *inode;
7687 	u64 start;
7688 	int uptodate;
7689 };
7690 
7691 static void btrfs_retry_endio_nocsum(struct bio *bio, int err)
7692 {
7693 	struct btrfs_retry_complete *done = bio->bi_private;
7694 	struct bio_vec *bvec;
7695 	int i;
7696 
7697 	if (err)
7698 		goto end;
7699 
7700 	done->uptodate = 1;
7701 	bio_for_each_segment_all(bvec, bio, i)
7702 		clean_io_failure(done->inode, done->start, bvec->bv_page, 0);
7703 end:
7704 	complete(&done->done);
7705 	bio_put(bio);
7706 }
7707 
7708 static int __btrfs_correct_data_nocsum(struct inode *inode,
7709 				       struct btrfs_io_bio *io_bio)
7710 {
7711 	struct bio_vec *bvec;
7712 	struct btrfs_retry_complete done;
7713 	u64 start;
7714 	int i;
7715 	int ret;
7716 
7717 	start = io_bio->logical;
7718 	done.inode = inode;
7719 
7720 	bio_for_each_segment_all(bvec, &io_bio->bio, i) {
7721 try_again:
7722 		done.uptodate = 0;
7723 		done.start = start;
7724 		init_completion(&done.done);
7725 
7726 		ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page, start,
7727 				     start + bvec->bv_len - 1,
7728 				     io_bio->mirror_num,
7729 				     btrfs_retry_endio_nocsum, &done);
7730 		if (ret)
7731 			return ret;
7732 
7733 		wait_for_completion(&done.done);
7734 
7735 		if (!done.uptodate) {
7736 			/* We might have another mirror, so try again */
7737 			goto try_again;
7738 		}
7739 
7740 		start += bvec->bv_len;
7741 	}
7742 
7743 	return 0;
7744 }
7745 
7746 static void btrfs_retry_endio(struct bio *bio, int err)
7747 {
7748 	struct btrfs_retry_complete *done = bio->bi_private;
7749 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
7750 	struct bio_vec *bvec;
7751 	int uptodate;
7752 	int ret;
7753 	int i;
7754 
7755 	if (err)
7756 		goto end;
7757 
7758 	uptodate = 1;
7759 	bio_for_each_segment_all(bvec, bio, i) {
7760 		ret = __readpage_endio_check(done->inode, io_bio, i,
7761 					     bvec->bv_page, 0,
7762 					     done->start, bvec->bv_len);
7763 		if (!ret)
7764 			clean_io_failure(done->inode, done->start,
7765 					 bvec->bv_page, 0);
7766 		else
7767 			uptodate = 0;
7768 	}
7769 
7770 	done->uptodate = uptodate;
7771 end:
7772 	complete(&done->done);
7773 	bio_put(bio);
7774 }
7775 
7776 static int __btrfs_subio_endio_read(struct inode *inode,
7777 				    struct btrfs_io_bio *io_bio, int err)
7778 {
7779 	struct bio_vec *bvec;
7780 	struct btrfs_retry_complete done;
7781 	u64 start;
7782 	u64 offset = 0;
7783 	int i;
7784 	int ret;
7785 
7786 	err = 0;
7787 	start = io_bio->logical;
7788 	done.inode = inode;
7789 
7790 	bio_for_each_segment_all(bvec, &io_bio->bio, i) {
7791 		ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
7792 					     0, start, bvec->bv_len);
7793 		if (likely(!ret))
7794 			goto next;
7795 try_again:
7796 		done.uptodate = 0;
7797 		done.start = start;
7798 		init_completion(&done.done);
7799 
7800 		ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page, start,
7801 				     start + bvec->bv_len - 1,
7802 				     io_bio->mirror_num,
7803 				     btrfs_retry_endio, &done);
7804 		if (ret) {
7805 			err = ret;
7806 			goto next;
7807 		}
7808 
7809 		wait_for_completion(&done.done);
7810 
7811 		if (!done.uptodate) {
7812 			/* We might have another mirror, so try again */
7813 			goto try_again;
7814 		}
7815 next:
7816 		offset += bvec->bv_len;
7817 		start += bvec->bv_len;
7818 	}
7819 
7820 	return err;
7821 }
7822 
7823 static int btrfs_subio_endio_read(struct inode *inode,
7824 				  struct btrfs_io_bio *io_bio, int err)
7825 {
7826 	bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
7827 
7828 	if (skip_csum) {
7829 		if (unlikely(err))
7830 			return __btrfs_correct_data_nocsum(inode, io_bio);
7831 		else
7832 			return 0;
7833 	} else {
7834 		return __btrfs_subio_endio_read(inode, io_bio, err);
7835 	}
7836 }
7837 
7838 static void btrfs_endio_direct_read(struct bio *bio, int err)
7839 {
7840 	struct btrfs_dio_private *dip = bio->bi_private;
7841 	struct inode *inode = dip->inode;
7842 	struct bio *dio_bio;
7843 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
7844 
7845 	if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
7846 		err = btrfs_subio_endio_read(inode, io_bio, err);
7847 
7848 	unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
7849 		      dip->logical_offset + dip->bytes - 1);
7850 	dio_bio = dip->dio_bio;
7851 
7852 	kfree(dip);
7853 
7854 	/* If we had a csum failure make sure to clear the uptodate flag */
7855 	if (err)
7856 		clear_bit(BIO_UPTODATE, &dio_bio->bi_flags);
7857 	dio_end_io(dio_bio, err);
7858 
7859 	if (io_bio->end_io)
7860 		io_bio->end_io(io_bio, err);
7861 	bio_put(bio);
7862 }
7863 
7864 static void btrfs_endio_direct_write(struct bio *bio, int err)
7865 {
7866 	struct btrfs_dio_private *dip = bio->bi_private;
7867 	struct inode *inode = dip->inode;
7868 	struct btrfs_root *root = BTRFS_I(inode)->root;
7869 	struct btrfs_ordered_extent *ordered = NULL;
7870 	u64 ordered_offset = dip->logical_offset;
7871 	u64 ordered_bytes = dip->bytes;
7872 	struct bio *dio_bio;
7873 	int ret;
7874 
7875 again:
7876 	ret = btrfs_dec_test_first_ordered_pending(inode, &ordered,
7877 						   &ordered_offset,
7878 						   ordered_bytes, !err);
7879 	if (!ret)
7880 		goto out_test;
7881 
7882 	btrfs_init_work(&ordered->work, btrfs_endio_write_helper,
7883 			finish_ordered_fn, NULL, NULL);
7884 	btrfs_queue_work(root->fs_info->endio_write_workers,
7885 			 &ordered->work);
7886 out_test:
7887 	/*
7888 	 * our bio might span multiple ordered extents.  If we haven't
7889 	 * completed the accounting for the whole dio, go back and try again
7890 	 */
7891 	if (ordered_offset < dip->logical_offset + dip->bytes) {
7892 		ordered_bytes = dip->logical_offset + dip->bytes -
7893 			ordered_offset;
7894 		ordered = NULL;
7895 		goto again;
7896 	}
7897 	dio_bio = dip->dio_bio;
7898 
7899 	kfree(dip);
7900 
7901 	/* If we had an error make sure to clear the uptodate flag */
7902 	if (err)
7903 		clear_bit(BIO_UPTODATE, &dio_bio->bi_flags);
7904 	dio_end_io(dio_bio, err);
7905 	bio_put(bio);
7906 }
7907 
7908 static int __btrfs_submit_bio_start_direct_io(struct inode *inode, int rw,
7909 				    struct bio *bio, int mirror_num,
7910 				    unsigned long bio_flags, u64 offset)
7911 {
7912 	int ret;
7913 	struct btrfs_root *root = BTRFS_I(inode)->root;
7914 	ret = btrfs_csum_one_bio(root, inode, bio, offset, 1);
7915 	BUG_ON(ret); /* -ENOMEM */
7916 	return 0;
7917 }
7918 
7919 static void btrfs_end_dio_bio(struct bio *bio, int err)
7920 {
7921 	struct btrfs_dio_private *dip = bio->bi_private;
7922 
7923 	if (err)
7924 		btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
7925 			   "direct IO failed ino %llu rw %lu sector %#Lx len %u err no %d",
7926 			   btrfs_ino(dip->inode), bio->bi_rw,
7927 			   (unsigned long long)bio->bi_iter.bi_sector,
7928 			   bio->bi_iter.bi_size, err);
7929 
7930 	if (dip->subio_endio)
7931 		err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
7932 
7933 	if (err) {
7934 		dip->errors = 1;
7935 
7936 		/*
7937 		 * before atomic variable goto zero, we must make sure
7938 		 * dip->errors is perceived to be set.
7939 		 */
7940 		smp_mb__before_atomic();
7941 	}
7942 
7943 	/* if there are more bios still pending for this dio, just exit */
7944 	if (!atomic_dec_and_test(&dip->pending_bios))
7945 		goto out;
7946 
7947 	if (dip->errors) {
7948 		bio_io_error(dip->orig_bio);
7949 	} else {
7950 		set_bit(BIO_UPTODATE, &dip->dio_bio->bi_flags);
7951 		bio_endio(dip->orig_bio, 0);
7952 	}
7953 out:
7954 	bio_put(bio);
7955 }
7956 
7957 static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev,
7958 				       u64 first_sector, gfp_t gfp_flags)
7959 {
7960 	int nr_vecs = bio_get_nr_vecs(bdev);
7961 	return btrfs_bio_alloc(bdev, first_sector, nr_vecs, gfp_flags);
7962 }
7963 
7964 static inline int btrfs_lookup_and_bind_dio_csum(struct btrfs_root *root,
7965 						 struct inode *inode,
7966 						 struct btrfs_dio_private *dip,
7967 						 struct bio *bio,
7968 						 u64 file_offset)
7969 {
7970 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
7971 	struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
7972 	int ret;
7973 
7974 	/*
7975 	 * We load all the csum data we need when we submit
7976 	 * the first bio to reduce the csum tree search and
7977 	 * contention.
7978 	 */
7979 	if (dip->logical_offset == file_offset) {
7980 		ret = btrfs_lookup_bio_sums_dio(root, inode, dip->orig_bio,
7981 						file_offset);
7982 		if (ret)
7983 			return ret;
7984 	}
7985 
7986 	if (bio == dip->orig_bio)
7987 		return 0;
7988 
7989 	file_offset -= dip->logical_offset;
7990 	file_offset >>= inode->i_sb->s_blocksize_bits;
7991 	io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset);
7992 
7993 	return 0;
7994 }
7995 
7996 static inline int __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode,
7997 					 int rw, u64 file_offset, int skip_sum,
7998 					 int async_submit)
7999 {
8000 	struct btrfs_dio_private *dip = bio->bi_private;
8001 	int write = rw & REQ_WRITE;
8002 	struct btrfs_root *root = BTRFS_I(inode)->root;
8003 	int ret;
8004 
8005 	if (async_submit)
8006 		async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);
8007 
8008 	bio_get(bio);
8009 
8010 	if (!write) {
8011 		ret = btrfs_bio_wq_end_io(root->fs_info, bio,
8012 				BTRFS_WQ_ENDIO_DATA);
8013 		if (ret)
8014 			goto err;
8015 	}
8016 
8017 	if (skip_sum)
8018 		goto map;
8019 
8020 	if (write && async_submit) {
8021 		ret = btrfs_wq_submit_bio(root->fs_info,
8022 				   inode, rw, bio, 0, 0,
8023 				   file_offset,
8024 				   __btrfs_submit_bio_start_direct_io,
8025 				   __btrfs_submit_bio_done);
8026 		goto err;
8027 	} else if (write) {
8028 		/*
8029 		 * If we aren't doing async submit, calculate the csum of the
8030 		 * bio now.
8031 		 */
8032 		ret = btrfs_csum_one_bio(root, inode, bio, file_offset, 1);
8033 		if (ret)
8034 			goto err;
8035 	} else {
8036 		ret = btrfs_lookup_and_bind_dio_csum(root, inode, dip, bio,
8037 						     file_offset);
8038 		if (ret)
8039 			goto err;
8040 	}
8041 map:
8042 	ret = btrfs_map_bio(root, rw, bio, 0, async_submit);
8043 err:
8044 	bio_put(bio);
8045 	return ret;
8046 }
8047 
8048 static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
8049 				    int skip_sum)
8050 {
8051 	struct inode *inode = dip->inode;
8052 	struct btrfs_root *root = BTRFS_I(inode)->root;
8053 	struct bio *bio;
8054 	struct bio *orig_bio = dip->orig_bio;
8055 	struct bio_vec *bvec = orig_bio->bi_io_vec;
8056 	u64 start_sector = orig_bio->bi_iter.bi_sector;
8057 	u64 file_offset = dip->logical_offset;
8058 	u64 submit_len = 0;
8059 	u64 map_length;
8060 	int nr_pages = 0;
8061 	int ret;
8062 	int async_submit = 0;
8063 
8064 	map_length = orig_bio->bi_iter.bi_size;
8065 	ret = btrfs_map_block(root->fs_info, rw, start_sector << 9,
8066 			      &map_length, NULL, 0);
8067 	if (ret)
8068 		return -EIO;
8069 
8070 	if (map_length >= orig_bio->bi_iter.bi_size) {
8071 		bio = orig_bio;
8072 		dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
8073 		goto submit;
8074 	}
8075 
8076 	/* async crcs make it difficult to collect full stripe writes. */
8077 	if (btrfs_get_alloc_profile(root, 1) & BTRFS_BLOCK_GROUP_RAID56_MASK)
8078 		async_submit = 0;
8079 	else
8080 		async_submit = 1;
8081 
8082 	bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev, start_sector, GFP_NOFS);
8083 	if (!bio)
8084 		return -ENOMEM;
8085 
8086 	bio->bi_private = dip;
8087 	bio->bi_end_io = btrfs_end_dio_bio;
8088 	btrfs_io_bio(bio)->logical = file_offset;
8089 	atomic_inc(&dip->pending_bios);
8090 
8091 	while (bvec <= (orig_bio->bi_io_vec + orig_bio->bi_vcnt - 1)) {
8092 		if (map_length < submit_len + bvec->bv_len ||
8093 		    bio_add_page(bio, bvec->bv_page, bvec->bv_len,
8094 				 bvec->bv_offset) < bvec->bv_len) {
8095 			/*
8096 			 * inc the count before we submit the bio so
8097 			 * we know the end IO handler won't happen before
8098 			 * we inc the count. Otherwise, the dip might get freed
8099 			 * before we're done setting it up
8100 			 */
8101 			atomic_inc(&dip->pending_bios);
8102 			ret = __btrfs_submit_dio_bio(bio, inode, rw,
8103 						     file_offset, skip_sum,
8104 						     async_submit);
8105 			if (ret) {
8106 				bio_put(bio);
8107 				atomic_dec(&dip->pending_bios);
8108 				goto out_err;
8109 			}
8110 
8111 			start_sector += submit_len >> 9;
8112 			file_offset += submit_len;
8113 
8114 			submit_len = 0;
8115 			nr_pages = 0;
8116 
8117 			bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev,
8118 						  start_sector, GFP_NOFS);
8119 			if (!bio)
8120 				goto out_err;
8121 			bio->bi_private = dip;
8122 			bio->bi_end_io = btrfs_end_dio_bio;
8123 			btrfs_io_bio(bio)->logical = file_offset;
8124 
8125 			map_length = orig_bio->bi_iter.bi_size;
8126 			ret = btrfs_map_block(root->fs_info, rw,
8127 					      start_sector << 9,
8128 					      &map_length, NULL, 0);
8129 			if (ret) {
8130 				bio_put(bio);
8131 				goto out_err;
8132 			}
8133 		} else {
8134 			submit_len += bvec->bv_len;
8135 			nr_pages++;
8136 			bvec++;
8137 		}
8138 	}
8139 
8140 submit:
8141 	ret = __btrfs_submit_dio_bio(bio, inode, rw, file_offset, skip_sum,
8142 				     async_submit);
8143 	if (!ret)
8144 		return 0;
8145 
8146 	bio_put(bio);
8147 out_err:
8148 	dip->errors = 1;
8149 	/*
8150 	 * before atomic variable goto zero, we must
8151 	 * make sure dip->errors is perceived to be set.
8152 	 */
8153 	smp_mb__before_atomic();
8154 	if (atomic_dec_and_test(&dip->pending_bios))
8155 		bio_io_error(dip->orig_bio);
8156 
8157 	/* bio_end_io() will handle error, so we needn't return it */
8158 	return 0;
8159 }
8160 
8161 static void btrfs_submit_direct(int rw, struct bio *dio_bio,
8162 				struct inode *inode, loff_t file_offset)
8163 {
8164 	struct btrfs_dio_private *dip = NULL;
8165 	struct bio *io_bio = NULL;
8166 	struct btrfs_io_bio *btrfs_bio;
8167 	int skip_sum;
8168 	int write = rw & REQ_WRITE;
8169 	int ret = 0;
8170 
8171 	skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
8172 
8173 	io_bio = btrfs_bio_clone(dio_bio, GFP_NOFS);
8174 	if (!io_bio) {
8175 		ret = -ENOMEM;
8176 		goto free_ordered;
8177 	}
8178 
8179 	dip = kzalloc(sizeof(*dip), GFP_NOFS);
8180 	if (!dip) {
8181 		ret = -ENOMEM;
8182 		goto free_ordered;
8183 	}
8184 
8185 	dip->private = dio_bio->bi_private;
8186 	dip->inode = inode;
8187 	dip->logical_offset = file_offset;
8188 	dip->bytes = dio_bio->bi_iter.bi_size;
8189 	dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
8190 	io_bio->bi_private = dip;
8191 	dip->orig_bio = io_bio;
8192 	dip->dio_bio = dio_bio;
8193 	atomic_set(&dip->pending_bios, 0);
8194 	btrfs_bio = btrfs_io_bio(io_bio);
8195 	btrfs_bio->logical = file_offset;
8196 
8197 	if (write) {
8198 		io_bio->bi_end_io = btrfs_endio_direct_write;
8199 	} else {
8200 		io_bio->bi_end_io = btrfs_endio_direct_read;
8201 		dip->subio_endio = btrfs_subio_endio_read;
8202 	}
8203 
8204 	ret = btrfs_submit_direct_hook(rw, dip, skip_sum);
8205 	if (!ret)
8206 		return;
8207 
8208 	if (btrfs_bio->end_io)
8209 		btrfs_bio->end_io(btrfs_bio, ret);
8210 
8211 free_ordered:
8212 	/*
8213 	 * If we arrived here it means either we failed to submit the dip
8214 	 * or we either failed to clone the dio_bio or failed to allocate the
8215 	 * dip. If we cloned the dio_bio and allocated the dip, we can just
8216 	 * call bio_endio against our io_bio so that we get proper resource
8217 	 * cleanup if we fail to submit the dip, otherwise, we must do the
8218 	 * same as btrfs_endio_direct_[write|read] because we can't call these
8219 	 * callbacks - they require an allocated dip and a clone of dio_bio.
8220 	 */
8221 	if (io_bio && dip) {
8222 		bio_endio(io_bio, ret);
8223 		/*
8224 		 * The end io callbacks free our dip, do the final put on io_bio
8225 		 * and all the cleanup and final put for dio_bio (through
8226 		 * dio_end_io()).
8227 		 */
8228 		dip = NULL;
8229 		io_bio = NULL;
8230 	} else {
8231 		if (write) {
8232 			struct btrfs_ordered_extent *ordered;
8233 
8234 			ordered = btrfs_lookup_ordered_extent(inode,
8235 							      file_offset);
8236 			set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);
8237 			/*
8238 			 * Decrements our ref on the ordered extent and removes
8239 			 * the ordered extent from the inode's ordered tree,
8240 			 * doing all the proper resource cleanup such as for the
8241 			 * reserved space and waking up any waiters for this
8242 			 * ordered extent (through btrfs_remove_ordered_extent).
8243 			 */
8244 			btrfs_finish_ordered_io(ordered);
8245 		} else {
8246 			unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
8247 			      file_offset + dio_bio->bi_iter.bi_size - 1);
8248 		}
8249 		clear_bit(BIO_UPTODATE, &dio_bio->bi_flags);
8250 		/*
8251 		 * Releases and cleans up our dio_bio, no need to bio_put()
8252 		 * nor bio_endio()/bio_io_error() against dio_bio.
8253 		 */
8254 		dio_end_io(dio_bio, ret);
8255 	}
8256 	if (io_bio)
8257 		bio_put(io_bio);
8258 	kfree(dip);
8259 }
8260 
8261 static ssize_t check_direct_IO(struct btrfs_root *root, struct kiocb *iocb,
8262 			const struct iov_iter *iter, loff_t offset)
8263 {
8264 	int seg;
8265 	int i;
8266 	unsigned blocksize_mask = root->sectorsize - 1;
8267 	ssize_t retval = -EINVAL;
8268 
8269 	if (offset & blocksize_mask)
8270 		goto out;
8271 
8272 	if (iov_iter_alignment(iter) & blocksize_mask)
8273 		goto out;
8274 
8275 	/* If this is a write we don't need to check anymore */
8276 	if (iov_iter_rw(iter) == WRITE)
8277 		return 0;
8278 	/*
8279 	 * Check to make sure we don't have duplicate iov_base's in this
8280 	 * iovec, if so return EINVAL, otherwise we'll get csum errors
8281 	 * when reading back.
8282 	 */
8283 	for (seg = 0; seg < iter->nr_segs; seg++) {
8284 		for (i = seg + 1; i < iter->nr_segs; i++) {
8285 			if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
8286 				goto out;
8287 		}
8288 	}
8289 	retval = 0;
8290 out:
8291 	return retval;
8292 }
8293 
8294 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
8295 			       loff_t offset)
8296 {
8297 	struct file *file = iocb->ki_filp;
8298 	struct inode *inode = file->f_mapping->host;
8299 	u64 outstanding_extents = 0;
8300 	size_t count = 0;
8301 	int flags = 0;
8302 	bool wakeup = true;
8303 	bool relock = false;
8304 	ssize_t ret;
8305 
8306 	if (check_direct_IO(BTRFS_I(inode)->root, iocb, iter, offset))
8307 		return 0;
8308 
8309 	inode_dio_begin(inode);
8310 	smp_mb__after_atomic();
8311 
8312 	/*
8313 	 * The generic stuff only does filemap_write_and_wait_range, which
8314 	 * isn't enough if we've written compressed pages to this area, so
8315 	 * we need to flush the dirty pages again to make absolutely sure
8316 	 * that any outstanding dirty pages are on disk.
8317 	 */
8318 	count = iov_iter_count(iter);
8319 	if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8320 		     &BTRFS_I(inode)->runtime_flags))
8321 		filemap_fdatawrite_range(inode->i_mapping, offset,
8322 					 offset + count - 1);
8323 
8324 	if (iov_iter_rw(iter) == WRITE) {
8325 		/*
8326 		 * If the write DIO is beyond the EOF, we need update
8327 		 * the isize, but it is protected by i_mutex. So we can
8328 		 * not unlock the i_mutex at this case.
8329 		 */
8330 		if (offset + count <= inode->i_size) {
8331 			mutex_unlock(&inode->i_mutex);
8332 			relock = true;
8333 		}
8334 		ret = btrfs_delalloc_reserve_space(inode, count);
8335 		if (ret)
8336 			goto out;
8337 		outstanding_extents = div64_u64(count +
8338 						BTRFS_MAX_EXTENT_SIZE - 1,
8339 						BTRFS_MAX_EXTENT_SIZE);
8340 
8341 		/*
8342 		 * We need to know how many extents we reserved so that we can
8343 		 * do the accounting properly if we go over the number we
8344 		 * originally calculated.  Abuse current->journal_info for this.
8345 		 */
8346 		current->journal_info = &outstanding_extents;
8347 	} else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8348 				     &BTRFS_I(inode)->runtime_flags)) {
8349 		inode_dio_end(inode);
8350 		flags = DIO_LOCKING | DIO_SKIP_HOLES;
8351 		wakeup = false;
8352 	}
8353 
8354 	ret = __blockdev_direct_IO(iocb, inode,
8355 				   BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev,
8356 				   iter, offset, btrfs_get_blocks_direct, NULL,
8357 				   btrfs_submit_direct, flags);
8358 	if (iov_iter_rw(iter) == WRITE) {
8359 		current->journal_info = NULL;
8360 		if (ret < 0 && ret != -EIOCBQUEUED) {
8361 			/*
8362 			 * If the error comes from submitting stage,
8363 			 * btrfs_get_blocsk_direct() has free'd data space,
8364 			 * and metadata space will be handled by
8365 			 * finish_ordered_fn, don't do that again to make
8366 			 * sure bytes_may_use is correct.
8367 			 */
8368 			if (!test_and_clear_bit(BTRFS_INODE_DIO_READY,
8369 				     &BTRFS_I(inode)->runtime_flags))
8370 				btrfs_delalloc_release_space(inode, count);
8371 		} else if (ret >= 0 && (size_t)ret < count)
8372 			btrfs_delalloc_release_space(inode,
8373 						     count - (size_t)ret);
8374 	}
8375 out:
8376 	if (wakeup)
8377 		inode_dio_end(inode);
8378 	if (relock)
8379 		mutex_lock(&inode->i_mutex);
8380 
8381 	return ret;
8382 }
8383 
8384 #define BTRFS_FIEMAP_FLAGS	(FIEMAP_FLAG_SYNC)
8385 
8386 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
8387 		__u64 start, __u64 len)
8388 {
8389 	int	ret;
8390 
8391 	ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS);
8392 	if (ret)
8393 		return ret;
8394 
8395 	return extent_fiemap(inode, fieinfo, start, len, btrfs_get_extent_fiemap);
8396 }
8397 
8398 int btrfs_readpage(struct file *file, struct page *page)
8399 {
8400 	struct extent_io_tree *tree;
8401 	tree = &BTRFS_I(page->mapping->host)->io_tree;
8402 	return extent_read_full_page(tree, page, btrfs_get_extent, 0);
8403 }
8404 
8405 static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
8406 {
8407 	struct extent_io_tree *tree;
8408 
8409 
8410 	if (current->flags & PF_MEMALLOC) {
8411 		redirty_page_for_writepage(wbc, page);
8412 		unlock_page(page);
8413 		return 0;
8414 	}
8415 	tree = &BTRFS_I(page->mapping->host)->io_tree;
8416 	return extent_write_full_page(tree, page, btrfs_get_extent, wbc);
8417 }
8418 
8419 static int btrfs_writepages(struct address_space *mapping,
8420 			    struct writeback_control *wbc)
8421 {
8422 	struct extent_io_tree *tree;
8423 
8424 	tree = &BTRFS_I(mapping->host)->io_tree;
8425 	return extent_writepages(tree, mapping, btrfs_get_extent, wbc);
8426 }
8427 
8428 static int
8429 btrfs_readpages(struct file *file, struct address_space *mapping,
8430 		struct list_head *pages, unsigned nr_pages)
8431 {
8432 	struct extent_io_tree *tree;
8433 	tree = &BTRFS_I(mapping->host)->io_tree;
8434 	return extent_readpages(tree, mapping, pages, nr_pages,
8435 				btrfs_get_extent);
8436 }
8437 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8438 {
8439 	struct extent_io_tree *tree;
8440 	struct extent_map_tree *map;
8441 	int ret;
8442 
8443 	tree = &BTRFS_I(page->mapping->host)->io_tree;
8444 	map = &BTRFS_I(page->mapping->host)->extent_tree;
8445 	ret = try_release_extent_mapping(map, tree, page, gfp_flags);
8446 	if (ret == 1) {
8447 		ClearPagePrivate(page);
8448 		set_page_private(page, 0);
8449 		page_cache_release(page);
8450 	}
8451 	return ret;
8452 }
8453 
8454 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8455 {
8456 	if (PageWriteback(page) || PageDirty(page))
8457 		return 0;
8458 	return __btrfs_releasepage(page, gfp_flags & GFP_NOFS);
8459 }
8460 
8461 static void btrfs_invalidatepage(struct page *page, unsigned int offset,
8462 				 unsigned int length)
8463 {
8464 	struct inode *inode = page->mapping->host;
8465 	struct extent_io_tree *tree;
8466 	struct btrfs_ordered_extent *ordered;
8467 	struct extent_state *cached_state = NULL;
8468 	u64 page_start = page_offset(page);
8469 	u64 page_end = page_start + PAGE_CACHE_SIZE - 1;
8470 	int inode_evicting = inode->i_state & I_FREEING;
8471 
8472 	/*
8473 	 * we have the page locked, so new writeback can't start,
8474 	 * and the dirty bit won't be cleared while we are here.
8475 	 *
8476 	 * Wait for IO on this page so that we can safely clear
8477 	 * the PagePrivate2 bit and do ordered accounting
8478 	 */
8479 	wait_on_page_writeback(page);
8480 
8481 	tree = &BTRFS_I(inode)->io_tree;
8482 	if (offset) {
8483 		btrfs_releasepage(page, GFP_NOFS);
8484 		return;
8485 	}
8486 
8487 	if (!inode_evicting)
8488 		lock_extent_bits(tree, page_start, page_end, 0, &cached_state);
8489 	ordered = btrfs_lookup_ordered_extent(inode, page_start);
8490 	if (ordered) {
8491 		/*
8492 		 * IO on this page will never be started, so we need
8493 		 * to account for any ordered extents now
8494 		 */
8495 		if (!inode_evicting)
8496 			clear_extent_bit(tree, page_start, page_end,
8497 					 EXTENT_DIRTY | EXTENT_DELALLOC |
8498 					 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
8499 					 EXTENT_DEFRAG, 1, 0, &cached_state,
8500 					 GFP_NOFS);
8501 		/*
8502 		 * whoever cleared the private bit is responsible
8503 		 * for the finish_ordered_io
8504 		 */
8505 		if (TestClearPagePrivate2(page)) {
8506 			struct btrfs_ordered_inode_tree *tree;
8507 			u64 new_len;
8508 
8509 			tree = &BTRFS_I(inode)->ordered_tree;
8510 
8511 			spin_lock_irq(&tree->lock);
8512 			set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
8513 			new_len = page_start - ordered->file_offset;
8514 			if (new_len < ordered->truncated_len)
8515 				ordered->truncated_len = new_len;
8516 			spin_unlock_irq(&tree->lock);
8517 
8518 			if (btrfs_dec_test_ordered_pending(inode, &ordered,
8519 							   page_start,
8520 							   PAGE_CACHE_SIZE, 1))
8521 				btrfs_finish_ordered_io(ordered);
8522 		}
8523 		btrfs_put_ordered_extent(ordered);
8524 		if (!inode_evicting) {
8525 			cached_state = NULL;
8526 			lock_extent_bits(tree, page_start, page_end, 0,
8527 					 &cached_state);
8528 		}
8529 	}
8530 
8531 	if (!inode_evicting) {
8532 		clear_extent_bit(tree, page_start, page_end,
8533 				 EXTENT_LOCKED | EXTENT_DIRTY |
8534 				 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
8535 				 EXTENT_DEFRAG, 1, 1,
8536 				 &cached_state, GFP_NOFS);
8537 
8538 		__btrfs_releasepage(page, GFP_NOFS);
8539 	}
8540 
8541 	ClearPageChecked(page);
8542 	if (PagePrivate(page)) {
8543 		ClearPagePrivate(page);
8544 		set_page_private(page, 0);
8545 		page_cache_release(page);
8546 	}
8547 }
8548 
8549 /*
8550  * btrfs_page_mkwrite() is not allowed to change the file size as it gets
8551  * called from a page fault handler when a page is first dirtied. Hence we must
8552  * be careful to check for EOF conditions here. We set the page up correctly
8553  * for a written page which means we get ENOSPC checking when writing into
8554  * holes and correct delalloc and unwritten extent mapping on filesystems that
8555  * support these features.
8556  *
8557  * We are not allowed to take the i_mutex here so we have to play games to
8558  * protect against truncate races as the page could now be beyond EOF.  Because
8559  * vmtruncate() writes the inode size before removing pages, once we have the
8560  * page lock we can determine safely if the page is beyond EOF. If it is not
8561  * beyond EOF, then the page is guaranteed safe against truncation until we
8562  * unlock the page.
8563  */
8564 int btrfs_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
8565 {
8566 	struct page *page = vmf->page;
8567 	struct inode *inode = file_inode(vma->vm_file);
8568 	struct btrfs_root *root = BTRFS_I(inode)->root;
8569 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
8570 	struct btrfs_ordered_extent *ordered;
8571 	struct extent_state *cached_state = NULL;
8572 	char *kaddr;
8573 	unsigned long zero_start;
8574 	loff_t size;
8575 	int ret;
8576 	int reserved = 0;
8577 	u64 page_start;
8578 	u64 page_end;
8579 
8580 	sb_start_pagefault(inode->i_sb);
8581 	ret  = btrfs_delalloc_reserve_space(inode, PAGE_CACHE_SIZE);
8582 	if (!ret) {
8583 		ret = file_update_time(vma->vm_file);
8584 		reserved = 1;
8585 	}
8586 	if (ret) {
8587 		if (ret == -ENOMEM)
8588 			ret = VM_FAULT_OOM;
8589 		else /* -ENOSPC, -EIO, etc */
8590 			ret = VM_FAULT_SIGBUS;
8591 		if (reserved)
8592 			goto out;
8593 		goto out_noreserve;
8594 	}
8595 
8596 	ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
8597 again:
8598 	lock_page(page);
8599 	size = i_size_read(inode);
8600 	page_start = page_offset(page);
8601 	page_end = page_start + PAGE_CACHE_SIZE - 1;
8602 
8603 	if ((page->mapping != inode->i_mapping) ||
8604 	    (page_start >= size)) {
8605 		/* page got truncated out from underneath us */
8606 		goto out_unlock;
8607 	}
8608 	wait_on_page_writeback(page);
8609 
8610 	lock_extent_bits(io_tree, page_start, page_end, 0, &cached_state);
8611 	set_page_extent_mapped(page);
8612 
8613 	/*
8614 	 * we can't set the delalloc bits if there are pending ordered
8615 	 * extents.  Drop our locks and wait for them to finish
8616 	 */
8617 	ordered = btrfs_lookup_ordered_extent(inode, page_start);
8618 	if (ordered) {
8619 		unlock_extent_cached(io_tree, page_start, page_end,
8620 				     &cached_state, GFP_NOFS);
8621 		unlock_page(page);
8622 		btrfs_start_ordered_extent(inode, ordered, 1);
8623 		btrfs_put_ordered_extent(ordered);
8624 		goto again;
8625 	}
8626 
8627 	/*
8628 	 * XXX - page_mkwrite gets called every time the page is dirtied, even
8629 	 * if it was already dirty, so for space accounting reasons we need to
8630 	 * clear any delalloc bits for the range we are fixing to save.  There
8631 	 * is probably a better way to do this, but for now keep consistent with
8632 	 * prepare_pages in the normal write path.
8633 	 */
8634 	clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, page_end,
8635 			  EXTENT_DIRTY | EXTENT_DELALLOC |
8636 			  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
8637 			  0, 0, &cached_state, GFP_NOFS);
8638 
8639 	ret = btrfs_set_extent_delalloc(inode, page_start, page_end,
8640 					&cached_state);
8641 	if (ret) {
8642 		unlock_extent_cached(io_tree, page_start, page_end,
8643 				     &cached_state, GFP_NOFS);
8644 		ret = VM_FAULT_SIGBUS;
8645 		goto out_unlock;
8646 	}
8647 	ret = 0;
8648 
8649 	/* page is wholly or partially inside EOF */
8650 	if (page_start + PAGE_CACHE_SIZE > size)
8651 		zero_start = size & ~PAGE_CACHE_MASK;
8652 	else
8653 		zero_start = PAGE_CACHE_SIZE;
8654 
8655 	if (zero_start != PAGE_CACHE_SIZE) {
8656 		kaddr = kmap(page);
8657 		memset(kaddr + zero_start, 0, PAGE_CACHE_SIZE - zero_start);
8658 		flush_dcache_page(page);
8659 		kunmap(page);
8660 	}
8661 	ClearPageChecked(page);
8662 	set_page_dirty(page);
8663 	SetPageUptodate(page);
8664 
8665 	BTRFS_I(inode)->last_trans = root->fs_info->generation;
8666 	BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid;
8667 	BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit;
8668 
8669 	unlock_extent_cached(io_tree, page_start, page_end, &cached_state, GFP_NOFS);
8670 
8671 out_unlock:
8672 	if (!ret) {
8673 		sb_end_pagefault(inode->i_sb);
8674 		return VM_FAULT_LOCKED;
8675 	}
8676 	unlock_page(page);
8677 out:
8678 	btrfs_delalloc_release_space(inode, PAGE_CACHE_SIZE);
8679 out_noreserve:
8680 	sb_end_pagefault(inode->i_sb);
8681 	return ret;
8682 }
8683 
8684 static int btrfs_truncate(struct inode *inode)
8685 {
8686 	struct btrfs_root *root = BTRFS_I(inode)->root;
8687 	struct btrfs_block_rsv *rsv;
8688 	int ret = 0;
8689 	int err = 0;
8690 	struct btrfs_trans_handle *trans;
8691 	u64 mask = root->sectorsize - 1;
8692 	u64 min_size = btrfs_calc_trunc_metadata_size(root, 1);
8693 
8694 	ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
8695 				       (u64)-1);
8696 	if (ret)
8697 		return ret;
8698 
8699 	/*
8700 	 * Yes ladies and gentelment, this is indeed ugly.  The fact is we have
8701 	 * 3 things going on here
8702 	 *
8703 	 * 1) We need to reserve space for our orphan item and the space to
8704 	 * delete our orphan item.  Lord knows we don't want to have a dangling
8705 	 * orphan item because we didn't reserve space to remove it.
8706 	 *
8707 	 * 2) We need to reserve space to update our inode.
8708 	 *
8709 	 * 3) We need to have something to cache all the space that is going to
8710 	 * be free'd up by the truncate operation, but also have some slack
8711 	 * space reserved in case it uses space during the truncate (thank you
8712 	 * very much snapshotting).
8713 	 *
8714 	 * And we need these to all be seperate.  The fact is we can use alot of
8715 	 * space doing the truncate, and we have no earthly idea how much space
8716 	 * we will use, so we need the truncate reservation to be seperate so it
8717 	 * doesn't end up using space reserved for updating the inode or
8718 	 * removing the orphan item.  We also need to be able to stop the
8719 	 * transaction and start a new one, which means we need to be able to
8720 	 * update the inode several times, and we have no idea of knowing how
8721 	 * many times that will be, so we can't just reserve 1 item for the
8722 	 * entirety of the opration, so that has to be done seperately as well.
8723 	 * Then there is the orphan item, which does indeed need to be held on
8724 	 * to for the whole operation, and we need nobody to touch this reserved
8725 	 * space except the orphan code.
8726 	 *
8727 	 * So that leaves us with
8728 	 *
8729 	 * 1) root->orphan_block_rsv - for the orphan deletion.
8730 	 * 2) rsv - for the truncate reservation, which we will steal from the
8731 	 * transaction reservation.
8732 	 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for
8733 	 * updating the inode.
8734 	 */
8735 	rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP);
8736 	if (!rsv)
8737 		return -ENOMEM;
8738 	rsv->size = min_size;
8739 	rsv->failfast = 1;
8740 
8741 	/*
8742 	 * 1 for the truncate slack space
8743 	 * 1 for updating the inode.
8744 	 */
8745 	trans = btrfs_start_transaction(root, 2);
8746 	if (IS_ERR(trans)) {
8747 		err = PTR_ERR(trans);
8748 		goto out;
8749 	}
8750 
8751 	/* Migrate the slack space for the truncate to our reserve */
8752 	ret = btrfs_block_rsv_migrate(&root->fs_info->trans_block_rsv, rsv,
8753 				      min_size);
8754 	BUG_ON(ret);
8755 
8756 	/*
8757 	 * So if we truncate and then write and fsync we normally would just
8758 	 * write the extents that changed, which is a problem if we need to
8759 	 * first truncate that entire inode.  So set this flag so we write out
8760 	 * all of the extents in the inode to the sync log so we're completely
8761 	 * safe.
8762 	 */
8763 	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
8764 	trans->block_rsv = rsv;
8765 
8766 	while (1) {
8767 		ret = btrfs_truncate_inode_items(trans, root, inode,
8768 						 inode->i_size,
8769 						 BTRFS_EXTENT_DATA_KEY);
8770 		if (ret != -ENOSPC && ret != -EAGAIN) {
8771 			err = ret;
8772 			break;
8773 		}
8774 
8775 		trans->block_rsv = &root->fs_info->trans_block_rsv;
8776 		ret = btrfs_update_inode(trans, root, inode);
8777 		if (ret) {
8778 			err = ret;
8779 			break;
8780 		}
8781 
8782 		btrfs_end_transaction(trans, root);
8783 		btrfs_btree_balance_dirty(root);
8784 
8785 		trans = btrfs_start_transaction(root, 2);
8786 		if (IS_ERR(trans)) {
8787 			ret = err = PTR_ERR(trans);
8788 			trans = NULL;
8789 			break;
8790 		}
8791 
8792 		ret = btrfs_block_rsv_migrate(&root->fs_info->trans_block_rsv,
8793 					      rsv, min_size);
8794 		BUG_ON(ret);	/* shouldn't happen */
8795 		trans->block_rsv = rsv;
8796 	}
8797 
8798 	if (ret == 0 && inode->i_nlink > 0) {
8799 		trans->block_rsv = root->orphan_block_rsv;
8800 		ret = btrfs_orphan_del(trans, inode);
8801 		if (ret)
8802 			err = ret;
8803 	}
8804 
8805 	if (trans) {
8806 		trans->block_rsv = &root->fs_info->trans_block_rsv;
8807 		ret = btrfs_update_inode(trans, root, inode);
8808 		if (ret && !err)
8809 			err = ret;
8810 
8811 		ret = btrfs_end_transaction(trans, root);
8812 		btrfs_btree_balance_dirty(root);
8813 	}
8814 
8815 out:
8816 	btrfs_free_block_rsv(root, rsv);
8817 
8818 	if (ret && !err)
8819 		err = ret;
8820 
8821 	return err;
8822 }
8823 
8824 /*
8825  * create a new subvolume directory/inode (helper for the ioctl).
8826  */
8827 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
8828 			     struct btrfs_root *new_root,
8829 			     struct btrfs_root *parent_root,
8830 			     u64 new_dirid)
8831 {
8832 	struct inode *inode;
8833 	int err;
8834 	u64 index = 0;
8835 
8836 	inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
8837 				new_dirid, new_dirid,
8838 				S_IFDIR | (~current_umask() & S_IRWXUGO),
8839 				&index);
8840 	if (IS_ERR(inode))
8841 		return PTR_ERR(inode);
8842 	inode->i_op = &btrfs_dir_inode_operations;
8843 	inode->i_fop = &btrfs_dir_file_operations;
8844 
8845 	set_nlink(inode, 1);
8846 	btrfs_i_size_write(inode, 0);
8847 	unlock_new_inode(inode);
8848 
8849 	err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
8850 	if (err)
8851 		btrfs_err(new_root->fs_info,
8852 			  "error inheriting subvolume %llu properties: %d",
8853 			  new_root->root_key.objectid, err);
8854 
8855 	err = btrfs_update_inode(trans, new_root, inode);
8856 
8857 	iput(inode);
8858 	return err;
8859 }
8860 
8861 struct inode *btrfs_alloc_inode(struct super_block *sb)
8862 {
8863 	struct btrfs_inode *ei;
8864 	struct inode *inode;
8865 
8866 	ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_NOFS);
8867 	if (!ei)
8868 		return NULL;
8869 
8870 	ei->root = NULL;
8871 	ei->generation = 0;
8872 	ei->last_trans = 0;
8873 	ei->last_sub_trans = 0;
8874 	ei->logged_trans = 0;
8875 	ei->delalloc_bytes = 0;
8876 	ei->defrag_bytes = 0;
8877 	ei->disk_i_size = 0;
8878 	ei->flags = 0;
8879 	ei->csum_bytes = 0;
8880 	ei->index_cnt = (u64)-1;
8881 	ei->dir_index = 0;
8882 	ei->last_unlink_trans = 0;
8883 	ei->last_log_commit = 0;
8884 
8885 	spin_lock_init(&ei->lock);
8886 	ei->outstanding_extents = 0;
8887 	ei->reserved_extents = 0;
8888 
8889 	ei->runtime_flags = 0;
8890 	ei->force_compress = BTRFS_COMPRESS_NONE;
8891 
8892 	ei->delayed_node = NULL;
8893 
8894 	ei->i_otime.tv_sec = 0;
8895 	ei->i_otime.tv_nsec = 0;
8896 
8897 	inode = &ei->vfs_inode;
8898 	extent_map_tree_init(&ei->extent_tree);
8899 	extent_io_tree_init(&ei->io_tree, &inode->i_data);
8900 	extent_io_tree_init(&ei->io_failure_tree, &inode->i_data);
8901 	ei->io_tree.track_uptodate = 1;
8902 	ei->io_failure_tree.track_uptodate = 1;
8903 	atomic_set(&ei->sync_writers, 0);
8904 	mutex_init(&ei->log_mutex);
8905 	mutex_init(&ei->delalloc_mutex);
8906 	btrfs_ordered_inode_tree_init(&ei->ordered_tree);
8907 	INIT_LIST_HEAD(&ei->delalloc_inodes);
8908 	RB_CLEAR_NODE(&ei->rb_node);
8909 
8910 	return inode;
8911 }
8912 
8913 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
8914 void btrfs_test_destroy_inode(struct inode *inode)
8915 {
8916 	btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
8917 	kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
8918 }
8919 #endif
8920 
8921 static void btrfs_i_callback(struct rcu_head *head)
8922 {
8923 	struct inode *inode = container_of(head, struct inode, i_rcu);
8924 	kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
8925 }
8926 
8927 void btrfs_destroy_inode(struct inode *inode)
8928 {
8929 	struct btrfs_ordered_extent *ordered;
8930 	struct btrfs_root *root = BTRFS_I(inode)->root;
8931 
8932 	WARN_ON(!hlist_empty(&inode->i_dentry));
8933 	WARN_ON(inode->i_data.nrpages);
8934 	WARN_ON(BTRFS_I(inode)->outstanding_extents);
8935 	WARN_ON(BTRFS_I(inode)->reserved_extents);
8936 	WARN_ON(BTRFS_I(inode)->delalloc_bytes);
8937 	WARN_ON(BTRFS_I(inode)->csum_bytes);
8938 	WARN_ON(BTRFS_I(inode)->defrag_bytes);
8939 
8940 	/*
8941 	 * This can happen where we create an inode, but somebody else also
8942 	 * created the same inode and we need to destroy the one we already
8943 	 * created.
8944 	 */
8945 	if (!root)
8946 		goto free;
8947 
8948 	if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
8949 		     &BTRFS_I(inode)->runtime_flags)) {
8950 		btrfs_info(root->fs_info, "inode %llu still on the orphan list",
8951 			btrfs_ino(inode));
8952 		atomic_dec(&root->orphan_inodes);
8953 	}
8954 
8955 	while (1) {
8956 		ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
8957 		if (!ordered)
8958 			break;
8959 		else {
8960 			btrfs_err(root->fs_info, "found ordered extent %llu %llu on inode cleanup",
8961 				ordered->file_offset, ordered->len);
8962 			btrfs_remove_ordered_extent(inode, ordered);
8963 			btrfs_put_ordered_extent(ordered);
8964 			btrfs_put_ordered_extent(ordered);
8965 		}
8966 	}
8967 	inode_tree_del(inode);
8968 	btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
8969 free:
8970 	call_rcu(&inode->i_rcu, btrfs_i_callback);
8971 }
8972 
8973 int btrfs_drop_inode(struct inode *inode)
8974 {
8975 	struct btrfs_root *root = BTRFS_I(inode)->root;
8976 
8977 	if (root == NULL)
8978 		return 1;
8979 
8980 	/* the snap/subvol tree is on deleting */
8981 	if (btrfs_root_refs(&root->root_item) == 0)
8982 		return 1;
8983 	else
8984 		return generic_drop_inode(inode);
8985 }
8986 
8987 static void init_once(void *foo)
8988 {
8989 	struct btrfs_inode *ei = (struct btrfs_inode *) foo;
8990 
8991 	inode_init_once(&ei->vfs_inode);
8992 }
8993 
8994 void btrfs_destroy_cachep(void)
8995 {
8996 	/*
8997 	 * Make sure all delayed rcu free inodes are flushed before we
8998 	 * destroy cache.
8999 	 */
9000 	rcu_barrier();
9001 	if (btrfs_inode_cachep)
9002 		kmem_cache_destroy(btrfs_inode_cachep);
9003 	if (btrfs_trans_handle_cachep)
9004 		kmem_cache_destroy(btrfs_trans_handle_cachep);
9005 	if (btrfs_transaction_cachep)
9006 		kmem_cache_destroy(btrfs_transaction_cachep);
9007 	if (btrfs_path_cachep)
9008 		kmem_cache_destroy(btrfs_path_cachep);
9009 	if (btrfs_free_space_cachep)
9010 		kmem_cache_destroy(btrfs_free_space_cachep);
9011 	if (btrfs_delalloc_work_cachep)
9012 		kmem_cache_destroy(btrfs_delalloc_work_cachep);
9013 }
9014 
9015 int btrfs_init_cachep(void)
9016 {
9017 	btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
9018 			sizeof(struct btrfs_inode), 0,
9019 			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, init_once);
9020 	if (!btrfs_inode_cachep)
9021 		goto fail;
9022 
9023 	btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
9024 			sizeof(struct btrfs_trans_handle), 0,
9025 			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
9026 	if (!btrfs_trans_handle_cachep)
9027 		goto fail;
9028 
9029 	btrfs_transaction_cachep = kmem_cache_create("btrfs_transaction",
9030 			sizeof(struct btrfs_transaction), 0,
9031 			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
9032 	if (!btrfs_transaction_cachep)
9033 		goto fail;
9034 
9035 	btrfs_path_cachep = kmem_cache_create("btrfs_path",
9036 			sizeof(struct btrfs_path), 0,
9037 			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
9038 	if (!btrfs_path_cachep)
9039 		goto fail;
9040 
9041 	btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
9042 			sizeof(struct btrfs_free_space), 0,
9043 			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
9044 	if (!btrfs_free_space_cachep)
9045 		goto fail;
9046 
9047 	btrfs_delalloc_work_cachep = kmem_cache_create("btrfs_delalloc_work",
9048 			sizeof(struct btrfs_delalloc_work), 0,
9049 			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
9050 			NULL);
9051 	if (!btrfs_delalloc_work_cachep)
9052 		goto fail;
9053 
9054 	return 0;
9055 fail:
9056 	btrfs_destroy_cachep();
9057 	return -ENOMEM;
9058 }
9059 
9060 static int btrfs_getattr(struct vfsmount *mnt,
9061 			 struct dentry *dentry, struct kstat *stat)
9062 {
9063 	u64 delalloc_bytes;
9064 	struct inode *inode = d_inode(dentry);
9065 	u32 blocksize = inode->i_sb->s_blocksize;
9066 
9067 	generic_fillattr(inode, stat);
9068 	stat->dev = BTRFS_I(inode)->root->anon_dev;
9069 	stat->blksize = PAGE_CACHE_SIZE;
9070 
9071 	spin_lock(&BTRFS_I(inode)->lock);
9072 	delalloc_bytes = BTRFS_I(inode)->delalloc_bytes;
9073 	spin_unlock(&BTRFS_I(inode)->lock);
9074 	stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
9075 			ALIGN(delalloc_bytes, blocksize)) >> 9;
9076 	return 0;
9077 }
9078 
9079 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
9080 			   struct inode *new_dir, struct dentry *new_dentry)
9081 {
9082 	struct btrfs_trans_handle *trans;
9083 	struct btrfs_root *root = BTRFS_I(old_dir)->root;
9084 	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9085 	struct inode *new_inode = d_inode(new_dentry);
9086 	struct inode *old_inode = d_inode(old_dentry);
9087 	struct timespec ctime = CURRENT_TIME;
9088 	u64 index = 0;
9089 	u64 root_objectid;
9090 	int ret;
9091 	u64 old_ino = btrfs_ino(old_inode);
9092 
9093 	if (btrfs_ino(new_dir) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
9094 		return -EPERM;
9095 
9096 	/* we only allow rename subvolume link between subvolumes */
9097 	if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9098 		return -EXDEV;
9099 
9100 	if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
9101 	    (new_inode && btrfs_ino(new_inode) == BTRFS_FIRST_FREE_OBJECTID))
9102 		return -ENOTEMPTY;
9103 
9104 	if (S_ISDIR(old_inode->i_mode) && new_inode &&
9105 	    new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
9106 		return -ENOTEMPTY;
9107 
9108 
9109 	/* check for collisions, even if the  name isn't there */
9110 	ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
9111 			     new_dentry->d_name.name,
9112 			     new_dentry->d_name.len);
9113 
9114 	if (ret) {
9115 		if (ret == -EEXIST) {
9116 			/* we shouldn't get
9117 			 * eexist without a new_inode */
9118 			if (WARN_ON(!new_inode)) {
9119 				return ret;
9120 			}
9121 		} else {
9122 			/* maybe -EOVERFLOW */
9123 			return ret;
9124 		}
9125 	}
9126 	ret = 0;
9127 
9128 	/*
9129 	 * we're using rename to replace one file with another.  Start IO on it
9130 	 * now so  we don't add too much work to the end of the transaction
9131 	 */
9132 	if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
9133 		filemap_flush(old_inode->i_mapping);
9134 
9135 	/* close the racy window with snapshot create/destroy ioctl */
9136 	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9137 		down_read(&root->fs_info->subvol_sem);
9138 	/*
9139 	 * We want to reserve the absolute worst case amount of items.  So if
9140 	 * both inodes are subvols and we need to unlink them then that would
9141 	 * require 4 item modifications, but if they are both normal inodes it
9142 	 * would require 5 item modifications, so we'll assume their normal
9143 	 * inodes.  So 5 * 2 is 10, plus 1 for the new link, so 11 total items
9144 	 * should cover the worst case number of items we'll modify.
9145 	 */
9146 	trans = btrfs_start_transaction(root, 11);
9147 	if (IS_ERR(trans)) {
9148                 ret = PTR_ERR(trans);
9149                 goto out_notrans;
9150         }
9151 
9152 	if (dest != root)
9153 		btrfs_record_root_in_trans(trans, dest);
9154 
9155 	ret = btrfs_set_inode_index(new_dir, &index);
9156 	if (ret)
9157 		goto out_fail;
9158 
9159 	BTRFS_I(old_inode)->dir_index = 0ULL;
9160 	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9161 		/* force full log commit if subvolume involved. */
9162 		btrfs_set_log_full_commit(root->fs_info, trans);
9163 	} else {
9164 		ret = btrfs_insert_inode_ref(trans, dest,
9165 					     new_dentry->d_name.name,
9166 					     new_dentry->d_name.len,
9167 					     old_ino,
9168 					     btrfs_ino(new_dir), index);
9169 		if (ret)
9170 			goto out_fail;
9171 		/*
9172 		 * this is an ugly little race, but the rename is required
9173 		 * to make sure that if we crash, the inode is either at the
9174 		 * old name or the new one.  pinning the log transaction lets
9175 		 * us make sure we don't allow a log commit to come in after
9176 		 * we unlink the name but before we add the new name back in.
9177 		 */
9178 		btrfs_pin_log_trans(root);
9179 	}
9180 
9181 	inode_inc_iversion(old_dir);
9182 	inode_inc_iversion(new_dir);
9183 	inode_inc_iversion(old_inode);
9184 	old_dir->i_ctime = old_dir->i_mtime = ctime;
9185 	new_dir->i_ctime = new_dir->i_mtime = ctime;
9186 	old_inode->i_ctime = ctime;
9187 
9188 	if (old_dentry->d_parent != new_dentry->d_parent)
9189 		btrfs_record_unlink_dir(trans, old_dir, old_inode, 1);
9190 
9191 	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9192 		root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
9193 		ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid,
9194 					old_dentry->d_name.name,
9195 					old_dentry->d_name.len);
9196 	} else {
9197 		ret = __btrfs_unlink_inode(trans, root, old_dir,
9198 					d_inode(old_dentry),
9199 					old_dentry->d_name.name,
9200 					old_dentry->d_name.len);
9201 		if (!ret)
9202 			ret = btrfs_update_inode(trans, root, old_inode);
9203 	}
9204 	if (ret) {
9205 		btrfs_abort_transaction(trans, root, ret);
9206 		goto out_fail;
9207 	}
9208 
9209 	if (new_inode) {
9210 		inode_inc_iversion(new_inode);
9211 		new_inode->i_ctime = CURRENT_TIME;
9212 		if (unlikely(btrfs_ino(new_inode) ==
9213 			     BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
9214 			root_objectid = BTRFS_I(new_inode)->location.objectid;
9215 			ret = btrfs_unlink_subvol(trans, dest, new_dir,
9216 						root_objectid,
9217 						new_dentry->d_name.name,
9218 						new_dentry->d_name.len);
9219 			BUG_ON(new_inode->i_nlink == 0);
9220 		} else {
9221 			ret = btrfs_unlink_inode(trans, dest, new_dir,
9222 						 d_inode(new_dentry),
9223 						 new_dentry->d_name.name,
9224 						 new_dentry->d_name.len);
9225 		}
9226 		if (!ret && new_inode->i_nlink == 0)
9227 			ret = btrfs_orphan_add(trans, d_inode(new_dentry));
9228 		if (ret) {
9229 			btrfs_abort_transaction(trans, root, ret);
9230 			goto out_fail;
9231 		}
9232 	}
9233 
9234 	ret = btrfs_add_link(trans, new_dir, old_inode,
9235 			     new_dentry->d_name.name,
9236 			     new_dentry->d_name.len, 0, index);
9237 	if (ret) {
9238 		btrfs_abort_transaction(trans, root, ret);
9239 		goto out_fail;
9240 	}
9241 
9242 	if (old_inode->i_nlink == 1)
9243 		BTRFS_I(old_inode)->dir_index = index;
9244 
9245 	if (old_ino != BTRFS_FIRST_FREE_OBJECTID) {
9246 		struct dentry *parent = new_dentry->d_parent;
9247 		btrfs_log_new_name(trans, old_inode, old_dir, parent);
9248 		btrfs_end_log_trans(root);
9249 	}
9250 out_fail:
9251 	btrfs_end_transaction(trans, root);
9252 out_notrans:
9253 	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9254 		up_read(&root->fs_info->subvol_sem);
9255 
9256 	return ret;
9257 }
9258 
9259 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
9260 			 struct inode *new_dir, struct dentry *new_dentry,
9261 			 unsigned int flags)
9262 {
9263 	if (flags & ~RENAME_NOREPLACE)
9264 		return -EINVAL;
9265 
9266 	return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry);
9267 }
9268 
9269 static void btrfs_run_delalloc_work(struct btrfs_work *work)
9270 {
9271 	struct btrfs_delalloc_work *delalloc_work;
9272 	struct inode *inode;
9273 
9274 	delalloc_work = container_of(work, struct btrfs_delalloc_work,
9275 				     work);
9276 	inode = delalloc_work->inode;
9277 	if (delalloc_work->wait) {
9278 		btrfs_wait_ordered_range(inode, 0, (u64)-1);
9279 	} else {
9280 		filemap_flush(inode->i_mapping);
9281 		if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
9282 			     &BTRFS_I(inode)->runtime_flags))
9283 			filemap_flush(inode->i_mapping);
9284 	}
9285 
9286 	if (delalloc_work->delay_iput)
9287 		btrfs_add_delayed_iput(inode);
9288 	else
9289 		iput(inode);
9290 	complete(&delalloc_work->completion);
9291 }
9292 
9293 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode,
9294 						    int wait, int delay_iput)
9295 {
9296 	struct btrfs_delalloc_work *work;
9297 
9298 	work = kmem_cache_zalloc(btrfs_delalloc_work_cachep, GFP_NOFS);
9299 	if (!work)
9300 		return NULL;
9301 
9302 	init_completion(&work->completion);
9303 	INIT_LIST_HEAD(&work->list);
9304 	work->inode = inode;
9305 	work->wait = wait;
9306 	work->delay_iput = delay_iput;
9307 	WARN_ON_ONCE(!inode);
9308 	btrfs_init_work(&work->work, btrfs_flush_delalloc_helper,
9309 			btrfs_run_delalloc_work, NULL, NULL);
9310 
9311 	return work;
9312 }
9313 
9314 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work)
9315 {
9316 	wait_for_completion(&work->completion);
9317 	kmem_cache_free(btrfs_delalloc_work_cachep, work);
9318 }
9319 
9320 /*
9321  * some fairly slow code that needs optimization. This walks the list
9322  * of all the inodes with pending delalloc and forces them to disk.
9323  */
9324 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput,
9325 				   int nr)
9326 {
9327 	struct btrfs_inode *binode;
9328 	struct inode *inode;
9329 	struct btrfs_delalloc_work *work, *next;
9330 	struct list_head works;
9331 	struct list_head splice;
9332 	int ret = 0;
9333 
9334 	INIT_LIST_HEAD(&works);
9335 	INIT_LIST_HEAD(&splice);
9336 
9337 	mutex_lock(&root->delalloc_mutex);
9338 	spin_lock(&root->delalloc_lock);
9339 	list_splice_init(&root->delalloc_inodes, &splice);
9340 	while (!list_empty(&splice)) {
9341 		binode = list_entry(splice.next, struct btrfs_inode,
9342 				    delalloc_inodes);
9343 
9344 		list_move_tail(&binode->delalloc_inodes,
9345 			       &root->delalloc_inodes);
9346 		inode = igrab(&binode->vfs_inode);
9347 		if (!inode) {
9348 			cond_resched_lock(&root->delalloc_lock);
9349 			continue;
9350 		}
9351 		spin_unlock(&root->delalloc_lock);
9352 
9353 		work = btrfs_alloc_delalloc_work(inode, 0, delay_iput);
9354 		if (!work) {
9355 			if (delay_iput)
9356 				btrfs_add_delayed_iput(inode);
9357 			else
9358 				iput(inode);
9359 			ret = -ENOMEM;
9360 			goto out;
9361 		}
9362 		list_add_tail(&work->list, &works);
9363 		btrfs_queue_work(root->fs_info->flush_workers,
9364 				 &work->work);
9365 		ret++;
9366 		if (nr != -1 && ret >= nr)
9367 			goto out;
9368 		cond_resched();
9369 		spin_lock(&root->delalloc_lock);
9370 	}
9371 	spin_unlock(&root->delalloc_lock);
9372 
9373 out:
9374 	list_for_each_entry_safe(work, next, &works, list) {
9375 		list_del_init(&work->list);
9376 		btrfs_wait_and_free_delalloc_work(work);
9377 	}
9378 
9379 	if (!list_empty_careful(&splice)) {
9380 		spin_lock(&root->delalloc_lock);
9381 		list_splice_tail(&splice, &root->delalloc_inodes);
9382 		spin_unlock(&root->delalloc_lock);
9383 	}
9384 	mutex_unlock(&root->delalloc_mutex);
9385 	return ret;
9386 }
9387 
9388 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput)
9389 {
9390 	int ret;
9391 
9392 	if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
9393 		return -EROFS;
9394 
9395 	ret = __start_delalloc_inodes(root, delay_iput, -1);
9396 	if (ret > 0)
9397 		ret = 0;
9398 	/*
9399 	 * the filemap_flush will queue IO into the worker threads, but
9400 	 * we have to make sure the IO is actually started and that
9401 	 * ordered extents get created before we return
9402 	 */
9403 	atomic_inc(&root->fs_info->async_submit_draining);
9404 	while (atomic_read(&root->fs_info->nr_async_submits) ||
9405 	      atomic_read(&root->fs_info->async_delalloc_pages)) {
9406 		wait_event(root->fs_info->async_submit_wait,
9407 		   (atomic_read(&root->fs_info->nr_async_submits) == 0 &&
9408 		    atomic_read(&root->fs_info->async_delalloc_pages) == 0));
9409 	}
9410 	atomic_dec(&root->fs_info->async_submit_draining);
9411 	return ret;
9412 }
9413 
9414 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput,
9415 			       int nr)
9416 {
9417 	struct btrfs_root *root;
9418 	struct list_head splice;
9419 	int ret;
9420 
9421 	if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
9422 		return -EROFS;
9423 
9424 	INIT_LIST_HEAD(&splice);
9425 
9426 	mutex_lock(&fs_info->delalloc_root_mutex);
9427 	spin_lock(&fs_info->delalloc_root_lock);
9428 	list_splice_init(&fs_info->delalloc_roots, &splice);
9429 	while (!list_empty(&splice) && nr) {
9430 		root = list_first_entry(&splice, struct btrfs_root,
9431 					delalloc_root);
9432 		root = btrfs_grab_fs_root(root);
9433 		BUG_ON(!root);
9434 		list_move_tail(&root->delalloc_root,
9435 			       &fs_info->delalloc_roots);
9436 		spin_unlock(&fs_info->delalloc_root_lock);
9437 
9438 		ret = __start_delalloc_inodes(root, delay_iput, nr);
9439 		btrfs_put_fs_root(root);
9440 		if (ret < 0)
9441 			goto out;
9442 
9443 		if (nr != -1) {
9444 			nr -= ret;
9445 			WARN_ON(nr < 0);
9446 		}
9447 		spin_lock(&fs_info->delalloc_root_lock);
9448 	}
9449 	spin_unlock(&fs_info->delalloc_root_lock);
9450 
9451 	ret = 0;
9452 	atomic_inc(&fs_info->async_submit_draining);
9453 	while (atomic_read(&fs_info->nr_async_submits) ||
9454 	      atomic_read(&fs_info->async_delalloc_pages)) {
9455 		wait_event(fs_info->async_submit_wait,
9456 		   (atomic_read(&fs_info->nr_async_submits) == 0 &&
9457 		    atomic_read(&fs_info->async_delalloc_pages) == 0));
9458 	}
9459 	atomic_dec(&fs_info->async_submit_draining);
9460 out:
9461 	if (!list_empty_careful(&splice)) {
9462 		spin_lock(&fs_info->delalloc_root_lock);
9463 		list_splice_tail(&splice, &fs_info->delalloc_roots);
9464 		spin_unlock(&fs_info->delalloc_root_lock);
9465 	}
9466 	mutex_unlock(&fs_info->delalloc_root_mutex);
9467 	return ret;
9468 }
9469 
9470 static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
9471 			 const char *symname)
9472 {
9473 	struct btrfs_trans_handle *trans;
9474 	struct btrfs_root *root = BTRFS_I(dir)->root;
9475 	struct btrfs_path *path;
9476 	struct btrfs_key key;
9477 	struct inode *inode = NULL;
9478 	int err;
9479 	int drop_inode = 0;
9480 	u64 objectid;
9481 	u64 index = 0;
9482 	int name_len;
9483 	int datasize;
9484 	unsigned long ptr;
9485 	struct btrfs_file_extent_item *ei;
9486 	struct extent_buffer *leaf;
9487 
9488 	name_len = strlen(symname);
9489 	if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(root))
9490 		return -ENAMETOOLONG;
9491 
9492 	/*
9493 	 * 2 items for inode item and ref
9494 	 * 2 items for dir items
9495 	 * 1 item for xattr if selinux is on
9496 	 */
9497 	trans = btrfs_start_transaction(root, 5);
9498 	if (IS_ERR(trans))
9499 		return PTR_ERR(trans);
9500 
9501 	err = btrfs_find_free_ino(root, &objectid);
9502 	if (err)
9503 		goto out_unlock;
9504 
9505 	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
9506 				dentry->d_name.len, btrfs_ino(dir), objectid,
9507 				S_IFLNK|S_IRWXUGO, &index);
9508 	if (IS_ERR(inode)) {
9509 		err = PTR_ERR(inode);
9510 		goto out_unlock;
9511 	}
9512 
9513 	/*
9514 	* If the active LSM wants to access the inode during
9515 	* d_instantiate it needs these. Smack checks to see
9516 	* if the filesystem supports xattrs by looking at the
9517 	* ops vector.
9518 	*/
9519 	inode->i_fop = &btrfs_file_operations;
9520 	inode->i_op = &btrfs_file_inode_operations;
9521 	inode->i_mapping->a_ops = &btrfs_aops;
9522 	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
9523 
9524 	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
9525 	if (err)
9526 		goto out_unlock_inode;
9527 
9528 	err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
9529 	if (err)
9530 		goto out_unlock_inode;
9531 
9532 	path = btrfs_alloc_path();
9533 	if (!path) {
9534 		err = -ENOMEM;
9535 		goto out_unlock_inode;
9536 	}
9537 	key.objectid = btrfs_ino(inode);
9538 	key.offset = 0;
9539 	key.type = BTRFS_EXTENT_DATA_KEY;
9540 	datasize = btrfs_file_extent_calc_inline_size(name_len);
9541 	err = btrfs_insert_empty_item(trans, root, path, &key,
9542 				      datasize);
9543 	if (err) {
9544 		btrfs_free_path(path);
9545 		goto out_unlock_inode;
9546 	}
9547 	leaf = path->nodes[0];
9548 	ei = btrfs_item_ptr(leaf, path->slots[0],
9549 			    struct btrfs_file_extent_item);
9550 	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
9551 	btrfs_set_file_extent_type(leaf, ei,
9552 				   BTRFS_FILE_EXTENT_INLINE);
9553 	btrfs_set_file_extent_encryption(leaf, ei, 0);
9554 	btrfs_set_file_extent_compression(leaf, ei, 0);
9555 	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
9556 	btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
9557 
9558 	ptr = btrfs_file_extent_inline_start(ei);
9559 	write_extent_buffer(leaf, symname, ptr, name_len);
9560 	btrfs_mark_buffer_dirty(leaf);
9561 	btrfs_free_path(path);
9562 
9563 	inode->i_op = &btrfs_symlink_inode_operations;
9564 	inode->i_mapping->a_ops = &btrfs_symlink_aops;
9565 	inode_set_bytes(inode, name_len);
9566 	btrfs_i_size_write(inode, name_len);
9567 	err = btrfs_update_inode(trans, root, inode);
9568 	if (err) {
9569 		drop_inode = 1;
9570 		goto out_unlock_inode;
9571 	}
9572 
9573 	unlock_new_inode(inode);
9574 	d_instantiate(dentry, inode);
9575 
9576 out_unlock:
9577 	btrfs_end_transaction(trans, root);
9578 	if (drop_inode) {
9579 		inode_dec_link_count(inode);
9580 		iput(inode);
9581 	}
9582 	btrfs_btree_balance_dirty(root);
9583 	return err;
9584 
9585 out_unlock_inode:
9586 	drop_inode = 1;
9587 	unlock_new_inode(inode);
9588 	goto out_unlock;
9589 }
9590 
9591 static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
9592 				       u64 start, u64 num_bytes, u64 min_size,
9593 				       loff_t actual_len, u64 *alloc_hint,
9594 				       struct btrfs_trans_handle *trans)
9595 {
9596 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
9597 	struct extent_map *em;
9598 	struct btrfs_root *root = BTRFS_I(inode)->root;
9599 	struct btrfs_key ins;
9600 	u64 cur_offset = start;
9601 	u64 i_size;
9602 	u64 cur_bytes;
9603 	int ret = 0;
9604 	bool own_trans = true;
9605 
9606 	if (trans)
9607 		own_trans = false;
9608 	while (num_bytes > 0) {
9609 		if (own_trans) {
9610 			trans = btrfs_start_transaction(root, 3);
9611 			if (IS_ERR(trans)) {
9612 				ret = PTR_ERR(trans);
9613 				break;
9614 			}
9615 		}
9616 
9617 		cur_bytes = min(num_bytes, 256ULL * 1024 * 1024);
9618 		cur_bytes = max(cur_bytes, min_size);
9619 		ret = btrfs_reserve_extent(root, cur_bytes, min_size, 0,
9620 					   *alloc_hint, &ins, 1, 0);
9621 		if (ret) {
9622 			if (own_trans)
9623 				btrfs_end_transaction(trans, root);
9624 			break;
9625 		}
9626 
9627 		ret = insert_reserved_file_extent(trans, inode,
9628 						  cur_offset, ins.objectid,
9629 						  ins.offset, ins.offset,
9630 						  ins.offset, 0, 0, 0,
9631 						  BTRFS_FILE_EXTENT_PREALLOC);
9632 		if (ret) {
9633 			btrfs_free_reserved_extent(root, ins.objectid,
9634 						   ins.offset, 0);
9635 			btrfs_abort_transaction(trans, root, ret);
9636 			if (own_trans)
9637 				btrfs_end_transaction(trans, root);
9638 			break;
9639 		}
9640 
9641 		btrfs_drop_extent_cache(inode, cur_offset,
9642 					cur_offset + ins.offset -1, 0);
9643 
9644 		em = alloc_extent_map();
9645 		if (!em) {
9646 			set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
9647 				&BTRFS_I(inode)->runtime_flags);
9648 			goto next;
9649 		}
9650 
9651 		em->start = cur_offset;
9652 		em->orig_start = cur_offset;
9653 		em->len = ins.offset;
9654 		em->block_start = ins.objectid;
9655 		em->block_len = ins.offset;
9656 		em->orig_block_len = ins.offset;
9657 		em->ram_bytes = ins.offset;
9658 		em->bdev = root->fs_info->fs_devices->latest_bdev;
9659 		set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
9660 		em->generation = trans->transid;
9661 
9662 		while (1) {
9663 			write_lock(&em_tree->lock);
9664 			ret = add_extent_mapping(em_tree, em, 1);
9665 			write_unlock(&em_tree->lock);
9666 			if (ret != -EEXIST)
9667 				break;
9668 			btrfs_drop_extent_cache(inode, cur_offset,
9669 						cur_offset + ins.offset - 1,
9670 						0);
9671 		}
9672 		free_extent_map(em);
9673 next:
9674 		num_bytes -= ins.offset;
9675 		cur_offset += ins.offset;
9676 		*alloc_hint = ins.objectid + ins.offset;
9677 
9678 		inode_inc_iversion(inode);
9679 		inode->i_ctime = CURRENT_TIME;
9680 		BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
9681 		if (!(mode & FALLOC_FL_KEEP_SIZE) &&
9682 		    (actual_len > inode->i_size) &&
9683 		    (cur_offset > inode->i_size)) {
9684 			if (cur_offset > actual_len)
9685 				i_size = actual_len;
9686 			else
9687 				i_size = cur_offset;
9688 			i_size_write(inode, i_size);
9689 			btrfs_ordered_update_i_size(inode, i_size, NULL);
9690 		}
9691 
9692 		ret = btrfs_update_inode(trans, root, inode);
9693 
9694 		if (ret) {
9695 			btrfs_abort_transaction(trans, root, ret);
9696 			if (own_trans)
9697 				btrfs_end_transaction(trans, root);
9698 			break;
9699 		}
9700 
9701 		if (own_trans)
9702 			btrfs_end_transaction(trans, root);
9703 	}
9704 	return ret;
9705 }
9706 
9707 int btrfs_prealloc_file_range(struct inode *inode, int mode,
9708 			      u64 start, u64 num_bytes, u64 min_size,
9709 			      loff_t actual_len, u64 *alloc_hint)
9710 {
9711 	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
9712 					   min_size, actual_len, alloc_hint,
9713 					   NULL);
9714 }
9715 
9716 int btrfs_prealloc_file_range_trans(struct inode *inode,
9717 				    struct btrfs_trans_handle *trans, int mode,
9718 				    u64 start, u64 num_bytes, u64 min_size,
9719 				    loff_t actual_len, u64 *alloc_hint)
9720 {
9721 	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
9722 					   min_size, actual_len, alloc_hint, trans);
9723 }
9724 
9725 static int btrfs_set_page_dirty(struct page *page)
9726 {
9727 	return __set_page_dirty_nobuffers(page);
9728 }
9729 
9730 static int btrfs_permission(struct inode *inode, int mask)
9731 {
9732 	struct btrfs_root *root = BTRFS_I(inode)->root;
9733 	umode_t mode = inode->i_mode;
9734 
9735 	if (mask & MAY_WRITE &&
9736 	    (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
9737 		if (btrfs_root_readonly(root))
9738 			return -EROFS;
9739 		if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
9740 			return -EACCES;
9741 	}
9742 	return generic_permission(inode, mask);
9743 }
9744 
9745 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
9746 {
9747 	struct btrfs_trans_handle *trans;
9748 	struct btrfs_root *root = BTRFS_I(dir)->root;
9749 	struct inode *inode = NULL;
9750 	u64 objectid;
9751 	u64 index;
9752 	int ret = 0;
9753 
9754 	/*
9755 	 * 5 units required for adding orphan entry
9756 	 */
9757 	trans = btrfs_start_transaction(root, 5);
9758 	if (IS_ERR(trans))
9759 		return PTR_ERR(trans);
9760 
9761 	ret = btrfs_find_free_ino(root, &objectid);
9762 	if (ret)
9763 		goto out;
9764 
9765 	inode = btrfs_new_inode(trans, root, dir, NULL, 0,
9766 				btrfs_ino(dir), objectid, mode, &index);
9767 	if (IS_ERR(inode)) {
9768 		ret = PTR_ERR(inode);
9769 		inode = NULL;
9770 		goto out;
9771 	}
9772 
9773 	inode->i_fop = &btrfs_file_operations;
9774 	inode->i_op = &btrfs_file_inode_operations;
9775 
9776 	inode->i_mapping->a_ops = &btrfs_aops;
9777 	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
9778 
9779 	ret = btrfs_init_inode_security(trans, inode, dir, NULL);
9780 	if (ret)
9781 		goto out_inode;
9782 
9783 	ret = btrfs_update_inode(trans, root, inode);
9784 	if (ret)
9785 		goto out_inode;
9786 	ret = btrfs_orphan_add(trans, inode);
9787 	if (ret)
9788 		goto out_inode;
9789 
9790 	/*
9791 	 * We set number of links to 0 in btrfs_new_inode(), and here we set
9792 	 * it to 1 because d_tmpfile() will issue a warning if the count is 0,
9793 	 * through:
9794 	 *
9795 	 *    d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
9796 	 */
9797 	set_nlink(inode, 1);
9798 	unlock_new_inode(inode);
9799 	d_tmpfile(dentry, inode);
9800 	mark_inode_dirty(inode);
9801 
9802 out:
9803 	btrfs_end_transaction(trans, root);
9804 	if (ret)
9805 		iput(inode);
9806 	btrfs_balance_delayed_items(root);
9807 	btrfs_btree_balance_dirty(root);
9808 	return ret;
9809 
9810 out_inode:
9811 	unlock_new_inode(inode);
9812 	goto out;
9813 
9814 }
9815 
9816 /* Inspired by filemap_check_errors() */
9817 int btrfs_inode_check_errors(struct inode *inode)
9818 {
9819 	int ret = 0;
9820 
9821 	if (test_bit(AS_ENOSPC, &inode->i_mapping->flags) &&
9822 	    test_and_clear_bit(AS_ENOSPC, &inode->i_mapping->flags))
9823 		ret = -ENOSPC;
9824 	if (test_bit(AS_EIO, &inode->i_mapping->flags) &&
9825 	    test_and_clear_bit(AS_EIO, &inode->i_mapping->flags))
9826 		ret = -EIO;
9827 
9828 	return ret;
9829 }
9830 
9831 static const struct inode_operations btrfs_dir_inode_operations = {
9832 	.getattr	= btrfs_getattr,
9833 	.lookup		= btrfs_lookup,
9834 	.create		= btrfs_create,
9835 	.unlink		= btrfs_unlink,
9836 	.link		= btrfs_link,
9837 	.mkdir		= btrfs_mkdir,
9838 	.rmdir		= btrfs_rmdir,
9839 	.rename2	= btrfs_rename2,
9840 	.symlink	= btrfs_symlink,
9841 	.setattr	= btrfs_setattr,
9842 	.mknod		= btrfs_mknod,
9843 	.setxattr	= btrfs_setxattr,
9844 	.getxattr	= btrfs_getxattr,
9845 	.listxattr	= btrfs_listxattr,
9846 	.removexattr	= btrfs_removexattr,
9847 	.permission	= btrfs_permission,
9848 	.get_acl	= btrfs_get_acl,
9849 	.set_acl	= btrfs_set_acl,
9850 	.update_time	= btrfs_update_time,
9851 	.tmpfile        = btrfs_tmpfile,
9852 };
9853 static const struct inode_operations btrfs_dir_ro_inode_operations = {
9854 	.lookup		= btrfs_lookup,
9855 	.permission	= btrfs_permission,
9856 	.get_acl	= btrfs_get_acl,
9857 	.set_acl	= btrfs_set_acl,
9858 	.update_time	= btrfs_update_time,
9859 };
9860 
9861 static const struct file_operations btrfs_dir_file_operations = {
9862 	.llseek		= generic_file_llseek,
9863 	.read		= generic_read_dir,
9864 	.iterate	= btrfs_real_readdir,
9865 	.unlocked_ioctl	= btrfs_ioctl,
9866 #ifdef CONFIG_COMPAT
9867 	.compat_ioctl	= btrfs_ioctl,
9868 #endif
9869 	.release        = btrfs_release_file,
9870 	.fsync		= btrfs_sync_file,
9871 };
9872 
9873 static struct extent_io_ops btrfs_extent_io_ops = {
9874 	.fill_delalloc = run_delalloc_range,
9875 	.submit_bio_hook = btrfs_submit_bio_hook,
9876 	.merge_bio_hook = btrfs_merge_bio_hook,
9877 	.readpage_end_io_hook = btrfs_readpage_end_io_hook,
9878 	.writepage_end_io_hook = btrfs_writepage_end_io_hook,
9879 	.writepage_start_hook = btrfs_writepage_start_hook,
9880 	.set_bit_hook = btrfs_set_bit_hook,
9881 	.clear_bit_hook = btrfs_clear_bit_hook,
9882 	.merge_extent_hook = btrfs_merge_extent_hook,
9883 	.split_extent_hook = btrfs_split_extent_hook,
9884 };
9885 
9886 /*
9887  * btrfs doesn't support the bmap operation because swapfiles
9888  * use bmap to make a mapping of extents in the file.  They assume
9889  * these extents won't change over the life of the file and they
9890  * use the bmap result to do IO directly to the drive.
9891  *
9892  * the btrfs bmap call would return logical addresses that aren't
9893  * suitable for IO and they also will change frequently as COW
9894  * operations happen.  So, swapfile + btrfs == corruption.
9895  *
9896  * For now we're avoiding this by dropping bmap.
9897  */
9898 static const struct address_space_operations btrfs_aops = {
9899 	.readpage	= btrfs_readpage,
9900 	.writepage	= btrfs_writepage,
9901 	.writepages	= btrfs_writepages,
9902 	.readpages	= btrfs_readpages,
9903 	.direct_IO	= btrfs_direct_IO,
9904 	.invalidatepage = btrfs_invalidatepage,
9905 	.releasepage	= btrfs_releasepage,
9906 	.set_page_dirty	= btrfs_set_page_dirty,
9907 	.error_remove_page = generic_error_remove_page,
9908 };
9909 
9910 static const struct address_space_operations btrfs_symlink_aops = {
9911 	.readpage	= btrfs_readpage,
9912 	.writepage	= btrfs_writepage,
9913 	.invalidatepage = btrfs_invalidatepage,
9914 	.releasepage	= btrfs_releasepage,
9915 };
9916 
9917 static const struct inode_operations btrfs_file_inode_operations = {
9918 	.getattr	= btrfs_getattr,
9919 	.setattr	= btrfs_setattr,
9920 	.setxattr	= btrfs_setxattr,
9921 	.getxattr	= btrfs_getxattr,
9922 	.listxattr      = btrfs_listxattr,
9923 	.removexattr	= btrfs_removexattr,
9924 	.permission	= btrfs_permission,
9925 	.fiemap		= btrfs_fiemap,
9926 	.get_acl	= btrfs_get_acl,
9927 	.set_acl	= btrfs_set_acl,
9928 	.update_time	= btrfs_update_time,
9929 };
9930 static const struct inode_operations btrfs_special_inode_operations = {
9931 	.getattr	= btrfs_getattr,
9932 	.setattr	= btrfs_setattr,
9933 	.permission	= btrfs_permission,
9934 	.setxattr	= btrfs_setxattr,
9935 	.getxattr	= btrfs_getxattr,
9936 	.listxattr	= btrfs_listxattr,
9937 	.removexattr	= btrfs_removexattr,
9938 	.get_acl	= btrfs_get_acl,
9939 	.set_acl	= btrfs_set_acl,
9940 	.update_time	= btrfs_update_time,
9941 };
9942 static const struct inode_operations btrfs_symlink_inode_operations = {
9943 	.readlink	= generic_readlink,
9944 	.follow_link	= page_follow_link_light,
9945 	.put_link	= page_put_link,
9946 	.getattr	= btrfs_getattr,
9947 	.setattr	= btrfs_setattr,
9948 	.permission	= btrfs_permission,
9949 	.setxattr	= btrfs_setxattr,
9950 	.getxattr	= btrfs_getxattr,
9951 	.listxattr	= btrfs_listxattr,
9952 	.removexattr	= btrfs_removexattr,
9953 	.update_time	= btrfs_update_time,
9954 };
9955 
9956 const struct dentry_operations btrfs_dentry_operations = {
9957 	.d_delete	= btrfs_dentry_delete,
9958 	.d_release	= btrfs_dentry_release,
9959 };
9960