xref: /openbmc/linux/fs/btrfs/inode.c (revision ae3473231e77a3f1909d48cd144cebe5e1d049b3)
1 /*
2  * Copyright (C) 2007 Oracle.  All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public
6  * License v2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11  * General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public
14  * License along with this program; if not, write to the
15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16  * Boston, MA 021110-1307, USA.
17  */
18 
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/buffer_head.h>
22 #include <linux/file.h>
23 #include <linux/fs.h>
24 #include <linux/pagemap.h>
25 #include <linux/highmem.h>
26 #include <linux/time.h>
27 #include <linux/init.h>
28 #include <linux/string.h>
29 #include <linux/backing-dev.h>
30 #include <linux/mpage.h>
31 #include <linux/swap.h>
32 #include <linux/writeback.h>
33 #include <linux/compat.h>
34 #include <linux/bit_spinlock.h>
35 #include <linux/xattr.h>
36 #include <linux/posix_acl.h>
37 #include <linux/falloc.h>
38 #include <linux/slab.h>
39 #include <linux/ratelimit.h>
40 #include <linux/mount.h>
41 #include <linux/btrfs.h>
42 #include <linux/blkdev.h>
43 #include <linux/posix_acl_xattr.h>
44 #include <linux/uio.h>
45 #include "ctree.h"
46 #include "disk-io.h"
47 #include "transaction.h"
48 #include "btrfs_inode.h"
49 #include "print-tree.h"
50 #include "ordered-data.h"
51 #include "xattr.h"
52 #include "tree-log.h"
53 #include "volumes.h"
54 #include "compression.h"
55 #include "locking.h"
56 #include "free-space-cache.h"
57 #include "inode-map.h"
58 #include "backref.h"
59 #include "hash.h"
60 #include "props.h"
61 #include "qgroup.h"
62 #include "dedupe.h"
63 
64 struct btrfs_iget_args {
65 	struct btrfs_key *location;
66 	struct btrfs_root *root;
67 };
68 
69 struct btrfs_dio_data {
70 	u64 outstanding_extents;
71 	u64 reserve;
72 	u64 unsubmitted_oe_range_start;
73 	u64 unsubmitted_oe_range_end;
74 };
75 
76 static const struct inode_operations btrfs_dir_inode_operations;
77 static const struct inode_operations btrfs_symlink_inode_operations;
78 static const struct inode_operations btrfs_dir_ro_inode_operations;
79 static const struct inode_operations btrfs_special_inode_operations;
80 static const struct inode_operations btrfs_file_inode_operations;
81 static const struct address_space_operations btrfs_aops;
82 static const struct address_space_operations btrfs_symlink_aops;
83 static const struct file_operations btrfs_dir_file_operations;
84 static const struct extent_io_ops btrfs_extent_io_ops;
85 
86 static struct kmem_cache *btrfs_inode_cachep;
87 struct kmem_cache *btrfs_trans_handle_cachep;
88 struct kmem_cache *btrfs_transaction_cachep;
89 struct kmem_cache *btrfs_path_cachep;
90 struct kmem_cache *btrfs_free_space_cachep;
91 
92 #define S_SHIFT 12
93 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
94 	[S_IFREG >> S_SHIFT]	= BTRFS_FT_REG_FILE,
95 	[S_IFDIR >> S_SHIFT]	= BTRFS_FT_DIR,
96 	[S_IFCHR >> S_SHIFT]	= BTRFS_FT_CHRDEV,
97 	[S_IFBLK >> S_SHIFT]	= BTRFS_FT_BLKDEV,
98 	[S_IFIFO >> S_SHIFT]	= BTRFS_FT_FIFO,
99 	[S_IFSOCK >> S_SHIFT]	= BTRFS_FT_SOCK,
100 	[S_IFLNK >> S_SHIFT]	= BTRFS_FT_SYMLINK,
101 };
102 
103 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
104 static int btrfs_truncate(struct inode *inode);
105 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
106 static noinline int cow_file_range(struct inode *inode,
107 				   struct page *locked_page,
108 				   u64 start, u64 end, u64 delalloc_end,
109 				   int *page_started, unsigned long *nr_written,
110 				   int unlock, struct btrfs_dedupe_hash *hash);
111 static struct extent_map *create_pinned_em(struct inode *inode, u64 start,
112 					   u64 len, u64 orig_start,
113 					   u64 block_start, u64 block_len,
114 					   u64 orig_block_len, u64 ram_bytes,
115 					   int type);
116 
117 static int btrfs_dirty_inode(struct inode *inode);
118 
119 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
120 void btrfs_test_inode_set_ops(struct inode *inode)
121 {
122 	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
123 }
124 #endif
125 
126 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
127 				     struct inode *inode,  struct inode *dir,
128 				     const struct qstr *qstr)
129 {
130 	int err;
131 
132 	err = btrfs_init_acl(trans, inode, dir);
133 	if (!err)
134 		err = btrfs_xattr_security_init(trans, inode, dir, qstr);
135 	return err;
136 }
137 
138 /*
139  * this does all the hard work for inserting an inline extent into
140  * the btree.  The caller should have done a btrfs_drop_extents so that
141  * no overlapping inline items exist in the btree
142  */
143 static int insert_inline_extent(struct btrfs_trans_handle *trans,
144 				struct btrfs_path *path, int extent_inserted,
145 				struct btrfs_root *root, struct inode *inode,
146 				u64 start, size_t size, size_t compressed_size,
147 				int compress_type,
148 				struct page **compressed_pages)
149 {
150 	struct extent_buffer *leaf;
151 	struct page *page = NULL;
152 	char *kaddr;
153 	unsigned long ptr;
154 	struct btrfs_file_extent_item *ei;
155 	int err = 0;
156 	int ret;
157 	size_t cur_size = size;
158 	unsigned long offset;
159 
160 	if (compressed_size && compressed_pages)
161 		cur_size = compressed_size;
162 
163 	inode_add_bytes(inode, size);
164 
165 	if (!extent_inserted) {
166 		struct btrfs_key key;
167 		size_t datasize;
168 
169 		key.objectid = btrfs_ino(inode);
170 		key.offset = start;
171 		key.type = BTRFS_EXTENT_DATA_KEY;
172 
173 		datasize = btrfs_file_extent_calc_inline_size(cur_size);
174 		path->leave_spinning = 1;
175 		ret = btrfs_insert_empty_item(trans, root, path, &key,
176 					      datasize);
177 		if (ret) {
178 			err = ret;
179 			goto fail;
180 		}
181 	}
182 	leaf = path->nodes[0];
183 	ei = btrfs_item_ptr(leaf, path->slots[0],
184 			    struct btrfs_file_extent_item);
185 	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
186 	btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
187 	btrfs_set_file_extent_encryption(leaf, ei, 0);
188 	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
189 	btrfs_set_file_extent_ram_bytes(leaf, ei, size);
190 	ptr = btrfs_file_extent_inline_start(ei);
191 
192 	if (compress_type != BTRFS_COMPRESS_NONE) {
193 		struct page *cpage;
194 		int i = 0;
195 		while (compressed_size > 0) {
196 			cpage = compressed_pages[i];
197 			cur_size = min_t(unsigned long, compressed_size,
198 				       PAGE_SIZE);
199 
200 			kaddr = kmap_atomic(cpage);
201 			write_extent_buffer(leaf, kaddr, ptr, cur_size);
202 			kunmap_atomic(kaddr);
203 
204 			i++;
205 			ptr += cur_size;
206 			compressed_size -= cur_size;
207 		}
208 		btrfs_set_file_extent_compression(leaf, ei,
209 						  compress_type);
210 	} else {
211 		page = find_get_page(inode->i_mapping,
212 				     start >> PAGE_SHIFT);
213 		btrfs_set_file_extent_compression(leaf, ei, 0);
214 		kaddr = kmap_atomic(page);
215 		offset = start & (PAGE_SIZE - 1);
216 		write_extent_buffer(leaf, kaddr + offset, ptr, size);
217 		kunmap_atomic(kaddr);
218 		put_page(page);
219 	}
220 	btrfs_mark_buffer_dirty(leaf);
221 	btrfs_release_path(path);
222 
223 	/*
224 	 * we're an inline extent, so nobody can
225 	 * extend the file past i_size without locking
226 	 * a page we already have locked.
227 	 *
228 	 * We must do any isize and inode updates
229 	 * before we unlock the pages.  Otherwise we
230 	 * could end up racing with unlink.
231 	 */
232 	BTRFS_I(inode)->disk_i_size = inode->i_size;
233 	ret = btrfs_update_inode(trans, root, inode);
234 
235 	return ret;
236 fail:
237 	return err;
238 }
239 
240 
241 /*
242  * conditionally insert an inline extent into the file.  This
243  * does the checks required to make sure the data is small enough
244  * to fit as an inline extent.
245  */
246 static noinline int cow_file_range_inline(struct btrfs_root *root,
247 					  struct inode *inode, u64 start,
248 					  u64 end, size_t compressed_size,
249 					  int compress_type,
250 					  struct page **compressed_pages)
251 {
252 	struct btrfs_fs_info *fs_info = root->fs_info;
253 	struct btrfs_trans_handle *trans;
254 	u64 isize = i_size_read(inode);
255 	u64 actual_end = min(end + 1, isize);
256 	u64 inline_len = actual_end - start;
257 	u64 aligned_end = ALIGN(end, fs_info->sectorsize);
258 	u64 data_len = inline_len;
259 	int ret;
260 	struct btrfs_path *path;
261 	int extent_inserted = 0;
262 	u32 extent_item_size;
263 
264 	if (compressed_size)
265 		data_len = compressed_size;
266 
267 	if (start > 0 ||
268 	    actual_end > fs_info->sectorsize ||
269 	    data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) ||
270 	    (!compressed_size &&
271 	    (actual_end & (fs_info->sectorsize - 1)) == 0) ||
272 	    end + 1 < isize ||
273 	    data_len > fs_info->max_inline) {
274 		return 1;
275 	}
276 
277 	path = btrfs_alloc_path();
278 	if (!path)
279 		return -ENOMEM;
280 
281 	trans = btrfs_join_transaction(root);
282 	if (IS_ERR(trans)) {
283 		btrfs_free_path(path);
284 		return PTR_ERR(trans);
285 	}
286 	trans->block_rsv = &fs_info->delalloc_block_rsv;
287 
288 	if (compressed_size && compressed_pages)
289 		extent_item_size = btrfs_file_extent_calc_inline_size(
290 		   compressed_size);
291 	else
292 		extent_item_size = btrfs_file_extent_calc_inline_size(
293 		    inline_len);
294 
295 	ret = __btrfs_drop_extents(trans, root, inode, path,
296 				   start, aligned_end, NULL,
297 				   1, 1, extent_item_size, &extent_inserted);
298 	if (ret) {
299 		btrfs_abort_transaction(trans, ret);
300 		goto out;
301 	}
302 
303 	if (isize > actual_end)
304 		inline_len = min_t(u64, isize, actual_end);
305 	ret = insert_inline_extent(trans, path, extent_inserted,
306 				   root, inode, start,
307 				   inline_len, compressed_size,
308 				   compress_type, compressed_pages);
309 	if (ret && ret != -ENOSPC) {
310 		btrfs_abort_transaction(trans, ret);
311 		goto out;
312 	} else if (ret == -ENOSPC) {
313 		ret = 1;
314 		goto out;
315 	}
316 
317 	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
318 	btrfs_delalloc_release_metadata(inode, end + 1 - start);
319 	btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0);
320 out:
321 	/*
322 	 * Don't forget to free the reserved space, as for inlined extent
323 	 * it won't count as data extent, free them directly here.
324 	 * And at reserve time, it's always aligned to page size, so
325 	 * just free one page here.
326 	 */
327 	btrfs_qgroup_free_data(inode, 0, PAGE_SIZE);
328 	btrfs_free_path(path);
329 	btrfs_end_transaction(trans);
330 	return ret;
331 }
332 
333 struct async_extent {
334 	u64 start;
335 	u64 ram_size;
336 	u64 compressed_size;
337 	struct page **pages;
338 	unsigned long nr_pages;
339 	int compress_type;
340 	struct list_head list;
341 };
342 
343 struct async_cow {
344 	struct inode *inode;
345 	struct btrfs_root *root;
346 	struct page *locked_page;
347 	u64 start;
348 	u64 end;
349 	struct list_head extents;
350 	struct btrfs_work work;
351 };
352 
353 static noinline int add_async_extent(struct async_cow *cow,
354 				     u64 start, u64 ram_size,
355 				     u64 compressed_size,
356 				     struct page **pages,
357 				     unsigned long nr_pages,
358 				     int compress_type)
359 {
360 	struct async_extent *async_extent;
361 
362 	async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
363 	BUG_ON(!async_extent); /* -ENOMEM */
364 	async_extent->start = start;
365 	async_extent->ram_size = ram_size;
366 	async_extent->compressed_size = compressed_size;
367 	async_extent->pages = pages;
368 	async_extent->nr_pages = nr_pages;
369 	async_extent->compress_type = compress_type;
370 	list_add_tail(&async_extent->list, &cow->extents);
371 	return 0;
372 }
373 
374 static inline int inode_need_compress(struct inode *inode)
375 {
376 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
377 
378 	/* force compress */
379 	if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
380 		return 1;
381 	/* bad compression ratios */
382 	if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
383 		return 0;
384 	if (btrfs_test_opt(fs_info, COMPRESS) ||
385 	    BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
386 	    BTRFS_I(inode)->force_compress)
387 		return 1;
388 	return 0;
389 }
390 
391 /*
392  * we create compressed extents in two phases.  The first
393  * phase compresses a range of pages that have already been
394  * locked (both pages and state bits are locked).
395  *
396  * This is done inside an ordered work queue, and the compression
397  * is spread across many cpus.  The actual IO submission is step
398  * two, and the ordered work queue takes care of making sure that
399  * happens in the same order things were put onto the queue by
400  * writepages and friends.
401  *
402  * If this code finds it can't get good compression, it puts an
403  * entry onto the work queue to write the uncompressed bytes.  This
404  * makes sure that both compressed inodes and uncompressed inodes
405  * are written in the same order that the flusher thread sent them
406  * down.
407  */
408 static noinline void compress_file_range(struct inode *inode,
409 					struct page *locked_page,
410 					u64 start, u64 end,
411 					struct async_cow *async_cow,
412 					int *num_added)
413 {
414 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
415 	struct btrfs_root *root = BTRFS_I(inode)->root;
416 	u64 num_bytes;
417 	u64 blocksize = fs_info->sectorsize;
418 	u64 actual_end;
419 	u64 isize = i_size_read(inode);
420 	int ret = 0;
421 	struct page **pages = NULL;
422 	unsigned long nr_pages;
423 	unsigned long nr_pages_ret = 0;
424 	unsigned long total_compressed = 0;
425 	unsigned long total_in = 0;
426 	unsigned long max_compressed = SZ_128K;
427 	unsigned long max_uncompressed = SZ_128K;
428 	int i;
429 	int will_compress;
430 	int compress_type = fs_info->compress_type;
431 	int redirty = 0;
432 
433 	/* if this is a small write inside eof, kick off a defrag */
434 	if ((end - start + 1) < SZ_16K &&
435 	    (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
436 		btrfs_add_inode_defrag(NULL, inode);
437 
438 	actual_end = min_t(u64, isize, end + 1);
439 again:
440 	will_compress = 0;
441 	nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
442 	nr_pages = min_t(unsigned long, nr_pages, SZ_128K / PAGE_SIZE);
443 
444 	/*
445 	 * we don't want to send crud past the end of i_size through
446 	 * compression, that's just a waste of CPU time.  So, if the
447 	 * end of the file is before the start of our current
448 	 * requested range of bytes, we bail out to the uncompressed
449 	 * cleanup code that can deal with all of this.
450 	 *
451 	 * It isn't really the fastest way to fix things, but this is a
452 	 * very uncommon corner.
453 	 */
454 	if (actual_end <= start)
455 		goto cleanup_and_bail_uncompressed;
456 
457 	total_compressed = actual_end - start;
458 
459 	/*
460 	 * skip compression for a small file range(<=blocksize) that
461 	 * isn't an inline extent, since it doesn't save disk space at all.
462 	 */
463 	if (total_compressed <= blocksize &&
464 	   (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
465 		goto cleanup_and_bail_uncompressed;
466 
467 	/* we want to make sure that amount of ram required to uncompress
468 	 * an extent is reasonable, so we limit the total size in ram
469 	 * of a compressed extent to 128k.  This is a crucial number
470 	 * because it also controls how easily we can spread reads across
471 	 * cpus for decompression.
472 	 *
473 	 * We also want to make sure the amount of IO required to do
474 	 * a random read is reasonably small, so we limit the size of
475 	 * a compressed extent to 128k.
476 	 */
477 	total_compressed = min(total_compressed, max_uncompressed);
478 	num_bytes = ALIGN(end - start + 1, blocksize);
479 	num_bytes = max(blocksize,  num_bytes);
480 	total_in = 0;
481 	ret = 0;
482 
483 	/*
484 	 * we do compression for mount -o compress and when the
485 	 * inode has not been flagged as nocompress.  This flag can
486 	 * change at any time if we discover bad compression ratios.
487 	 */
488 	if (inode_need_compress(inode)) {
489 		WARN_ON(pages);
490 		pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
491 		if (!pages) {
492 			/* just bail out to the uncompressed code */
493 			goto cont;
494 		}
495 
496 		if (BTRFS_I(inode)->force_compress)
497 			compress_type = BTRFS_I(inode)->force_compress;
498 
499 		/*
500 		 * we need to call clear_page_dirty_for_io on each
501 		 * page in the range.  Otherwise applications with the file
502 		 * mmap'd can wander in and change the page contents while
503 		 * we are compressing them.
504 		 *
505 		 * If the compression fails for any reason, we set the pages
506 		 * dirty again later on.
507 		 */
508 		extent_range_clear_dirty_for_io(inode, start, end);
509 		redirty = 1;
510 		ret = btrfs_compress_pages(compress_type,
511 					   inode->i_mapping, start,
512 					   total_compressed, pages,
513 					   nr_pages, &nr_pages_ret,
514 					   &total_in,
515 					   &total_compressed,
516 					   max_compressed);
517 
518 		if (!ret) {
519 			unsigned long offset = total_compressed &
520 				(PAGE_SIZE - 1);
521 			struct page *page = pages[nr_pages_ret - 1];
522 			char *kaddr;
523 
524 			/* zero the tail end of the last page, we might be
525 			 * sending it down to disk
526 			 */
527 			if (offset) {
528 				kaddr = kmap_atomic(page);
529 				memset(kaddr + offset, 0,
530 				       PAGE_SIZE - offset);
531 				kunmap_atomic(kaddr);
532 			}
533 			will_compress = 1;
534 		}
535 	}
536 cont:
537 	if (start == 0) {
538 		/* lets try to make an inline extent */
539 		if (ret || total_in < (actual_end - start)) {
540 			/* we didn't compress the entire range, try
541 			 * to make an uncompressed inline extent.
542 			 */
543 			ret = cow_file_range_inline(root, inode, start, end,
544 						    0, 0, NULL);
545 		} else {
546 			/* try making a compressed inline extent */
547 			ret = cow_file_range_inline(root, inode, start, end,
548 						    total_compressed,
549 						    compress_type, pages);
550 		}
551 		if (ret <= 0) {
552 			unsigned long clear_flags = EXTENT_DELALLOC |
553 				EXTENT_DEFRAG;
554 			unsigned long page_error_op;
555 
556 			clear_flags |= (ret < 0) ? EXTENT_DO_ACCOUNTING : 0;
557 			page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
558 
559 			/*
560 			 * inline extent creation worked or returned error,
561 			 * we don't need to create any more async work items.
562 			 * Unlock and free up our temp pages.
563 			 */
564 			extent_clear_unlock_delalloc(inode, start, end, end,
565 						     NULL, clear_flags,
566 						     PAGE_UNLOCK |
567 						     PAGE_CLEAR_DIRTY |
568 						     PAGE_SET_WRITEBACK |
569 						     page_error_op |
570 						     PAGE_END_WRITEBACK);
571 			btrfs_free_reserved_data_space_noquota(inode, start,
572 						end - start + 1);
573 			goto free_pages_out;
574 		}
575 	}
576 
577 	if (will_compress) {
578 		/*
579 		 * we aren't doing an inline extent round the compressed size
580 		 * up to a block size boundary so the allocator does sane
581 		 * things
582 		 */
583 		total_compressed = ALIGN(total_compressed, blocksize);
584 
585 		/*
586 		 * one last check to make sure the compression is really a
587 		 * win, compare the page count read with the blocks on disk
588 		 */
589 		total_in = ALIGN(total_in, PAGE_SIZE);
590 		if (total_compressed >= total_in) {
591 			will_compress = 0;
592 		} else {
593 			num_bytes = total_in;
594 			*num_added += 1;
595 
596 			/*
597 			 * The async work queues will take care of doing actual
598 			 * allocation on disk for these compressed pages, and
599 			 * will submit them to the elevator.
600 			 */
601 			add_async_extent(async_cow, start, num_bytes,
602 					total_compressed, pages, nr_pages_ret,
603 					compress_type);
604 
605 			if (start + num_bytes < end) {
606 				start += num_bytes;
607 				pages = NULL;
608 				cond_resched();
609 				goto again;
610 			}
611 			return;
612 		}
613 	}
614 	if (pages) {
615 		/*
616 		 * the compression code ran but failed to make things smaller,
617 		 * free any pages it allocated and our page pointer array
618 		 */
619 		for (i = 0; i < nr_pages_ret; i++) {
620 			WARN_ON(pages[i]->mapping);
621 			put_page(pages[i]);
622 		}
623 		kfree(pages);
624 		pages = NULL;
625 		total_compressed = 0;
626 		nr_pages_ret = 0;
627 
628 		/* flag the file so we don't compress in the future */
629 		if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
630 		    !(BTRFS_I(inode)->force_compress)) {
631 			BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
632 		}
633 	}
634 cleanup_and_bail_uncompressed:
635 	/*
636 	 * No compression, but we still need to write the pages in the file
637 	 * we've been given so far.  redirty the locked page if it corresponds
638 	 * to our extent and set things up for the async work queue to run
639 	 * cow_file_range to do the normal delalloc dance.
640 	 */
641 	if (page_offset(locked_page) >= start &&
642 	    page_offset(locked_page) <= end)
643 		__set_page_dirty_nobuffers(locked_page);
644 		/* unlocked later on in the async handlers */
645 
646 	if (redirty)
647 		extent_range_redirty_for_io(inode, start, end);
648 	add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0,
649 			 BTRFS_COMPRESS_NONE);
650 	*num_added += 1;
651 
652 	return;
653 
654 free_pages_out:
655 	for (i = 0; i < nr_pages_ret; i++) {
656 		WARN_ON(pages[i]->mapping);
657 		put_page(pages[i]);
658 	}
659 	kfree(pages);
660 }
661 
662 static void free_async_extent_pages(struct async_extent *async_extent)
663 {
664 	int i;
665 
666 	if (!async_extent->pages)
667 		return;
668 
669 	for (i = 0; i < async_extent->nr_pages; i++) {
670 		WARN_ON(async_extent->pages[i]->mapping);
671 		put_page(async_extent->pages[i]);
672 	}
673 	kfree(async_extent->pages);
674 	async_extent->nr_pages = 0;
675 	async_extent->pages = NULL;
676 }
677 
678 /*
679  * phase two of compressed writeback.  This is the ordered portion
680  * of the code, which only gets called in the order the work was
681  * queued.  We walk all the async extents created by compress_file_range
682  * and send them down to the disk.
683  */
684 static noinline void submit_compressed_extents(struct inode *inode,
685 					      struct async_cow *async_cow)
686 {
687 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
688 	struct async_extent *async_extent;
689 	u64 alloc_hint = 0;
690 	struct btrfs_key ins;
691 	struct extent_map *em;
692 	struct btrfs_root *root = BTRFS_I(inode)->root;
693 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
694 	struct extent_io_tree *io_tree;
695 	int ret = 0;
696 
697 again:
698 	while (!list_empty(&async_cow->extents)) {
699 		async_extent = list_entry(async_cow->extents.next,
700 					  struct async_extent, list);
701 		list_del(&async_extent->list);
702 
703 		io_tree = &BTRFS_I(inode)->io_tree;
704 
705 retry:
706 		/* did the compression code fall back to uncompressed IO? */
707 		if (!async_extent->pages) {
708 			int page_started = 0;
709 			unsigned long nr_written = 0;
710 
711 			lock_extent(io_tree, async_extent->start,
712 					 async_extent->start +
713 					 async_extent->ram_size - 1);
714 
715 			/* allocate blocks */
716 			ret = cow_file_range(inode, async_cow->locked_page,
717 					     async_extent->start,
718 					     async_extent->start +
719 					     async_extent->ram_size - 1,
720 					     async_extent->start +
721 					     async_extent->ram_size - 1,
722 					     &page_started, &nr_written, 0,
723 					     NULL);
724 
725 			/* JDM XXX */
726 
727 			/*
728 			 * if page_started, cow_file_range inserted an
729 			 * inline extent and took care of all the unlocking
730 			 * and IO for us.  Otherwise, we need to submit
731 			 * all those pages down to the drive.
732 			 */
733 			if (!page_started && !ret)
734 				extent_write_locked_range(io_tree,
735 						  inode, async_extent->start,
736 						  async_extent->start +
737 						  async_extent->ram_size - 1,
738 						  btrfs_get_extent,
739 						  WB_SYNC_ALL);
740 			else if (ret)
741 				unlock_page(async_cow->locked_page);
742 			kfree(async_extent);
743 			cond_resched();
744 			continue;
745 		}
746 
747 		lock_extent(io_tree, async_extent->start,
748 			    async_extent->start + async_extent->ram_size - 1);
749 
750 		ret = btrfs_reserve_extent(root, async_extent->ram_size,
751 					   async_extent->compressed_size,
752 					   async_extent->compressed_size,
753 					   0, alloc_hint, &ins, 1, 1);
754 		if (ret) {
755 			free_async_extent_pages(async_extent);
756 
757 			if (ret == -ENOSPC) {
758 				unlock_extent(io_tree, async_extent->start,
759 					      async_extent->start +
760 					      async_extent->ram_size - 1);
761 
762 				/*
763 				 * we need to redirty the pages if we decide to
764 				 * fallback to uncompressed IO, otherwise we
765 				 * will not submit these pages down to lower
766 				 * layers.
767 				 */
768 				extent_range_redirty_for_io(inode,
769 						async_extent->start,
770 						async_extent->start +
771 						async_extent->ram_size - 1);
772 
773 				goto retry;
774 			}
775 			goto out_free;
776 		}
777 		/*
778 		 * here we're doing allocation and writeback of the
779 		 * compressed pages
780 		 */
781 		btrfs_drop_extent_cache(inode, async_extent->start,
782 					async_extent->start +
783 					async_extent->ram_size - 1, 0);
784 
785 		em = alloc_extent_map();
786 		if (!em) {
787 			ret = -ENOMEM;
788 			goto out_free_reserve;
789 		}
790 		em->start = async_extent->start;
791 		em->len = async_extent->ram_size;
792 		em->orig_start = em->start;
793 		em->mod_start = em->start;
794 		em->mod_len = em->len;
795 
796 		em->block_start = ins.objectid;
797 		em->block_len = ins.offset;
798 		em->orig_block_len = ins.offset;
799 		em->ram_bytes = async_extent->ram_size;
800 		em->bdev = fs_info->fs_devices->latest_bdev;
801 		em->compress_type = async_extent->compress_type;
802 		set_bit(EXTENT_FLAG_PINNED, &em->flags);
803 		set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
804 		em->generation = -1;
805 
806 		while (1) {
807 			write_lock(&em_tree->lock);
808 			ret = add_extent_mapping(em_tree, em, 1);
809 			write_unlock(&em_tree->lock);
810 			if (ret != -EEXIST) {
811 				free_extent_map(em);
812 				break;
813 			}
814 			btrfs_drop_extent_cache(inode, async_extent->start,
815 						async_extent->start +
816 						async_extent->ram_size - 1, 0);
817 		}
818 
819 		if (ret)
820 			goto out_free_reserve;
821 
822 		ret = btrfs_add_ordered_extent_compress(inode,
823 						async_extent->start,
824 						ins.objectid,
825 						async_extent->ram_size,
826 						ins.offset,
827 						BTRFS_ORDERED_COMPRESSED,
828 						async_extent->compress_type);
829 		if (ret) {
830 			btrfs_drop_extent_cache(inode, async_extent->start,
831 						async_extent->start +
832 						async_extent->ram_size - 1, 0);
833 			goto out_free_reserve;
834 		}
835 		btrfs_dec_block_group_reservations(fs_info, ins.objectid);
836 
837 		/*
838 		 * clear dirty, set writeback and unlock the pages.
839 		 */
840 		extent_clear_unlock_delalloc(inode, async_extent->start,
841 				async_extent->start +
842 				async_extent->ram_size - 1,
843 				async_extent->start +
844 				async_extent->ram_size - 1,
845 				NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
846 				PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
847 				PAGE_SET_WRITEBACK);
848 		ret = btrfs_submit_compressed_write(inode,
849 				    async_extent->start,
850 				    async_extent->ram_size,
851 				    ins.objectid,
852 				    ins.offset, async_extent->pages,
853 				    async_extent->nr_pages);
854 		if (ret) {
855 			struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
856 			struct page *p = async_extent->pages[0];
857 			const u64 start = async_extent->start;
858 			const u64 end = start + async_extent->ram_size - 1;
859 
860 			p->mapping = inode->i_mapping;
861 			tree->ops->writepage_end_io_hook(p, start, end,
862 							 NULL, 0);
863 			p->mapping = NULL;
864 			extent_clear_unlock_delalloc(inode, start, end, end,
865 						     NULL, 0,
866 						     PAGE_END_WRITEBACK |
867 						     PAGE_SET_ERROR);
868 			free_async_extent_pages(async_extent);
869 		}
870 		alloc_hint = ins.objectid + ins.offset;
871 		kfree(async_extent);
872 		cond_resched();
873 	}
874 	return;
875 out_free_reserve:
876 	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
877 	btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
878 out_free:
879 	extent_clear_unlock_delalloc(inode, async_extent->start,
880 				     async_extent->start +
881 				     async_extent->ram_size - 1,
882 				     async_extent->start +
883 				     async_extent->ram_size - 1,
884 				     NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
885 				     EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
886 				     PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
887 				     PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
888 				     PAGE_SET_ERROR);
889 	free_async_extent_pages(async_extent);
890 	kfree(async_extent);
891 	goto again;
892 }
893 
894 static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
895 				      u64 num_bytes)
896 {
897 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
898 	struct extent_map *em;
899 	u64 alloc_hint = 0;
900 
901 	read_lock(&em_tree->lock);
902 	em = search_extent_mapping(em_tree, start, num_bytes);
903 	if (em) {
904 		/*
905 		 * if block start isn't an actual block number then find the
906 		 * first block in this inode and use that as a hint.  If that
907 		 * block is also bogus then just don't worry about it.
908 		 */
909 		if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
910 			free_extent_map(em);
911 			em = search_extent_mapping(em_tree, 0, 0);
912 			if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
913 				alloc_hint = em->block_start;
914 			if (em)
915 				free_extent_map(em);
916 		} else {
917 			alloc_hint = em->block_start;
918 			free_extent_map(em);
919 		}
920 	}
921 	read_unlock(&em_tree->lock);
922 
923 	return alloc_hint;
924 }
925 
926 /*
927  * when extent_io.c finds a delayed allocation range in the file,
928  * the call backs end up in this code.  The basic idea is to
929  * allocate extents on disk for the range, and create ordered data structs
930  * in ram to track those extents.
931  *
932  * locked_page is the page that writepage had locked already.  We use
933  * it to make sure we don't do extra locks or unlocks.
934  *
935  * *page_started is set to one if we unlock locked_page and do everything
936  * required to start IO on it.  It may be clean and already done with
937  * IO when we return.
938  */
939 static noinline int cow_file_range(struct inode *inode,
940 				   struct page *locked_page,
941 				   u64 start, u64 end, u64 delalloc_end,
942 				   int *page_started, unsigned long *nr_written,
943 				   int unlock, struct btrfs_dedupe_hash *hash)
944 {
945 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
946 	struct btrfs_root *root = BTRFS_I(inode)->root;
947 	u64 alloc_hint = 0;
948 	u64 num_bytes;
949 	unsigned long ram_size;
950 	u64 disk_num_bytes;
951 	u64 cur_alloc_size;
952 	u64 blocksize = fs_info->sectorsize;
953 	struct btrfs_key ins;
954 	struct extent_map *em;
955 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
956 	int ret = 0;
957 
958 	if (btrfs_is_free_space_inode(inode)) {
959 		WARN_ON_ONCE(1);
960 		ret = -EINVAL;
961 		goto out_unlock;
962 	}
963 
964 	num_bytes = ALIGN(end - start + 1, blocksize);
965 	num_bytes = max(blocksize,  num_bytes);
966 	disk_num_bytes = num_bytes;
967 
968 	/* if this is a small write inside eof, kick off defrag */
969 	if (num_bytes < SZ_64K &&
970 	    (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
971 		btrfs_add_inode_defrag(NULL, inode);
972 
973 	if (start == 0) {
974 		/* lets try to make an inline extent */
975 		ret = cow_file_range_inline(root, inode, start, end, 0, 0,
976 					    NULL);
977 		if (ret == 0) {
978 			extent_clear_unlock_delalloc(inode, start, end,
979 				     delalloc_end, NULL,
980 				     EXTENT_LOCKED | EXTENT_DELALLOC |
981 				     EXTENT_DEFRAG, PAGE_UNLOCK |
982 				     PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
983 				     PAGE_END_WRITEBACK);
984 			btrfs_free_reserved_data_space_noquota(inode, start,
985 						end - start + 1);
986 			*nr_written = *nr_written +
987 			     (end - start + PAGE_SIZE) / PAGE_SIZE;
988 			*page_started = 1;
989 			goto out;
990 		} else if (ret < 0) {
991 			goto out_unlock;
992 		}
993 	}
994 
995 	BUG_ON(disk_num_bytes >
996 	       btrfs_super_total_bytes(fs_info->super_copy));
997 
998 	alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
999 	btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0);
1000 
1001 	while (disk_num_bytes > 0) {
1002 		unsigned long op;
1003 
1004 		cur_alloc_size = disk_num_bytes;
1005 		ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
1006 					   fs_info->sectorsize, 0, alloc_hint,
1007 					   &ins, 1, 1);
1008 		if (ret < 0)
1009 			goto out_unlock;
1010 
1011 		em = alloc_extent_map();
1012 		if (!em) {
1013 			ret = -ENOMEM;
1014 			goto out_reserve;
1015 		}
1016 		em->start = start;
1017 		em->orig_start = em->start;
1018 		ram_size = ins.offset;
1019 		em->len = ins.offset;
1020 		em->mod_start = em->start;
1021 		em->mod_len = em->len;
1022 
1023 		em->block_start = ins.objectid;
1024 		em->block_len = ins.offset;
1025 		em->orig_block_len = ins.offset;
1026 		em->ram_bytes = ram_size;
1027 		em->bdev = fs_info->fs_devices->latest_bdev;
1028 		set_bit(EXTENT_FLAG_PINNED, &em->flags);
1029 		em->generation = -1;
1030 
1031 		while (1) {
1032 			write_lock(&em_tree->lock);
1033 			ret = add_extent_mapping(em_tree, em, 1);
1034 			write_unlock(&em_tree->lock);
1035 			if (ret != -EEXIST) {
1036 				free_extent_map(em);
1037 				break;
1038 			}
1039 			btrfs_drop_extent_cache(inode, start,
1040 						start + ram_size - 1, 0);
1041 		}
1042 		if (ret)
1043 			goto out_reserve;
1044 
1045 		cur_alloc_size = ins.offset;
1046 		ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
1047 					       ram_size, cur_alloc_size, 0);
1048 		if (ret)
1049 			goto out_drop_extent_cache;
1050 
1051 		if (root->root_key.objectid ==
1052 		    BTRFS_DATA_RELOC_TREE_OBJECTID) {
1053 			ret = btrfs_reloc_clone_csums(inode, start,
1054 						      cur_alloc_size);
1055 			if (ret)
1056 				goto out_drop_extent_cache;
1057 		}
1058 
1059 		btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1060 
1061 		if (disk_num_bytes < cur_alloc_size)
1062 			break;
1063 
1064 		/* we're not doing compressed IO, don't unlock the first
1065 		 * page (which the caller expects to stay locked), don't
1066 		 * clear any dirty bits and don't set any writeback bits
1067 		 *
1068 		 * Do set the Private2 bit so we know this page was properly
1069 		 * setup for writepage
1070 		 */
1071 		op = unlock ? PAGE_UNLOCK : 0;
1072 		op |= PAGE_SET_PRIVATE2;
1073 
1074 		extent_clear_unlock_delalloc(inode, start,
1075 					     start + ram_size - 1,
1076 					     delalloc_end, locked_page,
1077 					     EXTENT_LOCKED | EXTENT_DELALLOC,
1078 					     op);
1079 		disk_num_bytes -= cur_alloc_size;
1080 		num_bytes -= cur_alloc_size;
1081 		alloc_hint = ins.objectid + ins.offset;
1082 		start += cur_alloc_size;
1083 	}
1084 out:
1085 	return ret;
1086 
1087 out_drop_extent_cache:
1088 	btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0);
1089 out_reserve:
1090 	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1091 	btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1092 out_unlock:
1093 	extent_clear_unlock_delalloc(inode, start, end, delalloc_end,
1094 				     locked_page,
1095 				     EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
1096 				     EXTENT_DELALLOC | EXTENT_DEFRAG,
1097 				     PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
1098 				     PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK);
1099 	goto out;
1100 }
1101 
1102 /*
1103  * work queue call back to started compression on a file and pages
1104  */
1105 static noinline void async_cow_start(struct btrfs_work *work)
1106 {
1107 	struct async_cow *async_cow;
1108 	int num_added = 0;
1109 	async_cow = container_of(work, struct async_cow, work);
1110 
1111 	compress_file_range(async_cow->inode, async_cow->locked_page,
1112 			    async_cow->start, async_cow->end, async_cow,
1113 			    &num_added);
1114 	if (num_added == 0) {
1115 		btrfs_add_delayed_iput(async_cow->inode);
1116 		async_cow->inode = NULL;
1117 	}
1118 }
1119 
1120 /*
1121  * work queue call back to submit previously compressed pages
1122  */
1123 static noinline void async_cow_submit(struct btrfs_work *work)
1124 {
1125 	struct btrfs_fs_info *fs_info;
1126 	struct async_cow *async_cow;
1127 	struct btrfs_root *root;
1128 	unsigned long nr_pages;
1129 
1130 	async_cow = container_of(work, struct async_cow, work);
1131 
1132 	root = async_cow->root;
1133 	fs_info = root->fs_info;
1134 	nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >>
1135 		PAGE_SHIFT;
1136 
1137 	/*
1138 	 * atomic_sub_return implies a barrier for waitqueue_active
1139 	 */
1140 	if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
1141 	    5 * SZ_1M &&
1142 	    waitqueue_active(&fs_info->async_submit_wait))
1143 		wake_up(&fs_info->async_submit_wait);
1144 
1145 	if (async_cow->inode)
1146 		submit_compressed_extents(async_cow->inode, async_cow);
1147 }
1148 
1149 static noinline void async_cow_free(struct btrfs_work *work)
1150 {
1151 	struct async_cow *async_cow;
1152 	async_cow = container_of(work, struct async_cow, work);
1153 	if (async_cow->inode)
1154 		btrfs_add_delayed_iput(async_cow->inode);
1155 	kfree(async_cow);
1156 }
1157 
1158 static int cow_file_range_async(struct inode *inode, struct page *locked_page,
1159 				u64 start, u64 end, int *page_started,
1160 				unsigned long *nr_written)
1161 {
1162 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1163 	struct async_cow *async_cow;
1164 	struct btrfs_root *root = BTRFS_I(inode)->root;
1165 	unsigned long nr_pages;
1166 	u64 cur_end;
1167 	int limit = 10 * SZ_1M;
1168 
1169 	clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
1170 			 1, 0, NULL, GFP_NOFS);
1171 	while (start < end) {
1172 		async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
1173 		BUG_ON(!async_cow); /* -ENOMEM */
1174 		async_cow->inode = igrab(inode);
1175 		async_cow->root = root;
1176 		async_cow->locked_page = locked_page;
1177 		async_cow->start = start;
1178 
1179 		if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
1180 		    !btrfs_test_opt(fs_info, FORCE_COMPRESS))
1181 			cur_end = end;
1182 		else
1183 			cur_end = min(end, start + SZ_512K - 1);
1184 
1185 		async_cow->end = cur_end;
1186 		INIT_LIST_HEAD(&async_cow->extents);
1187 
1188 		btrfs_init_work(&async_cow->work,
1189 				btrfs_delalloc_helper,
1190 				async_cow_start, async_cow_submit,
1191 				async_cow_free);
1192 
1193 		nr_pages = (cur_end - start + PAGE_SIZE) >>
1194 			PAGE_SHIFT;
1195 		atomic_add(nr_pages, &fs_info->async_delalloc_pages);
1196 
1197 		btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work);
1198 
1199 		if (atomic_read(&fs_info->async_delalloc_pages) > limit) {
1200 			wait_event(fs_info->async_submit_wait,
1201 				   (atomic_read(&fs_info->async_delalloc_pages) <
1202 				    limit));
1203 		}
1204 
1205 		while (atomic_read(&fs_info->async_submit_draining) &&
1206 		       atomic_read(&fs_info->async_delalloc_pages)) {
1207 			wait_event(fs_info->async_submit_wait,
1208 				   (atomic_read(&fs_info->async_delalloc_pages) ==
1209 				    0));
1210 		}
1211 
1212 		*nr_written += nr_pages;
1213 		start = cur_end + 1;
1214 	}
1215 	*page_started = 1;
1216 	return 0;
1217 }
1218 
1219 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info,
1220 					u64 bytenr, u64 num_bytes)
1221 {
1222 	int ret;
1223 	struct btrfs_ordered_sum *sums;
1224 	LIST_HEAD(list);
1225 
1226 	ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr,
1227 				       bytenr + num_bytes - 1, &list, 0);
1228 	if (ret == 0 && list_empty(&list))
1229 		return 0;
1230 
1231 	while (!list_empty(&list)) {
1232 		sums = list_entry(list.next, struct btrfs_ordered_sum, list);
1233 		list_del(&sums->list);
1234 		kfree(sums);
1235 	}
1236 	return 1;
1237 }
1238 
1239 /*
1240  * when nowcow writeback call back.  This checks for snapshots or COW copies
1241  * of the extents that exist in the file, and COWs the file as required.
1242  *
1243  * If no cow copies or snapshots exist, we write directly to the existing
1244  * blocks on disk
1245  */
1246 static noinline int run_delalloc_nocow(struct inode *inode,
1247 				       struct page *locked_page,
1248 			      u64 start, u64 end, int *page_started, int force,
1249 			      unsigned long *nr_written)
1250 {
1251 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1252 	struct btrfs_root *root = BTRFS_I(inode)->root;
1253 	struct btrfs_trans_handle *trans;
1254 	struct extent_buffer *leaf;
1255 	struct btrfs_path *path;
1256 	struct btrfs_file_extent_item *fi;
1257 	struct btrfs_key found_key;
1258 	u64 cow_start;
1259 	u64 cur_offset;
1260 	u64 extent_end;
1261 	u64 extent_offset;
1262 	u64 disk_bytenr;
1263 	u64 num_bytes;
1264 	u64 disk_num_bytes;
1265 	u64 ram_bytes;
1266 	int extent_type;
1267 	int ret, err;
1268 	int type;
1269 	int nocow;
1270 	int check_prev = 1;
1271 	bool nolock;
1272 	u64 ino = btrfs_ino(inode);
1273 
1274 	path = btrfs_alloc_path();
1275 	if (!path) {
1276 		extent_clear_unlock_delalloc(inode, start, end, end,
1277 					     locked_page,
1278 					     EXTENT_LOCKED | EXTENT_DELALLOC |
1279 					     EXTENT_DO_ACCOUNTING |
1280 					     EXTENT_DEFRAG, PAGE_UNLOCK |
1281 					     PAGE_CLEAR_DIRTY |
1282 					     PAGE_SET_WRITEBACK |
1283 					     PAGE_END_WRITEBACK);
1284 		return -ENOMEM;
1285 	}
1286 
1287 	nolock = btrfs_is_free_space_inode(inode);
1288 
1289 	if (nolock)
1290 		trans = btrfs_join_transaction_nolock(root);
1291 	else
1292 		trans = btrfs_join_transaction(root);
1293 
1294 	if (IS_ERR(trans)) {
1295 		extent_clear_unlock_delalloc(inode, start, end, end,
1296 					     locked_page,
1297 					     EXTENT_LOCKED | EXTENT_DELALLOC |
1298 					     EXTENT_DO_ACCOUNTING |
1299 					     EXTENT_DEFRAG, PAGE_UNLOCK |
1300 					     PAGE_CLEAR_DIRTY |
1301 					     PAGE_SET_WRITEBACK |
1302 					     PAGE_END_WRITEBACK);
1303 		btrfs_free_path(path);
1304 		return PTR_ERR(trans);
1305 	}
1306 
1307 	trans->block_rsv = &fs_info->delalloc_block_rsv;
1308 
1309 	cow_start = (u64)-1;
1310 	cur_offset = start;
1311 	while (1) {
1312 		ret = btrfs_lookup_file_extent(trans, root, path, ino,
1313 					       cur_offset, 0);
1314 		if (ret < 0)
1315 			goto error;
1316 		if (ret > 0 && path->slots[0] > 0 && check_prev) {
1317 			leaf = path->nodes[0];
1318 			btrfs_item_key_to_cpu(leaf, &found_key,
1319 					      path->slots[0] - 1);
1320 			if (found_key.objectid == ino &&
1321 			    found_key.type == BTRFS_EXTENT_DATA_KEY)
1322 				path->slots[0]--;
1323 		}
1324 		check_prev = 0;
1325 next_slot:
1326 		leaf = path->nodes[0];
1327 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1328 			ret = btrfs_next_leaf(root, path);
1329 			if (ret < 0)
1330 				goto error;
1331 			if (ret > 0)
1332 				break;
1333 			leaf = path->nodes[0];
1334 		}
1335 
1336 		nocow = 0;
1337 		disk_bytenr = 0;
1338 		num_bytes = 0;
1339 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1340 
1341 		if (found_key.objectid > ino)
1342 			break;
1343 		if (WARN_ON_ONCE(found_key.objectid < ino) ||
1344 		    found_key.type < BTRFS_EXTENT_DATA_KEY) {
1345 			path->slots[0]++;
1346 			goto next_slot;
1347 		}
1348 		if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
1349 		    found_key.offset > end)
1350 			break;
1351 
1352 		if (found_key.offset > cur_offset) {
1353 			extent_end = found_key.offset;
1354 			extent_type = 0;
1355 			goto out_check;
1356 		}
1357 
1358 		fi = btrfs_item_ptr(leaf, path->slots[0],
1359 				    struct btrfs_file_extent_item);
1360 		extent_type = btrfs_file_extent_type(leaf, fi);
1361 
1362 		ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1363 		if (extent_type == BTRFS_FILE_EXTENT_REG ||
1364 		    extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1365 			disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1366 			extent_offset = btrfs_file_extent_offset(leaf, fi);
1367 			extent_end = found_key.offset +
1368 				btrfs_file_extent_num_bytes(leaf, fi);
1369 			disk_num_bytes =
1370 				btrfs_file_extent_disk_num_bytes(leaf, fi);
1371 			if (extent_end <= start) {
1372 				path->slots[0]++;
1373 				goto next_slot;
1374 			}
1375 			if (disk_bytenr == 0)
1376 				goto out_check;
1377 			if (btrfs_file_extent_compression(leaf, fi) ||
1378 			    btrfs_file_extent_encryption(leaf, fi) ||
1379 			    btrfs_file_extent_other_encoding(leaf, fi))
1380 				goto out_check;
1381 			if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
1382 				goto out_check;
1383 			if (btrfs_extent_readonly(fs_info, disk_bytenr))
1384 				goto out_check;
1385 			if (btrfs_cross_ref_exist(trans, root, ino,
1386 						  found_key.offset -
1387 						  extent_offset, disk_bytenr))
1388 				goto out_check;
1389 			disk_bytenr += extent_offset;
1390 			disk_bytenr += cur_offset - found_key.offset;
1391 			num_bytes = min(end + 1, extent_end) - cur_offset;
1392 			/*
1393 			 * if there are pending snapshots for this root,
1394 			 * we fall into common COW way.
1395 			 */
1396 			if (!nolock) {
1397 				err = btrfs_start_write_no_snapshoting(root);
1398 				if (!err)
1399 					goto out_check;
1400 			}
1401 			/*
1402 			 * force cow if csum exists in the range.
1403 			 * this ensure that csum for a given extent are
1404 			 * either valid or do not exist.
1405 			 */
1406 			if (csum_exist_in_range(fs_info, disk_bytenr,
1407 						num_bytes))
1408 				goto out_check;
1409 			if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr))
1410 				goto out_check;
1411 			nocow = 1;
1412 		} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1413 			extent_end = found_key.offset +
1414 				btrfs_file_extent_inline_len(leaf,
1415 						     path->slots[0], fi);
1416 			extent_end = ALIGN(extent_end,
1417 					   fs_info->sectorsize);
1418 		} else {
1419 			BUG_ON(1);
1420 		}
1421 out_check:
1422 		if (extent_end <= start) {
1423 			path->slots[0]++;
1424 			if (!nolock && nocow)
1425 				btrfs_end_write_no_snapshoting(root);
1426 			if (nocow)
1427 				btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1428 			goto next_slot;
1429 		}
1430 		if (!nocow) {
1431 			if (cow_start == (u64)-1)
1432 				cow_start = cur_offset;
1433 			cur_offset = extent_end;
1434 			if (cur_offset > end)
1435 				break;
1436 			path->slots[0]++;
1437 			goto next_slot;
1438 		}
1439 
1440 		btrfs_release_path(path);
1441 		if (cow_start != (u64)-1) {
1442 			ret = cow_file_range(inode, locked_page,
1443 					     cow_start, found_key.offset - 1,
1444 					     end, page_started, nr_written, 1,
1445 					     NULL);
1446 			if (ret) {
1447 				if (!nolock && nocow)
1448 					btrfs_end_write_no_snapshoting(root);
1449 				if (nocow)
1450 					btrfs_dec_nocow_writers(fs_info,
1451 								disk_bytenr);
1452 				goto error;
1453 			}
1454 			cow_start = (u64)-1;
1455 		}
1456 
1457 		if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1458 			struct extent_map *em;
1459 			struct extent_map_tree *em_tree;
1460 			em_tree = &BTRFS_I(inode)->extent_tree;
1461 			em = alloc_extent_map();
1462 			BUG_ON(!em); /* -ENOMEM */
1463 			em->start = cur_offset;
1464 			em->orig_start = found_key.offset - extent_offset;
1465 			em->len = num_bytes;
1466 			em->block_len = num_bytes;
1467 			em->block_start = disk_bytenr;
1468 			em->orig_block_len = disk_num_bytes;
1469 			em->ram_bytes = ram_bytes;
1470 			em->bdev = fs_info->fs_devices->latest_bdev;
1471 			em->mod_start = em->start;
1472 			em->mod_len = em->len;
1473 			set_bit(EXTENT_FLAG_PINNED, &em->flags);
1474 			set_bit(EXTENT_FLAG_FILLING, &em->flags);
1475 			em->generation = -1;
1476 			while (1) {
1477 				write_lock(&em_tree->lock);
1478 				ret = add_extent_mapping(em_tree, em, 1);
1479 				write_unlock(&em_tree->lock);
1480 				if (ret != -EEXIST) {
1481 					free_extent_map(em);
1482 					break;
1483 				}
1484 				btrfs_drop_extent_cache(inode, em->start,
1485 						em->start + em->len - 1, 0);
1486 			}
1487 			type = BTRFS_ORDERED_PREALLOC;
1488 		} else {
1489 			type = BTRFS_ORDERED_NOCOW;
1490 		}
1491 
1492 		ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
1493 					       num_bytes, num_bytes, type);
1494 		if (nocow)
1495 			btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1496 		BUG_ON(ret); /* -ENOMEM */
1497 
1498 		if (root->root_key.objectid ==
1499 		    BTRFS_DATA_RELOC_TREE_OBJECTID) {
1500 			ret = btrfs_reloc_clone_csums(inode, cur_offset,
1501 						      num_bytes);
1502 			if (ret) {
1503 				if (!nolock && nocow)
1504 					btrfs_end_write_no_snapshoting(root);
1505 				goto error;
1506 			}
1507 		}
1508 
1509 		extent_clear_unlock_delalloc(inode, cur_offset,
1510 					     cur_offset + num_bytes - 1, end,
1511 					     locked_page, EXTENT_LOCKED |
1512 					     EXTENT_DELALLOC |
1513 					     EXTENT_CLEAR_DATA_RESV,
1514 					     PAGE_UNLOCK | PAGE_SET_PRIVATE2);
1515 
1516 		if (!nolock && nocow)
1517 			btrfs_end_write_no_snapshoting(root);
1518 		cur_offset = extent_end;
1519 		if (cur_offset > end)
1520 			break;
1521 	}
1522 	btrfs_release_path(path);
1523 
1524 	if (cur_offset <= end && cow_start == (u64)-1) {
1525 		cow_start = cur_offset;
1526 		cur_offset = end;
1527 	}
1528 
1529 	if (cow_start != (u64)-1) {
1530 		ret = cow_file_range(inode, locked_page, cow_start, end, end,
1531 				     page_started, nr_written, 1, NULL);
1532 		if (ret)
1533 			goto error;
1534 	}
1535 
1536 error:
1537 	err = btrfs_end_transaction(trans);
1538 	if (!ret)
1539 		ret = err;
1540 
1541 	if (ret && cur_offset < end)
1542 		extent_clear_unlock_delalloc(inode, cur_offset, end, end,
1543 					     locked_page, EXTENT_LOCKED |
1544 					     EXTENT_DELALLOC | EXTENT_DEFRAG |
1545 					     EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
1546 					     PAGE_CLEAR_DIRTY |
1547 					     PAGE_SET_WRITEBACK |
1548 					     PAGE_END_WRITEBACK);
1549 	btrfs_free_path(path);
1550 	return ret;
1551 }
1552 
1553 static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
1554 {
1555 
1556 	if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
1557 	    !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
1558 		return 0;
1559 
1560 	/*
1561 	 * @defrag_bytes is a hint value, no spinlock held here,
1562 	 * if is not zero, it means the file is defragging.
1563 	 * Force cow if given extent needs to be defragged.
1564 	 */
1565 	if (BTRFS_I(inode)->defrag_bytes &&
1566 	    test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
1567 			   EXTENT_DEFRAG, 0, NULL))
1568 		return 1;
1569 
1570 	return 0;
1571 }
1572 
1573 /*
1574  * extent_io.c call back to do delayed allocation processing
1575  */
1576 static int run_delalloc_range(struct inode *inode, struct page *locked_page,
1577 			      u64 start, u64 end, int *page_started,
1578 			      unsigned long *nr_written)
1579 {
1580 	int ret;
1581 	int force_cow = need_force_cow(inode, start, end);
1582 
1583 	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
1584 		ret = run_delalloc_nocow(inode, locked_page, start, end,
1585 					 page_started, 1, nr_written);
1586 	} else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
1587 		ret = run_delalloc_nocow(inode, locked_page, start, end,
1588 					 page_started, 0, nr_written);
1589 	} else if (!inode_need_compress(inode)) {
1590 		ret = cow_file_range(inode, locked_page, start, end, end,
1591 				      page_started, nr_written, 1, NULL);
1592 	} else {
1593 		set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
1594 			&BTRFS_I(inode)->runtime_flags);
1595 		ret = cow_file_range_async(inode, locked_page, start, end,
1596 					   page_started, nr_written);
1597 	}
1598 	return ret;
1599 }
1600 
1601 static void btrfs_split_extent_hook(struct inode *inode,
1602 				    struct extent_state *orig, u64 split)
1603 {
1604 	u64 size;
1605 
1606 	/* not delalloc, ignore it */
1607 	if (!(orig->state & EXTENT_DELALLOC))
1608 		return;
1609 
1610 	size = orig->end - orig->start + 1;
1611 	if (size > BTRFS_MAX_EXTENT_SIZE) {
1612 		u64 num_extents;
1613 		u64 new_size;
1614 
1615 		/*
1616 		 * See the explanation in btrfs_merge_extent_hook, the same
1617 		 * applies here, just in reverse.
1618 		 */
1619 		new_size = orig->end - split + 1;
1620 		num_extents = div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
1621 					BTRFS_MAX_EXTENT_SIZE);
1622 		new_size = split - orig->start;
1623 		num_extents += div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
1624 					BTRFS_MAX_EXTENT_SIZE);
1625 		if (div64_u64(size + BTRFS_MAX_EXTENT_SIZE - 1,
1626 			      BTRFS_MAX_EXTENT_SIZE) >= num_extents)
1627 			return;
1628 	}
1629 
1630 	spin_lock(&BTRFS_I(inode)->lock);
1631 	BTRFS_I(inode)->outstanding_extents++;
1632 	spin_unlock(&BTRFS_I(inode)->lock);
1633 }
1634 
1635 /*
1636  * extent_io.c merge_extent_hook, used to track merged delayed allocation
1637  * extents so we can keep track of new extents that are just merged onto old
1638  * extents, such as when we are doing sequential writes, so we can properly
1639  * account for the metadata space we'll need.
1640  */
1641 static void btrfs_merge_extent_hook(struct inode *inode,
1642 				    struct extent_state *new,
1643 				    struct extent_state *other)
1644 {
1645 	u64 new_size, old_size;
1646 	u64 num_extents;
1647 
1648 	/* not delalloc, ignore it */
1649 	if (!(other->state & EXTENT_DELALLOC))
1650 		return;
1651 
1652 	if (new->start > other->start)
1653 		new_size = new->end - other->start + 1;
1654 	else
1655 		new_size = other->end - new->start + 1;
1656 
1657 	/* we're not bigger than the max, unreserve the space and go */
1658 	if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
1659 		spin_lock(&BTRFS_I(inode)->lock);
1660 		BTRFS_I(inode)->outstanding_extents--;
1661 		spin_unlock(&BTRFS_I(inode)->lock);
1662 		return;
1663 	}
1664 
1665 	/*
1666 	 * We have to add up either side to figure out how many extents were
1667 	 * accounted for before we merged into one big extent.  If the number of
1668 	 * extents we accounted for is <= the amount we need for the new range
1669 	 * then we can return, otherwise drop.  Think of it like this
1670 	 *
1671 	 * [ 4k][MAX_SIZE]
1672 	 *
1673 	 * So we've grown the extent by a MAX_SIZE extent, this would mean we
1674 	 * need 2 outstanding extents, on one side we have 1 and the other side
1675 	 * we have 1 so they are == and we can return.  But in this case
1676 	 *
1677 	 * [MAX_SIZE+4k][MAX_SIZE+4k]
1678 	 *
1679 	 * Each range on their own accounts for 2 extents, but merged together
1680 	 * they are only 3 extents worth of accounting, so we need to drop in
1681 	 * this case.
1682 	 */
1683 	old_size = other->end - other->start + 1;
1684 	num_extents = div64_u64(old_size + BTRFS_MAX_EXTENT_SIZE - 1,
1685 				BTRFS_MAX_EXTENT_SIZE);
1686 	old_size = new->end - new->start + 1;
1687 	num_extents += div64_u64(old_size + BTRFS_MAX_EXTENT_SIZE - 1,
1688 				 BTRFS_MAX_EXTENT_SIZE);
1689 
1690 	if (div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
1691 		      BTRFS_MAX_EXTENT_SIZE) >= num_extents)
1692 		return;
1693 
1694 	spin_lock(&BTRFS_I(inode)->lock);
1695 	BTRFS_I(inode)->outstanding_extents--;
1696 	spin_unlock(&BTRFS_I(inode)->lock);
1697 }
1698 
1699 static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
1700 				      struct inode *inode)
1701 {
1702 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1703 
1704 	spin_lock(&root->delalloc_lock);
1705 	if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1706 		list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
1707 			      &root->delalloc_inodes);
1708 		set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1709 			&BTRFS_I(inode)->runtime_flags);
1710 		root->nr_delalloc_inodes++;
1711 		if (root->nr_delalloc_inodes == 1) {
1712 			spin_lock(&fs_info->delalloc_root_lock);
1713 			BUG_ON(!list_empty(&root->delalloc_root));
1714 			list_add_tail(&root->delalloc_root,
1715 				      &fs_info->delalloc_roots);
1716 			spin_unlock(&fs_info->delalloc_root_lock);
1717 		}
1718 	}
1719 	spin_unlock(&root->delalloc_lock);
1720 }
1721 
1722 static void btrfs_del_delalloc_inode(struct btrfs_root *root,
1723 				     struct inode *inode)
1724 {
1725 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1726 
1727 	spin_lock(&root->delalloc_lock);
1728 	if (!list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1729 		list_del_init(&BTRFS_I(inode)->delalloc_inodes);
1730 		clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1731 			  &BTRFS_I(inode)->runtime_flags);
1732 		root->nr_delalloc_inodes--;
1733 		if (!root->nr_delalloc_inodes) {
1734 			spin_lock(&fs_info->delalloc_root_lock);
1735 			BUG_ON(list_empty(&root->delalloc_root));
1736 			list_del_init(&root->delalloc_root);
1737 			spin_unlock(&fs_info->delalloc_root_lock);
1738 		}
1739 	}
1740 	spin_unlock(&root->delalloc_lock);
1741 }
1742 
1743 /*
1744  * extent_io.c set_bit_hook, used to track delayed allocation
1745  * bytes in this file, and to maintain the list of inodes that
1746  * have pending delalloc work to be done.
1747  */
1748 static void btrfs_set_bit_hook(struct inode *inode,
1749 			       struct extent_state *state, unsigned *bits)
1750 {
1751 
1752 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1753 
1754 	if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
1755 		WARN_ON(1);
1756 	/*
1757 	 * set_bit and clear bit hooks normally require _irqsave/restore
1758 	 * but in this case, we are only testing for the DELALLOC
1759 	 * bit, which is only set or cleared with irqs on
1760 	 */
1761 	if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1762 		struct btrfs_root *root = BTRFS_I(inode)->root;
1763 		u64 len = state->end + 1 - state->start;
1764 		bool do_list = !btrfs_is_free_space_inode(inode);
1765 
1766 		if (*bits & EXTENT_FIRST_DELALLOC) {
1767 			*bits &= ~EXTENT_FIRST_DELALLOC;
1768 		} else {
1769 			spin_lock(&BTRFS_I(inode)->lock);
1770 			BTRFS_I(inode)->outstanding_extents++;
1771 			spin_unlock(&BTRFS_I(inode)->lock);
1772 		}
1773 
1774 		/* For sanity tests */
1775 		if (btrfs_is_testing(fs_info))
1776 			return;
1777 
1778 		__percpu_counter_add(&fs_info->delalloc_bytes, len,
1779 				     fs_info->delalloc_batch);
1780 		spin_lock(&BTRFS_I(inode)->lock);
1781 		BTRFS_I(inode)->delalloc_bytes += len;
1782 		if (*bits & EXTENT_DEFRAG)
1783 			BTRFS_I(inode)->defrag_bytes += len;
1784 		if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1785 					 &BTRFS_I(inode)->runtime_flags))
1786 			btrfs_add_delalloc_inodes(root, inode);
1787 		spin_unlock(&BTRFS_I(inode)->lock);
1788 	}
1789 }
1790 
1791 /*
1792  * extent_io.c clear_bit_hook, see set_bit_hook for why
1793  */
1794 static void btrfs_clear_bit_hook(struct inode *inode,
1795 				 struct extent_state *state,
1796 				 unsigned *bits)
1797 {
1798 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1799 	u64 len = state->end + 1 - state->start;
1800 	u64 num_extents = div64_u64(len + BTRFS_MAX_EXTENT_SIZE -1,
1801 				    BTRFS_MAX_EXTENT_SIZE);
1802 
1803 	spin_lock(&BTRFS_I(inode)->lock);
1804 	if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG))
1805 		BTRFS_I(inode)->defrag_bytes -= len;
1806 	spin_unlock(&BTRFS_I(inode)->lock);
1807 
1808 	/*
1809 	 * set_bit and clear bit hooks normally require _irqsave/restore
1810 	 * but in this case, we are only testing for the DELALLOC
1811 	 * bit, which is only set or cleared with irqs on
1812 	 */
1813 	if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1814 		struct btrfs_root *root = BTRFS_I(inode)->root;
1815 		bool do_list = !btrfs_is_free_space_inode(inode);
1816 
1817 		if (*bits & EXTENT_FIRST_DELALLOC) {
1818 			*bits &= ~EXTENT_FIRST_DELALLOC;
1819 		} else if (!(*bits & EXTENT_DO_ACCOUNTING)) {
1820 			spin_lock(&BTRFS_I(inode)->lock);
1821 			BTRFS_I(inode)->outstanding_extents -= num_extents;
1822 			spin_unlock(&BTRFS_I(inode)->lock);
1823 		}
1824 
1825 		/*
1826 		 * We don't reserve metadata space for space cache inodes so we
1827 		 * don't need to call dellalloc_release_metadata if there is an
1828 		 * error.
1829 		 */
1830 		if (*bits & EXTENT_DO_ACCOUNTING &&
1831 		    root != fs_info->tree_root)
1832 			btrfs_delalloc_release_metadata(inode, len);
1833 
1834 		/* For sanity tests. */
1835 		if (btrfs_is_testing(fs_info))
1836 			return;
1837 
1838 		if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
1839 		    && do_list && !(state->state & EXTENT_NORESERVE)
1840 		    && (*bits & (EXTENT_DO_ACCOUNTING |
1841 		    EXTENT_CLEAR_DATA_RESV)))
1842 			btrfs_free_reserved_data_space_noquota(inode,
1843 					state->start, len);
1844 
1845 		__percpu_counter_add(&fs_info->delalloc_bytes, -len,
1846 				     fs_info->delalloc_batch);
1847 		spin_lock(&BTRFS_I(inode)->lock);
1848 		BTRFS_I(inode)->delalloc_bytes -= len;
1849 		if (do_list && BTRFS_I(inode)->delalloc_bytes == 0 &&
1850 		    test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1851 			     &BTRFS_I(inode)->runtime_flags))
1852 			btrfs_del_delalloc_inode(root, inode);
1853 		spin_unlock(&BTRFS_I(inode)->lock);
1854 	}
1855 }
1856 
1857 /*
1858  * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
1859  * we don't create bios that span stripes or chunks
1860  *
1861  * return 1 if page cannot be merged to bio
1862  * return 0 if page can be merged to bio
1863  * return error otherwise
1864  */
1865 int btrfs_merge_bio_hook(struct page *page, unsigned long offset,
1866 			 size_t size, struct bio *bio,
1867 			 unsigned long bio_flags)
1868 {
1869 	struct inode *inode = page->mapping->host;
1870 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1871 	u64 logical = (u64)bio->bi_iter.bi_sector << 9;
1872 	u64 length = 0;
1873 	u64 map_length;
1874 	int ret;
1875 
1876 	if (bio_flags & EXTENT_BIO_COMPRESSED)
1877 		return 0;
1878 
1879 	length = bio->bi_iter.bi_size;
1880 	map_length = length;
1881 	ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length,
1882 			      NULL, 0);
1883 	if (ret < 0)
1884 		return ret;
1885 	if (map_length < length + size)
1886 		return 1;
1887 	return 0;
1888 }
1889 
1890 /*
1891  * in order to insert checksums into the metadata in large chunks,
1892  * we wait until bio submission time.   All the pages in the bio are
1893  * checksummed and sums are attached onto the ordered extent record.
1894  *
1895  * At IO completion time the cums attached on the ordered extent record
1896  * are inserted into the btree
1897  */
1898 static int __btrfs_submit_bio_start(struct inode *inode, struct bio *bio,
1899 				    int mirror_num, unsigned long bio_flags,
1900 				    u64 bio_offset)
1901 {
1902 	int ret = 0;
1903 
1904 	ret = btrfs_csum_one_bio(inode, bio, 0, 0);
1905 	BUG_ON(ret); /* -ENOMEM */
1906 	return 0;
1907 }
1908 
1909 /*
1910  * in order to insert checksums into the metadata in large chunks,
1911  * we wait until bio submission time.   All the pages in the bio are
1912  * checksummed and sums are attached onto the ordered extent record.
1913  *
1914  * At IO completion time the cums attached on the ordered extent record
1915  * are inserted into the btree
1916  */
1917 static int __btrfs_submit_bio_done(struct inode *inode, struct bio *bio,
1918 			  int mirror_num, unsigned long bio_flags,
1919 			  u64 bio_offset)
1920 {
1921 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1922 	int ret;
1923 
1924 	ret = btrfs_map_bio(fs_info, bio, mirror_num, 1);
1925 	if (ret) {
1926 		bio->bi_error = ret;
1927 		bio_endio(bio);
1928 	}
1929 	return ret;
1930 }
1931 
1932 /*
1933  * extent_io.c submission hook. This does the right thing for csum calculation
1934  * on write, or reading the csums from the tree before a read
1935  */
1936 static int btrfs_submit_bio_hook(struct inode *inode, struct bio *bio,
1937 			  int mirror_num, unsigned long bio_flags,
1938 			  u64 bio_offset)
1939 {
1940 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1941 	struct btrfs_root *root = BTRFS_I(inode)->root;
1942 	enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
1943 	int ret = 0;
1944 	int skip_sum;
1945 	int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
1946 
1947 	skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
1948 
1949 	if (btrfs_is_free_space_inode(inode))
1950 		metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
1951 
1952 	if (bio_op(bio) != REQ_OP_WRITE) {
1953 		ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);
1954 		if (ret)
1955 			goto out;
1956 
1957 		if (bio_flags & EXTENT_BIO_COMPRESSED) {
1958 			ret = btrfs_submit_compressed_read(inode, bio,
1959 							   mirror_num,
1960 							   bio_flags);
1961 			goto out;
1962 		} else if (!skip_sum) {
1963 			ret = btrfs_lookup_bio_sums(inode, bio, NULL);
1964 			if (ret)
1965 				goto out;
1966 		}
1967 		goto mapit;
1968 	} else if (async && !skip_sum) {
1969 		/* csum items have already been cloned */
1970 		if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
1971 			goto mapit;
1972 		/* we're doing a write, do the async checksumming */
1973 		ret = btrfs_wq_submit_bio(fs_info, inode, bio, mirror_num,
1974 					  bio_flags, bio_offset,
1975 					  __btrfs_submit_bio_start,
1976 					  __btrfs_submit_bio_done);
1977 		goto out;
1978 	} else if (!skip_sum) {
1979 		ret = btrfs_csum_one_bio(inode, bio, 0, 0);
1980 		if (ret)
1981 			goto out;
1982 	}
1983 
1984 mapit:
1985 	ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
1986 
1987 out:
1988 	if (ret < 0) {
1989 		bio->bi_error = ret;
1990 		bio_endio(bio);
1991 	}
1992 	return ret;
1993 }
1994 
1995 /*
1996  * given a list of ordered sums record them in the inode.  This happens
1997  * at IO completion time based on sums calculated at bio submission time.
1998  */
1999 static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
2000 			     struct inode *inode, u64 file_offset,
2001 			     struct list_head *list)
2002 {
2003 	struct btrfs_ordered_sum *sum;
2004 
2005 	list_for_each_entry(sum, list, list) {
2006 		trans->adding_csums = 1;
2007 		btrfs_csum_file_blocks(trans,
2008 		       BTRFS_I(inode)->root->fs_info->csum_root, sum);
2009 		trans->adding_csums = 0;
2010 	}
2011 	return 0;
2012 }
2013 
2014 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
2015 			      struct extent_state **cached_state, int dedupe)
2016 {
2017 	WARN_ON((end & (PAGE_SIZE - 1)) == 0);
2018 	return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
2019 				   cached_state);
2020 }
2021 
2022 /* see btrfs_writepage_start_hook for details on why this is required */
2023 struct btrfs_writepage_fixup {
2024 	struct page *page;
2025 	struct btrfs_work work;
2026 };
2027 
2028 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2029 {
2030 	struct btrfs_writepage_fixup *fixup;
2031 	struct btrfs_ordered_extent *ordered;
2032 	struct extent_state *cached_state = NULL;
2033 	struct page *page;
2034 	struct inode *inode;
2035 	u64 page_start;
2036 	u64 page_end;
2037 	int ret;
2038 
2039 	fixup = container_of(work, struct btrfs_writepage_fixup, work);
2040 	page = fixup->page;
2041 again:
2042 	lock_page(page);
2043 	if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
2044 		ClearPageChecked(page);
2045 		goto out_page;
2046 	}
2047 
2048 	inode = page->mapping->host;
2049 	page_start = page_offset(page);
2050 	page_end = page_offset(page) + PAGE_SIZE - 1;
2051 
2052 	lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end,
2053 			 &cached_state);
2054 
2055 	/* already ordered? We're done */
2056 	if (PagePrivate2(page))
2057 		goto out;
2058 
2059 	ordered = btrfs_lookup_ordered_range(inode, page_start,
2060 					PAGE_SIZE);
2061 	if (ordered) {
2062 		unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
2063 				     page_end, &cached_state, GFP_NOFS);
2064 		unlock_page(page);
2065 		btrfs_start_ordered_extent(inode, ordered, 1);
2066 		btrfs_put_ordered_extent(ordered);
2067 		goto again;
2068 	}
2069 
2070 	ret = btrfs_delalloc_reserve_space(inode, page_start,
2071 					   PAGE_SIZE);
2072 	if (ret) {
2073 		mapping_set_error(page->mapping, ret);
2074 		end_extent_writepage(page, ret, page_start, page_end);
2075 		ClearPageChecked(page);
2076 		goto out;
2077 	 }
2078 
2079 	btrfs_set_extent_delalloc(inode, page_start, page_end, &cached_state,
2080 				  0);
2081 	ClearPageChecked(page);
2082 	set_page_dirty(page);
2083 out:
2084 	unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
2085 			     &cached_state, GFP_NOFS);
2086 out_page:
2087 	unlock_page(page);
2088 	put_page(page);
2089 	kfree(fixup);
2090 }
2091 
2092 /*
2093  * There are a few paths in the higher layers of the kernel that directly
2094  * set the page dirty bit without asking the filesystem if it is a
2095  * good idea.  This causes problems because we want to make sure COW
2096  * properly happens and the data=ordered rules are followed.
2097  *
2098  * In our case any range that doesn't have the ORDERED bit set
2099  * hasn't been properly setup for IO.  We kick off an async process
2100  * to fix it up.  The async helper will wait for ordered extents, set
2101  * the delalloc bit and make it safe to write the page.
2102  */
2103 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
2104 {
2105 	struct inode *inode = page->mapping->host;
2106 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2107 	struct btrfs_writepage_fixup *fixup;
2108 
2109 	/* this page is properly in the ordered list */
2110 	if (TestClearPagePrivate2(page))
2111 		return 0;
2112 
2113 	if (PageChecked(page))
2114 		return -EAGAIN;
2115 
2116 	fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2117 	if (!fixup)
2118 		return -EAGAIN;
2119 
2120 	SetPageChecked(page);
2121 	get_page(page);
2122 	btrfs_init_work(&fixup->work, btrfs_fixup_helper,
2123 			btrfs_writepage_fixup_worker, NULL, NULL);
2124 	fixup->page = page;
2125 	btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
2126 	return -EBUSY;
2127 }
2128 
2129 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2130 				       struct inode *inode, u64 file_pos,
2131 				       u64 disk_bytenr, u64 disk_num_bytes,
2132 				       u64 num_bytes, u64 ram_bytes,
2133 				       u8 compression, u8 encryption,
2134 				       u16 other_encoding, int extent_type)
2135 {
2136 	struct btrfs_root *root = BTRFS_I(inode)->root;
2137 	struct btrfs_file_extent_item *fi;
2138 	struct btrfs_path *path;
2139 	struct extent_buffer *leaf;
2140 	struct btrfs_key ins;
2141 	int extent_inserted = 0;
2142 	int ret;
2143 
2144 	path = btrfs_alloc_path();
2145 	if (!path)
2146 		return -ENOMEM;
2147 
2148 	/*
2149 	 * we may be replacing one extent in the tree with another.
2150 	 * The new extent is pinned in the extent map, and we don't want
2151 	 * to drop it from the cache until it is completely in the btree.
2152 	 *
2153 	 * So, tell btrfs_drop_extents to leave this extent in the cache.
2154 	 * the caller is expected to unpin it and allow it to be merged
2155 	 * with the others.
2156 	 */
2157 	ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
2158 				   file_pos + num_bytes, NULL, 0,
2159 				   1, sizeof(*fi), &extent_inserted);
2160 	if (ret)
2161 		goto out;
2162 
2163 	if (!extent_inserted) {
2164 		ins.objectid = btrfs_ino(inode);
2165 		ins.offset = file_pos;
2166 		ins.type = BTRFS_EXTENT_DATA_KEY;
2167 
2168 		path->leave_spinning = 1;
2169 		ret = btrfs_insert_empty_item(trans, root, path, &ins,
2170 					      sizeof(*fi));
2171 		if (ret)
2172 			goto out;
2173 	}
2174 	leaf = path->nodes[0];
2175 	fi = btrfs_item_ptr(leaf, path->slots[0],
2176 			    struct btrfs_file_extent_item);
2177 	btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2178 	btrfs_set_file_extent_type(leaf, fi, extent_type);
2179 	btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
2180 	btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
2181 	btrfs_set_file_extent_offset(leaf, fi, 0);
2182 	btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2183 	btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
2184 	btrfs_set_file_extent_compression(leaf, fi, compression);
2185 	btrfs_set_file_extent_encryption(leaf, fi, encryption);
2186 	btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
2187 
2188 	btrfs_mark_buffer_dirty(leaf);
2189 	btrfs_release_path(path);
2190 
2191 	inode_add_bytes(inode, num_bytes);
2192 
2193 	ins.objectid = disk_bytenr;
2194 	ins.offset = disk_num_bytes;
2195 	ins.type = BTRFS_EXTENT_ITEM_KEY;
2196 	ret = btrfs_alloc_reserved_file_extent(trans, root->root_key.objectid,
2197 					       btrfs_ino(inode), file_pos,
2198 					       ram_bytes, &ins);
2199 	/*
2200 	 * Release the reserved range from inode dirty range map, as it is
2201 	 * already moved into delayed_ref_head
2202 	 */
2203 	btrfs_qgroup_release_data(inode, file_pos, ram_bytes);
2204 out:
2205 	btrfs_free_path(path);
2206 
2207 	return ret;
2208 }
2209 
2210 /* snapshot-aware defrag */
2211 struct sa_defrag_extent_backref {
2212 	struct rb_node node;
2213 	struct old_sa_defrag_extent *old;
2214 	u64 root_id;
2215 	u64 inum;
2216 	u64 file_pos;
2217 	u64 extent_offset;
2218 	u64 num_bytes;
2219 	u64 generation;
2220 };
2221 
2222 struct old_sa_defrag_extent {
2223 	struct list_head list;
2224 	struct new_sa_defrag_extent *new;
2225 
2226 	u64 extent_offset;
2227 	u64 bytenr;
2228 	u64 offset;
2229 	u64 len;
2230 	int count;
2231 };
2232 
2233 struct new_sa_defrag_extent {
2234 	struct rb_root root;
2235 	struct list_head head;
2236 	struct btrfs_path *path;
2237 	struct inode *inode;
2238 	u64 file_pos;
2239 	u64 len;
2240 	u64 bytenr;
2241 	u64 disk_len;
2242 	u8 compress_type;
2243 };
2244 
2245 static int backref_comp(struct sa_defrag_extent_backref *b1,
2246 			struct sa_defrag_extent_backref *b2)
2247 {
2248 	if (b1->root_id < b2->root_id)
2249 		return -1;
2250 	else if (b1->root_id > b2->root_id)
2251 		return 1;
2252 
2253 	if (b1->inum < b2->inum)
2254 		return -1;
2255 	else if (b1->inum > b2->inum)
2256 		return 1;
2257 
2258 	if (b1->file_pos < b2->file_pos)
2259 		return -1;
2260 	else if (b1->file_pos > b2->file_pos)
2261 		return 1;
2262 
2263 	/*
2264 	 * [------------------------------] ===> (a range of space)
2265 	 *     |<--->|   |<---->| =============> (fs/file tree A)
2266 	 * |<---------------------------->| ===> (fs/file tree B)
2267 	 *
2268 	 * A range of space can refer to two file extents in one tree while
2269 	 * refer to only one file extent in another tree.
2270 	 *
2271 	 * So we may process a disk offset more than one time(two extents in A)
2272 	 * and locate at the same extent(one extent in B), then insert two same
2273 	 * backrefs(both refer to the extent in B).
2274 	 */
2275 	return 0;
2276 }
2277 
2278 static void backref_insert(struct rb_root *root,
2279 			   struct sa_defrag_extent_backref *backref)
2280 {
2281 	struct rb_node **p = &root->rb_node;
2282 	struct rb_node *parent = NULL;
2283 	struct sa_defrag_extent_backref *entry;
2284 	int ret;
2285 
2286 	while (*p) {
2287 		parent = *p;
2288 		entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
2289 
2290 		ret = backref_comp(backref, entry);
2291 		if (ret < 0)
2292 			p = &(*p)->rb_left;
2293 		else
2294 			p = &(*p)->rb_right;
2295 	}
2296 
2297 	rb_link_node(&backref->node, parent, p);
2298 	rb_insert_color(&backref->node, root);
2299 }
2300 
2301 /*
2302  * Note the backref might has changed, and in this case we just return 0.
2303  */
2304 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
2305 				       void *ctx)
2306 {
2307 	struct btrfs_file_extent_item *extent;
2308 	struct old_sa_defrag_extent *old = ctx;
2309 	struct new_sa_defrag_extent *new = old->new;
2310 	struct btrfs_path *path = new->path;
2311 	struct btrfs_key key;
2312 	struct btrfs_root *root;
2313 	struct sa_defrag_extent_backref *backref;
2314 	struct extent_buffer *leaf;
2315 	struct inode *inode = new->inode;
2316 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2317 	int slot;
2318 	int ret;
2319 	u64 extent_offset;
2320 	u64 num_bytes;
2321 
2322 	if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
2323 	    inum == btrfs_ino(inode))
2324 		return 0;
2325 
2326 	key.objectid = root_id;
2327 	key.type = BTRFS_ROOT_ITEM_KEY;
2328 	key.offset = (u64)-1;
2329 
2330 	root = btrfs_read_fs_root_no_name(fs_info, &key);
2331 	if (IS_ERR(root)) {
2332 		if (PTR_ERR(root) == -ENOENT)
2333 			return 0;
2334 		WARN_ON(1);
2335 		btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu",
2336 			 inum, offset, root_id);
2337 		return PTR_ERR(root);
2338 	}
2339 
2340 	key.objectid = inum;
2341 	key.type = BTRFS_EXTENT_DATA_KEY;
2342 	if (offset > (u64)-1 << 32)
2343 		key.offset = 0;
2344 	else
2345 		key.offset = offset;
2346 
2347 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2348 	if (WARN_ON(ret < 0))
2349 		return ret;
2350 	ret = 0;
2351 
2352 	while (1) {
2353 		cond_resched();
2354 
2355 		leaf = path->nodes[0];
2356 		slot = path->slots[0];
2357 
2358 		if (slot >= btrfs_header_nritems(leaf)) {
2359 			ret = btrfs_next_leaf(root, path);
2360 			if (ret < 0) {
2361 				goto out;
2362 			} else if (ret > 0) {
2363 				ret = 0;
2364 				goto out;
2365 			}
2366 			continue;
2367 		}
2368 
2369 		path->slots[0]++;
2370 
2371 		btrfs_item_key_to_cpu(leaf, &key, slot);
2372 
2373 		if (key.objectid > inum)
2374 			goto out;
2375 
2376 		if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
2377 			continue;
2378 
2379 		extent = btrfs_item_ptr(leaf, slot,
2380 					struct btrfs_file_extent_item);
2381 
2382 		if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
2383 			continue;
2384 
2385 		/*
2386 		 * 'offset' refers to the exact key.offset,
2387 		 * NOT the 'offset' field in btrfs_extent_data_ref, ie.
2388 		 * (key.offset - extent_offset).
2389 		 */
2390 		if (key.offset != offset)
2391 			continue;
2392 
2393 		extent_offset = btrfs_file_extent_offset(leaf, extent);
2394 		num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2395 
2396 		if (extent_offset >= old->extent_offset + old->offset +
2397 		    old->len || extent_offset + num_bytes <=
2398 		    old->extent_offset + old->offset)
2399 			continue;
2400 		break;
2401 	}
2402 
2403 	backref = kmalloc(sizeof(*backref), GFP_NOFS);
2404 	if (!backref) {
2405 		ret = -ENOENT;
2406 		goto out;
2407 	}
2408 
2409 	backref->root_id = root_id;
2410 	backref->inum = inum;
2411 	backref->file_pos = offset;
2412 	backref->num_bytes = num_bytes;
2413 	backref->extent_offset = extent_offset;
2414 	backref->generation = btrfs_file_extent_generation(leaf, extent);
2415 	backref->old = old;
2416 	backref_insert(&new->root, backref);
2417 	old->count++;
2418 out:
2419 	btrfs_release_path(path);
2420 	WARN_ON(ret);
2421 	return ret;
2422 }
2423 
2424 static noinline bool record_extent_backrefs(struct btrfs_path *path,
2425 				   struct new_sa_defrag_extent *new)
2426 {
2427 	struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2428 	struct old_sa_defrag_extent *old, *tmp;
2429 	int ret;
2430 
2431 	new->path = path;
2432 
2433 	list_for_each_entry_safe(old, tmp, &new->head, list) {
2434 		ret = iterate_inodes_from_logical(old->bytenr +
2435 						  old->extent_offset, fs_info,
2436 						  path, record_one_backref,
2437 						  old);
2438 		if (ret < 0 && ret != -ENOENT)
2439 			return false;
2440 
2441 		/* no backref to be processed for this extent */
2442 		if (!old->count) {
2443 			list_del(&old->list);
2444 			kfree(old);
2445 		}
2446 	}
2447 
2448 	if (list_empty(&new->head))
2449 		return false;
2450 
2451 	return true;
2452 }
2453 
2454 static int relink_is_mergable(struct extent_buffer *leaf,
2455 			      struct btrfs_file_extent_item *fi,
2456 			      struct new_sa_defrag_extent *new)
2457 {
2458 	if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
2459 		return 0;
2460 
2461 	if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2462 		return 0;
2463 
2464 	if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
2465 		return 0;
2466 
2467 	if (btrfs_file_extent_encryption(leaf, fi) ||
2468 	    btrfs_file_extent_other_encoding(leaf, fi))
2469 		return 0;
2470 
2471 	return 1;
2472 }
2473 
2474 /*
2475  * Note the backref might has changed, and in this case we just return 0.
2476  */
2477 static noinline int relink_extent_backref(struct btrfs_path *path,
2478 				 struct sa_defrag_extent_backref *prev,
2479 				 struct sa_defrag_extent_backref *backref)
2480 {
2481 	struct btrfs_file_extent_item *extent;
2482 	struct btrfs_file_extent_item *item;
2483 	struct btrfs_ordered_extent *ordered;
2484 	struct btrfs_trans_handle *trans;
2485 	struct btrfs_root *root;
2486 	struct btrfs_key key;
2487 	struct extent_buffer *leaf;
2488 	struct old_sa_defrag_extent *old = backref->old;
2489 	struct new_sa_defrag_extent *new = old->new;
2490 	struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2491 	struct inode *inode;
2492 	struct extent_state *cached = NULL;
2493 	int ret = 0;
2494 	u64 start;
2495 	u64 len;
2496 	u64 lock_start;
2497 	u64 lock_end;
2498 	bool merge = false;
2499 	int index;
2500 
2501 	if (prev && prev->root_id == backref->root_id &&
2502 	    prev->inum == backref->inum &&
2503 	    prev->file_pos + prev->num_bytes == backref->file_pos)
2504 		merge = true;
2505 
2506 	/* step 1: get root */
2507 	key.objectid = backref->root_id;
2508 	key.type = BTRFS_ROOT_ITEM_KEY;
2509 	key.offset = (u64)-1;
2510 
2511 	index = srcu_read_lock(&fs_info->subvol_srcu);
2512 
2513 	root = btrfs_read_fs_root_no_name(fs_info, &key);
2514 	if (IS_ERR(root)) {
2515 		srcu_read_unlock(&fs_info->subvol_srcu, index);
2516 		if (PTR_ERR(root) == -ENOENT)
2517 			return 0;
2518 		return PTR_ERR(root);
2519 	}
2520 
2521 	if (btrfs_root_readonly(root)) {
2522 		srcu_read_unlock(&fs_info->subvol_srcu, index);
2523 		return 0;
2524 	}
2525 
2526 	/* step 2: get inode */
2527 	key.objectid = backref->inum;
2528 	key.type = BTRFS_INODE_ITEM_KEY;
2529 	key.offset = 0;
2530 
2531 	inode = btrfs_iget(fs_info->sb, &key, root, NULL);
2532 	if (IS_ERR(inode)) {
2533 		srcu_read_unlock(&fs_info->subvol_srcu, index);
2534 		return 0;
2535 	}
2536 
2537 	srcu_read_unlock(&fs_info->subvol_srcu, index);
2538 
2539 	/* step 3: relink backref */
2540 	lock_start = backref->file_pos;
2541 	lock_end = backref->file_pos + backref->num_bytes - 1;
2542 	lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2543 			 &cached);
2544 
2545 	ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
2546 	if (ordered) {
2547 		btrfs_put_ordered_extent(ordered);
2548 		goto out_unlock;
2549 	}
2550 
2551 	trans = btrfs_join_transaction(root);
2552 	if (IS_ERR(trans)) {
2553 		ret = PTR_ERR(trans);
2554 		goto out_unlock;
2555 	}
2556 
2557 	key.objectid = backref->inum;
2558 	key.type = BTRFS_EXTENT_DATA_KEY;
2559 	key.offset = backref->file_pos;
2560 
2561 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2562 	if (ret < 0) {
2563 		goto out_free_path;
2564 	} else if (ret > 0) {
2565 		ret = 0;
2566 		goto out_free_path;
2567 	}
2568 
2569 	extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
2570 				struct btrfs_file_extent_item);
2571 
2572 	if (btrfs_file_extent_generation(path->nodes[0], extent) !=
2573 	    backref->generation)
2574 		goto out_free_path;
2575 
2576 	btrfs_release_path(path);
2577 
2578 	start = backref->file_pos;
2579 	if (backref->extent_offset < old->extent_offset + old->offset)
2580 		start += old->extent_offset + old->offset -
2581 			 backref->extent_offset;
2582 
2583 	len = min(backref->extent_offset + backref->num_bytes,
2584 		  old->extent_offset + old->offset + old->len);
2585 	len -= max(backref->extent_offset, old->extent_offset + old->offset);
2586 
2587 	ret = btrfs_drop_extents(trans, root, inode, start,
2588 				 start + len, 1);
2589 	if (ret)
2590 		goto out_free_path;
2591 again:
2592 	key.objectid = btrfs_ino(inode);
2593 	key.type = BTRFS_EXTENT_DATA_KEY;
2594 	key.offset = start;
2595 
2596 	path->leave_spinning = 1;
2597 	if (merge) {
2598 		struct btrfs_file_extent_item *fi;
2599 		u64 extent_len;
2600 		struct btrfs_key found_key;
2601 
2602 		ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2603 		if (ret < 0)
2604 			goto out_free_path;
2605 
2606 		path->slots[0]--;
2607 		leaf = path->nodes[0];
2608 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2609 
2610 		fi = btrfs_item_ptr(leaf, path->slots[0],
2611 				    struct btrfs_file_extent_item);
2612 		extent_len = btrfs_file_extent_num_bytes(leaf, fi);
2613 
2614 		if (extent_len + found_key.offset == start &&
2615 		    relink_is_mergable(leaf, fi, new)) {
2616 			btrfs_set_file_extent_num_bytes(leaf, fi,
2617 							extent_len + len);
2618 			btrfs_mark_buffer_dirty(leaf);
2619 			inode_add_bytes(inode, len);
2620 
2621 			ret = 1;
2622 			goto out_free_path;
2623 		} else {
2624 			merge = false;
2625 			btrfs_release_path(path);
2626 			goto again;
2627 		}
2628 	}
2629 
2630 	ret = btrfs_insert_empty_item(trans, root, path, &key,
2631 					sizeof(*extent));
2632 	if (ret) {
2633 		btrfs_abort_transaction(trans, ret);
2634 		goto out_free_path;
2635 	}
2636 
2637 	leaf = path->nodes[0];
2638 	item = btrfs_item_ptr(leaf, path->slots[0],
2639 				struct btrfs_file_extent_item);
2640 	btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
2641 	btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
2642 	btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
2643 	btrfs_set_file_extent_num_bytes(leaf, item, len);
2644 	btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
2645 	btrfs_set_file_extent_generation(leaf, item, trans->transid);
2646 	btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
2647 	btrfs_set_file_extent_compression(leaf, item, new->compress_type);
2648 	btrfs_set_file_extent_encryption(leaf, item, 0);
2649 	btrfs_set_file_extent_other_encoding(leaf, item, 0);
2650 
2651 	btrfs_mark_buffer_dirty(leaf);
2652 	inode_add_bytes(inode, len);
2653 	btrfs_release_path(path);
2654 
2655 	ret = btrfs_inc_extent_ref(trans, fs_info, new->bytenr,
2656 			new->disk_len, 0,
2657 			backref->root_id, backref->inum,
2658 			new->file_pos);	/* start - extent_offset */
2659 	if (ret) {
2660 		btrfs_abort_transaction(trans, ret);
2661 		goto out_free_path;
2662 	}
2663 
2664 	ret = 1;
2665 out_free_path:
2666 	btrfs_release_path(path);
2667 	path->leave_spinning = 0;
2668 	btrfs_end_transaction(trans);
2669 out_unlock:
2670 	unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2671 			     &cached, GFP_NOFS);
2672 	iput(inode);
2673 	return ret;
2674 }
2675 
2676 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
2677 {
2678 	struct old_sa_defrag_extent *old, *tmp;
2679 
2680 	if (!new)
2681 		return;
2682 
2683 	list_for_each_entry_safe(old, tmp, &new->head, list) {
2684 		kfree(old);
2685 	}
2686 	kfree(new);
2687 }
2688 
2689 static void relink_file_extents(struct new_sa_defrag_extent *new)
2690 {
2691 	struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2692 	struct btrfs_path *path;
2693 	struct sa_defrag_extent_backref *backref;
2694 	struct sa_defrag_extent_backref *prev = NULL;
2695 	struct inode *inode;
2696 	struct btrfs_root *root;
2697 	struct rb_node *node;
2698 	int ret;
2699 
2700 	inode = new->inode;
2701 	root = BTRFS_I(inode)->root;
2702 
2703 	path = btrfs_alloc_path();
2704 	if (!path)
2705 		return;
2706 
2707 	if (!record_extent_backrefs(path, new)) {
2708 		btrfs_free_path(path);
2709 		goto out;
2710 	}
2711 	btrfs_release_path(path);
2712 
2713 	while (1) {
2714 		node = rb_first(&new->root);
2715 		if (!node)
2716 			break;
2717 		rb_erase(node, &new->root);
2718 
2719 		backref = rb_entry(node, struct sa_defrag_extent_backref, node);
2720 
2721 		ret = relink_extent_backref(path, prev, backref);
2722 		WARN_ON(ret < 0);
2723 
2724 		kfree(prev);
2725 
2726 		if (ret == 1)
2727 			prev = backref;
2728 		else
2729 			prev = NULL;
2730 		cond_resched();
2731 	}
2732 	kfree(prev);
2733 
2734 	btrfs_free_path(path);
2735 out:
2736 	free_sa_defrag_extent(new);
2737 
2738 	atomic_dec(&fs_info->defrag_running);
2739 	wake_up(&fs_info->transaction_wait);
2740 }
2741 
2742 static struct new_sa_defrag_extent *
2743 record_old_file_extents(struct inode *inode,
2744 			struct btrfs_ordered_extent *ordered)
2745 {
2746 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2747 	struct btrfs_root *root = BTRFS_I(inode)->root;
2748 	struct btrfs_path *path;
2749 	struct btrfs_key key;
2750 	struct old_sa_defrag_extent *old;
2751 	struct new_sa_defrag_extent *new;
2752 	int ret;
2753 
2754 	new = kmalloc(sizeof(*new), GFP_NOFS);
2755 	if (!new)
2756 		return NULL;
2757 
2758 	new->inode = inode;
2759 	new->file_pos = ordered->file_offset;
2760 	new->len = ordered->len;
2761 	new->bytenr = ordered->start;
2762 	new->disk_len = ordered->disk_len;
2763 	new->compress_type = ordered->compress_type;
2764 	new->root = RB_ROOT;
2765 	INIT_LIST_HEAD(&new->head);
2766 
2767 	path = btrfs_alloc_path();
2768 	if (!path)
2769 		goto out_kfree;
2770 
2771 	key.objectid = btrfs_ino(inode);
2772 	key.type = BTRFS_EXTENT_DATA_KEY;
2773 	key.offset = new->file_pos;
2774 
2775 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2776 	if (ret < 0)
2777 		goto out_free_path;
2778 	if (ret > 0 && path->slots[0] > 0)
2779 		path->slots[0]--;
2780 
2781 	/* find out all the old extents for the file range */
2782 	while (1) {
2783 		struct btrfs_file_extent_item *extent;
2784 		struct extent_buffer *l;
2785 		int slot;
2786 		u64 num_bytes;
2787 		u64 offset;
2788 		u64 end;
2789 		u64 disk_bytenr;
2790 		u64 extent_offset;
2791 
2792 		l = path->nodes[0];
2793 		slot = path->slots[0];
2794 
2795 		if (slot >= btrfs_header_nritems(l)) {
2796 			ret = btrfs_next_leaf(root, path);
2797 			if (ret < 0)
2798 				goto out_free_path;
2799 			else if (ret > 0)
2800 				break;
2801 			continue;
2802 		}
2803 
2804 		btrfs_item_key_to_cpu(l, &key, slot);
2805 
2806 		if (key.objectid != btrfs_ino(inode))
2807 			break;
2808 		if (key.type != BTRFS_EXTENT_DATA_KEY)
2809 			break;
2810 		if (key.offset >= new->file_pos + new->len)
2811 			break;
2812 
2813 		extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
2814 
2815 		num_bytes = btrfs_file_extent_num_bytes(l, extent);
2816 		if (key.offset + num_bytes < new->file_pos)
2817 			goto next;
2818 
2819 		disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
2820 		if (!disk_bytenr)
2821 			goto next;
2822 
2823 		extent_offset = btrfs_file_extent_offset(l, extent);
2824 
2825 		old = kmalloc(sizeof(*old), GFP_NOFS);
2826 		if (!old)
2827 			goto out_free_path;
2828 
2829 		offset = max(new->file_pos, key.offset);
2830 		end = min(new->file_pos + new->len, key.offset + num_bytes);
2831 
2832 		old->bytenr = disk_bytenr;
2833 		old->extent_offset = extent_offset;
2834 		old->offset = offset - key.offset;
2835 		old->len = end - offset;
2836 		old->new = new;
2837 		old->count = 0;
2838 		list_add_tail(&old->list, &new->head);
2839 next:
2840 		path->slots[0]++;
2841 		cond_resched();
2842 	}
2843 
2844 	btrfs_free_path(path);
2845 	atomic_inc(&fs_info->defrag_running);
2846 
2847 	return new;
2848 
2849 out_free_path:
2850 	btrfs_free_path(path);
2851 out_kfree:
2852 	free_sa_defrag_extent(new);
2853 	return NULL;
2854 }
2855 
2856 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
2857 					 u64 start, u64 len)
2858 {
2859 	struct btrfs_block_group_cache *cache;
2860 
2861 	cache = btrfs_lookup_block_group(fs_info, start);
2862 	ASSERT(cache);
2863 
2864 	spin_lock(&cache->lock);
2865 	cache->delalloc_bytes -= len;
2866 	spin_unlock(&cache->lock);
2867 
2868 	btrfs_put_block_group(cache);
2869 }
2870 
2871 /* as ordered data IO finishes, this gets called so we can finish
2872  * an ordered extent if the range of bytes in the file it covers are
2873  * fully written.
2874  */
2875 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
2876 {
2877 	struct inode *inode = ordered_extent->inode;
2878 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2879 	struct btrfs_root *root = BTRFS_I(inode)->root;
2880 	struct btrfs_trans_handle *trans = NULL;
2881 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2882 	struct extent_state *cached_state = NULL;
2883 	struct new_sa_defrag_extent *new = NULL;
2884 	int compress_type = 0;
2885 	int ret = 0;
2886 	u64 logical_len = ordered_extent->len;
2887 	bool nolock;
2888 	bool truncated = false;
2889 
2890 	nolock = btrfs_is_free_space_inode(inode);
2891 
2892 	if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
2893 		ret = -EIO;
2894 		goto out;
2895 	}
2896 
2897 	btrfs_free_io_failure_record(inode, ordered_extent->file_offset,
2898 				     ordered_extent->file_offset +
2899 				     ordered_extent->len - 1);
2900 
2901 	if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
2902 		truncated = true;
2903 		logical_len = ordered_extent->truncated_len;
2904 		/* Truncated the entire extent, don't bother adding */
2905 		if (!logical_len)
2906 			goto out;
2907 	}
2908 
2909 	if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
2910 		BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
2911 
2912 		/*
2913 		 * For mwrite(mmap + memset to write) case, we still reserve
2914 		 * space for NOCOW range.
2915 		 * As NOCOW won't cause a new delayed ref, just free the space
2916 		 */
2917 		btrfs_qgroup_free_data(inode, ordered_extent->file_offset,
2918 				       ordered_extent->len);
2919 		btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2920 		if (nolock)
2921 			trans = btrfs_join_transaction_nolock(root);
2922 		else
2923 			trans = btrfs_join_transaction(root);
2924 		if (IS_ERR(trans)) {
2925 			ret = PTR_ERR(trans);
2926 			trans = NULL;
2927 			goto out;
2928 		}
2929 		trans->block_rsv = &fs_info->delalloc_block_rsv;
2930 		ret = btrfs_update_inode_fallback(trans, root, inode);
2931 		if (ret) /* -ENOMEM or corruption */
2932 			btrfs_abort_transaction(trans, ret);
2933 		goto out;
2934 	}
2935 
2936 	lock_extent_bits(io_tree, ordered_extent->file_offset,
2937 			 ordered_extent->file_offset + ordered_extent->len - 1,
2938 			 &cached_state);
2939 
2940 	ret = test_range_bit(io_tree, ordered_extent->file_offset,
2941 			ordered_extent->file_offset + ordered_extent->len - 1,
2942 			EXTENT_DEFRAG, 1, cached_state);
2943 	if (ret) {
2944 		u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
2945 		if (0 && last_snapshot >= BTRFS_I(inode)->generation)
2946 			/* the inode is shared */
2947 			new = record_old_file_extents(inode, ordered_extent);
2948 
2949 		clear_extent_bit(io_tree, ordered_extent->file_offset,
2950 			ordered_extent->file_offset + ordered_extent->len - 1,
2951 			EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS);
2952 	}
2953 
2954 	if (nolock)
2955 		trans = btrfs_join_transaction_nolock(root);
2956 	else
2957 		trans = btrfs_join_transaction(root);
2958 	if (IS_ERR(trans)) {
2959 		ret = PTR_ERR(trans);
2960 		trans = NULL;
2961 		goto out_unlock;
2962 	}
2963 
2964 	trans->block_rsv = &fs_info->delalloc_block_rsv;
2965 
2966 	if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
2967 		compress_type = ordered_extent->compress_type;
2968 	if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
2969 		BUG_ON(compress_type);
2970 		ret = btrfs_mark_extent_written(trans, inode,
2971 						ordered_extent->file_offset,
2972 						ordered_extent->file_offset +
2973 						logical_len);
2974 	} else {
2975 		BUG_ON(root == fs_info->tree_root);
2976 		ret = insert_reserved_file_extent(trans, inode,
2977 						ordered_extent->file_offset,
2978 						ordered_extent->start,
2979 						ordered_extent->disk_len,
2980 						logical_len, logical_len,
2981 						compress_type, 0, 0,
2982 						BTRFS_FILE_EXTENT_REG);
2983 		if (!ret)
2984 			btrfs_release_delalloc_bytes(fs_info,
2985 						     ordered_extent->start,
2986 						     ordered_extent->disk_len);
2987 	}
2988 	unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
2989 			   ordered_extent->file_offset, ordered_extent->len,
2990 			   trans->transid);
2991 	if (ret < 0) {
2992 		btrfs_abort_transaction(trans, ret);
2993 		goto out_unlock;
2994 	}
2995 
2996 	add_pending_csums(trans, inode, ordered_extent->file_offset,
2997 			  &ordered_extent->list);
2998 
2999 	btrfs_ordered_update_i_size(inode, 0, ordered_extent);
3000 	ret = btrfs_update_inode_fallback(trans, root, inode);
3001 	if (ret) { /* -ENOMEM or corruption */
3002 		btrfs_abort_transaction(trans, ret);
3003 		goto out_unlock;
3004 	}
3005 	ret = 0;
3006 out_unlock:
3007 	unlock_extent_cached(io_tree, ordered_extent->file_offset,
3008 			     ordered_extent->file_offset +
3009 			     ordered_extent->len - 1, &cached_state, GFP_NOFS);
3010 out:
3011 	if (root != fs_info->tree_root)
3012 		btrfs_delalloc_release_metadata(inode, ordered_extent->len);
3013 	if (trans)
3014 		btrfs_end_transaction(trans);
3015 
3016 	if (ret || truncated) {
3017 		u64 start, end;
3018 
3019 		if (truncated)
3020 			start = ordered_extent->file_offset + logical_len;
3021 		else
3022 			start = ordered_extent->file_offset;
3023 		end = ordered_extent->file_offset + ordered_extent->len - 1;
3024 		clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS);
3025 
3026 		/* Drop the cache for the part of the extent we didn't write. */
3027 		btrfs_drop_extent_cache(inode, start, end, 0);
3028 
3029 		/*
3030 		 * If the ordered extent had an IOERR or something else went
3031 		 * wrong we need to return the space for this ordered extent
3032 		 * back to the allocator.  We only free the extent in the
3033 		 * truncated case if we didn't write out the extent at all.
3034 		 */
3035 		if ((ret || !logical_len) &&
3036 		    !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3037 		    !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
3038 			btrfs_free_reserved_extent(fs_info,
3039 						   ordered_extent->start,
3040 						   ordered_extent->disk_len, 1);
3041 	}
3042 
3043 
3044 	/*
3045 	 * This needs to be done to make sure anybody waiting knows we are done
3046 	 * updating everything for this ordered extent.
3047 	 */
3048 	btrfs_remove_ordered_extent(inode, ordered_extent);
3049 
3050 	/* for snapshot-aware defrag */
3051 	if (new) {
3052 		if (ret) {
3053 			free_sa_defrag_extent(new);
3054 			atomic_dec(&fs_info->defrag_running);
3055 		} else {
3056 			relink_file_extents(new);
3057 		}
3058 	}
3059 
3060 	/* once for us */
3061 	btrfs_put_ordered_extent(ordered_extent);
3062 	/* once for the tree */
3063 	btrfs_put_ordered_extent(ordered_extent);
3064 
3065 	return ret;
3066 }
3067 
3068 static void finish_ordered_fn(struct btrfs_work *work)
3069 {
3070 	struct btrfs_ordered_extent *ordered_extent;
3071 	ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
3072 	btrfs_finish_ordered_io(ordered_extent);
3073 }
3074 
3075 static int btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
3076 				struct extent_state *state, int uptodate)
3077 {
3078 	struct inode *inode = page->mapping->host;
3079 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3080 	struct btrfs_ordered_extent *ordered_extent = NULL;
3081 	struct btrfs_workqueue *wq;
3082 	btrfs_work_func_t func;
3083 
3084 	trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
3085 
3086 	ClearPagePrivate2(page);
3087 	if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
3088 					    end - start + 1, uptodate))
3089 		return 0;
3090 
3091 	if (btrfs_is_free_space_inode(inode)) {
3092 		wq = fs_info->endio_freespace_worker;
3093 		func = btrfs_freespace_write_helper;
3094 	} else {
3095 		wq = fs_info->endio_write_workers;
3096 		func = btrfs_endio_write_helper;
3097 	}
3098 
3099 	btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
3100 			NULL);
3101 	btrfs_queue_work(wq, &ordered_extent->work);
3102 
3103 	return 0;
3104 }
3105 
3106 static int __readpage_endio_check(struct inode *inode,
3107 				  struct btrfs_io_bio *io_bio,
3108 				  int icsum, struct page *page,
3109 				  int pgoff, u64 start, size_t len)
3110 {
3111 	char *kaddr;
3112 	u32 csum_expected;
3113 	u32 csum = ~(u32)0;
3114 
3115 	csum_expected = *(((u32 *)io_bio->csum) + icsum);
3116 
3117 	kaddr = kmap_atomic(page);
3118 	csum = btrfs_csum_data(kaddr + pgoff, csum,  len);
3119 	btrfs_csum_final(csum, (u8 *)&csum);
3120 	if (csum != csum_expected)
3121 		goto zeroit;
3122 
3123 	kunmap_atomic(kaddr);
3124 	return 0;
3125 zeroit:
3126 	btrfs_warn_rl(BTRFS_I(inode)->root->fs_info,
3127 		"csum failed ino %llu off %llu csum %u expected csum %u",
3128 			   btrfs_ino(inode), start, csum, csum_expected);
3129 	memset(kaddr + pgoff, 1, len);
3130 	flush_dcache_page(page);
3131 	kunmap_atomic(kaddr);
3132 	if (csum_expected == 0)
3133 		return 0;
3134 	return -EIO;
3135 }
3136 
3137 /*
3138  * when reads are done, we need to check csums to verify the data is correct
3139  * if there's a match, we allow the bio to finish.  If not, the code in
3140  * extent_io.c will try to find good copies for us.
3141  */
3142 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
3143 				      u64 phy_offset, struct page *page,
3144 				      u64 start, u64 end, int mirror)
3145 {
3146 	size_t offset = start - page_offset(page);
3147 	struct inode *inode = page->mapping->host;
3148 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3149 	struct btrfs_root *root = BTRFS_I(inode)->root;
3150 
3151 	if (PageChecked(page)) {
3152 		ClearPageChecked(page);
3153 		return 0;
3154 	}
3155 
3156 	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
3157 		return 0;
3158 
3159 	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
3160 	    test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
3161 		clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
3162 		return 0;
3163 	}
3164 
3165 	phy_offset >>= inode->i_sb->s_blocksize_bits;
3166 	return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
3167 				      start, (size_t)(end - start + 1));
3168 }
3169 
3170 void btrfs_add_delayed_iput(struct inode *inode)
3171 {
3172 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3173 	struct btrfs_inode *binode = BTRFS_I(inode);
3174 
3175 	if (atomic_add_unless(&inode->i_count, -1, 1))
3176 		return;
3177 
3178 	spin_lock(&fs_info->delayed_iput_lock);
3179 	if (binode->delayed_iput_count == 0) {
3180 		ASSERT(list_empty(&binode->delayed_iput));
3181 		list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
3182 	} else {
3183 		binode->delayed_iput_count++;
3184 	}
3185 	spin_unlock(&fs_info->delayed_iput_lock);
3186 }
3187 
3188 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3189 {
3190 
3191 	spin_lock(&fs_info->delayed_iput_lock);
3192 	while (!list_empty(&fs_info->delayed_iputs)) {
3193 		struct btrfs_inode *inode;
3194 
3195 		inode = list_first_entry(&fs_info->delayed_iputs,
3196 				struct btrfs_inode, delayed_iput);
3197 		if (inode->delayed_iput_count) {
3198 			inode->delayed_iput_count--;
3199 			list_move_tail(&inode->delayed_iput,
3200 					&fs_info->delayed_iputs);
3201 		} else {
3202 			list_del_init(&inode->delayed_iput);
3203 		}
3204 		spin_unlock(&fs_info->delayed_iput_lock);
3205 		iput(&inode->vfs_inode);
3206 		spin_lock(&fs_info->delayed_iput_lock);
3207 	}
3208 	spin_unlock(&fs_info->delayed_iput_lock);
3209 }
3210 
3211 /*
3212  * This is called in transaction commit time. If there are no orphan
3213  * files in the subvolume, it removes orphan item and frees block_rsv
3214  * structure.
3215  */
3216 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
3217 			      struct btrfs_root *root)
3218 {
3219 	struct btrfs_fs_info *fs_info = root->fs_info;
3220 	struct btrfs_block_rsv *block_rsv;
3221 	int ret;
3222 
3223 	if (atomic_read(&root->orphan_inodes) ||
3224 	    root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
3225 		return;
3226 
3227 	spin_lock(&root->orphan_lock);
3228 	if (atomic_read(&root->orphan_inodes)) {
3229 		spin_unlock(&root->orphan_lock);
3230 		return;
3231 	}
3232 
3233 	if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
3234 		spin_unlock(&root->orphan_lock);
3235 		return;
3236 	}
3237 
3238 	block_rsv = root->orphan_block_rsv;
3239 	root->orphan_block_rsv = NULL;
3240 	spin_unlock(&root->orphan_lock);
3241 
3242 	if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) &&
3243 	    btrfs_root_refs(&root->root_item) > 0) {
3244 		ret = btrfs_del_orphan_item(trans, fs_info->tree_root,
3245 					    root->root_key.objectid);
3246 		if (ret)
3247 			btrfs_abort_transaction(trans, ret);
3248 		else
3249 			clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
3250 				  &root->state);
3251 	}
3252 
3253 	if (block_rsv) {
3254 		WARN_ON(block_rsv->size > 0);
3255 		btrfs_free_block_rsv(fs_info, block_rsv);
3256 	}
3257 }
3258 
3259 /*
3260  * This creates an orphan entry for the given inode in case something goes
3261  * wrong in the middle of an unlink/truncate.
3262  *
3263  * NOTE: caller of this function should reserve 5 units of metadata for
3264  *	 this function.
3265  */
3266 int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct inode *inode)
3267 {
3268 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3269 	struct btrfs_root *root = BTRFS_I(inode)->root;
3270 	struct btrfs_block_rsv *block_rsv = NULL;
3271 	int reserve = 0;
3272 	int insert = 0;
3273 	int ret;
3274 
3275 	if (!root->orphan_block_rsv) {
3276 		block_rsv = btrfs_alloc_block_rsv(fs_info,
3277 						  BTRFS_BLOCK_RSV_TEMP);
3278 		if (!block_rsv)
3279 			return -ENOMEM;
3280 	}
3281 
3282 	spin_lock(&root->orphan_lock);
3283 	if (!root->orphan_block_rsv) {
3284 		root->orphan_block_rsv = block_rsv;
3285 	} else if (block_rsv) {
3286 		btrfs_free_block_rsv(fs_info, block_rsv);
3287 		block_rsv = NULL;
3288 	}
3289 
3290 	if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3291 			      &BTRFS_I(inode)->runtime_flags)) {
3292 #if 0
3293 		/*
3294 		 * For proper ENOSPC handling, we should do orphan
3295 		 * cleanup when mounting. But this introduces backward
3296 		 * compatibility issue.
3297 		 */
3298 		if (!xchg(&root->orphan_item_inserted, 1))
3299 			insert = 2;
3300 		else
3301 			insert = 1;
3302 #endif
3303 		insert = 1;
3304 		atomic_inc(&root->orphan_inodes);
3305 	}
3306 
3307 	if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3308 			      &BTRFS_I(inode)->runtime_flags))
3309 		reserve = 1;
3310 	spin_unlock(&root->orphan_lock);
3311 
3312 	/* grab metadata reservation from transaction handle */
3313 	if (reserve) {
3314 		ret = btrfs_orphan_reserve_metadata(trans, inode);
3315 		ASSERT(!ret);
3316 		if (ret) {
3317 			atomic_dec(&root->orphan_inodes);
3318 			clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3319 				  &BTRFS_I(inode)->runtime_flags);
3320 			if (insert)
3321 				clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3322 					  &BTRFS_I(inode)->runtime_flags);
3323 			return ret;
3324 		}
3325 	}
3326 
3327 	/* insert an orphan item to track this unlinked/truncated file */
3328 	if (insert >= 1) {
3329 		ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
3330 		if (ret) {
3331 			atomic_dec(&root->orphan_inodes);
3332 			if (reserve) {
3333 				clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3334 					  &BTRFS_I(inode)->runtime_flags);
3335 				btrfs_orphan_release_metadata(inode);
3336 			}
3337 			if (ret != -EEXIST) {
3338 				clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3339 					  &BTRFS_I(inode)->runtime_flags);
3340 				btrfs_abort_transaction(trans, ret);
3341 				return ret;
3342 			}
3343 		}
3344 		ret = 0;
3345 	}
3346 
3347 	/* insert an orphan item to track subvolume contains orphan files */
3348 	if (insert >= 2) {
3349 		ret = btrfs_insert_orphan_item(trans, fs_info->tree_root,
3350 					       root->root_key.objectid);
3351 		if (ret && ret != -EEXIST) {
3352 			btrfs_abort_transaction(trans, ret);
3353 			return ret;
3354 		}
3355 	}
3356 	return 0;
3357 }
3358 
3359 /*
3360  * We have done the truncate/delete so we can go ahead and remove the orphan
3361  * item for this particular inode.
3362  */
3363 static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3364 			    struct inode *inode)
3365 {
3366 	struct btrfs_root *root = BTRFS_I(inode)->root;
3367 	int delete_item = 0;
3368 	int release_rsv = 0;
3369 	int ret = 0;
3370 
3371 	spin_lock(&root->orphan_lock);
3372 	if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3373 			       &BTRFS_I(inode)->runtime_flags))
3374 		delete_item = 1;
3375 
3376 	if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3377 			       &BTRFS_I(inode)->runtime_flags))
3378 		release_rsv = 1;
3379 	spin_unlock(&root->orphan_lock);
3380 
3381 	if (delete_item) {
3382 		atomic_dec(&root->orphan_inodes);
3383 		if (trans)
3384 			ret = btrfs_del_orphan_item(trans, root,
3385 						    btrfs_ino(inode));
3386 	}
3387 
3388 	if (release_rsv)
3389 		btrfs_orphan_release_metadata(inode);
3390 
3391 	return ret;
3392 }
3393 
3394 /*
3395  * this cleans up any orphans that may be left on the list from the last use
3396  * of this root.
3397  */
3398 int btrfs_orphan_cleanup(struct btrfs_root *root)
3399 {
3400 	struct btrfs_fs_info *fs_info = root->fs_info;
3401 	struct btrfs_path *path;
3402 	struct extent_buffer *leaf;
3403 	struct btrfs_key key, found_key;
3404 	struct btrfs_trans_handle *trans;
3405 	struct inode *inode;
3406 	u64 last_objectid = 0;
3407 	int ret = 0, nr_unlink = 0, nr_truncate = 0;
3408 
3409 	if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3410 		return 0;
3411 
3412 	path = btrfs_alloc_path();
3413 	if (!path) {
3414 		ret = -ENOMEM;
3415 		goto out;
3416 	}
3417 	path->reada = READA_BACK;
3418 
3419 	key.objectid = BTRFS_ORPHAN_OBJECTID;
3420 	key.type = BTRFS_ORPHAN_ITEM_KEY;
3421 	key.offset = (u64)-1;
3422 
3423 	while (1) {
3424 		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3425 		if (ret < 0)
3426 			goto out;
3427 
3428 		/*
3429 		 * if ret == 0 means we found what we were searching for, which
3430 		 * is weird, but possible, so only screw with path if we didn't
3431 		 * find the key and see if we have stuff that matches
3432 		 */
3433 		if (ret > 0) {
3434 			ret = 0;
3435 			if (path->slots[0] == 0)
3436 				break;
3437 			path->slots[0]--;
3438 		}
3439 
3440 		/* pull out the item */
3441 		leaf = path->nodes[0];
3442 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3443 
3444 		/* make sure the item matches what we want */
3445 		if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3446 			break;
3447 		if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3448 			break;
3449 
3450 		/* release the path since we're done with it */
3451 		btrfs_release_path(path);
3452 
3453 		/*
3454 		 * this is where we are basically btrfs_lookup, without the
3455 		 * crossing root thing.  we store the inode number in the
3456 		 * offset of the orphan item.
3457 		 */
3458 
3459 		if (found_key.offset == last_objectid) {
3460 			btrfs_err(fs_info,
3461 				  "Error removing orphan entry, stopping orphan cleanup");
3462 			ret = -EINVAL;
3463 			goto out;
3464 		}
3465 
3466 		last_objectid = found_key.offset;
3467 
3468 		found_key.objectid = found_key.offset;
3469 		found_key.type = BTRFS_INODE_ITEM_KEY;
3470 		found_key.offset = 0;
3471 		inode = btrfs_iget(fs_info->sb, &found_key, root, NULL);
3472 		ret = PTR_ERR_OR_ZERO(inode);
3473 		if (ret && ret != -ENOENT)
3474 			goto out;
3475 
3476 		if (ret == -ENOENT && root == fs_info->tree_root) {
3477 			struct btrfs_root *dead_root;
3478 			struct btrfs_fs_info *fs_info = root->fs_info;
3479 			int is_dead_root = 0;
3480 
3481 			/*
3482 			 * this is an orphan in the tree root. Currently these
3483 			 * could come from 2 sources:
3484 			 *  a) a snapshot deletion in progress
3485 			 *  b) a free space cache inode
3486 			 * We need to distinguish those two, as the snapshot
3487 			 * orphan must not get deleted.
3488 			 * find_dead_roots already ran before us, so if this
3489 			 * is a snapshot deletion, we should find the root
3490 			 * in the dead_roots list
3491 			 */
3492 			spin_lock(&fs_info->trans_lock);
3493 			list_for_each_entry(dead_root, &fs_info->dead_roots,
3494 					    root_list) {
3495 				if (dead_root->root_key.objectid ==
3496 				    found_key.objectid) {
3497 					is_dead_root = 1;
3498 					break;
3499 				}
3500 			}
3501 			spin_unlock(&fs_info->trans_lock);
3502 			if (is_dead_root) {
3503 				/* prevent this orphan from being found again */
3504 				key.offset = found_key.objectid - 1;
3505 				continue;
3506 			}
3507 		}
3508 		/*
3509 		 * Inode is already gone but the orphan item is still there,
3510 		 * kill the orphan item.
3511 		 */
3512 		if (ret == -ENOENT) {
3513 			trans = btrfs_start_transaction(root, 1);
3514 			if (IS_ERR(trans)) {
3515 				ret = PTR_ERR(trans);
3516 				goto out;
3517 			}
3518 			btrfs_debug(fs_info, "auto deleting %Lu",
3519 				    found_key.objectid);
3520 			ret = btrfs_del_orphan_item(trans, root,
3521 						    found_key.objectid);
3522 			btrfs_end_transaction(trans);
3523 			if (ret)
3524 				goto out;
3525 			continue;
3526 		}
3527 
3528 		/*
3529 		 * add this inode to the orphan list so btrfs_orphan_del does
3530 		 * the proper thing when we hit it
3531 		 */
3532 		set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3533 			&BTRFS_I(inode)->runtime_flags);
3534 		atomic_inc(&root->orphan_inodes);
3535 
3536 		/* if we have links, this was a truncate, lets do that */
3537 		if (inode->i_nlink) {
3538 			if (WARN_ON(!S_ISREG(inode->i_mode))) {
3539 				iput(inode);
3540 				continue;
3541 			}
3542 			nr_truncate++;
3543 
3544 			/* 1 for the orphan item deletion. */
3545 			trans = btrfs_start_transaction(root, 1);
3546 			if (IS_ERR(trans)) {
3547 				iput(inode);
3548 				ret = PTR_ERR(trans);
3549 				goto out;
3550 			}
3551 			ret = btrfs_orphan_add(trans, inode);
3552 			btrfs_end_transaction(trans);
3553 			if (ret) {
3554 				iput(inode);
3555 				goto out;
3556 			}
3557 
3558 			ret = btrfs_truncate(inode);
3559 			if (ret)
3560 				btrfs_orphan_del(NULL, inode);
3561 		} else {
3562 			nr_unlink++;
3563 		}
3564 
3565 		/* this will do delete_inode and everything for us */
3566 		iput(inode);
3567 		if (ret)
3568 			goto out;
3569 	}
3570 	/* release the path since we're done with it */
3571 	btrfs_release_path(path);
3572 
3573 	root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
3574 
3575 	if (root->orphan_block_rsv)
3576 		btrfs_block_rsv_release(fs_info, root->orphan_block_rsv,
3577 					(u64)-1);
3578 
3579 	if (root->orphan_block_rsv ||
3580 	    test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3581 		trans = btrfs_join_transaction(root);
3582 		if (!IS_ERR(trans))
3583 			btrfs_end_transaction(trans);
3584 	}
3585 
3586 	if (nr_unlink)
3587 		btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3588 	if (nr_truncate)
3589 		btrfs_debug(fs_info, "truncated %d orphans", nr_truncate);
3590 
3591 out:
3592 	if (ret)
3593 		btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
3594 	btrfs_free_path(path);
3595 	return ret;
3596 }
3597 
3598 /*
3599  * very simple check to peek ahead in the leaf looking for xattrs.  If we
3600  * don't find any xattrs, we know there can't be any acls.
3601  *
3602  * slot is the slot the inode is in, objectid is the objectid of the inode
3603  */
3604 static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3605 					  int slot, u64 objectid,
3606 					  int *first_xattr_slot)
3607 {
3608 	u32 nritems = btrfs_header_nritems(leaf);
3609 	struct btrfs_key found_key;
3610 	static u64 xattr_access = 0;
3611 	static u64 xattr_default = 0;
3612 	int scanned = 0;
3613 
3614 	if (!xattr_access) {
3615 		xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3616 					strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3617 		xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3618 					strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3619 	}
3620 
3621 	slot++;
3622 	*first_xattr_slot = -1;
3623 	while (slot < nritems) {
3624 		btrfs_item_key_to_cpu(leaf, &found_key, slot);
3625 
3626 		/* we found a different objectid, there must not be acls */
3627 		if (found_key.objectid != objectid)
3628 			return 0;
3629 
3630 		/* we found an xattr, assume we've got an acl */
3631 		if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3632 			if (*first_xattr_slot == -1)
3633 				*first_xattr_slot = slot;
3634 			if (found_key.offset == xattr_access ||
3635 			    found_key.offset == xattr_default)
3636 				return 1;
3637 		}
3638 
3639 		/*
3640 		 * we found a key greater than an xattr key, there can't
3641 		 * be any acls later on
3642 		 */
3643 		if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3644 			return 0;
3645 
3646 		slot++;
3647 		scanned++;
3648 
3649 		/*
3650 		 * it goes inode, inode backrefs, xattrs, extents,
3651 		 * so if there are a ton of hard links to an inode there can
3652 		 * be a lot of backrefs.  Don't waste time searching too hard,
3653 		 * this is just an optimization
3654 		 */
3655 		if (scanned >= 8)
3656 			break;
3657 	}
3658 	/* we hit the end of the leaf before we found an xattr or
3659 	 * something larger than an xattr.  We have to assume the inode
3660 	 * has acls
3661 	 */
3662 	if (*first_xattr_slot == -1)
3663 		*first_xattr_slot = slot;
3664 	return 1;
3665 }
3666 
3667 /*
3668  * read an inode from the btree into the in-memory inode
3669  */
3670 static int btrfs_read_locked_inode(struct inode *inode)
3671 {
3672 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3673 	struct btrfs_path *path;
3674 	struct extent_buffer *leaf;
3675 	struct btrfs_inode_item *inode_item;
3676 	struct btrfs_root *root = BTRFS_I(inode)->root;
3677 	struct btrfs_key location;
3678 	unsigned long ptr;
3679 	int maybe_acls;
3680 	u32 rdev;
3681 	int ret;
3682 	bool filled = false;
3683 	int first_xattr_slot;
3684 
3685 	ret = btrfs_fill_inode(inode, &rdev);
3686 	if (!ret)
3687 		filled = true;
3688 
3689 	path = btrfs_alloc_path();
3690 	if (!path) {
3691 		ret = -ENOMEM;
3692 		goto make_bad;
3693 	}
3694 
3695 	memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
3696 
3697 	ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3698 	if (ret) {
3699 		if (ret > 0)
3700 			ret = -ENOENT;
3701 		goto make_bad;
3702 	}
3703 
3704 	leaf = path->nodes[0];
3705 
3706 	if (filled)
3707 		goto cache_index;
3708 
3709 	inode_item = btrfs_item_ptr(leaf, path->slots[0],
3710 				    struct btrfs_inode_item);
3711 	inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3712 	set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3713 	i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
3714 	i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3715 	btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item));
3716 
3717 	inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
3718 	inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
3719 
3720 	inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
3721 	inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
3722 
3723 	inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
3724 	inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
3725 
3726 	BTRFS_I(inode)->i_otime.tv_sec =
3727 		btrfs_timespec_sec(leaf, &inode_item->otime);
3728 	BTRFS_I(inode)->i_otime.tv_nsec =
3729 		btrfs_timespec_nsec(leaf, &inode_item->otime);
3730 
3731 	inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3732 	BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
3733 	BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
3734 
3735 	inode->i_version = btrfs_inode_sequence(leaf, inode_item);
3736 	inode->i_generation = BTRFS_I(inode)->generation;
3737 	inode->i_rdev = 0;
3738 	rdev = btrfs_inode_rdev(leaf, inode_item);
3739 
3740 	BTRFS_I(inode)->index_cnt = (u64)-1;
3741 	BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
3742 
3743 cache_index:
3744 	/*
3745 	 * If we were modified in the current generation and evicted from memory
3746 	 * and then re-read we need to do a full sync since we don't have any
3747 	 * idea about which extents were modified before we were evicted from
3748 	 * cache.
3749 	 *
3750 	 * This is required for both inode re-read from disk and delayed inode
3751 	 * in delayed_nodes_tree.
3752 	 */
3753 	if (BTRFS_I(inode)->last_trans == fs_info->generation)
3754 		set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
3755 			&BTRFS_I(inode)->runtime_flags);
3756 
3757 	/*
3758 	 * We don't persist the id of the transaction where an unlink operation
3759 	 * against the inode was last made. So here we assume the inode might
3760 	 * have been evicted, and therefore the exact value of last_unlink_trans
3761 	 * lost, and set it to last_trans to avoid metadata inconsistencies
3762 	 * between the inode and its parent if the inode is fsync'ed and the log
3763 	 * replayed. For example, in the scenario:
3764 	 *
3765 	 * touch mydir/foo
3766 	 * ln mydir/foo mydir/bar
3767 	 * sync
3768 	 * unlink mydir/bar
3769 	 * echo 2 > /proc/sys/vm/drop_caches   # evicts inode
3770 	 * xfs_io -c fsync mydir/foo
3771 	 * <power failure>
3772 	 * mount fs, triggers fsync log replay
3773 	 *
3774 	 * We must make sure that when we fsync our inode foo we also log its
3775 	 * parent inode, otherwise after log replay the parent still has the
3776 	 * dentry with the "bar" name but our inode foo has a link count of 1
3777 	 * and doesn't have an inode ref with the name "bar" anymore.
3778 	 *
3779 	 * Setting last_unlink_trans to last_trans is a pessimistic approach,
3780 	 * but it guarantees correctness at the expense of occasional full
3781 	 * transaction commits on fsync if our inode is a directory, or if our
3782 	 * inode is not a directory, logging its parent unnecessarily.
3783 	 */
3784 	BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
3785 
3786 	path->slots[0]++;
3787 	if (inode->i_nlink != 1 ||
3788 	    path->slots[0] >= btrfs_header_nritems(leaf))
3789 		goto cache_acl;
3790 
3791 	btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
3792 	if (location.objectid != btrfs_ino(inode))
3793 		goto cache_acl;
3794 
3795 	ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
3796 	if (location.type == BTRFS_INODE_REF_KEY) {
3797 		struct btrfs_inode_ref *ref;
3798 
3799 		ref = (struct btrfs_inode_ref *)ptr;
3800 		BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
3801 	} else if (location.type == BTRFS_INODE_EXTREF_KEY) {
3802 		struct btrfs_inode_extref *extref;
3803 
3804 		extref = (struct btrfs_inode_extref *)ptr;
3805 		BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
3806 								     extref);
3807 	}
3808 cache_acl:
3809 	/*
3810 	 * try to precache a NULL acl entry for files that don't have
3811 	 * any xattrs or acls
3812 	 */
3813 	maybe_acls = acls_after_inode_item(leaf, path->slots[0],
3814 					   btrfs_ino(inode), &first_xattr_slot);
3815 	if (first_xattr_slot != -1) {
3816 		path->slots[0] = first_xattr_slot;
3817 		ret = btrfs_load_inode_props(inode, path);
3818 		if (ret)
3819 			btrfs_err(fs_info,
3820 				  "error loading props for ino %llu (root %llu): %d",
3821 				  btrfs_ino(inode),
3822 				  root->root_key.objectid, ret);
3823 	}
3824 	btrfs_free_path(path);
3825 
3826 	if (!maybe_acls)
3827 		cache_no_acl(inode);
3828 
3829 	switch (inode->i_mode & S_IFMT) {
3830 	case S_IFREG:
3831 		inode->i_mapping->a_ops = &btrfs_aops;
3832 		BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
3833 		inode->i_fop = &btrfs_file_operations;
3834 		inode->i_op = &btrfs_file_inode_operations;
3835 		break;
3836 	case S_IFDIR:
3837 		inode->i_fop = &btrfs_dir_file_operations;
3838 		if (root == fs_info->tree_root)
3839 			inode->i_op = &btrfs_dir_ro_inode_operations;
3840 		else
3841 			inode->i_op = &btrfs_dir_inode_operations;
3842 		break;
3843 	case S_IFLNK:
3844 		inode->i_op = &btrfs_symlink_inode_operations;
3845 		inode_nohighmem(inode);
3846 		inode->i_mapping->a_ops = &btrfs_symlink_aops;
3847 		break;
3848 	default:
3849 		inode->i_op = &btrfs_special_inode_operations;
3850 		init_special_inode(inode, inode->i_mode, rdev);
3851 		break;
3852 	}
3853 
3854 	btrfs_update_iflags(inode);
3855 	return 0;
3856 
3857 make_bad:
3858 	btrfs_free_path(path);
3859 	make_bad_inode(inode);
3860 	return ret;
3861 }
3862 
3863 /*
3864  * given a leaf and an inode, copy the inode fields into the leaf
3865  */
3866 static void fill_inode_item(struct btrfs_trans_handle *trans,
3867 			    struct extent_buffer *leaf,
3868 			    struct btrfs_inode_item *item,
3869 			    struct inode *inode)
3870 {
3871 	struct btrfs_map_token token;
3872 
3873 	btrfs_init_map_token(&token);
3874 
3875 	btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
3876 	btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
3877 	btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
3878 				   &token);
3879 	btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
3880 	btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3881 
3882 	btrfs_set_token_timespec_sec(leaf, &item->atime,
3883 				     inode->i_atime.tv_sec, &token);
3884 	btrfs_set_token_timespec_nsec(leaf, &item->atime,
3885 				      inode->i_atime.tv_nsec, &token);
3886 
3887 	btrfs_set_token_timespec_sec(leaf, &item->mtime,
3888 				     inode->i_mtime.tv_sec, &token);
3889 	btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3890 				      inode->i_mtime.tv_nsec, &token);
3891 
3892 	btrfs_set_token_timespec_sec(leaf, &item->ctime,
3893 				     inode->i_ctime.tv_sec, &token);
3894 	btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3895 				      inode->i_ctime.tv_nsec, &token);
3896 
3897 	btrfs_set_token_timespec_sec(leaf, &item->otime,
3898 				     BTRFS_I(inode)->i_otime.tv_sec, &token);
3899 	btrfs_set_token_timespec_nsec(leaf, &item->otime,
3900 				      BTRFS_I(inode)->i_otime.tv_nsec, &token);
3901 
3902 	btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
3903 				     &token);
3904 	btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
3905 					 &token);
3906 	btrfs_set_token_inode_sequence(leaf, item, inode->i_version, &token);
3907 	btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
3908 	btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
3909 	btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
3910 	btrfs_set_token_inode_block_group(leaf, item, 0, &token);
3911 }
3912 
3913 /*
3914  * copy everything in the in-memory inode into the btree.
3915  */
3916 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
3917 				struct btrfs_root *root, struct inode *inode)
3918 {
3919 	struct btrfs_inode_item *inode_item;
3920 	struct btrfs_path *path;
3921 	struct extent_buffer *leaf;
3922 	int ret;
3923 
3924 	path = btrfs_alloc_path();
3925 	if (!path)
3926 		return -ENOMEM;
3927 
3928 	path->leave_spinning = 1;
3929 	ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
3930 				 1);
3931 	if (ret) {
3932 		if (ret > 0)
3933 			ret = -ENOENT;
3934 		goto failed;
3935 	}
3936 
3937 	leaf = path->nodes[0];
3938 	inode_item = btrfs_item_ptr(leaf, path->slots[0],
3939 				    struct btrfs_inode_item);
3940 
3941 	fill_inode_item(trans, leaf, inode_item, inode);
3942 	btrfs_mark_buffer_dirty(leaf);
3943 	btrfs_set_inode_last_trans(trans, inode);
3944 	ret = 0;
3945 failed:
3946 	btrfs_free_path(path);
3947 	return ret;
3948 }
3949 
3950 /*
3951  * copy everything in the in-memory inode into the btree.
3952  */
3953 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
3954 				struct btrfs_root *root, struct inode *inode)
3955 {
3956 	struct btrfs_fs_info *fs_info = root->fs_info;
3957 	int ret;
3958 
3959 	/*
3960 	 * If the inode is a free space inode, we can deadlock during commit
3961 	 * if we put it into the delayed code.
3962 	 *
3963 	 * The data relocation inode should also be directly updated
3964 	 * without delay
3965 	 */
3966 	if (!btrfs_is_free_space_inode(inode)
3967 	    && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
3968 	    && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
3969 		btrfs_update_root_times(trans, root);
3970 
3971 		ret = btrfs_delayed_update_inode(trans, root, inode);
3972 		if (!ret)
3973 			btrfs_set_inode_last_trans(trans, inode);
3974 		return ret;
3975 	}
3976 
3977 	return btrfs_update_inode_item(trans, root, inode);
3978 }
3979 
3980 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
3981 					 struct btrfs_root *root,
3982 					 struct inode *inode)
3983 {
3984 	int ret;
3985 
3986 	ret = btrfs_update_inode(trans, root, inode);
3987 	if (ret == -ENOSPC)
3988 		return btrfs_update_inode_item(trans, root, inode);
3989 	return ret;
3990 }
3991 
3992 /*
3993  * unlink helper that gets used here in inode.c and in the tree logging
3994  * recovery code.  It remove a link in a directory with a given name, and
3995  * also drops the back refs in the inode to the directory
3996  */
3997 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
3998 				struct btrfs_root *root,
3999 				struct inode *dir, struct inode *inode,
4000 				const char *name, int name_len)
4001 {
4002 	struct btrfs_fs_info *fs_info = root->fs_info;
4003 	struct btrfs_path *path;
4004 	int ret = 0;
4005 	struct extent_buffer *leaf;
4006 	struct btrfs_dir_item *di;
4007 	struct btrfs_key key;
4008 	u64 index;
4009 	u64 ino = btrfs_ino(inode);
4010 	u64 dir_ino = btrfs_ino(dir);
4011 
4012 	path = btrfs_alloc_path();
4013 	if (!path) {
4014 		ret = -ENOMEM;
4015 		goto out;
4016 	}
4017 
4018 	path->leave_spinning = 1;
4019 	di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4020 				    name, name_len, -1);
4021 	if (IS_ERR(di)) {
4022 		ret = PTR_ERR(di);
4023 		goto err;
4024 	}
4025 	if (!di) {
4026 		ret = -ENOENT;
4027 		goto err;
4028 	}
4029 	leaf = path->nodes[0];
4030 	btrfs_dir_item_key_to_cpu(leaf, di, &key);
4031 	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4032 	if (ret)
4033 		goto err;
4034 	btrfs_release_path(path);
4035 
4036 	/*
4037 	 * If we don't have dir index, we have to get it by looking up
4038 	 * the inode ref, since we get the inode ref, remove it directly,
4039 	 * it is unnecessary to do delayed deletion.
4040 	 *
4041 	 * But if we have dir index, needn't search inode ref to get it.
4042 	 * Since the inode ref is close to the inode item, it is better
4043 	 * that we delay to delete it, and just do this deletion when
4044 	 * we update the inode item.
4045 	 */
4046 	if (BTRFS_I(inode)->dir_index) {
4047 		ret = btrfs_delayed_delete_inode_ref(inode);
4048 		if (!ret) {
4049 			index = BTRFS_I(inode)->dir_index;
4050 			goto skip_backref;
4051 		}
4052 	}
4053 
4054 	ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
4055 				  dir_ino, &index);
4056 	if (ret) {
4057 		btrfs_info(fs_info,
4058 			"failed to delete reference to %.*s, inode %llu parent %llu",
4059 			name_len, name, ino, dir_ino);
4060 		btrfs_abort_transaction(trans, ret);
4061 		goto err;
4062 	}
4063 skip_backref:
4064 	ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index);
4065 	if (ret) {
4066 		btrfs_abort_transaction(trans, ret);
4067 		goto err;
4068 	}
4069 
4070 	ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len,
4071 					 inode, dir_ino);
4072 	if (ret != 0 && ret != -ENOENT) {
4073 		btrfs_abort_transaction(trans, ret);
4074 		goto err;
4075 	}
4076 
4077 	ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len,
4078 					   dir, index);
4079 	if (ret == -ENOENT)
4080 		ret = 0;
4081 	else if (ret)
4082 		btrfs_abort_transaction(trans, ret);
4083 err:
4084 	btrfs_free_path(path);
4085 	if (ret)
4086 		goto out;
4087 
4088 	btrfs_i_size_write(dir, dir->i_size - name_len * 2);
4089 	inode_inc_iversion(inode);
4090 	inode_inc_iversion(dir);
4091 	inode->i_ctime = dir->i_mtime =
4092 		dir->i_ctime = current_time(inode);
4093 	ret = btrfs_update_inode(trans, root, dir);
4094 out:
4095 	return ret;
4096 }
4097 
4098 int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4099 		       struct btrfs_root *root,
4100 		       struct inode *dir, struct inode *inode,
4101 		       const char *name, int name_len)
4102 {
4103 	int ret;
4104 	ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
4105 	if (!ret) {
4106 		drop_nlink(inode);
4107 		ret = btrfs_update_inode(trans, root, inode);
4108 	}
4109 	return ret;
4110 }
4111 
4112 /*
4113  * helper to start transaction for unlink and rmdir.
4114  *
4115  * unlink and rmdir are special in btrfs, they do not always free space, so
4116  * if we cannot make our reservations the normal way try and see if there is
4117  * plenty of slack room in the global reserve to migrate, otherwise we cannot
4118  * allow the unlink to occur.
4119  */
4120 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
4121 {
4122 	struct btrfs_root *root = BTRFS_I(dir)->root;
4123 
4124 	/*
4125 	 * 1 for the possible orphan item
4126 	 * 1 for the dir item
4127 	 * 1 for the dir index
4128 	 * 1 for the inode ref
4129 	 * 1 for the inode
4130 	 */
4131 	return btrfs_start_transaction_fallback_global_rsv(root, 5, 5);
4132 }
4133 
4134 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4135 {
4136 	struct btrfs_root *root = BTRFS_I(dir)->root;
4137 	struct btrfs_trans_handle *trans;
4138 	struct inode *inode = d_inode(dentry);
4139 	int ret;
4140 
4141 	trans = __unlink_start_trans(dir);
4142 	if (IS_ERR(trans))
4143 		return PTR_ERR(trans);
4144 
4145 	btrfs_record_unlink_dir(trans, dir, d_inode(dentry), 0);
4146 
4147 	ret = btrfs_unlink_inode(trans, root, dir, d_inode(dentry),
4148 				 dentry->d_name.name, dentry->d_name.len);
4149 	if (ret)
4150 		goto out;
4151 
4152 	if (inode->i_nlink == 0) {
4153 		ret = btrfs_orphan_add(trans, inode);
4154 		if (ret)
4155 			goto out;
4156 	}
4157 
4158 out:
4159 	btrfs_end_transaction(trans);
4160 	btrfs_btree_balance_dirty(root->fs_info);
4161 	return ret;
4162 }
4163 
4164 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4165 			struct btrfs_root *root,
4166 			struct inode *dir, u64 objectid,
4167 			const char *name, int name_len)
4168 {
4169 	struct btrfs_fs_info *fs_info = root->fs_info;
4170 	struct btrfs_path *path;
4171 	struct extent_buffer *leaf;
4172 	struct btrfs_dir_item *di;
4173 	struct btrfs_key key;
4174 	u64 index;
4175 	int ret;
4176 	u64 dir_ino = btrfs_ino(dir);
4177 
4178 	path = btrfs_alloc_path();
4179 	if (!path)
4180 		return -ENOMEM;
4181 
4182 	di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4183 				   name, name_len, -1);
4184 	if (IS_ERR_OR_NULL(di)) {
4185 		if (!di)
4186 			ret = -ENOENT;
4187 		else
4188 			ret = PTR_ERR(di);
4189 		goto out;
4190 	}
4191 
4192 	leaf = path->nodes[0];
4193 	btrfs_dir_item_key_to_cpu(leaf, di, &key);
4194 	WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4195 	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4196 	if (ret) {
4197 		btrfs_abort_transaction(trans, ret);
4198 		goto out;
4199 	}
4200 	btrfs_release_path(path);
4201 
4202 	ret = btrfs_del_root_ref(trans, fs_info, objectid,
4203 				 root->root_key.objectid, dir_ino,
4204 				 &index, name, name_len);
4205 	if (ret < 0) {
4206 		if (ret != -ENOENT) {
4207 			btrfs_abort_transaction(trans, ret);
4208 			goto out;
4209 		}
4210 		di = btrfs_search_dir_index_item(root, path, dir_ino,
4211 						 name, name_len);
4212 		if (IS_ERR_OR_NULL(di)) {
4213 			if (!di)
4214 				ret = -ENOENT;
4215 			else
4216 				ret = PTR_ERR(di);
4217 			btrfs_abort_transaction(trans, ret);
4218 			goto out;
4219 		}
4220 
4221 		leaf = path->nodes[0];
4222 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4223 		btrfs_release_path(path);
4224 		index = key.offset;
4225 	}
4226 	btrfs_release_path(path);
4227 
4228 	ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index);
4229 	if (ret) {
4230 		btrfs_abort_transaction(trans, ret);
4231 		goto out;
4232 	}
4233 
4234 	btrfs_i_size_write(dir, dir->i_size - name_len * 2);
4235 	inode_inc_iversion(dir);
4236 	dir->i_mtime = dir->i_ctime = current_time(dir);
4237 	ret = btrfs_update_inode_fallback(trans, root, dir);
4238 	if (ret)
4239 		btrfs_abort_transaction(trans, ret);
4240 out:
4241 	btrfs_free_path(path);
4242 	return ret;
4243 }
4244 
4245 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
4246 {
4247 	struct inode *inode = d_inode(dentry);
4248 	int err = 0;
4249 	struct btrfs_root *root = BTRFS_I(dir)->root;
4250 	struct btrfs_trans_handle *trans;
4251 	u64 last_unlink_trans;
4252 
4253 	if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
4254 		return -ENOTEMPTY;
4255 	if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID)
4256 		return -EPERM;
4257 
4258 	trans = __unlink_start_trans(dir);
4259 	if (IS_ERR(trans))
4260 		return PTR_ERR(trans);
4261 
4262 	if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4263 		err = btrfs_unlink_subvol(trans, root, dir,
4264 					  BTRFS_I(inode)->location.objectid,
4265 					  dentry->d_name.name,
4266 					  dentry->d_name.len);
4267 		goto out;
4268 	}
4269 
4270 	err = btrfs_orphan_add(trans, inode);
4271 	if (err)
4272 		goto out;
4273 
4274 	last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
4275 
4276 	/* now the directory is empty */
4277 	err = btrfs_unlink_inode(trans, root, dir, d_inode(dentry),
4278 				 dentry->d_name.name, dentry->d_name.len);
4279 	if (!err) {
4280 		btrfs_i_size_write(inode, 0);
4281 		/*
4282 		 * Propagate the last_unlink_trans value of the deleted dir to
4283 		 * its parent directory. This is to prevent an unrecoverable
4284 		 * log tree in the case we do something like this:
4285 		 * 1) create dir foo
4286 		 * 2) create snapshot under dir foo
4287 		 * 3) delete the snapshot
4288 		 * 4) rmdir foo
4289 		 * 5) mkdir foo
4290 		 * 6) fsync foo or some file inside foo
4291 		 */
4292 		if (last_unlink_trans >= trans->transid)
4293 			BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
4294 	}
4295 out:
4296 	btrfs_end_transaction(trans);
4297 	btrfs_btree_balance_dirty(root->fs_info);
4298 
4299 	return err;
4300 }
4301 
4302 static int truncate_space_check(struct btrfs_trans_handle *trans,
4303 				struct btrfs_root *root,
4304 				u64 bytes_deleted)
4305 {
4306 	struct btrfs_fs_info *fs_info = root->fs_info;
4307 	int ret;
4308 
4309 	/*
4310 	 * This is only used to apply pressure to the enospc system, we don't
4311 	 * intend to use this reservation at all.
4312 	 */
4313 	bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted);
4314 	bytes_deleted *= fs_info->nodesize;
4315 	ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv,
4316 				  bytes_deleted, BTRFS_RESERVE_NO_FLUSH);
4317 	if (!ret) {
4318 		trace_btrfs_space_reservation(fs_info, "transaction",
4319 					      trans->transid,
4320 					      bytes_deleted, 1);
4321 		trans->bytes_reserved += bytes_deleted;
4322 	}
4323 	return ret;
4324 
4325 }
4326 
4327 static int truncate_inline_extent(struct inode *inode,
4328 				  struct btrfs_path *path,
4329 				  struct btrfs_key *found_key,
4330 				  const u64 item_end,
4331 				  const u64 new_size)
4332 {
4333 	struct extent_buffer *leaf = path->nodes[0];
4334 	int slot = path->slots[0];
4335 	struct btrfs_file_extent_item *fi;
4336 	u32 size = (u32)(new_size - found_key->offset);
4337 	struct btrfs_root *root = BTRFS_I(inode)->root;
4338 
4339 	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
4340 
4341 	if (btrfs_file_extent_compression(leaf, fi) != BTRFS_COMPRESS_NONE) {
4342 		loff_t offset = new_size;
4343 		loff_t page_end = ALIGN(offset, PAGE_SIZE);
4344 
4345 		/*
4346 		 * Zero out the remaining of the last page of our inline extent,
4347 		 * instead of directly truncating our inline extent here - that
4348 		 * would be much more complex (decompressing all the data, then
4349 		 * compressing the truncated data, which might be bigger than
4350 		 * the size of the inline extent, resize the extent, etc).
4351 		 * We release the path because to get the page we might need to
4352 		 * read the extent item from disk (data not in the page cache).
4353 		 */
4354 		btrfs_release_path(path);
4355 		return btrfs_truncate_block(inode, offset, page_end - offset,
4356 					0);
4357 	}
4358 
4359 	btrfs_set_file_extent_ram_bytes(leaf, fi, size);
4360 	size = btrfs_file_extent_calc_inline_size(size);
4361 	btrfs_truncate_item(root->fs_info, path, size, 1);
4362 
4363 	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4364 		inode_sub_bytes(inode, item_end + 1 - new_size);
4365 
4366 	return 0;
4367 }
4368 
4369 /*
4370  * this can truncate away extent items, csum items and directory items.
4371  * It starts at a high offset and removes keys until it can't find
4372  * any higher than new_size
4373  *
4374  * csum items that cross the new i_size are truncated to the new size
4375  * as well.
4376  *
4377  * min_type is the minimum key type to truncate down to.  If set to 0, this
4378  * will kill all the items on this inode, including the INODE_ITEM_KEY.
4379  */
4380 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
4381 			       struct btrfs_root *root,
4382 			       struct inode *inode,
4383 			       u64 new_size, u32 min_type)
4384 {
4385 	struct btrfs_fs_info *fs_info = root->fs_info;
4386 	struct btrfs_path *path;
4387 	struct extent_buffer *leaf;
4388 	struct btrfs_file_extent_item *fi;
4389 	struct btrfs_key key;
4390 	struct btrfs_key found_key;
4391 	u64 extent_start = 0;
4392 	u64 extent_num_bytes = 0;
4393 	u64 extent_offset = 0;
4394 	u64 item_end = 0;
4395 	u64 last_size = new_size;
4396 	u32 found_type = (u8)-1;
4397 	int found_extent;
4398 	int del_item;
4399 	int pending_del_nr = 0;
4400 	int pending_del_slot = 0;
4401 	int extent_type = -1;
4402 	int ret;
4403 	int err = 0;
4404 	u64 ino = btrfs_ino(inode);
4405 	u64 bytes_deleted = 0;
4406 	bool be_nice = 0;
4407 	bool should_throttle = 0;
4408 	bool should_end = 0;
4409 
4410 	BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
4411 
4412 	/*
4413 	 * for non-free space inodes and ref cows, we want to back off from
4414 	 * time to time
4415 	 */
4416 	if (!btrfs_is_free_space_inode(inode) &&
4417 	    test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4418 		be_nice = 1;
4419 
4420 	path = btrfs_alloc_path();
4421 	if (!path)
4422 		return -ENOMEM;
4423 	path->reada = READA_BACK;
4424 
4425 	/*
4426 	 * We want to drop from the next block forward in case this new size is
4427 	 * not block aligned since we will be keeping the last block of the
4428 	 * extent just the way it is.
4429 	 */
4430 	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4431 	    root == fs_info->tree_root)
4432 		btrfs_drop_extent_cache(inode, ALIGN(new_size,
4433 					fs_info->sectorsize),
4434 					(u64)-1, 0);
4435 
4436 	/*
4437 	 * This function is also used to drop the items in the log tree before
4438 	 * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4439 	 * it is used to drop the loged items. So we shouldn't kill the delayed
4440 	 * items.
4441 	 */
4442 	if (min_type == 0 && root == BTRFS_I(inode)->root)
4443 		btrfs_kill_delayed_inode_items(inode);
4444 
4445 	key.objectid = ino;
4446 	key.offset = (u64)-1;
4447 	key.type = (u8)-1;
4448 
4449 search_again:
4450 	/*
4451 	 * with a 16K leaf size and 128MB extents, you can actually queue
4452 	 * up a huge file in a single leaf.  Most of the time that
4453 	 * bytes_deleted is > 0, it will be huge by the time we get here
4454 	 */
4455 	if (be_nice && bytes_deleted > SZ_32M) {
4456 		if (btrfs_should_end_transaction(trans)) {
4457 			err = -EAGAIN;
4458 			goto error;
4459 		}
4460 	}
4461 
4462 
4463 	path->leave_spinning = 1;
4464 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
4465 	if (ret < 0) {
4466 		err = ret;
4467 		goto out;
4468 	}
4469 
4470 	if (ret > 0) {
4471 		/* there are no items in the tree for us to truncate, we're
4472 		 * done
4473 		 */
4474 		if (path->slots[0] == 0)
4475 			goto out;
4476 		path->slots[0]--;
4477 	}
4478 
4479 	while (1) {
4480 		fi = NULL;
4481 		leaf = path->nodes[0];
4482 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
4483 		found_type = found_key.type;
4484 
4485 		if (found_key.objectid != ino)
4486 			break;
4487 
4488 		if (found_type < min_type)
4489 			break;
4490 
4491 		item_end = found_key.offset;
4492 		if (found_type == BTRFS_EXTENT_DATA_KEY) {
4493 			fi = btrfs_item_ptr(leaf, path->slots[0],
4494 					    struct btrfs_file_extent_item);
4495 			extent_type = btrfs_file_extent_type(leaf, fi);
4496 			if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4497 				item_end +=
4498 				    btrfs_file_extent_num_bytes(leaf, fi);
4499 			} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4500 				item_end += btrfs_file_extent_inline_len(leaf,
4501 							 path->slots[0], fi);
4502 			}
4503 			item_end--;
4504 		}
4505 		if (found_type > min_type) {
4506 			del_item = 1;
4507 		} else {
4508 			if (item_end < new_size)
4509 				break;
4510 			if (found_key.offset >= new_size)
4511 				del_item = 1;
4512 			else
4513 				del_item = 0;
4514 		}
4515 		found_extent = 0;
4516 		/* FIXME, shrink the extent if the ref count is only 1 */
4517 		if (found_type != BTRFS_EXTENT_DATA_KEY)
4518 			goto delete;
4519 
4520 		if (del_item)
4521 			last_size = found_key.offset;
4522 		else
4523 			last_size = new_size;
4524 
4525 		if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4526 			u64 num_dec;
4527 			extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
4528 			if (!del_item) {
4529 				u64 orig_num_bytes =
4530 					btrfs_file_extent_num_bytes(leaf, fi);
4531 				extent_num_bytes = ALIGN(new_size -
4532 						found_key.offset,
4533 						fs_info->sectorsize);
4534 				btrfs_set_file_extent_num_bytes(leaf, fi,
4535 							 extent_num_bytes);
4536 				num_dec = (orig_num_bytes -
4537 					   extent_num_bytes);
4538 				if (test_bit(BTRFS_ROOT_REF_COWS,
4539 					     &root->state) &&
4540 				    extent_start != 0)
4541 					inode_sub_bytes(inode, num_dec);
4542 				btrfs_mark_buffer_dirty(leaf);
4543 			} else {
4544 				extent_num_bytes =
4545 					btrfs_file_extent_disk_num_bytes(leaf,
4546 									 fi);
4547 				extent_offset = found_key.offset -
4548 					btrfs_file_extent_offset(leaf, fi);
4549 
4550 				/* FIXME blocksize != 4096 */
4551 				num_dec = btrfs_file_extent_num_bytes(leaf, fi);
4552 				if (extent_start != 0) {
4553 					found_extent = 1;
4554 					if (test_bit(BTRFS_ROOT_REF_COWS,
4555 						     &root->state))
4556 						inode_sub_bytes(inode, num_dec);
4557 				}
4558 			}
4559 		} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4560 			/*
4561 			 * we can't truncate inline items that have had
4562 			 * special encodings
4563 			 */
4564 			if (!del_item &&
4565 			    btrfs_file_extent_encryption(leaf, fi) == 0 &&
4566 			    btrfs_file_extent_other_encoding(leaf, fi) == 0) {
4567 
4568 				/*
4569 				 * Need to release path in order to truncate a
4570 				 * compressed extent. So delete any accumulated
4571 				 * extent items so far.
4572 				 */
4573 				if (btrfs_file_extent_compression(leaf, fi) !=
4574 				    BTRFS_COMPRESS_NONE && pending_del_nr) {
4575 					err = btrfs_del_items(trans, root, path,
4576 							      pending_del_slot,
4577 							      pending_del_nr);
4578 					if (err) {
4579 						btrfs_abort_transaction(trans,
4580 									err);
4581 						goto error;
4582 					}
4583 					pending_del_nr = 0;
4584 				}
4585 
4586 				err = truncate_inline_extent(inode, path,
4587 							     &found_key,
4588 							     item_end,
4589 							     new_size);
4590 				if (err) {
4591 					btrfs_abort_transaction(trans, err);
4592 					goto error;
4593 				}
4594 			} else if (test_bit(BTRFS_ROOT_REF_COWS,
4595 					    &root->state)) {
4596 				inode_sub_bytes(inode, item_end + 1 - new_size);
4597 			}
4598 		}
4599 delete:
4600 		if (del_item) {
4601 			if (!pending_del_nr) {
4602 				/* no pending yet, add ourselves */
4603 				pending_del_slot = path->slots[0];
4604 				pending_del_nr = 1;
4605 			} else if (pending_del_nr &&
4606 				   path->slots[0] + 1 == pending_del_slot) {
4607 				/* hop on the pending chunk */
4608 				pending_del_nr++;
4609 				pending_del_slot = path->slots[0];
4610 			} else {
4611 				BUG();
4612 			}
4613 		} else {
4614 			break;
4615 		}
4616 		should_throttle = 0;
4617 
4618 		if (found_extent &&
4619 		    (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4620 		     root == fs_info->tree_root)) {
4621 			btrfs_set_path_blocking(path);
4622 			bytes_deleted += extent_num_bytes;
4623 			ret = btrfs_free_extent(trans, fs_info, extent_start,
4624 						extent_num_bytes, 0,
4625 						btrfs_header_owner(leaf),
4626 						ino, extent_offset);
4627 			BUG_ON(ret);
4628 			if (btrfs_should_throttle_delayed_refs(trans, fs_info))
4629 				btrfs_async_run_delayed_refs(fs_info,
4630 					trans->delayed_ref_updates * 2,
4631 					trans->transid, 0);
4632 			if (be_nice) {
4633 				if (truncate_space_check(trans, root,
4634 							 extent_num_bytes)) {
4635 					should_end = 1;
4636 				}
4637 				if (btrfs_should_throttle_delayed_refs(trans,
4638 								       fs_info))
4639 					should_throttle = 1;
4640 			}
4641 		}
4642 
4643 		if (found_type == BTRFS_INODE_ITEM_KEY)
4644 			break;
4645 
4646 		if (path->slots[0] == 0 ||
4647 		    path->slots[0] != pending_del_slot ||
4648 		    should_throttle || should_end) {
4649 			if (pending_del_nr) {
4650 				ret = btrfs_del_items(trans, root, path,
4651 						pending_del_slot,
4652 						pending_del_nr);
4653 				if (ret) {
4654 					btrfs_abort_transaction(trans, ret);
4655 					goto error;
4656 				}
4657 				pending_del_nr = 0;
4658 			}
4659 			btrfs_release_path(path);
4660 			if (should_throttle) {
4661 				unsigned long updates = trans->delayed_ref_updates;
4662 				if (updates) {
4663 					trans->delayed_ref_updates = 0;
4664 					ret = btrfs_run_delayed_refs(trans,
4665 								   fs_info,
4666 								   updates * 2);
4667 					if (ret && !err)
4668 						err = ret;
4669 				}
4670 			}
4671 			/*
4672 			 * if we failed to refill our space rsv, bail out
4673 			 * and let the transaction restart
4674 			 */
4675 			if (should_end) {
4676 				err = -EAGAIN;
4677 				goto error;
4678 			}
4679 			goto search_again;
4680 		} else {
4681 			path->slots[0]--;
4682 		}
4683 	}
4684 out:
4685 	if (pending_del_nr) {
4686 		ret = btrfs_del_items(trans, root, path, pending_del_slot,
4687 				      pending_del_nr);
4688 		if (ret)
4689 			btrfs_abort_transaction(trans, ret);
4690 	}
4691 error:
4692 	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID)
4693 		btrfs_ordered_update_i_size(inode, last_size, NULL);
4694 
4695 	btrfs_free_path(path);
4696 
4697 	if (be_nice && bytes_deleted > SZ_32M) {
4698 		unsigned long updates = trans->delayed_ref_updates;
4699 		if (updates) {
4700 			trans->delayed_ref_updates = 0;
4701 			ret = btrfs_run_delayed_refs(trans, fs_info,
4702 						     updates * 2);
4703 			if (ret && !err)
4704 				err = ret;
4705 		}
4706 	}
4707 	return err;
4708 }
4709 
4710 /*
4711  * btrfs_truncate_block - read, zero a chunk and write a block
4712  * @inode - inode that we're zeroing
4713  * @from - the offset to start zeroing
4714  * @len - the length to zero, 0 to zero the entire range respective to the
4715  *	offset
4716  * @front - zero up to the offset instead of from the offset on
4717  *
4718  * This will find the block for the "from" offset and cow the block and zero the
4719  * part we want to zero.  This is used with truncate and hole punching.
4720  */
4721 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
4722 			int front)
4723 {
4724 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4725 	struct address_space *mapping = inode->i_mapping;
4726 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4727 	struct btrfs_ordered_extent *ordered;
4728 	struct extent_state *cached_state = NULL;
4729 	char *kaddr;
4730 	u32 blocksize = fs_info->sectorsize;
4731 	pgoff_t index = from >> PAGE_SHIFT;
4732 	unsigned offset = from & (blocksize - 1);
4733 	struct page *page;
4734 	gfp_t mask = btrfs_alloc_write_mask(mapping);
4735 	int ret = 0;
4736 	u64 block_start;
4737 	u64 block_end;
4738 
4739 	if ((offset & (blocksize - 1)) == 0 &&
4740 	    (!len || ((len & (blocksize - 1)) == 0)))
4741 		goto out;
4742 
4743 	ret = btrfs_delalloc_reserve_space(inode,
4744 			round_down(from, blocksize), blocksize);
4745 	if (ret)
4746 		goto out;
4747 
4748 again:
4749 	page = find_or_create_page(mapping, index, mask);
4750 	if (!page) {
4751 		btrfs_delalloc_release_space(inode,
4752 				round_down(from, blocksize),
4753 				blocksize);
4754 		ret = -ENOMEM;
4755 		goto out;
4756 	}
4757 
4758 	block_start = round_down(from, blocksize);
4759 	block_end = block_start + blocksize - 1;
4760 
4761 	if (!PageUptodate(page)) {
4762 		ret = btrfs_readpage(NULL, page);
4763 		lock_page(page);
4764 		if (page->mapping != mapping) {
4765 			unlock_page(page);
4766 			put_page(page);
4767 			goto again;
4768 		}
4769 		if (!PageUptodate(page)) {
4770 			ret = -EIO;
4771 			goto out_unlock;
4772 		}
4773 	}
4774 	wait_on_page_writeback(page);
4775 
4776 	lock_extent_bits(io_tree, block_start, block_end, &cached_state);
4777 	set_page_extent_mapped(page);
4778 
4779 	ordered = btrfs_lookup_ordered_extent(inode, block_start);
4780 	if (ordered) {
4781 		unlock_extent_cached(io_tree, block_start, block_end,
4782 				     &cached_state, GFP_NOFS);
4783 		unlock_page(page);
4784 		put_page(page);
4785 		btrfs_start_ordered_extent(inode, ordered, 1);
4786 		btrfs_put_ordered_extent(ordered);
4787 		goto again;
4788 	}
4789 
4790 	clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
4791 			  EXTENT_DIRTY | EXTENT_DELALLOC |
4792 			  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
4793 			  0, 0, &cached_state, GFP_NOFS);
4794 
4795 	ret = btrfs_set_extent_delalloc(inode, block_start, block_end,
4796 					&cached_state, 0);
4797 	if (ret) {
4798 		unlock_extent_cached(io_tree, block_start, block_end,
4799 				     &cached_state, GFP_NOFS);
4800 		goto out_unlock;
4801 	}
4802 
4803 	if (offset != blocksize) {
4804 		if (!len)
4805 			len = blocksize - offset;
4806 		kaddr = kmap(page);
4807 		if (front)
4808 			memset(kaddr + (block_start - page_offset(page)),
4809 				0, offset);
4810 		else
4811 			memset(kaddr + (block_start - page_offset(page)) +  offset,
4812 				0, len);
4813 		flush_dcache_page(page);
4814 		kunmap(page);
4815 	}
4816 	ClearPageChecked(page);
4817 	set_page_dirty(page);
4818 	unlock_extent_cached(io_tree, block_start, block_end, &cached_state,
4819 			     GFP_NOFS);
4820 
4821 out_unlock:
4822 	if (ret)
4823 		btrfs_delalloc_release_space(inode, block_start,
4824 					     blocksize);
4825 	unlock_page(page);
4826 	put_page(page);
4827 out:
4828 	return ret;
4829 }
4830 
4831 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
4832 			     u64 offset, u64 len)
4833 {
4834 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4835 	struct btrfs_trans_handle *trans;
4836 	int ret;
4837 
4838 	/*
4839 	 * Still need to make sure the inode looks like it's been updated so
4840 	 * that any holes get logged if we fsync.
4841 	 */
4842 	if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
4843 		BTRFS_I(inode)->last_trans = fs_info->generation;
4844 		BTRFS_I(inode)->last_sub_trans = root->log_transid;
4845 		BTRFS_I(inode)->last_log_commit = root->last_log_commit;
4846 		return 0;
4847 	}
4848 
4849 	/*
4850 	 * 1 - for the one we're dropping
4851 	 * 1 - for the one we're adding
4852 	 * 1 - for updating the inode.
4853 	 */
4854 	trans = btrfs_start_transaction(root, 3);
4855 	if (IS_ERR(trans))
4856 		return PTR_ERR(trans);
4857 
4858 	ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
4859 	if (ret) {
4860 		btrfs_abort_transaction(trans, ret);
4861 		btrfs_end_transaction(trans);
4862 		return ret;
4863 	}
4864 
4865 	ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), offset,
4866 				       0, 0, len, 0, len, 0, 0, 0);
4867 	if (ret)
4868 		btrfs_abort_transaction(trans, ret);
4869 	else
4870 		btrfs_update_inode(trans, root, inode);
4871 	btrfs_end_transaction(trans);
4872 	return ret;
4873 }
4874 
4875 /*
4876  * This function puts in dummy file extents for the area we're creating a hole
4877  * for.  So if we are truncating this file to a larger size we need to insert
4878  * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4879  * the range between oldsize and size
4880  */
4881 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
4882 {
4883 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4884 	struct btrfs_root *root = BTRFS_I(inode)->root;
4885 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4886 	struct extent_map *em = NULL;
4887 	struct extent_state *cached_state = NULL;
4888 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
4889 	u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
4890 	u64 block_end = ALIGN(size, fs_info->sectorsize);
4891 	u64 last_byte;
4892 	u64 cur_offset;
4893 	u64 hole_size;
4894 	int err = 0;
4895 
4896 	/*
4897 	 * If our size started in the middle of a block we need to zero out the
4898 	 * rest of the block before we expand the i_size, otherwise we could
4899 	 * expose stale data.
4900 	 */
4901 	err = btrfs_truncate_block(inode, oldsize, 0, 0);
4902 	if (err)
4903 		return err;
4904 
4905 	if (size <= hole_start)
4906 		return 0;
4907 
4908 	while (1) {
4909 		struct btrfs_ordered_extent *ordered;
4910 
4911 		lock_extent_bits(io_tree, hole_start, block_end - 1,
4912 				 &cached_state);
4913 		ordered = btrfs_lookup_ordered_range(inode, hole_start,
4914 						     block_end - hole_start);
4915 		if (!ordered)
4916 			break;
4917 		unlock_extent_cached(io_tree, hole_start, block_end - 1,
4918 				     &cached_state, GFP_NOFS);
4919 		btrfs_start_ordered_extent(inode, ordered, 1);
4920 		btrfs_put_ordered_extent(ordered);
4921 	}
4922 
4923 	cur_offset = hole_start;
4924 	while (1) {
4925 		em = btrfs_get_extent(inode, NULL, 0, cur_offset,
4926 				block_end - cur_offset, 0);
4927 		if (IS_ERR(em)) {
4928 			err = PTR_ERR(em);
4929 			em = NULL;
4930 			break;
4931 		}
4932 		last_byte = min(extent_map_end(em), block_end);
4933 		last_byte = ALIGN(last_byte, fs_info->sectorsize);
4934 		if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
4935 			struct extent_map *hole_em;
4936 			hole_size = last_byte - cur_offset;
4937 
4938 			err = maybe_insert_hole(root, inode, cur_offset,
4939 						hole_size);
4940 			if (err)
4941 				break;
4942 			btrfs_drop_extent_cache(inode, cur_offset,
4943 						cur_offset + hole_size - 1, 0);
4944 			hole_em = alloc_extent_map();
4945 			if (!hole_em) {
4946 				set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4947 					&BTRFS_I(inode)->runtime_flags);
4948 				goto next;
4949 			}
4950 			hole_em->start = cur_offset;
4951 			hole_em->len = hole_size;
4952 			hole_em->orig_start = cur_offset;
4953 
4954 			hole_em->block_start = EXTENT_MAP_HOLE;
4955 			hole_em->block_len = 0;
4956 			hole_em->orig_block_len = 0;
4957 			hole_em->ram_bytes = hole_size;
4958 			hole_em->bdev = fs_info->fs_devices->latest_bdev;
4959 			hole_em->compress_type = BTRFS_COMPRESS_NONE;
4960 			hole_em->generation = fs_info->generation;
4961 
4962 			while (1) {
4963 				write_lock(&em_tree->lock);
4964 				err = add_extent_mapping(em_tree, hole_em, 1);
4965 				write_unlock(&em_tree->lock);
4966 				if (err != -EEXIST)
4967 					break;
4968 				btrfs_drop_extent_cache(inode, cur_offset,
4969 							cur_offset +
4970 							hole_size - 1, 0);
4971 			}
4972 			free_extent_map(hole_em);
4973 		}
4974 next:
4975 		free_extent_map(em);
4976 		em = NULL;
4977 		cur_offset = last_byte;
4978 		if (cur_offset >= block_end)
4979 			break;
4980 	}
4981 	free_extent_map(em);
4982 	unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state,
4983 			     GFP_NOFS);
4984 	return err;
4985 }
4986 
4987 static int btrfs_setsize(struct inode *inode, struct iattr *attr)
4988 {
4989 	struct btrfs_root *root = BTRFS_I(inode)->root;
4990 	struct btrfs_trans_handle *trans;
4991 	loff_t oldsize = i_size_read(inode);
4992 	loff_t newsize = attr->ia_size;
4993 	int mask = attr->ia_valid;
4994 	int ret;
4995 
4996 	/*
4997 	 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
4998 	 * special case where we need to update the times despite not having
4999 	 * these flags set.  For all other operations the VFS set these flags
5000 	 * explicitly if it wants a timestamp update.
5001 	 */
5002 	if (newsize != oldsize) {
5003 		inode_inc_iversion(inode);
5004 		if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
5005 			inode->i_ctime = inode->i_mtime =
5006 				current_time(inode);
5007 	}
5008 
5009 	if (newsize > oldsize) {
5010 		/*
5011 		 * Don't do an expanding truncate while snapshoting is ongoing.
5012 		 * This is to ensure the snapshot captures a fully consistent
5013 		 * state of this file - if the snapshot captures this expanding
5014 		 * truncation, it must capture all writes that happened before
5015 		 * this truncation.
5016 		 */
5017 		btrfs_wait_for_snapshot_creation(root);
5018 		ret = btrfs_cont_expand(inode, oldsize, newsize);
5019 		if (ret) {
5020 			btrfs_end_write_no_snapshoting(root);
5021 			return ret;
5022 		}
5023 
5024 		trans = btrfs_start_transaction(root, 1);
5025 		if (IS_ERR(trans)) {
5026 			btrfs_end_write_no_snapshoting(root);
5027 			return PTR_ERR(trans);
5028 		}
5029 
5030 		i_size_write(inode, newsize);
5031 		btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
5032 		pagecache_isize_extended(inode, oldsize, newsize);
5033 		ret = btrfs_update_inode(trans, root, inode);
5034 		btrfs_end_write_no_snapshoting(root);
5035 		btrfs_end_transaction(trans);
5036 	} else {
5037 
5038 		/*
5039 		 * We're truncating a file that used to have good data down to
5040 		 * zero. Make sure it gets into the ordered flush list so that
5041 		 * any new writes get down to disk quickly.
5042 		 */
5043 		if (newsize == 0)
5044 			set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
5045 				&BTRFS_I(inode)->runtime_flags);
5046 
5047 		/*
5048 		 * 1 for the orphan item we're going to add
5049 		 * 1 for the orphan item deletion.
5050 		 */
5051 		trans = btrfs_start_transaction(root, 2);
5052 		if (IS_ERR(trans))
5053 			return PTR_ERR(trans);
5054 
5055 		/*
5056 		 * We need to do this in case we fail at _any_ point during the
5057 		 * actual truncate.  Once we do the truncate_setsize we could
5058 		 * invalidate pages which forces any outstanding ordered io to
5059 		 * be instantly completed which will give us extents that need
5060 		 * to be truncated.  If we fail to get an orphan inode down we
5061 		 * could have left over extents that were never meant to live,
5062 		 * so we need to guarantee from this point on that everything
5063 		 * will be consistent.
5064 		 */
5065 		ret = btrfs_orphan_add(trans, inode);
5066 		btrfs_end_transaction(trans);
5067 		if (ret)
5068 			return ret;
5069 
5070 		/* we don't support swapfiles, so vmtruncate shouldn't fail */
5071 		truncate_setsize(inode, newsize);
5072 
5073 		/* Disable nonlocked read DIO to avoid the end less truncate */
5074 		btrfs_inode_block_unlocked_dio(inode);
5075 		inode_dio_wait(inode);
5076 		btrfs_inode_resume_unlocked_dio(inode);
5077 
5078 		ret = btrfs_truncate(inode);
5079 		if (ret && inode->i_nlink) {
5080 			int err;
5081 
5082 			/*
5083 			 * failed to truncate, disk_i_size is only adjusted down
5084 			 * as we remove extents, so it should represent the true
5085 			 * size of the inode, so reset the in memory size and
5086 			 * delete our orphan entry.
5087 			 */
5088 			trans = btrfs_join_transaction(root);
5089 			if (IS_ERR(trans)) {
5090 				btrfs_orphan_del(NULL, inode);
5091 				return ret;
5092 			}
5093 			i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5094 			err = btrfs_orphan_del(trans, inode);
5095 			if (err)
5096 				btrfs_abort_transaction(trans, err);
5097 			btrfs_end_transaction(trans);
5098 		}
5099 	}
5100 
5101 	return ret;
5102 }
5103 
5104 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
5105 {
5106 	struct inode *inode = d_inode(dentry);
5107 	struct btrfs_root *root = BTRFS_I(inode)->root;
5108 	int err;
5109 
5110 	if (btrfs_root_readonly(root))
5111 		return -EROFS;
5112 
5113 	err = setattr_prepare(dentry, attr);
5114 	if (err)
5115 		return err;
5116 
5117 	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5118 		err = btrfs_setsize(inode, attr);
5119 		if (err)
5120 			return err;
5121 	}
5122 
5123 	if (attr->ia_valid) {
5124 		setattr_copy(inode, attr);
5125 		inode_inc_iversion(inode);
5126 		err = btrfs_dirty_inode(inode);
5127 
5128 		if (!err && attr->ia_valid & ATTR_MODE)
5129 			err = posix_acl_chmod(inode, inode->i_mode);
5130 	}
5131 
5132 	return err;
5133 }
5134 
5135 /*
5136  * While truncating the inode pages during eviction, we get the VFS calling
5137  * btrfs_invalidatepage() against each page of the inode. This is slow because
5138  * the calls to btrfs_invalidatepage() result in a huge amount of calls to
5139  * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
5140  * extent_state structures over and over, wasting lots of time.
5141  *
5142  * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
5143  * those expensive operations on a per page basis and do only the ordered io
5144  * finishing, while we release here the extent_map and extent_state structures,
5145  * without the excessive merging and splitting.
5146  */
5147 static void evict_inode_truncate_pages(struct inode *inode)
5148 {
5149 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5150 	struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
5151 	struct rb_node *node;
5152 
5153 	ASSERT(inode->i_state & I_FREEING);
5154 	truncate_inode_pages_final(&inode->i_data);
5155 
5156 	write_lock(&map_tree->lock);
5157 	while (!RB_EMPTY_ROOT(&map_tree->map)) {
5158 		struct extent_map *em;
5159 
5160 		node = rb_first(&map_tree->map);
5161 		em = rb_entry(node, struct extent_map, rb_node);
5162 		clear_bit(EXTENT_FLAG_PINNED, &em->flags);
5163 		clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
5164 		remove_extent_mapping(map_tree, em);
5165 		free_extent_map(em);
5166 		if (need_resched()) {
5167 			write_unlock(&map_tree->lock);
5168 			cond_resched();
5169 			write_lock(&map_tree->lock);
5170 		}
5171 	}
5172 	write_unlock(&map_tree->lock);
5173 
5174 	/*
5175 	 * Keep looping until we have no more ranges in the io tree.
5176 	 * We can have ongoing bios started by readpages (called from readahead)
5177 	 * that have their endio callback (extent_io.c:end_bio_extent_readpage)
5178 	 * still in progress (unlocked the pages in the bio but did not yet
5179 	 * unlocked the ranges in the io tree). Therefore this means some
5180 	 * ranges can still be locked and eviction started because before
5181 	 * submitting those bios, which are executed by a separate task (work
5182 	 * queue kthread), inode references (inode->i_count) were not taken
5183 	 * (which would be dropped in the end io callback of each bio).
5184 	 * Therefore here we effectively end up waiting for those bios and
5185 	 * anyone else holding locked ranges without having bumped the inode's
5186 	 * reference count - if we don't do it, when they access the inode's
5187 	 * io_tree to unlock a range it may be too late, leading to an
5188 	 * use-after-free issue.
5189 	 */
5190 	spin_lock(&io_tree->lock);
5191 	while (!RB_EMPTY_ROOT(&io_tree->state)) {
5192 		struct extent_state *state;
5193 		struct extent_state *cached_state = NULL;
5194 		u64 start;
5195 		u64 end;
5196 
5197 		node = rb_first(&io_tree->state);
5198 		state = rb_entry(node, struct extent_state, rb_node);
5199 		start = state->start;
5200 		end = state->end;
5201 		spin_unlock(&io_tree->lock);
5202 
5203 		lock_extent_bits(io_tree, start, end, &cached_state);
5204 
5205 		/*
5206 		 * If still has DELALLOC flag, the extent didn't reach disk,
5207 		 * and its reserved space won't be freed by delayed_ref.
5208 		 * So we need to free its reserved space here.
5209 		 * (Refer to comment in btrfs_invalidatepage, case 2)
5210 		 *
5211 		 * Note, end is the bytenr of last byte, so we need + 1 here.
5212 		 */
5213 		if (state->state & EXTENT_DELALLOC)
5214 			btrfs_qgroup_free_data(inode, start, end - start + 1);
5215 
5216 		clear_extent_bit(io_tree, start, end,
5217 				 EXTENT_LOCKED | EXTENT_DIRTY |
5218 				 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
5219 				 EXTENT_DEFRAG, 1, 1,
5220 				 &cached_state, GFP_NOFS);
5221 
5222 		cond_resched();
5223 		spin_lock(&io_tree->lock);
5224 	}
5225 	spin_unlock(&io_tree->lock);
5226 }
5227 
5228 void btrfs_evict_inode(struct inode *inode)
5229 {
5230 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5231 	struct btrfs_trans_handle *trans;
5232 	struct btrfs_root *root = BTRFS_I(inode)->root;
5233 	struct btrfs_block_rsv *rsv, *global_rsv;
5234 	int steal_from_global = 0;
5235 	u64 min_size;
5236 	int ret;
5237 
5238 	trace_btrfs_inode_evict(inode);
5239 
5240 	if (!root) {
5241 		kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
5242 		return;
5243 	}
5244 
5245 	min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
5246 
5247 	evict_inode_truncate_pages(inode);
5248 
5249 	if (inode->i_nlink &&
5250 	    ((btrfs_root_refs(&root->root_item) != 0 &&
5251 	      root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
5252 	     btrfs_is_free_space_inode(inode)))
5253 		goto no_delete;
5254 
5255 	if (is_bad_inode(inode)) {
5256 		btrfs_orphan_del(NULL, inode);
5257 		goto no_delete;
5258 	}
5259 	/* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
5260 	if (!special_file(inode->i_mode))
5261 		btrfs_wait_ordered_range(inode, 0, (u64)-1);
5262 
5263 	btrfs_free_io_failure_record(inode, 0, (u64)-1);
5264 
5265 	if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
5266 		BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
5267 				 &BTRFS_I(inode)->runtime_flags));
5268 		goto no_delete;
5269 	}
5270 
5271 	if (inode->i_nlink > 0) {
5272 		BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5273 		       root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
5274 		goto no_delete;
5275 	}
5276 
5277 	ret = btrfs_commit_inode_delayed_inode(inode);
5278 	if (ret) {
5279 		btrfs_orphan_del(NULL, inode);
5280 		goto no_delete;
5281 	}
5282 
5283 	rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
5284 	if (!rsv) {
5285 		btrfs_orphan_del(NULL, inode);
5286 		goto no_delete;
5287 	}
5288 	rsv->size = min_size;
5289 	rsv->failfast = 1;
5290 	global_rsv = &fs_info->global_block_rsv;
5291 
5292 	btrfs_i_size_write(inode, 0);
5293 
5294 	/*
5295 	 * This is a bit simpler than btrfs_truncate since we've already
5296 	 * reserved our space for our orphan item in the unlink, so we just
5297 	 * need to reserve some slack space in case we add bytes and update
5298 	 * inode item when doing the truncate.
5299 	 */
5300 	while (1) {
5301 		ret = btrfs_block_rsv_refill(root, rsv, min_size,
5302 					     BTRFS_RESERVE_FLUSH_LIMIT);
5303 
5304 		/*
5305 		 * Try and steal from the global reserve since we will
5306 		 * likely not use this space anyway, we want to try as
5307 		 * hard as possible to get this to work.
5308 		 */
5309 		if (ret)
5310 			steal_from_global++;
5311 		else
5312 			steal_from_global = 0;
5313 		ret = 0;
5314 
5315 		/*
5316 		 * steal_from_global == 0: we reserved stuff, hooray!
5317 		 * steal_from_global == 1: we didn't reserve stuff, boo!
5318 		 * steal_from_global == 2: we've committed, still not a lot of
5319 		 * room but maybe we'll have room in the global reserve this
5320 		 * time.
5321 		 * steal_from_global == 3: abandon all hope!
5322 		 */
5323 		if (steal_from_global > 2) {
5324 			btrfs_warn(fs_info,
5325 				   "Could not get space for a delete, will truncate on mount %d",
5326 				   ret);
5327 			btrfs_orphan_del(NULL, inode);
5328 			btrfs_free_block_rsv(fs_info, rsv);
5329 			goto no_delete;
5330 		}
5331 
5332 		trans = btrfs_join_transaction(root);
5333 		if (IS_ERR(trans)) {
5334 			btrfs_orphan_del(NULL, inode);
5335 			btrfs_free_block_rsv(fs_info, rsv);
5336 			goto no_delete;
5337 		}
5338 
5339 		/*
5340 		 * We can't just steal from the global reserve, we need to make
5341 		 * sure there is room to do it, if not we need to commit and try
5342 		 * again.
5343 		 */
5344 		if (steal_from_global) {
5345 			if (!btrfs_check_space_for_delayed_refs(trans, fs_info))
5346 				ret = btrfs_block_rsv_migrate(global_rsv, rsv,
5347 							      min_size, 0);
5348 			else
5349 				ret = -ENOSPC;
5350 		}
5351 
5352 		/*
5353 		 * Couldn't steal from the global reserve, we have too much
5354 		 * pending stuff built up, commit the transaction and try it
5355 		 * again.
5356 		 */
5357 		if (ret) {
5358 			ret = btrfs_commit_transaction(trans);
5359 			if (ret) {
5360 				btrfs_orphan_del(NULL, inode);
5361 				btrfs_free_block_rsv(fs_info, rsv);
5362 				goto no_delete;
5363 			}
5364 			continue;
5365 		} else {
5366 			steal_from_global = 0;
5367 		}
5368 
5369 		trans->block_rsv = rsv;
5370 
5371 		ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
5372 		if (ret != -ENOSPC && ret != -EAGAIN)
5373 			break;
5374 
5375 		trans->block_rsv = &fs_info->trans_block_rsv;
5376 		btrfs_end_transaction(trans);
5377 		trans = NULL;
5378 		btrfs_btree_balance_dirty(fs_info);
5379 	}
5380 
5381 	btrfs_free_block_rsv(fs_info, rsv);
5382 
5383 	/*
5384 	 * Errors here aren't a big deal, it just means we leave orphan items
5385 	 * in the tree.  They will be cleaned up on the next mount.
5386 	 */
5387 	if (ret == 0) {
5388 		trans->block_rsv = root->orphan_block_rsv;
5389 		btrfs_orphan_del(trans, inode);
5390 	} else {
5391 		btrfs_orphan_del(NULL, inode);
5392 	}
5393 
5394 	trans->block_rsv = &fs_info->trans_block_rsv;
5395 	if (!(root == fs_info->tree_root ||
5396 	      root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
5397 		btrfs_return_ino(root, btrfs_ino(inode));
5398 
5399 	btrfs_end_transaction(trans);
5400 	btrfs_btree_balance_dirty(fs_info);
5401 no_delete:
5402 	btrfs_remove_delayed_node(inode);
5403 	clear_inode(inode);
5404 }
5405 
5406 /*
5407  * this returns the key found in the dir entry in the location pointer.
5408  * If no dir entries were found, location->objectid is 0.
5409  */
5410 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
5411 			       struct btrfs_key *location)
5412 {
5413 	const char *name = dentry->d_name.name;
5414 	int namelen = dentry->d_name.len;
5415 	struct btrfs_dir_item *di;
5416 	struct btrfs_path *path;
5417 	struct btrfs_root *root = BTRFS_I(dir)->root;
5418 	int ret = 0;
5419 
5420 	path = btrfs_alloc_path();
5421 	if (!path)
5422 		return -ENOMEM;
5423 
5424 	di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir), name,
5425 				    namelen, 0);
5426 	if (IS_ERR(di))
5427 		ret = PTR_ERR(di);
5428 
5429 	if (IS_ERR_OR_NULL(di))
5430 		goto out_err;
5431 
5432 	btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5433 out:
5434 	btrfs_free_path(path);
5435 	return ret;
5436 out_err:
5437 	location->objectid = 0;
5438 	goto out;
5439 }
5440 
5441 /*
5442  * when we hit a tree root in a directory, the btrfs part of the inode
5443  * needs to be changed to reflect the root directory of the tree root.  This
5444  * is kind of like crossing a mount point.
5445  */
5446 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5447 				    struct inode *dir,
5448 				    struct dentry *dentry,
5449 				    struct btrfs_key *location,
5450 				    struct btrfs_root **sub_root)
5451 {
5452 	struct btrfs_path *path;
5453 	struct btrfs_root *new_root;
5454 	struct btrfs_root_ref *ref;
5455 	struct extent_buffer *leaf;
5456 	struct btrfs_key key;
5457 	int ret;
5458 	int err = 0;
5459 
5460 	path = btrfs_alloc_path();
5461 	if (!path) {
5462 		err = -ENOMEM;
5463 		goto out;
5464 	}
5465 
5466 	err = -ENOENT;
5467 	key.objectid = BTRFS_I(dir)->root->root_key.objectid;
5468 	key.type = BTRFS_ROOT_REF_KEY;
5469 	key.offset = location->objectid;
5470 
5471 	ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
5472 	if (ret) {
5473 		if (ret < 0)
5474 			err = ret;
5475 		goto out;
5476 	}
5477 
5478 	leaf = path->nodes[0];
5479 	ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5480 	if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) ||
5481 	    btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
5482 		goto out;
5483 
5484 	ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
5485 				   (unsigned long)(ref + 1),
5486 				   dentry->d_name.len);
5487 	if (ret)
5488 		goto out;
5489 
5490 	btrfs_release_path(path);
5491 
5492 	new_root = btrfs_read_fs_root_no_name(fs_info, location);
5493 	if (IS_ERR(new_root)) {
5494 		err = PTR_ERR(new_root);
5495 		goto out;
5496 	}
5497 
5498 	*sub_root = new_root;
5499 	location->objectid = btrfs_root_dirid(&new_root->root_item);
5500 	location->type = BTRFS_INODE_ITEM_KEY;
5501 	location->offset = 0;
5502 	err = 0;
5503 out:
5504 	btrfs_free_path(path);
5505 	return err;
5506 }
5507 
5508 static void inode_tree_add(struct inode *inode)
5509 {
5510 	struct btrfs_root *root = BTRFS_I(inode)->root;
5511 	struct btrfs_inode *entry;
5512 	struct rb_node **p;
5513 	struct rb_node *parent;
5514 	struct rb_node *new = &BTRFS_I(inode)->rb_node;
5515 	u64 ino = btrfs_ino(inode);
5516 
5517 	if (inode_unhashed(inode))
5518 		return;
5519 	parent = NULL;
5520 	spin_lock(&root->inode_lock);
5521 	p = &root->inode_tree.rb_node;
5522 	while (*p) {
5523 		parent = *p;
5524 		entry = rb_entry(parent, struct btrfs_inode, rb_node);
5525 
5526 		if (ino < btrfs_ino(&entry->vfs_inode))
5527 			p = &parent->rb_left;
5528 		else if (ino > btrfs_ino(&entry->vfs_inode))
5529 			p = &parent->rb_right;
5530 		else {
5531 			WARN_ON(!(entry->vfs_inode.i_state &
5532 				  (I_WILL_FREE | I_FREEING)));
5533 			rb_replace_node(parent, new, &root->inode_tree);
5534 			RB_CLEAR_NODE(parent);
5535 			spin_unlock(&root->inode_lock);
5536 			return;
5537 		}
5538 	}
5539 	rb_link_node(new, parent, p);
5540 	rb_insert_color(new, &root->inode_tree);
5541 	spin_unlock(&root->inode_lock);
5542 }
5543 
5544 static void inode_tree_del(struct inode *inode)
5545 {
5546 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5547 	struct btrfs_root *root = BTRFS_I(inode)->root;
5548 	int empty = 0;
5549 
5550 	spin_lock(&root->inode_lock);
5551 	if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
5552 		rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
5553 		RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
5554 		empty = RB_EMPTY_ROOT(&root->inode_tree);
5555 	}
5556 	spin_unlock(&root->inode_lock);
5557 
5558 	if (empty && btrfs_root_refs(&root->root_item) == 0) {
5559 		synchronize_srcu(&fs_info->subvol_srcu);
5560 		spin_lock(&root->inode_lock);
5561 		empty = RB_EMPTY_ROOT(&root->inode_tree);
5562 		spin_unlock(&root->inode_lock);
5563 		if (empty)
5564 			btrfs_add_dead_root(root);
5565 	}
5566 }
5567 
5568 void btrfs_invalidate_inodes(struct btrfs_root *root)
5569 {
5570 	struct btrfs_fs_info *fs_info = root->fs_info;
5571 	struct rb_node *node;
5572 	struct rb_node *prev;
5573 	struct btrfs_inode *entry;
5574 	struct inode *inode;
5575 	u64 objectid = 0;
5576 
5577 	if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
5578 		WARN_ON(btrfs_root_refs(&root->root_item) != 0);
5579 
5580 	spin_lock(&root->inode_lock);
5581 again:
5582 	node = root->inode_tree.rb_node;
5583 	prev = NULL;
5584 	while (node) {
5585 		prev = node;
5586 		entry = rb_entry(node, struct btrfs_inode, rb_node);
5587 
5588 		if (objectid < btrfs_ino(&entry->vfs_inode))
5589 			node = node->rb_left;
5590 		else if (objectid > btrfs_ino(&entry->vfs_inode))
5591 			node = node->rb_right;
5592 		else
5593 			break;
5594 	}
5595 	if (!node) {
5596 		while (prev) {
5597 			entry = rb_entry(prev, struct btrfs_inode, rb_node);
5598 			if (objectid <= btrfs_ino(&entry->vfs_inode)) {
5599 				node = prev;
5600 				break;
5601 			}
5602 			prev = rb_next(prev);
5603 		}
5604 	}
5605 	while (node) {
5606 		entry = rb_entry(node, struct btrfs_inode, rb_node);
5607 		objectid = btrfs_ino(&entry->vfs_inode) + 1;
5608 		inode = igrab(&entry->vfs_inode);
5609 		if (inode) {
5610 			spin_unlock(&root->inode_lock);
5611 			if (atomic_read(&inode->i_count) > 1)
5612 				d_prune_aliases(inode);
5613 			/*
5614 			 * btrfs_drop_inode will have it removed from
5615 			 * the inode cache when its usage count
5616 			 * hits zero.
5617 			 */
5618 			iput(inode);
5619 			cond_resched();
5620 			spin_lock(&root->inode_lock);
5621 			goto again;
5622 		}
5623 
5624 		if (cond_resched_lock(&root->inode_lock))
5625 			goto again;
5626 
5627 		node = rb_next(node);
5628 	}
5629 	spin_unlock(&root->inode_lock);
5630 }
5631 
5632 static int btrfs_init_locked_inode(struct inode *inode, void *p)
5633 {
5634 	struct btrfs_iget_args *args = p;
5635 	inode->i_ino = args->location->objectid;
5636 	memcpy(&BTRFS_I(inode)->location, args->location,
5637 	       sizeof(*args->location));
5638 	BTRFS_I(inode)->root = args->root;
5639 	return 0;
5640 }
5641 
5642 static int btrfs_find_actor(struct inode *inode, void *opaque)
5643 {
5644 	struct btrfs_iget_args *args = opaque;
5645 	return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5646 		args->root == BTRFS_I(inode)->root;
5647 }
5648 
5649 static struct inode *btrfs_iget_locked(struct super_block *s,
5650 				       struct btrfs_key *location,
5651 				       struct btrfs_root *root)
5652 {
5653 	struct inode *inode;
5654 	struct btrfs_iget_args args;
5655 	unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5656 
5657 	args.location = location;
5658 	args.root = root;
5659 
5660 	inode = iget5_locked(s, hashval, btrfs_find_actor,
5661 			     btrfs_init_locked_inode,
5662 			     (void *)&args);
5663 	return inode;
5664 }
5665 
5666 /* Get an inode object given its location and corresponding root.
5667  * Returns in *is_new if the inode was read from disk
5668  */
5669 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
5670 			 struct btrfs_root *root, int *new)
5671 {
5672 	struct inode *inode;
5673 
5674 	inode = btrfs_iget_locked(s, location, root);
5675 	if (!inode)
5676 		return ERR_PTR(-ENOMEM);
5677 
5678 	if (inode->i_state & I_NEW) {
5679 		int ret;
5680 
5681 		ret = btrfs_read_locked_inode(inode);
5682 		if (!is_bad_inode(inode)) {
5683 			inode_tree_add(inode);
5684 			unlock_new_inode(inode);
5685 			if (new)
5686 				*new = 1;
5687 		} else {
5688 			unlock_new_inode(inode);
5689 			iput(inode);
5690 			ASSERT(ret < 0);
5691 			inode = ERR_PTR(ret < 0 ? ret : -ESTALE);
5692 		}
5693 	}
5694 
5695 	return inode;
5696 }
5697 
5698 static struct inode *new_simple_dir(struct super_block *s,
5699 				    struct btrfs_key *key,
5700 				    struct btrfs_root *root)
5701 {
5702 	struct inode *inode = new_inode(s);
5703 
5704 	if (!inode)
5705 		return ERR_PTR(-ENOMEM);
5706 
5707 	BTRFS_I(inode)->root = root;
5708 	memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
5709 	set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
5710 
5711 	inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
5712 	inode->i_op = &btrfs_dir_ro_inode_operations;
5713 	inode->i_fop = &simple_dir_operations;
5714 	inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5715 	inode->i_mtime = current_time(inode);
5716 	inode->i_atime = inode->i_mtime;
5717 	inode->i_ctime = inode->i_mtime;
5718 	BTRFS_I(inode)->i_otime = inode->i_mtime;
5719 
5720 	return inode;
5721 }
5722 
5723 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
5724 {
5725 	struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
5726 	struct inode *inode;
5727 	struct btrfs_root *root = BTRFS_I(dir)->root;
5728 	struct btrfs_root *sub_root = root;
5729 	struct btrfs_key location;
5730 	int index;
5731 	int ret = 0;
5732 
5733 	if (dentry->d_name.len > BTRFS_NAME_LEN)
5734 		return ERR_PTR(-ENAMETOOLONG);
5735 
5736 	ret = btrfs_inode_by_name(dir, dentry, &location);
5737 	if (ret < 0)
5738 		return ERR_PTR(ret);
5739 
5740 	if (location.objectid == 0)
5741 		return ERR_PTR(-ENOENT);
5742 
5743 	if (location.type == BTRFS_INODE_ITEM_KEY) {
5744 		inode = btrfs_iget(dir->i_sb, &location, root, NULL);
5745 		return inode;
5746 	}
5747 
5748 	BUG_ON(location.type != BTRFS_ROOT_ITEM_KEY);
5749 
5750 	index = srcu_read_lock(&fs_info->subvol_srcu);
5751 	ret = fixup_tree_root_location(fs_info, dir, dentry,
5752 				       &location, &sub_root);
5753 	if (ret < 0) {
5754 		if (ret != -ENOENT)
5755 			inode = ERR_PTR(ret);
5756 		else
5757 			inode = new_simple_dir(dir->i_sb, &location, sub_root);
5758 	} else {
5759 		inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
5760 	}
5761 	srcu_read_unlock(&fs_info->subvol_srcu, index);
5762 
5763 	if (!IS_ERR(inode) && root != sub_root) {
5764 		down_read(&fs_info->cleanup_work_sem);
5765 		if (!(inode->i_sb->s_flags & MS_RDONLY))
5766 			ret = btrfs_orphan_cleanup(sub_root);
5767 		up_read(&fs_info->cleanup_work_sem);
5768 		if (ret) {
5769 			iput(inode);
5770 			inode = ERR_PTR(ret);
5771 		}
5772 	}
5773 
5774 	return inode;
5775 }
5776 
5777 static int btrfs_dentry_delete(const struct dentry *dentry)
5778 {
5779 	struct btrfs_root *root;
5780 	struct inode *inode = d_inode(dentry);
5781 
5782 	if (!inode && !IS_ROOT(dentry))
5783 		inode = d_inode(dentry->d_parent);
5784 
5785 	if (inode) {
5786 		root = BTRFS_I(inode)->root;
5787 		if (btrfs_root_refs(&root->root_item) == 0)
5788 			return 1;
5789 
5790 		if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
5791 			return 1;
5792 	}
5793 	return 0;
5794 }
5795 
5796 static void btrfs_dentry_release(struct dentry *dentry)
5797 {
5798 	kfree(dentry->d_fsdata);
5799 }
5800 
5801 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
5802 				   unsigned int flags)
5803 {
5804 	struct inode *inode;
5805 
5806 	inode = btrfs_lookup_dentry(dir, dentry);
5807 	if (IS_ERR(inode)) {
5808 		if (PTR_ERR(inode) == -ENOENT)
5809 			inode = NULL;
5810 		else
5811 			return ERR_CAST(inode);
5812 	}
5813 
5814 	return d_splice_alias(inode, dentry);
5815 }
5816 
5817 unsigned char btrfs_filetype_table[] = {
5818 	DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
5819 };
5820 
5821 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
5822 {
5823 	struct inode *inode = file_inode(file);
5824 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5825 	struct btrfs_root *root = BTRFS_I(inode)->root;
5826 	struct btrfs_item *item;
5827 	struct btrfs_dir_item *di;
5828 	struct btrfs_key key;
5829 	struct btrfs_key found_key;
5830 	struct btrfs_path *path;
5831 	struct list_head ins_list;
5832 	struct list_head del_list;
5833 	int ret;
5834 	struct extent_buffer *leaf;
5835 	int slot;
5836 	unsigned char d_type;
5837 	int over = 0;
5838 	char tmp_name[32];
5839 	char *name_ptr;
5840 	int name_len;
5841 	bool put = false;
5842 	struct btrfs_key location;
5843 
5844 	if (!dir_emit_dots(file, ctx))
5845 		return 0;
5846 
5847 	path = btrfs_alloc_path();
5848 	if (!path)
5849 		return -ENOMEM;
5850 
5851 	path->reada = READA_FORWARD;
5852 
5853 	INIT_LIST_HEAD(&ins_list);
5854 	INIT_LIST_HEAD(&del_list);
5855 	put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list);
5856 
5857 	key.type = BTRFS_DIR_INDEX_KEY;
5858 	key.offset = ctx->pos;
5859 	key.objectid = btrfs_ino(inode);
5860 
5861 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5862 	if (ret < 0)
5863 		goto err;
5864 
5865 	while (1) {
5866 		leaf = path->nodes[0];
5867 		slot = path->slots[0];
5868 		if (slot >= btrfs_header_nritems(leaf)) {
5869 			ret = btrfs_next_leaf(root, path);
5870 			if (ret < 0)
5871 				goto err;
5872 			else if (ret > 0)
5873 				break;
5874 			continue;
5875 		}
5876 
5877 		item = btrfs_item_nr(slot);
5878 		btrfs_item_key_to_cpu(leaf, &found_key, slot);
5879 
5880 		if (found_key.objectid != key.objectid)
5881 			break;
5882 		if (found_key.type != BTRFS_DIR_INDEX_KEY)
5883 			break;
5884 		if (found_key.offset < ctx->pos)
5885 			goto next;
5886 		if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
5887 			goto next;
5888 
5889 		ctx->pos = found_key.offset;
5890 
5891 		di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
5892 		if (verify_dir_item(fs_info, leaf, di))
5893 			goto next;
5894 
5895 		name_len = btrfs_dir_name_len(leaf, di);
5896 		if (name_len <= sizeof(tmp_name)) {
5897 			name_ptr = tmp_name;
5898 		} else {
5899 			name_ptr = kmalloc(name_len, GFP_KERNEL);
5900 			if (!name_ptr) {
5901 				ret = -ENOMEM;
5902 				goto err;
5903 			}
5904 		}
5905 		read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1),
5906 				   name_len);
5907 
5908 		d_type = btrfs_filetype_table[btrfs_dir_type(leaf, di)];
5909 		btrfs_dir_item_key_to_cpu(leaf, di, &location);
5910 
5911 		over = !dir_emit(ctx, name_ptr, name_len, location.objectid,
5912 				 d_type);
5913 
5914 		if (name_ptr != tmp_name)
5915 			kfree(name_ptr);
5916 
5917 		if (over)
5918 			goto nopos;
5919 		ctx->pos++;
5920 next:
5921 		path->slots[0]++;
5922 	}
5923 
5924 	ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
5925 	if (ret)
5926 		goto nopos;
5927 
5928 	/*
5929 	 * Stop new entries from being returned after we return the last
5930 	 * entry.
5931 	 *
5932 	 * New directory entries are assigned a strictly increasing
5933 	 * offset.  This means that new entries created during readdir
5934 	 * are *guaranteed* to be seen in the future by that readdir.
5935 	 * This has broken buggy programs which operate on names as
5936 	 * they're returned by readdir.  Until we re-use freed offsets
5937 	 * we have this hack to stop new entries from being returned
5938 	 * under the assumption that they'll never reach this huge
5939 	 * offset.
5940 	 *
5941 	 * This is being careful not to overflow 32bit loff_t unless the
5942 	 * last entry requires it because doing so has broken 32bit apps
5943 	 * in the past.
5944 	 */
5945 	if (ctx->pos >= INT_MAX)
5946 		ctx->pos = LLONG_MAX;
5947 	else
5948 		ctx->pos = INT_MAX;
5949 nopos:
5950 	ret = 0;
5951 err:
5952 	if (put)
5953 		btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
5954 	btrfs_free_path(path);
5955 	return ret;
5956 }
5957 
5958 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc)
5959 {
5960 	struct btrfs_root *root = BTRFS_I(inode)->root;
5961 	struct btrfs_trans_handle *trans;
5962 	int ret = 0;
5963 	bool nolock = false;
5964 
5965 	if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
5966 		return 0;
5967 
5968 	if (btrfs_fs_closing(root->fs_info) && btrfs_is_free_space_inode(inode))
5969 		nolock = true;
5970 
5971 	if (wbc->sync_mode == WB_SYNC_ALL) {
5972 		if (nolock)
5973 			trans = btrfs_join_transaction_nolock(root);
5974 		else
5975 			trans = btrfs_join_transaction(root);
5976 		if (IS_ERR(trans))
5977 			return PTR_ERR(trans);
5978 		ret = btrfs_commit_transaction(trans);
5979 	}
5980 	return ret;
5981 }
5982 
5983 /*
5984  * This is somewhat expensive, updating the tree every time the
5985  * inode changes.  But, it is most likely to find the inode in cache.
5986  * FIXME, needs more benchmarking...there are no reasons other than performance
5987  * to keep or drop this code.
5988  */
5989 static int btrfs_dirty_inode(struct inode *inode)
5990 {
5991 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5992 	struct btrfs_root *root = BTRFS_I(inode)->root;
5993 	struct btrfs_trans_handle *trans;
5994 	int ret;
5995 
5996 	if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
5997 		return 0;
5998 
5999 	trans = btrfs_join_transaction(root);
6000 	if (IS_ERR(trans))
6001 		return PTR_ERR(trans);
6002 
6003 	ret = btrfs_update_inode(trans, root, inode);
6004 	if (ret && ret == -ENOSPC) {
6005 		/* whoops, lets try again with the full transaction */
6006 		btrfs_end_transaction(trans);
6007 		trans = btrfs_start_transaction(root, 1);
6008 		if (IS_ERR(trans))
6009 			return PTR_ERR(trans);
6010 
6011 		ret = btrfs_update_inode(trans, root, inode);
6012 	}
6013 	btrfs_end_transaction(trans);
6014 	if (BTRFS_I(inode)->delayed_node)
6015 		btrfs_balance_delayed_items(fs_info);
6016 
6017 	return ret;
6018 }
6019 
6020 /*
6021  * This is a copy of file_update_time.  We need this so we can return error on
6022  * ENOSPC for updating the inode in the case of file write and mmap writes.
6023  */
6024 static int btrfs_update_time(struct inode *inode, struct timespec *now,
6025 			     int flags)
6026 {
6027 	struct btrfs_root *root = BTRFS_I(inode)->root;
6028 
6029 	if (btrfs_root_readonly(root))
6030 		return -EROFS;
6031 
6032 	if (flags & S_VERSION)
6033 		inode_inc_iversion(inode);
6034 	if (flags & S_CTIME)
6035 		inode->i_ctime = *now;
6036 	if (flags & S_MTIME)
6037 		inode->i_mtime = *now;
6038 	if (flags & S_ATIME)
6039 		inode->i_atime = *now;
6040 	return btrfs_dirty_inode(inode);
6041 }
6042 
6043 /*
6044  * find the highest existing sequence number in a directory
6045  * and then set the in-memory index_cnt variable to reflect
6046  * free sequence numbers
6047  */
6048 static int btrfs_set_inode_index_count(struct inode *inode)
6049 {
6050 	struct btrfs_root *root = BTRFS_I(inode)->root;
6051 	struct btrfs_key key, found_key;
6052 	struct btrfs_path *path;
6053 	struct extent_buffer *leaf;
6054 	int ret;
6055 
6056 	key.objectid = btrfs_ino(inode);
6057 	key.type = BTRFS_DIR_INDEX_KEY;
6058 	key.offset = (u64)-1;
6059 
6060 	path = btrfs_alloc_path();
6061 	if (!path)
6062 		return -ENOMEM;
6063 
6064 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6065 	if (ret < 0)
6066 		goto out;
6067 	/* FIXME: we should be able to handle this */
6068 	if (ret == 0)
6069 		goto out;
6070 	ret = 0;
6071 
6072 	/*
6073 	 * MAGIC NUMBER EXPLANATION:
6074 	 * since we search a directory based on f_pos we have to start at 2
6075 	 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
6076 	 * else has to start at 2
6077 	 */
6078 	if (path->slots[0] == 0) {
6079 		BTRFS_I(inode)->index_cnt = 2;
6080 		goto out;
6081 	}
6082 
6083 	path->slots[0]--;
6084 
6085 	leaf = path->nodes[0];
6086 	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6087 
6088 	if (found_key.objectid != btrfs_ino(inode) ||
6089 	    found_key.type != BTRFS_DIR_INDEX_KEY) {
6090 		BTRFS_I(inode)->index_cnt = 2;
6091 		goto out;
6092 	}
6093 
6094 	BTRFS_I(inode)->index_cnt = found_key.offset + 1;
6095 out:
6096 	btrfs_free_path(path);
6097 	return ret;
6098 }
6099 
6100 /*
6101  * helper to find a free sequence number in a given directory.  This current
6102  * code is very simple, later versions will do smarter things in the btree
6103  */
6104 int btrfs_set_inode_index(struct inode *dir, u64 *index)
6105 {
6106 	int ret = 0;
6107 
6108 	if (BTRFS_I(dir)->index_cnt == (u64)-1) {
6109 		ret = btrfs_inode_delayed_dir_index_count(dir);
6110 		if (ret) {
6111 			ret = btrfs_set_inode_index_count(dir);
6112 			if (ret)
6113 				return ret;
6114 		}
6115 	}
6116 
6117 	*index = BTRFS_I(dir)->index_cnt;
6118 	BTRFS_I(dir)->index_cnt++;
6119 
6120 	return ret;
6121 }
6122 
6123 static int btrfs_insert_inode_locked(struct inode *inode)
6124 {
6125 	struct btrfs_iget_args args;
6126 	args.location = &BTRFS_I(inode)->location;
6127 	args.root = BTRFS_I(inode)->root;
6128 
6129 	return insert_inode_locked4(inode,
6130 		   btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
6131 		   btrfs_find_actor, &args);
6132 }
6133 
6134 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
6135 				     struct btrfs_root *root,
6136 				     struct inode *dir,
6137 				     const char *name, int name_len,
6138 				     u64 ref_objectid, u64 objectid,
6139 				     umode_t mode, u64 *index)
6140 {
6141 	struct btrfs_fs_info *fs_info = root->fs_info;
6142 	struct inode *inode;
6143 	struct btrfs_inode_item *inode_item;
6144 	struct btrfs_key *location;
6145 	struct btrfs_path *path;
6146 	struct btrfs_inode_ref *ref;
6147 	struct btrfs_key key[2];
6148 	u32 sizes[2];
6149 	int nitems = name ? 2 : 1;
6150 	unsigned long ptr;
6151 	int ret;
6152 
6153 	path = btrfs_alloc_path();
6154 	if (!path)
6155 		return ERR_PTR(-ENOMEM);
6156 
6157 	inode = new_inode(fs_info->sb);
6158 	if (!inode) {
6159 		btrfs_free_path(path);
6160 		return ERR_PTR(-ENOMEM);
6161 	}
6162 
6163 	/*
6164 	 * O_TMPFILE, set link count to 0, so that after this point,
6165 	 * we fill in an inode item with the correct link count.
6166 	 */
6167 	if (!name)
6168 		set_nlink(inode, 0);
6169 
6170 	/*
6171 	 * we have to initialize this early, so we can reclaim the inode
6172 	 * number if we fail afterwards in this function.
6173 	 */
6174 	inode->i_ino = objectid;
6175 
6176 	if (dir && name) {
6177 		trace_btrfs_inode_request(dir);
6178 
6179 		ret = btrfs_set_inode_index(dir, index);
6180 		if (ret) {
6181 			btrfs_free_path(path);
6182 			iput(inode);
6183 			return ERR_PTR(ret);
6184 		}
6185 	} else if (dir) {
6186 		*index = 0;
6187 	}
6188 	/*
6189 	 * index_cnt is ignored for everything but a dir,
6190 	 * btrfs_get_inode_index_count has an explanation for the magic
6191 	 * number
6192 	 */
6193 	BTRFS_I(inode)->index_cnt = 2;
6194 	BTRFS_I(inode)->dir_index = *index;
6195 	BTRFS_I(inode)->root = root;
6196 	BTRFS_I(inode)->generation = trans->transid;
6197 	inode->i_generation = BTRFS_I(inode)->generation;
6198 
6199 	/*
6200 	 * We could have gotten an inode number from somebody who was fsynced
6201 	 * and then removed in this same transaction, so let's just set full
6202 	 * sync since it will be a full sync anyway and this will blow away the
6203 	 * old info in the log.
6204 	 */
6205 	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
6206 
6207 	key[0].objectid = objectid;
6208 	key[0].type = BTRFS_INODE_ITEM_KEY;
6209 	key[0].offset = 0;
6210 
6211 	sizes[0] = sizeof(struct btrfs_inode_item);
6212 
6213 	if (name) {
6214 		/*
6215 		 * Start new inodes with an inode_ref. This is slightly more
6216 		 * efficient for small numbers of hard links since they will
6217 		 * be packed into one item. Extended refs will kick in if we
6218 		 * add more hard links than can fit in the ref item.
6219 		 */
6220 		key[1].objectid = objectid;
6221 		key[1].type = BTRFS_INODE_REF_KEY;
6222 		key[1].offset = ref_objectid;
6223 
6224 		sizes[1] = name_len + sizeof(*ref);
6225 	}
6226 
6227 	location = &BTRFS_I(inode)->location;
6228 	location->objectid = objectid;
6229 	location->offset = 0;
6230 	location->type = BTRFS_INODE_ITEM_KEY;
6231 
6232 	ret = btrfs_insert_inode_locked(inode);
6233 	if (ret < 0)
6234 		goto fail;
6235 
6236 	path->leave_spinning = 1;
6237 	ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
6238 	if (ret != 0)
6239 		goto fail_unlock;
6240 
6241 	inode_init_owner(inode, dir, mode);
6242 	inode_set_bytes(inode, 0);
6243 
6244 	inode->i_mtime = current_time(inode);
6245 	inode->i_atime = inode->i_mtime;
6246 	inode->i_ctime = inode->i_mtime;
6247 	BTRFS_I(inode)->i_otime = inode->i_mtime;
6248 
6249 	inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6250 				  struct btrfs_inode_item);
6251 	memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
6252 			     sizeof(*inode_item));
6253 	fill_inode_item(trans, path->nodes[0], inode_item, inode);
6254 
6255 	if (name) {
6256 		ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6257 				     struct btrfs_inode_ref);
6258 		btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
6259 		btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
6260 		ptr = (unsigned long)(ref + 1);
6261 		write_extent_buffer(path->nodes[0], name, ptr, name_len);
6262 	}
6263 
6264 	btrfs_mark_buffer_dirty(path->nodes[0]);
6265 	btrfs_free_path(path);
6266 
6267 	btrfs_inherit_iflags(inode, dir);
6268 
6269 	if (S_ISREG(mode)) {
6270 		if (btrfs_test_opt(fs_info, NODATASUM))
6271 			BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6272 		if (btrfs_test_opt(fs_info, NODATACOW))
6273 			BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6274 				BTRFS_INODE_NODATASUM;
6275 	}
6276 
6277 	inode_tree_add(inode);
6278 
6279 	trace_btrfs_inode_new(inode);
6280 	btrfs_set_inode_last_trans(trans, inode);
6281 
6282 	btrfs_update_root_times(trans, root);
6283 
6284 	ret = btrfs_inode_inherit_props(trans, inode, dir);
6285 	if (ret)
6286 		btrfs_err(fs_info,
6287 			  "error inheriting props for ino %llu (root %llu): %d",
6288 			  btrfs_ino(inode), root->root_key.objectid, ret);
6289 
6290 	return inode;
6291 
6292 fail_unlock:
6293 	unlock_new_inode(inode);
6294 fail:
6295 	if (dir && name)
6296 		BTRFS_I(dir)->index_cnt--;
6297 	btrfs_free_path(path);
6298 	iput(inode);
6299 	return ERR_PTR(ret);
6300 }
6301 
6302 static inline u8 btrfs_inode_type(struct inode *inode)
6303 {
6304 	return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT];
6305 }
6306 
6307 /*
6308  * utility function to add 'inode' into 'parent_inode' with
6309  * a give name and a given sequence number.
6310  * if 'add_backref' is true, also insert a backref from the
6311  * inode to the parent directory.
6312  */
6313 int btrfs_add_link(struct btrfs_trans_handle *trans,
6314 		   struct inode *parent_inode, struct inode *inode,
6315 		   const char *name, int name_len, int add_backref, u64 index)
6316 {
6317 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6318 	int ret = 0;
6319 	struct btrfs_key key;
6320 	struct btrfs_root *root = BTRFS_I(parent_inode)->root;
6321 	u64 ino = btrfs_ino(inode);
6322 	u64 parent_ino = btrfs_ino(parent_inode);
6323 
6324 	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6325 		memcpy(&key, &BTRFS_I(inode)->root->root_key, sizeof(key));
6326 	} else {
6327 		key.objectid = ino;
6328 		key.type = BTRFS_INODE_ITEM_KEY;
6329 		key.offset = 0;
6330 	}
6331 
6332 	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6333 		ret = btrfs_add_root_ref(trans, fs_info, key.objectid,
6334 					 root->root_key.objectid, parent_ino,
6335 					 index, name, name_len);
6336 	} else if (add_backref) {
6337 		ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
6338 					     parent_ino, index);
6339 	}
6340 
6341 	/* Nothing to clean up yet */
6342 	if (ret)
6343 		return ret;
6344 
6345 	ret = btrfs_insert_dir_item(trans, root, name, name_len,
6346 				    parent_inode, &key,
6347 				    btrfs_inode_type(inode), index);
6348 	if (ret == -EEXIST || ret == -EOVERFLOW)
6349 		goto fail_dir_item;
6350 	else if (ret) {
6351 		btrfs_abort_transaction(trans, ret);
6352 		return ret;
6353 	}
6354 
6355 	btrfs_i_size_write(parent_inode, parent_inode->i_size +
6356 			   name_len * 2);
6357 	inode_inc_iversion(parent_inode);
6358 	parent_inode->i_mtime = parent_inode->i_ctime =
6359 		current_time(parent_inode);
6360 	ret = btrfs_update_inode(trans, root, parent_inode);
6361 	if (ret)
6362 		btrfs_abort_transaction(trans, ret);
6363 	return ret;
6364 
6365 fail_dir_item:
6366 	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6367 		u64 local_index;
6368 		int err;
6369 		err = btrfs_del_root_ref(trans, fs_info, key.objectid,
6370 					 root->root_key.objectid, parent_ino,
6371 					 &local_index, name, name_len);
6372 
6373 	} else if (add_backref) {
6374 		u64 local_index;
6375 		int err;
6376 
6377 		err = btrfs_del_inode_ref(trans, root, name, name_len,
6378 					  ino, parent_ino, &local_index);
6379 	}
6380 	return ret;
6381 }
6382 
6383 static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
6384 			    struct inode *dir, struct dentry *dentry,
6385 			    struct inode *inode, int backref, u64 index)
6386 {
6387 	int err = btrfs_add_link(trans, dir, inode,
6388 				 dentry->d_name.name, dentry->d_name.len,
6389 				 backref, index);
6390 	if (err > 0)
6391 		err = -EEXIST;
6392 	return err;
6393 }
6394 
6395 static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
6396 			umode_t mode, dev_t rdev)
6397 {
6398 	struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6399 	struct btrfs_trans_handle *trans;
6400 	struct btrfs_root *root = BTRFS_I(dir)->root;
6401 	struct inode *inode = NULL;
6402 	int err;
6403 	int drop_inode = 0;
6404 	u64 objectid;
6405 	u64 index = 0;
6406 
6407 	/*
6408 	 * 2 for inode item and ref
6409 	 * 2 for dir items
6410 	 * 1 for xattr if selinux is on
6411 	 */
6412 	trans = btrfs_start_transaction(root, 5);
6413 	if (IS_ERR(trans))
6414 		return PTR_ERR(trans);
6415 
6416 	err = btrfs_find_free_ino(root, &objectid);
6417 	if (err)
6418 		goto out_unlock;
6419 
6420 	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6421 				dentry->d_name.len, btrfs_ino(dir), objectid,
6422 				mode, &index);
6423 	if (IS_ERR(inode)) {
6424 		err = PTR_ERR(inode);
6425 		goto out_unlock;
6426 	}
6427 
6428 	/*
6429 	* If the active LSM wants to access the inode during
6430 	* d_instantiate it needs these. Smack checks to see
6431 	* if the filesystem supports xattrs by looking at the
6432 	* ops vector.
6433 	*/
6434 	inode->i_op = &btrfs_special_inode_operations;
6435 	init_special_inode(inode, inode->i_mode, rdev);
6436 
6437 	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6438 	if (err)
6439 		goto out_unlock_inode;
6440 
6441 	err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
6442 	if (err) {
6443 		goto out_unlock_inode;
6444 	} else {
6445 		btrfs_update_inode(trans, root, inode);
6446 		unlock_new_inode(inode);
6447 		d_instantiate(dentry, inode);
6448 	}
6449 
6450 out_unlock:
6451 	btrfs_end_transaction(trans);
6452 	btrfs_balance_delayed_items(fs_info);
6453 	btrfs_btree_balance_dirty(fs_info);
6454 	if (drop_inode) {
6455 		inode_dec_link_count(inode);
6456 		iput(inode);
6457 	}
6458 	return err;
6459 
6460 out_unlock_inode:
6461 	drop_inode = 1;
6462 	unlock_new_inode(inode);
6463 	goto out_unlock;
6464 
6465 }
6466 
6467 static int btrfs_create(struct inode *dir, struct dentry *dentry,
6468 			umode_t mode, bool excl)
6469 {
6470 	struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6471 	struct btrfs_trans_handle *trans;
6472 	struct btrfs_root *root = BTRFS_I(dir)->root;
6473 	struct inode *inode = NULL;
6474 	int drop_inode_on_err = 0;
6475 	int err;
6476 	u64 objectid;
6477 	u64 index = 0;
6478 
6479 	/*
6480 	 * 2 for inode item and ref
6481 	 * 2 for dir items
6482 	 * 1 for xattr if selinux is on
6483 	 */
6484 	trans = btrfs_start_transaction(root, 5);
6485 	if (IS_ERR(trans))
6486 		return PTR_ERR(trans);
6487 
6488 	err = btrfs_find_free_ino(root, &objectid);
6489 	if (err)
6490 		goto out_unlock;
6491 
6492 	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6493 				dentry->d_name.len, btrfs_ino(dir), objectid,
6494 				mode, &index);
6495 	if (IS_ERR(inode)) {
6496 		err = PTR_ERR(inode);
6497 		goto out_unlock;
6498 	}
6499 	drop_inode_on_err = 1;
6500 	/*
6501 	* If the active LSM wants to access the inode during
6502 	* d_instantiate it needs these. Smack checks to see
6503 	* if the filesystem supports xattrs by looking at the
6504 	* ops vector.
6505 	*/
6506 	inode->i_fop = &btrfs_file_operations;
6507 	inode->i_op = &btrfs_file_inode_operations;
6508 	inode->i_mapping->a_ops = &btrfs_aops;
6509 
6510 	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6511 	if (err)
6512 		goto out_unlock_inode;
6513 
6514 	err = btrfs_update_inode(trans, root, inode);
6515 	if (err)
6516 		goto out_unlock_inode;
6517 
6518 	err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
6519 	if (err)
6520 		goto out_unlock_inode;
6521 
6522 	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
6523 	unlock_new_inode(inode);
6524 	d_instantiate(dentry, inode);
6525 
6526 out_unlock:
6527 	btrfs_end_transaction(trans);
6528 	if (err && drop_inode_on_err) {
6529 		inode_dec_link_count(inode);
6530 		iput(inode);
6531 	}
6532 	btrfs_balance_delayed_items(fs_info);
6533 	btrfs_btree_balance_dirty(fs_info);
6534 	return err;
6535 
6536 out_unlock_inode:
6537 	unlock_new_inode(inode);
6538 	goto out_unlock;
6539 
6540 }
6541 
6542 static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6543 		      struct dentry *dentry)
6544 {
6545 	struct btrfs_trans_handle *trans = NULL;
6546 	struct btrfs_root *root = BTRFS_I(dir)->root;
6547 	struct inode *inode = d_inode(old_dentry);
6548 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6549 	u64 index;
6550 	int err;
6551 	int drop_inode = 0;
6552 
6553 	/* do not allow sys_link's with other subvols of the same device */
6554 	if (root->objectid != BTRFS_I(inode)->root->objectid)
6555 		return -EXDEV;
6556 
6557 	if (inode->i_nlink >= BTRFS_LINK_MAX)
6558 		return -EMLINK;
6559 
6560 	err = btrfs_set_inode_index(dir, &index);
6561 	if (err)
6562 		goto fail;
6563 
6564 	/*
6565 	 * 2 items for inode and inode ref
6566 	 * 2 items for dir items
6567 	 * 1 item for parent inode
6568 	 */
6569 	trans = btrfs_start_transaction(root, 5);
6570 	if (IS_ERR(trans)) {
6571 		err = PTR_ERR(trans);
6572 		trans = NULL;
6573 		goto fail;
6574 	}
6575 
6576 	/* There are several dir indexes for this inode, clear the cache. */
6577 	BTRFS_I(inode)->dir_index = 0ULL;
6578 	inc_nlink(inode);
6579 	inode_inc_iversion(inode);
6580 	inode->i_ctime = current_time(inode);
6581 	ihold(inode);
6582 	set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6583 
6584 	err = btrfs_add_nondir(trans, dir, dentry, inode, 1, index);
6585 
6586 	if (err) {
6587 		drop_inode = 1;
6588 	} else {
6589 		struct dentry *parent = dentry->d_parent;
6590 		err = btrfs_update_inode(trans, root, inode);
6591 		if (err)
6592 			goto fail;
6593 		if (inode->i_nlink == 1) {
6594 			/*
6595 			 * If new hard link count is 1, it's a file created
6596 			 * with open(2) O_TMPFILE flag.
6597 			 */
6598 			err = btrfs_orphan_del(trans, inode);
6599 			if (err)
6600 				goto fail;
6601 		}
6602 		d_instantiate(dentry, inode);
6603 		btrfs_log_new_name(trans, inode, NULL, parent);
6604 	}
6605 
6606 	btrfs_balance_delayed_items(fs_info);
6607 fail:
6608 	if (trans)
6609 		btrfs_end_transaction(trans);
6610 	if (drop_inode) {
6611 		inode_dec_link_count(inode);
6612 		iput(inode);
6613 	}
6614 	btrfs_btree_balance_dirty(fs_info);
6615 	return err;
6616 }
6617 
6618 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
6619 {
6620 	struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6621 	struct inode *inode = NULL;
6622 	struct btrfs_trans_handle *trans;
6623 	struct btrfs_root *root = BTRFS_I(dir)->root;
6624 	int err = 0;
6625 	int drop_on_err = 0;
6626 	u64 objectid = 0;
6627 	u64 index = 0;
6628 
6629 	/*
6630 	 * 2 items for inode and ref
6631 	 * 2 items for dir items
6632 	 * 1 for xattr if selinux is on
6633 	 */
6634 	trans = btrfs_start_transaction(root, 5);
6635 	if (IS_ERR(trans))
6636 		return PTR_ERR(trans);
6637 
6638 	err = btrfs_find_free_ino(root, &objectid);
6639 	if (err)
6640 		goto out_fail;
6641 
6642 	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6643 				dentry->d_name.len, btrfs_ino(dir), objectid,
6644 				S_IFDIR | mode, &index);
6645 	if (IS_ERR(inode)) {
6646 		err = PTR_ERR(inode);
6647 		goto out_fail;
6648 	}
6649 
6650 	drop_on_err = 1;
6651 	/* these must be set before we unlock the inode */
6652 	inode->i_op = &btrfs_dir_inode_operations;
6653 	inode->i_fop = &btrfs_dir_file_operations;
6654 
6655 	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6656 	if (err)
6657 		goto out_fail_inode;
6658 
6659 	btrfs_i_size_write(inode, 0);
6660 	err = btrfs_update_inode(trans, root, inode);
6661 	if (err)
6662 		goto out_fail_inode;
6663 
6664 	err = btrfs_add_link(trans, dir, inode, dentry->d_name.name,
6665 			     dentry->d_name.len, 0, index);
6666 	if (err)
6667 		goto out_fail_inode;
6668 
6669 	d_instantiate(dentry, inode);
6670 	/*
6671 	 * mkdir is special.  We're unlocking after we call d_instantiate
6672 	 * to avoid a race with nfsd calling d_instantiate.
6673 	 */
6674 	unlock_new_inode(inode);
6675 	drop_on_err = 0;
6676 
6677 out_fail:
6678 	btrfs_end_transaction(trans);
6679 	if (drop_on_err) {
6680 		inode_dec_link_count(inode);
6681 		iput(inode);
6682 	}
6683 	btrfs_balance_delayed_items(fs_info);
6684 	btrfs_btree_balance_dirty(fs_info);
6685 	return err;
6686 
6687 out_fail_inode:
6688 	unlock_new_inode(inode);
6689 	goto out_fail;
6690 }
6691 
6692 /* Find next extent map of a given extent map, caller needs to ensure locks */
6693 static struct extent_map *next_extent_map(struct extent_map *em)
6694 {
6695 	struct rb_node *next;
6696 
6697 	next = rb_next(&em->rb_node);
6698 	if (!next)
6699 		return NULL;
6700 	return container_of(next, struct extent_map, rb_node);
6701 }
6702 
6703 static struct extent_map *prev_extent_map(struct extent_map *em)
6704 {
6705 	struct rb_node *prev;
6706 
6707 	prev = rb_prev(&em->rb_node);
6708 	if (!prev)
6709 		return NULL;
6710 	return container_of(prev, struct extent_map, rb_node);
6711 }
6712 
6713 /* helper for btfs_get_extent.  Given an existing extent in the tree,
6714  * the existing extent is the nearest extent to map_start,
6715  * and an extent that you want to insert, deal with overlap and insert
6716  * the best fitted new extent into the tree.
6717  */
6718 static int merge_extent_mapping(struct extent_map_tree *em_tree,
6719 				struct extent_map *existing,
6720 				struct extent_map *em,
6721 				u64 map_start)
6722 {
6723 	struct extent_map *prev;
6724 	struct extent_map *next;
6725 	u64 start;
6726 	u64 end;
6727 	u64 start_diff;
6728 
6729 	BUG_ON(map_start < em->start || map_start >= extent_map_end(em));
6730 
6731 	if (existing->start > map_start) {
6732 		next = existing;
6733 		prev = prev_extent_map(next);
6734 	} else {
6735 		prev = existing;
6736 		next = next_extent_map(prev);
6737 	}
6738 
6739 	start = prev ? extent_map_end(prev) : em->start;
6740 	start = max_t(u64, start, em->start);
6741 	end = next ? next->start : extent_map_end(em);
6742 	end = min_t(u64, end, extent_map_end(em));
6743 	start_diff = start - em->start;
6744 	em->start = start;
6745 	em->len = end - start;
6746 	if (em->block_start < EXTENT_MAP_LAST_BYTE &&
6747 	    !test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
6748 		em->block_start += start_diff;
6749 		em->block_len -= start_diff;
6750 	}
6751 	return add_extent_mapping(em_tree, em, 0);
6752 }
6753 
6754 static noinline int uncompress_inline(struct btrfs_path *path,
6755 				      struct page *page,
6756 				      size_t pg_offset, u64 extent_offset,
6757 				      struct btrfs_file_extent_item *item)
6758 {
6759 	int ret;
6760 	struct extent_buffer *leaf = path->nodes[0];
6761 	char *tmp;
6762 	size_t max_size;
6763 	unsigned long inline_size;
6764 	unsigned long ptr;
6765 	int compress_type;
6766 
6767 	WARN_ON(pg_offset != 0);
6768 	compress_type = btrfs_file_extent_compression(leaf, item);
6769 	max_size = btrfs_file_extent_ram_bytes(leaf, item);
6770 	inline_size = btrfs_file_extent_inline_item_len(leaf,
6771 					btrfs_item_nr(path->slots[0]));
6772 	tmp = kmalloc(inline_size, GFP_NOFS);
6773 	if (!tmp)
6774 		return -ENOMEM;
6775 	ptr = btrfs_file_extent_inline_start(item);
6776 
6777 	read_extent_buffer(leaf, tmp, ptr, inline_size);
6778 
6779 	max_size = min_t(unsigned long, PAGE_SIZE, max_size);
6780 	ret = btrfs_decompress(compress_type, tmp, page,
6781 			       extent_offset, inline_size, max_size);
6782 	kfree(tmp);
6783 	return ret;
6784 }
6785 
6786 /*
6787  * a bit scary, this does extent mapping from logical file offset to the disk.
6788  * the ugly parts come from merging extents from the disk with the in-ram
6789  * representation.  This gets more complex because of the data=ordered code,
6790  * where the in-ram extents might be locked pending data=ordered completion.
6791  *
6792  * This also copies inline extents directly into the page.
6793  */
6794 
6795 struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
6796 				    size_t pg_offset, u64 start, u64 len,
6797 				    int create)
6798 {
6799 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6800 	int ret;
6801 	int err = 0;
6802 	u64 extent_start = 0;
6803 	u64 extent_end = 0;
6804 	u64 objectid = btrfs_ino(inode);
6805 	u32 found_type;
6806 	struct btrfs_path *path = NULL;
6807 	struct btrfs_root *root = BTRFS_I(inode)->root;
6808 	struct btrfs_file_extent_item *item;
6809 	struct extent_buffer *leaf;
6810 	struct btrfs_key found_key;
6811 	struct extent_map *em = NULL;
6812 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
6813 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
6814 	struct btrfs_trans_handle *trans = NULL;
6815 	const bool new_inline = !page || create;
6816 
6817 again:
6818 	read_lock(&em_tree->lock);
6819 	em = lookup_extent_mapping(em_tree, start, len);
6820 	if (em)
6821 		em->bdev = fs_info->fs_devices->latest_bdev;
6822 	read_unlock(&em_tree->lock);
6823 
6824 	if (em) {
6825 		if (em->start > start || em->start + em->len <= start)
6826 			free_extent_map(em);
6827 		else if (em->block_start == EXTENT_MAP_INLINE && page)
6828 			free_extent_map(em);
6829 		else
6830 			goto out;
6831 	}
6832 	em = alloc_extent_map();
6833 	if (!em) {
6834 		err = -ENOMEM;
6835 		goto out;
6836 	}
6837 	em->bdev = fs_info->fs_devices->latest_bdev;
6838 	em->start = EXTENT_MAP_HOLE;
6839 	em->orig_start = EXTENT_MAP_HOLE;
6840 	em->len = (u64)-1;
6841 	em->block_len = (u64)-1;
6842 
6843 	if (!path) {
6844 		path = btrfs_alloc_path();
6845 		if (!path) {
6846 			err = -ENOMEM;
6847 			goto out;
6848 		}
6849 		/*
6850 		 * Chances are we'll be called again, so go ahead and do
6851 		 * readahead
6852 		 */
6853 		path->reada = READA_FORWARD;
6854 	}
6855 
6856 	ret = btrfs_lookup_file_extent(trans, root, path,
6857 				       objectid, start, trans != NULL);
6858 	if (ret < 0) {
6859 		err = ret;
6860 		goto out;
6861 	}
6862 
6863 	if (ret != 0) {
6864 		if (path->slots[0] == 0)
6865 			goto not_found;
6866 		path->slots[0]--;
6867 	}
6868 
6869 	leaf = path->nodes[0];
6870 	item = btrfs_item_ptr(leaf, path->slots[0],
6871 			      struct btrfs_file_extent_item);
6872 	/* are we inside the extent that was found? */
6873 	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6874 	found_type = found_key.type;
6875 	if (found_key.objectid != objectid ||
6876 	    found_type != BTRFS_EXTENT_DATA_KEY) {
6877 		/*
6878 		 * If we backup past the first extent we want to move forward
6879 		 * and see if there is an extent in front of us, otherwise we'll
6880 		 * say there is a hole for our whole search range which can
6881 		 * cause problems.
6882 		 */
6883 		extent_end = start;
6884 		goto next;
6885 	}
6886 
6887 	found_type = btrfs_file_extent_type(leaf, item);
6888 	extent_start = found_key.offset;
6889 	if (found_type == BTRFS_FILE_EXTENT_REG ||
6890 	    found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6891 		extent_end = extent_start +
6892 		       btrfs_file_extent_num_bytes(leaf, item);
6893 	} else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
6894 		size_t size;
6895 		size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
6896 		extent_end = ALIGN(extent_start + size,
6897 				   fs_info->sectorsize);
6898 	}
6899 next:
6900 	if (start >= extent_end) {
6901 		path->slots[0]++;
6902 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
6903 			ret = btrfs_next_leaf(root, path);
6904 			if (ret < 0) {
6905 				err = ret;
6906 				goto out;
6907 			}
6908 			if (ret > 0)
6909 				goto not_found;
6910 			leaf = path->nodes[0];
6911 		}
6912 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6913 		if (found_key.objectid != objectid ||
6914 		    found_key.type != BTRFS_EXTENT_DATA_KEY)
6915 			goto not_found;
6916 		if (start + len <= found_key.offset)
6917 			goto not_found;
6918 		if (start > found_key.offset)
6919 			goto next;
6920 		em->start = start;
6921 		em->orig_start = start;
6922 		em->len = found_key.offset - start;
6923 		goto not_found_em;
6924 	}
6925 
6926 	btrfs_extent_item_to_extent_map(inode, path, item, new_inline, em);
6927 
6928 	if (found_type == BTRFS_FILE_EXTENT_REG ||
6929 	    found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6930 		goto insert;
6931 	} else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
6932 		unsigned long ptr;
6933 		char *map;
6934 		size_t size;
6935 		size_t extent_offset;
6936 		size_t copy_size;
6937 
6938 		if (new_inline)
6939 			goto out;
6940 
6941 		size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
6942 		extent_offset = page_offset(page) + pg_offset - extent_start;
6943 		copy_size = min_t(u64, PAGE_SIZE - pg_offset,
6944 				  size - extent_offset);
6945 		em->start = extent_start + extent_offset;
6946 		em->len = ALIGN(copy_size, fs_info->sectorsize);
6947 		em->orig_block_len = em->len;
6948 		em->orig_start = em->start;
6949 		ptr = btrfs_file_extent_inline_start(item) + extent_offset;
6950 		if (create == 0 && !PageUptodate(page)) {
6951 			if (btrfs_file_extent_compression(leaf, item) !=
6952 			    BTRFS_COMPRESS_NONE) {
6953 				ret = uncompress_inline(path, page, pg_offset,
6954 							extent_offset, item);
6955 				if (ret) {
6956 					err = ret;
6957 					goto out;
6958 				}
6959 			} else {
6960 				map = kmap(page);
6961 				read_extent_buffer(leaf, map + pg_offset, ptr,
6962 						   copy_size);
6963 				if (pg_offset + copy_size < PAGE_SIZE) {
6964 					memset(map + pg_offset + copy_size, 0,
6965 					       PAGE_SIZE - pg_offset -
6966 					       copy_size);
6967 				}
6968 				kunmap(page);
6969 			}
6970 			flush_dcache_page(page);
6971 		} else if (create && PageUptodate(page)) {
6972 			BUG();
6973 			if (!trans) {
6974 				kunmap(page);
6975 				free_extent_map(em);
6976 				em = NULL;
6977 
6978 				btrfs_release_path(path);
6979 				trans = btrfs_join_transaction(root);
6980 
6981 				if (IS_ERR(trans))
6982 					return ERR_CAST(trans);
6983 				goto again;
6984 			}
6985 			map = kmap(page);
6986 			write_extent_buffer(leaf, map + pg_offset, ptr,
6987 					    copy_size);
6988 			kunmap(page);
6989 			btrfs_mark_buffer_dirty(leaf);
6990 		}
6991 		set_extent_uptodate(io_tree, em->start,
6992 				    extent_map_end(em) - 1, NULL, GFP_NOFS);
6993 		goto insert;
6994 	}
6995 not_found:
6996 	em->start = start;
6997 	em->orig_start = start;
6998 	em->len = len;
6999 not_found_em:
7000 	em->block_start = EXTENT_MAP_HOLE;
7001 	set_bit(EXTENT_FLAG_VACANCY, &em->flags);
7002 insert:
7003 	btrfs_release_path(path);
7004 	if (em->start > start || extent_map_end(em) <= start) {
7005 		btrfs_err(fs_info,
7006 			  "bad extent! em: [%llu %llu] passed [%llu %llu]",
7007 			  em->start, em->len, start, len);
7008 		err = -EIO;
7009 		goto out;
7010 	}
7011 
7012 	err = 0;
7013 	write_lock(&em_tree->lock);
7014 	ret = add_extent_mapping(em_tree, em, 0);
7015 	/* it is possible that someone inserted the extent into the tree
7016 	 * while we had the lock dropped.  It is also possible that
7017 	 * an overlapping map exists in the tree
7018 	 */
7019 	if (ret == -EEXIST) {
7020 		struct extent_map *existing;
7021 
7022 		ret = 0;
7023 
7024 		existing = search_extent_mapping(em_tree, start, len);
7025 		/*
7026 		 * existing will always be non-NULL, since there must be
7027 		 * extent causing the -EEXIST.
7028 		 */
7029 		if (existing->start == em->start &&
7030 		    extent_map_end(existing) >= extent_map_end(em) &&
7031 		    em->block_start == existing->block_start) {
7032 			/*
7033 			 * The existing extent map already encompasses the
7034 			 * entire extent map we tried to add.
7035 			 */
7036 			free_extent_map(em);
7037 			em = existing;
7038 			err = 0;
7039 
7040 		} else if (start >= extent_map_end(existing) ||
7041 		    start <= existing->start) {
7042 			/*
7043 			 * The existing extent map is the one nearest to
7044 			 * the [start, start + len) range which overlaps
7045 			 */
7046 			err = merge_extent_mapping(em_tree, existing,
7047 						   em, start);
7048 			free_extent_map(existing);
7049 			if (err) {
7050 				free_extent_map(em);
7051 				em = NULL;
7052 			}
7053 		} else {
7054 			free_extent_map(em);
7055 			em = existing;
7056 			err = 0;
7057 		}
7058 	}
7059 	write_unlock(&em_tree->lock);
7060 out:
7061 
7062 	trace_btrfs_get_extent(root, em);
7063 
7064 	btrfs_free_path(path);
7065 	if (trans) {
7066 		ret = btrfs_end_transaction(trans);
7067 		if (!err)
7068 			err = ret;
7069 	}
7070 	if (err) {
7071 		free_extent_map(em);
7072 		return ERR_PTR(err);
7073 	}
7074 	BUG_ON(!em); /* Error is always set */
7075 	return em;
7076 }
7077 
7078 struct extent_map *btrfs_get_extent_fiemap(struct inode *inode, struct page *page,
7079 					   size_t pg_offset, u64 start, u64 len,
7080 					   int create)
7081 {
7082 	struct extent_map *em;
7083 	struct extent_map *hole_em = NULL;
7084 	u64 range_start = start;
7085 	u64 end;
7086 	u64 found;
7087 	u64 found_end;
7088 	int err = 0;
7089 
7090 	em = btrfs_get_extent(inode, page, pg_offset, start, len, create);
7091 	if (IS_ERR(em))
7092 		return em;
7093 	if (em) {
7094 		/*
7095 		 * if our em maps to
7096 		 * -  a hole or
7097 		 * -  a pre-alloc extent,
7098 		 * there might actually be delalloc bytes behind it.
7099 		 */
7100 		if (em->block_start != EXTENT_MAP_HOLE &&
7101 		    !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7102 			return em;
7103 		else
7104 			hole_em = em;
7105 	}
7106 
7107 	/* check to see if we've wrapped (len == -1 or similar) */
7108 	end = start + len;
7109 	if (end < start)
7110 		end = (u64)-1;
7111 	else
7112 		end -= 1;
7113 
7114 	em = NULL;
7115 
7116 	/* ok, we didn't find anything, lets look for delalloc */
7117 	found = count_range_bits(&BTRFS_I(inode)->io_tree, &range_start,
7118 				 end, len, EXTENT_DELALLOC, 1);
7119 	found_end = range_start + found;
7120 	if (found_end < range_start)
7121 		found_end = (u64)-1;
7122 
7123 	/*
7124 	 * we didn't find anything useful, return
7125 	 * the original results from get_extent()
7126 	 */
7127 	if (range_start > end || found_end <= start) {
7128 		em = hole_em;
7129 		hole_em = NULL;
7130 		goto out;
7131 	}
7132 
7133 	/* adjust the range_start to make sure it doesn't
7134 	 * go backwards from the start they passed in
7135 	 */
7136 	range_start = max(start, range_start);
7137 	found = found_end - range_start;
7138 
7139 	if (found > 0) {
7140 		u64 hole_start = start;
7141 		u64 hole_len = len;
7142 
7143 		em = alloc_extent_map();
7144 		if (!em) {
7145 			err = -ENOMEM;
7146 			goto out;
7147 		}
7148 		/*
7149 		 * when btrfs_get_extent can't find anything it
7150 		 * returns one huge hole
7151 		 *
7152 		 * make sure what it found really fits our range, and
7153 		 * adjust to make sure it is based on the start from
7154 		 * the caller
7155 		 */
7156 		if (hole_em) {
7157 			u64 calc_end = extent_map_end(hole_em);
7158 
7159 			if (calc_end <= start || (hole_em->start > end)) {
7160 				free_extent_map(hole_em);
7161 				hole_em = NULL;
7162 			} else {
7163 				hole_start = max(hole_em->start, start);
7164 				hole_len = calc_end - hole_start;
7165 			}
7166 		}
7167 		em->bdev = NULL;
7168 		if (hole_em && range_start > hole_start) {
7169 			/* our hole starts before our delalloc, so we
7170 			 * have to return just the parts of the hole
7171 			 * that go until  the delalloc starts
7172 			 */
7173 			em->len = min(hole_len,
7174 				      range_start - hole_start);
7175 			em->start = hole_start;
7176 			em->orig_start = hole_start;
7177 			/*
7178 			 * don't adjust block start at all,
7179 			 * it is fixed at EXTENT_MAP_HOLE
7180 			 */
7181 			em->block_start = hole_em->block_start;
7182 			em->block_len = hole_len;
7183 			if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
7184 				set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
7185 		} else {
7186 			em->start = range_start;
7187 			em->len = found;
7188 			em->orig_start = range_start;
7189 			em->block_start = EXTENT_MAP_DELALLOC;
7190 			em->block_len = found;
7191 		}
7192 	} else if (hole_em) {
7193 		return hole_em;
7194 	}
7195 out:
7196 
7197 	free_extent_map(hole_em);
7198 	if (err) {
7199 		free_extent_map(em);
7200 		return ERR_PTR(err);
7201 	}
7202 	return em;
7203 }
7204 
7205 static struct extent_map *btrfs_create_dio_extent(struct inode *inode,
7206 						  const u64 start,
7207 						  const u64 len,
7208 						  const u64 orig_start,
7209 						  const u64 block_start,
7210 						  const u64 block_len,
7211 						  const u64 orig_block_len,
7212 						  const u64 ram_bytes,
7213 						  const int type)
7214 {
7215 	struct extent_map *em = NULL;
7216 	int ret;
7217 
7218 	down_read(&BTRFS_I(inode)->dio_sem);
7219 	if (type != BTRFS_ORDERED_NOCOW) {
7220 		em = create_pinned_em(inode, start, len, orig_start,
7221 				      block_start, block_len, orig_block_len,
7222 				      ram_bytes, type);
7223 		if (IS_ERR(em))
7224 			goto out;
7225 	}
7226 	ret = btrfs_add_ordered_extent_dio(inode, start, block_start,
7227 					   len, block_len, type);
7228 	if (ret) {
7229 		if (em) {
7230 			free_extent_map(em);
7231 			btrfs_drop_extent_cache(inode, start,
7232 						start + len - 1, 0);
7233 		}
7234 		em = ERR_PTR(ret);
7235 	}
7236  out:
7237 	up_read(&BTRFS_I(inode)->dio_sem);
7238 
7239 	return em;
7240 }
7241 
7242 static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
7243 						  u64 start, u64 len)
7244 {
7245 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7246 	struct btrfs_root *root = BTRFS_I(inode)->root;
7247 	struct extent_map *em;
7248 	struct btrfs_key ins;
7249 	u64 alloc_hint;
7250 	int ret;
7251 
7252 	alloc_hint = get_extent_allocation_hint(inode, start, len);
7253 	ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
7254 				   0, alloc_hint, &ins, 1, 1);
7255 	if (ret)
7256 		return ERR_PTR(ret);
7257 
7258 	em = btrfs_create_dio_extent(inode, start, ins.offset, start,
7259 				     ins.objectid, ins.offset, ins.offset,
7260 				     ins.offset, 0);
7261 	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
7262 	if (IS_ERR(em))
7263 		btrfs_free_reserved_extent(fs_info, ins.objectid,
7264 					   ins.offset, 1);
7265 
7266 	return em;
7267 }
7268 
7269 /*
7270  * returns 1 when the nocow is safe, < 1 on error, 0 if the
7271  * block must be cow'd
7272  */
7273 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7274 			      u64 *orig_start, u64 *orig_block_len,
7275 			      u64 *ram_bytes)
7276 {
7277 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7278 	struct btrfs_trans_handle *trans;
7279 	struct btrfs_path *path;
7280 	int ret;
7281 	struct extent_buffer *leaf;
7282 	struct btrfs_root *root = BTRFS_I(inode)->root;
7283 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7284 	struct btrfs_file_extent_item *fi;
7285 	struct btrfs_key key;
7286 	u64 disk_bytenr;
7287 	u64 backref_offset;
7288 	u64 extent_end;
7289 	u64 num_bytes;
7290 	int slot;
7291 	int found_type;
7292 	bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
7293 
7294 	path = btrfs_alloc_path();
7295 	if (!path)
7296 		return -ENOMEM;
7297 
7298 	ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode),
7299 				       offset, 0);
7300 	if (ret < 0)
7301 		goto out;
7302 
7303 	slot = path->slots[0];
7304 	if (ret == 1) {
7305 		if (slot == 0) {
7306 			/* can't find the item, must cow */
7307 			ret = 0;
7308 			goto out;
7309 		}
7310 		slot--;
7311 	}
7312 	ret = 0;
7313 	leaf = path->nodes[0];
7314 	btrfs_item_key_to_cpu(leaf, &key, slot);
7315 	if (key.objectid != btrfs_ino(inode) ||
7316 	    key.type != BTRFS_EXTENT_DATA_KEY) {
7317 		/* not our file or wrong item type, must cow */
7318 		goto out;
7319 	}
7320 
7321 	if (key.offset > offset) {
7322 		/* Wrong offset, must cow */
7323 		goto out;
7324 	}
7325 
7326 	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
7327 	found_type = btrfs_file_extent_type(leaf, fi);
7328 	if (found_type != BTRFS_FILE_EXTENT_REG &&
7329 	    found_type != BTRFS_FILE_EXTENT_PREALLOC) {
7330 		/* not a regular extent, must cow */
7331 		goto out;
7332 	}
7333 
7334 	if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
7335 		goto out;
7336 
7337 	extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
7338 	if (extent_end <= offset)
7339 		goto out;
7340 
7341 	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7342 	if (disk_bytenr == 0)
7343 		goto out;
7344 
7345 	if (btrfs_file_extent_compression(leaf, fi) ||
7346 	    btrfs_file_extent_encryption(leaf, fi) ||
7347 	    btrfs_file_extent_other_encoding(leaf, fi))
7348 		goto out;
7349 
7350 	backref_offset = btrfs_file_extent_offset(leaf, fi);
7351 
7352 	if (orig_start) {
7353 		*orig_start = key.offset - backref_offset;
7354 		*orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
7355 		*ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
7356 	}
7357 
7358 	if (btrfs_extent_readonly(fs_info, disk_bytenr))
7359 		goto out;
7360 
7361 	num_bytes = min(offset + *len, extent_end) - offset;
7362 	if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7363 		u64 range_end;
7364 
7365 		range_end = round_up(offset + num_bytes,
7366 				     root->fs_info->sectorsize) - 1;
7367 		ret = test_range_bit(io_tree, offset, range_end,
7368 				     EXTENT_DELALLOC, 0, NULL);
7369 		if (ret) {
7370 			ret = -EAGAIN;
7371 			goto out;
7372 		}
7373 	}
7374 
7375 	btrfs_release_path(path);
7376 
7377 	/*
7378 	 * look for other files referencing this extent, if we
7379 	 * find any we must cow
7380 	 */
7381 	trans = btrfs_join_transaction(root);
7382 	if (IS_ERR(trans)) {
7383 		ret = 0;
7384 		goto out;
7385 	}
7386 
7387 	ret = btrfs_cross_ref_exist(trans, root, btrfs_ino(inode),
7388 				    key.offset - backref_offset, disk_bytenr);
7389 	btrfs_end_transaction(trans);
7390 	if (ret) {
7391 		ret = 0;
7392 		goto out;
7393 	}
7394 
7395 	/*
7396 	 * adjust disk_bytenr and num_bytes to cover just the bytes
7397 	 * in this extent we are about to write.  If there
7398 	 * are any csums in that range we have to cow in order
7399 	 * to keep the csums correct
7400 	 */
7401 	disk_bytenr += backref_offset;
7402 	disk_bytenr += offset - key.offset;
7403 	if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes))
7404 		goto out;
7405 	/*
7406 	 * all of the above have passed, it is safe to overwrite this extent
7407 	 * without cow
7408 	 */
7409 	*len = num_bytes;
7410 	ret = 1;
7411 out:
7412 	btrfs_free_path(path);
7413 	return ret;
7414 }
7415 
7416 bool btrfs_page_exists_in_range(struct inode *inode, loff_t start, loff_t end)
7417 {
7418 	struct radix_tree_root *root = &inode->i_mapping->page_tree;
7419 	int found = false;
7420 	void **pagep = NULL;
7421 	struct page *page = NULL;
7422 	int start_idx;
7423 	int end_idx;
7424 
7425 	start_idx = start >> PAGE_SHIFT;
7426 
7427 	/*
7428 	 * end is the last byte in the last page.  end == start is legal
7429 	 */
7430 	end_idx = end >> PAGE_SHIFT;
7431 
7432 	rcu_read_lock();
7433 
7434 	/* Most of the code in this while loop is lifted from
7435 	 * find_get_page.  It's been modified to begin searching from a
7436 	 * page and return just the first page found in that range.  If the
7437 	 * found idx is less than or equal to the end idx then we know that
7438 	 * a page exists.  If no pages are found or if those pages are
7439 	 * outside of the range then we're fine (yay!) */
7440 	while (page == NULL &&
7441 	       radix_tree_gang_lookup_slot(root, &pagep, NULL, start_idx, 1)) {
7442 		page = radix_tree_deref_slot(pagep);
7443 		if (unlikely(!page))
7444 			break;
7445 
7446 		if (radix_tree_exception(page)) {
7447 			if (radix_tree_deref_retry(page)) {
7448 				page = NULL;
7449 				continue;
7450 			}
7451 			/*
7452 			 * Otherwise, shmem/tmpfs must be storing a swap entry
7453 			 * here as an exceptional entry: so return it without
7454 			 * attempting to raise page count.
7455 			 */
7456 			page = NULL;
7457 			break; /* TODO: Is this relevant for this use case? */
7458 		}
7459 
7460 		if (!page_cache_get_speculative(page)) {
7461 			page = NULL;
7462 			continue;
7463 		}
7464 
7465 		/*
7466 		 * Has the page moved?
7467 		 * This is part of the lockless pagecache protocol. See
7468 		 * include/linux/pagemap.h for details.
7469 		 */
7470 		if (unlikely(page != *pagep)) {
7471 			put_page(page);
7472 			page = NULL;
7473 		}
7474 	}
7475 
7476 	if (page) {
7477 		if (page->index <= end_idx)
7478 			found = true;
7479 		put_page(page);
7480 	}
7481 
7482 	rcu_read_unlock();
7483 	return found;
7484 }
7485 
7486 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
7487 			      struct extent_state **cached_state, int writing)
7488 {
7489 	struct btrfs_ordered_extent *ordered;
7490 	int ret = 0;
7491 
7492 	while (1) {
7493 		lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7494 				 cached_state);
7495 		/*
7496 		 * We're concerned with the entire range that we're going to be
7497 		 * doing DIO to, so we need to make sure there's no ordered
7498 		 * extents in this range.
7499 		 */
7500 		ordered = btrfs_lookup_ordered_range(inode, lockstart,
7501 						     lockend - lockstart + 1);
7502 
7503 		/*
7504 		 * We need to make sure there are no buffered pages in this
7505 		 * range either, we could have raced between the invalidate in
7506 		 * generic_file_direct_write and locking the extent.  The
7507 		 * invalidate needs to happen so that reads after a write do not
7508 		 * get stale data.
7509 		 */
7510 		if (!ordered &&
7511 		    (!writing ||
7512 		     !btrfs_page_exists_in_range(inode, lockstart, lockend)))
7513 			break;
7514 
7515 		unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7516 				     cached_state, GFP_NOFS);
7517 
7518 		if (ordered) {
7519 			/*
7520 			 * If we are doing a DIO read and the ordered extent we
7521 			 * found is for a buffered write, we can not wait for it
7522 			 * to complete and retry, because if we do so we can
7523 			 * deadlock with concurrent buffered writes on page
7524 			 * locks. This happens only if our DIO read covers more
7525 			 * than one extent map, if at this point has already
7526 			 * created an ordered extent for a previous extent map
7527 			 * and locked its range in the inode's io tree, and a
7528 			 * concurrent write against that previous extent map's
7529 			 * range and this range started (we unlock the ranges
7530 			 * in the io tree only when the bios complete and
7531 			 * buffered writes always lock pages before attempting
7532 			 * to lock range in the io tree).
7533 			 */
7534 			if (writing ||
7535 			    test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
7536 				btrfs_start_ordered_extent(inode, ordered, 1);
7537 			else
7538 				ret = -ENOTBLK;
7539 			btrfs_put_ordered_extent(ordered);
7540 		} else {
7541 			/*
7542 			 * We could trigger writeback for this range (and wait
7543 			 * for it to complete) and then invalidate the pages for
7544 			 * this range (through invalidate_inode_pages2_range()),
7545 			 * but that can lead us to a deadlock with a concurrent
7546 			 * call to readpages() (a buffered read or a defrag call
7547 			 * triggered a readahead) on a page lock due to an
7548 			 * ordered dio extent we created before but did not have
7549 			 * yet a corresponding bio submitted (whence it can not
7550 			 * complete), which makes readpages() wait for that
7551 			 * ordered extent to complete while holding a lock on
7552 			 * that page.
7553 			 */
7554 			ret = -ENOTBLK;
7555 		}
7556 
7557 		if (ret)
7558 			break;
7559 
7560 		cond_resched();
7561 	}
7562 
7563 	return ret;
7564 }
7565 
7566 static struct extent_map *create_pinned_em(struct inode *inode, u64 start,
7567 					   u64 len, u64 orig_start,
7568 					   u64 block_start, u64 block_len,
7569 					   u64 orig_block_len, u64 ram_bytes,
7570 					   int type)
7571 {
7572 	struct extent_map_tree *em_tree;
7573 	struct extent_map *em;
7574 	struct btrfs_root *root = BTRFS_I(inode)->root;
7575 	int ret;
7576 
7577 	em_tree = &BTRFS_I(inode)->extent_tree;
7578 	em = alloc_extent_map();
7579 	if (!em)
7580 		return ERR_PTR(-ENOMEM);
7581 
7582 	em->start = start;
7583 	em->orig_start = orig_start;
7584 	em->mod_start = start;
7585 	em->mod_len = len;
7586 	em->len = len;
7587 	em->block_len = block_len;
7588 	em->block_start = block_start;
7589 	em->bdev = root->fs_info->fs_devices->latest_bdev;
7590 	em->orig_block_len = orig_block_len;
7591 	em->ram_bytes = ram_bytes;
7592 	em->generation = -1;
7593 	set_bit(EXTENT_FLAG_PINNED, &em->flags);
7594 	if (type == BTRFS_ORDERED_PREALLOC)
7595 		set_bit(EXTENT_FLAG_FILLING, &em->flags);
7596 
7597 	do {
7598 		btrfs_drop_extent_cache(inode, em->start,
7599 				em->start + em->len - 1, 0);
7600 		write_lock(&em_tree->lock);
7601 		ret = add_extent_mapping(em_tree, em, 1);
7602 		write_unlock(&em_tree->lock);
7603 	} while (ret == -EEXIST);
7604 
7605 	if (ret) {
7606 		free_extent_map(em);
7607 		return ERR_PTR(ret);
7608 	}
7609 
7610 	return em;
7611 }
7612 
7613 static void adjust_dio_outstanding_extents(struct inode *inode,
7614 					   struct btrfs_dio_data *dio_data,
7615 					   const u64 len)
7616 {
7617 	unsigned num_extents;
7618 
7619 	num_extents = (unsigned) div64_u64(len + BTRFS_MAX_EXTENT_SIZE - 1,
7620 					   BTRFS_MAX_EXTENT_SIZE);
7621 	/*
7622 	 * If we have an outstanding_extents count still set then we're
7623 	 * within our reservation, otherwise we need to adjust our inode
7624 	 * counter appropriately.
7625 	 */
7626 	if (dio_data->outstanding_extents) {
7627 		dio_data->outstanding_extents -= num_extents;
7628 	} else {
7629 		spin_lock(&BTRFS_I(inode)->lock);
7630 		BTRFS_I(inode)->outstanding_extents += num_extents;
7631 		spin_unlock(&BTRFS_I(inode)->lock);
7632 	}
7633 }
7634 
7635 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
7636 				   struct buffer_head *bh_result, int create)
7637 {
7638 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7639 	struct extent_map *em;
7640 	struct extent_state *cached_state = NULL;
7641 	struct btrfs_dio_data *dio_data = NULL;
7642 	u64 start = iblock << inode->i_blkbits;
7643 	u64 lockstart, lockend;
7644 	u64 len = bh_result->b_size;
7645 	int unlock_bits = EXTENT_LOCKED;
7646 	int ret = 0;
7647 
7648 	if (create)
7649 		unlock_bits |= EXTENT_DIRTY;
7650 	else
7651 		len = min_t(u64, len, fs_info->sectorsize);
7652 
7653 	lockstart = start;
7654 	lockend = start + len - 1;
7655 
7656 	if (current->journal_info) {
7657 		/*
7658 		 * Need to pull our outstanding extents and set journal_info to NULL so
7659 		 * that anything that needs to check if there's a transaction doesn't get
7660 		 * confused.
7661 		 */
7662 		dio_data = current->journal_info;
7663 		current->journal_info = NULL;
7664 	}
7665 
7666 	/*
7667 	 * If this errors out it's because we couldn't invalidate pagecache for
7668 	 * this range and we need to fallback to buffered.
7669 	 */
7670 	if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
7671 			       create)) {
7672 		ret = -ENOTBLK;
7673 		goto err;
7674 	}
7675 
7676 	em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
7677 	if (IS_ERR(em)) {
7678 		ret = PTR_ERR(em);
7679 		goto unlock_err;
7680 	}
7681 
7682 	/*
7683 	 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7684 	 * io.  INLINE is special, and we could probably kludge it in here, but
7685 	 * it's still buffered so for safety lets just fall back to the generic
7686 	 * buffered path.
7687 	 *
7688 	 * For COMPRESSED we _have_ to read the entire extent in so we can
7689 	 * decompress it, so there will be buffering required no matter what we
7690 	 * do, so go ahead and fallback to buffered.
7691 	 *
7692 	 * We return -ENOTBLK because that's what makes DIO go ahead and go back
7693 	 * to buffered IO.  Don't blame me, this is the price we pay for using
7694 	 * the generic code.
7695 	 */
7696 	if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
7697 	    em->block_start == EXTENT_MAP_INLINE) {
7698 		free_extent_map(em);
7699 		ret = -ENOTBLK;
7700 		goto unlock_err;
7701 	}
7702 
7703 	/* Just a good old fashioned hole, return */
7704 	if (!create && (em->block_start == EXTENT_MAP_HOLE ||
7705 			test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
7706 		free_extent_map(em);
7707 		goto unlock_err;
7708 	}
7709 
7710 	/*
7711 	 * We don't allocate a new extent in the following cases
7712 	 *
7713 	 * 1) The inode is marked as NODATACOW.  In this case we'll just use the
7714 	 * existing extent.
7715 	 * 2) The extent is marked as PREALLOC.  We're good to go here and can
7716 	 * just use the extent.
7717 	 *
7718 	 */
7719 	if (!create) {
7720 		len = min(len, em->len - (start - em->start));
7721 		lockstart = start + len;
7722 		goto unlock;
7723 	}
7724 
7725 	if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
7726 	    ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
7727 	     em->block_start != EXTENT_MAP_HOLE)) {
7728 		int type;
7729 		u64 block_start, orig_start, orig_block_len, ram_bytes;
7730 
7731 		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7732 			type = BTRFS_ORDERED_PREALLOC;
7733 		else
7734 			type = BTRFS_ORDERED_NOCOW;
7735 		len = min(len, em->len - (start - em->start));
7736 		block_start = em->block_start + (start - em->start);
7737 
7738 		if (can_nocow_extent(inode, start, &len, &orig_start,
7739 				     &orig_block_len, &ram_bytes) == 1 &&
7740 		    btrfs_inc_nocow_writers(fs_info, block_start)) {
7741 			struct extent_map *em2;
7742 
7743 			em2 = btrfs_create_dio_extent(inode, start, len,
7744 						      orig_start, block_start,
7745 						      len, orig_block_len,
7746 						      ram_bytes, type);
7747 			btrfs_dec_nocow_writers(fs_info, block_start);
7748 			if (type == BTRFS_ORDERED_PREALLOC) {
7749 				free_extent_map(em);
7750 				em = em2;
7751 			}
7752 			if (em2 && IS_ERR(em2)) {
7753 				ret = PTR_ERR(em2);
7754 				goto unlock_err;
7755 			}
7756 			/*
7757 			 * For inode marked NODATACOW or extent marked PREALLOC,
7758 			 * use the existing or preallocated extent, so does not
7759 			 * need to adjust btrfs_space_info's bytes_may_use.
7760 			 */
7761 			btrfs_free_reserved_data_space_noquota(inode,
7762 					start, len);
7763 			goto unlock;
7764 		}
7765 	}
7766 
7767 	/*
7768 	 * this will cow the extent, reset the len in case we changed
7769 	 * it above
7770 	 */
7771 	len = bh_result->b_size;
7772 	free_extent_map(em);
7773 	em = btrfs_new_extent_direct(inode, start, len);
7774 	if (IS_ERR(em)) {
7775 		ret = PTR_ERR(em);
7776 		goto unlock_err;
7777 	}
7778 	len = min(len, em->len - (start - em->start));
7779 unlock:
7780 	bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7781 		inode->i_blkbits;
7782 	bh_result->b_size = len;
7783 	bh_result->b_bdev = em->bdev;
7784 	set_buffer_mapped(bh_result);
7785 	if (create) {
7786 		if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7787 			set_buffer_new(bh_result);
7788 
7789 		/*
7790 		 * Need to update the i_size under the extent lock so buffered
7791 		 * readers will get the updated i_size when we unlock.
7792 		 */
7793 		if (start + len > i_size_read(inode))
7794 			i_size_write(inode, start + len);
7795 
7796 		adjust_dio_outstanding_extents(inode, dio_data, len);
7797 		WARN_ON(dio_data->reserve < len);
7798 		dio_data->reserve -= len;
7799 		dio_data->unsubmitted_oe_range_end = start + len;
7800 		current->journal_info = dio_data;
7801 	}
7802 
7803 	/*
7804 	 * In the case of write we need to clear and unlock the entire range,
7805 	 * in the case of read we need to unlock only the end area that we
7806 	 * aren't using if there is any left over space.
7807 	 */
7808 	if (lockstart < lockend) {
7809 		clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
7810 				 lockend, unlock_bits, 1, 0,
7811 				 &cached_state, GFP_NOFS);
7812 	} else {
7813 		free_extent_state(cached_state);
7814 	}
7815 
7816 	free_extent_map(em);
7817 
7818 	return 0;
7819 
7820 unlock_err:
7821 	clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7822 			 unlock_bits, 1, 0, &cached_state, GFP_NOFS);
7823 err:
7824 	if (dio_data)
7825 		current->journal_info = dio_data;
7826 	/*
7827 	 * Compensate the delalloc release we do in btrfs_direct_IO() when we
7828 	 * write less data then expected, so that we don't underflow our inode's
7829 	 * outstanding extents counter.
7830 	 */
7831 	if (create && dio_data)
7832 		adjust_dio_outstanding_extents(inode, dio_data, len);
7833 
7834 	return ret;
7835 }
7836 
7837 static inline int submit_dio_repair_bio(struct inode *inode, struct bio *bio,
7838 					int mirror_num)
7839 {
7840 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7841 	int ret;
7842 
7843 	BUG_ON(bio_op(bio) == REQ_OP_WRITE);
7844 
7845 	bio_get(bio);
7846 
7847 	ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR);
7848 	if (ret)
7849 		goto err;
7850 
7851 	ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
7852 err:
7853 	bio_put(bio);
7854 	return ret;
7855 }
7856 
7857 static int btrfs_check_dio_repairable(struct inode *inode,
7858 				      struct bio *failed_bio,
7859 				      struct io_failure_record *failrec,
7860 				      int failed_mirror)
7861 {
7862 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7863 	int num_copies;
7864 
7865 	num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
7866 	if (num_copies == 1) {
7867 		/*
7868 		 * we only have a single copy of the data, so don't bother with
7869 		 * all the retry and error correction code that follows. no
7870 		 * matter what the error is, it is very likely to persist.
7871 		 */
7872 		btrfs_debug(fs_info,
7873 			"Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
7874 			num_copies, failrec->this_mirror, failed_mirror);
7875 		return 0;
7876 	}
7877 
7878 	failrec->failed_mirror = failed_mirror;
7879 	failrec->this_mirror++;
7880 	if (failrec->this_mirror == failed_mirror)
7881 		failrec->this_mirror++;
7882 
7883 	if (failrec->this_mirror > num_copies) {
7884 		btrfs_debug(fs_info,
7885 			"Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
7886 			num_copies, failrec->this_mirror, failed_mirror);
7887 		return 0;
7888 	}
7889 
7890 	return 1;
7891 }
7892 
7893 static int dio_read_error(struct inode *inode, struct bio *failed_bio,
7894 			struct page *page, unsigned int pgoff,
7895 			u64 start, u64 end, int failed_mirror,
7896 			bio_end_io_t *repair_endio, void *repair_arg)
7897 {
7898 	struct io_failure_record *failrec;
7899 	struct bio *bio;
7900 	int isector;
7901 	int read_mode = 0;
7902 	int ret;
7903 
7904 	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
7905 
7906 	ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
7907 	if (ret)
7908 		return ret;
7909 
7910 	ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
7911 					 failed_mirror);
7912 	if (!ret) {
7913 		free_io_failure(inode, failrec);
7914 		return -EIO;
7915 	}
7916 
7917 	if ((failed_bio->bi_vcnt > 1)
7918 		|| (failed_bio->bi_io_vec->bv_len
7919 			> btrfs_inode_sectorsize(inode)))
7920 		read_mode |= REQ_FAILFAST_DEV;
7921 
7922 	isector = start - btrfs_io_bio(failed_bio)->logical;
7923 	isector >>= inode->i_sb->s_blocksize_bits;
7924 	bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
7925 				pgoff, isector, repair_endio, repair_arg);
7926 	if (!bio) {
7927 		free_io_failure(inode, failrec);
7928 		return -EIO;
7929 	}
7930 	bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
7931 
7932 	btrfs_debug(BTRFS_I(inode)->root->fs_info,
7933 		    "Repair DIO Read Error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d\n",
7934 		    read_mode, failrec->this_mirror, failrec->in_validation);
7935 
7936 	ret = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
7937 	if (ret) {
7938 		free_io_failure(inode, failrec);
7939 		bio_put(bio);
7940 	}
7941 
7942 	return ret;
7943 }
7944 
7945 struct btrfs_retry_complete {
7946 	struct completion done;
7947 	struct inode *inode;
7948 	u64 start;
7949 	int uptodate;
7950 };
7951 
7952 static void btrfs_retry_endio_nocsum(struct bio *bio)
7953 {
7954 	struct btrfs_retry_complete *done = bio->bi_private;
7955 	struct inode *inode;
7956 	struct bio_vec *bvec;
7957 	int i;
7958 
7959 	if (bio->bi_error)
7960 		goto end;
7961 
7962 	ASSERT(bio->bi_vcnt == 1);
7963 	inode = bio->bi_io_vec->bv_page->mapping->host;
7964 	ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
7965 
7966 	done->uptodate = 1;
7967 	bio_for_each_segment_all(bvec, bio, i)
7968 		clean_io_failure(done->inode, done->start, bvec->bv_page, 0);
7969 end:
7970 	complete(&done->done);
7971 	bio_put(bio);
7972 }
7973 
7974 static int __btrfs_correct_data_nocsum(struct inode *inode,
7975 				       struct btrfs_io_bio *io_bio)
7976 {
7977 	struct btrfs_fs_info *fs_info;
7978 	struct bio_vec *bvec;
7979 	struct btrfs_retry_complete done;
7980 	u64 start;
7981 	unsigned int pgoff;
7982 	u32 sectorsize;
7983 	int nr_sectors;
7984 	int i;
7985 	int ret;
7986 
7987 	fs_info = BTRFS_I(inode)->root->fs_info;
7988 	sectorsize = fs_info->sectorsize;
7989 
7990 	start = io_bio->logical;
7991 	done.inode = inode;
7992 
7993 	bio_for_each_segment_all(bvec, &io_bio->bio, i) {
7994 		nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len);
7995 		pgoff = bvec->bv_offset;
7996 
7997 next_block_or_try_again:
7998 		done.uptodate = 0;
7999 		done.start = start;
8000 		init_completion(&done.done);
8001 
8002 		ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page,
8003 				pgoff, start, start + sectorsize - 1,
8004 				io_bio->mirror_num,
8005 				btrfs_retry_endio_nocsum, &done);
8006 		if (ret)
8007 			return ret;
8008 
8009 		wait_for_completion(&done.done);
8010 
8011 		if (!done.uptodate) {
8012 			/* We might have another mirror, so try again */
8013 			goto next_block_or_try_again;
8014 		}
8015 
8016 		start += sectorsize;
8017 
8018 		if (nr_sectors--) {
8019 			pgoff += sectorsize;
8020 			goto next_block_or_try_again;
8021 		}
8022 	}
8023 
8024 	return 0;
8025 }
8026 
8027 static void btrfs_retry_endio(struct bio *bio)
8028 {
8029 	struct btrfs_retry_complete *done = bio->bi_private;
8030 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8031 	struct inode *inode;
8032 	struct bio_vec *bvec;
8033 	u64 start;
8034 	int uptodate;
8035 	int ret;
8036 	int i;
8037 
8038 	if (bio->bi_error)
8039 		goto end;
8040 
8041 	uptodate = 1;
8042 
8043 	start = done->start;
8044 
8045 	ASSERT(bio->bi_vcnt == 1);
8046 	inode = bio->bi_io_vec->bv_page->mapping->host;
8047 	ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
8048 
8049 	bio_for_each_segment_all(bvec, bio, i) {
8050 		ret = __readpage_endio_check(done->inode, io_bio, i,
8051 					bvec->bv_page, bvec->bv_offset,
8052 					done->start, bvec->bv_len);
8053 		if (!ret)
8054 			clean_io_failure(done->inode, done->start,
8055 					bvec->bv_page, bvec->bv_offset);
8056 		else
8057 			uptodate = 0;
8058 	}
8059 
8060 	done->uptodate = uptodate;
8061 end:
8062 	complete(&done->done);
8063 	bio_put(bio);
8064 }
8065 
8066 static int __btrfs_subio_endio_read(struct inode *inode,
8067 				    struct btrfs_io_bio *io_bio, int err)
8068 {
8069 	struct btrfs_fs_info *fs_info;
8070 	struct bio_vec *bvec;
8071 	struct btrfs_retry_complete done;
8072 	u64 start;
8073 	u64 offset = 0;
8074 	u32 sectorsize;
8075 	int nr_sectors;
8076 	unsigned int pgoff;
8077 	int csum_pos;
8078 	int i;
8079 	int ret;
8080 
8081 	fs_info = BTRFS_I(inode)->root->fs_info;
8082 	sectorsize = fs_info->sectorsize;
8083 
8084 	err = 0;
8085 	start = io_bio->logical;
8086 	done.inode = inode;
8087 
8088 	bio_for_each_segment_all(bvec, &io_bio->bio, i) {
8089 		nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len);
8090 
8091 		pgoff = bvec->bv_offset;
8092 next_block:
8093 		csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset);
8094 		ret = __readpage_endio_check(inode, io_bio, csum_pos,
8095 					bvec->bv_page, pgoff, start,
8096 					sectorsize);
8097 		if (likely(!ret))
8098 			goto next;
8099 try_again:
8100 		done.uptodate = 0;
8101 		done.start = start;
8102 		init_completion(&done.done);
8103 
8104 		ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page,
8105 				pgoff, start, start + sectorsize - 1,
8106 				io_bio->mirror_num,
8107 				btrfs_retry_endio, &done);
8108 		if (ret) {
8109 			err = ret;
8110 			goto next;
8111 		}
8112 
8113 		wait_for_completion(&done.done);
8114 
8115 		if (!done.uptodate) {
8116 			/* We might have another mirror, so try again */
8117 			goto try_again;
8118 		}
8119 next:
8120 		offset += sectorsize;
8121 		start += sectorsize;
8122 
8123 		ASSERT(nr_sectors);
8124 
8125 		if (--nr_sectors) {
8126 			pgoff += sectorsize;
8127 			goto next_block;
8128 		}
8129 	}
8130 
8131 	return err;
8132 }
8133 
8134 static int btrfs_subio_endio_read(struct inode *inode,
8135 				  struct btrfs_io_bio *io_bio, int err)
8136 {
8137 	bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
8138 
8139 	if (skip_csum) {
8140 		if (unlikely(err))
8141 			return __btrfs_correct_data_nocsum(inode, io_bio);
8142 		else
8143 			return 0;
8144 	} else {
8145 		return __btrfs_subio_endio_read(inode, io_bio, err);
8146 	}
8147 }
8148 
8149 static void btrfs_endio_direct_read(struct bio *bio)
8150 {
8151 	struct btrfs_dio_private *dip = bio->bi_private;
8152 	struct inode *inode = dip->inode;
8153 	struct bio *dio_bio;
8154 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8155 	int err = bio->bi_error;
8156 
8157 	if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
8158 		err = btrfs_subio_endio_read(inode, io_bio, err);
8159 
8160 	unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
8161 		      dip->logical_offset + dip->bytes - 1);
8162 	dio_bio = dip->dio_bio;
8163 
8164 	kfree(dip);
8165 
8166 	dio_bio->bi_error = bio->bi_error;
8167 	dio_end_io(dio_bio, bio->bi_error);
8168 
8169 	if (io_bio->end_io)
8170 		io_bio->end_io(io_bio, err);
8171 	bio_put(bio);
8172 }
8173 
8174 static void btrfs_endio_direct_write_update_ordered(struct inode *inode,
8175 						    const u64 offset,
8176 						    const u64 bytes,
8177 						    const int uptodate)
8178 {
8179 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8180 	struct btrfs_ordered_extent *ordered = NULL;
8181 	u64 ordered_offset = offset;
8182 	u64 ordered_bytes = bytes;
8183 	int ret;
8184 
8185 again:
8186 	ret = btrfs_dec_test_first_ordered_pending(inode, &ordered,
8187 						   &ordered_offset,
8188 						   ordered_bytes,
8189 						   uptodate);
8190 	if (!ret)
8191 		goto out_test;
8192 
8193 	btrfs_init_work(&ordered->work, btrfs_endio_write_helper,
8194 			finish_ordered_fn, NULL, NULL);
8195 	btrfs_queue_work(fs_info->endio_write_workers, &ordered->work);
8196 out_test:
8197 	/*
8198 	 * our bio might span multiple ordered extents.  If we haven't
8199 	 * completed the accounting for the whole dio, go back and try again
8200 	 */
8201 	if (ordered_offset < offset + bytes) {
8202 		ordered_bytes = offset + bytes - ordered_offset;
8203 		ordered = NULL;
8204 		goto again;
8205 	}
8206 }
8207 
8208 static void btrfs_endio_direct_write(struct bio *bio)
8209 {
8210 	struct btrfs_dio_private *dip = bio->bi_private;
8211 	struct bio *dio_bio = dip->dio_bio;
8212 
8213 	btrfs_endio_direct_write_update_ordered(dip->inode,
8214 						dip->logical_offset,
8215 						dip->bytes,
8216 						!bio->bi_error);
8217 
8218 	kfree(dip);
8219 
8220 	dio_bio->bi_error = bio->bi_error;
8221 	dio_end_io(dio_bio, bio->bi_error);
8222 	bio_put(bio);
8223 }
8224 
8225 static int __btrfs_submit_bio_start_direct_io(struct inode *inode,
8226 				    struct bio *bio, int mirror_num,
8227 				    unsigned long bio_flags, u64 offset)
8228 {
8229 	int ret;
8230 	ret = btrfs_csum_one_bio(inode, bio, offset, 1);
8231 	BUG_ON(ret); /* -ENOMEM */
8232 	return 0;
8233 }
8234 
8235 static void btrfs_end_dio_bio(struct bio *bio)
8236 {
8237 	struct btrfs_dio_private *dip = bio->bi_private;
8238 	int err = bio->bi_error;
8239 
8240 	if (err)
8241 		btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
8242 			   "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
8243 			   btrfs_ino(dip->inode), bio_op(bio), bio->bi_opf,
8244 			   (unsigned long long)bio->bi_iter.bi_sector,
8245 			   bio->bi_iter.bi_size, err);
8246 
8247 	if (dip->subio_endio)
8248 		err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
8249 
8250 	if (err) {
8251 		dip->errors = 1;
8252 
8253 		/*
8254 		 * before atomic variable goto zero, we must make sure
8255 		 * dip->errors is perceived to be set.
8256 		 */
8257 		smp_mb__before_atomic();
8258 	}
8259 
8260 	/* if there are more bios still pending for this dio, just exit */
8261 	if (!atomic_dec_and_test(&dip->pending_bios))
8262 		goto out;
8263 
8264 	if (dip->errors) {
8265 		bio_io_error(dip->orig_bio);
8266 	} else {
8267 		dip->dio_bio->bi_error = 0;
8268 		bio_endio(dip->orig_bio);
8269 	}
8270 out:
8271 	bio_put(bio);
8272 }
8273 
8274 static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev,
8275 				       u64 first_sector, gfp_t gfp_flags)
8276 {
8277 	struct bio *bio;
8278 	bio = btrfs_bio_alloc(bdev, first_sector, BIO_MAX_PAGES, gfp_flags);
8279 	if (bio)
8280 		bio_associate_current(bio);
8281 	return bio;
8282 }
8283 
8284 static inline int btrfs_lookup_and_bind_dio_csum(struct inode *inode,
8285 						 struct btrfs_dio_private *dip,
8286 						 struct bio *bio,
8287 						 u64 file_offset)
8288 {
8289 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8290 	struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
8291 	int ret;
8292 
8293 	/*
8294 	 * We load all the csum data we need when we submit
8295 	 * the first bio to reduce the csum tree search and
8296 	 * contention.
8297 	 */
8298 	if (dip->logical_offset == file_offset) {
8299 		ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio,
8300 						file_offset);
8301 		if (ret)
8302 			return ret;
8303 	}
8304 
8305 	if (bio == dip->orig_bio)
8306 		return 0;
8307 
8308 	file_offset -= dip->logical_offset;
8309 	file_offset >>= inode->i_sb->s_blocksize_bits;
8310 	io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset);
8311 
8312 	return 0;
8313 }
8314 
8315 static inline int __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode,
8316 					 u64 file_offset, int skip_sum,
8317 					 int async_submit)
8318 {
8319 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8320 	struct btrfs_dio_private *dip = bio->bi_private;
8321 	bool write = bio_op(bio) == REQ_OP_WRITE;
8322 	int ret;
8323 
8324 	if (async_submit)
8325 		async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);
8326 
8327 	bio_get(bio);
8328 
8329 	if (!write) {
8330 		ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
8331 		if (ret)
8332 			goto err;
8333 	}
8334 
8335 	if (skip_sum)
8336 		goto map;
8337 
8338 	if (write && async_submit) {
8339 		ret = btrfs_wq_submit_bio(fs_info, inode, bio, 0, 0,
8340 					  file_offset,
8341 					  __btrfs_submit_bio_start_direct_io,
8342 					  __btrfs_submit_bio_done);
8343 		goto err;
8344 	} else if (write) {
8345 		/*
8346 		 * If we aren't doing async submit, calculate the csum of the
8347 		 * bio now.
8348 		 */
8349 		ret = btrfs_csum_one_bio(inode, bio, file_offset, 1);
8350 		if (ret)
8351 			goto err;
8352 	} else {
8353 		ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio,
8354 						     file_offset);
8355 		if (ret)
8356 			goto err;
8357 	}
8358 map:
8359 	ret = btrfs_map_bio(fs_info, bio, 0, async_submit);
8360 err:
8361 	bio_put(bio);
8362 	return ret;
8363 }
8364 
8365 static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip,
8366 				    int skip_sum)
8367 {
8368 	struct inode *inode = dip->inode;
8369 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8370 	struct btrfs_root *root = BTRFS_I(inode)->root;
8371 	struct bio *bio;
8372 	struct bio *orig_bio = dip->orig_bio;
8373 	struct bio_vec *bvec;
8374 	u64 start_sector = orig_bio->bi_iter.bi_sector;
8375 	u64 file_offset = dip->logical_offset;
8376 	u64 submit_len = 0;
8377 	u64 map_length;
8378 	u32 blocksize = fs_info->sectorsize;
8379 	int async_submit = 0;
8380 	int nr_sectors;
8381 	int ret;
8382 	int i, j;
8383 
8384 	map_length = orig_bio->bi_iter.bi_size;
8385 	ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9,
8386 			      &map_length, NULL, 0);
8387 	if (ret)
8388 		return -EIO;
8389 
8390 	if (map_length >= orig_bio->bi_iter.bi_size) {
8391 		bio = orig_bio;
8392 		dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
8393 		goto submit;
8394 	}
8395 
8396 	/* async crcs make it difficult to collect full stripe writes. */
8397 	if (btrfs_get_alloc_profile(root, 1) & BTRFS_BLOCK_GROUP_RAID56_MASK)
8398 		async_submit = 0;
8399 	else
8400 		async_submit = 1;
8401 
8402 	bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev, start_sector, GFP_NOFS);
8403 	if (!bio)
8404 		return -ENOMEM;
8405 
8406 	bio->bi_opf = orig_bio->bi_opf;
8407 	bio->bi_private = dip;
8408 	bio->bi_end_io = btrfs_end_dio_bio;
8409 	btrfs_io_bio(bio)->logical = file_offset;
8410 	atomic_inc(&dip->pending_bios);
8411 
8412 	bio_for_each_segment_all(bvec, orig_bio, j) {
8413 		nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len);
8414 		i = 0;
8415 next_block:
8416 		if (unlikely(map_length < submit_len + blocksize ||
8417 		    bio_add_page(bio, bvec->bv_page, blocksize,
8418 			    bvec->bv_offset + (i * blocksize)) < blocksize)) {
8419 			/*
8420 			 * inc the count before we submit the bio so
8421 			 * we know the end IO handler won't happen before
8422 			 * we inc the count. Otherwise, the dip might get freed
8423 			 * before we're done setting it up
8424 			 */
8425 			atomic_inc(&dip->pending_bios);
8426 			ret = __btrfs_submit_dio_bio(bio, inode,
8427 						     file_offset, skip_sum,
8428 						     async_submit);
8429 			if (ret) {
8430 				bio_put(bio);
8431 				atomic_dec(&dip->pending_bios);
8432 				goto out_err;
8433 			}
8434 
8435 			start_sector += submit_len >> 9;
8436 			file_offset += submit_len;
8437 
8438 			submit_len = 0;
8439 
8440 			bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev,
8441 						  start_sector, GFP_NOFS);
8442 			if (!bio)
8443 				goto out_err;
8444 			bio->bi_opf = orig_bio->bi_opf;
8445 			bio->bi_private = dip;
8446 			bio->bi_end_io = btrfs_end_dio_bio;
8447 			btrfs_io_bio(bio)->logical = file_offset;
8448 
8449 			map_length = orig_bio->bi_iter.bi_size;
8450 			ret = btrfs_map_block(fs_info, btrfs_op(orig_bio),
8451 					      start_sector << 9,
8452 					      &map_length, NULL, 0);
8453 			if (ret) {
8454 				bio_put(bio);
8455 				goto out_err;
8456 			}
8457 
8458 			goto next_block;
8459 		} else {
8460 			submit_len += blocksize;
8461 			if (--nr_sectors) {
8462 				i++;
8463 				goto next_block;
8464 			}
8465 		}
8466 	}
8467 
8468 submit:
8469 	ret = __btrfs_submit_dio_bio(bio, inode, file_offset, skip_sum,
8470 				     async_submit);
8471 	if (!ret)
8472 		return 0;
8473 
8474 	bio_put(bio);
8475 out_err:
8476 	dip->errors = 1;
8477 	/*
8478 	 * before atomic variable goto zero, we must
8479 	 * make sure dip->errors is perceived to be set.
8480 	 */
8481 	smp_mb__before_atomic();
8482 	if (atomic_dec_and_test(&dip->pending_bios))
8483 		bio_io_error(dip->orig_bio);
8484 
8485 	/* bio_end_io() will handle error, so we needn't return it */
8486 	return 0;
8487 }
8488 
8489 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
8490 				loff_t file_offset)
8491 {
8492 	struct btrfs_dio_private *dip = NULL;
8493 	struct bio *io_bio = NULL;
8494 	struct btrfs_io_bio *btrfs_bio;
8495 	int skip_sum;
8496 	bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
8497 	int ret = 0;
8498 
8499 	skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
8500 
8501 	io_bio = btrfs_bio_clone(dio_bio, GFP_NOFS);
8502 	if (!io_bio) {
8503 		ret = -ENOMEM;
8504 		goto free_ordered;
8505 	}
8506 
8507 	dip = kzalloc(sizeof(*dip), GFP_NOFS);
8508 	if (!dip) {
8509 		ret = -ENOMEM;
8510 		goto free_ordered;
8511 	}
8512 
8513 	dip->private = dio_bio->bi_private;
8514 	dip->inode = inode;
8515 	dip->logical_offset = file_offset;
8516 	dip->bytes = dio_bio->bi_iter.bi_size;
8517 	dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
8518 	io_bio->bi_private = dip;
8519 	dip->orig_bio = io_bio;
8520 	dip->dio_bio = dio_bio;
8521 	atomic_set(&dip->pending_bios, 0);
8522 	btrfs_bio = btrfs_io_bio(io_bio);
8523 	btrfs_bio->logical = file_offset;
8524 
8525 	if (write) {
8526 		io_bio->bi_end_io = btrfs_endio_direct_write;
8527 	} else {
8528 		io_bio->bi_end_io = btrfs_endio_direct_read;
8529 		dip->subio_endio = btrfs_subio_endio_read;
8530 	}
8531 
8532 	/*
8533 	 * Reset the range for unsubmitted ordered extents (to a 0 length range)
8534 	 * even if we fail to submit a bio, because in such case we do the
8535 	 * corresponding error handling below and it must not be done a second
8536 	 * time by btrfs_direct_IO().
8537 	 */
8538 	if (write) {
8539 		struct btrfs_dio_data *dio_data = current->journal_info;
8540 
8541 		dio_data->unsubmitted_oe_range_end = dip->logical_offset +
8542 			dip->bytes;
8543 		dio_data->unsubmitted_oe_range_start =
8544 			dio_data->unsubmitted_oe_range_end;
8545 	}
8546 
8547 	ret = btrfs_submit_direct_hook(dip, skip_sum);
8548 	if (!ret)
8549 		return;
8550 
8551 	if (btrfs_bio->end_io)
8552 		btrfs_bio->end_io(btrfs_bio, ret);
8553 
8554 free_ordered:
8555 	/*
8556 	 * If we arrived here it means either we failed to submit the dip
8557 	 * or we either failed to clone the dio_bio or failed to allocate the
8558 	 * dip. If we cloned the dio_bio and allocated the dip, we can just
8559 	 * call bio_endio against our io_bio so that we get proper resource
8560 	 * cleanup if we fail to submit the dip, otherwise, we must do the
8561 	 * same as btrfs_endio_direct_[write|read] because we can't call these
8562 	 * callbacks - they require an allocated dip and a clone of dio_bio.
8563 	 */
8564 	if (io_bio && dip) {
8565 		io_bio->bi_error = -EIO;
8566 		bio_endio(io_bio);
8567 		/*
8568 		 * The end io callbacks free our dip, do the final put on io_bio
8569 		 * and all the cleanup and final put for dio_bio (through
8570 		 * dio_end_io()).
8571 		 */
8572 		dip = NULL;
8573 		io_bio = NULL;
8574 	} else {
8575 		if (write)
8576 			btrfs_endio_direct_write_update_ordered(inode,
8577 						file_offset,
8578 						dio_bio->bi_iter.bi_size,
8579 						0);
8580 		else
8581 			unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
8582 			      file_offset + dio_bio->bi_iter.bi_size - 1);
8583 
8584 		dio_bio->bi_error = -EIO;
8585 		/*
8586 		 * Releases and cleans up our dio_bio, no need to bio_put()
8587 		 * nor bio_endio()/bio_io_error() against dio_bio.
8588 		 */
8589 		dio_end_io(dio_bio, ret);
8590 	}
8591 	if (io_bio)
8592 		bio_put(io_bio);
8593 	kfree(dip);
8594 }
8595 
8596 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
8597 			       struct kiocb *iocb,
8598 			       const struct iov_iter *iter, loff_t offset)
8599 {
8600 	int seg;
8601 	int i;
8602 	unsigned int blocksize_mask = fs_info->sectorsize - 1;
8603 	ssize_t retval = -EINVAL;
8604 
8605 	if (offset & blocksize_mask)
8606 		goto out;
8607 
8608 	if (iov_iter_alignment(iter) & blocksize_mask)
8609 		goto out;
8610 
8611 	/* If this is a write we don't need to check anymore */
8612 	if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
8613 		return 0;
8614 	/*
8615 	 * Check to make sure we don't have duplicate iov_base's in this
8616 	 * iovec, if so return EINVAL, otherwise we'll get csum errors
8617 	 * when reading back.
8618 	 */
8619 	for (seg = 0; seg < iter->nr_segs; seg++) {
8620 		for (i = seg + 1; i < iter->nr_segs; i++) {
8621 			if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
8622 				goto out;
8623 		}
8624 	}
8625 	retval = 0;
8626 out:
8627 	return retval;
8628 }
8629 
8630 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
8631 {
8632 	struct file *file = iocb->ki_filp;
8633 	struct inode *inode = file->f_mapping->host;
8634 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8635 	struct btrfs_dio_data dio_data = { 0 };
8636 	loff_t offset = iocb->ki_pos;
8637 	size_t count = 0;
8638 	int flags = 0;
8639 	bool wakeup = true;
8640 	bool relock = false;
8641 	ssize_t ret;
8642 
8643 	if (check_direct_IO(fs_info, iocb, iter, offset))
8644 		return 0;
8645 
8646 	inode_dio_begin(inode);
8647 	smp_mb__after_atomic();
8648 
8649 	/*
8650 	 * The generic stuff only does filemap_write_and_wait_range, which
8651 	 * isn't enough if we've written compressed pages to this area, so
8652 	 * we need to flush the dirty pages again to make absolutely sure
8653 	 * that any outstanding dirty pages are on disk.
8654 	 */
8655 	count = iov_iter_count(iter);
8656 	if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8657 		     &BTRFS_I(inode)->runtime_flags))
8658 		filemap_fdatawrite_range(inode->i_mapping, offset,
8659 					 offset + count - 1);
8660 
8661 	if (iov_iter_rw(iter) == WRITE) {
8662 		/*
8663 		 * If the write DIO is beyond the EOF, we need update
8664 		 * the isize, but it is protected by i_mutex. So we can
8665 		 * not unlock the i_mutex at this case.
8666 		 */
8667 		if (offset + count <= inode->i_size) {
8668 			inode_unlock(inode);
8669 			relock = true;
8670 		}
8671 		ret = btrfs_delalloc_reserve_space(inode, offset, count);
8672 		if (ret)
8673 			goto out;
8674 		dio_data.outstanding_extents = div64_u64(count +
8675 						BTRFS_MAX_EXTENT_SIZE - 1,
8676 						BTRFS_MAX_EXTENT_SIZE);
8677 
8678 		/*
8679 		 * We need to know how many extents we reserved so that we can
8680 		 * do the accounting properly if we go over the number we
8681 		 * originally calculated.  Abuse current->journal_info for this.
8682 		 */
8683 		dio_data.reserve = round_up(count,
8684 					    fs_info->sectorsize);
8685 		dio_data.unsubmitted_oe_range_start = (u64)offset;
8686 		dio_data.unsubmitted_oe_range_end = (u64)offset;
8687 		current->journal_info = &dio_data;
8688 	} else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8689 				     &BTRFS_I(inode)->runtime_flags)) {
8690 		inode_dio_end(inode);
8691 		flags = DIO_LOCKING | DIO_SKIP_HOLES;
8692 		wakeup = false;
8693 	}
8694 
8695 	ret = __blockdev_direct_IO(iocb, inode,
8696 				   fs_info->fs_devices->latest_bdev,
8697 				   iter, btrfs_get_blocks_direct, NULL,
8698 				   btrfs_submit_direct, flags);
8699 	if (iov_iter_rw(iter) == WRITE) {
8700 		current->journal_info = NULL;
8701 		if (ret < 0 && ret != -EIOCBQUEUED) {
8702 			if (dio_data.reserve)
8703 				btrfs_delalloc_release_space(inode, offset,
8704 							     dio_data.reserve);
8705 			/*
8706 			 * On error we might have left some ordered extents
8707 			 * without submitting corresponding bios for them, so
8708 			 * cleanup them up to avoid other tasks getting them
8709 			 * and waiting for them to complete forever.
8710 			 */
8711 			if (dio_data.unsubmitted_oe_range_start <
8712 			    dio_data.unsubmitted_oe_range_end)
8713 				btrfs_endio_direct_write_update_ordered(inode,
8714 					dio_data.unsubmitted_oe_range_start,
8715 					dio_data.unsubmitted_oe_range_end -
8716 					dio_data.unsubmitted_oe_range_start,
8717 					0);
8718 		} else if (ret >= 0 && (size_t)ret < count)
8719 			btrfs_delalloc_release_space(inode, offset,
8720 						     count - (size_t)ret);
8721 	}
8722 out:
8723 	if (wakeup)
8724 		inode_dio_end(inode);
8725 	if (relock)
8726 		inode_lock(inode);
8727 
8728 	return ret;
8729 }
8730 
8731 #define BTRFS_FIEMAP_FLAGS	(FIEMAP_FLAG_SYNC)
8732 
8733 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
8734 		__u64 start, __u64 len)
8735 {
8736 	int	ret;
8737 
8738 	ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS);
8739 	if (ret)
8740 		return ret;
8741 
8742 	return extent_fiemap(inode, fieinfo, start, len, btrfs_get_extent_fiemap);
8743 }
8744 
8745 int btrfs_readpage(struct file *file, struct page *page)
8746 {
8747 	struct extent_io_tree *tree;
8748 	tree = &BTRFS_I(page->mapping->host)->io_tree;
8749 	return extent_read_full_page(tree, page, btrfs_get_extent, 0);
8750 }
8751 
8752 static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
8753 {
8754 	struct extent_io_tree *tree;
8755 	struct inode *inode = page->mapping->host;
8756 	int ret;
8757 
8758 	if (current->flags & PF_MEMALLOC) {
8759 		redirty_page_for_writepage(wbc, page);
8760 		unlock_page(page);
8761 		return 0;
8762 	}
8763 
8764 	/*
8765 	 * If we are under memory pressure we will call this directly from the
8766 	 * VM, we need to make sure we have the inode referenced for the ordered
8767 	 * extent.  If not just return like we didn't do anything.
8768 	 */
8769 	if (!igrab(inode)) {
8770 		redirty_page_for_writepage(wbc, page);
8771 		return AOP_WRITEPAGE_ACTIVATE;
8772 	}
8773 	tree = &BTRFS_I(page->mapping->host)->io_tree;
8774 	ret = extent_write_full_page(tree, page, btrfs_get_extent, wbc);
8775 	btrfs_add_delayed_iput(inode);
8776 	return ret;
8777 }
8778 
8779 static int btrfs_writepages(struct address_space *mapping,
8780 			    struct writeback_control *wbc)
8781 {
8782 	struct extent_io_tree *tree;
8783 
8784 	tree = &BTRFS_I(mapping->host)->io_tree;
8785 	return extent_writepages(tree, mapping, btrfs_get_extent, wbc);
8786 }
8787 
8788 static int
8789 btrfs_readpages(struct file *file, struct address_space *mapping,
8790 		struct list_head *pages, unsigned nr_pages)
8791 {
8792 	struct extent_io_tree *tree;
8793 	tree = &BTRFS_I(mapping->host)->io_tree;
8794 	return extent_readpages(tree, mapping, pages, nr_pages,
8795 				btrfs_get_extent);
8796 }
8797 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8798 {
8799 	struct extent_io_tree *tree;
8800 	struct extent_map_tree *map;
8801 	int ret;
8802 
8803 	tree = &BTRFS_I(page->mapping->host)->io_tree;
8804 	map = &BTRFS_I(page->mapping->host)->extent_tree;
8805 	ret = try_release_extent_mapping(map, tree, page, gfp_flags);
8806 	if (ret == 1) {
8807 		ClearPagePrivate(page);
8808 		set_page_private(page, 0);
8809 		put_page(page);
8810 	}
8811 	return ret;
8812 }
8813 
8814 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8815 {
8816 	if (PageWriteback(page) || PageDirty(page))
8817 		return 0;
8818 	return __btrfs_releasepage(page, gfp_flags & GFP_NOFS);
8819 }
8820 
8821 static void btrfs_invalidatepage(struct page *page, unsigned int offset,
8822 				 unsigned int length)
8823 {
8824 	struct inode *inode = page->mapping->host;
8825 	struct extent_io_tree *tree;
8826 	struct btrfs_ordered_extent *ordered;
8827 	struct extent_state *cached_state = NULL;
8828 	u64 page_start = page_offset(page);
8829 	u64 page_end = page_start + PAGE_SIZE - 1;
8830 	u64 start;
8831 	u64 end;
8832 	int inode_evicting = inode->i_state & I_FREEING;
8833 
8834 	/*
8835 	 * we have the page locked, so new writeback can't start,
8836 	 * and the dirty bit won't be cleared while we are here.
8837 	 *
8838 	 * Wait for IO on this page so that we can safely clear
8839 	 * the PagePrivate2 bit and do ordered accounting
8840 	 */
8841 	wait_on_page_writeback(page);
8842 
8843 	tree = &BTRFS_I(inode)->io_tree;
8844 	if (offset) {
8845 		btrfs_releasepage(page, GFP_NOFS);
8846 		return;
8847 	}
8848 
8849 	if (!inode_evicting)
8850 		lock_extent_bits(tree, page_start, page_end, &cached_state);
8851 again:
8852 	start = page_start;
8853 	ordered = btrfs_lookup_ordered_range(inode, start,
8854 					page_end - start + 1);
8855 	if (ordered) {
8856 		end = min(page_end, ordered->file_offset + ordered->len - 1);
8857 		/*
8858 		 * IO on this page will never be started, so we need
8859 		 * to account for any ordered extents now
8860 		 */
8861 		if (!inode_evicting)
8862 			clear_extent_bit(tree, start, end,
8863 					 EXTENT_DIRTY | EXTENT_DELALLOC |
8864 					 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
8865 					 EXTENT_DEFRAG, 1, 0, &cached_state,
8866 					 GFP_NOFS);
8867 		/*
8868 		 * whoever cleared the private bit is responsible
8869 		 * for the finish_ordered_io
8870 		 */
8871 		if (TestClearPagePrivate2(page)) {
8872 			struct btrfs_ordered_inode_tree *tree;
8873 			u64 new_len;
8874 
8875 			tree = &BTRFS_I(inode)->ordered_tree;
8876 
8877 			spin_lock_irq(&tree->lock);
8878 			set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
8879 			new_len = start - ordered->file_offset;
8880 			if (new_len < ordered->truncated_len)
8881 				ordered->truncated_len = new_len;
8882 			spin_unlock_irq(&tree->lock);
8883 
8884 			if (btrfs_dec_test_ordered_pending(inode, &ordered,
8885 							   start,
8886 							   end - start + 1, 1))
8887 				btrfs_finish_ordered_io(ordered);
8888 		}
8889 		btrfs_put_ordered_extent(ordered);
8890 		if (!inode_evicting) {
8891 			cached_state = NULL;
8892 			lock_extent_bits(tree, start, end,
8893 					 &cached_state);
8894 		}
8895 
8896 		start = end + 1;
8897 		if (start < page_end)
8898 			goto again;
8899 	}
8900 
8901 	/*
8902 	 * Qgroup reserved space handler
8903 	 * Page here will be either
8904 	 * 1) Already written to disk
8905 	 *    In this case, its reserved space is released from data rsv map
8906 	 *    and will be freed by delayed_ref handler finally.
8907 	 *    So even we call qgroup_free_data(), it won't decrease reserved
8908 	 *    space.
8909 	 * 2) Not written to disk
8910 	 *    This means the reserved space should be freed here. However,
8911 	 *    if a truncate invalidates the page (by clearing PageDirty)
8912 	 *    and the page is accounted for while allocating extent
8913 	 *    in btrfs_check_data_free_space() we let delayed_ref to
8914 	 *    free the entire extent.
8915 	 */
8916 	if (PageDirty(page))
8917 		btrfs_qgroup_free_data(inode, page_start, PAGE_SIZE);
8918 	if (!inode_evicting) {
8919 		clear_extent_bit(tree, page_start, page_end,
8920 				 EXTENT_LOCKED | EXTENT_DIRTY |
8921 				 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
8922 				 EXTENT_DEFRAG, 1, 1,
8923 				 &cached_state, GFP_NOFS);
8924 
8925 		__btrfs_releasepage(page, GFP_NOFS);
8926 	}
8927 
8928 	ClearPageChecked(page);
8929 	if (PagePrivate(page)) {
8930 		ClearPagePrivate(page);
8931 		set_page_private(page, 0);
8932 		put_page(page);
8933 	}
8934 }
8935 
8936 /*
8937  * btrfs_page_mkwrite() is not allowed to change the file size as it gets
8938  * called from a page fault handler when a page is first dirtied. Hence we must
8939  * be careful to check for EOF conditions here. We set the page up correctly
8940  * for a written page which means we get ENOSPC checking when writing into
8941  * holes and correct delalloc and unwritten extent mapping on filesystems that
8942  * support these features.
8943  *
8944  * We are not allowed to take the i_mutex here so we have to play games to
8945  * protect against truncate races as the page could now be beyond EOF.  Because
8946  * vmtruncate() writes the inode size before removing pages, once we have the
8947  * page lock we can determine safely if the page is beyond EOF. If it is not
8948  * beyond EOF, then the page is guaranteed safe against truncation until we
8949  * unlock the page.
8950  */
8951 int btrfs_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
8952 {
8953 	struct page *page = vmf->page;
8954 	struct inode *inode = file_inode(vma->vm_file);
8955 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8956 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
8957 	struct btrfs_ordered_extent *ordered;
8958 	struct extent_state *cached_state = NULL;
8959 	char *kaddr;
8960 	unsigned long zero_start;
8961 	loff_t size;
8962 	int ret;
8963 	int reserved = 0;
8964 	u64 reserved_space;
8965 	u64 page_start;
8966 	u64 page_end;
8967 	u64 end;
8968 
8969 	reserved_space = PAGE_SIZE;
8970 
8971 	sb_start_pagefault(inode->i_sb);
8972 	page_start = page_offset(page);
8973 	page_end = page_start + PAGE_SIZE - 1;
8974 	end = page_end;
8975 
8976 	/*
8977 	 * Reserving delalloc space after obtaining the page lock can lead to
8978 	 * deadlock. For example, if a dirty page is locked by this function
8979 	 * and the call to btrfs_delalloc_reserve_space() ends up triggering
8980 	 * dirty page write out, then the btrfs_writepage() function could
8981 	 * end up waiting indefinitely to get a lock on the page currently
8982 	 * being processed by btrfs_page_mkwrite() function.
8983 	 */
8984 	ret = btrfs_delalloc_reserve_space(inode, page_start,
8985 					   reserved_space);
8986 	if (!ret) {
8987 		ret = file_update_time(vma->vm_file);
8988 		reserved = 1;
8989 	}
8990 	if (ret) {
8991 		if (ret == -ENOMEM)
8992 			ret = VM_FAULT_OOM;
8993 		else /* -ENOSPC, -EIO, etc */
8994 			ret = VM_FAULT_SIGBUS;
8995 		if (reserved)
8996 			goto out;
8997 		goto out_noreserve;
8998 	}
8999 
9000 	ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
9001 again:
9002 	lock_page(page);
9003 	size = i_size_read(inode);
9004 
9005 	if ((page->mapping != inode->i_mapping) ||
9006 	    (page_start >= size)) {
9007 		/* page got truncated out from underneath us */
9008 		goto out_unlock;
9009 	}
9010 	wait_on_page_writeback(page);
9011 
9012 	lock_extent_bits(io_tree, page_start, page_end, &cached_state);
9013 	set_page_extent_mapped(page);
9014 
9015 	/*
9016 	 * we can't set the delalloc bits if there are pending ordered
9017 	 * extents.  Drop our locks and wait for them to finish
9018 	 */
9019 	ordered = btrfs_lookup_ordered_range(inode, page_start, page_end);
9020 	if (ordered) {
9021 		unlock_extent_cached(io_tree, page_start, page_end,
9022 				     &cached_state, GFP_NOFS);
9023 		unlock_page(page);
9024 		btrfs_start_ordered_extent(inode, ordered, 1);
9025 		btrfs_put_ordered_extent(ordered);
9026 		goto again;
9027 	}
9028 
9029 	if (page->index == ((size - 1) >> PAGE_SHIFT)) {
9030 		reserved_space = round_up(size - page_start,
9031 					  fs_info->sectorsize);
9032 		if (reserved_space < PAGE_SIZE) {
9033 			end = page_start + reserved_space - 1;
9034 			spin_lock(&BTRFS_I(inode)->lock);
9035 			BTRFS_I(inode)->outstanding_extents++;
9036 			spin_unlock(&BTRFS_I(inode)->lock);
9037 			btrfs_delalloc_release_space(inode, page_start,
9038 						PAGE_SIZE - reserved_space);
9039 		}
9040 	}
9041 
9042 	/*
9043 	 * XXX - page_mkwrite gets called every time the page is dirtied, even
9044 	 * if it was already dirty, so for space accounting reasons we need to
9045 	 * clear any delalloc bits for the range we are fixing to save.  There
9046 	 * is probably a better way to do this, but for now keep consistent with
9047 	 * prepare_pages in the normal write path.
9048 	 */
9049 	clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end,
9050 			  EXTENT_DIRTY | EXTENT_DELALLOC |
9051 			  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
9052 			  0, 0, &cached_state, GFP_NOFS);
9053 
9054 	ret = btrfs_set_extent_delalloc(inode, page_start, end,
9055 					&cached_state, 0);
9056 	if (ret) {
9057 		unlock_extent_cached(io_tree, page_start, page_end,
9058 				     &cached_state, GFP_NOFS);
9059 		ret = VM_FAULT_SIGBUS;
9060 		goto out_unlock;
9061 	}
9062 	ret = 0;
9063 
9064 	/* page is wholly or partially inside EOF */
9065 	if (page_start + PAGE_SIZE > size)
9066 		zero_start = size & ~PAGE_MASK;
9067 	else
9068 		zero_start = PAGE_SIZE;
9069 
9070 	if (zero_start != PAGE_SIZE) {
9071 		kaddr = kmap(page);
9072 		memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start);
9073 		flush_dcache_page(page);
9074 		kunmap(page);
9075 	}
9076 	ClearPageChecked(page);
9077 	set_page_dirty(page);
9078 	SetPageUptodate(page);
9079 
9080 	BTRFS_I(inode)->last_trans = fs_info->generation;
9081 	BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid;
9082 	BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit;
9083 
9084 	unlock_extent_cached(io_tree, page_start, page_end, &cached_state, GFP_NOFS);
9085 
9086 out_unlock:
9087 	if (!ret) {
9088 		sb_end_pagefault(inode->i_sb);
9089 		return VM_FAULT_LOCKED;
9090 	}
9091 	unlock_page(page);
9092 out:
9093 	btrfs_delalloc_release_space(inode, page_start, reserved_space);
9094 out_noreserve:
9095 	sb_end_pagefault(inode->i_sb);
9096 	return ret;
9097 }
9098 
9099 static int btrfs_truncate(struct inode *inode)
9100 {
9101 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9102 	struct btrfs_root *root = BTRFS_I(inode)->root;
9103 	struct btrfs_block_rsv *rsv;
9104 	int ret = 0;
9105 	int err = 0;
9106 	struct btrfs_trans_handle *trans;
9107 	u64 mask = fs_info->sectorsize - 1;
9108 	u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
9109 
9110 	ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
9111 				       (u64)-1);
9112 	if (ret)
9113 		return ret;
9114 
9115 	/*
9116 	 * Yes ladies and gentlemen, this is indeed ugly.  The fact is we have
9117 	 * 3 things going on here
9118 	 *
9119 	 * 1) We need to reserve space for our orphan item and the space to
9120 	 * delete our orphan item.  Lord knows we don't want to have a dangling
9121 	 * orphan item because we didn't reserve space to remove it.
9122 	 *
9123 	 * 2) We need to reserve space to update our inode.
9124 	 *
9125 	 * 3) We need to have something to cache all the space that is going to
9126 	 * be free'd up by the truncate operation, but also have some slack
9127 	 * space reserved in case it uses space during the truncate (thank you
9128 	 * very much snapshotting).
9129 	 *
9130 	 * And we need these to all be separate.  The fact is we can use a lot of
9131 	 * space doing the truncate, and we have no earthly idea how much space
9132 	 * we will use, so we need the truncate reservation to be separate so it
9133 	 * doesn't end up using space reserved for updating the inode or
9134 	 * removing the orphan item.  We also need to be able to stop the
9135 	 * transaction and start a new one, which means we need to be able to
9136 	 * update the inode several times, and we have no idea of knowing how
9137 	 * many times that will be, so we can't just reserve 1 item for the
9138 	 * entirety of the operation, so that has to be done separately as well.
9139 	 * Then there is the orphan item, which does indeed need to be held on
9140 	 * to for the whole operation, and we need nobody to touch this reserved
9141 	 * space except the orphan code.
9142 	 *
9143 	 * So that leaves us with
9144 	 *
9145 	 * 1) root->orphan_block_rsv - for the orphan deletion.
9146 	 * 2) rsv - for the truncate reservation, which we will steal from the
9147 	 * transaction reservation.
9148 	 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for
9149 	 * updating the inode.
9150 	 */
9151 	rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
9152 	if (!rsv)
9153 		return -ENOMEM;
9154 	rsv->size = min_size;
9155 	rsv->failfast = 1;
9156 
9157 	/*
9158 	 * 1 for the truncate slack space
9159 	 * 1 for updating the inode.
9160 	 */
9161 	trans = btrfs_start_transaction(root, 2);
9162 	if (IS_ERR(trans)) {
9163 		err = PTR_ERR(trans);
9164 		goto out;
9165 	}
9166 
9167 	/* Migrate the slack space for the truncate to our reserve */
9168 	ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
9169 				      min_size, 0);
9170 	BUG_ON(ret);
9171 
9172 	/*
9173 	 * So if we truncate and then write and fsync we normally would just
9174 	 * write the extents that changed, which is a problem if we need to
9175 	 * first truncate that entire inode.  So set this flag so we write out
9176 	 * all of the extents in the inode to the sync log so we're completely
9177 	 * safe.
9178 	 */
9179 	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
9180 	trans->block_rsv = rsv;
9181 
9182 	while (1) {
9183 		ret = btrfs_truncate_inode_items(trans, root, inode,
9184 						 inode->i_size,
9185 						 BTRFS_EXTENT_DATA_KEY);
9186 		if (ret != -ENOSPC && ret != -EAGAIN) {
9187 			err = ret;
9188 			break;
9189 		}
9190 
9191 		trans->block_rsv = &fs_info->trans_block_rsv;
9192 		ret = btrfs_update_inode(trans, root, inode);
9193 		if (ret) {
9194 			err = ret;
9195 			break;
9196 		}
9197 
9198 		btrfs_end_transaction(trans);
9199 		btrfs_btree_balance_dirty(fs_info);
9200 
9201 		trans = btrfs_start_transaction(root, 2);
9202 		if (IS_ERR(trans)) {
9203 			ret = err = PTR_ERR(trans);
9204 			trans = NULL;
9205 			break;
9206 		}
9207 
9208 		ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
9209 					      rsv, min_size, 0);
9210 		BUG_ON(ret);	/* shouldn't happen */
9211 		trans->block_rsv = rsv;
9212 	}
9213 
9214 	if (ret == 0 && inode->i_nlink > 0) {
9215 		trans->block_rsv = root->orphan_block_rsv;
9216 		ret = btrfs_orphan_del(trans, inode);
9217 		if (ret)
9218 			err = ret;
9219 	}
9220 
9221 	if (trans) {
9222 		trans->block_rsv = &fs_info->trans_block_rsv;
9223 		ret = btrfs_update_inode(trans, root, inode);
9224 		if (ret && !err)
9225 			err = ret;
9226 
9227 		ret = btrfs_end_transaction(trans);
9228 		btrfs_btree_balance_dirty(fs_info);
9229 	}
9230 out:
9231 	btrfs_free_block_rsv(fs_info, rsv);
9232 
9233 	if (ret && !err)
9234 		err = ret;
9235 
9236 	return err;
9237 }
9238 
9239 /*
9240  * create a new subvolume directory/inode (helper for the ioctl).
9241  */
9242 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
9243 			     struct btrfs_root *new_root,
9244 			     struct btrfs_root *parent_root,
9245 			     u64 new_dirid)
9246 {
9247 	struct inode *inode;
9248 	int err;
9249 	u64 index = 0;
9250 
9251 	inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
9252 				new_dirid, new_dirid,
9253 				S_IFDIR | (~current_umask() & S_IRWXUGO),
9254 				&index);
9255 	if (IS_ERR(inode))
9256 		return PTR_ERR(inode);
9257 	inode->i_op = &btrfs_dir_inode_operations;
9258 	inode->i_fop = &btrfs_dir_file_operations;
9259 
9260 	set_nlink(inode, 1);
9261 	btrfs_i_size_write(inode, 0);
9262 	unlock_new_inode(inode);
9263 
9264 	err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
9265 	if (err)
9266 		btrfs_err(new_root->fs_info,
9267 			  "error inheriting subvolume %llu properties: %d",
9268 			  new_root->root_key.objectid, err);
9269 
9270 	err = btrfs_update_inode(trans, new_root, inode);
9271 
9272 	iput(inode);
9273 	return err;
9274 }
9275 
9276 struct inode *btrfs_alloc_inode(struct super_block *sb)
9277 {
9278 	struct btrfs_inode *ei;
9279 	struct inode *inode;
9280 
9281 	ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_NOFS);
9282 	if (!ei)
9283 		return NULL;
9284 
9285 	ei->root = NULL;
9286 	ei->generation = 0;
9287 	ei->last_trans = 0;
9288 	ei->last_sub_trans = 0;
9289 	ei->logged_trans = 0;
9290 	ei->delalloc_bytes = 0;
9291 	ei->defrag_bytes = 0;
9292 	ei->disk_i_size = 0;
9293 	ei->flags = 0;
9294 	ei->csum_bytes = 0;
9295 	ei->index_cnt = (u64)-1;
9296 	ei->dir_index = 0;
9297 	ei->last_unlink_trans = 0;
9298 	ei->last_log_commit = 0;
9299 	ei->delayed_iput_count = 0;
9300 
9301 	spin_lock_init(&ei->lock);
9302 	ei->outstanding_extents = 0;
9303 	ei->reserved_extents = 0;
9304 
9305 	ei->runtime_flags = 0;
9306 	ei->force_compress = BTRFS_COMPRESS_NONE;
9307 
9308 	ei->delayed_node = NULL;
9309 
9310 	ei->i_otime.tv_sec = 0;
9311 	ei->i_otime.tv_nsec = 0;
9312 
9313 	inode = &ei->vfs_inode;
9314 	extent_map_tree_init(&ei->extent_tree);
9315 	extent_io_tree_init(&ei->io_tree, &inode->i_data);
9316 	extent_io_tree_init(&ei->io_failure_tree, &inode->i_data);
9317 	ei->io_tree.track_uptodate = 1;
9318 	ei->io_failure_tree.track_uptodate = 1;
9319 	atomic_set(&ei->sync_writers, 0);
9320 	mutex_init(&ei->log_mutex);
9321 	mutex_init(&ei->delalloc_mutex);
9322 	btrfs_ordered_inode_tree_init(&ei->ordered_tree);
9323 	INIT_LIST_HEAD(&ei->delalloc_inodes);
9324 	INIT_LIST_HEAD(&ei->delayed_iput);
9325 	RB_CLEAR_NODE(&ei->rb_node);
9326 	init_rwsem(&ei->dio_sem);
9327 
9328 	return inode;
9329 }
9330 
9331 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
9332 void btrfs_test_destroy_inode(struct inode *inode)
9333 {
9334 	btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
9335 	kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9336 }
9337 #endif
9338 
9339 static void btrfs_i_callback(struct rcu_head *head)
9340 {
9341 	struct inode *inode = container_of(head, struct inode, i_rcu);
9342 	kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9343 }
9344 
9345 void btrfs_destroy_inode(struct inode *inode)
9346 {
9347 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9348 	struct btrfs_ordered_extent *ordered;
9349 	struct btrfs_root *root = BTRFS_I(inode)->root;
9350 
9351 	WARN_ON(!hlist_empty(&inode->i_dentry));
9352 	WARN_ON(inode->i_data.nrpages);
9353 	WARN_ON(BTRFS_I(inode)->outstanding_extents);
9354 	WARN_ON(BTRFS_I(inode)->reserved_extents);
9355 	WARN_ON(BTRFS_I(inode)->delalloc_bytes);
9356 	WARN_ON(BTRFS_I(inode)->csum_bytes);
9357 	WARN_ON(BTRFS_I(inode)->defrag_bytes);
9358 
9359 	/*
9360 	 * This can happen where we create an inode, but somebody else also
9361 	 * created the same inode and we need to destroy the one we already
9362 	 * created.
9363 	 */
9364 	if (!root)
9365 		goto free;
9366 
9367 	if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
9368 		     &BTRFS_I(inode)->runtime_flags)) {
9369 		btrfs_info(fs_info, "inode %llu still on the orphan list",
9370 			   btrfs_ino(inode));
9371 		atomic_dec(&root->orphan_inodes);
9372 	}
9373 
9374 	while (1) {
9375 		ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
9376 		if (!ordered)
9377 			break;
9378 		else {
9379 			btrfs_err(fs_info,
9380 				  "found ordered extent %llu %llu on inode cleanup",
9381 				  ordered->file_offset, ordered->len);
9382 			btrfs_remove_ordered_extent(inode, ordered);
9383 			btrfs_put_ordered_extent(ordered);
9384 			btrfs_put_ordered_extent(ordered);
9385 		}
9386 	}
9387 	btrfs_qgroup_check_reserved_leak(inode);
9388 	inode_tree_del(inode);
9389 	btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
9390 free:
9391 	call_rcu(&inode->i_rcu, btrfs_i_callback);
9392 }
9393 
9394 int btrfs_drop_inode(struct inode *inode)
9395 {
9396 	struct btrfs_root *root = BTRFS_I(inode)->root;
9397 
9398 	if (root == NULL)
9399 		return 1;
9400 
9401 	/* the snap/subvol tree is on deleting */
9402 	if (btrfs_root_refs(&root->root_item) == 0)
9403 		return 1;
9404 	else
9405 		return generic_drop_inode(inode);
9406 }
9407 
9408 static void init_once(void *foo)
9409 {
9410 	struct btrfs_inode *ei = (struct btrfs_inode *) foo;
9411 
9412 	inode_init_once(&ei->vfs_inode);
9413 }
9414 
9415 void btrfs_destroy_cachep(void)
9416 {
9417 	/*
9418 	 * Make sure all delayed rcu free inodes are flushed before we
9419 	 * destroy cache.
9420 	 */
9421 	rcu_barrier();
9422 	kmem_cache_destroy(btrfs_inode_cachep);
9423 	kmem_cache_destroy(btrfs_trans_handle_cachep);
9424 	kmem_cache_destroy(btrfs_transaction_cachep);
9425 	kmem_cache_destroy(btrfs_path_cachep);
9426 	kmem_cache_destroy(btrfs_free_space_cachep);
9427 }
9428 
9429 int btrfs_init_cachep(void)
9430 {
9431 	btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
9432 			sizeof(struct btrfs_inode), 0,
9433 			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
9434 			init_once);
9435 	if (!btrfs_inode_cachep)
9436 		goto fail;
9437 
9438 	btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
9439 			sizeof(struct btrfs_trans_handle), 0,
9440 			SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
9441 	if (!btrfs_trans_handle_cachep)
9442 		goto fail;
9443 
9444 	btrfs_transaction_cachep = kmem_cache_create("btrfs_transaction",
9445 			sizeof(struct btrfs_transaction), 0,
9446 			SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
9447 	if (!btrfs_transaction_cachep)
9448 		goto fail;
9449 
9450 	btrfs_path_cachep = kmem_cache_create("btrfs_path",
9451 			sizeof(struct btrfs_path), 0,
9452 			SLAB_MEM_SPREAD, NULL);
9453 	if (!btrfs_path_cachep)
9454 		goto fail;
9455 
9456 	btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
9457 			sizeof(struct btrfs_free_space), 0,
9458 			SLAB_MEM_SPREAD, NULL);
9459 	if (!btrfs_free_space_cachep)
9460 		goto fail;
9461 
9462 	return 0;
9463 fail:
9464 	btrfs_destroy_cachep();
9465 	return -ENOMEM;
9466 }
9467 
9468 static int btrfs_getattr(struct vfsmount *mnt,
9469 			 struct dentry *dentry, struct kstat *stat)
9470 {
9471 	u64 delalloc_bytes;
9472 	struct inode *inode = d_inode(dentry);
9473 	u32 blocksize = inode->i_sb->s_blocksize;
9474 
9475 	generic_fillattr(inode, stat);
9476 	stat->dev = BTRFS_I(inode)->root->anon_dev;
9477 
9478 	spin_lock(&BTRFS_I(inode)->lock);
9479 	delalloc_bytes = BTRFS_I(inode)->delalloc_bytes;
9480 	spin_unlock(&BTRFS_I(inode)->lock);
9481 	stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
9482 			ALIGN(delalloc_bytes, blocksize)) >> 9;
9483 	return 0;
9484 }
9485 
9486 static int btrfs_rename_exchange(struct inode *old_dir,
9487 			      struct dentry *old_dentry,
9488 			      struct inode *new_dir,
9489 			      struct dentry *new_dentry)
9490 {
9491 	struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9492 	struct btrfs_trans_handle *trans;
9493 	struct btrfs_root *root = BTRFS_I(old_dir)->root;
9494 	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9495 	struct inode *new_inode = new_dentry->d_inode;
9496 	struct inode *old_inode = old_dentry->d_inode;
9497 	struct timespec ctime = current_time(old_inode);
9498 	struct dentry *parent;
9499 	u64 old_ino = btrfs_ino(old_inode);
9500 	u64 new_ino = btrfs_ino(new_inode);
9501 	u64 old_idx = 0;
9502 	u64 new_idx = 0;
9503 	u64 root_objectid;
9504 	int ret;
9505 	bool root_log_pinned = false;
9506 	bool dest_log_pinned = false;
9507 
9508 	/* we only allow rename subvolume link between subvolumes */
9509 	if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9510 		return -EXDEV;
9511 
9512 	/* close the race window with snapshot create/destroy ioctl */
9513 	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9514 		down_read(&fs_info->subvol_sem);
9515 	if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
9516 		down_read(&fs_info->subvol_sem);
9517 
9518 	/*
9519 	 * We want to reserve the absolute worst case amount of items.  So if
9520 	 * both inodes are subvols and we need to unlink them then that would
9521 	 * require 4 item modifications, but if they are both normal inodes it
9522 	 * would require 5 item modifications, so we'll assume their normal
9523 	 * inodes.  So 5 * 2 is 10, plus 2 for the new links, so 12 total items
9524 	 * should cover the worst case number of items we'll modify.
9525 	 */
9526 	trans = btrfs_start_transaction(root, 12);
9527 	if (IS_ERR(trans)) {
9528 		ret = PTR_ERR(trans);
9529 		goto out_notrans;
9530 	}
9531 
9532 	/*
9533 	 * We need to find a free sequence number both in the source and
9534 	 * in the destination directory for the exchange.
9535 	 */
9536 	ret = btrfs_set_inode_index(new_dir, &old_idx);
9537 	if (ret)
9538 		goto out_fail;
9539 	ret = btrfs_set_inode_index(old_dir, &new_idx);
9540 	if (ret)
9541 		goto out_fail;
9542 
9543 	BTRFS_I(old_inode)->dir_index = 0ULL;
9544 	BTRFS_I(new_inode)->dir_index = 0ULL;
9545 
9546 	/* Reference for the source. */
9547 	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9548 		/* force full log commit if subvolume involved. */
9549 		btrfs_set_log_full_commit(fs_info, trans);
9550 	} else {
9551 		btrfs_pin_log_trans(root);
9552 		root_log_pinned = true;
9553 		ret = btrfs_insert_inode_ref(trans, dest,
9554 					     new_dentry->d_name.name,
9555 					     new_dentry->d_name.len,
9556 					     old_ino,
9557 					     btrfs_ino(new_dir), old_idx);
9558 		if (ret)
9559 			goto out_fail;
9560 	}
9561 
9562 	/* And now for the dest. */
9563 	if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9564 		/* force full log commit if subvolume involved. */
9565 		btrfs_set_log_full_commit(fs_info, trans);
9566 	} else {
9567 		btrfs_pin_log_trans(dest);
9568 		dest_log_pinned = true;
9569 		ret = btrfs_insert_inode_ref(trans, root,
9570 					     old_dentry->d_name.name,
9571 					     old_dentry->d_name.len,
9572 					     new_ino,
9573 					     btrfs_ino(old_dir), new_idx);
9574 		if (ret)
9575 			goto out_fail;
9576 	}
9577 
9578 	/* Update inode version and ctime/mtime. */
9579 	inode_inc_iversion(old_dir);
9580 	inode_inc_iversion(new_dir);
9581 	inode_inc_iversion(old_inode);
9582 	inode_inc_iversion(new_inode);
9583 	old_dir->i_ctime = old_dir->i_mtime = ctime;
9584 	new_dir->i_ctime = new_dir->i_mtime = ctime;
9585 	old_inode->i_ctime = ctime;
9586 	new_inode->i_ctime = ctime;
9587 
9588 	if (old_dentry->d_parent != new_dentry->d_parent) {
9589 		btrfs_record_unlink_dir(trans, old_dir, old_inode, 1);
9590 		btrfs_record_unlink_dir(trans, new_dir, new_inode, 1);
9591 	}
9592 
9593 	/* src is a subvolume */
9594 	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9595 		root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
9596 		ret = btrfs_unlink_subvol(trans, root, old_dir,
9597 					  root_objectid,
9598 					  old_dentry->d_name.name,
9599 					  old_dentry->d_name.len);
9600 	} else { /* src is an inode */
9601 		ret = __btrfs_unlink_inode(trans, root, old_dir,
9602 					   old_dentry->d_inode,
9603 					   old_dentry->d_name.name,
9604 					   old_dentry->d_name.len);
9605 		if (!ret)
9606 			ret = btrfs_update_inode(trans, root, old_inode);
9607 	}
9608 	if (ret) {
9609 		btrfs_abort_transaction(trans, ret);
9610 		goto out_fail;
9611 	}
9612 
9613 	/* dest is a subvolume */
9614 	if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9615 		root_objectid = BTRFS_I(new_inode)->root->root_key.objectid;
9616 		ret = btrfs_unlink_subvol(trans, dest, new_dir,
9617 					  root_objectid,
9618 					  new_dentry->d_name.name,
9619 					  new_dentry->d_name.len);
9620 	} else { /* dest is an inode */
9621 		ret = __btrfs_unlink_inode(trans, dest, new_dir,
9622 					   new_dentry->d_inode,
9623 					   new_dentry->d_name.name,
9624 					   new_dentry->d_name.len);
9625 		if (!ret)
9626 			ret = btrfs_update_inode(trans, dest, new_inode);
9627 	}
9628 	if (ret) {
9629 		btrfs_abort_transaction(trans, ret);
9630 		goto out_fail;
9631 	}
9632 
9633 	ret = btrfs_add_link(trans, new_dir, old_inode,
9634 			     new_dentry->d_name.name,
9635 			     new_dentry->d_name.len, 0, old_idx);
9636 	if (ret) {
9637 		btrfs_abort_transaction(trans, ret);
9638 		goto out_fail;
9639 	}
9640 
9641 	ret = btrfs_add_link(trans, old_dir, new_inode,
9642 			     old_dentry->d_name.name,
9643 			     old_dentry->d_name.len, 0, new_idx);
9644 	if (ret) {
9645 		btrfs_abort_transaction(trans, ret);
9646 		goto out_fail;
9647 	}
9648 
9649 	if (old_inode->i_nlink == 1)
9650 		BTRFS_I(old_inode)->dir_index = old_idx;
9651 	if (new_inode->i_nlink == 1)
9652 		BTRFS_I(new_inode)->dir_index = new_idx;
9653 
9654 	if (root_log_pinned) {
9655 		parent = new_dentry->d_parent;
9656 		btrfs_log_new_name(trans, old_inode, old_dir, parent);
9657 		btrfs_end_log_trans(root);
9658 		root_log_pinned = false;
9659 	}
9660 	if (dest_log_pinned) {
9661 		parent = old_dentry->d_parent;
9662 		btrfs_log_new_name(trans, new_inode, new_dir, parent);
9663 		btrfs_end_log_trans(dest);
9664 		dest_log_pinned = false;
9665 	}
9666 out_fail:
9667 	/*
9668 	 * If we have pinned a log and an error happened, we unpin tasks
9669 	 * trying to sync the log and force them to fallback to a transaction
9670 	 * commit if the log currently contains any of the inodes involved in
9671 	 * this rename operation (to ensure we do not persist a log with an
9672 	 * inconsistent state for any of these inodes or leading to any
9673 	 * inconsistencies when replayed). If the transaction was aborted, the
9674 	 * abortion reason is propagated to userspace when attempting to commit
9675 	 * the transaction. If the log does not contain any of these inodes, we
9676 	 * allow the tasks to sync it.
9677 	 */
9678 	if (ret && (root_log_pinned || dest_log_pinned)) {
9679 		if (btrfs_inode_in_log(old_dir, fs_info->generation) ||
9680 		    btrfs_inode_in_log(new_dir, fs_info->generation) ||
9681 		    btrfs_inode_in_log(old_inode, fs_info->generation) ||
9682 		    (new_inode &&
9683 		     btrfs_inode_in_log(new_inode, fs_info->generation)))
9684 			btrfs_set_log_full_commit(fs_info, trans);
9685 
9686 		if (root_log_pinned) {
9687 			btrfs_end_log_trans(root);
9688 			root_log_pinned = false;
9689 		}
9690 		if (dest_log_pinned) {
9691 			btrfs_end_log_trans(dest);
9692 			dest_log_pinned = false;
9693 		}
9694 	}
9695 	ret = btrfs_end_transaction(trans);
9696 out_notrans:
9697 	if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
9698 		up_read(&fs_info->subvol_sem);
9699 	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9700 		up_read(&fs_info->subvol_sem);
9701 
9702 	return ret;
9703 }
9704 
9705 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans,
9706 				     struct btrfs_root *root,
9707 				     struct inode *dir,
9708 				     struct dentry *dentry)
9709 {
9710 	int ret;
9711 	struct inode *inode;
9712 	u64 objectid;
9713 	u64 index;
9714 
9715 	ret = btrfs_find_free_ino(root, &objectid);
9716 	if (ret)
9717 		return ret;
9718 
9719 	inode = btrfs_new_inode(trans, root, dir,
9720 				dentry->d_name.name,
9721 				dentry->d_name.len,
9722 				btrfs_ino(dir),
9723 				objectid,
9724 				S_IFCHR | WHITEOUT_MODE,
9725 				&index);
9726 
9727 	if (IS_ERR(inode)) {
9728 		ret = PTR_ERR(inode);
9729 		return ret;
9730 	}
9731 
9732 	inode->i_op = &btrfs_special_inode_operations;
9733 	init_special_inode(inode, inode->i_mode,
9734 		WHITEOUT_DEV);
9735 
9736 	ret = btrfs_init_inode_security(trans, inode, dir,
9737 				&dentry->d_name);
9738 	if (ret)
9739 		goto out;
9740 
9741 	ret = btrfs_add_nondir(trans, dir, dentry,
9742 				inode, 0, index);
9743 	if (ret)
9744 		goto out;
9745 
9746 	ret = btrfs_update_inode(trans, root, inode);
9747 out:
9748 	unlock_new_inode(inode);
9749 	if (ret)
9750 		inode_dec_link_count(inode);
9751 	iput(inode);
9752 
9753 	return ret;
9754 }
9755 
9756 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
9757 			   struct inode *new_dir, struct dentry *new_dentry,
9758 			   unsigned int flags)
9759 {
9760 	struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9761 	struct btrfs_trans_handle *trans;
9762 	unsigned int trans_num_items;
9763 	struct btrfs_root *root = BTRFS_I(old_dir)->root;
9764 	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9765 	struct inode *new_inode = d_inode(new_dentry);
9766 	struct inode *old_inode = d_inode(old_dentry);
9767 	u64 index = 0;
9768 	u64 root_objectid;
9769 	int ret;
9770 	u64 old_ino = btrfs_ino(old_inode);
9771 	bool log_pinned = false;
9772 
9773 	if (btrfs_ino(new_dir) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
9774 		return -EPERM;
9775 
9776 	/* we only allow rename subvolume link between subvolumes */
9777 	if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9778 		return -EXDEV;
9779 
9780 	if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
9781 	    (new_inode && btrfs_ino(new_inode) == BTRFS_FIRST_FREE_OBJECTID))
9782 		return -ENOTEMPTY;
9783 
9784 	if (S_ISDIR(old_inode->i_mode) && new_inode &&
9785 	    new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
9786 		return -ENOTEMPTY;
9787 
9788 
9789 	/* check for collisions, even if the  name isn't there */
9790 	ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
9791 			     new_dentry->d_name.name,
9792 			     new_dentry->d_name.len);
9793 
9794 	if (ret) {
9795 		if (ret == -EEXIST) {
9796 			/* we shouldn't get
9797 			 * eexist without a new_inode */
9798 			if (WARN_ON(!new_inode)) {
9799 				return ret;
9800 			}
9801 		} else {
9802 			/* maybe -EOVERFLOW */
9803 			return ret;
9804 		}
9805 	}
9806 	ret = 0;
9807 
9808 	/*
9809 	 * we're using rename to replace one file with another.  Start IO on it
9810 	 * now so  we don't add too much work to the end of the transaction
9811 	 */
9812 	if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
9813 		filemap_flush(old_inode->i_mapping);
9814 
9815 	/* close the racy window with snapshot create/destroy ioctl */
9816 	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9817 		down_read(&fs_info->subvol_sem);
9818 	/*
9819 	 * We want to reserve the absolute worst case amount of items.  So if
9820 	 * both inodes are subvols and we need to unlink them then that would
9821 	 * require 4 item modifications, but if they are both normal inodes it
9822 	 * would require 5 item modifications, so we'll assume they are normal
9823 	 * inodes.  So 5 * 2 is 10, plus 1 for the new link, so 11 total items
9824 	 * should cover the worst case number of items we'll modify.
9825 	 * If our rename has the whiteout flag, we need more 5 units for the
9826 	 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
9827 	 * when selinux is enabled).
9828 	 */
9829 	trans_num_items = 11;
9830 	if (flags & RENAME_WHITEOUT)
9831 		trans_num_items += 5;
9832 	trans = btrfs_start_transaction(root, trans_num_items);
9833 	if (IS_ERR(trans)) {
9834 		ret = PTR_ERR(trans);
9835 		goto out_notrans;
9836 	}
9837 
9838 	if (dest != root)
9839 		btrfs_record_root_in_trans(trans, dest);
9840 
9841 	ret = btrfs_set_inode_index(new_dir, &index);
9842 	if (ret)
9843 		goto out_fail;
9844 
9845 	BTRFS_I(old_inode)->dir_index = 0ULL;
9846 	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9847 		/* force full log commit if subvolume involved. */
9848 		btrfs_set_log_full_commit(fs_info, trans);
9849 	} else {
9850 		btrfs_pin_log_trans(root);
9851 		log_pinned = true;
9852 		ret = btrfs_insert_inode_ref(trans, dest,
9853 					     new_dentry->d_name.name,
9854 					     new_dentry->d_name.len,
9855 					     old_ino,
9856 					     btrfs_ino(new_dir), index);
9857 		if (ret)
9858 			goto out_fail;
9859 	}
9860 
9861 	inode_inc_iversion(old_dir);
9862 	inode_inc_iversion(new_dir);
9863 	inode_inc_iversion(old_inode);
9864 	old_dir->i_ctime = old_dir->i_mtime =
9865 	new_dir->i_ctime = new_dir->i_mtime =
9866 	old_inode->i_ctime = current_time(old_dir);
9867 
9868 	if (old_dentry->d_parent != new_dentry->d_parent)
9869 		btrfs_record_unlink_dir(trans, old_dir, old_inode, 1);
9870 
9871 	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9872 		root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
9873 		ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid,
9874 					old_dentry->d_name.name,
9875 					old_dentry->d_name.len);
9876 	} else {
9877 		ret = __btrfs_unlink_inode(trans, root, old_dir,
9878 					d_inode(old_dentry),
9879 					old_dentry->d_name.name,
9880 					old_dentry->d_name.len);
9881 		if (!ret)
9882 			ret = btrfs_update_inode(trans, root, old_inode);
9883 	}
9884 	if (ret) {
9885 		btrfs_abort_transaction(trans, ret);
9886 		goto out_fail;
9887 	}
9888 
9889 	if (new_inode) {
9890 		inode_inc_iversion(new_inode);
9891 		new_inode->i_ctime = current_time(new_inode);
9892 		if (unlikely(btrfs_ino(new_inode) ==
9893 			     BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
9894 			root_objectid = BTRFS_I(new_inode)->location.objectid;
9895 			ret = btrfs_unlink_subvol(trans, dest, new_dir,
9896 						root_objectid,
9897 						new_dentry->d_name.name,
9898 						new_dentry->d_name.len);
9899 			BUG_ON(new_inode->i_nlink == 0);
9900 		} else {
9901 			ret = btrfs_unlink_inode(trans, dest, new_dir,
9902 						 d_inode(new_dentry),
9903 						 new_dentry->d_name.name,
9904 						 new_dentry->d_name.len);
9905 		}
9906 		if (!ret && new_inode->i_nlink == 0)
9907 			ret = btrfs_orphan_add(trans, d_inode(new_dentry));
9908 		if (ret) {
9909 			btrfs_abort_transaction(trans, ret);
9910 			goto out_fail;
9911 		}
9912 	}
9913 
9914 	ret = btrfs_add_link(trans, new_dir, old_inode,
9915 			     new_dentry->d_name.name,
9916 			     new_dentry->d_name.len, 0, index);
9917 	if (ret) {
9918 		btrfs_abort_transaction(trans, ret);
9919 		goto out_fail;
9920 	}
9921 
9922 	if (old_inode->i_nlink == 1)
9923 		BTRFS_I(old_inode)->dir_index = index;
9924 
9925 	if (log_pinned) {
9926 		struct dentry *parent = new_dentry->d_parent;
9927 
9928 		btrfs_log_new_name(trans, old_inode, old_dir, parent);
9929 		btrfs_end_log_trans(root);
9930 		log_pinned = false;
9931 	}
9932 
9933 	if (flags & RENAME_WHITEOUT) {
9934 		ret = btrfs_whiteout_for_rename(trans, root, old_dir,
9935 						old_dentry);
9936 
9937 		if (ret) {
9938 			btrfs_abort_transaction(trans, ret);
9939 			goto out_fail;
9940 		}
9941 	}
9942 out_fail:
9943 	/*
9944 	 * If we have pinned the log and an error happened, we unpin tasks
9945 	 * trying to sync the log and force them to fallback to a transaction
9946 	 * commit if the log currently contains any of the inodes involved in
9947 	 * this rename operation (to ensure we do not persist a log with an
9948 	 * inconsistent state for any of these inodes or leading to any
9949 	 * inconsistencies when replayed). If the transaction was aborted, the
9950 	 * abortion reason is propagated to userspace when attempting to commit
9951 	 * the transaction. If the log does not contain any of these inodes, we
9952 	 * allow the tasks to sync it.
9953 	 */
9954 	if (ret && log_pinned) {
9955 		if (btrfs_inode_in_log(old_dir, fs_info->generation) ||
9956 		    btrfs_inode_in_log(new_dir, fs_info->generation) ||
9957 		    btrfs_inode_in_log(old_inode, fs_info->generation) ||
9958 		    (new_inode &&
9959 		     btrfs_inode_in_log(new_inode, fs_info->generation)))
9960 			btrfs_set_log_full_commit(fs_info, trans);
9961 
9962 		btrfs_end_log_trans(root);
9963 		log_pinned = false;
9964 	}
9965 	btrfs_end_transaction(trans);
9966 out_notrans:
9967 	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9968 		up_read(&fs_info->subvol_sem);
9969 
9970 	return ret;
9971 }
9972 
9973 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
9974 			 struct inode *new_dir, struct dentry *new_dentry,
9975 			 unsigned int flags)
9976 {
9977 	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
9978 		return -EINVAL;
9979 
9980 	if (flags & RENAME_EXCHANGE)
9981 		return btrfs_rename_exchange(old_dir, old_dentry, new_dir,
9982 					  new_dentry);
9983 
9984 	return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
9985 }
9986 
9987 static void btrfs_run_delalloc_work(struct btrfs_work *work)
9988 {
9989 	struct btrfs_delalloc_work *delalloc_work;
9990 	struct inode *inode;
9991 
9992 	delalloc_work = container_of(work, struct btrfs_delalloc_work,
9993 				     work);
9994 	inode = delalloc_work->inode;
9995 	filemap_flush(inode->i_mapping);
9996 	if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
9997 				&BTRFS_I(inode)->runtime_flags))
9998 		filemap_flush(inode->i_mapping);
9999 
10000 	if (delalloc_work->delay_iput)
10001 		btrfs_add_delayed_iput(inode);
10002 	else
10003 		iput(inode);
10004 	complete(&delalloc_work->completion);
10005 }
10006 
10007 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode,
10008 						    int delay_iput)
10009 {
10010 	struct btrfs_delalloc_work *work;
10011 
10012 	work = kmalloc(sizeof(*work), GFP_NOFS);
10013 	if (!work)
10014 		return NULL;
10015 
10016 	init_completion(&work->completion);
10017 	INIT_LIST_HEAD(&work->list);
10018 	work->inode = inode;
10019 	work->delay_iput = delay_iput;
10020 	WARN_ON_ONCE(!inode);
10021 	btrfs_init_work(&work->work, btrfs_flush_delalloc_helper,
10022 			btrfs_run_delalloc_work, NULL, NULL);
10023 
10024 	return work;
10025 }
10026 
10027 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work)
10028 {
10029 	wait_for_completion(&work->completion);
10030 	kfree(work);
10031 }
10032 
10033 /*
10034  * some fairly slow code that needs optimization. This walks the list
10035  * of all the inodes with pending delalloc and forces them to disk.
10036  */
10037 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput,
10038 				   int nr)
10039 {
10040 	struct btrfs_inode *binode;
10041 	struct inode *inode;
10042 	struct btrfs_delalloc_work *work, *next;
10043 	struct list_head works;
10044 	struct list_head splice;
10045 	int ret = 0;
10046 
10047 	INIT_LIST_HEAD(&works);
10048 	INIT_LIST_HEAD(&splice);
10049 
10050 	mutex_lock(&root->delalloc_mutex);
10051 	spin_lock(&root->delalloc_lock);
10052 	list_splice_init(&root->delalloc_inodes, &splice);
10053 	while (!list_empty(&splice)) {
10054 		binode = list_entry(splice.next, struct btrfs_inode,
10055 				    delalloc_inodes);
10056 
10057 		list_move_tail(&binode->delalloc_inodes,
10058 			       &root->delalloc_inodes);
10059 		inode = igrab(&binode->vfs_inode);
10060 		if (!inode) {
10061 			cond_resched_lock(&root->delalloc_lock);
10062 			continue;
10063 		}
10064 		spin_unlock(&root->delalloc_lock);
10065 
10066 		work = btrfs_alloc_delalloc_work(inode, delay_iput);
10067 		if (!work) {
10068 			if (delay_iput)
10069 				btrfs_add_delayed_iput(inode);
10070 			else
10071 				iput(inode);
10072 			ret = -ENOMEM;
10073 			goto out;
10074 		}
10075 		list_add_tail(&work->list, &works);
10076 		btrfs_queue_work(root->fs_info->flush_workers,
10077 				 &work->work);
10078 		ret++;
10079 		if (nr != -1 && ret >= nr)
10080 			goto out;
10081 		cond_resched();
10082 		spin_lock(&root->delalloc_lock);
10083 	}
10084 	spin_unlock(&root->delalloc_lock);
10085 
10086 out:
10087 	list_for_each_entry_safe(work, next, &works, list) {
10088 		list_del_init(&work->list);
10089 		btrfs_wait_and_free_delalloc_work(work);
10090 	}
10091 
10092 	if (!list_empty_careful(&splice)) {
10093 		spin_lock(&root->delalloc_lock);
10094 		list_splice_tail(&splice, &root->delalloc_inodes);
10095 		spin_unlock(&root->delalloc_lock);
10096 	}
10097 	mutex_unlock(&root->delalloc_mutex);
10098 	return ret;
10099 }
10100 
10101 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput)
10102 {
10103 	struct btrfs_fs_info *fs_info = root->fs_info;
10104 	int ret;
10105 
10106 	if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10107 		return -EROFS;
10108 
10109 	ret = __start_delalloc_inodes(root, delay_iput, -1);
10110 	if (ret > 0)
10111 		ret = 0;
10112 	/*
10113 	 * the filemap_flush will queue IO into the worker threads, but
10114 	 * we have to make sure the IO is actually started and that
10115 	 * ordered extents get created before we return
10116 	 */
10117 	atomic_inc(&fs_info->async_submit_draining);
10118 	while (atomic_read(&fs_info->nr_async_submits) ||
10119 	       atomic_read(&fs_info->async_delalloc_pages)) {
10120 		wait_event(fs_info->async_submit_wait,
10121 			   (atomic_read(&fs_info->nr_async_submits) == 0 &&
10122 			    atomic_read(&fs_info->async_delalloc_pages) == 0));
10123 	}
10124 	atomic_dec(&fs_info->async_submit_draining);
10125 	return ret;
10126 }
10127 
10128 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput,
10129 			       int nr)
10130 {
10131 	struct btrfs_root *root;
10132 	struct list_head splice;
10133 	int ret;
10134 
10135 	if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10136 		return -EROFS;
10137 
10138 	INIT_LIST_HEAD(&splice);
10139 
10140 	mutex_lock(&fs_info->delalloc_root_mutex);
10141 	spin_lock(&fs_info->delalloc_root_lock);
10142 	list_splice_init(&fs_info->delalloc_roots, &splice);
10143 	while (!list_empty(&splice) && nr) {
10144 		root = list_first_entry(&splice, struct btrfs_root,
10145 					delalloc_root);
10146 		root = btrfs_grab_fs_root(root);
10147 		BUG_ON(!root);
10148 		list_move_tail(&root->delalloc_root,
10149 			       &fs_info->delalloc_roots);
10150 		spin_unlock(&fs_info->delalloc_root_lock);
10151 
10152 		ret = __start_delalloc_inodes(root, delay_iput, nr);
10153 		btrfs_put_fs_root(root);
10154 		if (ret < 0)
10155 			goto out;
10156 
10157 		if (nr != -1) {
10158 			nr -= ret;
10159 			WARN_ON(nr < 0);
10160 		}
10161 		spin_lock(&fs_info->delalloc_root_lock);
10162 	}
10163 	spin_unlock(&fs_info->delalloc_root_lock);
10164 
10165 	ret = 0;
10166 	atomic_inc(&fs_info->async_submit_draining);
10167 	while (atomic_read(&fs_info->nr_async_submits) ||
10168 	      atomic_read(&fs_info->async_delalloc_pages)) {
10169 		wait_event(fs_info->async_submit_wait,
10170 		   (atomic_read(&fs_info->nr_async_submits) == 0 &&
10171 		    atomic_read(&fs_info->async_delalloc_pages) == 0));
10172 	}
10173 	atomic_dec(&fs_info->async_submit_draining);
10174 out:
10175 	if (!list_empty_careful(&splice)) {
10176 		spin_lock(&fs_info->delalloc_root_lock);
10177 		list_splice_tail(&splice, &fs_info->delalloc_roots);
10178 		spin_unlock(&fs_info->delalloc_root_lock);
10179 	}
10180 	mutex_unlock(&fs_info->delalloc_root_mutex);
10181 	return ret;
10182 }
10183 
10184 static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
10185 			 const char *symname)
10186 {
10187 	struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10188 	struct btrfs_trans_handle *trans;
10189 	struct btrfs_root *root = BTRFS_I(dir)->root;
10190 	struct btrfs_path *path;
10191 	struct btrfs_key key;
10192 	struct inode *inode = NULL;
10193 	int err;
10194 	int drop_inode = 0;
10195 	u64 objectid;
10196 	u64 index = 0;
10197 	int name_len;
10198 	int datasize;
10199 	unsigned long ptr;
10200 	struct btrfs_file_extent_item *ei;
10201 	struct extent_buffer *leaf;
10202 
10203 	name_len = strlen(symname);
10204 	if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
10205 		return -ENAMETOOLONG;
10206 
10207 	/*
10208 	 * 2 items for inode item and ref
10209 	 * 2 items for dir items
10210 	 * 1 item for updating parent inode item
10211 	 * 1 item for the inline extent item
10212 	 * 1 item for xattr if selinux is on
10213 	 */
10214 	trans = btrfs_start_transaction(root, 7);
10215 	if (IS_ERR(trans))
10216 		return PTR_ERR(trans);
10217 
10218 	err = btrfs_find_free_ino(root, &objectid);
10219 	if (err)
10220 		goto out_unlock;
10221 
10222 	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
10223 				dentry->d_name.len, btrfs_ino(dir), objectid,
10224 				S_IFLNK|S_IRWXUGO, &index);
10225 	if (IS_ERR(inode)) {
10226 		err = PTR_ERR(inode);
10227 		goto out_unlock;
10228 	}
10229 
10230 	/*
10231 	* If the active LSM wants to access the inode during
10232 	* d_instantiate it needs these. Smack checks to see
10233 	* if the filesystem supports xattrs by looking at the
10234 	* ops vector.
10235 	*/
10236 	inode->i_fop = &btrfs_file_operations;
10237 	inode->i_op = &btrfs_file_inode_operations;
10238 	inode->i_mapping->a_ops = &btrfs_aops;
10239 	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10240 
10241 	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
10242 	if (err)
10243 		goto out_unlock_inode;
10244 
10245 	path = btrfs_alloc_path();
10246 	if (!path) {
10247 		err = -ENOMEM;
10248 		goto out_unlock_inode;
10249 	}
10250 	key.objectid = btrfs_ino(inode);
10251 	key.offset = 0;
10252 	key.type = BTRFS_EXTENT_DATA_KEY;
10253 	datasize = btrfs_file_extent_calc_inline_size(name_len);
10254 	err = btrfs_insert_empty_item(trans, root, path, &key,
10255 				      datasize);
10256 	if (err) {
10257 		btrfs_free_path(path);
10258 		goto out_unlock_inode;
10259 	}
10260 	leaf = path->nodes[0];
10261 	ei = btrfs_item_ptr(leaf, path->slots[0],
10262 			    struct btrfs_file_extent_item);
10263 	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
10264 	btrfs_set_file_extent_type(leaf, ei,
10265 				   BTRFS_FILE_EXTENT_INLINE);
10266 	btrfs_set_file_extent_encryption(leaf, ei, 0);
10267 	btrfs_set_file_extent_compression(leaf, ei, 0);
10268 	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
10269 	btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
10270 
10271 	ptr = btrfs_file_extent_inline_start(ei);
10272 	write_extent_buffer(leaf, symname, ptr, name_len);
10273 	btrfs_mark_buffer_dirty(leaf);
10274 	btrfs_free_path(path);
10275 
10276 	inode->i_op = &btrfs_symlink_inode_operations;
10277 	inode_nohighmem(inode);
10278 	inode->i_mapping->a_ops = &btrfs_symlink_aops;
10279 	inode_set_bytes(inode, name_len);
10280 	btrfs_i_size_write(inode, name_len);
10281 	err = btrfs_update_inode(trans, root, inode);
10282 	/*
10283 	 * Last step, add directory indexes for our symlink inode. This is the
10284 	 * last step to avoid extra cleanup of these indexes if an error happens
10285 	 * elsewhere above.
10286 	 */
10287 	if (!err)
10288 		err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
10289 	if (err) {
10290 		drop_inode = 1;
10291 		goto out_unlock_inode;
10292 	}
10293 
10294 	unlock_new_inode(inode);
10295 	d_instantiate(dentry, inode);
10296 
10297 out_unlock:
10298 	btrfs_end_transaction(trans);
10299 	if (drop_inode) {
10300 		inode_dec_link_count(inode);
10301 		iput(inode);
10302 	}
10303 	btrfs_btree_balance_dirty(fs_info);
10304 	return err;
10305 
10306 out_unlock_inode:
10307 	drop_inode = 1;
10308 	unlock_new_inode(inode);
10309 	goto out_unlock;
10310 }
10311 
10312 static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
10313 				       u64 start, u64 num_bytes, u64 min_size,
10314 				       loff_t actual_len, u64 *alloc_hint,
10315 				       struct btrfs_trans_handle *trans)
10316 {
10317 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
10318 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
10319 	struct extent_map *em;
10320 	struct btrfs_root *root = BTRFS_I(inode)->root;
10321 	struct btrfs_key ins;
10322 	u64 cur_offset = start;
10323 	u64 i_size;
10324 	u64 cur_bytes;
10325 	u64 last_alloc = (u64)-1;
10326 	int ret = 0;
10327 	bool own_trans = true;
10328 	u64 end = start + num_bytes - 1;
10329 
10330 	if (trans)
10331 		own_trans = false;
10332 	while (num_bytes > 0) {
10333 		if (own_trans) {
10334 			trans = btrfs_start_transaction(root, 3);
10335 			if (IS_ERR(trans)) {
10336 				ret = PTR_ERR(trans);
10337 				break;
10338 			}
10339 		}
10340 
10341 		cur_bytes = min_t(u64, num_bytes, SZ_256M);
10342 		cur_bytes = max(cur_bytes, min_size);
10343 		/*
10344 		 * If we are severely fragmented we could end up with really
10345 		 * small allocations, so if the allocator is returning small
10346 		 * chunks lets make its job easier by only searching for those
10347 		 * sized chunks.
10348 		 */
10349 		cur_bytes = min(cur_bytes, last_alloc);
10350 		ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
10351 				min_size, 0, *alloc_hint, &ins, 1, 0);
10352 		if (ret) {
10353 			if (own_trans)
10354 				btrfs_end_transaction(trans);
10355 			break;
10356 		}
10357 		btrfs_dec_block_group_reservations(fs_info, ins.objectid);
10358 
10359 		last_alloc = ins.offset;
10360 		ret = insert_reserved_file_extent(trans, inode,
10361 						  cur_offset, ins.objectid,
10362 						  ins.offset, ins.offset,
10363 						  ins.offset, 0, 0, 0,
10364 						  BTRFS_FILE_EXTENT_PREALLOC);
10365 		if (ret) {
10366 			btrfs_free_reserved_extent(fs_info, ins.objectid,
10367 						   ins.offset, 0);
10368 			btrfs_abort_transaction(trans, ret);
10369 			if (own_trans)
10370 				btrfs_end_transaction(trans);
10371 			break;
10372 		}
10373 
10374 		btrfs_drop_extent_cache(inode, cur_offset,
10375 					cur_offset + ins.offset -1, 0);
10376 
10377 		em = alloc_extent_map();
10378 		if (!em) {
10379 			set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
10380 				&BTRFS_I(inode)->runtime_flags);
10381 			goto next;
10382 		}
10383 
10384 		em->start = cur_offset;
10385 		em->orig_start = cur_offset;
10386 		em->len = ins.offset;
10387 		em->block_start = ins.objectid;
10388 		em->block_len = ins.offset;
10389 		em->orig_block_len = ins.offset;
10390 		em->ram_bytes = ins.offset;
10391 		em->bdev = fs_info->fs_devices->latest_bdev;
10392 		set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
10393 		em->generation = trans->transid;
10394 
10395 		while (1) {
10396 			write_lock(&em_tree->lock);
10397 			ret = add_extent_mapping(em_tree, em, 1);
10398 			write_unlock(&em_tree->lock);
10399 			if (ret != -EEXIST)
10400 				break;
10401 			btrfs_drop_extent_cache(inode, cur_offset,
10402 						cur_offset + ins.offset - 1,
10403 						0);
10404 		}
10405 		free_extent_map(em);
10406 next:
10407 		num_bytes -= ins.offset;
10408 		cur_offset += ins.offset;
10409 		*alloc_hint = ins.objectid + ins.offset;
10410 
10411 		inode_inc_iversion(inode);
10412 		inode->i_ctime = current_time(inode);
10413 		BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
10414 		if (!(mode & FALLOC_FL_KEEP_SIZE) &&
10415 		    (actual_len > inode->i_size) &&
10416 		    (cur_offset > inode->i_size)) {
10417 			if (cur_offset > actual_len)
10418 				i_size = actual_len;
10419 			else
10420 				i_size = cur_offset;
10421 			i_size_write(inode, i_size);
10422 			btrfs_ordered_update_i_size(inode, i_size, NULL);
10423 		}
10424 
10425 		ret = btrfs_update_inode(trans, root, inode);
10426 
10427 		if (ret) {
10428 			btrfs_abort_transaction(trans, ret);
10429 			if (own_trans)
10430 				btrfs_end_transaction(trans);
10431 			break;
10432 		}
10433 
10434 		if (own_trans)
10435 			btrfs_end_transaction(trans);
10436 	}
10437 	if (cur_offset < end)
10438 		btrfs_free_reserved_data_space(inode, cur_offset,
10439 			end - cur_offset + 1);
10440 	return ret;
10441 }
10442 
10443 int btrfs_prealloc_file_range(struct inode *inode, int mode,
10444 			      u64 start, u64 num_bytes, u64 min_size,
10445 			      loff_t actual_len, u64 *alloc_hint)
10446 {
10447 	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10448 					   min_size, actual_len, alloc_hint,
10449 					   NULL);
10450 }
10451 
10452 int btrfs_prealloc_file_range_trans(struct inode *inode,
10453 				    struct btrfs_trans_handle *trans, int mode,
10454 				    u64 start, u64 num_bytes, u64 min_size,
10455 				    loff_t actual_len, u64 *alloc_hint)
10456 {
10457 	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10458 					   min_size, actual_len, alloc_hint, trans);
10459 }
10460 
10461 static int btrfs_set_page_dirty(struct page *page)
10462 {
10463 	return __set_page_dirty_nobuffers(page);
10464 }
10465 
10466 static int btrfs_permission(struct inode *inode, int mask)
10467 {
10468 	struct btrfs_root *root = BTRFS_I(inode)->root;
10469 	umode_t mode = inode->i_mode;
10470 
10471 	if (mask & MAY_WRITE &&
10472 	    (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
10473 		if (btrfs_root_readonly(root))
10474 			return -EROFS;
10475 		if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
10476 			return -EACCES;
10477 	}
10478 	return generic_permission(inode, mask);
10479 }
10480 
10481 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
10482 {
10483 	struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10484 	struct btrfs_trans_handle *trans;
10485 	struct btrfs_root *root = BTRFS_I(dir)->root;
10486 	struct inode *inode = NULL;
10487 	u64 objectid;
10488 	u64 index;
10489 	int ret = 0;
10490 
10491 	/*
10492 	 * 5 units required for adding orphan entry
10493 	 */
10494 	trans = btrfs_start_transaction(root, 5);
10495 	if (IS_ERR(trans))
10496 		return PTR_ERR(trans);
10497 
10498 	ret = btrfs_find_free_ino(root, &objectid);
10499 	if (ret)
10500 		goto out;
10501 
10502 	inode = btrfs_new_inode(trans, root, dir, NULL, 0,
10503 				btrfs_ino(dir), objectid, mode, &index);
10504 	if (IS_ERR(inode)) {
10505 		ret = PTR_ERR(inode);
10506 		inode = NULL;
10507 		goto out;
10508 	}
10509 
10510 	inode->i_fop = &btrfs_file_operations;
10511 	inode->i_op = &btrfs_file_inode_operations;
10512 
10513 	inode->i_mapping->a_ops = &btrfs_aops;
10514 	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10515 
10516 	ret = btrfs_init_inode_security(trans, inode, dir, NULL);
10517 	if (ret)
10518 		goto out_inode;
10519 
10520 	ret = btrfs_update_inode(trans, root, inode);
10521 	if (ret)
10522 		goto out_inode;
10523 	ret = btrfs_orphan_add(trans, inode);
10524 	if (ret)
10525 		goto out_inode;
10526 
10527 	/*
10528 	 * We set number of links to 0 in btrfs_new_inode(), and here we set
10529 	 * it to 1 because d_tmpfile() will issue a warning if the count is 0,
10530 	 * through:
10531 	 *
10532 	 *    d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
10533 	 */
10534 	set_nlink(inode, 1);
10535 	unlock_new_inode(inode);
10536 	d_tmpfile(dentry, inode);
10537 	mark_inode_dirty(inode);
10538 
10539 out:
10540 	btrfs_end_transaction(trans);
10541 	if (ret)
10542 		iput(inode);
10543 	btrfs_balance_delayed_items(fs_info);
10544 	btrfs_btree_balance_dirty(fs_info);
10545 	return ret;
10546 
10547 out_inode:
10548 	unlock_new_inode(inode);
10549 	goto out;
10550 
10551 }
10552 
10553 static const struct inode_operations btrfs_dir_inode_operations = {
10554 	.getattr	= btrfs_getattr,
10555 	.lookup		= btrfs_lookup,
10556 	.create		= btrfs_create,
10557 	.unlink		= btrfs_unlink,
10558 	.link		= btrfs_link,
10559 	.mkdir		= btrfs_mkdir,
10560 	.rmdir		= btrfs_rmdir,
10561 	.rename		= btrfs_rename2,
10562 	.symlink	= btrfs_symlink,
10563 	.setattr	= btrfs_setattr,
10564 	.mknod		= btrfs_mknod,
10565 	.listxattr	= btrfs_listxattr,
10566 	.permission	= btrfs_permission,
10567 	.get_acl	= btrfs_get_acl,
10568 	.set_acl	= btrfs_set_acl,
10569 	.update_time	= btrfs_update_time,
10570 	.tmpfile        = btrfs_tmpfile,
10571 };
10572 static const struct inode_operations btrfs_dir_ro_inode_operations = {
10573 	.lookup		= btrfs_lookup,
10574 	.permission	= btrfs_permission,
10575 	.get_acl	= btrfs_get_acl,
10576 	.set_acl	= btrfs_set_acl,
10577 	.update_time	= btrfs_update_time,
10578 };
10579 
10580 static const struct file_operations btrfs_dir_file_operations = {
10581 	.llseek		= generic_file_llseek,
10582 	.read		= generic_read_dir,
10583 	.iterate_shared	= btrfs_real_readdir,
10584 	.unlocked_ioctl	= btrfs_ioctl,
10585 #ifdef CONFIG_COMPAT
10586 	.compat_ioctl	= btrfs_compat_ioctl,
10587 #endif
10588 	.release        = btrfs_release_file,
10589 	.fsync		= btrfs_sync_file,
10590 };
10591 
10592 static const struct extent_io_ops btrfs_extent_io_ops = {
10593 	.fill_delalloc = run_delalloc_range,
10594 	.submit_bio_hook = btrfs_submit_bio_hook,
10595 	.merge_bio_hook = btrfs_merge_bio_hook,
10596 	.readpage_end_io_hook = btrfs_readpage_end_io_hook,
10597 	.writepage_end_io_hook = btrfs_writepage_end_io_hook,
10598 	.writepage_start_hook = btrfs_writepage_start_hook,
10599 	.set_bit_hook = btrfs_set_bit_hook,
10600 	.clear_bit_hook = btrfs_clear_bit_hook,
10601 	.merge_extent_hook = btrfs_merge_extent_hook,
10602 	.split_extent_hook = btrfs_split_extent_hook,
10603 };
10604 
10605 /*
10606  * btrfs doesn't support the bmap operation because swapfiles
10607  * use bmap to make a mapping of extents in the file.  They assume
10608  * these extents won't change over the life of the file and they
10609  * use the bmap result to do IO directly to the drive.
10610  *
10611  * the btrfs bmap call would return logical addresses that aren't
10612  * suitable for IO and they also will change frequently as COW
10613  * operations happen.  So, swapfile + btrfs == corruption.
10614  *
10615  * For now we're avoiding this by dropping bmap.
10616  */
10617 static const struct address_space_operations btrfs_aops = {
10618 	.readpage	= btrfs_readpage,
10619 	.writepage	= btrfs_writepage,
10620 	.writepages	= btrfs_writepages,
10621 	.readpages	= btrfs_readpages,
10622 	.direct_IO	= btrfs_direct_IO,
10623 	.invalidatepage = btrfs_invalidatepage,
10624 	.releasepage	= btrfs_releasepage,
10625 	.set_page_dirty	= btrfs_set_page_dirty,
10626 	.error_remove_page = generic_error_remove_page,
10627 };
10628 
10629 static const struct address_space_operations btrfs_symlink_aops = {
10630 	.readpage	= btrfs_readpage,
10631 	.writepage	= btrfs_writepage,
10632 	.invalidatepage = btrfs_invalidatepage,
10633 	.releasepage	= btrfs_releasepage,
10634 };
10635 
10636 static const struct inode_operations btrfs_file_inode_operations = {
10637 	.getattr	= btrfs_getattr,
10638 	.setattr	= btrfs_setattr,
10639 	.listxattr      = btrfs_listxattr,
10640 	.permission	= btrfs_permission,
10641 	.fiemap		= btrfs_fiemap,
10642 	.get_acl	= btrfs_get_acl,
10643 	.set_acl	= btrfs_set_acl,
10644 	.update_time	= btrfs_update_time,
10645 };
10646 static const struct inode_operations btrfs_special_inode_operations = {
10647 	.getattr	= btrfs_getattr,
10648 	.setattr	= btrfs_setattr,
10649 	.permission	= btrfs_permission,
10650 	.listxattr	= btrfs_listxattr,
10651 	.get_acl	= btrfs_get_acl,
10652 	.set_acl	= btrfs_set_acl,
10653 	.update_time	= btrfs_update_time,
10654 };
10655 static const struct inode_operations btrfs_symlink_inode_operations = {
10656 	.get_link	= page_get_link,
10657 	.getattr	= btrfs_getattr,
10658 	.setattr	= btrfs_setattr,
10659 	.permission	= btrfs_permission,
10660 	.listxattr	= btrfs_listxattr,
10661 	.update_time	= btrfs_update_time,
10662 };
10663 
10664 const struct dentry_operations btrfs_dentry_operations = {
10665 	.d_delete	= btrfs_dentry_delete,
10666 	.d_release	= btrfs_dentry_release,
10667 };
10668