xref: /openbmc/linux/fs/f2fs/data.c (revision ca79522c)
1 /*
2  * fs/f2fs/data.c
3  *
4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5  *             http://www.samsung.com/
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11 #include <linux/fs.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/buffer_head.h>
14 #include <linux/mpage.h>
15 #include <linux/aio.h>
16 #include <linux/writeback.h>
17 #include <linux/backing-dev.h>
18 #include <linux/blkdev.h>
19 #include <linux/bio.h>
20 #include <linux/prefetch.h>
21 
22 #include "f2fs.h"
23 #include "node.h"
24 #include "segment.h"
25 #include <trace/events/f2fs.h>
26 
27 /*
28  * Lock ordering for the change of data block address:
29  * ->data_page
30  *  ->node_page
31  *    update block addresses in the node page
32  */
33 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr)
34 {
35 	struct f2fs_node *rn;
36 	__le32 *addr_array;
37 	struct page *node_page = dn->node_page;
38 	unsigned int ofs_in_node = dn->ofs_in_node;
39 
40 	wait_on_page_writeback(node_page);
41 
42 	rn = (struct f2fs_node *)page_address(node_page);
43 
44 	/* Get physical address of data block */
45 	addr_array = blkaddr_in_node(rn);
46 	addr_array[ofs_in_node] = cpu_to_le32(new_addr);
47 	set_page_dirty(node_page);
48 }
49 
50 int reserve_new_block(struct dnode_of_data *dn)
51 {
52 	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
53 
54 	if (is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))
55 		return -EPERM;
56 	if (!inc_valid_block_count(sbi, dn->inode, 1))
57 		return -ENOSPC;
58 
59 	trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
60 
61 	__set_data_blkaddr(dn, NEW_ADDR);
62 	dn->data_blkaddr = NEW_ADDR;
63 	sync_inode_page(dn);
64 	return 0;
65 }
66 
67 static int check_extent_cache(struct inode *inode, pgoff_t pgofs,
68 					struct buffer_head *bh_result)
69 {
70 	struct f2fs_inode_info *fi = F2FS_I(inode);
71 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
72 	pgoff_t start_fofs, end_fofs;
73 	block_t start_blkaddr;
74 
75 	read_lock(&fi->ext.ext_lock);
76 	if (fi->ext.len == 0) {
77 		read_unlock(&fi->ext.ext_lock);
78 		return 0;
79 	}
80 
81 	sbi->total_hit_ext++;
82 	start_fofs = fi->ext.fofs;
83 	end_fofs = fi->ext.fofs + fi->ext.len - 1;
84 	start_blkaddr = fi->ext.blk_addr;
85 
86 	if (pgofs >= start_fofs && pgofs <= end_fofs) {
87 		unsigned int blkbits = inode->i_sb->s_blocksize_bits;
88 		size_t count;
89 
90 		clear_buffer_new(bh_result);
91 		map_bh(bh_result, inode->i_sb,
92 				start_blkaddr + pgofs - start_fofs);
93 		count = end_fofs - pgofs + 1;
94 		if (count < (UINT_MAX >> blkbits))
95 			bh_result->b_size = (count << blkbits);
96 		else
97 			bh_result->b_size = UINT_MAX;
98 
99 		sbi->read_hit_ext++;
100 		read_unlock(&fi->ext.ext_lock);
101 		return 1;
102 	}
103 	read_unlock(&fi->ext.ext_lock);
104 	return 0;
105 }
106 
107 void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
108 {
109 	struct f2fs_inode_info *fi = F2FS_I(dn->inode);
110 	pgoff_t fofs, start_fofs, end_fofs;
111 	block_t start_blkaddr, end_blkaddr;
112 
113 	BUG_ON(blk_addr == NEW_ADDR);
114 	fofs = start_bidx_of_node(ofs_of_node(dn->node_page)) + dn->ofs_in_node;
115 
116 	/* Update the page address in the parent node */
117 	__set_data_blkaddr(dn, blk_addr);
118 
119 	write_lock(&fi->ext.ext_lock);
120 
121 	start_fofs = fi->ext.fofs;
122 	end_fofs = fi->ext.fofs + fi->ext.len - 1;
123 	start_blkaddr = fi->ext.blk_addr;
124 	end_blkaddr = fi->ext.blk_addr + fi->ext.len - 1;
125 
126 	/* Drop and initialize the matched extent */
127 	if (fi->ext.len == 1 && fofs == start_fofs)
128 		fi->ext.len = 0;
129 
130 	/* Initial extent */
131 	if (fi->ext.len == 0) {
132 		if (blk_addr != NULL_ADDR) {
133 			fi->ext.fofs = fofs;
134 			fi->ext.blk_addr = blk_addr;
135 			fi->ext.len = 1;
136 		}
137 		goto end_update;
138 	}
139 
140 	/* Front merge */
141 	if (fofs == start_fofs - 1 && blk_addr == start_blkaddr - 1) {
142 		fi->ext.fofs--;
143 		fi->ext.blk_addr--;
144 		fi->ext.len++;
145 		goto end_update;
146 	}
147 
148 	/* Back merge */
149 	if (fofs == end_fofs + 1 && blk_addr == end_blkaddr + 1) {
150 		fi->ext.len++;
151 		goto end_update;
152 	}
153 
154 	/* Split the existing extent */
155 	if (fi->ext.len > 1 &&
156 		fofs >= start_fofs && fofs <= end_fofs) {
157 		if ((end_fofs - fofs) < (fi->ext.len >> 1)) {
158 			fi->ext.len = fofs - start_fofs;
159 		} else {
160 			fi->ext.fofs = fofs + 1;
161 			fi->ext.blk_addr = start_blkaddr +
162 					fofs - start_fofs + 1;
163 			fi->ext.len -= fofs - start_fofs + 1;
164 		}
165 		goto end_update;
166 	}
167 	write_unlock(&fi->ext.ext_lock);
168 	return;
169 
170 end_update:
171 	write_unlock(&fi->ext.ext_lock);
172 	sync_inode_page(dn);
173 	return;
174 }
175 
176 struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync)
177 {
178 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
179 	struct address_space *mapping = inode->i_mapping;
180 	struct dnode_of_data dn;
181 	struct page *page;
182 	int err;
183 
184 	page = find_get_page(mapping, index);
185 	if (page && PageUptodate(page))
186 		return page;
187 	f2fs_put_page(page, 0);
188 
189 	set_new_dnode(&dn, inode, NULL, NULL, 0);
190 	err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
191 	if (err)
192 		return ERR_PTR(err);
193 	f2fs_put_dnode(&dn);
194 
195 	if (dn.data_blkaddr == NULL_ADDR)
196 		return ERR_PTR(-ENOENT);
197 
198 	/* By fallocate(), there is no cached page, but with NEW_ADDR */
199 	if (dn.data_blkaddr == NEW_ADDR)
200 		return ERR_PTR(-EINVAL);
201 
202 	page = grab_cache_page(mapping, index);
203 	if (!page)
204 		return ERR_PTR(-ENOMEM);
205 
206 	if (PageUptodate(page)) {
207 		unlock_page(page);
208 		return page;
209 	}
210 
211 	err = f2fs_readpage(sbi, page, dn.data_blkaddr,
212 					sync ? READ_SYNC : READA);
213 	if (sync) {
214 		wait_on_page_locked(page);
215 		if (!PageUptodate(page)) {
216 			f2fs_put_page(page, 0);
217 			return ERR_PTR(-EIO);
218 		}
219 	}
220 	return page;
221 }
222 
223 /*
224  * If it tries to access a hole, return an error.
225  * Because, the callers, functions in dir.c and GC, should be able to know
226  * whether this page exists or not.
227  */
228 struct page *get_lock_data_page(struct inode *inode, pgoff_t index)
229 {
230 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
231 	struct address_space *mapping = inode->i_mapping;
232 	struct dnode_of_data dn;
233 	struct page *page;
234 	int err;
235 
236 	set_new_dnode(&dn, inode, NULL, NULL, 0);
237 	err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
238 	if (err)
239 		return ERR_PTR(err);
240 	f2fs_put_dnode(&dn);
241 
242 	if (dn.data_blkaddr == NULL_ADDR)
243 		return ERR_PTR(-ENOENT);
244 repeat:
245 	page = grab_cache_page(mapping, index);
246 	if (!page)
247 		return ERR_PTR(-ENOMEM);
248 
249 	if (PageUptodate(page))
250 		return page;
251 
252 	BUG_ON(dn.data_blkaddr == NEW_ADDR);
253 	BUG_ON(dn.data_blkaddr == NULL_ADDR);
254 
255 	err = f2fs_readpage(sbi, page, dn.data_blkaddr, READ_SYNC);
256 	if (err)
257 		return ERR_PTR(err);
258 
259 	lock_page(page);
260 	if (!PageUptodate(page)) {
261 		f2fs_put_page(page, 1);
262 		return ERR_PTR(-EIO);
263 	}
264 	if (page->mapping != mapping) {
265 		f2fs_put_page(page, 1);
266 		goto repeat;
267 	}
268 	return page;
269 }
270 
271 /*
272  * Caller ensures that this data page is never allocated.
273  * A new zero-filled data page is allocated in the page cache.
274  *
275  * Also, caller should grab and release a mutex by calling mutex_lock_op() and
276  * mutex_unlock_op().
277  */
278 struct page *get_new_data_page(struct inode *inode, pgoff_t index,
279 						bool new_i_size)
280 {
281 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
282 	struct address_space *mapping = inode->i_mapping;
283 	struct page *page;
284 	struct dnode_of_data dn;
285 	int err;
286 
287 	set_new_dnode(&dn, inode, NULL, NULL, 0);
288 	err = get_dnode_of_data(&dn, index, ALLOC_NODE);
289 	if (err)
290 		return ERR_PTR(err);
291 
292 	if (dn.data_blkaddr == NULL_ADDR) {
293 		if (reserve_new_block(&dn)) {
294 			f2fs_put_dnode(&dn);
295 			return ERR_PTR(-ENOSPC);
296 		}
297 	}
298 	f2fs_put_dnode(&dn);
299 repeat:
300 	page = grab_cache_page(mapping, index);
301 	if (!page)
302 		return ERR_PTR(-ENOMEM);
303 
304 	if (PageUptodate(page))
305 		return page;
306 
307 	if (dn.data_blkaddr == NEW_ADDR) {
308 		zero_user_segment(page, 0, PAGE_CACHE_SIZE);
309 		SetPageUptodate(page);
310 	} else {
311 		err = f2fs_readpage(sbi, page, dn.data_blkaddr, READ_SYNC);
312 		if (err)
313 			return ERR_PTR(err);
314 		lock_page(page);
315 		if (!PageUptodate(page)) {
316 			f2fs_put_page(page, 1);
317 			return ERR_PTR(-EIO);
318 		}
319 		if (page->mapping != mapping) {
320 			f2fs_put_page(page, 1);
321 			goto repeat;
322 		}
323 	}
324 
325 	if (new_i_size &&
326 		i_size_read(inode) < ((index + 1) << PAGE_CACHE_SHIFT)) {
327 		i_size_write(inode, ((index + 1) << PAGE_CACHE_SHIFT));
328 		mark_inode_dirty_sync(inode);
329 	}
330 	return page;
331 }
332 
333 static void read_end_io(struct bio *bio, int err)
334 {
335 	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
336 	struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
337 
338 	do {
339 		struct page *page = bvec->bv_page;
340 
341 		if (--bvec >= bio->bi_io_vec)
342 			prefetchw(&bvec->bv_page->flags);
343 
344 		if (uptodate) {
345 			SetPageUptodate(page);
346 		} else {
347 			ClearPageUptodate(page);
348 			SetPageError(page);
349 		}
350 		unlock_page(page);
351 	} while (bvec >= bio->bi_io_vec);
352 	kfree(bio->bi_private);
353 	bio_put(bio);
354 }
355 
356 /*
357  * Fill the locked page with data located in the block address.
358  * Return unlocked page.
359  */
360 int f2fs_readpage(struct f2fs_sb_info *sbi, struct page *page,
361 					block_t blk_addr, int type)
362 {
363 	struct block_device *bdev = sbi->sb->s_bdev;
364 	struct bio *bio;
365 
366 	trace_f2fs_readpage(page, blk_addr, type);
367 
368 	down_read(&sbi->bio_sem);
369 
370 	/* Allocate a new bio */
371 	bio = f2fs_bio_alloc(bdev, 1);
372 
373 	/* Initialize the bio */
374 	bio->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
375 	bio->bi_end_io = read_end_io;
376 
377 	if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
378 		kfree(bio->bi_private);
379 		bio_put(bio);
380 		up_read(&sbi->bio_sem);
381 		f2fs_put_page(page, 1);
382 		return -EFAULT;
383 	}
384 
385 	submit_bio(type, bio);
386 	up_read(&sbi->bio_sem);
387 	return 0;
388 }
389 
390 /*
391  * This function should be used by the data read flow only where it
392  * does not check the "create" flag that indicates block allocation.
393  * The reason for this special functionality is to exploit VFS readahead
394  * mechanism.
395  */
396 static int get_data_block_ro(struct inode *inode, sector_t iblock,
397 			struct buffer_head *bh_result, int create)
398 {
399 	unsigned int blkbits = inode->i_sb->s_blocksize_bits;
400 	unsigned maxblocks = bh_result->b_size >> blkbits;
401 	struct dnode_of_data dn;
402 	pgoff_t pgofs;
403 	int err;
404 
405 	/* Get the page offset from the block offset(iblock) */
406 	pgofs =	(pgoff_t)(iblock >> (PAGE_CACHE_SHIFT - blkbits));
407 
408 	if (check_extent_cache(inode, pgofs, bh_result)) {
409 		trace_f2fs_get_data_block(inode, iblock, bh_result, 0);
410 		return 0;
411 	}
412 
413 	/* When reading holes, we need its node page */
414 	set_new_dnode(&dn, inode, NULL, NULL, 0);
415 	err = get_dnode_of_data(&dn, pgofs, LOOKUP_NODE_RA);
416 	if (err) {
417 		trace_f2fs_get_data_block(inode, iblock, bh_result, err);
418 		return (err == -ENOENT) ? 0 : err;
419 	}
420 
421 	/* It does not support data allocation */
422 	BUG_ON(create);
423 
424 	if (dn.data_blkaddr != NEW_ADDR && dn.data_blkaddr != NULL_ADDR) {
425 		int i;
426 		unsigned int end_offset;
427 
428 		end_offset = IS_INODE(dn.node_page) ?
429 				ADDRS_PER_INODE :
430 				ADDRS_PER_BLOCK;
431 
432 		clear_buffer_new(bh_result);
433 
434 		/* Give more consecutive addresses for the read ahead */
435 		for (i = 0; i < end_offset - dn.ofs_in_node; i++)
436 			if (((datablock_addr(dn.node_page,
437 							dn.ofs_in_node + i))
438 				!= (dn.data_blkaddr + i)) || maxblocks == i)
439 				break;
440 		map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
441 		bh_result->b_size = (i << blkbits);
442 	}
443 	f2fs_put_dnode(&dn);
444 	trace_f2fs_get_data_block(inode, iblock, bh_result, 0);
445 	return 0;
446 }
447 
448 static int f2fs_read_data_page(struct file *file, struct page *page)
449 {
450 	return mpage_readpage(page, get_data_block_ro);
451 }
452 
453 static int f2fs_read_data_pages(struct file *file,
454 			struct address_space *mapping,
455 			struct list_head *pages, unsigned nr_pages)
456 {
457 	return mpage_readpages(mapping, pages, nr_pages, get_data_block_ro);
458 }
459 
460 int do_write_data_page(struct page *page)
461 {
462 	struct inode *inode = page->mapping->host;
463 	block_t old_blk_addr, new_blk_addr;
464 	struct dnode_of_data dn;
465 	int err = 0;
466 
467 	set_new_dnode(&dn, inode, NULL, NULL, 0);
468 	err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
469 	if (err)
470 		return err;
471 
472 	old_blk_addr = dn.data_blkaddr;
473 
474 	/* This page is already truncated */
475 	if (old_blk_addr == NULL_ADDR)
476 		goto out_writepage;
477 
478 	set_page_writeback(page);
479 
480 	/*
481 	 * If current allocation needs SSR,
482 	 * it had better in-place writes for updated data.
483 	 */
484 	if (old_blk_addr != NEW_ADDR && !is_cold_data(page) &&
485 				need_inplace_update(inode)) {
486 		rewrite_data_page(F2FS_SB(inode->i_sb), page,
487 						old_blk_addr);
488 	} else {
489 		write_data_page(inode, page, &dn,
490 				old_blk_addr, &new_blk_addr);
491 		update_extent_cache(new_blk_addr, &dn);
492 	}
493 out_writepage:
494 	f2fs_put_dnode(&dn);
495 	return err;
496 }
497 
498 static int f2fs_write_data_page(struct page *page,
499 					struct writeback_control *wbc)
500 {
501 	struct inode *inode = page->mapping->host;
502 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
503 	loff_t i_size = i_size_read(inode);
504 	const pgoff_t end_index = ((unsigned long long) i_size)
505 							>> PAGE_CACHE_SHIFT;
506 	unsigned offset;
507 	bool need_balance_fs = false;
508 	int err = 0;
509 
510 	if (page->index < end_index)
511 		goto write;
512 
513 	/*
514 	 * If the offset is out-of-range of file size,
515 	 * this page does not have to be written to disk.
516 	 */
517 	offset = i_size & (PAGE_CACHE_SIZE - 1);
518 	if ((page->index >= end_index + 1) || !offset) {
519 		if (S_ISDIR(inode->i_mode)) {
520 			dec_page_count(sbi, F2FS_DIRTY_DENTS);
521 			inode_dec_dirty_dents(inode);
522 		}
523 		goto out;
524 	}
525 
526 	zero_user_segment(page, offset, PAGE_CACHE_SIZE);
527 write:
528 	if (sbi->por_doing) {
529 		err = AOP_WRITEPAGE_ACTIVATE;
530 		goto redirty_out;
531 	}
532 
533 	/* Dentry blocks are controlled by checkpoint */
534 	if (S_ISDIR(inode->i_mode)) {
535 		dec_page_count(sbi, F2FS_DIRTY_DENTS);
536 		inode_dec_dirty_dents(inode);
537 		err = do_write_data_page(page);
538 	} else {
539 		int ilock = mutex_lock_op(sbi);
540 		err = do_write_data_page(page);
541 		mutex_unlock_op(sbi, ilock);
542 		need_balance_fs = true;
543 	}
544 	if (err == -ENOENT)
545 		goto out;
546 	else if (err)
547 		goto redirty_out;
548 
549 	if (wbc->for_reclaim)
550 		f2fs_submit_bio(sbi, DATA, true);
551 
552 	clear_cold_data(page);
553 out:
554 	unlock_page(page);
555 	if (need_balance_fs)
556 		f2fs_balance_fs(sbi);
557 	return 0;
558 
559 redirty_out:
560 	wbc->pages_skipped++;
561 	set_page_dirty(page);
562 	return err;
563 }
564 
565 #define MAX_DESIRED_PAGES_WP	4096
566 
567 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
568 			void *data)
569 {
570 	struct address_space *mapping = data;
571 	int ret = mapping->a_ops->writepage(page, wbc);
572 	mapping_set_error(mapping, ret);
573 	return ret;
574 }
575 
576 static int f2fs_write_data_pages(struct address_space *mapping,
577 			    struct writeback_control *wbc)
578 {
579 	struct inode *inode = mapping->host;
580 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
581 	bool locked = false;
582 	int ret;
583 	long excess_nrtw = 0, desired_nrtw;
584 
585 	/* deal with chardevs and other special file */
586 	if (!mapping->a_ops->writepage)
587 		return 0;
588 
589 	if (wbc->nr_to_write < MAX_DESIRED_PAGES_WP) {
590 		desired_nrtw = MAX_DESIRED_PAGES_WP;
591 		excess_nrtw = desired_nrtw - wbc->nr_to_write;
592 		wbc->nr_to_write = desired_nrtw;
593 	}
594 
595 	if (!S_ISDIR(inode->i_mode)) {
596 		mutex_lock(&sbi->writepages);
597 		locked = true;
598 	}
599 	ret = write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
600 	if (locked)
601 		mutex_unlock(&sbi->writepages);
602 	f2fs_submit_bio(sbi, DATA, (wbc->sync_mode == WB_SYNC_ALL));
603 
604 	remove_dirty_dir_inode(inode);
605 
606 	wbc->nr_to_write -= excess_nrtw;
607 	return ret;
608 }
609 
610 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
611 		loff_t pos, unsigned len, unsigned flags,
612 		struct page **pagep, void **fsdata)
613 {
614 	struct inode *inode = mapping->host;
615 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
616 	struct page *page;
617 	pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
618 	struct dnode_of_data dn;
619 	int err = 0;
620 	int ilock;
621 
622 	/* for nobh_write_end */
623 	*fsdata = NULL;
624 
625 	f2fs_balance_fs(sbi);
626 repeat:
627 	page = grab_cache_page_write_begin(mapping, index, flags);
628 	if (!page)
629 		return -ENOMEM;
630 	*pagep = page;
631 
632 	ilock = mutex_lock_op(sbi);
633 
634 	set_new_dnode(&dn, inode, NULL, NULL, 0);
635 	err = get_dnode_of_data(&dn, index, ALLOC_NODE);
636 	if (err)
637 		goto err;
638 
639 	if (dn.data_blkaddr == NULL_ADDR)
640 		err = reserve_new_block(&dn);
641 
642 	f2fs_put_dnode(&dn);
643 	if (err)
644 		goto err;
645 
646 	mutex_unlock_op(sbi, ilock);
647 
648 	if ((len == PAGE_CACHE_SIZE) || PageUptodate(page))
649 		return 0;
650 
651 	if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
652 		unsigned start = pos & (PAGE_CACHE_SIZE - 1);
653 		unsigned end = start + len;
654 
655 		/* Reading beyond i_size is simple: memset to zero */
656 		zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE);
657 		goto out;
658 	}
659 
660 	if (dn.data_blkaddr == NEW_ADDR) {
661 		zero_user_segment(page, 0, PAGE_CACHE_SIZE);
662 	} else {
663 		err = f2fs_readpage(sbi, page, dn.data_blkaddr, READ_SYNC);
664 		if (err)
665 			return err;
666 		lock_page(page);
667 		if (!PageUptodate(page)) {
668 			f2fs_put_page(page, 1);
669 			return -EIO;
670 		}
671 		if (page->mapping != mapping) {
672 			f2fs_put_page(page, 1);
673 			goto repeat;
674 		}
675 	}
676 out:
677 	SetPageUptodate(page);
678 	clear_cold_data(page);
679 	return 0;
680 
681 err:
682 	mutex_unlock_op(sbi, ilock);
683 	f2fs_put_page(page, 1);
684 	return err;
685 }
686 
687 static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
688 		const struct iovec *iov, loff_t offset, unsigned long nr_segs)
689 {
690 	struct file *file = iocb->ki_filp;
691 	struct inode *inode = file->f_mapping->host;
692 
693 	if (rw == WRITE)
694 		return 0;
695 
696 	/* Needs synchronization with the cleaner */
697 	return blockdev_direct_IO(rw, iocb, inode, iov, offset, nr_segs,
698 						  get_data_block_ro);
699 }
700 
701 static void f2fs_invalidate_data_page(struct page *page, unsigned long offset)
702 {
703 	struct inode *inode = page->mapping->host;
704 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
705 	if (S_ISDIR(inode->i_mode) && PageDirty(page)) {
706 		dec_page_count(sbi, F2FS_DIRTY_DENTS);
707 		inode_dec_dirty_dents(inode);
708 	}
709 	ClearPagePrivate(page);
710 }
711 
712 static int f2fs_release_data_page(struct page *page, gfp_t wait)
713 {
714 	ClearPagePrivate(page);
715 	return 1;
716 }
717 
718 static int f2fs_set_data_page_dirty(struct page *page)
719 {
720 	struct address_space *mapping = page->mapping;
721 	struct inode *inode = mapping->host;
722 
723 	SetPageUptodate(page);
724 	if (!PageDirty(page)) {
725 		__set_page_dirty_nobuffers(page);
726 		set_dirty_dir_page(inode, page);
727 		return 1;
728 	}
729 	return 0;
730 }
731 
732 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
733 {
734 	return generic_block_bmap(mapping, block, get_data_block_ro);
735 }
736 
737 const struct address_space_operations f2fs_dblock_aops = {
738 	.readpage	= f2fs_read_data_page,
739 	.readpages	= f2fs_read_data_pages,
740 	.writepage	= f2fs_write_data_page,
741 	.writepages	= f2fs_write_data_pages,
742 	.write_begin	= f2fs_write_begin,
743 	.write_end	= nobh_write_end,
744 	.set_page_dirty	= f2fs_set_data_page_dirty,
745 	.invalidatepage	= f2fs_invalidate_data_page,
746 	.releasepage	= f2fs_release_data_page,
747 	.direct_IO	= f2fs_direct_IO,
748 	.bmap		= f2fs_bmap,
749 };
750