Lines Matching +full:page +full:- +full:offset
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) International Business Machines Corp., 2000-2005
4 * Portions Copyright (C) Christoph Hellwig, 2001-2002
26 uint pagealloc; /* # of page allocations */
27 uint pagefree; /* # of page frees */
32 #define metapage_locked(mp) test_bit(META_locked, &(mp)->flag)
33 #define trylock_metapage(mp) test_and_set_bit_lock(META_locked, &(mp)->flag)
37 clear_bit_unlock(META_locked, &mp->flag); in unlock_metapage()
38 wake_up(&mp->wait); in unlock_metapage()
45 add_wait_queue_exclusive(&mp->wait, &wait); in __lock_metapage()
49 unlock_page(mp->page); in __lock_metapage()
51 lock_page(mp->page); in __lock_metapage()
55 remove_wait_queue(&mp->wait, &wait); in __lock_metapage()
59 * Must have mp->page locked
80 #define mp_anchor(page) ((struct meta_anchor *)page_private(page)) argument
82 static inline struct metapage *page_to_mp(struct page *page, int offset) in page_to_mp() argument
84 if (!PagePrivate(page)) in page_to_mp()
86 return mp_anchor(page)->mp[offset >> L2PSIZE]; in page_to_mp()
89 static inline int insert_metapage(struct page *page, struct metapage *mp) in insert_metapage() argument
95 if (PagePrivate(page)) in insert_metapage()
96 a = mp_anchor(page); in insert_metapage()
100 return -ENOMEM; in insert_metapage()
101 set_page_private(page, (unsigned long)a); in insert_metapage()
102 SetPagePrivate(page); in insert_metapage()
103 kmap(page); in insert_metapage()
107 l2mp_blocks = L2PSIZE - page->mapping->host->i_blkbits; in insert_metapage()
108 index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1); in insert_metapage()
109 a->mp_count++; in insert_metapage()
110 a->mp[index] = mp; in insert_metapage()
116 static inline void remove_metapage(struct page *page, struct metapage *mp) in remove_metapage() argument
118 struct meta_anchor *a = mp_anchor(page); in remove_metapage()
119 int l2mp_blocks = L2PSIZE - page->mapping->host->i_blkbits; in remove_metapage()
122 index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1); in remove_metapage()
124 BUG_ON(a->mp[index] != mp); in remove_metapage()
126 a->mp[index] = NULL; in remove_metapage()
127 if (--a->mp_count == 0) { in remove_metapage()
129 set_page_private(page, 0); in remove_metapage()
130 ClearPagePrivate(page); in remove_metapage()
131 kunmap(page); in remove_metapage()
135 static inline void inc_io(struct page *page) in inc_io() argument
137 atomic_inc(&mp_anchor(page)->io_count); in inc_io()
140 static inline void dec_io(struct page *page, void (*handler) (struct page *)) in dec_io() argument
142 if (atomic_dec_and_test(&mp_anchor(page)->io_count)) in dec_io()
143 handler(page); in dec_io()
147 static inline struct metapage *page_to_mp(struct page *page, int offset) in page_to_mp() argument
149 return PagePrivate(page) ? (struct metapage *)page_private(page) : NULL; in page_to_mp()
152 static inline int insert_metapage(struct page *page, struct metapage *mp) in insert_metapage() argument
155 set_page_private(page, (unsigned long)mp); in insert_metapage()
156 SetPagePrivate(page); in insert_metapage()
157 kmap(page); in insert_metapage()
162 static inline void remove_metapage(struct page *page, struct metapage *mp) in remove_metapage() argument
164 set_page_private(page, 0); in remove_metapage()
165 ClearPagePrivate(page); in remove_metapage()
166 kunmap(page); in remove_metapage()
169 #define inc_io(page) do {} while(0) argument
170 #define dec_io(page, handler) handler(page) argument
179 mp->lid = 0; in alloc_metapage()
180 mp->lsn = 0; in alloc_metapage()
181 mp->data = NULL; in alloc_metapage()
182 mp->clsn = 0; in alloc_metapage()
183 mp->log = NULL; in alloc_metapage()
184 init_waitqueue_head(&mp->wait); in alloc_metapage()
202 return -ENOMEM; in metapage_init()
209 return -ENOMEM; in metapage_init()
221 static inline void drop_metapage(struct page *page, struct metapage *mp) in drop_metapage() argument
223 if (mp->count || mp->nohomeok || test_bit(META_dirty, &mp->flag) || in drop_metapage()
224 test_bit(META_io, &mp->flag)) in drop_metapage()
226 remove_metapage(page, mp); in drop_metapage()
241 sector_t file_blocks = (inode->i_size + inode->i_sb->s_blocksize - 1) >> in metapage_get_blocks()
242 inode->i_blkbits; in metapage_get_blocks()
247 *len = file_blocks - lblock; in metapage_get_blocks()
249 if (inode->i_ino) { in metapage_get_blocks()
260 static void last_read_complete(struct page *page) in last_read_complete() argument
262 if (!PageError(page)) in last_read_complete()
263 SetPageUptodate(page); in last_read_complete()
264 unlock_page(page); in last_read_complete()
269 struct page *page = bio->bi_private; in metapage_read_end_io() local
271 if (bio->bi_status) { in metapage_read_end_io()
273 SetPageError(page); in metapage_read_end_io()
276 dec_io(page, last_read_complete); in metapage_read_end_io()
282 struct jfs_log *log = mp->log; in remove_from_logsync()
292 if (mp->lsn) { in remove_from_logsync()
293 mp->log = NULL; in remove_from_logsync()
294 mp->lsn = 0; in remove_from_logsync()
295 mp->clsn = 0; in remove_from_logsync()
296 log->count--; in remove_from_logsync()
297 list_del(&mp->synclist); in remove_from_logsync()
302 static void last_write_complete(struct page *page) in last_write_complete() argument
305 unsigned int offset; in last_write_complete() local
307 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { in last_write_complete()
308 mp = page_to_mp(page, offset); in last_write_complete()
309 if (mp && test_bit(META_io, &mp->flag)) { in last_write_complete()
310 if (mp->lsn) in last_write_complete()
312 clear_bit(META_io, &mp->flag); in last_write_complete()
316 * safe unless I have the page locked in last_write_complete()
319 end_page_writeback(page); in last_write_complete()
324 struct page *page = bio->bi_private; in metapage_write_end_io() local
326 BUG_ON(!PagePrivate(page)); in metapage_write_end_io()
328 if (bio->bi_status) { in metapage_write_end_io()
330 SetPageError(page); in metapage_write_end_io()
332 dec_io(page, last_write_complete); in metapage_write_end_io()
336 static int metapage_writepage(struct page *page, struct writeback_control *wbc) in metapage_writepage() argument
339 int block_offset; /* block offset of mp within page */ in metapage_writepage()
340 struct inode *inode = page->mapping->host; in metapage_writepage()
341 int blocks_per_mp = JFS_SBI(inode->i_sb)->nbperpage; in metapage_writepage()
353 int offset; in metapage_writepage() local
356 page_start = (sector_t)page->index << in metapage_writepage()
357 (PAGE_SHIFT - inode->i_blkbits); in metapage_writepage()
358 BUG_ON(!PageLocked(page)); in metapage_writepage()
359 BUG_ON(PageWriteback(page)); in metapage_writepage()
360 set_page_writeback(page); in metapage_writepage()
362 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { in metapage_writepage()
363 mp = page_to_mp(page, offset); in metapage_writepage()
365 if (!mp || !test_bit(META_dirty, &mp->flag)) in metapage_writepage()
368 if (mp->nohomeok && !test_bit(META_forcewrite, &mp->flag)) { in metapage_writepage()
371 * Make sure this page isn't blocked indefinitely. in metapage_writepage()
374 if (mp->log && !(mp->log->cflag & logGC_PAGEOUT)) in metapage_writepage()
375 jfs_flush_journal(mp->log, 0); in metapage_writepage()
379 clear_bit(META_dirty, &mp->flag); in metapage_writepage()
380 set_bit(META_io, &mp->flag); in metapage_writepage()
381 block_offset = offset >> inode->i_blkbits; in metapage_writepage()
387 xlen -= len; in metapage_writepage()
388 bio_bytes += len << inode->i_blkbits; in metapage_writepage()
392 if (bio_add_page(bio, page, bio_bytes, bio_offset) < in metapage_writepage()
399 inc_io(page); in metapage_writepage()
400 if (!bio->bi_iter.bi_size) in metapage_writepage()
406 inc_io(page); in metapage_writepage()
407 xlen = (PAGE_SIZE - offset) >> inode->i_blkbits; in metapage_writepage()
413 * with dec_io() until we're done with the page in metapage_writepage()
418 len = min(xlen, (int)JFS_SBI(inode->i_sb)->nbperpage); in metapage_writepage()
420 bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOFS); in metapage_writepage()
421 bio->bi_iter.bi_sector = pblock << (inode->i_blkbits - 9); in metapage_writepage()
422 bio->bi_end_io = metapage_write_end_io; in metapage_writepage()
423 bio->bi_private = page; in metapage_writepage()
426 bio_offset = offset; in metapage_writepage()
427 bio_bytes = len << inode->i_blkbits; in metapage_writepage()
429 xlen -= len; in metapage_writepage()
433 if (bio_add_page(bio, page, bio_bytes, bio_offset) < bio_bytes) in metapage_writepage()
435 if (!bio->bi_iter.bi_size) in metapage_writepage()
442 redirty_page_for_writepage(wbc, page); in metapage_writepage()
444 unlock_page(page); in metapage_writepage()
450 end_page_writeback(page); in metapage_writepage()
462 unlock_page(page); in metapage_writepage()
463 dec_io(page, last_write_complete); in metapage_writepage()
465 while (bad_blocks--) in metapage_writepage()
466 dec_io(page, last_write_complete); in metapage_writepage()
467 return -EIO; in metapage_writepage()
472 struct page *page = &folio->page; in metapage_read_folio() local
473 struct inode *inode = page->mapping->host; in metapage_read_folio()
476 int blocks_per_page = i_blocks_per_page(inode, page); in metapage_read_folio()
477 sector_t page_start; /* address of page in fs blocks */ in metapage_read_folio()
481 int offset; in metapage_read_folio() local
483 BUG_ON(!PageLocked(page)); in metapage_read_folio()
484 page_start = (sector_t)page->index << in metapage_read_folio()
485 (PAGE_SHIFT - inode->i_blkbits); in metapage_read_folio()
489 xlen = blocks_per_page - block_offset; in metapage_read_folio()
493 if (!PagePrivate(page)) in metapage_read_folio()
494 insert_metapage(page, NULL); in metapage_read_folio()
495 inc_io(page); in metapage_read_folio()
499 bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_READ, in metapage_read_folio()
501 bio->bi_iter.bi_sector = in metapage_read_folio()
502 pblock << (inode->i_blkbits - 9); in metapage_read_folio()
503 bio->bi_end_io = metapage_read_end_io; in metapage_read_folio()
504 bio->bi_private = page; in metapage_read_folio()
505 len = xlen << inode->i_blkbits; in metapage_read_folio()
506 offset = block_offset << inode->i_blkbits; in metapage_read_folio()
507 if (bio_add_page(bio, page, len, offset) < len) in metapage_read_folio()
516 unlock_page(page); in metapage_read_folio()
523 dec_io(page, last_read_complete); in metapage_read_folio()
524 return -EIO; in metapage_read_folio()
531 int offset; in metapage_release_folio() local
533 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { in metapage_release_folio()
534 mp = page_to_mp(&folio->page, offset); in metapage_release_folio()
540 if (mp->count || mp->nohomeok || in metapage_release_folio()
541 test_bit(META_dirty, &mp->flag)) { in metapage_release_folio()
542 jfs_info("count = %ld, nohomeok = %d", mp->count, in metapage_release_folio()
543 mp->nohomeok); in metapage_release_folio()
547 if (mp->lsn) in metapage_release_folio()
549 remove_metapage(&folio->page, mp); in metapage_release_folio()
556 static void metapage_invalidate_folio(struct folio *folio, size_t offset, in metapage_invalidate_folio() argument
559 BUG_ON(offset || length < folio_size(folio)); in metapage_invalidate_folio()
582 struct page *page; in __get_metapage() local
587 inode->i_ino, lblock, absolute); in __get_metapage()
589 l2bsize = inode->i_blkbits; in __get_metapage()
590 l2BlocksPerPage = PAGE_SHIFT - l2bsize; in __get_metapage()
592 page_offset = (lblock - (page_index << l2BlocksPerPage)) << l2bsize; in __get_metapage()
594 jfs_err("MetaData crosses page boundary!!"); in __get_metapage()
600 mapping = JFS_SBI(inode->i_sb)->direct_inode->i_mapping; in __get_metapage()
607 if ((lblock << inode->i_blkbits) >= inode->i_size) in __get_metapage()
609 mapping = inode->i_mapping; in __get_metapage()
613 page = grab_cache_page(mapping, page_index); in __get_metapage()
614 if (!page) { in __get_metapage()
618 SetPageUptodate(page); in __get_metapage()
620 page = read_mapping_page(mapping, page_index, NULL); in __get_metapage()
621 if (IS_ERR(page)) { in __get_metapage()
625 lock_page(page); in __get_metapage()
628 mp = page_to_mp(page, page_offset); in __get_metapage()
630 if (mp->logical_size != size) { in __get_metapage()
631 jfs_error(inode->i_sb, in __get_metapage()
632 "get_mp->logical_size != size\n"); in __get_metapage()
634 mp->logical_size, size); in __get_metapage()
638 mp->count++; in __get_metapage()
640 if (test_bit(META_discard, &mp->flag)) { in __get_metapage()
642 jfs_error(inode->i_sb, in __get_metapage()
647 clear_bit(META_discard, &mp->flag); in __get_metapage()
654 mp->page = page; in __get_metapage()
655 mp->sb = inode->i_sb; in __get_metapage()
656 mp->flag = 0; in __get_metapage()
657 mp->xflag = COMMIT_PAGE; in __get_metapage()
658 mp->count = 1; in __get_metapage()
659 mp->nohomeok = 0; in __get_metapage()
660 mp->logical_size = size; in __get_metapage()
661 mp->data = page_address(page) + page_offset; in __get_metapage()
662 mp->index = lblock; in __get_metapage()
663 if (unlikely(insert_metapage(page, mp))) { in __get_metapage()
672 memset(mp->data, 0, PSIZE); in __get_metapage()
675 unlock_page(page); in __get_metapage()
676 jfs_info("__get_metapage: returning = 0x%p data = 0x%p", mp, mp->data); in __get_metapage()
680 unlock_page(page); in __get_metapage()
687 get_page(mp->page); in grab_metapage()
688 lock_page(mp->page); in grab_metapage()
689 mp->count++; in grab_metapage()
691 unlock_page(mp->page); in grab_metapage()
694 static int metapage_write_one(struct page *page) in metapage_write_one() argument
696 struct folio *folio = page_folio(page); in metapage_write_one()
697 struct address_space *mapping = folio->mapping; in metapage_write_one()
710 ret = metapage_writepage(page, &wbc); in metapage_write_one()
725 struct page *page = mp->page; in force_metapage() local
727 set_bit(META_forcewrite, &mp->flag); in force_metapage()
728 clear_bit(META_sync, &mp->flag); in force_metapage()
729 get_page(page); in force_metapage()
730 lock_page(page); in force_metapage()
731 set_page_dirty(page); in force_metapage()
732 if (metapage_write_one(page)) in force_metapage()
733 jfs_error(mp->sb, "metapage_write_one() failed\n"); in force_metapage()
734 clear_bit(META_forcewrite, &mp->flag); in force_metapage()
735 put_page(page); in force_metapage()
740 lock_page(mp->page); in hold_metapage()
745 if (mp->count || mp->nohomeok) { in put_metapage()
747 unlock_page(mp->page); in put_metapage()
750 get_page(mp->page); in put_metapage()
751 mp->count++; in put_metapage()
753 unlock_page(mp->page); in put_metapage()
759 struct page *page = mp->page; in release_metapage() local
760 jfs_info("release_metapage: mp = 0x%p, flag = 0x%lx", mp, mp->flag); in release_metapage()
762 BUG_ON(!page); in release_metapage()
764 lock_page(page); in release_metapage()
767 assert(mp->count); in release_metapage()
768 if (--mp->count || mp->nohomeok) { in release_metapage()
769 unlock_page(page); in release_metapage()
770 put_page(page); in release_metapage()
774 if (test_bit(META_dirty, &mp->flag)) { in release_metapage()
775 set_page_dirty(page); in release_metapage()
776 if (test_bit(META_sync, &mp->flag)) { in release_metapage()
777 clear_bit(META_sync, &mp->flag); in release_metapage()
778 if (metapage_write_one(page)) in release_metapage()
779 jfs_error(mp->sb, "metapage_write_one() failed\n"); in release_metapage()
780 lock_page(page); in release_metapage()
782 } else if (mp->lsn) /* discard_metapage doesn't remove it */ in release_metapage()
786 drop_metapage(page, mp); in release_metapage()
788 unlock_page(page); in release_metapage()
789 put_page(page); in release_metapage()
795 int l2BlocksPerPage = PAGE_SHIFT - ip->i_blkbits; in __invalidate_metapages()
799 JFS_SBI(ip->i_sb)->direct_inode->i_mapping; in __invalidate_metapages()
801 struct page *page; in __invalidate_metapages() local
802 unsigned int offset; in __invalidate_metapages() local
808 for (lblock = addr & ~(BlocksPerPage - 1); lblock < addr + len; in __invalidate_metapages()
810 page = find_lock_page(mapping, lblock >> l2BlocksPerPage); in __invalidate_metapages()
811 if (!page) in __invalidate_metapages()
813 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { in __invalidate_metapages()
814 mp = page_to_mp(page, offset); in __invalidate_metapages()
817 if (mp->index < addr) in __invalidate_metapages()
819 if (mp->index >= addr + len) in __invalidate_metapages()
822 clear_bit(META_dirty, &mp->flag); in __invalidate_metapages()
823 set_bit(META_discard, &mp->flag); in __invalidate_metapages()
824 if (mp->lsn) in __invalidate_metapages()
827 unlock_page(page); in __invalidate_metapages()
828 put_page(page); in __invalidate_metapages()
838 "page allocations = %d\n" in jfs_mpstat_proc_show()
839 "page frees = %d\n" in jfs_mpstat_proc_show()