1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Buffer/page management specific to NILFS 4 * 5 * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. 6 * 7 * Written by Ryusuke Konishi and Seiji Kihara. 8 */ 9 10 #include <linux/pagemap.h> 11 #include <linux/writeback.h> 12 #include <linux/swap.h> 13 #include <linux/bitops.h> 14 #include <linux/page-flags.h> 15 #include <linux/list.h> 16 #include <linux/highmem.h> 17 #include <linux/pagevec.h> 18 #include <linux/gfp.h> 19 #include "nilfs.h" 20 #include "page.h" 21 #include "mdt.h" 22 23 24 #define NILFS_BUFFER_INHERENT_BITS \ 25 (BIT(BH_Uptodate) | BIT(BH_Mapped) | BIT(BH_NILFS_Node) | \ 26 BIT(BH_NILFS_Volatile) | BIT(BH_NILFS_Checked)) 27 28 static struct buffer_head * 29 __nilfs_get_page_block(struct page *page, unsigned long block, pgoff_t index, 30 int blkbits, unsigned long b_state) 31 32 { 33 unsigned long first_block; 34 struct buffer_head *bh; 35 36 if (!page_has_buffers(page)) 37 create_empty_buffers(page, 1 << blkbits, b_state); 38 39 first_block = (unsigned long)index << (PAGE_SHIFT - blkbits); 40 bh = nilfs_page_get_nth_block(page, block - first_block); 41 42 touch_buffer(bh); 43 wait_on_buffer(bh); 44 return bh; 45 } 46 47 struct buffer_head *nilfs_grab_buffer(struct inode *inode, 48 struct address_space *mapping, 49 unsigned long blkoff, 50 unsigned long b_state) 51 { 52 int blkbits = inode->i_blkbits; 53 pgoff_t index = blkoff >> (PAGE_SHIFT - blkbits); 54 struct page *page; 55 struct buffer_head *bh; 56 57 page = grab_cache_page(mapping, index); 58 if (unlikely(!page)) 59 return NULL; 60 61 bh = __nilfs_get_page_block(page, blkoff, index, blkbits, b_state); 62 if (unlikely(!bh)) { 63 unlock_page(page); 64 put_page(page); 65 return NULL; 66 } 67 return bh; 68 } 69 70 /** 71 * nilfs_forget_buffer - discard dirty state 72 * @bh: buffer head of the buffer to be discarded 73 */ 74 void nilfs_forget_buffer(struct buffer_head *bh) 75 { 76 struct page *page = bh->b_page; 77 const unsigned long clear_bits = 78 (BIT(BH_Uptodate) | BIT(BH_Dirty) | BIT(BH_Mapped) | 79 BIT(BH_Async_Write) | BIT(BH_NILFS_Volatile) | 80 BIT(BH_NILFS_Checked) | BIT(BH_NILFS_Redirected)); 81 82 lock_buffer(bh); 83 set_mask_bits(&bh->b_state, clear_bits, 0); 84 if (nilfs_page_buffers_clean(page)) 85 __nilfs_clear_page_dirty(page); 86 87 bh->b_blocknr = -1; 88 ClearPageUptodate(page); 89 ClearPageMappedToDisk(page); 90 unlock_buffer(bh); 91 brelse(bh); 92 } 93 94 /** 95 * nilfs_copy_buffer -- copy buffer data and flags 96 * @dbh: destination buffer 97 * @sbh: source buffer 98 */ 99 void nilfs_copy_buffer(struct buffer_head *dbh, struct buffer_head *sbh) 100 { 101 void *kaddr0, *kaddr1; 102 unsigned long bits; 103 struct page *spage = sbh->b_page, *dpage = dbh->b_page; 104 struct buffer_head *bh; 105 106 kaddr0 = kmap_atomic(spage); 107 kaddr1 = kmap_atomic(dpage); 108 memcpy(kaddr1 + bh_offset(dbh), kaddr0 + bh_offset(sbh), sbh->b_size); 109 kunmap_atomic(kaddr1); 110 kunmap_atomic(kaddr0); 111 112 dbh->b_state = sbh->b_state & NILFS_BUFFER_INHERENT_BITS; 113 dbh->b_blocknr = sbh->b_blocknr; 114 dbh->b_bdev = sbh->b_bdev; 115 116 bh = dbh; 117 bits = sbh->b_state & (BIT(BH_Uptodate) | BIT(BH_Mapped)); 118 while ((bh = bh->b_this_page) != dbh) { 119 lock_buffer(bh); 120 bits &= bh->b_state; 121 unlock_buffer(bh); 122 } 123 if (bits & BIT(BH_Uptodate)) 124 SetPageUptodate(dpage); 125 else 126 ClearPageUptodate(dpage); 127 if (bits & BIT(BH_Mapped)) 128 SetPageMappedToDisk(dpage); 129 else 130 ClearPageMappedToDisk(dpage); 131 } 132 133 /** 134 * nilfs_page_buffers_clean - check if a page has dirty buffers or not. 135 * @page: page to be checked 136 * 137 * nilfs_page_buffers_clean() returns zero if the page has dirty buffers. 138 * Otherwise, it returns non-zero value. 139 */ 140 int nilfs_page_buffers_clean(struct page *page) 141 { 142 struct buffer_head *bh, *head; 143 144 bh = head = page_buffers(page); 145 do { 146 if (buffer_dirty(bh)) 147 return 0; 148 bh = bh->b_this_page; 149 } while (bh != head); 150 return 1; 151 } 152 153 void nilfs_page_bug(struct page *page) 154 { 155 struct address_space *m; 156 unsigned long ino; 157 158 if (unlikely(!page)) { 159 printk(KERN_CRIT "NILFS_PAGE_BUG(NULL)\n"); 160 return; 161 } 162 163 m = page->mapping; 164 ino = m ? m->host->i_ino : 0; 165 166 printk(KERN_CRIT "NILFS_PAGE_BUG(%p): cnt=%d index#=%llu flags=0x%lx " 167 "mapping=%p ino=%lu\n", 168 page, page_ref_count(page), 169 (unsigned long long)page->index, page->flags, m, ino); 170 171 if (page_has_buffers(page)) { 172 struct buffer_head *bh, *head; 173 int i = 0; 174 175 bh = head = page_buffers(page); 176 do { 177 printk(KERN_CRIT 178 " BH[%d] %p: cnt=%d block#=%llu state=0x%lx\n", 179 i++, bh, atomic_read(&bh->b_count), 180 (unsigned long long)bh->b_blocknr, bh->b_state); 181 bh = bh->b_this_page; 182 } while (bh != head); 183 } 184 } 185 186 /** 187 * nilfs_copy_page -- copy the page with buffers 188 * @dst: destination page 189 * @src: source page 190 * @copy_dirty: flag whether to copy dirty states on the page's buffer heads. 191 * 192 * This function is for both data pages and btnode pages. The dirty flag 193 * should be treated by caller. The page must not be under i/o. 194 * Both src and dst page must be locked 195 */ 196 static void nilfs_copy_page(struct page *dst, struct page *src, int copy_dirty) 197 { 198 struct buffer_head *dbh, *dbufs, *sbh; 199 unsigned long mask = NILFS_BUFFER_INHERENT_BITS; 200 201 BUG_ON(PageWriteback(dst)); 202 203 sbh = page_buffers(src); 204 if (!page_has_buffers(dst)) 205 create_empty_buffers(dst, sbh->b_size, 0); 206 207 if (copy_dirty) 208 mask |= BIT(BH_Dirty); 209 210 dbh = dbufs = page_buffers(dst); 211 do { 212 lock_buffer(sbh); 213 lock_buffer(dbh); 214 dbh->b_state = sbh->b_state & mask; 215 dbh->b_blocknr = sbh->b_blocknr; 216 dbh->b_bdev = sbh->b_bdev; 217 sbh = sbh->b_this_page; 218 dbh = dbh->b_this_page; 219 } while (dbh != dbufs); 220 221 copy_highpage(dst, src); 222 223 if (PageUptodate(src) && !PageUptodate(dst)) 224 SetPageUptodate(dst); 225 else if (!PageUptodate(src) && PageUptodate(dst)) 226 ClearPageUptodate(dst); 227 if (PageMappedToDisk(src) && !PageMappedToDisk(dst)) 228 SetPageMappedToDisk(dst); 229 else if (!PageMappedToDisk(src) && PageMappedToDisk(dst)) 230 ClearPageMappedToDisk(dst); 231 232 do { 233 unlock_buffer(sbh); 234 unlock_buffer(dbh); 235 sbh = sbh->b_this_page; 236 dbh = dbh->b_this_page; 237 } while (dbh != dbufs); 238 } 239 240 int nilfs_copy_dirty_pages(struct address_space *dmap, 241 struct address_space *smap) 242 { 243 struct pagevec pvec; 244 unsigned int i; 245 pgoff_t index = 0; 246 int err = 0; 247 248 pagevec_init(&pvec); 249 repeat: 250 if (!pagevec_lookup_tag(&pvec, smap, &index, PAGECACHE_TAG_DIRTY)) 251 return 0; 252 253 for (i = 0; i < pagevec_count(&pvec); i++) { 254 struct page *page = pvec.pages[i], *dpage; 255 256 lock_page(page); 257 if (unlikely(!PageDirty(page))) 258 NILFS_PAGE_BUG(page, "inconsistent dirty state"); 259 260 dpage = grab_cache_page(dmap, page->index); 261 if (unlikely(!dpage)) { 262 /* No empty page is added to the page cache */ 263 err = -ENOMEM; 264 unlock_page(page); 265 break; 266 } 267 if (unlikely(!page_has_buffers(page))) 268 NILFS_PAGE_BUG(page, 269 "found empty page in dat page cache"); 270 271 nilfs_copy_page(dpage, page, 1); 272 __set_page_dirty_nobuffers(dpage); 273 274 unlock_page(dpage); 275 put_page(dpage); 276 unlock_page(page); 277 } 278 pagevec_release(&pvec); 279 cond_resched(); 280 281 if (likely(!err)) 282 goto repeat; 283 return err; 284 } 285 286 /** 287 * nilfs_copy_back_pages -- copy back pages to original cache from shadow cache 288 * @dmap: destination page cache 289 * @smap: source page cache 290 * 291 * No pages must be added to the cache during this process. 292 * This must be ensured by the caller. 293 */ 294 void nilfs_copy_back_pages(struct address_space *dmap, 295 struct address_space *smap) 296 { 297 struct pagevec pvec; 298 unsigned int i, n; 299 pgoff_t index = 0; 300 301 pagevec_init(&pvec); 302 repeat: 303 n = pagevec_lookup(&pvec, smap, &index); 304 if (!n) 305 return; 306 307 for (i = 0; i < pagevec_count(&pvec); i++) { 308 struct page *page = pvec.pages[i], *dpage; 309 pgoff_t offset = page->index; 310 311 lock_page(page); 312 dpage = find_lock_page(dmap, offset); 313 if (dpage) { 314 /* overwrite existing page in the destination cache */ 315 WARN_ON(PageDirty(dpage)); 316 nilfs_copy_page(dpage, page, 0); 317 unlock_page(dpage); 318 put_page(dpage); 319 /* Do we not need to remove page from smap here? */ 320 } else { 321 struct page *p; 322 323 /* move the page to the destination cache */ 324 xa_lock_irq(&smap->i_pages); 325 p = __xa_erase(&smap->i_pages, offset); 326 WARN_ON(page != p); 327 smap->nrpages--; 328 xa_unlock_irq(&smap->i_pages); 329 330 xa_lock_irq(&dmap->i_pages); 331 p = __xa_store(&dmap->i_pages, offset, page, GFP_NOFS); 332 if (unlikely(p)) { 333 /* Probably -ENOMEM */ 334 page->mapping = NULL; 335 put_page(page); 336 } else { 337 page->mapping = dmap; 338 dmap->nrpages++; 339 if (PageDirty(page)) 340 __xa_set_mark(&dmap->i_pages, offset, 341 PAGECACHE_TAG_DIRTY); 342 } 343 xa_unlock_irq(&dmap->i_pages); 344 } 345 unlock_page(page); 346 } 347 pagevec_release(&pvec); 348 cond_resched(); 349 350 goto repeat; 351 } 352 353 /** 354 * nilfs_clear_dirty_pages - discard dirty pages in address space 355 * @mapping: address space with dirty pages for discarding 356 * @silent: suppress [true] or print [false] warning messages 357 */ 358 void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent) 359 { 360 struct pagevec pvec; 361 unsigned int i; 362 pgoff_t index = 0; 363 364 pagevec_init(&pvec); 365 366 while (pagevec_lookup_tag(&pvec, mapping, &index, 367 PAGECACHE_TAG_DIRTY)) { 368 for (i = 0; i < pagevec_count(&pvec); i++) { 369 struct page *page = pvec.pages[i]; 370 371 lock_page(page); 372 nilfs_clear_dirty_page(page, silent); 373 unlock_page(page); 374 } 375 pagevec_release(&pvec); 376 cond_resched(); 377 } 378 } 379 380 /** 381 * nilfs_clear_dirty_page - discard dirty page 382 * @page: dirty page that will be discarded 383 * @silent: suppress [true] or print [false] warning messages 384 */ 385 void nilfs_clear_dirty_page(struct page *page, bool silent) 386 { 387 struct inode *inode = page->mapping->host; 388 struct super_block *sb = inode->i_sb; 389 390 BUG_ON(!PageLocked(page)); 391 392 if (!silent) 393 nilfs_warn(sb, "discard dirty page: offset=%lld, ino=%lu", 394 page_offset(page), inode->i_ino); 395 396 ClearPageUptodate(page); 397 ClearPageMappedToDisk(page); 398 399 if (page_has_buffers(page)) { 400 struct buffer_head *bh, *head; 401 const unsigned long clear_bits = 402 (BIT(BH_Uptodate) | BIT(BH_Dirty) | BIT(BH_Mapped) | 403 BIT(BH_Async_Write) | BIT(BH_NILFS_Volatile) | 404 BIT(BH_NILFS_Checked) | BIT(BH_NILFS_Redirected)); 405 406 bh = head = page_buffers(page); 407 do { 408 lock_buffer(bh); 409 if (!silent) 410 nilfs_warn(sb, 411 "discard dirty block: blocknr=%llu, size=%zu", 412 (u64)bh->b_blocknr, bh->b_size); 413 414 set_mask_bits(&bh->b_state, clear_bits, 0); 415 unlock_buffer(bh); 416 } while (bh = bh->b_this_page, bh != head); 417 } 418 419 __nilfs_clear_page_dirty(page); 420 } 421 422 unsigned int nilfs_page_count_clean_buffers(struct page *page, 423 unsigned int from, unsigned int to) 424 { 425 unsigned int block_start, block_end; 426 struct buffer_head *bh, *head; 427 unsigned int nc = 0; 428 429 for (bh = head = page_buffers(page), block_start = 0; 430 bh != head || !block_start; 431 block_start = block_end, bh = bh->b_this_page) { 432 block_end = block_start + bh->b_size; 433 if (block_end > from && block_start < to && !buffer_dirty(bh)) 434 nc++; 435 } 436 return nc; 437 } 438 439 /* 440 * NILFS2 needs clear_page_dirty() in the following two cases: 441 * 442 * 1) For B-tree node pages and data pages of DAT file, NILFS2 clears dirty 443 * flag of pages when it copies back pages from shadow cache to the 444 * original cache. 445 * 446 * 2) Some B-tree operations like insertion or deletion may dispose buffers 447 * in dirty state, and this needs to cancel the dirty state of their pages. 448 */ 449 int __nilfs_clear_page_dirty(struct page *page) 450 { 451 struct address_space *mapping = page->mapping; 452 453 if (mapping) { 454 xa_lock_irq(&mapping->i_pages); 455 if (test_bit(PG_dirty, &page->flags)) { 456 __xa_clear_mark(&mapping->i_pages, page_index(page), 457 PAGECACHE_TAG_DIRTY); 458 xa_unlock_irq(&mapping->i_pages); 459 return clear_page_dirty_for_io(page); 460 } 461 xa_unlock_irq(&mapping->i_pages); 462 return 0; 463 } 464 return TestClearPageDirty(page); 465 } 466 467 /** 468 * nilfs_find_uncommitted_extent - find extent of uncommitted data 469 * @inode: inode 470 * @start_blk: start block offset (in) 471 * @blkoff: start offset of the found extent (out) 472 * 473 * This function searches an extent of buffers marked "delayed" which 474 * starts from a block offset equal to or larger than @start_blk. If 475 * such an extent was found, this will store the start offset in 476 * @blkoff and return its length in blocks. Otherwise, zero is 477 * returned. 478 */ 479 unsigned long nilfs_find_uncommitted_extent(struct inode *inode, 480 sector_t start_blk, 481 sector_t *blkoff) 482 { 483 unsigned int i; 484 pgoff_t index; 485 unsigned int nblocks_in_page; 486 unsigned long length = 0; 487 sector_t b; 488 struct pagevec pvec; 489 struct page *page; 490 491 if (inode->i_mapping->nrpages == 0) 492 return 0; 493 494 index = start_blk >> (PAGE_SHIFT - inode->i_blkbits); 495 nblocks_in_page = 1U << (PAGE_SHIFT - inode->i_blkbits); 496 497 pagevec_init(&pvec); 498 499 repeat: 500 pvec.nr = find_get_pages_contig(inode->i_mapping, index, PAGEVEC_SIZE, 501 pvec.pages); 502 if (pvec.nr == 0) 503 return length; 504 505 if (length > 0 && pvec.pages[0]->index > index) 506 goto out; 507 508 b = pvec.pages[0]->index << (PAGE_SHIFT - inode->i_blkbits); 509 i = 0; 510 do { 511 page = pvec.pages[i]; 512 513 lock_page(page); 514 if (page_has_buffers(page)) { 515 struct buffer_head *bh, *head; 516 517 bh = head = page_buffers(page); 518 do { 519 if (b < start_blk) 520 continue; 521 if (buffer_delay(bh)) { 522 if (length == 0) 523 *blkoff = b; 524 length++; 525 } else if (length > 0) { 526 goto out_locked; 527 } 528 } while (++b, bh = bh->b_this_page, bh != head); 529 } else { 530 if (length > 0) 531 goto out_locked; 532 533 b += nblocks_in_page; 534 } 535 unlock_page(page); 536 537 } while (++i < pagevec_count(&pvec)); 538 539 index = page->index + 1; 540 pagevec_release(&pvec); 541 cond_resched(); 542 goto repeat; 543 544 out_locked: 545 unlock_page(page); 546 out: 547 pagevec_release(&pvec); 548 return length; 549 } 550