1 /* 2 * Copyright (c) 2000-2006 Silicon Graphics, Inc. 3 * All Rights Reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 #include "xfs.h" 19 #include <linux/stddef.h> 20 #include <linux/errno.h> 21 #include <linux/gfp.h> 22 #include <linux/pagemap.h> 23 #include <linux/init.h> 24 #include <linux/vmalloc.h> 25 #include <linux/bio.h> 26 #include <linux/sysctl.h> 27 #include <linux/proc_fs.h> 28 #include <linux/workqueue.h> 29 #include <linux/percpu.h> 30 #include <linux/blkdev.h> 31 #include <linux/hash.h> 32 #include <linux/kthread.h> 33 #include <linux/migrate.h> 34 #include <linux/backing-dev.h> 35 #include <linux/freezer.h> 36 37 #include "xfs_format.h" 38 #include "xfs_log_format.h" 39 #include "xfs_trans_resv.h" 40 #include "xfs_sb.h" 41 #include "xfs_mount.h" 42 #include "xfs_trace.h" 43 #include "xfs_log.h" 44 45 static kmem_zone_t *xfs_buf_zone; 46 47 #ifdef XFS_BUF_LOCK_TRACKING 48 # define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid) 49 # define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1) 50 # define XB_GET_OWNER(bp) ((bp)->b_last_holder) 51 #else 52 # define XB_SET_OWNER(bp) do { } while (0) 53 # define XB_CLEAR_OWNER(bp) do { } while (0) 54 # define XB_GET_OWNER(bp) do { } while (0) 55 #endif 56 57 #define xb_to_gfp(flags) \ 58 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : GFP_NOFS) | __GFP_NOWARN) 59 60 61 static inline int 62 xfs_buf_is_vmapped( 63 struct xfs_buf *bp) 64 { 65 /* 66 * Return true if the buffer is vmapped. 67 * 68 * b_addr is null if the buffer is not mapped, but the code is clever 69 * enough to know it doesn't have to map a single page, so the check has 70 * to be both for b_addr and bp->b_page_count > 1. 71 */ 72 return bp->b_addr && bp->b_page_count > 1; 73 } 74 75 static inline int 76 xfs_buf_vmap_len( 77 struct xfs_buf *bp) 78 { 79 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset; 80 } 81 82 /* 83 * Bump the I/O in flight count on the buftarg if we haven't yet done so for 84 * this buffer. The count is incremented once per buffer (per hold cycle) 85 * because the corresponding decrement is deferred to buffer release. Buffers 86 * can undergo I/O multiple times in a hold-release cycle and per buffer I/O 87 * tracking adds unnecessary overhead. This is used for sychronization purposes 88 * with unmount (see xfs_wait_buftarg()), so all we really need is a count of 89 * in-flight buffers. 90 * 91 * Buffers that are never released (e.g., superblock, iclog buffers) must set 92 * the XBF_NO_IOACCT flag before I/O submission. Otherwise, the buftarg count 93 * never reaches zero and unmount hangs indefinitely. 94 */ 95 static inline void 96 xfs_buf_ioacct_inc( 97 struct xfs_buf *bp) 98 { 99 if (bp->b_flags & (XBF_NO_IOACCT|_XBF_IN_FLIGHT)) 100 return; 101 102 ASSERT(bp->b_flags & XBF_ASYNC); 103 bp->b_flags |= _XBF_IN_FLIGHT; 104 percpu_counter_inc(&bp->b_target->bt_io_count); 105 } 106 107 /* 108 * Clear the in-flight state on a buffer about to be released to the LRU or 109 * freed and unaccount from the buftarg. 110 */ 111 static inline void 112 xfs_buf_ioacct_dec( 113 struct xfs_buf *bp) 114 { 115 if (!(bp->b_flags & _XBF_IN_FLIGHT)) 116 return; 117 118 bp->b_flags &= ~_XBF_IN_FLIGHT; 119 percpu_counter_dec(&bp->b_target->bt_io_count); 120 } 121 122 /* 123 * When we mark a buffer stale, we remove the buffer from the LRU and clear the 124 * b_lru_ref count so that the buffer is freed immediately when the buffer 125 * reference count falls to zero. If the buffer is already on the LRU, we need 126 * to remove the reference that LRU holds on the buffer. 127 * 128 * This prevents build-up of stale buffers on the LRU. 129 */ 130 void 131 xfs_buf_stale( 132 struct xfs_buf *bp) 133 { 134 ASSERT(xfs_buf_islocked(bp)); 135 136 bp->b_flags |= XBF_STALE; 137 138 /* 139 * Clear the delwri status so that a delwri queue walker will not 140 * flush this buffer to disk now that it is stale. The delwri queue has 141 * a reference to the buffer, so this is safe to do. 142 */ 143 bp->b_flags &= ~_XBF_DELWRI_Q; 144 145 /* 146 * Once the buffer is marked stale and unlocked, a subsequent lookup 147 * could reset b_flags. There is no guarantee that the buffer is 148 * unaccounted (released to LRU) before that occurs. Drop in-flight 149 * status now to preserve accounting consistency. 150 */ 151 xfs_buf_ioacct_dec(bp); 152 153 spin_lock(&bp->b_lock); 154 atomic_set(&bp->b_lru_ref, 0); 155 if (!(bp->b_state & XFS_BSTATE_DISPOSE) && 156 (list_lru_del(&bp->b_target->bt_lru, &bp->b_lru))) 157 atomic_dec(&bp->b_hold); 158 159 ASSERT(atomic_read(&bp->b_hold) >= 1); 160 spin_unlock(&bp->b_lock); 161 } 162 163 static int 164 xfs_buf_get_maps( 165 struct xfs_buf *bp, 166 int map_count) 167 { 168 ASSERT(bp->b_maps == NULL); 169 bp->b_map_count = map_count; 170 171 if (map_count == 1) { 172 bp->b_maps = &bp->__b_map; 173 return 0; 174 } 175 176 bp->b_maps = kmem_zalloc(map_count * sizeof(struct xfs_buf_map), 177 KM_NOFS); 178 if (!bp->b_maps) 179 return -ENOMEM; 180 return 0; 181 } 182 183 /* 184 * Frees b_pages if it was allocated. 185 */ 186 static void 187 xfs_buf_free_maps( 188 struct xfs_buf *bp) 189 { 190 if (bp->b_maps != &bp->__b_map) { 191 kmem_free(bp->b_maps); 192 bp->b_maps = NULL; 193 } 194 } 195 196 struct xfs_buf * 197 _xfs_buf_alloc( 198 struct xfs_buftarg *target, 199 struct xfs_buf_map *map, 200 int nmaps, 201 xfs_buf_flags_t flags) 202 { 203 struct xfs_buf *bp; 204 int error; 205 int i; 206 207 bp = kmem_zone_zalloc(xfs_buf_zone, KM_NOFS); 208 if (unlikely(!bp)) 209 return NULL; 210 211 /* 212 * We don't want certain flags to appear in b_flags unless they are 213 * specifically set by later operations on the buffer. 214 */ 215 flags &= ~(XBF_UNMAPPED | XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD); 216 217 atomic_set(&bp->b_hold, 1); 218 atomic_set(&bp->b_lru_ref, 1); 219 init_completion(&bp->b_iowait); 220 INIT_LIST_HEAD(&bp->b_lru); 221 INIT_LIST_HEAD(&bp->b_list); 222 sema_init(&bp->b_sema, 0); /* held, no waiters */ 223 spin_lock_init(&bp->b_lock); 224 XB_SET_OWNER(bp); 225 bp->b_target = target; 226 bp->b_flags = flags; 227 228 /* 229 * Set length and io_length to the same value initially. 230 * I/O routines should use io_length, which will be the same in 231 * most cases but may be reset (e.g. XFS recovery). 232 */ 233 error = xfs_buf_get_maps(bp, nmaps); 234 if (error) { 235 kmem_zone_free(xfs_buf_zone, bp); 236 return NULL; 237 } 238 239 bp->b_bn = map[0].bm_bn; 240 bp->b_length = 0; 241 for (i = 0; i < nmaps; i++) { 242 bp->b_maps[i].bm_bn = map[i].bm_bn; 243 bp->b_maps[i].bm_len = map[i].bm_len; 244 bp->b_length += map[i].bm_len; 245 } 246 bp->b_io_length = bp->b_length; 247 248 atomic_set(&bp->b_pin_count, 0); 249 init_waitqueue_head(&bp->b_waiters); 250 251 XFS_STATS_INC(target->bt_mount, xb_create); 252 trace_xfs_buf_init(bp, _RET_IP_); 253 254 return bp; 255 } 256 257 /* 258 * Allocate a page array capable of holding a specified number 259 * of pages, and point the page buf at it. 260 */ 261 STATIC int 262 _xfs_buf_get_pages( 263 xfs_buf_t *bp, 264 int page_count) 265 { 266 /* Make sure that we have a page list */ 267 if (bp->b_pages == NULL) { 268 bp->b_page_count = page_count; 269 if (page_count <= XB_PAGES) { 270 bp->b_pages = bp->b_page_array; 271 } else { 272 bp->b_pages = kmem_alloc(sizeof(struct page *) * 273 page_count, KM_NOFS); 274 if (bp->b_pages == NULL) 275 return -ENOMEM; 276 } 277 memset(bp->b_pages, 0, sizeof(struct page *) * page_count); 278 } 279 return 0; 280 } 281 282 /* 283 * Frees b_pages if it was allocated. 284 */ 285 STATIC void 286 _xfs_buf_free_pages( 287 xfs_buf_t *bp) 288 { 289 if (bp->b_pages != bp->b_page_array) { 290 kmem_free(bp->b_pages); 291 bp->b_pages = NULL; 292 } 293 } 294 295 /* 296 * Releases the specified buffer. 297 * 298 * The modification state of any associated pages is left unchanged. 299 * The buffer must not be on any hash - use xfs_buf_rele instead for 300 * hashed and refcounted buffers 301 */ 302 void 303 xfs_buf_free( 304 xfs_buf_t *bp) 305 { 306 trace_xfs_buf_free(bp, _RET_IP_); 307 308 ASSERT(list_empty(&bp->b_lru)); 309 310 if (bp->b_flags & _XBF_PAGES) { 311 uint i; 312 313 if (xfs_buf_is_vmapped(bp)) 314 vm_unmap_ram(bp->b_addr - bp->b_offset, 315 bp->b_page_count); 316 317 for (i = 0; i < bp->b_page_count; i++) { 318 struct page *page = bp->b_pages[i]; 319 320 __free_page(page); 321 } 322 } else if (bp->b_flags & _XBF_KMEM) 323 kmem_free(bp->b_addr); 324 _xfs_buf_free_pages(bp); 325 xfs_buf_free_maps(bp); 326 kmem_zone_free(xfs_buf_zone, bp); 327 } 328 329 /* 330 * Allocates all the pages for buffer in question and builds it's page list. 331 */ 332 STATIC int 333 xfs_buf_allocate_memory( 334 xfs_buf_t *bp, 335 uint flags) 336 { 337 size_t size; 338 size_t nbytes, offset; 339 gfp_t gfp_mask = xb_to_gfp(flags); 340 unsigned short page_count, i; 341 xfs_off_t start, end; 342 int error; 343 344 /* 345 * for buffers that are contained within a single page, just allocate 346 * the memory from the heap - there's no need for the complexity of 347 * page arrays to keep allocation down to order 0. 348 */ 349 size = BBTOB(bp->b_length); 350 if (size < PAGE_SIZE) { 351 bp->b_addr = kmem_alloc(size, KM_NOFS); 352 if (!bp->b_addr) { 353 /* low memory - use alloc_page loop instead */ 354 goto use_alloc_page; 355 } 356 357 if (((unsigned long)(bp->b_addr + size - 1) & PAGE_MASK) != 358 ((unsigned long)bp->b_addr & PAGE_MASK)) { 359 /* b_addr spans two pages - use alloc_page instead */ 360 kmem_free(bp->b_addr); 361 bp->b_addr = NULL; 362 goto use_alloc_page; 363 } 364 bp->b_offset = offset_in_page(bp->b_addr); 365 bp->b_pages = bp->b_page_array; 366 bp->b_pages[0] = virt_to_page(bp->b_addr); 367 bp->b_page_count = 1; 368 bp->b_flags |= _XBF_KMEM; 369 return 0; 370 } 371 372 use_alloc_page: 373 start = BBTOB(bp->b_maps[0].bm_bn) >> PAGE_SHIFT; 374 end = (BBTOB(bp->b_maps[0].bm_bn + bp->b_length) + PAGE_SIZE - 1) 375 >> PAGE_SHIFT; 376 page_count = end - start; 377 error = _xfs_buf_get_pages(bp, page_count); 378 if (unlikely(error)) 379 return error; 380 381 offset = bp->b_offset; 382 bp->b_flags |= _XBF_PAGES; 383 384 for (i = 0; i < bp->b_page_count; i++) { 385 struct page *page; 386 uint retries = 0; 387 retry: 388 page = alloc_page(gfp_mask); 389 if (unlikely(page == NULL)) { 390 if (flags & XBF_READ_AHEAD) { 391 bp->b_page_count = i; 392 error = -ENOMEM; 393 goto out_free_pages; 394 } 395 396 /* 397 * This could deadlock. 398 * 399 * But until all the XFS lowlevel code is revamped to 400 * handle buffer allocation failures we can't do much. 401 */ 402 if (!(++retries % 100)) 403 xfs_err(NULL, 404 "%s(%u) possible memory allocation deadlock in %s (mode:0x%x)", 405 current->comm, current->pid, 406 __func__, gfp_mask); 407 408 XFS_STATS_INC(bp->b_target->bt_mount, xb_page_retries); 409 congestion_wait(BLK_RW_ASYNC, HZ/50); 410 goto retry; 411 } 412 413 XFS_STATS_INC(bp->b_target->bt_mount, xb_page_found); 414 415 nbytes = min_t(size_t, size, PAGE_SIZE - offset); 416 size -= nbytes; 417 bp->b_pages[i] = page; 418 offset = 0; 419 } 420 return 0; 421 422 out_free_pages: 423 for (i = 0; i < bp->b_page_count; i++) 424 __free_page(bp->b_pages[i]); 425 return error; 426 } 427 428 /* 429 * Map buffer into kernel address-space if necessary. 430 */ 431 STATIC int 432 _xfs_buf_map_pages( 433 xfs_buf_t *bp, 434 uint flags) 435 { 436 ASSERT(bp->b_flags & _XBF_PAGES); 437 if (bp->b_page_count == 1) { 438 /* A single page buffer is always mappable */ 439 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset; 440 } else if (flags & XBF_UNMAPPED) { 441 bp->b_addr = NULL; 442 } else { 443 int retried = 0; 444 unsigned noio_flag; 445 446 /* 447 * vm_map_ram() will allocate auxillary structures (e.g. 448 * pagetables) with GFP_KERNEL, yet we are likely to be under 449 * GFP_NOFS context here. Hence we need to tell memory reclaim 450 * that we are in such a context via PF_MEMALLOC_NOIO to prevent 451 * memory reclaim re-entering the filesystem here and 452 * potentially deadlocking. 453 */ 454 noio_flag = memalloc_noio_save(); 455 do { 456 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count, 457 -1, PAGE_KERNEL); 458 if (bp->b_addr) 459 break; 460 vm_unmap_aliases(); 461 } while (retried++ <= 1); 462 memalloc_noio_restore(noio_flag); 463 464 if (!bp->b_addr) 465 return -ENOMEM; 466 bp->b_addr += bp->b_offset; 467 } 468 469 return 0; 470 } 471 472 /* 473 * Finding and Reading Buffers 474 */ 475 static int 476 _xfs_buf_obj_cmp( 477 struct rhashtable_compare_arg *arg, 478 const void *obj) 479 { 480 const struct xfs_buf_map *map = arg->key; 481 const struct xfs_buf *bp = obj; 482 483 /* 484 * The key hashing in the lookup path depends on the key being the 485 * first element of the compare_arg, make sure to assert this. 486 */ 487 BUILD_BUG_ON(offsetof(struct xfs_buf_map, bm_bn) != 0); 488 489 if (bp->b_bn != map->bm_bn) 490 return 1; 491 492 if (unlikely(bp->b_length != map->bm_len)) { 493 /* 494 * found a block number match. If the range doesn't 495 * match, the only way this is allowed is if the buffer 496 * in the cache is stale and the transaction that made 497 * it stale has not yet committed. i.e. we are 498 * reallocating a busy extent. Skip this buffer and 499 * continue searching for an exact match. 500 */ 501 ASSERT(bp->b_flags & XBF_STALE); 502 return 1; 503 } 504 return 0; 505 } 506 507 static const struct rhashtable_params xfs_buf_hash_params = { 508 .min_size = 32, /* empty AGs have minimal footprint */ 509 .nelem_hint = 16, 510 .key_len = sizeof(xfs_daddr_t), 511 .key_offset = offsetof(struct xfs_buf, b_bn), 512 .head_offset = offsetof(struct xfs_buf, b_rhash_head), 513 .automatic_shrinking = true, 514 .obj_cmpfn = _xfs_buf_obj_cmp, 515 }; 516 517 int 518 xfs_buf_hash_init( 519 struct xfs_perag *pag) 520 { 521 spin_lock_init(&pag->pag_buf_lock); 522 return rhashtable_init(&pag->pag_buf_hash, &xfs_buf_hash_params); 523 } 524 525 void 526 xfs_buf_hash_destroy( 527 struct xfs_perag *pag) 528 { 529 rhashtable_destroy(&pag->pag_buf_hash); 530 } 531 532 /* 533 * Look up, and creates if absent, a lockable buffer for 534 * a given range of an inode. The buffer is returned 535 * locked. No I/O is implied by this call. 536 */ 537 xfs_buf_t * 538 _xfs_buf_find( 539 struct xfs_buftarg *btp, 540 struct xfs_buf_map *map, 541 int nmaps, 542 xfs_buf_flags_t flags, 543 xfs_buf_t *new_bp) 544 { 545 struct xfs_perag *pag; 546 xfs_buf_t *bp; 547 struct xfs_buf_map cmap = { .bm_bn = map[0].bm_bn }; 548 xfs_daddr_t eofs; 549 int i; 550 551 for (i = 0; i < nmaps; i++) 552 cmap.bm_len += map[i].bm_len; 553 554 /* Check for IOs smaller than the sector size / not sector aligned */ 555 ASSERT(!(BBTOB(cmap.bm_len) < btp->bt_meta_sectorsize)); 556 ASSERT(!(BBTOB(cmap.bm_bn) & (xfs_off_t)btp->bt_meta_sectormask)); 557 558 /* 559 * Corrupted block numbers can get through to here, unfortunately, so we 560 * have to check that the buffer falls within the filesystem bounds. 561 */ 562 eofs = XFS_FSB_TO_BB(btp->bt_mount, btp->bt_mount->m_sb.sb_dblocks); 563 if (cmap.bm_bn < 0 || cmap.bm_bn >= eofs) { 564 /* 565 * XXX (dgc): we should really be returning -EFSCORRUPTED here, 566 * but none of the higher level infrastructure supports 567 * returning a specific error on buffer lookup failures. 568 */ 569 xfs_alert(btp->bt_mount, 570 "%s: Block out of range: block 0x%llx, EOFS 0x%llx ", 571 __func__, cmap.bm_bn, eofs); 572 WARN_ON(1); 573 return NULL; 574 } 575 576 pag = xfs_perag_get(btp->bt_mount, 577 xfs_daddr_to_agno(btp->bt_mount, cmap.bm_bn)); 578 579 spin_lock(&pag->pag_buf_lock); 580 bp = rhashtable_lookup_fast(&pag->pag_buf_hash, &cmap, 581 xfs_buf_hash_params); 582 if (bp) { 583 atomic_inc(&bp->b_hold); 584 goto found; 585 } 586 587 /* No match found */ 588 if (new_bp) { 589 /* the buffer keeps the perag reference until it is freed */ 590 new_bp->b_pag = pag; 591 rhashtable_insert_fast(&pag->pag_buf_hash, 592 &new_bp->b_rhash_head, 593 xfs_buf_hash_params); 594 spin_unlock(&pag->pag_buf_lock); 595 } else { 596 XFS_STATS_INC(btp->bt_mount, xb_miss_locked); 597 spin_unlock(&pag->pag_buf_lock); 598 xfs_perag_put(pag); 599 } 600 return new_bp; 601 602 found: 603 spin_unlock(&pag->pag_buf_lock); 604 xfs_perag_put(pag); 605 606 if (!xfs_buf_trylock(bp)) { 607 if (flags & XBF_TRYLOCK) { 608 xfs_buf_rele(bp); 609 XFS_STATS_INC(btp->bt_mount, xb_busy_locked); 610 return NULL; 611 } 612 xfs_buf_lock(bp); 613 XFS_STATS_INC(btp->bt_mount, xb_get_locked_waited); 614 } 615 616 /* 617 * if the buffer is stale, clear all the external state associated with 618 * it. We need to keep flags such as how we allocated the buffer memory 619 * intact here. 620 */ 621 if (bp->b_flags & XBF_STALE) { 622 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0); 623 ASSERT(bp->b_iodone == NULL); 624 bp->b_flags &= _XBF_KMEM | _XBF_PAGES; 625 bp->b_ops = NULL; 626 } 627 628 trace_xfs_buf_find(bp, flags, _RET_IP_); 629 XFS_STATS_INC(btp->bt_mount, xb_get_locked); 630 return bp; 631 } 632 633 /* 634 * Assembles a buffer covering the specified range. The code is optimised for 635 * cache hits, as metadata intensive workloads will see 3 orders of magnitude 636 * more hits than misses. 637 */ 638 struct xfs_buf * 639 xfs_buf_get_map( 640 struct xfs_buftarg *target, 641 struct xfs_buf_map *map, 642 int nmaps, 643 xfs_buf_flags_t flags) 644 { 645 struct xfs_buf *bp; 646 struct xfs_buf *new_bp; 647 int error = 0; 648 649 bp = _xfs_buf_find(target, map, nmaps, flags, NULL); 650 if (likely(bp)) 651 goto found; 652 653 new_bp = _xfs_buf_alloc(target, map, nmaps, flags); 654 if (unlikely(!new_bp)) 655 return NULL; 656 657 error = xfs_buf_allocate_memory(new_bp, flags); 658 if (error) { 659 xfs_buf_free(new_bp); 660 return NULL; 661 } 662 663 bp = _xfs_buf_find(target, map, nmaps, flags, new_bp); 664 if (!bp) { 665 xfs_buf_free(new_bp); 666 return NULL; 667 } 668 669 if (bp != new_bp) 670 xfs_buf_free(new_bp); 671 672 found: 673 if (!bp->b_addr) { 674 error = _xfs_buf_map_pages(bp, flags); 675 if (unlikely(error)) { 676 xfs_warn(target->bt_mount, 677 "%s: failed to map pagesn", __func__); 678 xfs_buf_relse(bp); 679 return NULL; 680 } 681 } 682 683 /* 684 * Clear b_error if this is a lookup from a caller that doesn't expect 685 * valid data to be found in the buffer. 686 */ 687 if (!(flags & XBF_READ)) 688 xfs_buf_ioerror(bp, 0); 689 690 XFS_STATS_INC(target->bt_mount, xb_get); 691 trace_xfs_buf_get(bp, flags, _RET_IP_); 692 return bp; 693 } 694 695 STATIC int 696 _xfs_buf_read( 697 xfs_buf_t *bp, 698 xfs_buf_flags_t flags) 699 { 700 ASSERT(!(flags & XBF_WRITE)); 701 ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL); 702 703 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD); 704 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD); 705 706 if (flags & XBF_ASYNC) { 707 xfs_buf_submit(bp); 708 return 0; 709 } 710 return xfs_buf_submit_wait(bp); 711 } 712 713 xfs_buf_t * 714 xfs_buf_read_map( 715 struct xfs_buftarg *target, 716 struct xfs_buf_map *map, 717 int nmaps, 718 xfs_buf_flags_t flags, 719 const struct xfs_buf_ops *ops) 720 { 721 struct xfs_buf *bp; 722 723 flags |= XBF_READ; 724 725 bp = xfs_buf_get_map(target, map, nmaps, flags); 726 if (bp) { 727 trace_xfs_buf_read(bp, flags, _RET_IP_); 728 729 if (!(bp->b_flags & XBF_DONE)) { 730 XFS_STATS_INC(target->bt_mount, xb_get_read); 731 bp->b_ops = ops; 732 _xfs_buf_read(bp, flags); 733 } else if (flags & XBF_ASYNC) { 734 /* 735 * Read ahead call which is already satisfied, 736 * drop the buffer 737 */ 738 xfs_buf_relse(bp); 739 return NULL; 740 } else { 741 /* We do not want read in the flags */ 742 bp->b_flags &= ~XBF_READ; 743 } 744 } 745 746 return bp; 747 } 748 749 /* 750 * If we are not low on memory then do the readahead in a deadlock 751 * safe manner. 752 */ 753 void 754 xfs_buf_readahead_map( 755 struct xfs_buftarg *target, 756 struct xfs_buf_map *map, 757 int nmaps, 758 const struct xfs_buf_ops *ops) 759 { 760 if (bdi_read_congested(target->bt_bdi)) 761 return; 762 763 xfs_buf_read_map(target, map, nmaps, 764 XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD, ops); 765 } 766 767 /* 768 * Read an uncached buffer from disk. Allocates and returns a locked 769 * buffer containing the disk contents or nothing. 770 */ 771 int 772 xfs_buf_read_uncached( 773 struct xfs_buftarg *target, 774 xfs_daddr_t daddr, 775 size_t numblks, 776 int flags, 777 struct xfs_buf **bpp, 778 const struct xfs_buf_ops *ops) 779 { 780 struct xfs_buf *bp; 781 782 *bpp = NULL; 783 784 bp = xfs_buf_get_uncached(target, numblks, flags); 785 if (!bp) 786 return -ENOMEM; 787 788 /* set up the buffer for a read IO */ 789 ASSERT(bp->b_map_count == 1); 790 bp->b_bn = XFS_BUF_DADDR_NULL; /* always null for uncached buffers */ 791 bp->b_maps[0].bm_bn = daddr; 792 bp->b_flags |= XBF_READ; 793 bp->b_ops = ops; 794 795 xfs_buf_submit_wait(bp); 796 if (bp->b_error) { 797 int error = bp->b_error; 798 xfs_buf_relse(bp); 799 return error; 800 } 801 802 *bpp = bp; 803 return 0; 804 } 805 806 /* 807 * Return a buffer allocated as an empty buffer and associated to external 808 * memory via xfs_buf_associate_memory() back to it's empty state. 809 */ 810 void 811 xfs_buf_set_empty( 812 struct xfs_buf *bp, 813 size_t numblks) 814 { 815 if (bp->b_pages) 816 _xfs_buf_free_pages(bp); 817 818 bp->b_pages = NULL; 819 bp->b_page_count = 0; 820 bp->b_addr = NULL; 821 bp->b_length = numblks; 822 bp->b_io_length = numblks; 823 824 ASSERT(bp->b_map_count == 1); 825 bp->b_bn = XFS_BUF_DADDR_NULL; 826 bp->b_maps[0].bm_bn = XFS_BUF_DADDR_NULL; 827 bp->b_maps[0].bm_len = bp->b_length; 828 } 829 830 static inline struct page * 831 mem_to_page( 832 void *addr) 833 { 834 if ((!is_vmalloc_addr(addr))) { 835 return virt_to_page(addr); 836 } else { 837 return vmalloc_to_page(addr); 838 } 839 } 840 841 int 842 xfs_buf_associate_memory( 843 xfs_buf_t *bp, 844 void *mem, 845 size_t len) 846 { 847 int rval; 848 int i = 0; 849 unsigned long pageaddr; 850 unsigned long offset; 851 size_t buflen; 852 int page_count; 853 854 pageaddr = (unsigned long)mem & PAGE_MASK; 855 offset = (unsigned long)mem - pageaddr; 856 buflen = PAGE_ALIGN(len + offset); 857 page_count = buflen >> PAGE_SHIFT; 858 859 /* Free any previous set of page pointers */ 860 if (bp->b_pages) 861 _xfs_buf_free_pages(bp); 862 863 bp->b_pages = NULL; 864 bp->b_addr = mem; 865 866 rval = _xfs_buf_get_pages(bp, page_count); 867 if (rval) 868 return rval; 869 870 bp->b_offset = offset; 871 872 for (i = 0; i < bp->b_page_count; i++) { 873 bp->b_pages[i] = mem_to_page((void *)pageaddr); 874 pageaddr += PAGE_SIZE; 875 } 876 877 bp->b_io_length = BTOBB(len); 878 bp->b_length = BTOBB(buflen); 879 880 return 0; 881 } 882 883 xfs_buf_t * 884 xfs_buf_get_uncached( 885 struct xfs_buftarg *target, 886 size_t numblks, 887 int flags) 888 { 889 unsigned long page_count; 890 int error, i; 891 struct xfs_buf *bp; 892 DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks); 893 894 /* flags might contain irrelevant bits, pass only what we care about */ 895 bp = _xfs_buf_alloc(target, &map, 1, flags & XBF_NO_IOACCT); 896 if (unlikely(bp == NULL)) 897 goto fail; 898 899 page_count = PAGE_ALIGN(numblks << BBSHIFT) >> PAGE_SHIFT; 900 error = _xfs_buf_get_pages(bp, page_count); 901 if (error) 902 goto fail_free_buf; 903 904 for (i = 0; i < page_count; i++) { 905 bp->b_pages[i] = alloc_page(xb_to_gfp(flags)); 906 if (!bp->b_pages[i]) 907 goto fail_free_mem; 908 } 909 bp->b_flags |= _XBF_PAGES; 910 911 error = _xfs_buf_map_pages(bp, 0); 912 if (unlikely(error)) { 913 xfs_warn(target->bt_mount, 914 "%s: failed to map pages", __func__); 915 goto fail_free_mem; 916 } 917 918 trace_xfs_buf_get_uncached(bp, _RET_IP_); 919 return bp; 920 921 fail_free_mem: 922 while (--i >= 0) 923 __free_page(bp->b_pages[i]); 924 _xfs_buf_free_pages(bp); 925 fail_free_buf: 926 xfs_buf_free_maps(bp); 927 kmem_zone_free(xfs_buf_zone, bp); 928 fail: 929 return NULL; 930 } 931 932 /* 933 * Increment reference count on buffer, to hold the buffer concurrently 934 * with another thread which may release (free) the buffer asynchronously. 935 * Must hold the buffer already to call this function. 936 */ 937 void 938 xfs_buf_hold( 939 xfs_buf_t *bp) 940 { 941 trace_xfs_buf_hold(bp, _RET_IP_); 942 atomic_inc(&bp->b_hold); 943 } 944 945 /* 946 * Release a hold on the specified buffer. If the hold count is 1, the buffer is 947 * placed on LRU or freed (depending on b_lru_ref). 948 */ 949 void 950 xfs_buf_rele( 951 xfs_buf_t *bp) 952 { 953 struct xfs_perag *pag = bp->b_pag; 954 bool release; 955 bool freebuf = false; 956 957 trace_xfs_buf_rele(bp, _RET_IP_); 958 959 if (!pag) { 960 ASSERT(list_empty(&bp->b_lru)); 961 if (atomic_dec_and_test(&bp->b_hold)) { 962 xfs_buf_ioacct_dec(bp); 963 xfs_buf_free(bp); 964 } 965 return; 966 } 967 968 ASSERT(atomic_read(&bp->b_hold) > 0); 969 970 release = atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock); 971 spin_lock(&bp->b_lock); 972 if (!release) { 973 /* 974 * Drop the in-flight state if the buffer is already on the LRU 975 * and it holds the only reference. This is racy because we 976 * haven't acquired the pag lock, but the use of _XBF_IN_FLIGHT 977 * ensures the decrement occurs only once per-buf. 978 */ 979 if ((atomic_read(&bp->b_hold) == 1) && !list_empty(&bp->b_lru)) 980 xfs_buf_ioacct_dec(bp); 981 goto out_unlock; 982 } 983 984 /* the last reference has been dropped ... */ 985 xfs_buf_ioacct_dec(bp); 986 if (!(bp->b_flags & XBF_STALE) && atomic_read(&bp->b_lru_ref)) { 987 /* 988 * If the buffer is added to the LRU take a new reference to the 989 * buffer for the LRU and clear the (now stale) dispose list 990 * state flag 991 */ 992 if (list_lru_add(&bp->b_target->bt_lru, &bp->b_lru)) { 993 bp->b_state &= ~XFS_BSTATE_DISPOSE; 994 atomic_inc(&bp->b_hold); 995 } 996 spin_unlock(&pag->pag_buf_lock); 997 } else { 998 /* 999 * most of the time buffers will already be removed from the 1000 * LRU, so optimise that case by checking for the 1001 * XFS_BSTATE_DISPOSE flag indicating the last list the buffer 1002 * was on was the disposal list 1003 */ 1004 if (!(bp->b_state & XFS_BSTATE_DISPOSE)) { 1005 list_lru_del(&bp->b_target->bt_lru, &bp->b_lru); 1006 } else { 1007 ASSERT(list_empty(&bp->b_lru)); 1008 } 1009 1010 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q)); 1011 rhashtable_remove_fast(&pag->pag_buf_hash, &bp->b_rhash_head, 1012 xfs_buf_hash_params); 1013 spin_unlock(&pag->pag_buf_lock); 1014 xfs_perag_put(pag); 1015 freebuf = true; 1016 } 1017 1018 out_unlock: 1019 spin_unlock(&bp->b_lock); 1020 1021 if (freebuf) 1022 xfs_buf_free(bp); 1023 } 1024 1025 1026 /* 1027 * Lock a buffer object, if it is not already locked. 1028 * 1029 * If we come across a stale, pinned, locked buffer, we know that we are 1030 * being asked to lock a buffer that has been reallocated. Because it is 1031 * pinned, we know that the log has not been pushed to disk and hence it 1032 * will still be locked. Rather than continuing to have trylock attempts 1033 * fail until someone else pushes the log, push it ourselves before 1034 * returning. This means that the xfsaild will not get stuck trying 1035 * to push on stale inode buffers. 1036 */ 1037 int 1038 xfs_buf_trylock( 1039 struct xfs_buf *bp) 1040 { 1041 int locked; 1042 1043 locked = down_trylock(&bp->b_sema) == 0; 1044 if (locked) { 1045 XB_SET_OWNER(bp); 1046 trace_xfs_buf_trylock(bp, _RET_IP_); 1047 } else { 1048 trace_xfs_buf_trylock_fail(bp, _RET_IP_); 1049 } 1050 return locked; 1051 } 1052 1053 /* 1054 * Lock a buffer object. 1055 * 1056 * If we come across a stale, pinned, locked buffer, we know that we 1057 * are being asked to lock a buffer that has been reallocated. Because 1058 * it is pinned, we know that the log has not been pushed to disk and 1059 * hence it will still be locked. Rather than sleeping until someone 1060 * else pushes the log, push it ourselves before trying to get the lock. 1061 */ 1062 void 1063 xfs_buf_lock( 1064 struct xfs_buf *bp) 1065 { 1066 trace_xfs_buf_lock(bp, _RET_IP_); 1067 1068 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE)) 1069 xfs_log_force(bp->b_target->bt_mount, 0); 1070 down(&bp->b_sema); 1071 XB_SET_OWNER(bp); 1072 1073 trace_xfs_buf_lock_done(bp, _RET_IP_); 1074 } 1075 1076 void 1077 xfs_buf_unlock( 1078 struct xfs_buf *bp) 1079 { 1080 XB_CLEAR_OWNER(bp); 1081 up(&bp->b_sema); 1082 1083 trace_xfs_buf_unlock(bp, _RET_IP_); 1084 } 1085 1086 STATIC void 1087 xfs_buf_wait_unpin( 1088 xfs_buf_t *bp) 1089 { 1090 DECLARE_WAITQUEUE (wait, current); 1091 1092 if (atomic_read(&bp->b_pin_count) == 0) 1093 return; 1094 1095 add_wait_queue(&bp->b_waiters, &wait); 1096 for (;;) { 1097 set_current_state(TASK_UNINTERRUPTIBLE); 1098 if (atomic_read(&bp->b_pin_count) == 0) 1099 break; 1100 io_schedule(); 1101 } 1102 remove_wait_queue(&bp->b_waiters, &wait); 1103 set_current_state(TASK_RUNNING); 1104 } 1105 1106 /* 1107 * Buffer Utility Routines 1108 */ 1109 1110 void 1111 xfs_buf_ioend( 1112 struct xfs_buf *bp) 1113 { 1114 bool read = bp->b_flags & XBF_READ; 1115 1116 trace_xfs_buf_iodone(bp, _RET_IP_); 1117 1118 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD); 1119 1120 /* 1121 * Pull in IO completion errors now. We are guaranteed to be running 1122 * single threaded, so we don't need the lock to read b_io_error. 1123 */ 1124 if (!bp->b_error && bp->b_io_error) 1125 xfs_buf_ioerror(bp, bp->b_io_error); 1126 1127 /* Only validate buffers that were read without errors */ 1128 if (read && !bp->b_error && bp->b_ops) { 1129 ASSERT(!bp->b_iodone); 1130 bp->b_ops->verify_read(bp); 1131 } 1132 1133 if (!bp->b_error) 1134 bp->b_flags |= XBF_DONE; 1135 1136 if (bp->b_iodone) 1137 (*(bp->b_iodone))(bp); 1138 else if (bp->b_flags & XBF_ASYNC) 1139 xfs_buf_relse(bp); 1140 else 1141 complete(&bp->b_iowait); 1142 } 1143 1144 static void 1145 xfs_buf_ioend_work( 1146 struct work_struct *work) 1147 { 1148 struct xfs_buf *bp = 1149 container_of(work, xfs_buf_t, b_ioend_work); 1150 1151 xfs_buf_ioend(bp); 1152 } 1153 1154 static void 1155 xfs_buf_ioend_async( 1156 struct xfs_buf *bp) 1157 { 1158 INIT_WORK(&bp->b_ioend_work, xfs_buf_ioend_work); 1159 queue_work(bp->b_ioend_wq, &bp->b_ioend_work); 1160 } 1161 1162 void 1163 xfs_buf_ioerror( 1164 xfs_buf_t *bp, 1165 int error) 1166 { 1167 ASSERT(error <= 0 && error >= -1000); 1168 bp->b_error = error; 1169 trace_xfs_buf_ioerror(bp, error, _RET_IP_); 1170 } 1171 1172 void 1173 xfs_buf_ioerror_alert( 1174 struct xfs_buf *bp, 1175 const char *func) 1176 { 1177 xfs_alert(bp->b_target->bt_mount, 1178 "metadata I/O error: block 0x%llx (\"%s\") error %d numblks %d", 1179 (__uint64_t)XFS_BUF_ADDR(bp), func, -bp->b_error, bp->b_length); 1180 } 1181 1182 int 1183 xfs_bwrite( 1184 struct xfs_buf *bp) 1185 { 1186 int error; 1187 1188 ASSERT(xfs_buf_islocked(bp)); 1189 1190 bp->b_flags |= XBF_WRITE; 1191 bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q | 1192 XBF_WRITE_FAIL | XBF_DONE); 1193 1194 error = xfs_buf_submit_wait(bp); 1195 if (error) { 1196 xfs_force_shutdown(bp->b_target->bt_mount, 1197 SHUTDOWN_META_IO_ERROR); 1198 } 1199 return error; 1200 } 1201 1202 static void 1203 xfs_buf_bio_end_io( 1204 struct bio *bio) 1205 { 1206 struct xfs_buf *bp = (struct xfs_buf *)bio->bi_private; 1207 1208 /* 1209 * don't overwrite existing errors - otherwise we can lose errors on 1210 * buffers that require multiple bios to complete. 1211 */ 1212 if (bio->bi_error) 1213 cmpxchg(&bp->b_io_error, 0, bio->bi_error); 1214 1215 if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ)) 1216 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp)); 1217 1218 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) 1219 xfs_buf_ioend_async(bp); 1220 bio_put(bio); 1221 } 1222 1223 static void 1224 xfs_buf_ioapply_map( 1225 struct xfs_buf *bp, 1226 int map, 1227 int *buf_offset, 1228 int *count, 1229 int op, 1230 int op_flags) 1231 { 1232 int page_index; 1233 int total_nr_pages = bp->b_page_count; 1234 int nr_pages; 1235 struct bio *bio; 1236 sector_t sector = bp->b_maps[map].bm_bn; 1237 int size; 1238 int offset; 1239 1240 total_nr_pages = bp->b_page_count; 1241 1242 /* skip the pages in the buffer before the start offset */ 1243 page_index = 0; 1244 offset = *buf_offset; 1245 while (offset >= PAGE_SIZE) { 1246 page_index++; 1247 offset -= PAGE_SIZE; 1248 } 1249 1250 /* 1251 * Limit the IO size to the length of the current vector, and update the 1252 * remaining IO count for the next time around. 1253 */ 1254 size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count); 1255 *count -= size; 1256 *buf_offset += size; 1257 1258 next_chunk: 1259 atomic_inc(&bp->b_io_remaining); 1260 nr_pages = min(total_nr_pages, BIO_MAX_PAGES); 1261 1262 bio = bio_alloc(GFP_NOIO, nr_pages); 1263 bio->bi_bdev = bp->b_target->bt_bdev; 1264 bio->bi_iter.bi_sector = sector; 1265 bio->bi_end_io = xfs_buf_bio_end_io; 1266 bio->bi_private = bp; 1267 bio_set_op_attrs(bio, op, op_flags); 1268 1269 for (; size && nr_pages; nr_pages--, page_index++) { 1270 int rbytes, nbytes = PAGE_SIZE - offset; 1271 1272 if (nbytes > size) 1273 nbytes = size; 1274 1275 rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes, 1276 offset); 1277 if (rbytes < nbytes) 1278 break; 1279 1280 offset = 0; 1281 sector += BTOBB(nbytes); 1282 size -= nbytes; 1283 total_nr_pages--; 1284 } 1285 1286 if (likely(bio->bi_iter.bi_size)) { 1287 if (xfs_buf_is_vmapped(bp)) { 1288 flush_kernel_vmap_range(bp->b_addr, 1289 xfs_buf_vmap_len(bp)); 1290 } 1291 submit_bio(bio); 1292 if (size) 1293 goto next_chunk; 1294 } else { 1295 /* 1296 * This is guaranteed not to be the last io reference count 1297 * because the caller (xfs_buf_submit) holds a count itself. 1298 */ 1299 atomic_dec(&bp->b_io_remaining); 1300 xfs_buf_ioerror(bp, -EIO); 1301 bio_put(bio); 1302 } 1303 1304 } 1305 1306 STATIC void 1307 _xfs_buf_ioapply( 1308 struct xfs_buf *bp) 1309 { 1310 struct blk_plug plug; 1311 int op; 1312 int op_flags = 0; 1313 int offset; 1314 int size; 1315 int i; 1316 1317 /* 1318 * Make sure we capture only current IO errors rather than stale errors 1319 * left over from previous use of the buffer (e.g. failed readahead). 1320 */ 1321 bp->b_error = 0; 1322 1323 /* 1324 * Initialize the I/O completion workqueue if we haven't yet or the 1325 * submitter has not opted to specify a custom one. 1326 */ 1327 if (!bp->b_ioend_wq) 1328 bp->b_ioend_wq = bp->b_target->bt_mount->m_buf_workqueue; 1329 1330 if (bp->b_flags & XBF_WRITE) { 1331 op = REQ_OP_WRITE; 1332 if (bp->b_flags & XBF_SYNCIO) 1333 op_flags = REQ_SYNC; 1334 if (bp->b_flags & XBF_FUA) 1335 op_flags |= REQ_FUA; 1336 if (bp->b_flags & XBF_FLUSH) 1337 op_flags |= REQ_PREFLUSH; 1338 1339 /* 1340 * Run the write verifier callback function if it exists. If 1341 * this function fails it will mark the buffer with an error and 1342 * the IO should not be dispatched. 1343 */ 1344 if (bp->b_ops) { 1345 bp->b_ops->verify_write(bp); 1346 if (bp->b_error) { 1347 xfs_force_shutdown(bp->b_target->bt_mount, 1348 SHUTDOWN_CORRUPT_INCORE); 1349 return; 1350 } 1351 } else if (bp->b_bn != XFS_BUF_DADDR_NULL) { 1352 struct xfs_mount *mp = bp->b_target->bt_mount; 1353 1354 /* 1355 * non-crc filesystems don't attach verifiers during 1356 * log recovery, so don't warn for such filesystems. 1357 */ 1358 if (xfs_sb_version_hascrc(&mp->m_sb)) { 1359 xfs_warn(mp, 1360 "%s: no ops on block 0x%llx/0x%x", 1361 __func__, bp->b_bn, bp->b_length); 1362 xfs_hex_dump(bp->b_addr, 64); 1363 dump_stack(); 1364 } 1365 } 1366 } else if (bp->b_flags & XBF_READ_AHEAD) { 1367 op = REQ_OP_READ; 1368 op_flags = REQ_RAHEAD; 1369 } else { 1370 op = REQ_OP_READ; 1371 } 1372 1373 /* we only use the buffer cache for meta-data */ 1374 op_flags |= REQ_META; 1375 1376 /* 1377 * Walk all the vectors issuing IO on them. Set up the initial offset 1378 * into the buffer and the desired IO size before we start - 1379 * _xfs_buf_ioapply_vec() will modify them appropriately for each 1380 * subsequent call. 1381 */ 1382 offset = bp->b_offset; 1383 size = BBTOB(bp->b_io_length); 1384 blk_start_plug(&plug); 1385 for (i = 0; i < bp->b_map_count; i++) { 1386 xfs_buf_ioapply_map(bp, i, &offset, &size, op, op_flags); 1387 if (bp->b_error) 1388 break; 1389 if (size <= 0) 1390 break; /* all done */ 1391 } 1392 blk_finish_plug(&plug); 1393 } 1394 1395 /* 1396 * Asynchronous IO submission path. This transfers the buffer lock ownership and 1397 * the current reference to the IO. It is not safe to reference the buffer after 1398 * a call to this function unless the caller holds an additional reference 1399 * itself. 1400 */ 1401 void 1402 xfs_buf_submit( 1403 struct xfs_buf *bp) 1404 { 1405 trace_xfs_buf_submit(bp, _RET_IP_); 1406 1407 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q)); 1408 ASSERT(bp->b_flags & XBF_ASYNC); 1409 1410 /* on shutdown we stale and complete the buffer immediately */ 1411 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) { 1412 xfs_buf_ioerror(bp, -EIO); 1413 bp->b_flags &= ~XBF_DONE; 1414 xfs_buf_stale(bp); 1415 xfs_buf_ioend(bp); 1416 return; 1417 } 1418 1419 if (bp->b_flags & XBF_WRITE) 1420 xfs_buf_wait_unpin(bp); 1421 1422 /* clear the internal error state to avoid spurious errors */ 1423 bp->b_io_error = 0; 1424 1425 /* 1426 * The caller's reference is released during I/O completion. 1427 * This occurs some time after the last b_io_remaining reference is 1428 * released, so after we drop our Io reference we have to have some 1429 * other reference to ensure the buffer doesn't go away from underneath 1430 * us. Take a direct reference to ensure we have safe access to the 1431 * buffer until we are finished with it. 1432 */ 1433 xfs_buf_hold(bp); 1434 1435 /* 1436 * Set the count to 1 initially, this will stop an I/O completion 1437 * callout which happens before we have started all the I/O from calling 1438 * xfs_buf_ioend too early. 1439 */ 1440 atomic_set(&bp->b_io_remaining, 1); 1441 xfs_buf_ioacct_inc(bp); 1442 _xfs_buf_ioapply(bp); 1443 1444 /* 1445 * If _xfs_buf_ioapply failed, we can get back here with only the IO 1446 * reference we took above. If we drop it to zero, run completion so 1447 * that we don't return to the caller with completion still pending. 1448 */ 1449 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) { 1450 if (bp->b_error) 1451 xfs_buf_ioend(bp); 1452 else 1453 xfs_buf_ioend_async(bp); 1454 } 1455 1456 xfs_buf_rele(bp); 1457 /* Note: it is not safe to reference bp now we've dropped our ref */ 1458 } 1459 1460 /* 1461 * Synchronous buffer IO submission path, read or write. 1462 */ 1463 int 1464 xfs_buf_submit_wait( 1465 struct xfs_buf *bp) 1466 { 1467 int error; 1468 1469 trace_xfs_buf_submit_wait(bp, _RET_IP_); 1470 1471 ASSERT(!(bp->b_flags & (_XBF_DELWRI_Q | XBF_ASYNC))); 1472 1473 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) { 1474 xfs_buf_ioerror(bp, -EIO); 1475 xfs_buf_stale(bp); 1476 bp->b_flags &= ~XBF_DONE; 1477 return -EIO; 1478 } 1479 1480 if (bp->b_flags & XBF_WRITE) 1481 xfs_buf_wait_unpin(bp); 1482 1483 /* clear the internal error state to avoid spurious errors */ 1484 bp->b_io_error = 0; 1485 1486 /* 1487 * For synchronous IO, the IO does not inherit the submitters reference 1488 * count, nor the buffer lock. Hence we cannot release the reference we 1489 * are about to take until we've waited for all IO completion to occur, 1490 * including any xfs_buf_ioend_async() work that may be pending. 1491 */ 1492 xfs_buf_hold(bp); 1493 1494 /* 1495 * Set the count to 1 initially, this will stop an I/O completion 1496 * callout which happens before we have started all the I/O from calling 1497 * xfs_buf_ioend too early. 1498 */ 1499 atomic_set(&bp->b_io_remaining, 1); 1500 _xfs_buf_ioapply(bp); 1501 1502 /* 1503 * make sure we run completion synchronously if it raced with us and is 1504 * already complete. 1505 */ 1506 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) 1507 xfs_buf_ioend(bp); 1508 1509 /* wait for completion before gathering the error from the buffer */ 1510 trace_xfs_buf_iowait(bp, _RET_IP_); 1511 wait_for_completion(&bp->b_iowait); 1512 trace_xfs_buf_iowait_done(bp, _RET_IP_); 1513 error = bp->b_error; 1514 1515 /* 1516 * all done now, we can release the hold that keeps the buffer 1517 * referenced for the entire IO. 1518 */ 1519 xfs_buf_rele(bp); 1520 return error; 1521 } 1522 1523 void * 1524 xfs_buf_offset( 1525 struct xfs_buf *bp, 1526 size_t offset) 1527 { 1528 struct page *page; 1529 1530 if (bp->b_addr) 1531 return bp->b_addr + offset; 1532 1533 offset += bp->b_offset; 1534 page = bp->b_pages[offset >> PAGE_SHIFT]; 1535 return page_address(page) + (offset & (PAGE_SIZE-1)); 1536 } 1537 1538 /* 1539 * Move data into or out of a buffer. 1540 */ 1541 void 1542 xfs_buf_iomove( 1543 xfs_buf_t *bp, /* buffer to process */ 1544 size_t boff, /* starting buffer offset */ 1545 size_t bsize, /* length to copy */ 1546 void *data, /* data address */ 1547 xfs_buf_rw_t mode) /* read/write/zero flag */ 1548 { 1549 size_t bend; 1550 1551 bend = boff + bsize; 1552 while (boff < bend) { 1553 struct page *page; 1554 int page_index, page_offset, csize; 1555 1556 page_index = (boff + bp->b_offset) >> PAGE_SHIFT; 1557 page_offset = (boff + bp->b_offset) & ~PAGE_MASK; 1558 page = bp->b_pages[page_index]; 1559 csize = min_t(size_t, PAGE_SIZE - page_offset, 1560 BBTOB(bp->b_io_length) - boff); 1561 1562 ASSERT((csize + page_offset) <= PAGE_SIZE); 1563 1564 switch (mode) { 1565 case XBRW_ZERO: 1566 memset(page_address(page) + page_offset, 0, csize); 1567 break; 1568 case XBRW_READ: 1569 memcpy(data, page_address(page) + page_offset, csize); 1570 break; 1571 case XBRW_WRITE: 1572 memcpy(page_address(page) + page_offset, data, csize); 1573 } 1574 1575 boff += csize; 1576 data += csize; 1577 } 1578 } 1579 1580 /* 1581 * Handling of buffer targets (buftargs). 1582 */ 1583 1584 /* 1585 * Wait for any bufs with callbacks that have been submitted but have not yet 1586 * returned. These buffers will have an elevated hold count, so wait on those 1587 * while freeing all the buffers only held by the LRU. 1588 */ 1589 static enum lru_status 1590 xfs_buftarg_wait_rele( 1591 struct list_head *item, 1592 struct list_lru_one *lru, 1593 spinlock_t *lru_lock, 1594 void *arg) 1595 1596 { 1597 struct xfs_buf *bp = container_of(item, struct xfs_buf, b_lru); 1598 struct list_head *dispose = arg; 1599 1600 if (atomic_read(&bp->b_hold) > 1) { 1601 /* need to wait, so skip it this pass */ 1602 trace_xfs_buf_wait_buftarg(bp, _RET_IP_); 1603 return LRU_SKIP; 1604 } 1605 if (!spin_trylock(&bp->b_lock)) 1606 return LRU_SKIP; 1607 1608 /* 1609 * clear the LRU reference count so the buffer doesn't get 1610 * ignored in xfs_buf_rele(). 1611 */ 1612 atomic_set(&bp->b_lru_ref, 0); 1613 bp->b_state |= XFS_BSTATE_DISPOSE; 1614 list_lru_isolate_move(lru, item, dispose); 1615 spin_unlock(&bp->b_lock); 1616 return LRU_REMOVED; 1617 } 1618 1619 void 1620 xfs_wait_buftarg( 1621 struct xfs_buftarg *btp) 1622 { 1623 LIST_HEAD(dispose); 1624 int loop = 0; 1625 1626 /* 1627 * First wait on the buftarg I/O count for all in-flight buffers to be 1628 * released. This is critical as new buffers do not make the LRU until 1629 * they are released. 1630 * 1631 * Next, flush the buffer workqueue to ensure all completion processing 1632 * has finished. Just waiting on buffer locks is not sufficient for 1633 * async IO as the reference count held over IO is not released until 1634 * after the buffer lock is dropped. Hence we need to ensure here that 1635 * all reference counts have been dropped before we start walking the 1636 * LRU list. 1637 */ 1638 while (percpu_counter_sum(&btp->bt_io_count)) 1639 delay(100); 1640 flush_workqueue(btp->bt_mount->m_buf_workqueue); 1641 1642 /* loop until there is nothing left on the lru list. */ 1643 while (list_lru_count(&btp->bt_lru)) { 1644 list_lru_walk(&btp->bt_lru, xfs_buftarg_wait_rele, 1645 &dispose, LONG_MAX); 1646 1647 while (!list_empty(&dispose)) { 1648 struct xfs_buf *bp; 1649 bp = list_first_entry(&dispose, struct xfs_buf, b_lru); 1650 list_del_init(&bp->b_lru); 1651 if (bp->b_flags & XBF_WRITE_FAIL) { 1652 xfs_alert(btp->bt_mount, 1653 "Corruption Alert: Buffer at block 0x%llx had permanent write failures!", 1654 (long long)bp->b_bn); 1655 xfs_alert(btp->bt_mount, 1656 "Please run xfs_repair to determine the extent of the problem."); 1657 } 1658 xfs_buf_rele(bp); 1659 } 1660 if (loop++ != 0) 1661 delay(100); 1662 } 1663 } 1664 1665 static enum lru_status 1666 xfs_buftarg_isolate( 1667 struct list_head *item, 1668 struct list_lru_one *lru, 1669 spinlock_t *lru_lock, 1670 void *arg) 1671 { 1672 struct xfs_buf *bp = container_of(item, struct xfs_buf, b_lru); 1673 struct list_head *dispose = arg; 1674 1675 /* 1676 * we are inverting the lru lock/bp->b_lock here, so use a trylock. 1677 * If we fail to get the lock, just skip it. 1678 */ 1679 if (!spin_trylock(&bp->b_lock)) 1680 return LRU_SKIP; 1681 /* 1682 * Decrement the b_lru_ref count unless the value is already 1683 * zero. If the value is already zero, we need to reclaim the 1684 * buffer, otherwise it gets another trip through the LRU. 1685 */ 1686 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) { 1687 spin_unlock(&bp->b_lock); 1688 return LRU_ROTATE; 1689 } 1690 1691 bp->b_state |= XFS_BSTATE_DISPOSE; 1692 list_lru_isolate_move(lru, item, dispose); 1693 spin_unlock(&bp->b_lock); 1694 return LRU_REMOVED; 1695 } 1696 1697 static unsigned long 1698 xfs_buftarg_shrink_scan( 1699 struct shrinker *shrink, 1700 struct shrink_control *sc) 1701 { 1702 struct xfs_buftarg *btp = container_of(shrink, 1703 struct xfs_buftarg, bt_shrinker); 1704 LIST_HEAD(dispose); 1705 unsigned long freed; 1706 1707 freed = list_lru_shrink_walk(&btp->bt_lru, sc, 1708 xfs_buftarg_isolate, &dispose); 1709 1710 while (!list_empty(&dispose)) { 1711 struct xfs_buf *bp; 1712 bp = list_first_entry(&dispose, struct xfs_buf, b_lru); 1713 list_del_init(&bp->b_lru); 1714 xfs_buf_rele(bp); 1715 } 1716 1717 return freed; 1718 } 1719 1720 static unsigned long 1721 xfs_buftarg_shrink_count( 1722 struct shrinker *shrink, 1723 struct shrink_control *sc) 1724 { 1725 struct xfs_buftarg *btp = container_of(shrink, 1726 struct xfs_buftarg, bt_shrinker); 1727 return list_lru_shrink_count(&btp->bt_lru, sc); 1728 } 1729 1730 void 1731 xfs_free_buftarg( 1732 struct xfs_mount *mp, 1733 struct xfs_buftarg *btp) 1734 { 1735 unregister_shrinker(&btp->bt_shrinker); 1736 ASSERT(percpu_counter_sum(&btp->bt_io_count) == 0); 1737 percpu_counter_destroy(&btp->bt_io_count); 1738 list_lru_destroy(&btp->bt_lru); 1739 1740 xfs_blkdev_issue_flush(btp); 1741 1742 kmem_free(btp); 1743 } 1744 1745 int 1746 xfs_setsize_buftarg( 1747 xfs_buftarg_t *btp, 1748 unsigned int sectorsize) 1749 { 1750 /* Set up metadata sector size info */ 1751 btp->bt_meta_sectorsize = sectorsize; 1752 btp->bt_meta_sectormask = sectorsize - 1; 1753 1754 if (set_blocksize(btp->bt_bdev, sectorsize)) { 1755 xfs_warn(btp->bt_mount, 1756 "Cannot set_blocksize to %u on device %pg", 1757 sectorsize, btp->bt_bdev); 1758 return -EINVAL; 1759 } 1760 1761 /* Set up device logical sector size mask */ 1762 btp->bt_logical_sectorsize = bdev_logical_block_size(btp->bt_bdev); 1763 btp->bt_logical_sectormask = bdev_logical_block_size(btp->bt_bdev) - 1; 1764 1765 return 0; 1766 } 1767 1768 /* 1769 * When allocating the initial buffer target we have not yet 1770 * read in the superblock, so don't know what sized sectors 1771 * are being used at this early stage. Play safe. 1772 */ 1773 STATIC int 1774 xfs_setsize_buftarg_early( 1775 xfs_buftarg_t *btp, 1776 struct block_device *bdev) 1777 { 1778 return xfs_setsize_buftarg(btp, bdev_logical_block_size(bdev)); 1779 } 1780 1781 xfs_buftarg_t * 1782 xfs_alloc_buftarg( 1783 struct xfs_mount *mp, 1784 struct block_device *bdev) 1785 { 1786 xfs_buftarg_t *btp; 1787 1788 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP | KM_NOFS); 1789 1790 btp->bt_mount = mp; 1791 btp->bt_dev = bdev->bd_dev; 1792 btp->bt_bdev = bdev; 1793 btp->bt_bdi = blk_get_backing_dev_info(bdev); 1794 1795 if (xfs_setsize_buftarg_early(btp, bdev)) 1796 goto error; 1797 1798 if (list_lru_init(&btp->bt_lru)) 1799 goto error; 1800 1801 if (percpu_counter_init(&btp->bt_io_count, 0, GFP_KERNEL)) 1802 goto error; 1803 1804 btp->bt_shrinker.count_objects = xfs_buftarg_shrink_count; 1805 btp->bt_shrinker.scan_objects = xfs_buftarg_shrink_scan; 1806 btp->bt_shrinker.seeks = DEFAULT_SEEKS; 1807 btp->bt_shrinker.flags = SHRINKER_NUMA_AWARE; 1808 register_shrinker(&btp->bt_shrinker); 1809 return btp; 1810 1811 error: 1812 kmem_free(btp); 1813 return NULL; 1814 } 1815 1816 /* 1817 * Add a buffer to the delayed write list. 1818 * 1819 * This queues a buffer for writeout if it hasn't already been. Note that 1820 * neither this routine nor the buffer list submission functions perform 1821 * any internal synchronization. It is expected that the lists are thread-local 1822 * to the callers. 1823 * 1824 * Returns true if we queued up the buffer, or false if it already had 1825 * been on the buffer list. 1826 */ 1827 bool 1828 xfs_buf_delwri_queue( 1829 struct xfs_buf *bp, 1830 struct list_head *list) 1831 { 1832 ASSERT(xfs_buf_islocked(bp)); 1833 ASSERT(!(bp->b_flags & XBF_READ)); 1834 1835 /* 1836 * If the buffer is already marked delwri it already is queued up 1837 * by someone else for imediate writeout. Just ignore it in that 1838 * case. 1839 */ 1840 if (bp->b_flags & _XBF_DELWRI_Q) { 1841 trace_xfs_buf_delwri_queued(bp, _RET_IP_); 1842 return false; 1843 } 1844 1845 trace_xfs_buf_delwri_queue(bp, _RET_IP_); 1846 1847 /* 1848 * If a buffer gets written out synchronously or marked stale while it 1849 * is on a delwri list we lazily remove it. To do this, the other party 1850 * clears the _XBF_DELWRI_Q flag but otherwise leaves the buffer alone. 1851 * It remains referenced and on the list. In a rare corner case it 1852 * might get readded to a delwri list after the synchronous writeout, in 1853 * which case we need just need to re-add the flag here. 1854 */ 1855 bp->b_flags |= _XBF_DELWRI_Q; 1856 if (list_empty(&bp->b_list)) { 1857 atomic_inc(&bp->b_hold); 1858 list_add_tail(&bp->b_list, list); 1859 } 1860 1861 return true; 1862 } 1863 1864 /* 1865 * Compare function is more complex than it needs to be because 1866 * the return value is only 32 bits and we are doing comparisons 1867 * on 64 bit values 1868 */ 1869 static int 1870 xfs_buf_cmp( 1871 void *priv, 1872 struct list_head *a, 1873 struct list_head *b) 1874 { 1875 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list); 1876 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list); 1877 xfs_daddr_t diff; 1878 1879 diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn; 1880 if (diff < 0) 1881 return -1; 1882 if (diff > 0) 1883 return 1; 1884 return 0; 1885 } 1886 1887 /* 1888 * submit buffers for write. 1889 * 1890 * When we have a large buffer list, we do not want to hold all the buffers 1891 * locked while we block on the request queue waiting for IO dispatch. To avoid 1892 * this problem, we lock and submit buffers in groups of 50, thereby minimising 1893 * the lock hold times for lists which may contain thousands of objects. 1894 * 1895 * To do this, we sort the buffer list before we walk the list to lock and 1896 * submit buffers, and we plug and unplug around each group of buffers we 1897 * submit. 1898 */ 1899 static int 1900 xfs_buf_delwri_submit_buffers( 1901 struct list_head *buffer_list, 1902 struct list_head *wait_list) 1903 { 1904 struct xfs_buf *bp, *n; 1905 LIST_HEAD (submit_list); 1906 int pinned = 0; 1907 struct blk_plug plug; 1908 1909 list_sort(NULL, buffer_list, xfs_buf_cmp); 1910 1911 blk_start_plug(&plug); 1912 list_for_each_entry_safe(bp, n, buffer_list, b_list) { 1913 if (!wait_list) { 1914 if (xfs_buf_ispinned(bp)) { 1915 pinned++; 1916 continue; 1917 } 1918 if (!xfs_buf_trylock(bp)) 1919 continue; 1920 } else { 1921 xfs_buf_lock(bp); 1922 } 1923 1924 /* 1925 * Someone else might have written the buffer synchronously or 1926 * marked it stale in the meantime. In that case only the 1927 * _XBF_DELWRI_Q flag got cleared, and we have to drop the 1928 * reference and remove it from the list here. 1929 */ 1930 if (!(bp->b_flags & _XBF_DELWRI_Q)) { 1931 list_del_init(&bp->b_list); 1932 xfs_buf_relse(bp); 1933 continue; 1934 } 1935 1936 trace_xfs_buf_delwri_split(bp, _RET_IP_); 1937 1938 /* 1939 * We do all IO submission async. This means if we need 1940 * to wait for IO completion we need to take an extra 1941 * reference so the buffer is still valid on the other 1942 * side. We need to move the buffer onto the io_list 1943 * at this point so the caller can still access it. 1944 */ 1945 bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_WRITE_FAIL); 1946 bp->b_flags |= XBF_WRITE | XBF_ASYNC; 1947 if (wait_list) { 1948 xfs_buf_hold(bp); 1949 list_move_tail(&bp->b_list, wait_list); 1950 } else 1951 list_del_init(&bp->b_list); 1952 1953 xfs_buf_submit(bp); 1954 } 1955 blk_finish_plug(&plug); 1956 1957 return pinned; 1958 } 1959 1960 /* 1961 * Write out a buffer list asynchronously. 1962 * 1963 * This will take the @buffer_list, write all non-locked and non-pinned buffers 1964 * out and not wait for I/O completion on any of the buffers. This interface 1965 * is only safely useable for callers that can track I/O completion by higher 1966 * level means, e.g. AIL pushing as the @buffer_list is consumed in this 1967 * function. 1968 */ 1969 int 1970 xfs_buf_delwri_submit_nowait( 1971 struct list_head *buffer_list) 1972 { 1973 return xfs_buf_delwri_submit_buffers(buffer_list, NULL); 1974 } 1975 1976 /* 1977 * Write out a buffer list synchronously. 1978 * 1979 * This will take the @buffer_list, write all buffers out and wait for I/O 1980 * completion on all of the buffers. @buffer_list is consumed by the function, 1981 * so callers must have some other way of tracking buffers if they require such 1982 * functionality. 1983 */ 1984 int 1985 xfs_buf_delwri_submit( 1986 struct list_head *buffer_list) 1987 { 1988 LIST_HEAD (wait_list); 1989 int error = 0, error2; 1990 struct xfs_buf *bp; 1991 1992 xfs_buf_delwri_submit_buffers(buffer_list, &wait_list); 1993 1994 /* Wait for IO to complete. */ 1995 while (!list_empty(&wait_list)) { 1996 bp = list_first_entry(&wait_list, struct xfs_buf, b_list); 1997 1998 list_del_init(&bp->b_list); 1999 2000 /* locking the buffer will wait for async IO completion. */ 2001 xfs_buf_lock(bp); 2002 error2 = bp->b_error; 2003 xfs_buf_relse(bp); 2004 if (!error) 2005 error = error2; 2006 } 2007 2008 return error; 2009 } 2010 2011 int __init 2012 xfs_buf_init(void) 2013 { 2014 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf", 2015 KM_ZONE_HWALIGN, NULL); 2016 if (!xfs_buf_zone) 2017 goto out; 2018 2019 return 0; 2020 2021 out: 2022 return -ENOMEM; 2023 } 2024 2025 void 2026 xfs_buf_terminate(void) 2027 { 2028 kmem_zone_destroy(xfs_buf_zone); 2029 } 2030