1 /* 2 * linux/fs/nfs/write.c 3 * 4 * Write file data over NFS. 5 * 6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de> 7 */ 8 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/mm.h> 12 #include <linux/pagemap.h> 13 #include <linux/file.h> 14 #include <linux/writeback.h> 15 #include <linux/swap.h> 16 #include <linux/migrate.h> 17 18 #include <linux/sunrpc/clnt.h> 19 #include <linux/nfs_fs.h> 20 #include <linux/nfs_mount.h> 21 #include <linux/nfs_page.h> 22 #include <linux/backing-dev.h> 23 #include <linux/export.h> 24 #include <linux/freezer.h> 25 #include <linux/wait.h> 26 27 #include <linux/uaccess.h> 28 29 #include "delegation.h" 30 #include "internal.h" 31 #include "iostat.h" 32 #include "nfs4_fs.h" 33 #include "fscache.h" 34 #include "pnfs.h" 35 36 #include "nfstrace.h" 37 38 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 39 40 #define MIN_POOL_WRITE (32) 41 #define MIN_POOL_COMMIT (4) 42 43 struct nfs_io_completion { 44 void (*complete)(void *data); 45 void *data; 46 struct kref refcount; 47 }; 48 49 /* 50 * Local function declarations 51 */ 52 static void nfs_redirty_request(struct nfs_page *req); 53 static const struct rpc_call_ops nfs_commit_ops; 54 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops; 55 static const struct nfs_commit_completion_ops nfs_commit_completion_ops; 56 static const struct nfs_rw_ops nfs_rw_write_ops; 57 static void nfs_clear_request_commit(struct nfs_page *req); 58 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 59 struct inode *inode); 60 static struct nfs_page * 61 nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi, 62 struct page *page); 63 64 static struct kmem_cache *nfs_wdata_cachep; 65 static mempool_t *nfs_wdata_mempool; 66 static struct kmem_cache *nfs_cdata_cachep; 67 static mempool_t *nfs_commit_mempool; 68 69 struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail) 70 { 71 struct nfs_commit_data *p; 72 73 if (never_fail) 74 p = mempool_alloc(nfs_commit_mempool, GFP_NOIO); 75 else { 76 /* It is OK to do some reclaim, not no safe to wait 77 * for anything to be returned to the pool. 78 * mempool_alloc() cannot handle that particular combination, 79 * so we need two separate attempts. 80 */ 81 p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT); 82 if (!p) 83 p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO | 84 __GFP_NOWARN | __GFP_NORETRY); 85 if (!p) 86 return NULL; 87 } 88 89 memset(p, 0, sizeof(*p)); 90 INIT_LIST_HEAD(&p->pages); 91 return p; 92 } 93 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); 94 95 void nfs_commit_free(struct nfs_commit_data *p) 96 { 97 mempool_free(p, nfs_commit_mempool); 98 } 99 EXPORT_SYMBOL_GPL(nfs_commit_free); 100 101 static struct nfs_pgio_header *nfs_writehdr_alloc(void) 102 { 103 struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO); 104 105 if (p) { 106 memset(p, 0, sizeof(*p)); 107 p->rw_mode = FMODE_WRITE; 108 } 109 return p; 110 } 111 112 static void nfs_writehdr_free(struct nfs_pgio_header *hdr) 113 { 114 mempool_free(hdr, nfs_wdata_mempool); 115 } 116 117 static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags) 118 { 119 return kmalloc(sizeof(struct nfs_io_completion), gfp_flags); 120 } 121 122 static void nfs_io_completion_init(struct nfs_io_completion *ioc, 123 void (*complete)(void *), void *data) 124 { 125 ioc->complete = complete; 126 ioc->data = data; 127 kref_init(&ioc->refcount); 128 } 129 130 static void nfs_io_completion_release(struct kref *kref) 131 { 132 struct nfs_io_completion *ioc = container_of(kref, 133 struct nfs_io_completion, refcount); 134 ioc->complete(ioc->data); 135 kfree(ioc); 136 } 137 138 static void nfs_io_completion_get(struct nfs_io_completion *ioc) 139 { 140 if (ioc != NULL) 141 kref_get(&ioc->refcount); 142 } 143 144 static void nfs_io_completion_put(struct nfs_io_completion *ioc) 145 { 146 if (ioc != NULL) 147 kref_put(&ioc->refcount, nfs_io_completion_release); 148 } 149 150 static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error) 151 { 152 ctx->error = error; 153 smp_wmb(); 154 set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags); 155 } 156 157 /* 158 * nfs_page_find_head_request_locked - find head request associated with @page 159 * 160 * must be called while holding the inode lock. 161 * 162 * returns matching head request with reference held, or NULL if not found. 163 */ 164 static struct nfs_page * 165 nfs_page_find_head_request_locked(struct nfs_inode *nfsi, struct page *page) 166 { 167 struct nfs_page *req = NULL; 168 169 if (PagePrivate(page)) 170 req = (struct nfs_page *)page_private(page); 171 else if (unlikely(PageSwapCache(page))) 172 req = nfs_page_search_commits_for_head_request_locked(nfsi, 173 page); 174 175 if (req) { 176 WARN_ON_ONCE(req->wb_head != req); 177 kref_get(&req->wb_kref); 178 } 179 180 return req; 181 } 182 183 /* 184 * nfs_page_find_head_request - find head request associated with @page 185 * 186 * returns matching head request with reference held, or NULL if not found. 187 */ 188 static struct nfs_page *nfs_page_find_head_request(struct page *page) 189 { 190 struct inode *inode = page_file_mapping(page)->host; 191 struct nfs_page *req = NULL; 192 193 spin_lock(&inode->i_lock); 194 req = nfs_page_find_head_request_locked(NFS_I(inode), page); 195 spin_unlock(&inode->i_lock); 196 return req; 197 } 198 199 /* Adjust the file length if we're writing beyond the end */ 200 static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count) 201 { 202 struct inode *inode = page_file_mapping(page)->host; 203 loff_t end, i_size; 204 pgoff_t end_index; 205 206 spin_lock(&inode->i_lock); 207 i_size = i_size_read(inode); 208 end_index = (i_size - 1) >> PAGE_SHIFT; 209 if (i_size > 0 && page_index(page) < end_index) 210 goto out; 211 end = page_file_offset(page) + ((loff_t)offset+count); 212 if (i_size >= end) 213 goto out; 214 i_size_write(inode, end); 215 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); 216 out: 217 spin_unlock(&inode->i_lock); 218 } 219 220 /* A writeback failed: mark the page as bad, and invalidate the page cache */ 221 static void nfs_set_pageerror(struct page *page) 222 { 223 nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page)); 224 } 225 226 /* 227 * nfs_page_group_search_locked 228 * @head - head request of page group 229 * @page_offset - offset into page 230 * 231 * Search page group with head @head to find a request that contains the 232 * page offset @page_offset. 233 * 234 * Returns a pointer to the first matching nfs request, or NULL if no 235 * match is found. 236 * 237 * Must be called with the page group lock held 238 */ 239 static struct nfs_page * 240 nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset) 241 { 242 struct nfs_page *req; 243 244 WARN_ON_ONCE(head != head->wb_head); 245 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_head->wb_flags)); 246 247 req = head; 248 do { 249 if (page_offset >= req->wb_pgbase && 250 page_offset < (req->wb_pgbase + req->wb_bytes)) 251 return req; 252 253 req = req->wb_this_page; 254 } while (req != head); 255 256 return NULL; 257 } 258 259 /* 260 * nfs_page_group_covers_page 261 * @head - head request of page group 262 * 263 * Return true if the page group with head @head covers the whole page, 264 * returns false otherwise 265 */ 266 static bool nfs_page_group_covers_page(struct nfs_page *req) 267 { 268 struct nfs_page *tmp; 269 unsigned int pos = 0; 270 unsigned int len = nfs_page_length(req->wb_page); 271 272 nfs_page_group_lock(req, false); 273 274 do { 275 tmp = nfs_page_group_search_locked(req->wb_head, pos); 276 if (tmp) { 277 /* no way this should happen */ 278 WARN_ON_ONCE(tmp->wb_pgbase != pos); 279 pos += tmp->wb_bytes - (pos - tmp->wb_pgbase); 280 } 281 } while (tmp && pos < len); 282 283 nfs_page_group_unlock(req); 284 WARN_ON_ONCE(pos > len); 285 return pos == len; 286 } 287 288 /* We can set the PG_uptodate flag if we see that a write request 289 * covers the full page. 290 */ 291 static void nfs_mark_uptodate(struct nfs_page *req) 292 { 293 if (PageUptodate(req->wb_page)) 294 return; 295 if (!nfs_page_group_covers_page(req)) 296 return; 297 SetPageUptodate(req->wb_page); 298 } 299 300 static int wb_priority(struct writeback_control *wbc) 301 { 302 int ret = 0; 303 304 if (wbc->sync_mode == WB_SYNC_ALL) 305 ret = FLUSH_COND_STABLE; 306 return ret; 307 } 308 309 /* 310 * NFS congestion control 311 */ 312 313 int nfs_congestion_kb; 314 315 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10)) 316 #define NFS_CONGESTION_OFF_THRESH \ 317 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) 318 319 static void nfs_set_page_writeback(struct page *page) 320 { 321 struct inode *inode = page_file_mapping(page)->host; 322 struct nfs_server *nfss = NFS_SERVER(inode); 323 int ret = test_set_page_writeback(page); 324 325 WARN_ON_ONCE(ret != 0); 326 327 if (atomic_long_inc_return(&nfss->writeback) > 328 NFS_CONGESTION_ON_THRESH) 329 set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 330 } 331 332 static void nfs_end_page_writeback(struct nfs_page *req) 333 { 334 struct inode *inode = page_file_mapping(req->wb_page)->host; 335 struct nfs_server *nfss = NFS_SERVER(inode); 336 337 if (!nfs_page_group_sync_on_bit(req, PG_WB_END)) 338 return; 339 340 end_page_writeback(req->wb_page); 341 if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 342 clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 343 } 344 345 346 /* nfs_page_group_clear_bits 347 * @req - an nfs request 348 * clears all page group related bits from @req 349 */ 350 static void 351 nfs_page_group_clear_bits(struct nfs_page *req) 352 { 353 clear_bit(PG_TEARDOWN, &req->wb_flags); 354 clear_bit(PG_UNLOCKPAGE, &req->wb_flags); 355 clear_bit(PG_UPTODATE, &req->wb_flags); 356 clear_bit(PG_WB_END, &req->wb_flags); 357 clear_bit(PG_REMOVE, &req->wb_flags); 358 } 359 360 361 /* 362 * nfs_unroll_locks_and_wait - unlock all newly locked reqs and wait on @req 363 * 364 * this is a helper function for nfs_lock_and_join_requests 365 * 366 * @inode - inode associated with request page group, must be holding inode lock 367 * @head - head request of page group, must be holding head lock 368 * @req - request that couldn't lock and needs to wait on the req bit lock 369 * @nonblock - if true, don't actually wait 370 * 371 * NOTE: this must be called holding page_group bit lock and inode spin lock 372 * and BOTH will be released before returning. 373 * 374 * returns 0 on success, < 0 on error. 375 */ 376 static int 377 nfs_unroll_locks_and_wait(struct inode *inode, struct nfs_page *head, 378 struct nfs_page *req, bool nonblock) 379 __releases(&inode->i_lock) 380 { 381 struct nfs_page *tmp; 382 int ret; 383 384 /* relinquish all the locks successfully grabbed this run */ 385 for (tmp = head ; tmp != req; tmp = tmp->wb_this_page) 386 nfs_unlock_request(tmp); 387 388 WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags)); 389 390 /* grab a ref on the request that will be waited on */ 391 kref_get(&req->wb_kref); 392 393 nfs_page_group_unlock(head); 394 spin_unlock(&inode->i_lock); 395 396 /* release ref from nfs_page_find_head_request_locked */ 397 nfs_release_request(head); 398 399 if (!nonblock) 400 ret = nfs_wait_on_request(req); 401 else 402 ret = -EAGAIN; 403 nfs_release_request(req); 404 405 return ret; 406 } 407 408 /* 409 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests 410 * 411 * @destroy_list - request list (using wb_this_page) terminated by @old_head 412 * @old_head - the old head of the list 413 * 414 * All subrequests must be locked and removed from all lists, so at this point 415 * they are only "active" in this function, and possibly in nfs_wait_on_request 416 * with a reference held by some other context. 417 */ 418 static void 419 nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, 420 struct nfs_page *old_head) 421 { 422 while (destroy_list) { 423 struct nfs_page *subreq = destroy_list; 424 425 destroy_list = (subreq->wb_this_page == old_head) ? 426 NULL : subreq->wb_this_page; 427 428 WARN_ON_ONCE(old_head != subreq->wb_head); 429 430 /* make sure old group is not used */ 431 subreq->wb_head = subreq; 432 subreq->wb_this_page = subreq; 433 434 /* subreq is now totally disconnected from page group or any 435 * write / commit lists. last chance to wake any waiters */ 436 nfs_unlock_request(subreq); 437 438 if (!test_bit(PG_TEARDOWN, &subreq->wb_flags)) { 439 /* release ref on old head request */ 440 nfs_release_request(old_head); 441 442 nfs_page_group_clear_bits(subreq); 443 444 /* release the PG_INODE_REF reference */ 445 if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) 446 nfs_release_request(subreq); 447 else 448 WARN_ON_ONCE(1); 449 } else { 450 WARN_ON_ONCE(test_bit(PG_CLEAN, &subreq->wb_flags)); 451 /* zombie requests have already released the last 452 * reference and were waiting on the rest of the 453 * group to complete. Since it's no longer part of a 454 * group, simply free the request */ 455 nfs_page_group_clear_bits(subreq); 456 nfs_free_request(subreq); 457 } 458 } 459 } 460 461 /* 462 * nfs_lock_and_join_requests - join all subreqs to the head req and return 463 * a locked reference, cancelling any pending 464 * operations for this page. 465 * 466 * @page - the page used to lookup the "page group" of nfs_page structures 467 * @nonblock - if true, don't block waiting for request locks 468 * 469 * This function joins all sub requests to the head request by first 470 * locking all requests in the group, cancelling any pending operations 471 * and finally updating the head request to cover the whole range covered by 472 * the (former) group. All subrequests are removed from any write or commit 473 * lists, unlinked from the group and destroyed. 474 * 475 * Returns a locked, referenced pointer to the head request - which after 476 * this call is guaranteed to be the only request associated with the page. 477 * Returns NULL if no requests are found for @page, or a ERR_PTR if an 478 * error was encountered. 479 */ 480 static struct nfs_page * 481 nfs_lock_and_join_requests(struct page *page, bool nonblock) 482 { 483 struct inode *inode = page_file_mapping(page)->host; 484 struct nfs_page *head, *subreq; 485 struct nfs_page *destroy_list = NULL; 486 unsigned int total_bytes; 487 int ret; 488 489 try_again: 490 total_bytes = 0; 491 492 WARN_ON_ONCE(destroy_list); 493 494 spin_lock(&inode->i_lock); 495 496 /* 497 * A reference is taken only on the head request which acts as a 498 * reference to the whole page group - the group will not be destroyed 499 * until the head reference is released. 500 */ 501 head = nfs_page_find_head_request_locked(NFS_I(inode), page); 502 503 if (!head) { 504 spin_unlock(&inode->i_lock); 505 return NULL; 506 } 507 508 /* holding inode lock, so always make a non-blocking call to try the 509 * page group lock */ 510 ret = nfs_page_group_lock(head, true); 511 if (ret < 0) { 512 spin_unlock(&inode->i_lock); 513 514 if (!nonblock && ret == -EAGAIN) { 515 nfs_page_group_lock_wait(head); 516 nfs_release_request(head); 517 goto try_again; 518 } 519 520 nfs_release_request(head); 521 return ERR_PTR(ret); 522 } 523 524 /* lock each request in the page group */ 525 subreq = head; 526 do { 527 /* 528 * Subrequests are always contiguous, non overlapping 529 * and in order - but may be repeated (mirrored writes). 530 */ 531 if (subreq->wb_offset == (head->wb_offset + total_bytes)) { 532 /* keep track of how many bytes this group covers */ 533 total_bytes += subreq->wb_bytes; 534 } else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset || 535 ((subreq->wb_offset + subreq->wb_bytes) > 536 (head->wb_offset + total_bytes)))) { 537 nfs_page_group_unlock(head); 538 spin_unlock(&inode->i_lock); 539 return ERR_PTR(-EIO); 540 } 541 542 if (!nfs_lock_request(subreq)) { 543 /* releases page group bit lock and 544 * inode spin lock and all references */ 545 ret = nfs_unroll_locks_and_wait(inode, head, 546 subreq, nonblock); 547 548 if (ret == 0) 549 goto try_again; 550 551 return ERR_PTR(ret); 552 } 553 554 subreq = subreq->wb_this_page; 555 } while (subreq != head); 556 557 /* Now that all requests are locked, make sure they aren't on any list. 558 * Commit list removal accounting is done after locks are dropped */ 559 subreq = head; 560 do { 561 nfs_clear_request_commit(subreq); 562 subreq = subreq->wb_this_page; 563 } while (subreq != head); 564 565 /* unlink subrequests from head, destroy them later */ 566 if (head->wb_this_page != head) { 567 /* destroy list will be terminated by head */ 568 destroy_list = head->wb_this_page; 569 head->wb_this_page = head; 570 571 /* change head request to cover whole range that 572 * the former page group covered */ 573 head->wb_bytes = total_bytes; 574 } 575 576 /* 577 * prepare head request to be added to new pgio descriptor 578 */ 579 nfs_page_group_clear_bits(head); 580 581 /* 582 * some part of the group was still on the inode list - otherwise 583 * the group wouldn't be involved in async write. 584 * grab a reference for the head request, iff it needs one. 585 */ 586 if (!test_and_set_bit(PG_INODE_REF, &head->wb_flags)) 587 kref_get(&head->wb_kref); 588 589 nfs_page_group_unlock(head); 590 591 /* drop lock to clean uprequests on destroy list */ 592 spin_unlock(&inode->i_lock); 593 594 nfs_destroy_unlinked_subrequests(destroy_list, head); 595 596 /* still holds ref on head from nfs_page_find_head_request_locked 597 * and still has lock on head from lock loop */ 598 return head; 599 } 600 601 static void nfs_write_error_remove_page(struct nfs_page *req) 602 { 603 nfs_unlock_request(req); 604 nfs_end_page_writeback(req); 605 generic_error_remove_page(page_file_mapping(req->wb_page), 606 req->wb_page); 607 nfs_release_request(req); 608 } 609 610 static bool 611 nfs_error_is_fatal_on_server(int err) 612 { 613 switch (err) { 614 case 0: 615 case -ERESTARTSYS: 616 case -EINTR: 617 return false; 618 } 619 return nfs_error_is_fatal(err); 620 } 621 622 /* 623 * Find an associated nfs write request, and prepare to flush it out 624 * May return an error if the user signalled nfs_wait_on_request(). 625 */ 626 static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio, 627 struct page *page, bool nonblock) 628 { 629 struct nfs_page *req; 630 int ret = 0; 631 632 req = nfs_lock_and_join_requests(page, nonblock); 633 if (!req) 634 goto out; 635 ret = PTR_ERR(req); 636 if (IS_ERR(req)) 637 goto out; 638 639 nfs_set_page_writeback(page); 640 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags)); 641 642 ret = 0; 643 /* If there is a fatal error that covers this write, just exit */ 644 if (nfs_error_is_fatal_on_server(req->wb_context->error)) 645 goto out_launder; 646 647 if (!nfs_pageio_add_request(pgio, req)) { 648 ret = pgio->pg_error; 649 /* 650 * Remove the problematic req upon fatal errors on the server 651 */ 652 if (nfs_error_is_fatal(ret)) { 653 nfs_context_set_write_error(req->wb_context, ret); 654 if (nfs_error_is_fatal_on_server(ret)) 655 goto out_launder; 656 } 657 nfs_redirty_request(req); 658 ret = -EAGAIN; 659 } else 660 nfs_add_stats(page_file_mapping(page)->host, 661 NFSIOS_WRITEPAGES, 1); 662 out: 663 return ret; 664 out_launder: 665 nfs_write_error_remove_page(req); 666 return ret; 667 } 668 669 static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, 670 struct nfs_pageio_descriptor *pgio) 671 { 672 int ret; 673 674 nfs_pageio_cond_complete(pgio, page_index(page)); 675 ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE); 676 if (ret == -EAGAIN) { 677 redirty_page_for_writepage(wbc, page); 678 ret = 0; 679 } 680 return ret; 681 } 682 683 /* 684 * Write an mmapped page to the server. 685 */ 686 static int nfs_writepage_locked(struct page *page, 687 struct writeback_control *wbc) 688 { 689 struct nfs_pageio_descriptor pgio; 690 struct inode *inode = page_file_mapping(page)->host; 691 int err; 692 693 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE); 694 nfs_pageio_init_write(&pgio, inode, 0, 695 false, &nfs_async_write_completion_ops); 696 err = nfs_do_writepage(page, wbc, &pgio); 697 nfs_pageio_complete(&pgio); 698 if (err < 0) 699 return err; 700 if (pgio.pg_error < 0) 701 return pgio.pg_error; 702 return 0; 703 } 704 705 int nfs_writepage(struct page *page, struct writeback_control *wbc) 706 { 707 int ret; 708 709 ret = nfs_writepage_locked(page, wbc); 710 unlock_page(page); 711 return ret; 712 } 713 714 static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data) 715 { 716 int ret; 717 718 ret = nfs_do_writepage(page, wbc, data); 719 unlock_page(page); 720 return ret; 721 } 722 723 static void nfs_io_completion_commit(void *inode) 724 { 725 nfs_commit_inode(inode, 0); 726 } 727 728 int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) 729 { 730 struct inode *inode = mapping->host; 731 struct nfs_pageio_descriptor pgio; 732 struct nfs_io_completion *ioc = nfs_io_completion_alloc(GFP_NOFS); 733 int err; 734 735 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES); 736 737 if (ioc) 738 nfs_io_completion_init(ioc, nfs_io_completion_commit, inode); 739 740 nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false, 741 &nfs_async_write_completion_ops); 742 pgio.pg_io_completion = ioc; 743 err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio); 744 nfs_pageio_complete(&pgio); 745 nfs_io_completion_put(ioc); 746 747 if (err < 0) 748 goto out_err; 749 err = pgio.pg_error; 750 if (err < 0) 751 goto out_err; 752 return 0; 753 out_err: 754 return err; 755 } 756 757 /* 758 * Insert a write request into an inode 759 */ 760 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req) 761 { 762 struct nfs_inode *nfsi = NFS_I(inode); 763 764 WARN_ON_ONCE(req->wb_this_page != req); 765 766 /* Lock the request! */ 767 nfs_lock_request(req); 768 769 spin_lock(&inode->i_lock); 770 if (!nfsi->nrequests && 771 NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) 772 inode->i_version++; 773 /* 774 * Swap-space should not get truncated. Hence no need to plug the race 775 * with invalidate/truncate. 776 */ 777 if (likely(!PageSwapCache(req->wb_page))) { 778 set_bit(PG_MAPPED, &req->wb_flags); 779 SetPagePrivate(req->wb_page); 780 set_page_private(req->wb_page, (unsigned long)req); 781 } 782 nfsi->nrequests++; 783 /* this a head request for a page group - mark it as having an 784 * extra reference so sub groups can follow suit. 785 * This flag also informs pgio layer when to bump nrequests when 786 * adding subrequests. */ 787 WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags)); 788 kref_get(&req->wb_kref); 789 spin_unlock(&inode->i_lock); 790 } 791 792 /* 793 * Remove a write request from an inode 794 */ 795 static void nfs_inode_remove_request(struct nfs_page *req) 796 { 797 struct inode *inode = d_inode(req->wb_context->dentry); 798 struct nfs_inode *nfsi = NFS_I(inode); 799 struct nfs_page *head; 800 801 if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) { 802 head = req->wb_head; 803 804 spin_lock(&inode->i_lock); 805 if (likely(head->wb_page && !PageSwapCache(head->wb_page))) { 806 set_page_private(head->wb_page, 0); 807 ClearPagePrivate(head->wb_page); 808 clear_bit(PG_MAPPED, &head->wb_flags); 809 } 810 nfsi->nrequests--; 811 spin_unlock(&inode->i_lock); 812 } else { 813 spin_lock(&inode->i_lock); 814 nfsi->nrequests--; 815 spin_unlock(&inode->i_lock); 816 } 817 818 if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) 819 nfs_release_request(req); 820 } 821 822 static void 823 nfs_mark_request_dirty(struct nfs_page *req) 824 { 825 if (req->wb_page) 826 __set_page_dirty_nobuffers(req->wb_page); 827 } 828 829 /* 830 * nfs_page_search_commits_for_head_request_locked 831 * 832 * Search through commit lists on @inode for the head request for @page. 833 * Must be called while holding the inode (which is cinfo) lock. 834 * 835 * Returns the head request if found, or NULL if not found. 836 */ 837 static struct nfs_page * 838 nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi, 839 struct page *page) 840 { 841 struct nfs_page *freq, *t; 842 struct nfs_commit_info cinfo; 843 struct inode *inode = &nfsi->vfs_inode; 844 845 nfs_init_cinfo_from_inode(&cinfo, inode); 846 847 /* search through pnfs commit lists */ 848 freq = pnfs_search_commit_reqs(inode, &cinfo, page); 849 if (freq) 850 return freq->wb_head; 851 852 /* Linearly search the commit list for the correct request */ 853 list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) { 854 if (freq->wb_page == page) 855 return freq->wb_head; 856 } 857 858 return NULL; 859 } 860 861 /** 862 * nfs_request_add_commit_list_locked - add request to a commit list 863 * @req: pointer to a struct nfs_page 864 * @dst: commit list head 865 * @cinfo: holds list lock and accounting info 866 * 867 * This sets the PG_CLEAN bit, updates the cinfo count of 868 * number of outstanding requests requiring a commit as well as 869 * the MM page stats. 870 * 871 * The caller must hold cinfo->inode->i_lock, and the nfs_page lock. 872 */ 873 void 874 nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst, 875 struct nfs_commit_info *cinfo) 876 { 877 set_bit(PG_CLEAN, &req->wb_flags); 878 nfs_list_add_request(req, dst); 879 cinfo->mds->ncommit++; 880 } 881 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked); 882 883 /** 884 * nfs_request_add_commit_list - add request to a commit list 885 * @req: pointer to a struct nfs_page 886 * @dst: commit list head 887 * @cinfo: holds list lock and accounting info 888 * 889 * This sets the PG_CLEAN bit, updates the cinfo count of 890 * number of outstanding requests requiring a commit as well as 891 * the MM page stats. 892 * 893 * The caller must _not_ hold the cinfo->lock, but must be 894 * holding the nfs_page lock. 895 */ 896 void 897 nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo) 898 { 899 spin_lock(&cinfo->inode->i_lock); 900 nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo); 901 spin_unlock(&cinfo->inode->i_lock); 902 if (req->wb_page) 903 nfs_mark_page_unstable(req->wb_page, cinfo); 904 } 905 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); 906 907 /** 908 * nfs_request_remove_commit_list - Remove request from a commit list 909 * @req: pointer to a nfs_page 910 * @cinfo: holds list lock and accounting info 911 * 912 * This clears the PG_CLEAN bit, and updates the cinfo's count of 913 * number of outstanding requests requiring a commit 914 * It does not update the MM page stats. 915 * 916 * The caller _must_ hold the cinfo->lock and the nfs_page lock. 917 */ 918 void 919 nfs_request_remove_commit_list(struct nfs_page *req, 920 struct nfs_commit_info *cinfo) 921 { 922 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags)) 923 return; 924 nfs_list_remove_request(req); 925 cinfo->mds->ncommit--; 926 } 927 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list); 928 929 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 930 struct inode *inode) 931 { 932 cinfo->inode = inode; 933 cinfo->mds = &NFS_I(inode)->commit_info; 934 cinfo->ds = pnfs_get_ds_info(inode); 935 cinfo->dreq = NULL; 936 cinfo->completion_ops = &nfs_commit_completion_ops; 937 } 938 939 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 940 struct inode *inode, 941 struct nfs_direct_req *dreq) 942 { 943 if (dreq) 944 nfs_init_cinfo_from_dreq(cinfo, dreq); 945 else 946 nfs_init_cinfo_from_inode(cinfo, inode); 947 } 948 EXPORT_SYMBOL_GPL(nfs_init_cinfo); 949 950 /* 951 * Add a request to the inode's commit list. 952 */ 953 void 954 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 955 struct nfs_commit_info *cinfo, u32 ds_commit_idx) 956 { 957 if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx)) 958 return; 959 nfs_request_add_commit_list(req, cinfo); 960 } 961 962 static void 963 nfs_clear_page_commit(struct page *page) 964 { 965 dec_node_page_state(page, NR_UNSTABLE_NFS); 966 dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb, 967 WB_RECLAIMABLE); 968 } 969 970 /* Called holding inode (/cinfo) lock */ 971 static void 972 nfs_clear_request_commit(struct nfs_page *req) 973 { 974 if (test_bit(PG_CLEAN, &req->wb_flags)) { 975 struct inode *inode = d_inode(req->wb_context->dentry); 976 struct nfs_commit_info cinfo; 977 978 nfs_init_cinfo_from_inode(&cinfo, inode); 979 if (!pnfs_clear_request_commit(req, &cinfo)) { 980 nfs_request_remove_commit_list(req, &cinfo); 981 } 982 nfs_clear_page_commit(req->wb_page); 983 } 984 } 985 986 int nfs_write_need_commit(struct nfs_pgio_header *hdr) 987 { 988 if (hdr->verf.committed == NFS_DATA_SYNC) 989 return hdr->lseg == NULL; 990 return hdr->verf.committed != NFS_FILE_SYNC; 991 } 992 993 static void nfs_async_write_init(struct nfs_pgio_header *hdr) 994 { 995 nfs_io_completion_get(hdr->io_completion); 996 } 997 998 static void nfs_write_completion(struct nfs_pgio_header *hdr) 999 { 1000 struct nfs_commit_info cinfo; 1001 unsigned long bytes = 0; 1002 1003 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) 1004 goto out; 1005 nfs_init_cinfo_from_inode(&cinfo, hdr->inode); 1006 while (!list_empty(&hdr->pages)) { 1007 struct nfs_page *req = nfs_list_entry(hdr->pages.next); 1008 1009 bytes += req->wb_bytes; 1010 nfs_list_remove_request(req); 1011 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && 1012 (hdr->good_bytes < bytes)) { 1013 nfs_set_pageerror(req->wb_page); 1014 nfs_context_set_write_error(req->wb_context, hdr->error); 1015 goto remove_req; 1016 } 1017 if (nfs_write_need_commit(hdr)) { 1018 memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf)); 1019 nfs_mark_request_commit(req, hdr->lseg, &cinfo, 1020 hdr->pgio_mirror_idx); 1021 goto next; 1022 } 1023 remove_req: 1024 nfs_inode_remove_request(req); 1025 next: 1026 nfs_unlock_request(req); 1027 nfs_end_page_writeback(req); 1028 nfs_release_request(req); 1029 } 1030 out: 1031 nfs_io_completion_put(hdr->io_completion); 1032 hdr->release(hdr); 1033 } 1034 1035 unsigned long 1036 nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 1037 { 1038 return cinfo->mds->ncommit; 1039 } 1040 1041 /* cinfo->inode->i_lock held by caller */ 1042 int 1043 nfs_scan_commit_list(struct list_head *src, struct list_head *dst, 1044 struct nfs_commit_info *cinfo, int max) 1045 { 1046 struct nfs_page *req, *tmp; 1047 int ret = 0; 1048 1049 list_for_each_entry_safe(req, tmp, src, wb_list) { 1050 if (!nfs_lock_request(req)) 1051 continue; 1052 kref_get(&req->wb_kref); 1053 if (cond_resched_lock(&cinfo->inode->i_lock)) 1054 list_safe_reset_next(req, tmp, wb_list); 1055 nfs_request_remove_commit_list(req, cinfo); 1056 nfs_list_add_request(req, dst); 1057 ret++; 1058 if ((ret == max) && !cinfo->dreq) 1059 break; 1060 } 1061 return ret; 1062 } 1063 1064 /* 1065 * nfs_scan_commit - Scan an inode for commit requests 1066 * @inode: NFS inode to scan 1067 * @dst: mds destination list 1068 * @cinfo: mds and ds lists of reqs ready to commit 1069 * 1070 * Moves requests from the inode's 'commit' request list. 1071 * The requests are *not* checked to ensure that they form a contiguous set. 1072 */ 1073 int 1074 nfs_scan_commit(struct inode *inode, struct list_head *dst, 1075 struct nfs_commit_info *cinfo) 1076 { 1077 int ret = 0; 1078 1079 spin_lock(&cinfo->inode->i_lock); 1080 if (cinfo->mds->ncommit > 0) { 1081 const int max = INT_MAX; 1082 1083 ret = nfs_scan_commit_list(&cinfo->mds->list, dst, 1084 cinfo, max); 1085 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret); 1086 } 1087 spin_unlock(&cinfo->inode->i_lock); 1088 return ret; 1089 } 1090 1091 /* 1092 * Search for an existing write request, and attempt to update 1093 * it to reflect a new dirty region on a given page. 1094 * 1095 * If the attempt fails, then the existing request is flushed out 1096 * to disk. 1097 */ 1098 static struct nfs_page *nfs_try_to_update_request(struct inode *inode, 1099 struct page *page, 1100 unsigned int offset, 1101 unsigned int bytes) 1102 { 1103 struct nfs_page *req; 1104 unsigned int rqend; 1105 unsigned int end; 1106 int error; 1107 1108 if (!PagePrivate(page)) 1109 return NULL; 1110 1111 end = offset + bytes; 1112 spin_lock(&inode->i_lock); 1113 1114 for (;;) { 1115 req = nfs_page_find_head_request_locked(NFS_I(inode), page); 1116 if (req == NULL) 1117 goto out_unlock; 1118 1119 /* should be handled by nfs_flush_incompatible */ 1120 WARN_ON_ONCE(req->wb_head != req); 1121 WARN_ON_ONCE(req->wb_this_page != req); 1122 1123 rqend = req->wb_offset + req->wb_bytes; 1124 /* 1125 * Tell the caller to flush out the request if 1126 * the offsets are non-contiguous. 1127 * Note: nfs_flush_incompatible() will already 1128 * have flushed out requests having wrong owners. 1129 */ 1130 if (offset > rqend 1131 || end < req->wb_offset) 1132 goto out_flushme; 1133 1134 if (nfs_lock_request(req)) 1135 break; 1136 1137 /* The request is locked, so wait and then retry */ 1138 spin_unlock(&inode->i_lock); 1139 error = nfs_wait_on_request(req); 1140 nfs_release_request(req); 1141 if (error != 0) 1142 goto out_err; 1143 spin_lock(&inode->i_lock); 1144 } 1145 1146 /* Okay, the request matches. Update the region */ 1147 if (offset < req->wb_offset) { 1148 req->wb_offset = offset; 1149 req->wb_pgbase = offset; 1150 } 1151 if (end > rqend) 1152 req->wb_bytes = end - req->wb_offset; 1153 else 1154 req->wb_bytes = rqend - req->wb_offset; 1155 out_unlock: 1156 if (req) 1157 nfs_clear_request_commit(req); 1158 spin_unlock(&inode->i_lock); 1159 return req; 1160 out_flushme: 1161 spin_unlock(&inode->i_lock); 1162 nfs_release_request(req); 1163 error = nfs_wb_page(inode, page); 1164 out_err: 1165 return ERR_PTR(error); 1166 } 1167 1168 /* 1169 * Try to update an existing write request, or create one if there is none. 1170 * 1171 * Note: Should always be called with the Page Lock held to prevent races 1172 * if we have to add a new request. Also assumes that the caller has 1173 * already called nfs_flush_incompatible() if necessary. 1174 */ 1175 static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx, 1176 struct page *page, unsigned int offset, unsigned int bytes) 1177 { 1178 struct inode *inode = page_file_mapping(page)->host; 1179 struct nfs_page *req; 1180 1181 req = nfs_try_to_update_request(inode, page, offset, bytes); 1182 if (req != NULL) 1183 goto out; 1184 req = nfs_create_request(ctx, page, NULL, offset, bytes); 1185 if (IS_ERR(req)) 1186 goto out; 1187 nfs_inode_add_request(inode, req); 1188 out: 1189 return req; 1190 } 1191 1192 static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page, 1193 unsigned int offset, unsigned int count) 1194 { 1195 struct nfs_page *req; 1196 1197 req = nfs_setup_write_request(ctx, page, offset, count); 1198 if (IS_ERR(req)) 1199 return PTR_ERR(req); 1200 /* Update file length */ 1201 nfs_grow_file(page, offset, count); 1202 nfs_mark_uptodate(req); 1203 nfs_mark_request_dirty(req); 1204 nfs_unlock_and_release_request(req); 1205 return 0; 1206 } 1207 1208 int nfs_flush_incompatible(struct file *file, struct page *page) 1209 { 1210 struct nfs_open_context *ctx = nfs_file_open_context(file); 1211 struct nfs_lock_context *l_ctx; 1212 struct file_lock_context *flctx = file_inode(file)->i_flctx; 1213 struct nfs_page *req; 1214 int do_flush, status; 1215 /* 1216 * Look for a request corresponding to this page. If there 1217 * is one, and it belongs to another file, we flush it out 1218 * before we try to copy anything into the page. Do this 1219 * due to the lack of an ACCESS-type call in NFSv2. 1220 * Also do the same if we find a request from an existing 1221 * dropped page. 1222 */ 1223 do { 1224 req = nfs_page_find_head_request(page); 1225 if (req == NULL) 1226 return 0; 1227 l_ctx = req->wb_lock_context; 1228 do_flush = req->wb_page != page || 1229 !nfs_match_open_context(req->wb_context, ctx); 1230 /* for now, flush if more than 1 request in page_group */ 1231 do_flush |= req->wb_this_page != req; 1232 if (l_ctx && flctx && 1233 !(list_empty_careful(&flctx->flc_posix) && 1234 list_empty_careful(&flctx->flc_flock))) { 1235 do_flush |= l_ctx->lockowner != current->files; 1236 } 1237 nfs_release_request(req); 1238 if (!do_flush) 1239 return 0; 1240 status = nfs_wb_page(page_file_mapping(page)->host, page); 1241 } while (status == 0); 1242 return status; 1243 } 1244 1245 /* 1246 * Avoid buffered writes when a open context credential's key would 1247 * expire soon. 1248 * 1249 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL. 1250 * 1251 * Return 0 and set a credential flag which triggers the inode to flush 1252 * and performs NFS_FILE_SYNC writes if the key will expired within 1253 * RPC_KEY_EXPIRE_TIMEO. 1254 */ 1255 int 1256 nfs_key_timeout_notify(struct file *filp, struct inode *inode) 1257 { 1258 struct nfs_open_context *ctx = nfs_file_open_context(filp); 1259 struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth; 1260 1261 return rpcauth_key_timeout_notify(auth, ctx->cred); 1262 } 1263 1264 /* 1265 * Test if the open context credential key is marked to expire soon. 1266 */ 1267 bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode) 1268 { 1269 struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth; 1270 1271 return rpcauth_cred_key_to_expire(auth, ctx->cred); 1272 } 1273 1274 /* 1275 * If the page cache is marked as unsafe or invalid, then we can't rely on 1276 * the PageUptodate() flag. In this case, we will need to turn off 1277 * write optimisations that depend on the page contents being correct. 1278 */ 1279 static bool nfs_write_pageuptodate(struct page *page, struct inode *inode) 1280 { 1281 struct nfs_inode *nfsi = NFS_I(inode); 1282 1283 if (nfs_have_delegated_attributes(inode)) 1284 goto out; 1285 if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE) 1286 return false; 1287 smp_rmb(); 1288 if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags)) 1289 return false; 1290 out: 1291 if (nfsi->cache_validity & NFS_INO_INVALID_DATA) 1292 return false; 1293 return PageUptodate(page) != 0; 1294 } 1295 1296 static bool 1297 is_whole_file_wrlock(struct file_lock *fl) 1298 { 1299 return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX && 1300 fl->fl_type == F_WRLCK; 1301 } 1302 1303 /* If we know the page is up to date, and we're not using byte range locks (or 1304 * if we have the whole file locked for writing), it may be more efficient to 1305 * extend the write to cover the entire page in order to avoid fragmentation 1306 * inefficiencies. 1307 * 1308 * If the file is opened for synchronous writes then we can just skip the rest 1309 * of the checks. 1310 */ 1311 static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode) 1312 { 1313 int ret; 1314 struct file_lock_context *flctx = inode->i_flctx; 1315 struct file_lock *fl; 1316 1317 if (file->f_flags & O_DSYNC) 1318 return 0; 1319 if (!nfs_write_pageuptodate(page, inode)) 1320 return 0; 1321 if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) 1322 return 1; 1323 if (!flctx || (list_empty_careful(&flctx->flc_flock) && 1324 list_empty_careful(&flctx->flc_posix))) 1325 return 1; 1326 1327 /* Check to see if there are whole file write locks */ 1328 ret = 0; 1329 spin_lock(&flctx->flc_lock); 1330 if (!list_empty(&flctx->flc_posix)) { 1331 fl = list_first_entry(&flctx->flc_posix, struct file_lock, 1332 fl_list); 1333 if (is_whole_file_wrlock(fl)) 1334 ret = 1; 1335 } else if (!list_empty(&flctx->flc_flock)) { 1336 fl = list_first_entry(&flctx->flc_flock, struct file_lock, 1337 fl_list); 1338 if (fl->fl_type == F_WRLCK) 1339 ret = 1; 1340 } 1341 spin_unlock(&flctx->flc_lock); 1342 return ret; 1343 } 1344 1345 /* 1346 * Update and possibly write a cached page of an NFS file. 1347 * 1348 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad 1349 * things with a page scheduled for an RPC call (e.g. invalidate it). 1350 */ 1351 int nfs_updatepage(struct file *file, struct page *page, 1352 unsigned int offset, unsigned int count) 1353 { 1354 struct nfs_open_context *ctx = nfs_file_open_context(file); 1355 struct inode *inode = page_file_mapping(page)->host; 1356 int status = 0; 1357 1358 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE); 1359 1360 dprintk("NFS: nfs_updatepage(%pD2 %d@%lld)\n", 1361 file, count, (long long)(page_file_offset(page) + offset)); 1362 1363 if (!count) 1364 goto out; 1365 1366 if (nfs_can_extend_write(file, page, inode)) { 1367 count = max(count + offset, nfs_page_length(page)); 1368 offset = 0; 1369 } 1370 1371 status = nfs_writepage_setup(ctx, page, offset, count); 1372 if (status < 0) 1373 nfs_set_pageerror(page); 1374 else 1375 __set_page_dirty_nobuffers(page); 1376 out: 1377 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n", 1378 status, (long long)i_size_read(inode)); 1379 return status; 1380 } 1381 1382 static int flush_task_priority(int how) 1383 { 1384 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) { 1385 case FLUSH_HIGHPRI: 1386 return RPC_PRIORITY_HIGH; 1387 case FLUSH_LOWPRI: 1388 return RPC_PRIORITY_LOW; 1389 } 1390 return RPC_PRIORITY_NORMAL; 1391 } 1392 1393 static void nfs_initiate_write(struct nfs_pgio_header *hdr, 1394 struct rpc_message *msg, 1395 const struct nfs_rpc_ops *rpc_ops, 1396 struct rpc_task_setup *task_setup_data, int how) 1397 { 1398 int priority = flush_task_priority(how); 1399 1400 task_setup_data->priority = priority; 1401 rpc_ops->write_setup(hdr, msg); 1402 1403 nfs4_state_protect_write(NFS_SERVER(hdr->inode)->nfs_client, 1404 &task_setup_data->rpc_client, msg, hdr); 1405 } 1406 1407 /* If a nfs_flush_* function fails, it should remove reqs from @head and 1408 * call this on each, which will prepare them to be retried on next 1409 * writeback using standard nfs. 1410 */ 1411 static void nfs_redirty_request(struct nfs_page *req) 1412 { 1413 nfs_mark_request_dirty(req); 1414 set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags); 1415 nfs_unlock_request(req); 1416 nfs_end_page_writeback(req); 1417 nfs_release_request(req); 1418 } 1419 1420 static void nfs_async_write_error(struct list_head *head) 1421 { 1422 struct nfs_page *req; 1423 1424 while (!list_empty(head)) { 1425 req = nfs_list_entry(head->next); 1426 nfs_list_remove_request(req); 1427 nfs_redirty_request(req); 1428 } 1429 } 1430 1431 static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr) 1432 { 1433 nfs_async_write_error(&hdr->pages); 1434 } 1435 1436 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = { 1437 .init_hdr = nfs_async_write_init, 1438 .error_cleanup = nfs_async_write_error, 1439 .completion = nfs_write_completion, 1440 .reschedule_io = nfs_async_write_reschedule_io, 1441 }; 1442 1443 void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, 1444 struct inode *inode, int ioflags, bool force_mds, 1445 const struct nfs_pgio_completion_ops *compl_ops) 1446 { 1447 struct nfs_server *server = NFS_SERVER(inode); 1448 const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops; 1449 1450 #ifdef CONFIG_NFS_V4_1 1451 if (server->pnfs_curr_ld && !force_mds) 1452 pg_ops = server->pnfs_curr_ld->pg_write_ops; 1453 #endif 1454 nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops, 1455 server->wsize, ioflags, GFP_NOIO); 1456 } 1457 EXPORT_SYMBOL_GPL(nfs_pageio_init_write); 1458 1459 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio) 1460 { 1461 struct nfs_pgio_mirror *mirror; 1462 1463 if (pgio->pg_ops && pgio->pg_ops->pg_cleanup) 1464 pgio->pg_ops->pg_cleanup(pgio); 1465 1466 pgio->pg_ops = &nfs_pgio_rw_ops; 1467 1468 nfs_pageio_stop_mirroring(pgio); 1469 1470 mirror = &pgio->pg_mirrors[0]; 1471 mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize; 1472 } 1473 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds); 1474 1475 1476 void nfs_commit_prepare(struct rpc_task *task, void *calldata) 1477 { 1478 struct nfs_commit_data *data = calldata; 1479 1480 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data); 1481 } 1482 1483 /* 1484 * Special version of should_remove_suid() that ignores capabilities. 1485 */ 1486 static int nfs_should_remove_suid(const struct inode *inode) 1487 { 1488 umode_t mode = inode->i_mode; 1489 int kill = 0; 1490 1491 /* suid always must be killed */ 1492 if (unlikely(mode & S_ISUID)) 1493 kill = ATTR_KILL_SUID; 1494 1495 /* 1496 * sgid without any exec bits is just a mandatory locking mark; leave 1497 * it alone. If some exec bits are set, it's a real sgid; kill it. 1498 */ 1499 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) 1500 kill |= ATTR_KILL_SGID; 1501 1502 if (unlikely(kill && S_ISREG(mode))) 1503 return kill; 1504 1505 return 0; 1506 } 1507 1508 static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr, 1509 struct nfs_fattr *fattr) 1510 { 1511 struct nfs_pgio_args *argp = &hdr->args; 1512 struct nfs_pgio_res *resp = &hdr->res; 1513 u64 size = argp->offset + resp->count; 1514 1515 if (!(fattr->valid & NFS_ATTR_FATTR_SIZE)) 1516 fattr->size = size; 1517 if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) { 1518 fattr->valid &= ~NFS_ATTR_FATTR_SIZE; 1519 return; 1520 } 1521 if (size != fattr->size) 1522 return; 1523 /* Set attribute barrier */ 1524 nfs_fattr_set_barrier(fattr); 1525 /* ...and update size */ 1526 fattr->valid |= NFS_ATTR_FATTR_SIZE; 1527 } 1528 1529 void nfs_writeback_update_inode(struct nfs_pgio_header *hdr) 1530 { 1531 struct nfs_fattr *fattr = &hdr->fattr; 1532 struct inode *inode = hdr->inode; 1533 1534 spin_lock(&inode->i_lock); 1535 nfs_writeback_check_extend(hdr, fattr); 1536 nfs_post_op_update_inode_force_wcc_locked(inode, fattr); 1537 spin_unlock(&inode->i_lock); 1538 } 1539 EXPORT_SYMBOL_GPL(nfs_writeback_update_inode); 1540 1541 /* 1542 * This function is called when the WRITE call is complete. 1543 */ 1544 static int nfs_writeback_done(struct rpc_task *task, 1545 struct nfs_pgio_header *hdr, 1546 struct inode *inode) 1547 { 1548 int status; 1549 1550 /* 1551 * ->write_done will attempt to use post-op attributes to detect 1552 * conflicting writes by other clients. A strict interpretation 1553 * of close-to-open would allow us to continue caching even if 1554 * another writer had changed the file, but some applications 1555 * depend on tighter cache coherency when writing. 1556 */ 1557 status = NFS_PROTO(inode)->write_done(task, hdr); 1558 if (status != 0) 1559 return status; 1560 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count); 1561 1562 if (hdr->res.verf->committed < hdr->args.stable && 1563 task->tk_status >= 0) { 1564 /* We tried a write call, but the server did not 1565 * commit data to stable storage even though we 1566 * requested it. 1567 * Note: There is a known bug in Tru64 < 5.0 in which 1568 * the server reports NFS_DATA_SYNC, but performs 1569 * NFS_FILE_SYNC. We therefore implement this checking 1570 * as a dprintk() in order to avoid filling syslog. 1571 */ 1572 static unsigned long complain; 1573 1574 /* Note this will print the MDS for a DS write */ 1575 if (time_before(complain, jiffies)) { 1576 dprintk("NFS: faulty NFS server %s:" 1577 " (committed = %d) != (stable = %d)\n", 1578 NFS_SERVER(inode)->nfs_client->cl_hostname, 1579 hdr->res.verf->committed, hdr->args.stable); 1580 complain = jiffies + 300 * HZ; 1581 } 1582 } 1583 1584 /* Deal with the suid/sgid bit corner case */ 1585 if (nfs_should_remove_suid(inode)) 1586 nfs_mark_for_revalidate(inode); 1587 return 0; 1588 } 1589 1590 /* 1591 * This function is called when the WRITE call is complete. 1592 */ 1593 static void nfs_writeback_result(struct rpc_task *task, 1594 struct nfs_pgio_header *hdr) 1595 { 1596 struct nfs_pgio_args *argp = &hdr->args; 1597 struct nfs_pgio_res *resp = &hdr->res; 1598 1599 if (resp->count < argp->count) { 1600 static unsigned long complain; 1601 1602 /* This a short write! */ 1603 nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE); 1604 1605 /* Has the server at least made some progress? */ 1606 if (resp->count == 0) { 1607 if (time_before(complain, jiffies)) { 1608 printk(KERN_WARNING 1609 "NFS: Server wrote zero bytes, expected %u.\n", 1610 argp->count); 1611 complain = jiffies + 300 * HZ; 1612 } 1613 nfs_set_pgio_error(hdr, -EIO, argp->offset); 1614 task->tk_status = -EIO; 1615 return; 1616 } 1617 1618 /* For non rpc-based layout drivers, retry-through-MDS */ 1619 if (!task->tk_ops) { 1620 hdr->pnfs_error = -EAGAIN; 1621 return; 1622 } 1623 1624 /* Was this an NFSv2 write or an NFSv3 stable write? */ 1625 if (resp->verf->committed != NFS_UNSTABLE) { 1626 /* Resend from where the server left off */ 1627 hdr->mds_offset += resp->count; 1628 argp->offset += resp->count; 1629 argp->pgbase += resp->count; 1630 argp->count -= resp->count; 1631 } else { 1632 /* Resend as a stable write in order to avoid 1633 * headaches in the case of a server crash. 1634 */ 1635 argp->stable = NFS_FILE_SYNC; 1636 } 1637 rpc_restart_call_prepare(task); 1638 } 1639 } 1640 1641 static int wait_on_commit(struct nfs_mds_commit_info *cinfo) 1642 { 1643 return wait_on_atomic_t(&cinfo->rpcs_out, 1644 nfs_wait_atomic_killable, TASK_KILLABLE); 1645 } 1646 1647 static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo) 1648 { 1649 atomic_inc(&cinfo->rpcs_out); 1650 } 1651 1652 static void nfs_commit_end(struct nfs_mds_commit_info *cinfo) 1653 { 1654 if (atomic_dec_and_test(&cinfo->rpcs_out)) 1655 wake_up_atomic_t(&cinfo->rpcs_out); 1656 } 1657 1658 void nfs_commitdata_release(struct nfs_commit_data *data) 1659 { 1660 put_nfs_open_context(data->context); 1661 nfs_commit_free(data); 1662 } 1663 EXPORT_SYMBOL_GPL(nfs_commitdata_release); 1664 1665 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, 1666 const struct nfs_rpc_ops *nfs_ops, 1667 const struct rpc_call_ops *call_ops, 1668 int how, int flags) 1669 { 1670 struct rpc_task *task; 1671 int priority = flush_task_priority(how); 1672 struct rpc_message msg = { 1673 .rpc_argp = &data->args, 1674 .rpc_resp = &data->res, 1675 .rpc_cred = data->cred, 1676 }; 1677 struct rpc_task_setup task_setup_data = { 1678 .task = &data->task, 1679 .rpc_client = clnt, 1680 .rpc_message = &msg, 1681 .callback_ops = call_ops, 1682 .callback_data = data, 1683 .workqueue = nfsiod_workqueue, 1684 .flags = RPC_TASK_ASYNC | flags, 1685 .priority = priority, 1686 }; 1687 /* Set up the initial task struct. */ 1688 nfs_ops->commit_setup(data, &msg); 1689 1690 dprintk("NFS: initiated commit call\n"); 1691 1692 nfs4_state_protect(NFS_SERVER(data->inode)->nfs_client, 1693 NFS_SP4_MACH_CRED_COMMIT, &task_setup_data.rpc_client, &msg); 1694 1695 task = rpc_run_task(&task_setup_data); 1696 if (IS_ERR(task)) 1697 return PTR_ERR(task); 1698 if (how & FLUSH_SYNC) 1699 rpc_wait_for_completion_task(task); 1700 rpc_put_task(task); 1701 return 0; 1702 } 1703 EXPORT_SYMBOL_GPL(nfs_initiate_commit); 1704 1705 static loff_t nfs_get_lwb(struct list_head *head) 1706 { 1707 loff_t lwb = 0; 1708 struct nfs_page *req; 1709 1710 list_for_each_entry(req, head, wb_list) 1711 if (lwb < (req_offset(req) + req->wb_bytes)) 1712 lwb = req_offset(req) + req->wb_bytes; 1713 1714 return lwb; 1715 } 1716 1717 /* 1718 * Set up the argument/result storage required for the RPC call. 1719 */ 1720 void nfs_init_commit(struct nfs_commit_data *data, 1721 struct list_head *head, 1722 struct pnfs_layout_segment *lseg, 1723 struct nfs_commit_info *cinfo) 1724 { 1725 struct nfs_page *first = nfs_list_entry(head->next); 1726 struct inode *inode = d_inode(first->wb_context->dentry); 1727 1728 /* Set up the RPC argument and reply structs 1729 * NB: take care not to mess about with data->commit et al. */ 1730 1731 list_splice_init(head, &data->pages); 1732 1733 data->inode = inode; 1734 data->cred = first->wb_context->cred; 1735 data->lseg = lseg; /* reference transferred */ 1736 /* only set lwb for pnfs commit */ 1737 if (lseg) 1738 data->lwb = nfs_get_lwb(&data->pages); 1739 data->mds_ops = &nfs_commit_ops; 1740 data->completion_ops = cinfo->completion_ops; 1741 data->dreq = cinfo->dreq; 1742 1743 data->args.fh = NFS_FH(data->inode); 1744 /* Note: we always request a commit of the entire inode */ 1745 data->args.offset = 0; 1746 data->args.count = 0; 1747 data->context = get_nfs_open_context(first->wb_context); 1748 data->res.fattr = &data->fattr; 1749 data->res.verf = &data->verf; 1750 nfs_fattr_init(&data->fattr); 1751 } 1752 EXPORT_SYMBOL_GPL(nfs_init_commit); 1753 1754 void nfs_retry_commit(struct list_head *page_list, 1755 struct pnfs_layout_segment *lseg, 1756 struct nfs_commit_info *cinfo, 1757 u32 ds_commit_idx) 1758 { 1759 struct nfs_page *req; 1760 1761 while (!list_empty(page_list)) { 1762 req = nfs_list_entry(page_list->next); 1763 nfs_list_remove_request(req); 1764 nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx); 1765 if (!cinfo->dreq) 1766 nfs_clear_page_commit(req->wb_page); 1767 nfs_unlock_and_release_request(req); 1768 } 1769 } 1770 EXPORT_SYMBOL_GPL(nfs_retry_commit); 1771 1772 static void 1773 nfs_commit_resched_write(struct nfs_commit_info *cinfo, 1774 struct nfs_page *req) 1775 { 1776 __set_page_dirty_nobuffers(req->wb_page); 1777 } 1778 1779 /* 1780 * Commit dirty pages 1781 */ 1782 static int 1783 nfs_commit_list(struct inode *inode, struct list_head *head, int how, 1784 struct nfs_commit_info *cinfo) 1785 { 1786 struct nfs_commit_data *data; 1787 1788 /* another commit raced with us */ 1789 if (list_empty(head)) 1790 return 0; 1791 1792 data = nfs_commitdata_alloc(true); 1793 1794 /* Set up the argument struct */ 1795 nfs_init_commit(data, head, NULL, cinfo); 1796 atomic_inc(&cinfo->mds->rpcs_out); 1797 return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode), 1798 data->mds_ops, how, 0); 1799 } 1800 1801 /* 1802 * COMMIT call returned 1803 */ 1804 static void nfs_commit_done(struct rpc_task *task, void *calldata) 1805 { 1806 struct nfs_commit_data *data = calldata; 1807 1808 dprintk("NFS: %5u nfs_commit_done (status %d)\n", 1809 task->tk_pid, task->tk_status); 1810 1811 /* Call the NFS version-specific code */ 1812 NFS_PROTO(data->inode)->commit_done(task, data); 1813 } 1814 1815 static void nfs_commit_release_pages(struct nfs_commit_data *data) 1816 { 1817 struct nfs_page *req; 1818 int status = data->task.tk_status; 1819 struct nfs_commit_info cinfo; 1820 struct nfs_server *nfss; 1821 1822 while (!list_empty(&data->pages)) { 1823 req = nfs_list_entry(data->pages.next); 1824 nfs_list_remove_request(req); 1825 if (req->wb_page) 1826 nfs_clear_page_commit(req->wb_page); 1827 1828 dprintk("NFS: commit (%s/%llu %d@%lld)", 1829 req->wb_context->dentry->d_sb->s_id, 1830 (unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)), 1831 req->wb_bytes, 1832 (long long)req_offset(req)); 1833 if (status < 0) { 1834 nfs_context_set_write_error(req->wb_context, status); 1835 if (req->wb_page) 1836 nfs_inode_remove_request(req); 1837 dprintk_cont(", error = %d\n", status); 1838 goto next; 1839 } 1840 1841 /* Okay, COMMIT succeeded, apparently. Check the verifier 1842 * returned by the server against all stored verfs. */ 1843 if (!nfs_write_verifier_cmp(&req->wb_verf, &data->verf.verifier)) { 1844 /* We have a match */ 1845 if (req->wb_page) 1846 nfs_inode_remove_request(req); 1847 dprintk_cont(" OK\n"); 1848 goto next; 1849 } 1850 /* We have a mismatch. Write the page again */ 1851 dprintk_cont(" mismatch\n"); 1852 nfs_mark_request_dirty(req); 1853 set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags); 1854 next: 1855 nfs_unlock_and_release_request(req); 1856 } 1857 nfss = NFS_SERVER(data->inode); 1858 if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 1859 clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC); 1860 1861 nfs_init_cinfo(&cinfo, data->inode, data->dreq); 1862 nfs_commit_end(cinfo.mds); 1863 } 1864 1865 static void nfs_commit_release(void *calldata) 1866 { 1867 struct nfs_commit_data *data = calldata; 1868 1869 data->completion_ops->completion(data); 1870 nfs_commitdata_release(calldata); 1871 } 1872 1873 static const struct rpc_call_ops nfs_commit_ops = { 1874 .rpc_call_prepare = nfs_commit_prepare, 1875 .rpc_call_done = nfs_commit_done, 1876 .rpc_release = nfs_commit_release, 1877 }; 1878 1879 static const struct nfs_commit_completion_ops nfs_commit_completion_ops = { 1880 .completion = nfs_commit_release_pages, 1881 .resched_write = nfs_commit_resched_write, 1882 }; 1883 1884 int nfs_generic_commit_list(struct inode *inode, struct list_head *head, 1885 int how, struct nfs_commit_info *cinfo) 1886 { 1887 int status; 1888 1889 status = pnfs_commit_list(inode, head, how, cinfo); 1890 if (status == PNFS_NOT_ATTEMPTED) 1891 status = nfs_commit_list(inode, head, how, cinfo); 1892 return status; 1893 } 1894 1895 int nfs_commit_inode(struct inode *inode, int how) 1896 { 1897 LIST_HEAD(head); 1898 struct nfs_commit_info cinfo; 1899 int may_wait = how & FLUSH_SYNC; 1900 int error = 0; 1901 int res; 1902 1903 nfs_init_cinfo_from_inode(&cinfo, inode); 1904 nfs_commit_begin(cinfo.mds); 1905 res = nfs_scan_commit(inode, &head, &cinfo); 1906 if (res) 1907 error = nfs_generic_commit_list(inode, &head, how, &cinfo); 1908 nfs_commit_end(cinfo.mds); 1909 if (error < 0) 1910 goto out_error; 1911 if (!may_wait) 1912 goto out_mark_dirty; 1913 error = wait_on_commit(cinfo.mds); 1914 if (error < 0) 1915 return error; 1916 return res; 1917 out_error: 1918 res = error; 1919 /* Note: If we exit without ensuring that the commit is complete, 1920 * we must mark the inode as dirty. Otherwise, future calls to 1921 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure 1922 * that the data is on the disk. 1923 */ 1924 out_mark_dirty: 1925 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1926 return res; 1927 } 1928 EXPORT_SYMBOL_GPL(nfs_commit_inode); 1929 1930 int nfs_write_inode(struct inode *inode, struct writeback_control *wbc) 1931 { 1932 struct nfs_inode *nfsi = NFS_I(inode); 1933 int flags = FLUSH_SYNC; 1934 int ret = 0; 1935 1936 /* no commits means nothing needs to be done */ 1937 if (!nfsi->commit_info.ncommit) 1938 return ret; 1939 1940 if (wbc->sync_mode == WB_SYNC_NONE) { 1941 /* Don't commit yet if this is a non-blocking flush and there 1942 * are a lot of outstanding writes for this mapping. 1943 */ 1944 if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) 1945 goto out_mark_dirty; 1946 1947 /* don't wait for the COMMIT response */ 1948 flags = 0; 1949 } 1950 1951 ret = nfs_commit_inode(inode, flags); 1952 if (ret >= 0) { 1953 if (wbc->sync_mode == WB_SYNC_NONE) { 1954 if (ret < wbc->nr_to_write) 1955 wbc->nr_to_write -= ret; 1956 else 1957 wbc->nr_to_write = 0; 1958 } 1959 return 0; 1960 } 1961 out_mark_dirty: 1962 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1963 return ret; 1964 } 1965 EXPORT_SYMBOL_GPL(nfs_write_inode); 1966 1967 /* 1968 * Wrapper for filemap_write_and_wait_range() 1969 * 1970 * Needed for pNFS in order to ensure data becomes visible to the 1971 * client. 1972 */ 1973 int nfs_filemap_write_and_wait_range(struct address_space *mapping, 1974 loff_t lstart, loff_t lend) 1975 { 1976 int ret; 1977 1978 ret = filemap_write_and_wait_range(mapping, lstart, lend); 1979 if (ret == 0) 1980 ret = pnfs_sync_inode(mapping->host, true); 1981 return ret; 1982 } 1983 EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range); 1984 1985 /* 1986 * flush the inode to disk. 1987 */ 1988 int nfs_wb_all(struct inode *inode) 1989 { 1990 int ret; 1991 1992 trace_nfs_writeback_inode_enter(inode); 1993 1994 ret = filemap_write_and_wait(inode->i_mapping); 1995 if (ret) 1996 goto out; 1997 ret = nfs_commit_inode(inode, FLUSH_SYNC); 1998 if (ret < 0) 1999 goto out; 2000 pnfs_sync_inode(inode, true); 2001 ret = 0; 2002 2003 out: 2004 trace_nfs_writeback_inode_exit(inode, ret); 2005 return ret; 2006 } 2007 EXPORT_SYMBOL_GPL(nfs_wb_all); 2008 2009 int nfs_wb_page_cancel(struct inode *inode, struct page *page) 2010 { 2011 struct nfs_page *req; 2012 int ret = 0; 2013 2014 wait_on_page_writeback(page); 2015 2016 /* blocking call to cancel all requests and join to a single (head) 2017 * request */ 2018 req = nfs_lock_and_join_requests(page, false); 2019 2020 if (IS_ERR(req)) { 2021 ret = PTR_ERR(req); 2022 } else if (req) { 2023 /* all requests from this page have been cancelled by 2024 * nfs_lock_and_join_requests, so just remove the head 2025 * request from the inode / page_private pointer and 2026 * release it */ 2027 nfs_inode_remove_request(req); 2028 nfs_unlock_and_release_request(req); 2029 } 2030 2031 return ret; 2032 } 2033 2034 /* 2035 * Write back all requests on one page - we do this before reading it. 2036 */ 2037 int nfs_wb_page(struct inode *inode, struct page *page) 2038 { 2039 loff_t range_start = page_file_offset(page); 2040 loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1); 2041 struct writeback_control wbc = { 2042 .sync_mode = WB_SYNC_ALL, 2043 .nr_to_write = 0, 2044 .range_start = range_start, 2045 .range_end = range_end, 2046 }; 2047 int ret; 2048 2049 trace_nfs_writeback_page_enter(inode); 2050 2051 for (;;) { 2052 wait_on_page_writeback(page); 2053 if (clear_page_dirty_for_io(page)) { 2054 ret = nfs_writepage_locked(page, &wbc); 2055 if (ret < 0) 2056 goto out_error; 2057 continue; 2058 } 2059 ret = 0; 2060 if (!PagePrivate(page)) 2061 break; 2062 ret = nfs_commit_inode(inode, FLUSH_SYNC); 2063 if (ret < 0) 2064 goto out_error; 2065 } 2066 out_error: 2067 trace_nfs_writeback_page_exit(inode, ret); 2068 return ret; 2069 } 2070 2071 #ifdef CONFIG_MIGRATION 2072 int nfs_migrate_page(struct address_space *mapping, struct page *newpage, 2073 struct page *page, enum migrate_mode mode) 2074 { 2075 /* 2076 * If PagePrivate is set, then the page is currently associated with 2077 * an in-progress read or write request. Don't try to migrate it. 2078 * 2079 * FIXME: we could do this in principle, but we'll need a way to ensure 2080 * that we can safely release the inode reference while holding 2081 * the page lock. 2082 */ 2083 if (PagePrivate(page)) 2084 return -EBUSY; 2085 2086 if (!nfs_fscache_release_page(page, GFP_KERNEL)) 2087 return -EBUSY; 2088 2089 return migrate_page(mapping, newpage, page, mode); 2090 } 2091 #endif 2092 2093 int __init nfs_init_writepagecache(void) 2094 { 2095 nfs_wdata_cachep = kmem_cache_create("nfs_write_data", 2096 sizeof(struct nfs_pgio_header), 2097 0, SLAB_HWCACHE_ALIGN, 2098 NULL); 2099 if (nfs_wdata_cachep == NULL) 2100 return -ENOMEM; 2101 2102 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE, 2103 nfs_wdata_cachep); 2104 if (nfs_wdata_mempool == NULL) 2105 goto out_destroy_write_cache; 2106 2107 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data", 2108 sizeof(struct nfs_commit_data), 2109 0, SLAB_HWCACHE_ALIGN, 2110 NULL); 2111 if (nfs_cdata_cachep == NULL) 2112 goto out_destroy_write_mempool; 2113 2114 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT, 2115 nfs_cdata_cachep); 2116 if (nfs_commit_mempool == NULL) 2117 goto out_destroy_commit_cache; 2118 2119 /* 2120 * NFS congestion size, scale with available memory. 2121 * 2122 * 64MB: 8192k 2123 * 128MB: 11585k 2124 * 256MB: 16384k 2125 * 512MB: 23170k 2126 * 1GB: 32768k 2127 * 2GB: 46340k 2128 * 4GB: 65536k 2129 * 8GB: 92681k 2130 * 16GB: 131072k 2131 * 2132 * This allows larger machines to have larger/more transfers. 2133 * Limit the default to 256M 2134 */ 2135 nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10); 2136 if (nfs_congestion_kb > 256*1024) 2137 nfs_congestion_kb = 256*1024; 2138 2139 return 0; 2140 2141 out_destroy_commit_cache: 2142 kmem_cache_destroy(nfs_cdata_cachep); 2143 out_destroy_write_mempool: 2144 mempool_destroy(nfs_wdata_mempool); 2145 out_destroy_write_cache: 2146 kmem_cache_destroy(nfs_wdata_cachep); 2147 return -ENOMEM; 2148 } 2149 2150 void nfs_destroy_writepagecache(void) 2151 { 2152 mempool_destroy(nfs_commit_mempool); 2153 kmem_cache_destroy(nfs_cdata_cachep); 2154 mempool_destroy(nfs_wdata_mempool); 2155 kmem_cache_destroy(nfs_wdata_cachep); 2156 } 2157 2158 static const struct nfs_rw_ops nfs_rw_write_ops = { 2159 .rw_alloc_header = nfs_writehdr_alloc, 2160 .rw_free_header = nfs_writehdr_free, 2161 .rw_done = nfs_writeback_done, 2162 .rw_result = nfs_writeback_result, 2163 .rw_initiate = nfs_initiate_write, 2164 }; 2165