1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/fs/nfs/write.c 4 * 5 * Write file data over NFS. 6 * 7 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de> 8 */ 9 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/mm.h> 13 #include <linux/pagemap.h> 14 #include <linux/file.h> 15 #include <linux/writeback.h> 16 #include <linux/swap.h> 17 #include <linux/migrate.h> 18 19 #include <linux/sunrpc/clnt.h> 20 #include <linux/nfs_fs.h> 21 #include <linux/nfs_mount.h> 22 #include <linux/nfs_page.h> 23 #include <linux/backing-dev.h> 24 #include <linux/export.h> 25 #include <linux/freezer.h> 26 #include <linux/wait.h> 27 #include <linux/iversion.h> 28 29 #include <linux/uaccess.h> 30 #include <linux/sched/mm.h> 31 32 #include "delegation.h" 33 #include "internal.h" 34 #include "iostat.h" 35 #include "nfs4_fs.h" 36 #include "fscache.h" 37 #include "pnfs.h" 38 39 #include "nfstrace.h" 40 41 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 42 43 #define MIN_POOL_WRITE (32) 44 #define MIN_POOL_COMMIT (4) 45 46 struct nfs_io_completion { 47 void (*complete)(void *data); 48 void *data; 49 struct kref refcount; 50 }; 51 52 /* 53 * Local function declarations 54 */ 55 static void nfs_redirty_request(struct nfs_page *req); 56 static const struct rpc_call_ops nfs_commit_ops; 57 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops; 58 static const struct nfs_commit_completion_ops nfs_commit_completion_ops; 59 static const struct nfs_rw_ops nfs_rw_write_ops; 60 static void nfs_inode_remove_request(struct nfs_page *req); 61 static void nfs_clear_request_commit(struct nfs_page *req); 62 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 63 struct inode *inode); 64 static struct nfs_page * 65 nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi, 66 struct page *page); 67 68 static struct kmem_cache *nfs_wdata_cachep; 69 static mempool_t *nfs_wdata_mempool; 70 static struct kmem_cache *nfs_cdata_cachep; 71 static mempool_t *nfs_commit_mempool; 72 73 struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail) 74 { 75 struct nfs_commit_data *p; 76 77 if (never_fail) 78 p = mempool_alloc(nfs_commit_mempool, GFP_NOIO); 79 else { 80 /* It is OK to do some reclaim, not no safe to wait 81 * for anything to be returned to the pool. 82 * mempool_alloc() cannot handle that particular combination, 83 * so we need two separate attempts. 84 */ 85 p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT); 86 if (!p) 87 p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO | 88 __GFP_NOWARN | __GFP_NORETRY); 89 if (!p) 90 return NULL; 91 } 92 93 memset(p, 0, sizeof(*p)); 94 INIT_LIST_HEAD(&p->pages); 95 return p; 96 } 97 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); 98 99 void nfs_commit_free(struct nfs_commit_data *p) 100 { 101 mempool_free(p, nfs_commit_mempool); 102 } 103 EXPORT_SYMBOL_GPL(nfs_commit_free); 104 105 static struct nfs_pgio_header *nfs_writehdr_alloc(void) 106 { 107 struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_KERNEL); 108 109 memset(p, 0, sizeof(*p)); 110 p->rw_mode = FMODE_WRITE; 111 return p; 112 } 113 114 static void nfs_writehdr_free(struct nfs_pgio_header *hdr) 115 { 116 mempool_free(hdr, nfs_wdata_mempool); 117 } 118 119 static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags) 120 { 121 return kmalloc(sizeof(struct nfs_io_completion), gfp_flags); 122 } 123 124 static void nfs_io_completion_init(struct nfs_io_completion *ioc, 125 void (*complete)(void *), void *data) 126 { 127 ioc->complete = complete; 128 ioc->data = data; 129 kref_init(&ioc->refcount); 130 } 131 132 static void nfs_io_completion_release(struct kref *kref) 133 { 134 struct nfs_io_completion *ioc = container_of(kref, 135 struct nfs_io_completion, refcount); 136 ioc->complete(ioc->data); 137 kfree(ioc); 138 } 139 140 static void nfs_io_completion_get(struct nfs_io_completion *ioc) 141 { 142 if (ioc != NULL) 143 kref_get(&ioc->refcount); 144 } 145 146 static void nfs_io_completion_put(struct nfs_io_completion *ioc) 147 { 148 if (ioc != NULL) 149 kref_put(&ioc->refcount, nfs_io_completion_release); 150 } 151 152 static void 153 nfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode) 154 { 155 if (!test_and_set_bit(PG_INODE_REF, &req->wb_flags)) { 156 kref_get(&req->wb_kref); 157 atomic_long_inc(&NFS_I(inode)->nrequests); 158 } 159 } 160 161 static int 162 nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode) 163 { 164 int ret; 165 166 if (!test_bit(PG_REMOVE, &req->wb_flags)) 167 return 0; 168 ret = nfs_page_group_lock(req); 169 if (ret) 170 return ret; 171 if (test_and_clear_bit(PG_REMOVE, &req->wb_flags)) 172 nfs_page_set_inode_ref(req, inode); 173 nfs_page_group_unlock(req); 174 return 0; 175 } 176 177 static struct nfs_page * 178 nfs_page_private_request(struct page *page) 179 { 180 if (!PagePrivate(page)) 181 return NULL; 182 return (struct nfs_page *)page_private(page); 183 } 184 185 /* 186 * nfs_page_find_head_request_locked - find head request associated with @page 187 * 188 * must be called while holding the inode lock. 189 * 190 * returns matching head request with reference held, or NULL if not found. 191 */ 192 static struct nfs_page * 193 nfs_page_find_private_request(struct page *page) 194 { 195 struct address_space *mapping = page_file_mapping(page); 196 struct nfs_page *req; 197 198 if (!PagePrivate(page)) 199 return NULL; 200 spin_lock(&mapping->private_lock); 201 req = nfs_page_private_request(page); 202 if (req) { 203 WARN_ON_ONCE(req->wb_head != req); 204 kref_get(&req->wb_kref); 205 } 206 spin_unlock(&mapping->private_lock); 207 return req; 208 } 209 210 static struct nfs_page * 211 nfs_page_find_swap_request(struct page *page) 212 { 213 struct inode *inode = page_file_mapping(page)->host; 214 struct nfs_inode *nfsi = NFS_I(inode); 215 struct nfs_page *req = NULL; 216 if (!PageSwapCache(page)) 217 return NULL; 218 mutex_lock(&nfsi->commit_mutex); 219 if (PageSwapCache(page)) { 220 req = nfs_page_search_commits_for_head_request_locked(nfsi, 221 page); 222 if (req) { 223 WARN_ON_ONCE(req->wb_head != req); 224 kref_get(&req->wb_kref); 225 } 226 } 227 mutex_unlock(&nfsi->commit_mutex); 228 return req; 229 } 230 231 /* 232 * nfs_page_find_head_request - find head request associated with @page 233 * 234 * returns matching head request with reference held, or NULL if not found. 235 */ 236 static struct nfs_page *nfs_page_find_head_request(struct page *page) 237 { 238 struct nfs_page *req; 239 240 req = nfs_page_find_private_request(page); 241 if (!req) 242 req = nfs_page_find_swap_request(page); 243 return req; 244 } 245 246 static struct nfs_page *nfs_find_and_lock_page_request(struct page *page) 247 { 248 struct inode *inode = page_file_mapping(page)->host; 249 struct nfs_page *req, *head; 250 int ret; 251 252 for (;;) { 253 req = nfs_page_find_head_request(page); 254 if (!req) 255 return req; 256 head = nfs_page_group_lock_head(req); 257 if (head != req) 258 nfs_release_request(req); 259 if (IS_ERR(head)) 260 return head; 261 ret = nfs_cancel_remove_inode(head, inode); 262 if (ret < 0) { 263 nfs_unlock_and_release_request(head); 264 return ERR_PTR(ret); 265 } 266 /* Ensure that nobody removed the request before we locked it */ 267 if (head == nfs_page_private_request(page)) 268 break; 269 if (PageSwapCache(page)) 270 break; 271 nfs_unlock_and_release_request(head); 272 } 273 return head; 274 } 275 276 /* Adjust the file length if we're writing beyond the end */ 277 static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count) 278 { 279 struct inode *inode = page_file_mapping(page)->host; 280 loff_t end, i_size; 281 pgoff_t end_index; 282 283 spin_lock(&inode->i_lock); 284 i_size = i_size_read(inode); 285 end_index = (i_size - 1) >> PAGE_SHIFT; 286 if (i_size > 0 && page_index(page) < end_index) 287 goto out; 288 end = page_file_offset(page) + ((loff_t)offset+count); 289 if (i_size >= end) 290 goto out; 291 i_size_write(inode, end); 292 NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; 293 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); 294 out: 295 spin_unlock(&inode->i_lock); 296 } 297 298 /* A writeback failed: mark the page as bad, and invalidate the page cache */ 299 static void nfs_set_pageerror(struct address_space *mapping) 300 { 301 struct inode *inode = mapping->host; 302 303 nfs_zap_mapping(mapping->host, mapping); 304 /* Force file size revalidation */ 305 spin_lock(&inode->i_lock); 306 NFS_I(inode)->cache_validity |= NFS_INO_REVAL_FORCED | 307 NFS_INO_REVAL_PAGECACHE | 308 NFS_INO_INVALID_SIZE; 309 spin_unlock(&inode->i_lock); 310 } 311 312 static void nfs_mapping_set_error(struct page *page, int error) 313 { 314 struct address_space *mapping = page_file_mapping(page); 315 316 SetPageError(page); 317 mapping_set_error(mapping, error); 318 nfs_set_pageerror(mapping); 319 } 320 321 /* 322 * nfs_page_group_search_locked 323 * @head - head request of page group 324 * @page_offset - offset into page 325 * 326 * Search page group with head @head to find a request that contains the 327 * page offset @page_offset. 328 * 329 * Returns a pointer to the first matching nfs request, or NULL if no 330 * match is found. 331 * 332 * Must be called with the page group lock held 333 */ 334 static struct nfs_page * 335 nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset) 336 { 337 struct nfs_page *req; 338 339 req = head; 340 do { 341 if (page_offset >= req->wb_pgbase && 342 page_offset < (req->wb_pgbase + req->wb_bytes)) 343 return req; 344 345 req = req->wb_this_page; 346 } while (req != head); 347 348 return NULL; 349 } 350 351 /* 352 * nfs_page_group_covers_page 353 * @head - head request of page group 354 * 355 * Return true if the page group with head @head covers the whole page, 356 * returns false otherwise 357 */ 358 static bool nfs_page_group_covers_page(struct nfs_page *req) 359 { 360 struct nfs_page *tmp; 361 unsigned int pos = 0; 362 unsigned int len = nfs_page_length(req->wb_page); 363 364 nfs_page_group_lock(req); 365 366 for (;;) { 367 tmp = nfs_page_group_search_locked(req->wb_head, pos); 368 if (!tmp) 369 break; 370 pos = tmp->wb_pgbase + tmp->wb_bytes; 371 } 372 373 nfs_page_group_unlock(req); 374 return pos >= len; 375 } 376 377 /* We can set the PG_uptodate flag if we see that a write request 378 * covers the full page. 379 */ 380 static void nfs_mark_uptodate(struct nfs_page *req) 381 { 382 if (PageUptodate(req->wb_page)) 383 return; 384 if (!nfs_page_group_covers_page(req)) 385 return; 386 SetPageUptodate(req->wb_page); 387 } 388 389 static int wb_priority(struct writeback_control *wbc) 390 { 391 int ret = 0; 392 393 if (wbc->sync_mode == WB_SYNC_ALL) 394 ret = FLUSH_COND_STABLE; 395 return ret; 396 } 397 398 /* 399 * NFS congestion control 400 */ 401 402 int nfs_congestion_kb; 403 404 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10)) 405 #define NFS_CONGESTION_OFF_THRESH \ 406 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) 407 408 static void nfs_set_page_writeback(struct page *page) 409 { 410 struct inode *inode = page_file_mapping(page)->host; 411 struct nfs_server *nfss = NFS_SERVER(inode); 412 int ret = test_set_page_writeback(page); 413 414 WARN_ON_ONCE(ret != 0); 415 416 if (atomic_long_inc_return(&nfss->writeback) > 417 NFS_CONGESTION_ON_THRESH) 418 set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 419 } 420 421 static void nfs_end_page_writeback(struct nfs_page *req) 422 { 423 struct inode *inode = page_file_mapping(req->wb_page)->host; 424 struct nfs_server *nfss = NFS_SERVER(inode); 425 bool is_done; 426 427 is_done = nfs_page_group_sync_on_bit(req, PG_WB_END); 428 nfs_unlock_request(req); 429 if (!is_done) 430 return; 431 432 end_page_writeback(req->wb_page); 433 if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 434 clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 435 } 436 437 /* 438 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests 439 * 440 * @destroy_list - request list (using wb_this_page) terminated by @old_head 441 * @old_head - the old head of the list 442 * 443 * All subrequests must be locked and removed from all lists, so at this point 444 * they are only "active" in this function, and possibly in nfs_wait_on_request 445 * with a reference held by some other context. 446 */ 447 static void 448 nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, 449 struct nfs_page *old_head, 450 struct inode *inode) 451 { 452 while (destroy_list) { 453 struct nfs_page *subreq = destroy_list; 454 455 destroy_list = (subreq->wb_this_page == old_head) ? 456 NULL : subreq->wb_this_page; 457 458 /* Note: lock subreq in order to change subreq->wb_head */ 459 nfs_page_set_headlock(subreq); 460 WARN_ON_ONCE(old_head != subreq->wb_head); 461 462 /* make sure old group is not used */ 463 subreq->wb_this_page = subreq; 464 subreq->wb_head = subreq; 465 466 clear_bit(PG_REMOVE, &subreq->wb_flags); 467 468 /* Note: races with nfs_page_group_destroy() */ 469 if (!kref_read(&subreq->wb_kref)) { 470 /* Check if we raced with nfs_page_group_destroy() */ 471 if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) { 472 nfs_page_clear_headlock(subreq); 473 nfs_free_request(subreq); 474 } else 475 nfs_page_clear_headlock(subreq); 476 continue; 477 } 478 nfs_page_clear_headlock(subreq); 479 480 nfs_release_request(old_head); 481 482 if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) { 483 nfs_release_request(subreq); 484 atomic_long_dec(&NFS_I(inode)->nrequests); 485 } 486 487 /* subreq is now totally disconnected from page group or any 488 * write / commit lists. last chance to wake any waiters */ 489 nfs_unlock_and_release_request(subreq); 490 } 491 } 492 493 /* 494 * nfs_join_page_group - destroy subrequests of the head req 495 * @head: the page used to lookup the "page group" of nfs_page structures 496 * @inode: Inode to which the request belongs. 497 * 498 * This function joins all sub requests to the head request by first 499 * locking all requests in the group, cancelling any pending operations 500 * and finally updating the head request to cover the whole range covered by 501 * the (former) group. All subrequests are removed from any write or commit 502 * lists, unlinked from the group and destroyed. 503 */ 504 void 505 nfs_join_page_group(struct nfs_page *head, struct inode *inode) 506 { 507 struct nfs_page *subreq; 508 struct nfs_page *destroy_list = NULL; 509 unsigned int pgbase, off, bytes; 510 511 pgbase = head->wb_pgbase; 512 bytes = head->wb_bytes; 513 off = head->wb_offset; 514 for (subreq = head->wb_this_page; subreq != head; 515 subreq = subreq->wb_this_page) { 516 /* Subrequests should always form a contiguous range */ 517 if (pgbase > subreq->wb_pgbase) { 518 off -= pgbase - subreq->wb_pgbase; 519 bytes += pgbase - subreq->wb_pgbase; 520 pgbase = subreq->wb_pgbase; 521 } 522 bytes = max(subreq->wb_pgbase + subreq->wb_bytes 523 - pgbase, bytes); 524 } 525 526 /* Set the head request's range to cover the former page group */ 527 head->wb_pgbase = pgbase; 528 head->wb_bytes = bytes; 529 head->wb_offset = off; 530 531 /* Now that all requests are locked, make sure they aren't on any list. 532 * Commit list removal accounting is done after locks are dropped */ 533 subreq = head; 534 do { 535 nfs_clear_request_commit(subreq); 536 subreq = subreq->wb_this_page; 537 } while (subreq != head); 538 539 /* unlink subrequests from head, destroy them later */ 540 if (head->wb_this_page != head) { 541 /* destroy list will be terminated by head */ 542 destroy_list = head->wb_this_page; 543 head->wb_this_page = head; 544 } 545 546 nfs_destroy_unlinked_subrequests(destroy_list, head, inode); 547 } 548 549 /* 550 * nfs_lock_and_join_requests - join all subreqs to the head req 551 * @page: the page used to lookup the "page group" of nfs_page structures 552 * 553 * This function joins all sub requests to the head request by first 554 * locking all requests in the group, cancelling any pending operations 555 * and finally updating the head request to cover the whole range covered by 556 * the (former) group. All subrequests are removed from any write or commit 557 * lists, unlinked from the group and destroyed. 558 * 559 * Returns a locked, referenced pointer to the head request - which after 560 * this call is guaranteed to be the only request associated with the page. 561 * Returns NULL if no requests are found for @page, or a ERR_PTR if an 562 * error was encountered. 563 */ 564 static struct nfs_page * 565 nfs_lock_and_join_requests(struct page *page) 566 { 567 struct inode *inode = page_file_mapping(page)->host; 568 struct nfs_page *head; 569 int ret; 570 571 /* 572 * A reference is taken only on the head request which acts as a 573 * reference to the whole page group - the group will not be destroyed 574 * until the head reference is released. 575 */ 576 head = nfs_find_and_lock_page_request(page); 577 if (IS_ERR_OR_NULL(head)) 578 return head; 579 580 /* lock each request in the page group */ 581 ret = nfs_page_group_lock_subrequests(head); 582 if (ret < 0) { 583 nfs_unlock_and_release_request(head); 584 return ERR_PTR(ret); 585 } 586 587 nfs_join_page_group(head, inode); 588 589 return head; 590 } 591 592 static void nfs_write_error(struct nfs_page *req, int error) 593 { 594 trace_nfs_write_error(req, error); 595 nfs_mapping_set_error(req->wb_page, error); 596 nfs_inode_remove_request(req); 597 nfs_end_page_writeback(req); 598 nfs_release_request(req); 599 } 600 601 /* 602 * Find an associated nfs write request, and prepare to flush it out 603 * May return an error if the user signalled nfs_wait_on_request(). 604 */ 605 static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio, 606 struct page *page) 607 { 608 struct nfs_page *req; 609 int ret = 0; 610 611 req = nfs_lock_and_join_requests(page); 612 if (!req) 613 goto out; 614 ret = PTR_ERR(req); 615 if (IS_ERR(req)) 616 goto out; 617 618 nfs_set_page_writeback(page); 619 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags)); 620 621 /* If there is a fatal error that covers this write, just exit */ 622 ret = pgio->pg_error; 623 if (nfs_error_is_fatal_on_server(ret)) 624 goto out_launder; 625 626 ret = 0; 627 if (!nfs_pageio_add_request(pgio, req)) { 628 ret = pgio->pg_error; 629 /* 630 * Remove the problematic req upon fatal errors on the server 631 */ 632 if (nfs_error_is_fatal(ret)) { 633 if (nfs_error_is_fatal_on_server(ret)) 634 goto out_launder; 635 } else 636 ret = -EAGAIN; 637 nfs_redirty_request(req); 638 pgio->pg_error = 0; 639 } else 640 nfs_add_stats(page_file_mapping(page)->host, 641 NFSIOS_WRITEPAGES, 1); 642 out: 643 return ret; 644 out_launder: 645 nfs_write_error(req, ret); 646 return 0; 647 } 648 649 static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, 650 struct nfs_pageio_descriptor *pgio) 651 { 652 int ret; 653 654 nfs_pageio_cond_complete(pgio, page_index(page)); 655 ret = nfs_page_async_flush(pgio, page); 656 if (ret == -EAGAIN) { 657 redirty_page_for_writepage(wbc, page); 658 ret = AOP_WRITEPAGE_ACTIVATE; 659 } 660 return ret; 661 } 662 663 /* 664 * Write an mmapped page to the server. 665 */ 666 static int nfs_writepage_locked(struct page *page, 667 struct writeback_control *wbc) 668 { 669 struct nfs_pageio_descriptor pgio; 670 struct inode *inode = page_file_mapping(page)->host; 671 int err; 672 673 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE); 674 nfs_pageio_init_write(&pgio, inode, 0, 675 false, &nfs_async_write_completion_ops); 676 err = nfs_do_writepage(page, wbc, &pgio); 677 pgio.pg_error = 0; 678 nfs_pageio_complete(&pgio); 679 if (err < 0) 680 return err; 681 if (nfs_error_is_fatal(pgio.pg_error)) 682 return pgio.pg_error; 683 return 0; 684 } 685 686 int nfs_writepage(struct page *page, struct writeback_control *wbc) 687 { 688 int ret; 689 690 ret = nfs_writepage_locked(page, wbc); 691 if (ret != AOP_WRITEPAGE_ACTIVATE) 692 unlock_page(page); 693 return ret; 694 } 695 696 static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data) 697 { 698 int ret; 699 700 ret = nfs_do_writepage(page, wbc, data); 701 if (ret != AOP_WRITEPAGE_ACTIVATE) 702 unlock_page(page); 703 return ret; 704 } 705 706 static void nfs_io_completion_commit(void *inode) 707 { 708 nfs_commit_inode(inode, 0); 709 } 710 711 int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) 712 { 713 struct inode *inode = mapping->host; 714 struct nfs_pageio_descriptor pgio; 715 struct nfs_io_completion *ioc; 716 int err; 717 718 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES); 719 720 ioc = nfs_io_completion_alloc(GFP_KERNEL); 721 if (ioc) 722 nfs_io_completion_init(ioc, nfs_io_completion_commit, inode); 723 724 nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false, 725 &nfs_async_write_completion_ops); 726 pgio.pg_io_completion = ioc; 727 err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio); 728 pgio.pg_error = 0; 729 nfs_pageio_complete(&pgio); 730 nfs_io_completion_put(ioc); 731 732 if (err < 0) 733 goto out_err; 734 err = pgio.pg_error; 735 if (nfs_error_is_fatal(err)) 736 goto out_err; 737 return 0; 738 out_err: 739 return err; 740 } 741 742 /* 743 * Insert a write request into an inode 744 */ 745 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req) 746 { 747 struct address_space *mapping = page_file_mapping(req->wb_page); 748 struct nfs_inode *nfsi = NFS_I(inode); 749 750 WARN_ON_ONCE(req->wb_this_page != req); 751 752 /* Lock the request! */ 753 nfs_lock_request(req); 754 755 /* 756 * Swap-space should not get truncated. Hence no need to plug the race 757 * with invalidate/truncate. 758 */ 759 spin_lock(&mapping->private_lock); 760 if (!nfs_have_writebacks(inode) && 761 NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) 762 inode_inc_iversion_raw(inode); 763 if (likely(!PageSwapCache(req->wb_page))) { 764 set_bit(PG_MAPPED, &req->wb_flags); 765 SetPagePrivate(req->wb_page); 766 set_page_private(req->wb_page, (unsigned long)req); 767 } 768 spin_unlock(&mapping->private_lock); 769 atomic_long_inc(&nfsi->nrequests); 770 /* this a head request for a page group - mark it as having an 771 * extra reference so sub groups can follow suit. 772 * This flag also informs pgio layer when to bump nrequests when 773 * adding subrequests. */ 774 WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags)); 775 kref_get(&req->wb_kref); 776 } 777 778 /* 779 * Remove a write request from an inode 780 */ 781 static void nfs_inode_remove_request(struct nfs_page *req) 782 { 783 struct address_space *mapping = page_file_mapping(req->wb_page); 784 struct inode *inode = mapping->host; 785 struct nfs_inode *nfsi = NFS_I(inode); 786 struct nfs_page *head; 787 788 if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) { 789 head = req->wb_head; 790 791 spin_lock(&mapping->private_lock); 792 if (likely(head->wb_page && !PageSwapCache(head->wb_page))) { 793 set_page_private(head->wb_page, 0); 794 ClearPagePrivate(head->wb_page); 795 clear_bit(PG_MAPPED, &head->wb_flags); 796 } 797 spin_unlock(&mapping->private_lock); 798 } 799 800 if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) { 801 nfs_release_request(req); 802 atomic_long_dec(&nfsi->nrequests); 803 } 804 } 805 806 static void 807 nfs_mark_request_dirty(struct nfs_page *req) 808 { 809 if (req->wb_page) 810 __set_page_dirty_nobuffers(req->wb_page); 811 } 812 813 /* 814 * nfs_page_search_commits_for_head_request_locked 815 * 816 * Search through commit lists on @inode for the head request for @page. 817 * Must be called while holding the inode (which is cinfo) lock. 818 * 819 * Returns the head request if found, or NULL if not found. 820 */ 821 static struct nfs_page * 822 nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi, 823 struct page *page) 824 { 825 struct nfs_page *freq, *t; 826 struct nfs_commit_info cinfo; 827 struct inode *inode = &nfsi->vfs_inode; 828 829 nfs_init_cinfo_from_inode(&cinfo, inode); 830 831 /* search through pnfs commit lists */ 832 freq = pnfs_search_commit_reqs(inode, &cinfo, page); 833 if (freq) 834 return freq->wb_head; 835 836 /* Linearly search the commit list for the correct request */ 837 list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) { 838 if (freq->wb_page == page) 839 return freq->wb_head; 840 } 841 842 return NULL; 843 } 844 845 /** 846 * nfs_request_add_commit_list_locked - add request to a commit list 847 * @req: pointer to a struct nfs_page 848 * @dst: commit list head 849 * @cinfo: holds list lock and accounting info 850 * 851 * This sets the PG_CLEAN bit, updates the cinfo count of 852 * number of outstanding requests requiring a commit as well as 853 * the MM page stats. 854 * 855 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the 856 * nfs_page lock. 857 */ 858 void 859 nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst, 860 struct nfs_commit_info *cinfo) 861 { 862 set_bit(PG_CLEAN, &req->wb_flags); 863 nfs_list_add_request(req, dst); 864 atomic_long_inc(&cinfo->mds->ncommit); 865 } 866 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked); 867 868 /** 869 * nfs_request_add_commit_list - add request to a commit list 870 * @req: pointer to a struct nfs_page 871 * @cinfo: holds list lock and accounting info 872 * 873 * This sets the PG_CLEAN bit, updates the cinfo count of 874 * number of outstanding requests requiring a commit as well as 875 * the MM page stats. 876 * 877 * The caller must _not_ hold the cinfo->lock, but must be 878 * holding the nfs_page lock. 879 */ 880 void 881 nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo) 882 { 883 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); 884 nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo); 885 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); 886 if (req->wb_page) 887 nfs_mark_page_unstable(req->wb_page, cinfo); 888 } 889 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); 890 891 /** 892 * nfs_request_remove_commit_list - Remove request from a commit list 893 * @req: pointer to a nfs_page 894 * @cinfo: holds list lock and accounting info 895 * 896 * This clears the PG_CLEAN bit, and updates the cinfo's count of 897 * number of outstanding requests requiring a commit 898 * It does not update the MM page stats. 899 * 900 * The caller _must_ hold the cinfo->lock and the nfs_page lock. 901 */ 902 void 903 nfs_request_remove_commit_list(struct nfs_page *req, 904 struct nfs_commit_info *cinfo) 905 { 906 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags)) 907 return; 908 nfs_list_remove_request(req); 909 atomic_long_dec(&cinfo->mds->ncommit); 910 } 911 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list); 912 913 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 914 struct inode *inode) 915 { 916 cinfo->inode = inode; 917 cinfo->mds = &NFS_I(inode)->commit_info; 918 cinfo->ds = pnfs_get_ds_info(inode); 919 cinfo->dreq = NULL; 920 cinfo->completion_ops = &nfs_commit_completion_ops; 921 } 922 923 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 924 struct inode *inode, 925 struct nfs_direct_req *dreq) 926 { 927 if (dreq) 928 nfs_init_cinfo_from_dreq(cinfo, dreq); 929 else 930 nfs_init_cinfo_from_inode(cinfo, inode); 931 } 932 EXPORT_SYMBOL_GPL(nfs_init_cinfo); 933 934 /* 935 * Add a request to the inode's commit list. 936 */ 937 void 938 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 939 struct nfs_commit_info *cinfo, u32 ds_commit_idx) 940 { 941 if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx)) 942 return; 943 nfs_request_add_commit_list(req, cinfo); 944 } 945 946 static void 947 nfs_clear_page_commit(struct page *page) 948 { 949 dec_node_page_state(page, NR_WRITEBACK); 950 dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb, 951 WB_WRITEBACK); 952 } 953 954 /* Called holding the request lock on @req */ 955 static void 956 nfs_clear_request_commit(struct nfs_page *req) 957 { 958 if (test_bit(PG_CLEAN, &req->wb_flags)) { 959 struct nfs_open_context *ctx = nfs_req_openctx(req); 960 struct inode *inode = d_inode(ctx->dentry); 961 struct nfs_commit_info cinfo; 962 963 nfs_init_cinfo_from_inode(&cinfo, inode); 964 mutex_lock(&NFS_I(inode)->commit_mutex); 965 if (!pnfs_clear_request_commit(req, &cinfo)) { 966 nfs_request_remove_commit_list(req, &cinfo); 967 } 968 mutex_unlock(&NFS_I(inode)->commit_mutex); 969 nfs_clear_page_commit(req->wb_page); 970 } 971 } 972 973 int nfs_write_need_commit(struct nfs_pgio_header *hdr) 974 { 975 if (hdr->verf.committed == NFS_DATA_SYNC) 976 return hdr->lseg == NULL; 977 return hdr->verf.committed != NFS_FILE_SYNC; 978 } 979 980 static void nfs_async_write_init(struct nfs_pgio_header *hdr) 981 { 982 nfs_io_completion_get(hdr->io_completion); 983 } 984 985 static void nfs_write_completion(struct nfs_pgio_header *hdr) 986 { 987 struct nfs_commit_info cinfo; 988 unsigned long bytes = 0; 989 990 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) 991 goto out; 992 nfs_init_cinfo_from_inode(&cinfo, hdr->inode); 993 while (!list_empty(&hdr->pages)) { 994 struct nfs_page *req = nfs_list_entry(hdr->pages.next); 995 996 bytes += req->wb_bytes; 997 nfs_list_remove_request(req); 998 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && 999 (hdr->good_bytes < bytes)) { 1000 trace_nfs_comp_error(req, hdr->error); 1001 nfs_mapping_set_error(req->wb_page, hdr->error); 1002 goto remove_req; 1003 } 1004 if (nfs_write_need_commit(hdr)) { 1005 /* Reset wb_nio, since the write was successful. */ 1006 req->wb_nio = 0; 1007 memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf)); 1008 nfs_mark_request_commit(req, hdr->lseg, &cinfo, 1009 hdr->pgio_mirror_idx); 1010 goto next; 1011 } 1012 remove_req: 1013 nfs_inode_remove_request(req); 1014 next: 1015 nfs_end_page_writeback(req); 1016 nfs_release_request(req); 1017 } 1018 out: 1019 nfs_io_completion_put(hdr->io_completion); 1020 hdr->release(hdr); 1021 } 1022 1023 unsigned long 1024 nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 1025 { 1026 return atomic_long_read(&cinfo->mds->ncommit); 1027 } 1028 1029 /* NFS_I(cinfo->inode)->commit_mutex held by caller */ 1030 int 1031 nfs_scan_commit_list(struct list_head *src, struct list_head *dst, 1032 struct nfs_commit_info *cinfo, int max) 1033 { 1034 struct nfs_page *req, *tmp; 1035 int ret = 0; 1036 1037 restart: 1038 list_for_each_entry_safe(req, tmp, src, wb_list) { 1039 kref_get(&req->wb_kref); 1040 if (!nfs_lock_request(req)) { 1041 int status; 1042 1043 /* Prevent deadlock with nfs_lock_and_join_requests */ 1044 if (!list_empty(dst)) { 1045 nfs_release_request(req); 1046 continue; 1047 } 1048 /* Ensure we make progress to prevent livelock */ 1049 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); 1050 status = nfs_wait_on_request(req); 1051 nfs_release_request(req); 1052 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); 1053 if (status < 0) 1054 break; 1055 goto restart; 1056 } 1057 nfs_request_remove_commit_list(req, cinfo); 1058 clear_bit(PG_COMMIT_TO_DS, &req->wb_flags); 1059 nfs_list_add_request(req, dst); 1060 ret++; 1061 if ((ret == max) && !cinfo->dreq) 1062 break; 1063 cond_resched(); 1064 } 1065 return ret; 1066 } 1067 EXPORT_SYMBOL_GPL(nfs_scan_commit_list); 1068 1069 /* 1070 * nfs_scan_commit - Scan an inode for commit requests 1071 * @inode: NFS inode to scan 1072 * @dst: mds destination list 1073 * @cinfo: mds and ds lists of reqs ready to commit 1074 * 1075 * Moves requests from the inode's 'commit' request list. 1076 * The requests are *not* checked to ensure that they form a contiguous set. 1077 */ 1078 int 1079 nfs_scan_commit(struct inode *inode, struct list_head *dst, 1080 struct nfs_commit_info *cinfo) 1081 { 1082 int ret = 0; 1083 1084 if (!atomic_long_read(&cinfo->mds->ncommit)) 1085 return 0; 1086 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); 1087 if (atomic_long_read(&cinfo->mds->ncommit) > 0) { 1088 const int max = INT_MAX; 1089 1090 ret = nfs_scan_commit_list(&cinfo->mds->list, dst, 1091 cinfo, max); 1092 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret); 1093 } 1094 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); 1095 return ret; 1096 } 1097 1098 /* 1099 * Search for an existing write request, and attempt to update 1100 * it to reflect a new dirty region on a given page. 1101 * 1102 * If the attempt fails, then the existing request is flushed out 1103 * to disk. 1104 */ 1105 static struct nfs_page *nfs_try_to_update_request(struct inode *inode, 1106 struct page *page, 1107 unsigned int offset, 1108 unsigned int bytes) 1109 { 1110 struct nfs_page *req; 1111 unsigned int rqend; 1112 unsigned int end; 1113 int error; 1114 1115 end = offset + bytes; 1116 1117 req = nfs_lock_and_join_requests(page); 1118 if (IS_ERR_OR_NULL(req)) 1119 return req; 1120 1121 rqend = req->wb_offset + req->wb_bytes; 1122 /* 1123 * Tell the caller to flush out the request if 1124 * the offsets are non-contiguous. 1125 * Note: nfs_flush_incompatible() will already 1126 * have flushed out requests having wrong owners. 1127 */ 1128 if (offset > rqend || end < req->wb_offset) 1129 goto out_flushme; 1130 1131 /* Okay, the request matches. Update the region */ 1132 if (offset < req->wb_offset) { 1133 req->wb_offset = offset; 1134 req->wb_pgbase = offset; 1135 } 1136 if (end > rqend) 1137 req->wb_bytes = end - req->wb_offset; 1138 else 1139 req->wb_bytes = rqend - req->wb_offset; 1140 req->wb_nio = 0; 1141 return req; 1142 out_flushme: 1143 /* 1144 * Note: we mark the request dirty here because 1145 * nfs_lock_and_join_requests() cannot preserve 1146 * commit flags, so we have to replay the write. 1147 */ 1148 nfs_mark_request_dirty(req); 1149 nfs_unlock_and_release_request(req); 1150 error = nfs_wb_page(inode, page); 1151 return (error < 0) ? ERR_PTR(error) : NULL; 1152 } 1153 1154 /* 1155 * Try to update an existing write request, or create one if there is none. 1156 * 1157 * Note: Should always be called with the Page Lock held to prevent races 1158 * if we have to add a new request. Also assumes that the caller has 1159 * already called nfs_flush_incompatible() if necessary. 1160 */ 1161 static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx, 1162 struct page *page, unsigned int offset, unsigned int bytes) 1163 { 1164 struct inode *inode = page_file_mapping(page)->host; 1165 struct nfs_page *req; 1166 1167 req = nfs_try_to_update_request(inode, page, offset, bytes); 1168 if (req != NULL) 1169 goto out; 1170 req = nfs_create_request(ctx, page, offset, bytes); 1171 if (IS_ERR(req)) 1172 goto out; 1173 nfs_inode_add_request(inode, req); 1174 out: 1175 return req; 1176 } 1177 1178 static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page, 1179 unsigned int offset, unsigned int count) 1180 { 1181 struct nfs_page *req; 1182 1183 req = nfs_setup_write_request(ctx, page, offset, count); 1184 if (IS_ERR(req)) 1185 return PTR_ERR(req); 1186 /* Update file length */ 1187 nfs_grow_file(page, offset, count); 1188 nfs_mark_uptodate(req); 1189 nfs_mark_request_dirty(req); 1190 nfs_unlock_and_release_request(req); 1191 return 0; 1192 } 1193 1194 int nfs_flush_incompatible(struct file *file, struct page *page) 1195 { 1196 struct nfs_open_context *ctx = nfs_file_open_context(file); 1197 struct nfs_lock_context *l_ctx; 1198 struct file_lock_context *flctx = file_inode(file)->i_flctx; 1199 struct nfs_page *req; 1200 int do_flush, status; 1201 /* 1202 * Look for a request corresponding to this page. If there 1203 * is one, and it belongs to another file, we flush it out 1204 * before we try to copy anything into the page. Do this 1205 * due to the lack of an ACCESS-type call in NFSv2. 1206 * Also do the same if we find a request from an existing 1207 * dropped page. 1208 */ 1209 do { 1210 req = nfs_page_find_head_request(page); 1211 if (req == NULL) 1212 return 0; 1213 l_ctx = req->wb_lock_context; 1214 do_flush = req->wb_page != page || 1215 !nfs_match_open_context(nfs_req_openctx(req), ctx); 1216 if (l_ctx && flctx && 1217 !(list_empty_careful(&flctx->flc_posix) && 1218 list_empty_careful(&flctx->flc_flock))) { 1219 do_flush |= l_ctx->lockowner != current->files; 1220 } 1221 nfs_release_request(req); 1222 if (!do_flush) 1223 return 0; 1224 status = nfs_wb_page(page_file_mapping(page)->host, page); 1225 } while (status == 0); 1226 return status; 1227 } 1228 1229 /* 1230 * Avoid buffered writes when a open context credential's key would 1231 * expire soon. 1232 * 1233 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL. 1234 * 1235 * Return 0 and set a credential flag which triggers the inode to flush 1236 * and performs NFS_FILE_SYNC writes if the key will expired within 1237 * RPC_KEY_EXPIRE_TIMEO. 1238 */ 1239 int 1240 nfs_key_timeout_notify(struct file *filp, struct inode *inode) 1241 { 1242 struct nfs_open_context *ctx = nfs_file_open_context(filp); 1243 1244 if (nfs_ctx_key_to_expire(ctx, inode) && 1245 !ctx->ll_cred) 1246 /* Already expired! */ 1247 return -EACCES; 1248 return 0; 1249 } 1250 1251 /* 1252 * Test if the open context credential key is marked to expire soon. 1253 */ 1254 bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode) 1255 { 1256 struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth; 1257 struct rpc_cred *cred = ctx->ll_cred; 1258 struct auth_cred acred = { 1259 .cred = ctx->cred, 1260 }; 1261 1262 if (cred && !cred->cr_ops->crmatch(&acred, cred, 0)) { 1263 put_rpccred(cred); 1264 ctx->ll_cred = NULL; 1265 cred = NULL; 1266 } 1267 if (!cred) 1268 cred = auth->au_ops->lookup_cred(auth, &acred, 0); 1269 if (!cred || IS_ERR(cred)) 1270 return true; 1271 ctx->ll_cred = cred; 1272 return !!(cred->cr_ops->crkey_timeout && 1273 cred->cr_ops->crkey_timeout(cred)); 1274 } 1275 1276 /* 1277 * If the page cache is marked as unsafe or invalid, then we can't rely on 1278 * the PageUptodate() flag. In this case, we will need to turn off 1279 * write optimisations that depend on the page contents being correct. 1280 */ 1281 static bool nfs_write_pageuptodate(struct page *page, struct inode *inode, 1282 unsigned int pagelen) 1283 { 1284 struct nfs_inode *nfsi = NFS_I(inode); 1285 1286 if (nfs_have_delegated_attributes(inode)) 1287 goto out; 1288 if (nfsi->cache_validity & 1289 (NFS_INO_REVAL_PAGECACHE | NFS_INO_INVALID_SIZE)) 1290 return false; 1291 smp_rmb(); 1292 if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags) && pagelen != 0) 1293 return false; 1294 out: 1295 if (nfsi->cache_validity & NFS_INO_INVALID_DATA && pagelen != 0) 1296 return false; 1297 return PageUptodate(page) != 0; 1298 } 1299 1300 static bool 1301 is_whole_file_wrlock(struct file_lock *fl) 1302 { 1303 return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX && 1304 fl->fl_type == F_WRLCK; 1305 } 1306 1307 /* If we know the page is up to date, and we're not using byte range locks (or 1308 * if we have the whole file locked for writing), it may be more efficient to 1309 * extend the write to cover the entire page in order to avoid fragmentation 1310 * inefficiencies. 1311 * 1312 * If the file is opened for synchronous writes then we can just skip the rest 1313 * of the checks. 1314 */ 1315 static int nfs_can_extend_write(struct file *file, struct page *page, 1316 struct inode *inode, unsigned int pagelen) 1317 { 1318 int ret; 1319 struct file_lock_context *flctx = inode->i_flctx; 1320 struct file_lock *fl; 1321 1322 if (file->f_flags & O_DSYNC) 1323 return 0; 1324 if (!nfs_write_pageuptodate(page, inode, pagelen)) 1325 return 0; 1326 if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) 1327 return 1; 1328 if (!flctx || (list_empty_careful(&flctx->flc_flock) && 1329 list_empty_careful(&flctx->flc_posix))) 1330 return 1; 1331 1332 /* Check to see if there are whole file write locks */ 1333 ret = 0; 1334 spin_lock(&flctx->flc_lock); 1335 if (!list_empty(&flctx->flc_posix)) { 1336 fl = list_first_entry(&flctx->flc_posix, struct file_lock, 1337 fl_list); 1338 if (is_whole_file_wrlock(fl)) 1339 ret = 1; 1340 } else if (!list_empty(&flctx->flc_flock)) { 1341 fl = list_first_entry(&flctx->flc_flock, struct file_lock, 1342 fl_list); 1343 if (fl->fl_type == F_WRLCK) 1344 ret = 1; 1345 } 1346 spin_unlock(&flctx->flc_lock); 1347 return ret; 1348 } 1349 1350 /* 1351 * Update and possibly write a cached page of an NFS file. 1352 * 1353 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad 1354 * things with a page scheduled for an RPC call (e.g. invalidate it). 1355 */ 1356 int nfs_updatepage(struct file *file, struct page *page, 1357 unsigned int offset, unsigned int count) 1358 { 1359 struct nfs_open_context *ctx = nfs_file_open_context(file); 1360 struct address_space *mapping = page_file_mapping(page); 1361 struct inode *inode = mapping->host; 1362 unsigned int pagelen = nfs_page_length(page); 1363 int status = 0; 1364 1365 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE); 1366 1367 dprintk("NFS: nfs_updatepage(%pD2 %d@%lld)\n", 1368 file, count, (long long)(page_file_offset(page) + offset)); 1369 1370 if (!count) 1371 goto out; 1372 1373 if (nfs_can_extend_write(file, page, inode, pagelen)) { 1374 count = max(count + offset, pagelen); 1375 offset = 0; 1376 } 1377 1378 status = nfs_writepage_setup(ctx, page, offset, count); 1379 if (status < 0) 1380 nfs_set_pageerror(mapping); 1381 else 1382 __set_page_dirty_nobuffers(page); 1383 out: 1384 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n", 1385 status, (long long)i_size_read(inode)); 1386 return status; 1387 } 1388 1389 static int flush_task_priority(int how) 1390 { 1391 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) { 1392 case FLUSH_HIGHPRI: 1393 return RPC_PRIORITY_HIGH; 1394 case FLUSH_LOWPRI: 1395 return RPC_PRIORITY_LOW; 1396 } 1397 return RPC_PRIORITY_NORMAL; 1398 } 1399 1400 static void nfs_initiate_write(struct nfs_pgio_header *hdr, 1401 struct rpc_message *msg, 1402 const struct nfs_rpc_ops *rpc_ops, 1403 struct rpc_task_setup *task_setup_data, int how) 1404 { 1405 int priority = flush_task_priority(how); 1406 1407 task_setup_data->priority = priority; 1408 rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client); 1409 trace_nfs_initiate_write(hdr); 1410 } 1411 1412 /* If a nfs_flush_* function fails, it should remove reqs from @head and 1413 * call this on each, which will prepare them to be retried on next 1414 * writeback using standard nfs. 1415 */ 1416 static void nfs_redirty_request(struct nfs_page *req) 1417 { 1418 /* Bump the transmission count */ 1419 req->wb_nio++; 1420 nfs_mark_request_dirty(req); 1421 set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags); 1422 nfs_end_page_writeback(req); 1423 nfs_release_request(req); 1424 } 1425 1426 static void nfs_async_write_error(struct list_head *head, int error) 1427 { 1428 struct nfs_page *req; 1429 1430 while (!list_empty(head)) { 1431 req = nfs_list_entry(head->next); 1432 nfs_list_remove_request(req); 1433 if (nfs_error_is_fatal(error)) 1434 nfs_write_error(req, error); 1435 else 1436 nfs_redirty_request(req); 1437 } 1438 } 1439 1440 static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr) 1441 { 1442 nfs_async_write_error(&hdr->pages, 0); 1443 filemap_fdatawrite_range(hdr->inode->i_mapping, hdr->args.offset, 1444 hdr->args.offset + hdr->args.count - 1); 1445 } 1446 1447 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = { 1448 .init_hdr = nfs_async_write_init, 1449 .error_cleanup = nfs_async_write_error, 1450 .completion = nfs_write_completion, 1451 .reschedule_io = nfs_async_write_reschedule_io, 1452 }; 1453 1454 void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, 1455 struct inode *inode, int ioflags, bool force_mds, 1456 const struct nfs_pgio_completion_ops *compl_ops) 1457 { 1458 struct nfs_server *server = NFS_SERVER(inode); 1459 const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops; 1460 1461 #ifdef CONFIG_NFS_V4_1 1462 if (server->pnfs_curr_ld && !force_mds) 1463 pg_ops = server->pnfs_curr_ld->pg_write_ops; 1464 #endif 1465 nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops, 1466 server->wsize, ioflags); 1467 } 1468 EXPORT_SYMBOL_GPL(nfs_pageio_init_write); 1469 1470 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio) 1471 { 1472 struct nfs_pgio_mirror *mirror; 1473 1474 if (pgio->pg_ops && pgio->pg_ops->pg_cleanup) 1475 pgio->pg_ops->pg_cleanup(pgio); 1476 1477 pgio->pg_ops = &nfs_pgio_rw_ops; 1478 1479 nfs_pageio_stop_mirroring(pgio); 1480 1481 mirror = &pgio->pg_mirrors[0]; 1482 mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize; 1483 } 1484 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds); 1485 1486 1487 void nfs_commit_prepare(struct rpc_task *task, void *calldata) 1488 { 1489 struct nfs_commit_data *data = calldata; 1490 1491 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data); 1492 } 1493 1494 /* 1495 * Special version of should_remove_suid() that ignores capabilities. 1496 */ 1497 static int nfs_should_remove_suid(const struct inode *inode) 1498 { 1499 umode_t mode = inode->i_mode; 1500 int kill = 0; 1501 1502 /* suid always must be killed */ 1503 if (unlikely(mode & S_ISUID)) 1504 kill = ATTR_KILL_SUID; 1505 1506 /* 1507 * sgid without any exec bits is just a mandatory locking mark; leave 1508 * it alone. If some exec bits are set, it's a real sgid; kill it. 1509 */ 1510 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) 1511 kill |= ATTR_KILL_SGID; 1512 1513 if (unlikely(kill && S_ISREG(mode))) 1514 return kill; 1515 1516 return 0; 1517 } 1518 1519 static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr, 1520 struct nfs_fattr *fattr) 1521 { 1522 struct nfs_pgio_args *argp = &hdr->args; 1523 struct nfs_pgio_res *resp = &hdr->res; 1524 u64 size = argp->offset + resp->count; 1525 1526 if (!(fattr->valid & NFS_ATTR_FATTR_SIZE)) 1527 fattr->size = size; 1528 if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) { 1529 fattr->valid &= ~NFS_ATTR_FATTR_SIZE; 1530 return; 1531 } 1532 if (size != fattr->size) 1533 return; 1534 /* Set attribute barrier */ 1535 nfs_fattr_set_barrier(fattr); 1536 /* ...and update size */ 1537 fattr->valid |= NFS_ATTR_FATTR_SIZE; 1538 } 1539 1540 void nfs_writeback_update_inode(struct nfs_pgio_header *hdr) 1541 { 1542 struct nfs_fattr *fattr = &hdr->fattr; 1543 struct inode *inode = hdr->inode; 1544 1545 spin_lock(&inode->i_lock); 1546 nfs_writeback_check_extend(hdr, fattr); 1547 nfs_post_op_update_inode_force_wcc_locked(inode, fattr); 1548 spin_unlock(&inode->i_lock); 1549 } 1550 EXPORT_SYMBOL_GPL(nfs_writeback_update_inode); 1551 1552 /* 1553 * This function is called when the WRITE call is complete. 1554 */ 1555 static int nfs_writeback_done(struct rpc_task *task, 1556 struct nfs_pgio_header *hdr, 1557 struct inode *inode) 1558 { 1559 int status; 1560 1561 /* 1562 * ->write_done will attempt to use post-op attributes to detect 1563 * conflicting writes by other clients. A strict interpretation 1564 * of close-to-open would allow us to continue caching even if 1565 * another writer had changed the file, but some applications 1566 * depend on tighter cache coherency when writing. 1567 */ 1568 status = NFS_PROTO(inode)->write_done(task, hdr); 1569 if (status != 0) 1570 return status; 1571 1572 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count); 1573 trace_nfs_writeback_done(task, hdr); 1574 1575 if (hdr->res.verf->committed < hdr->args.stable && 1576 task->tk_status >= 0) { 1577 /* We tried a write call, but the server did not 1578 * commit data to stable storage even though we 1579 * requested it. 1580 * Note: There is a known bug in Tru64 < 5.0 in which 1581 * the server reports NFS_DATA_SYNC, but performs 1582 * NFS_FILE_SYNC. We therefore implement this checking 1583 * as a dprintk() in order to avoid filling syslog. 1584 */ 1585 static unsigned long complain; 1586 1587 /* Note this will print the MDS for a DS write */ 1588 if (time_before(complain, jiffies)) { 1589 dprintk("NFS: faulty NFS server %s:" 1590 " (committed = %d) != (stable = %d)\n", 1591 NFS_SERVER(inode)->nfs_client->cl_hostname, 1592 hdr->res.verf->committed, hdr->args.stable); 1593 complain = jiffies + 300 * HZ; 1594 } 1595 } 1596 1597 /* Deal with the suid/sgid bit corner case */ 1598 if (nfs_should_remove_suid(inode)) { 1599 spin_lock(&inode->i_lock); 1600 NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER; 1601 spin_unlock(&inode->i_lock); 1602 } 1603 return 0; 1604 } 1605 1606 /* 1607 * This function is called when the WRITE call is complete. 1608 */ 1609 static void nfs_writeback_result(struct rpc_task *task, 1610 struct nfs_pgio_header *hdr) 1611 { 1612 struct nfs_pgio_args *argp = &hdr->args; 1613 struct nfs_pgio_res *resp = &hdr->res; 1614 1615 if (resp->count < argp->count) { 1616 static unsigned long complain; 1617 1618 /* This a short write! */ 1619 nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE); 1620 1621 /* Has the server at least made some progress? */ 1622 if (resp->count == 0) { 1623 if (time_before(complain, jiffies)) { 1624 printk(KERN_WARNING 1625 "NFS: Server wrote zero bytes, expected %u.\n", 1626 argp->count); 1627 complain = jiffies + 300 * HZ; 1628 } 1629 nfs_set_pgio_error(hdr, -EIO, argp->offset); 1630 task->tk_status = -EIO; 1631 return; 1632 } 1633 1634 /* For non rpc-based layout drivers, retry-through-MDS */ 1635 if (!task->tk_ops) { 1636 hdr->pnfs_error = -EAGAIN; 1637 return; 1638 } 1639 1640 /* Was this an NFSv2 write or an NFSv3 stable write? */ 1641 if (resp->verf->committed != NFS_UNSTABLE) { 1642 /* Resend from where the server left off */ 1643 hdr->mds_offset += resp->count; 1644 argp->offset += resp->count; 1645 argp->pgbase += resp->count; 1646 argp->count -= resp->count; 1647 } else { 1648 /* Resend as a stable write in order to avoid 1649 * headaches in the case of a server crash. 1650 */ 1651 argp->stable = NFS_FILE_SYNC; 1652 } 1653 resp->count = 0; 1654 resp->verf->committed = 0; 1655 rpc_restart_call_prepare(task); 1656 } 1657 } 1658 1659 static int wait_on_commit(struct nfs_mds_commit_info *cinfo) 1660 { 1661 return wait_var_event_killable(&cinfo->rpcs_out, 1662 !atomic_read(&cinfo->rpcs_out)); 1663 } 1664 1665 static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo) 1666 { 1667 atomic_inc(&cinfo->rpcs_out); 1668 } 1669 1670 static void nfs_commit_end(struct nfs_mds_commit_info *cinfo) 1671 { 1672 if (atomic_dec_and_test(&cinfo->rpcs_out)) 1673 wake_up_var(&cinfo->rpcs_out); 1674 } 1675 1676 void nfs_commitdata_release(struct nfs_commit_data *data) 1677 { 1678 put_nfs_open_context(data->context); 1679 nfs_commit_free(data); 1680 } 1681 EXPORT_SYMBOL_GPL(nfs_commitdata_release); 1682 1683 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, 1684 const struct nfs_rpc_ops *nfs_ops, 1685 const struct rpc_call_ops *call_ops, 1686 int how, int flags) 1687 { 1688 struct rpc_task *task; 1689 int priority = flush_task_priority(how); 1690 struct rpc_message msg = { 1691 .rpc_argp = &data->args, 1692 .rpc_resp = &data->res, 1693 .rpc_cred = data->cred, 1694 }; 1695 struct rpc_task_setup task_setup_data = { 1696 .task = &data->task, 1697 .rpc_client = clnt, 1698 .rpc_message = &msg, 1699 .callback_ops = call_ops, 1700 .callback_data = data, 1701 .workqueue = nfsiod_workqueue, 1702 .flags = RPC_TASK_ASYNC | flags, 1703 .priority = priority, 1704 }; 1705 /* Set up the initial task struct. */ 1706 nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client); 1707 trace_nfs_initiate_commit(data); 1708 1709 dprintk("NFS: initiated commit call\n"); 1710 1711 task = rpc_run_task(&task_setup_data); 1712 if (IS_ERR(task)) 1713 return PTR_ERR(task); 1714 if (how & FLUSH_SYNC) 1715 rpc_wait_for_completion_task(task); 1716 rpc_put_task(task); 1717 return 0; 1718 } 1719 EXPORT_SYMBOL_GPL(nfs_initiate_commit); 1720 1721 static loff_t nfs_get_lwb(struct list_head *head) 1722 { 1723 loff_t lwb = 0; 1724 struct nfs_page *req; 1725 1726 list_for_each_entry(req, head, wb_list) 1727 if (lwb < (req_offset(req) + req->wb_bytes)) 1728 lwb = req_offset(req) + req->wb_bytes; 1729 1730 return lwb; 1731 } 1732 1733 /* 1734 * Set up the argument/result storage required for the RPC call. 1735 */ 1736 void nfs_init_commit(struct nfs_commit_data *data, 1737 struct list_head *head, 1738 struct pnfs_layout_segment *lseg, 1739 struct nfs_commit_info *cinfo) 1740 { 1741 struct nfs_page *first; 1742 struct nfs_open_context *ctx; 1743 struct inode *inode; 1744 1745 /* Set up the RPC argument and reply structs 1746 * NB: take care not to mess about with data->commit et al. */ 1747 1748 if (head) 1749 list_splice_init(head, &data->pages); 1750 1751 first = nfs_list_entry(data->pages.next); 1752 ctx = nfs_req_openctx(first); 1753 inode = d_inode(ctx->dentry); 1754 1755 data->inode = inode; 1756 data->cred = ctx->cred; 1757 data->lseg = lseg; /* reference transferred */ 1758 /* only set lwb for pnfs commit */ 1759 if (lseg) 1760 data->lwb = nfs_get_lwb(&data->pages); 1761 data->mds_ops = &nfs_commit_ops; 1762 data->completion_ops = cinfo->completion_ops; 1763 data->dreq = cinfo->dreq; 1764 1765 data->args.fh = NFS_FH(data->inode); 1766 /* Note: we always request a commit of the entire inode */ 1767 data->args.offset = 0; 1768 data->args.count = 0; 1769 data->context = get_nfs_open_context(ctx); 1770 data->res.fattr = &data->fattr; 1771 data->res.verf = &data->verf; 1772 nfs_fattr_init(&data->fattr); 1773 } 1774 EXPORT_SYMBOL_GPL(nfs_init_commit); 1775 1776 void nfs_retry_commit(struct list_head *page_list, 1777 struct pnfs_layout_segment *lseg, 1778 struct nfs_commit_info *cinfo, 1779 u32 ds_commit_idx) 1780 { 1781 struct nfs_page *req; 1782 1783 while (!list_empty(page_list)) { 1784 req = nfs_list_entry(page_list->next); 1785 nfs_list_remove_request(req); 1786 nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx); 1787 if (!cinfo->dreq) 1788 nfs_clear_page_commit(req->wb_page); 1789 nfs_unlock_and_release_request(req); 1790 } 1791 } 1792 EXPORT_SYMBOL_GPL(nfs_retry_commit); 1793 1794 static void 1795 nfs_commit_resched_write(struct nfs_commit_info *cinfo, 1796 struct nfs_page *req) 1797 { 1798 __set_page_dirty_nobuffers(req->wb_page); 1799 } 1800 1801 /* 1802 * Commit dirty pages 1803 */ 1804 static int 1805 nfs_commit_list(struct inode *inode, struct list_head *head, int how, 1806 struct nfs_commit_info *cinfo) 1807 { 1808 struct nfs_commit_data *data; 1809 1810 /* another commit raced with us */ 1811 if (list_empty(head)) 1812 return 0; 1813 1814 data = nfs_commitdata_alloc(true); 1815 1816 /* Set up the argument struct */ 1817 nfs_init_commit(data, head, NULL, cinfo); 1818 atomic_inc(&cinfo->mds->rpcs_out); 1819 return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode), 1820 data->mds_ops, how, RPC_TASK_CRED_NOREF); 1821 } 1822 1823 /* 1824 * COMMIT call returned 1825 */ 1826 static void nfs_commit_done(struct rpc_task *task, void *calldata) 1827 { 1828 struct nfs_commit_data *data = calldata; 1829 1830 dprintk("NFS: %5u nfs_commit_done (status %d)\n", 1831 task->tk_pid, task->tk_status); 1832 1833 /* Call the NFS version-specific code */ 1834 NFS_PROTO(data->inode)->commit_done(task, data); 1835 trace_nfs_commit_done(task, data); 1836 } 1837 1838 static void nfs_commit_release_pages(struct nfs_commit_data *data) 1839 { 1840 const struct nfs_writeverf *verf = data->res.verf; 1841 struct nfs_page *req; 1842 int status = data->task.tk_status; 1843 struct nfs_commit_info cinfo; 1844 struct nfs_server *nfss; 1845 1846 while (!list_empty(&data->pages)) { 1847 req = nfs_list_entry(data->pages.next); 1848 nfs_list_remove_request(req); 1849 if (req->wb_page) 1850 nfs_clear_page_commit(req->wb_page); 1851 1852 dprintk("NFS: commit (%s/%llu %d@%lld)", 1853 nfs_req_openctx(req)->dentry->d_sb->s_id, 1854 (unsigned long long)NFS_FILEID(d_inode(nfs_req_openctx(req)->dentry)), 1855 req->wb_bytes, 1856 (long long)req_offset(req)); 1857 if (status < 0) { 1858 if (req->wb_page) { 1859 trace_nfs_commit_error(req, status); 1860 nfs_mapping_set_error(req->wb_page, status); 1861 nfs_inode_remove_request(req); 1862 } 1863 dprintk_cont(", error = %d\n", status); 1864 goto next; 1865 } 1866 1867 /* Okay, COMMIT succeeded, apparently. Check the verifier 1868 * returned by the server against all stored verfs. */ 1869 if (nfs_write_match_verf(verf, req)) { 1870 /* We have a match */ 1871 if (req->wb_page) 1872 nfs_inode_remove_request(req); 1873 dprintk_cont(" OK\n"); 1874 goto next; 1875 } 1876 /* We have a mismatch. Write the page again */ 1877 dprintk_cont(" mismatch\n"); 1878 nfs_mark_request_dirty(req); 1879 set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags); 1880 next: 1881 nfs_unlock_and_release_request(req); 1882 /* Latency breaker */ 1883 cond_resched(); 1884 } 1885 nfss = NFS_SERVER(data->inode); 1886 if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 1887 clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC); 1888 1889 nfs_init_cinfo(&cinfo, data->inode, data->dreq); 1890 nfs_commit_end(cinfo.mds); 1891 } 1892 1893 static void nfs_commit_release(void *calldata) 1894 { 1895 struct nfs_commit_data *data = calldata; 1896 1897 data->completion_ops->completion(data); 1898 nfs_commitdata_release(calldata); 1899 } 1900 1901 static const struct rpc_call_ops nfs_commit_ops = { 1902 .rpc_call_prepare = nfs_commit_prepare, 1903 .rpc_call_done = nfs_commit_done, 1904 .rpc_release = nfs_commit_release, 1905 }; 1906 1907 static const struct nfs_commit_completion_ops nfs_commit_completion_ops = { 1908 .completion = nfs_commit_release_pages, 1909 .resched_write = nfs_commit_resched_write, 1910 }; 1911 1912 int nfs_generic_commit_list(struct inode *inode, struct list_head *head, 1913 int how, struct nfs_commit_info *cinfo) 1914 { 1915 int status; 1916 1917 status = pnfs_commit_list(inode, head, how, cinfo); 1918 if (status == PNFS_NOT_ATTEMPTED) 1919 status = nfs_commit_list(inode, head, how, cinfo); 1920 return status; 1921 } 1922 1923 static int __nfs_commit_inode(struct inode *inode, int how, 1924 struct writeback_control *wbc) 1925 { 1926 LIST_HEAD(head); 1927 struct nfs_commit_info cinfo; 1928 int may_wait = how & FLUSH_SYNC; 1929 int ret, nscan; 1930 1931 nfs_init_cinfo_from_inode(&cinfo, inode); 1932 nfs_commit_begin(cinfo.mds); 1933 for (;;) { 1934 ret = nscan = nfs_scan_commit(inode, &head, &cinfo); 1935 if (ret <= 0) 1936 break; 1937 ret = nfs_generic_commit_list(inode, &head, how, &cinfo); 1938 if (ret < 0) 1939 break; 1940 ret = 0; 1941 if (wbc && wbc->sync_mode == WB_SYNC_NONE) { 1942 if (nscan < wbc->nr_to_write) 1943 wbc->nr_to_write -= nscan; 1944 else 1945 wbc->nr_to_write = 0; 1946 } 1947 if (nscan < INT_MAX) 1948 break; 1949 cond_resched(); 1950 } 1951 nfs_commit_end(cinfo.mds); 1952 if (ret || !may_wait) 1953 return ret; 1954 return wait_on_commit(cinfo.mds); 1955 } 1956 1957 int nfs_commit_inode(struct inode *inode, int how) 1958 { 1959 return __nfs_commit_inode(inode, how, NULL); 1960 } 1961 EXPORT_SYMBOL_GPL(nfs_commit_inode); 1962 1963 int nfs_write_inode(struct inode *inode, struct writeback_control *wbc) 1964 { 1965 struct nfs_inode *nfsi = NFS_I(inode); 1966 int flags = FLUSH_SYNC; 1967 int ret = 0; 1968 1969 if (wbc->sync_mode == WB_SYNC_NONE) { 1970 /* no commits means nothing needs to be done */ 1971 if (!atomic_long_read(&nfsi->commit_info.ncommit)) 1972 goto check_requests_outstanding; 1973 1974 /* Don't commit yet if this is a non-blocking flush and there 1975 * are a lot of outstanding writes for this mapping. 1976 */ 1977 if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) 1978 goto out_mark_dirty; 1979 1980 /* don't wait for the COMMIT response */ 1981 flags = 0; 1982 } 1983 1984 ret = __nfs_commit_inode(inode, flags, wbc); 1985 if (!ret) { 1986 if (flags & FLUSH_SYNC) 1987 return 0; 1988 } else if (atomic_long_read(&nfsi->commit_info.ncommit)) 1989 goto out_mark_dirty; 1990 1991 check_requests_outstanding: 1992 if (!atomic_read(&nfsi->commit_info.rpcs_out)) 1993 return ret; 1994 out_mark_dirty: 1995 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1996 return ret; 1997 } 1998 EXPORT_SYMBOL_GPL(nfs_write_inode); 1999 2000 /* 2001 * Wrapper for filemap_write_and_wait_range() 2002 * 2003 * Needed for pNFS in order to ensure data becomes visible to the 2004 * client. 2005 */ 2006 int nfs_filemap_write_and_wait_range(struct address_space *mapping, 2007 loff_t lstart, loff_t lend) 2008 { 2009 int ret; 2010 2011 ret = filemap_write_and_wait_range(mapping, lstart, lend); 2012 if (ret == 0) 2013 ret = pnfs_sync_inode(mapping->host, true); 2014 return ret; 2015 } 2016 EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range); 2017 2018 /* 2019 * flush the inode to disk. 2020 */ 2021 int nfs_wb_all(struct inode *inode) 2022 { 2023 int ret; 2024 2025 trace_nfs_writeback_inode_enter(inode); 2026 2027 ret = filemap_write_and_wait(inode->i_mapping); 2028 if (ret) 2029 goto out; 2030 ret = nfs_commit_inode(inode, FLUSH_SYNC); 2031 if (ret < 0) 2032 goto out; 2033 pnfs_sync_inode(inode, true); 2034 ret = 0; 2035 2036 out: 2037 trace_nfs_writeback_inode_exit(inode, ret); 2038 return ret; 2039 } 2040 EXPORT_SYMBOL_GPL(nfs_wb_all); 2041 2042 int nfs_wb_page_cancel(struct inode *inode, struct page *page) 2043 { 2044 struct nfs_page *req; 2045 int ret = 0; 2046 2047 wait_on_page_writeback(page); 2048 2049 /* blocking call to cancel all requests and join to a single (head) 2050 * request */ 2051 req = nfs_lock_and_join_requests(page); 2052 2053 if (IS_ERR(req)) { 2054 ret = PTR_ERR(req); 2055 } else if (req) { 2056 /* all requests from this page have been cancelled by 2057 * nfs_lock_and_join_requests, so just remove the head 2058 * request from the inode / page_private pointer and 2059 * release it */ 2060 nfs_inode_remove_request(req); 2061 nfs_unlock_and_release_request(req); 2062 } 2063 2064 return ret; 2065 } 2066 2067 /* 2068 * Write back all requests on one page - we do this before reading it. 2069 */ 2070 int nfs_wb_page(struct inode *inode, struct page *page) 2071 { 2072 loff_t range_start = page_file_offset(page); 2073 loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1); 2074 struct writeback_control wbc = { 2075 .sync_mode = WB_SYNC_ALL, 2076 .nr_to_write = 0, 2077 .range_start = range_start, 2078 .range_end = range_end, 2079 }; 2080 int ret; 2081 2082 trace_nfs_writeback_page_enter(inode); 2083 2084 for (;;) { 2085 wait_on_page_writeback(page); 2086 if (clear_page_dirty_for_io(page)) { 2087 ret = nfs_writepage_locked(page, &wbc); 2088 if (ret < 0) 2089 goto out_error; 2090 continue; 2091 } 2092 ret = 0; 2093 if (!PagePrivate(page)) 2094 break; 2095 ret = nfs_commit_inode(inode, FLUSH_SYNC); 2096 if (ret < 0) 2097 goto out_error; 2098 } 2099 out_error: 2100 trace_nfs_writeback_page_exit(inode, ret); 2101 return ret; 2102 } 2103 2104 #ifdef CONFIG_MIGRATION 2105 int nfs_migrate_page(struct address_space *mapping, struct page *newpage, 2106 struct page *page, enum migrate_mode mode) 2107 { 2108 /* 2109 * If PagePrivate is set, then the page is currently associated with 2110 * an in-progress read or write request. Don't try to migrate it. 2111 * 2112 * FIXME: we could do this in principle, but we'll need a way to ensure 2113 * that we can safely release the inode reference while holding 2114 * the page lock. 2115 */ 2116 if (PagePrivate(page)) 2117 return -EBUSY; 2118 2119 if (!nfs_fscache_release_page(page, GFP_KERNEL)) 2120 return -EBUSY; 2121 2122 return migrate_page(mapping, newpage, page, mode); 2123 } 2124 #endif 2125 2126 int __init nfs_init_writepagecache(void) 2127 { 2128 nfs_wdata_cachep = kmem_cache_create("nfs_write_data", 2129 sizeof(struct nfs_pgio_header), 2130 0, SLAB_HWCACHE_ALIGN, 2131 NULL); 2132 if (nfs_wdata_cachep == NULL) 2133 return -ENOMEM; 2134 2135 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE, 2136 nfs_wdata_cachep); 2137 if (nfs_wdata_mempool == NULL) 2138 goto out_destroy_write_cache; 2139 2140 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data", 2141 sizeof(struct nfs_commit_data), 2142 0, SLAB_HWCACHE_ALIGN, 2143 NULL); 2144 if (nfs_cdata_cachep == NULL) 2145 goto out_destroy_write_mempool; 2146 2147 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT, 2148 nfs_cdata_cachep); 2149 if (nfs_commit_mempool == NULL) 2150 goto out_destroy_commit_cache; 2151 2152 /* 2153 * NFS congestion size, scale with available memory. 2154 * 2155 * 64MB: 8192k 2156 * 128MB: 11585k 2157 * 256MB: 16384k 2158 * 512MB: 23170k 2159 * 1GB: 32768k 2160 * 2GB: 46340k 2161 * 4GB: 65536k 2162 * 8GB: 92681k 2163 * 16GB: 131072k 2164 * 2165 * This allows larger machines to have larger/more transfers. 2166 * Limit the default to 256M 2167 */ 2168 nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10); 2169 if (nfs_congestion_kb > 256*1024) 2170 nfs_congestion_kb = 256*1024; 2171 2172 return 0; 2173 2174 out_destroy_commit_cache: 2175 kmem_cache_destroy(nfs_cdata_cachep); 2176 out_destroy_write_mempool: 2177 mempool_destroy(nfs_wdata_mempool); 2178 out_destroy_write_cache: 2179 kmem_cache_destroy(nfs_wdata_cachep); 2180 return -ENOMEM; 2181 } 2182 2183 void nfs_destroy_writepagecache(void) 2184 { 2185 mempool_destroy(nfs_commit_mempool); 2186 kmem_cache_destroy(nfs_cdata_cachep); 2187 mempool_destroy(nfs_wdata_mempool); 2188 kmem_cache_destroy(nfs_wdata_cachep); 2189 } 2190 2191 static const struct nfs_rw_ops nfs_rw_write_ops = { 2192 .rw_alloc_header = nfs_writehdr_alloc, 2193 .rw_free_header = nfs_writehdr_free, 2194 .rw_done = nfs_writeback_done, 2195 .rw_result = nfs_writeback_result, 2196 .rw_initiate = nfs_initiate_write, 2197 }; 2198