1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/fs/nfs/write.c 4 * 5 * Write file data over NFS. 6 * 7 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de> 8 */ 9 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/mm.h> 13 #include <linux/pagemap.h> 14 #include <linux/file.h> 15 #include <linux/writeback.h> 16 #include <linux/swap.h> 17 #include <linux/migrate.h> 18 19 #include <linux/sunrpc/clnt.h> 20 #include <linux/nfs_fs.h> 21 #include <linux/nfs_mount.h> 22 #include <linux/nfs_page.h> 23 #include <linux/backing-dev.h> 24 #include <linux/export.h> 25 #include <linux/freezer.h> 26 #include <linux/wait.h> 27 #include <linux/iversion.h> 28 29 #include <linux/uaccess.h> 30 #include <linux/sched/mm.h> 31 32 #include "delegation.h" 33 #include "internal.h" 34 #include "iostat.h" 35 #include "nfs4_fs.h" 36 #include "fscache.h" 37 #include "pnfs.h" 38 39 #include "nfstrace.h" 40 41 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 42 43 #define MIN_POOL_WRITE (32) 44 #define MIN_POOL_COMMIT (4) 45 46 struct nfs_io_completion { 47 void (*complete)(void *data); 48 void *data; 49 struct kref refcount; 50 }; 51 52 /* 53 * Local function declarations 54 */ 55 static void nfs_redirty_request(struct nfs_page *req); 56 static const struct rpc_call_ops nfs_commit_ops; 57 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops; 58 static const struct nfs_commit_completion_ops nfs_commit_completion_ops; 59 static const struct nfs_rw_ops nfs_rw_write_ops; 60 static void nfs_inode_remove_request(struct nfs_page *req); 61 static void nfs_clear_request_commit(struct nfs_page *req); 62 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 63 struct inode *inode); 64 static struct nfs_page * 65 nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi, 66 struct page *page); 67 68 static struct kmem_cache *nfs_wdata_cachep; 69 static mempool_t *nfs_wdata_mempool; 70 static struct kmem_cache *nfs_cdata_cachep; 71 static mempool_t *nfs_commit_mempool; 72 73 struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail) 74 { 75 struct nfs_commit_data *p; 76 77 if (never_fail) 78 p = mempool_alloc(nfs_commit_mempool, GFP_NOIO); 79 else { 80 /* It is OK to do some reclaim, not no safe to wait 81 * for anything to be returned to the pool. 82 * mempool_alloc() cannot handle that particular combination, 83 * so we need two separate attempts. 84 */ 85 p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT); 86 if (!p) 87 p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO | 88 __GFP_NOWARN | __GFP_NORETRY); 89 if (!p) 90 return NULL; 91 } 92 93 memset(p, 0, sizeof(*p)); 94 INIT_LIST_HEAD(&p->pages); 95 return p; 96 } 97 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); 98 99 void nfs_commit_free(struct nfs_commit_data *p) 100 { 101 mempool_free(p, nfs_commit_mempool); 102 } 103 EXPORT_SYMBOL_GPL(nfs_commit_free); 104 105 static struct nfs_pgio_header *nfs_writehdr_alloc(void) 106 { 107 struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_KERNEL); 108 109 memset(p, 0, sizeof(*p)); 110 p->rw_mode = FMODE_WRITE; 111 return p; 112 } 113 114 static void nfs_writehdr_free(struct nfs_pgio_header *hdr) 115 { 116 mempool_free(hdr, nfs_wdata_mempool); 117 } 118 119 static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags) 120 { 121 return kmalloc(sizeof(struct nfs_io_completion), gfp_flags); 122 } 123 124 static void nfs_io_completion_init(struct nfs_io_completion *ioc, 125 void (*complete)(void *), void *data) 126 { 127 ioc->complete = complete; 128 ioc->data = data; 129 kref_init(&ioc->refcount); 130 } 131 132 static void nfs_io_completion_release(struct kref *kref) 133 { 134 struct nfs_io_completion *ioc = container_of(kref, 135 struct nfs_io_completion, refcount); 136 ioc->complete(ioc->data); 137 kfree(ioc); 138 } 139 140 static void nfs_io_completion_get(struct nfs_io_completion *ioc) 141 { 142 if (ioc != NULL) 143 kref_get(&ioc->refcount); 144 } 145 146 static void nfs_io_completion_put(struct nfs_io_completion *ioc) 147 { 148 if (ioc != NULL) 149 kref_put(&ioc->refcount, nfs_io_completion_release); 150 } 151 152 static void 153 nfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode) 154 { 155 if (!test_and_set_bit(PG_INODE_REF, &req->wb_flags)) { 156 kref_get(&req->wb_kref); 157 atomic_long_inc(&NFS_I(inode)->nrequests); 158 } 159 } 160 161 static int 162 nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode) 163 { 164 int ret; 165 166 if (!test_bit(PG_REMOVE, &req->wb_flags)) 167 return 0; 168 ret = nfs_page_group_lock(req); 169 if (ret) 170 return ret; 171 if (test_and_clear_bit(PG_REMOVE, &req->wb_flags)) 172 nfs_page_set_inode_ref(req, inode); 173 nfs_page_group_unlock(req); 174 return 0; 175 } 176 177 static struct nfs_page * 178 nfs_page_private_request(struct page *page) 179 { 180 if (!PagePrivate(page)) 181 return NULL; 182 return (struct nfs_page *)page_private(page); 183 } 184 185 /* 186 * nfs_page_find_head_request_locked - find head request associated with @page 187 * 188 * must be called while holding the inode lock. 189 * 190 * returns matching head request with reference held, or NULL if not found. 191 */ 192 static struct nfs_page * 193 nfs_page_find_private_request(struct page *page) 194 { 195 struct address_space *mapping = page_file_mapping(page); 196 struct nfs_page *req; 197 198 if (!PagePrivate(page)) 199 return NULL; 200 spin_lock(&mapping->private_lock); 201 req = nfs_page_private_request(page); 202 if (req) { 203 WARN_ON_ONCE(req->wb_head != req); 204 kref_get(&req->wb_kref); 205 } 206 spin_unlock(&mapping->private_lock); 207 return req; 208 } 209 210 static struct nfs_page * 211 nfs_page_find_swap_request(struct page *page) 212 { 213 struct inode *inode = page_file_mapping(page)->host; 214 struct nfs_inode *nfsi = NFS_I(inode); 215 struct nfs_page *req = NULL; 216 if (!PageSwapCache(page)) 217 return NULL; 218 mutex_lock(&nfsi->commit_mutex); 219 if (PageSwapCache(page)) { 220 req = nfs_page_search_commits_for_head_request_locked(nfsi, 221 page); 222 if (req) { 223 WARN_ON_ONCE(req->wb_head != req); 224 kref_get(&req->wb_kref); 225 } 226 } 227 mutex_unlock(&nfsi->commit_mutex); 228 return req; 229 } 230 231 /* 232 * nfs_page_find_head_request - find head request associated with @page 233 * 234 * returns matching head request with reference held, or NULL if not found. 235 */ 236 static struct nfs_page *nfs_page_find_head_request(struct page *page) 237 { 238 struct nfs_page *req; 239 240 req = nfs_page_find_private_request(page); 241 if (!req) 242 req = nfs_page_find_swap_request(page); 243 return req; 244 } 245 246 static struct nfs_page *nfs_find_and_lock_page_request(struct page *page) 247 { 248 struct inode *inode = page_file_mapping(page)->host; 249 struct nfs_page *req, *head; 250 int ret; 251 252 for (;;) { 253 req = nfs_page_find_head_request(page); 254 if (!req) 255 return req; 256 head = nfs_page_group_lock_head(req); 257 if (head != req) 258 nfs_release_request(req); 259 if (IS_ERR(head)) 260 return head; 261 ret = nfs_cancel_remove_inode(head, inode); 262 if (ret < 0) { 263 nfs_unlock_and_release_request(head); 264 return ERR_PTR(ret); 265 } 266 /* Ensure that nobody removed the request before we locked it */ 267 if (head == nfs_page_private_request(page)) 268 break; 269 if (PageSwapCache(page)) 270 break; 271 nfs_unlock_and_release_request(head); 272 } 273 return head; 274 } 275 276 /* Adjust the file length if we're writing beyond the end */ 277 static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count) 278 { 279 struct inode *inode = page_file_mapping(page)->host; 280 loff_t end, i_size; 281 pgoff_t end_index; 282 283 spin_lock(&inode->i_lock); 284 i_size = i_size_read(inode); 285 end_index = (i_size - 1) >> PAGE_SHIFT; 286 if (i_size > 0 && page_index(page) < end_index) 287 goto out; 288 end = page_file_offset(page) + ((loff_t)offset+count); 289 if (i_size >= end) 290 goto out; 291 i_size_write(inode, end); 292 NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; 293 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); 294 out: 295 spin_unlock(&inode->i_lock); 296 } 297 298 /* A writeback failed: mark the page as bad, and invalidate the page cache */ 299 static void nfs_set_pageerror(struct address_space *mapping) 300 { 301 struct inode *inode = mapping->host; 302 303 nfs_zap_mapping(mapping->host, mapping); 304 /* Force file size revalidation */ 305 spin_lock(&inode->i_lock); 306 nfs_set_cache_invalid(inode, NFS_INO_REVAL_FORCED | 307 NFS_INO_REVAL_PAGECACHE | 308 NFS_INO_INVALID_SIZE); 309 spin_unlock(&inode->i_lock); 310 } 311 312 static void nfs_mapping_set_error(struct page *page, int error) 313 { 314 struct address_space *mapping = page_file_mapping(page); 315 316 SetPageError(page); 317 filemap_set_wb_err(mapping, error); 318 if (mapping->host) 319 errseq_set(&mapping->host->i_sb->s_wb_err, 320 error == -ENOSPC ? -ENOSPC : -EIO); 321 nfs_set_pageerror(mapping); 322 } 323 324 /* 325 * nfs_page_group_search_locked 326 * @head - head request of page group 327 * @page_offset - offset into page 328 * 329 * Search page group with head @head to find a request that contains the 330 * page offset @page_offset. 331 * 332 * Returns a pointer to the first matching nfs request, or NULL if no 333 * match is found. 334 * 335 * Must be called with the page group lock held 336 */ 337 static struct nfs_page * 338 nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset) 339 { 340 struct nfs_page *req; 341 342 req = head; 343 do { 344 if (page_offset >= req->wb_pgbase && 345 page_offset < (req->wb_pgbase + req->wb_bytes)) 346 return req; 347 348 req = req->wb_this_page; 349 } while (req != head); 350 351 return NULL; 352 } 353 354 /* 355 * nfs_page_group_covers_page 356 * @head - head request of page group 357 * 358 * Return true if the page group with head @head covers the whole page, 359 * returns false otherwise 360 */ 361 static bool nfs_page_group_covers_page(struct nfs_page *req) 362 { 363 struct nfs_page *tmp; 364 unsigned int pos = 0; 365 unsigned int len = nfs_page_length(req->wb_page); 366 367 nfs_page_group_lock(req); 368 369 for (;;) { 370 tmp = nfs_page_group_search_locked(req->wb_head, pos); 371 if (!tmp) 372 break; 373 pos = tmp->wb_pgbase + tmp->wb_bytes; 374 } 375 376 nfs_page_group_unlock(req); 377 return pos >= len; 378 } 379 380 /* We can set the PG_uptodate flag if we see that a write request 381 * covers the full page. 382 */ 383 static void nfs_mark_uptodate(struct nfs_page *req) 384 { 385 if (PageUptodate(req->wb_page)) 386 return; 387 if (!nfs_page_group_covers_page(req)) 388 return; 389 SetPageUptodate(req->wb_page); 390 } 391 392 static int wb_priority(struct writeback_control *wbc) 393 { 394 int ret = 0; 395 396 if (wbc->sync_mode == WB_SYNC_ALL) 397 ret = FLUSH_COND_STABLE; 398 return ret; 399 } 400 401 /* 402 * NFS congestion control 403 */ 404 405 int nfs_congestion_kb; 406 407 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10)) 408 #define NFS_CONGESTION_OFF_THRESH \ 409 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) 410 411 static void nfs_set_page_writeback(struct page *page) 412 { 413 struct inode *inode = page_file_mapping(page)->host; 414 struct nfs_server *nfss = NFS_SERVER(inode); 415 int ret = test_set_page_writeback(page); 416 417 WARN_ON_ONCE(ret != 0); 418 419 if (atomic_long_inc_return(&nfss->writeback) > 420 NFS_CONGESTION_ON_THRESH) 421 set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 422 } 423 424 static void nfs_end_page_writeback(struct nfs_page *req) 425 { 426 struct inode *inode = page_file_mapping(req->wb_page)->host; 427 struct nfs_server *nfss = NFS_SERVER(inode); 428 bool is_done; 429 430 is_done = nfs_page_group_sync_on_bit(req, PG_WB_END); 431 nfs_unlock_request(req); 432 if (!is_done) 433 return; 434 435 end_page_writeback(req->wb_page); 436 if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 437 clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC); 438 } 439 440 /* 441 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests 442 * 443 * @destroy_list - request list (using wb_this_page) terminated by @old_head 444 * @old_head - the old head of the list 445 * 446 * All subrequests must be locked and removed from all lists, so at this point 447 * they are only "active" in this function, and possibly in nfs_wait_on_request 448 * with a reference held by some other context. 449 */ 450 static void 451 nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, 452 struct nfs_page *old_head, 453 struct inode *inode) 454 { 455 while (destroy_list) { 456 struct nfs_page *subreq = destroy_list; 457 458 destroy_list = (subreq->wb_this_page == old_head) ? 459 NULL : subreq->wb_this_page; 460 461 /* Note: lock subreq in order to change subreq->wb_head */ 462 nfs_page_set_headlock(subreq); 463 WARN_ON_ONCE(old_head != subreq->wb_head); 464 465 /* make sure old group is not used */ 466 subreq->wb_this_page = subreq; 467 subreq->wb_head = subreq; 468 469 clear_bit(PG_REMOVE, &subreq->wb_flags); 470 471 /* Note: races with nfs_page_group_destroy() */ 472 if (!kref_read(&subreq->wb_kref)) { 473 /* Check if we raced with nfs_page_group_destroy() */ 474 if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) { 475 nfs_page_clear_headlock(subreq); 476 nfs_free_request(subreq); 477 } else 478 nfs_page_clear_headlock(subreq); 479 continue; 480 } 481 nfs_page_clear_headlock(subreq); 482 483 nfs_release_request(old_head); 484 485 if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) { 486 nfs_release_request(subreq); 487 atomic_long_dec(&NFS_I(inode)->nrequests); 488 } 489 490 /* subreq is now totally disconnected from page group or any 491 * write / commit lists. last chance to wake any waiters */ 492 nfs_unlock_and_release_request(subreq); 493 } 494 } 495 496 /* 497 * nfs_join_page_group - destroy subrequests of the head req 498 * @head: the page used to lookup the "page group" of nfs_page structures 499 * @inode: Inode to which the request belongs. 500 * 501 * This function joins all sub requests to the head request by first 502 * locking all requests in the group, cancelling any pending operations 503 * and finally updating the head request to cover the whole range covered by 504 * the (former) group. All subrequests are removed from any write or commit 505 * lists, unlinked from the group and destroyed. 506 */ 507 void 508 nfs_join_page_group(struct nfs_page *head, struct inode *inode) 509 { 510 struct nfs_page *subreq; 511 struct nfs_page *destroy_list = NULL; 512 unsigned int pgbase, off, bytes; 513 514 pgbase = head->wb_pgbase; 515 bytes = head->wb_bytes; 516 off = head->wb_offset; 517 for (subreq = head->wb_this_page; subreq != head; 518 subreq = subreq->wb_this_page) { 519 /* Subrequests should always form a contiguous range */ 520 if (pgbase > subreq->wb_pgbase) { 521 off -= pgbase - subreq->wb_pgbase; 522 bytes += pgbase - subreq->wb_pgbase; 523 pgbase = subreq->wb_pgbase; 524 } 525 bytes = max(subreq->wb_pgbase + subreq->wb_bytes 526 - pgbase, bytes); 527 } 528 529 /* Set the head request's range to cover the former page group */ 530 head->wb_pgbase = pgbase; 531 head->wb_bytes = bytes; 532 head->wb_offset = off; 533 534 /* Now that all requests are locked, make sure they aren't on any list. 535 * Commit list removal accounting is done after locks are dropped */ 536 subreq = head; 537 do { 538 nfs_clear_request_commit(subreq); 539 subreq = subreq->wb_this_page; 540 } while (subreq != head); 541 542 /* unlink subrequests from head, destroy them later */ 543 if (head->wb_this_page != head) { 544 /* destroy list will be terminated by head */ 545 destroy_list = head->wb_this_page; 546 head->wb_this_page = head; 547 } 548 549 nfs_destroy_unlinked_subrequests(destroy_list, head, inode); 550 } 551 552 /* 553 * nfs_lock_and_join_requests - join all subreqs to the head req 554 * @page: the page used to lookup the "page group" of nfs_page structures 555 * 556 * This function joins all sub requests to the head request by first 557 * locking all requests in the group, cancelling any pending operations 558 * and finally updating the head request to cover the whole range covered by 559 * the (former) group. All subrequests are removed from any write or commit 560 * lists, unlinked from the group and destroyed. 561 * 562 * Returns a locked, referenced pointer to the head request - which after 563 * this call is guaranteed to be the only request associated with the page. 564 * Returns NULL if no requests are found for @page, or a ERR_PTR if an 565 * error was encountered. 566 */ 567 static struct nfs_page * 568 nfs_lock_and_join_requests(struct page *page) 569 { 570 struct inode *inode = page_file_mapping(page)->host; 571 struct nfs_page *head; 572 int ret; 573 574 /* 575 * A reference is taken only on the head request which acts as a 576 * reference to the whole page group - the group will not be destroyed 577 * until the head reference is released. 578 */ 579 head = nfs_find_and_lock_page_request(page); 580 if (IS_ERR_OR_NULL(head)) 581 return head; 582 583 /* lock each request in the page group */ 584 ret = nfs_page_group_lock_subrequests(head); 585 if (ret < 0) { 586 nfs_unlock_and_release_request(head); 587 return ERR_PTR(ret); 588 } 589 590 nfs_join_page_group(head, inode); 591 592 return head; 593 } 594 595 static void nfs_write_error(struct nfs_page *req, int error) 596 { 597 trace_nfs_write_error(req, error); 598 nfs_mapping_set_error(req->wb_page, error); 599 nfs_inode_remove_request(req); 600 nfs_end_page_writeback(req); 601 nfs_release_request(req); 602 } 603 604 /* 605 * Find an associated nfs write request, and prepare to flush it out 606 * May return an error if the user signalled nfs_wait_on_request(). 607 */ 608 static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio, 609 struct page *page) 610 { 611 struct nfs_page *req; 612 int ret = 0; 613 614 req = nfs_lock_and_join_requests(page); 615 if (!req) 616 goto out; 617 ret = PTR_ERR(req); 618 if (IS_ERR(req)) 619 goto out; 620 621 nfs_set_page_writeback(page); 622 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags)); 623 624 /* If there is a fatal error that covers this write, just exit */ 625 ret = pgio->pg_error; 626 if (nfs_error_is_fatal_on_server(ret)) 627 goto out_launder; 628 629 ret = 0; 630 if (!nfs_pageio_add_request(pgio, req)) { 631 ret = pgio->pg_error; 632 /* 633 * Remove the problematic req upon fatal errors on the server 634 */ 635 if (nfs_error_is_fatal(ret)) { 636 if (nfs_error_is_fatal_on_server(ret)) 637 goto out_launder; 638 } else 639 ret = -EAGAIN; 640 nfs_redirty_request(req); 641 pgio->pg_error = 0; 642 } else 643 nfs_add_stats(page_file_mapping(page)->host, 644 NFSIOS_WRITEPAGES, 1); 645 out: 646 return ret; 647 out_launder: 648 nfs_write_error(req, ret); 649 return 0; 650 } 651 652 static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, 653 struct nfs_pageio_descriptor *pgio) 654 { 655 int ret; 656 657 nfs_pageio_cond_complete(pgio, page_index(page)); 658 ret = nfs_page_async_flush(pgio, page); 659 if (ret == -EAGAIN) { 660 redirty_page_for_writepage(wbc, page); 661 ret = AOP_WRITEPAGE_ACTIVATE; 662 } 663 return ret; 664 } 665 666 /* 667 * Write an mmapped page to the server. 668 */ 669 static int nfs_writepage_locked(struct page *page, 670 struct writeback_control *wbc) 671 { 672 struct nfs_pageio_descriptor pgio; 673 struct inode *inode = page_file_mapping(page)->host; 674 int err; 675 676 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE); 677 nfs_pageio_init_write(&pgio, inode, 0, 678 false, &nfs_async_write_completion_ops); 679 err = nfs_do_writepage(page, wbc, &pgio); 680 pgio.pg_error = 0; 681 nfs_pageio_complete(&pgio); 682 if (err < 0) 683 return err; 684 if (nfs_error_is_fatal(pgio.pg_error)) 685 return pgio.pg_error; 686 return 0; 687 } 688 689 int nfs_writepage(struct page *page, struct writeback_control *wbc) 690 { 691 int ret; 692 693 ret = nfs_writepage_locked(page, wbc); 694 if (ret != AOP_WRITEPAGE_ACTIVATE) 695 unlock_page(page); 696 return ret; 697 } 698 699 static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data) 700 { 701 int ret; 702 703 ret = nfs_do_writepage(page, wbc, data); 704 if (ret != AOP_WRITEPAGE_ACTIVATE) 705 unlock_page(page); 706 return ret; 707 } 708 709 static void nfs_io_completion_commit(void *inode) 710 { 711 nfs_commit_inode(inode, 0); 712 } 713 714 int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) 715 { 716 struct inode *inode = mapping->host; 717 struct nfs_pageio_descriptor pgio; 718 struct nfs_io_completion *ioc = NULL; 719 unsigned int mntflags = NFS_SERVER(inode)->flags; 720 int priority = 0; 721 int err; 722 723 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES); 724 725 if (!(mntflags & NFS_MOUNT_WRITE_EAGER) || wbc->for_kupdate || 726 wbc->for_background || wbc->for_sync || wbc->for_reclaim) { 727 ioc = nfs_io_completion_alloc(GFP_KERNEL); 728 if (ioc) 729 nfs_io_completion_init(ioc, nfs_io_completion_commit, 730 inode); 731 priority = wb_priority(wbc); 732 } 733 734 nfs_pageio_init_write(&pgio, inode, priority, false, 735 &nfs_async_write_completion_ops); 736 pgio.pg_io_completion = ioc; 737 err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio); 738 pgio.pg_error = 0; 739 nfs_pageio_complete(&pgio); 740 nfs_io_completion_put(ioc); 741 742 if (err < 0) 743 goto out_err; 744 err = pgio.pg_error; 745 if (nfs_error_is_fatal(err)) 746 goto out_err; 747 return 0; 748 out_err: 749 return err; 750 } 751 752 /* 753 * Insert a write request into an inode 754 */ 755 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req) 756 { 757 struct address_space *mapping = page_file_mapping(req->wb_page); 758 struct nfs_inode *nfsi = NFS_I(inode); 759 760 WARN_ON_ONCE(req->wb_this_page != req); 761 762 /* Lock the request! */ 763 nfs_lock_request(req); 764 765 /* 766 * Swap-space should not get truncated. Hence no need to plug the race 767 * with invalidate/truncate. 768 */ 769 spin_lock(&mapping->private_lock); 770 if (likely(!PageSwapCache(req->wb_page))) { 771 set_bit(PG_MAPPED, &req->wb_flags); 772 SetPagePrivate(req->wb_page); 773 set_page_private(req->wb_page, (unsigned long)req); 774 } 775 spin_unlock(&mapping->private_lock); 776 atomic_long_inc(&nfsi->nrequests); 777 /* this a head request for a page group - mark it as having an 778 * extra reference so sub groups can follow suit. 779 * This flag also informs pgio layer when to bump nrequests when 780 * adding subrequests. */ 781 WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags)); 782 kref_get(&req->wb_kref); 783 } 784 785 /* 786 * Remove a write request from an inode 787 */ 788 static void nfs_inode_remove_request(struct nfs_page *req) 789 { 790 struct address_space *mapping = page_file_mapping(req->wb_page); 791 struct inode *inode = mapping->host; 792 struct nfs_inode *nfsi = NFS_I(inode); 793 struct nfs_page *head; 794 795 if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) { 796 head = req->wb_head; 797 798 spin_lock(&mapping->private_lock); 799 if (likely(head->wb_page && !PageSwapCache(head->wb_page))) { 800 set_page_private(head->wb_page, 0); 801 ClearPagePrivate(head->wb_page); 802 clear_bit(PG_MAPPED, &head->wb_flags); 803 } 804 spin_unlock(&mapping->private_lock); 805 } 806 807 if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) { 808 nfs_release_request(req); 809 atomic_long_dec(&nfsi->nrequests); 810 } 811 } 812 813 static void 814 nfs_mark_request_dirty(struct nfs_page *req) 815 { 816 if (req->wb_page) 817 __set_page_dirty_nobuffers(req->wb_page); 818 } 819 820 /* 821 * nfs_page_search_commits_for_head_request_locked 822 * 823 * Search through commit lists on @inode for the head request for @page. 824 * Must be called while holding the inode (which is cinfo) lock. 825 * 826 * Returns the head request if found, or NULL if not found. 827 */ 828 static struct nfs_page * 829 nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi, 830 struct page *page) 831 { 832 struct nfs_page *freq, *t; 833 struct nfs_commit_info cinfo; 834 struct inode *inode = &nfsi->vfs_inode; 835 836 nfs_init_cinfo_from_inode(&cinfo, inode); 837 838 /* search through pnfs commit lists */ 839 freq = pnfs_search_commit_reqs(inode, &cinfo, page); 840 if (freq) 841 return freq->wb_head; 842 843 /* Linearly search the commit list for the correct request */ 844 list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) { 845 if (freq->wb_page == page) 846 return freq->wb_head; 847 } 848 849 return NULL; 850 } 851 852 /** 853 * nfs_request_add_commit_list_locked - add request to a commit list 854 * @req: pointer to a struct nfs_page 855 * @dst: commit list head 856 * @cinfo: holds list lock and accounting info 857 * 858 * This sets the PG_CLEAN bit, updates the cinfo count of 859 * number of outstanding requests requiring a commit as well as 860 * the MM page stats. 861 * 862 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the 863 * nfs_page lock. 864 */ 865 void 866 nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst, 867 struct nfs_commit_info *cinfo) 868 { 869 set_bit(PG_CLEAN, &req->wb_flags); 870 nfs_list_add_request(req, dst); 871 atomic_long_inc(&cinfo->mds->ncommit); 872 } 873 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked); 874 875 /** 876 * nfs_request_add_commit_list - add request to a commit list 877 * @req: pointer to a struct nfs_page 878 * @cinfo: holds list lock and accounting info 879 * 880 * This sets the PG_CLEAN bit, updates the cinfo count of 881 * number of outstanding requests requiring a commit as well as 882 * the MM page stats. 883 * 884 * The caller must _not_ hold the cinfo->lock, but must be 885 * holding the nfs_page lock. 886 */ 887 void 888 nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo) 889 { 890 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); 891 nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo); 892 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); 893 if (req->wb_page) 894 nfs_mark_page_unstable(req->wb_page, cinfo); 895 } 896 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); 897 898 /** 899 * nfs_request_remove_commit_list - Remove request from a commit list 900 * @req: pointer to a nfs_page 901 * @cinfo: holds list lock and accounting info 902 * 903 * This clears the PG_CLEAN bit, and updates the cinfo's count of 904 * number of outstanding requests requiring a commit 905 * It does not update the MM page stats. 906 * 907 * The caller _must_ hold the cinfo->lock and the nfs_page lock. 908 */ 909 void 910 nfs_request_remove_commit_list(struct nfs_page *req, 911 struct nfs_commit_info *cinfo) 912 { 913 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags)) 914 return; 915 nfs_list_remove_request(req); 916 atomic_long_dec(&cinfo->mds->ncommit); 917 } 918 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list); 919 920 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 921 struct inode *inode) 922 { 923 cinfo->inode = inode; 924 cinfo->mds = &NFS_I(inode)->commit_info; 925 cinfo->ds = pnfs_get_ds_info(inode); 926 cinfo->dreq = NULL; 927 cinfo->completion_ops = &nfs_commit_completion_ops; 928 } 929 930 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 931 struct inode *inode, 932 struct nfs_direct_req *dreq) 933 { 934 if (dreq) 935 nfs_init_cinfo_from_dreq(cinfo, dreq); 936 else 937 nfs_init_cinfo_from_inode(cinfo, inode); 938 } 939 EXPORT_SYMBOL_GPL(nfs_init_cinfo); 940 941 /* 942 * Add a request to the inode's commit list. 943 */ 944 void 945 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 946 struct nfs_commit_info *cinfo, u32 ds_commit_idx) 947 { 948 if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx)) 949 return; 950 nfs_request_add_commit_list(req, cinfo); 951 } 952 953 static void 954 nfs_clear_page_commit(struct page *page) 955 { 956 dec_node_page_state(page, NR_WRITEBACK); 957 dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb, 958 WB_WRITEBACK); 959 } 960 961 /* Called holding the request lock on @req */ 962 static void 963 nfs_clear_request_commit(struct nfs_page *req) 964 { 965 if (test_bit(PG_CLEAN, &req->wb_flags)) { 966 struct nfs_open_context *ctx = nfs_req_openctx(req); 967 struct inode *inode = d_inode(ctx->dentry); 968 struct nfs_commit_info cinfo; 969 970 nfs_init_cinfo_from_inode(&cinfo, inode); 971 mutex_lock(&NFS_I(inode)->commit_mutex); 972 if (!pnfs_clear_request_commit(req, &cinfo)) { 973 nfs_request_remove_commit_list(req, &cinfo); 974 } 975 mutex_unlock(&NFS_I(inode)->commit_mutex); 976 nfs_clear_page_commit(req->wb_page); 977 } 978 } 979 980 int nfs_write_need_commit(struct nfs_pgio_header *hdr) 981 { 982 if (hdr->verf.committed == NFS_DATA_SYNC) 983 return hdr->lseg == NULL; 984 return hdr->verf.committed != NFS_FILE_SYNC; 985 } 986 987 static void nfs_async_write_init(struct nfs_pgio_header *hdr) 988 { 989 nfs_io_completion_get(hdr->io_completion); 990 } 991 992 static void nfs_write_completion(struct nfs_pgio_header *hdr) 993 { 994 struct nfs_commit_info cinfo; 995 unsigned long bytes = 0; 996 997 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) 998 goto out; 999 nfs_init_cinfo_from_inode(&cinfo, hdr->inode); 1000 while (!list_empty(&hdr->pages)) { 1001 struct nfs_page *req = nfs_list_entry(hdr->pages.next); 1002 1003 bytes += req->wb_bytes; 1004 nfs_list_remove_request(req); 1005 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && 1006 (hdr->good_bytes < bytes)) { 1007 trace_nfs_comp_error(req, hdr->error); 1008 nfs_mapping_set_error(req->wb_page, hdr->error); 1009 goto remove_req; 1010 } 1011 if (nfs_write_need_commit(hdr)) { 1012 /* Reset wb_nio, since the write was successful. */ 1013 req->wb_nio = 0; 1014 memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf)); 1015 nfs_mark_request_commit(req, hdr->lseg, &cinfo, 1016 hdr->pgio_mirror_idx); 1017 goto next; 1018 } 1019 remove_req: 1020 nfs_inode_remove_request(req); 1021 next: 1022 nfs_end_page_writeback(req); 1023 nfs_release_request(req); 1024 } 1025 out: 1026 nfs_io_completion_put(hdr->io_completion); 1027 hdr->release(hdr); 1028 } 1029 1030 unsigned long 1031 nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 1032 { 1033 return atomic_long_read(&cinfo->mds->ncommit); 1034 } 1035 1036 /* NFS_I(cinfo->inode)->commit_mutex held by caller */ 1037 int 1038 nfs_scan_commit_list(struct list_head *src, struct list_head *dst, 1039 struct nfs_commit_info *cinfo, int max) 1040 { 1041 struct nfs_page *req, *tmp; 1042 int ret = 0; 1043 1044 list_for_each_entry_safe(req, tmp, src, wb_list) { 1045 kref_get(&req->wb_kref); 1046 if (!nfs_lock_request(req)) { 1047 nfs_release_request(req); 1048 continue; 1049 } 1050 nfs_request_remove_commit_list(req, cinfo); 1051 clear_bit(PG_COMMIT_TO_DS, &req->wb_flags); 1052 nfs_list_add_request(req, dst); 1053 ret++; 1054 if ((ret == max) && !cinfo->dreq) 1055 break; 1056 cond_resched(); 1057 } 1058 return ret; 1059 } 1060 EXPORT_SYMBOL_GPL(nfs_scan_commit_list); 1061 1062 /* 1063 * nfs_scan_commit - Scan an inode for commit requests 1064 * @inode: NFS inode to scan 1065 * @dst: mds destination list 1066 * @cinfo: mds and ds lists of reqs ready to commit 1067 * 1068 * Moves requests from the inode's 'commit' request list. 1069 * The requests are *not* checked to ensure that they form a contiguous set. 1070 */ 1071 int 1072 nfs_scan_commit(struct inode *inode, struct list_head *dst, 1073 struct nfs_commit_info *cinfo) 1074 { 1075 int ret = 0; 1076 1077 if (!atomic_long_read(&cinfo->mds->ncommit)) 1078 return 0; 1079 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); 1080 if (atomic_long_read(&cinfo->mds->ncommit) > 0) { 1081 const int max = INT_MAX; 1082 1083 ret = nfs_scan_commit_list(&cinfo->mds->list, dst, 1084 cinfo, max); 1085 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret); 1086 } 1087 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); 1088 return ret; 1089 } 1090 1091 /* 1092 * Search for an existing write request, and attempt to update 1093 * it to reflect a new dirty region on a given page. 1094 * 1095 * If the attempt fails, then the existing request is flushed out 1096 * to disk. 1097 */ 1098 static struct nfs_page *nfs_try_to_update_request(struct inode *inode, 1099 struct page *page, 1100 unsigned int offset, 1101 unsigned int bytes) 1102 { 1103 struct nfs_page *req; 1104 unsigned int rqend; 1105 unsigned int end; 1106 int error; 1107 1108 end = offset + bytes; 1109 1110 req = nfs_lock_and_join_requests(page); 1111 if (IS_ERR_OR_NULL(req)) 1112 return req; 1113 1114 rqend = req->wb_offset + req->wb_bytes; 1115 /* 1116 * Tell the caller to flush out the request if 1117 * the offsets are non-contiguous. 1118 * Note: nfs_flush_incompatible() will already 1119 * have flushed out requests having wrong owners. 1120 */ 1121 if (offset > rqend || end < req->wb_offset) 1122 goto out_flushme; 1123 1124 /* Okay, the request matches. Update the region */ 1125 if (offset < req->wb_offset) { 1126 req->wb_offset = offset; 1127 req->wb_pgbase = offset; 1128 } 1129 if (end > rqend) 1130 req->wb_bytes = end - req->wb_offset; 1131 else 1132 req->wb_bytes = rqend - req->wb_offset; 1133 req->wb_nio = 0; 1134 return req; 1135 out_flushme: 1136 /* 1137 * Note: we mark the request dirty here because 1138 * nfs_lock_and_join_requests() cannot preserve 1139 * commit flags, so we have to replay the write. 1140 */ 1141 nfs_mark_request_dirty(req); 1142 nfs_unlock_and_release_request(req); 1143 error = nfs_wb_page(inode, page); 1144 return (error < 0) ? ERR_PTR(error) : NULL; 1145 } 1146 1147 /* 1148 * Try to update an existing write request, or create one if there is none. 1149 * 1150 * Note: Should always be called with the Page Lock held to prevent races 1151 * if we have to add a new request. Also assumes that the caller has 1152 * already called nfs_flush_incompatible() if necessary. 1153 */ 1154 static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx, 1155 struct page *page, unsigned int offset, unsigned int bytes) 1156 { 1157 struct inode *inode = page_file_mapping(page)->host; 1158 struct nfs_page *req; 1159 1160 req = nfs_try_to_update_request(inode, page, offset, bytes); 1161 if (req != NULL) 1162 goto out; 1163 req = nfs_create_request(ctx, page, offset, bytes); 1164 if (IS_ERR(req)) 1165 goto out; 1166 nfs_inode_add_request(inode, req); 1167 out: 1168 return req; 1169 } 1170 1171 static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page, 1172 unsigned int offset, unsigned int count) 1173 { 1174 struct nfs_page *req; 1175 1176 req = nfs_setup_write_request(ctx, page, offset, count); 1177 if (IS_ERR(req)) 1178 return PTR_ERR(req); 1179 /* Update file length */ 1180 nfs_grow_file(page, offset, count); 1181 nfs_mark_uptodate(req); 1182 nfs_mark_request_dirty(req); 1183 nfs_unlock_and_release_request(req); 1184 return 0; 1185 } 1186 1187 int nfs_flush_incompatible(struct file *file, struct page *page) 1188 { 1189 struct nfs_open_context *ctx = nfs_file_open_context(file); 1190 struct nfs_lock_context *l_ctx; 1191 struct file_lock_context *flctx = file_inode(file)->i_flctx; 1192 struct nfs_page *req; 1193 int do_flush, status; 1194 /* 1195 * Look for a request corresponding to this page. If there 1196 * is one, and it belongs to another file, we flush it out 1197 * before we try to copy anything into the page. Do this 1198 * due to the lack of an ACCESS-type call in NFSv2. 1199 * Also do the same if we find a request from an existing 1200 * dropped page. 1201 */ 1202 do { 1203 req = nfs_page_find_head_request(page); 1204 if (req == NULL) 1205 return 0; 1206 l_ctx = req->wb_lock_context; 1207 do_flush = req->wb_page != page || 1208 !nfs_match_open_context(nfs_req_openctx(req), ctx); 1209 if (l_ctx && flctx && 1210 !(list_empty_careful(&flctx->flc_posix) && 1211 list_empty_careful(&flctx->flc_flock))) { 1212 do_flush |= l_ctx->lockowner != current->files; 1213 } 1214 nfs_release_request(req); 1215 if (!do_flush) 1216 return 0; 1217 status = nfs_wb_page(page_file_mapping(page)->host, page); 1218 } while (status == 0); 1219 return status; 1220 } 1221 1222 /* 1223 * Avoid buffered writes when a open context credential's key would 1224 * expire soon. 1225 * 1226 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL. 1227 * 1228 * Return 0 and set a credential flag which triggers the inode to flush 1229 * and performs NFS_FILE_SYNC writes if the key will expired within 1230 * RPC_KEY_EXPIRE_TIMEO. 1231 */ 1232 int 1233 nfs_key_timeout_notify(struct file *filp, struct inode *inode) 1234 { 1235 struct nfs_open_context *ctx = nfs_file_open_context(filp); 1236 1237 if (nfs_ctx_key_to_expire(ctx, inode) && 1238 !ctx->ll_cred) 1239 /* Already expired! */ 1240 return -EACCES; 1241 return 0; 1242 } 1243 1244 /* 1245 * Test if the open context credential key is marked to expire soon. 1246 */ 1247 bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode) 1248 { 1249 struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth; 1250 struct rpc_cred *cred = ctx->ll_cred; 1251 struct auth_cred acred = { 1252 .cred = ctx->cred, 1253 }; 1254 1255 if (cred && !cred->cr_ops->crmatch(&acred, cred, 0)) { 1256 put_rpccred(cred); 1257 ctx->ll_cred = NULL; 1258 cred = NULL; 1259 } 1260 if (!cred) 1261 cred = auth->au_ops->lookup_cred(auth, &acred, 0); 1262 if (!cred || IS_ERR(cred)) 1263 return true; 1264 ctx->ll_cred = cred; 1265 return !!(cred->cr_ops->crkey_timeout && 1266 cred->cr_ops->crkey_timeout(cred)); 1267 } 1268 1269 /* 1270 * If the page cache is marked as unsafe or invalid, then we can't rely on 1271 * the PageUptodate() flag. In this case, we will need to turn off 1272 * write optimisations that depend on the page contents being correct. 1273 */ 1274 static bool nfs_write_pageuptodate(struct page *page, struct inode *inode, 1275 unsigned int pagelen) 1276 { 1277 struct nfs_inode *nfsi = NFS_I(inode); 1278 1279 if (nfs_have_delegated_attributes(inode)) 1280 goto out; 1281 if (nfsi->cache_validity & 1282 (NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE)) 1283 return false; 1284 smp_rmb(); 1285 if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags) && pagelen != 0) 1286 return false; 1287 out: 1288 if (nfsi->cache_validity & NFS_INO_INVALID_DATA && pagelen != 0) 1289 return false; 1290 return PageUptodate(page) != 0; 1291 } 1292 1293 static bool 1294 is_whole_file_wrlock(struct file_lock *fl) 1295 { 1296 return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX && 1297 fl->fl_type == F_WRLCK; 1298 } 1299 1300 /* If we know the page is up to date, and we're not using byte range locks (or 1301 * if we have the whole file locked for writing), it may be more efficient to 1302 * extend the write to cover the entire page in order to avoid fragmentation 1303 * inefficiencies. 1304 * 1305 * If the file is opened for synchronous writes then we can just skip the rest 1306 * of the checks. 1307 */ 1308 static int nfs_can_extend_write(struct file *file, struct page *page, 1309 struct inode *inode, unsigned int pagelen) 1310 { 1311 int ret; 1312 struct file_lock_context *flctx = inode->i_flctx; 1313 struct file_lock *fl; 1314 1315 if (file->f_flags & O_DSYNC) 1316 return 0; 1317 if (!nfs_write_pageuptodate(page, inode, pagelen)) 1318 return 0; 1319 if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) 1320 return 1; 1321 if (!flctx || (list_empty_careful(&flctx->flc_flock) && 1322 list_empty_careful(&flctx->flc_posix))) 1323 return 1; 1324 1325 /* Check to see if there are whole file write locks */ 1326 ret = 0; 1327 spin_lock(&flctx->flc_lock); 1328 if (!list_empty(&flctx->flc_posix)) { 1329 fl = list_first_entry(&flctx->flc_posix, struct file_lock, 1330 fl_list); 1331 if (is_whole_file_wrlock(fl)) 1332 ret = 1; 1333 } else if (!list_empty(&flctx->flc_flock)) { 1334 fl = list_first_entry(&flctx->flc_flock, struct file_lock, 1335 fl_list); 1336 if (fl->fl_type == F_WRLCK) 1337 ret = 1; 1338 } 1339 spin_unlock(&flctx->flc_lock); 1340 return ret; 1341 } 1342 1343 /* 1344 * Update and possibly write a cached page of an NFS file. 1345 * 1346 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad 1347 * things with a page scheduled for an RPC call (e.g. invalidate it). 1348 */ 1349 int nfs_updatepage(struct file *file, struct page *page, 1350 unsigned int offset, unsigned int count) 1351 { 1352 struct nfs_open_context *ctx = nfs_file_open_context(file); 1353 struct address_space *mapping = page_file_mapping(page); 1354 struct inode *inode = mapping->host; 1355 unsigned int pagelen = nfs_page_length(page); 1356 int status = 0; 1357 1358 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE); 1359 1360 dprintk("NFS: nfs_updatepage(%pD2 %d@%lld)\n", 1361 file, count, (long long)(page_file_offset(page) + offset)); 1362 1363 if (!count) 1364 goto out; 1365 1366 if (nfs_can_extend_write(file, page, inode, pagelen)) { 1367 count = max(count + offset, pagelen); 1368 offset = 0; 1369 } 1370 1371 status = nfs_writepage_setup(ctx, page, offset, count); 1372 if (status < 0) 1373 nfs_set_pageerror(mapping); 1374 else 1375 __set_page_dirty_nobuffers(page); 1376 out: 1377 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n", 1378 status, (long long)i_size_read(inode)); 1379 return status; 1380 } 1381 1382 static int flush_task_priority(int how) 1383 { 1384 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) { 1385 case FLUSH_HIGHPRI: 1386 return RPC_PRIORITY_HIGH; 1387 case FLUSH_LOWPRI: 1388 return RPC_PRIORITY_LOW; 1389 } 1390 return RPC_PRIORITY_NORMAL; 1391 } 1392 1393 static void nfs_initiate_write(struct nfs_pgio_header *hdr, 1394 struct rpc_message *msg, 1395 const struct nfs_rpc_ops *rpc_ops, 1396 struct rpc_task_setup *task_setup_data, int how) 1397 { 1398 int priority = flush_task_priority(how); 1399 1400 task_setup_data->priority = priority; 1401 rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client); 1402 trace_nfs_initiate_write(hdr); 1403 } 1404 1405 /* If a nfs_flush_* function fails, it should remove reqs from @head and 1406 * call this on each, which will prepare them to be retried on next 1407 * writeback using standard nfs. 1408 */ 1409 static void nfs_redirty_request(struct nfs_page *req) 1410 { 1411 /* Bump the transmission count */ 1412 req->wb_nio++; 1413 nfs_mark_request_dirty(req); 1414 set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags); 1415 nfs_end_page_writeback(req); 1416 nfs_release_request(req); 1417 } 1418 1419 static void nfs_async_write_error(struct list_head *head, int error) 1420 { 1421 struct nfs_page *req; 1422 1423 while (!list_empty(head)) { 1424 req = nfs_list_entry(head->next); 1425 nfs_list_remove_request(req); 1426 if (nfs_error_is_fatal(error)) 1427 nfs_write_error(req, error); 1428 else 1429 nfs_redirty_request(req); 1430 } 1431 } 1432 1433 static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr) 1434 { 1435 nfs_async_write_error(&hdr->pages, 0); 1436 filemap_fdatawrite_range(hdr->inode->i_mapping, hdr->args.offset, 1437 hdr->args.offset + hdr->args.count - 1); 1438 } 1439 1440 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = { 1441 .init_hdr = nfs_async_write_init, 1442 .error_cleanup = nfs_async_write_error, 1443 .completion = nfs_write_completion, 1444 .reschedule_io = nfs_async_write_reschedule_io, 1445 }; 1446 1447 void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, 1448 struct inode *inode, int ioflags, bool force_mds, 1449 const struct nfs_pgio_completion_ops *compl_ops) 1450 { 1451 struct nfs_server *server = NFS_SERVER(inode); 1452 const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops; 1453 1454 #ifdef CONFIG_NFS_V4_1 1455 if (server->pnfs_curr_ld && !force_mds) 1456 pg_ops = server->pnfs_curr_ld->pg_write_ops; 1457 #endif 1458 nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops, 1459 server->wsize, ioflags); 1460 } 1461 EXPORT_SYMBOL_GPL(nfs_pageio_init_write); 1462 1463 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio) 1464 { 1465 struct nfs_pgio_mirror *mirror; 1466 1467 if (pgio->pg_ops && pgio->pg_ops->pg_cleanup) 1468 pgio->pg_ops->pg_cleanup(pgio); 1469 1470 pgio->pg_ops = &nfs_pgio_rw_ops; 1471 1472 nfs_pageio_stop_mirroring(pgio); 1473 1474 mirror = &pgio->pg_mirrors[0]; 1475 mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize; 1476 } 1477 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds); 1478 1479 1480 void nfs_commit_prepare(struct rpc_task *task, void *calldata) 1481 { 1482 struct nfs_commit_data *data = calldata; 1483 1484 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data); 1485 } 1486 1487 /* 1488 * Special version of should_remove_suid() that ignores capabilities. 1489 */ 1490 static int nfs_should_remove_suid(const struct inode *inode) 1491 { 1492 umode_t mode = inode->i_mode; 1493 int kill = 0; 1494 1495 /* suid always must be killed */ 1496 if (unlikely(mode & S_ISUID)) 1497 kill = ATTR_KILL_SUID; 1498 1499 /* 1500 * sgid without any exec bits is just a mandatory locking mark; leave 1501 * it alone. If some exec bits are set, it's a real sgid; kill it. 1502 */ 1503 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) 1504 kill |= ATTR_KILL_SGID; 1505 1506 if (unlikely(kill && S_ISREG(mode))) 1507 return kill; 1508 1509 return 0; 1510 } 1511 1512 static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr, 1513 struct nfs_fattr *fattr) 1514 { 1515 struct nfs_pgio_args *argp = &hdr->args; 1516 struct nfs_pgio_res *resp = &hdr->res; 1517 u64 size = argp->offset + resp->count; 1518 1519 if (!(fattr->valid & NFS_ATTR_FATTR_SIZE)) 1520 fattr->size = size; 1521 if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) { 1522 fattr->valid &= ~NFS_ATTR_FATTR_SIZE; 1523 return; 1524 } 1525 if (size != fattr->size) 1526 return; 1527 /* Set attribute barrier */ 1528 nfs_fattr_set_barrier(fattr); 1529 /* ...and update size */ 1530 fattr->valid |= NFS_ATTR_FATTR_SIZE; 1531 } 1532 1533 void nfs_writeback_update_inode(struct nfs_pgio_header *hdr) 1534 { 1535 struct nfs_fattr *fattr = &hdr->fattr; 1536 struct inode *inode = hdr->inode; 1537 1538 spin_lock(&inode->i_lock); 1539 nfs_writeback_check_extend(hdr, fattr); 1540 nfs_post_op_update_inode_force_wcc_locked(inode, fattr); 1541 spin_unlock(&inode->i_lock); 1542 } 1543 EXPORT_SYMBOL_GPL(nfs_writeback_update_inode); 1544 1545 /* 1546 * This function is called when the WRITE call is complete. 1547 */ 1548 static int nfs_writeback_done(struct rpc_task *task, 1549 struct nfs_pgio_header *hdr, 1550 struct inode *inode) 1551 { 1552 int status; 1553 1554 /* 1555 * ->write_done will attempt to use post-op attributes to detect 1556 * conflicting writes by other clients. A strict interpretation 1557 * of close-to-open would allow us to continue caching even if 1558 * another writer had changed the file, but some applications 1559 * depend on tighter cache coherency when writing. 1560 */ 1561 status = NFS_PROTO(inode)->write_done(task, hdr); 1562 if (status != 0) 1563 return status; 1564 1565 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count); 1566 trace_nfs_writeback_done(task, hdr); 1567 1568 if (hdr->res.verf->committed < hdr->args.stable && 1569 task->tk_status >= 0) { 1570 /* We tried a write call, but the server did not 1571 * commit data to stable storage even though we 1572 * requested it. 1573 * Note: There is a known bug in Tru64 < 5.0 in which 1574 * the server reports NFS_DATA_SYNC, but performs 1575 * NFS_FILE_SYNC. We therefore implement this checking 1576 * as a dprintk() in order to avoid filling syslog. 1577 */ 1578 static unsigned long complain; 1579 1580 /* Note this will print the MDS for a DS write */ 1581 if (time_before(complain, jiffies)) { 1582 dprintk("NFS: faulty NFS server %s:" 1583 " (committed = %d) != (stable = %d)\n", 1584 NFS_SERVER(inode)->nfs_client->cl_hostname, 1585 hdr->res.verf->committed, hdr->args.stable); 1586 complain = jiffies + 300 * HZ; 1587 } 1588 } 1589 1590 /* Deal with the suid/sgid bit corner case */ 1591 if (nfs_should_remove_suid(inode)) { 1592 spin_lock(&inode->i_lock); 1593 nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE); 1594 spin_unlock(&inode->i_lock); 1595 } 1596 return 0; 1597 } 1598 1599 /* 1600 * This function is called when the WRITE call is complete. 1601 */ 1602 static void nfs_writeback_result(struct rpc_task *task, 1603 struct nfs_pgio_header *hdr) 1604 { 1605 struct nfs_pgio_args *argp = &hdr->args; 1606 struct nfs_pgio_res *resp = &hdr->res; 1607 1608 if (resp->count < argp->count) { 1609 static unsigned long complain; 1610 1611 /* This a short write! */ 1612 nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE); 1613 1614 /* Has the server at least made some progress? */ 1615 if (resp->count == 0) { 1616 if (time_before(complain, jiffies)) { 1617 printk(KERN_WARNING 1618 "NFS: Server wrote zero bytes, expected %u.\n", 1619 argp->count); 1620 complain = jiffies + 300 * HZ; 1621 } 1622 nfs_set_pgio_error(hdr, -EIO, argp->offset); 1623 task->tk_status = -EIO; 1624 return; 1625 } 1626 1627 /* For non rpc-based layout drivers, retry-through-MDS */ 1628 if (!task->tk_ops) { 1629 hdr->pnfs_error = -EAGAIN; 1630 return; 1631 } 1632 1633 /* Was this an NFSv2 write or an NFSv3 stable write? */ 1634 if (resp->verf->committed != NFS_UNSTABLE) { 1635 /* Resend from where the server left off */ 1636 hdr->mds_offset += resp->count; 1637 argp->offset += resp->count; 1638 argp->pgbase += resp->count; 1639 argp->count -= resp->count; 1640 } else { 1641 /* Resend as a stable write in order to avoid 1642 * headaches in the case of a server crash. 1643 */ 1644 argp->stable = NFS_FILE_SYNC; 1645 } 1646 resp->count = 0; 1647 resp->verf->committed = 0; 1648 rpc_restart_call_prepare(task); 1649 } 1650 } 1651 1652 static int wait_on_commit(struct nfs_mds_commit_info *cinfo) 1653 { 1654 return wait_var_event_killable(&cinfo->rpcs_out, 1655 !atomic_read(&cinfo->rpcs_out)); 1656 } 1657 1658 static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo) 1659 { 1660 atomic_inc(&cinfo->rpcs_out); 1661 } 1662 1663 bool nfs_commit_end(struct nfs_mds_commit_info *cinfo) 1664 { 1665 if (atomic_dec_and_test(&cinfo->rpcs_out)) { 1666 wake_up_var(&cinfo->rpcs_out); 1667 return true; 1668 } 1669 return false; 1670 } 1671 1672 void nfs_commitdata_release(struct nfs_commit_data *data) 1673 { 1674 put_nfs_open_context(data->context); 1675 nfs_commit_free(data); 1676 } 1677 EXPORT_SYMBOL_GPL(nfs_commitdata_release); 1678 1679 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, 1680 const struct nfs_rpc_ops *nfs_ops, 1681 const struct rpc_call_ops *call_ops, 1682 int how, int flags) 1683 { 1684 struct rpc_task *task; 1685 int priority = flush_task_priority(how); 1686 struct rpc_message msg = { 1687 .rpc_argp = &data->args, 1688 .rpc_resp = &data->res, 1689 .rpc_cred = data->cred, 1690 }; 1691 struct rpc_task_setup task_setup_data = { 1692 .task = &data->task, 1693 .rpc_client = clnt, 1694 .rpc_message = &msg, 1695 .callback_ops = call_ops, 1696 .callback_data = data, 1697 .workqueue = nfsiod_workqueue, 1698 .flags = RPC_TASK_ASYNC | flags, 1699 .priority = priority, 1700 }; 1701 /* Set up the initial task struct. */ 1702 nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client); 1703 trace_nfs_initiate_commit(data); 1704 1705 dprintk("NFS: initiated commit call\n"); 1706 1707 task = rpc_run_task(&task_setup_data); 1708 if (IS_ERR(task)) 1709 return PTR_ERR(task); 1710 if (how & FLUSH_SYNC) 1711 rpc_wait_for_completion_task(task); 1712 rpc_put_task(task); 1713 return 0; 1714 } 1715 EXPORT_SYMBOL_GPL(nfs_initiate_commit); 1716 1717 static loff_t nfs_get_lwb(struct list_head *head) 1718 { 1719 loff_t lwb = 0; 1720 struct nfs_page *req; 1721 1722 list_for_each_entry(req, head, wb_list) 1723 if (lwb < (req_offset(req) + req->wb_bytes)) 1724 lwb = req_offset(req) + req->wb_bytes; 1725 1726 return lwb; 1727 } 1728 1729 /* 1730 * Set up the argument/result storage required for the RPC call. 1731 */ 1732 void nfs_init_commit(struct nfs_commit_data *data, 1733 struct list_head *head, 1734 struct pnfs_layout_segment *lseg, 1735 struct nfs_commit_info *cinfo) 1736 { 1737 struct nfs_page *first; 1738 struct nfs_open_context *ctx; 1739 struct inode *inode; 1740 1741 /* Set up the RPC argument and reply structs 1742 * NB: take care not to mess about with data->commit et al. */ 1743 1744 if (head) 1745 list_splice_init(head, &data->pages); 1746 1747 first = nfs_list_entry(data->pages.next); 1748 ctx = nfs_req_openctx(first); 1749 inode = d_inode(ctx->dentry); 1750 1751 data->inode = inode; 1752 data->cred = ctx->cred; 1753 data->lseg = lseg; /* reference transferred */ 1754 /* only set lwb for pnfs commit */ 1755 if (lseg) 1756 data->lwb = nfs_get_lwb(&data->pages); 1757 data->mds_ops = &nfs_commit_ops; 1758 data->completion_ops = cinfo->completion_ops; 1759 data->dreq = cinfo->dreq; 1760 1761 data->args.fh = NFS_FH(data->inode); 1762 /* Note: we always request a commit of the entire inode */ 1763 data->args.offset = 0; 1764 data->args.count = 0; 1765 data->context = get_nfs_open_context(ctx); 1766 data->res.fattr = &data->fattr; 1767 data->res.verf = &data->verf; 1768 nfs_fattr_init(&data->fattr); 1769 nfs_commit_begin(cinfo->mds); 1770 } 1771 EXPORT_SYMBOL_GPL(nfs_init_commit); 1772 1773 void nfs_retry_commit(struct list_head *page_list, 1774 struct pnfs_layout_segment *lseg, 1775 struct nfs_commit_info *cinfo, 1776 u32 ds_commit_idx) 1777 { 1778 struct nfs_page *req; 1779 1780 while (!list_empty(page_list)) { 1781 req = nfs_list_entry(page_list->next); 1782 nfs_list_remove_request(req); 1783 nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx); 1784 if (!cinfo->dreq) 1785 nfs_clear_page_commit(req->wb_page); 1786 nfs_unlock_and_release_request(req); 1787 } 1788 } 1789 EXPORT_SYMBOL_GPL(nfs_retry_commit); 1790 1791 static void 1792 nfs_commit_resched_write(struct nfs_commit_info *cinfo, 1793 struct nfs_page *req) 1794 { 1795 __set_page_dirty_nobuffers(req->wb_page); 1796 } 1797 1798 /* 1799 * Commit dirty pages 1800 */ 1801 static int 1802 nfs_commit_list(struct inode *inode, struct list_head *head, int how, 1803 struct nfs_commit_info *cinfo) 1804 { 1805 struct nfs_commit_data *data; 1806 unsigned short task_flags = 0; 1807 1808 /* another commit raced with us */ 1809 if (list_empty(head)) 1810 return 0; 1811 1812 data = nfs_commitdata_alloc(true); 1813 1814 /* Set up the argument struct */ 1815 nfs_init_commit(data, head, NULL, cinfo); 1816 if (NFS_SERVER(inode)->nfs_client->cl_minorversion) 1817 task_flags = RPC_TASK_MOVEABLE; 1818 return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode), 1819 data->mds_ops, how, 1820 RPC_TASK_CRED_NOREF | task_flags); 1821 } 1822 1823 /* 1824 * COMMIT call returned 1825 */ 1826 static void nfs_commit_done(struct rpc_task *task, void *calldata) 1827 { 1828 struct nfs_commit_data *data = calldata; 1829 1830 dprintk("NFS: %5u nfs_commit_done (status %d)\n", 1831 task->tk_pid, task->tk_status); 1832 1833 /* Call the NFS version-specific code */ 1834 NFS_PROTO(data->inode)->commit_done(task, data); 1835 trace_nfs_commit_done(task, data); 1836 } 1837 1838 static void nfs_commit_release_pages(struct nfs_commit_data *data) 1839 { 1840 const struct nfs_writeverf *verf = data->res.verf; 1841 struct nfs_page *req; 1842 int status = data->task.tk_status; 1843 struct nfs_commit_info cinfo; 1844 struct nfs_server *nfss; 1845 1846 while (!list_empty(&data->pages)) { 1847 req = nfs_list_entry(data->pages.next); 1848 nfs_list_remove_request(req); 1849 if (req->wb_page) 1850 nfs_clear_page_commit(req->wb_page); 1851 1852 dprintk("NFS: commit (%s/%llu %d@%lld)", 1853 nfs_req_openctx(req)->dentry->d_sb->s_id, 1854 (unsigned long long)NFS_FILEID(d_inode(nfs_req_openctx(req)->dentry)), 1855 req->wb_bytes, 1856 (long long)req_offset(req)); 1857 if (status < 0) { 1858 if (req->wb_page) { 1859 trace_nfs_commit_error(req, status); 1860 nfs_mapping_set_error(req->wb_page, status); 1861 nfs_inode_remove_request(req); 1862 } 1863 dprintk_cont(", error = %d\n", status); 1864 goto next; 1865 } 1866 1867 /* Okay, COMMIT succeeded, apparently. Check the verifier 1868 * returned by the server against all stored verfs. */ 1869 if (nfs_write_match_verf(verf, req)) { 1870 /* We have a match */ 1871 if (req->wb_page) 1872 nfs_inode_remove_request(req); 1873 dprintk_cont(" OK\n"); 1874 goto next; 1875 } 1876 /* We have a mismatch. Write the page again */ 1877 dprintk_cont(" mismatch\n"); 1878 nfs_mark_request_dirty(req); 1879 set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags); 1880 next: 1881 nfs_unlock_and_release_request(req); 1882 /* Latency breaker */ 1883 cond_resched(); 1884 } 1885 nfss = NFS_SERVER(data->inode); 1886 if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 1887 clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC); 1888 1889 nfs_init_cinfo(&cinfo, data->inode, data->dreq); 1890 nfs_commit_end(cinfo.mds); 1891 } 1892 1893 static void nfs_commit_release(void *calldata) 1894 { 1895 struct nfs_commit_data *data = calldata; 1896 1897 data->completion_ops->completion(data); 1898 nfs_commitdata_release(calldata); 1899 } 1900 1901 static const struct rpc_call_ops nfs_commit_ops = { 1902 .rpc_call_prepare = nfs_commit_prepare, 1903 .rpc_call_done = nfs_commit_done, 1904 .rpc_release = nfs_commit_release, 1905 }; 1906 1907 static const struct nfs_commit_completion_ops nfs_commit_completion_ops = { 1908 .completion = nfs_commit_release_pages, 1909 .resched_write = nfs_commit_resched_write, 1910 }; 1911 1912 int nfs_generic_commit_list(struct inode *inode, struct list_head *head, 1913 int how, struct nfs_commit_info *cinfo) 1914 { 1915 int status; 1916 1917 status = pnfs_commit_list(inode, head, how, cinfo); 1918 if (status == PNFS_NOT_ATTEMPTED) 1919 status = nfs_commit_list(inode, head, how, cinfo); 1920 return status; 1921 } 1922 1923 static int __nfs_commit_inode(struct inode *inode, int how, 1924 struct writeback_control *wbc) 1925 { 1926 LIST_HEAD(head); 1927 struct nfs_commit_info cinfo; 1928 int may_wait = how & FLUSH_SYNC; 1929 int ret, nscan; 1930 1931 how &= ~FLUSH_SYNC; 1932 nfs_init_cinfo_from_inode(&cinfo, inode); 1933 nfs_commit_begin(cinfo.mds); 1934 for (;;) { 1935 ret = nscan = nfs_scan_commit(inode, &head, &cinfo); 1936 if (ret <= 0) 1937 break; 1938 ret = nfs_generic_commit_list(inode, &head, how, &cinfo); 1939 if (ret < 0) 1940 break; 1941 ret = 0; 1942 if (wbc && wbc->sync_mode == WB_SYNC_NONE) { 1943 if (nscan < wbc->nr_to_write) 1944 wbc->nr_to_write -= nscan; 1945 else 1946 wbc->nr_to_write = 0; 1947 } 1948 if (nscan < INT_MAX) 1949 break; 1950 cond_resched(); 1951 } 1952 nfs_commit_end(cinfo.mds); 1953 if (ret || !may_wait) 1954 return ret; 1955 return wait_on_commit(cinfo.mds); 1956 } 1957 1958 int nfs_commit_inode(struct inode *inode, int how) 1959 { 1960 return __nfs_commit_inode(inode, how, NULL); 1961 } 1962 EXPORT_SYMBOL_GPL(nfs_commit_inode); 1963 1964 int nfs_write_inode(struct inode *inode, struct writeback_control *wbc) 1965 { 1966 struct nfs_inode *nfsi = NFS_I(inode); 1967 int flags = FLUSH_SYNC; 1968 int ret = 0; 1969 1970 if (wbc->sync_mode == WB_SYNC_NONE) { 1971 /* no commits means nothing needs to be done */ 1972 if (!atomic_long_read(&nfsi->commit_info.ncommit)) 1973 goto check_requests_outstanding; 1974 1975 /* Don't commit yet if this is a non-blocking flush and there 1976 * are a lot of outstanding writes for this mapping. 1977 */ 1978 if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) 1979 goto out_mark_dirty; 1980 1981 /* don't wait for the COMMIT response */ 1982 flags = 0; 1983 } 1984 1985 ret = __nfs_commit_inode(inode, flags, wbc); 1986 if (!ret) { 1987 if (flags & FLUSH_SYNC) 1988 return 0; 1989 } else if (atomic_long_read(&nfsi->commit_info.ncommit)) 1990 goto out_mark_dirty; 1991 1992 check_requests_outstanding: 1993 if (!atomic_read(&nfsi->commit_info.rpcs_out)) 1994 return ret; 1995 out_mark_dirty: 1996 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1997 return ret; 1998 } 1999 EXPORT_SYMBOL_GPL(nfs_write_inode); 2000 2001 /* 2002 * Wrapper for filemap_write_and_wait_range() 2003 * 2004 * Needed for pNFS in order to ensure data becomes visible to the 2005 * client. 2006 */ 2007 int nfs_filemap_write_and_wait_range(struct address_space *mapping, 2008 loff_t lstart, loff_t lend) 2009 { 2010 int ret; 2011 2012 ret = filemap_write_and_wait_range(mapping, lstart, lend); 2013 if (ret == 0) 2014 ret = pnfs_sync_inode(mapping->host, true); 2015 return ret; 2016 } 2017 EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range); 2018 2019 /* 2020 * flush the inode to disk. 2021 */ 2022 int nfs_wb_all(struct inode *inode) 2023 { 2024 int ret; 2025 2026 trace_nfs_writeback_inode_enter(inode); 2027 2028 ret = filemap_write_and_wait(inode->i_mapping); 2029 if (ret) 2030 goto out; 2031 ret = nfs_commit_inode(inode, FLUSH_SYNC); 2032 if (ret < 0) 2033 goto out; 2034 pnfs_sync_inode(inode, true); 2035 ret = 0; 2036 2037 out: 2038 trace_nfs_writeback_inode_exit(inode, ret); 2039 return ret; 2040 } 2041 EXPORT_SYMBOL_GPL(nfs_wb_all); 2042 2043 int nfs_wb_page_cancel(struct inode *inode, struct page *page) 2044 { 2045 struct nfs_page *req; 2046 int ret = 0; 2047 2048 wait_on_page_writeback(page); 2049 2050 /* blocking call to cancel all requests and join to a single (head) 2051 * request */ 2052 req = nfs_lock_and_join_requests(page); 2053 2054 if (IS_ERR(req)) { 2055 ret = PTR_ERR(req); 2056 } else if (req) { 2057 /* all requests from this page have been cancelled by 2058 * nfs_lock_and_join_requests, so just remove the head 2059 * request from the inode / page_private pointer and 2060 * release it */ 2061 nfs_inode_remove_request(req); 2062 nfs_unlock_and_release_request(req); 2063 } 2064 2065 return ret; 2066 } 2067 2068 /* 2069 * Write back all requests on one page - we do this before reading it. 2070 */ 2071 int nfs_wb_page(struct inode *inode, struct page *page) 2072 { 2073 loff_t range_start = page_file_offset(page); 2074 loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1); 2075 struct writeback_control wbc = { 2076 .sync_mode = WB_SYNC_ALL, 2077 .nr_to_write = 0, 2078 .range_start = range_start, 2079 .range_end = range_end, 2080 }; 2081 int ret; 2082 2083 trace_nfs_writeback_page_enter(inode); 2084 2085 for (;;) { 2086 wait_on_page_writeback(page); 2087 if (clear_page_dirty_for_io(page)) { 2088 ret = nfs_writepage_locked(page, &wbc); 2089 if (ret < 0) 2090 goto out_error; 2091 continue; 2092 } 2093 ret = 0; 2094 if (!PagePrivate(page)) 2095 break; 2096 ret = nfs_commit_inode(inode, FLUSH_SYNC); 2097 if (ret < 0) 2098 goto out_error; 2099 } 2100 out_error: 2101 trace_nfs_writeback_page_exit(inode, ret); 2102 return ret; 2103 } 2104 2105 #ifdef CONFIG_MIGRATION 2106 int nfs_migrate_page(struct address_space *mapping, struct page *newpage, 2107 struct page *page, enum migrate_mode mode) 2108 { 2109 /* 2110 * If PagePrivate is set, then the page is currently associated with 2111 * an in-progress read or write request. Don't try to migrate it. 2112 * 2113 * FIXME: we could do this in principle, but we'll need a way to ensure 2114 * that we can safely release the inode reference while holding 2115 * the page lock. 2116 */ 2117 if (PagePrivate(page)) 2118 return -EBUSY; 2119 2120 if (!nfs_fscache_release_page(page, GFP_KERNEL)) 2121 return -EBUSY; 2122 2123 return migrate_page(mapping, newpage, page, mode); 2124 } 2125 #endif 2126 2127 int __init nfs_init_writepagecache(void) 2128 { 2129 nfs_wdata_cachep = kmem_cache_create("nfs_write_data", 2130 sizeof(struct nfs_pgio_header), 2131 0, SLAB_HWCACHE_ALIGN, 2132 NULL); 2133 if (nfs_wdata_cachep == NULL) 2134 return -ENOMEM; 2135 2136 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE, 2137 nfs_wdata_cachep); 2138 if (nfs_wdata_mempool == NULL) 2139 goto out_destroy_write_cache; 2140 2141 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data", 2142 sizeof(struct nfs_commit_data), 2143 0, SLAB_HWCACHE_ALIGN, 2144 NULL); 2145 if (nfs_cdata_cachep == NULL) 2146 goto out_destroy_write_mempool; 2147 2148 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT, 2149 nfs_cdata_cachep); 2150 if (nfs_commit_mempool == NULL) 2151 goto out_destroy_commit_cache; 2152 2153 /* 2154 * NFS congestion size, scale with available memory. 2155 * 2156 * 64MB: 8192k 2157 * 128MB: 11585k 2158 * 256MB: 16384k 2159 * 512MB: 23170k 2160 * 1GB: 32768k 2161 * 2GB: 46340k 2162 * 4GB: 65536k 2163 * 8GB: 92681k 2164 * 16GB: 131072k 2165 * 2166 * This allows larger machines to have larger/more transfers. 2167 * Limit the default to 256M 2168 */ 2169 nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10); 2170 if (nfs_congestion_kb > 256*1024) 2171 nfs_congestion_kb = 256*1024; 2172 2173 return 0; 2174 2175 out_destroy_commit_cache: 2176 kmem_cache_destroy(nfs_cdata_cachep); 2177 out_destroy_write_mempool: 2178 mempool_destroy(nfs_wdata_mempool); 2179 out_destroy_write_cache: 2180 kmem_cache_destroy(nfs_wdata_cachep); 2181 return -ENOMEM; 2182 } 2183 2184 void nfs_destroy_writepagecache(void) 2185 { 2186 mempool_destroy(nfs_commit_mempool); 2187 kmem_cache_destroy(nfs_cdata_cachep); 2188 mempool_destroy(nfs_wdata_mempool); 2189 kmem_cache_destroy(nfs_wdata_cachep); 2190 } 2191 2192 static const struct nfs_rw_ops nfs_rw_write_ops = { 2193 .rw_alloc_header = nfs_writehdr_alloc, 2194 .rw_free_header = nfs_writehdr_free, 2195 .rw_done = nfs_writeback_done, 2196 .rw_result = nfs_writeback_result, 2197 .rw_initiate = nfs_initiate_write, 2198 }; 2199