1 /* 2 * linux/fs/nfs/write.c 3 * 4 * Write file data over NFS. 5 * 6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de> 7 */ 8 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/mm.h> 12 #include <linux/pagemap.h> 13 #include <linux/file.h> 14 #include <linux/writeback.h> 15 #include <linux/swap.h> 16 #include <linux/migrate.h> 17 18 #include <linux/sunrpc/clnt.h> 19 #include <linux/nfs_fs.h> 20 #include <linux/nfs_mount.h> 21 #include <linux/nfs_page.h> 22 #include <linux/backing-dev.h> 23 #include <linux/export.h> 24 25 #include <asm/uaccess.h> 26 27 #include "delegation.h" 28 #include "internal.h" 29 #include "iostat.h" 30 #include "nfs4_fs.h" 31 #include "fscache.h" 32 #include "pnfs.h" 33 34 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 35 36 #define MIN_POOL_WRITE (32) 37 #define MIN_POOL_COMMIT (4) 38 39 /* 40 * Local function declarations 41 */ 42 static void nfs_redirty_request(struct nfs_page *req); 43 static const struct rpc_call_ops nfs_write_common_ops; 44 static const struct rpc_call_ops nfs_commit_ops; 45 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops; 46 static const struct nfs_commit_completion_ops nfs_commit_completion_ops; 47 48 static struct kmem_cache *nfs_wdata_cachep; 49 static mempool_t *nfs_wdata_mempool; 50 static struct kmem_cache *nfs_cdata_cachep; 51 static mempool_t *nfs_commit_mempool; 52 53 struct nfs_commit_data *nfs_commitdata_alloc(void) 54 { 55 struct nfs_commit_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOIO); 56 57 if (p) { 58 memset(p, 0, sizeof(*p)); 59 INIT_LIST_HEAD(&p->pages); 60 } 61 return p; 62 } 63 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); 64 65 void nfs_commit_free(struct nfs_commit_data *p) 66 { 67 mempool_free(p, nfs_commit_mempool); 68 } 69 EXPORT_SYMBOL_GPL(nfs_commit_free); 70 71 struct nfs_write_header *nfs_writehdr_alloc(void) 72 { 73 struct nfs_write_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO); 74 75 if (p) { 76 struct nfs_pgio_header *hdr = &p->header; 77 78 memset(p, 0, sizeof(*p)); 79 INIT_LIST_HEAD(&hdr->pages); 80 INIT_LIST_HEAD(&hdr->rpc_list); 81 spin_lock_init(&hdr->lock); 82 atomic_set(&hdr->refcnt, 0); 83 hdr->verf = &p->verf; 84 } 85 return p; 86 } 87 EXPORT_SYMBOL_GPL(nfs_writehdr_alloc); 88 89 static struct nfs_write_data *nfs_writedata_alloc(struct nfs_pgio_header *hdr, 90 unsigned int pagecount) 91 { 92 struct nfs_write_data *data, *prealloc; 93 94 prealloc = &container_of(hdr, struct nfs_write_header, header)->rpc_data; 95 if (prealloc->header == NULL) 96 data = prealloc; 97 else 98 data = kzalloc(sizeof(*data), GFP_KERNEL); 99 if (!data) 100 goto out; 101 102 if (nfs_pgarray_set(&data->pages, pagecount)) { 103 data->header = hdr; 104 atomic_inc(&hdr->refcnt); 105 } else { 106 if (data != prealloc) 107 kfree(data); 108 data = NULL; 109 } 110 out: 111 return data; 112 } 113 114 void nfs_writehdr_free(struct nfs_pgio_header *hdr) 115 { 116 struct nfs_write_header *whdr = container_of(hdr, struct nfs_write_header, header); 117 mempool_free(whdr, nfs_wdata_mempool); 118 } 119 EXPORT_SYMBOL_GPL(nfs_writehdr_free); 120 121 void nfs_writedata_release(struct nfs_write_data *wdata) 122 { 123 struct nfs_pgio_header *hdr = wdata->header; 124 struct nfs_write_header *write_header = container_of(hdr, struct nfs_write_header, header); 125 126 put_nfs_open_context(wdata->args.context); 127 if (wdata->pages.pagevec != wdata->pages.page_array) 128 kfree(wdata->pages.pagevec); 129 if (wdata == &write_header->rpc_data) { 130 wdata->header = NULL; 131 wdata = NULL; 132 } 133 if (atomic_dec_and_test(&hdr->refcnt)) 134 hdr->completion_ops->completion(hdr); 135 /* Note: we only free the rpc_task after callbacks are done. 136 * See the comment in rpc_free_task() for why 137 */ 138 kfree(wdata); 139 } 140 EXPORT_SYMBOL_GPL(nfs_writedata_release); 141 142 static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error) 143 { 144 ctx->error = error; 145 smp_wmb(); 146 set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags); 147 } 148 149 static struct nfs_page * 150 nfs_page_find_request_locked(struct nfs_inode *nfsi, struct page *page) 151 { 152 struct nfs_page *req = NULL; 153 154 if (PagePrivate(page)) 155 req = (struct nfs_page *)page_private(page); 156 else if (unlikely(PageSwapCache(page))) { 157 struct nfs_page *freq, *t; 158 159 /* Linearly search the commit list for the correct req */ 160 list_for_each_entry_safe(freq, t, &nfsi->commit_info.list, wb_list) { 161 if (freq->wb_page == page) { 162 req = freq; 163 break; 164 } 165 } 166 } 167 168 if (req) 169 kref_get(&req->wb_kref); 170 171 return req; 172 } 173 174 static struct nfs_page *nfs_page_find_request(struct page *page) 175 { 176 struct inode *inode = page_file_mapping(page)->host; 177 struct nfs_page *req = NULL; 178 179 spin_lock(&inode->i_lock); 180 req = nfs_page_find_request_locked(NFS_I(inode), page); 181 spin_unlock(&inode->i_lock); 182 return req; 183 } 184 185 /* Adjust the file length if we're writing beyond the end */ 186 static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count) 187 { 188 struct inode *inode = page_file_mapping(page)->host; 189 loff_t end, i_size; 190 pgoff_t end_index; 191 192 spin_lock(&inode->i_lock); 193 i_size = i_size_read(inode); 194 end_index = (i_size - 1) >> PAGE_CACHE_SHIFT; 195 if (i_size > 0 && page_file_index(page) < end_index) 196 goto out; 197 end = page_file_offset(page) + ((loff_t)offset+count); 198 if (i_size >= end) 199 goto out; 200 i_size_write(inode, end); 201 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); 202 out: 203 spin_unlock(&inode->i_lock); 204 } 205 206 /* A writeback failed: mark the page as bad, and invalidate the page cache */ 207 static void nfs_set_pageerror(struct page *page) 208 { 209 nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page)); 210 } 211 212 /* We can set the PG_uptodate flag if we see that a write request 213 * covers the full page. 214 */ 215 static void nfs_mark_uptodate(struct page *page, unsigned int base, unsigned int count) 216 { 217 if (PageUptodate(page)) 218 return; 219 if (base != 0) 220 return; 221 if (count != nfs_page_length(page)) 222 return; 223 SetPageUptodate(page); 224 } 225 226 static int wb_priority(struct writeback_control *wbc) 227 { 228 if (wbc->for_reclaim) 229 return FLUSH_HIGHPRI | FLUSH_STABLE; 230 if (wbc->for_kupdate || wbc->for_background) 231 return FLUSH_LOWPRI | FLUSH_COND_STABLE; 232 return FLUSH_COND_STABLE; 233 } 234 235 /* 236 * NFS congestion control 237 */ 238 239 int nfs_congestion_kb; 240 241 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10)) 242 #define NFS_CONGESTION_OFF_THRESH \ 243 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) 244 245 static void nfs_set_page_writeback(struct page *page) 246 { 247 struct nfs_server *nfss = NFS_SERVER(page_file_mapping(page)->host); 248 int ret = test_set_page_writeback(page); 249 250 WARN_ON_ONCE(ret != 0); 251 252 if (atomic_long_inc_return(&nfss->writeback) > 253 NFS_CONGESTION_ON_THRESH) { 254 set_bdi_congested(&nfss->backing_dev_info, 255 BLK_RW_ASYNC); 256 } 257 } 258 259 static void nfs_end_page_writeback(struct page *page) 260 { 261 struct inode *inode = page_file_mapping(page)->host; 262 struct nfs_server *nfss = NFS_SERVER(inode); 263 264 end_page_writeback(page); 265 if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 266 clear_bdi_congested(&nfss->backing_dev_info, BLK_RW_ASYNC); 267 } 268 269 static struct nfs_page *nfs_find_and_lock_request(struct page *page, bool nonblock) 270 { 271 struct inode *inode = page_file_mapping(page)->host; 272 struct nfs_page *req; 273 int ret; 274 275 spin_lock(&inode->i_lock); 276 for (;;) { 277 req = nfs_page_find_request_locked(NFS_I(inode), page); 278 if (req == NULL) 279 break; 280 if (nfs_lock_request(req)) 281 break; 282 /* Note: If we hold the page lock, as is the case in nfs_writepage, 283 * then the call to nfs_lock_request() will always 284 * succeed provided that someone hasn't already marked the 285 * request as dirty (in which case we don't care). 286 */ 287 spin_unlock(&inode->i_lock); 288 if (!nonblock) 289 ret = nfs_wait_on_request(req); 290 else 291 ret = -EAGAIN; 292 nfs_release_request(req); 293 if (ret != 0) 294 return ERR_PTR(ret); 295 spin_lock(&inode->i_lock); 296 } 297 spin_unlock(&inode->i_lock); 298 return req; 299 } 300 301 /* 302 * Find an associated nfs write request, and prepare to flush it out 303 * May return an error if the user signalled nfs_wait_on_request(). 304 */ 305 static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio, 306 struct page *page, bool nonblock) 307 { 308 struct nfs_page *req; 309 int ret = 0; 310 311 req = nfs_find_and_lock_request(page, nonblock); 312 if (!req) 313 goto out; 314 ret = PTR_ERR(req); 315 if (IS_ERR(req)) 316 goto out; 317 318 nfs_set_page_writeback(page); 319 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags)); 320 321 ret = 0; 322 if (!nfs_pageio_add_request(pgio, req)) { 323 nfs_redirty_request(req); 324 ret = pgio->pg_error; 325 } 326 out: 327 return ret; 328 } 329 330 static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio) 331 { 332 struct inode *inode = page_file_mapping(page)->host; 333 int ret; 334 335 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE); 336 nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1); 337 338 nfs_pageio_cond_complete(pgio, page_file_index(page)); 339 ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE); 340 if (ret == -EAGAIN) { 341 redirty_page_for_writepage(wbc, page); 342 ret = 0; 343 } 344 return ret; 345 } 346 347 /* 348 * Write an mmapped page to the server. 349 */ 350 static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc) 351 { 352 struct nfs_pageio_descriptor pgio; 353 int err; 354 355 NFS_PROTO(page_file_mapping(page)->host)->write_pageio_init(&pgio, 356 page->mapping->host, 357 wb_priority(wbc), 358 &nfs_async_write_completion_ops); 359 err = nfs_do_writepage(page, wbc, &pgio); 360 nfs_pageio_complete(&pgio); 361 if (err < 0) 362 return err; 363 if (pgio.pg_error < 0) 364 return pgio.pg_error; 365 return 0; 366 } 367 368 int nfs_writepage(struct page *page, struct writeback_control *wbc) 369 { 370 int ret; 371 372 ret = nfs_writepage_locked(page, wbc); 373 unlock_page(page); 374 return ret; 375 } 376 377 static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data) 378 { 379 int ret; 380 381 ret = nfs_do_writepage(page, wbc, data); 382 unlock_page(page); 383 return ret; 384 } 385 386 int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) 387 { 388 struct inode *inode = mapping->host; 389 unsigned long *bitlock = &NFS_I(inode)->flags; 390 struct nfs_pageio_descriptor pgio; 391 int err; 392 393 /* Stop dirtying of new pages while we sync */ 394 err = wait_on_bit_lock(bitlock, NFS_INO_FLUSHING, 395 nfs_wait_bit_killable, TASK_KILLABLE); 396 if (err) 397 goto out_err; 398 399 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES); 400 401 NFS_PROTO(inode)->write_pageio_init(&pgio, inode, wb_priority(wbc), &nfs_async_write_completion_ops); 402 err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio); 403 nfs_pageio_complete(&pgio); 404 405 clear_bit_unlock(NFS_INO_FLUSHING, bitlock); 406 smp_mb__after_clear_bit(); 407 wake_up_bit(bitlock, NFS_INO_FLUSHING); 408 409 if (err < 0) 410 goto out_err; 411 err = pgio.pg_error; 412 if (err < 0) 413 goto out_err; 414 return 0; 415 out_err: 416 return err; 417 } 418 419 /* 420 * Insert a write request into an inode 421 */ 422 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req) 423 { 424 struct nfs_inode *nfsi = NFS_I(inode); 425 426 /* Lock the request! */ 427 nfs_lock_request(req); 428 429 spin_lock(&inode->i_lock); 430 if (!nfsi->npages && NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) 431 inode->i_version++; 432 /* 433 * Swap-space should not get truncated. Hence no need to plug the race 434 * with invalidate/truncate. 435 */ 436 if (likely(!PageSwapCache(req->wb_page))) { 437 set_bit(PG_MAPPED, &req->wb_flags); 438 SetPagePrivate(req->wb_page); 439 set_page_private(req->wb_page, (unsigned long)req); 440 } 441 nfsi->npages++; 442 kref_get(&req->wb_kref); 443 spin_unlock(&inode->i_lock); 444 } 445 446 /* 447 * Remove a write request from an inode 448 */ 449 static void nfs_inode_remove_request(struct nfs_page *req) 450 { 451 struct inode *inode = req->wb_context->dentry->d_inode; 452 struct nfs_inode *nfsi = NFS_I(inode); 453 454 spin_lock(&inode->i_lock); 455 if (likely(!PageSwapCache(req->wb_page))) { 456 set_page_private(req->wb_page, 0); 457 ClearPagePrivate(req->wb_page); 458 clear_bit(PG_MAPPED, &req->wb_flags); 459 } 460 nfsi->npages--; 461 spin_unlock(&inode->i_lock); 462 nfs_release_request(req); 463 } 464 465 static void 466 nfs_mark_request_dirty(struct nfs_page *req) 467 { 468 __set_page_dirty_nobuffers(req->wb_page); 469 } 470 471 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 472 /** 473 * nfs_request_add_commit_list - add request to a commit list 474 * @req: pointer to a struct nfs_page 475 * @dst: commit list head 476 * @cinfo: holds list lock and accounting info 477 * 478 * This sets the PG_CLEAN bit, updates the cinfo count of 479 * number of outstanding requests requiring a commit as well as 480 * the MM page stats. 481 * 482 * The caller must _not_ hold the cinfo->lock, but must be 483 * holding the nfs_page lock. 484 */ 485 void 486 nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst, 487 struct nfs_commit_info *cinfo) 488 { 489 set_bit(PG_CLEAN, &(req)->wb_flags); 490 spin_lock(cinfo->lock); 491 nfs_list_add_request(req, dst); 492 cinfo->mds->ncommit++; 493 spin_unlock(cinfo->lock); 494 if (!cinfo->dreq) { 495 inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS); 496 inc_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info, 497 BDI_RECLAIMABLE); 498 __mark_inode_dirty(req->wb_context->dentry->d_inode, 499 I_DIRTY_DATASYNC); 500 } 501 } 502 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); 503 504 /** 505 * nfs_request_remove_commit_list - Remove request from a commit list 506 * @req: pointer to a nfs_page 507 * @cinfo: holds list lock and accounting info 508 * 509 * This clears the PG_CLEAN bit, and updates the cinfo's count of 510 * number of outstanding requests requiring a commit 511 * It does not update the MM page stats. 512 * 513 * The caller _must_ hold the cinfo->lock and the nfs_page lock. 514 */ 515 void 516 nfs_request_remove_commit_list(struct nfs_page *req, 517 struct nfs_commit_info *cinfo) 518 { 519 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags)) 520 return; 521 nfs_list_remove_request(req); 522 cinfo->mds->ncommit--; 523 } 524 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list); 525 526 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 527 struct inode *inode) 528 { 529 cinfo->lock = &inode->i_lock; 530 cinfo->mds = &NFS_I(inode)->commit_info; 531 cinfo->ds = pnfs_get_ds_info(inode); 532 cinfo->dreq = NULL; 533 cinfo->completion_ops = &nfs_commit_completion_ops; 534 } 535 536 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 537 struct inode *inode, 538 struct nfs_direct_req *dreq) 539 { 540 if (dreq) 541 nfs_init_cinfo_from_dreq(cinfo, dreq); 542 else 543 nfs_init_cinfo_from_inode(cinfo, inode); 544 } 545 EXPORT_SYMBOL_GPL(nfs_init_cinfo); 546 547 /* 548 * Add a request to the inode's commit list. 549 */ 550 void 551 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 552 struct nfs_commit_info *cinfo) 553 { 554 if (pnfs_mark_request_commit(req, lseg, cinfo)) 555 return; 556 nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo); 557 } 558 559 static void 560 nfs_clear_page_commit(struct page *page) 561 { 562 dec_zone_page_state(page, NR_UNSTABLE_NFS); 563 dec_bdi_stat(page_file_mapping(page)->backing_dev_info, BDI_RECLAIMABLE); 564 } 565 566 static void 567 nfs_clear_request_commit(struct nfs_page *req) 568 { 569 if (test_bit(PG_CLEAN, &req->wb_flags)) { 570 struct inode *inode = req->wb_context->dentry->d_inode; 571 struct nfs_commit_info cinfo; 572 573 nfs_init_cinfo_from_inode(&cinfo, inode); 574 if (!pnfs_clear_request_commit(req, &cinfo)) { 575 spin_lock(cinfo.lock); 576 nfs_request_remove_commit_list(req, &cinfo); 577 spin_unlock(cinfo.lock); 578 } 579 nfs_clear_page_commit(req->wb_page); 580 } 581 } 582 583 static inline 584 int nfs_write_need_commit(struct nfs_write_data *data) 585 { 586 if (data->verf.committed == NFS_DATA_SYNC) 587 return data->header->lseg == NULL; 588 return data->verf.committed != NFS_FILE_SYNC; 589 } 590 591 #else 592 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 593 struct inode *inode) 594 { 595 } 596 597 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 598 struct inode *inode, 599 struct nfs_direct_req *dreq) 600 { 601 } 602 603 void 604 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 605 struct nfs_commit_info *cinfo) 606 { 607 } 608 609 static void 610 nfs_clear_request_commit(struct nfs_page *req) 611 { 612 } 613 614 static inline 615 int nfs_write_need_commit(struct nfs_write_data *data) 616 { 617 return 0; 618 } 619 620 #endif 621 622 static void nfs_write_completion(struct nfs_pgio_header *hdr) 623 { 624 struct nfs_commit_info cinfo; 625 unsigned long bytes = 0; 626 627 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) 628 goto out; 629 nfs_init_cinfo_from_inode(&cinfo, hdr->inode); 630 while (!list_empty(&hdr->pages)) { 631 struct nfs_page *req = nfs_list_entry(hdr->pages.next); 632 633 bytes += req->wb_bytes; 634 nfs_list_remove_request(req); 635 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && 636 (hdr->good_bytes < bytes)) { 637 nfs_set_pageerror(req->wb_page); 638 nfs_context_set_write_error(req->wb_context, hdr->error); 639 goto remove_req; 640 } 641 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) { 642 nfs_mark_request_dirty(req); 643 goto next; 644 } 645 if (test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) { 646 memcpy(&req->wb_verf, &hdr->verf->verifier, sizeof(req->wb_verf)); 647 nfs_mark_request_commit(req, hdr->lseg, &cinfo); 648 goto next; 649 } 650 remove_req: 651 nfs_inode_remove_request(req); 652 next: 653 nfs_unlock_request(req); 654 nfs_end_page_writeback(req->wb_page); 655 nfs_release_request(req); 656 } 657 out: 658 hdr->release(hdr); 659 } 660 661 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 662 static unsigned long 663 nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 664 { 665 return cinfo->mds->ncommit; 666 } 667 668 /* cinfo->lock held by caller */ 669 int 670 nfs_scan_commit_list(struct list_head *src, struct list_head *dst, 671 struct nfs_commit_info *cinfo, int max) 672 { 673 struct nfs_page *req, *tmp; 674 int ret = 0; 675 676 list_for_each_entry_safe(req, tmp, src, wb_list) { 677 if (!nfs_lock_request(req)) 678 continue; 679 kref_get(&req->wb_kref); 680 if (cond_resched_lock(cinfo->lock)) 681 list_safe_reset_next(req, tmp, wb_list); 682 nfs_request_remove_commit_list(req, cinfo); 683 nfs_list_add_request(req, dst); 684 ret++; 685 if ((ret == max) && !cinfo->dreq) 686 break; 687 } 688 return ret; 689 } 690 691 /* 692 * nfs_scan_commit - Scan an inode for commit requests 693 * @inode: NFS inode to scan 694 * @dst: mds destination list 695 * @cinfo: mds and ds lists of reqs ready to commit 696 * 697 * Moves requests from the inode's 'commit' request list. 698 * The requests are *not* checked to ensure that they form a contiguous set. 699 */ 700 int 701 nfs_scan_commit(struct inode *inode, struct list_head *dst, 702 struct nfs_commit_info *cinfo) 703 { 704 int ret = 0; 705 706 spin_lock(cinfo->lock); 707 if (cinfo->mds->ncommit > 0) { 708 const int max = INT_MAX; 709 710 ret = nfs_scan_commit_list(&cinfo->mds->list, dst, 711 cinfo, max); 712 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret); 713 } 714 spin_unlock(cinfo->lock); 715 return ret; 716 } 717 718 #else 719 static unsigned long nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 720 { 721 return 0; 722 } 723 724 int nfs_scan_commit(struct inode *inode, struct list_head *dst, 725 struct nfs_commit_info *cinfo) 726 { 727 return 0; 728 } 729 #endif 730 731 /* 732 * Search for an existing write request, and attempt to update 733 * it to reflect a new dirty region on a given page. 734 * 735 * If the attempt fails, then the existing request is flushed out 736 * to disk. 737 */ 738 static struct nfs_page *nfs_try_to_update_request(struct inode *inode, 739 struct page *page, 740 unsigned int offset, 741 unsigned int bytes) 742 { 743 struct nfs_page *req; 744 unsigned int rqend; 745 unsigned int end; 746 int error; 747 748 if (!PagePrivate(page)) 749 return NULL; 750 751 end = offset + bytes; 752 spin_lock(&inode->i_lock); 753 754 for (;;) { 755 req = nfs_page_find_request_locked(NFS_I(inode), page); 756 if (req == NULL) 757 goto out_unlock; 758 759 rqend = req->wb_offset + req->wb_bytes; 760 /* 761 * Tell the caller to flush out the request if 762 * the offsets are non-contiguous. 763 * Note: nfs_flush_incompatible() will already 764 * have flushed out requests having wrong owners. 765 */ 766 if (offset > rqend 767 || end < req->wb_offset) 768 goto out_flushme; 769 770 if (nfs_lock_request(req)) 771 break; 772 773 /* The request is locked, so wait and then retry */ 774 spin_unlock(&inode->i_lock); 775 error = nfs_wait_on_request(req); 776 nfs_release_request(req); 777 if (error != 0) 778 goto out_err; 779 spin_lock(&inode->i_lock); 780 } 781 782 /* Okay, the request matches. Update the region */ 783 if (offset < req->wb_offset) { 784 req->wb_offset = offset; 785 req->wb_pgbase = offset; 786 } 787 if (end > rqend) 788 req->wb_bytes = end - req->wb_offset; 789 else 790 req->wb_bytes = rqend - req->wb_offset; 791 out_unlock: 792 spin_unlock(&inode->i_lock); 793 if (req) 794 nfs_clear_request_commit(req); 795 return req; 796 out_flushme: 797 spin_unlock(&inode->i_lock); 798 nfs_release_request(req); 799 error = nfs_wb_page(inode, page); 800 out_err: 801 return ERR_PTR(error); 802 } 803 804 /* 805 * Try to update an existing write request, or create one if there is none. 806 * 807 * Note: Should always be called with the Page Lock held to prevent races 808 * if we have to add a new request. Also assumes that the caller has 809 * already called nfs_flush_incompatible() if necessary. 810 */ 811 static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx, 812 struct page *page, unsigned int offset, unsigned int bytes) 813 { 814 struct inode *inode = page_file_mapping(page)->host; 815 struct nfs_page *req; 816 817 req = nfs_try_to_update_request(inode, page, offset, bytes); 818 if (req != NULL) 819 goto out; 820 req = nfs_create_request(ctx, inode, page, offset, bytes); 821 if (IS_ERR(req)) 822 goto out; 823 nfs_inode_add_request(inode, req); 824 out: 825 return req; 826 } 827 828 static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page, 829 unsigned int offset, unsigned int count) 830 { 831 struct nfs_page *req; 832 833 req = nfs_setup_write_request(ctx, page, offset, count); 834 if (IS_ERR(req)) 835 return PTR_ERR(req); 836 /* Update file length */ 837 nfs_grow_file(page, offset, count); 838 nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes); 839 nfs_mark_request_dirty(req); 840 nfs_unlock_and_release_request(req); 841 return 0; 842 } 843 844 int nfs_flush_incompatible(struct file *file, struct page *page) 845 { 846 struct nfs_open_context *ctx = nfs_file_open_context(file); 847 struct nfs_lock_context *l_ctx; 848 struct nfs_page *req; 849 int do_flush, status; 850 /* 851 * Look for a request corresponding to this page. If there 852 * is one, and it belongs to another file, we flush it out 853 * before we try to copy anything into the page. Do this 854 * due to the lack of an ACCESS-type call in NFSv2. 855 * Also do the same if we find a request from an existing 856 * dropped page. 857 */ 858 do { 859 req = nfs_page_find_request(page); 860 if (req == NULL) 861 return 0; 862 l_ctx = req->wb_lock_context; 863 do_flush = req->wb_page != page || req->wb_context != ctx; 864 if (l_ctx) { 865 do_flush |= l_ctx->lockowner.l_owner != current->files 866 || l_ctx->lockowner.l_pid != current->tgid; 867 } 868 nfs_release_request(req); 869 if (!do_flush) 870 return 0; 871 status = nfs_wb_page(page_file_mapping(page)->host, page); 872 } while (status == 0); 873 return status; 874 } 875 876 /* 877 * If the page cache is marked as unsafe or invalid, then we can't rely on 878 * the PageUptodate() flag. In this case, we will need to turn off 879 * write optimisations that depend on the page contents being correct. 880 */ 881 static bool nfs_write_pageuptodate(struct page *page, struct inode *inode) 882 { 883 if (nfs_have_delegated_attributes(inode)) 884 goto out; 885 if (NFS_I(inode)->cache_validity & (NFS_INO_INVALID_DATA|NFS_INO_REVAL_PAGECACHE)) 886 return false; 887 out: 888 return PageUptodate(page) != 0; 889 } 890 891 /* 892 * Update and possibly write a cached page of an NFS file. 893 * 894 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad 895 * things with a page scheduled for an RPC call (e.g. invalidate it). 896 */ 897 int nfs_updatepage(struct file *file, struct page *page, 898 unsigned int offset, unsigned int count) 899 { 900 struct nfs_open_context *ctx = nfs_file_open_context(file); 901 struct inode *inode = page_file_mapping(page)->host; 902 int status = 0; 903 904 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE); 905 906 dprintk("NFS: nfs_updatepage(%s/%s %d@%lld)\n", 907 file->f_path.dentry->d_parent->d_name.name, 908 file->f_path.dentry->d_name.name, count, 909 (long long)(page_file_offset(page) + offset)); 910 911 /* If we're not using byte range locks, and we know the page 912 * is up to date, it may be more efficient to extend the write 913 * to cover the entire page in order to avoid fragmentation 914 * inefficiencies. 915 */ 916 if (nfs_write_pageuptodate(page, inode) && 917 inode->i_flock == NULL && 918 !(file->f_flags & O_DSYNC)) { 919 count = max(count + offset, nfs_page_length(page)); 920 offset = 0; 921 } 922 923 status = nfs_writepage_setup(ctx, page, offset, count); 924 if (status < 0) 925 nfs_set_pageerror(page); 926 else 927 __set_page_dirty_nobuffers(page); 928 929 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n", 930 status, (long long)i_size_read(inode)); 931 return status; 932 } 933 934 static int flush_task_priority(int how) 935 { 936 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) { 937 case FLUSH_HIGHPRI: 938 return RPC_PRIORITY_HIGH; 939 case FLUSH_LOWPRI: 940 return RPC_PRIORITY_LOW; 941 } 942 return RPC_PRIORITY_NORMAL; 943 } 944 945 int nfs_initiate_write(struct rpc_clnt *clnt, 946 struct nfs_write_data *data, 947 const struct rpc_call_ops *call_ops, 948 int how, int flags) 949 { 950 struct inode *inode = data->header->inode; 951 int priority = flush_task_priority(how); 952 struct rpc_task *task; 953 struct rpc_message msg = { 954 .rpc_argp = &data->args, 955 .rpc_resp = &data->res, 956 .rpc_cred = data->header->cred, 957 }; 958 struct rpc_task_setup task_setup_data = { 959 .rpc_client = clnt, 960 .task = &data->task, 961 .rpc_message = &msg, 962 .callback_ops = call_ops, 963 .callback_data = data, 964 .workqueue = nfsiod_workqueue, 965 .flags = RPC_TASK_ASYNC | flags, 966 .priority = priority, 967 }; 968 int ret = 0; 969 970 /* Set up the initial task struct. */ 971 NFS_PROTO(inode)->write_setup(data, &msg); 972 973 dprintk("NFS: %5u initiated write call " 974 "(req %s/%lld, %u bytes @ offset %llu)\n", 975 data->task.tk_pid, 976 inode->i_sb->s_id, 977 (long long)NFS_FILEID(inode), 978 data->args.count, 979 (unsigned long long)data->args.offset); 980 981 task = rpc_run_task(&task_setup_data); 982 if (IS_ERR(task)) { 983 ret = PTR_ERR(task); 984 goto out; 985 } 986 if (how & FLUSH_SYNC) { 987 ret = rpc_wait_for_completion_task(task); 988 if (ret == 0) 989 ret = task->tk_status; 990 } 991 rpc_put_task(task); 992 out: 993 return ret; 994 } 995 EXPORT_SYMBOL_GPL(nfs_initiate_write); 996 997 /* 998 * Set up the argument/result storage required for the RPC call. 999 */ 1000 static void nfs_write_rpcsetup(struct nfs_write_data *data, 1001 unsigned int count, unsigned int offset, 1002 int how, struct nfs_commit_info *cinfo) 1003 { 1004 struct nfs_page *req = data->header->req; 1005 1006 /* Set up the RPC argument and reply structs 1007 * NB: take care not to mess about with data->commit et al. */ 1008 1009 data->args.fh = NFS_FH(data->header->inode); 1010 data->args.offset = req_offset(req) + offset; 1011 /* pnfs_set_layoutcommit needs this */ 1012 data->mds_offset = data->args.offset; 1013 data->args.pgbase = req->wb_pgbase + offset; 1014 data->args.pages = data->pages.pagevec; 1015 data->args.count = count; 1016 data->args.context = get_nfs_open_context(req->wb_context); 1017 data->args.lock_context = req->wb_lock_context; 1018 data->args.stable = NFS_UNSTABLE; 1019 switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) { 1020 case 0: 1021 break; 1022 case FLUSH_COND_STABLE: 1023 if (nfs_reqs_to_commit(cinfo)) 1024 break; 1025 default: 1026 data->args.stable = NFS_FILE_SYNC; 1027 } 1028 1029 data->res.fattr = &data->fattr; 1030 data->res.count = count; 1031 data->res.verf = &data->verf; 1032 nfs_fattr_init(&data->fattr); 1033 } 1034 1035 static int nfs_do_write(struct nfs_write_data *data, 1036 const struct rpc_call_ops *call_ops, 1037 int how) 1038 { 1039 struct inode *inode = data->header->inode; 1040 1041 return nfs_initiate_write(NFS_CLIENT(inode), data, call_ops, how, 0); 1042 } 1043 1044 static int nfs_do_multiple_writes(struct list_head *head, 1045 const struct rpc_call_ops *call_ops, 1046 int how) 1047 { 1048 struct nfs_write_data *data; 1049 int ret = 0; 1050 1051 while (!list_empty(head)) { 1052 int ret2; 1053 1054 data = list_first_entry(head, struct nfs_write_data, list); 1055 list_del_init(&data->list); 1056 1057 ret2 = nfs_do_write(data, call_ops, how); 1058 if (ret == 0) 1059 ret = ret2; 1060 } 1061 return ret; 1062 } 1063 1064 /* If a nfs_flush_* function fails, it should remove reqs from @head and 1065 * call this on each, which will prepare them to be retried on next 1066 * writeback using standard nfs. 1067 */ 1068 static void nfs_redirty_request(struct nfs_page *req) 1069 { 1070 nfs_mark_request_dirty(req); 1071 nfs_unlock_request(req); 1072 nfs_end_page_writeback(req->wb_page); 1073 nfs_release_request(req); 1074 } 1075 1076 static void nfs_async_write_error(struct list_head *head) 1077 { 1078 struct nfs_page *req; 1079 1080 while (!list_empty(head)) { 1081 req = nfs_list_entry(head->next); 1082 nfs_list_remove_request(req); 1083 nfs_redirty_request(req); 1084 } 1085 } 1086 1087 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = { 1088 .error_cleanup = nfs_async_write_error, 1089 .completion = nfs_write_completion, 1090 }; 1091 1092 static void nfs_flush_error(struct nfs_pageio_descriptor *desc, 1093 struct nfs_pgio_header *hdr) 1094 { 1095 set_bit(NFS_IOHDR_REDO, &hdr->flags); 1096 while (!list_empty(&hdr->rpc_list)) { 1097 struct nfs_write_data *data = list_first_entry(&hdr->rpc_list, 1098 struct nfs_write_data, list); 1099 list_del(&data->list); 1100 nfs_writedata_release(data); 1101 } 1102 desc->pg_completion_ops->error_cleanup(&desc->pg_list); 1103 } 1104 1105 /* 1106 * Generate multiple small requests to write out a single 1107 * contiguous dirty area on one page. 1108 */ 1109 static int nfs_flush_multi(struct nfs_pageio_descriptor *desc, 1110 struct nfs_pgio_header *hdr) 1111 { 1112 struct nfs_page *req = hdr->req; 1113 struct page *page = req->wb_page; 1114 struct nfs_write_data *data; 1115 size_t wsize = desc->pg_bsize, nbytes; 1116 unsigned int offset; 1117 int requests = 0; 1118 struct nfs_commit_info cinfo; 1119 1120 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); 1121 1122 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && 1123 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo) || 1124 desc->pg_count > wsize)) 1125 desc->pg_ioflags &= ~FLUSH_COND_STABLE; 1126 1127 1128 offset = 0; 1129 nbytes = desc->pg_count; 1130 do { 1131 size_t len = min(nbytes, wsize); 1132 1133 data = nfs_writedata_alloc(hdr, 1); 1134 if (!data) { 1135 nfs_flush_error(desc, hdr); 1136 return -ENOMEM; 1137 } 1138 data->pages.pagevec[0] = page; 1139 nfs_write_rpcsetup(data, len, offset, desc->pg_ioflags, &cinfo); 1140 list_add(&data->list, &hdr->rpc_list); 1141 requests++; 1142 nbytes -= len; 1143 offset += len; 1144 } while (nbytes != 0); 1145 nfs_list_remove_request(req); 1146 nfs_list_add_request(req, &hdr->pages); 1147 desc->pg_rpc_callops = &nfs_write_common_ops; 1148 return 0; 1149 } 1150 1151 /* 1152 * Create an RPC task for the given write request and kick it. 1153 * The page must have been locked by the caller. 1154 * 1155 * It may happen that the page we're passed is not marked dirty. 1156 * This is the case if nfs_updatepage detects a conflicting request 1157 * that has been written but not committed. 1158 */ 1159 static int nfs_flush_one(struct nfs_pageio_descriptor *desc, 1160 struct nfs_pgio_header *hdr) 1161 { 1162 struct nfs_page *req; 1163 struct page **pages; 1164 struct nfs_write_data *data; 1165 struct list_head *head = &desc->pg_list; 1166 struct nfs_commit_info cinfo; 1167 1168 data = nfs_writedata_alloc(hdr, nfs_page_array_len(desc->pg_base, 1169 desc->pg_count)); 1170 if (!data) { 1171 nfs_flush_error(desc, hdr); 1172 return -ENOMEM; 1173 } 1174 1175 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); 1176 pages = data->pages.pagevec; 1177 while (!list_empty(head)) { 1178 req = nfs_list_entry(head->next); 1179 nfs_list_remove_request(req); 1180 nfs_list_add_request(req, &hdr->pages); 1181 *pages++ = req->wb_page; 1182 } 1183 1184 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && 1185 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo))) 1186 desc->pg_ioflags &= ~FLUSH_COND_STABLE; 1187 1188 /* Set up the argument struct */ 1189 nfs_write_rpcsetup(data, desc->pg_count, 0, desc->pg_ioflags, &cinfo); 1190 list_add(&data->list, &hdr->rpc_list); 1191 desc->pg_rpc_callops = &nfs_write_common_ops; 1192 return 0; 1193 } 1194 1195 int nfs_generic_flush(struct nfs_pageio_descriptor *desc, 1196 struct nfs_pgio_header *hdr) 1197 { 1198 if (desc->pg_bsize < PAGE_CACHE_SIZE) 1199 return nfs_flush_multi(desc, hdr); 1200 return nfs_flush_one(desc, hdr); 1201 } 1202 EXPORT_SYMBOL_GPL(nfs_generic_flush); 1203 1204 static int nfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 1205 { 1206 struct nfs_write_header *whdr; 1207 struct nfs_pgio_header *hdr; 1208 int ret; 1209 1210 whdr = nfs_writehdr_alloc(); 1211 if (!whdr) { 1212 desc->pg_completion_ops->error_cleanup(&desc->pg_list); 1213 return -ENOMEM; 1214 } 1215 hdr = &whdr->header; 1216 nfs_pgheader_init(desc, hdr, nfs_writehdr_free); 1217 atomic_inc(&hdr->refcnt); 1218 ret = nfs_generic_flush(desc, hdr); 1219 if (ret == 0) 1220 ret = nfs_do_multiple_writes(&hdr->rpc_list, 1221 desc->pg_rpc_callops, 1222 desc->pg_ioflags); 1223 if (atomic_dec_and_test(&hdr->refcnt)) 1224 hdr->completion_ops->completion(hdr); 1225 return ret; 1226 } 1227 1228 static const struct nfs_pageio_ops nfs_pageio_write_ops = { 1229 .pg_test = nfs_generic_pg_test, 1230 .pg_doio = nfs_generic_pg_writepages, 1231 }; 1232 1233 void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, 1234 struct inode *inode, int ioflags, 1235 const struct nfs_pgio_completion_ops *compl_ops) 1236 { 1237 nfs_pageio_init(pgio, inode, &nfs_pageio_write_ops, compl_ops, 1238 NFS_SERVER(inode)->wsize, ioflags); 1239 } 1240 EXPORT_SYMBOL_GPL(nfs_pageio_init_write); 1241 1242 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio) 1243 { 1244 pgio->pg_ops = &nfs_pageio_write_ops; 1245 pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize; 1246 } 1247 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds); 1248 1249 1250 void nfs_write_prepare(struct rpc_task *task, void *calldata) 1251 { 1252 struct nfs_write_data *data = calldata; 1253 NFS_PROTO(data->header->inode)->write_rpc_prepare(task, data); 1254 } 1255 1256 void nfs_commit_prepare(struct rpc_task *task, void *calldata) 1257 { 1258 struct nfs_commit_data *data = calldata; 1259 1260 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data); 1261 } 1262 1263 /* 1264 * Handle a write reply that flushes a whole page. 1265 * 1266 * FIXME: There is an inherent race with invalidate_inode_pages and 1267 * writebacks since the page->count is kept > 1 for as long 1268 * as the page has a write request pending. 1269 */ 1270 static void nfs_writeback_done_common(struct rpc_task *task, void *calldata) 1271 { 1272 struct nfs_write_data *data = calldata; 1273 1274 nfs_writeback_done(task, data); 1275 } 1276 1277 static void nfs_writeback_release_common(void *calldata) 1278 { 1279 struct nfs_write_data *data = calldata; 1280 struct nfs_pgio_header *hdr = data->header; 1281 int status = data->task.tk_status; 1282 1283 if ((status >= 0) && nfs_write_need_commit(data)) { 1284 spin_lock(&hdr->lock); 1285 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) 1286 ; /* Do nothing */ 1287 else if (!test_and_set_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) 1288 memcpy(hdr->verf, &data->verf, sizeof(*hdr->verf)); 1289 else if (memcmp(hdr->verf, &data->verf, sizeof(*hdr->verf))) 1290 set_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags); 1291 spin_unlock(&hdr->lock); 1292 } 1293 nfs_writedata_release(data); 1294 } 1295 1296 static const struct rpc_call_ops nfs_write_common_ops = { 1297 .rpc_call_prepare = nfs_write_prepare, 1298 .rpc_call_done = nfs_writeback_done_common, 1299 .rpc_release = nfs_writeback_release_common, 1300 }; 1301 1302 1303 /* 1304 * This function is called when the WRITE call is complete. 1305 */ 1306 void nfs_writeback_done(struct rpc_task *task, struct nfs_write_data *data) 1307 { 1308 struct nfs_writeargs *argp = &data->args; 1309 struct nfs_writeres *resp = &data->res; 1310 struct inode *inode = data->header->inode; 1311 int status; 1312 1313 dprintk("NFS: %5u nfs_writeback_done (status %d)\n", 1314 task->tk_pid, task->tk_status); 1315 1316 /* 1317 * ->write_done will attempt to use post-op attributes to detect 1318 * conflicting writes by other clients. A strict interpretation 1319 * of close-to-open would allow us to continue caching even if 1320 * another writer had changed the file, but some applications 1321 * depend on tighter cache coherency when writing. 1322 */ 1323 status = NFS_PROTO(inode)->write_done(task, data); 1324 if (status != 0) 1325 return; 1326 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, resp->count); 1327 1328 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 1329 if (resp->verf->committed < argp->stable && task->tk_status >= 0) { 1330 /* We tried a write call, but the server did not 1331 * commit data to stable storage even though we 1332 * requested it. 1333 * Note: There is a known bug in Tru64 < 5.0 in which 1334 * the server reports NFS_DATA_SYNC, but performs 1335 * NFS_FILE_SYNC. We therefore implement this checking 1336 * as a dprintk() in order to avoid filling syslog. 1337 */ 1338 static unsigned long complain; 1339 1340 /* Note this will print the MDS for a DS write */ 1341 if (time_before(complain, jiffies)) { 1342 dprintk("NFS: faulty NFS server %s:" 1343 " (committed = %d) != (stable = %d)\n", 1344 NFS_SERVER(inode)->nfs_client->cl_hostname, 1345 resp->verf->committed, argp->stable); 1346 complain = jiffies + 300 * HZ; 1347 } 1348 } 1349 #endif 1350 if (task->tk_status < 0) 1351 nfs_set_pgio_error(data->header, task->tk_status, argp->offset); 1352 else if (resp->count < argp->count) { 1353 static unsigned long complain; 1354 1355 /* This a short write! */ 1356 nfs_inc_stats(inode, NFSIOS_SHORTWRITE); 1357 1358 /* Has the server at least made some progress? */ 1359 if (resp->count == 0) { 1360 if (time_before(complain, jiffies)) { 1361 printk(KERN_WARNING 1362 "NFS: Server wrote zero bytes, expected %u.\n", 1363 argp->count); 1364 complain = jiffies + 300 * HZ; 1365 } 1366 nfs_set_pgio_error(data->header, -EIO, argp->offset); 1367 task->tk_status = -EIO; 1368 return; 1369 } 1370 /* Was this an NFSv2 write or an NFSv3 stable write? */ 1371 if (resp->verf->committed != NFS_UNSTABLE) { 1372 /* Resend from where the server left off */ 1373 data->mds_offset += resp->count; 1374 argp->offset += resp->count; 1375 argp->pgbase += resp->count; 1376 argp->count -= resp->count; 1377 } else { 1378 /* Resend as a stable write in order to avoid 1379 * headaches in the case of a server crash. 1380 */ 1381 argp->stable = NFS_FILE_SYNC; 1382 } 1383 rpc_restart_call_prepare(task); 1384 } 1385 } 1386 1387 1388 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 1389 static int nfs_commit_set_lock(struct nfs_inode *nfsi, int may_wait) 1390 { 1391 int ret; 1392 1393 if (!test_and_set_bit(NFS_INO_COMMIT, &nfsi->flags)) 1394 return 1; 1395 if (!may_wait) 1396 return 0; 1397 ret = out_of_line_wait_on_bit_lock(&nfsi->flags, 1398 NFS_INO_COMMIT, 1399 nfs_wait_bit_killable, 1400 TASK_KILLABLE); 1401 return (ret < 0) ? ret : 1; 1402 } 1403 1404 static void nfs_commit_clear_lock(struct nfs_inode *nfsi) 1405 { 1406 clear_bit(NFS_INO_COMMIT, &nfsi->flags); 1407 smp_mb__after_clear_bit(); 1408 wake_up_bit(&nfsi->flags, NFS_INO_COMMIT); 1409 } 1410 1411 void nfs_commitdata_release(struct nfs_commit_data *data) 1412 { 1413 put_nfs_open_context(data->context); 1414 nfs_commit_free(data); 1415 } 1416 EXPORT_SYMBOL_GPL(nfs_commitdata_release); 1417 1418 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, 1419 const struct rpc_call_ops *call_ops, 1420 int how, int flags) 1421 { 1422 struct rpc_task *task; 1423 int priority = flush_task_priority(how); 1424 struct rpc_message msg = { 1425 .rpc_argp = &data->args, 1426 .rpc_resp = &data->res, 1427 .rpc_cred = data->cred, 1428 }; 1429 struct rpc_task_setup task_setup_data = { 1430 .task = &data->task, 1431 .rpc_client = clnt, 1432 .rpc_message = &msg, 1433 .callback_ops = call_ops, 1434 .callback_data = data, 1435 .workqueue = nfsiod_workqueue, 1436 .flags = RPC_TASK_ASYNC | flags, 1437 .priority = priority, 1438 }; 1439 /* Set up the initial task struct. */ 1440 NFS_PROTO(data->inode)->commit_setup(data, &msg); 1441 1442 dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid); 1443 1444 task = rpc_run_task(&task_setup_data); 1445 if (IS_ERR(task)) 1446 return PTR_ERR(task); 1447 if (how & FLUSH_SYNC) 1448 rpc_wait_for_completion_task(task); 1449 rpc_put_task(task); 1450 return 0; 1451 } 1452 EXPORT_SYMBOL_GPL(nfs_initiate_commit); 1453 1454 /* 1455 * Set up the argument/result storage required for the RPC call. 1456 */ 1457 void nfs_init_commit(struct nfs_commit_data *data, 1458 struct list_head *head, 1459 struct pnfs_layout_segment *lseg, 1460 struct nfs_commit_info *cinfo) 1461 { 1462 struct nfs_page *first = nfs_list_entry(head->next); 1463 struct inode *inode = first->wb_context->dentry->d_inode; 1464 1465 /* Set up the RPC argument and reply structs 1466 * NB: take care not to mess about with data->commit et al. */ 1467 1468 list_splice_init(head, &data->pages); 1469 1470 data->inode = inode; 1471 data->cred = first->wb_context->cred; 1472 data->lseg = lseg; /* reference transferred */ 1473 data->mds_ops = &nfs_commit_ops; 1474 data->completion_ops = cinfo->completion_ops; 1475 data->dreq = cinfo->dreq; 1476 1477 data->args.fh = NFS_FH(data->inode); 1478 /* Note: we always request a commit of the entire inode */ 1479 data->args.offset = 0; 1480 data->args.count = 0; 1481 data->context = get_nfs_open_context(first->wb_context); 1482 data->res.fattr = &data->fattr; 1483 data->res.verf = &data->verf; 1484 nfs_fattr_init(&data->fattr); 1485 } 1486 EXPORT_SYMBOL_GPL(nfs_init_commit); 1487 1488 void nfs_retry_commit(struct list_head *page_list, 1489 struct pnfs_layout_segment *lseg, 1490 struct nfs_commit_info *cinfo) 1491 { 1492 struct nfs_page *req; 1493 1494 while (!list_empty(page_list)) { 1495 req = nfs_list_entry(page_list->next); 1496 nfs_list_remove_request(req); 1497 nfs_mark_request_commit(req, lseg, cinfo); 1498 if (!cinfo->dreq) { 1499 dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS); 1500 dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info, 1501 BDI_RECLAIMABLE); 1502 } 1503 nfs_unlock_and_release_request(req); 1504 } 1505 } 1506 EXPORT_SYMBOL_GPL(nfs_retry_commit); 1507 1508 /* 1509 * Commit dirty pages 1510 */ 1511 static int 1512 nfs_commit_list(struct inode *inode, struct list_head *head, int how, 1513 struct nfs_commit_info *cinfo) 1514 { 1515 struct nfs_commit_data *data; 1516 1517 data = nfs_commitdata_alloc(); 1518 1519 if (!data) 1520 goto out_bad; 1521 1522 /* Set up the argument struct */ 1523 nfs_init_commit(data, head, NULL, cinfo); 1524 atomic_inc(&cinfo->mds->rpcs_out); 1525 return nfs_initiate_commit(NFS_CLIENT(inode), data, data->mds_ops, 1526 how, 0); 1527 out_bad: 1528 nfs_retry_commit(head, NULL, cinfo); 1529 cinfo->completion_ops->error_cleanup(NFS_I(inode)); 1530 return -ENOMEM; 1531 } 1532 1533 /* 1534 * COMMIT call returned 1535 */ 1536 static void nfs_commit_done(struct rpc_task *task, void *calldata) 1537 { 1538 struct nfs_commit_data *data = calldata; 1539 1540 dprintk("NFS: %5u nfs_commit_done (status %d)\n", 1541 task->tk_pid, task->tk_status); 1542 1543 /* Call the NFS version-specific code */ 1544 NFS_PROTO(data->inode)->commit_done(task, data); 1545 } 1546 1547 static void nfs_commit_release_pages(struct nfs_commit_data *data) 1548 { 1549 struct nfs_page *req; 1550 int status = data->task.tk_status; 1551 struct nfs_commit_info cinfo; 1552 1553 while (!list_empty(&data->pages)) { 1554 req = nfs_list_entry(data->pages.next); 1555 nfs_list_remove_request(req); 1556 nfs_clear_page_commit(req->wb_page); 1557 1558 dprintk("NFS: commit (%s/%lld %d@%lld)", 1559 req->wb_context->dentry->d_sb->s_id, 1560 (long long)NFS_FILEID(req->wb_context->dentry->d_inode), 1561 req->wb_bytes, 1562 (long long)req_offset(req)); 1563 if (status < 0) { 1564 nfs_context_set_write_error(req->wb_context, status); 1565 nfs_inode_remove_request(req); 1566 dprintk(", error = %d\n", status); 1567 goto next; 1568 } 1569 1570 /* Okay, COMMIT succeeded, apparently. Check the verifier 1571 * returned by the server against all stored verfs. */ 1572 if (!memcmp(&req->wb_verf, &data->verf.verifier, sizeof(req->wb_verf))) { 1573 /* We have a match */ 1574 nfs_inode_remove_request(req); 1575 dprintk(" OK\n"); 1576 goto next; 1577 } 1578 /* We have a mismatch. Write the page again */ 1579 dprintk(" mismatch\n"); 1580 nfs_mark_request_dirty(req); 1581 set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags); 1582 next: 1583 nfs_unlock_and_release_request(req); 1584 } 1585 nfs_init_cinfo(&cinfo, data->inode, data->dreq); 1586 if (atomic_dec_and_test(&cinfo.mds->rpcs_out)) 1587 nfs_commit_clear_lock(NFS_I(data->inode)); 1588 } 1589 1590 static void nfs_commit_release(void *calldata) 1591 { 1592 struct nfs_commit_data *data = calldata; 1593 1594 data->completion_ops->completion(data); 1595 nfs_commitdata_release(calldata); 1596 } 1597 1598 static const struct rpc_call_ops nfs_commit_ops = { 1599 .rpc_call_prepare = nfs_commit_prepare, 1600 .rpc_call_done = nfs_commit_done, 1601 .rpc_release = nfs_commit_release, 1602 }; 1603 1604 static const struct nfs_commit_completion_ops nfs_commit_completion_ops = { 1605 .completion = nfs_commit_release_pages, 1606 .error_cleanup = nfs_commit_clear_lock, 1607 }; 1608 1609 int nfs_generic_commit_list(struct inode *inode, struct list_head *head, 1610 int how, struct nfs_commit_info *cinfo) 1611 { 1612 int status; 1613 1614 status = pnfs_commit_list(inode, head, how, cinfo); 1615 if (status == PNFS_NOT_ATTEMPTED) 1616 status = nfs_commit_list(inode, head, how, cinfo); 1617 return status; 1618 } 1619 1620 int nfs_commit_inode(struct inode *inode, int how) 1621 { 1622 LIST_HEAD(head); 1623 struct nfs_commit_info cinfo; 1624 int may_wait = how & FLUSH_SYNC; 1625 int res; 1626 1627 res = nfs_commit_set_lock(NFS_I(inode), may_wait); 1628 if (res <= 0) 1629 goto out_mark_dirty; 1630 nfs_init_cinfo_from_inode(&cinfo, inode); 1631 res = nfs_scan_commit(inode, &head, &cinfo); 1632 if (res) { 1633 int error; 1634 1635 error = nfs_generic_commit_list(inode, &head, how, &cinfo); 1636 if (error < 0) 1637 return error; 1638 if (!may_wait) 1639 goto out_mark_dirty; 1640 error = wait_on_bit(&NFS_I(inode)->flags, 1641 NFS_INO_COMMIT, 1642 nfs_wait_bit_killable, 1643 TASK_KILLABLE); 1644 if (error < 0) 1645 return error; 1646 } else 1647 nfs_commit_clear_lock(NFS_I(inode)); 1648 return res; 1649 /* Note: If we exit without ensuring that the commit is complete, 1650 * we must mark the inode as dirty. Otherwise, future calls to 1651 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure 1652 * that the data is on the disk. 1653 */ 1654 out_mark_dirty: 1655 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1656 return res; 1657 } 1658 1659 static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc) 1660 { 1661 struct nfs_inode *nfsi = NFS_I(inode); 1662 int flags = FLUSH_SYNC; 1663 int ret = 0; 1664 1665 /* no commits means nothing needs to be done */ 1666 if (!nfsi->commit_info.ncommit) 1667 return ret; 1668 1669 if (wbc->sync_mode == WB_SYNC_NONE) { 1670 /* Don't commit yet if this is a non-blocking flush and there 1671 * are a lot of outstanding writes for this mapping. 1672 */ 1673 if (nfsi->commit_info.ncommit <= (nfsi->npages >> 1)) 1674 goto out_mark_dirty; 1675 1676 /* don't wait for the COMMIT response */ 1677 flags = 0; 1678 } 1679 1680 ret = nfs_commit_inode(inode, flags); 1681 if (ret >= 0) { 1682 if (wbc->sync_mode == WB_SYNC_NONE) { 1683 if (ret < wbc->nr_to_write) 1684 wbc->nr_to_write -= ret; 1685 else 1686 wbc->nr_to_write = 0; 1687 } 1688 return 0; 1689 } 1690 out_mark_dirty: 1691 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1692 return ret; 1693 } 1694 #else 1695 static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc) 1696 { 1697 return 0; 1698 } 1699 #endif 1700 1701 int nfs_write_inode(struct inode *inode, struct writeback_control *wbc) 1702 { 1703 return nfs_commit_unstable_pages(inode, wbc); 1704 } 1705 EXPORT_SYMBOL_GPL(nfs_write_inode); 1706 1707 /* 1708 * flush the inode to disk. 1709 */ 1710 int nfs_wb_all(struct inode *inode) 1711 { 1712 struct writeback_control wbc = { 1713 .sync_mode = WB_SYNC_ALL, 1714 .nr_to_write = LONG_MAX, 1715 .range_start = 0, 1716 .range_end = LLONG_MAX, 1717 }; 1718 1719 return sync_inode(inode, &wbc); 1720 } 1721 EXPORT_SYMBOL_GPL(nfs_wb_all); 1722 1723 int nfs_wb_page_cancel(struct inode *inode, struct page *page) 1724 { 1725 struct nfs_page *req; 1726 int ret = 0; 1727 1728 for (;;) { 1729 wait_on_page_writeback(page); 1730 req = nfs_page_find_request(page); 1731 if (req == NULL) 1732 break; 1733 if (nfs_lock_request(req)) { 1734 nfs_clear_request_commit(req); 1735 nfs_inode_remove_request(req); 1736 /* 1737 * In case nfs_inode_remove_request has marked the 1738 * page as being dirty 1739 */ 1740 cancel_dirty_page(page, PAGE_CACHE_SIZE); 1741 nfs_unlock_and_release_request(req); 1742 break; 1743 } 1744 ret = nfs_wait_on_request(req); 1745 nfs_release_request(req); 1746 if (ret < 0) 1747 break; 1748 } 1749 return ret; 1750 } 1751 1752 /* 1753 * Write back all requests on one page - we do this before reading it. 1754 */ 1755 int nfs_wb_page(struct inode *inode, struct page *page) 1756 { 1757 loff_t range_start = page_file_offset(page); 1758 loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1); 1759 struct writeback_control wbc = { 1760 .sync_mode = WB_SYNC_ALL, 1761 .nr_to_write = 0, 1762 .range_start = range_start, 1763 .range_end = range_end, 1764 }; 1765 int ret; 1766 1767 for (;;) { 1768 wait_on_page_writeback(page); 1769 if (clear_page_dirty_for_io(page)) { 1770 ret = nfs_writepage_locked(page, &wbc); 1771 if (ret < 0) 1772 goto out_error; 1773 continue; 1774 } 1775 if (!PagePrivate(page)) 1776 break; 1777 ret = nfs_commit_inode(inode, FLUSH_SYNC); 1778 if (ret < 0) 1779 goto out_error; 1780 } 1781 return 0; 1782 out_error: 1783 return ret; 1784 } 1785 1786 #ifdef CONFIG_MIGRATION 1787 int nfs_migrate_page(struct address_space *mapping, struct page *newpage, 1788 struct page *page, enum migrate_mode mode) 1789 { 1790 /* 1791 * If PagePrivate is set, then the page is currently associated with 1792 * an in-progress read or write request. Don't try to migrate it. 1793 * 1794 * FIXME: we could do this in principle, but we'll need a way to ensure 1795 * that we can safely release the inode reference while holding 1796 * the page lock. 1797 */ 1798 if (PagePrivate(page)) 1799 return -EBUSY; 1800 1801 if (!nfs_fscache_release_page(page, GFP_KERNEL)) 1802 return -EBUSY; 1803 1804 return migrate_page(mapping, newpage, page, mode); 1805 } 1806 #endif 1807 1808 int __init nfs_init_writepagecache(void) 1809 { 1810 nfs_wdata_cachep = kmem_cache_create("nfs_write_data", 1811 sizeof(struct nfs_write_header), 1812 0, SLAB_HWCACHE_ALIGN, 1813 NULL); 1814 if (nfs_wdata_cachep == NULL) 1815 return -ENOMEM; 1816 1817 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE, 1818 nfs_wdata_cachep); 1819 if (nfs_wdata_mempool == NULL) 1820 goto out_destroy_write_cache; 1821 1822 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data", 1823 sizeof(struct nfs_commit_data), 1824 0, SLAB_HWCACHE_ALIGN, 1825 NULL); 1826 if (nfs_cdata_cachep == NULL) 1827 goto out_destroy_write_mempool; 1828 1829 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT, 1830 nfs_cdata_cachep); 1831 if (nfs_commit_mempool == NULL) 1832 goto out_destroy_commit_cache; 1833 1834 /* 1835 * NFS congestion size, scale with available memory. 1836 * 1837 * 64MB: 8192k 1838 * 128MB: 11585k 1839 * 256MB: 16384k 1840 * 512MB: 23170k 1841 * 1GB: 32768k 1842 * 2GB: 46340k 1843 * 4GB: 65536k 1844 * 8GB: 92681k 1845 * 16GB: 131072k 1846 * 1847 * This allows larger machines to have larger/more transfers. 1848 * Limit the default to 256M 1849 */ 1850 nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10); 1851 if (nfs_congestion_kb > 256*1024) 1852 nfs_congestion_kb = 256*1024; 1853 1854 return 0; 1855 1856 out_destroy_commit_cache: 1857 kmem_cache_destroy(nfs_cdata_cachep); 1858 out_destroy_write_mempool: 1859 mempool_destroy(nfs_wdata_mempool); 1860 out_destroy_write_cache: 1861 kmem_cache_destroy(nfs_wdata_cachep); 1862 return -ENOMEM; 1863 } 1864 1865 void nfs_destroy_writepagecache(void) 1866 { 1867 mempool_destroy(nfs_commit_mempool); 1868 kmem_cache_destroy(nfs_cdata_cachep); 1869 mempool_destroy(nfs_wdata_mempool); 1870 kmem_cache_destroy(nfs_wdata_cachep); 1871 } 1872 1873