1 /* 2 * linux/fs/nfs/write.c 3 * 4 * Write file data over NFS. 5 * 6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de> 7 */ 8 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/mm.h> 12 #include <linux/pagemap.h> 13 #include <linux/file.h> 14 #include <linux/writeback.h> 15 #include <linux/swap.h> 16 #include <linux/migrate.h> 17 18 #include <linux/sunrpc/clnt.h> 19 #include <linux/nfs_fs.h> 20 #include <linux/nfs_mount.h> 21 #include <linux/nfs_page.h> 22 #include <linux/backing-dev.h> 23 #include <linux/export.h> 24 25 #include <asm/uaccess.h> 26 27 #include "delegation.h" 28 #include "internal.h" 29 #include "iostat.h" 30 #include "nfs4_fs.h" 31 #include "fscache.h" 32 #include "pnfs.h" 33 34 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 35 36 #define MIN_POOL_WRITE (32) 37 #define MIN_POOL_COMMIT (4) 38 39 /* 40 * Local function declarations 41 */ 42 static void nfs_redirty_request(struct nfs_page *req); 43 static const struct rpc_call_ops nfs_write_common_ops; 44 static const struct rpc_call_ops nfs_commit_ops; 45 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops; 46 static const struct nfs_commit_completion_ops nfs_commit_completion_ops; 47 48 static struct kmem_cache *nfs_wdata_cachep; 49 static mempool_t *nfs_wdata_mempool; 50 static struct kmem_cache *nfs_cdata_cachep; 51 static mempool_t *nfs_commit_mempool; 52 53 struct nfs_commit_data *nfs_commitdata_alloc(void) 54 { 55 struct nfs_commit_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOIO); 56 57 if (p) { 58 memset(p, 0, sizeof(*p)); 59 INIT_LIST_HEAD(&p->pages); 60 } 61 return p; 62 } 63 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); 64 65 void nfs_commit_free(struct nfs_commit_data *p) 66 { 67 mempool_free(p, nfs_commit_mempool); 68 } 69 EXPORT_SYMBOL_GPL(nfs_commit_free); 70 71 struct nfs_write_header *nfs_writehdr_alloc(void) 72 { 73 struct nfs_write_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO); 74 75 if (p) { 76 struct nfs_pgio_header *hdr = &p->header; 77 78 memset(p, 0, sizeof(*p)); 79 INIT_LIST_HEAD(&hdr->pages); 80 INIT_LIST_HEAD(&hdr->rpc_list); 81 spin_lock_init(&hdr->lock); 82 atomic_set(&hdr->refcnt, 0); 83 hdr->verf = &p->verf; 84 } 85 return p; 86 } 87 EXPORT_SYMBOL_GPL(nfs_writehdr_alloc); 88 89 static struct nfs_write_data *nfs_writedata_alloc(struct nfs_pgio_header *hdr, 90 unsigned int pagecount) 91 { 92 struct nfs_write_data *data, *prealloc; 93 94 prealloc = &container_of(hdr, struct nfs_write_header, header)->rpc_data; 95 if (prealloc->header == NULL) 96 data = prealloc; 97 else 98 data = kzalloc(sizeof(*data), GFP_KERNEL); 99 if (!data) 100 goto out; 101 102 if (nfs_pgarray_set(&data->pages, pagecount)) { 103 data->header = hdr; 104 atomic_inc(&hdr->refcnt); 105 } else { 106 if (data != prealloc) 107 kfree(data); 108 data = NULL; 109 } 110 out: 111 return data; 112 } 113 114 void nfs_writehdr_free(struct nfs_pgio_header *hdr) 115 { 116 struct nfs_write_header *whdr = container_of(hdr, struct nfs_write_header, header); 117 mempool_free(whdr, nfs_wdata_mempool); 118 } 119 EXPORT_SYMBOL_GPL(nfs_writehdr_free); 120 121 void nfs_writedata_release(struct nfs_write_data *wdata) 122 { 123 struct nfs_pgio_header *hdr = wdata->header; 124 struct nfs_write_header *write_header = container_of(hdr, struct nfs_write_header, header); 125 126 put_nfs_open_context(wdata->args.context); 127 if (wdata->pages.pagevec != wdata->pages.page_array) 128 kfree(wdata->pages.pagevec); 129 if (wdata == &write_header->rpc_data) { 130 wdata->header = NULL; 131 wdata = NULL; 132 } 133 if (atomic_dec_and_test(&hdr->refcnt)) 134 hdr->completion_ops->completion(hdr); 135 /* Note: we only free the rpc_task after callbacks are done. 136 * See the comment in rpc_free_task() for why 137 */ 138 kfree(wdata); 139 } 140 EXPORT_SYMBOL_GPL(nfs_writedata_release); 141 142 static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error) 143 { 144 ctx->error = error; 145 smp_wmb(); 146 set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags); 147 } 148 149 static struct nfs_page * 150 nfs_page_find_request_locked(struct nfs_inode *nfsi, struct page *page) 151 { 152 struct nfs_page *req = NULL; 153 154 if (PagePrivate(page)) 155 req = (struct nfs_page *)page_private(page); 156 else if (unlikely(PageSwapCache(page))) { 157 struct nfs_page *freq, *t; 158 159 /* Linearly search the commit list for the correct req */ 160 list_for_each_entry_safe(freq, t, &nfsi->commit_info.list, wb_list) { 161 if (freq->wb_page == page) { 162 req = freq; 163 break; 164 } 165 } 166 } 167 168 if (req) 169 kref_get(&req->wb_kref); 170 171 return req; 172 } 173 174 static struct nfs_page *nfs_page_find_request(struct page *page) 175 { 176 struct inode *inode = page_file_mapping(page)->host; 177 struct nfs_page *req = NULL; 178 179 spin_lock(&inode->i_lock); 180 req = nfs_page_find_request_locked(NFS_I(inode), page); 181 spin_unlock(&inode->i_lock); 182 return req; 183 } 184 185 /* Adjust the file length if we're writing beyond the end */ 186 static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count) 187 { 188 struct inode *inode = page_file_mapping(page)->host; 189 loff_t end, i_size; 190 pgoff_t end_index; 191 192 spin_lock(&inode->i_lock); 193 i_size = i_size_read(inode); 194 end_index = (i_size - 1) >> PAGE_CACHE_SHIFT; 195 if (i_size > 0 && page_file_index(page) < end_index) 196 goto out; 197 end = page_file_offset(page) + ((loff_t)offset+count); 198 if (i_size >= end) 199 goto out; 200 i_size_write(inode, end); 201 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); 202 out: 203 spin_unlock(&inode->i_lock); 204 } 205 206 /* A writeback failed: mark the page as bad, and invalidate the page cache */ 207 static void nfs_set_pageerror(struct page *page) 208 { 209 nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page)); 210 } 211 212 /* We can set the PG_uptodate flag if we see that a write request 213 * covers the full page. 214 */ 215 static void nfs_mark_uptodate(struct page *page, unsigned int base, unsigned int count) 216 { 217 if (PageUptodate(page)) 218 return; 219 if (base != 0) 220 return; 221 if (count != nfs_page_length(page)) 222 return; 223 SetPageUptodate(page); 224 } 225 226 static int wb_priority(struct writeback_control *wbc) 227 { 228 if (wbc->for_reclaim) 229 return FLUSH_HIGHPRI | FLUSH_STABLE; 230 if (wbc->for_kupdate || wbc->for_background) 231 return FLUSH_LOWPRI | FLUSH_COND_STABLE; 232 return FLUSH_COND_STABLE; 233 } 234 235 /* 236 * NFS congestion control 237 */ 238 239 int nfs_congestion_kb; 240 241 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10)) 242 #define NFS_CONGESTION_OFF_THRESH \ 243 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) 244 245 static void nfs_set_page_writeback(struct page *page) 246 { 247 struct nfs_server *nfss = NFS_SERVER(page_file_mapping(page)->host); 248 int ret = test_set_page_writeback(page); 249 250 WARN_ON_ONCE(ret != 0); 251 252 if (atomic_long_inc_return(&nfss->writeback) > 253 NFS_CONGESTION_ON_THRESH) { 254 set_bdi_congested(&nfss->backing_dev_info, 255 BLK_RW_ASYNC); 256 } 257 } 258 259 static void nfs_end_page_writeback(struct page *page) 260 { 261 struct inode *inode = page_file_mapping(page)->host; 262 struct nfs_server *nfss = NFS_SERVER(inode); 263 264 end_page_writeback(page); 265 if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) 266 clear_bdi_congested(&nfss->backing_dev_info, BLK_RW_ASYNC); 267 } 268 269 static struct nfs_page *nfs_find_and_lock_request(struct page *page, bool nonblock) 270 { 271 struct inode *inode = page_file_mapping(page)->host; 272 struct nfs_page *req; 273 int ret; 274 275 spin_lock(&inode->i_lock); 276 for (;;) { 277 req = nfs_page_find_request_locked(NFS_I(inode), page); 278 if (req == NULL) 279 break; 280 if (nfs_lock_request(req)) 281 break; 282 /* Note: If we hold the page lock, as is the case in nfs_writepage, 283 * then the call to nfs_lock_request() will always 284 * succeed provided that someone hasn't already marked the 285 * request as dirty (in which case we don't care). 286 */ 287 spin_unlock(&inode->i_lock); 288 if (!nonblock) 289 ret = nfs_wait_on_request(req); 290 else 291 ret = -EAGAIN; 292 nfs_release_request(req); 293 if (ret != 0) 294 return ERR_PTR(ret); 295 spin_lock(&inode->i_lock); 296 } 297 spin_unlock(&inode->i_lock); 298 return req; 299 } 300 301 /* 302 * Find an associated nfs write request, and prepare to flush it out 303 * May return an error if the user signalled nfs_wait_on_request(). 304 */ 305 static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio, 306 struct page *page, bool nonblock) 307 { 308 struct nfs_page *req; 309 int ret = 0; 310 311 req = nfs_find_and_lock_request(page, nonblock); 312 if (!req) 313 goto out; 314 ret = PTR_ERR(req); 315 if (IS_ERR(req)) 316 goto out; 317 318 nfs_set_page_writeback(page); 319 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags)); 320 321 ret = 0; 322 if (!nfs_pageio_add_request(pgio, req)) { 323 nfs_redirty_request(req); 324 ret = pgio->pg_error; 325 } 326 out: 327 return ret; 328 } 329 330 static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio) 331 { 332 struct inode *inode = page_file_mapping(page)->host; 333 int ret; 334 335 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE); 336 nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1); 337 338 nfs_pageio_cond_complete(pgio, page_file_index(page)); 339 ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE); 340 if (ret == -EAGAIN) { 341 redirty_page_for_writepage(wbc, page); 342 ret = 0; 343 } 344 return ret; 345 } 346 347 /* 348 * Write an mmapped page to the server. 349 */ 350 static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc) 351 { 352 struct nfs_pageio_descriptor pgio; 353 int err; 354 355 NFS_PROTO(page_file_mapping(page)->host)->write_pageio_init(&pgio, 356 page->mapping->host, 357 wb_priority(wbc), 358 &nfs_async_write_completion_ops); 359 err = nfs_do_writepage(page, wbc, &pgio); 360 nfs_pageio_complete(&pgio); 361 if (err < 0) 362 return err; 363 if (pgio.pg_error < 0) 364 return pgio.pg_error; 365 return 0; 366 } 367 368 int nfs_writepage(struct page *page, struct writeback_control *wbc) 369 { 370 int ret; 371 372 ret = nfs_writepage_locked(page, wbc); 373 unlock_page(page); 374 return ret; 375 } 376 377 static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data) 378 { 379 int ret; 380 381 ret = nfs_do_writepage(page, wbc, data); 382 unlock_page(page); 383 return ret; 384 } 385 386 int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) 387 { 388 struct inode *inode = mapping->host; 389 unsigned long *bitlock = &NFS_I(inode)->flags; 390 struct nfs_pageio_descriptor pgio; 391 int err; 392 393 /* Stop dirtying of new pages while we sync */ 394 err = wait_on_bit_lock(bitlock, NFS_INO_FLUSHING, 395 nfs_wait_bit_killable, TASK_KILLABLE); 396 if (err) 397 goto out_err; 398 399 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES); 400 401 NFS_PROTO(inode)->write_pageio_init(&pgio, inode, wb_priority(wbc), &nfs_async_write_completion_ops); 402 err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio); 403 nfs_pageio_complete(&pgio); 404 405 clear_bit_unlock(NFS_INO_FLUSHING, bitlock); 406 smp_mb__after_clear_bit(); 407 wake_up_bit(bitlock, NFS_INO_FLUSHING); 408 409 if (err < 0) 410 goto out_err; 411 err = pgio.pg_error; 412 if (err < 0) 413 goto out_err; 414 return 0; 415 out_err: 416 return err; 417 } 418 419 /* 420 * Insert a write request into an inode 421 */ 422 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req) 423 { 424 struct nfs_inode *nfsi = NFS_I(inode); 425 426 /* Lock the request! */ 427 nfs_lock_request(req); 428 429 spin_lock(&inode->i_lock); 430 if (!nfsi->npages && NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) 431 inode->i_version++; 432 /* 433 * Swap-space should not get truncated. Hence no need to plug the race 434 * with invalidate/truncate. 435 */ 436 if (likely(!PageSwapCache(req->wb_page))) { 437 set_bit(PG_MAPPED, &req->wb_flags); 438 SetPagePrivate(req->wb_page); 439 set_page_private(req->wb_page, (unsigned long)req); 440 } 441 nfsi->npages++; 442 kref_get(&req->wb_kref); 443 spin_unlock(&inode->i_lock); 444 } 445 446 /* 447 * Remove a write request from an inode 448 */ 449 static void nfs_inode_remove_request(struct nfs_page *req) 450 { 451 struct inode *inode = req->wb_context->dentry->d_inode; 452 struct nfs_inode *nfsi = NFS_I(inode); 453 454 spin_lock(&inode->i_lock); 455 if (likely(!PageSwapCache(req->wb_page))) { 456 set_page_private(req->wb_page, 0); 457 ClearPagePrivate(req->wb_page); 458 clear_bit(PG_MAPPED, &req->wb_flags); 459 } 460 nfsi->npages--; 461 spin_unlock(&inode->i_lock); 462 nfs_release_request(req); 463 } 464 465 static void 466 nfs_mark_request_dirty(struct nfs_page *req) 467 { 468 __set_page_dirty_nobuffers(req->wb_page); 469 } 470 471 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 472 /** 473 * nfs_request_add_commit_list - add request to a commit list 474 * @req: pointer to a struct nfs_page 475 * @dst: commit list head 476 * @cinfo: holds list lock and accounting info 477 * 478 * This sets the PG_CLEAN bit, updates the cinfo count of 479 * number of outstanding requests requiring a commit as well as 480 * the MM page stats. 481 * 482 * The caller must _not_ hold the cinfo->lock, but must be 483 * holding the nfs_page lock. 484 */ 485 void 486 nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst, 487 struct nfs_commit_info *cinfo) 488 { 489 set_bit(PG_CLEAN, &(req)->wb_flags); 490 spin_lock(cinfo->lock); 491 nfs_list_add_request(req, dst); 492 cinfo->mds->ncommit++; 493 spin_unlock(cinfo->lock); 494 if (!cinfo->dreq) { 495 inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS); 496 inc_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info, 497 BDI_RECLAIMABLE); 498 __mark_inode_dirty(req->wb_context->dentry->d_inode, 499 I_DIRTY_DATASYNC); 500 } 501 } 502 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); 503 504 /** 505 * nfs_request_remove_commit_list - Remove request from a commit list 506 * @req: pointer to a nfs_page 507 * @cinfo: holds list lock and accounting info 508 * 509 * This clears the PG_CLEAN bit, and updates the cinfo's count of 510 * number of outstanding requests requiring a commit 511 * It does not update the MM page stats. 512 * 513 * The caller _must_ hold the cinfo->lock and the nfs_page lock. 514 */ 515 void 516 nfs_request_remove_commit_list(struct nfs_page *req, 517 struct nfs_commit_info *cinfo) 518 { 519 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags)) 520 return; 521 nfs_list_remove_request(req); 522 cinfo->mds->ncommit--; 523 } 524 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list); 525 526 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 527 struct inode *inode) 528 { 529 cinfo->lock = &inode->i_lock; 530 cinfo->mds = &NFS_I(inode)->commit_info; 531 cinfo->ds = pnfs_get_ds_info(inode); 532 cinfo->dreq = NULL; 533 cinfo->completion_ops = &nfs_commit_completion_ops; 534 } 535 536 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 537 struct inode *inode, 538 struct nfs_direct_req *dreq) 539 { 540 if (dreq) 541 nfs_init_cinfo_from_dreq(cinfo, dreq); 542 else 543 nfs_init_cinfo_from_inode(cinfo, inode); 544 } 545 EXPORT_SYMBOL_GPL(nfs_init_cinfo); 546 547 /* 548 * Add a request to the inode's commit list. 549 */ 550 void 551 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 552 struct nfs_commit_info *cinfo) 553 { 554 if (pnfs_mark_request_commit(req, lseg, cinfo)) 555 return; 556 nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo); 557 } 558 559 static void 560 nfs_clear_page_commit(struct page *page) 561 { 562 dec_zone_page_state(page, NR_UNSTABLE_NFS); 563 dec_bdi_stat(page_file_mapping(page)->backing_dev_info, BDI_RECLAIMABLE); 564 } 565 566 static void 567 nfs_clear_request_commit(struct nfs_page *req) 568 { 569 if (test_bit(PG_CLEAN, &req->wb_flags)) { 570 struct inode *inode = req->wb_context->dentry->d_inode; 571 struct nfs_commit_info cinfo; 572 573 nfs_init_cinfo_from_inode(&cinfo, inode); 574 if (!pnfs_clear_request_commit(req, &cinfo)) { 575 spin_lock(cinfo.lock); 576 nfs_request_remove_commit_list(req, &cinfo); 577 spin_unlock(cinfo.lock); 578 } 579 nfs_clear_page_commit(req->wb_page); 580 } 581 } 582 583 static inline 584 int nfs_write_need_commit(struct nfs_write_data *data) 585 { 586 if (data->verf.committed == NFS_DATA_SYNC) 587 return data->header->lseg == NULL; 588 return data->verf.committed != NFS_FILE_SYNC; 589 } 590 591 #else 592 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, 593 struct inode *inode) 594 { 595 } 596 597 void nfs_init_cinfo(struct nfs_commit_info *cinfo, 598 struct inode *inode, 599 struct nfs_direct_req *dreq) 600 { 601 } 602 603 void 604 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, 605 struct nfs_commit_info *cinfo) 606 { 607 } 608 609 static void 610 nfs_clear_request_commit(struct nfs_page *req) 611 { 612 } 613 614 static inline 615 int nfs_write_need_commit(struct nfs_write_data *data) 616 { 617 return 0; 618 } 619 620 #endif 621 622 static void nfs_write_completion(struct nfs_pgio_header *hdr) 623 { 624 struct nfs_commit_info cinfo; 625 unsigned long bytes = 0; 626 627 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) 628 goto out; 629 nfs_init_cinfo_from_inode(&cinfo, hdr->inode); 630 while (!list_empty(&hdr->pages)) { 631 struct nfs_page *req = nfs_list_entry(hdr->pages.next); 632 633 bytes += req->wb_bytes; 634 nfs_list_remove_request(req); 635 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && 636 (hdr->good_bytes < bytes)) { 637 nfs_set_pageerror(req->wb_page); 638 nfs_context_set_write_error(req->wb_context, hdr->error); 639 goto remove_req; 640 } 641 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) { 642 nfs_mark_request_dirty(req); 643 goto next; 644 } 645 if (test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) { 646 memcpy(&req->wb_verf, &hdr->verf->verifier, sizeof(req->wb_verf)); 647 nfs_mark_request_commit(req, hdr->lseg, &cinfo); 648 goto next; 649 } 650 remove_req: 651 nfs_inode_remove_request(req); 652 next: 653 nfs_unlock_request(req); 654 nfs_end_page_writeback(req->wb_page); 655 nfs_release_request(req); 656 } 657 out: 658 hdr->release(hdr); 659 } 660 661 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 662 static unsigned long 663 nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 664 { 665 return cinfo->mds->ncommit; 666 } 667 668 /* cinfo->lock held by caller */ 669 int 670 nfs_scan_commit_list(struct list_head *src, struct list_head *dst, 671 struct nfs_commit_info *cinfo, int max) 672 { 673 struct nfs_page *req, *tmp; 674 int ret = 0; 675 676 list_for_each_entry_safe(req, tmp, src, wb_list) { 677 if (!nfs_lock_request(req)) 678 continue; 679 kref_get(&req->wb_kref); 680 if (cond_resched_lock(cinfo->lock)) 681 list_safe_reset_next(req, tmp, wb_list); 682 nfs_request_remove_commit_list(req, cinfo); 683 nfs_list_add_request(req, dst); 684 ret++; 685 if ((ret == max) && !cinfo->dreq) 686 break; 687 } 688 return ret; 689 } 690 691 /* 692 * nfs_scan_commit - Scan an inode for commit requests 693 * @inode: NFS inode to scan 694 * @dst: mds destination list 695 * @cinfo: mds and ds lists of reqs ready to commit 696 * 697 * Moves requests from the inode's 'commit' request list. 698 * The requests are *not* checked to ensure that they form a contiguous set. 699 */ 700 int 701 nfs_scan_commit(struct inode *inode, struct list_head *dst, 702 struct nfs_commit_info *cinfo) 703 { 704 int ret = 0; 705 706 spin_lock(cinfo->lock); 707 if (cinfo->mds->ncommit > 0) { 708 const int max = INT_MAX; 709 710 ret = nfs_scan_commit_list(&cinfo->mds->list, dst, 711 cinfo, max); 712 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret); 713 } 714 spin_unlock(cinfo->lock); 715 return ret; 716 } 717 718 #else 719 static unsigned long nfs_reqs_to_commit(struct nfs_commit_info *cinfo) 720 { 721 return 0; 722 } 723 724 int nfs_scan_commit(struct inode *inode, struct list_head *dst, 725 struct nfs_commit_info *cinfo) 726 { 727 return 0; 728 } 729 #endif 730 731 /* 732 * Search for an existing write request, and attempt to update 733 * it to reflect a new dirty region on a given page. 734 * 735 * If the attempt fails, then the existing request is flushed out 736 * to disk. 737 */ 738 static struct nfs_page *nfs_try_to_update_request(struct inode *inode, 739 struct page *page, 740 unsigned int offset, 741 unsigned int bytes) 742 { 743 struct nfs_page *req; 744 unsigned int rqend; 745 unsigned int end; 746 int error; 747 748 if (!PagePrivate(page)) 749 return NULL; 750 751 end = offset + bytes; 752 spin_lock(&inode->i_lock); 753 754 for (;;) { 755 req = nfs_page_find_request_locked(NFS_I(inode), page); 756 if (req == NULL) 757 goto out_unlock; 758 759 rqend = req->wb_offset + req->wb_bytes; 760 /* 761 * Tell the caller to flush out the request if 762 * the offsets are non-contiguous. 763 * Note: nfs_flush_incompatible() will already 764 * have flushed out requests having wrong owners. 765 */ 766 if (offset > rqend 767 || end < req->wb_offset) 768 goto out_flushme; 769 770 if (nfs_lock_request(req)) 771 break; 772 773 /* The request is locked, so wait and then retry */ 774 spin_unlock(&inode->i_lock); 775 error = nfs_wait_on_request(req); 776 nfs_release_request(req); 777 if (error != 0) 778 goto out_err; 779 spin_lock(&inode->i_lock); 780 } 781 782 /* Okay, the request matches. Update the region */ 783 if (offset < req->wb_offset) { 784 req->wb_offset = offset; 785 req->wb_pgbase = offset; 786 } 787 if (end > rqend) 788 req->wb_bytes = end - req->wb_offset; 789 else 790 req->wb_bytes = rqend - req->wb_offset; 791 out_unlock: 792 spin_unlock(&inode->i_lock); 793 if (req) 794 nfs_clear_request_commit(req); 795 return req; 796 out_flushme: 797 spin_unlock(&inode->i_lock); 798 nfs_release_request(req); 799 error = nfs_wb_page(inode, page); 800 out_err: 801 return ERR_PTR(error); 802 } 803 804 /* 805 * Try to update an existing write request, or create one if there is none. 806 * 807 * Note: Should always be called with the Page Lock held to prevent races 808 * if we have to add a new request. Also assumes that the caller has 809 * already called nfs_flush_incompatible() if necessary. 810 */ 811 static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx, 812 struct page *page, unsigned int offset, unsigned int bytes) 813 { 814 struct inode *inode = page_file_mapping(page)->host; 815 struct nfs_page *req; 816 817 req = nfs_try_to_update_request(inode, page, offset, bytes); 818 if (req != NULL) 819 goto out; 820 req = nfs_create_request(ctx, inode, page, offset, bytes); 821 if (IS_ERR(req)) 822 goto out; 823 nfs_inode_add_request(inode, req); 824 out: 825 return req; 826 } 827 828 static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page, 829 unsigned int offset, unsigned int count) 830 { 831 struct nfs_page *req; 832 833 req = nfs_setup_write_request(ctx, page, offset, count); 834 if (IS_ERR(req)) 835 return PTR_ERR(req); 836 /* Update file length */ 837 nfs_grow_file(page, offset, count); 838 nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes); 839 nfs_mark_request_dirty(req); 840 nfs_unlock_and_release_request(req); 841 return 0; 842 } 843 844 int nfs_flush_incompatible(struct file *file, struct page *page) 845 { 846 struct nfs_open_context *ctx = nfs_file_open_context(file); 847 struct nfs_lock_context *l_ctx; 848 struct nfs_page *req; 849 int do_flush, status; 850 /* 851 * Look for a request corresponding to this page. If there 852 * is one, and it belongs to another file, we flush it out 853 * before we try to copy anything into the page. Do this 854 * due to the lack of an ACCESS-type call in NFSv2. 855 * Also do the same if we find a request from an existing 856 * dropped page. 857 */ 858 do { 859 req = nfs_page_find_request(page); 860 if (req == NULL) 861 return 0; 862 l_ctx = req->wb_lock_context; 863 do_flush = req->wb_page != page || req->wb_context != ctx; 864 if (l_ctx) { 865 do_flush |= l_ctx->lockowner.l_owner != current->files 866 || l_ctx->lockowner.l_pid != current->tgid; 867 } 868 nfs_release_request(req); 869 if (!do_flush) 870 return 0; 871 status = nfs_wb_page(page_file_mapping(page)->host, page); 872 } while (status == 0); 873 return status; 874 } 875 876 /* 877 * If the page cache is marked as unsafe or invalid, then we can't rely on 878 * the PageUptodate() flag. In this case, we will need to turn off 879 * write optimisations that depend on the page contents being correct. 880 */ 881 static bool nfs_write_pageuptodate(struct page *page, struct inode *inode) 882 { 883 if (nfs_have_delegated_attributes(inode)) 884 goto out; 885 if (NFS_I(inode)->cache_validity & (NFS_INO_INVALID_DATA|NFS_INO_REVAL_PAGECACHE)) 886 return false; 887 out: 888 return PageUptodate(page) != 0; 889 } 890 891 /* 892 * Update and possibly write a cached page of an NFS file. 893 * 894 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad 895 * things with a page scheduled for an RPC call (e.g. invalidate it). 896 */ 897 int nfs_updatepage(struct file *file, struct page *page, 898 unsigned int offset, unsigned int count) 899 { 900 struct nfs_open_context *ctx = nfs_file_open_context(file); 901 struct inode *inode = page_file_mapping(page)->host; 902 int status = 0; 903 904 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE); 905 906 dprintk("NFS: nfs_updatepage(%s/%s %d@%lld)\n", 907 file->f_path.dentry->d_parent->d_name.name, 908 file->f_path.dentry->d_name.name, count, 909 (long long)(page_file_offset(page) + offset)); 910 911 /* If we're not using byte range locks, and we know the page 912 * is up to date, it may be more efficient to extend the write 913 * to cover the entire page in order to avoid fragmentation 914 * inefficiencies. 915 */ 916 if (nfs_write_pageuptodate(page, inode) && 917 inode->i_flock == NULL && 918 !(file->f_flags & O_DSYNC)) { 919 count = max(count + offset, nfs_page_length(page)); 920 offset = 0; 921 } 922 923 status = nfs_writepage_setup(ctx, page, offset, count); 924 if (status < 0) 925 nfs_set_pageerror(page); 926 else 927 __set_page_dirty_nobuffers(page); 928 929 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n", 930 status, (long long)i_size_read(inode)); 931 return status; 932 } 933 934 static int flush_task_priority(int how) 935 { 936 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) { 937 case FLUSH_HIGHPRI: 938 return RPC_PRIORITY_HIGH; 939 case FLUSH_LOWPRI: 940 return RPC_PRIORITY_LOW; 941 } 942 return RPC_PRIORITY_NORMAL; 943 } 944 945 int nfs_initiate_write(struct rpc_clnt *clnt, 946 struct nfs_write_data *data, 947 const struct rpc_call_ops *call_ops, 948 int how, int flags) 949 { 950 struct inode *inode = data->header->inode; 951 int priority = flush_task_priority(how); 952 struct rpc_task *task; 953 struct rpc_message msg = { 954 .rpc_argp = &data->args, 955 .rpc_resp = &data->res, 956 .rpc_cred = data->header->cred, 957 }; 958 struct rpc_task_setup task_setup_data = { 959 .rpc_client = clnt, 960 .task = &data->task, 961 .rpc_message = &msg, 962 .callback_ops = call_ops, 963 .callback_data = data, 964 .workqueue = nfsiod_workqueue, 965 .flags = RPC_TASK_ASYNC | flags, 966 .priority = priority, 967 }; 968 int ret = 0; 969 970 /* Set up the initial task struct. */ 971 NFS_PROTO(inode)->write_setup(data, &msg); 972 973 dprintk("NFS: %5u initiated write call " 974 "(req %s/%lld, %u bytes @ offset %llu)\n", 975 data->task.tk_pid, 976 inode->i_sb->s_id, 977 (long long)NFS_FILEID(inode), 978 data->args.count, 979 (unsigned long long)data->args.offset); 980 981 task = rpc_run_task(&task_setup_data); 982 if (IS_ERR(task)) { 983 ret = PTR_ERR(task); 984 goto out; 985 } 986 if (how & FLUSH_SYNC) { 987 ret = rpc_wait_for_completion_task(task); 988 if (ret == 0) 989 ret = task->tk_status; 990 } 991 rpc_put_task(task); 992 out: 993 return ret; 994 } 995 EXPORT_SYMBOL_GPL(nfs_initiate_write); 996 997 /* 998 * Set up the argument/result storage required for the RPC call. 999 */ 1000 static void nfs_write_rpcsetup(struct nfs_write_data *data, 1001 unsigned int count, unsigned int offset, 1002 int how, struct nfs_commit_info *cinfo) 1003 { 1004 struct nfs_page *req = data->header->req; 1005 1006 /* Set up the RPC argument and reply structs 1007 * NB: take care not to mess about with data->commit et al. */ 1008 1009 data->args.fh = NFS_FH(data->header->inode); 1010 data->args.offset = req_offset(req) + offset; 1011 /* pnfs_set_layoutcommit needs this */ 1012 data->mds_offset = data->args.offset; 1013 data->args.pgbase = req->wb_pgbase + offset; 1014 data->args.pages = data->pages.pagevec; 1015 data->args.count = count; 1016 data->args.context = get_nfs_open_context(req->wb_context); 1017 data->args.lock_context = req->wb_lock_context; 1018 data->args.stable = NFS_UNSTABLE; 1019 switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) { 1020 case 0: 1021 break; 1022 case FLUSH_COND_STABLE: 1023 if (nfs_reqs_to_commit(cinfo)) 1024 break; 1025 default: 1026 data->args.stable = NFS_FILE_SYNC; 1027 } 1028 1029 data->res.fattr = &data->fattr; 1030 data->res.count = count; 1031 data->res.verf = &data->verf; 1032 nfs_fattr_init(&data->fattr); 1033 } 1034 1035 static int nfs_do_write(struct nfs_write_data *data, 1036 const struct rpc_call_ops *call_ops, 1037 int how) 1038 { 1039 struct inode *inode = data->header->inode; 1040 1041 return nfs_initiate_write(NFS_CLIENT(inode), data, call_ops, how, 0); 1042 } 1043 1044 static int nfs_do_multiple_writes(struct list_head *head, 1045 const struct rpc_call_ops *call_ops, 1046 int how) 1047 { 1048 struct nfs_write_data *data; 1049 int ret = 0; 1050 1051 while (!list_empty(head)) { 1052 int ret2; 1053 1054 data = list_first_entry(head, struct nfs_write_data, list); 1055 list_del_init(&data->list); 1056 1057 ret2 = nfs_do_write(data, call_ops, how); 1058 if (ret == 0) 1059 ret = ret2; 1060 } 1061 return ret; 1062 } 1063 1064 /* If a nfs_flush_* function fails, it should remove reqs from @head and 1065 * call this on each, which will prepare them to be retried on next 1066 * writeback using standard nfs. 1067 */ 1068 static void nfs_redirty_request(struct nfs_page *req) 1069 { 1070 nfs_mark_request_dirty(req); 1071 nfs_unlock_request(req); 1072 nfs_end_page_writeback(req->wb_page); 1073 nfs_release_request(req); 1074 } 1075 1076 static void nfs_async_write_error(struct list_head *head) 1077 { 1078 struct nfs_page *req; 1079 1080 while (!list_empty(head)) { 1081 req = nfs_list_entry(head->next); 1082 nfs_list_remove_request(req); 1083 nfs_redirty_request(req); 1084 } 1085 } 1086 1087 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = { 1088 .error_cleanup = nfs_async_write_error, 1089 .completion = nfs_write_completion, 1090 }; 1091 1092 static void nfs_flush_error(struct nfs_pageio_descriptor *desc, 1093 struct nfs_pgio_header *hdr) 1094 { 1095 set_bit(NFS_IOHDR_REDO, &hdr->flags); 1096 while (!list_empty(&hdr->rpc_list)) { 1097 struct nfs_write_data *data = list_first_entry(&hdr->rpc_list, 1098 struct nfs_write_data, list); 1099 list_del(&data->list); 1100 nfs_writedata_release(data); 1101 } 1102 desc->pg_completion_ops->error_cleanup(&desc->pg_list); 1103 } 1104 1105 /* 1106 * Generate multiple small requests to write out a single 1107 * contiguous dirty area on one page. 1108 */ 1109 static int nfs_flush_multi(struct nfs_pageio_descriptor *desc, 1110 struct nfs_pgio_header *hdr) 1111 { 1112 struct nfs_page *req = hdr->req; 1113 struct page *page = req->wb_page; 1114 struct nfs_write_data *data; 1115 size_t wsize = desc->pg_bsize, nbytes; 1116 unsigned int offset; 1117 int requests = 0; 1118 struct nfs_commit_info cinfo; 1119 1120 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); 1121 1122 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && 1123 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo) || 1124 desc->pg_count > wsize)) 1125 desc->pg_ioflags &= ~FLUSH_COND_STABLE; 1126 1127 1128 offset = 0; 1129 nbytes = desc->pg_count; 1130 do { 1131 size_t len = min(nbytes, wsize); 1132 1133 data = nfs_writedata_alloc(hdr, 1); 1134 if (!data) { 1135 nfs_flush_error(desc, hdr); 1136 return -ENOMEM; 1137 } 1138 data->pages.pagevec[0] = page; 1139 nfs_write_rpcsetup(data, len, offset, desc->pg_ioflags, &cinfo); 1140 list_add(&data->list, &hdr->rpc_list); 1141 requests++; 1142 nbytes -= len; 1143 offset += len; 1144 } while (nbytes != 0); 1145 nfs_list_remove_request(req); 1146 nfs_list_add_request(req, &hdr->pages); 1147 desc->pg_rpc_callops = &nfs_write_common_ops; 1148 return 0; 1149 } 1150 1151 /* 1152 * Create an RPC task for the given write request and kick it. 1153 * The page must have been locked by the caller. 1154 * 1155 * It may happen that the page we're passed is not marked dirty. 1156 * This is the case if nfs_updatepage detects a conflicting request 1157 * that has been written but not committed. 1158 */ 1159 static int nfs_flush_one(struct nfs_pageio_descriptor *desc, 1160 struct nfs_pgio_header *hdr) 1161 { 1162 struct nfs_page *req; 1163 struct page **pages; 1164 struct nfs_write_data *data; 1165 struct list_head *head = &desc->pg_list; 1166 struct nfs_commit_info cinfo; 1167 1168 data = nfs_writedata_alloc(hdr, nfs_page_array_len(desc->pg_base, 1169 desc->pg_count)); 1170 if (!data) { 1171 nfs_flush_error(desc, hdr); 1172 return -ENOMEM; 1173 } 1174 1175 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); 1176 pages = data->pages.pagevec; 1177 while (!list_empty(head)) { 1178 req = nfs_list_entry(head->next); 1179 nfs_list_remove_request(req); 1180 nfs_list_add_request(req, &hdr->pages); 1181 *pages++ = req->wb_page; 1182 } 1183 1184 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && 1185 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo))) 1186 desc->pg_ioflags &= ~FLUSH_COND_STABLE; 1187 1188 /* Set up the argument struct */ 1189 nfs_write_rpcsetup(data, desc->pg_count, 0, desc->pg_ioflags, &cinfo); 1190 list_add(&data->list, &hdr->rpc_list); 1191 desc->pg_rpc_callops = &nfs_write_common_ops; 1192 return 0; 1193 } 1194 1195 int nfs_generic_flush(struct nfs_pageio_descriptor *desc, 1196 struct nfs_pgio_header *hdr) 1197 { 1198 if (desc->pg_bsize < PAGE_CACHE_SIZE) 1199 return nfs_flush_multi(desc, hdr); 1200 return nfs_flush_one(desc, hdr); 1201 } 1202 EXPORT_SYMBOL_GPL(nfs_generic_flush); 1203 1204 static int nfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 1205 { 1206 struct nfs_write_header *whdr; 1207 struct nfs_pgio_header *hdr; 1208 int ret; 1209 1210 whdr = nfs_writehdr_alloc(); 1211 if (!whdr) { 1212 desc->pg_completion_ops->error_cleanup(&desc->pg_list); 1213 return -ENOMEM; 1214 } 1215 hdr = &whdr->header; 1216 nfs_pgheader_init(desc, hdr, nfs_writehdr_free); 1217 atomic_inc(&hdr->refcnt); 1218 ret = nfs_generic_flush(desc, hdr); 1219 if (ret == 0) 1220 ret = nfs_do_multiple_writes(&hdr->rpc_list, 1221 desc->pg_rpc_callops, 1222 desc->pg_ioflags); 1223 if (atomic_dec_and_test(&hdr->refcnt)) 1224 hdr->completion_ops->completion(hdr); 1225 return ret; 1226 } 1227 1228 static const struct nfs_pageio_ops nfs_pageio_write_ops = { 1229 .pg_test = nfs_generic_pg_test, 1230 .pg_doio = nfs_generic_pg_writepages, 1231 }; 1232 1233 void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, 1234 struct inode *inode, int ioflags, 1235 const struct nfs_pgio_completion_ops *compl_ops) 1236 { 1237 nfs_pageio_init(pgio, inode, &nfs_pageio_write_ops, compl_ops, 1238 NFS_SERVER(inode)->wsize, ioflags); 1239 } 1240 EXPORT_SYMBOL_GPL(nfs_pageio_init_write); 1241 1242 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio) 1243 { 1244 pgio->pg_ops = &nfs_pageio_write_ops; 1245 pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize; 1246 } 1247 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds); 1248 1249 1250 void nfs_write_prepare(struct rpc_task *task, void *calldata) 1251 { 1252 struct nfs_write_data *data = calldata; 1253 NFS_PROTO(data->header->inode)->write_rpc_prepare(task, data); 1254 if (unlikely(test_bit(NFS_CONTEXT_BAD, &data->args.context->flags))) 1255 rpc_exit(task, -EIO); 1256 } 1257 1258 void nfs_commit_prepare(struct rpc_task *task, void *calldata) 1259 { 1260 struct nfs_commit_data *data = calldata; 1261 1262 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data); 1263 } 1264 1265 /* 1266 * Handle a write reply that flushes a whole page. 1267 * 1268 * FIXME: There is an inherent race with invalidate_inode_pages and 1269 * writebacks since the page->count is kept > 1 for as long 1270 * as the page has a write request pending. 1271 */ 1272 static void nfs_writeback_done_common(struct rpc_task *task, void *calldata) 1273 { 1274 struct nfs_write_data *data = calldata; 1275 1276 nfs_writeback_done(task, data); 1277 } 1278 1279 static void nfs_writeback_release_common(void *calldata) 1280 { 1281 struct nfs_write_data *data = calldata; 1282 struct nfs_pgio_header *hdr = data->header; 1283 int status = data->task.tk_status; 1284 1285 if ((status >= 0) && nfs_write_need_commit(data)) { 1286 spin_lock(&hdr->lock); 1287 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) 1288 ; /* Do nothing */ 1289 else if (!test_and_set_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) 1290 memcpy(hdr->verf, &data->verf, sizeof(*hdr->verf)); 1291 else if (memcmp(hdr->verf, &data->verf, sizeof(*hdr->verf))) 1292 set_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags); 1293 spin_unlock(&hdr->lock); 1294 } 1295 nfs_writedata_release(data); 1296 } 1297 1298 static const struct rpc_call_ops nfs_write_common_ops = { 1299 .rpc_call_prepare = nfs_write_prepare, 1300 .rpc_call_done = nfs_writeback_done_common, 1301 .rpc_release = nfs_writeback_release_common, 1302 }; 1303 1304 1305 /* 1306 * This function is called when the WRITE call is complete. 1307 */ 1308 void nfs_writeback_done(struct rpc_task *task, struct nfs_write_data *data) 1309 { 1310 struct nfs_writeargs *argp = &data->args; 1311 struct nfs_writeres *resp = &data->res; 1312 struct inode *inode = data->header->inode; 1313 int status; 1314 1315 dprintk("NFS: %5u nfs_writeback_done (status %d)\n", 1316 task->tk_pid, task->tk_status); 1317 1318 /* 1319 * ->write_done will attempt to use post-op attributes to detect 1320 * conflicting writes by other clients. A strict interpretation 1321 * of close-to-open would allow us to continue caching even if 1322 * another writer had changed the file, but some applications 1323 * depend on tighter cache coherency when writing. 1324 */ 1325 status = NFS_PROTO(inode)->write_done(task, data); 1326 if (status != 0) 1327 return; 1328 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, resp->count); 1329 1330 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 1331 if (resp->verf->committed < argp->stable && task->tk_status >= 0) { 1332 /* We tried a write call, but the server did not 1333 * commit data to stable storage even though we 1334 * requested it. 1335 * Note: There is a known bug in Tru64 < 5.0 in which 1336 * the server reports NFS_DATA_SYNC, but performs 1337 * NFS_FILE_SYNC. We therefore implement this checking 1338 * as a dprintk() in order to avoid filling syslog. 1339 */ 1340 static unsigned long complain; 1341 1342 /* Note this will print the MDS for a DS write */ 1343 if (time_before(complain, jiffies)) { 1344 dprintk("NFS: faulty NFS server %s:" 1345 " (committed = %d) != (stable = %d)\n", 1346 NFS_SERVER(inode)->nfs_client->cl_hostname, 1347 resp->verf->committed, argp->stable); 1348 complain = jiffies + 300 * HZ; 1349 } 1350 } 1351 #endif 1352 if (task->tk_status < 0) 1353 nfs_set_pgio_error(data->header, task->tk_status, argp->offset); 1354 else if (resp->count < argp->count) { 1355 static unsigned long complain; 1356 1357 /* This a short write! */ 1358 nfs_inc_stats(inode, NFSIOS_SHORTWRITE); 1359 1360 /* Has the server at least made some progress? */ 1361 if (resp->count == 0) { 1362 if (time_before(complain, jiffies)) { 1363 printk(KERN_WARNING 1364 "NFS: Server wrote zero bytes, expected %u.\n", 1365 argp->count); 1366 complain = jiffies + 300 * HZ; 1367 } 1368 nfs_set_pgio_error(data->header, -EIO, argp->offset); 1369 task->tk_status = -EIO; 1370 return; 1371 } 1372 /* Was this an NFSv2 write or an NFSv3 stable write? */ 1373 if (resp->verf->committed != NFS_UNSTABLE) { 1374 /* Resend from where the server left off */ 1375 data->mds_offset += resp->count; 1376 argp->offset += resp->count; 1377 argp->pgbase += resp->count; 1378 argp->count -= resp->count; 1379 } else { 1380 /* Resend as a stable write in order to avoid 1381 * headaches in the case of a server crash. 1382 */ 1383 argp->stable = NFS_FILE_SYNC; 1384 } 1385 rpc_restart_call_prepare(task); 1386 } 1387 } 1388 1389 1390 #if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) 1391 static int nfs_commit_set_lock(struct nfs_inode *nfsi, int may_wait) 1392 { 1393 int ret; 1394 1395 if (!test_and_set_bit(NFS_INO_COMMIT, &nfsi->flags)) 1396 return 1; 1397 if (!may_wait) 1398 return 0; 1399 ret = out_of_line_wait_on_bit_lock(&nfsi->flags, 1400 NFS_INO_COMMIT, 1401 nfs_wait_bit_killable, 1402 TASK_KILLABLE); 1403 return (ret < 0) ? ret : 1; 1404 } 1405 1406 static void nfs_commit_clear_lock(struct nfs_inode *nfsi) 1407 { 1408 clear_bit(NFS_INO_COMMIT, &nfsi->flags); 1409 smp_mb__after_clear_bit(); 1410 wake_up_bit(&nfsi->flags, NFS_INO_COMMIT); 1411 } 1412 1413 void nfs_commitdata_release(struct nfs_commit_data *data) 1414 { 1415 put_nfs_open_context(data->context); 1416 nfs_commit_free(data); 1417 } 1418 EXPORT_SYMBOL_GPL(nfs_commitdata_release); 1419 1420 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, 1421 const struct rpc_call_ops *call_ops, 1422 int how, int flags) 1423 { 1424 struct rpc_task *task; 1425 int priority = flush_task_priority(how); 1426 struct rpc_message msg = { 1427 .rpc_argp = &data->args, 1428 .rpc_resp = &data->res, 1429 .rpc_cred = data->cred, 1430 }; 1431 struct rpc_task_setup task_setup_data = { 1432 .task = &data->task, 1433 .rpc_client = clnt, 1434 .rpc_message = &msg, 1435 .callback_ops = call_ops, 1436 .callback_data = data, 1437 .workqueue = nfsiod_workqueue, 1438 .flags = RPC_TASK_ASYNC | flags, 1439 .priority = priority, 1440 }; 1441 /* Set up the initial task struct. */ 1442 NFS_PROTO(data->inode)->commit_setup(data, &msg); 1443 1444 dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid); 1445 1446 task = rpc_run_task(&task_setup_data); 1447 if (IS_ERR(task)) 1448 return PTR_ERR(task); 1449 if (how & FLUSH_SYNC) 1450 rpc_wait_for_completion_task(task); 1451 rpc_put_task(task); 1452 return 0; 1453 } 1454 EXPORT_SYMBOL_GPL(nfs_initiate_commit); 1455 1456 /* 1457 * Set up the argument/result storage required for the RPC call. 1458 */ 1459 void nfs_init_commit(struct nfs_commit_data *data, 1460 struct list_head *head, 1461 struct pnfs_layout_segment *lseg, 1462 struct nfs_commit_info *cinfo) 1463 { 1464 struct nfs_page *first = nfs_list_entry(head->next); 1465 struct inode *inode = first->wb_context->dentry->d_inode; 1466 1467 /* Set up the RPC argument and reply structs 1468 * NB: take care not to mess about with data->commit et al. */ 1469 1470 list_splice_init(head, &data->pages); 1471 1472 data->inode = inode; 1473 data->cred = first->wb_context->cred; 1474 data->lseg = lseg; /* reference transferred */ 1475 data->mds_ops = &nfs_commit_ops; 1476 data->completion_ops = cinfo->completion_ops; 1477 data->dreq = cinfo->dreq; 1478 1479 data->args.fh = NFS_FH(data->inode); 1480 /* Note: we always request a commit of the entire inode */ 1481 data->args.offset = 0; 1482 data->args.count = 0; 1483 data->context = get_nfs_open_context(first->wb_context); 1484 data->res.fattr = &data->fattr; 1485 data->res.verf = &data->verf; 1486 nfs_fattr_init(&data->fattr); 1487 } 1488 EXPORT_SYMBOL_GPL(nfs_init_commit); 1489 1490 void nfs_retry_commit(struct list_head *page_list, 1491 struct pnfs_layout_segment *lseg, 1492 struct nfs_commit_info *cinfo) 1493 { 1494 struct nfs_page *req; 1495 1496 while (!list_empty(page_list)) { 1497 req = nfs_list_entry(page_list->next); 1498 nfs_list_remove_request(req); 1499 nfs_mark_request_commit(req, lseg, cinfo); 1500 if (!cinfo->dreq) { 1501 dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS); 1502 dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info, 1503 BDI_RECLAIMABLE); 1504 } 1505 nfs_unlock_and_release_request(req); 1506 } 1507 } 1508 EXPORT_SYMBOL_GPL(nfs_retry_commit); 1509 1510 /* 1511 * Commit dirty pages 1512 */ 1513 static int 1514 nfs_commit_list(struct inode *inode, struct list_head *head, int how, 1515 struct nfs_commit_info *cinfo) 1516 { 1517 struct nfs_commit_data *data; 1518 1519 data = nfs_commitdata_alloc(); 1520 1521 if (!data) 1522 goto out_bad; 1523 1524 /* Set up the argument struct */ 1525 nfs_init_commit(data, head, NULL, cinfo); 1526 atomic_inc(&cinfo->mds->rpcs_out); 1527 return nfs_initiate_commit(NFS_CLIENT(inode), data, data->mds_ops, 1528 how, 0); 1529 out_bad: 1530 nfs_retry_commit(head, NULL, cinfo); 1531 cinfo->completion_ops->error_cleanup(NFS_I(inode)); 1532 return -ENOMEM; 1533 } 1534 1535 /* 1536 * COMMIT call returned 1537 */ 1538 static void nfs_commit_done(struct rpc_task *task, void *calldata) 1539 { 1540 struct nfs_commit_data *data = calldata; 1541 1542 dprintk("NFS: %5u nfs_commit_done (status %d)\n", 1543 task->tk_pid, task->tk_status); 1544 1545 /* Call the NFS version-specific code */ 1546 NFS_PROTO(data->inode)->commit_done(task, data); 1547 } 1548 1549 static void nfs_commit_release_pages(struct nfs_commit_data *data) 1550 { 1551 struct nfs_page *req; 1552 int status = data->task.tk_status; 1553 struct nfs_commit_info cinfo; 1554 1555 while (!list_empty(&data->pages)) { 1556 req = nfs_list_entry(data->pages.next); 1557 nfs_list_remove_request(req); 1558 nfs_clear_page_commit(req->wb_page); 1559 1560 dprintk("NFS: commit (%s/%lld %d@%lld)", 1561 req->wb_context->dentry->d_sb->s_id, 1562 (long long)NFS_FILEID(req->wb_context->dentry->d_inode), 1563 req->wb_bytes, 1564 (long long)req_offset(req)); 1565 if (status < 0) { 1566 nfs_context_set_write_error(req->wb_context, status); 1567 nfs_inode_remove_request(req); 1568 dprintk(", error = %d\n", status); 1569 goto next; 1570 } 1571 1572 /* Okay, COMMIT succeeded, apparently. Check the verifier 1573 * returned by the server against all stored verfs. */ 1574 if (!memcmp(&req->wb_verf, &data->verf.verifier, sizeof(req->wb_verf))) { 1575 /* We have a match */ 1576 nfs_inode_remove_request(req); 1577 dprintk(" OK\n"); 1578 goto next; 1579 } 1580 /* We have a mismatch. Write the page again */ 1581 dprintk(" mismatch\n"); 1582 nfs_mark_request_dirty(req); 1583 set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags); 1584 next: 1585 nfs_unlock_and_release_request(req); 1586 } 1587 nfs_init_cinfo(&cinfo, data->inode, data->dreq); 1588 if (atomic_dec_and_test(&cinfo.mds->rpcs_out)) 1589 nfs_commit_clear_lock(NFS_I(data->inode)); 1590 } 1591 1592 static void nfs_commit_release(void *calldata) 1593 { 1594 struct nfs_commit_data *data = calldata; 1595 1596 data->completion_ops->completion(data); 1597 nfs_commitdata_release(calldata); 1598 } 1599 1600 static const struct rpc_call_ops nfs_commit_ops = { 1601 .rpc_call_prepare = nfs_commit_prepare, 1602 .rpc_call_done = nfs_commit_done, 1603 .rpc_release = nfs_commit_release, 1604 }; 1605 1606 static const struct nfs_commit_completion_ops nfs_commit_completion_ops = { 1607 .completion = nfs_commit_release_pages, 1608 .error_cleanup = nfs_commit_clear_lock, 1609 }; 1610 1611 int nfs_generic_commit_list(struct inode *inode, struct list_head *head, 1612 int how, struct nfs_commit_info *cinfo) 1613 { 1614 int status; 1615 1616 status = pnfs_commit_list(inode, head, how, cinfo); 1617 if (status == PNFS_NOT_ATTEMPTED) 1618 status = nfs_commit_list(inode, head, how, cinfo); 1619 return status; 1620 } 1621 1622 int nfs_commit_inode(struct inode *inode, int how) 1623 { 1624 LIST_HEAD(head); 1625 struct nfs_commit_info cinfo; 1626 int may_wait = how & FLUSH_SYNC; 1627 int res; 1628 1629 res = nfs_commit_set_lock(NFS_I(inode), may_wait); 1630 if (res <= 0) 1631 goto out_mark_dirty; 1632 nfs_init_cinfo_from_inode(&cinfo, inode); 1633 res = nfs_scan_commit(inode, &head, &cinfo); 1634 if (res) { 1635 int error; 1636 1637 error = nfs_generic_commit_list(inode, &head, how, &cinfo); 1638 if (error < 0) 1639 return error; 1640 if (!may_wait) 1641 goto out_mark_dirty; 1642 error = wait_on_bit(&NFS_I(inode)->flags, 1643 NFS_INO_COMMIT, 1644 nfs_wait_bit_killable, 1645 TASK_KILLABLE); 1646 if (error < 0) 1647 return error; 1648 } else 1649 nfs_commit_clear_lock(NFS_I(inode)); 1650 return res; 1651 /* Note: If we exit without ensuring that the commit is complete, 1652 * we must mark the inode as dirty. Otherwise, future calls to 1653 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure 1654 * that the data is on the disk. 1655 */ 1656 out_mark_dirty: 1657 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1658 return res; 1659 } 1660 1661 static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc) 1662 { 1663 struct nfs_inode *nfsi = NFS_I(inode); 1664 int flags = FLUSH_SYNC; 1665 int ret = 0; 1666 1667 /* no commits means nothing needs to be done */ 1668 if (!nfsi->commit_info.ncommit) 1669 return ret; 1670 1671 if (wbc->sync_mode == WB_SYNC_NONE) { 1672 /* Don't commit yet if this is a non-blocking flush and there 1673 * are a lot of outstanding writes for this mapping. 1674 */ 1675 if (nfsi->commit_info.ncommit <= (nfsi->npages >> 1)) 1676 goto out_mark_dirty; 1677 1678 /* don't wait for the COMMIT response */ 1679 flags = 0; 1680 } 1681 1682 ret = nfs_commit_inode(inode, flags); 1683 if (ret >= 0) { 1684 if (wbc->sync_mode == WB_SYNC_NONE) { 1685 if (ret < wbc->nr_to_write) 1686 wbc->nr_to_write -= ret; 1687 else 1688 wbc->nr_to_write = 0; 1689 } 1690 return 0; 1691 } 1692 out_mark_dirty: 1693 __mark_inode_dirty(inode, I_DIRTY_DATASYNC); 1694 return ret; 1695 } 1696 #else 1697 static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc) 1698 { 1699 return 0; 1700 } 1701 #endif 1702 1703 int nfs_write_inode(struct inode *inode, struct writeback_control *wbc) 1704 { 1705 return nfs_commit_unstable_pages(inode, wbc); 1706 } 1707 EXPORT_SYMBOL_GPL(nfs_write_inode); 1708 1709 /* 1710 * flush the inode to disk. 1711 */ 1712 int nfs_wb_all(struct inode *inode) 1713 { 1714 struct writeback_control wbc = { 1715 .sync_mode = WB_SYNC_ALL, 1716 .nr_to_write = LONG_MAX, 1717 .range_start = 0, 1718 .range_end = LLONG_MAX, 1719 }; 1720 1721 return sync_inode(inode, &wbc); 1722 } 1723 EXPORT_SYMBOL_GPL(nfs_wb_all); 1724 1725 int nfs_wb_page_cancel(struct inode *inode, struct page *page) 1726 { 1727 struct nfs_page *req; 1728 int ret = 0; 1729 1730 for (;;) { 1731 wait_on_page_writeback(page); 1732 req = nfs_page_find_request(page); 1733 if (req == NULL) 1734 break; 1735 if (nfs_lock_request(req)) { 1736 nfs_clear_request_commit(req); 1737 nfs_inode_remove_request(req); 1738 /* 1739 * In case nfs_inode_remove_request has marked the 1740 * page as being dirty 1741 */ 1742 cancel_dirty_page(page, PAGE_CACHE_SIZE); 1743 nfs_unlock_and_release_request(req); 1744 break; 1745 } 1746 ret = nfs_wait_on_request(req); 1747 nfs_release_request(req); 1748 if (ret < 0) 1749 break; 1750 } 1751 return ret; 1752 } 1753 1754 /* 1755 * Write back all requests on one page - we do this before reading it. 1756 */ 1757 int nfs_wb_page(struct inode *inode, struct page *page) 1758 { 1759 loff_t range_start = page_file_offset(page); 1760 loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1); 1761 struct writeback_control wbc = { 1762 .sync_mode = WB_SYNC_ALL, 1763 .nr_to_write = 0, 1764 .range_start = range_start, 1765 .range_end = range_end, 1766 }; 1767 int ret; 1768 1769 for (;;) { 1770 wait_on_page_writeback(page); 1771 if (clear_page_dirty_for_io(page)) { 1772 ret = nfs_writepage_locked(page, &wbc); 1773 if (ret < 0) 1774 goto out_error; 1775 continue; 1776 } 1777 if (!PagePrivate(page)) 1778 break; 1779 ret = nfs_commit_inode(inode, FLUSH_SYNC); 1780 if (ret < 0) 1781 goto out_error; 1782 } 1783 return 0; 1784 out_error: 1785 return ret; 1786 } 1787 1788 #ifdef CONFIG_MIGRATION 1789 int nfs_migrate_page(struct address_space *mapping, struct page *newpage, 1790 struct page *page, enum migrate_mode mode) 1791 { 1792 /* 1793 * If PagePrivate is set, then the page is currently associated with 1794 * an in-progress read or write request. Don't try to migrate it. 1795 * 1796 * FIXME: we could do this in principle, but we'll need a way to ensure 1797 * that we can safely release the inode reference while holding 1798 * the page lock. 1799 */ 1800 if (PagePrivate(page)) 1801 return -EBUSY; 1802 1803 if (!nfs_fscache_release_page(page, GFP_KERNEL)) 1804 return -EBUSY; 1805 1806 return migrate_page(mapping, newpage, page, mode); 1807 } 1808 #endif 1809 1810 int __init nfs_init_writepagecache(void) 1811 { 1812 nfs_wdata_cachep = kmem_cache_create("nfs_write_data", 1813 sizeof(struct nfs_write_header), 1814 0, SLAB_HWCACHE_ALIGN, 1815 NULL); 1816 if (nfs_wdata_cachep == NULL) 1817 return -ENOMEM; 1818 1819 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE, 1820 nfs_wdata_cachep); 1821 if (nfs_wdata_mempool == NULL) 1822 goto out_destroy_write_cache; 1823 1824 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data", 1825 sizeof(struct nfs_commit_data), 1826 0, SLAB_HWCACHE_ALIGN, 1827 NULL); 1828 if (nfs_cdata_cachep == NULL) 1829 goto out_destroy_write_mempool; 1830 1831 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT, 1832 nfs_cdata_cachep); 1833 if (nfs_commit_mempool == NULL) 1834 goto out_destroy_commit_cache; 1835 1836 /* 1837 * NFS congestion size, scale with available memory. 1838 * 1839 * 64MB: 8192k 1840 * 128MB: 11585k 1841 * 256MB: 16384k 1842 * 512MB: 23170k 1843 * 1GB: 32768k 1844 * 2GB: 46340k 1845 * 4GB: 65536k 1846 * 8GB: 92681k 1847 * 16GB: 131072k 1848 * 1849 * This allows larger machines to have larger/more transfers. 1850 * Limit the default to 256M 1851 */ 1852 nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10); 1853 if (nfs_congestion_kb > 256*1024) 1854 nfs_congestion_kb = 256*1024; 1855 1856 return 0; 1857 1858 out_destroy_commit_cache: 1859 kmem_cache_destroy(nfs_cdata_cachep); 1860 out_destroy_write_mempool: 1861 mempool_destroy(nfs_wdata_mempool); 1862 out_destroy_write_cache: 1863 kmem_cache_destroy(nfs_wdata_cachep); 1864 return -ENOMEM; 1865 } 1866 1867 void nfs_destroy_writepagecache(void) 1868 { 1869 mempool_destroy(nfs_commit_mempool); 1870 kmem_cache_destroy(nfs_cdata_cachep); 1871 mempool_destroy(nfs_wdata_mempool); 1872 kmem_cache_destroy(nfs_wdata_cachep); 1873 } 1874 1875