1 /* 2 * linux/fs/nfs/read.c 3 * 4 * Block I/O for NFS 5 * 6 * Partial copy of Linus' read cache modifications to fs/nfs/file.c 7 * modified for async RPC by okir@monad.swb.de 8 */ 9 10 #include <linux/time.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/fcntl.h> 14 #include <linux/stat.h> 15 #include <linux/mm.h> 16 #include <linux/slab.h> 17 #include <linux/pagemap.h> 18 #include <linux/sunrpc/clnt.h> 19 #include <linux/nfs_fs.h> 20 #include <linux/nfs_page.h> 21 #include <linux/module.h> 22 23 #include <asm/system.h> 24 #include "pnfs.h" 25 26 #include "nfs4_fs.h" 27 #include "internal.h" 28 #include "iostat.h" 29 #include "fscache.h" 30 31 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 32 33 static int nfs_pagein_multi(struct nfs_pageio_descriptor *desc); 34 static int nfs_pagein_one(struct nfs_pageio_descriptor *desc); 35 static const struct rpc_call_ops nfs_read_partial_ops; 36 static const struct rpc_call_ops nfs_read_full_ops; 37 38 static struct kmem_cache *nfs_rdata_cachep; 39 static mempool_t *nfs_rdata_mempool; 40 41 #define MIN_POOL_READ (32) 42 43 struct nfs_read_data *nfs_readdata_alloc(unsigned int pagecount) 44 { 45 struct nfs_read_data *p = mempool_alloc(nfs_rdata_mempool, GFP_KERNEL); 46 47 if (p) { 48 memset(p, 0, sizeof(*p)); 49 INIT_LIST_HEAD(&p->pages); 50 p->npages = pagecount; 51 if (pagecount <= ARRAY_SIZE(p->page_array)) 52 p->pagevec = p->page_array; 53 else { 54 p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL); 55 if (!p->pagevec) { 56 mempool_free(p, nfs_rdata_mempool); 57 p = NULL; 58 } 59 } 60 } 61 return p; 62 } 63 64 void nfs_readdata_free(struct nfs_read_data *p) 65 { 66 if (p && (p->pagevec != &p->page_array[0])) 67 kfree(p->pagevec); 68 mempool_free(p, nfs_rdata_mempool); 69 } 70 71 static void nfs_readdata_release(struct nfs_read_data *rdata) 72 { 73 put_lseg(rdata->lseg); 74 put_nfs_open_context(rdata->args.context); 75 nfs_readdata_free(rdata); 76 } 77 78 static 79 int nfs_return_empty_page(struct page *page) 80 { 81 zero_user(page, 0, PAGE_CACHE_SIZE); 82 SetPageUptodate(page); 83 unlock_page(page); 84 return 0; 85 } 86 87 static void nfs_readpage_truncate_uninitialised_page(struct nfs_read_data *data) 88 { 89 unsigned int remainder = data->args.count - data->res.count; 90 unsigned int base = data->args.pgbase + data->res.count; 91 unsigned int pglen; 92 struct page **pages; 93 94 if (data->res.eof == 0 || remainder == 0) 95 return; 96 /* 97 * Note: "remainder" can never be negative, since we check for 98 * this in the XDR code. 99 */ 100 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT]; 101 base &= ~PAGE_CACHE_MASK; 102 pglen = PAGE_CACHE_SIZE - base; 103 for (;;) { 104 if (remainder <= pglen) { 105 zero_user(*pages, base, remainder); 106 break; 107 } 108 zero_user(*pages, base, pglen); 109 pages++; 110 remainder -= pglen; 111 pglen = PAGE_CACHE_SIZE; 112 base = 0; 113 } 114 } 115 116 int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode, 117 struct page *page) 118 { 119 struct nfs_page *new; 120 unsigned int len; 121 struct nfs_pageio_descriptor pgio; 122 123 len = nfs_page_length(page); 124 if (len == 0) 125 return nfs_return_empty_page(page); 126 new = nfs_create_request(ctx, inode, page, 0, len); 127 if (IS_ERR(new)) { 128 unlock_page(page); 129 return PTR_ERR(new); 130 } 131 if (len < PAGE_CACHE_SIZE) 132 zero_user_segment(page, len, PAGE_CACHE_SIZE); 133 134 nfs_pageio_init(&pgio, inode, NULL, 0, 0); 135 nfs_list_add_request(new, &pgio.pg_list); 136 pgio.pg_count = len; 137 138 if (NFS_SERVER(inode)->rsize < PAGE_CACHE_SIZE) 139 nfs_pagein_multi(&pgio); 140 else 141 nfs_pagein_one(&pgio); 142 return 0; 143 } 144 145 static void nfs_readpage_release(struct nfs_page *req) 146 { 147 struct inode *d_inode = req->wb_context->path.dentry->d_inode; 148 149 if (PageUptodate(req->wb_page)) 150 nfs_readpage_to_fscache(d_inode, req->wb_page, 0); 151 152 unlock_page(req->wb_page); 153 154 dprintk("NFS: read done (%s/%Ld %d@%Ld)\n", 155 req->wb_context->path.dentry->d_inode->i_sb->s_id, 156 (long long)NFS_FILEID(req->wb_context->path.dentry->d_inode), 157 req->wb_bytes, 158 (long long)req_offset(req)); 159 nfs_release_request(req); 160 } 161 162 int nfs_initiate_read(struct nfs_read_data *data, struct rpc_clnt *clnt, 163 const struct rpc_call_ops *call_ops) 164 { 165 struct inode *inode = data->inode; 166 int swap_flags = IS_SWAPFILE(inode) ? NFS_RPC_SWAPFLAGS : 0; 167 struct rpc_task *task; 168 struct rpc_message msg = { 169 .rpc_argp = &data->args, 170 .rpc_resp = &data->res, 171 .rpc_cred = data->cred, 172 }; 173 struct rpc_task_setup task_setup_data = { 174 .task = &data->task, 175 .rpc_client = clnt, 176 .rpc_message = &msg, 177 .callback_ops = call_ops, 178 .callback_data = data, 179 .workqueue = nfsiod_workqueue, 180 .flags = RPC_TASK_ASYNC | swap_flags, 181 }; 182 183 /* Set up the initial task struct. */ 184 NFS_PROTO(inode)->read_setup(data, &msg); 185 186 dprintk("NFS: %5u initiated read call (req %s/%lld, %u bytes @ " 187 "offset %llu)\n", 188 data->task.tk_pid, 189 inode->i_sb->s_id, 190 (long long)NFS_FILEID(inode), 191 data->args.count, 192 (unsigned long long)data->args.offset); 193 194 task = rpc_run_task(&task_setup_data); 195 if (IS_ERR(task)) 196 return PTR_ERR(task); 197 rpc_put_task(task); 198 return 0; 199 } 200 EXPORT_SYMBOL_GPL(nfs_initiate_read); 201 202 /* 203 * Set up the NFS read request struct 204 */ 205 static int nfs_read_rpcsetup(struct nfs_page *req, struct nfs_read_data *data, 206 const struct rpc_call_ops *call_ops, 207 unsigned int count, unsigned int offset, 208 struct pnfs_layout_segment *lseg) 209 { 210 struct inode *inode = req->wb_context->path.dentry->d_inode; 211 212 data->req = req; 213 data->inode = inode; 214 data->cred = req->wb_context->cred; 215 data->lseg = get_lseg(lseg); 216 217 data->args.fh = NFS_FH(inode); 218 data->args.offset = req_offset(req) + offset; 219 data->args.pgbase = req->wb_pgbase + offset; 220 data->args.pages = data->pagevec; 221 data->args.count = count; 222 data->args.context = get_nfs_open_context(req->wb_context); 223 data->args.lock_context = req->wb_lock_context; 224 225 data->res.fattr = &data->fattr; 226 data->res.count = count; 227 data->res.eof = 0; 228 nfs_fattr_init(&data->fattr); 229 230 if (data->lseg && 231 (pnfs_try_to_read_data(data, call_ops) == PNFS_ATTEMPTED)) 232 return 0; 233 234 return nfs_initiate_read(data, NFS_CLIENT(inode), call_ops); 235 } 236 237 static void 238 nfs_async_read_error(struct list_head *head) 239 { 240 struct nfs_page *req; 241 242 while (!list_empty(head)) { 243 req = nfs_list_entry(head->next); 244 nfs_list_remove_request(req); 245 SetPageError(req->wb_page); 246 nfs_readpage_release(req); 247 } 248 } 249 250 /* 251 * Generate multiple requests to fill a single page. 252 * 253 * We optimize to reduce the number of read operations on the wire. If we 254 * detect that we're reading a page, or an area of a page, that is past the 255 * end of file, we do not generate NFS read operations but just clear the 256 * parts of the page that would have come back zero from the server anyway. 257 * 258 * We rely on the cached value of i_size to make this determination; another 259 * client can fill pages on the server past our cached end-of-file, but we 260 * won't see the new data until our attribute cache is updated. This is more 261 * or less conventional NFS client behavior. 262 */ 263 static int nfs_pagein_multi(struct nfs_pageio_descriptor *desc) 264 { 265 struct nfs_page *req = nfs_list_entry(desc->pg_list.next); 266 struct page *page = req->wb_page; 267 struct nfs_read_data *data; 268 size_t rsize = NFS_SERVER(desc->pg_inode)->rsize, nbytes; 269 unsigned int offset; 270 int requests = 0; 271 int ret = 0; 272 struct pnfs_layout_segment *lseg; 273 LIST_HEAD(list); 274 275 nfs_list_remove_request(req); 276 277 nbytes = desc->pg_count; 278 do { 279 size_t len = min(nbytes,rsize); 280 281 data = nfs_readdata_alloc(1); 282 if (!data) 283 goto out_bad; 284 list_add(&data->pages, &list); 285 requests++; 286 nbytes -= len; 287 } while(nbytes != 0); 288 atomic_set(&req->wb_complete, requests); 289 290 BUG_ON(desc->pg_lseg != NULL); 291 lseg = pnfs_update_layout(desc->pg_inode, req->wb_context, 292 req_offset(req), desc->pg_count, 293 IOMODE_READ, GFP_KERNEL); 294 ClearPageError(page); 295 offset = 0; 296 nbytes = desc->pg_count; 297 do { 298 int ret2; 299 300 data = list_entry(list.next, struct nfs_read_data, pages); 301 list_del_init(&data->pages); 302 303 data->pagevec[0] = page; 304 305 if (nbytes < rsize) 306 rsize = nbytes; 307 ret2 = nfs_read_rpcsetup(req, data, &nfs_read_partial_ops, 308 rsize, offset, lseg); 309 if (ret == 0) 310 ret = ret2; 311 offset += rsize; 312 nbytes -= rsize; 313 } while (nbytes != 0); 314 put_lseg(lseg); 315 desc->pg_lseg = NULL; 316 317 return ret; 318 319 out_bad: 320 while (!list_empty(&list)) { 321 data = list_entry(list.next, struct nfs_read_data, pages); 322 list_del(&data->pages); 323 nfs_readdata_free(data); 324 } 325 SetPageError(page); 326 nfs_readpage_release(req); 327 return -ENOMEM; 328 } 329 330 static int nfs_pagein_one(struct nfs_pageio_descriptor *desc) 331 { 332 struct nfs_page *req; 333 struct page **pages; 334 struct nfs_read_data *data; 335 struct list_head *head = &desc->pg_list; 336 struct pnfs_layout_segment *lseg = desc->pg_lseg; 337 int ret = -ENOMEM; 338 339 data = nfs_readdata_alloc(nfs_page_array_len(desc->pg_base, 340 desc->pg_count)); 341 if (!data) { 342 nfs_async_read_error(head); 343 goto out; 344 } 345 346 pages = data->pagevec; 347 while (!list_empty(head)) { 348 req = nfs_list_entry(head->next); 349 nfs_list_remove_request(req); 350 nfs_list_add_request(req, &data->pages); 351 ClearPageError(req->wb_page); 352 *pages++ = req->wb_page; 353 } 354 req = nfs_list_entry(data->pages.next); 355 if ((!lseg) && list_is_singular(&data->pages)) 356 lseg = pnfs_update_layout(desc->pg_inode, req->wb_context, 357 req_offset(req), desc->pg_count, 358 IOMODE_READ, GFP_KERNEL); 359 360 ret = nfs_read_rpcsetup(req, data, &nfs_read_full_ops, desc->pg_count, 361 0, lseg); 362 out: 363 put_lseg(lseg); 364 desc->pg_lseg = NULL; 365 return ret; 366 } 367 368 /* 369 * This is the callback from RPC telling us whether a reply was 370 * received or some error occurred (timeout or socket shutdown). 371 */ 372 int nfs_readpage_result(struct rpc_task *task, struct nfs_read_data *data) 373 { 374 int status; 375 376 dprintk("NFS: %s: %5u, (status %d)\n", __func__, task->tk_pid, 377 task->tk_status); 378 379 status = NFS_PROTO(data->inode)->read_done(task, data); 380 if (status != 0) 381 return status; 382 383 nfs_add_stats(data->inode, NFSIOS_SERVERREADBYTES, data->res.count); 384 385 if (task->tk_status == -ESTALE) { 386 set_bit(NFS_INO_STALE, &NFS_I(data->inode)->flags); 387 nfs_mark_for_revalidate(data->inode); 388 } 389 return 0; 390 } 391 392 static void nfs_readpage_retry(struct rpc_task *task, struct nfs_read_data *data) 393 { 394 struct nfs_readargs *argp = &data->args; 395 struct nfs_readres *resp = &data->res; 396 397 if (resp->eof || resp->count == argp->count) 398 return; 399 400 /* This is a short read! */ 401 nfs_inc_stats(data->inode, NFSIOS_SHORTREAD); 402 /* Has the server at least made some progress? */ 403 if (resp->count == 0) 404 return; 405 406 /* Yes, so retry the read at the end of the data */ 407 data->mds_offset += resp->count; 408 argp->offset += resp->count; 409 argp->pgbase += resp->count; 410 argp->count -= resp->count; 411 nfs_restart_rpc(task, NFS_SERVER(data->inode)->nfs_client); 412 } 413 414 /* 415 * Handle a read reply that fills part of a page. 416 */ 417 static void nfs_readpage_result_partial(struct rpc_task *task, void *calldata) 418 { 419 struct nfs_read_data *data = calldata; 420 421 if (nfs_readpage_result(task, data) != 0) 422 return; 423 if (task->tk_status < 0) 424 return; 425 426 nfs_readpage_truncate_uninitialised_page(data); 427 nfs_readpage_retry(task, data); 428 } 429 430 static void nfs_readpage_release_partial(void *calldata) 431 { 432 struct nfs_read_data *data = calldata; 433 struct nfs_page *req = data->req; 434 struct page *page = req->wb_page; 435 int status = data->task.tk_status; 436 437 if (status < 0) 438 SetPageError(page); 439 440 if (atomic_dec_and_test(&req->wb_complete)) { 441 if (!PageError(page)) 442 SetPageUptodate(page); 443 nfs_readpage_release(req); 444 } 445 nfs_readdata_release(calldata); 446 } 447 448 #if defined(CONFIG_NFS_V4_1) 449 void nfs_read_prepare(struct rpc_task *task, void *calldata) 450 { 451 struct nfs_read_data *data = calldata; 452 453 if (nfs4_setup_sequence(NFS_SERVER(data->inode), 454 &data->args.seq_args, &data->res.seq_res, 455 0, task)) 456 return; 457 rpc_call_start(task); 458 } 459 #endif /* CONFIG_NFS_V4_1 */ 460 461 static const struct rpc_call_ops nfs_read_partial_ops = { 462 #if defined(CONFIG_NFS_V4_1) 463 .rpc_call_prepare = nfs_read_prepare, 464 #endif /* CONFIG_NFS_V4_1 */ 465 .rpc_call_done = nfs_readpage_result_partial, 466 .rpc_release = nfs_readpage_release_partial, 467 }; 468 469 static void nfs_readpage_set_pages_uptodate(struct nfs_read_data *data) 470 { 471 unsigned int count = data->res.count; 472 unsigned int base = data->args.pgbase; 473 struct page **pages; 474 475 if (data->res.eof) 476 count = data->args.count; 477 if (unlikely(count == 0)) 478 return; 479 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT]; 480 base &= ~PAGE_CACHE_MASK; 481 count += base; 482 for (;count >= PAGE_CACHE_SIZE; count -= PAGE_CACHE_SIZE, pages++) 483 SetPageUptodate(*pages); 484 if (count == 0) 485 return; 486 /* Was this a short read? */ 487 if (data->res.eof || data->res.count == data->args.count) 488 SetPageUptodate(*pages); 489 } 490 491 /* 492 * This is the callback from RPC telling us whether a reply was 493 * received or some error occurred (timeout or socket shutdown). 494 */ 495 static void nfs_readpage_result_full(struct rpc_task *task, void *calldata) 496 { 497 struct nfs_read_data *data = calldata; 498 499 if (nfs_readpage_result(task, data) != 0) 500 return; 501 if (task->tk_status < 0) 502 return; 503 /* 504 * Note: nfs_readpage_retry may change the values of 505 * data->args. In the multi-page case, we therefore need 506 * to ensure that we call nfs_readpage_set_pages_uptodate() 507 * first. 508 */ 509 nfs_readpage_truncate_uninitialised_page(data); 510 nfs_readpage_set_pages_uptodate(data); 511 nfs_readpage_retry(task, data); 512 } 513 514 static void nfs_readpage_release_full(void *calldata) 515 { 516 struct nfs_read_data *data = calldata; 517 518 while (!list_empty(&data->pages)) { 519 struct nfs_page *req = nfs_list_entry(data->pages.next); 520 521 nfs_list_remove_request(req); 522 nfs_readpage_release(req); 523 } 524 nfs_readdata_release(calldata); 525 } 526 527 static const struct rpc_call_ops nfs_read_full_ops = { 528 #if defined(CONFIG_NFS_V4_1) 529 .rpc_call_prepare = nfs_read_prepare, 530 #endif /* CONFIG_NFS_V4_1 */ 531 .rpc_call_done = nfs_readpage_result_full, 532 .rpc_release = nfs_readpage_release_full, 533 }; 534 535 /* 536 * Read a page over NFS. 537 * We read the page synchronously in the following case: 538 * - The error flag is set for this page. This happens only when a 539 * previous async read operation failed. 540 */ 541 int nfs_readpage(struct file *file, struct page *page) 542 { 543 struct nfs_open_context *ctx; 544 struct inode *inode = page->mapping->host; 545 int error; 546 547 dprintk("NFS: nfs_readpage (%p %ld@%lu)\n", 548 page, PAGE_CACHE_SIZE, page->index); 549 nfs_inc_stats(inode, NFSIOS_VFSREADPAGE); 550 nfs_add_stats(inode, NFSIOS_READPAGES, 1); 551 552 /* 553 * Try to flush any pending writes to the file.. 554 * 555 * NOTE! Because we own the page lock, there cannot 556 * be any new pending writes generated at this point 557 * for this page (other pages can be written to). 558 */ 559 error = nfs_wb_page(inode, page); 560 if (error) 561 goto out_unlock; 562 if (PageUptodate(page)) 563 goto out_unlock; 564 565 error = -ESTALE; 566 if (NFS_STALE(inode)) 567 goto out_unlock; 568 569 if (file == NULL) { 570 error = -EBADF; 571 ctx = nfs_find_open_context(inode, NULL, FMODE_READ); 572 if (ctx == NULL) 573 goto out_unlock; 574 } else 575 ctx = get_nfs_open_context(nfs_file_open_context(file)); 576 577 if (!IS_SYNC(inode)) { 578 error = nfs_readpage_from_fscache(ctx, inode, page); 579 if (error == 0) 580 goto out; 581 } 582 583 error = nfs_readpage_async(ctx, inode, page); 584 585 out: 586 put_nfs_open_context(ctx); 587 return error; 588 out_unlock: 589 unlock_page(page); 590 return error; 591 } 592 593 struct nfs_readdesc { 594 struct nfs_pageio_descriptor *pgio; 595 struct nfs_open_context *ctx; 596 }; 597 598 static int 599 readpage_async_filler(void *data, struct page *page) 600 { 601 struct nfs_readdesc *desc = (struct nfs_readdesc *)data; 602 struct inode *inode = page->mapping->host; 603 struct nfs_page *new; 604 unsigned int len; 605 int error; 606 607 len = nfs_page_length(page); 608 if (len == 0) 609 return nfs_return_empty_page(page); 610 611 new = nfs_create_request(desc->ctx, inode, page, 0, len); 612 if (IS_ERR(new)) 613 goto out_error; 614 615 if (len < PAGE_CACHE_SIZE) 616 zero_user_segment(page, len, PAGE_CACHE_SIZE); 617 if (!nfs_pageio_add_request(desc->pgio, new)) { 618 error = desc->pgio->pg_error; 619 goto out_unlock; 620 } 621 return 0; 622 out_error: 623 error = PTR_ERR(new); 624 SetPageError(page); 625 out_unlock: 626 unlock_page(page); 627 return error; 628 } 629 630 int nfs_readpages(struct file *filp, struct address_space *mapping, 631 struct list_head *pages, unsigned nr_pages) 632 { 633 struct nfs_pageio_descriptor pgio; 634 struct nfs_readdesc desc = { 635 .pgio = &pgio, 636 }; 637 struct inode *inode = mapping->host; 638 struct nfs_server *server = NFS_SERVER(inode); 639 size_t rsize = server->rsize; 640 unsigned long npages; 641 int ret = -ESTALE; 642 643 dprintk("NFS: nfs_readpages (%s/%Ld %d)\n", 644 inode->i_sb->s_id, 645 (long long)NFS_FILEID(inode), 646 nr_pages); 647 nfs_inc_stats(inode, NFSIOS_VFSREADPAGES); 648 649 if (NFS_STALE(inode)) 650 goto out; 651 652 if (filp == NULL) { 653 desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ); 654 if (desc.ctx == NULL) 655 return -EBADF; 656 } else 657 desc.ctx = get_nfs_open_context(nfs_file_open_context(filp)); 658 659 /* attempt to read as many of the pages as possible from the cache 660 * - this returns -ENOBUFS immediately if the cookie is negative 661 */ 662 ret = nfs_readpages_from_fscache(desc.ctx, inode, mapping, 663 pages, &nr_pages); 664 if (ret == 0) 665 goto read_complete; /* all pages were read */ 666 667 if (rsize < PAGE_CACHE_SIZE) 668 nfs_pageio_init(&pgio, inode, nfs_pagein_multi, rsize, 0); 669 else 670 nfs_pageio_init(&pgio, inode, nfs_pagein_one, rsize, 0); 671 672 ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc); 673 674 nfs_pageio_complete(&pgio); 675 npages = (pgio.pg_bytes_written + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 676 nfs_add_stats(inode, NFSIOS_READPAGES, npages); 677 read_complete: 678 put_nfs_open_context(desc.ctx); 679 out: 680 return ret; 681 } 682 683 int __init nfs_init_readpagecache(void) 684 { 685 nfs_rdata_cachep = kmem_cache_create("nfs_read_data", 686 sizeof(struct nfs_read_data), 687 0, SLAB_HWCACHE_ALIGN, 688 NULL); 689 if (nfs_rdata_cachep == NULL) 690 return -ENOMEM; 691 692 nfs_rdata_mempool = mempool_create_slab_pool(MIN_POOL_READ, 693 nfs_rdata_cachep); 694 if (nfs_rdata_mempool == NULL) 695 return -ENOMEM; 696 697 return 0; 698 } 699 700 void nfs_destroy_readpagecache(void) 701 { 702 mempool_destroy(nfs_rdata_mempool); 703 kmem_cache_destroy(nfs_rdata_cachep); 704 } 705