1 /* 2 * linux/fs/nfs/read.c 3 * 4 * Block I/O for NFS 5 * 6 * Partial copy of Linus' read cache modifications to fs/nfs/file.c 7 * modified for async RPC by okir@monad.swb.de 8 */ 9 10 #include <linux/time.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/fcntl.h> 14 #include <linux/stat.h> 15 #include <linux/mm.h> 16 #include <linux/slab.h> 17 #include <linux/pagemap.h> 18 #include <linux/sunrpc/clnt.h> 19 #include <linux/nfs_fs.h> 20 #include <linux/nfs_page.h> 21 #include <linux/smp_lock.h> 22 23 #include <asm/system.h> 24 25 #include "internal.h" 26 #include "iostat.h" 27 #include "fscache.h" 28 29 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 30 31 static int nfs_pagein_multi(struct inode *, struct list_head *, unsigned int, size_t, int); 32 static int nfs_pagein_one(struct inode *, struct list_head *, unsigned int, size_t, int); 33 static const struct rpc_call_ops nfs_read_partial_ops; 34 static const struct rpc_call_ops nfs_read_full_ops; 35 36 static struct kmem_cache *nfs_rdata_cachep; 37 static mempool_t *nfs_rdata_mempool; 38 39 #define MIN_POOL_READ (32) 40 41 struct nfs_read_data *nfs_readdata_alloc(unsigned int pagecount) 42 { 43 struct nfs_read_data *p = mempool_alloc(nfs_rdata_mempool, GFP_NOFS); 44 45 if (p) { 46 memset(p, 0, sizeof(*p)); 47 INIT_LIST_HEAD(&p->pages); 48 p->npages = pagecount; 49 if (pagecount <= ARRAY_SIZE(p->page_array)) 50 p->pagevec = p->page_array; 51 else { 52 p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_NOFS); 53 if (!p->pagevec) { 54 mempool_free(p, nfs_rdata_mempool); 55 p = NULL; 56 } 57 } 58 } 59 return p; 60 } 61 62 static void nfs_readdata_free(struct nfs_read_data *p) 63 { 64 if (p && (p->pagevec != &p->page_array[0])) 65 kfree(p->pagevec); 66 mempool_free(p, nfs_rdata_mempool); 67 } 68 69 void nfs_readdata_release(void *data) 70 { 71 struct nfs_read_data *rdata = data; 72 73 put_nfs_open_context(rdata->args.context); 74 nfs_readdata_free(rdata); 75 } 76 77 static 78 int nfs_return_empty_page(struct page *page) 79 { 80 zero_user(page, 0, PAGE_CACHE_SIZE); 81 SetPageUptodate(page); 82 unlock_page(page); 83 return 0; 84 } 85 86 static void nfs_readpage_truncate_uninitialised_page(struct nfs_read_data *data) 87 { 88 unsigned int remainder = data->args.count - data->res.count; 89 unsigned int base = data->args.pgbase + data->res.count; 90 unsigned int pglen; 91 struct page **pages; 92 93 if (data->res.eof == 0 || remainder == 0) 94 return; 95 /* 96 * Note: "remainder" can never be negative, since we check for 97 * this in the XDR code. 98 */ 99 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT]; 100 base &= ~PAGE_CACHE_MASK; 101 pglen = PAGE_CACHE_SIZE - base; 102 for (;;) { 103 if (remainder <= pglen) { 104 zero_user(*pages, base, remainder); 105 break; 106 } 107 zero_user(*pages, base, pglen); 108 pages++; 109 remainder -= pglen; 110 pglen = PAGE_CACHE_SIZE; 111 base = 0; 112 } 113 } 114 115 int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode, 116 struct page *page) 117 { 118 LIST_HEAD(one_request); 119 struct nfs_page *new; 120 unsigned int len; 121 122 len = nfs_page_length(page); 123 if (len == 0) 124 return nfs_return_empty_page(page); 125 new = nfs_create_request(ctx, inode, page, 0, len); 126 if (IS_ERR(new)) { 127 unlock_page(page); 128 return PTR_ERR(new); 129 } 130 if (len < PAGE_CACHE_SIZE) 131 zero_user_segment(page, len, PAGE_CACHE_SIZE); 132 133 nfs_list_add_request(new, &one_request); 134 if (NFS_SERVER(inode)->rsize < PAGE_CACHE_SIZE) 135 nfs_pagein_multi(inode, &one_request, 1, len, 0); 136 else 137 nfs_pagein_one(inode, &one_request, 1, len, 0); 138 return 0; 139 } 140 141 static void nfs_readpage_release(struct nfs_page *req) 142 { 143 struct inode *d_inode = req->wb_context->path.dentry->d_inode; 144 145 if (PageUptodate(req->wb_page)) 146 nfs_readpage_to_fscache(d_inode, req->wb_page, 0); 147 148 unlock_page(req->wb_page); 149 150 dprintk("NFS: read done (%s/%Ld %d@%Ld)\n", 151 req->wb_context->path.dentry->d_inode->i_sb->s_id, 152 (long long)NFS_FILEID(req->wb_context->path.dentry->d_inode), 153 req->wb_bytes, 154 (long long)req_offset(req)); 155 nfs_clear_request(req); 156 nfs_release_request(req); 157 } 158 159 /* 160 * Set up the NFS read request struct 161 */ 162 static int nfs_read_rpcsetup(struct nfs_page *req, struct nfs_read_data *data, 163 const struct rpc_call_ops *call_ops, 164 unsigned int count, unsigned int offset) 165 { 166 struct inode *inode = req->wb_context->path.dentry->d_inode; 167 int swap_flags = IS_SWAPFILE(inode) ? NFS_RPC_SWAPFLAGS : 0; 168 struct rpc_task *task; 169 struct rpc_message msg = { 170 .rpc_argp = &data->args, 171 .rpc_resp = &data->res, 172 .rpc_cred = req->wb_context->cred, 173 }; 174 struct rpc_task_setup task_setup_data = { 175 .task = &data->task, 176 .rpc_client = NFS_CLIENT(inode), 177 .rpc_message = &msg, 178 .callback_ops = call_ops, 179 .callback_data = data, 180 .workqueue = nfsiod_workqueue, 181 .flags = RPC_TASK_ASYNC | swap_flags, 182 }; 183 184 data->req = req; 185 data->inode = inode; 186 data->cred = msg.rpc_cred; 187 188 data->args.fh = NFS_FH(inode); 189 data->args.offset = req_offset(req) + offset; 190 data->args.pgbase = req->wb_pgbase + offset; 191 data->args.pages = data->pagevec; 192 data->args.count = count; 193 data->args.context = get_nfs_open_context(req->wb_context); 194 195 data->res.fattr = &data->fattr; 196 data->res.count = count; 197 data->res.eof = 0; 198 nfs_fattr_init(&data->fattr); 199 200 /* Set up the initial task struct. */ 201 NFS_PROTO(inode)->read_setup(data, &msg); 202 203 dprintk("NFS: %5u initiated read call (req %s/%Ld, %u bytes @ offset %Lu)\n", 204 data->task.tk_pid, 205 inode->i_sb->s_id, 206 (long long)NFS_FILEID(inode), 207 count, 208 (unsigned long long)data->args.offset); 209 210 task = rpc_run_task(&task_setup_data); 211 if (IS_ERR(task)) 212 return PTR_ERR(task); 213 rpc_put_task(task); 214 return 0; 215 } 216 217 static void 218 nfs_async_read_error(struct list_head *head) 219 { 220 struct nfs_page *req; 221 222 while (!list_empty(head)) { 223 req = nfs_list_entry(head->next); 224 nfs_list_remove_request(req); 225 SetPageError(req->wb_page); 226 nfs_readpage_release(req); 227 } 228 } 229 230 /* 231 * Generate multiple requests to fill a single page. 232 * 233 * We optimize to reduce the number of read operations on the wire. If we 234 * detect that we're reading a page, or an area of a page, that is past the 235 * end of file, we do not generate NFS read operations but just clear the 236 * parts of the page that would have come back zero from the server anyway. 237 * 238 * We rely on the cached value of i_size to make this determination; another 239 * client can fill pages on the server past our cached end-of-file, but we 240 * won't see the new data until our attribute cache is updated. This is more 241 * or less conventional NFS client behavior. 242 */ 243 static int nfs_pagein_multi(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int flags) 244 { 245 struct nfs_page *req = nfs_list_entry(head->next); 246 struct page *page = req->wb_page; 247 struct nfs_read_data *data; 248 size_t rsize = NFS_SERVER(inode)->rsize, nbytes; 249 unsigned int offset; 250 int requests = 0; 251 int ret = 0; 252 LIST_HEAD(list); 253 254 nfs_list_remove_request(req); 255 256 nbytes = count; 257 do { 258 size_t len = min(nbytes,rsize); 259 260 data = nfs_readdata_alloc(1); 261 if (!data) 262 goto out_bad; 263 list_add(&data->pages, &list); 264 requests++; 265 nbytes -= len; 266 } while(nbytes != 0); 267 atomic_set(&req->wb_complete, requests); 268 269 ClearPageError(page); 270 offset = 0; 271 nbytes = count; 272 do { 273 int ret2; 274 275 data = list_entry(list.next, struct nfs_read_data, pages); 276 list_del_init(&data->pages); 277 278 data->pagevec[0] = page; 279 280 if (nbytes < rsize) 281 rsize = nbytes; 282 ret2 = nfs_read_rpcsetup(req, data, &nfs_read_partial_ops, 283 rsize, offset); 284 if (ret == 0) 285 ret = ret2; 286 offset += rsize; 287 nbytes -= rsize; 288 } while (nbytes != 0); 289 290 return ret; 291 292 out_bad: 293 while (!list_empty(&list)) { 294 data = list_entry(list.next, struct nfs_read_data, pages); 295 list_del(&data->pages); 296 nfs_readdata_free(data); 297 } 298 SetPageError(page); 299 nfs_readpage_release(req); 300 return -ENOMEM; 301 } 302 303 static int nfs_pagein_one(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int flags) 304 { 305 struct nfs_page *req; 306 struct page **pages; 307 struct nfs_read_data *data; 308 int ret = -ENOMEM; 309 310 data = nfs_readdata_alloc(npages); 311 if (!data) 312 goto out_bad; 313 314 pages = data->pagevec; 315 while (!list_empty(head)) { 316 req = nfs_list_entry(head->next); 317 nfs_list_remove_request(req); 318 nfs_list_add_request(req, &data->pages); 319 ClearPageError(req->wb_page); 320 *pages++ = req->wb_page; 321 } 322 req = nfs_list_entry(data->pages.next); 323 324 return nfs_read_rpcsetup(req, data, &nfs_read_full_ops, count, 0); 325 out_bad: 326 nfs_async_read_error(head); 327 return ret; 328 } 329 330 /* 331 * This is the callback from RPC telling us whether a reply was 332 * received or some error occurred (timeout or socket shutdown). 333 */ 334 int nfs_readpage_result(struct rpc_task *task, struct nfs_read_data *data) 335 { 336 int status; 337 338 dprintk("NFS: %s: %5u, (status %d)\n", __func__, task->tk_pid, 339 task->tk_status); 340 341 status = NFS_PROTO(data->inode)->read_done(task, data); 342 if (status != 0) 343 return status; 344 345 nfs_add_stats(data->inode, NFSIOS_SERVERREADBYTES, data->res.count); 346 347 if (task->tk_status == -ESTALE) { 348 set_bit(NFS_INO_STALE, &NFS_I(data->inode)->flags); 349 nfs_mark_for_revalidate(data->inode); 350 } 351 return 0; 352 } 353 354 static void nfs_readpage_retry(struct rpc_task *task, struct nfs_read_data *data) 355 { 356 struct nfs_readargs *argp = &data->args; 357 struct nfs_readres *resp = &data->res; 358 359 if (resp->eof || resp->count == argp->count) 360 return; 361 362 /* This is a short read! */ 363 nfs_inc_stats(data->inode, NFSIOS_SHORTREAD); 364 /* Has the server at least made some progress? */ 365 if (resp->count == 0) 366 return; 367 368 /* Yes, so retry the read at the end of the data */ 369 argp->offset += resp->count; 370 argp->pgbase += resp->count; 371 argp->count -= resp->count; 372 rpc_restart_call(task); 373 } 374 375 /* 376 * Handle a read reply that fills part of a page. 377 */ 378 static void nfs_readpage_result_partial(struct rpc_task *task, void *calldata) 379 { 380 struct nfs_read_data *data = calldata; 381 382 if (nfs_readpage_result(task, data) != 0) 383 return; 384 if (task->tk_status < 0) 385 return; 386 387 nfs_readpage_truncate_uninitialised_page(data); 388 nfs_readpage_retry(task, data); 389 } 390 391 static void nfs_readpage_release_partial(void *calldata) 392 { 393 struct nfs_read_data *data = calldata; 394 struct nfs_page *req = data->req; 395 struct page *page = req->wb_page; 396 int status = data->task.tk_status; 397 398 if (status < 0) 399 SetPageError(page); 400 401 if (atomic_dec_and_test(&req->wb_complete)) { 402 if (!PageError(page)) 403 SetPageUptodate(page); 404 nfs_readpage_release(req); 405 } 406 nfs_readdata_release(calldata); 407 } 408 409 static const struct rpc_call_ops nfs_read_partial_ops = { 410 .rpc_call_done = nfs_readpage_result_partial, 411 .rpc_release = nfs_readpage_release_partial, 412 }; 413 414 static void nfs_readpage_set_pages_uptodate(struct nfs_read_data *data) 415 { 416 unsigned int count = data->res.count; 417 unsigned int base = data->args.pgbase; 418 struct page **pages; 419 420 if (data->res.eof) 421 count = data->args.count; 422 if (unlikely(count == 0)) 423 return; 424 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT]; 425 base &= ~PAGE_CACHE_MASK; 426 count += base; 427 for (;count >= PAGE_CACHE_SIZE; count -= PAGE_CACHE_SIZE, pages++) 428 SetPageUptodate(*pages); 429 if (count == 0) 430 return; 431 /* Was this a short read? */ 432 if (data->res.eof || data->res.count == data->args.count) 433 SetPageUptodate(*pages); 434 } 435 436 /* 437 * This is the callback from RPC telling us whether a reply was 438 * received or some error occurred (timeout or socket shutdown). 439 */ 440 static void nfs_readpage_result_full(struct rpc_task *task, void *calldata) 441 { 442 struct nfs_read_data *data = calldata; 443 444 if (nfs_readpage_result(task, data) != 0) 445 return; 446 if (task->tk_status < 0) 447 return; 448 /* 449 * Note: nfs_readpage_retry may change the values of 450 * data->args. In the multi-page case, we therefore need 451 * to ensure that we call nfs_readpage_set_pages_uptodate() 452 * first. 453 */ 454 nfs_readpage_truncate_uninitialised_page(data); 455 nfs_readpage_set_pages_uptodate(data); 456 nfs_readpage_retry(task, data); 457 } 458 459 static void nfs_readpage_release_full(void *calldata) 460 { 461 struct nfs_read_data *data = calldata; 462 463 while (!list_empty(&data->pages)) { 464 struct nfs_page *req = nfs_list_entry(data->pages.next); 465 466 nfs_list_remove_request(req); 467 nfs_readpage_release(req); 468 } 469 nfs_readdata_release(calldata); 470 } 471 472 static const struct rpc_call_ops nfs_read_full_ops = { 473 .rpc_call_done = nfs_readpage_result_full, 474 .rpc_release = nfs_readpage_release_full, 475 }; 476 477 /* 478 * Read a page over NFS. 479 * We read the page synchronously in the following case: 480 * - The error flag is set for this page. This happens only when a 481 * previous async read operation failed. 482 */ 483 int nfs_readpage(struct file *file, struct page *page) 484 { 485 struct nfs_open_context *ctx; 486 struct inode *inode = page->mapping->host; 487 int error; 488 489 dprintk("NFS: nfs_readpage (%p %ld@%lu)\n", 490 page, PAGE_CACHE_SIZE, page->index); 491 nfs_inc_stats(inode, NFSIOS_VFSREADPAGE); 492 nfs_add_stats(inode, NFSIOS_READPAGES, 1); 493 494 /* 495 * Try to flush any pending writes to the file.. 496 * 497 * NOTE! Because we own the page lock, there cannot 498 * be any new pending writes generated at this point 499 * for this page (other pages can be written to). 500 */ 501 error = nfs_wb_page(inode, page); 502 if (error) 503 goto out_unlock; 504 if (PageUptodate(page)) 505 goto out_unlock; 506 507 error = -ESTALE; 508 if (NFS_STALE(inode)) 509 goto out_unlock; 510 511 if (file == NULL) { 512 error = -EBADF; 513 ctx = nfs_find_open_context(inode, NULL, FMODE_READ); 514 if (ctx == NULL) 515 goto out_unlock; 516 } else 517 ctx = get_nfs_open_context(nfs_file_open_context(file)); 518 519 if (!IS_SYNC(inode)) { 520 error = nfs_readpage_from_fscache(ctx, inode, page); 521 if (error == 0) 522 goto out; 523 } 524 525 error = nfs_readpage_async(ctx, inode, page); 526 527 out: 528 put_nfs_open_context(ctx); 529 return error; 530 out_unlock: 531 unlock_page(page); 532 return error; 533 } 534 535 struct nfs_readdesc { 536 struct nfs_pageio_descriptor *pgio; 537 struct nfs_open_context *ctx; 538 }; 539 540 static int 541 readpage_async_filler(void *data, struct page *page) 542 { 543 struct nfs_readdesc *desc = (struct nfs_readdesc *)data; 544 struct inode *inode = page->mapping->host; 545 struct nfs_page *new; 546 unsigned int len; 547 int error; 548 549 len = nfs_page_length(page); 550 if (len == 0) 551 return nfs_return_empty_page(page); 552 553 new = nfs_create_request(desc->ctx, inode, page, 0, len); 554 if (IS_ERR(new)) 555 goto out_error; 556 557 if (len < PAGE_CACHE_SIZE) 558 zero_user_segment(page, len, PAGE_CACHE_SIZE); 559 if (!nfs_pageio_add_request(desc->pgio, new)) { 560 error = desc->pgio->pg_error; 561 goto out_unlock; 562 } 563 return 0; 564 out_error: 565 error = PTR_ERR(new); 566 SetPageError(page); 567 out_unlock: 568 unlock_page(page); 569 return error; 570 } 571 572 int nfs_readpages(struct file *filp, struct address_space *mapping, 573 struct list_head *pages, unsigned nr_pages) 574 { 575 struct nfs_pageio_descriptor pgio; 576 struct nfs_readdesc desc = { 577 .pgio = &pgio, 578 }; 579 struct inode *inode = mapping->host; 580 struct nfs_server *server = NFS_SERVER(inode); 581 size_t rsize = server->rsize; 582 unsigned long npages; 583 int ret = -ESTALE; 584 585 dprintk("NFS: nfs_readpages (%s/%Ld %d)\n", 586 inode->i_sb->s_id, 587 (long long)NFS_FILEID(inode), 588 nr_pages); 589 nfs_inc_stats(inode, NFSIOS_VFSREADPAGES); 590 591 if (NFS_STALE(inode)) 592 goto out; 593 594 if (filp == NULL) { 595 desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ); 596 if (desc.ctx == NULL) 597 return -EBADF; 598 } else 599 desc.ctx = get_nfs_open_context(nfs_file_open_context(filp)); 600 601 /* attempt to read as many of the pages as possible from the cache 602 * - this returns -ENOBUFS immediately if the cookie is negative 603 */ 604 ret = nfs_readpages_from_fscache(desc.ctx, inode, mapping, 605 pages, &nr_pages); 606 if (ret == 0) 607 goto read_complete; /* all pages were read */ 608 609 if (rsize < PAGE_CACHE_SIZE) 610 nfs_pageio_init(&pgio, inode, nfs_pagein_multi, rsize, 0); 611 else 612 nfs_pageio_init(&pgio, inode, nfs_pagein_one, rsize, 0); 613 614 ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc); 615 616 nfs_pageio_complete(&pgio); 617 npages = (pgio.pg_bytes_written + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 618 nfs_add_stats(inode, NFSIOS_READPAGES, npages); 619 read_complete: 620 put_nfs_open_context(desc.ctx); 621 out: 622 return ret; 623 } 624 625 int __init nfs_init_readpagecache(void) 626 { 627 nfs_rdata_cachep = kmem_cache_create("nfs_read_data", 628 sizeof(struct nfs_read_data), 629 0, SLAB_HWCACHE_ALIGN, 630 NULL); 631 if (nfs_rdata_cachep == NULL) 632 return -ENOMEM; 633 634 nfs_rdata_mempool = mempool_create_slab_pool(MIN_POOL_READ, 635 nfs_rdata_cachep); 636 if (nfs_rdata_mempool == NULL) 637 return -ENOMEM; 638 639 return 0; 640 } 641 642 void nfs_destroy_readpagecache(void) 643 { 644 mempool_destroy(nfs_rdata_mempool); 645 kmem_cache_destroy(nfs_rdata_cachep); 646 } 647