1 /* 2 * linux/fs/nfs/read.c 3 * 4 * Block I/O for NFS 5 * 6 * Partial copy of Linus' read cache modifications to fs/nfs/file.c 7 * modified for async RPC by okir@monad.swb.de 8 */ 9 10 #include <linux/time.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/fcntl.h> 14 #include <linux/stat.h> 15 #include <linux/mm.h> 16 #include <linux/slab.h> 17 #include <linux/pagemap.h> 18 #include <linux/sunrpc/clnt.h> 19 #include <linux/nfs_fs.h> 20 #include <linux/nfs_page.h> 21 #include <linux/smp_lock.h> 22 23 #include <asm/system.h> 24 25 #include "internal.h" 26 #include "iostat.h" 27 28 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 29 30 static int nfs_pagein_multi(struct inode *, struct list_head *, unsigned int, size_t, int); 31 static int nfs_pagein_one(struct inode *, struct list_head *, unsigned int, size_t, int); 32 static const struct rpc_call_ops nfs_read_partial_ops; 33 static const struct rpc_call_ops nfs_read_full_ops; 34 35 static struct kmem_cache *nfs_rdata_cachep; 36 static mempool_t *nfs_rdata_mempool; 37 38 #define MIN_POOL_READ (32) 39 40 struct nfs_read_data *nfs_readdata_alloc(unsigned int pagecount) 41 { 42 struct nfs_read_data *p = mempool_alloc(nfs_rdata_mempool, GFP_NOFS); 43 44 if (p) { 45 memset(p, 0, sizeof(*p)); 46 INIT_LIST_HEAD(&p->pages); 47 p->npages = pagecount; 48 if (pagecount <= ARRAY_SIZE(p->page_array)) 49 p->pagevec = p->page_array; 50 else { 51 p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_NOFS); 52 if (!p->pagevec) { 53 mempool_free(p, nfs_rdata_mempool); 54 p = NULL; 55 } 56 } 57 } 58 return p; 59 } 60 61 static void nfs_readdata_free(struct nfs_read_data *p) 62 { 63 if (p && (p->pagevec != &p->page_array[0])) 64 kfree(p->pagevec); 65 mempool_free(p, nfs_rdata_mempool); 66 } 67 68 void nfs_readdata_release(void *data) 69 { 70 struct nfs_read_data *rdata = data; 71 72 put_nfs_open_context(rdata->args.context); 73 nfs_readdata_free(rdata); 74 } 75 76 static 77 int nfs_return_empty_page(struct page *page) 78 { 79 zero_user(page, 0, PAGE_CACHE_SIZE); 80 SetPageUptodate(page); 81 unlock_page(page); 82 return 0; 83 } 84 85 static void nfs_readpage_truncate_uninitialised_page(struct nfs_read_data *data) 86 { 87 unsigned int remainder = data->args.count - data->res.count; 88 unsigned int base = data->args.pgbase + data->res.count; 89 unsigned int pglen; 90 struct page **pages; 91 92 if (data->res.eof == 0 || remainder == 0) 93 return; 94 /* 95 * Note: "remainder" can never be negative, since we check for 96 * this in the XDR code. 97 */ 98 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT]; 99 base &= ~PAGE_CACHE_MASK; 100 pglen = PAGE_CACHE_SIZE - base; 101 for (;;) { 102 if (remainder <= pglen) { 103 zero_user(*pages, base, remainder); 104 break; 105 } 106 zero_user(*pages, base, pglen); 107 pages++; 108 remainder -= pglen; 109 pglen = PAGE_CACHE_SIZE; 110 base = 0; 111 } 112 } 113 114 static int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode, 115 struct page *page) 116 { 117 LIST_HEAD(one_request); 118 struct nfs_page *new; 119 unsigned int len; 120 121 len = nfs_page_length(page); 122 if (len == 0) 123 return nfs_return_empty_page(page); 124 new = nfs_create_request(ctx, inode, page, 0, len); 125 if (IS_ERR(new)) { 126 unlock_page(page); 127 return PTR_ERR(new); 128 } 129 if (len < PAGE_CACHE_SIZE) 130 zero_user_segment(page, len, PAGE_CACHE_SIZE); 131 132 nfs_list_add_request(new, &one_request); 133 if (NFS_SERVER(inode)->rsize < PAGE_CACHE_SIZE) 134 nfs_pagein_multi(inode, &one_request, 1, len, 0); 135 else 136 nfs_pagein_one(inode, &one_request, 1, len, 0); 137 return 0; 138 } 139 140 static void nfs_readpage_release(struct nfs_page *req) 141 { 142 unlock_page(req->wb_page); 143 144 dprintk("NFS: read done (%s/%Ld %d@%Ld)\n", 145 req->wb_context->path.dentry->d_inode->i_sb->s_id, 146 (long long)NFS_FILEID(req->wb_context->path.dentry->d_inode), 147 req->wb_bytes, 148 (long long)req_offset(req)); 149 nfs_clear_request(req); 150 nfs_release_request(req); 151 } 152 153 /* 154 * Set up the NFS read request struct 155 */ 156 static int nfs_read_rpcsetup(struct nfs_page *req, struct nfs_read_data *data, 157 const struct rpc_call_ops *call_ops, 158 unsigned int count, unsigned int offset) 159 { 160 struct inode *inode = req->wb_context->path.dentry->d_inode; 161 int swap_flags = IS_SWAPFILE(inode) ? NFS_RPC_SWAPFLAGS : 0; 162 struct rpc_task *task; 163 struct rpc_message msg = { 164 .rpc_argp = &data->args, 165 .rpc_resp = &data->res, 166 .rpc_cred = req->wb_context->cred, 167 }; 168 struct rpc_task_setup task_setup_data = { 169 .task = &data->task, 170 .rpc_client = NFS_CLIENT(inode), 171 .rpc_message = &msg, 172 .callback_ops = call_ops, 173 .callback_data = data, 174 .workqueue = nfsiod_workqueue, 175 .flags = RPC_TASK_ASYNC | swap_flags, 176 }; 177 178 data->req = req; 179 data->inode = inode; 180 data->cred = msg.rpc_cred; 181 182 data->args.fh = NFS_FH(inode); 183 data->args.offset = req_offset(req) + offset; 184 data->args.pgbase = req->wb_pgbase + offset; 185 data->args.pages = data->pagevec; 186 data->args.count = count; 187 data->args.context = get_nfs_open_context(req->wb_context); 188 189 data->res.fattr = &data->fattr; 190 data->res.count = count; 191 data->res.eof = 0; 192 nfs_fattr_init(&data->fattr); 193 194 /* Set up the initial task struct. */ 195 NFS_PROTO(inode)->read_setup(data, &msg); 196 197 dprintk("NFS: %5u initiated read call (req %s/%Ld, %u bytes @ offset %Lu)\n", 198 data->task.tk_pid, 199 inode->i_sb->s_id, 200 (long long)NFS_FILEID(inode), 201 count, 202 (unsigned long long)data->args.offset); 203 204 task = rpc_run_task(&task_setup_data); 205 if (IS_ERR(task)) 206 return PTR_ERR(task); 207 rpc_put_task(task); 208 return 0; 209 } 210 211 static void 212 nfs_async_read_error(struct list_head *head) 213 { 214 struct nfs_page *req; 215 216 while (!list_empty(head)) { 217 req = nfs_list_entry(head->next); 218 nfs_list_remove_request(req); 219 SetPageError(req->wb_page); 220 nfs_readpage_release(req); 221 } 222 } 223 224 /* 225 * Generate multiple requests to fill a single page. 226 * 227 * We optimize to reduce the number of read operations on the wire. If we 228 * detect that we're reading a page, or an area of a page, that is past the 229 * end of file, we do not generate NFS read operations but just clear the 230 * parts of the page that would have come back zero from the server anyway. 231 * 232 * We rely on the cached value of i_size to make this determination; another 233 * client can fill pages on the server past our cached end-of-file, but we 234 * won't see the new data until our attribute cache is updated. This is more 235 * or less conventional NFS client behavior. 236 */ 237 static int nfs_pagein_multi(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int flags) 238 { 239 struct nfs_page *req = nfs_list_entry(head->next); 240 struct page *page = req->wb_page; 241 struct nfs_read_data *data; 242 size_t rsize = NFS_SERVER(inode)->rsize, nbytes; 243 unsigned int offset; 244 int requests = 0; 245 int ret = 0; 246 LIST_HEAD(list); 247 248 nfs_list_remove_request(req); 249 250 nbytes = count; 251 do { 252 size_t len = min(nbytes,rsize); 253 254 data = nfs_readdata_alloc(1); 255 if (!data) 256 goto out_bad; 257 list_add(&data->pages, &list); 258 requests++; 259 nbytes -= len; 260 } while(nbytes != 0); 261 atomic_set(&req->wb_complete, requests); 262 263 ClearPageError(page); 264 offset = 0; 265 nbytes = count; 266 do { 267 int ret2; 268 269 data = list_entry(list.next, struct nfs_read_data, pages); 270 list_del_init(&data->pages); 271 272 data->pagevec[0] = page; 273 274 if (nbytes < rsize) 275 rsize = nbytes; 276 ret2 = nfs_read_rpcsetup(req, data, &nfs_read_partial_ops, 277 rsize, offset); 278 if (ret == 0) 279 ret = ret2; 280 offset += rsize; 281 nbytes -= rsize; 282 } while (nbytes != 0); 283 284 return ret; 285 286 out_bad: 287 while (!list_empty(&list)) { 288 data = list_entry(list.next, struct nfs_read_data, pages); 289 list_del(&data->pages); 290 nfs_readdata_free(data); 291 } 292 SetPageError(page); 293 nfs_readpage_release(req); 294 return -ENOMEM; 295 } 296 297 static int nfs_pagein_one(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int flags) 298 { 299 struct nfs_page *req; 300 struct page **pages; 301 struct nfs_read_data *data; 302 int ret = -ENOMEM; 303 304 data = nfs_readdata_alloc(npages); 305 if (!data) 306 goto out_bad; 307 308 pages = data->pagevec; 309 while (!list_empty(head)) { 310 req = nfs_list_entry(head->next); 311 nfs_list_remove_request(req); 312 nfs_list_add_request(req, &data->pages); 313 ClearPageError(req->wb_page); 314 *pages++ = req->wb_page; 315 } 316 req = nfs_list_entry(data->pages.next); 317 318 return nfs_read_rpcsetup(req, data, &nfs_read_full_ops, count, 0); 319 out_bad: 320 nfs_async_read_error(head); 321 return ret; 322 } 323 324 /* 325 * This is the callback from RPC telling us whether a reply was 326 * received or some error occurred (timeout or socket shutdown). 327 */ 328 int nfs_readpage_result(struct rpc_task *task, struct nfs_read_data *data) 329 { 330 int status; 331 332 dprintk("NFS: %s: %5u, (status %d)\n", __FUNCTION__, task->tk_pid, 333 task->tk_status); 334 335 status = NFS_PROTO(data->inode)->read_done(task, data); 336 if (status != 0) 337 return status; 338 339 nfs_add_stats(data->inode, NFSIOS_SERVERREADBYTES, data->res.count); 340 341 if (task->tk_status == -ESTALE) { 342 set_bit(NFS_INO_STALE, &NFS_I(data->inode)->flags); 343 nfs_mark_for_revalidate(data->inode); 344 } 345 return 0; 346 } 347 348 static void nfs_readpage_retry(struct rpc_task *task, struct nfs_read_data *data) 349 { 350 struct nfs_readargs *argp = &data->args; 351 struct nfs_readres *resp = &data->res; 352 353 if (resp->eof || resp->count == argp->count) 354 return; 355 356 /* This is a short read! */ 357 nfs_inc_stats(data->inode, NFSIOS_SHORTREAD); 358 /* Has the server at least made some progress? */ 359 if (resp->count == 0) 360 return; 361 362 /* Yes, so retry the read at the end of the data */ 363 argp->offset += resp->count; 364 argp->pgbase += resp->count; 365 argp->count -= resp->count; 366 rpc_restart_call(task); 367 } 368 369 /* 370 * Handle a read reply that fills part of a page. 371 */ 372 static void nfs_readpage_result_partial(struct rpc_task *task, void *calldata) 373 { 374 struct nfs_read_data *data = calldata; 375 376 if (nfs_readpage_result(task, data) != 0) 377 return; 378 if (task->tk_status < 0) 379 return; 380 381 nfs_readpage_truncate_uninitialised_page(data); 382 nfs_readpage_retry(task, data); 383 } 384 385 static void nfs_readpage_release_partial(void *calldata) 386 { 387 struct nfs_read_data *data = calldata; 388 struct nfs_page *req = data->req; 389 struct page *page = req->wb_page; 390 int status = data->task.tk_status; 391 392 if (status < 0) 393 SetPageError(page); 394 395 if (atomic_dec_and_test(&req->wb_complete)) { 396 if (!PageError(page)) 397 SetPageUptodate(page); 398 nfs_readpage_release(req); 399 } 400 nfs_readdata_release(calldata); 401 } 402 403 static const struct rpc_call_ops nfs_read_partial_ops = { 404 .rpc_call_done = nfs_readpage_result_partial, 405 .rpc_release = nfs_readpage_release_partial, 406 }; 407 408 static void nfs_readpage_set_pages_uptodate(struct nfs_read_data *data) 409 { 410 unsigned int count = data->res.count; 411 unsigned int base = data->args.pgbase; 412 struct page **pages; 413 414 if (data->res.eof) 415 count = data->args.count; 416 if (unlikely(count == 0)) 417 return; 418 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT]; 419 base &= ~PAGE_CACHE_MASK; 420 count += base; 421 for (;count >= PAGE_CACHE_SIZE; count -= PAGE_CACHE_SIZE, pages++) 422 SetPageUptodate(*pages); 423 if (count == 0) 424 return; 425 /* Was this a short read? */ 426 if (data->res.eof || data->res.count == data->args.count) 427 SetPageUptodate(*pages); 428 } 429 430 /* 431 * This is the callback from RPC telling us whether a reply was 432 * received or some error occurred (timeout or socket shutdown). 433 */ 434 static void nfs_readpage_result_full(struct rpc_task *task, void *calldata) 435 { 436 struct nfs_read_data *data = calldata; 437 438 if (nfs_readpage_result(task, data) != 0) 439 return; 440 if (task->tk_status < 0) 441 return; 442 /* 443 * Note: nfs_readpage_retry may change the values of 444 * data->args. In the multi-page case, we therefore need 445 * to ensure that we call nfs_readpage_set_pages_uptodate() 446 * first. 447 */ 448 nfs_readpage_truncate_uninitialised_page(data); 449 nfs_readpage_set_pages_uptodate(data); 450 nfs_readpage_retry(task, data); 451 } 452 453 static void nfs_readpage_release_full(void *calldata) 454 { 455 struct nfs_read_data *data = calldata; 456 457 while (!list_empty(&data->pages)) { 458 struct nfs_page *req = nfs_list_entry(data->pages.next); 459 460 nfs_list_remove_request(req); 461 nfs_readpage_release(req); 462 } 463 nfs_readdata_release(calldata); 464 } 465 466 static const struct rpc_call_ops nfs_read_full_ops = { 467 .rpc_call_done = nfs_readpage_result_full, 468 .rpc_release = nfs_readpage_release_full, 469 }; 470 471 /* 472 * Read a page over NFS. 473 * We read the page synchronously in the following case: 474 * - The error flag is set for this page. This happens only when a 475 * previous async read operation failed. 476 */ 477 int nfs_readpage(struct file *file, struct page *page) 478 { 479 struct nfs_open_context *ctx; 480 struct inode *inode = page->mapping->host; 481 int error; 482 483 dprintk("NFS: nfs_readpage (%p %ld@%lu)\n", 484 page, PAGE_CACHE_SIZE, page->index); 485 nfs_inc_stats(inode, NFSIOS_VFSREADPAGE); 486 nfs_add_stats(inode, NFSIOS_READPAGES, 1); 487 488 /* 489 * Try to flush any pending writes to the file.. 490 * 491 * NOTE! Because we own the page lock, there cannot 492 * be any new pending writes generated at this point 493 * for this page (other pages can be written to). 494 */ 495 error = nfs_wb_page(inode, page); 496 if (error) 497 goto out_unlock; 498 if (PageUptodate(page)) 499 goto out_unlock; 500 501 error = -ESTALE; 502 if (NFS_STALE(inode)) 503 goto out_unlock; 504 505 if (file == NULL) { 506 error = -EBADF; 507 ctx = nfs_find_open_context(inode, NULL, FMODE_READ); 508 if (ctx == NULL) 509 goto out_unlock; 510 } else 511 ctx = get_nfs_open_context(nfs_file_open_context(file)); 512 513 error = nfs_readpage_async(ctx, inode, page); 514 515 put_nfs_open_context(ctx); 516 return error; 517 out_unlock: 518 unlock_page(page); 519 return error; 520 } 521 522 struct nfs_readdesc { 523 struct nfs_pageio_descriptor *pgio; 524 struct nfs_open_context *ctx; 525 }; 526 527 static int 528 readpage_async_filler(void *data, struct page *page) 529 { 530 struct nfs_readdesc *desc = (struct nfs_readdesc *)data; 531 struct inode *inode = page->mapping->host; 532 struct nfs_page *new; 533 unsigned int len; 534 int error; 535 536 error = nfs_wb_page(inode, page); 537 if (error) 538 goto out_unlock; 539 if (PageUptodate(page)) 540 goto out_unlock; 541 542 len = nfs_page_length(page); 543 if (len == 0) 544 return nfs_return_empty_page(page); 545 546 new = nfs_create_request(desc->ctx, inode, page, 0, len); 547 if (IS_ERR(new)) 548 goto out_error; 549 550 if (len < PAGE_CACHE_SIZE) 551 zero_user_segment(page, len, PAGE_CACHE_SIZE); 552 if (!nfs_pageio_add_request(desc->pgio, new)) { 553 error = desc->pgio->pg_error; 554 goto out_unlock; 555 } 556 return 0; 557 out_error: 558 error = PTR_ERR(new); 559 SetPageError(page); 560 out_unlock: 561 unlock_page(page); 562 return error; 563 } 564 565 int nfs_readpages(struct file *filp, struct address_space *mapping, 566 struct list_head *pages, unsigned nr_pages) 567 { 568 struct nfs_pageio_descriptor pgio; 569 struct nfs_readdesc desc = { 570 .pgio = &pgio, 571 }; 572 struct inode *inode = mapping->host; 573 struct nfs_server *server = NFS_SERVER(inode); 574 size_t rsize = server->rsize; 575 unsigned long npages; 576 int ret = -ESTALE; 577 578 dprintk("NFS: nfs_readpages (%s/%Ld %d)\n", 579 inode->i_sb->s_id, 580 (long long)NFS_FILEID(inode), 581 nr_pages); 582 nfs_inc_stats(inode, NFSIOS_VFSREADPAGES); 583 584 if (NFS_STALE(inode)) 585 goto out; 586 587 if (filp == NULL) { 588 desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ); 589 if (desc.ctx == NULL) 590 return -EBADF; 591 } else 592 desc.ctx = get_nfs_open_context(nfs_file_open_context(filp)); 593 if (rsize < PAGE_CACHE_SIZE) 594 nfs_pageio_init(&pgio, inode, nfs_pagein_multi, rsize, 0); 595 else 596 nfs_pageio_init(&pgio, inode, nfs_pagein_one, rsize, 0); 597 598 ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc); 599 600 nfs_pageio_complete(&pgio); 601 npages = (pgio.pg_bytes_written + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 602 nfs_add_stats(inode, NFSIOS_READPAGES, npages); 603 put_nfs_open_context(desc.ctx); 604 out: 605 return ret; 606 } 607 608 int __init nfs_init_readpagecache(void) 609 { 610 nfs_rdata_cachep = kmem_cache_create("nfs_read_data", 611 sizeof(struct nfs_read_data), 612 0, SLAB_HWCACHE_ALIGN, 613 NULL); 614 if (nfs_rdata_cachep == NULL) 615 return -ENOMEM; 616 617 nfs_rdata_mempool = mempool_create_slab_pool(MIN_POOL_READ, 618 nfs_rdata_cachep); 619 if (nfs_rdata_mempool == NULL) 620 return -ENOMEM; 621 622 return 0; 623 } 624 625 void nfs_destroy_readpagecache(void) 626 { 627 mempool_destroy(nfs_rdata_mempool); 628 kmem_cache_destroy(nfs_rdata_cachep); 629 } 630