1 /* 2 * linux/fs/nfs/pagelist.c 3 * 4 * A set of helper functions for managing NFS read and write requests. 5 * The main purpose of these routines is to provide support for the 6 * coalescing of several requests into a single RPC call. 7 * 8 * Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no> 9 * 10 */ 11 12 #include <linux/slab.h> 13 #include <linux/file.h> 14 #include <linux/sched.h> 15 #include <linux/sunrpc/clnt.h> 16 #include <linux/nfs.h> 17 #include <linux/nfs3.h> 18 #include <linux/nfs4.h> 19 #include <linux/nfs_page.h> 20 #include <linux/nfs_fs.h> 21 #include <linux/nfs_mount.h> 22 #include <linux/export.h> 23 24 #include "internal.h" 25 #include "pnfs.h" 26 27 #define NFSDBG_FACILITY NFSDBG_PAGECACHE 28 29 static struct kmem_cache *nfs_page_cachep; 30 static const struct rpc_call_ops nfs_pgio_common_ops; 31 32 static void nfs_free_request(struct nfs_page *); 33 34 static bool nfs_pgarray_set(struct nfs_page_array *p, unsigned int pagecount) 35 { 36 p->npages = pagecount; 37 if (pagecount <= ARRAY_SIZE(p->page_array)) 38 p->pagevec = p->page_array; 39 else { 40 p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL); 41 if (!p->pagevec) 42 p->npages = 0; 43 } 44 return p->pagevec != NULL; 45 } 46 47 void nfs_pgheader_init(struct nfs_pageio_descriptor *desc, 48 struct nfs_pgio_header *hdr, 49 void (*release)(struct nfs_pgio_header *hdr)) 50 { 51 hdr->req = nfs_list_entry(desc->pg_list.next); 52 hdr->inode = desc->pg_inode; 53 hdr->cred = hdr->req->wb_context->cred; 54 hdr->io_start = req_offset(hdr->req); 55 hdr->good_bytes = desc->pg_count; 56 hdr->dreq = desc->pg_dreq; 57 hdr->layout_private = desc->pg_layout_private; 58 hdr->release = release; 59 hdr->completion_ops = desc->pg_completion_ops; 60 if (hdr->completion_ops->init_hdr) 61 hdr->completion_ops->init_hdr(hdr); 62 } 63 EXPORT_SYMBOL_GPL(nfs_pgheader_init); 64 65 void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos) 66 { 67 spin_lock(&hdr->lock); 68 if (pos < hdr->io_start + hdr->good_bytes) { 69 set_bit(NFS_IOHDR_ERROR, &hdr->flags); 70 clear_bit(NFS_IOHDR_EOF, &hdr->flags); 71 hdr->good_bytes = pos - hdr->io_start; 72 hdr->error = error; 73 } 74 spin_unlock(&hdr->lock); 75 } 76 77 static inline struct nfs_page * 78 nfs_page_alloc(void) 79 { 80 struct nfs_page *p = kmem_cache_zalloc(nfs_page_cachep, GFP_NOIO); 81 if (p) 82 INIT_LIST_HEAD(&p->wb_list); 83 return p; 84 } 85 86 static inline void 87 nfs_page_free(struct nfs_page *p) 88 { 89 kmem_cache_free(nfs_page_cachep, p); 90 } 91 92 static void 93 nfs_iocounter_inc(struct nfs_io_counter *c) 94 { 95 atomic_inc(&c->io_count); 96 } 97 98 static void 99 nfs_iocounter_dec(struct nfs_io_counter *c) 100 { 101 if (atomic_dec_and_test(&c->io_count)) { 102 clear_bit(NFS_IO_INPROGRESS, &c->flags); 103 smp_mb__after_atomic(); 104 wake_up_bit(&c->flags, NFS_IO_INPROGRESS); 105 } 106 } 107 108 static int 109 __nfs_iocounter_wait(struct nfs_io_counter *c) 110 { 111 wait_queue_head_t *wq = bit_waitqueue(&c->flags, NFS_IO_INPROGRESS); 112 DEFINE_WAIT_BIT(q, &c->flags, NFS_IO_INPROGRESS); 113 int ret = 0; 114 115 do { 116 prepare_to_wait(wq, &q.wait, TASK_KILLABLE); 117 set_bit(NFS_IO_INPROGRESS, &c->flags); 118 if (atomic_read(&c->io_count) == 0) 119 break; 120 ret = nfs_wait_bit_killable(&c->flags); 121 } while (atomic_read(&c->io_count) != 0); 122 finish_wait(wq, &q.wait); 123 return ret; 124 } 125 126 /** 127 * nfs_iocounter_wait - wait for i/o to complete 128 * @c: nfs_io_counter to use 129 * 130 * returns -ERESTARTSYS if interrupted by a fatal signal. 131 * Otherwise returns 0 once the io_count hits 0. 132 */ 133 int 134 nfs_iocounter_wait(struct nfs_io_counter *c) 135 { 136 if (atomic_read(&c->io_count) == 0) 137 return 0; 138 return __nfs_iocounter_wait(c); 139 } 140 141 static int nfs_wait_bit_uninterruptible(void *word) 142 { 143 io_schedule(); 144 return 0; 145 } 146 147 /* 148 * nfs_page_group_lock - lock the head of the page group 149 * @req - request in group that is to be locked 150 * 151 * this lock must be held if modifying the page group list 152 */ 153 void 154 nfs_page_group_lock(struct nfs_page *req) 155 { 156 struct nfs_page *head = req->wb_head; 157 158 WARN_ON_ONCE(head != head->wb_head); 159 160 wait_on_bit_lock(&head->wb_flags, PG_HEADLOCK, 161 nfs_wait_bit_uninterruptible, 162 TASK_UNINTERRUPTIBLE); 163 } 164 165 /* 166 * nfs_page_group_unlock - unlock the head of the page group 167 * @req - request in group that is to be unlocked 168 */ 169 void 170 nfs_page_group_unlock(struct nfs_page *req) 171 { 172 struct nfs_page *head = req->wb_head; 173 174 WARN_ON_ONCE(head != head->wb_head); 175 176 smp_mb__before_atomic(); 177 clear_bit(PG_HEADLOCK, &head->wb_flags); 178 smp_mb__after_atomic(); 179 wake_up_bit(&head->wb_flags, PG_HEADLOCK); 180 } 181 182 /* 183 * nfs_page_group_sync_on_bit_locked 184 * 185 * must be called with page group lock held 186 */ 187 static bool 188 nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit) 189 { 190 struct nfs_page *head = req->wb_head; 191 struct nfs_page *tmp; 192 193 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags)); 194 WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags)); 195 196 tmp = req->wb_this_page; 197 while (tmp != req) { 198 if (!test_bit(bit, &tmp->wb_flags)) 199 return false; 200 tmp = tmp->wb_this_page; 201 } 202 203 /* true! reset all bits */ 204 tmp = req; 205 do { 206 clear_bit(bit, &tmp->wb_flags); 207 tmp = tmp->wb_this_page; 208 } while (tmp != req); 209 210 return true; 211 } 212 213 /* 214 * nfs_page_group_sync_on_bit - set bit on current request, but only 215 * return true if the bit is set for all requests in page group 216 * @req - request in page group 217 * @bit - PG_* bit that is used to sync page group 218 */ 219 bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit) 220 { 221 bool ret; 222 223 nfs_page_group_lock(req); 224 ret = nfs_page_group_sync_on_bit_locked(req, bit); 225 nfs_page_group_unlock(req); 226 227 return ret; 228 } 229 230 /* 231 * nfs_page_group_init - Initialize the page group linkage for @req 232 * @req - a new nfs request 233 * @prev - the previous request in page group, or NULL if @req is the first 234 * or only request in the group (the head). 235 */ 236 static inline void 237 nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev) 238 { 239 WARN_ON_ONCE(prev == req); 240 241 if (!prev) { 242 req->wb_head = req; 243 req->wb_this_page = req; 244 } else { 245 WARN_ON_ONCE(prev->wb_this_page != prev->wb_head); 246 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags)); 247 req->wb_head = prev->wb_head; 248 req->wb_this_page = prev->wb_this_page; 249 prev->wb_this_page = req; 250 251 /* grab extra ref if head request has extra ref from 252 * the write/commit path to handle handoff between write 253 * and commit lists */ 254 if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) 255 kref_get(&req->wb_kref); 256 } 257 } 258 259 /* 260 * nfs_page_group_destroy - sync the destruction of page groups 261 * @req - request that no longer needs the page group 262 * 263 * releases the page group reference from each member once all 264 * members have called this function. 265 */ 266 static void 267 nfs_page_group_destroy(struct kref *kref) 268 { 269 struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref); 270 struct nfs_page *tmp, *next; 271 272 if (!nfs_page_group_sync_on_bit(req, PG_TEARDOWN)) 273 return; 274 275 tmp = req; 276 do { 277 next = tmp->wb_this_page; 278 /* unlink and free */ 279 tmp->wb_this_page = tmp; 280 tmp->wb_head = tmp; 281 nfs_free_request(tmp); 282 tmp = next; 283 } while (tmp != req); 284 } 285 286 /** 287 * nfs_create_request - Create an NFS read/write request. 288 * @ctx: open context to use 289 * @page: page to write 290 * @last: last nfs request created for this page group or NULL if head 291 * @offset: starting offset within the page for the write 292 * @count: number of bytes to read/write 293 * 294 * The page must be locked by the caller. This makes sure we never 295 * create two different requests for the same page. 296 * User should ensure it is safe to sleep in this function. 297 */ 298 struct nfs_page * 299 nfs_create_request(struct nfs_open_context *ctx, struct page *page, 300 struct nfs_page *last, unsigned int offset, 301 unsigned int count) 302 { 303 struct nfs_page *req; 304 struct nfs_lock_context *l_ctx; 305 306 if (test_bit(NFS_CONTEXT_BAD, &ctx->flags)) 307 return ERR_PTR(-EBADF); 308 /* try to allocate the request struct */ 309 req = nfs_page_alloc(); 310 if (req == NULL) 311 return ERR_PTR(-ENOMEM); 312 313 /* get lock context early so we can deal with alloc failures */ 314 l_ctx = nfs_get_lock_context(ctx); 315 if (IS_ERR(l_ctx)) { 316 nfs_page_free(req); 317 return ERR_CAST(l_ctx); 318 } 319 req->wb_lock_context = l_ctx; 320 nfs_iocounter_inc(&l_ctx->io_count); 321 322 /* Initialize the request struct. Initially, we assume a 323 * long write-back delay. This will be adjusted in 324 * update_nfs_request below if the region is not locked. */ 325 req->wb_page = page; 326 req->wb_index = page_file_index(page); 327 page_cache_get(page); 328 req->wb_offset = offset; 329 req->wb_pgbase = offset; 330 req->wb_bytes = count; 331 req->wb_context = get_nfs_open_context(ctx); 332 kref_init(&req->wb_kref); 333 nfs_page_group_init(req, last); 334 return req; 335 } 336 337 /** 338 * nfs_unlock_request - Unlock request and wake up sleepers. 339 * @req: 340 */ 341 void nfs_unlock_request(struct nfs_page *req) 342 { 343 if (!NFS_WBACK_BUSY(req)) { 344 printk(KERN_ERR "NFS: Invalid unlock attempted\n"); 345 BUG(); 346 } 347 smp_mb__before_atomic(); 348 clear_bit(PG_BUSY, &req->wb_flags); 349 smp_mb__after_atomic(); 350 wake_up_bit(&req->wb_flags, PG_BUSY); 351 } 352 353 /** 354 * nfs_unlock_and_release_request - Unlock request and release the nfs_page 355 * @req: 356 */ 357 void nfs_unlock_and_release_request(struct nfs_page *req) 358 { 359 nfs_unlock_request(req); 360 nfs_release_request(req); 361 } 362 363 /* 364 * nfs_clear_request - Free up all resources allocated to the request 365 * @req: 366 * 367 * Release page and open context resources associated with a read/write 368 * request after it has completed. 369 */ 370 static void nfs_clear_request(struct nfs_page *req) 371 { 372 struct page *page = req->wb_page; 373 struct nfs_open_context *ctx = req->wb_context; 374 struct nfs_lock_context *l_ctx = req->wb_lock_context; 375 376 if (page != NULL) { 377 page_cache_release(page); 378 req->wb_page = NULL; 379 } 380 if (l_ctx != NULL) { 381 nfs_iocounter_dec(&l_ctx->io_count); 382 nfs_put_lock_context(l_ctx); 383 req->wb_lock_context = NULL; 384 } 385 if (ctx != NULL) { 386 put_nfs_open_context(ctx); 387 req->wb_context = NULL; 388 } 389 } 390 391 /** 392 * nfs_release_request - Release the count on an NFS read/write request 393 * @req: request to release 394 * 395 * Note: Should never be called with the spinlock held! 396 */ 397 static void nfs_free_request(struct nfs_page *req) 398 { 399 WARN_ON_ONCE(req->wb_this_page != req); 400 401 /* extra debug: make sure no sync bits are still set */ 402 WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags)); 403 WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags)); 404 WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags)); 405 WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags)); 406 WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags)); 407 408 /* Release struct file and open context */ 409 nfs_clear_request(req); 410 nfs_page_free(req); 411 } 412 413 void nfs_release_request(struct nfs_page *req) 414 { 415 kref_put(&req->wb_kref, nfs_page_group_destroy); 416 } 417 418 /** 419 * nfs_wait_on_request - Wait for a request to complete. 420 * @req: request to wait upon. 421 * 422 * Interruptible by fatal signals only. 423 * The user is responsible for holding a count on the request. 424 */ 425 int 426 nfs_wait_on_request(struct nfs_page *req) 427 { 428 return wait_on_bit(&req->wb_flags, PG_BUSY, 429 nfs_wait_bit_uninterruptible, 430 TASK_UNINTERRUPTIBLE); 431 } 432 433 /* 434 * nfs_generic_pg_test - determine if requests can be coalesced 435 * @desc: pointer to descriptor 436 * @prev: previous request in desc, or NULL 437 * @req: this request 438 * 439 * Returns zero if @req can be coalesced into @desc, otherwise it returns 440 * the size of the request. 441 */ 442 size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc, 443 struct nfs_page *prev, struct nfs_page *req) 444 { 445 if (desc->pg_count > desc->pg_bsize) { 446 /* should never happen */ 447 WARN_ON_ONCE(1); 448 return 0; 449 } 450 451 return min(desc->pg_bsize - desc->pg_count, (size_t)req->wb_bytes); 452 } 453 EXPORT_SYMBOL_GPL(nfs_generic_pg_test); 454 455 static inline struct nfs_rw_header *NFS_RW_HEADER(struct nfs_pgio_header *hdr) 456 { 457 return container_of(hdr, struct nfs_rw_header, header); 458 } 459 460 /** 461 * nfs_rw_header_alloc - Allocate a header for a read or write 462 * @ops: Read or write function vector 463 */ 464 struct nfs_rw_header *nfs_rw_header_alloc(const struct nfs_rw_ops *ops) 465 { 466 struct nfs_rw_header *header = ops->rw_alloc_header(); 467 468 if (header) { 469 struct nfs_pgio_header *hdr = &header->header; 470 471 INIT_LIST_HEAD(&hdr->pages); 472 spin_lock_init(&hdr->lock); 473 atomic_set(&hdr->refcnt, 0); 474 hdr->rw_ops = ops; 475 } 476 return header; 477 } 478 EXPORT_SYMBOL_GPL(nfs_rw_header_alloc); 479 480 /* 481 * nfs_rw_header_free - Free a read or write header 482 * @hdr: The header to free 483 */ 484 void nfs_rw_header_free(struct nfs_pgio_header *hdr) 485 { 486 hdr->rw_ops->rw_free_header(NFS_RW_HEADER(hdr)); 487 } 488 EXPORT_SYMBOL_GPL(nfs_rw_header_free); 489 490 /** 491 * nfs_pgio_data_alloc - Allocate pageio data 492 * @hdr: The header making a request 493 * @pagecount: Number of pages to create 494 */ 495 static struct nfs_pgio_data *nfs_pgio_data_alloc(struct nfs_pgio_header *hdr, 496 unsigned int pagecount) 497 { 498 struct nfs_pgio_data *data, *prealloc; 499 500 prealloc = &NFS_RW_HEADER(hdr)->rpc_data; 501 if (prealloc->header == NULL) 502 data = prealloc; 503 else 504 data = kzalloc(sizeof(*data), GFP_KERNEL); 505 if (!data) 506 goto out; 507 508 if (nfs_pgarray_set(&data->pages, pagecount)) { 509 data->header = hdr; 510 atomic_inc(&hdr->refcnt); 511 } else { 512 if (data != prealloc) 513 kfree(data); 514 data = NULL; 515 } 516 out: 517 return data; 518 } 519 520 /** 521 * nfs_pgio_data_release - Properly free pageio data 522 * @data: The data to release 523 */ 524 void nfs_pgio_data_release(struct nfs_pgio_data *data) 525 { 526 struct nfs_pgio_header *hdr = data->header; 527 struct nfs_rw_header *pageio_header = NFS_RW_HEADER(hdr); 528 529 put_nfs_open_context(data->args.context); 530 if (data->pages.pagevec != data->pages.page_array) 531 kfree(data->pages.pagevec); 532 if (data == &pageio_header->rpc_data) { 533 data->header = NULL; 534 data = NULL; 535 } 536 if (atomic_dec_and_test(&hdr->refcnt)) 537 hdr->completion_ops->completion(hdr); 538 /* Note: we only free the rpc_task after callbacks are done. 539 * See the comment in rpc_free_task() for why 540 */ 541 kfree(data); 542 } 543 EXPORT_SYMBOL_GPL(nfs_pgio_data_release); 544 545 /** 546 * nfs_pgio_rpcsetup - Set up arguments for a pageio call 547 * @data: The pageio data 548 * @count: Number of bytes to read 549 * @offset: Initial offset 550 * @how: How to commit data (writes only) 551 * @cinfo: Commit information for the call (writes only) 552 */ 553 static void nfs_pgio_rpcsetup(struct nfs_pgio_data *data, 554 unsigned int count, unsigned int offset, 555 int how, struct nfs_commit_info *cinfo) 556 { 557 struct nfs_page *req = data->header->req; 558 559 /* Set up the RPC argument and reply structs 560 * NB: take care not to mess about with data->commit et al. */ 561 562 data->args.fh = NFS_FH(data->header->inode); 563 data->args.offset = req_offset(req) + offset; 564 /* pnfs_set_layoutcommit needs this */ 565 data->mds_offset = data->args.offset; 566 data->args.pgbase = req->wb_pgbase + offset; 567 data->args.pages = data->pages.pagevec; 568 data->args.count = count; 569 data->args.context = get_nfs_open_context(req->wb_context); 570 data->args.lock_context = req->wb_lock_context; 571 data->args.stable = NFS_UNSTABLE; 572 switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) { 573 case 0: 574 break; 575 case FLUSH_COND_STABLE: 576 if (nfs_reqs_to_commit(cinfo)) 577 break; 578 default: 579 data->args.stable = NFS_FILE_SYNC; 580 } 581 582 data->res.fattr = &data->fattr; 583 data->res.count = count; 584 data->res.eof = 0; 585 data->res.verf = &data->verf; 586 nfs_fattr_init(&data->fattr); 587 } 588 589 /** 590 * nfs_pgio_prepare - Prepare pageio data to go over the wire 591 * @task: The current task 592 * @calldata: pageio data to prepare 593 */ 594 static void nfs_pgio_prepare(struct rpc_task *task, void *calldata) 595 { 596 struct nfs_pgio_data *data = calldata; 597 int err; 598 err = NFS_PROTO(data->header->inode)->pgio_rpc_prepare(task, data); 599 if (err) 600 rpc_exit(task, err); 601 } 602 603 int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_data *data, 604 const struct rpc_call_ops *call_ops, int how, int flags) 605 { 606 struct rpc_task *task; 607 struct rpc_message msg = { 608 .rpc_argp = &data->args, 609 .rpc_resp = &data->res, 610 .rpc_cred = data->header->cred, 611 }; 612 struct rpc_task_setup task_setup_data = { 613 .rpc_client = clnt, 614 .task = &data->task, 615 .rpc_message = &msg, 616 .callback_ops = call_ops, 617 .callback_data = data, 618 .workqueue = nfsiod_workqueue, 619 .flags = RPC_TASK_ASYNC | flags, 620 }; 621 int ret = 0; 622 623 data->header->rw_ops->rw_initiate(data, &msg, &task_setup_data, how); 624 625 dprintk("NFS: %5u initiated pgio call " 626 "(req %s/%llu, %u bytes @ offset %llu)\n", 627 data->task.tk_pid, 628 data->header->inode->i_sb->s_id, 629 (unsigned long long)NFS_FILEID(data->header->inode), 630 data->args.count, 631 (unsigned long long)data->args.offset); 632 633 task = rpc_run_task(&task_setup_data); 634 if (IS_ERR(task)) { 635 ret = PTR_ERR(task); 636 goto out; 637 } 638 if (how & FLUSH_SYNC) { 639 ret = rpc_wait_for_completion_task(task); 640 if (ret == 0) 641 ret = task->tk_status; 642 } 643 rpc_put_task(task); 644 out: 645 return ret; 646 } 647 EXPORT_SYMBOL_GPL(nfs_initiate_pgio); 648 649 /** 650 * nfs_pgio_error - Clean up from a pageio error 651 * @desc: IO descriptor 652 * @hdr: pageio header 653 */ 654 static int nfs_pgio_error(struct nfs_pageio_descriptor *desc, 655 struct nfs_pgio_header *hdr) 656 { 657 set_bit(NFS_IOHDR_REDO, &hdr->flags); 658 nfs_pgio_data_release(hdr->data); 659 hdr->data = NULL; 660 desc->pg_completion_ops->error_cleanup(&desc->pg_list); 661 return -ENOMEM; 662 } 663 664 /** 665 * nfs_pgio_release - Release pageio data 666 * @calldata: The pageio data to release 667 */ 668 static void nfs_pgio_release(void *calldata) 669 { 670 struct nfs_pgio_data *data = calldata; 671 if (data->header->rw_ops->rw_release) 672 data->header->rw_ops->rw_release(data); 673 nfs_pgio_data_release(data); 674 } 675 676 /** 677 * nfs_pageio_init - initialise a page io descriptor 678 * @desc: pointer to descriptor 679 * @inode: pointer to inode 680 * @doio: pointer to io function 681 * @bsize: io block size 682 * @io_flags: extra parameters for the io function 683 */ 684 void nfs_pageio_init(struct nfs_pageio_descriptor *desc, 685 struct inode *inode, 686 const struct nfs_pageio_ops *pg_ops, 687 const struct nfs_pgio_completion_ops *compl_ops, 688 const struct nfs_rw_ops *rw_ops, 689 size_t bsize, 690 int io_flags) 691 { 692 INIT_LIST_HEAD(&desc->pg_list); 693 desc->pg_bytes_written = 0; 694 desc->pg_count = 0; 695 desc->pg_bsize = bsize; 696 desc->pg_base = 0; 697 desc->pg_moreio = 0; 698 desc->pg_recoalesce = 0; 699 desc->pg_inode = inode; 700 desc->pg_ops = pg_ops; 701 desc->pg_completion_ops = compl_ops; 702 desc->pg_rw_ops = rw_ops; 703 desc->pg_ioflags = io_flags; 704 desc->pg_error = 0; 705 desc->pg_lseg = NULL; 706 desc->pg_dreq = NULL; 707 desc->pg_layout_private = NULL; 708 } 709 EXPORT_SYMBOL_GPL(nfs_pageio_init); 710 711 /** 712 * nfs_pgio_result - Basic pageio error handling 713 * @task: The task that ran 714 * @calldata: Pageio data to check 715 */ 716 static void nfs_pgio_result(struct rpc_task *task, void *calldata) 717 { 718 struct nfs_pgio_data *data = calldata; 719 struct inode *inode = data->header->inode; 720 721 dprintk("NFS: %s: %5u, (status %d)\n", __func__, 722 task->tk_pid, task->tk_status); 723 724 if (data->header->rw_ops->rw_done(task, data, inode) != 0) 725 return; 726 if (task->tk_status < 0) 727 nfs_set_pgio_error(data->header, task->tk_status, data->args.offset); 728 else 729 data->header->rw_ops->rw_result(task, data); 730 } 731 732 /* 733 * Create an RPC task for the given read or write request and kick it. 734 * The page must have been locked by the caller. 735 * 736 * It may happen that the page we're passed is not marked dirty. 737 * This is the case if nfs_updatepage detects a conflicting request 738 * that has been written but not committed. 739 */ 740 int nfs_generic_pgio(struct nfs_pageio_descriptor *desc, 741 struct nfs_pgio_header *hdr) 742 { 743 struct nfs_page *req; 744 struct page **pages; 745 struct nfs_pgio_data *data; 746 struct list_head *head = &desc->pg_list; 747 struct nfs_commit_info cinfo; 748 749 data = nfs_pgio_data_alloc(hdr, nfs_page_array_len(desc->pg_base, 750 desc->pg_count)); 751 if (!data) 752 return nfs_pgio_error(desc, hdr); 753 754 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); 755 pages = data->pages.pagevec; 756 while (!list_empty(head)) { 757 req = nfs_list_entry(head->next); 758 nfs_list_remove_request(req); 759 nfs_list_add_request(req, &hdr->pages); 760 *pages++ = req->wb_page; 761 } 762 763 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && 764 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo))) 765 desc->pg_ioflags &= ~FLUSH_COND_STABLE; 766 767 /* Set up the argument struct */ 768 nfs_pgio_rpcsetup(data, desc->pg_count, 0, desc->pg_ioflags, &cinfo); 769 hdr->data = data; 770 desc->pg_rpc_callops = &nfs_pgio_common_ops; 771 return 0; 772 } 773 EXPORT_SYMBOL_GPL(nfs_generic_pgio); 774 775 static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc) 776 { 777 struct nfs_rw_header *rw_hdr; 778 struct nfs_pgio_header *hdr; 779 int ret; 780 781 rw_hdr = nfs_rw_header_alloc(desc->pg_rw_ops); 782 if (!rw_hdr) { 783 desc->pg_completion_ops->error_cleanup(&desc->pg_list); 784 return -ENOMEM; 785 } 786 hdr = &rw_hdr->header; 787 nfs_pgheader_init(desc, hdr, nfs_rw_header_free); 788 atomic_inc(&hdr->refcnt); 789 ret = nfs_generic_pgio(desc, hdr); 790 if (ret == 0) 791 ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode), 792 hdr->data, desc->pg_rpc_callops, 793 desc->pg_ioflags, 0); 794 if (atomic_dec_and_test(&hdr->refcnt)) 795 hdr->completion_ops->completion(hdr); 796 return ret; 797 } 798 799 static bool nfs_match_open_context(const struct nfs_open_context *ctx1, 800 const struct nfs_open_context *ctx2) 801 { 802 return ctx1->cred == ctx2->cred && ctx1->state == ctx2->state; 803 } 804 805 static bool nfs_match_lock_context(const struct nfs_lock_context *l1, 806 const struct nfs_lock_context *l2) 807 { 808 return l1->lockowner.l_owner == l2->lockowner.l_owner 809 && l1->lockowner.l_pid == l2->lockowner.l_pid; 810 } 811 812 /** 813 * nfs_can_coalesce_requests - test two requests for compatibility 814 * @prev: pointer to nfs_page 815 * @req: pointer to nfs_page 816 * 817 * The nfs_page structures 'prev' and 'req' are compared to ensure that the 818 * page data area they describe is contiguous, and that their RPC 819 * credentials, NFSv4 open state, and lockowners are the same. 820 * 821 * Return 'true' if this is the case, else return 'false'. 822 */ 823 static bool nfs_can_coalesce_requests(struct nfs_page *prev, 824 struct nfs_page *req, 825 struct nfs_pageio_descriptor *pgio) 826 { 827 size_t size; 828 829 if (prev) { 830 if (!nfs_match_open_context(req->wb_context, prev->wb_context)) 831 return false; 832 if (req->wb_context->dentry->d_inode->i_flock != NULL && 833 !nfs_match_lock_context(req->wb_lock_context, 834 prev->wb_lock_context)) 835 return false; 836 if (req_offset(req) != req_offset(prev) + prev->wb_bytes) 837 return false; 838 } 839 size = pgio->pg_ops->pg_test(pgio, prev, req); 840 WARN_ON_ONCE(size > req->wb_bytes); 841 if (size && size < req->wb_bytes) 842 req->wb_bytes = size; 843 return size > 0; 844 } 845 846 /** 847 * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list. 848 * @desc: destination io descriptor 849 * @req: request 850 * 851 * Returns true if the request 'req' was successfully coalesced into the 852 * existing list of pages 'desc'. 853 */ 854 static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc, 855 struct nfs_page *req) 856 { 857 struct nfs_page *prev = NULL; 858 if (desc->pg_count != 0) { 859 prev = nfs_list_entry(desc->pg_list.prev); 860 } else { 861 if (desc->pg_ops->pg_init) 862 desc->pg_ops->pg_init(desc, req); 863 desc->pg_base = req->wb_pgbase; 864 } 865 if (!nfs_can_coalesce_requests(prev, req, desc)) 866 return 0; 867 nfs_list_remove_request(req); 868 nfs_list_add_request(req, &desc->pg_list); 869 desc->pg_count += req->wb_bytes; 870 return 1; 871 } 872 873 /* 874 * Helper for nfs_pageio_add_request and nfs_pageio_complete 875 */ 876 static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc) 877 { 878 if (!list_empty(&desc->pg_list)) { 879 int error = desc->pg_ops->pg_doio(desc); 880 if (error < 0) 881 desc->pg_error = error; 882 else 883 desc->pg_bytes_written += desc->pg_count; 884 } 885 if (list_empty(&desc->pg_list)) { 886 desc->pg_count = 0; 887 desc->pg_base = 0; 888 } 889 } 890 891 /** 892 * nfs_pageio_add_request - Attempt to coalesce a request into a page list. 893 * @desc: destination io descriptor 894 * @req: request 895 * 896 * This may split a request into subrequests which are all part of the 897 * same page group. 898 * 899 * Returns true if the request 'req' was successfully coalesced into the 900 * existing list of pages 'desc'. 901 */ 902 static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, 903 struct nfs_page *req) 904 { 905 struct nfs_page *subreq; 906 unsigned int bytes_left = 0; 907 unsigned int offset, pgbase; 908 909 nfs_page_group_lock(req); 910 911 subreq = req; 912 bytes_left = subreq->wb_bytes; 913 offset = subreq->wb_offset; 914 pgbase = subreq->wb_pgbase; 915 916 do { 917 if (!nfs_pageio_do_add_request(desc, subreq)) { 918 /* make sure pg_test call(s) did nothing */ 919 WARN_ON_ONCE(subreq->wb_bytes != bytes_left); 920 WARN_ON_ONCE(subreq->wb_offset != offset); 921 WARN_ON_ONCE(subreq->wb_pgbase != pgbase); 922 923 nfs_page_group_unlock(req); 924 desc->pg_moreio = 1; 925 nfs_pageio_doio(desc); 926 if (desc->pg_error < 0) 927 return 0; 928 desc->pg_moreio = 0; 929 if (desc->pg_recoalesce) 930 return 0; 931 /* retry add_request for this subreq */ 932 nfs_page_group_lock(req); 933 continue; 934 } 935 936 /* check for buggy pg_test call(s) */ 937 WARN_ON_ONCE(subreq->wb_bytes + subreq->wb_pgbase > PAGE_SIZE); 938 WARN_ON_ONCE(subreq->wb_bytes > bytes_left); 939 WARN_ON_ONCE(subreq->wb_bytes == 0); 940 941 bytes_left -= subreq->wb_bytes; 942 offset += subreq->wb_bytes; 943 pgbase += subreq->wb_bytes; 944 945 if (bytes_left) { 946 subreq = nfs_create_request(req->wb_context, 947 req->wb_page, 948 subreq, pgbase, bytes_left); 949 if (IS_ERR(subreq)) 950 goto err_ptr; 951 nfs_lock_request(subreq); 952 subreq->wb_offset = offset; 953 subreq->wb_index = req->wb_index; 954 } 955 } while (bytes_left > 0); 956 957 nfs_page_group_unlock(req); 958 return 1; 959 err_ptr: 960 desc->pg_error = PTR_ERR(subreq); 961 nfs_page_group_unlock(req); 962 return 0; 963 } 964 965 static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc) 966 { 967 LIST_HEAD(head); 968 969 do { 970 list_splice_init(&desc->pg_list, &head); 971 desc->pg_bytes_written -= desc->pg_count; 972 desc->pg_count = 0; 973 desc->pg_base = 0; 974 desc->pg_recoalesce = 0; 975 976 while (!list_empty(&head)) { 977 struct nfs_page *req; 978 979 req = list_first_entry(&head, struct nfs_page, wb_list); 980 nfs_list_remove_request(req); 981 if (__nfs_pageio_add_request(desc, req)) 982 continue; 983 if (desc->pg_error < 0) 984 return 0; 985 break; 986 } 987 } while (desc->pg_recoalesce); 988 return 1; 989 } 990 991 int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, 992 struct nfs_page *req) 993 { 994 int ret; 995 996 do { 997 ret = __nfs_pageio_add_request(desc, req); 998 if (ret) 999 break; 1000 if (desc->pg_error < 0) 1001 break; 1002 ret = nfs_do_recoalesce(desc); 1003 } while (ret); 1004 return ret; 1005 } 1006 EXPORT_SYMBOL_GPL(nfs_pageio_add_request); 1007 1008 /** 1009 * nfs_pageio_complete - Complete I/O on an nfs_pageio_descriptor 1010 * @desc: pointer to io descriptor 1011 */ 1012 void nfs_pageio_complete(struct nfs_pageio_descriptor *desc) 1013 { 1014 for (;;) { 1015 nfs_pageio_doio(desc); 1016 if (!desc->pg_recoalesce) 1017 break; 1018 if (!nfs_do_recoalesce(desc)) 1019 break; 1020 } 1021 } 1022 EXPORT_SYMBOL_GPL(nfs_pageio_complete); 1023 1024 /** 1025 * nfs_pageio_cond_complete - Conditional I/O completion 1026 * @desc: pointer to io descriptor 1027 * @index: page index 1028 * 1029 * It is important to ensure that processes don't try to take locks 1030 * on non-contiguous ranges of pages as that might deadlock. This 1031 * function should be called before attempting to wait on a locked 1032 * nfs_page. It will complete the I/O if the page index 'index' 1033 * is not contiguous with the existing list of pages in 'desc'. 1034 */ 1035 void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index) 1036 { 1037 if (!list_empty(&desc->pg_list)) { 1038 struct nfs_page *prev = nfs_list_entry(desc->pg_list.prev); 1039 if (index != prev->wb_index + 1) 1040 nfs_pageio_complete(desc); 1041 } 1042 } 1043 1044 int __init nfs_init_nfspagecache(void) 1045 { 1046 nfs_page_cachep = kmem_cache_create("nfs_page", 1047 sizeof(struct nfs_page), 1048 0, SLAB_HWCACHE_ALIGN, 1049 NULL); 1050 if (nfs_page_cachep == NULL) 1051 return -ENOMEM; 1052 1053 return 0; 1054 } 1055 1056 void nfs_destroy_nfspagecache(void) 1057 { 1058 kmem_cache_destroy(nfs_page_cachep); 1059 } 1060 1061 static const struct rpc_call_ops nfs_pgio_common_ops = { 1062 .rpc_call_prepare = nfs_pgio_prepare, 1063 .rpc_call_done = nfs_pgio_result, 1064 .rpc_release = nfs_pgio_release, 1065 }; 1066 1067 const struct nfs_pageio_ops nfs_pgio_rw_ops = { 1068 .pg_test = nfs_generic_pg_test, 1069 .pg_doio = nfs_generic_pg_pgios, 1070 }; 1071