1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include "internal.h" 34 #include "pnfs.h" 35 #include "iostat.h" 36 #include "nfs4trace.h" 37 #include "delegation.h" 38 #include "nfs42.h" 39 40 #define NFSDBG_FACILITY NFSDBG_PNFS 41 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 42 43 /* Locking: 44 * 45 * pnfs_spinlock: 46 * protects pnfs_modules_tbl. 47 */ 48 static DEFINE_SPINLOCK(pnfs_spinlock); 49 50 /* 51 * pnfs_modules_tbl holds all pnfs modules 52 */ 53 static LIST_HEAD(pnfs_modules_tbl); 54 55 static int 56 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid, 57 enum pnfs_iomode iomode, bool sync); 58 59 /* Return the registered pnfs layout driver module matching given id */ 60 static struct pnfs_layoutdriver_type * 61 find_pnfs_driver_locked(u32 id) 62 { 63 struct pnfs_layoutdriver_type *local; 64 65 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 66 if (local->id == id) 67 goto out; 68 local = NULL; 69 out: 70 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 71 return local; 72 } 73 74 static struct pnfs_layoutdriver_type * 75 find_pnfs_driver(u32 id) 76 { 77 struct pnfs_layoutdriver_type *local; 78 79 spin_lock(&pnfs_spinlock); 80 local = find_pnfs_driver_locked(id); 81 if (local != NULL && !try_module_get(local->owner)) { 82 dprintk("%s: Could not grab reference on module\n", __func__); 83 local = NULL; 84 } 85 spin_unlock(&pnfs_spinlock); 86 return local; 87 } 88 89 void 90 unset_pnfs_layoutdriver(struct nfs_server *nfss) 91 { 92 if (nfss->pnfs_curr_ld) { 93 if (nfss->pnfs_curr_ld->clear_layoutdriver) 94 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 95 /* Decrement the MDS count. Purge the deviceid cache if zero */ 96 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 97 nfs4_deviceid_purge_client(nfss->nfs_client); 98 module_put(nfss->pnfs_curr_ld->owner); 99 } 100 nfss->pnfs_curr_ld = NULL; 101 } 102 103 /* 104 * Try to set the server's pnfs module to the pnfs layout type specified by id. 105 * Currently only one pNFS layout driver per filesystem is supported. 106 * 107 * @id layout type. Zero (illegal layout type) indicates pNFS not in use. 108 */ 109 void 110 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 111 u32 id) 112 { 113 struct pnfs_layoutdriver_type *ld_type = NULL; 114 115 if (id == 0) 116 goto out_no_driver; 117 if (!(server->nfs_client->cl_exchange_flags & 118 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 119 printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n", 120 __func__, id, server->nfs_client->cl_exchange_flags); 121 goto out_no_driver; 122 } 123 ld_type = find_pnfs_driver(id); 124 if (!ld_type) { 125 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id); 126 ld_type = find_pnfs_driver(id); 127 if (!ld_type) { 128 dprintk("%s: No pNFS module found for %u.\n", 129 __func__, id); 130 goto out_no_driver; 131 } 132 } 133 server->pnfs_curr_ld = ld_type; 134 if (ld_type->set_layoutdriver 135 && ld_type->set_layoutdriver(server, mntfh)) { 136 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 137 "driver %u.\n", __func__, id); 138 module_put(ld_type->owner); 139 goto out_no_driver; 140 } 141 /* Bump the MDS count */ 142 atomic_inc(&server->nfs_client->cl_mds_count); 143 144 dprintk("%s: pNFS module for %u set\n", __func__, id); 145 return; 146 147 out_no_driver: 148 dprintk("%s: Using NFSv4 I/O\n", __func__); 149 server->pnfs_curr_ld = NULL; 150 } 151 152 int 153 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 154 { 155 int status = -EINVAL; 156 struct pnfs_layoutdriver_type *tmp; 157 158 if (ld_type->id == 0) { 159 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 160 return status; 161 } 162 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 163 printk(KERN_ERR "NFS: %s Layout driver must provide " 164 "alloc_lseg and free_lseg.\n", __func__); 165 return status; 166 } 167 168 spin_lock(&pnfs_spinlock); 169 tmp = find_pnfs_driver_locked(ld_type->id); 170 if (!tmp) { 171 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 172 status = 0; 173 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 174 ld_type->name); 175 } else { 176 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 177 __func__, ld_type->id); 178 } 179 spin_unlock(&pnfs_spinlock); 180 181 return status; 182 } 183 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 184 185 void 186 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 187 { 188 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 189 spin_lock(&pnfs_spinlock); 190 list_del(&ld_type->pnfs_tblid); 191 spin_unlock(&pnfs_spinlock); 192 } 193 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 194 195 /* 196 * pNFS client layout cache 197 */ 198 199 /* Need to hold i_lock if caller does not already hold reference */ 200 void 201 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 202 { 203 atomic_inc(&lo->plh_refcount); 204 } 205 206 static struct pnfs_layout_hdr * 207 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 208 { 209 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 210 return ld->alloc_layout_hdr(ino, gfp_flags); 211 } 212 213 static void 214 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 215 { 216 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 217 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 218 219 if (!list_empty(&lo->plh_layouts)) { 220 struct nfs_client *clp = server->nfs_client; 221 222 spin_lock(&clp->cl_lock); 223 list_del_init(&lo->plh_layouts); 224 spin_unlock(&clp->cl_lock); 225 } 226 put_rpccred(lo->plh_lc_cred); 227 return ld->free_layout_hdr(lo); 228 } 229 230 static void 231 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 232 { 233 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 234 dprintk("%s: freeing layout cache %p\n", __func__, lo); 235 nfsi->layout = NULL; 236 /* Reset MDS Threshold I/O counters */ 237 nfsi->write_io = 0; 238 nfsi->read_io = 0; 239 } 240 241 void 242 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 243 { 244 struct inode *inode = lo->plh_inode; 245 246 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 247 if (!list_empty(&lo->plh_segs)) 248 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 249 pnfs_detach_layout_hdr(lo); 250 spin_unlock(&inode->i_lock); 251 pnfs_free_layout_hdr(lo); 252 } 253 } 254 255 static int 256 pnfs_iomode_to_fail_bit(u32 iomode) 257 { 258 return iomode == IOMODE_RW ? 259 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 260 } 261 262 static void 263 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 264 { 265 lo->plh_retry_timestamp = jiffies; 266 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 267 atomic_inc(&lo->plh_refcount); 268 } 269 270 static void 271 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 272 { 273 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 274 atomic_dec(&lo->plh_refcount); 275 } 276 277 static void 278 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 279 { 280 struct inode *inode = lo->plh_inode; 281 struct pnfs_layout_range range = { 282 .iomode = iomode, 283 .offset = 0, 284 .length = NFS4_MAX_UINT64, 285 }; 286 LIST_HEAD(head); 287 288 spin_lock(&inode->i_lock); 289 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 290 pnfs_mark_matching_lsegs_invalid(lo, &head, &range); 291 spin_unlock(&inode->i_lock); 292 pnfs_free_lseg_list(&head); 293 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 294 iomode == IOMODE_RW ? "RW" : "READ"); 295 } 296 297 static bool 298 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 299 { 300 unsigned long start, end; 301 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 302 303 if (test_bit(fail_bit, &lo->plh_flags) == 0) 304 return false; 305 end = jiffies; 306 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 307 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 308 /* It is time to retry the failed layoutgets */ 309 pnfs_layout_clear_fail_bit(lo, fail_bit); 310 return false; 311 } 312 return true; 313 } 314 315 static void 316 init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg) 317 { 318 INIT_LIST_HEAD(&lseg->pls_list); 319 INIT_LIST_HEAD(&lseg->pls_lc_list); 320 atomic_set(&lseg->pls_refcount, 1); 321 smp_mb(); 322 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 323 lseg->pls_layout = lo; 324 } 325 326 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 327 { 328 struct inode *ino = lseg->pls_layout->plh_inode; 329 330 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 331 } 332 333 static void 334 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 335 struct pnfs_layout_segment *lseg) 336 { 337 struct inode *inode = lo->plh_inode; 338 339 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 340 list_del_init(&lseg->pls_list); 341 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 342 atomic_dec(&lo->plh_refcount); 343 if (list_empty(&lo->plh_segs)) 344 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 345 rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq); 346 } 347 348 /* Return true if layoutreturn is needed */ 349 static bool 350 pnfs_layout_need_return(struct pnfs_layout_hdr *lo, 351 struct pnfs_layout_segment *lseg) 352 { 353 struct pnfs_layout_segment *s; 354 355 if (!test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 356 return false; 357 358 list_for_each_entry(s, &lo->plh_segs, pls_list) 359 if (s != lseg && test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags)) 360 return false; 361 362 return true; 363 } 364 365 static bool 366 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo) 367 { 368 if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 369 return false; 370 lo->plh_return_iomode = 0; 371 pnfs_get_layout_hdr(lo); 372 clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, &lo->plh_flags); 373 return true; 374 } 375 376 static void pnfs_layoutreturn_before_put_lseg(struct pnfs_layout_segment *lseg, 377 struct pnfs_layout_hdr *lo, struct inode *inode) 378 { 379 lo = lseg->pls_layout; 380 inode = lo->plh_inode; 381 382 spin_lock(&inode->i_lock); 383 if (pnfs_layout_need_return(lo, lseg)) { 384 nfs4_stateid stateid; 385 enum pnfs_iomode iomode; 386 bool send; 387 388 stateid = lo->plh_stateid; 389 iomode = lo->plh_return_iomode; 390 send = pnfs_prepare_layoutreturn(lo); 391 spin_unlock(&inode->i_lock); 392 if (send) { 393 /* Send an async layoutreturn so we dont deadlock */ 394 pnfs_send_layoutreturn(lo, stateid, iomode, false); 395 } 396 } else 397 spin_unlock(&inode->i_lock); 398 } 399 400 void 401 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 402 { 403 struct pnfs_layout_hdr *lo; 404 struct inode *inode; 405 406 if (!lseg) 407 return; 408 409 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 410 atomic_read(&lseg->pls_refcount), 411 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 412 413 /* Handle the case where refcount != 1 */ 414 if (atomic_add_unless(&lseg->pls_refcount, -1, 1)) 415 return; 416 417 lo = lseg->pls_layout; 418 inode = lo->plh_inode; 419 /* Do we need a layoutreturn? */ 420 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 421 pnfs_layoutreturn_before_put_lseg(lseg, lo, inode); 422 423 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 424 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 425 spin_unlock(&inode->i_lock); 426 return; 427 } 428 pnfs_get_layout_hdr(lo); 429 pnfs_layout_remove_lseg(lo, lseg); 430 spin_unlock(&inode->i_lock); 431 pnfs_free_lseg(lseg); 432 pnfs_put_layout_hdr(lo); 433 } 434 } 435 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 436 437 static void pnfs_free_lseg_async_work(struct work_struct *work) 438 { 439 struct pnfs_layout_segment *lseg; 440 struct pnfs_layout_hdr *lo; 441 442 lseg = container_of(work, struct pnfs_layout_segment, pls_work); 443 lo = lseg->pls_layout; 444 445 pnfs_free_lseg(lseg); 446 pnfs_put_layout_hdr(lo); 447 } 448 449 static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg) 450 { 451 INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work); 452 schedule_work(&lseg->pls_work); 453 } 454 455 void 456 pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg) 457 { 458 if (!lseg) 459 return; 460 461 assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock); 462 463 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 464 atomic_read(&lseg->pls_refcount), 465 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 466 if (atomic_dec_and_test(&lseg->pls_refcount)) { 467 struct pnfs_layout_hdr *lo = lseg->pls_layout; 468 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 469 return; 470 pnfs_get_layout_hdr(lo); 471 pnfs_layout_remove_lseg(lo, lseg); 472 pnfs_free_lseg_async(lseg); 473 } 474 } 475 EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked); 476 477 static u64 478 end_offset(u64 start, u64 len) 479 { 480 u64 end; 481 482 end = start + len; 483 return end >= start ? end : NFS4_MAX_UINT64; 484 } 485 486 /* 487 * is l2 fully contained in l1? 488 * start1 end1 489 * [----------------------------------) 490 * start2 end2 491 * [----------------) 492 */ 493 static bool 494 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 495 const struct pnfs_layout_range *l2) 496 { 497 u64 start1 = l1->offset; 498 u64 end1 = end_offset(start1, l1->length); 499 u64 start2 = l2->offset; 500 u64 end2 = end_offset(start2, l2->length); 501 502 return (start1 <= start2) && (end1 >= end2); 503 } 504 505 /* 506 * is l1 and l2 intersecting? 507 * start1 end1 508 * [----------------------------------) 509 * start2 end2 510 * [----------------) 511 */ 512 static bool 513 pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1, 514 const struct pnfs_layout_range *l2) 515 { 516 u64 start1 = l1->offset; 517 u64 end1 = end_offset(start1, l1->length); 518 u64 start2 = l2->offset; 519 u64 end2 = end_offset(start2, l2->length); 520 521 return (end1 == NFS4_MAX_UINT64 || end1 > start2) && 522 (end2 == NFS4_MAX_UINT64 || end2 > start1); 523 } 524 525 static bool 526 should_free_lseg(const struct pnfs_layout_range *lseg_range, 527 const struct pnfs_layout_range *recall_range) 528 { 529 return (recall_range->iomode == IOMODE_ANY || 530 lseg_range->iomode == recall_range->iomode) && 531 pnfs_lseg_range_intersecting(lseg_range, recall_range); 532 } 533 534 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 535 struct list_head *tmp_list) 536 { 537 if (!atomic_dec_and_test(&lseg->pls_refcount)) 538 return false; 539 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 540 list_add(&lseg->pls_list, tmp_list); 541 return true; 542 } 543 544 /* Returns 1 if lseg is removed from list, 0 otherwise */ 545 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 546 struct list_head *tmp_list) 547 { 548 int rv = 0; 549 550 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 551 /* Remove the reference keeping the lseg in the 552 * list. It will now be removed when all 553 * outstanding io is finished. 554 */ 555 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 556 atomic_read(&lseg->pls_refcount)); 557 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 558 rv = 1; 559 } 560 return rv; 561 } 562 563 /* Returns count of number of matching invalid lsegs remaining in list 564 * after call. 565 */ 566 int 567 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 568 struct list_head *tmp_list, 569 struct pnfs_layout_range *recall_range) 570 { 571 struct pnfs_layout_segment *lseg, *next; 572 int invalid = 0, removed = 0; 573 574 dprintk("%s:Begin lo %p\n", __func__, lo); 575 576 if (list_empty(&lo->plh_segs)) 577 return 0; 578 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 579 if (!recall_range || 580 should_free_lseg(&lseg->pls_range, recall_range)) { 581 dprintk("%s: freeing lseg %p iomode %d " 582 "offset %llu length %llu\n", __func__, 583 lseg, lseg->pls_range.iomode, lseg->pls_range.offset, 584 lseg->pls_range.length); 585 invalid++; 586 removed += mark_lseg_invalid(lseg, tmp_list); 587 } 588 dprintk("%s:Return %i\n", __func__, invalid - removed); 589 return invalid - removed; 590 } 591 592 /* note free_me must contain lsegs from a single layout_hdr */ 593 void 594 pnfs_free_lseg_list(struct list_head *free_me) 595 { 596 struct pnfs_layout_segment *lseg, *tmp; 597 598 if (list_empty(free_me)) 599 return; 600 601 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 602 list_del(&lseg->pls_list); 603 pnfs_free_lseg(lseg); 604 } 605 } 606 607 void 608 pnfs_destroy_layout(struct nfs_inode *nfsi) 609 { 610 struct pnfs_layout_hdr *lo; 611 LIST_HEAD(tmp_list); 612 613 spin_lock(&nfsi->vfs_inode.i_lock); 614 lo = nfsi->layout; 615 if (lo) { 616 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */ 617 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL); 618 pnfs_get_layout_hdr(lo); 619 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 620 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 621 pnfs_clear_retry_layoutget(lo); 622 spin_unlock(&nfsi->vfs_inode.i_lock); 623 pnfs_free_lseg_list(&tmp_list); 624 pnfs_put_layout_hdr(lo); 625 } else 626 spin_unlock(&nfsi->vfs_inode.i_lock); 627 } 628 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 629 630 static bool 631 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 632 struct list_head *layout_list) 633 { 634 struct pnfs_layout_hdr *lo; 635 bool ret = false; 636 637 spin_lock(&inode->i_lock); 638 lo = NFS_I(inode)->layout; 639 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 640 pnfs_get_layout_hdr(lo); 641 list_add(&lo->plh_bulk_destroy, layout_list); 642 ret = true; 643 } 644 spin_unlock(&inode->i_lock); 645 return ret; 646 } 647 648 /* Caller must hold rcu_read_lock and clp->cl_lock */ 649 static int 650 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 651 struct nfs_server *server, 652 struct list_head *layout_list) 653 { 654 struct pnfs_layout_hdr *lo, *next; 655 struct inode *inode; 656 657 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 658 inode = igrab(lo->plh_inode); 659 if (inode == NULL) 660 continue; 661 list_del_init(&lo->plh_layouts); 662 if (pnfs_layout_add_bulk_destroy_list(inode, layout_list)) 663 continue; 664 rcu_read_unlock(); 665 spin_unlock(&clp->cl_lock); 666 iput(inode); 667 spin_lock(&clp->cl_lock); 668 rcu_read_lock(); 669 return -EAGAIN; 670 } 671 return 0; 672 } 673 674 static int 675 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 676 bool is_bulk_recall) 677 { 678 struct pnfs_layout_hdr *lo; 679 struct inode *inode; 680 struct pnfs_layout_range range = { 681 .iomode = IOMODE_ANY, 682 .offset = 0, 683 .length = NFS4_MAX_UINT64, 684 }; 685 LIST_HEAD(lseg_list); 686 int ret = 0; 687 688 while (!list_empty(layout_list)) { 689 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 690 plh_bulk_destroy); 691 dprintk("%s freeing layout for inode %lu\n", __func__, 692 lo->plh_inode->i_ino); 693 inode = lo->plh_inode; 694 695 pnfs_layoutcommit_inode(inode, false); 696 697 spin_lock(&inode->i_lock); 698 list_del_init(&lo->plh_bulk_destroy); 699 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */ 700 if (is_bulk_recall) 701 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 702 if (pnfs_mark_matching_lsegs_invalid(lo, &lseg_list, &range)) 703 ret = -EAGAIN; 704 spin_unlock(&inode->i_lock); 705 pnfs_free_lseg_list(&lseg_list); 706 pnfs_put_layout_hdr(lo); 707 iput(inode); 708 } 709 return ret; 710 } 711 712 int 713 pnfs_destroy_layouts_byfsid(struct nfs_client *clp, 714 struct nfs_fsid *fsid, 715 bool is_recall) 716 { 717 struct nfs_server *server; 718 LIST_HEAD(layout_list); 719 720 spin_lock(&clp->cl_lock); 721 rcu_read_lock(); 722 restart: 723 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 724 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 725 continue; 726 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 727 server, 728 &layout_list) != 0) 729 goto restart; 730 } 731 rcu_read_unlock(); 732 spin_unlock(&clp->cl_lock); 733 734 if (list_empty(&layout_list)) 735 return 0; 736 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall); 737 } 738 739 int 740 pnfs_destroy_layouts_byclid(struct nfs_client *clp, 741 bool is_recall) 742 { 743 struct nfs_server *server; 744 LIST_HEAD(layout_list); 745 746 spin_lock(&clp->cl_lock); 747 rcu_read_lock(); 748 restart: 749 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 750 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 751 server, 752 &layout_list) != 0) 753 goto restart; 754 } 755 rcu_read_unlock(); 756 spin_unlock(&clp->cl_lock); 757 758 if (list_empty(&layout_list)) 759 return 0; 760 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall); 761 } 762 763 /* 764 * Called by the state manger to remove all layouts established under an 765 * expired lease. 766 */ 767 void 768 pnfs_destroy_all_layouts(struct nfs_client *clp) 769 { 770 nfs4_deviceid_mark_client_invalid(clp); 771 nfs4_deviceid_purge_client(clp); 772 773 pnfs_destroy_layouts_byclid(clp, false); 774 } 775 776 /* 777 * Compare 2 layout stateid sequence ids, to see which is newer, 778 * taking into account wraparound issues. 779 */ 780 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 781 { 782 return (s32)(s1 - s2) > 0; 783 } 784 785 /* update lo->plh_stateid with new if is more recent */ 786 void 787 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 788 bool update_barrier) 789 { 790 u32 oldseq, newseq, new_barrier; 791 int empty = list_empty(&lo->plh_segs); 792 793 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 794 newseq = be32_to_cpu(new->seqid); 795 if (empty || pnfs_seqid_is_newer(newseq, oldseq)) { 796 nfs4_stateid_copy(&lo->plh_stateid, new); 797 if (update_barrier) { 798 new_barrier = be32_to_cpu(new->seqid); 799 } else { 800 /* Because of wraparound, we want to keep the barrier 801 * "close" to the current seqids. 802 */ 803 new_barrier = newseq - atomic_read(&lo->plh_outstanding); 804 } 805 if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier)) 806 lo->plh_barrier = new_barrier; 807 } 808 } 809 810 static bool 811 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 812 const nfs4_stateid *stateid) 813 { 814 u32 seqid = be32_to_cpu(stateid->seqid); 815 816 return !pnfs_seqid_is_newer(seqid, lo->plh_barrier); 817 } 818 819 /* lget is set to 1 if called from inside send_layoutget call chain */ 820 static bool 821 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 822 { 823 return lo->plh_block_lgets || 824 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 825 } 826 827 int 828 pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo, 829 struct pnfs_layout_range *range, 830 struct nfs4_state *open_state) 831 { 832 int status = 0; 833 834 dprintk("--> %s\n", __func__); 835 spin_lock(&lo->plh_inode->i_lock); 836 if (pnfs_layoutgets_blocked(lo)) { 837 status = -EAGAIN; 838 } else if (!nfs4_valid_open_stateid(open_state)) { 839 status = -EBADF; 840 } else if (list_empty(&lo->plh_segs) || 841 test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 842 int seq; 843 844 do { 845 seq = read_seqbegin(&open_state->seqlock); 846 nfs4_stateid_copy(dst, &open_state->stateid); 847 } while (read_seqretry(&open_state->seqlock, seq)); 848 } else 849 nfs4_stateid_copy(dst, &lo->plh_stateid); 850 spin_unlock(&lo->plh_inode->i_lock); 851 dprintk("<-- %s\n", __func__); 852 return status; 853 } 854 855 /* 856 * Get layout from server. 857 * for now, assume that whole file layouts are requested. 858 * arg->offset: 0 859 * arg->length: all ones 860 */ 861 static struct pnfs_layout_segment * 862 send_layoutget(struct pnfs_layout_hdr *lo, 863 struct nfs_open_context *ctx, 864 struct pnfs_layout_range *range, 865 gfp_t gfp_flags) 866 { 867 struct inode *ino = lo->plh_inode; 868 struct nfs_server *server = NFS_SERVER(ino); 869 struct nfs4_layoutget *lgp; 870 struct pnfs_layout_segment *lseg; 871 872 dprintk("--> %s\n", __func__); 873 874 lgp = kzalloc(sizeof(*lgp), gfp_flags); 875 if (lgp == NULL) 876 return NULL; 877 878 lgp->args.minlength = PAGE_CACHE_SIZE; 879 if (lgp->args.minlength > range->length) 880 lgp->args.minlength = range->length; 881 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 882 lgp->args.range = *range; 883 lgp->args.type = server->pnfs_curr_ld->id; 884 lgp->args.inode = ino; 885 lgp->args.ctx = get_nfs_open_context(ctx); 886 lgp->gfp_flags = gfp_flags; 887 lgp->cred = lo->plh_lc_cred; 888 889 /* Synchronously retrieve layout information from server and 890 * store in lseg. 891 */ 892 lseg = nfs4_proc_layoutget(lgp, gfp_flags); 893 if (IS_ERR(lseg)) { 894 switch (PTR_ERR(lseg)) { 895 case -ENOMEM: 896 case -ERESTARTSYS: 897 break; 898 default: 899 /* remember that LAYOUTGET failed and suspend trying */ 900 pnfs_layout_io_set_failed(lo, range->iomode); 901 } 902 return NULL; 903 } else 904 pnfs_layout_clear_fail_bit(lo, 905 pnfs_iomode_to_fail_bit(range->iomode)); 906 907 return lseg; 908 } 909 910 static void pnfs_clear_layoutcommit(struct inode *inode, 911 struct list_head *head) 912 { 913 struct nfs_inode *nfsi = NFS_I(inode); 914 struct pnfs_layout_segment *lseg, *tmp; 915 916 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 917 return; 918 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 919 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 920 continue; 921 pnfs_lseg_dec_and_remove_zero(lseg, head); 922 } 923 } 924 925 void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 926 { 927 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 928 smp_mb__after_atomic(); 929 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 930 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 931 } 932 933 static int 934 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid, 935 enum pnfs_iomode iomode, bool sync) 936 { 937 struct inode *ino = lo->plh_inode; 938 struct nfs4_layoutreturn *lrp; 939 int status = 0; 940 941 lrp = kzalloc(sizeof(*lrp), GFP_NOFS); 942 if (unlikely(lrp == NULL)) { 943 status = -ENOMEM; 944 spin_lock(&ino->i_lock); 945 pnfs_clear_layoutreturn_waitbit(lo); 946 spin_unlock(&ino->i_lock); 947 pnfs_put_layout_hdr(lo); 948 goto out; 949 } 950 951 lrp->args.stateid = stateid; 952 lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id; 953 lrp->args.inode = ino; 954 lrp->args.range.iomode = iomode; 955 lrp->args.range.offset = 0; 956 lrp->args.range.length = NFS4_MAX_UINT64; 957 lrp->args.layout = lo; 958 lrp->clp = NFS_SERVER(ino)->nfs_client; 959 lrp->cred = lo->plh_lc_cred; 960 961 status = nfs4_proc_layoutreturn(lrp, sync); 962 out: 963 dprintk("<-- %s status: %d\n", __func__, status); 964 return status; 965 } 966 967 /* 968 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 969 * when the layout segment list is empty. 970 * 971 * Note that a pnfs_layout_hdr can exist with an empty layout segment 972 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 973 * deviceid is marked invalid. 974 */ 975 int 976 _pnfs_return_layout(struct inode *ino) 977 { 978 struct pnfs_layout_hdr *lo = NULL; 979 struct nfs_inode *nfsi = NFS_I(ino); 980 LIST_HEAD(tmp_list); 981 nfs4_stateid stateid; 982 int status = 0, empty; 983 bool send; 984 985 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino); 986 987 spin_lock(&ino->i_lock); 988 lo = nfsi->layout; 989 if (!lo) { 990 spin_unlock(&ino->i_lock); 991 dprintk("NFS: %s no layout to return\n", __func__); 992 goto out; 993 } 994 stateid = nfsi->layout->plh_stateid; 995 /* Reference matched in nfs4_layoutreturn_release */ 996 pnfs_get_layout_hdr(lo); 997 empty = list_empty(&lo->plh_segs); 998 pnfs_clear_layoutcommit(ino, &tmp_list); 999 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL); 1000 1001 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) { 1002 struct pnfs_layout_range range = { 1003 .iomode = IOMODE_ANY, 1004 .offset = 0, 1005 .length = NFS4_MAX_UINT64, 1006 }; 1007 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1008 } 1009 1010 /* Don't send a LAYOUTRETURN if list was initially empty */ 1011 if (empty) { 1012 spin_unlock(&ino->i_lock); 1013 dprintk("NFS: %s no layout segments to return\n", __func__); 1014 goto out_put_layout_hdr; 1015 } 1016 1017 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1018 send = pnfs_prepare_layoutreturn(lo); 1019 spin_unlock(&ino->i_lock); 1020 pnfs_free_lseg_list(&tmp_list); 1021 if (send) 1022 status = pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, true); 1023 out_put_layout_hdr: 1024 pnfs_put_layout_hdr(lo); 1025 out: 1026 dprintk("<-- %s status: %d\n", __func__, status); 1027 return status; 1028 } 1029 EXPORT_SYMBOL_GPL(_pnfs_return_layout); 1030 1031 int 1032 pnfs_commit_and_return_layout(struct inode *inode) 1033 { 1034 struct pnfs_layout_hdr *lo; 1035 int ret; 1036 1037 spin_lock(&inode->i_lock); 1038 lo = NFS_I(inode)->layout; 1039 if (lo == NULL) { 1040 spin_unlock(&inode->i_lock); 1041 return 0; 1042 } 1043 pnfs_get_layout_hdr(lo); 1044 /* Block new layoutgets and read/write to ds */ 1045 lo->plh_block_lgets++; 1046 spin_unlock(&inode->i_lock); 1047 filemap_fdatawait(inode->i_mapping); 1048 ret = pnfs_layoutcommit_inode(inode, true); 1049 if (ret == 0) 1050 ret = _pnfs_return_layout(inode); 1051 spin_lock(&inode->i_lock); 1052 lo->plh_block_lgets--; 1053 spin_unlock(&inode->i_lock); 1054 pnfs_put_layout_hdr(lo); 1055 return ret; 1056 } 1057 1058 bool pnfs_roc(struct inode *ino) 1059 { 1060 struct nfs_inode *nfsi = NFS_I(ino); 1061 struct nfs_open_context *ctx; 1062 struct nfs4_state *state; 1063 struct pnfs_layout_hdr *lo; 1064 struct pnfs_layout_segment *lseg, *tmp; 1065 nfs4_stateid stateid; 1066 LIST_HEAD(tmp_list); 1067 bool found = false, layoutreturn = false, roc = false; 1068 1069 spin_lock(&ino->i_lock); 1070 lo = nfsi->layout; 1071 if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) 1072 goto out_noroc; 1073 1074 /* no roc if we hold a delegation */ 1075 if (nfs4_check_delegation(ino, FMODE_READ)) 1076 goto out_noroc; 1077 1078 list_for_each_entry(ctx, &nfsi->open_files, list) { 1079 state = ctx->state; 1080 /* Don't return layout if there is open file state */ 1081 if (state != NULL && state->state != 0) 1082 goto out_noroc; 1083 } 1084 1085 stateid = lo->plh_stateid; 1086 /* always send layoutreturn if being marked so */ 1087 if (test_and_clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, 1088 &lo->plh_flags)) 1089 layoutreturn = pnfs_prepare_layoutreturn(lo); 1090 1091 pnfs_clear_retry_layoutget(lo); 1092 list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list) 1093 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1094 if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) { 1095 mark_lseg_invalid(lseg, &tmp_list); 1096 found = true; 1097 } 1098 /* pnfs_prepare_layoutreturn() grabs lo ref and it will be put 1099 * in pnfs_roc_release(). We don't really send a layoutreturn but 1100 * still want others to view us like we are sending one! 1101 * 1102 * If pnfs_prepare_layoutreturn() fails, it means someone else is doing 1103 * LAYOUTRETURN, so we proceed like there are no layouts to return. 1104 * 1105 * ROC in three conditions: 1106 * 1. there are ROC lsegs 1107 * 2. we don't send layoutreturn 1108 * 3. no others are sending layoutreturn 1109 */ 1110 if (found && !layoutreturn && pnfs_prepare_layoutreturn(lo)) 1111 roc = true; 1112 1113 out_noroc: 1114 spin_unlock(&ino->i_lock); 1115 pnfs_free_lseg_list(&tmp_list); 1116 pnfs_layoutcommit_inode(ino, true); 1117 if (layoutreturn) 1118 pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, true); 1119 return roc; 1120 } 1121 1122 void pnfs_roc_release(struct inode *ino) 1123 { 1124 struct pnfs_layout_hdr *lo; 1125 1126 spin_lock(&ino->i_lock); 1127 lo = NFS_I(ino)->layout; 1128 pnfs_clear_layoutreturn_waitbit(lo); 1129 if (atomic_dec_and_test(&lo->plh_refcount)) { 1130 pnfs_detach_layout_hdr(lo); 1131 spin_unlock(&ino->i_lock); 1132 pnfs_free_layout_hdr(lo); 1133 } else 1134 spin_unlock(&ino->i_lock); 1135 } 1136 1137 void pnfs_roc_set_barrier(struct inode *ino, u32 barrier) 1138 { 1139 struct pnfs_layout_hdr *lo; 1140 1141 spin_lock(&ino->i_lock); 1142 lo = NFS_I(ino)->layout; 1143 if (pnfs_seqid_is_newer(barrier, lo->plh_barrier)) 1144 lo->plh_barrier = barrier; 1145 spin_unlock(&ino->i_lock); 1146 trace_nfs4_layoutreturn_on_close(ino, 0); 1147 } 1148 1149 void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier) 1150 { 1151 struct nfs_inode *nfsi = NFS_I(ino); 1152 struct pnfs_layout_hdr *lo; 1153 u32 current_seqid; 1154 1155 spin_lock(&ino->i_lock); 1156 lo = nfsi->layout; 1157 current_seqid = be32_to_cpu(lo->plh_stateid.seqid); 1158 1159 /* Since close does not return a layout stateid for use as 1160 * a barrier, we choose the worst-case barrier. 1161 */ 1162 *barrier = current_seqid + atomic_read(&lo->plh_outstanding); 1163 spin_unlock(&ino->i_lock); 1164 } 1165 1166 /* 1167 * Compare two layout segments for sorting into layout cache. 1168 * We want to preferentially return RW over RO layouts, so ensure those 1169 * are seen first. 1170 */ 1171 static s64 1172 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1173 const struct pnfs_layout_range *l2) 1174 { 1175 s64 d; 1176 1177 /* high offset > low offset */ 1178 d = l1->offset - l2->offset; 1179 if (d) 1180 return d; 1181 1182 /* short length > long length */ 1183 d = l2->length - l1->length; 1184 if (d) 1185 return d; 1186 1187 /* read > read/write */ 1188 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1189 } 1190 1191 static void 1192 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1193 struct pnfs_layout_segment *lseg) 1194 { 1195 struct pnfs_layout_segment *lp; 1196 1197 dprintk("%s:Begin\n", __func__); 1198 1199 list_for_each_entry(lp, &lo->plh_segs, pls_list) { 1200 if (pnfs_lseg_range_cmp(&lseg->pls_range, &lp->pls_range) > 0) 1201 continue; 1202 list_add_tail(&lseg->pls_list, &lp->pls_list); 1203 dprintk("%s: inserted lseg %p " 1204 "iomode %d offset %llu length %llu before " 1205 "lp %p iomode %d offset %llu length %llu\n", 1206 __func__, lseg, lseg->pls_range.iomode, 1207 lseg->pls_range.offset, lseg->pls_range.length, 1208 lp, lp->pls_range.iomode, lp->pls_range.offset, 1209 lp->pls_range.length); 1210 goto out; 1211 } 1212 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1213 dprintk("%s: inserted lseg %p " 1214 "iomode %d offset %llu length %llu at tail\n", 1215 __func__, lseg, lseg->pls_range.iomode, 1216 lseg->pls_range.offset, lseg->pls_range.length); 1217 out: 1218 pnfs_get_layout_hdr(lo); 1219 1220 dprintk("%s:Return\n", __func__); 1221 } 1222 1223 static struct pnfs_layout_hdr * 1224 alloc_init_layout_hdr(struct inode *ino, 1225 struct nfs_open_context *ctx, 1226 gfp_t gfp_flags) 1227 { 1228 struct pnfs_layout_hdr *lo; 1229 1230 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1231 if (!lo) 1232 return NULL; 1233 atomic_set(&lo->plh_refcount, 1); 1234 INIT_LIST_HEAD(&lo->plh_layouts); 1235 INIT_LIST_HEAD(&lo->plh_segs); 1236 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1237 lo->plh_inode = ino; 1238 lo->plh_lc_cred = get_rpccred(ctx->cred); 1239 return lo; 1240 } 1241 1242 static struct pnfs_layout_hdr * 1243 pnfs_find_alloc_layout(struct inode *ino, 1244 struct nfs_open_context *ctx, 1245 gfp_t gfp_flags) 1246 { 1247 struct nfs_inode *nfsi = NFS_I(ino); 1248 struct pnfs_layout_hdr *new = NULL; 1249 1250 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1251 1252 if (nfsi->layout != NULL) 1253 goto out_existing; 1254 spin_unlock(&ino->i_lock); 1255 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1256 spin_lock(&ino->i_lock); 1257 1258 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1259 nfsi->layout = new; 1260 return new; 1261 } else if (new != NULL) 1262 pnfs_free_layout_hdr(new); 1263 out_existing: 1264 pnfs_get_layout_hdr(nfsi->layout); 1265 return nfsi->layout; 1266 } 1267 1268 /* 1269 * iomode matching rules: 1270 * iomode lseg match 1271 * ----- ----- ----- 1272 * ANY READ true 1273 * ANY RW true 1274 * RW READ false 1275 * RW RW true 1276 * READ READ true 1277 * READ RW true 1278 */ 1279 static bool 1280 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1281 const struct pnfs_layout_range *range) 1282 { 1283 struct pnfs_layout_range range1; 1284 1285 if ((range->iomode == IOMODE_RW && 1286 ls_range->iomode != IOMODE_RW) || 1287 !pnfs_lseg_range_intersecting(ls_range, range)) 1288 return 0; 1289 1290 /* range1 covers only the first byte in the range */ 1291 range1 = *range; 1292 range1.length = 1; 1293 return pnfs_lseg_range_contained(ls_range, &range1); 1294 } 1295 1296 /* 1297 * lookup range in layout 1298 */ 1299 static struct pnfs_layout_segment * 1300 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1301 struct pnfs_layout_range *range) 1302 { 1303 struct pnfs_layout_segment *lseg, *ret = NULL; 1304 1305 dprintk("%s:Begin\n", __func__); 1306 1307 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1308 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1309 !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 1310 pnfs_lseg_range_match(&lseg->pls_range, range)) { 1311 ret = pnfs_get_lseg(lseg); 1312 break; 1313 } 1314 if (lseg->pls_range.offset > range->offset) 1315 break; 1316 } 1317 1318 dprintk("%s:Return lseg %p ref %d\n", 1319 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0); 1320 return ret; 1321 } 1322 1323 /* 1324 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1325 * to the MDS or over pNFS 1326 * 1327 * The nfs_inode read_io and write_io fields are cumulative counters reset 1328 * when there are no layout segments. Note that in pnfs_update_layout iomode 1329 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 1330 * WRITE request. 1331 * 1332 * A return of true means use MDS I/O. 1333 * 1334 * From rfc 5661: 1335 * If a file's size is smaller than the file size threshold, data accesses 1336 * SHOULD be sent to the metadata server. If an I/O request has a length that 1337 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 1338 * server. If both file size and I/O size are provided, the client SHOULD 1339 * reach or exceed both thresholds before sending its read or write 1340 * requests to the data server. 1341 */ 1342 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 1343 struct inode *ino, int iomode) 1344 { 1345 struct nfs4_threshold *t = ctx->mdsthreshold; 1346 struct nfs_inode *nfsi = NFS_I(ino); 1347 loff_t fsize = i_size_read(ino); 1348 bool size = false, size_set = false, io = false, io_set = false, ret = false; 1349 1350 if (t == NULL) 1351 return ret; 1352 1353 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 1354 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 1355 1356 switch (iomode) { 1357 case IOMODE_READ: 1358 if (t->bm & THRESHOLD_RD) { 1359 dprintk("%s fsize %llu\n", __func__, fsize); 1360 size_set = true; 1361 if (fsize < t->rd_sz) 1362 size = true; 1363 } 1364 if (t->bm & THRESHOLD_RD_IO) { 1365 dprintk("%s nfsi->read_io %llu\n", __func__, 1366 nfsi->read_io); 1367 io_set = true; 1368 if (nfsi->read_io < t->rd_io_sz) 1369 io = true; 1370 } 1371 break; 1372 case IOMODE_RW: 1373 if (t->bm & THRESHOLD_WR) { 1374 dprintk("%s fsize %llu\n", __func__, fsize); 1375 size_set = true; 1376 if (fsize < t->wr_sz) 1377 size = true; 1378 } 1379 if (t->bm & THRESHOLD_WR_IO) { 1380 dprintk("%s nfsi->write_io %llu\n", __func__, 1381 nfsi->write_io); 1382 io_set = true; 1383 if (nfsi->write_io < t->wr_io_sz) 1384 io = true; 1385 } 1386 break; 1387 } 1388 if (size_set && io_set) { 1389 if (size && io) 1390 ret = true; 1391 } else if (size || io) 1392 ret = true; 1393 1394 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 1395 return ret; 1396 } 1397 1398 /* stop waiting if someone clears NFS_LAYOUT_RETRY_LAYOUTGET bit. */ 1399 static int pnfs_layoutget_retry_bit_wait(struct wait_bit_key *key) 1400 { 1401 if (!test_bit(NFS_LAYOUT_RETRY_LAYOUTGET, key->flags)) 1402 return 1; 1403 return nfs_wait_bit_killable(key); 1404 } 1405 1406 static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 1407 { 1408 if (!pnfs_should_retry_layoutget(lo)) 1409 return false; 1410 /* 1411 * send layoutcommit as it can hold up layoutreturn due to lseg 1412 * reference 1413 */ 1414 pnfs_layoutcommit_inode(lo->plh_inode, false); 1415 return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 1416 pnfs_layoutget_retry_bit_wait, 1417 TASK_UNINTERRUPTIBLE); 1418 } 1419 1420 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 1421 { 1422 unsigned long *bitlock = &lo->plh_flags; 1423 1424 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 1425 smp_mb__after_atomic(); 1426 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 1427 } 1428 1429 /* 1430 * Layout segment is retreived from the server if not cached. 1431 * The appropriate layout segment is referenced and returned to the caller. 1432 */ 1433 struct pnfs_layout_segment * 1434 pnfs_update_layout(struct inode *ino, 1435 struct nfs_open_context *ctx, 1436 loff_t pos, 1437 u64 count, 1438 enum pnfs_iomode iomode, 1439 gfp_t gfp_flags) 1440 { 1441 struct pnfs_layout_range arg = { 1442 .iomode = iomode, 1443 .offset = pos, 1444 .length = count, 1445 }; 1446 unsigned pg_offset; 1447 struct nfs_server *server = NFS_SERVER(ino); 1448 struct nfs_client *clp = server->nfs_client; 1449 struct pnfs_layout_hdr *lo; 1450 struct pnfs_layout_segment *lseg = NULL; 1451 bool first; 1452 1453 if (!pnfs_enabled_sb(NFS_SERVER(ino))) 1454 goto out; 1455 1456 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) 1457 goto out; 1458 1459 lookup_again: 1460 first = false; 1461 spin_lock(&ino->i_lock); 1462 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 1463 if (lo == NULL) { 1464 spin_unlock(&ino->i_lock); 1465 goto out; 1466 } 1467 1468 /* Do we even need to bother with this? */ 1469 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1470 dprintk("%s matches recall, use MDS\n", __func__); 1471 goto out_unlock; 1472 } 1473 1474 /* if LAYOUTGET already failed once we don't try again */ 1475 if (pnfs_layout_io_test_failed(lo, iomode) && 1476 !pnfs_should_retry_layoutget(lo)) 1477 goto out_unlock; 1478 1479 first = list_empty(&lo->plh_segs); 1480 if (first) { 1481 /* The first layoutget for the file. Need to serialize per 1482 * RFC 5661 Errata 3208. 1483 */ 1484 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 1485 &lo->plh_flags)) { 1486 spin_unlock(&ino->i_lock); 1487 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET, 1488 TASK_UNINTERRUPTIBLE); 1489 pnfs_put_layout_hdr(lo); 1490 goto lookup_again; 1491 } 1492 } else { 1493 /* Check to see if the layout for the given range 1494 * already exists 1495 */ 1496 lseg = pnfs_find_lseg(lo, &arg); 1497 if (lseg) 1498 goto out_unlock; 1499 } 1500 1501 /* 1502 * Because we free lsegs before sending LAYOUTRETURN, we need to wait 1503 * for LAYOUTRETURN even if first is true. 1504 */ 1505 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1506 spin_unlock(&ino->i_lock); 1507 dprintk("%s wait for layoutreturn\n", __func__); 1508 if (pnfs_prepare_to_retry_layoutget(lo)) { 1509 if (first) 1510 pnfs_clear_first_layoutget(lo); 1511 pnfs_put_layout_hdr(lo); 1512 dprintk("%s retrying\n", __func__); 1513 goto lookup_again; 1514 } 1515 goto out_put_layout_hdr; 1516 } 1517 1518 if (pnfs_layoutgets_blocked(lo)) 1519 goto out_unlock; 1520 atomic_inc(&lo->plh_outstanding); 1521 spin_unlock(&ino->i_lock); 1522 1523 if (list_empty(&lo->plh_layouts)) { 1524 /* The lo must be on the clp list if there is any 1525 * chance of a CB_LAYOUTRECALL(FILE) coming in. 1526 */ 1527 spin_lock(&clp->cl_lock); 1528 if (list_empty(&lo->plh_layouts)) 1529 list_add_tail(&lo->plh_layouts, &server->layouts); 1530 spin_unlock(&clp->cl_lock); 1531 } 1532 1533 pg_offset = arg.offset & ~PAGE_CACHE_MASK; 1534 if (pg_offset) { 1535 arg.offset -= pg_offset; 1536 arg.length += pg_offset; 1537 } 1538 if (arg.length != NFS4_MAX_UINT64) 1539 arg.length = PAGE_CACHE_ALIGN(arg.length); 1540 1541 lseg = send_layoutget(lo, ctx, &arg, gfp_flags); 1542 pnfs_clear_retry_layoutget(lo); 1543 atomic_dec(&lo->plh_outstanding); 1544 out_put_layout_hdr: 1545 if (first) 1546 pnfs_clear_first_layoutget(lo); 1547 pnfs_put_layout_hdr(lo); 1548 out: 1549 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 1550 "(%s, offset: %llu, length: %llu)\n", 1551 __func__, ino->i_sb->s_id, 1552 (unsigned long long)NFS_FILEID(ino), 1553 lseg == NULL ? "not found" : "found", 1554 iomode==IOMODE_RW ? "read/write" : "read-only", 1555 (unsigned long long)pos, 1556 (unsigned long long)count); 1557 return lseg; 1558 out_unlock: 1559 spin_unlock(&ino->i_lock); 1560 goto out_put_layout_hdr; 1561 } 1562 EXPORT_SYMBOL_GPL(pnfs_update_layout); 1563 1564 struct pnfs_layout_segment * 1565 pnfs_layout_process(struct nfs4_layoutget *lgp) 1566 { 1567 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout; 1568 struct nfs4_layoutget_res *res = &lgp->res; 1569 struct pnfs_layout_segment *lseg; 1570 struct inode *ino = lo->plh_inode; 1571 LIST_HEAD(free_me); 1572 int status = 0; 1573 1574 /* Inject layout blob into I/O device driver */ 1575 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 1576 if (!lseg || IS_ERR(lseg)) { 1577 if (!lseg) 1578 status = -ENOMEM; 1579 else 1580 status = PTR_ERR(lseg); 1581 dprintk("%s: Could not allocate layout: error %d\n", 1582 __func__, status); 1583 goto out; 1584 } 1585 1586 init_lseg(lo, lseg); 1587 lseg->pls_range = res->range; 1588 1589 spin_lock(&ino->i_lock); 1590 if (pnfs_layoutgets_blocked(lo)) { 1591 dprintk("%s forget reply due to state\n", __func__); 1592 goto out_forget_reply; 1593 } 1594 1595 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 1596 /* existing state ID, make sure the sequence number matches. */ 1597 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 1598 dprintk("%s forget reply due to sequence\n", __func__); 1599 goto out_forget_reply; 1600 } 1601 pnfs_set_layout_stateid(lo, &res->stateid, false); 1602 } else { 1603 /* 1604 * We got an entirely new state ID. Mark all segments for the 1605 * inode invalid, and don't bother validating the stateid 1606 * sequence number. 1607 */ 1608 pnfs_mark_matching_lsegs_invalid(lo, &free_me, NULL); 1609 1610 nfs4_stateid_copy(&lo->plh_stateid, &res->stateid); 1611 lo->plh_barrier = be32_to_cpu(res->stateid.seqid); 1612 } 1613 1614 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1615 1616 pnfs_get_lseg(lseg); 1617 pnfs_layout_insert_lseg(lo, lseg); 1618 1619 if (res->return_on_close) 1620 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 1621 1622 spin_unlock(&ino->i_lock); 1623 pnfs_free_lseg_list(&free_me); 1624 return lseg; 1625 out: 1626 return ERR_PTR(status); 1627 1628 out_forget_reply: 1629 spin_unlock(&ino->i_lock); 1630 lseg->pls_layout = lo; 1631 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 1632 goto out; 1633 } 1634 1635 static void 1636 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 1637 struct list_head *tmp_list, 1638 struct pnfs_layout_range *return_range) 1639 { 1640 struct pnfs_layout_segment *lseg, *next; 1641 1642 dprintk("%s:Begin lo %p\n", __func__, lo); 1643 1644 if (list_empty(&lo->plh_segs)) 1645 return; 1646 1647 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 1648 if (should_free_lseg(&lseg->pls_range, return_range)) { 1649 dprintk("%s: marking lseg %p iomode %d " 1650 "offset %llu length %llu\n", __func__, 1651 lseg, lseg->pls_range.iomode, 1652 lseg->pls_range.offset, 1653 lseg->pls_range.length); 1654 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1655 mark_lseg_invalid(lseg, tmp_list); 1656 } 1657 } 1658 1659 void pnfs_error_mark_layout_for_return(struct inode *inode, 1660 struct pnfs_layout_segment *lseg) 1661 { 1662 struct pnfs_layout_hdr *lo = NFS_I(inode)->layout; 1663 int iomode = pnfs_iomode_to_fail_bit(lseg->pls_range.iomode); 1664 struct pnfs_layout_range range = { 1665 .iomode = lseg->pls_range.iomode, 1666 .offset = 0, 1667 .length = NFS4_MAX_UINT64, 1668 }; 1669 LIST_HEAD(free_me); 1670 1671 spin_lock(&inode->i_lock); 1672 /* set failure bit so that pnfs path will be retried later */ 1673 pnfs_layout_set_fail_bit(lo, iomode); 1674 if (lo->plh_return_iomode == 0) 1675 lo->plh_return_iomode = range.iomode; 1676 else if (lo->plh_return_iomode != range.iomode) 1677 lo->plh_return_iomode = IOMODE_ANY; 1678 /* 1679 * mark all matching lsegs so that we are sure to have no live 1680 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 1681 * for how it works. 1682 */ 1683 pnfs_mark_matching_lsegs_return(lo, &free_me, &range); 1684 spin_unlock(&inode->i_lock); 1685 pnfs_free_lseg_list(&free_me); 1686 } 1687 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 1688 1689 void 1690 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 1691 { 1692 u64 rd_size = req->wb_bytes; 1693 1694 if (pgio->pg_lseg == NULL) { 1695 if (pgio->pg_dreq == NULL) 1696 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 1697 else 1698 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq); 1699 1700 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 1701 req->wb_context, 1702 req_offset(req), 1703 rd_size, 1704 IOMODE_READ, 1705 GFP_KERNEL); 1706 } 1707 /* If no lseg, fall back to read through mds */ 1708 if (pgio->pg_lseg == NULL) 1709 nfs_pageio_reset_read_mds(pgio); 1710 1711 } 1712 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 1713 1714 void 1715 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 1716 struct nfs_page *req, u64 wb_size) 1717 { 1718 if (pgio->pg_lseg == NULL) 1719 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 1720 req->wb_context, 1721 req_offset(req), 1722 wb_size, 1723 IOMODE_RW, 1724 GFP_NOFS); 1725 /* If no lseg, fall back to write through mds */ 1726 if (pgio->pg_lseg == NULL) 1727 nfs_pageio_reset_write_mds(pgio); 1728 } 1729 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 1730 1731 void 1732 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 1733 { 1734 if (desc->pg_lseg) { 1735 pnfs_put_lseg(desc->pg_lseg); 1736 desc->pg_lseg = NULL; 1737 } 1738 } 1739 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 1740 1741 /* 1742 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 1743 * of bytes (maximum @req->wb_bytes) that can be coalesced. 1744 */ 1745 size_t 1746 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 1747 struct nfs_page *prev, struct nfs_page *req) 1748 { 1749 unsigned int size; 1750 u64 seg_end, req_start, seg_left; 1751 1752 size = nfs_generic_pg_test(pgio, prev, req); 1753 if (!size) 1754 return 0; 1755 1756 /* 1757 * 'size' contains the number of bytes left in the current page (up 1758 * to the original size asked for in @req->wb_bytes). 1759 * 1760 * Calculate how many bytes are left in the layout segment 1761 * and if there are less bytes than 'size', return that instead. 1762 * 1763 * Please also note that 'end_offset' is actually the offset of the 1764 * first byte that lies outside the pnfs_layout_range. FIXME? 1765 * 1766 */ 1767 if (pgio->pg_lseg) { 1768 seg_end = end_offset(pgio->pg_lseg->pls_range.offset, 1769 pgio->pg_lseg->pls_range.length); 1770 req_start = req_offset(req); 1771 WARN_ON_ONCE(req_start >= seg_end); 1772 /* start of request is past the last byte of this segment */ 1773 if (req_start >= seg_end) { 1774 /* reference the new lseg */ 1775 if (pgio->pg_ops->pg_cleanup) 1776 pgio->pg_ops->pg_cleanup(pgio); 1777 if (pgio->pg_ops->pg_init) 1778 pgio->pg_ops->pg_init(pgio, req); 1779 return 0; 1780 } 1781 1782 /* adjust 'size' iff there are fewer bytes left in the 1783 * segment than what nfs_generic_pg_test returned */ 1784 seg_left = seg_end - req_start; 1785 if (seg_left < size) 1786 size = (unsigned int)seg_left; 1787 } 1788 1789 return size; 1790 } 1791 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 1792 1793 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 1794 { 1795 struct nfs_pageio_descriptor pgio; 1796 1797 /* Resend all requests through the MDS */ 1798 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 1799 hdr->completion_ops); 1800 set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags); 1801 return nfs_pageio_resend(&pgio, hdr); 1802 } 1803 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 1804 1805 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 1806 { 1807 1808 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 1809 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 1810 PNFS_LAYOUTRET_ON_ERROR) { 1811 pnfs_return_layout(hdr->inode); 1812 } 1813 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 1814 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 1815 } 1816 1817 /* 1818 * Called by non rpc-based layout drivers 1819 */ 1820 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 1821 { 1822 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 1823 if (!hdr->pnfs_error) { 1824 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 1825 hdr->mds_offset + hdr->res.count); 1826 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 1827 } else 1828 pnfs_ld_handle_write_error(hdr); 1829 hdr->mds_ops->rpc_release(hdr); 1830 } 1831 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 1832 1833 static void 1834 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 1835 struct nfs_pgio_header *hdr) 1836 { 1837 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 1838 1839 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 1840 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 1841 nfs_pageio_reset_write_mds(desc); 1842 mirror->pg_recoalesce = 1; 1843 } 1844 nfs_pgio_data_destroy(hdr); 1845 hdr->release(hdr); 1846 } 1847 1848 static enum pnfs_try_status 1849 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 1850 const struct rpc_call_ops *call_ops, 1851 struct pnfs_layout_segment *lseg, 1852 int how) 1853 { 1854 struct inode *inode = hdr->inode; 1855 enum pnfs_try_status trypnfs; 1856 struct nfs_server *nfss = NFS_SERVER(inode); 1857 1858 hdr->mds_ops = call_ops; 1859 1860 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__, 1861 inode->i_ino, hdr->args.count, hdr->args.offset, how); 1862 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 1863 if (trypnfs != PNFS_NOT_ATTEMPTED) 1864 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 1865 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 1866 return trypnfs; 1867 } 1868 1869 static void 1870 pnfs_do_write(struct nfs_pageio_descriptor *desc, 1871 struct nfs_pgio_header *hdr, int how) 1872 { 1873 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 1874 struct pnfs_layout_segment *lseg = desc->pg_lseg; 1875 enum pnfs_try_status trypnfs; 1876 1877 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 1878 if (trypnfs == PNFS_NOT_ATTEMPTED) 1879 pnfs_write_through_mds(desc, hdr); 1880 } 1881 1882 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 1883 { 1884 pnfs_put_lseg(hdr->lseg); 1885 nfs_pgio_header_free(hdr); 1886 } 1887 1888 int 1889 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 1890 { 1891 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 1892 1893 struct nfs_pgio_header *hdr; 1894 int ret; 1895 1896 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 1897 if (!hdr) { 1898 desc->pg_completion_ops->error_cleanup(&mirror->pg_list); 1899 return -ENOMEM; 1900 } 1901 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 1902 1903 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 1904 ret = nfs_generic_pgio(desc, hdr); 1905 if (!ret) 1906 pnfs_do_write(desc, hdr, desc->pg_ioflags); 1907 1908 return ret; 1909 } 1910 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 1911 1912 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 1913 { 1914 struct nfs_pageio_descriptor pgio; 1915 1916 /* Resend all requests through the MDS */ 1917 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 1918 return nfs_pageio_resend(&pgio, hdr); 1919 } 1920 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 1921 1922 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 1923 { 1924 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 1925 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 1926 PNFS_LAYOUTRET_ON_ERROR) { 1927 pnfs_return_layout(hdr->inode); 1928 } 1929 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 1930 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 1931 } 1932 1933 /* 1934 * Called by non rpc-based layout drivers 1935 */ 1936 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 1937 { 1938 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 1939 if (likely(!hdr->pnfs_error)) { 1940 __nfs4_read_done_cb(hdr); 1941 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 1942 } else 1943 pnfs_ld_handle_read_error(hdr); 1944 hdr->mds_ops->rpc_release(hdr); 1945 } 1946 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 1947 1948 static void 1949 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 1950 struct nfs_pgio_header *hdr) 1951 { 1952 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 1953 1954 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 1955 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 1956 nfs_pageio_reset_read_mds(desc); 1957 mirror->pg_recoalesce = 1; 1958 } 1959 nfs_pgio_data_destroy(hdr); 1960 hdr->release(hdr); 1961 } 1962 1963 /* 1964 * Call the appropriate parallel I/O subsystem read function. 1965 */ 1966 static enum pnfs_try_status 1967 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 1968 const struct rpc_call_ops *call_ops, 1969 struct pnfs_layout_segment *lseg) 1970 { 1971 struct inode *inode = hdr->inode; 1972 struct nfs_server *nfss = NFS_SERVER(inode); 1973 enum pnfs_try_status trypnfs; 1974 1975 hdr->mds_ops = call_ops; 1976 1977 dprintk("%s: Reading ino:%lu %u@%llu\n", 1978 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 1979 1980 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 1981 if (trypnfs != PNFS_NOT_ATTEMPTED) 1982 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 1983 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 1984 return trypnfs; 1985 } 1986 1987 /* Resend all requests through pnfs. */ 1988 int pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr) 1989 { 1990 struct nfs_pageio_descriptor pgio; 1991 1992 nfs_pageio_init_read(&pgio, hdr->inode, false, hdr->completion_ops); 1993 return nfs_pageio_resend(&pgio, hdr); 1994 } 1995 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 1996 1997 static void 1998 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 1999 { 2000 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 2001 struct pnfs_layout_segment *lseg = desc->pg_lseg; 2002 enum pnfs_try_status trypnfs; 2003 int err = 0; 2004 2005 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 2006 if (trypnfs == PNFS_TRY_AGAIN) 2007 err = pnfs_read_resend_pnfs(hdr); 2008 if (trypnfs == PNFS_NOT_ATTEMPTED || err) 2009 pnfs_read_through_mds(desc, hdr); 2010 } 2011 2012 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 2013 { 2014 pnfs_put_lseg(hdr->lseg); 2015 nfs_pgio_header_free(hdr); 2016 } 2017 2018 int 2019 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 2020 { 2021 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2022 2023 struct nfs_pgio_header *hdr; 2024 int ret; 2025 2026 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 2027 if (!hdr) { 2028 desc->pg_completion_ops->error_cleanup(&mirror->pg_list); 2029 return -ENOMEM; 2030 } 2031 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 2032 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 2033 ret = nfs_generic_pgio(desc, hdr); 2034 if (!ret) 2035 pnfs_do_read(desc, hdr); 2036 return ret; 2037 } 2038 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 2039 2040 static void pnfs_clear_layoutcommitting(struct inode *inode) 2041 { 2042 unsigned long *bitlock = &NFS_I(inode)->flags; 2043 2044 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 2045 smp_mb__after_atomic(); 2046 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 2047 } 2048 2049 /* 2050 * There can be multiple RW segments. 2051 */ 2052 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 2053 { 2054 struct pnfs_layout_segment *lseg; 2055 2056 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 2057 if (lseg->pls_range.iomode == IOMODE_RW && 2058 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 2059 list_add(&lseg->pls_lc_list, listp); 2060 } 2061 } 2062 2063 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 2064 { 2065 struct pnfs_layout_segment *lseg, *tmp; 2066 2067 /* Matched by references in pnfs_set_layoutcommit */ 2068 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 2069 list_del_init(&lseg->pls_lc_list); 2070 pnfs_put_lseg(lseg); 2071 } 2072 2073 pnfs_clear_layoutcommitting(inode); 2074 } 2075 2076 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 2077 { 2078 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 2079 } 2080 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 2081 2082 void 2083 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 2084 loff_t end_pos) 2085 { 2086 struct nfs_inode *nfsi = NFS_I(inode); 2087 bool mark_as_dirty = false; 2088 2089 spin_lock(&inode->i_lock); 2090 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 2091 nfsi->layout->plh_lwb = end_pos; 2092 mark_as_dirty = true; 2093 dprintk("%s: Set layoutcommit for inode %lu ", 2094 __func__, inode->i_ino); 2095 } else if (end_pos > nfsi->layout->plh_lwb) 2096 nfsi->layout->plh_lwb = end_pos; 2097 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 2098 /* references matched in nfs4_layoutcommit_release */ 2099 pnfs_get_lseg(lseg); 2100 } 2101 spin_unlock(&inode->i_lock); 2102 dprintk("%s: lseg %p end_pos %llu\n", 2103 __func__, lseg, nfsi->layout->plh_lwb); 2104 2105 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 2106 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 2107 if (mark_as_dirty) 2108 mark_inode_dirty_sync(inode); 2109 } 2110 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 2111 2112 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 2113 { 2114 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 2115 2116 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 2117 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 2118 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 2119 } 2120 2121 /* 2122 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 2123 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 2124 * data to disk to allow the server to recover the data if it crashes. 2125 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 2126 * is off, and a COMMIT is sent to a data server, or 2127 * if WRITEs to a data server return NFS_DATA_SYNC. 2128 */ 2129 int 2130 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 2131 { 2132 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 2133 struct nfs4_layoutcommit_data *data; 2134 struct nfs_inode *nfsi = NFS_I(inode); 2135 loff_t end_pos; 2136 int status; 2137 2138 if (!pnfs_layoutcommit_outstanding(inode)) 2139 return 0; 2140 2141 dprintk("--> %s inode %lu\n", __func__, inode->i_ino); 2142 2143 status = -EAGAIN; 2144 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 2145 if (!sync) 2146 goto out; 2147 status = wait_on_bit_lock_action(&nfsi->flags, 2148 NFS_INO_LAYOUTCOMMITTING, 2149 nfs_wait_bit_killable, 2150 TASK_KILLABLE); 2151 if (status) 2152 goto out; 2153 } 2154 2155 status = -ENOMEM; 2156 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 2157 data = kzalloc(sizeof(*data), GFP_NOFS); 2158 if (!data) 2159 goto clear_layoutcommitting; 2160 2161 status = 0; 2162 spin_lock(&inode->i_lock); 2163 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 2164 goto out_unlock; 2165 2166 INIT_LIST_HEAD(&data->lseg_list); 2167 pnfs_list_write_lseg(inode, &data->lseg_list); 2168 2169 end_pos = nfsi->layout->plh_lwb; 2170 2171 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 2172 spin_unlock(&inode->i_lock); 2173 2174 data->args.inode = inode; 2175 data->cred = get_rpccred(nfsi->layout->plh_lc_cred); 2176 nfs_fattr_init(&data->fattr); 2177 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 2178 data->res.fattr = &data->fattr; 2179 data->args.lastbytewritten = end_pos - 1; 2180 data->res.server = NFS_SERVER(inode); 2181 2182 if (ld->prepare_layoutcommit) { 2183 status = ld->prepare_layoutcommit(&data->args); 2184 if (status) { 2185 put_rpccred(data->cred); 2186 spin_lock(&inode->i_lock); 2187 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 2188 if (end_pos > nfsi->layout->plh_lwb) 2189 nfsi->layout->plh_lwb = end_pos; 2190 goto out_unlock; 2191 } 2192 } 2193 2194 2195 status = nfs4_proc_layoutcommit(data, sync); 2196 out: 2197 if (status) 2198 mark_inode_dirty_sync(inode); 2199 dprintk("<-- %s status %d\n", __func__, status); 2200 return status; 2201 out_unlock: 2202 spin_unlock(&inode->i_lock); 2203 kfree(data); 2204 clear_layoutcommitting: 2205 pnfs_clear_layoutcommitting(inode); 2206 goto out; 2207 } 2208 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 2209 2210 int 2211 pnfs_generic_sync(struct inode *inode, bool datasync) 2212 { 2213 return pnfs_layoutcommit_inode(inode, true); 2214 } 2215 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 2216 2217 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 2218 { 2219 struct nfs4_threshold *thp; 2220 2221 thp = kzalloc(sizeof(*thp), GFP_NOFS); 2222 if (!thp) { 2223 dprintk("%s mdsthreshold allocation failed\n", __func__); 2224 return NULL; 2225 } 2226 return thp; 2227 } 2228 2229 #if IS_ENABLED(CONFIG_NFS_V4_2) 2230 int 2231 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 2232 { 2233 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 2234 struct nfs_server *server = NFS_SERVER(inode); 2235 struct nfs_inode *nfsi = NFS_I(inode); 2236 struct nfs42_layoutstat_data *data; 2237 struct pnfs_layout_hdr *hdr; 2238 int status = 0; 2239 2240 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 2241 goto out; 2242 2243 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 2244 goto out; 2245 2246 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 2247 goto out; 2248 2249 spin_lock(&inode->i_lock); 2250 if (!NFS_I(inode)->layout) { 2251 spin_unlock(&inode->i_lock); 2252 goto out; 2253 } 2254 hdr = NFS_I(inode)->layout; 2255 pnfs_get_layout_hdr(hdr); 2256 spin_unlock(&inode->i_lock); 2257 2258 data = kzalloc(sizeof(*data), gfp_flags); 2259 if (!data) { 2260 status = -ENOMEM; 2261 goto out_put; 2262 } 2263 2264 data->args.fh = NFS_FH(inode); 2265 data->args.inode = inode; 2266 nfs4_stateid_copy(&data->args.stateid, &hdr->plh_stateid); 2267 status = ld->prepare_layoutstats(&data->args); 2268 if (status) 2269 goto out_free; 2270 2271 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 2272 2273 out: 2274 dprintk("%s returns %d\n", __func__, status); 2275 return status; 2276 2277 out_free: 2278 kfree(data); 2279 out_put: 2280 pnfs_put_layout_hdr(hdr); 2281 smp_mb__before_atomic(); 2282 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 2283 smp_mb__after_atomic(); 2284 goto out; 2285 } 2286 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 2287 #endif 2288