1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include <linux/sort.h> 34 #include "internal.h" 35 #include "pnfs.h" 36 #include "iostat.h" 37 #include "nfs4trace.h" 38 #include "delegation.h" 39 #include "nfs42.h" 40 41 #define NFSDBG_FACILITY NFSDBG_PNFS 42 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 43 44 /* Locking: 45 * 46 * pnfs_spinlock: 47 * protects pnfs_modules_tbl. 48 */ 49 static DEFINE_SPINLOCK(pnfs_spinlock); 50 51 /* 52 * pnfs_modules_tbl holds all pnfs modules 53 */ 54 static LIST_HEAD(pnfs_modules_tbl); 55 56 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); 57 58 /* Return the registered pnfs layout driver module matching given id */ 59 static struct pnfs_layoutdriver_type * 60 find_pnfs_driver_locked(u32 id) 61 { 62 struct pnfs_layoutdriver_type *local; 63 64 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 65 if (local->id == id) 66 goto out; 67 local = NULL; 68 out: 69 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 70 return local; 71 } 72 73 static struct pnfs_layoutdriver_type * 74 find_pnfs_driver(u32 id) 75 { 76 struct pnfs_layoutdriver_type *local; 77 78 spin_lock(&pnfs_spinlock); 79 local = find_pnfs_driver_locked(id); 80 if (local != NULL && !try_module_get(local->owner)) { 81 dprintk("%s: Could not grab reference on module\n", __func__); 82 local = NULL; 83 } 84 spin_unlock(&pnfs_spinlock); 85 return local; 86 } 87 88 void 89 unset_pnfs_layoutdriver(struct nfs_server *nfss) 90 { 91 if (nfss->pnfs_curr_ld) { 92 if (nfss->pnfs_curr_ld->clear_layoutdriver) 93 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 94 /* Decrement the MDS count. Purge the deviceid cache if zero */ 95 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 96 nfs4_deviceid_purge_client(nfss->nfs_client); 97 module_put(nfss->pnfs_curr_ld->owner); 98 } 99 nfss->pnfs_curr_ld = NULL; 100 } 101 102 /* 103 * When the server sends a list of layout types, we choose one in the order 104 * given in the list below. 105 * 106 * FIXME: should this list be configurable in some fashion? module param? 107 * mount option? something else? 108 */ 109 static const u32 ld_prefs[] = { 110 LAYOUT_SCSI, 111 LAYOUT_BLOCK_VOLUME, 112 LAYOUT_OSD2_OBJECTS, 113 LAYOUT_FLEX_FILES, 114 LAYOUT_NFSV4_1_FILES, 115 0 116 }; 117 118 static int 119 ld_cmp(const void *e1, const void *e2) 120 { 121 u32 ld1 = *((u32 *)e1); 122 u32 ld2 = *((u32 *)e2); 123 int i; 124 125 for (i = 0; ld_prefs[i] != 0; i++) { 126 if (ld1 == ld_prefs[i]) 127 return -1; 128 129 if (ld2 == ld_prefs[i]) 130 return 1; 131 } 132 return 0; 133 } 134 135 /* 136 * Try to set the server's pnfs module to the pnfs layout type specified by id. 137 * Currently only one pNFS layout driver per filesystem is supported. 138 * 139 * @ids array of layout types supported by MDS. 140 */ 141 void 142 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 143 struct nfs_fsinfo *fsinfo) 144 { 145 struct pnfs_layoutdriver_type *ld_type = NULL; 146 u32 id; 147 int i; 148 149 if (fsinfo->nlayouttypes == 0) 150 goto out_no_driver; 151 if (!(server->nfs_client->cl_exchange_flags & 152 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 153 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", 154 __func__, server->nfs_client->cl_exchange_flags); 155 goto out_no_driver; 156 } 157 158 sort(fsinfo->layouttype, fsinfo->nlayouttypes, 159 sizeof(*fsinfo->layouttype), ld_cmp, NULL); 160 161 for (i = 0; i < fsinfo->nlayouttypes; i++) { 162 id = fsinfo->layouttype[i]; 163 ld_type = find_pnfs_driver(id); 164 if (!ld_type) { 165 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, 166 id); 167 ld_type = find_pnfs_driver(id); 168 } 169 if (ld_type) 170 break; 171 } 172 173 if (!ld_type) { 174 dprintk("%s: No pNFS module found!\n", __func__); 175 goto out_no_driver; 176 } 177 178 server->pnfs_curr_ld = ld_type; 179 if (ld_type->set_layoutdriver 180 && ld_type->set_layoutdriver(server, mntfh)) { 181 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 182 "driver %u.\n", __func__, id); 183 module_put(ld_type->owner); 184 goto out_no_driver; 185 } 186 /* Bump the MDS count */ 187 atomic_inc(&server->nfs_client->cl_mds_count); 188 189 dprintk("%s: pNFS module for %u set\n", __func__, id); 190 return; 191 192 out_no_driver: 193 dprintk("%s: Using NFSv4 I/O\n", __func__); 194 server->pnfs_curr_ld = NULL; 195 } 196 197 int 198 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 199 { 200 int status = -EINVAL; 201 struct pnfs_layoutdriver_type *tmp; 202 203 if (ld_type->id == 0) { 204 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 205 return status; 206 } 207 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 208 printk(KERN_ERR "NFS: %s Layout driver must provide " 209 "alloc_lseg and free_lseg.\n", __func__); 210 return status; 211 } 212 213 spin_lock(&pnfs_spinlock); 214 tmp = find_pnfs_driver_locked(ld_type->id); 215 if (!tmp) { 216 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 217 status = 0; 218 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 219 ld_type->name); 220 } else { 221 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 222 __func__, ld_type->id); 223 } 224 spin_unlock(&pnfs_spinlock); 225 226 return status; 227 } 228 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 229 230 void 231 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 232 { 233 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 234 spin_lock(&pnfs_spinlock); 235 list_del(&ld_type->pnfs_tblid); 236 spin_unlock(&pnfs_spinlock); 237 } 238 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 239 240 /* 241 * pNFS client layout cache 242 */ 243 244 /* Need to hold i_lock if caller does not already hold reference */ 245 void 246 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 247 { 248 atomic_inc(&lo->plh_refcount); 249 } 250 251 static struct pnfs_layout_hdr * 252 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 253 { 254 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 255 return ld->alloc_layout_hdr(ino, gfp_flags); 256 } 257 258 static void 259 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 260 { 261 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 262 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 263 264 if (!list_empty(&lo->plh_layouts)) { 265 struct nfs_client *clp = server->nfs_client; 266 267 spin_lock(&clp->cl_lock); 268 list_del_init(&lo->plh_layouts); 269 spin_unlock(&clp->cl_lock); 270 } 271 put_rpccred(lo->plh_lc_cred); 272 return ld->free_layout_hdr(lo); 273 } 274 275 static void 276 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 277 { 278 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 279 dprintk("%s: freeing layout cache %p\n", __func__, lo); 280 nfsi->layout = NULL; 281 /* Reset MDS Threshold I/O counters */ 282 nfsi->write_io = 0; 283 nfsi->read_io = 0; 284 } 285 286 void 287 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 288 { 289 struct inode *inode = lo->plh_inode; 290 291 pnfs_layoutreturn_before_put_layout_hdr(lo); 292 293 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 294 if (!list_empty(&lo->plh_segs)) 295 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 296 pnfs_detach_layout_hdr(lo); 297 spin_unlock(&inode->i_lock); 298 pnfs_free_layout_hdr(lo); 299 } 300 } 301 302 /* 303 * Mark a pnfs_layout_hdr and all associated layout segments as invalid 304 * 305 * In order to continue using the pnfs_layout_hdr, a full recovery 306 * is required. 307 * Note that caller must hold inode->i_lock. 308 */ 309 int 310 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, 311 struct list_head *lseg_list) 312 { 313 struct pnfs_layout_range range = { 314 .iomode = IOMODE_ANY, 315 .offset = 0, 316 .length = NFS4_MAX_UINT64, 317 }; 318 319 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 320 return pnfs_mark_matching_lsegs_invalid(lo, lseg_list, &range, 0); 321 } 322 323 static int 324 pnfs_iomode_to_fail_bit(u32 iomode) 325 { 326 return iomode == IOMODE_RW ? 327 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 328 } 329 330 static void 331 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 332 { 333 lo->plh_retry_timestamp = jiffies; 334 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 335 atomic_inc(&lo->plh_refcount); 336 } 337 338 static void 339 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 340 { 341 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 342 atomic_dec(&lo->plh_refcount); 343 } 344 345 static void 346 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 347 { 348 struct inode *inode = lo->plh_inode; 349 struct pnfs_layout_range range = { 350 .iomode = iomode, 351 .offset = 0, 352 .length = NFS4_MAX_UINT64, 353 }; 354 LIST_HEAD(head); 355 356 spin_lock(&inode->i_lock); 357 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 358 pnfs_mark_matching_lsegs_invalid(lo, &head, &range, 0); 359 spin_unlock(&inode->i_lock); 360 pnfs_free_lseg_list(&head); 361 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 362 iomode == IOMODE_RW ? "RW" : "READ"); 363 } 364 365 static bool 366 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 367 { 368 unsigned long start, end; 369 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 370 371 if (test_bit(fail_bit, &lo->plh_flags) == 0) 372 return false; 373 end = jiffies; 374 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 375 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 376 /* It is time to retry the failed layoutgets */ 377 pnfs_layout_clear_fail_bit(lo, fail_bit); 378 return false; 379 } 380 return true; 381 } 382 383 static void 384 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, 385 const struct pnfs_layout_range *range, 386 const nfs4_stateid *stateid) 387 { 388 INIT_LIST_HEAD(&lseg->pls_list); 389 INIT_LIST_HEAD(&lseg->pls_lc_list); 390 atomic_set(&lseg->pls_refcount, 1); 391 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 392 lseg->pls_layout = lo; 393 lseg->pls_range = *range; 394 lseg->pls_seq = be32_to_cpu(stateid->seqid); 395 } 396 397 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 398 { 399 struct inode *ino = lseg->pls_layout->plh_inode; 400 401 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 402 } 403 404 static void 405 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 406 struct pnfs_layout_segment *lseg) 407 { 408 struct inode *inode = lo->plh_inode; 409 410 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 411 list_del_init(&lseg->pls_list); 412 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 413 atomic_dec(&lo->plh_refcount); 414 if (list_empty(&lo->plh_segs)) { 415 if (atomic_read(&lo->plh_outstanding) == 0) 416 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 417 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 418 } 419 rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq); 420 } 421 422 void 423 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 424 { 425 struct pnfs_layout_hdr *lo; 426 struct inode *inode; 427 428 if (!lseg) 429 return; 430 431 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 432 atomic_read(&lseg->pls_refcount), 433 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 434 435 lo = lseg->pls_layout; 436 inode = lo->plh_inode; 437 438 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 439 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 440 spin_unlock(&inode->i_lock); 441 return; 442 } 443 pnfs_get_layout_hdr(lo); 444 pnfs_layout_remove_lseg(lo, lseg); 445 spin_unlock(&inode->i_lock); 446 pnfs_free_lseg(lseg); 447 pnfs_put_layout_hdr(lo); 448 } 449 } 450 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 451 452 static void pnfs_free_lseg_async_work(struct work_struct *work) 453 { 454 struct pnfs_layout_segment *lseg; 455 struct pnfs_layout_hdr *lo; 456 457 lseg = container_of(work, struct pnfs_layout_segment, pls_work); 458 lo = lseg->pls_layout; 459 460 pnfs_free_lseg(lseg); 461 pnfs_put_layout_hdr(lo); 462 } 463 464 static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg) 465 { 466 INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work); 467 schedule_work(&lseg->pls_work); 468 } 469 470 void 471 pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg) 472 { 473 if (!lseg) 474 return; 475 476 assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock); 477 478 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 479 atomic_read(&lseg->pls_refcount), 480 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 481 if (atomic_dec_and_test(&lseg->pls_refcount)) { 482 struct pnfs_layout_hdr *lo = lseg->pls_layout; 483 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 484 return; 485 pnfs_get_layout_hdr(lo); 486 pnfs_layout_remove_lseg(lo, lseg); 487 pnfs_free_lseg_async(lseg); 488 } 489 } 490 EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked); 491 492 static u64 493 end_offset(u64 start, u64 len) 494 { 495 u64 end; 496 497 end = start + len; 498 return end >= start ? end : NFS4_MAX_UINT64; 499 } 500 501 /* 502 * is l2 fully contained in l1? 503 * start1 end1 504 * [----------------------------------) 505 * start2 end2 506 * [----------------) 507 */ 508 static bool 509 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 510 const struct pnfs_layout_range *l2) 511 { 512 u64 start1 = l1->offset; 513 u64 end1 = end_offset(start1, l1->length); 514 u64 start2 = l2->offset; 515 u64 end2 = end_offset(start2, l2->length); 516 517 return (start1 <= start2) && (end1 >= end2); 518 } 519 520 /* 521 * is l1 and l2 intersecting? 522 * start1 end1 523 * [----------------------------------) 524 * start2 end2 525 * [----------------) 526 */ 527 static bool 528 pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1, 529 const struct pnfs_layout_range *l2) 530 { 531 u64 start1 = l1->offset; 532 u64 end1 = end_offset(start1, l1->length); 533 u64 start2 = l2->offset; 534 u64 end2 = end_offset(start2, l2->length); 535 536 return (end1 == NFS4_MAX_UINT64 || end1 > start2) && 537 (end2 == NFS4_MAX_UINT64 || end2 > start1); 538 } 539 540 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 541 struct list_head *tmp_list) 542 { 543 if (!atomic_dec_and_test(&lseg->pls_refcount)) 544 return false; 545 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 546 list_add(&lseg->pls_list, tmp_list); 547 return true; 548 } 549 550 /* Returns 1 if lseg is removed from list, 0 otherwise */ 551 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 552 struct list_head *tmp_list) 553 { 554 int rv = 0; 555 556 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 557 /* Remove the reference keeping the lseg in the 558 * list. It will now be removed when all 559 * outstanding io is finished. 560 */ 561 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 562 atomic_read(&lseg->pls_refcount)); 563 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 564 rv = 1; 565 } 566 return rv; 567 } 568 569 /* 570 * Compare 2 layout stateid sequence ids, to see which is newer, 571 * taking into account wraparound issues. 572 */ 573 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 574 { 575 return (s32)(s1 - s2) > 0; 576 } 577 578 static bool 579 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, 580 const struct pnfs_layout_range *recall_range) 581 { 582 return (recall_range->iomode == IOMODE_ANY || 583 lseg_range->iomode == recall_range->iomode) && 584 pnfs_lseg_range_intersecting(lseg_range, recall_range); 585 } 586 587 static bool 588 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, 589 const struct pnfs_layout_range *recall_range, 590 u32 seq) 591 { 592 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) 593 return false; 594 if (recall_range == NULL) 595 return true; 596 return pnfs_should_free_range(&lseg->pls_range, recall_range); 597 } 598 599 /** 600 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later 601 * @lo: layout header containing the lsegs 602 * @tmp_list: list head where doomed lsegs should go 603 * @recall_range: optional recall range argument to match (may be NULL) 604 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) 605 * 606 * Walk the list of lsegs in the layout header, and tear down any that should 607 * be destroyed. If "recall_range" is specified then the segment must match 608 * that range. If "seq" is non-zero, then only match segments that were handed 609 * out at or before that sequence. 610 * 611 * Returns number of matching invalid lsegs remaining in list after scanning 612 * it and purging them. 613 */ 614 int 615 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 616 struct list_head *tmp_list, 617 const struct pnfs_layout_range *recall_range, 618 u32 seq) 619 { 620 struct pnfs_layout_segment *lseg, *next; 621 int remaining = 0; 622 623 dprintk("%s:Begin lo %p\n", __func__, lo); 624 625 if (list_empty(&lo->plh_segs)) 626 return 0; 627 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 628 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { 629 dprintk("%s: freeing lseg %p iomode %d seq %u" 630 "offset %llu length %llu\n", __func__, 631 lseg, lseg->pls_range.iomode, lseg->pls_seq, 632 lseg->pls_range.offset, lseg->pls_range.length); 633 if (!mark_lseg_invalid(lseg, tmp_list)) 634 remaining++; 635 } 636 dprintk("%s:Return %i\n", __func__, remaining); 637 return remaining; 638 } 639 640 /* note free_me must contain lsegs from a single layout_hdr */ 641 void 642 pnfs_free_lseg_list(struct list_head *free_me) 643 { 644 struct pnfs_layout_segment *lseg, *tmp; 645 646 if (list_empty(free_me)) 647 return; 648 649 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 650 list_del(&lseg->pls_list); 651 pnfs_free_lseg(lseg); 652 } 653 } 654 655 void 656 pnfs_destroy_layout(struct nfs_inode *nfsi) 657 { 658 struct pnfs_layout_hdr *lo; 659 LIST_HEAD(tmp_list); 660 661 spin_lock(&nfsi->vfs_inode.i_lock); 662 lo = nfsi->layout; 663 if (lo) { 664 pnfs_get_layout_hdr(lo); 665 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); 666 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 667 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 668 spin_unlock(&nfsi->vfs_inode.i_lock); 669 pnfs_free_lseg_list(&tmp_list); 670 pnfs_put_layout_hdr(lo); 671 } else 672 spin_unlock(&nfsi->vfs_inode.i_lock); 673 } 674 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 675 676 static bool 677 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 678 struct list_head *layout_list) 679 { 680 struct pnfs_layout_hdr *lo; 681 bool ret = false; 682 683 spin_lock(&inode->i_lock); 684 lo = NFS_I(inode)->layout; 685 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 686 pnfs_get_layout_hdr(lo); 687 list_add(&lo->plh_bulk_destroy, layout_list); 688 ret = true; 689 } 690 spin_unlock(&inode->i_lock); 691 return ret; 692 } 693 694 /* Caller must hold rcu_read_lock and clp->cl_lock */ 695 static int 696 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 697 struct nfs_server *server, 698 struct list_head *layout_list) 699 { 700 struct pnfs_layout_hdr *lo, *next; 701 struct inode *inode; 702 703 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 704 inode = igrab(lo->plh_inode); 705 if (inode == NULL) 706 continue; 707 list_del_init(&lo->plh_layouts); 708 if (pnfs_layout_add_bulk_destroy_list(inode, layout_list)) 709 continue; 710 rcu_read_unlock(); 711 spin_unlock(&clp->cl_lock); 712 iput(inode); 713 spin_lock(&clp->cl_lock); 714 rcu_read_lock(); 715 return -EAGAIN; 716 } 717 return 0; 718 } 719 720 static int 721 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 722 bool is_bulk_recall) 723 { 724 struct pnfs_layout_hdr *lo; 725 struct inode *inode; 726 LIST_HEAD(lseg_list); 727 int ret = 0; 728 729 while (!list_empty(layout_list)) { 730 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 731 plh_bulk_destroy); 732 dprintk("%s freeing layout for inode %lu\n", __func__, 733 lo->plh_inode->i_ino); 734 inode = lo->plh_inode; 735 736 pnfs_layoutcommit_inode(inode, false); 737 738 spin_lock(&inode->i_lock); 739 list_del_init(&lo->plh_bulk_destroy); 740 if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { 741 if (is_bulk_recall) 742 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 743 ret = -EAGAIN; 744 } 745 spin_unlock(&inode->i_lock); 746 pnfs_free_lseg_list(&lseg_list); 747 /* Free all lsegs that are attached to commit buckets */ 748 nfs_commit_inode(inode, 0); 749 pnfs_put_layout_hdr(lo); 750 iput(inode); 751 } 752 return ret; 753 } 754 755 int 756 pnfs_destroy_layouts_byfsid(struct nfs_client *clp, 757 struct nfs_fsid *fsid, 758 bool is_recall) 759 { 760 struct nfs_server *server; 761 LIST_HEAD(layout_list); 762 763 spin_lock(&clp->cl_lock); 764 rcu_read_lock(); 765 restart: 766 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 767 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 768 continue; 769 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 770 server, 771 &layout_list) != 0) 772 goto restart; 773 } 774 rcu_read_unlock(); 775 spin_unlock(&clp->cl_lock); 776 777 if (list_empty(&layout_list)) 778 return 0; 779 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall); 780 } 781 782 int 783 pnfs_destroy_layouts_byclid(struct nfs_client *clp, 784 bool is_recall) 785 { 786 struct nfs_server *server; 787 LIST_HEAD(layout_list); 788 789 spin_lock(&clp->cl_lock); 790 rcu_read_lock(); 791 restart: 792 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 793 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 794 server, 795 &layout_list) != 0) 796 goto restart; 797 } 798 rcu_read_unlock(); 799 spin_unlock(&clp->cl_lock); 800 801 if (list_empty(&layout_list)) 802 return 0; 803 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall); 804 } 805 806 /* 807 * Called by the state manger to remove all layouts established under an 808 * expired lease. 809 */ 810 void 811 pnfs_destroy_all_layouts(struct nfs_client *clp) 812 { 813 nfs4_deviceid_mark_client_invalid(clp); 814 nfs4_deviceid_purge_client(clp); 815 816 pnfs_destroy_layouts_byclid(clp, false); 817 } 818 819 static void 820 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) 821 { 822 lo->plh_return_iomode = 0; 823 lo->plh_return_seq = 0; 824 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 825 } 826 827 /* update lo->plh_stateid with new if is more recent */ 828 void 829 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 830 bool update_barrier) 831 { 832 u32 oldseq, newseq, new_barrier = 0; 833 834 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 835 newseq = be32_to_cpu(new->seqid); 836 837 if (!pnfs_layout_is_valid(lo)) { 838 nfs4_stateid_copy(&lo->plh_stateid, new); 839 lo->plh_barrier = newseq; 840 pnfs_clear_layoutreturn_info(lo); 841 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 842 return; 843 } 844 if (pnfs_seqid_is_newer(newseq, oldseq)) { 845 nfs4_stateid_copy(&lo->plh_stateid, new); 846 /* 847 * Because of wraparound, we want to keep the barrier 848 * "close" to the current seqids. 849 */ 850 new_barrier = newseq - atomic_read(&lo->plh_outstanding); 851 } 852 if (update_barrier) 853 new_barrier = be32_to_cpu(new->seqid); 854 else if (new_barrier == 0) 855 return; 856 if (pnfs_seqid_is_newer(new_barrier, lo->plh_barrier)) 857 lo->plh_barrier = new_barrier; 858 } 859 860 static bool 861 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 862 const nfs4_stateid *stateid) 863 { 864 u32 seqid = be32_to_cpu(stateid->seqid); 865 866 return !pnfs_seqid_is_newer(seqid, lo->plh_barrier); 867 } 868 869 /* lget is set to 1 if called from inside send_layoutget call chain */ 870 static bool 871 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 872 { 873 return lo->plh_block_lgets || 874 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 875 } 876 877 /* 878 * Get layout from server. 879 * for now, assume that whole file layouts are requested. 880 * arg->offset: 0 881 * arg->length: all ones 882 */ 883 static struct pnfs_layout_segment * 884 send_layoutget(struct pnfs_layout_hdr *lo, 885 struct nfs_open_context *ctx, 886 nfs4_stateid *stateid, 887 const struct pnfs_layout_range *range, 888 long *timeout, gfp_t gfp_flags) 889 { 890 struct inode *ino = lo->plh_inode; 891 struct nfs_server *server = NFS_SERVER(ino); 892 struct nfs4_layoutget *lgp; 893 loff_t i_size; 894 895 dprintk("--> %s\n", __func__); 896 897 /* 898 * Synchronously retrieve layout information from server and 899 * store in lseg. If we race with a concurrent seqid morphing 900 * op, then re-send the LAYOUTGET. 901 */ 902 lgp = kzalloc(sizeof(*lgp), gfp_flags); 903 if (lgp == NULL) 904 return ERR_PTR(-ENOMEM); 905 906 i_size = i_size_read(ino); 907 908 lgp->args.minlength = PAGE_SIZE; 909 if (lgp->args.minlength > range->length) 910 lgp->args.minlength = range->length; 911 if (range->iomode == IOMODE_READ) { 912 if (range->offset >= i_size) 913 lgp->args.minlength = 0; 914 else if (i_size - range->offset < lgp->args.minlength) 915 lgp->args.minlength = i_size - range->offset; 916 } 917 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 918 pnfs_copy_range(&lgp->args.range, range); 919 lgp->args.type = server->pnfs_curr_ld->id; 920 lgp->args.inode = ino; 921 lgp->args.ctx = get_nfs_open_context(ctx); 922 nfs4_stateid_copy(&lgp->args.stateid, stateid); 923 lgp->gfp_flags = gfp_flags; 924 lgp->cred = lo->plh_lc_cred; 925 926 return nfs4_proc_layoutget(lgp, timeout, gfp_flags); 927 } 928 929 static void pnfs_clear_layoutcommit(struct inode *inode, 930 struct list_head *head) 931 { 932 struct nfs_inode *nfsi = NFS_I(inode); 933 struct pnfs_layout_segment *lseg, *tmp; 934 935 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 936 return; 937 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 938 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 939 continue; 940 pnfs_lseg_dec_and_remove_zero(lseg, head); 941 } 942 } 943 944 void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 945 { 946 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 947 smp_mb__after_atomic(); 948 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 949 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 950 } 951 952 static bool 953 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, 954 nfs4_stateid *stateid, 955 enum pnfs_iomode *iomode) 956 { 957 /* Serialise LAYOUTGET/LAYOUTRETURN */ 958 if (atomic_read(&lo->plh_outstanding) != 0) 959 return false; 960 if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 961 return false; 962 pnfs_get_layout_hdr(lo); 963 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 964 if (stateid != NULL) { 965 nfs4_stateid_copy(stateid, &lo->plh_stateid); 966 if (lo->plh_return_seq != 0) 967 stateid->seqid = cpu_to_be32(lo->plh_return_seq); 968 } 969 if (iomode != NULL) 970 *iomode = lo->plh_return_iomode; 971 pnfs_clear_layoutreturn_info(lo); 972 return true; 973 } 974 if (stateid != NULL) 975 nfs4_stateid_copy(stateid, &lo->plh_stateid); 976 if (iomode != NULL) 977 *iomode = IOMODE_ANY; 978 return true; 979 } 980 981 static int 982 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid, 983 enum pnfs_iomode iomode, bool sync) 984 { 985 struct inode *ino = lo->plh_inode; 986 struct nfs4_layoutreturn *lrp; 987 int status = 0; 988 989 lrp = kzalloc(sizeof(*lrp), GFP_NOFS); 990 if (unlikely(lrp == NULL)) { 991 status = -ENOMEM; 992 spin_lock(&ino->i_lock); 993 pnfs_clear_layoutreturn_waitbit(lo); 994 spin_unlock(&ino->i_lock); 995 pnfs_put_layout_hdr(lo); 996 goto out; 997 } 998 999 nfs4_stateid_copy(&lrp->args.stateid, stateid); 1000 lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id; 1001 lrp->args.inode = ino; 1002 lrp->args.range.iomode = iomode; 1003 lrp->args.range.offset = 0; 1004 lrp->args.range.length = NFS4_MAX_UINT64; 1005 lrp->args.layout = lo; 1006 lrp->clp = NFS_SERVER(ino)->nfs_client; 1007 lrp->cred = lo->plh_lc_cred; 1008 1009 status = nfs4_proc_layoutreturn(lrp, sync); 1010 out: 1011 dprintk("<-- %s status: %d\n", __func__, status); 1012 return status; 1013 } 1014 1015 /* Return true if layoutreturn is needed */ 1016 static bool 1017 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) 1018 { 1019 struct pnfs_layout_segment *s; 1020 1021 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1022 return false; 1023 1024 /* Defer layoutreturn until all lsegs are done */ 1025 list_for_each_entry(s, &lo->plh_segs, pls_list) { 1026 if (test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags)) 1027 return false; 1028 } 1029 1030 return true; 1031 } 1032 1033 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) 1034 { 1035 struct inode *inode= lo->plh_inode; 1036 1037 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1038 return; 1039 spin_lock(&inode->i_lock); 1040 if (pnfs_layout_need_return(lo)) { 1041 nfs4_stateid stateid; 1042 enum pnfs_iomode iomode; 1043 bool send; 1044 1045 send = pnfs_prepare_layoutreturn(lo, &stateid, &iomode); 1046 spin_unlock(&inode->i_lock); 1047 if (send) { 1048 /* Send an async layoutreturn so we dont deadlock */ 1049 pnfs_send_layoutreturn(lo, &stateid, iomode, false); 1050 } 1051 } else 1052 spin_unlock(&inode->i_lock); 1053 } 1054 1055 /* 1056 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 1057 * when the layout segment list is empty. 1058 * 1059 * Note that a pnfs_layout_hdr can exist with an empty layout segment 1060 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 1061 * deviceid is marked invalid. 1062 */ 1063 int 1064 _pnfs_return_layout(struct inode *ino) 1065 { 1066 struct pnfs_layout_hdr *lo = NULL; 1067 struct nfs_inode *nfsi = NFS_I(ino); 1068 LIST_HEAD(tmp_list); 1069 nfs4_stateid stateid; 1070 int status = 0, empty; 1071 bool send; 1072 1073 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino); 1074 1075 spin_lock(&ino->i_lock); 1076 lo = nfsi->layout; 1077 if (!lo) { 1078 spin_unlock(&ino->i_lock); 1079 dprintk("NFS: %s no layout to return\n", __func__); 1080 goto out; 1081 } 1082 /* Reference matched in nfs4_layoutreturn_release */ 1083 pnfs_get_layout_hdr(lo); 1084 empty = list_empty(&lo->plh_segs); 1085 pnfs_clear_layoutcommit(ino, &tmp_list); 1086 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0); 1087 1088 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) { 1089 struct pnfs_layout_range range = { 1090 .iomode = IOMODE_ANY, 1091 .offset = 0, 1092 .length = NFS4_MAX_UINT64, 1093 }; 1094 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1095 } 1096 1097 /* Don't send a LAYOUTRETURN if list was initially empty */ 1098 if (empty) { 1099 spin_unlock(&ino->i_lock); 1100 dprintk("NFS: %s no layout segments to return\n", __func__); 1101 goto out_put_layout_hdr; 1102 } 1103 1104 send = pnfs_prepare_layoutreturn(lo, &stateid, NULL); 1105 spin_unlock(&ino->i_lock); 1106 pnfs_free_lseg_list(&tmp_list); 1107 if (send) 1108 status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true); 1109 out_put_layout_hdr: 1110 pnfs_put_layout_hdr(lo); 1111 out: 1112 dprintk("<-- %s status: %d\n", __func__, status); 1113 return status; 1114 } 1115 EXPORT_SYMBOL_GPL(_pnfs_return_layout); 1116 1117 int 1118 pnfs_commit_and_return_layout(struct inode *inode) 1119 { 1120 struct pnfs_layout_hdr *lo; 1121 int ret; 1122 1123 spin_lock(&inode->i_lock); 1124 lo = NFS_I(inode)->layout; 1125 if (lo == NULL) { 1126 spin_unlock(&inode->i_lock); 1127 return 0; 1128 } 1129 pnfs_get_layout_hdr(lo); 1130 /* Block new layoutgets and read/write to ds */ 1131 lo->plh_block_lgets++; 1132 spin_unlock(&inode->i_lock); 1133 filemap_fdatawait(inode->i_mapping); 1134 ret = pnfs_layoutcommit_inode(inode, true); 1135 if (ret == 0) 1136 ret = _pnfs_return_layout(inode); 1137 spin_lock(&inode->i_lock); 1138 lo->plh_block_lgets--; 1139 spin_unlock(&inode->i_lock); 1140 pnfs_put_layout_hdr(lo); 1141 return ret; 1142 } 1143 1144 bool pnfs_roc(struct inode *ino) 1145 { 1146 struct nfs_inode *nfsi = NFS_I(ino); 1147 struct nfs_open_context *ctx; 1148 struct nfs4_state *state; 1149 struct pnfs_layout_hdr *lo; 1150 struct pnfs_layout_segment *lseg, *tmp; 1151 nfs4_stateid stateid; 1152 LIST_HEAD(tmp_list); 1153 bool found = false, layoutreturn = false, roc = false; 1154 1155 spin_lock(&ino->i_lock); 1156 lo = nfsi->layout; 1157 if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) 1158 goto out_noroc; 1159 1160 /* no roc if we hold a delegation */ 1161 if (nfs4_check_delegation(ino, FMODE_READ)) 1162 goto out_noroc; 1163 1164 list_for_each_entry(ctx, &nfsi->open_files, list) { 1165 state = ctx->state; 1166 /* Don't return layout if there is open file state */ 1167 if (state != NULL && state->state != 0) 1168 goto out_noroc; 1169 } 1170 1171 /* always send layoutreturn if being marked so */ 1172 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1173 layoutreturn = pnfs_prepare_layoutreturn(lo, 1174 &stateid, NULL); 1175 1176 list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list) 1177 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1178 if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) { 1179 mark_lseg_invalid(lseg, &tmp_list); 1180 found = true; 1181 } 1182 /* ROC in two conditions: 1183 * 1. there are ROC lsegs 1184 * 2. we don't send layoutreturn 1185 */ 1186 if (found && !layoutreturn) { 1187 /* lo ref dropped in pnfs_roc_release() */ 1188 pnfs_get_layout_hdr(lo); 1189 roc = true; 1190 } 1191 1192 out_noroc: 1193 spin_unlock(&ino->i_lock); 1194 pnfs_free_lseg_list(&tmp_list); 1195 pnfs_layoutcommit_inode(ino, true); 1196 if (layoutreturn) 1197 pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true); 1198 return roc; 1199 } 1200 1201 void pnfs_roc_release(struct inode *ino) 1202 { 1203 struct pnfs_layout_hdr *lo; 1204 1205 spin_lock(&ino->i_lock); 1206 lo = NFS_I(ino)->layout; 1207 pnfs_clear_layoutreturn_waitbit(lo); 1208 if (atomic_dec_and_test(&lo->plh_refcount)) { 1209 pnfs_detach_layout_hdr(lo); 1210 spin_unlock(&ino->i_lock); 1211 pnfs_free_layout_hdr(lo); 1212 } else 1213 spin_unlock(&ino->i_lock); 1214 } 1215 1216 void pnfs_roc_set_barrier(struct inode *ino, u32 barrier) 1217 { 1218 struct pnfs_layout_hdr *lo; 1219 1220 spin_lock(&ino->i_lock); 1221 lo = NFS_I(ino)->layout; 1222 if (pnfs_seqid_is_newer(barrier, lo->plh_barrier)) 1223 lo->plh_barrier = barrier; 1224 spin_unlock(&ino->i_lock); 1225 trace_nfs4_layoutreturn_on_close(ino, 0); 1226 } 1227 1228 void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier) 1229 { 1230 struct nfs_inode *nfsi = NFS_I(ino); 1231 struct pnfs_layout_hdr *lo; 1232 u32 current_seqid; 1233 1234 spin_lock(&ino->i_lock); 1235 lo = nfsi->layout; 1236 current_seqid = be32_to_cpu(lo->plh_stateid.seqid); 1237 1238 /* Since close does not return a layout stateid for use as 1239 * a barrier, we choose the worst-case barrier. 1240 */ 1241 *barrier = current_seqid + atomic_read(&lo->plh_outstanding); 1242 spin_unlock(&ino->i_lock); 1243 } 1244 1245 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1246 { 1247 struct nfs_inode *nfsi = NFS_I(ino); 1248 struct pnfs_layout_hdr *lo; 1249 bool sleep = false; 1250 1251 /* we might not have grabbed lo reference. so need to check under 1252 * i_lock */ 1253 spin_lock(&ino->i_lock); 1254 lo = nfsi->layout; 1255 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 1256 sleep = true; 1257 spin_unlock(&ino->i_lock); 1258 1259 if (sleep) 1260 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1261 1262 return sleep; 1263 } 1264 1265 /* 1266 * Compare two layout segments for sorting into layout cache. 1267 * We want to preferentially return RW over RO layouts, so ensure those 1268 * are seen first. 1269 */ 1270 static s64 1271 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1272 const struct pnfs_layout_range *l2) 1273 { 1274 s64 d; 1275 1276 /* high offset > low offset */ 1277 d = l1->offset - l2->offset; 1278 if (d) 1279 return d; 1280 1281 /* short length > long length */ 1282 d = l2->length - l1->length; 1283 if (d) 1284 return d; 1285 1286 /* read > read/write */ 1287 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1288 } 1289 1290 static bool 1291 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1292 const struct pnfs_layout_range *l2) 1293 { 1294 return pnfs_lseg_range_cmp(l1, l2) > 0; 1295 } 1296 1297 static bool 1298 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1299 struct pnfs_layout_segment *old) 1300 { 1301 return false; 1302 } 1303 1304 void 1305 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1306 struct pnfs_layout_segment *lseg, 1307 bool (*is_after)(const struct pnfs_layout_range *, 1308 const struct pnfs_layout_range *), 1309 bool (*do_merge)(struct pnfs_layout_segment *, 1310 struct pnfs_layout_segment *), 1311 struct list_head *free_me) 1312 { 1313 struct pnfs_layout_segment *lp, *tmp; 1314 1315 dprintk("%s:Begin\n", __func__); 1316 1317 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1318 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1319 continue; 1320 if (do_merge(lseg, lp)) { 1321 mark_lseg_invalid(lp, free_me); 1322 continue; 1323 } 1324 if (is_after(&lseg->pls_range, &lp->pls_range)) 1325 continue; 1326 list_add_tail(&lseg->pls_list, &lp->pls_list); 1327 dprintk("%s: inserted lseg %p " 1328 "iomode %d offset %llu length %llu before " 1329 "lp %p iomode %d offset %llu length %llu\n", 1330 __func__, lseg, lseg->pls_range.iomode, 1331 lseg->pls_range.offset, lseg->pls_range.length, 1332 lp, lp->pls_range.iomode, lp->pls_range.offset, 1333 lp->pls_range.length); 1334 goto out; 1335 } 1336 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1337 dprintk("%s: inserted lseg %p " 1338 "iomode %d offset %llu length %llu at tail\n", 1339 __func__, lseg, lseg->pls_range.iomode, 1340 lseg->pls_range.offset, lseg->pls_range.length); 1341 out: 1342 pnfs_get_layout_hdr(lo); 1343 1344 dprintk("%s:Return\n", __func__); 1345 } 1346 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1347 1348 static void 1349 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1350 struct pnfs_layout_segment *lseg, 1351 struct list_head *free_me) 1352 { 1353 struct inode *inode = lo->plh_inode; 1354 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1355 1356 if (ld->add_lseg != NULL) 1357 ld->add_lseg(lo, lseg, free_me); 1358 else 1359 pnfs_generic_layout_insert_lseg(lo, lseg, 1360 pnfs_lseg_range_is_after, 1361 pnfs_lseg_no_merge, 1362 free_me); 1363 } 1364 1365 static struct pnfs_layout_hdr * 1366 alloc_init_layout_hdr(struct inode *ino, 1367 struct nfs_open_context *ctx, 1368 gfp_t gfp_flags) 1369 { 1370 struct pnfs_layout_hdr *lo; 1371 1372 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1373 if (!lo) 1374 return NULL; 1375 atomic_set(&lo->plh_refcount, 1); 1376 INIT_LIST_HEAD(&lo->plh_layouts); 1377 INIT_LIST_HEAD(&lo->plh_segs); 1378 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1379 lo->plh_inode = ino; 1380 lo->plh_lc_cred = get_rpccred(ctx->cred); 1381 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; 1382 return lo; 1383 } 1384 1385 static struct pnfs_layout_hdr * 1386 pnfs_find_alloc_layout(struct inode *ino, 1387 struct nfs_open_context *ctx, 1388 gfp_t gfp_flags) 1389 __releases(&ino->i_lock) 1390 __acquires(&ino->i_lock) 1391 { 1392 struct nfs_inode *nfsi = NFS_I(ino); 1393 struct pnfs_layout_hdr *new = NULL; 1394 1395 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1396 1397 if (nfsi->layout != NULL) 1398 goto out_existing; 1399 spin_unlock(&ino->i_lock); 1400 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1401 spin_lock(&ino->i_lock); 1402 1403 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1404 nfsi->layout = new; 1405 return new; 1406 } else if (new != NULL) 1407 pnfs_free_layout_hdr(new); 1408 out_existing: 1409 pnfs_get_layout_hdr(nfsi->layout); 1410 return nfsi->layout; 1411 } 1412 1413 /* 1414 * iomode matching rules: 1415 * iomode lseg strict match 1416 * iomode 1417 * ----- ----- ------ ----- 1418 * ANY READ N/A true 1419 * ANY RW N/A true 1420 * RW READ N/A false 1421 * RW RW N/A true 1422 * READ READ N/A true 1423 * READ RW true false 1424 * READ RW false true 1425 */ 1426 static bool 1427 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1428 const struct pnfs_layout_range *range, 1429 bool strict_iomode) 1430 { 1431 struct pnfs_layout_range range1; 1432 1433 if ((range->iomode == IOMODE_RW && 1434 ls_range->iomode != IOMODE_RW) || 1435 (range->iomode != ls_range->iomode && 1436 strict_iomode == true) || 1437 !pnfs_lseg_range_intersecting(ls_range, range)) 1438 return 0; 1439 1440 /* range1 covers only the first byte in the range */ 1441 range1 = *range; 1442 range1.length = 1; 1443 return pnfs_lseg_range_contained(ls_range, &range1); 1444 } 1445 1446 /* 1447 * lookup range in layout 1448 */ 1449 static struct pnfs_layout_segment * 1450 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1451 struct pnfs_layout_range *range, 1452 bool strict_iomode) 1453 { 1454 struct pnfs_layout_segment *lseg, *ret = NULL; 1455 1456 dprintk("%s:Begin\n", __func__); 1457 1458 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1459 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1460 !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 1461 pnfs_lseg_range_match(&lseg->pls_range, range, 1462 strict_iomode)) { 1463 ret = pnfs_get_lseg(lseg); 1464 break; 1465 } 1466 } 1467 1468 dprintk("%s:Return lseg %p ref %d\n", 1469 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0); 1470 return ret; 1471 } 1472 1473 /* 1474 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1475 * to the MDS or over pNFS 1476 * 1477 * The nfs_inode read_io and write_io fields are cumulative counters reset 1478 * when there are no layout segments. Note that in pnfs_update_layout iomode 1479 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 1480 * WRITE request. 1481 * 1482 * A return of true means use MDS I/O. 1483 * 1484 * From rfc 5661: 1485 * If a file's size is smaller than the file size threshold, data accesses 1486 * SHOULD be sent to the metadata server. If an I/O request has a length that 1487 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 1488 * server. If both file size and I/O size are provided, the client SHOULD 1489 * reach or exceed both thresholds before sending its read or write 1490 * requests to the data server. 1491 */ 1492 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 1493 struct inode *ino, int iomode) 1494 { 1495 struct nfs4_threshold *t = ctx->mdsthreshold; 1496 struct nfs_inode *nfsi = NFS_I(ino); 1497 loff_t fsize = i_size_read(ino); 1498 bool size = false, size_set = false, io = false, io_set = false, ret = false; 1499 1500 if (t == NULL) 1501 return ret; 1502 1503 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 1504 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 1505 1506 switch (iomode) { 1507 case IOMODE_READ: 1508 if (t->bm & THRESHOLD_RD) { 1509 dprintk("%s fsize %llu\n", __func__, fsize); 1510 size_set = true; 1511 if (fsize < t->rd_sz) 1512 size = true; 1513 } 1514 if (t->bm & THRESHOLD_RD_IO) { 1515 dprintk("%s nfsi->read_io %llu\n", __func__, 1516 nfsi->read_io); 1517 io_set = true; 1518 if (nfsi->read_io < t->rd_io_sz) 1519 io = true; 1520 } 1521 break; 1522 case IOMODE_RW: 1523 if (t->bm & THRESHOLD_WR) { 1524 dprintk("%s fsize %llu\n", __func__, fsize); 1525 size_set = true; 1526 if (fsize < t->wr_sz) 1527 size = true; 1528 } 1529 if (t->bm & THRESHOLD_WR_IO) { 1530 dprintk("%s nfsi->write_io %llu\n", __func__, 1531 nfsi->write_io); 1532 io_set = true; 1533 if (nfsi->write_io < t->wr_io_sz) 1534 io = true; 1535 } 1536 break; 1537 } 1538 if (size_set && io_set) { 1539 if (size && io) 1540 ret = true; 1541 } else if (size || io) 1542 ret = true; 1543 1544 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 1545 return ret; 1546 } 1547 1548 static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 1549 { 1550 /* 1551 * send layoutcommit as it can hold up layoutreturn due to lseg 1552 * reference 1553 */ 1554 pnfs_layoutcommit_inode(lo->plh_inode, false); 1555 return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 1556 nfs_wait_bit_killable, 1557 TASK_UNINTERRUPTIBLE); 1558 } 1559 1560 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 1561 { 1562 unsigned long *bitlock = &lo->plh_flags; 1563 1564 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 1565 smp_mb__after_atomic(); 1566 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 1567 } 1568 1569 /* 1570 * Layout segment is retreived from the server if not cached. 1571 * The appropriate layout segment is referenced and returned to the caller. 1572 */ 1573 struct pnfs_layout_segment * 1574 pnfs_update_layout(struct inode *ino, 1575 struct nfs_open_context *ctx, 1576 loff_t pos, 1577 u64 count, 1578 enum pnfs_iomode iomode, 1579 bool strict_iomode, 1580 gfp_t gfp_flags) 1581 { 1582 struct pnfs_layout_range arg = { 1583 .iomode = iomode, 1584 .offset = pos, 1585 .length = count, 1586 }; 1587 unsigned pg_offset, seq; 1588 struct nfs_server *server = NFS_SERVER(ino); 1589 struct nfs_client *clp = server->nfs_client; 1590 struct pnfs_layout_hdr *lo = NULL; 1591 struct pnfs_layout_segment *lseg = NULL; 1592 nfs4_stateid stateid; 1593 long timeout = 0; 1594 unsigned long giveup = jiffies + (clp->cl_lease_time << 1); 1595 bool first; 1596 1597 if (!pnfs_enabled_sb(NFS_SERVER(ino))) { 1598 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1599 PNFS_UPDATE_LAYOUT_NO_PNFS); 1600 goto out; 1601 } 1602 1603 if (iomode == IOMODE_READ && i_size_read(ino) == 0) { 1604 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1605 PNFS_UPDATE_LAYOUT_RD_ZEROLEN); 1606 goto out; 1607 } 1608 1609 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { 1610 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1611 PNFS_UPDATE_LAYOUT_MDSTHRESH); 1612 goto out; 1613 } 1614 1615 lookup_again: 1616 nfs4_client_recover_expired_lease(clp); 1617 first = false; 1618 spin_lock(&ino->i_lock); 1619 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 1620 if (lo == NULL) { 1621 spin_unlock(&ino->i_lock); 1622 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1623 PNFS_UPDATE_LAYOUT_NOMEM); 1624 goto out; 1625 } 1626 1627 /* Do we even need to bother with this? */ 1628 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1629 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1630 PNFS_UPDATE_LAYOUT_BULK_RECALL); 1631 dprintk("%s matches recall, use MDS\n", __func__); 1632 goto out_unlock; 1633 } 1634 1635 /* if LAYOUTGET already failed once we don't try again */ 1636 if (pnfs_layout_io_test_failed(lo, iomode)) { 1637 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1638 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); 1639 goto out_unlock; 1640 } 1641 1642 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); 1643 if (lseg) { 1644 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1645 PNFS_UPDATE_LAYOUT_FOUND_CACHED); 1646 goto out_unlock; 1647 } 1648 1649 if (!nfs4_valid_open_stateid(ctx->state)) { 1650 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1651 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 1652 goto out_unlock; 1653 } 1654 1655 /* 1656 * Choose a stateid for the LAYOUTGET. If we don't have a layout 1657 * stateid, or it has been invalidated, then we must use the open 1658 * stateid. 1659 */ 1660 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 1661 1662 /* 1663 * The first layoutget for the file. Need to serialize per 1664 * RFC 5661 Errata 3208. 1665 */ 1666 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 1667 &lo->plh_flags)) { 1668 spin_unlock(&ino->i_lock); 1669 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET, 1670 TASK_UNINTERRUPTIBLE); 1671 pnfs_put_layout_hdr(lo); 1672 dprintk("%s retrying\n", __func__); 1673 goto lookup_again; 1674 } 1675 1676 first = true; 1677 do { 1678 seq = read_seqbegin(&ctx->state->seqlock); 1679 nfs4_stateid_copy(&stateid, &ctx->state->stateid); 1680 } while (read_seqretry(&ctx->state->seqlock, seq)); 1681 } else { 1682 nfs4_stateid_copy(&stateid, &lo->plh_stateid); 1683 } 1684 1685 /* 1686 * Because we free lsegs before sending LAYOUTRETURN, we need to wait 1687 * for LAYOUTRETURN even if first is true. 1688 */ 1689 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1690 spin_unlock(&ino->i_lock); 1691 dprintk("%s wait for layoutreturn\n", __func__); 1692 if (pnfs_prepare_to_retry_layoutget(lo)) { 1693 if (first) 1694 pnfs_clear_first_layoutget(lo); 1695 pnfs_put_layout_hdr(lo); 1696 dprintk("%s retrying\n", __func__); 1697 trace_pnfs_update_layout(ino, pos, count, iomode, lo, 1698 lseg, PNFS_UPDATE_LAYOUT_RETRY); 1699 goto lookup_again; 1700 } 1701 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1702 PNFS_UPDATE_LAYOUT_RETURN); 1703 goto out_put_layout_hdr; 1704 } 1705 1706 if (pnfs_layoutgets_blocked(lo)) { 1707 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1708 PNFS_UPDATE_LAYOUT_BLOCKED); 1709 goto out_unlock; 1710 } 1711 atomic_inc(&lo->plh_outstanding); 1712 spin_unlock(&ino->i_lock); 1713 1714 if (list_empty(&lo->plh_layouts)) { 1715 /* The lo must be on the clp list if there is any 1716 * chance of a CB_LAYOUTRECALL(FILE) coming in. 1717 */ 1718 spin_lock(&clp->cl_lock); 1719 if (list_empty(&lo->plh_layouts)) 1720 list_add_tail(&lo->plh_layouts, &server->layouts); 1721 spin_unlock(&clp->cl_lock); 1722 } 1723 1724 pg_offset = arg.offset & ~PAGE_MASK; 1725 if (pg_offset) { 1726 arg.offset -= pg_offset; 1727 arg.length += pg_offset; 1728 } 1729 if (arg.length != NFS4_MAX_UINT64) 1730 arg.length = PAGE_ALIGN(arg.length); 1731 1732 lseg = send_layoutget(lo, ctx, &stateid, &arg, &timeout, gfp_flags); 1733 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1734 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); 1735 atomic_dec(&lo->plh_outstanding); 1736 if (IS_ERR(lseg)) { 1737 switch(PTR_ERR(lseg)) { 1738 case -EBUSY: 1739 if (time_after(jiffies, giveup)) 1740 lseg = NULL; 1741 break; 1742 case -ERECALLCONFLICT: 1743 /* Huh? We hold no layouts, how is there a recall? */ 1744 if (first) { 1745 lseg = NULL; 1746 break; 1747 } 1748 /* Destroy the existing layout and start over */ 1749 if (time_after(jiffies, giveup)) 1750 pnfs_destroy_layout(NFS_I(ino)); 1751 /* Fallthrough */ 1752 case -EAGAIN: 1753 break; 1754 default: 1755 if (!nfs_error_is_fatal(PTR_ERR(lseg))) { 1756 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 1757 lseg = NULL; 1758 } 1759 goto out_put_layout_hdr; 1760 } 1761 if (lseg) { 1762 if (first) 1763 pnfs_clear_first_layoutget(lo); 1764 trace_pnfs_update_layout(ino, pos, count, 1765 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); 1766 pnfs_put_layout_hdr(lo); 1767 goto lookup_again; 1768 } 1769 } else { 1770 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 1771 } 1772 1773 out_put_layout_hdr: 1774 if (first) 1775 pnfs_clear_first_layoutget(lo); 1776 pnfs_put_layout_hdr(lo); 1777 out: 1778 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 1779 "(%s, offset: %llu, length: %llu)\n", 1780 __func__, ino->i_sb->s_id, 1781 (unsigned long long)NFS_FILEID(ino), 1782 IS_ERR_OR_NULL(lseg) ? "not found" : "found", 1783 iomode==IOMODE_RW ? "read/write" : "read-only", 1784 (unsigned long long)pos, 1785 (unsigned long long)count); 1786 return lseg; 1787 out_unlock: 1788 spin_unlock(&ino->i_lock); 1789 goto out_put_layout_hdr; 1790 } 1791 EXPORT_SYMBOL_GPL(pnfs_update_layout); 1792 1793 static bool 1794 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 1795 { 1796 switch (range->iomode) { 1797 case IOMODE_READ: 1798 case IOMODE_RW: 1799 break; 1800 default: 1801 return false; 1802 } 1803 if (range->offset == NFS4_MAX_UINT64) 1804 return false; 1805 if (range->length == 0) 1806 return false; 1807 if (range->length != NFS4_MAX_UINT64 && 1808 range->length > NFS4_MAX_UINT64 - range->offset) 1809 return false; 1810 return true; 1811 } 1812 1813 struct pnfs_layout_segment * 1814 pnfs_layout_process(struct nfs4_layoutget *lgp) 1815 { 1816 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout; 1817 struct nfs4_layoutget_res *res = &lgp->res; 1818 struct pnfs_layout_segment *lseg; 1819 struct inode *ino = lo->plh_inode; 1820 LIST_HEAD(free_me); 1821 1822 if (!pnfs_sanity_check_layout_range(&res->range)) 1823 return ERR_PTR(-EINVAL); 1824 1825 /* Inject layout blob into I/O device driver */ 1826 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 1827 if (IS_ERR_OR_NULL(lseg)) { 1828 if (!lseg) 1829 lseg = ERR_PTR(-ENOMEM); 1830 1831 dprintk("%s: Could not allocate layout: error %ld\n", 1832 __func__, PTR_ERR(lseg)); 1833 return lseg; 1834 } 1835 1836 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); 1837 1838 spin_lock(&ino->i_lock); 1839 if (pnfs_layoutgets_blocked(lo)) { 1840 dprintk("%s forget reply due to state\n", __func__); 1841 goto out_forget; 1842 } 1843 1844 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 1845 /* existing state ID, make sure the sequence number matches. */ 1846 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 1847 dprintk("%s forget reply due to sequence\n", __func__); 1848 goto out_forget; 1849 } 1850 pnfs_set_layout_stateid(lo, &res->stateid, false); 1851 } else { 1852 /* 1853 * We got an entirely new state ID. Mark all segments for the 1854 * inode invalid, and don't bother validating the stateid 1855 * sequence number. 1856 */ 1857 pnfs_mark_layout_stateid_invalid(lo, &free_me); 1858 1859 pnfs_set_layout_stateid(lo, &res->stateid, true); 1860 } 1861 1862 pnfs_get_lseg(lseg); 1863 pnfs_layout_insert_lseg(lo, lseg, &free_me); 1864 1865 1866 if (res->return_on_close) 1867 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 1868 1869 spin_unlock(&ino->i_lock); 1870 pnfs_free_lseg_list(&free_me); 1871 return lseg; 1872 1873 out_forget: 1874 spin_unlock(&ino->i_lock); 1875 lseg->pls_layout = lo; 1876 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 1877 return ERR_PTR(-EAGAIN); 1878 } 1879 1880 static void 1881 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, 1882 u32 seq) 1883 { 1884 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) 1885 iomode = IOMODE_ANY; 1886 lo->plh_return_iomode = iomode; 1887 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 1888 if (seq != 0) { 1889 WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq); 1890 lo->plh_return_seq = seq; 1891 } 1892 } 1893 1894 /** 1895 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments 1896 * @lo: pointer to layout header 1897 * @tmp_list: list header to be used with pnfs_free_lseg_list() 1898 * @return_range: describe layout segment ranges to be returned 1899 * 1900 * This function is mainly intended for use by layoutrecall. It attempts 1901 * to free the layout segment immediately, or else to mark it for return 1902 * as soon as its reference count drops to zero. 1903 */ 1904 int 1905 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 1906 struct list_head *tmp_list, 1907 const struct pnfs_layout_range *return_range, 1908 u32 seq) 1909 { 1910 struct pnfs_layout_segment *lseg, *next; 1911 int remaining = 0; 1912 1913 dprintk("%s:Begin lo %p\n", __func__, lo); 1914 1915 if (list_empty(&lo->plh_segs)) 1916 return 0; 1917 1918 assert_spin_locked(&lo->plh_inode->i_lock); 1919 1920 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 1921 if (pnfs_match_lseg_recall(lseg, return_range, seq)) { 1922 dprintk("%s: marking lseg %p iomode %d " 1923 "offset %llu length %llu\n", __func__, 1924 lseg, lseg->pls_range.iomode, 1925 lseg->pls_range.offset, 1926 lseg->pls_range.length); 1927 if (mark_lseg_invalid(lseg, tmp_list)) 1928 continue; 1929 remaining++; 1930 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1931 } 1932 1933 if (remaining) 1934 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 1935 1936 return remaining; 1937 } 1938 1939 void pnfs_error_mark_layout_for_return(struct inode *inode, 1940 struct pnfs_layout_segment *lseg) 1941 { 1942 struct pnfs_layout_hdr *lo = NFS_I(inode)->layout; 1943 struct pnfs_layout_range range = { 1944 .iomode = lseg->pls_range.iomode, 1945 .offset = 0, 1946 .length = NFS4_MAX_UINT64, 1947 }; 1948 LIST_HEAD(free_me); 1949 bool return_now = false; 1950 1951 spin_lock(&inode->i_lock); 1952 pnfs_set_plh_return_info(lo, range.iomode, 0); 1953 /* 1954 * mark all matching lsegs so that we are sure to have no live 1955 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 1956 * for how it works. 1957 */ 1958 if (!pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0)) { 1959 nfs4_stateid stateid; 1960 enum pnfs_iomode iomode; 1961 1962 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &iomode); 1963 spin_unlock(&inode->i_lock); 1964 if (return_now) 1965 pnfs_send_layoutreturn(lo, &stateid, iomode, false); 1966 } else { 1967 spin_unlock(&inode->i_lock); 1968 nfs_commit_inode(inode, 0); 1969 } 1970 pnfs_free_lseg_list(&free_me); 1971 } 1972 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 1973 1974 void 1975 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 1976 { 1977 u64 rd_size = req->wb_bytes; 1978 1979 if (pgio->pg_lseg == NULL) { 1980 if (pgio->pg_dreq == NULL) 1981 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 1982 else 1983 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq); 1984 1985 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 1986 req->wb_context, 1987 req_offset(req), 1988 rd_size, 1989 IOMODE_READ, 1990 false, 1991 GFP_KERNEL); 1992 if (IS_ERR(pgio->pg_lseg)) { 1993 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 1994 pgio->pg_lseg = NULL; 1995 return; 1996 } 1997 } 1998 /* If no lseg, fall back to read through mds */ 1999 if (pgio->pg_lseg == NULL) 2000 nfs_pageio_reset_read_mds(pgio); 2001 2002 } 2003 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 2004 2005 void 2006 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 2007 struct nfs_page *req, u64 wb_size) 2008 { 2009 if (pgio->pg_lseg == NULL) { 2010 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 2011 req->wb_context, 2012 req_offset(req), 2013 wb_size, 2014 IOMODE_RW, 2015 false, 2016 GFP_NOFS); 2017 if (IS_ERR(pgio->pg_lseg)) { 2018 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2019 pgio->pg_lseg = NULL; 2020 return; 2021 } 2022 } 2023 /* If no lseg, fall back to write through mds */ 2024 if (pgio->pg_lseg == NULL) 2025 nfs_pageio_reset_write_mds(pgio); 2026 } 2027 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 2028 2029 void 2030 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 2031 { 2032 if (desc->pg_lseg) { 2033 pnfs_put_lseg(desc->pg_lseg); 2034 desc->pg_lseg = NULL; 2035 } 2036 } 2037 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 2038 2039 /* 2040 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 2041 * of bytes (maximum @req->wb_bytes) that can be coalesced. 2042 */ 2043 size_t 2044 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 2045 struct nfs_page *prev, struct nfs_page *req) 2046 { 2047 unsigned int size; 2048 u64 seg_end, req_start, seg_left; 2049 2050 size = nfs_generic_pg_test(pgio, prev, req); 2051 if (!size) 2052 return 0; 2053 2054 /* 2055 * 'size' contains the number of bytes left in the current page (up 2056 * to the original size asked for in @req->wb_bytes). 2057 * 2058 * Calculate how many bytes are left in the layout segment 2059 * and if there are less bytes than 'size', return that instead. 2060 * 2061 * Please also note that 'end_offset' is actually the offset of the 2062 * first byte that lies outside the pnfs_layout_range. FIXME? 2063 * 2064 */ 2065 if (pgio->pg_lseg) { 2066 seg_end = end_offset(pgio->pg_lseg->pls_range.offset, 2067 pgio->pg_lseg->pls_range.length); 2068 req_start = req_offset(req); 2069 WARN_ON_ONCE(req_start >= seg_end); 2070 /* start of request is past the last byte of this segment */ 2071 if (req_start >= seg_end) { 2072 /* reference the new lseg */ 2073 if (pgio->pg_ops->pg_cleanup) 2074 pgio->pg_ops->pg_cleanup(pgio); 2075 if (pgio->pg_ops->pg_init) 2076 pgio->pg_ops->pg_init(pgio, req); 2077 return 0; 2078 } 2079 2080 /* adjust 'size' iff there are fewer bytes left in the 2081 * segment than what nfs_generic_pg_test returned */ 2082 seg_left = seg_end - req_start; 2083 if (seg_left < size) 2084 size = (unsigned int)seg_left; 2085 } 2086 2087 return size; 2088 } 2089 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 2090 2091 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 2092 { 2093 struct nfs_pageio_descriptor pgio; 2094 2095 /* Resend all requests through the MDS */ 2096 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 2097 hdr->completion_ops); 2098 set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags); 2099 return nfs_pageio_resend(&pgio, hdr); 2100 } 2101 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 2102 2103 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 2104 { 2105 2106 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 2107 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 2108 PNFS_LAYOUTRET_ON_ERROR) { 2109 pnfs_return_layout(hdr->inode); 2110 } 2111 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 2112 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 2113 } 2114 2115 /* 2116 * Called by non rpc-based layout drivers 2117 */ 2118 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 2119 { 2120 if (likely(!hdr->pnfs_error)) { 2121 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 2122 hdr->mds_offset + hdr->res.count); 2123 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 2124 } 2125 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 2126 if (unlikely(hdr->pnfs_error)) 2127 pnfs_ld_handle_write_error(hdr); 2128 hdr->mds_ops->rpc_release(hdr); 2129 } 2130 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 2131 2132 static void 2133 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 2134 struct nfs_pgio_header *hdr) 2135 { 2136 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2137 2138 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2139 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 2140 nfs_pageio_reset_write_mds(desc); 2141 mirror->pg_recoalesce = 1; 2142 } 2143 nfs_pgio_data_destroy(hdr); 2144 hdr->release(hdr); 2145 } 2146 2147 static enum pnfs_try_status 2148 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 2149 const struct rpc_call_ops *call_ops, 2150 struct pnfs_layout_segment *lseg, 2151 int how) 2152 { 2153 struct inode *inode = hdr->inode; 2154 enum pnfs_try_status trypnfs; 2155 struct nfs_server *nfss = NFS_SERVER(inode); 2156 2157 hdr->mds_ops = call_ops; 2158 2159 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__, 2160 inode->i_ino, hdr->args.count, hdr->args.offset, how); 2161 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 2162 if (trypnfs != PNFS_NOT_ATTEMPTED) 2163 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 2164 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 2165 return trypnfs; 2166 } 2167 2168 static void 2169 pnfs_do_write(struct nfs_pageio_descriptor *desc, 2170 struct nfs_pgio_header *hdr, int how) 2171 { 2172 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 2173 struct pnfs_layout_segment *lseg = desc->pg_lseg; 2174 enum pnfs_try_status trypnfs; 2175 2176 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 2177 if (trypnfs == PNFS_NOT_ATTEMPTED) 2178 pnfs_write_through_mds(desc, hdr); 2179 } 2180 2181 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 2182 { 2183 pnfs_put_lseg(hdr->lseg); 2184 nfs_pgio_header_free(hdr); 2185 } 2186 2187 int 2188 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 2189 { 2190 struct nfs_pgio_header *hdr; 2191 int ret; 2192 2193 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 2194 if (!hdr) { 2195 desc->pg_error = -ENOMEM; 2196 return desc->pg_error; 2197 } 2198 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 2199 2200 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 2201 ret = nfs_generic_pgio(desc, hdr); 2202 if (!ret) 2203 pnfs_do_write(desc, hdr, desc->pg_ioflags); 2204 2205 return ret; 2206 } 2207 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 2208 2209 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 2210 { 2211 struct nfs_pageio_descriptor pgio; 2212 2213 /* Resend all requests through the MDS */ 2214 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 2215 return nfs_pageio_resend(&pgio, hdr); 2216 } 2217 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 2218 2219 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 2220 { 2221 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 2222 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 2223 PNFS_LAYOUTRET_ON_ERROR) { 2224 pnfs_return_layout(hdr->inode); 2225 } 2226 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 2227 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 2228 } 2229 2230 /* 2231 * Called by non rpc-based layout drivers 2232 */ 2233 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 2234 { 2235 if (likely(!hdr->pnfs_error)) 2236 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 2237 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 2238 if (unlikely(hdr->pnfs_error)) 2239 pnfs_ld_handle_read_error(hdr); 2240 hdr->mds_ops->rpc_release(hdr); 2241 } 2242 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 2243 2244 static void 2245 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 2246 struct nfs_pgio_header *hdr) 2247 { 2248 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2249 2250 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2251 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 2252 nfs_pageio_reset_read_mds(desc); 2253 mirror->pg_recoalesce = 1; 2254 } 2255 nfs_pgio_data_destroy(hdr); 2256 hdr->release(hdr); 2257 } 2258 2259 /* 2260 * Call the appropriate parallel I/O subsystem read function. 2261 */ 2262 static enum pnfs_try_status 2263 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 2264 const struct rpc_call_ops *call_ops, 2265 struct pnfs_layout_segment *lseg) 2266 { 2267 struct inode *inode = hdr->inode; 2268 struct nfs_server *nfss = NFS_SERVER(inode); 2269 enum pnfs_try_status trypnfs; 2270 2271 hdr->mds_ops = call_ops; 2272 2273 dprintk("%s: Reading ino:%lu %u@%llu\n", 2274 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 2275 2276 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 2277 if (trypnfs != PNFS_NOT_ATTEMPTED) 2278 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 2279 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 2280 return trypnfs; 2281 } 2282 2283 /* Resend all requests through pnfs. */ 2284 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr) 2285 { 2286 struct nfs_pageio_descriptor pgio; 2287 2288 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2289 nfs_pageio_init_read(&pgio, hdr->inode, false, 2290 hdr->completion_ops); 2291 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); 2292 } 2293 } 2294 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 2295 2296 static void 2297 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 2298 { 2299 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 2300 struct pnfs_layout_segment *lseg = desc->pg_lseg; 2301 enum pnfs_try_status trypnfs; 2302 2303 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 2304 if (trypnfs == PNFS_TRY_AGAIN) 2305 pnfs_read_resend_pnfs(hdr); 2306 if (trypnfs == PNFS_NOT_ATTEMPTED || hdr->task.tk_status) 2307 pnfs_read_through_mds(desc, hdr); 2308 } 2309 2310 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 2311 { 2312 pnfs_put_lseg(hdr->lseg); 2313 nfs_pgio_header_free(hdr); 2314 } 2315 2316 int 2317 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 2318 { 2319 struct nfs_pgio_header *hdr; 2320 int ret; 2321 2322 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 2323 if (!hdr) { 2324 desc->pg_error = -ENOMEM; 2325 return desc->pg_error; 2326 } 2327 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 2328 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 2329 ret = nfs_generic_pgio(desc, hdr); 2330 if (!ret) 2331 pnfs_do_read(desc, hdr); 2332 return ret; 2333 } 2334 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 2335 2336 static void pnfs_clear_layoutcommitting(struct inode *inode) 2337 { 2338 unsigned long *bitlock = &NFS_I(inode)->flags; 2339 2340 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 2341 smp_mb__after_atomic(); 2342 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 2343 } 2344 2345 /* 2346 * There can be multiple RW segments. 2347 */ 2348 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 2349 { 2350 struct pnfs_layout_segment *lseg; 2351 2352 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 2353 if (lseg->pls_range.iomode == IOMODE_RW && 2354 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 2355 list_add(&lseg->pls_lc_list, listp); 2356 } 2357 } 2358 2359 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 2360 { 2361 struct pnfs_layout_segment *lseg, *tmp; 2362 2363 /* Matched by references in pnfs_set_layoutcommit */ 2364 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 2365 list_del_init(&lseg->pls_lc_list); 2366 pnfs_put_lseg(lseg); 2367 } 2368 2369 pnfs_clear_layoutcommitting(inode); 2370 } 2371 2372 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 2373 { 2374 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 2375 } 2376 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 2377 2378 void 2379 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 2380 loff_t end_pos) 2381 { 2382 struct nfs_inode *nfsi = NFS_I(inode); 2383 bool mark_as_dirty = false; 2384 2385 spin_lock(&inode->i_lock); 2386 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 2387 nfsi->layout->plh_lwb = end_pos; 2388 mark_as_dirty = true; 2389 dprintk("%s: Set layoutcommit for inode %lu ", 2390 __func__, inode->i_ino); 2391 } else if (end_pos > nfsi->layout->plh_lwb) 2392 nfsi->layout->plh_lwb = end_pos; 2393 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 2394 /* references matched in nfs4_layoutcommit_release */ 2395 pnfs_get_lseg(lseg); 2396 } 2397 spin_unlock(&inode->i_lock); 2398 dprintk("%s: lseg %p end_pos %llu\n", 2399 __func__, lseg, nfsi->layout->plh_lwb); 2400 2401 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 2402 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 2403 if (mark_as_dirty) 2404 mark_inode_dirty_sync(inode); 2405 } 2406 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 2407 2408 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 2409 { 2410 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 2411 2412 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 2413 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 2414 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 2415 } 2416 2417 /* 2418 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 2419 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 2420 * data to disk to allow the server to recover the data if it crashes. 2421 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 2422 * is off, and a COMMIT is sent to a data server, or 2423 * if WRITEs to a data server return NFS_DATA_SYNC. 2424 */ 2425 int 2426 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 2427 { 2428 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 2429 struct nfs4_layoutcommit_data *data; 2430 struct nfs_inode *nfsi = NFS_I(inode); 2431 loff_t end_pos; 2432 int status; 2433 2434 if (!pnfs_layoutcommit_outstanding(inode)) 2435 return 0; 2436 2437 dprintk("--> %s inode %lu\n", __func__, inode->i_ino); 2438 2439 status = -EAGAIN; 2440 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 2441 if (!sync) 2442 goto out; 2443 status = wait_on_bit_lock_action(&nfsi->flags, 2444 NFS_INO_LAYOUTCOMMITTING, 2445 nfs_wait_bit_killable, 2446 TASK_KILLABLE); 2447 if (status) 2448 goto out; 2449 } 2450 2451 status = -ENOMEM; 2452 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 2453 data = kzalloc(sizeof(*data), GFP_NOFS); 2454 if (!data) 2455 goto clear_layoutcommitting; 2456 2457 status = 0; 2458 spin_lock(&inode->i_lock); 2459 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 2460 goto out_unlock; 2461 2462 INIT_LIST_HEAD(&data->lseg_list); 2463 pnfs_list_write_lseg(inode, &data->lseg_list); 2464 2465 end_pos = nfsi->layout->plh_lwb; 2466 2467 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 2468 spin_unlock(&inode->i_lock); 2469 2470 data->args.inode = inode; 2471 data->cred = get_rpccred(nfsi->layout->plh_lc_cred); 2472 nfs_fattr_init(&data->fattr); 2473 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 2474 data->res.fattr = &data->fattr; 2475 if (end_pos != 0) 2476 data->args.lastbytewritten = end_pos - 1; 2477 else 2478 data->args.lastbytewritten = U64_MAX; 2479 data->res.server = NFS_SERVER(inode); 2480 2481 if (ld->prepare_layoutcommit) { 2482 status = ld->prepare_layoutcommit(&data->args); 2483 if (status) { 2484 put_rpccred(data->cred); 2485 spin_lock(&inode->i_lock); 2486 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 2487 if (end_pos > nfsi->layout->plh_lwb) 2488 nfsi->layout->plh_lwb = end_pos; 2489 goto out_unlock; 2490 } 2491 } 2492 2493 2494 status = nfs4_proc_layoutcommit(data, sync); 2495 out: 2496 if (status) 2497 mark_inode_dirty_sync(inode); 2498 dprintk("<-- %s status %d\n", __func__, status); 2499 return status; 2500 out_unlock: 2501 spin_unlock(&inode->i_lock); 2502 kfree(data); 2503 clear_layoutcommitting: 2504 pnfs_clear_layoutcommitting(inode); 2505 goto out; 2506 } 2507 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 2508 2509 int 2510 pnfs_generic_sync(struct inode *inode, bool datasync) 2511 { 2512 return pnfs_layoutcommit_inode(inode, true); 2513 } 2514 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 2515 2516 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 2517 { 2518 struct nfs4_threshold *thp; 2519 2520 thp = kzalloc(sizeof(*thp), GFP_NOFS); 2521 if (!thp) { 2522 dprintk("%s mdsthreshold allocation failed\n", __func__); 2523 return NULL; 2524 } 2525 return thp; 2526 } 2527 2528 #if IS_ENABLED(CONFIG_NFS_V4_2) 2529 int 2530 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 2531 { 2532 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 2533 struct nfs_server *server = NFS_SERVER(inode); 2534 struct nfs_inode *nfsi = NFS_I(inode); 2535 struct nfs42_layoutstat_data *data; 2536 struct pnfs_layout_hdr *hdr; 2537 int status = 0; 2538 2539 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 2540 goto out; 2541 2542 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 2543 goto out; 2544 2545 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 2546 goto out; 2547 2548 spin_lock(&inode->i_lock); 2549 if (!NFS_I(inode)->layout) { 2550 spin_unlock(&inode->i_lock); 2551 goto out_clear_layoutstats; 2552 } 2553 hdr = NFS_I(inode)->layout; 2554 pnfs_get_layout_hdr(hdr); 2555 spin_unlock(&inode->i_lock); 2556 2557 data = kzalloc(sizeof(*data), gfp_flags); 2558 if (!data) { 2559 status = -ENOMEM; 2560 goto out_put; 2561 } 2562 2563 data->args.fh = NFS_FH(inode); 2564 data->args.inode = inode; 2565 status = ld->prepare_layoutstats(&data->args); 2566 if (status) 2567 goto out_free; 2568 2569 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 2570 2571 out: 2572 dprintk("%s returns %d\n", __func__, status); 2573 return status; 2574 2575 out_free: 2576 kfree(data); 2577 out_put: 2578 pnfs_put_layout_hdr(hdr); 2579 out_clear_layoutstats: 2580 smp_mb__before_atomic(); 2581 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 2582 smp_mb__after_atomic(); 2583 goto out; 2584 } 2585 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 2586 #endif 2587 2588 unsigned int layoutstats_timer; 2589 module_param(layoutstats_timer, uint, 0644); 2590 EXPORT_SYMBOL_GPL(layoutstats_timer); 2591