1 /* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30 #include <linux/nfs_fs.h> 31 #include <linux/nfs_page.h> 32 #include <linux/module.h> 33 #include <linux/sort.h> 34 #include "internal.h" 35 #include "pnfs.h" 36 #include "iostat.h" 37 #include "nfs4trace.h" 38 #include "delegation.h" 39 #include "nfs42.h" 40 41 #define NFSDBG_FACILITY NFSDBG_PNFS 42 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 43 44 /* Locking: 45 * 46 * pnfs_spinlock: 47 * protects pnfs_modules_tbl. 48 */ 49 static DEFINE_SPINLOCK(pnfs_spinlock); 50 51 /* 52 * pnfs_modules_tbl holds all pnfs modules 53 */ 54 static LIST_HEAD(pnfs_modules_tbl); 55 56 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); 57 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 58 struct list_head *free_me, 59 const struct pnfs_layout_range *range, 60 u32 seq); 61 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 62 struct list_head *tmp_list); 63 64 /* Return the registered pnfs layout driver module matching given id */ 65 static struct pnfs_layoutdriver_type * 66 find_pnfs_driver_locked(u32 id) 67 { 68 struct pnfs_layoutdriver_type *local; 69 70 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 71 if (local->id == id) 72 goto out; 73 local = NULL; 74 out: 75 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 76 return local; 77 } 78 79 static struct pnfs_layoutdriver_type * 80 find_pnfs_driver(u32 id) 81 { 82 struct pnfs_layoutdriver_type *local; 83 84 spin_lock(&pnfs_spinlock); 85 local = find_pnfs_driver_locked(id); 86 if (local != NULL && !try_module_get(local->owner)) { 87 dprintk("%s: Could not grab reference on module\n", __func__); 88 local = NULL; 89 } 90 spin_unlock(&pnfs_spinlock); 91 return local; 92 } 93 94 void 95 unset_pnfs_layoutdriver(struct nfs_server *nfss) 96 { 97 if (nfss->pnfs_curr_ld) { 98 if (nfss->pnfs_curr_ld->clear_layoutdriver) 99 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 100 /* Decrement the MDS count. Purge the deviceid cache if zero */ 101 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 102 nfs4_deviceid_purge_client(nfss->nfs_client); 103 module_put(nfss->pnfs_curr_ld->owner); 104 } 105 nfss->pnfs_curr_ld = NULL; 106 } 107 108 /* 109 * When the server sends a list of layout types, we choose one in the order 110 * given in the list below. 111 * 112 * FIXME: should this list be configurable in some fashion? module param? 113 * mount option? something else? 114 */ 115 static const u32 ld_prefs[] = { 116 LAYOUT_SCSI, 117 LAYOUT_BLOCK_VOLUME, 118 LAYOUT_OSD2_OBJECTS, 119 LAYOUT_FLEX_FILES, 120 LAYOUT_NFSV4_1_FILES, 121 0 122 }; 123 124 static int 125 ld_cmp(const void *e1, const void *e2) 126 { 127 u32 ld1 = *((u32 *)e1); 128 u32 ld2 = *((u32 *)e2); 129 int i; 130 131 for (i = 0; ld_prefs[i] != 0; i++) { 132 if (ld1 == ld_prefs[i]) 133 return -1; 134 135 if (ld2 == ld_prefs[i]) 136 return 1; 137 } 138 return 0; 139 } 140 141 /* 142 * Try to set the server's pnfs module to the pnfs layout type specified by id. 143 * Currently only one pNFS layout driver per filesystem is supported. 144 * 145 * @ids array of layout types supported by MDS. 146 */ 147 void 148 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 149 struct nfs_fsinfo *fsinfo) 150 { 151 struct pnfs_layoutdriver_type *ld_type = NULL; 152 u32 id; 153 int i; 154 155 if (fsinfo->nlayouttypes == 0) 156 goto out_no_driver; 157 if (!(server->nfs_client->cl_exchange_flags & 158 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 159 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", 160 __func__, server->nfs_client->cl_exchange_flags); 161 goto out_no_driver; 162 } 163 164 sort(fsinfo->layouttype, fsinfo->nlayouttypes, 165 sizeof(*fsinfo->layouttype), ld_cmp, NULL); 166 167 for (i = 0; i < fsinfo->nlayouttypes; i++) { 168 id = fsinfo->layouttype[i]; 169 ld_type = find_pnfs_driver(id); 170 if (!ld_type) { 171 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, 172 id); 173 ld_type = find_pnfs_driver(id); 174 } 175 if (ld_type) 176 break; 177 } 178 179 if (!ld_type) { 180 dprintk("%s: No pNFS module found!\n", __func__); 181 goto out_no_driver; 182 } 183 184 server->pnfs_curr_ld = ld_type; 185 if (ld_type->set_layoutdriver 186 && ld_type->set_layoutdriver(server, mntfh)) { 187 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 188 "driver %u.\n", __func__, id); 189 module_put(ld_type->owner); 190 goto out_no_driver; 191 } 192 /* Bump the MDS count */ 193 atomic_inc(&server->nfs_client->cl_mds_count); 194 195 dprintk("%s: pNFS module for %u set\n", __func__, id); 196 return; 197 198 out_no_driver: 199 dprintk("%s: Using NFSv4 I/O\n", __func__); 200 server->pnfs_curr_ld = NULL; 201 } 202 203 int 204 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 205 { 206 int status = -EINVAL; 207 struct pnfs_layoutdriver_type *tmp; 208 209 if (ld_type->id == 0) { 210 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 211 return status; 212 } 213 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 214 printk(KERN_ERR "NFS: %s Layout driver must provide " 215 "alloc_lseg and free_lseg.\n", __func__); 216 return status; 217 } 218 219 spin_lock(&pnfs_spinlock); 220 tmp = find_pnfs_driver_locked(ld_type->id); 221 if (!tmp) { 222 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 223 status = 0; 224 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 225 ld_type->name); 226 } else { 227 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 228 __func__, ld_type->id); 229 } 230 spin_unlock(&pnfs_spinlock); 231 232 return status; 233 } 234 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 235 236 void 237 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 238 { 239 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 240 spin_lock(&pnfs_spinlock); 241 list_del(&ld_type->pnfs_tblid); 242 spin_unlock(&pnfs_spinlock); 243 } 244 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 245 246 /* 247 * pNFS client layout cache 248 */ 249 250 /* Need to hold i_lock if caller does not already hold reference */ 251 void 252 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 253 { 254 atomic_inc(&lo->plh_refcount); 255 } 256 257 static struct pnfs_layout_hdr * 258 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 259 { 260 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 261 return ld->alloc_layout_hdr(ino, gfp_flags); 262 } 263 264 static void 265 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 266 { 267 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 268 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 269 270 if (!list_empty(&lo->plh_layouts)) { 271 struct nfs_client *clp = server->nfs_client; 272 273 spin_lock(&clp->cl_lock); 274 list_del_init(&lo->plh_layouts); 275 spin_unlock(&clp->cl_lock); 276 } 277 put_rpccred(lo->plh_lc_cred); 278 return ld->free_layout_hdr(lo); 279 } 280 281 static void 282 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 283 { 284 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 285 dprintk("%s: freeing layout cache %p\n", __func__, lo); 286 nfsi->layout = NULL; 287 /* Reset MDS Threshold I/O counters */ 288 nfsi->write_io = 0; 289 nfsi->read_io = 0; 290 } 291 292 void 293 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 294 { 295 struct inode *inode = lo->plh_inode; 296 297 pnfs_layoutreturn_before_put_layout_hdr(lo); 298 299 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 300 if (!list_empty(&lo->plh_segs)) 301 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 302 pnfs_detach_layout_hdr(lo); 303 spin_unlock(&inode->i_lock); 304 pnfs_free_layout_hdr(lo); 305 } 306 } 307 308 static void 309 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, 310 u32 seq) 311 { 312 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) 313 iomode = IOMODE_ANY; 314 lo->plh_return_iomode = iomode; 315 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 316 if (seq != 0) { 317 WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq); 318 lo->plh_return_seq = seq; 319 } 320 } 321 322 static void 323 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) 324 { 325 struct pnfs_layout_segment *lseg; 326 lo->plh_return_iomode = 0; 327 lo->plh_return_seq = 0; 328 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); 329 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 330 if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 331 continue; 332 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 333 } 334 } 335 336 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 337 { 338 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 339 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); 340 smp_mb__after_atomic(); 341 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 342 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 343 } 344 345 static void 346 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg, 347 struct list_head *free_me) 348 { 349 clear_bit(NFS_LSEG_ROC, &lseg->pls_flags); 350 clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 351 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 352 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 353 if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 354 pnfs_lseg_dec_and_remove_zero(lseg, free_me); 355 } 356 357 /* 358 * Mark a pnfs_layout_hdr and all associated layout segments as invalid 359 * 360 * In order to continue using the pnfs_layout_hdr, a full recovery 361 * is required. 362 * Note that caller must hold inode->i_lock. 363 */ 364 int 365 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, 366 struct list_head *lseg_list) 367 { 368 struct pnfs_layout_range range = { 369 .iomode = IOMODE_ANY, 370 .offset = 0, 371 .length = NFS4_MAX_UINT64, 372 }; 373 struct pnfs_layout_segment *lseg, *next; 374 375 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 376 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 377 pnfs_clear_lseg_state(lseg, lseg_list); 378 pnfs_clear_layoutreturn_info(lo); 379 pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); 380 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && 381 !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 382 pnfs_clear_layoutreturn_waitbit(lo); 383 return !list_empty(&lo->plh_segs); 384 } 385 386 static int 387 pnfs_iomode_to_fail_bit(u32 iomode) 388 { 389 return iomode == IOMODE_RW ? 390 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 391 } 392 393 static void 394 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 395 { 396 lo->plh_retry_timestamp = jiffies; 397 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 398 atomic_inc(&lo->plh_refcount); 399 } 400 401 static void 402 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 403 { 404 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 405 atomic_dec(&lo->plh_refcount); 406 } 407 408 static void 409 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 410 { 411 struct inode *inode = lo->plh_inode; 412 struct pnfs_layout_range range = { 413 .iomode = iomode, 414 .offset = 0, 415 .length = NFS4_MAX_UINT64, 416 }; 417 LIST_HEAD(head); 418 419 spin_lock(&inode->i_lock); 420 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 421 pnfs_mark_matching_lsegs_invalid(lo, &head, &range, 0); 422 spin_unlock(&inode->i_lock); 423 pnfs_free_lseg_list(&head); 424 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 425 iomode == IOMODE_RW ? "RW" : "READ"); 426 } 427 428 static bool 429 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 430 { 431 unsigned long start, end; 432 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 433 434 if (test_bit(fail_bit, &lo->plh_flags) == 0) 435 return false; 436 end = jiffies; 437 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 438 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 439 /* It is time to retry the failed layoutgets */ 440 pnfs_layout_clear_fail_bit(lo, fail_bit); 441 return false; 442 } 443 return true; 444 } 445 446 static void 447 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, 448 const struct pnfs_layout_range *range, 449 const nfs4_stateid *stateid) 450 { 451 INIT_LIST_HEAD(&lseg->pls_list); 452 INIT_LIST_HEAD(&lseg->pls_lc_list); 453 atomic_set(&lseg->pls_refcount, 1); 454 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 455 lseg->pls_layout = lo; 456 lseg->pls_range = *range; 457 lseg->pls_seq = be32_to_cpu(stateid->seqid); 458 } 459 460 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 461 { 462 if (lseg != NULL) { 463 struct inode *inode = lseg->pls_layout->plh_inode; 464 NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg); 465 } 466 } 467 468 static void 469 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 470 struct pnfs_layout_segment *lseg) 471 { 472 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 473 list_del_init(&lseg->pls_list); 474 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 475 atomic_dec(&lo->plh_refcount); 476 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 477 return; 478 if (list_empty(&lo->plh_segs) && 479 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && 480 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 481 if (atomic_read(&lo->plh_outstanding) == 0) 482 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 483 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 484 } 485 } 486 487 static bool 488 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, 489 struct pnfs_layout_segment *lseg) 490 { 491 if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 492 pnfs_layout_is_valid(lo)) { 493 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 494 list_move_tail(&lseg->pls_list, &lo->plh_return_segs); 495 return true; 496 } 497 return false; 498 } 499 500 void 501 pnfs_put_lseg(struct pnfs_layout_segment *lseg) 502 { 503 struct pnfs_layout_hdr *lo; 504 struct inode *inode; 505 506 if (!lseg) 507 return; 508 509 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 510 atomic_read(&lseg->pls_refcount), 511 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 512 513 lo = lseg->pls_layout; 514 inode = lo->plh_inode; 515 516 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 517 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 518 spin_unlock(&inode->i_lock); 519 return; 520 } 521 pnfs_get_layout_hdr(lo); 522 pnfs_layout_remove_lseg(lo, lseg); 523 if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) 524 lseg = NULL; 525 spin_unlock(&inode->i_lock); 526 pnfs_free_lseg(lseg); 527 pnfs_put_layout_hdr(lo); 528 } 529 } 530 EXPORT_SYMBOL_GPL(pnfs_put_lseg); 531 532 static void pnfs_free_lseg_async_work(struct work_struct *work) 533 { 534 struct pnfs_layout_segment *lseg; 535 struct pnfs_layout_hdr *lo; 536 537 lseg = container_of(work, struct pnfs_layout_segment, pls_work); 538 lo = lseg->pls_layout; 539 540 pnfs_free_lseg(lseg); 541 pnfs_put_layout_hdr(lo); 542 } 543 544 static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg) 545 { 546 INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work); 547 schedule_work(&lseg->pls_work); 548 } 549 550 void 551 pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg) 552 { 553 if (!lseg) 554 return; 555 556 assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock); 557 558 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 559 atomic_read(&lseg->pls_refcount), 560 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 561 if (atomic_dec_and_test(&lseg->pls_refcount)) { 562 struct pnfs_layout_hdr *lo = lseg->pls_layout; 563 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 564 return; 565 pnfs_layout_remove_lseg(lo, lseg); 566 if (!pnfs_cache_lseg_for_layoutreturn(lo, lseg)) { 567 pnfs_get_layout_hdr(lo); 568 pnfs_free_lseg_async(lseg); 569 } 570 } 571 } 572 573 /* 574 * is l2 fully contained in l1? 575 * start1 end1 576 * [----------------------------------) 577 * start2 end2 578 * [----------------) 579 */ 580 static bool 581 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 582 const struct pnfs_layout_range *l2) 583 { 584 u64 start1 = l1->offset; 585 u64 end1 = pnfs_end_offset(start1, l1->length); 586 u64 start2 = l2->offset; 587 u64 end2 = pnfs_end_offset(start2, l2->length); 588 589 return (start1 <= start2) && (end1 >= end2); 590 } 591 592 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 593 struct list_head *tmp_list) 594 { 595 if (!atomic_dec_and_test(&lseg->pls_refcount)) 596 return false; 597 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 598 list_add(&lseg->pls_list, tmp_list); 599 return true; 600 } 601 602 /* Returns 1 if lseg is removed from list, 0 otherwise */ 603 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 604 struct list_head *tmp_list) 605 { 606 int rv = 0; 607 608 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 609 /* Remove the reference keeping the lseg in the 610 * list. It will now be removed when all 611 * outstanding io is finished. 612 */ 613 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 614 atomic_read(&lseg->pls_refcount)); 615 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 616 rv = 1; 617 } 618 return rv; 619 } 620 621 /* 622 * Compare 2 layout stateid sequence ids, to see which is newer, 623 * taking into account wraparound issues. 624 */ 625 static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 626 { 627 return (s32)(s1 - s2) > 0; 628 } 629 630 static bool 631 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, 632 const struct pnfs_layout_range *recall_range) 633 { 634 return (recall_range->iomode == IOMODE_ANY || 635 lseg_range->iomode == recall_range->iomode) && 636 pnfs_lseg_range_intersecting(lseg_range, recall_range); 637 } 638 639 static bool 640 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, 641 const struct pnfs_layout_range *recall_range, 642 u32 seq) 643 { 644 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) 645 return false; 646 if (recall_range == NULL) 647 return true; 648 return pnfs_should_free_range(&lseg->pls_range, recall_range); 649 } 650 651 /** 652 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later 653 * @lo: layout header containing the lsegs 654 * @tmp_list: list head where doomed lsegs should go 655 * @recall_range: optional recall range argument to match (may be NULL) 656 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) 657 * 658 * Walk the list of lsegs in the layout header, and tear down any that should 659 * be destroyed. If "recall_range" is specified then the segment must match 660 * that range. If "seq" is non-zero, then only match segments that were handed 661 * out at or before that sequence. 662 * 663 * Returns number of matching invalid lsegs remaining in list after scanning 664 * it and purging them. 665 */ 666 int 667 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 668 struct list_head *tmp_list, 669 const struct pnfs_layout_range *recall_range, 670 u32 seq) 671 { 672 struct pnfs_layout_segment *lseg, *next; 673 int remaining = 0; 674 675 dprintk("%s:Begin lo %p\n", __func__, lo); 676 677 if (list_empty(&lo->plh_segs)) 678 return 0; 679 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 680 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { 681 dprintk("%s: freeing lseg %p iomode %d seq %u" 682 "offset %llu length %llu\n", __func__, 683 lseg, lseg->pls_range.iomode, lseg->pls_seq, 684 lseg->pls_range.offset, lseg->pls_range.length); 685 if (!mark_lseg_invalid(lseg, tmp_list)) 686 remaining++; 687 } 688 dprintk("%s:Return %i\n", __func__, remaining); 689 return remaining; 690 } 691 692 static void 693 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, 694 struct list_head *free_me, 695 const struct pnfs_layout_range *range, 696 u32 seq) 697 { 698 struct pnfs_layout_segment *lseg, *next; 699 700 list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { 701 if (pnfs_match_lseg_recall(lseg, range, seq)) 702 list_move_tail(&lseg->pls_list, free_me); 703 } 704 } 705 706 /* note free_me must contain lsegs from a single layout_hdr */ 707 void 708 pnfs_free_lseg_list(struct list_head *free_me) 709 { 710 struct pnfs_layout_segment *lseg, *tmp; 711 712 if (list_empty(free_me)) 713 return; 714 715 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 716 list_del(&lseg->pls_list); 717 pnfs_free_lseg(lseg); 718 } 719 } 720 721 void 722 pnfs_destroy_layout(struct nfs_inode *nfsi) 723 { 724 struct pnfs_layout_hdr *lo; 725 LIST_HEAD(tmp_list); 726 727 spin_lock(&nfsi->vfs_inode.i_lock); 728 lo = nfsi->layout; 729 if (lo) { 730 pnfs_get_layout_hdr(lo); 731 pnfs_mark_layout_stateid_invalid(lo, &tmp_list); 732 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 733 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 734 spin_unlock(&nfsi->vfs_inode.i_lock); 735 pnfs_free_lseg_list(&tmp_list); 736 nfs_commit_inode(&nfsi->vfs_inode, 0); 737 pnfs_put_layout_hdr(lo); 738 } else 739 spin_unlock(&nfsi->vfs_inode.i_lock); 740 } 741 EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 742 743 static bool 744 pnfs_layout_add_bulk_destroy_list(struct inode *inode, 745 struct list_head *layout_list) 746 { 747 struct pnfs_layout_hdr *lo; 748 bool ret = false; 749 750 spin_lock(&inode->i_lock); 751 lo = NFS_I(inode)->layout; 752 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 753 pnfs_get_layout_hdr(lo); 754 list_add(&lo->plh_bulk_destroy, layout_list); 755 ret = true; 756 } 757 spin_unlock(&inode->i_lock); 758 return ret; 759 } 760 761 /* Caller must hold rcu_read_lock and clp->cl_lock */ 762 static int 763 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 764 struct nfs_server *server, 765 struct list_head *layout_list) 766 { 767 struct pnfs_layout_hdr *lo, *next; 768 struct inode *inode; 769 770 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 771 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) 772 continue; 773 inode = igrab(lo->plh_inode); 774 if (inode == NULL) 775 continue; 776 list_del_init(&lo->plh_layouts); 777 if (pnfs_layout_add_bulk_destroy_list(inode, layout_list)) 778 continue; 779 rcu_read_unlock(); 780 spin_unlock(&clp->cl_lock); 781 iput(inode); 782 spin_lock(&clp->cl_lock); 783 rcu_read_lock(); 784 return -EAGAIN; 785 } 786 return 0; 787 } 788 789 static int 790 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 791 bool is_bulk_recall) 792 { 793 struct pnfs_layout_hdr *lo; 794 struct inode *inode; 795 LIST_HEAD(lseg_list); 796 int ret = 0; 797 798 while (!list_empty(layout_list)) { 799 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 800 plh_bulk_destroy); 801 dprintk("%s freeing layout for inode %lu\n", __func__, 802 lo->plh_inode->i_ino); 803 inode = lo->plh_inode; 804 805 pnfs_layoutcommit_inode(inode, false); 806 807 spin_lock(&inode->i_lock); 808 list_del_init(&lo->plh_bulk_destroy); 809 if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { 810 if (is_bulk_recall) 811 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 812 ret = -EAGAIN; 813 } 814 spin_unlock(&inode->i_lock); 815 pnfs_free_lseg_list(&lseg_list); 816 /* Free all lsegs that are attached to commit buckets */ 817 nfs_commit_inode(inode, 0); 818 pnfs_put_layout_hdr(lo); 819 iput(inode); 820 } 821 return ret; 822 } 823 824 int 825 pnfs_destroy_layouts_byfsid(struct nfs_client *clp, 826 struct nfs_fsid *fsid, 827 bool is_recall) 828 { 829 struct nfs_server *server; 830 LIST_HEAD(layout_list); 831 832 spin_lock(&clp->cl_lock); 833 rcu_read_lock(); 834 restart: 835 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 836 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 837 continue; 838 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 839 server, 840 &layout_list) != 0) 841 goto restart; 842 } 843 rcu_read_unlock(); 844 spin_unlock(&clp->cl_lock); 845 846 if (list_empty(&layout_list)) 847 return 0; 848 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall); 849 } 850 851 int 852 pnfs_destroy_layouts_byclid(struct nfs_client *clp, 853 bool is_recall) 854 { 855 struct nfs_server *server; 856 LIST_HEAD(layout_list); 857 858 spin_lock(&clp->cl_lock); 859 rcu_read_lock(); 860 restart: 861 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 862 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 863 server, 864 &layout_list) != 0) 865 goto restart; 866 } 867 rcu_read_unlock(); 868 spin_unlock(&clp->cl_lock); 869 870 if (list_empty(&layout_list)) 871 return 0; 872 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall); 873 } 874 875 /* 876 * Called by the state manger to remove all layouts established under an 877 * expired lease. 878 */ 879 void 880 pnfs_destroy_all_layouts(struct nfs_client *clp) 881 { 882 nfs4_deviceid_mark_client_invalid(clp); 883 nfs4_deviceid_purge_client(clp); 884 885 pnfs_destroy_layouts_byclid(clp, false); 886 } 887 888 /* update lo->plh_stateid with new if is more recent */ 889 void 890 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 891 bool update_barrier) 892 { 893 u32 oldseq, newseq, new_barrier = 0; 894 895 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 896 newseq = be32_to_cpu(new->seqid); 897 898 if (!pnfs_layout_is_valid(lo)) { 899 nfs4_stateid_copy(&lo->plh_stateid, new); 900 lo->plh_barrier = newseq; 901 pnfs_clear_layoutreturn_info(lo); 902 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 903 return; 904 } 905 if (pnfs_seqid_is_newer(newseq, oldseq)) { 906 nfs4_stateid_copy(&lo->plh_stateid, new); 907 /* 908 * Because of wraparound, we want to keep the barrier 909 * "close" to the current seqids. 910 */ 911 new_barrier = newseq - atomic_read(&lo->plh_outstanding); 912 } 913 if (update_barrier) 914 new_barrier = be32_to_cpu(new->seqid); 915 else if (new_barrier == 0) 916 return; 917 if (pnfs_seqid_is_newer(new_barrier, lo->plh_barrier)) 918 lo->plh_barrier = new_barrier; 919 } 920 921 static bool 922 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 923 const nfs4_stateid *stateid) 924 { 925 u32 seqid = be32_to_cpu(stateid->seqid); 926 927 return !pnfs_seqid_is_newer(seqid, lo->plh_barrier); 928 } 929 930 /* lget is set to 1 if called from inside send_layoutget call chain */ 931 static bool 932 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 933 { 934 return lo->plh_block_lgets || 935 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 936 } 937 938 /* 939 * Get layout from server. 940 * for now, assume that whole file layouts are requested. 941 * arg->offset: 0 942 * arg->length: all ones 943 */ 944 static struct pnfs_layout_segment * 945 send_layoutget(struct pnfs_layout_hdr *lo, 946 struct nfs_open_context *ctx, 947 nfs4_stateid *stateid, 948 const struct pnfs_layout_range *range, 949 long *timeout, gfp_t gfp_flags) 950 { 951 struct inode *ino = lo->plh_inode; 952 struct nfs_server *server = NFS_SERVER(ino); 953 struct nfs4_layoutget *lgp; 954 loff_t i_size; 955 956 dprintk("--> %s\n", __func__); 957 958 /* 959 * Synchronously retrieve layout information from server and 960 * store in lseg. If we race with a concurrent seqid morphing 961 * op, then re-send the LAYOUTGET. 962 */ 963 lgp = kzalloc(sizeof(*lgp), gfp_flags); 964 if (lgp == NULL) 965 return ERR_PTR(-ENOMEM); 966 967 i_size = i_size_read(ino); 968 969 lgp->args.minlength = PAGE_SIZE; 970 if (lgp->args.minlength > range->length) 971 lgp->args.minlength = range->length; 972 if (range->iomode == IOMODE_READ) { 973 if (range->offset >= i_size) 974 lgp->args.minlength = 0; 975 else if (i_size - range->offset < lgp->args.minlength) 976 lgp->args.minlength = i_size - range->offset; 977 } 978 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 979 pnfs_copy_range(&lgp->args.range, range); 980 lgp->args.type = server->pnfs_curr_ld->id; 981 lgp->args.inode = ino; 982 lgp->args.ctx = get_nfs_open_context(ctx); 983 nfs4_stateid_copy(&lgp->args.stateid, stateid); 984 lgp->gfp_flags = gfp_flags; 985 lgp->cred = lo->plh_lc_cred; 986 987 return nfs4_proc_layoutget(lgp, timeout, gfp_flags); 988 } 989 990 static void pnfs_clear_layoutcommit(struct inode *inode, 991 struct list_head *head) 992 { 993 struct nfs_inode *nfsi = NFS_I(inode); 994 struct pnfs_layout_segment *lseg, *tmp; 995 996 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 997 return; 998 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 999 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 1000 continue; 1001 pnfs_lseg_dec_and_remove_zero(lseg, head); 1002 } 1003 } 1004 1005 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, 1006 const nfs4_stateid *arg_stateid, 1007 const struct pnfs_layout_range *range, 1008 const nfs4_stateid *stateid) 1009 { 1010 struct inode *inode = lo->plh_inode; 1011 LIST_HEAD(freeme); 1012 1013 spin_lock(&inode->i_lock); 1014 if (!pnfs_layout_is_valid(lo) || !arg_stateid || 1015 !nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) 1016 goto out_unlock; 1017 if (stateid) { 1018 u32 seq = be32_to_cpu(arg_stateid->seqid); 1019 1020 pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); 1021 pnfs_free_returned_lsegs(lo, &freeme, range, seq); 1022 pnfs_set_layout_stateid(lo, stateid, true); 1023 } else 1024 pnfs_mark_layout_stateid_invalid(lo, &freeme); 1025 out_unlock: 1026 pnfs_clear_layoutreturn_waitbit(lo); 1027 spin_unlock(&inode->i_lock); 1028 pnfs_free_lseg_list(&freeme); 1029 1030 } 1031 1032 static bool 1033 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, 1034 nfs4_stateid *stateid, 1035 enum pnfs_iomode *iomode) 1036 { 1037 /* Serialise LAYOUTGET/LAYOUTRETURN */ 1038 if (atomic_read(&lo->plh_outstanding) != 0) 1039 return false; 1040 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) 1041 return false; 1042 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); 1043 pnfs_get_layout_hdr(lo); 1044 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 1045 if (stateid != NULL) { 1046 nfs4_stateid_copy(stateid, &lo->plh_stateid); 1047 if (lo->plh_return_seq != 0) 1048 stateid->seqid = cpu_to_be32(lo->plh_return_seq); 1049 } 1050 if (iomode != NULL) 1051 *iomode = lo->plh_return_iomode; 1052 pnfs_clear_layoutreturn_info(lo); 1053 return true; 1054 } 1055 if (stateid != NULL) 1056 nfs4_stateid_copy(stateid, &lo->plh_stateid); 1057 if (iomode != NULL) 1058 *iomode = IOMODE_ANY; 1059 return true; 1060 } 1061 1062 static void 1063 pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, 1064 struct pnfs_layout_hdr *lo, 1065 const nfs4_stateid *stateid, 1066 enum pnfs_iomode iomode) 1067 { 1068 struct inode *inode = lo->plh_inode; 1069 1070 args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id; 1071 args->inode = inode; 1072 args->range.iomode = iomode; 1073 args->range.offset = 0; 1074 args->range.length = NFS4_MAX_UINT64; 1075 args->layout = lo; 1076 nfs4_stateid_copy(&args->stateid, stateid); 1077 } 1078 1079 static int 1080 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid, 1081 enum pnfs_iomode iomode, bool sync) 1082 { 1083 struct inode *ino = lo->plh_inode; 1084 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1085 struct nfs4_layoutreturn *lrp; 1086 int status = 0; 1087 1088 lrp = kzalloc(sizeof(*lrp), GFP_NOFS); 1089 if (unlikely(lrp == NULL)) { 1090 status = -ENOMEM; 1091 spin_lock(&ino->i_lock); 1092 pnfs_clear_layoutreturn_waitbit(lo); 1093 spin_unlock(&ino->i_lock); 1094 pnfs_put_layout_hdr(lo); 1095 goto out; 1096 } 1097 1098 pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); 1099 lrp->args.ld_private = &lrp->ld_private; 1100 lrp->clp = NFS_SERVER(ino)->nfs_client; 1101 lrp->cred = lo->plh_lc_cred; 1102 if (ld->prepare_layoutreturn) 1103 ld->prepare_layoutreturn(&lrp->args); 1104 1105 status = nfs4_proc_layoutreturn(lrp, sync); 1106 out: 1107 dprintk("<-- %s status: %d\n", __func__, status); 1108 return status; 1109 } 1110 1111 /* Return true if layoutreturn is needed */ 1112 static bool 1113 pnfs_layout_need_return(struct pnfs_layout_hdr *lo) 1114 { 1115 struct pnfs_layout_segment *s; 1116 1117 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1118 return false; 1119 1120 /* Defer layoutreturn until all lsegs are done */ 1121 list_for_each_entry(s, &lo->plh_segs, pls_list) { 1122 if (test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags)) 1123 return false; 1124 } 1125 1126 return true; 1127 } 1128 1129 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) 1130 { 1131 struct inode *inode= lo->plh_inode; 1132 1133 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1134 return; 1135 spin_lock(&inode->i_lock); 1136 if (pnfs_layout_need_return(lo)) { 1137 nfs4_stateid stateid; 1138 enum pnfs_iomode iomode; 1139 bool send; 1140 1141 send = pnfs_prepare_layoutreturn(lo, &stateid, &iomode); 1142 spin_unlock(&inode->i_lock); 1143 if (send) { 1144 /* Send an async layoutreturn so we dont deadlock */ 1145 pnfs_send_layoutreturn(lo, &stateid, iomode, false); 1146 } 1147 } else 1148 spin_unlock(&inode->i_lock); 1149 } 1150 1151 /* 1152 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 1153 * when the layout segment list is empty. 1154 * 1155 * Note that a pnfs_layout_hdr can exist with an empty layout segment 1156 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 1157 * deviceid is marked invalid. 1158 */ 1159 int 1160 _pnfs_return_layout(struct inode *ino) 1161 { 1162 struct pnfs_layout_hdr *lo = NULL; 1163 struct nfs_inode *nfsi = NFS_I(ino); 1164 LIST_HEAD(tmp_list); 1165 nfs4_stateid stateid; 1166 int status = 0; 1167 bool send; 1168 1169 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino); 1170 1171 spin_lock(&ino->i_lock); 1172 lo = nfsi->layout; 1173 if (!lo) { 1174 spin_unlock(&ino->i_lock); 1175 dprintk("NFS: %s no layout to return\n", __func__); 1176 goto out; 1177 } 1178 /* Reference matched in nfs4_layoutreturn_release */ 1179 pnfs_get_layout_hdr(lo); 1180 /* Is there an outstanding layoutreturn ? */ 1181 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1182 spin_unlock(&ino->i_lock); 1183 if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1184 TASK_UNINTERRUPTIBLE)) 1185 goto out_put_layout_hdr; 1186 spin_lock(&ino->i_lock); 1187 } 1188 pnfs_clear_layoutcommit(ino, &tmp_list); 1189 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0); 1190 1191 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) { 1192 struct pnfs_layout_range range = { 1193 .iomode = IOMODE_ANY, 1194 .offset = 0, 1195 .length = NFS4_MAX_UINT64, 1196 }; 1197 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1198 } 1199 1200 /* Don't send a LAYOUTRETURN if list was initially empty */ 1201 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { 1202 spin_unlock(&ino->i_lock); 1203 dprintk("NFS: %s no layout segments to return\n", __func__); 1204 goto out_put_layout_hdr; 1205 } 1206 1207 send = pnfs_prepare_layoutreturn(lo, &stateid, NULL); 1208 spin_unlock(&ino->i_lock); 1209 if (send) 1210 status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true); 1211 out_put_layout_hdr: 1212 pnfs_free_lseg_list(&tmp_list); 1213 pnfs_put_layout_hdr(lo); 1214 out: 1215 dprintk("<-- %s status: %d\n", __func__, status); 1216 return status; 1217 } 1218 1219 int 1220 pnfs_commit_and_return_layout(struct inode *inode) 1221 { 1222 struct pnfs_layout_hdr *lo; 1223 int ret; 1224 1225 spin_lock(&inode->i_lock); 1226 lo = NFS_I(inode)->layout; 1227 if (lo == NULL) { 1228 spin_unlock(&inode->i_lock); 1229 return 0; 1230 } 1231 pnfs_get_layout_hdr(lo); 1232 /* Block new layoutgets and read/write to ds */ 1233 lo->plh_block_lgets++; 1234 spin_unlock(&inode->i_lock); 1235 filemap_fdatawait(inode->i_mapping); 1236 ret = pnfs_layoutcommit_inode(inode, true); 1237 if (ret == 0) 1238 ret = _pnfs_return_layout(inode); 1239 spin_lock(&inode->i_lock); 1240 lo->plh_block_lgets--; 1241 spin_unlock(&inode->i_lock); 1242 pnfs_put_layout_hdr(lo); 1243 return ret; 1244 } 1245 1246 bool pnfs_roc(struct inode *ino, 1247 struct nfs4_layoutreturn_args *args, 1248 struct nfs4_layoutreturn_res *res, 1249 const struct rpc_cred *cred) 1250 { 1251 struct nfs_inode *nfsi = NFS_I(ino); 1252 struct nfs_open_context *ctx; 1253 struct nfs4_state *state; 1254 struct pnfs_layout_hdr *lo; 1255 struct pnfs_layout_segment *lseg, *next; 1256 nfs4_stateid stateid; 1257 enum pnfs_iomode iomode = 0; 1258 bool layoutreturn = false, roc = false; 1259 bool skip_read = false; 1260 1261 if (!nfs_have_layout(ino)) 1262 return false; 1263 retry: 1264 spin_lock(&ino->i_lock); 1265 lo = nfsi->layout; 1266 if (!lo || !pnfs_layout_is_valid(lo) || 1267 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) 1268 goto out_noroc; 1269 if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { 1270 pnfs_get_layout_hdr(lo); 1271 spin_unlock(&ino->i_lock); 1272 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, 1273 TASK_UNINTERRUPTIBLE); 1274 pnfs_put_layout_hdr(lo); 1275 goto retry; 1276 } 1277 1278 /* no roc if we hold a delegation */ 1279 if (nfs4_check_delegation(ino, FMODE_READ)) { 1280 if (nfs4_check_delegation(ino, FMODE_WRITE)) 1281 goto out_noroc; 1282 skip_read = true; 1283 } 1284 1285 list_for_each_entry(ctx, &nfsi->open_files, list) { 1286 state = ctx->state; 1287 if (state == NULL) 1288 continue; 1289 /* Don't return layout if there is open file state */ 1290 if (state->state & FMODE_WRITE) 1291 goto out_noroc; 1292 if (state->state & FMODE_READ) 1293 skip_read = true; 1294 } 1295 1296 1297 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { 1298 if (skip_read && lseg->pls_range.iomode == IOMODE_READ) 1299 continue; 1300 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1301 if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags)) 1302 continue; 1303 /* 1304 * Note: mark lseg for return so pnfs_layout_remove_lseg 1305 * doesn't invalidate the layout for us. 1306 */ 1307 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1308 if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) 1309 continue; 1310 pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); 1311 } 1312 1313 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) 1314 goto out_noroc; 1315 1316 /* ROC in two conditions: 1317 * 1. there are ROC lsegs 1318 * 2. we don't send layoutreturn 1319 */ 1320 /* lo ref dropped in pnfs_roc_release() */ 1321 layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &iomode); 1322 /* If the creds don't match, we can't compound the layoutreturn */ 1323 if (!layoutreturn || cred != lo->plh_lc_cred) 1324 goto out_noroc; 1325 1326 roc = layoutreturn; 1327 pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); 1328 res->lrs_present = 0; 1329 layoutreturn = false; 1330 1331 out_noroc: 1332 spin_unlock(&ino->i_lock); 1333 pnfs_layoutcommit_inode(ino, true); 1334 if (roc) { 1335 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 1336 if (ld->prepare_layoutreturn) 1337 ld->prepare_layoutreturn(args); 1338 return true; 1339 } 1340 if (layoutreturn) 1341 pnfs_send_layoutreturn(lo, &stateid, iomode, true); 1342 return false; 1343 } 1344 1345 void pnfs_roc_release(struct nfs4_layoutreturn_args *args, 1346 struct nfs4_layoutreturn_res *res, 1347 int ret) 1348 { 1349 struct pnfs_layout_hdr *lo = args->layout; 1350 const nfs4_stateid *arg_stateid = NULL; 1351 const nfs4_stateid *res_stateid = NULL; 1352 struct nfs4_xdr_opaque_data *ld_private = args->ld_private; 1353 1354 if (ret == 0) { 1355 arg_stateid = &args->stateid; 1356 if (res->lrs_present) 1357 res_stateid = &res->stateid; 1358 } 1359 pnfs_layoutreturn_free_lsegs(lo, arg_stateid, &args->range, 1360 res_stateid); 1361 if (ld_private && ld_private->ops && ld_private->ops->free) 1362 ld_private->ops->free(ld_private); 1363 pnfs_put_layout_hdr(lo); 1364 trace_nfs4_layoutreturn_on_close(args->inode, 0); 1365 } 1366 1367 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1368 { 1369 struct nfs_inode *nfsi = NFS_I(ino); 1370 struct pnfs_layout_hdr *lo; 1371 bool sleep = false; 1372 1373 /* we might not have grabbed lo reference. so need to check under 1374 * i_lock */ 1375 spin_lock(&ino->i_lock); 1376 lo = nfsi->layout; 1377 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1378 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1379 sleep = true; 1380 } 1381 spin_unlock(&ino->i_lock); 1382 return sleep; 1383 } 1384 1385 /* 1386 * Compare two layout segments for sorting into layout cache. 1387 * We want to preferentially return RW over RO layouts, so ensure those 1388 * are seen first. 1389 */ 1390 static s64 1391 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1392 const struct pnfs_layout_range *l2) 1393 { 1394 s64 d; 1395 1396 /* high offset > low offset */ 1397 d = l1->offset - l2->offset; 1398 if (d) 1399 return d; 1400 1401 /* short length > long length */ 1402 d = l2->length - l1->length; 1403 if (d) 1404 return d; 1405 1406 /* read > read/write */ 1407 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1408 } 1409 1410 static bool 1411 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1412 const struct pnfs_layout_range *l2) 1413 { 1414 return pnfs_lseg_range_cmp(l1, l2) > 0; 1415 } 1416 1417 static bool 1418 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1419 struct pnfs_layout_segment *old) 1420 { 1421 return false; 1422 } 1423 1424 void 1425 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1426 struct pnfs_layout_segment *lseg, 1427 bool (*is_after)(const struct pnfs_layout_range *, 1428 const struct pnfs_layout_range *), 1429 bool (*do_merge)(struct pnfs_layout_segment *, 1430 struct pnfs_layout_segment *), 1431 struct list_head *free_me) 1432 { 1433 struct pnfs_layout_segment *lp, *tmp; 1434 1435 dprintk("%s:Begin\n", __func__); 1436 1437 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1438 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1439 continue; 1440 if (do_merge(lseg, lp)) { 1441 mark_lseg_invalid(lp, free_me); 1442 continue; 1443 } 1444 if (is_after(&lseg->pls_range, &lp->pls_range)) 1445 continue; 1446 list_add_tail(&lseg->pls_list, &lp->pls_list); 1447 dprintk("%s: inserted lseg %p " 1448 "iomode %d offset %llu length %llu before " 1449 "lp %p iomode %d offset %llu length %llu\n", 1450 __func__, lseg, lseg->pls_range.iomode, 1451 lseg->pls_range.offset, lseg->pls_range.length, 1452 lp, lp->pls_range.iomode, lp->pls_range.offset, 1453 lp->pls_range.length); 1454 goto out; 1455 } 1456 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1457 dprintk("%s: inserted lseg %p " 1458 "iomode %d offset %llu length %llu at tail\n", 1459 __func__, lseg, lseg->pls_range.iomode, 1460 lseg->pls_range.offset, lseg->pls_range.length); 1461 out: 1462 pnfs_get_layout_hdr(lo); 1463 1464 dprintk("%s:Return\n", __func__); 1465 } 1466 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1467 1468 static void 1469 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1470 struct pnfs_layout_segment *lseg, 1471 struct list_head *free_me) 1472 { 1473 struct inode *inode = lo->plh_inode; 1474 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1475 1476 if (ld->add_lseg != NULL) 1477 ld->add_lseg(lo, lseg, free_me); 1478 else 1479 pnfs_generic_layout_insert_lseg(lo, lseg, 1480 pnfs_lseg_range_is_after, 1481 pnfs_lseg_no_merge, 1482 free_me); 1483 } 1484 1485 static struct pnfs_layout_hdr * 1486 alloc_init_layout_hdr(struct inode *ino, 1487 struct nfs_open_context *ctx, 1488 gfp_t gfp_flags) 1489 { 1490 struct pnfs_layout_hdr *lo; 1491 1492 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1493 if (!lo) 1494 return NULL; 1495 atomic_set(&lo->plh_refcount, 1); 1496 INIT_LIST_HEAD(&lo->plh_layouts); 1497 INIT_LIST_HEAD(&lo->plh_segs); 1498 INIT_LIST_HEAD(&lo->plh_return_segs); 1499 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1500 lo->plh_inode = ino; 1501 lo->plh_lc_cred = get_rpccred(ctx->cred); 1502 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; 1503 return lo; 1504 } 1505 1506 static struct pnfs_layout_hdr * 1507 pnfs_find_alloc_layout(struct inode *ino, 1508 struct nfs_open_context *ctx, 1509 gfp_t gfp_flags) 1510 __releases(&ino->i_lock) 1511 __acquires(&ino->i_lock) 1512 { 1513 struct nfs_inode *nfsi = NFS_I(ino); 1514 struct pnfs_layout_hdr *new = NULL; 1515 1516 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1517 1518 if (nfsi->layout != NULL) 1519 goto out_existing; 1520 spin_unlock(&ino->i_lock); 1521 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1522 spin_lock(&ino->i_lock); 1523 1524 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1525 nfsi->layout = new; 1526 return new; 1527 } else if (new != NULL) 1528 pnfs_free_layout_hdr(new); 1529 out_existing: 1530 pnfs_get_layout_hdr(nfsi->layout); 1531 return nfsi->layout; 1532 } 1533 1534 /* 1535 * iomode matching rules: 1536 * iomode lseg strict match 1537 * iomode 1538 * ----- ----- ------ ----- 1539 * ANY READ N/A true 1540 * ANY RW N/A true 1541 * RW READ N/A false 1542 * RW RW N/A true 1543 * READ READ N/A true 1544 * READ RW true false 1545 * READ RW false true 1546 */ 1547 static bool 1548 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1549 const struct pnfs_layout_range *range, 1550 bool strict_iomode) 1551 { 1552 struct pnfs_layout_range range1; 1553 1554 if ((range->iomode == IOMODE_RW && 1555 ls_range->iomode != IOMODE_RW) || 1556 (range->iomode != ls_range->iomode && 1557 strict_iomode == true) || 1558 !pnfs_lseg_range_intersecting(ls_range, range)) 1559 return 0; 1560 1561 /* range1 covers only the first byte in the range */ 1562 range1 = *range; 1563 range1.length = 1; 1564 return pnfs_lseg_range_contained(ls_range, &range1); 1565 } 1566 1567 /* 1568 * lookup range in layout 1569 */ 1570 static struct pnfs_layout_segment * 1571 pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1572 struct pnfs_layout_range *range, 1573 bool strict_iomode) 1574 { 1575 struct pnfs_layout_segment *lseg, *ret = NULL; 1576 1577 dprintk("%s:Begin\n", __func__); 1578 1579 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1580 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1581 !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 1582 pnfs_lseg_range_match(&lseg->pls_range, range, 1583 strict_iomode)) { 1584 ret = pnfs_get_lseg(lseg); 1585 break; 1586 } 1587 } 1588 1589 dprintk("%s:Return lseg %p ref %d\n", 1590 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0); 1591 return ret; 1592 } 1593 1594 /* 1595 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1596 * to the MDS or over pNFS 1597 * 1598 * The nfs_inode read_io and write_io fields are cumulative counters reset 1599 * when there are no layout segments. Note that in pnfs_update_layout iomode 1600 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 1601 * WRITE request. 1602 * 1603 * A return of true means use MDS I/O. 1604 * 1605 * From rfc 5661: 1606 * If a file's size is smaller than the file size threshold, data accesses 1607 * SHOULD be sent to the metadata server. If an I/O request has a length that 1608 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 1609 * server. If both file size and I/O size are provided, the client SHOULD 1610 * reach or exceed both thresholds before sending its read or write 1611 * requests to the data server. 1612 */ 1613 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 1614 struct inode *ino, int iomode) 1615 { 1616 struct nfs4_threshold *t = ctx->mdsthreshold; 1617 struct nfs_inode *nfsi = NFS_I(ino); 1618 loff_t fsize = i_size_read(ino); 1619 bool size = false, size_set = false, io = false, io_set = false, ret = false; 1620 1621 if (t == NULL) 1622 return ret; 1623 1624 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 1625 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 1626 1627 switch (iomode) { 1628 case IOMODE_READ: 1629 if (t->bm & THRESHOLD_RD) { 1630 dprintk("%s fsize %llu\n", __func__, fsize); 1631 size_set = true; 1632 if (fsize < t->rd_sz) 1633 size = true; 1634 } 1635 if (t->bm & THRESHOLD_RD_IO) { 1636 dprintk("%s nfsi->read_io %llu\n", __func__, 1637 nfsi->read_io); 1638 io_set = true; 1639 if (nfsi->read_io < t->rd_io_sz) 1640 io = true; 1641 } 1642 break; 1643 case IOMODE_RW: 1644 if (t->bm & THRESHOLD_WR) { 1645 dprintk("%s fsize %llu\n", __func__, fsize); 1646 size_set = true; 1647 if (fsize < t->wr_sz) 1648 size = true; 1649 } 1650 if (t->bm & THRESHOLD_WR_IO) { 1651 dprintk("%s nfsi->write_io %llu\n", __func__, 1652 nfsi->write_io); 1653 io_set = true; 1654 if (nfsi->write_io < t->wr_io_sz) 1655 io = true; 1656 } 1657 break; 1658 } 1659 if (size_set && io_set) { 1660 if (size && io) 1661 ret = true; 1662 } else if (size || io) 1663 ret = true; 1664 1665 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 1666 return ret; 1667 } 1668 1669 static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 1670 { 1671 /* 1672 * send layoutcommit as it can hold up layoutreturn due to lseg 1673 * reference 1674 */ 1675 pnfs_layoutcommit_inode(lo->plh_inode, false); 1676 return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 1677 nfs_wait_bit_killable, 1678 TASK_UNINTERRUPTIBLE); 1679 } 1680 1681 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 1682 { 1683 unsigned long *bitlock = &lo->plh_flags; 1684 1685 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 1686 smp_mb__after_atomic(); 1687 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 1688 } 1689 1690 /* 1691 * Layout segment is retreived from the server if not cached. 1692 * The appropriate layout segment is referenced and returned to the caller. 1693 */ 1694 struct pnfs_layout_segment * 1695 pnfs_update_layout(struct inode *ino, 1696 struct nfs_open_context *ctx, 1697 loff_t pos, 1698 u64 count, 1699 enum pnfs_iomode iomode, 1700 bool strict_iomode, 1701 gfp_t gfp_flags) 1702 { 1703 struct pnfs_layout_range arg = { 1704 .iomode = iomode, 1705 .offset = pos, 1706 .length = count, 1707 }; 1708 unsigned pg_offset, seq; 1709 struct nfs_server *server = NFS_SERVER(ino); 1710 struct nfs_client *clp = server->nfs_client; 1711 struct pnfs_layout_hdr *lo = NULL; 1712 struct pnfs_layout_segment *lseg = NULL; 1713 nfs4_stateid stateid; 1714 long timeout = 0; 1715 unsigned long giveup = jiffies + (clp->cl_lease_time << 1); 1716 bool first; 1717 1718 if (!pnfs_enabled_sb(NFS_SERVER(ino))) { 1719 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1720 PNFS_UPDATE_LAYOUT_NO_PNFS); 1721 goto out; 1722 } 1723 1724 if (iomode == IOMODE_READ && i_size_read(ino) == 0) { 1725 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1726 PNFS_UPDATE_LAYOUT_RD_ZEROLEN); 1727 goto out; 1728 } 1729 1730 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { 1731 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1732 PNFS_UPDATE_LAYOUT_MDSTHRESH); 1733 goto out; 1734 } 1735 1736 lookup_again: 1737 nfs4_client_recover_expired_lease(clp); 1738 first = false; 1739 spin_lock(&ino->i_lock); 1740 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 1741 if (lo == NULL) { 1742 spin_unlock(&ino->i_lock); 1743 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1744 PNFS_UPDATE_LAYOUT_NOMEM); 1745 goto out; 1746 } 1747 1748 /* Do we even need to bother with this? */ 1749 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1750 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1751 PNFS_UPDATE_LAYOUT_BULK_RECALL); 1752 dprintk("%s matches recall, use MDS\n", __func__); 1753 goto out_unlock; 1754 } 1755 1756 /* if LAYOUTGET already failed once we don't try again */ 1757 if (pnfs_layout_io_test_failed(lo, iomode)) { 1758 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1759 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); 1760 goto out_unlock; 1761 } 1762 1763 lseg = pnfs_find_lseg(lo, &arg, strict_iomode); 1764 if (lseg) { 1765 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1766 PNFS_UPDATE_LAYOUT_FOUND_CACHED); 1767 goto out_unlock; 1768 } 1769 1770 if (!nfs4_valid_open_stateid(ctx->state)) { 1771 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1772 PNFS_UPDATE_LAYOUT_INVALID_OPEN); 1773 goto out_unlock; 1774 } 1775 1776 /* 1777 * Choose a stateid for the LAYOUTGET. If we don't have a layout 1778 * stateid, or it has been invalidated, then we must use the open 1779 * stateid. 1780 */ 1781 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 1782 1783 /* 1784 * The first layoutget for the file. Need to serialize per 1785 * RFC 5661 Errata 3208. 1786 */ 1787 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 1788 &lo->plh_flags)) { 1789 spin_unlock(&ino->i_lock); 1790 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET, 1791 TASK_UNINTERRUPTIBLE); 1792 pnfs_put_layout_hdr(lo); 1793 dprintk("%s retrying\n", __func__); 1794 goto lookup_again; 1795 } 1796 1797 first = true; 1798 do { 1799 seq = read_seqbegin(&ctx->state->seqlock); 1800 nfs4_stateid_copy(&stateid, &ctx->state->stateid); 1801 } while (read_seqretry(&ctx->state->seqlock, seq)); 1802 } else { 1803 nfs4_stateid_copy(&stateid, &lo->plh_stateid); 1804 } 1805 1806 /* 1807 * Because we free lsegs before sending LAYOUTRETURN, we need to wait 1808 * for LAYOUTRETURN even if first is true. 1809 */ 1810 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1811 spin_unlock(&ino->i_lock); 1812 dprintk("%s wait for layoutreturn\n", __func__); 1813 if (pnfs_prepare_to_retry_layoutget(lo)) { 1814 if (first) 1815 pnfs_clear_first_layoutget(lo); 1816 pnfs_put_layout_hdr(lo); 1817 dprintk("%s retrying\n", __func__); 1818 trace_pnfs_update_layout(ino, pos, count, iomode, lo, 1819 lseg, PNFS_UPDATE_LAYOUT_RETRY); 1820 goto lookup_again; 1821 } 1822 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1823 PNFS_UPDATE_LAYOUT_RETURN); 1824 goto out_put_layout_hdr; 1825 } 1826 1827 if (pnfs_layoutgets_blocked(lo)) { 1828 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1829 PNFS_UPDATE_LAYOUT_BLOCKED); 1830 goto out_unlock; 1831 } 1832 atomic_inc(&lo->plh_outstanding); 1833 spin_unlock(&ino->i_lock); 1834 1835 if (list_empty(&lo->plh_layouts)) { 1836 /* The lo must be on the clp list if there is any 1837 * chance of a CB_LAYOUTRECALL(FILE) coming in. 1838 */ 1839 spin_lock(&clp->cl_lock); 1840 if (list_empty(&lo->plh_layouts)) 1841 list_add_tail(&lo->plh_layouts, &server->layouts); 1842 spin_unlock(&clp->cl_lock); 1843 } 1844 1845 pg_offset = arg.offset & ~PAGE_MASK; 1846 if (pg_offset) { 1847 arg.offset -= pg_offset; 1848 arg.length += pg_offset; 1849 } 1850 if (arg.length != NFS4_MAX_UINT64) 1851 arg.length = PAGE_ALIGN(arg.length); 1852 1853 lseg = send_layoutget(lo, ctx, &stateid, &arg, &timeout, gfp_flags); 1854 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, 1855 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); 1856 atomic_dec(&lo->plh_outstanding); 1857 if (IS_ERR(lseg)) { 1858 switch(PTR_ERR(lseg)) { 1859 case -EBUSY: 1860 if (time_after(jiffies, giveup)) 1861 lseg = NULL; 1862 break; 1863 case -ERECALLCONFLICT: 1864 /* Huh? We hold no layouts, how is there a recall? */ 1865 if (first) { 1866 lseg = NULL; 1867 break; 1868 } 1869 /* Destroy the existing layout and start over */ 1870 if (time_after(jiffies, giveup)) 1871 pnfs_destroy_layout(NFS_I(ino)); 1872 /* Fallthrough */ 1873 case -EAGAIN: 1874 break; 1875 default: 1876 if (!nfs_error_is_fatal(PTR_ERR(lseg))) { 1877 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 1878 lseg = NULL; 1879 } 1880 goto out_put_layout_hdr; 1881 } 1882 if (lseg) { 1883 if (first) 1884 pnfs_clear_first_layoutget(lo); 1885 trace_pnfs_update_layout(ino, pos, count, 1886 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); 1887 pnfs_put_layout_hdr(lo); 1888 goto lookup_again; 1889 } 1890 } else { 1891 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 1892 } 1893 1894 out_put_layout_hdr: 1895 if (first) 1896 pnfs_clear_first_layoutget(lo); 1897 pnfs_put_layout_hdr(lo); 1898 out: 1899 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 1900 "(%s, offset: %llu, length: %llu)\n", 1901 __func__, ino->i_sb->s_id, 1902 (unsigned long long)NFS_FILEID(ino), 1903 IS_ERR_OR_NULL(lseg) ? "not found" : "found", 1904 iomode==IOMODE_RW ? "read/write" : "read-only", 1905 (unsigned long long)pos, 1906 (unsigned long long)count); 1907 return lseg; 1908 out_unlock: 1909 spin_unlock(&ino->i_lock); 1910 goto out_put_layout_hdr; 1911 } 1912 EXPORT_SYMBOL_GPL(pnfs_update_layout); 1913 1914 static bool 1915 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 1916 { 1917 switch (range->iomode) { 1918 case IOMODE_READ: 1919 case IOMODE_RW: 1920 break; 1921 default: 1922 return false; 1923 } 1924 if (range->offset == NFS4_MAX_UINT64) 1925 return false; 1926 if (range->length == 0) 1927 return false; 1928 if (range->length != NFS4_MAX_UINT64 && 1929 range->length > NFS4_MAX_UINT64 - range->offset) 1930 return false; 1931 return true; 1932 } 1933 1934 struct pnfs_layout_segment * 1935 pnfs_layout_process(struct nfs4_layoutget *lgp) 1936 { 1937 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout; 1938 struct nfs4_layoutget_res *res = &lgp->res; 1939 struct pnfs_layout_segment *lseg; 1940 struct inode *ino = lo->plh_inode; 1941 LIST_HEAD(free_me); 1942 1943 if (!pnfs_sanity_check_layout_range(&res->range)) 1944 return ERR_PTR(-EINVAL); 1945 1946 /* Inject layout blob into I/O device driver */ 1947 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 1948 if (IS_ERR_OR_NULL(lseg)) { 1949 if (!lseg) 1950 lseg = ERR_PTR(-ENOMEM); 1951 1952 dprintk("%s: Could not allocate layout: error %ld\n", 1953 __func__, PTR_ERR(lseg)); 1954 return lseg; 1955 } 1956 1957 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); 1958 1959 spin_lock(&ino->i_lock); 1960 if (pnfs_layoutgets_blocked(lo)) { 1961 dprintk("%s forget reply due to state\n", __func__); 1962 goto out_forget; 1963 } 1964 1965 if (!pnfs_layout_is_valid(lo)) { 1966 /* We have a completely new layout */ 1967 pnfs_set_layout_stateid(lo, &res->stateid, true); 1968 } else if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 1969 /* existing state ID, make sure the sequence number matches. */ 1970 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 1971 dprintk("%s forget reply due to sequence\n", __func__); 1972 goto out_forget; 1973 } 1974 pnfs_set_layout_stateid(lo, &res->stateid, false); 1975 } else { 1976 /* 1977 * We got an entirely new state ID. Mark all segments for the 1978 * inode invalid, and retry the layoutget 1979 */ 1980 pnfs_mark_layout_stateid_invalid(lo, &free_me); 1981 goto out_forget; 1982 } 1983 1984 pnfs_get_lseg(lseg); 1985 pnfs_layout_insert_lseg(lo, lseg, &free_me); 1986 1987 1988 if (res->return_on_close) 1989 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 1990 1991 spin_unlock(&ino->i_lock); 1992 pnfs_free_lseg_list(&free_me); 1993 return lseg; 1994 1995 out_forget: 1996 spin_unlock(&ino->i_lock); 1997 lseg->pls_layout = lo; 1998 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 1999 if (!pnfs_layout_is_valid(lo)) 2000 nfs_commit_inode(ino, 0); 2001 return ERR_PTR(-EAGAIN); 2002 } 2003 2004 /** 2005 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments 2006 * @lo: pointer to layout header 2007 * @tmp_list: list header to be used with pnfs_free_lseg_list() 2008 * @return_range: describe layout segment ranges to be returned 2009 * 2010 * This function is mainly intended for use by layoutrecall. It attempts 2011 * to free the layout segment immediately, or else to mark it for return 2012 * as soon as its reference count drops to zero. 2013 */ 2014 int 2015 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 2016 struct list_head *tmp_list, 2017 const struct pnfs_layout_range *return_range, 2018 u32 seq) 2019 { 2020 struct pnfs_layout_segment *lseg, *next; 2021 int remaining = 0; 2022 2023 dprintk("%s:Begin lo %p\n", __func__, lo); 2024 2025 if (list_empty(&lo->plh_segs)) 2026 return 0; 2027 2028 assert_spin_locked(&lo->plh_inode->i_lock); 2029 2030 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 2031 if (pnfs_match_lseg_recall(lseg, return_range, seq)) { 2032 dprintk("%s: marking lseg %p iomode %d " 2033 "offset %llu length %llu\n", __func__, 2034 lseg, lseg->pls_range.iomode, 2035 lseg->pls_range.offset, 2036 lseg->pls_range.length); 2037 if (mark_lseg_invalid(lseg, tmp_list)) 2038 continue; 2039 remaining++; 2040 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 2041 } 2042 2043 if (remaining) 2044 pnfs_set_plh_return_info(lo, return_range->iomode, seq); 2045 2046 return remaining; 2047 } 2048 2049 void pnfs_error_mark_layout_for_return(struct inode *inode, 2050 struct pnfs_layout_segment *lseg) 2051 { 2052 struct pnfs_layout_hdr *lo = NFS_I(inode)->layout; 2053 struct pnfs_layout_range range = { 2054 .iomode = lseg->pls_range.iomode, 2055 .offset = 0, 2056 .length = NFS4_MAX_UINT64, 2057 }; 2058 bool return_now = false; 2059 2060 spin_lock(&inode->i_lock); 2061 if (!pnfs_layout_is_valid(lo)) { 2062 spin_unlock(&inode->i_lock); 2063 return; 2064 } 2065 pnfs_set_plh_return_info(lo, range.iomode, 0); 2066 /* 2067 * mark all matching lsegs so that we are sure to have no live 2068 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 2069 * for how it works. 2070 */ 2071 if (!pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, &range, 0)) { 2072 nfs4_stateid stateid; 2073 enum pnfs_iomode iomode; 2074 2075 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &iomode); 2076 spin_unlock(&inode->i_lock); 2077 if (return_now) 2078 pnfs_send_layoutreturn(lo, &stateid, iomode, false); 2079 } else { 2080 spin_unlock(&inode->i_lock); 2081 nfs_commit_inode(inode, 0); 2082 } 2083 } 2084 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 2085 2086 void 2087 pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio) 2088 { 2089 if (pgio->pg_lseg == NULL || 2090 test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags)) 2091 return; 2092 pnfs_put_lseg(pgio->pg_lseg); 2093 pgio->pg_lseg = NULL; 2094 } 2095 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); 2096 2097 /* 2098 * Check for any intersection between the request and the pgio->pg_lseg, 2099 * and if none, put this pgio->pg_lseg away. 2100 */ 2101 static void 2102 pnfs_generic_pg_check_range(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 2103 { 2104 if (pgio->pg_lseg && !pnfs_lseg_request_intersecting(pgio->pg_lseg, req)) { 2105 pnfs_put_lseg(pgio->pg_lseg); 2106 pgio->pg_lseg = NULL; 2107 } 2108 } 2109 2110 void 2111 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 2112 { 2113 u64 rd_size = req->wb_bytes; 2114 2115 pnfs_generic_pg_check_layout(pgio); 2116 pnfs_generic_pg_check_range(pgio, req); 2117 if (pgio->pg_lseg == NULL) { 2118 if (pgio->pg_dreq == NULL) 2119 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 2120 else 2121 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq); 2122 2123 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 2124 req->wb_context, 2125 req_offset(req), 2126 rd_size, 2127 IOMODE_READ, 2128 false, 2129 GFP_KERNEL); 2130 if (IS_ERR(pgio->pg_lseg)) { 2131 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2132 pgio->pg_lseg = NULL; 2133 return; 2134 } 2135 } 2136 /* If no lseg, fall back to read through mds */ 2137 if (pgio->pg_lseg == NULL) 2138 nfs_pageio_reset_read_mds(pgio); 2139 2140 } 2141 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 2142 2143 void 2144 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 2145 struct nfs_page *req, u64 wb_size) 2146 { 2147 pnfs_generic_pg_check_layout(pgio); 2148 pnfs_generic_pg_check_range(pgio, req); 2149 if (pgio->pg_lseg == NULL) { 2150 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 2151 req->wb_context, 2152 req_offset(req), 2153 wb_size, 2154 IOMODE_RW, 2155 false, 2156 GFP_NOFS); 2157 if (IS_ERR(pgio->pg_lseg)) { 2158 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 2159 pgio->pg_lseg = NULL; 2160 return; 2161 } 2162 } 2163 /* If no lseg, fall back to write through mds */ 2164 if (pgio->pg_lseg == NULL) 2165 nfs_pageio_reset_write_mds(pgio); 2166 } 2167 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 2168 2169 void 2170 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 2171 { 2172 if (desc->pg_lseg) { 2173 pnfs_put_lseg(desc->pg_lseg); 2174 desc->pg_lseg = NULL; 2175 } 2176 } 2177 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 2178 2179 /* 2180 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 2181 * of bytes (maximum @req->wb_bytes) that can be coalesced. 2182 */ 2183 size_t 2184 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 2185 struct nfs_page *prev, struct nfs_page *req) 2186 { 2187 unsigned int size; 2188 u64 seg_end, req_start, seg_left; 2189 2190 size = nfs_generic_pg_test(pgio, prev, req); 2191 if (!size) 2192 return 0; 2193 2194 /* 2195 * 'size' contains the number of bytes left in the current page (up 2196 * to the original size asked for in @req->wb_bytes). 2197 * 2198 * Calculate how many bytes are left in the layout segment 2199 * and if there are less bytes than 'size', return that instead. 2200 * 2201 * Please also note that 'end_offset' is actually the offset of the 2202 * first byte that lies outside the pnfs_layout_range. FIXME? 2203 * 2204 */ 2205 if (pgio->pg_lseg) { 2206 seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset, 2207 pgio->pg_lseg->pls_range.length); 2208 req_start = req_offset(req); 2209 2210 /* start of request is past the last byte of this segment */ 2211 if (req_start >= seg_end) 2212 return 0; 2213 2214 /* adjust 'size' iff there are fewer bytes left in the 2215 * segment than what nfs_generic_pg_test returned */ 2216 seg_left = seg_end - req_start; 2217 if (seg_left < size) 2218 size = (unsigned int)seg_left; 2219 } 2220 2221 return size; 2222 } 2223 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 2224 2225 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 2226 { 2227 struct nfs_pageio_descriptor pgio; 2228 2229 /* Resend all requests through the MDS */ 2230 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 2231 hdr->completion_ops); 2232 set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags); 2233 return nfs_pageio_resend(&pgio, hdr); 2234 } 2235 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 2236 2237 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 2238 { 2239 2240 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 2241 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 2242 PNFS_LAYOUTRET_ON_ERROR) { 2243 pnfs_return_layout(hdr->inode); 2244 } 2245 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 2246 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 2247 } 2248 2249 /* 2250 * Called by non rpc-based layout drivers 2251 */ 2252 void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 2253 { 2254 if (likely(!hdr->pnfs_error)) { 2255 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 2256 hdr->mds_offset + hdr->res.count); 2257 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 2258 } 2259 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 2260 if (unlikely(hdr->pnfs_error)) 2261 pnfs_ld_handle_write_error(hdr); 2262 hdr->mds_ops->rpc_release(hdr); 2263 } 2264 EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 2265 2266 static void 2267 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 2268 struct nfs_pgio_header *hdr) 2269 { 2270 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2271 2272 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2273 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 2274 nfs_pageio_reset_write_mds(desc); 2275 mirror->pg_recoalesce = 1; 2276 } 2277 nfs_pgio_data_destroy(hdr); 2278 hdr->release(hdr); 2279 } 2280 2281 static enum pnfs_try_status 2282 pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 2283 const struct rpc_call_ops *call_ops, 2284 struct pnfs_layout_segment *lseg, 2285 int how) 2286 { 2287 struct inode *inode = hdr->inode; 2288 enum pnfs_try_status trypnfs; 2289 struct nfs_server *nfss = NFS_SERVER(inode); 2290 2291 hdr->mds_ops = call_ops; 2292 2293 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__, 2294 inode->i_ino, hdr->args.count, hdr->args.offset, how); 2295 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 2296 if (trypnfs != PNFS_NOT_ATTEMPTED) 2297 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 2298 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 2299 return trypnfs; 2300 } 2301 2302 static void 2303 pnfs_do_write(struct nfs_pageio_descriptor *desc, 2304 struct nfs_pgio_header *hdr, int how) 2305 { 2306 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 2307 struct pnfs_layout_segment *lseg = desc->pg_lseg; 2308 enum pnfs_try_status trypnfs; 2309 2310 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 2311 switch (trypnfs) { 2312 case PNFS_NOT_ATTEMPTED: 2313 pnfs_write_through_mds(desc, hdr); 2314 case PNFS_ATTEMPTED: 2315 break; 2316 case PNFS_TRY_AGAIN: 2317 /* cleanup hdr and prepare to redo pnfs */ 2318 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2319 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2320 list_splice_init(&hdr->pages, &mirror->pg_list); 2321 mirror->pg_recoalesce = 1; 2322 } 2323 hdr->mds_ops->rpc_release(hdr); 2324 } 2325 } 2326 2327 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 2328 { 2329 pnfs_put_lseg(hdr->lseg); 2330 nfs_pgio_header_free(hdr); 2331 } 2332 2333 int 2334 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 2335 { 2336 struct nfs_pgio_header *hdr; 2337 int ret; 2338 2339 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 2340 if (!hdr) { 2341 desc->pg_error = -ENOMEM; 2342 return desc->pg_error; 2343 } 2344 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 2345 2346 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 2347 ret = nfs_generic_pgio(desc, hdr); 2348 if (!ret) 2349 pnfs_do_write(desc, hdr, desc->pg_ioflags); 2350 2351 return ret; 2352 } 2353 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 2354 2355 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 2356 { 2357 struct nfs_pageio_descriptor pgio; 2358 2359 /* Resend all requests through the MDS */ 2360 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 2361 return nfs_pageio_resend(&pgio, hdr); 2362 } 2363 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 2364 2365 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 2366 { 2367 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 2368 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 2369 PNFS_LAYOUTRET_ON_ERROR) { 2370 pnfs_return_layout(hdr->inode); 2371 } 2372 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 2373 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 2374 } 2375 2376 /* 2377 * Called by non rpc-based layout drivers 2378 */ 2379 void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 2380 { 2381 if (likely(!hdr->pnfs_error)) 2382 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 2383 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 2384 if (unlikely(hdr->pnfs_error)) 2385 pnfs_ld_handle_read_error(hdr); 2386 hdr->mds_ops->rpc_release(hdr); 2387 } 2388 EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 2389 2390 static void 2391 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 2392 struct nfs_pgio_header *hdr) 2393 { 2394 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2395 2396 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2397 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 2398 nfs_pageio_reset_read_mds(desc); 2399 mirror->pg_recoalesce = 1; 2400 } 2401 nfs_pgio_data_destroy(hdr); 2402 hdr->release(hdr); 2403 } 2404 2405 /* 2406 * Call the appropriate parallel I/O subsystem read function. 2407 */ 2408 static enum pnfs_try_status 2409 pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 2410 const struct rpc_call_ops *call_ops, 2411 struct pnfs_layout_segment *lseg) 2412 { 2413 struct inode *inode = hdr->inode; 2414 struct nfs_server *nfss = NFS_SERVER(inode); 2415 enum pnfs_try_status trypnfs; 2416 2417 hdr->mds_ops = call_ops; 2418 2419 dprintk("%s: Reading ino:%lu %u@%llu\n", 2420 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 2421 2422 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 2423 if (trypnfs != PNFS_NOT_ATTEMPTED) 2424 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 2425 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 2426 return trypnfs; 2427 } 2428 2429 /* Resend all requests through pnfs. */ 2430 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr) 2431 { 2432 struct nfs_pageio_descriptor pgio; 2433 2434 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2435 /* Prevent deadlocks with layoutreturn! */ 2436 pnfs_put_lseg(hdr->lseg); 2437 hdr->lseg = NULL; 2438 2439 nfs_pageio_init_read(&pgio, hdr->inode, false, 2440 hdr->completion_ops); 2441 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); 2442 } 2443 } 2444 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 2445 2446 static void 2447 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 2448 { 2449 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 2450 struct pnfs_layout_segment *lseg = desc->pg_lseg; 2451 enum pnfs_try_status trypnfs; 2452 2453 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 2454 switch (trypnfs) { 2455 case PNFS_NOT_ATTEMPTED: 2456 pnfs_read_through_mds(desc, hdr); 2457 case PNFS_ATTEMPTED: 2458 break; 2459 case PNFS_TRY_AGAIN: 2460 /* cleanup hdr and prepare to redo pnfs */ 2461 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2462 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2463 list_splice_init(&hdr->pages, &mirror->pg_list); 2464 mirror->pg_recoalesce = 1; 2465 } 2466 hdr->mds_ops->rpc_release(hdr); 2467 } 2468 } 2469 2470 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 2471 { 2472 pnfs_put_lseg(hdr->lseg); 2473 nfs_pgio_header_free(hdr); 2474 } 2475 2476 int 2477 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 2478 { 2479 struct nfs_pgio_header *hdr; 2480 int ret; 2481 2482 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 2483 if (!hdr) { 2484 desc->pg_error = -ENOMEM; 2485 return desc->pg_error; 2486 } 2487 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 2488 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 2489 ret = nfs_generic_pgio(desc, hdr); 2490 if (!ret) 2491 pnfs_do_read(desc, hdr); 2492 return ret; 2493 } 2494 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 2495 2496 static void pnfs_clear_layoutcommitting(struct inode *inode) 2497 { 2498 unsigned long *bitlock = &NFS_I(inode)->flags; 2499 2500 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 2501 smp_mb__after_atomic(); 2502 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 2503 } 2504 2505 /* 2506 * There can be multiple RW segments. 2507 */ 2508 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 2509 { 2510 struct pnfs_layout_segment *lseg; 2511 2512 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 2513 if (lseg->pls_range.iomode == IOMODE_RW && 2514 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 2515 list_add(&lseg->pls_lc_list, listp); 2516 } 2517 } 2518 2519 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 2520 { 2521 struct pnfs_layout_segment *lseg, *tmp; 2522 2523 /* Matched by references in pnfs_set_layoutcommit */ 2524 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 2525 list_del_init(&lseg->pls_lc_list); 2526 pnfs_put_lseg(lseg); 2527 } 2528 2529 pnfs_clear_layoutcommitting(inode); 2530 } 2531 2532 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 2533 { 2534 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 2535 } 2536 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 2537 2538 void 2539 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 2540 loff_t end_pos) 2541 { 2542 struct nfs_inode *nfsi = NFS_I(inode); 2543 bool mark_as_dirty = false; 2544 2545 spin_lock(&inode->i_lock); 2546 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 2547 nfsi->layout->plh_lwb = end_pos; 2548 mark_as_dirty = true; 2549 dprintk("%s: Set layoutcommit for inode %lu ", 2550 __func__, inode->i_ino); 2551 } else if (end_pos > nfsi->layout->plh_lwb) 2552 nfsi->layout->plh_lwb = end_pos; 2553 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 2554 /* references matched in nfs4_layoutcommit_release */ 2555 pnfs_get_lseg(lseg); 2556 } 2557 spin_unlock(&inode->i_lock); 2558 dprintk("%s: lseg %p end_pos %llu\n", 2559 __func__, lseg, nfsi->layout->plh_lwb); 2560 2561 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 2562 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 2563 if (mark_as_dirty) 2564 mark_inode_dirty_sync(inode); 2565 } 2566 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 2567 2568 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 2569 { 2570 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 2571 2572 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 2573 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 2574 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 2575 } 2576 2577 /* 2578 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 2579 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 2580 * data to disk to allow the server to recover the data if it crashes. 2581 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 2582 * is off, and a COMMIT is sent to a data server, or 2583 * if WRITEs to a data server return NFS_DATA_SYNC. 2584 */ 2585 int 2586 pnfs_layoutcommit_inode(struct inode *inode, bool sync) 2587 { 2588 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 2589 struct nfs4_layoutcommit_data *data; 2590 struct nfs_inode *nfsi = NFS_I(inode); 2591 loff_t end_pos; 2592 int status; 2593 2594 if (!pnfs_layoutcommit_outstanding(inode)) 2595 return 0; 2596 2597 dprintk("--> %s inode %lu\n", __func__, inode->i_ino); 2598 2599 status = -EAGAIN; 2600 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 2601 if (!sync) 2602 goto out; 2603 status = wait_on_bit_lock_action(&nfsi->flags, 2604 NFS_INO_LAYOUTCOMMITTING, 2605 nfs_wait_bit_killable, 2606 TASK_KILLABLE); 2607 if (status) 2608 goto out; 2609 } 2610 2611 status = -ENOMEM; 2612 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 2613 data = kzalloc(sizeof(*data), GFP_NOFS); 2614 if (!data) 2615 goto clear_layoutcommitting; 2616 2617 status = 0; 2618 spin_lock(&inode->i_lock); 2619 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 2620 goto out_unlock; 2621 2622 INIT_LIST_HEAD(&data->lseg_list); 2623 pnfs_list_write_lseg(inode, &data->lseg_list); 2624 2625 end_pos = nfsi->layout->plh_lwb; 2626 2627 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 2628 spin_unlock(&inode->i_lock); 2629 2630 data->args.inode = inode; 2631 data->cred = get_rpccred(nfsi->layout->plh_lc_cred); 2632 nfs_fattr_init(&data->fattr); 2633 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 2634 data->res.fattr = &data->fattr; 2635 if (end_pos != 0) 2636 data->args.lastbytewritten = end_pos - 1; 2637 else 2638 data->args.lastbytewritten = U64_MAX; 2639 data->res.server = NFS_SERVER(inode); 2640 2641 if (ld->prepare_layoutcommit) { 2642 status = ld->prepare_layoutcommit(&data->args); 2643 if (status) { 2644 put_rpccred(data->cred); 2645 spin_lock(&inode->i_lock); 2646 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 2647 if (end_pos > nfsi->layout->plh_lwb) 2648 nfsi->layout->plh_lwb = end_pos; 2649 goto out_unlock; 2650 } 2651 } 2652 2653 2654 status = nfs4_proc_layoutcommit(data, sync); 2655 out: 2656 if (status) 2657 mark_inode_dirty_sync(inode); 2658 dprintk("<-- %s status %d\n", __func__, status); 2659 return status; 2660 out_unlock: 2661 spin_unlock(&inode->i_lock); 2662 kfree(data); 2663 clear_layoutcommitting: 2664 pnfs_clear_layoutcommitting(inode); 2665 goto out; 2666 } 2667 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 2668 2669 int 2670 pnfs_generic_sync(struct inode *inode, bool datasync) 2671 { 2672 return pnfs_layoutcommit_inode(inode, true); 2673 } 2674 EXPORT_SYMBOL_GPL(pnfs_generic_sync); 2675 2676 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 2677 { 2678 struct nfs4_threshold *thp; 2679 2680 thp = kzalloc(sizeof(*thp), GFP_NOFS); 2681 if (!thp) { 2682 dprintk("%s mdsthreshold allocation failed\n", __func__); 2683 return NULL; 2684 } 2685 return thp; 2686 } 2687 2688 #if IS_ENABLED(CONFIG_NFS_V4_2) 2689 int 2690 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 2691 { 2692 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 2693 struct nfs_server *server = NFS_SERVER(inode); 2694 struct nfs_inode *nfsi = NFS_I(inode); 2695 struct nfs42_layoutstat_data *data; 2696 struct pnfs_layout_hdr *hdr; 2697 int status = 0; 2698 2699 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 2700 goto out; 2701 2702 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 2703 goto out; 2704 2705 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 2706 goto out; 2707 2708 spin_lock(&inode->i_lock); 2709 if (!NFS_I(inode)->layout) { 2710 spin_unlock(&inode->i_lock); 2711 goto out_clear_layoutstats; 2712 } 2713 hdr = NFS_I(inode)->layout; 2714 pnfs_get_layout_hdr(hdr); 2715 spin_unlock(&inode->i_lock); 2716 2717 data = kzalloc(sizeof(*data), gfp_flags); 2718 if (!data) { 2719 status = -ENOMEM; 2720 goto out_put; 2721 } 2722 2723 data->args.fh = NFS_FH(inode); 2724 data->args.inode = inode; 2725 status = ld->prepare_layoutstats(&data->args); 2726 if (status) 2727 goto out_free; 2728 2729 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 2730 2731 out: 2732 dprintk("%s returns %d\n", __func__, status); 2733 return status; 2734 2735 out_free: 2736 kfree(data); 2737 out_put: 2738 pnfs_put_layout_hdr(hdr); 2739 out_clear_layoutstats: 2740 smp_mb__before_atomic(); 2741 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 2742 smp_mb__after_atomic(); 2743 goto out; 2744 } 2745 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 2746 #endif 2747 2748 unsigned int layoutstats_timer; 2749 module_param(layoutstats_timer, uint, 0644); 2750 EXPORT_SYMBOL_GPL(layoutstats_timer); 2751