1 /* 2 * Module for pnfs flexfile layout driver. 3 * 4 * Copyright (c) 2014, Primary Data, Inc. All rights reserved. 5 * 6 * Tao Peng <bergwolf@primarydata.com> 7 */ 8 9 #include <linux/nfs_fs.h> 10 #include <linux/nfs_page.h> 11 #include <linux/module.h> 12 13 #include <linux/sunrpc/metrics.h> 14 15 #include "flexfilelayout.h" 16 #include "../nfs4session.h" 17 #include "../nfs4idmap.h" 18 #include "../internal.h" 19 #include "../delegation.h" 20 #include "../nfs4trace.h" 21 #include "../iostat.h" 22 #include "../nfs.h" 23 #include "../nfs42.h" 24 25 #define NFSDBG_FACILITY NFSDBG_PNFS_LD 26 27 #define FF_LAYOUT_POLL_RETRY_MAX (15*HZ) 28 29 static struct pnfs_layout_hdr * 30 ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) 31 { 32 struct nfs4_flexfile_layout *ffl; 33 34 ffl = kzalloc(sizeof(*ffl), gfp_flags); 35 if (ffl) { 36 INIT_LIST_HEAD(&ffl->error_list); 37 INIT_LIST_HEAD(&ffl->mirrors); 38 return &ffl->generic_hdr; 39 } else 40 return NULL; 41 } 42 43 static void 44 ff_layout_free_layout_hdr(struct pnfs_layout_hdr *lo) 45 { 46 struct nfs4_ff_layout_ds_err *err, *n; 47 48 list_for_each_entry_safe(err, n, &FF_LAYOUT_FROM_HDR(lo)->error_list, 49 list) { 50 list_del(&err->list); 51 kfree(err); 52 } 53 kfree(FF_LAYOUT_FROM_HDR(lo)); 54 } 55 56 static int decode_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) 57 { 58 __be32 *p; 59 60 p = xdr_inline_decode(xdr, NFS4_STATEID_SIZE); 61 if (unlikely(p == NULL)) 62 return -ENOBUFS; 63 memcpy(stateid, p, NFS4_STATEID_SIZE); 64 dprintk("%s: stateid id= [%x%x%x%x]\n", __func__, 65 p[0], p[1], p[2], p[3]); 66 return 0; 67 } 68 69 static int decode_deviceid(struct xdr_stream *xdr, struct nfs4_deviceid *devid) 70 { 71 __be32 *p; 72 73 p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE); 74 if (unlikely(!p)) 75 return -ENOBUFS; 76 memcpy(devid, p, NFS4_DEVICEID4_SIZE); 77 nfs4_print_deviceid(devid); 78 return 0; 79 } 80 81 static int decode_nfs_fh(struct xdr_stream *xdr, struct nfs_fh *fh) 82 { 83 __be32 *p; 84 85 p = xdr_inline_decode(xdr, 4); 86 if (unlikely(!p)) 87 return -ENOBUFS; 88 fh->size = be32_to_cpup(p++); 89 if (fh->size > sizeof(struct nfs_fh)) { 90 printk(KERN_ERR "NFS flexfiles: Too big fh received %d\n", 91 fh->size); 92 return -EOVERFLOW; 93 } 94 /* fh.data */ 95 p = xdr_inline_decode(xdr, fh->size); 96 if (unlikely(!p)) 97 return -ENOBUFS; 98 memcpy(&fh->data, p, fh->size); 99 dprintk("%s: fh len %d\n", __func__, fh->size); 100 101 return 0; 102 } 103 104 /* 105 * Currently only stringified uids and gids are accepted. 106 * I.e., kerberos is not supported to the DSes, so no pricipals. 107 * 108 * That means that one common function will suffice, but when 109 * principals are added, this should be split to accomodate 110 * calls to both nfs_map_name_to_uid() and nfs_map_group_to_gid(). 111 */ 112 static int 113 decode_name(struct xdr_stream *xdr, u32 *id) 114 { 115 __be32 *p; 116 int len; 117 118 /* opaque_length(4)*/ 119 p = xdr_inline_decode(xdr, 4); 120 if (unlikely(!p)) 121 return -ENOBUFS; 122 len = be32_to_cpup(p++); 123 if (len < 0) 124 return -EINVAL; 125 126 dprintk("%s: len %u\n", __func__, len); 127 128 /* opaque body */ 129 p = xdr_inline_decode(xdr, len); 130 if (unlikely(!p)) 131 return -ENOBUFS; 132 133 if (!nfs_map_string_to_numeric((char *)p, len, id)) 134 return -EINVAL; 135 136 return 0; 137 } 138 139 static bool ff_mirror_match_fh(const struct nfs4_ff_layout_mirror *m1, 140 const struct nfs4_ff_layout_mirror *m2) 141 { 142 int i, j; 143 144 if (m1->fh_versions_cnt != m2->fh_versions_cnt) 145 return false; 146 for (i = 0; i < m1->fh_versions_cnt; i++) { 147 bool found_fh = false; 148 for (j = 0; j < m2->fh_versions_cnt; j++) { 149 if (nfs_compare_fh(&m1->fh_versions[i], 150 &m2->fh_versions[j]) == 0) { 151 found_fh = true; 152 break; 153 } 154 } 155 if (!found_fh) 156 return false; 157 } 158 return true; 159 } 160 161 static struct nfs4_ff_layout_mirror * 162 ff_layout_add_mirror(struct pnfs_layout_hdr *lo, 163 struct nfs4_ff_layout_mirror *mirror) 164 { 165 struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo); 166 struct nfs4_ff_layout_mirror *pos; 167 struct inode *inode = lo->plh_inode; 168 169 spin_lock(&inode->i_lock); 170 list_for_each_entry(pos, &ff_layout->mirrors, mirrors) { 171 if (mirror->mirror_ds != pos->mirror_ds) 172 continue; 173 if (!ff_mirror_match_fh(mirror, pos)) 174 continue; 175 if (atomic_inc_not_zero(&pos->ref)) { 176 spin_unlock(&inode->i_lock); 177 return pos; 178 } 179 } 180 list_add(&mirror->mirrors, &ff_layout->mirrors); 181 mirror->layout = lo; 182 spin_unlock(&inode->i_lock); 183 return mirror; 184 } 185 186 static void 187 ff_layout_remove_mirror(struct nfs4_ff_layout_mirror *mirror) 188 { 189 struct inode *inode; 190 if (mirror->layout == NULL) 191 return; 192 inode = mirror->layout->plh_inode; 193 spin_lock(&inode->i_lock); 194 list_del(&mirror->mirrors); 195 spin_unlock(&inode->i_lock); 196 mirror->layout = NULL; 197 } 198 199 static struct nfs4_ff_layout_mirror *ff_layout_alloc_mirror(gfp_t gfp_flags) 200 { 201 struct nfs4_ff_layout_mirror *mirror; 202 203 mirror = kzalloc(sizeof(*mirror), gfp_flags); 204 if (mirror != NULL) { 205 spin_lock_init(&mirror->lock); 206 atomic_set(&mirror->ref, 1); 207 INIT_LIST_HEAD(&mirror->mirrors); 208 } 209 return mirror; 210 } 211 212 static void ff_layout_free_mirror(struct nfs4_ff_layout_mirror *mirror) 213 { 214 ff_layout_remove_mirror(mirror); 215 kfree(mirror->fh_versions); 216 if (mirror->cred) 217 put_rpccred(mirror->cred); 218 nfs4_ff_layout_put_deviceid(mirror->mirror_ds); 219 kfree(mirror); 220 } 221 222 static void ff_layout_put_mirror(struct nfs4_ff_layout_mirror *mirror) 223 { 224 if (mirror != NULL && atomic_dec_and_test(&mirror->ref)) 225 ff_layout_free_mirror(mirror); 226 } 227 228 static void ff_layout_free_mirror_array(struct nfs4_ff_layout_segment *fls) 229 { 230 int i; 231 232 if (fls->mirror_array) { 233 for (i = 0; i < fls->mirror_array_cnt; i++) { 234 /* normally mirror_ds is freed in 235 * .free_deviceid_node but we still do it here 236 * for .alloc_lseg error path */ 237 ff_layout_put_mirror(fls->mirror_array[i]); 238 } 239 kfree(fls->mirror_array); 240 fls->mirror_array = NULL; 241 } 242 } 243 244 static int ff_layout_check_layout(struct nfs4_layoutget_res *lgr) 245 { 246 int ret = 0; 247 248 dprintk("--> %s\n", __func__); 249 250 /* FIXME: remove this check when layout segment support is added */ 251 if (lgr->range.offset != 0 || 252 lgr->range.length != NFS4_MAX_UINT64) { 253 dprintk("%s Only whole file layouts supported. Use MDS i/o\n", 254 __func__); 255 ret = -EINVAL; 256 } 257 258 dprintk("--> %s returns %d\n", __func__, ret); 259 return ret; 260 } 261 262 static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls) 263 { 264 if (fls) { 265 ff_layout_free_mirror_array(fls); 266 kfree(fls); 267 } 268 } 269 270 static bool 271 ff_lseg_range_is_after(const struct pnfs_layout_range *l1, 272 const struct pnfs_layout_range *l2) 273 { 274 u64 end1, end2; 275 276 if (l1->iomode != l2->iomode) 277 return l1->iomode != IOMODE_READ; 278 end1 = pnfs_calc_offset_end(l1->offset, l1->length); 279 end2 = pnfs_calc_offset_end(l2->offset, l2->length); 280 if (end1 < l2->offset) 281 return false; 282 if (end2 < l1->offset) 283 return true; 284 return l2->offset <= l1->offset; 285 } 286 287 static bool 288 ff_lseg_merge(struct pnfs_layout_segment *new, 289 struct pnfs_layout_segment *old) 290 { 291 u64 new_end, old_end; 292 293 if (new->pls_range.iomode != old->pls_range.iomode) 294 return false; 295 old_end = pnfs_calc_offset_end(old->pls_range.offset, 296 old->pls_range.length); 297 if (old_end < new->pls_range.offset) 298 return false; 299 new_end = pnfs_calc_offset_end(new->pls_range.offset, 300 new->pls_range.length); 301 if (new_end < old->pls_range.offset) 302 return false; 303 304 /* Mergeable: copy info from 'old' to 'new' */ 305 if (new_end < old_end) 306 new_end = old_end; 307 if (new->pls_range.offset < old->pls_range.offset) 308 new->pls_range.offset = old->pls_range.offset; 309 new->pls_range.length = pnfs_calc_offset_length(new->pls_range.offset, 310 new_end); 311 if (test_bit(NFS_LSEG_ROC, &old->pls_flags)) 312 set_bit(NFS_LSEG_ROC, &new->pls_flags); 313 if (test_bit(NFS_LSEG_LAYOUTRETURN, &old->pls_flags)) 314 set_bit(NFS_LSEG_LAYOUTRETURN, &new->pls_flags); 315 return true; 316 } 317 318 static void 319 ff_layout_add_lseg(struct pnfs_layout_hdr *lo, 320 struct pnfs_layout_segment *lseg, 321 struct list_head *free_me) 322 { 323 pnfs_generic_layout_insert_lseg(lo, lseg, 324 ff_lseg_range_is_after, 325 ff_lseg_merge, 326 free_me); 327 } 328 329 static void ff_layout_sort_mirrors(struct nfs4_ff_layout_segment *fls) 330 { 331 int i, j; 332 333 for (i = 0; i < fls->mirror_array_cnt - 1; i++) { 334 for (j = i + 1; j < fls->mirror_array_cnt; j++) 335 if (fls->mirror_array[i]->efficiency < 336 fls->mirror_array[j]->efficiency) 337 swap(fls->mirror_array[i], 338 fls->mirror_array[j]); 339 } 340 } 341 342 static void ff_layout_mark_devices_valid(struct nfs4_ff_layout_segment *fls) 343 { 344 struct nfs4_deviceid_node *node; 345 int i; 346 347 if (!(fls->flags & FF_FLAGS_NO_IO_THRU_MDS)) 348 return; 349 for (i = 0; i < fls->mirror_array_cnt; i++) { 350 node = &fls->mirror_array[i]->mirror_ds->id_node; 351 clear_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags); 352 } 353 } 354 355 static struct pnfs_layout_segment * 356 ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, 357 struct nfs4_layoutget_res *lgr, 358 gfp_t gfp_flags) 359 { 360 struct pnfs_layout_segment *ret; 361 struct nfs4_ff_layout_segment *fls = NULL; 362 struct xdr_stream stream; 363 struct xdr_buf buf; 364 struct page *scratch; 365 u64 stripe_unit; 366 u32 mirror_array_cnt; 367 __be32 *p; 368 int i, rc; 369 370 dprintk("--> %s\n", __func__); 371 scratch = alloc_page(gfp_flags); 372 if (!scratch) 373 return ERR_PTR(-ENOMEM); 374 375 xdr_init_decode_pages(&stream, &buf, lgr->layoutp->pages, 376 lgr->layoutp->len); 377 xdr_set_scratch_buffer(&stream, page_address(scratch), PAGE_SIZE); 378 379 /* stripe unit and mirror_array_cnt */ 380 rc = -EIO; 381 p = xdr_inline_decode(&stream, 8 + 4); 382 if (!p) 383 goto out_err_free; 384 385 p = xdr_decode_hyper(p, &stripe_unit); 386 mirror_array_cnt = be32_to_cpup(p++); 387 dprintk("%s: stripe_unit=%llu mirror_array_cnt=%u\n", __func__, 388 stripe_unit, mirror_array_cnt); 389 390 if (mirror_array_cnt > NFS4_FLEXFILE_LAYOUT_MAX_MIRROR_CNT || 391 mirror_array_cnt == 0) 392 goto out_err_free; 393 394 rc = -ENOMEM; 395 fls = kzalloc(sizeof(*fls), gfp_flags); 396 if (!fls) 397 goto out_err_free; 398 399 fls->mirror_array_cnt = mirror_array_cnt; 400 fls->stripe_unit = stripe_unit; 401 fls->mirror_array = kcalloc(fls->mirror_array_cnt, 402 sizeof(fls->mirror_array[0]), gfp_flags); 403 if (fls->mirror_array == NULL) 404 goto out_err_free; 405 406 for (i = 0; i < fls->mirror_array_cnt; i++) { 407 struct nfs4_ff_layout_mirror *mirror; 408 struct nfs4_deviceid devid; 409 struct nfs4_deviceid_node *idnode; 410 u32 ds_count; 411 u32 fh_count; 412 int j; 413 414 rc = -EIO; 415 p = xdr_inline_decode(&stream, 4); 416 if (!p) 417 goto out_err_free; 418 ds_count = be32_to_cpup(p); 419 420 /* FIXME: allow for striping? */ 421 if (ds_count != 1) 422 goto out_err_free; 423 424 fls->mirror_array[i] = ff_layout_alloc_mirror(gfp_flags); 425 if (fls->mirror_array[i] == NULL) { 426 rc = -ENOMEM; 427 goto out_err_free; 428 } 429 430 fls->mirror_array[i]->ds_count = ds_count; 431 432 /* deviceid */ 433 rc = decode_deviceid(&stream, &devid); 434 if (rc) 435 goto out_err_free; 436 437 idnode = nfs4_find_get_deviceid(NFS_SERVER(lh->plh_inode), 438 &devid, lh->plh_lc_cred, 439 gfp_flags); 440 /* 441 * upon success, mirror_ds is allocated by previous 442 * getdeviceinfo, or newly by .alloc_deviceid_node 443 * nfs4_find_get_deviceid failure is indeed getdeviceinfo falure 444 */ 445 if (idnode) 446 fls->mirror_array[i]->mirror_ds = 447 FF_LAYOUT_MIRROR_DS(idnode); 448 else 449 goto out_err_free; 450 451 /* efficiency */ 452 rc = -EIO; 453 p = xdr_inline_decode(&stream, 4); 454 if (!p) 455 goto out_err_free; 456 fls->mirror_array[i]->efficiency = be32_to_cpup(p); 457 458 /* stateid */ 459 rc = decode_stateid(&stream, &fls->mirror_array[i]->stateid); 460 if (rc) 461 goto out_err_free; 462 463 /* fh */ 464 p = xdr_inline_decode(&stream, 4); 465 if (!p) 466 goto out_err_free; 467 fh_count = be32_to_cpup(p); 468 469 fls->mirror_array[i]->fh_versions = 470 kzalloc(fh_count * sizeof(struct nfs_fh), 471 gfp_flags); 472 if (fls->mirror_array[i]->fh_versions == NULL) { 473 rc = -ENOMEM; 474 goto out_err_free; 475 } 476 477 for (j = 0; j < fh_count; j++) { 478 rc = decode_nfs_fh(&stream, 479 &fls->mirror_array[i]->fh_versions[j]); 480 if (rc) 481 goto out_err_free; 482 } 483 484 fls->mirror_array[i]->fh_versions_cnt = fh_count; 485 486 /* user */ 487 rc = decode_name(&stream, &fls->mirror_array[i]->uid); 488 if (rc) 489 goto out_err_free; 490 491 /* group */ 492 rc = decode_name(&stream, &fls->mirror_array[i]->gid); 493 if (rc) 494 goto out_err_free; 495 496 mirror = ff_layout_add_mirror(lh, fls->mirror_array[i]); 497 if (mirror != fls->mirror_array[i]) { 498 ff_layout_free_mirror(fls->mirror_array[i]); 499 fls->mirror_array[i] = mirror; 500 } 501 502 dprintk("%s: uid %d gid %d\n", __func__, 503 fls->mirror_array[i]->uid, 504 fls->mirror_array[i]->gid); 505 } 506 507 p = xdr_inline_decode(&stream, 4); 508 if (!p) 509 goto out_sort_mirrors; 510 fls->flags = be32_to_cpup(p); 511 512 p = xdr_inline_decode(&stream, 4); 513 if (!p) 514 goto out_sort_mirrors; 515 for (i=0; i < fls->mirror_array_cnt; i++) 516 fls->mirror_array[i]->report_interval = be32_to_cpup(p); 517 518 out_sort_mirrors: 519 ff_layout_sort_mirrors(fls); 520 rc = ff_layout_check_layout(lgr); 521 if (rc) 522 goto out_err_free; 523 ff_layout_mark_devices_valid(fls); 524 525 ret = &fls->generic_hdr; 526 dprintk("<-- %s (success)\n", __func__); 527 out_free_page: 528 __free_page(scratch); 529 return ret; 530 out_err_free: 531 _ff_layout_free_lseg(fls); 532 ret = ERR_PTR(rc); 533 dprintk("<-- %s (%d)\n", __func__, rc); 534 goto out_free_page; 535 } 536 537 static bool ff_layout_has_rw_segments(struct pnfs_layout_hdr *layout) 538 { 539 struct pnfs_layout_segment *lseg; 540 541 list_for_each_entry(lseg, &layout->plh_segs, pls_list) 542 if (lseg->pls_range.iomode == IOMODE_RW) 543 return true; 544 545 return false; 546 } 547 548 static void 549 ff_layout_free_lseg(struct pnfs_layout_segment *lseg) 550 { 551 struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); 552 553 dprintk("--> %s\n", __func__); 554 555 if (lseg->pls_range.iomode == IOMODE_RW) { 556 struct nfs4_flexfile_layout *ffl; 557 struct inode *inode; 558 559 ffl = FF_LAYOUT_FROM_HDR(lseg->pls_layout); 560 inode = ffl->generic_hdr.plh_inode; 561 spin_lock(&inode->i_lock); 562 if (!ff_layout_has_rw_segments(lseg->pls_layout)) { 563 ffl->commit_info.nbuckets = 0; 564 kfree(ffl->commit_info.buckets); 565 ffl->commit_info.buckets = NULL; 566 } 567 spin_unlock(&inode->i_lock); 568 } 569 _ff_layout_free_lseg(fls); 570 } 571 572 /* Return 1 until we have multiple lsegs support */ 573 static int 574 ff_layout_get_lseg_count(struct nfs4_ff_layout_segment *fls) 575 { 576 return 1; 577 } 578 579 static void 580 nfs4_ff_start_busy_timer(struct nfs4_ff_busy_timer *timer, ktime_t now) 581 { 582 /* first IO request? */ 583 if (atomic_inc_return(&timer->n_ops) == 1) { 584 timer->start_time = now; 585 } 586 } 587 588 static ktime_t 589 nfs4_ff_end_busy_timer(struct nfs4_ff_busy_timer *timer, ktime_t now) 590 { 591 ktime_t start; 592 593 if (atomic_dec_return(&timer->n_ops) < 0) 594 WARN_ON_ONCE(1); 595 596 start = timer->start_time; 597 timer->start_time = now; 598 return ktime_sub(now, start); 599 } 600 601 static bool 602 nfs4_ff_layoutstat_start_io(struct nfs4_ff_layout_mirror *mirror, 603 struct nfs4_ff_layoutstat *layoutstat, 604 ktime_t now) 605 { 606 static const ktime_t notime = {0}; 607 s64 report_interval = FF_LAYOUTSTATS_REPORT_INTERVAL; 608 609 nfs4_ff_start_busy_timer(&layoutstat->busy_timer, now); 610 if (ktime_equal(mirror->start_time, notime)) 611 mirror->start_time = now; 612 if (ktime_equal(mirror->last_report_time, notime)) 613 mirror->last_report_time = now; 614 if (mirror->report_interval != 0) 615 report_interval = (s64)mirror->report_interval * 1000LL; 616 else if (layoutstats_timer != 0) 617 report_interval = (s64)layoutstats_timer * 1000LL; 618 if (ktime_to_ms(ktime_sub(now, mirror->last_report_time)) >= 619 report_interval) { 620 mirror->last_report_time = now; 621 return true; 622 } 623 624 return false; 625 } 626 627 static void 628 nfs4_ff_layout_stat_io_update_requested(struct nfs4_ff_layoutstat *layoutstat, 629 __u64 requested) 630 { 631 struct nfs4_ff_io_stat *iostat = &layoutstat->io_stat; 632 633 iostat->ops_requested++; 634 iostat->bytes_requested += requested; 635 } 636 637 static void 638 nfs4_ff_layout_stat_io_update_completed(struct nfs4_ff_layoutstat *layoutstat, 639 __u64 requested, 640 __u64 completed, 641 ktime_t time_completed, 642 ktime_t time_started) 643 { 644 struct nfs4_ff_io_stat *iostat = &layoutstat->io_stat; 645 ktime_t completion_time = ktime_sub(time_completed, time_started); 646 ktime_t timer; 647 648 iostat->ops_completed++; 649 iostat->bytes_completed += completed; 650 iostat->bytes_not_delivered += requested - completed; 651 652 timer = nfs4_ff_end_busy_timer(&layoutstat->busy_timer, time_completed); 653 iostat->total_busy_time = 654 ktime_add(iostat->total_busy_time, timer); 655 iostat->aggregate_completion_time = 656 ktime_add(iostat->aggregate_completion_time, 657 completion_time); 658 } 659 660 static void 661 nfs4_ff_layout_stat_io_start_read(struct inode *inode, 662 struct nfs4_ff_layout_mirror *mirror, 663 __u64 requested, ktime_t now) 664 { 665 bool report; 666 667 spin_lock(&mirror->lock); 668 report = nfs4_ff_layoutstat_start_io(mirror, &mirror->read_stat, now); 669 nfs4_ff_layout_stat_io_update_requested(&mirror->read_stat, requested); 670 spin_unlock(&mirror->lock); 671 672 if (report) 673 pnfs_report_layoutstat(inode, GFP_KERNEL); 674 } 675 676 static void 677 nfs4_ff_layout_stat_io_end_read(struct rpc_task *task, 678 struct nfs4_ff_layout_mirror *mirror, 679 __u64 requested, 680 __u64 completed) 681 { 682 spin_lock(&mirror->lock); 683 nfs4_ff_layout_stat_io_update_completed(&mirror->read_stat, 684 requested, completed, 685 ktime_get(), task->tk_start); 686 spin_unlock(&mirror->lock); 687 } 688 689 static void 690 nfs4_ff_layout_stat_io_start_write(struct inode *inode, 691 struct nfs4_ff_layout_mirror *mirror, 692 __u64 requested, ktime_t now) 693 { 694 bool report; 695 696 spin_lock(&mirror->lock); 697 report = nfs4_ff_layoutstat_start_io(mirror , &mirror->write_stat, now); 698 nfs4_ff_layout_stat_io_update_requested(&mirror->write_stat, requested); 699 spin_unlock(&mirror->lock); 700 701 if (report) 702 pnfs_report_layoutstat(inode, GFP_NOIO); 703 } 704 705 static void 706 nfs4_ff_layout_stat_io_end_write(struct rpc_task *task, 707 struct nfs4_ff_layout_mirror *mirror, 708 __u64 requested, 709 __u64 completed, 710 enum nfs3_stable_how committed) 711 { 712 if (committed == NFS_UNSTABLE) 713 requested = completed = 0; 714 715 spin_lock(&mirror->lock); 716 nfs4_ff_layout_stat_io_update_completed(&mirror->write_stat, 717 requested, completed, ktime_get(), task->tk_start); 718 spin_unlock(&mirror->lock); 719 } 720 721 static int 722 ff_layout_alloc_commit_info(struct pnfs_layout_segment *lseg, 723 struct nfs_commit_info *cinfo, 724 gfp_t gfp_flags) 725 { 726 struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); 727 struct pnfs_commit_bucket *buckets; 728 int size; 729 730 if (cinfo->ds->nbuckets != 0) { 731 /* This assumes there is only one RW lseg per file. 732 * To support multiple lseg per file, we need to 733 * change struct pnfs_commit_bucket to allow dynamic 734 * increasing nbuckets. 735 */ 736 return 0; 737 } 738 739 size = ff_layout_get_lseg_count(fls) * FF_LAYOUT_MIRROR_COUNT(lseg); 740 741 buckets = kcalloc(size, sizeof(struct pnfs_commit_bucket), 742 gfp_flags); 743 if (!buckets) 744 return -ENOMEM; 745 else { 746 int i; 747 748 spin_lock(cinfo->lock); 749 if (cinfo->ds->nbuckets != 0) 750 kfree(buckets); 751 else { 752 cinfo->ds->buckets = buckets; 753 cinfo->ds->nbuckets = size; 754 for (i = 0; i < size; i++) { 755 INIT_LIST_HEAD(&buckets[i].written); 756 INIT_LIST_HEAD(&buckets[i].committing); 757 /* mark direct verifier as unset */ 758 buckets[i].direct_verf.committed = 759 NFS_INVALID_STABLE_HOW; 760 } 761 } 762 spin_unlock(cinfo->lock); 763 return 0; 764 } 765 } 766 767 static struct nfs4_pnfs_ds * 768 ff_layout_choose_best_ds_for_read(struct pnfs_layout_segment *lseg, 769 int start_idx, 770 int *best_idx) 771 { 772 struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); 773 struct nfs4_pnfs_ds *ds; 774 int idx; 775 776 /* mirrors are sorted by efficiency */ 777 for (idx = start_idx; idx < fls->mirror_array_cnt; idx++) { 778 ds = nfs4_ff_layout_prepare_ds(lseg, idx, false); 779 if (ds) { 780 *best_idx = idx; 781 return ds; 782 } 783 } 784 785 return NULL; 786 } 787 788 static void 789 ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio, 790 struct nfs_page *req) 791 { 792 struct nfs_pgio_mirror *pgm; 793 struct nfs4_ff_layout_mirror *mirror; 794 struct nfs4_pnfs_ds *ds; 795 int ds_idx; 796 797 /* Use full layout for now */ 798 if (!pgio->pg_lseg) { 799 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 800 req->wb_context, 801 0, 802 NFS4_MAX_UINT64, 803 IOMODE_READ, 804 GFP_KERNEL); 805 if (IS_ERR(pgio->pg_lseg)) { 806 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 807 pgio->pg_lseg = NULL; 808 return; 809 } 810 } 811 /* If no lseg, fall back to read through mds */ 812 if (pgio->pg_lseg == NULL) 813 goto out_mds; 814 815 ds = ff_layout_choose_best_ds_for_read(pgio->pg_lseg, 0, &ds_idx); 816 if (!ds) 817 goto out_mds; 818 mirror = FF_LAYOUT_COMP(pgio->pg_lseg, ds_idx); 819 820 pgio->pg_mirror_idx = ds_idx; 821 822 /* read always uses only one mirror - idx 0 for pgio layer */ 823 pgm = &pgio->pg_mirrors[0]; 824 pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].rsize; 825 826 return; 827 out_mds: 828 pnfs_put_lseg(pgio->pg_lseg); 829 pgio->pg_lseg = NULL; 830 nfs_pageio_reset_read_mds(pgio); 831 } 832 833 static void 834 ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio, 835 struct nfs_page *req) 836 { 837 struct nfs4_ff_layout_mirror *mirror; 838 struct nfs_pgio_mirror *pgm; 839 struct nfs_commit_info cinfo; 840 struct nfs4_pnfs_ds *ds; 841 int i; 842 int status; 843 844 if (!pgio->pg_lseg) { 845 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 846 req->wb_context, 847 0, 848 NFS4_MAX_UINT64, 849 IOMODE_RW, 850 GFP_NOFS); 851 if (IS_ERR(pgio->pg_lseg)) { 852 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 853 pgio->pg_lseg = NULL; 854 return; 855 } 856 } 857 /* If no lseg, fall back to write through mds */ 858 if (pgio->pg_lseg == NULL) 859 goto out_mds; 860 861 nfs_init_cinfo(&cinfo, pgio->pg_inode, pgio->pg_dreq); 862 status = ff_layout_alloc_commit_info(pgio->pg_lseg, &cinfo, GFP_NOFS); 863 if (status < 0) 864 goto out_mds; 865 866 /* Use a direct mapping of ds_idx to pgio mirror_idx */ 867 if (WARN_ON_ONCE(pgio->pg_mirror_count != 868 FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg))) 869 goto out_mds; 870 871 for (i = 0; i < pgio->pg_mirror_count; i++) { 872 ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, i, true); 873 if (!ds) 874 goto out_mds; 875 pgm = &pgio->pg_mirrors[i]; 876 mirror = FF_LAYOUT_COMP(pgio->pg_lseg, i); 877 pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].wsize; 878 } 879 880 return; 881 882 out_mds: 883 pnfs_put_lseg(pgio->pg_lseg); 884 pgio->pg_lseg = NULL; 885 nfs_pageio_reset_write_mds(pgio); 886 } 887 888 static unsigned int 889 ff_layout_pg_get_mirror_count_write(struct nfs_pageio_descriptor *pgio, 890 struct nfs_page *req) 891 { 892 if (!pgio->pg_lseg) { 893 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 894 req->wb_context, 895 0, 896 NFS4_MAX_UINT64, 897 IOMODE_RW, 898 GFP_NOFS); 899 if (IS_ERR(pgio->pg_lseg)) { 900 pgio->pg_error = PTR_ERR(pgio->pg_lseg); 901 pgio->pg_lseg = NULL; 902 goto out; 903 } 904 } 905 if (pgio->pg_lseg) 906 return FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg); 907 908 /* no lseg means that pnfs is not in use, so no mirroring here */ 909 nfs_pageio_reset_write_mds(pgio); 910 out: 911 return 1; 912 } 913 914 static const struct nfs_pageio_ops ff_layout_pg_read_ops = { 915 .pg_init = ff_layout_pg_init_read, 916 .pg_test = pnfs_generic_pg_test, 917 .pg_doio = pnfs_generic_pg_readpages, 918 .pg_cleanup = pnfs_generic_pg_cleanup, 919 }; 920 921 static const struct nfs_pageio_ops ff_layout_pg_write_ops = { 922 .pg_init = ff_layout_pg_init_write, 923 .pg_test = pnfs_generic_pg_test, 924 .pg_doio = pnfs_generic_pg_writepages, 925 .pg_get_mirror_count = ff_layout_pg_get_mirror_count_write, 926 .pg_cleanup = pnfs_generic_pg_cleanup, 927 }; 928 929 static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs) 930 { 931 struct rpc_task *task = &hdr->task; 932 933 pnfs_layoutcommit_inode(hdr->inode, false); 934 935 if (retry_pnfs) { 936 dprintk("%s Reset task %5u for i/o through pNFS " 937 "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, 938 hdr->task.tk_pid, 939 hdr->inode->i_sb->s_id, 940 (unsigned long long)NFS_FILEID(hdr->inode), 941 hdr->args.count, 942 (unsigned long long)hdr->args.offset); 943 944 hdr->completion_ops->reschedule_io(hdr); 945 return; 946 } 947 948 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 949 dprintk("%s Reset task %5u for i/o through MDS " 950 "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, 951 hdr->task.tk_pid, 952 hdr->inode->i_sb->s_id, 953 (unsigned long long)NFS_FILEID(hdr->inode), 954 hdr->args.count, 955 (unsigned long long)hdr->args.offset); 956 957 task->tk_status = pnfs_write_done_resend_to_mds(hdr); 958 } 959 } 960 961 static void ff_layout_reset_read(struct nfs_pgio_header *hdr) 962 { 963 struct rpc_task *task = &hdr->task; 964 965 pnfs_layoutcommit_inode(hdr->inode, false); 966 967 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 968 dprintk("%s Reset task %5u for i/o through MDS " 969 "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, 970 hdr->task.tk_pid, 971 hdr->inode->i_sb->s_id, 972 (unsigned long long)NFS_FILEID(hdr->inode), 973 hdr->args.count, 974 (unsigned long long)hdr->args.offset); 975 976 task->tk_status = pnfs_read_done_resend_to_mds(hdr); 977 } 978 } 979 980 static int ff_layout_async_handle_error_v4(struct rpc_task *task, 981 struct nfs4_state *state, 982 struct nfs_client *clp, 983 struct pnfs_layout_segment *lseg, 984 int idx) 985 { 986 struct pnfs_layout_hdr *lo = lseg->pls_layout; 987 struct inode *inode = lo->plh_inode; 988 struct nfs_server *mds_server = NFS_SERVER(inode); 989 990 struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx); 991 struct nfs_client *mds_client = mds_server->nfs_client; 992 struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table; 993 994 if (task->tk_status >= 0) 995 return 0; 996 997 switch (task->tk_status) { 998 /* MDS state errors */ 999 case -NFS4ERR_DELEG_REVOKED: 1000 case -NFS4ERR_ADMIN_REVOKED: 1001 case -NFS4ERR_BAD_STATEID: 1002 if (state == NULL) 1003 break; 1004 nfs_remove_bad_delegation(state->inode); 1005 case -NFS4ERR_OPENMODE: 1006 if (state == NULL) 1007 break; 1008 if (nfs4_schedule_stateid_recovery(mds_server, state) < 0) 1009 goto out_bad_stateid; 1010 goto wait_on_recovery; 1011 case -NFS4ERR_EXPIRED: 1012 if (state != NULL) { 1013 if (nfs4_schedule_stateid_recovery(mds_server, state) < 0) 1014 goto out_bad_stateid; 1015 } 1016 nfs4_schedule_lease_recovery(mds_client); 1017 goto wait_on_recovery; 1018 /* DS session errors */ 1019 case -NFS4ERR_BADSESSION: 1020 case -NFS4ERR_BADSLOT: 1021 case -NFS4ERR_BAD_HIGH_SLOT: 1022 case -NFS4ERR_DEADSESSION: 1023 case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: 1024 case -NFS4ERR_SEQ_FALSE_RETRY: 1025 case -NFS4ERR_SEQ_MISORDERED: 1026 dprintk("%s ERROR %d, Reset session. Exchangeid " 1027 "flags 0x%x\n", __func__, task->tk_status, 1028 clp->cl_exchange_flags); 1029 nfs4_schedule_session_recovery(clp->cl_session, task->tk_status); 1030 break; 1031 case -NFS4ERR_DELAY: 1032 case -NFS4ERR_GRACE: 1033 rpc_delay(task, FF_LAYOUT_POLL_RETRY_MAX); 1034 break; 1035 case -NFS4ERR_RETRY_UNCACHED_REP: 1036 break; 1037 /* Invalidate Layout errors */ 1038 case -NFS4ERR_PNFS_NO_LAYOUT: 1039 case -ESTALE: /* mapped NFS4ERR_STALE */ 1040 case -EBADHANDLE: /* mapped NFS4ERR_BADHANDLE */ 1041 case -EISDIR: /* mapped NFS4ERR_ISDIR */ 1042 case -NFS4ERR_FHEXPIRED: 1043 case -NFS4ERR_WRONG_TYPE: 1044 dprintk("%s Invalid layout error %d\n", __func__, 1045 task->tk_status); 1046 /* 1047 * Destroy layout so new i/o will get a new layout. 1048 * Layout will not be destroyed until all current lseg 1049 * references are put. Mark layout as invalid to resend failed 1050 * i/o and all i/o waiting on the slot table to the MDS until 1051 * layout is destroyed and a new valid layout is obtained. 1052 */ 1053 pnfs_destroy_layout(NFS_I(inode)); 1054 rpc_wake_up(&tbl->slot_tbl_waitq); 1055 goto reset; 1056 /* RPC connection errors */ 1057 case -ECONNREFUSED: 1058 case -EHOSTDOWN: 1059 case -EHOSTUNREACH: 1060 case -ENETUNREACH: 1061 case -EIO: 1062 case -ETIMEDOUT: 1063 case -EPIPE: 1064 dprintk("%s DS connection error %d\n", __func__, 1065 task->tk_status); 1066 nfs4_mark_deviceid_unavailable(devid); 1067 rpc_wake_up(&tbl->slot_tbl_waitq); 1068 /* fall through */ 1069 default: 1070 if (ff_layout_no_fallback_to_mds(lseg) || 1071 ff_layout_has_available_ds(lseg)) 1072 return -NFS4ERR_RESET_TO_PNFS; 1073 reset: 1074 dprintk("%s Retry through MDS. Error %d\n", __func__, 1075 task->tk_status); 1076 return -NFS4ERR_RESET_TO_MDS; 1077 } 1078 out: 1079 task->tk_status = 0; 1080 return -EAGAIN; 1081 out_bad_stateid: 1082 task->tk_status = -EIO; 1083 return 0; 1084 wait_on_recovery: 1085 rpc_sleep_on(&mds_client->cl_rpcwaitq, task, NULL); 1086 if (test_bit(NFS4CLNT_MANAGER_RUNNING, &mds_client->cl_state) == 0) 1087 rpc_wake_up_queued_task(&mds_client->cl_rpcwaitq, task); 1088 goto out; 1089 } 1090 1091 /* Retry all errors through either pNFS or MDS except for -EJUKEBOX */ 1092 static int ff_layout_async_handle_error_v3(struct rpc_task *task, 1093 struct pnfs_layout_segment *lseg, 1094 int idx) 1095 { 1096 struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx); 1097 1098 if (task->tk_status >= 0) 1099 return 0; 1100 1101 switch (task->tk_status) { 1102 /* File access problems. Don't mark the device as unavailable */ 1103 case -EACCES: 1104 case -ESTALE: 1105 case -EISDIR: 1106 case -EBADHANDLE: 1107 case -ELOOP: 1108 case -ENOSPC: 1109 break; 1110 case -EJUKEBOX: 1111 nfs_inc_stats(lseg->pls_layout->plh_inode, NFSIOS_DELAY); 1112 goto out_retry; 1113 default: 1114 dprintk("%s DS connection error %d\n", __func__, 1115 task->tk_status); 1116 nfs4_mark_deviceid_unavailable(devid); 1117 } 1118 /* FIXME: Need to prevent infinite looping here. */ 1119 return -NFS4ERR_RESET_TO_PNFS; 1120 out_retry: 1121 task->tk_status = 0; 1122 rpc_restart_call_prepare(task); 1123 rpc_delay(task, NFS_JUKEBOX_RETRY_TIME); 1124 return -EAGAIN; 1125 } 1126 1127 static int ff_layout_async_handle_error(struct rpc_task *task, 1128 struct nfs4_state *state, 1129 struct nfs_client *clp, 1130 struct pnfs_layout_segment *lseg, 1131 int idx) 1132 { 1133 int vers = clp->cl_nfs_mod->rpc_vers->number; 1134 1135 switch (vers) { 1136 case 3: 1137 return ff_layout_async_handle_error_v3(task, lseg, idx); 1138 case 4: 1139 return ff_layout_async_handle_error_v4(task, state, clp, 1140 lseg, idx); 1141 default: 1142 /* should never happen */ 1143 WARN_ON_ONCE(1); 1144 return 0; 1145 } 1146 } 1147 1148 static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, 1149 int idx, u64 offset, u64 length, 1150 u32 status, int opnum, int error) 1151 { 1152 struct nfs4_ff_layout_mirror *mirror; 1153 int err; 1154 1155 if (status == 0) { 1156 switch (error) { 1157 case -ETIMEDOUT: 1158 case -EPFNOSUPPORT: 1159 case -EPROTONOSUPPORT: 1160 case -EOPNOTSUPP: 1161 case -ECONNREFUSED: 1162 case -ECONNRESET: 1163 case -EHOSTDOWN: 1164 case -EHOSTUNREACH: 1165 case -ENETUNREACH: 1166 case -EADDRINUSE: 1167 case -ENOBUFS: 1168 case -EPIPE: 1169 case -EPERM: 1170 status = NFS4ERR_NXIO; 1171 break; 1172 case -EACCES: 1173 status = NFS4ERR_ACCESS; 1174 break; 1175 default: 1176 return; 1177 } 1178 } 1179 1180 switch (status) { 1181 case NFS4ERR_DELAY: 1182 case NFS4ERR_GRACE: 1183 return; 1184 default: 1185 break; 1186 } 1187 1188 mirror = FF_LAYOUT_COMP(lseg, idx); 1189 err = ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout), 1190 mirror, offset, length, status, opnum, 1191 GFP_NOIO); 1192 pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode, lseg); 1193 dprintk("%s: err %d op %d status %u\n", __func__, err, opnum, status); 1194 } 1195 1196 /* NFS_PROTO call done callback routines */ 1197 static int ff_layout_read_done_cb(struct rpc_task *task, 1198 struct nfs_pgio_header *hdr) 1199 { 1200 int err; 1201 1202 trace_nfs4_pnfs_read(hdr, task->tk_status); 1203 if (task->tk_status < 0) 1204 ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx, 1205 hdr->args.offset, hdr->args.count, 1206 hdr->res.op_status, OP_READ, 1207 task->tk_status); 1208 err = ff_layout_async_handle_error(task, hdr->args.context->state, 1209 hdr->ds_clp, hdr->lseg, 1210 hdr->pgio_mirror_idx); 1211 1212 switch (err) { 1213 case -NFS4ERR_RESET_TO_PNFS: 1214 if (ff_layout_choose_best_ds_for_read(hdr->lseg, 1215 hdr->pgio_mirror_idx + 1, 1216 &hdr->pgio_mirror_idx)) 1217 goto out_eagain; 1218 set_bit(NFS_LAYOUT_RETURN_REQUESTED, 1219 &hdr->lseg->pls_layout->plh_flags); 1220 pnfs_read_resend_pnfs(hdr); 1221 return task->tk_status; 1222 case -NFS4ERR_RESET_TO_MDS: 1223 ff_layout_reset_read(hdr); 1224 return task->tk_status; 1225 case -EAGAIN: 1226 goto out_eagain; 1227 } 1228 1229 return 0; 1230 out_eagain: 1231 rpc_restart_call_prepare(task); 1232 return -EAGAIN; 1233 } 1234 1235 static bool 1236 ff_layout_need_layoutcommit(struct pnfs_layout_segment *lseg) 1237 { 1238 return !(FF_LAYOUT_LSEG(lseg)->flags & FF_FLAGS_NO_LAYOUTCOMMIT); 1239 } 1240 1241 /* 1242 * We reference the rpc_cred of the first WRITE that triggers the need for 1243 * a LAYOUTCOMMIT, and use it to send the layoutcommit compound. 1244 * rfc5661 is not clear about which credential should be used. 1245 * 1246 * Flexlayout client should treat DS replied FILE_SYNC as DATA_SYNC, so 1247 * to follow http://www.rfc-editor.org/errata_search.php?rfc=5661&eid=2751 1248 * we always send layoutcommit after DS writes. 1249 */ 1250 static void 1251 ff_layout_set_layoutcommit(struct nfs_pgio_header *hdr) 1252 { 1253 if (!ff_layout_need_layoutcommit(hdr->lseg)) 1254 return; 1255 1256 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 1257 hdr->mds_offset + hdr->res.count); 1258 dprintk("%s inode %lu pls_end_pos %lu\n", __func__, hdr->inode->i_ino, 1259 (unsigned long) NFS_I(hdr->inode)->layout->plh_lwb); 1260 } 1261 1262 static bool 1263 ff_layout_reset_to_mds(struct pnfs_layout_segment *lseg, int idx) 1264 { 1265 /* No mirroring for now */ 1266 struct nfs4_deviceid_node *node = FF_LAYOUT_DEVID_NODE(lseg, idx); 1267 1268 return ff_layout_test_devid_unavailable(node); 1269 } 1270 1271 static void ff_layout_read_record_layoutstats_start(struct rpc_task *task, 1272 struct nfs_pgio_header *hdr) 1273 { 1274 if (test_and_set_bit(NFS_IOHDR_STAT, &hdr->flags)) 1275 return; 1276 nfs4_ff_layout_stat_io_start_read(hdr->inode, 1277 FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx), 1278 hdr->args.count, 1279 task->tk_start); 1280 } 1281 1282 static void ff_layout_read_record_layoutstats_done(struct rpc_task *task, 1283 struct nfs_pgio_header *hdr) 1284 { 1285 if (!test_and_clear_bit(NFS_IOHDR_STAT, &hdr->flags)) 1286 return; 1287 nfs4_ff_layout_stat_io_end_read(task, 1288 FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx), 1289 hdr->args.count, 1290 hdr->res.count); 1291 } 1292 1293 static int ff_layout_read_prepare_common(struct rpc_task *task, 1294 struct nfs_pgio_header *hdr) 1295 { 1296 if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) { 1297 rpc_exit(task, -EIO); 1298 return -EIO; 1299 } 1300 if (ff_layout_reset_to_mds(hdr->lseg, hdr->pgio_mirror_idx)) { 1301 dprintk("%s task %u reset io to MDS\n", __func__, task->tk_pid); 1302 if (ff_layout_has_available_ds(hdr->lseg)) 1303 pnfs_read_resend_pnfs(hdr); 1304 else 1305 ff_layout_reset_read(hdr); 1306 rpc_exit(task, 0); 1307 return -EAGAIN; 1308 } 1309 hdr->pgio_done_cb = ff_layout_read_done_cb; 1310 1311 ff_layout_read_record_layoutstats_start(task, hdr); 1312 return 0; 1313 } 1314 1315 /* 1316 * Call ops for the async read/write cases 1317 * In the case of dense layouts, the offset needs to be reset to its 1318 * original value. 1319 */ 1320 static void ff_layout_read_prepare_v3(struct rpc_task *task, void *data) 1321 { 1322 struct nfs_pgio_header *hdr = data; 1323 1324 if (ff_layout_read_prepare_common(task, hdr)) 1325 return; 1326 1327 rpc_call_start(task); 1328 } 1329 1330 static int ff_layout_setup_sequence(struct nfs_client *ds_clp, 1331 struct nfs4_sequence_args *args, 1332 struct nfs4_sequence_res *res, 1333 struct rpc_task *task) 1334 { 1335 if (ds_clp->cl_session) 1336 return nfs41_setup_sequence(ds_clp->cl_session, 1337 args, 1338 res, 1339 task); 1340 return nfs40_setup_sequence(ds_clp->cl_slot_tbl, 1341 args, 1342 res, 1343 task); 1344 } 1345 1346 static void ff_layout_read_prepare_v4(struct rpc_task *task, void *data) 1347 { 1348 struct nfs_pgio_header *hdr = data; 1349 1350 if (ff_layout_setup_sequence(hdr->ds_clp, 1351 &hdr->args.seq_args, 1352 &hdr->res.seq_res, 1353 task)) 1354 return; 1355 1356 if (ff_layout_read_prepare_common(task, hdr)) 1357 return; 1358 1359 if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context, 1360 hdr->args.lock_context, FMODE_READ) == -EIO) 1361 rpc_exit(task, -EIO); /* lost lock, terminate I/O */ 1362 } 1363 1364 static void ff_layout_read_call_done(struct rpc_task *task, void *data) 1365 { 1366 struct nfs_pgio_header *hdr = data; 1367 1368 dprintk("--> %s task->tk_status %d\n", __func__, task->tk_status); 1369 1370 if (test_bit(NFS_IOHDR_REDO, &hdr->flags) && 1371 task->tk_status == 0) { 1372 nfs4_sequence_done(task, &hdr->res.seq_res); 1373 return; 1374 } 1375 1376 /* Note this may cause RPC to be resent */ 1377 hdr->mds_ops->rpc_call_done(task, hdr); 1378 } 1379 1380 static void ff_layout_read_count_stats(struct rpc_task *task, void *data) 1381 { 1382 struct nfs_pgio_header *hdr = data; 1383 1384 ff_layout_read_record_layoutstats_done(task, hdr); 1385 rpc_count_iostats_metrics(task, 1386 &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_READ]); 1387 } 1388 1389 static void ff_layout_read_release(void *data) 1390 { 1391 struct nfs_pgio_header *hdr = data; 1392 1393 ff_layout_read_record_layoutstats_done(&hdr->task, hdr); 1394 pnfs_generic_rw_release(data); 1395 } 1396 1397 1398 static int ff_layout_write_done_cb(struct rpc_task *task, 1399 struct nfs_pgio_header *hdr) 1400 { 1401 int err; 1402 1403 trace_nfs4_pnfs_write(hdr, task->tk_status); 1404 if (task->tk_status < 0) 1405 ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx, 1406 hdr->args.offset, hdr->args.count, 1407 hdr->res.op_status, OP_WRITE, 1408 task->tk_status); 1409 err = ff_layout_async_handle_error(task, hdr->args.context->state, 1410 hdr->ds_clp, hdr->lseg, 1411 hdr->pgio_mirror_idx); 1412 1413 switch (err) { 1414 case -NFS4ERR_RESET_TO_PNFS: 1415 ff_layout_reset_write(hdr, true); 1416 return task->tk_status; 1417 case -NFS4ERR_RESET_TO_MDS: 1418 ff_layout_reset_write(hdr, false); 1419 return task->tk_status; 1420 case -EAGAIN: 1421 return -EAGAIN; 1422 } 1423 1424 if (hdr->res.verf->committed == NFS_FILE_SYNC || 1425 hdr->res.verf->committed == NFS_DATA_SYNC) 1426 ff_layout_set_layoutcommit(hdr); 1427 1428 /* zero out fattr since we don't care DS attr at all */ 1429 hdr->fattr.valid = 0; 1430 if (task->tk_status >= 0) 1431 nfs_writeback_update_inode(hdr); 1432 1433 return 0; 1434 } 1435 1436 static int ff_layout_commit_done_cb(struct rpc_task *task, 1437 struct nfs_commit_data *data) 1438 { 1439 int err; 1440 1441 trace_nfs4_pnfs_commit_ds(data, task->tk_status); 1442 if (task->tk_status < 0) 1443 ff_layout_io_track_ds_error(data->lseg, data->ds_commit_index, 1444 data->args.offset, data->args.count, 1445 data->res.op_status, OP_COMMIT, 1446 task->tk_status); 1447 err = ff_layout_async_handle_error(task, NULL, data->ds_clp, 1448 data->lseg, data->ds_commit_index); 1449 1450 switch (err) { 1451 case -NFS4ERR_RESET_TO_PNFS: 1452 pnfs_generic_prepare_to_resend_writes(data); 1453 return -EAGAIN; 1454 case -NFS4ERR_RESET_TO_MDS: 1455 pnfs_generic_prepare_to_resend_writes(data); 1456 return -EAGAIN; 1457 case -EAGAIN: 1458 rpc_restart_call_prepare(task); 1459 return -EAGAIN; 1460 } 1461 1462 if (data->verf.committed == NFS_UNSTABLE 1463 && ff_layout_need_layoutcommit(data->lseg)) 1464 pnfs_set_layoutcommit(data->inode, data->lseg, data->lwb); 1465 1466 return 0; 1467 } 1468 1469 static void ff_layout_write_record_layoutstats_start(struct rpc_task *task, 1470 struct nfs_pgio_header *hdr) 1471 { 1472 if (test_and_set_bit(NFS_IOHDR_STAT, &hdr->flags)) 1473 return; 1474 nfs4_ff_layout_stat_io_start_write(hdr->inode, 1475 FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx), 1476 hdr->args.count, 1477 task->tk_start); 1478 } 1479 1480 static void ff_layout_write_record_layoutstats_done(struct rpc_task *task, 1481 struct nfs_pgio_header *hdr) 1482 { 1483 if (!test_and_clear_bit(NFS_IOHDR_STAT, &hdr->flags)) 1484 return; 1485 nfs4_ff_layout_stat_io_end_write(task, 1486 FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx), 1487 hdr->args.count, hdr->res.count, 1488 hdr->res.verf->committed); 1489 } 1490 1491 static int ff_layout_write_prepare_common(struct rpc_task *task, 1492 struct nfs_pgio_header *hdr) 1493 { 1494 if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) { 1495 rpc_exit(task, -EIO); 1496 return -EIO; 1497 } 1498 1499 if (ff_layout_reset_to_mds(hdr->lseg, hdr->pgio_mirror_idx)) { 1500 bool retry_pnfs; 1501 1502 retry_pnfs = ff_layout_has_available_ds(hdr->lseg); 1503 dprintk("%s task %u reset io to %s\n", __func__, 1504 task->tk_pid, retry_pnfs ? "pNFS" : "MDS"); 1505 ff_layout_reset_write(hdr, retry_pnfs); 1506 rpc_exit(task, 0); 1507 return -EAGAIN; 1508 } 1509 1510 ff_layout_write_record_layoutstats_start(task, hdr); 1511 return 0; 1512 } 1513 1514 static void ff_layout_write_prepare_v3(struct rpc_task *task, void *data) 1515 { 1516 struct nfs_pgio_header *hdr = data; 1517 1518 if (ff_layout_write_prepare_common(task, hdr)) 1519 return; 1520 1521 rpc_call_start(task); 1522 } 1523 1524 static void ff_layout_write_prepare_v4(struct rpc_task *task, void *data) 1525 { 1526 struct nfs_pgio_header *hdr = data; 1527 1528 if (ff_layout_setup_sequence(hdr->ds_clp, 1529 &hdr->args.seq_args, 1530 &hdr->res.seq_res, 1531 task)) 1532 return; 1533 1534 if (ff_layout_write_prepare_common(task, hdr)) 1535 return; 1536 1537 if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context, 1538 hdr->args.lock_context, FMODE_WRITE) == -EIO) 1539 rpc_exit(task, -EIO); /* lost lock, terminate I/O */ 1540 } 1541 1542 static void ff_layout_write_call_done(struct rpc_task *task, void *data) 1543 { 1544 struct nfs_pgio_header *hdr = data; 1545 1546 if (test_bit(NFS_IOHDR_REDO, &hdr->flags) && 1547 task->tk_status == 0) { 1548 nfs4_sequence_done(task, &hdr->res.seq_res); 1549 return; 1550 } 1551 1552 /* Note this may cause RPC to be resent */ 1553 hdr->mds_ops->rpc_call_done(task, hdr); 1554 } 1555 1556 static void ff_layout_write_count_stats(struct rpc_task *task, void *data) 1557 { 1558 struct nfs_pgio_header *hdr = data; 1559 1560 ff_layout_write_record_layoutstats_done(task, hdr); 1561 rpc_count_iostats_metrics(task, 1562 &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_WRITE]); 1563 } 1564 1565 static void ff_layout_write_release(void *data) 1566 { 1567 struct nfs_pgio_header *hdr = data; 1568 1569 ff_layout_write_record_layoutstats_done(&hdr->task, hdr); 1570 pnfs_generic_rw_release(data); 1571 } 1572 1573 static void ff_layout_commit_record_layoutstats_start(struct rpc_task *task, 1574 struct nfs_commit_data *cdata) 1575 { 1576 if (test_and_set_bit(NFS_IOHDR_STAT, &cdata->flags)) 1577 return; 1578 nfs4_ff_layout_stat_io_start_write(cdata->inode, 1579 FF_LAYOUT_COMP(cdata->lseg, cdata->ds_commit_index), 1580 0, task->tk_start); 1581 } 1582 1583 static void ff_layout_commit_record_layoutstats_done(struct rpc_task *task, 1584 struct nfs_commit_data *cdata) 1585 { 1586 struct nfs_page *req; 1587 __u64 count = 0; 1588 1589 if (!test_and_clear_bit(NFS_IOHDR_STAT, &cdata->flags)) 1590 return; 1591 1592 if (task->tk_status == 0) { 1593 list_for_each_entry(req, &cdata->pages, wb_list) 1594 count += req->wb_bytes; 1595 } 1596 nfs4_ff_layout_stat_io_end_write(task, 1597 FF_LAYOUT_COMP(cdata->lseg, cdata->ds_commit_index), 1598 count, count, NFS_FILE_SYNC); 1599 } 1600 1601 static void ff_layout_commit_prepare_common(struct rpc_task *task, 1602 struct nfs_commit_data *cdata) 1603 { 1604 ff_layout_commit_record_layoutstats_start(task, cdata); 1605 } 1606 1607 static void ff_layout_commit_prepare_v3(struct rpc_task *task, void *data) 1608 { 1609 ff_layout_commit_prepare_common(task, data); 1610 rpc_call_start(task); 1611 } 1612 1613 static void ff_layout_commit_prepare_v4(struct rpc_task *task, void *data) 1614 { 1615 struct nfs_commit_data *wdata = data; 1616 1617 if (ff_layout_setup_sequence(wdata->ds_clp, 1618 &wdata->args.seq_args, 1619 &wdata->res.seq_res, 1620 task)) 1621 return; 1622 ff_layout_commit_prepare_common(task, data); 1623 } 1624 1625 static void ff_layout_commit_done(struct rpc_task *task, void *data) 1626 { 1627 pnfs_generic_write_commit_done(task, data); 1628 } 1629 1630 static void ff_layout_commit_count_stats(struct rpc_task *task, void *data) 1631 { 1632 struct nfs_commit_data *cdata = data; 1633 1634 ff_layout_commit_record_layoutstats_done(task, cdata); 1635 rpc_count_iostats_metrics(task, 1636 &NFS_CLIENT(cdata->inode)->cl_metrics[NFSPROC4_CLNT_COMMIT]); 1637 } 1638 1639 static void ff_layout_commit_release(void *data) 1640 { 1641 struct nfs_commit_data *cdata = data; 1642 1643 ff_layout_commit_record_layoutstats_done(&cdata->task, cdata); 1644 pnfs_generic_commit_release(data); 1645 } 1646 1647 static const struct rpc_call_ops ff_layout_read_call_ops_v3 = { 1648 .rpc_call_prepare = ff_layout_read_prepare_v3, 1649 .rpc_call_done = ff_layout_read_call_done, 1650 .rpc_count_stats = ff_layout_read_count_stats, 1651 .rpc_release = ff_layout_read_release, 1652 }; 1653 1654 static const struct rpc_call_ops ff_layout_read_call_ops_v4 = { 1655 .rpc_call_prepare = ff_layout_read_prepare_v4, 1656 .rpc_call_done = ff_layout_read_call_done, 1657 .rpc_count_stats = ff_layout_read_count_stats, 1658 .rpc_release = ff_layout_read_release, 1659 }; 1660 1661 static const struct rpc_call_ops ff_layout_write_call_ops_v3 = { 1662 .rpc_call_prepare = ff_layout_write_prepare_v3, 1663 .rpc_call_done = ff_layout_write_call_done, 1664 .rpc_count_stats = ff_layout_write_count_stats, 1665 .rpc_release = ff_layout_write_release, 1666 }; 1667 1668 static const struct rpc_call_ops ff_layout_write_call_ops_v4 = { 1669 .rpc_call_prepare = ff_layout_write_prepare_v4, 1670 .rpc_call_done = ff_layout_write_call_done, 1671 .rpc_count_stats = ff_layout_write_count_stats, 1672 .rpc_release = ff_layout_write_release, 1673 }; 1674 1675 static const struct rpc_call_ops ff_layout_commit_call_ops_v3 = { 1676 .rpc_call_prepare = ff_layout_commit_prepare_v3, 1677 .rpc_call_done = ff_layout_commit_done, 1678 .rpc_count_stats = ff_layout_commit_count_stats, 1679 .rpc_release = ff_layout_commit_release, 1680 }; 1681 1682 static const struct rpc_call_ops ff_layout_commit_call_ops_v4 = { 1683 .rpc_call_prepare = ff_layout_commit_prepare_v4, 1684 .rpc_call_done = ff_layout_commit_done, 1685 .rpc_count_stats = ff_layout_commit_count_stats, 1686 .rpc_release = ff_layout_commit_release, 1687 }; 1688 1689 static enum pnfs_try_status 1690 ff_layout_read_pagelist(struct nfs_pgio_header *hdr) 1691 { 1692 struct pnfs_layout_segment *lseg = hdr->lseg; 1693 struct nfs4_pnfs_ds *ds; 1694 struct rpc_clnt *ds_clnt; 1695 struct rpc_cred *ds_cred; 1696 loff_t offset = hdr->args.offset; 1697 u32 idx = hdr->pgio_mirror_idx; 1698 int vers; 1699 struct nfs_fh *fh; 1700 1701 dprintk("--> %s ino %lu pgbase %u req %Zu@%llu\n", 1702 __func__, hdr->inode->i_ino, 1703 hdr->args.pgbase, (size_t)hdr->args.count, offset); 1704 1705 ds = nfs4_ff_layout_prepare_ds(lseg, idx, false); 1706 if (!ds) 1707 goto out_failed; 1708 1709 ds_clnt = nfs4_ff_find_or_create_ds_client(lseg, idx, ds->ds_clp, 1710 hdr->inode); 1711 if (IS_ERR(ds_clnt)) 1712 goto out_failed; 1713 1714 ds_cred = ff_layout_get_ds_cred(lseg, idx, hdr->cred); 1715 if (IS_ERR(ds_cred)) 1716 goto out_failed; 1717 1718 vers = nfs4_ff_layout_ds_version(lseg, idx); 1719 1720 dprintk("%s USE DS: %s cl_count %d vers %d\n", __func__, 1721 ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count), vers); 1722 1723 atomic_inc(&ds->ds_clp->cl_count); 1724 hdr->ds_clp = ds->ds_clp; 1725 fh = nfs4_ff_layout_select_ds_fh(lseg, idx); 1726 if (fh) 1727 hdr->args.fh = fh; 1728 /* 1729 * Note that if we ever decide to split across DSes, 1730 * then we may need to handle dense-like offsets. 1731 */ 1732 hdr->args.offset = offset; 1733 hdr->mds_offset = offset; 1734 1735 /* Perform an asynchronous read to ds */ 1736 nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, 1737 vers == 3 ? &ff_layout_read_call_ops_v3 : 1738 &ff_layout_read_call_ops_v4, 1739 0, RPC_TASK_SOFTCONN); 1740 1741 return PNFS_ATTEMPTED; 1742 1743 out_failed: 1744 if (ff_layout_has_available_ds(lseg)) 1745 return PNFS_TRY_AGAIN; 1746 return PNFS_NOT_ATTEMPTED; 1747 } 1748 1749 /* Perform async writes. */ 1750 static enum pnfs_try_status 1751 ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) 1752 { 1753 struct pnfs_layout_segment *lseg = hdr->lseg; 1754 struct nfs4_pnfs_ds *ds; 1755 struct rpc_clnt *ds_clnt; 1756 struct rpc_cred *ds_cred; 1757 loff_t offset = hdr->args.offset; 1758 int vers; 1759 struct nfs_fh *fh; 1760 int idx = hdr->pgio_mirror_idx; 1761 1762 ds = nfs4_ff_layout_prepare_ds(lseg, idx, true); 1763 if (!ds) 1764 return PNFS_NOT_ATTEMPTED; 1765 1766 ds_clnt = nfs4_ff_find_or_create_ds_client(lseg, idx, ds->ds_clp, 1767 hdr->inode); 1768 if (IS_ERR(ds_clnt)) 1769 return PNFS_NOT_ATTEMPTED; 1770 1771 ds_cred = ff_layout_get_ds_cred(lseg, idx, hdr->cred); 1772 if (IS_ERR(ds_cred)) 1773 return PNFS_NOT_ATTEMPTED; 1774 1775 vers = nfs4_ff_layout_ds_version(lseg, idx); 1776 1777 dprintk("%s ino %lu sync %d req %Zu@%llu DS: %s cl_count %d vers %d\n", 1778 __func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count, 1779 offset, ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count), 1780 vers); 1781 1782 hdr->pgio_done_cb = ff_layout_write_done_cb; 1783 atomic_inc(&ds->ds_clp->cl_count); 1784 hdr->ds_clp = ds->ds_clp; 1785 hdr->ds_commit_idx = idx; 1786 fh = nfs4_ff_layout_select_ds_fh(lseg, idx); 1787 if (fh) 1788 hdr->args.fh = fh; 1789 1790 /* 1791 * Note that if we ever decide to split across DSes, 1792 * then we may need to handle dense-like offsets. 1793 */ 1794 hdr->args.offset = offset; 1795 1796 /* Perform an asynchronous write */ 1797 nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, 1798 vers == 3 ? &ff_layout_write_call_ops_v3 : 1799 &ff_layout_write_call_ops_v4, 1800 sync, RPC_TASK_SOFTCONN); 1801 return PNFS_ATTEMPTED; 1802 } 1803 1804 static u32 calc_ds_index_from_commit(struct pnfs_layout_segment *lseg, u32 i) 1805 { 1806 return i; 1807 } 1808 1809 static struct nfs_fh * 1810 select_ds_fh_from_commit(struct pnfs_layout_segment *lseg, u32 i) 1811 { 1812 struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg); 1813 1814 /* FIXME: Assume that there is only one NFS version available 1815 * for the DS. 1816 */ 1817 return &flseg->mirror_array[i]->fh_versions[0]; 1818 } 1819 1820 static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) 1821 { 1822 struct pnfs_layout_segment *lseg = data->lseg; 1823 struct nfs4_pnfs_ds *ds; 1824 struct rpc_clnt *ds_clnt; 1825 struct rpc_cred *ds_cred; 1826 u32 idx; 1827 int vers; 1828 struct nfs_fh *fh; 1829 1830 idx = calc_ds_index_from_commit(lseg, data->ds_commit_index); 1831 ds = nfs4_ff_layout_prepare_ds(lseg, idx, true); 1832 if (!ds) 1833 goto out_err; 1834 1835 ds_clnt = nfs4_ff_find_or_create_ds_client(lseg, idx, ds->ds_clp, 1836 data->inode); 1837 if (IS_ERR(ds_clnt)) 1838 goto out_err; 1839 1840 ds_cred = ff_layout_get_ds_cred(lseg, idx, data->cred); 1841 if (IS_ERR(ds_cred)) 1842 goto out_err; 1843 1844 vers = nfs4_ff_layout_ds_version(lseg, idx); 1845 1846 dprintk("%s ino %lu, how %d cl_count %d vers %d\n", __func__, 1847 data->inode->i_ino, how, atomic_read(&ds->ds_clp->cl_count), 1848 vers); 1849 data->commit_done_cb = ff_layout_commit_done_cb; 1850 data->cred = ds_cred; 1851 atomic_inc(&ds->ds_clp->cl_count); 1852 data->ds_clp = ds->ds_clp; 1853 fh = select_ds_fh_from_commit(lseg, data->ds_commit_index); 1854 if (fh) 1855 data->args.fh = fh; 1856 1857 return nfs_initiate_commit(ds_clnt, data, ds->ds_clp->rpc_ops, 1858 vers == 3 ? &ff_layout_commit_call_ops_v3 : 1859 &ff_layout_commit_call_ops_v4, 1860 how, RPC_TASK_SOFTCONN); 1861 out_err: 1862 pnfs_generic_prepare_to_resend_writes(data); 1863 pnfs_generic_commit_release(data); 1864 return -EAGAIN; 1865 } 1866 1867 static int 1868 ff_layout_commit_pagelist(struct inode *inode, struct list_head *mds_pages, 1869 int how, struct nfs_commit_info *cinfo) 1870 { 1871 return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo, 1872 ff_layout_initiate_commit); 1873 } 1874 1875 static struct pnfs_ds_commit_info * 1876 ff_layout_get_ds_info(struct inode *inode) 1877 { 1878 struct pnfs_layout_hdr *layout = NFS_I(inode)->layout; 1879 1880 if (layout == NULL) 1881 return NULL; 1882 1883 return &FF_LAYOUT_FROM_HDR(layout)->commit_info; 1884 } 1885 1886 static void 1887 ff_layout_free_deviceid_node(struct nfs4_deviceid_node *d) 1888 { 1889 nfs4_ff_layout_free_deviceid(container_of(d, struct nfs4_ff_layout_ds, 1890 id_node)); 1891 } 1892 1893 static int ff_layout_encode_ioerr(struct nfs4_flexfile_layout *flo, 1894 struct xdr_stream *xdr, 1895 const struct nfs4_layoutreturn_args *args) 1896 { 1897 struct pnfs_layout_hdr *hdr = &flo->generic_hdr; 1898 __be32 *start; 1899 int count = 0, ret = 0; 1900 1901 start = xdr_reserve_space(xdr, 4); 1902 if (unlikely(!start)) 1903 return -E2BIG; 1904 1905 /* This assume we always return _ALL_ layouts */ 1906 spin_lock(&hdr->plh_inode->i_lock); 1907 ret = ff_layout_encode_ds_ioerr(flo, xdr, &count, &args->range); 1908 spin_unlock(&hdr->plh_inode->i_lock); 1909 1910 *start = cpu_to_be32(count); 1911 1912 return ret; 1913 } 1914 1915 /* report nothing for now */ 1916 static void ff_layout_encode_iostats(struct nfs4_flexfile_layout *flo, 1917 struct xdr_stream *xdr, 1918 const struct nfs4_layoutreturn_args *args) 1919 { 1920 __be32 *p; 1921 1922 p = xdr_reserve_space(xdr, 4); 1923 if (likely(p)) 1924 *p = cpu_to_be32(0); 1925 } 1926 1927 static struct nfs4_deviceid_node * 1928 ff_layout_alloc_deviceid_node(struct nfs_server *server, 1929 struct pnfs_device *pdev, gfp_t gfp_flags) 1930 { 1931 struct nfs4_ff_layout_ds *dsaddr; 1932 1933 dsaddr = nfs4_ff_alloc_deviceid_node(server, pdev, gfp_flags); 1934 if (!dsaddr) 1935 return NULL; 1936 return &dsaddr->id_node; 1937 } 1938 1939 static void 1940 ff_layout_encode_layoutreturn(struct pnfs_layout_hdr *lo, 1941 struct xdr_stream *xdr, 1942 const struct nfs4_layoutreturn_args *args) 1943 { 1944 struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo); 1945 __be32 *start; 1946 1947 dprintk("%s: Begin\n", __func__); 1948 start = xdr_reserve_space(xdr, 4); 1949 BUG_ON(!start); 1950 1951 ff_layout_encode_ioerr(flo, xdr, args); 1952 ff_layout_encode_iostats(flo, xdr, args); 1953 1954 *start = cpu_to_be32((xdr->p - start - 1) * 4); 1955 dprintk("%s: Return\n", __func__); 1956 } 1957 1958 static int 1959 ff_layout_ntop4(const struct sockaddr *sap, char *buf, const size_t buflen) 1960 { 1961 const struct sockaddr_in *sin = (struct sockaddr_in *)sap; 1962 1963 return snprintf(buf, buflen, "%pI4", &sin->sin_addr); 1964 } 1965 1966 static size_t 1967 ff_layout_ntop6_noscopeid(const struct sockaddr *sap, char *buf, 1968 const int buflen) 1969 { 1970 const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap; 1971 const struct in6_addr *addr = &sin6->sin6_addr; 1972 1973 /* 1974 * RFC 4291, Section 2.2.2 1975 * 1976 * Shorthanded ANY address 1977 */ 1978 if (ipv6_addr_any(addr)) 1979 return snprintf(buf, buflen, "::"); 1980 1981 /* 1982 * RFC 4291, Section 2.2.2 1983 * 1984 * Shorthanded loopback address 1985 */ 1986 if (ipv6_addr_loopback(addr)) 1987 return snprintf(buf, buflen, "::1"); 1988 1989 /* 1990 * RFC 4291, Section 2.2.3 1991 * 1992 * Special presentation address format for mapped v4 1993 * addresses. 1994 */ 1995 if (ipv6_addr_v4mapped(addr)) 1996 return snprintf(buf, buflen, "::ffff:%pI4", 1997 &addr->s6_addr32[3]); 1998 1999 /* 2000 * RFC 4291, Section 2.2.1 2001 */ 2002 return snprintf(buf, buflen, "%pI6c", addr); 2003 } 2004 2005 /* Derived from rpc_sockaddr2uaddr */ 2006 static void 2007 ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da) 2008 { 2009 struct sockaddr *sap = (struct sockaddr *)&da->da_addr; 2010 char portbuf[RPCBIND_MAXUADDRPLEN]; 2011 char addrbuf[RPCBIND_MAXUADDRLEN]; 2012 char *netid; 2013 unsigned short port; 2014 int len, netid_len; 2015 __be32 *p; 2016 2017 switch (sap->sa_family) { 2018 case AF_INET: 2019 if (ff_layout_ntop4(sap, addrbuf, sizeof(addrbuf)) == 0) 2020 return; 2021 port = ntohs(((struct sockaddr_in *)sap)->sin_port); 2022 netid = "tcp"; 2023 netid_len = 3; 2024 break; 2025 case AF_INET6: 2026 if (ff_layout_ntop6_noscopeid(sap, addrbuf, sizeof(addrbuf)) == 0) 2027 return; 2028 port = ntohs(((struct sockaddr_in6 *)sap)->sin6_port); 2029 netid = "tcp6"; 2030 netid_len = 4; 2031 break; 2032 default: 2033 /* we only support tcp and tcp6 */ 2034 WARN_ON_ONCE(1); 2035 return; 2036 } 2037 2038 snprintf(portbuf, sizeof(portbuf), ".%u.%u", port >> 8, port & 0xff); 2039 len = strlcat(addrbuf, portbuf, sizeof(addrbuf)); 2040 2041 p = xdr_reserve_space(xdr, 4 + netid_len); 2042 xdr_encode_opaque(p, netid, netid_len); 2043 2044 p = xdr_reserve_space(xdr, 4 + len); 2045 xdr_encode_opaque(p, addrbuf, len); 2046 } 2047 2048 static void 2049 ff_layout_encode_nfstime(struct xdr_stream *xdr, 2050 ktime_t t) 2051 { 2052 struct timespec64 ts; 2053 __be32 *p; 2054 2055 p = xdr_reserve_space(xdr, 12); 2056 ts = ktime_to_timespec64(t); 2057 p = xdr_encode_hyper(p, ts.tv_sec); 2058 *p++ = cpu_to_be32(ts.tv_nsec); 2059 } 2060 2061 static void 2062 ff_layout_encode_io_latency(struct xdr_stream *xdr, 2063 struct nfs4_ff_io_stat *stat) 2064 { 2065 __be32 *p; 2066 2067 p = xdr_reserve_space(xdr, 5 * 8); 2068 p = xdr_encode_hyper(p, stat->ops_requested); 2069 p = xdr_encode_hyper(p, stat->bytes_requested); 2070 p = xdr_encode_hyper(p, stat->ops_completed); 2071 p = xdr_encode_hyper(p, stat->bytes_completed); 2072 p = xdr_encode_hyper(p, stat->bytes_not_delivered); 2073 ff_layout_encode_nfstime(xdr, stat->total_busy_time); 2074 ff_layout_encode_nfstime(xdr, stat->aggregate_completion_time); 2075 } 2076 2077 static void 2078 ff_layout_encode_layoutstats(struct xdr_stream *xdr, 2079 struct nfs42_layoutstat_args *args, 2080 struct nfs42_layoutstat_devinfo *devinfo) 2081 { 2082 struct nfs4_ff_layout_mirror *mirror = devinfo->layout_private; 2083 struct nfs4_pnfs_ds_addr *da; 2084 struct nfs4_pnfs_ds *ds = mirror->mirror_ds->ds; 2085 struct nfs_fh *fh = &mirror->fh_versions[0]; 2086 __be32 *p, *start; 2087 2088 da = list_first_entry(&ds->ds_addrs, struct nfs4_pnfs_ds_addr, da_node); 2089 dprintk("%s: DS %s: encoding address %s\n", 2090 __func__, ds->ds_remotestr, da->da_remotestr); 2091 /* layoutupdate length */ 2092 start = xdr_reserve_space(xdr, 4); 2093 /* netaddr4 */ 2094 ff_layout_encode_netaddr(xdr, da); 2095 /* nfs_fh4 */ 2096 p = xdr_reserve_space(xdr, 4 + fh->size); 2097 xdr_encode_opaque(p, fh->data, fh->size); 2098 /* ff_io_latency4 read */ 2099 spin_lock(&mirror->lock); 2100 ff_layout_encode_io_latency(xdr, &mirror->read_stat.io_stat); 2101 /* ff_io_latency4 write */ 2102 ff_layout_encode_io_latency(xdr, &mirror->write_stat.io_stat); 2103 spin_unlock(&mirror->lock); 2104 /* nfstime4 */ 2105 ff_layout_encode_nfstime(xdr, ktime_sub(ktime_get(), mirror->start_time)); 2106 /* bool */ 2107 p = xdr_reserve_space(xdr, 4); 2108 *p = cpu_to_be32(false); 2109 2110 *start = cpu_to_be32((xdr->p - start - 1) * 4); 2111 } 2112 2113 static int 2114 ff_layout_mirror_prepare_stats(struct nfs42_layoutstat_args *args, 2115 struct pnfs_layout_hdr *lo, 2116 int dev_limit) 2117 { 2118 struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo); 2119 struct nfs4_ff_layout_mirror *mirror; 2120 struct nfs4_deviceid_node *dev; 2121 struct nfs42_layoutstat_devinfo *devinfo; 2122 int i = 0; 2123 2124 list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) { 2125 if (i >= dev_limit) 2126 break; 2127 if (!mirror->mirror_ds) 2128 continue; 2129 /* mirror refcount put in cleanup_layoutstats */ 2130 if (!atomic_inc_not_zero(&mirror->ref)) 2131 continue; 2132 dev = &mirror->mirror_ds->id_node; 2133 devinfo = &args->devinfo[i]; 2134 memcpy(&devinfo->dev_id, &dev->deviceid, NFS4_DEVICEID4_SIZE); 2135 devinfo->offset = 0; 2136 devinfo->length = NFS4_MAX_UINT64; 2137 devinfo->read_count = mirror->read_stat.io_stat.ops_completed; 2138 devinfo->read_bytes = mirror->read_stat.io_stat.bytes_completed; 2139 devinfo->write_count = mirror->write_stat.io_stat.ops_completed; 2140 devinfo->write_bytes = mirror->write_stat.io_stat.bytes_completed; 2141 devinfo->layout_type = LAYOUT_FLEX_FILES; 2142 devinfo->layoutstats_encode = ff_layout_encode_layoutstats; 2143 devinfo->layout_private = mirror; 2144 2145 i++; 2146 } 2147 return i; 2148 } 2149 2150 static int 2151 ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args) 2152 { 2153 struct nfs4_flexfile_layout *ff_layout; 2154 struct nfs4_ff_layout_mirror *mirror; 2155 int dev_count = 0; 2156 2157 spin_lock(&args->inode->i_lock); 2158 ff_layout = FF_LAYOUT_FROM_HDR(NFS_I(args->inode)->layout); 2159 list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) { 2160 if (atomic_read(&mirror->ref) != 0) 2161 dev_count ++; 2162 } 2163 spin_unlock(&args->inode->i_lock); 2164 /* For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics */ 2165 if (dev_count > PNFS_LAYOUTSTATS_MAXDEV) { 2166 dprintk("%s: truncating devinfo to limit (%d:%d)\n", 2167 __func__, dev_count, PNFS_LAYOUTSTATS_MAXDEV); 2168 dev_count = PNFS_LAYOUTSTATS_MAXDEV; 2169 } 2170 args->devinfo = kmalloc_array(dev_count, sizeof(*args->devinfo), GFP_NOIO); 2171 if (!args->devinfo) 2172 return -ENOMEM; 2173 2174 spin_lock(&args->inode->i_lock); 2175 args->num_dev = ff_layout_mirror_prepare_stats(args, 2176 &ff_layout->generic_hdr, dev_count); 2177 spin_unlock(&args->inode->i_lock); 2178 2179 return 0; 2180 } 2181 2182 static void 2183 ff_layout_cleanup_layoutstats(struct nfs42_layoutstat_data *data) 2184 { 2185 struct nfs4_ff_layout_mirror *mirror; 2186 int i; 2187 2188 for (i = 0; i < data->args.num_dev; i++) { 2189 mirror = data->args.devinfo[i].layout_private; 2190 data->args.devinfo[i].layout_private = NULL; 2191 ff_layout_put_mirror(mirror); 2192 } 2193 } 2194 2195 static struct pnfs_layoutdriver_type flexfilelayout_type = { 2196 .id = LAYOUT_FLEX_FILES, 2197 .name = "LAYOUT_FLEX_FILES", 2198 .owner = THIS_MODULE, 2199 .alloc_layout_hdr = ff_layout_alloc_layout_hdr, 2200 .free_layout_hdr = ff_layout_free_layout_hdr, 2201 .alloc_lseg = ff_layout_alloc_lseg, 2202 .free_lseg = ff_layout_free_lseg, 2203 .add_lseg = ff_layout_add_lseg, 2204 .pg_read_ops = &ff_layout_pg_read_ops, 2205 .pg_write_ops = &ff_layout_pg_write_ops, 2206 .get_ds_info = ff_layout_get_ds_info, 2207 .free_deviceid_node = ff_layout_free_deviceid_node, 2208 .mark_request_commit = pnfs_layout_mark_request_commit, 2209 .clear_request_commit = pnfs_generic_clear_request_commit, 2210 .scan_commit_lists = pnfs_generic_scan_commit_lists, 2211 .recover_commit_reqs = pnfs_generic_recover_commit_reqs, 2212 .commit_pagelist = ff_layout_commit_pagelist, 2213 .read_pagelist = ff_layout_read_pagelist, 2214 .write_pagelist = ff_layout_write_pagelist, 2215 .alloc_deviceid_node = ff_layout_alloc_deviceid_node, 2216 .encode_layoutreturn = ff_layout_encode_layoutreturn, 2217 .sync = pnfs_nfs_generic_sync, 2218 .prepare_layoutstats = ff_layout_prepare_layoutstats, 2219 .cleanup_layoutstats = ff_layout_cleanup_layoutstats, 2220 }; 2221 2222 static int __init nfs4flexfilelayout_init(void) 2223 { 2224 printk(KERN_INFO "%s: NFSv4 Flexfile Layout Driver Registering...\n", 2225 __func__); 2226 return pnfs_register_layoutdriver(&flexfilelayout_type); 2227 } 2228 2229 static void __exit nfs4flexfilelayout_exit(void) 2230 { 2231 printk(KERN_INFO "%s: NFSv4 Flexfile Layout Driver Unregistering...\n", 2232 __func__); 2233 pnfs_unregister_layoutdriver(&flexfilelayout_type); 2234 } 2235 2236 MODULE_ALIAS("nfs-layouttype4-4"); 2237 2238 MODULE_LICENSE("GPL"); 2239 MODULE_DESCRIPTION("The NFSv4 flexfile layout driver"); 2240 2241 module_init(nfs4flexfilelayout_init); 2242 module_exit(nfs4flexfilelayout_exit); 2243