1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/net/sunrpc/xdr.c 4 * 5 * Generic XDR support. 6 * 7 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> 8 */ 9 10 #include <linux/module.h> 11 #include <linux/slab.h> 12 #include <linux/types.h> 13 #include <linux/string.h> 14 #include <linux/kernel.h> 15 #include <linux/pagemap.h> 16 #include <linux/errno.h> 17 #include <linux/sunrpc/xdr.h> 18 #include <linux/sunrpc/msg_prot.h> 19 #include <linux/bvec.h> 20 #include <trace/events/sunrpc.h> 21 22 static void _copy_to_pages(struct page **, size_t, const char *, size_t); 23 24 25 /* 26 * XDR functions for basic NFS types 27 */ 28 __be32 * 29 xdr_encode_netobj(__be32 *p, const struct xdr_netobj *obj) 30 { 31 unsigned int quadlen = XDR_QUADLEN(obj->len); 32 33 p[quadlen] = 0; /* zero trailing bytes */ 34 *p++ = cpu_to_be32(obj->len); 35 memcpy(p, obj->data, obj->len); 36 return p + XDR_QUADLEN(obj->len); 37 } 38 EXPORT_SYMBOL_GPL(xdr_encode_netobj); 39 40 __be32 * 41 xdr_decode_netobj(__be32 *p, struct xdr_netobj *obj) 42 { 43 unsigned int len; 44 45 if ((len = be32_to_cpu(*p++)) > XDR_MAX_NETOBJ) 46 return NULL; 47 obj->len = len; 48 obj->data = (u8 *) p; 49 return p + XDR_QUADLEN(len); 50 } 51 EXPORT_SYMBOL_GPL(xdr_decode_netobj); 52 53 /** 54 * xdr_encode_opaque_fixed - Encode fixed length opaque data 55 * @p: pointer to current position in XDR buffer. 56 * @ptr: pointer to data to encode (or NULL) 57 * @nbytes: size of data. 58 * 59 * Copy the array of data of length nbytes at ptr to the XDR buffer 60 * at position p, then align to the next 32-bit boundary by padding 61 * with zero bytes (see RFC1832). 62 * Note: if ptr is NULL, only the padding is performed. 63 * 64 * Returns the updated current XDR buffer position 65 * 66 */ 67 __be32 *xdr_encode_opaque_fixed(__be32 *p, const void *ptr, unsigned int nbytes) 68 { 69 if (likely(nbytes != 0)) { 70 unsigned int quadlen = XDR_QUADLEN(nbytes); 71 unsigned int padding = (quadlen << 2) - nbytes; 72 73 if (ptr != NULL) 74 memcpy(p, ptr, nbytes); 75 if (padding != 0) 76 memset((char *)p + nbytes, 0, padding); 77 p += quadlen; 78 } 79 return p; 80 } 81 EXPORT_SYMBOL_GPL(xdr_encode_opaque_fixed); 82 83 /** 84 * xdr_encode_opaque - Encode variable length opaque data 85 * @p: pointer to current position in XDR buffer. 86 * @ptr: pointer to data to encode (or NULL) 87 * @nbytes: size of data. 88 * 89 * Returns the updated current XDR buffer position 90 */ 91 __be32 *xdr_encode_opaque(__be32 *p, const void *ptr, unsigned int nbytes) 92 { 93 *p++ = cpu_to_be32(nbytes); 94 return xdr_encode_opaque_fixed(p, ptr, nbytes); 95 } 96 EXPORT_SYMBOL_GPL(xdr_encode_opaque); 97 98 __be32 * 99 xdr_encode_string(__be32 *p, const char *string) 100 { 101 return xdr_encode_array(p, string, strlen(string)); 102 } 103 EXPORT_SYMBOL_GPL(xdr_encode_string); 104 105 __be32 * 106 xdr_decode_string_inplace(__be32 *p, char **sp, 107 unsigned int *lenp, unsigned int maxlen) 108 { 109 u32 len; 110 111 len = be32_to_cpu(*p++); 112 if (len > maxlen) 113 return NULL; 114 *lenp = len; 115 *sp = (char *) p; 116 return p + XDR_QUADLEN(len); 117 } 118 EXPORT_SYMBOL_GPL(xdr_decode_string_inplace); 119 120 /** 121 * xdr_terminate_string - '\0'-terminate a string residing in an xdr_buf 122 * @buf: XDR buffer where string resides 123 * @len: length of string, in bytes 124 * 125 */ 126 void xdr_terminate_string(const struct xdr_buf *buf, const u32 len) 127 { 128 char *kaddr; 129 130 kaddr = kmap_atomic(buf->pages[0]); 131 kaddr[buf->page_base + len] = '\0'; 132 kunmap_atomic(kaddr); 133 } 134 EXPORT_SYMBOL_GPL(xdr_terminate_string); 135 136 size_t xdr_buf_pagecount(const struct xdr_buf *buf) 137 { 138 if (!buf->page_len) 139 return 0; 140 return (buf->page_base + buf->page_len + PAGE_SIZE - 1) >> PAGE_SHIFT; 141 } 142 143 int 144 xdr_alloc_bvec(struct xdr_buf *buf, gfp_t gfp) 145 { 146 size_t i, n = xdr_buf_pagecount(buf); 147 148 if (n != 0 && buf->bvec == NULL) { 149 buf->bvec = kmalloc_array(n, sizeof(buf->bvec[0]), gfp); 150 if (!buf->bvec) 151 return -ENOMEM; 152 for (i = 0; i < n; i++) { 153 bvec_set_page(&buf->bvec[i], buf->pages[i], PAGE_SIZE, 154 0); 155 } 156 } 157 return 0; 158 } 159 160 void 161 xdr_free_bvec(struct xdr_buf *buf) 162 { 163 kfree(buf->bvec); 164 buf->bvec = NULL; 165 } 166 167 /** 168 * xdr_inline_pages - Prepare receive buffer for a large reply 169 * @xdr: xdr_buf into which reply will be placed 170 * @offset: expected offset where data payload will start, in bytes 171 * @pages: vector of struct page pointers 172 * @base: offset in first page where receive should start, in bytes 173 * @len: expected size of the upper layer data payload, in bytes 174 * 175 */ 176 void 177 xdr_inline_pages(struct xdr_buf *xdr, unsigned int offset, 178 struct page **pages, unsigned int base, unsigned int len) 179 { 180 struct kvec *head = xdr->head; 181 struct kvec *tail = xdr->tail; 182 char *buf = (char *)head->iov_base; 183 unsigned int buflen = head->iov_len; 184 185 head->iov_len = offset; 186 187 xdr->pages = pages; 188 xdr->page_base = base; 189 xdr->page_len = len; 190 191 tail->iov_base = buf + offset; 192 tail->iov_len = buflen - offset; 193 xdr->buflen += len; 194 } 195 EXPORT_SYMBOL_GPL(xdr_inline_pages); 196 197 /* 198 * Helper routines for doing 'memmove' like operations on a struct xdr_buf 199 */ 200 201 /** 202 * _shift_data_left_pages 203 * @pages: vector of pages containing both the source and dest memory area. 204 * @pgto_base: page vector address of destination 205 * @pgfrom_base: page vector address of source 206 * @len: number of bytes to copy 207 * 208 * Note: the addresses pgto_base and pgfrom_base are both calculated in 209 * the same way: 210 * if a memory area starts at byte 'base' in page 'pages[i]', 211 * then its address is given as (i << PAGE_CACHE_SHIFT) + base 212 * Alse note: pgto_base must be < pgfrom_base, but the memory areas 213 * they point to may overlap. 214 */ 215 static void 216 _shift_data_left_pages(struct page **pages, size_t pgto_base, 217 size_t pgfrom_base, size_t len) 218 { 219 struct page **pgfrom, **pgto; 220 char *vfrom, *vto; 221 size_t copy; 222 223 BUG_ON(pgfrom_base <= pgto_base); 224 225 if (!len) 226 return; 227 228 pgto = pages + (pgto_base >> PAGE_SHIFT); 229 pgfrom = pages + (pgfrom_base >> PAGE_SHIFT); 230 231 pgto_base &= ~PAGE_MASK; 232 pgfrom_base &= ~PAGE_MASK; 233 234 do { 235 if (pgto_base >= PAGE_SIZE) { 236 pgto_base = 0; 237 pgto++; 238 } 239 if (pgfrom_base >= PAGE_SIZE){ 240 pgfrom_base = 0; 241 pgfrom++; 242 } 243 244 copy = len; 245 if (copy > (PAGE_SIZE - pgto_base)) 246 copy = PAGE_SIZE - pgto_base; 247 if (copy > (PAGE_SIZE - pgfrom_base)) 248 copy = PAGE_SIZE - pgfrom_base; 249 250 vto = kmap_atomic(*pgto); 251 if (*pgto != *pgfrom) { 252 vfrom = kmap_atomic(*pgfrom); 253 memcpy(vto + pgto_base, vfrom + pgfrom_base, copy); 254 kunmap_atomic(vfrom); 255 } else 256 memmove(vto + pgto_base, vto + pgfrom_base, copy); 257 flush_dcache_page(*pgto); 258 kunmap_atomic(vto); 259 260 pgto_base += copy; 261 pgfrom_base += copy; 262 263 } while ((len -= copy) != 0); 264 } 265 266 /** 267 * _shift_data_right_pages 268 * @pages: vector of pages containing both the source and dest memory area. 269 * @pgto_base: page vector address of destination 270 * @pgfrom_base: page vector address of source 271 * @len: number of bytes to copy 272 * 273 * Note: the addresses pgto_base and pgfrom_base are both calculated in 274 * the same way: 275 * if a memory area starts at byte 'base' in page 'pages[i]', 276 * then its address is given as (i << PAGE_SHIFT) + base 277 * Also note: pgfrom_base must be < pgto_base, but the memory areas 278 * they point to may overlap. 279 */ 280 static void 281 _shift_data_right_pages(struct page **pages, size_t pgto_base, 282 size_t pgfrom_base, size_t len) 283 { 284 struct page **pgfrom, **pgto; 285 char *vfrom, *vto; 286 size_t copy; 287 288 BUG_ON(pgto_base <= pgfrom_base); 289 290 if (!len) 291 return; 292 293 pgto_base += len; 294 pgfrom_base += len; 295 296 pgto = pages + (pgto_base >> PAGE_SHIFT); 297 pgfrom = pages + (pgfrom_base >> PAGE_SHIFT); 298 299 pgto_base &= ~PAGE_MASK; 300 pgfrom_base &= ~PAGE_MASK; 301 302 do { 303 /* Are any pointers crossing a page boundary? */ 304 if (pgto_base == 0) { 305 pgto_base = PAGE_SIZE; 306 pgto--; 307 } 308 if (pgfrom_base == 0) { 309 pgfrom_base = PAGE_SIZE; 310 pgfrom--; 311 } 312 313 copy = len; 314 if (copy > pgto_base) 315 copy = pgto_base; 316 if (copy > pgfrom_base) 317 copy = pgfrom_base; 318 pgto_base -= copy; 319 pgfrom_base -= copy; 320 321 vto = kmap_atomic(*pgto); 322 if (*pgto != *pgfrom) { 323 vfrom = kmap_atomic(*pgfrom); 324 memcpy(vto + pgto_base, vfrom + pgfrom_base, copy); 325 kunmap_atomic(vfrom); 326 } else 327 memmove(vto + pgto_base, vto + pgfrom_base, copy); 328 flush_dcache_page(*pgto); 329 kunmap_atomic(vto); 330 331 } while ((len -= copy) != 0); 332 } 333 334 /** 335 * _copy_to_pages 336 * @pages: array of pages 337 * @pgbase: page vector address of destination 338 * @p: pointer to source data 339 * @len: length 340 * 341 * Copies data from an arbitrary memory location into an array of pages 342 * The copy is assumed to be non-overlapping. 343 */ 344 static void 345 _copy_to_pages(struct page **pages, size_t pgbase, const char *p, size_t len) 346 { 347 struct page **pgto; 348 char *vto; 349 size_t copy; 350 351 if (!len) 352 return; 353 354 pgto = pages + (pgbase >> PAGE_SHIFT); 355 pgbase &= ~PAGE_MASK; 356 357 for (;;) { 358 copy = PAGE_SIZE - pgbase; 359 if (copy > len) 360 copy = len; 361 362 vto = kmap_atomic(*pgto); 363 memcpy(vto + pgbase, p, copy); 364 kunmap_atomic(vto); 365 366 len -= copy; 367 if (len == 0) 368 break; 369 370 pgbase += copy; 371 if (pgbase == PAGE_SIZE) { 372 flush_dcache_page(*pgto); 373 pgbase = 0; 374 pgto++; 375 } 376 p += copy; 377 } 378 flush_dcache_page(*pgto); 379 } 380 381 /** 382 * _copy_from_pages 383 * @p: pointer to destination 384 * @pages: array of pages 385 * @pgbase: offset of source data 386 * @len: length 387 * 388 * Copies data into an arbitrary memory location from an array of pages 389 * The copy is assumed to be non-overlapping. 390 */ 391 void 392 _copy_from_pages(char *p, struct page **pages, size_t pgbase, size_t len) 393 { 394 struct page **pgfrom; 395 char *vfrom; 396 size_t copy; 397 398 if (!len) 399 return; 400 401 pgfrom = pages + (pgbase >> PAGE_SHIFT); 402 pgbase &= ~PAGE_MASK; 403 404 do { 405 copy = PAGE_SIZE - pgbase; 406 if (copy > len) 407 copy = len; 408 409 vfrom = kmap_atomic(*pgfrom); 410 memcpy(p, vfrom + pgbase, copy); 411 kunmap_atomic(vfrom); 412 413 pgbase += copy; 414 if (pgbase == PAGE_SIZE) { 415 pgbase = 0; 416 pgfrom++; 417 } 418 p += copy; 419 420 } while ((len -= copy) != 0); 421 } 422 EXPORT_SYMBOL_GPL(_copy_from_pages); 423 424 static void xdr_buf_iov_zero(const struct kvec *iov, unsigned int base, 425 unsigned int len) 426 { 427 if (base >= iov->iov_len) 428 return; 429 if (len > iov->iov_len - base) 430 len = iov->iov_len - base; 431 memset(iov->iov_base + base, 0, len); 432 } 433 434 /** 435 * xdr_buf_pages_zero 436 * @buf: xdr_buf 437 * @pgbase: beginning offset 438 * @len: length 439 */ 440 static void xdr_buf_pages_zero(const struct xdr_buf *buf, unsigned int pgbase, 441 unsigned int len) 442 { 443 struct page **pages = buf->pages; 444 struct page **page; 445 char *vpage; 446 unsigned int zero; 447 448 if (!len) 449 return; 450 if (pgbase >= buf->page_len) { 451 xdr_buf_iov_zero(buf->tail, pgbase - buf->page_len, len); 452 return; 453 } 454 if (pgbase + len > buf->page_len) { 455 xdr_buf_iov_zero(buf->tail, 0, pgbase + len - buf->page_len); 456 len = buf->page_len - pgbase; 457 } 458 459 pgbase += buf->page_base; 460 461 page = pages + (pgbase >> PAGE_SHIFT); 462 pgbase &= ~PAGE_MASK; 463 464 do { 465 zero = PAGE_SIZE - pgbase; 466 if (zero > len) 467 zero = len; 468 469 vpage = kmap_atomic(*page); 470 memset(vpage + pgbase, 0, zero); 471 kunmap_atomic(vpage); 472 473 flush_dcache_page(*page); 474 pgbase = 0; 475 page++; 476 477 } while ((len -= zero) != 0); 478 } 479 480 static unsigned int xdr_buf_pages_fill_sparse(const struct xdr_buf *buf, 481 unsigned int buflen, gfp_t gfp) 482 { 483 unsigned int i, npages, pagelen; 484 485 if (!(buf->flags & XDRBUF_SPARSE_PAGES)) 486 return buflen; 487 if (buflen <= buf->head->iov_len) 488 return buflen; 489 pagelen = buflen - buf->head->iov_len; 490 if (pagelen > buf->page_len) 491 pagelen = buf->page_len; 492 npages = (pagelen + buf->page_base + PAGE_SIZE - 1) >> PAGE_SHIFT; 493 for (i = 0; i < npages; i++) { 494 if (!buf->pages[i]) 495 continue; 496 buf->pages[i] = alloc_page(gfp); 497 if (likely(buf->pages[i])) 498 continue; 499 buflen -= pagelen; 500 pagelen = i << PAGE_SHIFT; 501 if (pagelen > buf->page_base) 502 buflen += pagelen - buf->page_base; 503 break; 504 } 505 return buflen; 506 } 507 508 static void xdr_buf_try_expand(struct xdr_buf *buf, unsigned int len) 509 { 510 struct kvec *head = buf->head; 511 struct kvec *tail = buf->tail; 512 unsigned int sum = head->iov_len + buf->page_len + tail->iov_len; 513 unsigned int free_space, newlen; 514 515 if (sum > buf->len) { 516 free_space = min_t(unsigned int, sum - buf->len, len); 517 newlen = xdr_buf_pages_fill_sparse(buf, buf->len + free_space, 518 GFP_KERNEL); 519 free_space = newlen - buf->len; 520 buf->len = newlen; 521 len -= free_space; 522 if (!len) 523 return; 524 } 525 526 if (buf->buflen > sum) { 527 /* Expand the tail buffer */ 528 free_space = min_t(unsigned int, buf->buflen - sum, len); 529 tail->iov_len += free_space; 530 buf->len += free_space; 531 } 532 } 533 534 static void xdr_buf_tail_copy_right(const struct xdr_buf *buf, 535 unsigned int base, unsigned int len, 536 unsigned int shift) 537 { 538 const struct kvec *tail = buf->tail; 539 unsigned int to = base + shift; 540 541 if (to >= tail->iov_len) 542 return; 543 if (len + to > tail->iov_len) 544 len = tail->iov_len - to; 545 memmove(tail->iov_base + to, tail->iov_base + base, len); 546 } 547 548 static void xdr_buf_pages_copy_right(const struct xdr_buf *buf, 549 unsigned int base, unsigned int len, 550 unsigned int shift) 551 { 552 const struct kvec *tail = buf->tail; 553 unsigned int to = base + shift; 554 unsigned int pglen = 0; 555 unsigned int talen = 0, tato = 0; 556 557 if (base >= buf->page_len) 558 return; 559 if (len > buf->page_len - base) 560 len = buf->page_len - base; 561 if (to >= buf->page_len) { 562 tato = to - buf->page_len; 563 if (tail->iov_len >= len + tato) 564 talen = len; 565 else if (tail->iov_len > tato) 566 talen = tail->iov_len - tato; 567 } else if (len + to >= buf->page_len) { 568 pglen = buf->page_len - to; 569 talen = len - pglen; 570 if (talen > tail->iov_len) 571 talen = tail->iov_len; 572 } else 573 pglen = len; 574 575 _copy_from_pages(tail->iov_base + tato, buf->pages, 576 buf->page_base + base + pglen, talen); 577 _shift_data_right_pages(buf->pages, buf->page_base + to, 578 buf->page_base + base, pglen); 579 } 580 581 static void xdr_buf_head_copy_right(const struct xdr_buf *buf, 582 unsigned int base, unsigned int len, 583 unsigned int shift) 584 { 585 const struct kvec *head = buf->head; 586 const struct kvec *tail = buf->tail; 587 unsigned int to = base + shift; 588 unsigned int pglen = 0, pgto = 0; 589 unsigned int talen = 0, tato = 0; 590 591 if (base >= head->iov_len) 592 return; 593 if (len > head->iov_len - base) 594 len = head->iov_len - base; 595 if (to >= buf->page_len + head->iov_len) { 596 tato = to - buf->page_len - head->iov_len; 597 talen = len; 598 } else if (to >= head->iov_len) { 599 pgto = to - head->iov_len; 600 pglen = len; 601 if (pgto + pglen > buf->page_len) { 602 talen = pgto + pglen - buf->page_len; 603 pglen -= talen; 604 } 605 } else { 606 pglen = len - to; 607 if (pglen > buf->page_len) { 608 talen = pglen - buf->page_len; 609 pglen = buf->page_len; 610 } 611 } 612 613 len -= talen; 614 base += len; 615 if (talen + tato > tail->iov_len) 616 talen = tail->iov_len > tato ? tail->iov_len - tato : 0; 617 memcpy(tail->iov_base + tato, head->iov_base + base, talen); 618 619 len -= pglen; 620 base -= pglen; 621 _copy_to_pages(buf->pages, buf->page_base + pgto, head->iov_base + base, 622 pglen); 623 624 base -= len; 625 memmove(head->iov_base + to, head->iov_base + base, len); 626 } 627 628 static void xdr_buf_tail_shift_right(const struct xdr_buf *buf, 629 unsigned int base, unsigned int len, 630 unsigned int shift) 631 { 632 const struct kvec *tail = buf->tail; 633 634 if (base >= tail->iov_len || !shift || !len) 635 return; 636 xdr_buf_tail_copy_right(buf, base, len, shift); 637 } 638 639 static void xdr_buf_pages_shift_right(const struct xdr_buf *buf, 640 unsigned int base, unsigned int len, 641 unsigned int shift) 642 { 643 if (!shift || !len) 644 return; 645 if (base >= buf->page_len) { 646 xdr_buf_tail_shift_right(buf, base - buf->page_len, len, shift); 647 return; 648 } 649 if (base + len > buf->page_len) 650 xdr_buf_tail_shift_right(buf, 0, base + len - buf->page_len, 651 shift); 652 xdr_buf_pages_copy_right(buf, base, len, shift); 653 } 654 655 static void xdr_buf_head_shift_right(const struct xdr_buf *buf, 656 unsigned int base, unsigned int len, 657 unsigned int shift) 658 { 659 const struct kvec *head = buf->head; 660 661 if (!shift) 662 return; 663 if (base >= head->iov_len) { 664 xdr_buf_pages_shift_right(buf, head->iov_len - base, len, 665 shift); 666 return; 667 } 668 if (base + len > head->iov_len) 669 xdr_buf_pages_shift_right(buf, 0, base + len - head->iov_len, 670 shift); 671 xdr_buf_head_copy_right(buf, base, len, shift); 672 } 673 674 static void xdr_buf_tail_copy_left(const struct xdr_buf *buf, unsigned int base, 675 unsigned int len, unsigned int shift) 676 { 677 const struct kvec *tail = buf->tail; 678 679 if (base >= tail->iov_len) 680 return; 681 if (len > tail->iov_len - base) 682 len = tail->iov_len - base; 683 /* Shift data into head */ 684 if (shift > buf->page_len + base) { 685 const struct kvec *head = buf->head; 686 unsigned int hdto = 687 head->iov_len + buf->page_len + base - shift; 688 unsigned int hdlen = len; 689 690 if (WARN_ONCE(shift > head->iov_len + buf->page_len + base, 691 "SUNRPC: Misaligned data.\n")) 692 return; 693 if (hdto + hdlen > head->iov_len) 694 hdlen = head->iov_len - hdto; 695 memcpy(head->iov_base + hdto, tail->iov_base + base, hdlen); 696 base += hdlen; 697 len -= hdlen; 698 if (!len) 699 return; 700 } 701 /* Shift data into pages */ 702 if (shift > base) { 703 unsigned int pgto = buf->page_len + base - shift; 704 unsigned int pglen = len; 705 706 if (pgto + pglen > buf->page_len) 707 pglen = buf->page_len - pgto; 708 _copy_to_pages(buf->pages, buf->page_base + pgto, 709 tail->iov_base + base, pglen); 710 base += pglen; 711 len -= pglen; 712 if (!len) 713 return; 714 } 715 memmove(tail->iov_base + base - shift, tail->iov_base + base, len); 716 } 717 718 static void xdr_buf_pages_copy_left(const struct xdr_buf *buf, 719 unsigned int base, unsigned int len, 720 unsigned int shift) 721 { 722 unsigned int pgto; 723 724 if (base >= buf->page_len) 725 return; 726 if (len > buf->page_len - base) 727 len = buf->page_len - base; 728 /* Shift data into head */ 729 if (shift > base) { 730 const struct kvec *head = buf->head; 731 unsigned int hdto = head->iov_len + base - shift; 732 unsigned int hdlen = len; 733 734 if (WARN_ONCE(shift > head->iov_len + base, 735 "SUNRPC: Misaligned data.\n")) 736 return; 737 if (hdto + hdlen > head->iov_len) 738 hdlen = head->iov_len - hdto; 739 _copy_from_pages(head->iov_base + hdto, buf->pages, 740 buf->page_base + base, hdlen); 741 base += hdlen; 742 len -= hdlen; 743 if (!len) 744 return; 745 } 746 pgto = base - shift; 747 _shift_data_left_pages(buf->pages, buf->page_base + pgto, 748 buf->page_base + base, len); 749 } 750 751 static void xdr_buf_tail_shift_left(const struct xdr_buf *buf, 752 unsigned int base, unsigned int len, 753 unsigned int shift) 754 { 755 if (!shift || !len) 756 return; 757 xdr_buf_tail_copy_left(buf, base, len, shift); 758 } 759 760 static void xdr_buf_pages_shift_left(const struct xdr_buf *buf, 761 unsigned int base, unsigned int len, 762 unsigned int shift) 763 { 764 if (!shift || !len) 765 return; 766 if (base >= buf->page_len) { 767 xdr_buf_tail_shift_left(buf, base - buf->page_len, len, shift); 768 return; 769 } 770 xdr_buf_pages_copy_left(buf, base, len, shift); 771 len += base; 772 if (len <= buf->page_len) 773 return; 774 xdr_buf_tail_copy_left(buf, 0, len - buf->page_len, shift); 775 } 776 777 static void xdr_buf_head_shift_left(const struct xdr_buf *buf, 778 unsigned int base, unsigned int len, 779 unsigned int shift) 780 { 781 const struct kvec *head = buf->head; 782 unsigned int bytes; 783 784 if (!shift || !len) 785 return; 786 787 if (shift > base) { 788 bytes = (shift - base); 789 if (bytes >= len) 790 return; 791 base += bytes; 792 len -= bytes; 793 } 794 795 if (base < head->iov_len) { 796 bytes = min_t(unsigned int, len, head->iov_len - base); 797 memmove(head->iov_base + (base - shift), 798 head->iov_base + base, bytes); 799 base += bytes; 800 len -= bytes; 801 } 802 xdr_buf_pages_shift_left(buf, base - head->iov_len, len, shift); 803 } 804 805 /** 806 * xdr_shrink_bufhead 807 * @buf: xdr_buf 808 * @len: new length of buf->head[0] 809 * 810 * Shrinks XDR buffer's header kvec buf->head[0], setting it to 811 * 'len' bytes. The extra data is not lost, but is instead 812 * moved into the inlined pages and/or the tail. 813 */ 814 static unsigned int xdr_shrink_bufhead(struct xdr_buf *buf, unsigned int len) 815 { 816 struct kvec *head = buf->head; 817 unsigned int shift, buflen = max(buf->len, len); 818 819 WARN_ON_ONCE(len > head->iov_len); 820 if (head->iov_len > buflen) { 821 buf->buflen -= head->iov_len - buflen; 822 head->iov_len = buflen; 823 } 824 if (len >= head->iov_len) 825 return 0; 826 shift = head->iov_len - len; 827 xdr_buf_try_expand(buf, shift); 828 xdr_buf_head_shift_right(buf, len, buflen - len, shift); 829 head->iov_len = len; 830 buf->buflen -= shift; 831 buf->len -= shift; 832 return shift; 833 } 834 835 /** 836 * xdr_shrink_pagelen - shrinks buf->pages to @len bytes 837 * @buf: xdr_buf 838 * @len: new page buffer length 839 * 840 * The extra data is not lost, but is instead moved into buf->tail. 841 * Returns the actual number of bytes moved. 842 */ 843 static unsigned int xdr_shrink_pagelen(struct xdr_buf *buf, unsigned int len) 844 { 845 unsigned int shift, buflen = buf->len - buf->head->iov_len; 846 847 WARN_ON_ONCE(len > buf->page_len); 848 if (buf->head->iov_len >= buf->len || len > buflen) 849 buflen = len; 850 if (buf->page_len > buflen) { 851 buf->buflen -= buf->page_len - buflen; 852 buf->page_len = buflen; 853 } 854 if (len >= buf->page_len) 855 return 0; 856 shift = buf->page_len - len; 857 xdr_buf_try_expand(buf, shift); 858 xdr_buf_pages_shift_right(buf, len, buflen - len, shift); 859 buf->page_len = len; 860 buf->len -= shift; 861 buf->buflen -= shift; 862 return shift; 863 } 864 865 void 866 xdr_shift_buf(struct xdr_buf *buf, size_t len) 867 { 868 xdr_shrink_bufhead(buf, buf->head->iov_len - len); 869 } 870 EXPORT_SYMBOL_GPL(xdr_shift_buf); 871 872 /** 873 * xdr_stream_pos - Return the current offset from the start of the xdr_stream 874 * @xdr: pointer to struct xdr_stream 875 */ 876 unsigned int xdr_stream_pos(const struct xdr_stream *xdr) 877 { 878 return (unsigned int)(XDR_QUADLEN(xdr->buf->len) - xdr->nwords) << 2; 879 } 880 EXPORT_SYMBOL_GPL(xdr_stream_pos); 881 882 static void xdr_stream_set_pos(struct xdr_stream *xdr, unsigned int pos) 883 { 884 unsigned int blen = xdr->buf->len; 885 886 xdr->nwords = blen > pos ? XDR_QUADLEN(blen) - XDR_QUADLEN(pos) : 0; 887 } 888 889 static void xdr_stream_page_set_pos(struct xdr_stream *xdr, unsigned int pos) 890 { 891 xdr_stream_set_pos(xdr, pos + xdr->buf->head[0].iov_len); 892 } 893 894 /** 895 * xdr_page_pos - Return the current offset from the start of the xdr pages 896 * @xdr: pointer to struct xdr_stream 897 */ 898 unsigned int xdr_page_pos(const struct xdr_stream *xdr) 899 { 900 unsigned int pos = xdr_stream_pos(xdr); 901 902 WARN_ON(pos < xdr->buf->head[0].iov_len); 903 return pos - xdr->buf->head[0].iov_len; 904 } 905 EXPORT_SYMBOL_GPL(xdr_page_pos); 906 907 /** 908 * xdr_init_encode - Initialize a struct xdr_stream for sending data. 909 * @xdr: pointer to xdr_stream struct 910 * @buf: pointer to XDR buffer in which to encode data 911 * @p: current pointer inside XDR buffer 912 * @rqst: pointer to controlling rpc_rqst, for debugging 913 * 914 * Note: at the moment the RPC client only passes the length of our 915 * scratch buffer in the xdr_buf's header kvec. Previously this 916 * meant we needed to call xdr_adjust_iovec() after encoding the 917 * data. With the new scheme, the xdr_stream manages the details 918 * of the buffer length, and takes care of adjusting the kvec 919 * length for us. 920 */ 921 void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p, 922 struct rpc_rqst *rqst) 923 { 924 struct kvec *iov = buf->head; 925 int scratch_len = buf->buflen - buf->page_len - buf->tail[0].iov_len; 926 927 xdr_reset_scratch_buffer(xdr); 928 BUG_ON(scratch_len < 0); 929 xdr->buf = buf; 930 xdr->iov = iov; 931 xdr->p = (__be32 *)((char *)iov->iov_base + iov->iov_len); 932 xdr->end = (__be32 *)((char *)iov->iov_base + scratch_len); 933 BUG_ON(iov->iov_len > scratch_len); 934 935 if (p != xdr->p && p != NULL) { 936 size_t len; 937 938 BUG_ON(p < xdr->p || p > xdr->end); 939 len = (char *)p - (char *)xdr->p; 940 xdr->p = p; 941 buf->len += len; 942 iov->iov_len += len; 943 } 944 xdr->rqst = rqst; 945 } 946 EXPORT_SYMBOL_GPL(xdr_init_encode); 947 948 /** 949 * xdr_init_encode_pages - Initialize an xdr_stream for encoding into pages 950 * @xdr: pointer to xdr_stream struct 951 * @buf: pointer to XDR buffer into which to encode data 952 * @pages: list of pages to decode into 953 * @rqst: pointer to controlling rpc_rqst, for debugging 954 * 955 */ 956 void xdr_init_encode_pages(struct xdr_stream *xdr, struct xdr_buf *buf, 957 struct page **pages, struct rpc_rqst *rqst) 958 { 959 xdr_reset_scratch_buffer(xdr); 960 961 xdr->buf = buf; 962 xdr->page_ptr = pages; 963 xdr->iov = NULL; 964 xdr->p = page_address(*pages); 965 xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE); 966 xdr->rqst = rqst; 967 } 968 EXPORT_SYMBOL_GPL(xdr_init_encode_pages); 969 970 /** 971 * __xdr_commit_encode - Ensure all data is written to buffer 972 * @xdr: pointer to xdr_stream 973 * 974 * We handle encoding across page boundaries by giving the caller a 975 * temporary location to write to, then later copying the data into 976 * place; xdr_commit_encode does that copying. 977 * 978 * Normally the caller doesn't need to call this directly, as the 979 * following xdr_reserve_space will do it. But an explicit call may be 980 * required at the end of encoding, or any other time when the xdr_buf 981 * data might be read. 982 */ 983 void __xdr_commit_encode(struct xdr_stream *xdr) 984 { 985 size_t shift = xdr->scratch.iov_len; 986 void *page; 987 988 page = page_address(*xdr->page_ptr); 989 memcpy(xdr->scratch.iov_base, page, shift); 990 memmove(page, page + shift, (void *)xdr->p - page); 991 xdr_reset_scratch_buffer(xdr); 992 } 993 EXPORT_SYMBOL_GPL(__xdr_commit_encode); 994 995 /* 996 * The buffer space to be reserved crosses the boundary between 997 * xdr->buf->head and xdr->buf->pages, or between two pages 998 * in xdr->buf->pages. 999 */ 1000 static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, 1001 size_t nbytes) 1002 { 1003 int space_left; 1004 int frag1bytes, frag2bytes; 1005 void *p; 1006 1007 if (nbytes > PAGE_SIZE) 1008 goto out_overflow; /* Bigger buffers require special handling */ 1009 if (xdr->buf->len + nbytes > xdr->buf->buflen) 1010 goto out_overflow; /* Sorry, we're totally out of space */ 1011 frag1bytes = (xdr->end - xdr->p) << 2; 1012 frag2bytes = nbytes - frag1bytes; 1013 if (xdr->iov) 1014 xdr->iov->iov_len += frag1bytes; 1015 else 1016 xdr->buf->page_len += frag1bytes; 1017 xdr->page_ptr++; 1018 xdr->iov = NULL; 1019 1020 /* 1021 * If the last encode didn't end exactly on a page boundary, the 1022 * next one will straddle boundaries. Encode into the next 1023 * page, then copy it back later in xdr_commit_encode. We use 1024 * the "scratch" iov to track any temporarily unused fragment of 1025 * space at the end of the previous buffer: 1026 */ 1027 xdr_set_scratch_buffer(xdr, xdr->p, frag1bytes); 1028 1029 /* 1030 * xdr->p is where the next encode will start after 1031 * xdr_commit_encode() has shifted this one back: 1032 */ 1033 p = page_address(*xdr->page_ptr); 1034 xdr->p = p + frag2bytes; 1035 space_left = xdr->buf->buflen - xdr->buf->len; 1036 if (space_left - frag1bytes >= PAGE_SIZE) 1037 xdr->end = p + PAGE_SIZE; 1038 else 1039 xdr->end = p + space_left - frag1bytes; 1040 1041 xdr->buf->page_len += frag2bytes; 1042 xdr->buf->len += nbytes; 1043 return p; 1044 out_overflow: 1045 trace_rpc_xdr_overflow(xdr, nbytes); 1046 return NULL; 1047 } 1048 1049 /** 1050 * xdr_reserve_space - Reserve buffer space for sending 1051 * @xdr: pointer to xdr_stream 1052 * @nbytes: number of bytes to reserve 1053 * 1054 * Checks that we have enough buffer space to encode 'nbytes' more 1055 * bytes of data. If so, update the total xdr_buf length, and 1056 * adjust the length of the current kvec. 1057 */ 1058 __be32 * xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes) 1059 { 1060 __be32 *p = xdr->p; 1061 __be32 *q; 1062 1063 xdr_commit_encode(xdr); 1064 /* align nbytes on the next 32-bit boundary */ 1065 nbytes += 3; 1066 nbytes &= ~3; 1067 q = p + (nbytes >> 2); 1068 if (unlikely(q > xdr->end || q < p)) 1069 return xdr_get_next_encode_buffer(xdr, nbytes); 1070 xdr->p = q; 1071 if (xdr->iov) 1072 xdr->iov->iov_len += nbytes; 1073 else 1074 xdr->buf->page_len += nbytes; 1075 xdr->buf->len += nbytes; 1076 return p; 1077 } 1078 EXPORT_SYMBOL_GPL(xdr_reserve_space); 1079 1080 1081 /** 1082 * xdr_reserve_space_vec - Reserves a large amount of buffer space for sending 1083 * @xdr: pointer to xdr_stream 1084 * @vec: pointer to a kvec array 1085 * @nbytes: number of bytes to reserve 1086 * 1087 * Reserves enough buffer space to encode 'nbytes' of data and stores the 1088 * pointers in 'vec'. The size argument passed to xdr_reserve_space() is 1089 * determined based on the number of bytes remaining in the current page to 1090 * avoid invalidating iov_base pointers when xdr_commit_encode() is called. 1091 */ 1092 int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec, size_t nbytes) 1093 { 1094 int thislen; 1095 int v = 0; 1096 __be32 *p; 1097 1098 /* 1099 * svcrdma requires every READ payload to start somewhere 1100 * in xdr->pages. 1101 */ 1102 if (xdr->iov == xdr->buf->head) { 1103 xdr->iov = NULL; 1104 xdr->end = xdr->p; 1105 } 1106 1107 while (nbytes) { 1108 thislen = xdr->buf->page_len % PAGE_SIZE; 1109 thislen = min_t(size_t, nbytes, PAGE_SIZE - thislen); 1110 1111 p = xdr_reserve_space(xdr, thislen); 1112 if (!p) 1113 return -EIO; 1114 1115 vec[v].iov_base = p; 1116 vec[v].iov_len = thislen; 1117 v++; 1118 nbytes -= thislen; 1119 } 1120 1121 return v; 1122 } 1123 EXPORT_SYMBOL_GPL(xdr_reserve_space_vec); 1124 1125 /** 1126 * xdr_truncate_encode - truncate an encode buffer 1127 * @xdr: pointer to xdr_stream 1128 * @len: new length of buffer 1129 * 1130 * Truncates the xdr stream, so that xdr->buf->len == len, 1131 * and xdr->p points at offset len from the start of the buffer, and 1132 * head, tail, and page lengths are adjusted to correspond. 1133 * 1134 * If this means moving xdr->p to a different buffer, we assume that 1135 * the end pointer should be set to the end of the current page, 1136 * except in the case of the head buffer when we assume the head 1137 * buffer's current length represents the end of the available buffer. 1138 * 1139 * This is *not* safe to use on a buffer that already has inlined page 1140 * cache pages (as in a zero-copy server read reply), except for the 1141 * simple case of truncating from one position in the tail to another. 1142 * 1143 */ 1144 void xdr_truncate_encode(struct xdr_stream *xdr, size_t len) 1145 { 1146 struct xdr_buf *buf = xdr->buf; 1147 struct kvec *head = buf->head; 1148 struct kvec *tail = buf->tail; 1149 int fraglen; 1150 int new; 1151 1152 if (len > buf->len) { 1153 WARN_ON_ONCE(1); 1154 return; 1155 } 1156 xdr_commit_encode(xdr); 1157 1158 fraglen = min_t(int, buf->len - len, tail->iov_len); 1159 tail->iov_len -= fraglen; 1160 buf->len -= fraglen; 1161 if (tail->iov_len) { 1162 xdr->p = tail->iov_base + tail->iov_len; 1163 WARN_ON_ONCE(!xdr->end); 1164 WARN_ON_ONCE(!xdr->iov); 1165 return; 1166 } 1167 WARN_ON_ONCE(fraglen); 1168 fraglen = min_t(int, buf->len - len, buf->page_len); 1169 buf->page_len -= fraglen; 1170 buf->len -= fraglen; 1171 1172 new = buf->page_base + buf->page_len; 1173 1174 xdr->page_ptr = buf->pages + (new >> PAGE_SHIFT); 1175 1176 if (buf->page_len) { 1177 xdr->p = page_address(*xdr->page_ptr); 1178 xdr->end = (void *)xdr->p + PAGE_SIZE; 1179 xdr->p = (void *)xdr->p + (new % PAGE_SIZE); 1180 WARN_ON_ONCE(xdr->iov); 1181 return; 1182 } 1183 if (fraglen) 1184 xdr->end = head->iov_base + head->iov_len; 1185 /* (otherwise assume xdr->end is already set) */ 1186 xdr->page_ptr--; 1187 head->iov_len = len; 1188 buf->len = len; 1189 xdr->p = head->iov_base + head->iov_len; 1190 xdr->iov = buf->head; 1191 } 1192 EXPORT_SYMBOL(xdr_truncate_encode); 1193 1194 /** 1195 * xdr_restrict_buflen - decrease available buffer space 1196 * @xdr: pointer to xdr_stream 1197 * @newbuflen: new maximum number of bytes available 1198 * 1199 * Adjust our idea of how much space is available in the buffer. 1200 * If we've already used too much space in the buffer, returns -1. 1201 * If the available space is already smaller than newbuflen, returns 0 1202 * and does nothing. Otherwise, adjusts xdr->buf->buflen to newbuflen 1203 * and ensures xdr->end is set at most offset newbuflen from the start 1204 * of the buffer. 1205 */ 1206 int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen) 1207 { 1208 struct xdr_buf *buf = xdr->buf; 1209 int left_in_this_buf = (void *)xdr->end - (void *)xdr->p; 1210 int end_offset = buf->len + left_in_this_buf; 1211 1212 if (newbuflen < 0 || newbuflen < buf->len) 1213 return -1; 1214 if (newbuflen > buf->buflen) 1215 return 0; 1216 if (newbuflen < end_offset) 1217 xdr->end = (void *)xdr->end + newbuflen - end_offset; 1218 buf->buflen = newbuflen; 1219 return 0; 1220 } 1221 EXPORT_SYMBOL(xdr_restrict_buflen); 1222 1223 /** 1224 * xdr_write_pages - Insert a list of pages into an XDR buffer for sending 1225 * @xdr: pointer to xdr_stream 1226 * @pages: array of pages to insert 1227 * @base: starting offset of first data byte in @pages 1228 * @len: number of data bytes in @pages to insert 1229 * 1230 * After the @pages are added, the tail iovec is instantiated pointing to 1231 * end of the head buffer, and the stream is set up to encode subsequent 1232 * items into the tail. 1233 */ 1234 void xdr_write_pages(struct xdr_stream *xdr, struct page **pages, unsigned int base, 1235 unsigned int len) 1236 { 1237 struct xdr_buf *buf = xdr->buf; 1238 struct kvec *tail = buf->tail; 1239 1240 buf->pages = pages; 1241 buf->page_base = base; 1242 buf->page_len = len; 1243 1244 tail->iov_base = xdr->p; 1245 tail->iov_len = 0; 1246 xdr->iov = tail; 1247 1248 if (len & 3) { 1249 unsigned int pad = 4 - (len & 3); 1250 1251 BUG_ON(xdr->p >= xdr->end); 1252 tail->iov_base = (char *)xdr->p + (len & 3); 1253 tail->iov_len += pad; 1254 len += pad; 1255 *xdr->p++ = 0; 1256 } 1257 buf->buflen += len; 1258 buf->len += len; 1259 } 1260 EXPORT_SYMBOL_GPL(xdr_write_pages); 1261 1262 static unsigned int xdr_set_iov(struct xdr_stream *xdr, struct kvec *iov, 1263 unsigned int base, unsigned int len) 1264 { 1265 if (len > iov->iov_len) 1266 len = iov->iov_len; 1267 if (unlikely(base > len)) 1268 base = len; 1269 xdr->p = (__be32*)(iov->iov_base + base); 1270 xdr->end = (__be32*)(iov->iov_base + len); 1271 xdr->iov = iov; 1272 xdr->page_ptr = NULL; 1273 return len - base; 1274 } 1275 1276 static unsigned int xdr_set_tail_base(struct xdr_stream *xdr, 1277 unsigned int base, unsigned int len) 1278 { 1279 struct xdr_buf *buf = xdr->buf; 1280 1281 xdr_stream_set_pos(xdr, base + buf->page_len + buf->head->iov_len); 1282 return xdr_set_iov(xdr, buf->tail, base, len); 1283 } 1284 1285 static unsigned int xdr_set_page_base(struct xdr_stream *xdr, 1286 unsigned int base, unsigned int len) 1287 { 1288 unsigned int pgnr; 1289 unsigned int maxlen; 1290 unsigned int pgoff; 1291 unsigned int pgend; 1292 void *kaddr; 1293 1294 maxlen = xdr->buf->page_len; 1295 if (base >= maxlen) 1296 return 0; 1297 else 1298 maxlen -= base; 1299 if (len > maxlen) 1300 len = maxlen; 1301 1302 xdr_stream_page_set_pos(xdr, base); 1303 base += xdr->buf->page_base; 1304 1305 pgnr = base >> PAGE_SHIFT; 1306 xdr->page_ptr = &xdr->buf->pages[pgnr]; 1307 kaddr = page_address(*xdr->page_ptr); 1308 1309 pgoff = base & ~PAGE_MASK; 1310 xdr->p = (__be32*)(kaddr + pgoff); 1311 1312 pgend = pgoff + len; 1313 if (pgend > PAGE_SIZE) 1314 pgend = PAGE_SIZE; 1315 xdr->end = (__be32*)(kaddr + pgend); 1316 xdr->iov = NULL; 1317 return len; 1318 } 1319 1320 static void xdr_set_page(struct xdr_stream *xdr, unsigned int base, 1321 unsigned int len) 1322 { 1323 if (xdr_set_page_base(xdr, base, len) == 0) { 1324 base -= xdr->buf->page_len; 1325 xdr_set_tail_base(xdr, base, len); 1326 } 1327 } 1328 1329 static void xdr_set_next_page(struct xdr_stream *xdr) 1330 { 1331 unsigned int newbase; 1332 1333 newbase = (1 + xdr->page_ptr - xdr->buf->pages) << PAGE_SHIFT; 1334 newbase -= xdr->buf->page_base; 1335 if (newbase < xdr->buf->page_len) 1336 xdr_set_page_base(xdr, newbase, xdr_stream_remaining(xdr)); 1337 else 1338 xdr_set_tail_base(xdr, 0, xdr_stream_remaining(xdr)); 1339 } 1340 1341 static bool xdr_set_next_buffer(struct xdr_stream *xdr) 1342 { 1343 if (xdr->page_ptr != NULL) 1344 xdr_set_next_page(xdr); 1345 else if (xdr->iov == xdr->buf->head) 1346 xdr_set_page(xdr, 0, xdr_stream_remaining(xdr)); 1347 return xdr->p != xdr->end; 1348 } 1349 1350 /** 1351 * xdr_init_decode - Initialize an xdr_stream for decoding data. 1352 * @xdr: pointer to xdr_stream struct 1353 * @buf: pointer to XDR buffer from which to decode data 1354 * @p: current pointer inside XDR buffer 1355 * @rqst: pointer to controlling rpc_rqst, for debugging 1356 */ 1357 void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p, 1358 struct rpc_rqst *rqst) 1359 { 1360 xdr->buf = buf; 1361 xdr_reset_scratch_buffer(xdr); 1362 xdr->nwords = XDR_QUADLEN(buf->len); 1363 if (xdr_set_iov(xdr, buf->head, 0, buf->len) == 0 && 1364 xdr_set_page_base(xdr, 0, buf->len) == 0) 1365 xdr_set_iov(xdr, buf->tail, 0, buf->len); 1366 if (p != NULL && p > xdr->p && xdr->end >= p) { 1367 xdr->nwords -= p - xdr->p; 1368 xdr->p = p; 1369 } 1370 xdr->rqst = rqst; 1371 } 1372 EXPORT_SYMBOL_GPL(xdr_init_decode); 1373 1374 /** 1375 * xdr_init_decode_pages - Initialize an xdr_stream for decoding into pages 1376 * @xdr: pointer to xdr_stream struct 1377 * @buf: pointer to XDR buffer from which to decode data 1378 * @pages: list of pages to decode into 1379 * @len: length in bytes of buffer in pages 1380 */ 1381 void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf, 1382 struct page **pages, unsigned int len) 1383 { 1384 memset(buf, 0, sizeof(*buf)); 1385 buf->pages = pages; 1386 buf->page_len = len; 1387 buf->buflen = len; 1388 buf->len = len; 1389 xdr_init_decode(xdr, buf, NULL, NULL); 1390 } 1391 EXPORT_SYMBOL_GPL(xdr_init_decode_pages); 1392 1393 static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes) 1394 { 1395 unsigned int nwords = XDR_QUADLEN(nbytes); 1396 __be32 *p = xdr->p; 1397 __be32 *q = p + nwords; 1398 1399 if (unlikely(nwords > xdr->nwords || q > xdr->end || q < p)) 1400 return NULL; 1401 xdr->p = q; 1402 xdr->nwords -= nwords; 1403 return p; 1404 } 1405 1406 static __be32 *xdr_copy_to_scratch(struct xdr_stream *xdr, size_t nbytes) 1407 { 1408 __be32 *p; 1409 char *cpdest = xdr->scratch.iov_base; 1410 size_t cplen = (char *)xdr->end - (char *)xdr->p; 1411 1412 if (nbytes > xdr->scratch.iov_len) 1413 goto out_overflow; 1414 p = __xdr_inline_decode(xdr, cplen); 1415 if (p == NULL) 1416 return NULL; 1417 memcpy(cpdest, p, cplen); 1418 if (!xdr_set_next_buffer(xdr)) 1419 goto out_overflow; 1420 cpdest += cplen; 1421 nbytes -= cplen; 1422 p = __xdr_inline_decode(xdr, nbytes); 1423 if (p == NULL) 1424 return NULL; 1425 memcpy(cpdest, p, nbytes); 1426 return xdr->scratch.iov_base; 1427 out_overflow: 1428 trace_rpc_xdr_overflow(xdr, nbytes); 1429 return NULL; 1430 } 1431 1432 /** 1433 * xdr_inline_decode - Retrieve XDR data to decode 1434 * @xdr: pointer to xdr_stream struct 1435 * @nbytes: number of bytes of data to decode 1436 * 1437 * Check if the input buffer is long enough to enable us to decode 1438 * 'nbytes' more bytes of data starting at the current position. 1439 * If so return the current pointer, then update the current 1440 * pointer position. 1441 */ 1442 __be32 * xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes) 1443 { 1444 __be32 *p; 1445 1446 if (unlikely(nbytes == 0)) 1447 return xdr->p; 1448 if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr)) 1449 goto out_overflow; 1450 p = __xdr_inline_decode(xdr, nbytes); 1451 if (p != NULL) 1452 return p; 1453 return xdr_copy_to_scratch(xdr, nbytes); 1454 out_overflow: 1455 trace_rpc_xdr_overflow(xdr, nbytes); 1456 return NULL; 1457 } 1458 EXPORT_SYMBOL_GPL(xdr_inline_decode); 1459 1460 static void xdr_realign_pages(struct xdr_stream *xdr) 1461 { 1462 struct xdr_buf *buf = xdr->buf; 1463 struct kvec *iov = buf->head; 1464 unsigned int cur = xdr_stream_pos(xdr); 1465 unsigned int copied; 1466 1467 /* Realign pages to current pointer position */ 1468 if (iov->iov_len > cur) { 1469 copied = xdr_shrink_bufhead(buf, cur); 1470 trace_rpc_xdr_alignment(xdr, cur, copied); 1471 xdr_set_page(xdr, 0, buf->page_len); 1472 } 1473 } 1474 1475 static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len) 1476 { 1477 struct xdr_buf *buf = xdr->buf; 1478 unsigned int nwords = XDR_QUADLEN(len); 1479 unsigned int copied; 1480 1481 if (xdr->nwords == 0) 1482 return 0; 1483 1484 xdr_realign_pages(xdr); 1485 if (nwords > xdr->nwords) { 1486 nwords = xdr->nwords; 1487 len = nwords << 2; 1488 } 1489 if (buf->page_len <= len) 1490 len = buf->page_len; 1491 else if (nwords < xdr->nwords) { 1492 /* Truncate page data and move it into the tail */ 1493 copied = xdr_shrink_pagelen(buf, len); 1494 trace_rpc_xdr_alignment(xdr, len, copied); 1495 } 1496 return len; 1497 } 1498 1499 /** 1500 * xdr_read_pages - align page-based XDR data to current pointer position 1501 * @xdr: pointer to xdr_stream struct 1502 * @len: number of bytes of page data 1503 * 1504 * Moves data beyond the current pointer position from the XDR head[] buffer 1505 * into the page list. Any data that lies beyond current position + @len 1506 * bytes is moved into the XDR tail[]. The xdr_stream current position is 1507 * then advanced past that data to align to the next XDR object in the tail. 1508 * 1509 * Returns the number of XDR encoded bytes now contained in the pages 1510 */ 1511 unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len) 1512 { 1513 unsigned int nwords = XDR_QUADLEN(len); 1514 unsigned int base, end, pglen; 1515 1516 pglen = xdr_align_pages(xdr, nwords << 2); 1517 if (pglen == 0) 1518 return 0; 1519 1520 base = (nwords << 2) - pglen; 1521 end = xdr_stream_remaining(xdr) - pglen; 1522 1523 xdr_set_tail_base(xdr, base, end); 1524 return len <= pglen ? len : pglen; 1525 } 1526 EXPORT_SYMBOL_GPL(xdr_read_pages); 1527 1528 /** 1529 * xdr_set_pagelen - Sets the length of the XDR pages 1530 * @xdr: pointer to xdr_stream struct 1531 * @len: new length of the XDR page data 1532 * 1533 * Either grows or shrinks the length of the xdr pages by setting pagelen to 1534 * @len bytes. When shrinking, any extra data is moved into buf->tail, whereas 1535 * when growing any data beyond the current pointer is moved into the tail. 1536 * 1537 * Returns True if the operation was successful, and False otherwise. 1538 */ 1539 void xdr_set_pagelen(struct xdr_stream *xdr, unsigned int len) 1540 { 1541 struct xdr_buf *buf = xdr->buf; 1542 size_t remaining = xdr_stream_remaining(xdr); 1543 size_t base = 0; 1544 1545 if (len < buf->page_len) { 1546 base = buf->page_len - len; 1547 xdr_shrink_pagelen(buf, len); 1548 } else { 1549 xdr_buf_head_shift_right(buf, xdr_stream_pos(xdr), 1550 buf->page_len, remaining); 1551 if (len > buf->page_len) 1552 xdr_buf_try_expand(buf, len - buf->page_len); 1553 } 1554 xdr_set_tail_base(xdr, base, remaining); 1555 } 1556 EXPORT_SYMBOL_GPL(xdr_set_pagelen); 1557 1558 /** 1559 * xdr_enter_page - decode data from the XDR page 1560 * @xdr: pointer to xdr_stream struct 1561 * @len: number of bytes of page data 1562 * 1563 * Moves data beyond the current pointer position from the XDR head[] buffer 1564 * into the page list. Any data that lies beyond current position + "len" 1565 * bytes is moved into the XDR tail[]. The current pointer is then 1566 * repositioned at the beginning of the first XDR page. 1567 */ 1568 void xdr_enter_page(struct xdr_stream *xdr, unsigned int len) 1569 { 1570 len = xdr_align_pages(xdr, len); 1571 /* 1572 * Position current pointer at beginning of tail, and 1573 * set remaining message length. 1574 */ 1575 if (len != 0) 1576 xdr_set_page_base(xdr, 0, len); 1577 } 1578 EXPORT_SYMBOL_GPL(xdr_enter_page); 1579 1580 static const struct kvec empty_iov = {.iov_base = NULL, .iov_len = 0}; 1581 1582 void xdr_buf_from_iov(const struct kvec *iov, struct xdr_buf *buf) 1583 { 1584 buf->head[0] = *iov; 1585 buf->tail[0] = empty_iov; 1586 buf->page_len = 0; 1587 buf->buflen = buf->len = iov->iov_len; 1588 } 1589 EXPORT_SYMBOL_GPL(xdr_buf_from_iov); 1590 1591 /** 1592 * xdr_buf_subsegment - set subbuf to a portion of buf 1593 * @buf: an xdr buffer 1594 * @subbuf: the result buffer 1595 * @base: beginning of range in bytes 1596 * @len: length of range in bytes 1597 * 1598 * sets @subbuf to an xdr buffer representing the portion of @buf of 1599 * length @len starting at offset @base. 1600 * 1601 * @buf and @subbuf may be pointers to the same struct xdr_buf. 1602 * 1603 * Returns -1 if base or length are out of bounds. 1604 */ 1605 int xdr_buf_subsegment(const struct xdr_buf *buf, struct xdr_buf *subbuf, 1606 unsigned int base, unsigned int len) 1607 { 1608 subbuf->buflen = subbuf->len = len; 1609 if (base < buf->head[0].iov_len) { 1610 subbuf->head[0].iov_base = buf->head[0].iov_base + base; 1611 subbuf->head[0].iov_len = min_t(unsigned int, len, 1612 buf->head[0].iov_len - base); 1613 len -= subbuf->head[0].iov_len; 1614 base = 0; 1615 } else { 1616 base -= buf->head[0].iov_len; 1617 subbuf->head[0].iov_base = buf->head[0].iov_base; 1618 subbuf->head[0].iov_len = 0; 1619 } 1620 1621 if (base < buf->page_len) { 1622 subbuf->page_len = min(buf->page_len - base, len); 1623 base += buf->page_base; 1624 subbuf->page_base = base & ~PAGE_MASK; 1625 subbuf->pages = &buf->pages[base >> PAGE_SHIFT]; 1626 len -= subbuf->page_len; 1627 base = 0; 1628 } else { 1629 base -= buf->page_len; 1630 subbuf->pages = buf->pages; 1631 subbuf->page_base = 0; 1632 subbuf->page_len = 0; 1633 } 1634 1635 if (base < buf->tail[0].iov_len) { 1636 subbuf->tail[0].iov_base = buf->tail[0].iov_base + base; 1637 subbuf->tail[0].iov_len = min_t(unsigned int, len, 1638 buf->tail[0].iov_len - base); 1639 len -= subbuf->tail[0].iov_len; 1640 base = 0; 1641 } else { 1642 base -= buf->tail[0].iov_len; 1643 subbuf->tail[0].iov_base = buf->tail[0].iov_base; 1644 subbuf->tail[0].iov_len = 0; 1645 } 1646 1647 if (base || len) 1648 return -1; 1649 return 0; 1650 } 1651 EXPORT_SYMBOL_GPL(xdr_buf_subsegment); 1652 1653 /** 1654 * xdr_stream_subsegment - set @subbuf to a portion of @xdr 1655 * @xdr: an xdr_stream set up for decoding 1656 * @subbuf: the result buffer 1657 * @nbytes: length of @xdr to extract, in bytes 1658 * 1659 * Sets up @subbuf to represent a portion of @xdr. The portion 1660 * starts at the current offset in @xdr, and extends for a length 1661 * of @nbytes. If this is successful, @xdr is advanced to the next 1662 * XDR data item following that portion. 1663 * 1664 * Return values: 1665 * %true: @subbuf has been initialized, and @xdr has been advanced. 1666 * %false: a bounds error has occurred 1667 */ 1668 bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf, 1669 unsigned int nbytes) 1670 { 1671 unsigned int start = xdr_stream_pos(xdr); 1672 unsigned int remaining, len; 1673 1674 /* Extract @subbuf and bounds-check the fn arguments */ 1675 if (xdr_buf_subsegment(xdr->buf, subbuf, start, nbytes)) 1676 return false; 1677 1678 /* Advance @xdr by @nbytes */ 1679 for (remaining = nbytes; remaining;) { 1680 if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr)) 1681 return false; 1682 1683 len = (char *)xdr->end - (char *)xdr->p; 1684 if (remaining <= len) { 1685 xdr->p = (__be32 *)((char *)xdr->p + 1686 (remaining + xdr_pad_size(nbytes))); 1687 break; 1688 } 1689 1690 xdr->p = (__be32 *)((char *)xdr->p + len); 1691 xdr->end = xdr->p; 1692 remaining -= len; 1693 } 1694 1695 xdr_stream_set_pos(xdr, start + nbytes); 1696 return true; 1697 } 1698 EXPORT_SYMBOL_GPL(xdr_stream_subsegment); 1699 1700 /** 1701 * xdr_stream_move_subsegment - Move part of a stream to another position 1702 * @xdr: the source xdr_stream 1703 * @offset: the source offset of the segment 1704 * @target: the target offset of the segment 1705 * @length: the number of bytes to move 1706 * 1707 * Moves @length bytes from @offset to @target in the xdr_stream, overwriting 1708 * anything in its space. Returns the number of bytes in the segment. 1709 */ 1710 unsigned int xdr_stream_move_subsegment(struct xdr_stream *xdr, unsigned int offset, 1711 unsigned int target, unsigned int length) 1712 { 1713 struct xdr_buf buf; 1714 unsigned int shift; 1715 1716 if (offset < target) { 1717 shift = target - offset; 1718 if (xdr_buf_subsegment(xdr->buf, &buf, offset, shift + length) < 0) 1719 return 0; 1720 xdr_buf_head_shift_right(&buf, 0, length, shift); 1721 } else if (offset > target) { 1722 shift = offset - target; 1723 if (xdr_buf_subsegment(xdr->buf, &buf, target, shift + length) < 0) 1724 return 0; 1725 xdr_buf_head_shift_left(&buf, shift, length, shift); 1726 } 1727 return length; 1728 } 1729 EXPORT_SYMBOL_GPL(xdr_stream_move_subsegment); 1730 1731 /** 1732 * xdr_stream_zero - zero out a portion of an xdr_stream 1733 * @xdr: an xdr_stream to zero out 1734 * @offset: the starting point in the stream 1735 * @length: the number of bytes to zero 1736 */ 1737 unsigned int xdr_stream_zero(struct xdr_stream *xdr, unsigned int offset, 1738 unsigned int length) 1739 { 1740 struct xdr_buf buf; 1741 1742 if (xdr_buf_subsegment(xdr->buf, &buf, offset, length) < 0) 1743 return 0; 1744 if (buf.head[0].iov_len) 1745 xdr_buf_iov_zero(buf.head, 0, buf.head[0].iov_len); 1746 if (buf.page_len > 0) 1747 xdr_buf_pages_zero(&buf, 0, buf.page_len); 1748 if (buf.tail[0].iov_len) 1749 xdr_buf_iov_zero(buf.tail, 0, buf.tail[0].iov_len); 1750 return length; 1751 } 1752 EXPORT_SYMBOL_GPL(xdr_stream_zero); 1753 1754 /** 1755 * xdr_buf_trim - lop at most "len" bytes off the end of "buf" 1756 * @buf: buf to be trimmed 1757 * @len: number of bytes to reduce "buf" by 1758 * 1759 * Trim an xdr_buf by the given number of bytes by fixing up the lengths. Note 1760 * that it's possible that we'll trim less than that amount if the xdr_buf is 1761 * too small, or if (for instance) it's all in the head and the parser has 1762 * already read too far into it. 1763 */ 1764 void xdr_buf_trim(struct xdr_buf *buf, unsigned int len) 1765 { 1766 size_t cur; 1767 unsigned int trim = len; 1768 1769 if (buf->tail[0].iov_len) { 1770 cur = min_t(size_t, buf->tail[0].iov_len, trim); 1771 buf->tail[0].iov_len -= cur; 1772 trim -= cur; 1773 if (!trim) 1774 goto fix_len; 1775 } 1776 1777 if (buf->page_len) { 1778 cur = min_t(unsigned int, buf->page_len, trim); 1779 buf->page_len -= cur; 1780 trim -= cur; 1781 if (!trim) 1782 goto fix_len; 1783 } 1784 1785 if (buf->head[0].iov_len) { 1786 cur = min_t(size_t, buf->head[0].iov_len, trim); 1787 buf->head[0].iov_len -= cur; 1788 trim -= cur; 1789 } 1790 fix_len: 1791 buf->len -= (len - trim); 1792 } 1793 EXPORT_SYMBOL_GPL(xdr_buf_trim); 1794 1795 static void __read_bytes_from_xdr_buf(const struct xdr_buf *subbuf, 1796 void *obj, unsigned int len) 1797 { 1798 unsigned int this_len; 1799 1800 this_len = min_t(unsigned int, len, subbuf->head[0].iov_len); 1801 memcpy(obj, subbuf->head[0].iov_base, this_len); 1802 len -= this_len; 1803 obj += this_len; 1804 this_len = min_t(unsigned int, len, subbuf->page_len); 1805 _copy_from_pages(obj, subbuf->pages, subbuf->page_base, this_len); 1806 len -= this_len; 1807 obj += this_len; 1808 this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len); 1809 memcpy(obj, subbuf->tail[0].iov_base, this_len); 1810 } 1811 1812 /* obj is assumed to point to allocated memory of size at least len: */ 1813 int read_bytes_from_xdr_buf(const struct xdr_buf *buf, unsigned int base, 1814 void *obj, unsigned int len) 1815 { 1816 struct xdr_buf subbuf; 1817 int status; 1818 1819 status = xdr_buf_subsegment(buf, &subbuf, base, len); 1820 if (status != 0) 1821 return status; 1822 __read_bytes_from_xdr_buf(&subbuf, obj, len); 1823 return 0; 1824 } 1825 EXPORT_SYMBOL_GPL(read_bytes_from_xdr_buf); 1826 1827 static void __write_bytes_to_xdr_buf(const struct xdr_buf *subbuf, 1828 void *obj, unsigned int len) 1829 { 1830 unsigned int this_len; 1831 1832 this_len = min_t(unsigned int, len, subbuf->head[0].iov_len); 1833 memcpy(subbuf->head[0].iov_base, obj, this_len); 1834 len -= this_len; 1835 obj += this_len; 1836 this_len = min_t(unsigned int, len, subbuf->page_len); 1837 _copy_to_pages(subbuf->pages, subbuf->page_base, obj, this_len); 1838 len -= this_len; 1839 obj += this_len; 1840 this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len); 1841 memcpy(subbuf->tail[0].iov_base, obj, this_len); 1842 } 1843 1844 /* obj is assumed to point to allocated memory of size at least len: */ 1845 int write_bytes_to_xdr_buf(const struct xdr_buf *buf, unsigned int base, 1846 void *obj, unsigned int len) 1847 { 1848 struct xdr_buf subbuf; 1849 int status; 1850 1851 status = xdr_buf_subsegment(buf, &subbuf, base, len); 1852 if (status != 0) 1853 return status; 1854 __write_bytes_to_xdr_buf(&subbuf, obj, len); 1855 return 0; 1856 } 1857 EXPORT_SYMBOL_GPL(write_bytes_to_xdr_buf); 1858 1859 int xdr_decode_word(const struct xdr_buf *buf, unsigned int base, u32 *obj) 1860 { 1861 __be32 raw; 1862 int status; 1863 1864 status = read_bytes_from_xdr_buf(buf, base, &raw, sizeof(*obj)); 1865 if (status) 1866 return status; 1867 *obj = be32_to_cpu(raw); 1868 return 0; 1869 } 1870 EXPORT_SYMBOL_GPL(xdr_decode_word); 1871 1872 int xdr_encode_word(const struct xdr_buf *buf, unsigned int base, u32 obj) 1873 { 1874 __be32 raw = cpu_to_be32(obj); 1875 1876 return write_bytes_to_xdr_buf(buf, base, &raw, sizeof(obj)); 1877 } 1878 EXPORT_SYMBOL_GPL(xdr_encode_word); 1879 1880 /* Returns 0 on success, or else a negative error code. */ 1881 static int xdr_xcode_array2(const struct xdr_buf *buf, unsigned int base, 1882 struct xdr_array2_desc *desc, int encode) 1883 { 1884 char *elem = NULL, *c; 1885 unsigned int copied = 0, todo, avail_here; 1886 struct page **ppages = NULL; 1887 int err; 1888 1889 if (encode) { 1890 if (xdr_encode_word(buf, base, desc->array_len) != 0) 1891 return -EINVAL; 1892 } else { 1893 if (xdr_decode_word(buf, base, &desc->array_len) != 0 || 1894 desc->array_len > desc->array_maxlen || 1895 (unsigned long) base + 4 + desc->array_len * 1896 desc->elem_size > buf->len) 1897 return -EINVAL; 1898 } 1899 base += 4; 1900 1901 if (!desc->xcode) 1902 return 0; 1903 1904 todo = desc->array_len * desc->elem_size; 1905 1906 /* process head */ 1907 if (todo && base < buf->head->iov_len) { 1908 c = buf->head->iov_base + base; 1909 avail_here = min_t(unsigned int, todo, 1910 buf->head->iov_len - base); 1911 todo -= avail_here; 1912 1913 while (avail_here >= desc->elem_size) { 1914 err = desc->xcode(desc, c); 1915 if (err) 1916 goto out; 1917 c += desc->elem_size; 1918 avail_here -= desc->elem_size; 1919 } 1920 if (avail_here) { 1921 if (!elem) { 1922 elem = kmalloc(desc->elem_size, GFP_KERNEL); 1923 err = -ENOMEM; 1924 if (!elem) 1925 goto out; 1926 } 1927 if (encode) { 1928 err = desc->xcode(desc, elem); 1929 if (err) 1930 goto out; 1931 memcpy(c, elem, avail_here); 1932 } else 1933 memcpy(elem, c, avail_here); 1934 copied = avail_here; 1935 } 1936 base = buf->head->iov_len; /* align to start of pages */ 1937 } 1938 1939 /* process pages array */ 1940 base -= buf->head->iov_len; 1941 if (todo && base < buf->page_len) { 1942 unsigned int avail_page; 1943 1944 avail_here = min(todo, buf->page_len - base); 1945 todo -= avail_here; 1946 1947 base += buf->page_base; 1948 ppages = buf->pages + (base >> PAGE_SHIFT); 1949 base &= ~PAGE_MASK; 1950 avail_page = min_t(unsigned int, PAGE_SIZE - base, 1951 avail_here); 1952 c = kmap(*ppages) + base; 1953 1954 while (avail_here) { 1955 avail_here -= avail_page; 1956 if (copied || avail_page < desc->elem_size) { 1957 unsigned int l = min(avail_page, 1958 desc->elem_size - copied); 1959 if (!elem) { 1960 elem = kmalloc(desc->elem_size, 1961 GFP_KERNEL); 1962 err = -ENOMEM; 1963 if (!elem) 1964 goto out; 1965 } 1966 if (encode) { 1967 if (!copied) { 1968 err = desc->xcode(desc, elem); 1969 if (err) 1970 goto out; 1971 } 1972 memcpy(c, elem + copied, l); 1973 copied += l; 1974 if (copied == desc->elem_size) 1975 copied = 0; 1976 } else { 1977 memcpy(elem + copied, c, l); 1978 copied += l; 1979 if (copied == desc->elem_size) { 1980 err = desc->xcode(desc, elem); 1981 if (err) 1982 goto out; 1983 copied = 0; 1984 } 1985 } 1986 avail_page -= l; 1987 c += l; 1988 } 1989 while (avail_page >= desc->elem_size) { 1990 err = desc->xcode(desc, c); 1991 if (err) 1992 goto out; 1993 c += desc->elem_size; 1994 avail_page -= desc->elem_size; 1995 } 1996 if (avail_page) { 1997 unsigned int l = min(avail_page, 1998 desc->elem_size - copied); 1999 if (!elem) { 2000 elem = kmalloc(desc->elem_size, 2001 GFP_KERNEL); 2002 err = -ENOMEM; 2003 if (!elem) 2004 goto out; 2005 } 2006 if (encode) { 2007 if (!copied) { 2008 err = desc->xcode(desc, elem); 2009 if (err) 2010 goto out; 2011 } 2012 memcpy(c, elem + copied, l); 2013 copied += l; 2014 if (copied == desc->elem_size) 2015 copied = 0; 2016 } else { 2017 memcpy(elem + copied, c, l); 2018 copied += l; 2019 if (copied == desc->elem_size) { 2020 err = desc->xcode(desc, elem); 2021 if (err) 2022 goto out; 2023 copied = 0; 2024 } 2025 } 2026 } 2027 if (avail_here) { 2028 kunmap(*ppages); 2029 ppages++; 2030 c = kmap(*ppages); 2031 } 2032 2033 avail_page = min(avail_here, 2034 (unsigned int) PAGE_SIZE); 2035 } 2036 base = buf->page_len; /* align to start of tail */ 2037 } 2038 2039 /* process tail */ 2040 base -= buf->page_len; 2041 if (todo) { 2042 c = buf->tail->iov_base + base; 2043 if (copied) { 2044 unsigned int l = desc->elem_size - copied; 2045 2046 if (encode) 2047 memcpy(c, elem + copied, l); 2048 else { 2049 memcpy(elem + copied, c, l); 2050 err = desc->xcode(desc, elem); 2051 if (err) 2052 goto out; 2053 } 2054 todo -= l; 2055 c += l; 2056 } 2057 while (todo) { 2058 err = desc->xcode(desc, c); 2059 if (err) 2060 goto out; 2061 c += desc->elem_size; 2062 todo -= desc->elem_size; 2063 } 2064 } 2065 err = 0; 2066 2067 out: 2068 kfree(elem); 2069 if (ppages) 2070 kunmap(*ppages); 2071 return err; 2072 } 2073 2074 int xdr_decode_array2(const struct xdr_buf *buf, unsigned int base, 2075 struct xdr_array2_desc *desc) 2076 { 2077 if (base >= buf->len) 2078 return -EINVAL; 2079 2080 return xdr_xcode_array2(buf, base, desc, 0); 2081 } 2082 EXPORT_SYMBOL_GPL(xdr_decode_array2); 2083 2084 int xdr_encode_array2(const struct xdr_buf *buf, unsigned int base, 2085 struct xdr_array2_desc *desc) 2086 { 2087 if ((unsigned long) base + 4 + desc->array_len * desc->elem_size > 2088 buf->head->iov_len + buf->page_len + buf->tail->iov_len) 2089 return -EINVAL; 2090 2091 return xdr_xcode_array2(buf, base, desc, 1); 2092 } 2093 EXPORT_SYMBOL_GPL(xdr_encode_array2); 2094 2095 int xdr_process_buf(const struct xdr_buf *buf, unsigned int offset, 2096 unsigned int len, 2097 int (*actor)(struct scatterlist *, void *), void *data) 2098 { 2099 int i, ret = 0; 2100 unsigned int page_len, thislen, page_offset; 2101 struct scatterlist sg[1]; 2102 2103 sg_init_table(sg, 1); 2104 2105 if (offset >= buf->head[0].iov_len) { 2106 offset -= buf->head[0].iov_len; 2107 } else { 2108 thislen = buf->head[0].iov_len - offset; 2109 if (thislen > len) 2110 thislen = len; 2111 sg_set_buf(sg, buf->head[0].iov_base + offset, thislen); 2112 ret = actor(sg, data); 2113 if (ret) 2114 goto out; 2115 offset = 0; 2116 len -= thislen; 2117 } 2118 if (len == 0) 2119 goto out; 2120 2121 if (offset >= buf->page_len) { 2122 offset -= buf->page_len; 2123 } else { 2124 page_len = buf->page_len - offset; 2125 if (page_len > len) 2126 page_len = len; 2127 len -= page_len; 2128 page_offset = (offset + buf->page_base) & (PAGE_SIZE - 1); 2129 i = (offset + buf->page_base) >> PAGE_SHIFT; 2130 thislen = PAGE_SIZE - page_offset; 2131 do { 2132 if (thislen > page_len) 2133 thislen = page_len; 2134 sg_set_page(sg, buf->pages[i], thislen, page_offset); 2135 ret = actor(sg, data); 2136 if (ret) 2137 goto out; 2138 page_len -= thislen; 2139 i++; 2140 page_offset = 0; 2141 thislen = PAGE_SIZE; 2142 } while (page_len != 0); 2143 offset = 0; 2144 } 2145 if (len == 0) 2146 goto out; 2147 if (offset < buf->tail[0].iov_len) { 2148 thislen = buf->tail[0].iov_len - offset; 2149 if (thislen > len) 2150 thislen = len; 2151 sg_set_buf(sg, buf->tail[0].iov_base + offset, thislen); 2152 ret = actor(sg, data); 2153 len -= thislen; 2154 } 2155 if (len != 0) 2156 ret = -EINVAL; 2157 out: 2158 return ret; 2159 } 2160 EXPORT_SYMBOL_GPL(xdr_process_buf); 2161 2162 /** 2163 * xdr_stream_decode_opaque - Decode variable length opaque 2164 * @xdr: pointer to xdr_stream 2165 * @ptr: location to store opaque data 2166 * @size: size of storage buffer @ptr 2167 * 2168 * Return values: 2169 * On success, returns size of object stored in *@ptr 2170 * %-EBADMSG on XDR buffer overflow 2171 * %-EMSGSIZE on overflow of storage buffer @ptr 2172 */ 2173 ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, size_t size) 2174 { 2175 ssize_t ret; 2176 void *p; 2177 2178 ret = xdr_stream_decode_opaque_inline(xdr, &p, size); 2179 if (ret <= 0) 2180 return ret; 2181 memcpy(ptr, p, ret); 2182 return ret; 2183 } 2184 EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque); 2185 2186 /** 2187 * xdr_stream_decode_opaque_dup - Decode and duplicate variable length opaque 2188 * @xdr: pointer to xdr_stream 2189 * @ptr: location to store pointer to opaque data 2190 * @maxlen: maximum acceptable object size 2191 * @gfp_flags: GFP mask to use 2192 * 2193 * Return values: 2194 * On success, returns size of object stored in *@ptr 2195 * %-EBADMSG on XDR buffer overflow 2196 * %-EMSGSIZE if the size of the object would exceed @maxlen 2197 * %-ENOMEM on memory allocation failure 2198 */ 2199 ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr, 2200 size_t maxlen, gfp_t gfp_flags) 2201 { 2202 ssize_t ret; 2203 void *p; 2204 2205 ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen); 2206 if (ret > 0) { 2207 *ptr = kmemdup(p, ret, gfp_flags); 2208 if (*ptr != NULL) 2209 return ret; 2210 ret = -ENOMEM; 2211 } 2212 *ptr = NULL; 2213 return ret; 2214 } 2215 EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque_dup); 2216 2217 /** 2218 * xdr_stream_decode_string - Decode variable length string 2219 * @xdr: pointer to xdr_stream 2220 * @str: location to store string 2221 * @size: size of storage buffer @str 2222 * 2223 * Return values: 2224 * On success, returns length of NUL-terminated string stored in *@str 2225 * %-EBADMSG on XDR buffer overflow 2226 * %-EMSGSIZE on overflow of storage buffer @str 2227 */ 2228 ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, size_t size) 2229 { 2230 ssize_t ret; 2231 void *p; 2232 2233 ret = xdr_stream_decode_opaque_inline(xdr, &p, size); 2234 if (ret > 0) { 2235 memcpy(str, p, ret); 2236 str[ret] = '\0'; 2237 return strlen(str); 2238 } 2239 *str = '\0'; 2240 return ret; 2241 } 2242 EXPORT_SYMBOL_GPL(xdr_stream_decode_string); 2243 2244 /** 2245 * xdr_stream_decode_string_dup - Decode and duplicate variable length string 2246 * @xdr: pointer to xdr_stream 2247 * @str: location to store pointer to string 2248 * @maxlen: maximum acceptable string length 2249 * @gfp_flags: GFP mask to use 2250 * 2251 * Return values: 2252 * On success, returns length of NUL-terminated string stored in *@ptr 2253 * %-EBADMSG on XDR buffer overflow 2254 * %-EMSGSIZE if the size of the string would exceed @maxlen 2255 * %-ENOMEM on memory allocation failure 2256 */ 2257 ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str, 2258 size_t maxlen, gfp_t gfp_flags) 2259 { 2260 void *p; 2261 ssize_t ret; 2262 2263 ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen); 2264 if (ret > 0) { 2265 char *s = kmemdup_nul(p, ret, gfp_flags); 2266 if (s != NULL) { 2267 *str = s; 2268 return strlen(s); 2269 } 2270 ret = -ENOMEM; 2271 } 2272 *str = NULL; 2273 return ret; 2274 } 2275 EXPORT_SYMBOL_GPL(xdr_stream_decode_string_dup); 2276