1 /* 2 * af_alg: User-space algorithm interface 3 * 4 * This file provides the user-space API for algorithms. 5 * 6 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License as published by the Free 10 * Software Foundation; either version 2 of the License, or (at your option) 11 * any later version. 12 * 13 */ 14 15 #include <linux/atomic.h> 16 #include <crypto/if_alg.h> 17 #include <linux/crypto.h> 18 #include <linux/init.h> 19 #include <linux/kernel.h> 20 #include <linux/list.h> 21 #include <linux/module.h> 22 #include <linux/net.h> 23 #include <linux/rwsem.h> 24 #include <linux/sched/signal.h> 25 #include <linux/security.h> 26 27 struct alg_type_list { 28 const struct af_alg_type *type; 29 struct list_head list; 30 }; 31 32 static atomic_long_t alg_memory_allocated; 33 34 static struct proto alg_proto = { 35 .name = "ALG", 36 .owner = THIS_MODULE, 37 .memory_allocated = &alg_memory_allocated, 38 .obj_size = sizeof(struct alg_sock), 39 }; 40 41 static LIST_HEAD(alg_types); 42 static DECLARE_RWSEM(alg_types_sem); 43 44 static const struct af_alg_type *alg_get_type(const char *name) 45 { 46 const struct af_alg_type *type = ERR_PTR(-ENOENT); 47 struct alg_type_list *node; 48 49 down_read(&alg_types_sem); 50 list_for_each_entry(node, &alg_types, list) { 51 if (strcmp(node->type->name, name)) 52 continue; 53 54 if (try_module_get(node->type->owner)) 55 type = node->type; 56 break; 57 } 58 up_read(&alg_types_sem); 59 60 return type; 61 } 62 63 int af_alg_register_type(const struct af_alg_type *type) 64 { 65 struct alg_type_list *node; 66 int err = -EEXIST; 67 68 down_write(&alg_types_sem); 69 list_for_each_entry(node, &alg_types, list) { 70 if (!strcmp(node->type->name, type->name)) 71 goto unlock; 72 } 73 74 node = kmalloc(sizeof(*node), GFP_KERNEL); 75 err = -ENOMEM; 76 if (!node) 77 goto unlock; 78 79 type->ops->owner = THIS_MODULE; 80 if (type->ops_nokey) 81 type->ops_nokey->owner = THIS_MODULE; 82 node->type = type; 83 list_add(&node->list, &alg_types); 84 err = 0; 85 86 unlock: 87 up_write(&alg_types_sem); 88 89 return err; 90 } 91 EXPORT_SYMBOL_GPL(af_alg_register_type); 92 93 int af_alg_unregister_type(const struct af_alg_type *type) 94 { 95 struct alg_type_list *node; 96 int err = -ENOENT; 97 98 down_write(&alg_types_sem); 99 list_for_each_entry(node, &alg_types, list) { 100 if (strcmp(node->type->name, type->name)) 101 continue; 102 103 list_del(&node->list); 104 kfree(node); 105 err = 0; 106 break; 107 } 108 up_write(&alg_types_sem); 109 110 return err; 111 } 112 EXPORT_SYMBOL_GPL(af_alg_unregister_type); 113 114 static void alg_do_release(const struct af_alg_type *type, void *private) 115 { 116 if (!type) 117 return; 118 119 type->release(private); 120 module_put(type->owner); 121 } 122 123 int af_alg_release(struct socket *sock) 124 { 125 if (sock->sk) 126 sock_put(sock->sk); 127 return 0; 128 } 129 EXPORT_SYMBOL_GPL(af_alg_release); 130 131 void af_alg_release_parent(struct sock *sk) 132 { 133 struct alg_sock *ask = alg_sk(sk); 134 unsigned int nokey = ask->nokey_refcnt; 135 bool last = nokey && !ask->refcnt; 136 137 sk = ask->parent; 138 ask = alg_sk(sk); 139 140 lock_sock(sk); 141 ask->nokey_refcnt -= nokey; 142 if (!last) 143 last = !--ask->refcnt; 144 release_sock(sk); 145 146 if (last) 147 sock_put(sk); 148 } 149 EXPORT_SYMBOL_GPL(af_alg_release_parent); 150 151 static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 152 { 153 const u32 forbidden = CRYPTO_ALG_INTERNAL; 154 struct sock *sk = sock->sk; 155 struct alg_sock *ask = alg_sk(sk); 156 struct sockaddr_alg *sa = (void *)uaddr; 157 const struct af_alg_type *type; 158 void *private; 159 int err; 160 161 if (sock->state == SS_CONNECTED) 162 return -EINVAL; 163 164 if (addr_len < sizeof(*sa)) 165 return -EINVAL; 166 167 sa->salg_type[sizeof(sa->salg_type) - 1] = 0; 168 sa->salg_name[sizeof(sa->salg_name) + addr_len - sizeof(*sa) - 1] = 0; 169 170 type = alg_get_type(sa->salg_type); 171 if (IS_ERR(type) && PTR_ERR(type) == -ENOENT) { 172 request_module("algif-%s", sa->salg_type); 173 type = alg_get_type(sa->salg_type); 174 } 175 176 if (IS_ERR(type)) 177 return PTR_ERR(type); 178 179 private = type->bind(sa->salg_name, 180 sa->salg_feat & ~forbidden, 181 sa->salg_mask & ~forbidden); 182 if (IS_ERR(private)) { 183 module_put(type->owner); 184 return PTR_ERR(private); 185 } 186 187 err = -EBUSY; 188 lock_sock(sk); 189 if (ask->refcnt | ask->nokey_refcnt) 190 goto unlock; 191 192 swap(ask->type, type); 193 swap(ask->private, private); 194 195 err = 0; 196 197 unlock: 198 release_sock(sk); 199 200 alg_do_release(type, private); 201 202 return err; 203 } 204 205 static int alg_setkey(struct sock *sk, char __user *ukey, 206 unsigned int keylen) 207 { 208 struct alg_sock *ask = alg_sk(sk); 209 const struct af_alg_type *type = ask->type; 210 u8 *key; 211 int err; 212 213 key = sock_kmalloc(sk, keylen, GFP_KERNEL); 214 if (!key) 215 return -ENOMEM; 216 217 err = -EFAULT; 218 if (copy_from_user(key, ukey, keylen)) 219 goto out; 220 221 err = type->setkey(ask->private, key, keylen); 222 223 out: 224 sock_kzfree_s(sk, key, keylen); 225 226 return err; 227 } 228 229 static int alg_setsockopt(struct socket *sock, int level, int optname, 230 char __user *optval, unsigned int optlen) 231 { 232 struct sock *sk = sock->sk; 233 struct alg_sock *ask = alg_sk(sk); 234 const struct af_alg_type *type; 235 int err = -EBUSY; 236 237 lock_sock(sk); 238 if (ask->refcnt) 239 goto unlock; 240 241 type = ask->type; 242 243 err = -ENOPROTOOPT; 244 if (level != SOL_ALG || !type) 245 goto unlock; 246 247 switch (optname) { 248 case ALG_SET_KEY: 249 if (sock->state == SS_CONNECTED) 250 goto unlock; 251 if (!type->setkey) 252 goto unlock; 253 254 err = alg_setkey(sk, optval, optlen); 255 break; 256 case ALG_SET_AEAD_AUTHSIZE: 257 if (sock->state == SS_CONNECTED) 258 goto unlock; 259 if (!type->setauthsize) 260 goto unlock; 261 err = type->setauthsize(ask->private, optlen); 262 } 263 264 unlock: 265 release_sock(sk); 266 267 return err; 268 } 269 270 int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern) 271 { 272 struct alg_sock *ask = alg_sk(sk); 273 const struct af_alg_type *type; 274 struct sock *sk2; 275 unsigned int nokey; 276 int err; 277 278 lock_sock(sk); 279 type = ask->type; 280 281 err = -EINVAL; 282 if (!type) 283 goto unlock; 284 285 sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto, kern); 286 err = -ENOMEM; 287 if (!sk2) 288 goto unlock; 289 290 sock_init_data(newsock, sk2); 291 security_sock_graft(sk2, newsock); 292 security_sk_clone(sk, sk2); 293 294 err = type->accept(ask->private, sk2); 295 296 nokey = err == -ENOKEY; 297 if (nokey && type->accept_nokey) 298 err = type->accept_nokey(ask->private, sk2); 299 300 if (err) 301 goto unlock; 302 303 sk2->sk_family = PF_ALG; 304 305 if (nokey || !ask->refcnt++) 306 sock_hold(sk); 307 ask->nokey_refcnt += nokey; 308 alg_sk(sk2)->parent = sk; 309 alg_sk(sk2)->type = type; 310 alg_sk(sk2)->nokey_refcnt = nokey; 311 312 newsock->ops = type->ops; 313 newsock->state = SS_CONNECTED; 314 315 if (nokey) 316 newsock->ops = type->ops_nokey; 317 318 err = 0; 319 320 unlock: 321 release_sock(sk); 322 323 return err; 324 } 325 EXPORT_SYMBOL_GPL(af_alg_accept); 326 327 static int alg_accept(struct socket *sock, struct socket *newsock, int flags, 328 bool kern) 329 { 330 return af_alg_accept(sock->sk, newsock, kern); 331 } 332 333 static const struct proto_ops alg_proto_ops = { 334 .family = PF_ALG, 335 .owner = THIS_MODULE, 336 337 .connect = sock_no_connect, 338 .socketpair = sock_no_socketpair, 339 .getname = sock_no_getname, 340 .ioctl = sock_no_ioctl, 341 .listen = sock_no_listen, 342 .shutdown = sock_no_shutdown, 343 .getsockopt = sock_no_getsockopt, 344 .mmap = sock_no_mmap, 345 .sendpage = sock_no_sendpage, 346 .sendmsg = sock_no_sendmsg, 347 .recvmsg = sock_no_recvmsg, 348 .poll = sock_no_poll, 349 350 .bind = alg_bind, 351 .release = af_alg_release, 352 .setsockopt = alg_setsockopt, 353 .accept = alg_accept, 354 }; 355 356 static void alg_sock_destruct(struct sock *sk) 357 { 358 struct alg_sock *ask = alg_sk(sk); 359 360 alg_do_release(ask->type, ask->private); 361 } 362 363 static int alg_create(struct net *net, struct socket *sock, int protocol, 364 int kern) 365 { 366 struct sock *sk; 367 int err; 368 369 if (sock->type != SOCK_SEQPACKET) 370 return -ESOCKTNOSUPPORT; 371 if (protocol != 0) 372 return -EPROTONOSUPPORT; 373 374 err = -ENOMEM; 375 sk = sk_alloc(net, PF_ALG, GFP_KERNEL, &alg_proto, kern); 376 if (!sk) 377 goto out; 378 379 sock->ops = &alg_proto_ops; 380 sock_init_data(sock, sk); 381 382 sk->sk_family = PF_ALG; 383 sk->sk_destruct = alg_sock_destruct; 384 385 return 0; 386 out: 387 return err; 388 } 389 390 static const struct net_proto_family alg_family = { 391 .family = PF_ALG, 392 .create = alg_create, 393 .owner = THIS_MODULE, 394 }; 395 396 int af_alg_make_sg(struct af_alg_sgl *sgl, struct iov_iter *iter, int len) 397 { 398 size_t off; 399 ssize_t n; 400 int npages, i; 401 402 n = iov_iter_get_pages(iter, sgl->pages, len, ALG_MAX_PAGES, &off); 403 if (n < 0) 404 return n; 405 406 npages = (off + n + PAGE_SIZE - 1) >> PAGE_SHIFT; 407 if (WARN_ON(npages == 0)) 408 return -EINVAL; 409 /* Add one extra for linking */ 410 sg_init_table(sgl->sg, npages + 1); 411 412 for (i = 0, len = n; i < npages; i++) { 413 int plen = min_t(int, len, PAGE_SIZE - off); 414 415 sg_set_page(sgl->sg + i, sgl->pages[i], plen, off); 416 417 off = 0; 418 len -= plen; 419 } 420 sg_mark_end(sgl->sg + npages - 1); 421 sgl->npages = npages; 422 423 return n; 424 } 425 EXPORT_SYMBOL_GPL(af_alg_make_sg); 426 427 void af_alg_link_sg(struct af_alg_sgl *sgl_prev, struct af_alg_sgl *sgl_new) 428 { 429 sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1); 430 sg_chain(sgl_prev->sg, sgl_prev->npages + 1, sgl_new->sg); 431 } 432 EXPORT_SYMBOL_GPL(af_alg_link_sg); 433 434 void af_alg_free_sg(struct af_alg_sgl *sgl) 435 { 436 int i; 437 438 for (i = 0; i < sgl->npages; i++) 439 put_page(sgl->pages[i]); 440 } 441 EXPORT_SYMBOL_GPL(af_alg_free_sg); 442 443 int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) 444 { 445 struct cmsghdr *cmsg; 446 447 for_each_cmsghdr(cmsg, msg) { 448 if (!CMSG_OK(msg, cmsg)) 449 return -EINVAL; 450 if (cmsg->cmsg_level != SOL_ALG) 451 continue; 452 453 switch (cmsg->cmsg_type) { 454 case ALG_SET_IV: 455 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv))) 456 return -EINVAL; 457 con->iv = (void *)CMSG_DATA(cmsg); 458 if (cmsg->cmsg_len < CMSG_LEN(con->iv->ivlen + 459 sizeof(*con->iv))) 460 return -EINVAL; 461 break; 462 463 case ALG_SET_OP: 464 if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) 465 return -EINVAL; 466 con->op = *(u32 *)CMSG_DATA(cmsg); 467 break; 468 469 case ALG_SET_AEAD_ASSOCLEN: 470 if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) 471 return -EINVAL; 472 con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg); 473 break; 474 475 default: 476 return -EINVAL; 477 } 478 } 479 480 return 0; 481 } 482 EXPORT_SYMBOL_GPL(af_alg_cmsg_send); 483 484 /** 485 * af_alg_alloc_tsgl - allocate the TX SGL 486 * 487 * @sk socket of connection to user space 488 * @return: 0 upon success, < 0 upon error 489 */ 490 int af_alg_alloc_tsgl(struct sock *sk) 491 { 492 struct alg_sock *ask = alg_sk(sk); 493 struct af_alg_ctx *ctx = ask->private; 494 struct af_alg_tsgl *sgl; 495 struct scatterlist *sg = NULL; 496 497 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); 498 if (!list_empty(&ctx->tsgl_list)) 499 sg = sgl->sg; 500 501 if (!sg || sgl->cur >= MAX_SGL_ENTS) { 502 sgl = sock_kmalloc(sk, sizeof(*sgl) + 503 sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1), 504 GFP_KERNEL); 505 if (!sgl) 506 return -ENOMEM; 507 508 sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); 509 sgl->cur = 0; 510 511 if (sg) 512 sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); 513 514 list_add_tail(&sgl->list, &ctx->tsgl_list); 515 } 516 517 return 0; 518 } 519 EXPORT_SYMBOL_GPL(af_alg_alloc_tsgl); 520 521 /** 522 * aead_count_tsgl - Count number of TX SG entries 523 * 524 * The counting starts from the beginning of the SGL to @bytes. If 525 * an offset is provided, the counting of the SG entries starts at the offset. 526 * 527 * @sk socket of connection to user space 528 * @bytes Count the number of SG entries holding given number of bytes. 529 * @offset Start the counting of SG entries from the given offset. 530 * @return Number of TX SG entries found given the constraints 531 */ 532 unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset) 533 { 534 struct alg_sock *ask = alg_sk(sk); 535 struct af_alg_ctx *ctx = ask->private; 536 struct af_alg_tsgl *sgl, *tmp; 537 unsigned int i; 538 unsigned int sgl_count = 0; 539 540 if (!bytes) 541 return 0; 542 543 list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) { 544 struct scatterlist *sg = sgl->sg; 545 546 for (i = 0; i < sgl->cur; i++) { 547 size_t bytes_count; 548 549 /* Skip offset */ 550 if (offset >= sg[i].length) { 551 offset -= sg[i].length; 552 bytes -= sg[i].length; 553 continue; 554 } 555 556 bytes_count = sg[i].length - offset; 557 558 offset = 0; 559 sgl_count++; 560 561 /* If we have seen requested number of bytes, stop */ 562 if (bytes_count >= bytes) 563 return sgl_count; 564 565 bytes -= bytes_count; 566 } 567 } 568 569 return sgl_count; 570 } 571 EXPORT_SYMBOL_GPL(af_alg_count_tsgl); 572 573 /** 574 * aead_pull_tsgl - Release the specified buffers from TX SGL 575 * 576 * If @dst is non-null, reassign the pages to dst. The caller must release 577 * the pages. If @dst_offset is given only reassign the pages to @dst starting 578 * at the @dst_offset (byte). The caller must ensure that @dst is large 579 * enough (e.g. by using af_alg_count_tsgl with the same offset). 580 * 581 * @sk socket of connection to user space 582 * @used Number of bytes to pull from TX SGL 583 * @dst If non-NULL, buffer is reassigned to dst SGL instead of releasing. The 584 * caller must release the buffers in dst. 585 * @dst_offset Reassign the TX SGL from given offset. All buffers before 586 * reaching the offset is released. 587 */ 588 void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, 589 size_t dst_offset) 590 { 591 struct alg_sock *ask = alg_sk(sk); 592 struct af_alg_ctx *ctx = ask->private; 593 struct af_alg_tsgl *sgl; 594 struct scatterlist *sg; 595 unsigned int i, j = 0; 596 597 while (!list_empty(&ctx->tsgl_list)) { 598 sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl, 599 list); 600 sg = sgl->sg; 601 602 for (i = 0; i < sgl->cur; i++) { 603 size_t plen = min_t(size_t, used, sg[i].length); 604 struct page *page = sg_page(sg + i); 605 606 if (!page) 607 continue; 608 609 /* 610 * Assumption: caller created af_alg_count_tsgl(len) 611 * SG entries in dst. 612 */ 613 if (dst) { 614 if (dst_offset >= plen) { 615 /* discard page before offset */ 616 dst_offset -= plen; 617 } else { 618 /* reassign page to dst after offset */ 619 get_page(page); 620 sg_set_page(dst + j, page, 621 plen - dst_offset, 622 sg[i].offset + dst_offset); 623 dst_offset = 0; 624 j++; 625 } 626 } 627 628 sg[i].length -= plen; 629 sg[i].offset += plen; 630 631 used -= plen; 632 ctx->used -= plen; 633 634 if (sg[i].length) 635 return; 636 637 put_page(page); 638 sg_assign_page(sg + i, NULL); 639 } 640 641 list_del(&sgl->list); 642 sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) * 643 (MAX_SGL_ENTS + 1)); 644 } 645 646 if (!ctx->used) 647 ctx->merge = 0; 648 } 649 EXPORT_SYMBOL_GPL(af_alg_pull_tsgl); 650 651 /** 652 * af_alg_free_areq_sgls - Release TX and RX SGLs of the request 653 * 654 * @areq Request holding the TX and RX SGL 655 */ 656 void af_alg_free_areq_sgls(struct af_alg_async_req *areq) 657 { 658 struct sock *sk = areq->sk; 659 struct alg_sock *ask = alg_sk(sk); 660 struct af_alg_ctx *ctx = ask->private; 661 struct af_alg_rsgl *rsgl, *tmp; 662 struct scatterlist *tsgl; 663 struct scatterlist *sg; 664 unsigned int i; 665 666 list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { 667 ctx->rcvused -= rsgl->sg_num_bytes; 668 af_alg_free_sg(&rsgl->sgl); 669 list_del(&rsgl->list); 670 if (rsgl != &areq->first_rsgl) 671 sock_kfree_s(sk, rsgl, sizeof(*rsgl)); 672 } 673 674 tsgl = areq->tsgl; 675 for_each_sg(tsgl, sg, areq->tsgl_entries, i) { 676 if (!sg_page(sg)) 677 continue; 678 put_page(sg_page(sg)); 679 } 680 681 if (areq->tsgl && areq->tsgl_entries) 682 sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); 683 } 684 EXPORT_SYMBOL_GPL(af_alg_free_areq_sgls); 685 686 /** 687 * af_alg_wait_for_wmem - wait for availability of writable memory 688 * 689 * @sk socket of connection to user space 690 * @flags If MSG_DONTWAIT is set, then only report if function would sleep 691 * @return 0 when writable memory is available, < 0 upon error 692 */ 693 int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags) 694 { 695 DEFINE_WAIT_FUNC(wait, woken_wake_function); 696 int err = -ERESTARTSYS; 697 long timeout; 698 699 if (flags & MSG_DONTWAIT) 700 return -EAGAIN; 701 702 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 703 704 add_wait_queue(sk_sleep(sk), &wait); 705 for (;;) { 706 if (signal_pending(current)) 707 break; 708 timeout = MAX_SCHEDULE_TIMEOUT; 709 if (sk_wait_event(sk, &timeout, af_alg_writable(sk), &wait)) { 710 err = 0; 711 break; 712 } 713 } 714 remove_wait_queue(sk_sleep(sk), &wait); 715 716 return err; 717 } 718 EXPORT_SYMBOL_GPL(af_alg_wait_for_wmem); 719 720 /** 721 * af_alg_wmem_wakeup - wakeup caller when writable memory is available 722 * 723 * @sk socket of connection to user space 724 */ 725 void af_alg_wmem_wakeup(struct sock *sk) 726 { 727 struct socket_wq *wq; 728 729 if (!af_alg_writable(sk)) 730 return; 731 732 rcu_read_lock(); 733 wq = rcu_dereference(sk->sk_wq); 734 if (skwq_has_sleeper(wq)) 735 wake_up_interruptible_sync_poll(&wq->wait, POLLIN | 736 POLLRDNORM | 737 POLLRDBAND); 738 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 739 rcu_read_unlock(); 740 } 741 EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup); 742 743 /** 744 * af_alg_wait_for_data - wait for availability of TX data 745 * 746 * @sk socket of connection to user space 747 * @flags If MSG_DONTWAIT is set, then only report if function would sleep 748 * @return 0 when writable memory is available, < 0 upon error 749 */ 750 int af_alg_wait_for_data(struct sock *sk, unsigned flags) 751 { 752 DEFINE_WAIT_FUNC(wait, woken_wake_function); 753 struct alg_sock *ask = alg_sk(sk); 754 struct af_alg_ctx *ctx = ask->private; 755 long timeout; 756 int err = -ERESTARTSYS; 757 758 if (flags & MSG_DONTWAIT) 759 return -EAGAIN; 760 761 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 762 763 add_wait_queue(sk_sleep(sk), &wait); 764 for (;;) { 765 if (signal_pending(current)) 766 break; 767 timeout = MAX_SCHEDULE_TIMEOUT; 768 if (sk_wait_event(sk, &timeout, (ctx->used || !ctx->more), 769 &wait)) { 770 err = 0; 771 break; 772 } 773 } 774 remove_wait_queue(sk_sleep(sk), &wait); 775 776 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 777 778 return err; 779 } 780 EXPORT_SYMBOL_GPL(af_alg_wait_for_data); 781 782 /** 783 * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel 784 * 785 * @sk socket of connection to user space 786 */ 787 788 void af_alg_data_wakeup(struct sock *sk) 789 { 790 struct alg_sock *ask = alg_sk(sk); 791 struct af_alg_ctx *ctx = ask->private; 792 struct socket_wq *wq; 793 794 if (!ctx->used) 795 return; 796 797 rcu_read_lock(); 798 wq = rcu_dereference(sk->sk_wq); 799 if (skwq_has_sleeper(wq)) 800 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT | 801 POLLRDNORM | 802 POLLRDBAND); 803 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 804 rcu_read_unlock(); 805 } 806 EXPORT_SYMBOL_GPL(af_alg_data_wakeup); 807 808 /** 809 * af_alg_sendmsg - implementation of sendmsg system call handler 810 * 811 * The sendmsg system call handler obtains the user data and stores it 812 * in ctx->tsgl_list. This implies allocation of the required numbers of 813 * struct af_alg_tsgl. 814 * 815 * In addition, the ctx is filled with the information sent via CMSG. 816 * 817 * @sock socket of connection to user space 818 * @msg message from user space 819 * @size size of message from user space 820 * @ivsize the size of the IV for the cipher operation to verify that the 821 * user-space-provided IV has the right size 822 * @return the number of copied data upon success, < 0 upon error 823 */ 824 int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, 825 unsigned int ivsize) 826 { 827 struct sock *sk = sock->sk; 828 struct alg_sock *ask = alg_sk(sk); 829 struct af_alg_ctx *ctx = ask->private; 830 struct af_alg_tsgl *sgl; 831 struct af_alg_control con = {}; 832 long copied = 0; 833 bool enc = 0; 834 bool init = 0; 835 int err = 0; 836 837 if (msg->msg_controllen) { 838 err = af_alg_cmsg_send(msg, &con); 839 if (err) 840 return err; 841 842 init = 1; 843 switch (con.op) { 844 case ALG_OP_ENCRYPT: 845 enc = 1; 846 break; 847 case ALG_OP_DECRYPT: 848 enc = 0; 849 break; 850 default: 851 return -EINVAL; 852 } 853 854 if (con.iv && con.iv->ivlen != ivsize) 855 return -EINVAL; 856 } 857 858 lock_sock(sk); 859 if (!ctx->more && ctx->used) { 860 err = -EINVAL; 861 goto unlock; 862 } 863 864 if (init) { 865 ctx->enc = enc; 866 if (con.iv) 867 memcpy(ctx->iv, con.iv->iv, ivsize); 868 869 ctx->aead_assoclen = con.aead_assoclen; 870 } 871 872 while (size) { 873 struct scatterlist *sg; 874 size_t len = size; 875 size_t plen; 876 877 /* use the existing memory in an allocated page */ 878 if (ctx->merge) { 879 sgl = list_entry(ctx->tsgl_list.prev, 880 struct af_alg_tsgl, list); 881 sg = sgl->sg + sgl->cur - 1; 882 len = min_t(size_t, len, 883 PAGE_SIZE - sg->offset - sg->length); 884 885 err = memcpy_from_msg(page_address(sg_page(sg)) + 886 sg->offset + sg->length, 887 msg, len); 888 if (err) 889 goto unlock; 890 891 sg->length += len; 892 ctx->merge = (sg->offset + sg->length) & 893 (PAGE_SIZE - 1); 894 895 ctx->used += len; 896 copied += len; 897 size -= len; 898 continue; 899 } 900 901 if (!af_alg_writable(sk)) { 902 err = af_alg_wait_for_wmem(sk, msg->msg_flags); 903 if (err) 904 goto unlock; 905 } 906 907 /* allocate a new page */ 908 len = min_t(unsigned long, len, af_alg_sndbuf(sk)); 909 910 err = af_alg_alloc_tsgl(sk); 911 if (err) 912 goto unlock; 913 914 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, 915 list); 916 sg = sgl->sg; 917 if (sgl->cur) 918 sg_unmark_end(sg + sgl->cur - 1); 919 920 do { 921 unsigned int i = sgl->cur; 922 923 plen = min_t(size_t, len, PAGE_SIZE); 924 925 sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); 926 if (!sg_page(sg + i)) { 927 err = -ENOMEM; 928 goto unlock; 929 } 930 931 err = memcpy_from_msg(page_address(sg_page(sg + i)), 932 msg, plen); 933 if (err) { 934 __free_page(sg_page(sg + i)); 935 sg_assign_page(sg + i, NULL); 936 goto unlock; 937 } 938 939 sg[i].length = plen; 940 len -= plen; 941 ctx->used += plen; 942 copied += plen; 943 size -= plen; 944 sgl->cur++; 945 } while (len && sgl->cur < MAX_SGL_ENTS); 946 947 if (!size) 948 sg_mark_end(sg + sgl->cur - 1); 949 950 ctx->merge = plen & (PAGE_SIZE - 1); 951 } 952 953 err = 0; 954 955 ctx->more = msg->msg_flags & MSG_MORE; 956 957 unlock: 958 af_alg_data_wakeup(sk); 959 release_sock(sk); 960 961 return copied ?: err; 962 } 963 EXPORT_SYMBOL_GPL(af_alg_sendmsg); 964 965 /** 966 * af_alg_sendpage - sendpage system call handler 967 * 968 * This is a generic implementation of sendpage to fill ctx->tsgl_list. 969 */ 970 ssize_t af_alg_sendpage(struct socket *sock, struct page *page, 971 int offset, size_t size, int flags) 972 { 973 struct sock *sk = sock->sk; 974 struct alg_sock *ask = alg_sk(sk); 975 struct af_alg_ctx *ctx = ask->private; 976 struct af_alg_tsgl *sgl; 977 int err = -EINVAL; 978 979 if (flags & MSG_SENDPAGE_NOTLAST) 980 flags |= MSG_MORE; 981 982 lock_sock(sk); 983 if (!ctx->more && ctx->used) 984 goto unlock; 985 986 if (!size) 987 goto done; 988 989 if (!af_alg_writable(sk)) { 990 err = af_alg_wait_for_wmem(sk, flags); 991 if (err) 992 goto unlock; 993 } 994 995 err = af_alg_alloc_tsgl(sk); 996 if (err) 997 goto unlock; 998 999 ctx->merge = 0; 1000 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); 1001 1002 if (sgl->cur) 1003 sg_unmark_end(sgl->sg + sgl->cur - 1); 1004 1005 sg_mark_end(sgl->sg + sgl->cur); 1006 1007 get_page(page); 1008 sg_set_page(sgl->sg + sgl->cur, page, size, offset); 1009 sgl->cur++; 1010 ctx->used += size; 1011 1012 done: 1013 ctx->more = flags & MSG_MORE; 1014 1015 unlock: 1016 af_alg_data_wakeup(sk); 1017 release_sock(sk); 1018 1019 return err ?: size; 1020 } 1021 EXPORT_SYMBOL_GPL(af_alg_sendpage); 1022 1023 /** 1024 * af_alg_async_cb - AIO callback handler 1025 * 1026 * This handler cleans up the struct af_alg_async_req upon completion of the 1027 * AIO operation. 1028 * 1029 * The number of bytes to be generated with the AIO operation must be set 1030 * in areq->outlen before the AIO callback handler is invoked. 1031 */ 1032 void af_alg_async_cb(struct crypto_async_request *_req, int err) 1033 { 1034 struct af_alg_async_req *areq = _req->data; 1035 struct sock *sk = areq->sk; 1036 struct kiocb *iocb = areq->iocb; 1037 unsigned int resultlen; 1038 1039 lock_sock(sk); 1040 1041 /* Buffer size written by crypto operation. */ 1042 resultlen = areq->outlen; 1043 1044 af_alg_free_areq_sgls(areq); 1045 sock_kfree_s(sk, areq, areq->areqlen); 1046 __sock_put(sk); 1047 1048 iocb->ki_complete(iocb, err ? err : resultlen, 0); 1049 1050 release_sock(sk); 1051 } 1052 EXPORT_SYMBOL_GPL(af_alg_async_cb); 1053 1054 /** 1055 * af_alg_poll - poll system call handler 1056 */ 1057 __poll_t af_alg_poll(struct file *file, struct socket *sock, 1058 poll_table *wait) 1059 { 1060 struct sock *sk = sock->sk; 1061 struct alg_sock *ask = alg_sk(sk); 1062 struct af_alg_ctx *ctx = ask->private; 1063 __poll_t mask; 1064 1065 sock_poll_wait(file, sk_sleep(sk), wait); 1066 mask = 0; 1067 1068 if (!ctx->more || ctx->used) 1069 mask |= POLLIN | POLLRDNORM; 1070 1071 if (af_alg_writable(sk)) 1072 mask |= POLLOUT | POLLWRNORM | POLLWRBAND; 1073 1074 return mask; 1075 } 1076 EXPORT_SYMBOL_GPL(af_alg_poll); 1077 1078 /** 1079 * af_alg_alloc_areq - allocate struct af_alg_async_req 1080 * 1081 * @sk socket of connection to user space 1082 * @areqlen size of struct af_alg_async_req + crypto_*_reqsize 1083 * @return allocated data structure or ERR_PTR upon error 1084 */ 1085 struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, 1086 unsigned int areqlen) 1087 { 1088 struct af_alg_async_req *areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); 1089 1090 if (unlikely(!areq)) 1091 return ERR_PTR(-ENOMEM); 1092 1093 areq->areqlen = areqlen; 1094 areq->sk = sk; 1095 areq->last_rsgl = NULL; 1096 INIT_LIST_HEAD(&areq->rsgl_list); 1097 areq->tsgl = NULL; 1098 areq->tsgl_entries = 0; 1099 1100 return areq; 1101 } 1102 EXPORT_SYMBOL_GPL(af_alg_alloc_areq); 1103 1104 /** 1105 * af_alg_get_rsgl - create the RX SGL for the output data from the crypto 1106 * operation 1107 * 1108 * @sk socket of connection to user space 1109 * @msg user space message 1110 * @flags flags used to invoke recvmsg with 1111 * @areq instance of the cryptographic request that will hold the RX SGL 1112 * @maxsize maximum number of bytes to be pulled from user space 1113 * @outlen number of bytes in the RX SGL 1114 * @return 0 on success, < 0 upon error 1115 */ 1116 int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, 1117 struct af_alg_async_req *areq, size_t maxsize, 1118 size_t *outlen) 1119 { 1120 struct alg_sock *ask = alg_sk(sk); 1121 struct af_alg_ctx *ctx = ask->private; 1122 size_t len = 0; 1123 1124 while (maxsize > len && msg_data_left(msg)) { 1125 struct af_alg_rsgl *rsgl; 1126 size_t seglen; 1127 int err; 1128 1129 /* limit the amount of readable buffers */ 1130 if (!af_alg_readable(sk)) 1131 break; 1132 1133 if (!ctx->used) { 1134 err = af_alg_wait_for_data(sk, flags); 1135 if (err) 1136 return err; 1137 } 1138 1139 seglen = min_t(size_t, (maxsize - len), 1140 msg_data_left(msg)); 1141 1142 if (list_empty(&areq->rsgl_list)) { 1143 rsgl = &areq->first_rsgl; 1144 } else { 1145 rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); 1146 if (unlikely(!rsgl)) 1147 return -ENOMEM; 1148 } 1149 1150 rsgl->sgl.npages = 0; 1151 list_add_tail(&rsgl->list, &areq->rsgl_list); 1152 1153 /* make one iovec available as scatterlist */ 1154 err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); 1155 if (err < 0) 1156 return err; 1157 1158 /* chain the new scatterlist with previous one */ 1159 if (areq->last_rsgl) 1160 af_alg_link_sg(&areq->last_rsgl->sgl, &rsgl->sgl); 1161 1162 areq->last_rsgl = rsgl; 1163 len += err; 1164 ctx->rcvused += err; 1165 rsgl->sg_num_bytes = err; 1166 iov_iter_advance(&msg->msg_iter, err); 1167 } 1168 1169 *outlen = len; 1170 return 0; 1171 } 1172 EXPORT_SYMBOL_GPL(af_alg_get_rsgl); 1173 1174 static int __init af_alg_init(void) 1175 { 1176 int err = proto_register(&alg_proto, 0); 1177 1178 if (err) 1179 goto out; 1180 1181 err = sock_register(&alg_family); 1182 if (err != 0) 1183 goto out_unregister_proto; 1184 1185 out: 1186 return err; 1187 1188 out_unregister_proto: 1189 proto_unregister(&alg_proto); 1190 goto out; 1191 } 1192 1193 static void __exit af_alg_exit(void) 1194 { 1195 sock_unregister(PF_ALG); 1196 proto_unregister(&alg_proto); 1197 } 1198 1199 module_init(af_alg_init); 1200 module_exit(af_alg_exit); 1201 MODULE_LICENSE("GPL"); 1202 MODULE_ALIAS_NETPROTO(AF_ALG); 1203