1 /* 2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved. 5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved. 6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved. 7 * 8 * This software is available to you under a choice of one of two 9 * licenses. You may choose to be licensed under the terms of the GNU 10 * General Public License (GPL) Version 2, available from the file 11 * COPYING in the main directory of this source tree, or the 12 * OpenIB.org BSD license below: 13 * 14 * Redistribution and use in source and binary forms, with or 15 * without modification, are permitted provided that the following 16 * conditions are met: 17 * 18 * - Redistributions of source code must retain the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer. 21 * 22 * - Redistributions in binary form must reproduce the above 23 * copyright notice, this list of conditions and the following 24 * disclaimer in the documentation and/or other materials 25 * provided with the distribution. 26 * 27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34 * SOFTWARE. 35 */ 36 37 #include <linux/sched/signal.h> 38 #include <linux/module.h> 39 #include <crypto/aead.h> 40 41 #include <net/strparser.h> 42 #include <net/tls.h> 43 44 #define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE 45 46 static int tls_do_decryption(struct sock *sk, 47 struct scatterlist *sgin, 48 struct scatterlist *sgout, 49 char *iv_recv, 50 size_t data_len, 51 struct sk_buff *skb, 52 gfp_t flags) 53 { 54 struct tls_context *tls_ctx = tls_get_ctx(sk); 55 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 56 struct aead_request *aead_req; 57 58 int ret; 59 60 aead_req = aead_request_alloc(ctx->aead_recv, flags); 61 if (!aead_req) 62 return -ENOMEM; 63 64 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); 65 aead_request_set_crypt(aead_req, sgin, sgout, 66 data_len + tls_ctx->rx.tag_size, 67 (u8 *)iv_recv); 68 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 69 crypto_req_done, &ctx->async_wait); 70 71 ret = crypto_wait_req(crypto_aead_decrypt(aead_req), &ctx->async_wait); 72 73 aead_request_free(aead_req); 74 return ret; 75 } 76 77 static void trim_sg(struct sock *sk, struct scatterlist *sg, 78 int *sg_num_elem, unsigned int *sg_size, int target_size) 79 { 80 int i = *sg_num_elem - 1; 81 int trim = *sg_size - target_size; 82 83 if (trim <= 0) { 84 WARN_ON(trim < 0); 85 return; 86 } 87 88 *sg_size = target_size; 89 while (trim >= sg[i].length) { 90 trim -= sg[i].length; 91 sk_mem_uncharge(sk, sg[i].length); 92 put_page(sg_page(&sg[i])); 93 i--; 94 95 if (i < 0) 96 goto out; 97 } 98 99 sg[i].length -= trim; 100 sk_mem_uncharge(sk, trim); 101 102 out: 103 *sg_num_elem = i + 1; 104 } 105 106 static void trim_both_sgl(struct sock *sk, int target_size) 107 { 108 struct tls_context *tls_ctx = tls_get_ctx(sk); 109 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 110 111 trim_sg(sk, ctx->sg_plaintext_data, 112 &ctx->sg_plaintext_num_elem, 113 &ctx->sg_plaintext_size, 114 target_size); 115 116 if (target_size > 0) 117 target_size += tls_ctx->tx.overhead_size; 118 119 trim_sg(sk, ctx->sg_encrypted_data, 120 &ctx->sg_encrypted_num_elem, 121 &ctx->sg_encrypted_size, 122 target_size); 123 } 124 125 static int alloc_encrypted_sg(struct sock *sk, int len) 126 { 127 struct tls_context *tls_ctx = tls_get_ctx(sk); 128 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 129 int rc = 0; 130 131 rc = sk_alloc_sg(sk, len, 132 ctx->sg_encrypted_data, 0, 133 &ctx->sg_encrypted_num_elem, 134 &ctx->sg_encrypted_size, 0); 135 136 return rc; 137 } 138 139 static int alloc_plaintext_sg(struct sock *sk, int len) 140 { 141 struct tls_context *tls_ctx = tls_get_ctx(sk); 142 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 143 int rc = 0; 144 145 rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0, 146 &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size, 147 tls_ctx->pending_open_record_frags); 148 149 return rc; 150 } 151 152 static void free_sg(struct sock *sk, struct scatterlist *sg, 153 int *sg_num_elem, unsigned int *sg_size) 154 { 155 int i, n = *sg_num_elem; 156 157 for (i = 0; i < n; ++i) { 158 sk_mem_uncharge(sk, sg[i].length); 159 put_page(sg_page(&sg[i])); 160 } 161 *sg_num_elem = 0; 162 *sg_size = 0; 163 } 164 165 static void tls_free_both_sg(struct sock *sk) 166 { 167 struct tls_context *tls_ctx = tls_get_ctx(sk); 168 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 169 170 free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem, 171 &ctx->sg_encrypted_size); 172 173 free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, 174 &ctx->sg_plaintext_size); 175 } 176 177 static int tls_do_encryption(struct tls_context *tls_ctx, 178 struct tls_sw_context_tx *ctx, 179 struct aead_request *aead_req, 180 size_t data_len) 181 { 182 int rc; 183 184 ctx->sg_encrypted_data[0].offset += tls_ctx->tx.prepend_size; 185 ctx->sg_encrypted_data[0].length -= tls_ctx->tx.prepend_size; 186 187 aead_request_set_tfm(aead_req, ctx->aead_send); 188 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); 189 aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out, 190 data_len, tls_ctx->tx.iv); 191 192 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 193 crypto_req_done, &ctx->async_wait); 194 195 rc = crypto_wait_req(crypto_aead_encrypt(aead_req), &ctx->async_wait); 196 197 ctx->sg_encrypted_data[0].offset -= tls_ctx->tx.prepend_size; 198 ctx->sg_encrypted_data[0].length += tls_ctx->tx.prepend_size; 199 200 return rc; 201 } 202 203 static int tls_push_record(struct sock *sk, int flags, 204 unsigned char record_type) 205 { 206 struct tls_context *tls_ctx = tls_get_ctx(sk); 207 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 208 struct aead_request *req; 209 int rc; 210 211 req = aead_request_alloc(ctx->aead_send, sk->sk_allocation); 212 if (!req) 213 return -ENOMEM; 214 215 sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1); 216 sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1); 217 218 tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size, 219 tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size, 220 record_type); 221 222 tls_fill_prepend(tls_ctx, 223 page_address(sg_page(&ctx->sg_encrypted_data[0])) + 224 ctx->sg_encrypted_data[0].offset, 225 ctx->sg_plaintext_size, record_type); 226 227 tls_ctx->pending_open_record_frags = 0; 228 set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags); 229 230 rc = tls_do_encryption(tls_ctx, ctx, req, ctx->sg_plaintext_size); 231 if (rc < 0) { 232 /* If we are called from write_space and 233 * we fail, we need to set this SOCK_NOSPACE 234 * to trigger another write_space in the future. 235 */ 236 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 237 goto out_req; 238 } 239 240 free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, 241 &ctx->sg_plaintext_size); 242 243 ctx->sg_encrypted_num_elem = 0; 244 ctx->sg_encrypted_size = 0; 245 246 /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */ 247 rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags); 248 if (rc < 0 && rc != -EAGAIN) 249 tls_err_abort(sk, EBADMSG); 250 251 tls_advance_record_sn(sk, &tls_ctx->tx); 252 out_req: 253 aead_request_free(req); 254 return rc; 255 } 256 257 static int tls_sw_push_pending_record(struct sock *sk, int flags) 258 { 259 return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA); 260 } 261 262 static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from, 263 int length, int *pages_used, 264 unsigned int *size_used, 265 struct scatterlist *to, int to_max_pages, 266 bool charge, bool revert) 267 { 268 struct page *pages[MAX_SKB_FRAGS]; 269 270 size_t offset; 271 ssize_t copied, use; 272 int i = 0; 273 unsigned int size = *size_used; 274 int num_elem = *pages_used; 275 int rc = 0; 276 int maxpages; 277 278 while (length > 0) { 279 i = 0; 280 maxpages = to_max_pages - num_elem; 281 if (maxpages == 0) { 282 rc = -EFAULT; 283 goto out; 284 } 285 copied = iov_iter_get_pages(from, pages, 286 length, 287 maxpages, &offset); 288 if (copied <= 0) { 289 rc = -EFAULT; 290 goto out; 291 } 292 293 iov_iter_advance(from, copied); 294 295 length -= copied; 296 size += copied; 297 while (copied) { 298 use = min_t(int, copied, PAGE_SIZE - offset); 299 300 sg_set_page(&to[num_elem], 301 pages[i], use, offset); 302 sg_unmark_end(&to[num_elem]); 303 if (charge) 304 sk_mem_charge(sk, use); 305 306 offset = 0; 307 copied -= use; 308 309 ++i; 310 ++num_elem; 311 } 312 } 313 314 out: 315 *size_used = size; 316 *pages_used = num_elem; 317 if (revert) 318 iov_iter_revert(from, size); 319 320 return rc; 321 } 322 323 static int memcopy_from_iter(struct sock *sk, struct iov_iter *from, 324 int bytes) 325 { 326 struct tls_context *tls_ctx = tls_get_ctx(sk); 327 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 328 struct scatterlist *sg = ctx->sg_plaintext_data; 329 int copy, i, rc = 0; 330 331 for (i = tls_ctx->pending_open_record_frags; 332 i < ctx->sg_plaintext_num_elem; ++i) { 333 copy = sg[i].length; 334 if (copy_from_iter( 335 page_address(sg_page(&sg[i])) + sg[i].offset, 336 copy, from) != copy) { 337 rc = -EFAULT; 338 goto out; 339 } 340 bytes -= copy; 341 342 ++tls_ctx->pending_open_record_frags; 343 344 if (!bytes) 345 break; 346 } 347 348 out: 349 return rc; 350 } 351 352 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 353 { 354 struct tls_context *tls_ctx = tls_get_ctx(sk); 355 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 356 int ret = 0; 357 int required_size; 358 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 359 bool eor = !(msg->msg_flags & MSG_MORE); 360 size_t try_to_copy, copied = 0; 361 unsigned char record_type = TLS_RECORD_TYPE_DATA; 362 int record_room; 363 bool full_record; 364 int orig_size; 365 366 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 367 return -ENOTSUPP; 368 369 lock_sock(sk); 370 371 if (tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo)) 372 goto send_end; 373 374 if (unlikely(msg->msg_controllen)) { 375 ret = tls_proccess_cmsg(sk, msg, &record_type); 376 if (ret) 377 goto send_end; 378 } 379 380 while (msg_data_left(msg)) { 381 if (sk->sk_err) { 382 ret = -sk->sk_err; 383 goto send_end; 384 } 385 386 orig_size = ctx->sg_plaintext_size; 387 full_record = false; 388 try_to_copy = msg_data_left(msg); 389 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size; 390 if (try_to_copy >= record_room) { 391 try_to_copy = record_room; 392 full_record = true; 393 } 394 395 required_size = ctx->sg_plaintext_size + try_to_copy + 396 tls_ctx->tx.overhead_size; 397 398 if (!sk_stream_memory_free(sk)) 399 goto wait_for_sndbuf; 400 alloc_encrypted: 401 ret = alloc_encrypted_sg(sk, required_size); 402 if (ret) { 403 if (ret != -ENOSPC) 404 goto wait_for_memory; 405 406 /* Adjust try_to_copy according to the amount that was 407 * actually allocated. The difference is due 408 * to max sg elements limit 409 */ 410 try_to_copy -= required_size - ctx->sg_encrypted_size; 411 full_record = true; 412 } 413 414 if (full_record || eor) { 415 ret = zerocopy_from_iter(sk, &msg->msg_iter, 416 try_to_copy, &ctx->sg_plaintext_num_elem, 417 &ctx->sg_plaintext_size, 418 ctx->sg_plaintext_data, 419 ARRAY_SIZE(ctx->sg_plaintext_data), 420 true, false); 421 if (ret) 422 goto fallback_to_reg_send; 423 424 copied += try_to_copy; 425 ret = tls_push_record(sk, msg->msg_flags, record_type); 426 if (!ret) 427 continue; 428 if (ret == -EAGAIN) 429 goto send_end; 430 431 copied -= try_to_copy; 432 fallback_to_reg_send: 433 iov_iter_revert(&msg->msg_iter, 434 ctx->sg_plaintext_size - orig_size); 435 trim_sg(sk, ctx->sg_plaintext_data, 436 &ctx->sg_plaintext_num_elem, 437 &ctx->sg_plaintext_size, 438 orig_size); 439 } 440 441 required_size = ctx->sg_plaintext_size + try_to_copy; 442 alloc_plaintext: 443 ret = alloc_plaintext_sg(sk, required_size); 444 if (ret) { 445 if (ret != -ENOSPC) 446 goto wait_for_memory; 447 448 /* Adjust try_to_copy according to the amount that was 449 * actually allocated. The difference is due 450 * to max sg elements limit 451 */ 452 try_to_copy -= required_size - ctx->sg_plaintext_size; 453 full_record = true; 454 455 trim_sg(sk, ctx->sg_encrypted_data, 456 &ctx->sg_encrypted_num_elem, 457 &ctx->sg_encrypted_size, 458 ctx->sg_plaintext_size + 459 tls_ctx->tx.overhead_size); 460 } 461 462 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy); 463 if (ret) 464 goto trim_sgl; 465 466 copied += try_to_copy; 467 if (full_record || eor) { 468 push_record: 469 ret = tls_push_record(sk, msg->msg_flags, record_type); 470 if (ret) { 471 if (ret == -ENOMEM) 472 goto wait_for_memory; 473 474 goto send_end; 475 } 476 } 477 478 continue; 479 480 wait_for_sndbuf: 481 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 482 wait_for_memory: 483 ret = sk_stream_wait_memory(sk, &timeo); 484 if (ret) { 485 trim_sgl: 486 trim_both_sgl(sk, orig_size); 487 goto send_end; 488 } 489 490 if (tls_is_pending_closed_record(tls_ctx)) 491 goto push_record; 492 493 if (ctx->sg_encrypted_size < required_size) 494 goto alloc_encrypted; 495 496 goto alloc_plaintext; 497 } 498 499 send_end: 500 ret = sk_stream_error(sk, msg->msg_flags, ret); 501 502 release_sock(sk); 503 return copied ? copied : ret; 504 } 505 506 int tls_sw_sendpage(struct sock *sk, struct page *page, 507 int offset, size_t size, int flags) 508 { 509 struct tls_context *tls_ctx = tls_get_ctx(sk); 510 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 511 int ret = 0; 512 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 513 bool eor; 514 size_t orig_size = size; 515 unsigned char record_type = TLS_RECORD_TYPE_DATA; 516 struct scatterlist *sg; 517 bool full_record; 518 int record_room; 519 520 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 521 MSG_SENDPAGE_NOTLAST)) 522 return -ENOTSUPP; 523 524 /* No MSG_EOR from splice, only look at MSG_MORE */ 525 eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST)); 526 527 lock_sock(sk); 528 529 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 530 531 if (tls_complete_pending_work(sk, tls_ctx, flags, &timeo)) 532 goto sendpage_end; 533 534 /* Call the sk_stream functions to manage the sndbuf mem. */ 535 while (size > 0) { 536 size_t copy, required_size; 537 538 if (sk->sk_err) { 539 ret = -sk->sk_err; 540 goto sendpage_end; 541 } 542 543 full_record = false; 544 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size; 545 copy = size; 546 if (copy >= record_room) { 547 copy = record_room; 548 full_record = true; 549 } 550 required_size = ctx->sg_plaintext_size + copy + 551 tls_ctx->tx.overhead_size; 552 553 if (!sk_stream_memory_free(sk)) 554 goto wait_for_sndbuf; 555 alloc_payload: 556 ret = alloc_encrypted_sg(sk, required_size); 557 if (ret) { 558 if (ret != -ENOSPC) 559 goto wait_for_memory; 560 561 /* Adjust copy according to the amount that was 562 * actually allocated. The difference is due 563 * to max sg elements limit 564 */ 565 copy -= required_size - ctx->sg_plaintext_size; 566 full_record = true; 567 } 568 569 get_page(page); 570 sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem; 571 sg_set_page(sg, page, copy, offset); 572 sg_unmark_end(sg); 573 574 ctx->sg_plaintext_num_elem++; 575 576 sk_mem_charge(sk, copy); 577 offset += copy; 578 size -= copy; 579 ctx->sg_plaintext_size += copy; 580 tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem; 581 582 if (full_record || eor || 583 ctx->sg_plaintext_num_elem == 584 ARRAY_SIZE(ctx->sg_plaintext_data)) { 585 push_record: 586 ret = tls_push_record(sk, flags, record_type); 587 if (ret) { 588 if (ret == -ENOMEM) 589 goto wait_for_memory; 590 591 goto sendpage_end; 592 } 593 } 594 continue; 595 wait_for_sndbuf: 596 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 597 wait_for_memory: 598 ret = sk_stream_wait_memory(sk, &timeo); 599 if (ret) { 600 trim_both_sgl(sk, ctx->sg_plaintext_size); 601 goto sendpage_end; 602 } 603 604 if (tls_is_pending_closed_record(tls_ctx)) 605 goto push_record; 606 607 goto alloc_payload; 608 } 609 610 sendpage_end: 611 if (orig_size > size) 612 ret = orig_size - size; 613 else 614 ret = sk_stream_error(sk, flags, ret); 615 616 release_sock(sk); 617 return ret; 618 } 619 620 static struct sk_buff *tls_wait_data(struct sock *sk, int flags, 621 long timeo, int *err) 622 { 623 struct tls_context *tls_ctx = tls_get_ctx(sk); 624 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 625 struct sk_buff *skb; 626 DEFINE_WAIT_FUNC(wait, woken_wake_function); 627 628 while (!(skb = ctx->recv_pkt)) { 629 if (sk->sk_err) { 630 *err = sock_error(sk); 631 return NULL; 632 } 633 634 if (sock_flag(sk, SOCK_DONE)) 635 return NULL; 636 637 if ((flags & MSG_DONTWAIT) || !timeo) { 638 *err = -EAGAIN; 639 return NULL; 640 } 641 642 add_wait_queue(sk_sleep(sk), &wait); 643 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 644 sk_wait_event(sk, &timeo, ctx->recv_pkt != skb, &wait); 645 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 646 remove_wait_queue(sk_sleep(sk), &wait); 647 648 /* Handle signals */ 649 if (signal_pending(current)) { 650 *err = sock_intr_errno(timeo); 651 return NULL; 652 } 653 } 654 655 return skb; 656 } 657 658 static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, 659 struct scatterlist *sgout, bool *zc) 660 { 661 struct tls_context *tls_ctx = tls_get_ctx(sk); 662 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 663 struct strp_msg *rxm = strp_msg(skb); 664 int err = 0; 665 666 #ifdef CONFIG_TLS_DEVICE 667 err = tls_device_decrypted(sk, skb); 668 if (err < 0) 669 return err; 670 #endif 671 if (!ctx->decrypted) { 672 err = decrypt_skb(sk, skb, sgout); 673 if (err < 0) 674 return err; 675 } else { 676 *zc = false; 677 } 678 679 rxm->offset += tls_ctx->rx.prepend_size; 680 rxm->full_len -= tls_ctx->rx.overhead_size; 681 tls_advance_record_sn(sk, &tls_ctx->rx); 682 ctx->decrypted = true; 683 ctx->saved_data_ready(sk); 684 685 return err; 686 } 687 688 int decrypt_skb(struct sock *sk, struct sk_buff *skb, 689 struct scatterlist *sgout) 690 { 691 struct tls_context *tls_ctx = tls_get_ctx(sk); 692 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 693 char iv[TLS_CIPHER_AES_GCM_128_SALT_SIZE + MAX_IV_SIZE]; 694 struct scatterlist sgin_arr[MAX_SKB_FRAGS + 2]; 695 struct scatterlist *sgin = &sgin_arr[0]; 696 struct strp_msg *rxm = strp_msg(skb); 697 int ret, nsg = ARRAY_SIZE(sgin_arr); 698 struct sk_buff *unused; 699 700 ret = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE, 701 iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 702 tls_ctx->rx.iv_size); 703 if (ret < 0) 704 return ret; 705 706 memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 707 if (!sgout) { 708 nsg = skb_cow_data(skb, 0, &unused) + 1; 709 sgin = kmalloc_array(nsg, sizeof(*sgin), sk->sk_allocation); 710 sgout = sgin; 711 } 712 713 sg_init_table(sgin, nsg); 714 sg_set_buf(&sgin[0], ctx->rx_aad_ciphertext, TLS_AAD_SPACE_SIZE); 715 716 nsg = skb_to_sgvec(skb, &sgin[1], 717 rxm->offset + tls_ctx->rx.prepend_size, 718 rxm->full_len - tls_ctx->rx.prepend_size); 719 720 tls_make_aad(ctx->rx_aad_ciphertext, 721 rxm->full_len - tls_ctx->rx.overhead_size, 722 tls_ctx->rx.rec_seq, 723 tls_ctx->rx.rec_seq_size, 724 ctx->control); 725 726 ret = tls_do_decryption(sk, sgin, sgout, iv, 727 rxm->full_len - tls_ctx->rx.overhead_size, 728 skb, sk->sk_allocation); 729 730 if (sgin != &sgin_arr[0]) 731 kfree(sgin); 732 733 return ret; 734 } 735 736 static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb, 737 unsigned int len) 738 { 739 struct tls_context *tls_ctx = tls_get_ctx(sk); 740 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 741 struct strp_msg *rxm = strp_msg(skb); 742 743 if (len < rxm->full_len) { 744 rxm->offset += len; 745 rxm->full_len -= len; 746 747 return false; 748 } 749 750 /* Finished with message */ 751 ctx->recv_pkt = NULL; 752 kfree_skb(skb); 753 __strp_unpause(&ctx->strp); 754 755 return true; 756 } 757 758 int tls_sw_recvmsg(struct sock *sk, 759 struct msghdr *msg, 760 size_t len, 761 int nonblock, 762 int flags, 763 int *addr_len) 764 { 765 struct tls_context *tls_ctx = tls_get_ctx(sk); 766 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 767 unsigned char control; 768 struct strp_msg *rxm; 769 struct sk_buff *skb; 770 ssize_t copied = 0; 771 bool cmsg = false; 772 int target, err = 0; 773 long timeo; 774 775 flags |= nonblock; 776 777 if (unlikely(flags & MSG_ERRQUEUE)) 778 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR); 779 780 lock_sock(sk); 781 782 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 783 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 784 do { 785 bool zc = false; 786 int chunk = 0; 787 788 skb = tls_wait_data(sk, flags, timeo, &err); 789 if (!skb) 790 goto recv_end; 791 792 rxm = strp_msg(skb); 793 if (!cmsg) { 794 int cerr; 795 796 cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, 797 sizeof(ctx->control), &ctx->control); 798 cmsg = true; 799 control = ctx->control; 800 if (ctx->control != TLS_RECORD_TYPE_DATA) { 801 if (cerr || msg->msg_flags & MSG_CTRUNC) { 802 err = -EIO; 803 goto recv_end; 804 } 805 } 806 } else if (control != ctx->control) { 807 goto recv_end; 808 } 809 810 if (!ctx->decrypted) { 811 int page_count; 812 int to_copy; 813 814 page_count = iov_iter_npages(&msg->msg_iter, 815 MAX_SKB_FRAGS); 816 to_copy = rxm->full_len - tls_ctx->rx.overhead_size; 817 if (to_copy <= len && page_count < MAX_SKB_FRAGS && 818 likely(!(flags & MSG_PEEK))) { 819 struct scatterlist sgin[MAX_SKB_FRAGS + 1]; 820 int pages = 0; 821 822 zc = true; 823 sg_init_table(sgin, MAX_SKB_FRAGS + 1); 824 sg_set_buf(&sgin[0], ctx->rx_aad_plaintext, 825 TLS_AAD_SPACE_SIZE); 826 827 err = zerocopy_from_iter(sk, &msg->msg_iter, 828 to_copy, &pages, 829 &chunk, &sgin[1], 830 MAX_SKB_FRAGS, false, true); 831 if (err < 0) 832 goto fallback_to_reg_recv; 833 834 err = decrypt_skb_update(sk, skb, sgin, &zc); 835 for (; pages > 0; pages--) 836 put_page(sg_page(&sgin[pages])); 837 if (err < 0) { 838 tls_err_abort(sk, EBADMSG); 839 goto recv_end; 840 } 841 } else { 842 fallback_to_reg_recv: 843 err = decrypt_skb_update(sk, skb, NULL, &zc); 844 if (err < 0) { 845 tls_err_abort(sk, EBADMSG); 846 goto recv_end; 847 } 848 } 849 ctx->decrypted = true; 850 } 851 852 if (!zc) { 853 chunk = min_t(unsigned int, rxm->full_len, len); 854 err = skb_copy_datagram_msg(skb, rxm->offset, msg, 855 chunk); 856 if (err < 0) 857 goto recv_end; 858 } 859 860 copied += chunk; 861 len -= chunk; 862 if (likely(!(flags & MSG_PEEK))) { 863 u8 control = ctx->control; 864 865 if (tls_sw_advance_skb(sk, skb, chunk)) { 866 /* Return full control message to 867 * userspace before trying to parse 868 * another message type 869 */ 870 msg->msg_flags |= MSG_EOR; 871 if (control != TLS_RECORD_TYPE_DATA) 872 goto recv_end; 873 } 874 } 875 /* If we have a new message from strparser, continue now. */ 876 if (copied >= target && !ctx->recv_pkt) 877 break; 878 } while (len); 879 880 recv_end: 881 release_sock(sk); 882 return copied ? : err; 883 } 884 885 ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, 886 struct pipe_inode_info *pipe, 887 size_t len, unsigned int flags) 888 { 889 struct tls_context *tls_ctx = tls_get_ctx(sock->sk); 890 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 891 struct strp_msg *rxm = NULL; 892 struct sock *sk = sock->sk; 893 struct sk_buff *skb; 894 ssize_t copied = 0; 895 int err = 0; 896 long timeo; 897 int chunk; 898 bool zc; 899 900 lock_sock(sk); 901 902 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 903 904 skb = tls_wait_data(sk, flags, timeo, &err); 905 if (!skb) 906 goto splice_read_end; 907 908 /* splice does not support reading control messages */ 909 if (ctx->control != TLS_RECORD_TYPE_DATA) { 910 err = -ENOTSUPP; 911 goto splice_read_end; 912 } 913 914 if (!ctx->decrypted) { 915 err = decrypt_skb_update(sk, skb, NULL, &zc); 916 917 if (err < 0) { 918 tls_err_abort(sk, EBADMSG); 919 goto splice_read_end; 920 } 921 ctx->decrypted = true; 922 } 923 rxm = strp_msg(skb); 924 925 chunk = min_t(unsigned int, rxm->full_len, len); 926 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags); 927 if (copied < 0) 928 goto splice_read_end; 929 930 if (likely(!(flags & MSG_PEEK))) 931 tls_sw_advance_skb(sk, skb, copied); 932 933 splice_read_end: 934 release_sock(sk); 935 return copied ? : err; 936 } 937 938 unsigned int tls_sw_poll(struct file *file, struct socket *sock, 939 struct poll_table_struct *wait) 940 { 941 unsigned int ret; 942 struct sock *sk = sock->sk; 943 struct tls_context *tls_ctx = tls_get_ctx(sk); 944 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 945 946 /* Grab POLLOUT and POLLHUP from the underlying socket */ 947 ret = ctx->sk_poll(file, sock, wait); 948 949 /* Clear POLLIN bits, and set based on recv_pkt */ 950 ret &= ~(POLLIN | POLLRDNORM); 951 if (ctx->recv_pkt) 952 ret |= POLLIN | POLLRDNORM; 953 954 return ret; 955 } 956 957 static int tls_read_size(struct strparser *strp, struct sk_buff *skb) 958 { 959 struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 960 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 961 char header[TLS_HEADER_SIZE + MAX_IV_SIZE]; 962 struct strp_msg *rxm = strp_msg(skb); 963 size_t cipher_overhead; 964 size_t data_len = 0; 965 int ret; 966 967 /* Verify that we have a full TLS header, or wait for more data */ 968 if (rxm->offset + tls_ctx->rx.prepend_size > skb->len) 969 return 0; 970 971 /* Sanity-check size of on-stack buffer. */ 972 if (WARN_ON(tls_ctx->rx.prepend_size > sizeof(header))) { 973 ret = -EINVAL; 974 goto read_failure; 975 } 976 977 /* Linearize header to local buffer */ 978 ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size); 979 980 if (ret < 0) 981 goto read_failure; 982 983 ctx->control = header[0]; 984 985 data_len = ((header[4] & 0xFF) | (header[3] << 8)); 986 987 cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size; 988 989 if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) { 990 ret = -EMSGSIZE; 991 goto read_failure; 992 } 993 if (data_len < cipher_overhead) { 994 ret = -EBADMSG; 995 goto read_failure; 996 } 997 998 if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.version) || 999 header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.version)) { 1000 ret = -EINVAL; 1001 goto read_failure; 1002 } 1003 1004 #ifdef CONFIG_TLS_DEVICE 1005 handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset, 1006 *(u64*)tls_ctx->rx.rec_seq); 1007 #endif 1008 return data_len + TLS_HEADER_SIZE; 1009 1010 read_failure: 1011 tls_err_abort(strp->sk, ret); 1012 1013 return ret; 1014 } 1015 1016 static void tls_queue(struct strparser *strp, struct sk_buff *skb) 1017 { 1018 struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 1019 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1020 1021 ctx->decrypted = false; 1022 1023 ctx->recv_pkt = skb; 1024 strp_pause(strp); 1025 1026 strp->sk->sk_state_change(strp->sk); 1027 } 1028 1029 static void tls_data_ready(struct sock *sk) 1030 { 1031 struct tls_context *tls_ctx = tls_get_ctx(sk); 1032 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1033 1034 strp_data_ready(&ctx->strp); 1035 } 1036 1037 void tls_sw_free_resources_tx(struct sock *sk) 1038 { 1039 struct tls_context *tls_ctx = tls_get_ctx(sk); 1040 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1041 1042 if (ctx->aead_send) 1043 crypto_free_aead(ctx->aead_send); 1044 tls_free_both_sg(sk); 1045 1046 kfree(ctx); 1047 } 1048 1049 void tls_sw_release_resources_rx(struct sock *sk) 1050 { 1051 struct tls_context *tls_ctx = tls_get_ctx(sk); 1052 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1053 1054 if (ctx->aead_recv) { 1055 if (ctx->recv_pkt) { 1056 kfree_skb(ctx->recv_pkt); 1057 ctx->recv_pkt = NULL; 1058 } 1059 crypto_free_aead(ctx->aead_recv); 1060 strp_stop(&ctx->strp); 1061 write_lock_bh(&sk->sk_callback_lock); 1062 sk->sk_data_ready = ctx->saved_data_ready; 1063 write_unlock_bh(&sk->sk_callback_lock); 1064 release_sock(sk); 1065 strp_done(&ctx->strp); 1066 lock_sock(sk); 1067 } 1068 } 1069 1070 void tls_sw_free_resources_rx(struct sock *sk) 1071 { 1072 struct tls_context *tls_ctx = tls_get_ctx(sk); 1073 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1074 1075 tls_sw_release_resources_rx(sk); 1076 1077 kfree(ctx); 1078 } 1079 1080 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) 1081 { 1082 char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE]; 1083 struct tls_crypto_info *crypto_info; 1084 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; 1085 struct tls_sw_context_tx *sw_ctx_tx = NULL; 1086 struct tls_sw_context_rx *sw_ctx_rx = NULL; 1087 struct cipher_context *cctx; 1088 struct crypto_aead **aead; 1089 struct strp_callbacks cb; 1090 u16 nonce_size, tag_size, iv_size, rec_seq_size; 1091 char *iv, *rec_seq; 1092 int rc = 0; 1093 1094 if (!ctx) { 1095 rc = -EINVAL; 1096 goto out; 1097 } 1098 1099 if (tx) { 1100 if (!ctx->priv_ctx_tx) { 1101 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL); 1102 if (!sw_ctx_tx) { 1103 rc = -ENOMEM; 1104 goto out; 1105 } 1106 ctx->priv_ctx_tx = sw_ctx_tx; 1107 } else { 1108 sw_ctx_tx = 1109 (struct tls_sw_context_tx *)ctx->priv_ctx_tx; 1110 } 1111 } else { 1112 if (!ctx->priv_ctx_rx) { 1113 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL); 1114 if (!sw_ctx_rx) { 1115 rc = -ENOMEM; 1116 goto out; 1117 } 1118 ctx->priv_ctx_rx = sw_ctx_rx; 1119 } else { 1120 sw_ctx_rx = 1121 (struct tls_sw_context_rx *)ctx->priv_ctx_rx; 1122 } 1123 } 1124 1125 if (tx) { 1126 crypto_init_wait(&sw_ctx_tx->async_wait); 1127 crypto_info = &ctx->crypto_send; 1128 cctx = &ctx->tx; 1129 aead = &sw_ctx_tx->aead_send; 1130 } else { 1131 crypto_init_wait(&sw_ctx_rx->async_wait); 1132 crypto_info = &ctx->crypto_recv; 1133 cctx = &ctx->rx; 1134 aead = &sw_ctx_rx->aead_recv; 1135 } 1136 1137 switch (crypto_info->cipher_type) { 1138 case TLS_CIPHER_AES_GCM_128: { 1139 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 1140 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; 1141 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 1142 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; 1143 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; 1144 rec_seq = 1145 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; 1146 gcm_128_info = 1147 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 1148 break; 1149 } 1150 default: 1151 rc = -EINVAL; 1152 goto free_priv; 1153 } 1154 1155 /* Sanity-check the IV size for stack allocations. */ 1156 if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE) { 1157 rc = -EINVAL; 1158 goto free_priv; 1159 } 1160 1161 cctx->prepend_size = TLS_HEADER_SIZE + nonce_size; 1162 cctx->tag_size = tag_size; 1163 cctx->overhead_size = cctx->prepend_size + cctx->tag_size; 1164 cctx->iv_size = iv_size; 1165 cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 1166 GFP_KERNEL); 1167 if (!cctx->iv) { 1168 rc = -ENOMEM; 1169 goto free_priv; 1170 } 1171 memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 1172 memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size); 1173 cctx->rec_seq_size = rec_seq_size; 1174 cctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL); 1175 if (!cctx->rec_seq) { 1176 rc = -ENOMEM; 1177 goto free_iv; 1178 } 1179 memcpy(cctx->rec_seq, rec_seq, rec_seq_size); 1180 1181 if (sw_ctx_tx) { 1182 sg_init_table(sw_ctx_tx->sg_encrypted_data, 1183 ARRAY_SIZE(sw_ctx_tx->sg_encrypted_data)); 1184 sg_init_table(sw_ctx_tx->sg_plaintext_data, 1185 ARRAY_SIZE(sw_ctx_tx->sg_plaintext_data)); 1186 1187 sg_init_table(sw_ctx_tx->sg_aead_in, 2); 1188 sg_set_buf(&sw_ctx_tx->sg_aead_in[0], sw_ctx_tx->aad_space, 1189 sizeof(sw_ctx_tx->aad_space)); 1190 sg_unmark_end(&sw_ctx_tx->sg_aead_in[1]); 1191 sg_chain(sw_ctx_tx->sg_aead_in, 2, 1192 sw_ctx_tx->sg_plaintext_data); 1193 sg_init_table(sw_ctx_tx->sg_aead_out, 2); 1194 sg_set_buf(&sw_ctx_tx->sg_aead_out[0], sw_ctx_tx->aad_space, 1195 sizeof(sw_ctx_tx->aad_space)); 1196 sg_unmark_end(&sw_ctx_tx->sg_aead_out[1]); 1197 sg_chain(sw_ctx_tx->sg_aead_out, 2, 1198 sw_ctx_tx->sg_encrypted_data); 1199 } 1200 1201 if (!*aead) { 1202 *aead = crypto_alloc_aead("gcm(aes)", 0, 0); 1203 if (IS_ERR(*aead)) { 1204 rc = PTR_ERR(*aead); 1205 *aead = NULL; 1206 goto free_rec_seq; 1207 } 1208 } 1209 1210 ctx->push_pending_record = tls_sw_push_pending_record; 1211 1212 memcpy(keyval, gcm_128_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE); 1213 1214 rc = crypto_aead_setkey(*aead, keyval, 1215 TLS_CIPHER_AES_GCM_128_KEY_SIZE); 1216 if (rc) 1217 goto free_aead; 1218 1219 rc = crypto_aead_setauthsize(*aead, cctx->tag_size); 1220 if (rc) 1221 goto free_aead; 1222 1223 if (sw_ctx_rx) { 1224 /* Set up strparser */ 1225 memset(&cb, 0, sizeof(cb)); 1226 cb.rcv_msg = tls_queue; 1227 cb.parse_msg = tls_read_size; 1228 1229 strp_init(&sw_ctx_rx->strp, sk, &cb); 1230 1231 write_lock_bh(&sk->sk_callback_lock); 1232 sw_ctx_rx->saved_data_ready = sk->sk_data_ready; 1233 sk->sk_data_ready = tls_data_ready; 1234 write_unlock_bh(&sk->sk_callback_lock); 1235 1236 sw_ctx_rx->sk_poll = sk->sk_socket->ops->poll; 1237 1238 strp_check_rcv(&sw_ctx_rx->strp); 1239 } 1240 1241 goto out; 1242 1243 free_aead: 1244 crypto_free_aead(*aead); 1245 *aead = NULL; 1246 free_rec_seq: 1247 kfree(cctx->rec_seq); 1248 cctx->rec_seq = NULL; 1249 free_iv: 1250 kfree(cctx->iv); 1251 cctx->iv = NULL; 1252 free_priv: 1253 if (tx) { 1254 kfree(ctx->priv_ctx_tx); 1255 ctx->priv_ctx_tx = NULL; 1256 } else { 1257 kfree(ctx->priv_ctx_rx); 1258 ctx->priv_ctx_rx = NULL; 1259 } 1260 out: 1261 return rc; 1262 } 1263