13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved. 53c4d7559SDave Watson * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved. 63c4d7559SDave Watson * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved. 73c4d7559SDave Watson * 83c4d7559SDave Watson * This software is available to you under a choice of one of two 93c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 103c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 113c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 123c4d7559SDave Watson * OpenIB.org BSD license below: 133c4d7559SDave Watson * 143c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 153c4d7559SDave Watson * without modification, are permitted provided that the following 163c4d7559SDave Watson * conditions are met: 173c4d7559SDave Watson * 183c4d7559SDave Watson * - Redistributions of source code must retain the above 193c4d7559SDave Watson * copyright notice, this list of conditions and the following 203c4d7559SDave Watson * disclaimer. 213c4d7559SDave Watson * 223c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 233c4d7559SDave Watson * copyright notice, this list of conditions and the following 243c4d7559SDave Watson * disclaimer in the documentation and/or other materials 253c4d7559SDave Watson * provided with the distribution. 263c4d7559SDave Watson * 273c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 283c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 293c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 303c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 313c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 323c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 333c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 343c4d7559SDave Watson * SOFTWARE. 353c4d7559SDave Watson */ 363c4d7559SDave Watson 37c46234ebSDave Watson #include <linux/sched/signal.h> 383c4d7559SDave Watson #include <linux/module.h> 393c4d7559SDave Watson #include <crypto/aead.h> 403c4d7559SDave Watson 41c46234ebSDave Watson #include <net/strparser.h> 423c4d7559SDave Watson #include <net/tls.h> 433c4d7559SDave Watson 44b16520f7SKees Cook #define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE 45b16520f7SKees Cook 46c46234ebSDave Watson static int tls_do_decryption(struct sock *sk, 47c46234ebSDave Watson struct scatterlist *sgin, 48c46234ebSDave Watson struct scatterlist *sgout, 49c46234ebSDave Watson char *iv_recv, 50c46234ebSDave Watson size_t data_len, 51c46234ebSDave Watson struct sk_buff *skb, 52c46234ebSDave Watson gfp_t flags) 53c46234ebSDave Watson { 54c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 55f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 56c46234ebSDave Watson struct aead_request *aead_req; 57c46234ebSDave Watson 58c46234ebSDave Watson int ret; 59c46234ebSDave Watson 60d2bdd268SVakul Garg aead_req = aead_request_alloc(ctx->aead_recv, flags); 61c46234ebSDave Watson if (!aead_req) 62c46234ebSDave Watson return -ENOMEM; 63c46234ebSDave Watson 64c46234ebSDave Watson aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); 65c46234ebSDave Watson aead_request_set_crypt(aead_req, sgin, sgout, 66c46234ebSDave Watson data_len + tls_ctx->rx.tag_size, 67c46234ebSDave Watson (u8 *)iv_recv); 68c46234ebSDave Watson aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 69c46234ebSDave Watson crypto_req_done, &ctx->async_wait); 70c46234ebSDave Watson 71c46234ebSDave Watson ret = crypto_wait_req(crypto_aead_decrypt(aead_req), &ctx->async_wait); 72c46234ebSDave Watson 73d2bdd268SVakul Garg aead_request_free(aead_req); 74c46234ebSDave Watson return ret; 75c46234ebSDave Watson } 76c46234ebSDave Watson 773c4d7559SDave Watson static void trim_sg(struct sock *sk, struct scatterlist *sg, 783c4d7559SDave Watson int *sg_num_elem, unsigned int *sg_size, int target_size) 793c4d7559SDave Watson { 803c4d7559SDave Watson int i = *sg_num_elem - 1; 813c4d7559SDave Watson int trim = *sg_size - target_size; 823c4d7559SDave Watson 833c4d7559SDave Watson if (trim <= 0) { 843c4d7559SDave Watson WARN_ON(trim < 0); 853c4d7559SDave Watson return; 863c4d7559SDave Watson } 873c4d7559SDave Watson 883c4d7559SDave Watson *sg_size = target_size; 893c4d7559SDave Watson while (trim >= sg[i].length) { 903c4d7559SDave Watson trim -= sg[i].length; 913c4d7559SDave Watson sk_mem_uncharge(sk, sg[i].length); 923c4d7559SDave Watson put_page(sg_page(&sg[i])); 933c4d7559SDave Watson i--; 943c4d7559SDave Watson 953c4d7559SDave Watson if (i < 0) 963c4d7559SDave Watson goto out; 973c4d7559SDave Watson } 983c4d7559SDave Watson 993c4d7559SDave Watson sg[i].length -= trim; 1003c4d7559SDave Watson sk_mem_uncharge(sk, trim); 1013c4d7559SDave Watson 1023c4d7559SDave Watson out: 1033c4d7559SDave Watson *sg_num_elem = i + 1; 1043c4d7559SDave Watson } 1053c4d7559SDave Watson 1063c4d7559SDave Watson static void trim_both_sgl(struct sock *sk, int target_size) 1073c4d7559SDave Watson { 1083c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 109f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1103c4d7559SDave Watson 1113c4d7559SDave Watson trim_sg(sk, ctx->sg_plaintext_data, 1123c4d7559SDave Watson &ctx->sg_plaintext_num_elem, 1133c4d7559SDave Watson &ctx->sg_plaintext_size, 1143c4d7559SDave Watson target_size); 1153c4d7559SDave Watson 1163c4d7559SDave Watson if (target_size > 0) 117dbe42559SDave Watson target_size += tls_ctx->tx.overhead_size; 1183c4d7559SDave Watson 1193c4d7559SDave Watson trim_sg(sk, ctx->sg_encrypted_data, 1203c4d7559SDave Watson &ctx->sg_encrypted_num_elem, 1213c4d7559SDave Watson &ctx->sg_encrypted_size, 1223c4d7559SDave Watson target_size); 1233c4d7559SDave Watson } 1243c4d7559SDave Watson 1253c4d7559SDave Watson static int alloc_encrypted_sg(struct sock *sk, int len) 1263c4d7559SDave Watson { 1273c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 128f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1293c4d7559SDave Watson int rc = 0; 1303c4d7559SDave Watson 1312c3682f0SJohn Fastabend rc = sk_alloc_sg(sk, len, 1328c05dbf0SJohn Fastabend ctx->sg_encrypted_data, 0, 1332c3682f0SJohn Fastabend &ctx->sg_encrypted_num_elem, 1342c3682f0SJohn Fastabend &ctx->sg_encrypted_size, 0); 1353c4d7559SDave Watson 1363c4d7559SDave Watson return rc; 1373c4d7559SDave Watson } 1383c4d7559SDave Watson 1393c4d7559SDave Watson static int alloc_plaintext_sg(struct sock *sk, int len) 1403c4d7559SDave Watson { 1413c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 142f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1433c4d7559SDave Watson int rc = 0; 1443c4d7559SDave Watson 1458c05dbf0SJohn Fastabend rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0, 1463c4d7559SDave Watson &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size, 1473c4d7559SDave Watson tls_ctx->pending_open_record_frags); 1483c4d7559SDave Watson 1493c4d7559SDave Watson return rc; 1503c4d7559SDave Watson } 1513c4d7559SDave Watson 1523c4d7559SDave Watson static void free_sg(struct sock *sk, struct scatterlist *sg, 1533c4d7559SDave Watson int *sg_num_elem, unsigned int *sg_size) 1543c4d7559SDave Watson { 1553c4d7559SDave Watson int i, n = *sg_num_elem; 1563c4d7559SDave Watson 1573c4d7559SDave Watson for (i = 0; i < n; ++i) { 1583c4d7559SDave Watson sk_mem_uncharge(sk, sg[i].length); 1593c4d7559SDave Watson put_page(sg_page(&sg[i])); 1603c4d7559SDave Watson } 1613c4d7559SDave Watson *sg_num_elem = 0; 1623c4d7559SDave Watson *sg_size = 0; 1633c4d7559SDave Watson } 1643c4d7559SDave Watson 1653c4d7559SDave Watson static void tls_free_both_sg(struct sock *sk) 1663c4d7559SDave Watson { 1673c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 168f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1693c4d7559SDave Watson 1703c4d7559SDave Watson free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem, 1713c4d7559SDave Watson &ctx->sg_encrypted_size); 1723c4d7559SDave Watson 1733c4d7559SDave Watson free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, 1743c4d7559SDave Watson &ctx->sg_plaintext_size); 1753c4d7559SDave Watson } 1763c4d7559SDave Watson 1773c4d7559SDave Watson static int tls_do_encryption(struct tls_context *tls_ctx, 178a447da7dSDaniel Borkmann struct tls_sw_context_tx *ctx, 179a447da7dSDaniel Borkmann struct aead_request *aead_req, 180a447da7dSDaniel Borkmann size_t data_len) 1813c4d7559SDave Watson { 1823c4d7559SDave Watson int rc; 1833c4d7559SDave Watson 184dbe42559SDave Watson ctx->sg_encrypted_data[0].offset += tls_ctx->tx.prepend_size; 185dbe42559SDave Watson ctx->sg_encrypted_data[0].length -= tls_ctx->tx.prepend_size; 1863c4d7559SDave Watson 1873c4d7559SDave Watson aead_request_set_tfm(aead_req, ctx->aead_send); 1883c4d7559SDave Watson aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); 1893c4d7559SDave Watson aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out, 190dbe42559SDave Watson data_len, tls_ctx->tx.iv); 191a54667f6SVakul Garg 192a54667f6SVakul Garg aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 193a54667f6SVakul Garg crypto_req_done, &ctx->async_wait); 194a54667f6SVakul Garg 195a54667f6SVakul Garg rc = crypto_wait_req(crypto_aead_encrypt(aead_req), &ctx->async_wait); 1963c4d7559SDave Watson 197dbe42559SDave Watson ctx->sg_encrypted_data[0].offset -= tls_ctx->tx.prepend_size; 198dbe42559SDave Watson ctx->sg_encrypted_data[0].length += tls_ctx->tx.prepend_size; 1993c4d7559SDave Watson 2003c4d7559SDave Watson return rc; 2013c4d7559SDave Watson } 2023c4d7559SDave Watson 2033c4d7559SDave Watson static int tls_push_record(struct sock *sk, int flags, 2043c4d7559SDave Watson unsigned char record_type) 2053c4d7559SDave Watson { 2063c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 207f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 208a447da7dSDaniel Borkmann struct aead_request *req; 2093c4d7559SDave Watson int rc; 2103c4d7559SDave Watson 211d2bdd268SVakul Garg req = aead_request_alloc(ctx->aead_send, sk->sk_allocation); 212a447da7dSDaniel Borkmann if (!req) 213a447da7dSDaniel Borkmann return -ENOMEM; 214a447da7dSDaniel Borkmann 2153c4d7559SDave Watson sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1); 2163c4d7559SDave Watson sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1); 2173c4d7559SDave Watson 218213ef6e7SIlya Lesokhin tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size, 219dbe42559SDave Watson tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size, 2203c4d7559SDave Watson record_type); 2213c4d7559SDave Watson 2223c4d7559SDave Watson tls_fill_prepend(tls_ctx, 2233c4d7559SDave Watson page_address(sg_page(&ctx->sg_encrypted_data[0])) + 2243c4d7559SDave Watson ctx->sg_encrypted_data[0].offset, 2253c4d7559SDave Watson ctx->sg_plaintext_size, record_type); 2263c4d7559SDave Watson 2273c4d7559SDave Watson tls_ctx->pending_open_record_frags = 0; 2283c4d7559SDave Watson set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags); 2293c4d7559SDave Watson 230a447da7dSDaniel Borkmann rc = tls_do_encryption(tls_ctx, ctx, req, ctx->sg_plaintext_size); 2313c4d7559SDave Watson if (rc < 0) { 2323c4d7559SDave Watson /* If we are called from write_space and 2333c4d7559SDave Watson * we fail, we need to set this SOCK_NOSPACE 2343c4d7559SDave Watson * to trigger another write_space in the future. 2353c4d7559SDave Watson */ 2363c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 237a447da7dSDaniel Borkmann goto out_req; 2383c4d7559SDave Watson } 2393c4d7559SDave Watson 2403c4d7559SDave Watson free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, 2413c4d7559SDave Watson &ctx->sg_plaintext_size); 2423c4d7559SDave Watson 2433c4d7559SDave Watson ctx->sg_encrypted_num_elem = 0; 2443c4d7559SDave Watson ctx->sg_encrypted_size = 0; 2453c4d7559SDave Watson 2463c4d7559SDave Watson /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */ 2473c4d7559SDave Watson rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags); 2483c4d7559SDave Watson if (rc < 0 && rc != -EAGAIN) 249f4a8e43fSDave Watson tls_err_abort(sk, EBADMSG); 2503c4d7559SDave Watson 251dbe42559SDave Watson tls_advance_record_sn(sk, &tls_ctx->tx); 252a447da7dSDaniel Borkmann out_req: 253d2bdd268SVakul Garg aead_request_free(req); 2543c4d7559SDave Watson return rc; 2553c4d7559SDave Watson } 2563c4d7559SDave Watson 2573c4d7559SDave Watson static int tls_sw_push_pending_record(struct sock *sk, int flags) 2583c4d7559SDave Watson { 2593c4d7559SDave Watson return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA); 2603c4d7559SDave Watson } 2613c4d7559SDave Watson 2623c4d7559SDave Watson static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from, 26369ca9293SDave Watson int length, int *pages_used, 26469ca9293SDave Watson unsigned int *size_used, 26569ca9293SDave Watson struct scatterlist *to, int to_max_pages, 2662da19ed3SDoron Roberts-Kedes bool charge) 2673c4d7559SDave Watson { 2683c4d7559SDave Watson struct page *pages[MAX_SKB_FRAGS]; 2693c4d7559SDave Watson 2703c4d7559SDave Watson size_t offset; 2713c4d7559SDave Watson ssize_t copied, use; 2723c4d7559SDave Watson int i = 0; 27369ca9293SDave Watson unsigned int size = *size_used; 27469ca9293SDave Watson int num_elem = *pages_used; 2753c4d7559SDave Watson int rc = 0; 2763c4d7559SDave Watson int maxpages; 2773c4d7559SDave Watson 2783c4d7559SDave Watson while (length > 0) { 2793c4d7559SDave Watson i = 0; 28069ca9293SDave Watson maxpages = to_max_pages - num_elem; 2813c4d7559SDave Watson if (maxpages == 0) { 2823c4d7559SDave Watson rc = -EFAULT; 2833c4d7559SDave Watson goto out; 2843c4d7559SDave Watson } 2853c4d7559SDave Watson copied = iov_iter_get_pages(from, pages, 2863c4d7559SDave Watson length, 2873c4d7559SDave Watson maxpages, &offset); 2883c4d7559SDave Watson if (copied <= 0) { 2893c4d7559SDave Watson rc = -EFAULT; 2903c4d7559SDave Watson goto out; 2913c4d7559SDave Watson } 2923c4d7559SDave Watson 2933c4d7559SDave Watson iov_iter_advance(from, copied); 2943c4d7559SDave Watson 2953c4d7559SDave Watson length -= copied; 2963c4d7559SDave Watson size += copied; 2973c4d7559SDave Watson while (copied) { 2983c4d7559SDave Watson use = min_t(int, copied, PAGE_SIZE - offset); 2993c4d7559SDave Watson 30069ca9293SDave Watson sg_set_page(&to[num_elem], 3013c4d7559SDave Watson pages[i], use, offset); 30269ca9293SDave Watson sg_unmark_end(&to[num_elem]); 30369ca9293SDave Watson if (charge) 3043c4d7559SDave Watson sk_mem_charge(sk, use); 3053c4d7559SDave Watson 3063c4d7559SDave Watson offset = 0; 3073c4d7559SDave Watson copied -= use; 3083c4d7559SDave Watson 3093c4d7559SDave Watson ++i; 3103c4d7559SDave Watson ++num_elem; 3113c4d7559SDave Watson } 3123c4d7559SDave Watson } 3133c4d7559SDave Watson 3143c4d7559SDave Watson out: 3152da19ed3SDoron Roberts-Kedes if (rc) 3162da19ed3SDoron Roberts-Kedes iov_iter_revert(from, size - *size_used); 31769ca9293SDave Watson *size_used = size; 31869ca9293SDave Watson *pages_used = num_elem; 31969ca9293SDave Watson 3203c4d7559SDave Watson return rc; 3213c4d7559SDave Watson } 3223c4d7559SDave Watson 3233c4d7559SDave Watson static int memcopy_from_iter(struct sock *sk, struct iov_iter *from, 3243c4d7559SDave Watson int bytes) 3253c4d7559SDave Watson { 3263c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 327f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 3283c4d7559SDave Watson struct scatterlist *sg = ctx->sg_plaintext_data; 3293c4d7559SDave Watson int copy, i, rc = 0; 3303c4d7559SDave Watson 3313c4d7559SDave Watson for (i = tls_ctx->pending_open_record_frags; 3323c4d7559SDave Watson i < ctx->sg_plaintext_num_elem; ++i) { 3333c4d7559SDave Watson copy = sg[i].length; 3343c4d7559SDave Watson if (copy_from_iter( 3353c4d7559SDave Watson page_address(sg_page(&sg[i])) + sg[i].offset, 3363c4d7559SDave Watson copy, from) != copy) { 3373c4d7559SDave Watson rc = -EFAULT; 3383c4d7559SDave Watson goto out; 3393c4d7559SDave Watson } 3403c4d7559SDave Watson bytes -= copy; 3413c4d7559SDave Watson 3423c4d7559SDave Watson ++tls_ctx->pending_open_record_frags; 3433c4d7559SDave Watson 3443c4d7559SDave Watson if (!bytes) 3453c4d7559SDave Watson break; 3463c4d7559SDave Watson } 3473c4d7559SDave Watson 3483c4d7559SDave Watson out: 3493c4d7559SDave Watson return rc; 3503c4d7559SDave Watson } 3513c4d7559SDave Watson 3523c4d7559SDave Watson int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 3533c4d7559SDave Watson { 3543c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 355f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 3563c4d7559SDave Watson int ret = 0; 3573c4d7559SDave Watson int required_size; 3583c4d7559SDave Watson long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 3593c4d7559SDave Watson bool eor = !(msg->msg_flags & MSG_MORE); 3603c4d7559SDave Watson size_t try_to_copy, copied = 0; 3613c4d7559SDave Watson unsigned char record_type = TLS_RECORD_TYPE_DATA; 3623c4d7559SDave Watson int record_room; 3633c4d7559SDave Watson bool full_record; 3643c4d7559SDave Watson int orig_size; 3650a26cf3fSDoron Roberts-Kedes bool is_kvec = msg->msg_iter.type & ITER_KVEC; 3663c4d7559SDave Watson 3673c4d7559SDave Watson if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 3683c4d7559SDave Watson return -ENOTSUPP; 3693c4d7559SDave Watson 3703c4d7559SDave Watson lock_sock(sk); 3713c4d7559SDave Watson 3723c4d7559SDave Watson if (tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo)) 3733c4d7559SDave Watson goto send_end; 3743c4d7559SDave Watson 3753c4d7559SDave Watson if (unlikely(msg->msg_controllen)) { 3763c4d7559SDave Watson ret = tls_proccess_cmsg(sk, msg, &record_type); 3773c4d7559SDave Watson if (ret) 3783c4d7559SDave Watson goto send_end; 3793c4d7559SDave Watson } 3803c4d7559SDave Watson 3813c4d7559SDave Watson while (msg_data_left(msg)) { 3823c4d7559SDave Watson if (sk->sk_err) { 38330be8f8dSr.hering@avm.de ret = -sk->sk_err; 3843c4d7559SDave Watson goto send_end; 3853c4d7559SDave Watson } 3863c4d7559SDave Watson 3873c4d7559SDave Watson orig_size = ctx->sg_plaintext_size; 3883c4d7559SDave Watson full_record = false; 3893c4d7559SDave Watson try_to_copy = msg_data_left(msg); 3903c4d7559SDave Watson record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size; 3913c4d7559SDave Watson if (try_to_copy >= record_room) { 3923c4d7559SDave Watson try_to_copy = record_room; 3933c4d7559SDave Watson full_record = true; 3943c4d7559SDave Watson } 3953c4d7559SDave Watson 3963c4d7559SDave Watson required_size = ctx->sg_plaintext_size + try_to_copy + 397dbe42559SDave Watson tls_ctx->tx.overhead_size; 3983c4d7559SDave Watson 3993c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 4003c4d7559SDave Watson goto wait_for_sndbuf; 4013c4d7559SDave Watson alloc_encrypted: 4023c4d7559SDave Watson ret = alloc_encrypted_sg(sk, required_size); 4033c4d7559SDave Watson if (ret) { 4043c4d7559SDave Watson if (ret != -ENOSPC) 4053c4d7559SDave Watson goto wait_for_memory; 4063c4d7559SDave Watson 4073c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 4083c4d7559SDave Watson * actually allocated. The difference is due 4093c4d7559SDave Watson * to max sg elements limit 4103c4d7559SDave Watson */ 4113c4d7559SDave Watson try_to_copy -= required_size - ctx->sg_encrypted_size; 4123c4d7559SDave Watson full_record = true; 4133c4d7559SDave Watson } 4140a26cf3fSDoron Roberts-Kedes if (!is_kvec && (full_record || eor)) { 4153c4d7559SDave Watson ret = zerocopy_from_iter(sk, &msg->msg_iter, 41669ca9293SDave Watson try_to_copy, &ctx->sg_plaintext_num_elem, 41769ca9293SDave Watson &ctx->sg_plaintext_size, 41869ca9293SDave Watson ctx->sg_plaintext_data, 41969ca9293SDave Watson ARRAY_SIZE(ctx->sg_plaintext_data), 4202da19ed3SDoron Roberts-Kedes true); 4213c4d7559SDave Watson if (ret) 4223c4d7559SDave Watson goto fallback_to_reg_send; 4233c4d7559SDave Watson 4243c4d7559SDave Watson copied += try_to_copy; 4253c4d7559SDave Watson ret = tls_push_record(sk, msg->msg_flags, record_type); 4265a3611efSDoron Roberts-Kedes if (ret) 4273c4d7559SDave Watson goto send_end; 4285a3611efSDoron Roberts-Kedes continue; 4293c4d7559SDave Watson 4303c4d7559SDave Watson fallback_to_reg_send: 4313c4d7559SDave Watson trim_sg(sk, ctx->sg_plaintext_data, 4323c4d7559SDave Watson &ctx->sg_plaintext_num_elem, 4333c4d7559SDave Watson &ctx->sg_plaintext_size, 4343c4d7559SDave Watson orig_size); 4353c4d7559SDave Watson } 4363c4d7559SDave Watson 4373c4d7559SDave Watson required_size = ctx->sg_plaintext_size + try_to_copy; 4383c4d7559SDave Watson alloc_plaintext: 4393c4d7559SDave Watson ret = alloc_plaintext_sg(sk, required_size); 4403c4d7559SDave Watson if (ret) { 4413c4d7559SDave Watson if (ret != -ENOSPC) 4423c4d7559SDave Watson goto wait_for_memory; 4433c4d7559SDave Watson 4443c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 4453c4d7559SDave Watson * actually allocated. The difference is due 4463c4d7559SDave Watson * to max sg elements limit 4473c4d7559SDave Watson */ 4483c4d7559SDave Watson try_to_copy -= required_size - ctx->sg_plaintext_size; 4493c4d7559SDave Watson full_record = true; 4503c4d7559SDave Watson 4513c4d7559SDave Watson trim_sg(sk, ctx->sg_encrypted_data, 4523c4d7559SDave Watson &ctx->sg_encrypted_num_elem, 4533c4d7559SDave Watson &ctx->sg_encrypted_size, 4543c4d7559SDave Watson ctx->sg_plaintext_size + 455dbe42559SDave Watson tls_ctx->tx.overhead_size); 4563c4d7559SDave Watson } 4573c4d7559SDave Watson 4583c4d7559SDave Watson ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy); 4593c4d7559SDave Watson if (ret) 4603c4d7559SDave Watson goto trim_sgl; 4613c4d7559SDave Watson 4623c4d7559SDave Watson copied += try_to_copy; 4633c4d7559SDave Watson if (full_record || eor) { 4643c4d7559SDave Watson push_record: 4653c4d7559SDave Watson ret = tls_push_record(sk, msg->msg_flags, record_type); 4663c4d7559SDave Watson if (ret) { 4673c4d7559SDave Watson if (ret == -ENOMEM) 4683c4d7559SDave Watson goto wait_for_memory; 4693c4d7559SDave Watson 4703c4d7559SDave Watson goto send_end; 4713c4d7559SDave Watson } 4723c4d7559SDave Watson } 4733c4d7559SDave Watson 4743c4d7559SDave Watson continue; 4753c4d7559SDave Watson 4763c4d7559SDave Watson wait_for_sndbuf: 4773c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 4783c4d7559SDave Watson wait_for_memory: 4793c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 4803c4d7559SDave Watson if (ret) { 4813c4d7559SDave Watson trim_sgl: 4823c4d7559SDave Watson trim_both_sgl(sk, orig_size); 4833c4d7559SDave Watson goto send_end; 4843c4d7559SDave Watson } 4853c4d7559SDave Watson 4863c4d7559SDave Watson if (tls_is_pending_closed_record(tls_ctx)) 4873c4d7559SDave Watson goto push_record; 4883c4d7559SDave Watson 4893c4d7559SDave Watson if (ctx->sg_encrypted_size < required_size) 4903c4d7559SDave Watson goto alloc_encrypted; 4913c4d7559SDave Watson 4923c4d7559SDave Watson goto alloc_plaintext; 4933c4d7559SDave Watson } 4943c4d7559SDave Watson 4953c4d7559SDave Watson send_end: 4963c4d7559SDave Watson ret = sk_stream_error(sk, msg->msg_flags, ret); 4973c4d7559SDave Watson 4983c4d7559SDave Watson release_sock(sk); 4993c4d7559SDave Watson return copied ? copied : ret; 5003c4d7559SDave Watson } 5013c4d7559SDave Watson 5023c4d7559SDave Watson int tls_sw_sendpage(struct sock *sk, struct page *page, 5033c4d7559SDave Watson int offset, size_t size, int flags) 5043c4d7559SDave Watson { 5053c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 506f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 5073c4d7559SDave Watson int ret = 0; 5083c4d7559SDave Watson long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 5093c4d7559SDave Watson bool eor; 5103c4d7559SDave Watson size_t orig_size = size; 5113c4d7559SDave Watson unsigned char record_type = TLS_RECORD_TYPE_DATA; 5123c4d7559SDave Watson struct scatterlist *sg; 5133c4d7559SDave Watson bool full_record; 5143c4d7559SDave Watson int record_room; 5153c4d7559SDave Watson 5163c4d7559SDave Watson if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 5173c4d7559SDave Watson MSG_SENDPAGE_NOTLAST)) 5183c4d7559SDave Watson return -ENOTSUPP; 5193c4d7559SDave Watson 5203c4d7559SDave Watson /* No MSG_EOR from splice, only look at MSG_MORE */ 5213c4d7559SDave Watson eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST)); 5223c4d7559SDave Watson 5233c4d7559SDave Watson lock_sock(sk); 5243c4d7559SDave Watson 5253c4d7559SDave Watson sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 5263c4d7559SDave Watson 5273c4d7559SDave Watson if (tls_complete_pending_work(sk, tls_ctx, flags, &timeo)) 5283c4d7559SDave Watson goto sendpage_end; 5293c4d7559SDave Watson 5303c4d7559SDave Watson /* Call the sk_stream functions to manage the sndbuf mem. */ 5313c4d7559SDave Watson while (size > 0) { 5323c4d7559SDave Watson size_t copy, required_size; 5333c4d7559SDave Watson 5343c4d7559SDave Watson if (sk->sk_err) { 53530be8f8dSr.hering@avm.de ret = -sk->sk_err; 5363c4d7559SDave Watson goto sendpage_end; 5373c4d7559SDave Watson } 5383c4d7559SDave Watson 5393c4d7559SDave Watson full_record = false; 5403c4d7559SDave Watson record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size; 5413c4d7559SDave Watson copy = size; 5423c4d7559SDave Watson if (copy >= record_room) { 5433c4d7559SDave Watson copy = record_room; 5443c4d7559SDave Watson full_record = true; 5453c4d7559SDave Watson } 5463c4d7559SDave Watson required_size = ctx->sg_plaintext_size + copy + 547dbe42559SDave Watson tls_ctx->tx.overhead_size; 5483c4d7559SDave Watson 5493c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 5503c4d7559SDave Watson goto wait_for_sndbuf; 5513c4d7559SDave Watson alloc_payload: 5523c4d7559SDave Watson ret = alloc_encrypted_sg(sk, required_size); 5533c4d7559SDave Watson if (ret) { 5543c4d7559SDave Watson if (ret != -ENOSPC) 5553c4d7559SDave Watson goto wait_for_memory; 5563c4d7559SDave Watson 5573c4d7559SDave Watson /* Adjust copy according to the amount that was 5583c4d7559SDave Watson * actually allocated. The difference is due 5593c4d7559SDave Watson * to max sg elements limit 5603c4d7559SDave Watson */ 5613c4d7559SDave Watson copy -= required_size - ctx->sg_plaintext_size; 5623c4d7559SDave Watson full_record = true; 5633c4d7559SDave Watson } 5643c4d7559SDave Watson 5653c4d7559SDave Watson get_page(page); 5663c4d7559SDave Watson sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem; 5673c4d7559SDave Watson sg_set_page(sg, page, copy, offset); 5687a8c4dd9SDave Watson sg_unmark_end(sg); 5697a8c4dd9SDave Watson 5703c4d7559SDave Watson ctx->sg_plaintext_num_elem++; 5713c4d7559SDave Watson 5723c4d7559SDave Watson sk_mem_charge(sk, copy); 5733c4d7559SDave Watson offset += copy; 5743c4d7559SDave Watson size -= copy; 5753c4d7559SDave Watson ctx->sg_plaintext_size += copy; 5763c4d7559SDave Watson tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem; 5773c4d7559SDave Watson 5783c4d7559SDave Watson if (full_record || eor || 5793c4d7559SDave Watson ctx->sg_plaintext_num_elem == 5803c4d7559SDave Watson ARRAY_SIZE(ctx->sg_plaintext_data)) { 5813c4d7559SDave Watson push_record: 5823c4d7559SDave Watson ret = tls_push_record(sk, flags, record_type); 5833c4d7559SDave Watson if (ret) { 5843c4d7559SDave Watson if (ret == -ENOMEM) 5853c4d7559SDave Watson goto wait_for_memory; 5863c4d7559SDave Watson 5873c4d7559SDave Watson goto sendpage_end; 5883c4d7559SDave Watson } 5893c4d7559SDave Watson } 5903c4d7559SDave Watson continue; 5913c4d7559SDave Watson wait_for_sndbuf: 5923c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 5933c4d7559SDave Watson wait_for_memory: 5943c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 5953c4d7559SDave Watson if (ret) { 5963c4d7559SDave Watson trim_both_sgl(sk, ctx->sg_plaintext_size); 5973c4d7559SDave Watson goto sendpage_end; 5983c4d7559SDave Watson } 5993c4d7559SDave Watson 6003c4d7559SDave Watson if (tls_is_pending_closed_record(tls_ctx)) 6013c4d7559SDave Watson goto push_record; 6023c4d7559SDave Watson 6033c4d7559SDave Watson goto alloc_payload; 6043c4d7559SDave Watson } 6053c4d7559SDave Watson 6063c4d7559SDave Watson sendpage_end: 6073c4d7559SDave Watson if (orig_size > size) 6083c4d7559SDave Watson ret = orig_size - size; 6093c4d7559SDave Watson else 6103c4d7559SDave Watson ret = sk_stream_error(sk, flags, ret); 6113c4d7559SDave Watson 6123c4d7559SDave Watson release_sock(sk); 6133c4d7559SDave Watson return ret; 6143c4d7559SDave Watson } 6153c4d7559SDave Watson 616c46234ebSDave Watson static struct sk_buff *tls_wait_data(struct sock *sk, int flags, 617c46234ebSDave Watson long timeo, int *err) 618c46234ebSDave Watson { 619c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 620f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 621c46234ebSDave Watson struct sk_buff *skb; 622c46234ebSDave Watson DEFINE_WAIT_FUNC(wait, woken_wake_function); 623c46234ebSDave Watson 624c46234ebSDave Watson while (!(skb = ctx->recv_pkt)) { 625c46234ebSDave Watson if (sk->sk_err) { 626c46234ebSDave Watson *err = sock_error(sk); 627c46234ebSDave Watson return NULL; 628c46234ebSDave Watson } 629c46234ebSDave Watson 630fcf4793eSDoron Roberts-Kedes if (sk->sk_shutdown & RCV_SHUTDOWN) 631fcf4793eSDoron Roberts-Kedes return NULL; 632fcf4793eSDoron Roberts-Kedes 633c46234ebSDave Watson if (sock_flag(sk, SOCK_DONE)) 634c46234ebSDave Watson return NULL; 635c46234ebSDave Watson 636c46234ebSDave Watson if ((flags & MSG_DONTWAIT) || !timeo) { 637c46234ebSDave Watson *err = -EAGAIN; 638c46234ebSDave Watson return NULL; 639c46234ebSDave Watson } 640c46234ebSDave Watson 641c46234ebSDave Watson add_wait_queue(sk_sleep(sk), &wait); 642c46234ebSDave Watson sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 643c46234ebSDave Watson sk_wait_event(sk, &timeo, ctx->recv_pkt != skb, &wait); 644c46234ebSDave Watson sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 645c46234ebSDave Watson remove_wait_queue(sk_sleep(sk), &wait); 646c46234ebSDave Watson 647c46234ebSDave Watson /* Handle signals */ 648c46234ebSDave Watson if (signal_pending(current)) { 649c46234ebSDave Watson *err = sock_intr_errno(timeo); 650c46234ebSDave Watson return NULL; 651c46234ebSDave Watson } 652c46234ebSDave Watson } 653c46234ebSDave Watson 654c46234ebSDave Watson return skb; 655c46234ebSDave Watson } 656c46234ebSDave Watson 657dafb67f3SBoris Pismenny static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, 6584799ac81SBoris Pismenny struct scatterlist *sgout, bool *zc) 659dafb67f3SBoris Pismenny { 660dafb67f3SBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 661dafb67f3SBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 662dafb67f3SBoris Pismenny struct strp_msg *rxm = strp_msg(skb); 663dafb67f3SBoris Pismenny int err = 0; 664dafb67f3SBoris Pismenny 6654799ac81SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 6664799ac81SBoris Pismenny err = tls_device_decrypted(sk, skb); 6674799ac81SBoris Pismenny if (err < 0) 6684799ac81SBoris Pismenny return err; 6694799ac81SBoris Pismenny #endif 6704799ac81SBoris Pismenny if (!ctx->decrypted) { 671dafb67f3SBoris Pismenny err = decrypt_skb(sk, skb, sgout); 672dafb67f3SBoris Pismenny if (err < 0) 673dafb67f3SBoris Pismenny return err; 6744799ac81SBoris Pismenny } else { 6754799ac81SBoris Pismenny *zc = false; 6764799ac81SBoris Pismenny } 677dafb67f3SBoris Pismenny 678dafb67f3SBoris Pismenny rxm->offset += tls_ctx->rx.prepend_size; 679dafb67f3SBoris Pismenny rxm->full_len -= tls_ctx->rx.overhead_size; 680dafb67f3SBoris Pismenny tls_advance_record_sn(sk, &tls_ctx->rx); 681dafb67f3SBoris Pismenny ctx->decrypted = true; 682dafb67f3SBoris Pismenny ctx->saved_data_ready(sk); 683dafb67f3SBoris Pismenny 684dafb67f3SBoris Pismenny return err; 685dafb67f3SBoris Pismenny } 686dafb67f3SBoris Pismenny 687dafb67f3SBoris Pismenny int decrypt_skb(struct sock *sk, struct sk_buff *skb, 688c46234ebSDave Watson struct scatterlist *sgout) 689c46234ebSDave Watson { 690c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 691f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 692b16520f7SKees Cook char iv[TLS_CIPHER_AES_GCM_128_SALT_SIZE + MAX_IV_SIZE]; 693c46234ebSDave Watson struct scatterlist sgin_arr[MAX_SKB_FRAGS + 2]; 694c46234ebSDave Watson struct scatterlist *sgin = &sgin_arr[0]; 695c46234ebSDave Watson struct strp_msg *rxm = strp_msg(skb); 696c46234ebSDave Watson int ret, nsg = ARRAY_SIZE(sgin_arr); 697c46234ebSDave Watson struct sk_buff *unused; 698c46234ebSDave Watson 699c46234ebSDave Watson ret = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE, 700c46234ebSDave Watson iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 701c46234ebSDave Watson tls_ctx->rx.iv_size); 702c46234ebSDave Watson if (ret < 0) 703c46234ebSDave Watson return ret; 704c46234ebSDave Watson 705c46234ebSDave Watson memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 706c46234ebSDave Watson if (!sgout) { 707c46234ebSDave Watson nsg = skb_cow_data(skb, 0, &unused) + 1; 708c46234ebSDave Watson sgin = kmalloc_array(nsg, sizeof(*sgin), sk->sk_allocation); 709c46234ebSDave Watson sgout = sgin; 710c46234ebSDave Watson } 711c46234ebSDave Watson 712c46234ebSDave Watson sg_init_table(sgin, nsg); 7138ab6ffbaSMatt Mullins sg_set_buf(&sgin[0], ctx->rx_aad_ciphertext, TLS_AAD_SPACE_SIZE); 714c46234ebSDave Watson 715c46234ebSDave Watson nsg = skb_to_sgvec(skb, &sgin[1], 716c46234ebSDave Watson rxm->offset + tls_ctx->rx.prepend_size, 717c46234ebSDave Watson rxm->full_len - tls_ctx->rx.prepend_size); 71852ee6ef3SDoron Roberts-Kedes if (nsg < 0) { 71952ee6ef3SDoron Roberts-Kedes ret = nsg; 72052ee6ef3SDoron Roberts-Kedes goto out; 72152ee6ef3SDoron Roberts-Kedes } 722c46234ebSDave Watson 7238ab6ffbaSMatt Mullins tls_make_aad(ctx->rx_aad_ciphertext, 724c46234ebSDave Watson rxm->full_len - tls_ctx->rx.overhead_size, 725c46234ebSDave Watson tls_ctx->rx.rec_seq, 726c46234ebSDave Watson tls_ctx->rx.rec_seq_size, 727c46234ebSDave Watson ctx->control); 728c46234ebSDave Watson 729c46234ebSDave Watson ret = tls_do_decryption(sk, sgin, sgout, iv, 730c46234ebSDave Watson rxm->full_len - tls_ctx->rx.overhead_size, 731c46234ebSDave Watson skb, sk->sk_allocation); 732c46234ebSDave Watson 73352ee6ef3SDoron Roberts-Kedes out: 734c46234ebSDave Watson if (sgin != &sgin_arr[0]) 735c46234ebSDave Watson kfree(sgin); 736c46234ebSDave Watson 737c46234ebSDave Watson return ret; 738c46234ebSDave Watson } 739c46234ebSDave Watson 740c46234ebSDave Watson static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb, 741c46234ebSDave Watson unsigned int len) 742c46234ebSDave Watson { 743c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 744f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 745c46234ebSDave Watson struct strp_msg *rxm = strp_msg(skb); 746c46234ebSDave Watson 747c46234ebSDave Watson if (len < rxm->full_len) { 748c46234ebSDave Watson rxm->offset += len; 749c46234ebSDave Watson rxm->full_len -= len; 750c46234ebSDave Watson 751c46234ebSDave Watson return false; 752c46234ebSDave Watson } 753c46234ebSDave Watson 754c46234ebSDave Watson /* Finished with message */ 755c46234ebSDave Watson ctx->recv_pkt = NULL; 756c46234ebSDave Watson kfree_skb(skb); 7577170e604SDoron Roberts-Kedes __strp_unpause(&ctx->strp); 758c46234ebSDave Watson 759c46234ebSDave Watson return true; 760c46234ebSDave Watson } 761c46234ebSDave Watson 762c46234ebSDave Watson int tls_sw_recvmsg(struct sock *sk, 763c46234ebSDave Watson struct msghdr *msg, 764c46234ebSDave Watson size_t len, 765c46234ebSDave Watson int nonblock, 766c46234ebSDave Watson int flags, 767c46234ebSDave Watson int *addr_len) 768c46234ebSDave Watson { 769c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 770f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 771c46234ebSDave Watson unsigned char control; 772c46234ebSDave Watson struct strp_msg *rxm; 773c46234ebSDave Watson struct sk_buff *skb; 774c46234ebSDave Watson ssize_t copied = 0; 775c46234ebSDave Watson bool cmsg = false; 77606030dbaSDaniel Borkmann int target, err = 0; 777c46234ebSDave Watson long timeo; 7780a26cf3fSDoron Roberts-Kedes bool is_kvec = msg->msg_iter.type & ITER_KVEC; 779c46234ebSDave Watson 780c46234ebSDave Watson flags |= nonblock; 781c46234ebSDave Watson 782c46234ebSDave Watson if (unlikely(flags & MSG_ERRQUEUE)) 783c46234ebSDave Watson return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR); 784c46234ebSDave Watson 785c46234ebSDave Watson lock_sock(sk); 786c46234ebSDave Watson 78706030dbaSDaniel Borkmann target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 788c46234ebSDave Watson timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 789c46234ebSDave Watson do { 790c46234ebSDave Watson bool zc = false; 791c46234ebSDave Watson int chunk = 0; 792c46234ebSDave Watson 793c46234ebSDave Watson skb = tls_wait_data(sk, flags, timeo, &err); 794c46234ebSDave Watson if (!skb) 795c46234ebSDave Watson goto recv_end; 796c46234ebSDave Watson 797c46234ebSDave Watson rxm = strp_msg(skb); 798c46234ebSDave Watson if (!cmsg) { 799c46234ebSDave Watson int cerr; 800c46234ebSDave Watson 801c46234ebSDave Watson cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, 802c46234ebSDave Watson sizeof(ctx->control), &ctx->control); 803c46234ebSDave Watson cmsg = true; 804c46234ebSDave Watson control = ctx->control; 805c46234ebSDave Watson if (ctx->control != TLS_RECORD_TYPE_DATA) { 806c46234ebSDave Watson if (cerr || msg->msg_flags & MSG_CTRUNC) { 807c46234ebSDave Watson err = -EIO; 808c46234ebSDave Watson goto recv_end; 809c46234ebSDave Watson } 810c46234ebSDave Watson } 811c46234ebSDave Watson } else if (control != ctx->control) { 812c46234ebSDave Watson goto recv_end; 813c46234ebSDave Watson } 814c46234ebSDave Watson 815c46234ebSDave Watson if (!ctx->decrypted) { 816c46234ebSDave Watson int page_count; 817c46234ebSDave Watson int to_copy; 818c46234ebSDave Watson 819c46234ebSDave Watson page_count = iov_iter_npages(&msg->msg_iter, 820c46234ebSDave Watson MAX_SKB_FRAGS); 821c46234ebSDave Watson to_copy = rxm->full_len - tls_ctx->rx.overhead_size; 8220a26cf3fSDoron Roberts-Kedes if (!is_kvec && to_copy <= len && page_count < MAX_SKB_FRAGS && 823c46234ebSDave Watson likely(!(flags & MSG_PEEK))) { 824c46234ebSDave Watson struct scatterlist sgin[MAX_SKB_FRAGS + 1]; 825c46234ebSDave Watson int pages = 0; 826c46234ebSDave Watson 827c46234ebSDave Watson zc = true; 828c46234ebSDave Watson sg_init_table(sgin, MAX_SKB_FRAGS + 1); 8298ab6ffbaSMatt Mullins sg_set_buf(&sgin[0], ctx->rx_aad_plaintext, 8308ab6ffbaSMatt Mullins TLS_AAD_SPACE_SIZE); 831c46234ebSDave Watson 832c46234ebSDave Watson err = zerocopy_from_iter(sk, &msg->msg_iter, 833c46234ebSDave Watson to_copy, &pages, 834c46234ebSDave Watson &chunk, &sgin[1], 8352da19ed3SDoron Roberts-Kedes MAX_SKB_FRAGS, false); 836c46234ebSDave Watson if (err < 0) 837c46234ebSDave Watson goto fallback_to_reg_recv; 838c46234ebSDave Watson 8394799ac81SBoris Pismenny err = decrypt_skb_update(sk, skb, sgin, &zc); 840c46234ebSDave Watson for (; pages > 0; pages--) 841c46234ebSDave Watson put_page(sg_page(&sgin[pages])); 842c46234ebSDave Watson if (err < 0) { 843c46234ebSDave Watson tls_err_abort(sk, EBADMSG); 844c46234ebSDave Watson goto recv_end; 845c46234ebSDave Watson } 846c46234ebSDave Watson } else { 847c46234ebSDave Watson fallback_to_reg_recv: 8484799ac81SBoris Pismenny err = decrypt_skb_update(sk, skb, NULL, &zc); 849c46234ebSDave Watson if (err < 0) { 850c46234ebSDave Watson tls_err_abort(sk, EBADMSG); 851c46234ebSDave Watson goto recv_end; 852c46234ebSDave Watson } 853c46234ebSDave Watson } 854c46234ebSDave Watson ctx->decrypted = true; 855c46234ebSDave Watson } 856c46234ebSDave Watson 857c46234ebSDave Watson if (!zc) { 858c46234ebSDave Watson chunk = min_t(unsigned int, rxm->full_len, len); 859c46234ebSDave Watson err = skb_copy_datagram_msg(skb, rxm->offset, msg, 860c46234ebSDave Watson chunk); 861c46234ebSDave Watson if (err < 0) 862c46234ebSDave Watson goto recv_end; 863c46234ebSDave Watson } 864c46234ebSDave Watson 865c46234ebSDave Watson copied += chunk; 866c46234ebSDave Watson len -= chunk; 867c46234ebSDave Watson if (likely(!(flags & MSG_PEEK))) { 868c46234ebSDave Watson u8 control = ctx->control; 869c46234ebSDave Watson 870c46234ebSDave Watson if (tls_sw_advance_skb(sk, skb, chunk)) { 871c46234ebSDave Watson /* Return full control message to 872c46234ebSDave Watson * userspace before trying to parse 873c46234ebSDave Watson * another message type 874c46234ebSDave Watson */ 875c46234ebSDave Watson msg->msg_flags |= MSG_EOR; 876c46234ebSDave Watson if (control != TLS_RECORD_TYPE_DATA) 877c46234ebSDave Watson goto recv_end; 878c46234ebSDave Watson } 879c46234ebSDave Watson } 88006030dbaSDaniel Borkmann /* If we have a new message from strparser, continue now. */ 88106030dbaSDaniel Borkmann if (copied >= target && !ctx->recv_pkt) 88206030dbaSDaniel Borkmann break; 883c46234ebSDave Watson } while (len); 884c46234ebSDave Watson 885c46234ebSDave Watson recv_end: 886c46234ebSDave Watson release_sock(sk); 887c46234ebSDave Watson return copied ? : err; 888c46234ebSDave Watson } 889c46234ebSDave Watson 890c46234ebSDave Watson ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, 891c46234ebSDave Watson struct pipe_inode_info *pipe, 892c46234ebSDave Watson size_t len, unsigned int flags) 893c46234ebSDave Watson { 894c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sock->sk); 895f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 896c46234ebSDave Watson struct strp_msg *rxm = NULL; 897c46234ebSDave Watson struct sock *sk = sock->sk; 898c46234ebSDave Watson struct sk_buff *skb; 899c46234ebSDave Watson ssize_t copied = 0; 900c46234ebSDave Watson int err = 0; 901c46234ebSDave Watson long timeo; 902c46234ebSDave Watson int chunk; 9034799ac81SBoris Pismenny bool zc; 904c46234ebSDave Watson 905c46234ebSDave Watson lock_sock(sk); 906c46234ebSDave Watson 907c46234ebSDave Watson timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 908c46234ebSDave Watson 909c46234ebSDave Watson skb = tls_wait_data(sk, flags, timeo, &err); 910c46234ebSDave Watson if (!skb) 911c46234ebSDave Watson goto splice_read_end; 912c46234ebSDave Watson 913c46234ebSDave Watson /* splice does not support reading control messages */ 914c46234ebSDave Watson if (ctx->control != TLS_RECORD_TYPE_DATA) { 915c46234ebSDave Watson err = -ENOTSUPP; 916c46234ebSDave Watson goto splice_read_end; 917c46234ebSDave Watson } 918c46234ebSDave Watson 919c46234ebSDave Watson if (!ctx->decrypted) { 9204799ac81SBoris Pismenny err = decrypt_skb_update(sk, skb, NULL, &zc); 921c46234ebSDave Watson 922c46234ebSDave Watson if (err < 0) { 923c46234ebSDave Watson tls_err_abort(sk, EBADMSG); 924c46234ebSDave Watson goto splice_read_end; 925c46234ebSDave Watson } 926c46234ebSDave Watson ctx->decrypted = true; 927c46234ebSDave Watson } 928c46234ebSDave Watson rxm = strp_msg(skb); 929c46234ebSDave Watson 930c46234ebSDave Watson chunk = min_t(unsigned int, rxm->full_len, len); 931c46234ebSDave Watson copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags); 932c46234ebSDave Watson if (copied < 0) 933c46234ebSDave Watson goto splice_read_end; 934c46234ebSDave Watson 935c46234ebSDave Watson if (likely(!(flags & MSG_PEEK))) 936c46234ebSDave Watson tls_sw_advance_skb(sk, skb, copied); 937c46234ebSDave Watson 938c46234ebSDave Watson splice_read_end: 939c46234ebSDave Watson release_sock(sk); 940c46234ebSDave Watson return copied ? : err; 941c46234ebSDave Watson } 942c46234ebSDave Watson 943a11e1d43SLinus Torvalds unsigned int tls_sw_poll(struct file *file, struct socket *sock, 944a11e1d43SLinus Torvalds struct poll_table_struct *wait) 945c46234ebSDave Watson { 946a11e1d43SLinus Torvalds unsigned int ret; 947c46234ebSDave Watson struct sock *sk = sock->sk; 948c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 949f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 950c46234ebSDave Watson 951a11e1d43SLinus Torvalds /* Grab POLLOUT and POLLHUP from the underlying socket */ 952a11e1d43SLinus Torvalds ret = ctx->sk_poll(file, sock, wait); 953c46234ebSDave Watson 954a11e1d43SLinus Torvalds /* Clear POLLIN bits, and set based on recv_pkt */ 955a11e1d43SLinus Torvalds ret &= ~(POLLIN | POLLRDNORM); 956c46234ebSDave Watson if (ctx->recv_pkt) 957a11e1d43SLinus Torvalds ret |= POLLIN | POLLRDNORM; 958c46234ebSDave Watson 959a11e1d43SLinus Torvalds return ret; 960c46234ebSDave Watson } 961c46234ebSDave Watson 962c46234ebSDave Watson static int tls_read_size(struct strparser *strp, struct sk_buff *skb) 963c46234ebSDave Watson { 964c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 965f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 9663463e51dSKees Cook char header[TLS_HEADER_SIZE + MAX_IV_SIZE]; 967c46234ebSDave Watson struct strp_msg *rxm = strp_msg(skb); 968c46234ebSDave Watson size_t cipher_overhead; 969c46234ebSDave Watson size_t data_len = 0; 970c46234ebSDave Watson int ret; 971c46234ebSDave Watson 972c46234ebSDave Watson /* Verify that we have a full TLS header, or wait for more data */ 973c46234ebSDave Watson if (rxm->offset + tls_ctx->rx.prepend_size > skb->len) 974c46234ebSDave Watson return 0; 975c46234ebSDave Watson 9763463e51dSKees Cook /* Sanity-check size of on-stack buffer. */ 9773463e51dSKees Cook if (WARN_ON(tls_ctx->rx.prepend_size > sizeof(header))) { 9783463e51dSKees Cook ret = -EINVAL; 9793463e51dSKees Cook goto read_failure; 9803463e51dSKees Cook } 9813463e51dSKees Cook 982c46234ebSDave Watson /* Linearize header to local buffer */ 983c46234ebSDave Watson ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size); 984c46234ebSDave Watson 985c46234ebSDave Watson if (ret < 0) 986c46234ebSDave Watson goto read_failure; 987c46234ebSDave Watson 988c46234ebSDave Watson ctx->control = header[0]; 989c46234ebSDave Watson 990c46234ebSDave Watson data_len = ((header[4] & 0xFF) | (header[3] << 8)); 991c46234ebSDave Watson 992c46234ebSDave Watson cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size; 993c46234ebSDave Watson 994c46234ebSDave Watson if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) { 995c46234ebSDave Watson ret = -EMSGSIZE; 996c46234ebSDave Watson goto read_failure; 997c46234ebSDave Watson } 998c46234ebSDave Watson if (data_len < cipher_overhead) { 999c46234ebSDave Watson ret = -EBADMSG; 1000c46234ebSDave Watson goto read_failure; 1001c46234ebSDave Watson } 1002c46234ebSDave Watson 1003c46234ebSDave Watson if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.version) || 1004c46234ebSDave Watson header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.version)) { 1005c46234ebSDave Watson ret = -EINVAL; 1006c46234ebSDave Watson goto read_failure; 1007c46234ebSDave Watson } 1008c46234ebSDave Watson 10094799ac81SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 10104799ac81SBoris Pismenny handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset, 10114799ac81SBoris Pismenny *(u64*)tls_ctx->rx.rec_seq); 10124799ac81SBoris Pismenny #endif 1013c46234ebSDave Watson return data_len + TLS_HEADER_SIZE; 1014c46234ebSDave Watson 1015c46234ebSDave Watson read_failure: 1016c46234ebSDave Watson tls_err_abort(strp->sk, ret); 1017c46234ebSDave Watson 1018c46234ebSDave Watson return ret; 1019c46234ebSDave Watson } 1020c46234ebSDave Watson 1021c46234ebSDave Watson static void tls_queue(struct strparser *strp, struct sk_buff *skb) 1022c46234ebSDave Watson { 1023c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 1024f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1025c46234ebSDave Watson 1026c46234ebSDave Watson ctx->decrypted = false; 1027c46234ebSDave Watson 1028c46234ebSDave Watson ctx->recv_pkt = skb; 1029c46234ebSDave Watson strp_pause(strp); 1030c46234ebSDave Watson 1031ad13acceSVakul Garg ctx->saved_data_ready(strp->sk); 1032c46234ebSDave Watson } 1033c46234ebSDave Watson 1034c46234ebSDave Watson static void tls_data_ready(struct sock *sk) 1035c46234ebSDave Watson { 1036c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1037f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1038c46234ebSDave Watson 1039c46234ebSDave Watson strp_data_ready(&ctx->strp); 1040c46234ebSDave Watson } 1041c46234ebSDave Watson 1042f66de3eeSBoris Pismenny void tls_sw_free_resources_tx(struct sock *sk) 10433c4d7559SDave Watson { 10443c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1045f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 10463c4d7559SDave Watson 10473c4d7559SDave Watson crypto_free_aead(ctx->aead_send); 1048f66de3eeSBoris Pismenny tls_free_both_sg(sk); 1049f66de3eeSBoris Pismenny 1050f66de3eeSBoris Pismenny kfree(ctx); 1051f66de3eeSBoris Pismenny } 1052f66de3eeSBoris Pismenny 105339f56e1aSBoris Pismenny void tls_sw_release_resources_rx(struct sock *sk) 1054f66de3eeSBoris Pismenny { 1055f66de3eeSBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 1056f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1057f66de3eeSBoris Pismenny 1058c46234ebSDave Watson if (ctx->aead_recv) { 1059c46234ebSDave Watson kfree_skb(ctx->recv_pkt); 1060c46234ebSDave Watson ctx->recv_pkt = NULL; 1061c46234ebSDave Watson crypto_free_aead(ctx->aead_recv); 1062c46234ebSDave Watson strp_stop(&ctx->strp); 1063c46234ebSDave Watson write_lock_bh(&sk->sk_callback_lock); 1064c46234ebSDave Watson sk->sk_data_ready = ctx->saved_data_ready; 1065c46234ebSDave Watson write_unlock_bh(&sk->sk_callback_lock); 1066c46234ebSDave Watson release_sock(sk); 1067c46234ebSDave Watson strp_done(&ctx->strp); 1068c46234ebSDave Watson lock_sock(sk); 1069c46234ebSDave Watson } 107039f56e1aSBoris Pismenny } 107139f56e1aSBoris Pismenny 107239f56e1aSBoris Pismenny void tls_sw_free_resources_rx(struct sock *sk) 107339f56e1aSBoris Pismenny { 107439f56e1aSBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 107539f56e1aSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 107639f56e1aSBoris Pismenny 107739f56e1aSBoris Pismenny tls_sw_release_resources_rx(sk); 10783c4d7559SDave Watson 10793c4d7559SDave Watson kfree(ctx); 10803c4d7559SDave Watson } 10813c4d7559SDave Watson 1082c46234ebSDave Watson int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) 10833c4d7559SDave Watson { 10843c4d7559SDave Watson char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE]; 10853c4d7559SDave Watson struct tls_crypto_info *crypto_info; 10863c4d7559SDave Watson struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; 1087f66de3eeSBoris Pismenny struct tls_sw_context_tx *sw_ctx_tx = NULL; 1088f66de3eeSBoris Pismenny struct tls_sw_context_rx *sw_ctx_rx = NULL; 1089c46234ebSDave Watson struct cipher_context *cctx; 1090c46234ebSDave Watson struct crypto_aead **aead; 1091c46234ebSDave Watson struct strp_callbacks cb; 10923c4d7559SDave Watson u16 nonce_size, tag_size, iv_size, rec_seq_size; 10933c4d7559SDave Watson char *iv, *rec_seq; 10943c4d7559SDave Watson int rc = 0; 10953c4d7559SDave Watson 10963c4d7559SDave Watson if (!ctx) { 10973c4d7559SDave Watson rc = -EINVAL; 10983c4d7559SDave Watson goto out; 10993c4d7559SDave Watson } 11003c4d7559SDave Watson 1101f66de3eeSBoris Pismenny if (tx) { 1102b190a587SBoris Pismenny if (!ctx->priv_ctx_tx) { 1103f66de3eeSBoris Pismenny sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL); 1104f66de3eeSBoris Pismenny if (!sw_ctx_tx) { 11053c4d7559SDave Watson rc = -ENOMEM; 11063c4d7559SDave Watson goto out; 11073c4d7559SDave Watson } 1108f66de3eeSBoris Pismenny ctx->priv_ctx_tx = sw_ctx_tx; 1109c46234ebSDave Watson } else { 1110b190a587SBoris Pismenny sw_ctx_tx = 1111b190a587SBoris Pismenny (struct tls_sw_context_tx *)ctx->priv_ctx_tx; 1112b190a587SBoris Pismenny } 1113b190a587SBoris Pismenny } else { 1114b190a587SBoris Pismenny if (!ctx->priv_ctx_rx) { 1115f66de3eeSBoris Pismenny sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL); 1116f66de3eeSBoris Pismenny if (!sw_ctx_rx) { 1117f66de3eeSBoris Pismenny rc = -ENOMEM; 1118f66de3eeSBoris Pismenny goto out; 1119c46234ebSDave Watson } 1120f66de3eeSBoris Pismenny ctx->priv_ctx_rx = sw_ctx_rx; 1121b190a587SBoris Pismenny } else { 1122b190a587SBoris Pismenny sw_ctx_rx = 1123b190a587SBoris Pismenny (struct tls_sw_context_rx *)ctx->priv_ctx_rx; 1124b190a587SBoris Pismenny } 1125f66de3eeSBoris Pismenny } 11263c4d7559SDave Watson 1127c46234ebSDave Watson if (tx) { 1128b190a587SBoris Pismenny crypto_init_wait(&sw_ctx_tx->async_wait); 11293c4d7559SDave Watson crypto_info = &ctx->crypto_send; 1130c46234ebSDave Watson cctx = &ctx->tx; 1131f66de3eeSBoris Pismenny aead = &sw_ctx_tx->aead_send; 1132c46234ebSDave Watson } else { 1133b190a587SBoris Pismenny crypto_init_wait(&sw_ctx_rx->async_wait); 1134c46234ebSDave Watson crypto_info = &ctx->crypto_recv; 1135c46234ebSDave Watson cctx = &ctx->rx; 1136f66de3eeSBoris Pismenny aead = &sw_ctx_rx->aead_recv; 1137c46234ebSDave Watson } 1138c46234ebSDave Watson 11393c4d7559SDave Watson switch (crypto_info->cipher_type) { 11403c4d7559SDave Watson case TLS_CIPHER_AES_GCM_128: { 11413c4d7559SDave Watson nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 11423c4d7559SDave Watson tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; 11433c4d7559SDave Watson iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 11443c4d7559SDave Watson iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; 11453c4d7559SDave Watson rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; 11463c4d7559SDave Watson rec_seq = 11473c4d7559SDave Watson ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; 11483c4d7559SDave Watson gcm_128_info = 11493c4d7559SDave Watson (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 11503c4d7559SDave Watson break; 11513c4d7559SDave Watson } 11523c4d7559SDave Watson default: 11533c4d7559SDave Watson rc = -EINVAL; 1154cf6d43efSSabrina Dubroca goto free_priv; 11553c4d7559SDave Watson } 11563c4d7559SDave Watson 1157b16520f7SKees Cook /* Sanity-check the IV size for stack allocations. */ 11583463e51dSKees Cook if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE) { 1159b16520f7SKees Cook rc = -EINVAL; 1160b16520f7SKees Cook goto free_priv; 1161b16520f7SKees Cook } 1162b16520f7SKees Cook 1163c46234ebSDave Watson cctx->prepend_size = TLS_HEADER_SIZE + nonce_size; 1164c46234ebSDave Watson cctx->tag_size = tag_size; 1165c46234ebSDave Watson cctx->overhead_size = cctx->prepend_size + cctx->tag_size; 1166c46234ebSDave Watson cctx->iv_size = iv_size; 1167c46234ebSDave Watson cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 1168dbe42559SDave Watson GFP_KERNEL); 1169c46234ebSDave Watson if (!cctx->iv) { 11703c4d7559SDave Watson rc = -ENOMEM; 1171cf6d43efSSabrina Dubroca goto free_priv; 11723c4d7559SDave Watson } 1173c46234ebSDave Watson memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 1174c46234ebSDave Watson memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size); 1175c46234ebSDave Watson cctx->rec_seq_size = rec_seq_size; 1176969d5090Szhong jiang cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL); 1177c46234ebSDave Watson if (!cctx->rec_seq) { 11783c4d7559SDave Watson rc = -ENOMEM; 11793c4d7559SDave Watson goto free_iv; 11803c4d7559SDave Watson } 11813c4d7559SDave Watson 1182f66de3eeSBoris Pismenny if (sw_ctx_tx) { 1183f66de3eeSBoris Pismenny sg_init_table(sw_ctx_tx->sg_encrypted_data, 1184f66de3eeSBoris Pismenny ARRAY_SIZE(sw_ctx_tx->sg_encrypted_data)); 1185f66de3eeSBoris Pismenny sg_init_table(sw_ctx_tx->sg_plaintext_data, 1186f66de3eeSBoris Pismenny ARRAY_SIZE(sw_ctx_tx->sg_plaintext_data)); 11873c4d7559SDave Watson 1188f66de3eeSBoris Pismenny sg_init_table(sw_ctx_tx->sg_aead_in, 2); 1189f66de3eeSBoris Pismenny sg_set_buf(&sw_ctx_tx->sg_aead_in[0], sw_ctx_tx->aad_space, 1190f66de3eeSBoris Pismenny sizeof(sw_ctx_tx->aad_space)); 1191f66de3eeSBoris Pismenny sg_unmark_end(&sw_ctx_tx->sg_aead_in[1]); 1192f66de3eeSBoris Pismenny sg_chain(sw_ctx_tx->sg_aead_in, 2, 1193f66de3eeSBoris Pismenny sw_ctx_tx->sg_plaintext_data); 1194f66de3eeSBoris Pismenny sg_init_table(sw_ctx_tx->sg_aead_out, 2); 1195f66de3eeSBoris Pismenny sg_set_buf(&sw_ctx_tx->sg_aead_out[0], sw_ctx_tx->aad_space, 1196f66de3eeSBoris Pismenny sizeof(sw_ctx_tx->aad_space)); 1197f66de3eeSBoris Pismenny sg_unmark_end(&sw_ctx_tx->sg_aead_out[1]); 1198f66de3eeSBoris Pismenny sg_chain(sw_ctx_tx->sg_aead_out, 2, 1199f66de3eeSBoris Pismenny sw_ctx_tx->sg_encrypted_data); 1200c46234ebSDave Watson } 12013c4d7559SDave Watson 1202c46234ebSDave Watson if (!*aead) { 1203c46234ebSDave Watson *aead = crypto_alloc_aead("gcm(aes)", 0, 0); 1204c46234ebSDave Watson if (IS_ERR(*aead)) { 1205c46234ebSDave Watson rc = PTR_ERR(*aead); 1206c46234ebSDave Watson *aead = NULL; 12073c4d7559SDave Watson goto free_rec_seq; 12083c4d7559SDave Watson } 12093c4d7559SDave Watson } 12103c4d7559SDave Watson 12113c4d7559SDave Watson ctx->push_pending_record = tls_sw_push_pending_record; 12123c4d7559SDave Watson 12133c4d7559SDave Watson memcpy(keyval, gcm_128_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE); 12143c4d7559SDave Watson 1215c46234ebSDave Watson rc = crypto_aead_setkey(*aead, keyval, 12163c4d7559SDave Watson TLS_CIPHER_AES_GCM_128_KEY_SIZE); 12173c4d7559SDave Watson if (rc) 12183c4d7559SDave Watson goto free_aead; 12193c4d7559SDave Watson 1220c46234ebSDave Watson rc = crypto_aead_setauthsize(*aead, cctx->tag_size); 1221c46234ebSDave Watson if (rc) 1222c46234ebSDave Watson goto free_aead; 1223c46234ebSDave Watson 1224f66de3eeSBoris Pismenny if (sw_ctx_rx) { 1225c46234ebSDave Watson /* Set up strparser */ 1226c46234ebSDave Watson memset(&cb, 0, sizeof(cb)); 1227c46234ebSDave Watson cb.rcv_msg = tls_queue; 1228c46234ebSDave Watson cb.parse_msg = tls_read_size; 1229c46234ebSDave Watson 1230f66de3eeSBoris Pismenny strp_init(&sw_ctx_rx->strp, sk, &cb); 1231c46234ebSDave Watson 1232c46234ebSDave Watson write_lock_bh(&sk->sk_callback_lock); 1233f66de3eeSBoris Pismenny sw_ctx_rx->saved_data_ready = sk->sk_data_ready; 1234c46234ebSDave Watson sk->sk_data_ready = tls_data_ready; 1235c46234ebSDave Watson write_unlock_bh(&sk->sk_callback_lock); 1236c46234ebSDave Watson 1237a11e1d43SLinus Torvalds sw_ctx_rx->sk_poll = sk->sk_socket->ops->poll; 1238c46234ebSDave Watson 1239f66de3eeSBoris Pismenny strp_check_rcv(&sw_ctx_rx->strp); 1240c46234ebSDave Watson } 1241c46234ebSDave Watson 1242c46234ebSDave Watson goto out; 12433c4d7559SDave Watson 12443c4d7559SDave Watson free_aead: 1245c46234ebSDave Watson crypto_free_aead(*aead); 1246c46234ebSDave Watson *aead = NULL; 12473c4d7559SDave Watson free_rec_seq: 1248c46234ebSDave Watson kfree(cctx->rec_seq); 1249c46234ebSDave Watson cctx->rec_seq = NULL; 12503c4d7559SDave Watson free_iv: 1251f66de3eeSBoris Pismenny kfree(cctx->iv); 1252f66de3eeSBoris Pismenny cctx->iv = NULL; 1253cf6d43efSSabrina Dubroca free_priv: 1254f66de3eeSBoris Pismenny if (tx) { 1255f66de3eeSBoris Pismenny kfree(ctx->priv_ctx_tx); 1256f66de3eeSBoris Pismenny ctx->priv_ctx_tx = NULL; 1257f66de3eeSBoris Pismenny } else { 1258f66de3eeSBoris Pismenny kfree(ctx->priv_ctx_rx); 1259f66de3eeSBoris Pismenny ctx->priv_ctx_rx = NULL; 1260f66de3eeSBoris Pismenny } 12613c4d7559SDave Watson out: 12623c4d7559SDave Watson return rc; 12633c4d7559SDave Watson } 1264