13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved. 53c4d7559SDave Watson * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved. 63c4d7559SDave Watson * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved. 73c4d7559SDave Watson * 83c4d7559SDave Watson * This software is available to you under a choice of one of two 93c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 103c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 113c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 123c4d7559SDave Watson * OpenIB.org BSD license below: 133c4d7559SDave Watson * 143c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 153c4d7559SDave Watson * without modification, are permitted provided that the following 163c4d7559SDave Watson * conditions are met: 173c4d7559SDave Watson * 183c4d7559SDave Watson * - Redistributions of source code must retain the above 193c4d7559SDave Watson * copyright notice, this list of conditions and the following 203c4d7559SDave Watson * disclaimer. 213c4d7559SDave Watson * 223c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 233c4d7559SDave Watson * copyright notice, this list of conditions and the following 243c4d7559SDave Watson * disclaimer in the documentation and/or other materials 253c4d7559SDave Watson * provided with the distribution. 263c4d7559SDave Watson * 273c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 283c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 293c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 303c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 313c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 323c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 333c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 343c4d7559SDave Watson * SOFTWARE. 353c4d7559SDave Watson */ 363c4d7559SDave Watson 373c4d7559SDave Watson #include <linux/module.h> 383c4d7559SDave Watson #include <crypto/aead.h> 393c4d7559SDave Watson 403c4d7559SDave Watson #include <net/tls.h> 413c4d7559SDave Watson 423c4d7559SDave Watson static void trim_sg(struct sock *sk, struct scatterlist *sg, 433c4d7559SDave Watson int *sg_num_elem, unsigned int *sg_size, int target_size) 443c4d7559SDave Watson { 453c4d7559SDave Watson int i = *sg_num_elem - 1; 463c4d7559SDave Watson int trim = *sg_size - target_size; 473c4d7559SDave Watson 483c4d7559SDave Watson if (trim <= 0) { 493c4d7559SDave Watson WARN_ON(trim < 0); 503c4d7559SDave Watson return; 513c4d7559SDave Watson } 523c4d7559SDave Watson 533c4d7559SDave Watson *sg_size = target_size; 543c4d7559SDave Watson while (trim >= sg[i].length) { 553c4d7559SDave Watson trim -= sg[i].length; 563c4d7559SDave Watson sk_mem_uncharge(sk, sg[i].length); 573c4d7559SDave Watson put_page(sg_page(&sg[i])); 583c4d7559SDave Watson i--; 593c4d7559SDave Watson 603c4d7559SDave Watson if (i < 0) 613c4d7559SDave Watson goto out; 623c4d7559SDave Watson } 633c4d7559SDave Watson 643c4d7559SDave Watson sg[i].length -= trim; 653c4d7559SDave Watson sk_mem_uncharge(sk, trim); 663c4d7559SDave Watson 673c4d7559SDave Watson out: 683c4d7559SDave Watson *sg_num_elem = i + 1; 693c4d7559SDave Watson } 703c4d7559SDave Watson 713c4d7559SDave Watson static void trim_both_sgl(struct sock *sk, int target_size) 723c4d7559SDave Watson { 733c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 743c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 753c4d7559SDave Watson 763c4d7559SDave Watson trim_sg(sk, ctx->sg_plaintext_data, 773c4d7559SDave Watson &ctx->sg_plaintext_num_elem, 783c4d7559SDave Watson &ctx->sg_plaintext_size, 793c4d7559SDave Watson target_size); 803c4d7559SDave Watson 813c4d7559SDave Watson if (target_size > 0) 823c4d7559SDave Watson target_size += tls_ctx->overhead_size; 833c4d7559SDave Watson 843c4d7559SDave Watson trim_sg(sk, ctx->sg_encrypted_data, 853c4d7559SDave Watson &ctx->sg_encrypted_num_elem, 863c4d7559SDave Watson &ctx->sg_encrypted_size, 873c4d7559SDave Watson target_size); 883c4d7559SDave Watson } 893c4d7559SDave Watson 903c4d7559SDave Watson static int alloc_encrypted_sg(struct sock *sk, int len) 913c4d7559SDave Watson { 923c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 933c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 943c4d7559SDave Watson int rc = 0; 953c4d7559SDave Watson 962c3682f0SJohn Fastabend rc = sk_alloc_sg(sk, len, 978c05dbf0SJohn Fastabend ctx->sg_encrypted_data, 0, 982c3682f0SJohn Fastabend &ctx->sg_encrypted_num_elem, 992c3682f0SJohn Fastabend &ctx->sg_encrypted_size, 0); 1003c4d7559SDave Watson 1013c4d7559SDave Watson return rc; 1023c4d7559SDave Watson } 1033c4d7559SDave Watson 1043c4d7559SDave Watson static int alloc_plaintext_sg(struct sock *sk, int len) 1053c4d7559SDave Watson { 1063c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1073c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 1083c4d7559SDave Watson int rc = 0; 1093c4d7559SDave Watson 1108c05dbf0SJohn Fastabend rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0, 1113c4d7559SDave Watson &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size, 1123c4d7559SDave Watson tls_ctx->pending_open_record_frags); 1133c4d7559SDave Watson 1143c4d7559SDave Watson return rc; 1153c4d7559SDave Watson } 1163c4d7559SDave Watson 1173c4d7559SDave Watson static void free_sg(struct sock *sk, struct scatterlist *sg, 1183c4d7559SDave Watson int *sg_num_elem, unsigned int *sg_size) 1193c4d7559SDave Watson { 1203c4d7559SDave Watson int i, n = *sg_num_elem; 1213c4d7559SDave Watson 1223c4d7559SDave Watson for (i = 0; i < n; ++i) { 1233c4d7559SDave Watson sk_mem_uncharge(sk, sg[i].length); 1243c4d7559SDave Watson put_page(sg_page(&sg[i])); 1253c4d7559SDave Watson } 1263c4d7559SDave Watson *sg_num_elem = 0; 1273c4d7559SDave Watson *sg_size = 0; 1283c4d7559SDave Watson } 1293c4d7559SDave Watson 1303c4d7559SDave Watson static void tls_free_both_sg(struct sock *sk) 1313c4d7559SDave Watson { 1323c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1333c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 1343c4d7559SDave Watson 1353c4d7559SDave Watson free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem, 1363c4d7559SDave Watson &ctx->sg_encrypted_size); 1373c4d7559SDave Watson 1383c4d7559SDave Watson free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, 1393c4d7559SDave Watson &ctx->sg_plaintext_size); 1403c4d7559SDave Watson } 1413c4d7559SDave Watson 1423c4d7559SDave Watson static int tls_do_encryption(struct tls_context *tls_ctx, 1433c4d7559SDave Watson struct tls_sw_context *ctx, size_t data_len, 1443c4d7559SDave Watson gfp_t flags) 1453c4d7559SDave Watson { 1463c4d7559SDave Watson unsigned int req_size = sizeof(struct aead_request) + 1473c4d7559SDave Watson crypto_aead_reqsize(ctx->aead_send); 1483c4d7559SDave Watson struct aead_request *aead_req; 1493c4d7559SDave Watson int rc; 1503c4d7559SDave Watson 15161ef6da6SIlya Lesokhin aead_req = kzalloc(req_size, flags); 1523c4d7559SDave Watson if (!aead_req) 1533c4d7559SDave Watson return -ENOMEM; 1543c4d7559SDave Watson 1553c4d7559SDave Watson ctx->sg_encrypted_data[0].offset += tls_ctx->prepend_size; 1563c4d7559SDave Watson ctx->sg_encrypted_data[0].length -= tls_ctx->prepend_size; 1573c4d7559SDave Watson 1583c4d7559SDave Watson aead_request_set_tfm(aead_req, ctx->aead_send); 1593c4d7559SDave Watson aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); 1603c4d7559SDave Watson aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out, 1613c4d7559SDave Watson data_len, tls_ctx->iv); 162a54667f6SVakul Garg 163a54667f6SVakul Garg aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 164a54667f6SVakul Garg crypto_req_done, &ctx->async_wait); 165a54667f6SVakul Garg 166a54667f6SVakul Garg rc = crypto_wait_req(crypto_aead_encrypt(aead_req), &ctx->async_wait); 1673c4d7559SDave Watson 1683c4d7559SDave Watson ctx->sg_encrypted_data[0].offset -= tls_ctx->prepend_size; 1693c4d7559SDave Watson ctx->sg_encrypted_data[0].length += tls_ctx->prepend_size; 1703c4d7559SDave Watson 1713c4d7559SDave Watson kfree(aead_req); 1723c4d7559SDave Watson return rc; 1733c4d7559SDave Watson } 1743c4d7559SDave Watson 1753c4d7559SDave Watson static int tls_push_record(struct sock *sk, int flags, 1763c4d7559SDave Watson unsigned char record_type) 1773c4d7559SDave Watson { 1783c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1793c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 1803c4d7559SDave Watson int rc; 1813c4d7559SDave Watson 1823c4d7559SDave Watson sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1); 1833c4d7559SDave Watson sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1); 1843c4d7559SDave Watson 185213ef6e7SIlya Lesokhin tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size, 1863c4d7559SDave Watson tls_ctx->rec_seq, tls_ctx->rec_seq_size, 1873c4d7559SDave Watson record_type); 1883c4d7559SDave Watson 1893c4d7559SDave Watson tls_fill_prepend(tls_ctx, 1903c4d7559SDave Watson page_address(sg_page(&ctx->sg_encrypted_data[0])) + 1913c4d7559SDave Watson ctx->sg_encrypted_data[0].offset, 1923c4d7559SDave Watson ctx->sg_plaintext_size, record_type); 1933c4d7559SDave Watson 1943c4d7559SDave Watson tls_ctx->pending_open_record_frags = 0; 1953c4d7559SDave Watson set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags); 1963c4d7559SDave Watson 1973c4d7559SDave Watson rc = tls_do_encryption(tls_ctx, ctx, ctx->sg_plaintext_size, 1983c4d7559SDave Watson sk->sk_allocation); 1993c4d7559SDave Watson if (rc < 0) { 2003c4d7559SDave Watson /* If we are called from write_space and 2013c4d7559SDave Watson * we fail, we need to set this SOCK_NOSPACE 2023c4d7559SDave Watson * to trigger another write_space in the future. 2033c4d7559SDave Watson */ 2043c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2053c4d7559SDave Watson return rc; 2063c4d7559SDave Watson } 2073c4d7559SDave Watson 2083c4d7559SDave Watson free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, 2093c4d7559SDave Watson &ctx->sg_plaintext_size); 2103c4d7559SDave Watson 2113c4d7559SDave Watson ctx->sg_encrypted_num_elem = 0; 2123c4d7559SDave Watson ctx->sg_encrypted_size = 0; 2133c4d7559SDave Watson 2143c4d7559SDave Watson /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */ 2153c4d7559SDave Watson rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags); 2163c4d7559SDave Watson if (rc < 0 && rc != -EAGAIN) 2173c4d7559SDave Watson tls_err_abort(sk); 2183c4d7559SDave Watson 2193c4d7559SDave Watson tls_advance_record_sn(sk, tls_ctx); 2203c4d7559SDave Watson return rc; 2213c4d7559SDave Watson } 2223c4d7559SDave Watson 2233c4d7559SDave Watson static int tls_sw_push_pending_record(struct sock *sk, int flags) 2243c4d7559SDave Watson { 2253c4d7559SDave Watson return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA); 2263c4d7559SDave Watson } 2273c4d7559SDave Watson 2283c4d7559SDave Watson static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from, 22969ca9293SDave Watson int length, int *pages_used, 23069ca9293SDave Watson unsigned int *size_used, 23169ca9293SDave Watson struct scatterlist *to, int to_max_pages, 23269ca9293SDave Watson bool charge) 2333c4d7559SDave Watson { 2343c4d7559SDave Watson struct page *pages[MAX_SKB_FRAGS]; 2353c4d7559SDave Watson 2363c4d7559SDave Watson size_t offset; 2373c4d7559SDave Watson ssize_t copied, use; 2383c4d7559SDave Watson int i = 0; 23969ca9293SDave Watson unsigned int size = *size_used; 24069ca9293SDave Watson int num_elem = *pages_used; 2413c4d7559SDave Watson int rc = 0; 2423c4d7559SDave Watson int maxpages; 2433c4d7559SDave Watson 2443c4d7559SDave Watson while (length > 0) { 2453c4d7559SDave Watson i = 0; 24669ca9293SDave Watson maxpages = to_max_pages - num_elem; 2473c4d7559SDave Watson if (maxpages == 0) { 2483c4d7559SDave Watson rc = -EFAULT; 2493c4d7559SDave Watson goto out; 2503c4d7559SDave Watson } 2513c4d7559SDave Watson copied = iov_iter_get_pages(from, pages, 2523c4d7559SDave Watson length, 2533c4d7559SDave Watson maxpages, &offset); 2543c4d7559SDave Watson if (copied <= 0) { 2553c4d7559SDave Watson rc = -EFAULT; 2563c4d7559SDave Watson goto out; 2573c4d7559SDave Watson } 2583c4d7559SDave Watson 2593c4d7559SDave Watson iov_iter_advance(from, copied); 2603c4d7559SDave Watson 2613c4d7559SDave Watson length -= copied; 2623c4d7559SDave Watson size += copied; 2633c4d7559SDave Watson while (copied) { 2643c4d7559SDave Watson use = min_t(int, copied, PAGE_SIZE - offset); 2653c4d7559SDave Watson 26669ca9293SDave Watson sg_set_page(&to[num_elem], 2673c4d7559SDave Watson pages[i], use, offset); 26869ca9293SDave Watson sg_unmark_end(&to[num_elem]); 26969ca9293SDave Watson if (charge) 2703c4d7559SDave Watson sk_mem_charge(sk, use); 2713c4d7559SDave Watson 2723c4d7559SDave Watson offset = 0; 2733c4d7559SDave Watson copied -= use; 2743c4d7559SDave Watson 2753c4d7559SDave Watson ++i; 2763c4d7559SDave Watson ++num_elem; 2773c4d7559SDave Watson } 2783c4d7559SDave Watson } 2793c4d7559SDave Watson 2803c4d7559SDave Watson out: 28169ca9293SDave Watson *size_used = size; 28269ca9293SDave Watson *pages_used = num_elem; 28369ca9293SDave Watson 2843c4d7559SDave Watson return rc; 2853c4d7559SDave Watson } 2863c4d7559SDave Watson 2873c4d7559SDave Watson static int memcopy_from_iter(struct sock *sk, struct iov_iter *from, 2883c4d7559SDave Watson int bytes) 2893c4d7559SDave Watson { 2903c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2913c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 2923c4d7559SDave Watson struct scatterlist *sg = ctx->sg_plaintext_data; 2933c4d7559SDave Watson int copy, i, rc = 0; 2943c4d7559SDave Watson 2953c4d7559SDave Watson for (i = tls_ctx->pending_open_record_frags; 2963c4d7559SDave Watson i < ctx->sg_plaintext_num_elem; ++i) { 2973c4d7559SDave Watson copy = sg[i].length; 2983c4d7559SDave Watson if (copy_from_iter( 2993c4d7559SDave Watson page_address(sg_page(&sg[i])) + sg[i].offset, 3003c4d7559SDave Watson copy, from) != copy) { 3013c4d7559SDave Watson rc = -EFAULT; 3023c4d7559SDave Watson goto out; 3033c4d7559SDave Watson } 3043c4d7559SDave Watson bytes -= copy; 3053c4d7559SDave Watson 3063c4d7559SDave Watson ++tls_ctx->pending_open_record_frags; 3073c4d7559SDave Watson 3083c4d7559SDave Watson if (!bytes) 3093c4d7559SDave Watson break; 3103c4d7559SDave Watson } 3113c4d7559SDave Watson 3123c4d7559SDave Watson out: 3133c4d7559SDave Watson return rc; 3143c4d7559SDave Watson } 3153c4d7559SDave Watson 3163c4d7559SDave Watson int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 3173c4d7559SDave Watson { 3183c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 3193c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 3203c4d7559SDave Watson int ret = 0; 3213c4d7559SDave Watson int required_size; 3223c4d7559SDave Watson long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 3233c4d7559SDave Watson bool eor = !(msg->msg_flags & MSG_MORE); 3243c4d7559SDave Watson size_t try_to_copy, copied = 0; 3253c4d7559SDave Watson unsigned char record_type = TLS_RECORD_TYPE_DATA; 3263c4d7559SDave Watson int record_room; 3273c4d7559SDave Watson bool full_record; 3283c4d7559SDave Watson int orig_size; 3293c4d7559SDave Watson 3303c4d7559SDave Watson if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 3313c4d7559SDave Watson return -ENOTSUPP; 3323c4d7559SDave Watson 3333c4d7559SDave Watson lock_sock(sk); 3343c4d7559SDave Watson 3353c4d7559SDave Watson if (tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo)) 3363c4d7559SDave Watson goto send_end; 3373c4d7559SDave Watson 3383c4d7559SDave Watson if (unlikely(msg->msg_controllen)) { 3393c4d7559SDave Watson ret = tls_proccess_cmsg(sk, msg, &record_type); 3403c4d7559SDave Watson if (ret) 3413c4d7559SDave Watson goto send_end; 3423c4d7559SDave Watson } 3433c4d7559SDave Watson 3443c4d7559SDave Watson while (msg_data_left(msg)) { 3453c4d7559SDave Watson if (sk->sk_err) { 34630be8f8dSr.hering@avm.de ret = -sk->sk_err; 3473c4d7559SDave Watson goto send_end; 3483c4d7559SDave Watson } 3493c4d7559SDave Watson 3503c4d7559SDave Watson orig_size = ctx->sg_plaintext_size; 3513c4d7559SDave Watson full_record = false; 3523c4d7559SDave Watson try_to_copy = msg_data_left(msg); 3533c4d7559SDave Watson record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size; 3543c4d7559SDave Watson if (try_to_copy >= record_room) { 3553c4d7559SDave Watson try_to_copy = record_room; 3563c4d7559SDave Watson full_record = true; 3573c4d7559SDave Watson } 3583c4d7559SDave Watson 3593c4d7559SDave Watson required_size = ctx->sg_plaintext_size + try_to_copy + 3603c4d7559SDave Watson tls_ctx->overhead_size; 3613c4d7559SDave Watson 3623c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 3633c4d7559SDave Watson goto wait_for_sndbuf; 3643c4d7559SDave Watson alloc_encrypted: 3653c4d7559SDave Watson ret = alloc_encrypted_sg(sk, required_size); 3663c4d7559SDave Watson if (ret) { 3673c4d7559SDave Watson if (ret != -ENOSPC) 3683c4d7559SDave Watson goto wait_for_memory; 3693c4d7559SDave Watson 3703c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 3713c4d7559SDave Watson * actually allocated. The difference is due 3723c4d7559SDave Watson * to max sg elements limit 3733c4d7559SDave Watson */ 3743c4d7559SDave Watson try_to_copy -= required_size - ctx->sg_encrypted_size; 3753c4d7559SDave Watson full_record = true; 3763c4d7559SDave Watson } 3773c4d7559SDave Watson 3783c4d7559SDave Watson if (full_record || eor) { 3793c4d7559SDave Watson ret = zerocopy_from_iter(sk, &msg->msg_iter, 38069ca9293SDave Watson try_to_copy, &ctx->sg_plaintext_num_elem, 38169ca9293SDave Watson &ctx->sg_plaintext_size, 38269ca9293SDave Watson ctx->sg_plaintext_data, 38369ca9293SDave Watson ARRAY_SIZE(ctx->sg_plaintext_data), 38469ca9293SDave Watson true); 3853c4d7559SDave Watson if (ret) 3863c4d7559SDave Watson goto fallback_to_reg_send; 3873c4d7559SDave Watson 3883c4d7559SDave Watson copied += try_to_copy; 3893c4d7559SDave Watson ret = tls_push_record(sk, msg->msg_flags, record_type); 3903c4d7559SDave Watson if (!ret) 3913c4d7559SDave Watson continue; 3923c4d7559SDave Watson if (ret == -EAGAIN) 3933c4d7559SDave Watson goto send_end; 3943c4d7559SDave Watson 3953c4d7559SDave Watson copied -= try_to_copy; 3963c4d7559SDave Watson fallback_to_reg_send: 3973c4d7559SDave Watson iov_iter_revert(&msg->msg_iter, 3983c4d7559SDave Watson ctx->sg_plaintext_size - orig_size); 3993c4d7559SDave Watson trim_sg(sk, ctx->sg_plaintext_data, 4003c4d7559SDave Watson &ctx->sg_plaintext_num_elem, 4013c4d7559SDave Watson &ctx->sg_plaintext_size, 4023c4d7559SDave Watson orig_size); 4033c4d7559SDave Watson } 4043c4d7559SDave Watson 4053c4d7559SDave Watson required_size = ctx->sg_plaintext_size + try_to_copy; 4063c4d7559SDave Watson alloc_plaintext: 4073c4d7559SDave Watson ret = alloc_plaintext_sg(sk, required_size); 4083c4d7559SDave Watson if (ret) { 4093c4d7559SDave Watson if (ret != -ENOSPC) 4103c4d7559SDave Watson goto wait_for_memory; 4113c4d7559SDave Watson 4123c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 4133c4d7559SDave Watson * actually allocated. The difference is due 4143c4d7559SDave Watson * to max sg elements limit 4153c4d7559SDave Watson */ 4163c4d7559SDave Watson try_to_copy -= required_size - ctx->sg_plaintext_size; 4173c4d7559SDave Watson full_record = true; 4183c4d7559SDave Watson 4193c4d7559SDave Watson trim_sg(sk, ctx->sg_encrypted_data, 4203c4d7559SDave Watson &ctx->sg_encrypted_num_elem, 4213c4d7559SDave Watson &ctx->sg_encrypted_size, 4223c4d7559SDave Watson ctx->sg_plaintext_size + 4233c4d7559SDave Watson tls_ctx->overhead_size); 4243c4d7559SDave Watson } 4253c4d7559SDave Watson 4263c4d7559SDave Watson ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy); 4273c4d7559SDave Watson if (ret) 4283c4d7559SDave Watson goto trim_sgl; 4293c4d7559SDave Watson 4303c4d7559SDave Watson copied += try_to_copy; 4313c4d7559SDave Watson if (full_record || eor) { 4323c4d7559SDave Watson push_record: 4333c4d7559SDave Watson ret = tls_push_record(sk, msg->msg_flags, record_type); 4343c4d7559SDave Watson if (ret) { 4353c4d7559SDave Watson if (ret == -ENOMEM) 4363c4d7559SDave Watson goto wait_for_memory; 4373c4d7559SDave Watson 4383c4d7559SDave Watson goto send_end; 4393c4d7559SDave Watson } 4403c4d7559SDave Watson } 4413c4d7559SDave Watson 4423c4d7559SDave Watson continue; 4433c4d7559SDave Watson 4443c4d7559SDave Watson wait_for_sndbuf: 4453c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 4463c4d7559SDave Watson wait_for_memory: 4473c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 4483c4d7559SDave Watson if (ret) { 4493c4d7559SDave Watson trim_sgl: 4503c4d7559SDave Watson trim_both_sgl(sk, orig_size); 4513c4d7559SDave Watson goto send_end; 4523c4d7559SDave Watson } 4533c4d7559SDave Watson 4543c4d7559SDave Watson if (tls_is_pending_closed_record(tls_ctx)) 4553c4d7559SDave Watson goto push_record; 4563c4d7559SDave Watson 4573c4d7559SDave Watson if (ctx->sg_encrypted_size < required_size) 4583c4d7559SDave Watson goto alloc_encrypted; 4593c4d7559SDave Watson 4603c4d7559SDave Watson goto alloc_plaintext; 4613c4d7559SDave Watson } 4623c4d7559SDave Watson 4633c4d7559SDave Watson send_end: 4643c4d7559SDave Watson ret = sk_stream_error(sk, msg->msg_flags, ret); 4653c4d7559SDave Watson 4663c4d7559SDave Watson release_sock(sk); 4673c4d7559SDave Watson return copied ? copied : ret; 4683c4d7559SDave Watson } 4693c4d7559SDave Watson 4703c4d7559SDave Watson int tls_sw_sendpage(struct sock *sk, struct page *page, 4713c4d7559SDave Watson int offset, size_t size, int flags) 4723c4d7559SDave Watson { 4733c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 4743c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 4753c4d7559SDave Watson int ret = 0; 4763c4d7559SDave Watson long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 4773c4d7559SDave Watson bool eor; 4783c4d7559SDave Watson size_t orig_size = size; 4793c4d7559SDave Watson unsigned char record_type = TLS_RECORD_TYPE_DATA; 4803c4d7559SDave Watson struct scatterlist *sg; 4813c4d7559SDave Watson bool full_record; 4823c4d7559SDave Watson int record_room; 4833c4d7559SDave Watson 4843c4d7559SDave Watson if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 4853c4d7559SDave Watson MSG_SENDPAGE_NOTLAST)) 4863c4d7559SDave Watson return -ENOTSUPP; 4873c4d7559SDave Watson 4883c4d7559SDave Watson /* No MSG_EOR from splice, only look at MSG_MORE */ 4893c4d7559SDave Watson eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST)); 4903c4d7559SDave Watson 4913c4d7559SDave Watson lock_sock(sk); 4923c4d7559SDave Watson 4933c4d7559SDave Watson sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 4943c4d7559SDave Watson 4953c4d7559SDave Watson if (tls_complete_pending_work(sk, tls_ctx, flags, &timeo)) 4963c4d7559SDave Watson goto sendpage_end; 4973c4d7559SDave Watson 4983c4d7559SDave Watson /* Call the sk_stream functions to manage the sndbuf mem. */ 4993c4d7559SDave Watson while (size > 0) { 5003c4d7559SDave Watson size_t copy, required_size; 5013c4d7559SDave Watson 5023c4d7559SDave Watson if (sk->sk_err) { 50330be8f8dSr.hering@avm.de ret = -sk->sk_err; 5043c4d7559SDave Watson goto sendpage_end; 5053c4d7559SDave Watson } 5063c4d7559SDave Watson 5073c4d7559SDave Watson full_record = false; 5083c4d7559SDave Watson record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size; 5093c4d7559SDave Watson copy = size; 5103c4d7559SDave Watson if (copy >= record_room) { 5113c4d7559SDave Watson copy = record_room; 5123c4d7559SDave Watson full_record = true; 5133c4d7559SDave Watson } 5143c4d7559SDave Watson required_size = ctx->sg_plaintext_size + copy + 5153c4d7559SDave Watson tls_ctx->overhead_size; 5163c4d7559SDave Watson 5173c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 5183c4d7559SDave Watson goto wait_for_sndbuf; 5193c4d7559SDave Watson alloc_payload: 5203c4d7559SDave Watson ret = alloc_encrypted_sg(sk, required_size); 5213c4d7559SDave Watson if (ret) { 5223c4d7559SDave Watson if (ret != -ENOSPC) 5233c4d7559SDave Watson goto wait_for_memory; 5243c4d7559SDave Watson 5253c4d7559SDave Watson /* Adjust copy according to the amount that was 5263c4d7559SDave Watson * actually allocated. The difference is due 5273c4d7559SDave Watson * to max sg elements limit 5283c4d7559SDave Watson */ 5293c4d7559SDave Watson copy -= required_size - ctx->sg_plaintext_size; 5303c4d7559SDave Watson full_record = true; 5313c4d7559SDave Watson } 5323c4d7559SDave Watson 5333c4d7559SDave Watson get_page(page); 5343c4d7559SDave Watson sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem; 5353c4d7559SDave Watson sg_set_page(sg, page, copy, offset); 5367a8c4dd9SDave Watson sg_unmark_end(sg); 5377a8c4dd9SDave Watson 5383c4d7559SDave Watson ctx->sg_plaintext_num_elem++; 5393c4d7559SDave Watson 5403c4d7559SDave Watson sk_mem_charge(sk, copy); 5413c4d7559SDave Watson offset += copy; 5423c4d7559SDave Watson size -= copy; 5433c4d7559SDave Watson ctx->sg_plaintext_size += copy; 5443c4d7559SDave Watson tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem; 5453c4d7559SDave Watson 5463c4d7559SDave Watson if (full_record || eor || 5473c4d7559SDave Watson ctx->sg_plaintext_num_elem == 5483c4d7559SDave Watson ARRAY_SIZE(ctx->sg_plaintext_data)) { 5493c4d7559SDave Watson push_record: 5503c4d7559SDave Watson ret = tls_push_record(sk, flags, record_type); 5513c4d7559SDave Watson if (ret) { 5523c4d7559SDave Watson if (ret == -ENOMEM) 5533c4d7559SDave Watson goto wait_for_memory; 5543c4d7559SDave Watson 5553c4d7559SDave Watson goto sendpage_end; 5563c4d7559SDave Watson } 5573c4d7559SDave Watson } 5583c4d7559SDave Watson continue; 5593c4d7559SDave Watson wait_for_sndbuf: 5603c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 5613c4d7559SDave Watson wait_for_memory: 5623c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 5633c4d7559SDave Watson if (ret) { 5643c4d7559SDave Watson trim_both_sgl(sk, ctx->sg_plaintext_size); 5653c4d7559SDave Watson goto sendpage_end; 5663c4d7559SDave Watson } 5673c4d7559SDave Watson 5683c4d7559SDave Watson if (tls_is_pending_closed_record(tls_ctx)) 5693c4d7559SDave Watson goto push_record; 5703c4d7559SDave Watson 5713c4d7559SDave Watson goto alloc_payload; 5723c4d7559SDave Watson } 5733c4d7559SDave Watson 5743c4d7559SDave Watson sendpage_end: 5753c4d7559SDave Watson if (orig_size > size) 5763c4d7559SDave Watson ret = orig_size - size; 5773c4d7559SDave Watson else 5783c4d7559SDave Watson ret = sk_stream_error(sk, flags, ret); 5793c4d7559SDave Watson 5803c4d7559SDave Watson release_sock(sk); 5813c4d7559SDave Watson return ret; 5823c4d7559SDave Watson } 5833c4d7559SDave Watson 584ff45d820SIlya Lesokhin void tls_sw_free_tx_resources(struct sock *sk) 5853c4d7559SDave Watson { 5863c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 5873c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 5883c4d7559SDave Watson 5893c4d7559SDave Watson if (ctx->aead_send) 5903c4d7559SDave Watson crypto_free_aead(ctx->aead_send); 5913c4d7559SDave Watson 5923c4d7559SDave Watson tls_free_both_sg(sk); 5933c4d7559SDave Watson 5943c4d7559SDave Watson kfree(ctx); 595ff45d820SIlya Lesokhin kfree(tls_ctx); 5963c4d7559SDave Watson } 5973c4d7559SDave Watson 5983c4d7559SDave Watson int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx) 5993c4d7559SDave Watson { 6003c4d7559SDave Watson char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE]; 6013c4d7559SDave Watson struct tls_crypto_info *crypto_info; 6023c4d7559SDave Watson struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; 6033c4d7559SDave Watson struct tls_sw_context *sw_ctx; 6043c4d7559SDave Watson u16 nonce_size, tag_size, iv_size, rec_seq_size; 6053c4d7559SDave Watson char *iv, *rec_seq; 6063c4d7559SDave Watson int rc = 0; 6073c4d7559SDave Watson 6083c4d7559SDave Watson if (!ctx) { 6093c4d7559SDave Watson rc = -EINVAL; 6103c4d7559SDave Watson goto out; 6113c4d7559SDave Watson } 6123c4d7559SDave Watson 6133c4d7559SDave Watson if (ctx->priv_ctx) { 6143c4d7559SDave Watson rc = -EEXIST; 6153c4d7559SDave Watson goto out; 6163c4d7559SDave Watson } 6173c4d7559SDave Watson 6183c4d7559SDave Watson sw_ctx = kzalloc(sizeof(*sw_ctx), GFP_KERNEL); 6193c4d7559SDave Watson if (!sw_ctx) { 6203c4d7559SDave Watson rc = -ENOMEM; 6213c4d7559SDave Watson goto out; 6223c4d7559SDave Watson } 6233c4d7559SDave Watson 624a54667f6SVakul Garg crypto_init_wait(&sw_ctx->async_wait); 625a54667f6SVakul Garg 6263c4d7559SDave Watson ctx->priv_ctx = (struct tls_offload_context *)sw_ctx; 6273c4d7559SDave Watson 6283c4d7559SDave Watson crypto_info = &ctx->crypto_send; 6293c4d7559SDave Watson switch (crypto_info->cipher_type) { 6303c4d7559SDave Watson case TLS_CIPHER_AES_GCM_128: { 6313c4d7559SDave Watson nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 6323c4d7559SDave Watson tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; 6333c4d7559SDave Watson iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 6343c4d7559SDave Watson iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; 6353c4d7559SDave Watson rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; 6363c4d7559SDave Watson rec_seq = 6373c4d7559SDave Watson ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; 6383c4d7559SDave Watson gcm_128_info = 6393c4d7559SDave Watson (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 6403c4d7559SDave Watson break; 6413c4d7559SDave Watson } 6423c4d7559SDave Watson default: 6433c4d7559SDave Watson rc = -EINVAL; 644cf6d43efSSabrina Dubroca goto free_priv; 6453c4d7559SDave Watson } 6463c4d7559SDave Watson 6473c4d7559SDave Watson ctx->prepend_size = TLS_HEADER_SIZE + nonce_size; 6483c4d7559SDave Watson ctx->tag_size = tag_size; 6493c4d7559SDave Watson ctx->overhead_size = ctx->prepend_size + ctx->tag_size; 6503c4d7559SDave Watson ctx->iv_size = iv_size; 651cf6d43efSSabrina Dubroca ctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, GFP_KERNEL); 6523c4d7559SDave Watson if (!ctx->iv) { 6533c4d7559SDave Watson rc = -ENOMEM; 654cf6d43efSSabrina Dubroca goto free_priv; 6553c4d7559SDave Watson } 6563c4d7559SDave Watson memcpy(ctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 6573c4d7559SDave Watson memcpy(ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size); 6583c4d7559SDave Watson ctx->rec_seq_size = rec_seq_size; 6593c4d7559SDave Watson ctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL); 6603c4d7559SDave Watson if (!ctx->rec_seq) { 6613c4d7559SDave Watson rc = -ENOMEM; 6623c4d7559SDave Watson goto free_iv; 6633c4d7559SDave Watson } 6643c4d7559SDave Watson memcpy(ctx->rec_seq, rec_seq, rec_seq_size); 6653c4d7559SDave Watson 6663c4d7559SDave Watson sg_init_table(sw_ctx->sg_encrypted_data, 6673c4d7559SDave Watson ARRAY_SIZE(sw_ctx->sg_encrypted_data)); 6683c4d7559SDave Watson sg_init_table(sw_ctx->sg_plaintext_data, 6693c4d7559SDave Watson ARRAY_SIZE(sw_ctx->sg_plaintext_data)); 6703c4d7559SDave Watson 6713c4d7559SDave Watson sg_init_table(sw_ctx->sg_aead_in, 2); 6723c4d7559SDave Watson sg_set_buf(&sw_ctx->sg_aead_in[0], sw_ctx->aad_space, 6733c4d7559SDave Watson sizeof(sw_ctx->aad_space)); 6743c4d7559SDave Watson sg_unmark_end(&sw_ctx->sg_aead_in[1]); 6753c4d7559SDave Watson sg_chain(sw_ctx->sg_aead_in, 2, sw_ctx->sg_plaintext_data); 6763c4d7559SDave Watson sg_init_table(sw_ctx->sg_aead_out, 2); 6773c4d7559SDave Watson sg_set_buf(&sw_ctx->sg_aead_out[0], sw_ctx->aad_space, 6783c4d7559SDave Watson sizeof(sw_ctx->aad_space)); 6793c4d7559SDave Watson sg_unmark_end(&sw_ctx->sg_aead_out[1]); 6803c4d7559SDave Watson sg_chain(sw_ctx->sg_aead_out, 2, sw_ctx->sg_encrypted_data); 6813c4d7559SDave Watson 6823c4d7559SDave Watson if (!sw_ctx->aead_send) { 6833c4d7559SDave Watson sw_ctx->aead_send = crypto_alloc_aead("gcm(aes)", 0, 0); 6843c4d7559SDave Watson if (IS_ERR(sw_ctx->aead_send)) { 6853c4d7559SDave Watson rc = PTR_ERR(sw_ctx->aead_send); 6863c4d7559SDave Watson sw_ctx->aead_send = NULL; 6873c4d7559SDave Watson goto free_rec_seq; 6883c4d7559SDave Watson } 6893c4d7559SDave Watson } 6903c4d7559SDave Watson 6913c4d7559SDave Watson ctx->push_pending_record = tls_sw_push_pending_record; 6923c4d7559SDave Watson 6933c4d7559SDave Watson memcpy(keyval, gcm_128_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE); 6943c4d7559SDave Watson 6953c4d7559SDave Watson rc = crypto_aead_setkey(sw_ctx->aead_send, keyval, 6963c4d7559SDave Watson TLS_CIPHER_AES_GCM_128_KEY_SIZE); 6973c4d7559SDave Watson if (rc) 6983c4d7559SDave Watson goto free_aead; 6993c4d7559SDave Watson 7003c4d7559SDave Watson rc = crypto_aead_setauthsize(sw_ctx->aead_send, ctx->tag_size); 7013c4d7559SDave Watson if (!rc) 702cf6d43efSSabrina Dubroca return 0; 7033c4d7559SDave Watson 7043c4d7559SDave Watson free_aead: 7053c4d7559SDave Watson crypto_free_aead(sw_ctx->aead_send); 7063c4d7559SDave Watson sw_ctx->aead_send = NULL; 7073c4d7559SDave Watson free_rec_seq: 7083c4d7559SDave Watson kfree(ctx->rec_seq); 7093c4d7559SDave Watson ctx->rec_seq = NULL; 7103c4d7559SDave Watson free_iv: 7113c4d7559SDave Watson kfree(ctx->iv); 7123c4d7559SDave Watson ctx->iv = NULL; 713cf6d43efSSabrina Dubroca free_priv: 714cf6d43efSSabrina Dubroca kfree(ctx->priv_ctx); 715cf6d43efSSabrina Dubroca ctx->priv_ctx = NULL; 7163c4d7559SDave Watson out: 7173c4d7559SDave Watson return rc; 7183c4d7559SDave Watson } 719