13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved. 53c4d7559SDave Watson * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved. 63c4d7559SDave Watson * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved. 73c4d7559SDave Watson * 83c4d7559SDave Watson * This software is available to you under a choice of one of two 93c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 103c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 113c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 123c4d7559SDave Watson * OpenIB.org BSD license below: 133c4d7559SDave Watson * 143c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 153c4d7559SDave Watson * without modification, are permitted provided that the following 163c4d7559SDave Watson * conditions are met: 173c4d7559SDave Watson * 183c4d7559SDave Watson * - Redistributions of source code must retain the above 193c4d7559SDave Watson * copyright notice, this list of conditions and the following 203c4d7559SDave Watson * disclaimer. 213c4d7559SDave Watson * 223c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 233c4d7559SDave Watson * copyright notice, this list of conditions and the following 243c4d7559SDave Watson * disclaimer in the documentation and/or other materials 253c4d7559SDave Watson * provided with the distribution. 263c4d7559SDave Watson * 273c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 283c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 293c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 303c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 313c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 323c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 333c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 343c4d7559SDave Watson * SOFTWARE. 353c4d7559SDave Watson */ 363c4d7559SDave Watson 373c4d7559SDave Watson #include <linux/module.h> 383c4d7559SDave Watson #include <crypto/aead.h> 393c4d7559SDave Watson 403c4d7559SDave Watson #include <net/tls.h> 413c4d7559SDave Watson 423c4d7559SDave Watson static inline void tls_make_aad(int recv, 433c4d7559SDave Watson char *buf, 443c4d7559SDave Watson size_t size, 453c4d7559SDave Watson char *record_sequence, 463c4d7559SDave Watson int record_sequence_size, 473c4d7559SDave Watson unsigned char record_type) 483c4d7559SDave Watson { 493c4d7559SDave Watson memcpy(buf, record_sequence, record_sequence_size); 503c4d7559SDave Watson 513c4d7559SDave Watson buf[8] = record_type; 523c4d7559SDave Watson buf[9] = TLS_1_2_VERSION_MAJOR; 533c4d7559SDave Watson buf[10] = TLS_1_2_VERSION_MINOR; 543c4d7559SDave Watson buf[11] = size >> 8; 553c4d7559SDave Watson buf[12] = size & 0xFF; 563c4d7559SDave Watson } 573c4d7559SDave Watson 583c4d7559SDave Watson static void trim_sg(struct sock *sk, struct scatterlist *sg, 593c4d7559SDave Watson int *sg_num_elem, unsigned int *sg_size, int target_size) 603c4d7559SDave Watson { 613c4d7559SDave Watson int i = *sg_num_elem - 1; 623c4d7559SDave Watson int trim = *sg_size - target_size; 633c4d7559SDave Watson 643c4d7559SDave Watson if (trim <= 0) { 653c4d7559SDave Watson WARN_ON(trim < 0); 663c4d7559SDave Watson return; 673c4d7559SDave Watson } 683c4d7559SDave Watson 693c4d7559SDave Watson *sg_size = target_size; 703c4d7559SDave Watson while (trim >= sg[i].length) { 713c4d7559SDave Watson trim -= sg[i].length; 723c4d7559SDave Watson sk_mem_uncharge(sk, sg[i].length); 733c4d7559SDave Watson put_page(sg_page(&sg[i])); 743c4d7559SDave Watson i--; 753c4d7559SDave Watson 763c4d7559SDave Watson if (i < 0) 773c4d7559SDave Watson goto out; 783c4d7559SDave Watson } 793c4d7559SDave Watson 803c4d7559SDave Watson sg[i].length -= trim; 813c4d7559SDave Watson sk_mem_uncharge(sk, trim); 823c4d7559SDave Watson 833c4d7559SDave Watson out: 843c4d7559SDave Watson *sg_num_elem = i + 1; 853c4d7559SDave Watson } 863c4d7559SDave Watson 873c4d7559SDave Watson static void trim_both_sgl(struct sock *sk, int target_size) 883c4d7559SDave Watson { 893c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 903c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 913c4d7559SDave Watson 923c4d7559SDave Watson trim_sg(sk, ctx->sg_plaintext_data, 933c4d7559SDave Watson &ctx->sg_plaintext_num_elem, 943c4d7559SDave Watson &ctx->sg_plaintext_size, 953c4d7559SDave Watson target_size); 963c4d7559SDave Watson 973c4d7559SDave Watson if (target_size > 0) 983c4d7559SDave Watson target_size += tls_ctx->overhead_size; 993c4d7559SDave Watson 1003c4d7559SDave Watson trim_sg(sk, ctx->sg_encrypted_data, 1013c4d7559SDave Watson &ctx->sg_encrypted_num_elem, 1023c4d7559SDave Watson &ctx->sg_encrypted_size, 1033c4d7559SDave Watson target_size); 1043c4d7559SDave Watson } 1053c4d7559SDave Watson 1063c4d7559SDave Watson static int alloc_sg(struct sock *sk, int len, struct scatterlist *sg, 1073c4d7559SDave Watson int *sg_num_elem, unsigned int *sg_size, 1083c4d7559SDave Watson int first_coalesce) 1093c4d7559SDave Watson { 1103c4d7559SDave Watson struct page_frag *pfrag; 1113c4d7559SDave Watson unsigned int size = *sg_size; 1123c4d7559SDave Watson int num_elem = *sg_num_elem, use = 0, rc = 0; 1133c4d7559SDave Watson struct scatterlist *sge; 1143c4d7559SDave Watson unsigned int orig_offset; 1153c4d7559SDave Watson 1163c4d7559SDave Watson len -= size; 1173c4d7559SDave Watson pfrag = sk_page_frag(sk); 1183c4d7559SDave Watson 1193c4d7559SDave Watson while (len > 0) { 1203c4d7559SDave Watson if (!sk_page_frag_refill(sk, pfrag)) { 1213c4d7559SDave Watson rc = -ENOMEM; 1223c4d7559SDave Watson goto out; 1233c4d7559SDave Watson } 1243c4d7559SDave Watson 1253c4d7559SDave Watson use = min_t(int, len, pfrag->size - pfrag->offset); 1263c4d7559SDave Watson 1273c4d7559SDave Watson if (!sk_wmem_schedule(sk, use)) { 1283c4d7559SDave Watson rc = -ENOMEM; 1293c4d7559SDave Watson goto out; 1303c4d7559SDave Watson } 1313c4d7559SDave Watson 1323c4d7559SDave Watson sk_mem_charge(sk, use); 1333c4d7559SDave Watson size += use; 1343c4d7559SDave Watson orig_offset = pfrag->offset; 1353c4d7559SDave Watson pfrag->offset += use; 1363c4d7559SDave Watson 1373c4d7559SDave Watson sge = sg + num_elem - 1; 1383c4d7559SDave Watson if (num_elem > first_coalesce && sg_page(sg) == pfrag->page && 1393c4d7559SDave Watson sg->offset + sg->length == orig_offset) { 1403c4d7559SDave Watson sg->length += use; 1413c4d7559SDave Watson } else { 1423c4d7559SDave Watson sge++; 1433c4d7559SDave Watson sg_unmark_end(sge); 1443c4d7559SDave Watson sg_set_page(sge, pfrag->page, use, orig_offset); 1453c4d7559SDave Watson get_page(pfrag->page); 1463c4d7559SDave Watson ++num_elem; 1473c4d7559SDave Watson if (num_elem == MAX_SKB_FRAGS) { 1483c4d7559SDave Watson rc = -ENOSPC; 1493c4d7559SDave Watson break; 1503c4d7559SDave Watson } 1513c4d7559SDave Watson } 1523c4d7559SDave Watson 1533c4d7559SDave Watson len -= use; 1543c4d7559SDave Watson } 1553c4d7559SDave Watson goto out; 1563c4d7559SDave Watson 1573c4d7559SDave Watson out: 1583c4d7559SDave Watson *sg_size = size; 1593c4d7559SDave Watson *sg_num_elem = num_elem; 1603c4d7559SDave Watson return rc; 1613c4d7559SDave Watson } 1623c4d7559SDave Watson 1633c4d7559SDave Watson static int alloc_encrypted_sg(struct sock *sk, int len) 1643c4d7559SDave Watson { 1653c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1663c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 1673c4d7559SDave Watson int rc = 0; 1683c4d7559SDave Watson 1693c4d7559SDave Watson rc = alloc_sg(sk, len, ctx->sg_encrypted_data, 1703c4d7559SDave Watson &ctx->sg_encrypted_num_elem, &ctx->sg_encrypted_size, 0); 1713c4d7559SDave Watson 1723c4d7559SDave Watson return rc; 1733c4d7559SDave Watson } 1743c4d7559SDave Watson 1753c4d7559SDave Watson static int alloc_plaintext_sg(struct sock *sk, int len) 1763c4d7559SDave Watson { 1773c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1783c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 1793c4d7559SDave Watson int rc = 0; 1803c4d7559SDave Watson 1813c4d7559SDave Watson rc = alloc_sg(sk, len, ctx->sg_plaintext_data, 1823c4d7559SDave Watson &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size, 1833c4d7559SDave Watson tls_ctx->pending_open_record_frags); 1843c4d7559SDave Watson 1853c4d7559SDave Watson return rc; 1863c4d7559SDave Watson } 1873c4d7559SDave Watson 1883c4d7559SDave Watson static void free_sg(struct sock *sk, struct scatterlist *sg, 1893c4d7559SDave Watson int *sg_num_elem, unsigned int *sg_size) 1903c4d7559SDave Watson { 1913c4d7559SDave Watson int i, n = *sg_num_elem; 1923c4d7559SDave Watson 1933c4d7559SDave Watson for (i = 0; i < n; ++i) { 1943c4d7559SDave Watson sk_mem_uncharge(sk, sg[i].length); 1953c4d7559SDave Watson put_page(sg_page(&sg[i])); 1963c4d7559SDave Watson } 1973c4d7559SDave Watson *sg_num_elem = 0; 1983c4d7559SDave Watson *sg_size = 0; 1993c4d7559SDave Watson } 2003c4d7559SDave Watson 2013c4d7559SDave Watson static void tls_free_both_sg(struct sock *sk) 2023c4d7559SDave Watson { 2033c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2043c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 2053c4d7559SDave Watson 2063c4d7559SDave Watson free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem, 2073c4d7559SDave Watson &ctx->sg_encrypted_size); 2083c4d7559SDave Watson 2093c4d7559SDave Watson free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, 2103c4d7559SDave Watson &ctx->sg_plaintext_size); 2113c4d7559SDave Watson } 2123c4d7559SDave Watson 2133c4d7559SDave Watson static int tls_do_encryption(struct tls_context *tls_ctx, 2143c4d7559SDave Watson struct tls_sw_context *ctx, size_t data_len, 2153c4d7559SDave Watson gfp_t flags) 2163c4d7559SDave Watson { 2173c4d7559SDave Watson unsigned int req_size = sizeof(struct aead_request) + 2183c4d7559SDave Watson crypto_aead_reqsize(ctx->aead_send); 2193c4d7559SDave Watson struct aead_request *aead_req; 2203c4d7559SDave Watson int rc; 2213c4d7559SDave Watson 2223c4d7559SDave Watson aead_req = kmalloc(req_size, flags); 2233c4d7559SDave Watson if (!aead_req) 2243c4d7559SDave Watson return -ENOMEM; 2253c4d7559SDave Watson 2263c4d7559SDave Watson ctx->sg_encrypted_data[0].offset += tls_ctx->prepend_size; 2273c4d7559SDave Watson ctx->sg_encrypted_data[0].length -= tls_ctx->prepend_size; 2283c4d7559SDave Watson 2293c4d7559SDave Watson aead_request_set_tfm(aead_req, ctx->aead_send); 2303c4d7559SDave Watson aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); 2313c4d7559SDave Watson aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out, 2323c4d7559SDave Watson data_len, tls_ctx->iv); 2333c4d7559SDave Watson rc = crypto_aead_encrypt(aead_req); 2343c4d7559SDave Watson 2353c4d7559SDave Watson ctx->sg_encrypted_data[0].offset -= tls_ctx->prepend_size; 2363c4d7559SDave Watson ctx->sg_encrypted_data[0].length += tls_ctx->prepend_size; 2373c4d7559SDave Watson 2383c4d7559SDave Watson kfree(aead_req); 2393c4d7559SDave Watson return rc; 2403c4d7559SDave Watson } 2413c4d7559SDave Watson 2423c4d7559SDave Watson static int tls_push_record(struct sock *sk, int flags, 2433c4d7559SDave Watson unsigned char record_type) 2443c4d7559SDave Watson { 2453c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2463c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 2473c4d7559SDave Watson int rc; 2483c4d7559SDave Watson 2493c4d7559SDave Watson sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1); 2503c4d7559SDave Watson sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1); 2513c4d7559SDave Watson 2523c4d7559SDave Watson tls_make_aad(0, ctx->aad_space, ctx->sg_plaintext_size, 2533c4d7559SDave Watson tls_ctx->rec_seq, tls_ctx->rec_seq_size, 2543c4d7559SDave Watson record_type); 2553c4d7559SDave Watson 2563c4d7559SDave Watson tls_fill_prepend(tls_ctx, 2573c4d7559SDave Watson page_address(sg_page(&ctx->sg_encrypted_data[0])) + 2583c4d7559SDave Watson ctx->sg_encrypted_data[0].offset, 2593c4d7559SDave Watson ctx->sg_plaintext_size, record_type); 2603c4d7559SDave Watson 2613c4d7559SDave Watson tls_ctx->pending_open_record_frags = 0; 2623c4d7559SDave Watson set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags); 2633c4d7559SDave Watson 2643c4d7559SDave Watson rc = tls_do_encryption(tls_ctx, ctx, ctx->sg_plaintext_size, 2653c4d7559SDave Watson sk->sk_allocation); 2663c4d7559SDave Watson if (rc < 0) { 2673c4d7559SDave Watson /* If we are called from write_space and 2683c4d7559SDave Watson * we fail, we need to set this SOCK_NOSPACE 2693c4d7559SDave Watson * to trigger another write_space in the future. 2703c4d7559SDave Watson */ 2713c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2723c4d7559SDave Watson return rc; 2733c4d7559SDave Watson } 2743c4d7559SDave Watson 2753c4d7559SDave Watson free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem, 2763c4d7559SDave Watson &ctx->sg_plaintext_size); 2773c4d7559SDave Watson 2783c4d7559SDave Watson ctx->sg_encrypted_num_elem = 0; 2793c4d7559SDave Watson ctx->sg_encrypted_size = 0; 2803c4d7559SDave Watson 2813c4d7559SDave Watson /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */ 2823c4d7559SDave Watson rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags); 2833c4d7559SDave Watson if (rc < 0 && rc != -EAGAIN) 2843c4d7559SDave Watson tls_err_abort(sk); 2853c4d7559SDave Watson 2863c4d7559SDave Watson tls_advance_record_sn(sk, tls_ctx); 2873c4d7559SDave Watson return rc; 2883c4d7559SDave Watson } 2893c4d7559SDave Watson 2903c4d7559SDave Watson static int tls_sw_push_pending_record(struct sock *sk, int flags) 2913c4d7559SDave Watson { 2923c4d7559SDave Watson return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA); 2933c4d7559SDave Watson } 2943c4d7559SDave Watson 2953c4d7559SDave Watson static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from, 2963c4d7559SDave Watson int length) 2973c4d7559SDave Watson { 2983c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2993c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 3003c4d7559SDave Watson struct page *pages[MAX_SKB_FRAGS]; 3013c4d7559SDave Watson 3023c4d7559SDave Watson size_t offset; 3033c4d7559SDave Watson ssize_t copied, use; 3043c4d7559SDave Watson int i = 0; 3053c4d7559SDave Watson unsigned int size = ctx->sg_plaintext_size; 3063c4d7559SDave Watson int num_elem = ctx->sg_plaintext_num_elem; 3073c4d7559SDave Watson int rc = 0; 3083c4d7559SDave Watson int maxpages; 3093c4d7559SDave Watson 3103c4d7559SDave Watson while (length > 0) { 3113c4d7559SDave Watson i = 0; 3123c4d7559SDave Watson maxpages = ARRAY_SIZE(ctx->sg_plaintext_data) - num_elem; 3133c4d7559SDave Watson if (maxpages == 0) { 3143c4d7559SDave Watson rc = -EFAULT; 3153c4d7559SDave Watson goto out; 3163c4d7559SDave Watson } 3173c4d7559SDave Watson copied = iov_iter_get_pages(from, pages, 3183c4d7559SDave Watson length, 3193c4d7559SDave Watson maxpages, &offset); 3203c4d7559SDave Watson if (copied <= 0) { 3213c4d7559SDave Watson rc = -EFAULT; 3223c4d7559SDave Watson goto out; 3233c4d7559SDave Watson } 3243c4d7559SDave Watson 3253c4d7559SDave Watson iov_iter_advance(from, copied); 3263c4d7559SDave Watson 3273c4d7559SDave Watson length -= copied; 3283c4d7559SDave Watson size += copied; 3293c4d7559SDave Watson while (copied) { 3303c4d7559SDave Watson use = min_t(int, copied, PAGE_SIZE - offset); 3313c4d7559SDave Watson 3323c4d7559SDave Watson sg_set_page(&ctx->sg_plaintext_data[num_elem], 3333c4d7559SDave Watson pages[i], use, offset); 3343c4d7559SDave Watson sg_unmark_end(&ctx->sg_plaintext_data[num_elem]); 3353c4d7559SDave Watson sk_mem_charge(sk, use); 3363c4d7559SDave Watson 3373c4d7559SDave Watson offset = 0; 3383c4d7559SDave Watson copied -= use; 3393c4d7559SDave Watson 3403c4d7559SDave Watson ++i; 3413c4d7559SDave Watson ++num_elem; 3423c4d7559SDave Watson } 3433c4d7559SDave Watson } 3443c4d7559SDave Watson 3453c4d7559SDave Watson out: 3463c4d7559SDave Watson ctx->sg_plaintext_size = size; 3473c4d7559SDave Watson ctx->sg_plaintext_num_elem = num_elem; 3483c4d7559SDave Watson return rc; 3493c4d7559SDave Watson } 3503c4d7559SDave Watson 3513c4d7559SDave Watson static int memcopy_from_iter(struct sock *sk, struct iov_iter *from, 3523c4d7559SDave Watson int bytes) 3533c4d7559SDave Watson { 3543c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 3553c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 3563c4d7559SDave Watson struct scatterlist *sg = ctx->sg_plaintext_data; 3573c4d7559SDave Watson int copy, i, rc = 0; 3583c4d7559SDave Watson 3593c4d7559SDave Watson for (i = tls_ctx->pending_open_record_frags; 3603c4d7559SDave Watson i < ctx->sg_plaintext_num_elem; ++i) { 3613c4d7559SDave Watson copy = sg[i].length; 3623c4d7559SDave Watson if (copy_from_iter( 3633c4d7559SDave Watson page_address(sg_page(&sg[i])) + sg[i].offset, 3643c4d7559SDave Watson copy, from) != copy) { 3653c4d7559SDave Watson rc = -EFAULT; 3663c4d7559SDave Watson goto out; 3673c4d7559SDave Watson } 3683c4d7559SDave Watson bytes -= copy; 3693c4d7559SDave Watson 3703c4d7559SDave Watson ++tls_ctx->pending_open_record_frags; 3713c4d7559SDave Watson 3723c4d7559SDave Watson if (!bytes) 3733c4d7559SDave Watson break; 3743c4d7559SDave Watson } 3753c4d7559SDave Watson 3763c4d7559SDave Watson out: 3773c4d7559SDave Watson return rc; 3783c4d7559SDave Watson } 3793c4d7559SDave Watson 3803c4d7559SDave Watson int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 3813c4d7559SDave Watson { 3823c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 3833c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 3843c4d7559SDave Watson int ret = 0; 3853c4d7559SDave Watson int required_size; 3863c4d7559SDave Watson long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 3873c4d7559SDave Watson bool eor = !(msg->msg_flags & MSG_MORE); 3883c4d7559SDave Watson size_t try_to_copy, copied = 0; 3893c4d7559SDave Watson unsigned char record_type = TLS_RECORD_TYPE_DATA; 3903c4d7559SDave Watson int record_room; 3913c4d7559SDave Watson bool full_record; 3923c4d7559SDave Watson int orig_size; 3933c4d7559SDave Watson 3943c4d7559SDave Watson if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 3953c4d7559SDave Watson return -ENOTSUPP; 3963c4d7559SDave Watson 3973c4d7559SDave Watson lock_sock(sk); 3983c4d7559SDave Watson 3993c4d7559SDave Watson if (tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo)) 4003c4d7559SDave Watson goto send_end; 4013c4d7559SDave Watson 4023c4d7559SDave Watson if (unlikely(msg->msg_controllen)) { 4033c4d7559SDave Watson ret = tls_proccess_cmsg(sk, msg, &record_type); 4043c4d7559SDave Watson if (ret) 4053c4d7559SDave Watson goto send_end; 4063c4d7559SDave Watson } 4073c4d7559SDave Watson 4083c4d7559SDave Watson while (msg_data_left(msg)) { 4093c4d7559SDave Watson if (sk->sk_err) { 4103c4d7559SDave Watson ret = sk->sk_err; 4113c4d7559SDave Watson goto send_end; 4123c4d7559SDave Watson } 4133c4d7559SDave Watson 4143c4d7559SDave Watson orig_size = ctx->sg_plaintext_size; 4153c4d7559SDave Watson full_record = false; 4163c4d7559SDave Watson try_to_copy = msg_data_left(msg); 4173c4d7559SDave Watson record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size; 4183c4d7559SDave Watson if (try_to_copy >= record_room) { 4193c4d7559SDave Watson try_to_copy = record_room; 4203c4d7559SDave Watson full_record = true; 4213c4d7559SDave Watson } 4223c4d7559SDave Watson 4233c4d7559SDave Watson required_size = ctx->sg_plaintext_size + try_to_copy + 4243c4d7559SDave Watson tls_ctx->overhead_size; 4253c4d7559SDave Watson 4263c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 4273c4d7559SDave Watson goto wait_for_sndbuf; 4283c4d7559SDave Watson alloc_encrypted: 4293c4d7559SDave Watson ret = alloc_encrypted_sg(sk, required_size); 4303c4d7559SDave Watson if (ret) { 4313c4d7559SDave Watson if (ret != -ENOSPC) 4323c4d7559SDave Watson goto wait_for_memory; 4333c4d7559SDave Watson 4343c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 4353c4d7559SDave Watson * actually allocated. The difference is due 4363c4d7559SDave Watson * to max sg elements limit 4373c4d7559SDave Watson */ 4383c4d7559SDave Watson try_to_copy -= required_size - ctx->sg_encrypted_size; 4393c4d7559SDave Watson full_record = true; 4403c4d7559SDave Watson } 4413c4d7559SDave Watson 4423c4d7559SDave Watson if (full_record || eor) { 4433c4d7559SDave Watson ret = zerocopy_from_iter(sk, &msg->msg_iter, 4443c4d7559SDave Watson try_to_copy); 4453c4d7559SDave Watson if (ret) 4463c4d7559SDave Watson goto fallback_to_reg_send; 4473c4d7559SDave Watson 4483c4d7559SDave Watson copied += try_to_copy; 4493c4d7559SDave Watson ret = tls_push_record(sk, msg->msg_flags, record_type); 4503c4d7559SDave Watson if (!ret) 4513c4d7559SDave Watson continue; 4523c4d7559SDave Watson if (ret == -EAGAIN) 4533c4d7559SDave Watson goto send_end; 4543c4d7559SDave Watson 4553c4d7559SDave Watson copied -= try_to_copy; 4563c4d7559SDave Watson fallback_to_reg_send: 4573c4d7559SDave Watson iov_iter_revert(&msg->msg_iter, 4583c4d7559SDave Watson ctx->sg_plaintext_size - orig_size); 4593c4d7559SDave Watson trim_sg(sk, ctx->sg_plaintext_data, 4603c4d7559SDave Watson &ctx->sg_plaintext_num_elem, 4613c4d7559SDave Watson &ctx->sg_plaintext_size, 4623c4d7559SDave Watson orig_size); 4633c4d7559SDave Watson } 4643c4d7559SDave Watson 4653c4d7559SDave Watson required_size = ctx->sg_plaintext_size + try_to_copy; 4663c4d7559SDave Watson alloc_plaintext: 4673c4d7559SDave Watson ret = alloc_plaintext_sg(sk, required_size); 4683c4d7559SDave Watson if (ret) { 4693c4d7559SDave Watson if (ret != -ENOSPC) 4703c4d7559SDave Watson goto wait_for_memory; 4713c4d7559SDave Watson 4723c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 4733c4d7559SDave Watson * actually allocated. The difference is due 4743c4d7559SDave Watson * to max sg elements limit 4753c4d7559SDave Watson */ 4763c4d7559SDave Watson try_to_copy -= required_size - ctx->sg_plaintext_size; 4773c4d7559SDave Watson full_record = true; 4783c4d7559SDave Watson 4793c4d7559SDave Watson trim_sg(sk, ctx->sg_encrypted_data, 4803c4d7559SDave Watson &ctx->sg_encrypted_num_elem, 4813c4d7559SDave Watson &ctx->sg_encrypted_size, 4823c4d7559SDave Watson ctx->sg_plaintext_size + 4833c4d7559SDave Watson tls_ctx->overhead_size); 4843c4d7559SDave Watson } 4853c4d7559SDave Watson 4863c4d7559SDave Watson ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy); 4873c4d7559SDave Watson if (ret) 4883c4d7559SDave Watson goto trim_sgl; 4893c4d7559SDave Watson 4903c4d7559SDave Watson copied += try_to_copy; 4913c4d7559SDave Watson if (full_record || eor) { 4923c4d7559SDave Watson push_record: 4933c4d7559SDave Watson ret = tls_push_record(sk, msg->msg_flags, record_type); 4943c4d7559SDave Watson if (ret) { 4953c4d7559SDave Watson if (ret == -ENOMEM) 4963c4d7559SDave Watson goto wait_for_memory; 4973c4d7559SDave Watson 4983c4d7559SDave Watson goto send_end; 4993c4d7559SDave Watson } 5003c4d7559SDave Watson } 5013c4d7559SDave Watson 5023c4d7559SDave Watson continue; 5033c4d7559SDave Watson 5043c4d7559SDave Watson wait_for_sndbuf: 5053c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 5063c4d7559SDave Watson wait_for_memory: 5073c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 5083c4d7559SDave Watson if (ret) { 5093c4d7559SDave Watson trim_sgl: 5103c4d7559SDave Watson trim_both_sgl(sk, orig_size); 5113c4d7559SDave Watson goto send_end; 5123c4d7559SDave Watson } 5133c4d7559SDave Watson 5143c4d7559SDave Watson if (tls_is_pending_closed_record(tls_ctx)) 5153c4d7559SDave Watson goto push_record; 5163c4d7559SDave Watson 5173c4d7559SDave Watson if (ctx->sg_encrypted_size < required_size) 5183c4d7559SDave Watson goto alloc_encrypted; 5193c4d7559SDave Watson 5203c4d7559SDave Watson goto alloc_plaintext; 5213c4d7559SDave Watson } 5223c4d7559SDave Watson 5233c4d7559SDave Watson send_end: 5243c4d7559SDave Watson ret = sk_stream_error(sk, msg->msg_flags, ret); 5253c4d7559SDave Watson 5263c4d7559SDave Watson release_sock(sk); 5273c4d7559SDave Watson return copied ? copied : ret; 5283c4d7559SDave Watson } 5293c4d7559SDave Watson 5303c4d7559SDave Watson int tls_sw_sendpage(struct sock *sk, struct page *page, 5313c4d7559SDave Watson int offset, size_t size, int flags) 5323c4d7559SDave Watson { 5333c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 5343c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 5353c4d7559SDave Watson int ret = 0; 5363c4d7559SDave Watson long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 5373c4d7559SDave Watson bool eor; 5383c4d7559SDave Watson size_t orig_size = size; 5393c4d7559SDave Watson unsigned char record_type = TLS_RECORD_TYPE_DATA; 5403c4d7559SDave Watson struct scatterlist *sg; 5413c4d7559SDave Watson bool full_record; 5423c4d7559SDave Watson int record_room; 5433c4d7559SDave Watson 5443c4d7559SDave Watson if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 5453c4d7559SDave Watson MSG_SENDPAGE_NOTLAST)) 5463c4d7559SDave Watson return -ENOTSUPP; 5473c4d7559SDave Watson 5483c4d7559SDave Watson /* No MSG_EOR from splice, only look at MSG_MORE */ 5493c4d7559SDave Watson eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST)); 5503c4d7559SDave Watson 5513c4d7559SDave Watson lock_sock(sk); 5523c4d7559SDave Watson 5533c4d7559SDave Watson sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 5543c4d7559SDave Watson 5553c4d7559SDave Watson if (tls_complete_pending_work(sk, tls_ctx, flags, &timeo)) 5563c4d7559SDave Watson goto sendpage_end; 5573c4d7559SDave Watson 5583c4d7559SDave Watson /* Call the sk_stream functions to manage the sndbuf mem. */ 5593c4d7559SDave Watson while (size > 0) { 5603c4d7559SDave Watson size_t copy, required_size; 5613c4d7559SDave Watson 5623c4d7559SDave Watson if (sk->sk_err) { 5633c4d7559SDave Watson ret = sk->sk_err; 5643c4d7559SDave Watson goto sendpage_end; 5653c4d7559SDave Watson } 5663c4d7559SDave Watson 5673c4d7559SDave Watson full_record = false; 5683c4d7559SDave Watson record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size; 5693c4d7559SDave Watson copy = size; 5703c4d7559SDave Watson if (copy >= record_room) { 5713c4d7559SDave Watson copy = record_room; 5723c4d7559SDave Watson full_record = true; 5733c4d7559SDave Watson } 5743c4d7559SDave Watson required_size = ctx->sg_plaintext_size + copy + 5753c4d7559SDave Watson tls_ctx->overhead_size; 5763c4d7559SDave Watson 5773c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 5783c4d7559SDave Watson goto wait_for_sndbuf; 5793c4d7559SDave Watson alloc_payload: 5803c4d7559SDave Watson ret = alloc_encrypted_sg(sk, required_size); 5813c4d7559SDave Watson if (ret) { 5823c4d7559SDave Watson if (ret != -ENOSPC) 5833c4d7559SDave Watson goto wait_for_memory; 5843c4d7559SDave Watson 5853c4d7559SDave Watson /* Adjust copy according to the amount that was 5863c4d7559SDave Watson * actually allocated. The difference is due 5873c4d7559SDave Watson * to max sg elements limit 5883c4d7559SDave Watson */ 5893c4d7559SDave Watson copy -= required_size - ctx->sg_plaintext_size; 5903c4d7559SDave Watson full_record = true; 5913c4d7559SDave Watson } 5923c4d7559SDave Watson 5933c4d7559SDave Watson get_page(page); 5943c4d7559SDave Watson sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem; 5953c4d7559SDave Watson sg_set_page(sg, page, copy, offset); 5963c4d7559SDave Watson ctx->sg_plaintext_num_elem++; 5973c4d7559SDave Watson 5983c4d7559SDave Watson sk_mem_charge(sk, copy); 5993c4d7559SDave Watson offset += copy; 6003c4d7559SDave Watson size -= copy; 6013c4d7559SDave Watson ctx->sg_plaintext_size += copy; 6023c4d7559SDave Watson tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem; 6033c4d7559SDave Watson 6043c4d7559SDave Watson if (full_record || eor || 6053c4d7559SDave Watson ctx->sg_plaintext_num_elem == 6063c4d7559SDave Watson ARRAY_SIZE(ctx->sg_plaintext_data)) { 6073c4d7559SDave Watson push_record: 6083c4d7559SDave Watson ret = tls_push_record(sk, flags, record_type); 6093c4d7559SDave Watson if (ret) { 6103c4d7559SDave Watson if (ret == -ENOMEM) 6113c4d7559SDave Watson goto wait_for_memory; 6123c4d7559SDave Watson 6133c4d7559SDave Watson goto sendpage_end; 6143c4d7559SDave Watson } 6153c4d7559SDave Watson } 6163c4d7559SDave Watson continue; 6173c4d7559SDave Watson wait_for_sndbuf: 6183c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 6193c4d7559SDave Watson wait_for_memory: 6203c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 6213c4d7559SDave Watson if (ret) { 6223c4d7559SDave Watson trim_both_sgl(sk, ctx->sg_plaintext_size); 6233c4d7559SDave Watson goto sendpage_end; 6243c4d7559SDave Watson } 6253c4d7559SDave Watson 6263c4d7559SDave Watson if (tls_is_pending_closed_record(tls_ctx)) 6273c4d7559SDave Watson goto push_record; 6283c4d7559SDave Watson 6293c4d7559SDave Watson goto alloc_payload; 6303c4d7559SDave Watson } 6313c4d7559SDave Watson 6323c4d7559SDave Watson sendpage_end: 6333c4d7559SDave Watson if (orig_size > size) 6343c4d7559SDave Watson ret = orig_size - size; 6353c4d7559SDave Watson else 6363c4d7559SDave Watson ret = sk_stream_error(sk, flags, ret); 6373c4d7559SDave Watson 6383c4d7559SDave Watson release_sock(sk); 6393c4d7559SDave Watson return ret; 6403c4d7559SDave Watson } 6413c4d7559SDave Watson 642a5135676STobias Klauser static void tls_sw_free_resources(struct sock *sk) 6433c4d7559SDave Watson { 6443c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 6453c4d7559SDave Watson struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); 6463c4d7559SDave Watson 6473c4d7559SDave Watson if (ctx->aead_send) 6483c4d7559SDave Watson crypto_free_aead(ctx->aead_send); 6493c4d7559SDave Watson 6503c4d7559SDave Watson tls_free_both_sg(sk); 6513c4d7559SDave Watson 6523c4d7559SDave Watson kfree(ctx); 6533c4d7559SDave Watson } 6543c4d7559SDave Watson 6553c4d7559SDave Watson int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx) 6563c4d7559SDave Watson { 6573c4d7559SDave Watson char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE]; 6583c4d7559SDave Watson struct tls_crypto_info *crypto_info; 6593c4d7559SDave Watson struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; 6603c4d7559SDave Watson struct tls_sw_context *sw_ctx; 6613c4d7559SDave Watson u16 nonce_size, tag_size, iv_size, rec_seq_size; 6623c4d7559SDave Watson char *iv, *rec_seq; 6633c4d7559SDave Watson int rc = 0; 6643c4d7559SDave Watson 6653c4d7559SDave Watson if (!ctx) { 6663c4d7559SDave Watson rc = -EINVAL; 6673c4d7559SDave Watson goto out; 6683c4d7559SDave Watson } 6693c4d7559SDave Watson 6703c4d7559SDave Watson if (ctx->priv_ctx) { 6713c4d7559SDave Watson rc = -EEXIST; 6723c4d7559SDave Watson goto out; 6733c4d7559SDave Watson } 6743c4d7559SDave Watson 6753c4d7559SDave Watson sw_ctx = kzalloc(sizeof(*sw_ctx), GFP_KERNEL); 6763c4d7559SDave Watson if (!sw_ctx) { 6773c4d7559SDave Watson rc = -ENOMEM; 6783c4d7559SDave Watson goto out; 6793c4d7559SDave Watson } 6803c4d7559SDave Watson 6813c4d7559SDave Watson ctx->priv_ctx = (struct tls_offload_context *)sw_ctx; 6823c4d7559SDave Watson ctx->free_resources = tls_sw_free_resources; 6833c4d7559SDave Watson 6843c4d7559SDave Watson crypto_info = &ctx->crypto_send; 6853c4d7559SDave Watson switch (crypto_info->cipher_type) { 6863c4d7559SDave Watson case TLS_CIPHER_AES_GCM_128: { 6873c4d7559SDave Watson nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 6883c4d7559SDave Watson tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; 6893c4d7559SDave Watson iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 6903c4d7559SDave Watson iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; 6913c4d7559SDave Watson rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; 6923c4d7559SDave Watson rec_seq = 6933c4d7559SDave Watson ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; 6943c4d7559SDave Watson gcm_128_info = 6953c4d7559SDave Watson (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 6963c4d7559SDave Watson break; 6973c4d7559SDave Watson } 6983c4d7559SDave Watson default: 6993c4d7559SDave Watson rc = -EINVAL; 7003c4d7559SDave Watson goto out; 7013c4d7559SDave Watson } 7023c4d7559SDave Watson 7033c4d7559SDave Watson ctx->prepend_size = TLS_HEADER_SIZE + nonce_size; 7043c4d7559SDave Watson ctx->tag_size = tag_size; 7053c4d7559SDave Watson ctx->overhead_size = ctx->prepend_size + ctx->tag_size; 7063c4d7559SDave Watson ctx->iv_size = iv_size; 7073c4d7559SDave Watson ctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 7083c4d7559SDave Watson GFP_KERNEL); 7093c4d7559SDave Watson if (!ctx->iv) { 7103c4d7559SDave Watson rc = -ENOMEM; 7113c4d7559SDave Watson goto out; 7123c4d7559SDave Watson } 7133c4d7559SDave Watson memcpy(ctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 7143c4d7559SDave Watson memcpy(ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size); 7153c4d7559SDave Watson ctx->rec_seq_size = rec_seq_size; 7163c4d7559SDave Watson ctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL); 7173c4d7559SDave Watson if (!ctx->rec_seq) { 7183c4d7559SDave Watson rc = -ENOMEM; 7193c4d7559SDave Watson goto free_iv; 7203c4d7559SDave Watson } 7213c4d7559SDave Watson memcpy(ctx->rec_seq, rec_seq, rec_seq_size); 7223c4d7559SDave Watson 7233c4d7559SDave Watson sg_init_table(sw_ctx->sg_encrypted_data, 7243c4d7559SDave Watson ARRAY_SIZE(sw_ctx->sg_encrypted_data)); 7253c4d7559SDave Watson sg_init_table(sw_ctx->sg_plaintext_data, 7263c4d7559SDave Watson ARRAY_SIZE(sw_ctx->sg_plaintext_data)); 7273c4d7559SDave Watson 7283c4d7559SDave Watson sg_init_table(sw_ctx->sg_aead_in, 2); 7293c4d7559SDave Watson sg_set_buf(&sw_ctx->sg_aead_in[0], sw_ctx->aad_space, 7303c4d7559SDave Watson sizeof(sw_ctx->aad_space)); 7313c4d7559SDave Watson sg_unmark_end(&sw_ctx->sg_aead_in[1]); 7323c4d7559SDave Watson sg_chain(sw_ctx->sg_aead_in, 2, sw_ctx->sg_plaintext_data); 7333c4d7559SDave Watson sg_init_table(sw_ctx->sg_aead_out, 2); 7343c4d7559SDave Watson sg_set_buf(&sw_ctx->sg_aead_out[0], sw_ctx->aad_space, 7353c4d7559SDave Watson sizeof(sw_ctx->aad_space)); 7363c4d7559SDave Watson sg_unmark_end(&sw_ctx->sg_aead_out[1]); 7373c4d7559SDave Watson sg_chain(sw_ctx->sg_aead_out, 2, sw_ctx->sg_encrypted_data); 7383c4d7559SDave Watson 7393c4d7559SDave Watson if (!sw_ctx->aead_send) { 7403c4d7559SDave Watson sw_ctx->aead_send = crypto_alloc_aead("gcm(aes)", 0, 0); 7413c4d7559SDave Watson if (IS_ERR(sw_ctx->aead_send)) { 7423c4d7559SDave Watson rc = PTR_ERR(sw_ctx->aead_send); 7433c4d7559SDave Watson sw_ctx->aead_send = NULL; 7443c4d7559SDave Watson goto free_rec_seq; 7453c4d7559SDave Watson } 7463c4d7559SDave Watson } 7473c4d7559SDave Watson 7483c4d7559SDave Watson ctx->push_pending_record = tls_sw_push_pending_record; 7493c4d7559SDave Watson 7503c4d7559SDave Watson memcpy(keyval, gcm_128_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE); 7513c4d7559SDave Watson 7523c4d7559SDave Watson rc = crypto_aead_setkey(sw_ctx->aead_send, keyval, 7533c4d7559SDave Watson TLS_CIPHER_AES_GCM_128_KEY_SIZE); 7543c4d7559SDave Watson if (rc) 7553c4d7559SDave Watson goto free_aead; 7563c4d7559SDave Watson 7573c4d7559SDave Watson rc = crypto_aead_setauthsize(sw_ctx->aead_send, ctx->tag_size); 7583c4d7559SDave Watson if (!rc) 7593c4d7559SDave Watson goto out; 7603c4d7559SDave Watson 7613c4d7559SDave Watson free_aead: 7623c4d7559SDave Watson crypto_free_aead(sw_ctx->aead_send); 7633c4d7559SDave Watson sw_ctx->aead_send = NULL; 7643c4d7559SDave Watson free_rec_seq: 7653c4d7559SDave Watson kfree(ctx->rec_seq); 7663c4d7559SDave Watson ctx->rec_seq = NULL; 7673c4d7559SDave Watson free_iv: 7683c4d7559SDave Watson kfree(ctx->iv); 7693c4d7559SDave Watson ctx->iv = NULL; 7703c4d7559SDave Watson out: 7713c4d7559SDave Watson return rc; 7723c4d7559SDave Watson } 773