13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved. 53c4d7559SDave Watson * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved. 63c4d7559SDave Watson * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved. 7d3b18ad3SJohn Fastabend * Copyright (c) 2018, Covalent IO, Inc. http://covalent.io 83c4d7559SDave Watson * 93c4d7559SDave Watson * This software is available to you under a choice of one of two 103c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 113c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 123c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 133c4d7559SDave Watson * OpenIB.org BSD license below: 143c4d7559SDave Watson * 153c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 163c4d7559SDave Watson * without modification, are permitted provided that the following 173c4d7559SDave Watson * conditions are met: 183c4d7559SDave Watson * 193c4d7559SDave Watson * - Redistributions of source code must retain the above 203c4d7559SDave Watson * copyright notice, this list of conditions and the following 213c4d7559SDave Watson * disclaimer. 223c4d7559SDave Watson * 233c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 243c4d7559SDave Watson * copyright notice, this list of conditions and the following 253c4d7559SDave Watson * disclaimer in the documentation and/or other materials 263c4d7559SDave Watson * provided with the distribution. 273c4d7559SDave Watson * 283c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 293c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 303c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 313c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 323c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 333c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 343c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 353c4d7559SDave Watson * SOFTWARE. 363c4d7559SDave Watson */ 373c4d7559SDave Watson 38c46234ebSDave Watson #include <linux/sched/signal.h> 393c4d7559SDave Watson #include <linux/module.h> 403c4d7559SDave Watson #include <crypto/aead.h> 413c4d7559SDave Watson 42c46234ebSDave Watson #include <net/strparser.h> 433c4d7559SDave Watson #include <net/tls.h> 443c4d7559SDave Watson 45b16520f7SKees Cook #define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE 46b16520f7SKees Cook 470927f71dSDoron Roberts-Kedes static int __skb_nsg(struct sk_buff *skb, int offset, int len, 480927f71dSDoron Roberts-Kedes unsigned int recursion_level) 490927f71dSDoron Roberts-Kedes { 500927f71dSDoron Roberts-Kedes int start = skb_headlen(skb); 510927f71dSDoron Roberts-Kedes int i, chunk = start - offset; 520927f71dSDoron Roberts-Kedes struct sk_buff *frag_iter; 530927f71dSDoron Roberts-Kedes int elt = 0; 540927f71dSDoron Roberts-Kedes 550927f71dSDoron Roberts-Kedes if (unlikely(recursion_level >= 24)) 560927f71dSDoron Roberts-Kedes return -EMSGSIZE; 570927f71dSDoron Roberts-Kedes 580927f71dSDoron Roberts-Kedes if (chunk > 0) { 590927f71dSDoron Roberts-Kedes if (chunk > len) 600927f71dSDoron Roberts-Kedes chunk = len; 610927f71dSDoron Roberts-Kedes elt++; 620927f71dSDoron Roberts-Kedes len -= chunk; 630927f71dSDoron Roberts-Kedes if (len == 0) 640927f71dSDoron Roberts-Kedes return elt; 650927f71dSDoron Roberts-Kedes offset += chunk; 660927f71dSDoron Roberts-Kedes } 670927f71dSDoron Roberts-Kedes 680927f71dSDoron Roberts-Kedes for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 690927f71dSDoron Roberts-Kedes int end; 700927f71dSDoron Roberts-Kedes 710927f71dSDoron Roberts-Kedes WARN_ON(start > offset + len); 720927f71dSDoron Roberts-Kedes 730927f71dSDoron Roberts-Kedes end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 740927f71dSDoron Roberts-Kedes chunk = end - offset; 750927f71dSDoron Roberts-Kedes if (chunk > 0) { 760927f71dSDoron Roberts-Kedes if (chunk > len) 770927f71dSDoron Roberts-Kedes chunk = len; 780927f71dSDoron Roberts-Kedes elt++; 790927f71dSDoron Roberts-Kedes len -= chunk; 800927f71dSDoron Roberts-Kedes if (len == 0) 810927f71dSDoron Roberts-Kedes return elt; 820927f71dSDoron Roberts-Kedes offset += chunk; 830927f71dSDoron Roberts-Kedes } 840927f71dSDoron Roberts-Kedes start = end; 850927f71dSDoron Roberts-Kedes } 860927f71dSDoron Roberts-Kedes 870927f71dSDoron Roberts-Kedes if (unlikely(skb_has_frag_list(skb))) { 880927f71dSDoron Roberts-Kedes skb_walk_frags(skb, frag_iter) { 890927f71dSDoron Roberts-Kedes int end, ret; 900927f71dSDoron Roberts-Kedes 910927f71dSDoron Roberts-Kedes WARN_ON(start > offset + len); 920927f71dSDoron Roberts-Kedes 930927f71dSDoron Roberts-Kedes end = start + frag_iter->len; 940927f71dSDoron Roberts-Kedes chunk = end - offset; 950927f71dSDoron Roberts-Kedes if (chunk > 0) { 960927f71dSDoron Roberts-Kedes if (chunk > len) 970927f71dSDoron Roberts-Kedes chunk = len; 980927f71dSDoron Roberts-Kedes ret = __skb_nsg(frag_iter, offset - start, chunk, 990927f71dSDoron Roberts-Kedes recursion_level + 1); 1000927f71dSDoron Roberts-Kedes if (unlikely(ret < 0)) 1010927f71dSDoron Roberts-Kedes return ret; 1020927f71dSDoron Roberts-Kedes elt += ret; 1030927f71dSDoron Roberts-Kedes len -= chunk; 1040927f71dSDoron Roberts-Kedes if (len == 0) 1050927f71dSDoron Roberts-Kedes return elt; 1060927f71dSDoron Roberts-Kedes offset += chunk; 1070927f71dSDoron Roberts-Kedes } 1080927f71dSDoron Roberts-Kedes start = end; 1090927f71dSDoron Roberts-Kedes } 1100927f71dSDoron Roberts-Kedes } 1110927f71dSDoron Roberts-Kedes BUG_ON(len); 1120927f71dSDoron Roberts-Kedes return elt; 1130927f71dSDoron Roberts-Kedes } 1140927f71dSDoron Roberts-Kedes 1150927f71dSDoron Roberts-Kedes /* Return the number of scatterlist elements required to completely map the 1160927f71dSDoron Roberts-Kedes * skb, or -EMSGSIZE if the recursion depth is exceeded. 1170927f71dSDoron Roberts-Kedes */ 1180927f71dSDoron Roberts-Kedes static int skb_nsg(struct sk_buff *skb, int offset, int len) 1190927f71dSDoron Roberts-Kedes { 1200927f71dSDoron Roberts-Kedes return __skb_nsg(skb, offset, len, 0); 1210927f71dSDoron Roberts-Kedes } 1220927f71dSDoron Roberts-Kedes 123130b392cSDave Watson static int padding_length(struct tls_sw_context_rx *ctx, 124130b392cSDave Watson struct tls_context *tls_ctx, struct sk_buff *skb) 125130b392cSDave Watson { 126130b392cSDave Watson struct strp_msg *rxm = strp_msg(skb); 127130b392cSDave Watson int sub = 0; 128130b392cSDave Watson 129130b392cSDave Watson /* Determine zero-padding length */ 1304509de14SVakul Garg if (tls_ctx->prot_info.version == TLS_1_3_VERSION) { 131130b392cSDave Watson char content_type = 0; 132130b392cSDave Watson int err; 133130b392cSDave Watson int back = 17; 134130b392cSDave Watson 135130b392cSDave Watson while (content_type == 0) { 136130b392cSDave Watson if (back > rxm->full_len) 137130b392cSDave Watson return -EBADMSG; 138130b392cSDave Watson err = skb_copy_bits(skb, 139130b392cSDave Watson rxm->offset + rxm->full_len - back, 140130b392cSDave Watson &content_type, 1); 141130b392cSDave Watson if (content_type) 142130b392cSDave Watson break; 143130b392cSDave Watson sub++; 144130b392cSDave Watson back++; 145130b392cSDave Watson } 146130b392cSDave Watson ctx->control = content_type; 147130b392cSDave Watson } 148130b392cSDave Watson return sub; 149130b392cSDave Watson } 150130b392cSDave Watson 15194524d8fSVakul Garg static void tls_decrypt_done(struct crypto_async_request *req, int err) 15294524d8fSVakul Garg { 15394524d8fSVakul Garg struct aead_request *aead_req = (struct aead_request *)req; 15494524d8fSVakul Garg struct scatterlist *sgout = aead_req->dst; 155692d7b5dSVakul Garg struct scatterlist *sgin = aead_req->src; 1567a3dd8c8SJohn Fastabend struct tls_sw_context_rx *ctx; 1577a3dd8c8SJohn Fastabend struct tls_context *tls_ctx; 1584509de14SVakul Garg struct tls_prot_info *prot; 15994524d8fSVakul Garg struct scatterlist *sg; 1607a3dd8c8SJohn Fastabend struct sk_buff *skb; 16194524d8fSVakul Garg unsigned int pages; 1627a3dd8c8SJohn Fastabend int pending; 1637a3dd8c8SJohn Fastabend 1647a3dd8c8SJohn Fastabend skb = (struct sk_buff *)req->data; 1657a3dd8c8SJohn Fastabend tls_ctx = tls_get_ctx(skb->sk); 1667a3dd8c8SJohn Fastabend ctx = tls_sw_ctx_rx(tls_ctx); 1674509de14SVakul Garg prot = &tls_ctx->prot_info; 16894524d8fSVakul Garg 16994524d8fSVakul Garg /* Propagate if there was an err */ 17094524d8fSVakul Garg if (err) { 17194524d8fSVakul Garg ctx->async_wait.err = err; 1727a3dd8c8SJohn Fastabend tls_err_abort(skb->sk, err); 173692d7b5dSVakul Garg } else { 174692d7b5dSVakul Garg struct strp_msg *rxm = strp_msg(skb); 175130b392cSDave Watson rxm->full_len -= padding_length(ctx, tls_ctx, skb); 1764509de14SVakul Garg rxm->offset += prot->prepend_size; 1774509de14SVakul Garg rxm->full_len -= prot->overhead_size; 17894524d8fSVakul Garg } 17994524d8fSVakul Garg 1807a3dd8c8SJohn Fastabend /* After using skb->sk to propagate sk through crypto async callback 1817a3dd8c8SJohn Fastabend * we need to NULL it again. 1827a3dd8c8SJohn Fastabend */ 1837a3dd8c8SJohn Fastabend skb->sk = NULL; 1847a3dd8c8SJohn Fastabend 18594524d8fSVakul Garg 186692d7b5dSVakul Garg /* Free the destination pages if skb was not decrypted inplace */ 187692d7b5dSVakul Garg if (sgout != sgin) { 18894524d8fSVakul Garg /* Skip the first S/G entry as it points to AAD */ 18994524d8fSVakul Garg for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { 19094524d8fSVakul Garg if (!sg) 19194524d8fSVakul Garg break; 19294524d8fSVakul Garg put_page(sg_page(sg)); 19394524d8fSVakul Garg } 194692d7b5dSVakul Garg } 19594524d8fSVakul Garg 19694524d8fSVakul Garg kfree(aead_req); 19794524d8fSVakul Garg 198692d7b5dSVakul Garg pending = atomic_dec_return(&ctx->decrypt_pending); 199692d7b5dSVakul Garg 20094524d8fSVakul Garg if (!pending && READ_ONCE(ctx->async_notify)) 20194524d8fSVakul Garg complete(&ctx->async_wait.completion); 20294524d8fSVakul Garg } 20394524d8fSVakul Garg 204c46234ebSDave Watson static int tls_do_decryption(struct sock *sk, 20594524d8fSVakul Garg struct sk_buff *skb, 206c46234ebSDave Watson struct scatterlist *sgin, 207c46234ebSDave Watson struct scatterlist *sgout, 208c46234ebSDave Watson char *iv_recv, 209c46234ebSDave Watson size_t data_len, 21094524d8fSVakul Garg struct aead_request *aead_req, 21194524d8fSVakul Garg bool async) 212c46234ebSDave Watson { 213c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2144509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 215f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 216c46234ebSDave Watson int ret; 217c46234ebSDave Watson 2180b243d00SVakul Garg aead_request_set_tfm(aead_req, ctx->aead_recv); 2194509de14SVakul Garg aead_request_set_ad(aead_req, prot->aad_size); 220c46234ebSDave Watson aead_request_set_crypt(aead_req, sgin, sgout, 2214509de14SVakul Garg data_len + prot->tag_size, 222c46234ebSDave Watson (u8 *)iv_recv); 223c46234ebSDave Watson 22494524d8fSVakul Garg if (async) { 2257a3dd8c8SJohn Fastabend /* Using skb->sk to push sk through to crypto async callback 2267a3dd8c8SJohn Fastabend * handler. This allows propagating errors up to the socket 2277a3dd8c8SJohn Fastabend * if needed. It _must_ be cleared in the async handler 2287a3dd8c8SJohn Fastabend * before kfree_skb is called. We _know_ skb->sk is NULL 2297a3dd8c8SJohn Fastabend * because it is a clone from strparser. 2307a3dd8c8SJohn Fastabend */ 2317a3dd8c8SJohn Fastabend skb->sk = sk; 23294524d8fSVakul Garg aead_request_set_callback(aead_req, 23394524d8fSVakul Garg CRYPTO_TFM_REQ_MAY_BACKLOG, 23494524d8fSVakul Garg tls_decrypt_done, skb); 23594524d8fSVakul Garg atomic_inc(&ctx->decrypt_pending); 23694524d8fSVakul Garg } else { 23794524d8fSVakul Garg aead_request_set_callback(aead_req, 23894524d8fSVakul Garg CRYPTO_TFM_REQ_MAY_BACKLOG, 23994524d8fSVakul Garg crypto_req_done, &ctx->async_wait); 24094524d8fSVakul Garg } 24194524d8fSVakul Garg 24294524d8fSVakul Garg ret = crypto_aead_decrypt(aead_req); 24394524d8fSVakul Garg if (ret == -EINPROGRESS) { 24494524d8fSVakul Garg if (async) 24594524d8fSVakul Garg return ret; 24694524d8fSVakul Garg 24794524d8fSVakul Garg ret = crypto_wait_req(ret, &ctx->async_wait); 24894524d8fSVakul Garg } 24994524d8fSVakul Garg 25094524d8fSVakul Garg if (async) 25194524d8fSVakul Garg atomic_dec(&ctx->decrypt_pending); 25294524d8fSVakul Garg 253c46234ebSDave Watson return ret; 254c46234ebSDave Watson } 255c46234ebSDave Watson 256d829e9c4SDaniel Borkmann static void tls_trim_both_msgs(struct sock *sk, int target_size) 2573c4d7559SDave Watson { 2583c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2594509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 260f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 261a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 2623c4d7559SDave Watson 263d829e9c4SDaniel Borkmann sk_msg_trim(sk, &rec->msg_plaintext, target_size); 2643c4d7559SDave Watson if (target_size > 0) 2654509de14SVakul Garg target_size += prot->overhead_size; 266d829e9c4SDaniel Borkmann sk_msg_trim(sk, &rec->msg_encrypted, target_size); 2673c4d7559SDave Watson } 2683c4d7559SDave Watson 269d829e9c4SDaniel Borkmann static int tls_alloc_encrypted_msg(struct sock *sk, int len) 2703c4d7559SDave Watson { 2713c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 272f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 273a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 274d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 2753c4d7559SDave Watson 276d829e9c4SDaniel Borkmann return sk_msg_alloc(sk, msg_en, len, 0); 2773c4d7559SDave Watson } 2783c4d7559SDave Watson 279d829e9c4SDaniel Borkmann static int tls_clone_plaintext_msg(struct sock *sk, int required) 2803c4d7559SDave Watson { 2813c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2824509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 283f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 284a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 285d829e9c4SDaniel Borkmann struct sk_msg *msg_pl = &rec->msg_plaintext; 286d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 2874e6d4720SVakul Garg int skip, len; 2883c4d7559SDave Watson 289d829e9c4SDaniel Borkmann /* We add page references worth len bytes from encrypted sg 290d829e9c4SDaniel Borkmann * at the end of plaintext sg. It is guaranteed that msg_en 2914e6d4720SVakul Garg * has enough required room (ensured by caller). 2924e6d4720SVakul Garg */ 293d829e9c4SDaniel Borkmann len = required - msg_pl->sg.size; 29452ea992cSVakul Garg 295d829e9c4SDaniel Borkmann /* Skip initial bytes in msg_en's data to be able to use 296d829e9c4SDaniel Borkmann * same offset of both plain and encrypted data. 2974e6d4720SVakul Garg */ 2984509de14SVakul Garg skip = prot->prepend_size + msg_pl->sg.size; 2994e6d4720SVakul Garg 300d829e9c4SDaniel Borkmann return sk_msg_clone(sk, msg_pl, msg_en, skip, len); 3013c4d7559SDave Watson } 3023c4d7559SDave Watson 303d3b18ad3SJohn Fastabend static struct tls_rec *tls_get_rec(struct sock *sk) 304d3b18ad3SJohn Fastabend { 305d3b18ad3SJohn Fastabend struct tls_context *tls_ctx = tls_get_ctx(sk); 3064509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 307d3b18ad3SJohn Fastabend struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 308d3b18ad3SJohn Fastabend struct sk_msg *msg_pl, *msg_en; 309d3b18ad3SJohn Fastabend struct tls_rec *rec; 310d3b18ad3SJohn Fastabend int mem_size; 311d3b18ad3SJohn Fastabend 312d3b18ad3SJohn Fastabend mem_size = sizeof(struct tls_rec) + crypto_aead_reqsize(ctx->aead_send); 313d3b18ad3SJohn Fastabend 314d3b18ad3SJohn Fastabend rec = kzalloc(mem_size, sk->sk_allocation); 315d3b18ad3SJohn Fastabend if (!rec) 316d3b18ad3SJohn Fastabend return NULL; 317d3b18ad3SJohn Fastabend 318d3b18ad3SJohn Fastabend msg_pl = &rec->msg_plaintext; 319d3b18ad3SJohn Fastabend msg_en = &rec->msg_encrypted; 320d3b18ad3SJohn Fastabend 321d3b18ad3SJohn Fastabend sk_msg_init(msg_pl); 322d3b18ad3SJohn Fastabend sk_msg_init(msg_en); 323d3b18ad3SJohn Fastabend 324d3b18ad3SJohn Fastabend sg_init_table(rec->sg_aead_in, 2); 3254509de14SVakul Garg sg_set_buf(&rec->sg_aead_in[0], rec->aad_space, prot->aad_size); 326d3b18ad3SJohn Fastabend sg_unmark_end(&rec->sg_aead_in[1]); 327d3b18ad3SJohn Fastabend 328d3b18ad3SJohn Fastabend sg_init_table(rec->sg_aead_out, 2); 3294509de14SVakul Garg sg_set_buf(&rec->sg_aead_out[0], rec->aad_space, prot->aad_size); 330d3b18ad3SJohn Fastabend sg_unmark_end(&rec->sg_aead_out[1]); 331d3b18ad3SJohn Fastabend 332d3b18ad3SJohn Fastabend return rec; 333d3b18ad3SJohn Fastabend } 334d3b18ad3SJohn Fastabend 335d3b18ad3SJohn Fastabend static void tls_free_rec(struct sock *sk, struct tls_rec *rec) 336d3b18ad3SJohn Fastabend { 337d3b18ad3SJohn Fastabend sk_msg_free(sk, &rec->msg_encrypted); 338d3b18ad3SJohn Fastabend sk_msg_free(sk, &rec->msg_plaintext); 339d3b18ad3SJohn Fastabend kfree(rec); 340d3b18ad3SJohn Fastabend } 341d3b18ad3SJohn Fastabend 342c774973eSVakul Garg static void tls_free_open_rec(struct sock *sk) 3433c4d7559SDave Watson { 3443c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 345f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 346a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 3473c4d7559SDave Watson 348d3b18ad3SJohn Fastabend if (rec) { 349d3b18ad3SJohn Fastabend tls_free_rec(sk, rec); 350d3b18ad3SJohn Fastabend ctx->open_rec = NULL; 351d3b18ad3SJohn Fastabend } 3523c4d7559SDave Watson } 3533c4d7559SDave Watson 354a42055e8SVakul Garg int tls_tx_records(struct sock *sk, int flags) 355a42055e8SVakul Garg { 356a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 357a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 358a42055e8SVakul Garg struct tls_rec *rec, *tmp; 359d829e9c4SDaniel Borkmann struct sk_msg *msg_en; 360a42055e8SVakul Garg int tx_flags, rc = 0; 361a42055e8SVakul Garg 362a42055e8SVakul Garg if (tls_is_partially_sent_record(tls_ctx)) { 3639932a29aSVakul Garg rec = list_first_entry(&ctx->tx_list, 364a42055e8SVakul Garg struct tls_rec, list); 365a42055e8SVakul Garg 366a42055e8SVakul Garg if (flags == -1) 367a42055e8SVakul Garg tx_flags = rec->tx_flags; 368a42055e8SVakul Garg else 369a42055e8SVakul Garg tx_flags = flags; 370a42055e8SVakul Garg 371a42055e8SVakul Garg rc = tls_push_partial_record(sk, tls_ctx, tx_flags); 372a42055e8SVakul Garg if (rc) 373a42055e8SVakul Garg goto tx_err; 374a42055e8SVakul Garg 375a42055e8SVakul Garg /* Full record has been transmitted. 3769932a29aSVakul Garg * Remove the head of tx_list 377a42055e8SVakul Garg */ 378a42055e8SVakul Garg list_del(&rec->list); 379d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 380a42055e8SVakul Garg kfree(rec); 381a42055e8SVakul Garg } 382a42055e8SVakul Garg 3839932a29aSVakul Garg /* Tx all ready records */ 3849932a29aSVakul Garg list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { 3859932a29aSVakul Garg if (READ_ONCE(rec->tx_ready)) { 386a42055e8SVakul Garg if (flags == -1) 387a42055e8SVakul Garg tx_flags = rec->tx_flags; 388a42055e8SVakul Garg else 389a42055e8SVakul Garg tx_flags = flags; 390a42055e8SVakul Garg 391d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 392a42055e8SVakul Garg rc = tls_push_sg(sk, tls_ctx, 393d829e9c4SDaniel Borkmann &msg_en->sg.data[msg_en->sg.curr], 394a42055e8SVakul Garg 0, tx_flags); 395a42055e8SVakul Garg if (rc) 396a42055e8SVakul Garg goto tx_err; 397a42055e8SVakul Garg 398a42055e8SVakul Garg list_del(&rec->list); 399d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 400a42055e8SVakul Garg kfree(rec); 401a42055e8SVakul Garg } else { 402a42055e8SVakul Garg break; 403a42055e8SVakul Garg } 404a42055e8SVakul Garg } 405a42055e8SVakul Garg 406a42055e8SVakul Garg tx_err: 407a42055e8SVakul Garg if (rc < 0 && rc != -EAGAIN) 408a42055e8SVakul Garg tls_err_abort(sk, EBADMSG); 409a42055e8SVakul Garg 410a42055e8SVakul Garg return rc; 411a42055e8SVakul Garg } 412a42055e8SVakul Garg 413a42055e8SVakul Garg static void tls_encrypt_done(struct crypto_async_request *req, int err) 414a42055e8SVakul Garg { 415a42055e8SVakul Garg struct aead_request *aead_req = (struct aead_request *)req; 416a42055e8SVakul Garg struct sock *sk = req->data; 417a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 4184509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 419a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 420d829e9c4SDaniel Borkmann struct scatterlist *sge; 421d829e9c4SDaniel Borkmann struct sk_msg *msg_en; 422a42055e8SVakul Garg struct tls_rec *rec; 423a42055e8SVakul Garg bool ready = false; 424a42055e8SVakul Garg int pending; 425a42055e8SVakul Garg 426a42055e8SVakul Garg rec = container_of(aead_req, struct tls_rec, aead_req); 427d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 428a42055e8SVakul Garg 429d829e9c4SDaniel Borkmann sge = sk_msg_elem(msg_en, msg_en->sg.curr); 4304509de14SVakul Garg sge->offset -= prot->prepend_size; 4314509de14SVakul Garg sge->length += prot->prepend_size; 432a42055e8SVakul Garg 43380ece6a0SVakul Garg /* Check if error is previously set on socket */ 434a42055e8SVakul Garg if (err || sk->sk_err) { 435a42055e8SVakul Garg rec = NULL; 436a42055e8SVakul Garg 437a42055e8SVakul Garg /* If err is already set on socket, return the same code */ 438a42055e8SVakul Garg if (sk->sk_err) { 439a42055e8SVakul Garg ctx->async_wait.err = sk->sk_err; 440a42055e8SVakul Garg } else { 441a42055e8SVakul Garg ctx->async_wait.err = err; 442a42055e8SVakul Garg tls_err_abort(sk, err); 443a42055e8SVakul Garg } 444a42055e8SVakul Garg } 445a42055e8SVakul Garg 4469932a29aSVakul Garg if (rec) { 4479932a29aSVakul Garg struct tls_rec *first_rec; 4489932a29aSVakul Garg 4499932a29aSVakul Garg /* Mark the record as ready for transmission */ 4509932a29aSVakul Garg smp_store_mb(rec->tx_ready, true); 4519932a29aSVakul Garg 4529932a29aSVakul Garg /* If received record is at head of tx_list, schedule tx */ 4539932a29aSVakul Garg first_rec = list_first_entry(&ctx->tx_list, 4549932a29aSVakul Garg struct tls_rec, list); 4559932a29aSVakul Garg if (rec == first_rec) 4569932a29aSVakul Garg ready = true; 4579932a29aSVakul Garg } 458a42055e8SVakul Garg 459a42055e8SVakul Garg pending = atomic_dec_return(&ctx->encrypt_pending); 460a42055e8SVakul Garg 461a42055e8SVakul Garg if (!pending && READ_ONCE(ctx->async_notify)) 462a42055e8SVakul Garg complete(&ctx->async_wait.completion); 463a42055e8SVakul Garg 464a42055e8SVakul Garg if (!ready) 465a42055e8SVakul Garg return; 466a42055e8SVakul Garg 467a42055e8SVakul Garg /* Schedule the transmission */ 468a42055e8SVakul Garg if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) 469d829e9c4SDaniel Borkmann schedule_delayed_work(&ctx->tx_work.work, 1); 470a42055e8SVakul Garg } 471a42055e8SVakul Garg 472a42055e8SVakul Garg static int tls_do_encryption(struct sock *sk, 473a42055e8SVakul Garg struct tls_context *tls_ctx, 474a447da7dSDaniel Borkmann struct tls_sw_context_tx *ctx, 475a447da7dSDaniel Borkmann struct aead_request *aead_req, 476d829e9c4SDaniel Borkmann size_t data_len, u32 start) 4773c4d7559SDave Watson { 4784509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 479a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 480d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 481d829e9c4SDaniel Borkmann struct scatterlist *sge = sk_msg_elem(msg_en, start); 4823c4d7559SDave Watson int rc; 4833c4d7559SDave Watson 48432eb67b9SDave Watson memcpy(rec->iv_data, tls_ctx->tx.iv, sizeof(rec->iv_data)); 4854509de14SVakul Garg xor_iv_with_seq(prot->version, rec->iv_data, 486130b392cSDave Watson tls_ctx->tx.rec_seq); 48732eb67b9SDave Watson 4884509de14SVakul Garg sge->offset += prot->prepend_size; 4894509de14SVakul Garg sge->length -= prot->prepend_size; 4903c4d7559SDave Watson 491d829e9c4SDaniel Borkmann msg_en->sg.curr = start; 4924e6d4720SVakul Garg 4933c4d7559SDave Watson aead_request_set_tfm(aead_req, ctx->aead_send); 4944509de14SVakul Garg aead_request_set_ad(aead_req, prot->aad_size); 495d829e9c4SDaniel Borkmann aead_request_set_crypt(aead_req, rec->sg_aead_in, 496d829e9c4SDaniel Borkmann rec->sg_aead_out, 49732eb67b9SDave Watson data_len, rec->iv_data); 498a54667f6SVakul Garg 499a54667f6SVakul Garg aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 500a42055e8SVakul Garg tls_encrypt_done, sk); 501a54667f6SVakul Garg 5029932a29aSVakul Garg /* Add the record in tx_list */ 5039932a29aSVakul Garg list_add_tail((struct list_head *)&rec->list, &ctx->tx_list); 504a42055e8SVakul Garg atomic_inc(&ctx->encrypt_pending); 5053c4d7559SDave Watson 506a42055e8SVakul Garg rc = crypto_aead_encrypt(aead_req); 507a42055e8SVakul Garg if (!rc || rc != -EINPROGRESS) { 508a42055e8SVakul Garg atomic_dec(&ctx->encrypt_pending); 5094509de14SVakul Garg sge->offset -= prot->prepend_size; 5104509de14SVakul Garg sge->length += prot->prepend_size; 511a42055e8SVakul Garg } 5123c4d7559SDave Watson 5139932a29aSVakul Garg if (!rc) { 5149932a29aSVakul Garg WRITE_ONCE(rec->tx_ready, true); 5159932a29aSVakul Garg } else if (rc != -EINPROGRESS) { 5169932a29aSVakul Garg list_del(&rec->list); 517a42055e8SVakul Garg return rc; 5189932a29aSVakul Garg } 519a42055e8SVakul Garg 520a42055e8SVakul Garg /* Unhook the record from context if encryption is not failure */ 521a42055e8SVakul Garg ctx->open_rec = NULL; 5224509de14SVakul Garg tls_advance_record_sn(sk, &tls_ctx->tx, prot->version); 5233c4d7559SDave Watson return rc; 5243c4d7559SDave Watson } 5253c4d7559SDave Watson 526d3b18ad3SJohn Fastabend static int tls_split_open_record(struct sock *sk, struct tls_rec *from, 527d3b18ad3SJohn Fastabend struct tls_rec **to, struct sk_msg *msg_opl, 528d3b18ad3SJohn Fastabend struct sk_msg *msg_oen, u32 split_point, 529d3b18ad3SJohn Fastabend u32 tx_overhead_size, u32 *orig_end) 530d3b18ad3SJohn Fastabend { 531d3b18ad3SJohn Fastabend u32 i, j, bytes = 0, apply = msg_opl->apply_bytes; 532d3b18ad3SJohn Fastabend struct scatterlist *sge, *osge, *nsge; 533d3b18ad3SJohn Fastabend u32 orig_size = msg_opl->sg.size; 534d3b18ad3SJohn Fastabend struct scatterlist tmp = { }; 535d3b18ad3SJohn Fastabend struct sk_msg *msg_npl; 536d3b18ad3SJohn Fastabend struct tls_rec *new; 537d3b18ad3SJohn Fastabend int ret; 538d3b18ad3SJohn Fastabend 539d3b18ad3SJohn Fastabend new = tls_get_rec(sk); 540d3b18ad3SJohn Fastabend if (!new) 541d3b18ad3SJohn Fastabend return -ENOMEM; 542d3b18ad3SJohn Fastabend ret = sk_msg_alloc(sk, &new->msg_encrypted, msg_opl->sg.size + 543d3b18ad3SJohn Fastabend tx_overhead_size, 0); 544d3b18ad3SJohn Fastabend if (ret < 0) { 545d3b18ad3SJohn Fastabend tls_free_rec(sk, new); 546d3b18ad3SJohn Fastabend return ret; 547d3b18ad3SJohn Fastabend } 548d3b18ad3SJohn Fastabend 549d3b18ad3SJohn Fastabend *orig_end = msg_opl->sg.end; 550d3b18ad3SJohn Fastabend i = msg_opl->sg.start; 551d3b18ad3SJohn Fastabend sge = sk_msg_elem(msg_opl, i); 552d3b18ad3SJohn Fastabend while (apply && sge->length) { 553d3b18ad3SJohn Fastabend if (sge->length > apply) { 554d3b18ad3SJohn Fastabend u32 len = sge->length - apply; 555d3b18ad3SJohn Fastabend 556d3b18ad3SJohn Fastabend get_page(sg_page(sge)); 557d3b18ad3SJohn Fastabend sg_set_page(&tmp, sg_page(sge), len, 558d3b18ad3SJohn Fastabend sge->offset + apply); 559d3b18ad3SJohn Fastabend sge->length = apply; 560d3b18ad3SJohn Fastabend bytes += apply; 561d3b18ad3SJohn Fastabend apply = 0; 562d3b18ad3SJohn Fastabend } else { 563d3b18ad3SJohn Fastabend apply -= sge->length; 564d3b18ad3SJohn Fastabend bytes += sge->length; 565d3b18ad3SJohn Fastabend } 566d3b18ad3SJohn Fastabend 567d3b18ad3SJohn Fastabend sk_msg_iter_var_next(i); 568d3b18ad3SJohn Fastabend if (i == msg_opl->sg.end) 569d3b18ad3SJohn Fastabend break; 570d3b18ad3SJohn Fastabend sge = sk_msg_elem(msg_opl, i); 571d3b18ad3SJohn Fastabend } 572d3b18ad3SJohn Fastabend 573d3b18ad3SJohn Fastabend msg_opl->sg.end = i; 574d3b18ad3SJohn Fastabend msg_opl->sg.curr = i; 575d3b18ad3SJohn Fastabend msg_opl->sg.copybreak = 0; 576d3b18ad3SJohn Fastabend msg_opl->apply_bytes = 0; 577d3b18ad3SJohn Fastabend msg_opl->sg.size = bytes; 578d3b18ad3SJohn Fastabend 579d3b18ad3SJohn Fastabend msg_npl = &new->msg_plaintext; 580d3b18ad3SJohn Fastabend msg_npl->apply_bytes = apply; 581d3b18ad3SJohn Fastabend msg_npl->sg.size = orig_size - bytes; 582d3b18ad3SJohn Fastabend 583d3b18ad3SJohn Fastabend j = msg_npl->sg.start; 584d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 585d3b18ad3SJohn Fastabend if (tmp.length) { 586d3b18ad3SJohn Fastabend memcpy(nsge, &tmp, sizeof(*nsge)); 587d3b18ad3SJohn Fastabend sk_msg_iter_var_next(j); 588d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 589d3b18ad3SJohn Fastabend } 590d3b18ad3SJohn Fastabend 591d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 592d3b18ad3SJohn Fastabend while (osge->length) { 593d3b18ad3SJohn Fastabend memcpy(nsge, osge, sizeof(*nsge)); 594d3b18ad3SJohn Fastabend sg_unmark_end(nsge); 595d3b18ad3SJohn Fastabend sk_msg_iter_var_next(i); 596d3b18ad3SJohn Fastabend sk_msg_iter_var_next(j); 597d3b18ad3SJohn Fastabend if (i == *orig_end) 598d3b18ad3SJohn Fastabend break; 599d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 600d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 601d3b18ad3SJohn Fastabend } 602d3b18ad3SJohn Fastabend 603d3b18ad3SJohn Fastabend msg_npl->sg.end = j; 604d3b18ad3SJohn Fastabend msg_npl->sg.curr = j; 605d3b18ad3SJohn Fastabend msg_npl->sg.copybreak = 0; 606d3b18ad3SJohn Fastabend 607d3b18ad3SJohn Fastabend *to = new; 608d3b18ad3SJohn Fastabend return 0; 609d3b18ad3SJohn Fastabend } 610d3b18ad3SJohn Fastabend 611d3b18ad3SJohn Fastabend static void tls_merge_open_record(struct sock *sk, struct tls_rec *to, 612d3b18ad3SJohn Fastabend struct tls_rec *from, u32 orig_end) 613d3b18ad3SJohn Fastabend { 614d3b18ad3SJohn Fastabend struct sk_msg *msg_npl = &from->msg_plaintext; 615d3b18ad3SJohn Fastabend struct sk_msg *msg_opl = &to->msg_plaintext; 616d3b18ad3SJohn Fastabend struct scatterlist *osge, *nsge; 617d3b18ad3SJohn Fastabend u32 i, j; 618d3b18ad3SJohn Fastabend 619d3b18ad3SJohn Fastabend i = msg_opl->sg.end; 620d3b18ad3SJohn Fastabend sk_msg_iter_var_prev(i); 621d3b18ad3SJohn Fastabend j = msg_npl->sg.start; 622d3b18ad3SJohn Fastabend 623d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 624d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 625d3b18ad3SJohn Fastabend 626d3b18ad3SJohn Fastabend if (sg_page(osge) == sg_page(nsge) && 627d3b18ad3SJohn Fastabend osge->offset + osge->length == nsge->offset) { 628d3b18ad3SJohn Fastabend osge->length += nsge->length; 629d3b18ad3SJohn Fastabend put_page(sg_page(nsge)); 630d3b18ad3SJohn Fastabend } 631d3b18ad3SJohn Fastabend 632d3b18ad3SJohn Fastabend msg_opl->sg.end = orig_end; 633d3b18ad3SJohn Fastabend msg_opl->sg.curr = orig_end; 634d3b18ad3SJohn Fastabend msg_opl->sg.copybreak = 0; 635d3b18ad3SJohn Fastabend msg_opl->apply_bytes = msg_opl->sg.size + msg_npl->sg.size; 636d3b18ad3SJohn Fastabend msg_opl->sg.size += msg_npl->sg.size; 637d3b18ad3SJohn Fastabend 638d3b18ad3SJohn Fastabend sk_msg_free(sk, &to->msg_encrypted); 639d3b18ad3SJohn Fastabend sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted); 640d3b18ad3SJohn Fastabend 641d3b18ad3SJohn Fastabend kfree(from); 642d3b18ad3SJohn Fastabend } 643d3b18ad3SJohn Fastabend 6443c4d7559SDave Watson static int tls_push_record(struct sock *sk, int flags, 6453c4d7559SDave Watson unsigned char record_type) 6463c4d7559SDave Watson { 6473c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 6484509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 649f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 650d3b18ad3SJohn Fastabend struct tls_rec *rec = ctx->open_rec, *tmp = NULL; 651d3b18ad3SJohn Fastabend u32 i, split_point, uninitialized_var(orig_end); 652d829e9c4SDaniel Borkmann struct sk_msg *msg_pl, *msg_en; 653a447da7dSDaniel Borkmann struct aead_request *req; 654d3b18ad3SJohn Fastabend bool split; 6553c4d7559SDave Watson int rc; 6563c4d7559SDave Watson 657a42055e8SVakul Garg if (!rec) 658a42055e8SVakul Garg return 0; 659a447da7dSDaniel Borkmann 660d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 661d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 662d829e9c4SDaniel Borkmann 663d3b18ad3SJohn Fastabend split_point = msg_pl->apply_bytes; 664d3b18ad3SJohn Fastabend split = split_point && split_point < msg_pl->sg.size; 665d3b18ad3SJohn Fastabend if (split) { 666d3b18ad3SJohn Fastabend rc = tls_split_open_record(sk, rec, &tmp, msg_pl, msg_en, 6674509de14SVakul Garg split_point, prot->overhead_size, 668d3b18ad3SJohn Fastabend &orig_end); 669d3b18ad3SJohn Fastabend if (rc < 0) 670d3b18ad3SJohn Fastabend return rc; 671d3b18ad3SJohn Fastabend sk_msg_trim(sk, msg_en, msg_pl->sg.size + 6724509de14SVakul Garg prot->overhead_size); 673d3b18ad3SJohn Fastabend } 674d3b18ad3SJohn Fastabend 675a42055e8SVakul Garg rec->tx_flags = flags; 676a42055e8SVakul Garg req = &rec->aead_req; 6773c4d7559SDave Watson 678d829e9c4SDaniel Borkmann i = msg_pl->sg.end; 679d829e9c4SDaniel Borkmann sk_msg_iter_var_prev(i); 680130b392cSDave Watson 681130b392cSDave Watson rec->content_type = record_type; 6824509de14SVakul Garg if (prot->version == TLS_1_3_VERSION) { 683130b392cSDave Watson /* Add content type to end of message. No padding added */ 684130b392cSDave Watson sg_set_buf(&rec->sg_content_type, &rec->content_type, 1); 685130b392cSDave Watson sg_mark_end(&rec->sg_content_type); 686130b392cSDave Watson sg_chain(msg_pl->sg.data, msg_pl->sg.end + 1, 687130b392cSDave Watson &rec->sg_content_type); 688130b392cSDave Watson } else { 689d829e9c4SDaniel Borkmann sg_mark_end(sk_msg_elem(msg_pl, i)); 690130b392cSDave Watson } 691a42055e8SVakul Garg 692d829e9c4SDaniel Borkmann i = msg_pl->sg.start; 693d829e9c4SDaniel Borkmann sg_chain(rec->sg_aead_in, 2, rec->inplace_crypto ? 694d829e9c4SDaniel Borkmann &msg_en->sg.data[i] : &msg_pl->sg.data[i]); 695d829e9c4SDaniel Borkmann 696d829e9c4SDaniel Borkmann i = msg_en->sg.end; 697d829e9c4SDaniel Borkmann sk_msg_iter_var_prev(i); 698d829e9c4SDaniel Borkmann sg_mark_end(sk_msg_elem(msg_en, i)); 699d829e9c4SDaniel Borkmann 700d829e9c4SDaniel Borkmann i = msg_en->sg.start; 701d829e9c4SDaniel Borkmann sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]); 702d829e9c4SDaniel Borkmann 7034509de14SVakul Garg tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size, 7044509de14SVakul Garg tls_ctx->tx.rec_seq, prot->rec_seq_size, 7054509de14SVakul Garg record_type, prot->version); 7063c4d7559SDave Watson 7073c4d7559SDave Watson tls_fill_prepend(tls_ctx, 708d829e9c4SDaniel Borkmann page_address(sg_page(&msg_en->sg.data[i])) + 709130b392cSDave Watson msg_en->sg.data[i].offset, 7104509de14SVakul Garg msg_pl->sg.size + prot->tail_size, 7114509de14SVakul Garg record_type, prot->version); 7123c4d7559SDave Watson 713d829e9c4SDaniel Borkmann tls_ctx->pending_open_record_frags = false; 7143c4d7559SDave Watson 715130b392cSDave Watson rc = tls_do_encryption(sk, tls_ctx, ctx, req, 7164509de14SVakul Garg msg_pl->sg.size + prot->tail_size, i); 7173c4d7559SDave Watson if (rc < 0) { 718d3b18ad3SJohn Fastabend if (rc != -EINPROGRESS) { 719a42055e8SVakul Garg tls_err_abort(sk, EBADMSG); 720d3b18ad3SJohn Fastabend if (split) { 721d3b18ad3SJohn Fastabend tls_ctx->pending_open_record_frags = true; 722d3b18ad3SJohn Fastabend tls_merge_open_record(sk, rec, tmp, orig_end); 723d3b18ad3SJohn Fastabend } 724d3b18ad3SJohn Fastabend } 7255b053e12SDave Watson ctx->async_capable = 1; 726a42055e8SVakul Garg return rc; 727d3b18ad3SJohn Fastabend } else if (split) { 728d3b18ad3SJohn Fastabend msg_pl = &tmp->msg_plaintext; 729d3b18ad3SJohn Fastabend msg_en = &tmp->msg_encrypted; 7304509de14SVakul Garg sk_msg_trim(sk, msg_en, msg_pl->sg.size + prot->overhead_size); 731d3b18ad3SJohn Fastabend tls_ctx->pending_open_record_frags = true; 732d3b18ad3SJohn Fastabend ctx->open_rec = tmp; 7333c4d7559SDave Watson } 7343c4d7559SDave Watson 735a42055e8SVakul Garg return tls_tx_records(sk, flags); 7363c4d7559SDave Watson } 7373c4d7559SDave Watson 738d3b18ad3SJohn Fastabend static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, 739d3b18ad3SJohn Fastabend bool full_record, u8 record_type, 740d3b18ad3SJohn Fastabend size_t *copied, int flags) 7413c4d7559SDave Watson { 7423c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 743f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 744d3b18ad3SJohn Fastabend struct sk_msg msg_redir = { }; 745d3b18ad3SJohn Fastabend struct sk_psock *psock; 746d3b18ad3SJohn Fastabend struct sock *sk_redir; 747a42055e8SVakul Garg struct tls_rec *rec; 7480608c69cSJohn Fastabend bool enospc, policy; 749d3b18ad3SJohn Fastabend int err = 0, send; 7507246d8edSJohn Fastabend u32 delta = 0; 751a42055e8SVakul Garg 7520608c69cSJohn Fastabend policy = !(flags & MSG_SENDPAGE_NOPOLICY); 753d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 7540608c69cSJohn Fastabend if (!psock || !policy) 755d3b18ad3SJohn Fastabend return tls_push_record(sk, flags, record_type); 756d3b18ad3SJohn Fastabend more_data: 757d3b18ad3SJohn Fastabend enospc = sk_msg_full(msg); 7587246d8edSJohn Fastabend if (psock->eval == __SK_NONE) { 7597246d8edSJohn Fastabend delta = msg->sg.size; 760d3b18ad3SJohn Fastabend psock->eval = sk_psock_msg_verdict(sk, psock, msg); 7617246d8edSJohn Fastabend if (delta < msg->sg.size) 7627246d8edSJohn Fastabend delta -= msg->sg.size; 7637246d8edSJohn Fastabend else 7647246d8edSJohn Fastabend delta = 0; 7657246d8edSJohn Fastabend } 766d3b18ad3SJohn Fastabend if (msg->cork_bytes && msg->cork_bytes > msg->sg.size && 767d3b18ad3SJohn Fastabend !enospc && !full_record) { 768d3b18ad3SJohn Fastabend err = -ENOSPC; 769d3b18ad3SJohn Fastabend goto out_err; 770d3b18ad3SJohn Fastabend } 771d3b18ad3SJohn Fastabend msg->cork_bytes = 0; 772d3b18ad3SJohn Fastabend send = msg->sg.size; 773d3b18ad3SJohn Fastabend if (msg->apply_bytes && msg->apply_bytes < send) 774d3b18ad3SJohn Fastabend send = msg->apply_bytes; 775a42055e8SVakul Garg 776d3b18ad3SJohn Fastabend switch (psock->eval) { 777d3b18ad3SJohn Fastabend case __SK_PASS: 778d3b18ad3SJohn Fastabend err = tls_push_record(sk, flags, record_type); 779d3b18ad3SJohn Fastabend if (err < 0) { 780d3b18ad3SJohn Fastabend *copied -= sk_msg_free(sk, msg); 781d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 782d3b18ad3SJohn Fastabend goto out_err; 783d3b18ad3SJohn Fastabend } 784d3b18ad3SJohn Fastabend break; 785d3b18ad3SJohn Fastabend case __SK_REDIRECT: 786d3b18ad3SJohn Fastabend sk_redir = psock->sk_redir; 787d3b18ad3SJohn Fastabend memcpy(&msg_redir, msg, sizeof(*msg)); 788d3b18ad3SJohn Fastabend if (msg->apply_bytes < send) 789d3b18ad3SJohn Fastabend msg->apply_bytes = 0; 790d3b18ad3SJohn Fastabend else 791d3b18ad3SJohn Fastabend msg->apply_bytes -= send; 792d3b18ad3SJohn Fastabend sk_msg_return_zero(sk, msg, send); 793d3b18ad3SJohn Fastabend msg->sg.size -= send; 794d3b18ad3SJohn Fastabend release_sock(sk); 795d3b18ad3SJohn Fastabend err = tcp_bpf_sendmsg_redir(sk_redir, &msg_redir, send, flags); 796d3b18ad3SJohn Fastabend lock_sock(sk); 797d3b18ad3SJohn Fastabend if (err < 0) { 798d3b18ad3SJohn Fastabend *copied -= sk_msg_free_nocharge(sk, &msg_redir); 799d3b18ad3SJohn Fastabend msg->sg.size = 0; 800d3b18ad3SJohn Fastabend } 801d3b18ad3SJohn Fastabend if (msg->sg.size == 0) 802d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 803d3b18ad3SJohn Fastabend break; 804d3b18ad3SJohn Fastabend case __SK_DROP: 805d3b18ad3SJohn Fastabend default: 806d3b18ad3SJohn Fastabend sk_msg_free_partial(sk, msg, send); 807d3b18ad3SJohn Fastabend if (msg->apply_bytes < send) 808d3b18ad3SJohn Fastabend msg->apply_bytes = 0; 809d3b18ad3SJohn Fastabend else 810d3b18ad3SJohn Fastabend msg->apply_bytes -= send; 811d3b18ad3SJohn Fastabend if (msg->sg.size == 0) 812d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 8137246d8edSJohn Fastabend *copied -= (send + delta); 814d3b18ad3SJohn Fastabend err = -EACCES; 815d3b18ad3SJohn Fastabend } 816a42055e8SVakul Garg 817d3b18ad3SJohn Fastabend if (likely(!err)) { 818d3b18ad3SJohn Fastabend bool reset_eval = !ctx->open_rec; 819d3b18ad3SJohn Fastabend 820d3b18ad3SJohn Fastabend rec = ctx->open_rec; 821d3b18ad3SJohn Fastabend if (rec) { 822d3b18ad3SJohn Fastabend msg = &rec->msg_plaintext; 823d3b18ad3SJohn Fastabend if (!msg->apply_bytes) 824d3b18ad3SJohn Fastabend reset_eval = true; 825d3b18ad3SJohn Fastabend } 826d3b18ad3SJohn Fastabend if (reset_eval) { 827d3b18ad3SJohn Fastabend psock->eval = __SK_NONE; 828d3b18ad3SJohn Fastabend if (psock->sk_redir) { 829d3b18ad3SJohn Fastabend sock_put(psock->sk_redir); 830d3b18ad3SJohn Fastabend psock->sk_redir = NULL; 831d3b18ad3SJohn Fastabend } 832d3b18ad3SJohn Fastabend } 833d3b18ad3SJohn Fastabend if (rec) 834d3b18ad3SJohn Fastabend goto more_data; 835d3b18ad3SJohn Fastabend } 836d3b18ad3SJohn Fastabend out_err: 837d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 838d3b18ad3SJohn Fastabend return err; 839d3b18ad3SJohn Fastabend } 840d3b18ad3SJohn Fastabend 841d3b18ad3SJohn Fastabend static int tls_sw_push_pending_record(struct sock *sk, int flags) 842d3b18ad3SJohn Fastabend { 843d3b18ad3SJohn Fastabend struct tls_context *tls_ctx = tls_get_ctx(sk); 844d3b18ad3SJohn Fastabend struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 845d3b18ad3SJohn Fastabend struct tls_rec *rec = ctx->open_rec; 846d3b18ad3SJohn Fastabend struct sk_msg *msg_pl; 847d3b18ad3SJohn Fastabend size_t copied; 848d3b18ad3SJohn Fastabend 849a42055e8SVakul Garg if (!rec) 850d3b18ad3SJohn Fastabend return 0; 851a42055e8SVakul Garg 852d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 853d3b18ad3SJohn Fastabend copied = msg_pl->sg.size; 854d3b18ad3SJohn Fastabend if (!copied) 855d3b18ad3SJohn Fastabend return 0; 856a42055e8SVakul Garg 857d3b18ad3SJohn Fastabend return bpf_exec_tx_verdict(msg_pl, sk, true, TLS_RECORD_TYPE_DATA, 858d3b18ad3SJohn Fastabend &copied, flags); 859a42055e8SVakul Garg } 860a42055e8SVakul Garg 861a42055e8SVakul Garg int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 862a42055e8SVakul Garg { 8633c4d7559SDave Watson long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 864a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 8654509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 866a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 8675b053e12SDave Watson bool async_capable = ctx->async_capable; 868a42055e8SVakul Garg unsigned char record_type = TLS_RECORD_TYPE_DATA; 86900e23707SDavid Howells bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); 8703c4d7559SDave Watson bool eor = !(msg->msg_flags & MSG_MORE); 8713c4d7559SDave Watson size_t try_to_copy, copied = 0; 872d829e9c4SDaniel Borkmann struct sk_msg *msg_pl, *msg_en; 873a42055e8SVakul Garg struct tls_rec *rec; 874a42055e8SVakul Garg int required_size; 875a42055e8SVakul Garg int num_async = 0; 8763c4d7559SDave Watson bool full_record; 877a42055e8SVakul Garg int record_room; 878a42055e8SVakul Garg int num_zc = 0; 8793c4d7559SDave Watson int orig_size; 8804128c0cfSVakul Garg int ret = 0; 8813c4d7559SDave Watson 8823c4d7559SDave Watson if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 8833c4d7559SDave Watson return -ENOTSUPP; 8843c4d7559SDave Watson 8853c4d7559SDave Watson lock_sock(sk); 8863c4d7559SDave Watson 887a42055e8SVakul Garg /* Wait till there is any pending write on socket */ 888a42055e8SVakul Garg if (unlikely(sk->sk_write_pending)) { 889a42055e8SVakul Garg ret = wait_on_pending_writer(sk, &timeo); 890a42055e8SVakul Garg if (unlikely(ret)) 8913c4d7559SDave Watson goto send_end; 892a42055e8SVakul Garg } 8933c4d7559SDave Watson 8943c4d7559SDave Watson if (unlikely(msg->msg_controllen)) { 8953c4d7559SDave Watson ret = tls_proccess_cmsg(sk, msg, &record_type); 896a42055e8SVakul Garg if (ret) { 897a42055e8SVakul Garg if (ret == -EINPROGRESS) 898a42055e8SVakul Garg num_async++; 899a42055e8SVakul Garg else if (ret != -EAGAIN) 9003c4d7559SDave Watson goto send_end; 9013c4d7559SDave Watson } 902a42055e8SVakul Garg } 9033c4d7559SDave Watson 9043c4d7559SDave Watson while (msg_data_left(msg)) { 9053c4d7559SDave Watson if (sk->sk_err) { 90630be8f8dSr.hering@avm.de ret = -sk->sk_err; 9073c4d7559SDave Watson goto send_end; 9083c4d7559SDave Watson } 9093c4d7559SDave Watson 910d3b18ad3SJohn Fastabend if (ctx->open_rec) 911d3b18ad3SJohn Fastabend rec = ctx->open_rec; 912d3b18ad3SJohn Fastabend else 913d3b18ad3SJohn Fastabend rec = ctx->open_rec = tls_get_rec(sk); 914a42055e8SVakul Garg if (!rec) { 915a42055e8SVakul Garg ret = -ENOMEM; 916a42055e8SVakul Garg goto send_end; 917a42055e8SVakul Garg } 918a42055e8SVakul Garg 919d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 920d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 921d829e9c4SDaniel Borkmann 922d829e9c4SDaniel Borkmann orig_size = msg_pl->sg.size; 9233c4d7559SDave Watson full_record = false; 9243c4d7559SDave Watson try_to_copy = msg_data_left(msg); 925d829e9c4SDaniel Borkmann record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size; 9263c4d7559SDave Watson if (try_to_copy >= record_room) { 9273c4d7559SDave Watson try_to_copy = record_room; 9283c4d7559SDave Watson full_record = true; 9293c4d7559SDave Watson } 9303c4d7559SDave Watson 931d829e9c4SDaniel Borkmann required_size = msg_pl->sg.size + try_to_copy + 9324509de14SVakul Garg prot->overhead_size; 9333c4d7559SDave Watson 9343c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 9353c4d7559SDave Watson goto wait_for_sndbuf; 936a42055e8SVakul Garg 9373c4d7559SDave Watson alloc_encrypted: 938d829e9c4SDaniel Borkmann ret = tls_alloc_encrypted_msg(sk, required_size); 9393c4d7559SDave Watson if (ret) { 9403c4d7559SDave Watson if (ret != -ENOSPC) 9413c4d7559SDave Watson goto wait_for_memory; 9423c4d7559SDave Watson 9433c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 9443c4d7559SDave Watson * actually allocated. The difference is due 9453c4d7559SDave Watson * to max sg elements limit 9463c4d7559SDave Watson */ 947d829e9c4SDaniel Borkmann try_to_copy -= required_size - msg_en->sg.size; 9483c4d7559SDave Watson full_record = true; 9493c4d7559SDave Watson } 950a42055e8SVakul Garg 951a42055e8SVakul Garg if (!is_kvec && (full_record || eor) && !async_capable) { 952d3b18ad3SJohn Fastabend u32 first = msg_pl->sg.end; 953d3b18ad3SJohn Fastabend 954d829e9c4SDaniel Borkmann ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter, 955d829e9c4SDaniel Borkmann msg_pl, try_to_copy); 9563c4d7559SDave Watson if (ret) 9573c4d7559SDave Watson goto fallback_to_reg_send; 9583c4d7559SDave Watson 9594e6d4720SVakul Garg rec->inplace_crypto = 0; 9604e6d4720SVakul Garg 961a42055e8SVakul Garg num_zc++; 9623c4d7559SDave Watson copied += try_to_copy; 963d3b18ad3SJohn Fastabend 964d3b18ad3SJohn Fastabend sk_msg_sg_copy_set(msg_pl, first); 965d3b18ad3SJohn Fastabend ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 966d3b18ad3SJohn Fastabend record_type, &copied, 967d3b18ad3SJohn Fastabend msg->msg_flags); 968a42055e8SVakul Garg if (ret) { 969a42055e8SVakul Garg if (ret == -EINPROGRESS) 970a42055e8SVakul Garg num_async++; 971d3b18ad3SJohn Fastabend else if (ret == -ENOMEM) 972d3b18ad3SJohn Fastabend goto wait_for_memory; 973d3b18ad3SJohn Fastabend else if (ret == -ENOSPC) 974d3b18ad3SJohn Fastabend goto rollback_iter; 975a42055e8SVakul Garg else if (ret != -EAGAIN) 9763c4d7559SDave Watson goto send_end; 977a42055e8SVakul Garg } 9785a3611efSDoron Roberts-Kedes continue; 979d3b18ad3SJohn Fastabend rollback_iter: 980d3b18ad3SJohn Fastabend copied -= try_to_copy; 981d3b18ad3SJohn Fastabend sk_msg_sg_copy_clear(msg_pl, first); 982d3b18ad3SJohn Fastabend iov_iter_revert(&msg->msg_iter, 983d3b18ad3SJohn Fastabend msg_pl->sg.size - orig_size); 9843c4d7559SDave Watson fallback_to_reg_send: 985d829e9c4SDaniel Borkmann sk_msg_trim(sk, msg_pl, orig_size); 9863c4d7559SDave Watson } 9873c4d7559SDave Watson 988d829e9c4SDaniel Borkmann required_size = msg_pl->sg.size + try_to_copy; 9894e6d4720SVakul Garg 990d829e9c4SDaniel Borkmann ret = tls_clone_plaintext_msg(sk, required_size); 9913c4d7559SDave Watson if (ret) { 9923c4d7559SDave Watson if (ret != -ENOSPC) 9934e6d4720SVakul Garg goto send_end; 9943c4d7559SDave Watson 9953c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 9963c4d7559SDave Watson * actually allocated. The difference is due 9973c4d7559SDave Watson * to max sg elements limit 9983c4d7559SDave Watson */ 999d829e9c4SDaniel Borkmann try_to_copy -= required_size - msg_pl->sg.size; 10003c4d7559SDave Watson full_record = true; 10014509de14SVakul Garg sk_msg_trim(sk, msg_en, 10024509de14SVakul Garg msg_pl->sg.size + prot->overhead_size); 10033c4d7559SDave Watson } 10043c4d7559SDave Watson 100565a10e28SVakul Garg if (try_to_copy) { 100665a10e28SVakul Garg ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, 100765a10e28SVakul Garg msg_pl, try_to_copy); 1008d829e9c4SDaniel Borkmann if (ret < 0) 10093c4d7559SDave Watson goto trim_sgl; 101065a10e28SVakul Garg } 10113c4d7559SDave Watson 1012d829e9c4SDaniel Borkmann /* Open records defined only if successfully copied, otherwise 1013d829e9c4SDaniel Borkmann * we would trim the sg but not reset the open record frags. 1014d829e9c4SDaniel Borkmann */ 1015d829e9c4SDaniel Borkmann tls_ctx->pending_open_record_frags = true; 10163c4d7559SDave Watson copied += try_to_copy; 10173c4d7559SDave Watson if (full_record || eor) { 1018d3b18ad3SJohn Fastabend ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 1019d3b18ad3SJohn Fastabend record_type, &copied, 1020d3b18ad3SJohn Fastabend msg->msg_flags); 10213c4d7559SDave Watson if (ret) { 1022a42055e8SVakul Garg if (ret == -EINPROGRESS) 1023a42055e8SVakul Garg num_async++; 1024d3b18ad3SJohn Fastabend else if (ret == -ENOMEM) 1025d3b18ad3SJohn Fastabend goto wait_for_memory; 1026d3b18ad3SJohn Fastabend else if (ret != -EAGAIN) { 1027d3b18ad3SJohn Fastabend if (ret == -ENOSPC) 1028d3b18ad3SJohn Fastabend ret = 0; 10293c4d7559SDave Watson goto send_end; 10303c4d7559SDave Watson } 10313c4d7559SDave Watson } 1032d3b18ad3SJohn Fastabend } 10333c4d7559SDave Watson 10343c4d7559SDave Watson continue; 10353c4d7559SDave Watson 10363c4d7559SDave Watson wait_for_sndbuf: 10373c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 10383c4d7559SDave Watson wait_for_memory: 10393c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 10403c4d7559SDave Watson if (ret) { 10413c4d7559SDave Watson trim_sgl: 1042d829e9c4SDaniel Borkmann tls_trim_both_msgs(sk, orig_size); 10433c4d7559SDave Watson goto send_end; 10443c4d7559SDave Watson } 10453c4d7559SDave Watson 1046d829e9c4SDaniel Borkmann if (msg_en->sg.size < required_size) 10473c4d7559SDave Watson goto alloc_encrypted; 10483c4d7559SDave Watson } 10493c4d7559SDave Watson 1050a42055e8SVakul Garg if (!num_async) { 1051a42055e8SVakul Garg goto send_end; 1052a42055e8SVakul Garg } else if (num_zc) { 1053a42055e8SVakul Garg /* Wait for pending encryptions to get completed */ 1054a42055e8SVakul Garg smp_store_mb(ctx->async_notify, true); 1055a42055e8SVakul Garg 1056a42055e8SVakul Garg if (atomic_read(&ctx->encrypt_pending)) 1057a42055e8SVakul Garg crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 1058a42055e8SVakul Garg else 1059a42055e8SVakul Garg reinit_completion(&ctx->async_wait.completion); 1060a42055e8SVakul Garg 1061a42055e8SVakul Garg WRITE_ONCE(ctx->async_notify, false); 1062a42055e8SVakul Garg 1063a42055e8SVakul Garg if (ctx->async_wait.err) { 1064a42055e8SVakul Garg ret = ctx->async_wait.err; 1065a42055e8SVakul Garg copied = 0; 1066a42055e8SVakul Garg } 1067a42055e8SVakul Garg } 1068a42055e8SVakul Garg 1069a42055e8SVakul Garg /* Transmit if any encryptions have completed */ 1070a42055e8SVakul Garg if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 1071a42055e8SVakul Garg cancel_delayed_work(&ctx->tx_work.work); 1072a42055e8SVakul Garg tls_tx_records(sk, msg->msg_flags); 1073a42055e8SVakul Garg } 1074a42055e8SVakul Garg 10753c4d7559SDave Watson send_end: 10763c4d7559SDave Watson ret = sk_stream_error(sk, msg->msg_flags, ret); 10773c4d7559SDave Watson 10783c4d7559SDave Watson release_sock(sk); 10793c4d7559SDave Watson return copied ? copied : ret; 10803c4d7559SDave Watson } 10813c4d7559SDave Watson 108201cb8a1aSYueHaibing static int tls_sw_do_sendpage(struct sock *sk, struct page *page, 10833c4d7559SDave Watson int offset, size_t size, int flags) 10843c4d7559SDave Watson { 1085a42055e8SVakul Garg long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 10863c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1087f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 10884509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 10893c4d7559SDave Watson unsigned char record_type = TLS_RECORD_TYPE_DATA; 1090d829e9c4SDaniel Borkmann struct sk_msg *msg_pl; 1091a42055e8SVakul Garg struct tls_rec *rec; 1092a42055e8SVakul Garg int num_async = 0; 1093d3b18ad3SJohn Fastabend size_t copied = 0; 10943c4d7559SDave Watson bool full_record; 10953c4d7559SDave Watson int record_room; 10964128c0cfSVakul Garg int ret = 0; 1097a42055e8SVakul Garg bool eor; 10983c4d7559SDave Watson 10993c4d7559SDave Watson eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST)); 11003c4d7559SDave Watson sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 11013c4d7559SDave Watson 1102a42055e8SVakul Garg /* Wait till there is any pending write on socket */ 1103a42055e8SVakul Garg if (unlikely(sk->sk_write_pending)) { 1104a42055e8SVakul Garg ret = wait_on_pending_writer(sk, &timeo); 1105a42055e8SVakul Garg if (unlikely(ret)) 11063c4d7559SDave Watson goto sendpage_end; 1107a42055e8SVakul Garg } 11083c4d7559SDave Watson 11093c4d7559SDave Watson /* Call the sk_stream functions to manage the sndbuf mem. */ 11103c4d7559SDave Watson while (size > 0) { 11113c4d7559SDave Watson size_t copy, required_size; 11123c4d7559SDave Watson 11133c4d7559SDave Watson if (sk->sk_err) { 111430be8f8dSr.hering@avm.de ret = -sk->sk_err; 11153c4d7559SDave Watson goto sendpage_end; 11163c4d7559SDave Watson } 11173c4d7559SDave Watson 1118d3b18ad3SJohn Fastabend if (ctx->open_rec) 1119d3b18ad3SJohn Fastabend rec = ctx->open_rec; 1120d3b18ad3SJohn Fastabend else 1121d3b18ad3SJohn Fastabend rec = ctx->open_rec = tls_get_rec(sk); 1122a42055e8SVakul Garg if (!rec) { 1123a42055e8SVakul Garg ret = -ENOMEM; 1124a42055e8SVakul Garg goto sendpage_end; 1125a42055e8SVakul Garg } 1126a42055e8SVakul Garg 1127d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 1128d829e9c4SDaniel Borkmann 11293c4d7559SDave Watson full_record = false; 1130d829e9c4SDaniel Borkmann record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size; 1131d3b18ad3SJohn Fastabend copied = 0; 11323c4d7559SDave Watson copy = size; 11333c4d7559SDave Watson if (copy >= record_room) { 11343c4d7559SDave Watson copy = record_room; 11353c4d7559SDave Watson full_record = true; 11363c4d7559SDave Watson } 1137d829e9c4SDaniel Borkmann 11384509de14SVakul Garg required_size = msg_pl->sg.size + copy + prot->overhead_size; 11393c4d7559SDave Watson 11403c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 11413c4d7559SDave Watson goto wait_for_sndbuf; 11423c4d7559SDave Watson alloc_payload: 1143d829e9c4SDaniel Borkmann ret = tls_alloc_encrypted_msg(sk, required_size); 11443c4d7559SDave Watson if (ret) { 11453c4d7559SDave Watson if (ret != -ENOSPC) 11463c4d7559SDave Watson goto wait_for_memory; 11473c4d7559SDave Watson 11483c4d7559SDave Watson /* Adjust copy according to the amount that was 11493c4d7559SDave Watson * actually allocated. The difference is due 11503c4d7559SDave Watson * to max sg elements limit 11513c4d7559SDave Watson */ 1152d829e9c4SDaniel Borkmann copy -= required_size - msg_pl->sg.size; 11533c4d7559SDave Watson full_record = true; 11543c4d7559SDave Watson } 11553c4d7559SDave Watson 1156d829e9c4SDaniel Borkmann sk_msg_page_add(msg_pl, page, copy, offset); 11573c4d7559SDave Watson sk_mem_charge(sk, copy); 1158d829e9c4SDaniel Borkmann 11593c4d7559SDave Watson offset += copy; 11603c4d7559SDave Watson size -= copy; 1161d3b18ad3SJohn Fastabend copied += copy; 11623c4d7559SDave Watson 1163d829e9c4SDaniel Borkmann tls_ctx->pending_open_record_frags = true; 1164d829e9c4SDaniel Borkmann if (full_record || eor || sk_msg_full(msg_pl)) { 11654e6d4720SVakul Garg rec->inplace_crypto = 0; 1166d3b18ad3SJohn Fastabend ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 1167d3b18ad3SJohn Fastabend record_type, &copied, flags); 11683c4d7559SDave Watson if (ret) { 1169a42055e8SVakul Garg if (ret == -EINPROGRESS) 1170a42055e8SVakul Garg num_async++; 1171d3b18ad3SJohn Fastabend else if (ret == -ENOMEM) 1172d3b18ad3SJohn Fastabend goto wait_for_memory; 1173d3b18ad3SJohn Fastabend else if (ret != -EAGAIN) { 1174d3b18ad3SJohn Fastabend if (ret == -ENOSPC) 1175d3b18ad3SJohn Fastabend ret = 0; 11763c4d7559SDave Watson goto sendpage_end; 11773c4d7559SDave Watson } 11783c4d7559SDave Watson } 1179d3b18ad3SJohn Fastabend } 11803c4d7559SDave Watson continue; 11813c4d7559SDave Watson wait_for_sndbuf: 11823c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 11833c4d7559SDave Watson wait_for_memory: 11843c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 11853c4d7559SDave Watson if (ret) { 1186d829e9c4SDaniel Borkmann tls_trim_both_msgs(sk, msg_pl->sg.size); 11873c4d7559SDave Watson goto sendpage_end; 11883c4d7559SDave Watson } 11893c4d7559SDave Watson 11903c4d7559SDave Watson goto alloc_payload; 11913c4d7559SDave Watson } 11923c4d7559SDave Watson 1193a42055e8SVakul Garg if (num_async) { 1194a42055e8SVakul Garg /* Transmit if any encryptions have completed */ 1195a42055e8SVakul Garg if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 1196a42055e8SVakul Garg cancel_delayed_work(&ctx->tx_work.work); 1197a42055e8SVakul Garg tls_tx_records(sk, flags); 1198a42055e8SVakul Garg } 1199a42055e8SVakul Garg } 12003c4d7559SDave Watson sendpage_end: 12013c4d7559SDave Watson ret = sk_stream_error(sk, flags, ret); 1202d3b18ad3SJohn Fastabend return copied ? copied : ret; 12033c4d7559SDave Watson } 12043c4d7559SDave Watson 12050608c69cSJohn Fastabend int tls_sw_sendpage(struct sock *sk, struct page *page, 12060608c69cSJohn Fastabend int offset, size_t size, int flags) 12070608c69cSJohn Fastabend { 12080608c69cSJohn Fastabend int ret; 12090608c69cSJohn Fastabend 12100608c69cSJohn Fastabend if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 12110608c69cSJohn Fastabend MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY)) 12120608c69cSJohn Fastabend return -ENOTSUPP; 12130608c69cSJohn Fastabend 12140608c69cSJohn Fastabend lock_sock(sk); 12150608c69cSJohn Fastabend ret = tls_sw_do_sendpage(sk, page, offset, size, flags); 12160608c69cSJohn Fastabend release_sock(sk); 12170608c69cSJohn Fastabend return ret; 12180608c69cSJohn Fastabend } 12190608c69cSJohn Fastabend 1220d3b18ad3SJohn Fastabend static struct sk_buff *tls_wait_data(struct sock *sk, struct sk_psock *psock, 1221d3b18ad3SJohn Fastabend int flags, long timeo, int *err) 1222c46234ebSDave Watson { 1223c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1224f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1225c46234ebSDave Watson struct sk_buff *skb; 1226c46234ebSDave Watson DEFINE_WAIT_FUNC(wait, woken_wake_function); 1227c46234ebSDave Watson 1228d3b18ad3SJohn Fastabend while (!(skb = ctx->recv_pkt) && sk_psock_queue_empty(psock)) { 1229c46234ebSDave Watson if (sk->sk_err) { 1230c46234ebSDave Watson *err = sock_error(sk); 1231c46234ebSDave Watson return NULL; 1232c46234ebSDave Watson } 1233c46234ebSDave Watson 1234fcf4793eSDoron Roberts-Kedes if (sk->sk_shutdown & RCV_SHUTDOWN) 1235fcf4793eSDoron Roberts-Kedes return NULL; 1236fcf4793eSDoron Roberts-Kedes 1237c46234ebSDave Watson if (sock_flag(sk, SOCK_DONE)) 1238c46234ebSDave Watson return NULL; 1239c46234ebSDave Watson 1240c46234ebSDave Watson if ((flags & MSG_DONTWAIT) || !timeo) { 1241c46234ebSDave Watson *err = -EAGAIN; 1242c46234ebSDave Watson return NULL; 1243c46234ebSDave Watson } 1244c46234ebSDave Watson 1245c46234ebSDave Watson add_wait_queue(sk_sleep(sk), &wait); 1246c46234ebSDave Watson sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1247d3b18ad3SJohn Fastabend sk_wait_event(sk, &timeo, 1248d3b18ad3SJohn Fastabend ctx->recv_pkt != skb || 1249d3b18ad3SJohn Fastabend !sk_psock_queue_empty(psock), 1250d3b18ad3SJohn Fastabend &wait); 1251c46234ebSDave Watson sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1252c46234ebSDave Watson remove_wait_queue(sk_sleep(sk), &wait); 1253c46234ebSDave Watson 1254c46234ebSDave Watson /* Handle signals */ 1255c46234ebSDave Watson if (signal_pending(current)) { 1256c46234ebSDave Watson *err = sock_intr_errno(timeo); 1257c46234ebSDave Watson return NULL; 1258c46234ebSDave Watson } 1259c46234ebSDave Watson } 1260c46234ebSDave Watson 1261c46234ebSDave Watson return skb; 1262c46234ebSDave Watson } 1263c46234ebSDave Watson 1264d829e9c4SDaniel Borkmann static int tls_setup_from_iter(struct sock *sk, struct iov_iter *from, 1265d829e9c4SDaniel Borkmann int length, int *pages_used, 1266d829e9c4SDaniel Borkmann unsigned int *size_used, 1267d829e9c4SDaniel Borkmann struct scatterlist *to, 1268d829e9c4SDaniel Borkmann int to_max_pages) 1269d829e9c4SDaniel Borkmann { 1270d829e9c4SDaniel Borkmann int rc = 0, i = 0, num_elem = *pages_used, maxpages; 1271d829e9c4SDaniel Borkmann struct page *pages[MAX_SKB_FRAGS]; 1272d829e9c4SDaniel Borkmann unsigned int size = *size_used; 1273d829e9c4SDaniel Borkmann ssize_t copied, use; 1274d829e9c4SDaniel Borkmann size_t offset; 1275d829e9c4SDaniel Borkmann 1276d829e9c4SDaniel Borkmann while (length > 0) { 1277d829e9c4SDaniel Borkmann i = 0; 1278d829e9c4SDaniel Borkmann maxpages = to_max_pages - num_elem; 1279d829e9c4SDaniel Borkmann if (maxpages == 0) { 1280d829e9c4SDaniel Borkmann rc = -EFAULT; 1281d829e9c4SDaniel Borkmann goto out; 1282d829e9c4SDaniel Borkmann } 1283d829e9c4SDaniel Borkmann copied = iov_iter_get_pages(from, pages, 1284d829e9c4SDaniel Borkmann length, 1285d829e9c4SDaniel Borkmann maxpages, &offset); 1286d829e9c4SDaniel Borkmann if (copied <= 0) { 1287d829e9c4SDaniel Borkmann rc = -EFAULT; 1288d829e9c4SDaniel Borkmann goto out; 1289d829e9c4SDaniel Borkmann } 1290d829e9c4SDaniel Borkmann 1291d829e9c4SDaniel Borkmann iov_iter_advance(from, copied); 1292d829e9c4SDaniel Borkmann 1293d829e9c4SDaniel Borkmann length -= copied; 1294d829e9c4SDaniel Borkmann size += copied; 1295d829e9c4SDaniel Borkmann while (copied) { 1296d829e9c4SDaniel Borkmann use = min_t(int, copied, PAGE_SIZE - offset); 1297d829e9c4SDaniel Borkmann 1298d829e9c4SDaniel Borkmann sg_set_page(&to[num_elem], 1299d829e9c4SDaniel Borkmann pages[i], use, offset); 1300d829e9c4SDaniel Borkmann sg_unmark_end(&to[num_elem]); 1301d829e9c4SDaniel Borkmann /* We do not uncharge memory from this API */ 1302d829e9c4SDaniel Borkmann 1303d829e9c4SDaniel Borkmann offset = 0; 1304d829e9c4SDaniel Borkmann copied -= use; 1305d829e9c4SDaniel Borkmann 1306d829e9c4SDaniel Borkmann i++; 1307d829e9c4SDaniel Borkmann num_elem++; 1308d829e9c4SDaniel Borkmann } 1309d829e9c4SDaniel Borkmann } 1310d829e9c4SDaniel Borkmann /* Mark the end in the last sg entry if newly added */ 1311d829e9c4SDaniel Borkmann if (num_elem > *pages_used) 1312d829e9c4SDaniel Borkmann sg_mark_end(&to[num_elem - 1]); 1313d829e9c4SDaniel Borkmann out: 1314d829e9c4SDaniel Borkmann if (rc) 1315d829e9c4SDaniel Borkmann iov_iter_revert(from, size - *size_used); 1316d829e9c4SDaniel Borkmann *size_used = size; 1317d829e9c4SDaniel Borkmann *pages_used = num_elem; 1318d829e9c4SDaniel Borkmann 1319d829e9c4SDaniel Borkmann return rc; 1320d829e9c4SDaniel Borkmann } 1321d829e9c4SDaniel Borkmann 13220b243d00SVakul Garg /* This function decrypts the input skb into either out_iov or in out_sg 13230b243d00SVakul Garg * or in skb buffers itself. The input parameter 'zc' indicates if 13240b243d00SVakul Garg * zero-copy mode needs to be tried or not. With zero-copy mode, either 13250b243d00SVakul Garg * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are 13260b243d00SVakul Garg * NULL, then the decryption happens inside skb buffers itself, i.e. 13270b243d00SVakul Garg * zero-copy gets disabled and 'zc' is updated. 13280b243d00SVakul Garg */ 13290b243d00SVakul Garg 13300b243d00SVakul Garg static int decrypt_internal(struct sock *sk, struct sk_buff *skb, 13310b243d00SVakul Garg struct iov_iter *out_iov, 13320b243d00SVakul Garg struct scatterlist *out_sg, 1333692d7b5dSVakul Garg int *chunk, bool *zc, bool async) 13340b243d00SVakul Garg { 13350b243d00SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 13360b243d00SVakul Garg struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 13374509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 13380b243d00SVakul Garg struct strp_msg *rxm = strp_msg(skb); 13390b243d00SVakul Garg int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0; 13400b243d00SVakul Garg struct aead_request *aead_req; 13410b243d00SVakul Garg struct sk_buff *unused; 13420b243d00SVakul Garg u8 *aad, *iv, *mem = NULL; 13430b243d00SVakul Garg struct scatterlist *sgin = NULL; 13440b243d00SVakul Garg struct scatterlist *sgout = NULL; 13454509de14SVakul Garg const int data_len = rxm->full_len - prot->overhead_size + 13464509de14SVakul Garg prot->tail_size; 13470b243d00SVakul Garg 13480b243d00SVakul Garg if (*zc && (out_iov || out_sg)) { 13490b243d00SVakul Garg if (out_iov) 13500b243d00SVakul Garg n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1; 13510b243d00SVakul Garg else 13520b243d00SVakul Garg n_sgout = sg_nents(out_sg); 13534509de14SVakul Garg n_sgin = skb_nsg(skb, rxm->offset + prot->prepend_size, 13544509de14SVakul Garg rxm->full_len - prot->prepend_size); 13550b243d00SVakul Garg } else { 13560b243d00SVakul Garg n_sgout = 0; 13570b243d00SVakul Garg *zc = false; 13580927f71dSDoron Roberts-Kedes n_sgin = skb_cow_data(skb, 0, &unused); 13590b243d00SVakul Garg } 13600b243d00SVakul Garg 13610b243d00SVakul Garg if (n_sgin < 1) 13620b243d00SVakul Garg return -EBADMSG; 13630b243d00SVakul Garg 13640b243d00SVakul Garg /* Increment to accommodate AAD */ 13650b243d00SVakul Garg n_sgin = n_sgin + 1; 13660b243d00SVakul Garg 13670b243d00SVakul Garg nsg = n_sgin + n_sgout; 13680b243d00SVakul Garg 13690b243d00SVakul Garg aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv); 13700b243d00SVakul Garg mem_size = aead_size + (nsg * sizeof(struct scatterlist)); 13714509de14SVakul Garg mem_size = mem_size + prot->aad_size; 13720b243d00SVakul Garg mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv); 13730b243d00SVakul Garg 13740b243d00SVakul Garg /* Allocate a single block of memory which contains 13750b243d00SVakul Garg * aead_req || sgin[] || sgout[] || aad || iv. 13760b243d00SVakul Garg * This order achieves correct alignment for aead_req, sgin, sgout. 13770b243d00SVakul Garg */ 13780b243d00SVakul Garg mem = kmalloc(mem_size, sk->sk_allocation); 13790b243d00SVakul Garg if (!mem) 13800b243d00SVakul Garg return -ENOMEM; 13810b243d00SVakul Garg 13820b243d00SVakul Garg /* Segment the allocated memory */ 13830b243d00SVakul Garg aead_req = (struct aead_request *)mem; 13840b243d00SVakul Garg sgin = (struct scatterlist *)(mem + aead_size); 13850b243d00SVakul Garg sgout = sgin + n_sgin; 13860b243d00SVakul Garg aad = (u8 *)(sgout + n_sgout); 13874509de14SVakul Garg iv = aad + prot->aad_size; 13880b243d00SVakul Garg 13890b243d00SVakul Garg /* Prepare IV */ 13900b243d00SVakul Garg err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE, 13910b243d00SVakul Garg iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 13924509de14SVakul Garg prot->iv_size); 13930b243d00SVakul Garg if (err < 0) { 13940b243d00SVakul Garg kfree(mem); 13950b243d00SVakul Garg return err; 13960b243d00SVakul Garg } 13974509de14SVakul Garg if (prot->version == TLS_1_3_VERSION) 1398130b392cSDave Watson memcpy(iv, tls_ctx->rx.iv, crypto_aead_ivsize(ctx->aead_recv)); 1399130b392cSDave Watson else 14000b243d00SVakul Garg memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 14010b243d00SVakul Garg 14024509de14SVakul Garg xor_iv_with_seq(prot->version, iv, tls_ctx->rx.rec_seq); 1403130b392cSDave Watson 14040b243d00SVakul Garg /* Prepare AAD */ 14054509de14SVakul Garg tls_make_aad(aad, rxm->full_len - prot->overhead_size + 14064509de14SVakul Garg prot->tail_size, 14074509de14SVakul Garg tls_ctx->rx.rec_seq, prot->rec_seq_size, 14084509de14SVakul Garg ctx->control, prot->version); 14090b243d00SVakul Garg 14100b243d00SVakul Garg /* Prepare sgin */ 14110b243d00SVakul Garg sg_init_table(sgin, n_sgin); 14124509de14SVakul Garg sg_set_buf(&sgin[0], aad, prot->aad_size); 14130b243d00SVakul Garg err = skb_to_sgvec(skb, &sgin[1], 14144509de14SVakul Garg rxm->offset + prot->prepend_size, 14154509de14SVakul Garg rxm->full_len - prot->prepend_size); 14160b243d00SVakul Garg if (err < 0) { 14170b243d00SVakul Garg kfree(mem); 14180b243d00SVakul Garg return err; 14190b243d00SVakul Garg } 14200b243d00SVakul Garg 14210b243d00SVakul Garg if (n_sgout) { 14220b243d00SVakul Garg if (out_iov) { 14230b243d00SVakul Garg sg_init_table(sgout, n_sgout); 14244509de14SVakul Garg sg_set_buf(&sgout[0], aad, prot->aad_size); 14250b243d00SVakul Garg 14260b243d00SVakul Garg *chunk = 0; 1427d829e9c4SDaniel Borkmann err = tls_setup_from_iter(sk, out_iov, data_len, 1428d829e9c4SDaniel Borkmann &pages, chunk, &sgout[1], 1429d829e9c4SDaniel Borkmann (n_sgout - 1)); 14300b243d00SVakul Garg if (err < 0) 14310b243d00SVakul Garg goto fallback_to_reg_recv; 14320b243d00SVakul Garg } else if (out_sg) { 14330b243d00SVakul Garg memcpy(sgout, out_sg, n_sgout * sizeof(*sgout)); 14340b243d00SVakul Garg } else { 14350b243d00SVakul Garg goto fallback_to_reg_recv; 14360b243d00SVakul Garg } 14370b243d00SVakul Garg } else { 14380b243d00SVakul Garg fallback_to_reg_recv: 14390b243d00SVakul Garg sgout = sgin; 14400b243d00SVakul Garg pages = 0; 1441692d7b5dSVakul Garg *chunk = data_len; 14420b243d00SVakul Garg *zc = false; 14430b243d00SVakul Garg } 14440b243d00SVakul Garg 14450b243d00SVakul Garg /* Prepare and submit AEAD request */ 144694524d8fSVakul Garg err = tls_do_decryption(sk, skb, sgin, sgout, iv, 1447692d7b5dSVakul Garg data_len, aead_req, async); 144894524d8fSVakul Garg if (err == -EINPROGRESS) 144994524d8fSVakul Garg return err; 14500b243d00SVakul Garg 14510b243d00SVakul Garg /* Release the pages in case iov was mapped to pages */ 14520b243d00SVakul Garg for (; pages > 0; pages--) 14530b243d00SVakul Garg put_page(sg_page(&sgout[pages])); 14540b243d00SVakul Garg 14550b243d00SVakul Garg kfree(mem); 14560b243d00SVakul Garg return err; 14570b243d00SVakul Garg } 14580b243d00SVakul Garg 1459dafb67f3SBoris Pismenny static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, 1460692d7b5dSVakul Garg struct iov_iter *dest, int *chunk, bool *zc, 1461692d7b5dSVakul Garg bool async) 1462dafb67f3SBoris Pismenny { 1463dafb67f3SBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 1464dafb67f3SBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 14654509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 14664509de14SVakul Garg int version = prot->version; 1467dafb67f3SBoris Pismenny struct strp_msg *rxm = strp_msg(skb); 1468dafb67f3SBoris Pismenny int err = 0; 1469dafb67f3SBoris Pismenny 1470d069b780SBoris Pismenny if (!ctx->decrypted) { 14714799ac81SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 14724799ac81SBoris Pismenny err = tls_device_decrypted(sk, skb); 14734799ac81SBoris Pismenny if (err < 0) 14744799ac81SBoris Pismenny return err; 14754799ac81SBoris Pismenny #endif 1476d069b780SBoris Pismenny /* Still not decrypted after tls_device */ 14774799ac81SBoris Pismenny if (!ctx->decrypted) { 1478d069b780SBoris Pismenny err = decrypt_internal(sk, skb, dest, NULL, chunk, zc, 1479d069b780SBoris Pismenny async); 148094524d8fSVakul Garg if (err < 0) { 148194524d8fSVakul Garg if (err == -EINPROGRESS) 1482130b392cSDave Watson tls_advance_record_sn(sk, &tls_ctx->rx, 1483130b392cSDave Watson version); 148494524d8fSVakul Garg 1485dafb67f3SBoris Pismenny return err; 148694524d8fSVakul Garg } 1487c43ac97bSJakub Kicinski } else { 1488c43ac97bSJakub Kicinski *zc = false; 1489d069b780SBoris Pismenny } 1490130b392cSDave Watson 1491130b392cSDave Watson rxm->full_len -= padding_length(ctx, tls_ctx, skb); 14924509de14SVakul Garg rxm->offset += prot->prepend_size; 14934509de14SVakul Garg rxm->full_len -= prot->overhead_size; 1494130b392cSDave Watson tls_advance_record_sn(sk, &tls_ctx->rx, version); 1495dafb67f3SBoris Pismenny ctx->decrypted = true; 1496dafb67f3SBoris Pismenny ctx->saved_data_ready(sk); 1497fedf201eSDave Watson } else { 1498fedf201eSDave Watson *zc = false; 1499fedf201eSDave Watson } 1500dafb67f3SBoris Pismenny 1501dafb67f3SBoris Pismenny return err; 1502dafb67f3SBoris Pismenny } 1503dafb67f3SBoris Pismenny 1504dafb67f3SBoris Pismenny int decrypt_skb(struct sock *sk, struct sk_buff *skb, 1505c46234ebSDave Watson struct scatterlist *sgout) 1506c46234ebSDave Watson { 15070b243d00SVakul Garg bool zc = true; 15080b243d00SVakul Garg int chunk; 1509c46234ebSDave Watson 1510692d7b5dSVakul Garg return decrypt_internal(sk, skb, NULL, sgout, &chunk, &zc, false); 1511c46234ebSDave Watson } 1512c46234ebSDave Watson 1513c46234ebSDave Watson static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb, 1514c46234ebSDave Watson unsigned int len) 1515c46234ebSDave Watson { 1516c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1517f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 151894524d8fSVakul Garg 151994524d8fSVakul Garg if (skb) { 1520c46234ebSDave Watson struct strp_msg *rxm = strp_msg(skb); 1521c46234ebSDave Watson 1522c46234ebSDave Watson if (len < rxm->full_len) { 1523c46234ebSDave Watson rxm->offset += len; 1524c46234ebSDave Watson rxm->full_len -= len; 1525c46234ebSDave Watson return false; 1526c46234ebSDave Watson } 152794524d8fSVakul Garg kfree_skb(skb); 152894524d8fSVakul Garg } 1529c46234ebSDave Watson 1530c46234ebSDave Watson /* Finished with message */ 1531c46234ebSDave Watson ctx->recv_pkt = NULL; 15327170e604SDoron Roberts-Kedes __strp_unpause(&ctx->strp); 1533c46234ebSDave Watson 1534c46234ebSDave Watson return true; 1535c46234ebSDave Watson } 1536c46234ebSDave Watson 1537692d7b5dSVakul Garg /* This function traverses the rx_list in tls receive context to copies the 15382b794c40SVakul Garg * decrypted records into the buffer provided by caller zero copy is not 1539692d7b5dSVakul Garg * true. Further, the records are removed from the rx_list if it is not a peek 1540692d7b5dSVakul Garg * case and the record has been consumed completely. 1541692d7b5dSVakul Garg */ 1542692d7b5dSVakul Garg static int process_rx_list(struct tls_sw_context_rx *ctx, 1543692d7b5dSVakul Garg struct msghdr *msg, 15442b794c40SVakul Garg u8 *control, 15452b794c40SVakul Garg bool *cmsg, 1546692d7b5dSVakul Garg size_t skip, 1547692d7b5dSVakul Garg size_t len, 1548692d7b5dSVakul Garg bool zc, 1549692d7b5dSVakul Garg bool is_peek) 1550692d7b5dSVakul Garg { 1551692d7b5dSVakul Garg struct sk_buff *skb = skb_peek(&ctx->rx_list); 15522b794c40SVakul Garg u8 ctrl = *control; 15532b794c40SVakul Garg u8 msgc = *cmsg; 15542b794c40SVakul Garg struct tls_msg *tlm; 1555692d7b5dSVakul Garg ssize_t copied = 0; 1556692d7b5dSVakul Garg 15572b794c40SVakul Garg /* Set the record type in 'control' if caller didn't pass it */ 15582b794c40SVakul Garg if (!ctrl && skb) { 15592b794c40SVakul Garg tlm = tls_msg(skb); 15602b794c40SVakul Garg ctrl = tlm->control; 15612b794c40SVakul Garg } 15622b794c40SVakul Garg 1563692d7b5dSVakul Garg while (skip && skb) { 1564692d7b5dSVakul Garg struct strp_msg *rxm = strp_msg(skb); 15652b794c40SVakul Garg tlm = tls_msg(skb); 15662b794c40SVakul Garg 15672b794c40SVakul Garg /* Cannot process a record of different type */ 15682b794c40SVakul Garg if (ctrl != tlm->control) 15692b794c40SVakul Garg return 0; 1570692d7b5dSVakul Garg 1571692d7b5dSVakul Garg if (skip < rxm->full_len) 1572692d7b5dSVakul Garg break; 1573692d7b5dSVakul Garg 1574692d7b5dSVakul Garg skip = skip - rxm->full_len; 1575692d7b5dSVakul Garg skb = skb_peek_next(skb, &ctx->rx_list); 1576692d7b5dSVakul Garg } 1577692d7b5dSVakul Garg 1578692d7b5dSVakul Garg while (len && skb) { 1579692d7b5dSVakul Garg struct sk_buff *next_skb; 1580692d7b5dSVakul Garg struct strp_msg *rxm = strp_msg(skb); 1581692d7b5dSVakul Garg int chunk = min_t(unsigned int, rxm->full_len - skip, len); 1582692d7b5dSVakul Garg 15832b794c40SVakul Garg tlm = tls_msg(skb); 15842b794c40SVakul Garg 15852b794c40SVakul Garg /* Cannot process a record of different type */ 15862b794c40SVakul Garg if (ctrl != tlm->control) 15872b794c40SVakul Garg return 0; 15882b794c40SVakul Garg 15892b794c40SVakul Garg /* Set record type if not already done. For a non-data record, 15902b794c40SVakul Garg * do not proceed if record type could not be copied. 15912b794c40SVakul Garg */ 15922b794c40SVakul Garg if (!msgc) { 15932b794c40SVakul Garg int cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, 15942b794c40SVakul Garg sizeof(ctrl), &ctrl); 15952b794c40SVakul Garg msgc = true; 15962b794c40SVakul Garg if (ctrl != TLS_RECORD_TYPE_DATA) { 15972b794c40SVakul Garg if (cerr || msg->msg_flags & MSG_CTRUNC) 15982b794c40SVakul Garg return -EIO; 15992b794c40SVakul Garg 16002b794c40SVakul Garg *cmsg = msgc; 16012b794c40SVakul Garg } 16022b794c40SVakul Garg } 16032b794c40SVakul Garg 1604692d7b5dSVakul Garg if (!zc || (rxm->full_len - skip) > len) { 1605692d7b5dSVakul Garg int err = skb_copy_datagram_msg(skb, rxm->offset + skip, 1606692d7b5dSVakul Garg msg, chunk); 1607692d7b5dSVakul Garg if (err < 0) 1608692d7b5dSVakul Garg return err; 1609692d7b5dSVakul Garg } 1610692d7b5dSVakul Garg 1611692d7b5dSVakul Garg len = len - chunk; 1612692d7b5dSVakul Garg copied = copied + chunk; 1613692d7b5dSVakul Garg 1614692d7b5dSVakul Garg /* Consume the data from record if it is non-peek case*/ 1615692d7b5dSVakul Garg if (!is_peek) { 1616692d7b5dSVakul Garg rxm->offset = rxm->offset + chunk; 1617692d7b5dSVakul Garg rxm->full_len = rxm->full_len - chunk; 1618692d7b5dSVakul Garg 1619692d7b5dSVakul Garg /* Return if there is unconsumed data in the record */ 1620692d7b5dSVakul Garg if (rxm->full_len - skip) 1621692d7b5dSVakul Garg break; 1622692d7b5dSVakul Garg } 1623692d7b5dSVakul Garg 1624692d7b5dSVakul Garg /* The remaining skip-bytes must lie in 1st record in rx_list. 1625692d7b5dSVakul Garg * So from the 2nd record, 'skip' should be 0. 1626692d7b5dSVakul Garg */ 1627692d7b5dSVakul Garg skip = 0; 1628692d7b5dSVakul Garg 1629692d7b5dSVakul Garg if (msg) 1630692d7b5dSVakul Garg msg->msg_flags |= MSG_EOR; 1631692d7b5dSVakul Garg 1632692d7b5dSVakul Garg next_skb = skb_peek_next(skb, &ctx->rx_list); 1633692d7b5dSVakul Garg 1634692d7b5dSVakul Garg if (!is_peek) { 1635692d7b5dSVakul Garg skb_unlink(skb, &ctx->rx_list); 1636692d7b5dSVakul Garg kfree_skb(skb); 1637692d7b5dSVakul Garg } 1638692d7b5dSVakul Garg 1639692d7b5dSVakul Garg skb = next_skb; 1640692d7b5dSVakul Garg } 1641692d7b5dSVakul Garg 16422b794c40SVakul Garg *control = ctrl; 1643692d7b5dSVakul Garg return copied; 1644692d7b5dSVakul Garg } 1645692d7b5dSVakul Garg 1646c46234ebSDave Watson int tls_sw_recvmsg(struct sock *sk, 1647c46234ebSDave Watson struct msghdr *msg, 1648c46234ebSDave Watson size_t len, 1649c46234ebSDave Watson int nonblock, 1650c46234ebSDave Watson int flags, 1651c46234ebSDave Watson int *addr_len) 1652c46234ebSDave Watson { 1653c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1654f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 16554509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 1656d3b18ad3SJohn Fastabend struct sk_psock *psock; 1657692d7b5dSVakul Garg unsigned char control = 0; 1658692d7b5dSVakul Garg ssize_t decrypted = 0; 1659c46234ebSDave Watson struct strp_msg *rxm; 16602b794c40SVakul Garg struct tls_msg *tlm; 1661c46234ebSDave Watson struct sk_buff *skb; 1662c46234ebSDave Watson ssize_t copied = 0; 1663c46234ebSDave Watson bool cmsg = false; 166406030dbaSDaniel Borkmann int target, err = 0; 1665c46234ebSDave Watson long timeo; 166600e23707SDavid Howells bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); 1667692d7b5dSVakul Garg bool is_peek = flags & MSG_PEEK; 166894524d8fSVakul Garg int num_async = 0; 1669c46234ebSDave Watson 1670c46234ebSDave Watson flags |= nonblock; 1671c46234ebSDave Watson 1672c46234ebSDave Watson if (unlikely(flags & MSG_ERRQUEUE)) 1673c46234ebSDave Watson return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR); 1674c46234ebSDave Watson 1675d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 1676c46234ebSDave Watson lock_sock(sk); 1677c46234ebSDave Watson 1678692d7b5dSVakul Garg /* Process pending decrypted records. It must be non-zero-copy */ 16792b794c40SVakul Garg err = process_rx_list(ctx, msg, &control, &cmsg, 0, len, false, 16802b794c40SVakul Garg is_peek); 1681692d7b5dSVakul Garg if (err < 0) { 1682692d7b5dSVakul Garg tls_err_abort(sk, err); 1683692d7b5dSVakul Garg goto end; 1684692d7b5dSVakul Garg } else { 1685692d7b5dSVakul Garg copied = err; 1686692d7b5dSVakul Garg } 1687692d7b5dSVakul Garg 1688692d7b5dSVakul Garg len = len - copied; 1689692d7b5dSVakul Garg if (len) { 169006030dbaSDaniel Borkmann target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1691c46234ebSDave Watson timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 1692692d7b5dSVakul Garg } else { 1693692d7b5dSVakul Garg goto recv_end; 1694692d7b5dSVakul Garg } 1695692d7b5dSVakul Garg 1696c46234ebSDave Watson do { 1697692d7b5dSVakul Garg bool retain_skb = false; 1698692d7b5dSVakul Garg bool zc = false; 1699692d7b5dSVakul Garg int to_decrypt; 1700c46234ebSDave Watson int chunk = 0; 17017754bd63SEran Ben Elisha bool async_capable; 17027754bd63SEran Ben Elisha bool async = false; 1703c46234ebSDave Watson 1704d3b18ad3SJohn Fastabend skb = tls_wait_data(sk, psock, flags, timeo, &err); 1705d3b18ad3SJohn Fastabend if (!skb) { 1706d3b18ad3SJohn Fastabend if (psock) { 170702c558b2SJohn Fastabend int ret = __tcp_bpf_recvmsg(sk, psock, 170802c558b2SJohn Fastabend msg, len, flags); 1709d3b18ad3SJohn Fastabend 1710d3b18ad3SJohn Fastabend if (ret > 0) { 1711692d7b5dSVakul Garg decrypted += ret; 1712d3b18ad3SJohn Fastabend len -= ret; 1713d3b18ad3SJohn Fastabend continue; 1714d3b18ad3SJohn Fastabend } 1715d3b18ad3SJohn Fastabend } 1716c46234ebSDave Watson goto recv_end; 17172b794c40SVakul Garg } else { 17182b794c40SVakul Garg tlm = tls_msg(skb); 17192b794c40SVakul Garg if (prot->version == TLS_1_3_VERSION) 17202b794c40SVakul Garg tlm->control = 0; 17212b794c40SVakul Garg else 17222b794c40SVakul Garg tlm->control = ctx->control; 1723d3b18ad3SJohn Fastabend } 1724c46234ebSDave Watson 1725c46234ebSDave Watson rxm = strp_msg(skb); 172694524d8fSVakul Garg 17274509de14SVakul Garg to_decrypt = rxm->full_len - prot->overhead_size; 1728fedf201eSDave Watson 1729fedf201eSDave Watson if (to_decrypt <= len && !is_kvec && !is_peek && 1730130b392cSDave Watson ctx->control == TLS_RECORD_TYPE_DATA && 17314509de14SVakul Garg prot->version != TLS_1_3_VERSION) 1732fedf201eSDave Watson zc = true; 1733fedf201eSDave Watson 1734c0ab4732SVakul Garg /* Do not use async mode if record is non-data */ 1735c0ab4732SVakul Garg if (ctx->control == TLS_RECORD_TYPE_DATA) 17367754bd63SEran Ben Elisha async_capable = ctx->async_capable; 1737c0ab4732SVakul Garg else 17387754bd63SEran Ben Elisha async_capable = false; 1739c0ab4732SVakul Garg 1740fedf201eSDave Watson err = decrypt_skb_update(sk, skb, &msg->msg_iter, 17417754bd63SEran Ben Elisha &chunk, &zc, async_capable); 1742fedf201eSDave Watson if (err < 0 && err != -EINPROGRESS) { 1743fedf201eSDave Watson tls_err_abort(sk, EBADMSG); 1744fedf201eSDave Watson goto recv_end; 1745fedf201eSDave Watson } 1746fedf201eSDave Watson 17477754bd63SEran Ben Elisha if (err == -EINPROGRESS) { 17487754bd63SEran Ben Elisha async = true; 1749fedf201eSDave Watson num_async++; 17507754bd63SEran Ben Elisha } else if (prot->version == TLS_1_3_VERSION) { 17512b794c40SVakul Garg tlm->control = ctx->control; 17527754bd63SEran Ben Elisha } 17532b794c40SVakul Garg 17542b794c40SVakul Garg /* If the type of records being processed is not known yet, 17552b794c40SVakul Garg * set it to record type just dequeued. If it is already known, 17562b794c40SVakul Garg * but does not match the record type just dequeued, go to end. 17572b794c40SVakul Garg * We always get record type here since for tls1.2, record type 17582b794c40SVakul Garg * is known just after record is dequeued from stream parser. 17592b794c40SVakul Garg * For tls1.3, we disable async. 17602b794c40SVakul Garg */ 17612b794c40SVakul Garg 17622b794c40SVakul Garg if (!control) 17632b794c40SVakul Garg control = tlm->control; 17642b794c40SVakul Garg else if (control != tlm->control) 17652b794c40SVakul Garg goto recv_end; 1766fedf201eSDave Watson 1767c46234ebSDave Watson if (!cmsg) { 1768c46234ebSDave Watson int cerr; 1769c46234ebSDave Watson 1770c46234ebSDave Watson cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, 17712b794c40SVakul Garg sizeof(control), &control); 1772c46234ebSDave Watson cmsg = true; 17732b794c40SVakul Garg if (control != TLS_RECORD_TYPE_DATA) { 1774c46234ebSDave Watson if (cerr || msg->msg_flags & MSG_CTRUNC) { 1775c46234ebSDave Watson err = -EIO; 1776c46234ebSDave Watson goto recv_end; 1777c46234ebSDave Watson } 1778c46234ebSDave Watson } 1779c46234ebSDave Watson } 1780c46234ebSDave Watson 1781c0ab4732SVakul Garg if (async) 1782c0ab4732SVakul Garg goto pick_next_record; 1783c0ab4732SVakul Garg 1784c46234ebSDave Watson if (!zc) { 1785692d7b5dSVakul Garg if (rxm->full_len > len) { 1786692d7b5dSVakul Garg retain_skb = true; 1787692d7b5dSVakul Garg chunk = len; 1788692d7b5dSVakul Garg } else { 1789692d7b5dSVakul Garg chunk = rxm->full_len; 1790692d7b5dSVakul Garg } 179194524d8fSVakul Garg 1792692d7b5dSVakul Garg err = skb_copy_datagram_msg(skb, rxm->offset, 1793692d7b5dSVakul Garg msg, chunk); 1794c46234ebSDave Watson if (err < 0) 1795c46234ebSDave Watson goto recv_end; 1796692d7b5dSVakul Garg 1797692d7b5dSVakul Garg if (!is_peek) { 1798692d7b5dSVakul Garg rxm->offset = rxm->offset + chunk; 1799692d7b5dSVakul Garg rxm->full_len = rxm->full_len - chunk; 1800692d7b5dSVakul Garg } 1801692d7b5dSVakul Garg } 1802c46234ebSDave Watson 180394524d8fSVakul Garg pick_next_record: 1804692d7b5dSVakul Garg if (chunk > len) 1805692d7b5dSVakul Garg chunk = len; 1806c46234ebSDave Watson 1807692d7b5dSVakul Garg decrypted += chunk; 1808692d7b5dSVakul Garg len -= chunk; 1809692d7b5dSVakul Garg 1810692d7b5dSVakul Garg /* For async or peek case, queue the current skb */ 1811692d7b5dSVakul Garg if (async || is_peek || retain_skb) { 1812692d7b5dSVakul Garg skb_queue_tail(&ctx->rx_list, skb); 181394524d8fSVakul Garg skb = NULL; 1814692d7b5dSVakul Garg } 181594524d8fSVakul Garg 1816c46234ebSDave Watson if (tls_sw_advance_skb(sk, skb, chunk)) { 1817c46234ebSDave Watson /* Return full control message to 1818c46234ebSDave Watson * userspace before trying to parse 1819c46234ebSDave Watson * another message type 1820c46234ebSDave Watson */ 1821c46234ebSDave Watson msg->msg_flags |= MSG_EOR; 1822692d7b5dSVakul Garg if (ctx->control != TLS_RECORD_TYPE_DATA) 1823c46234ebSDave Watson goto recv_end; 182494524d8fSVakul Garg } else { 182594524d8fSVakul Garg break; 1826c46234ebSDave Watson } 182794524d8fSVakul Garg 182806030dbaSDaniel Borkmann /* If we have a new message from strparser, continue now. */ 1829692d7b5dSVakul Garg if (decrypted >= target && !ctx->recv_pkt) 183006030dbaSDaniel Borkmann break; 1831c46234ebSDave Watson } while (len); 1832c46234ebSDave Watson 1833c46234ebSDave Watson recv_end: 183494524d8fSVakul Garg if (num_async) { 183594524d8fSVakul Garg /* Wait for all previously submitted records to be decrypted */ 183694524d8fSVakul Garg smp_store_mb(ctx->async_notify, true); 183794524d8fSVakul Garg if (atomic_read(&ctx->decrypt_pending)) { 183894524d8fSVakul Garg err = crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 183994524d8fSVakul Garg if (err) { 184094524d8fSVakul Garg /* one of async decrypt failed */ 184194524d8fSVakul Garg tls_err_abort(sk, err); 184294524d8fSVakul Garg copied = 0; 1843692d7b5dSVakul Garg decrypted = 0; 1844692d7b5dSVakul Garg goto end; 184594524d8fSVakul Garg } 184694524d8fSVakul Garg } else { 184794524d8fSVakul Garg reinit_completion(&ctx->async_wait.completion); 184894524d8fSVakul Garg } 184994524d8fSVakul Garg WRITE_ONCE(ctx->async_notify, false); 1850692d7b5dSVakul Garg 1851692d7b5dSVakul Garg /* Drain records from the rx_list & copy if required */ 1852692d7b5dSVakul Garg if (is_peek || is_kvec) 18532b794c40SVakul Garg err = process_rx_list(ctx, msg, &control, &cmsg, copied, 1854692d7b5dSVakul Garg decrypted, false, is_peek); 1855692d7b5dSVakul Garg else 18562b794c40SVakul Garg err = process_rx_list(ctx, msg, &control, &cmsg, 0, 1857692d7b5dSVakul Garg decrypted, true, is_peek); 1858692d7b5dSVakul Garg if (err < 0) { 1859692d7b5dSVakul Garg tls_err_abort(sk, err); 1860692d7b5dSVakul Garg copied = 0; 1861692d7b5dSVakul Garg goto end; 186294524d8fSVakul Garg } 1863692d7b5dSVakul Garg } 1864692d7b5dSVakul Garg 1865692d7b5dSVakul Garg copied += decrypted; 1866692d7b5dSVakul Garg 1867692d7b5dSVakul Garg end: 1868c46234ebSDave Watson release_sock(sk); 1869d3b18ad3SJohn Fastabend if (psock) 1870d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 1871c46234ebSDave Watson return copied ? : err; 1872c46234ebSDave Watson } 1873c46234ebSDave Watson 1874c46234ebSDave Watson ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, 1875c46234ebSDave Watson struct pipe_inode_info *pipe, 1876c46234ebSDave Watson size_t len, unsigned int flags) 1877c46234ebSDave Watson { 1878c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sock->sk); 1879f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1880c46234ebSDave Watson struct strp_msg *rxm = NULL; 1881c46234ebSDave Watson struct sock *sk = sock->sk; 1882c46234ebSDave Watson struct sk_buff *skb; 1883c46234ebSDave Watson ssize_t copied = 0; 1884c46234ebSDave Watson int err = 0; 1885c46234ebSDave Watson long timeo; 1886c46234ebSDave Watson int chunk; 18870b243d00SVakul Garg bool zc = false; 1888c46234ebSDave Watson 1889c46234ebSDave Watson lock_sock(sk); 1890c46234ebSDave Watson 1891c46234ebSDave Watson timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 1892c46234ebSDave Watson 1893d3b18ad3SJohn Fastabend skb = tls_wait_data(sk, NULL, flags, timeo, &err); 1894c46234ebSDave Watson if (!skb) 1895c46234ebSDave Watson goto splice_read_end; 1896c46234ebSDave Watson 1897fedf201eSDave Watson if (!ctx->decrypted) { 1898fedf201eSDave Watson err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc, false); 1899fedf201eSDave Watson 1900c46234ebSDave Watson /* splice does not support reading control messages */ 1901c46234ebSDave Watson if (ctx->control != TLS_RECORD_TYPE_DATA) { 1902c46234ebSDave Watson err = -ENOTSUPP; 1903c46234ebSDave Watson goto splice_read_end; 1904c46234ebSDave Watson } 1905c46234ebSDave Watson 1906c46234ebSDave Watson if (err < 0) { 1907c46234ebSDave Watson tls_err_abort(sk, EBADMSG); 1908c46234ebSDave Watson goto splice_read_end; 1909c46234ebSDave Watson } 1910c46234ebSDave Watson ctx->decrypted = true; 1911c46234ebSDave Watson } 1912c46234ebSDave Watson rxm = strp_msg(skb); 1913c46234ebSDave Watson 1914c46234ebSDave Watson chunk = min_t(unsigned int, rxm->full_len, len); 1915c46234ebSDave Watson copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags); 1916c46234ebSDave Watson if (copied < 0) 1917c46234ebSDave Watson goto splice_read_end; 1918c46234ebSDave Watson 1919c46234ebSDave Watson if (likely(!(flags & MSG_PEEK))) 1920c46234ebSDave Watson tls_sw_advance_skb(sk, skb, copied); 1921c46234ebSDave Watson 1922c46234ebSDave Watson splice_read_end: 1923c46234ebSDave Watson release_sock(sk); 1924c46234ebSDave Watson return copied ? : err; 1925c46234ebSDave Watson } 1926c46234ebSDave Watson 1927924ad65eSJohn Fastabend bool tls_sw_stream_read(const struct sock *sk) 1928c46234ebSDave Watson { 1929c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1930f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1931d3b18ad3SJohn Fastabend bool ingress_empty = true; 1932d3b18ad3SJohn Fastabend struct sk_psock *psock; 1933c46234ebSDave Watson 1934d3b18ad3SJohn Fastabend rcu_read_lock(); 1935d3b18ad3SJohn Fastabend psock = sk_psock(sk); 1936d3b18ad3SJohn Fastabend if (psock) 1937d3b18ad3SJohn Fastabend ingress_empty = list_empty(&psock->ingress_msg); 1938d3b18ad3SJohn Fastabend rcu_read_unlock(); 1939c46234ebSDave Watson 1940d3b18ad3SJohn Fastabend return !ingress_empty || ctx->recv_pkt; 1941c46234ebSDave Watson } 1942c46234ebSDave Watson 1943c46234ebSDave Watson static int tls_read_size(struct strparser *strp, struct sk_buff *skb) 1944c46234ebSDave Watson { 1945c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 1946f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 19474509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 19483463e51dSKees Cook char header[TLS_HEADER_SIZE + MAX_IV_SIZE]; 1949c46234ebSDave Watson struct strp_msg *rxm = strp_msg(skb); 1950c46234ebSDave Watson size_t cipher_overhead; 1951c46234ebSDave Watson size_t data_len = 0; 1952c46234ebSDave Watson int ret; 1953c46234ebSDave Watson 1954c46234ebSDave Watson /* Verify that we have a full TLS header, or wait for more data */ 19554509de14SVakul Garg if (rxm->offset + prot->prepend_size > skb->len) 1956c46234ebSDave Watson return 0; 1957c46234ebSDave Watson 19583463e51dSKees Cook /* Sanity-check size of on-stack buffer. */ 19594509de14SVakul Garg if (WARN_ON(prot->prepend_size > sizeof(header))) { 19603463e51dSKees Cook ret = -EINVAL; 19613463e51dSKees Cook goto read_failure; 19623463e51dSKees Cook } 19633463e51dSKees Cook 1964c46234ebSDave Watson /* Linearize header to local buffer */ 19654509de14SVakul Garg ret = skb_copy_bits(skb, rxm->offset, header, prot->prepend_size); 1966c46234ebSDave Watson 1967c46234ebSDave Watson if (ret < 0) 1968c46234ebSDave Watson goto read_failure; 1969c46234ebSDave Watson 1970c46234ebSDave Watson ctx->control = header[0]; 1971c46234ebSDave Watson 1972c46234ebSDave Watson data_len = ((header[4] & 0xFF) | (header[3] << 8)); 1973c46234ebSDave Watson 19744509de14SVakul Garg cipher_overhead = prot->tag_size; 19754509de14SVakul Garg if (prot->version != TLS_1_3_VERSION) 19764509de14SVakul Garg cipher_overhead += prot->iv_size; 1977c46234ebSDave Watson 1978130b392cSDave Watson if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead + 19794509de14SVakul Garg prot->tail_size) { 1980c46234ebSDave Watson ret = -EMSGSIZE; 1981c46234ebSDave Watson goto read_failure; 1982c46234ebSDave Watson } 1983c46234ebSDave Watson if (data_len < cipher_overhead) { 1984c46234ebSDave Watson ret = -EBADMSG; 1985c46234ebSDave Watson goto read_failure; 1986c46234ebSDave Watson } 1987c46234ebSDave Watson 1988130b392cSDave Watson /* Note that both TLS1.3 and TLS1.2 use TLS_1_2 version here */ 1989130b392cSDave Watson if (header[1] != TLS_1_2_VERSION_MINOR || 1990130b392cSDave Watson header[2] != TLS_1_2_VERSION_MAJOR) { 1991c46234ebSDave Watson ret = -EINVAL; 1992c46234ebSDave Watson goto read_failure; 1993c46234ebSDave Watson } 19944799ac81SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 19954799ac81SBoris Pismenny handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset, 19964799ac81SBoris Pismenny *(u64*)tls_ctx->rx.rec_seq); 19974799ac81SBoris Pismenny #endif 1998c46234ebSDave Watson return data_len + TLS_HEADER_SIZE; 1999c46234ebSDave Watson 2000c46234ebSDave Watson read_failure: 2001c46234ebSDave Watson tls_err_abort(strp->sk, ret); 2002c46234ebSDave Watson 2003c46234ebSDave Watson return ret; 2004c46234ebSDave Watson } 2005c46234ebSDave Watson 2006c46234ebSDave Watson static void tls_queue(struct strparser *strp, struct sk_buff *skb) 2007c46234ebSDave Watson { 2008c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 2009f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2010c46234ebSDave Watson 2011c46234ebSDave Watson ctx->decrypted = false; 2012c46234ebSDave Watson 2013c46234ebSDave Watson ctx->recv_pkt = skb; 2014c46234ebSDave Watson strp_pause(strp); 2015c46234ebSDave Watson 2016ad13acceSVakul Garg ctx->saved_data_ready(strp->sk); 2017c46234ebSDave Watson } 2018c46234ebSDave Watson 2019c46234ebSDave Watson static void tls_data_ready(struct sock *sk) 2020c46234ebSDave Watson { 2021c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2022f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2023d3b18ad3SJohn Fastabend struct sk_psock *psock; 2024c46234ebSDave Watson 2025c46234ebSDave Watson strp_data_ready(&ctx->strp); 2026d3b18ad3SJohn Fastabend 2027d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 2028d3b18ad3SJohn Fastabend if (psock && !list_empty(&psock->ingress_msg)) { 2029d3b18ad3SJohn Fastabend ctx->saved_data_ready(sk); 2030d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 2031d3b18ad3SJohn Fastabend } 2032c46234ebSDave Watson } 2033c46234ebSDave Watson 2034f66de3eeSBoris Pismenny void tls_sw_free_resources_tx(struct sock *sk) 20353c4d7559SDave Watson { 20363c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2037f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 2038a42055e8SVakul Garg struct tls_rec *rec, *tmp; 2039a42055e8SVakul Garg 2040a42055e8SVakul Garg /* Wait for any pending async encryptions to complete */ 2041a42055e8SVakul Garg smp_store_mb(ctx->async_notify, true); 2042a42055e8SVakul Garg if (atomic_read(&ctx->encrypt_pending)) 2043a42055e8SVakul Garg crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 2044a42055e8SVakul Garg 204510231213SDave Watson release_sock(sk); 2046a42055e8SVakul Garg cancel_delayed_work_sync(&ctx->tx_work.work); 204710231213SDave Watson lock_sock(sk); 2048a42055e8SVakul Garg 2049a42055e8SVakul Garg /* Tx whatever records we can transmit and abandon the rest */ 2050a42055e8SVakul Garg tls_tx_records(sk, -1); 2051a42055e8SVakul Garg 20529932a29aSVakul Garg /* Free up un-sent records in tx_list. First, free 2053a42055e8SVakul Garg * the partially sent record if any at head of tx_list. 2054a42055e8SVakul Garg */ 205535b71a34SJakub Kicinski if (tls_free_partial_record(sk, tls_ctx)) { 20569932a29aSVakul Garg rec = list_first_entry(&ctx->tx_list, 2057a42055e8SVakul Garg struct tls_rec, list); 2058a42055e8SVakul Garg list_del(&rec->list); 2059d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 2060a42055e8SVakul Garg kfree(rec); 2061a42055e8SVakul Garg } 2062a42055e8SVakul Garg 20639932a29aSVakul Garg list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { 2064a42055e8SVakul Garg list_del(&rec->list); 2065d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_encrypted); 2066d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 2067a42055e8SVakul Garg kfree(rec); 2068a42055e8SVakul Garg } 20693c4d7559SDave Watson 20703c4d7559SDave Watson crypto_free_aead(ctx->aead_send); 2071c774973eSVakul Garg tls_free_open_rec(sk); 2072f66de3eeSBoris Pismenny 2073f66de3eeSBoris Pismenny kfree(ctx); 2074f66de3eeSBoris Pismenny } 2075f66de3eeSBoris Pismenny 207639f56e1aSBoris Pismenny void tls_sw_release_resources_rx(struct sock *sk) 2077f66de3eeSBoris Pismenny { 2078f66de3eeSBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 2079f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2080f66de3eeSBoris Pismenny 2081c46234ebSDave Watson if (ctx->aead_recv) { 2082c46234ebSDave Watson kfree_skb(ctx->recv_pkt); 2083c46234ebSDave Watson ctx->recv_pkt = NULL; 2084692d7b5dSVakul Garg skb_queue_purge(&ctx->rx_list); 2085c46234ebSDave Watson crypto_free_aead(ctx->aead_recv); 2086c46234ebSDave Watson strp_stop(&ctx->strp); 2087c46234ebSDave Watson write_lock_bh(&sk->sk_callback_lock); 2088c46234ebSDave Watson sk->sk_data_ready = ctx->saved_data_ready; 2089c46234ebSDave Watson write_unlock_bh(&sk->sk_callback_lock); 2090c46234ebSDave Watson release_sock(sk); 2091c46234ebSDave Watson strp_done(&ctx->strp); 2092c46234ebSDave Watson lock_sock(sk); 2093c46234ebSDave Watson } 209439f56e1aSBoris Pismenny } 209539f56e1aSBoris Pismenny 209639f56e1aSBoris Pismenny void tls_sw_free_resources_rx(struct sock *sk) 209739f56e1aSBoris Pismenny { 209839f56e1aSBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 209939f56e1aSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 210039f56e1aSBoris Pismenny 210139f56e1aSBoris Pismenny tls_sw_release_resources_rx(sk); 21023c4d7559SDave Watson 21033c4d7559SDave Watson kfree(ctx); 21043c4d7559SDave Watson } 21053c4d7559SDave Watson 21069932a29aSVakul Garg /* The work handler to transmitt the encrypted records in tx_list */ 2107a42055e8SVakul Garg static void tx_work_handler(struct work_struct *work) 2108a42055e8SVakul Garg { 2109a42055e8SVakul Garg struct delayed_work *delayed_work = to_delayed_work(work); 2110a42055e8SVakul Garg struct tx_work *tx_work = container_of(delayed_work, 2111a42055e8SVakul Garg struct tx_work, work); 2112a42055e8SVakul Garg struct sock *sk = tx_work->sk; 2113a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 2114a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 2115a42055e8SVakul Garg 2116a42055e8SVakul Garg if (!test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) 2117a42055e8SVakul Garg return; 2118a42055e8SVakul Garg 2119a42055e8SVakul Garg lock_sock(sk); 2120a42055e8SVakul Garg tls_tx_records(sk, -1); 2121a42055e8SVakul Garg release_sock(sk); 2122a42055e8SVakul Garg } 2123a42055e8SVakul Garg 21247463d3a2SBoris Pismenny void tls_sw_write_space(struct sock *sk, struct tls_context *ctx) 21257463d3a2SBoris Pismenny { 21267463d3a2SBoris Pismenny struct tls_sw_context_tx *tx_ctx = tls_sw_ctx_tx(ctx); 21277463d3a2SBoris Pismenny 21287463d3a2SBoris Pismenny /* Schedule the transmission if tx list is ready */ 21297463d3a2SBoris Pismenny if (is_tx_ready(tx_ctx) && !sk->sk_write_pending) { 21307463d3a2SBoris Pismenny /* Schedule the transmission */ 21317463d3a2SBoris Pismenny if (!test_and_set_bit(BIT_TX_SCHEDULED, 21327463d3a2SBoris Pismenny &tx_ctx->tx_bitmask)) 21337463d3a2SBoris Pismenny schedule_delayed_work(&tx_ctx->tx_work.work, 0); 21347463d3a2SBoris Pismenny } 21357463d3a2SBoris Pismenny } 21367463d3a2SBoris Pismenny 2137c46234ebSDave Watson int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) 21383c4d7559SDave Watson { 21394509de14SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 21404509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 21413c4d7559SDave Watson struct tls_crypto_info *crypto_info; 21423c4d7559SDave Watson struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; 2143fb99bce7SDave Watson struct tls12_crypto_info_aes_gcm_256 *gcm_256_info; 2144f66de3eeSBoris Pismenny struct tls_sw_context_tx *sw_ctx_tx = NULL; 2145f66de3eeSBoris Pismenny struct tls_sw_context_rx *sw_ctx_rx = NULL; 2146c46234ebSDave Watson struct cipher_context *cctx; 2147c46234ebSDave Watson struct crypto_aead **aead; 2148c46234ebSDave Watson struct strp_callbacks cb; 21493c4d7559SDave Watson u16 nonce_size, tag_size, iv_size, rec_seq_size; 2150692d7b5dSVakul Garg struct crypto_tfm *tfm; 2151fb99bce7SDave Watson char *iv, *rec_seq, *key, *salt; 2152fb99bce7SDave Watson size_t keysize; 21533c4d7559SDave Watson int rc = 0; 21543c4d7559SDave Watson 21553c4d7559SDave Watson if (!ctx) { 21563c4d7559SDave Watson rc = -EINVAL; 21573c4d7559SDave Watson goto out; 21583c4d7559SDave Watson } 21593c4d7559SDave Watson 2160f66de3eeSBoris Pismenny if (tx) { 2161b190a587SBoris Pismenny if (!ctx->priv_ctx_tx) { 2162f66de3eeSBoris Pismenny sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL); 2163f66de3eeSBoris Pismenny if (!sw_ctx_tx) { 21643c4d7559SDave Watson rc = -ENOMEM; 21653c4d7559SDave Watson goto out; 21663c4d7559SDave Watson } 2167f66de3eeSBoris Pismenny ctx->priv_ctx_tx = sw_ctx_tx; 2168c46234ebSDave Watson } else { 2169b190a587SBoris Pismenny sw_ctx_tx = 2170b190a587SBoris Pismenny (struct tls_sw_context_tx *)ctx->priv_ctx_tx; 2171b190a587SBoris Pismenny } 2172b190a587SBoris Pismenny } else { 2173b190a587SBoris Pismenny if (!ctx->priv_ctx_rx) { 2174f66de3eeSBoris Pismenny sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL); 2175f66de3eeSBoris Pismenny if (!sw_ctx_rx) { 2176f66de3eeSBoris Pismenny rc = -ENOMEM; 2177f66de3eeSBoris Pismenny goto out; 2178c46234ebSDave Watson } 2179f66de3eeSBoris Pismenny ctx->priv_ctx_rx = sw_ctx_rx; 2180b190a587SBoris Pismenny } else { 2181b190a587SBoris Pismenny sw_ctx_rx = 2182b190a587SBoris Pismenny (struct tls_sw_context_rx *)ctx->priv_ctx_rx; 2183b190a587SBoris Pismenny } 2184f66de3eeSBoris Pismenny } 21853c4d7559SDave Watson 2186c46234ebSDave Watson if (tx) { 2187b190a587SBoris Pismenny crypto_init_wait(&sw_ctx_tx->async_wait); 218886029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 2189c46234ebSDave Watson cctx = &ctx->tx; 2190f66de3eeSBoris Pismenny aead = &sw_ctx_tx->aead_send; 21919932a29aSVakul Garg INIT_LIST_HEAD(&sw_ctx_tx->tx_list); 2192a42055e8SVakul Garg INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler); 2193a42055e8SVakul Garg sw_ctx_tx->tx_work.sk = sk; 2194c46234ebSDave Watson } else { 2195b190a587SBoris Pismenny crypto_init_wait(&sw_ctx_rx->async_wait); 219686029d10SSabrina Dubroca crypto_info = &ctx->crypto_recv.info; 2197c46234ebSDave Watson cctx = &ctx->rx; 2198692d7b5dSVakul Garg skb_queue_head_init(&sw_ctx_rx->rx_list); 2199f66de3eeSBoris Pismenny aead = &sw_ctx_rx->aead_recv; 2200c46234ebSDave Watson } 2201c46234ebSDave Watson 22023c4d7559SDave Watson switch (crypto_info->cipher_type) { 22033c4d7559SDave Watson case TLS_CIPHER_AES_GCM_128: { 22043c4d7559SDave Watson nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 22053c4d7559SDave Watson tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; 22063c4d7559SDave Watson iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 22073c4d7559SDave Watson iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; 22083c4d7559SDave Watson rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; 22093c4d7559SDave Watson rec_seq = 22103c4d7559SDave Watson ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; 22113c4d7559SDave Watson gcm_128_info = 22123c4d7559SDave Watson (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 2213fb99bce7SDave Watson keysize = TLS_CIPHER_AES_GCM_128_KEY_SIZE; 2214fb99bce7SDave Watson key = gcm_128_info->key; 2215fb99bce7SDave Watson salt = gcm_128_info->salt; 2216fb99bce7SDave Watson break; 2217fb99bce7SDave Watson } 2218fb99bce7SDave Watson case TLS_CIPHER_AES_GCM_256: { 2219fb99bce7SDave Watson nonce_size = TLS_CIPHER_AES_GCM_256_IV_SIZE; 2220fb99bce7SDave Watson tag_size = TLS_CIPHER_AES_GCM_256_TAG_SIZE; 2221fb99bce7SDave Watson iv_size = TLS_CIPHER_AES_GCM_256_IV_SIZE; 2222fb99bce7SDave Watson iv = ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->iv; 2223fb99bce7SDave Watson rec_seq_size = TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE; 2224fb99bce7SDave Watson rec_seq = 2225fb99bce7SDave Watson ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->rec_seq; 2226fb99bce7SDave Watson gcm_256_info = 2227fb99bce7SDave Watson (struct tls12_crypto_info_aes_gcm_256 *)crypto_info; 2228fb99bce7SDave Watson keysize = TLS_CIPHER_AES_GCM_256_KEY_SIZE; 2229fb99bce7SDave Watson key = gcm_256_info->key; 2230fb99bce7SDave Watson salt = gcm_256_info->salt; 22313c4d7559SDave Watson break; 22323c4d7559SDave Watson } 22333c4d7559SDave Watson default: 22343c4d7559SDave Watson rc = -EINVAL; 2235cf6d43efSSabrina Dubroca goto free_priv; 22363c4d7559SDave Watson } 22373c4d7559SDave Watson 2238b16520f7SKees Cook /* Sanity-check the IV size for stack allocations. */ 22393463e51dSKees Cook if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE) { 2240b16520f7SKees Cook rc = -EINVAL; 2241b16520f7SKees Cook goto free_priv; 2242b16520f7SKees Cook } 2243b16520f7SKees Cook 2244130b392cSDave Watson if (crypto_info->version == TLS_1_3_VERSION) { 2245130b392cSDave Watson nonce_size = 0; 22464509de14SVakul Garg prot->aad_size = TLS_HEADER_SIZE; 22474509de14SVakul Garg prot->tail_size = 1; 2248130b392cSDave Watson } else { 22494509de14SVakul Garg prot->aad_size = TLS_AAD_SPACE_SIZE; 22504509de14SVakul Garg prot->tail_size = 0; 2251130b392cSDave Watson } 2252130b392cSDave Watson 22534509de14SVakul Garg prot->version = crypto_info->version; 22544509de14SVakul Garg prot->cipher_type = crypto_info->cipher_type; 22554509de14SVakul Garg prot->prepend_size = TLS_HEADER_SIZE + nonce_size; 22564509de14SVakul Garg prot->tag_size = tag_size; 22574509de14SVakul Garg prot->overhead_size = prot->prepend_size + 22584509de14SVakul Garg prot->tag_size + prot->tail_size; 22594509de14SVakul Garg prot->iv_size = iv_size; 2260c46234ebSDave Watson cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 2261dbe42559SDave Watson GFP_KERNEL); 2262c46234ebSDave Watson if (!cctx->iv) { 22633c4d7559SDave Watson rc = -ENOMEM; 2264cf6d43efSSabrina Dubroca goto free_priv; 22653c4d7559SDave Watson } 2266fb99bce7SDave Watson /* Note: 128 & 256 bit salt are the same size */ 2267fb99bce7SDave Watson memcpy(cctx->iv, salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 2268c46234ebSDave Watson memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size); 22694509de14SVakul Garg prot->rec_seq_size = rec_seq_size; 2270969d5090Szhong jiang cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL); 2271c46234ebSDave Watson if (!cctx->rec_seq) { 22723c4d7559SDave Watson rc = -ENOMEM; 22733c4d7559SDave Watson goto free_iv; 22743c4d7559SDave Watson } 22753c4d7559SDave Watson 2276c46234ebSDave Watson if (!*aead) { 2277c46234ebSDave Watson *aead = crypto_alloc_aead("gcm(aes)", 0, 0); 2278c46234ebSDave Watson if (IS_ERR(*aead)) { 2279c46234ebSDave Watson rc = PTR_ERR(*aead); 2280c46234ebSDave Watson *aead = NULL; 22813c4d7559SDave Watson goto free_rec_seq; 22823c4d7559SDave Watson } 22833c4d7559SDave Watson } 22843c4d7559SDave Watson 22853c4d7559SDave Watson ctx->push_pending_record = tls_sw_push_pending_record; 22863c4d7559SDave Watson 2287fb99bce7SDave Watson rc = crypto_aead_setkey(*aead, key, keysize); 2288fb99bce7SDave Watson 22893c4d7559SDave Watson if (rc) 22903c4d7559SDave Watson goto free_aead; 22913c4d7559SDave Watson 22924509de14SVakul Garg rc = crypto_aead_setauthsize(*aead, prot->tag_size); 2293c46234ebSDave Watson if (rc) 2294c46234ebSDave Watson goto free_aead; 2295c46234ebSDave Watson 2296f66de3eeSBoris Pismenny if (sw_ctx_rx) { 2297692d7b5dSVakul Garg tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv); 22988497ded2SVakul Garg 22998497ded2SVakul Garg if (crypto_info->version == TLS_1_3_VERSION) 23008497ded2SVakul Garg sw_ctx_rx->async_capable = false; 23018497ded2SVakul Garg else 2302692d7b5dSVakul Garg sw_ctx_rx->async_capable = 2303692d7b5dSVakul Garg tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; 2304692d7b5dSVakul Garg 2305c46234ebSDave Watson /* Set up strparser */ 2306c46234ebSDave Watson memset(&cb, 0, sizeof(cb)); 2307c46234ebSDave Watson cb.rcv_msg = tls_queue; 2308c46234ebSDave Watson cb.parse_msg = tls_read_size; 2309c46234ebSDave Watson 2310f66de3eeSBoris Pismenny strp_init(&sw_ctx_rx->strp, sk, &cb); 2311c46234ebSDave Watson 2312c46234ebSDave Watson write_lock_bh(&sk->sk_callback_lock); 2313f66de3eeSBoris Pismenny sw_ctx_rx->saved_data_ready = sk->sk_data_ready; 2314c46234ebSDave Watson sk->sk_data_ready = tls_data_ready; 2315c46234ebSDave Watson write_unlock_bh(&sk->sk_callback_lock); 2316c46234ebSDave Watson 2317f66de3eeSBoris Pismenny strp_check_rcv(&sw_ctx_rx->strp); 2318c46234ebSDave Watson } 2319c46234ebSDave Watson 2320c46234ebSDave Watson goto out; 23213c4d7559SDave Watson 23223c4d7559SDave Watson free_aead: 2323c46234ebSDave Watson crypto_free_aead(*aead); 2324c46234ebSDave Watson *aead = NULL; 23253c4d7559SDave Watson free_rec_seq: 2326c46234ebSDave Watson kfree(cctx->rec_seq); 2327c46234ebSDave Watson cctx->rec_seq = NULL; 23283c4d7559SDave Watson free_iv: 2329f66de3eeSBoris Pismenny kfree(cctx->iv); 2330f66de3eeSBoris Pismenny cctx->iv = NULL; 2331cf6d43efSSabrina Dubroca free_priv: 2332f66de3eeSBoris Pismenny if (tx) { 2333f66de3eeSBoris Pismenny kfree(ctx->priv_ctx_tx); 2334f66de3eeSBoris Pismenny ctx->priv_ctx_tx = NULL; 2335f66de3eeSBoris Pismenny } else { 2336f66de3eeSBoris Pismenny kfree(ctx->priv_ctx_rx); 2337f66de3eeSBoris Pismenny ctx->priv_ctx_rx = NULL; 2338f66de3eeSBoris Pismenny } 23393c4d7559SDave Watson out: 23403c4d7559SDave Watson return rc; 23413c4d7559SDave Watson } 2342