13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved. 53c4d7559SDave Watson * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved. 63c4d7559SDave Watson * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved. 7d3b18ad3SJohn Fastabend * Copyright (c) 2018, Covalent IO, Inc. http://covalent.io 83c4d7559SDave Watson * 93c4d7559SDave Watson * This software is available to you under a choice of one of two 103c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 113c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 123c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 133c4d7559SDave Watson * OpenIB.org BSD license below: 143c4d7559SDave Watson * 153c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 163c4d7559SDave Watson * without modification, are permitted provided that the following 173c4d7559SDave Watson * conditions are met: 183c4d7559SDave Watson * 193c4d7559SDave Watson * - Redistributions of source code must retain the above 203c4d7559SDave Watson * copyright notice, this list of conditions and the following 213c4d7559SDave Watson * disclaimer. 223c4d7559SDave Watson * 233c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 243c4d7559SDave Watson * copyright notice, this list of conditions and the following 253c4d7559SDave Watson * disclaimer in the documentation and/or other materials 263c4d7559SDave Watson * provided with the distribution. 273c4d7559SDave Watson * 283c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 293c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 303c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 313c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 323c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 333c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 343c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 353c4d7559SDave Watson * SOFTWARE. 363c4d7559SDave Watson */ 373c4d7559SDave Watson 38da353facSDaniel Jordan #include <linux/bug.h> 39c46234ebSDave Watson #include <linux/sched/signal.h> 403c4d7559SDave Watson #include <linux/module.h> 418d338c76SHerbert Xu #include <linux/kernel.h> 42974271e5SJim Ma #include <linux/splice.h> 433c4d7559SDave Watson #include <crypto/aead.h> 443c4d7559SDave Watson 45c46234ebSDave Watson #include <net/strparser.h> 463c4d7559SDave Watson #include <net/tls.h> 4740e0b090SPeilin Ye #include <trace/events/sock.h> 483c4d7559SDave Watson 4958790314SJakub Kicinski #include "tls.h" 5058790314SJakub Kicinski 514175eac3SJakub Kicinski struct tls_decrypt_arg { 526bd116c8SJakub Kicinski struct_group(inargs, 534175eac3SJakub Kicinski bool zc; 544175eac3SJakub Kicinski bool async; 5581be8535SSabrina Dubroca bool async_done; 56ce61327cSJakub Kicinski u8 tail; 576bd116c8SJakub Kicinski ); 586bd116c8SJakub Kicinski 596bd116c8SJakub Kicinski struct sk_buff *skb; 604175eac3SJakub Kicinski }; 614175eac3SJakub Kicinski 62b89fec54SJakub Kicinski struct tls_decrypt_ctx { 638d338c76SHerbert Xu struct sock *sk; 64b89fec54SJakub Kicinski u8 iv[MAX_IV_SIZE]; 65b89fec54SJakub Kicinski u8 aad[TLS_MAX_AAD_SIZE]; 66b89fec54SJakub Kicinski u8 tail; 67d6847635SSabrina Dubroca bool free_sgout; 68b89fec54SJakub Kicinski struct scatterlist sg[]; 69b89fec54SJakub Kicinski }; 70b89fec54SJakub Kicinski 71da353facSDaniel Jordan noinline void tls_err_abort(struct sock *sk, int err) 72da353facSDaniel Jordan { 73da353facSDaniel Jordan WARN_ON_ONCE(err >= 0); 74da353facSDaniel Jordan /* sk->sk_err should contain a positive error code. */ 758a0d57dfSJakub Kicinski WRITE_ONCE(sk->sk_err, -err); 768a0d57dfSJakub Kicinski /* Paired with smp_rmb() in tcp_poll() */ 778a0d57dfSJakub Kicinski smp_wmb(); 78da353facSDaniel Jordan sk_error_report(sk); 79da353facSDaniel Jordan } 80da353facSDaniel Jordan 810927f71dSDoron Roberts-Kedes static int __skb_nsg(struct sk_buff *skb, int offset, int len, 820927f71dSDoron Roberts-Kedes unsigned int recursion_level) 830927f71dSDoron Roberts-Kedes { 840927f71dSDoron Roberts-Kedes int start = skb_headlen(skb); 850927f71dSDoron Roberts-Kedes int i, chunk = start - offset; 860927f71dSDoron Roberts-Kedes struct sk_buff *frag_iter; 870927f71dSDoron Roberts-Kedes int elt = 0; 880927f71dSDoron Roberts-Kedes 890927f71dSDoron Roberts-Kedes if (unlikely(recursion_level >= 24)) 900927f71dSDoron Roberts-Kedes return -EMSGSIZE; 910927f71dSDoron Roberts-Kedes 920927f71dSDoron Roberts-Kedes if (chunk > 0) { 930927f71dSDoron Roberts-Kedes if (chunk > len) 940927f71dSDoron Roberts-Kedes chunk = len; 950927f71dSDoron Roberts-Kedes elt++; 960927f71dSDoron Roberts-Kedes len -= chunk; 970927f71dSDoron Roberts-Kedes if (len == 0) 980927f71dSDoron Roberts-Kedes return elt; 990927f71dSDoron Roberts-Kedes offset += chunk; 1000927f71dSDoron Roberts-Kedes } 1010927f71dSDoron Roberts-Kedes 1020927f71dSDoron Roberts-Kedes for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1030927f71dSDoron Roberts-Kedes int end; 1040927f71dSDoron Roberts-Kedes 1050927f71dSDoron Roberts-Kedes WARN_ON(start > offset + len); 1060927f71dSDoron Roberts-Kedes 1070927f71dSDoron Roberts-Kedes end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 1080927f71dSDoron Roberts-Kedes chunk = end - offset; 1090927f71dSDoron Roberts-Kedes if (chunk > 0) { 1100927f71dSDoron Roberts-Kedes if (chunk > len) 1110927f71dSDoron Roberts-Kedes chunk = len; 1120927f71dSDoron Roberts-Kedes elt++; 1130927f71dSDoron Roberts-Kedes len -= chunk; 1140927f71dSDoron Roberts-Kedes if (len == 0) 1150927f71dSDoron Roberts-Kedes return elt; 1160927f71dSDoron Roberts-Kedes offset += chunk; 1170927f71dSDoron Roberts-Kedes } 1180927f71dSDoron Roberts-Kedes start = end; 1190927f71dSDoron Roberts-Kedes } 1200927f71dSDoron Roberts-Kedes 1210927f71dSDoron Roberts-Kedes if (unlikely(skb_has_frag_list(skb))) { 1220927f71dSDoron Roberts-Kedes skb_walk_frags(skb, frag_iter) { 1230927f71dSDoron Roberts-Kedes int end, ret; 1240927f71dSDoron Roberts-Kedes 1250927f71dSDoron Roberts-Kedes WARN_ON(start > offset + len); 1260927f71dSDoron Roberts-Kedes 1270927f71dSDoron Roberts-Kedes end = start + frag_iter->len; 1280927f71dSDoron Roberts-Kedes chunk = end - offset; 1290927f71dSDoron Roberts-Kedes if (chunk > 0) { 1300927f71dSDoron Roberts-Kedes if (chunk > len) 1310927f71dSDoron Roberts-Kedes chunk = len; 1320927f71dSDoron Roberts-Kedes ret = __skb_nsg(frag_iter, offset - start, chunk, 1330927f71dSDoron Roberts-Kedes recursion_level + 1); 1340927f71dSDoron Roberts-Kedes if (unlikely(ret < 0)) 1350927f71dSDoron Roberts-Kedes return ret; 1360927f71dSDoron Roberts-Kedes elt += ret; 1370927f71dSDoron Roberts-Kedes len -= chunk; 1380927f71dSDoron Roberts-Kedes if (len == 0) 1390927f71dSDoron Roberts-Kedes return elt; 1400927f71dSDoron Roberts-Kedes offset += chunk; 1410927f71dSDoron Roberts-Kedes } 1420927f71dSDoron Roberts-Kedes start = end; 1430927f71dSDoron Roberts-Kedes } 1440927f71dSDoron Roberts-Kedes } 1450927f71dSDoron Roberts-Kedes BUG_ON(len); 1460927f71dSDoron Roberts-Kedes return elt; 1470927f71dSDoron Roberts-Kedes } 1480927f71dSDoron Roberts-Kedes 1490927f71dSDoron Roberts-Kedes /* Return the number of scatterlist elements required to completely map the 1500927f71dSDoron Roberts-Kedes * skb, or -EMSGSIZE if the recursion depth is exceeded. 1510927f71dSDoron Roberts-Kedes */ 1520927f71dSDoron Roberts-Kedes static int skb_nsg(struct sk_buff *skb, int offset, int len) 1530927f71dSDoron Roberts-Kedes { 1540927f71dSDoron Roberts-Kedes return __skb_nsg(skb, offset, len, 0); 1550927f71dSDoron Roberts-Kedes } 1560927f71dSDoron Roberts-Kedes 157ce61327cSJakub Kicinski static int tls_padding_length(struct tls_prot_info *prot, struct sk_buff *skb, 158ce61327cSJakub Kicinski struct tls_decrypt_arg *darg) 159130b392cSDave Watson { 160130b392cSDave Watson struct strp_msg *rxm = strp_msg(skb); 161c3f6bb74SJakub Kicinski struct tls_msg *tlm = tls_msg(skb); 162130b392cSDave Watson int sub = 0; 163130b392cSDave Watson 164130b392cSDave Watson /* Determine zero-padding length */ 165b53f4976SJakub Kicinski if (prot->version == TLS_1_3_VERSION) { 1665deee41bSJakub Kicinski int offset = rxm->full_len - TLS_TAG_SIZE - 1; 167ce61327cSJakub Kicinski char content_type = darg->zc ? darg->tail : 0; 168130b392cSDave Watson int err; 169130b392cSDave Watson 170130b392cSDave Watson while (content_type == 0) { 1715deee41bSJakub Kicinski if (offset < prot->prepend_size) 172130b392cSDave Watson return -EBADMSG; 1735deee41bSJakub Kicinski err = skb_copy_bits(skb, rxm->offset + offset, 174130b392cSDave Watson &content_type, 1); 175b53f4976SJakub Kicinski if (err) 176b53f4976SJakub Kicinski return err; 177130b392cSDave Watson if (content_type) 178130b392cSDave Watson break; 179130b392cSDave Watson sub++; 1805deee41bSJakub Kicinski offset--; 181130b392cSDave Watson } 182c3f6bb74SJakub Kicinski tlm->control = content_type; 183130b392cSDave Watson } 184130b392cSDave Watson return sub; 185130b392cSDave Watson } 186130b392cSDave Watson 1878580e55aSHerbert Xu static void tls_decrypt_done(void *data, int err) 18894524d8fSVakul Garg { 1898580e55aSHerbert Xu struct aead_request *aead_req = data; 1908d338c76SHerbert Xu struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); 19194524d8fSVakul Garg struct scatterlist *sgout = aead_req->dst; 1927a3dd8c8SJohn Fastabend struct tls_sw_context_rx *ctx; 1938d338c76SHerbert Xu struct tls_decrypt_ctx *dctx; 1947a3dd8c8SJohn Fastabend struct tls_context *tls_ctx; 19594524d8fSVakul Garg struct scatterlist *sg; 19694524d8fSVakul Garg unsigned int pages; 1976ececdc5SJakub Kicinski struct sock *sk; 1988d338c76SHerbert Xu int aead_size; 1997a3dd8c8SJohn Fastabend 20013eca403SJakub Kicinski /* If requests get too backlogged crypto API returns -EBUSY and calls 20113eca403SJakub Kicinski * ->complete(-EINPROGRESS) immediately followed by ->complete(0) 20213eca403SJakub Kicinski * to make waiting for backlog to flush with crypto_wait_req() easier. 20313eca403SJakub Kicinski * First wait converts -EBUSY -> -EINPROGRESS, and the second one 20413eca403SJakub Kicinski * -EINPROGRESS -> 0. 20513eca403SJakub Kicinski * We have a single struct crypto_async_request per direction, this 20613eca403SJakub Kicinski * scheme doesn't help us, so just ignore the first ->complete(). 20713eca403SJakub Kicinski */ 20813eca403SJakub Kicinski if (err == -EINPROGRESS) 20913eca403SJakub Kicinski return; 21013eca403SJakub Kicinski 2118d338c76SHerbert Xu aead_size = sizeof(*aead_req) + crypto_aead_reqsize(aead); 2128d338c76SHerbert Xu aead_size = ALIGN(aead_size, __alignof__(*dctx)); 2138d338c76SHerbert Xu dctx = (void *)((u8 *)aead_req + aead_size); 2148d338c76SHerbert Xu 2158d338c76SHerbert Xu sk = dctx->sk; 2166ececdc5SJakub Kicinski tls_ctx = tls_get_ctx(sk); 2177a3dd8c8SJohn Fastabend ctx = tls_sw_ctx_rx(tls_ctx); 21894524d8fSVakul Garg 21994524d8fSVakul Garg /* Propagate if there was an err */ 22094524d8fSVakul Garg if (err) { 2215c5ec668SJakub Kicinski if (err == -EBADMSG) 2226ececdc5SJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTERROR); 22394524d8fSVakul Garg ctx->async_wait.err = err; 2246ececdc5SJakub Kicinski tls_err_abort(sk, err); 22594524d8fSVakul Garg } 22694524d8fSVakul Garg 227692d7b5dSVakul Garg /* Free the destination pages if skb was not decrypted inplace */ 228d6847635SSabrina Dubroca if (dctx->free_sgout) { 22994524d8fSVakul Garg /* Skip the first S/G entry as it points to AAD */ 23094524d8fSVakul Garg for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { 23194524d8fSVakul Garg if (!sg) 23294524d8fSVakul Garg break; 23394524d8fSVakul Garg put_page(sg_page(sg)); 23494524d8fSVakul Garg } 235692d7b5dSVakul Garg } 23694524d8fSVakul Garg 23794524d8fSVakul Garg kfree(aead_req); 23894524d8fSVakul Garg 23986dc27eeSJakub Kicinski if (atomic_dec_and_test(&ctx->decrypt_pending)) 24094524d8fSVakul Garg complete(&ctx->async_wait.completion); 24194524d8fSVakul Garg } 24294524d8fSVakul Garg 243de31df6fSJakub Kicinski static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) 244de31df6fSJakub Kicinski { 24586dc27eeSJakub Kicinski if (!atomic_dec_and_test(&ctx->decrypt_pending)) 246de31df6fSJakub Kicinski crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 24786dc27eeSJakub Kicinski atomic_inc(&ctx->decrypt_pending); 248de31df6fSJakub Kicinski 249de31df6fSJakub Kicinski return ctx->async_wait.err; 250de31df6fSJakub Kicinski } 251de31df6fSJakub Kicinski 252c46234ebSDave Watson static int tls_do_decryption(struct sock *sk, 253c46234ebSDave Watson struct scatterlist *sgin, 254c46234ebSDave Watson struct scatterlist *sgout, 255c46234ebSDave Watson char *iv_recv, 256c46234ebSDave Watson size_t data_len, 25794524d8fSVakul Garg struct aead_request *aead_req, 2583547a1f9SJakub Kicinski struct tls_decrypt_arg *darg) 259c46234ebSDave Watson { 260c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2614509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 262f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 263c46234ebSDave Watson int ret; 264c46234ebSDave Watson 2650b243d00SVakul Garg aead_request_set_tfm(aead_req, ctx->aead_recv); 2664509de14SVakul Garg aead_request_set_ad(aead_req, prot->aad_size); 267c46234ebSDave Watson aead_request_set_crypt(aead_req, sgin, sgout, 2684509de14SVakul Garg data_len + prot->tag_size, 269c46234ebSDave Watson (u8 *)iv_recv); 270c46234ebSDave Watson 2713547a1f9SJakub Kicinski if (darg->async) { 27294524d8fSVakul Garg aead_request_set_callback(aead_req, 27394524d8fSVakul Garg CRYPTO_TFM_REQ_MAY_BACKLOG, 2748d338c76SHerbert Xu tls_decrypt_done, aead_req); 27586dc27eeSJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&ctx->decrypt_pending) < 1); 27694524d8fSVakul Garg atomic_inc(&ctx->decrypt_pending); 27794524d8fSVakul Garg } else { 278dec5b6e7SSabrina Dubroca DECLARE_CRYPTO_WAIT(wait); 279dec5b6e7SSabrina Dubroca 28094524d8fSVakul Garg aead_request_set_callback(aead_req, 28194524d8fSVakul Garg CRYPTO_TFM_REQ_MAY_BACKLOG, 282dec5b6e7SSabrina Dubroca crypto_req_done, &wait); 283dec5b6e7SSabrina Dubroca ret = crypto_aead_decrypt(aead_req); 284dec5b6e7SSabrina Dubroca if (ret == -EINPROGRESS || ret == -EBUSY) 285dec5b6e7SSabrina Dubroca ret = crypto_wait_req(ret, &wait); 286dec5b6e7SSabrina Dubroca return ret; 28794524d8fSVakul Garg } 28894524d8fSVakul Garg 28994524d8fSVakul Garg ret = crypto_aead_decrypt(aead_req); 29081be8535SSabrina Dubroca if (ret == -EINPROGRESS) 29181be8535SSabrina Dubroca return 0; 29281be8535SSabrina Dubroca 29313eca403SJakub Kicinski if (ret == -EBUSY) { 29413eca403SJakub Kicinski ret = tls_decrypt_async_wait(ctx); 29581be8535SSabrina Dubroca darg->async_done = true; 29681be8535SSabrina Dubroca /* all completions have run, we're not doing async anymore */ 29781be8535SSabrina Dubroca darg->async = false; 29881be8535SSabrina Dubroca return ret; 29913eca403SJakub Kicinski } 30081be8535SSabrina Dubroca 301c19fdb06SSabrina Dubroca atomic_dec(&ctx->decrypt_pending); 3023547a1f9SJakub Kicinski darg->async = false; 3033547a1f9SJakub Kicinski 304c46234ebSDave Watson return ret; 305c46234ebSDave Watson } 306c46234ebSDave Watson 307d829e9c4SDaniel Borkmann static void tls_trim_both_msgs(struct sock *sk, int target_size) 3083c4d7559SDave Watson { 3093c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 3104509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 311f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 312a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 3133c4d7559SDave Watson 314d829e9c4SDaniel Borkmann sk_msg_trim(sk, &rec->msg_plaintext, target_size); 3153c4d7559SDave Watson if (target_size > 0) 3164509de14SVakul Garg target_size += prot->overhead_size; 317d829e9c4SDaniel Borkmann sk_msg_trim(sk, &rec->msg_encrypted, target_size); 3183c4d7559SDave Watson } 3193c4d7559SDave Watson 320d829e9c4SDaniel Borkmann static int tls_alloc_encrypted_msg(struct sock *sk, int len) 3213c4d7559SDave Watson { 3223c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 323f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 324a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 325d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 3263c4d7559SDave Watson 327d829e9c4SDaniel Borkmann return sk_msg_alloc(sk, msg_en, len, 0); 3283c4d7559SDave Watson } 3293c4d7559SDave Watson 330d829e9c4SDaniel Borkmann static int tls_clone_plaintext_msg(struct sock *sk, int required) 3313c4d7559SDave Watson { 3323c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 3334509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 334f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 335a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 336d829e9c4SDaniel Borkmann struct sk_msg *msg_pl = &rec->msg_plaintext; 337d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 3384e6d4720SVakul Garg int skip, len; 3393c4d7559SDave Watson 340d829e9c4SDaniel Borkmann /* We add page references worth len bytes from encrypted sg 341d829e9c4SDaniel Borkmann * at the end of plaintext sg. It is guaranteed that msg_en 3424e6d4720SVakul Garg * has enough required room (ensured by caller). 3434e6d4720SVakul Garg */ 344d829e9c4SDaniel Borkmann len = required - msg_pl->sg.size; 34552ea992cSVakul Garg 346d829e9c4SDaniel Borkmann /* Skip initial bytes in msg_en's data to be able to use 347d829e9c4SDaniel Borkmann * same offset of both plain and encrypted data. 3484e6d4720SVakul Garg */ 3494509de14SVakul Garg skip = prot->prepend_size + msg_pl->sg.size; 3504e6d4720SVakul Garg 351d829e9c4SDaniel Borkmann return sk_msg_clone(sk, msg_pl, msg_en, skip, len); 3523c4d7559SDave Watson } 3533c4d7559SDave Watson 354d3b18ad3SJohn Fastabend static struct tls_rec *tls_get_rec(struct sock *sk) 355d3b18ad3SJohn Fastabend { 356d3b18ad3SJohn Fastabend struct tls_context *tls_ctx = tls_get_ctx(sk); 3574509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 358d3b18ad3SJohn Fastabend struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 359d3b18ad3SJohn Fastabend struct sk_msg *msg_pl, *msg_en; 360d3b18ad3SJohn Fastabend struct tls_rec *rec; 361d3b18ad3SJohn Fastabend int mem_size; 362d3b18ad3SJohn Fastabend 363d3b18ad3SJohn Fastabend mem_size = sizeof(struct tls_rec) + crypto_aead_reqsize(ctx->aead_send); 364d3b18ad3SJohn Fastabend 365d3b18ad3SJohn Fastabend rec = kzalloc(mem_size, sk->sk_allocation); 366d3b18ad3SJohn Fastabend if (!rec) 367d3b18ad3SJohn Fastabend return NULL; 368d3b18ad3SJohn Fastabend 369d3b18ad3SJohn Fastabend msg_pl = &rec->msg_plaintext; 370d3b18ad3SJohn Fastabend msg_en = &rec->msg_encrypted; 371d3b18ad3SJohn Fastabend 372d3b18ad3SJohn Fastabend sk_msg_init(msg_pl); 373d3b18ad3SJohn Fastabend sk_msg_init(msg_en); 374d3b18ad3SJohn Fastabend 375d3b18ad3SJohn Fastabend sg_init_table(rec->sg_aead_in, 2); 3764509de14SVakul Garg sg_set_buf(&rec->sg_aead_in[0], rec->aad_space, prot->aad_size); 377d3b18ad3SJohn Fastabend sg_unmark_end(&rec->sg_aead_in[1]); 378d3b18ad3SJohn Fastabend 379d3b18ad3SJohn Fastabend sg_init_table(rec->sg_aead_out, 2); 3804509de14SVakul Garg sg_set_buf(&rec->sg_aead_out[0], rec->aad_space, prot->aad_size); 381d3b18ad3SJohn Fastabend sg_unmark_end(&rec->sg_aead_out[1]); 382d3b18ad3SJohn Fastabend 3838d338c76SHerbert Xu rec->sk = sk; 3848d338c76SHerbert Xu 385d3b18ad3SJohn Fastabend return rec; 386d3b18ad3SJohn Fastabend } 387d3b18ad3SJohn Fastabend 388d3b18ad3SJohn Fastabend static void tls_free_rec(struct sock *sk, struct tls_rec *rec) 389d3b18ad3SJohn Fastabend { 390d3b18ad3SJohn Fastabend sk_msg_free(sk, &rec->msg_encrypted); 391d3b18ad3SJohn Fastabend sk_msg_free(sk, &rec->msg_plaintext); 392d3b18ad3SJohn Fastabend kfree(rec); 393d3b18ad3SJohn Fastabend } 394d3b18ad3SJohn Fastabend 395c774973eSVakul Garg static void tls_free_open_rec(struct sock *sk) 3963c4d7559SDave Watson { 3973c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 398f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 399a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 4003c4d7559SDave Watson 401d3b18ad3SJohn Fastabend if (rec) { 402d3b18ad3SJohn Fastabend tls_free_rec(sk, rec); 403d3b18ad3SJohn Fastabend ctx->open_rec = NULL; 404d3b18ad3SJohn Fastabend } 4053c4d7559SDave Watson } 4063c4d7559SDave Watson 407a42055e8SVakul Garg int tls_tx_records(struct sock *sk, int flags) 408a42055e8SVakul Garg { 409a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 410a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 411a42055e8SVakul Garg struct tls_rec *rec, *tmp; 412d829e9c4SDaniel Borkmann struct sk_msg *msg_en; 413a42055e8SVakul Garg int tx_flags, rc = 0; 414a42055e8SVakul Garg 415a42055e8SVakul Garg if (tls_is_partially_sent_record(tls_ctx)) { 4169932a29aSVakul Garg rec = list_first_entry(&ctx->tx_list, 417a42055e8SVakul Garg struct tls_rec, list); 418a42055e8SVakul Garg 419a42055e8SVakul Garg if (flags == -1) 420a42055e8SVakul Garg tx_flags = rec->tx_flags; 421a42055e8SVakul Garg else 422a42055e8SVakul Garg tx_flags = flags; 423a42055e8SVakul Garg 424a42055e8SVakul Garg rc = tls_push_partial_record(sk, tls_ctx, tx_flags); 425a42055e8SVakul Garg if (rc) 426a42055e8SVakul Garg goto tx_err; 427a42055e8SVakul Garg 428a42055e8SVakul Garg /* Full record has been transmitted. 4299932a29aSVakul Garg * Remove the head of tx_list 430a42055e8SVakul Garg */ 431a42055e8SVakul Garg list_del(&rec->list); 432d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 433a42055e8SVakul Garg kfree(rec); 434a42055e8SVakul Garg } 435a42055e8SVakul Garg 4369932a29aSVakul Garg /* Tx all ready records */ 4379932a29aSVakul Garg list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { 4389932a29aSVakul Garg if (READ_ONCE(rec->tx_ready)) { 439a42055e8SVakul Garg if (flags == -1) 440a42055e8SVakul Garg tx_flags = rec->tx_flags; 441a42055e8SVakul Garg else 442a42055e8SVakul Garg tx_flags = flags; 443a42055e8SVakul Garg 444d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 445a42055e8SVakul Garg rc = tls_push_sg(sk, tls_ctx, 446d829e9c4SDaniel Borkmann &msg_en->sg.data[msg_en->sg.curr], 447a42055e8SVakul Garg 0, tx_flags); 448a42055e8SVakul Garg if (rc) 449a42055e8SVakul Garg goto tx_err; 450a42055e8SVakul Garg 451a42055e8SVakul Garg list_del(&rec->list); 452d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 453a42055e8SVakul Garg kfree(rec); 454a42055e8SVakul Garg } else { 455a42055e8SVakul Garg break; 456a42055e8SVakul Garg } 457a42055e8SVakul Garg } 458a42055e8SVakul Garg 459a42055e8SVakul Garg tx_err: 460a42055e8SVakul Garg if (rc < 0 && rc != -EAGAIN) 461da353facSDaniel Jordan tls_err_abort(sk, -EBADMSG); 462a42055e8SVakul Garg 463a42055e8SVakul Garg return rc; 464a42055e8SVakul Garg } 465a42055e8SVakul Garg 4668580e55aSHerbert Xu static void tls_encrypt_done(void *data, int err) 467a42055e8SVakul Garg { 4688d338c76SHerbert Xu struct tls_sw_context_tx *ctx; 4698d338c76SHerbert Xu struct tls_context *tls_ctx; 4708d338c76SHerbert Xu struct tls_prot_info *prot; 471d3777ceaSHerbert Xu struct tls_rec *rec = data; 472d829e9c4SDaniel Borkmann struct scatterlist *sge; 473d829e9c4SDaniel Borkmann struct sk_msg *msg_en; 4748d338c76SHerbert Xu struct sock *sk; 475a42055e8SVakul Garg 47613eca403SJakub Kicinski if (err == -EINPROGRESS) /* see the comment in tls_decrypt_done() */ 47713eca403SJakub Kicinski return; 47813eca403SJakub Kicinski 479d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 480a42055e8SVakul Garg 4818d338c76SHerbert Xu sk = rec->sk; 4828d338c76SHerbert Xu tls_ctx = tls_get_ctx(sk); 4838d338c76SHerbert Xu prot = &tls_ctx->prot_info; 4848d338c76SHerbert Xu ctx = tls_sw_ctx_tx(tls_ctx); 4858d338c76SHerbert Xu 486d829e9c4SDaniel Borkmann sge = sk_msg_elem(msg_en, msg_en->sg.curr); 4874509de14SVakul Garg sge->offset -= prot->prepend_size; 4884509de14SVakul Garg sge->length += prot->prepend_size; 489a42055e8SVakul Garg 49080ece6a0SVakul Garg /* Check if error is previously set on socket */ 491a42055e8SVakul Garg if (err || sk->sk_err) { 492a42055e8SVakul Garg rec = NULL; 493a42055e8SVakul Garg 494a42055e8SVakul Garg /* If err is already set on socket, return the same code */ 495a42055e8SVakul Garg if (sk->sk_err) { 4961d9d6fd2SDaniel Jordan ctx->async_wait.err = -sk->sk_err; 497a42055e8SVakul Garg } else { 498a42055e8SVakul Garg ctx->async_wait.err = err; 499a42055e8SVakul Garg tls_err_abort(sk, err); 500a42055e8SVakul Garg } 501a42055e8SVakul Garg } 502a42055e8SVakul Garg 5039932a29aSVakul Garg if (rec) { 5049932a29aSVakul Garg struct tls_rec *first_rec; 5059932a29aSVakul Garg 5069932a29aSVakul Garg /* Mark the record as ready for transmission */ 5079932a29aSVakul Garg smp_store_mb(rec->tx_ready, true); 5089932a29aSVakul Garg 5099932a29aSVakul Garg /* If received record is at head of tx_list, schedule tx */ 5109932a29aSVakul Garg first_rec = list_first_entry(&ctx->tx_list, 5119932a29aSVakul Garg struct tls_rec, list); 5126db22d6cSJakub Kicinski if (rec == first_rec) { 5136db22d6cSJakub Kicinski /* Schedule the transmission */ 5146db22d6cSJakub Kicinski if (!test_and_set_bit(BIT_TX_SCHEDULED, 5156db22d6cSJakub Kicinski &ctx->tx_bitmask)) 5166db22d6cSJakub Kicinski schedule_delayed_work(&ctx->tx_work.work, 1); 5176db22d6cSJakub Kicinski } 5189932a29aSVakul Garg } 519a42055e8SVakul Garg 52086dc27eeSJakub Kicinski if (atomic_dec_and_test(&ctx->encrypt_pending)) 521a42055e8SVakul Garg complete(&ctx->async_wait.completion); 522a42055e8SVakul Garg } 523a42055e8SVakul Garg 524de31df6fSJakub Kicinski static int tls_encrypt_async_wait(struct tls_sw_context_tx *ctx) 525de31df6fSJakub Kicinski { 52686dc27eeSJakub Kicinski if (!atomic_dec_and_test(&ctx->encrypt_pending)) 527de31df6fSJakub Kicinski crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 52886dc27eeSJakub Kicinski atomic_inc(&ctx->encrypt_pending); 529de31df6fSJakub Kicinski 530de31df6fSJakub Kicinski return ctx->async_wait.err; 531de31df6fSJakub Kicinski } 532de31df6fSJakub Kicinski 533a42055e8SVakul Garg static int tls_do_encryption(struct sock *sk, 534a42055e8SVakul Garg struct tls_context *tls_ctx, 535a447da7dSDaniel Borkmann struct tls_sw_context_tx *ctx, 536a447da7dSDaniel Borkmann struct aead_request *aead_req, 537d829e9c4SDaniel Borkmann size_t data_len, u32 start) 5383c4d7559SDave Watson { 5394509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 540a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 541d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 542d829e9c4SDaniel Borkmann struct scatterlist *sge = sk_msg_elem(msg_en, start); 543f295b3aeSVakul Garg int rc, iv_offset = 0; 5443c4d7559SDave Watson 545f295b3aeSVakul Garg /* For CCM based ciphers, first byte of IV is a constant */ 546128cfb88STianjia Zhang switch (prot->cipher_type) { 547128cfb88STianjia Zhang case TLS_CIPHER_AES_CCM_128: 548f295b3aeSVakul Garg rec->iv_data[0] = TLS_AES_CCM_IV_B0_BYTE; 549f295b3aeSVakul Garg iv_offset = 1; 550128cfb88STianjia Zhang break; 551128cfb88STianjia Zhang case TLS_CIPHER_SM4_CCM: 552128cfb88STianjia Zhang rec->iv_data[0] = TLS_SM4_CCM_IV_B0_BYTE; 553128cfb88STianjia Zhang iv_offset = 1; 554128cfb88STianjia Zhang break; 555f295b3aeSVakul Garg } 556f295b3aeSVakul Garg 557f295b3aeSVakul Garg memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv, 558f295b3aeSVakul Garg prot->iv_size + prot->salt_size); 559f295b3aeSVakul Garg 56058790314SJakub Kicinski tls_xor_iv_with_seq(prot, rec->iv_data + iv_offset, 56158790314SJakub Kicinski tls_ctx->tx.rec_seq); 56232eb67b9SDave Watson 5634509de14SVakul Garg sge->offset += prot->prepend_size; 5644509de14SVakul Garg sge->length -= prot->prepend_size; 5653c4d7559SDave Watson 566d829e9c4SDaniel Borkmann msg_en->sg.curr = start; 5674e6d4720SVakul Garg 5683c4d7559SDave Watson aead_request_set_tfm(aead_req, ctx->aead_send); 5694509de14SVakul Garg aead_request_set_ad(aead_req, prot->aad_size); 570d829e9c4SDaniel Borkmann aead_request_set_crypt(aead_req, rec->sg_aead_in, 571d829e9c4SDaniel Borkmann rec->sg_aead_out, 57232eb67b9SDave Watson data_len, rec->iv_data); 573a54667f6SVakul Garg 574a54667f6SVakul Garg aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 575d3777ceaSHerbert Xu tls_encrypt_done, rec); 576a54667f6SVakul Garg 5779932a29aSVakul Garg /* Add the record in tx_list */ 5789932a29aSVakul Garg list_add_tail((struct list_head *)&rec->list, &ctx->tx_list); 57986dc27eeSJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&ctx->encrypt_pending) < 1); 580a42055e8SVakul Garg atomic_inc(&ctx->encrypt_pending); 5813c4d7559SDave Watson 582a42055e8SVakul Garg rc = crypto_aead_encrypt(aead_req); 58313eca403SJakub Kicinski if (rc == -EBUSY) { 58413eca403SJakub Kicinski rc = tls_encrypt_async_wait(ctx); 58513eca403SJakub Kicinski rc = rc ?: -EINPROGRESS; 58613eca403SJakub Kicinski } 587a42055e8SVakul Garg if (!rc || rc != -EINPROGRESS) { 588a42055e8SVakul Garg atomic_dec(&ctx->encrypt_pending); 5894509de14SVakul Garg sge->offset -= prot->prepend_size; 5904509de14SVakul Garg sge->length += prot->prepend_size; 591a42055e8SVakul Garg } 5923c4d7559SDave Watson 5939932a29aSVakul Garg if (!rc) { 5949932a29aSVakul Garg WRITE_ONCE(rec->tx_ready, true); 5959932a29aSVakul Garg } else if (rc != -EINPROGRESS) { 5969932a29aSVakul Garg list_del(&rec->list); 597a42055e8SVakul Garg return rc; 5989932a29aSVakul Garg } 599a42055e8SVakul Garg 600a42055e8SVakul Garg /* Unhook the record from context if encryption is not failure */ 601a42055e8SVakul Garg ctx->open_rec = NULL; 602fb0f886fSJakub Kicinski tls_advance_record_sn(sk, prot, &tls_ctx->tx); 6033c4d7559SDave Watson return rc; 6043c4d7559SDave Watson } 6053c4d7559SDave Watson 606d3b18ad3SJohn Fastabend static int tls_split_open_record(struct sock *sk, struct tls_rec *from, 607d3b18ad3SJohn Fastabend struct tls_rec **to, struct sk_msg *msg_opl, 608d3b18ad3SJohn Fastabend struct sk_msg *msg_oen, u32 split_point, 609d3b18ad3SJohn Fastabend u32 tx_overhead_size, u32 *orig_end) 610d3b18ad3SJohn Fastabend { 611d3b18ad3SJohn Fastabend u32 i, j, bytes = 0, apply = msg_opl->apply_bytes; 612d3b18ad3SJohn Fastabend struct scatterlist *sge, *osge, *nsge; 613d3b18ad3SJohn Fastabend u32 orig_size = msg_opl->sg.size; 614d3b18ad3SJohn Fastabend struct scatterlist tmp = { }; 615d3b18ad3SJohn Fastabend struct sk_msg *msg_npl; 616d3b18ad3SJohn Fastabend struct tls_rec *new; 617d3b18ad3SJohn Fastabend int ret; 618d3b18ad3SJohn Fastabend 619d3b18ad3SJohn Fastabend new = tls_get_rec(sk); 620d3b18ad3SJohn Fastabend if (!new) 621d3b18ad3SJohn Fastabend return -ENOMEM; 622d3b18ad3SJohn Fastabend ret = sk_msg_alloc(sk, &new->msg_encrypted, msg_opl->sg.size + 623d3b18ad3SJohn Fastabend tx_overhead_size, 0); 624d3b18ad3SJohn Fastabend if (ret < 0) { 625d3b18ad3SJohn Fastabend tls_free_rec(sk, new); 626d3b18ad3SJohn Fastabend return ret; 627d3b18ad3SJohn Fastabend } 628d3b18ad3SJohn Fastabend 629d3b18ad3SJohn Fastabend *orig_end = msg_opl->sg.end; 630d3b18ad3SJohn Fastabend i = msg_opl->sg.start; 631d3b18ad3SJohn Fastabend sge = sk_msg_elem(msg_opl, i); 632d3b18ad3SJohn Fastabend while (apply && sge->length) { 633d3b18ad3SJohn Fastabend if (sge->length > apply) { 634d3b18ad3SJohn Fastabend u32 len = sge->length - apply; 635d3b18ad3SJohn Fastabend 636d3b18ad3SJohn Fastabend get_page(sg_page(sge)); 637d3b18ad3SJohn Fastabend sg_set_page(&tmp, sg_page(sge), len, 638d3b18ad3SJohn Fastabend sge->offset + apply); 639d3b18ad3SJohn Fastabend sge->length = apply; 640d3b18ad3SJohn Fastabend bytes += apply; 641d3b18ad3SJohn Fastabend apply = 0; 642d3b18ad3SJohn Fastabend } else { 643d3b18ad3SJohn Fastabend apply -= sge->length; 644d3b18ad3SJohn Fastabend bytes += sge->length; 645d3b18ad3SJohn Fastabend } 646d3b18ad3SJohn Fastabend 647d3b18ad3SJohn Fastabend sk_msg_iter_var_next(i); 648d3b18ad3SJohn Fastabend if (i == msg_opl->sg.end) 649d3b18ad3SJohn Fastabend break; 650d3b18ad3SJohn Fastabend sge = sk_msg_elem(msg_opl, i); 651d3b18ad3SJohn Fastabend } 652d3b18ad3SJohn Fastabend 653d3b18ad3SJohn Fastabend msg_opl->sg.end = i; 654d3b18ad3SJohn Fastabend msg_opl->sg.curr = i; 655d3b18ad3SJohn Fastabend msg_opl->sg.copybreak = 0; 656d3b18ad3SJohn Fastabend msg_opl->apply_bytes = 0; 657d3b18ad3SJohn Fastabend msg_opl->sg.size = bytes; 658d3b18ad3SJohn Fastabend 659d3b18ad3SJohn Fastabend msg_npl = &new->msg_plaintext; 660d3b18ad3SJohn Fastabend msg_npl->apply_bytes = apply; 661d3b18ad3SJohn Fastabend msg_npl->sg.size = orig_size - bytes; 662d3b18ad3SJohn Fastabend 663d3b18ad3SJohn Fastabend j = msg_npl->sg.start; 664d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 665d3b18ad3SJohn Fastabend if (tmp.length) { 666d3b18ad3SJohn Fastabend memcpy(nsge, &tmp, sizeof(*nsge)); 667d3b18ad3SJohn Fastabend sk_msg_iter_var_next(j); 668d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 669d3b18ad3SJohn Fastabend } 670d3b18ad3SJohn Fastabend 671d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 672d3b18ad3SJohn Fastabend while (osge->length) { 673d3b18ad3SJohn Fastabend memcpy(nsge, osge, sizeof(*nsge)); 674d3b18ad3SJohn Fastabend sg_unmark_end(nsge); 675d3b18ad3SJohn Fastabend sk_msg_iter_var_next(i); 676d3b18ad3SJohn Fastabend sk_msg_iter_var_next(j); 677d3b18ad3SJohn Fastabend if (i == *orig_end) 678d3b18ad3SJohn Fastabend break; 679d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 680d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 681d3b18ad3SJohn Fastabend } 682d3b18ad3SJohn Fastabend 683d3b18ad3SJohn Fastabend msg_npl->sg.end = j; 684d3b18ad3SJohn Fastabend msg_npl->sg.curr = j; 685d3b18ad3SJohn Fastabend msg_npl->sg.copybreak = 0; 686d3b18ad3SJohn Fastabend 687d3b18ad3SJohn Fastabend *to = new; 688d3b18ad3SJohn Fastabend return 0; 689d3b18ad3SJohn Fastabend } 690d3b18ad3SJohn Fastabend 691d3b18ad3SJohn Fastabend static void tls_merge_open_record(struct sock *sk, struct tls_rec *to, 692d3b18ad3SJohn Fastabend struct tls_rec *from, u32 orig_end) 693d3b18ad3SJohn Fastabend { 694d3b18ad3SJohn Fastabend struct sk_msg *msg_npl = &from->msg_plaintext; 695d3b18ad3SJohn Fastabend struct sk_msg *msg_opl = &to->msg_plaintext; 696d3b18ad3SJohn Fastabend struct scatterlist *osge, *nsge; 697d3b18ad3SJohn Fastabend u32 i, j; 698d3b18ad3SJohn Fastabend 699d3b18ad3SJohn Fastabend i = msg_opl->sg.end; 700d3b18ad3SJohn Fastabend sk_msg_iter_var_prev(i); 701d3b18ad3SJohn Fastabend j = msg_npl->sg.start; 702d3b18ad3SJohn Fastabend 703d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 704d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 705d3b18ad3SJohn Fastabend 706d3b18ad3SJohn Fastabend if (sg_page(osge) == sg_page(nsge) && 707d3b18ad3SJohn Fastabend osge->offset + osge->length == nsge->offset) { 708d3b18ad3SJohn Fastabend osge->length += nsge->length; 709d3b18ad3SJohn Fastabend put_page(sg_page(nsge)); 710d3b18ad3SJohn Fastabend } 711d3b18ad3SJohn Fastabend 712d3b18ad3SJohn Fastabend msg_opl->sg.end = orig_end; 713d3b18ad3SJohn Fastabend msg_opl->sg.curr = orig_end; 714d3b18ad3SJohn Fastabend msg_opl->sg.copybreak = 0; 715d3b18ad3SJohn Fastabend msg_opl->apply_bytes = msg_opl->sg.size + msg_npl->sg.size; 716d3b18ad3SJohn Fastabend msg_opl->sg.size += msg_npl->sg.size; 717d3b18ad3SJohn Fastabend 718d3b18ad3SJohn Fastabend sk_msg_free(sk, &to->msg_encrypted); 719d3b18ad3SJohn Fastabend sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted); 720d3b18ad3SJohn Fastabend 721d3b18ad3SJohn Fastabend kfree(from); 722d3b18ad3SJohn Fastabend } 723d3b18ad3SJohn Fastabend 7243c4d7559SDave Watson static int tls_push_record(struct sock *sk, int flags, 7253c4d7559SDave Watson unsigned char record_type) 7263c4d7559SDave Watson { 7273c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 7284509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 729f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 730d3b18ad3SJohn Fastabend struct tls_rec *rec = ctx->open_rec, *tmp = NULL; 7313f649ab7SKees Cook u32 i, split_point, orig_end; 732d829e9c4SDaniel Borkmann struct sk_msg *msg_pl, *msg_en; 733a447da7dSDaniel Borkmann struct aead_request *req; 734d3b18ad3SJohn Fastabend bool split; 7353c4d7559SDave Watson int rc; 7363c4d7559SDave Watson 737a42055e8SVakul Garg if (!rec) 738a42055e8SVakul Garg return 0; 739a447da7dSDaniel Borkmann 740d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 741d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 742d829e9c4SDaniel Borkmann 743d3b18ad3SJohn Fastabend split_point = msg_pl->apply_bytes; 744d3b18ad3SJohn Fastabend split = split_point && split_point < msg_pl->sg.size; 745d468e477SJohn Fastabend if (unlikely((!split && 746d468e477SJohn Fastabend msg_pl->sg.size + 747d468e477SJohn Fastabend prot->overhead_size > msg_en->sg.size) || 748d468e477SJohn Fastabend (split && 749d468e477SJohn Fastabend split_point + 750d468e477SJohn Fastabend prot->overhead_size > msg_en->sg.size))) { 751d468e477SJohn Fastabend split = true; 752d468e477SJohn Fastabend split_point = msg_en->sg.size; 753d468e477SJohn Fastabend } 754d3b18ad3SJohn Fastabend if (split) { 755d3b18ad3SJohn Fastabend rc = tls_split_open_record(sk, rec, &tmp, msg_pl, msg_en, 7564509de14SVakul Garg split_point, prot->overhead_size, 757d3b18ad3SJohn Fastabend &orig_end); 758d3b18ad3SJohn Fastabend if (rc < 0) 759d3b18ad3SJohn Fastabend return rc; 760d468e477SJohn Fastabend /* This can happen if above tls_split_open_record allocates 761d468e477SJohn Fastabend * a single large encryption buffer instead of two smaller 762d468e477SJohn Fastabend * ones. In this case adjust pointers and continue without 763d468e477SJohn Fastabend * split. 764d468e477SJohn Fastabend */ 765d468e477SJohn Fastabend if (!msg_pl->sg.size) { 766d468e477SJohn Fastabend tls_merge_open_record(sk, rec, tmp, orig_end); 767d468e477SJohn Fastabend msg_pl = &rec->msg_plaintext; 768d468e477SJohn Fastabend msg_en = &rec->msg_encrypted; 769d468e477SJohn Fastabend split = false; 770d468e477SJohn Fastabend } 771d3b18ad3SJohn Fastabend sk_msg_trim(sk, msg_en, msg_pl->sg.size + 7724509de14SVakul Garg prot->overhead_size); 773d3b18ad3SJohn Fastabend } 774d3b18ad3SJohn Fastabend 775a42055e8SVakul Garg rec->tx_flags = flags; 776a42055e8SVakul Garg req = &rec->aead_req; 7773c4d7559SDave Watson 778d829e9c4SDaniel Borkmann i = msg_pl->sg.end; 779d829e9c4SDaniel Borkmann sk_msg_iter_var_prev(i); 780130b392cSDave Watson 781130b392cSDave Watson rec->content_type = record_type; 7824509de14SVakul Garg if (prot->version == TLS_1_3_VERSION) { 783130b392cSDave Watson /* Add content type to end of message. No padding added */ 784130b392cSDave Watson sg_set_buf(&rec->sg_content_type, &rec->content_type, 1); 785130b392cSDave Watson sg_mark_end(&rec->sg_content_type); 786130b392cSDave Watson sg_chain(msg_pl->sg.data, msg_pl->sg.end + 1, 787130b392cSDave Watson &rec->sg_content_type); 788130b392cSDave Watson } else { 789d829e9c4SDaniel Borkmann sg_mark_end(sk_msg_elem(msg_pl, i)); 790130b392cSDave Watson } 791a42055e8SVakul Garg 7929aaaa568SJohn Fastabend if (msg_pl->sg.end < msg_pl->sg.start) { 7939aaaa568SJohn Fastabend sg_chain(&msg_pl->sg.data[msg_pl->sg.start], 7949aaaa568SJohn Fastabend MAX_SKB_FRAGS - msg_pl->sg.start + 1, 7959aaaa568SJohn Fastabend msg_pl->sg.data); 7969aaaa568SJohn Fastabend } 7979aaaa568SJohn Fastabend 798d829e9c4SDaniel Borkmann i = msg_pl->sg.start; 7999e5ffed3SJakub Kicinski sg_chain(rec->sg_aead_in, 2, &msg_pl->sg.data[i]); 800d829e9c4SDaniel Borkmann 801d829e9c4SDaniel Borkmann i = msg_en->sg.end; 802d829e9c4SDaniel Borkmann sk_msg_iter_var_prev(i); 803d829e9c4SDaniel Borkmann sg_mark_end(sk_msg_elem(msg_en, i)); 804d829e9c4SDaniel Borkmann 805d829e9c4SDaniel Borkmann i = msg_en->sg.start; 806d829e9c4SDaniel Borkmann sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]); 807d829e9c4SDaniel Borkmann 8084509de14SVakul Garg tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size, 8096942a284SVadim Fedorenko tls_ctx->tx.rec_seq, record_type, prot); 8103c4d7559SDave Watson 8113c4d7559SDave Watson tls_fill_prepend(tls_ctx, 812d829e9c4SDaniel Borkmann page_address(sg_page(&msg_en->sg.data[i])) + 813130b392cSDave Watson msg_en->sg.data[i].offset, 8144509de14SVakul Garg msg_pl->sg.size + prot->tail_size, 8156942a284SVadim Fedorenko record_type); 8163c4d7559SDave Watson 817d829e9c4SDaniel Borkmann tls_ctx->pending_open_record_frags = false; 8183c4d7559SDave Watson 819130b392cSDave Watson rc = tls_do_encryption(sk, tls_ctx, ctx, req, 8204509de14SVakul Garg msg_pl->sg.size + prot->tail_size, i); 8213c4d7559SDave Watson if (rc < 0) { 822d3b18ad3SJohn Fastabend if (rc != -EINPROGRESS) { 823da353facSDaniel Jordan tls_err_abort(sk, -EBADMSG); 824d3b18ad3SJohn Fastabend if (split) { 825d3b18ad3SJohn Fastabend tls_ctx->pending_open_record_frags = true; 826d3b18ad3SJohn Fastabend tls_merge_open_record(sk, rec, tmp, orig_end); 827d3b18ad3SJohn Fastabend } 828d3b18ad3SJohn Fastabend } 8295b053e12SDave Watson ctx->async_capable = 1; 830a42055e8SVakul Garg return rc; 831d3b18ad3SJohn Fastabend } else if (split) { 832d3b18ad3SJohn Fastabend msg_pl = &tmp->msg_plaintext; 833d3b18ad3SJohn Fastabend msg_en = &tmp->msg_encrypted; 8344509de14SVakul Garg sk_msg_trim(sk, msg_en, msg_pl->sg.size + prot->overhead_size); 835d3b18ad3SJohn Fastabend tls_ctx->pending_open_record_frags = true; 836d3b18ad3SJohn Fastabend ctx->open_rec = tmp; 8373c4d7559SDave Watson } 8383c4d7559SDave Watson 839a42055e8SVakul Garg return tls_tx_records(sk, flags); 8403c4d7559SDave Watson } 8413c4d7559SDave Watson 842d3b18ad3SJohn Fastabend static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, 843d3b18ad3SJohn Fastabend bool full_record, u8 record_type, 844a7bff11fSVadim Fedorenko ssize_t *copied, int flags) 8453c4d7559SDave Watson { 8463c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 847f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 848d3b18ad3SJohn Fastabend struct sk_msg msg_redir = { }; 849d3b18ad3SJohn Fastabend struct sk_psock *psock; 850d3b18ad3SJohn Fastabend struct sock *sk_redir; 851a42055e8SVakul Garg struct tls_rec *rec; 852a351d608SPengcheng Yang bool enospc, policy, redir_ingress; 853d3b18ad3SJohn Fastabend int err = 0, send; 8547246d8edSJohn Fastabend u32 delta = 0; 855a42055e8SVakul Garg 8560608c69cSJohn Fastabend policy = !(flags & MSG_SENDPAGE_NOPOLICY); 857d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 858d10523d0SJakub Kicinski if (!psock || !policy) { 859d10523d0SJakub Kicinski err = tls_push_record(sk, flags, record_type); 860cfaa80c9SLiu Jian if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) { 861d10523d0SJakub Kicinski *copied -= sk_msg_free(sk, msg); 862d10523d0SJakub Kicinski tls_free_open_rec(sk); 863635d9398SVadim Fedorenko err = -sk->sk_err; 864d10523d0SJakub Kicinski } 865095f5614SXiyu Yang if (psock) 866095f5614SXiyu Yang sk_psock_put(sk, psock); 867d10523d0SJakub Kicinski return err; 868d10523d0SJakub Kicinski } 869d3b18ad3SJohn Fastabend more_data: 870d3b18ad3SJohn Fastabend enospc = sk_msg_full(msg); 8717246d8edSJohn Fastabend if (psock->eval == __SK_NONE) { 8727246d8edSJohn Fastabend delta = msg->sg.size; 873d3b18ad3SJohn Fastabend psock->eval = sk_psock_msg_verdict(sk, psock, msg); 8747246d8edSJohn Fastabend delta -= msg->sg.size; 8757246d8edSJohn Fastabend } 876d3b18ad3SJohn Fastabend if (msg->cork_bytes && msg->cork_bytes > msg->sg.size && 877d3b18ad3SJohn Fastabend !enospc && !full_record) { 878d3b18ad3SJohn Fastabend err = -ENOSPC; 879d3b18ad3SJohn Fastabend goto out_err; 880d3b18ad3SJohn Fastabend } 881d3b18ad3SJohn Fastabend msg->cork_bytes = 0; 882d3b18ad3SJohn Fastabend send = msg->sg.size; 883d3b18ad3SJohn Fastabend if (msg->apply_bytes && msg->apply_bytes < send) 884d3b18ad3SJohn Fastabend send = msg->apply_bytes; 885a42055e8SVakul Garg 886d3b18ad3SJohn Fastabend switch (psock->eval) { 887d3b18ad3SJohn Fastabend case __SK_PASS: 888d3b18ad3SJohn Fastabend err = tls_push_record(sk, flags, record_type); 889cfaa80c9SLiu Jian if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) { 890d3b18ad3SJohn Fastabend *copied -= sk_msg_free(sk, msg); 891d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 892635d9398SVadim Fedorenko err = -sk->sk_err; 893d3b18ad3SJohn Fastabend goto out_err; 894d3b18ad3SJohn Fastabend } 895d3b18ad3SJohn Fastabend break; 896d3b18ad3SJohn Fastabend case __SK_REDIRECT: 897a351d608SPengcheng Yang redir_ingress = psock->redir_ingress; 898d3b18ad3SJohn Fastabend sk_redir = psock->sk_redir; 899d3b18ad3SJohn Fastabend memcpy(&msg_redir, msg, sizeof(*msg)); 900d3b18ad3SJohn Fastabend if (msg->apply_bytes < send) 901d3b18ad3SJohn Fastabend msg->apply_bytes = 0; 902d3b18ad3SJohn Fastabend else 903d3b18ad3SJohn Fastabend msg->apply_bytes -= send; 904d3b18ad3SJohn Fastabend sk_msg_return_zero(sk, msg, send); 905d3b18ad3SJohn Fastabend msg->sg.size -= send; 906d3b18ad3SJohn Fastabend release_sock(sk); 907a351d608SPengcheng Yang err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress, 908a351d608SPengcheng Yang &msg_redir, send, flags); 909d3b18ad3SJohn Fastabend lock_sock(sk); 910d3b18ad3SJohn Fastabend if (err < 0) { 911d3b18ad3SJohn Fastabend *copied -= sk_msg_free_nocharge(sk, &msg_redir); 912d3b18ad3SJohn Fastabend msg->sg.size = 0; 913d3b18ad3SJohn Fastabend } 914d3b18ad3SJohn Fastabend if (msg->sg.size == 0) 915d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 916d3b18ad3SJohn Fastabend break; 917d3b18ad3SJohn Fastabend case __SK_DROP: 918d3b18ad3SJohn Fastabend default: 919d3b18ad3SJohn Fastabend sk_msg_free_partial(sk, msg, send); 920d3b18ad3SJohn Fastabend if (msg->apply_bytes < send) 921d3b18ad3SJohn Fastabend msg->apply_bytes = 0; 922d3b18ad3SJohn Fastabend else 923d3b18ad3SJohn Fastabend msg->apply_bytes -= send; 924d3b18ad3SJohn Fastabend if (msg->sg.size == 0) 925d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 9267246d8edSJohn Fastabend *copied -= (send + delta); 927d3b18ad3SJohn Fastabend err = -EACCES; 928d3b18ad3SJohn Fastabend } 929a42055e8SVakul Garg 930d3b18ad3SJohn Fastabend if (likely(!err)) { 931d3b18ad3SJohn Fastabend bool reset_eval = !ctx->open_rec; 932d3b18ad3SJohn Fastabend 933d3b18ad3SJohn Fastabend rec = ctx->open_rec; 934d3b18ad3SJohn Fastabend if (rec) { 935d3b18ad3SJohn Fastabend msg = &rec->msg_plaintext; 936d3b18ad3SJohn Fastabend if (!msg->apply_bytes) 937d3b18ad3SJohn Fastabend reset_eval = true; 938d3b18ad3SJohn Fastabend } 939d3b18ad3SJohn Fastabend if (reset_eval) { 940d3b18ad3SJohn Fastabend psock->eval = __SK_NONE; 941d3b18ad3SJohn Fastabend if (psock->sk_redir) { 942d3b18ad3SJohn Fastabend sock_put(psock->sk_redir); 943d3b18ad3SJohn Fastabend psock->sk_redir = NULL; 944d3b18ad3SJohn Fastabend } 945d3b18ad3SJohn Fastabend } 946d3b18ad3SJohn Fastabend if (rec) 947d3b18ad3SJohn Fastabend goto more_data; 948d3b18ad3SJohn Fastabend } 949d3b18ad3SJohn Fastabend out_err: 950d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 951d3b18ad3SJohn Fastabend return err; 952d3b18ad3SJohn Fastabend } 953d3b18ad3SJohn Fastabend 954d3b18ad3SJohn Fastabend static int tls_sw_push_pending_record(struct sock *sk, int flags) 955d3b18ad3SJohn Fastabend { 956d3b18ad3SJohn Fastabend struct tls_context *tls_ctx = tls_get_ctx(sk); 957d3b18ad3SJohn Fastabend struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 958d3b18ad3SJohn Fastabend struct tls_rec *rec = ctx->open_rec; 959d3b18ad3SJohn Fastabend struct sk_msg *msg_pl; 960d3b18ad3SJohn Fastabend size_t copied; 961d3b18ad3SJohn Fastabend 962a42055e8SVakul Garg if (!rec) 963d3b18ad3SJohn Fastabend return 0; 964a42055e8SVakul Garg 965d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 966d3b18ad3SJohn Fastabend copied = msg_pl->sg.size; 967d3b18ad3SJohn Fastabend if (!copied) 968d3b18ad3SJohn Fastabend return 0; 969a42055e8SVakul Garg 970d3b18ad3SJohn Fastabend return bpf_exec_tx_verdict(msg_pl, sk, true, TLS_RECORD_TYPE_DATA, 971d3b18ad3SJohn Fastabend &copied, flags); 972a42055e8SVakul Garg } 973a42055e8SVakul Garg 974fe1e81d4SDavid Howells static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg, 975fe1e81d4SDavid Howells struct sk_msg *msg_pl, size_t try_to_copy, 976fe1e81d4SDavid Howells ssize_t *copied) 977fe1e81d4SDavid Howells { 978fe1e81d4SDavid Howells struct page *page = NULL, **pages = &page; 979fe1e81d4SDavid Howells 980fe1e81d4SDavid Howells do { 981fe1e81d4SDavid Howells ssize_t part; 982fe1e81d4SDavid Howells size_t off; 983fe1e81d4SDavid Howells 984fe1e81d4SDavid Howells part = iov_iter_extract_pages(&msg->msg_iter, &pages, 985fe1e81d4SDavid Howells try_to_copy, 1, 0, &off); 986fe1e81d4SDavid Howells if (part <= 0) 987fe1e81d4SDavid Howells return part ?: -EIO; 988fe1e81d4SDavid Howells 989fe1e81d4SDavid Howells if (WARN_ON_ONCE(!sendpage_ok(page))) { 990fe1e81d4SDavid Howells iov_iter_revert(&msg->msg_iter, part); 991fe1e81d4SDavid Howells return -EIO; 992fe1e81d4SDavid Howells } 993fe1e81d4SDavid Howells 994fe1e81d4SDavid Howells sk_msg_page_add(msg_pl, page, part, off); 995eb30a025SJohn Fastabend msg_pl->sg.copybreak = 0; 996eb30a025SJohn Fastabend msg_pl->sg.curr = msg_pl->sg.end; 997fe1e81d4SDavid Howells sk_mem_charge(sk, part); 998fe1e81d4SDavid Howells *copied += part; 999fe1e81d4SDavid Howells try_to_copy -= part; 1000fe1e81d4SDavid Howells } while (try_to_copy && !sk_msg_full(msg_pl)); 1001fe1e81d4SDavid Howells 1002fe1e81d4SDavid Howells return 0; 1003fe1e81d4SDavid Howells } 1004fe1e81d4SDavid Howells 100545e5be84SDavid Howells static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, 100645e5be84SDavid Howells size_t size) 1007a42055e8SVakul Garg { 10083c4d7559SDave Watson long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 1009a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 10104509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 1011a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 10125b053e12SDave Watson bool async_capable = ctx->async_capable; 1013a42055e8SVakul Garg unsigned char record_type = TLS_RECORD_TYPE_DATA; 101400e23707SDavid Howells bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); 10153c4d7559SDave Watson bool eor = !(msg->msg_flags & MSG_MORE); 1016a7bff11fSVadim Fedorenko size_t try_to_copy; 1017a7bff11fSVadim Fedorenko ssize_t copied = 0; 1018d829e9c4SDaniel Borkmann struct sk_msg *msg_pl, *msg_en; 1019a42055e8SVakul Garg struct tls_rec *rec; 1020a42055e8SVakul Garg int required_size; 1021a42055e8SVakul Garg int num_async = 0; 10223c4d7559SDave Watson bool full_record; 1023a42055e8SVakul Garg int record_room; 1024a42055e8SVakul Garg int num_zc = 0; 10253c4d7559SDave Watson int orig_size; 10264128c0cfSVakul Garg int ret = 0; 10273c4d7559SDave Watson 1028e22e358bSHannes Reinecke if (!eor && (msg->msg_flags & MSG_EOR)) 1029e22e358bSHannes Reinecke return -EINVAL; 1030e22e358bSHannes Reinecke 10313c4d7559SDave Watson if (unlikely(msg->msg_controllen)) { 103258790314SJakub Kicinski ret = tls_process_cmsg(sk, msg, &record_type); 1033a42055e8SVakul Garg if (ret) { 1034a42055e8SVakul Garg if (ret == -EINPROGRESS) 1035a42055e8SVakul Garg num_async++; 1036a42055e8SVakul Garg else if (ret != -EAGAIN) 10373c4d7559SDave Watson goto send_end; 10383c4d7559SDave Watson } 1039a42055e8SVakul Garg } 10403c4d7559SDave Watson 10413c4d7559SDave Watson while (msg_data_left(msg)) { 10423c4d7559SDave Watson if (sk->sk_err) { 104330be8f8dSr.hering@avm.de ret = -sk->sk_err; 10443c4d7559SDave Watson goto send_end; 10453c4d7559SDave Watson } 10463c4d7559SDave Watson 1047d3b18ad3SJohn Fastabend if (ctx->open_rec) 1048d3b18ad3SJohn Fastabend rec = ctx->open_rec; 1049d3b18ad3SJohn Fastabend else 1050d3b18ad3SJohn Fastabend rec = ctx->open_rec = tls_get_rec(sk); 1051a42055e8SVakul Garg if (!rec) { 1052a42055e8SVakul Garg ret = -ENOMEM; 1053a42055e8SVakul Garg goto send_end; 1054a42055e8SVakul Garg } 1055a42055e8SVakul Garg 1056d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 1057d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 1058d829e9c4SDaniel Borkmann 1059d829e9c4SDaniel Borkmann orig_size = msg_pl->sg.size; 10603c4d7559SDave Watson full_record = false; 10613c4d7559SDave Watson try_to_copy = msg_data_left(msg); 1062d829e9c4SDaniel Borkmann record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size; 10633c4d7559SDave Watson if (try_to_copy >= record_room) { 10643c4d7559SDave Watson try_to_copy = record_room; 10653c4d7559SDave Watson full_record = true; 10663c4d7559SDave Watson } 10673c4d7559SDave Watson 1068d829e9c4SDaniel Borkmann required_size = msg_pl->sg.size + try_to_copy + 10694509de14SVakul Garg prot->overhead_size; 10703c4d7559SDave Watson 10713c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 10723c4d7559SDave Watson goto wait_for_sndbuf; 1073a42055e8SVakul Garg 10743c4d7559SDave Watson alloc_encrypted: 1075d829e9c4SDaniel Borkmann ret = tls_alloc_encrypted_msg(sk, required_size); 10763c4d7559SDave Watson if (ret) { 10773c4d7559SDave Watson if (ret != -ENOSPC) 10783c4d7559SDave Watson goto wait_for_memory; 10793c4d7559SDave Watson 10803c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 10813c4d7559SDave Watson * actually allocated. The difference is due 10823c4d7559SDave Watson * to max sg elements limit 10833c4d7559SDave Watson */ 1084d829e9c4SDaniel Borkmann try_to_copy -= required_size - msg_en->sg.size; 10853c4d7559SDave Watson full_record = true; 10863c4d7559SDave Watson } 1087a42055e8SVakul Garg 1088fe1e81d4SDavid Howells if (try_to_copy && (msg->msg_flags & MSG_SPLICE_PAGES)) { 1089fe1e81d4SDavid Howells ret = tls_sw_sendmsg_splice(sk, msg, msg_pl, 1090fe1e81d4SDavid Howells try_to_copy, &copied); 1091fe1e81d4SDavid Howells if (ret < 0) 1092fe1e81d4SDavid Howells goto send_end; 1093fe1e81d4SDavid Howells tls_ctx->pending_open_record_frags = true; 109402e368ebSJohn Fastabend 109502e368ebSJohn Fastabend if (sk_msg_full(msg_pl)) 109602e368ebSJohn Fastabend full_record = true; 109702e368ebSJohn Fastabend 109802e368ebSJohn Fastabend if (full_record || eor) 1099fe1e81d4SDavid Howells goto copied; 1100fe1e81d4SDavid Howells continue; 1101fe1e81d4SDavid Howells } 1102fe1e81d4SDavid Howells 1103a42055e8SVakul Garg if (!is_kvec && (full_record || eor) && !async_capable) { 1104d3b18ad3SJohn Fastabend u32 first = msg_pl->sg.end; 1105d3b18ad3SJohn Fastabend 1106d829e9c4SDaniel Borkmann ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter, 1107d829e9c4SDaniel Borkmann msg_pl, try_to_copy); 11083c4d7559SDave Watson if (ret) 11093c4d7559SDave Watson goto fallback_to_reg_send; 11103c4d7559SDave Watson 1111a42055e8SVakul Garg num_zc++; 11123c4d7559SDave Watson copied += try_to_copy; 1113d3b18ad3SJohn Fastabend 1114d3b18ad3SJohn Fastabend sk_msg_sg_copy_set(msg_pl, first); 1115d3b18ad3SJohn Fastabend ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 1116d3b18ad3SJohn Fastabend record_type, &copied, 1117d3b18ad3SJohn Fastabend msg->msg_flags); 1118a42055e8SVakul Garg if (ret) { 1119a42055e8SVakul Garg if (ret == -EINPROGRESS) 1120a42055e8SVakul Garg num_async++; 1121d3b18ad3SJohn Fastabend else if (ret == -ENOMEM) 1122d3b18ad3SJohn Fastabend goto wait_for_memory; 1123c329ef96SJakub Kicinski else if (ctx->open_rec && ret == -ENOSPC) 1124d3b18ad3SJohn Fastabend goto rollback_iter; 1125a42055e8SVakul Garg else if (ret != -EAGAIN) 11263c4d7559SDave Watson goto send_end; 1127a42055e8SVakul Garg } 11285a3611efSDoron Roberts-Kedes continue; 1129d3b18ad3SJohn Fastabend rollback_iter: 1130d3b18ad3SJohn Fastabend copied -= try_to_copy; 1131d3b18ad3SJohn Fastabend sk_msg_sg_copy_clear(msg_pl, first); 1132d3b18ad3SJohn Fastabend iov_iter_revert(&msg->msg_iter, 1133d3b18ad3SJohn Fastabend msg_pl->sg.size - orig_size); 11343c4d7559SDave Watson fallback_to_reg_send: 1135d829e9c4SDaniel Borkmann sk_msg_trim(sk, msg_pl, orig_size); 11363c4d7559SDave Watson } 11373c4d7559SDave Watson 1138d829e9c4SDaniel Borkmann required_size = msg_pl->sg.size + try_to_copy; 11394e6d4720SVakul Garg 1140d829e9c4SDaniel Borkmann ret = tls_clone_plaintext_msg(sk, required_size); 11413c4d7559SDave Watson if (ret) { 11423c4d7559SDave Watson if (ret != -ENOSPC) 11434e6d4720SVakul Garg goto send_end; 11443c4d7559SDave Watson 11453c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 11463c4d7559SDave Watson * actually allocated. The difference is due 11473c4d7559SDave Watson * to max sg elements limit 11483c4d7559SDave Watson */ 1149d829e9c4SDaniel Borkmann try_to_copy -= required_size - msg_pl->sg.size; 11503c4d7559SDave Watson full_record = true; 11514509de14SVakul Garg sk_msg_trim(sk, msg_en, 11524509de14SVakul Garg msg_pl->sg.size + prot->overhead_size); 11533c4d7559SDave Watson } 11543c4d7559SDave Watson 115565a10e28SVakul Garg if (try_to_copy) { 115665a10e28SVakul Garg ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, 115765a10e28SVakul Garg msg_pl, try_to_copy); 1158d829e9c4SDaniel Borkmann if (ret < 0) 11593c4d7559SDave Watson goto trim_sgl; 116065a10e28SVakul Garg } 11613c4d7559SDave Watson 1162d829e9c4SDaniel Borkmann /* Open records defined only if successfully copied, otherwise 1163d829e9c4SDaniel Borkmann * we would trim the sg but not reset the open record frags. 1164d829e9c4SDaniel Borkmann */ 1165d829e9c4SDaniel Borkmann tls_ctx->pending_open_record_frags = true; 11663c4d7559SDave Watson copied += try_to_copy; 1167fe1e81d4SDavid Howells copied: 11683c4d7559SDave Watson if (full_record || eor) { 1169d3b18ad3SJohn Fastabend ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 1170d3b18ad3SJohn Fastabend record_type, &copied, 1171d3b18ad3SJohn Fastabend msg->msg_flags); 11723c4d7559SDave Watson if (ret) { 1173a42055e8SVakul Garg if (ret == -EINPROGRESS) 1174a42055e8SVakul Garg num_async++; 1175d3b18ad3SJohn Fastabend else if (ret == -ENOMEM) 1176d3b18ad3SJohn Fastabend goto wait_for_memory; 1177d3b18ad3SJohn Fastabend else if (ret != -EAGAIN) { 1178d3b18ad3SJohn Fastabend if (ret == -ENOSPC) 1179d3b18ad3SJohn Fastabend ret = 0; 11803c4d7559SDave Watson goto send_end; 11813c4d7559SDave Watson } 11823c4d7559SDave Watson } 1183d3b18ad3SJohn Fastabend } 11843c4d7559SDave Watson 11853c4d7559SDave Watson continue; 11863c4d7559SDave Watson 11873c4d7559SDave Watson wait_for_sndbuf: 11883c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 11893c4d7559SDave Watson wait_for_memory: 11903c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 11913c4d7559SDave Watson if (ret) { 11923c4d7559SDave Watson trim_sgl: 1193c329ef96SJakub Kicinski if (ctx->open_rec) 1194d829e9c4SDaniel Borkmann tls_trim_both_msgs(sk, orig_size); 11953c4d7559SDave Watson goto send_end; 11963c4d7559SDave Watson } 11973c4d7559SDave Watson 1198c329ef96SJakub Kicinski if (ctx->open_rec && msg_en->sg.size < required_size) 11993c4d7559SDave Watson goto alloc_encrypted; 12003c4d7559SDave Watson } 12013c4d7559SDave Watson 1202a42055e8SVakul Garg if (!num_async) { 1203a42055e8SVakul Garg goto send_end; 1204a42055e8SVakul Garg } else if (num_zc) { 1205de31df6fSJakub Kicinski int err; 1206de31df6fSJakub Kicinski 1207a42055e8SVakul Garg /* Wait for pending encryptions to get completed */ 1208de31df6fSJakub Kicinski err = tls_encrypt_async_wait(ctx); 1209de31df6fSJakub Kicinski if (err) { 1210de31df6fSJakub Kicinski ret = err; 1211a42055e8SVakul Garg copied = 0; 1212a42055e8SVakul Garg } 1213a42055e8SVakul Garg } 1214a42055e8SVakul Garg 1215a42055e8SVakul Garg /* Transmit if any encryptions have completed */ 1216a42055e8SVakul Garg if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 1217a42055e8SVakul Garg cancel_delayed_work(&ctx->tx_work.work); 1218a42055e8SVakul Garg tls_tx_records(sk, msg->msg_flags); 1219a42055e8SVakul Garg } 1220a42055e8SVakul Garg 12213c4d7559SDave Watson send_end: 12223c4d7559SDave Watson ret = sk_stream_error(sk, msg->msg_flags, ret); 122345e5be84SDavid Howells return copied > 0 ? copied : ret; 122445e5be84SDavid Howells } 12253c4d7559SDave Watson 122645e5be84SDavid Howells int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 122745e5be84SDavid Howells { 122845e5be84SDavid Howells struct tls_context *tls_ctx = tls_get_ctx(sk); 122945e5be84SDavid Howells int ret; 123045e5be84SDavid Howells 123145e5be84SDavid Howells if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 1232e22e358bSHannes Reinecke MSG_CMSG_COMPAT | MSG_SPLICE_PAGES | MSG_EOR | 1233b848b26cSDavid Howells MSG_SENDPAGE_NOPOLICY)) 123445e5be84SDavid Howells return -EOPNOTSUPP; 123545e5be84SDavid Howells 123645e5be84SDavid Howells ret = mutex_lock_interruptible(&tls_ctx->tx_lock); 123745e5be84SDavid Howells if (ret) 123845e5be84SDavid Howells return ret; 123945e5be84SDavid Howells lock_sock(sk); 124045e5be84SDavid Howells ret = tls_sw_sendmsg_locked(sk, msg, size); 12413c4d7559SDave Watson release_sock(sk); 124279ffe608SJakub Kicinski mutex_unlock(&tls_ctx->tx_lock); 124345e5be84SDavid Howells return ret; 12443c4d7559SDave Watson } 12453c4d7559SDave Watson 1246df720d28SDavid Howells /* 1247df720d28SDavid Howells * Handle unexpected EOF during splice without SPLICE_F_MORE set. 1248df720d28SDavid Howells */ 1249df720d28SDavid Howells void tls_sw_splice_eof(struct socket *sock) 1250df720d28SDavid Howells { 1251df720d28SDavid Howells struct sock *sk = sock->sk; 1252df720d28SDavid Howells struct tls_context *tls_ctx = tls_get_ctx(sk); 1253df720d28SDavid Howells struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1254df720d28SDavid Howells struct tls_rec *rec; 1255df720d28SDavid Howells struct sk_msg *msg_pl; 1256df720d28SDavid Howells ssize_t copied = 0; 1257df720d28SDavid Howells bool retrying = false; 1258df720d28SDavid Howells int ret = 0; 1259df720d28SDavid Howells 1260df720d28SDavid Howells if (!ctx->open_rec) 1261df720d28SDavid Howells return; 1262df720d28SDavid Howells 1263df720d28SDavid Howells mutex_lock(&tls_ctx->tx_lock); 1264df720d28SDavid Howells lock_sock(sk); 1265df720d28SDavid Howells 1266df720d28SDavid Howells retry: 12672214e2bbSJann Horn /* same checks as in tls_sw_push_pending_record() */ 1268df720d28SDavid Howells rec = ctx->open_rec; 1269df720d28SDavid Howells if (!rec) 1270df720d28SDavid Howells goto unlock; 1271df720d28SDavid Howells 1272df720d28SDavid Howells msg_pl = &rec->msg_plaintext; 12732214e2bbSJann Horn if (msg_pl->sg.size == 0) 12742214e2bbSJann Horn goto unlock; 1275df720d28SDavid Howells 1276df720d28SDavid Howells /* Check the BPF advisor and perform transmission. */ 1277df720d28SDavid Howells ret = bpf_exec_tx_verdict(msg_pl, sk, false, TLS_RECORD_TYPE_DATA, 1278df720d28SDavid Howells &copied, 0); 1279df720d28SDavid Howells switch (ret) { 1280df720d28SDavid Howells case 0: 1281df720d28SDavid Howells case -EAGAIN: 1282df720d28SDavid Howells if (retrying) 1283df720d28SDavid Howells goto unlock; 1284df720d28SDavid Howells retrying = true; 1285df720d28SDavid Howells goto retry; 1286df720d28SDavid Howells case -EINPROGRESS: 1287df720d28SDavid Howells break; 1288df720d28SDavid Howells default: 1289df720d28SDavid Howells goto unlock; 1290df720d28SDavid Howells } 1291df720d28SDavid Howells 1292df720d28SDavid Howells /* Wait for pending encryptions to get completed */ 1293de31df6fSJakub Kicinski if (tls_encrypt_async_wait(ctx)) 1294df720d28SDavid Howells goto unlock; 1295df720d28SDavid Howells 1296df720d28SDavid Howells /* Transmit if any encryptions have completed */ 1297df720d28SDavid Howells if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 1298df720d28SDavid Howells cancel_delayed_work(&ctx->tx_work.work); 1299df720d28SDavid Howells tls_tx_records(sk, 0); 1300df720d28SDavid Howells } 1301df720d28SDavid Howells 1302df720d28SDavid Howells unlock: 1303df720d28SDavid Howells release_sock(sk); 1304df720d28SDavid Howells mutex_unlock(&tls_ctx->tx_lock); 1305df720d28SDavid Howells } 1306df720d28SDavid Howells 130735560b7fSJakub Kicinski static int 130835560b7fSJakub Kicinski tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock, 130970f03fc2SJakub Kicinski bool released) 1310c46234ebSDave Watson { 1311c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1312f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1313c46234ebSDave Watson DEFINE_WAIT_FUNC(wait, woken_wake_function); 1314419ce133SPaolo Abeni int ret = 0; 131570f03fc2SJakub Kicinski long timeo; 131670f03fc2SJakub Kicinski 131770f03fc2SJakub Kicinski timeo = sock_rcvtimeo(sk, nonblock); 1318c46234ebSDave Watson 1319b92a13d4SJakub Kicinski while (!tls_strp_msg_ready(ctx)) { 132035560b7fSJakub Kicinski if (!sk_psock_queue_empty(psock)) 132135560b7fSJakub Kicinski return 0; 132235560b7fSJakub Kicinski 132335560b7fSJakub Kicinski if (sk->sk_err) 132435560b7fSJakub Kicinski return sock_error(sk); 1325c46234ebSDave Watson 1326419ce133SPaolo Abeni if (ret < 0) 1327419ce133SPaolo Abeni return ret; 1328419ce133SPaolo Abeni 132920ffc7adSVadim Fedorenko if (!skb_queue_empty(&sk->sk_receive_queue)) { 133084c61fe1SJakub Kicinski tls_strp_check_rcv(&ctx->strp); 1331b92a13d4SJakub Kicinski if (tls_strp_msg_ready(ctx)) 133235560b7fSJakub Kicinski break; 133320ffc7adSVadim Fedorenko } 133420ffc7adSVadim Fedorenko 1335fcf4793eSDoron Roberts-Kedes if (sk->sk_shutdown & RCV_SHUTDOWN) 133635560b7fSJakub Kicinski return 0; 1337fcf4793eSDoron Roberts-Kedes 1338c46234ebSDave Watson if (sock_flag(sk, SOCK_DONE)) 133935560b7fSJakub Kicinski return 0; 1340c46234ebSDave Watson 134170f03fc2SJakub Kicinski if (!timeo) 134235560b7fSJakub Kicinski return -EAGAIN; 1343c46234ebSDave Watson 134484c61fe1SJakub Kicinski released = true; 1345c46234ebSDave Watson add_wait_queue(sk_sleep(sk), &wait); 1346c46234ebSDave Watson sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1347419ce133SPaolo Abeni ret = sk_wait_event(sk, &timeo, 1348b92a13d4SJakub Kicinski tls_strp_msg_ready(ctx) || 1349b92a13d4SJakub Kicinski !sk_psock_queue_empty(psock), 1350d3b18ad3SJohn Fastabend &wait); 1351c46234ebSDave Watson sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1352c46234ebSDave Watson remove_wait_queue(sk_sleep(sk), &wait); 1353c46234ebSDave Watson 1354c46234ebSDave Watson /* Handle signals */ 135535560b7fSJakub Kicinski if (signal_pending(current)) 135635560b7fSJakub Kicinski return sock_intr_errno(timeo); 1357c46234ebSDave Watson } 1358c46234ebSDave Watson 135984c61fe1SJakub Kicinski tls_strp_msg_load(&ctx->strp, released); 136084c61fe1SJakub Kicinski 136135560b7fSJakub Kicinski return 1; 1362c46234ebSDave Watson } 1363c46234ebSDave Watson 1364d4bd88e6SJakub Kicinski static int tls_setup_from_iter(struct iov_iter *from, 1365d829e9c4SDaniel Borkmann int length, int *pages_used, 1366d829e9c4SDaniel Borkmann struct scatterlist *to, 1367d829e9c4SDaniel Borkmann int to_max_pages) 1368d829e9c4SDaniel Borkmann { 1369d829e9c4SDaniel Borkmann int rc = 0, i = 0, num_elem = *pages_used, maxpages; 1370d829e9c4SDaniel Borkmann struct page *pages[MAX_SKB_FRAGS]; 1371d4bd88e6SJakub Kicinski unsigned int size = 0; 1372d829e9c4SDaniel Borkmann ssize_t copied, use; 1373d829e9c4SDaniel Borkmann size_t offset; 1374d829e9c4SDaniel Borkmann 1375d829e9c4SDaniel Borkmann while (length > 0) { 1376d829e9c4SDaniel Borkmann i = 0; 1377d829e9c4SDaniel Borkmann maxpages = to_max_pages - num_elem; 1378d829e9c4SDaniel Borkmann if (maxpages == 0) { 1379d829e9c4SDaniel Borkmann rc = -EFAULT; 1380d829e9c4SDaniel Borkmann goto out; 1381d829e9c4SDaniel Borkmann } 13821ef255e2SAl Viro copied = iov_iter_get_pages2(from, pages, 1383d829e9c4SDaniel Borkmann length, 1384d829e9c4SDaniel Borkmann maxpages, &offset); 1385d829e9c4SDaniel Borkmann if (copied <= 0) { 1386d829e9c4SDaniel Borkmann rc = -EFAULT; 1387d829e9c4SDaniel Borkmann goto out; 1388d829e9c4SDaniel Borkmann } 1389d829e9c4SDaniel Borkmann 1390d829e9c4SDaniel Borkmann length -= copied; 1391d829e9c4SDaniel Borkmann size += copied; 1392d829e9c4SDaniel Borkmann while (copied) { 1393d829e9c4SDaniel Borkmann use = min_t(int, copied, PAGE_SIZE - offset); 1394d829e9c4SDaniel Borkmann 1395d829e9c4SDaniel Borkmann sg_set_page(&to[num_elem], 1396d829e9c4SDaniel Borkmann pages[i], use, offset); 1397d829e9c4SDaniel Borkmann sg_unmark_end(&to[num_elem]); 1398d829e9c4SDaniel Borkmann /* We do not uncharge memory from this API */ 1399d829e9c4SDaniel Borkmann 1400d829e9c4SDaniel Borkmann offset = 0; 1401d829e9c4SDaniel Borkmann copied -= use; 1402d829e9c4SDaniel Borkmann 1403d829e9c4SDaniel Borkmann i++; 1404d829e9c4SDaniel Borkmann num_elem++; 1405d829e9c4SDaniel Borkmann } 1406d829e9c4SDaniel Borkmann } 1407d829e9c4SDaniel Borkmann /* Mark the end in the last sg entry if newly added */ 1408d829e9c4SDaniel Borkmann if (num_elem > *pages_used) 1409d829e9c4SDaniel Borkmann sg_mark_end(&to[num_elem - 1]); 1410d829e9c4SDaniel Borkmann out: 1411d829e9c4SDaniel Borkmann if (rc) 1412d4bd88e6SJakub Kicinski iov_iter_revert(from, size); 1413d829e9c4SDaniel Borkmann *pages_used = num_elem; 1414d829e9c4SDaniel Borkmann 1415d829e9c4SDaniel Borkmann return rc; 1416d829e9c4SDaniel Borkmann } 1417d829e9c4SDaniel Borkmann 1418fd31f399SJakub Kicinski static struct sk_buff * 1419fd31f399SJakub Kicinski tls_alloc_clrtxt_skb(struct sock *sk, struct sk_buff *skb, 1420fd31f399SJakub Kicinski unsigned int full_len) 1421fd31f399SJakub Kicinski { 1422fd31f399SJakub Kicinski struct strp_msg *clr_rxm; 1423fd31f399SJakub Kicinski struct sk_buff *clr_skb; 1424fd31f399SJakub Kicinski int err; 1425fd31f399SJakub Kicinski 1426fd31f399SJakub Kicinski clr_skb = alloc_skb_with_frags(0, full_len, TLS_PAGE_ORDER, 1427fd31f399SJakub Kicinski &err, sk->sk_allocation); 1428fd31f399SJakub Kicinski if (!clr_skb) 1429fd31f399SJakub Kicinski return NULL; 1430fd31f399SJakub Kicinski 1431fd31f399SJakub Kicinski skb_copy_header(clr_skb, skb); 1432fd31f399SJakub Kicinski clr_skb->len = full_len; 1433fd31f399SJakub Kicinski clr_skb->data_len = full_len; 1434fd31f399SJakub Kicinski 1435fd31f399SJakub Kicinski clr_rxm = strp_msg(clr_skb); 1436fd31f399SJakub Kicinski clr_rxm->offset = 0; 1437fd31f399SJakub Kicinski 1438fd31f399SJakub Kicinski return clr_skb; 1439fd31f399SJakub Kicinski } 1440fd31f399SJakub Kicinski 14418a958732SJakub Kicinski /* Decrypt handlers 14428a958732SJakub Kicinski * 1443dd47ed36SJakub Kicinski * tls_decrypt_sw() and tls_decrypt_device() are decrypt handlers. 14448a958732SJakub Kicinski * They must transform the darg in/out argument are as follows: 14458a958732SJakub Kicinski * | Input | Output 14468a958732SJakub Kicinski * ------------------------------------------------------------------- 14478a958732SJakub Kicinski * zc | Zero-copy decrypt allowed | Zero-copy performed 14488a958732SJakub Kicinski * async | Async decrypt allowed | Async crypto used / in progress 14496bd116c8SJakub Kicinski * skb | * | Output skb 1450b93f5700SJakub Kicinski * 1451b93f5700SJakub Kicinski * If ZC decryption was performed darg.skb will point to the input skb. 14528a958732SJakub Kicinski */ 14538a958732SJakub Kicinski 14540b243d00SVakul Garg /* This function decrypts the input skb into either out_iov or in out_sg 14558a958732SJakub Kicinski * or in skb buffers itself. The input parameter 'darg->zc' indicates if 14560b243d00SVakul Garg * zero-copy mode needs to be tried or not. With zero-copy mode, either 14570b243d00SVakul Garg * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are 14580b243d00SVakul Garg * NULL, then the decryption happens inside skb buffers itself, i.e. 14598a958732SJakub Kicinski * zero-copy gets disabled and 'darg->zc' is updated. 14600b243d00SVakul Garg */ 1461541cc48bSJakub Kicinski static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, 14620b243d00SVakul Garg struct scatterlist *out_sg, 14634175eac3SJakub Kicinski struct tls_decrypt_arg *darg) 14640b243d00SVakul Garg { 14650b243d00SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 14660b243d00SVakul Garg struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 14674509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 1468b89fec54SJakub Kicinski int n_sgin, n_sgout, aead_size, err, pages = 0; 1469541cc48bSJakub Kicinski struct sk_buff *skb = tls_strp_msg(ctx); 1470fd31f399SJakub Kicinski const struct strp_msg *rxm = strp_msg(skb); 1471fd31f399SJakub Kicinski const struct tls_msg *tlm = tls_msg(skb); 14720b243d00SVakul Garg struct aead_request *aead_req; 14730b243d00SVakul Garg struct scatterlist *sgin = NULL; 14740b243d00SVakul Garg struct scatterlist *sgout = NULL; 1475603380f5SJakub Kicinski const int data_len = rxm->full_len - prot->overhead_size; 1476ce61327cSJakub Kicinski int tail_pages = !!prot->tail_size; 1477b89fec54SJakub Kicinski struct tls_decrypt_ctx *dctx; 1478fd31f399SJakub Kicinski struct sk_buff *clear_skb; 1479f295b3aeSVakul Garg int iv_offset = 0; 1480b89fec54SJakub Kicinski u8 *mem; 14810b243d00SVakul Garg 1482fd31f399SJakub Kicinski n_sgin = skb_nsg(skb, rxm->offset + prot->prepend_size, 1483fd31f399SJakub Kicinski rxm->full_len - prot->prepend_size); 1484fd31f399SJakub Kicinski if (n_sgin < 1) 1485fd31f399SJakub Kicinski return n_sgin ?: -EBADMSG; 1486fd31f399SJakub Kicinski 14874175eac3SJakub Kicinski if (darg->zc && (out_iov || out_sg)) { 1488fd31f399SJakub Kicinski clear_skb = NULL; 1489fd31f399SJakub Kicinski 14900b243d00SVakul Garg if (out_iov) 1491ce61327cSJakub Kicinski n_sgout = 1 + tail_pages + 1492b93235e6SJakub Kicinski iov_iter_npages_cap(out_iov, INT_MAX, data_len); 14930b243d00SVakul Garg else 14940b243d00SVakul Garg n_sgout = sg_nents(out_sg); 14950b243d00SVakul Garg } else { 14964175eac3SJakub Kicinski darg->zc = false; 14970b243d00SVakul Garg 1498fd31f399SJakub Kicinski clear_skb = tls_alloc_clrtxt_skb(sk, skb, rxm->full_len); 1499fd31f399SJakub Kicinski if (!clear_skb) 1500fd31f399SJakub Kicinski return -ENOMEM; 1501fd31f399SJakub Kicinski 1502fd31f399SJakub Kicinski n_sgout = 1 + skb_shinfo(clear_skb)->nr_frags; 1503fd31f399SJakub Kicinski } 15040b243d00SVakul Garg 15050b243d00SVakul Garg /* Increment to accommodate AAD */ 15060b243d00SVakul Garg n_sgin = n_sgin + 1; 15070b243d00SVakul Garg 15080b243d00SVakul Garg /* Allocate a single block of memory which contains 1509b89fec54SJakub Kicinski * aead_req || tls_decrypt_ctx. 1510b89fec54SJakub Kicinski * Both structs are variable length. 15110b243d00SVakul Garg */ 1512b89fec54SJakub Kicinski aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv); 15138d338c76SHerbert Xu aead_size = ALIGN(aead_size, __alignof__(*dctx)); 15144c7f066bSGustavo A. R. Silva mem = kmalloc(aead_size + struct_size(dctx, sg, size_add(n_sgin, n_sgout)), 1515b89fec54SJakub Kicinski sk->sk_allocation); 1516fd31f399SJakub Kicinski if (!mem) { 1517fd31f399SJakub Kicinski err = -ENOMEM; 1518fd31f399SJakub Kicinski goto exit_free_skb; 1519fd31f399SJakub Kicinski } 15200b243d00SVakul Garg 15210b243d00SVakul Garg /* Segment the allocated memory */ 15220b243d00SVakul Garg aead_req = (struct aead_request *)mem; 1523b89fec54SJakub Kicinski dctx = (struct tls_decrypt_ctx *)(mem + aead_size); 15248d338c76SHerbert Xu dctx->sk = sk; 1525b89fec54SJakub Kicinski sgin = &dctx->sg[0]; 1526b89fec54SJakub Kicinski sgout = &dctx->sg[n_sgin]; 15270b243d00SVakul Garg 1528128cfb88STianjia Zhang /* For CCM based ciphers, first byte of nonce+iv is a constant */ 1529128cfb88STianjia Zhang switch (prot->cipher_type) { 1530128cfb88STianjia Zhang case TLS_CIPHER_AES_CCM_128: 1531b89fec54SJakub Kicinski dctx->iv[0] = TLS_AES_CCM_IV_B0_BYTE; 1532f295b3aeSVakul Garg iv_offset = 1; 1533128cfb88STianjia Zhang break; 1534128cfb88STianjia Zhang case TLS_CIPHER_SM4_CCM: 1535b89fec54SJakub Kicinski dctx->iv[0] = TLS_SM4_CCM_IV_B0_BYTE; 1536128cfb88STianjia Zhang iv_offset = 1; 1537128cfb88STianjia Zhang break; 1538f295b3aeSVakul Garg } 1539f295b3aeSVakul Garg 15400b243d00SVakul Garg /* Prepare IV */ 1541a4ae58cdSJakub Kicinski if (prot->version == TLS_1_3_VERSION || 1542a4ae58cdSJakub Kicinski prot->cipher_type == TLS_CIPHER_CHACHA20_POLY1305) { 1543b89fec54SJakub Kicinski memcpy(&dctx->iv[iv_offset], tls_ctx->rx.iv, 1544a4ae58cdSJakub Kicinski prot->iv_size + prot->salt_size); 1545a4ae58cdSJakub Kicinski } else { 15460b243d00SVakul Garg err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE, 1547b89fec54SJakub Kicinski &dctx->iv[iv_offset] + prot->salt_size, 15484509de14SVakul Garg prot->iv_size); 154903957d84SJakub Kicinski if (err < 0) 155003957d84SJakub Kicinski goto exit_free; 1551b89fec54SJakub Kicinski memcpy(&dctx->iv[iv_offset], tls_ctx->rx.iv, prot->salt_size); 1552a4ae58cdSJakub Kicinski } 155358790314SJakub Kicinski tls_xor_iv_with_seq(prot, &dctx->iv[iv_offset], tls_ctx->rx.rec_seq); 1554130b392cSDave Watson 15550b243d00SVakul Garg /* Prepare AAD */ 1556b89fec54SJakub Kicinski tls_make_aad(dctx->aad, rxm->full_len - prot->overhead_size + 15574509de14SVakul Garg prot->tail_size, 1558c3f6bb74SJakub Kicinski tls_ctx->rx.rec_seq, tlm->control, prot); 15590b243d00SVakul Garg 15600b243d00SVakul Garg /* Prepare sgin */ 15610b243d00SVakul Garg sg_init_table(sgin, n_sgin); 1562b89fec54SJakub Kicinski sg_set_buf(&sgin[0], dctx->aad, prot->aad_size); 15630b243d00SVakul Garg err = skb_to_sgvec(skb, &sgin[1], 15644509de14SVakul Garg rxm->offset + prot->prepend_size, 15654509de14SVakul Garg rxm->full_len - prot->prepend_size); 156603957d84SJakub Kicinski if (err < 0) 156703957d84SJakub Kicinski goto exit_free; 15680b243d00SVakul Garg 1569fd31f399SJakub Kicinski if (clear_skb) { 15700b243d00SVakul Garg sg_init_table(sgout, n_sgout); 1571b89fec54SJakub Kicinski sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); 15720b243d00SVakul Garg 1573fd31f399SJakub Kicinski err = skb_to_sgvec(clear_skb, &sgout[1], prot->prepend_size, 1574fd31f399SJakub Kicinski data_len + prot->tail_size); 1575fd31f399SJakub Kicinski if (err < 0) 1576fd31f399SJakub Kicinski goto exit_free; 1577fd31f399SJakub Kicinski } else if (out_iov) { 1578fd31f399SJakub Kicinski sg_init_table(sgout, n_sgout); 1579fd31f399SJakub Kicinski sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); 1580fd31f399SJakub Kicinski 1581fd31f399SJakub Kicinski err = tls_setup_from_iter(out_iov, data_len, &pages, &sgout[1], 1582ce61327cSJakub Kicinski (n_sgout - 1 - tail_pages)); 15830b243d00SVakul Garg if (err < 0) 1584fd31f399SJakub Kicinski goto exit_free_pages; 1585ce61327cSJakub Kicinski 1586ce61327cSJakub Kicinski if (prot->tail_size) { 1587ce61327cSJakub Kicinski sg_unmark_end(&sgout[pages]); 1588b89fec54SJakub Kicinski sg_set_buf(&sgout[pages + 1], &dctx->tail, 1589ce61327cSJakub Kicinski prot->tail_size); 1590ce61327cSJakub Kicinski sg_mark_end(&sgout[pages + 1]); 1591ce61327cSJakub Kicinski } 15920b243d00SVakul Garg } else if (out_sg) { 15930b243d00SVakul Garg memcpy(sgout, out_sg, n_sgout * sizeof(*sgout)); 15940b243d00SVakul Garg } 1595d6847635SSabrina Dubroca dctx->free_sgout = !!pages; 15960b243d00SVakul Garg 15970b243d00SVakul Garg /* Prepare and submit AEAD request */ 15986ececdc5SJakub Kicinski err = tls_do_decryption(sk, sgin, sgout, dctx->iv, 1599603380f5SJakub Kicinski data_len + prot->tail_size, aead_req, darg); 160081be8535SSabrina Dubroca if (err) { 160181be8535SSabrina Dubroca if (darg->async_done) 160281be8535SSabrina Dubroca goto exit_free_skb; 16036bd116c8SJakub Kicinski goto exit_free_pages; 160481be8535SSabrina Dubroca } 16056bd116c8SJakub Kicinski 1606fd31f399SJakub Kicinski darg->skb = clear_skb ?: tls_strp_msg(ctx); 1607fd31f399SJakub Kicinski clear_skb = NULL; 1608c618db2aSJakub Kicinski 1609c618db2aSJakub Kicinski if (unlikely(darg->async)) { 161084c61fe1SJakub Kicinski err = tls_strp_msg_hold(&ctx->strp, &ctx->async_hold); 1611c618db2aSJakub Kicinski if (err) 1612c618db2aSJakub Kicinski __skb_queue_tail(&ctx->async_hold, darg->skb); 1613c618db2aSJakub Kicinski return err; 1614c618db2aSJakub Kicinski } 16150b243d00SVakul Garg 161681be8535SSabrina Dubroca if (unlikely(darg->async_done)) 161781be8535SSabrina Dubroca return 0; 161881be8535SSabrina Dubroca 1619ce61327cSJakub Kicinski if (prot->tail_size) 1620b89fec54SJakub Kicinski darg->tail = dctx->tail; 1621ce61327cSJakub Kicinski 16226bd116c8SJakub Kicinski exit_free_pages: 16230b243d00SVakul Garg /* Release the pages in case iov was mapped to pages */ 16240b243d00SVakul Garg for (; pages > 0; pages--) 16250b243d00SVakul Garg put_page(sg_page(&sgout[pages])); 162603957d84SJakub Kicinski exit_free: 16270b243d00SVakul Garg kfree(mem); 1628fd31f399SJakub Kicinski exit_free_skb: 1629fd31f399SJakub Kicinski consume_skb(clear_skb); 16300b243d00SVakul Garg return err; 16310b243d00SVakul Garg } 16320b243d00SVakul Garg 16338a958732SJakub Kicinski static int 1634dd47ed36SJakub Kicinski tls_decrypt_sw(struct sock *sk, struct tls_context *tls_ctx, 1635dd47ed36SJakub Kicinski struct msghdr *msg, struct tls_decrypt_arg *darg) 16368a958732SJakub Kicinski { 16376bd116c8SJakub Kicinski struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 16384509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 1639541cc48bSJakub Kicinski struct strp_msg *rxm; 16403764ae5bSJakub Kicinski int pad, err; 1641dafb67f3SBoris Pismenny 1642dd47ed36SJakub Kicinski err = tls_decrypt_sg(sk, &msg->msg_iter, NULL, darg); 1643a069a905SGal Pressman if (err < 0) { 1644a069a905SGal Pressman if (err == -EBADMSG) 1645a069a905SGal Pressman TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTERROR); 1646dafb67f3SBoris Pismenny return err; 1647a069a905SGal Pressman } 1648dd47ed36SJakub Kicinski /* keep going even for ->async, the code below is TLS 1.3 */ 1649dd47ed36SJakub Kicinski 1650ce61327cSJakub Kicinski /* If opportunistic TLS 1.3 ZC failed retry without ZC */ 1651ce61327cSJakub Kicinski if (unlikely(darg->zc && prot->version == TLS_1_3_VERSION && 1652ce61327cSJakub Kicinski darg->tail != TLS_RECORD_TYPE_DATA)) { 1653ce61327cSJakub Kicinski darg->zc = false; 1654bb56cea9SJakub Kicinski if (!darg->tail) 1655bb56cea9SJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXNOPADVIOL); 16561090c1eaSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTRETRY); 1657dd47ed36SJakub Kicinski return tls_decrypt_sw(sk, tls_ctx, msg, darg); 1658ce61327cSJakub Kicinski } 1659130b392cSDave Watson 16606bd116c8SJakub Kicinski pad = tls_padding_length(prot, darg->skb, darg); 16616bd116c8SJakub Kicinski if (pad < 0) { 1662b93f5700SJakub Kicinski if (darg->skb != tls_strp_msg(ctx)) 16636bd116c8SJakub Kicinski consume_skb(darg->skb); 16646bd116c8SJakub Kicinski return pad; 16656bd116c8SJakub Kicinski } 16666bd116c8SJakub Kicinski 16676bd116c8SJakub Kicinski rxm = strp_msg(darg->skb); 1668b53f4976SJakub Kicinski rxm->full_len -= pad; 1669dd47ed36SJakub Kicinski 1670dd47ed36SJakub Kicinski return 0; 1671dd47ed36SJakub Kicinski } 1672dd47ed36SJakub Kicinski 1673dd47ed36SJakub Kicinski static int 1674d4e5db64SJakub Kicinski tls_decrypt_device(struct sock *sk, struct msghdr *msg, 1675d4e5db64SJakub Kicinski struct tls_context *tls_ctx, struct tls_decrypt_arg *darg) 1676dd47ed36SJakub Kicinski { 1677dd47ed36SJakub Kicinski struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1678dd47ed36SJakub Kicinski struct tls_prot_info *prot = &tls_ctx->prot_info; 1679dd47ed36SJakub Kicinski struct strp_msg *rxm; 1680dd47ed36SJakub Kicinski int pad, err; 1681dd47ed36SJakub Kicinski 1682dd47ed36SJakub Kicinski if (tls_ctx->rx_conf != TLS_HW) 1683dd47ed36SJakub Kicinski return 0; 1684dd47ed36SJakub Kicinski 1685dd47ed36SJakub Kicinski err = tls_device_decrypted(sk, tls_ctx); 1686dd47ed36SJakub Kicinski if (err <= 0) 1687dd47ed36SJakub Kicinski return err; 1688dd47ed36SJakub Kicinski 1689dd47ed36SJakub Kicinski pad = tls_padding_length(prot, tls_strp_msg(ctx), darg); 1690dd47ed36SJakub Kicinski if (pad < 0) 1691dd47ed36SJakub Kicinski return pad; 1692dd47ed36SJakub Kicinski 1693dd47ed36SJakub Kicinski darg->async = false; 1694dd47ed36SJakub Kicinski darg->skb = tls_strp_msg(ctx); 1695d4e5db64SJakub Kicinski /* ->zc downgrade check, in case TLS 1.3 gets here */ 1696d4e5db64SJakub Kicinski darg->zc &= !(prot->version == TLS_1_3_VERSION && 1697d4e5db64SJakub Kicinski tls_msg(darg->skb)->control != TLS_RECORD_TYPE_DATA); 1698dd47ed36SJakub Kicinski 1699dd47ed36SJakub Kicinski rxm = strp_msg(darg->skb); 1700dd47ed36SJakub Kicinski rxm->full_len -= pad; 1701d4e5db64SJakub Kicinski 1702d4e5db64SJakub Kicinski if (!darg->zc) { 1703d4e5db64SJakub Kicinski /* Non-ZC case needs a real skb */ 1704d4e5db64SJakub Kicinski darg->skb = tls_strp_msg_detach(ctx); 1705d4e5db64SJakub Kicinski if (!darg->skb) 1706d4e5db64SJakub Kicinski return -ENOMEM; 1707d4e5db64SJakub Kicinski } else { 1708d4e5db64SJakub Kicinski unsigned int off, len; 1709d4e5db64SJakub Kicinski 1710d4e5db64SJakub Kicinski /* In ZC case nobody cares about the output skb. 1711d4e5db64SJakub Kicinski * Just copy the data here. Note the skb is not fully trimmed. 1712d4e5db64SJakub Kicinski */ 1713d4e5db64SJakub Kicinski off = rxm->offset + prot->prepend_size; 1714d4e5db64SJakub Kicinski len = rxm->full_len - prot->overhead_size; 1715d4e5db64SJakub Kicinski 1716d4e5db64SJakub Kicinski err = skb_copy_datagram_msg(darg->skb, off, msg, len); 1717d4e5db64SJakub Kicinski if (err) 1718d4e5db64SJakub Kicinski return err; 1719d4e5db64SJakub Kicinski } 1720dd47ed36SJakub Kicinski return 1; 1721dd47ed36SJakub Kicinski } 1722dd47ed36SJakub Kicinski 1723dd47ed36SJakub Kicinski static int tls_rx_one_record(struct sock *sk, struct msghdr *msg, 1724dd47ed36SJakub Kicinski struct tls_decrypt_arg *darg) 1725dd47ed36SJakub Kicinski { 1726dd47ed36SJakub Kicinski struct tls_context *tls_ctx = tls_get_ctx(sk); 1727dd47ed36SJakub Kicinski struct tls_prot_info *prot = &tls_ctx->prot_info; 1728dd47ed36SJakub Kicinski struct strp_msg *rxm; 1729dd47ed36SJakub Kicinski int err; 1730dd47ed36SJakub Kicinski 1731d4e5db64SJakub Kicinski err = tls_decrypt_device(sk, msg, tls_ctx, darg); 1732dd47ed36SJakub Kicinski if (!err) 1733dd47ed36SJakub Kicinski err = tls_decrypt_sw(sk, tls_ctx, msg, darg); 1734dd47ed36SJakub Kicinski if (err < 0) 1735dd47ed36SJakub Kicinski return err; 1736dd47ed36SJakub Kicinski 1737dd47ed36SJakub Kicinski rxm = strp_msg(darg->skb); 17384509de14SVakul Garg rxm->offset += prot->prepend_size; 17394509de14SVakul Garg rxm->full_len -= prot->overhead_size; 17403547a1f9SJakub Kicinski tls_advance_record_sn(sk, prot, &tls_ctx->rx); 1741dafb67f3SBoris Pismenny 17423764ae5bSJakub Kicinski return 0; 1743dafb67f3SBoris Pismenny } 1744dafb67f3SBoris Pismenny 1745541cc48bSJakub Kicinski int decrypt_skb(struct sock *sk, struct scatterlist *sgout) 1746c46234ebSDave Watson { 17474175eac3SJakub Kicinski struct tls_decrypt_arg darg = { .zc = true, }; 1748c46234ebSDave Watson 1749541cc48bSJakub Kicinski return tls_decrypt_sg(sk, NULL, sgout, &darg); 1750c46234ebSDave Watson } 1751c46234ebSDave Watson 175206554f4fSJakub Kicinski static int tls_record_content_type(struct msghdr *msg, struct tls_msg *tlm, 175306554f4fSJakub Kicinski u8 *control) 175406554f4fSJakub Kicinski { 175506554f4fSJakub Kicinski int err; 175606554f4fSJakub Kicinski 175706554f4fSJakub Kicinski if (!*control) { 175806554f4fSJakub Kicinski *control = tlm->control; 175906554f4fSJakub Kicinski if (!*control) 176006554f4fSJakub Kicinski return -EBADMSG; 176106554f4fSJakub Kicinski 176206554f4fSJakub Kicinski err = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, 176306554f4fSJakub Kicinski sizeof(*control), control); 176406554f4fSJakub Kicinski if (*control != TLS_RECORD_TYPE_DATA) { 176506554f4fSJakub Kicinski if (err || msg->msg_flags & MSG_CTRUNC) 176606554f4fSJakub Kicinski return -EIO; 176706554f4fSJakub Kicinski } 176806554f4fSJakub Kicinski } else if (*control != tlm->control) { 176906554f4fSJakub Kicinski return 0; 177006554f4fSJakub Kicinski } 177106554f4fSJakub Kicinski 177206554f4fSJakub Kicinski return 1; 177306554f4fSJakub Kicinski } 177406554f4fSJakub Kicinski 1775abb47dc9SJakub Kicinski static void tls_rx_rec_done(struct tls_sw_context_rx *ctx) 1776abb47dc9SJakub Kicinski { 177784c61fe1SJakub Kicinski tls_strp_msg_done(&ctx->strp); 1778abb47dc9SJakub Kicinski } 1779abb47dc9SJakub Kicinski 1780692d7b5dSVakul Garg /* This function traverses the rx_list in tls receive context to copies the 17812b794c40SVakul Garg * decrypted records into the buffer provided by caller zero copy is not 1782692d7b5dSVakul Garg * true. Further, the records are removed from the rx_list if it is not a peek 1783692d7b5dSVakul Garg * case and the record has been consumed completely. 1784692d7b5dSVakul Garg */ 1785692d7b5dSVakul Garg static int process_rx_list(struct tls_sw_context_rx *ctx, 1786692d7b5dSVakul Garg struct msghdr *msg, 17872b794c40SVakul Garg u8 *control, 1788692d7b5dSVakul Garg size_t skip, 1789692d7b5dSVakul Garg size_t len, 17904f13a79eSSabrina Dubroca bool is_peek, 17914f13a79eSSabrina Dubroca bool *more) 1792692d7b5dSVakul Garg { 1793692d7b5dSVakul Garg struct sk_buff *skb = skb_peek(&ctx->rx_list); 17942b794c40SVakul Garg struct tls_msg *tlm; 1795692d7b5dSVakul Garg ssize_t copied = 0; 179606554f4fSJakub Kicinski int err; 17972b794c40SVakul Garg 1798692d7b5dSVakul Garg while (skip && skb) { 1799692d7b5dSVakul Garg struct strp_msg *rxm = strp_msg(skb); 18002b794c40SVakul Garg tlm = tls_msg(skb); 18012b794c40SVakul Garg 180206554f4fSJakub Kicinski err = tls_record_content_type(msg, tlm, control); 180306554f4fSJakub Kicinski if (err <= 0) 18044f13a79eSSabrina Dubroca goto more; 1805692d7b5dSVakul Garg 1806692d7b5dSVakul Garg if (skip < rxm->full_len) 1807692d7b5dSVakul Garg break; 1808692d7b5dSVakul Garg 1809692d7b5dSVakul Garg skip = skip - rxm->full_len; 1810692d7b5dSVakul Garg skb = skb_peek_next(skb, &ctx->rx_list); 1811692d7b5dSVakul Garg } 1812692d7b5dSVakul Garg 1813692d7b5dSVakul Garg while (len && skb) { 1814692d7b5dSVakul Garg struct sk_buff *next_skb; 1815692d7b5dSVakul Garg struct strp_msg *rxm = strp_msg(skb); 1816692d7b5dSVakul Garg int chunk = min_t(unsigned int, rxm->full_len - skip, len); 1817692d7b5dSVakul Garg 18182b794c40SVakul Garg tlm = tls_msg(skb); 18192b794c40SVakul Garg 182006554f4fSJakub Kicinski err = tls_record_content_type(msg, tlm, control); 182106554f4fSJakub Kicinski if (err <= 0) 18224f13a79eSSabrina Dubroca goto more; 18232b794c40SVakul Garg 182406554f4fSJakub Kicinski err = skb_copy_datagram_msg(skb, rxm->offset + skip, 1825692d7b5dSVakul Garg msg, chunk); 1826692d7b5dSVakul Garg if (err < 0) 18274f13a79eSSabrina Dubroca goto more; 1828692d7b5dSVakul Garg 1829692d7b5dSVakul Garg len = len - chunk; 1830692d7b5dSVakul Garg copied = copied + chunk; 1831692d7b5dSVakul Garg 1832692d7b5dSVakul Garg /* Consume the data from record if it is non-peek case*/ 1833692d7b5dSVakul Garg if (!is_peek) { 1834692d7b5dSVakul Garg rxm->offset = rxm->offset + chunk; 1835692d7b5dSVakul Garg rxm->full_len = rxm->full_len - chunk; 1836692d7b5dSVakul Garg 1837692d7b5dSVakul Garg /* Return if there is unconsumed data in the record */ 1838692d7b5dSVakul Garg if (rxm->full_len - skip) 1839692d7b5dSVakul Garg break; 1840692d7b5dSVakul Garg } 1841692d7b5dSVakul Garg 1842692d7b5dSVakul Garg /* The remaining skip-bytes must lie in 1st record in rx_list. 1843692d7b5dSVakul Garg * So from the 2nd record, 'skip' should be 0. 1844692d7b5dSVakul Garg */ 1845692d7b5dSVakul Garg skip = 0; 1846692d7b5dSVakul Garg 1847692d7b5dSVakul Garg if (msg) 1848692d7b5dSVakul Garg msg->msg_flags |= MSG_EOR; 1849692d7b5dSVakul Garg 1850692d7b5dSVakul Garg next_skb = skb_peek_next(skb, &ctx->rx_list); 1851692d7b5dSVakul Garg 1852692d7b5dSVakul Garg if (!is_peek) { 1853a30295c4SJakub Kicinski __skb_unlink(skb, &ctx->rx_list); 1854a88c26f6SVakul Garg consume_skb(skb); 1855692d7b5dSVakul Garg } 1856692d7b5dSVakul Garg 1857692d7b5dSVakul Garg skb = next_skb; 1858692d7b5dSVakul Garg } 18594dcdd971SJakub Kicinski err = 0; 1860692d7b5dSVakul Garg 18614dcdd971SJakub Kicinski out: 18624dcdd971SJakub Kicinski return copied ? : err; 18634f13a79eSSabrina Dubroca more: 18644f13a79eSSabrina Dubroca if (more) 18654f13a79eSSabrina Dubroca *more = true; 18664f13a79eSSabrina Dubroca goto out; 1867692d7b5dSVakul Garg } 1868692d7b5dSVakul Garg 186984c61fe1SJakub Kicinski static bool 1870c46b0183SJakub Kicinski tls_read_flush_backlog(struct sock *sk, struct tls_prot_info *prot, 1871c46b0183SJakub Kicinski size_t len_left, size_t decrypted, ssize_t done, 1872c46b0183SJakub Kicinski size_t *flushed_at) 1873c46b0183SJakub Kicinski { 1874c46b0183SJakub Kicinski size_t max_rec; 1875c46b0183SJakub Kicinski 1876c46b0183SJakub Kicinski if (len_left <= decrypted) 187784c61fe1SJakub Kicinski return false; 1878c46b0183SJakub Kicinski 1879c46b0183SJakub Kicinski max_rec = prot->overhead_size - prot->tail_size + TLS_MAX_PAYLOAD_SIZE; 1880c46b0183SJakub Kicinski if (done - *flushed_at < SZ_128K && tcp_inq(sk) > max_rec) 188184c61fe1SJakub Kicinski return false; 1882c46b0183SJakub Kicinski 1883c46b0183SJakub Kicinski *flushed_at = done; 188484c61fe1SJakub Kicinski return sk_flush_backlog(sk); 1885c46b0183SJakub Kicinski } 1886c46b0183SJakub Kicinski 1887f9ae3204SHannes Reinecke static int tls_rx_reader_acquire(struct sock *sk, struct tls_sw_context_rx *ctx, 18884cbc325eSJakub Kicinski bool nonblock) 18894cbc325eSJakub Kicinski { 18904cbc325eSJakub Kicinski long timeo; 1891419ce133SPaolo Abeni int ret; 18924cbc325eSJakub Kicinski 18934cbc325eSJakub Kicinski timeo = sock_rcvtimeo(sk, nonblock); 18944cbc325eSJakub Kicinski 18954cbc325eSJakub Kicinski while (unlikely(ctx->reader_present)) { 18964cbc325eSJakub Kicinski DEFINE_WAIT_FUNC(wait, woken_wake_function); 18974cbc325eSJakub Kicinski 18984cbc325eSJakub Kicinski ctx->reader_contended = 1; 18994cbc325eSJakub Kicinski 19004cbc325eSJakub Kicinski add_wait_queue(&ctx->wq, &wait); 1901419ce133SPaolo Abeni ret = sk_wait_event(sk, &timeo, 19024cbc325eSJakub Kicinski !READ_ONCE(ctx->reader_present), &wait); 19034cbc325eSJakub Kicinski remove_wait_queue(&ctx->wq, &wait); 19044cbc325eSJakub Kicinski 1905f9ae3204SHannes Reinecke if (timeo <= 0) 1906f9ae3204SHannes Reinecke return -EAGAIN; 1907f9ae3204SHannes Reinecke if (signal_pending(current)) 1908f9ae3204SHannes Reinecke return sock_intr_errno(timeo); 1909419ce133SPaolo Abeni if (ret < 0) 1910419ce133SPaolo Abeni return ret; 19114cbc325eSJakub Kicinski } 19124cbc325eSJakub Kicinski 19134cbc325eSJakub Kicinski WRITE_ONCE(ctx->reader_present, 1); 19144cbc325eSJakub Kicinski 191570f03fc2SJakub Kicinski return 0; 1916f9ae3204SHannes Reinecke } 1917dde06aaaSJakub Kicinski 1918f9ae3204SHannes Reinecke static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx, 1919f9ae3204SHannes Reinecke bool nonblock) 1920f9ae3204SHannes Reinecke { 1921f9ae3204SHannes Reinecke int err; 1922f9ae3204SHannes Reinecke 1923f9ae3204SHannes Reinecke lock_sock(sk); 1924f9ae3204SHannes Reinecke err = tls_rx_reader_acquire(sk, ctx, nonblock); 1925f9ae3204SHannes Reinecke if (err) 1926dde06aaaSJakub Kicinski release_sock(sk); 1927dde06aaaSJakub Kicinski return err; 19284cbc325eSJakub Kicinski } 19294cbc325eSJakub Kicinski 1930f9ae3204SHannes Reinecke static void tls_rx_reader_release(struct sock *sk, struct tls_sw_context_rx *ctx) 19314cbc325eSJakub Kicinski { 19324cbc325eSJakub Kicinski if (unlikely(ctx->reader_contended)) { 19334cbc325eSJakub Kicinski if (wq_has_sleeper(&ctx->wq)) 19344cbc325eSJakub Kicinski wake_up(&ctx->wq); 19354cbc325eSJakub Kicinski else 19364cbc325eSJakub Kicinski ctx->reader_contended = 0; 19374cbc325eSJakub Kicinski 19384cbc325eSJakub Kicinski WARN_ON_ONCE(!ctx->reader_present); 19394cbc325eSJakub Kicinski } 19404cbc325eSJakub Kicinski 19414cbc325eSJakub Kicinski WRITE_ONCE(ctx->reader_present, 0); 1942f9ae3204SHannes Reinecke } 1943f9ae3204SHannes Reinecke 1944f9ae3204SHannes Reinecke static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx) 1945f9ae3204SHannes Reinecke { 1946f9ae3204SHannes Reinecke tls_rx_reader_release(sk, ctx); 19474cbc325eSJakub Kicinski release_sock(sk); 19484cbc325eSJakub Kicinski } 19494cbc325eSJakub Kicinski 1950c46234ebSDave Watson int tls_sw_recvmsg(struct sock *sk, 1951c46234ebSDave Watson struct msghdr *msg, 1952c46234ebSDave Watson size_t len, 1953c46234ebSDave Watson int flags, 1954c46234ebSDave Watson int *addr_len) 1955c46234ebSDave Watson { 1956c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1957f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 19584509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 1959cbbdee99SJakub Kicinski ssize_t decrypted = 0, async_copy_bytes = 0; 1960d3b18ad3SJohn Fastabend struct sk_psock *psock; 1961692d7b5dSVakul Garg unsigned char control = 0; 1962c46b0183SJakub Kicinski size_t flushed_at = 0; 1963c46234ebSDave Watson struct strp_msg *rxm; 19642b794c40SVakul Garg struct tls_msg *tlm; 1965c46234ebSDave Watson ssize_t copied = 0; 196611121c2cSSabrina Dubroca ssize_t peeked = 0; 19677da18bccSJakub Kicinski bool async = false; 196870f03fc2SJakub Kicinski int target, err; 196900e23707SDavid Howells bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); 1970692d7b5dSVakul Garg bool is_peek = flags & MSG_PEEK; 19714f13a79eSSabrina Dubroca bool rx_more = false; 197284c61fe1SJakub Kicinski bool released = true; 1973e91de6afSJohn Fastabend bool bpf_strp_enabled; 1974ba13609dSJakub Kicinski bool zc_capable; 1975c46234ebSDave Watson 1976c46234ebSDave Watson if (unlikely(flags & MSG_ERRQUEUE)) 1977c46234ebSDave Watson return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR); 1978c46234ebSDave Watson 1979d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 198070f03fc2SJakub Kicinski err = tls_rx_reader_lock(sk, ctx, flags & MSG_DONTWAIT); 198170f03fc2SJakub Kicinski if (err < 0) 198270f03fc2SJakub Kicinski return err; 1983e91de6afSJohn Fastabend bpf_strp_enabled = sk_psock_strp_enabled(psock); 1984c46234ebSDave Watson 1985f314bfeeSJakub Kicinski /* If crypto failed the connection is broken */ 1986f314bfeeSJakub Kicinski err = ctx->async_wait.err; 1987f314bfeeSJakub Kicinski if (err) 1988f314bfeeSJakub Kicinski goto end; 1989f314bfeeSJakub Kicinski 1990692d7b5dSVakul Garg /* Process pending decrypted records. It must be non-zero-copy */ 19914f13a79eSSabrina Dubroca err = process_rx_list(ctx, msg, &control, 0, len, is_peek, &rx_more); 19924dcdd971SJakub Kicinski if (err < 0) 1993692d7b5dSVakul Garg goto end; 1994692d7b5dSVakul Garg 1995d5123eddSJakub Kicinski copied = err; 19964f13a79eSSabrina Dubroca if (len <= copied || (copied && control != TLS_RECORD_TYPE_DATA) || rx_more) 1997bfc06e1aSJakub Kicinski goto end; 199846a16959SJakub Kicinski 199946a16959SJakub Kicinski target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 200046a16959SJakub Kicinski len = len - copied; 2001692d7b5dSVakul Garg 2002ba13609dSJakub Kicinski zc_capable = !bpf_strp_enabled && !is_kvec && !is_peek && 200388527790SJakub Kicinski ctx->zc_capable; 2004bfc06e1aSJakub Kicinski decrypted = 0; 2005b92a13d4SJakub Kicinski while (len && (decrypted + copied < target || tls_strp_msg_ready(ctx))) { 20066bd116c8SJakub Kicinski struct tls_decrypt_arg darg; 20079bdf75ccSJakub Kicinski int to_decrypt, chunk; 2008c46234ebSDave Watson 200970f03fc2SJakub Kicinski err = tls_rx_rec_wait(sk, psock, flags & MSG_DONTWAIT, 201070f03fc2SJakub Kicinski released); 201135560b7fSJakub Kicinski if (err <= 0) { 2012d3b18ad3SJohn Fastabend if (psock) { 20130775639cSJakub Kicinski chunk = sk_msg_recvmsg(sk, psock, msg, len, 20142bc793e3SCong Wang flags); 2015008141deSJakub Kicinski if (chunk > 0) { 2016008141deSJakub Kicinski decrypted += chunk; 2017008141deSJakub Kicinski len -= chunk; 2018008141deSJakub Kicinski continue; 2019008141deSJakub Kicinski } 2020d3b18ad3SJohn Fastabend } 2021c46234ebSDave Watson goto recv_end; 2022d3b18ad3SJohn Fastabend } 2023c46234ebSDave Watson 20246bd116c8SJakub Kicinski memset(&darg.inargs, 0, sizeof(darg.inargs)); 20256bd116c8SJakub Kicinski 202684c61fe1SJakub Kicinski rxm = strp_msg(tls_strp_msg(ctx)); 202784c61fe1SJakub Kicinski tlm = tls_msg(tls_strp_msg(ctx)); 202894524d8fSVakul Garg 20294509de14SVakul Garg to_decrypt = rxm->full_len - prot->overhead_size; 2030fedf201eSDave Watson 2031ba13609dSJakub Kicinski if (zc_capable && to_decrypt <= len && 2032ba13609dSJakub Kicinski tlm->control == TLS_RECORD_TYPE_DATA) 20334175eac3SJakub Kicinski darg.zc = true; 2034fedf201eSDave Watson 2035c0ab4732SVakul Garg /* Do not use async mode if record is non-data */ 2036c3f6bb74SJakub Kicinski if (tlm->control == TLS_RECORD_TYPE_DATA && !bpf_strp_enabled) 20374175eac3SJakub Kicinski darg.async = ctx->async_capable; 2038c0ab4732SVakul Garg else 20394175eac3SJakub Kicinski darg.async = false; 2040c0ab4732SVakul Garg 2041dd47ed36SJakub Kicinski err = tls_rx_one_record(sk, msg, &darg); 20423547a1f9SJakub Kicinski if (err < 0) { 2043da353facSDaniel Jordan tls_err_abort(sk, -EBADMSG); 2044fedf201eSDave Watson goto recv_end; 2045fedf201eSDave Watson } 2046fedf201eSDave Watson 20473547a1f9SJakub Kicinski async |= darg.async; 20482b794c40SVakul Garg 20492b794c40SVakul Garg /* If the type of records being processed is not known yet, 20502b794c40SVakul Garg * set it to record type just dequeued. If it is already known, 20512b794c40SVakul Garg * but does not match the record type just dequeued, go to end. 20522b794c40SVakul Garg * We always get record type here since for tls1.2, record type 20532b794c40SVakul Garg * is known just after record is dequeued from stream parser. 20542b794c40SVakul Garg * For tls1.3, we disable async. 20552b794c40SVakul Garg */ 2056b93f5700SJakub Kicinski err = tls_record_content_type(msg, tls_msg(darg.skb), &control); 2057abb47dc9SJakub Kicinski if (err <= 0) { 2058b93f5700SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(darg.zc); 2059abb47dc9SJakub Kicinski tls_rx_rec_done(ctx); 2060abb47dc9SJakub Kicinski put_on_rx_list_err: 2061b93f5700SJakub Kicinski __skb_queue_tail(&ctx->rx_list, darg.skb); 20622b794c40SVakul Garg goto recv_end; 2063abb47dc9SJakub Kicinski } 2064fedf201eSDave Watson 2065c46b0183SJakub Kicinski /* periodically flush backlog, and feed strparser */ 206684c61fe1SJakub Kicinski released = tls_read_flush_backlog(sk, prot, len, to_decrypt, 206784c61fe1SJakub Kicinski decrypted + copied, 206884c61fe1SJakub Kicinski &flushed_at); 2069c46b0183SJakub Kicinski 2070abb47dc9SJakub Kicinski /* TLS 1.3 may have updated the length by more than overhead */ 2071b93f5700SJakub Kicinski rxm = strp_msg(darg.skb); 2072abb47dc9SJakub Kicinski chunk = rxm->full_len; 2073abb47dc9SJakub Kicinski tls_rx_rec_done(ctx); 2074b1a2c178SJakub Kicinski 2075cbbdee99SJakub Kicinski if (!darg.zc) { 2076cbbdee99SJakub Kicinski bool partially_consumed = chunk > len; 2077b93f5700SJakub Kicinski struct sk_buff *skb = darg.skb; 2078b93f5700SJakub Kicinski 2079e20691faSJakub Kicinski DEBUG_NET_WARN_ON_ONCE(darg.skb == ctx->strp.anchor); 2080cbbdee99SJakub Kicinski 20819bdf75ccSJakub Kicinski if (async) { 2082cbbdee99SJakub Kicinski /* TLS 1.2-only, to_decrypt must be text len */ 20839bdf75ccSJakub Kicinski chunk = min_t(int, to_decrypt, len); 2084cbbdee99SJakub Kicinski async_copy_bytes += chunk; 2085008141deSJakub Kicinski put_on_rx_list: 2086f940b6efSJakub Kicinski decrypted += chunk; 2087f940b6efSJakub Kicinski len -= chunk; 2088008141deSJakub Kicinski __skb_queue_tail(&ctx->rx_list, skb); 208980b1d6a0SSabrina Dubroca if (unlikely(control != TLS_RECORD_TYPE_DATA)) 209080b1d6a0SSabrina Dubroca break; 2091f940b6efSJakub Kicinski continue; 20929bdf75ccSJakub Kicinski } 2093c0ab4732SVakul Garg 2094e91de6afSJohn Fastabend if (bpf_strp_enabled) { 209584c61fe1SJakub Kicinski released = true; 2096e91de6afSJohn Fastabend err = sk_psock_tls_strp_read(psock, skb); 2097e91de6afSJohn Fastabend if (err != __SK_PASS) { 2098e91de6afSJohn Fastabend rxm->offset = rxm->offset + rxm->full_len; 2099e91de6afSJohn Fastabend rxm->full_len = 0; 2100e91de6afSJohn Fastabend if (err == __SK_DROP) 2101e91de6afSJohn Fastabend consume_skb(skb); 2102e91de6afSJohn Fastabend continue; 2103e91de6afSJohn Fastabend } 2104e91de6afSJohn Fastabend } 2105e91de6afSJohn Fastabend 2106f940b6efSJakub Kicinski if (partially_consumed) 2107692d7b5dSVakul Garg chunk = len; 210894524d8fSVakul Garg 2109692d7b5dSVakul Garg err = skb_copy_datagram_msg(skb, rxm->offset, 2110692d7b5dSVakul Garg msg, chunk); 2111abb47dc9SJakub Kicinski if (err < 0) 2112abb47dc9SJakub Kicinski goto put_on_rx_list_err; 2113692d7b5dSVakul Garg 211411121c2cSSabrina Dubroca if (is_peek) { 211511121c2cSSabrina Dubroca peeked += chunk; 2116008141deSJakub Kicinski goto put_on_rx_list; 211711121c2cSSabrina Dubroca } 2118f940b6efSJakub Kicinski 2119f940b6efSJakub Kicinski if (partially_consumed) { 2120f940b6efSJakub Kicinski rxm->offset += chunk; 2121f940b6efSJakub Kicinski rxm->full_len -= chunk; 2122008141deSJakub Kicinski goto put_on_rx_list; 2123692d7b5dSVakul Garg } 2124b93f5700SJakub Kicinski 2125b93f5700SJakub Kicinski consume_skb(skb); 2126692d7b5dSVakul Garg } 2127c46234ebSDave Watson 2128692d7b5dSVakul Garg decrypted += chunk; 2129692d7b5dSVakul Garg len -= chunk; 2130692d7b5dSVakul Garg 2131f940b6efSJakub Kicinski /* Return full control message to userspace before trying 2132f940b6efSJakub Kicinski * to parse another message type 2133c46234ebSDave Watson */ 2134c46234ebSDave Watson msg->msg_flags |= MSG_EOR; 21353fe16edfSVadim Fedorenko if (control != TLS_RECORD_TYPE_DATA) 2136f940b6efSJakub Kicinski break; 213704b25a54SJakub Kicinski } 2138c46234ebSDave Watson 2139c46234ebSDave Watson recv_end: 21407da18bccSJakub Kicinski if (async) { 2141de31df6fSJakub Kicinski int ret; 21427da18bccSJakub Kicinski 214394524d8fSVakul Garg /* Wait for all previously submitted records to be decrypted */ 2144de31df6fSJakub Kicinski ret = tls_decrypt_async_wait(ctx); 2145c618db2aSJakub Kicinski __skb_queue_purge(&ctx->async_hold); 2146c618db2aSJakub Kicinski 2147f314bfeeSJakub Kicinski if (ret) { 2148f314bfeeSJakub Kicinski if (err >= 0 || err == -EINPROGRESS) 2149f314bfeeSJakub Kicinski err = ret; 2150692d7b5dSVakul Garg decrypted = 0; 2151692d7b5dSVakul Garg goto end; 215294524d8fSVakul Garg } 21530cada332SVinay Kumar Yadav 2154692d7b5dSVakul Garg /* Drain records from the rx_list & copy if required */ 2155*dc4bce20SSabrina Dubroca if (is_peek) 215611121c2cSSabrina Dubroca err = process_rx_list(ctx, msg, &control, copied + peeked, 215711121c2cSSabrina Dubroca decrypted - peeked, is_peek, NULL); 2158692d7b5dSVakul Garg else 215906554f4fSJakub Kicinski err = process_rx_list(ctx, msg, &control, 0, 21604f13a79eSSabrina Dubroca async_copy_bytes, is_peek, NULL); 2161692d7b5dSVakul Garg } 2162692d7b5dSVakul Garg 2163692d7b5dSVakul Garg copied += decrypted; 2164692d7b5dSVakul Garg 2165692d7b5dSVakul Garg end: 21664cbc325eSJakub Kicinski tls_rx_reader_unlock(sk, ctx); 2167d3b18ad3SJohn Fastabend if (psock) 2168d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 2169c46234ebSDave Watson return copied ? : err; 2170c46234ebSDave Watson } 2171c46234ebSDave Watson 2172c46234ebSDave Watson ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, 2173c46234ebSDave Watson struct pipe_inode_info *pipe, 2174c46234ebSDave Watson size_t len, unsigned int flags) 2175c46234ebSDave Watson { 2176c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sock->sk); 2177f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2178c46234ebSDave Watson struct strp_msg *rxm = NULL; 2179c46234ebSDave Watson struct sock *sk = sock->sk; 2180c3f6bb74SJakub Kicinski struct tls_msg *tlm; 2181c46234ebSDave Watson struct sk_buff *skb; 2182c46234ebSDave Watson ssize_t copied = 0; 2183c46234ebSDave Watson int chunk; 218470f03fc2SJakub Kicinski int err; 2185c46234ebSDave Watson 218670f03fc2SJakub Kicinski err = tls_rx_reader_lock(sk, ctx, flags & SPLICE_F_NONBLOCK); 218770f03fc2SJakub Kicinski if (err < 0) 218870f03fc2SJakub Kicinski return err; 2189c46234ebSDave Watson 2190abb47dc9SJakub Kicinski if (!skb_queue_empty(&ctx->rx_list)) { 2191e062fe99SJakub Kicinski skb = __skb_dequeue(&ctx->rx_list); 2192e062fe99SJakub Kicinski } else { 21936bd116c8SJakub Kicinski struct tls_decrypt_arg darg; 21944175eac3SJakub Kicinski 219535560b7fSJakub Kicinski err = tls_rx_rec_wait(sk, NULL, flags & SPLICE_F_NONBLOCK, 219670f03fc2SJakub Kicinski true); 219735560b7fSJakub Kicinski if (err <= 0) 2198c46234ebSDave Watson goto splice_read_end; 2199c46234ebSDave Watson 22006bd116c8SJakub Kicinski memset(&darg.inargs, 0, sizeof(darg.inargs)); 22016bd116c8SJakub Kicinski 2202541cc48bSJakub Kicinski err = tls_rx_one_record(sk, NULL, &darg); 2203520493f6SJakub Kicinski if (err < 0) { 2204520493f6SJakub Kicinski tls_err_abort(sk, -EBADMSG); 2205520493f6SJakub Kicinski goto splice_read_end; 2206520493f6SJakub Kicinski } 2207abb47dc9SJakub Kicinski 2208abb47dc9SJakub Kicinski tls_rx_rec_done(ctx); 22096bd116c8SJakub Kicinski skb = darg.skb; 2210e062fe99SJakub Kicinski } 2211fedf201eSDave Watson 2212c3f6bb74SJakub Kicinski rxm = strp_msg(skb); 2213c3f6bb74SJakub Kicinski tlm = tls_msg(skb); 2214c3f6bb74SJakub Kicinski 2215c46234ebSDave Watson /* splice does not support reading control messages */ 2216c3f6bb74SJakub Kicinski if (tlm->control != TLS_RECORD_TYPE_DATA) { 22174a5cdc60SValentin Vidic err = -EINVAL; 2218abb47dc9SJakub Kicinski goto splice_requeue; 2219c46234ebSDave Watson } 2220c46234ebSDave Watson 2221c46234ebSDave Watson chunk = min_t(unsigned int, rxm->full_len, len); 2222c46234ebSDave Watson copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags); 2223c46234ebSDave Watson if (copied < 0) 2224abb47dc9SJakub Kicinski goto splice_requeue; 2225c46234ebSDave Watson 2226e062fe99SJakub Kicinski if (chunk < rxm->full_len) { 2227e062fe99SJakub Kicinski rxm->offset += len; 2228e062fe99SJakub Kicinski rxm->full_len -= len; 2229abb47dc9SJakub Kicinski goto splice_requeue; 2230e062fe99SJakub Kicinski } 2231c46234ebSDave Watson 2232abb47dc9SJakub Kicinski consume_skb(skb); 2233abb47dc9SJakub Kicinski 2234c46234ebSDave Watson splice_read_end: 22354cbc325eSJakub Kicinski tls_rx_reader_unlock(sk, ctx); 2236c46234ebSDave Watson return copied ? : err; 2237abb47dc9SJakub Kicinski 2238abb47dc9SJakub Kicinski splice_requeue: 2239abb47dc9SJakub Kicinski __skb_queue_head(&ctx->rx_list, skb); 2240abb47dc9SJakub Kicinski goto splice_read_end; 2241c46234ebSDave Watson } 2242c46234ebSDave Watson 2243662fbcecSHannes Reinecke int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, 2244662fbcecSHannes Reinecke sk_read_actor_t read_actor) 2245662fbcecSHannes Reinecke { 2246662fbcecSHannes Reinecke struct tls_context *tls_ctx = tls_get_ctx(sk); 2247662fbcecSHannes Reinecke struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2248662fbcecSHannes Reinecke struct tls_prot_info *prot = &tls_ctx->prot_info; 2249662fbcecSHannes Reinecke struct strp_msg *rxm = NULL; 2250662fbcecSHannes Reinecke struct sk_buff *skb = NULL; 2251662fbcecSHannes Reinecke struct sk_psock *psock; 2252662fbcecSHannes Reinecke size_t flushed_at = 0; 2253662fbcecSHannes Reinecke bool released = true; 2254662fbcecSHannes Reinecke struct tls_msg *tlm; 2255662fbcecSHannes Reinecke ssize_t copied = 0; 2256662fbcecSHannes Reinecke ssize_t decrypted; 2257662fbcecSHannes Reinecke int err, used; 2258662fbcecSHannes Reinecke 2259662fbcecSHannes Reinecke psock = sk_psock_get(sk); 2260662fbcecSHannes Reinecke if (psock) { 2261662fbcecSHannes Reinecke sk_psock_put(sk, psock); 2262662fbcecSHannes Reinecke return -EINVAL; 2263662fbcecSHannes Reinecke } 2264662fbcecSHannes Reinecke err = tls_rx_reader_acquire(sk, ctx, true); 2265662fbcecSHannes Reinecke if (err < 0) 2266662fbcecSHannes Reinecke return err; 2267662fbcecSHannes Reinecke 2268662fbcecSHannes Reinecke /* If crypto failed the connection is broken */ 2269662fbcecSHannes Reinecke err = ctx->async_wait.err; 2270662fbcecSHannes Reinecke if (err) 2271662fbcecSHannes Reinecke goto read_sock_end; 2272662fbcecSHannes Reinecke 2273662fbcecSHannes Reinecke decrypted = 0; 2274662fbcecSHannes Reinecke do { 2275662fbcecSHannes Reinecke if (!skb_queue_empty(&ctx->rx_list)) { 2276662fbcecSHannes Reinecke skb = __skb_dequeue(&ctx->rx_list); 2277662fbcecSHannes Reinecke rxm = strp_msg(skb); 2278662fbcecSHannes Reinecke tlm = tls_msg(skb); 2279662fbcecSHannes Reinecke } else { 2280662fbcecSHannes Reinecke struct tls_decrypt_arg darg; 2281662fbcecSHannes Reinecke 2282662fbcecSHannes Reinecke err = tls_rx_rec_wait(sk, NULL, true, released); 2283662fbcecSHannes Reinecke if (err <= 0) 2284662fbcecSHannes Reinecke goto read_sock_end; 2285662fbcecSHannes Reinecke 2286662fbcecSHannes Reinecke memset(&darg.inargs, 0, sizeof(darg.inargs)); 2287662fbcecSHannes Reinecke 2288662fbcecSHannes Reinecke err = tls_rx_one_record(sk, NULL, &darg); 2289662fbcecSHannes Reinecke if (err < 0) { 2290662fbcecSHannes Reinecke tls_err_abort(sk, -EBADMSG); 2291662fbcecSHannes Reinecke goto read_sock_end; 2292662fbcecSHannes Reinecke } 2293662fbcecSHannes Reinecke 2294ba4a734eSHannes Reinecke released = tls_read_flush_backlog(sk, prot, INT_MAX, 2295ba4a734eSHannes Reinecke 0, decrypted, 2296ba4a734eSHannes Reinecke &flushed_at); 2297662fbcecSHannes Reinecke skb = darg.skb; 2298ba4a734eSHannes Reinecke rxm = strp_msg(skb); 2299ba4a734eSHannes Reinecke tlm = tls_msg(skb); 2300662fbcecSHannes Reinecke decrypted += rxm->full_len; 2301662fbcecSHannes Reinecke 2302662fbcecSHannes Reinecke tls_rx_rec_done(ctx); 2303662fbcecSHannes Reinecke } 2304662fbcecSHannes Reinecke 2305662fbcecSHannes Reinecke /* read_sock does not support reading control messages */ 2306662fbcecSHannes Reinecke if (tlm->control != TLS_RECORD_TYPE_DATA) { 2307662fbcecSHannes Reinecke err = -EINVAL; 2308662fbcecSHannes Reinecke goto read_sock_requeue; 2309662fbcecSHannes Reinecke } 2310662fbcecSHannes Reinecke 2311662fbcecSHannes Reinecke used = read_actor(desc, skb, rxm->offset, rxm->full_len); 2312662fbcecSHannes Reinecke if (used <= 0) { 2313662fbcecSHannes Reinecke if (!copied) 2314662fbcecSHannes Reinecke err = used; 2315662fbcecSHannes Reinecke goto read_sock_requeue; 2316662fbcecSHannes Reinecke } 2317662fbcecSHannes Reinecke copied += used; 2318662fbcecSHannes Reinecke if (used < rxm->full_len) { 2319662fbcecSHannes Reinecke rxm->offset += used; 2320662fbcecSHannes Reinecke rxm->full_len -= used; 2321662fbcecSHannes Reinecke if (!desc->count) 2322662fbcecSHannes Reinecke goto read_sock_requeue; 2323662fbcecSHannes Reinecke } else { 2324662fbcecSHannes Reinecke consume_skb(skb); 2325662fbcecSHannes Reinecke if (!desc->count) 2326662fbcecSHannes Reinecke skb = NULL; 2327662fbcecSHannes Reinecke } 2328662fbcecSHannes Reinecke } while (skb); 2329662fbcecSHannes Reinecke 2330662fbcecSHannes Reinecke read_sock_end: 2331662fbcecSHannes Reinecke tls_rx_reader_release(sk, ctx); 2332662fbcecSHannes Reinecke return copied ? : err; 2333662fbcecSHannes Reinecke 2334662fbcecSHannes Reinecke read_sock_requeue: 2335662fbcecSHannes Reinecke __skb_queue_head(&ctx->rx_list, skb); 2336662fbcecSHannes Reinecke goto read_sock_end; 2337662fbcecSHannes Reinecke } 2338662fbcecSHannes Reinecke 23397b50ecfcSCong Wang bool tls_sw_sock_is_readable(struct sock *sk) 2340c46234ebSDave Watson { 2341c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2342f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2343d3b18ad3SJohn Fastabend bool ingress_empty = true; 2344d3b18ad3SJohn Fastabend struct sk_psock *psock; 2345c46234ebSDave Watson 2346d3b18ad3SJohn Fastabend rcu_read_lock(); 2347d3b18ad3SJohn Fastabend psock = sk_psock(sk); 2348d3b18ad3SJohn Fastabend if (psock) 2349d3b18ad3SJohn Fastabend ingress_empty = list_empty(&psock->ingress_msg); 2350d3b18ad3SJohn Fastabend rcu_read_unlock(); 2351c46234ebSDave Watson 2352b92a13d4SJakub Kicinski return !ingress_empty || tls_strp_msg_ready(ctx) || 235313aecb17SJakub Kicinski !skb_queue_empty(&ctx->rx_list); 2354c46234ebSDave Watson } 2355c46234ebSDave Watson 235684c61fe1SJakub Kicinski int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb) 2357c46234ebSDave Watson { 2358c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 23594509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 23603463e51dSKees Cook char header[TLS_HEADER_SIZE + MAX_IV_SIZE]; 2361c46234ebSDave Watson size_t cipher_overhead; 2362c46234ebSDave Watson size_t data_len = 0; 2363c46234ebSDave Watson int ret; 2364c46234ebSDave Watson 2365c46234ebSDave Watson /* Verify that we have a full TLS header, or wait for more data */ 236684c61fe1SJakub Kicinski if (strp->stm.offset + prot->prepend_size > skb->len) 2367c46234ebSDave Watson return 0; 2368c46234ebSDave Watson 23693463e51dSKees Cook /* Sanity-check size of on-stack buffer. */ 23704509de14SVakul Garg if (WARN_ON(prot->prepend_size > sizeof(header))) { 23713463e51dSKees Cook ret = -EINVAL; 23723463e51dSKees Cook goto read_failure; 23733463e51dSKees Cook } 23743463e51dSKees Cook 2375c46234ebSDave Watson /* Linearize header to local buffer */ 237684c61fe1SJakub Kicinski ret = skb_copy_bits(skb, strp->stm.offset, header, prot->prepend_size); 2377c46234ebSDave Watson if (ret < 0) 2378c46234ebSDave Watson goto read_failure; 2379c46234ebSDave Watson 238084c61fe1SJakub Kicinski strp->mark = header[0]; 2381c46234ebSDave Watson 2382c46234ebSDave Watson data_len = ((header[4] & 0xFF) | (header[3] << 8)); 2383c46234ebSDave Watson 23844509de14SVakul Garg cipher_overhead = prot->tag_size; 2385a6acbe62SVadim Fedorenko if (prot->version != TLS_1_3_VERSION && 2386a6acbe62SVadim Fedorenko prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305) 23874509de14SVakul Garg cipher_overhead += prot->iv_size; 2388c46234ebSDave Watson 2389130b392cSDave Watson if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead + 23904509de14SVakul Garg prot->tail_size) { 2391c46234ebSDave Watson ret = -EMSGSIZE; 2392c46234ebSDave Watson goto read_failure; 2393c46234ebSDave Watson } 2394c46234ebSDave Watson if (data_len < cipher_overhead) { 2395c46234ebSDave Watson ret = -EBADMSG; 2396c46234ebSDave Watson goto read_failure; 2397c46234ebSDave Watson } 2398c46234ebSDave Watson 2399130b392cSDave Watson /* Note that both TLS1.3 and TLS1.2 use TLS_1_2 version here */ 2400130b392cSDave Watson if (header[1] != TLS_1_2_VERSION_MINOR || 2401130b392cSDave Watson header[2] != TLS_1_2_VERSION_MAJOR) { 2402c46234ebSDave Watson ret = -EINVAL; 2403c46234ebSDave Watson goto read_failure; 2404c46234ebSDave Watson } 2405be2fbc15SJakub Kicinski 2406f953d33bSJakub Kicinski tls_device_rx_resync_new_rec(strp->sk, data_len + TLS_HEADER_SIZE, 240784c61fe1SJakub Kicinski TCP_SKB_CB(skb)->seq + strp->stm.offset); 2408c46234ebSDave Watson return data_len + TLS_HEADER_SIZE; 2409c46234ebSDave Watson 2410c46234ebSDave Watson read_failure: 2411c46234ebSDave Watson tls_err_abort(strp->sk, ret); 2412c46234ebSDave Watson 2413c46234ebSDave Watson return ret; 2414c46234ebSDave Watson } 2415c46234ebSDave Watson 241684c61fe1SJakub Kicinski void tls_rx_msg_ready(struct tls_strparser *strp) 2417c46234ebSDave Watson { 241884c61fe1SJakub Kicinski struct tls_sw_context_rx *ctx; 2419c46234ebSDave Watson 242084c61fe1SJakub Kicinski ctx = container_of(strp, struct tls_sw_context_rx, strp); 2421ad13acceSVakul Garg ctx->saved_data_ready(strp->sk); 2422c46234ebSDave Watson } 2423c46234ebSDave Watson 2424c46234ebSDave Watson static void tls_data_ready(struct sock *sk) 2425c46234ebSDave Watson { 2426c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2427f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2428d3b18ad3SJohn Fastabend struct sk_psock *psock; 242974836ec8SJakub Kicinski gfp_t alloc_save; 2430c46234ebSDave Watson 243140e0b090SPeilin Ye trace_sk_data_ready(sk); 243240e0b090SPeilin Ye 243374836ec8SJakub Kicinski alloc_save = sk->sk_allocation; 243474836ec8SJakub Kicinski sk->sk_allocation = GFP_ATOMIC; 243584c61fe1SJakub Kicinski tls_strp_data_ready(&ctx->strp); 243674836ec8SJakub Kicinski sk->sk_allocation = alloc_save; 2437d3b18ad3SJohn Fastabend 2438d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 243962b4011fSXiyu Yang if (psock) { 244062b4011fSXiyu Yang if (!list_empty(&psock->ingress_msg)) 2441d3b18ad3SJohn Fastabend ctx->saved_data_ready(sk); 2442d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 2443d3b18ad3SJohn Fastabend } 2444c46234ebSDave Watson } 2445c46234ebSDave Watson 2446f87e62d4SJohn Fastabend void tls_sw_cancel_work_tx(struct tls_context *tls_ctx) 2447f87e62d4SJohn Fastabend { 2448f87e62d4SJohn Fastabend struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 2449f87e62d4SJohn Fastabend 2450f87e62d4SJohn Fastabend set_bit(BIT_TX_CLOSING, &ctx->tx_bitmask); 2451f87e62d4SJohn Fastabend set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask); 2452f87e62d4SJohn Fastabend cancel_delayed_work_sync(&ctx->tx_work.work); 2453f87e62d4SJohn Fastabend } 2454f87e62d4SJohn Fastabend 2455313ab004SJohn Fastabend void tls_sw_release_resources_tx(struct sock *sk) 24563c4d7559SDave Watson { 24573c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 2458f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 2459a42055e8SVakul Garg struct tls_rec *rec, *tmp; 2460a42055e8SVakul Garg 2461a42055e8SVakul Garg /* Wait for any pending async encryptions to complete */ 2462de31df6fSJakub Kicinski tls_encrypt_async_wait(ctx); 2463a42055e8SVakul Garg 2464a42055e8SVakul Garg tls_tx_records(sk, -1); 2465a42055e8SVakul Garg 24669932a29aSVakul Garg /* Free up un-sent records in tx_list. First, free 2467a42055e8SVakul Garg * the partially sent record if any at head of tx_list. 2468a42055e8SVakul Garg */ 2469c5daa6ccSJakub Kicinski if (tls_ctx->partially_sent_record) { 2470c5daa6ccSJakub Kicinski tls_free_partial_record(sk, tls_ctx); 24719932a29aSVakul Garg rec = list_first_entry(&ctx->tx_list, 2472a42055e8SVakul Garg struct tls_rec, list); 2473a42055e8SVakul Garg list_del(&rec->list); 2474d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 2475a42055e8SVakul Garg kfree(rec); 2476a42055e8SVakul Garg } 2477a42055e8SVakul Garg 24789932a29aSVakul Garg list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { 2479a42055e8SVakul Garg list_del(&rec->list); 2480d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_encrypted); 2481d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 2482a42055e8SVakul Garg kfree(rec); 2483a42055e8SVakul Garg } 24843c4d7559SDave Watson 24853c4d7559SDave Watson crypto_free_aead(ctx->aead_send); 2486c774973eSVakul Garg tls_free_open_rec(sk); 2487313ab004SJohn Fastabend } 2488313ab004SJohn Fastabend 2489313ab004SJohn Fastabend void tls_sw_free_ctx_tx(struct tls_context *tls_ctx) 2490313ab004SJohn Fastabend { 2491313ab004SJohn Fastabend struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 2492f66de3eeSBoris Pismenny 2493f66de3eeSBoris Pismenny kfree(ctx); 2494f66de3eeSBoris Pismenny } 2495f66de3eeSBoris Pismenny 249639f56e1aSBoris Pismenny void tls_sw_release_resources_rx(struct sock *sk) 2497f66de3eeSBoris Pismenny { 2498f66de3eeSBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 2499f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2500f66de3eeSBoris Pismenny 250112c76861SJakub Kicinski kfree(tls_ctx->rx.rec_seq); 250212c76861SJakub Kicinski kfree(tls_ctx->rx.iv); 250312c76861SJakub Kicinski 2504c46234ebSDave Watson if (ctx->aead_recv) { 2505a30295c4SJakub Kicinski __skb_queue_purge(&ctx->rx_list); 2506c46234ebSDave Watson crypto_free_aead(ctx->aead_recv); 250784c61fe1SJakub Kicinski tls_strp_stop(&ctx->strp); 2508313ab004SJohn Fastabend /* If tls_sw_strparser_arm() was not called (cleanup paths) 250984c61fe1SJakub Kicinski * we still want to tls_strp_stop(), but sk->sk_data_ready was 2510313ab004SJohn Fastabend * never swapped. 2511313ab004SJohn Fastabend */ 2512313ab004SJohn Fastabend if (ctx->saved_data_ready) { 2513c46234ebSDave Watson write_lock_bh(&sk->sk_callback_lock); 2514c46234ebSDave Watson sk->sk_data_ready = ctx->saved_data_ready; 2515c46234ebSDave Watson write_unlock_bh(&sk->sk_callback_lock); 2516c46234ebSDave Watson } 251739f56e1aSBoris Pismenny } 2518313ab004SJohn Fastabend } 2519313ab004SJohn Fastabend 2520313ab004SJohn Fastabend void tls_sw_strparser_done(struct tls_context *tls_ctx) 2521313ab004SJohn Fastabend { 2522313ab004SJohn Fastabend struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2523313ab004SJohn Fastabend 252484c61fe1SJakub Kicinski tls_strp_done(&ctx->strp); 2525313ab004SJohn Fastabend } 2526313ab004SJohn Fastabend 2527313ab004SJohn Fastabend void tls_sw_free_ctx_rx(struct tls_context *tls_ctx) 2528313ab004SJohn Fastabend { 2529313ab004SJohn Fastabend struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 2530313ab004SJohn Fastabend 2531313ab004SJohn Fastabend kfree(ctx); 2532313ab004SJohn Fastabend } 253339f56e1aSBoris Pismenny 253439f56e1aSBoris Pismenny void tls_sw_free_resources_rx(struct sock *sk) 253539f56e1aSBoris Pismenny { 253639f56e1aSBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 253739f56e1aSBoris Pismenny 253839f56e1aSBoris Pismenny tls_sw_release_resources_rx(sk); 2539313ab004SJohn Fastabend tls_sw_free_ctx_rx(tls_ctx); 25403c4d7559SDave Watson } 25413c4d7559SDave Watson 25429932a29aSVakul Garg /* The work handler to transmitt the encrypted records in tx_list */ 2543a42055e8SVakul Garg static void tx_work_handler(struct work_struct *work) 2544a42055e8SVakul Garg { 2545a42055e8SVakul Garg struct delayed_work *delayed_work = to_delayed_work(work); 2546a42055e8SVakul Garg struct tx_work *tx_work = container_of(delayed_work, 2547a42055e8SVakul Garg struct tx_work, work); 2548a42055e8SVakul Garg struct sock *sk = tx_work->sk; 2549a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 2550f87e62d4SJohn Fastabend struct tls_sw_context_tx *ctx; 2551f87e62d4SJohn Fastabend 2552f87e62d4SJohn Fastabend if (unlikely(!tls_ctx)) 2553f87e62d4SJohn Fastabend return; 2554f87e62d4SJohn Fastabend 2555f87e62d4SJohn Fastabend ctx = tls_sw_ctx_tx(tls_ctx); 2556f87e62d4SJohn Fastabend if (test_bit(BIT_TX_CLOSING, &ctx->tx_bitmask)) 2557f87e62d4SJohn Fastabend return; 2558a42055e8SVakul Garg 2559a42055e8SVakul Garg if (!test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) 2560a42055e8SVakul Garg return; 2561f3221361SJakub Kicinski 2562f3221361SJakub Kicinski if (mutex_trylock(&tls_ctx->tx_lock)) { 2563a42055e8SVakul Garg lock_sock(sk); 2564a42055e8SVakul Garg tls_tx_records(sk, -1); 2565a42055e8SVakul Garg release_sock(sk); 256679ffe608SJakub Kicinski mutex_unlock(&tls_ctx->tx_lock); 2567f3221361SJakub Kicinski } else if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 2568f3221361SJakub Kicinski /* Someone is holding the tx_lock, they will likely run Tx 2569f3221361SJakub Kicinski * and cancel the work on their way out of the lock section. 2570f3221361SJakub Kicinski * Schedule a long delay just in case. 2571f3221361SJakub Kicinski */ 2572f3221361SJakub Kicinski schedule_delayed_work(&ctx->tx_work.work, msecs_to_jiffies(10)); 2573f3221361SJakub Kicinski } 2574a42055e8SVakul Garg } 2575a42055e8SVakul Garg 257658790314SJakub Kicinski static bool tls_is_tx_ready(struct tls_sw_context_tx *ctx) 257758790314SJakub Kicinski { 257858790314SJakub Kicinski struct tls_rec *rec; 257958790314SJakub Kicinski 2580ffe2a225SPietro Borrello rec = list_first_entry_or_null(&ctx->tx_list, struct tls_rec, list); 258158790314SJakub Kicinski if (!rec) 258258790314SJakub Kicinski return false; 258358790314SJakub Kicinski 258458790314SJakub Kicinski return READ_ONCE(rec->tx_ready); 258558790314SJakub Kicinski } 258658790314SJakub Kicinski 25877463d3a2SBoris Pismenny void tls_sw_write_space(struct sock *sk, struct tls_context *ctx) 25887463d3a2SBoris Pismenny { 25897463d3a2SBoris Pismenny struct tls_sw_context_tx *tx_ctx = tls_sw_ctx_tx(ctx); 25907463d3a2SBoris Pismenny 25917463d3a2SBoris Pismenny /* Schedule the transmission if tx list is ready */ 259258790314SJakub Kicinski if (tls_is_tx_ready(tx_ctx) && 259302b1fa07SJakub Kicinski !test_and_set_bit(BIT_TX_SCHEDULED, &tx_ctx->tx_bitmask)) 25947463d3a2SBoris Pismenny schedule_delayed_work(&tx_ctx->tx_work.work, 0); 25957463d3a2SBoris Pismenny } 25967463d3a2SBoris Pismenny 2597318892acSJakub Kicinski void tls_sw_strparser_arm(struct sock *sk, struct tls_context *tls_ctx) 2598318892acSJakub Kicinski { 2599318892acSJakub Kicinski struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx); 2600318892acSJakub Kicinski 2601318892acSJakub Kicinski write_lock_bh(&sk->sk_callback_lock); 2602318892acSJakub Kicinski rx_ctx->saved_data_ready = sk->sk_data_ready; 2603318892acSJakub Kicinski sk->sk_data_ready = tls_data_ready; 2604318892acSJakub Kicinski write_unlock_bh(&sk->sk_callback_lock); 2605318892acSJakub Kicinski } 2606318892acSJakub Kicinski 260788527790SJakub Kicinski void tls_update_rx_zc_capable(struct tls_context *tls_ctx) 260888527790SJakub Kicinski { 260988527790SJakub Kicinski struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx); 261088527790SJakub Kicinski 261188527790SJakub Kicinski rx_ctx->zc_capable = tls_ctx->rx_no_pad || 261288527790SJakub Kicinski tls_ctx->prot_info.version != TLS_1_3_VERSION; 261388527790SJakub Kicinski } 261488527790SJakub Kicinski 26155e01c54eSSabrina Dubroca static struct tls_sw_context_tx *init_ctx_tx(struct tls_context *ctx, struct sock *sk) 26165e01c54eSSabrina Dubroca { 26175e01c54eSSabrina Dubroca struct tls_sw_context_tx *sw_ctx_tx; 26185e01c54eSSabrina Dubroca 26195e01c54eSSabrina Dubroca if (!ctx->priv_ctx_tx) { 26205e01c54eSSabrina Dubroca sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL); 26215e01c54eSSabrina Dubroca if (!sw_ctx_tx) 26225e01c54eSSabrina Dubroca return NULL; 26235e01c54eSSabrina Dubroca } else { 26245e01c54eSSabrina Dubroca sw_ctx_tx = ctx->priv_ctx_tx; 26255e01c54eSSabrina Dubroca } 26265e01c54eSSabrina Dubroca 26275e01c54eSSabrina Dubroca crypto_init_wait(&sw_ctx_tx->async_wait); 262886dc27eeSJakub Kicinski atomic_set(&sw_ctx_tx->encrypt_pending, 1); 26295e01c54eSSabrina Dubroca INIT_LIST_HEAD(&sw_ctx_tx->tx_list); 26305e01c54eSSabrina Dubroca INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler); 26315e01c54eSSabrina Dubroca sw_ctx_tx->tx_work.sk = sk; 26325e01c54eSSabrina Dubroca 26335e01c54eSSabrina Dubroca return sw_ctx_tx; 26345e01c54eSSabrina Dubroca } 26355e01c54eSSabrina Dubroca 26365e01c54eSSabrina Dubroca static struct tls_sw_context_rx *init_ctx_rx(struct tls_context *ctx) 26375e01c54eSSabrina Dubroca { 26385e01c54eSSabrina Dubroca struct tls_sw_context_rx *sw_ctx_rx; 26395e01c54eSSabrina Dubroca 26405e01c54eSSabrina Dubroca if (!ctx->priv_ctx_rx) { 26415e01c54eSSabrina Dubroca sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL); 26425e01c54eSSabrina Dubroca if (!sw_ctx_rx) 26435e01c54eSSabrina Dubroca return NULL; 26445e01c54eSSabrina Dubroca } else { 26455e01c54eSSabrina Dubroca sw_ctx_rx = ctx->priv_ctx_rx; 26465e01c54eSSabrina Dubroca } 26475e01c54eSSabrina Dubroca 26485e01c54eSSabrina Dubroca crypto_init_wait(&sw_ctx_rx->async_wait); 264986dc27eeSJakub Kicinski atomic_set(&sw_ctx_rx->decrypt_pending, 1); 26505e01c54eSSabrina Dubroca init_waitqueue_head(&sw_ctx_rx->wq); 26515e01c54eSSabrina Dubroca skb_queue_head_init(&sw_ctx_rx->rx_list); 26525e01c54eSSabrina Dubroca skb_queue_head_init(&sw_ctx_rx->async_hold); 26535e01c54eSSabrina Dubroca 26545e01c54eSSabrina Dubroca return sw_ctx_rx; 26555e01c54eSSabrina Dubroca } 26565e01c54eSSabrina Dubroca 2657c46234ebSDave Watson int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) 26583c4d7559SDave Watson { 26594509de14SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 26604509de14SVakul Garg struct tls_prot_info *prot = &tls_ctx->prot_info; 26613c4d7559SDave Watson struct tls_crypto_info *crypto_info; 2662f66de3eeSBoris Pismenny struct tls_sw_context_tx *sw_ctx_tx = NULL; 2663f66de3eeSBoris Pismenny struct tls_sw_context_rx *sw_ctx_rx = NULL; 2664c46234ebSDave Watson struct cipher_context *cctx; 2665c46234ebSDave Watson struct crypto_aead **aead; 2666692d7b5dSVakul Garg struct crypto_tfm *tfm; 2667f3e444e3SSabrina Dubroca char *iv, *rec_seq, *key, *salt; 2668d9a6ca1aSSabrina Dubroca const struct tls_cipher_desc *cipher_desc; 2669d9a6ca1aSSabrina Dubroca u16 nonce_size; 26703c4d7559SDave Watson int rc = 0; 26713c4d7559SDave Watson 26723c4d7559SDave Watson if (!ctx) { 26733c4d7559SDave Watson rc = -EINVAL; 26743c4d7559SDave Watson goto out; 26753c4d7559SDave Watson } 26763c4d7559SDave Watson 2677f66de3eeSBoris Pismenny if (tx) { 26785e01c54eSSabrina Dubroca ctx->priv_ctx_tx = init_ctx_tx(ctx, sk); 26795e01c54eSSabrina Dubroca if (!ctx->priv_ctx_tx) 26805e01c54eSSabrina Dubroca return -ENOMEM; 26813c4d7559SDave Watson 26825e01c54eSSabrina Dubroca sw_ctx_tx = ctx->priv_ctx_tx; 268386029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 2684c46234ebSDave Watson cctx = &ctx->tx; 2685f66de3eeSBoris Pismenny aead = &sw_ctx_tx->aead_send; 2686c46234ebSDave Watson } else { 26875e01c54eSSabrina Dubroca ctx->priv_ctx_rx = init_ctx_rx(ctx); 26885e01c54eSSabrina Dubroca if (!ctx->priv_ctx_rx) 26895e01c54eSSabrina Dubroca return -ENOMEM; 26905e01c54eSSabrina Dubroca 26915e01c54eSSabrina Dubroca sw_ctx_rx = ctx->priv_ctx_rx; 269286029d10SSabrina Dubroca crypto_info = &ctx->crypto_recv.info; 2693c46234ebSDave Watson cctx = &ctx->rx; 2694f66de3eeSBoris Pismenny aead = &sw_ctx_rx->aead_recv; 2695c46234ebSDave Watson } 2696c46234ebSDave Watson 2697f3e444e3SSabrina Dubroca cipher_desc = get_cipher_desc(crypto_info->cipher_type); 2698f3e444e3SSabrina Dubroca if (!cipher_desc) { 26993c4d7559SDave Watson rc = -EINVAL; 2700cf6d43efSSabrina Dubroca goto free_priv; 27013c4d7559SDave Watson } 27023c4d7559SDave Watson 2703d9a6ca1aSSabrina Dubroca nonce_size = cipher_desc->nonce; 2704d9a6ca1aSSabrina Dubroca 270548dfad27SSabrina Dubroca iv = crypto_info_iv(crypto_info, cipher_desc); 270648dfad27SSabrina Dubroca key = crypto_info_key(crypto_info, cipher_desc); 270748dfad27SSabrina Dubroca salt = crypto_info_salt(crypto_info, cipher_desc); 270848dfad27SSabrina Dubroca rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc); 270948dfad27SSabrina Dubroca 2710130b392cSDave Watson if (crypto_info->version == TLS_1_3_VERSION) { 2711130b392cSDave Watson nonce_size = 0; 27124509de14SVakul Garg prot->aad_size = TLS_HEADER_SIZE; 27134509de14SVakul Garg prot->tail_size = 1; 2714130b392cSDave Watson } else { 27154509de14SVakul Garg prot->aad_size = TLS_AAD_SPACE_SIZE; 27164509de14SVakul Garg prot->tail_size = 0; 2717130b392cSDave Watson } 2718130b392cSDave Watson 271950a07aa5SJakub Kicinski /* Sanity-check the sizes for stack allocations. */ 2720d9a6ca1aSSabrina Dubroca if (nonce_size > MAX_IV_SIZE || prot->aad_size > TLS_MAX_AAD_SIZE) { 272150a07aa5SJakub Kicinski rc = -EINVAL; 272250a07aa5SJakub Kicinski goto free_priv; 272350a07aa5SJakub Kicinski } 272450a07aa5SJakub Kicinski 27254509de14SVakul Garg prot->version = crypto_info->version; 27264509de14SVakul Garg prot->cipher_type = crypto_info->cipher_type; 27274509de14SVakul Garg prot->prepend_size = TLS_HEADER_SIZE + nonce_size; 2728d9a6ca1aSSabrina Dubroca prot->tag_size = cipher_desc->tag; 27294509de14SVakul Garg prot->overhead_size = prot->prepend_size + 27304509de14SVakul Garg prot->tag_size + prot->tail_size; 2731d9a6ca1aSSabrina Dubroca prot->iv_size = cipher_desc->iv; 2732d9a6ca1aSSabrina Dubroca prot->salt_size = cipher_desc->salt; 2733d9a6ca1aSSabrina Dubroca cctx->iv = kmalloc(cipher_desc->iv + cipher_desc->salt, GFP_KERNEL); 2734c46234ebSDave Watson if (!cctx->iv) { 27353c4d7559SDave Watson rc = -ENOMEM; 2736cf6d43efSSabrina Dubroca goto free_priv; 27373c4d7559SDave Watson } 2738fb99bce7SDave Watson /* Note: 128 & 256 bit salt are the same size */ 2739d9a6ca1aSSabrina Dubroca prot->rec_seq_size = cipher_desc->rec_seq; 2740d9a6ca1aSSabrina Dubroca memcpy(cctx->iv, salt, cipher_desc->salt); 2741d9a6ca1aSSabrina Dubroca memcpy(cctx->iv + cipher_desc->salt, iv, cipher_desc->iv); 2742d9a6ca1aSSabrina Dubroca 2743d9a6ca1aSSabrina Dubroca cctx->rec_seq = kmemdup(rec_seq, cipher_desc->rec_seq, GFP_KERNEL); 2744c46234ebSDave Watson if (!cctx->rec_seq) { 27453c4d7559SDave Watson rc = -ENOMEM; 27463c4d7559SDave Watson goto free_iv; 27473c4d7559SDave Watson } 27483c4d7559SDave Watson 2749c46234ebSDave Watson if (!*aead) { 2750f3e444e3SSabrina Dubroca *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); 2751c46234ebSDave Watson if (IS_ERR(*aead)) { 2752c46234ebSDave Watson rc = PTR_ERR(*aead); 2753c46234ebSDave Watson *aead = NULL; 27543c4d7559SDave Watson goto free_rec_seq; 27553c4d7559SDave Watson } 27563c4d7559SDave Watson } 27573c4d7559SDave Watson 27583c4d7559SDave Watson ctx->push_pending_record = tls_sw_push_pending_record; 27593c4d7559SDave Watson 2760d9a6ca1aSSabrina Dubroca rc = crypto_aead_setkey(*aead, key, cipher_desc->key); 27613c4d7559SDave Watson if (rc) 27623c4d7559SDave Watson goto free_aead; 27633c4d7559SDave Watson 27644509de14SVakul Garg rc = crypto_aead_setauthsize(*aead, prot->tag_size); 2765c46234ebSDave Watson if (rc) 2766c46234ebSDave Watson goto free_aead; 2767c46234ebSDave Watson 2768f66de3eeSBoris Pismenny if (sw_ctx_rx) { 2769692d7b5dSVakul Garg tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv); 27708497ded2SVakul Garg 277188527790SJakub Kicinski tls_update_rx_zc_capable(ctx); 2772692d7b5dSVakul Garg sw_ctx_rx->async_capable = 277388527790SJakub Kicinski crypto_info->version != TLS_1_3_VERSION && 277488527790SJakub Kicinski !!(tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC); 2775692d7b5dSVakul Garg 2776849f16bbSJakub Kicinski rc = tls_strp_init(&sw_ctx_rx->strp, sk); 2777849f16bbSJakub Kicinski if (rc) 2778849f16bbSJakub Kicinski goto free_aead; 2779c46234ebSDave Watson } 2780c46234ebSDave Watson 2781c46234ebSDave Watson goto out; 27823c4d7559SDave Watson 27833c4d7559SDave Watson free_aead: 2784c46234ebSDave Watson crypto_free_aead(*aead); 2785c46234ebSDave Watson *aead = NULL; 27863c4d7559SDave Watson free_rec_seq: 2787c46234ebSDave Watson kfree(cctx->rec_seq); 2788c46234ebSDave Watson cctx->rec_seq = NULL; 27893c4d7559SDave Watson free_iv: 2790f66de3eeSBoris Pismenny kfree(cctx->iv); 2791f66de3eeSBoris Pismenny cctx->iv = NULL; 2792cf6d43efSSabrina Dubroca free_priv: 2793f66de3eeSBoris Pismenny if (tx) { 2794f66de3eeSBoris Pismenny kfree(ctx->priv_ctx_tx); 2795f66de3eeSBoris Pismenny ctx->priv_ctx_tx = NULL; 2796f66de3eeSBoris Pismenny } else { 2797f66de3eeSBoris Pismenny kfree(ctx->priv_ctx_rx); 2798f66de3eeSBoris Pismenny ctx->priv_ctx_rx = NULL; 2799f66de3eeSBoris Pismenny } 28003c4d7559SDave Watson out: 28013c4d7559SDave Watson return rc; 28023c4d7559SDave Watson } 2803