13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved. 53c4d7559SDave Watson * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved. 63c4d7559SDave Watson * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved. 7d3b18ad3SJohn Fastabend * Copyright (c) 2018, Covalent IO, Inc. http://covalent.io 83c4d7559SDave Watson * 93c4d7559SDave Watson * This software is available to you under a choice of one of two 103c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 113c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 123c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 133c4d7559SDave Watson * OpenIB.org BSD license below: 143c4d7559SDave Watson * 153c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 163c4d7559SDave Watson * without modification, are permitted provided that the following 173c4d7559SDave Watson * conditions are met: 183c4d7559SDave Watson * 193c4d7559SDave Watson * - Redistributions of source code must retain the above 203c4d7559SDave Watson * copyright notice, this list of conditions and the following 213c4d7559SDave Watson * disclaimer. 223c4d7559SDave Watson * 233c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 243c4d7559SDave Watson * copyright notice, this list of conditions and the following 253c4d7559SDave Watson * disclaimer in the documentation and/or other materials 263c4d7559SDave Watson * provided with the distribution. 273c4d7559SDave Watson * 283c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 293c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 303c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 313c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 323c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 333c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 343c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 353c4d7559SDave Watson * SOFTWARE. 363c4d7559SDave Watson */ 373c4d7559SDave Watson 38c46234ebSDave Watson #include <linux/sched/signal.h> 393c4d7559SDave Watson #include <linux/module.h> 403c4d7559SDave Watson #include <crypto/aead.h> 413c4d7559SDave Watson 42c46234ebSDave Watson #include <net/strparser.h> 433c4d7559SDave Watson #include <net/tls.h> 443c4d7559SDave Watson 45b16520f7SKees Cook #define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE 46b16520f7SKees Cook 470927f71dSDoron Roberts-Kedes static int __skb_nsg(struct sk_buff *skb, int offset, int len, 480927f71dSDoron Roberts-Kedes unsigned int recursion_level) 490927f71dSDoron Roberts-Kedes { 500927f71dSDoron Roberts-Kedes int start = skb_headlen(skb); 510927f71dSDoron Roberts-Kedes int i, chunk = start - offset; 520927f71dSDoron Roberts-Kedes struct sk_buff *frag_iter; 530927f71dSDoron Roberts-Kedes int elt = 0; 540927f71dSDoron Roberts-Kedes 550927f71dSDoron Roberts-Kedes if (unlikely(recursion_level >= 24)) 560927f71dSDoron Roberts-Kedes return -EMSGSIZE; 570927f71dSDoron Roberts-Kedes 580927f71dSDoron Roberts-Kedes if (chunk > 0) { 590927f71dSDoron Roberts-Kedes if (chunk > len) 600927f71dSDoron Roberts-Kedes chunk = len; 610927f71dSDoron Roberts-Kedes elt++; 620927f71dSDoron Roberts-Kedes len -= chunk; 630927f71dSDoron Roberts-Kedes if (len == 0) 640927f71dSDoron Roberts-Kedes return elt; 650927f71dSDoron Roberts-Kedes offset += chunk; 660927f71dSDoron Roberts-Kedes } 670927f71dSDoron Roberts-Kedes 680927f71dSDoron Roberts-Kedes for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 690927f71dSDoron Roberts-Kedes int end; 700927f71dSDoron Roberts-Kedes 710927f71dSDoron Roberts-Kedes WARN_ON(start > offset + len); 720927f71dSDoron Roberts-Kedes 730927f71dSDoron Roberts-Kedes end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 740927f71dSDoron Roberts-Kedes chunk = end - offset; 750927f71dSDoron Roberts-Kedes if (chunk > 0) { 760927f71dSDoron Roberts-Kedes if (chunk > len) 770927f71dSDoron Roberts-Kedes chunk = len; 780927f71dSDoron Roberts-Kedes elt++; 790927f71dSDoron Roberts-Kedes len -= chunk; 800927f71dSDoron Roberts-Kedes if (len == 0) 810927f71dSDoron Roberts-Kedes return elt; 820927f71dSDoron Roberts-Kedes offset += chunk; 830927f71dSDoron Roberts-Kedes } 840927f71dSDoron Roberts-Kedes start = end; 850927f71dSDoron Roberts-Kedes } 860927f71dSDoron Roberts-Kedes 870927f71dSDoron Roberts-Kedes if (unlikely(skb_has_frag_list(skb))) { 880927f71dSDoron Roberts-Kedes skb_walk_frags(skb, frag_iter) { 890927f71dSDoron Roberts-Kedes int end, ret; 900927f71dSDoron Roberts-Kedes 910927f71dSDoron Roberts-Kedes WARN_ON(start > offset + len); 920927f71dSDoron Roberts-Kedes 930927f71dSDoron Roberts-Kedes end = start + frag_iter->len; 940927f71dSDoron Roberts-Kedes chunk = end - offset; 950927f71dSDoron Roberts-Kedes if (chunk > 0) { 960927f71dSDoron Roberts-Kedes if (chunk > len) 970927f71dSDoron Roberts-Kedes chunk = len; 980927f71dSDoron Roberts-Kedes ret = __skb_nsg(frag_iter, offset - start, chunk, 990927f71dSDoron Roberts-Kedes recursion_level + 1); 1000927f71dSDoron Roberts-Kedes if (unlikely(ret < 0)) 1010927f71dSDoron Roberts-Kedes return ret; 1020927f71dSDoron Roberts-Kedes elt += ret; 1030927f71dSDoron Roberts-Kedes len -= chunk; 1040927f71dSDoron Roberts-Kedes if (len == 0) 1050927f71dSDoron Roberts-Kedes return elt; 1060927f71dSDoron Roberts-Kedes offset += chunk; 1070927f71dSDoron Roberts-Kedes } 1080927f71dSDoron Roberts-Kedes start = end; 1090927f71dSDoron Roberts-Kedes } 1100927f71dSDoron Roberts-Kedes } 1110927f71dSDoron Roberts-Kedes BUG_ON(len); 1120927f71dSDoron Roberts-Kedes return elt; 1130927f71dSDoron Roberts-Kedes } 1140927f71dSDoron Roberts-Kedes 1150927f71dSDoron Roberts-Kedes /* Return the number of scatterlist elements required to completely map the 1160927f71dSDoron Roberts-Kedes * skb, or -EMSGSIZE if the recursion depth is exceeded. 1170927f71dSDoron Roberts-Kedes */ 1180927f71dSDoron Roberts-Kedes static int skb_nsg(struct sk_buff *skb, int offset, int len) 1190927f71dSDoron Roberts-Kedes { 1200927f71dSDoron Roberts-Kedes return __skb_nsg(skb, offset, len, 0); 1210927f71dSDoron Roberts-Kedes } 1220927f71dSDoron Roberts-Kedes 12394524d8fSVakul Garg static void tls_decrypt_done(struct crypto_async_request *req, int err) 12494524d8fSVakul Garg { 12594524d8fSVakul Garg struct aead_request *aead_req = (struct aead_request *)req; 12694524d8fSVakul Garg struct scatterlist *sgout = aead_req->dst; 1277a3dd8c8SJohn Fastabend struct tls_sw_context_rx *ctx; 1287a3dd8c8SJohn Fastabend struct tls_context *tls_ctx; 12994524d8fSVakul Garg struct scatterlist *sg; 1307a3dd8c8SJohn Fastabend struct sk_buff *skb; 13194524d8fSVakul Garg unsigned int pages; 1327a3dd8c8SJohn Fastabend int pending; 1337a3dd8c8SJohn Fastabend 1347a3dd8c8SJohn Fastabend skb = (struct sk_buff *)req->data; 1357a3dd8c8SJohn Fastabend tls_ctx = tls_get_ctx(skb->sk); 1367a3dd8c8SJohn Fastabend ctx = tls_sw_ctx_rx(tls_ctx); 1377a3dd8c8SJohn Fastabend pending = atomic_dec_return(&ctx->decrypt_pending); 13894524d8fSVakul Garg 13994524d8fSVakul Garg /* Propagate if there was an err */ 14094524d8fSVakul Garg if (err) { 14194524d8fSVakul Garg ctx->async_wait.err = err; 1427a3dd8c8SJohn Fastabend tls_err_abort(skb->sk, err); 14394524d8fSVakul Garg } 14494524d8fSVakul Garg 1457a3dd8c8SJohn Fastabend /* After using skb->sk to propagate sk through crypto async callback 1467a3dd8c8SJohn Fastabend * we need to NULL it again. 1477a3dd8c8SJohn Fastabend */ 1487a3dd8c8SJohn Fastabend skb->sk = NULL; 1497a3dd8c8SJohn Fastabend 15094524d8fSVakul Garg /* Release the skb, pages and memory allocated for crypto req */ 1517a3dd8c8SJohn Fastabend kfree_skb(skb); 15294524d8fSVakul Garg 15394524d8fSVakul Garg /* Skip the first S/G entry as it points to AAD */ 15494524d8fSVakul Garg for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { 15594524d8fSVakul Garg if (!sg) 15694524d8fSVakul Garg break; 15794524d8fSVakul Garg put_page(sg_page(sg)); 15894524d8fSVakul Garg } 15994524d8fSVakul Garg 16094524d8fSVakul Garg kfree(aead_req); 16194524d8fSVakul Garg 16294524d8fSVakul Garg if (!pending && READ_ONCE(ctx->async_notify)) 16394524d8fSVakul Garg complete(&ctx->async_wait.completion); 16494524d8fSVakul Garg } 16594524d8fSVakul Garg 166c46234ebSDave Watson static int tls_do_decryption(struct sock *sk, 16794524d8fSVakul Garg struct sk_buff *skb, 168c46234ebSDave Watson struct scatterlist *sgin, 169c46234ebSDave Watson struct scatterlist *sgout, 170c46234ebSDave Watson char *iv_recv, 171c46234ebSDave Watson size_t data_len, 17294524d8fSVakul Garg struct aead_request *aead_req, 17394524d8fSVakul Garg bool async) 174c46234ebSDave Watson { 175c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 176f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 177c46234ebSDave Watson int ret; 178c46234ebSDave Watson 1790b243d00SVakul Garg aead_request_set_tfm(aead_req, ctx->aead_recv); 180c46234ebSDave Watson aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); 181c46234ebSDave Watson aead_request_set_crypt(aead_req, sgin, sgout, 182c46234ebSDave Watson data_len + tls_ctx->rx.tag_size, 183c46234ebSDave Watson (u8 *)iv_recv); 184c46234ebSDave Watson 18594524d8fSVakul Garg if (async) { 1867a3dd8c8SJohn Fastabend /* Using skb->sk to push sk through to crypto async callback 1877a3dd8c8SJohn Fastabend * handler. This allows propagating errors up to the socket 1887a3dd8c8SJohn Fastabend * if needed. It _must_ be cleared in the async handler 1897a3dd8c8SJohn Fastabend * before kfree_skb is called. We _know_ skb->sk is NULL 1907a3dd8c8SJohn Fastabend * because it is a clone from strparser. 1917a3dd8c8SJohn Fastabend */ 1927a3dd8c8SJohn Fastabend skb->sk = sk; 19394524d8fSVakul Garg aead_request_set_callback(aead_req, 19494524d8fSVakul Garg CRYPTO_TFM_REQ_MAY_BACKLOG, 19594524d8fSVakul Garg tls_decrypt_done, skb); 19694524d8fSVakul Garg atomic_inc(&ctx->decrypt_pending); 19794524d8fSVakul Garg } else { 19894524d8fSVakul Garg aead_request_set_callback(aead_req, 19994524d8fSVakul Garg CRYPTO_TFM_REQ_MAY_BACKLOG, 20094524d8fSVakul Garg crypto_req_done, &ctx->async_wait); 20194524d8fSVakul Garg } 20294524d8fSVakul Garg 20394524d8fSVakul Garg ret = crypto_aead_decrypt(aead_req); 20494524d8fSVakul Garg if (ret == -EINPROGRESS) { 20594524d8fSVakul Garg if (async) 20694524d8fSVakul Garg return ret; 20794524d8fSVakul Garg 20894524d8fSVakul Garg ret = crypto_wait_req(ret, &ctx->async_wait); 20994524d8fSVakul Garg } 21094524d8fSVakul Garg 21194524d8fSVakul Garg if (async) 21294524d8fSVakul Garg atomic_dec(&ctx->decrypt_pending); 21394524d8fSVakul Garg 214c46234ebSDave Watson return ret; 215c46234ebSDave Watson } 216c46234ebSDave Watson 217d829e9c4SDaniel Borkmann static void tls_trim_both_msgs(struct sock *sk, int target_size) 2183c4d7559SDave Watson { 2193c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 220f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 221a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 2223c4d7559SDave Watson 223d829e9c4SDaniel Borkmann sk_msg_trim(sk, &rec->msg_plaintext, target_size); 2243c4d7559SDave Watson if (target_size > 0) 225dbe42559SDave Watson target_size += tls_ctx->tx.overhead_size; 226d829e9c4SDaniel Borkmann sk_msg_trim(sk, &rec->msg_encrypted, target_size); 2273c4d7559SDave Watson } 2283c4d7559SDave Watson 229d829e9c4SDaniel Borkmann static int tls_alloc_encrypted_msg(struct sock *sk, int len) 2303c4d7559SDave Watson { 2313c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 232f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 233a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 234d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 2353c4d7559SDave Watson 236d829e9c4SDaniel Borkmann return sk_msg_alloc(sk, msg_en, len, 0); 2373c4d7559SDave Watson } 2383c4d7559SDave Watson 239d829e9c4SDaniel Borkmann static int tls_clone_plaintext_msg(struct sock *sk, int required) 2403c4d7559SDave Watson { 2413c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 242f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 243a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 244d829e9c4SDaniel Borkmann struct sk_msg *msg_pl = &rec->msg_plaintext; 245d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 2464e6d4720SVakul Garg int skip, len; 2473c4d7559SDave Watson 248d829e9c4SDaniel Borkmann /* We add page references worth len bytes from encrypted sg 249d829e9c4SDaniel Borkmann * at the end of plaintext sg. It is guaranteed that msg_en 2504e6d4720SVakul Garg * has enough required room (ensured by caller). 2514e6d4720SVakul Garg */ 252d829e9c4SDaniel Borkmann len = required - msg_pl->sg.size; 25352ea992cSVakul Garg 254d829e9c4SDaniel Borkmann /* Skip initial bytes in msg_en's data to be able to use 255d829e9c4SDaniel Borkmann * same offset of both plain and encrypted data. 2564e6d4720SVakul Garg */ 257d829e9c4SDaniel Borkmann skip = tls_ctx->tx.prepend_size + msg_pl->sg.size; 2584e6d4720SVakul Garg 259d829e9c4SDaniel Borkmann return sk_msg_clone(sk, msg_pl, msg_en, skip, len); 2603c4d7559SDave Watson } 2613c4d7559SDave Watson 262d3b18ad3SJohn Fastabend static struct tls_rec *tls_get_rec(struct sock *sk) 263d3b18ad3SJohn Fastabend { 264d3b18ad3SJohn Fastabend struct tls_context *tls_ctx = tls_get_ctx(sk); 265d3b18ad3SJohn Fastabend struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 266d3b18ad3SJohn Fastabend struct sk_msg *msg_pl, *msg_en; 267d3b18ad3SJohn Fastabend struct tls_rec *rec; 268d3b18ad3SJohn Fastabend int mem_size; 269d3b18ad3SJohn Fastabend 270d3b18ad3SJohn Fastabend mem_size = sizeof(struct tls_rec) + crypto_aead_reqsize(ctx->aead_send); 271d3b18ad3SJohn Fastabend 272d3b18ad3SJohn Fastabend rec = kzalloc(mem_size, sk->sk_allocation); 273d3b18ad3SJohn Fastabend if (!rec) 274d3b18ad3SJohn Fastabend return NULL; 275d3b18ad3SJohn Fastabend 276d3b18ad3SJohn Fastabend msg_pl = &rec->msg_plaintext; 277d3b18ad3SJohn Fastabend msg_en = &rec->msg_encrypted; 278d3b18ad3SJohn Fastabend 279d3b18ad3SJohn Fastabend sk_msg_init(msg_pl); 280d3b18ad3SJohn Fastabend sk_msg_init(msg_en); 281d3b18ad3SJohn Fastabend 282d3b18ad3SJohn Fastabend sg_init_table(rec->sg_aead_in, 2); 283d3b18ad3SJohn Fastabend sg_set_buf(&rec->sg_aead_in[0], rec->aad_space, 284d3b18ad3SJohn Fastabend sizeof(rec->aad_space)); 285d3b18ad3SJohn Fastabend sg_unmark_end(&rec->sg_aead_in[1]); 286d3b18ad3SJohn Fastabend 287d3b18ad3SJohn Fastabend sg_init_table(rec->sg_aead_out, 2); 288d3b18ad3SJohn Fastabend sg_set_buf(&rec->sg_aead_out[0], rec->aad_space, 289d3b18ad3SJohn Fastabend sizeof(rec->aad_space)); 290d3b18ad3SJohn Fastabend sg_unmark_end(&rec->sg_aead_out[1]); 291d3b18ad3SJohn Fastabend 292d3b18ad3SJohn Fastabend return rec; 293d3b18ad3SJohn Fastabend } 294d3b18ad3SJohn Fastabend 295d3b18ad3SJohn Fastabend static void tls_free_rec(struct sock *sk, struct tls_rec *rec) 296d3b18ad3SJohn Fastabend { 297d3b18ad3SJohn Fastabend sk_msg_free(sk, &rec->msg_encrypted); 298d3b18ad3SJohn Fastabend sk_msg_free(sk, &rec->msg_plaintext); 299d3b18ad3SJohn Fastabend kfree(rec); 300d3b18ad3SJohn Fastabend } 301d3b18ad3SJohn Fastabend 302c774973eSVakul Garg static void tls_free_open_rec(struct sock *sk) 3033c4d7559SDave Watson { 3043c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 305f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 306a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 3073c4d7559SDave Watson 308d3b18ad3SJohn Fastabend if (rec) { 309d3b18ad3SJohn Fastabend tls_free_rec(sk, rec); 310d3b18ad3SJohn Fastabend ctx->open_rec = NULL; 311d3b18ad3SJohn Fastabend } 3123c4d7559SDave Watson } 3133c4d7559SDave Watson 314a42055e8SVakul Garg int tls_tx_records(struct sock *sk, int flags) 315a42055e8SVakul Garg { 316a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 317a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 318a42055e8SVakul Garg struct tls_rec *rec, *tmp; 319d829e9c4SDaniel Borkmann struct sk_msg *msg_en; 320a42055e8SVakul Garg int tx_flags, rc = 0; 321a42055e8SVakul Garg 322a42055e8SVakul Garg if (tls_is_partially_sent_record(tls_ctx)) { 3239932a29aSVakul Garg rec = list_first_entry(&ctx->tx_list, 324a42055e8SVakul Garg struct tls_rec, list); 325a42055e8SVakul Garg 326a42055e8SVakul Garg if (flags == -1) 327a42055e8SVakul Garg tx_flags = rec->tx_flags; 328a42055e8SVakul Garg else 329a42055e8SVakul Garg tx_flags = flags; 330a42055e8SVakul Garg 331a42055e8SVakul Garg rc = tls_push_partial_record(sk, tls_ctx, tx_flags); 332a42055e8SVakul Garg if (rc) 333a42055e8SVakul Garg goto tx_err; 334a42055e8SVakul Garg 335a42055e8SVakul Garg /* Full record has been transmitted. 3369932a29aSVakul Garg * Remove the head of tx_list 337a42055e8SVakul Garg */ 338a42055e8SVakul Garg list_del(&rec->list); 339d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 340a42055e8SVakul Garg kfree(rec); 341a42055e8SVakul Garg } 342a42055e8SVakul Garg 3439932a29aSVakul Garg /* Tx all ready records */ 3449932a29aSVakul Garg list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { 3459932a29aSVakul Garg if (READ_ONCE(rec->tx_ready)) { 346a42055e8SVakul Garg if (flags == -1) 347a42055e8SVakul Garg tx_flags = rec->tx_flags; 348a42055e8SVakul Garg else 349a42055e8SVakul Garg tx_flags = flags; 350a42055e8SVakul Garg 351d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 352a42055e8SVakul Garg rc = tls_push_sg(sk, tls_ctx, 353d829e9c4SDaniel Borkmann &msg_en->sg.data[msg_en->sg.curr], 354a42055e8SVakul Garg 0, tx_flags); 355a42055e8SVakul Garg if (rc) 356a42055e8SVakul Garg goto tx_err; 357a42055e8SVakul Garg 358a42055e8SVakul Garg list_del(&rec->list); 359d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 360a42055e8SVakul Garg kfree(rec); 361a42055e8SVakul Garg } else { 362a42055e8SVakul Garg break; 363a42055e8SVakul Garg } 364a42055e8SVakul Garg } 365a42055e8SVakul Garg 366a42055e8SVakul Garg tx_err: 367a42055e8SVakul Garg if (rc < 0 && rc != -EAGAIN) 368a42055e8SVakul Garg tls_err_abort(sk, EBADMSG); 369a42055e8SVakul Garg 370a42055e8SVakul Garg return rc; 371a42055e8SVakul Garg } 372a42055e8SVakul Garg 373a42055e8SVakul Garg static void tls_encrypt_done(struct crypto_async_request *req, int err) 374a42055e8SVakul Garg { 375a42055e8SVakul Garg struct aead_request *aead_req = (struct aead_request *)req; 376a42055e8SVakul Garg struct sock *sk = req->data; 377a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 378a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 379d829e9c4SDaniel Borkmann struct scatterlist *sge; 380d829e9c4SDaniel Borkmann struct sk_msg *msg_en; 381a42055e8SVakul Garg struct tls_rec *rec; 382a42055e8SVakul Garg bool ready = false; 383a42055e8SVakul Garg int pending; 384a42055e8SVakul Garg 385a42055e8SVakul Garg rec = container_of(aead_req, struct tls_rec, aead_req); 386d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 387a42055e8SVakul Garg 388d829e9c4SDaniel Borkmann sge = sk_msg_elem(msg_en, msg_en->sg.curr); 389d829e9c4SDaniel Borkmann sge->offset -= tls_ctx->tx.prepend_size; 390d829e9c4SDaniel Borkmann sge->length += tls_ctx->tx.prepend_size; 391a42055e8SVakul Garg 39280ece6a0SVakul Garg /* Check if error is previously set on socket */ 393a42055e8SVakul Garg if (err || sk->sk_err) { 394a42055e8SVakul Garg rec = NULL; 395a42055e8SVakul Garg 396a42055e8SVakul Garg /* If err is already set on socket, return the same code */ 397a42055e8SVakul Garg if (sk->sk_err) { 398a42055e8SVakul Garg ctx->async_wait.err = sk->sk_err; 399a42055e8SVakul Garg } else { 400a42055e8SVakul Garg ctx->async_wait.err = err; 401a42055e8SVakul Garg tls_err_abort(sk, err); 402a42055e8SVakul Garg } 403a42055e8SVakul Garg } 404a42055e8SVakul Garg 4059932a29aSVakul Garg if (rec) { 4069932a29aSVakul Garg struct tls_rec *first_rec; 4079932a29aSVakul Garg 4089932a29aSVakul Garg /* Mark the record as ready for transmission */ 4099932a29aSVakul Garg smp_store_mb(rec->tx_ready, true); 4109932a29aSVakul Garg 4119932a29aSVakul Garg /* If received record is at head of tx_list, schedule tx */ 4129932a29aSVakul Garg first_rec = list_first_entry(&ctx->tx_list, 4139932a29aSVakul Garg struct tls_rec, list); 4149932a29aSVakul Garg if (rec == first_rec) 4159932a29aSVakul Garg ready = true; 4169932a29aSVakul Garg } 417a42055e8SVakul Garg 418a42055e8SVakul Garg pending = atomic_dec_return(&ctx->encrypt_pending); 419a42055e8SVakul Garg 420a42055e8SVakul Garg if (!pending && READ_ONCE(ctx->async_notify)) 421a42055e8SVakul Garg complete(&ctx->async_wait.completion); 422a42055e8SVakul Garg 423a42055e8SVakul Garg if (!ready) 424a42055e8SVakul Garg return; 425a42055e8SVakul Garg 426a42055e8SVakul Garg /* Schedule the transmission */ 427a42055e8SVakul Garg if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) 428d829e9c4SDaniel Borkmann schedule_delayed_work(&ctx->tx_work.work, 1); 429a42055e8SVakul Garg } 430a42055e8SVakul Garg 431a42055e8SVakul Garg static int tls_do_encryption(struct sock *sk, 432a42055e8SVakul Garg struct tls_context *tls_ctx, 433a447da7dSDaniel Borkmann struct tls_sw_context_tx *ctx, 434a447da7dSDaniel Borkmann struct aead_request *aead_req, 435d829e9c4SDaniel Borkmann size_t data_len, u32 start) 4363c4d7559SDave Watson { 437a42055e8SVakul Garg struct tls_rec *rec = ctx->open_rec; 438d829e9c4SDaniel Borkmann struct sk_msg *msg_en = &rec->msg_encrypted; 439d829e9c4SDaniel Borkmann struct scatterlist *sge = sk_msg_elem(msg_en, start); 4403c4d7559SDave Watson int rc; 4413c4d7559SDave Watson 44232eb67b9SDave Watson memcpy(rec->iv_data, tls_ctx->tx.iv, sizeof(rec->iv_data)); 44332eb67b9SDave Watson 444d829e9c4SDaniel Borkmann sge->offset += tls_ctx->tx.prepend_size; 445d829e9c4SDaniel Borkmann sge->length -= tls_ctx->tx.prepend_size; 4463c4d7559SDave Watson 447d829e9c4SDaniel Borkmann msg_en->sg.curr = start; 4484e6d4720SVakul Garg 4493c4d7559SDave Watson aead_request_set_tfm(aead_req, ctx->aead_send); 4503c4d7559SDave Watson aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); 451d829e9c4SDaniel Borkmann aead_request_set_crypt(aead_req, rec->sg_aead_in, 452d829e9c4SDaniel Borkmann rec->sg_aead_out, 45332eb67b9SDave Watson data_len, rec->iv_data); 454a54667f6SVakul Garg 455a54667f6SVakul Garg aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 456a42055e8SVakul Garg tls_encrypt_done, sk); 457a54667f6SVakul Garg 4589932a29aSVakul Garg /* Add the record in tx_list */ 4599932a29aSVakul Garg list_add_tail((struct list_head *)&rec->list, &ctx->tx_list); 460a42055e8SVakul Garg atomic_inc(&ctx->encrypt_pending); 4613c4d7559SDave Watson 462a42055e8SVakul Garg rc = crypto_aead_encrypt(aead_req); 463a42055e8SVakul Garg if (!rc || rc != -EINPROGRESS) { 464a42055e8SVakul Garg atomic_dec(&ctx->encrypt_pending); 465d829e9c4SDaniel Borkmann sge->offset -= tls_ctx->tx.prepend_size; 466d829e9c4SDaniel Borkmann sge->length += tls_ctx->tx.prepend_size; 467a42055e8SVakul Garg } 4683c4d7559SDave Watson 4699932a29aSVakul Garg if (!rc) { 4709932a29aSVakul Garg WRITE_ONCE(rec->tx_ready, true); 4719932a29aSVakul Garg } else if (rc != -EINPROGRESS) { 4729932a29aSVakul Garg list_del(&rec->list); 473a42055e8SVakul Garg return rc; 4749932a29aSVakul Garg } 475a42055e8SVakul Garg 476a42055e8SVakul Garg /* Unhook the record from context if encryption is not failure */ 477a42055e8SVakul Garg ctx->open_rec = NULL; 478a42055e8SVakul Garg tls_advance_record_sn(sk, &tls_ctx->tx); 4793c4d7559SDave Watson return rc; 4803c4d7559SDave Watson } 4813c4d7559SDave Watson 482d3b18ad3SJohn Fastabend static int tls_split_open_record(struct sock *sk, struct tls_rec *from, 483d3b18ad3SJohn Fastabend struct tls_rec **to, struct sk_msg *msg_opl, 484d3b18ad3SJohn Fastabend struct sk_msg *msg_oen, u32 split_point, 485d3b18ad3SJohn Fastabend u32 tx_overhead_size, u32 *orig_end) 486d3b18ad3SJohn Fastabend { 487d3b18ad3SJohn Fastabend u32 i, j, bytes = 0, apply = msg_opl->apply_bytes; 488d3b18ad3SJohn Fastabend struct scatterlist *sge, *osge, *nsge; 489d3b18ad3SJohn Fastabend u32 orig_size = msg_opl->sg.size; 490d3b18ad3SJohn Fastabend struct scatterlist tmp = { }; 491d3b18ad3SJohn Fastabend struct sk_msg *msg_npl; 492d3b18ad3SJohn Fastabend struct tls_rec *new; 493d3b18ad3SJohn Fastabend int ret; 494d3b18ad3SJohn Fastabend 495d3b18ad3SJohn Fastabend new = tls_get_rec(sk); 496d3b18ad3SJohn Fastabend if (!new) 497d3b18ad3SJohn Fastabend return -ENOMEM; 498d3b18ad3SJohn Fastabend ret = sk_msg_alloc(sk, &new->msg_encrypted, msg_opl->sg.size + 499d3b18ad3SJohn Fastabend tx_overhead_size, 0); 500d3b18ad3SJohn Fastabend if (ret < 0) { 501d3b18ad3SJohn Fastabend tls_free_rec(sk, new); 502d3b18ad3SJohn Fastabend return ret; 503d3b18ad3SJohn Fastabend } 504d3b18ad3SJohn Fastabend 505d3b18ad3SJohn Fastabend *orig_end = msg_opl->sg.end; 506d3b18ad3SJohn Fastabend i = msg_opl->sg.start; 507d3b18ad3SJohn Fastabend sge = sk_msg_elem(msg_opl, i); 508d3b18ad3SJohn Fastabend while (apply && sge->length) { 509d3b18ad3SJohn Fastabend if (sge->length > apply) { 510d3b18ad3SJohn Fastabend u32 len = sge->length - apply; 511d3b18ad3SJohn Fastabend 512d3b18ad3SJohn Fastabend get_page(sg_page(sge)); 513d3b18ad3SJohn Fastabend sg_set_page(&tmp, sg_page(sge), len, 514d3b18ad3SJohn Fastabend sge->offset + apply); 515d3b18ad3SJohn Fastabend sge->length = apply; 516d3b18ad3SJohn Fastabend bytes += apply; 517d3b18ad3SJohn Fastabend apply = 0; 518d3b18ad3SJohn Fastabend } else { 519d3b18ad3SJohn Fastabend apply -= sge->length; 520d3b18ad3SJohn Fastabend bytes += sge->length; 521d3b18ad3SJohn Fastabend } 522d3b18ad3SJohn Fastabend 523d3b18ad3SJohn Fastabend sk_msg_iter_var_next(i); 524d3b18ad3SJohn Fastabend if (i == msg_opl->sg.end) 525d3b18ad3SJohn Fastabend break; 526d3b18ad3SJohn Fastabend sge = sk_msg_elem(msg_opl, i); 527d3b18ad3SJohn Fastabend } 528d3b18ad3SJohn Fastabend 529d3b18ad3SJohn Fastabend msg_opl->sg.end = i; 530d3b18ad3SJohn Fastabend msg_opl->sg.curr = i; 531d3b18ad3SJohn Fastabend msg_opl->sg.copybreak = 0; 532d3b18ad3SJohn Fastabend msg_opl->apply_bytes = 0; 533d3b18ad3SJohn Fastabend msg_opl->sg.size = bytes; 534d3b18ad3SJohn Fastabend 535d3b18ad3SJohn Fastabend msg_npl = &new->msg_plaintext; 536d3b18ad3SJohn Fastabend msg_npl->apply_bytes = apply; 537d3b18ad3SJohn Fastabend msg_npl->sg.size = orig_size - bytes; 538d3b18ad3SJohn Fastabend 539d3b18ad3SJohn Fastabend j = msg_npl->sg.start; 540d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 541d3b18ad3SJohn Fastabend if (tmp.length) { 542d3b18ad3SJohn Fastabend memcpy(nsge, &tmp, sizeof(*nsge)); 543d3b18ad3SJohn Fastabend sk_msg_iter_var_next(j); 544d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 545d3b18ad3SJohn Fastabend } 546d3b18ad3SJohn Fastabend 547d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 548d3b18ad3SJohn Fastabend while (osge->length) { 549d3b18ad3SJohn Fastabend memcpy(nsge, osge, sizeof(*nsge)); 550d3b18ad3SJohn Fastabend sg_unmark_end(nsge); 551d3b18ad3SJohn Fastabend sk_msg_iter_var_next(i); 552d3b18ad3SJohn Fastabend sk_msg_iter_var_next(j); 553d3b18ad3SJohn Fastabend if (i == *orig_end) 554d3b18ad3SJohn Fastabend break; 555d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 556d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 557d3b18ad3SJohn Fastabend } 558d3b18ad3SJohn Fastabend 559d3b18ad3SJohn Fastabend msg_npl->sg.end = j; 560d3b18ad3SJohn Fastabend msg_npl->sg.curr = j; 561d3b18ad3SJohn Fastabend msg_npl->sg.copybreak = 0; 562d3b18ad3SJohn Fastabend 563d3b18ad3SJohn Fastabend *to = new; 564d3b18ad3SJohn Fastabend return 0; 565d3b18ad3SJohn Fastabend } 566d3b18ad3SJohn Fastabend 567d3b18ad3SJohn Fastabend static void tls_merge_open_record(struct sock *sk, struct tls_rec *to, 568d3b18ad3SJohn Fastabend struct tls_rec *from, u32 orig_end) 569d3b18ad3SJohn Fastabend { 570d3b18ad3SJohn Fastabend struct sk_msg *msg_npl = &from->msg_plaintext; 571d3b18ad3SJohn Fastabend struct sk_msg *msg_opl = &to->msg_plaintext; 572d3b18ad3SJohn Fastabend struct scatterlist *osge, *nsge; 573d3b18ad3SJohn Fastabend u32 i, j; 574d3b18ad3SJohn Fastabend 575d3b18ad3SJohn Fastabend i = msg_opl->sg.end; 576d3b18ad3SJohn Fastabend sk_msg_iter_var_prev(i); 577d3b18ad3SJohn Fastabend j = msg_npl->sg.start; 578d3b18ad3SJohn Fastabend 579d3b18ad3SJohn Fastabend osge = sk_msg_elem(msg_opl, i); 580d3b18ad3SJohn Fastabend nsge = sk_msg_elem(msg_npl, j); 581d3b18ad3SJohn Fastabend 582d3b18ad3SJohn Fastabend if (sg_page(osge) == sg_page(nsge) && 583d3b18ad3SJohn Fastabend osge->offset + osge->length == nsge->offset) { 584d3b18ad3SJohn Fastabend osge->length += nsge->length; 585d3b18ad3SJohn Fastabend put_page(sg_page(nsge)); 586d3b18ad3SJohn Fastabend } 587d3b18ad3SJohn Fastabend 588d3b18ad3SJohn Fastabend msg_opl->sg.end = orig_end; 589d3b18ad3SJohn Fastabend msg_opl->sg.curr = orig_end; 590d3b18ad3SJohn Fastabend msg_opl->sg.copybreak = 0; 591d3b18ad3SJohn Fastabend msg_opl->apply_bytes = msg_opl->sg.size + msg_npl->sg.size; 592d3b18ad3SJohn Fastabend msg_opl->sg.size += msg_npl->sg.size; 593d3b18ad3SJohn Fastabend 594d3b18ad3SJohn Fastabend sk_msg_free(sk, &to->msg_encrypted); 595d3b18ad3SJohn Fastabend sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted); 596d3b18ad3SJohn Fastabend 597d3b18ad3SJohn Fastabend kfree(from); 598d3b18ad3SJohn Fastabend } 599d3b18ad3SJohn Fastabend 6003c4d7559SDave Watson static int tls_push_record(struct sock *sk, int flags, 6013c4d7559SDave Watson unsigned char record_type) 6023c4d7559SDave Watson { 6033c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 604f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 605d3b18ad3SJohn Fastabend struct tls_rec *rec = ctx->open_rec, *tmp = NULL; 606d3b18ad3SJohn Fastabend u32 i, split_point, uninitialized_var(orig_end); 607d829e9c4SDaniel Borkmann struct sk_msg *msg_pl, *msg_en; 608a447da7dSDaniel Borkmann struct aead_request *req; 609d3b18ad3SJohn Fastabend bool split; 6103c4d7559SDave Watson int rc; 6113c4d7559SDave Watson 612a42055e8SVakul Garg if (!rec) 613a42055e8SVakul Garg return 0; 614a447da7dSDaniel Borkmann 615d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 616d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 617d829e9c4SDaniel Borkmann 618d3b18ad3SJohn Fastabend split_point = msg_pl->apply_bytes; 619d3b18ad3SJohn Fastabend split = split_point && split_point < msg_pl->sg.size; 620d3b18ad3SJohn Fastabend if (split) { 621d3b18ad3SJohn Fastabend rc = tls_split_open_record(sk, rec, &tmp, msg_pl, msg_en, 622d3b18ad3SJohn Fastabend split_point, tls_ctx->tx.overhead_size, 623d3b18ad3SJohn Fastabend &orig_end); 624d3b18ad3SJohn Fastabend if (rc < 0) 625d3b18ad3SJohn Fastabend return rc; 626d3b18ad3SJohn Fastabend sk_msg_trim(sk, msg_en, msg_pl->sg.size + 627d3b18ad3SJohn Fastabend tls_ctx->tx.overhead_size); 628d3b18ad3SJohn Fastabend } 629d3b18ad3SJohn Fastabend 630a42055e8SVakul Garg rec->tx_flags = flags; 631a42055e8SVakul Garg req = &rec->aead_req; 6323c4d7559SDave Watson 633d829e9c4SDaniel Borkmann i = msg_pl->sg.end; 634d829e9c4SDaniel Borkmann sk_msg_iter_var_prev(i); 635d829e9c4SDaniel Borkmann sg_mark_end(sk_msg_elem(msg_pl, i)); 636a42055e8SVakul Garg 637d829e9c4SDaniel Borkmann i = msg_pl->sg.start; 638d829e9c4SDaniel Borkmann sg_chain(rec->sg_aead_in, 2, rec->inplace_crypto ? 639d829e9c4SDaniel Borkmann &msg_en->sg.data[i] : &msg_pl->sg.data[i]); 640d829e9c4SDaniel Borkmann 641d829e9c4SDaniel Borkmann i = msg_en->sg.end; 642d829e9c4SDaniel Borkmann sk_msg_iter_var_prev(i); 643d829e9c4SDaniel Borkmann sg_mark_end(sk_msg_elem(msg_en, i)); 644d829e9c4SDaniel Borkmann 645d829e9c4SDaniel Borkmann i = msg_en->sg.start; 646d829e9c4SDaniel Borkmann sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]); 647d829e9c4SDaniel Borkmann 648d829e9c4SDaniel Borkmann tls_make_aad(rec->aad_space, msg_pl->sg.size, 649dbe42559SDave Watson tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size, 6503c4d7559SDave Watson record_type); 6513c4d7559SDave Watson 6523c4d7559SDave Watson tls_fill_prepend(tls_ctx, 653d829e9c4SDaniel Borkmann page_address(sg_page(&msg_en->sg.data[i])) + 654d829e9c4SDaniel Borkmann msg_en->sg.data[i].offset, msg_pl->sg.size, 655d829e9c4SDaniel Borkmann record_type); 6563c4d7559SDave Watson 657d829e9c4SDaniel Borkmann tls_ctx->pending_open_record_frags = false; 6583c4d7559SDave Watson 659d829e9c4SDaniel Borkmann rc = tls_do_encryption(sk, tls_ctx, ctx, req, msg_pl->sg.size, i); 6603c4d7559SDave Watson if (rc < 0) { 661d3b18ad3SJohn Fastabend if (rc != -EINPROGRESS) { 662a42055e8SVakul Garg tls_err_abort(sk, EBADMSG); 663d3b18ad3SJohn Fastabend if (split) { 664d3b18ad3SJohn Fastabend tls_ctx->pending_open_record_frags = true; 665d3b18ad3SJohn Fastabend tls_merge_open_record(sk, rec, tmp, orig_end); 666d3b18ad3SJohn Fastabend } 667d3b18ad3SJohn Fastabend } 668a42055e8SVakul Garg return rc; 669d3b18ad3SJohn Fastabend } else if (split) { 670d3b18ad3SJohn Fastabend msg_pl = &tmp->msg_plaintext; 671d3b18ad3SJohn Fastabend msg_en = &tmp->msg_encrypted; 672d3b18ad3SJohn Fastabend sk_msg_trim(sk, msg_en, msg_pl->sg.size + 673d3b18ad3SJohn Fastabend tls_ctx->tx.overhead_size); 674d3b18ad3SJohn Fastabend tls_ctx->pending_open_record_frags = true; 675d3b18ad3SJohn Fastabend ctx->open_rec = tmp; 6763c4d7559SDave Watson } 6773c4d7559SDave Watson 678a42055e8SVakul Garg return tls_tx_records(sk, flags); 6793c4d7559SDave Watson } 6803c4d7559SDave Watson 681d3b18ad3SJohn Fastabend static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, 682d3b18ad3SJohn Fastabend bool full_record, u8 record_type, 683d3b18ad3SJohn Fastabend size_t *copied, int flags) 6843c4d7559SDave Watson { 6853c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 686f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 687d3b18ad3SJohn Fastabend struct sk_msg msg_redir = { }; 688d3b18ad3SJohn Fastabend struct sk_psock *psock; 689d3b18ad3SJohn Fastabend struct sock *sk_redir; 690a42055e8SVakul Garg struct tls_rec *rec; 6910608c69cSJohn Fastabend bool enospc, policy; 692d3b18ad3SJohn Fastabend int err = 0, send; 6937246d8edSJohn Fastabend u32 delta = 0; 694a42055e8SVakul Garg 6950608c69cSJohn Fastabend policy = !(flags & MSG_SENDPAGE_NOPOLICY); 696d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 6970608c69cSJohn Fastabend if (!psock || !policy) 698d3b18ad3SJohn Fastabend return tls_push_record(sk, flags, record_type); 699d3b18ad3SJohn Fastabend more_data: 700d3b18ad3SJohn Fastabend enospc = sk_msg_full(msg); 7017246d8edSJohn Fastabend if (psock->eval == __SK_NONE) { 7027246d8edSJohn Fastabend delta = msg->sg.size; 703d3b18ad3SJohn Fastabend psock->eval = sk_psock_msg_verdict(sk, psock, msg); 7047246d8edSJohn Fastabend if (delta < msg->sg.size) 7057246d8edSJohn Fastabend delta -= msg->sg.size; 7067246d8edSJohn Fastabend else 7077246d8edSJohn Fastabend delta = 0; 7087246d8edSJohn Fastabend } 709d3b18ad3SJohn Fastabend if (msg->cork_bytes && msg->cork_bytes > msg->sg.size && 710d3b18ad3SJohn Fastabend !enospc && !full_record) { 711d3b18ad3SJohn Fastabend err = -ENOSPC; 712d3b18ad3SJohn Fastabend goto out_err; 713d3b18ad3SJohn Fastabend } 714d3b18ad3SJohn Fastabend msg->cork_bytes = 0; 715d3b18ad3SJohn Fastabend send = msg->sg.size; 716d3b18ad3SJohn Fastabend if (msg->apply_bytes && msg->apply_bytes < send) 717d3b18ad3SJohn Fastabend send = msg->apply_bytes; 718a42055e8SVakul Garg 719d3b18ad3SJohn Fastabend switch (psock->eval) { 720d3b18ad3SJohn Fastabend case __SK_PASS: 721d3b18ad3SJohn Fastabend err = tls_push_record(sk, flags, record_type); 722d3b18ad3SJohn Fastabend if (err < 0) { 723d3b18ad3SJohn Fastabend *copied -= sk_msg_free(sk, msg); 724d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 725d3b18ad3SJohn Fastabend goto out_err; 726d3b18ad3SJohn Fastabend } 727d3b18ad3SJohn Fastabend break; 728d3b18ad3SJohn Fastabend case __SK_REDIRECT: 729d3b18ad3SJohn Fastabend sk_redir = psock->sk_redir; 730d3b18ad3SJohn Fastabend memcpy(&msg_redir, msg, sizeof(*msg)); 731d3b18ad3SJohn Fastabend if (msg->apply_bytes < send) 732d3b18ad3SJohn Fastabend msg->apply_bytes = 0; 733d3b18ad3SJohn Fastabend else 734d3b18ad3SJohn Fastabend msg->apply_bytes -= send; 735d3b18ad3SJohn Fastabend sk_msg_return_zero(sk, msg, send); 736d3b18ad3SJohn Fastabend msg->sg.size -= send; 737d3b18ad3SJohn Fastabend release_sock(sk); 738d3b18ad3SJohn Fastabend err = tcp_bpf_sendmsg_redir(sk_redir, &msg_redir, send, flags); 739d3b18ad3SJohn Fastabend lock_sock(sk); 740d3b18ad3SJohn Fastabend if (err < 0) { 741d3b18ad3SJohn Fastabend *copied -= sk_msg_free_nocharge(sk, &msg_redir); 742d3b18ad3SJohn Fastabend msg->sg.size = 0; 743d3b18ad3SJohn Fastabend } 744d3b18ad3SJohn Fastabend if (msg->sg.size == 0) 745d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 746d3b18ad3SJohn Fastabend break; 747d3b18ad3SJohn Fastabend case __SK_DROP: 748d3b18ad3SJohn Fastabend default: 749d3b18ad3SJohn Fastabend sk_msg_free_partial(sk, msg, send); 750d3b18ad3SJohn Fastabend if (msg->apply_bytes < send) 751d3b18ad3SJohn Fastabend msg->apply_bytes = 0; 752d3b18ad3SJohn Fastabend else 753d3b18ad3SJohn Fastabend msg->apply_bytes -= send; 754d3b18ad3SJohn Fastabend if (msg->sg.size == 0) 755d3b18ad3SJohn Fastabend tls_free_open_rec(sk); 7567246d8edSJohn Fastabend *copied -= (send + delta); 757d3b18ad3SJohn Fastabend err = -EACCES; 758d3b18ad3SJohn Fastabend } 759a42055e8SVakul Garg 760d3b18ad3SJohn Fastabend if (likely(!err)) { 761d3b18ad3SJohn Fastabend bool reset_eval = !ctx->open_rec; 762d3b18ad3SJohn Fastabend 763d3b18ad3SJohn Fastabend rec = ctx->open_rec; 764d3b18ad3SJohn Fastabend if (rec) { 765d3b18ad3SJohn Fastabend msg = &rec->msg_plaintext; 766d3b18ad3SJohn Fastabend if (!msg->apply_bytes) 767d3b18ad3SJohn Fastabend reset_eval = true; 768d3b18ad3SJohn Fastabend } 769d3b18ad3SJohn Fastabend if (reset_eval) { 770d3b18ad3SJohn Fastabend psock->eval = __SK_NONE; 771d3b18ad3SJohn Fastabend if (psock->sk_redir) { 772d3b18ad3SJohn Fastabend sock_put(psock->sk_redir); 773d3b18ad3SJohn Fastabend psock->sk_redir = NULL; 774d3b18ad3SJohn Fastabend } 775d3b18ad3SJohn Fastabend } 776d3b18ad3SJohn Fastabend if (rec) 777d3b18ad3SJohn Fastabend goto more_data; 778d3b18ad3SJohn Fastabend } 779d3b18ad3SJohn Fastabend out_err: 780d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 781d3b18ad3SJohn Fastabend return err; 782d3b18ad3SJohn Fastabend } 783d3b18ad3SJohn Fastabend 784d3b18ad3SJohn Fastabend static int tls_sw_push_pending_record(struct sock *sk, int flags) 785d3b18ad3SJohn Fastabend { 786d3b18ad3SJohn Fastabend struct tls_context *tls_ctx = tls_get_ctx(sk); 787d3b18ad3SJohn Fastabend struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 788d3b18ad3SJohn Fastabend struct tls_rec *rec = ctx->open_rec; 789d3b18ad3SJohn Fastabend struct sk_msg *msg_pl; 790d3b18ad3SJohn Fastabend size_t copied; 791d3b18ad3SJohn Fastabend 792a42055e8SVakul Garg if (!rec) 793d3b18ad3SJohn Fastabend return 0; 794a42055e8SVakul Garg 795d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 796d3b18ad3SJohn Fastabend copied = msg_pl->sg.size; 797d3b18ad3SJohn Fastabend if (!copied) 798d3b18ad3SJohn Fastabend return 0; 799a42055e8SVakul Garg 800d3b18ad3SJohn Fastabend return bpf_exec_tx_verdict(msg_pl, sk, true, TLS_RECORD_TYPE_DATA, 801d3b18ad3SJohn Fastabend &copied, flags); 802a42055e8SVakul Garg } 803a42055e8SVakul Garg 804a42055e8SVakul Garg int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 805a42055e8SVakul Garg { 8063c4d7559SDave Watson long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 807a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 808a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 809a42055e8SVakul Garg struct crypto_tfm *tfm = crypto_aead_tfm(ctx->aead_send); 810a42055e8SVakul Garg bool async_capable = tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; 811a42055e8SVakul Garg unsigned char record_type = TLS_RECORD_TYPE_DATA; 81200e23707SDavid Howells bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); 8133c4d7559SDave Watson bool eor = !(msg->msg_flags & MSG_MORE); 8143c4d7559SDave Watson size_t try_to_copy, copied = 0; 815d829e9c4SDaniel Borkmann struct sk_msg *msg_pl, *msg_en; 816a42055e8SVakul Garg struct tls_rec *rec; 817a42055e8SVakul Garg int required_size; 818a42055e8SVakul Garg int num_async = 0; 8193c4d7559SDave Watson bool full_record; 820a42055e8SVakul Garg int record_room; 821a42055e8SVakul Garg int num_zc = 0; 8223c4d7559SDave Watson int orig_size; 8234128c0cfSVakul Garg int ret = 0; 8243c4d7559SDave Watson 8253c4d7559SDave Watson if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 8263c4d7559SDave Watson return -ENOTSUPP; 8273c4d7559SDave Watson 8283c4d7559SDave Watson lock_sock(sk); 8293c4d7559SDave Watson 830a42055e8SVakul Garg /* Wait till there is any pending write on socket */ 831a42055e8SVakul Garg if (unlikely(sk->sk_write_pending)) { 832a42055e8SVakul Garg ret = wait_on_pending_writer(sk, &timeo); 833a42055e8SVakul Garg if (unlikely(ret)) 8343c4d7559SDave Watson goto send_end; 835a42055e8SVakul Garg } 8363c4d7559SDave Watson 8373c4d7559SDave Watson if (unlikely(msg->msg_controllen)) { 8383c4d7559SDave Watson ret = tls_proccess_cmsg(sk, msg, &record_type); 839a42055e8SVakul Garg if (ret) { 840a42055e8SVakul Garg if (ret == -EINPROGRESS) 841a42055e8SVakul Garg num_async++; 842a42055e8SVakul Garg else if (ret != -EAGAIN) 8433c4d7559SDave Watson goto send_end; 8443c4d7559SDave Watson } 845a42055e8SVakul Garg } 8463c4d7559SDave Watson 8473c4d7559SDave Watson while (msg_data_left(msg)) { 8483c4d7559SDave Watson if (sk->sk_err) { 84930be8f8dSr.hering@avm.de ret = -sk->sk_err; 8503c4d7559SDave Watson goto send_end; 8513c4d7559SDave Watson } 8523c4d7559SDave Watson 853d3b18ad3SJohn Fastabend if (ctx->open_rec) 854d3b18ad3SJohn Fastabend rec = ctx->open_rec; 855d3b18ad3SJohn Fastabend else 856d3b18ad3SJohn Fastabend rec = ctx->open_rec = tls_get_rec(sk); 857a42055e8SVakul Garg if (!rec) { 858a42055e8SVakul Garg ret = -ENOMEM; 859a42055e8SVakul Garg goto send_end; 860a42055e8SVakul Garg } 861a42055e8SVakul Garg 862d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 863d829e9c4SDaniel Borkmann msg_en = &rec->msg_encrypted; 864d829e9c4SDaniel Borkmann 865d829e9c4SDaniel Borkmann orig_size = msg_pl->sg.size; 8663c4d7559SDave Watson full_record = false; 8673c4d7559SDave Watson try_to_copy = msg_data_left(msg); 868d829e9c4SDaniel Borkmann record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size; 8693c4d7559SDave Watson if (try_to_copy >= record_room) { 8703c4d7559SDave Watson try_to_copy = record_room; 8713c4d7559SDave Watson full_record = true; 8723c4d7559SDave Watson } 8733c4d7559SDave Watson 874d829e9c4SDaniel Borkmann required_size = msg_pl->sg.size + try_to_copy + 875dbe42559SDave Watson tls_ctx->tx.overhead_size; 8763c4d7559SDave Watson 8773c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 8783c4d7559SDave Watson goto wait_for_sndbuf; 879a42055e8SVakul Garg 8803c4d7559SDave Watson alloc_encrypted: 881d829e9c4SDaniel Borkmann ret = tls_alloc_encrypted_msg(sk, required_size); 8823c4d7559SDave Watson if (ret) { 8833c4d7559SDave Watson if (ret != -ENOSPC) 8843c4d7559SDave Watson goto wait_for_memory; 8853c4d7559SDave Watson 8863c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 8873c4d7559SDave Watson * actually allocated. The difference is due 8883c4d7559SDave Watson * to max sg elements limit 8893c4d7559SDave Watson */ 890d829e9c4SDaniel Borkmann try_to_copy -= required_size - msg_en->sg.size; 8913c4d7559SDave Watson full_record = true; 8923c4d7559SDave Watson } 893a42055e8SVakul Garg 894a42055e8SVakul Garg if (!is_kvec && (full_record || eor) && !async_capable) { 895d3b18ad3SJohn Fastabend u32 first = msg_pl->sg.end; 896d3b18ad3SJohn Fastabend 897d829e9c4SDaniel Borkmann ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter, 898d829e9c4SDaniel Borkmann msg_pl, try_to_copy); 8993c4d7559SDave Watson if (ret) 9003c4d7559SDave Watson goto fallback_to_reg_send; 9013c4d7559SDave Watson 9024e6d4720SVakul Garg rec->inplace_crypto = 0; 9034e6d4720SVakul Garg 904a42055e8SVakul Garg num_zc++; 9053c4d7559SDave Watson copied += try_to_copy; 906d3b18ad3SJohn Fastabend 907d3b18ad3SJohn Fastabend sk_msg_sg_copy_set(msg_pl, first); 908d3b18ad3SJohn Fastabend ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 909d3b18ad3SJohn Fastabend record_type, &copied, 910d3b18ad3SJohn Fastabend msg->msg_flags); 911a42055e8SVakul Garg if (ret) { 912a42055e8SVakul Garg if (ret == -EINPROGRESS) 913a42055e8SVakul Garg num_async++; 914d3b18ad3SJohn Fastabend else if (ret == -ENOMEM) 915d3b18ad3SJohn Fastabend goto wait_for_memory; 916d3b18ad3SJohn Fastabend else if (ret == -ENOSPC) 917d3b18ad3SJohn Fastabend goto rollback_iter; 918a42055e8SVakul Garg else if (ret != -EAGAIN) 9193c4d7559SDave Watson goto send_end; 920a42055e8SVakul Garg } 9215a3611efSDoron Roberts-Kedes continue; 922d3b18ad3SJohn Fastabend rollback_iter: 923d3b18ad3SJohn Fastabend copied -= try_to_copy; 924d3b18ad3SJohn Fastabend sk_msg_sg_copy_clear(msg_pl, first); 925d3b18ad3SJohn Fastabend iov_iter_revert(&msg->msg_iter, 926d3b18ad3SJohn Fastabend msg_pl->sg.size - orig_size); 9273c4d7559SDave Watson fallback_to_reg_send: 928d829e9c4SDaniel Borkmann sk_msg_trim(sk, msg_pl, orig_size); 9293c4d7559SDave Watson } 9303c4d7559SDave Watson 931d829e9c4SDaniel Borkmann required_size = msg_pl->sg.size + try_to_copy; 9324e6d4720SVakul Garg 933d829e9c4SDaniel Borkmann ret = tls_clone_plaintext_msg(sk, required_size); 9343c4d7559SDave Watson if (ret) { 9353c4d7559SDave Watson if (ret != -ENOSPC) 9364e6d4720SVakul Garg goto send_end; 9373c4d7559SDave Watson 9383c4d7559SDave Watson /* Adjust try_to_copy according to the amount that was 9393c4d7559SDave Watson * actually allocated. The difference is due 9403c4d7559SDave Watson * to max sg elements limit 9413c4d7559SDave Watson */ 942d829e9c4SDaniel Borkmann try_to_copy -= required_size - msg_pl->sg.size; 9433c4d7559SDave Watson full_record = true; 944d829e9c4SDaniel Borkmann sk_msg_trim(sk, msg_en, msg_pl->sg.size + 945dbe42559SDave Watson tls_ctx->tx.overhead_size); 9463c4d7559SDave Watson } 9473c4d7559SDave Watson 94865a10e28SVakul Garg if (try_to_copy) { 94965a10e28SVakul Garg ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, 95065a10e28SVakul Garg msg_pl, try_to_copy); 951d829e9c4SDaniel Borkmann if (ret < 0) 9523c4d7559SDave Watson goto trim_sgl; 95365a10e28SVakul Garg } 9543c4d7559SDave Watson 955d829e9c4SDaniel Borkmann /* Open records defined only if successfully copied, otherwise 956d829e9c4SDaniel Borkmann * we would trim the sg but not reset the open record frags. 957d829e9c4SDaniel Borkmann */ 958d829e9c4SDaniel Borkmann tls_ctx->pending_open_record_frags = true; 9593c4d7559SDave Watson copied += try_to_copy; 9603c4d7559SDave Watson if (full_record || eor) { 961d3b18ad3SJohn Fastabend ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 962d3b18ad3SJohn Fastabend record_type, &copied, 963d3b18ad3SJohn Fastabend msg->msg_flags); 9643c4d7559SDave Watson if (ret) { 965a42055e8SVakul Garg if (ret == -EINPROGRESS) 966a42055e8SVakul Garg num_async++; 967d3b18ad3SJohn Fastabend else if (ret == -ENOMEM) 968d3b18ad3SJohn Fastabend goto wait_for_memory; 969d3b18ad3SJohn Fastabend else if (ret != -EAGAIN) { 970d3b18ad3SJohn Fastabend if (ret == -ENOSPC) 971d3b18ad3SJohn Fastabend ret = 0; 9723c4d7559SDave Watson goto send_end; 9733c4d7559SDave Watson } 9743c4d7559SDave Watson } 975d3b18ad3SJohn Fastabend } 9763c4d7559SDave Watson 9773c4d7559SDave Watson continue; 9783c4d7559SDave Watson 9793c4d7559SDave Watson wait_for_sndbuf: 9803c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 9813c4d7559SDave Watson wait_for_memory: 9823c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 9833c4d7559SDave Watson if (ret) { 9843c4d7559SDave Watson trim_sgl: 985d829e9c4SDaniel Borkmann tls_trim_both_msgs(sk, orig_size); 9863c4d7559SDave Watson goto send_end; 9873c4d7559SDave Watson } 9883c4d7559SDave Watson 989d829e9c4SDaniel Borkmann if (msg_en->sg.size < required_size) 9903c4d7559SDave Watson goto alloc_encrypted; 9913c4d7559SDave Watson } 9923c4d7559SDave Watson 993a42055e8SVakul Garg if (!num_async) { 994a42055e8SVakul Garg goto send_end; 995a42055e8SVakul Garg } else if (num_zc) { 996a42055e8SVakul Garg /* Wait for pending encryptions to get completed */ 997a42055e8SVakul Garg smp_store_mb(ctx->async_notify, true); 998a42055e8SVakul Garg 999a42055e8SVakul Garg if (atomic_read(&ctx->encrypt_pending)) 1000a42055e8SVakul Garg crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 1001a42055e8SVakul Garg else 1002a42055e8SVakul Garg reinit_completion(&ctx->async_wait.completion); 1003a42055e8SVakul Garg 1004a42055e8SVakul Garg WRITE_ONCE(ctx->async_notify, false); 1005a42055e8SVakul Garg 1006a42055e8SVakul Garg if (ctx->async_wait.err) { 1007a42055e8SVakul Garg ret = ctx->async_wait.err; 1008a42055e8SVakul Garg copied = 0; 1009a42055e8SVakul Garg } 1010a42055e8SVakul Garg } 1011a42055e8SVakul Garg 1012a42055e8SVakul Garg /* Transmit if any encryptions have completed */ 1013a42055e8SVakul Garg if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 1014a42055e8SVakul Garg cancel_delayed_work(&ctx->tx_work.work); 1015a42055e8SVakul Garg tls_tx_records(sk, msg->msg_flags); 1016a42055e8SVakul Garg } 1017a42055e8SVakul Garg 10183c4d7559SDave Watson send_end: 10193c4d7559SDave Watson ret = sk_stream_error(sk, msg->msg_flags, ret); 10203c4d7559SDave Watson 10213c4d7559SDave Watson release_sock(sk); 10223c4d7559SDave Watson return copied ? copied : ret; 10233c4d7559SDave Watson } 10243c4d7559SDave Watson 10250608c69cSJohn Fastabend int tls_sw_do_sendpage(struct sock *sk, struct page *page, 10263c4d7559SDave Watson int offset, size_t size, int flags) 10273c4d7559SDave Watson { 1028a42055e8SVakul Garg long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 10293c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1030f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 10313c4d7559SDave Watson unsigned char record_type = TLS_RECORD_TYPE_DATA; 1032d829e9c4SDaniel Borkmann struct sk_msg *msg_pl; 1033a42055e8SVakul Garg struct tls_rec *rec; 1034a42055e8SVakul Garg int num_async = 0; 1035d3b18ad3SJohn Fastabend size_t copied = 0; 10363c4d7559SDave Watson bool full_record; 10373c4d7559SDave Watson int record_room; 10384128c0cfSVakul Garg int ret = 0; 1039a42055e8SVakul Garg bool eor; 10403c4d7559SDave Watson 10413c4d7559SDave Watson eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST)); 10423c4d7559SDave Watson sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 10433c4d7559SDave Watson 1044a42055e8SVakul Garg /* Wait till there is any pending write on socket */ 1045a42055e8SVakul Garg if (unlikely(sk->sk_write_pending)) { 1046a42055e8SVakul Garg ret = wait_on_pending_writer(sk, &timeo); 1047a42055e8SVakul Garg if (unlikely(ret)) 10483c4d7559SDave Watson goto sendpage_end; 1049a42055e8SVakul Garg } 10503c4d7559SDave Watson 10513c4d7559SDave Watson /* Call the sk_stream functions to manage the sndbuf mem. */ 10523c4d7559SDave Watson while (size > 0) { 10533c4d7559SDave Watson size_t copy, required_size; 10543c4d7559SDave Watson 10553c4d7559SDave Watson if (sk->sk_err) { 105630be8f8dSr.hering@avm.de ret = -sk->sk_err; 10573c4d7559SDave Watson goto sendpage_end; 10583c4d7559SDave Watson } 10593c4d7559SDave Watson 1060d3b18ad3SJohn Fastabend if (ctx->open_rec) 1061d3b18ad3SJohn Fastabend rec = ctx->open_rec; 1062d3b18ad3SJohn Fastabend else 1063d3b18ad3SJohn Fastabend rec = ctx->open_rec = tls_get_rec(sk); 1064a42055e8SVakul Garg if (!rec) { 1065a42055e8SVakul Garg ret = -ENOMEM; 1066a42055e8SVakul Garg goto sendpage_end; 1067a42055e8SVakul Garg } 1068a42055e8SVakul Garg 1069d829e9c4SDaniel Borkmann msg_pl = &rec->msg_plaintext; 1070d829e9c4SDaniel Borkmann 10713c4d7559SDave Watson full_record = false; 1072d829e9c4SDaniel Borkmann record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size; 1073d3b18ad3SJohn Fastabend copied = 0; 10743c4d7559SDave Watson copy = size; 10753c4d7559SDave Watson if (copy >= record_room) { 10763c4d7559SDave Watson copy = record_room; 10773c4d7559SDave Watson full_record = true; 10783c4d7559SDave Watson } 1079d829e9c4SDaniel Borkmann 1080d829e9c4SDaniel Borkmann required_size = msg_pl->sg.size + copy + 1081dbe42559SDave Watson tls_ctx->tx.overhead_size; 10823c4d7559SDave Watson 10833c4d7559SDave Watson if (!sk_stream_memory_free(sk)) 10843c4d7559SDave Watson goto wait_for_sndbuf; 10853c4d7559SDave Watson alloc_payload: 1086d829e9c4SDaniel Borkmann ret = tls_alloc_encrypted_msg(sk, required_size); 10873c4d7559SDave Watson if (ret) { 10883c4d7559SDave Watson if (ret != -ENOSPC) 10893c4d7559SDave Watson goto wait_for_memory; 10903c4d7559SDave Watson 10913c4d7559SDave Watson /* Adjust copy according to the amount that was 10923c4d7559SDave Watson * actually allocated. The difference is due 10933c4d7559SDave Watson * to max sg elements limit 10943c4d7559SDave Watson */ 1095d829e9c4SDaniel Borkmann copy -= required_size - msg_pl->sg.size; 10963c4d7559SDave Watson full_record = true; 10973c4d7559SDave Watson } 10983c4d7559SDave Watson 1099d829e9c4SDaniel Borkmann sk_msg_page_add(msg_pl, page, copy, offset); 11003c4d7559SDave Watson sk_mem_charge(sk, copy); 1101d829e9c4SDaniel Borkmann 11023c4d7559SDave Watson offset += copy; 11033c4d7559SDave Watson size -= copy; 1104d3b18ad3SJohn Fastabend copied += copy; 11053c4d7559SDave Watson 1106d829e9c4SDaniel Borkmann tls_ctx->pending_open_record_frags = true; 1107d829e9c4SDaniel Borkmann if (full_record || eor || sk_msg_full(msg_pl)) { 11084e6d4720SVakul Garg rec->inplace_crypto = 0; 1109d3b18ad3SJohn Fastabend ret = bpf_exec_tx_verdict(msg_pl, sk, full_record, 1110d3b18ad3SJohn Fastabend record_type, &copied, flags); 11113c4d7559SDave Watson if (ret) { 1112a42055e8SVakul Garg if (ret == -EINPROGRESS) 1113a42055e8SVakul Garg num_async++; 1114d3b18ad3SJohn Fastabend else if (ret == -ENOMEM) 1115d3b18ad3SJohn Fastabend goto wait_for_memory; 1116d3b18ad3SJohn Fastabend else if (ret != -EAGAIN) { 1117d3b18ad3SJohn Fastabend if (ret == -ENOSPC) 1118d3b18ad3SJohn Fastabend ret = 0; 11193c4d7559SDave Watson goto sendpage_end; 11203c4d7559SDave Watson } 11213c4d7559SDave Watson } 1122d3b18ad3SJohn Fastabend } 11233c4d7559SDave Watson continue; 11243c4d7559SDave Watson wait_for_sndbuf: 11253c4d7559SDave Watson set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 11263c4d7559SDave Watson wait_for_memory: 11273c4d7559SDave Watson ret = sk_stream_wait_memory(sk, &timeo); 11283c4d7559SDave Watson if (ret) { 1129d829e9c4SDaniel Borkmann tls_trim_both_msgs(sk, msg_pl->sg.size); 11303c4d7559SDave Watson goto sendpage_end; 11313c4d7559SDave Watson } 11323c4d7559SDave Watson 11333c4d7559SDave Watson goto alloc_payload; 11343c4d7559SDave Watson } 11353c4d7559SDave Watson 1136a42055e8SVakul Garg if (num_async) { 1137a42055e8SVakul Garg /* Transmit if any encryptions have completed */ 1138a42055e8SVakul Garg if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) { 1139a42055e8SVakul Garg cancel_delayed_work(&ctx->tx_work.work); 1140a42055e8SVakul Garg tls_tx_records(sk, flags); 1141a42055e8SVakul Garg } 1142a42055e8SVakul Garg } 11433c4d7559SDave Watson sendpage_end: 11443c4d7559SDave Watson ret = sk_stream_error(sk, flags, ret); 1145d3b18ad3SJohn Fastabend return copied ? copied : ret; 11463c4d7559SDave Watson } 11473c4d7559SDave Watson 11480608c69cSJohn Fastabend int tls_sw_sendpage_locked(struct sock *sk, struct page *page, 11490608c69cSJohn Fastabend int offset, size_t size, int flags) 11500608c69cSJohn Fastabend { 11510608c69cSJohn Fastabend if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 11520608c69cSJohn Fastabend MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY)) 11530608c69cSJohn Fastabend return -ENOTSUPP; 11540608c69cSJohn Fastabend 11550608c69cSJohn Fastabend return tls_sw_do_sendpage(sk, page, offset, size, flags); 11560608c69cSJohn Fastabend } 11570608c69cSJohn Fastabend 11580608c69cSJohn Fastabend int tls_sw_sendpage(struct sock *sk, struct page *page, 11590608c69cSJohn Fastabend int offset, size_t size, int flags) 11600608c69cSJohn Fastabend { 11610608c69cSJohn Fastabend int ret; 11620608c69cSJohn Fastabend 11630608c69cSJohn Fastabend if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | 11640608c69cSJohn Fastabend MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY)) 11650608c69cSJohn Fastabend return -ENOTSUPP; 11660608c69cSJohn Fastabend 11670608c69cSJohn Fastabend lock_sock(sk); 11680608c69cSJohn Fastabend ret = tls_sw_do_sendpage(sk, page, offset, size, flags); 11690608c69cSJohn Fastabend release_sock(sk); 11700608c69cSJohn Fastabend return ret; 11710608c69cSJohn Fastabend } 11720608c69cSJohn Fastabend 1173d3b18ad3SJohn Fastabend static struct sk_buff *tls_wait_data(struct sock *sk, struct sk_psock *psock, 1174d3b18ad3SJohn Fastabend int flags, long timeo, int *err) 1175c46234ebSDave Watson { 1176c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1177f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1178c46234ebSDave Watson struct sk_buff *skb; 1179c46234ebSDave Watson DEFINE_WAIT_FUNC(wait, woken_wake_function); 1180c46234ebSDave Watson 1181d3b18ad3SJohn Fastabend while (!(skb = ctx->recv_pkt) && sk_psock_queue_empty(psock)) { 1182c46234ebSDave Watson if (sk->sk_err) { 1183c46234ebSDave Watson *err = sock_error(sk); 1184c46234ebSDave Watson return NULL; 1185c46234ebSDave Watson } 1186c46234ebSDave Watson 1187fcf4793eSDoron Roberts-Kedes if (sk->sk_shutdown & RCV_SHUTDOWN) 1188fcf4793eSDoron Roberts-Kedes return NULL; 1189fcf4793eSDoron Roberts-Kedes 1190c46234ebSDave Watson if (sock_flag(sk, SOCK_DONE)) 1191c46234ebSDave Watson return NULL; 1192c46234ebSDave Watson 1193c46234ebSDave Watson if ((flags & MSG_DONTWAIT) || !timeo) { 1194c46234ebSDave Watson *err = -EAGAIN; 1195c46234ebSDave Watson return NULL; 1196c46234ebSDave Watson } 1197c46234ebSDave Watson 1198c46234ebSDave Watson add_wait_queue(sk_sleep(sk), &wait); 1199c46234ebSDave Watson sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1200d3b18ad3SJohn Fastabend sk_wait_event(sk, &timeo, 1201d3b18ad3SJohn Fastabend ctx->recv_pkt != skb || 1202d3b18ad3SJohn Fastabend !sk_psock_queue_empty(psock), 1203d3b18ad3SJohn Fastabend &wait); 1204c46234ebSDave Watson sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1205c46234ebSDave Watson remove_wait_queue(sk_sleep(sk), &wait); 1206c46234ebSDave Watson 1207c46234ebSDave Watson /* Handle signals */ 1208c46234ebSDave Watson if (signal_pending(current)) { 1209c46234ebSDave Watson *err = sock_intr_errno(timeo); 1210c46234ebSDave Watson return NULL; 1211c46234ebSDave Watson } 1212c46234ebSDave Watson } 1213c46234ebSDave Watson 1214c46234ebSDave Watson return skb; 1215c46234ebSDave Watson } 1216c46234ebSDave Watson 1217d829e9c4SDaniel Borkmann static int tls_setup_from_iter(struct sock *sk, struct iov_iter *from, 1218d829e9c4SDaniel Borkmann int length, int *pages_used, 1219d829e9c4SDaniel Borkmann unsigned int *size_used, 1220d829e9c4SDaniel Borkmann struct scatterlist *to, 1221d829e9c4SDaniel Borkmann int to_max_pages) 1222d829e9c4SDaniel Borkmann { 1223d829e9c4SDaniel Borkmann int rc = 0, i = 0, num_elem = *pages_used, maxpages; 1224d829e9c4SDaniel Borkmann struct page *pages[MAX_SKB_FRAGS]; 1225d829e9c4SDaniel Borkmann unsigned int size = *size_used; 1226d829e9c4SDaniel Borkmann ssize_t copied, use; 1227d829e9c4SDaniel Borkmann size_t offset; 1228d829e9c4SDaniel Borkmann 1229d829e9c4SDaniel Borkmann while (length > 0) { 1230d829e9c4SDaniel Borkmann i = 0; 1231d829e9c4SDaniel Borkmann maxpages = to_max_pages - num_elem; 1232d829e9c4SDaniel Borkmann if (maxpages == 0) { 1233d829e9c4SDaniel Borkmann rc = -EFAULT; 1234d829e9c4SDaniel Borkmann goto out; 1235d829e9c4SDaniel Borkmann } 1236d829e9c4SDaniel Borkmann copied = iov_iter_get_pages(from, pages, 1237d829e9c4SDaniel Borkmann length, 1238d829e9c4SDaniel Borkmann maxpages, &offset); 1239d829e9c4SDaniel Borkmann if (copied <= 0) { 1240d829e9c4SDaniel Borkmann rc = -EFAULT; 1241d829e9c4SDaniel Borkmann goto out; 1242d829e9c4SDaniel Borkmann } 1243d829e9c4SDaniel Borkmann 1244d829e9c4SDaniel Borkmann iov_iter_advance(from, copied); 1245d829e9c4SDaniel Borkmann 1246d829e9c4SDaniel Borkmann length -= copied; 1247d829e9c4SDaniel Borkmann size += copied; 1248d829e9c4SDaniel Borkmann while (copied) { 1249d829e9c4SDaniel Borkmann use = min_t(int, copied, PAGE_SIZE - offset); 1250d829e9c4SDaniel Borkmann 1251d829e9c4SDaniel Borkmann sg_set_page(&to[num_elem], 1252d829e9c4SDaniel Borkmann pages[i], use, offset); 1253d829e9c4SDaniel Borkmann sg_unmark_end(&to[num_elem]); 1254d829e9c4SDaniel Borkmann /* We do not uncharge memory from this API */ 1255d829e9c4SDaniel Borkmann 1256d829e9c4SDaniel Borkmann offset = 0; 1257d829e9c4SDaniel Borkmann copied -= use; 1258d829e9c4SDaniel Borkmann 1259d829e9c4SDaniel Borkmann i++; 1260d829e9c4SDaniel Borkmann num_elem++; 1261d829e9c4SDaniel Borkmann } 1262d829e9c4SDaniel Borkmann } 1263d829e9c4SDaniel Borkmann /* Mark the end in the last sg entry if newly added */ 1264d829e9c4SDaniel Borkmann if (num_elem > *pages_used) 1265d829e9c4SDaniel Borkmann sg_mark_end(&to[num_elem - 1]); 1266d829e9c4SDaniel Borkmann out: 1267d829e9c4SDaniel Borkmann if (rc) 1268d829e9c4SDaniel Borkmann iov_iter_revert(from, size - *size_used); 1269d829e9c4SDaniel Borkmann *size_used = size; 1270d829e9c4SDaniel Borkmann *pages_used = num_elem; 1271d829e9c4SDaniel Borkmann 1272d829e9c4SDaniel Borkmann return rc; 1273d829e9c4SDaniel Borkmann } 1274d829e9c4SDaniel Borkmann 12750b243d00SVakul Garg /* This function decrypts the input skb into either out_iov or in out_sg 12760b243d00SVakul Garg * or in skb buffers itself. The input parameter 'zc' indicates if 12770b243d00SVakul Garg * zero-copy mode needs to be tried or not. With zero-copy mode, either 12780b243d00SVakul Garg * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are 12790b243d00SVakul Garg * NULL, then the decryption happens inside skb buffers itself, i.e. 12800b243d00SVakul Garg * zero-copy gets disabled and 'zc' is updated. 12810b243d00SVakul Garg */ 12820b243d00SVakul Garg 12830b243d00SVakul Garg static int decrypt_internal(struct sock *sk, struct sk_buff *skb, 12840b243d00SVakul Garg struct iov_iter *out_iov, 12850b243d00SVakul Garg struct scatterlist *out_sg, 12860b243d00SVakul Garg int *chunk, bool *zc) 12870b243d00SVakul Garg { 12880b243d00SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 12890b243d00SVakul Garg struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 12900b243d00SVakul Garg struct strp_msg *rxm = strp_msg(skb); 12910b243d00SVakul Garg int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0; 12920b243d00SVakul Garg struct aead_request *aead_req; 12930b243d00SVakul Garg struct sk_buff *unused; 12940b243d00SVakul Garg u8 *aad, *iv, *mem = NULL; 12950b243d00SVakul Garg struct scatterlist *sgin = NULL; 12960b243d00SVakul Garg struct scatterlist *sgout = NULL; 12970b243d00SVakul Garg const int data_len = rxm->full_len - tls_ctx->rx.overhead_size; 12980b243d00SVakul Garg 12990b243d00SVakul Garg if (*zc && (out_iov || out_sg)) { 13000b243d00SVakul Garg if (out_iov) 13010b243d00SVakul Garg n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1; 13020b243d00SVakul Garg else 13030b243d00SVakul Garg n_sgout = sg_nents(out_sg); 13040927f71dSDoron Roberts-Kedes n_sgin = skb_nsg(skb, rxm->offset + tls_ctx->rx.prepend_size, 13050927f71dSDoron Roberts-Kedes rxm->full_len - tls_ctx->rx.prepend_size); 13060b243d00SVakul Garg } else { 13070b243d00SVakul Garg n_sgout = 0; 13080b243d00SVakul Garg *zc = false; 13090927f71dSDoron Roberts-Kedes n_sgin = skb_cow_data(skb, 0, &unused); 13100b243d00SVakul Garg } 13110b243d00SVakul Garg 13120b243d00SVakul Garg if (n_sgin < 1) 13130b243d00SVakul Garg return -EBADMSG; 13140b243d00SVakul Garg 13150b243d00SVakul Garg /* Increment to accommodate AAD */ 13160b243d00SVakul Garg n_sgin = n_sgin + 1; 13170b243d00SVakul Garg 13180b243d00SVakul Garg nsg = n_sgin + n_sgout; 13190b243d00SVakul Garg 13200b243d00SVakul Garg aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv); 13210b243d00SVakul Garg mem_size = aead_size + (nsg * sizeof(struct scatterlist)); 13220b243d00SVakul Garg mem_size = mem_size + TLS_AAD_SPACE_SIZE; 13230b243d00SVakul Garg mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv); 13240b243d00SVakul Garg 13250b243d00SVakul Garg /* Allocate a single block of memory which contains 13260b243d00SVakul Garg * aead_req || sgin[] || sgout[] || aad || iv. 13270b243d00SVakul Garg * This order achieves correct alignment for aead_req, sgin, sgout. 13280b243d00SVakul Garg */ 13290b243d00SVakul Garg mem = kmalloc(mem_size, sk->sk_allocation); 13300b243d00SVakul Garg if (!mem) 13310b243d00SVakul Garg return -ENOMEM; 13320b243d00SVakul Garg 13330b243d00SVakul Garg /* Segment the allocated memory */ 13340b243d00SVakul Garg aead_req = (struct aead_request *)mem; 13350b243d00SVakul Garg sgin = (struct scatterlist *)(mem + aead_size); 13360b243d00SVakul Garg sgout = sgin + n_sgin; 13370b243d00SVakul Garg aad = (u8 *)(sgout + n_sgout); 13380b243d00SVakul Garg iv = aad + TLS_AAD_SPACE_SIZE; 13390b243d00SVakul Garg 13400b243d00SVakul Garg /* Prepare IV */ 13410b243d00SVakul Garg err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE, 13420b243d00SVakul Garg iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 13430b243d00SVakul Garg tls_ctx->rx.iv_size); 13440b243d00SVakul Garg if (err < 0) { 13450b243d00SVakul Garg kfree(mem); 13460b243d00SVakul Garg return err; 13470b243d00SVakul Garg } 13480b243d00SVakul Garg memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 13490b243d00SVakul Garg 13500b243d00SVakul Garg /* Prepare AAD */ 13510b243d00SVakul Garg tls_make_aad(aad, rxm->full_len - tls_ctx->rx.overhead_size, 13520b243d00SVakul Garg tls_ctx->rx.rec_seq, tls_ctx->rx.rec_seq_size, 13530b243d00SVakul Garg ctx->control); 13540b243d00SVakul Garg 13550b243d00SVakul Garg /* Prepare sgin */ 13560b243d00SVakul Garg sg_init_table(sgin, n_sgin); 13570b243d00SVakul Garg sg_set_buf(&sgin[0], aad, TLS_AAD_SPACE_SIZE); 13580b243d00SVakul Garg err = skb_to_sgvec(skb, &sgin[1], 13590b243d00SVakul Garg rxm->offset + tls_ctx->rx.prepend_size, 13600b243d00SVakul Garg rxm->full_len - tls_ctx->rx.prepend_size); 13610b243d00SVakul Garg if (err < 0) { 13620b243d00SVakul Garg kfree(mem); 13630b243d00SVakul Garg return err; 13640b243d00SVakul Garg } 13650b243d00SVakul Garg 13660b243d00SVakul Garg if (n_sgout) { 13670b243d00SVakul Garg if (out_iov) { 13680b243d00SVakul Garg sg_init_table(sgout, n_sgout); 13690b243d00SVakul Garg sg_set_buf(&sgout[0], aad, TLS_AAD_SPACE_SIZE); 13700b243d00SVakul Garg 13710b243d00SVakul Garg *chunk = 0; 1372d829e9c4SDaniel Borkmann err = tls_setup_from_iter(sk, out_iov, data_len, 1373d829e9c4SDaniel Borkmann &pages, chunk, &sgout[1], 1374d829e9c4SDaniel Borkmann (n_sgout - 1)); 13750b243d00SVakul Garg if (err < 0) 13760b243d00SVakul Garg goto fallback_to_reg_recv; 13770b243d00SVakul Garg } else if (out_sg) { 13780b243d00SVakul Garg memcpy(sgout, out_sg, n_sgout * sizeof(*sgout)); 13790b243d00SVakul Garg } else { 13800b243d00SVakul Garg goto fallback_to_reg_recv; 13810b243d00SVakul Garg } 13820b243d00SVakul Garg } else { 13830b243d00SVakul Garg fallback_to_reg_recv: 13840b243d00SVakul Garg sgout = sgin; 13850b243d00SVakul Garg pages = 0; 13860b243d00SVakul Garg *chunk = 0; 13870b243d00SVakul Garg *zc = false; 13880b243d00SVakul Garg } 13890b243d00SVakul Garg 13900b243d00SVakul Garg /* Prepare and submit AEAD request */ 139194524d8fSVakul Garg err = tls_do_decryption(sk, skb, sgin, sgout, iv, 139294524d8fSVakul Garg data_len, aead_req, *zc); 139394524d8fSVakul Garg if (err == -EINPROGRESS) 139494524d8fSVakul Garg return err; 13950b243d00SVakul Garg 13960b243d00SVakul Garg /* Release the pages in case iov was mapped to pages */ 13970b243d00SVakul Garg for (; pages > 0; pages--) 13980b243d00SVakul Garg put_page(sg_page(&sgout[pages])); 13990b243d00SVakul Garg 14000b243d00SVakul Garg kfree(mem); 14010b243d00SVakul Garg return err; 14020b243d00SVakul Garg } 14030b243d00SVakul Garg 1404dafb67f3SBoris Pismenny static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, 14050b243d00SVakul Garg struct iov_iter *dest, int *chunk, bool *zc) 1406dafb67f3SBoris Pismenny { 1407dafb67f3SBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 1408dafb67f3SBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1409dafb67f3SBoris Pismenny struct strp_msg *rxm = strp_msg(skb); 1410dafb67f3SBoris Pismenny int err = 0; 1411dafb67f3SBoris Pismenny 14124799ac81SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 14134799ac81SBoris Pismenny err = tls_device_decrypted(sk, skb); 14144799ac81SBoris Pismenny if (err < 0) 14154799ac81SBoris Pismenny return err; 14164799ac81SBoris Pismenny #endif 14174799ac81SBoris Pismenny if (!ctx->decrypted) { 14180b243d00SVakul Garg err = decrypt_internal(sk, skb, dest, NULL, chunk, zc); 141994524d8fSVakul Garg if (err < 0) { 142094524d8fSVakul Garg if (err == -EINPROGRESS) 142194524d8fSVakul Garg tls_advance_record_sn(sk, &tls_ctx->rx); 142294524d8fSVakul Garg 1423dafb67f3SBoris Pismenny return err; 142494524d8fSVakul Garg } 14254799ac81SBoris Pismenny } else { 14264799ac81SBoris Pismenny *zc = false; 14274799ac81SBoris Pismenny } 1428dafb67f3SBoris Pismenny 1429dafb67f3SBoris Pismenny rxm->offset += tls_ctx->rx.prepend_size; 1430dafb67f3SBoris Pismenny rxm->full_len -= tls_ctx->rx.overhead_size; 1431dafb67f3SBoris Pismenny tls_advance_record_sn(sk, &tls_ctx->rx); 1432dafb67f3SBoris Pismenny ctx->decrypted = true; 1433dafb67f3SBoris Pismenny ctx->saved_data_ready(sk); 1434dafb67f3SBoris Pismenny 1435dafb67f3SBoris Pismenny return err; 1436dafb67f3SBoris Pismenny } 1437dafb67f3SBoris Pismenny 1438dafb67f3SBoris Pismenny int decrypt_skb(struct sock *sk, struct sk_buff *skb, 1439c46234ebSDave Watson struct scatterlist *sgout) 1440c46234ebSDave Watson { 14410b243d00SVakul Garg bool zc = true; 14420b243d00SVakul Garg int chunk; 1443c46234ebSDave Watson 14440b243d00SVakul Garg return decrypt_internal(sk, skb, NULL, sgout, &chunk, &zc); 1445c46234ebSDave Watson } 1446c46234ebSDave Watson 1447c46234ebSDave Watson static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb, 1448c46234ebSDave Watson unsigned int len) 1449c46234ebSDave Watson { 1450c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1451f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 145294524d8fSVakul Garg 145394524d8fSVakul Garg if (skb) { 1454c46234ebSDave Watson struct strp_msg *rxm = strp_msg(skb); 1455c46234ebSDave Watson 1456c46234ebSDave Watson if (len < rxm->full_len) { 1457c46234ebSDave Watson rxm->offset += len; 1458c46234ebSDave Watson rxm->full_len -= len; 1459c46234ebSDave Watson return false; 1460c46234ebSDave Watson } 146194524d8fSVakul Garg kfree_skb(skb); 146294524d8fSVakul Garg } 1463c46234ebSDave Watson 1464c46234ebSDave Watson /* Finished with message */ 1465c46234ebSDave Watson ctx->recv_pkt = NULL; 14667170e604SDoron Roberts-Kedes __strp_unpause(&ctx->strp); 1467c46234ebSDave Watson 1468c46234ebSDave Watson return true; 1469c46234ebSDave Watson } 1470c46234ebSDave Watson 1471c46234ebSDave Watson int tls_sw_recvmsg(struct sock *sk, 1472c46234ebSDave Watson struct msghdr *msg, 1473c46234ebSDave Watson size_t len, 1474c46234ebSDave Watson int nonblock, 1475c46234ebSDave Watson int flags, 1476c46234ebSDave Watson int *addr_len) 1477c46234ebSDave Watson { 1478c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1479f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1480d3b18ad3SJohn Fastabend struct sk_psock *psock; 1481c46234ebSDave Watson unsigned char control; 1482c46234ebSDave Watson struct strp_msg *rxm; 1483c46234ebSDave Watson struct sk_buff *skb; 1484c46234ebSDave Watson ssize_t copied = 0; 1485c46234ebSDave Watson bool cmsg = false; 148606030dbaSDaniel Borkmann int target, err = 0; 1487c46234ebSDave Watson long timeo; 148800e23707SDavid Howells bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); 148994524d8fSVakul Garg int num_async = 0; 1490c46234ebSDave Watson 1491c46234ebSDave Watson flags |= nonblock; 1492c46234ebSDave Watson 1493c46234ebSDave Watson if (unlikely(flags & MSG_ERRQUEUE)) 1494c46234ebSDave Watson return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR); 1495c46234ebSDave Watson 1496d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 1497c46234ebSDave Watson lock_sock(sk); 1498c46234ebSDave Watson 149906030dbaSDaniel Borkmann target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1500c46234ebSDave Watson timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 1501c46234ebSDave Watson do { 1502c46234ebSDave Watson bool zc = false; 150394524d8fSVakul Garg bool async = false; 1504c46234ebSDave Watson int chunk = 0; 1505c46234ebSDave Watson 1506d3b18ad3SJohn Fastabend skb = tls_wait_data(sk, psock, flags, timeo, &err); 1507d3b18ad3SJohn Fastabend if (!skb) { 1508d3b18ad3SJohn Fastabend if (psock) { 150902c558b2SJohn Fastabend int ret = __tcp_bpf_recvmsg(sk, psock, 151002c558b2SJohn Fastabend msg, len, flags); 1511d3b18ad3SJohn Fastabend 1512d3b18ad3SJohn Fastabend if (ret > 0) { 1513d3b18ad3SJohn Fastabend copied += ret; 1514d3b18ad3SJohn Fastabend len -= ret; 1515d3b18ad3SJohn Fastabend continue; 1516d3b18ad3SJohn Fastabend } 1517d3b18ad3SJohn Fastabend } 1518c46234ebSDave Watson goto recv_end; 1519d3b18ad3SJohn Fastabend } 1520c46234ebSDave Watson 1521c46234ebSDave Watson rxm = strp_msg(skb); 152294524d8fSVakul Garg 1523c46234ebSDave Watson if (!cmsg) { 1524c46234ebSDave Watson int cerr; 1525c46234ebSDave Watson 1526c46234ebSDave Watson cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, 1527c46234ebSDave Watson sizeof(ctx->control), &ctx->control); 1528c46234ebSDave Watson cmsg = true; 1529c46234ebSDave Watson control = ctx->control; 1530c46234ebSDave Watson if (ctx->control != TLS_RECORD_TYPE_DATA) { 1531c46234ebSDave Watson if (cerr || msg->msg_flags & MSG_CTRUNC) { 1532c46234ebSDave Watson err = -EIO; 1533c46234ebSDave Watson goto recv_end; 1534c46234ebSDave Watson } 1535c46234ebSDave Watson } 1536c46234ebSDave Watson } else if (control != ctx->control) { 1537c46234ebSDave Watson goto recv_end; 1538c46234ebSDave Watson } 1539c46234ebSDave Watson 1540c46234ebSDave Watson if (!ctx->decrypted) { 15410b243d00SVakul Garg int to_copy = rxm->full_len - tls_ctx->rx.overhead_size; 1542c46234ebSDave Watson 15430b243d00SVakul Garg if (!is_kvec && to_copy <= len && 15440b243d00SVakul Garg likely(!(flags & MSG_PEEK))) 1545c46234ebSDave Watson zc = true; 1546c46234ebSDave Watson 15470b243d00SVakul Garg err = decrypt_skb_update(sk, skb, &msg->msg_iter, 15480b243d00SVakul Garg &chunk, &zc); 154994524d8fSVakul Garg if (err < 0 && err != -EINPROGRESS) { 1550c46234ebSDave Watson tls_err_abort(sk, EBADMSG); 1551c46234ebSDave Watson goto recv_end; 1552c46234ebSDave Watson } 155394524d8fSVakul Garg 155494524d8fSVakul Garg if (err == -EINPROGRESS) { 155594524d8fSVakul Garg async = true; 155694524d8fSVakul Garg num_async++; 155794524d8fSVakul Garg goto pick_next_record; 155894524d8fSVakul Garg } 155994524d8fSVakul Garg 1560c46234ebSDave Watson ctx->decrypted = true; 1561c46234ebSDave Watson } 1562c46234ebSDave Watson 1563c46234ebSDave Watson if (!zc) { 1564c46234ebSDave Watson chunk = min_t(unsigned int, rxm->full_len, len); 156594524d8fSVakul Garg 1566c46234ebSDave Watson err = skb_copy_datagram_msg(skb, rxm->offset, msg, 1567c46234ebSDave Watson chunk); 1568c46234ebSDave Watson if (err < 0) 1569c46234ebSDave Watson goto recv_end; 1570c46234ebSDave Watson } 1571c46234ebSDave Watson 157294524d8fSVakul Garg pick_next_record: 1573c46234ebSDave Watson copied += chunk; 1574c46234ebSDave Watson len -= chunk; 1575c46234ebSDave Watson if (likely(!(flags & MSG_PEEK))) { 1576c46234ebSDave Watson u8 control = ctx->control; 1577c46234ebSDave Watson 157894524d8fSVakul Garg /* For async, drop current skb reference */ 157994524d8fSVakul Garg if (async) 158094524d8fSVakul Garg skb = NULL; 158194524d8fSVakul Garg 1582c46234ebSDave Watson if (tls_sw_advance_skb(sk, skb, chunk)) { 1583c46234ebSDave Watson /* Return full control message to 1584c46234ebSDave Watson * userspace before trying to parse 1585c46234ebSDave Watson * another message type 1586c46234ebSDave Watson */ 1587c46234ebSDave Watson msg->msg_flags |= MSG_EOR; 1588c46234ebSDave Watson if (control != TLS_RECORD_TYPE_DATA) 1589c46234ebSDave Watson goto recv_end; 159094524d8fSVakul Garg } else { 159194524d8fSVakul Garg break; 1592c46234ebSDave Watson } 159350c6b58aSDaniel Borkmann } else { 159450c6b58aSDaniel Borkmann /* MSG_PEEK right now cannot look beyond current skb 159550c6b58aSDaniel Borkmann * from strparser, meaning we cannot advance skb here 159650c6b58aSDaniel Borkmann * and thus unpause strparser since we'd loose original 159750c6b58aSDaniel Borkmann * one. 159850c6b58aSDaniel Borkmann */ 159950c6b58aSDaniel Borkmann break; 1600c46234ebSDave Watson } 160194524d8fSVakul Garg 160206030dbaSDaniel Borkmann /* If we have a new message from strparser, continue now. */ 160306030dbaSDaniel Borkmann if (copied >= target && !ctx->recv_pkt) 160406030dbaSDaniel Borkmann break; 1605c46234ebSDave Watson } while (len); 1606c46234ebSDave Watson 1607c46234ebSDave Watson recv_end: 160894524d8fSVakul Garg if (num_async) { 160994524d8fSVakul Garg /* Wait for all previously submitted records to be decrypted */ 161094524d8fSVakul Garg smp_store_mb(ctx->async_notify, true); 161194524d8fSVakul Garg if (atomic_read(&ctx->decrypt_pending)) { 161294524d8fSVakul Garg err = crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 161394524d8fSVakul Garg if (err) { 161494524d8fSVakul Garg /* one of async decrypt failed */ 161594524d8fSVakul Garg tls_err_abort(sk, err); 161694524d8fSVakul Garg copied = 0; 161794524d8fSVakul Garg } 161894524d8fSVakul Garg } else { 161994524d8fSVakul Garg reinit_completion(&ctx->async_wait.completion); 162094524d8fSVakul Garg } 162194524d8fSVakul Garg WRITE_ONCE(ctx->async_notify, false); 162294524d8fSVakul Garg } 162394524d8fSVakul Garg 1624c46234ebSDave Watson release_sock(sk); 1625d3b18ad3SJohn Fastabend if (psock) 1626d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 1627c46234ebSDave Watson return copied ? : err; 1628c46234ebSDave Watson } 1629c46234ebSDave Watson 1630c46234ebSDave Watson ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, 1631c46234ebSDave Watson struct pipe_inode_info *pipe, 1632c46234ebSDave Watson size_t len, unsigned int flags) 1633c46234ebSDave Watson { 1634c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sock->sk); 1635f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1636c46234ebSDave Watson struct strp_msg *rxm = NULL; 1637c46234ebSDave Watson struct sock *sk = sock->sk; 1638c46234ebSDave Watson struct sk_buff *skb; 1639c46234ebSDave Watson ssize_t copied = 0; 1640c46234ebSDave Watson int err = 0; 1641c46234ebSDave Watson long timeo; 1642c46234ebSDave Watson int chunk; 16430b243d00SVakul Garg bool zc = false; 1644c46234ebSDave Watson 1645c46234ebSDave Watson lock_sock(sk); 1646c46234ebSDave Watson 1647c46234ebSDave Watson timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 1648c46234ebSDave Watson 1649d3b18ad3SJohn Fastabend skb = tls_wait_data(sk, NULL, flags, timeo, &err); 1650c46234ebSDave Watson if (!skb) 1651c46234ebSDave Watson goto splice_read_end; 1652c46234ebSDave Watson 1653c46234ebSDave Watson /* splice does not support reading control messages */ 1654c46234ebSDave Watson if (ctx->control != TLS_RECORD_TYPE_DATA) { 1655c46234ebSDave Watson err = -ENOTSUPP; 1656c46234ebSDave Watson goto splice_read_end; 1657c46234ebSDave Watson } 1658c46234ebSDave Watson 1659c46234ebSDave Watson if (!ctx->decrypted) { 16600b243d00SVakul Garg err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc); 1661c46234ebSDave Watson 1662c46234ebSDave Watson if (err < 0) { 1663c46234ebSDave Watson tls_err_abort(sk, EBADMSG); 1664c46234ebSDave Watson goto splice_read_end; 1665c46234ebSDave Watson } 1666c46234ebSDave Watson ctx->decrypted = true; 1667c46234ebSDave Watson } 1668c46234ebSDave Watson rxm = strp_msg(skb); 1669c46234ebSDave Watson 1670c46234ebSDave Watson chunk = min_t(unsigned int, rxm->full_len, len); 1671c46234ebSDave Watson copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags); 1672c46234ebSDave Watson if (copied < 0) 1673c46234ebSDave Watson goto splice_read_end; 1674c46234ebSDave Watson 1675c46234ebSDave Watson if (likely(!(flags & MSG_PEEK))) 1676c46234ebSDave Watson tls_sw_advance_skb(sk, skb, copied); 1677c46234ebSDave Watson 1678c46234ebSDave Watson splice_read_end: 1679c46234ebSDave Watson release_sock(sk); 1680c46234ebSDave Watson return copied ? : err; 1681c46234ebSDave Watson } 1682c46234ebSDave Watson 1683924ad65eSJohn Fastabend bool tls_sw_stream_read(const struct sock *sk) 1684c46234ebSDave Watson { 1685c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1686f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1687d3b18ad3SJohn Fastabend bool ingress_empty = true; 1688d3b18ad3SJohn Fastabend struct sk_psock *psock; 1689c46234ebSDave Watson 1690d3b18ad3SJohn Fastabend rcu_read_lock(); 1691d3b18ad3SJohn Fastabend psock = sk_psock(sk); 1692d3b18ad3SJohn Fastabend if (psock) 1693d3b18ad3SJohn Fastabend ingress_empty = list_empty(&psock->ingress_msg); 1694d3b18ad3SJohn Fastabend rcu_read_unlock(); 1695c46234ebSDave Watson 1696d3b18ad3SJohn Fastabend return !ingress_empty || ctx->recv_pkt; 1697c46234ebSDave Watson } 1698c46234ebSDave Watson 1699c46234ebSDave Watson static int tls_read_size(struct strparser *strp, struct sk_buff *skb) 1700c46234ebSDave Watson { 1701c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 1702f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 17033463e51dSKees Cook char header[TLS_HEADER_SIZE + MAX_IV_SIZE]; 1704c46234ebSDave Watson struct strp_msg *rxm = strp_msg(skb); 1705c46234ebSDave Watson size_t cipher_overhead; 1706c46234ebSDave Watson size_t data_len = 0; 1707c46234ebSDave Watson int ret; 1708c46234ebSDave Watson 1709c46234ebSDave Watson /* Verify that we have a full TLS header, or wait for more data */ 1710c46234ebSDave Watson if (rxm->offset + tls_ctx->rx.prepend_size > skb->len) 1711c46234ebSDave Watson return 0; 1712c46234ebSDave Watson 17133463e51dSKees Cook /* Sanity-check size of on-stack buffer. */ 17143463e51dSKees Cook if (WARN_ON(tls_ctx->rx.prepend_size > sizeof(header))) { 17153463e51dSKees Cook ret = -EINVAL; 17163463e51dSKees Cook goto read_failure; 17173463e51dSKees Cook } 17183463e51dSKees Cook 1719c46234ebSDave Watson /* Linearize header to local buffer */ 1720c46234ebSDave Watson ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size); 1721c46234ebSDave Watson 1722c46234ebSDave Watson if (ret < 0) 1723c46234ebSDave Watson goto read_failure; 1724c46234ebSDave Watson 1725c46234ebSDave Watson ctx->control = header[0]; 1726c46234ebSDave Watson 1727c46234ebSDave Watson data_len = ((header[4] & 0xFF) | (header[3] << 8)); 1728c46234ebSDave Watson 1729c46234ebSDave Watson cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size; 1730c46234ebSDave Watson 1731c46234ebSDave Watson if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) { 1732c46234ebSDave Watson ret = -EMSGSIZE; 1733c46234ebSDave Watson goto read_failure; 1734c46234ebSDave Watson } 1735c46234ebSDave Watson if (data_len < cipher_overhead) { 1736c46234ebSDave Watson ret = -EBADMSG; 1737c46234ebSDave Watson goto read_failure; 1738c46234ebSDave Watson } 1739c46234ebSDave Watson 174086029d10SSabrina Dubroca if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.info.version) || 174186029d10SSabrina Dubroca header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.info.version)) { 1742c46234ebSDave Watson ret = -EINVAL; 1743c46234ebSDave Watson goto read_failure; 1744c46234ebSDave Watson } 1745c46234ebSDave Watson 17464799ac81SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 17474799ac81SBoris Pismenny handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset, 17484799ac81SBoris Pismenny *(u64*)tls_ctx->rx.rec_seq); 17494799ac81SBoris Pismenny #endif 1750c46234ebSDave Watson return data_len + TLS_HEADER_SIZE; 1751c46234ebSDave Watson 1752c46234ebSDave Watson read_failure: 1753c46234ebSDave Watson tls_err_abort(strp->sk, ret); 1754c46234ebSDave Watson 1755c46234ebSDave Watson return ret; 1756c46234ebSDave Watson } 1757c46234ebSDave Watson 1758c46234ebSDave Watson static void tls_queue(struct strparser *strp, struct sk_buff *skb) 1759c46234ebSDave Watson { 1760c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(strp->sk); 1761f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1762c46234ebSDave Watson 1763c46234ebSDave Watson ctx->decrypted = false; 1764c46234ebSDave Watson 1765c46234ebSDave Watson ctx->recv_pkt = skb; 1766c46234ebSDave Watson strp_pause(strp); 1767c46234ebSDave Watson 1768ad13acceSVakul Garg ctx->saved_data_ready(strp->sk); 1769c46234ebSDave Watson } 1770c46234ebSDave Watson 1771c46234ebSDave Watson static void tls_data_ready(struct sock *sk) 1772c46234ebSDave Watson { 1773c46234ebSDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1774f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1775d3b18ad3SJohn Fastabend struct sk_psock *psock; 1776c46234ebSDave Watson 1777c46234ebSDave Watson strp_data_ready(&ctx->strp); 1778d3b18ad3SJohn Fastabend 1779d3b18ad3SJohn Fastabend psock = sk_psock_get(sk); 1780d3b18ad3SJohn Fastabend if (psock && !list_empty(&psock->ingress_msg)) { 1781d3b18ad3SJohn Fastabend ctx->saved_data_ready(sk); 1782d3b18ad3SJohn Fastabend sk_psock_put(sk, psock); 1783d3b18ad3SJohn Fastabend } 1784c46234ebSDave Watson } 1785c46234ebSDave Watson 1786f66de3eeSBoris Pismenny void tls_sw_free_resources_tx(struct sock *sk) 17873c4d7559SDave Watson { 17883c4d7559SDave Watson struct tls_context *tls_ctx = tls_get_ctx(sk); 1789f66de3eeSBoris Pismenny struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1790a42055e8SVakul Garg struct tls_rec *rec, *tmp; 1791a42055e8SVakul Garg 1792a42055e8SVakul Garg /* Wait for any pending async encryptions to complete */ 1793a42055e8SVakul Garg smp_store_mb(ctx->async_notify, true); 1794a42055e8SVakul Garg if (atomic_read(&ctx->encrypt_pending)) 1795a42055e8SVakul Garg crypto_wait_req(-EINPROGRESS, &ctx->async_wait); 1796a42055e8SVakul Garg 179710231213SDave Watson release_sock(sk); 1798a42055e8SVakul Garg cancel_delayed_work_sync(&ctx->tx_work.work); 179910231213SDave Watson lock_sock(sk); 1800a42055e8SVakul Garg 1801a42055e8SVakul Garg /* Tx whatever records we can transmit and abandon the rest */ 1802a42055e8SVakul Garg tls_tx_records(sk, -1); 1803a42055e8SVakul Garg 18049932a29aSVakul Garg /* Free up un-sent records in tx_list. First, free 1805a42055e8SVakul Garg * the partially sent record if any at head of tx_list. 1806a42055e8SVakul Garg */ 1807a42055e8SVakul Garg if (tls_ctx->partially_sent_record) { 1808a42055e8SVakul Garg struct scatterlist *sg = tls_ctx->partially_sent_record; 1809a42055e8SVakul Garg 1810a42055e8SVakul Garg while (1) { 1811a42055e8SVakul Garg put_page(sg_page(sg)); 1812a42055e8SVakul Garg sk_mem_uncharge(sk, sg->length); 1813a42055e8SVakul Garg 1814a42055e8SVakul Garg if (sg_is_last(sg)) 1815a42055e8SVakul Garg break; 1816a42055e8SVakul Garg sg++; 1817a42055e8SVakul Garg } 1818a42055e8SVakul Garg 1819a42055e8SVakul Garg tls_ctx->partially_sent_record = NULL; 1820a42055e8SVakul Garg 18219932a29aSVakul Garg rec = list_first_entry(&ctx->tx_list, 1822a42055e8SVakul Garg struct tls_rec, list); 1823a42055e8SVakul Garg list_del(&rec->list); 1824d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 1825a42055e8SVakul Garg kfree(rec); 1826a42055e8SVakul Garg } 1827a42055e8SVakul Garg 18289932a29aSVakul Garg list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { 1829a42055e8SVakul Garg list_del(&rec->list); 1830d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_encrypted); 1831d829e9c4SDaniel Borkmann sk_msg_free(sk, &rec->msg_plaintext); 1832a42055e8SVakul Garg kfree(rec); 1833a42055e8SVakul Garg } 18343c4d7559SDave Watson 18353c4d7559SDave Watson crypto_free_aead(ctx->aead_send); 1836c774973eSVakul Garg tls_free_open_rec(sk); 1837f66de3eeSBoris Pismenny 1838f66de3eeSBoris Pismenny kfree(ctx); 1839f66de3eeSBoris Pismenny } 1840f66de3eeSBoris Pismenny 184139f56e1aSBoris Pismenny void tls_sw_release_resources_rx(struct sock *sk) 1842f66de3eeSBoris Pismenny { 1843f66de3eeSBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 1844f66de3eeSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 1845f66de3eeSBoris Pismenny 1846c46234ebSDave Watson if (ctx->aead_recv) { 1847c46234ebSDave Watson kfree_skb(ctx->recv_pkt); 1848c46234ebSDave Watson ctx->recv_pkt = NULL; 1849c46234ebSDave Watson crypto_free_aead(ctx->aead_recv); 1850c46234ebSDave Watson strp_stop(&ctx->strp); 1851c46234ebSDave Watson write_lock_bh(&sk->sk_callback_lock); 1852c46234ebSDave Watson sk->sk_data_ready = ctx->saved_data_ready; 1853c46234ebSDave Watson write_unlock_bh(&sk->sk_callback_lock); 1854c46234ebSDave Watson release_sock(sk); 1855c46234ebSDave Watson strp_done(&ctx->strp); 1856c46234ebSDave Watson lock_sock(sk); 1857c46234ebSDave Watson } 185839f56e1aSBoris Pismenny } 185939f56e1aSBoris Pismenny 186039f56e1aSBoris Pismenny void tls_sw_free_resources_rx(struct sock *sk) 186139f56e1aSBoris Pismenny { 186239f56e1aSBoris Pismenny struct tls_context *tls_ctx = tls_get_ctx(sk); 186339f56e1aSBoris Pismenny struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); 186439f56e1aSBoris Pismenny 186539f56e1aSBoris Pismenny tls_sw_release_resources_rx(sk); 18663c4d7559SDave Watson 18673c4d7559SDave Watson kfree(ctx); 18683c4d7559SDave Watson } 18693c4d7559SDave Watson 18709932a29aSVakul Garg /* The work handler to transmitt the encrypted records in tx_list */ 1871a42055e8SVakul Garg static void tx_work_handler(struct work_struct *work) 1872a42055e8SVakul Garg { 1873a42055e8SVakul Garg struct delayed_work *delayed_work = to_delayed_work(work); 1874a42055e8SVakul Garg struct tx_work *tx_work = container_of(delayed_work, 1875a42055e8SVakul Garg struct tx_work, work); 1876a42055e8SVakul Garg struct sock *sk = tx_work->sk; 1877a42055e8SVakul Garg struct tls_context *tls_ctx = tls_get_ctx(sk); 1878a42055e8SVakul Garg struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); 1879a42055e8SVakul Garg 1880a42055e8SVakul Garg if (!test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) 1881a42055e8SVakul Garg return; 1882a42055e8SVakul Garg 1883a42055e8SVakul Garg lock_sock(sk); 1884a42055e8SVakul Garg tls_tx_records(sk, -1); 1885a42055e8SVakul Garg release_sock(sk); 1886a42055e8SVakul Garg } 1887a42055e8SVakul Garg 1888c46234ebSDave Watson int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) 18893c4d7559SDave Watson { 18903c4d7559SDave Watson struct tls_crypto_info *crypto_info; 18913c4d7559SDave Watson struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; 1892f66de3eeSBoris Pismenny struct tls_sw_context_tx *sw_ctx_tx = NULL; 1893f66de3eeSBoris Pismenny struct tls_sw_context_rx *sw_ctx_rx = NULL; 1894c46234ebSDave Watson struct cipher_context *cctx; 1895c46234ebSDave Watson struct crypto_aead **aead; 1896c46234ebSDave Watson struct strp_callbacks cb; 18973c4d7559SDave Watson u16 nonce_size, tag_size, iv_size, rec_seq_size; 18983c4d7559SDave Watson char *iv, *rec_seq; 18993c4d7559SDave Watson int rc = 0; 19003c4d7559SDave Watson 19013c4d7559SDave Watson if (!ctx) { 19023c4d7559SDave Watson rc = -EINVAL; 19033c4d7559SDave Watson goto out; 19043c4d7559SDave Watson } 19053c4d7559SDave Watson 1906f66de3eeSBoris Pismenny if (tx) { 1907b190a587SBoris Pismenny if (!ctx->priv_ctx_tx) { 1908f66de3eeSBoris Pismenny sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL); 1909f66de3eeSBoris Pismenny if (!sw_ctx_tx) { 19103c4d7559SDave Watson rc = -ENOMEM; 19113c4d7559SDave Watson goto out; 19123c4d7559SDave Watson } 1913f66de3eeSBoris Pismenny ctx->priv_ctx_tx = sw_ctx_tx; 1914c46234ebSDave Watson } else { 1915b190a587SBoris Pismenny sw_ctx_tx = 1916b190a587SBoris Pismenny (struct tls_sw_context_tx *)ctx->priv_ctx_tx; 1917b190a587SBoris Pismenny } 1918b190a587SBoris Pismenny } else { 1919b190a587SBoris Pismenny if (!ctx->priv_ctx_rx) { 1920f66de3eeSBoris Pismenny sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL); 1921f66de3eeSBoris Pismenny if (!sw_ctx_rx) { 1922f66de3eeSBoris Pismenny rc = -ENOMEM; 1923f66de3eeSBoris Pismenny goto out; 1924c46234ebSDave Watson } 1925f66de3eeSBoris Pismenny ctx->priv_ctx_rx = sw_ctx_rx; 1926b190a587SBoris Pismenny } else { 1927b190a587SBoris Pismenny sw_ctx_rx = 1928b190a587SBoris Pismenny (struct tls_sw_context_rx *)ctx->priv_ctx_rx; 1929b190a587SBoris Pismenny } 1930f66de3eeSBoris Pismenny } 19313c4d7559SDave Watson 1932c46234ebSDave Watson if (tx) { 1933b190a587SBoris Pismenny crypto_init_wait(&sw_ctx_tx->async_wait); 193486029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 1935c46234ebSDave Watson cctx = &ctx->tx; 1936f66de3eeSBoris Pismenny aead = &sw_ctx_tx->aead_send; 19379932a29aSVakul Garg INIT_LIST_HEAD(&sw_ctx_tx->tx_list); 1938a42055e8SVakul Garg INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler); 1939a42055e8SVakul Garg sw_ctx_tx->tx_work.sk = sk; 1940c46234ebSDave Watson } else { 1941b190a587SBoris Pismenny crypto_init_wait(&sw_ctx_rx->async_wait); 194286029d10SSabrina Dubroca crypto_info = &ctx->crypto_recv.info; 1943c46234ebSDave Watson cctx = &ctx->rx; 1944f66de3eeSBoris Pismenny aead = &sw_ctx_rx->aead_recv; 1945c46234ebSDave Watson } 1946c46234ebSDave Watson 19473c4d7559SDave Watson switch (crypto_info->cipher_type) { 19483c4d7559SDave Watson case TLS_CIPHER_AES_GCM_128: { 19493c4d7559SDave Watson nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 19503c4d7559SDave Watson tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; 19513c4d7559SDave Watson iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; 19523c4d7559SDave Watson iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; 19533c4d7559SDave Watson rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; 19543c4d7559SDave Watson rec_seq = 19553c4d7559SDave Watson ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; 19563c4d7559SDave Watson gcm_128_info = 19573c4d7559SDave Watson (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 19583c4d7559SDave Watson break; 19593c4d7559SDave Watson } 19603c4d7559SDave Watson default: 19613c4d7559SDave Watson rc = -EINVAL; 1962cf6d43efSSabrina Dubroca goto free_priv; 19633c4d7559SDave Watson } 19643c4d7559SDave Watson 1965b16520f7SKees Cook /* Sanity-check the IV size for stack allocations. */ 19663463e51dSKees Cook if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE) { 1967b16520f7SKees Cook rc = -EINVAL; 1968b16520f7SKees Cook goto free_priv; 1969b16520f7SKees Cook } 1970b16520f7SKees Cook 1971c46234ebSDave Watson cctx->prepend_size = TLS_HEADER_SIZE + nonce_size; 1972c46234ebSDave Watson cctx->tag_size = tag_size; 1973c46234ebSDave Watson cctx->overhead_size = cctx->prepend_size + cctx->tag_size; 1974c46234ebSDave Watson cctx->iv_size = iv_size; 1975c46234ebSDave Watson cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 1976dbe42559SDave Watson GFP_KERNEL); 1977c46234ebSDave Watson if (!cctx->iv) { 19783c4d7559SDave Watson rc = -ENOMEM; 1979cf6d43efSSabrina Dubroca goto free_priv; 19803c4d7559SDave Watson } 1981c46234ebSDave Watson memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 1982c46234ebSDave Watson memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size); 1983c46234ebSDave Watson cctx->rec_seq_size = rec_seq_size; 1984969d5090Szhong jiang cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL); 1985c46234ebSDave Watson if (!cctx->rec_seq) { 19863c4d7559SDave Watson rc = -ENOMEM; 19873c4d7559SDave Watson goto free_iv; 19883c4d7559SDave Watson } 19893c4d7559SDave Watson 1990c46234ebSDave Watson if (!*aead) { 1991c46234ebSDave Watson *aead = crypto_alloc_aead("gcm(aes)", 0, 0); 1992c46234ebSDave Watson if (IS_ERR(*aead)) { 1993c46234ebSDave Watson rc = PTR_ERR(*aead); 1994c46234ebSDave Watson *aead = NULL; 19953c4d7559SDave Watson goto free_rec_seq; 19963c4d7559SDave Watson } 19973c4d7559SDave Watson } 19983c4d7559SDave Watson 19993c4d7559SDave Watson ctx->push_pending_record = tls_sw_push_pending_record; 20003c4d7559SDave Watson 20017cba09c6SSabrina Dubroca rc = crypto_aead_setkey(*aead, gcm_128_info->key, 20023c4d7559SDave Watson TLS_CIPHER_AES_GCM_128_KEY_SIZE); 20033c4d7559SDave Watson if (rc) 20043c4d7559SDave Watson goto free_aead; 20053c4d7559SDave Watson 2006c46234ebSDave Watson rc = crypto_aead_setauthsize(*aead, cctx->tag_size); 2007c46234ebSDave Watson if (rc) 2008c46234ebSDave Watson goto free_aead; 2009c46234ebSDave Watson 2010f66de3eeSBoris Pismenny if (sw_ctx_rx) { 2011c46234ebSDave Watson /* Set up strparser */ 2012c46234ebSDave Watson memset(&cb, 0, sizeof(cb)); 2013c46234ebSDave Watson cb.rcv_msg = tls_queue; 2014c46234ebSDave Watson cb.parse_msg = tls_read_size; 2015c46234ebSDave Watson 2016f66de3eeSBoris Pismenny strp_init(&sw_ctx_rx->strp, sk, &cb); 2017c46234ebSDave Watson 2018c46234ebSDave Watson write_lock_bh(&sk->sk_callback_lock); 2019f66de3eeSBoris Pismenny sw_ctx_rx->saved_data_ready = sk->sk_data_ready; 2020c46234ebSDave Watson sk->sk_data_ready = tls_data_ready; 2021c46234ebSDave Watson write_unlock_bh(&sk->sk_callback_lock); 2022c46234ebSDave Watson 2023f66de3eeSBoris Pismenny strp_check_rcv(&sw_ctx_rx->strp); 2024c46234ebSDave Watson } 2025c46234ebSDave Watson 2026c46234ebSDave Watson goto out; 20273c4d7559SDave Watson 20283c4d7559SDave Watson free_aead: 2029c46234ebSDave Watson crypto_free_aead(*aead); 2030c46234ebSDave Watson *aead = NULL; 20313c4d7559SDave Watson free_rec_seq: 2032c46234ebSDave Watson kfree(cctx->rec_seq); 2033c46234ebSDave Watson cctx->rec_seq = NULL; 20343c4d7559SDave Watson free_iv: 2035f66de3eeSBoris Pismenny kfree(cctx->iv); 2036f66de3eeSBoris Pismenny cctx->iv = NULL; 2037cf6d43efSSabrina Dubroca free_priv: 2038f66de3eeSBoris Pismenny if (tx) { 2039f66de3eeSBoris Pismenny kfree(ctx->priv_ctx_tx); 2040f66de3eeSBoris Pismenny ctx->priv_ctx_tx = NULL; 2041f66de3eeSBoris Pismenny } else { 2042f66de3eeSBoris Pismenny kfree(ctx->priv_ctx_rx); 2043f66de3eeSBoris Pismenny ctx->priv_ctx_rx = NULL; 2044f66de3eeSBoris Pismenny } 20453c4d7559SDave Watson out: 20463c4d7559SDave Watson return rc; 20473c4d7559SDave Watson } 2048