13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * 53c4d7559SDave Watson * This software is available to you under a choice of one of two 63c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 73c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 83c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 93c4d7559SDave Watson * OpenIB.org BSD license below: 103c4d7559SDave Watson * 113c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 123c4d7559SDave Watson * without modification, are permitted provided that the following 133c4d7559SDave Watson * conditions are met: 143c4d7559SDave Watson * 153c4d7559SDave Watson * - Redistributions of source code must retain the above 163c4d7559SDave Watson * copyright notice, this list of conditions and the following 173c4d7559SDave Watson * disclaimer. 183c4d7559SDave Watson * 193c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 203c4d7559SDave Watson * copyright notice, this list of conditions and the following 213c4d7559SDave Watson * disclaimer in the documentation and/or other materials 223c4d7559SDave Watson * provided with the distribution. 233c4d7559SDave Watson * 243c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 253c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 263c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 273c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 283c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 293c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 303c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 313c4d7559SDave Watson * SOFTWARE. 323c4d7559SDave Watson */ 333c4d7559SDave Watson 343c4d7559SDave Watson #include <linux/module.h> 353c4d7559SDave Watson 363c4d7559SDave Watson #include <net/tcp.h> 373c4d7559SDave Watson #include <net/inet_common.h> 383c4d7559SDave Watson #include <linux/highmem.h> 393c4d7559SDave Watson #include <linux/netdevice.h> 403c4d7559SDave Watson #include <linux/sched/signal.h> 41dd0bed16SAtul Gupta #include <linux/inetdevice.h> 4226811cc9SDavide Caratti #include <linux/inet_diag.h> 433c4d7559SDave Watson 44d26b698dSJakub Kicinski #include <net/snmp.h> 453c4d7559SDave Watson #include <net/tls.h> 4625a3cd81SJakub Kicinski #include <net/tls_toe.h> 473c4d7559SDave Watson 4858790314SJakub Kicinski #include "tls.h" 4958790314SJakub Kicinski 503c4d7559SDave Watson MODULE_AUTHOR("Mellanox Technologies"); 513c4d7559SDave Watson MODULE_DESCRIPTION("Transport Layer Security Support"); 523c4d7559SDave Watson MODULE_LICENSE("Dual BSD/GPL"); 53037b0b86SDaniel Borkmann MODULE_ALIAS_TCP_ULP("tls"); 543c4d7559SDave Watson 556d88207fSIlya Lesokhin enum { 56c113187dSBoris Pismenny TLSV4, 57c113187dSBoris Pismenny TLSV6, 58c113187dSBoris Pismenny TLS_NUM_PROTS, 59c113187dSBoris Pismenny }; 606d88207fSIlya Lesokhin 610d98cc02SSabrina Dubroca #define CHECK_CIPHER_DESC(cipher,ci) \ 620d98cc02SSabrina Dubroca static_assert(cipher ## _IV_SIZE <= MAX_IV_SIZE); \ 630d98cc02SSabrina Dubroca static_assert(cipher ## _REC_SEQ_SIZE <= TLS_MAX_REC_SEQ_SIZE); \ 640d98cc02SSabrina Dubroca static_assert(cipher ## _TAG_SIZE == TLS_TAG_SIZE); \ 650d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, iv) == cipher ## _IV_SIZE); \ 660d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, key) == cipher ## _KEY_SIZE); \ 670d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, salt) == cipher ## _SALT_SIZE); \ 680d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, rec_seq) == cipher ## _REC_SEQ_SIZE); 690d98cc02SSabrina Dubroca 70176a3f50SSabrina Dubroca #define __CIPHER_DESC(ci) \ 71176a3f50SSabrina Dubroca .iv_offset = offsetof(struct ci, iv), \ 72176a3f50SSabrina Dubroca .key_offset = offsetof(struct ci, key), \ 73176a3f50SSabrina Dubroca .salt_offset = offsetof(struct ci, salt), \ 74176a3f50SSabrina Dubroca .rec_seq_offset = offsetof(struct ci, rec_seq), \ 75176a3f50SSabrina Dubroca .crypto_info = sizeof(struct ci) 76176a3f50SSabrina Dubroca 77176a3f50SSabrina Dubroca #define CIPHER_DESC(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \ 78176a3f50SSabrina Dubroca .nonce = cipher ## _IV_SIZE, \ 792d2c5ea2STariq Toukan .iv = cipher ## _IV_SIZE, \ 802d2c5ea2STariq Toukan .key = cipher ## _KEY_SIZE, \ 812d2c5ea2STariq Toukan .salt = cipher ## _SALT_SIZE, \ 822d2c5ea2STariq Toukan .tag = cipher ## _TAG_SIZE, \ 832d2c5ea2STariq Toukan .rec_seq = cipher ## _REC_SEQ_SIZE, \ 84176a3f50SSabrina Dubroca .cipher_name = algname, \ 85176a3f50SSabrina Dubroca .offloadable = _offloadable, \ 86176a3f50SSabrina Dubroca __CIPHER_DESC(ci), \ 87176a3f50SSabrina Dubroca } 88176a3f50SSabrina Dubroca 89176a3f50SSabrina Dubroca #define CIPHER_DESC_NONCE0(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \ 90176a3f50SSabrina Dubroca .nonce = 0, \ 91176a3f50SSabrina Dubroca .iv = cipher ## _IV_SIZE, \ 92176a3f50SSabrina Dubroca .key = cipher ## _KEY_SIZE, \ 93176a3f50SSabrina Dubroca .salt = cipher ## _SALT_SIZE, \ 94176a3f50SSabrina Dubroca .tag = cipher ## _TAG_SIZE, \ 95176a3f50SSabrina Dubroca .rec_seq = cipher ## _REC_SEQ_SIZE, \ 96176a3f50SSabrina Dubroca .cipher_name = algname, \ 97176a3f50SSabrina Dubroca .offloadable = _offloadable, \ 98176a3f50SSabrina Dubroca __CIPHER_DESC(ci), \ 992d2c5ea2STariq Toukan } 1002d2c5ea2STariq Toukan 1018db44ab2SSabrina Dubroca const struct tls_cipher_desc tls_cipher_desc[TLS_CIPHER_MAX + 1 - TLS_CIPHER_MIN] = { 102176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128, "gcm(aes)", true), 103176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256, "gcm(aes)", true), 104176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128, "ccm(aes)", false), 105176a3f50SSabrina Dubroca CIPHER_DESC_NONCE0(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305, "rfc7539(chacha20,poly1305)", false), 106176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm, "gcm(sm4)", false), 107176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm, "ccm(sm4)", false), 108176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128, "gcm(aria)", false), 109176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256, "gcm(aria)", false), 1102d2c5ea2STariq Toukan }; 1112d2c5ea2STariq Toukan 1120d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128); 1130d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256); 1140d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128); 1150d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305); 1160d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm); 1170d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm); 1180d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128); 1190d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256); 1200d98cc02SSabrina Dubroca 121f691a25cSArnd Bergmann static const struct proto *saved_tcpv6_prot; 122c113187dSBoris Pismenny static DEFINE_MUTEX(tcpv6_prot_mutex); 123f691a25cSArnd Bergmann static const struct proto *saved_tcpv4_prot; 12428cb6f1eSJohn Fastabend static DEFINE_MUTEX(tcpv4_prot_mutex); 125f66de3eeSBoris Pismenny static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 126f3911f73SJakub Kicinski static struct proto_ops tls_proto_ops[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 12763a6b3feSAtul Gupta static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 128f13fe3e6SJakub Sitnicki const struct proto *base); 1296d88207fSIlya Lesokhin 13008700dabSJakub Kicinski void update_sk_prot(struct sock *sk, struct tls_context *ctx) 1316d88207fSIlya Lesokhin { 132c113187dSBoris Pismenny int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 133c113187dSBoris Pismenny 134d5bee737SJakub Sitnicki WRITE_ONCE(sk->sk_prot, 135d5bee737SJakub Sitnicki &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]); 136f3911f73SJakub Kicinski WRITE_ONCE(sk->sk_socket->ops, 137f3911f73SJakub Kicinski &tls_proto_ops[ip_ver][ctx->tx_conf][ctx->rx_conf]); 1386d88207fSIlya Lesokhin } 1393c4d7559SDave Watson 1403c4d7559SDave Watson int wait_on_pending_writer(struct sock *sk, long *timeo) 1413c4d7559SDave Watson { 1423c4d7559SDave Watson int rc = 0; 1433c4d7559SDave Watson DEFINE_WAIT_FUNC(wait, woken_wake_function); 1443c4d7559SDave Watson 1453c4d7559SDave Watson add_wait_queue(sk_sleep(sk), &wait); 1463c4d7559SDave Watson while (1) { 1473c4d7559SDave Watson if (!*timeo) { 1483c4d7559SDave Watson rc = -EAGAIN; 1493c4d7559SDave Watson break; 1503c4d7559SDave Watson } 1513c4d7559SDave Watson 1523c4d7559SDave Watson if (signal_pending(current)) { 1533c4d7559SDave Watson rc = sock_intr_errno(*timeo); 1543c4d7559SDave Watson break; 1553c4d7559SDave Watson } 1563c4d7559SDave Watson 157d0ac89f6SEric Dumazet if (sk_wait_event(sk, timeo, 158d0ac89f6SEric Dumazet !READ_ONCE(sk->sk_write_pending), &wait)) 1593c4d7559SDave Watson break; 1603c4d7559SDave Watson } 1613c4d7559SDave Watson remove_wait_queue(sk_sleep(sk), &wait); 1623c4d7559SDave Watson return rc; 1633c4d7559SDave Watson } 1643c4d7559SDave Watson 1653c4d7559SDave Watson int tls_push_sg(struct sock *sk, 1663c4d7559SDave Watson struct tls_context *ctx, 1673c4d7559SDave Watson struct scatterlist *sg, 1683c4d7559SDave Watson u16 first_offset, 1693c4d7559SDave Watson int flags) 1703c4d7559SDave Watson { 171e117dcfdSDavid Howells struct bio_vec bvec; 172e117dcfdSDavid Howells struct msghdr msg = { 173b848b26cSDavid Howells .msg_flags = MSG_SPLICE_PAGES | flags, 174e117dcfdSDavid Howells }; 1753c4d7559SDave Watson int ret = 0; 1763c4d7559SDave Watson struct page *p; 1773c4d7559SDave Watson size_t size; 1783c4d7559SDave Watson int offset = first_offset; 1793c4d7559SDave Watson 1803c4d7559SDave Watson size = sg->length - offset; 1813c4d7559SDave Watson offset += sg->offset; 1823c4d7559SDave Watson 183e117dcfdSDavid Howells ctx->splicing_pages = true; 1843c4d7559SDave Watson while (1) { 1853c4d7559SDave Watson /* is sending application-limited? */ 1863c4d7559SDave Watson tcp_rate_check_app_limited(sk); 1873c4d7559SDave Watson p = sg_page(sg); 1883c4d7559SDave Watson retry: 189e117dcfdSDavid Howells bvec_set_page(&bvec, p, size, offset); 190e117dcfdSDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size); 191e117dcfdSDavid Howells 192e117dcfdSDavid Howells ret = tcp_sendmsg_locked(sk, &msg, size); 1933c4d7559SDave Watson 1943c4d7559SDave Watson if (ret != size) { 1953c4d7559SDave Watson if (ret > 0) { 1963c4d7559SDave Watson offset += ret; 1973c4d7559SDave Watson size -= ret; 1983c4d7559SDave Watson goto retry; 1993c4d7559SDave Watson } 2003c4d7559SDave Watson 2013c4d7559SDave Watson offset -= sg->offset; 2023c4d7559SDave Watson ctx->partially_sent_offset = offset; 2033c4d7559SDave Watson ctx->partially_sent_record = (void *)sg; 204e117dcfdSDavid Howells ctx->splicing_pages = false; 2053c4d7559SDave Watson return ret; 2063c4d7559SDave Watson } 2073c4d7559SDave Watson 2083c4d7559SDave Watson put_page(p); 2093c4d7559SDave Watson sk_mem_uncharge(sk, sg->length); 2103c4d7559SDave Watson sg = sg_next(sg); 2113c4d7559SDave Watson if (!sg) 2123c4d7559SDave Watson break; 2133c4d7559SDave Watson 2143c4d7559SDave Watson offset = sg->offset; 2153c4d7559SDave Watson size = sg->length; 2163c4d7559SDave Watson } 2173c4d7559SDave Watson 218e117dcfdSDavid Howells ctx->splicing_pages = false; 2193c4d7559SDave Watson 2203c4d7559SDave Watson return 0; 2213c4d7559SDave Watson } 2223c4d7559SDave Watson 2233c4d7559SDave Watson static int tls_handle_open_record(struct sock *sk, int flags) 2243c4d7559SDave Watson { 2253c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 2263c4d7559SDave Watson 2273c4d7559SDave Watson if (tls_is_pending_open_record(ctx)) 2283c4d7559SDave Watson return ctx->push_pending_record(sk, flags); 2293c4d7559SDave Watson 2303c4d7559SDave Watson return 0; 2313c4d7559SDave Watson } 2323c4d7559SDave Watson 23358790314SJakub Kicinski int tls_process_cmsg(struct sock *sk, struct msghdr *msg, 2343c4d7559SDave Watson unsigned char *record_type) 2353c4d7559SDave Watson { 2363c4d7559SDave Watson struct cmsghdr *cmsg; 2373c4d7559SDave Watson int rc = -EINVAL; 2383c4d7559SDave Watson 2393c4d7559SDave Watson for_each_cmsghdr(cmsg, msg) { 2403c4d7559SDave Watson if (!CMSG_OK(msg, cmsg)) 2413c4d7559SDave Watson return -EINVAL; 2423c4d7559SDave Watson if (cmsg->cmsg_level != SOL_TLS) 2433c4d7559SDave Watson continue; 2443c4d7559SDave Watson 2453c4d7559SDave Watson switch (cmsg->cmsg_type) { 2463c4d7559SDave Watson case TLS_SET_RECORD_TYPE: 2473c4d7559SDave Watson if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) 2483c4d7559SDave Watson return -EINVAL; 2493c4d7559SDave Watson 2503c4d7559SDave Watson if (msg->msg_flags & MSG_MORE) 2513c4d7559SDave Watson return -EINVAL; 2523c4d7559SDave Watson 2533c4d7559SDave Watson rc = tls_handle_open_record(sk, msg->msg_flags); 2543c4d7559SDave Watson if (rc) 2553c4d7559SDave Watson return rc; 2563c4d7559SDave Watson 2573c4d7559SDave Watson *record_type = *(unsigned char *)CMSG_DATA(cmsg); 2583c4d7559SDave Watson rc = 0; 2593c4d7559SDave Watson break; 2603c4d7559SDave Watson default: 2613c4d7559SDave Watson return -EINVAL; 2623c4d7559SDave Watson } 2633c4d7559SDave Watson } 2643c4d7559SDave Watson 2653c4d7559SDave Watson return rc; 2663c4d7559SDave Watson } 2673c4d7559SDave Watson 268a42055e8SVakul Garg int tls_push_partial_record(struct sock *sk, struct tls_context *ctx, 269a42055e8SVakul Garg int flags) 2703c4d7559SDave Watson { 2713c4d7559SDave Watson struct scatterlist *sg; 2723c4d7559SDave Watson u16 offset; 2733c4d7559SDave Watson 2743c4d7559SDave Watson sg = ctx->partially_sent_record; 2753c4d7559SDave Watson offset = ctx->partially_sent_offset; 2763c4d7559SDave Watson 2773c4d7559SDave Watson ctx->partially_sent_record = NULL; 2783c4d7559SDave Watson return tls_push_sg(sk, ctx, sg, offset, flags); 2793c4d7559SDave Watson } 2803c4d7559SDave Watson 281c5daa6ccSJakub Kicinski void tls_free_partial_record(struct sock *sk, struct tls_context *ctx) 28235b71a34SJakub Kicinski { 28335b71a34SJakub Kicinski struct scatterlist *sg; 28435b71a34SJakub Kicinski 285c5daa6ccSJakub Kicinski for (sg = ctx->partially_sent_record; sg; sg = sg_next(sg)) { 28635b71a34SJakub Kicinski put_page(sg_page(sg)); 28735b71a34SJakub Kicinski sk_mem_uncharge(sk, sg->length); 28835b71a34SJakub Kicinski } 28935b71a34SJakub Kicinski ctx->partially_sent_record = NULL; 29035b71a34SJakub Kicinski } 29135b71a34SJakub Kicinski 2923c4d7559SDave Watson static void tls_write_space(struct sock *sk) 2933c4d7559SDave Watson { 2943c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 2953c4d7559SDave Watson 296e117dcfdSDavid Howells /* If splicing_pages call lower protocol write space handler 29767db7cd2SJohn Fastabend * to ensure we wake up any waiting operations there. For example 298e117dcfdSDavid Howells * if splicing pages where to call sk_wait_event. 29967db7cd2SJohn Fastabend */ 300e117dcfdSDavid Howells if (ctx->splicing_pages) { 30167db7cd2SJohn Fastabend ctx->sk_write_space(sk); 302c212d2c7SDave Watson return; 30367db7cd2SJohn Fastabend } 304c212d2c7SDave Watson 3057463d3a2SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 3067463d3a2SBoris Pismenny if (ctx->tx_conf == TLS_HW) 3077463d3a2SBoris Pismenny tls_device_write_space(sk, ctx); 3087463d3a2SBoris Pismenny else 3097463d3a2SBoris Pismenny #endif 3107463d3a2SBoris Pismenny tls_sw_write_space(sk, ctx); 3114504ab0eSVakul Garg 3124504ab0eSVakul Garg ctx->sk_write_space(sk); 3133c4d7559SDave Watson } 3143c4d7559SDave Watson 31515a7dea7SJakub Kicinski /** 31615a7dea7SJakub Kicinski * tls_ctx_free() - free TLS ULP context 31715a7dea7SJakub Kicinski * @sk: socket to with @ctx is attached 31815a7dea7SJakub Kicinski * @ctx: TLS context structure 31915a7dea7SJakub Kicinski * 32015a7dea7SJakub Kicinski * Free TLS context. If @sk is %NULL caller guarantees that the socket 32115a7dea7SJakub Kicinski * to which @ctx was attached has no outstanding references. 32215a7dea7SJakub Kicinski */ 32315a7dea7SJakub Kicinski void tls_ctx_free(struct sock *sk, struct tls_context *ctx) 32486029d10SSabrina Dubroca { 32586029d10SSabrina Dubroca if (!ctx) 32686029d10SSabrina Dubroca return; 32786029d10SSabrina Dubroca 32886029d10SSabrina Dubroca memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send)); 32986029d10SSabrina Dubroca memzero_explicit(&ctx->crypto_recv, sizeof(ctx->crypto_recv)); 33079ffe608SJakub Kicinski mutex_destroy(&ctx->tx_lock); 33115a7dea7SJakub Kicinski 33215a7dea7SJakub Kicinski if (sk) 33315a7dea7SJakub Kicinski kfree_rcu(ctx, rcu); 33415a7dea7SJakub Kicinski else 33586029d10SSabrina Dubroca kfree(ctx); 33686029d10SSabrina Dubroca } 33786029d10SSabrina Dubroca 338313ab004SJohn Fastabend static void tls_sk_proto_cleanup(struct sock *sk, 339313ab004SJohn Fastabend struct tls_context *ctx, long timeo) 3403c4d7559SDave Watson { 3419354544cSDirk van der Merwe if (unlikely(sk->sk_write_pending) && 3429354544cSDirk van der Merwe !wait_on_pending_writer(sk, &timeo)) 3433c4d7559SDave Watson tls_handle_open_record(sk, 0); 3443c4d7559SDave Watson 345f66de3eeSBoris Pismenny /* We need these for tls_sw_fallback handling of other packets */ 346f66de3eeSBoris Pismenny if (ctx->tx_conf == TLS_SW) { 347dbe42559SDave Watson kfree(ctx->tx.rec_seq); 348dbe42559SDave Watson kfree(ctx->tx.iv); 349313ab004SJohn Fastabend tls_sw_release_resources_tx(sk); 350b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW); 35135b71a34SJakub Kicinski } else if (ctx->tx_conf == TLS_HW) { 35235b71a34SJakub Kicinski tls_device_free_resources_tx(sk); 353b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE); 354f66de3eeSBoris Pismenny } 355f66de3eeSBoris Pismenny 356b32fd3ccSJakub Kicinski if (ctx->rx_conf == TLS_SW) { 357313ab004SJohn Fastabend tls_sw_release_resources_rx(sk); 358b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW); 359b32fd3ccSJakub Kicinski } else if (ctx->rx_conf == TLS_HW) { 3604799ac81SBoris Pismenny tls_device_offload_cleanup_rx(sk); 361b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE); 362b32fd3ccSJakub Kicinski } 363e8f69799SIlya Lesokhin } 364e8f69799SIlya Lesokhin 365313ab004SJohn Fastabend static void tls_sk_proto_close(struct sock *sk, long timeout) 366313ab004SJohn Fastabend { 36795fa1454SJohn Fastabend struct inet_connection_sock *icsk = inet_csk(sk); 368313ab004SJohn Fastabend struct tls_context *ctx = tls_get_ctx(sk); 369313ab004SJohn Fastabend long timeo = sock_sndtimeo(sk, 0); 370313ab004SJohn Fastabend bool free_ctx; 371313ab004SJohn Fastabend 372313ab004SJohn Fastabend if (ctx->tx_conf == TLS_SW) 373313ab004SJohn Fastabend tls_sw_cancel_work_tx(ctx); 374313ab004SJohn Fastabend 375313ab004SJohn Fastabend lock_sock(sk); 376313ab004SJohn Fastabend free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW; 377313ab004SJohn Fastabend 378313ab004SJohn Fastabend if (ctx->tx_conf != TLS_BASE || ctx->rx_conf != TLS_BASE) 379313ab004SJohn Fastabend tls_sk_proto_cleanup(sk, ctx, timeo); 380313ab004SJohn Fastabend 38195fa1454SJohn Fastabend write_lock_bh(&sk->sk_callback_lock); 38295fa1454SJohn Fastabend if (free_ctx) 38315a7dea7SJakub Kicinski rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 384d5bee737SJakub Sitnicki WRITE_ONCE(sk->sk_prot, ctx->sk_proto); 385d85f0177SJohn Fastabend if (sk->sk_write_space == tls_write_space) 38657c722e9SJakub Kicinski sk->sk_write_space = ctx->sk_write_space; 38795fa1454SJohn Fastabend write_unlock_bh(&sk->sk_callback_lock); 3883c4d7559SDave Watson release_sock(sk); 389313ab004SJohn Fastabend if (ctx->tx_conf == TLS_SW) 390313ab004SJohn Fastabend tls_sw_free_ctx_tx(ctx); 391313ab004SJohn Fastabend if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) 392313ab004SJohn Fastabend tls_sw_strparser_done(ctx); 393313ab004SJohn Fastabend if (ctx->rx_conf == TLS_SW) 394313ab004SJohn Fastabend tls_sw_free_ctx_rx(ctx); 395be7bbea1SJakub Kicinski ctx->sk_proto->close(sk, timeout); 396313ab004SJohn Fastabend 39798f0a395SEric Dumazet if (free_ctx) 39815a7dea7SJakub Kicinski tls_ctx_free(sk, ctx); 3993c4d7559SDave Watson } 4003c4d7559SDave Watson 401121dca78SJakub Kicinski static __poll_t tls_sk_poll(struct file *file, struct socket *sock, 402121dca78SJakub Kicinski struct poll_table_struct *wait) 403121dca78SJakub Kicinski { 404121dca78SJakub Kicinski struct tls_sw_context_rx *ctx; 405121dca78SJakub Kicinski struct tls_context *tls_ctx; 406121dca78SJakub Kicinski struct sock *sk = sock->sk; 407121dca78SJakub Kicinski struct sk_psock *psock; 408121dca78SJakub Kicinski __poll_t mask = 0; 409121dca78SJakub Kicinski u8 shutdown; 410121dca78SJakub Kicinski int state; 411121dca78SJakub Kicinski 412121dca78SJakub Kicinski mask = tcp_poll(file, sock, wait); 413121dca78SJakub Kicinski 414121dca78SJakub Kicinski state = inet_sk_state_load(sk); 415121dca78SJakub Kicinski shutdown = READ_ONCE(sk->sk_shutdown); 416121dca78SJakub Kicinski if (unlikely(state != TCP_ESTABLISHED || shutdown & RCV_SHUTDOWN)) 417121dca78SJakub Kicinski return mask; 418121dca78SJakub Kicinski 419121dca78SJakub Kicinski tls_ctx = tls_get_ctx(sk); 420121dca78SJakub Kicinski ctx = tls_sw_ctx_rx(tls_ctx); 421121dca78SJakub Kicinski psock = sk_psock_get(sk); 422121dca78SJakub Kicinski 423121dca78SJakub Kicinski if (skb_queue_empty_lockless(&ctx->rx_list) && 424121dca78SJakub Kicinski !tls_strp_msg_ready(ctx) && 425121dca78SJakub Kicinski sk_psock_queue_empty(psock)) 426121dca78SJakub Kicinski mask &= ~(EPOLLIN | EPOLLRDNORM); 427121dca78SJakub Kicinski 428121dca78SJakub Kicinski if (psock) 429121dca78SJakub Kicinski sk_psock_put(sk, psock); 430121dca78SJakub Kicinski 431121dca78SJakub Kicinski return mask; 432121dca78SJakub Kicinski } 433121dca78SJakub Kicinski 434ffa81fa4SYutaro Hayakawa static int do_tls_getsockopt_conf(struct sock *sk, char __user *optval, 435ffa81fa4SYutaro Hayakawa int __user *optlen, int tx) 4363c4d7559SDave Watson { 4373c4d7559SDave Watson int rc = 0; 4383c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 4393c4d7559SDave Watson struct tls_crypto_info *crypto_info; 440ffa81fa4SYutaro Hayakawa struct cipher_context *cctx; 4413c4d7559SDave Watson int len; 4423c4d7559SDave Watson 4433c4d7559SDave Watson if (get_user(len, optlen)) 4443c4d7559SDave Watson return -EFAULT; 4453c4d7559SDave Watson 4463c4d7559SDave Watson if (!optval || (len < sizeof(*crypto_info))) { 4473c4d7559SDave Watson rc = -EINVAL; 4483c4d7559SDave Watson goto out; 4493c4d7559SDave Watson } 4503c4d7559SDave Watson 4513c4d7559SDave Watson if (!ctx) { 4523c4d7559SDave Watson rc = -EBUSY; 4533c4d7559SDave Watson goto out; 4543c4d7559SDave Watson } 4553c4d7559SDave Watson 4563c4d7559SDave Watson /* get user crypto info */ 457ffa81fa4SYutaro Hayakawa if (tx) { 45886029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 459ffa81fa4SYutaro Hayakawa cctx = &ctx->tx; 460ffa81fa4SYutaro Hayakawa } else { 461ffa81fa4SYutaro Hayakawa crypto_info = &ctx->crypto_recv.info; 462ffa81fa4SYutaro Hayakawa cctx = &ctx->rx; 463ffa81fa4SYutaro Hayakawa } 4643c4d7559SDave Watson 4653c4d7559SDave Watson if (!TLS_CRYPTO_INFO_READY(crypto_info)) { 4663c4d7559SDave Watson rc = -EBUSY; 4673c4d7559SDave Watson goto out; 4683c4d7559SDave Watson } 4693c4d7559SDave Watson 4705a3b886cSMatthias Rosenfelder if (len == sizeof(*crypto_info)) { 471ac55cd61SDan Carpenter if (copy_to_user(optval, crypto_info, sizeof(*crypto_info))) 472ac55cd61SDan Carpenter rc = -EFAULT; 4733c4d7559SDave Watson goto out; 4743c4d7559SDave Watson } 4753c4d7559SDave Watson 4763c4d7559SDave Watson switch (crypto_info->cipher_type) { 4773c4d7559SDave Watson case TLS_CIPHER_AES_GCM_128: { 4783c4d7559SDave Watson struct tls12_crypto_info_aes_gcm_128 * 4793c4d7559SDave Watson crypto_info_aes_gcm_128 = 4803c4d7559SDave Watson container_of(crypto_info, 4813c4d7559SDave Watson struct tls12_crypto_info_aes_gcm_128, 4823c4d7559SDave Watson info); 4833c4d7559SDave Watson 4843c4d7559SDave Watson if (len != sizeof(*crypto_info_aes_gcm_128)) { 4853c4d7559SDave Watson rc = -EINVAL; 4863c4d7559SDave Watson goto out; 4873c4d7559SDave Watson } 488a1dfa681SBoris Pismenny memcpy(crypto_info_aes_gcm_128->iv, 489ffa81fa4SYutaro Hayakawa cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 4903c4d7559SDave Watson TLS_CIPHER_AES_GCM_128_IV_SIZE); 491ffa81fa4SYutaro Hayakawa memcpy(crypto_info_aes_gcm_128->rec_seq, cctx->rec_seq, 492c410c196SBoris Pismenny TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); 493ac55cd61SDan Carpenter if (copy_to_user(optval, 4943c4d7559SDave Watson crypto_info_aes_gcm_128, 495ac55cd61SDan Carpenter sizeof(*crypto_info_aes_gcm_128))) 496ac55cd61SDan Carpenter rc = -EFAULT; 4973c4d7559SDave Watson break; 4983c4d7559SDave Watson } 499fb99bce7SDave Watson case TLS_CIPHER_AES_GCM_256: { 500fb99bce7SDave Watson struct tls12_crypto_info_aes_gcm_256 * 501fb99bce7SDave Watson crypto_info_aes_gcm_256 = 502fb99bce7SDave Watson container_of(crypto_info, 503fb99bce7SDave Watson struct tls12_crypto_info_aes_gcm_256, 504fb99bce7SDave Watson info); 505fb99bce7SDave Watson 506fb99bce7SDave Watson if (len != sizeof(*crypto_info_aes_gcm_256)) { 507fb99bce7SDave Watson rc = -EINVAL; 508fb99bce7SDave Watson goto out; 509fb99bce7SDave Watson } 510fb99bce7SDave Watson memcpy(crypto_info_aes_gcm_256->iv, 511ffa81fa4SYutaro Hayakawa cctx->iv + TLS_CIPHER_AES_GCM_256_SALT_SIZE, 512fb99bce7SDave Watson TLS_CIPHER_AES_GCM_256_IV_SIZE); 513ffa81fa4SYutaro Hayakawa memcpy(crypto_info_aes_gcm_256->rec_seq, cctx->rec_seq, 514fb99bce7SDave Watson TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE); 515fb99bce7SDave Watson if (copy_to_user(optval, 516fb99bce7SDave Watson crypto_info_aes_gcm_256, 517fb99bce7SDave Watson sizeof(*crypto_info_aes_gcm_256))) 518fb99bce7SDave Watson rc = -EFAULT; 519fb99bce7SDave Watson break; 520fb99bce7SDave Watson } 5213fb59a5dSTianjia Zhang case TLS_CIPHER_AES_CCM_128: { 5223fb59a5dSTianjia Zhang struct tls12_crypto_info_aes_ccm_128 *aes_ccm_128 = 5233fb59a5dSTianjia Zhang container_of(crypto_info, 5243fb59a5dSTianjia Zhang struct tls12_crypto_info_aes_ccm_128, info); 5253fb59a5dSTianjia Zhang 5263fb59a5dSTianjia Zhang if (len != sizeof(*aes_ccm_128)) { 5273fb59a5dSTianjia Zhang rc = -EINVAL; 5283fb59a5dSTianjia Zhang goto out; 5293fb59a5dSTianjia Zhang } 5303fb59a5dSTianjia Zhang memcpy(aes_ccm_128->iv, 5313fb59a5dSTianjia Zhang cctx->iv + TLS_CIPHER_AES_CCM_128_SALT_SIZE, 5323fb59a5dSTianjia Zhang TLS_CIPHER_AES_CCM_128_IV_SIZE); 5333fb59a5dSTianjia Zhang memcpy(aes_ccm_128->rec_seq, cctx->rec_seq, 5343fb59a5dSTianjia Zhang TLS_CIPHER_AES_CCM_128_REC_SEQ_SIZE); 5353fb59a5dSTianjia Zhang if (copy_to_user(optval, aes_ccm_128, sizeof(*aes_ccm_128))) 5363fb59a5dSTianjia Zhang rc = -EFAULT; 5373fb59a5dSTianjia Zhang break; 5383fb59a5dSTianjia Zhang } 5393fb59a5dSTianjia Zhang case TLS_CIPHER_CHACHA20_POLY1305: { 5403fb59a5dSTianjia Zhang struct tls12_crypto_info_chacha20_poly1305 *chacha20_poly1305 = 5413fb59a5dSTianjia Zhang container_of(crypto_info, 5423fb59a5dSTianjia Zhang struct tls12_crypto_info_chacha20_poly1305, 5433fb59a5dSTianjia Zhang info); 5443fb59a5dSTianjia Zhang 5453fb59a5dSTianjia Zhang if (len != sizeof(*chacha20_poly1305)) { 5463fb59a5dSTianjia Zhang rc = -EINVAL; 5473fb59a5dSTianjia Zhang goto out; 5483fb59a5dSTianjia Zhang } 5493fb59a5dSTianjia Zhang memcpy(chacha20_poly1305->iv, 5503fb59a5dSTianjia Zhang cctx->iv + TLS_CIPHER_CHACHA20_POLY1305_SALT_SIZE, 5513fb59a5dSTianjia Zhang TLS_CIPHER_CHACHA20_POLY1305_IV_SIZE); 5523fb59a5dSTianjia Zhang memcpy(chacha20_poly1305->rec_seq, cctx->rec_seq, 5533fb59a5dSTianjia Zhang TLS_CIPHER_CHACHA20_POLY1305_REC_SEQ_SIZE); 5543fb59a5dSTianjia Zhang if (copy_to_user(optval, chacha20_poly1305, 5553fb59a5dSTianjia Zhang sizeof(*chacha20_poly1305))) 5563fb59a5dSTianjia Zhang rc = -EFAULT; 5573fb59a5dSTianjia Zhang break; 5583fb59a5dSTianjia Zhang } 559227b9644STianjia Zhang case TLS_CIPHER_SM4_GCM: { 560227b9644STianjia Zhang struct tls12_crypto_info_sm4_gcm *sm4_gcm_info = 561227b9644STianjia Zhang container_of(crypto_info, 562227b9644STianjia Zhang struct tls12_crypto_info_sm4_gcm, info); 563227b9644STianjia Zhang 564227b9644STianjia Zhang if (len != sizeof(*sm4_gcm_info)) { 565227b9644STianjia Zhang rc = -EINVAL; 566227b9644STianjia Zhang goto out; 567227b9644STianjia Zhang } 568227b9644STianjia Zhang memcpy(sm4_gcm_info->iv, 569227b9644STianjia Zhang cctx->iv + TLS_CIPHER_SM4_GCM_SALT_SIZE, 570227b9644STianjia Zhang TLS_CIPHER_SM4_GCM_IV_SIZE); 571227b9644STianjia Zhang memcpy(sm4_gcm_info->rec_seq, cctx->rec_seq, 572227b9644STianjia Zhang TLS_CIPHER_SM4_GCM_REC_SEQ_SIZE); 573227b9644STianjia Zhang if (copy_to_user(optval, sm4_gcm_info, sizeof(*sm4_gcm_info))) 574227b9644STianjia Zhang rc = -EFAULT; 575227b9644STianjia Zhang break; 576227b9644STianjia Zhang } 577227b9644STianjia Zhang case TLS_CIPHER_SM4_CCM: { 578227b9644STianjia Zhang struct tls12_crypto_info_sm4_ccm *sm4_ccm_info = 579227b9644STianjia Zhang container_of(crypto_info, 580227b9644STianjia Zhang struct tls12_crypto_info_sm4_ccm, info); 581227b9644STianjia Zhang 582227b9644STianjia Zhang if (len != sizeof(*sm4_ccm_info)) { 583227b9644STianjia Zhang rc = -EINVAL; 584227b9644STianjia Zhang goto out; 585227b9644STianjia Zhang } 586227b9644STianjia Zhang memcpy(sm4_ccm_info->iv, 587227b9644STianjia Zhang cctx->iv + TLS_CIPHER_SM4_CCM_SALT_SIZE, 588227b9644STianjia Zhang TLS_CIPHER_SM4_CCM_IV_SIZE); 589227b9644STianjia Zhang memcpy(sm4_ccm_info->rec_seq, cctx->rec_seq, 590227b9644STianjia Zhang TLS_CIPHER_SM4_CCM_REC_SEQ_SIZE); 591227b9644STianjia Zhang if (copy_to_user(optval, sm4_ccm_info, sizeof(*sm4_ccm_info))) 592227b9644STianjia Zhang rc = -EFAULT; 593227b9644STianjia Zhang break; 594227b9644STianjia Zhang } 59562e56ef5STaehee Yoo case TLS_CIPHER_ARIA_GCM_128: { 59662e56ef5STaehee Yoo struct tls12_crypto_info_aria_gcm_128 * 59762e56ef5STaehee Yoo crypto_info_aria_gcm_128 = 59862e56ef5STaehee Yoo container_of(crypto_info, 59962e56ef5STaehee Yoo struct tls12_crypto_info_aria_gcm_128, 60062e56ef5STaehee Yoo info); 60162e56ef5STaehee Yoo 60262e56ef5STaehee Yoo if (len != sizeof(*crypto_info_aria_gcm_128)) { 60362e56ef5STaehee Yoo rc = -EINVAL; 60462e56ef5STaehee Yoo goto out; 60562e56ef5STaehee Yoo } 60662e56ef5STaehee Yoo memcpy(crypto_info_aria_gcm_128->iv, 60762e56ef5STaehee Yoo cctx->iv + TLS_CIPHER_ARIA_GCM_128_SALT_SIZE, 60862e56ef5STaehee Yoo TLS_CIPHER_ARIA_GCM_128_IV_SIZE); 60962e56ef5STaehee Yoo memcpy(crypto_info_aria_gcm_128->rec_seq, cctx->rec_seq, 61062e56ef5STaehee Yoo TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE); 61162e56ef5STaehee Yoo if (copy_to_user(optval, 61262e56ef5STaehee Yoo crypto_info_aria_gcm_128, 61362e56ef5STaehee Yoo sizeof(*crypto_info_aria_gcm_128))) 61462e56ef5STaehee Yoo rc = -EFAULT; 61562e56ef5STaehee Yoo break; 61662e56ef5STaehee Yoo } 61762e56ef5STaehee Yoo case TLS_CIPHER_ARIA_GCM_256: { 61862e56ef5STaehee Yoo struct tls12_crypto_info_aria_gcm_256 * 61962e56ef5STaehee Yoo crypto_info_aria_gcm_256 = 62062e56ef5STaehee Yoo container_of(crypto_info, 62162e56ef5STaehee Yoo struct tls12_crypto_info_aria_gcm_256, 62262e56ef5STaehee Yoo info); 62362e56ef5STaehee Yoo 62462e56ef5STaehee Yoo if (len != sizeof(*crypto_info_aria_gcm_256)) { 62562e56ef5STaehee Yoo rc = -EINVAL; 62662e56ef5STaehee Yoo goto out; 62762e56ef5STaehee Yoo } 62862e56ef5STaehee Yoo memcpy(crypto_info_aria_gcm_256->iv, 62962e56ef5STaehee Yoo cctx->iv + TLS_CIPHER_ARIA_GCM_256_SALT_SIZE, 63062e56ef5STaehee Yoo TLS_CIPHER_ARIA_GCM_256_IV_SIZE); 63162e56ef5STaehee Yoo memcpy(crypto_info_aria_gcm_256->rec_seq, cctx->rec_seq, 63262e56ef5STaehee Yoo TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE); 63362e56ef5STaehee Yoo if (copy_to_user(optval, 63462e56ef5STaehee Yoo crypto_info_aria_gcm_256, 63562e56ef5STaehee Yoo sizeof(*crypto_info_aria_gcm_256))) 63662e56ef5STaehee Yoo rc = -EFAULT; 63762e56ef5STaehee Yoo break; 63862e56ef5STaehee Yoo } 6393c4d7559SDave Watson default: 6403c4d7559SDave Watson rc = -EINVAL; 6413c4d7559SDave Watson } 6423c4d7559SDave Watson 6433c4d7559SDave Watson out: 6443c4d7559SDave Watson return rc; 6453c4d7559SDave Watson } 6463c4d7559SDave Watson 647c1318b39SBoris Pismenny static int do_tls_getsockopt_tx_zc(struct sock *sk, char __user *optval, 648c1318b39SBoris Pismenny int __user *optlen) 649c1318b39SBoris Pismenny { 650c1318b39SBoris Pismenny struct tls_context *ctx = tls_get_ctx(sk); 651c1318b39SBoris Pismenny unsigned int value; 652c1318b39SBoris Pismenny int len; 653c1318b39SBoris Pismenny 654c1318b39SBoris Pismenny if (get_user(len, optlen)) 655c1318b39SBoris Pismenny return -EFAULT; 656c1318b39SBoris Pismenny 657c1318b39SBoris Pismenny if (len != sizeof(value)) 658c1318b39SBoris Pismenny return -EINVAL; 659c1318b39SBoris Pismenny 660c1318b39SBoris Pismenny value = ctx->zerocopy_sendfile; 661c1318b39SBoris Pismenny if (copy_to_user(optval, &value, sizeof(value))) 662c1318b39SBoris Pismenny return -EFAULT; 663c1318b39SBoris Pismenny 664c1318b39SBoris Pismenny return 0; 665c1318b39SBoris Pismenny } 666c1318b39SBoris Pismenny 66788527790SJakub Kicinski static int do_tls_getsockopt_no_pad(struct sock *sk, char __user *optval, 66888527790SJakub Kicinski int __user *optlen) 66988527790SJakub Kicinski { 67088527790SJakub Kicinski struct tls_context *ctx = tls_get_ctx(sk); 67157128e98SJakub Kicinski int value, len; 67288527790SJakub Kicinski 67388527790SJakub Kicinski if (ctx->prot_info.version != TLS_1_3_VERSION) 67488527790SJakub Kicinski return -EINVAL; 67588527790SJakub Kicinski 67688527790SJakub Kicinski if (get_user(len, optlen)) 67788527790SJakub Kicinski return -EFAULT; 67888527790SJakub Kicinski if (len < sizeof(value)) 67988527790SJakub Kicinski return -EINVAL; 68088527790SJakub Kicinski 68157128e98SJakub Kicinski value = -EINVAL; 68288527790SJakub Kicinski if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) 68388527790SJakub Kicinski value = ctx->rx_no_pad; 68457128e98SJakub Kicinski if (value < 0) 68557128e98SJakub Kicinski return value; 68688527790SJakub Kicinski 68788527790SJakub Kicinski if (put_user(sizeof(value), optlen)) 68888527790SJakub Kicinski return -EFAULT; 68988527790SJakub Kicinski if (copy_to_user(optval, &value, sizeof(value))) 69088527790SJakub Kicinski return -EFAULT; 69188527790SJakub Kicinski 69288527790SJakub Kicinski return 0; 69388527790SJakub Kicinski } 69488527790SJakub Kicinski 6953c4d7559SDave Watson static int do_tls_getsockopt(struct sock *sk, int optname, 6963c4d7559SDave Watson char __user *optval, int __user *optlen) 6973c4d7559SDave Watson { 6983c4d7559SDave Watson int rc = 0; 6993c4d7559SDave Watson 70049c47cc2SHangyu Hua lock_sock(sk); 70149c47cc2SHangyu Hua 7023c4d7559SDave Watson switch (optname) { 7033c4d7559SDave Watson case TLS_TX: 704ffa81fa4SYutaro Hayakawa case TLS_RX: 705ffa81fa4SYutaro Hayakawa rc = do_tls_getsockopt_conf(sk, optval, optlen, 706ffa81fa4SYutaro Hayakawa optname == TLS_TX); 7073c4d7559SDave Watson break; 708b489a6e5SMaxim Mikityanskiy case TLS_TX_ZEROCOPY_RO: 709c1318b39SBoris Pismenny rc = do_tls_getsockopt_tx_zc(sk, optval, optlen); 710c1318b39SBoris Pismenny break; 71188527790SJakub Kicinski case TLS_RX_EXPECT_NO_PAD: 71288527790SJakub Kicinski rc = do_tls_getsockopt_no_pad(sk, optval, optlen); 71388527790SJakub Kicinski break; 7143c4d7559SDave Watson default: 7153c4d7559SDave Watson rc = -ENOPROTOOPT; 7163c4d7559SDave Watson break; 7173c4d7559SDave Watson } 71849c47cc2SHangyu Hua 71949c47cc2SHangyu Hua release_sock(sk); 72049c47cc2SHangyu Hua 7213c4d7559SDave Watson return rc; 7223c4d7559SDave Watson } 7233c4d7559SDave Watson 7243c4d7559SDave Watson static int tls_getsockopt(struct sock *sk, int level, int optname, 7253c4d7559SDave Watson char __user *optval, int __user *optlen) 7263c4d7559SDave Watson { 7273c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 7283c4d7559SDave Watson 7293c4d7559SDave Watson if (level != SOL_TLS) 730be7bbea1SJakub Kicinski return ctx->sk_proto->getsockopt(sk, level, 731be7bbea1SJakub Kicinski optname, optval, optlen); 7323c4d7559SDave Watson 7333c4d7559SDave Watson return do_tls_getsockopt(sk, optname, optval, optlen); 7343c4d7559SDave Watson } 7353c4d7559SDave Watson 736a7b75c5aSChristoph Hellwig static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval, 737c46234ebSDave Watson unsigned int optlen, int tx) 7383c4d7559SDave Watson { 739196c31b4SIlya Lesokhin struct tls_crypto_info *crypto_info; 7404509de14SVakul Garg struct tls_crypto_info *alt_crypto_info; 7413c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 742*5f309adeSSabrina Dubroca const struct tls_cipher_desc *cipher_desc; 7433c4d7559SDave Watson int rc = 0; 74458371585SDave Watson int conf; 7453c4d7559SDave Watson 7461ddcbfbfSZiyang Xuan if (sockptr_is_null(optval) || (optlen < sizeof(*crypto_info))) 7471ddcbfbfSZiyang Xuan return -EINVAL; 7483c4d7559SDave Watson 7494509de14SVakul Garg if (tx) { 75086029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 7514509de14SVakul Garg alt_crypto_info = &ctx->crypto_recv.info; 7524509de14SVakul Garg } else { 75386029d10SSabrina Dubroca crypto_info = &ctx->crypto_recv.info; 7544509de14SVakul Garg alt_crypto_info = &ctx->crypto_send.info; 7554509de14SVakul Garg } 756c46234ebSDave Watson 757196c31b4SIlya Lesokhin /* Currently we don't support set crypto info more than one time */ 7581ddcbfbfSZiyang Xuan if (TLS_CRYPTO_INFO_READY(crypto_info)) 7591ddcbfbfSZiyang Xuan return -EBUSY; 760196c31b4SIlya Lesokhin 761a7b75c5aSChristoph Hellwig rc = copy_from_sockptr(crypto_info, optval, sizeof(*crypto_info)); 7623c4d7559SDave Watson if (rc) { 7633c4d7559SDave Watson rc = -EFAULT; 764257082e6SBoris Pismenny goto err_crypto_info; 7653c4d7559SDave Watson } 7663c4d7559SDave Watson 7673c4d7559SDave Watson /* check version */ 768130b392cSDave Watson if (crypto_info->version != TLS_1_2_VERSION && 769130b392cSDave Watson crypto_info->version != TLS_1_3_VERSION) { 7704a5cdc60SValentin Vidic rc = -EINVAL; 771196c31b4SIlya Lesokhin goto err_crypto_info; 7723c4d7559SDave Watson } 7733c4d7559SDave Watson 7744509de14SVakul Garg /* Ensure that TLS version and ciphers are same in both directions */ 7754509de14SVakul Garg if (TLS_CRYPTO_INFO_READY(alt_crypto_info)) { 7764509de14SVakul Garg if (alt_crypto_info->version != crypto_info->version || 7774509de14SVakul Garg alt_crypto_info->cipher_type != crypto_info->cipher_type) { 7784509de14SVakul Garg rc = -EINVAL; 7794509de14SVakul Garg goto err_crypto_info; 7804509de14SVakul Garg } 7814509de14SVakul Garg } 7824509de14SVakul Garg 783*5f309adeSSabrina Dubroca cipher_desc = get_cipher_desc(crypto_info->cipher_type); 784*5f309adeSSabrina Dubroca if (!cipher_desc) { 78562e56ef5STaehee Yoo rc = -EINVAL; 78662e56ef5STaehee Yoo goto err_crypto_info; 78762e56ef5STaehee Yoo } 788*5f309adeSSabrina Dubroca 789*5f309adeSSabrina Dubroca switch (crypto_info->cipher_type) { 790*5f309adeSSabrina Dubroca case TLS_CIPHER_ARIA_GCM_128: 79162e56ef5STaehee Yoo case TLS_CIPHER_ARIA_GCM_256: 79262e56ef5STaehee Yoo if (crypto_info->version != TLS_1_2_VERSION) { 79362e56ef5STaehee Yoo rc = -EINVAL; 79462e56ef5STaehee Yoo goto err_crypto_info; 79562e56ef5STaehee Yoo } 79662e56ef5STaehee Yoo break; 797f295b3aeSVakul Garg } 798f295b3aeSVakul Garg 799*5f309adeSSabrina Dubroca if (optlen != cipher_desc->crypto_info) { 8003c4d7559SDave Watson rc = -EINVAL; 8016db959c8SSabrina Dubroca goto err_crypto_info; 8023c4d7559SDave Watson } 803f295b3aeSVakul Garg 804d3c48151SChristoph Hellwig rc = copy_from_sockptr_offset(crypto_info + 1, optval, 805d3c48151SChristoph Hellwig sizeof(*crypto_info), 806196c31b4SIlya Lesokhin optlen - sizeof(*crypto_info)); 8073c4d7559SDave Watson if (rc) { 8083c4d7559SDave Watson rc = -EFAULT; 8093c4d7559SDave Watson goto err_crypto_info; 8103c4d7559SDave Watson } 8113c4d7559SDave Watson 812c46234ebSDave Watson if (tx) { 813e8f69799SIlya Lesokhin rc = tls_set_device_offload(sk, ctx); 814e8f69799SIlya Lesokhin conf = TLS_HW; 815b32fd3ccSJakub Kicinski if (!rc) { 816b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE); 817b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE); 818b32fd3ccSJakub Kicinski } else { 819c46234ebSDave Watson rc = tls_set_sw_offload(sk, ctx, 1); 820318892acSJakub Kicinski if (rc) 821318892acSJakub Kicinski goto err_crypto_info; 822b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW); 823b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW); 824f66de3eeSBoris Pismenny conf = TLS_SW; 825e8f69799SIlya Lesokhin } 826c46234ebSDave Watson } else { 8274799ac81SBoris Pismenny rc = tls_set_device_offload_rx(sk, ctx); 8284799ac81SBoris Pismenny conf = TLS_HW; 829b32fd3ccSJakub Kicinski if (!rc) { 830b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE); 831b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE); 832b32fd3ccSJakub Kicinski } else { 833c46234ebSDave Watson rc = tls_set_sw_offload(sk, ctx, 0); 834318892acSJakub Kicinski if (rc) 835318892acSJakub Kicinski goto err_crypto_info; 836b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW); 837b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW); 838f66de3eeSBoris Pismenny conf = TLS_SW; 839c46234ebSDave Watson } 840313ab004SJohn Fastabend tls_sw_strparser_arm(sk, ctx); 8414799ac81SBoris Pismenny } 842c46234ebSDave Watson 843f66de3eeSBoris Pismenny if (tx) 844f66de3eeSBoris Pismenny ctx->tx_conf = conf; 845f66de3eeSBoris Pismenny else 846f66de3eeSBoris Pismenny ctx->rx_conf = conf; 8476d88207fSIlya Lesokhin update_sk_prot(sk, ctx); 848c46234ebSDave Watson if (tx) { 849ee181e52SIlya Lesokhin ctx->sk_write_space = sk->sk_write_space; 850ee181e52SIlya Lesokhin sk->sk_write_space = tls_write_space; 85184c61fe1SJakub Kicinski } else { 85284c61fe1SJakub Kicinski struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(ctx); 85384c61fe1SJakub Kicinski 85484c61fe1SJakub Kicinski tls_strp_check_rcv(&rx_ctx->strp); 855c46234ebSDave Watson } 8561ddcbfbfSZiyang Xuan return 0; 8573c4d7559SDave Watson 8583c4d7559SDave Watson err_crypto_info: 859c844eb46SSabrina Dubroca memzero_explicit(crypto_info, sizeof(union tls_crypto_context)); 8603c4d7559SDave Watson return rc; 8613c4d7559SDave Watson } 8623c4d7559SDave Watson 863c1318b39SBoris Pismenny static int do_tls_setsockopt_tx_zc(struct sock *sk, sockptr_t optval, 864c1318b39SBoris Pismenny unsigned int optlen) 865c1318b39SBoris Pismenny { 866c1318b39SBoris Pismenny struct tls_context *ctx = tls_get_ctx(sk); 867c1318b39SBoris Pismenny unsigned int value; 868c1318b39SBoris Pismenny 869c1318b39SBoris Pismenny if (sockptr_is_null(optval) || optlen != sizeof(value)) 870c1318b39SBoris Pismenny return -EINVAL; 871c1318b39SBoris Pismenny 872c1318b39SBoris Pismenny if (copy_from_sockptr(&value, optval, sizeof(value))) 873c1318b39SBoris Pismenny return -EFAULT; 874c1318b39SBoris Pismenny 875c1318b39SBoris Pismenny if (value > 1) 876c1318b39SBoris Pismenny return -EINVAL; 877c1318b39SBoris Pismenny 878c1318b39SBoris Pismenny ctx->zerocopy_sendfile = value; 879c1318b39SBoris Pismenny 880c1318b39SBoris Pismenny return 0; 881c1318b39SBoris Pismenny } 882c1318b39SBoris Pismenny 88388527790SJakub Kicinski static int do_tls_setsockopt_no_pad(struct sock *sk, sockptr_t optval, 88488527790SJakub Kicinski unsigned int optlen) 88588527790SJakub Kicinski { 88688527790SJakub Kicinski struct tls_context *ctx = tls_get_ctx(sk); 88788527790SJakub Kicinski u32 val; 88888527790SJakub Kicinski int rc; 88988527790SJakub Kicinski 89088527790SJakub Kicinski if (ctx->prot_info.version != TLS_1_3_VERSION || 89188527790SJakub Kicinski sockptr_is_null(optval) || optlen < sizeof(val)) 89288527790SJakub Kicinski return -EINVAL; 89388527790SJakub Kicinski 89488527790SJakub Kicinski rc = copy_from_sockptr(&val, optval, sizeof(val)); 89588527790SJakub Kicinski if (rc) 89688527790SJakub Kicinski return -EFAULT; 89788527790SJakub Kicinski if (val > 1) 89888527790SJakub Kicinski return -EINVAL; 89988527790SJakub Kicinski rc = check_zeroed_sockptr(optval, sizeof(val), optlen - sizeof(val)); 90088527790SJakub Kicinski if (rc < 1) 90188527790SJakub Kicinski return rc == 0 ? -EINVAL : rc; 90288527790SJakub Kicinski 90388527790SJakub Kicinski lock_sock(sk); 90488527790SJakub Kicinski rc = -EINVAL; 90588527790SJakub Kicinski if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) { 90688527790SJakub Kicinski ctx->rx_no_pad = val; 90788527790SJakub Kicinski tls_update_rx_zc_capable(ctx); 90888527790SJakub Kicinski rc = 0; 90988527790SJakub Kicinski } 91088527790SJakub Kicinski release_sock(sk); 91188527790SJakub Kicinski 91288527790SJakub Kicinski return rc; 91388527790SJakub Kicinski } 91488527790SJakub Kicinski 915a7b75c5aSChristoph Hellwig static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval, 916a7b75c5aSChristoph Hellwig unsigned int optlen) 9173c4d7559SDave Watson { 9183c4d7559SDave Watson int rc = 0; 9193c4d7559SDave Watson 9203c4d7559SDave Watson switch (optname) { 9213c4d7559SDave Watson case TLS_TX: 922c46234ebSDave Watson case TLS_RX: 9233c4d7559SDave Watson lock_sock(sk); 924c46234ebSDave Watson rc = do_tls_setsockopt_conf(sk, optval, optlen, 925c46234ebSDave Watson optname == TLS_TX); 9263c4d7559SDave Watson release_sock(sk); 9273c4d7559SDave Watson break; 928b489a6e5SMaxim Mikityanskiy case TLS_TX_ZEROCOPY_RO: 929c1318b39SBoris Pismenny lock_sock(sk); 930c1318b39SBoris Pismenny rc = do_tls_setsockopt_tx_zc(sk, optval, optlen); 931c1318b39SBoris Pismenny release_sock(sk); 932c1318b39SBoris Pismenny break; 93388527790SJakub Kicinski case TLS_RX_EXPECT_NO_PAD: 93488527790SJakub Kicinski rc = do_tls_setsockopt_no_pad(sk, optval, optlen); 93588527790SJakub Kicinski break; 9363c4d7559SDave Watson default: 9373c4d7559SDave Watson rc = -ENOPROTOOPT; 9383c4d7559SDave Watson break; 9393c4d7559SDave Watson } 9403c4d7559SDave Watson return rc; 9413c4d7559SDave Watson } 9423c4d7559SDave Watson 9433c4d7559SDave Watson static int tls_setsockopt(struct sock *sk, int level, int optname, 944a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 9453c4d7559SDave Watson { 9463c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 9473c4d7559SDave Watson 9483c4d7559SDave Watson if (level != SOL_TLS) 949be7bbea1SJakub Kicinski return ctx->sk_proto->setsockopt(sk, level, optname, optval, 950be7bbea1SJakub Kicinski optlen); 9513c4d7559SDave Watson 9523c4d7559SDave Watson return do_tls_setsockopt(sk, optname, optval, optlen); 9533c4d7559SDave Watson } 9543c4d7559SDave Watson 95508700dabSJakub Kicinski struct tls_context *tls_ctx_create(struct sock *sk) 956dd0bed16SAtul Gupta { 957dd0bed16SAtul Gupta struct inet_connection_sock *icsk = inet_csk(sk); 958dd0bed16SAtul Gupta struct tls_context *ctx; 959dd0bed16SAtul Gupta 960c6ec179aSGanesh Goudar ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC); 961dd0bed16SAtul Gupta if (!ctx) 962dd0bed16SAtul Gupta return NULL; 963dd0bed16SAtul Gupta 96479ffe608SJakub Kicinski mutex_init(&ctx->tx_lock); 96515a7dea7SJakub Kicinski rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 966d5bee737SJakub Sitnicki ctx->sk_proto = READ_ONCE(sk->sk_prot); 967c55dcdd4SMaxim Mikityanskiy ctx->sk = sk; 968dd0bed16SAtul Gupta return ctx; 969dd0bed16SAtul Gupta } 970dd0bed16SAtul Gupta 971f3911f73SJakub Kicinski static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 972f3911f73SJakub Kicinski const struct proto_ops *base) 973f3911f73SJakub Kicinski { 974f3911f73SJakub Kicinski ops[TLS_BASE][TLS_BASE] = *base; 975f3911f73SJakub Kicinski 976f3911f73SJakub Kicinski ops[TLS_SW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; 977df720d28SDavid Howells ops[TLS_SW ][TLS_BASE].splice_eof = tls_sw_splice_eof; 978f3911f73SJakub Kicinski 979f3911f73SJakub Kicinski ops[TLS_BASE][TLS_SW ] = ops[TLS_BASE][TLS_BASE]; 980f3911f73SJakub Kicinski ops[TLS_BASE][TLS_SW ].splice_read = tls_sw_splice_read; 981121dca78SJakub Kicinski ops[TLS_BASE][TLS_SW ].poll = tls_sk_poll; 982662fbcecSHannes Reinecke ops[TLS_BASE][TLS_SW ].read_sock = tls_sw_read_sock; 983f3911f73SJakub Kicinski 984f3911f73SJakub Kicinski ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE]; 985f3911f73SJakub Kicinski ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read; 986121dca78SJakub Kicinski ops[TLS_SW ][TLS_SW ].poll = tls_sk_poll; 987662fbcecSHannes Reinecke ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock; 988f3911f73SJakub Kicinski 989f3911f73SJakub Kicinski #ifdef CONFIG_TLS_DEVICE 990f3911f73SJakub Kicinski ops[TLS_HW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; 991f3911f73SJakub Kicinski 992f3911f73SJakub Kicinski ops[TLS_HW ][TLS_SW ] = ops[TLS_BASE][TLS_SW ]; 993f3911f73SJakub Kicinski 994f3911f73SJakub Kicinski ops[TLS_BASE][TLS_HW ] = ops[TLS_BASE][TLS_SW ]; 995f3911f73SJakub Kicinski 996f3911f73SJakub Kicinski ops[TLS_SW ][TLS_HW ] = ops[TLS_SW ][TLS_SW ]; 997f3911f73SJakub Kicinski 998f3911f73SJakub Kicinski ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ]; 999f3911f73SJakub Kicinski #endif 1000f3911f73SJakub Kicinski #ifdef CONFIG_TLS_TOE 1001f3911f73SJakub Kicinski ops[TLS_HW_RECORD][TLS_HW_RECORD] = *base; 1002f3911f73SJakub Kicinski #endif 1003f3911f73SJakub Kicinski } 1004f3911f73SJakub Kicinski 100563a6b3feSAtul Gupta static void tls_build_proto(struct sock *sk) 100663a6b3feSAtul Gupta { 100763a6b3feSAtul Gupta int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 10089a893949SWill Deacon struct proto *prot = READ_ONCE(sk->sk_prot); 100963a6b3feSAtul Gupta 101063a6b3feSAtul Gupta /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */ 101163a6b3feSAtul Gupta if (ip_ver == TLSV6 && 10125bb4c45dSJakub Sitnicki unlikely(prot != smp_load_acquire(&saved_tcpv6_prot))) { 101363a6b3feSAtul Gupta mutex_lock(&tcpv6_prot_mutex); 10145bb4c45dSJakub Sitnicki if (likely(prot != saved_tcpv6_prot)) { 10155bb4c45dSJakub Sitnicki build_protos(tls_prots[TLSV6], prot); 1016f3911f73SJakub Kicinski build_proto_ops(tls_proto_ops[TLSV6], 1017f3911f73SJakub Kicinski sk->sk_socket->ops); 10185bb4c45dSJakub Sitnicki smp_store_release(&saved_tcpv6_prot, prot); 101963a6b3feSAtul Gupta } 102063a6b3feSAtul Gupta mutex_unlock(&tcpv6_prot_mutex); 102163a6b3feSAtul Gupta } 102263a6b3feSAtul Gupta 102363a6b3feSAtul Gupta if (ip_ver == TLSV4 && 10245bb4c45dSJakub Sitnicki unlikely(prot != smp_load_acquire(&saved_tcpv4_prot))) { 102563a6b3feSAtul Gupta mutex_lock(&tcpv4_prot_mutex); 10265bb4c45dSJakub Sitnicki if (likely(prot != saved_tcpv4_prot)) { 10275bb4c45dSJakub Sitnicki build_protos(tls_prots[TLSV4], prot); 1028f3911f73SJakub Kicinski build_proto_ops(tls_proto_ops[TLSV4], 1029f3911f73SJakub Kicinski sk->sk_socket->ops); 10305bb4c45dSJakub Sitnicki smp_store_release(&saved_tcpv4_prot, prot); 103163a6b3feSAtul Gupta } 103263a6b3feSAtul Gupta mutex_unlock(&tcpv4_prot_mutex); 103363a6b3feSAtul Gupta } 103463a6b3feSAtul Gupta } 103563a6b3feSAtul Gupta 1036f66de3eeSBoris Pismenny static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 1037f13fe3e6SJakub Sitnicki const struct proto *base) 1038c113187dSBoris Pismenny { 1039f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE] = *base; 1040f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt; 1041f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].getsockopt = tls_getsockopt; 1042f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].close = tls_sk_proto_close; 1043c113187dSBoris Pismenny 1044f66de3eeSBoris Pismenny prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 1045f66de3eeSBoris Pismenny prot[TLS_SW][TLS_BASE].sendmsg = tls_sw_sendmsg; 1046df720d28SDavid Howells prot[TLS_SW][TLS_BASE].splice_eof = tls_sw_splice_eof; 1047c46234ebSDave Watson 1048f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE]; 1049f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg; 10507b50ecfcSCong Wang prot[TLS_BASE][TLS_SW].sock_is_readable = tls_sw_sock_is_readable; 1051f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close; 1052c46234ebSDave Watson 1053f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE]; 1054f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg; 10557b50ecfcSCong Wang prot[TLS_SW][TLS_SW].sock_is_readable = tls_sw_sock_is_readable; 1056f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW].close = tls_sk_proto_close; 1057dd0bed16SAtul Gupta 1058e8f69799SIlya Lesokhin #ifdef CONFIG_TLS_DEVICE 1059e8f69799SIlya Lesokhin prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 1060e8f69799SIlya Lesokhin prot[TLS_HW][TLS_BASE].sendmsg = tls_device_sendmsg; 1061d4c1e80bSDavid Howells prot[TLS_HW][TLS_BASE].splice_eof = tls_device_splice_eof; 1062e8f69799SIlya Lesokhin 1063e8f69799SIlya Lesokhin prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW]; 1064e8f69799SIlya Lesokhin prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg; 1065d4c1e80bSDavid Howells prot[TLS_HW][TLS_SW].splice_eof = tls_device_splice_eof; 10664799ac81SBoris Pismenny 10674799ac81SBoris Pismenny prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW]; 10684799ac81SBoris Pismenny 10694799ac81SBoris Pismenny prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW]; 10704799ac81SBoris Pismenny 10714799ac81SBoris Pismenny prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW]; 1072e8f69799SIlya Lesokhin #endif 107353b4414aSJakub Kicinski #ifdef CONFIG_TLS_TOE 1074f66de3eeSBoris Pismenny prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base; 10750eb8745eSJakub Kicinski prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_toe_hash; 10760eb8745eSJakub Kicinski prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_toe_unhash; 107753b4414aSJakub Kicinski #endif 1078c113187dSBoris Pismenny } 1079c113187dSBoris Pismenny 10803c4d7559SDave Watson static int tls_init(struct sock *sk) 10813c4d7559SDave Watson { 10823c4d7559SDave Watson struct tls_context *ctx; 10833c4d7559SDave Watson int rc = 0; 10843c4d7559SDave Watson 108516bed0e6SJakub Kicinski tls_build_proto(sk); 108616bed0e6SJakub Kicinski 108753b4414aSJakub Kicinski #ifdef CONFIG_TLS_TOE 10880eb8745eSJakub Kicinski if (tls_toe_bypass(sk)) 108995fa1454SJohn Fastabend return 0; 109053b4414aSJakub Kicinski #endif 1091dd0bed16SAtul Gupta 1092d91c3e17SIlya Lesokhin /* The TLS ulp is currently supported only for TCP sockets 1093d91c3e17SIlya Lesokhin * in ESTABLISHED state. 1094d91c3e17SIlya Lesokhin * Supporting sockets in LISTEN state will require us 1095d91c3e17SIlya Lesokhin * to modify the accept implementation to clone rather then 1096d91c3e17SIlya Lesokhin * share the ulp context. 1097d91c3e17SIlya Lesokhin */ 1098d91c3e17SIlya Lesokhin if (sk->sk_state != TCP_ESTABLISHED) 10994a5cdc60SValentin Vidic return -ENOTCONN; 1100d91c3e17SIlya Lesokhin 11013c4d7559SDave Watson /* allocate tls context */ 110295fa1454SJohn Fastabend write_lock_bh(&sk->sk_callback_lock); 110308700dabSJakub Kicinski ctx = tls_ctx_create(sk); 11043c4d7559SDave Watson if (!ctx) { 11053c4d7559SDave Watson rc = -ENOMEM; 11063c4d7559SDave Watson goto out; 11073c4d7559SDave Watson } 11086d88207fSIlya Lesokhin 1109f66de3eeSBoris Pismenny ctx->tx_conf = TLS_BASE; 1110f66de3eeSBoris Pismenny ctx->rx_conf = TLS_BASE; 11116d88207fSIlya Lesokhin update_sk_prot(sk, ctx); 11123c4d7559SDave Watson out: 111395fa1454SJohn Fastabend write_unlock_bh(&sk->sk_callback_lock); 11143c4d7559SDave Watson return rc; 11153c4d7559SDave Watson } 11163c4d7559SDave Watson 111733bfe20dSJohn Fastabend static void tls_update(struct sock *sk, struct proto *p, 111833bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)) 111995fa1454SJohn Fastabend { 112095fa1454SJohn Fastabend struct tls_context *ctx; 112195fa1454SJohn Fastabend 1122e34a07c0SJakub Kicinski WARN_ON_ONCE(sk->sk_prot == p); 1123e34a07c0SJakub Kicinski 112495fa1454SJohn Fastabend ctx = tls_get_ctx(sk); 112533bfe20dSJohn Fastabend if (likely(ctx)) { 112633bfe20dSJohn Fastabend ctx->sk_write_space = write_space; 112795fa1454SJohn Fastabend ctx->sk_proto = p; 112833bfe20dSJohn Fastabend } else { 1129b8e202d1SJakub Sitnicki /* Pairs with lockless read in sk_clone_lock(). */ 1130b8e202d1SJakub Sitnicki WRITE_ONCE(sk->sk_prot, p); 113133bfe20dSJohn Fastabend sk->sk_write_space = write_space; 113233bfe20dSJohn Fastabend } 113395fa1454SJohn Fastabend } 113495fa1454SJohn Fastabend 113558790314SJakub Kicinski static u16 tls_user_config(struct tls_context *ctx, bool tx) 113658790314SJakub Kicinski { 113758790314SJakub Kicinski u16 config = tx ? ctx->tx_conf : ctx->rx_conf; 113858790314SJakub Kicinski 113958790314SJakub Kicinski switch (config) { 114058790314SJakub Kicinski case TLS_BASE: 114158790314SJakub Kicinski return TLS_CONF_BASE; 114258790314SJakub Kicinski case TLS_SW: 114358790314SJakub Kicinski return TLS_CONF_SW; 114458790314SJakub Kicinski case TLS_HW: 114558790314SJakub Kicinski return TLS_CONF_HW; 114658790314SJakub Kicinski case TLS_HW_RECORD: 114758790314SJakub Kicinski return TLS_CONF_HW_RECORD; 114858790314SJakub Kicinski } 114958790314SJakub Kicinski return 0; 115058790314SJakub Kicinski } 115158790314SJakub Kicinski 115226811cc9SDavide Caratti static int tls_get_info(const struct sock *sk, struct sk_buff *skb) 115326811cc9SDavide Caratti { 115426811cc9SDavide Caratti u16 version, cipher_type; 115526811cc9SDavide Caratti struct tls_context *ctx; 115626811cc9SDavide Caratti struct nlattr *start; 115726811cc9SDavide Caratti int err; 115826811cc9SDavide Caratti 115926811cc9SDavide Caratti start = nla_nest_start_noflag(skb, INET_ULP_INFO_TLS); 116026811cc9SDavide Caratti if (!start) 116126811cc9SDavide Caratti return -EMSGSIZE; 116226811cc9SDavide Caratti 116326811cc9SDavide Caratti rcu_read_lock(); 116426811cc9SDavide Caratti ctx = rcu_dereference(inet_csk(sk)->icsk_ulp_data); 116526811cc9SDavide Caratti if (!ctx) { 116626811cc9SDavide Caratti err = 0; 116726811cc9SDavide Caratti goto nla_failure; 116826811cc9SDavide Caratti } 116926811cc9SDavide Caratti version = ctx->prot_info.version; 117026811cc9SDavide Caratti if (version) { 117126811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_VERSION, version); 117226811cc9SDavide Caratti if (err) 117326811cc9SDavide Caratti goto nla_failure; 117426811cc9SDavide Caratti } 117526811cc9SDavide Caratti cipher_type = ctx->prot_info.cipher_type; 117626811cc9SDavide Caratti if (cipher_type) { 117726811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_CIPHER, cipher_type); 117826811cc9SDavide Caratti if (err) 117926811cc9SDavide Caratti goto nla_failure; 118026811cc9SDavide Caratti } 118126811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_TXCONF, tls_user_config(ctx, true)); 118226811cc9SDavide Caratti if (err) 118326811cc9SDavide Caratti goto nla_failure; 118426811cc9SDavide Caratti 118526811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_RXCONF, tls_user_config(ctx, false)); 118626811cc9SDavide Caratti if (err) 118726811cc9SDavide Caratti goto nla_failure; 118826811cc9SDavide Caratti 1189c1318b39SBoris Pismenny if (ctx->tx_conf == TLS_HW && ctx->zerocopy_sendfile) { 1190b489a6e5SMaxim Mikityanskiy err = nla_put_flag(skb, TLS_INFO_ZC_RO_TX); 1191c1318b39SBoris Pismenny if (err) 1192c1318b39SBoris Pismenny goto nla_failure; 1193c1318b39SBoris Pismenny } 119488527790SJakub Kicinski if (ctx->rx_no_pad) { 119588527790SJakub Kicinski err = nla_put_flag(skb, TLS_INFO_RX_NO_PAD); 119688527790SJakub Kicinski if (err) 119788527790SJakub Kicinski goto nla_failure; 119888527790SJakub Kicinski } 1199c1318b39SBoris Pismenny 120026811cc9SDavide Caratti rcu_read_unlock(); 120126811cc9SDavide Caratti nla_nest_end(skb, start); 120226811cc9SDavide Caratti return 0; 120326811cc9SDavide Caratti 120426811cc9SDavide Caratti nla_failure: 120526811cc9SDavide Caratti rcu_read_unlock(); 120626811cc9SDavide Caratti nla_nest_cancel(skb, start); 120726811cc9SDavide Caratti return err; 120826811cc9SDavide Caratti } 120926811cc9SDavide Caratti 121026811cc9SDavide Caratti static size_t tls_get_info_size(const struct sock *sk) 121126811cc9SDavide Caratti { 121226811cc9SDavide Caratti size_t size = 0; 121326811cc9SDavide Caratti 121426811cc9SDavide Caratti size += nla_total_size(0) + /* INET_ULP_INFO_TLS */ 121526811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_VERSION */ 121626811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_CIPHER */ 121726811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_RXCONF */ 121826811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_TXCONF */ 1219b489a6e5SMaxim Mikityanskiy nla_total_size(0) + /* TLS_INFO_ZC_RO_TX */ 122088527790SJakub Kicinski nla_total_size(0) + /* TLS_INFO_RX_NO_PAD */ 122126811cc9SDavide Caratti 0; 122226811cc9SDavide Caratti 122326811cc9SDavide Caratti return size; 122426811cc9SDavide Caratti } 122526811cc9SDavide Caratti 1226d26b698dSJakub Kicinski static int __net_init tls_init_net(struct net *net) 1227d26b698dSJakub Kicinski { 1228d26b698dSJakub Kicinski int err; 1229d26b698dSJakub Kicinski 1230d26b698dSJakub Kicinski net->mib.tls_statistics = alloc_percpu(struct linux_tls_mib); 1231d26b698dSJakub Kicinski if (!net->mib.tls_statistics) 1232d26b698dSJakub Kicinski return -ENOMEM; 1233d26b698dSJakub Kicinski 1234d26b698dSJakub Kicinski err = tls_proc_init(net); 1235d26b698dSJakub Kicinski if (err) 1236d26b698dSJakub Kicinski goto err_free_stats; 1237d26b698dSJakub Kicinski 1238d26b698dSJakub Kicinski return 0; 1239d26b698dSJakub Kicinski err_free_stats: 1240d26b698dSJakub Kicinski free_percpu(net->mib.tls_statistics); 1241d26b698dSJakub Kicinski return err; 1242d26b698dSJakub Kicinski } 1243d26b698dSJakub Kicinski 1244d26b698dSJakub Kicinski static void __net_exit tls_exit_net(struct net *net) 1245d26b698dSJakub Kicinski { 1246d26b698dSJakub Kicinski tls_proc_fini(net); 1247d26b698dSJakub Kicinski free_percpu(net->mib.tls_statistics); 1248d26b698dSJakub Kicinski } 1249d26b698dSJakub Kicinski 1250d26b698dSJakub Kicinski static struct pernet_operations tls_proc_ops = { 1251d26b698dSJakub Kicinski .init = tls_init_net, 1252d26b698dSJakub Kicinski .exit = tls_exit_net, 1253d26b698dSJakub Kicinski }; 1254d26b698dSJakub Kicinski 12553c4d7559SDave Watson static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = { 12563c4d7559SDave Watson .name = "tls", 12573c4d7559SDave Watson .owner = THIS_MODULE, 12583c4d7559SDave Watson .init = tls_init, 125995fa1454SJohn Fastabend .update = tls_update, 126026811cc9SDavide Caratti .get_info = tls_get_info, 126126811cc9SDavide Caratti .get_info_size = tls_get_info_size, 12623c4d7559SDave Watson }; 12633c4d7559SDave Watson 12643c4d7559SDave Watson static int __init tls_register(void) 12653c4d7559SDave Watson { 1266d26b698dSJakub Kicinski int err; 1267d26b698dSJakub Kicinski 1268d26b698dSJakub Kicinski err = register_pernet_subsys(&tls_proc_ops); 1269d26b698dSJakub Kicinski if (err) 1270d26b698dSJakub Kicinski return err; 1271d26b698dSJakub Kicinski 127284c61fe1SJakub Kicinski err = tls_strp_dev_init(); 127384c61fe1SJakub Kicinski if (err) 127484c61fe1SJakub Kicinski goto err_pernet; 127584c61fe1SJakub Kicinski 12763d8c51b2STariq Toukan err = tls_device_init(); 127784c61fe1SJakub Kicinski if (err) 127884c61fe1SJakub Kicinski goto err_strp; 12793d8c51b2STariq Toukan 12803c4d7559SDave Watson tcp_register_ulp(&tcp_tls_ulp_ops); 12813c4d7559SDave Watson 12823c4d7559SDave Watson return 0; 128384c61fe1SJakub Kicinski err_strp: 128484c61fe1SJakub Kicinski tls_strp_dev_exit(); 128584c61fe1SJakub Kicinski err_pernet: 128684c61fe1SJakub Kicinski unregister_pernet_subsys(&tls_proc_ops); 128784c61fe1SJakub Kicinski return err; 12883c4d7559SDave Watson } 12893c4d7559SDave Watson 12903c4d7559SDave Watson static void __exit tls_unregister(void) 12913c4d7559SDave Watson { 12923c4d7559SDave Watson tcp_unregister_ulp(&tcp_tls_ulp_ops); 129384c61fe1SJakub Kicinski tls_strp_dev_exit(); 1294e8f69799SIlya Lesokhin tls_device_cleanup(); 1295d26b698dSJakub Kicinski unregister_pernet_subsys(&tls_proc_ops); 12963c4d7559SDave Watson } 12973c4d7559SDave Watson 12983c4d7559SDave Watson module_init(tls_register); 12993c4d7559SDave Watson module_exit(tls_unregister); 1300