13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * 53c4d7559SDave Watson * This software is available to you under a choice of one of two 63c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 73c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 83c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 93c4d7559SDave Watson * OpenIB.org BSD license below: 103c4d7559SDave Watson * 113c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 123c4d7559SDave Watson * without modification, are permitted provided that the following 133c4d7559SDave Watson * conditions are met: 143c4d7559SDave Watson * 153c4d7559SDave Watson * - Redistributions of source code must retain the above 163c4d7559SDave Watson * copyright notice, this list of conditions and the following 173c4d7559SDave Watson * disclaimer. 183c4d7559SDave Watson * 193c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 203c4d7559SDave Watson * copyright notice, this list of conditions and the following 213c4d7559SDave Watson * disclaimer in the documentation and/or other materials 223c4d7559SDave Watson * provided with the distribution. 233c4d7559SDave Watson * 243c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 253c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 263c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 273c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 283c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 293c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 303c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 313c4d7559SDave Watson * SOFTWARE. 323c4d7559SDave Watson */ 333c4d7559SDave Watson 343c4d7559SDave Watson #include <linux/module.h> 353c4d7559SDave Watson 363c4d7559SDave Watson #include <net/tcp.h> 373c4d7559SDave Watson #include <net/inet_common.h> 383c4d7559SDave Watson #include <linux/highmem.h> 393c4d7559SDave Watson #include <linux/netdevice.h> 403c4d7559SDave Watson #include <linux/sched/signal.h> 41dd0bed16SAtul Gupta #include <linux/inetdevice.h> 4226811cc9SDavide Caratti #include <linux/inet_diag.h> 433c4d7559SDave Watson 44d26b698dSJakub Kicinski #include <net/snmp.h> 453c4d7559SDave Watson #include <net/tls.h> 4625a3cd81SJakub Kicinski #include <net/tls_toe.h> 473c4d7559SDave Watson 4858790314SJakub Kicinski #include "tls.h" 4958790314SJakub Kicinski 503c4d7559SDave Watson MODULE_AUTHOR("Mellanox Technologies"); 513c4d7559SDave Watson MODULE_DESCRIPTION("Transport Layer Security Support"); 523c4d7559SDave Watson MODULE_LICENSE("Dual BSD/GPL"); 53037b0b86SDaniel Borkmann MODULE_ALIAS_TCP_ULP("tls"); 543c4d7559SDave Watson 556d88207fSIlya Lesokhin enum { 56c113187dSBoris Pismenny TLSV4, 57c113187dSBoris Pismenny TLSV6, 58c113187dSBoris Pismenny TLS_NUM_PROTS, 59c113187dSBoris Pismenny }; 606d88207fSIlya Lesokhin 610d98cc02SSabrina Dubroca #define CHECK_CIPHER_DESC(cipher,ci) \ 620d98cc02SSabrina Dubroca static_assert(cipher ## _IV_SIZE <= MAX_IV_SIZE); \ 630d98cc02SSabrina Dubroca static_assert(cipher ## _REC_SEQ_SIZE <= TLS_MAX_REC_SEQ_SIZE); \ 640d98cc02SSabrina Dubroca static_assert(cipher ## _TAG_SIZE == TLS_TAG_SIZE); \ 650d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, iv) == cipher ## _IV_SIZE); \ 660d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, key) == cipher ## _KEY_SIZE); \ 670d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, salt) == cipher ## _SALT_SIZE); \ 680d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, rec_seq) == cipher ## _REC_SEQ_SIZE); 690d98cc02SSabrina Dubroca 70176a3f50SSabrina Dubroca #define __CIPHER_DESC(ci) \ 71176a3f50SSabrina Dubroca .iv_offset = offsetof(struct ci, iv), \ 72176a3f50SSabrina Dubroca .key_offset = offsetof(struct ci, key), \ 73176a3f50SSabrina Dubroca .salt_offset = offsetof(struct ci, salt), \ 74176a3f50SSabrina Dubroca .rec_seq_offset = offsetof(struct ci, rec_seq), \ 75176a3f50SSabrina Dubroca .crypto_info = sizeof(struct ci) 76176a3f50SSabrina Dubroca 77176a3f50SSabrina Dubroca #define CIPHER_DESC(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \ 78176a3f50SSabrina Dubroca .nonce = cipher ## _IV_SIZE, \ 792d2c5ea2STariq Toukan .iv = cipher ## _IV_SIZE, \ 802d2c5ea2STariq Toukan .key = cipher ## _KEY_SIZE, \ 812d2c5ea2STariq Toukan .salt = cipher ## _SALT_SIZE, \ 822d2c5ea2STariq Toukan .tag = cipher ## _TAG_SIZE, \ 832d2c5ea2STariq Toukan .rec_seq = cipher ## _REC_SEQ_SIZE, \ 84176a3f50SSabrina Dubroca .cipher_name = algname, \ 85176a3f50SSabrina Dubroca .offloadable = _offloadable, \ 86176a3f50SSabrina Dubroca __CIPHER_DESC(ci), \ 87176a3f50SSabrina Dubroca } 88176a3f50SSabrina Dubroca 89176a3f50SSabrina Dubroca #define CIPHER_DESC_NONCE0(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \ 90176a3f50SSabrina Dubroca .nonce = 0, \ 91176a3f50SSabrina Dubroca .iv = cipher ## _IV_SIZE, \ 92176a3f50SSabrina Dubroca .key = cipher ## _KEY_SIZE, \ 93176a3f50SSabrina Dubroca .salt = cipher ## _SALT_SIZE, \ 94176a3f50SSabrina Dubroca .tag = cipher ## _TAG_SIZE, \ 95176a3f50SSabrina Dubroca .rec_seq = cipher ## _REC_SEQ_SIZE, \ 96176a3f50SSabrina Dubroca .cipher_name = algname, \ 97176a3f50SSabrina Dubroca .offloadable = _offloadable, \ 98176a3f50SSabrina Dubroca __CIPHER_DESC(ci), \ 992d2c5ea2STariq Toukan } 1002d2c5ea2STariq Toukan 1018db44ab2SSabrina Dubroca const struct tls_cipher_desc tls_cipher_desc[TLS_CIPHER_MAX + 1 - TLS_CIPHER_MIN] = { 102176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128, "gcm(aes)", true), 103176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256, "gcm(aes)", true), 104176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128, "ccm(aes)", false), 105176a3f50SSabrina Dubroca CIPHER_DESC_NONCE0(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305, "rfc7539(chacha20,poly1305)", false), 106176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm, "gcm(sm4)", false), 107176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm, "ccm(sm4)", false), 108176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128, "gcm(aria)", false), 109176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256, "gcm(aria)", false), 1102d2c5ea2STariq Toukan }; 1112d2c5ea2STariq Toukan 1120d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128); 1130d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256); 1140d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128); 1150d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305); 1160d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm); 1170d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm); 1180d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128); 1190d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256); 1200d98cc02SSabrina Dubroca 121f691a25cSArnd Bergmann static const struct proto *saved_tcpv6_prot; 122c113187dSBoris Pismenny static DEFINE_MUTEX(tcpv6_prot_mutex); 123f691a25cSArnd Bergmann static const struct proto *saved_tcpv4_prot; 12428cb6f1eSJohn Fastabend static DEFINE_MUTEX(tcpv4_prot_mutex); 125f66de3eeSBoris Pismenny static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 126f3911f73SJakub Kicinski static struct proto_ops tls_proto_ops[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 12763a6b3feSAtul Gupta static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 128f13fe3e6SJakub Sitnicki const struct proto *base); 1296d88207fSIlya Lesokhin 13008700dabSJakub Kicinski void update_sk_prot(struct sock *sk, struct tls_context *ctx) 1316d88207fSIlya Lesokhin { 132c113187dSBoris Pismenny int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 133c113187dSBoris Pismenny 134d5bee737SJakub Sitnicki WRITE_ONCE(sk->sk_prot, 135d5bee737SJakub Sitnicki &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]); 136f3911f73SJakub Kicinski WRITE_ONCE(sk->sk_socket->ops, 137f3911f73SJakub Kicinski &tls_proto_ops[ip_ver][ctx->tx_conf][ctx->rx_conf]); 1386d88207fSIlya Lesokhin } 1393c4d7559SDave Watson 1403c4d7559SDave Watson int wait_on_pending_writer(struct sock *sk, long *timeo) 1413c4d7559SDave Watson { 1423c4d7559SDave Watson DEFINE_WAIT_FUNC(wait, woken_wake_function); 143419ce133SPaolo Abeni int ret, rc = 0; 1443c4d7559SDave Watson 1453c4d7559SDave Watson add_wait_queue(sk_sleep(sk), &wait); 1463c4d7559SDave Watson while (1) { 1473c4d7559SDave Watson if (!*timeo) { 1483c4d7559SDave Watson rc = -EAGAIN; 1493c4d7559SDave Watson break; 1503c4d7559SDave Watson } 1513c4d7559SDave Watson 1523c4d7559SDave Watson if (signal_pending(current)) { 1533c4d7559SDave Watson rc = sock_intr_errno(*timeo); 1543c4d7559SDave Watson break; 1553c4d7559SDave Watson } 1563c4d7559SDave Watson 157419ce133SPaolo Abeni ret = sk_wait_event(sk, timeo, 158419ce133SPaolo Abeni !READ_ONCE(sk->sk_write_pending), &wait); 159419ce133SPaolo Abeni if (ret) { 160419ce133SPaolo Abeni if (ret < 0) 161419ce133SPaolo Abeni rc = ret; 1623c4d7559SDave Watson break; 1633c4d7559SDave Watson } 164419ce133SPaolo Abeni } 1653c4d7559SDave Watson remove_wait_queue(sk_sleep(sk), &wait); 1663c4d7559SDave Watson return rc; 1673c4d7559SDave Watson } 1683c4d7559SDave Watson 1693c4d7559SDave Watson int tls_push_sg(struct sock *sk, 1703c4d7559SDave Watson struct tls_context *ctx, 1713c4d7559SDave Watson struct scatterlist *sg, 1723c4d7559SDave Watson u16 first_offset, 1733c4d7559SDave Watson int flags) 1743c4d7559SDave Watson { 175e117dcfdSDavid Howells struct bio_vec bvec; 176e117dcfdSDavid Howells struct msghdr msg = { 177b848b26cSDavid Howells .msg_flags = MSG_SPLICE_PAGES | flags, 178e117dcfdSDavid Howells }; 1793c4d7559SDave Watson int ret = 0; 1803c4d7559SDave Watson struct page *p; 1813c4d7559SDave Watson size_t size; 1823c4d7559SDave Watson int offset = first_offset; 1833c4d7559SDave Watson 1843c4d7559SDave Watson size = sg->length - offset; 1853c4d7559SDave Watson offset += sg->offset; 1863c4d7559SDave Watson 187e117dcfdSDavid Howells ctx->splicing_pages = true; 1883c4d7559SDave Watson while (1) { 1893c4d7559SDave Watson /* is sending application-limited? */ 1903c4d7559SDave Watson tcp_rate_check_app_limited(sk); 1913c4d7559SDave Watson p = sg_page(sg); 1923c4d7559SDave Watson retry: 193e117dcfdSDavid Howells bvec_set_page(&bvec, p, size, offset); 194e117dcfdSDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size); 195e117dcfdSDavid Howells 196e117dcfdSDavid Howells ret = tcp_sendmsg_locked(sk, &msg, size); 1973c4d7559SDave Watson 1983c4d7559SDave Watson if (ret != size) { 1993c4d7559SDave Watson if (ret > 0) { 2003c4d7559SDave Watson offset += ret; 2013c4d7559SDave Watson size -= ret; 2023c4d7559SDave Watson goto retry; 2033c4d7559SDave Watson } 2043c4d7559SDave Watson 2053c4d7559SDave Watson offset -= sg->offset; 2063c4d7559SDave Watson ctx->partially_sent_offset = offset; 2073c4d7559SDave Watson ctx->partially_sent_record = (void *)sg; 208e117dcfdSDavid Howells ctx->splicing_pages = false; 2093c4d7559SDave Watson return ret; 2103c4d7559SDave Watson } 2113c4d7559SDave Watson 2123c4d7559SDave Watson put_page(p); 2133c4d7559SDave Watson sk_mem_uncharge(sk, sg->length); 2143c4d7559SDave Watson sg = sg_next(sg); 2153c4d7559SDave Watson if (!sg) 2163c4d7559SDave Watson break; 2173c4d7559SDave Watson 2183c4d7559SDave Watson offset = sg->offset; 2193c4d7559SDave Watson size = sg->length; 2203c4d7559SDave Watson } 2213c4d7559SDave Watson 222e117dcfdSDavid Howells ctx->splicing_pages = false; 2233c4d7559SDave Watson 2243c4d7559SDave Watson return 0; 2253c4d7559SDave Watson } 2263c4d7559SDave Watson 2273c4d7559SDave Watson static int tls_handle_open_record(struct sock *sk, int flags) 2283c4d7559SDave Watson { 2293c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 2303c4d7559SDave Watson 2313c4d7559SDave Watson if (tls_is_pending_open_record(ctx)) 2323c4d7559SDave Watson return ctx->push_pending_record(sk, flags); 2333c4d7559SDave Watson 2343c4d7559SDave Watson return 0; 2353c4d7559SDave Watson } 2363c4d7559SDave Watson 23758790314SJakub Kicinski int tls_process_cmsg(struct sock *sk, struct msghdr *msg, 2383c4d7559SDave Watson unsigned char *record_type) 2393c4d7559SDave Watson { 2403c4d7559SDave Watson struct cmsghdr *cmsg; 2413c4d7559SDave Watson int rc = -EINVAL; 2423c4d7559SDave Watson 2433c4d7559SDave Watson for_each_cmsghdr(cmsg, msg) { 2443c4d7559SDave Watson if (!CMSG_OK(msg, cmsg)) 2453c4d7559SDave Watson return -EINVAL; 2463c4d7559SDave Watson if (cmsg->cmsg_level != SOL_TLS) 2473c4d7559SDave Watson continue; 2483c4d7559SDave Watson 2493c4d7559SDave Watson switch (cmsg->cmsg_type) { 2503c4d7559SDave Watson case TLS_SET_RECORD_TYPE: 2513c4d7559SDave Watson if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) 2523c4d7559SDave Watson return -EINVAL; 2533c4d7559SDave Watson 2543c4d7559SDave Watson if (msg->msg_flags & MSG_MORE) 2553c4d7559SDave Watson return -EINVAL; 2563c4d7559SDave Watson 2573c4d7559SDave Watson rc = tls_handle_open_record(sk, msg->msg_flags); 2583c4d7559SDave Watson if (rc) 2593c4d7559SDave Watson return rc; 2603c4d7559SDave Watson 2613c4d7559SDave Watson *record_type = *(unsigned char *)CMSG_DATA(cmsg); 2623c4d7559SDave Watson rc = 0; 2633c4d7559SDave Watson break; 2643c4d7559SDave Watson default: 2653c4d7559SDave Watson return -EINVAL; 2663c4d7559SDave Watson } 2673c4d7559SDave Watson } 2683c4d7559SDave Watson 2693c4d7559SDave Watson return rc; 2703c4d7559SDave Watson } 2713c4d7559SDave Watson 272a42055e8SVakul Garg int tls_push_partial_record(struct sock *sk, struct tls_context *ctx, 273a42055e8SVakul Garg int flags) 2743c4d7559SDave Watson { 2753c4d7559SDave Watson struct scatterlist *sg; 2763c4d7559SDave Watson u16 offset; 2773c4d7559SDave Watson 2783c4d7559SDave Watson sg = ctx->partially_sent_record; 2793c4d7559SDave Watson offset = ctx->partially_sent_offset; 2803c4d7559SDave Watson 2813c4d7559SDave Watson ctx->partially_sent_record = NULL; 2823c4d7559SDave Watson return tls_push_sg(sk, ctx, sg, offset, flags); 2833c4d7559SDave Watson } 2843c4d7559SDave Watson 285c5daa6ccSJakub Kicinski void tls_free_partial_record(struct sock *sk, struct tls_context *ctx) 28635b71a34SJakub Kicinski { 28735b71a34SJakub Kicinski struct scatterlist *sg; 28835b71a34SJakub Kicinski 289c5daa6ccSJakub Kicinski for (sg = ctx->partially_sent_record; sg; sg = sg_next(sg)) { 29035b71a34SJakub Kicinski put_page(sg_page(sg)); 29135b71a34SJakub Kicinski sk_mem_uncharge(sk, sg->length); 29235b71a34SJakub Kicinski } 29335b71a34SJakub Kicinski ctx->partially_sent_record = NULL; 29435b71a34SJakub Kicinski } 29535b71a34SJakub Kicinski 2963c4d7559SDave Watson static void tls_write_space(struct sock *sk) 2973c4d7559SDave Watson { 2983c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 2993c4d7559SDave Watson 300e117dcfdSDavid Howells /* If splicing_pages call lower protocol write space handler 30167db7cd2SJohn Fastabend * to ensure we wake up any waiting operations there. For example 302e117dcfdSDavid Howells * if splicing pages where to call sk_wait_event. 30367db7cd2SJohn Fastabend */ 304e117dcfdSDavid Howells if (ctx->splicing_pages) { 30567db7cd2SJohn Fastabend ctx->sk_write_space(sk); 306c212d2c7SDave Watson return; 30767db7cd2SJohn Fastabend } 308c212d2c7SDave Watson 3097463d3a2SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 3107463d3a2SBoris Pismenny if (ctx->tx_conf == TLS_HW) 3117463d3a2SBoris Pismenny tls_device_write_space(sk, ctx); 3127463d3a2SBoris Pismenny else 3137463d3a2SBoris Pismenny #endif 3147463d3a2SBoris Pismenny tls_sw_write_space(sk, ctx); 3154504ab0eSVakul Garg 3164504ab0eSVakul Garg ctx->sk_write_space(sk); 3173c4d7559SDave Watson } 3183c4d7559SDave Watson 31915a7dea7SJakub Kicinski /** 32015a7dea7SJakub Kicinski * tls_ctx_free() - free TLS ULP context 32115a7dea7SJakub Kicinski * @sk: socket to with @ctx is attached 32215a7dea7SJakub Kicinski * @ctx: TLS context structure 32315a7dea7SJakub Kicinski * 32415a7dea7SJakub Kicinski * Free TLS context. If @sk is %NULL caller guarantees that the socket 32515a7dea7SJakub Kicinski * to which @ctx was attached has no outstanding references. 32615a7dea7SJakub Kicinski */ 32715a7dea7SJakub Kicinski void tls_ctx_free(struct sock *sk, struct tls_context *ctx) 32886029d10SSabrina Dubroca { 32986029d10SSabrina Dubroca if (!ctx) 33086029d10SSabrina Dubroca return; 33186029d10SSabrina Dubroca 33286029d10SSabrina Dubroca memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send)); 33386029d10SSabrina Dubroca memzero_explicit(&ctx->crypto_recv, sizeof(ctx->crypto_recv)); 33479ffe608SJakub Kicinski mutex_destroy(&ctx->tx_lock); 33515a7dea7SJakub Kicinski 33615a7dea7SJakub Kicinski if (sk) 33715a7dea7SJakub Kicinski kfree_rcu(ctx, rcu); 33815a7dea7SJakub Kicinski else 33986029d10SSabrina Dubroca kfree(ctx); 34086029d10SSabrina Dubroca } 34186029d10SSabrina Dubroca 342313ab004SJohn Fastabend static void tls_sk_proto_cleanup(struct sock *sk, 343313ab004SJohn Fastabend struct tls_context *ctx, long timeo) 3443c4d7559SDave Watson { 3459354544cSDirk van der Merwe if (unlikely(sk->sk_write_pending) && 3469354544cSDirk van der Merwe !wait_on_pending_writer(sk, &timeo)) 3473c4d7559SDave Watson tls_handle_open_record(sk, 0); 3483c4d7559SDave Watson 349f66de3eeSBoris Pismenny /* We need these for tls_sw_fallback handling of other packets */ 350f66de3eeSBoris Pismenny if (ctx->tx_conf == TLS_SW) { 351dbe42559SDave Watson kfree(ctx->tx.rec_seq); 352dbe42559SDave Watson kfree(ctx->tx.iv); 353313ab004SJohn Fastabend tls_sw_release_resources_tx(sk); 354b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW); 35535b71a34SJakub Kicinski } else if (ctx->tx_conf == TLS_HW) { 35635b71a34SJakub Kicinski tls_device_free_resources_tx(sk); 357b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE); 358f66de3eeSBoris Pismenny } 359f66de3eeSBoris Pismenny 360b32fd3ccSJakub Kicinski if (ctx->rx_conf == TLS_SW) { 361313ab004SJohn Fastabend tls_sw_release_resources_rx(sk); 362b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW); 363b32fd3ccSJakub Kicinski } else if (ctx->rx_conf == TLS_HW) { 3644799ac81SBoris Pismenny tls_device_offload_cleanup_rx(sk); 365b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE); 366b32fd3ccSJakub Kicinski } 367e8f69799SIlya Lesokhin } 368e8f69799SIlya Lesokhin 369313ab004SJohn Fastabend static void tls_sk_proto_close(struct sock *sk, long timeout) 370313ab004SJohn Fastabend { 37195fa1454SJohn Fastabend struct inet_connection_sock *icsk = inet_csk(sk); 372313ab004SJohn Fastabend struct tls_context *ctx = tls_get_ctx(sk); 373313ab004SJohn Fastabend long timeo = sock_sndtimeo(sk, 0); 374313ab004SJohn Fastabend bool free_ctx; 375313ab004SJohn Fastabend 376313ab004SJohn Fastabend if (ctx->tx_conf == TLS_SW) 377313ab004SJohn Fastabend tls_sw_cancel_work_tx(ctx); 378313ab004SJohn Fastabend 379313ab004SJohn Fastabend lock_sock(sk); 380313ab004SJohn Fastabend free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW; 381313ab004SJohn Fastabend 382313ab004SJohn Fastabend if (ctx->tx_conf != TLS_BASE || ctx->rx_conf != TLS_BASE) 383313ab004SJohn Fastabend tls_sk_proto_cleanup(sk, ctx, timeo); 384313ab004SJohn Fastabend 38595fa1454SJohn Fastabend write_lock_bh(&sk->sk_callback_lock); 38695fa1454SJohn Fastabend if (free_ctx) 38715a7dea7SJakub Kicinski rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 388d5bee737SJakub Sitnicki WRITE_ONCE(sk->sk_prot, ctx->sk_proto); 389d85f0177SJohn Fastabend if (sk->sk_write_space == tls_write_space) 39057c722e9SJakub Kicinski sk->sk_write_space = ctx->sk_write_space; 39195fa1454SJohn Fastabend write_unlock_bh(&sk->sk_callback_lock); 3923c4d7559SDave Watson release_sock(sk); 393313ab004SJohn Fastabend if (ctx->tx_conf == TLS_SW) 394313ab004SJohn Fastabend tls_sw_free_ctx_tx(ctx); 395313ab004SJohn Fastabend if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) 396313ab004SJohn Fastabend tls_sw_strparser_done(ctx); 397313ab004SJohn Fastabend if (ctx->rx_conf == TLS_SW) 398313ab004SJohn Fastabend tls_sw_free_ctx_rx(ctx); 399be7bbea1SJakub Kicinski ctx->sk_proto->close(sk, timeout); 400313ab004SJohn Fastabend 40198f0a395SEric Dumazet if (free_ctx) 40215a7dea7SJakub Kicinski tls_ctx_free(sk, ctx); 4033c4d7559SDave Watson } 4043c4d7559SDave Watson 405121dca78SJakub Kicinski static __poll_t tls_sk_poll(struct file *file, struct socket *sock, 406121dca78SJakub Kicinski struct poll_table_struct *wait) 407121dca78SJakub Kicinski { 408121dca78SJakub Kicinski struct tls_sw_context_rx *ctx; 409121dca78SJakub Kicinski struct tls_context *tls_ctx; 410121dca78SJakub Kicinski struct sock *sk = sock->sk; 411121dca78SJakub Kicinski struct sk_psock *psock; 412121dca78SJakub Kicinski __poll_t mask = 0; 413121dca78SJakub Kicinski u8 shutdown; 414121dca78SJakub Kicinski int state; 415121dca78SJakub Kicinski 416121dca78SJakub Kicinski mask = tcp_poll(file, sock, wait); 417121dca78SJakub Kicinski 418121dca78SJakub Kicinski state = inet_sk_state_load(sk); 419121dca78SJakub Kicinski shutdown = READ_ONCE(sk->sk_shutdown); 420121dca78SJakub Kicinski if (unlikely(state != TCP_ESTABLISHED || shutdown & RCV_SHUTDOWN)) 421121dca78SJakub Kicinski return mask; 422121dca78SJakub Kicinski 423121dca78SJakub Kicinski tls_ctx = tls_get_ctx(sk); 424121dca78SJakub Kicinski ctx = tls_sw_ctx_rx(tls_ctx); 425121dca78SJakub Kicinski psock = sk_psock_get(sk); 426121dca78SJakub Kicinski 427121dca78SJakub Kicinski if (skb_queue_empty_lockless(&ctx->rx_list) && 428121dca78SJakub Kicinski !tls_strp_msg_ready(ctx) && 429121dca78SJakub Kicinski sk_psock_queue_empty(psock)) 430121dca78SJakub Kicinski mask &= ~(EPOLLIN | EPOLLRDNORM); 431121dca78SJakub Kicinski 432121dca78SJakub Kicinski if (psock) 433121dca78SJakub Kicinski sk_psock_put(sk, psock); 434121dca78SJakub Kicinski 435121dca78SJakub Kicinski return mask; 436121dca78SJakub Kicinski } 437121dca78SJakub Kicinski 438ffa81fa4SYutaro Hayakawa static int do_tls_getsockopt_conf(struct sock *sk, char __user *optval, 439ffa81fa4SYutaro Hayakawa int __user *optlen, int tx) 4403c4d7559SDave Watson { 4413c4d7559SDave Watson int rc = 0; 442077e05d1SSabrina Dubroca const struct tls_cipher_desc *cipher_desc; 4433c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 4443c4d7559SDave Watson struct tls_crypto_info *crypto_info; 445ffa81fa4SYutaro Hayakawa struct cipher_context *cctx; 4463c4d7559SDave Watson int len; 4473c4d7559SDave Watson 4483c4d7559SDave Watson if (get_user(len, optlen)) 4493c4d7559SDave Watson return -EFAULT; 4503c4d7559SDave Watson 4513c4d7559SDave Watson if (!optval || (len < sizeof(*crypto_info))) { 4523c4d7559SDave Watson rc = -EINVAL; 4533c4d7559SDave Watson goto out; 4543c4d7559SDave Watson } 4553c4d7559SDave Watson 4563c4d7559SDave Watson if (!ctx) { 4573c4d7559SDave Watson rc = -EBUSY; 4583c4d7559SDave Watson goto out; 4593c4d7559SDave Watson } 4603c4d7559SDave Watson 4613c4d7559SDave Watson /* get user crypto info */ 462ffa81fa4SYutaro Hayakawa if (tx) { 46386029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 464ffa81fa4SYutaro Hayakawa cctx = &ctx->tx; 465ffa81fa4SYutaro Hayakawa } else { 466ffa81fa4SYutaro Hayakawa crypto_info = &ctx->crypto_recv.info; 467ffa81fa4SYutaro Hayakawa cctx = &ctx->rx; 468ffa81fa4SYutaro Hayakawa } 4693c4d7559SDave Watson 4703c4d7559SDave Watson if (!TLS_CRYPTO_INFO_READY(crypto_info)) { 4713c4d7559SDave Watson rc = -EBUSY; 4723c4d7559SDave Watson goto out; 4733c4d7559SDave Watson } 4743c4d7559SDave Watson 4755a3b886cSMatthias Rosenfelder if (len == sizeof(*crypto_info)) { 476ac55cd61SDan Carpenter if (copy_to_user(optval, crypto_info, sizeof(*crypto_info))) 477ac55cd61SDan Carpenter rc = -EFAULT; 4783c4d7559SDave Watson goto out; 4793c4d7559SDave Watson } 4803c4d7559SDave Watson 481077e05d1SSabrina Dubroca cipher_desc = get_cipher_desc(crypto_info->cipher_type); 482077e05d1SSabrina Dubroca if (!cipher_desc || len != cipher_desc->crypto_info) { 4833c4d7559SDave Watson rc = -EINVAL; 4843c4d7559SDave Watson goto out; 4853c4d7559SDave Watson } 486fb99bce7SDave Watson 487077e05d1SSabrina Dubroca memcpy(crypto_info_iv(crypto_info, cipher_desc), 488077e05d1SSabrina Dubroca cctx->iv + cipher_desc->salt, cipher_desc->iv); 489077e05d1SSabrina Dubroca memcpy(crypto_info_rec_seq(crypto_info, cipher_desc), 490077e05d1SSabrina Dubroca cctx->rec_seq, cipher_desc->rec_seq); 4913fb59a5dSTianjia Zhang 492077e05d1SSabrina Dubroca if (copy_to_user(optval, crypto_info, cipher_desc->crypto_info)) 4933fb59a5dSTianjia Zhang rc = -EFAULT; 4943c4d7559SDave Watson 4953c4d7559SDave Watson out: 4963c4d7559SDave Watson return rc; 4973c4d7559SDave Watson } 4983c4d7559SDave Watson 499c1318b39SBoris Pismenny static int do_tls_getsockopt_tx_zc(struct sock *sk, char __user *optval, 500c1318b39SBoris Pismenny int __user *optlen) 501c1318b39SBoris Pismenny { 502c1318b39SBoris Pismenny struct tls_context *ctx = tls_get_ctx(sk); 503c1318b39SBoris Pismenny unsigned int value; 504c1318b39SBoris Pismenny int len; 505c1318b39SBoris Pismenny 506c1318b39SBoris Pismenny if (get_user(len, optlen)) 507c1318b39SBoris Pismenny return -EFAULT; 508c1318b39SBoris Pismenny 509c1318b39SBoris Pismenny if (len != sizeof(value)) 510c1318b39SBoris Pismenny return -EINVAL; 511c1318b39SBoris Pismenny 512c1318b39SBoris Pismenny value = ctx->zerocopy_sendfile; 513c1318b39SBoris Pismenny if (copy_to_user(optval, &value, sizeof(value))) 514c1318b39SBoris Pismenny return -EFAULT; 515c1318b39SBoris Pismenny 516c1318b39SBoris Pismenny return 0; 517c1318b39SBoris Pismenny } 518c1318b39SBoris Pismenny 51988527790SJakub Kicinski static int do_tls_getsockopt_no_pad(struct sock *sk, char __user *optval, 52088527790SJakub Kicinski int __user *optlen) 52188527790SJakub Kicinski { 52288527790SJakub Kicinski struct tls_context *ctx = tls_get_ctx(sk); 52357128e98SJakub Kicinski int value, len; 52488527790SJakub Kicinski 52588527790SJakub Kicinski if (ctx->prot_info.version != TLS_1_3_VERSION) 52688527790SJakub Kicinski return -EINVAL; 52788527790SJakub Kicinski 52888527790SJakub Kicinski if (get_user(len, optlen)) 52988527790SJakub Kicinski return -EFAULT; 53088527790SJakub Kicinski if (len < sizeof(value)) 53188527790SJakub Kicinski return -EINVAL; 53288527790SJakub Kicinski 53357128e98SJakub Kicinski value = -EINVAL; 53488527790SJakub Kicinski if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) 53588527790SJakub Kicinski value = ctx->rx_no_pad; 53657128e98SJakub Kicinski if (value < 0) 53757128e98SJakub Kicinski return value; 53888527790SJakub Kicinski 53988527790SJakub Kicinski if (put_user(sizeof(value), optlen)) 54088527790SJakub Kicinski return -EFAULT; 54188527790SJakub Kicinski if (copy_to_user(optval, &value, sizeof(value))) 54288527790SJakub Kicinski return -EFAULT; 54388527790SJakub Kicinski 54488527790SJakub Kicinski return 0; 54588527790SJakub Kicinski } 54688527790SJakub Kicinski 5473c4d7559SDave Watson static int do_tls_getsockopt(struct sock *sk, int optname, 5483c4d7559SDave Watson char __user *optval, int __user *optlen) 5493c4d7559SDave Watson { 5503c4d7559SDave Watson int rc = 0; 5513c4d7559SDave Watson 55249c47cc2SHangyu Hua lock_sock(sk); 55349c47cc2SHangyu Hua 5543c4d7559SDave Watson switch (optname) { 5553c4d7559SDave Watson case TLS_TX: 556ffa81fa4SYutaro Hayakawa case TLS_RX: 557ffa81fa4SYutaro Hayakawa rc = do_tls_getsockopt_conf(sk, optval, optlen, 558ffa81fa4SYutaro Hayakawa optname == TLS_TX); 5593c4d7559SDave Watson break; 560b489a6e5SMaxim Mikityanskiy case TLS_TX_ZEROCOPY_RO: 561c1318b39SBoris Pismenny rc = do_tls_getsockopt_tx_zc(sk, optval, optlen); 562c1318b39SBoris Pismenny break; 56388527790SJakub Kicinski case TLS_RX_EXPECT_NO_PAD: 56488527790SJakub Kicinski rc = do_tls_getsockopt_no_pad(sk, optval, optlen); 56588527790SJakub Kicinski break; 5663c4d7559SDave Watson default: 5673c4d7559SDave Watson rc = -ENOPROTOOPT; 5683c4d7559SDave Watson break; 5693c4d7559SDave Watson } 57049c47cc2SHangyu Hua 57149c47cc2SHangyu Hua release_sock(sk); 57249c47cc2SHangyu Hua 5733c4d7559SDave Watson return rc; 5743c4d7559SDave Watson } 5753c4d7559SDave Watson 5763c4d7559SDave Watson static int tls_getsockopt(struct sock *sk, int level, int optname, 5773c4d7559SDave Watson char __user *optval, int __user *optlen) 5783c4d7559SDave Watson { 5793c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 5803c4d7559SDave Watson 5813c4d7559SDave Watson if (level != SOL_TLS) 582be7bbea1SJakub Kicinski return ctx->sk_proto->getsockopt(sk, level, 583be7bbea1SJakub Kicinski optname, optval, optlen); 5843c4d7559SDave Watson 5853c4d7559SDave Watson return do_tls_getsockopt(sk, optname, optval, optlen); 5863c4d7559SDave Watson } 5873c4d7559SDave Watson 588a7b75c5aSChristoph Hellwig static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval, 589c46234ebSDave Watson unsigned int optlen, int tx) 5903c4d7559SDave Watson { 591196c31b4SIlya Lesokhin struct tls_crypto_info *crypto_info; 5924509de14SVakul Garg struct tls_crypto_info *alt_crypto_info; 5933c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 5945f309adeSSabrina Dubroca const struct tls_cipher_desc *cipher_desc; 5953c4d7559SDave Watson int rc = 0; 59658371585SDave Watson int conf; 5973c4d7559SDave Watson 5981ddcbfbfSZiyang Xuan if (sockptr_is_null(optval) || (optlen < sizeof(*crypto_info))) 5991ddcbfbfSZiyang Xuan return -EINVAL; 6003c4d7559SDave Watson 6014509de14SVakul Garg if (tx) { 60286029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 6034509de14SVakul Garg alt_crypto_info = &ctx->crypto_recv.info; 6044509de14SVakul Garg } else { 60586029d10SSabrina Dubroca crypto_info = &ctx->crypto_recv.info; 6064509de14SVakul Garg alt_crypto_info = &ctx->crypto_send.info; 6074509de14SVakul Garg } 608c46234ebSDave Watson 609196c31b4SIlya Lesokhin /* Currently we don't support set crypto info more than one time */ 6101ddcbfbfSZiyang Xuan if (TLS_CRYPTO_INFO_READY(crypto_info)) 6111ddcbfbfSZiyang Xuan return -EBUSY; 612196c31b4SIlya Lesokhin 613a7b75c5aSChristoph Hellwig rc = copy_from_sockptr(crypto_info, optval, sizeof(*crypto_info)); 6143c4d7559SDave Watson if (rc) { 6153c4d7559SDave Watson rc = -EFAULT; 616257082e6SBoris Pismenny goto err_crypto_info; 6173c4d7559SDave Watson } 6183c4d7559SDave Watson 6193c4d7559SDave Watson /* check version */ 620130b392cSDave Watson if (crypto_info->version != TLS_1_2_VERSION && 621130b392cSDave Watson crypto_info->version != TLS_1_3_VERSION) { 6224a5cdc60SValentin Vidic rc = -EINVAL; 623196c31b4SIlya Lesokhin goto err_crypto_info; 6243c4d7559SDave Watson } 6253c4d7559SDave Watson 6264509de14SVakul Garg /* Ensure that TLS version and ciphers are same in both directions */ 6274509de14SVakul Garg if (TLS_CRYPTO_INFO_READY(alt_crypto_info)) { 6284509de14SVakul Garg if (alt_crypto_info->version != crypto_info->version || 6294509de14SVakul Garg alt_crypto_info->cipher_type != crypto_info->cipher_type) { 6304509de14SVakul Garg rc = -EINVAL; 6314509de14SVakul Garg goto err_crypto_info; 6324509de14SVakul Garg } 6334509de14SVakul Garg } 6344509de14SVakul Garg 6355f309adeSSabrina Dubroca cipher_desc = get_cipher_desc(crypto_info->cipher_type); 6365f309adeSSabrina Dubroca if (!cipher_desc) { 63762e56ef5STaehee Yoo rc = -EINVAL; 63862e56ef5STaehee Yoo goto err_crypto_info; 63962e56ef5STaehee Yoo } 6405f309adeSSabrina Dubroca 6415f309adeSSabrina Dubroca switch (crypto_info->cipher_type) { 6425f309adeSSabrina Dubroca case TLS_CIPHER_ARIA_GCM_128: 64362e56ef5STaehee Yoo case TLS_CIPHER_ARIA_GCM_256: 64462e56ef5STaehee Yoo if (crypto_info->version != TLS_1_2_VERSION) { 64562e56ef5STaehee Yoo rc = -EINVAL; 64662e56ef5STaehee Yoo goto err_crypto_info; 64762e56ef5STaehee Yoo } 64862e56ef5STaehee Yoo break; 649f295b3aeSVakul Garg } 650f295b3aeSVakul Garg 6515f309adeSSabrina Dubroca if (optlen != cipher_desc->crypto_info) { 6523c4d7559SDave Watson rc = -EINVAL; 6536db959c8SSabrina Dubroca goto err_crypto_info; 6543c4d7559SDave Watson } 655f295b3aeSVakul Garg 656d3c48151SChristoph Hellwig rc = copy_from_sockptr_offset(crypto_info + 1, optval, 657d3c48151SChristoph Hellwig sizeof(*crypto_info), 658196c31b4SIlya Lesokhin optlen - sizeof(*crypto_info)); 6593c4d7559SDave Watson if (rc) { 6603c4d7559SDave Watson rc = -EFAULT; 6613c4d7559SDave Watson goto err_crypto_info; 6623c4d7559SDave Watson } 6633c4d7559SDave Watson 664c46234ebSDave Watson if (tx) { 665e8f69799SIlya Lesokhin rc = tls_set_device_offload(sk, ctx); 666e8f69799SIlya Lesokhin conf = TLS_HW; 667b32fd3ccSJakub Kicinski if (!rc) { 668b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE); 669b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE); 670b32fd3ccSJakub Kicinski } else { 671c46234ebSDave Watson rc = tls_set_sw_offload(sk, ctx, 1); 672318892acSJakub Kicinski if (rc) 673318892acSJakub Kicinski goto err_crypto_info; 674b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW); 675b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW); 676f66de3eeSBoris Pismenny conf = TLS_SW; 677e8f69799SIlya Lesokhin } 678c46234ebSDave Watson } else { 6794799ac81SBoris Pismenny rc = tls_set_device_offload_rx(sk, ctx); 6804799ac81SBoris Pismenny conf = TLS_HW; 681b32fd3ccSJakub Kicinski if (!rc) { 682b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE); 683b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE); 684b32fd3ccSJakub Kicinski } else { 685c46234ebSDave Watson rc = tls_set_sw_offload(sk, ctx, 0); 686318892acSJakub Kicinski if (rc) 687318892acSJakub Kicinski goto err_crypto_info; 688b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW); 689b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW); 690f66de3eeSBoris Pismenny conf = TLS_SW; 691c46234ebSDave Watson } 692313ab004SJohn Fastabend tls_sw_strparser_arm(sk, ctx); 6934799ac81SBoris Pismenny } 694c46234ebSDave Watson 695f66de3eeSBoris Pismenny if (tx) 696f66de3eeSBoris Pismenny ctx->tx_conf = conf; 697f66de3eeSBoris Pismenny else 698f66de3eeSBoris Pismenny ctx->rx_conf = conf; 6996d88207fSIlya Lesokhin update_sk_prot(sk, ctx); 700c46234ebSDave Watson if (tx) { 701ee181e52SIlya Lesokhin ctx->sk_write_space = sk->sk_write_space; 702ee181e52SIlya Lesokhin sk->sk_write_space = tls_write_space; 70384c61fe1SJakub Kicinski } else { 70484c61fe1SJakub Kicinski struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(ctx); 70584c61fe1SJakub Kicinski 70684c61fe1SJakub Kicinski tls_strp_check_rcv(&rx_ctx->strp); 707c46234ebSDave Watson } 7081ddcbfbfSZiyang Xuan return 0; 7093c4d7559SDave Watson 7103c4d7559SDave Watson err_crypto_info: 711c844eb46SSabrina Dubroca memzero_explicit(crypto_info, sizeof(union tls_crypto_context)); 7123c4d7559SDave Watson return rc; 7133c4d7559SDave Watson } 7143c4d7559SDave Watson 715c1318b39SBoris Pismenny static int do_tls_setsockopt_tx_zc(struct sock *sk, sockptr_t optval, 716c1318b39SBoris Pismenny unsigned int optlen) 717c1318b39SBoris Pismenny { 718c1318b39SBoris Pismenny struct tls_context *ctx = tls_get_ctx(sk); 719c1318b39SBoris Pismenny unsigned int value; 720c1318b39SBoris Pismenny 721c1318b39SBoris Pismenny if (sockptr_is_null(optval) || optlen != sizeof(value)) 722c1318b39SBoris Pismenny return -EINVAL; 723c1318b39SBoris Pismenny 724c1318b39SBoris Pismenny if (copy_from_sockptr(&value, optval, sizeof(value))) 725c1318b39SBoris Pismenny return -EFAULT; 726c1318b39SBoris Pismenny 727c1318b39SBoris Pismenny if (value > 1) 728c1318b39SBoris Pismenny return -EINVAL; 729c1318b39SBoris Pismenny 730c1318b39SBoris Pismenny ctx->zerocopy_sendfile = value; 731c1318b39SBoris Pismenny 732c1318b39SBoris Pismenny return 0; 733c1318b39SBoris Pismenny } 734c1318b39SBoris Pismenny 73588527790SJakub Kicinski static int do_tls_setsockopt_no_pad(struct sock *sk, sockptr_t optval, 73688527790SJakub Kicinski unsigned int optlen) 73788527790SJakub Kicinski { 73888527790SJakub Kicinski struct tls_context *ctx = tls_get_ctx(sk); 73988527790SJakub Kicinski u32 val; 74088527790SJakub Kicinski int rc; 74188527790SJakub Kicinski 74288527790SJakub Kicinski if (ctx->prot_info.version != TLS_1_3_VERSION || 74388527790SJakub Kicinski sockptr_is_null(optval) || optlen < sizeof(val)) 74488527790SJakub Kicinski return -EINVAL; 74588527790SJakub Kicinski 74688527790SJakub Kicinski rc = copy_from_sockptr(&val, optval, sizeof(val)); 74788527790SJakub Kicinski if (rc) 74888527790SJakub Kicinski return -EFAULT; 74988527790SJakub Kicinski if (val > 1) 75088527790SJakub Kicinski return -EINVAL; 75188527790SJakub Kicinski rc = check_zeroed_sockptr(optval, sizeof(val), optlen - sizeof(val)); 75288527790SJakub Kicinski if (rc < 1) 75388527790SJakub Kicinski return rc == 0 ? -EINVAL : rc; 75488527790SJakub Kicinski 75588527790SJakub Kicinski lock_sock(sk); 75688527790SJakub Kicinski rc = -EINVAL; 75788527790SJakub Kicinski if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) { 75888527790SJakub Kicinski ctx->rx_no_pad = val; 75988527790SJakub Kicinski tls_update_rx_zc_capable(ctx); 76088527790SJakub Kicinski rc = 0; 76188527790SJakub Kicinski } 76288527790SJakub Kicinski release_sock(sk); 76388527790SJakub Kicinski 76488527790SJakub Kicinski return rc; 76588527790SJakub Kicinski } 76688527790SJakub Kicinski 767a7b75c5aSChristoph Hellwig static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval, 768a7b75c5aSChristoph Hellwig unsigned int optlen) 7693c4d7559SDave Watson { 7703c4d7559SDave Watson int rc = 0; 7713c4d7559SDave Watson 7723c4d7559SDave Watson switch (optname) { 7733c4d7559SDave Watson case TLS_TX: 774c46234ebSDave Watson case TLS_RX: 7753c4d7559SDave Watson lock_sock(sk); 776c46234ebSDave Watson rc = do_tls_setsockopt_conf(sk, optval, optlen, 777c46234ebSDave Watson optname == TLS_TX); 7783c4d7559SDave Watson release_sock(sk); 7793c4d7559SDave Watson break; 780b489a6e5SMaxim Mikityanskiy case TLS_TX_ZEROCOPY_RO: 781c1318b39SBoris Pismenny lock_sock(sk); 782c1318b39SBoris Pismenny rc = do_tls_setsockopt_tx_zc(sk, optval, optlen); 783c1318b39SBoris Pismenny release_sock(sk); 784c1318b39SBoris Pismenny break; 78588527790SJakub Kicinski case TLS_RX_EXPECT_NO_PAD: 78688527790SJakub Kicinski rc = do_tls_setsockopt_no_pad(sk, optval, optlen); 78788527790SJakub Kicinski break; 7883c4d7559SDave Watson default: 7893c4d7559SDave Watson rc = -ENOPROTOOPT; 7903c4d7559SDave Watson break; 7913c4d7559SDave Watson } 7923c4d7559SDave Watson return rc; 7933c4d7559SDave Watson } 7943c4d7559SDave Watson 7953c4d7559SDave Watson static int tls_setsockopt(struct sock *sk, int level, int optname, 796a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 7973c4d7559SDave Watson { 7983c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 7993c4d7559SDave Watson 8003c4d7559SDave Watson if (level != SOL_TLS) 801be7bbea1SJakub Kicinski return ctx->sk_proto->setsockopt(sk, level, optname, optval, 802be7bbea1SJakub Kicinski optlen); 8033c4d7559SDave Watson 8043c4d7559SDave Watson return do_tls_setsockopt(sk, optname, optval, optlen); 8053c4d7559SDave Watson } 8063c4d7559SDave Watson 80708700dabSJakub Kicinski struct tls_context *tls_ctx_create(struct sock *sk) 808dd0bed16SAtul Gupta { 809dd0bed16SAtul Gupta struct inet_connection_sock *icsk = inet_csk(sk); 810dd0bed16SAtul Gupta struct tls_context *ctx; 811dd0bed16SAtul Gupta 812c6ec179aSGanesh Goudar ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC); 813dd0bed16SAtul Gupta if (!ctx) 814dd0bed16SAtul Gupta return NULL; 815dd0bed16SAtul Gupta 81679ffe608SJakub Kicinski mutex_init(&ctx->tx_lock); 81715a7dea7SJakub Kicinski rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 818d5bee737SJakub Sitnicki ctx->sk_proto = READ_ONCE(sk->sk_prot); 819c55dcdd4SMaxim Mikityanskiy ctx->sk = sk; 820dd0bed16SAtul Gupta return ctx; 821dd0bed16SAtul Gupta } 822dd0bed16SAtul Gupta 823f3911f73SJakub Kicinski static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 824f3911f73SJakub Kicinski const struct proto_ops *base) 825f3911f73SJakub Kicinski { 826f3911f73SJakub Kicinski ops[TLS_BASE][TLS_BASE] = *base; 827f3911f73SJakub Kicinski 828f3911f73SJakub Kicinski ops[TLS_SW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; 829df720d28SDavid Howells ops[TLS_SW ][TLS_BASE].splice_eof = tls_sw_splice_eof; 830f3911f73SJakub Kicinski 831f3911f73SJakub Kicinski ops[TLS_BASE][TLS_SW ] = ops[TLS_BASE][TLS_BASE]; 832f3911f73SJakub Kicinski ops[TLS_BASE][TLS_SW ].splice_read = tls_sw_splice_read; 833121dca78SJakub Kicinski ops[TLS_BASE][TLS_SW ].poll = tls_sk_poll; 834662fbcecSHannes Reinecke ops[TLS_BASE][TLS_SW ].read_sock = tls_sw_read_sock; 835f3911f73SJakub Kicinski 836f3911f73SJakub Kicinski ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE]; 837f3911f73SJakub Kicinski ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read; 838121dca78SJakub Kicinski ops[TLS_SW ][TLS_SW ].poll = tls_sk_poll; 839662fbcecSHannes Reinecke ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock; 840f3911f73SJakub Kicinski 841f3911f73SJakub Kicinski #ifdef CONFIG_TLS_DEVICE 842f3911f73SJakub Kicinski ops[TLS_HW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; 843f3911f73SJakub Kicinski 844f3911f73SJakub Kicinski ops[TLS_HW ][TLS_SW ] = ops[TLS_BASE][TLS_SW ]; 845f3911f73SJakub Kicinski 846f3911f73SJakub Kicinski ops[TLS_BASE][TLS_HW ] = ops[TLS_BASE][TLS_SW ]; 847f3911f73SJakub Kicinski 848f3911f73SJakub Kicinski ops[TLS_SW ][TLS_HW ] = ops[TLS_SW ][TLS_SW ]; 849f3911f73SJakub Kicinski 850f3911f73SJakub Kicinski ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ]; 851f3911f73SJakub Kicinski #endif 852f3911f73SJakub Kicinski #ifdef CONFIG_TLS_TOE 853f3911f73SJakub Kicinski ops[TLS_HW_RECORD][TLS_HW_RECORD] = *base; 854f3911f73SJakub Kicinski #endif 855f3911f73SJakub Kicinski } 856f3911f73SJakub Kicinski 85763a6b3feSAtul Gupta static void tls_build_proto(struct sock *sk) 85863a6b3feSAtul Gupta { 85963a6b3feSAtul Gupta int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 8609a893949SWill Deacon struct proto *prot = READ_ONCE(sk->sk_prot); 86163a6b3feSAtul Gupta 86263a6b3feSAtul Gupta /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */ 86363a6b3feSAtul Gupta if (ip_ver == TLSV6 && 8645bb4c45dSJakub Sitnicki unlikely(prot != smp_load_acquire(&saved_tcpv6_prot))) { 86563a6b3feSAtul Gupta mutex_lock(&tcpv6_prot_mutex); 8665bb4c45dSJakub Sitnicki if (likely(prot != saved_tcpv6_prot)) { 8675bb4c45dSJakub Sitnicki build_protos(tls_prots[TLSV6], prot); 868f3911f73SJakub Kicinski build_proto_ops(tls_proto_ops[TLSV6], 869f3911f73SJakub Kicinski sk->sk_socket->ops); 8705bb4c45dSJakub Sitnicki smp_store_release(&saved_tcpv6_prot, prot); 87163a6b3feSAtul Gupta } 87263a6b3feSAtul Gupta mutex_unlock(&tcpv6_prot_mutex); 87363a6b3feSAtul Gupta } 87463a6b3feSAtul Gupta 87563a6b3feSAtul Gupta if (ip_ver == TLSV4 && 8765bb4c45dSJakub Sitnicki unlikely(prot != smp_load_acquire(&saved_tcpv4_prot))) { 87763a6b3feSAtul Gupta mutex_lock(&tcpv4_prot_mutex); 8785bb4c45dSJakub Sitnicki if (likely(prot != saved_tcpv4_prot)) { 8795bb4c45dSJakub Sitnicki build_protos(tls_prots[TLSV4], prot); 880f3911f73SJakub Kicinski build_proto_ops(tls_proto_ops[TLSV4], 881f3911f73SJakub Kicinski sk->sk_socket->ops); 8825bb4c45dSJakub Sitnicki smp_store_release(&saved_tcpv4_prot, prot); 88363a6b3feSAtul Gupta } 88463a6b3feSAtul Gupta mutex_unlock(&tcpv4_prot_mutex); 88563a6b3feSAtul Gupta } 88663a6b3feSAtul Gupta } 88763a6b3feSAtul Gupta 888f66de3eeSBoris Pismenny static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 889f13fe3e6SJakub Sitnicki const struct proto *base) 890c113187dSBoris Pismenny { 891f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE] = *base; 892f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt; 893f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].getsockopt = tls_getsockopt; 894f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].close = tls_sk_proto_close; 895c113187dSBoris Pismenny 896f66de3eeSBoris Pismenny prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 897f66de3eeSBoris Pismenny prot[TLS_SW][TLS_BASE].sendmsg = tls_sw_sendmsg; 898df720d28SDavid Howells prot[TLS_SW][TLS_BASE].splice_eof = tls_sw_splice_eof; 899c46234ebSDave Watson 900f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE]; 901f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg; 9027b50ecfcSCong Wang prot[TLS_BASE][TLS_SW].sock_is_readable = tls_sw_sock_is_readable; 903f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close; 904c46234ebSDave Watson 905f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE]; 906f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg; 9077b50ecfcSCong Wang prot[TLS_SW][TLS_SW].sock_is_readable = tls_sw_sock_is_readable; 908f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW].close = tls_sk_proto_close; 909dd0bed16SAtul Gupta 910e8f69799SIlya Lesokhin #ifdef CONFIG_TLS_DEVICE 911e8f69799SIlya Lesokhin prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 912e8f69799SIlya Lesokhin prot[TLS_HW][TLS_BASE].sendmsg = tls_device_sendmsg; 913d4c1e80bSDavid Howells prot[TLS_HW][TLS_BASE].splice_eof = tls_device_splice_eof; 914e8f69799SIlya Lesokhin 915e8f69799SIlya Lesokhin prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW]; 916e8f69799SIlya Lesokhin prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg; 917d4c1e80bSDavid Howells prot[TLS_HW][TLS_SW].splice_eof = tls_device_splice_eof; 9184799ac81SBoris Pismenny 9194799ac81SBoris Pismenny prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW]; 9204799ac81SBoris Pismenny 9214799ac81SBoris Pismenny prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW]; 9224799ac81SBoris Pismenny 9234799ac81SBoris Pismenny prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW]; 924e8f69799SIlya Lesokhin #endif 92553b4414aSJakub Kicinski #ifdef CONFIG_TLS_TOE 926f66de3eeSBoris Pismenny prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base; 9270eb8745eSJakub Kicinski prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_toe_hash; 9280eb8745eSJakub Kicinski prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_toe_unhash; 92953b4414aSJakub Kicinski #endif 930c113187dSBoris Pismenny } 931c113187dSBoris Pismenny 9323c4d7559SDave Watson static int tls_init(struct sock *sk) 9333c4d7559SDave Watson { 9343c4d7559SDave Watson struct tls_context *ctx; 9353c4d7559SDave Watson int rc = 0; 9363c4d7559SDave Watson 93716bed0e6SJakub Kicinski tls_build_proto(sk); 93816bed0e6SJakub Kicinski 93953b4414aSJakub Kicinski #ifdef CONFIG_TLS_TOE 9400eb8745eSJakub Kicinski if (tls_toe_bypass(sk)) 94195fa1454SJohn Fastabend return 0; 94253b4414aSJakub Kicinski #endif 943dd0bed16SAtul Gupta 944d91c3e17SIlya Lesokhin /* The TLS ulp is currently supported only for TCP sockets 945d91c3e17SIlya Lesokhin * in ESTABLISHED state. 946d91c3e17SIlya Lesokhin * Supporting sockets in LISTEN state will require us 947d91c3e17SIlya Lesokhin * to modify the accept implementation to clone rather then 948d91c3e17SIlya Lesokhin * share the ulp context. 949d91c3e17SIlya Lesokhin */ 950d91c3e17SIlya Lesokhin if (sk->sk_state != TCP_ESTABLISHED) 9514a5cdc60SValentin Vidic return -ENOTCONN; 952d91c3e17SIlya Lesokhin 9533c4d7559SDave Watson /* allocate tls context */ 95495fa1454SJohn Fastabend write_lock_bh(&sk->sk_callback_lock); 95508700dabSJakub Kicinski ctx = tls_ctx_create(sk); 9563c4d7559SDave Watson if (!ctx) { 9573c4d7559SDave Watson rc = -ENOMEM; 9583c4d7559SDave Watson goto out; 9593c4d7559SDave Watson } 9606d88207fSIlya Lesokhin 961f66de3eeSBoris Pismenny ctx->tx_conf = TLS_BASE; 962f66de3eeSBoris Pismenny ctx->rx_conf = TLS_BASE; 9636d88207fSIlya Lesokhin update_sk_prot(sk, ctx); 9643c4d7559SDave Watson out: 96595fa1454SJohn Fastabend write_unlock_bh(&sk->sk_callback_lock); 9663c4d7559SDave Watson return rc; 9673c4d7559SDave Watson } 9683c4d7559SDave Watson 96933bfe20dSJohn Fastabend static void tls_update(struct sock *sk, struct proto *p, 97033bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)) 97195fa1454SJohn Fastabend { 97295fa1454SJohn Fastabend struct tls_context *ctx; 97395fa1454SJohn Fastabend 974e34a07c0SJakub Kicinski WARN_ON_ONCE(sk->sk_prot == p); 975e34a07c0SJakub Kicinski 97695fa1454SJohn Fastabend ctx = tls_get_ctx(sk); 97733bfe20dSJohn Fastabend if (likely(ctx)) { 97833bfe20dSJohn Fastabend ctx->sk_write_space = write_space; 97995fa1454SJohn Fastabend ctx->sk_proto = p; 98033bfe20dSJohn Fastabend } else { 981b8e202d1SJakub Sitnicki /* Pairs with lockless read in sk_clone_lock(). */ 982b8e202d1SJakub Sitnicki WRITE_ONCE(sk->sk_prot, p); 98333bfe20dSJohn Fastabend sk->sk_write_space = write_space; 98433bfe20dSJohn Fastabend } 98595fa1454SJohn Fastabend } 98695fa1454SJohn Fastabend 98758790314SJakub Kicinski static u16 tls_user_config(struct tls_context *ctx, bool tx) 98858790314SJakub Kicinski { 98958790314SJakub Kicinski u16 config = tx ? ctx->tx_conf : ctx->rx_conf; 99058790314SJakub Kicinski 99158790314SJakub Kicinski switch (config) { 99258790314SJakub Kicinski case TLS_BASE: 99358790314SJakub Kicinski return TLS_CONF_BASE; 99458790314SJakub Kicinski case TLS_SW: 99558790314SJakub Kicinski return TLS_CONF_SW; 99658790314SJakub Kicinski case TLS_HW: 99758790314SJakub Kicinski return TLS_CONF_HW; 99858790314SJakub Kicinski case TLS_HW_RECORD: 99958790314SJakub Kicinski return TLS_CONF_HW_RECORD; 100058790314SJakub Kicinski } 100158790314SJakub Kicinski return 0; 100258790314SJakub Kicinski } 100358790314SJakub Kicinski 1004*e074c829SPaolo Abeni static int tls_get_info(struct sock *sk, struct sk_buff *skb) 100526811cc9SDavide Caratti { 100626811cc9SDavide Caratti u16 version, cipher_type; 100726811cc9SDavide Caratti struct tls_context *ctx; 100826811cc9SDavide Caratti struct nlattr *start; 100926811cc9SDavide Caratti int err; 101026811cc9SDavide Caratti 101126811cc9SDavide Caratti start = nla_nest_start_noflag(skb, INET_ULP_INFO_TLS); 101226811cc9SDavide Caratti if (!start) 101326811cc9SDavide Caratti return -EMSGSIZE; 101426811cc9SDavide Caratti 101526811cc9SDavide Caratti rcu_read_lock(); 101626811cc9SDavide Caratti ctx = rcu_dereference(inet_csk(sk)->icsk_ulp_data); 101726811cc9SDavide Caratti if (!ctx) { 101826811cc9SDavide Caratti err = 0; 101926811cc9SDavide Caratti goto nla_failure; 102026811cc9SDavide Caratti } 102126811cc9SDavide Caratti version = ctx->prot_info.version; 102226811cc9SDavide Caratti if (version) { 102326811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_VERSION, version); 102426811cc9SDavide Caratti if (err) 102526811cc9SDavide Caratti goto nla_failure; 102626811cc9SDavide Caratti } 102726811cc9SDavide Caratti cipher_type = ctx->prot_info.cipher_type; 102826811cc9SDavide Caratti if (cipher_type) { 102926811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_CIPHER, cipher_type); 103026811cc9SDavide Caratti if (err) 103126811cc9SDavide Caratti goto nla_failure; 103226811cc9SDavide Caratti } 103326811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_TXCONF, tls_user_config(ctx, true)); 103426811cc9SDavide Caratti if (err) 103526811cc9SDavide Caratti goto nla_failure; 103626811cc9SDavide Caratti 103726811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_RXCONF, tls_user_config(ctx, false)); 103826811cc9SDavide Caratti if (err) 103926811cc9SDavide Caratti goto nla_failure; 104026811cc9SDavide Caratti 1041c1318b39SBoris Pismenny if (ctx->tx_conf == TLS_HW && ctx->zerocopy_sendfile) { 1042b489a6e5SMaxim Mikityanskiy err = nla_put_flag(skb, TLS_INFO_ZC_RO_TX); 1043c1318b39SBoris Pismenny if (err) 1044c1318b39SBoris Pismenny goto nla_failure; 1045c1318b39SBoris Pismenny } 104688527790SJakub Kicinski if (ctx->rx_no_pad) { 104788527790SJakub Kicinski err = nla_put_flag(skb, TLS_INFO_RX_NO_PAD); 104888527790SJakub Kicinski if (err) 104988527790SJakub Kicinski goto nla_failure; 105088527790SJakub Kicinski } 1051c1318b39SBoris Pismenny 105226811cc9SDavide Caratti rcu_read_unlock(); 105326811cc9SDavide Caratti nla_nest_end(skb, start); 105426811cc9SDavide Caratti return 0; 105526811cc9SDavide Caratti 105626811cc9SDavide Caratti nla_failure: 105726811cc9SDavide Caratti rcu_read_unlock(); 105826811cc9SDavide Caratti nla_nest_cancel(skb, start); 105926811cc9SDavide Caratti return err; 106026811cc9SDavide Caratti } 106126811cc9SDavide Caratti 106226811cc9SDavide Caratti static size_t tls_get_info_size(const struct sock *sk) 106326811cc9SDavide Caratti { 106426811cc9SDavide Caratti size_t size = 0; 106526811cc9SDavide Caratti 106626811cc9SDavide Caratti size += nla_total_size(0) + /* INET_ULP_INFO_TLS */ 106726811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_VERSION */ 106826811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_CIPHER */ 106926811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_RXCONF */ 107026811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_TXCONF */ 1071b489a6e5SMaxim Mikityanskiy nla_total_size(0) + /* TLS_INFO_ZC_RO_TX */ 107288527790SJakub Kicinski nla_total_size(0) + /* TLS_INFO_RX_NO_PAD */ 107326811cc9SDavide Caratti 0; 107426811cc9SDavide Caratti 107526811cc9SDavide Caratti return size; 107626811cc9SDavide Caratti } 107726811cc9SDavide Caratti 1078d26b698dSJakub Kicinski static int __net_init tls_init_net(struct net *net) 1079d26b698dSJakub Kicinski { 1080d26b698dSJakub Kicinski int err; 1081d26b698dSJakub Kicinski 1082d26b698dSJakub Kicinski net->mib.tls_statistics = alloc_percpu(struct linux_tls_mib); 1083d26b698dSJakub Kicinski if (!net->mib.tls_statistics) 1084d26b698dSJakub Kicinski return -ENOMEM; 1085d26b698dSJakub Kicinski 1086d26b698dSJakub Kicinski err = tls_proc_init(net); 1087d26b698dSJakub Kicinski if (err) 1088d26b698dSJakub Kicinski goto err_free_stats; 1089d26b698dSJakub Kicinski 1090d26b698dSJakub Kicinski return 0; 1091d26b698dSJakub Kicinski err_free_stats: 1092d26b698dSJakub Kicinski free_percpu(net->mib.tls_statistics); 1093d26b698dSJakub Kicinski return err; 1094d26b698dSJakub Kicinski } 1095d26b698dSJakub Kicinski 1096d26b698dSJakub Kicinski static void __net_exit tls_exit_net(struct net *net) 1097d26b698dSJakub Kicinski { 1098d26b698dSJakub Kicinski tls_proc_fini(net); 1099d26b698dSJakub Kicinski free_percpu(net->mib.tls_statistics); 1100d26b698dSJakub Kicinski } 1101d26b698dSJakub Kicinski 1102d26b698dSJakub Kicinski static struct pernet_operations tls_proc_ops = { 1103d26b698dSJakub Kicinski .init = tls_init_net, 1104d26b698dSJakub Kicinski .exit = tls_exit_net, 1105d26b698dSJakub Kicinski }; 1106d26b698dSJakub Kicinski 11073c4d7559SDave Watson static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = { 11083c4d7559SDave Watson .name = "tls", 11093c4d7559SDave Watson .owner = THIS_MODULE, 11103c4d7559SDave Watson .init = tls_init, 111195fa1454SJohn Fastabend .update = tls_update, 111226811cc9SDavide Caratti .get_info = tls_get_info, 111326811cc9SDavide Caratti .get_info_size = tls_get_info_size, 11143c4d7559SDave Watson }; 11153c4d7559SDave Watson 11163c4d7559SDave Watson static int __init tls_register(void) 11173c4d7559SDave Watson { 1118d26b698dSJakub Kicinski int err; 1119d26b698dSJakub Kicinski 1120d26b698dSJakub Kicinski err = register_pernet_subsys(&tls_proc_ops); 1121d26b698dSJakub Kicinski if (err) 1122d26b698dSJakub Kicinski return err; 1123d26b698dSJakub Kicinski 112484c61fe1SJakub Kicinski err = tls_strp_dev_init(); 112584c61fe1SJakub Kicinski if (err) 112684c61fe1SJakub Kicinski goto err_pernet; 112784c61fe1SJakub Kicinski 11283d8c51b2STariq Toukan err = tls_device_init(); 112984c61fe1SJakub Kicinski if (err) 113084c61fe1SJakub Kicinski goto err_strp; 11313d8c51b2STariq Toukan 11323c4d7559SDave Watson tcp_register_ulp(&tcp_tls_ulp_ops); 11333c4d7559SDave Watson 11343c4d7559SDave Watson return 0; 113584c61fe1SJakub Kicinski err_strp: 113684c61fe1SJakub Kicinski tls_strp_dev_exit(); 113784c61fe1SJakub Kicinski err_pernet: 113884c61fe1SJakub Kicinski unregister_pernet_subsys(&tls_proc_ops); 113984c61fe1SJakub Kicinski return err; 11403c4d7559SDave Watson } 11413c4d7559SDave Watson 11423c4d7559SDave Watson static void __exit tls_unregister(void) 11433c4d7559SDave Watson { 11443c4d7559SDave Watson tcp_unregister_ulp(&tcp_tls_ulp_ops); 114584c61fe1SJakub Kicinski tls_strp_dev_exit(); 1146e8f69799SIlya Lesokhin tls_device_cleanup(); 1147d26b698dSJakub Kicinski unregister_pernet_subsys(&tls_proc_ops); 11483c4d7559SDave Watson } 11493c4d7559SDave Watson 11503c4d7559SDave Watson module_init(tls_register); 11513c4d7559SDave Watson module_exit(tls_unregister); 1152