13c4d7559SDave Watson /* 23c4d7559SDave Watson * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 33c4d7559SDave Watson * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 43c4d7559SDave Watson * 53c4d7559SDave Watson * This software is available to you under a choice of one of two 63c4d7559SDave Watson * licenses. You may choose to be licensed under the terms of the GNU 73c4d7559SDave Watson * General Public License (GPL) Version 2, available from the file 83c4d7559SDave Watson * COPYING in the main directory of this source tree, or the 93c4d7559SDave Watson * OpenIB.org BSD license below: 103c4d7559SDave Watson * 113c4d7559SDave Watson * Redistribution and use in source and binary forms, with or 123c4d7559SDave Watson * without modification, are permitted provided that the following 133c4d7559SDave Watson * conditions are met: 143c4d7559SDave Watson * 153c4d7559SDave Watson * - Redistributions of source code must retain the above 163c4d7559SDave Watson * copyright notice, this list of conditions and the following 173c4d7559SDave Watson * disclaimer. 183c4d7559SDave Watson * 193c4d7559SDave Watson * - Redistributions in binary form must reproduce the above 203c4d7559SDave Watson * copyright notice, this list of conditions and the following 213c4d7559SDave Watson * disclaimer in the documentation and/or other materials 223c4d7559SDave Watson * provided with the distribution. 233c4d7559SDave Watson * 243c4d7559SDave Watson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 253c4d7559SDave Watson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 263c4d7559SDave Watson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 273c4d7559SDave Watson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 283c4d7559SDave Watson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 293c4d7559SDave Watson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 303c4d7559SDave Watson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 313c4d7559SDave Watson * SOFTWARE. 323c4d7559SDave Watson */ 333c4d7559SDave Watson 343c4d7559SDave Watson #include <linux/module.h> 353c4d7559SDave Watson 363c4d7559SDave Watson #include <net/tcp.h> 373c4d7559SDave Watson #include <net/inet_common.h> 383c4d7559SDave Watson #include <linux/highmem.h> 393c4d7559SDave Watson #include <linux/netdevice.h> 403c4d7559SDave Watson #include <linux/sched/signal.h> 41dd0bed16SAtul Gupta #include <linux/inetdevice.h> 4226811cc9SDavide Caratti #include <linux/inet_diag.h> 433c4d7559SDave Watson 44d26b698dSJakub Kicinski #include <net/snmp.h> 453c4d7559SDave Watson #include <net/tls.h> 4625a3cd81SJakub Kicinski #include <net/tls_toe.h> 473c4d7559SDave Watson 4858790314SJakub Kicinski #include "tls.h" 4958790314SJakub Kicinski 503c4d7559SDave Watson MODULE_AUTHOR("Mellanox Technologies"); 513c4d7559SDave Watson MODULE_DESCRIPTION("Transport Layer Security Support"); 523c4d7559SDave Watson MODULE_LICENSE("Dual BSD/GPL"); 53037b0b86SDaniel Borkmann MODULE_ALIAS_TCP_ULP("tls"); 543c4d7559SDave Watson 556d88207fSIlya Lesokhin enum { 56c113187dSBoris Pismenny TLSV4, 57c113187dSBoris Pismenny TLSV6, 58c113187dSBoris Pismenny TLS_NUM_PROTS, 59c113187dSBoris Pismenny }; 606d88207fSIlya Lesokhin 610d98cc02SSabrina Dubroca #define CHECK_CIPHER_DESC(cipher,ci) \ 620d98cc02SSabrina Dubroca static_assert(cipher ## _IV_SIZE <= MAX_IV_SIZE); \ 630d98cc02SSabrina Dubroca static_assert(cipher ## _REC_SEQ_SIZE <= TLS_MAX_REC_SEQ_SIZE); \ 640d98cc02SSabrina Dubroca static_assert(cipher ## _TAG_SIZE == TLS_TAG_SIZE); \ 650d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, iv) == cipher ## _IV_SIZE); \ 660d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, key) == cipher ## _KEY_SIZE); \ 670d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, salt) == cipher ## _SALT_SIZE); \ 680d98cc02SSabrina Dubroca static_assert(sizeof_field(struct ci, rec_seq) == cipher ## _REC_SEQ_SIZE); 690d98cc02SSabrina Dubroca 70176a3f50SSabrina Dubroca #define __CIPHER_DESC(ci) \ 71176a3f50SSabrina Dubroca .iv_offset = offsetof(struct ci, iv), \ 72176a3f50SSabrina Dubroca .key_offset = offsetof(struct ci, key), \ 73176a3f50SSabrina Dubroca .salt_offset = offsetof(struct ci, salt), \ 74176a3f50SSabrina Dubroca .rec_seq_offset = offsetof(struct ci, rec_seq), \ 75176a3f50SSabrina Dubroca .crypto_info = sizeof(struct ci) 76176a3f50SSabrina Dubroca 77176a3f50SSabrina Dubroca #define CIPHER_DESC(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \ 78176a3f50SSabrina Dubroca .nonce = cipher ## _IV_SIZE, \ 792d2c5ea2STariq Toukan .iv = cipher ## _IV_SIZE, \ 802d2c5ea2STariq Toukan .key = cipher ## _KEY_SIZE, \ 812d2c5ea2STariq Toukan .salt = cipher ## _SALT_SIZE, \ 822d2c5ea2STariq Toukan .tag = cipher ## _TAG_SIZE, \ 832d2c5ea2STariq Toukan .rec_seq = cipher ## _REC_SEQ_SIZE, \ 84176a3f50SSabrina Dubroca .cipher_name = algname, \ 85176a3f50SSabrina Dubroca .offloadable = _offloadable, \ 86176a3f50SSabrina Dubroca __CIPHER_DESC(ci), \ 87176a3f50SSabrina Dubroca } 88176a3f50SSabrina Dubroca 89176a3f50SSabrina Dubroca #define CIPHER_DESC_NONCE0(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \ 90176a3f50SSabrina Dubroca .nonce = 0, \ 91176a3f50SSabrina Dubroca .iv = cipher ## _IV_SIZE, \ 92176a3f50SSabrina Dubroca .key = cipher ## _KEY_SIZE, \ 93176a3f50SSabrina Dubroca .salt = cipher ## _SALT_SIZE, \ 94176a3f50SSabrina Dubroca .tag = cipher ## _TAG_SIZE, \ 95176a3f50SSabrina Dubroca .rec_seq = cipher ## _REC_SEQ_SIZE, \ 96176a3f50SSabrina Dubroca .cipher_name = algname, \ 97176a3f50SSabrina Dubroca .offloadable = _offloadable, \ 98176a3f50SSabrina Dubroca __CIPHER_DESC(ci), \ 992d2c5ea2STariq Toukan } 1002d2c5ea2STariq Toukan 1018db44ab2SSabrina Dubroca const struct tls_cipher_desc tls_cipher_desc[TLS_CIPHER_MAX + 1 - TLS_CIPHER_MIN] = { 102176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128, "gcm(aes)", true), 103176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256, "gcm(aes)", true), 104176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128, "ccm(aes)", false), 105176a3f50SSabrina Dubroca CIPHER_DESC_NONCE0(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305, "rfc7539(chacha20,poly1305)", false), 106176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm, "gcm(sm4)", false), 107176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm, "ccm(sm4)", false), 108176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128, "gcm(aria)", false), 109176a3f50SSabrina Dubroca CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256, "gcm(aria)", false), 1102d2c5ea2STariq Toukan }; 1112d2c5ea2STariq Toukan 1120d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128); 1130d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256); 1140d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128); 1150d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305); 1160d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm); 1170d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm); 1180d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128); 1190d98cc02SSabrina Dubroca CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256); 1200d98cc02SSabrina Dubroca 121f691a25cSArnd Bergmann static const struct proto *saved_tcpv6_prot; 122c113187dSBoris Pismenny static DEFINE_MUTEX(tcpv6_prot_mutex); 123f691a25cSArnd Bergmann static const struct proto *saved_tcpv4_prot; 12428cb6f1eSJohn Fastabend static DEFINE_MUTEX(tcpv4_prot_mutex); 125f66de3eeSBoris Pismenny static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 126f3911f73SJakub Kicinski static struct proto_ops tls_proto_ops[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 12763a6b3feSAtul Gupta static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 128f13fe3e6SJakub Sitnicki const struct proto *base); 1296d88207fSIlya Lesokhin 13008700dabSJakub Kicinski void update_sk_prot(struct sock *sk, struct tls_context *ctx) 1316d88207fSIlya Lesokhin { 132c113187dSBoris Pismenny int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 133c113187dSBoris Pismenny 134d5bee737SJakub Sitnicki WRITE_ONCE(sk->sk_prot, 135d5bee737SJakub Sitnicki &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]); 136f3911f73SJakub Kicinski WRITE_ONCE(sk->sk_socket->ops, 137f3911f73SJakub Kicinski &tls_proto_ops[ip_ver][ctx->tx_conf][ctx->rx_conf]); 1386d88207fSIlya Lesokhin } 1393c4d7559SDave Watson 1403c4d7559SDave Watson int wait_on_pending_writer(struct sock *sk, long *timeo) 1413c4d7559SDave Watson { 1423c4d7559SDave Watson int rc = 0; 1433c4d7559SDave Watson DEFINE_WAIT_FUNC(wait, woken_wake_function); 1443c4d7559SDave Watson 1453c4d7559SDave Watson add_wait_queue(sk_sleep(sk), &wait); 1463c4d7559SDave Watson while (1) { 1473c4d7559SDave Watson if (!*timeo) { 1483c4d7559SDave Watson rc = -EAGAIN; 1493c4d7559SDave Watson break; 1503c4d7559SDave Watson } 1513c4d7559SDave Watson 1523c4d7559SDave Watson if (signal_pending(current)) { 1533c4d7559SDave Watson rc = sock_intr_errno(*timeo); 1543c4d7559SDave Watson break; 1553c4d7559SDave Watson } 1563c4d7559SDave Watson 157d0ac89f6SEric Dumazet if (sk_wait_event(sk, timeo, 158d0ac89f6SEric Dumazet !READ_ONCE(sk->sk_write_pending), &wait)) 1593c4d7559SDave Watson break; 1603c4d7559SDave Watson } 1613c4d7559SDave Watson remove_wait_queue(sk_sleep(sk), &wait); 1623c4d7559SDave Watson return rc; 1633c4d7559SDave Watson } 1643c4d7559SDave Watson 1653c4d7559SDave Watson int tls_push_sg(struct sock *sk, 1663c4d7559SDave Watson struct tls_context *ctx, 1673c4d7559SDave Watson struct scatterlist *sg, 1683c4d7559SDave Watson u16 first_offset, 1693c4d7559SDave Watson int flags) 1703c4d7559SDave Watson { 171e117dcfdSDavid Howells struct bio_vec bvec; 172e117dcfdSDavid Howells struct msghdr msg = { 173b848b26cSDavid Howells .msg_flags = MSG_SPLICE_PAGES | flags, 174e117dcfdSDavid Howells }; 1753c4d7559SDave Watson int ret = 0; 1763c4d7559SDave Watson struct page *p; 1773c4d7559SDave Watson size_t size; 1783c4d7559SDave Watson int offset = first_offset; 1793c4d7559SDave Watson 1803c4d7559SDave Watson size = sg->length - offset; 1813c4d7559SDave Watson offset += sg->offset; 1823c4d7559SDave Watson 183e117dcfdSDavid Howells ctx->splicing_pages = true; 1843c4d7559SDave Watson while (1) { 1853c4d7559SDave Watson /* is sending application-limited? */ 1863c4d7559SDave Watson tcp_rate_check_app_limited(sk); 1873c4d7559SDave Watson p = sg_page(sg); 1883c4d7559SDave Watson retry: 189e117dcfdSDavid Howells bvec_set_page(&bvec, p, size, offset); 190e117dcfdSDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size); 191e117dcfdSDavid Howells 192e117dcfdSDavid Howells ret = tcp_sendmsg_locked(sk, &msg, size); 1933c4d7559SDave Watson 1943c4d7559SDave Watson if (ret != size) { 1953c4d7559SDave Watson if (ret > 0) { 1963c4d7559SDave Watson offset += ret; 1973c4d7559SDave Watson size -= ret; 1983c4d7559SDave Watson goto retry; 1993c4d7559SDave Watson } 2003c4d7559SDave Watson 2013c4d7559SDave Watson offset -= sg->offset; 2023c4d7559SDave Watson ctx->partially_sent_offset = offset; 2033c4d7559SDave Watson ctx->partially_sent_record = (void *)sg; 204e117dcfdSDavid Howells ctx->splicing_pages = false; 2053c4d7559SDave Watson return ret; 2063c4d7559SDave Watson } 2073c4d7559SDave Watson 2083c4d7559SDave Watson put_page(p); 2093c4d7559SDave Watson sk_mem_uncharge(sk, sg->length); 2103c4d7559SDave Watson sg = sg_next(sg); 2113c4d7559SDave Watson if (!sg) 2123c4d7559SDave Watson break; 2133c4d7559SDave Watson 2143c4d7559SDave Watson offset = sg->offset; 2153c4d7559SDave Watson size = sg->length; 2163c4d7559SDave Watson } 2173c4d7559SDave Watson 218e117dcfdSDavid Howells ctx->splicing_pages = false; 2193c4d7559SDave Watson 2203c4d7559SDave Watson return 0; 2213c4d7559SDave Watson } 2223c4d7559SDave Watson 2233c4d7559SDave Watson static int tls_handle_open_record(struct sock *sk, int flags) 2243c4d7559SDave Watson { 2253c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 2263c4d7559SDave Watson 2273c4d7559SDave Watson if (tls_is_pending_open_record(ctx)) 2283c4d7559SDave Watson return ctx->push_pending_record(sk, flags); 2293c4d7559SDave Watson 2303c4d7559SDave Watson return 0; 2313c4d7559SDave Watson } 2323c4d7559SDave Watson 23358790314SJakub Kicinski int tls_process_cmsg(struct sock *sk, struct msghdr *msg, 2343c4d7559SDave Watson unsigned char *record_type) 2353c4d7559SDave Watson { 2363c4d7559SDave Watson struct cmsghdr *cmsg; 2373c4d7559SDave Watson int rc = -EINVAL; 2383c4d7559SDave Watson 2393c4d7559SDave Watson for_each_cmsghdr(cmsg, msg) { 2403c4d7559SDave Watson if (!CMSG_OK(msg, cmsg)) 2413c4d7559SDave Watson return -EINVAL; 2423c4d7559SDave Watson if (cmsg->cmsg_level != SOL_TLS) 2433c4d7559SDave Watson continue; 2443c4d7559SDave Watson 2453c4d7559SDave Watson switch (cmsg->cmsg_type) { 2463c4d7559SDave Watson case TLS_SET_RECORD_TYPE: 2473c4d7559SDave Watson if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) 2483c4d7559SDave Watson return -EINVAL; 2493c4d7559SDave Watson 2503c4d7559SDave Watson if (msg->msg_flags & MSG_MORE) 2513c4d7559SDave Watson return -EINVAL; 2523c4d7559SDave Watson 2533c4d7559SDave Watson rc = tls_handle_open_record(sk, msg->msg_flags); 2543c4d7559SDave Watson if (rc) 2553c4d7559SDave Watson return rc; 2563c4d7559SDave Watson 2573c4d7559SDave Watson *record_type = *(unsigned char *)CMSG_DATA(cmsg); 2583c4d7559SDave Watson rc = 0; 2593c4d7559SDave Watson break; 2603c4d7559SDave Watson default: 2613c4d7559SDave Watson return -EINVAL; 2623c4d7559SDave Watson } 2633c4d7559SDave Watson } 2643c4d7559SDave Watson 2653c4d7559SDave Watson return rc; 2663c4d7559SDave Watson } 2673c4d7559SDave Watson 268a42055e8SVakul Garg int tls_push_partial_record(struct sock *sk, struct tls_context *ctx, 269a42055e8SVakul Garg int flags) 2703c4d7559SDave Watson { 2713c4d7559SDave Watson struct scatterlist *sg; 2723c4d7559SDave Watson u16 offset; 2733c4d7559SDave Watson 2743c4d7559SDave Watson sg = ctx->partially_sent_record; 2753c4d7559SDave Watson offset = ctx->partially_sent_offset; 2763c4d7559SDave Watson 2773c4d7559SDave Watson ctx->partially_sent_record = NULL; 2783c4d7559SDave Watson return tls_push_sg(sk, ctx, sg, offset, flags); 2793c4d7559SDave Watson } 2803c4d7559SDave Watson 281c5daa6ccSJakub Kicinski void tls_free_partial_record(struct sock *sk, struct tls_context *ctx) 28235b71a34SJakub Kicinski { 28335b71a34SJakub Kicinski struct scatterlist *sg; 28435b71a34SJakub Kicinski 285c5daa6ccSJakub Kicinski for (sg = ctx->partially_sent_record; sg; sg = sg_next(sg)) { 28635b71a34SJakub Kicinski put_page(sg_page(sg)); 28735b71a34SJakub Kicinski sk_mem_uncharge(sk, sg->length); 28835b71a34SJakub Kicinski } 28935b71a34SJakub Kicinski ctx->partially_sent_record = NULL; 29035b71a34SJakub Kicinski } 29135b71a34SJakub Kicinski 2923c4d7559SDave Watson static void tls_write_space(struct sock *sk) 2933c4d7559SDave Watson { 2943c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 2953c4d7559SDave Watson 296e117dcfdSDavid Howells /* If splicing_pages call lower protocol write space handler 29767db7cd2SJohn Fastabend * to ensure we wake up any waiting operations there. For example 298e117dcfdSDavid Howells * if splicing pages where to call sk_wait_event. 29967db7cd2SJohn Fastabend */ 300e117dcfdSDavid Howells if (ctx->splicing_pages) { 30167db7cd2SJohn Fastabend ctx->sk_write_space(sk); 302c212d2c7SDave Watson return; 30367db7cd2SJohn Fastabend } 304c212d2c7SDave Watson 3057463d3a2SBoris Pismenny #ifdef CONFIG_TLS_DEVICE 3067463d3a2SBoris Pismenny if (ctx->tx_conf == TLS_HW) 3077463d3a2SBoris Pismenny tls_device_write_space(sk, ctx); 3087463d3a2SBoris Pismenny else 3097463d3a2SBoris Pismenny #endif 3107463d3a2SBoris Pismenny tls_sw_write_space(sk, ctx); 3114504ab0eSVakul Garg 3124504ab0eSVakul Garg ctx->sk_write_space(sk); 3133c4d7559SDave Watson } 3143c4d7559SDave Watson 31515a7dea7SJakub Kicinski /** 31615a7dea7SJakub Kicinski * tls_ctx_free() - free TLS ULP context 31715a7dea7SJakub Kicinski * @sk: socket to with @ctx is attached 31815a7dea7SJakub Kicinski * @ctx: TLS context structure 31915a7dea7SJakub Kicinski * 32015a7dea7SJakub Kicinski * Free TLS context. If @sk is %NULL caller guarantees that the socket 32115a7dea7SJakub Kicinski * to which @ctx was attached has no outstanding references. 32215a7dea7SJakub Kicinski */ 32315a7dea7SJakub Kicinski void tls_ctx_free(struct sock *sk, struct tls_context *ctx) 32486029d10SSabrina Dubroca { 32586029d10SSabrina Dubroca if (!ctx) 32686029d10SSabrina Dubroca return; 32786029d10SSabrina Dubroca 32886029d10SSabrina Dubroca memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send)); 32986029d10SSabrina Dubroca memzero_explicit(&ctx->crypto_recv, sizeof(ctx->crypto_recv)); 33079ffe608SJakub Kicinski mutex_destroy(&ctx->tx_lock); 33115a7dea7SJakub Kicinski 33215a7dea7SJakub Kicinski if (sk) 33315a7dea7SJakub Kicinski kfree_rcu(ctx, rcu); 33415a7dea7SJakub Kicinski else 33586029d10SSabrina Dubroca kfree(ctx); 33686029d10SSabrina Dubroca } 33786029d10SSabrina Dubroca 338313ab004SJohn Fastabend static void tls_sk_proto_cleanup(struct sock *sk, 339313ab004SJohn Fastabend struct tls_context *ctx, long timeo) 3403c4d7559SDave Watson { 3419354544cSDirk van der Merwe if (unlikely(sk->sk_write_pending) && 3429354544cSDirk van der Merwe !wait_on_pending_writer(sk, &timeo)) 3433c4d7559SDave Watson tls_handle_open_record(sk, 0); 3443c4d7559SDave Watson 345f66de3eeSBoris Pismenny /* We need these for tls_sw_fallback handling of other packets */ 346f66de3eeSBoris Pismenny if (ctx->tx_conf == TLS_SW) { 347dbe42559SDave Watson kfree(ctx->tx.rec_seq); 348dbe42559SDave Watson kfree(ctx->tx.iv); 349313ab004SJohn Fastabend tls_sw_release_resources_tx(sk); 350b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW); 35135b71a34SJakub Kicinski } else if (ctx->tx_conf == TLS_HW) { 35235b71a34SJakub Kicinski tls_device_free_resources_tx(sk); 353b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE); 354f66de3eeSBoris Pismenny } 355f66de3eeSBoris Pismenny 356b32fd3ccSJakub Kicinski if (ctx->rx_conf == TLS_SW) { 357313ab004SJohn Fastabend tls_sw_release_resources_rx(sk); 358b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW); 359b32fd3ccSJakub Kicinski } else if (ctx->rx_conf == TLS_HW) { 3604799ac81SBoris Pismenny tls_device_offload_cleanup_rx(sk); 361b32fd3ccSJakub Kicinski TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE); 362b32fd3ccSJakub Kicinski } 363e8f69799SIlya Lesokhin } 364e8f69799SIlya Lesokhin 365313ab004SJohn Fastabend static void tls_sk_proto_close(struct sock *sk, long timeout) 366313ab004SJohn Fastabend { 36795fa1454SJohn Fastabend struct inet_connection_sock *icsk = inet_csk(sk); 368313ab004SJohn Fastabend struct tls_context *ctx = tls_get_ctx(sk); 369313ab004SJohn Fastabend long timeo = sock_sndtimeo(sk, 0); 370313ab004SJohn Fastabend bool free_ctx; 371313ab004SJohn Fastabend 372313ab004SJohn Fastabend if (ctx->tx_conf == TLS_SW) 373313ab004SJohn Fastabend tls_sw_cancel_work_tx(ctx); 374313ab004SJohn Fastabend 375313ab004SJohn Fastabend lock_sock(sk); 376313ab004SJohn Fastabend free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW; 377313ab004SJohn Fastabend 378313ab004SJohn Fastabend if (ctx->tx_conf != TLS_BASE || ctx->rx_conf != TLS_BASE) 379313ab004SJohn Fastabend tls_sk_proto_cleanup(sk, ctx, timeo); 380313ab004SJohn Fastabend 38195fa1454SJohn Fastabend write_lock_bh(&sk->sk_callback_lock); 38295fa1454SJohn Fastabend if (free_ctx) 38315a7dea7SJakub Kicinski rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 384d5bee737SJakub Sitnicki WRITE_ONCE(sk->sk_prot, ctx->sk_proto); 385d85f0177SJohn Fastabend if (sk->sk_write_space == tls_write_space) 38657c722e9SJakub Kicinski sk->sk_write_space = ctx->sk_write_space; 38795fa1454SJohn Fastabend write_unlock_bh(&sk->sk_callback_lock); 3883c4d7559SDave Watson release_sock(sk); 389313ab004SJohn Fastabend if (ctx->tx_conf == TLS_SW) 390313ab004SJohn Fastabend tls_sw_free_ctx_tx(ctx); 391313ab004SJohn Fastabend if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) 392313ab004SJohn Fastabend tls_sw_strparser_done(ctx); 393313ab004SJohn Fastabend if (ctx->rx_conf == TLS_SW) 394313ab004SJohn Fastabend tls_sw_free_ctx_rx(ctx); 395be7bbea1SJakub Kicinski ctx->sk_proto->close(sk, timeout); 396313ab004SJohn Fastabend 39798f0a395SEric Dumazet if (free_ctx) 39815a7dea7SJakub Kicinski tls_ctx_free(sk, ctx); 3993c4d7559SDave Watson } 4003c4d7559SDave Watson 401121dca78SJakub Kicinski static __poll_t tls_sk_poll(struct file *file, struct socket *sock, 402121dca78SJakub Kicinski struct poll_table_struct *wait) 403121dca78SJakub Kicinski { 404121dca78SJakub Kicinski struct tls_sw_context_rx *ctx; 405121dca78SJakub Kicinski struct tls_context *tls_ctx; 406121dca78SJakub Kicinski struct sock *sk = sock->sk; 407121dca78SJakub Kicinski struct sk_psock *psock; 408121dca78SJakub Kicinski __poll_t mask = 0; 409121dca78SJakub Kicinski u8 shutdown; 410121dca78SJakub Kicinski int state; 411121dca78SJakub Kicinski 412121dca78SJakub Kicinski mask = tcp_poll(file, sock, wait); 413121dca78SJakub Kicinski 414121dca78SJakub Kicinski state = inet_sk_state_load(sk); 415121dca78SJakub Kicinski shutdown = READ_ONCE(sk->sk_shutdown); 416121dca78SJakub Kicinski if (unlikely(state != TCP_ESTABLISHED || shutdown & RCV_SHUTDOWN)) 417121dca78SJakub Kicinski return mask; 418121dca78SJakub Kicinski 419121dca78SJakub Kicinski tls_ctx = tls_get_ctx(sk); 420121dca78SJakub Kicinski ctx = tls_sw_ctx_rx(tls_ctx); 421121dca78SJakub Kicinski psock = sk_psock_get(sk); 422121dca78SJakub Kicinski 423121dca78SJakub Kicinski if (skb_queue_empty_lockless(&ctx->rx_list) && 424121dca78SJakub Kicinski !tls_strp_msg_ready(ctx) && 425121dca78SJakub Kicinski sk_psock_queue_empty(psock)) 426121dca78SJakub Kicinski mask &= ~(EPOLLIN | EPOLLRDNORM); 427121dca78SJakub Kicinski 428121dca78SJakub Kicinski if (psock) 429121dca78SJakub Kicinski sk_psock_put(sk, psock); 430121dca78SJakub Kicinski 431121dca78SJakub Kicinski return mask; 432121dca78SJakub Kicinski } 433121dca78SJakub Kicinski 434ffa81fa4SYutaro Hayakawa static int do_tls_getsockopt_conf(struct sock *sk, char __user *optval, 435ffa81fa4SYutaro Hayakawa int __user *optlen, int tx) 4363c4d7559SDave Watson { 4373c4d7559SDave Watson int rc = 0; 438*077e05d1SSabrina Dubroca const struct tls_cipher_desc *cipher_desc; 4393c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 4403c4d7559SDave Watson struct tls_crypto_info *crypto_info; 441ffa81fa4SYutaro Hayakawa struct cipher_context *cctx; 4423c4d7559SDave Watson int len; 4433c4d7559SDave Watson 4443c4d7559SDave Watson if (get_user(len, optlen)) 4453c4d7559SDave Watson return -EFAULT; 4463c4d7559SDave Watson 4473c4d7559SDave Watson if (!optval || (len < sizeof(*crypto_info))) { 4483c4d7559SDave Watson rc = -EINVAL; 4493c4d7559SDave Watson goto out; 4503c4d7559SDave Watson } 4513c4d7559SDave Watson 4523c4d7559SDave Watson if (!ctx) { 4533c4d7559SDave Watson rc = -EBUSY; 4543c4d7559SDave Watson goto out; 4553c4d7559SDave Watson } 4563c4d7559SDave Watson 4573c4d7559SDave Watson /* get user crypto info */ 458ffa81fa4SYutaro Hayakawa if (tx) { 45986029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 460ffa81fa4SYutaro Hayakawa cctx = &ctx->tx; 461ffa81fa4SYutaro Hayakawa } else { 462ffa81fa4SYutaro Hayakawa crypto_info = &ctx->crypto_recv.info; 463ffa81fa4SYutaro Hayakawa cctx = &ctx->rx; 464ffa81fa4SYutaro Hayakawa } 4653c4d7559SDave Watson 4663c4d7559SDave Watson if (!TLS_CRYPTO_INFO_READY(crypto_info)) { 4673c4d7559SDave Watson rc = -EBUSY; 4683c4d7559SDave Watson goto out; 4693c4d7559SDave Watson } 4703c4d7559SDave Watson 4715a3b886cSMatthias Rosenfelder if (len == sizeof(*crypto_info)) { 472ac55cd61SDan Carpenter if (copy_to_user(optval, crypto_info, sizeof(*crypto_info))) 473ac55cd61SDan Carpenter rc = -EFAULT; 4743c4d7559SDave Watson goto out; 4753c4d7559SDave Watson } 4763c4d7559SDave Watson 477*077e05d1SSabrina Dubroca cipher_desc = get_cipher_desc(crypto_info->cipher_type); 478*077e05d1SSabrina Dubroca if (!cipher_desc || len != cipher_desc->crypto_info) { 4793c4d7559SDave Watson rc = -EINVAL; 4803c4d7559SDave Watson goto out; 4813c4d7559SDave Watson } 482fb99bce7SDave Watson 483*077e05d1SSabrina Dubroca memcpy(crypto_info_iv(crypto_info, cipher_desc), 484*077e05d1SSabrina Dubroca cctx->iv + cipher_desc->salt, cipher_desc->iv); 485*077e05d1SSabrina Dubroca memcpy(crypto_info_rec_seq(crypto_info, cipher_desc), 486*077e05d1SSabrina Dubroca cctx->rec_seq, cipher_desc->rec_seq); 4873fb59a5dSTianjia Zhang 488*077e05d1SSabrina Dubroca if (copy_to_user(optval, crypto_info, cipher_desc->crypto_info)) 4893fb59a5dSTianjia Zhang rc = -EFAULT; 4903c4d7559SDave Watson 4913c4d7559SDave Watson out: 4923c4d7559SDave Watson return rc; 4933c4d7559SDave Watson } 4943c4d7559SDave Watson 495c1318b39SBoris Pismenny static int do_tls_getsockopt_tx_zc(struct sock *sk, char __user *optval, 496c1318b39SBoris Pismenny int __user *optlen) 497c1318b39SBoris Pismenny { 498c1318b39SBoris Pismenny struct tls_context *ctx = tls_get_ctx(sk); 499c1318b39SBoris Pismenny unsigned int value; 500c1318b39SBoris Pismenny int len; 501c1318b39SBoris Pismenny 502c1318b39SBoris Pismenny if (get_user(len, optlen)) 503c1318b39SBoris Pismenny return -EFAULT; 504c1318b39SBoris Pismenny 505c1318b39SBoris Pismenny if (len != sizeof(value)) 506c1318b39SBoris Pismenny return -EINVAL; 507c1318b39SBoris Pismenny 508c1318b39SBoris Pismenny value = ctx->zerocopy_sendfile; 509c1318b39SBoris Pismenny if (copy_to_user(optval, &value, sizeof(value))) 510c1318b39SBoris Pismenny return -EFAULT; 511c1318b39SBoris Pismenny 512c1318b39SBoris Pismenny return 0; 513c1318b39SBoris Pismenny } 514c1318b39SBoris Pismenny 51588527790SJakub Kicinski static int do_tls_getsockopt_no_pad(struct sock *sk, char __user *optval, 51688527790SJakub Kicinski int __user *optlen) 51788527790SJakub Kicinski { 51888527790SJakub Kicinski struct tls_context *ctx = tls_get_ctx(sk); 51957128e98SJakub Kicinski int value, len; 52088527790SJakub Kicinski 52188527790SJakub Kicinski if (ctx->prot_info.version != TLS_1_3_VERSION) 52288527790SJakub Kicinski return -EINVAL; 52388527790SJakub Kicinski 52488527790SJakub Kicinski if (get_user(len, optlen)) 52588527790SJakub Kicinski return -EFAULT; 52688527790SJakub Kicinski if (len < sizeof(value)) 52788527790SJakub Kicinski return -EINVAL; 52888527790SJakub Kicinski 52957128e98SJakub Kicinski value = -EINVAL; 53088527790SJakub Kicinski if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) 53188527790SJakub Kicinski value = ctx->rx_no_pad; 53257128e98SJakub Kicinski if (value < 0) 53357128e98SJakub Kicinski return value; 53488527790SJakub Kicinski 53588527790SJakub Kicinski if (put_user(sizeof(value), optlen)) 53688527790SJakub Kicinski return -EFAULT; 53788527790SJakub Kicinski if (copy_to_user(optval, &value, sizeof(value))) 53888527790SJakub Kicinski return -EFAULT; 53988527790SJakub Kicinski 54088527790SJakub Kicinski return 0; 54188527790SJakub Kicinski } 54288527790SJakub Kicinski 5433c4d7559SDave Watson static int do_tls_getsockopt(struct sock *sk, int optname, 5443c4d7559SDave Watson char __user *optval, int __user *optlen) 5453c4d7559SDave Watson { 5463c4d7559SDave Watson int rc = 0; 5473c4d7559SDave Watson 54849c47cc2SHangyu Hua lock_sock(sk); 54949c47cc2SHangyu Hua 5503c4d7559SDave Watson switch (optname) { 5513c4d7559SDave Watson case TLS_TX: 552ffa81fa4SYutaro Hayakawa case TLS_RX: 553ffa81fa4SYutaro Hayakawa rc = do_tls_getsockopt_conf(sk, optval, optlen, 554ffa81fa4SYutaro Hayakawa optname == TLS_TX); 5553c4d7559SDave Watson break; 556b489a6e5SMaxim Mikityanskiy case TLS_TX_ZEROCOPY_RO: 557c1318b39SBoris Pismenny rc = do_tls_getsockopt_tx_zc(sk, optval, optlen); 558c1318b39SBoris Pismenny break; 55988527790SJakub Kicinski case TLS_RX_EXPECT_NO_PAD: 56088527790SJakub Kicinski rc = do_tls_getsockopt_no_pad(sk, optval, optlen); 56188527790SJakub Kicinski break; 5623c4d7559SDave Watson default: 5633c4d7559SDave Watson rc = -ENOPROTOOPT; 5643c4d7559SDave Watson break; 5653c4d7559SDave Watson } 56649c47cc2SHangyu Hua 56749c47cc2SHangyu Hua release_sock(sk); 56849c47cc2SHangyu Hua 5693c4d7559SDave Watson return rc; 5703c4d7559SDave Watson } 5713c4d7559SDave Watson 5723c4d7559SDave Watson static int tls_getsockopt(struct sock *sk, int level, int optname, 5733c4d7559SDave Watson char __user *optval, int __user *optlen) 5743c4d7559SDave Watson { 5753c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 5763c4d7559SDave Watson 5773c4d7559SDave Watson if (level != SOL_TLS) 578be7bbea1SJakub Kicinski return ctx->sk_proto->getsockopt(sk, level, 579be7bbea1SJakub Kicinski optname, optval, optlen); 5803c4d7559SDave Watson 5813c4d7559SDave Watson return do_tls_getsockopt(sk, optname, optval, optlen); 5823c4d7559SDave Watson } 5833c4d7559SDave Watson 584a7b75c5aSChristoph Hellwig static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval, 585c46234ebSDave Watson unsigned int optlen, int tx) 5863c4d7559SDave Watson { 587196c31b4SIlya Lesokhin struct tls_crypto_info *crypto_info; 5884509de14SVakul Garg struct tls_crypto_info *alt_crypto_info; 5893c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 5905f309adeSSabrina Dubroca const struct tls_cipher_desc *cipher_desc; 5913c4d7559SDave Watson int rc = 0; 59258371585SDave Watson int conf; 5933c4d7559SDave Watson 5941ddcbfbfSZiyang Xuan if (sockptr_is_null(optval) || (optlen < sizeof(*crypto_info))) 5951ddcbfbfSZiyang Xuan return -EINVAL; 5963c4d7559SDave Watson 5974509de14SVakul Garg if (tx) { 59886029d10SSabrina Dubroca crypto_info = &ctx->crypto_send.info; 5994509de14SVakul Garg alt_crypto_info = &ctx->crypto_recv.info; 6004509de14SVakul Garg } else { 60186029d10SSabrina Dubroca crypto_info = &ctx->crypto_recv.info; 6024509de14SVakul Garg alt_crypto_info = &ctx->crypto_send.info; 6034509de14SVakul Garg } 604c46234ebSDave Watson 605196c31b4SIlya Lesokhin /* Currently we don't support set crypto info more than one time */ 6061ddcbfbfSZiyang Xuan if (TLS_CRYPTO_INFO_READY(crypto_info)) 6071ddcbfbfSZiyang Xuan return -EBUSY; 608196c31b4SIlya Lesokhin 609a7b75c5aSChristoph Hellwig rc = copy_from_sockptr(crypto_info, optval, sizeof(*crypto_info)); 6103c4d7559SDave Watson if (rc) { 6113c4d7559SDave Watson rc = -EFAULT; 612257082e6SBoris Pismenny goto err_crypto_info; 6133c4d7559SDave Watson } 6143c4d7559SDave Watson 6153c4d7559SDave Watson /* check version */ 616130b392cSDave Watson if (crypto_info->version != TLS_1_2_VERSION && 617130b392cSDave Watson crypto_info->version != TLS_1_3_VERSION) { 6184a5cdc60SValentin Vidic rc = -EINVAL; 619196c31b4SIlya Lesokhin goto err_crypto_info; 6203c4d7559SDave Watson } 6213c4d7559SDave Watson 6224509de14SVakul Garg /* Ensure that TLS version and ciphers are same in both directions */ 6234509de14SVakul Garg if (TLS_CRYPTO_INFO_READY(alt_crypto_info)) { 6244509de14SVakul Garg if (alt_crypto_info->version != crypto_info->version || 6254509de14SVakul Garg alt_crypto_info->cipher_type != crypto_info->cipher_type) { 6264509de14SVakul Garg rc = -EINVAL; 6274509de14SVakul Garg goto err_crypto_info; 6284509de14SVakul Garg } 6294509de14SVakul Garg } 6304509de14SVakul Garg 6315f309adeSSabrina Dubroca cipher_desc = get_cipher_desc(crypto_info->cipher_type); 6325f309adeSSabrina Dubroca if (!cipher_desc) { 63362e56ef5STaehee Yoo rc = -EINVAL; 63462e56ef5STaehee Yoo goto err_crypto_info; 63562e56ef5STaehee Yoo } 6365f309adeSSabrina Dubroca 6375f309adeSSabrina Dubroca switch (crypto_info->cipher_type) { 6385f309adeSSabrina Dubroca case TLS_CIPHER_ARIA_GCM_128: 63962e56ef5STaehee Yoo case TLS_CIPHER_ARIA_GCM_256: 64062e56ef5STaehee Yoo if (crypto_info->version != TLS_1_2_VERSION) { 64162e56ef5STaehee Yoo rc = -EINVAL; 64262e56ef5STaehee Yoo goto err_crypto_info; 64362e56ef5STaehee Yoo } 64462e56ef5STaehee Yoo break; 645f295b3aeSVakul Garg } 646f295b3aeSVakul Garg 6475f309adeSSabrina Dubroca if (optlen != cipher_desc->crypto_info) { 6483c4d7559SDave Watson rc = -EINVAL; 6496db959c8SSabrina Dubroca goto err_crypto_info; 6503c4d7559SDave Watson } 651f295b3aeSVakul Garg 652d3c48151SChristoph Hellwig rc = copy_from_sockptr_offset(crypto_info + 1, optval, 653d3c48151SChristoph Hellwig sizeof(*crypto_info), 654196c31b4SIlya Lesokhin optlen - sizeof(*crypto_info)); 6553c4d7559SDave Watson if (rc) { 6563c4d7559SDave Watson rc = -EFAULT; 6573c4d7559SDave Watson goto err_crypto_info; 6583c4d7559SDave Watson } 6593c4d7559SDave Watson 660c46234ebSDave Watson if (tx) { 661e8f69799SIlya Lesokhin rc = tls_set_device_offload(sk, ctx); 662e8f69799SIlya Lesokhin conf = TLS_HW; 663b32fd3ccSJakub Kicinski if (!rc) { 664b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE); 665b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE); 666b32fd3ccSJakub Kicinski } else { 667c46234ebSDave Watson rc = tls_set_sw_offload(sk, ctx, 1); 668318892acSJakub Kicinski if (rc) 669318892acSJakub Kicinski goto err_crypto_info; 670b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW); 671b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW); 672f66de3eeSBoris Pismenny conf = TLS_SW; 673e8f69799SIlya Lesokhin } 674c46234ebSDave Watson } else { 6754799ac81SBoris Pismenny rc = tls_set_device_offload_rx(sk, ctx); 6764799ac81SBoris Pismenny conf = TLS_HW; 677b32fd3ccSJakub Kicinski if (!rc) { 678b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE); 679b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE); 680b32fd3ccSJakub Kicinski } else { 681c46234ebSDave Watson rc = tls_set_sw_offload(sk, ctx, 0); 682318892acSJakub Kicinski if (rc) 683318892acSJakub Kicinski goto err_crypto_info; 684b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW); 685b32fd3ccSJakub Kicinski TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW); 686f66de3eeSBoris Pismenny conf = TLS_SW; 687c46234ebSDave Watson } 688313ab004SJohn Fastabend tls_sw_strparser_arm(sk, ctx); 6894799ac81SBoris Pismenny } 690c46234ebSDave Watson 691f66de3eeSBoris Pismenny if (tx) 692f66de3eeSBoris Pismenny ctx->tx_conf = conf; 693f66de3eeSBoris Pismenny else 694f66de3eeSBoris Pismenny ctx->rx_conf = conf; 6956d88207fSIlya Lesokhin update_sk_prot(sk, ctx); 696c46234ebSDave Watson if (tx) { 697ee181e52SIlya Lesokhin ctx->sk_write_space = sk->sk_write_space; 698ee181e52SIlya Lesokhin sk->sk_write_space = tls_write_space; 69984c61fe1SJakub Kicinski } else { 70084c61fe1SJakub Kicinski struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(ctx); 70184c61fe1SJakub Kicinski 70284c61fe1SJakub Kicinski tls_strp_check_rcv(&rx_ctx->strp); 703c46234ebSDave Watson } 7041ddcbfbfSZiyang Xuan return 0; 7053c4d7559SDave Watson 7063c4d7559SDave Watson err_crypto_info: 707c844eb46SSabrina Dubroca memzero_explicit(crypto_info, sizeof(union tls_crypto_context)); 7083c4d7559SDave Watson return rc; 7093c4d7559SDave Watson } 7103c4d7559SDave Watson 711c1318b39SBoris Pismenny static int do_tls_setsockopt_tx_zc(struct sock *sk, sockptr_t optval, 712c1318b39SBoris Pismenny unsigned int optlen) 713c1318b39SBoris Pismenny { 714c1318b39SBoris Pismenny struct tls_context *ctx = tls_get_ctx(sk); 715c1318b39SBoris Pismenny unsigned int value; 716c1318b39SBoris Pismenny 717c1318b39SBoris Pismenny if (sockptr_is_null(optval) || optlen != sizeof(value)) 718c1318b39SBoris Pismenny return -EINVAL; 719c1318b39SBoris Pismenny 720c1318b39SBoris Pismenny if (copy_from_sockptr(&value, optval, sizeof(value))) 721c1318b39SBoris Pismenny return -EFAULT; 722c1318b39SBoris Pismenny 723c1318b39SBoris Pismenny if (value > 1) 724c1318b39SBoris Pismenny return -EINVAL; 725c1318b39SBoris Pismenny 726c1318b39SBoris Pismenny ctx->zerocopy_sendfile = value; 727c1318b39SBoris Pismenny 728c1318b39SBoris Pismenny return 0; 729c1318b39SBoris Pismenny } 730c1318b39SBoris Pismenny 73188527790SJakub Kicinski static int do_tls_setsockopt_no_pad(struct sock *sk, sockptr_t optval, 73288527790SJakub Kicinski unsigned int optlen) 73388527790SJakub Kicinski { 73488527790SJakub Kicinski struct tls_context *ctx = tls_get_ctx(sk); 73588527790SJakub Kicinski u32 val; 73688527790SJakub Kicinski int rc; 73788527790SJakub Kicinski 73888527790SJakub Kicinski if (ctx->prot_info.version != TLS_1_3_VERSION || 73988527790SJakub Kicinski sockptr_is_null(optval) || optlen < sizeof(val)) 74088527790SJakub Kicinski return -EINVAL; 74188527790SJakub Kicinski 74288527790SJakub Kicinski rc = copy_from_sockptr(&val, optval, sizeof(val)); 74388527790SJakub Kicinski if (rc) 74488527790SJakub Kicinski return -EFAULT; 74588527790SJakub Kicinski if (val > 1) 74688527790SJakub Kicinski return -EINVAL; 74788527790SJakub Kicinski rc = check_zeroed_sockptr(optval, sizeof(val), optlen - sizeof(val)); 74888527790SJakub Kicinski if (rc < 1) 74988527790SJakub Kicinski return rc == 0 ? -EINVAL : rc; 75088527790SJakub Kicinski 75188527790SJakub Kicinski lock_sock(sk); 75288527790SJakub Kicinski rc = -EINVAL; 75388527790SJakub Kicinski if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) { 75488527790SJakub Kicinski ctx->rx_no_pad = val; 75588527790SJakub Kicinski tls_update_rx_zc_capable(ctx); 75688527790SJakub Kicinski rc = 0; 75788527790SJakub Kicinski } 75888527790SJakub Kicinski release_sock(sk); 75988527790SJakub Kicinski 76088527790SJakub Kicinski return rc; 76188527790SJakub Kicinski } 76288527790SJakub Kicinski 763a7b75c5aSChristoph Hellwig static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval, 764a7b75c5aSChristoph Hellwig unsigned int optlen) 7653c4d7559SDave Watson { 7663c4d7559SDave Watson int rc = 0; 7673c4d7559SDave Watson 7683c4d7559SDave Watson switch (optname) { 7693c4d7559SDave Watson case TLS_TX: 770c46234ebSDave Watson case TLS_RX: 7713c4d7559SDave Watson lock_sock(sk); 772c46234ebSDave Watson rc = do_tls_setsockopt_conf(sk, optval, optlen, 773c46234ebSDave Watson optname == TLS_TX); 7743c4d7559SDave Watson release_sock(sk); 7753c4d7559SDave Watson break; 776b489a6e5SMaxim Mikityanskiy case TLS_TX_ZEROCOPY_RO: 777c1318b39SBoris Pismenny lock_sock(sk); 778c1318b39SBoris Pismenny rc = do_tls_setsockopt_tx_zc(sk, optval, optlen); 779c1318b39SBoris Pismenny release_sock(sk); 780c1318b39SBoris Pismenny break; 78188527790SJakub Kicinski case TLS_RX_EXPECT_NO_PAD: 78288527790SJakub Kicinski rc = do_tls_setsockopt_no_pad(sk, optval, optlen); 78388527790SJakub Kicinski break; 7843c4d7559SDave Watson default: 7853c4d7559SDave Watson rc = -ENOPROTOOPT; 7863c4d7559SDave Watson break; 7873c4d7559SDave Watson } 7883c4d7559SDave Watson return rc; 7893c4d7559SDave Watson } 7903c4d7559SDave Watson 7913c4d7559SDave Watson static int tls_setsockopt(struct sock *sk, int level, int optname, 792a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 7933c4d7559SDave Watson { 7943c4d7559SDave Watson struct tls_context *ctx = tls_get_ctx(sk); 7953c4d7559SDave Watson 7963c4d7559SDave Watson if (level != SOL_TLS) 797be7bbea1SJakub Kicinski return ctx->sk_proto->setsockopt(sk, level, optname, optval, 798be7bbea1SJakub Kicinski optlen); 7993c4d7559SDave Watson 8003c4d7559SDave Watson return do_tls_setsockopt(sk, optname, optval, optlen); 8013c4d7559SDave Watson } 8023c4d7559SDave Watson 80308700dabSJakub Kicinski struct tls_context *tls_ctx_create(struct sock *sk) 804dd0bed16SAtul Gupta { 805dd0bed16SAtul Gupta struct inet_connection_sock *icsk = inet_csk(sk); 806dd0bed16SAtul Gupta struct tls_context *ctx; 807dd0bed16SAtul Gupta 808c6ec179aSGanesh Goudar ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC); 809dd0bed16SAtul Gupta if (!ctx) 810dd0bed16SAtul Gupta return NULL; 811dd0bed16SAtul Gupta 81279ffe608SJakub Kicinski mutex_init(&ctx->tx_lock); 81315a7dea7SJakub Kicinski rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 814d5bee737SJakub Sitnicki ctx->sk_proto = READ_ONCE(sk->sk_prot); 815c55dcdd4SMaxim Mikityanskiy ctx->sk = sk; 816dd0bed16SAtul Gupta return ctx; 817dd0bed16SAtul Gupta } 818dd0bed16SAtul Gupta 819f3911f73SJakub Kicinski static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 820f3911f73SJakub Kicinski const struct proto_ops *base) 821f3911f73SJakub Kicinski { 822f3911f73SJakub Kicinski ops[TLS_BASE][TLS_BASE] = *base; 823f3911f73SJakub Kicinski 824f3911f73SJakub Kicinski ops[TLS_SW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; 825df720d28SDavid Howells ops[TLS_SW ][TLS_BASE].splice_eof = tls_sw_splice_eof; 826f3911f73SJakub Kicinski 827f3911f73SJakub Kicinski ops[TLS_BASE][TLS_SW ] = ops[TLS_BASE][TLS_BASE]; 828f3911f73SJakub Kicinski ops[TLS_BASE][TLS_SW ].splice_read = tls_sw_splice_read; 829121dca78SJakub Kicinski ops[TLS_BASE][TLS_SW ].poll = tls_sk_poll; 830662fbcecSHannes Reinecke ops[TLS_BASE][TLS_SW ].read_sock = tls_sw_read_sock; 831f3911f73SJakub Kicinski 832f3911f73SJakub Kicinski ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE]; 833f3911f73SJakub Kicinski ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read; 834121dca78SJakub Kicinski ops[TLS_SW ][TLS_SW ].poll = tls_sk_poll; 835662fbcecSHannes Reinecke ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock; 836f3911f73SJakub Kicinski 837f3911f73SJakub Kicinski #ifdef CONFIG_TLS_DEVICE 838f3911f73SJakub Kicinski ops[TLS_HW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; 839f3911f73SJakub Kicinski 840f3911f73SJakub Kicinski ops[TLS_HW ][TLS_SW ] = ops[TLS_BASE][TLS_SW ]; 841f3911f73SJakub Kicinski 842f3911f73SJakub Kicinski ops[TLS_BASE][TLS_HW ] = ops[TLS_BASE][TLS_SW ]; 843f3911f73SJakub Kicinski 844f3911f73SJakub Kicinski ops[TLS_SW ][TLS_HW ] = ops[TLS_SW ][TLS_SW ]; 845f3911f73SJakub Kicinski 846f3911f73SJakub Kicinski ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ]; 847f3911f73SJakub Kicinski #endif 848f3911f73SJakub Kicinski #ifdef CONFIG_TLS_TOE 849f3911f73SJakub Kicinski ops[TLS_HW_RECORD][TLS_HW_RECORD] = *base; 850f3911f73SJakub Kicinski #endif 851f3911f73SJakub Kicinski } 852f3911f73SJakub Kicinski 85363a6b3feSAtul Gupta static void tls_build_proto(struct sock *sk) 85463a6b3feSAtul Gupta { 85563a6b3feSAtul Gupta int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 8569a893949SWill Deacon struct proto *prot = READ_ONCE(sk->sk_prot); 85763a6b3feSAtul Gupta 85863a6b3feSAtul Gupta /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */ 85963a6b3feSAtul Gupta if (ip_ver == TLSV6 && 8605bb4c45dSJakub Sitnicki unlikely(prot != smp_load_acquire(&saved_tcpv6_prot))) { 86163a6b3feSAtul Gupta mutex_lock(&tcpv6_prot_mutex); 8625bb4c45dSJakub Sitnicki if (likely(prot != saved_tcpv6_prot)) { 8635bb4c45dSJakub Sitnicki build_protos(tls_prots[TLSV6], prot); 864f3911f73SJakub Kicinski build_proto_ops(tls_proto_ops[TLSV6], 865f3911f73SJakub Kicinski sk->sk_socket->ops); 8665bb4c45dSJakub Sitnicki smp_store_release(&saved_tcpv6_prot, prot); 86763a6b3feSAtul Gupta } 86863a6b3feSAtul Gupta mutex_unlock(&tcpv6_prot_mutex); 86963a6b3feSAtul Gupta } 87063a6b3feSAtul Gupta 87163a6b3feSAtul Gupta if (ip_ver == TLSV4 && 8725bb4c45dSJakub Sitnicki unlikely(prot != smp_load_acquire(&saved_tcpv4_prot))) { 87363a6b3feSAtul Gupta mutex_lock(&tcpv4_prot_mutex); 8745bb4c45dSJakub Sitnicki if (likely(prot != saved_tcpv4_prot)) { 8755bb4c45dSJakub Sitnicki build_protos(tls_prots[TLSV4], prot); 876f3911f73SJakub Kicinski build_proto_ops(tls_proto_ops[TLSV4], 877f3911f73SJakub Kicinski sk->sk_socket->ops); 8785bb4c45dSJakub Sitnicki smp_store_release(&saved_tcpv4_prot, prot); 87963a6b3feSAtul Gupta } 88063a6b3feSAtul Gupta mutex_unlock(&tcpv4_prot_mutex); 88163a6b3feSAtul Gupta } 88263a6b3feSAtul Gupta } 88363a6b3feSAtul Gupta 884f66de3eeSBoris Pismenny static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 885f13fe3e6SJakub Sitnicki const struct proto *base) 886c113187dSBoris Pismenny { 887f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE] = *base; 888f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt; 889f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].getsockopt = tls_getsockopt; 890f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_BASE].close = tls_sk_proto_close; 891c113187dSBoris Pismenny 892f66de3eeSBoris Pismenny prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 893f66de3eeSBoris Pismenny prot[TLS_SW][TLS_BASE].sendmsg = tls_sw_sendmsg; 894df720d28SDavid Howells prot[TLS_SW][TLS_BASE].splice_eof = tls_sw_splice_eof; 895c46234ebSDave Watson 896f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE]; 897f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg; 8987b50ecfcSCong Wang prot[TLS_BASE][TLS_SW].sock_is_readable = tls_sw_sock_is_readable; 899f66de3eeSBoris Pismenny prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close; 900c46234ebSDave Watson 901f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE]; 902f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg; 9037b50ecfcSCong Wang prot[TLS_SW][TLS_SW].sock_is_readable = tls_sw_sock_is_readable; 904f66de3eeSBoris Pismenny prot[TLS_SW][TLS_SW].close = tls_sk_proto_close; 905dd0bed16SAtul Gupta 906e8f69799SIlya Lesokhin #ifdef CONFIG_TLS_DEVICE 907e8f69799SIlya Lesokhin prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 908e8f69799SIlya Lesokhin prot[TLS_HW][TLS_BASE].sendmsg = tls_device_sendmsg; 909d4c1e80bSDavid Howells prot[TLS_HW][TLS_BASE].splice_eof = tls_device_splice_eof; 910e8f69799SIlya Lesokhin 911e8f69799SIlya Lesokhin prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW]; 912e8f69799SIlya Lesokhin prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg; 913d4c1e80bSDavid Howells prot[TLS_HW][TLS_SW].splice_eof = tls_device_splice_eof; 9144799ac81SBoris Pismenny 9154799ac81SBoris Pismenny prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW]; 9164799ac81SBoris Pismenny 9174799ac81SBoris Pismenny prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW]; 9184799ac81SBoris Pismenny 9194799ac81SBoris Pismenny prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW]; 920e8f69799SIlya Lesokhin #endif 92153b4414aSJakub Kicinski #ifdef CONFIG_TLS_TOE 922f66de3eeSBoris Pismenny prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base; 9230eb8745eSJakub Kicinski prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_toe_hash; 9240eb8745eSJakub Kicinski prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_toe_unhash; 92553b4414aSJakub Kicinski #endif 926c113187dSBoris Pismenny } 927c113187dSBoris Pismenny 9283c4d7559SDave Watson static int tls_init(struct sock *sk) 9293c4d7559SDave Watson { 9303c4d7559SDave Watson struct tls_context *ctx; 9313c4d7559SDave Watson int rc = 0; 9323c4d7559SDave Watson 93316bed0e6SJakub Kicinski tls_build_proto(sk); 93416bed0e6SJakub Kicinski 93553b4414aSJakub Kicinski #ifdef CONFIG_TLS_TOE 9360eb8745eSJakub Kicinski if (tls_toe_bypass(sk)) 93795fa1454SJohn Fastabend return 0; 93853b4414aSJakub Kicinski #endif 939dd0bed16SAtul Gupta 940d91c3e17SIlya Lesokhin /* The TLS ulp is currently supported only for TCP sockets 941d91c3e17SIlya Lesokhin * in ESTABLISHED state. 942d91c3e17SIlya Lesokhin * Supporting sockets in LISTEN state will require us 943d91c3e17SIlya Lesokhin * to modify the accept implementation to clone rather then 944d91c3e17SIlya Lesokhin * share the ulp context. 945d91c3e17SIlya Lesokhin */ 946d91c3e17SIlya Lesokhin if (sk->sk_state != TCP_ESTABLISHED) 9474a5cdc60SValentin Vidic return -ENOTCONN; 948d91c3e17SIlya Lesokhin 9493c4d7559SDave Watson /* allocate tls context */ 95095fa1454SJohn Fastabend write_lock_bh(&sk->sk_callback_lock); 95108700dabSJakub Kicinski ctx = tls_ctx_create(sk); 9523c4d7559SDave Watson if (!ctx) { 9533c4d7559SDave Watson rc = -ENOMEM; 9543c4d7559SDave Watson goto out; 9553c4d7559SDave Watson } 9566d88207fSIlya Lesokhin 957f66de3eeSBoris Pismenny ctx->tx_conf = TLS_BASE; 958f66de3eeSBoris Pismenny ctx->rx_conf = TLS_BASE; 9596d88207fSIlya Lesokhin update_sk_prot(sk, ctx); 9603c4d7559SDave Watson out: 96195fa1454SJohn Fastabend write_unlock_bh(&sk->sk_callback_lock); 9623c4d7559SDave Watson return rc; 9633c4d7559SDave Watson } 9643c4d7559SDave Watson 96533bfe20dSJohn Fastabend static void tls_update(struct sock *sk, struct proto *p, 96633bfe20dSJohn Fastabend void (*write_space)(struct sock *sk)) 96795fa1454SJohn Fastabend { 96895fa1454SJohn Fastabend struct tls_context *ctx; 96995fa1454SJohn Fastabend 970e34a07c0SJakub Kicinski WARN_ON_ONCE(sk->sk_prot == p); 971e34a07c0SJakub Kicinski 97295fa1454SJohn Fastabend ctx = tls_get_ctx(sk); 97333bfe20dSJohn Fastabend if (likely(ctx)) { 97433bfe20dSJohn Fastabend ctx->sk_write_space = write_space; 97595fa1454SJohn Fastabend ctx->sk_proto = p; 97633bfe20dSJohn Fastabend } else { 977b8e202d1SJakub Sitnicki /* Pairs with lockless read in sk_clone_lock(). */ 978b8e202d1SJakub Sitnicki WRITE_ONCE(sk->sk_prot, p); 97933bfe20dSJohn Fastabend sk->sk_write_space = write_space; 98033bfe20dSJohn Fastabend } 98195fa1454SJohn Fastabend } 98295fa1454SJohn Fastabend 98358790314SJakub Kicinski static u16 tls_user_config(struct tls_context *ctx, bool tx) 98458790314SJakub Kicinski { 98558790314SJakub Kicinski u16 config = tx ? ctx->tx_conf : ctx->rx_conf; 98658790314SJakub Kicinski 98758790314SJakub Kicinski switch (config) { 98858790314SJakub Kicinski case TLS_BASE: 98958790314SJakub Kicinski return TLS_CONF_BASE; 99058790314SJakub Kicinski case TLS_SW: 99158790314SJakub Kicinski return TLS_CONF_SW; 99258790314SJakub Kicinski case TLS_HW: 99358790314SJakub Kicinski return TLS_CONF_HW; 99458790314SJakub Kicinski case TLS_HW_RECORD: 99558790314SJakub Kicinski return TLS_CONF_HW_RECORD; 99658790314SJakub Kicinski } 99758790314SJakub Kicinski return 0; 99858790314SJakub Kicinski } 99958790314SJakub Kicinski 100026811cc9SDavide Caratti static int tls_get_info(const struct sock *sk, struct sk_buff *skb) 100126811cc9SDavide Caratti { 100226811cc9SDavide Caratti u16 version, cipher_type; 100326811cc9SDavide Caratti struct tls_context *ctx; 100426811cc9SDavide Caratti struct nlattr *start; 100526811cc9SDavide Caratti int err; 100626811cc9SDavide Caratti 100726811cc9SDavide Caratti start = nla_nest_start_noflag(skb, INET_ULP_INFO_TLS); 100826811cc9SDavide Caratti if (!start) 100926811cc9SDavide Caratti return -EMSGSIZE; 101026811cc9SDavide Caratti 101126811cc9SDavide Caratti rcu_read_lock(); 101226811cc9SDavide Caratti ctx = rcu_dereference(inet_csk(sk)->icsk_ulp_data); 101326811cc9SDavide Caratti if (!ctx) { 101426811cc9SDavide Caratti err = 0; 101526811cc9SDavide Caratti goto nla_failure; 101626811cc9SDavide Caratti } 101726811cc9SDavide Caratti version = ctx->prot_info.version; 101826811cc9SDavide Caratti if (version) { 101926811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_VERSION, version); 102026811cc9SDavide Caratti if (err) 102126811cc9SDavide Caratti goto nla_failure; 102226811cc9SDavide Caratti } 102326811cc9SDavide Caratti cipher_type = ctx->prot_info.cipher_type; 102426811cc9SDavide Caratti if (cipher_type) { 102526811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_CIPHER, cipher_type); 102626811cc9SDavide Caratti if (err) 102726811cc9SDavide Caratti goto nla_failure; 102826811cc9SDavide Caratti } 102926811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_TXCONF, tls_user_config(ctx, true)); 103026811cc9SDavide Caratti if (err) 103126811cc9SDavide Caratti goto nla_failure; 103226811cc9SDavide Caratti 103326811cc9SDavide Caratti err = nla_put_u16(skb, TLS_INFO_RXCONF, tls_user_config(ctx, false)); 103426811cc9SDavide Caratti if (err) 103526811cc9SDavide Caratti goto nla_failure; 103626811cc9SDavide Caratti 1037c1318b39SBoris Pismenny if (ctx->tx_conf == TLS_HW && ctx->zerocopy_sendfile) { 1038b489a6e5SMaxim Mikityanskiy err = nla_put_flag(skb, TLS_INFO_ZC_RO_TX); 1039c1318b39SBoris Pismenny if (err) 1040c1318b39SBoris Pismenny goto nla_failure; 1041c1318b39SBoris Pismenny } 104288527790SJakub Kicinski if (ctx->rx_no_pad) { 104388527790SJakub Kicinski err = nla_put_flag(skb, TLS_INFO_RX_NO_PAD); 104488527790SJakub Kicinski if (err) 104588527790SJakub Kicinski goto nla_failure; 104688527790SJakub Kicinski } 1047c1318b39SBoris Pismenny 104826811cc9SDavide Caratti rcu_read_unlock(); 104926811cc9SDavide Caratti nla_nest_end(skb, start); 105026811cc9SDavide Caratti return 0; 105126811cc9SDavide Caratti 105226811cc9SDavide Caratti nla_failure: 105326811cc9SDavide Caratti rcu_read_unlock(); 105426811cc9SDavide Caratti nla_nest_cancel(skb, start); 105526811cc9SDavide Caratti return err; 105626811cc9SDavide Caratti } 105726811cc9SDavide Caratti 105826811cc9SDavide Caratti static size_t tls_get_info_size(const struct sock *sk) 105926811cc9SDavide Caratti { 106026811cc9SDavide Caratti size_t size = 0; 106126811cc9SDavide Caratti 106226811cc9SDavide Caratti size += nla_total_size(0) + /* INET_ULP_INFO_TLS */ 106326811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_VERSION */ 106426811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_CIPHER */ 106526811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_RXCONF */ 106626811cc9SDavide Caratti nla_total_size(sizeof(u16)) + /* TLS_INFO_TXCONF */ 1067b489a6e5SMaxim Mikityanskiy nla_total_size(0) + /* TLS_INFO_ZC_RO_TX */ 106888527790SJakub Kicinski nla_total_size(0) + /* TLS_INFO_RX_NO_PAD */ 106926811cc9SDavide Caratti 0; 107026811cc9SDavide Caratti 107126811cc9SDavide Caratti return size; 107226811cc9SDavide Caratti } 107326811cc9SDavide Caratti 1074d26b698dSJakub Kicinski static int __net_init tls_init_net(struct net *net) 1075d26b698dSJakub Kicinski { 1076d26b698dSJakub Kicinski int err; 1077d26b698dSJakub Kicinski 1078d26b698dSJakub Kicinski net->mib.tls_statistics = alloc_percpu(struct linux_tls_mib); 1079d26b698dSJakub Kicinski if (!net->mib.tls_statistics) 1080d26b698dSJakub Kicinski return -ENOMEM; 1081d26b698dSJakub Kicinski 1082d26b698dSJakub Kicinski err = tls_proc_init(net); 1083d26b698dSJakub Kicinski if (err) 1084d26b698dSJakub Kicinski goto err_free_stats; 1085d26b698dSJakub Kicinski 1086d26b698dSJakub Kicinski return 0; 1087d26b698dSJakub Kicinski err_free_stats: 1088d26b698dSJakub Kicinski free_percpu(net->mib.tls_statistics); 1089d26b698dSJakub Kicinski return err; 1090d26b698dSJakub Kicinski } 1091d26b698dSJakub Kicinski 1092d26b698dSJakub Kicinski static void __net_exit tls_exit_net(struct net *net) 1093d26b698dSJakub Kicinski { 1094d26b698dSJakub Kicinski tls_proc_fini(net); 1095d26b698dSJakub Kicinski free_percpu(net->mib.tls_statistics); 1096d26b698dSJakub Kicinski } 1097d26b698dSJakub Kicinski 1098d26b698dSJakub Kicinski static struct pernet_operations tls_proc_ops = { 1099d26b698dSJakub Kicinski .init = tls_init_net, 1100d26b698dSJakub Kicinski .exit = tls_exit_net, 1101d26b698dSJakub Kicinski }; 1102d26b698dSJakub Kicinski 11033c4d7559SDave Watson static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = { 11043c4d7559SDave Watson .name = "tls", 11053c4d7559SDave Watson .owner = THIS_MODULE, 11063c4d7559SDave Watson .init = tls_init, 110795fa1454SJohn Fastabend .update = tls_update, 110826811cc9SDavide Caratti .get_info = tls_get_info, 110926811cc9SDavide Caratti .get_info_size = tls_get_info_size, 11103c4d7559SDave Watson }; 11113c4d7559SDave Watson 11123c4d7559SDave Watson static int __init tls_register(void) 11133c4d7559SDave Watson { 1114d26b698dSJakub Kicinski int err; 1115d26b698dSJakub Kicinski 1116d26b698dSJakub Kicinski err = register_pernet_subsys(&tls_proc_ops); 1117d26b698dSJakub Kicinski if (err) 1118d26b698dSJakub Kicinski return err; 1119d26b698dSJakub Kicinski 112084c61fe1SJakub Kicinski err = tls_strp_dev_init(); 112184c61fe1SJakub Kicinski if (err) 112284c61fe1SJakub Kicinski goto err_pernet; 112384c61fe1SJakub Kicinski 11243d8c51b2STariq Toukan err = tls_device_init(); 112584c61fe1SJakub Kicinski if (err) 112684c61fe1SJakub Kicinski goto err_strp; 11273d8c51b2STariq Toukan 11283c4d7559SDave Watson tcp_register_ulp(&tcp_tls_ulp_ops); 11293c4d7559SDave Watson 11303c4d7559SDave Watson return 0; 113184c61fe1SJakub Kicinski err_strp: 113284c61fe1SJakub Kicinski tls_strp_dev_exit(); 113384c61fe1SJakub Kicinski err_pernet: 113484c61fe1SJakub Kicinski unregister_pernet_subsys(&tls_proc_ops); 113584c61fe1SJakub Kicinski return err; 11363c4d7559SDave Watson } 11373c4d7559SDave Watson 11383c4d7559SDave Watson static void __exit tls_unregister(void) 11393c4d7559SDave Watson { 11403c4d7559SDave Watson tcp_unregister_ulp(&tcp_tls_ulp_ops); 114184c61fe1SJakub Kicinski tls_strp_dev_exit(); 1142e8f69799SIlya Lesokhin tls_device_cleanup(); 1143d26b698dSJakub Kicinski unregister_pernet_subsys(&tls_proc_ops); 11443c4d7559SDave Watson } 11453c4d7559SDave Watson 11463c4d7559SDave Watson module_init(tls_register); 11473c4d7559SDave Watson module_exit(tls_unregister); 1148