1 /* 2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 36 #include <net/tcp.h> 37 #include <net/inet_common.h> 38 #include <linux/highmem.h> 39 #include <linux/netdevice.h> 40 #include <linux/sched/signal.h> 41 #include <linux/inetdevice.h> 42 43 #include <net/tls.h> 44 45 MODULE_AUTHOR("Mellanox Technologies"); 46 MODULE_DESCRIPTION("Transport Layer Security Support"); 47 MODULE_LICENSE("Dual BSD/GPL"); 48 49 enum { 50 TLSV4, 51 TLSV6, 52 TLS_NUM_PROTS, 53 }; 54 55 static struct proto *saved_tcpv6_prot; 56 static DEFINE_MUTEX(tcpv6_prot_mutex); 57 static LIST_HEAD(device_list); 58 static DEFINE_MUTEX(device_mutex); 59 static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 60 static struct proto_ops tls_sw_proto_ops; 61 62 static void update_sk_prot(struct sock *sk, struct tls_context *ctx) 63 { 64 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 65 66 sk->sk_prot = &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]; 67 } 68 69 int wait_on_pending_writer(struct sock *sk, long *timeo) 70 { 71 int rc = 0; 72 DEFINE_WAIT_FUNC(wait, woken_wake_function); 73 74 add_wait_queue(sk_sleep(sk), &wait); 75 while (1) { 76 if (!*timeo) { 77 rc = -EAGAIN; 78 break; 79 } 80 81 if (signal_pending(current)) { 82 rc = sock_intr_errno(*timeo); 83 break; 84 } 85 86 if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait)) 87 break; 88 } 89 remove_wait_queue(sk_sleep(sk), &wait); 90 return rc; 91 } 92 93 int tls_push_sg(struct sock *sk, 94 struct tls_context *ctx, 95 struct scatterlist *sg, 96 u16 first_offset, 97 int flags) 98 { 99 int sendpage_flags = flags | MSG_SENDPAGE_NOTLAST; 100 int ret = 0; 101 struct page *p; 102 size_t size; 103 int offset = first_offset; 104 105 size = sg->length - offset; 106 offset += sg->offset; 107 108 ctx->in_tcp_sendpages = true; 109 while (1) { 110 if (sg_is_last(sg)) 111 sendpage_flags = flags; 112 113 /* is sending application-limited? */ 114 tcp_rate_check_app_limited(sk); 115 p = sg_page(sg); 116 retry: 117 ret = do_tcp_sendpages(sk, p, offset, size, sendpage_flags); 118 119 if (ret != size) { 120 if (ret > 0) { 121 offset += ret; 122 size -= ret; 123 goto retry; 124 } 125 126 offset -= sg->offset; 127 ctx->partially_sent_offset = offset; 128 ctx->partially_sent_record = (void *)sg; 129 ctx->in_tcp_sendpages = false; 130 return ret; 131 } 132 133 put_page(p); 134 sk_mem_uncharge(sk, sg->length); 135 sg = sg_next(sg); 136 if (!sg) 137 break; 138 139 offset = sg->offset; 140 size = sg->length; 141 } 142 143 clear_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags); 144 ctx->in_tcp_sendpages = false; 145 ctx->sk_write_space(sk); 146 147 return 0; 148 } 149 150 static int tls_handle_open_record(struct sock *sk, int flags) 151 { 152 struct tls_context *ctx = tls_get_ctx(sk); 153 154 if (tls_is_pending_open_record(ctx)) 155 return ctx->push_pending_record(sk, flags); 156 157 return 0; 158 } 159 160 int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg, 161 unsigned char *record_type) 162 { 163 struct cmsghdr *cmsg; 164 int rc = -EINVAL; 165 166 for_each_cmsghdr(cmsg, msg) { 167 if (!CMSG_OK(msg, cmsg)) 168 return -EINVAL; 169 if (cmsg->cmsg_level != SOL_TLS) 170 continue; 171 172 switch (cmsg->cmsg_type) { 173 case TLS_SET_RECORD_TYPE: 174 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) 175 return -EINVAL; 176 177 if (msg->msg_flags & MSG_MORE) 178 return -EINVAL; 179 180 rc = tls_handle_open_record(sk, msg->msg_flags); 181 if (rc) 182 return rc; 183 184 *record_type = *(unsigned char *)CMSG_DATA(cmsg); 185 rc = 0; 186 break; 187 default: 188 return -EINVAL; 189 } 190 } 191 192 return rc; 193 } 194 195 int tls_push_pending_closed_record(struct sock *sk, struct tls_context *ctx, 196 int flags, long *timeo) 197 { 198 struct scatterlist *sg; 199 u16 offset; 200 201 if (!tls_is_partially_sent_record(ctx)) 202 return ctx->push_pending_record(sk, flags); 203 204 sg = ctx->partially_sent_record; 205 offset = ctx->partially_sent_offset; 206 207 ctx->partially_sent_record = NULL; 208 return tls_push_sg(sk, ctx, sg, offset, flags); 209 } 210 211 static void tls_write_space(struct sock *sk) 212 { 213 struct tls_context *ctx = tls_get_ctx(sk); 214 215 /* We are already sending pages, ignore notification */ 216 if (ctx->in_tcp_sendpages) 217 return; 218 219 if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) { 220 gfp_t sk_allocation = sk->sk_allocation; 221 int rc; 222 long timeo = 0; 223 224 sk->sk_allocation = GFP_ATOMIC; 225 rc = tls_push_pending_closed_record(sk, ctx, 226 MSG_DONTWAIT | 227 MSG_NOSIGNAL, 228 &timeo); 229 sk->sk_allocation = sk_allocation; 230 231 if (rc < 0) 232 return; 233 } 234 235 ctx->sk_write_space(sk); 236 } 237 238 static void tls_sk_proto_close(struct sock *sk, long timeout) 239 { 240 struct tls_context *ctx = tls_get_ctx(sk); 241 long timeo = sock_sndtimeo(sk, 0); 242 void (*sk_proto_close)(struct sock *sk, long timeout); 243 bool free_ctx = false; 244 245 lock_sock(sk); 246 sk_proto_close = ctx->sk_proto_close; 247 248 if ((ctx->tx_conf == TLS_HW_RECORD && ctx->rx_conf == TLS_HW_RECORD) || 249 (ctx->tx_conf == TLS_BASE && ctx->rx_conf == TLS_BASE)) { 250 free_ctx = true; 251 goto skip_tx_cleanup; 252 } 253 254 if (!tls_complete_pending_work(sk, ctx, 0, &timeo)) 255 tls_handle_open_record(sk, 0); 256 257 if (ctx->partially_sent_record) { 258 struct scatterlist *sg = ctx->partially_sent_record; 259 260 while (1) { 261 put_page(sg_page(sg)); 262 sk_mem_uncharge(sk, sg->length); 263 264 if (sg_is_last(sg)) 265 break; 266 sg++; 267 } 268 } 269 270 /* We need these for tls_sw_fallback handling of other packets */ 271 if (ctx->tx_conf == TLS_SW) { 272 kfree(ctx->tx.rec_seq); 273 kfree(ctx->tx.iv); 274 tls_sw_free_resources_tx(sk); 275 } 276 277 if (ctx->rx_conf == TLS_SW) { 278 kfree(ctx->rx.rec_seq); 279 kfree(ctx->rx.iv); 280 tls_sw_free_resources_rx(sk); 281 } 282 283 #ifdef CONFIG_TLS_DEVICE 284 if (ctx->rx_conf == TLS_HW) 285 tls_device_offload_cleanup_rx(sk); 286 287 if (ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW) { 288 #else 289 { 290 #endif 291 kfree(ctx); 292 ctx = NULL; 293 } 294 295 skip_tx_cleanup: 296 release_sock(sk); 297 sk_proto_close(sk, timeout); 298 /* free ctx for TLS_HW_RECORD, used by tcp_set_state 299 * for sk->sk_prot->unhash [tls_hw_unhash] 300 */ 301 if (free_ctx) 302 kfree(ctx); 303 } 304 305 static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval, 306 int __user *optlen) 307 { 308 int rc = 0; 309 struct tls_context *ctx = tls_get_ctx(sk); 310 struct tls_crypto_info *crypto_info; 311 int len; 312 313 if (get_user(len, optlen)) 314 return -EFAULT; 315 316 if (!optval || (len < sizeof(*crypto_info))) { 317 rc = -EINVAL; 318 goto out; 319 } 320 321 if (!ctx) { 322 rc = -EBUSY; 323 goto out; 324 } 325 326 /* get user crypto info */ 327 crypto_info = &ctx->crypto_send; 328 329 if (!TLS_CRYPTO_INFO_READY(crypto_info)) { 330 rc = -EBUSY; 331 goto out; 332 } 333 334 if (len == sizeof(*crypto_info)) { 335 if (copy_to_user(optval, crypto_info, sizeof(*crypto_info))) 336 rc = -EFAULT; 337 goto out; 338 } 339 340 switch (crypto_info->cipher_type) { 341 case TLS_CIPHER_AES_GCM_128: { 342 struct tls12_crypto_info_aes_gcm_128 * 343 crypto_info_aes_gcm_128 = 344 container_of(crypto_info, 345 struct tls12_crypto_info_aes_gcm_128, 346 info); 347 348 if (len != sizeof(*crypto_info_aes_gcm_128)) { 349 rc = -EINVAL; 350 goto out; 351 } 352 lock_sock(sk); 353 memcpy(crypto_info_aes_gcm_128->iv, 354 ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 355 TLS_CIPHER_AES_GCM_128_IV_SIZE); 356 memcpy(crypto_info_aes_gcm_128->rec_seq, ctx->tx.rec_seq, 357 TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); 358 release_sock(sk); 359 if (copy_to_user(optval, 360 crypto_info_aes_gcm_128, 361 sizeof(*crypto_info_aes_gcm_128))) 362 rc = -EFAULT; 363 break; 364 } 365 default: 366 rc = -EINVAL; 367 } 368 369 out: 370 return rc; 371 } 372 373 static int do_tls_getsockopt(struct sock *sk, int optname, 374 char __user *optval, int __user *optlen) 375 { 376 int rc = 0; 377 378 switch (optname) { 379 case TLS_TX: 380 rc = do_tls_getsockopt_tx(sk, optval, optlen); 381 break; 382 default: 383 rc = -ENOPROTOOPT; 384 break; 385 } 386 return rc; 387 } 388 389 static int tls_getsockopt(struct sock *sk, int level, int optname, 390 char __user *optval, int __user *optlen) 391 { 392 struct tls_context *ctx = tls_get_ctx(sk); 393 394 if (level != SOL_TLS) 395 return ctx->getsockopt(sk, level, optname, optval, optlen); 396 397 return do_tls_getsockopt(sk, optname, optval, optlen); 398 } 399 400 static int do_tls_setsockopt_conf(struct sock *sk, char __user *optval, 401 unsigned int optlen, int tx) 402 { 403 struct tls_crypto_info *crypto_info; 404 struct tls_context *ctx = tls_get_ctx(sk); 405 int rc = 0; 406 int conf; 407 408 if (!optval || (optlen < sizeof(*crypto_info))) { 409 rc = -EINVAL; 410 goto out; 411 } 412 413 if (tx) 414 crypto_info = &ctx->crypto_send; 415 else 416 crypto_info = &ctx->crypto_recv; 417 418 /* Currently we don't support set crypto info more than one time */ 419 if (TLS_CRYPTO_INFO_READY(crypto_info)) { 420 rc = -EBUSY; 421 goto out; 422 } 423 424 rc = copy_from_user(crypto_info, optval, sizeof(*crypto_info)); 425 if (rc) { 426 rc = -EFAULT; 427 goto err_crypto_info; 428 } 429 430 /* check version */ 431 if (crypto_info->version != TLS_1_2_VERSION) { 432 rc = -ENOTSUPP; 433 goto err_crypto_info; 434 } 435 436 switch (crypto_info->cipher_type) { 437 case TLS_CIPHER_AES_GCM_128: { 438 if (optlen != sizeof(struct tls12_crypto_info_aes_gcm_128)) { 439 rc = -EINVAL; 440 goto err_crypto_info; 441 } 442 rc = copy_from_user(crypto_info + 1, optval + sizeof(*crypto_info), 443 optlen - sizeof(*crypto_info)); 444 if (rc) { 445 rc = -EFAULT; 446 goto err_crypto_info; 447 } 448 break; 449 } 450 default: 451 rc = -EINVAL; 452 goto err_crypto_info; 453 } 454 455 if (tx) { 456 #ifdef CONFIG_TLS_DEVICE 457 rc = tls_set_device_offload(sk, ctx); 458 conf = TLS_HW; 459 if (rc) { 460 #else 461 { 462 #endif 463 rc = tls_set_sw_offload(sk, ctx, 1); 464 conf = TLS_SW; 465 } 466 } else { 467 #ifdef CONFIG_TLS_DEVICE 468 rc = tls_set_device_offload_rx(sk, ctx); 469 conf = TLS_HW; 470 if (rc) { 471 #else 472 { 473 #endif 474 rc = tls_set_sw_offload(sk, ctx, 0); 475 conf = TLS_SW; 476 } 477 } 478 479 if (rc) 480 goto err_crypto_info; 481 482 if (tx) 483 ctx->tx_conf = conf; 484 else 485 ctx->rx_conf = conf; 486 update_sk_prot(sk, ctx); 487 if (tx) { 488 ctx->sk_write_space = sk->sk_write_space; 489 sk->sk_write_space = tls_write_space; 490 } else { 491 sk->sk_socket->ops = &tls_sw_proto_ops; 492 } 493 goto out; 494 495 err_crypto_info: 496 memset(crypto_info, 0, sizeof(*crypto_info)); 497 out: 498 return rc; 499 } 500 501 static int do_tls_setsockopt(struct sock *sk, int optname, 502 char __user *optval, unsigned int optlen) 503 { 504 int rc = 0; 505 506 switch (optname) { 507 case TLS_TX: 508 case TLS_RX: 509 lock_sock(sk); 510 rc = do_tls_setsockopt_conf(sk, optval, optlen, 511 optname == TLS_TX); 512 release_sock(sk); 513 break; 514 default: 515 rc = -ENOPROTOOPT; 516 break; 517 } 518 return rc; 519 } 520 521 static int tls_setsockopt(struct sock *sk, int level, int optname, 522 char __user *optval, unsigned int optlen) 523 { 524 struct tls_context *ctx = tls_get_ctx(sk); 525 526 if (level != SOL_TLS) 527 return ctx->setsockopt(sk, level, optname, optval, optlen); 528 529 return do_tls_setsockopt(sk, optname, optval, optlen); 530 } 531 532 static struct tls_context *create_ctx(struct sock *sk) 533 { 534 struct inet_connection_sock *icsk = inet_csk(sk); 535 struct tls_context *ctx; 536 537 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 538 if (!ctx) 539 return NULL; 540 541 icsk->icsk_ulp_data = ctx; 542 return ctx; 543 } 544 545 static int tls_hw_prot(struct sock *sk) 546 { 547 struct tls_context *ctx; 548 struct tls_device *dev; 549 int rc = 0; 550 551 mutex_lock(&device_mutex); 552 list_for_each_entry(dev, &device_list, dev_list) { 553 if (dev->feature && dev->feature(dev)) { 554 ctx = create_ctx(sk); 555 if (!ctx) 556 goto out; 557 558 ctx->hash = sk->sk_prot->hash; 559 ctx->unhash = sk->sk_prot->unhash; 560 ctx->sk_proto_close = sk->sk_prot->close; 561 ctx->rx_conf = TLS_HW_RECORD; 562 ctx->tx_conf = TLS_HW_RECORD; 563 update_sk_prot(sk, ctx); 564 rc = 1; 565 break; 566 } 567 } 568 out: 569 mutex_unlock(&device_mutex); 570 return rc; 571 } 572 573 static void tls_hw_unhash(struct sock *sk) 574 { 575 struct tls_context *ctx = tls_get_ctx(sk); 576 struct tls_device *dev; 577 578 mutex_lock(&device_mutex); 579 list_for_each_entry(dev, &device_list, dev_list) { 580 if (dev->unhash) 581 dev->unhash(dev, sk); 582 } 583 mutex_unlock(&device_mutex); 584 ctx->unhash(sk); 585 } 586 587 static int tls_hw_hash(struct sock *sk) 588 { 589 struct tls_context *ctx = tls_get_ctx(sk); 590 struct tls_device *dev; 591 int err; 592 593 err = ctx->hash(sk); 594 mutex_lock(&device_mutex); 595 list_for_each_entry(dev, &device_list, dev_list) { 596 if (dev->hash) 597 err |= dev->hash(dev, sk); 598 } 599 mutex_unlock(&device_mutex); 600 601 if (err) 602 tls_hw_unhash(sk); 603 return err; 604 } 605 606 static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 607 struct proto *base) 608 { 609 prot[TLS_BASE][TLS_BASE] = *base; 610 prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt; 611 prot[TLS_BASE][TLS_BASE].getsockopt = tls_getsockopt; 612 prot[TLS_BASE][TLS_BASE].close = tls_sk_proto_close; 613 614 prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 615 prot[TLS_SW][TLS_BASE].sendmsg = tls_sw_sendmsg; 616 prot[TLS_SW][TLS_BASE].sendpage = tls_sw_sendpage; 617 618 prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE]; 619 prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg; 620 prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close; 621 622 prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE]; 623 prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg; 624 prot[TLS_SW][TLS_SW].close = tls_sk_proto_close; 625 626 #ifdef CONFIG_TLS_DEVICE 627 prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 628 prot[TLS_HW][TLS_BASE].sendmsg = tls_device_sendmsg; 629 prot[TLS_HW][TLS_BASE].sendpage = tls_device_sendpage; 630 631 prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW]; 632 prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg; 633 prot[TLS_HW][TLS_SW].sendpage = tls_device_sendpage; 634 635 prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW]; 636 637 prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW]; 638 639 prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW]; 640 #endif 641 642 prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base; 643 prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_hw_hash; 644 prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_hw_unhash; 645 prot[TLS_HW_RECORD][TLS_HW_RECORD].close = tls_sk_proto_close; 646 } 647 648 static int tls_init(struct sock *sk) 649 { 650 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 651 struct tls_context *ctx; 652 int rc = 0; 653 654 if (tls_hw_prot(sk)) 655 goto out; 656 657 /* The TLS ulp is currently supported only for TCP sockets 658 * in ESTABLISHED state. 659 * Supporting sockets in LISTEN state will require us 660 * to modify the accept implementation to clone rather then 661 * share the ulp context. 662 */ 663 if (sk->sk_state != TCP_ESTABLISHED) 664 return -ENOTSUPP; 665 666 /* allocate tls context */ 667 ctx = create_ctx(sk); 668 if (!ctx) { 669 rc = -ENOMEM; 670 goto out; 671 } 672 ctx->setsockopt = sk->sk_prot->setsockopt; 673 ctx->getsockopt = sk->sk_prot->getsockopt; 674 ctx->sk_proto_close = sk->sk_prot->close; 675 676 /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */ 677 if (ip_ver == TLSV6 && 678 unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) { 679 mutex_lock(&tcpv6_prot_mutex); 680 if (likely(sk->sk_prot != saved_tcpv6_prot)) { 681 build_protos(tls_prots[TLSV6], sk->sk_prot); 682 smp_store_release(&saved_tcpv6_prot, sk->sk_prot); 683 } 684 mutex_unlock(&tcpv6_prot_mutex); 685 } 686 687 ctx->tx_conf = TLS_BASE; 688 ctx->rx_conf = TLS_BASE; 689 update_sk_prot(sk, ctx); 690 out: 691 return rc; 692 } 693 694 void tls_register_device(struct tls_device *device) 695 { 696 mutex_lock(&device_mutex); 697 list_add_tail(&device->dev_list, &device_list); 698 mutex_unlock(&device_mutex); 699 } 700 EXPORT_SYMBOL(tls_register_device); 701 702 void tls_unregister_device(struct tls_device *device) 703 { 704 mutex_lock(&device_mutex); 705 list_del(&device->dev_list); 706 mutex_unlock(&device_mutex); 707 } 708 EXPORT_SYMBOL(tls_unregister_device); 709 710 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = { 711 .name = "tls", 712 .uid = TCP_ULP_TLS, 713 .user_visible = true, 714 .owner = THIS_MODULE, 715 .init = tls_init, 716 }; 717 718 static int __init tls_register(void) 719 { 720 build_protos(tls_prots[TLSV4], &tcp_prot); 721 722 tls_sw_proto_ops = inet_stream_ops; 723 tls_sw_proto_ops.poll = tls_sw_poll; 724 tls_sw_proto_ops.splice_read = tls_sw_splice_read; 725 726 #ifdef CONFIG_TLS_DEVICE 727 tls_device_init(); 728 #endif 729 tcp_register_ulp(&tcp_tls_ulp_ops); 730 731 return 0; 732 } 733 734 static void __exit tls_unregister(void) 735 { 736 tcp_unregister_ulp(&tcp_tls_ulp_ops); 737 #ifdef CONFIG_TLS_DEVICE 738 tls_device_cleanup(); 739 #endif 740 } 741 742 module_init(tls_register); 743 module_exit(tls_unregister); 744