1 /* 2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 36 #include <net/tcp.h> 37 #include <net/inet_common.h> 38 #include <linux/highmem.h> 39 #include <linux/netdevice.h> 40 #include <linux/sched/signal.h> 41 42 #include <net/tls.h> 43 44 MODULE_AUTHOR("Mellanox Technologies"); 45 MODULE_DESCRIPTION("Transport Layer Security Support"); 46 MODULE_LICENSE("Dual BSD/GPL"); 47 48 enum { 49 TLSV4, 50 TLSV6, 51 TLS_NUM_PROTS, 52 }; 53 54 enum { 55 TLS_BASE_TX, 56 TLS_SW_TX, 57 TLS_NUM_CONFIG, 58 }; 59 60 static struct proto *saved_tcpv6_prot; 61 static DEFINE_MUTEX(tcpv6_prot_mutex); 62 static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG]; 63 64 static inline void update_sk_prot(struct sock *sk, struct tls_context *ctx) 65 { 66 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 67 68 sk->sk_prot = &tls_prots[ip_ver][ctx->tx_conf]; 69 } 70 71 int wait_on_pending_writer(struct sock *sk, long *timeo) 72 { 73 int rc = 0; 74 DEFINE_WAIT_FUNC(wait, woken_wake_function); 75 76 add_wait_queue(sk_sleep(sk), &wait); 77 while (1) { 78 if (!*timeo) { 79 rc = -EAGAIN; 80 break; 81 } 82 83 if (signal_pending(current)) { 84 rc = sock_intr_errno(*timeo); 85 break; 86 } 87 88 if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait)) 89 break; 90 } 91 remove_wait_queue(sk_sleep(sk), &wait); 92 return rc; 93 } 94 95 int tls_push_sg(struct sock *sk, 96 struct tls_context *ctx, 97 struct scatterlist *sg, 98 u16 first_offset, 99 int flags) 100 { 101 int sendpage_flags = flags | MSG_SENDPAGE_NOTLAST; 102 int ret = 0; 103 struct page *p; 104 size_t size; 105 int offset = first_offset; 106 107 size = sg->length - offset; 108 offset += sg->offset; 109 110 while (1) { 111 if (sg_is_last(sg)) 112 sendpage_flags = flags; 113 114 /* is sending application-limited? */ 115 tcp_rate_check_app_limited(sk); 116 p = sg_page(sg); 117 retry: 118 ret = do_tcp_sendpages(sk, p, offset, size, sendpage_flags); 119 120 if (ret != size) { 121 if (ret > 0) { 122 offset += ret; 123 size -= ret; 124 goto retry; 125 } 126 127 offset -= sg->offset; 128 ctx->partially_sent_offset = offset; 129 ctx->partially_sent_record = (void *)sg; 130 return ret; 131 } 132 133 put_page(p); 134 sk_mem_uncharge(sk, sg->length); 135 sg = sg_next(sg); 136 if (!sg) 137 break; 138 139 offset = sg->offset; 140 size = sg->length; 141 } 142 143 clear_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags); 144 145 return 0; 146 } 147 148 static int tls_handle_open_record(struct sock *sk, int flags) 149 { 150 struct tls_context *ctx = tls_get_ctx(sk); 151 152 if (tls_is_pending_open_record(ctx)) 153 return ctx->push_pending_record(sk, flags); 154 155 return 0; 156 } 157 158 int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg, 159 unsigned char *record_type) 160 { 161 struct cmsghdr *cmsg; 162 int rc = -EINVAL; 163 164 for_each_cmsghdr(cmsg, msg) { 165 if (!CMSG_OK(msg, cmsg)) 166 return -EINVAL; 167 if (cmsg->cmsg_level != SOL_TLS) 168 continue; 169 170 switch (cmsg->cmsg_type) { 171 case TLS_SET_RECORD_TYPE: 172 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) 173 return -EINVAL; 174 175 if (msg->msg_flags & MSG_MORE) 176 return -EINVAL; 177 178 rc = tls_handle_open_record(sk, msg->msg_flags); 179 if (rc) 180 return rc; 181 182 *record_type = *(unsigned char *)CMSG_DATA(cmsg); 183 rc = 0; 184 break; 185 default: 186 return -EINVAL; 187 } 188 } 189 190 return rc; 191 } 192 193 int tls_push_pending_closed_record(struct sock *sk, struct tls_context *ctx, 194 int flags, long *timeo) 195 { 196 struct scatterlist *sg; 197 u16 offset; 198 199 if (!tls_is_partially_sent_record(ctx)) 200 return ctx->push_pending_record(sk, flags); 201 202 sg = ctx->partially_sent_record; 203 offset = ctx->partially_sent_offset; 204 205 ctx->partially_sent_record = NULL; 206 return tls_push_sg(sk, ctx, sg, offset, flags); 207 } 208 209 static void tls_write_space(struct sock *sk) 210 { 211 struct tls_context *ctx = tls_get_ctx(sk); 212 213 if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) { 214 gfp_t sk_allocation = sk->sk_allocation; 215 int rc; 216 long timeo = 0; 217 218 sk->sk_allocation = GFP_ATOMIC; 219 rc = tls_push_pending_closed_record(sk, ctx, 220 MSG_DONTWAIT | 221 MSG_NOSIGNAL, 222 &timeo); 223 sk->sk_allocation = sk_allocation; 224 225 if (rc < 0) 226 return; 227 } 228 229 ctx->sk_write_space(sk); 230 } 231 232 static void tls_sk_proto_close(struct sock *sk, long timeout) 233 { 234 struct tls_context *ctx = tls_get_ctx(sk); 235 long timeo = sock_sndtimeo(sk, 0); 236 void (*sk_proto_close)(struct sock *sk, long timeout); 237 238 lock_sock(sk); 239 sk_proto_close = ctx->sk_proto_close; 240 241 if (ctx->tx_conf == TLS_BASE_TX) { 242 kfree(ctx); 243 goto skip_tx_cleanup; 244 } 245 246 if (!tls_complete_pending_work(sk, ctx, 0, &timeo)) 247 tls_handle_open_record(sk, 0); 248 249 if (ctx->partially_sent_record) { 250 struct scatterlist *sg = ctx->partially_sent_record; 251 252 while (1) { 253 put_page(sg_page(sg)); 254 sk_mem_uncharge(sk, sg->length); 255 256 if (sg_is_last(sg)) 257 break; 258 sg++; 259 } 260 } 261 262 kfree(ctx->rec_seq); 263 kfree(ctx->iv); 264 265 if (ctx->tx_conf == TLS_SW_TX) 266 tls_sw_free_tx_resources(sk); 267 268 skip_tx_cleanup: 269 release_sock(sk); 270 sk_proto_close(sk, timeout); 271 } 272 273 static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval, 274 int __user *optlen) 275 { 276 int rc = 0; 277 struct tls_context *ctx = tls_get_ctx(sk); 278 struct tls_crypto_info *crypto_info; 279 int len; 280 281 if (get_user(len, optlen)) 282 return -EFAULT; 283 284 if (!optval || (len < sizeof(*crypto_info))) { 285 rc = -EINVAL; 286 goto out; 287 } 288 289 if (!ctx) { 290 rc = -EBUSY; 291 goto out; 292 } 293 294 /* get user crypto info */ 295 crypto_info = &ctx->crypto_send; 296 297 if (!TLS_CRYPTO_INFO_READY(crypto_info)) { 298 rc = -EBUSY; 299 goto out; 300 } 301 302 if (len == sizeof(*crypto_info)) { 303 if (copy_to_user(optval, crypto_info, sizeof(*crypto_info))) 304 rc = -EFAULT; 305 goto out; 306 } 307 308 switch (crypto_info->cipher_type) { 309 case TLS_CIPHER_AES_GCM_128: { 310 struct tls12_crypto_info_aes_gcm_128 * 311 crypto_info_aes_gcm_128 = 312 container_of(crypto_info, 313 struct tls12_crypto_info_aes_gcm_128, 314 info); 315 316 if (len != sizeof(*crypto_info_aes_gcm_128)) { 317 rc = -EINVAL; 318 goto out; 319 } 320 lock_sock(sk); 321 memcpy(crypto_info_aes_gcm_128->iv, 322 ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 323 TLS_CIPHER_AES_GCM_128_IV_SIZE); 324 memcpy(crypto_info_aes_gcm_128->rec_seq, ctx->rec_seq, 325 TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); 326 release_sock(sk); 327 if (copy_to_user(optval, 328 crypto_info_aes_gcm_128, 329 sizeof(*crypto_info_aes_gcm_128))) 330 rc = -EFAULT; 331 break; 332 } 333 default: 334 rc = -EINVAL; 335 } 336 337 out: 338 return rc; 339 } 340 341 static int do_tls_getsockopt(struct sock *sk, int optname, 342 char __user *optval, int __user *optlen) 343 { 344 int rc = 0; 345 346 switch (optname) { 347 case TLS_TX: 348 rc = do_tls_getsockopt_tx(sk, optval, optlen); 349 break; 350 default: 351 rc = -ENOPROTOOPT; 352 break; 353 } 354 return rc; 355 } 356 357 static int tls_getsockopt(struct sock *sk, int level, int optname, 358 char __user *optval, int __user *optlen) 359 { 360 struct tls_context *ctx = tls_get_ctx(sk); 361 362 if (level != SOL_TLS) 363 return ctx->getsockopt(sk, level, optname, optval, optlen); 364 365 return do_tls_getsockopt(sk, optname, optval, optlen); 366 } 367 368 static int do_tls_setsockopt_tx(struct sock *sk, char __user *optval, 369 unsigned int optlen) 370 { 371 struct tls_crypto_info *crypto_info; 372 struct tls_context *ctx = tls_get_ctx(sk); 373 int rc = 0; 374 int tx_conf; 375 376 if (!optval || (optlen < sizeof(*crypto_info))) { 377 rc = -EINVAL; 378 goto out; 379 } 380 381 crypto_info = &ctx->crypto_send; 382 /* Currently we don't support set crypto info more than one time */ 383 if (TLS_CRYPTO_INFO_READY(crypto_info)) { 384 rc = -EBUSY; 385 goto out; 386 } 387 388 rc = copy_from_user(crypto_info, optval, sizeof(*crypto_info)); 389 if (rc) { 390 rc = -EFAULT; 391 goto err_crypto_info; 392 } 393 394 /* check version */ 395 if (crypto_info->version != TLS_1_2_VERSION) { 396 rc = -ENOTSUPP; 397 goto err_crypto_info; 398 } 399 400 switch (crypto_info->cipher_type) { 401 case TLS_CIPHER_AES_GCM_128: { 402 if (optlen != sizeof(struct tls12_crypto_info_aes_gcm_128)) { 403 rc = -EINVAL; 404 goto err_crypto_info; 405 } 406 rc = copy_from_user(crypto_info + 1, optval + sizeof(*crypto_info), 407 optlen - sizeof(*crypto_info)); 408 if (rc) { 409 rc = -EFAULT; 410 goto err_crypto_info; 411 } 412 break; 413 } 414 default: 415 rc = -EINVAL; 416 goto err_crypto_info; 417 } 418 419 /* currently SW is default, we will have ethtool in future */ 420 rc = tls_set_sw_offload(sk, ctx); 421 tx_conf = TLS_SW_TX; 422 if (rc) 423 goto err_crypto_info; 424 425 ctx->tx_conf = tx_conf; 426 update_sk_prot(sk, ctx); 427 ctx->sk_write_space = sk->sk_write_space; 428 sk->sk_write_space = tls_write_space; 429 goto out; 430 431 err_crypto_info: 432 memset(crypto_info, 0, sizeof(*crypto_info)); 433 out: 434 return rc; 435 } 436 437 static int do_tls_setsockopt(struct sock *sk, int optname, 438 char __user *optval, unsigned int optlen) 439 { 440 int rc = 0; 441 442 switch (optname) { 443 case TLS_TX: 444 lock_sock(sk); 445 rc = do_tls_setsockopt_tx(sk, optval, optlen); 446 release_sock(sk); 447 break; 448 default: 449 rc = -ENOPROTOOPT; 450 break; 451 } 452 return rc; 453 } 454 455 static int tls_setsockopt(struct sock *sk, int level, int optname, 456 char __user *optval, unsigned int optlen) 457 { 458 struct tls_context *ctx = tls_get_ctx(sk); 459 460 if (level != SOL_TLS) 461 return ctx->setsockopt(sk, level, optname, optval, optlen); 462 463 return do_tls_setsockopt(sk, optname, optval, optlen); 464 } 465 466 static void build_protos(struct proto *prot, struct proto *base) 467 { 468 prot[TLS_BASE_TX] = *base; 469 prot[TLS_BASE_TX].setsockopt = tls_setsockopt; 470 prot[TLS_BASE_TX].getsockopt = tls_getsockopt; 471 prot[TLS_BASE_TX].close = tls_sk_proto_close; 472 473 prot[TLS_SW_TX] = prot[TLS_BASE_TX]; 474 prot[TLS_SW_TX].sendmsg = tls_sw_sendmsg; 475 prot[TLS_SW_TX].sendpage = tls_sw_sendpage; 476 } 477 478 static int tls_init(struct sock *sk) 479 { 480 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 481 struct inet_connection_sock *icsk = inet_csk(sk); 482 struct tls_context *ctx; 483 int rc = 0; 484 485 /* The TLS ulp is currently supported only for TCP sockets 486 * in ESTABLISHED state. 487 * Supporting sockets in LISTEN state will require us 488 * to modify the accept implementation to clone rather then 489 * share the ulp context. 490 */ 491 if (sk->sk_state != TCP_ESTABLISHED) 492 return -ENOTSUPP; 493 494 /* allocate tls context */ 495 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 496 if (!ctx) { 497 rc = -ENOMEM; 498 goto out; 499 } 500 icsk->icsk_ulp_data = ctx; 501 ctx->setsockopt = sk->sk_prot->setsockopt; 502 ctx->getsockopt = sk->sk_prot->getsockopt; 503 ctx->sk_proto_close = sk->sk_prot->close; 504 505 /* Build IPv6 TLS whenever the address of tcpv6_prot changes */ 506 if (ip_ver == TLSV6 && 507 unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) { 508 mutex_lock(&tcpv6_prot_mutex); 509 if (likely(sk->sk_prot != saved_tcpv6_prot)) { 510 build_protos(tls_prots[TLSV6], sk->sk_prot); 511 smp_store_release(&saved_tcpv6_prot, sk->sk_prot); 512 } 513 mutex_unlock(&tcpv6_prot_mutex); 514 } 515 516 ctx->tx_conf = TLS_BASE_TX; 517 update_sk_prot(sk, ctx); 518 out: 519 return rc; 520 } 521 522 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = { 523 .name = "tls", 524 .uid = TCP_ULP_TLS, 525 .user_visible = true, 526 .owner = THIS_MODULE, 527 .init = tls_init, 528 }; 529 530 static int __init tls_register(void) 531 { 532 build_protos(tls_prots[TLSV4], &tcp_prot); 533 534 tcp_register_ulp(&tcp_tls_ulp_ops); 535 536 return 0; 537 } 538 539 static void __exit tls_unregister(void) 540 { 541 tcp_unregister_ulp(&tcp_tls_ulp_ops); 542 } 543 544 module_init(tls_register); 545 module_exit(tls_unregister); 546