1 /* 2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 36 #include <net/tcp.h> 37 #include <net/inet_common.h> 38 #include <linux/highmem.h> 39 #include <linux/netdevice.h> 40 #include <linux/sched/signal.h> 41 42 #include <net/tls.h> 43 44 MODULE_AUTHOR("Mellanox Technologies"); 45 MODULE_DESCRIPTION("Transport Layer Security Support"); 46 MODULE_LICENSE("Dual BSD/GPL"); 47 48 enum { 49 TLS_BASE_TX, 50 TLS_SW_TX, 51 TLS_NUM_CONFIG, 52 }; 53 54 static struct proto tls_prots[TLS_NUM_CONFIG]; 55 56 static inline void update_sk_prot(struct sock *sk, struct tls_context *ctx) 57 { 58 sk->sk_prot = &tls_prots[ctx->tx_conf]; 59 } 60 61 int wait_on_pending_writer(struct sock *sk, long *timeo) 62 { 63 int rc = 0; 64 DEFINE_WAIT_FUNC(wait, woken_wake_function); 65 66 add_wait_queue(sk_sleep(sk), &wait); 67 while (1) { 68 if (!*timeo) { 69 rc = -EAGAIN; 70 break; 71 } 72 73 if (signal_pending(current)) { 74 rc = sock_intr_errno(*timeo); 75 break; 76 } 77 78 if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait)) 79 break; 80 } 81 remove_wait_queue(sk_sleep(sk), &wait); 82 return rc; 83 } 84 85 int tls_push_sg(struct sock *sk, 86 struct tls_context *ctx, 87 struct scatterlist *sg, 88 u16 first_offset, 89 int flags) 90 { 91 int sendpage_flags = flags | MSG_SENDPAGE_NOTLAST; 92 int ret = 0; 93 struct page *p; 94 size_t size; 95 int offset = first_offset; 96 97 size = sg->length - offset; 98 offset += sg->offset; 99 100 while (1) { 101 if (sg_is_last(sg)) 102 sendpage_flags = flags; 103 104 /* is sending application-limited? */ 105 tcp_rate_check_app_limited(sk); 106 p = sg_page(sg); 107 retry: 108 ret = do_tcp_sendpages(sk, p, offset, size, sendpage_flags); 109 110 if (ret != size) { 111 if (ret > 0) { 112 offset += ret; 113 size -= ret; 114 goto retry; 115 } 116 117 offset -= sg->offset; 118 ctx->partially_sent_offset = offset; 119 ctx->partially_sent_record = (void *)sg; 120 return ret; 121 } 122 123 put_page(p); 124 sk_mem_uncharge(sk, sg->length); 125 sg = sg_next(sg); 126 if (!sg) 127 break; 128 129 offset = sg->offset; 130 size = sg->length; 131 } 132 133 clear_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags); 134 135 return 0; 136 } 137 138 static int tls_handle_open_record(struct sock *sk, int flags) 139 { 140 struct tls_context *ctx = tls_get_ctx(sk); 141 142 if (tls_is_pending_open_record(ctx)) 143 return ctx->push_pending_record(sk, flags); 144 145 return 0; 146 } 147 148 int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg, 149 unsigned char *record_type) 150 { 151 struct cmsghdr *cmsg; 152 int rc = -EINVAL; 153 154 for_each_cmsghdr(cmsg, msg) { 155 if (!CMSG_OK(msg, cmsg)) 156 return -EINVAL; 157 if (cmsg->cmsg_level != SOL_TLS) 158 continue; 159 160 switch (cmsg->cmsg_type) { 161 case TLS_SET_RECORD_TYPE: 162 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) 163 return -EINVAL; 164 165 if (msg->msg_flags & MSG_MORE) 166 return -EINVAL; 167 168 rc = tls_handle_open_record(sk, msg->msg_flags); 169 if (rc) 170 return rc; 171 172 *record_type = *(unsigned char *)CMSG_DATA(cmsg); 173 rc = 0; 174 break; 175 default: 176 return -EINVAL; 177 } 178 } 179 180 return rc; 181 } 182 183 int tls_push_pending_closed_record(struct sock *sk, struct tls_context *ctx, 184 int flags, long *timeo) 185 { 186 struct scatterlist *sg; 187 u16 offset; 188 189 if (!tls_is_partially_sent_record(ctx)) 190 return ctx->push_pending_record(sk, flags); 191 192 sg = ctx->partially_sent_record; 193 offset = ctx->partially_sent_offset; 194 195 ctx->partially_sent_record = NULL; 196 return tls_push_sg(sk, ctx, sg, offset, flags); 197 } 198 199 static void tls_write_space(struct sock *sk) 200 { 201 struct tls_context *ctx = tls_get_ctx(sk); 202 203 if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) { 204 gfp_t sk_allocation = sk->sk_allocation; 205 int rc; 206 long timeo = 0; 207 208 sk->sk_allocation = GFP_ATOMIC; 209 rc = tls_push_pending_closed_record(sk, ctx, 210 MSG_DONTWAIT | 211 MSG_NOSIGNAL, 212 &timeo); 213 sk->sk_allocation = sk_allocation; 214 215 if (rc < 0) 216 return; 217 } 218 219 ctx->sk_write_space(sk); 220 } 221 222 static void tls_sk_proto_close(struct sock *sk, long timeout) 223 { 224 struct tls_context *ctx = tls_get_ctx(sk); 225 long timeo = sock_sndtimeo(sk, 0); 226 void (*sk_proto_close)(struct sock *sk, long timeout); 227 228 lock_sock(sk); 229 sk_proto_close = ctx->sk_proto_close; 230 231 if (ctx->tx_conf == TLS_BASE_TX) { 232 kfree(ctx); 233 goto skip_tx_cleanup; 234 } 235 236 if (!tls_complete_pending_work(sk, ctx, 0, &timeo)) 237 tls_handle_open_record(sk, 0); 238 239 if (ctx->partially_sent_record) { 240 struct scatterlist *sg = ctx->partially_sent_record; 241 242 while (1) { 243 put_page(sg_page(sg)); 244 sk_mem_uncharge(sk, sg->length); 245 246 if (sg_is_last(sg)) 247 break; 248 sg++; 249 } 250 } 251 252 kfree(ctx->rec_seq); 253 kfree(ctx->iv); 254 255 if (ctx->tx_conf == TLS_SW_TX) 256 tls_sw_free_tx_resources(sk); 257 258 skip_tx_cleanup: 259 release_sock(sk); 260 sk_proto_close(sk, timeout); 261 } 262 263 static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval, 264 int __user *optlen) 265 { 266 int rc = 0; 267 struct tls_context *ctx = tls_get_ctx(sk); 268 struct tls_crypto_info *crypto_info; 269 int len; 270 271 if (get_user(len, optlen)) 272 return -EFAULT; 273 274 if (!optval || (len < sizeof(*crypto_info))) { 275 rc = -EINVAL; 276 goto out; 277 } 278 279 if (!ctx) { 280 rc = -EBUSY; 281 goto out; 282 } 283 284 /* get user crypto info */ 285 crypto_info = &ctx->crypto_send; 286 287 if (!TLS_CRYPTO_INFO_READY(crypto_info)) { 288 rc = -EBUSY; 289 goto out; 290 } 291 292 if (len == sizeof(*crypto_info)) { 293 if (copy_to_user(optval, crypto_info, sizeof(*crypto_info))) 294 rc = -EFAULT; 295 goto out; 296 } 297 298 switch (crypto_info->cipher_type) { 299 case TLS_CIPHER_AES_GCM_128: { 300 struct tls12_crypto_info_aes_gcm_128 * 301 crypto_info_aes_gcm_128 = 302 container_of(crypto_info, 303 struct tls12_crypto_info_aes_gcm_128, 304 info); 305 306 if (len != sizeof(*crypto_info_aes_gcm_128)) { 307 rc = -EINVAL; 308 goto out; 309 } 310 lock_sock(sk); 311 memcpy(crypto_info_aes_gcm_128->iv, 312 ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, 313 TLS_CIPHER_AES_GCM_128_IV_SIZE); 314 memcpy(crypto_info_aes_gcm_128->rec_seq, ctx->rec_seq, 315 TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); 316 release_sock(sk); 317 if (copy_to_user(optval, 318 crypto_info_aes_gcm_128, 319 sizeof(*crypto_info_aes_gcm_128))) 320 rc = -EFAULT; 321 break; 322 } 323 default: 324 rc = -EINVAL; 325 } 326 327 out: 328 return rc; 329 } 330 331 static int do_tls_getsockopt(struct sock *sk, int optname, 332 char __user *optval, int __user *optlen) 333 { 334 int rc = 0; 335 336 switch (optname) { 337 case TLS_TX: 338 rc = do_tls_getsockopt_tx(sk, optval, optlen); 339 break; 340 default: 341 rc = -ENOPROTOOPT; 342 break; 343 } 344 return rc; 345 } 346 347 static int tls_getsockopt(struct sock *sk, int level, int optname, 348 char __user *optval, int __user *optlen) 349 { 350 struct tls_context *ctx = tls_get_ctx(sk); 351 352 if (level != SOL_TLS) 353 return ctx->getsockopt(sk, level, optname, optval, optlen); 354 355 return do_tls_getsockopt(sk, optname, optval, optlen); 356 } 357 358 static int do_tls_setsockopt_tx(struct sock *sk, char __user *optval, 359 unsigned int optlen) 360 { 361 struct tls_crypto_info *crypto_info; 362 struct tls_context *ctx = tls_get_ctx(sk); 363 int rc = 0; 364 int tx_conf; 365 366 if (!optval || (optlen < sizeof(*crypto_info))) { 367 rc = -EINVAL; 368 goto out; 369 } 370 371 crypto_info = &ctx->crypto_send; 372 /* Currently we don't support set crypto info more than one time */ 373 if (TLS_CRYPTO_INFO_READY(crypto_info)) { 374 rc = -EBUSY; 375 goto out; 376 } 377 378 rc = copy_from_user(crypto_info, optval, sizeof(*crypto_info)); 379 if (rc) { 380 rc = -EFAULT; 381 goto err_crypto_info; 382 } 383 384 /* check version */ 385 if (crypto_info->version != TLS_1_2_VERSION) { 386 rc = -ENOTSUPP; 387 goto err_crypto_info; 388 } 389 390 switch (crypto_info->cipher_type) { 391 case TLS_CIPHER_AES_GCM_128: { 392 if (optlen != sizeof(struct tls12_crypto_info_aes_gcm_128)) { 393 rc = -EINVAL; 394 goto err_crypto_info; 395 } 396 rc = copy_from_user(crypto_info + 1, optval + sizeof(*crypto_info), 397 optlen - sizeof(*crypto_info)); 398 if (rc) { 399 rc = -EFAULT; 400 goto err_crypto_info; 401 } 402 break; 403 } 404 default: 405 rc = -EINVAL; 406 goto err_crypto_info; 407 } 408 409 /* currently SW is default, we will have ethtool in future */ 410 rc = tls_set_sw_offload(sk, ctx); 411 tx_conf = TLS_SW_TX; 412 if (rc) 413 goto err_crypto_info; 414 415 ctx->tx_conf = tx_conf; 416 update_sk_prot(sk, ctx); 417 ctx->sk_write_space = sk->sk_write_space; 418 sk->sk_write_space = tls_write_space; 419 goto out; 420 421 err_crypto_info: 422 memset(crypto_info, 0, sizeof(*crypto_info)); 423 out: 424 return rc; 425 } 426 427 static int do_tls_setsockopt(struct sock *sk, int optname, 428 char __user *optval, unsigned int optlen) 429 { 430 int rc = 0; 431 432 switch (optname) { 433 case TLS_TX: 434 lock_sock(sk); 435 rc = do_tls_setsockopt_tx(sk, optval, optlen); 436 release_sock(sk); 437 break; 438 default: 439 rc = -ENOPROTOOPT; 440 break; 441 } 442 return rc; 443 } 444 445 static int tls_setsockopt(struct sock *sk, int level, int optname, 446 char __user *optval, unsigned int optlen) 447 { 448 struct tls_context *ctx = tls_get_ctx(sk); 449 450 if (level != SOL_TLS) 451 return ctx->setsockopt(sk, level, optname, optval, optlen); 452 453 return do_tls_setsockopt(sk, optname, optval, optlen); 454 } 455 456 static int tls_init(struct sock *sk) 457 { 458 struct inet_connection_sock *icsk = inet_csk(sk); 459 struct tls_context *ctx; 460 int rc = 0; 461 462 /* The TLS ulp is currently supported only for TCP sockets 463 * in ESTABLISHED state. 464 * Supporting sockets in LISTEN state will require us 465 * to modify the accept implementation to clone rather then 466 * share the ulp context. 467 */ 468 if (sk->sk_state != TCP_ESTABLISHED) 469 return -ENOTSUPP; 470 471 /* allocate tls context */ 472 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 473 if (!ctx) { 474 rc = -ENOMEM; 475 goto out; 476 } 477 icsk->icsk_ulp_data = ctx; 478 ctx->setsockopt = sk->sk_prot->setsockopt; 479 ctx->getsockopt = sk->sk_prot->getsockopt; 480 ctx->sk_proto_close = sk->sk_prot->close; 481 482 ctx->tx_conf = TLS_BASE_TX; 483 update_sk_prot(sk, ctx); 484 out: 485 return rc; 486 } 487 488 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = { 489 .name = "tls", 490 .uid = TCP_ULP_TLS, 491 .user_visible = true, 492 .owner = THIS_MODULE, 493 .init = tls_init, 494 }; 495 496 static void build_protos(struct proto *prot, struct proto *base) 497 { 498 prot[TLS_BASE_TX] = *base; 499 prot[TLS_BASE_TX].setsockopt = tls_setsockopt; 500 prot[TLS_BASE_TX].getsockopt = tls_getsockopt; 501 prot[TLS_BASE_TX].close = tls_sk_proto_close; 502 503 prot[TLS_SW_TX] = prot[TLS_BASE_TX]; 504 prot[TLS_SW_TX].sendmsg = tls_sw_sendmsg; 505 prot[TLS_SW_TX].sendpage = tls_sw_sendpage; 506 } 507 508 static int __init tls_register(void) 509 { 510 build_protos(tls_prots, &tcp_prot); 511 512 tcp_register_ulp(&tcp_tls_ulp_ops); 513 514 return 0; 515 } 516 517 static void __exit tls_unregister(void) 518 { 519 tcp_unregister_ulp(&tcp_tls_ulp_ops); 520 } 521 522 module_init(tls_register); 523 module_exit(tls_unregister); 524