1 // SPDX-License-Identifier: GPL-2.0 2 #include <net/tcp.h> 3 #include <net/strparser.h> 4 #include <net/xfrm.h> 5 #include <net/esp.h> 6 #include <net/espintcp.h> 7 #include <linux/skmsg.h> 8 #include <net/inet_common.h> 9 #include <trace/events/sock.h> 10 #if IS_ENABLED(CONFIG_IPV6) 11 #include <net/ipv6_stubs.h> 12 #endif 13 14 static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb, 15 struct sock *sk) 16 { 17 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf || 18 !sk_rmem_schedule(sk, skb, skb->truesize)) { 19 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); 20 kfree_skb(skb); 21 return; 22 } 23 24 skb_set_owner_r(skb, sk); 25 26 memset(skb->cb, 0, sizeof(skb->cb)); 27 skb_queue_tail(&ctx->ike_queue, skb); 28 ctx->saved_data_ready(sk); 29 } 30 31 static void handle_esp(struct sk_buff *skb, struct sock *sk) 32 { 33 struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb; 34 35 skb_reset_transport_header(skb); 36 37 /* restore IP CB, we need at least IP6CB->nhoff */ 38 memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header)); 39 40 rcu_read_lock(); 41 skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif); 42 local_bh_disable(); 43 #if IS_ENABLED(CONFIG_IPV6) 44 if (sk->sk_family == AF_INET6) 45 ipv6_stub->xfrm6_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); 46 else 47 #endif 48 xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); 49 local_bh_enable(); 50 rcu_read_unlock(); 51 } 52 53 static void espintcp_rcv(struct strparser *strp, struct sk_buff *skb) 54 { 55 struct espintcp_ctx *ctx = container_of(strp, struct espintcp_ctx, 56 strp); 57 struct strp_msg *rxm = strp_msg(skb); 58 int len = rxm->full_len - 2; 59 u32 nonesp_marker; 60 int err; 61 62 /* keepalive packet? */ 63 if (unlikely(len == 1)) { 64 u8 data; 65 66 err = skb_copy_bits(skb, rxm->offset + 2, &data, 1); 67 if (err < 0) { 68 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 69 kfree_skb(skb); 70 return; 71 } 72 73 if (data == 0xff) { 74 kfree_skb(skb); 75 return; 76 } 77 } 78 79 /* drop other short messages */ 80 if (unlikely(len <= sizeof(nonesp_marker))) { 81 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 82 kfree_skb(skb); 83 return; 84 } 85 86 err = skb_copy_bits(skb, rxm->offset + 2, &nonesp_marker, 87 sizeof(nonesp_marker)); 88 if (err < 0) { 89 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 90 kfree_skb(skb); 91 return; 92 } 93 94 /* remove header, leave non-ESP marker/SPI */ 95 if (!pskb_pull(skb, rxm->offset + 2)) { 96 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR); 97 kfree_skb(skb); 98 return; 99 } 100 101 if (pskb_trim(skb, rxm->full_len - 2) != 0) { 102 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR); 103 kfree_skb(skb); 104 return; 105 } 106 107 if (nonesp_marker == 0) 108 handle_nonesp(ctx, skb, strp->sk); 109 else 110 handle_esp(skb, strp->sk); 111 } 112 113 static int espintcp_parse(struct strparser *strp, struct sk_buff *skb) 114 { 115 struct strp_msg *rxm = strp_msg(skb); 116 __be16 blen; 117 u16 len; 118 int err; 119 120 if (skb->len < rxm->offset + 2) 121 return 0; 122 123 err = skb_copy_bits(skb, rxm->offset, &blen, sizeof(blen)); 124 if (err < 0) 125 return err; 126 127 len = be16_to_cpu(blen); 128 if (len < 2) 129 return -EINVAL; 130 131 return len; 132 } 133 134 static int espintcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 135 int flags, int *addr_len) 136 { 137 struct espintcp_ctx *ctx = espintcp_getctx(sk); 138 struct sk_buff *skb; 139 int err = 0; 140 int copied; 141 int off = 0; 142 143 skb = __skb_recv_datagram(sk, &ctx->ike_queue, flags, &off, &err); 144 if (!skb) { 145 if (err == -EAGAIN && sk->sk_shutdown & RCV_SHUTDOWN) 146 return 0; 147 return err; 148 } 149 150 copied = len; 151 if (copied > skb->len) 152 copied = skb->len; 153 else if (copied < skb->len) 154 msg->msg_flags |= MSG_TRUNC; 155 156 err = skb_copy_datagram_msg(skb, 0, msg, copied); 157 if (unlikely(err)) { 158 kfree_skb(skb); 159 return err; 160 } 161 162 if (flags & MSG_TRUNC) 163 copied = skb->len; 164 kfree_skb(skb); 165 return copied; 166 } 167 168 int espintcp_queue_out(struct sock *sk, struct sk_buff *skb) 169 { 170 struct espintcp_ctx *ctx = espintcp_getctx(sk); 171 172 if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog)) 173 return -ENOBUFS; 174 175 __skb_queue_tail(&ctx->out_queue, skb); 176 177 return 0; 178 } 179 EXPORT_SYMBOL_GPL(espintcp_queue_out); 180 181 /* espintcp length field is 2B and length includes the length field's size */ 182 #define MAX_ESPINTCP_MSG (((1 << 16) - 1) - 2) 183 184 static int espintcp_sendskb_locked(struct sock *sk, struct espintcp_msg *emsg, 185 int flags) 186 { 187 do { 188 int ret; 189 190 ret = skb_send_sock_locked(sk, emsg->skb, 191 emsg->offset, emsg->len); 192 if (ret < 0) 193 return ret; 194 195 emsg->len -= ret; 196 emsg->offset += ret; 197 } while (emsg->len > 0); 198 199 kfree_skb(emsg->skb); 200 memset(emsg, 0, sizeof(*emsg)); 201 202 return 0; 203 } 204 205 static int espintcp_sendskmsg_locked(struct sock *sk, 206 struct espintcp_msg *emsg, int flags) 207 { 208 struct msghdr msghdr = { .msg_flags = flags | MSG_SPLICE_PAGES, }; 209 struct sk_msg *skmsg = &emsg->skmsg; 210 struct scatterlist *sg; 211 int done = 0; 212 int ret; 213 214 msghdr.msg_flags |= MSG_SENDPAGE_NOTLAST; 215 sg = &skmsg->sg.data[skmsg->sg.start]; 216 do { 217 struct bio_vec bvec; 218 size_t size = sg->length - emsg->offset; 219 int offset = sg->offset + emsg->offset; 220 struct page *p; 221 222 emsg->offset = 0; 223 224 if (sg_is_last(sg)) 225 msghdr.msg_flags &= ~MSG_SENDPAGE_NOTLAST; 226 227 p = sg_page(sg); 228 retry: 229 bvec_set_page(&bvec, p, size, offset); 230 iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, size); 231 ret = tcp_sendmsg_locked(sk, &msghdr, size); 232 if (ret < 0) { 233 emsg->offset = offset - sg->offset; 234 skmsg->sg.start += done; 235 return ret; 236 } 237 238 if (ret != size) { 239 offset += ret; 240 size -= ret; 241 goto retry; 242 } 243 244 done++; 245 put_page(p); 246 sk_mem_uncharge(sk, sg->length); 247 sg = sg_next(sg); 248 } while (sg); 249 250 memset(emsg, 0, sizeof(*emsg)); 251 252 return 0; 253 } 254 255 static int espintcp_push_msgs(struct sock *sk, int flags) 256 { 257 struct espintcp_ctx *ctx = espintcp_getctx(sk); 258 struct espintcp_msg *emsg = &ctx->partial; 259 int err; 260 261 if (!emsg->len) 262 return 0; 263 264 if (ctx->tx_running) 265 return -EAGAIN; 266 ctx->tx_running = 1; 267 268 if (emsg->skb) 269 err = espintcp_sendskb_locked(sk, emsg, flags); 270 else 271 err = espintcp_sendskmsg_locked(sk, emsg, flags); 272 if (err == -EAGAIN) { 273 ctx->tx_running = 0; 274 return flags & MSG_DONTWAIT ? -EAGAIN : 0; 275 } 276 if (!err) 277 memset(emsg, 0, sizeof(*emsg)); 278 279 ctx->tx_running = 0; 280 281 return err; 282 } 283 284 int espintcp_push_skb(struct sock *sk, struct sk_buff *skb) 285 { 286 struct espintcp_ctx *ctx = espintcp_getctx(sk); 287 struct espintcp_msg *emsg = &ctx->partial; 288 unsigned int len; 289 int offset; 290 291 if (sk->sk_state != TCP_ESTABLISHED) { 292 kfree_skb(skb); 293 return -ECONNRESET; 294 } 295 296 offset = skb_transport_offset(skb); 297 len = skb->len - offset; 298 299 espintcp_push_msgs(sk, 0); 300 301 if (emsg->len) { 302 kfree_skb(skb); 303 return -ENOBUFS; 304 } 305 306 skb_set_owner_w(skb, sk); 307 308 emsg->offset = offset; 309 emsg->len = len; 310 emsg->skb = skb; 311 312 espintcp_push_msgs(sk, 0); 313 314 return 0; 315 } 316 EXPORT_SYMBOL_GPL(espintcp_push_skb); 317 318 static int espintcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 319 { 320 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 321 struct espintcp_ctx *ctx = espintcp_getctx(sk); 322 struct espintcp_msg *emsg = &ctx->partial; 323 struct iov_iter pfx_iter; 324 struct kvec pfx_iov = {}; 325 size_t msglen = size + 2; 326 char buf[2] = {0}; 327 int err, end; 328 329 if (msg->msg_flags & ~MSG_DONTWAIT) 330 return -EOPNOTSUPP; 331 332 if (size > MAX_ESPINTCP_MSG) 333 return -EMSGSIZE; 334 335 if (msg->msg_controllen) 336 return -EOPNOTSUPP; 337 338 lock_sock(sk); 339 340 err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT); 341 if (err < 0) { 342 if (err != -EAGAIN || !(msg->msg_flags & MSG_DONTWAIT)) 343 err = -ENOBUFS; 344 goto unlock; 345 } 346 347 sk_msg_init(&emsg->skmsg); 348 while (1) { 349 /* only -ENOMEM is possible since we don't coalesce */ 350 err = sk_msg_alloc(sk, &emsg->skmsg, msglen, 0); 351 if (!err) 352 break; 353 354 err = sk_stream_wait_memory(sk, &timeo); 355 if (err) 356 goto fail; 357 } 358 359 *((__be16 *)buf) = cpu_to_be16(msglen); 360 pfx_iov.iov_base = buf; 361 pfx_iov.iov_len = sizeof(buf); 362 iov_iter_kvec(&pfx_iter, ITER_SOURCE, &pfx_iov, 1, pfx_iov.iov_len); 363 364 err = sk_msg_memcopy_from_iter(sk, &pfx_iter, &emsg->skmsg, 365 pfx_iov.iov_len); 366 if (err < 0) 367 goto fail; 368 369 err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, &emsg->skmsg, size); 370 if (err < 0) 371 goto fail; 372 373 end = emsg->skmsg.sg.end; 374 emsg->len = size; 375 sk_msg_iter_var_prev(end); 376 sg_mark_end(sk_msg_elem(&emsg->skmsg, end)); 377 378 tcp_rate_check_app_limited(sk); 379 380 err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT); 381 /* this message could be partially sent, keep it */ 382 383 release_sock(sk); 384 385 return size; 386 387 fail: 388 sk_msg_free(sk, &emsg->skmsg); 389 memset(emsg, 0, sizeof(*emsg)); 390 unlock: 391 release_sock(sk); 392 return err; 393 } 394 395 static struct proto espintcp_prot __ro_after_init; 396 static struct proto_ops espintcp_ops __ro_after_init; 397 static struct proto espintcp6_prot; 398 static struct proto_ops espintcp6_ops; 399 static DEFINE_MUTEX(tcpv6_prot_mutex); 400 401 static void espintcp_data_ready(struct sock *sk) 402 { 403 struct espintcp_ctx *ctx = espintcp_getctx(sk); 404 405 trace_sk_data_ready(sk); 406 407 strp_data_ready(&ctx->strp); 408 } 409 410 static void espintcp_tx_work(struct work_struct *work) 411 { 412 struct espintcp_ctx *ctx = container_of(work, 413 struct espintcp_ctx, work); 414 struct sock *sk = ctx->strp.sk; 415 416 lock_sock(sk); 417 if (!ctx->tx_running) 418 espintcp_push_msgs(sk, 0); 419 release_sock(sk); 420 } 421 422 static void espintcp_write_space(struct sock *sk) 423 { 424 struct espintcp_ctx *ctx = espintcp_getctx(sk); 425 426 schedule_work(&ctx->work); 427 ctx->saved_write_space(sk); 428 } 429 430 static void espintcp_destruct(struct sock *sk) 431 { 432 struct espintcp_ctx *ctx = espintcp_getctx(sk); 433 434 ctx->saved_destruct(sk); 435 kfree(ctx); 436 } 437 438 bool tcp_is_ulp_esp(struct sock *sk) 439 { 440 return sk->sk_prot == &espintcp_prot || sk->sk_prot == &espintcp6_prot; 441 } 442 EXPORT_SYMBOL_GPL(tcp_is_ulp_esp); 443 444 static void build_protos(struct proto *espintcp_prot, 445 struct proto_ops *espintcp_ops, 446 const struct proto *orig_prot, 447 const struct proto_ops *orig_ops); 448 static int espintcp_init_sk(struct sock *sk) 449 { 450 struct inet_connection_sock *icsk = inet_csk(sk); 451 struct strp_callbacks cb = { 452 .rcv_msg = espintcp_rcv, 453 .parse_msg = espintcp_parse, 454 }; 455 struct espintcp_ctx *ctx; 456 int err; 457 458 /* sockmap is not compatible with espintcp */ 459 if (sk->sk_user_data) 460 return -EBUSY; 461 462 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 463 if (!ctx) 464 return -ENOMEM; 465 466 err = strp_init(&ctx->strp, sk, &cb); 467 if (err) 468 goto free; 469 470 __sk_dst_reset(sk); 471 472 strp_check_rcv(&ctx->strp); 473 skb_queue_head_init(&ctx->ike_queue); 474 skb_queue_head_init(&ctx->out_queue); 475 476 if (sk->sk_family == AF_INET) { 477 sk->sk_prot = &espintcp_prot; 478 sk->sk_socket->ops = &espintcp_ops; 479 } else { 480 mutex_lock(&tcpv6_prot_mutex); 481 if (!espintcp6_prot.recvmsg) 482 build_protos(&espintcp6_prot, &espintcp6_ops, sk->sk_prot, sk->sk_socket->ops); 483 mutex_unlock(&tcpv6_prot_mutex); 484 485 sk->sk_prot = &espintcp6_prot; 486 sk->sk_socket->ops = &espintcp6_ops; 487 } 488 ctx->saved_data_ready = sk->sk_data_ready; 489 ctx->saved_write_space = sk->sk_write_space; 490 ctx->saved_destruct = sk->sk_destruct; 491 sk->sk_data_ready = espintcp_data_ready; 492 sk->sk_write_space = espintcp_write_space; 493 sk->sk_destruct = espintcp_destruct; 494 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 495 INIT_WORK(&ctx->work, espintcp_tx_work); 496 497 /* avoid using task_frag */ 498 sk->sk_allocation = GFP_ATOMIC; 499 sk->sk_use_task_frag = false; 500 501 return 0; 502 503 free: 504 kfree(ctx); 505 return err; 506 } 507 508 static void espintcp_release(struct sock *sk) 509 { 510 struct espintcp_ctx *ctx = espintcp_getctx(sk); 511 struct sk_buff_head queue; 512 struct sk_buff *skb; 513 514 __skb_queue_head_init(&queue); 515 skb_queue_splice_init(&ctx->out_queue, &queue); 516 517 while ((skb = __skb_dequeue(&queue))) 518 espintcp_push_skb(sk, skb); 519 520 tcp_release_cb(sk); 521 } 522 523 static void espintcp_close(struct sock *sk, long timeout) 524 { 525 struct espintcp_ctx *ctx = espintcp_getctx(sk); 526 struct espintcp_msg *emsg = &ctx->partial; 527 528 strp_stop(&ctx->strp); 529 530 sk->sk_prot = &tcp_prot; 531 barrier(); 532 533 cancel_work_sync(&ctx->work); 534 strp_done(&ctx->strp); 535 536 skb_queue_purge(&ctx->out_queue); 537 skb_queue_purge(&ctx->ike_queue); 538 539 if (emsg->len) { 540 if (emsg->skb) 541 kfree_skb(emsg->skb); 542 else 543 sk_msg_free(sk, &emsg->skmsg); 544 } 545 546 tcp_close(sk, timeout); 547 } 548 549 static __poll_t espintcp_poll(struct file *file, struct socket *sock, 550 poll_table *wait) 551 { 552 __poll_t mask = datagram_poll(file, sock, wait); 553 struct sock *sk = sock->sk; 554 struct espintcp_ctx *ctx = espintcp_getctx(sk); 555 556 if (!skb_queue_empty(&ctx->ike_queue)) 557 mask |= EPOLLIN | EPOLLRDNORM; 558 559 return mask; 560 } 561 562 static void build_protos(struct proto *espintcp_prot, 563 struct proto_ops *espintcp_ops, 564 const struct proto *orig_prot, 565 const struct proto_ops *orig_ops) 566 { 567 memcpy(espintcp_prot, orig_prot, sizeof(struct proto)); 568 memcpy(espintcp_ops, orig_ops, sizeof(struct proto_ops)); 569 espintcp_prot->sendmsg = espintcp_sendmsg; 570 espintcp_prot->recvmsg = espintcp_recvmsg; 571 espintcp_prot->close = espintcp_close; 572 espintcp_prot->release_cb = espintcp_release; 573 espintcp_ops->poll = espintcp_poll; 574 } 575 576 static struct tcp_ulp_ops espintcp_ulp __read_mostly = { 577 .name = "espintcp", 578 .owner = THIS_MODULE, 579 .init = espintcp_init_sk, 580 }; 581 582 void __init espintcp_init(void) 583 { 584 build_protos(&espintcp_prot, &espintcp_ops, &tcp_prot, &inet_stream_ops); 585 586 tcp_register_ulp(&espintcp_ulp); 587 } 588