1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ 3 4 #include <linux/skmsg.h> 5 #include <linux/filter.h> 6 #include <linux/bpf.h> 7 #include <linux/init.h> 8 #include <linux/wait.h> 9 #include <linux/util_macros.h> 10 11 #include <net/inet_common.h> 12 #include <net/tls.h> 13 14 void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) 15 { 16 struct tcp_sock *tcp; 17 int copied; 18 19 if (!skb || !skb->len || !sk_is_tcp(sk)) 20 return; 21 22 if (skb_bpf_strparser(skb)) 23 return; 24 25 tcp = tcp_sk(sk); 26 copied = tcp->copied_seq + skb->len; 27 WRITE_ONCE(tcp->copied_seq, copied); 28 tcp_rcv_space_adjust(sk); 29 __tcp_cleanup_rbuf(sk, skb->len); 30 } 31 32 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, 33 struct sk_msg *msg, u32 apply_bytes, int flags) 34 { 35 bool apply = apply_bytes; 36 struct scatterlist *sge; 37 u32 size, copied = 0; 38 struct sk_msg *tmp; 39 int i, ret = 0; 40 41 tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL); 42 if (unlikely(!tmp)) 43 return -ENOMEM; 44 45 lock_sock(sk); 46 tmp->sg.start = msg->sg.start; 47 i = msg->sg.start; 48 do { 49 sge = sk_msg_elem(msg, i); 50 size = (apply && apply_bytes < sge->length) ? 51 apply_bytes : sge->length; 52 if (!sk_wmem_schedule(sk, size)) { 53 if (!copied) 54 ret = -ENOMEM; 55 break; 56 } 57 58 sk_mem_charge(sk, size); 59 sk_msg_xfer(tmp, msg, i, size); 60 copied += size; 61 if (sge->length) 62 get_page(sk_msg_page(tmp, i)); 63 sk_msg_iter_var_next(i); 64 tmp->sg.end = i; 65 if (apply) { 66 apply_bytes -= size; 67 if (!apply_bytes) { 68 if (sge->length) 69 sk_msg_iter_var_prev(i); 70 break; 71 } 72 } 73 } while (i != msg->sg.end); 74 75 if (!ret) { 76 msg->sg.start = i; 77 sk_psock_queue_msg(psock, tmp); 78 sk_psock_data_ready(sk, psock); 79 } else { 80 sk_msg_free(sk, tmp); 81 kfree(tmp); 82 } 83 84 release_sock(sk); 85 return ret; 86 } 87 88 static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, 89 int flags, bool uncharge) 90 { 91 bool apply = apply_bytes; 92 struct scatterlist *sge; 93 struct page *page; 94 int size, ret = 0; 95 u32 off; 96 97 while (1) { 98 bool has_tx_ulp; 99 100 sge = sk_msg_elem(msg, msg->sg.start); 101 size = (apply && apply_bytes < sge->length) ? 102 apply_bytes : sge->length; 103 off = sge->offset; 104 page = sg_page(sge); 105 106 tcp_rate_check_app_limited(sk); 107 retry: 108 has_tx_ulp = tls_sw_has_ctx_tx(sk); 109 if (has_tx_ulp) { 110 flags |= MSG_SENDPAGE_NOPOLICY; 111 ret = kernel_sendpage_locked(sk, 112 page, off, size, flags); 113 } else { 114 ret = do_tcp_sendpages(sk, page, off, size, flags); 115 } 116 117 if (ret <= 0) 118 return ret; 119 if (apply) 120 apply_bytes -= ret; 121 msg->sg.size -= ret; 122 sge->offset += ret; 123 sge->length -= ret; 124 if (uncharge) 125 sk_mem_uncharge(sk, ret); 126 if (ret != size) { 127 size -= ret; 128 off += ret; 129 goto retry; 130 } 131 if (!sge->length) { 132 put_page(page); 133 sk_msg_iter_next(msg, start); 134 sg_init_table(sge, 1); 135 if (msg->sg.start == msg->sg.end) 136 break; 137 } 138 if (apply && !apply_bytes) 139 break; 140 } 141 142 return 0; 143 } 144 145 static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg, 146 u32 apply_bytes, int flags, bool uncharge) 147 { 148 int ret; 149 150 lock_sock(sk); 151 ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge); 152 release_sock(sk); 153 return ret; 154 } 155 156 int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 157 struct sk_msg *msg, u32 bytes, int flags) 158 { 159 struct sk_psock *psock = sk_psock_get(sk); 160 int ret; 161 162 if (unlikely(!psock)) 163 return -EPIPE; 164 165 ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) : 166 tcp_bpf_push_locked(sk, msg, bytes, flags, false); 167 sk_psock_put(sk, psock); 168 return ret; 169 } 170 EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir); 171 172 #ifdef CONFIG_BPF_SYSCALL 173 static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock, 174 long timeo) 175 { 176 DEFINE_WAIT_FUNC(wait, woken_wake_function); 177 int ret = 0; 178 179 if (sk->sk_shutdown & RCV_SHUTDOWN) 180 return 1; 181 182 if (!timeo) 183 return ret; 184 185 add_wait_queue(sk_sleep(sk), &wait); 186 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 187 ret = sk_wait_event(sk, &timeo, 188 !list_empty(&psock->ingress_msg) || 189 !skb_queue_empty_lockless(&sk->sk_receive_queue), &wait); 190 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 191 remove_wait_queue(sk_sleep(sk), &wait); 192 return ret; 193 } 194 195 static bool is_next_msg_fin(struct sk_psock *psock) 196 { 197 struct scatterlist *sge; 198 struct sk_msg *msg_rx; 199 int i; 200 201 msg_rx = sk_psock_peek_msg(psock); 202 i = msg_rx->sg.start; 203 sge = sk_msg_elem(msg_rx, i); 204 if (!sge->length) { 205 struct sk_buff *skb = msg_rx->skb; 206 207 if (skb && TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 208 return true; 209 } 210 return false; 211 } 212 213 static int tcp_bpf_recvmsg_parser(struct sock *sk, 214 struct msghdr *msg, 215 size_t len, 216 int flags, 217 int *addr_len) 218 { 219 struct tcp_sock *tcp = tcp_sk(sk); 220 u32 seq = tcp->copied_seq; 221 struct sk_psock *psock; 222 int copied = 0; 223 224 if (unlikely(flags & MSG_ERRQUEUE)) 225 return inet_recv_error(sk, msg, len, addr_len); 226 227 if (!len) 228 return 0; 229 230 psock = sk_psock_get(sk); 231 if (unlikely(!psock)) 232 return tcp_recvmsg(sk, msg, len, flags, addr_len); 233 234 lock_sock(sk); 235 236 /* We may have received data on the sk_receive_queue pre-accept and 237 * then we can not use read_skb in this context because we haven't 238 * assigned a sk_socket yet so have no link to the ops. The work-around 239 * is to check the sk_receive_queue and in these cases read skbs off 240 * queue again. The read_skb hook is not running at this point because 241 * of lock_sock so we avoid having multiple runners in read_skb. 242 */ 243 if (unlikely(!skb_queue_empty(&sk->sk_receive_queue))) { 244 tcp_data_ready(sk); 245 /* This handles the ENOMEM errors if we both receive data 246 * pre accept and are already under memory pressure. At least 247 * let user know to retry. 248 */ 249 if (unlikely(!skb_queue_empty(&sk->sk_receive_queue))) { 250 copied = -EAGAIN; 251 goto out; 252 } 253 } 254 255 msg_bytes_ready: 256 copied = sk_msg_recvmsg(sk, psock, msg, len, flags); 257 /* The typical case for EFAULT is the socket was gracefully 258 * shutdown with a FIN pkt. So check here the other case is 259 * some error on copy_page_to_iter which would be unexpected. 260 * On fin return correct return code to zero. 261 */ 262 if (copied == -EFAULT) { 263 bool is_fin = is_next_msg_fin(psock); 264 265 if (is_fin) { 266 copied = 0; 267 seq++; 268 goto out; 269 } 270 } 271 seq += copied; 272 if (!copied) { 273 long timeo; 274 int data; 275 276 if (sock_flag(sk, SOCK_DONE)) 277 goto out; 278 279 if (sk->sk_err) { 280 copied = sock_error(sk); 281 goto out; 282 } 283 284 if (sk->sk_shutdown & RCV_SHUTDOWN) 285 goto out; 286 287 if (sk->sk_state == TCP_CLOSE) { 288 copied = -ENOTCONN; 289 goto out; 290 } 291 292 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 293 if (!timeo) { 294 copied = -EAGAIN; 295 goto out; 296 } 297 298 if (signal_pending(current)) { 299 copied = sock_intr_errno(timeo); 300 goto out; 301 } 302 303 data = tcp_msg_wait_data(sk, psock, timeo); 304 if (data && !sk_psock_queue_empty(psock)) 305 goto msg_bytes_ready; 306 copied = -EAGAIN; 307 } 308 out: 309 WRITE_ONCE(tcp->copied_seq, seq); 310 tcp_rcv_space_adjust(sk); 311 if (copied > 0) 312 __tcp_cleanup_rbuf(sk, copied); 313 release_sock(sk); 314 sk_psock_put(sk, psock); 315 return copied; 316 } 317 318 static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 319 int flags, int *addr_len) 320 { 321 struct sk_psock *psock; 322 int copied, ret; 323 324 if (unlikely(flags & MSG_ERRQUEUE)) 325 return inet_recv_error(sk, msg, len, addr_len); 326 327 if (!len) 328 return 0; 329 330 psock = sk_psock_get(sk); 331 if (unlikely(!psock)) 332 return tcp_recvmsg(sk, msg, len, flags, addr_len); 333 if (!skb_queue_empty(&sk->sk_receive_queue) && 334 sk_psock_queue_empty(psock)) { 335 sk_psock_put(sk, psock); 336 return tcp_recvmsg(sk, msg, len, flags, addr_len); 337 } 338 lock_sock(sk); 339 msg_bytes_ready: 340 copied = sk_msg_recvmsg(sk, psock, msg, len, flags); 341 if (!copied) { 342 long timeo; 343 int data; 344 345 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 346 data = tcp_msg_wait_data(sk, psock, timeo); 347 if (data) { 348 if (!sk_psock_queue_empty(psock)) 349 goto msg_bytes_ready; 350 release_sock(sk); 351 sk_psock_put(sk, psock); 352 return tcp_recvmsg(sk, msg, len, flags, addr_len); 353 } 354 copied = -EAGAIN; 355 } 356 ret = copied; 357 release_sock(sk); 358 sk_psock_put(sk, psock); 359 return ret; 360 } 361 362 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, 363 struct sk_msg *msg, int *copied, int flags) 364 { 365 bool cork = false, enospc = sk_msg_full(msg), redir_ingress; 366 struct sock *sk_redir; 367 u32 tosend, origsize, sent, delta = 0; 368 u32 eval; 369 int ret; 370 371 more_data: 372 if (psock->eval == __SK_NONE) { 373 /* Track delta in msg size to add/subtract it on SK_DROP from 374 * returned to user copied size. This ensures user doesn't 375 * get a positive return code with msg_cut_data and SK_DROP 376 * verdict. 377 */ 378 delta = msg->sg.size; 379 psock->eval = sk_psock_msg_verdict(sk, psock, msg); 380 delta -= msg->sg.size; 381 } 382 383 if (msg->cork_bytes && 384 msg->cork_bytes > msg->sg.size && !enospc) { 385 psock->cork_bytes = msg->cork_bytes - msg->sg.size; 386 if (!psock->cork) { 387 psock->cork = kzalloc(sizeof(*psock->cork), 388 GFP_ATOMIC | __GFP_NOWARN); 389 if (!psock->cork) 390 return -ENOMEM; 391 } 392 memcpy(psock->cork, msg, sizeof(*msg)); 393 return 0; 394 } 395 396 tosend = msg->sg.size; 397 if (psock->apply_bytes && psock->apply_bytes < tosend) 398 tosend = psock->apply_bytes; 399 eval = __SK_NONE; 400 401 switch (psock->eval) { 402 case __SK_PASS: 403 ret = tcp_bpf_push(sk, msg, tosend, flags, true); 404 if (unlikely(ret)) { 405 *copied -= sk_msg_free(sk, msg); 406 break; 407 } 408 sk_msg_apply_bytes(psock, tosend); 409 break; 410 case __SK_REDIRECT: 411 redir_ingress = psock->redir_ingress; 412 sk_redir = psock->sk_redir; 413 sk_msg_apply_bytes(psock, tosend); 414 if (!psock->apply_bytes) { 415 /* Clean up before releasing the sock lock. */ 416 eval = psock->eval; 417 psock->eval = __SK_NONE; 418 psock->sk_redir = NULL; 419 } 420 if (psock->cork) { 421 cork = true; 422 psock->cork = NULL; 423 } 424 sk_msg_return(sk, msg, tosend); 425 release_sock(sk); 426 427 origsize = msg->sg.size; 428 ret = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress, 429 msg, tosend, flags); 430 sent = origsize - msg->sg.size; 431 432 if (eval == __SK_REDIRECT) 433 sock_put(sk_redir); 434 435 lock_sock(sk); 436 if (unlikely(ret < 0)) { 437 int free = sk_msg_free_nocharge(sk, msg); 438 439 if (!cork) 440 *copied -= free; 441 } 442 if (cork) { 443 sk_msg_free(sk, msg); 444 kfree(msg); 445 msg = NULL; 446 ret = 0; 447 } 448 break; 449 case __SK_DROP: 450 default: 451 sk_msg_free_partial(sk, msg, tosend); 452 sk_msg_apply_bytes(psock, tosend); 453 *copied -= (tosend + delta); 454 return -EACCES; 455 } 456 457 if (likely(!ret)) { 458 if (!psock->apply_bytes) { 459 psock->eval = __SK_NONE; 460 if (psock->sk_redir) { 461 sock_put(psock->sk_redir); 462 psock->sk_redir = NULL; 463 } 464 } 465 if (msg && 466 msg->sg.data[msg->sg.start].page_link && 467 msg->sg.data[msg->sg.start].length) { 468 if (eval == __SK_REDIRECT) 469 sk_mem_charge(sk, tosend - sent); 470 goto more_data; 471 } 472 } 473 return ret; 474 } 475 476 static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 477 { 478 struct sk_msg tmp, *msg_tx = NULL; 479 int copied = 0, err = 0; 480 struct sk_psock *psock; 481 long timeo; 482 int flags; 483 484 /* Don't let internal do_tcp_sendpages() flags through */ 485 flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED); 486 flags |= MSG_NO_SHARED_FRAGS; 487 488 psock = sk_psock_get(sk); 489 if (unlikely(!psock)) 490 return tcp_sendmsg(sk, msg, size); 491 492 lock_sock(sk); 493 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 494 while (msg_data_left(msg)) { 495 bool enospc = false; 496 u32 copy, osize; 497 498 if (sk->sk_err) { 499 err = -sk->sk_err; 500 goto out_err; 501 } 502 503 copy = msg_data_left(msg); 504 if (!sk_stream_memory_free(sk)) 505 goto wait_for_sndbuf; 506 if (psock->cork) { 507 msg_tx = psock->cork; 508 } else { 509 msg_tx = &tmp; 510 sk_msg_init(msg_tx); 511 } 512 513 osize = msg_tx->sg.size; 514 err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1); 515 if (err) { 516 if (err != -ENOSPC) 517 goto wait_for_memory; 518 enospc = true; 519 copy = msg_tx->sg.size - osize; 520 } 521 522 err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx, 523 copy); 524 if (err < 0) { 525 sk_msg_trim(sk, msg_tx, osize); 526 goto out_err; 527 } 528 529 copied += copy; 530 if (psock->cork_bytes) { 531 if (size > psock->cork_bytes) 532 psock->cork_bytes = 0; 533 else 534 psock->cork_bytes -= size; 535 if (psock->cork_bytes && !enospc) 536 goto out_err; 537 /* All cork bytes are accounted, rerun the prog. */ 538 psock->eval = __SK_NONE; 539 psock->cork_bytes = 0; 540 } 541 542 err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags); 543 if (unlikely(err < 0)) 544 goto out_err; 545 continue; 546 wait_for_sndbuf: 547 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 548 wait_for_memory: 549 err = sk_stream_wait_memory(sk, &timeo); 550 if (err) { 551 if (msg_tx && msg_tx != psock->cork) 552 sk_msg_free(sk, msg_tx); 553 goto out_err; 554 } 555 } 556 out_err: 557 if (err < 0) 558 err = sk_stream_error(sk, msg->msg_flags, err); 559 release_sock(sk); 560 sk_psock_put(sk, psock); 561 return copied ? copied : err; 562 } 563 564 static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset, 565 size_t size, int flags) 566 { 567 struct sk_msg tmp, *msg = NULL; 568 int err = 0, copied = 0; 569 struct sk_psock *psock; 570 bool enospc = false; 571 572 psock = sk_psock_get(sk); 573 if (unlikely(!psock)) 574 return tcp_sendpage(sk, page, offset, size, flags); 575 576 lock_sock(sk); 577 if (psock->cork) { 578 msg = psock->cork; 579 } else { 580 msg = &tmp; 581 sk_msg_init(msg); 582 } 583 584 /* Catch case where ring is full and sendpage is stalled. */ 585 if (unlikely(sk_msg_full(msg))) 586 goto out_err; 587 588 sk_msg_page_add(msg, page, size, offset); 589 sk_mem_charge(sk, size); 590 copied = size; 591 if (sk_msg_full(msg)) 592 enospc = true; 593 if (psock->cork_bytes) { 594 if (size > psock->cork_bytes) 595 psock->cork_bytes = 0; 596 else 597 psock->cork_bytes -= size; 598 if (psock->cork_bytes && !enospc) 599 goto out_err; 600 /* All cork bytes are accounted, rerun the prog. */ 601 psock->eval = __SK_NONE; 602 psock->cork_bytes = 0; 603 } 604 605 err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags); 606 out_err: 607 release_sock(sk); 608 sk_psock_put(sk, psock); 609 return copied ? copied : err; 610 } 611 612 enum { 613 TCP_BPF_IPV4, 614 TCP_BPF_IPV6, 615 TCP_BPF_NUM_PROTS, 616 }; 617 618 enum { 619 TCP_BPF_BASE, 620 TCP_BPF_TX, 621 TCP_BPF_RX, 622 TCP_BPF_TXRX, 623 TCP_BPF_NUM_CFGS, 624 }; 625 626 static struct proto *tcpv6_prot_saved __read_mostly; 627 static DEFINE_SPINLOCK(tcpv6_prot_lock); 628 static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS]; 629 630 static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], 631 struct proto *base) 632 { 633 prot[TCP_BPF_BASE] = *base; 634 prot[TCP_BPF_BASE].destroy = sock_map_destroy; 635 prot[TCP_BPF_BASE].close = sock_map_close; 636 prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; 637 prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable; 638 639 prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; 640 prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; 641 prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage; 642 643 prot[TCP_BPF_RX] = prot[TCP_BPF_BASE]; 644 prot[TCP_BPF_RX].recvmsg = tcp_bpf_recvmsg_parser; 645 646 prot[TCP_BPF_TXRX] = prot[TCP_BPF_TX]; 647 prot[TCP_BPF_TXRX].recvmsg = tcp_bpf_recvmsg_parser; 648 } 649 650 static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops) 651 { 652 if (unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) { 653 spin_lock_bh(&tcpv6_prot_lock); 654 if (likely(ops != tcpv6_prot_saved)) { 655 tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops); 656 smp_store_release(&tcpv6_prot_saved, ops); 657 } 658 spin_unlock_bh(&tcpv6_prot_lock); 659 } 660 } 661 662 static int __init tcp_bpf_v4_build_proto(void) 663 { 664 tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot); 665 return 0; 666 } 667 late_initcall(tcp_bpf_v4_build_proto); 668 669 static int tcp_bpf_assert_proto_ops(struct proto *ops) 670 { 671 /* In order to avoid retpoline, we make assumptions when we call 672 * into ops if e.g. a psock is not present. Make sure they are 673 * indeed valid assumptions. 674 */ 675 return ops->recvmsg == tcp_recvmsg && 676 ops->sendmsg == tcp_sendmsg && 677 ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP; 678 } 679 680 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) 681 { 682 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; 683 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; 684 685 if (psock->progs.stream_verdict || psock->progs.skb_verdict) { 686 config = (config == TCP_BPF_TX) ? TCP_BPF_TXRX : TCP_BPF_RX; 687 } 688 689 if (restore) { 690 if (inet_csk_has_ulp(sk)) { 691 /* TLS does not have an unhash proto in SW cases, 692 * but we need to ensure we stop using the sock_map 693 * unhash routine because the associated psock is being 694 * removed. So use the original unhash handler. 695 */ 696 WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash); 697 tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space); 698 } else { 699 sk->sk_write_space = psock->saved_write_space; 700 /* Pairs with lockless read in sk_clone_lock() */ 701 sock_replace_proto(sk, psock->sk_proto); 702 } 703 return 0; 704 } 705 706 if (sk->sk_family == AF_INET6) { 707 if (tcp_bpf_assert_proto_ops(psock->sk_proto)) 708 return -EINVAL; 709 710 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto); 711 } 712 713 /* Pairs with lockless read in sk_clone_lock() */ 714 sock_replace_proto(sk, &tcp_bpf_prots[family][config]); 715 return 0; 716 } 717 EXPORT_SYMBOL_GPL(tcp_bpf_update_proto); 718 719 /* If a child got cloned from a listening socket that had tcp_bpf 720 * protocol callbacks installed, we need to restore the callbacks to 721 * the default ones because the child does not inherit the psock state 722 * that tcp_bpf callbacks expect. 723 */ 724 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 725 { 726 struct proto *prot = newsk->sk_prot; 727 728 if (is_insidevar(prot, tcp_bpf_prots)) 729 newsk->sk_prot = sk->sk_prot_creator; 730 } 731 #endif /* CONFIG_BPF_SYSCALL */ 732