skmsg.c (cdd38c5f1ce4398ec58fec95904b75824daab7b5) skmsg.c (887596095ec2a9ea39ffcf98f27bf2e77c5eb512)
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4#include <linux/skmsg.h>
5#include <linux/skbuff.h>
6#include <linux/scatterlist.h>
7
8#include <net/sock.h>

--- 631 unchanged lines hidden (view full) ---

640 struct sk_psock_link *link, *tmp;
641
642 list_for_each_entry_safe(link, tmp, &psock->link, list) {
643 list_del(&link->list);
644 sk_psock_free_link(link);
645 }
646}
647
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4#include <linux/skmsg.h>
5#include <linux/skbuff.h>
6#include <linux/scatterlist.h>
7
8#include <net/sock.h>

--- 631 unchanged lines hidden (view full) ---

640 struct sk_psock_link *link, *tmp;
641
642 list_for_each_entry_safe(link, tmp, &psock->link, list) {
643 list_del(&link->list);
644 sk_psock_free_link(link);
645 }
646}
647
648static void sk_psock_done_strp(struct sk_psock *psock);
649
648static void sk_psock_destroy_deferred(struct work_struct *gc)
649{
650 struct sk_psock *psock = container_of(gc, struct sk_psock, gc);
651
652 /* No sk_callback_lock since already detached. */
653
650static void sk_psock_destroy_deferred(struct work_struct *gc)
651{
652 struct sk_psock *psock = container_of(gc, struct sk_psock, gc);
653
654 /* No sk_callback_lock since already detached. */
655
654 /* Parser has been stopped */
655 if (psock->progs.skb_parser)
656 strp_done(&psock->parser.strp);
656 sk_psock_done_strp(psock);
657
658 cancel_work_sync(&psock->work);
659
660 psock_progs_drop(&psock->progs);
661
662 sk_psock_link_destroy(psock);
663 sk_psock_cork_free(psock);
664 sk_psock_zap_ingress(psock);

--- 80 unchanged lines hidden (view full) ---

745
746static int sk_psock_bpf_run(struct sk_psock *psock, struct bpf_prog *prog,
747 struct sk_buff *skb)
748{
749 bpf_compute_data_end_sk_skb(skb);
750 return bpf_prog_run_pin_on_cpu(prog, skb);
751}
752
657
658 cancel_work_sync(&psock->work);
659
660 psock_progs_drop(&psock->progs);
661
662 sk_psock_link_destroy(psock);
663 sk_psock_cork_free(psock);
664 sk_psock_zap_ingress(psock);

--- 80 unchanged lines hidden (view full) ---

745
746static int sk_psock_bpf_run(struct sk_psock *psock, struct bpf_prog *prog,
747 struct sk_buff *skb)
748{
749 bpf_compute_data_end_sk_skb(skb);
750 return bpf_prog_run_pin_on_cpu(prog, skb);
751}
752
753static struct sk_psock *sk_psock_from_strp(struct strparser *strp)
754{
755 struct sk_psock_parser *parser;
756
757 parser = container_of(strp, struct sk_psock_parser, strp);
758 return container_of(parser, struct sk_psock, parser);
759}
760
761static void sk_psock_skb_redirect(struct sk_buff *skb)
762{
763 struct sk_psock *psock_other;
764 struct sock *sk_other;
765
766 sk_other = tcp_skb_bpf_redirect_fetch(skb);
767 /* This error is a buggy BPF program, it returned a redirect
768 * return code, but then didn't set a redirect interface.

--- 92 unchanged lines hidden (view full) ---

861 break;
862 case __SK_DROP:
863 default:
864out_free:
865 kfree_skb(skb);
866 }
867}
868
753static void sk_psock_skb_redirect(struct sk_buff *skb)
754{
755 struct sk_psock *psock_other;
756 struct sock *sk_other;
757
758 sk_other = tcp_skb_bpf_redirect_fetch(skb);
759 /* This error is a buggy BPF program, it returned a redirect
760 * return code, but then didn't set a redirect interface.

--- 92 unchanged lines hidden (view full) ---

853 break;
854 case __SK_DROP:
855 default:
856out_free:
857 kfree_skb(skb);
858 }
859}
860
861static void sk_psock_write_space(struct sock *sk)
862{
863 struct sk_psock *psock;
864 void (*write_space)(struct sock *sk) = NULL;
865
866 rcu_read_lock();
867 psock = sk_psock(sk);
868 if (likely(psock)) {
869 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
870 schedule_work(&psock->work);
871 write_space = psock->saved_write_space;
872 }
873 rcu_read_unlock();
874 if (write_space)
875 write_space(sk);
876}
877
878#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
869static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb)
870{
871 struct sk_psock *psock;
872 struct bpf_prog *prog;
873 int ret = __SK_DROP;
874 struct sock *sk;
875
876 rcu_read_lock();

--- 15 unchanged lines hidden (view full) ---

892 rcu_read_unlock();
893}
894
895static int sk_psock_strp_read_done(struct strparser *strp, int err)
896{
897 return err;
898}
899
879static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb)
880{
881 struct sk_psock *psock;
882 struct bpf_prog *prog;
883 int ret = __SK_DROP;
884 struct sock *sk;
885
886 rcu_read_lock();

--- 15 unchanged lines hidden (view full) ---

902 rcu_read_unlock();
903}
904
905static int sk_psock_strp_read_done(struct strparser *strp, int err)
906{
907 return err;
908}
909
910static struct sk_psock *sk_psock_from_strp(struct strparser *strp)
911{
912 struct sk_psock_parser *parser;
913
914 parser = container_of(strp, struct sk_psock_parser, strp);
915 return container_of(parser, struct sk_psock, parser);
916}
917
900static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb)
901{
902 struct sk_psock *psock = sk_psock_from_strp(strp);
903 struct bpf_prog *prog;
904 int ret = skb->len;
905
906 rcu_read_lock();
907 prog = READ_ONCE(psock->progs.skb_parser);

--- 20 unchanged lines hidden (view full) ---

928 write_lock_bh(&sk->sk_callback_lock);
929 strp_data_ready(&psock->parser.strp);
930 write_unlock_bh(&sk->sk_callback_lock);
931 }
932 }
933 rcu_read_unlock();
934}
935
918static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb)
919{
920 struct sk_psock *psock = sk_psock_from_strp(strp);
921 struct bpf_prog *prog;
922 int ret = skb->len;
923
924 rcu_read_lock();
925 prog = READ_ONCE(psock->progs.skb_parser);

--- 20 unchanged lines hidden (view full) ---

946 write_lock_bh(&sk->sk_callback_lock);
947 strp_data_ready(&psock->parser.strp);
948 write_unlock_bh(&sk->sk_callback_lock);
949 }
950 }
951 rcu_read_unlock();
952}
953
954int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
955{
956 static const struct strp_callbacks cb = {
957 .rcv_msg = sk_psock_strp_read,
958 .read_sock_done = sk_psock_strp_read_done,
959 .parse_msg = sk_psock_strp_parse,
960 };
961
962 psock->parser.enabled = false;
963 return strp_init(&psock->parser.strp, sk, &cb);
964}
965
966void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
967{
968 struct sk_psock_parser *parser = &psock->parser;
969
970 if (parser->enabled)
971 return;
972
973 parser->saved_data_ready = sk->sk_data_ready;
974 sk->sk_data_ready = sk_psock_strp_data_ready;
975 sk->sk_write_space = sk_psock_write_space;
976 parser->enabled = true;
977}
978
979void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
980{
981 struct sk_psock_parser *parser = &psock->parser;
982
983 if (!parser->enabled)
984 return;
985
986 sk->sk_data_ready = parser->saved_data_ready;
987 parser->saved_data_ready = NULL;
988 strp_stop(&parser->strp);
989 parser->enabled = false;
990}
991
992static void sk_psock_done_strp(struct sk_psock *psock)
993{
994 /* Parser has been stopped */
995 if (psock->progs.skb_parser)
996 strp_done(&psock->parser.strp);
997}
998#else
999static void sk_psock_done_strp(struct sk_psock *psock)
1000{
1001}
1002#endif /* CONFIG_BPF_STREAM_PARSER */
1003
936static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb,
937 unsigned int offset, size_t orig_len)
938{
939 struct sock *sk = (struct sock *)desc->arg.data;
940 struct sk_psock *psock;
941 struct bpf_prog *prog;
942 int ret = __SK_DROP;
943 int len = skb->len;

--- 35 unchanged lines hidden (view full) ---

979
980 desc.arg.data = sk;
981 desc.error = 0;
982 desc.count = 1;
983
984 sock->ops->read_sock(sk, &desc, sk_psock_verdict_recv);
985}
986
1004static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb,
1005 unsigned int offset, size_t orig_len)
1006{
1007 struct sock *sk = (struct sock *)desc->arg.data;
1008 struct sk_psock *psock;
1009 struct bpf_prog *prog;
1010 int ret = __SK_DROP;
1011 int len = skb->len;

--- 35 unchanged lines hidden (view full) ---

1047
1048 desc.arg.data = sk;
1049 desc.error = 0;
1050 desc.count = 1;
1051
1052 sock->ops->read_sock(sk, &desc, sk_psock_verdict_recv);
1053}
1054
987static void sk_psock_write_space(struct sock *sk)
988{
989 struct sk_psock *psock;
990 void (*write_space)(struct sock *sk) = NULL;
991
992 rcu_read_lock();
993 psock = sk_psock(sk);
994 if (likely(psock)) {
995 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
996 schedule_work(&psock->work);
997 write_space = psock->saved_write_space;
998 }
999 rcu_read_unlock();
1000 if (write_space)
1001 write_space(sk);
1002}
1003
1004int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
1005{
1006 static const struct strp_callbacks cb = {
1007 .rcv_msg = sk_psock_strp_read,
1008 .read_sock_done = sk_psock_strp_read_done,
1009 .parse_msg = sk_psock_strp_parse,
1010 };
1011
1012 psock->parser.enabled = false;
1013 return strp_init(&psock->parser.strp, sk, &cb);
1014}
1015
1016void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
1017{
1018 struct sk_psock_parser *parser = &psock->parser;
1019
1020 if (parser->enabled)
1021 return;
1022
1023 parser->saved_data_ready = sk->sk_data_ready;
1024 sk->sk_data_ready = sk_psock_verdict_data_ready;
1025 sk->sk_write_space = sk_psock_write_space;
1026 parser->enabled = true;
1027}
1028
1055void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
1056{
1057 struct sk_psock_parser *parser = &psock->parser;
1058
1059 if (parser->enabled)
1060 return;
1061
1062 parser->saved_data_ready = sk->sk_data_ready;
1063 sk->sk_data_ready = sk_psock_verdict_data_ready;
1064 sk->sk_write_space = sk_psock_write_space;
1065 parser->enabled = true;
1066}
1067
1029void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
1030{
1031 struct sk_psock_parser *parser = &psock->parser;
1032
1033 if (parser->enabled)
1034 return;
1035
1036 parser->saved_data_ready = sk->sk_data_ready;
1037 sk->sk_data_ready = sk_psock_strp_data_ready;
1038 sk->sk_write_space = sk_psock_write_space;
1039 parser->enabled = true;
1040}
1041
1042void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
1043{
1044 struct sk_psock_parser *parser = &psock->parser;
1045
1046 if (!parser->enabled)
1047 return;
1048
1049 sk->sk_data_ready = parser->saved_data_ready;
1050 parser->saved_data_ready = NULL;
1051 strp_stop(&parser->strp);
1052 parser->enabled = false;
1053}
1054
1055void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
1056{
1057 struct sk_psock_parser *parser = &psock->parser;
1058
1059 if (!parser->enabled)
1060 return;
1061
1062 sk->sk_data_ready = parser->saved_data_ready;
1063 parser->saved_data_ready = NULL;
1064 parser->enabled = false;
1065}
1068void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
1069{
1070 struct sk_psock_parser *parser = &psock->parser;
1071
1072 if (!parser->enabled)
1073 return;
1074
1075 sk->sk_data_ready = parser->saved_data_ready;
1076 parser->saved_data_ready = NULL;
1077 parser->enabled = false;
1078}