1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2023 Isovalent */
3 #ifndef __NET_TCX_H
4 #define __NET_TCX_H
5
6 #include <linux/bpf.h>
7 #include <linux/bpf_mprog.h>
8
9 #include <net/sch_generic.h>
10
11 struct mini_Qdisc;
12
13 struct tcx_entry {
14 struct mini_Qdisc __rcu *miniq;
15 struct bpf_mprog_bundle bundle;
16 u32 miniq_active;
17 struct rcu_head rcu;
18 };
19
20 struct tcx_link {
21 struct bpf_link link;
22 struct net_device *dev;
23 u32 location;
24 };
25
tcx_set_ingress(struct sk_buff * skb,bool ingress)26 static inline void tcx_set_ingress(struct sk_buff *skb, bool ingress)
27 {
28 #ifdef CONFIG_NET_XGRESS
29 skb->tc_at_ingress = ingress;
30 #endif
31 }
32
33 #ifdef CONFIG_NET_XGRESS
tcx_entry(struct bpf_mprog_entry * entry)34 static inline struct tcx_entry *tcx_entry(struct bpf_mprog_entry *entry)
35 {
36 struct bpf_mprog_bundle *bundle = entry->parent;
37
38 return container_of(bundle, struct tcx_entry, bundle);
39 }
40
tcx_link(struct bpf_link * link)41 static inline struct tcx_link *tcx_link(struct bpf_link *link)
42 {
43 return container_of(link, struct tcx_link, link);
44 }
45
tcx_link_const(const struct bpf_link * link)46 static inline const struct tcx_link *tcx_link_const(const struct bpf_link *link)
47 {
48 return tcx_link((struct bpf_link *)link);
49 }
50
51 void tcx_inc(void);
52 void tcx_dec(void);
53
tcx_entry_sync(void)54 static inline void tcx_entry_sync(void)
55 {
56 /* bpf_mprog_entry got a/b swapped, therefore ensure that
57 * there are no inflight users on the old one anymore.
58 */
59 synchronize_rcu();
60 }
61
62 static inline void
tcx_entry_update(struct net_device * dev,struct bpf_mprog_entry * entry,bool ingress)63 tcx_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry,
64 bool ingress)
65 {
66 ASSERT_RTNL();
67 if (ingress)
68 rcu_assign_pointer(dev->tcx_ingress, entry);
69 else
70 rcu_assign_pointer(dev->tcx_egress, entry);
71 }
72
73 static inline struct bpf_mprog_entry *
tcx_entry_fetch(struct net_device * dev,bool ingress)74 tcx_entry_fetch(struct net_device *dev, bool ingress)
75 {
76 ASSERT_RTNL();
77 if (ingress)
78 return rcu_dereference_rtnl(dev->tcx_ingress);
79 else
80 return rcu_dereference_rtnl(dev->tcx_egress);
81 }
82
tcx_entry_create(void)83 static inline struct bpf_mprog_entry *tcx_entry_create(void)
84 {
85 struct tcx_entry *tcx = kzalloc(sizeof(*tcx), GFP_KERNEL);
86
87 if (tcx) {
88 bpf_mprog_bundle_init(&tcx->bundle);
89 return &tcx->bundle.a;
90 }
91 return NULL;
92 }
93
tcx_entry_free(struct bpf_mprog_entry * entry)94 static inline void tcx_entry_free(struct bpf_mprog_entry *entry)
95 {
96 kfree_rcu(tcx_entry(entry), rcu);
97 }
98
99 static inline struct bpf_mprog_entry *
tcx_entry_fetch_or_create(struct net_device * dev,bool ingress,bool * created)100 tcx_entry_fetch_or_create(struct net_device *dev, bool ingress, bool *created)
101 {
102 struct bpf_mprog_entry *entry = tcx_entry_fetch(dev, ingress);
103
104 *created = false;
105 if (!entry) {
106 entry = tcx_entry_create();
107 if (!entry)
108 return NULL;
109 *created = true;
110 }
111 return entry;
112 }
113
tcx_skeys_inc(bool ingress)114 static inline void tcx_skeys_inc(bool ingress)
115 {
116 tcx_inc();
117 if (ingress)
118 net_inc_ingress_queue();
119 else
120 net_inc_egress_queue();
121 }
122
tcx_skeys_dec(bool ingress)123 static inline void tcx_skeys_dec(bool ingress)
124 {
125 if (ingress)
126 net_dec_ingress_queue();
127 else
128 net_dec_egress_queue();
129 tcx_dec();
130 }
131
tcx_miniq_inc(struct bpf_mprog_entry * entry)132 static inline void tcx_miniq_inc(struct bpf_mprog_entry *entry)
133 {
134 ASSERT_RTNL();
135 tcx_entry(entry)->miniq_active++;
136 }
137
tcx_miniq_dec(struct bpf_mprog_entry * entry)138 static inline void tcx_miniq_dec(struct bpf_mprog_entry *entry)
139 {
140 ASSERT_RTNL();
141 tcx_entry(entry)->miniq_active--;
142 }
143
tcx_entry_is_active(struct bpf_mprog_entry * entry)144 static inline bool tcx_entry_is_active(struct bpf_mprog_entry *entry)
145 {
146 ASSERT_RTNL();
147 return bpf_mprog_total(entry) || tcx_entry(entry)->miniq_active;
148 }
149
tcx_action_code(struct sk_buff * skb,int code)150 static inline enum tcx_action_base tcx_action_code(struct sk_buff *skb,
151 int code)
152 {
153 switch (code) {
154 case TCX_PASS:
155 skb->tc_index = qdisc_skb_cb(skb)->tc_classid;
156 fallthrough;
157 case TCX_DROP:
158 case TCX_REDIRECT:
159 return code;
160 case TCX_NEXT:
161 default:
162 return TCX_NEXT;
163 }
164 }
165 #endif /* CONFIG_NET_XGRESS */
166
167 #if defined(CONFIG_NET_XGRESS) && defined(CONFIG_BPF_SYSCALL)
168 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog);
169 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
170 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog);
171 void tcx_uninstall(struct net_device *dev, bool ingress);
172
173 int tcx_prog_query(const union bpf_attr *attr,
174 union bpf_attr __user *uattr);
175
dev_tcx_uninstall(struct net_device * dev)176 static inline void dev_tcx_uninstall(struct net_device *dev)
177 {
178 ASSERT_RTNL();
179 tcx_uninstall(dev, true);
180 tcx_uninstall(dev, false);
181 }
182 #else
tcx_prog_attach(const union bpf_attr * attr,struct bpf_prog * prog)183 static inline int tcx_prog_attach(const union bpf_attr *attr,
184 struct bpf_prog *prog)
185 {
186 return -EINVAL;
187 }
188
tcx_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)189 static inline int tcx_link_attach(const union bpf_attr *attr,
190 struct bpf_prog *prog)
191 {
192 return -EINVAL;
193 }
194
tcx_prog_detach(const union bpf_attr * attr,struct bpf_prog * prog)195 static inline int tcx_prog_detach(const union bpf_attr *attr,
196 struct bpf_prog *prog)
197 {
198 return -EINVAL;
199 }
200
tcx_prog_query(const union bpf_attr * attr,union bpf_attr __user * uattr)201 static inline int tcx_prog_query(const union bpf_attr *attr,
202 union bpf_attr __user *uattr)
203 {
204 return -EINVAL;
205 }
206
dev_tcx_uninstall(struct net_device * dev)207 static inline void dev_tcx_uninstall(struct net_device *dev)
208 {
209 }
210 #endif /* CONFIG_NET_XGRESS && CONFIG_BPF_SYSCALL */
211 #endif /* __NET_TCX_H */
212