1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2023 Isovalent */ 3 #ifndef __NET_TCX_H 4 #define __NET_TCX_H 5 6 #include <linux/bpf.h> 7 #include <linux/bpf_mprog.h> 8 9 #include <net/sch_generic.h> 10 11 struct mini_Qdisc; 12 13 struct tcx_entry { 14 struct mini_Qdisc __rcu *miniq; 15 struct bpf_mprog_bundle bundle; 16 u32 miniq_active; 17 struct rcu_head rcu; 18 }; 19 20 struct tcx_link { 21 struct bpf_link link; 22 struct net_device *dev; 23 u32 location; 24 }; 25 26 static inline void tcx_set_ingress(struct sk_buff *skb, bool ingress) 27 { 28 #ifdef CONFIG_NET_XGRESS 29 skb->tc_at_ingress = ingress; 30 #endif 31 } 32 33 #ifdef CONFIG_NET_XGRESS 34 static inline struct tcx_entry *tcx_entry(struct bpf_mprog_entry *entry) 35 { 36 struct bpf_mprog_bundle *bundle = entry->parent; 37 38 return container_of(bundle, struct tcx_entry, bundle); 39 } 40 41 static inline struct tcx_link *tcx_link(struct bpf_link *link) 42 { 43 return container_of(link, struct tcx_link, link); 44 } 45 46 static inline const struct tcx_link *tcx_link_const(const struct bpf_link *link) 47 { 48 return tcx_link((struct bpf_link *)link); 49 } 50 51 void tcx_inc(void); 52 void tcx_dec(void); 53 54 static inline void tcx_entry_sync(void) 55 { 56 /* bpf_mprog_entry got a/b swapped, therefore ensure that 57 * there are no inflight users on the old one anymore. 58 */ 59 synchronize_rcu(); 60 } 61 62 static inline void 63 tcx_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry, 64 bool ingress) 65 { 66 ASSERT_RTNL(); 67 if (ingress) 68 rcu_assign_pointer(dev->tcx_ingress, entry); 69 else 70 rcu_assign_pointer(dev->tcx_egress, entry); 71 } 72 73 static inline struct bpf_mprog_entry * 74 tcx_entry_fetch(struct net_device *dev, bool ingress) 75 { 76 ASSERT_RTNL(); 77 if (ingress) 78 return rcu_dereference_rtnl(dev->tcx_ingress); 79 else 80 return rcu_dereference_rtnl(dev->tcx_egress); 81 } 82 83 static inline struct bpf_mprog_entry *tcx_entry_create(void) 84 { 85 struct tcx_entry *tcx = kzalloc(sizeof(*tcx), GFP_KERNEL); 86 87 if (tcx) { 88 bpf_mprog_bundle_init(&tcx->bundle); 89 return &tcx->bundle.a; 90 } 91 return NULL; 92 } 93 94 static inline void tcx_entry_free(struct bpf_mprog_entry *entry) 95 { 96 kfree_rcu(tcx_entry(entry), rcu); 97 } 98 99 static inline struct bpf_mprog_entry * 100 tcx_entry_fetch_or_create(struct net_device *dev, bool ingress, bool *created) 101 { 102 struct bpf_mprog_entry *entry = tcx_entry_fetch(dev, ingress); 103 104 *created = false; 105 if (!entry) { 106 entry = tcx_entry_create(); 107 if (!entry) 108 return NULL; 109 *created = true; 110 } 111 return entry; 112 } 113 114 static inline void tcx_skeys_inc(bool ingress) 115 { 116 tcx_inc(); 117 if (ingress) 118 net_inc_ingress_queue(); 119 else 120 net_inc_egress_queue(); 121 } 122 123 static inline void tcx_skeys_dec(bool ingress) 124 { 125 if (ingress) 126 net_dec_ingress_queue(); 127 else 128 net_dec_egress_queue(); 129 tcx_dec(); 130 } 131 132 static inline void tcx_miniq_inc(struct bpf_mprog_entry *entry) 133 { 134 ASSERT_RTNL(); 135 tcx_entry(entry)->miniq_active++; 136 } 137 138 static inline void tcx_miniq_dec(struct bpf_mprog_entry *entry) 139 { 140 ASSERT_RTNL(); 141 tcx_entry(entry)->miniq_active--; 142 } 143 144 static inline bool tcx_entry_is_active(struct bpf_mprog_entry *entry) 145 { 146 ASSERT_RTNL(); 147 return bpf_mprog_total(entry) || tcx_entry(entry)->miniq_active; 148 } 149 150 static inline enum tcx_action_base tcx_action_code(struct sk_buff *skb, 151 int code) 152 { 153 switch (code) { 154 case TCX_PASS: 155 skb->tc_index = qdisc_skb_cb(skb)->tc_classid; 156 fallthrough; 157 case TCX_DROP: 158 case TCX_REDIRECT: 159 return code; 160 case TCX_NEXT: 161 default: 162 return TCX_NEXT; 163 } 164 } 165 #endif /* CONFIG_NET_XGRESS */ 166 167 #if defined(CONFIG_NET_XGRESS) && defined(CONFIG_BPF_SYSCALL) 168 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog); 169 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 170 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog); 171 void tcx_uninstall(struct net_device *dev, bool ingress); 172 173 int tcx_prog_query(const union bpf_attr *attr, 174 union bpf_attr __user *uattr); 175 176 static inline void dev_tcx_uninstall(struct net_device *dev) 177 { 178 ASSERT_RTNL(); 179 tcx_uninstall(dev, true); 180 tcx_uninstall(dev, false); 181 } 182 #else 183 static inline int tcx_prog_attach(const union bpf_attr *attr, 184 struct bpf_prog *prog) 185 { 186 return -EINVAL; 187 } 188 189 static inline int tcx_link_attach(const union bpf_attr *attr, 190 struct bpf_prog *prog) 191 { 192 return -EINVAL; 193 } 194 195 static inline int tcx_prog_detach(const union bpf_attr *attr, 196 struct bpf_prog *prog) 197 { 198 return -EINVAL; 199 } 200 201 static inline int tcx_prog_query(const union bpf_attr *attr, 202 union bpf_attr __user *uattr) 203 { 204 return -EINVAL; 205 } 206 207 static inline void dev_tcx_uninstall(struct net_device *dev) 208 { 209 } 210 #endif /* CONFIG_NET_XGRESS && CONFIG_BPF_SYSCALL */ 211 #endif /* __NET_TCX_H */ 212