1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008 Patrick McHardy <kaber@trash.net> 4 * 5 * Development of this code funded by Astaro AG (http://www.astaro.com/) 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/init.h> 11 #include <linux/list.h> 12 #include <linux/rculist.h> 13 #include <linux/skbuff.h> 14 #include <linux/netlink.h> 15 #include <linux/netfilter.h> 16 #include <linux/static_key.h> 17 #include <linux/netfilter/nfnetlink.h> 18 #include <linux/netfilter/nf_tables.h> 19 #include <net/netfilter/nf_tables_core.h> 20 #include <net/netfilter/nf_tables.h> 21 #include <net/netfilter/nf_log.h> 22 #include <net/netfilter/nft_meta.h> 23 24 static noinline void __nft_trace_packet(struct nft_traceinfo *info, 25 const struct nft_chain *chain, 26 enum nft_trace_types type) 27 { 28 const struct nft_pktinfo *pkt = info->pkt; 29 30 if (!info->trace || !pkt->skb->nf_trace) 31 return; 32 33 info->chain = chain; 34 info->type = type; 35 36 nft_trace_notify(info); 37 } 38 39 static inline void nft_trace_packet(struct nft_traceinfo *info, 40 const struct nft_chain *chain, 41 const struct nft_rule *rule, 42 enum nft_trace_types type) 43 { 44 if (static_branch_unlikely(&nft_trace_enabled)) { 45 info->rule = rule; 46 __nft_trace_packet(info, chain, type); 47 } 48 } 49 50 static void nft_cmp_fast_eval(const struct nft_expr *expr, 51 struct nft_regs *regs) 52 { 53 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr); 54 u32 mask = nft_cmp_fast_mask(priv->len); 55 56 if ((regs->data[priv->sreg] & mask) == priv->data) 57 return; 58 regs->verdict.code = NFT_BREAK; 59 } 60 61 static bool nft_payload_fast_eval(const struct nft_expr *expr, 62 struct nft_regs *regs, 63 const struct nft_pktinfo *pkt) 64 { 65 const struct nft_payload *priv = nft_expr_priv(expr); 66 const struct sk_buff *skb = pkt->skb; 67 u32 *dest = ®s->data[priv->dreg]; 68 unsigned char *ptr; 69 70 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER) 71 ptr = skb_network_header(skb); 72 else { 73 if (!pkt->tprot_set) 74 return false; 75 ptr = skb_network_header(skb) + pkt->xt.thoff; 76 } 77 78 ptr += priv->offset; 79 80 if (unlikely(ptr + priv->len > skb_tail_pointer(skb))) 81 return false; 82 83 *dest = 0; 84 if (priv->len == 2) 85 *(u16 *)dest = *(u16 *)ptr; 86 else if (priv->len == 4) 87 *(u32 *)dest = *(u32 *)ptr; 88 else 89 *(u8 *)dest = *(u8 *)ptr; 90 return true; 91 } 92 93 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled); 94 95 static noinline void nft_update_chain_stats(const struct nft_chain *chain, 96 const struct nft_pktinfo *pkt) 97 { 98 struct nft_base_chain *base_chain; 99 struct nft_stats __percpu *pstats; 100 struct nft_stats *stats; 101 102 base_chain = nft_base_chain(chain); 103 104 rcu_read_lock(); 105 pstats = READ_ONCE(base_chain->stats); 106 if (pstats) { 107 local_bh_disable(); 108 stats = this_cpu_ptr(pstats); 109 u64_stats_update_begin(&stats->syncp); 110 stats->pkts++; 111 stats->bytes += pkt->skb->len; 112 u64_stats_update_end(&stats->syncp); 113 local_bh_enable(); 114 } 115 rcu_read_unlock(); 116 } 117 118 struct nft_jumpstack { 119 const struct nft_chain *chain; 120 struct nft_rule *const *rules; 121 }; 122 123 static void expr_call_ops_eval(const struct nft_expr *expr, 124 struct nft_regs *regs, 125 struct nft_pktinfo *pkt) 126 { 127 #ifdef CONFIG_RETPOLINE 128 unsigned long e = (unsigned long)expr->ops->eval; 129 #define X(e, fun) \ 130 do { if ((e) == (unsigned long)(fun)) \ 131 return fun(expr, regs, pkt); } while (0) 132 133 X(e, nft_payload_eval); 134 X(e, nft_cmp_eval); 135 X(e, nft_meta_get_eval); 136 X(e, nft_lookup_eval); 137 X(e, nft_range_eval); 138 X(e, nft_immediate_eval); 139 X(e, nft_byteorder_eval); 140 X(e, nft_dynset_eval); 141 X(e, nft_rt_get_eval); 142 X(e, nft_bitwise_eval); 143 #undef X 144 #endif /* CONFIG_RETPOLINE */ 145 expr->ops->eval(expr, regs, pkt); 146 } 147 148 unsigned int 149 nft_do_chain(struct nft_pktinfo *pkt, void *priv) 150 { 151 const struct nft_chain *chain = priv, *basechain = chain; 152 const struct net *net = nft_net(pkt); 153 struct nft_rule *const *rules; 154 const struct nft_rule *rule; 155 const struct nft_expr *expr, *last; 156 struct nft_regs regs; 157 unsigned int stackptr = 0; 158 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE]; 159 bool genbit = READ_ONCE(net->nft.gencursor); 160 struct nft_traceinfo info; 161 162 info.trace = false; 163 if (static_branch_unlikely(&nft_trace_enabled)) 164 nft_trace_init(&info, pkt, ®s.verdict, basechain); 165 do_chain: 166 if (genbit) 167 rules = rcu_dereference(chain->rules_gen_1); 168 else 169 rules = rcu_dereference(chain->rules_gen_0); 170 171 next_rule: 172 rule = *rules; 173 regs.verdict.code = NFT_CONTINUE; 174 for (; *rules ; rules++) { 175 rule = *rules; 176 nft_rule_for_each_expr(expr, last, rule) { 177 if (expr->ops == &nft_cmp_fast_ops) 178 nft_cmp_fast_eval(expr, ®s); 179 else if (expr->ops != &nft_payload_fast_ops || 180 !nft_payload_fast_eval(expr, ®s, pkt)) 181 expr_call_ops_eval(expr, ®s, pkt); 182 183 if (regs.verdict.code != NFT_CONTINUE) 184 break; 185 } 186 187 switch (regs.verdict.code) { 188 case NFT_BREAK: 189 regs.verdict.code = NFT_CONTINUE; 190 continue; 191 case NFT_CONTINUE: 192 nft_trace_packet(&info, chain, rule, 193 NFT_TRACETYPE_RULE); 194 continue; 195 } 196 break; 197 } 198 199 switch (regs.verdict.code & NF_VERDICT_MASK) { 200 case NF_ACCEPT: 201 case NF_DROP: 202 case NF_QUEUE: 203 case NF_STOLEN: 204 nft_trace_packet(&info, chain, rule, 205 NFT_TRACETYPE_RULE); 206 return regs.verdict.code; 207 } 208 209 switch (regs.verdict.code) { 210 case NFT_JUMP: 211 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE)) 212 return NF_DROP; 213 jumpstack[stackptr].chain = chain; 214 jumpstack[stackptr].rules = rules + 1; 215 stackptr++; 216 fallthrough; 217 case NFT_GOTO: 218 nft_trace_packet(&info, chain, rule, 219 NFT_TRACETYPE_RULE); 220 221 chain = regs.verdict.chain; 222 goto do_chain; 223 case NFT_CONTINUE: 224 case NFT_RETURN: 225 nft_trace_packet(&info, chain, rule, 226 NFT_TRACETYPE_RETURN); 227 break; 228 default: 229 WARN_ON(1); 230 } 231 232 if (stackptr > 0) { 233 stackptr--; 234 chain = jumpstack[stackptr].chain; 235 rules = jumpstack[stackptr].rules; 236 goto next_rule; 237 } 238 239 nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY); 240 241 if (static_branch_unlikely(&nft_counters_enabled)) 242 nft_update_chain_stats(basechain, pkt); 243 244 return nft_base_chain(basechain)->policy; 245 } 246 EXPORT_SYMBOL_GPL(nft_do_chain); 247 248 static struct nft_expr_type *nft_basic_types[] = { 249 &nft_imm_type, 250 &nft_cmp_type, 251 &nft_lookup_type, 252 &nft_bitwise_type, 253 &nft_byteorder_type, 254 &nft_payload_type, 255 &nft_dynset_type, 256 &nft_range_type, 257 &nft_meta_type, 258 &nft_rt_type, 259 &nft_exthdr_type, 260 }; 261 262 static struct nft_object_type *nft_basic_objects[] = { 263 #ifdef CONFIG_NETWORK_SECMARK 264 &nft_secmark_obj_type, 265 #endif 266 }; 267 268 int __init nf_tables_core_module_init(void) 269 { 270 int err, i, j = 0; 271 272 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) { 273 err = nft_register_obj(nft_basic_objects[i]); 274 if (err) 275 goto err; 276 } 277 278 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) { 279 err = nft_register_expr(nft_basic_types[j]); 280 if (err) 281 goto err; 282 } 283 284 return 0; 285 286 err: 287 while (j-- > 0) 288 nft_unregister_expr(nft_basic_types[j]); 289 290 while (i-- > 0) 291 nft_unregister_obj(nft_basic_objects[i]); 292 293 return err; 294 } 295 296 void nf_tables_core_module_exit(void) 297 { 298 int i; 299 300 i = ARRAY_SIZE(nft_basic_types); 301 while (i-- > 0) 302 nft_unregister_expr(nft_basic_types[i]); 303 304 i = ARRAY_SIZE(nft_basic_objects); 305 while (i-- > 0) 306 nft_unregister_obj(nft_basic_objects[i]); 307 } 308