1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008 Patrick McHardy <kaber@trash.net> 4 * 5 * Development of this code funded by Astaro AG (http://www.astaro.com/) 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/init.h> 11 #include <linux/list.h> 12 #include <linux/rculist.h> 13 #include <linux/skbuff.h> 14 #include <linux/netlink.h> 15 #include <linux/netfilter.h> 16 #include <linux/static_key.h> 17 #include <linux/netfilter/nfnetlink.h> 18 #include <linux/netfilter/nf_tables.h> 19 #include <net/netfilter/nf_tables_core.h> 20 #include <net/netfilter/nf_tables.h> 21 #include <net/netfilter/nf_log.h> 22 #include <net/netfilter/nft_meta.h> 23 24 static noinline void __nft_trace_packet(struct nft_traceinfo *info, 25 const struct nft_chain *chain, 26 enum nft_trace_types type) 27 { 28 if (!info->trace || !info->nf_trace) 29 return; 30 31 info->chain = chain; 32 info->type = type; 33 34 nft_trace_notify(info); 35 } 36 37 static inline void nft_trace_packet(struct nft_traceinfo *info, 38 const struct nft_chain *chain, 39 const struct nft_rule_dp *rule, 40 enum nft_trace_types type) 41 { 42 if (static_branch_unlikely(&nft_trace_enabled)) { 43 const struct nft_pktinfo *pkt = info->pkt; 44 45 info->nf_trace = pkt->skb->nf_trace; 46 info->rule = rule; 47 __nft_trace_packet(info, chain, type); 48 } 49 } 50 51 static inline void nft_trace_copy_nftrace(struct nft_traceinfo *info) 52 { 53 if (static_branch_unlikely(&nft_trace_enabled)) { 54 const struct nft_pktinfo *pkt = info->pkt; 55 56 if (info->trace) 57 info->nf_trace = pkt->skb->nf_trace; 58 } 59 } 60 61 static void nft_bitwise_fast_eval(const struct nft_expr *expr, 62 struct nft_regs *regs) 63 { 64 const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr); 65 u32 *src = ®s->data[priv->sreg]; 66 u32 *dst = ®s->data[priv->dreg]; 67 68 *dst = (*src & priv->mask) ^ priv->xor; 69 } 70 71 static void nft_cmp_fast_eval(const struct nft_expr *expr, 72 struct nft_regs *regs) 73 { 74 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr); 75 76 if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv) 77 return; 78 regs->verdict.code = NFT_BREAK; 79 } 80 81 static void nft_cmp16_fast_eval(const struct nft_expr *expr, 82 struct nft_regs *regs) 83 { 84 const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr); 85 const u64 *reg_data = (const u64 *)®s->data[priv->sreg]; 86 const u64 *mask = (const u64 *)&priv->mask; 87 const u64 *data = (const u64 *)&priv->data; 88 89 if (((reg_data[0] & mask[0]) == data[0] && 90 ((reg_data[1] & mask[1]) == data[1])) ^ priv->inv) 91 return; 92 regs->verdict.code = NFT_BREAK; 93 } 94 95 static noinline void __nft_trace_verdict(struct nft_traceinfo *info, 96 const struct nft_chain *chain, 97 const struct nft_regs *regs) 98 { 99 const struct nft_pktinfo *pkt = info->pkt; 100 enum nft_trace_types type; 101 102 switch (regs->verdict.code) { 103 case NFT_CONTINUE: 104 case NFT_RETURN: 105 type = NFT_TRACETYPE_RETURN; 106 break; 107 case NF_STOLEN: 108 type = NFT_TRACETYPE_RULE; 109 /* can't access skb->nf_trace; use copy */ 110 break; 111 default: 112 type = NFT_TRACETYPE_RULE; 113 info->nf_trace = pkt->skb->nf_trace; 114 break; 115 } 116 117 __nft_trace_packet(info, chain, type); 118 } 119 120 static inline void nft_trace_verdict(struct nft_traceinfo *info, 121 const struct nft_chain *chain, 122 const struct nft_rule_dp *rule, 123 const struct nft_regs *regs) 124 { 125 if (static_branch_unlikely(&nft_trace_enabled)) { 126 info->rule = rule; 127 __nft_trace_verdict(info, chain, regs); 128 } 129 } 130 131 static bool nft_payload_fast_eval(const struct nft_expr *expr, 132 struct nft_regs *regs, 133 const struct nft_pktinfo *pkt) 134 { 135 const struct nft_payload *priv = nft_expr_priv(expr); 136 const struct sk_buff *skb = pkt->skb; 137 u32 *dest = ®s->data[priv->dreg]; 138 unsigned char *ptr; 139 140 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER) 141 ptr = skb_network_header(skb); 142 else { 143 if (!(pkt->flags & NFT_PKTINFO_L4PROTO)) 144 return false; 145 ptr = skb_network_header(skb) + nft_thoff(pkt); 146 } 147 148 ptr += priv->offset; 149 150 if (unlikely(ptr + priv->len > skb_tail_pointer(skb))) 151 return false; 152 153 *dest = 0; 154 if (priv->len == 2) 155 *(u16 *)dest = *(u16 *)ptr; 156 else if (priv->len == 4) 157 *(u32 *)dest = *(u32 *)ptr; 158 else 159 *(u8 *)dest = *(u8 *)ptr; 160 return true; 161 } 162 163 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled); 164 165 static noinline void nft_update_chain_stats(const struct nft_chain *chain, 166 const struct nft_pktinfo *pkt) 167 { 168 struct nft_base_chain *base_chain; 169 struct nft_stats __percpu *pstats; 170 struct nft_stats *stats; 171 172 base_chain = nft_base_chain(chain); 173 174 pstats = READ_ONCE(base_chain->stats); 175 if (pstats) { 176 local_bh_disable(); 177 stats = this_cpu_ptr(pstats); 178 u64_stats_update_begin(&stats->syncp); 179 stats->pkts++; 180 stats->bytes += pkt->skb->len; 181 u64_stats_update_end(&stats->syncp); 182 local_bh_enable(); 183 } 184 } 185 186 struct nft_jumpstack { 187 const struct nft_chain *chain; 188 const struct nft_rule_dp *rule; 189 const struct nft_rule_dp *last_rule; 190 }; 191 192 static void expr_call_ops_eval(const struct nft_expr *expr, 193 struct nft_regs *regs, 194 struct nft_pktinfo *pkt) 195 { 196 #ifdef CONFIG_RETPOLINE 197 unsigned long e = (unsigned long)expr->ops->eval; 198 #define X(e, fun) \ 199 do { if ((e) == (unsigned long)(fun)) \ 200 return fun(expr, regs, pkt); } while (0) 201 202 X(e, nft_payload_eval); 203 X(e, nft_cmp_eval); 204 X(e, nft_counter_eval); 205 X(e, nft_meta_get_eval); 206 X(e, nft_lookup_eval); 207 X(e, nft_range_eval); 208 X(e, nft_immediate_eval); 209 X(e, nft_byteorder_eval); 210 X(e, nft_dynset_eval); 211 X(e, nft_rt_get_eval); 212 X(e, nft_bitwise_eval); 213 #undef X 214 #endif /* CONFIG_RETPOLINE */ 215 expr->ops->eval(expr, regs, pkt); 216 } 217 218 #define nft_rule_expr_first(rule) (struct nft_expr *)&rule->data[0] 219 #define nft_rule_expr_next(expr) ((void *)expr) + expr->ops->size 220 #define nft_rule_expr_last(rule) (struct nft_expr *)&rule->data[rule->dlen] 221 #define nft_rule_next(rule) (void *)rule + sizeof(*rule) + rule->dlen 222 223 #define nft_rule_dp_for_each_expr(expr, last, rule) \ 224 for ((expr) = nft_rule_expr_first(rule), (last) = nft_rule_expr_last(rule); \ 225 (expr) != (last); \ 226 (expr) = nft_rule_expr_next(expr)) 227 228 unsigned int 229 nft_do_chain(struct nft_pktinfo *pkt, void *priv) 230 { 231 const struct nft_chain *chain = priv, *basechain = chain; 232 const struct nft_rule_dp *rule, *last_rule; 233 const struct net *net = nft_net(pkt); 234 const struct nft_expr *expr, *last; 235 struct nft_regs regs = {}; 236 unsigned int stackptr = 0; 237 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE]; 238 bool genbit = READ_ONCE(net->nft.gencursor); 239 struct nft_rule_blob *blob; 240 struct nft_traceinfo info; 241 242 info.trace = false; 243 if (static_branch_unlikely(&nft_trace_enabled)) 244 nft_trace_init(&info, pkt, ®s.verdict, basechain); 245 do_chain: 246 if (genbit) 247 blob = rcu_dereference(chain->blob_gen_1); 248 else 249 blob = rcu_dereference(chain->blob_gen_0); 250 251 rule = (struct nft_rule_dp *)blob->data; 252 last_rule = (void *)blob->data + blob->size; 253 next_rule: 254 regs.verdict.code = NFT_CONTINUE; 255 for (; rule < last_rule; rule = nft_rule_next(rule)) { 256 nft_rule_dp_for_each_expr(expr, last, rule) { 257 if (expr->ops == &nft_cmp_fast_ops) 258 nft_cmp_fast_eval(expr, ®s); 259 else if (expr->ops == &nft_cmp16_fast_ops) 260 nft_cmp16_fast_eval(expr, ®s); 261 else if (expr->ops == &nft_bitwise_fast_ops) 262 nft_bitwise_fast_eval(expr, ®s); 263 else if (expr->ops != &nft_payload_fast_ops || 264 !nft_payload_fast_eval(expr, ®s, pkt)) 265 expr_call_ops_eval(expr, ®s, pkt); 266 267 if (regs.verdict.code != NFT_CONTINUE) 268 break; 269 } 270 271 switch (regs.verdict.code) { 272 case NFT_BREAK: 273 regs.verdict.code = NFT_CONTINUE; 274 nft_trace_copy_nftrace(&info); 275 continue; 276 case NFT_CONTINUE: 277 nft_trace_packet(&info, chain, rule, 278 NFT_TRACETYPE_RULE); 279 continue; 280 } 281 break; 282 } 283 284 nft_trace_verdict(&info, chain, rule, ®s); 285 286 switch (regs.verdict.code & NF_VERDICT_MASK) { 287 case NF_ACCEPT: 288 case NF_DROP: 289 case NF_QUEUE: 290 case NF_STOLEN: 291 return regs.verdict.code; 292 } 293 294 switch (regs.verdict.code) { 295 case NFT_JUMP: 296 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE)) 297 return NF_DROP; 298 jumpstack[stackptr].chain = chain; 299 jumpstack[stackptr].rule = nft_rule_next(rule); 300 jumpstack[stackptr].last_rule = last_rule; 301 stackptr++; 302 fallthrough; 303 case NFT_GOTO: 304 chain = regs.verdict.chain; 305 goto do_chain; 306 case NFT_CONTINUE: 307 case NFT_RETURN: 308 break; 309 default: 310 WARN_ON_ONCE(1); 311 } 312 313 if (stackptr > 0) { 314 stackptr--; 315 chain = jumpstack[stackptr].chain; 316 rule = jumpstack[stackptr].rule; 317 last_rule = jumpstack[stackptr].last_rule; 318 goto next_rule; 319 } 320 321 nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY); 322 323 if (static_branch_unlikely(&nft_counters_enabled)) 324 nft_update_chain_stats(basechain, pkt); 325 326 return nft_base_chain(basechain)->policy; 327 } 328 EXPORT_SYMBOL_GPL(nft_do_chain); 329 330 static struct nft_expr_type *nft_basic_types[] = { 331 &nft_imm_type, 332 &nft_cmp_type, 333 &nft_lookup_type, 334 &nft_bitwise_type, 335 &nft_byteorder_type, 336 &nft_payload_type, 337 &nft_dynset_type, 338 &nft_range_type, 339 &nft_meta_type, 340 &nft_rt_type, 341 &nft_exthdr_type, 342 &nft_last_type, 343 &nft_counter_type, 344 }; 345 346 static struct nft_object_type *nft_basic_objects[] = { 347 #ifdef CONFIG_NETWORK_SECMARK 348 &nft_secmark_obj_type, 349 #endif 350 &nft_counter_obj_type, 351 }; 352 353 int __init nf_tables_core_module_init(void) 354 { 355 int err, i, j = 0; 356 357 nft_counter_init_seqcount(); 358 359 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) { 360 err = nft_register_obj(nft_basic_objects[i]); 361 if (err) 362 goto err; 363 } 364 365 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) { 366 err = nft_register_expr(nft_basic_types[j]); 367 if (err) 368 goto err; 369 } 370 371 return 0; 372 373 err: 374 while (j-- > 0) 375 nft_unregister_expr(nft_basic_types[j]); 376 377 while (i-- > 0) 378 nft_unregister_obj(nft_basic_objects[i]); 379 380 return err; 381 } 382 383 void nf_tables_core_module_exit(void) 384 { 385 int i; 386 387 i = ARRAY_SIZE(nft_basic_types); 388 while (i-- > 0) 389 nft_unregister_expr(nft_basic_types[i]); 390 391 i = ARRAY_SIZE(nft_basic_objects); 392 while (i-- > 0) 393 nft_unregister_obj(nft_basic_objects[i]); 394 } 395