1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008 Patrick McHardy <kaber@trash.net> 4 * 5 * Development of this code funded by Astaro AG (http://www.astaro.com/) 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/init.h> 11 #include <linux/list.h> 12 #include <linux/rculist.h> 13 #include <linux/skbuff.h> 14 #include <linux/netlink.h> 15 #include <linux/netfilter.h> 16 #include <linux/static_key.h> 17 #include <linux/netfilter/nfnetlink.h> 18 #include <linux/netfilter/nf_tables.h> 19 #include <net/netfilter/nf_tables_core.h> 20 #include <net/netfilter/nf_tables.h> 21 #include <net/netfilter/nf_log.h> 22 #include <net/netfilter/nft_meta.h> 23 24 static noinline void __nft_trace_packet(struct nft_traceinfo *info, 25 const struct nft_chain *chain, 26 enum nft_trace_types type) 27 { 28 const struct nft_pktinfo *pkt = info->pkt; 29 30 if (!info->trace || !pkt->skb->nf_trace) 31 return; 32 33 info->chain = chain; 34 info->type = type; 35 36 nft_trace_notify(info); 37 } 38 39 static inline void nft_trace_packet(struct nft_traceinfo *info, 40 const struct nft_chain *chain, 41 const struct nft_rule *rule, 42 enum nft_trace_types type) 43 { 44 if (static_branch_unlikely(&nft_trace_enabled)) { 45 info->rule = rule; 46 __nft_trace_packet(info, chain, type); 47 } 48 } 49 50 static void nft_bitwise_fast_eval(const struct nft_expr *expr, 51 struct nft_regs *regs) 52 { 53 const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr); 54 u32 *src = ®s->data[priv->sreg]; 55 u32 *dst = ®s->data[priv->dreg]; 56 57 *dst = (*src & priv->mask) ^ priv->xor; 58 } 59 60 static void nft_cmp_fast_eval(const struct nft_expr *expr, 61 struct nft_regs *regs) 62 { 63 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr); 64 65 if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv) 66 return; 67 regs->verdict.code = NFT_BREAK; 68 } 69 70 static bool nft_payload_fast_eval(const struct nft_expr *expr, 71 struct nft_regs *regs, 72 const struct nft_pktinfo *pkt) 73 { 74 const struct nft_payload *priv = nft_expr_priv(expr); 75 const struct sk_buff *skb = pkt->skb; 76 u32 *dest = ®s->data[priv->dreg]; 77 unsigned char *ptr; 78 79 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER) 80 ptr = skb_network_header(skb); 81 else { 82 if (!pkt->tprot_set) 83 return false; 84 ptr = skb_network_header(skb) + pkt->xt.thoff; 85 } 86 87 ptr += priv->offset; 88 89 if (unlikely(ptr + priv->len > skb_tail_pointer(skb))) 90 return false; 91 92 *dest = 0; 93 if (priv->len == 2) 94 *(u16 *)dest = *(u16 *)ptr; 95 else if (priv->len == 4) 96 *(u32 *)dest = *(u32 *)ptr; 97 else 98 *(u8 *)dest = *(u8 *)ptr; 99 return true; 100 } 101 102 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled); 103 104 static noinline void nft_update_chain_stats(const struct nft_chain *chain, 105 const struct nft_pktinfo *pkt) 106 { 107 struct nft_base_chain *base_chain; 108 struct nft_stats __percpu *pstats; 109 struct nft_stats *stats; 110 111 base_chain = nft_base_chain(chain); 112 113 rcu_read_lock(); 114 pstats = READ_ONCE(base_chain->stats); 115 if (pstats) { 116 local_bh_disable(); 117 stats = this_cpu_ptr(pstats); 118 u64_stats_update_begin(&stats->syncp); 119 stats->pkts++; 120 stats->bytes += pkt->skb->len; 121 u64_stats_update_end(&stats->syncp); 122 local_bh_enable(); 123 } 124 rcu_read_unlock(); 125 } 126 127 struct nft_jumpstack { 128 const struct nft_chain *chain; 129 struct nft_rule *const *rules; 130 }; 131 132 static void expr_call_ops_eval(const struct nft_expr *expr, 133 struct nft_regs *regs, 134 struct nft_pktinfo *pkt) 135 { 136 #ifdef CONFIG_RETPOLINE 137 unsigned long e = (unsigned long)expr->ops->eval; 138 #define X(e, fun) \ 139 do { if ((e) == (unsigned long)(fun)) \ 140 return fun(expr, regs, pkt); } while (0) 141 142 X(e, nft_payload_eval); 143 X(e, nft_cmp_eval); 144 X(e, nft_meta_get_eval); 145 X(e, nft_lookup_eval); 146 X(e, nft_range_eval); 147 X(e, nft_immediate_eval); 148 X(e, nft_byteorder_eval); 149 X(e, nft_dynset_eval); 150 X(e, nft_rt_get_eval); 151 X(e, nft_bitwise_eval); 152 #undef X 153 #endif /* CONFIG_RETPOLINE */ 154 expr->ops->eval(expr, regs, pkt); 155 } 156 157 unsigned int 158 nft_do_chain(struct nft_pktinfo *pkt, void *priv) 159 { 160 const struct nft_chain *chain = priv, *basechain = chain; 161 const struct net *net = nft_net(pkt); 162 struct nft_rule *const *rules; 163 const struct nft_rule *rule; 164 const struct nft_expr *expr, *last; 165 struct nft_regs regs; 166 unsigned int stackptr = 0; 167 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE]; 168 bool genbit = READ_ONCE(net->nft.gencursor); 169 struct nft_traceinfo info; 170 171 info.trace = false; 172 if (static_branch_unlikely(&nft_trace_enabled)) 173 nft_trace_init(&info, pkt, ®s.verdict, basechain); 174 do_chain: 175 if (genbit) 176 rules = rcu_dereference(chain->rules_gen_1); 177 else 178 rules = rcu_dereference(chain->rules_gen_0); 179 180 next_rule: 181 rule = *rules; 182 regs.verdict.code = NFT_CONTINUE; 183 for (; *rules ; rules++) { 184 rule = *rules; 185 nft_rule_for_each_expr(expr, last, rule) { 186 if (expr->ops == &nft_cmp_fast_ops) 187 nft_cmp_fast_eval(expr, ®s); 188 else if (expr->ops == &nft_bitwise_fast_ops) 189 nft_bitwise_fast_eval(expr, ®s); 190 else if (expr->ops != &nft_payload_fast_ops || 191 !nft_payload_fast_eval(expr, ®s, pkt)) 192 expr_call_ops_eval(expr, ®s, pkt); 193 194 if (regs.verdict.code != NFT_CONTINUE) 195 break; 196 } 197 198 switch (regs.verdict.code) { 199 case NFT_BREAK: 200 regs.verdict.code = NFT_CONTINUE; 201 continue; 202 case NFT_CONTINUE: 203 nft_trace_packet(&info, chain, rule, 204 NFT_TRACETYPE_RULE); 205 continue; 206 } 207 break; 208 } 209 210 switch (regs.verdict.code & NF_VERDICT_MASK) { 211 case NF_ACCEPT: 212 case NF_DROP: 213 case NF_QUEUE: 214 case NF_STOLEN: 215 nft_trace_packet(&info, chain, rule, 216 NFT_TRACETYPE_RULE); 217 return regs.verdict.code; 218 } 219 220 switch (regs.verdict.code) { 221 case NFT_JUMP: 222 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE)) 223 return NF_DROP; 224 jumpstack[stackptr].chain = chain; 225 jumpstack[stackptr].rules = rules + 1; 226 stackptr++; 227 fallthrough; 228 case NFT_GOTO: 229 nft_trace_packet(&info, chain, rule, 230 NFT_TRACETYPE_RULE); 231 232 chain = regs.verdict.chain; 233 goto do_chain; 234 case NFT_CONTINUE: 235 case NFT_RETURN: 236 nft_trace_packet(&info, chain, rule, 237 NFT_TRACETYPE_RETURN); 238 break; 239 default: 240 WARN_ON(1); 241 } 242 243 if (stackptr > 0) { 244 stackptr--; 245 chain = jumpstack[stackptr].chain; 246 rules = jumpstack[stackptr].rules; 247 goto next_rule; 248 } 249 250 nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY); 251 252 if (static_branch_unlikely(&nft_counters_enabled)) 253 nft_update_chain_stats(basechain, pkt); 254 255 return nft_base_chain(basechain)->policy; 256 } 257 EXPORT_SYMBOL_GPL(nft_do_chain); 258 259 static struct nft_expr_type *nft_basic_types[] = { 260 &nft_imm_type, 261 &nft_cmp_type, 262 &nft_lookup_type, 263 &nft_bitwise_type, 264 &nft_byteorder_type, 265 &nft_payload_type, 266 &nft_dynset_type, 267 &nft_range_type, 268 &nft_meta_type, 269 &nft_rt_type, 270 &nft_exthdr_type, 271 }; 272 273 static struct nft_object_type *nft_basic_objects[] = { 274 #ifdef CONFIG_NETWORK_SECMARK 275 &nft_secmark_obj_type, 276 #endif 277 }; 278 279 int __init nf_tables_core_module_init(void) 280 { 281 int err, i, j = 0; 282 283 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) { 284 err = nft_register_obj(nft_basic_objects[i]); 285 if (err) 286 goto err; 287 } 288 289 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) { 290 err = nft_register_expr(nft_basic_types[j]); 291 if (err) 292 goto err; 293 } 294 295 return 0; 296 297 err: 298 while (j-- > 0) 299 nft_unregister_expr(nft_basic_types[j]); 300 301 while (i-- > 0) 302 nft_unregister_obj(nft_basic_objects[i]); 303 304 return err; 305 } 306 307 void nf_tables_core_module_exit(void) 308 { 309 int i; 310 311 i = ARRAY_SIZE(nft_basic_types); 312 while (i-- > 0) 313 nft_unregister_expr(nft_basic_types[i]); 314 315 i = ARRAY_SIZE(nft_basic_objects); 316 while (i-- > 0) 317 nft_unregister_obj(nft_basic_objects[i]); 318 } 319