1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
4  *
5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
6  */
7 
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/list.h>
12 #include <linux/rculist.h>
13 #include <linux/skbuff.h>
14 #include <linux/netlink.h>
15 #include <linux/netfilter.h>
16 #include <linux/static_key.h>
17 #include <linux/netfilter/nfnetlink.h>
18 #include <linux/netfilter/nf_tables.h>
19 #include <net/netfilter/nf_tables_core.h>
20 #include <net/netfilter/nf_tables.h>
21 #include <net/netfilter/nf_log.h>
22 #include <net/netfilter/nft_meta.h>
23 
24 static noinline void __nft_trace_packet(struct nft_traceinfo *info,
25 					const struct nft_chain *chain,
26 					enum nft_trace_types type)
27 {
28 	const struct nft_pktinfo *pkt = info->pkt;
29 
30 	if (!info->trace || !pkt->skb->nf_trace)
31 		return;
32 
33 	info->chain = chain;
34 	info->type = type;
35 
36 	nft_trace_notify(info);
37 }
38 
39 static inline void nft_trace_packet(struct nft_traceinfo *info,
40 				    const struct nft_chain *chain,
41 				    const struct nft_rule *rule,
42 				    enum nft_trace_types type)
43 {
44 	if (static_branch_unlikely(&nft_trace_enabled)) {
45 		info->rule = rule;
46 		__nft_trace_packet(info, chain, type);
47 	}
48 }
49 
50 static void nft_cmp_fast_eval(const struct nft_expr *expr,
51 			      struct nft_regs *regs)
52 {
53 	const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
54 	u32 mask = nft_cmp_fast_mask(priv->len);
55 
56 	if ((regs->data[priv->sreg] & mask) == priv->data)
57 		return;
58 	regs->verdict.code = NFT_BREAK;
59 }
60 
61 static bool nft_payload_fast_eval(const struct nft_expr *expr,
62 				  struct nft_regs *regs,
63 				  const struct nft_pktinfo *pkt)
64 {
65 	const struct nft_payload *priv = nft_expr_priv(expr);
66 	const struct sk_buff *skb = pkt->skb;
67 	u32 *dest = &regs->data[priv->dreg];
68 	unsigned char *ptr;
69 
70 	if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
71 		ptr = skb_network_header(skb);
72 	else {
73 		if (!pkt->tprot_set)
74 			return false;
75 		ptr = skb_network_header(skb) + pkt->xt.thoff;
76 	}
77 
78 	ptr += priv->offset;
79 
80 	if (unlikely(ptr + priv->len > skb_tail_pointer(skb)))
81 		return false;
82 
83 	*dest = 0;
84 	if (priv->len == 2)
85 		*(u16 *)dest = *(u16 *)ptr;
86 	else if (priv->len == 4)
87 		*(u32 *)dest = *(u32 *)ptr;
88 	else
89 		*(u8 *)dest = *(u8 *)ptr;
90 	return true;
91 }
92 
93 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled);
94 
95 static noinline void nft_update_chain_stats(const struct nft_chain *chain,
96 					    const struct nft_pktinfo *pkt)
97 {
98 	struct nft_base_chain *base_chain;
99 	struct nft_stats __percpu *pstats;
100 	struct nft_stats *stats;
101 
102 	base_chain = nft_base_chain(chain);
103 
104 	rcu_read_lock();
105 	pstats = READ_ONCE(base_chain->stats);
106 	if (pstats) {
107 		local_bh_disable();
108 		stats = this_cpu_ptr(pstats);
109 		u64_stats_update_begin(&stats->syncp);
110 		stats->pkts++;
111 		stats->bytes += pkt->skb->len;
112 		u64_stats_update_end(&stats->syncp);
113 		local_bh_enable();
114 	}
115 	rcu_read_unlock();
116 }
117 
118 struct nft_jumpstack {
119 	const struct nft_chain	*chain;
120 	struct nft_rule	*const *rules;
121 };
122 
123 static void expr_call_ops_eval(const struct nft_expr *expr,
124 			       struct nft_regs *regs,
125 			       struct nft_pktinfo *pkt)
126 {
127 #ifdef CONFIG_RETPOLINE
128 	unsigned long e = (unsigned long)expr->ops->eval;
129 #define X(e, fun) \
130 	do { if ((e) == (unsigned long)(fun)) \
131 		return fun(expr, regs, pkt); } while (0)
132 
133 	X(e, nft_payload_eval);
134 	X(e, nft_cmp_eval);
135 	X(e, nft_meta_get_eval);
136 	X(e, nft_lookup_eval);
137 	X(e, nft_range_eval);
138 	X(e, nft_immediate_eval);
139 	X(e, nft_byteorder_eval);
140 	X(e, nft_dynset_eval);
141 	X(e, nft_rt_get_eval);
142 	X(e, nft_bitwise_eval);
143 #undef  X
144 #endif /* CONFIG_RETPOLINE */
145 	expr->ops->eval(expr, regs, pkt);
146 }
147 
148 unsigned int
149 nft_do_chain(struct nft_pktinfo *pkt, void *priv)
150 {
151 	const struct nft_chain *chain = priv, *basechain = chain;
152 	const struct net *net = nft_net(pkt);
153 	struct nft_rule *const *rules;
154 	const struct nft_rule *rule;
155 	const struct nft_expr *expr, *last;
156 	struct nft_regs regs;
157 	unsigned int stackptr = 0;
158 	struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
159 	bool genbit = READ_ONCE(net->nft.gencursor);
160 	struct nft_traceinfo info;
161 
162 	info.trace = false;
163 	if (static_branch_unlikely(&nft_trace_enabled))
164 		nft_trace_init(&info, pkt, &regs.verdict, basechain);
165 do_chain:
166 	if (genbit)
167 		rules = rcu_dereference(chain->rules_gen_1);
168 	else
169 		rules = rcu_dereference(chain->rules_gen_0);
170 
171 next_rule:
172 	rule = *rules;
173 	regs.verdict.code = NFT_CONTINUE;
174 	for (; *rules ; rules++) {
175 		rule = *rules;
176 		nft_rule_for_each_expr(expr, last, rule) {
177 			if (expr->ops == &nft_cmp_fast_ops)
178 				nft_cmp_fast_eval(expr, &regs);
179 			else if (expr->ops != &nft_payload_fast_ops ||
180 				 !nft_payload_fast_eval(expr, &regs, pkt))
181 				expr_call_ops_eval(expr, &regs, pkt);
182 
183 			if (regs.verdict.code != NFT_CONTINUE)
184 				break;
185 		}
186 
187 		switch (regs.verdict.code) {
188 		case NFT_BREAK:
189 			regs.verdict.code = NFT_CONTINUE;
190 			continue;
191 		case NFT_CONTINUE:
192 			nft_trace_packet(&info, chain, rule,
193 					 NFT_TRACETYPE_RULE);
194 			continue;
195 		}
196 		break;
197 	}
198 
199 	switch (regs.verdict.code & NF_VERDICT_MASK) {
200 	case NF_ACCEPT:
201 	case NF_DROP:
202 	case NF_QUEUE:
203 	case NF_STOLEN:
204 		nft_trace_packet(&info, chain, rule,
205 				 NFT_TRACETYPE_RULE);
206 		return regs.verdict.code;
207 	}
208 
209 	switch (regs.verdict.code) {
210 	case NFT_JUMP:
211 		if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
212 			return NF_DROP;
213 		jumpstack[stackptr].chain = chain;
214 		jumpstack[stackptr].rules = rules + 1;
215 		stackptr++;
216 		fallthrough;
217 	case NFT_GOTO:
218 		nft_trace_packet(&info, chain, rule,
219 				 NFT_TRACETYPE_RULE);
220 
221 		chain = regs.verdict.chain;
222 		goto do_chain;
223 	case NFT_CONTINUE:
224 	case NFT_RETURN:
225 		nft_trace_packet(&info, chain, rule,
226 				 NFT_TRACETYPE_RETURN);
227 		break;
228 	default:
229 		WARN_ON(1);
230 	}
231 
232 	if (stackptr > 0) {
233 		stackptr--;
234 		chain = jumpstack[stackptr].chain;
235 		rules = jumpstack[stackptr].rules;
236 		goto next_rule;
237 	}
238 
239 	nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY);
240 
241 	if (static_branch_unlikely(&nft_counters_enabled))
242 		nft_update_chain_stats(basechain, pkt);
243 
244 	return nft_base_chain(basechain)->policy;
245 }
246 EXPORT_SYMBOL_GPL(nft_do_chain);
247 
248 static struct nft_expr_type *nft_basic_types[] = {
249 	&nft_imm_type,
250 	&nft_cmp_type,
251 	&nft_lookup_type,
252 	&nft_bitwise_type,
253 	&nft_byteorder_type,
254 	&nft_payload_type,
255 	&nft_dynset_type,
256 	&nft_range_type,
257 	&nft_meta_type,
258 	&nft_rt_type,
259 	&nft_exthdr_type,
260 };
261 
262 static struct nft_object_type *nft_basic_objects[] = {
263 #ifdef CONFIG_NETWORK_SECMARK
264 	&nft_secmark_obj_type,
265 #endif
266 };
267 
268 int __init nf_tables_core_module_init(void)
269 {
270 	int err, i, j = 0;
271 
272 	for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) {
273 		err = nft_register_obj(nft_basic_objects[i]);
274 		if (err)
275 			goto err;
276 	}
277 
278 	for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) {
279 		err = nft_register_expr(nft_basic_types[j]);
280 		if (err)
281 			goto err;
282 	}
283 
284 	return 0;
285 
286 err:
287 	while (j-- > 0)
288 		nft_unregister_expr(nft_basic_types[j]);
289 
290 	while (i-- > 0)
291 		nft_unregister_obj(nft_basic_objects[i]);
292 
293 	return err;
294 }
295 
296 void nf_tables_core_module_exit(void)
297 {
298 	int i;
299 
300 	i = ARRAY_SIZE(nft_basic_types);
301 	while (i-- > 0)
302 		nft_unregister_expr(nft_basic_types[i]);
303 
304 	i = ARRAY_SIZE(nft_basic_objects);
305 	while (i-- > 0)
306 		nft_unregister_obj(nft_basic_objects[i]);
307 }
308