1 #include <linux/kernel.h>
2 #include <linux/init.h>
3 #include <linux/module.h>
4 #include <linux/netfilter.h>
5 #include <linux/rhashtable.h>
6 #include <linux/netdevice.h>
7 #include <linux/tc_act/tc_csum.h>
8 #include <net/flow_offload.h>
9 #include <net/netfilter/nf_flow_table.h>
10 #include <net/netfilter/nf_tables.h>
11 #include <net/netfilter/nf_conntrack.h>
12 #include <net/netfilter/nf_conntrack_acct.h>
13 #include <net/netfilter/nf_conntrack_core.h>
14 #include <net/netfilter/nf_conntrack_tuple.h>
15 
16 static struct workqueue_struct *nf_flow_offload_add_wq;
17 static struct workqueue_struct *nf_flow_offload_del_wq;
18 static struct workqueue_struct *nf_flow_offload_stats_wq;
19 
20 struct flow_offload_work {
21 	struct list_head	list;
22 	enum flow_cls_command	cmd;
23 	struct nf_flowtable	*flowtable;
24 	struct flow_offload	*flow;
25 	struct work_struct	work;
26 };
27 
28 #define NF_FLOW_DISSECTOR(__match, __type, __field)	\
29 	(__match)->dissector.offset[__type] =		\
30 		offsetof(struct nf_flow_key, __field)
31 
nf_flow_rule_lwt_match(struct nf_flow_match * match,struct ip_tunnel_info * tun_info)32 static void nf_flow_rule_lwt_match(struct nf_flow_match *match,
33 				   struct ip_tunnel_info *tun_info)
34 {
35 	struct nf_flow_key *mask = &match->mask;
36 	struct nf_flow_key *key = &match->key;
37 	unsigned long long enc_keys;
38 
39 	if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX))
40 		return;
41 
42 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_CONTROL, enc_control);
43 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
44 	key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id);
45 	mask->enc_key_id.keyid = 0xffffffff;
46 	enc_keys = BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
47 		   BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL);
48 
49 	if (ip_tunnel_info_af(tun_info) == AF_INET) {
50 		NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
51 				  enc_ipv4);
52 		key->enc_ipv4.src = tun_info->key.u.ipv4.dst;
53 		key->enc_ipv4.dst = tun_info->key.u.ipv4.src;
54 		if (key->enc_ipv4.src)
55 			mask->enc_ipv4.src = 0xffffffff;
56 		if (key->enc_ipv4.dst)
57 			mask->enc_ipv4.dst = 0xffffffff;
58 		enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS);
59 		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
60 	} else {
61 		memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst,
62 		       sizeof(struct in6_addr));
63 		memcpy(&key->enc_ipv6.dst, &tun_info->key.u.ipv6.src,
64 		       sizeof(struct in6_addr));
65 		if (memcmp(&key->enc_ipv6.src, &in6addr_any,
66 			   sizeof(struct in6_addr)))
67 			memset(&mask->enc_ipv6.src, 0xff,
68 			       sizeof(struct in6_addr));
69 		if (memcmp(&key->enc_ipv6.dst, &in6addr_any,
70 			   sizeof(struct in6_addr)))
71 			memset(&mask->enc_ipv6.dst, 0xff,
72 			       sizeof(struct in6_addr));
73 		enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS);
74 		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
75 	}
76 
77 	match->dissector.used_keys |= enc_keys;
78 }
79 
nf_flow_rule_vlan_match(struct flow_dissector_key_vlan * key,struct flow_dissector_key_vlan * mask,u16 vlan_id,__be16 proto)80 static void nf_flow_rule_vlan_match(struct flow_dissector_key_vlan *key,
81 				    struct flow_dissector_key_vlan *mask,
82 				    u16 vlan_id, __be16 proto)
83 {
84 	key->vlan_id = vlan_id;
85 	mask->vlan_id = VLAN_VID_MASK;
86 	key->vlan_tpid = proto;
87 	mask->vlan_tpid = 0xffff;
88 }
89 
nf_flow_rule_match(struct nf_flow_match * match,const struct flow_offload_tuple * tuple,struct dst_entry * other_dst)90 static int nf_flow_rule_match(struct nf_flow_match *match,
91 			      const struct flow_offload_tuple *tuple,
92 			      struct dst_entry *other_dst)
93 {
94 	struct nf_flow_key *mask = &match->mask;
95 	struct nf_flow_key *key = &match->key;
96 	struct ip_tunnel_info *tun_info;
97 	bool vlan_encap = false;
98 
99 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_META, meta);
100 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control);
101 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic);
102 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
103 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
104 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp);
105 	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp);
106 
107 	if (other_dst && other_dst->lwtstate) {
108 		tun_info = lwt_tun_info(other_dst->lwtstate);
109 		nf_flow_rule_lwt_match(match, tun_info);
110 	}
111 
112 	if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_TC)
113 		key->meta.ingress_ifindex = tuple->tc.iifidx;
114 	else
115 		key->meta.ingress_ifindex = tuple->iifidx;
116 
117 	mask->meta.ingress_ifindex = 0xffffffff;
118 
119 	if (tuple->encap_num > 0 && !(tuple->in_vlan_ingress & BIT(0)) &&
120 	    tuple->encap[0].proto == htons(ETH_P_8021Q)) {
121 		NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, vlan);
122 		nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
123 					tuple->encap[0].id,
124 					tuple->encap[0].proto);
125 		vlan_encap = true;
126 	}
127 
128 	if (tuple->encap_num > 1 && !(tuple->in_vlan_ingress & BIT(1)) &&
129 	    tuple->encap[1].proto == htons(ETH_P_8021Q)) {
130 		if (vlan_encap) {
131 			NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CVLAN,
132 					  cvlan);
133 			nf_flow_rule_vlan_match(&key->cvlan, &mask->cvlan,
134 						tuple->encap[1].id,
135 						tuple->encap[1].proto);
136 		} else {
137 			NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN,
138 					  vlan);
139 			nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
140 						tuple->encap[1].id,
141 						tuple->encap[1].proto);
142 		}
143 	}
144 
145 	switch (tuple->l3proto) {
146 	case AF_INET:
147 		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
148 		key->basic.n_proto = htons(ETH_P_IP);
149 		key->ipv4.src = tuple->src_v4.s_addr;
150 		mask->ipv4.src = 0xffffffff;
151 		key->ipv4.dst = tuple->dst_v4.s_addr;
152 		mask->ipv4.dst = 0xffffffff;
153 		break;
154        case AF_INET6:
155 		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
156 		key->basic.n_proto = htons(ETH_P_IPV6);
157 		key->ipv6.src = tuple->src_v6;
158 		memset(&mask->ipv6.src, 0xff, sizeof(mask->ipv6.src));
159 		key->ipv6.dst = tuple->dst_v6;
160 		memset(&mask->ipv6.dst, 0xff, sizeof(mask->ipv6.dst));
161 		break;
162 	default:
163 		return -EOPNOTSUPP;
164 	}
165 	mask->control.addr_type = 0xffff;
166 	match->dissector.used_keys |= BIT_ULL(key->control.addr_type);
167 	mask->basic.n_proto = 0xffff;
168 
169 	switch (tuple->l4proto) {
170 	case IPPROTO_TCP:
171 		key->tcp.flags = 0;
172 		mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16);
173 		match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_TCP);
174 		break;
175 	case IPPROTO_UDP:
176 	case IPPROTO_GRE:
177 		break;
178 	default:
179 		return -EOPNOTSUPP;
180 	}
181 
182 	key->basic.ip_proto = tuple->l4proto;
183 	mask->basic.ip_proto = 0xff;
184 
185 	match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_META) |
186 				      BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
187 				      BIT_ULL(FLOW_DISSECTOR_KEY_BASIC);
188 
189 	switch (tuple->l4proto) {
190 	case IPPROTO_TCP:
191 	case IPPROTO_UDP:
192 		key->tp.src = tuple->src_port;
193 		mask->tp.src = 0xffff;
194 		key->tp.dst = tuple->dst_port;
195 		mask->tp.dst = 0xffff;
196 
197 		match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_PORTS);
198 		break;
199 	}
200 
201 	return 0;
202 }
203 
flow_offload_mangle(struct flow_action_entry * entry,enum flow_action_mangle_base htype,u32 offset,const __be32 * value,const __be32 * mask)204 static void flow_offload_mangle(struct flow_action_entry *entry,
205 				enum flow_action_mangle_base htype, u32 offset,
206 				const __be32 *value, const __be32 *mask)
207 {
208 	entry->id = FLOW_ACTION_MANGLE;
209 	entry->mangle.htype = htype;
210 	entry->mangle.offset = offset;
211 	memcpy(&entry->mangle.mask, mask, sizeof(u32));
212 	memcpy(&entry->mangle.val, value, sizeof(u32));
213 }
214 
215 static inline struct flow_action_entry *
flow_action_entry_next(struct nf_flow_rule * flow_rule)216 flow_action_entry_next(struct nf_flow_rule *flow_rule)
217 {
218 	int i = flow_rule->rule->action.num_entries++;
219 
220 	return &flow_rule->rule->action.entries[i];
221 }
222 
flow_offload_eth_src(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)223 static int flow_offload_eth_src(struct net *net,
224 				const struct flow_offload *flow,
225 				enum flow_offload_tuple_dir dir,
226 				struct nf_flow_rule *flow_rule)
227 {
228 	struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
229 	struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
230 	const struct flow_offload_tuple *other_tuple, *this_tuple;
231 	struct net_device *dev = NULL;
232 	const unsigned char *addr;
233 	u32 mask, val;
234 	u16 val16;
235 
236 	this_tuple = &flow->tuplehash[dir].tuple;
237 
238 	switch (this_tuple->xmit_type) {
239 	case FLOW_OFFLOAD_XMIT_DIRECT:
240 		addr = this_tuple->out.h_source;
241 		break;
242 	case FLOW_OFFLOAD_XMIT_NEIGH:
243 		other_tuple = &flow->tuplehash[!dir].tuple;
244 		dev = dev_get_by_index(net, other_tuple->iifidx);
245 		if (!dev)
246 			return -ENOENT;
247 
248 		addr = dev->dev_addr;
249 		break;
250 	default:
251 		return -EOPNOTSUPP;
252 	}
253 
254 	mask = ~0xffff0000;
255 	memcpy(&val16, addr, 2);
256 	val = val16 << 16;
257 	flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
258 			    &val, &mask);
259 
260 	mask = ~0xffffffff;
261 	memcpy(&val, addr + 2, 4);
262 	flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8,
263 			    &val, &mask);
264 
265 	dev_put(dev);
266 
267 	return 0;
268 }
269 
flow_offload_eth_dst(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)270 static int flow_offload_eth_dst(struct net *net,
271 				const struct flow_offload *flow,
272 				enum flow_offload_tuple_dir dir,
273 				struct nf_flow_rule *flow_rule)
274 {
275 	struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
276 	struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
277 	const struct flow_offload_tuple *other_tuple, *this_tuple;
278 	const struct dst_entry *dst_cache;
279 	unsigned char ha[ETH_ALEN];
280 	struct neighbour *n;
281 	const void *daddr;
282 	u32 mask, val;
283 	u8 nud_state;
284 	u16 val16;
285 
286 	this_tuple = &flow->tuplehash[dir].tuple;
287 
288 	switch (this_tuple->xmit_type) {
289 	case FLOW_OFFLOAD_XMIT_DIRECT:
290 		ether_addr_copy(ha, this_tuple->out.h_dest);
291 		break;
292 	case FLOW_OFFLOAD_XMIT_NEIGH:
293 		other_tuple = &flow->tuplehash[!dir].tuple;
294 		daddr = &other_tuple->src_v4;
295 		dst_cache = this_tuple->dst_cache;
296 		n = dst_neigh_lookup(dst_cache, daddr);
297 		if (!n)
298 			return -ENOENT;
299 
300 		read_lock_bh(&n->lock);
301 		nud_state = n->nud_state;
302 		ether_addr_copy(ha, n->ha);
303 		read_unlock_bh(&n->lock);
304 		neigh_release(n);
305 
306 		if (!(nud_state & NUD_VALID))
307 			return -ENOENT;
308 		break;
309 	default:
310 		return -EOPNOTSUPP;
311 	}
312 
313 	mask = ~0xffffffff;
314 	memcpy(&val, ha, 4);
315 	flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0,
316 			    &val, &mask);
317 
318 	mask = ~0x0000ffff;
319 	memcpy(&val16, ha + 4, 2);
320 	val = val16;
321 	flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
322 			    &val, &mask);
323 
324 	return 0;
325 }
326 
flow_offload_ipv4_snat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)327 static void flow_offload_ipv4_snat(struct net *net,
328 				   const struct flow_offload *flow,
329 				   enum flow_offload_tuple_dir dir,
330 				   struct nf_flow_rule *flow_rule)
331 {
332 	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
333 	u32 mask = ~htonl(0xffffffff);
334 	__be32 addr;
335 	u32 offset;
336 
337 	switch (dir) {
338 	case FLOW_OFFLOAD_DIR_ORIGINAL:
339 		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
340 		offset = offsetof(struct iphdr, saddr);
341 		break;
342 	case FLOW_OFFLOAD_DIR_REPLY:
343 		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
344 		offset = offsetof(struct iphdr, daddr);
345 		break;
346 	default:
347 		return;
348 	}
349 
350 	flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
351 			    &addr, &mask);
352 }
353 
flow_offload_ipv4_dnat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)354 static void flow_offload_ipv4_dnat(struct net *net,
355 				   const struct flow_offload *flow,
356 				   enum flow_offload_tuple_dir dir,
357 				   struct nf_flow_rule *flow_rule)
358 {
359 	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
360 	u32 mask = ~htonl(0xffffffff);
361 	__be32 addr;
362 	u32 offset;
363 
364 	switch (dir) {
365 	case FLOW_OFFLOAD_DIR_ORIGINAL:
366 		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
367 		offset = offsetof(struct iphdr, daddr);
368 		break;
369 	case FLOW_OFFLOAD_DIR_REPLY:
370 		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
371 		offset = offsetof(struct iphdr, saddr);
372 		break;
373 	default:
374 		return;
375 	}
376 
377 	flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
378 			    &addr, &mask);
379 }
380 
flow_offload_ipv6_mangle(struct nf_flow_rule * flow_rule,unsigned int offset,const __be32 * addr,const __be32 * mask)381 static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule,
382 				     unsigned int offset,
383 				     const __be32 *addr, const __be32 *mask)
384 {
385 	struct flow_action_entry *entry;
386 	int i;
387 
388 	for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++) {
389 		entry = flow_action_entry_next(flow_rule);
390 		flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6,
391 				    offset + i * sizeof(u32), &addr[i], mask);
392 	}
393 }
394 
flow_offload_ipv6_snat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)395 static void flow_offload_ipv6_snat(struct net *net,
396 				   const struct flow_offload *flow,
397 				   enum flow_offload_tuple_dir dir,
398 				   struct nf_flow_rule *flow_rule)
399 {
400 	u32 mask = ~htonl(0xffffffff);
401 	const __be32 *addr;
402 	u32 offset;
403 
404 	switch (dir) {
405 	case FLOW_OFFLOAD_DIR_ORIGINAL:
406 		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr32;
407 		offset = offsetof(struct ipv6hdr, saddr);
408 		break;
409 	case FLOW_OFFLOAD_DIR_REPLY:
410 		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr32;
411 		offset = offsetof(struct ipv6hdr, daddr);
412 		break;
413 	default:
414 		return;
415 	}
416 
417 	flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
418 }
419 
flow_offload_ipv6_dnat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)420 static void flow_offload_ipv6_dnat(struct net *net,
421 				   const struct flow_offload *flow,
422 				   enum flow_offload_tuple_dir dir,
423 				   struct nf_flow_rule *flow_rule)
424 {
425 	u32 mask = ~htonl(0xffffffff);
426 	const __be32 *addr;
427 	u32 offset;
428 
429 	switch (dir) {
430 	case FLOW_OFFLOAD_DIR_ORIGINAL:
431 		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr32;
432 		offset = offsetof(struct ipv6hdr, daddr);
433 		break;
434 	case FLOW_OFFLOAD_DIR_REPLY:
435 		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr32;
436 		offset = offsetof(struct ipv6hdr, saddr);
437 		break;
438 	default:
439 		return;
440 	}
441 
442 	flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
443 }
444 
flow_offload_l4proto(const struct flow_offload * flow)445 static int flow_offload_l4proto(const struct flow_offload *flow)
446 {
447 	u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
448 	u8 type = 0;
449 
450 	switch (protonum) {
451 	case IPPROTO_TCP:
452 		type = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
453 		break;
454 	case IPPROTO_UDP:
455 		type = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
456 		break;
457 	default:
458 		break;
459 	}
460 
461 	return type;
462 }
463 
flow_offload_port_snat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)464 static void flow_offload_port_snat(struct net *net,
465 				   const struct flow_offload *flow,
466 				   enum flow_offload_tuple_dir dir,
467 				   struct nf_flow_rule *flow_rule)
468 {
469 	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
470 	u32 mask, port;
471 	u32 offset;
472 
473 	switch (dir) {
474 	case FLOW_OFFLOAD_DIR_ORIGINAL:
475 		port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port);
476 		offset = 0; /* offsetof(struct tcphdr, source); */
477 		port = htonl(port << 16);
478 		mask = ~htonl(0xffff0000);
479 		break;
480 	case FLOW_OFFLOAD_DIR_REPLY:
481 		port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port);
482 		offset = 0; /* offsetof(struct tcphdr, dest); */
483 		port = htonl(port);
484 		mask = ~htonl(0xffff);
485 		break;
486 	default:
487 		return;
488 	}
489 
490 	flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
491 			    &port, &mask);
492 }
493 
flow_offload_port_dnat(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)494 static void flow_offload_port_dnat(struct net *net,
495 				   const struct flow_offload *flow,
496 				   enum flow_offload_tuple_dir dir,
497 				   struct nf_flow_rule *flow_rule)
498 {
499 	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
500 	u32 mask, port;
501 	u32 offset;
502 
503 	switch (dir) {
504 	case FLOW_OFFLOAD_DIR_ORIGINAL:
505 		port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port);
506 		offset = 0; /* offsetof(struct tcphdr, dest); */
507 		port = htonl(port);
508 		mask = ~htonl(0xffff);
509 		break;
510 	case FLOW_OFFLOAD_DIR_REPLY:
511 		port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port);
512 		offset = 0; /* offsetof(struct tcphdr, source); */
513 		port = htonl(port << 16);
514 		mask = ~htonl(0xffff0000);
515 		break;
516 	default:
517 		return;
518 	}
519 
520 	flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
521 			    &port, &mask);
522 }
523 
flow_offload_ipv4_checksum(struct net * net,const struct flow_offload * flow,struct nf_flow_rule * flow_rule)524 static void flow_offload_ipv4_checksum(struct net *net,
525 				       const struct flow_offload *flow,
526 				       struct nf_flow_rule *flow_rule)
527 {
528 	u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
529 	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
530 
531 	entry->id = FLOW_ACTION_CSUM;
532 	entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR;
533 
534 	switch (protonum) {
535 	case IPPROTO_TCP:
536 		entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP;
537 		break;
538 	case IPPROTO_UDP:
539 		entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP;
540 		break;
541 	}
542 }
543 
flow_offload_redirect(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)544 static void flow_offload_redirect(struct net *net,
545 				  const struct flow_offload *flow,
546 				  enum flow_offload_tuple_dir dir,
547 				  struct nf_flow_rule *flow_rule)
548 {
549 	const struct flow_offload_tuple *this_tuple, *other_tuple;
550 	struct flow_action_entry *entry;
551 	struct net_device *dev;
552 	int ifindex;
553 
554 	this_tuple = &flow->tuplehash[dir].tuple;
555 	switch (this_tuple->xmit_type) {
556 	case FLOW_OFFLOAD_XMIT_DIRECT:
557 		this_tuple = &flow->tuplehash[dir].tuple;
558 		ifindex = this_tuple->out.hw_ifidx;
559 		break;
560 	case FLOW_OFFLOAD_XMIT_NEIGH:
561 		other_tuple = &flow->tuplehash[!dir].tuple;
562 		ifindex = other_tuple->iifidx;
563 		break;
564 	default:
565 		return;
566 	}
567 
568 	dev = dev_get_by_index(net, ifindex);
569 	if (!dev)
570 		return;
571 
572 	entry = flow_action_entry_next(flow_rule);
573 	entry->id = FLOW_ACTION_REDIRECT;
574 	entry->dev = dev;
575 }
576 
flow_offload_encap_tunnel(const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)577 static void flow_offload_encap_tunnel(const struct flow_offload *flow,
578 				      enum flow_offload_tuple_dir dir,
579 				      struct nf_flow_rule *flow_rule)
580 {
581 	const struct flow_offload_tuple *this_tuple;
582 	struct flow_action_entry *entry;
583 	struct dst_entry *dst;
584 
585 	this_tuple = &flow->tuplehash[dir].tuple;
586 	if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
587 		return;
588 
589 	dst = this_tuple->dst_cache;
590 	if (dst && dst->lwtstate) {
591 		struct ip_tunnel_info *tun_info;
592 
593 		tun_info = lwt_tun_info(dst->lwtstate);
594 		if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
595 			entry = flow_action_entry_next(flow_rule);
596 			entry->id = FLOW_ACTION_TUNNEL_ENCAP;
597 			entry->tunnel = tun_info;
598 		}
599 	}
600 }
601 
flow_offload_decap_tunnel(const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)602 static void flow_offload_decap_tunnel(const struct flow_offload *flow,
603 				      enum flow_offload_tuple_dir dir,
604 				      struct nf_flow_rule *flow_rule)
605 {
606 	const struct flow_offload_tuple *other_tuple;
607 	struct flow_action_entry *entry;
608 	struct dst_entry *dst;
609 
610 	other_tuple = &flow->tuplehash[!dir].tuple;
611 	if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
612 		return;
613 
614 	dst = other_tuple->dst_cache;
615 	if (dst && dst->lwtstate) {
616 		struct ip_tunnel_info *tun_info;
617 
618 		tun_info = lwt_tun_info(dst->lwtstate);
619 		if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
620 			entry = flow_action_entry_next(flow_rule);
621 			entry->id = FLOW_ACTION_TUNNEL_DECAP;
622 		}
623 	}
624 }
625 
626 static int
nf_flow_rule_route_common(struct net * net,const struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)627 nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow,
628 			  enum flow_offload_tuple_dir dir,
629 			  struct nf_flow_rule *flow_rule)
630 {
631 	const struct flow_offload_tuple *other_tuple;
632 	const struct flow_offload_tuple *tuple;
633 	int i;
634 
635 	flow_offload_decap_tunnel(flow, dir, flow_rule);
636 	flow_offload_encap_tunnel(flow, dir, flow_rule);
637 
638 	if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 ||
639 	    flow_offload_eth_dst(net, flow, dir, flow_rule) < 0)
640 		return -1;
641 
642 	tuple = &flow->tuplehash[dir].tuple;
643 
644 	for (i = 0; i < tuple->encap_num; i++) {
645 		struct flow_action_entry *entry;
646 
647 		if (tuple->in_vlan_ingress & BIT(i))
648 			continue;
649 
650 		if (tuple->encap[i].proto == htons(ETH_P_8021Q)) {
651 			entry = flow_action_entry_next(flow_rule);
652 			entry->id = FLOW_ACTION_VLAN_POP;
653 		}
654 	}
655 
656 	other_tuple = &flow->tuplehash[!dir].tuple;
657 
658 	for (i = 0; i < other_tuple->encap_num; i++) {
659 		struct flow_action_entry *entry;
660 
661 		if (other_tuple->in_vlan_ingress & BIT(i))
662 			continue;
663 
664 		entry = flow_action_entry_next(flow_rule);
665 
666 		switch (other_tuple->encap[i].proto) {
667 		case htons(ETH_P_PPP_SES):
668 			entry->id = FLOW_ACTION_PPPOE_PUSH;
669 			entry->pppoe.sid = other_tuple->encap[i].id;
670 			break;
671 		case htons(ETH_P_8021Q):
672 			entry->id = FLOW_ACTION_VLAN_PUSH;
673 			entry->vlan.vid = other_tuple->encap[i].id;
674 			entry->vlan.proto = other_tuple->encap[i].proto;
675 			break;
676 		}
677 	}
678 
679 	return 0;
680 }
681 
nf_flow_rule_route_ipv4(struct net * net,struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)682 int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow,
683 			    enum flow_offload_tuple_dir dir,
684 			    struct nf_flow_rule *flow_rule)
685 {
686 	if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
687 		return -1;
688 
689 	if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
690 		flow_offload_ipv4_snat(net, flow, dir, flow_rule);
691 		flow_offload_port_snat(net, flow, dir, flow_rule);
692 	}
693 	if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
694 		flow_offload_ipv4_dnat(net, flow, dir, flow_rule);
695 		flow_offload_port_dnat(net, flow, dir, flow_rule);
696 	}
697 	if (test_bit(NF_FLOW_SNAT, &flow->flags) ||
698 	    test_bit(NF_FLOW_DNAT, &flow->flags))
699 		flow_offload_ipv4_checksum(net, flow, flow_rule);
700 
701 	flow_offload_redirect(net, flow, dir, flow_rule);
702 
703 	return 0;
704 }
705 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4);
706 
nf_flow_rule_route_ipv6(struct net * net,struct flow_offload * flow,enum flow_offload_tuple_dir dir,struct nf_flow_rule * flow_rule)707 int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow,
708 			    enum flow_offload_tuple_dir dir,
709 			    struct nf_flow_rule *flow_rule)
710 {
711 	if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
712 		return -1;
713 
714 	if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
715 		flow_offload_ipv6_snat(net, flow, dir, flow_rule);
716 		flow_offload_port_snat(net, flow, dir, flow_rule);
717 	}
718 	if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
719 		flow_offload_ipv6_dnat(net, flow, dir, flow_rule);
720 		flow_offload_port_dnat(net, flow, dir, flow_rule);
721 	}
722 
723 	flow_offload_redirect(net, flow, dir, flow_rule);
724 
725 	return 0;
726 }
727 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6);
728 
729 #define NF_FLOW_RULE_ACTION_MAX	16
730 
731 static struct nf_flow_rule *
nf_flow_offload_rule_alloc(struct net * net,const struct flow_offload_work * offload,enum flow_offload_tuple_dir dir)732 nf_flow_offload_rule_alloc(struct net *net,
733 			   const struct flow_offload_work *offload,
734 			   enum flow_offload_tuple_dir dir)
735 {
736 	const struct nf_flowtable *flowtable = offload->flowtable;
737 	const struct flow_offload_tuple *tuple, *other_tuple;
738 	struct flow_offload *flow = offload->flow;
739 	struct dst_entry *other_dst = NULL;
740 	struct nf_flow_rule *flow_rule;
741 	int err = -ENOMEM;
742 
743 	flow_rule = kzalloc(sizeof(*flow_rule), GFP_KERNEL);
744 	if (!flow_rule)
745 		goto err_flow;
746 
747 	flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX);
748 	if (!flow_rule->rule)
749 		goto err_flow_rule;
750 
751 	flow_rule->rule->match.dissector = &flow_rule->match.dissector;
752 	flow_rule->rule->match.mask = &flow_rule->match.mask;
753 	flow_rule->rule->match.key = &flow_rule->match.key;
754 
755 	tuple = &flow->tuplehash[dir].tuple;
756 	other_tuple = &flow->tuplehash[!dir].tuple;
757 	if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH)
758 		other_dst = other_tuple->dst_cache;
759 
760 	err = nf_flow_rule_match(&flow_rule->match, tuple, other_dst);
761 	if (err < 0)
762 		goto err_flow_match;
763 
764 	flow_rule->rule->action.num_entries = 0;
765 	if (flowtable->type->action(net, flow, dir, flow_rule) < 0)
766 		goto err_flow_match;
767 
768 	return flow_rule;
769 
770 err_flow_match:
771 	kfree(flow_rule->rule);
772 err_flow_rule:
773 	kfree(flow_rule);
774 err_flow:
775 	return NULL;
776 }
777 
__nf_flow_offload_destroy(struct nf_flow_rule * flow_rule)778 static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule)
779 {
780 	struct flow_action_entry *entry;
781 	int i;
782 
783 	for (i = 0; i < flow_rule->rule->action.num_entries; i++) {
784 		entry = &flow_rule->rule->action.entries[i];
785 		if (entry->id != FLOW_ACTION_REDIRECT)
786 			continue;
787 
788 		dev_put(entry->dev);
789 	}
790 	kfree(flow_rule->rule);
791 	kfree(flow_rule);
792 }
793 
nf_flow_offload_destroy(struct nf_flow_rule * flow_rule[])794 static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[])
795 {
796 	int i;
797 
798 	for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++)
799 		__nf_flow_offload_destroy(flow_rule[i]);
800 }
801 
nf_flow_offload_alloc(const struct flow_offload_work * offload,struct nf_flow_rule * flow_rule[])802 static int nf_flow_offload_alloc(const struct flow_offload_work *offload,
803 				 struct nf_flow_rule *flow_rule[])
804 {
805 	struct net *net = read_pnet(&offload->flowtable->net);
806 
807 	flow_rule[0] = nf_flow_offload_rule_alloc(net, offload,
808 						  FLOW_OFFLOAD_DIR_ORIGINAL);
809 	if (!flow_rule[0])
810 		return -ENOMEM;
811 
812 	flow_rule[1] = nf_flow_offload_rule_alloc(net, offload,
813 						  FLOW_OFFLOAD_DIR_REPLY);
814 	if (!flow_rule[1]) {
815 		__nf_flow_offload_destroy(flow_rule[0]);
816 		return -ENOMEM;
817 	}
818 
819 	return 0;
820 }
821 
nf_flow_offload_init(struct flow_cls_offload * cls_flow,__be16 proto,int priority,enum flow_cls_command cmd,const struct flow_offload_tuple * tuple,struct netlink_ext_ack * extack)822 static void nf_flow_offload_init(struct flow_cls_offload *cls_flow,
823 				 __be16 proto, int priority,
824 				 enum flow_cls_command cmd,
825 				 const struct flow_offload_tuple *tuple,
826 				 struct netlink_ext_ack *extack)
827 {
828 	cls_flow->common.protocol = proto;
829 	cls_flow->common.prio = priority;
830 	cls_flow->common.extack = extack;
831 	cls_flow->command = cmd;
832 	cls_flow->cookie = (unsigned long)tuple;
833 }
834 
nf_flow_offload_tuple(struct nf_flowtable * flowtable,struct flow_offload * flow,struct nf_flow_rule * flow_rule,enum flow_offload_tuple_dir dir,int priority,int cmd,struct flow_stats * stats,struct list_head * block_cb_list)835 static int nf_flow_offload_tuple(struct nf_flowtable *flowtable,
836 				 struct flow_offload *flow,
837 				 struct nf_flow_rule *flow_rule,
838 				 enum flow_offload_tuple_dir dir,
839 				 int priority, int cmd,
840 				 struct flow_stats *stats,
841 				 struct list_head *block_cb_list)
842 {
843 	struct flow_cls_offload cls_flow = {};
844 	struct netlink_ext_ack extack = {};
845 	struct flow_block_cb *block_cb;
846 	__be16 proto = ETH_P_ALL;
847 	int err, i = 0;
848 
849 	nf_flow_offload_init(&cls_flow, proto, priority, cmd,
850 			     &flow->tuplehash[dir].tuple, &extack);
851 	if (cmd == FLOW_CLS_REPLACE)
852 		cls_flow.rule = flow_rule->rule;
853 
854 	down_read(&flowtable->flow_block_lock);
855 	list_for_each_entry(block_cb, block_cb_list, list) {
856 		err = block_cb->cb(TC_SETUP_CLSFLOWER, &cls_flow,
857 				   block_cb->cb_priv);
858 		if (err < 0)
859 			continue;
860 
861 		i++;
862 	}
863 	up_read(&flowtable->flow_block_lock);
864 
865 	if (cmd == FLOW_CLS_STATS)
866 		memcpy(stats, &cls_flow.stats, sizeof(*stats));
867 
868 	return i;
869 }
870 
flow_offload_tuple_add(struct flow_offload_work * offload,struct nf_flow_rule * flow_rule,enum flow_offload_tuple_dir dir)871 static int flow_offload_tuple_add(struct flow_offload_work *offload,
872 				  struct nf_flow_rule *flow_rule,
873 				  enum flow_offload_tuple_dir dir)
874 {
875 	return nf_flow_offload_tuple(offload->flowtable, offload->flow,
876 				     flow_rule, dir,
877 				     offload->flowtable->priority,
878 				     FLOW_CLS_REPLACE, NULL,
879 				     &offload->flowtable->flow_block.cb_list);
880 }
881 
flow_offload_tuple_del(struct flow_offload_work * offload,enum flow_offload_tuple_dir dir)882 static void flow_offload_tuple_del(struct flow_offload_work *offload,
883 				   enum flow_offload_tuple_dir dir)
884 {
885 	nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
886 			      offload->flowtable->priority,
887 			      FLOW_CLS_DESTROY, NULL,
888 			      &offload->flowtable->flow_block.cb_list);
889 }
890 
flow_offload_rule_add(struct flow_offload_work * offload,struct nf_flow_rule * flow_rule[])891 static int flow_offload_rule_add(struct flow_offload_work *offload,
892 				 struct nf_flow_rule *flow_rule[])
893 {
894 	int ok_count = 0;
895 
896 	ok_count += flow_offload_tuple_add(offload, flow_rule[0],
897 					   FLOW_OFFLOAD_DIR_ORIGINAL);
898 	if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
899 		ok_count += flow_offload_tuple_add(offload, flow_rule[1],
900 						   FLOW_OFFLOAD_DIR_REPLY);
901 	if (ok_count == 0)
902 		return -ENOENT;
903 
904 	return 0;
905 }
906 
flow_offload_work_add(struct flow_offload_work * offload)907 static void flow_offload_work_add(struct flow_offload_work *offload)
908 {
909 	struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX];
910 	int err;
911 
912 	err = nf_flow_offload_alloc(offload, flow_rule);
913 	if (err < 0)
914 		return;
915 
916 	err = flow_offload_rule_add(offload, flow_rule);
917 	if (err < 0)
918 		goto out;
919 
920 	set_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
921 
922 out:
923 	nf_flow_offload_destroy(flow_rule);
924 }
925 
flow_offload_work_del(struct flow_offload_work * offload)926 static void flow_offload_work_del(struct flow_offload_work *offload)
927 {
928 	clear_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
929 	flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL);
930 	if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
931 		flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY);
932 	set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags);
933 }
934 
flow_offload_tuple_stats(struct flow_offload_work * offload,enum flow_offload_tuple_dir dir,struct flow_stats * stats)935 static void flow_offload_tuple_stats(struct flow_offload_work *offload,
936 				     enum flow_offload_tuple_dir dir,
937 				     struct flow_stats *stats)
938 {
939 	nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
940 			      offload->flowtable->priority,
941 			      FLOW_CLS_STATS, stats,
942 			      &offload->flowtable->flow_block.cb_list);
943 }
944 
flow_offload_work_stats(struct flow_offload_work * offload)945 static void flow_offload_work_stats(struct flow_offload_work *offload)
946 {
947 	struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {};
948 	u64 lastused;
949 
950 	flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]);
951 	if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
952 		flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY,
953 					 &stats[1]);
954 
955 	lastused = max_t(u64, stats[0].lastused, stats[1].lastused);
956 	offload->flow->timeout = max_t(u64, offload->flow->timeout,
957 				       lastused + flow_offload_get_timeout(offload->flow));
958 
959 	if (offload->flowtable->flags & NF_FLOWTABLE_COUNTER) {
960 		if (stats[0].pkts)
961 			nf_ct_acct_add(offload->flow->ct,
962 				       FLOW_OFFLOAD_DIR_ORIGINAL,
963 				       stats[0].pkts, stats[0].bytes);
964 		if (stats[1].pkts)
965 			nf_ct_acct_add(offload->flow->ct,
966 				       FLOW_OFFLOAD_DIR_REPLY,
967 				       stats[1].pkts, stats[1].bytes);
968 	}
969 }
970 
flow_offload_work_handler(struct work_struct * work)971 static void flow_offload_work_handler(struct work_struct *work)
972 {
973 	struct flow_offload_work *offload;
974 	struct net *net;
975 
976 	offload = container_of(work, struct flow_offload_work, work);
977 	net = read_pnet(&offload->flowtable->net);
978 	switch (offload->cmd) {
979 		case FLOW_CLS_REPLACE:
980 			flow_offload_work_add(offload);
981 			NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_add);
982 			break;
983 		case FLOW_CLS_DESTROY:
984 			flow_offload_work_del(offload);
985 			NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_del);
986 			break;
987 		case FLOW_CLS_STATS:
988 			flow_offload_work_stats(offload);
989 			NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_stats);
990 			break;
991 		default:
992 			WARN_ON_ONCE(1);
993 	}
994 
995 	clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags);
996 	kfree(offload);
997 }
998 
flow_offload_queue_work(struct flow_offload_work * offload)999 static void flow_offload_queue_work(struct flow_offload_work *offload)
1000 {
1001 	struct net *net = read_pnet(&offload->flowtable->net);
1002 
1003 	if (offload->cmd == FLOW_CLS_REPLACE) {
1004 		NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_add);
1005 		queue_work(nf_flow_offload_add_wq, &offload->work);
1006 	} else if (offload->cmd == FLOW_CLS_DESTROY) {
1007 		NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_del);
1008 		queue_work(nf_flow_offload_del_wq, &offload->work);
1009 	} else {
1010 		NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_stats);
1011 		queue_work(nf_flow_offload_stats_wq, &offload->work);
1012 	}
1013 }
1014 
1015 static struct flow_offload_work *
nf_flow_offload_work_alloc(struct nf_flowtable * flowtable,struct flow_offload * flow,unsigned int cmd)1016 nf_flow_offload_work_alloc(struct nf_flowtable *flowtable,
1017 			   struct flow_offload *flow, unsigned int cmd)
1018 {
1019 	struct flow_offload_work *offload;
1020 
1021 	if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags))
1022 		return NULL;
1023 
1024 	offload = kmalloc(sizeof(struct flow_offload_work), GFP_ATOMIC);
1025 	if (!offload) {
1026 		clear_bit(NF_FLOW_HW_PENDING, &flow->flags);
1027 		return NULL;
1028 	}
1029 
1030 	offload->cmd = cmd;
1031 	offload->flow = flow;
1032 	offload->flowtable = flowtable;
1033 	INIT_WORK(&offload->work, flow_offload_work_handler);
1034 
1035 	return offload;
1036 }
1037 
1038 
nf_flow_offload_add(struct nf_flowtable * flowtable,struct flow_offload * flow)1039 void nf_flow_offload_add(struct nf_flowtable *flowtable,
1040 			 struct flow_offload *flow)
1041 {
1042 	struct flow_offload_work *offload;
1043 
1044 	offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_REPLACE);
1045 	if (!offload)
1046 		return;
1047 
1048 	flow_offload_queue_work(offload);
1049 }
1050 
nf_flow_offload_del(struct nf_flowtable * flowtable,struct flow_offload * flow)1051 void nf_flow_offload_del(struct nf_flowtable *flowtable,
1052 			 struct flow_offload *flow)
1053 {
1054 	struct flow_offload_work *offload;
1055 
1056 	offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_DESTROY);
1057 	if (!offload)
1058 		return;
1059 
1060 	set_bit(NF_FLOW_HW_DYING, &flow->flags);
1061 	flow_offload_queue_work(offload);
1062 }
1063 
nf_flow_offload_stats(struct nf_flowtable * flowtable,struct flow_offload * flow)1064 void nf_flow_offload_stats(struct nf_flowtable *flowtable,
1065 			   struct flow_offload *flow)
1066 {
1067 	struct flow_offload_work *offload;
1068 	__s32 delta;
1069 
1070 	delta = nf_flow_timeout_delta(flow->timeout);
1071 	if ((delta >= (9 * flow_offload_get_timeout(flow)) / 10))
1072 		return;
1073 
1074 	offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_STATS);
1075 	if (!offload)
1076 		return;
1077 
1078 	flow_offload_queue_work(offload);
1079 }
1080 
nf_flow_table_offload_flush_cleanup(struct nf_flowtable * flowtable)1081 void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable)
1082 {
1083 	if (nf_flowtable_hw_offload(flowtable)) {
1084 		flush_workqueue(nf_flow_offload_del_wq);
1085 		nf_flow_table_gc_run(flowtable);
1086 	}
1087 }
1088 
nf_flow_table_offload_flush(struct nf_flowtable * flowtable)1089 void nf_flow_table_offload_flush(struct nf_flowtable *flowtable)
1090 {
1091 	if (nf_flowtable_hw_offload(flowtable)) {
1092 		flush_workqueue(nf_flow_offload_add_wq);
1093 		flush_workqueue(nf_flow_offload_del_wq);
1094 		flush_workqueue(nf_flow_offload_stats_wq);
1095 	}
1096 }
1097 
nf_flow_table_block_setup(struct nf_flowtable * flowtable,struct flow_block_offload * bo,enum flow_block_command cmd)1098 static int nf_flow_table_block_setup(struct nf_flowtable *flowtable,
1099 				     struct flow_block_offload *bo,
1100 				     enum flow_block_command cmd)
1101 {
1102 	struct flow_block_cb *block_cb, *next;
1103 	int err = 0;
1104 
1105 	down_write(&flowtable->flow_block_lock);
1106 	switch (cmd) {
1107 	case FLOW_BLOCK_BIND:
1108 		list_splice(&bo->cb_list, &flowtable->flow_block.cb_list);
1109 		break;
1110 	case FLOW_BLOCK_UNBIND:
1111 		list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1112 			list_del(&block_cb->list);
1113 			flow_block_cb_free(block_cb);
1114 		}
1115 		break;
1116 	default:
1117 		WARN_ON_ONCE(1);
1118 		err = -EOPNOTSUPP;
1119 	}
1120 	up_write(&flowtable->flow_block_lock);
1121 
1122 	return err;
1123 }
1124 
nf_flow_table_block_offload_init(struct flow_block_offload * bo,struct net * net,enum flow_block_command cmd,struct nf_flowtable * flowtable,struct netlink_ext_ack * extack)1125 static void nf_flow_table_block_offload_init(struct flow_block_offload *bo,
1126 					     struct net *net,
1127 					     enum flow_block_command cmd,
1128 					     struct nf_flowtable *flowtable,
1129 					     struct netlink_ext_ack *extack)
1130 {
1131 	memset(bo, 0, sizeof(*bo));
1132 	bo->net		= net;
1133 	bo->block	= &flowtable->flow_block;
1134 	bo->command	= cmd;
1135 	bo->binder_type	= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
1136 	bo->extack	= extack;
1137 	bo->cb_list_head = &flowtable->flow_block.cb_list;
1138 	INIT_LIST_HEAD(&bo->cb_list);
1139 }
1140 
nf_flow_table_indr_cleanup(struct flow_block_cb * block_cb)1141 static void nf_flow_table_indr_cleanup(struct flow_block_cb *block_cb)
1142 {
1143 	struct nf_flowtable *flowtable = block_cb->indr.data;
1144 	struct net_device *dev = block_cb->indr.dev;
1145 
1146 	nf_flow_table_gc_cleanup(flowtable, dev);
1147 	down_write(&flowtable->flow_block_lock);
1148 	list_del(&block_cb->list);
1149 	list_del(&block_cb->driver_list);
1150 	flow_block_cb_free(block_cb);
1151 	up_write(&flowtable->flow_block_lock);
1152 }
1153 
nf_flow_table_indr_offload_cmd(struct flow_block_offload * bo,struct nf_flowtable * flowtable,struct net_device * dev,enum flow_block_command cmd,struct netlink_ext_ack * extack)1154 static int nf_flow_table_indr_offload_cmd(struct flow_block_offload *bo,
1155 					  struct nf_flowtable *flowtable,
1156 					  struct net_device *dev,
1157 					  enum flow_block_command cmd,
1158 					  struct netlink_ext_ack *extack)
1159 {
1160 	nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
1161 					 extack);
1162 
1163 	return flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_FT, flowtable, bo,
1164 					   nf_flow_table_indr_cleanup);
1165 }
1166 
nf_flow_table_offload_cmd(struct flow_block_offload * bo,struct nf_flowtable * flowtable,struct net_device * dev,enum flow_block_command cmd,struct netlink_ext_ack * extack)1167 static int nf_flow_table_offload_cmd(struct flow_block_offload *bo,
1168 				     struct nf_flowtable *flowtable,
1169 				     struct net_device *dev,
1170 				     enum flow_block_command cmd,
1171 				     struct netlink_ext_ack *extack)
1172 {
1173 	int err;
1174 
1175 	nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
1176 					 extack);
1177 	down_write(&flowtable->flow_block_lock);
1178 	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_FT, bo);
1179 	up_write(&flowtable->flow_block_lock);
1180 	if (err < 0)
1181 		return err;
1182 
1183 	return 0;
1184 }
1185 
nf_flow_table_offload_setup(struct nf_flowtable * flowtable,struct net_device * dev,enum flow_block_command cmd)1186 int nf_flow_table_offload_setup(struct nf_flowtable *flowtable,
1187 				struct net_device *dev,
1188 				enum flow_block_command cmd)
1189 {
1190 	struct netlink_ext_ack extack = {};
1191 	struct flow_block_offload bo;
1192 	int err;
1193 
1194 	if (!nf_flowtable_hw_offload(flowtable))
1195 		return 0;
1196 
1197 	if (dev->netdev_ops->ndo_setup_tc)
1198 		err = nf_flow_table_offload_cmd(&bo, flowtable, dev, cmd,
1199 						&extack);
1200 	else
1201 		err = nf_flow_table_indr_offload_cmd(&bo, flowtable, dev, cmd,
1202 						     &extack);
1203 	if (err < 0)
1204 		return err;
1205 
1206 	return nf_flow_table_block_setup(flowtable, &bo, cmd);
1207 }
1208 EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup);
1209 
nf_flow_table_offload_init(void)1210 int nf_flow_table_offload_init(void)
1211 {
1212 	nf_flow_offload_add_wq  = alloc_workqueue("nf_ft_offload_add",
1213 						  WQ_UNBOUND | WQ_SYSFS, 0);
1214 	if (!nf_flow_offload_add_wq)
1215 		return -ENOMEM;
1216 
1217 	nf_flow_offload_del_wq  = alloc_workqueue("nf_ft_offload_del",
1218 						  WQ_UNBOUND | WQ_SYSFS, 0);
1219 	if (!nf_flow_offload_del_wq)
1220 		goto err_del_wq;
1221 
1222 	nf_flow_offload_stats_wq  = alloc_workqueue("nf_ft_offload_stats",
1223 						    WQ_UNBOUND | WQ_SYSFS, 0);
1224 	if (!nf_flow_offload_stats_wq)
1225 		goto err_stats_wq;
1226 
1227 	return 0;
1228 
1229 err_stats_wq:
1230 	destroy_workqueue(nf_flow_offload_del_wq);
1231 err_del_wq:
1232 	destroy_workqueue(nf_flow_offload_add_wq);
1233 	return -ENOMEM;
1234 }
1235 
nf_flow_table_offload_exit(void)1236 void nf_flow_table_offload_exit(void)
1237 {
1238 	destroy_workqueue(nf_flow_offload_add_wq);
1239 	destroy_workqueue(nf_flow_offload_del_wq);
1240 	destroy_workqueue(nf_flow_offload_stats_wq);
1241 }
1242