xref: /openbmc/linux/net/sched/cls_api.c (revision b8265621)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * net/sched/cls_api.c	Packet classifier API.
4  *
5  * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6  *
7  * Changes:
8  *
9  * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
10  */
11 
12 #include <linux/module.h>
13 #include <linux/types.h>
14 #include <linux/kernel.h>
15 #include <linux/string.h>
16 #include <linux/errno.h>
17 #include <linux/err.h>
18 #include <linux/skbuff.h>
19 #include <linux/init.h>
20 #include <linux/kmod.h>
21 #include <linux/slab.h>
22 #include <linux/idr.h>
23 #include <linux/jhash.h>
24 #include <linux/rculist.h>
25 #include <net/net_namespace.h>
26 #include <net/sock.h>
27 #include <net/netlink.h>
28 #include <net/pkt_sched.h>
29 #include <net/pkt_cls.h>
30 #include <net/tc_act/tc_pedit.h>
31 #include <net/tc_act/tc_mirred.h>
32 #include <net/tc_act/tc_vlan.h>
33 #include <net/tc_act/tc_tunnel_key.h>
34 #include <net/tc_act/tc_csum.h>
35 #include <net/tc_act/tc_gact.h>
36 #include <net/tc_act/tc_police.h>
37 #include <net/tc_act/tc_sample.h>
38 #include <net/tc_act/tc_skbedit.h>
39 #include <net/tc_act/tc_ct.h>
40 #include <net/tc_act/tc_mpls.h>
41 #include <net/tc_act/tc_gate.h>
42 #include <net/flow_offload.h>
43 
44 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
45 
46 /* The list of all installed classifier types */
47 static LIST_HEAD(tcf_proto_base);
48 
49 /* Protects list of registered TC modules. It is pure SMP lock. */
50 static DEFINE_RWLOCK(cls_mod_lock);
51 
52 static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
53 {
54 	return jhash_3words(tp->chain->index, tp->prio,
55 			    (__force __u32)tp->protocol, 0);
56 }
57 
58 static void tcf_proto_signal_destroying(struct tcf_chain *chain,
59 					struct tcf_proto *tp)
60 {
61 	struct tcf_block *block = chain->block;
62 
63 	mutex_lock(&block->proto_destroy_lock);
64 	hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
65 		     destroy_obj_hashfn(tp));
66 	mutex_unlock(&block->proto_destroy_lock);
67 }
68 
69 static bool tcf_proto_cmp(const struct tcf_proto *tp1,
70 			  const struct tcf_proto *tp2)
71 {
72 	return tp1->chain->index == tp2->chain->index &&
73 	       tp1->prio == tp2->prio &&
74 	       tp1->protocol == tp2->protocol;
75 }
76 
77 static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
78 					struct tcf_proto *tp)
79 {
80 	u32 hash = destroy_obj_hashfn(tp);
81 	struct tcf_proto *iter;
82 	bool found = false;
83 
84 	rcu_read_lock();
85 	hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
86 				   destroy_ht_node, hash) {
87 		if (tcf_proto_cmp(tp, iter)) {
88 			found = true;
89 			break;
90 		}
91 	}
92 	rcu_read_unlock();
93 
94 	return found;
95 }
96 
97 static void
98 tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
99 {
100 	struct tcf_block *block = chain->block;
101 
102 	mutex_lock(&block->proto_destroy_lock);
103 	if (hash_hashed(&tp->destroy_ht_node))
104 		hash_del_rcu(&tp->destroy_ht_node);
105 	mutex_unlock(&block->proto_destroy_lock);
106 }
107 
108 /* Find classifier type by string name */
109 
110 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
111 {
112 	const struct tcf_proto_ops *t, *res = NULL;
113 
114 	if (kind) {
115 		read_lock(&cls_mod_lock);
116 		list_for_each_entry(t, &tcf_proto_base, head) {
117 			if (strcmp(kind, t->kind) == 0) {
118 				if (try_module_get(t->owner))
119 					res = t;
120 				break;
121 			}
122 		}
123 		read_unlock(&cls_mod_lock);
124 	}
125 	return res;
126 }
127 
128 static const struct tcf_proto_ops *
129 tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
130 		     struct netlink_ext_ack *extack)
131 {
132 	const struct tcf_proto_ops *ops;
133 
134 	ops = __tcf_proto_lookup_ops(kind);
135 	if (ops)
136 		return ops;
137 #ifdef CONFIG_MODULES
138 	if (rtnl_held)
139 		rtnl_unlock();
140 	request_module("cls_%s", kind);
141 	if (rtnl_held)
142 		rtnl_lock();
143 	ops = __tcf_proto_lookup_ops(kind);
144 	/* We dropped the RTNL semaphore in order to perform
145 	 * the module load. So, even if we succeeded in loading
146 	 * the module we have to replay the request. We indicate
147 	 * this using -EAGAIN.
148 	 */
149 	if (ops) {
150 		module_put(ops->owner);
151 		return ERR_PTR(-EAGAIN);
152 	}
153 #endif
154 	NL_SET_ERR_MSG(extack, "TC classifier not found");
155 	return ERR_PTR(-ENOENT);
156 }
157 
158 /* Register(unregister) new classifier type */
159 
160 int register_tcf_proto_ops(struct tcf_proto_ops *ops)
161 {
162 	struct tcf_proto_ops *t;
163 	int rc = -EEXIST;
164 
165 	write_lock(&cls_mod_lock);
166 	list_for_each_entry(t, &tcf_proto_base, head)
167 		if (!strcmp(ops->kind, t->kind))
168 			goto out;
169 
170 	list_add_tail(&ops->head, &tcf_proto_base);
171 	rc = 0;
172 out:
173 	write_unlock(&cls_mod_lock);
174 	return rc;
175 }
176 EXPORT_SYMBOL(register_tcf_proto_ops);
177 
178 static struct workqueue_struct *tc_filter_wq;
179 
180 int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
181 {
182 	struct tcf_proto_ops *t;
183 	int rc = -ENOENT;
184 
185 	/* Wait for outstanding call_rcu()s, if any, from a
186 	 * tcf_proto_ops's destroy() handler.
187 	 */
188 	rcu_barrier();
189 	flush_workqueue(tc_filter_wq);
190 
191 	write_lock(&cls_mod_lock);
192 	list_for_each_entry(t, &tcf_proto_base, head) {
193 		if (t == ops) {
194 			list_del(&t->head);
195 			rc = 0;
196 			break;
197 		}
198 	}
199 	write_unlock(&cls_mod_lock);
200 	return rc;
201 }
202 EXPORT_SYMBOL(unregister_tcf_proto_ops);
203 
204 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
205 {
206 	INIT_RCU_WORK(rwork, func);
207 	return queue_rcu_work(tc_filter_wq, rwork);
208 }
209 EXPORT_SYMBOL(tcf_queue_work);
210 
211 /* Select new prio value from the range, managed by kernel. */
212 
213 static inline u32 tcf_auto_prio(struct tcf_proto *tp)
214 {
215 	u32 first = TC_H_MAKE(0xC0000000U, 0U);
216 
217 	if (tp)
218 		first = tp->prio - 1;
219 
220 	return TC_H_MAJ(first);
221 }
222 
223 static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
224 {
225 	if (kind)
226 		return nla_strlcpy(name, kind, IFNAMSIZ) >= IFNAMSIZ;
227 	memset(name, 0, IFNAMSIZ);
228 	return false;
229 }
230 
231 static bool tcf_proto_is_unlocked(const char *kind)
232 {
233 	const struct tcf_proto_ops *ops;
234 	bool ret;
235 
236 	if (strlen(kind) == 0)
237 		return false;
238 
239 	ops = tcf_proto_lookup_ops(kind, false, NULL);
240 	/* On error return false to take rtnl lock. Proto lookup/create
241 	 * functions will perform lookup again and properly handle errors.
242 	 */
243 	if (IS_ERR(ops))
244 		return false;
245 
246 	ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
247 	module_put(ops->owner);
248 	return ret;
249 }
250 
251 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
252 					  u32 prio, struct tcf_chain *chain,
253 					  bool rtnl_held,
254 					  struct netlink_ext_ack *extack)
255 {
256 	struct tcf_proto *tp;
257 	int err;
258 
259 	tp = kzalloc(sizeof(*tp), GFP_KERNEL);
260 	if (!tp)
261 		return ERR_PTR(-ENOBUFS);
262 
263 	tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
264 	if (IS_ERR(tp->ops)) {
265 		err = PTR_ERR(tp->ops);
266 		goto errout;
267 	}
268 	tp->classify = tp->ops->classify;
269 	tp->protocol = protocol;
270 	tp->prio = prio;
271 	tp->chain = chain;
272 	spin_lock_init(&tp->lock);
273 	refcount_set(&tp->refcnt, 1);
274 
275 	err = tp->ops->init(tp);
276 	if (err) {
277 		module_put(tp->ops->owner);
278 		goto errout;
279 	}
280 	return tp;
281 
282 errout:
283 	kfree(tp);
284 	return ERR_PTR(err);
285 }
286 
287 static void tcf_proto_get(struct tcf_proto *tp)
288 {
289 	refcount_inc(&tp->refcnt);
290 }
291 
292 static void tcf_chain_put(struct tcf_chain *chain);
293 
294 static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
295 			      bool sig_destroy, struct netlink_ext_ack *extack)
296 {
297 	tp->ops->destroy(tp, rtnl_held, extack);
298 	if (sig_destroy)
299 		tcf_proto_signal_destroyed(tp->chain, tp);
300 	tcf_chain_put(tp->chain);
301 	module_put(tp->ops->owner);
302 	kfree_rcu(tp, rcu);
303 }
304 
305 static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
306 			  struct netlink_ext_ack *extack)
307 {
308 	if (refcount_dec_and_test(&tp->refcnt))
309 		tcf_proto_destroy(tp, rtnl_held, true, extack);
310 }
311 
312 static bool tcf_proto_check_delete(struct tcf_proto *tp)
313 {
314 	if (tp->ops->delete_empty)
315 		return tp->ops->delete_empty(tp);
316 
317 	tp->deleting = true;
318 	return tp->deleting;
319 }
320 
321 static void tcf_proto_mark_delete(struct tcf_proto *tp)
322 {
323 	spin_lock(&tp->lock);
324 	tp->deleting = true;
325 	spin_unlock(&tp->lock);
326 }
327 
328 static bool tcf_proto_is_deleting(struct tcf_proto *tp)
329 {
330 	bool deleting;
331 
332 	spin_lock(&tp->lock);
333 	deleting = tp->deleting;
334 	spin_unlock(&tp->lock);
335 
336 	return deleting;
337 }
338 
339 #define ASSERT_BLOCK_LOCKED(block)					\
340 	lockdep_assert_held(&(block)->lock)
341 
342 struct tcf_filter_chain_list_item {
343 	struct list_head list;
344 	tcf_chain_head_change_t *chain_head_change;
345 	void *chain_head_change_priv;
346 };
347 
348 static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
349 					  u32 chain_index)
350 {
351 	struct tcf_chain *chain;
352 
353 	ASSERT_BLOCK_LOCKED(block);
354 
355 	chain = kzalloc(sizeof(*chain), GFP_KERNEL);
356 	if (!chain)
357 		return NULL;
358 	list_add_tail_rcu(&chain->list, &block->chain_list);
359 	mutex_init(&chain->filter_chain_lock);
360 	chain->block = block;
361 	chain->index = chain_index;
362 	chain->refcnt = 1;
363 	if (!chain->index)
364 		block->chain0.chain = chain;
365 	return chain;
366 }
367 
368 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
369 				       struct tcf_proto *tp_head)
370 {
371 	if (item->chain_head_change)
372 		item->chain_head_change(tp_head, item->chain_head_change_priv);
373 }
374 
375 static void tcf_chain0_head_change(struct tcf_chain *chain,
376 				   struct tcf_proto *tp_head)
377 {
378 	struct tcf_filter_chain_list_item *item;
379 	struct tcf_block *block = chain->block;
380 
381 	if (chain->index)
382 		return;
383 
384 	mutex_lock(&block->lock);
385 	list_for_each_entry(item, &block->chain0.filter_chain_list, list)
386 		tcf_chain_head_change_item(item, tp_head);
387 	mutex_unlock(&block->lock);
388 }
389 
390 /* Returns true if block can be safely freed. */
391 
392 static bool tcf_chain_detach(struct tcf_chain *chain)
393 {
394 	struct tcf_block *block = chain->block;
395 
396 	ASSERT_BLOCK_LOCKED(block);
397 
398 	list_del_rcu(&chain->list);
399 	if (!chain->index)
400 		block->chain0.chain = NULL;
401 
402 	if (list_empty(&block->chain_list) &&
403 	    refcount_read(&block->refcnt) == 0)
404 		return true;
405 
406 	return false;
407 }
408 
409 static void tcf_block_destroy(struct tcf_block *block)
410 {
411 	mutex_destroy(&block->lock);
412 	mutex_destroy(&block->proto_destroy_lock);
413 	kfree_rcu(block, rcu);
414 }
415 
416 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
417 {
418 	struct tcf_block *block = chain->block;
419 
420 	mutex_destroy(&chain->filter_chain_lock);
421 	kfree_rcu(chain, rcu);
422 	if (free_block)
423 		tcf_block_destroy(block);
424 }
425 
426 static void tcf_chain_hold(struct tcf_chain *chain)
427 {
428 	ASSERT_BLOCK_LOCKED(chain->block);
429 
430 	++chain->refcnt;
431 }
432 
433 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
434 {
435 	ASSERT_BLOCK_LOCKED(chain->block);
436 
437 	/* In case all the references are action references, this
438 	 * chain should not be shown to the user.
439 	 */
440 	return chain->refcnt == chain->action_refcnt;
441 }
442 
443 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
444 					  u32 chain_index)
445 {
446 	struct tcf_chain *chain;
447 
448 	ASSERT_BLOCK_LOCKED(block);
449 
450 	list_for_each_entry(chain, &block->chain_list, list) {
451 		if (chain->index == chain_index)
452 			return chain;
453 	}
454 	return NULL;
455 }
456 
457 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
458 static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block,
459 					      u32 chain_index)
460 {
461 	struct tcf_chain *chain;
462 
463 	list_for_each_entry_rcu(chain, &block->chain_list, list) {
464 		if (chain->index == chain_index)
465 			return chain;
466 	}
467 	return NULL;
468 }
469 #endif
470 
471 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
472 			   u32 seq, u16 flags, int event, bool unicast);
473 
474 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
475 					 u32 chain_index, bool create,
476 					 bool by_act)
477 {
478 	struct tcf_chain *chain = NULL;
479 	bool is_first_reference;
480 
481 	mutex_lock(&block->lock);
482 	chain = tcf_chain_lookup(block, chain_index);
483 	if (chain) {
484 		tcf_chain_hold(chain);
485 	} else {
486 		if (!create)
487 			goto errout;
488 		chain = tcf_chain_create(block, chain_index);
489 		if (!chain)
490 			goto errout;
491 	}
492 
493 	if (by_act)
494 		++chain->action_refcnt;
495 	is_first_reference = chain->refcnt - chain->action_refcnt == 1;
496 	mutex_unlock(&block->lock);
497 
498 	/* Send notification only in case we got the first
499 	 * non-action reference. Until then, the chain acts only as
500 	 * a placeholder for actions pointing to it and user ought
501 	 * not know about them.
502 	 */
503 	if (is_first_reference && !by_act)
504 		tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
505 				RTM_NEWCHAIN, false);
506 
507 	return chain;
508 
509 errout:
510 	mutex_unlock(&block->lock);
511 	return chain;
512 }
513 
514 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
515 				       bool create)
516 {
517 	return __tcf_chain_get(block, chain_index, create, false);
518 }
519 
520 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
521 {
522 	return __tcf_chain_get(block, chain_index, true, true);
523 }
524 EXPORT_SYMBOL(tcf_chain_get_by_act);
525 
526 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
527 			       void *tmplt_priv);
528 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
529 				  void *tmplt_priv, u32 chain_index,
530 				  struct tcf_block *block, struct sk_buff *oskb,
531 				  u32 seq, u16 flags, bool unicast);
532 
533 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
534 			    bool explicitly_created)
535 {
536 	struct tcf_block *block = chain->block;
537 	const struct tcf_proto_ops *tmplt_ops;
538 	bool free_block = false;
539 	unsigned int refcnt;
540 	void *tmplt_priv;
541 
542 	mutex_lock(&block->lock);
543 	if (explicitly_created) {
544 		if (!chain->explicitly_created) {
545 			mutex_unlock(&block->lock);
546 			return;
547 		}
548 		chain->explicitly_created = false;
549 	}
550 
551 	if (by_act)
552 		chain->action_refcnt--;
553 
554 	/* tc_chain_notify_delete can't be called while holding block lock.
555 	 * However, when block is unlocked chain can be changed concurrently, so
556 	 * save these to temporary variables.
557 	 */
558 	refcnt = --chain->refcnt;
559 	tmplt_ops = chain->tmplt_ops;
560 	tmplt_priv = chain->tmplt_priv;
561 
562 	/* The last dropped non-action reference will trigger notification. */
563 	if (refcnt - chain->action_refcnt == 0 && !by_act) {
564 		tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index,
565 				       block, NULL, 0, 0, false);
566 		/* Last reference to chain, no need to lock. */
567 		chain->flushing = false;
568 	}
569 
570 	if (refcnt == 0)
571 		free_block = tcf_chain_detach(chain);
572 	mutex_unlock(&block->lock);
573 
574 	if (refcnt == 0) {
575 		tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
576 		tcf_chain_destroy(chain, free_block);
577 	}
578 }
579 
580 static void tcf_chain_put(struct tcf_chain *chain)
581 {
582 	__tcf_chain_put(chain, false, false);
583 }
584 
585 void tcf_chain_put_by_act(struct tcf_chain *chain)
586 {
587 	__tcf_chain_put(chain, true, false);
588 }
589 EXPORT_SYMBOL(tcf_chain_put_by_act);
590 
591 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
592 {
593 	__tcf_chain_put(chain, false, true);
594 }
595 
596 static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
597 {
598 	struct tcf_proto *tp, *tp_next;
599 
600 	mutex_lock(&chain->filter_chain_lock);
601 	tp = tcf_chain_dereference(chain->filter_chain, chain);
602 	while (tp) {
603 		tp_next = rcu_dereference_protected(tp->next, 1);
604 		tcf_proto_signal_destroying(chain, tp);
605 		tp = tp_next;
606 	}
607 	tp = tcf_chain_dereference(chain->filter_chain, chain);
608 	RCU_INIT_POINTER(chain->filter_chain, NULL);
609 	tcf_chain0_head_change(chain, NULL);
610 	chain->flushing = true;
611 	mutex_unlock(&chain->filter_chain_lock);
612 
613 	while (tp) {
614 		tp_next = rcu_dereference_protected(tp->next, 1);
615 		tcf_proto_put(tp, rtnl_held, NULL);
616 		tp = tp_next;
617 	}
618 }
619 
620 static int tcf_block_setup(struct tcf_block *block,
621 			   struct flow_block_offload *bo);
622 
623 static void tcf_block_offload_init(struct flow_block_offload *bo,
624 				   struct net_device *dev, struct Qdisc *sch,
625 				   enum flow_block_command command,
626 				   enum flow_block_binder_type binder_type,
627 				   struct flow_block *flow_block,
628 				   bool shared, struct netlink_ext_ack *extack)
629 {
630 	bo->net = dev_net(dev);
631 	bo->command = command;
632 	bo->binder_type = binder_type;
633 	bo->block = flow_block;
634 	bo->block_shared = shared;
635 	bo->extack = extack;
636 	bo->sch = sch;
637 	INIT_LIST_HEAD(&bo->cb_list);
638 }
639 
640 static void tcf_block_unbind(struct tcf_block *block,
641 			     struct flow_block_offload *bo);
642 
643 static void tc_block_indr_cleanup(struct flow_block_cb *block_cb)
644 {
645 	struct tcf_block *block = block_cb->indr.data;
646 	struct net_device *dev = block_cb->indr.dev;
647 	struct Qdisc *sch = block_cb->indr.sch;
648 	struct netlink_ext_ack extack = {};
649 	struct flow_block_offload bo;
650 
651 	tcf_block_offload_init(&bo, dev, sch, FLOW_BLOCK_UNBIND,
652 			       block_cb->indr.binder_type,
653 			       &block->flow_block, tcf_block_shared(block),
654 			       &extack);
655 	down_write(&block->cb_lock);
656 	list_del(&block_cb->driver_list);
657 	list_move(&block_cb->list, &bo.cb_list);
658 	up_write(&block->cb_lock);
659 	rtnl_lock();
660 	tcf_block_unbind(block, &bo);
661 	rtnl_unlock();
662 }
663 
664 static bool tcf_block_offload_in_use(struct tcf_block *block)
665 {
666 	return atomic_read(&block->offloadcnt);
667 }
668 
669 static int tcf_block_offload_cmd(struct tcf_block *block,
670 				 struct net_device *dev, struct Qdisc *sch,
671 				 struct tcf_block_ext_info *ei,
672 				 enum flow_block_command command,
673 				 struct netlink_ext_ack *extack)
674 {
675 	struct flow_block_offload bo = {};
676 
677 	tcf_block_offload_init(&bo, dev, sch, command, ei->binder_type,
678 			       &block->flow_block, tcf_block_shared(block),
679 			       extack);
680 
681 	if (dev->netdev_ops->ndo_setup_tc) {
682 		int err;
683 
684 		err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
685 		if (err < 0) {
686 			if (err != -EOPNOTSUPP)
687 				NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed");
688 			return err;
689 		}
690 
691 		return tcf_block_setup(block, &bo);
692 	}
693 
694 	flow_indr_dev_setup_offload(dev, sch, TC_SETUP_BLOCK, block, &bo,
695 				    tc_block_indr_cleanup);
696 	tcf_block_setup(block, &bo);
697 
698 	return -EOPNOTSUPP;
699 }
700 
701 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
702 				  struct tcf_block_ext_info *ei,
703 				  struct netlink_ext_ack *extack)
704 {
705 	struct net_device *dev = q->dev_queue->dev;
706 	int err;
707 
708 	down_write(&block->cb_lock);
709 
710 	/* If tc offload feature is disabled and the block we try to bind
711 	 * to already has some offloaded filters, forbid to bind.
712 	 */
713 	if (dev->netdev_ops->ndo_setup_tc &&
714 	    !tc_can_offload(dev) &&
715 	    tcf_block_offload_in_use(block)) {
716 		NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
717 		err = -EOPNOTSUPP;
718 		goto err_unlock;
719 	}
720 
721 	err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_BIND, extack);
722 	if (err == -EOPNOTSUPP)
723 		goto no_offload_dev_inc;
724 	if (err)
725 		goto err_unlock;
726 
727 	up_write(&block->cb_lock);
728 	return 0;
729 
730 no_offload_dev_inc:
731 	if (tcf_block_offload_in_use(block))
732 		goto err_unlock;
733 
734 	err = 0;
735 	block->nooffloaddevcnt++;
736 err_unlock:
737 	up_write(&block->cb_lock);
738 	return err;
739 }
740 
741 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
742 				     struct tcf_block_ext_info *ei)
743 {
744 	struct net_device *dev = q->dev_queue->dev;
745 	int err;
746 
747 	down_write(&block->cb_lock);
748 	err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_UNBIND, NULL);
749 	if (err == -EOPNOTSUPP)
750 		goto no_offload_dev_dec;
751 	up_write(&block->cb_lock);
752 	return;
753 
754 no_offload_dev_dec:
755 	WARN_ON(block->nooffloaddevcnt-- == 0);
756 	up_write(&block->cb_lock);
757 }
758 
759 static int
760 tcf_chain0_head_change_cb_add(struct tcf_block *block,
761 			      struct tcf_block_ext_info *ei,
762 			      struct netlink_ext_ack *extack)
763 {
764 	struct tcf_filter_chain_list_item *item;
765 	struct tcf_chain *chain0;
766 
767 	item = kmalloc(sizeof(*item), GFP_KERNEL);
768 	if (!item) {
769 		NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
770 		return -ENOMEM;
771 	}
772 	item->chain_head_change = ei->chain_head_change;
773 	item->chain_head_change_priv = ei->chain_head_change_priv;
774 
775 	mutex_lock(&block->lock);
776 	chain0 = block->chain0.chain;
777 	if (chain0)
778 		tcf_chain_hold(chain0);
779 	else
780 		list_add(&item->list, &block->chain0.filter_chain_list);
781 	mutex_unlock(&block->lock);
782 
783 	if (chain0) {
784 		struct tcf_proto *tp_head;
785 
786 		mutex_lock(&chain0->filter_chain_lock);
787 
788 		tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
789 		if (tp_head)
790 			tcf_chain_head_change_item(item, tp_head);
791 
792 		mutex_lock(&block->lock);
793 		list_add(&item->list, &block->chain0.filter_chain_list);
794 		mutex_unlock(&block->lock);
795 
796 		mutex_unlock(&chain0->filter_chain_lock);
797 		tcf_chain_put(chain0);
798 	}
799 
800 	return 0;
801 }
802 
803 static void
804 tcf_chain0_head_change_cb_del(struct tcf_block *block,
805 			      struct tcf_block_ext_info *ei)
806 {
807 	struct tcf_filter_chain_list_item *item;
808 
809 	mutex_lock(&block->lock);
810 	list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
811 		if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
812 		    (item->chain_head_change == ei->chain_head_change &&
813 		     item->chain_head_change_priv == ei->chain_head_change_priv)) {
814 			if (block->chain0.chain)
815 				tcf_chain_head_change_item(item, NULL);
816 			list_del(&item->list);
817 			mutex_unlock(&block->lock);
818 
819 			kfree(item);
820 			return;
821 		}
822 	}
823 	mutex_unlock(&block->lock);
824 	WARN_ON(1);
825 }
826 
827 struct tcf_net {
828 	spinlock_t idr_lock; /* Protects idr */
829 	struct idr idr;
830 };
831 
832 static unsigned int tcf_net_id;
833 
834 static int tcf_block_insert(struct tcf_block *block, struct net *net,
835 			    struct netlink_ext_ack *extack)
836 {
837 	struct tcf_net *tn = net_generic(net, tcf_net_id);
838 	int err;
839 
840 	idr_preload(GFP_KERNEL);
841 	spin_lock(&tn->idr_lock);
842 	err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
843 			    GFP_NOWAIT);
844 	spin_unlock(&tn->idr_lock);
845 	idr_preload_end();
846 
847 	return err;
848 }
849 
850 static void tcf_block_remove(struct tcf_block *block, struct net *net)
851 {
852 	struct tcf_net *tn = net_generic(net, tcf_net_id);
853 
854 	spin_lock(&tn->idr_lock);
855 	idr_remove(&tn->idr, block->index);
856 	spin_unlock(&tn->idr_lock);
857 }
858 
859 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
860 					  u32 block_index,
861 					  struct netlink_ext_ack *extack)
862 {
863 	struct tcf_block *block;
864 
865 	block = kzalloc(sizeof(*block), GFP_KERNEL);
866 	if (!block) {
867 		NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
868 		return ERR_PTR(-ENOMEM);
869 	}
870 	mutex_init(&block->lock);
871 	mutex_init(&block->proto_destroy_lock);
872 	init_rwsem(&block->cb_lock);
873 	flow_block_init(&block->flow_block);
874 	INIT_LIST_HEAD(&block->chain_list);
875 	INIT_LIST_HEAD(&block->owner_list);
876 	INIT_LIST_HEAD(&block->chain0.filter_chain_list);
877 
878 	refcount_set(&block->refcnt, 1);
879 	block->net = net;
880 	block->index = block_index;
881 
882 	/* Don't store q pointer for blocks which are shared */
883 	if (!tcf_block_shared(block))
884 		block->q = q;
885 	return block;
886 }
887 
888 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
889 {
890 	struct tcf_net *tn = net_generic(net, tcf_net_id);
891 
892 	return idr_find(&tn->idr, block_index);
893 }
894 
895 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
896 {
897 	struct tcf_block *block;
898 
899 	rcu_read_lock();
900 	block = tcf_block_lookup(net, block_index);
901 	if (block && !refcount_inc_not_zero(&block->refcnt))
902 		block = NULL;
903 	rcu_read_unlock();
904 
905 	return block;
906 }
907 
908 static struct tcf_chain *
909 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
910 {
911 	mutex_lock(&block->lock);
912 	if (chain)
913 		chain = list_is_last(&chain->list, &block->chain_list) ?
914 			NULL : list_next_entry(chain, list);
915 	else
916 		chain = list_first_entry_or_null(&block->chain_list,
917 						 struct tcf_chain, list);
918 
919 	/* skip all action-only chains */
920 	while (chain && tcf_chain_held_by_acts_only(chain))
921 		chain = list_is_last(&chain->list, &block->chain_list) ?
922 			NULL : list_next_entry(chain, list);
923 
924 	if (chain)
925 		tcf_chain_hold(chain);
926 	mutex_unlock(&block->lock);
927 
928 	return chain;
929 }
930 
931 /* Function to be used by all clients that want to iterate over all chains on
932  * block. It properly obtains block->lock and takes reference to chain before
933  * returning it. Users of this function must be tolerant to concurrent chain
934  * insertion/deletion or ensure that no concurrent chain modification is
935  * possible. Note that all netlink dump callbacks cannot guarantee to provide
936  * consistent dump because rtnl lock is released each time skb is filled with
937  * data and sent to user-space.
938  */
939 
940 struct tcf_chain *
941 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
942 {
943 	struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
944 
945 	if (chain)
946 		tcf_chain_put(chain);
947 
948 	return chain_next;
949 }
950 EXPORT_SYMBOL(tcf_get_next_chain);
951 
952 static struct tcf_proto *
953 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
954 {
955 	u32 prio = 0;
956 
957 	ASSERT_RTNL();
958 	mutex_lock(&chain->filter_chain_lock);
959 
960 	if (!tp) {
961 		tp = tcf_chain_dereference(chain->filter_chain, chain);
962 	} else if (tcf_proto_is_deleting(tp)) {
963 		/* 'deleting' flag is set and chain->filter_chain_lock was
964 		 * unlocked, which means next pointer could be invalid. Restart
965 		 * search.
966 		 */
967 		prio = tp->prio + 1;
968 		tp = tcf_chain_dereference(chain->filter_chain, chain);
969 
970 		for (; tp; tp = tcf_chain_dereference(tp->next, chain))
971 			if (!tp->deleting && tp->prio >= prio)
972 				break;
973 	} else {
974 		tp = tcf_chain_dereference(tp->next, chain);
975 	}
976 
977 	if (tp)
978 		tcf_proto_get(tp);
979 
980 	mutex_unlock(&chain->filter_chain_lock);
981 
982 	return tp;
983 }
984 
985 /* Function to be used by all clients that want to iterate over all tp's on
986  * chain. Users of this function must be tolerant to concurrent tp
987  * insertion/deletion or ensure that no concurrent chain modification is
988  * possible. Note that all netlink dump callbacks cannot guarantee to provide
989  * consistent dump because rtnl lock is released each time skb is filled with
990  * data and sent to user-space.
991  */
992 
993 struct tcf_proto *
994 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp,
995 		   bool rtnl_held)
996 {
997 	struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
998 
999 	if (tp)
1000 		tcf_proto_put(tp, rtnl_held, NULL);
1001 
1002 	return tp_next;
1003 }
1004 EXPORT_SYMBOL(tcf_get_next_proto);
1005 
1006 static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
1007 {
1008 	struct tcf_chain *chain;
1009 
1010 	/* Last reference to block. At this point chains cannot be added or
1011 	 * removed concurrently.
1012 	 */
1013 	for (chain = tcf_get_next_chain(block, NULL);
1014 	     chain;
1015 	     chain = tcf_get_next_chain(block, chain)) {
1016 		tcf_chain_put_explicitly_created(chain);
1017 		tcf_chain_flush(chain, rtnl_held);
1018 	}
1019 }
1020 
1021 /* Lookup Qdisc and increments its reference counter.
1022  * Set parent, if necessary.
1023  */
1024 
1025 static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
1026 			    u32 *parent, int ifindex, bool rtnl_held,
1027 			    struct netlink_ext_ack *extack)
1028 {
1029 	const struct Qdisc_class_ops *cops;
1030 	struct net_device *dev;
1031 	int err = 0;
1032 
1033 	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1034 		return 0;
1035 
1036 	rcu_read_lock();
1037 
1038 	/* Find link */
1039 	dev = dev_get_by_index_rcu(net, ifindex);
1040 	if (!dev) {
1041 		rcu_read_unlock();
1042 		return -ENODEV;
1043 	}
1044 
1045 	/* Find qdisc */
1046 	if (!*parent) {
1047 		*q = dev->qdisc;
1048 		*parent = (*q)->handle;
1049 	} else {
1050 		*q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1051 		if (!*q) {
1052 			NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1053 			err = -EINVAL;
1054 			goto errout_rcu;
1055 		}
1056 	}
1057 
1058 	*q = qdisc_refcount_inc_nz(*q);
1059 	if (!*q) {
1060 		NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1061 		err = -EINVAL;
1062 		goto errout_rcu;
1063 	}
1064 
1065 	/* Is it classful? */
1066 	cops = (*q)->ops->cl_ops;
1067 	if (!cops) {
1068 		NL_SET_ERR_MSG(extack, "Qdisc not classful");
1069 		err = -EINVAL;
1070 		goto errout_qdisc;
1071 	}
1072 
1073 	if (!cops->tcf_block) {
1074 		NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1075 		err = -EOPNOTSUPP;
1076 		goto errout_qdisc;
1077 	}
1078 
1079 errout_rcu:
1080 	/* At this point we know that qdisc is not noop_qdisc,
1081 	 * which means that qdisc holds a reference to net_device
1082 	 * and we hold a reference to qdisc, so it is safe to release
1083 	 * rcu read lock.
1084 	 */
1085 	rcu_read_unlock();
1086 	return err;
1087 
1088 errout_qdisc:
1089 	rcu_read_unlock();
1090 
1091 	if (rtnl_held)
1092 		qdisc_put(*q);
1093 	else
1094 		qdisc_put_unlocked(*q);
1095 	*q = NULL;
1096 
1097 	return err;
1098 }
1099 
1100 static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
1101 			       int ifindex, struct netlink_ext_ack *extack)
1102 {
1103 	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1104 		return 0;
1105 
1106 	/* Do we search for filter, attached to class? */
1107 	if (TC_H_MIN(parent)) {
1108 		const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1109 
1110 		*cl = cops->find(q, parent);
1111 		if (*cl == 0) {
1112 			NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1113 			return -ENOENT;
1114 		}
1115 	}
1116 
1117 	return 0;
1118 }
1119 
1120 static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
1121 					  unsigned long cl, int ifindex,
1122 					  u32 block_index,
1123 					  struct netlink_ext_ack *extack)
1124 {
1125 	struct tcf_block *block;
1126 
1127 	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1128 		block = tcf_block_refcnt_get(net, block_index);
1129 		if (!block) {
1130 			NL_SET_ERR_MSG(extack, "Block of given index was not found");
1131 			return ERR_PTR(-EINVAL);
1132 		}
1133 	} else {
1134 		const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1135 
1136 		block = cops->tcf_block(q, cl, extack);
1137 		if (!block)
1138 			return ERR_PTR(-EINVAL);
1139 
1140 		if (tcf_block_shared(block)) {
1141 			NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1142 			return ERR_PTR(-EOPNOTSUPP);
1143 		}
1144 
1145 		/* Always take reference to block in order to support execution
1146 		 * of rules update path of cls API without rtnl lock. Caller
1147 		 * must release block when it is finished using it. 'if' block
1148 		 * of this conditional obtain reference to block by calling
1149 		 * tcf_block_refcnt_get().
1150 		 */
1151 		refcount_inc(&block->refcnt);
1152 	}
1153 
1154 	return block;
1155 }
1156 
1157 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
1158 			    struct tcf_block_ext_info *ei, bool rtnl_held)
1159 {
1160 	if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
1161 		/* Flushing/putting all chains will cause the block to be
1162 		 * deallocated when last chain is freed. However, if chain_list
1163 		 * is empty, block has to be manually deallocated. After block
1164 		 * reference counter reached 0, it is no longer possible to
1165 		 * increment it or add new chains to block.
1166 		 */
1167 		bool free_block = list_empty(&block->chain_list);
1168 
1169 		mutex_unlock(&block->lock);
1170 		if (tcf_block_shared(block))
1171 			tcf_block_remove(block, block->net);
1172 
1173 		if (q)
1174 			tcf_block_offload_unbind(block, q, ei);
1175 
1176 		if (free_block)
1177 			tcf_block_destroy(block);
1178 		else
1179 			tcf_block_flush_all_chains(block, rtnl_held);
1180 	} else if (q) {
1181 		tcf_block_offload_unbind(block, q, ei);
1182 	}
1183 }
1184 
1185 static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
1186 {
1187 	__tcf_block_put(block, NULL, NULL, rtnl_held);
1188 }
1189 
1190 /* Find tcf block.
1191  * Set q, parent, cl when appropriate.
1192  */
1193 
1194 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1195 					u32 *parent, unsigned long *cl,
1196 					int ifindex, u32 block_index,
1197 					struct netlink_ext_ack *extack)
1198 {
1199 	struct tcf_block *block;
1200 	int err = 0;
1201 
1202 	ASSERT_RTNL();
1203 
1204 	err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
1205 	if (err)
1206 		goto errout;
1207 
1208 	err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack);
1209 	if (err)
1210 		goto errout_qdisc;
1211 
1212 	block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack);
1213 	if (IS_ERR(block)) {
1214 		err = PTR_ERR(block);
1215 		goto errout_qdisc;
1216 	}
1217 
1218 	return block;
1219 
1220 errout_qdisc:
1221 	if (*q)
1222 		qdisc_put(*q);
1223 errout:
1224 	*q = NULL;
1225 	return ERR_PTR(err);
1226 }
1227 
1228 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
1229 			      bool rtnl_held)
1230 {
1231 	if (!IS_ERR_OR_NULL(block))
1232 		tcf_block_refcnt_put(block, rtnl_held);
1233 
1234 	if (q) {
1235 		if (rtnl_held)
1236 			qdisc_put(q);
1237 		else
1238 			qdisc_put_unlocked(q);
1239 	}
1240 }
1241 
1242 struct tcf_block_owner_item {
1243 	struct list_head list;
1244 	struct Qdisc *q;
1245 	enum flow_block_binder_type binder_type;
1246 };
1247 
1248 static void
1249 tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1250 			       struct Qdisc *q,
1251 			       enum flow_block_binder_type binder_type)
1252 {
1253 	if (block->keep_dst &&
1254 	    binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1255 	    binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
1256 		netif_keep_dst(qdisc_dev(q));
1257 }
1258 
1259 void tcf_block_netif_keep_dst(struct tcf_block *block)
1260 {
1261 	struct tcf_block_owner_item *item;
1262 
1263 	block->keep_dst = true;
1264 	list_for_each_entry(item, &block->owner_list, list)
1265 		tcf_block_owner_netif_keep_dst(block, item->q,
1266 					       item->binder_type);
1267 }
1268 EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1269 
1270 static int tcf_block_owner_add(struct tcf_block *block,
1271 			       struct Qdisc *q,
1272 			       enum flow_block_binder_type binder_type)
1273 {
1274 	struct tcf_block_owner_item *item;
1275 
1276 	item = kmalloc(sizeof(*item), GFP_KERNEL);
1277 	if (!item)
1278 		return -ENOMEM;
1279 	item->q = q;
1280 	item->binder_type = binder_type;
1281 	list_add(&item->list, &block->owner_list);
1282 	return 0;
1283 }
1284 
1285 static void tcf_block_owner_del(struct tcf_block *block,
1286 				struct Qdisc *q,
1287 				enum flow_block_binder_type binder_type)
1288 {
1289 	struct tcf_block_owner_item *item;
1290 
1291 	list_for_each_entry(item, &block->owner_list, list) {
1292 		if (item->q == q && item->binder_type == binder_type) {
1293 			list_del(&item->list);
1294 			kfree(item);
1295 			return;
1296 		}
1297 	}
1298 	WARN_ON(1);
1299 }
1300 
1301 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1302 		      struct tcf_block_ext_info *ei,
1303 		      struct netlink_ext_ack *extack)
1304 {
1305 	struct net *net = qdisc_net(q);
1306 	struct tcf_block *block = NULL;
1307 	int err;
1308 
1309 	if (ei->block_index)
1310 		/* block_index not 0 means the shared block is requested */
1311 		block = tcf_block_refcnt_get(net, ei->block_index);
1312 
1313 	if (!block) {
1314 		block = tcf_block_create(net, q, ei->block_index, extack);
1315 		if (IS_ERR(block))
1316 			return PTR_ERR(block);
1317 		if (tcf_block_shared(block)) {
1318 			err = tcf_block_insert(block, net, extack);
1319 			if (err)
1320 				goto err_block_insert;
1321 		}
1322 	}
1323 
1324 	err = tcf_block_owner_add(block, q, ei->binder_type);
1325 	if (err)
1326 		goto err_block_owner_add;
1327 
1328 	tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1329 
1330 	err = tcf_chain0_head_change_cb_add(block, ei, extack);
1331 	if (err)
1332 		goto err_chain0_head_change_cb_add;
1333 
1334 	err = tcf_block_offload_bind(block, q, ei, extack);
1335 	if (err)
1336 		goto err_block_offload_bind;
1337 
1338 	*p_block = block;
1339 	return 0;
1340 
1341 err_block_offload_bind:
1342 	tcf_chain0_head_change_cb_del(block, ei);
1343 err_chain0_head_change_cb_add:
1344 	tcf_block_owner_del(block, q, ei->binder_type);
1345 err_block_owner_add:
1346 err_block_insert:
1347 	tcf_block_refcnt_put(block, true);
1348 	return err;
1349 }
1350 EXPORT_SYMBOL(tcf_block_get_ext);
1351 
1352 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1353 {
1354 	struct tcf_proto __rcu **p_filter_chain = priv;
1355 
1356 	rcu_assign_pointer(*p_filter_chain, tp_head);
1357 }
1358 
1359 int tcf_block_get(struct tcf_block **p_block,
1360 		  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1361 		  struct netlink_ext_ack *extack)
1362 {
1363 	struct tcf_block_ext_info ei = {
1364 		.chain_head_change = tcf_chain_head_change_dflt,
1365 		.chain_head_change_priv = p_filter_chain,
1366 	};
1367 
1368 	WARN_ON(!p_filter_chain);
1369 	return tcf_block_get_ext(p_block, q, &ei, extack);
1370 }
1371 EXPORT_SYMBOL(tcf_block_get);
1372 
1373 /* XXX: Standalone actions are not allowed to jump to any chain, and bound
1374  * actions should be all removed after flushing.
1375  */
1376 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
1377 		       struct tcf_block_ext_info *ei)
1378 {
1379 	if (!block)
1380 		return;
1381 	tcf_chain0_head_change_cb_del(block, ei);
1382 	tcf_block_owner_del(block, q, ei->binder_type);
1383 
1384 	__tcf_block_put(block, q, ei, true);
1385 }
1386 EXPORT_SYMBOL(tcf_block_put_ext);
1387 
1388 void tcf_block_put(struct tcf_block *block)
1389 {
1390 	struct tcf_block_ext_info ei = {0, };
1391 
1392 	if (!block)
1393 		return;
1394 	tcf_block_put_ext(block, block->q, &ei);
1395 }
1396 
1397 EXPORT_SYMBOL(tcf_block_put);
1398 
1399 static int
1400 tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
1401 			    void *cb_priv, bool add, bool offload_in_use,
1402 			    struct netlink_ext_ack *extack)
1403 {
1404 	struct tcf_chain *chain, *chain_prev;
1405 	struct tcf_proto *tp, *tp_prev;
1406 	int err;
1407 
1408 	lockdep_assert_held(&block->cb_lock);
1409 
1410 	for (chain = __tcf_get_next_chain(block, NULL);
1411 	     chain;
1412 	     chain_prev = chain,
1413 		     chain = __tcf_get_next_chain(block, chain),
1414 		     tcf_chain_put(chain_prev)) {
1415 		for (tp = __tcf_get_next_proto(chain, NULL); tp;
1416 		     tp_prev = tp,
1417 			     tp = __tcf_get_next_proto(chain, tp),
1418 			     tcf_proto_put(tp_prev, true, NULL)) {
1419 			if (tp->ops->reoffload) {
1420 				err = tp->ops->reoffload(tp, add, cb, cb_priv,
1421 							 extack);
1422 				if (err && add)
1423 					goto err_playback_remove;
1424 			} else if (add && offload_in_use) {
1425 				err = -EOPNOTSUPP;
1426 				NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1427 				goto err_playback_remove;
1428 			}
1429 		}
1430 	}
1431 
1432 	return 0;
1433 
1434 err_playback_remove:
1435 	tcf_proto_put(tp, true, NULL);
1436 	tcf_chain_put(chain);
1437 	tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1438 				    extack);
1439 	return err;
1440 }
1441 
1442 static int tcf_block_bind(struct tcf_block *block,
1443 			  struct flow_block_offload *bo)
1444 {
1445 	struct flow_block_cb *block_cb, *next;
1446 	int err, i = 0;
1447 
1448 	lockdep_assert_held(&block->cb_lock);
1449 
1450 	list_for_each_entry(block_cb, &bo->cb_list, list) {
1451 		err = tcf_block_playback_offloads(block, block_cb->cb,
1452 						  block_cb->cb_priv, true,
1453 						  tcf_block_offload_in_use(block),
1454 						  bo->extack);
1455 		if (err)
1456 			goto err_unroll;
1457 		if (!bo->unlocked_driver_cb)
1458 			block->lockeddevcnt++;
1459 
1460 		i++;
1461 	}
1462 	list_splice(&bo->cb_list, &block->flow_block.cb_list);
1463 
1464 	return 0;
1465 
1466 err_unroll:
1467 	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1468 		if (i-- > 0) {
1469 			list_del(&block_cb->list);
1470 			tcf_block_playback_offloads(block, block_cb->cb,
1471 						    block_cb->cb_priv, false,
1472 						    tcf_block_offload_in_use(block),
1473 						    NULL);
1474 			if (!bo->unlocked_driver_cb)
1475 				block->lockeddevcnt--;
1476 		}
1477 		flow_block_cb_free(block_cb);
1478 	}
1479 
1480 	return err;
1481 }
1482 
1483 static void tcf_block_unbind(struct tcf_block *block,
1484 			     struct flow_block_offload *bo)
1485 {
1486 	struct flow_block_cb *block_cb, *next;
1487 
1488 	lockdep_assert_held(&block->cb_lock);
1489 
1490 	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1491 		tcf_block_playback_offloads(block, block_cb->cb,
1492 					    block_cb->cb_priv, false,
1493 					    tcf_block_offload_in_use(block),
1494 					    NULL);
1495 		list_del(&block_cb->list);
1496 		flow_block_cb_free(block_cb);
1497 		if (!bo->unlocked_driver_cb)
1498 			block->lockeddevcnt--;
1499 	}
1500 }
1501 
1502 static int tcf_block_setup(struct tcf_block *block,
1503 			   struct flow_block_offload *bo)
1504 {
1505 	int err;
1506 
1507 	switch (bo->command) {
1508 	case FLOW_BLOCK_BIND:
1509 		err = tcf_block_bind(block, bo);
1510 		break;
1511 	case FLOW_BLOCK_UNBIND:
1512 		err = 0;
1513 		tcf_block_unbind(block, bo);
1514 		break;
1515 	default:
1516 		WARN_ON_ONCE(1);
1517 		err = -EOPNOTSUPP;
1518 	}
1519 
1520 	return err;
1521 }
1522 
1523 /* Main classifier routine: scans classifier chain attached
1524  * to this qdisc, (optionally) tests for protocol and asks
1525  * specific classifiers.
1526  */
1527 static inline int __tcf_classify(struct sk_buff *skb,
1528 				 const struct tcf_proto *tp,
1529 				 const struct tcf_proto *orig_tp,
1530 				 struct tcf_result *res,
1531 				 bool compat_mode,
1532 				 u32 *last_executed_chain)
1533 {
1534 #ifdef CONFIG_NET_CLS_ACT
1535 	const int max_reclassify_loop = 4;
1536 	const struct tcf_proto *first_tp;
1537 	int limit = 0;
1538 
1539 reclassify:
1540 #endif
1541 	for (; tp; tp = rcu_dereference_bh(tp->next)) {
1542 		__be16 protocol = skb_protocol(skb, false);
1543 		int err;
1544 
1545 		if (tp->protocol != protocol &&
1546 		    tp->protocol != htons(ETH_P_ALL))
1547 			continue;
1548 
1549 		err = tp->classify(skb, tp, res);
1550 #ifdef CONFIG_NET_CLS_ACT
1551 		if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1552 			first_tp = orig_tp;
1553 			*last_executed_chain = first_tp->chain->index;
1554 			goto reset;
1555 		} else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1556 			first_tp = res->goto_tp;
1557 			*last_executed_chain = err & TC_ACT_EXT_VAL_MASK;
1558 			goto reset;
1559 		}
1560 #endif
1561 		if (err >= 0)
1562 			return err;
1563 	}
1564 
1565 	return TC_ACT_UNSPEC; /* signal: continue lookup */
1566 #ifdef CONFIG_NET_CLS_ACT
1567 reset:
1568 	if (unlikely(limit++ >= max_reclassify_loop)) {
1569 		net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1570 				       tp->chain->block->index,
1571 				       tp->prio & 0xffff,
1572 				       ntohs(tp->protocol));
1573 		return TC_ACT_SHOT;
1574 	}
1575 
1576 	tp = first_tp;
1577 	goto reclassify;
1578 #endif
1579 }
1580 
1581 int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1582 		 struct tcf_result *res, bool compat_mode)
1583 {
1584 	u32 last_executed_chain = 0;
1585 
1586 	return __tcf_classify(skb, tp, tp, res, compat_mode,
1587 			      &last_executed_chain);
1588 }
1589 EXPORT_SYMBOL(tcf_classify);
1590 
1591 int tcf_classify_ingress(struct sk_buff *skb,
1592 			 const struct tcf_block *ingress_block,
1593 			 const struct tcf_proto *tp,
1594 			 struct tcf_result *res, bool compat_mode)
1595 {
1596 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1597 	u32 last_executed_chain = 0;
1598 
1599 	return __tcf_classify(skb, tp, tp, res, compat_mode,
1600 			      &last_executed_chain);
1601 #else
1602 	u32 last_executed_chain = tp ? tp->chain->index : 0;
1603 	const struct tcf_proto *orig_tp = tp;
1604 	struct tc_skb_ext *ext;
1605 	int ret;
1606 
1607 	ext = skb_ext_find(skb, TC_SKB_EXT);
1608 
1609 	if (ext && ext->chain) {
1610 		struct tcf_chain *fchain;
1611 
1612 		fchain = tcf_chain_lookup_rcu(ingress_block, ext->chain);
1613 		if (!fchain)
1614 			return TC_ACT_SHOT;
1615 
1616 		/* Consume, so cloned/redirect skbs won't inherit ext */
1617 		skb_ext_del(skb, TC_SKB_EXT);
1618 
1619 		tp = rcu_dereference_bh(fchain->filter_chain);
1620 		last_executed_chain = fchain->index;
1621 	}
1622 
1623 	ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode,
1624 			     &last_executed_chain);
1625 
1626 	/* If we missed on some chain */
1627 	if (ret == TC_ACT_UNSPEC && last_executed_chain) {
1628 		ext = skb_ext_add(skb, TC_SKB_EXT);
1629 		if (WARN_ON_ONCE(!ext))
1630 			return TC_ACT_SHOT;
1631 		ext->chain = last_executed_chain;
1632 	}
1633 
1634 	return ret;
1635 #endif
1636 }
1637 EXPORT_SYMBOL(tcf_classify_ingress);
1638 
1639 struct tcf_chain_info {
1640 	struct tcf_proto __rcu **pprev;
1641 	struct tcf_proto __rcu *next;
1642 };
1643 
1644 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1645 					   struct tcf_chain_info *chain_info)
1646 {
1647 	return tcf_chain_dereference(*chain_info->pprev, chain);
1648 }
1649 
1650 static int tcf_chain_tp_insert(struct tcf_chain *chain,
1651 			       struct tcf_chain_info *chain_info,
1652 			       struct tcf_proto *tp)
1653 {
1654 	if (chain->flushing)
1655 		return -EAGAIN;
1656 
1657 	if (*chain_info->pprev == chain->filter_chain)
1658 		tcf_chain0_head_change(chain, tp);
1659 	tcf_proto_get(tp);
1660 	RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1661 	rcu_assign_pointer(*chain_info->pprev, tp);
1662 
1663 	return 0;
1664 }
1665 
1666 static void tcf_chain_tp_remove(struct tcf_chain *chain,
1667 				struct tcf_chain_info *chain_info,
1668 				struct tcf_proto *tp)
1669 {
1670 	struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
1671 
1672 	tcf_proto_mark_delete(tp);
1673 	if (tp == chain->filter_chain)
1674 		tcf_chain0_head_change(chain, next);
1675 	RCU_INIT_POINTER(*chain_info->pprev, next);
1676 }
1677 
1678 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1679 					   struct tcf_chain_info *chain_info,
1680 					   u32 protocol, u32 prio,
1681 					   bool prio_allocate);
1682 
1683 /* Try to insert new proto.
1684  * If proto with specified priority already exists, free new proto
1685  * and return existing one.
1686  */
1687 
1688 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1689 						    struct tcf_proto *tp_new,
1690 						    u32 protocol, u32 prio,
1691 						    bool rtnl_held)
1692 {
1693 	struct tcf_chain_info chain_info;
1694 	struct tcf_proto *tp;
1695 	int err = 0;
1696 
1697 	mutex_lock(&chain->filter_chain_lock);
1698 
1699 	if (tcf_proto_exists_destroying(chain, tp_new)) {
1700 		mutex_unlock(&chain->filter_chain_lock);
1701 		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1702 		return ERR_PTR(-EAGAIN);
1703 	}
1704 
1705 	tp = tcf_chain_tp_find(chain, &chain_info,
1706 			       protocol, prio, false);
1707 	if (!tp)
1708 		err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
1709 	mutex_unlock(&chain->filter_chain_lock);
1710 
1711 	if (tp) {
1712 		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1713 		tp_new = tp;
1714 	} else if (err) {
1715 		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1716 		tp_new = ERR_PTR(err);
1717 	}
1718 
1719 	return tp_new;
1720 }
1721 
1722 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
1723 				      struct tcf_proto *tp, bool rtnl_held,
1724 				      struct netlink_ext_ack *extack)
1725 {
1726 	struct tcf_chain_info chain_info;
1727 	struct tcf_proto *tp_iter;
1728 	struct tcf_proto **pprev;
1729 	struct tcf_proto *next;
1730 
1731 	mutex_lock(&chain->filter_chain_lock);
1732 
1733 	/* Atomically find and remove tp from chain. */
1734 	for (pprev = &chain->filter_chain;
1735 	     (tp_iter = tcf_chain_dereference(*pprev, chain));
1736 	     pprev = &tp_iter->next) {
1737 		if (tp_iter == tp) {
1738 			chain_info.pprev = pprev;
1739 			chain_info.next = tp_iter->next;
1740 			WARN_ON(tp_iter->deleting);
1741 			break;
1742 		}
1743 	}
1744 	/* Verify that tp still exists and no new filters were inserted
1745 	 * concurrently.
1746 	 * Mark tp for deletion if it is empty.
1747 	 */
1748 	if (!tp_iter || !tcf_proto_check_delete(tp)) {
1749 		mutex_unlock(&chain->filter_chain_lock);
1750 		return;
1751 	}
1752 
1753 	tcf_proto_signal_destroying(chain, tp);
1754 	next = tcf_chain_dereference(chain_info.next, chain);
1755 	if (tp == chain->filter_chain)
1756 		tcf_chain0_head_change(chain, next);
1757 	RCU_INIT_POINTER(*chain_info.pprev, next);
1758 	mutex_unlock(&chain->filter_chain_lock);
1759 
1760 	tcf_proto_put(tp, rtnl_held, extack);
1761 }
1762 
1763 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1764 					   struct tcf_chain_info *chain_info,
1765 					   u32 protocol, u32 prio,
1766 					   bool prio_allocate)
1767 {
1768 	struct tcf_proto **pprev;
1769 	struct tcf_proto *tp;
1770 
1771 	/* Check the chain for existence of proto-tcf with this priority */
1772 	for (pprev = &chain->filter_chain;
1773 	     (tp = tcf_chain_dereference(*pprev, chain));
1774 	     pprev = &tp->next) {
1775 		if (tp->prio >= prio) {
1776 			if (tp->prio == prio) {
1777 				if (prio_allocate ||
1778 				    (tp->protocol != protocol && protocol))
1779 					return ERR_PTR(-EINVAL);
1780 			} else {
1781 				tp = NULL;
1782 			}
1783 			break;
1784 		}
1785 	}
1786 	chain_info->pprev = pprev;
1787 	if (tp) {
1788 		chain_info->next = tp->next;
1789 		tcf_proto_get(tp);
1790 	} else {
1791 		chain_info->next = NULL;
1792 	}
1793 	return tp;
1794 }
1795 
1796 static int tcf_fill_node(struct net *net, struct sk_buff *skb,
1797 			 struct tcf_proto *tp, struct tcf_block *block,
1798 			 struct Qdisc *q, u32 parent, void *fh,
1799 			 u32 portid, u32 seq, u16 flags, int event,
1800 			 bool terse_dump, bool rtnl_held)
1801 {
1802 	struct tcmsg *tcm;
1803 	struct nlmsghdr  *nlh;
1804 	unsigned char *b = skb_tail_pointer(skb);
1805 
1806 	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1807 	if (!nlh)
1808 		goto out_nlmsg_trim;
1809 	tcm = nlmsg_data(nlh);
1810 	tcm->tcm_family = AF_UNSPEC;
1811 	tcm->tcm__pad1 = 0;
1812 	tcm->tcm__pad2 = 0;
1813 	if (q) {
1814 		tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1815 		tcm->tcm_parent = parent;
1816 	} else {
1817 		tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1818 		tcm->tcm_block_index = block->index;
1819 	}
1820 	tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1821 	if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1822 		goto nla_put_failure;
1823 	if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1824 		goto nla_put_failure;
1825 	if (!fh) {
1826 		tcm->tcm_handle = 0;
1827 	} else if (terse_dump) {
1828 		if (tp->ops->terse_dump) {
1829 			if (tp->ops->terse_dump(net, tp, fh, skb, tcm,
1830 						rtnl_held) < 0)
1831 				goto nla_put_failure;
1832 		} else {
1833 			goto cls_op_not_supp;
1834 		}
1835 	} else {
1836 		if (tp->ops->dump &&
1837 		    tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
1838 			goto nla_put_failure;
1839 	}
1840 	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1841 	return skb->len;
1842 
1843 out_nlmsg_trim:
1844 nla_put_failure:
1845 cls_op_not_supp:
1846 	nlmsg_trim(skb, b);
1847 	return -1;
1848 }
1849 
1850 static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1851 			  struct nlmsghdr *n, struct tcf_proto *tp,
1852 			  struct tcf_block *block, struct Qdisc *q,
1853 			  u32 parent, void *fh, int event, bool unicast,
1854 			  bool rtnl_held)
1855 {
1856 	struct sk_buff *skb;
1857 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1858 	int err = 0;
1859 
1860 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1861 	if (!skb)
1862 		return -ENOBUFS;
1863 
1864 	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1865 			  n->nlmsg_seq, n->nlmsg_flags, event,
1866 			  false, rtnl_held) <= 0) {
1867 		kfree_skb(skb);
1868 		return -EINVAL;
1869 	}
1870 
1871 	if (unicast)
1872 		err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1873 	else
1874 		err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1875 				     n->nlmsg_flags & NLM_F_ECHO);
1876 
1877 	if (err > 0)
1878 		err = 0;
1879 	return err;
1880 }
1881 
1882 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1883 			      struct nlmsghdr *n, struct tcf_proto *tp,
1884 			      struct tcf_block *block, struct Qdisc *q,
1885 			      u32 parent, void *fh, bool unicast, bool *last,
1886 			      bool rtnl_held, struct netlink_ext_ack *extack)
1887 {
1888 	struct sk_buff *skb;
1889 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1890 	int err;
1891 
1892 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1893 	if (!skb)
1894 		return -ENOBUFS;
1895 
1896 	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1897 			  n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER,
1898 			  false, rtnl_held) <= 0) {
1899 		NL_SET_ERR_MSG(extack, "Failed to build del event notification");
1900 		kfree_skb(skb);
1901 		return -EINVAL;
1902 	}
1903 
1904 	err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
1905 	if (err) {
1906 		kfree_skb(skb);
1907 		return err;
1908 	}
1909 
1910 	if (unicast)
1911 		err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1912 	else
1913 		err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1914 				     n->nlmsg_flags & NLM_F_ECHO);
1915 	if (err < 0)
1916 		NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
1917 
1918 	if (err > 0)
1919 		err = 0;
1920 	return err;
1921 }
1922 
1923 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
1924 				 struct tcf_block *block, struct Qdisc *q,
1925 				 u32 parent, struct nlmsghdr *n,
1926 				 struct tcf_chain *chain, int event,
1927 				 bool rtnl_held)
1928 {
1929 	struct tcf_proto *tp;
1930 
1931 	for (tp = tcf_get_next_proto(chain, NULL, rtnl_held);
1932 	     tp; tp = tcf_get_next_proto(chain, tp, rtnl_held))
1933 		tfilter_notify(net, oskb, n, tp, block,
1934 			       q, parent, NULL, event, false, rtnl_held);
1935 }
1936 
1937 static void tfilter_put(struct tcf_proto *tp, void *fh)
1938 {
1939 	if (tp->ops->put && fh)
1940 		tp->ops->put(tp, fh);
1941 }
1942 
1943 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1944 			  struct netlink_ext_ack *extack)
1945 {
1946 	struct net *net = sock_net(skb->sk);
1947 	struct nlattr *tca[TCA_MAX + 1];
1948 	char name[IFNAMSIZ];
1949 	struct tcmsg *t;
1950 	u32 protocol;
1951 	u32 prio;
1952 	bool prio_allocate;
1953 	u32 parent;
1954 	u32 chain_index;
1955 	struct Qdisc *q = NULL;
1956 	struct tcf_chain_info chain_info;
1957 	struct tcf_chain *chain = NULL;
1958 	struct tcf_block *block;
1959 	struct tcf_proto *tp;
1960 	unsigned long cl;
1961 	void *fh;
1962 	int err;
1963 	int tp_created;
1964 	bool rtnl_held = false;
1965 
1966 	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1967 		return -EPERM;
1968 
1969 replay:
1970 	tp_created = 0;
1971 
1972 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
1973 				     rtm_tca_policy, extack);
1974 	if (err < 0)
1975 		return err;
1976 
1977 	t = nlmsg_data(n);
1978 	protocol = TC_H_MIN(t->tcm_info);
1979 	prio = TC_H_MAJ(t->tcm_info);
1980 	prio_allocate = false;
1981 	parent = t->tcm_parent;
1982 	tp = NULL;
1983 	cl = 0;
1984 	block = NULL;
1985 
1986 	if (prio == 0) {
1987 		/* If no priority is provided by the user,
1988 		 * we allocate one.
1989 		 */
1990 		if (n->nlmsg_flags & NLM_F_CREATE) {
1991 			prio = TC_H_MAKE(0x80000000U, 0U);
1992 			prio_allocate = true;
1993 		} else {
1994 			NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1995 			return -ENOENT;
1996 		}
1997 	}
1998 
1999 	/* Find head of filter chain. */
2000 
2001 	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2002 	if (err)
2003 		return err;
2004 
2005 	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2006 		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2007 		err = -EINVAL;
2008 		goto errout;
2009 	}
2010 
2011 	/* Take rtnl mutex if rtnl_held was set to true on previous iteration,
2012 	 * block is shared (no qdisc found), qdisc is not unlocked, classifier
2013 	 * type is not specified, classifier is not unlocked.
2014 	 */
2015 	if (rtnl_held ||
2016 	    (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2017 	    !tcf_proto_is_unlocked(name)) {
2018 		rtnl_held = true;
2019 		rtnl_lock();
2020 	}
2021 
2022 	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2023 	if (err)
2024 		goto errout;
2025 
2026 	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2027 				 extack);
2028 	if (IS_ERR(block)) {
2029 		err = PTR_ERR(block);
2030 		goto errout;
2031 	}
2032 	block->classid = parent;
2033 
2034 	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2035 	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2036 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2037 		err = -EINVAL;
2038 		goto errout;
2039 	}
2040 	chain = tcf_chain_get(block, chain_index, true);
2041 	if (!chain) {
2042 		NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
2043 		err = -ENOMEM;
2044 		goto errout;
2045 	}
2046 
2047 	mutex_lock(&chain->filter_chain_lock);
2048 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2049 			       prio, prio_allocate);
2050 	if (IS_ERR(tp)) {
2051 		NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2052 		err = PTR_ERR(tp);
2053 		goto errout_locked;
2054 	}
2055 
2056 	if (tp == NULL) {
2057 		struct tcf_proto *tp_new = NULL;
2058 
2059 		if (chain->flushing) {
2060 			err = -EAGAIN;
2061 			goto errout_locked;
2062 		}
2063 
2064 		/* Proto-tcf does not exist, create new one */
2065 
2066 		if (tca[TCA_KIND] == NULL || !protocol) {
2067 			NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
2068 			err = -EINVAL;
2069 			goto errout_locked;
2070 		}
2071 
2072 		if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2073 			NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2074 			err = -ENOENT;
2075 			goto errout_locked;
2076 		}
2077 
2078 		if (prio_allocate)
2079 			prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
2080 							       &chain_info));
2081 
2082 		mutex_unlock(&chain->filter_chain_lock);
2083 		tp_new = tcf_proto_create(name, protocol, prio, chain,
2084 					  rtnl_held, extack);
2085 		if (IS_ERR(tp_new)) {
2086 			err = PTR_ERR(tp_new);
2087 			goto errout_tp;
2088 		}
2089 
2090 		tp_created = 1;
2091 		tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2092 						rtnl_held);
2093 		if (IS_ERR(tp)) {
2094 			err = PTR_ERR(tp);
2095 			goto errout_tp;
2096 		}
2097 	} else {
2098 		mutex_unlock(&chain->filter_chain_lock);
2099 	}
2100 
2101 	if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2102 		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2103 		err = -EINVAL;
2104 		goto errout;
2105 	}
2106 
2107 	fh = tp->ops->get(tp, t->tcm_handle);
2108 
2109 	if (!fh) {
2110 		if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2111 			NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2112 			err = -ENOENT;
2113 			goto errout;
2114 		}
2115 	} else if (n->nlmsg_flags & NLM_F_EXCL) {
2116 		tfilter_put(tp, fh);
2117 		NL_SET_ERR_MSG(extack, "Filter already exists");
2118 		err = -EEXIST;
2119 		goto errout;
2120 	}
2121 
2122 	if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2123 		NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2124 		err = -EINVAL;
2125 		goto errout;
2126 	}
2127 
2128 	err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
2129 			      n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
2130 			      rtnl_held, extack);
2131 	if (err == 0) {
2132 		tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2133 			       RTM_NEWTFILTER, false, rtnl_held);
2134 		tfilter_put(tp, fh);
2135 		/* q pointer is NULL for shared blocks */
2136 		if (q)
2137 			q->flags &= ~TCQ_F_CAN_BYPASS;
2138 	}
2139 
2140 errout:
2141 	if (err && tp_created)
2142 		tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
2143 errout_tp:
2144 	if (chain) {
2145 		if (tp && !IS_ERR(tp))
2146 			tcf_proto_put(tp, rtnl_held, NULL);
2147 		if (!tp_created)
2148 			tcf_chain_put(chain);
2149 	}
2150 	tcf_block_release(q, block, rtnl_held);
2151 
2152 	if (rtnl_held)
2153 		rtnl_unlock();
2154 
2155 	if (err == -EAGAIN) {
2156 		/* Take rtnl lock in case EAGAIN is caused by concurrent flush
2157 		 * of target chain.
2158 		 */
2159 		rtnl_held = true;
2160 		/* Replay the request. */
2161 		goto replay;
2162 	}
2163 	return err;
2164 
2165 errout_locked:
2166 	mutex_unlock(&chain->filter_chain_lock);
2167 	goto errout;
2168 }
2169 
2170 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2171 			  struct netlink_ext_ack *extack)
2172 {
2173 	struct net *net = sock_net(skb->sk);
2174 	struct nlattr *tca[TCA_MAX + 1];
2175 	char name[IFNAMSIZ];
2176 	struct tcmsg *t;
2177 	u32 protocol;
2178 	u32 prio;
2179 	u32 parent;
2180 	u32 chain_index;
2181 	struct Qdisc *q = NULL;
2182 	struct tcf_chain_info chain_info;
2183 	struct tcf_chain *chain = NULL;
2184 	struct tcf_block *block = NULL;
2185 	struct tcf_proto *tp = NULL;
2186 	unsigned long cl = 0;
2187 	void *fh = NULL;
2188 	int err;
2189 	bool rtnl_held = false;
2190 
2191 	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2192 		return -EPERM;
2193 
2194 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2195 				     rtm_tca_policy, extack);
2196 	if (err < 0)
2197 		return err;
2198 
2199 	t = nlmsg_data(n);
2200 	protocol = TC_H_MIN(t->tcm_info);
2201 	prio = TC_H_MAJ(t->tcm_info);
2202 	parent = t->tcm_parent;
2203 
2204 	if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2205 		NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2206 		return -ENOENT;
2207 	}
2208 
2209 	/* Find head of filter chain. */
2210 
2211 	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2212 	if (err)
2213 		return err;
2214 
2215 	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2216 		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2217 		err = -EINVAL;
2218 		goto errout;
2219 	}
2220 	/* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2221 	 * found), qdisc is not unlocked, classifier type is not specified,
2222 	 * classifier is not unlocked.
2223 	 */
2224 	if (!prio ||
2225 	    (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2226 	    !tcf_proto_is_unlocked(name)) {
2227 		rtnl_held = true;
2228 		rtnl_lock();
2229 	}
2230 
2231 	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2232 	if (err)
2233 		goto errout;
2234 
2235 	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2236 				 extack);
2237 	if (IS_ERR(block)) {
2238 		err = PTR_ERR(block);
2239 		goto errout;
2240 	}
2241 
2242 	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2243 	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2244 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2245 		err = -EINVAL;
2246 		goto errout;
2247 	}
2248 	chain = tcf_chain_get(block, chain_index, false);
2249 	if (!chain) {
2250 		/* User requested flush on non-existent chain. Nothing to do,
2251 		 * so just return success.
2252 		 */
2253 		if (prio == 0) {
2254 			err = 0;
2255 			goto errout;
2256 		}
2257 		NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2258 		err = -ENOENT;
2259 		goto errout;
2260 	}
2261 
2262 	if (prio == 0) {
2263 		tfilter_notify_chain(net, skb, block, q, parent, n,
2264 				     chain, RTM_DELTFILTER, rtnl_held);
2265 		tcf_chain_flush(chain, rtnl_held);
2266 		err = 0;
2267 		goto errout;
2268 	}
2269 
2270 	mutex_lock(&chain->filter_chain_lock);
2271 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2272 			       prio, false);
2273 	if (!tp || IS_ERR(tp)) {
2274 		NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2275 		err = tp ? PTR_ERR(tp) : -ENOENT;
2276 		goto errout_locked;
2277 	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2278 		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2279 		err = -EINVAL;
2280 		goto errout_locked;
2281 	} else if (t->tcm_handle == 0) {
2282 		tcf_proto_signal_destroying(chain, tp);
2283 		tcf_chain_tp_remove(chain, &chain_info, tp);
2284 		mutex_unlock(&chain->filter_chain_lock);
2285 
2286 		tcf_proto_put(tp, rtnl_held, NULL);
2287 		tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2288 			       RTM_DELTFILTER, false, rtnl_held);
2289 		err = 0;
2290 		goto errout;
2291 	}
2292 	mutex_unlock(&chain->filter_chain_lock);
2293 
2294 	fh = tp->ops->get(tp, t->tcm_handle);
2295 
2296 	if (!fh) {
2297 		NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2298 		err = -ENOENT;
2299 	} else {
2300 		bool last;
2301 
2302 		err = tfilter_del_notify(net, skb, n, tp, block,
2303 					 q, parent, fh, false, &last,
2304 					 rtnl_held, extack);
2305 
2306 		if (err)
2307 			goto errout;
2308 		if (last)
2309 			tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
2310 	}
2311 
2312 errout:
2313 	if (chain) {
2314 		if (tp && !IS_ERR(tp))
2315 			tcf_proto_put(tp, rtnl_held, NULL);
2316 		tcf_chain_put(chain);
2317 	}
2318 	tcf_block_release(q, block, rtnl_held);
2319 
2320 	if (rtnl_held)
2321 		rtnl_unlock();
2322 
2323 	return err;
2324 
2325 errout_locked:
2326 	mutex_unlock(&chain->filter_chain_lock);
2327 	goto errout;
2328 }
2329 
2330 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2331 			  struct netlink_ext_ack *extack)
2332 {
2333 	struct net *net = sock_net(skb->sk);
2334 	struct nlattr *tca[TCA_MAX + 1];
2335 	char name[IFNAMSIZ];
2336 	struct tcmsg *t;
2337 	u32 protocol;
2338 	u32 prio;
2339 	u32 parent;
2340 	u32 chain_index;
2341 	struct Qdisc *q = NULL;
2342 	struct tcf_chain_info chain_info;
2343 	struct tcf_chain *chain = NULL;
2344 	struct tcf_block *block = NULL;
2345 	struct tcf_proto *tp = NULL;
2346 	unsigned long cl = 0;
2347 	void *fh = NULL;
2348 	int err;
2349 	bool rtnl_held = false;
2350 
2351 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2352 				     rtm_tca_policy, extack);
2353 	if (err < 0)
2354 		return err;
2355 
2356 	t = nlmsg_data(n);
2357 	protocol = TC_H_MIN(t->tcm_info);
2358 	prio = TC_H_MAJ(t->tcm_info);
2359 	parent = t->tcm_parent;
2360 
2361 	if (prio == 0) {
2362 		NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2363 		return -ENOENT;
2364 	}
2365 
2366 	/* Find head of filter chain. */
2367 
2368 	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2369 	if (err)
2370 		return err;
2371 
2372 	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2373 		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2374 		err = -EINVAL;
2375 		goto errout;
2376 	}
2377 	/* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
2378 	 * unlocked, classifier type is not specified, classifier is not
2379 	 * unlocked.
2380 	 */
2381 	if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2382 	    !tcf_proto_is_unlocked(name)) {
2383 		rtnl_held = true;
2384 		rtnl_lock();
2385 	}
2386 
2387 	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2388 	if (err)
2389 		goto errout;
2390 
2391 	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2392 				 extack);
2393 	if (IS_ERR(block)) {
2394 		err = PTR_ERR(block);
2395 		goto errout;
2396 	}
2397 
2398 	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2399 	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2400 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2401 		err = -EINVAL;
2402 		goto errout;
2403 	}
2404 	chain = tcf_chain_get(block, chain_index, false);
2405 	if (!chain) {
2406 		NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2407 		err = -EINVAL;
2408 		goto errout;
2409 	}
2410 
2411 	mutex_lock(&chain->filter_chain_lock);
2412 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2413 			       prio, false);
2414 	mutex_unlock(&chain->filter_chain_lock);
2415 	if (!tp || IS_ERR(tp)) {
2416 		NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2417 		err = tp ? PTR_ERR(tp) : -ENOENT;
2418 		goto errout;
2419 	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2420 		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2421 		err = -EINVAL;
2422 		goto errout;
2423 	}
2424 
2425 	fh = tp->ops->get(tp, t->tcm_handle);
2426 
2427 	if (!fh) {
2428 		NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2429 		err = -ENOENT;
2430 	} else {
2431 		err = tfilter_notify(net, skb, n, tp, block, q, parent,
2432 				     fh, RTM_NEWTFILTER, true, rtnl_held);
2433 		if (err < 0)
2434 			NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2435 	}
2436 
2437 	tfilter_put(tp, fh);
2438 errout:
2439 	if (chain) {
2440 		if (tp && !IS_ERR(tp))
2441 			tcf_proto_put(tp, rtnl_held, NULL);
2442 		tcf_chain_put(chain);
2443 	}
2444 	tcf_block_release(q, block, rtnl_held);
2445 
2446 	if (rtnl_held)
2447 		rtnl_unlock();
2448 
2449 	return err;
2450 }
2451 
2452 struct tcf_dump_args {
2453 	struct tcf_walker w;
2454 	struct sk_buff *skb;
2455 	struct netlink_callback *cb;
2456 	struct tcf_block *block;
2457 	struct Qdisc *q;
2458 	u32 parent;
2459 	bool terse_dump;
2460 };
2461 
2462 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
2463 {
2464 	struct tcf_dump_args *a = (void *)arg;
2465 	struct net *net = sock_net(a->skb->sk);
2466 
2467 	return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
2468 			     n, NETLINK_CB(a->cb->skb).portid,
2469 			     a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2470 			     RTM_NEWTFILTER, a->terse_dump, true);
2471 }
2472 
2473 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2474 			   struct sk_buff *skb, struct netlink_callback *cb,
2475 			   long index_start, long *p_index, bool terse)
2476 {
2477 	struct net *net = sock_net(skb->sk);
2478 	struct tcf_block *block = chain->block;
2479 	struct tcmsg *tcm = nlmsg_data(cb->nlh);
2480 	struct tcf_proto *tp, *tp_prev;
2481 	struct tcf_dump_args arg;
2482 
2483 	for (tp = __tcf_get_next_proto(chain, NULL);
2484 	     tp;
2485 	     tp_prev = tp,
2486 		     tp = __tcf_get_next_proto(chain, tp),
2487 		     tcf_proto_put(tp_prev, true, NULL),
2488 		     (*p_index)++) {
2489 		if (*p_index < index_start)
2490 			continue;
2491 		if (TC_H_MAJ(tcm->tcm_info) &&
2492 		    TC_H_MAJ(tcm->tcm_info) != tp->prio)
2493 			continue;
2494 		if (TC_H_MIN(tcm->tcm_info) &&
2495 		    TC_H_MIN(tcm->tcm_info) != tp->protocol)
2496 			continue;
2497 		if (*p_index > index_start)
2498 			memset(&cb->args[1], 0,
2499 			       sizeof(cb->args) - sizeof(cb->args[0]));
2500 		if (cb->args[1] == 0) {
2501 			if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
2502 					  NETLINK_CB(cb->skb).portid,
2503 					  cb->nlh->nlmsg_seq, NLM_F_MULTI,
2504 					  RTM_NEWTFILTER, false, true) <= 0)
2505 				goto errout;
2506 			cb->args[1] = 1;
2507 		}
2508 		if (!tp->ops->walk)
2509 			continue;
2510 		arg.w.fn = tcf_node_dump;
2511 		arg.skb = skb;
2512 		arg.cb = cb;
2513 		arg.block = block;
2514 		arg.q = q;
2515 		arg.parent = parent;
2516 		arg.w.stop = 0;
2517 		arg.w.skip = cb->args[1] - 1;
2518 		arg.w.count = 0;
2519 		arg.w.cookie = cb->args[2];
2520 		arg.terse_dump = terse;
2521 		tp->ops->walk(tp, &arg.w, true);
2522 		cb->args[2] = arg.w.cookie;
2523 		cb->args[1] = arg.w.count + 1;
2524 		if (arg.w.stop)
2525 			goto errout;
2526 	}
2527 	return true;
2528 
2529 errout:
2530 	tcf_proto_put(tp, true, NULL);
2531 	return false;
2532 }
2533 
2534 static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = {
2535 	[TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE),
2536 };
2537 
2538 /* called with RTNL */
2539 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2540 {
2541 	struct tcf_chain *chain, *chain_prev;
2542 	struct net *net = sock_net(skb->sk);
2543 	struct nlattr *tca[TCA_MAX + 1];
2544 	struct Qdisc *q = NULL;
2545 	struct tcf_block *block;
2546 	struct tcmsg *tcm = nlmsg_data(cb->nlh);
2547 	bool terse_dump = false;
2548 	long index_start;
2549 	long index;
2550 	u32 parent;
2551 	int err;
2552 
2553 	if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2554 		return skb->len;
2555 
2556 	err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2557 				     tcf_tfilter_dump_policy, cb->extack);
2558 	if (err)
2559 		return err;
2560 
2561 	if (tca[TCA_DUMP_FLAGS]) {
2562 		struct nla_bitfield32 flags =
2563 			nla_get_bitfield32(tca[TCA_DUMP_FLAGS]);
2564 
2565 		terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE;
2566 	}
2567 
2568 	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2569 		block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2570 		if (!block)
2571 			goto out;
2572 		/* If we work with block index, q is NULL and parent value
2573 		 * will never be used in the following code. The check
2574 		 * in tcf_fill_node prevents it. However, compiler does not
2575 		 * see that far, so set parent to zero to silence the warning
2576 		 * about parent being uninitialized.
2577 		 */
2578 		parent = 0;
2579 	} else {
2580 		const struct Qdisc_class_ops *cops;
2581 		struct net_device *dev;
2582 		unsigned long cl = 0;
2583 
2584 		dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2585 		if (!dev)
2586 			return skb->len;
2587 
2588 		parent = tcm->tcm_parent;
2589 		if (!parent)
2590 			q = dev->qdisc;
2591 		else
2592 			q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2593 		if (!q)
2594 			goto out;
2595 		cops = q->ops->cl_ops;
2596 		if (!cops)
2597 			goto out;
2598 		if (!cops->tcf_block)
2599 			goto out;
2600 		if (TC_H_MIN(tcm->tcm_parent)) {
2601 			cl = cops->find(q, tcm->tcm_parent);
2602 			if (cl == 0)
2603 				goto out;
2604 		}
2605 		block = cops->tcf_block(q, cl, NULL);
2606 		if (!block)
2607 			goto out;
2608 		parent = block->classid;
2609 		if (tcf_block_shared(block))
2610 			q = NULL;
2611 	}
2612 
2613 	index_start = cb->args[0];
2614 	index = 0;
2615 
2616 	for (chain = __tcf_get_next_chain(block, NULL);
2617 	     chain;
2618 	     chain_prev = chain,
2619 		     chain = __tcf_get_next_chain(block, chain),
2620 		     tcf_chain_put(chain_prev)) {
2621 		if (tca[TCA_CHAIN] &&
2622 		    nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2623 			continue;
2624 		if (!tcf_chain_dump(chain, q, parent, skb, cb,
2625 				    index_start, &index, terse_dump)) {
2626 			tcf_chain_put(chain);
2627 			err = -EMSGSIZE;
2628 			break;
2629 		}
2630 	}
2631 
2632 	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2633 		tcf_block_refcnt_put(block, true);
2634 	cb->args[0] = index;
2635 
2636 out:
2637 	/* If we did no progress, the error (EMSGSIZE) is real */
2638 	if (skb->len == 0 && err)
2639 		return err;
2640 	return skb->len;
2641 }
2642 
2643 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2644 			      void *tmplt_priv, u32 chain_index,
2645 			      struct net *net, struct sk_buff *skb,
2646 			      struct tcf_block *block,
2647 			      u32 portid, u32 seq, u16 flags, int event)
2648 {
2649 	unsigned char *b = skb_tail_pointer(skb);
2650 	const struct tcf_proto_ops *ops;
2651 	struct nlmsghdr *nlh;
2652 	struct tcmsg *tcm;
2653 	void *priv;
2654 
2655 	ops = tmplt_ops;
2656 	priv = tmplt_priv;
2657 
2658 	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2659 	if (!nlh)
2660 		goto out_nlmsg_trim;
2661 	tcm = nlmsg_data(nlh);
2662 	tcm->tcm_family = AF_UNSPEC;
2663 	tcm->tcm__pad1 = 0;
2664 	tcm->tcm__pad2 = 0;
2665 	tcm->tcm_handle = 0;
2666 	if (block->q) {
2667 		tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2668 		tcm->tcm_parent = block->q->handle;
2669 	} else {
2670 		tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2671 		tcm->tcm_block_index = block->index;
2672 	}
2673 
2674 	if (nla_put_u32(skb, TCA_CHAIN, chain_index))
2675 		goto nla_put_failure;
2676 
2677 	if (ops) {
2678 		if (nla_put_string(skb, TCA_KIND, ops->kind))
2679 			goto nla_put_failure;
2680 		if (ops->tmplt_dump(skb, net, priv) < 0)
2681 			goto nla_put_failure;
2682 	}
2683 
2684 	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2685 	return skb->len;
2686 
2687 out_nlmsg_trim:
2688 nla_put_failure:
2689 	nlmsg_trim(skb, b);
2690 	return -EMSGSIZE;
2691 }
2692 
2693 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2694 			   u32 seq, u16 flags, int event, bool unicast)
2695 {
2696 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2697 	struct tcf_block *block = chain->block;
2698 	struct net *net = block->net;
2699 	struct sk_buff *skb;
2700 	int err = 0;
2701 
2702 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2703 	if (!skb)
2704 		return -ENOBUFS;
2705 
2706 	if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2707 			       chain->index, net, skb, block, portid,
2708 			       seq, flags, event) <= 0) {
2709 		kfree_skb(skb);
2710 		return -EINVAL;
2711 	}
2712 
2713 	if (unicast)
2714 		err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2715 	else
2716 		err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
2717 				     flags & NLM_F_ECHO);
2718 
2719 	if (err > 0)
2720 		err = 0;
2721 	return err;
2722 }
2723 
2724 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2725 				  void *tmplt_priv, u32 chain_index,
2726 				  struct tcf_block *block, struct sk_buff *oskb,
2727 				  u32 seq, u16 flags, bool unicast)
2728 {
2729 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2730 	struct net *net = block->net;
2731 	struct sk_buff *skb;
2732 
2733 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2734 	if (!skb)
2735 		return -ENOBUFS;
2736 
2737 	if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2738 			       block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
2739 		kfree_skb(skb);
2740 		return -EINVAL;
2741 	}
2742 
2743 	if (unicast)
2744 		return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2745 
2746 	return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2747 }
2748 
2749 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2750 			      struct nlattr **tca,
2751 			      struct netlink_ext_ack *extack)
2752 {
2753 	const struct tcf_proto_ops *ops;
2754 	char name[IFNAMSIZ];
2755 	void *tmplt_priv;
2756 
2757 	/* If kind is not set, user did not specify template. */
2758 	if (!tca[TCA_KIND])
2759 		return 0;
2760 
2761 	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2762 		NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
2763 		return -EINVAL;
2764 	}
2765 
2766 	ops = tcf_proto_lookup_ops(name, true, extack);
2767 	if (IS_ERR(ops))
2768 		return PTR_ERR(ops);
2769 	if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2770 		NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2771 		return -EOPNOTSUPP;
2772 	}
2773 
2774 	tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2775 	if (IS_ERR(tmplt_priv)) {
2776 		module_put(ops->owner);
2777 		return PTR_ERR(tmplt_priv);
2778 	}
2779 	chain->tmplt_ops = ops;
2780 	chain->tmplt_priv = tmplt_priv;
2781 	return 0;
2782 }
2783 
2784 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
2785 			       void *tmplt_priv)
2786 {
2787 	/* If template ops are set, no work to do for us. */
2788 	if (!tmplt_ops)
2789 		return;
2790 
2791 	tmplt_ops->tmplt_destroy(tmplt_priv);
2792 	module_put(tmplt_ops->owner);
2793 }
2794 
2795 /* Add/delete/get a chain */
2796 
2797 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2798 			struct netlink_ext_ack *extack)
2799 {
2800 	struct net *net = sock_net(skb->sk);
2801 	struct nlattr *tca[TCA_MAX + 1];
2802 	struct tcmsg *t;
2803 	u32 parent;
2804 	u32 chain_index;
2805 	struct Qdisc *q = NULL;
2806 	struct tcf_chain *chain = NULL;
2807 	struct tcf_block *block;
2808 	unsigned long cl;
2809 	int err;
2810 
2811 	if (n->nlmsg_type != RTM_GETCHAIN &&
2812 	    !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2813 		return -EPERM;
2814 
2815 replay:
2816 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2817 				     rtm_tca_policy, extack);
2818 	if (err < 0)
2819 		return err;
2820 
2821 	t = nlmsg_data(n);
2822 	parent = t->tcm_parent;
2823 	cl = 0;
2824 
2825 	block = tcf_block_find(net, &q, &parent, &cl,
2826 			       t->tcm_ifindex, t->tcm_block_index, extack);
2827 	if (IS_ERR(block))
2828 		return PTR_ERR(block);
2829 
2830 	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2831 	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2832 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2833 		err = -EINVAL;
2834 		goto errout_block;
2835 	}
2836 
2837 	mutex_lock(&block->lock);
2838 	chain = tcf_chain_lookup(block, chain_index);
2839 	if (n->nlmsg_type == RTM_NEWCHAIN) {
2840 		if (chain) {
2841 			if (tcf_chain_held_by_acts_only(chain)) {
2842 				/* The chain exists only because there is
2843 				 * some action referencing it.
2844 				 */
2845 				tcf_chain_hold(chain);
2846 			} else {
2847 				NL_SET_ERR_MSG(extack, "Filter chain already exists");
2848 				err = -EEXIST;
2849 				goto errout_block_locked;
2850 			}
2851 		} else {
2852 			if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2853 				NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
2854 				err = -ENOENT;
2855 				goto errout_block_locked;
2856 			}
2857 			chain = tcf_chain_create(block, chain_index);
2858 			if (!chain) {
2859 				NL_SET_ERR_MSG(extack, "Failed to create filter chain");
2860 				err = -ENOMEM;
2861 				goto errout_block_locked;
2862 			}
2863 		}
2864 	} else {
2865 		if (!chain || tcf_chain_held_by_acts_only(chain)) {
2866 			NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2867 			err = -EINVAL;
2868 			goto errout_block_locked;
2869 		}
2870 		tcf_chain_hold(chain);
2871 	}
2872 
2873 	if (n->nlmsg_type == RTM_NEWCHAIN) {
2874 		/* Modifying chain requires holding parent block lock. In case
2875 		 * the chain was successfully added, take a reference to the
2876 		 * chain. This ensures that an empty chain does not disappear at
2877 		 * the end of this function.
2878 		 */
2879 		tcf_chain_hold(chain);
2880 		chain->explicitly_created = true;
2881 	}
2882 	mutex_unlock(&block->lock);
2883 
2884 	switch (n->nlmsg_type) {
2885 	case RTM_NEWCHAIN:
2886 		err = tc_chain_tmplt_add(chain, net, tca, extack);
2887 		if (err) {
2888 			tcf_chain_put_explicitly_created(chain);
2889 			goto errout;
2890 		}
2891 
2892 		tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2893 				RTM_NEWCHAIN, false);
2894 		break;
2895 	case RTM_DELCHAIN:
2896 		tfilter_notify_chain(net, skb, block, q, parent, n,
2897 				     chain, RTM_DELTFILTER, true);
2898 		/* Flush the chain first as the user requested chain removal. */
2899 		tcf_chain_flush(chain, true);
2900 		/* In case the chain was successfully deleted, put a reference
2901 		 * to the chain previously taken during addition.
2902 		 */
2903 		tcf_chain_put_explicitly_created(chain);
2904 		break;
2905 	case RTM_GETCHAIN:
2906 		err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2907 				      n->nlmsg_seq, n->nlmsg_type, true);
2908 		if (err < 0)
2909 			NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2910 		break;
2911 	default:
2912 		err = -EOPNOTSUPP;
2913 		NL_SET_ERR_MSG(extack, "Unsupported message type");
2914 		goto errout;
2915 	}
2916 
2917 errout:
2918 	tcf_chain_put(chain);
2919 errout_block:
2920 	tcf_block_release(q, block, true);
2921 	if (err == -EAGAIN)
2922 		/* Replay the request. */
2923 		goto replay;
2924 	return err;
2925 
2926 errout_block_locked:
2927 	mutex_unlock(&block->lock);
2928 	goto errout_block;
2929 }
2930 
2931 /* called with RTNL */
2932 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2933 {
2934 	struct net *net = sock_net(skb->sk);
2935 	struct nlattr *tca[TCA_MAX + 1];
2936 	struct Qdisc *q = NULL;
2937 	struct tcf_block *block;
2938 	struct tcmsg *tcm = nlmsg_data(cb->nlh);
2939 	struct tcf_chain *chain;
2940 	long index_start;
2941 	long index;
2942 	u32 parent;
2943 	int err;
2944 
2945 	if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2946 		return skb->len;
2947 
2948 	err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2949 				     rtm_tca_policy, cb->extack);
2950 	if (err)
2951 		return err;
2952 
2953 	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2954 		block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2955 		if (!block)
2956 			goto out;
2957 		/* If we work with block index, q is NULL and parent value
2958 		 * will never be used in the following code. The check
2959 		 * in tcf_fill_node prevents it. However, compiler does not
2960 		 * see that far, so set parent to zero to silence the warning
2961 		 * about parent being uninitialized.
2962 		 */
2963 		parent = 0;
2964 	} else {
2965 		const struct Qdisc_class_ops *cops;
2966 		struct net_device *dev;
2967 		unsigned long cl = 0;
2968 
2969 		dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2970 		if (!dev)
2971 			return skb->len;
2972 
2973 		parent = tcm->tcm_parent;
2974 		if (!parent) {
2975 			q = dev->qdisc;
2976 			parent = q->handle;
2977 		} else {
2978 			q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2979 		}
2980 		if (!q)
2981 			goto out;
2982 		cops = q->ops->cl_ops;
2983 		if (!cops)
2984 			goto out;
2985 		if (!cops->tcf_block)
2986 			goto out;
2987 		if (TC_H_MIN(tcm->tcm_parent)) {
2988 			cl = cops->find(q, tcm->tcm_parent);
2989 			if (cl == 0)
2990 				goto out;
2991 		}
2992 		block = cops->tcf_block(q, cl, NULL);
2993 		if (!block)
2994 			goto out;
2995 		if (tcf_block_shared(block))
2996 			q = NULL;
2997 	}
2998 
2999 	index_start = cb->args[0];
3000 	index = 0;
3001 
3002 	mutex_lock(&block->lock);
3003 	list_for_each_entry(chain, &block->chain_list, list) {
3004 		if ((tca[TCA_CHAIN] &&
3005 		     nla_get_u32(tca[TCA_CHAIN]) != chain->index))
3006 			continue;
3007 		if (index < index_start) {
3008 			index++;
3009 			continue;
3010 		}
3011 		if (tcf_chain_held_by_acts_only(chain))
3012 			continue;
3013 		err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
3014 					 chain->index, net, skb, block,
3015 					 NETLINK_CB(cb->skb).portid,
3016 					 cb->nlh->nlmsg_seq, NLM_F_MULTI,
3017 					 RTM_NEWCHAIN);
3018 		if (err <= 0)
3019 			break;
3020 		index++;
3021 	}
3022 	mutex_unlock(&block->lock);
3023 
3024 	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
3025 		tcf_block_refcnt_put(block, true);
3026 	cb->args[0] = index;
3027 
3028 out:
3029 	/* If we did no progress, the error (EMSGSIZE) is real */
3030 	if (skb->len == 0 && err)
3031 		return err;
3032 	return skb->len;
3033 }
3034 
3035 void tcf_exts_destroy(struct tcf_exts *exts)
3036 {
3037 #ifdef CONFIG_NET_CLS_ACT
3038 	if (exts->actions) {
3039 		tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
3040 		kfree(exts->actions);
3041 	}
3042 	exts->nr_actions = 0;
3043 #endif
3044 }
3045 EXPORT_SYMBOL(tcf_exts_destroy);
3046 
3047 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3048 		      struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
3049 		      bool rtnl_held, struct netlink_ext_ack *extack)
3050 {
3051 #ifdef CONFIG_NET_CLS_ACT
3052 	{
3053 		struct tc_action *act;
3054 		size_t attr_size = 0;
3055 
3056 		if (exts->police && tb[exts->police]) {
3057 			act = tcf_action_init_1(net, tp, tb[exts->police],
3058 						rate_tlv, "police", ovr,
3059 						TCA_ACT_BIND, rtnl_held,
3060 						extack);
3061 			if (IS_ERR(act))
3062 				return PTR_ERR(act);
3063 
3064 			act->type = exts->type = TCA_OLD_COMPAT;
3065 			exts->actions[0] = act;
3066 			exts->nr_actions = 1;
3067 		} else if (exts->action && tb[exts->action]) {
3068 			int err;
3069 
3070 			err = tcf_action_init(net, tp, tb[exts->action],
3071 					      rate_tlv, NULL, ovr, TCA_ACT_BIND,
3072 					      exts->actions, &attr_size,
3073 					      rtnl_held, extack);
3074 			if (err < 0)
3075 				return err;
3076 			exts->nr_actions = err;
3077 		}
3078 	}
3079 #else
3080 	if ((exts->action && tb[exts->action]) ||
3081 	    (exts->police && tb[exts->police])) {
3082 		NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
3083 		return -EOPNOTSUPP;
3084 	}
3085 #endif
3086 
3087 	return 0;
3088 }
3089 EXPORT_SYMBOL(tcf_exts_validate);
3090 
3091 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
3092 {
3093 #ifdef CONFIG_NET_CLS_ACT
3094 	struct tcf_exts old = *dst;
3095 
3096 	*dst = *src;
3097 	tcf_exts_destroy(&old);
3098 #endif
3099 }
3100 EXPORT_SYMBOL(tcf_exts_change);
3101 
3102 #ifdef CONFIG_NET_CLS_ACT
3103 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
3104 {
3105 	if (exts->nr_actions == 0)
3106 		return NULL;
3107 	else
3108 		return exts->actions[0];
3109 }
3110 #endif
3111 
3112 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
3113 {
3114 #ifdef CONFIG_NET_CLS_ACT
3115 	struct nlattr *nest;
3116 
3117 	if (exts->action && tcf_exts_has_actions(exts)) {
3118 		/*
3119 		 * again for backward compatible mode - we want
3120 		 * to work with both old and new modes of entering
3121 		 * tc data even if iproute2  was newer - jhs
3122 		 */
3123 		if (exts->type != TCA_OLD_COMPAT) {
3124 			nest = nla_nest_start_noflag(skb, exts->action);
3125 			if (nest == NULL)
3126 				goto nla_put_failure;
3127 
3128 			if (tcf_action_dump(skb, exts->actions, 0, 0, false)
3129 			    < 0)
3130 				goto nla_put_failure;
3131 			nla_nest_end(skb, nest);
3132 		} else if (exts->police) {
3133 			struct tc_action *act = tcf_exts_first_act(exts);
3134 			nest = nla_nest_start_noflag(skb, exts->police);
3135 			if (nest == NULL || !act)
3136 				goto nla_put_failure;
3137 			if (tcf_action_dump_old(skb, act, 0, 0) < 0)
3138 				goto nla_put_failure;
3139 			nla_nest_end(skb, nest);
3140 		}
3141 	}
3142 	return 0;
3143 
3144 nla_put_failure:
3145 	nla_nest_cancel(skb, nest);
3146 	return -1;
3147 #else
3148 	return 0;
3149 #endif
3150 }
3151 EXPORT_SYMBOL(tcf_exts_dump);
3152 
3153 int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts)
3154 {
3155 #ifdef CONFIG_NET_CLS_ACT
3156 	struct nlattr *nest;
3157 
3158 	if (!exts->action || !tcf_exts_has_actions(exts))
3159 		return 0;
3160 
3161 	nest = nla_nest_start_noflag(skb, exts->action);
3162 	if (!nest)
3163 		goto nla_put_failure;
3164 
3165 	if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0)
3166 		goto nla_put_failure;
3167 	nla_nest_end(skb, nest);
3168 	return 0;
3169 
3170 nla_put_failure:
3171 	nla_nest_cancel(skb, nest);
3172 	return -1;
3173 #else
3174 	return 0;
3175 #endif
3176 }
3177 EXPORT_SYMBOL(tcf_exts_terse_dump);
3178 
3179 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
3180 {
3181 #ifdef CONFIG_NET_CLS_ACT
3182 	struct tc_action *a = tcf_exts_first_act(exts);
3183 	if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
3184 		return -1;
3185 #endif
3186 	return 0;
3187 }
3188 EXPORT_SYMBOL(tcf_exts_dump_stats);
3189 
3190 static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
3191 {
3192 	if (*flags & TCA_CLS_FLAGS_IN_HW)
3193 		return;
3194 	*flags |= TCA_CLS_FLAGS_IN_HW;
3195 	atomic_inc(&block->offloadcnt);
3196 }
3197 
3198 static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
3199 {
3200 	if (!(*flags & TCA_CLS_FLAGS_IN_HW))
3201 		return;
3202 	*flags &= ~TCA_CLS_FLAGS_IN_HW;
3203 	atomic_dec(&block->offloadcnt);
3204 }
3205 
3206 static void tc_cls_offload_cnt_update(struct tcf_block *block,
3207 				      struct tcf_proto *tp, u32 *cnt,
3208 				      u32 *flags, u32 diff, bool add)
3209 {
3210 	lockdep_assert_held(&block->cb_lock);
3211 
3212 	spin_lock(&tp->lock);
3213 	if (add) {
3214 		if (!*cnt)
3215 			tcf_block_offload_inc(block, flags);
3216 		*cnt += diff;
3217 	} else {
3218 		*cnt -= diff;
3219 		if (!*cnt)
3220 			tcf_block_offload_dec(block, flags);
3221 	}
3222 	spin_unlock(&tp->lock);
3223 }
3224 
3225 static void
3226 tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
3227 			 u32 *cnt, u32 *flags)
3228 {
3229 	lockdep_assert_held(&block->cb_lock);
3230 
3231 	spin_lock(&tp->lock);
3232 	tcf_block_offload_dec(block, flags);
3233 	*cnt = 0;
3234 	spin_unlock(&tp->lock);
3235 }
3236 
3237 static int
3238 __tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3239 		   void *type_data, bool err_stop)
3240 {
3241 	struct flow_block_cb *block_cb;
3242 	int ok_count = 0;
3243 	int err;
3244 
3245 	list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
3246 		err = block_cb->cb(type, type_data, block_cb->cb_priv);
3247 		if (err) {
3248 			if (err_stop)
3249 				return err;
3250 		} else {
3251 			ok_count++;
3252 		}
3253 	}
3254 	return ok_count;
3255 }
3256 
3257 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3258 		     void *type_data, bool err_stop, bool rtnl_held)
3259 {
3260 	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3261 	int ok_count;
3262 
3263 retry:
3264 	if (take_rtnl)
3265 		rtnl_lock();
3266 	down_read(&block->cb_lock);
3267 	/* Need to obtain rtnl lock if block is bound to devs that require it.
3268 	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3269 	 * obtain the locks in same order here.
3270 	 */
3271 	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3272 		up_read(&block->cb_lock);
3273 		take_rtnl = true;
3274 		goto retry;
3275 	}
3276 
3277 	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3278 
3279 	up_read(&block->cb_lock);
3280 	if (take_rtnl)
3281 		rtnl_unlock();
3282 	return ok_count;
3283 }
3284 EXPORT_SYMBOL(tc_setup_cb_call);
3285 
3286 /* Non-destructive filter add. If filter that wasn't already in hardware is
3287  * successfully offloaded, increment block offloads counter. On failure,
3288  * previously offloaded filter is considered to be intact and offloads counter
3289  * is not decremented.
3290  */
3291 
3292 int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
3293 		    enum tc_setup_type type, void *type_data, bool err_stop,
3294 		    u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3295 {
3296 	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3297 	int ok_count;
3298 
3299 retry:
3300 	if (take_rtnl)
3301 		rtnl_lock();
3302 	down_read(&block->cb_lock);
3303 	/* Need to obtain rtnl lock if block is bound to devs that require it.
3304 	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3305 	 * obtain the locks in same order here.
3306 	 */
3307 	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3308 		up_read(&block->cb_lock);
3309 		take_rtnl = true;
3310 		goto retry;
3311 	}
3312 
3313 	/* Make sure all netdevs sharing this block are offload-capable. */
3314 	if (block->nooffloaddevcnt && err_stop) {
3315 		ok_count = -EOPNOTSUPP;
3316 		goto err_unlock;
3317 	}
3318 
3319 	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3320 	if (ok_count < 0)
3321 		goto err_unlock;
3322 
3323 	if (tp->ops->hw_add)
3324 		tp->ops->hw_add(tp, type_data);
3325 	if (ok_count > 0)
3326 		tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
3327 					  ok_count, true);
3328 err_unlock:
3329 	up_read(&block->cb_lock);
3330 	if (take_rtnl)
3331 		rtnl_unlock();
3332 	return ok_count < 0 ? ok_count : 0;
3333 }
3334 EXPORT_SYMBOL(tc_setup_cb_add);
3335 
3336 /* Destructive filter replace. If filter that wasn't already in hardware is
3337  * successfully offloaded, increment block offload counter. On failure,
3338  * previously offloaded filter is considered to be destroyed and offload counter
3339  * is decremented.
3340  */
3341 
3342 int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
3343 			enum tc_setup_type type, void *type_data, bool err_stop,
3344 			u32 *old_flags, unsigned int *old_in_hw_count,
3345 			u32 *new_flags, unsigned int *new_in_hw_count,
3346 			bool rtnl_held)
3347 {
3348 	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3349 	int ok_count;
3350 
3351 retry:
3352 	if (take_rtnl)
3353 		rtnl_lock();
3354 	down_read(&block->cb_lock);
3355 	/* Need to obtain rtnl lock if block is bound to devs that require it.
3356 	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3357 	 * obtain the locks in same order here.
3358 	 */
3359 	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3360 		up_read(&block->cb_lock);
3361 		take_rtnl = true;
3362 		goto retry;
3363 	}
3364 
3365 	/* Make sure all netdevs sharing this block are offload-capable. */
3366 	if (block->nooffloaddevcnt && err_stop) {
3367 		ok_count = -EOPNOTSUPP;
3368 		goto err_unlock;
3369 	}
3370 
3371 	tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
3372 	if (tp->ops->hw_del)
3373 		tp->ops->hw_del(tp, type_data);
3374 
3375 	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3376 	if (ok_count < 0)
3377 		goto err_unlock;
3378 
3379 	if (tp->ops->hw_add)
3380 		tp->ops->hw_add(tp, type_data);
3381 	if (ok_count > 0)
3382 		tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
3383 					  new_flags, ok_count, true);
3384 err_unlock:
3385 	up_read(&block->cb_lock);
3386 	if (take_rtnl)
3387 		rtnl_unlock();
3388 	return ok_count < 0 ? ok_count : 0;
3389 }
3390 EXPORT_SYMBOL(tc_setup_cb_replace);
3391 
3392 /* Destroy filter and decrement block offload counter, if filter was previously
3393  * offloaded.
3394  */
3395 
3396 int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
3397 			enum tc_setup_type type, void *type_data, bool err_stop,
3398 			u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3399 {
3400 	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3401 	int ok_count;
3402 
3403 retry:
3404 	if (take_rtnl)
3405 		rtnl_lock();
3406 	down_read(&block->cb_lock);
3407 	/* Need to obtain rtnl lock if block is bound to devs that require it.
3408 	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3409 	 * obtain the locks in same order here.
3410 	 */
3411 	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3412 		up_read(&block->cb_lock);
3413 		take_rtnl = true;
3414 		goto retry;
3415 	}
3416 
3417 	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3418 
3419 	tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
3420 	if (tp->ops->hw_del)
3421 		tp->ops->hw_del(tp, type_data);
3422 
3423 	up_read(&block->cb_lock);
3424 	if (take_rtnl)
3425 		rtnl_unlock();
3426 	return ok_count < 0 ? ok_count : 0;
3427 }
3428 EXPORT_SYMBOL(tc_setup_cb_destroy);
3429 
3430 int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
3431 			  bool add, flow_setup_cb_t *cb,
3432 			  enum tc_setup_type type, void *type_data,
3433 			  void *cb_priv, u32 *flags, unsigned int *in_hw_count)
3434 {
3435 	int err = cb(type, type_data, cb_priv);
3436 
3437 	if (err) {
3438 		if (add && tc_skip_sw(*flags))
3439 			return err;
3440 	} else {
3441 		tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,
3442 					  add);
3443 	}
3444 
3445 	return 0;
3446 }
3447 EXPORT_SYMBOL(tc_setup_cb_reoffload);
3448 
3449 static int tcf_act_get_cookie(struct flow_action_entry *entry,
3450 			      const struct tc_action *act)
3451 {
3452 	struct tc_cookie *cookie;
3453 	int err = 0;
3454 
3455 	rcu_read_lock();
3456 	cookie = rcu_dereference(act->act_cookie);
3457 	if (cookie) {
3458 		entry->cookie = flow_action_cookie_create(cookie->data,
3459 							  cookie->len,
3460 							  GFP_ATOMIC);
3461 		if (!entry->cookie)
3462 			err = -ENOMEM;
3463 	}
3464 	rcu_read_unlock();
3465 	return err;
3466 }
3467 
3468 static void tcf_act_put_cookie(struct flow_action_entry *entry)
3469 {
3470 	flow_action_cookie_destroy(entry->cookie);
3471 }
3472 
3473 void tc_cleanup_flow_action(struct flow_action *flow_action)
3474 {
3475 	struct flow_action_entry *entry;
3476 	int i;
3477 
3478 	flow_action_for_each(i, entry, flow_action) {
3479 		tcf_act_put_cookie(entry);
3480 		if (entry->destructor)
3481 			entry->destructor(entry->destructor_priv);
3482 	}
3483 }
3484 EXPORT_SYMBOL(tc_cleanup_flow_action);
3485 
3486 static void tcf_mirred_get_dev(struct flow_action_entry *entry,
3487 			       const struct tc_action *act)
3488 {
3489 #ifdef CONFIG_NET_CLS_ACT
3490 	entry->dev = act->ops->get_dev(act, &entry->destructor);
3491 	if (!entry->dev)
3492 		return;
3493 	entry->destructor_priv = entry->dev;
3494 #endif
3495 }
3496 
3497 static void tcf_tunnel_encap_put_tunnel(void *priv)
3498 {
3499 	struct ip_tunnel_info *tunnel = priv;
3500 
3501 	kfree(tunnel);
3502 }
3503 
3504 static int tcf_tunnel_encap_get_tunnel(struct flow_action_entry *entry,
3505 				       const struct tc_action *act)
3506 {
3507 	entry->tunnel = tcf_tunnel_info_copy(act);
3508 	if (!entry->tunnel)
3509 		return -ENOMEM;
3510 	entry->destructor = tcf_tunnel_encap_put_tunnel;
3511 	entry->destructor_priv = entry->tunnel;
3512 	return 0;
3513 }
3514 
3515 static void tcf_sample_get_group(struct flow_action_entry *entry,
3516 				 const struct tc_action *act)
3517 {
3518 #ifdef CONFIG_NET_CLS_ACT
3519 	entry->sample.psample_group =
3520 		act->ops->get_psample_group(act, &entry->destructor);
3521 	entry->destructor_priv = entry->sample.psample_group;
3522 #endif
3523 }
3524 
3525 static void tcf_gate_entry_destructor(void *priv)
3526 {
3527 	struct action_gate_entry *oe = priv;
3528 
3529 	kfree(oe);
3530 }
3531 
3532 static int tcf_gate_get_entries(struct flow_action_entry *entry,
3533 				const struct tc_action *act)
3534 {
3535 	entry->gate.entries = tcf_gate_get_list(act);
3536 
3537 	if (!entry->gate.entries)
3538 		return -EINVAL;
3539 
3540 	entry->destructor = tcf_gate_entry_destructor;
3541 	entry->destructor_priv = entry->gate.entries;
3542 
3543 	return 0;
3544 }
3545 
3546 static enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats)
3547 {
3548 	if (WARN_ON_ONCE(hw_stats > TCA_ACT_HW_STATS_ANY))
3549 		return FLOW_ACTION_HW_STATS_DONT_CARE;
3550 	else if (!hw_stats)
3551 		return FLOW_ACTION_HW_STATS_DISABLED;
3552 
3553 	return hw_stats;
3554 }
3555 
3556 int tc_setup_flow_action(struct flow_action *flow_action,
3557 			 const struct tcf_exts *exts)
3558 {
3559 	struct tc_action *act;
3560 	int i, j, k, err = 0;
3561 
3562 	BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY);
3563 	BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE);
3564 	BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED);
3565 
3566 	if (!exts)
3567 		return 0;
3568 
3569 	j = 0;
3570 	tcf_exts_for_each_action(i, act, exts) {
3571 		struct flow_action_entry *entry;
3572 
3573 		entry = &flow_action->entries[j];
3574 		spin_lock_bh(&act->tcfa_lock);
3575 		err = tcf_act_get_cookie(entry, act);
3576 		if (err)
3577 			goto err_out_locked;
3578 
3579 		entry->hw_stats = tc_act_hw_stats(act->hw_stats);
3580 
3581 		if (is_tcf_gact_ok(act)) {
3582 			entry->id = FLOW_ACTION_ACCEPT;
3583 		} else if (is_tcf_gact_shot(act)) {
3584 			entry->id = FLOW_ACTION_DROP;
3585 		} else if (is_tcf_gact_trap(act)) {
3586 			entry->id = FLOW_ACTION_TRAP;
3587 		} else if (is_tcf_gact_goto_chain(act)) {
3588 			entry->id = FLOW_ACTION_GOTO;
3589 			entry->chain_index = tcf_gact_goto_chain_index(act);
3590 		} else if (is_tcf_mirred_egress_redirect(act)) {
3591 			entry->id = FLOW_ACTION_REDIRECT;
3592 			tcf_mirred_get_dev(entry, act);
3593 		} else if (is_tcf_mirred_egress_mirror(act)) {
3594 			entry->id = FLOW_ACTION_MIRRED;
3595 			tcf_mirred_get_dev(entry, act);
3596 		} else if (is_tcf_mirred_ingress_redirect(act)) {
3597 			entry->id = FLOW_ACTION_REDIRECT_INGRESS;
3598 			tcf_mirred_get_dev(entry, act);
3599 		} else if (is_tcf_mirred_ingress_mirror(act)) {
3600 			entry->id = FLOW_ACTION_MIRRED_INGRESS;
3601 			tcf_mirred_get_dev(entry, act);
3602 		} else if (is_tcf_vlan(act)) {
3603 			switch (tcf_vlan_action(act)) {
3604 			case TCA_VLAN_ACT_PUSH:
3605 				entry->id = FLOW_ACTION_VLAN_PUSH;
3606 				entry->vlan.vid = tcf_vlan_push_vid(act);
3607 				entry->vlan.proto = tcf_vlan_push_proto(act);
3608 				entry->vlan.prio = tcf_vlan_push_prio(act);
3609 				break;
3610 			case TCA_VLAN_ACT_POP:
3611 				entry->id = FLOW_ACTION_VLAN_POP;
3612 				break;
3613 			case TCA_VLAN_ACT_MODIFY:
3614 				entry->id = FLOW_ACTION_VLAN_MANGLE;
3615 				entry->vlan.vid = tcf_vlan_push_vid(act);
3616 				entry->vlan.proto = tcf_vlan_push_proto(act);
3617 				entry->vlan.prio = tcf_vlan_push_prio(act);
3618 				break;
3619 			default:
3620 				err = -EOPNOTSUPP;
3621 				goto err_out_locked;
3622 			}
3623 		} else if (is_tcf_tunnel_set(act)) {
3624 			entry->id = FLOW_ACTION_TUNNEL_ENCAP;
3625 			err = tcf_tunnel_encap_get_tunnel(entry, act);
3626 			if (err)
3627 				goto err_out_locked;
3628 		} else if (is_tcf_tunnel_release(act)) {
3629 			entry->id = FLOW_ACTION_TUNNEL_DECAP;
3630 		} else if (is_tcf_pedit(act)) {
3631 			for (k = 0; k < tcf_pedit_nkeys(act); k++) {
3632 				switch (tcf_pedit_cmd(act, k)) {
3633 				case TCA_PEDIT_KEY_EX_CMD_SET:
3634 					entry->id = FLOW_ACTION_MANGLE;
3635 					break;
3636 				case TCA_PEDIT_KEY_EX_CMD_ADD:
3637 					entry->id = FLOW_ACTION_ADD;
3638 					break;
3639 				default:
3640 					err = -EOPNOTSUPP;
3641 					goto err_out_locked;
3642 				}
3643 				entry->mangle.htype = tcf_pedit_htype(act, k);
3644 				entry->mangle.mask = tcf_pedit_mask(act, k);
3645 				entry->mangle.val = tcf_pedit_val(act, k);
3646 				entry->mangle.offset = tcf_pedit_offset(act, k);
3647 				entry->hw_stats = tc_act_hw_stats(act->hw_stats);
3648 				entry = &flow_action->entries[++j];
3649 			}
3650 		} else if (is_tcf_csum(act)) {
3651 			entry->id = FLOW_ACTION_CSUM;
3652 			entry->csum_flags = tcf_csum_update_flags(act);
3653 		} else if (is_tcf_skbedit_mark(act)) {
3654 			entry->id = FLOW_ACTION_MARK;
3655 			entry->mark = tcf_skbedit_mark(act);
3656 		} else if (is_tcf_sample(act)) {
3657 			entry->id = FLOW_ACTION_SAMPLE;
3658 			entry->sample.trunc_size = tcf_sample_trunc_size(act);
3659 			entry->sample.truncate = tcf_sample_truncate(act);
3660 			entry->sample.rate = tcf_sample_rate(act);
3661 			tcf_sample_get_group(entry, act);
3662 		} else if (is_tcf_police(act)) {
3663 			entry->id = FLOW_ACTION_POLICE;
3664 			entry->police.burst = tcf_police_burst(act);
3665 			entry->police.rate_bytes_ps =
3666 				tcf_police_rate_bytes_ps(act);
3667 			entry->police.mtu = tcf_police_tcfp_mtu(act);
3668 			entry->police.index = act->tcfa_index;
3669 		} else if (is_tcf_ct(act)) {
3670 			entry->id = FLOW_ACTION_CT;
3671 			entry->ct.action = tcf_ct_action(act);
3672 			entry->ct.zone = tcf_ct_zone(act);
3673 			entry->ct.flow_table = tcf_ct_ft(act);
3674 		} else if (is_tcf_mpls(act)) {
3675 			switch (tcf_mpls_action(act)) {
3676 			case TCA_MPLS_ACT_PUSH:
3677 				entry->id = FLOW_ACTION_MPLS_PUSH;
3678 				entry->mpls_push.proto = tcf_mpls_proto(act);
3679 				entry->mpls_push.label = tcf_mpls_label(act);
3680 				entry->mpls_push.tc = tcf_mpls_tc(act);
3681 				entry->mpls_push.bos = tcf_mpls_bos(act);
3682 				entry->mpls_push.ttl = tcf_mpls_ttl(act);
3683 				break;
3684 			case TCA_MPLS_ACT_POP:
3685 				entry->id = FLOW_ACTION_MPLS_POP;
3686 				entry->mpls_pop.proto = tcf_mpls_proto(act);
3687 				break;
3688 			case TCA_MPLS_ACT_MODIFY:
3689 				entry->id = FLOW_ACTION_MPLS_MANGLE;
3690 				entry->mpls_mangle.label = tcf_mpls_label(act);
3691 				entry->mpls_mangle.tc = tcf_mpls_tc(act);
3692 				entry->mpls_mangle.bos = tcf_mpls_bos(act);
3693 				entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
3694 				break;
3695 			default:
3696 				goto err_out_locked;
3697 			}
3698 		} else if (is_tcf_skbedit_ptype(act)) {
3699 			entry->id = FLOW_ACTION_PTYPE;
3700 			entry->ptype = tcf_skbedit_ptype(act);
3701 		} else if (is_tcf_skbedit_priority(act)) {
3702 			entry->id = FLOW_ACTION_PRIORITY;
3703 			entry->priority = tcf_skbedit_priority(act);
3704 		} else if (is_tcf_gate(act)) {
3705 			entry->id = FLOW_ACTION_GATE;
3706 			entry->gate.index = tcf_gate_index(act);
3707 			entry->gate.prio = tcf_gate_prio(act);
3708 			entry->gate.basetime = tcf_gate_basetime(act);
3709 			entry->gate.cycletime = tcf_gate_cycletime(act);
3710 			entry->gate.cycletimeext = tcf_gate_cycletimeext(act);
3711 			entry->gate.num_entries = tcf_gate_num_entries(act);
3712 			err = tcf_gate_get_entries(entry, act);
3713 			if (err)
3714 				goto err_out;
3715 		} else {
3716 			err = -EOPNOTSUPP;
3717 			goto err_out_locked;
3718 		}
3719 		spin_unlock_bh(&act->tcfa_lock);
3720 
3721 		if (!is_tcf_pedit(act))
3722 			j++;
3723 	}
3724 
3725 err_out:
3726 	if (err)
3727 		tc_cleanup_flow_action(flow_action);
3728 
3729 	return err;
3730 err_out_locked:
3731 	spin_unlock_bh(&act->tcfa_lock);
3732 	goto err_out;
3733 }
3734 EXPORT_SYMBOL(tc_setup_flow_action);
3735 
3736 unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3737 {
3738 	unsigned int num_acts = 0;
3739 	struct tc_action *act;
3740 	int i;
3741 
3742 	tcf_exts_for_each_action(i, act, exts) {
3743 		if (is_tcf_pedit(act))
3744 			num_acts += tcf_pedit_nkeys(act);
3745 		else
3746 			num_acts++;
3747 	}
3748 	return num_acts;
3749 }
3750 EXPORT_SYMBOL(tcf_exts_num_actions);
3751 
3752 #ifdef CONFIG_NET_CLS_ACT
3753 static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr,
3754 					u32 *p_block_index,
3755 					struct netlink_ext_ack *extack)
3756 {
3757 	*p_block_index = nla_get_u32(block_index_attr);
3758 	if (!*p_block_index) {
3759 		NL_SET_ERR_MSG(extack, "Block number may not be zero");
3760 		return -EINVAL;
3761 	}
3762 
3763 	return 0;
3764 }
3765 
3766 int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
3767 		    enum flow_block_binder_type binder_type,
3768 		    struct nlattr *block_index_attr,
3769 		    struct netlink_ext_ack *extack)
3770 {
3771 	u32 block_index;
3772 	int err;
3773 
3774 	if (!block_index_attr)
3775 		return 0;
3776 
3777 	err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
3778 	if (err)
3779 		return err;
3780 
3781 	if (!block_index)
3782 		return 0;
3783 
3784 	qe->info.binder_type = binder_type;
3785 	qe->info.chain_head_change = tcf_chain_head_change_dflt;
3786 	qe->info.chain_head_change_priv = &qe->filter_chain;
3787 	qe->info.block_index = block_index;
3788 
3789 	return tcf_block_get_ext(&qe->block, sch, &qe->info, extack);
3790 }
3791 EXPORT_SYMBOL(tcf_qevent_init);
3792 
3793 void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
3794 {
3795 	if (qe->info.block_index)
3796 		tcf_block_put_ext(qe->block, sch, &qe->info);
3797 }
3798 EXPORT_SYMBOL(tcf_qevent_destroy);
3799 
3800 int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
3801 			       struct netlink_ext_ack *extack)
3802 {
3803 	u32 block_index;
3804 	int err;
3805 
3806 	if (!block_index_attr)
3807 		return 0;
3808 
3809 	err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
3810 	if (err)
3811 		return err;
3812 
3813 	/* Bounce newly-configured block or change in block. */
3814 	if (block_index != qe->info.block_index) {
3815 		NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
3816 		return -EINVAL;
3817 	}
3818 
3819 	return 0;
3820 }
3821 EXPORT_SYMBOL(tcf_qevent_validate_change);
3822 
3823 struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
3824 				  struct sk_buff **to_free, int *ret)
3825 {
3826 	struct tcf_result cl_res;
3827 	struct tcf_proto *fl;
3828 
3829 	if (!qe->info.block_index)
3830 		return skb;
3831 
3832 	fl = rcu_dereference_bh(qe->filter_chain);
3833 
3834 	switch (tcf_classify(skb, fl, &cl_res, false)) {
3835 	case TC_ACT_SHOT:
3836 		qdisc_qstats_drop(sch);
3837 		__qdisc_drop(skb, to_free);
3838 		*ret = __NET_XMIT_BYPASS;
3839 		return NULL;
3840 	case TC_ACT_STOLEN:
3841 	case TC_ACT_QUEUED:
3842 	case TC_ACT_TRAP:
3843 		__qdisc_drop(skb, to_free);
3844 		*ret = __NET_XMIT_STOLEN;
3845 		return NULL;
3846 	case TC_ACT_REDIRECT:
3847 		skb_do_redirect(skb);
3848 		*ret = __NET_XMIT_STOLEN;
3849 		return NULL;
3850 	}
3851 
3852 	return skb;
3853 }
3854 EXPORT_SYMBOL(tcf_qevent_handle);
3855 
3856 int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
3857 {
3858 	if (!qe->info.block_index)
3859 		return 0;
3860 	return nla_put_u32(skb, attr_name, qe->info.block_index);
3861 }
3862 EXPORT_SYMBOL(tcf_qevent_dump);
3863 #endif
3864 
3865 static __net_init int tcf_net_init(struct net *net)
3866 {
3867 	struct tcf_net *tn = net_generic(net, tcf_net_id);
3868 
3869 	spin_lock_init(&tn->idr_lock);
3870 	idr_init(&tn->idr);
3871 	return 0;
3872 }
3873 
3874 static void __net_exit tcf_net_exit(struct net *net)
3875 {
3876 	struct tcf_net *tn = net_generic(net, tcf_net_id);
3877 
3878 	idr_destroy(&tn->idr);
3879 }
3880 
3881 static struct pernet_operations tcf_net_ops = {
3882 	.init = tcf_net_init,
3883 	.exit = tcf_net_exit,
3884 	.id   = &tcf_net_id,
3885 	.size = sizeof(struct tcf_net),
3886 };
3887 
3888 static int __init tc_filter_init(void)
3889 {
3890 	int err;
3891 
3892 	tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
3893 	if (!tc_filter_wq)
3894 		return -ENOMEM;
3895 
3896 	err = register_pernet_subsys(&tcf_net_ops);
3897 	if (err)
3898 		goto err_register_pernet_subsys;
3899 
3900 	rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
3901 		      RTNL_FLAG_DOIT_UNLOCKED);
3902 	rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
3903 		      RTNL_FLAG_DOIT_UNLOCKED);
3904 	rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
3905 		      tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED);
3906 	rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
3907 	rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
3908 	rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
3909 		      tc_dump_chain, 0);
3910 
3911 	return 0;
3912 
3913 err_register_pernet_subsys:
3914 	destroy_workqueue(tc_filter_wq);
3915 	return err;
3916 }
3917 
3918 subsys_initcall(tc_filter_init);
3919