133 	struct Qdisc_ops *q, **qp;  in register_qdisc()  local
137 	for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)  in register_qdisc()
138 		if (!strcmp(qops->id, q->id))  in register_qdisc()
177 	struct Qdisc_ops *q, **qp;  in unregister_qdisc()  local
181 	for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)  in unregister_qdisc()
182 		if (q == qops)  in unregister_qdisc()
184 	if (q) {  in unregister_qdisc()
185 		*qp = q->next;  in unregister_qdisc()
186 		q->next = NULL;  in unregister_qdisc()
205 	struct Qdisc_ops *q = NULL;  in qdisc_lookup_default()  local
207 	for (q = qdisc_base; q; q = q->next) {  in qdisc_lookup_default()
208 		if (!strcmp(name, q->id)) {  in qdisc_lookup_default()
209 			if (!try_module_get(q->owner))  in qdisc_lookup_default()
210 				q = NULL;  in qdisc_lookup_default()
215 	return q;  in qdisc_lookup_default()
263 	struct Qdisc *q;  in qdisc_match_from_root()  local
272 	hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle,  in qdisc_match_from_root()
274 		if (q->handle == handle)  in qdisc_match_from_root()
275 			return q;  in qdisc_match_from_root()
280 void qdisc_hash_add(struct Qdisc *q, bool invisible)  in qdisc_hash_add()  argument
282 	if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {  in qdisc_hash_add()
284 		hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);  in qdisc_hash_add()
286 			q->flags |= TCQ_F_INVISIBLE;  in qdisc_hash_add()
291 void qdisc_hash_del(struct Qdisc *q)  in qdisc_hash_del()  argument
293 	if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {  in qdisc_hash_del()
295 		hash_del_rcu(&q->hash);  in qdisc_hash_del()
302 	struct Qdisc *q;  in qdisc_lookup()  local
306 	q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle);  in qdisc_lookup()
307 	if (q)  in qdisc_lookup()
311 		q = qdisc_match_from_root(  in qdisc_lookup()
315 	return q;  in qdisc_lookup()
321 	struct Qdisc *q;  in qdisc_lookup_rcu()  local
325 	q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle);  in qdisc_lookup_rcu()
326 	if (q)  in qdisc_lookup_rcu()
331 		q = qdisc_match_from_root(rcu_dereference(nq->qdisc_sleeping),  in qdisc_lookup_rcu()
334 	return q;  in qdisc_lookup_rcu()
355 	struct Qdisc_ops *q = NULL;  in qdisc_lookup_ops()  local
359 		for (q = qdisc_base; q; q = q->next) {  in qdisc_lookup_ops()
360 			if (nla_strcmp(kind, q->id) == 0) {  in qdisc_lookup_ops()
361 				if (!try_module_get(q->owner))  in qdisc_lookup_ops()
362 					q = NULL;  in qdisc_lookup_ops()
368 	return q;  in qdisc_lookup_ops()
809 		notify = !sch->q.qlen && !WARN_ON_ONCE(!n &&  in qdisc_tree_reduce_backlog()
822 		sch->q.qlen -= n;  in qdisc_tree_reduce_backlog()
912 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,  in tc_fill_qdisc()  argument
934 	tcm->tcm_ifindex = qdisc_dev(q)->ifindex;  in tc_fill_qdisc()
936 	tcm->tcm_handle = q->handle;  in tc_fill_qdisc()
937 	tcm->tcm_info = refcount_read(&q->refcnt);  in tc_fill_qdisc()
938 	if (nla_put_string(skb, TCA_KIND, q->ops->id))  in tc_fill_qdisc()
940 	if (q->ops->ingress_block_get) {  in tc_fill_qdisc()
941 		block_index = q->ops->ingress_block_get(q);  in tc_fill_qdisc()
946 	if (q->ops->egress_block_get) {  in tc_fill_qdisc()
947 		block_index = q->ops->egress_block_get(q);  in tc_fill_qdisc()
952 	if (q->ops->dump && q->ops->dump(q, skb) < 0)  in tc_fill_qdisc()
954 	if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED)))  in tc_fill_qdisc()
956 	qlen = qdisc_qlen_sum(q);  in tc_fill_qdisc()
958 	stab = rtnl_dereference(q->stab);  in tc_fill_qdisc()
966 	if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)  in tc_fill_qdisc()
969 	if (qdisc_is_percpu_stats(q)) {  in tc_fill_qdisc()
970 		cpu_bstats = q->cpu_bstats;  in tc_fill_qdisc()
971 		cpu_qstats = q->cpu_qstats;  in tc_fill_qdisc()
974 	if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats, true) < 0 ||  in tc_fill_qdisc()
975 	    gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||  in tc_fill_qdisc()
976 	    gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)  in tc_fill_qdisc()
996 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)  in tc_qdisc_dump_ignore()  argument
998 	if (q->flags & TCQ_F_BUILTIN)  in tc_qdisc_dump_ignore()
1000 	if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)  in tc_qdisc_dump_ignore()
1077 	struct Qdisc *q = old;  in qdisc_graft()  local
1086 		if ((q && q->flags & TCQ_F_INGRESS) ||  in qdisc_graft()
1095 			q = rtnl_dereference(dev_queue->qdisc_sleeping);  in qdisc_graft()
1100 			if (!qdisc_refcount_dec_if_one(q)) {  in qdisc_graft()
1425 static int check_loop_fn(struct Qdisc *q, unsigned long cl,
1428 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)  in check_loop()  argument
1432 	if (q->ops->cl_ops == NULL)  in check_loop()
1439 	q->ops->cl_ops->walk(q, &arg.w);  in check_loop()
1444 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)  in check_loop_fn()  argument
1447 	const struct Qdisc_class_ops *cops = q->ops->cl_ops;  in check_loop_fn()
1450 	leaf = cops->leaf(q, cl);  in check_loop_fn()
1482 	struct Qdisc *q = NULL;  in tc_get_qdisc()  local
1504 				q = qdisc_leaf(p, clid);  in tc_get_qdisc()
1506 				q = rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping);  in tc_get_qdisc()
1509 			q = rtnl_dereference(dev->qdisc);  in tc_get_qdisc()
1511 		if (!q) {  in tc_get_qdisc()
1516 		if (tcm->tcm_handle && q->handle != tcm->tcm_handle) {  in tc_get_qdisc()
1521 		q = qdisc_lookup(dev, tcm->tcm_handle);  in tc_get_qdisc()
1522 		if (!q) {  in tc_get_qdisc()
1528 	if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {  in tc_get_qdisc()
1538 		if (q->handle == 0) {  in tc_get_qdisc()
1542 		err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack);  in tc_get_qdisc()
1546 		qdisc_notify(net, skb, n, clid, NULL, q, NULL);  in tc_get_qdisc()
1581 	struct Qdisc *q, *p;  in tc_modify_qdisc()  local
1593 	q = p = NULL;  in tc_modify_qdisc()
1608 				q = qdisc_leaf(p, clid);  in tc_modify_qdisc()
1610 				q = rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping);  in tc_modify_qdisc()
1613 			q = rtnl_dereference(dev->qdisc);  in tc_modify_qdisc()
1617 		if (q && q->handle == 0)  in tc_modify_qdisc()
1618 			q = NULL;  in tc_modify_qdisc()
1620 		if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {  in tc_modify_qdisc()
1622 				if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) {  in tc_modify_qdisc()
1630 				q = qdisc_lookup(dev, tcm->tcm_handle);  in tc_modify_qdisc()
1631 				if (!q)  in tc_modify_qdisc()
1638 				    nla_strcmp(tca[TCA_KIND], q->ops->id)) {  in tc_modify_qdisc()
1642 				if (q->flags & TCQ_F_INGRESS) {  in tc_modify_qdisc()
1647 				if (q == p ||  in tc_modify_qdisc()
1648 				    (p && check_loop(q, p, 0))) {  in tc_modify_qdisc()
1656 				qdisc_refcount_inc(q);  in tc_modify_qdisc()
1659 				if (!q)  in tc_modify_qdisc()
1688 				    nla_strcmp(tca[TCA_KIND], q->ops->id)) {  in tc_modify_qdisc()
1702 		q = qdisc_lookup(dev, tcm->tcm_handle);  in tc_modify_qdisc()
1706 	if (!q) {  in tc_modify_qdisc()
1714 	if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {  in tc_modify_qdisc()
1718 	err = qdisc_change(q, tca, extack);  in tc_modify_qdisc()
1720 		qdisc_notify(net, skb, n, clid, NULL, q, extack);  in tc_modify_qdisc()
1731 			q = qdisc_create(dev, dev_ingress_queue(dev),  in tc_modify_qdisc()
1748 		q = qdisc_create(dev, dev_queue,  in tc_modify_qdisc()
1752 	if (q == NULL) {  in tc_modify_qdisc()
1759 	err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack);  in tc_modify_qdisc()
1761 		if (q)  in tc_modify_qdisc()
1762 			qdisc_put(q);  in tc_modify_qdisc()
1775 	struct Qdisc *q;  in tc_dump_qdisc_root()  local
1781 	q = root;  in tc_dump_qdisc_root()
1785 		if (!tc_qdisc_dump_ignore(q, dump_invisible) &&  in tc_dump_qdisc_root()
1786 		    tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,  in tc_dump_qdisc_root()
1802 	hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {  in tc_dump_qdisc_root()
1807 		if (!tc_qdisc_dump_ignore(q, dump_invisible) &&  in tc_dump_qdisc_root()
1808 		    tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,  in tc_dump_qdisc_root()
1882 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,  in tc_fill_tclass()  argument
1890 	const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;  in tc_fill_tclass()
1900 	tcm->tcm_ifindex = qdisc_dev(q)->ifindex;  in tc_fill_tclass()
1901 	tcm->tcm_parent = q->handle;  in tc_fill_tclass()
1902 	tcm->tcm_handle = q->handle;  in tc_fill_tclass()
1904 	if (nla_put_string(skb, TCA_KIND, q->ops->id))  in tc_fill_tclass()
1906 	if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)  in tc_fill_tclass()
1913 	if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)  in tc_fill_tclass()
1934 			 struct nlmsghdr *n, struct Qdisc *q,  in tclass_notify()  argument
1944 	if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event, extack) < 0) {  in tclass_notify()
1956 			     struct Qdisc *q, unsigned long cl,  in tclass_del_notify()  argument
1970 	if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,  in tclass_del_notify()
1976 	err = cops->delete(q, cl, extack);  in tclass_del_notify()
2001 		struct Qdisc *q = tcf_block_q(tp->chain->block);  in tcf_node_bind()  local
2003 		sch_tree_lock(q);  in tcf_node_bind()
2004 		tp->ops->bind_class(n, a->classid, a->cl, q, a->base);  in tcf_node_bind()
2005 		sch_tree_unlock(q);  in tcf_node_bind()
2017 static int tc_bind_class_walker(struct Qdisc *q, unsigned long cl,  in tc_bind_class_walker()  argument
2021 	const struct Qdisc_class_ops *cops = q->ops->cl_ops;  in tc_bind_class_walker()
2025 	block = cops->tcf_block(q, cl, NULL);  in tc_bind_class_walker()
2048 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,  in tc_bind_tclass()  argument
2051 	const struct Qdisc_class_ops *cops = q->ops->cl_ops;  in tc_bind_tclass()
2060 	q->ops->cl_ops->walk(q, &args.w);  in tc_bind_tclass()
2065 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,  in tc_bind_tclass()  argument
2079 	struct Qdisc *q = NULL;  in tc_ctl_tclass()  local
2141 	q = qdisc_lookup(dev, qid);  in tc_ctl_tclass()
2142 	if (!q)  in tc_ctl_tclass()
2146 	cops = q->ops->cl_ops;  in tc_ctl_tclass()
2158 		cl = cops->find(q, clid);  in tc_ctl_tclass()
2173 			err = tclass_del_notify(net, cops, skb, n, q, cl, extack);  in tc_ctl_tclass()
2175 			tc_bind_tclass(q, portid, clid, 0);  in tc_ctl_tclass()
2178 			err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS, extack);  in tc_ctl_tclass()
2194 		err = cops->change(q, clid, portid, tca, &new_cl, extack);  in tc_ctl_tclass()
2196 		tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS, extack);  in tc_ctl_tclass()
2199 			tc_bind_tclass(q, portid, clid, new_cl);  in tc_ctl_tclass()
2211 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl,  in qdisc_class_dump()  argument
2216 	return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,  in qdisc_class_dump()
2221 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,  in tc_dump_tclass_qdisc()  argument
2227 	if (tc_qdisc_dump_ignore(q, false) ||  in tc_dump_tclass_qdisc()
2228 	    *t_p < s_t || !q->ops->cl_ops ||  in tc_dump_tclass_qdisc()
2230 	     TC_H_MAJ(tcm->tcm_parent) != q->handle)) {  in tc_dump_tclass_qdisc()
2242 	q->ops->cl_ops->walk(q, &arg.w);  in tc_dump_tclass_qdisc()
2254 	struct Qdisc *q;  in tc_dump_tclass_root()  local
2267 		q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent));  in tc_dump_tclass_root()
2268 		if (q && q != root &&  in tc_dump_tclass_root()
2269 		    tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)  in tc_dump_tclass_root()
2273 	hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {  in tc_dump_tclass_root()
2274 		if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)  in tc_dump_tclass_root()