1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/cls_api.c Packet classifier API. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * 7 * Changes: 8 * 9 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support 10 */ 11 12 #include <linux/module.h> 13 #include <linux/types.h> 14 #include <linux/kernel.h> 15 #include <linux/string.h> 16 #include <linux/errno.h> 17 #include <linux/err.h> 18 #include <linux/skbuff.h> 19 #include <linux/init.h> 20 #include <linux/kmod.h> 21 #include <linux/slab.h> 22 #include <linux/idr.h> 23 #include <linux/rhashtable.h> 24 #include <net/net_namespace.h> 25 #include <net/sock.h> 26 #include <net/netlink.h> 27 #include <net/pkt_sched.h> 28 #include <net/pkt_cls.h> 29 #include <net/tc_act/tc_pedit.h> 30 #include <net/tc_act/tc_mirred.h> 31 #include <net/tc_act/tc_vlan.h> 32 #include <net/tc_act/tc_tunnel_key.h> 33 #include <net/tc_act/tc_csum.h> 34 #include <net/tc_act/tc_gact.h> 35 #include <net/tc_act/tc_police.h> 36 #include <net/tc_act/tc_sample.h> 37 #include <net/tc_act/tc_skbedit.h> 38 39 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1]; 40 41 /* The list of all installed classifier types */ 42 static LIST_HEAD(tcf_proto_base); 43 44 /* Protects list of registered TC modules. It is pure SMP lock. */ 45 static DEFINE_RWLOCK(cls_mod_lock); 46 47 /* Find classifier type by string name */ 48 49 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind) 50 { 51 const struct tcf_proto_ops *t, *res = NULL; 52 53 if (kind) { 54 read_lock(&cls_mod_lock); 55 list_for_each_entry(t, &tcf_proto_base, head) { 56 if (strcmp(kind, t->kind) == 0) { 57 if (try_module_get(t->owner)) 58 res = t; 59 break; 60 } 61 } 62 read_unlock(&cls_mod_lock); 63 } 64 return res; 65 } 66 67 static const struct tcf_proto_ops * 68 tcf_proto_lookup_ops(const char *kind, bool rtnl_held, 69 struct netlink_ext_ack *extack) 70 { 71 const struct tcf_proto_ops *ops; 72 73 ops = __tcf_proto_lookup_ops(kind); 74 if (ops) 75 return ops; 76 #ifdef CONFIG_MODULES 77 if (rtnl_held) 78 rtnl_unlock(); 79 request_module("cls_%s", kind); 80 if (rtnl_held) 81 rtnl_lock(); 82 ops = __tcf_proto_lookup_ops(kind); 83 /* We dropped the RTNL semaphore in order to perform 84 * the module load. So, even if we succeeded in loading 85 * the module we have to replay the request. We indicate 86 * this using -EAGAIN. 87 */ 88 if (ops) { 89 module_put(ops->owner); 90 return ERR_PTR(-EAGAIN); 91 } 92 #endif 93 NL_SET_ERR_MSG(extack, "TC classifier not found"); 94 return ERR_PTR(-ENOENT); 95 } 96 97 /* Register(unregister) new classifier type */ 98 99 int register_tcf_proto_ops(struct tcf_proto_ops *ops) 100 { 101 struct tcf_proto_ops *t; 102 int rc = -EEXIST; 103 104 write_lock(&cls_mod_lock); 105 list_for_each_entry(t, &tcf_proto_base, head) 106 if (!strcmp(ops->kind, t->kind)) 107 goto out; 108 109 list_add_tail(&ops->head, &tcf_proto_base); 110 rc = 0; 111 out: 112 write_unlock(&cls_mod_lock); 113 return rc; 114 } 115 EXPORT_SYMBOL(register_tcf_proto_ops); 116 117 static struct workqueue_struct *tc_filter_wq; 118 119 int unregister_tcf_proto_ops(struct tcf_proto_ops *ops) 120 { 121 struct tcf_proto_ops *t; 122 int rc = -ENOENT; 123 124 /* Wait for outstanding call_rcu()s, if any, from a 125 * tcf_proto_ops's destroy() handler. 126 */ 127 rcu_barrier(); 128 flush_workqueue(tc_filter_wq); 129 130 write_lock(&cls_mod_lock); 131 list_for_each_entry(t, &tcf_proto_base, head) { 132 if (t == ops) { 133 list_del(&t->head); 134 rc = 0; 135 break; 136 } 137 } 138 write_unlock(&cls_mod_lock); 139 return rc; 140 } 141 EXPORT_SYMBOL(unregister_tcf_proto_ops); 142 143 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func) 144 { 145 INIT_RCU_WORK(rwork, func); 146 return queue_rcu_work(tc_filter_wq, rwork); 147 } 148 EXPORT_SYMBOL(tcf_queue_work); 149 150 /* Select new prio value from the range, managed by kernel. */ 151 152 static inline u32 tcf_auto_prio(struct tcf_proto *tp) 153 { 154 u32 first = TC_H_MAKE(0xC0000000U, 0U); 155 156 if (tp) 157 first = tp->prio - 1; 158 159 return TC_H_MAJ(first); 160 } 161 162 static bool tcf_proto_is_unlocked(const char *kind) 163 { 164 const struct tcf_proto_ops *ops; 165 bool ret; 166 167 ops = tcf_proto_lookup_ops(kind, false, NULL); 168 /* On error return false to take rtnl lock. Proto lookup/create 169 * functions will perform lookup again and properly handle errors. 170 */ 171 if (IS_ERR(ops)) 172 return false; 173 174 ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED); 175 module_put(ops->owner); 176 return ret; 177 } 178 179 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol, 180 u32 prio, struct tcf_chain *chain, 181 bool rtnl_held, 182 struct netlink_ext_ack *extack) 183 { 184 struct tcf_proto *tp; 185 int err; 186 187 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 188 if (!tp) 189 return ERR_PTR(-ENOBUFS); 190 191 tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack); 192 if (IS_ERR(tp->ops)) { 193 err = PTR_ERR(tp->ops); 194 goto errout; 195 } 196 tp->classify = tp->ops->classify; 197 tp->protocol = protocol; 198 tp->prio = prio; 199 tp->chain = chain; 200 spin_lock_init(&tp->lock); 201 refcount_set(&tp->refcnt, 1); 202 203 err = tp->ops->init(tp); 204 if (err) { 205 module_put(tp->ops->owner); 206 goto errout; 207 } 208 return tp; 209 210 errout: 211 kfree(tp); 212 return ERR_PTR(err); 213 } 214 215 static void tcf_proto_get(struct tcf_proto *tp) 216 { 217 refcount_inc(&tp->refcnt); 218 } 219 220 static void tcf_chain_put(struct tcf_chain *chain); 221 222 static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held, 223 struct netlink_ext_ack *extack) 224 { 225 tp->ops->destroy(tp, rtnl_held, extack); 226 tcf_chain_put(tp->chain); 227 module_put(tp->ops->owner); 228 kfree_rcu(tp, rcu); 229 } 230 231 static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held, 232 struct netlink_ext_ack *extack) 233 { 234 if (refcount_dec_and_test(&tp->refcnt)) 235 tcf_proto_destroy(tp, rtnl_held, extack); 236 } 237 238 static int walker_check_empty(struct tcf_proto *tp, void *fh, 239 struct tcf_walker *arg) 240 { 241 if (fh) { 242 arg->nonempty = true; 243 return -1; 244 } 245 return 0; 246 } 247 248 static bool tcf_proto_is_empty(struct tcf_proto *tp, bool rtnl_held) 249 { 250 struct tcf_walker walker = { .fn = walker_check_empty, }; 251 252 if (tp->ops->walk) { 253 tp->ops->walk(tp, &walker, rtnl_held); 254 return !walker.nonempty; 255 } 256 return true; 257 } 258 259 static bool tcf_proto_check_delete(struct tcf_proto *tp, bool rtnl_held) 260 { 261 spin_lock(&tp->lock); 262 if (tcf_proto_is_empty(tp, rtnl_held)) 263 tp->deleting = true; 264 spin_unlock(&tp->lock); 265 return tp->deleting; 266 } 267 268 static void tcf_proto_mark_delete(struct tcf_proto *tp) 269 { 270 spin_lock(&tp->lock); 271 tp->deleting = true; 272 spin_unlock(&tp->lock); 273 } 274 275 static bool tcf_proto_is_deleting(struct tcf_proto *tp) 276 { 277 bool deleting; 278 279 spin_lock(&tp->lock); 280 deleting = tp->deleting; 281 spin_unlock(&tp->lock); 282 283 return deleting; 284 } 285 286 #define ASSERT_BLOCK_LOCKED(block) \ 287 lockdep_assert_held(&(block)->lock) 288 289 struct tcf_filter_chain_list_item { 290 struct list_head list; 291 tcf_chain_head_change_t *chain_head_change; 292 void *chain_head_change_priv; 293 }; 294 295 static struct tcf_chain *tcf_chain_create(struct tcf_block *block, 296 u32 chain_index) 297 { 298 struct tcf_chain *chain; 299 300 ASSERT_BLOCK_LOCKED(block); 301 302 chain = kzalloc(sizeof(*chain), GFP_KERNEL); 303 if (!chain) 304 return NULL; 305 list_add_tail(&chain->list, &block->chain_list); 306 mutex_init(&chain->filter_chain_lock); 307 chain->block = block; 308 chain->index = chain_index; 309 chain->refcnt = 1; 310 if (!chain->index) 311 block->chain0.chain = chain; 312 return chain; 313 } 314 315 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item, 316 struct tcf_proto *tp_head) 317 { 318 if (item->chain_head_change) 319 item->chain_head_change(tp_head, item->chain_head_change_priv); 320 } 321 322 static void tcf_chain0_head_change(struct tcf_chain *chain, 323 struct tcf_proto *tp_head) 324 { 325 struct tcf_filter_chain_list_item *item; 326 struct tcf_block *block = chain->block; 327 328 if (chain->index) 329 return; 330 331 mutex_lock(&block->lock); 332 list_for_each_entry(item, &block->chain0.filter_chain_list, list) 333 tcf_chain_head_change_item(item, tp_head); 334 mutex_unlock(&block->lock); 335 } 336 337 /* Returns true if block can be safely freed. */ 338 339 static bool tcf_chain_detach(struct tcf_chain *chain) 340 { 341 struct tcf_block *block = chain->block; 342 343 ASSERT_BLOCK_LOCKED(block); 344 345 list_del(&chain->list); 346 if (!chain->index) 347 block->chain0.chain = NULL; 348 349 if (list_empty(&block->chain_list) && 350 refcount_read(&block->refcnt) == 0) 351 return true; 352 353 return false; 354 } 355 356 static void tcf_block_destroy(struct tcf_block *block) 357 { 358 mutex_destroy(&block->lock); 359 kfree_rcu(block, rcu); 360 } 361 362 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block) 363 { 364 struct tcf_block *block = chain->block; 365 366 mutex_destroy(&chain->filter_chain_lock); 367 kfree_rcu(chain, rcu); 368 if (free_block) 369 tcf_block_destroy(block); 370 } 371 372 static void tcf_chain_hold(struct tcf_chain *chain) 373 { 374 ASSERT_BLOCK_LOCKED(chain->block); 375 376 ++chain->refcnt; 377 } 378 379 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain) 380 { 381 ASSERT_BLOCK_LOCKED(chain->block); 382 383 /* In case all the references are action references, this 384 * chain should not be shown to the user. 385 */ 386 return chain->refcnt == chain->action_refcnt; 387 } 388 389 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block, 390 u32 chain_index) 391 { 392 struct tcf_chain *chain; 393 394 ASSERT_BLOCK_LOCKED(block); 395 396 list_for_each_entry(chain, &block->chain_list, list) { 397 if (chain->index == chain_index) 398 return chain; 399 } 400 return NULL; 401 } 402 403 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, 404 u32 seq, u16 flags, int event, bool unicast); 405 406 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block, 407 u32 chain_index, bool create, 408 bool by_act) 409 { 410 struct tcf_chain *chain = NULL; 411 bool is_first_reference; 412 413 mutex_lock(&block->lock); 414 chain = tcf_chain_lookup(block, chain_index); 415 if (chain) { 416 tcf_chain_hold(chain); 417 } else { 418 if (!create) 419 goto errout; 420 chain = tcf_chain_create(block, chain_index); 421 if (!chain) 422 goto errout; 423 } 424 425 if (by_act) 426 ++chain->action_refcnt; 427 is_first_reference = chain->refcnt - chain->action_refcnt == 1; 428 mutex_unlock(&block->lock); 429 430 /* Send notification only in case we got the first 431 * non-action reference. Until then, the chain acts only as 432 * a placeholder for actions pointing to it and user ought 433 * not know about them. 434 */ 435 if (is_first_reference && !by_act) 436 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL, 437 RTM_NEWCHAIN, false); 438 439 return chain; 440 441 errout: 442 mutex_unlock(&block->lock); 443 return chain; 444 } 445 446 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index, 447 bool create) 448 { 449 return __tcf_chain_get(block, chain_index, create, false); 450 } 451 452 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index) 453 { 454 return __tcf_chain_get(block, chain_index, true, true); 455 } 456 EXPORT_SYMBOL(tcf_chain_get_by_act); 457 458 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops, 459 void *tmplt_priv); 460 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, 461 void *tmplt_priv, u32 chain_index, 462 struct tcf_block *block, struct sk_buff *oskb, 463 u32 seq, u16 flags, bool unicast); 464 465 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act, 466 bool explicitly_created) 467 { 468 struct tcf_block *block = chain->block; 469 const struct tcf_proto_ops *tmplt_ops; 470 bool free_block = false; 471 unsigned int refcnt; 472 void *tmplt_priv; 473 474 mutex_lock(&block->lock); 475 if (explicitly_created) { 476 if (!chain->explicitly_created) { 477 mutex_unlock(&block->lock); 478 return; 479 } 480 chain->explicitly_created = false; 481 } 482 483 if (by_act) 484 chain->action_refcnt--; 485 486 /* tc_chain_notify_delete can't be called while holding block lock. 487 * However, when block is unlocked chain can be changed concurrently, so 488 * save these to temporary variables. 489 */ 490 refcnt = --chain->refcnt; 491 tmplt_ops = chain->tmplt_ops; 492 tmplt_priv = chain->tmplt_priv; 493 494 /* The last dropped non-action reference will trigger notification. */ 495 if (refcnt - chain->action_refcnt == 0 && !by_act) { 496 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index, 497 block, NULL, 0, 0, false); 498 /* Last reference to chain, no need to lock. */ 499 chain->flushing = false; 500 } 501 502 if (refcnt == 0) 503 free_block = tcf_chain_detach(chain); 504 mutex_unlock(&block->lock); 505 506 if (refcnt == 0) { 507 tc_chain_tmplt_del(tmplt_ops, tmplt_priv); 508 tcf_chain_destroy(chain, free_block); 509 } 510 } 511 512 static void tcf_chain_put(struct tcf_chain *chain) 513 { 514 __tcf_chain_put(chain, false, false); 515 } 516 517 void tcf_chain_put_by_act(struct tcf_chain *chain) 518 { 519 __tcf_chain_put(chain, true, false); 520 } 521 EXPORT_SYMBOL(tcf_chain_put_by_act); 522 523 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain) 524 { 525 __tcf_chain_put(chain, false, true); 526 } 527 528 static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held) 529 { 530 struct tcf_proto *tp, *tp_next; 531 532 mutex_lock(&chain->filter_chain_lock); 533 tp = tcf_chain_dereference(chain->filter_chain, chain); 534 RCU_INIT_POINTER(chain->filter_chain, NULL); 535 tcf_chain0_head_change(chain, NULL); 536 chain->flushing = true; 537 mutex_unlock(&chain->filter_chain_lock); 538 539 while (tp) { 540 tp_next = rcu_dereference_protected(tp->next, 1); 541 tcf_proto_put(tp, rtnl_held, NULL); 542 tp = tp_next; 543 } 544 } 545 546 static struct tcf_block *tc_dev_ingress_block(struct net_device *dev) 547 { 548 const struct Qdisc_class_ops *cops; 549 struct Qdisc *qdisc; 550 551 if (!dev_ingress_queue(dev)) 552 return NULL; 553 554 qdisc = dev_ingress_queue(dev)->qdisc_sleeping; 555 if (!qdisc) 556 return NULL; 557 558 cops = qdisc->ops->cl_ops; 559 if (!cops) 560 return NULL; 561 562 if (!cops->tcf_block) 563 return NULL; 564 565 return cops->tcf_block(qdisc, TC_H_MIN_INGRESS, NULL); 566 } 567 568 static struct rhashtable indr_setup_block_ht; 569 570 struct tc_indr_block_dev { 571 struct rhash_head ht_node; 572 struct net_device *dev; 573 unsigned int refcnt; 574 struct list_head cb_list; 575 struct tcf_block *block; 576 }; 577 578 struct tc_indr_block_cb { 579 struct list_head list; 580 void *cb_priv; 581 tc_indr_block_bind_cb_t *cb; 582 void *cb_ident; 583 }; 584 585 static const struct rhashtable_params tc_indr_setup_block_ht_params = { 586 .key_offset = offsetof(struct tc_indr_block_dev, dev), 587 .head_offset = offsetof(struct tc_indr_block_dev, ht_node), 588 .key_len = sizeof(struct net_device *), 589 }; 590 591 static struct tc_indr_block_dev * 592 tc_indr_block_dev_lookup(struct net_device *dev) 593 { 594 return rhashtable_lookup_fast(&indr_setup_block_ht, &dev, 595 tc_indr_setup_block_ht_params); 596 } 597 598 static struct tc_indr_block_dev *tc_indr_block_dev_get(struct net_device *dev) 599 { 600 struct tc_indr_block_dev *indr_dev; 601 602 indr_dev = tc_indr_block_dev_lookup(dev); 603 if (indr_dev) 604 goto inc_ref; 605 606 indr_dev = kzalloc(sizeof(*indr_dev), GFP_KERNEL); 607 if (!indr_dev) 608 return NULL; 609 610 INIT_LIST_HEAD(&indr_dev->cb_list); 611 indr_dev->dev = dev; 612 indr_dev->block = tc_dev_ingress_block(dev); 613 if (rhashtable_insert_fast(&indr_setup_block_ht, &indr_dev->ht_node, 614 tc_indr_setup_block_ht_params)) { 615 kfree(indr_dev); 616 return NULL; 617 } 618 619 inc_ref: 620 indr_dev->refcnt++; 621 return indr_dev; 622 } 623 624 static void tc_indr_block_dev_put(struct tc_indr_block_dev *indr_dev) 625 { 626 if (--indr_dev->refcnt) 627 return; 628 629 rhashtable_remove_fast(&indr_setup_block_ht, &indr_dev->ht_node, 630 tc_indr_setup_block_ht_params); 631 kfree(indr_dev); 632 } 633 634 static struct tc_indr_block_cb * 635 tc_indr_block_cb_lookup(struct tc_indr_block_dev *indr_dev, 636 tc_indr_block_bind_cb_t *cb, void *cb_ident) 637 { 638 struct tc_indr_block_cb *indr_block_cb; 639 640 list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list) 641 if (indr_block_cb->cb == cb && 642 indr_block_cb->cb_ident == cb_ident) 643 return indr_block_cb; 644 return NULL; 645 } 646 647 static struct tc_indr_block_cb * 648 tc_indr_block_cb_add(struct tc_indr_block_dev *indr_dev, void *cb_priv, 649 tc_indr_block_bind_cb_t *cb, void *cb_ident) 650 { 651 struct tc_indr_block_cb *indr_block_cb; 652 653 indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident); 654 if (indr_block_cb) 655 return ERR_PTR(-EEXIST); 656 657 indr_block_cb = kzalloc(sizeof(*indr_block_cb), GFP_KERNEL); 658 if (!indr_block_cb) 659 return ERR_PTR(-ENOMEM); 660 661 indr_block_cb->cb_priv = cb_priv; 662 indr_block_cb->cb = cb; 663 indr_block_cb->cb_ident = cb_ident; 664 list_add(&indr_block_cb->list, &indr_dev->cb_list); 665 666 return indr_block_cb; 667 } 668 669 static void tc_indr_block_cb_del(struct tc_indr_block_cb *indr_block_cb) 670 { 671 list_del(&indr_block_cb->list); 672 kfree(indr_block_cb); 673 } 674 675 static void tc_indr_block_ing_cmd(struct tc_indr_block_dev *indr_dev, 676 struct tc_indr_block_cb *indr_block_cb, 677 enum tc_block_command command) 678 { 679 struct tc_block_offload bo = { 680 .command = command, 681 .binder_type = TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 682 .block = indr_dev->block, 683 }; 684 685 if (!indr_dev->block) 686 return; 687 688 indr_block_cb->cb(indr_dev->dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK, 689 &bo); 690 } 691 692 int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv, 693 tc_indr_block_bind_cb_t *cb, void *cb_ident) 694 { 695 struct tc_indr_block_cb *indr_block_cb; 696 struct tc_indr_block_dev *indr_dev; 697 int err; 698 699 indr_dev = tc_indr_block_dev_get(dev); 700 if (!indr_dev) 701 return -ENOMEM; 702 703 indr_block_cb = tc_indr_block_cb_add(indr_dev, cb_priv, cb, cb_ident); 704 err = PTR_ERR_OR_ZERO(indr_block_cb); 705 if (err) 706 goto err_dev_put; 707 708 tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_BIND); 709 return 0; 710 711 err_dev_put: 712 tc_indr_block_dev_put(indr_dev); 713 return err; 714 } 715 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_register); 716 717 int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv, 718 tc_indr_block_bind_cb_t *cb, void *cb_ident) 719 { 720 int err; 721 722 rtnl_lock(); 723 err = __tc_indr_block_cb_register(dev, cb_priv, cb, cb_ident); 724 rtnl_unlock(); 725 726 return err; 727 } 728 EXPORT_SYMBOL_GPL(tc_indr_block_cb_register); 729 730 void __tc_indr_block_cb_unregister(struct net_device *dev, 731 tc_indr_block_bind_cb_t *cb, void *cb_ident) 732 { 733 struct tc_indr_block_cb *indr_block_cb; 734 struct tc_indr_block_dev *indr_dev; 735 736 indr_dev = tc_indr_block_dev_lookup(dev); 737 if (!indr_dev) 738 return; 739 740 indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident); 741 if (!indr_block_cb) 742 return; 743 744 /* Send unbind message if required to free any block cbs. */ 745 tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_UNBIND); 746 tc_indr_block_cb_del(indr_block_cb); 747 tc_indr_block_dev_put(indr_dev); 748 } 749 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_unregister); 750 751 void tc_indr_block_cb_unregister(struct net_device *dev, 752 tc_indr_block_bind_cb_t *cb, void *cb_ident) 753 { 754 rtnl_lock(); 755 __tc_indr_block_cb_unregister(dev, cb, cb_ident); 756 rtnl_unlock(); 757 } 758 EXPORT_SYMBOL_GPL(tc_indr_block_cb_unregister); 759 760 static void tc_indr_block_call(struct tcf_block *block, struct net_device *dev, 761 struct tcf_block_ext_info *ei, 762 enum tc_block_command command, 763 struct netlink_ext_ack *extack) 764 { 765 struct tc_indr_block_cb *indr_block_cb; 766 struct tc_indr_block_dev *indr_dev; 767 struct tc_block_offload bo = { 768 .command = command, 769 .binder_type = ei->binder_type, 770 .block = block, 771 .extack = extack, 772 }; 773 774 indr_dev = tc_indr_block_dev_lookup(dev); 775 if (!indr_dev) 776 return; 777 778 indr_dev->block = command == TC_BLOCK_BIND ? block : NULL; 779 780 list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list) 781 indr_block_cb->cb(dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK, 782 &bo); 783 } 784 785 static bool tcf_block_offload_in_use(struct tcf_block *block) 786 { 787 return block->offloadcnt; 788 } 789 790 static int tcf_block_offload_cmd(struct tcf_block *block, 791 struct net_device *dev, 792 struct tcf_block_ext_info *ei, 793 enum tc_block_command command, 794 struct netlink_ext_ack *extack) 795 { 796 struct tc_block_offload bo = {}; 797 798 bo.command = command; 799 bo.binder_type = ei->binder_type; 800 bo.block = block; 801 bo.extack = extack; 802 return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo); 803 } 804 805 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q, 806 struct tcf_block_ext_info *ei, 807 struct netlink_ext_ack *extack) 808 { 809 struct net_device *dev = q->dev_queue->dev; 810 int err; 811 812 if (!dev->netdev_ops->ndo_setup_tc) 813 goto no_offload_dev_inc; 814 815 /* If tc offload feature is disabled and the block we try to bind 816 * to already has some offloaded filters, forbid to bind. 817 */ 818 if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) { 819 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled"); 820 return -EOPNOTSUPP; 821 } 822 823 err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_BIND, extack); 824 if (err == -EOPNOTSUPP) 825 goto no_offload_dev_inc; 826 if (err) 827 return err; 828 829 tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack); 830 return 0; 831 832 no_offload_dev_inc: 833 if (tcf_block_offload_in_use(block)) 834 return -EOPNOTSUPP; 835 block->nooffloaddevcnt++; 836 tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack); 837 return 0; 838 } 839 840 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q, 841 struct tcf_block_ext_info *ei) 842 { 843 struct net_device *dev = q->dev_queue->dev; 844 int err; 845 846 tc_indr_block_call(block, dev, ei, TC_BLOCK_UNBIND, NULL); 847 848 if (!dev->netdev_ops->ndo_setup_tc) 849 goto no_offload_dev_dec; 850 err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_UNBIND, NULL); 851 if (err == -EOPNOTSUPP) 852 goto no_offload_dev_dec; 853 return; 854 855 no_offload_dev_dec: 856 WARN_ON(block->nooffloaddevcnt-- == 0); 857 } 858 859 static int 860 tcf_chain0_head_change_cb_add(struct tcf_block *block, 861 struct tcf_block_ext_info *ei, 862 struct netlink_ext_ack *extack) 863 { 864 struct tcf_filter_chain_list_item *item; 865 struct tcf_chain *chain0; 866 867 item = kmalloc(sizeof(*item), GFP_KERNEL); 868 if (!item) { 869 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed"); 870 return -ENOMEM; 871 } 872 item->chain_head_change = ei->chain_head_change; 873 item->chain_head_change_priv = ei->chain_head_change_priv; 874 875 mutex_lock(&block->lock); 876 chain0 = block->chain0.chain; 877 if (chain0) 878 tcf_chain_hold(chain0); 879 else 880 list_add(&item->list, &block->chain0.filter_chain_list); 881 mutex_unlock(&block->lock); 882 883 if (chain0) { 884 struct tcf_proto *tp_head; 885 886 mutex_lock(&chain0->filter_chain_lock); 887 888 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0); 889 if (tp_head) 890 tcf_chain_head_change_item(item, tp_head); 891 892 mutex_lock(&block->lock); 893 list_add(&item->list, &block->chain0.filter_chain_list); 894 mutex_unlock(&block->lock); 895 896 mutex_unlock(&chain0->filter_chain_lock); 897 tcf_chain_put(chain0); 898 } 899 900 return 0; 901 } 902 903 static void 904 tcf_chain0_head_change_cb_del(struct tcf_block *block, 905 struct tcf_block_ext_info *ei) 906 { 907 struct tcf_filter_chain_list_item *item; 908 909 mutex_lock(&block->lock); 910 list_for_each_entry(item, &block->chain0.filter_chain_list, list) { 911 if ((!ei->chain_head_change && !ei->chain_head_change_priv) || 912 (item->chain_head_change == ei->chain_head_change && 913 item->chain_head_change_priv == ei->chain_head_change_priv)) { 914 if (block->chain0.chain) 915 tcf_chain_head_change_item(item, NULL); 916 list_del(&item->list); 917 mutex_unlock(&block->lock); 918 919 kfree(item); 920 return; 921 } 922 } 923 mutex_unlock(&block->lock); 924 WARN_ON(1); 925 } 926 927 struct tcf_net { 928 spinlock_t idr_lock; /* Protects idr */ 929 struct idr idr; 930 }; 931 932 static unsigned int tcf_net_id; 933 934 static int tcf_block_insert(struct tcf_block *block, struct net *net, 935 struct netlink_ext_ack *extack) 936 { 937 struct tcf_net *tn = net_generic(net, tcf_net_id); 938 int err; 939 940 idr_preload(GFP_KERNEL); 941 spin_lock(&tn->idr_lock); 942 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index, 943 GFP_NOWAIT); 944 spin_unlock(&tn->idr_lock); 945 idr_preload_end(); 946 947 return err; 948 } 949 950 static void tcf_block_remove(struct tcf_block *block, struct net *net) 951 { 952 struct tcf_net *tn = net_generic(net, tcf_net_id); 953 954 spin_lock(&tn->idr_lock); 955 idr_remove(&tn->idr, block->index); 956 spin_unlock(&tn->idr_lock); 957 } 958 959 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q, 960 u32 block_index, 961 struct netlink_ext_ack *extack) 962 { 963 struct tcf_block *block; 964 965 block = kzalloc(sizeof(*block), GFP_KERNEL); 966 if (!block) { 967 NL_SET_ERR_MSG(extack, "Memory allocation for block failed"); 968 return ERR_PTR(-ENOMEM); 969 } 970 mutex_init(&block->lock); 971 INIT_LIST_HEAD(&block->chain_list); 972 INIT_LIST_HEAD(&block->cb_list); 973 INIT_LIST_HEAD(&block->owner_list); 974 INIT_LIST_HEAD(&block->chain0.filter_chain_list); 975 976 refcount_set(&block->refcnt, 1); 977 block->net = net; 978 block->index = block_index; 979 980 /* Don't store q pointer for blocks which are shared */ 981 if (!tcf_block_shared(block)) 982 block->q = q; 983 return block; 984 } 985 986 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index) 987 { 988 struct tcf_net *tn = net_generic(net, tcf_net_id); 989 990 return idr_find(&tn->idr, block_index); 991 } 992 993 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index) 994 { 995 struct tcf_block *block; 996 997 rcu_read_lock(); 998 block = tcf_block_lookup(net, block_index); 999 if (block && !refcount_inc_not_zero(&block->refcnt)) 1000 block = NULL; 1001 rcu_read_unlock(); 1002 1003 return block; 1004 } 1005 1006 static struct tcf_chain * 1007 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain) 1008 { 1009 mutex_lock(&block->lock); 1010 if (chain) 1011 chain = list_is_last(&chain->list, &block->chain_list) ? 1012 NULL : list_next_entry(chain, list); 1013 else 1014 chain = list_first_entry_or_null(&block->chain_list, 1015 struct tcf_chain, list); 1016 1017 /* skip all action-only chains */ 1018 while (chain && tcf_chain_held_by_acts_only(chain)) 1019 chain = list_is_last(&chain->list, &block->chain_list) ? 1020 NULL : list_next_entry(chain, list); 1021 1022 if (chain) 1023 tcf_chain_hold(chain); 1024 mutex_unlock(&block->lock); 1025 1026 return chain; 1027 } 1028 1029 /* Function to be used by all clients that want to iterate over all chains on 1030 * block. It properly obtains block->lock and takes reference to chain before 1031 * returning it. Users of this function must be tolerant to concurrent chain 1032 * insertion/deletion or ensure that no concurrent chain modification is 1033 * possible. Note that all netlink dump callbacks cannot guarantee to provide 1034 * consistent dump because rtnl lock is released each time skb is filled with 1035 * data and sent to user-space. 1036 */ 1037 1038 struct tcf_chain * 1039 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain) 1040 { 1041 struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain); 1042 1043 if (chain) 1044 tcf_chain_put(chain); 1045 1046 return chain_next; 1047 } 1048 EXPORT_SYMBOL(tcf_get_next_chain); 1049 1050 static struct tcf_proto * 1051 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp) 1052 { 1053 u32 prio = 0; 1054 1055 ASSERT_RTNL(); 1056 mutex_lock(&chain->filter_chain_lock); 1057 1058 if (!tp) { 1059 tp = tcf_chain_dereference(chain->filter_chain, chain); 1060 } else if (tcf_proto_is_deleting(tp)) { 1061 /* 'deleting' flag is set and chain->filter_chain_lock was 1062 * unlocked, which means next pointer could be invalid. Restart 1063 * search. 1064 */ 1065 prio = tp->prio + 1; 1066 tp = tcf_chain_dereference(chain->filter_chain, chain); 1067 1068 for (; tp; tp = tcf_chain_dereference(tp->next, chain)) 1069 if (!tp->deleting && tp->prio >= prio) 1070 break; 1071 } else { 1072 tp = tcf_chain_dereference(tp->next, chain); 1073 } 1074 1075 if (tp) 1076 tcf_proto_get(tp); 1077 1078 mutex_unlock(&chain->filter_chain_lock); 1079 1080 return tp; 1081 } 1082 1083 /* Function to be used by all clients that want to iterate over all tp's on 1084 * chain. Users of this function must be tolerant to concurrent tp 1085 * insertion/deletion or ensure that no concurrent chain modification is 1086 * possible. Note that all netlink dump callbacks cannot guarantee to provide 1087 * consistent dump because rtnl lock is released each time skb is filled with 1088 * data and sent to user-space. 1089 */ 1090 1091 struct tcf_proto * 1092 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp, 1093 bool rtnl_held) 1094 { 1095 struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp); 1096 1097 if (tp) 1098 tcf_proto_put(tp, rtnl_held, NULL); 1099 1100 return tp_next; 1101 } 1102 EXPORT_SYMBOL(tcf_get_next_proto); 1103 1104 static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held) 1105 { 1106 struct tcf_chain *chain; 1107 1108 /* Last reference to block. At this point chains cannot be added or 1109 * removed concurrently. 1110 */ 1111 for (chain = tcf_get_next_chain(block, NULL); 1112 chain; 1113 chain = tcf_get_next_chain(block, chain)) { 1114 tcf_chain_put_explicitly_created(chain); 1115 tcf_chain_flush(chain, rtnl_held); 1116 } 1117 } 1118 1119 /* Lookup Qdisc and increments its reference counter. 1120 * Set parent, if necessary. 1121 */ 1122 1123 static int __tcf_qdisc_find(struct net *net, struct Qdisc **q, 1124 u32 *parent, int ifindex, bool rtnl_held, 1125 struct netlink_ext_ack *extack) 1126 { 1127 const struct Qdisc_class_ops *cops; 1128 struct net_device *dev; 1129 int err = 0; 1130 1131 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) 1132 return 0; 1133 1134 rcu_read_lock(); 1135 1136 /* Find link */ 1137 dev = dev_get_by_index_rcu(net, ifindex); 1138 if (!dev) { 1139 rcu_read_unlock(); 1140 return -ENODEV; 1141 } 1142 1143 /* Find qdisc */ 1144 if (!*parent) { 1145 *q = dev->qdisc; 1146 *parent = (*q)->handle; 1147 } else { 1148 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent)); 1149 if (!*q) { 1150 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists"); 1151 err = -EINVAL; 1152 goto errout_rcu; 1153 } 1154 } 1155 1156 *q = qdisc_refcount_inc_nz(*q); 1157 if (!*q) { 1158 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists"); 1159 err = -EINVAL; 1160 goto errout_rcu; 1161 } 1162 1163 /* Is it classful? */ 1164 cops = (*q)->ops->cl_ops; 1165 if (!cops) { 1166 NL_SET_ERR_MSG(extack, "Qdisc not classful"); 1167 err = -EINVAL; 1168 goto errout_qdisc; 1169 } 1170 1171 if (!cops->tcf_block) { 1172 NL_SET_ERR_MSG(extack, "Class doesn't support blocks"); 1173 err = -EOPNOTSUPP; 1174 goto errout_qdisc; 1175 } 1176 1177 errout_rcu: 1178 /* At this point we know that qdisc is not noop_qdisc, 1179 * which means that qdisc holds a reference to net_device 1180 * and we hold a reference to qdisc, so it is safe to release 1181 * rcu read lock. 1182 */ 1183 rcu_read_unlock(); 1184 return err; 1185 1186 errout_qdisc: 1187 rcu_read_unlock(); 1188 1189 if (rtnl_held) 1190 qdisc_put(*q); 1191 else 1192 qdisc_put_unlocked(*q); 1193 *q = NULL; 1194 1195 return err; 1196 } 1197 1198 static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl, 1199 int ifindex, struct netlink_ext_ack *extack) 1200 { 1201 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) 1202 return 0; 1203 1204 /* Do we search for filter, attached to class? */ 1205 if (TC_H_MIN(parent)) { 1206 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1207 1208 *cl = cops->find(q, parent); 1209 if (*cl == 0) { 1210 NL_SET_ERR_MSG(extack, "Specified class doesn't exist"); 1211 return -ENOENT; 1212 } 1213 } 1214 1215 return 0; 1216 } 1217 1218 static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q, 1219 unsigned long cl, int ifindex, 1220 u32 block_index, 1221 struct netlink_ext_ack *extack) 1222 { 1223 struct tcf_block *block; 1224 1225 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 1226 block = tcf_block_refcnt_get(net, block_index); 1227 if (!block) { 1228 NL_SET_ERR_MSG(extack, "Block of given index was not found"); 1229 return ERR_PTR(-EINVAL); 1230 } 1231 } else { 1232 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1233 1234 block = cops->tcf_block(q, cl, extack); 1235 if (!block) 1236 return ERR_PTR(-EINVAL); 1237 1238 if (tcf_block_shared(block)) { 1239 NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters"); 1240 return ERR_PTR(-EOPNOTSUPP); 1241 } 1242 1243 /* Always take reference to block in order to support execution 1244 * of rules update path of cls API without rtnl lock. Caller 1245 * must release block when it is finished using it. 'if' block 1246 * of this conditional obtain reference to block by calling 1247 * tcf_block_refcnt_get(). 1248 */ 1249 refcount_inc(&block->refcnt); 1250 } 1251 1252 return block; 1253 } 1254 1255 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q, 1256 struct tcf_block_ext_info *ei, bool rtnl_held) 1257 { 1258 if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) { 1259 /* Flushing/putting all chains will cause the block to be 1260 * deallocated when last chain is freed. However, if chain_list 1261 * is empty, block has to be manually deallocated. After block 1262 * reference counter reached 0, it is no longer possible to 1263 * increment it or add new chains to block. 1264 */ 1265 bool free_block = list_empty(&block->chain_list); 1266 1267 mutex_unlock(&block->lock); 1268 if (tcf_block_shared(block)) 1269 tcf_block_remove(block, block->net); 1270 1271 if (q) 1272 tcf_block_offload_unbind(block, q, ei); 1273 1274 if (free_block) 1275 tcf_block_destroy(block); 1276 else 1277 tcf_block_flush_all_chains(block, rtnl_held); 1278 } else if (q) { 1279 tcf_block_offload_unbind(block, q, ei); 1280 } 1281 } 1282 1283 static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held) 1284 { 1285 __tcf_block_put(block, NULL, NULL, rtnl_held); 1286 } 1287 1288 /* Find tcf block. 1289 * Set q, parent, cl when appropriate. 1290 */ 1291 1292 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q, 1293 u32 *parent, unsigned long *cl, 1294 int ifindex, u32 block_index, 1295 struct netlink_ext_ack *extack) 1296 { 1297 struct tcf_block *block; 1298 int err = 0; 1299 1300 ASSERT_RTNL(); 1301 1302 err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack); 1303 if (err) 1304 goto errout; 1305 1306 err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack); 1307 if (err) 1308 goto errout_qdisc; 1309 1310 block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack); 1311 if (IS_ERR(block)) { 1312 err = PTR_ERR(block); 1313 goto errout_qdisc; 1314 } 1315 1316 return block; 1317 1318 errout_qdisc: 1319 if (*q) 1320 qdisc_put(*q); 1321 errout: 1322 *q = NULL; 1323 return ERR_PTR(err); 1324 } 1325 1326 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block, 1327 bool rtnl_held) 1328 { 1329 if (!IS_ERR_OR_NULL(block)) 1330 tcf_block_refcnt_put(block, rtnl_held); 1331 1332 if (q) { 1333 if (rtnl_held) 1334 qdisc_put(q); 1335 else 1336 qdisc_put_unlocked(q); 1337 } 1338 } 1339 1340 struct tcf_block_owner_item { 1341 struct list_head list; 1342 struct Qdisc *q; 1343 enum tcf_block_binder_type binder_type; 1344 }; 1345 1346 static void 1347 tcf_block_owner_netif_keep_dst(struct tcf_block *block, 1348 struct Qdisc *q, 1349 enum tcf_block_binder_type binder_type) 1350 { 1351 if (block->keep_dst && 1352 binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS && 1353 binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS) 1354 netif_keep_dst(qdisc_dev(q)); 1355 } 1356 1357 void tcf_block_netif_keep_dst(struct tcf_block *block) 1358 { 1359 struct tcf_block_owner_item *item; 1360 1361 block->keep_dst = true; 1362 list_for_each_entry(item, &block->owner_list, list) 1363 tcf_block_owner_netif_keep_dst(block, item->q, 1364 item->binder_type); 1365 } 1366 EXPORT_SYMBOL(tcf_block_netif_keep_dst); 1367 1368 static int tcf_block_owner_add(struct tcf_block *block, 1369 struct Qdisc *q, 1370 enum tcf_block_binder_type binder_type) 1371 { 1372 struct tcf_block_owner_item *item; 1373 1374 item = kmalloc(sizeof(*item), GFP_KERNEL); 1375 if (!item) 1376 return -ENOMEM; 1377 item->q = q; 1378 item->binder_type = binder_type; 1379 list_add(&item->list, &block->owner_list); 1380 return 0; 1381 } 1382 1383 static void tcf_block_owner_del(struct tcf_block *block, 1384 struct Qdisc *q, 1385 enum tcf_block_binder_type binder_type) 1386 { 1387 struct tcf_block_owner_item *item; 1388 1389 list_for_each_entry(item, &block->owner_list, list) { 1390 if (item->q == q && item->binder_type == binder_type) { 1391 list_del(&item->list); 1392 kfree(item); 1393 return; 1394 } 1395 } 1396 WARN_ON(1); 1397 } 1398 1399 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q, 1400 struct tcf_block_ext_info *ei, 1401 struct netlink_ext_ack *extack) 1402 { 1403 struct net *net = qdisc_net(q); 1404 struct tcf_block *block = NULL; 1405 int err; 1406 1407 if (ei->block_index) 1408 /* block_index not 0 means the shared block is requested */ 1409 block = tcf_block_refcnt_get(net, ei->block_index); 1410 1411 if (!block) { 1412 block = tcf_block_create(net, q, ei->block_index, extack); 1413 if (IS_ERR(block)) 1414 return PTR_ERR(block); 1415 if (tcf_block_shared(block)) { 1416 err = tcf_block_insert(block, net, extack); 1417 if (err) 1418 goto err_block_insert; 1419 } 1420 } 1421 1422 err = tcf_block_owner_add(block, q, ei->binder_type); 1423 if (err) 1424 goto err_block_owner_add; 1425 1426 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type); 1427 1428 err = tcf_chain0_head_change_cb_add(block, ei, extack); 1429 if (err) 1430 goto err_chain0_head_change_cb_add; 1431 1432 err = tcf_block_offload_bind(block, q, ei, extack); 1433 if (err) 1434 goto err_block_offload_bind; 1435 1436 *p_block = block; 1437 return 0; 1438 1439 err_block_offload_bind: 1440 tcf_chain0_head_change_cb_del(block, ei); 1441 err_chain0_head_change_cb_add: 1442 tcf_block_owner_del(block, q, ei->binder_type); 1443 err_block_owner_add: 1444 err_block_insert: 1445 tcf_block_refcnt_put(block, true); 1446 return err; 1447 } 1448 EXPORT_SYMBOL(tcf_block_get_ext); 1449 1450 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv) 1451 { 1452 struct tcf_proto __rcu **p_filter_chain = priv; 1453 1454 rcu_assign_pointer(*p_filter_chain, tp_head); 1455 } 1456 1457 int tcf_block_get(struct tcf_block **p_block, 1458 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q, 1459 struct netlink_ext_ack *extack) 1460 { 1461 struct tcf_block_ext_info ei = { 1462 .chain_head_change = tcf_chain_head_change_dflt, 1463 .chain_head_change_priv = p_filter_chain, 1464 }; 1465 1466 WARN_ON(!p_filter_chain); 1467 return tcf_block_get_ext(p_block, q, &ei, extack); 1468 } 1469 EXPORT_SYMBOL(tcf_block_get); 1470 1471 /* XXX: Standalone actions are not allowed to jump to any chain, and bound 1472 * actions should be all removed after flushing. 1473 */ 1474 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q, 1475 struct tcf_block_ext_info *ei) 1476 { 1477 if (!block) 1478 return; 1479 tcf_chain0_head_change_cb_del(block, ei); 1480 tcf_block_owner_del(block, q, ei->binder_type); 1481 1482 __tcf_block_put(block, q, ei, true); 1483 } 1484 EXPORT_SYMBOL(tcf_block_put_ext); 1485 1486 void tcf_block_put(struct tcf_block *block) 1487 { 1488 struct tcf_block_ext_info ei = {0, }; 1489 1490 if (!block) 1491 return; 1492 tcf_block_put_ext(block, block->q, &ei); 1493 } 1494 1495 EXPORT_SYMBOL(tcf_block_put); 1496 1497 struct tcf_block_cb { 1498 struct list_head list; 1499 tc_setup_cb_t *cb; 1500 void *cb_ident; 1501 void *cb_priv; 1502 unsigned int refcnt; 1503 }; 1504 1505 void *tcf_block_cb_priv(struct tcf_block_cb *block_cb) 1506 { 1507 return block_cb->cb_priv; 1508 } 1509 EXPORT_SYMBOL(tcf_block_cb_priv); 1510 1511 struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block, 1512 tc_setup_cb_t *cb, void *cb_ident) 1513 { struct tcf_block_cb *block_cb; 1514 1515 list_for_each_entry(block_cb, &block->cb_list, list) 1516 if (block_cb->cb == cb && block_cb->cb_ident == cb_ident) 1517 return block_cb; 1518 return NULL; 1519 } 1520 EXPORT_SYMBOL(tcf_block_cb_lookup); 1521 1522 void tcf_block_cb_incref(struct tcf_block_cb *block_cb) 1523 { 1524 block_cb->refcnt++; 1525 } 1526 EXPORT_SYMBOL(tcf_block_cb_incref); 1527 1528 unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb) 1529 { 1530 return --block_cb->refcnt; 1531 } 1532 EXPORT_SYMBOL(tcf_block_cb_decref); 1533 1534 static int 1535 tcf_block_playback_offloads(struct tcf_block *block, tc_setup_cb_t *cb, 1536 void *cb_priv, bool add, bool offload_in_use, 1537 struct netlink_ext_ack *extack) 1538 { 1539 struct tcf_chain *chain, *chain_prev; 1540 struct tcf_proto *tp, *tp_prev; 1541 int err; 1542 1543 for (chain = __tcf_get_next_chain(block, NULL); 1544 chain; 1545 chain_prev = chain, 1546 chain = __tcf_get_next_chain(block, chain), 1547 tcf_chain_put(chain_prev)) { 1548 for (tp = __tcf_get_next_proto(chain, NULL); tp; 1549 tp_prev = tp, 1550 tp = __tcf_get_next_proto(chain, tp), 1551 tcf_proto_put(tp_prev, true, NULL)) { 1552 if (tp->ops->reoffload) { 1553 err = tp->ops->reoffload(tp, add, cb, cb_priv, 1554 extack); 1555 if (err && add) 1556 goto err_playback_remove; 1557 } else if (add && offload_in_use) { 1558 err = -EOPNOTSUPP; 1559 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support"); 1560 goto err_playback_remove; 1561 } 1562 } 1563 } 1564 1565 return 0; 1566 1567 err_playback_remove: 1568 tcf_proto_put(tp, true, NULL); 1569 tcf_chain_put(chain); 1570 tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use, 1571 extack); 1572 return err; 1573 } 1574 1575 struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block, 1576 tc_setup_cb_t *cb, void *cb_ident, 1577 void *cb_priv, 1578 struct netlink_ext_ack *extack) 1579 { 1580 struct tcf_block_cb *block_cb; 1581 int err; 1582 1583 /* Replay any already present rules */ 1584 err = tcf_block_playback_offloads(block, cb, cb_priv, true, 1585 tcf_block_offload_in_use(block), 1586 extack); 1587 if (err) 1588 return ERR_PTR(err); 1589 1590 block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL); 1591 if (!block_cb) 1592 return ERR_PTR(-ENOMEM); 1593 block_cb->cb = cb; 1594 block_cb->cb_ident = cb_ident; 1595 block_cb->cb_priv = cb_priv; 1596 list_add(&block_cb->list, &block->cb_list); 1597 return block_cb; 1598 } 1599 EXPORT_SYMBOL(__tcf_block_cb_register); 1600 1601 int tcf_block_cb_register(struct tcf_block *block, 1602 tc_setup_cb_t *cb, void *cb_ident, 1603 void *cb_priv, struct netlink_ext_ack *extack) 1604 { 1605 struct tcf_block_cb *block_cb; 1606 1607 block_cb = __tcf_block_cb_register(block, cb, cb_ident, cb_priv, 1608 extack); 1609 return PTR_ERR_OR_ZERO(block_cb); 1610 } 1611 EXPORT_SYMBOL(tcf_block_cb_register); 1612 1613 void __tcf_block_cb_unregister(struct tcf_block *block, 1614 struct tcf_block_cb *block_cb) 1615 { 1616 tcf_block_playback_offloads(block, block_cb->cb, block_cb->cb_priv, 1617 false, tcf_block_offload_in_use(block), 1618 NULL); 1619 list_del(&block_cb->list); 1620 kfree(block_cb); 1621 } 1622 EXPORT_SYMBOL(__tcf_block_cb_unregister); 1623 1624 void tcf_block_cb_unregister(struct tcf_block *block, 1625 tc_setup_cb_t *cb, void *cb_ident) 1626 { 1627 struct tcf_block_cb *block_cb; 1628 1629 block_cb = tcf_block_cb_lookup(block, cb, cb_ident); 1630 if (!block_cb) 1631 return; 1632 __tcf_block_cb_unregister(block, block_cb); 1633 } 1634 EXPORT_SYMBOL(tcf_block_cb_unregister); 1635 1636 /* Main classifier routine: scans classifier chain attached 1637 * to this qdisc, (optionally) tests for protocol and asks 1638 * specific classifiers. 1639 */ 1640 int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp, 1641 struct tcf_result *res, bool compat_mode) 1642 { 1643 #ifdef CONFIG_NET_CLS_ACT 1644 const int max_reclassify_loop = 4; 1645 const struct tcf_proto *orig_tp = tp; 1646 const struct tcf_proto *first_tp; 1647 int limit = 0; 1648 1649 reclassify: 1650 #endif 1651 for (; tp; tp = rcu_dereference_bh(tp->next)) { 1652 __be16 protocol = tc_skb_protocol(skb); 1653 int err; 1654 1655 if (tp->protocol != protocol && 1656 tp->protocol != htons(ETH_P_ALL)) 1657 continue; 1658 1659 err = tp->classify(skb, tp, res); 1660 #ifdef CONFIG_NET_CLS_ACT 1661 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) { 1662 first_tp = orig_tp; 1663 goto reset; 1664 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) { 1665 first_tp = res->goto_tp; 1666 goto reset; 1667 } 1668 #endif 1669 if (err >= 0) 1670 return err; 1671 } 1672 1673 return TC_ACT_UNSPEC; /* signal: continue lookup */ 1674 #ifdef CONFIG_NET_CLS_ACT 1675 reset: 1676 if (unlikely(limit++ >= max_reclassify_loop)) { 1677 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n", 1678 tp->chain->block->index, 1679 tp->prio & 0xffff, 1680 ntohs(tp->protocol)); 1681 return TC_ACT_SHOT; 1682 } 1683 1684 tp = first_tp; 1685 goto reclassify; 1686 #endif 1687 } 1688 EXPORT_SYMBOL(tcf_classify); 1689 1690 struct tcf_chain_info { 1691 struct tcf_proto __rcu **pprev; 1692 struct tcf_proto __rcu *next; 1693 }; 1694 1695 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain, 1696 struct tcf_chain_info *chain_info) 1697 { 1698 return tcf_chain_dereference(*chain_info->pprev, chain); 1699 } 1700 1701 static int tcf_chain_tp_insert(struct tcf_chain *chain, 1702 struct tcf_chain_info *chain_info, 1703 struct tcf_proto *tp) 1704 { 1705 if (chain->flushing) 1706 return -EAGAIN; 1707 1708 if (*chain_info->pprev == chain->filter_chain) 1709 tcf_chain0_head_change(chain, tp); 1710 tcf_proto_get(tp); 1711 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info)); 1712 rcu_assign_pointer(*chain_info->pprev, tp); 1713 1714 return 0; 1715 } 1716 1717 static void tcf_chain_tp_remove(struct tcf_chain *chain, 1718 struct tcf_chain_info *chain_info, 1719 struct tcf_proto *tp) 1720 { 1721 struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain); 1722 1723 tcf_proto_mark_delete(tp); 1724 if (tp == chain->filter_chain) 1725 tcf_chain0_head_change(chain, next); 1726 RCU_INIT_POINTER(*chain_info->pprev, next); 1727 } 1728 1729 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, 1730 struct tcf_chain_info *chain_info, 1731 u32 protocol, u32 prio, 1732 bool prio_allocate); 1733 1734 /* Try to insert new proto. 1735 * If proto with specified priority already exists, free new proto 1736 * and return existing one. 1737 */ 1738 1739 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain, 1740 struct tcf_proto *tp_new, 1741 u32 protocol, u32 prio, 1742 bool rtnl_held) 1743 { 1744 struct tcf_chain_info chain_info; 1745 struct tcf_proto *tp; 1746 int err = 0; 1747 1748 mutex_lock(&chain->filter_chain_lock); 1749 1750 tp = tcf_chain_tp_find(chain, &chain_info, 1751 protocol, prio, false); 1752 if (!tp) 1753 err = tcf_chain_tp_insert(chain, &chain_info, tp_new); 1754 mutex_unlock(&chain->filter_chain_lock); 1755 1756 if (tp) { 1757 tcf_proto_destroy(tp_new, rtnl_held, NULL); 1758 tp_new = tp; 1759 } else if (err) { 1760 tcf_proto_destroy(tp_new, rtnl_held, NULL); 1761 tp_new = ERR_PTR(err); 1762 } 1763 1764 return tp_new; 1765 } 1766 1767 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain, 1768 struct tcf_proto *tp, bool rtnl_held, 1769 struct netlink_ext_ack *extack) 1770 { 1771 struct tcf_chain_info chain_info; 1772 struct tcf_proto *tp_iter; 1773 struct tcf_proto **pprev; 1774 struct tcf_proto *next; 1775 1776 mutex_lock(&chain->filter_chain_lock); 1777 1778 /* Atomically find and remove tp from chain. */ 1779 for (pprev = &chain->filter_chain; 1780 (tp_iter = tcf_chain_dereference(*pprev, chain)); 1781 pprev = &tp_iter->next) { 1782 if (tp_iter == tp) { 1783 chain_info.pprev = pprev; 1784 chain_info.next = tp_iter->next; 1785 WARN_ON(tp_iter->deleting); 1786 break; 1787 } 1788 } 1789 /* Verify that tp still exists and no new filters were inserted 1790 * concurrently. 1791 * Mark tp for deletion if it is empty. 1792 */ 1793 if (!tp_iter || !tcf_proto_check_delete(tp, rtnl_held)) { 1794 mutex_unlock(&chain->filter_chain_lock); 1795 return; 1796 } 1797 1798 next = tcf_chain_dereference(chain_info.next, chain); 1799 if (tp == chain->filter_chain) 1800 tcf_chain0_head_change(chain, next); 1801 RCU_INIT_POINTER(*chain_info.pprev, next); 1802 mutex_unlock(&chain->filter_chain_lock); 1803 1804 tcf_proto_put(tp, rtnl_held, extack); 1805 } 1806 1807 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, 1808 struct tcf_chain_info *chain_info, 1809 u32 protocol, u32 prio, 1810 bool prio_allocate) 1811 { 1812 struct tcf_proto **pprev; 1813 struct tcf_proto *tp; 1814 1815 /* Check the chain for existence of proto-tcf with this priority */ 1816 for (pprev = &chain->filter_chain; 1817 (tp = tcf_chain_dereference(*pprev, chain)); 1818 pprev = &tp->next) { 1819 if (tp->prio >= prio) { 1820 if (tp->prio == prio) { 1821 if (prio_allocate || 1822 (tp->protocol != protocol && protocol)) 1823 return ERR_PTR(-EINVAL); 1824 } else { 1825 tp = NULL; 1826 } 1827 break; 1828 } 1829 } 1830 chain_info->pprev = pprev; 1831 if (tp) { 1832 chain_info->next = tp->next; 1833 tcf_proto_get(tp); 1834 } else { 1835 chain_info->next = NULL; 1836 } 1837 return tp; 1838 } 1839 1840 static int tcf_fill_node(struct net *net, struct sk_buff *skb, 1841 struct tcf_proto *tp, struct tcf_block *block, 1842 struct Qdisc *q, u32 parent, void *fh, 1843 u32 portid, u32 seq, u16 flags, int event, 1844 bool rtnl_held) 1845 { 1846 struct tcmsg *tcm; 1847 struct nlmsghdr *nlh; 1848 unsigned char *b = skb_tail_pointer(skb); 1849 1850 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 1851 if (!nlh) 1852 goto out_nlmsg_trim; 1853 tcm = nlmsg_data(nlh); 1854 tcm->tcm_family = AF_UNSPEC; 1855 tcm->tcm__pad1 = 0; 1856 tcm->tcm__pad2 = 0; 1857 if (q) { 1858 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 1859 tcm->tcm_parent = parent; 1860 } else { 1861 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; 1862 tcm->tcm_block_index = block->index; 1863 } 1864 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol); 1865 if (nla_put_string(skb, TCA_KIND, tp->ops->kind)) 1866 goto nla_put_failure; 1867 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index)) 1868 goto nla_put_failure; 1869 if (!fh) { 1870 tcm->tcm_handle = 0; 1871 } else { 1872 if (tp->ops->dump && 1873 tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0) 1874 goto nla_put_failure; 1875 } 1876 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1877 return skb->len; 1878 1879 out_nlmsg_trim: 1880 nla_put_failure: 1881 nlmsg_trim(skb, b); 1882 return -1; 1883 } 1884 1885 static int tfilter_notify(struct net *net, struct sk_buff *oskb, 1886 struct nlmsghdr *n, struct tcf_proto *tp, 1887 struct tcf_block *block, struct Qdisc *q, 1888 u32 parent, void *fh, int event, bool unicast, 1889 bool rtnl_held) 1890 { 1891 struct sk_buff *skb; 1892 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1893 int err = 0; 1894 1895 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1896 if (!skb) 1897 return -ENOBUFS; 1898 1899 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid, 1900 n->nlmsg_seq, n->nlmsg_flags, event, 1901 rtnl_held) <= 0) { 1902 kfree_skb(skb); 1903 return -EINVAL; 1904 } 1905 1906 if (unicast) 1907 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); 1908 else 1909 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1910 n->nlmsg_flags & NLM_F_ECHO); 1911 1912 if (err > 0) 1913 err = 0; 1914 return err; 1915 } 1916 1917 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb, 1918 struct nlmsghdr *n, struct tcf_proto *tp, 1919 struct tcf_block *block, struct Qdisc *q, 1920 u32 parent, void *fh, bool unicast, bool *last, 1921 bool rtnl_held, struct netlink_ext_ack *extack) 1922 { 1923 struct sk_buff *skb; 1924 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1925 int err; 1926 1927 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1928 if (!skb) 1929 return -ENOBUFS; 1930 1931 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid, 1932 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER, 1933 rtnl_held) <= 0) { 1934 NL_SET_ERR_MSG(extack, "Failed to build del event notification"); 1935 kfree_skb(skb); 1936 return -EINVAL; 1937 } 1938 1939 err = tp->ops->delete(tp, fh, last, rtnl_held, extack); 1940 if (err) { 1941 kfree_skb(skb); 1942 return err; 1943 } 1944 1945 if (unicast) 1946 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); 1947 else 1948 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1949 n->nlmsg_flags & NLM_F_ECHO); 1950 if (err < 0) 1951 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification"); 1952 1953 if (err > 0) 1954 err = 0; 1955 return err; 1956 } 1957 1958 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb, 1959 struct tcf_block *block, struct Qdisc *q, 1960 u32 parent, struct nlmsghdr *n, 1961 struct tcf_chain *chain, int event, 1962 bool rtnl_held) 1963 { 1964 struct tcf_proto *tp; 1965 1966 for (tp = tcf_get_next_proto(chain, NULL, rtnl_held); 1967 tp; tp = tcf_get_next_proto(chain, tp, rtnl_held)) 1968 tfilter_notify(net, oskb, n, tp, block, 1969 q, parent, NULL, event, false, rtnl_held); 1970 } 1971 1972 static void tfilter_put(struct tcf_proto *tp, void *fh) 1973 { 1974 if (tp->ops->put && fh) 1975 tp->ops->put(tp, fh); 1976 } 1977 1978 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 1979 struct netlink_ext_ack *extack) 1980 { 1981 struct net *net = sock_net(skb->sk); 1982 struct nlattr *tca[TCA_MAX + 1]; 1983 struct tcmsg *t; 1984 u32 protocol; 1985 u32 prio; 1986 bool prio_allocate; 1987 u32 parent; 1988 u32 chain_index; 1989 struct Qdisc *q = NULL; 1990 struct tcf_chain_info chain_info; 1991 struct tcf_chain *chain = NULL; 1992 struct tcf_block *block; 1993 struct tcf_proto *tp; 1994 unsigned long cl; 1995 void *fh; 1996 int err; 1997 int tp_created; 1998 bool rtnl_held = false; 1999 2000 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) 2001 return -EPERM; 2002 2003 replay: 2004 tp_created = 0; 2005 2006 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2007 rtm_tca_policy, extack); 2008 if (err < 0) 2009 return err; 2010 2011 t = nlmsg_data(n); 2012 protocol = TC_H_MIN(t->tcm_info); 2013 prio = TC_H_MAJ(t->tcm_info); 2014 prio_allocate = false; 2015 parent = t->tcm_parent; 2016 tp = NULL; 2017 cl = 0; 2018 block = NULL; 2019 2020 if (prio == 0) { 2021 /* If no priority is provided by the user, 2022 * we allocate one. 2023 */ 2024 if (n->nlmsg_flags & NLM_F_CREATE) { 2025 prio = TC_H_MAKE(0x80000000U, 0U); 2026 prio_allocate = true; 2027 } else { 2028 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero"); 2029 return -ENOENT; 2030 } 2031 } 2032 2033 /* Find head of filter chain. */ 2034 2035 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2036 if (err) 2037 return err; 2038 2039 /* Take rtnl mutex if rtnl_held was set to true on previous iteration, 2040 * block is shared (no qdisc found), qdisc is not unlocked, classifier 2041 * type is not specified, classifier is not unlocked. 2042 */ 2043 if (rtnl_held || 2044 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2045 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) { 2046 rtnl_held = true; 2047 rtnl_lock(); 2048 } 2049 2050 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2051 if (err) 2052 goto errout; 2053 2054 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2055 extack); 2056 if (IS_ERR(block)) { 2057 err = PTR_ERR(block); 2058 goto errout; 2059 } 2060 2061 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2062 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2063 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2064 err = -EINVAL; 2065 goto errout; 2066 } 2067 chain = tcf_chain_get(block, chain_index, true); 2068 if (!chain) { 2069 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain"); 2070 err = -ENOMEM; 2071 goto errout; 2072 } 2073 2074 mutex_lock(&chain->filter_chain_lock); 2075 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2076 prio, prio_allocate); 2077 if (IS_ERR(tp)) { 2078 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2079 err = PTR_ERR(tp); 2080 goto errout_locked; 2081 } 2082 2083 if (tp == NULL) { 2084 struct tcf_proto *tp_new = NULL; 2085 2086 if (chain->flushing) { 2087 err = -EAGAIN; 2088 goto errout_locked; 2089 } 2090 2091 /* Proto-tcf does not exist, create new one */ 2092 2093 if (tca[TCA_KIND] == NULL || !protocol) { 2094 NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified"); 2095 err = -EINVAL; 2096 goto errout_locked; 2097 } 2098 2099 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2100 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); 2101 err = -ENOENT; 2102 goto errout_locked; 2103 } 2104 2105 if (prio_allocate) 2106 prio = tcf_auto_prio(tcf_chain_tp_prev(chain, 2107 &chain_info)); 2108 2109 mutex_unlock(&chain->filter_chain_lock); 2110 tp_new = tcf_proto_create(nla_data(tca[TCA_KIND]), 2111 protocol, prio, chain, rtnl_held, 2112 extack); 2113 if (IS_ERR(tp_new)) { 2114 err = PTR_ERR(tp_new); 2115 goto errout_tp; 2116 } 2117 2118 tp_created = 1; 2119 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio, 2120 rtnl_held); 2121 if (IS_ERR(tp)) { 2122 err = PTR_ERR(tp); 2123 goto errout_tp; 2124 } 2125 } else { 2126 mutex_unlock(&chain->filter_chain_lock); 2127 } 2128 2129 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2130 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2131 err = -EINVAL; 2132 goto errout; 2133 } 2134 2135 fh = tp->ops->get(tp, t->tcm_handle); 2136 2137 if (!fh) { 2138 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2139 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); 2140 err = -ENOENT; 2141 goto errout; 2142 } 2143 } else if (n->nlmsg_flags & NLM_F_EXCL) { 2144 tfilter_put(tp, fh); 2145 NL_SET_ERR_MSG(extack, "Filter already exists"); 2146 err = -EEXIST; 2147 goto errout; 2148 } 2149 2150 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) { 2151 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind"); 2152 err = -EINVAL; 2153 goto errout; 2154 } 2155 2156 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, 2157 n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE, 2158 rtnl_held, extack); 2159 if (err == 0) { 2160 tfilter_notify(net, skb, n, tp, block, q, parent, fh, 2161 RTM_NEWTFILTER, false, rtnl_held); 2162 tfilter_put(tp, fh); 2163 } 2164 2165 errout: 2166 if (err && tp_created) 2167 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL); 2168 errout_tp: 2169 if (chain) { 2170 if (tp && !IS_ERR(tp)) 2171 tcf_proto_put(tp, rtnl_held, NULL); 2172 if (!tp_created) 2173 tcf_chain_put(chain); 2174 } 2175 tcf_block_release(q, block, rtnl_held); 2176 2177 if (rtnl_held) 2178 rtnl_unlock(); 2179 2180 if (err == -EAGAIN) { 2181 /* Take rtnl lock in case EAGAIN is caused by concurrent flush 2182 * of target chain. 2183 */ 2184 rtnl_held = true; 2185 /* Replay the request. */ 2186 goto replay; 2187 } 2188 return err; 2189 2190 errout_locked: 2191 mutex_unlock(&chain->filter_chain_lock); 2192 goto errout; 2193 } 2194 2195 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2196 struct netlink_ext_ack *extack) 2197 { 2198 struct net *net = sock_net(skb->sk); 2199 struct nlattr *tca[TCA_MAX + 1]; 2200 struct tcmsg *t; 2201 u32 protocol; 2202 u32 prio; 2203 u32 parent; 2204 u32 chain_index; 2205 struct Qdisc *q = NULL; 2206 struct tcf_chain_info chain_info; 2207 struct tcf_chain *chain = NULL; 2208 struct tcf_block *block = NULL; 2209 struct tcf_proto *tp = NULL; 2210 unsigned long cl = 0; 2211 void *fh = NULL; 2212 int err; 2213 bool rtnl_held = false; 2214 2215 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) 2216 return -EPERM; 2217 2218 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2219 rtm_tca_policy, extack); 2220 if (err < 0) 2221 return err; 2222 2223 t = nlmsg_data(n); 2224 protocol = TC_H_MIN(t->tcm_info); 2225 prio = TC_H_MAJ(t->tcm_info); 2226 parent = t->tcm_parent; 2227 2228 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) { 2229 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set"); 2230 return -ENOENT; 2231 } 2232 2233 /* Find head of filter chain. */ 2234 2235 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2236 if (err) 2237 return err; 2238 2239 /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc 2240 * found), qdisc is not unlocked, classifier type is not specified, 2241 * classifier is not unlocked. 2242 */ 2243 if (!prio || 2244 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2245 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) { 2246 rtnl_held = true; 2247 rtnl_lock(); 2248 } 2249 2250 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2251 if (err) 2252 goto errout; 2253 2254 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2255 extack); 2256 if (IS_ERR(block)) { 2257 err = PTR_ERR(block); 2258 goto errout; 2259 } 2260 2261 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2262 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2263 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2264 err = -EINVAL; 2265 goto errout; 2266 } 2267 chain = tcf_chain_get(block, chain_index, false); 2268 if (!chain) { 2269 /* User requested flush on non-existent chain. Nothing to do, 2270 * so just return success. 2271 */ 2272 if (prio == 0) { 2273 err = 0; 2274 goto errout; 2275 } 2276 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2277 err = -ENOENT; 2278 goto errout; 2279 } 2280 2281 if (prio == 0) { 2282 tfilter_notify_chain(net, skb, block, q, parent, n, 2283 chain, RTM_DELTFILTER, rtnl_held); 2284 tcf_chain_flush(chain, rtnl_held); 2285 err = 0; 2286 goto errout; 2287 } 2288 2289 mutex_lock(&chain->filter_chain_lock); 2290 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2291 prio, false); 2292 if (!tp || IS_ERR(tp)) { 2293 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2294 err = tp ? PTR_ERR(tp) : -ENOENT; 2295 goto errout_locked; 2296 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2297 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2298 err = -EINVAL; 2299 goto errout_locked; 2300 } else if (t->tcm_handle == 0) { 2301 tcf_chain_tp_remove(chain, &chain_info, tp); 2302 mutex_unlock(&chain->filter_chain_lock); 2303 2304 tcf_proto_put(tp, rtnl_held, NULL); 2305 tfilter_notify(net, skb, n, tp, block, q, parent, fh, 2306 RTM_DELTFILTER, false, rtnl_held); 2307 err = 0; 2308 goto errout; 2309 } 2310 mutex_unlock(&chain->filter_chain_lock); 2311 2312 fh = tp->ops->get(tp, t->tcm_handle); 2313 2314 if (!fh) { 2315 NL_SET_ERR_MSG(extack, "Specified filter handle not found"); 2316 err = -ENOENT; 2317 } else { 2318 bool last; 2319 2320 err = tfilter_del_notify(net, skb, n, tp, block, 2321 q, parent, fh, false, &last, 2322 rtnl_held, extack); 2323 2324 if (err) 2325 goto errout; 2326 if (last) 2327 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack); 2328 } 2329 2330 errout: 2331 if (chain) { 2332 if (tp && !IS_ERR(tp)) 2333 tcf_proto_put(tp, rtnl_held, NULL); 2334 tcf_chain_put(chain); 2335 } 2336 tcf_block_release(q, block, rtnl_held); 2337 2338 if (rtnl_held) 2339 rtnl_unlock(); 2340 2341 return err; 2342 2343 errout_locked: 2344 mutex_unlock(&chain->filter_chain_lock); 2345 goto errout; 2346 } 2347 2348 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2349 struct netlink_ext_ack *extack) 2350 { 2351 struct net *net = sock_net(skb->sk); 2352 struct nlattr *tca[TCA_MAX + 1]; 2353 struct tcmsg *t; 2354 u32 protocol; 2355 u32 prio; 2356 u32 parent; 2357 u32 chain_index; 2358 struct Qdisc *q = NULL; 2359 struct tcf_chain_info chain_info; 2360 struct tcf_chain *chain = NULL; 2361 struct tcf_block *block = NULL; 2362 struct tcf_proto *tp = NULL; 2363 unsigned long cl = 0; 2364 void *fh = NULL; 2365 int err; 2366 bool rtnl_held = false; 2367 2368 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2369 rtm_tca_policy, extack); 2370 if (err < 0) 2371 return err; 2372 2373 t = nlmsg_data(n); 2374 protocol = TC_H_MIN(t->tcm_info); 2375 prio = TC_H_MAJ(t->tcm_info); 2376 parent = t->tcm_parent; 2377 2378 if (prio == 0) { 2379 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero"); 2380 return -ENOENT; 2381 } 2382 2383 /* Find head of filter chain. */ 2384 2385 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2386 if (err) 2387 return err; 2388 2389 /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not 2390 * unlocked, classifier type is not specified, classifier is not 2391 * unlocked. 2392 */ 2393 if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2394 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) { 2395 rtnl_held = true; 2396 rtnl_lock(); 2397 } 2398 2399 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2400 if (err) 2401 goto errout; 2402 2403 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2404 extack); 2405 if (IS_ERR(block)) { 2406 err = PTR_ERR(block); 2407 goto errout; 2408 } 2409 2410 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2411 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2412 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2413 err = -EINVAL; 2414 goto errout; 2415 } 2416 chain = tcf_chain_get(block, chain_index, false); 2417 if (!chain) { 2418 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2419 err = -EINVAL; 2420 goto errout; 2421 } 2422 2423 mutex_lock(&chain->filter_chain_lock); 2424 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2425 prio, false); 2426 mutex_unlock(&chain->filter_chain_lock); 2427 if (!tp || IS_ERR(tp)) { 2428 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2429 err = tp ? PTR_ERR(tp) : -ENOENT; 2430 goto errout; 2431 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2432 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2433 err = -EINVAL; 2434 goto errout; 2435 } 2436 2437 fh = tp->ops->get(tp, t->tcm_handle); 2438 2439 if (!fh) { 2440 NL_SET_ERR_MSG(extack, "Specified filter handle not found"); 2441 err = -ENOENT; 2442 } else { 2443 err = tfilter_notify(net, skb, n, tp, block, q, parent, 2444 fh, RTM_NEWTFILTER, true, rtnl_held); 2445 if (err < 0) 2446 NL_SET_ERR_MSG(extack, "Failed to send filter notify message"); 2447 } 2448 2449 tfilter_put(tp, fh); 2450 errout: 2451 if (chain) { 2452 if (tp && !IS_ERR(tp)) 2453 tcf_proto_put(tp, rtnl_held, NULL); 2454 tcf_chain_put(chain); 2455 } 2456 tcf_block_release(q, block, rtnl_held); 2457 2458 if (rtnl_held) 2459 rtnl_unlock(); 2460 2461 return err; 2462 } 2463 2464 struct tcf_dump_args { 2465 struct tcf_walker w; 2466 struct sk_buff *skb; 2467 struct netlink_callback *cb; 2468 struct tcf_block *block; 2469 struct Qdisc *q; 2470 u32 parent; 2471 }; 2472 2473 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg) 2474 { 2475 struct tcf_dump_args *a = (void *)arg; 2476 struct net *net = sock_net(a->skb->sk); 2477 2478 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent, 2479 n, NETLINK_CB(a->cb->skb).portid, 2480 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, 2481 RTM_NEWTFILTER, true); 2482 } 2483 2484 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent, 2485 struct sk_buff *skb, struct netlink_callback *cb, 2486 long index_start, long *p_index) 2487 { 2488 struct net *net = sock_net(skb->sk); 2489 struct tcf_block *block = chain->block; 2490 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2491 struct tcf_proto *tp, *tp_prev; 2492 struct tcf_dump_args arg; 2493 2494 for (tp = __tcf_get_next_proto(chain, NULL); 2495 tp; 2496 tp_prev = tp, 2497 tp = __tcf_get_next_proto(chain, tp), 2498 tcf_proto_put(tp_prev, true, NULL), 2499 (*p_index)++) { 2500 if (*p_index < index_start) 2501 continue; 2502 if (TC_H_MAJ(tcm->tcm_info) && 2503 TC_H_MAJ(tcm->tcm_info) != tp->prio) 2504 continue; 2505 if (TC_H_MIN(tcm->tcm_info) && 2506 TC_H_MIN(tcm->tcm_info) != tp->protocol) 2507 continue; 2508 if (*p_index > index_start) 2509 memset(&cb->args[1], 0, 2510 sizeof(cb->args) - sizeof(cb->args[0])); 2511 if (cb->args[1] == 0) { 2512 if (tcf_fill_node(net, skb, tp, block, q, parent, NULL, 2513 NETLINK_CB(cb->skb).portid, 2514 cb->nlh->nlmsg_seq, NLM_F_MULTI, 2515 RTM_NEWTFILTER, true) <= 0) 2516 goto errout; 2517 cb->args[1] = 1; 2518 } 2519 if (!tp->ops->walk) 2520 continue; 2521 arg.w.fn = tcf_node_dump; 2522 arg.skb = skb; 2523 arg.cb = cb; 2524 arg.block = block; 2525 arg.q = q; 2526 arg.parent = parent; 2527 arg.w.stop = 0; 2528 arg.w.skip = cb->args[1] - 1; 2529 arg.w.count = 0; 2530 arg.w.cookie = cb->args[2]; 2531 tp->ops->walk(tp, &arg.w, true); 2532 cb->args[2] = arg.w.cookie; 2533 cb->args[1] = arg.w.count + 1; 2534 if (arg.w.stop) 2535 goto errout; 2536 } 2537 return true; 2538 2539 errout: 2540 tcf_proto_put(tp, true, NULL); 2541 return false; 2542 } 2543 2544 /* called with RTNL */ 2545 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb) 2546 { 2547 struct tcf_chain *chain, *chain_prev; 2548 struct net *net = sock_net(skb->sk); 2549 struct nlattr *tca[TCA_MAX + 1]; 2550 struct Qdisc *q = NULL; 2551 struct tcf_block *block; 2552 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2553 long index_start; 2554 long index; 2555 u32 parent; 2556 int err; 2557 2558 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2559 return skb->len; 2560 2561 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX, 2562 NULL, cb->extack); 2563 if (err) 2564 return err; 2565 2566 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 2567 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); 2568 if (!block) 2569 goto out; 2570 /* If we work with block index, q is NULL and parent value 2571 * will never be used in the following code. The check 2572 * in tcf_fill_node prevents it. However, compiler does not 2573 * see that far, so set parent to zero to silence the warning 2574 * about parent being uninitialized. 2575 */ 2576 parent = 0; 2577 } else { 2578 const struct Qdisc_class_ops *cops; 2579 struct net_device *dev; 2580 unsigned long cl = 0; 2581 2582 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2583 if (!dev) 2584 return skb->len; 2585 2586 parent = tcm->tcm_parent; 2587 if (!parent) { 2588 q = dev->qdisc; 2589 parent = q->handle; 2590 } else { 2591 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); 2592 } 2593 if (!q) 2594 goto out; 2595 cops = q->ops->cl_ops; 2596 if (!cops) 2597 goto out; 2598 if (!cops->tcf_block) 2599 goto out; 2600 if (TC_H_MIN(tcm->tcm_parent)) { 2601 cl = cops->find(q, tcm->tcm_parent); 2602 if (cl == 0) 2603 goto out; 2604 } 2605 block = cops->tcf_block(q, cl, NULL); 2606 if (!block) 2607 goto out; 2608 if (tcf_block_shared(block)) 2609 q = NULL; 2610 } 2611 2612 index_start = cb->args[0]; 2613 index = 0; 2614 2615 for (chain = __tcf_get_next_chain(block, NULL); 2616 chain; 2617 chain_prev = chain, 2618 chain = __tcf_get_next_chain(block, chain), 2619 tcf_chain_put(chain_prev)) { 2620 if (tca[TCA_CHAIN] && 2621 nla_get_u32(tca[TCA_CHAIN]) != chain->index) 2622 continue; 2623 if (!tcf_chain_dump(chain, q, parent, skb, cb, 2624 index_start, &index)) { 2625 tcf_chain_put(chain); 2626 err = -EMSGSIZE; 2627 break; 2628 } 2629 } 2630 2631 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) 2632 tcf_block_refcnt_put(block, true); 2633 cb->args[0] = index; 2634 2635 out: 2636 /* If we did no progress, the error (EMSGSIZE) is real */ 2637 if (skb->len == 0 && err) 2638 return err; 2639 return skb->len; 2640 } 2641 2642 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops, 2643 void *tmplt_priv, u32 chain_index, 2644 struct net *net, struct sk_buff *skb, 2645 struct tcf_block *block, 2646 u32 portid, u32 seq, u16 flags, int event) 2647 { 2648 unsigned char *b = skb_tail_pointer(skb); 2649 const struct tcf_proto_ops *ops; 2650 struct nlmsghdr *nlh; 2651 struct tcmsg *tcm; 2652 void *priv; 2653 2654 ops = tmplt_ops; 2655 priv = tmplt_priv; 2656 2657 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 2658 if (!nlh) 2659 goto out_nlmsg_trim; 2660 tcm = nlmsg_data(nlh); 2661 tcm->tcm_family = AF_UNSPEC; 2662 tcm->tcm__pad1 = 0; 2663 tcm->tcm__pad2 = 0; 2664 tcm->tcm_handle = 0; 2665 if (block->q) { 2666 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex; 2667 tcm->tcm_parent = block->q->handle; 2668 } else { 2669 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; 2670 tcm->tcm_block_index = block->index; 2671 } 2672 2673 if (nla_put_u32(skb, TCA_CHAIN, chain_index)) 2674 goto nla_put_failure; 2675 2676 if (ops) { 2677 if (nla_put_string(skb, TCA_KIND, ops->kind)) 2678 goto nla_put_failure; 2679 if (ops->tmplt_dump(skb, net, priv) < 0) 2680 goto nla_put_failure; 2681 } 2682 2683 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2684 return skb->len; 2685 2686 out_nlmsg_trim: 2687 nla_put_failure: 2688 nlmsg_trim(skb, b); 2689 return -EMSGSIZE; 2690 } 2691 2692 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, 2693 u32 seq, u16 flags, int event, bool unicast) 2694 { 2695 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2696 struct tcf_block *block = chain->block; 2697 struct net *net = block->net; 2698 struct sk_buff *skb; 2699 int err = 0; 2700 2701 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2702 if (!skb) 2703 return -ENOBUFS; 2704 2705 if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, 2706 chain->index, net, skb, block, portid, 2707 seq, flags, event) <= 0) { 2708 kfree_skb(skb); 2709 return -EINVAL; 2710 } 2711 2712 if (unicast) 2713 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); 2714 else 2715 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2716 flags & NLM_F_ECHO); 2717 2718 if (err > 0) 2719 err = 0; 2720 return err; 2721 } 2722 2723 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, 2724 void *tmplt_priv, u32 chain_index, 2725 struct tcf_block *block, struct sk_buff *oskb, 2726 u32 seq, u16 flags, bool unicast) 2727 { 2728 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2729 struct net *net = block->net; 2730 struct sk_buff *skb; 2731 2732 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2733 if (!skb) 2734 return -ENOBUFS; 2735 2736 if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb, 2737 block, portid, seq, flags, RTM_DELCHAIN) <= 0) { 2738 kfree_skb(skb); 2739 return -EINVAL; 2740 } 2741 2742 if (unicast) 2743 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); 2744 2745 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO); 2746 } 2747 2748 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net, 2749 struct nlattr **tca, 2750 struct netlink_ext_ack *extack) 2751 { 2752 const struct tcf_proto_ops *ops; 2753 void *tmplt_priv; 2754 2755 /* If kind is not set, user did not specify template. */ 2756 if (!tca[TCA_KIND]) 2757 return 0; 2758 2759 ops = tcf_proto_lookup_ops(nla_data(tca[TCA_KIND]), true, extack); 2760 if (IS_ERR(ops)) 2761 return PTR_ERR(ops); 2762 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) { 2763 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier"); 2764 return -EOPNOTSUPP; 2765 } 2766 2767 tmplt_priv = ops->tmplt_create(net, chain, tca, extack); 2768 if (IS_ERR(tmplt_priv)) { 2769 module_put(ops->owner); 2770 return PTR_ERR(tmplt_priv); 2771 } 2772 chain->tmplt_ops = ops; 2773 chain->tmplt_priv = tmplt_priv; 2774 return 0; 2775 } 2776 2777 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops, 2778 void *tmplt_priv) 2779 { 2780 /* If template ops are set, no work to do for us. */ 2781 if (!tmplt_ops) 2782 return; 2783 2784 tmplt_ops->tmplt_destroy(tmplt_priv); 2785 module_put(tmplt_ops->owner); 2786 } 2787 2788 /* Add/delete/get a chain */ 2789 2790 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n, 2791 struct netlink_ext_ack *extack) 2792 { 2793 struct net *net = sock_net(skb->sk); 2794 struct nlattr *tca[TCA_MAX + 1]; 2795 struct tcmsg *t; 2796 u32 parent; 2797 u32 chain_index; 2798 struct Qdisc *q = NULL; 2799 struct tcf_chain *chain = NULL; 2800 struct tcf_block *block; 2801 unsigned long cl; 2802 int err; 2803 2804 if (n->nlmsg_type != RTM_GETCHAIN && 2805 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) 2806 return -EPERM; 2807 2808 replay: 2809 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2810 rtm_tca_policy, extack); 2811 if (err < 0) 2812 return err; 2813 2814 t = nlmsg_data(n); 2815 parent = t->tcm_parent; 2816 cl = 0; 2817 2818 block = tcf_block_find(net, &q, &parent, &cl, 2819 t->tcm_ifindex, t->tcm_block_index, extack); 2820 if (IS_ERR(block)) 2821 return PTR_ERR(block); 2822 2823 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2824 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2825 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2826 err = -EINVAL; 2827 goto errout_block; 2828 } 2829 2830 mutex_lock(&block->lock); 2831 chain = tcf_chain_lookup(block, chain_index); 2832 if (n->nlmsg_type == RTM_NEWCHAIN) { 2833 if (chain) { 2834 if (tcf_chain_held_by_acts_only(chain)) { 2835 /* The chain exists only because there is 2836 * some action referencing it. 2837 */ 2838 tcf_chain_hold(chain); 2839 } else { 2840 NL_SET_ERR_MSG(extack, "Filter chain already exists"); 2841 err = -EEXIST; 2842 goto errout_block_locked; 2843 } 2844 } else { 2845 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2846 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain"); 2847 err = -ENOENT; 2848 goto errout_block_locked; 2849 } 2850 chain = tcf_chain_create(block, chain_index); 2851 if (!chain) { 2852 NL_SET_ERR_MSG(extack, "Failed to create filter chain"); 2853 err = -ENOMEM; 2854 goto errout_block_locked; 2855 } 2856 } 2857 } else { 2858 if (!chain || tcf_chain_held_by_acts_only(chain)) { 2859 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2860 err = -EINVAL; 2861 goto errout_block_locked; 2862 } 2863 tcf_chain_hold(chain); 2864 } 2865 2866 if (n->nlmsg_type == RTM_NEWCHAIN) { 2867 /* Modifying chain requires holding parent block lock. In case 2868 * the chain was successfully added, take a reference to the 2869 * chain. This ensures that an empty chain does not disappear at 2870 * the end of this function. 2871 */ 2872 tcf_chain_hold(chain); 2873 chain->explicitly_created = true; 2874 } 2875 mutex_unlock(&block->lock); 2876 2877 switch (n->nlmsg_type) { 2878 case RTM_NEWCHAIN: 2879 err = tc_chain_tmplt_add(chain, net, tca, extack); 2880 if (err) { 2881 tcf_chain_put_explicitly_created(chain); 2882 goto errout; 2883 } 2884 2885 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL, 2886 RTM_NEWCHAIN, false); 2887 break; 2888 case RTM_DELCHAIN: 2889 tfilter_notify_chain(net, skb, block, q, parent, n, 2890 chain, RTM_DELTFILTER, true); 2891 /* Flush the chain first as the user requested chain removal. */ 2892 tcf_chain_flush(chain, true); 2893 /* In case the chain was successfully deleted, put a reference 2894 * to the chain previously taken during addition. 2895 */ 2896 tcf_chain_put_explicitly_created(chain); 2897 break; 2898 case RTM_GETCHAIN: 2899 err = tc_chain_notify(chain, skb, n->nlmsg_seq, 2900 n->nlmsg_seq, n->nlmsg_type, true); 2901 if (err < 0) 2902 NL_SET_ERR_MSG(extack, "Failed to send chain notify message"); 2903 break; 2904 default: 2905 err = -EOPNOTSUPP; 2906 NL_SET_ERR_MSG(extack, "Unsupported message type"); 2907 goto errout; 2908 } 2909 2910 errout: 2911 tcf_chain_put(chain); 2912 errout_block: 2913 tcf_block_release(q, block, true); 2914 if (err == -EAGAIN) 2915 /* Replay the request. */ 2916 goto replay; 2917 return err; 2918 2919 errout_block_locked: 2920 mutex_unlock(&block->lock); 2921 goto errout_block; 2922 } 2923 2924 /* called with RTNL */ 2925 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb) 2926 { 2927 struct net *net = sock_net(skb->sk); 2928 struct nlattr *tca[TCA_MAX + 1]; 2929 struct Qdisc *q = NULL; 2930 struct tcf_block *block; 2931 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2932 struct tcf_chain *chain; 2933 long index_start; 2934 long index; 2935 u32 parent; 2936 int err; 2937 2938 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2939 return skb->len; 2940 2941 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX, 2942 rtm_tca_policy, cb->extack); 2943 if (err) 2944 return err; 2945 2946 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 2947 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); 2948 if (!block) 2949 goto out; 2950 /* If we work with block index, q is NULL and parent value 2951 * will never be used in the following code. The check 2952 * in tcf_fill_node prevents it. However, compiler does not 2953 * see that far, so set parent to zero to silence the warning 2954 * about parent being uninitialized. 2955 */ 2956 parent = 0; 2957 } else { 2958 const struct Qdisc_class_ops *cops; 2959 struct net_device *dev; 2960 unsigned long cl = 0; 2961 2962 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2963 if (!dev) 2964 return skb->len; 2965 2966 parent = tcm->tcm_parent; 2967 if (!parent) { 2968 q = dev->qdisc; 2969 parent = q->handle; 2970 } else { 2971 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); 2972 } 2973 if (!q) 2974 goto out; 2975 cops = q->ops->cl_ops; 2976 if (!cops) 2977 goto out; 2978 if (!cops->tcf_block) 2979 goto out; 2980 if (TC_H_MIN(tcm->tcm_parent)) { 2981 cl = cops->find(q, tcm->tcm_parent); 2982 if (cl == 0) 2983 goto out; 2984 } 2985 block = cops->tcf_block(q, cl, NULL); 2986 if (!block) 2987 goto out; 2988 if (tcf_block_shared(block)) 2989 q = NULL; 2990 } 2991 2992 index_start = cb->args[0]; 2993 index = 0; 2994 2995 mutex_lock(&block->lock); 2996 list_for_each_entry(chain, &block->chain_list, list) { 2997 if ((tca[TCA_CHAIN] && 2998 nla_get_u32(tca[TCA_CHAIN]) != chain->index)) 2999 continue; 3000 if (index < index_start) { 3001 index++; 3002 continue; 3003 } 3004 if (tcf_chain_held_by_acts_only(chain)) 3005 continue; 3006 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, 3007 chain->index, net, skb, block, 3008 NETLINK_CB(cb->skb).portid, 3009 cb->nlh->nlmsg_seq, NLM_F_MULTI, 3010 RTM_NEWCHAIN); 3011 if (err <= 0) 3012 break; 3013 index++; 3014 } 3015 mutex_unlock(&block->lock); 3016 3017 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) 3018 tcf_block_refcnt_put(block, true); 3019 cb->args[0] = index; 3020 3021 out: 3022 /* If we did no progress, the error (EMSGSIZE) is real */ 3023 if (skb->len == 0 && err) 3024 return err; 3025 return skb->len; 3026 } 3027 3028 void tcf_exts_destroy(struct tcf_exts *exts) 3029 { 3030 #ifdef CONFIG_NET_CLS_ACT 3031 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND); 3032 kfree(exts->actions); 3033 exts->nr_actions = 0; 3034 #endif 3035 } 3036 EXPORT_SYMBOL(tcf_exts_destroy); 3037 3038 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb, 3039 struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr, 3040 bool rtnl_held, struct netlink_ext_ack *extack) 3041 { 3042 #ifdef CONFIG_NET_CLS_ACT 3043 { 3044 struct tc_action *act; 3045 size_t attr_size = 0; 3046 3047 if (exts->police && tb[exts->police]) { 3048 act = tcf_action_init_1(net, tp, tb[exts->police], 3049 rate_tlv, "police", ovr, 3050 TCA_ACT_BIND, rtnl_held, 3051 extack); 3052 if (IS_ERR(act)) 3053 return PTR_ERR(act); 3054 3055 act->type = exts->type = TCA_OLD_COMPAT; 3056 exts->actions[0] = act; 3057 exts->nr_actions = 1; 3058 } else if (exts->action && tb[exts->action]) { 3059 int err; 3060 3061 err = tcf_action_init(net, tp, tb[exts->action], 3062 rate_tlv, NULL, ovr, TCA_ACT_BIND, 3063 exts->actions, &attr_size, 3064 rtnl_held, extack); 3065 if (err < 0) 3066 return err; 3067 exts->nr_actions = err; 3068 } 3069 } 3070 #else 3071 if ((exts->action && tb[exts->action]) || 3072 (exts->police && tb[exts->police])) { 3073 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)"); 3074 return -EOPNOTSUPP; 3075 } 3076 #endif 3077 3078 return 0; 3079 } 3080 EXPORT_SYMBOL(tcf_exts_validate); 3081 3082 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src) 3083 { 3084 #ifdef CONFIG_NET_CLS_ACT 3085 struct tcf_exts old = *dst; 3086 3087 *dst = *src; 3088 tcf_exts_destroy(&old); 3089 #endif 3090 } 3091 EXPORT_SYMBOL(tcf_exts_change); 3092 3093 #ifdef CONFIG_NET_CLS_ACT 3094 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts) 3095 { 3096 if (exts->nr_actions == 0) 3097 return NULL; 3098 else 3099 return exts->actions[0]; 3100 } 3101 #endif 3102 3103 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts) 3104 { 3105 #ifdef CONFIG_NET_CLS_ACT 3106 struct nlattr *nest; 3107 3108 if (exts->action && tcf_exts_has_actions(exts)) { 3109 /* 3110 * again for backward compatible mode - we want 3111 * to work with both old and new modes of entering 3112 * tc data even if iproute2 was newer - jhs 3113 */ 3114 if (exts->type != TCA_OLD_COMPAT) { 3115 nest = nla_nest_start_noflag(skb, exts->action); 3116 if (nest == NULL) 3117 goto nla_put_failure; 3118 3119 if (tcf_action_dump(skb, exts->actions, 0, 0) < 0) 3120 goto nla_put_failure; 3121 nla_nest_end(skb, nest); 3122 } else if (exts->police) { 3123 struct tc_action *act = tcf_exts_first_act(exts); 3124 nest = nla_nest_start_noflag(skb, exts->police); 3125 if (nest == NULL || !act) 3126 goto nla_put_failure; 3127 if (tcf_action_dump_old(skb, act, 0, 0) < 0) 3128 goto nla_put_failure; 3129 nla_nest_end(skb, nest); 3130 } 3131 } 3132 return 0; 3133 3134 nla_put_failure: 3135 nla_nest_cancel(skb, nest); 3136 return -1; 3137 #else 3138 return 0; 3139 #endif 3140 } 3141 EXPORT_SYMBOL(tcf_exts_dump); 3142 3143 3144 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts) 3145 { 3146 #ifdef CONFIG_NET_CLS_ACT 3147 struct tc_action *a = tcf_exts_first_act(exts); 3148 if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0) 3149 return -1; 3150 #endif 3151 return 0; 3152 } 3153 EXPORT_SYMBOL(tcf_exts_dump_stats); 3154 3155 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, 3156 void *type_data, bool err_stop) 3157 { 3158 struct tcf_block_cb *block_cb; 3159 int ok_count = 0; 3160 int err; 3161 3162 /* Make sure all netdevs sharing this block are offload-capable. */ 3163 if (block->nooffloaddevcnt && err_stop) 3164 return -EOPNOTSUPP; 3165 3166 list_for_each_entry(block_cb, &block->cb_list, list) { 3167 err = block_cb->cb(type, type_data, block_cb->cb_priv); 3168 if (err) { 3169 if (err_stop) 3170 return err; 3171 } else { 3172 ok_count++; 3173 } 3174 } 3175 return ok_count; 3176 } 3177 EXPORT_SYMBOL(tc_setup_cb_call); 3178 3179 int tc_setup_flow_action(struct flow_action *flow_action, 3180 const struct tcf_exts *exts) 3181 { 3182 const struct tc_action *act; 3183 int i, j, k; 3184 3185 if (!exts) 3186 return 0; 3187 3188 j = 0; 3189 tcf_exts_for_each_action(i, act, exts) { 3190 struct flow_action_entry *entry; 3191 3192 entry = &flow_action->entries[j]; 3193 if (is_tcf_gact_ok(act)) { 3194 entry->id = FLOW_ACTION_ACCEPT; 3195 } else if (is_tcf_gact_shot(act)) { 3196 entry->id = FLOW_ACTION_DROP; 3197 } else if (is_tcf_gact_trap(act)) { 3198 entry->id = FLOW_ACTION_TRAP; 3199 } else if (is_tcf_gact_goto_chain(act)) { 3200 entry->id = FLOW_ACTION_GOTO; 3201 entry->chain_index = tcf_gact_goto_chain_index(act); 3202 } else if (is_tcf_mirred_egress_redirect(act)) { 3203 entry->id = FLOW_ACTION_REDIRECT; 3204 entry->dev = tcf_mirred_dev(act); 3205 } else if (is_tcf_mirred_egress_mirror(act)) { 3206 entry->id = FLOW_ACTION_MIRRED; 3207 entry->dev = tcf_mirred_dev(act); 3208 } else if (is_tcf_vlan(act)) { 3209 switch (tcf_vlan_action(act)) { 3210 case TCA_VLAN_ACT_PUSH: 3211 entry->id = FLOW_ACTION_VLAN_PUSH; 3212 entry->vlan.vid = tcf_vlan_push_vid(act); 3213 entry->vlan.proto = tcf_vlan_push_proto(act); 3214 entry->vlan.prio = tcf_vlan_push_prio(act); 3215 break; 3216 case TCA_VLAN_ACT_POP: 3217 entry->id = FLOW_ACTION_VLAN_POP; 3218 break; 3219 case TCA_VLAN_ACT_MODIFY: 3220 entry->id = FLOW_ACTION_VLAN_MANGLE; 3221 entry->vlan.vid = tcf_vlan_push_vid(act); 3222 entry->vlan.proto = tcf_vlan_push_proto(act); 3223 entry->vlan.prio = tcf_vlan_push_prio(act); 3224 break; 3225 default: 3226 goto err_out; 3227 } 3228 } else if (is_tcf_tunnel_set(act)) { 3229 entry->id = FLOW_ACTION_TUNNEL_ENCAP; 3230 entry->tunnel = tcf_tunnel_info(act); 3231 } else if (is_tcf_tunnel_release(act)) { 3232 entry->id = FLOW_ACTION_TUNNEL_DECAP; 3233 } else if (is_tcf_pedit(act)) { 3234 for (k = 0; k < tcf_pedit_nkeys(act); k++) { 3235 switch (tcf_pedit_cmd(act, k)) { 3236 case TCA_PEDIT_KEY_EX_CMD_SET: 3237 entry->id = FLOW_ACTION_MANGLE; 3238 break; 3239 case TCA_PEDIT_KEY_EX_CMD_ADD: 3240 entry->id = FLOW_ACTION_ADD; 3241 break; 3242 default: 3243 goto err_out; 3244 } 3245 entry->mangle.htype = tcf_pedit_htype(act, k); 3246 entry->mangle.mask = tcf_pedit_mask(act, k); 3247 entry->mangle.val = tcf_pedit_val(act, k); 3248 entry->mangle.offset = tcf_pedit_offset(act, k); 3249 entry = &flow_action->entries[++j]; 3250 } 3251 } else if (is_tcf_csum(act)) { 3252 entry->id = FLOW_ACTION_CSUM; 3253 entry->csum_flags = tcf_csum_update_flags(act); 3254 } else if (is_tcf_skbedit_mark(act)) { 3255 entry->id = FLOW_ACTION_MARK; 3256 entry->mark = tcf_skbedit_mark(act); 3257 } else if (is_tcf_sample(act)) { 3258 entry->id = FLOW_ACTION_SAMPLE; 3259 entry->sample.psample_group = 3260 tcf_sample_psample_group(act); 3261 entry->sample.trunc_size = tcf_sample_trunc_size(act); 3262 entry->sample.truncate = tcf_sample_truncate(act); 3263 entry->sample.rate = tcf_sample_rate(act); 3264 } else if (is_tcf_police(act)) { 3265 entry->id = FLOW_ACTION_POLICE; 3266 entry->police.burst = tcf_police_tcfp_burst(act); 3267 entry->police.rate_bytes_ps = 3268 tcf_police_rate_bytes_ps(act); 3269 } else { 3270 goto err_out; 3271 } 3272 3273 if (!is_tcf_pedit(act)) 3274 j++; 3275 } 3276 return 0; 3277 err_out: 3278 return -EOPNOTSUPP; 3279 } 3280 EXPORT_SYMBOL(tc_setup_flow_action); 3281 3282 unsigned int tcf_exts_num_actions(struct tcf_exts *exts) 3283 { 3284 unsigned int num_acts = 0; 3285 struct tc_action *act; 3286 int i; 3287 3288 tcf_exts_for_each_action(i, act, exts) { 3289 if (is_tcf_pedit(act)) 3290 num_acts += tcf_pedit_nkeys(act); 3291 else 3292 num_acts++; 3293 } 3294 return num_acts; 3295 } 3296 EXPORT_SYMBOL(tcf_exts_num_actions); 3297 3298 static __net_init int tcf_net_init(struct net *net) 3299 { 3300 struct tcf_net *tn = net_generic(net, tcf_net_id); 3301 3302 spin_lock_init(&tn->idr_lock); 3303 idr_init(&tn->idr); 3304 return 0; 3305 } 3306 3307 static void __net_exit tcf_net_exit(struct net *net) 3308 { 3309 struct tcf_net *tn = net_generic(net, tcf_net_id); 3310 3311 idr_destroy(&tn->idr); 3312 } 3313 3314 static struct pernet_operations tcf_net_ops = { 3315 .init = tcf_net_init, 3316 .exit = tcf_net_exit, 3317 .id = &tcf_net_id, 3318 .size = sizeof(struct tcf_net), 3319 }; 3320 3321 static int __init tc_filter_init(void) 3322 { 3323 int err; 3324 3325 tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0); 3326 if (!tc_filter_wq) 3327 return -ENOMEM; 3328 3329 err = register_pernet_subsys(&tcf_net_ops); 3330 if (err) 3331 goto err_register_pernet_subsys; 3332 3333 err = rhashtable_init(&indr_setup_block_ht, 3334 &tc_indr_setup_block_ht_params); 3335 if (err) 3336 goto err_rhash_setup_block_ht; 3337 3338 rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, 3339 RTNL_FLAG_DOIT_UNLOCKED); 3340 rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, 3341 RTNL_FLAG_DOIT_UNLOCKED); 3342 rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter, 3343 tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED); 3344 rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0); 3345 rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0); 3346 rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain, 3347 tc_dump_chain, 0); 3348 3349 return 0; 3350 3351 err_rhash_setup_block_ht: 3352 unregister_pernet_subsys(&tcf_net_ops); 3353 err_register_pernet_subsys: 3354 destroy_workqueue(tc_filter_wq); 3355 return err; 3356 } 3357 3358 subsys_initcall(tc_filter_init); 3359