1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/cls_api.c Packet classifier API. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * 7 * Changes: 8 * 9 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support 10 */ 11 12 #include <linux/module.h> 13 #include <linux/types.h> 14 #include <linux/kernel.h> 15 #include <linux/string.h> 16 #include <linux/errno.h> 17 #include <linux/err.h> 18 #include <linux/skbuff.h> 19 #include <linux/init.h> 20 #include <linux/kmod.h> 21 #include <linux/slab.h> 22 #include <linux/idr.h> 23 #include <linux/jhash.h> 24 #include <linux/rculist.h> 25 #include <linux/rhashtable.h> 26 #include <net/net_namespace.h> 27 #include <net/sock.h> 28 #include <net/netlink.h> 29 #include <net/pkt_sched.h> 30 #include <net/pkt_cls.h> 31 #include <net/tc_act/tc_pedit.h> 32 #include <net/tc_act/tc_mirred.h> 33 #include <net/tc_act/tc_vlan.h> 34 #include <net/tc_act/tc_tunnel_key.h> 35 #include <net/tc_act/tc_csum.h> 36 #include <net/tc_act/tc_gact.h> 37 #include <net/tc_act/tc_police.h> 38 #include <net/tc_act/tc_sample.h> 39 #include <net/tc_act/tc_skbedit.h> 40 #include <net/tc_act/tc_ct.h> 41 #include <net/tc_act/tc_mpls.h> 42 #include <net/tc_act/tc_gate.h> 43 #include <net/flow_offload.h> 44 #include <net/tc_wrapper.h> 45 46 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1]; 47 48 /* The list of all installed classifier types */ 49 static LIST_HEAD(tcf_proto_base); 50 51 /* Protects list of registered TC modules. It is pure SMP lock. */ 52 static DEFINE_RWLOCK(cls_mod_lock); 53 54 static struct xarray tcf_exts_miss_cookies_xa; 55 struct tcf_exts_miss_cookie_node { 56 const struct tcf_chain *chain; 57 const struct tcf_proto *tp; 58 const struct tcf_exts *exts; 59 u32 chain_index; 60 u32 tp_prio; 61 u32 handle; 62 u32 miss_cookie_base; 63 struct rcu_head rcu; 64 }; 65 66 /* Each tc action entry cookie will be comprised of 32bit miss_cookie_base + 67 * action index in the exts tc actions array. 68 */ 69 union tcf_exts_miss_cookie { 70 struct { 71 u32 miss_cookie_base; 72 u32 act_index; 73 }; 74 u64 miss_cookie; 75 }; 76 77 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 78 static int 79 tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp, 80 u32 handle) 81 { 82 struct tcf_exts_miss_cookie_node *n; 83 static u32 next; 84 int err; 85 86 if (WARN_ON(!handle || !tp->ops->get_exts)) 87 return -EINVAL; 88 89 n = kzalloc(sizeof(*n), GFP_KERNEL); 90 if (!n) 91 return -ENOMEM; 92 93 n->chain_index = tp->chain->index; 94 n->chain = tp->chain; 95 n->tp_prio = tp->prio; 96 n->tp = tp; 97 n->exts = exts; 98 n->handle = handle; 99 100 err = xa_alloc_cyclic(&tcf_exts_miss_cookies_xa, &n->miss_cookie_base, 101 n, xa_limit_32b, &next, GFP_KERNEL); 102 if (err) 103 goto err_xa_alloc; 104 105 exts->miss_cookie_node = n; 106 return 0; 107 108 err_xa_alloc: 109 kfree(n); 110 return err; 111 } 112 113 static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts) 114 { 115 struct tcf_exts_miss_cookie_node *n; 116 117 if (!exts->miss_cookie_node) 118 return; 119 120 n = exts->miss_cookie_node; 121 xa_erase(&tcf_exts_miss_cookies_xa, n->miss_cookie_base); 122 kfree_rcu(n, rcu); 123 } 124 125 static struct tcf_exts_miss_cookie_node * 126 tcf_exts_miss_cookie_lookup(u64 miss_cookie, int *act_index) 127 { 128 union tcf_exts_miss_cookie mc = { .miss_cookie = miss_cookie, }; 129 130 *act_index = mc.act_index; 131 return xa_load(&tcf_exts_miss_cookies_xa, mc.miss_cookie_base); 132 } 133 #else /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */ 134 static int 135 tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp, 136 u32 handle) 137 { 138 return 0; 139 } 140 141 static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts) 142 { 143 } 144 #endif /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */ 145 146 static u64 tcf_exts_miss_cookie_get(u32 miss_cookie_base, int act_index) 147 { 148 union tcf_exts_miss_cookie mc = { .act_index = act_index, }; 149 150 if (!miss_cookie_base) 151 return 0; 152 153 mc.miss_cookie_base = miss_cookie_base; 154 return mc.miss_cookie; 155 } 156 157 #ifdef CONFIG_NET_CLS_ACT 158 DEFINE_STATIC_KEY_FALSE(tc_skb_ext_tc); 159 EXPORT_SYMBOL(tc_skb_ext_tc); 160 161 void tc_skb_ext_tc_enable(void) 162 { 163 static_branch_inc(&tc_skb_ext_tc); 164 } 165 EXPORT_SYMBOL(tc_skb_ext_tc_enable); 166 167 void tc_skb_ext_tc_disable(void) 168 { 169 static_branch_dec(&tc_skb_ext_tc); 170 } 171 EXPORT_SYMBOL(tc_skb_ext_tc_disable); 172 #endif 173 174 static u32 destroy_obj_hashfn(const struct tcf_proto *tp) 175 { 176 return jhash_3words(tp->chain->index, tp->prio, 177 (__force __u32)tp->protocol, 0); 178 } 179 180 static void tcf_proto_signal_destroying(struct tcf_chain *chain, 181 struct tcf_proto *tp) 182 { 183 struct tcf_block *block = chain->block; 184 185 mutex_lock(&block->proto_destroy_lock); 186 hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node, 187 destroy_obj_hashfn(tp)); 188 mutex_unlock(&block->proto_destroy_lock); 189 } 190 191 static bool tcf_proto_cmp(const struct tcf_proto *tp1, 192 const struct tcf_proto *tp2) 193 { 194 return tp1->chain->index == tp2->chain->index && 195 tp1->prio == tp2->prio && 196 tp1->protocol == tp2->protocol; 197 } 198 199 static bool tcf_proto_exists_destroying(struct tcf_chain *chain, 200 struct tcf_proto *tp) 201 { 202 u32 hash = destroy_obj_hashfn(tp); 203 struct tcf_proto *iter; 204 bool found = false; 205 206 rcu_read_lock(); 207 hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter, 208 destroy_ht_node, hash) { 209 if (tcf_proto_cmp(tp, iter)) { 210 found = true; 211 break; 212 } 213 } 214 rcu_read_unlock(); 215 216 return found; 217 } 218 219 static void 220 tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp) 221 { 222 struct tcf_block *block = chain->block; 223 224 mutex_lock(&block->proto_destroy_lock); 225 if (hash_hashed(&tp->destroy_ht_node)) 226 hash_del_rcu(&tp->destroy_ht_node); 227 mutex_unlock(&block->proto_destroy_lock); 228 } 229 230 /* Find classifier type by string name */ 231 232 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind) 233 { 234 const struct tcf_proto_ops *t, *res = NULL; 235 236 if (kind) { 237 read_lock(&cls_mod_lock); 238 list_for_each_entry(t, &tcf_proto_base, head) { 239 if (strcmp(kind, t->kind) == 0) { 240 if (try_module_get(t->owner)) 241 res = t; 242 break; 243 } 244 } 245 read_unlock(&cls_mod_lock); 246 } 247 return res; 248 } 249 250 static const struct tcf_proto_ops * 251 tcf_proto_lookup_ops(const char *kind, bool rtnl_held, 252 struct netlink_ext_ack *extack) 253 { 254 const struct tcf_proto_ops *ops; 255 256 ops = __tcf_proto_lookup_ops(kind); 257 if (ops) 258 return ops; 259 #ifdef CONFIG_MODULES 260 if (rtnl_held) 261 rtnl_unlock(); 262 request_module("cls_%s", kind); 263 if (rtnl_held) 264 rtnl_lock(); 265 ops = __tcf_proto_lookup_ops(kind); 266 /* We dropped the RTNL semaphore in order to perform 267 * the module load. So, even if we succeeded in loading 268 * the module we have to replay the request. We indicate 269 * this using -EAGAIN. 270 */ 271 if (ops) { 272 module_put(ops->owner); 273 return ERR_PTR(-EAGAIN); 274 } 275 #endif 276 NL_SET_ERR_MSG(extack, "TC classifier not found"); 277 return ERR_PTR(-ENOENT); 278 } 279 280 /* Register(unregister) new classifier type */ 281 282 int register_tcf_proto_ops(struct tcf_proto_ops *ops) 283 { 284 struct tcf_proto_ops *t; 285 int rc = -EEXIST; 286 287 write_lock(&cls_mod_lock); 288 list_for_each_entry(t, &tcf_proto_base, head) 289 if (!strcmp(ops->kind, t->kind)) 290 goto out; 291 292 list_add_tail(&ops->head, &tcf_proto_base); 293 rc = 0; 294 out: 295 write_unlock(&cls_mod_lock); 296 return rc; 297 } 298 EXPORT_SYMBOL(register_tcf_proto_ops); 299 300 static struct workqueue_struct *tc_filter_wq; 301 302 void unregister_tcf_proto_ops(struct tcf_proto_ops *ops) 303 { 304 struct tcf_proto_ops *t; 305 int rc = -ENOENT; 306 307 /* Wait for outstanding call_rcu()s, if any, from a 308 * tcf_proto_ops's destroy() handler. 309 */ 310 rcu_barrier(); 311 flush_workqueue(tc_filter_wq); 312 313 write_lock(&cls_mod_lock); 314 list_for_each_entry(t, &tcf_proto_base, head) { 315 if (t == ops) { 316 list_del(&t->head); 317 rc = 0; 318 break; 319 } 320 } 321 write_unlock(&cls_mod_lock); 322 323 WARN(rc, "unregister tc filter kind(%s) failed %d\n", ops->kind, rc); 324 } 325 EXPORT_SYMBOL(unregister_tcf_proto_ops); 326 327 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func) 328 { 329 INIT_RCU_WORK(rwork, func); 330 return queue_rcu_work(tc_filter_wq, rwork); 331 } 332 EXPORT_SYMBOL(tcf_queue_work); 333 334 /* Select new prio value from the range, managed by kernel. */ 335 336 static inline u32 tcf_auto_prio(struct tcf_proto *tp) 337 { 338 u32 first = TC_H_MAKE(0xC0000000U, 0U); 339 340 if (tp) 341 first = tp->prio - 1; 342 343 return TC_H_MAJ(first); 344 } 345 346 static bool tcf_proto_check_kind(struct nlattr *kind, char *name) 347 { 348 if (kind) 349 return nla_strscpy(name, kind, IFNAMSIZ) < 0; 350 memset(name, 0, IFNAMSIZ); 351 return false; 352 } 353 354 static bool tcf_proto_is_unlocked(const char *kind) 355 { 356 const struct tcf_proto_ops *ops; 357 bool ret; 358 359 if (strlen(kind) == 0) 360 return false; 361 362 ops = tcf_proto_lookup_ops(kind, false, NULL); 363 /* On error return false to take rtnl lock. Proto lookup/create 364 * functions will perform lookup again and properly handle errors. 365 */ 366 if (IS_ERR(ops)) 367 return false; 368 369 ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED); 370 module_put(ops->owner); 371 return ret; 372 } 373 374 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol, 375 u32 prio, struct tcf_chain *chain, 376 bool rtnl_held, 377 struct netlink_ext_ack *extack) 378 { 379 struct tcf_proto *tp; 380 int err; 381 382 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 383 if (!tp) 384 return ERR_PTR(-ENOBUFS); 385 386 tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack); 387 if (IS_ERR(tp->ops)) { 388 err = PTR_ERR(tp->ops); 389 goto errout; 390 } 391 tp->classify = tp->ops->classify; 392 tp->protocol = protocol; 393 tp->prio = prio; 394 tp->chain = chain; 395 spin_lock_init(&tp->lock); 396 refcount_set(&tp->refcnt, 1); 397 398 err = tp->ops->init(tp); 399 if (err) { 400 module_put(tp->ops->owner); 401 goto errout; 402 } 403 return tp; 404 405 errout: 406 kfree(tp); 407 return ERR_PTR(err); 408 } 409 410 static void tcf_proto_get(struct tcf_proto *tp) 411 { 412 refcount_inc(&tp->refcnt); 413 } 414 415 static void tcf_chain_put(struct tcf_chain *chain); 416 417 static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held, 418 bool sig_destroy, struct netlink_ext_ack *extack) 419 { 420 tp->ops->destroy(tp, rtnl_held, extack); 421 if (sig_destroy) 422 tcf_proto_signal_destroyed(tp->chain, tp); 423 tcf_chain_put(tp->chain); 424 module_put(tp->ops->owner); 425 kfree_rcu(tp, rcu); 426 } 427 428 static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held, 429 struct netlink_ext_ack *extack) 430 { 431 if (refcount_dec_and_test(&tp->refcnt)) 432 tcf_proto_destroy(tp, rtnl_held, true, extack); 433 } 434 435 static bool tcf_proto_check_delete(struct tcf_proto *tp) 436 { 437 if (tp->ops->delete_empty) 438 return tp->ops->delete_empty(tp); 439 440 tp->deleting = true; 441 return tp->deleting; 442 } 443 444 static void tcf_proto_mark_delete(struct tcf_proto *tp) 445 { 446 spin_lock(&tp->lock); 447 tp->deleting = true; 448 spin_unlock(&tp->lock); 449 } 450 451 static bool tcf_proto_is_deleting(struct tcf_proto *tp) 452 { 453 bool deleting; 454 455 spin_lock(&tp->lock); 456 deleting = tp->deleting; 457 spin_unlock(&tp->lock); 458 459 return deleting; 460 } 461 462 #define ASSERT_BLOCK_LOCKED(block) \ 463 lockdep_assert_held(&(block)->lock) 464 465 struct tcf_filter_chain_list_item { 466 struct list_head list; 467 tcf_chain_head_change_t *chain_head_change; 468 void *chain_head_change_priv; 469 }; 470 471 static struct tcf_chain *tcf_chain_create(struct tcf_block *block, 472 u32 chain_index) 473 { 474 struct tcf_chain *chain; 475 476 ASSERT_BLOCK_LOCKED(block); 477 478 chain = kzalloc(sizeof(*chain), GFP_KERNEL); 479 if (!chain) 480 return NULL; 481 list_add_tail_rcu(&chain->list, &block->chain_list); 482 mutex_init(&chain->filter_chain_lock); 483 chain->block = block; 484 chain->index = chain_index; 485 chain->refcnt = 1; 486 if (!chain->index) 487 block->chain0.chain = chain; 488 return chain; 489 } 490 491 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item, 492 struct tcf_proto *tp_head) 493 { 494 if (item->chain_head_change) 495 item->chain_head_change(tp_head, item->chain_head_change_priv); 496 } 497 498 static void tcf_chain0_head_change(struct tcf_chain *chain, 499 struct tcf_proto *tp_head) 500 { 501 struct tcf_filter_chain_list_item *item; 502 struct tcf_block *block = chain->block; 503 504 if (chain->index) 505 return; 506 507 mutex_lock(&block->lock); 508 list_for_each_entry(item, &block->chain0.filter_chain_list, list) 509 tcf_chain_head_change_item(item, tp_head); 510 mutex_unlock(&block->lock); 511 } 512 513 /* Returns true if block can be safely freed. */ 514 515 static bool tcf_chain_detach(struct tcf_chain *chain) 516 { 517 struct tcf_block *block = chain->block; 518 519 ASSERT_BLOCK_LOCKED(block); 520 521 list_del_rcu(&chain->list); 522 if (!chain->index) 523 block->chain0.chain = NULL; 524 525 if (list_empty(&block->chain_list) && 526 refcount_read(&block->refcnt) == 0) 527 return true; 528 529 return false; 530 } 531 532 static void tcf_block_destroy(struct tcf_block *block) 533 { 534 mutex_destroy(&block->lock); 535 mutex_destroy(&block->proto_destroy_lock); 536 kfree_rcu(block, rcu); 537 } 538 539 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block) 540 { 541 struct tcf_block *block = chain->block; 542 543 mutex_destroy(&chain->filter_chain_lock); 544 kfree_rcu(chain, rcu); 545 if (free_block) 546 tcf_block_destroy(block); 547 } 548 549 static void tcf_chain_hold(struct tcf_chain *chain) 550 { 551 ASSERT_BLOCK_LOCKED(chain->block); 552 553 ++chain->refcnt; 554 } 555 556 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain) 557 { 558 ASSERT_BLOCK_LOCKED(chain->block); 559 560 /* In case all the references are action references, this 561 * chain should not be shown to the user. 562 */ 563 return chain->refcnt == chain->action_refcnt; 564 } 565 566 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block, 567 u32 chain_index) 568 { 569 struct tcf_chain *chain; 570 571 ASSERT_BLOCK_LOCKED(block); 572 573 list_for_each_entry(chain, &block->chain_list, list) { 574 if (chain->index == chain_index) 575 return chain; 576 } 577 return NULL; 578 } 579 580 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 581 static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block, 582 u32 chain_index) 583 { 584 struct tcf_chain *chain; 585 586 list_for_each_entry_rcu(chain, &block->chain_list, list) { 587 if (chain->index == chain_index) 588 return chain; 589 } 590 return NULL; 591 } 592 #endif 593 594 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, 595 u32 seq, u16 flags, int event, bool unicast, 596 struct netlink_ext_ack *extack); 597 598 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block, 599 u32 chain_index, bool create, 600 bool by_act) 601 { 602 struct tcf_chain *chain = NULL; 603 bool is_first_reference; 604 605 mutex_lock(&block->lock); 606 chain = tcf_chain_lookup(block, chain_index); 607 if (chain) { 608 tcf_chain_hold(chain); 609 } else { 610 if (!create) 611 goto errout; 612 chain = tcf_chain_create(block, chain_index); 613 if (!chain) 614 goto errout; 615 } 616 617 if (by_act) 618 ++chain->action_refcnt; 619 is_first_reference = chain->refcnt - chain->action_refcnt == 1; 620 mutex_unlock(&block->lock); 621 622 /* Send notification only in case we got the first 623 * non-action reference. Until then, the chain acts only as 624 * a placeholder for actions pointing to it and user ought 625 * not know about them. 626 */ 627 if (is_first_reference && !by_act) 628 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL, 629 RTM_NEWCHAIN, false, NULL); 630 631 return chain; 632 633 errout: 634 mutex_unlock(&block->lock); 635 return chain; 636 } 637 638 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index, 639 bool create) 640 { 641 return __tcf_chain_get(block, chain_index, create, false); 642 } 643 644 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index) 645 { 646 return __tcf_chain_get(block, chain_index, true, true); 647 } 648 EXPORT_SYMBOL(tcf_chain_get_by_act); 649 650 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops, 651 void *tmplt_priv); 652 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, 653 void *tmplt_priv, u32 chain_index, 654 struct tcf_block *block, struct sk_buff *oskb, 655 u32 seq, u16 flags, bool unicast); 656 657 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act, 658 bool explicitly_created) 659 { 660 struct tcf_block *block = chain->block; 661 const struct tcf_proto_ops *tmplt_ops; 662 bool free_block = false; 663 unsigned int refcnt; 664 void *tmplt_priv; 665 666 mutex_lock(&block->lock); 667 if (explicitly_created) { 668 if (!chain->explicitly_created) { 669 mutex_unlock(&block->lock); 670 return; 671 } 672 chain->explicitly_created = false; 673 } 674 675 if (by_act) 676 chain->action_refcnt--; 677 678 /* tc_chain_notify_delete can't be called while holding block lock. 679 * However, when block is unlocked chain can be changed concurrently, so 680 * save these to temporary variables. 681 */ 682 refcnt = --chain->refcnt; 683 tmplt_ops = chain->tmplt_ops; 684 tmplt_priv = chain->tmplt_priv; 685 686 /* The last dropped non-action reference will trigger notification. */ 687 if (refcnt - chain->action_refcnt == 0 && !by_act) { 688 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index, 689 block, NULL, 0, 0, false); 690 /* Last reference to chain, no need to lock. */ 691 chain->flushing = false; 692 } 693 694 if (refcnt == 0) 695 free_block = tcf_chain_detach(chain); 696 mutex_unlock(&block->lock); 697 698 if (refcnt == 0) { 699 tc_chain_tmplt_del(tmplt_ops, tmplt_priv); 700 tcf_chain_destroy(chain, free_block); 701 } 702 } 703 704 static void tcf_chain_put(struct tcf_chain *chain) 705 { 706 __tcf_chain_put(chain, false, false); 707 } 708 709 void tcf_chain_put_by_act(struct tcf_chain *chain) 710 { 711 __tcf_chain_put(chain, true, false); 712 } 713 EXPORT_SYMBOL(tcf_chain_put_by_act); 714 715 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain) 716 { 717 __tcf_chain_put(chain, false, true); 718 } 719 720 static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held) 721 { 722 struct tcf_proto *tp, *tp_next; 723 724 mutex_lock(&chain->filter_chain_lock); 725 tp = tcf_chain_dereference(chain->filter_chain, chain); 726 while (tp) { 727 tp_next = rcu_dereference_protected(tp->next, 1); 728 tcf_proto_signal_destroying(chain, tp); 729 tp = tp_next; 730 } 731 tp = tcf_chain_dereference(chain->filter_chain, chain); 732 RCU_INIT_POINTER(chain->filter_chain, NULL); 733 tcf_chain0_head_change(chain, NULL); 734 chain->flushing = true; 735 mutex_unlock(&chain->filter_chain_lock); 736 737 while (tp) { 738 tp_next = rcu_dereference_protected(tp->next, 1); 739 tcf_proto_put(tp, rtnl_held, NULL); 740 tp = tp_next; 741 } 742 } 743 744 static int tcf_block_setup(struct tcf_block *block, 745 struct flow_block_offload *bo); 746 747 static void tcf_block_offload_init(struct flow_block_offload *bo, 748 struct net_device *dev, struct Qdisc *sch, 749 enum flow_block_command command, 750 enum flow_block_binder_type binder_type, 751 struct flow_block *flow_block, 752 bool shared, struct netlink_ext_ack *extack) 753 { 754 bo->net = dev_net(dev); 755 bo->command = command; 756 bo->binder_type = binder_type; 757 bo->block = flow_block; 758 bo->block_shared = shared; 759 bo->extack = extack; 760 bo->sch = sch; 761 bo->cb_list_head = &flow_block->cb_list; 762 INIT_LIST_HEAD(&bo->cb_list); 763 } 764 765 static void tcf_block_unbind(struct tcf_block *block, 766 struct flow_block_offload *bo); 767 768 static void tc_block_indr_cleanup(struct flow_block_cb *block_cb) 769 { 770 struct tcf_block *block = block_cb->indr.data; 771 struct net_device *dev = block_cb->indr.dev; 772 struct Qdisc *sch = block_cb->indr.sch; 773 struct netlink_ext_ack extack = {}; 774 struct flow_block_offload bo = {}; 775 776 tcf_block_offload_init(&bo, dev, sch, FLOW_BLOCK_UNBIND, 777 block_cb->indr.binder_type, 778 &block->flow_block, tcf_block_shared(block), 779 &extack); 780 rtnl_lock(); 781 down_write(&block->cb_lock); 782 list_del(&block_cb->driver_list); 783 list_move(&block_cb->list, &bo.cb_list); 784 tcf_block_unbind(block, &bo); 785 up_write(&block->cb_lock); 786 rtnl_unlock(); 787 } 788 789 static bool tcf_block_offload_in_use(struct tcf_block *block) 790 { 791 return atomic_read(&block->offloadcnt); 792 } 793 794 static int tcf_block_offload_cmd(struct tcf_block *block, 795 struct net_device *dev, struct Qdisc *sch, 796 struct tcf_block_ext_info *ei, 797 enum flow_block_command command, 798 struct netlink_ext_ack *extack) 799 { 800 struct flow_block_offload bo = {}; 801 802 tcf_block_offload_init(&bo, dev, sch, command, ei->binder_type, 803 &block->flow_block, tcf_block_shared(block), 804 extack); 805 806 if (dev->netdev_ops->ndo_setup_tc) { 807 int err; 808 809 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo); 810 if (err < 0) { 811 if (err != -EOPNOTSUPP) 812 NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed"); 813 return err; 814 } 815 816 return tcf_block_setup(block, &bo); 817 } 818 819 flow_indr_dev_setup_offload(dev, sch, TC_SETUP_BLOCK, block, &bo, 820 tc_block_indr_cleanup); 821 tcf_block_setup(block, &bo); 822 823 return -EOPNOTSUPP; 824 } 825 826 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q, 827 struct tcf_block_ext_info *ei, 828 struct netlink_ext_ack *extack) 829 { 830 struct net_device *dev = q->dev_queue->dev; 831 int err; 832 833 down_write(&block->cb_lock); 834 835 /* If tc offload feature is disabled and the block we try to bind 836 * to already has some offloaded filters, forbid to bind. 837 */ 838 if (dev->netdev_ops->ndo_setup_tc && 839 !tc_can_offload(dev) && 840 tcf_block_offload_in_use(block)) { 841 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled"); 842 err = -EOPNOTSUPP; 843 goto err_unlock; 844 } 845 846 err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_BIND, extack); 847 if (err == -EOPNOTSUPP) 848 goto no_offload_dev_inc; 849 if (err) 850 goto err_unlock; 851 852 up_write(&block->cb_lock); 853 return 0; 854 855 no_offload_dev_inc: 856 if (tcf_block_offload_in_use(block)) 857 goto err_unlock; 858 859 err = 0; 860 block->nooffloaddevcnt++; 861 err_unlock: 862 up_write(&block->cb_lock); 863 return err; 864 } 865 866 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q, 867 struct tcf_block_ext_info *ei) 868 { 869 struct net_device *dev = q->dev_queue->dev; 870 int err; 871 872 down_write(&block->cb_lock); 873 err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_UNBIND, NULL); 874 if (err == -EOPNOTSUPP) 875 goto no_offload_dev_dec; 876 up_write(&block->cb_lock); 877 return; 878 879 no_offload_dev_dec: 880 WARN_ON(block->nooffloaddevcnt-- == 0); 881 up_write(&block->cb_lock); 882 } 883 884 static int 885 tcf_chain0_head_change_cb_add(struct tcf_block *block, 886 struct tcf_block_ext_info *ei, 887 struct netlink_ext_ack *extack) 888 { 889 struct tcf_filter_chain_list_item *item; 890 struct tcf_chain *chain0; 891 892 item = kmalloc(sizeof(*item), GFP_KERNEL); 893 if (!item) { 894 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed"); 895 return -ENOMEM; 896 } 897 item->chain_head_change = ei->chain_head_change; 898 item->chain_head_change_priv = ei->chain_head_change_priv; 899 900 mutex_lock(&block->lock); 901 chain0 = block->chain0.chain; 902 if (chain0) 903 tcf_chain_hold(chain0); 904 else 905 list_add(&item->list, &block->chain0.filter_chain_list); 906 mutex_unlock(&block->lock); 907 908 if (chain0) { 909 struct tcf_proto *tp_head; 910 911 mutex_lock(&chain0->filter_chain_lock); 912 913 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0); 914 if (tp_head) 915 tcf_chain_head_change_item(item, tp_head); 916 917 mutex_lock(&block->lock); 918 list_add(&item->list, &block->chain0.filter_chain_list); 919 mutex_unlock(&block->lock); 920 921 mutex_unlock(&chain0->filter_chain_lock); 922 tcf_chain_put(chain0); 923 } 924 925 return 0; 926 } 927 928 static void 929 tcf_chain0_head_change_cb_del(struct tcf_block *block, 930 struct tcf_block_ext_info *ei) 931 { 932 struct tcf_filter_chain_list_item *item; 933 934 mutex_lock(&block->lock); 935 list_for_each_entry(item, &block->chain0.filter_chain_list, list) { 936 if ((!ei->chain_head_change && !ei->chain_head_change_priv) || 937 (item->chain_head_change == ei->chain_head_change && 938 item->chain_head_change_priv == ei->chain_head_change_priv)) { 939 if (block->chain0.chain) 940 tcf_chain_head_change_item(item, NULL); 941 list_del(&item->list); 942 mutex_unlock(&block->lock); 943 944 kfree(item); 945 return; 946 } 947 } 948 mutex_unlock(&block->lock); 949 WARN_ON(1); 950 } 951 952 struct tcf_net { 953 spinlock_t idr_lock; /* Protects idr */ 954 struct idr idr; 955 }; 956 957 static unsigned int tcf_net_id; 958 959 static int tcf_block_insert(struct tcf_block *block, struct net *net, 960 struct netlink_ext_ack *extack) 961 { 962 struct tcf_net *tn = net_generic(net, tcf_net_id); 963 int err; 964 965 idr_preload(GFP_KERNEL); 966 spin_lock(&tn->idr_lock); 967 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index, 968 GFP_NOWAIT); 969 spin_unlock(&tn->idr_lock); 970 idr_preload_end(); 971 972 return err; 973 } 974 975 static void tcf_block_remove(struct tcf_block *block, struct net *net) 976 { 977 struct tcf_net *tn = net_generic(net, tcf_net_id); 978 979 spin_lock(&tn->idr_lock); 980 idr_remove(&tn->idr, block->index); 981 spin_unlock(&tn->idr_lock); 982 } 983 984 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q, 985 u32 block_index, 986 struct netlink_ext_ack *extack) 987 { 988 struct tcf_block *block; 989 990 block = kzalloc(sizeof(*block), GFP_KERNEL); 991 if (!block) { 992 NL_SET_ERR_MSG(extack, "Memory allocation for block failed"); 993 return ERR_PTR(-ENOMEM); 994 } 995 mutex_init(&block->lock); 996 mutex_init(&block->proto_destroy_lock); 997 init_rwsem(&block->cb_lock); 998 flow_block_init(&block->flow_block); 999 INIT_LIST_HEAD(&block->chain_list); 1000 INIT_LIST_HEAD(&block->owner_list); 1001 INIT_LIST_HEAD(&block->chain0.filter_chain_list); 1002 1003 refcount_set(&block->refcnt, 1); 1004 block->net = net; 1005 block->index = block_index; 1006 1007 /* Don't store q pointer for blocks which are shared */ 1008 if (!tcf_block_shared(block)) 1009 block->q = q; 1010 return block; 1011 } 1012 1013 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index) 1014 { 1015 struct tcf_net *tn = net_generic(net, tcf_net_id); 1016 1017 return idr_find(&tn->idr, block_index); 1018 } 1019 1020 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index) 1021 { 1022 struct tcf_block *block; 1023 1024 rcu_read_lock(); 1025 block = tcf_block_lookup(net, block_index); 1026 if (block && !refcount_inc_not_zero(&block->refcnt)) 1027 block = NULL; 1028 rcu_read_unlock(); 1029 1030 return block; 1031 } 1032 1033 static struct tcf_chain * 1034 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain) 1035 { 1036 mutex_lock(&block->lock); 1037 if (chain) 1038 chain = list_is_last(&chain->list, &block->chain_list) ? 1039 NULL : list_next_entry(chain, list); 1040 else 1041 chain = list_first_entry_or_null(&block->chain_list, 1042 struct tcf_chain, list); 1043 1044 /* skip all action-only chains */ 1045 while (chain && tcf_chain_held_by_acts_only(chain)) 1046 chain = list_is_last(&chain->list, &block->chain_list) ? 1047 NULL : list_next_entry(chain, list); 1048 1049 if (chain) 1050 tcf_chain_hold(chain); 1051 mutex_unlock(&block->lock); 1052 1053 return chain; 1054 } 1055 1056 /* Function to be used by all clients that want to iterate over all chains on 1057 * block. It properly obtains block->lock and takes reference to chain before 1058 * returning it. Users of this function must be tolerant to concurrent chain 1059 * insertion/deletion or ensure that no concurrent chain modification is 1060 * possible. Note that all netlink dump callbacks cannot guarantee to provide 1061 * consistent dump because rtnl lock is released each time skb is filled with 1062 * data and sent to user-space. 1063 */ 1064 1065 struct tcf_chain * 1066 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain) 1067 { 1068 struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain); 1069 1070 if (chain) 1071 tcf_chain_put(chain); 1072 1073 return chain_next; 1074 } 1075 EXPORT_SYMBOL(tcf_get_next_chain); 1076 1077 static struct tcf_proto * 1078 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp) 1079 { 1080 u32 prio = 0; 1081 1082 ASSERT_RTNL(); 1083 mutex_lock(&chain->filter_chain_lock); 1084 1085 if (!tp) { 1086 tp = tcf_chain_dereference(chain->filter_chain, chain); 1087 } else if (tcf_proto_is_deleting(tp)) { 1088 /* 'deleting' flag is set and chain->filter_chain_lock was 1089 * unlocked, which means next pointer could be invalid. Restart 1090 * search. 1091 */ 1092 prio = tp->prio + 1; 1093 tp = tcf_chain_dereference(chain->filter_chain, chain); 1094 1095 for (; tp; tp = tcf_chain_dereference(tp->next, chain)) 1096 if (!tp->deleting && tp->prio >= prio) 1097 break; 1098 } else { 1099 tp = tcf_chain_dereference(tp->next, chain); 1100 } 1101 1102 if (tp) 1103 tcf_proto_get(tp); 1104 1105 mutex_unlock(&chain->filter_chain_lock); 1106 1107 return tp; 1108 } 1109 1110 /* Function to be used by all clients that want to iterate over all tp's on 1111 * chain. Users of this function must be tolerant to concurrent tp 1112 * insertion/deletion or ensure that no concurrent chain modification is 1113 * possible. Note that all netlink dump callbacks cannot guarantee to provide 1114 * consistent dump because rtnl lock is released each time skb is filled with 1115 * data and sent to user-space. 1116 */ 1117 1118 struct tcf_proto * 1119 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp) 1120 { 1121 struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp); 1122 1123 if (tp) 1124 tcf_proto_put(tp, true, NULL); 1125 1126 return tp_next; 1127 } 1128 EXPORT_SYMBOL(tcf_get_next_proto); 1129 1130 static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held) 1131 { 1132 struct tcf_chain *chain; 1133 1134 /* Last reference to block. At this point chains cannot be added or 1135 * removed concurrently. 1136 */ 1137 for (chain = tcf_get_next_chain(block, NULL); 1138 chain; 1139 chain = tcf_get_next_chain(block, chain)) { 1140 tcf_chain_put_explicitly_created(chain); 1141 tcf_chain_flush(chain, rtnl_held); 1142 } 1143 } 1144 1145 /* Lookup Qdisc and increments its reference counter. 1146 * Set parent, if necessary. 1147 */ 1148 1149 static int __tcf_qdisc_find(struct net *net, struct Qdisc **q, 1150 u32 *parent, int ifindex, bool rtnl_held, 1151 struct netlink_ext_ack *extack) 1152 { 1153 const struct Qdisc_class_ops *cops; 1154 struct net_device *dev; 1155 int err = 0; 1156 1157 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) 1158 return 0; 1159 1160 rcu_read_lock(); 1161 1162 /* Find link */ 1163 dev = dev_get_by_index_rcu(net, ifindex); 1164 if (!dev) { 1165 rcu_read_unlock(); 1166 return -ENODEV; 1167 } 1168 1169 /* Find qdisc */ 1170 if (!*parent) { 1171 *q = rcu_dereference(dev->qdisc); 1172 *parent = (*q)->handle; 1173 } else { 1174 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent)); 1175 if (!*q) { 1176 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists"); 1177 err = -EINVAL; 1178 goto errout_rcu; 1179 } 1180 } 1181 1182 *q = qdisc_refcount_inc_nz(*q); 1183 if (!*q) { 1184 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists"); 1185 err = -EINVAL; 1186 goto errout_rcu; 1187 } 1188 1189 /* Is it classful? */ 1190 cops = (*q)->ops->cl_ops; 1191 if (!cops) { 1192 NL_SET_ERR_MSG(extack, "Qdisc not classful"); 1193 err = -EINVAL; 1194 goto errout_qdisc; 1195 } 1196 1197 if (!cops->tcf_block) { 1198 NL_SET_ERR_MSG(extack, "Class doesn't support blocks"); 1199 err = -EOPNOTSUPP; 1200 goto errout_qdisc; 1201 } 1202 1203 errout_rcu: 1204 /* At this point we know that qdisc is not noop_qdisc, 1205 * which means that qdisc holds a reference to net_device 1206 * and we hold a reference to qdisc, so it is safe to release 1207 * rcu read lock. 1208 */ 1209 rcu_read_unlock(); 1210 return err; 1211 1212 errout_qdisc: 1213 rcu_read_unlock(); 1214 1215 if (rtnl_held) 1216 qdisc_put(*q); 1217 else 1218 qdisc_put_unlocked(*q); 1219 *q = NULL; 1220 1221 return err; 1222 } 1223 1224 static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl, 1225 int ifindex, struct netlink_ext_ack *extack) 1226 { 1227 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) 1228 return 0; 1229 1230 /* Do we search for filter, attached to class? */ 1231 if (TC_H_MIN(parent)) { 1232 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1233 1234 *cl = cops->find(q, parent); 1235 if (*cl == 0) { 1236 NL_SET_ERR_MSG(extack, "Specified class doesn't exist"); 1237 return -ENOENT; 1238 } 1239 } 1240 1241 return 0; 1242 } 1243 1244 static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q, 1245 unsigned long cl, int ifindex, 1246 u32 block_index, 1247 struct netlink_ext_ack *extack) 1248 { 1249 struct tcf_block *block; 1250 1251 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 1252 block = tcf_block_refcnt_get(net, block_index); 1253 if (!block) { 1254 NL_SET_ERR_MSG(extack, "Block of given index was not found"); 1255 return ERR_PTR(-EINVAL); 1256 } 1257 } else { 1258 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1259 1260 block = cops->tcf_block(q, cl, extack); 1261 if (!block) 1262 return ERR_PTR(-EINVAL); 1263 1264 if (tcf_block_shared(block)) { 1265 NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters"); 1266 return ERR_PTR(-EOPNOTSUPP); 1267 } 1268 1269 /* Always take reference to block in order to support execution 1270 * of rules update path of cls API without rtnl lock. Caller 1271 * must release block when it is finished using it. 'if' block 1272 * of this conditional obtain reference to block by calling 1273 * tcf_block_refcnt_get(). 1274 */ 1275 refcount_inc(&block->refcnt); 1276 } 1277 1278 return block; 1279 } 1280 1281 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q, 1282 struct tcf_block_ext_info *ei, bool rtnl_held) 1283 { 1284 if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) { 1285 /* Flushing/putting all chains will cause the block to be 1286 * deallocated when last chain is freed. However, if chain_list 1287 * is empty, block has to be manually deallocated. After block 1288 * reference counter reached 0, it is no longer possible to 1289 * increment it or add new chains to block. 1290 */ 1291 bool free_block = list_empty(&block->chain_list); 1292 1293 mutex_unlock(&block->lock); 1294 if (tcf_block_shared(block)) 1295 tcf_block_remove(block, block->net); 1296 1297 if (q) 1298 tcf_block_offload_unbind(block, q, ei); 1299 1300 if (free_block) 1301 tcf_block_destroy(block); 1302 else 1303 tcf_block_flush_all_chains(block, rtnl_held); 1304 } else if (q) { 1305 tcf_block_offload_unbind(block, q, ei); 1306 } 1307 } 1308 1309 static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held) 1310 { 1311 __tcf_block_put(block, NULL, NULL, rtnl_held); 1312 } 1313 1314 /* Find tcf block. 1315 * Set q, parent, cl when appropriate. 1316 */ 1317 1318 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q, 1319 u32 *parent, unsigned long *cl, 1320 int ifindex, u32 block_index, 1321 struct netlink_ext_ack *extack) 1322 { 1323 struct tcf_block *block; 1324 int err = 0; 1325 1326 ASSERT_RTNL(); 1327 1328 err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack); 1329 if (err) 1330 goto errout; 1331 1332 err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack); 1333 if (err) 1334 goto errout_qdisc; 1335 1336 block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack); 1337 if (IS_ERR(block)) { 1338 err = PTR_ERR(block); 1339 goto errout_qdisc; 1340 } 1341 1342 return block; 1343 1344 errout_qdisc: 1345 if (*q) 1346 qdisc_put(*q); 1347 errout: 1348 *q = NULL; 1349 return ERR_PTR(err); 1350 } 1351 1352 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block, 1353 bool rtnl_held) 1354 { 1355 if (!IS_ERR_OR_NULL(block)) 1356 tcf_block_refcnt_put(block, rtnl_held); 1357 1358 if (q) { 1359 if (rtnl_held) 1360 qdisc_put(q); 1361 else 1362 qdisc_put_unlocked(q); 1363 } 1364 } 1365 1366 struct tcf_block_owner_item { 1367 struct list_head list; 1368 struct Qdisc *q; 1369 enum flow_block_binder_type binder_type; 1370 }; 1371 1372 static void 1373 tcf_block_owner_netif_keep_dst(struct tcf_block *block, 1374 struct Qdisc *q, 1375 enum flow_block_binder_type binder_type) 1376 { 1377 if (block->keep_dst && 1378 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS && 1379 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) 1380 netif_keep_dst(qdisc_dev(q)); 1381 } 1382 1383 void tcf_block_netif_keep_dst(struct tcf_block *block) 1384 { 1385 struct tcf_block_owner_item *item; 1386 1387 block->keep_dst = true; 1388 list_for_each_entry(item, &block->owner_list, list) 1389 tcf_block_owner_netif_keep_dst(block, item->q, 1390 item->binder_type); 1391 } 1392 EXPORT_SYMBOL(tcf_block_netif_keep_dst); 1393 1394 static int tcf_block_owner_add(struct tcf_block *block, 1395 struct Qdisc *q, 1396 enum flow_block_binder_type binder_type) 1397 { 1398 struct tcf_block_owner_item *item; 1399 1400 item = kmalloc(sizeof(*item), GFP_KERNEL); 1401 if (!item) 1402 return -ENOMEM; 1403 item->q = q; 1404 item->binder_type = binder_type; 1405 list_add(&item->list, &block->owner_list); 1406 return 0; 1407 } 1408 1409 static void tcf_block_owner_del(struct tcf_block *block, 1410 struct Qdisc *q, 1411 enum flow_block_binder_type binder_type) 1412 { 1413 struct tcf_block_owner_item *item; 1414 1415 list_for_each_entry(item, &block->owner_list, list) { 1416 if (item->q == q && item->binder_type == binder_type) { 1417 list_del(&item->list); 1418 kfree(item); 1419 return; 1420 } 1421 } 1422 WARN_ON(1); 1423 } 1424 1425 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q, 1426 struct tcf_block_ext_info *ei, 1427 struct netlink_ext_ack *extack) 1428 { 1429 struct net *net = qdisc_net(q); 1430 struct tcf_block *block = NULL; 1431 int err; 1432 1433 if (ei->block_index) 1434 /* block_index not 0 means the shared block is requested */ 1435 block = tcf_block_refcnt_get(net, ei->block_index); 1436 1437 if (!block) { 1438 block = tcf_block_create(net, q, ei->block_index, extack); 1439 if (IS_ERR(block)) 1440 return PTR_ERR(block); 1441 if (tcf_block_shared(block)) { 1442 err = tcf_block_insert(block, net, extack); 1443 if (err) 1444 goto err_block_insert; 1445 } 1446 } 1447 1448 err = tcf_block_owner_add(block, q, ei->binder_type); 1449 if (err) 1450 goto err_block_owner_add; 1451 1452 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type); 1453 1454 err = tcf_chain0_head_change_cb_add(block, ei, extack); 1455 if (err) 1456 goto err_chain0_head_change_cb_add; 1457 1458 err = tcf_block_offload_bind(block, q, ei, extack); 1459 if (err) 1460 goto err_block_offload_bind; 1461 1462 *p_block = block; 1463 return 0; 1464 1465 err_block_offload_bind: 1466 tcf_chain0_head_change_cb_del(block, ei); 1467 err_chain0_head_change_cb_add: 1468 tcf_block_owner_del(block, q, ei->binder_type); 1469 err_block_owner_add: 1470 err_block_insert: 1471 tcf_block_refcnt_put(block, true); 1472 return err; 1473 } 1474 EXPORT_SYMBOL(tcf_block_get_ext); 1475 1476 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv) 1477 { 1478 struct tcf_proto __rcu **p_filter_chain = priv; 1479 1480 rcu_assign_pointer(*p_filter_chain, tp_head); 1481 } 1482 1483 int tcf_block_get(struct tcf_block **p_block, 1484 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q, 1485 struct netlink_ext_ack *extack) 1486 { 1487 struct tcf_block_ext_info ei = { 1488 .chain_head_change = tcf_chain_head_change_dflt, 1489 .chain_head_change_priv = p_filter_chain, 1490 }; 1491 1492 WARN_ON(!p_filter_chain); 1493 return tcf_block_get_ext(p_block, q, &ei, extack); 1494 } 1495 EXPORT_SYMBOL(tcf_block_get); 1496 1497 /* XXX: Standalone actions are not allowed to jump to any chain, and bound 1498 * actions should be all removed after flushing. 1499 */ 1500 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q, 1501 struct tcf_block_ext_info *ei) 1502 { 1503 if (!block) 1504 return; 1505 tcf_chain0_head_change_cb_del(block, ei); 1506 tcf_block_owner_del(block, q, ei->binder_type); 1507 1508 __tcf_block_put(block, q, ei, true); 1509 } 1510 EXPORT_SYMBOL(tcf_block_put_ext); 1511 1512 void tcf_block_put(struct tcf_block *block) 1513 { 1514 struct tcf_block_ext_info ei = {0, }; 1515 1516 if (!block) 1517 return; 1518 tcf_block_put_ext(block, block->q, &ei); 1519 } 1520 1521 EXPORT_SYMBOL(tcf_block_put); 1522 1523 static int 1524 tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb, 1525 void *cb_priv, bool add, bool offload_in_use, 1526 struct netlink_ext_ack *extack) 1527 { 1528 struct tcf_chain *chain, *chain_prev; 1529 struct tcf_proto *tp, *tp_prev; 1530 int err; 1531 1532 lockdep_assert_held(&block->cb_lock); 1533 1534 for (chain = __tcf_get_next_chain(block, NULL); 1535 chain; 1536 chain_prev = chain, 1537 chain = __tcf_get_next_chain(block, chain), 1538 tcf_chain_put(chain_prev)) { 1539 for (tp = __tcf_get_next_proto(chain, NULL); tp; 1540 tp_prev = tp, 1541 tp = __tcf_get_next_proto(chain, tp), 1542 tcf_proto_put(tp_prev, true, NULL)) { 1543 if (tp->ops->reoffload) { 1544 err = tp->ops->reoffload(tp, add, cb, cb_priv, 1545 extack); 1546 if (err && add) 1547 goto err_playback_remove; 1548 } else if (add && offload_in_use) { 1549 err = -EOPNOTSUPP; 1550 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support"); 1551 goto err_playback_remove; 1552 } 1553 } 1554 } 1555 1556 return 0; 1557 1558 err_playback_remove: 1559 tcf_proto_put(tp, true, NULL); 1560 tcf_chain_put(chain); 1561 tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use, 1562 extack); 1563 return err; 1564 } 1565 1566 static int tcf_block_bind(struct tcf_block *block, 1567 struct flow_block_offload *bo) 1568 { 1569 struct flow_block_cb *block_cb, *next; 1570 int err, i = 0; 1571 1572 lockdep_assert_held(&block->cb_lock); 1573 1574 list_for_each_entry(block_cb, &bo->cb_list, list) { 1575 err = tcf_block_playback_offloads(block, block_cb->cb, 1576 block_cb->cb_priv, true, 1577 tcf_block_offload_in_use(block), 1578 bo->extack); 1579 if (err) 1580 goto err_unroll; 1581 if (!bo->unlocked_driver_cb) 1582 block->lockeddevcnt++; 1583 1584 i++; 1585 } 1586 list_splice(&bo->cb_list, &block->flow_block.cb_list); 1587 1588 return 0; 1589 1590 err_unroll: 1591 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { 1592 list_del(&block_cb->driver_list); 1593 if (i-- > 0) { 1594 list_del(&block_cb->list); 1595 tcf_block_playback_offloads(block, block_cb->cb, 1596 block_cb->cb_priv, false, 1597 tcf_block_offload_in_use(block), 1598 NULL); 1599 if (!bo->unlocked_driver_cb) 1600 block->lockeddevcnt--; 1601 } 1602 flow_block_cb_free(block_cb); 1603 } 1604 1605 return err; 1606 } 1607 1608 static void tcf_block_unbind(struct tcf_block *block, 1609 struct flow_block_offload *bo) 1610 { 1611 struct flow_block_cb *block_cb, *next; 1612 1613 lockdep_assert_held(&block->cb_lock); 1614 1615 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { 1616 tcf_block_playback_offloads(block, block_cb->cb, 1617 block_cb->cb_priv, false, 1618 tcf_block_offload_in_use(block), 1619 NULL); 1620 list_del(&block_cb->list); 1621 flow_block_cb_free(block_cb); 1622 if (!bo->unlocked_driver_cb) 1623 block->lockeddevcnt--; 1624 } 1625 } 1626 1627 static int tcf_block_setup(struct tcf_block *block, 1628 struct flow_block_offload *bo) 1629 { 1630 int err; 1631 1632 switch (bo->command) { 1633 case FLOW_BLOCK_BIND: 1634 err = tcf_block_bind(block, bo); 1635 break; 1636 case FLOW_BLOCK_UNBIND: 1637 err = 0; 1638 tcf_block_unbind(block, bo); 1639 break; 1640 default: 1641 WARN_ON_ONCE(1); 1642 err = -EOPNOTSUPP; 1643 } 1644 1645 return err; 1646 } 1647 1648 /* Main classifier routine: scans classifier chain attached 1649 * to this qdisc, (optionally) tests for protocol and asks 1650 * specific classifiers. 1651 */ 1652 static inline int __tcf_classify(struct sk_buff *skb, 1653 const struct tcf_proto *tp, 1654 const struct tcf_proto *orig_tp, 1655 struct tcf_result *res, 1656 bool compat_mode, 1657 struct tcf_exts_miss_cookie_node *n, 1658 int act_index, 1659 u32 *last_executed_chain) 1660 { 1661 #ifdef CONFIG_NET_CLS_ACT 1662 const int max_reclassify_loop = 16; 1663 const struct tcf_proto *first_tp; 1664 int limit = 0; 1665 1666 reclassify: 1667 #endif 1668 for (; tp; tp = rcu_dereference_bh(tp->next)) { 1669 __be16 protocol = skb_protocol(skb, false); 1670 int err = 0; 1671 1672 if (n) { 1673 struct tcf_exts *exts; 1674 1675 if (n->tp_prio != tp->prio) 1676 continue; 1677 1678 /* We re-lookup the tp and chain based on index instead 1679 * of having hard refs and locks to them, so do a sanity 1680 * check if any of tp,chain,exts was replaced by the 1681 * time we got here with a cookie from hardware. 1682 */ 1683 if (unlikely(n->tp != tp || n->tp->chain != n->chain || 1684 !tp->ops->get_exts)) 1685 return TC_ACT_SHOT; 1686 1687 exts = tp->ops->get_exts(tp, n->handle); 1688 if (unlikely(!exts || n->exts != exts)) 1689 return TC_ACT_SHOT; 1690 1691 n = NULL; 1692 err = tcf_exts_exec_ex(skb, exts, act_index, res); 1693 } else { 1694 if (tp->protocol != protocol && 1695 tp->protocol != htons(ETH_P_ALL)) 1696 continue; 1697 1698 err = tc_classify(skb, tp, res); 1699 } 1700 #ifdef CONFIG_NET_CLS_ACT 1701 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) { 1702 first_tp = orig_tp; 1703 *last_executed_chain = first_tp->chain->index; 1704 goto reset; 1705 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) { 1706 first_tp = res->goto_tp; 1707 *last_executed_chain = err & TC_ACT_EXT_VAL_MASK; 1708 goto reset; 1709 } 1710 #endif 1711 if (err >= 0) 1712 return err; 1713 } 1714 1715 if (unlikely(n)) 1716 return TC_ACT_SHOT; 1717 1718 return TC_ACT_UNSPEC; /* signal: continue lookup */ 1719 #ifdef CONFIG_NET_CLS_ACT 1720 reset: 1721 if (unlikely(limit++ >= max_reclassify_loop)) { 1722 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n", 1723 tp->chain->block->index, 1724 tp->prio & 0xffff, 1725 ntohs(tp->protocol)); 1726 return TC_ACT_SHOT; 1727 } 1728 1729 tp = first_tp; 1730 goto reclassify; 1731 #endif 1732 } 1733 1734 int tcf_classify(struct sk_buff *skb, 1735 const struct tcf_block *block, 1736 const struct tcf_proto *tp, 1737 struct tcf_result *res, bool compat_mode) 1738 { 1739 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 1740 u32 last_executed_chain = 0; 1741 1742 return __tcf_classify(skb, tp, tp, res, compat_mode, NULL, 0, 1743 &last_executed_chain); 1744 #else 1745 u32 last_executed_chain = tp ? tp->chain->index : 0; 1746 struct tcf_exts_miss_cookie_node *n = NULL; 1747 const struct tcf_proto *orig_tp = tp; 1748 struct tc_skb_ext *ext; 1749 int act_index = 0; 1750 int ret; 1751 1752 if (block) { 1753 ext = skb_ext_find(skb, TC_SKB_EXT); 1754 1755 if (ext && (ext->chain || ext->act_miss)) { 1756 struct tcf_chain *fchain; 1757 u32 chain; 1758 1759 if (ext->act_miss) { 1760 n = tcf_exts_miss_cookie_lookup(ext->act_miss_cookie, 1761 &act_index); 1762 if (!n) 1763 return TC_ACT_SHOT; 1764 1765 chain = n->chain_index; 1766 } else { 1767 chain = ext->chain; 1768 } 1769 1770 fchain = tcf_chain_lookup_rcu(block, chain); 1771 if (!fchain) 1772 return TC_ACT_SHOT; 1773 1774 /* Consume, so cloned/redirect skbs won't inherit ext */ 1775 skb_ext_del(skb, TC_SKB_EXT); 1776 1777 tp = rcu_dereference_bh(fchain->filter_chain); 1778 last_executed_chain = fchain->index; 1779 } 1780 } 1781 1782 ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode, n, act_index, 1783 &last_executed_chain); 1784 1785 if (tc_skb_ext_tc_enabled()) { 1786 /* If we missed on some chain */ 1787 if (ret == TC_ACT_UNSPEC && last_executed_chain) { 1788 struct tc_skb_cb *cb = tc_skb_cb(skb); 1789 1790 ext = tc_skb_ext_alloc(skb); 1791 if (WARN_ON_ONCE(!ext)) 1792 return TC_ACT_SHOT; 1793 ext->chain = last_executed_chain; 1794 ext->mru = cb->mru; 1795 ext->post_ct = cb->post_ct; 1796 ext->post_ct_snat = cb->post_ct_snat; 1797 ext->post_ct_dnat = cb->post_ct_dnat; 1798 ext->zone = cb->zone; 1799 } 1800 } 1801 1802 return ret; 1803 #endif 1804 } 1805 EXPORT_SYMBOL(tcf_classify); 1806 1807 struct tcf_chain_info { 1808 struct tcf_proto __rcu **pprev; 1809 struct tcf_proto __rcu *next; 1810 }; 1811 1812 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain, 1813 struct tcf_chain_info *chain_info) 1814 { 1815 return tcf_chain_dereference(*chain_info->pprev, chain); 1816 } 1817 1818 static int tcf_chain_tp_insert(struct tcf_chain *chain, 1819 struct tcf_chain_info *chain_info, 1820 struct tcf_proto *tp) 1821 { 1822 if (chain->flushing) 1823 return -EAGAIN; 1824 1825 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info)); 1826 if (*chain_info->pprev == chain->filter_chain) 1827 tcf_chain0_head_change(chain, tp); 1828 tcf_proto_get(tp); 1829 rcu_assign_pointer(*chain_info->pprev, tp); 1830 1831 return 0; 1832 } 1833 1834 static void tcf_chain_tp_remove(struct tcf_chain *chain, 1835 struct tcf_chain_info *chain_info, 1836 struct tcf_proto *tp) 1837 { 1838 struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain); 1839 1840 tcf_proto_mark_delete(tp); 1841 if (tp == chain->filter_chain) 1842 tcf_chain0_head_change(chain, next); 1843 RCU_INIT_POINTER(*chain_info->pprev, next); 1844 } 1845 1846 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, 1847 struct tcf_chain_info *chain_info, 1848 u32 protocol, u32 prio, 1849 bool prio_allocate); 1850 1851 /* Try to insert new proto. 1852 * If proto with specified priority already exists, free new proto 1853 * and return existing one. 1854 */ 1855 1856 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain, 1857 struct tcf_proto *tp_new, 1858 u32 protocol, u32 prio, 1859 bool rtnl_held) 1860 { 1861 struct tcf_chain_info chain_info; 1862 struct tcf_proto *tp; 1863 int err = 0; 1864 1865 mutex_lock(&chain->filter_chain_lock); 1866 1867 if (tcf_proto_exists_destroying(chain, tp_new)) { 1868 mutex_unlock(&chain->filter_chain_lock); 1869 tcf_proto_destroy(tp_new, rtnl_held, false, NULL); 1870 return ERR_PTR(-EAGAIN); 1871 } 1872 1873 tp = tcf_chain_tp_find(chain, &chain_info, 1874 protocol, prio, false); 1875 if (!tp) 1876 err = tcf_chain_tp_insert(chain, &chain_info, tp_new); 1877 mutex_unlock(&chain->filter_chain_lock); 1878 1879 if (tp) { 1880 tcf_proto_destroy(tp_new, rtnl_held, false, NULL); 1881 tp_new = tp; 1882 } else if (err) { 1883 tcf_proto_destroy(tp_new, rtnl_held, false, NULL); 1884 tp_new = ERR_PTR(err); 1885 } 1886 1887 return tp_new; 1888 } 1889 1890 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain, 1891 struct tcf_proto *tp, bool rtnl_held, 1892 struct netlink_ext_ack *extack) 1893 { 1894 struct tcf_chain_info chain_info; 1895 struct tcf_proto *tp_iter; 1896 struct tcf_proto **pprev; 1897 struct tcf_proto *next; 1898 1899 mutex_lock(&chain->filter_chain_lock); 1900 1901 /* Atomically find and remove tp from chain. */ 1902 for (pprev = &chain->filter_chain; 1903 (tp_iter = tcf_chain_dereference(*pprev, chain)); 1904 pprev = &tp_iter->next) { 1905 if (tp_iter == tp) { 1906 chain_info.pprev = pprev; 1907 chain_info.next = tp_iter->next; 1908 WARN_ON(tp_iter->deleting); 1909 break; 1910 } 1911 } 1912 /* Verify that tp still exists and no new filters were inserted 1913 * concurrently. 1914 * Mark tp for deletion if it is empty. 1915 */ 1916 if (!tp_iter || !tcf_proto_check_delete(tp)) { 1917 mutex_unlock(&chain->filter_chain_lock); 1918 return; 1919 } 1920 1921 tcf_proto_signal_destroying(chain, tp); 1922 next = tcf_chain_dereference(chain_info.next, chain); 1923 if (tp == chain->filter_chain) 1924 tcf_chain0_head_change(chain, next); 1925 RCU_INIT_POINTER(*chain_info.pprev, next); 1926 mutex_unlock(&chain->filter_chain_lock); 1927 1928 tcf_proto_put(tp, rtnl_held, extack); 1929 } 1930 1931 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, 1932 struct tcf_chain_info *chain_info, 1933 u32 protocol, u32 prio, 1934 bool prio_allocate) 1935 { 1936 struct tcf_proto **pprev; 1937 struct tcf_proto *tp; 1938 1939 /* Check the chain for existence of proto-tcf with this priority */ 1940 for (pprev = &chain->filter_chain; 1941 (tp = tcf_chain_dereference(*pprev, chain)); 1942 pprev = &tp->next) { 1943 if (tp->prio >= prio) { 1944 if (tp->prio == prio) { 1945 if (prio_allocate || 1946 (tp->protocol != protocol && protocol)) 1947 return ERR_PTR(-EINVAL); 1948 } else { 1949 tp = NULL; 1950 } 1951 break; 1952 } 1953 } 1954 chain_info->pprev = pprev; 1955 if (tp) { 1956 chain_info->next = tp->next; 1957 tcf_proto_get(tp); 1958 } else { 1959 chain_info->next = NULL; 1960 } 1961 return tp; 1962 } 1963 1964 static int tcf_fill_node(struct net *net, struct sk_buff *skb, 1965 struct tcf_proto *tp, struct tcf_block *block, 1966 struct Qdisc *q, u32 parent, void *fh, 1967 u32 portid, u32 seq, u16 flags, int event, 1968 bool terse_dump, bool rtnl_held, 1969 struct netlink_ext_ack *extack) 1970 { 1971 struct tcmsg *tcm; 1972 struct nlmsghdr *nlh; 1973 unsigned char *b = skb_tail_pointer(skb); 1974 1975 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 1976 if (!nlh) 1977 goto out_nlmsg_trim; 1978 tcm = nlmsg_data(nlh); 1979 tcm->tcm_family = AF_UNSPEC; 1980 tcm->tcm__pad1 = 0; 1981 tcm->tcm__pad2 = 0; 1982 if (q) { 1983 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 1984 tcm->tcm_parent = parent; 1985 } else { 1986 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; 1987 tcm->tcm_block_index = block->index; 1988 } 1989 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol); 1990 if (nla_put_string(skb, TCA_KIND, tp->ops->kind)) 1991 goto nla_put_failure; 1992 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index)) 1993 goto nla_put_failure; 1994 if (!fh) { 1995 tcm->tcm_handle = 0; 1996 } else if (terse_dump) { 1997 if (tp->ops->terse_dump) { 1998 if (tp->ops->terse_dump(net, tp, fh, skb, tcm, 1999 rtnl_held) < 0) 2000 goto nla_put_failure; 2001 } else { 2002 goto cls_op_not_supp; 2003 } 2004 } else { 2005 if (tp->ops->dump && 2006 tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0) 2007 goto nla_put_failure; 2008 } 2009 2010 if (extack && extack->_msg && 2011 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 2012 goto nla_put_failure; 2013 2014 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2015 2016 return skb->len; 2017 2018 out_nlmsg_trim: 2019 nla_put_failure: 2020 cls_op_not_supp: 2021 nlmsg_trim(skb, b); 2022 return -1; 2023 } 2024 2025 static int tfilter_notify(struct net *net, struct sk_buff *oskb, 2026 struct nlmsghdr *n, struct tcf_proto *tp, 2027 struct tcf_block *block, struct Qdisc *q, 2028 u32 parent, void *fh, int event, bool unicast, 2029 bool rtnl_held, struct netlink_ext_ack *extack) 2030 { 2031 struct sk_buff *skb; 2032 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2033 int err = 0; 2034 2035 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2036 if (!skb) 2037 return -ENOBUFS; 2038 2039 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid, 2040 n->nlmsg_seq, n->nlmsg_flags, event, 2041 false, rtnl_held, extack) <= 0) { 2042 kfree_skb(skb); 2043 return -EINVAL; 2044 } 2045 2046 if (unicast) 2047 err = rtnl_unicast(skb, net, portid); 2048 else 2049 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2050 n->nlmsg_flags & NLM_F_ECHO); 2051 return err; 2052 } 2053 2054 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb, 2055 struct nlmsghdr *n, struct tcf_proto *tp, 2056 struct tcf_block *block, struct Qdisc *q, 2057 u32 parent, void *fh, bool unicast, bool *last, 2058 bool rtnl_held, struct netlink_ext_ack *extack) 2059 { 2060 struct sk_buff *skb; 2061 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2062 int err; 2063 2064 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2065 if (!skb) 2066 return -ENOBUFS; 2067 2068 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid, 2069 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER, 2070 false, rtnl_held, extack) <= 0) { 2071 NL_SET_ERR_MSG(extack, "Failed to build del event notification"); 2072 kfree_skb(skb); 2073 return -EINVAL; 2074 } 2075 2076 err = tp->ops->delete(tp, fh, last, rtnl_held, extack); 2077 if (err) { 2078 kfree_skb(skb); 2079 return err; 2080 } 2081 2082 if (unicast) 2083 err = rtnl_unicast(skb, net, portid); 2084 else 2085 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2086 n->nlmsg_flags & NLM_F_ECHO); 2087 if (err < 0) 2088 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification"); 2089 2090 return err; 2091 } 2092 2093 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb, 2094 struct tcf_block *block, struct Qdisc *q, 2095 u32 parent, struct nlmsghdr *n, 2096 struct tcf_chain *chain, int event, 2097 struct netlink_ext_ack *extack) 2098 { 2099 struct tcf_proto *tp; 2100 2101 for (tp = tcf_get_next_proto(chain, NULL); 2102 tp; tp = tcf_get_next_proto(chain, tp)) 2103 tfilter_notify(net, oskb, n, tp, block, q, parent, NULL, 2104 event, false, true, extack); 2105 } 2106 2107 static void tfilter_put(struct tcf_proto *tp, void *fh) 2108 { 2109 if (tp->ops->put && fh) 2110 tp->ops->put(tp, fh); 2111 } 2112 2113 static bool is_qdisc_ingress(__u32 classid) 2114 { 2115 return (TC_H_MIN(classid) == TC_H_MIN(TC_H_MIN_INGRESS)); 2116 } 2117 2118 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2119 struct netlink_ext_ack *extack) 2120 { 2121 struct net *net = sock_net(skb->sk); 2122 struct nlattr *tca[TCA_MAX + 1]; 2123 char name[IFNAMSIZ]; 2124 struct tcmsg *t; 2125 u32 protocol; 2126 u32 prio; 2127 bool prio_allocate; 2128 u32 parent; 2129 u32 chain_index; 2130 struct Qdisc *q; 2131 struct tcf_chain_info chain_info; 2132 struct tcf_chain *chain; 2133 struct tcf_block *block; 2134 struct tcf_proto *tp; 2135 unsigned long cl; 2136 void *fh; 2137 int err; 2138 int tp_created; 2139 bool rtnl_held = false; 2140 u32 flags; 2141 2142 replay: 2143 tp_created = 0; 2144 2145 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2146 rtm_tca_policy, extack); 2147 if (err < 0) 2148 return err; 2149 2150 t = nlmsg_data(n); 2151 protocol = TC_H_MIN(t->tcm_info); 2152 prio = TC_H_MAJ(t->tcm_info); 2153 prio_allocate = false; 2154 parent = t->tcm_parent; 2155 tp = NULL; 2156 cl = 0; 2157 block = NULL; 2158 q = NULL; 2159 chain = NULL; 2160 flags = 0; 2161 2162 if (prio == 0) { 2163 /* If no priority is provided by the user, 2164 * we allocate one. 2165 */ 2166 if (n->nlmsg_flags & NLM_F_CREATE) { 2167 prio = TC_H_MAKE(0x80000000U, 0U); 2168 prio_allocate = true; 2169 } else { 2170 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero"); 2171 return -ENOENT; 2172 } 2173 } 2174 2175 /* Find head of filter chain. */ 2176 2177 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2178 if (err) 2179 return err; 2180 2181 if (tcf_proto_check_kind(tca[TCA_KIND], name)) { 2182 NL_SET_ERR_MSG(extack, "Specified TC filter name too long"); 2183 err = -EINVAL; 2184 goto errout; 2185 } 2186 2187 /* Take rtnl mutex if rtnl_held was set to true on previous iteration, 2188 * block is shared (no qdisc found), qdisc is not unlocked, classifier 2189 * type is not specified, classifier is not unlocked. 2190 */ 2191 if (rtnl_held || 2192 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2193 !tcf_proto_is_unlocked(name)) { 2194 rtnl_held = true; 2195 rtnl_lock(); 2196 } 2197 2198 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2199 if (err) 2200 goto errout; 2201 2202 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2203 extack); 2204 if (IS_ERR(block)) { 2205 err = PTR_ERR(block); 2206 goto errout; 2207 } 2208 block->classid = parent; 2209 2210 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2211 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2212 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2213 err = -EINVAL; 2214 goto errout; 2215 } 2216 chain = tcf_chain_get(block, chain_index, true); 2217 if (!chain) { 2218 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain"); 2219 err = -ENOMEM; 2220 goto errout; 2221 } 2222 2223 mutex_lock(&chain->filter_chain_lock); 2224 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2225 prio, prio_allocate); 2226 if (IS_ERR(tp)) { 2227 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2228 err = PTR_ERR(tp); 2229 goto errout_locked; 2230 } 2231 2232 if (tp == NULL) { 2233 struct tcf_proto *tp_new = NULL; 2234 2235 if (chain->flushing) { 2236 err = -EAGAIN; 2237 goto errout_locked; 2238 } 2239 2240 /* Proto-tcf does not exist, create new one */ 2241 2242 if (tca[TCA_KIND] == NULL || !protocol) { 2243 NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified"); 2244 err = -EINVAL; 2245 goto errout_locked; 2246 } 2247 2248 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2249 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); 2250 err = -ENOENT; 2251 goto errout_locked; 2252 } 2253 2254 if (prio_allocate) 2255 prio = tcf_auto_prio(tcf_chain_tp_prev(chain, 2256 &chain_info)); 2257 2258 mutex_unlock(&chain->filter_chain_lock); 2259 tp_new = tcf_proto_create(name, protocol, prio, chain, 2260 rtnl_held, extack); 2261 if (IS_ERR(tp_new)) { 2262 err = PTR_ERR(tp_new); 2263 goto errout_tp; 2264 } 2265 2266 tp_created = 1; 2267 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio, 2268 rtnl_held); 2269 if (IS_ERR(tp)) { 2270 err = PTR_ERR(tp); 2271 goto errout_tp; 2272 } 2273 } else { 2274 mutex_unlock(&chain->filter_chain_lock); 2275 } 2276 2277 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2278 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2279 err = -EINVAL; 2280 goto errout; 2281 } 2282 2283 fh = tp->ops->get(tp, t->tcm_handle); 2284 2285 if (!fh) { 2286 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2287 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); 2288 err = -ENOENT; 2289 goto errout; 2290 } 2291 } else if (n->nlmsg_flags & NLM_F_EXCL) { 2292 tfilter_put(tp, fh); 2293 NL_SET_ERR_MSG(extack, "Filter already exists"); 2294 err = -EEXIST; 2295 goto errout; 2296 } 2297 2298 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) { 2299 tfilter_put(tp, fh); 2300 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind"); 2301 err = -EINVAL; 2302 goto errout; 2303 } 2304 2305 if (!(n->nlmsg_flags & NLM_F_CREATE)) 2306 flags |= TCA_ACT_FLAGS_REPLACE; 2307 if (!rtnl_held) 2308 flags |= TCA_ACT_FLAGS_NO_RTNL; 2309 if (is_qdisc_ingress(parent)) 2310 flags |= TCA_ACT_FLAGS_AT_INGRESS; 2311 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, 2312 flags, extack); 2313 if (err == 0) { 2314 tfilter_notify(net, skb, n, tp, block, q, parent, fh, 2315 RTM_NEWTFILTER, false, rtnl_held, extack); 2316 tfilter_put(tp, fh); 2317 /* q pointer is NULL for shared blocks */ 2318 if (q) 2319 q->flags &= ~TCQ_F_CAN_BYPASS; 2320 } 2321 2322 errout: 2323 if (err && tp_created) 2324 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL); 2325 errout_tp: 2326 if (chain) { 2327 if (tp && !IS_ERR(tp)) 2328 tcf_proto_put(tp, rtnl_held, NULL); 2329 if (!tp_created) 2330 tcf_chain_put(chain); 2331 } 2332 tcf_block_release(q, block, rtnl_held); 2333 2334 if (rtnl_held) 2335 rtnl_unlock(); 2336 2337 if (err == -EAGAIN) { 2338 /* Take rtnl lock in case EAGAIN is caused by concurrent flush 2339 * of target chain. 2340 */ 2341 rtnl_held = true; 2342 /* Replay the request. */ 2343 goto replay; 2344 } 2345 return err; 2346 2347 errout_locked: 2348 mutex_unlock(&chain->filter_chain_lock); 2349 goto errout; 2350 } 2351 2352 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2353 struct netlink_ext_ack *extack) 2354 { 2355 struct net *net = sock_net(skb->sk); 2356 struct nlattr *tca[TCA_MAX + 1]; 2357 char name[IFNAMSIZ]; 2358 struct tcmsg *t; 2359 u32 protocol; 2360 u32 prio; 2361 u32 parent; 2362 u32 chain_index; 2363 struct Qdisc *q = NULL; 2364 struct tcf_chain_info chain_info; 2365 struct tcf_chain *chain = NULL; 2366 struct tcf_block *block = NULL; 2367 struct tcf_proto *tp = NULL; 2368 unsigned long cl = 0; 2369 void *fh = NULL; 2370 int err; 2371 bool rtnl_held = false; 2372 2373 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2374 rtm_tca_policy, extack); 2375 if (err < 0) 2376 return err; 2377 2378 t = nlmsg_data(n); 2379 protocol = TC_H_MIN(t->tcm_info); 2380 prio = TC_H_MAJ(t->tcm_info); 2381 parent = t->tcm_parent; 2382 2383 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) { 2384 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set"); 2385 return -ENOENT; 2386 } 2387 2388 /* Find head of filter chain. */ 2389 2390 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2391 if (err) 2392 return err; 2393 2394 if (tcf_proto_check_kind(tca[TCA_KIND], name)) { 2395 NL_SET_ERR_MSG(extack, "Specified TC filter name too long"); 2396 err = -EINVAL; 2397 goto errout; 2398 } 2399 /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc 2400 * found), qdisc is not unlocked, classifier type is not specified, 2401 * classifier is not unlocked. 2402 */ 2403 if (!prio || 2404 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2405 !tcf_proto_is_unlocked(name)) { 2406 rtnl_held = true; 2407 rtnl_lock(); 2408 } 2409 2410 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2411 if (err) 2412 goto errout; 2413 2414 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2415 extack); 2416 if (IS_ERR(block)) { 2417 err = PTR_ERR(block); 2418 goto errout; 2419 } 2420 2421 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2422 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2423 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2424 err = -EINVAL; 2425 goto errout; 2426 } 2427 chain = tcf_chain_get(block, chain_index, false); 2428 if (!chain) { 2429 /* User requested flush on non-existent chain. Nothing to do, 2430 * so just return success. 2431 */ 2432 if (prio == 0) { 2433 err = 0; 2434 goto errout; 2435 } 2436 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2437 err = -ENOENT; 2438 goto errout; 2439 } 2440 2441 if (prio == 0) { 2442 tfilter_notify_chain(net, skb, block, q, parent, n, 2443 chain, RTM_DELTFILTER, extack); 2444 tcf_chain_flush(chain, rtnl_held); 2445 err = 0; 2446 goto errout; 2447 } 2448 2449 mutex_lock(&chain->filter_chain_lock); 2450 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2451 prio, false); 2452 if (!tp || IS_ERR(tp)) { 2453 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2454 err = tp ? PTR_ERR(tp) : -ENOENT; 2455 goto errout_locked; 2456 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2457 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2458 err = -EINVAL; 2459 goto errout_locked; 2460 } else if (t->tcm_handle == 0) { 2461 tcf_proto_signal_destroying(chain, tp); 2462 tcf_chain_tp_remove(chain, &chain_info, tp); 2463 mutex_unlock(&chain->filter_chain_lock); 2464 2465 tcf_proto_put(tp, rtnl_held, NULL); 2466 tfilter_notify(net, skb, n, tp, block, q, parent, fh, 2467 RTM_DELTFILTER, false, rtnl_held, extack); 2468 err = 0; 2469 goto errout; 2470 } 2471 mutex_unlock(&chain->filter_chain_lock); 2472 2473 fh = tp->ops->get(tp, t->tcm_handle); 2474 2475 if (!fh) { 2476 NL_SET_ERR_MSG(extack, "Specified filter handle not found"); 2477 err = -ENOENT; 2478 } else { 2479 bool last; 2480 2481 err = tfilter_del_notify(net, skb, n, tp, block, 2482 q, parent, fh, false, &last, 2483 rtnl_held, extack); 2484 2485 if (err) 2486 goto errout; 2487 if (last) 2488 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack); 2489 } 2490 2491 errout: 2492 if (chain) { 2493 if (tp && !IS_ERR(tp)) 2494 tcf_proto_put(tp, rtnl_held, NULL); 2495 tcf_chain_put(chain); 2496 } 2497 tcf_block_release(q, block, rtnl_held); 2498 2499 if (rtnl_held) 2500 rtnl_unlock(); 2501 2502 return err; 2503 2504 errout_locked: 2505 mutex_unlock(&chain->filter_chain_lock); 2506 goto errout; 2507 } 2508 2509 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2510 struct netlink_ext_ack *extack) 2511 { 2512 struct net *net = sock_net(skb->sk); 2513 struct nlattr *tca[TCA_MAX + 1]; 2514 char name[IFNAMSIZ]; 2515 struct tcmsg *t; 2516 u32 protocol; 2517 u32 prio; 2518 u32 parent; 2519 u32 chain_index; 2520 struct Qdisc *q = NULL; 2521 struct tcf_chain_info chain_info; 2522 struct tcf_chain *chain = NULL; 2523 struct tcf_block *block = NULL; 2524 struct tcf_proto *tp = NULL; 2525 unsigned long cl = 0; 2526 void *fh = NULL; 2527 int err; 2528 bool rtnl_held = false; 2529 2530 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2531 rtm_tca_policy, extack); 2532 if (err < 0) 2533 return err; 2534 2535 t = nlmsg_data(n); 2536 protocol = TC_H_MIN(t->tcm_info); 2537 prio = TC_H_MAJ(t->tcm_info); 2538 parent = t->tcm_parent; 2539 2540 if (prio == 0) { 2541 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero"); 2542 return -ENOENT; 2543 } 2544 2545 /* Find head of filter chain. */ 2546 2547 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2548 if (err) 2549 return err; 2550 2551 if (tcf_proto_check_kind(tca[TCA_KIND], name)) { 2552 NL_SET_ERR_MSG(extack, "Specified TC filter name too long"); 2553 err = -EINVAL; 2554 goto errout; 2555 } 2556 /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not 2557 * unlocked, classifier type is not specified, classifier is not 2558 * unlocked. 2559 */ 2560 if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2561 !tcf_proto_is_unlocked(name)) { 2562 rtnl_held = true; 2563 rtnl_lock(); 2564 } 2565 2566 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2567 if (err) 2568 goto errout; 2569 2570 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2571 extack); 2572 if (IS_ERR(block)) { 2573 err = PTR_ERR(block); 2574 goto errout; 2575 } 2576 2577 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2578 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2579 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2580 err = -EINVAL; 2581 goto errout; 2582 } 2583 chain = tcf_chain_get(block, chain_index, false); 2584 if (!chain) { 2585 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2586 err = -EINVAL; 2587 goto errout; 2588 } 2589 2590 mutex_lock(&chain->filter_chain_lock); 2591 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2592 prio, false); 2593 mutex_unlock(&chain->filter_chain_lock); 2594 if (!tp || IS_ERR(tp)) { 2595 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2596 err = tp ? PTR_ERR(tp) : -ENOENT; 2597 goto errout; 2598 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2599 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2600 err = -EINVAL; 2601 goto errout; 2602 } 2603 2604 fh = tp->ops->get(tp, t->tcm_handle); 2605 2606 if (!fh) { 2607 NL_SET_ERR_MSG(extack, "Specified filter handle not found"); 2608 err = -ENOENT; 2609 } else { 2610 err = tfilter_notify(net, skb, n, tp, block, q, parent, 2611 fh, RTM_NEWTFILTER, true, rtnl_held, NULL); 2612 if (err < 0) 2613 NL_SET_ERR_MSG(extack, "Failed to send filter notify message"); 2614 } 2615 2616 tfilter_put(tp, fh); 2617 errout: 2618 if (chain) { 2619 if (tp && !IS_ERR(tp)) 2620 tcf_proto_put(tp, rtnl_held, NULL); 2621 tcf_chain_put(chain); 2622 } 2623 tcf_block_release(q, block, rtnl_held); 2624 2625 if (rtnl_held) 2626 rtnl_unlock(); 2627 2628 return err; 2629 } 2630 2631 struct tcf_dump_args { 2632 struct tcf_walker w; 2633 struct sk_buff *skb; 2634 struct netlink_callback *cb; 2635 struct tcf_block *block; 2636 struct Qdisc *q; 2637 u32 parent; 2638 bool terse_dump; 2639 }; 2640 2641 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg) 2642 { 2643 struct tcf_dump_args *a = (void *)arg; 2644 struct net *net = sock_net(a->skb->sk); 2645 2646 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent, 2647 n, NETLINK_CB(a->cb->skb).portid, 2648 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, 2649 RTM_NEWTFILTER, a->terse_dump, true, NULL); 2650 } 2651 2652 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent, 2653 struct sk_buff *skb, struct netlink_callback *cb, 2654 long index_start, long *p_index, bool terse) 2655 { 2656 struct net *net = sock_net(skb->sk); 2657 struct tcf_block *block = chain->block; 2658 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2659 struct tcf_proto *tp, *tp_prev; 2660 struct tcf_dump_args arg; 2661 2662 for (tp = __tcf_get_next_proto(chain, NULL); 2663 tp; 2664 tp_prev = tp, 2665 tp = __tcf_get_next_proto(chain, tp), 2666 tcf_proto_put(tp_prev, true, NULL), 2667 (*p_index)++) { 2668 if (*p_index < index_start) 2669 continue; 2670 if (TC_H_MAJ(tcm->tcm_info) && 2671 TC_H_MAJ(tcm->tcm_info) != tp->prio) 2672 continue; 2673 if (TC_H_MIN(tcm->tcm_info) && 2674 TC_H_MIN(tcm->tcm_info) != tp->protocol) 2675 continue; 2676 if (*p_index > index_start) 2677 memset(&cb->args[1], 0, 2678 sizeof(cb->args) - sizeof(cb->args[0])); 2679 if (cb->args[1] == 0) { 2680 if (tcf_fill_node(net, skb, tp, block, q, parent, NULL, 2681 NETLINK_CB(cb->skb).portid, 2682 cb->nlh->nlmsg_seq, NLM_F_MULTI, 2683 RTM_NEWTFILTER, false, true, NULL) <= 0) 2684 goto errout; 2685 cb->args[1] = 1; 2686 } 2687 if (!tp->ops->walk) 2688 continue; 2689 arg.w.fn = tcf_node_dump; 2690 arg.skb = skb; 2691 arg.cb = cb; 2692 arg.block = block; 2693 arg.q = q; 2694 arg.parent = parent; 2695 arg.w.stop = 0; 2696 arg.w.skip = cb->args[1] - 1; 2697 arg.w.count = 0; 2698 arg.w.cookie = cb->args[2]; 2699 arg.terse_dump = terse; 2700 tp->ops->walk(tp, &arg.w, true); 2701 cb->args[2] = arg.w.cookie; 2702 cb->args[1] = arg.w.count + 1; 2703 if (arg.w.stop) 2704 goto errout; 2705 } 2706 return true; 2707 2708 errout: 2709 tcf_proto_put(tp, true, NULL); 2710 return false; 2711 } 2712 2713 static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = { 2714 [TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE), 2715 }; 2716 2717 /* called with RTNL */ 2718 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb) 2719 { 2720 struct tcf_chain *chain, *chain_prev; 2721 struct net *net = sock_net(skb->sk); 2722 struct nlattr *tca[TCA_MAX + 1]; 2723 struct Qdisc *q = NULL; 2724 struct tcf_block *block; 2725 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2726 bool terse_dump = false; 2727 long index_start; 2728 long index; 2729 u32 parent; 2730 int err; 2731 2732 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2733 return skb->len; 2734 2735 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX, 2736 tcf_tfilter_dump_policy, cb->extack); 2737 if (err) 2738 return err; 2739 2740 if (tca[TCA_DUMP_FLAGS]) { 2741 struct nla_bitfield32 flags = 2742 nla_get_bitfield32(tca[TCA_DUMP_FLAGS]); 2743 2744 terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE; 2745 } 2746 2747 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 2748 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); 2749 if (!block) 2750 goto out; 2751 /* If we work with block index, q is NULL and parent value 2752 * will never be used in the following code. The check 2753 * in tcf_fill_node prevents it. However, compiler does not 2754 * see that far, so set parent to zero to silence the warning 2755 * about parent being uninitialized. 2756 */ 2757 parent = 0; 2758 } else { 2759 const struct Qdisc_class_ops *cops; 2760 struct net_device *dev; 2761 unsigned long cl = 0; 2762 2763 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2764 if (!dev) 2765 return skb->len; 2766 2767 parent = tcm->tcm_parent; 2768 if (!parent) 2769 q = rtnl_dereference(dev->qdisc); 2770 else 2771 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); 2772 if (!q) 2773 goto out; 2774 cops = q->ops->cl_ops; 2775 if (!cops) 2776 goto out; 2777 if (!cops->tcf_block) 2778 goto out; 2779 if (TC_H_MIN(tcm->tcm_parent)) { 2780 cl = cops->find(q, tcm->tcm_parent); 2781 if (cl == 0) 2782 goto out; 2783 } 2784 block = cops->tcf_block(q, cl, NULL); 2785 if (!block) 2786 goto out; 2787 parent = block->classid; 2788 if (tcf_block_shared(block)) 2789 q = NULL; 2790 } 2791 2792 index_start = cb->args[0]; 2793 index = 0; 2794 2795 for (chain = __tcf_get_next_chain(block, NULL); 2796 chain; 2797 chain_prev = chain, 2798 chain = __tcf_get_next_chain(block, chain), 2799 tcf_chain_put(chain_prev)) { 2800 if (tca[TCA_CHAIN] && 2801 nla_get_u32(tca[TCA_CHAIN]) != chain->index) 2802 continue; 2803 if (!tcf_chain_dump(chain, q, parent, skb, cb, 2804 index_start, &index, terse_dump)) { 2805 tcf_chain_put(chain); 2806 err = -EMSGSIZE; 2807 break; 2808 } 2809 } 2810 2811 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) 2812 tcf_block_refcnt_put(block, true); 2813 cb->args[0] = index; 2814 2815 out: 2816 /* If we did no progress, the error (EMSGSIZE) is real */ 2817 if (skb->len == 0 && err) 2818 return err; 2819 return skb->len; 2820 } 2821 2822 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops, 2823 void *tmplt_priv, u32 chain_index, 2824 struct net *net, struct sk_buff *skb, 2825 struct tcf_block *block, 2826 u32 portid, u32 seq, u16 flags, int event, 2827 struct netlink_ext_ack *extack) 2828 { 2829 unsigned char *b = skb_tail_pointer(skb); 2830 const struct tcf_proto_ops *ops; 2831 struct nlmsghdr *nlh; 2832 struct tcmsg *tcm; 2833 void *priv; 2834 2835 ops = tmplt_ops; 2836 priv = tmplt_priv; 2837 2838 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 2839 if (!nlh) 2840 goto out_nlmsg_trim; 2841 tcm = nlmsg_data(nlh); 2842 tcm->tcm_family = AF_UNSPEC; 2843 tcm->tcm__pad1 = 0; 2844 tcm->tcm__pad2 = 0; 2845 tcm->tcm_handle = 0; 2846 if (block->q) { 2847 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex; 2848 tcm->tcm_parent = block->q->handle; 2849 } else { 2850 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; 2851 tcm->tcm_block_index = block->index; 2852 } 2853 2854 if (nla_put_u32(skb, TCA_CHAIN, chain_index)) 2855 goto nla_put_failure; 2856 2857 if (ops) { 2858 if (nla_put_string(skb, TCA_KIND, ops->kind)) 2859 goto nla_put_failure; 2860 if (ops->tmplt_dump(skb, net, priv) < 0) 2861 goto nla_put_failure; 2862 } 2863 2864 if (extack && extack->_msg && 2865 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 2866 goto out_nlmsg_trim; 2867 2868 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2869 2870 return skb->len; 2871 2872 out_nlmsg_trim: 2873 nla_put_failure: 2874 nlmsg_trim(skb, b); 2875 return -EMSGSIZE; 2876 } 2877 2878 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, 2879 u32 seq, u16 flags, int event, bool unicast, 2880 struct netlink_ext_ack *extack) 2881 { 2882 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2883 struct tcf_block *block = chain->block; 2884 struct net *net = block->net; 2885 struct sk_buff *skb; 2886 int err = 0; 2887 2888 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2889 if (!skb) 2890 return -ENOBUFS; 2891 2892 if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, 2893 chain->index, net, skb, block, portid, 2894 seq, flags, event, extack) <= 0) { 2895 kfree_skb(skb); 2896 return -EINVAL; 2897 } 2898 2899 if (unicast) 2900 err = rtnl_unicast(skb, net, portid); 2901 else 2902 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2903 flags & NLM_F_ECHO); 2904 2905 return err; 2906 } 2907 2908 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, 2909 void *tmplt_priv, u32 chain_index, 2910 struct tcf_block *block, struct sk_buff *oskb, 2911 u32 seq, u16 flags, bool unicast) 2912 { 2913 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2914 struct net *net = block->net; 2915 struct sk_buff *skb; 2916 2917 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2918 if (!skb) 2919 return -ENOBUFS; 2920 2921 if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb, 2922 block, portid, seq, flags, RTM_DELCHAIN, NULL) <= 0) { 2923 kfree_skb(skb); 2924 return -EINVAL; 2925 } 2926 2927 if (unicast) 2928 return rtnl_unicast(skb, net, portid); 2929 2930 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO); 2931 } 2932 2933 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net, 2934 struct nlattr **tca, 2935 struct netlink_ext_ack *extack) 2936 { 2937 const struct tcf_proto_ops *ops; 2938 char name[IFNAMSIZ]; 2939 void *tmplt_priv; 2940 2941 /* If kind is not set, user did not specify template. */ 2942 if (!tca[TCA_KIND]) 2943 return 0; 2944 2945 if (tcf_proto_check_kind(tca[TCA_KIND], name)) { 2946 NL_SET_ERR_MSG(extack, "Specified TC chain template name too long"); 2947 return -EINVAL; 2948 } 2949 2950 ops = tcf_proto_lookup_ops(name, true, extack); 2951 if (IS_ERR(ops)) 2952 return PTR_ERR(ops); 2953 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) { 2954 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier"); 2955 return -EOPNOTSUPP; 2956 } 2957 2958 tmplt_priv = ops->tmplt_create(net, chain, tca, extack); 2959 if (IS_ERR(tmplt_priv)) { 2960 module_put(ops->owner); 2961 return PTR_ERR(tmplt_priv); 2962 } 2963 chain->tmplt_ops = ops; 2964 chain->tmplt_priv = tmplt_priv; 2965 return 0; 2966 } 2967 2968 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops, 2969 void *tmplt_priv) 2970 { 2971 /* If template ops are set, no work to do for us. */ 2972 if (!tmplt_ops) 2973 return; 2974 2975 tmplt_ops->tmplt_destroy(tmplt_priv); 2976 module_put(tmplt_ops->owner); 2977 } 2978 2979 /* Add/delete/get a chain */ 2980 2981 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n, 2982 struct netlink_ext_ack *extack) 2983 { 2984 struct net *net = sock_net(skb->sk); 2985 struct nlattr *tca[TCA_MAX + 1]; 2986 struct tcmsg *t; 2987 u32 parent; 2988 u32 chain_index; 2989 struct Qdisc *q; 2990 struct tcf_chain *chain; 2991 struct tcf_block *block; 2992 unsigned long cl; 2993 int err; 2994 2995 replay: 2996 q = NULL; 2997 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2998 rtm_tca_policy, extack); 2999 if (err < 0) 3000 return err; 3001 3002 t = nlmsg_data(n); 3003 parent = t->tcm_parent; 3004 cl = 0; 3005 3006 block = tcf_block_find(net, &q, &parent, &cl, 3007 t->tcm_ifindex, t->tcm_block_index, extack); 3008 if (IS_ERR(block)) 3009 return PTR_ERR(block); 3010 3011 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 3012 if (chain_index > TC_ACT_EXT_VAL_MASK) { 3013 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 3014 err = -EINVAL; 3015 goto errout_block; 3016 } 3017 3018 mutex_lock(&block->lock); 3019 chain = tcf_chain_lookup(block, chain_index); 3020 if (n->nlmsg_type == RTM_NEWCHAIN) { 3021 if (chain) { 3022 if (tcf_chain_held_by_acts_only(chain)) { 3023 /* The chain exists only because there is 3024 * some action referencing it. 3025 */ 3026 tcf_chain_hold(chain); 3027 } else { 3028 NL_SET_ERR_MSG(extack, "Filter chain already exists"); 3029 err = -EEXIST; 3030 goto errout_block_locked; 3031 } 3032 } else { 3033 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 3034 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain"); 3035 err = -ENOENT; 3036 goto errout_block_locked; 3037 } 3038 chain = tcf_chain_create(block, chain_index); 3039 if (!chain) { 3040 NL_SET_ERR_MSG(extack, "Failed to create filter chain"); 3041 err = -ENOMEM; 3042 goto errout_block_locked; 3043 } 3044 } 3045 } else { 3046 if (!chain || tcf_chain_held_by_acts_only(chain)) { 3047 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 3048 err = -EINVAL; 3049 goto errout_block_locked; 3050 } 3051 tcf_chain_hold(chain); 3052 } 3053 3054 if (n->nlmsg_type == RTM_NEWCHAIN) { 3055 /* Modifying chain requires holding parent block lock. In case 3056 * the chain was successfully added, take a reference to the 3057 * chain. This ensures that an empty chain does not disappear at 3058 * the end of this function. 3059 */ 3060 tcf_chain_hold(chain); 3061 chain->explicitly_created = true; 3062 } 3063 mutex_unlock(&block->lock); 3064 3065 switch (n->nlmsg_type) { 3066 case RTM_NEWCHAIN: 3067 err = tc_chain_tmplt_add(chain, net, tca, extack); 3068 if (err) { 3069 tcf_chain_put_explicitly_created(chain); 3070 goto errout; 3071 } 3072 3073 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL, 3074 RTM_NEWCHAIN, false, extack); 3075 break; 3076 case RTM_DELCHAIN: 3077 tfilter_notify_chain(net, skb, block, q, parent, n, 3078 chain, RTM_DELTFILTER, extack); 3079 /* Flush the chain first as the user requested chain removal. */ 3080 tcf_chain_flush(chain, true); 3081 /* In case the chain was successfully deleted, put a reference 3082 * to the chain previously taken during addition. 3083 */ 3084 tcf_chain_put_explicitly_created(chain); 3085 break; 3086 case RTM_GETCHAIN: 3087 err = tc_chain_notify(chain, skb, n->nlmsg_seq, 3088 n->nlmsg_flags, n->nlmsg_type, true, extack); 3089 if (err < 0) 3090 NL_SET_ERR_MSG(extack, "Failed to send chain notify message"); 3091 break; 3092 default: 3093 err = -EOPNOTSUPP; 3094 NL_SET_ERR_MSG(extack, "Unsupported message type"); 3095 goto errout; 3096 } 3097 3098 errout: 3099 tcf_chain_put(chain); 3100 errout_block: 3101 tcf_block_release(q, block, true); 3102 if (err == -EAGAIN) 3103 /* Replay the request. */ 3104 goto replay; 3105 return err; 3106 3107 errout_block_locked: 3108 mutex_unlock(&block->lock); 3109 goto errout_block; 3110 } 3111 3112 /* called with RTNL */ 3113 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb) 3114 { 3115 struct net *net = sock_net(skb->sk); 3116 struct nlattr *tca[TCA_MAX + 1]; 3117 struct Qdisc *q = NULL; 3118 struct tcf_block *block; 3119 struct tcmsg *tcm = nlmsg_data(cb->nlh); 3120 struct tcf_chain *chain; 3121 long index_start; 3122 long index; 3123 int err; 3124 3125 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 3126 return skb->len; 3127 3128 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX, 3129 rtm_tca_policy, cb->extack); 3130 if (err) 3131 return err; 3132 3133 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 3134 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); 3135 if (!block) 3136 goto out; 3137 } else { 3138 const struct Qdisc_class_ops *cops; 3139 struct net_device *dev; 3140 unsigned long cl = 0; 3141 3142 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 3143 if (!dev) 3144 return skb->len; 3145 3146 if (!tcm->tcm_parent) 3147 q = rtnl_dereference(dev->qdisc); 3148 else 3149 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); 3150 3151 if (!q) 3152 goto out; 3153 cops = q->ops->cl_ops; 3154 if (!cops) 3155 goto out; 3156 if (!cops->tcf_block) 3157 goto out; 3158 if (TC_H_MIN(tcm->tcm_parent)) { 3159 cl = cops->find(q, tcm->tcm_parent); 3160 if (cl == 0) 3161 goto out; 3162 } 3163 block = cops->tcf_block(q, cl, NULL); 3164 if (!block) 3165 goto out; 3166 if (tcf_block_shared(block)) 3167 q = NULL; 3168 } 3169 3170 index_start = cb->args[0]; 3171 index = 0; 3172 3173 mutex_lock(&block->lock); 3174 list_for_each_entry(chain, &block->chain_list, list) { 3175 if ((tca[TCA_CHAIN] && 3176 nla_get_u32(tca[TCA_CHAIN]) != chain->index)) 3177 continue; 3178 if (index < index_start) { 3179 index++; 3180 continue; 3181 } 3182 if (tcf_chain_held_by_acts_only(chain)) 3183 continue; 3184 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, 3185 chain->index, net, skb, block, 3186 NETLINK_CB(cb->skb).portid, 3187 cb->nlh->nlmsg_seq, NLM_F_MULTI, 3188 RTM_NEWCHAIN, NULL); 3189 if (err <= 0) 3190 break; 3191 index++; 3192 } 3193 mutex_unlock(&block->lock); 3194 3195 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) 3196 tcf_block_refcnt_put(block, true); 3197 cb->args[0] = index; 3198 3199 out: 3200 /* If we did no progress, the error (EMSGSIZE) is real */ 3201 if (skb->len == 0 && err) 3202 return err; 3203 return skb->len; 3204 } 3205 3206 int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action, 3207 int police, struct tcf_proto *tp, u32 handle, 3208 bool use_action_miss) 3209 { 3210 int err = 0; 3211 3212 #ifdef CONFIG_NET_CLS_ACT 3213 exts->type = 0; 3214 exts->nr_actions = 0; 3215 exts->miss_cookie_node = NULL; 3216 /* Note: we do not own yet a reference on net. 3217 * This reference might be taken later from tcf_exts_get_net(). 3218 */ 3219 exts->net = net; 3220 exts->actions = kcalloc(TCA_ACT_MAX_PRIO, sizeof(struct tc_action *), 3221 GFP_KERNEL); 3222 if (!exts->actions) 3223 return -ENOMEM; 3224 #endif 3225 3226 exts->action = action; 3227 exts->police = police; 3228 3229 if (!use_action_miss) 3230 return 0; 3231 3232 err = tcf_exts_miss_cookie_base_alloc(exts, tp, handle); 3233 if (err) 3234 goto err_miss_alloc; 3235 3236 return 0; 3237 3238 err_miss_alloc: 3239 tcf_exts_destroy(exts); 3240 #ifdef CONFIG_NET_CLS_ACT 3241 exts->actions = NULL; 3242 #endif 3243 return err; 3244 } 3245 EXPORT_SYMBOL(tcf_exts_init_ex); 3246 3247 void tcf_exts_destroy(struct tcf_exts *exts) 3248 { 3249 tcf_exts_miss_cookie_base_destroy(exts); 3250 3251 #ifdef CONFIG_NET_CLS_ACT 3252 if (exts->actions) { 3253 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND); 3254 kfree(exts->actions); 3255 } 3256 exts->nr_actions = 0; 3257 #endif 3258 } 3259 EXPORT_SYMBOL(tcf_exts_destroy); 3260 3261 int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb, 3262 struct nlattr *rate_tlv, struct tcf_exts *exts, 3263 u32 flags, u32 fl_flags, struct netlink_ext_ack *extack) 3264 { 3265 #ifdef CONFIG_NET_CLS_ACT 3266 { 3267 int init_res[TCA_ACT_MAX_PRIO] = {}; 3268 struct tc_action *act; 3269 size_t attr_size = 0; 3270 3271 if (exts->police && tb[exts->police]) { 3272 struct tc_action_ops *a_o; 3273 3274 a_o = tc_action_load_ops(tb[exts->police], true, 3275 !(flags & TCA_ACT_FLAGS_NO_RTNL), 3276 extack); 3277 if (IS_ERR(a_o)) 3278 return PTR_ERR(a_o); 3279 flags |= TCA_ACT_FLAGS_POLICE | TCA_ACT_FLAGS_BIND; 3280 act = tcf_action_init_1(net, tp, tb[exts->police], 3281 rate_tlv, a_o, init_res, flags, 3282 extack); 3283 module_put(a_o->owner); 3284 if (IS_ERR(act)) 3285 return PTR_ERR(act); 3286 3287 act->type = exts->type = TCA_OLD_COMPAT; 3288 exts->actions[0] = act; 3289 exts->nr_actions = 1; 3290 tcf_idr_insert_many(exts->actions); 3291 } else if (exts->action && tb[exts->action]) { 3292 int err; 3293 3294 flags |= TCA_ACT_FLAGS_BIND; 3295 err = tcf_action_init(net, tp, tb[exts->action], 3296 rate_tlv, exts->actions, init_res, 3297 &attr_size, flags, fl_flags, 3298 extack); 3299 if (err < 0) 3300 return err; 3301 exts->nr_actions = err; 3302 } 3303 } 3304 #else 3305 if ((exts->action && tb[exts->action]) || 3306 (exts->police && tb[exts->police])) { 3307 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)"); 3308 return -EOPNOTSUPP; 3309 } 3310 #endif 3311 3312 return 0; 3313 } 3314 EXPORT_SYMBOL(tcf_exts_validate_ex); 3315 3316 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb, 3317 struct nlattr *rate_tlv, struct tcf_exts *exts, 3318 u32 flags, struct netlink_ext_ack *extack) 3319 { 3320 return tcf_exts_validate_ex(net, tp, tb, rate_tlv, exts, 3321 flags, 0, extack); 3322 } 3323 EXPORT_SYMBOL(tcf_exts_validate); 3324 3325 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src) 3326 { 3327 #ifdef CONFIG_NET_CLS_ACT 3328 struct tcf_exts old = *dst; 3329 3330 *dst = *src; 3331 tcf_exts_destroy(&old); 3332 #endif 3333 } 3334 EXPORT_SYMBOL(tcf_exts_change); 3335 3336 #ifdef CONFIG_NET_CLS_ACT 3337 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts) 3338 { 3339 if (exts->nr_actions == 0) 3340 return NULL; 3341 else 3342 return exts->actions[0]; 3343 } 3344 #endif 3345 3346 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts) 3347 { 3348 #ifdef CONFIG_NET_CLS_ACT 3349 struct nlattr *nest; 3350 3351 if (exts->action && tcf_exts_has_actions(exts)) { 3352 /* 3353 * again for backward compatible mode - we want 3354 * to work with both old and new modes of entering 3355 * tc data even if iproute2 was newer - jhs 3356 */ 3357 if (exts->type != TCA_OLD_COMPAT) { 3358 nest = nla_nest_start_noflag(skb, exts->action); 3359 if (nest == NULL) 3360 goto nla_put_failure; 3361 3362 if (tcf_action_dump(skb, exts->actions, 0, 0, false) 3363 < 0) 3364 goto nla_put_failure; 3365 nla_nest_end(skb, nest); 3366 } else if (exts->police) { 3367 struct tc_action *act = tcf_exts_first_act(exts); 3368 nest = nla_nest_start_noflag(skb, exts->police); 3369 if (nest == NULL || !act) 3370 goto nla_put_failure; 3371 if (tcf_action_dump_old(skb, act, 0, 0) < 0) 3372 goto nla_put_failure; 3373 nla_nest_end(skb, nest); 3374 } 3375 } 3376 return 0; 3377 3378 nla_put_failure: 3379 nla_nest_cancel(skb, nest); 3380 return -1; 3381 #else 3382 return 0; 3383 #endif 3384 } 3385 EXPORT_SYMBOL(tcf_exts_dump); 3386 3387 int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts) 3388 { 3389 #ifdef CONFIG_NET_CLS_ACT 3390 struct nlattr *nest; 3391 3392 if (!exts->action || !tcf_exts_has_actions(exts)) 3393 return 0; 3394 3395 nest = nla_nest_start_noflag(skb, exts->action); 3396 if (!nest) 3397 goto nla_put_failure; 3398 3399 if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0) 3400 goto nla_put_failure; 3401 nla_nest_end(skb, nest); 3402 return 0; 3403 3404 nla_put_failure: 3405 nla_nest_cancel(skb, nest); 3406 return -1; 3407 #else 3408 return 0; 3409 #endif 3410 } 3411 EXPORT_SYMBOL(tcf_exts_terse_dump); 3412 3413 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts) 3414 { 3415 #ifdef CONFIG_NET_CLS_ACT 3416 struct tc_action *a = tcf_exts_first_act(exts); 3417 if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0) 3418 return -1; 3419 #endif 3420 return 0; 3421 } 3422 EXPORT_SYMBOL(tcf_exts_dump_stats); 3423 3424 static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) 3425 { 3426 if (*flags & TCA_CLS_FLAGS_IN_HW) 3427 return; 3428 *flags |= TCA_CLS_FLAGS_IN_HW; 3429 atomic_inc(&block->offloadcnt); 3430 } 3431 3432 static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags) 3433 { 3434 if (!(*flags & TCA_CLS_FLAGS_IN_HW)) 3435 return; 3436 *flags &= ~TCA_CLS_FLAGS_IN_HW; 3437 atomic_dec(&block->offloadcnt); 3438 } 3439 3440 static void tc_cls_offload_cnt_update(struct tcf_block *block, 3441 struct tcf_proto *tp, u32 *cnt, 3442 u32 *flags, u32 diff, bool add) 3443 { 3444 lockdep_assert_held(&block->cb_lock); 3445 3446 spin_lock(&tp->lock); 3447 if (add) { 3448 if (!*cnt) 3449 tcf_block_offload_inc(block, flags); 3450 *cnt += diff; 3451 } else { 3452 *cnt -= diff; 3453 if (!*cnt) 3454 tcf_block_offload_dec(block, flags); 3455 } 3456 spin_unlock(&tp->lock); 3457 } 3458 3459 static void 3460 tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp, 3461 u32 *cnt, u32 *flags) 3462 { 3463 lockdep_assert_held(&block->cb_lock); 3464 3465 spin_lock(&tp->lock); 3466 tcf_block_offload_dec(block, flags); 3467 *cnt = 0; 3468 spin_unlock(&tp->lock); 3469 } 3470 3471 static int 3472 __tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, 3473 void *type_data, bool err_stop) 3474 { 3475 struct flow_block_cb *block_cb; 3476 int ok_count = 0; 3477 int err; 3478 3479 list_for_each_entry(block_cb, &block->flow_block.cb_list, list) { 3480 err = block_cb->cb(type, type_data, block_cb->cb_priv); 3481 if (err) { 3482 if (err_stop) 3483 return err; 3484 } else { 3485 ok_count++; 3486 } 3487 } 3488 return ok_count; 3489 } 3490 3491 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, 3492 void *type_data, bool err_stop, bool rtnl_held) 3493 { 3494 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; 3495 int ok_count; 3496 3497 retry: 3498 if (take_rtnl) 3499 rtnl_lock(); 3500 down_read(&block->cb_lock); 3501 /* Need to obtain rtnl lock if block is bound to devs that require it. 3502 * In block bind code cb_lock is obtained while holding rtnl, so we must 3503 * obtain the locks in same order here. 3504 */ 3505 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { 3506 up_read(&block->cb_lock); 3507 take_rtnl = true; 3508 goto retry; 3509 } 3510 3511 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop); 3512 3513 up_read(&block->cb_lock); 3514 if (take_rtnl) 3515 rtnl_unlock(); 3516 return ok_count; 3517 } 3518 EXPORT_SYMBOL(tc_setup_cb_call); 3519 3520 /* Non-destructive filter add. If filter that wasn't already in hardware is 3521 * successfully offloaded, increment block offloads counter. On failure, 3522 * previously offloaded filter is considered to be intact and offloads counter 3523 * is not decremented. 3524 */ 3525 3526 int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp, 3527 enum tc_setup_type type, void *type_data, bool err_stop, 3528 u32 *flags, unsigned int *in_hw_count, bool rtnl_held) 3529 { 3530 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; 3531 int ok_count; 3532 3533 retry: 3534 if (take_rtnl) 3535 rtnl_lock(); 3536 down_read(&block->cb_lock); 3537 /* Need to obtain rtnl lock if block is bound to devs that require it. 3538 * In block bind code cb_lock is obtained while holding rtnl, so we must 3539 * obtain the locks in same order here. 3540 */ 3541 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { 3542 up_read(&block->cb_lock); 3543 take_rtnl = true; 3544 goto retry; 3545 } 3546 3547 /* Make sure all netdevs sharing this block are offload-capable. */ 3548 if (block->nooffloaddevcnt && err_stop) { 3549 ok_count = -EOPNOTSUPP; 3550 goto err_unlock; 3551 } 3552 3553 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop); 3554 if (ok_count < 0) 3555 goto err_unlock; 3556 3557 if (tp->ops->hw_add) 3558 tp->ops->hw_add(tp, type_data); 3559 if (ok_count > 0) 3560 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 3561 ok_count, true); 3562 err_unlock: 3563 up_read(&block->cb_lock); 3564 if (take_rtnl) 3565 rtnl_unlock(); 3566 return min(ok_count, 0); 3567 } 3568 EXPORT_SYMBOL(tc_setup_cb_add); 3569 3570 /* Destructive filter replace. If filter that wasn't already in hardware is 3571 * successfully offloaded, increment block offload counter. On failure, 3572 * previously offloaded filter is considered to be destroyed and offload counter 3573 * is decremented. 3574 */ 3575 3576 int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp, 3577 enum tc_setup_type type, void *type_data, bool err_stop, 3578 u32 *old_flags, unsigned int *old_in_hw_count, 3579 u32 *new_flags, unsigned int *new_in_hw_count, 3580 bool rtnl_held) 3581 { 3582 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; 3583 int ok_count; 3584 3585 retry: 3586 if (take_rtnl) 3587 rtnl_lock(); 3588 down_read(&block->cb_lock); 3589 /* Need to obtain rtnl lock if block is bound to devs that require it. 3590 * In block bind code cb_lock is obtained while holding rtnl, so we must 3591 * obtain the locks in same order here. 3592 */ 3593 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { 3594 up_read(&block->cb_lock); 3595 take_rtnl = true; 3596 goto retry; 3597 } 3598 3599 /* Make sure all netdevs sharing this block are offload-capable. */ 3600 if (block->nooffloaddevcnt && err_stop) { 3601 ok_count = -EOPNOTSUPP; 3602 goto err_unlock; 3603 } 3604 3605 tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags); 3606 if (tp->ops->hw_del) 3607 tp->ops->hw_del(tp, type_data); 3608 3609 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop); 3610 if (ok_count < 0) 3611 goto err_unlock; 3612 3613 if (tp->ops->hw_add) 3614 tp->ops->hw_add(tp, type_data); 3615 if (ok_count > 0) 3616 tc_cls_offload_cnt_update(block, tp, new_in_hw_count, 3617 new_flags, ok_count, true); 3618 err_unlock: 3619 up_read(&block->cb_lock); 3620 if (take_rtnl) 3621 rtnl_unlock(); 3622 return min(ok_count, 0); 3623 } 3624 EXPORT_SYMBOL(tc_setup_cb_replace); 3625 3626 /* Destroy filter and decrement block offload counter, if filter was previously 3627 * offloaded. 3628 */ 3629 3630 int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp, 3631 enum tc_setup_type type, void *type_data, bool err_stop, 3632 u32 *flags, unsigned int *in_hw_count, bool rtnl_held) 3633 { 3634 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; 3635 int ok_count; 3636 3637 retry: 3638 if (take_rtnl) 3639 rtnl_lock(); 3640 down_read(&block->cb_lock); 3641 /* Need to obtain rtnl lock if block is bound to devs that require it. 3642 * In block bind code cb_lock is obtained while holding rtnl, so we must 3643 * obtain the locks in same order here. 3644 */ 3645 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { 3646 up_read(&block->cb_lock); 3647 take_rtnl = true; 3648 goto retry; 3649 } 3650 3651 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop); 3652 3653 tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags); 3654 if (tp->ops->hw_del) 3655 tp->ops->hw_del(tp, type_data); 3656 3657 up_read(&block->cb_lock); 3658 if (take_rtnl) 3659 rtnl_unlock(); 3660 return min(ok_count, 0); 3661 } 3662 EXPORT_SYMBOL(tc_setup_cb_destroy); 3663 3664 int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp, 3665 bool add, flow_setup_cb_t *cb, 3666 enum tc_setup_type type, void *type_data, 3667 void *cb_priv, u32 *flags, unsigned int *in_hw_count) 3668 { 3669 int err = cb(type, type_data, cb_priv); 3670 3671 if (err) { 3672 if (add && tc_skip_sw(*flags)) 3673 return err; 3674 } else { 3675 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1, 3676 add); 3677 } 3678 3679 return 0; 3680 } 3681 EXPORT_SYMBOL(tc_setup_cb_reoffload); 3682 3683 static int tcf_act_get_user_cookie(struct flow_action_entry *entry, 3684 const struct tc_action *act) 3685 { 3686 struct tc_cookie *user_cookie; 3687 int err = 0; 3688 3689 rcu_read_lock(); 3690 user_cookie = rcu_dereference(act->user_cookie); 3691 if (user_cookie) { 3692 entry->user_cookie = flow_action_cookie_create(user_cookie->data, 3693 user_cookie->len, 3694 GFP_ATOMIC); 3695 if (!entry->user_cookie) 3696 err = -ENOMEM; 3697 } 3698 rcu_read_unlock(); 3699 return err; 3700 } 3701 3702 static void tcf_act_put_user_cookie(struct flow_action_entry *entry) 3703 { 3704 flow_action_cookie_destroy(entry->user_cookie); 3705 } 3706 3707 void tc_cleanup_offload_action(struct flow_action *flow_action) 3708 { 3709 struct flow_action_entry *entry; 3710 int i; 3711 3712 flow_action_for_each(i, entry, flow_action) { 3713 tcf_act_put_user_cookie(entry); 3714 if (entry->destructor) 3715 entry->destructor(entry->destructor_priv); 3716 } 3717 } 3718 EXPORT_SYMBOL(tc_cleanup_offload_action); 3719 3720 static int tc_setup_offload_act(struct tc_action *act, 3721 struct flow_action_entry *entry, 3722 u32 *index_inc, 3723 struct netlink_ext_ack *extack) 3724 { 3725 #ifdef CONFIG_NET_CLS_ACT 3726 if (act->ops->offload_act_setup) { 3727 return act->ops->offload_act_setup(act, entry, index_inc, true, 3728 extack); 3729 } else { 3730 NL_SET_ERR_MSG(extack, "Action does not support offload"); 3731 return -EOPNOTSUPP; 3732 } 3733 #else 3734 return 0; 3735 #endif 3736 } 3737 3738 int tc_setup_action(struct flow_action *flow_action, 3739 struct tc_action *actions[], 3740 u32 miss_cookie_base, 3741 struct netlink_ext_ack *extack) 3742 { 3743 int i, j, k, index, err = 0; 3744 struct tc_action *act; 3745 3746 BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY); 3747 BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE); 3748 BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED); 3749 3750 if (!actions) 3751 return 0; 3752 3753 j = 0; 3754 tcf_act_for_each_action(i, act, actions) { 3755 struct flow_action_entry *entry; 3756 3757 entry = &flow_action->entries[j]; 3758 spin_lock_bh(&act->tcfa_lock); 3759 err = tcf_act_get_user_cookie(entry, act); 3760 if (err) 3761 goto err_out_locked; 3762 3763 index = 0; 3764 err = tc_setup_offload_act(act, entry, &index, extack); 3765 if (err) 3766 goto err_out_locked; 3767 3768 for (k = 0; k < index ; k++) { 3769 entry[k].hw_stats = tc_act_hw_stats(act->hw_stats); 3770 entry[k].hw_index = act->tcfa_index; 3771 entry[k].cookie = (unsigned long)act; 3772 entry[k].miss_cookie = 3773 tcf_exts_miss_cookie_get(miss_cookie_base, i); 3774 } 3775 3776 j += index; 3777 3778 spin_unlock_bh(&act->tcfa_lock); 3779 } 3780 3781 err_out: 3782 if (err) 3783 tc_cleanup_offload_action(flow_action); 3784 3785 return err; 3786 err_out_locked: 3787 spin_unlock_bh(&act->tcfa_lock); 3788 goto err_out; 3789 } 3790 3791 int tc_setup_offload_action(struct flow_action *flow_action, 3792 const struct tcf_exts *exts, 3793 struct netlink_ext_ack *extack) 3794 { 3795 #ifdef CONFIG_NET_CLS_ACT 3796 u32 miss_cookie_base; 3797 3798 if (!exts) 3799 return 0; 3800 3801 miss_cookie_base = exts->miss_cookie_node ? 3802 exts->miss_cookie_node->miss_cookie_base : 0; 3803 return tc_setup_action(flow_action, exts->actions, miss_cookie_base, 3804 extack); 3805 #else 3806 return 0; 3807 #endif 3808 } 3809 EXPORT_SYMBOL(tc_setup_offload_action); 3810 3811 unsigned int tcf_exts_num_actions(struct tcf_exts *exts) 3812 { 3813 unsigned int num_acts = 0; 3814 struct tc_action *act; 3815 int i; 3816 3817 tcf_exts_for_each_action(i, act, exts) { 3818 if (is_tcf_pedit(act)) 3819 num_acts += tcf_pedit_nkeys(act); 3820 else 3821 num_acts++; 3822 } 3823 return num_acts; 3824 } 3825 EXPORT_SYMBOL(tcf_exts_num_actions); 3826 3827 #ifdef CONFIG_NET_CLS_ACT 3828 static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr, 3829 u32 *p_block_index, 3830 struct netlink_ext_ack *extack) 3831 { 3832 *p_block_index = nla_get_u32(block_index_attr); 3833 if (!*p_block_index) { 3834 NL_SET_ERR_MSG(extack, "Block number may not be zero"); 3835 return -EINVAL; 3836 } 3837 3838 return 0; 3839 } 3840 3841 int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch, 3842 enum flow_block_binder_type binder_type, 3843 struct nlattr *block_index_attr, 3844 struct netlink_ext_ack *extack) 3845 { 3846 u32 block_index; 3847 int err; 3848 3849 if (!block_index_attr) 3850 return 0; 3851 3852 err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack); 3853 if (err) 3854 return err; 3855 3856 qe->info.binder_type = binder_type; 3857 qe->info.chain_head_change = tcf_chain_head_change_dflt; 3858 qe->info.chain_head_change_priv = &qe->filter_chain; 3859 qe->info.block_index = block_index; 3860 3861 return tcf_block_get_ext(&qe->block, sch, &qe->info, extack); 3862 } 3863 EXPORT_SYMBOL(tcf_qevent_init); 3864 3865 void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch) 3866 { 3867 if (qe->info.block_index) 3868 tcf_block_put_ext(qe->block, sch, &qe->info); 3869 } 3870 EXPORT_SYMBOL(tcf_qevent_destroy); 3871 3872 int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr, 3873 struct netlink_ext_ack *extack) 3874 { 3875 u32 block_index; 3876 int err; 3877 3878 if (!block_index_attr) 3879 return 0; 3880 3881 err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack); 3882 if (err) 3883 return err; 3884 3885 /* Bounce newly-configured block or change in block. */ 3886 if (block_index != qe->info.block_index) { 3887 NL_SET_ERR_MSG(extack, "Change of blocks is not supported"); 3888 return -EINVAL; 3889 } 3890 3891 return 0; 3892 } 3893 EXPORT_SYMBOL(tcf_qevent_validate_change); 3894 3895 struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb, 3896 struct sk_buff **to_free, int *ret) 3897 { 3898 struct tcf_result cl_res; 3899 struct tcf_proto *fl; 3900 3901 if (!qe->info.block_index) 3902 return skb; 3903 3904 fl = rcu_dereference_bh(qe->filter_chain); 3905 3906 switch (tcf_classify(skb, NULL, fl, &cl_res, false)) { 3907 case TC_ACT_SHOT: 3908 qdisc_qstats_drop(sch); 3909 __qdisc_drop(skb, to_free); 3910 *ret = __NET_XMIT_BYPASS; 3911 return NULL; 3912 case TC_ACT_STOLEN: 3913 case TC_ACT_QUEUED: 3914 case TC_ACT_TRAP: 3915 __qdisc_drop(skb, to_free); 3916 *ret = __NET_XMIT_STOLEN; 3917 return NULL; 3918 case TC_ACT_REDIRECT: 3919 skb_do_redirect(skb); 3920 *ret = __NET_XMIT_STOLEN; 3921 return NULL; 3922 } 3923 3924 return skb; 3925 } 3926 EXPORT_SYMBOL(tcf_qevent_handle); 3927 3928 int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe) 3929 { 3930 if (!qe->info.block_index) 3931 return 0; 3932 return nla_put_u32(skb, attr_name, qe->info.block_index); 3933 } 3934 EXPORT_SYMBOL(tcf_qevent_dump); 3935 #endif 3936 3937 static __net_init int tcf_net_init(struct net *net) 3938 { 3939 struct tcf_net *tn = net_generic(net, tcf_net_id); 3940 3941 spin_lock_init(&tn->idr_lock); 3942 idr_init(&tn->idr); 3943 return 0; 3944 } 3945 3946 static void __net_exit tcf_net_exit(struct net *net) 3947 { 3948 struct tcf_net *tn = net_generic(net, tcf_net_id); 3949 3950 idr_destroy(&tn->idr); 3951 } 3952 3953 static struct pernet_operations tcf_net_ops = { 3954 .init = tcf_net_init, 3955 .exit = tcf_net_exit, 3956 .id = &tcf_net_id, 3957 .size = sizeof(struct tcf_net), 3958 }; 3959 3960 static int __init tc_filter_init(void) 3961 { 3962 int err; 3963 3964 tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0); 3965 if (!tc_filter_wq) 3966 return -ENOMEM; 3967 3968 err = register_pernet_subsys(&tcf_net_ops); 3969 if (err) 3970 goto err_register_pernet_subsys; 3971 3972 xa_init_flags(&tcf_exts_miss_cookies_xa, XA_FLAGS_ALLOC1); 3973 3974 rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, 3975 RTNL_FLAG_DOIT_UNLOCKED); 3976 rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, 3977 RTNL_FLAG_DOIT_UNLOCKED); 3978 rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter, 3979 tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED); 3980 rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0); 3981 rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0); 3982 rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain, 3983 tc_dump_chain, 0); 3984 3985 return 0; 3986 3987 err_register_pernet_subsys: 3988 destroy_workqueue(tc_filter_wq); 3989 return err; 3990 } 3991 3992 subsys_initcall(tc_filter_init); 3993