1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <net/gen_stats.h> 16 #include <net/rtnetlink.h> 17 18 struct Qdisc_ops; 19 struct qdisc_walker; 20 struct tcf_walker; 21 struct module; 22 struct bpf_flow_keys; 23 24 typedef int tc_setup_cb_t(enum tc_setup_type type, 25 void *type_data, void *cb_priv); 26 27 typedef int tc_indr_block_bind_cb_t(struct net_device *dev, void *cb_priv, 28 enum tc_setup_type type, void *type_data); 29 30 struct qdisc_rate_table { 31 struct tc_ratespec rate; 32 u32 data[256]; 33 struct qdisc_rate_table *next; 34 int refcnt; 35 }; 36 37 enum qdisc_state_t { 38 __QDISC_STATE_SCHED, 39 __QDISC_STATE_DEACTIVATED, 40 }; 41 42 struct qdisc_size_table { 43 struct rcu_head rcu; 44 struct list_head list; 45 struct tc_sizespec szopts; 46 int refcnt; 47 u16 data[]; 48 }; 49 50 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 51 struct qdisc_skb_head { 52 struct sk_buff *head; 53 struct sk_buff *tail; 54 __u32 qlen; 55 spinlock_t lock; 56 }; 57 58 struct Qdisc { 59 int (*enqueue)(struct sk_buff *skb, 60 struct Qdisc *sch, 61 struct sk_buff **to_free); 62 struct sk_buff * (*dequeue)(struct Qdisc *sch); 63 unsigned int flags; 64 #define TCQ_F_BUILTIN 1 65 #define TCQ_F_INGRESS 2 66 #define TCQ_F_CAN_BYPASS 4 67 #define TCQ_F_MQROOT 8 68 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 69 * q->dev_queue : It can test 70 * netif_xmit_frozen_or_stopped() before 71 * dequeueing next packet. 72 * Its true for MQ/MQPRIO slaves, or non 73 * multiqueue device. 74 */ 75 #define TCQ_F_WARN_NONWC (1 << 16) 76 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 77 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 78 * qdisc_tree_decrease_qlen() should stop. 79 */ 80 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 81 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 82 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */ 83 u32 limit; 84 const struct Qdisc_ops *ops; 85 struct qdisc_size_table __rcu *stab; 86 struct hlist_node hash; 87 u32 handle; 88 u32 parent; 89 90 struct netdev_queue *dev_queue; 91 92 struct net_rate_estimator __rcu *rate_est; 93 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 94 struct gnet_stats_queue __percpu *cpu_qstats; 95 int padded; 96 refcount_t refcnt; 97 98 /* 99 * For performance sake on SMP, we put highly modified fields at the end 100 */ 101 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 102 struct qdisc_skb_head q; 103 struct gnet_stats_basic_packed bstats; 104 seqcount_t running; 105 struct gnet_stats_queue qstats; 106 unsigned long state; 107 struct Qdisc *next_sched; 108 struct sk_buff_head skb_bad_txq; 109 110 spinlock_t busylock ____cacheline_aligned_in_smp; 111 spinlock_t seqlock; 112 struct rcu_head rcu; 113 }; 114 115 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 116 { 117 if (qdisc->flags & TCQ_F_BUILTIN) 118 return; 119 refcount_inc(&qdisc->refcnt); 120 } 121 122 /* Intended to be used by unlocked users, when concurrent qdisc release is 123 * possible. 124 */ 125 126 static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc) 127 { 128 if (qdisc->flags & TCQ_F_BUILTIN) 129 return qdisc; 130 if (refcount_inc_not_zero(&qdisc->refcnt)) 131 return qdisc; 132 return NULL; 133 } 134 135 static inline bool qdisc_is_running(struct Qdisc *qdisc) 136 { 137 if (qdisc->flags & TCQ_F_NOLOCK) 138 return spin_is_locked(&qdisc->seqlock); 139 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; 140 } 141 142 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 143 { 144 if (qdisc->flags & TCQ_F_NOLOCK) { 145 if (!spin_trylock(&qdisc->seqlock)) 146 return false; 147 } else if (qdisc_is_running(qdisc)) { 148 return false; 149 } 150 /* Variant of write_seqcount_begin() telling lockdep a trylock 151 * was attempted. 152 */ 153 raw_write_seqcount_begin(&qdisc->running); 154 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_); 155 return true; 156 } 157 158 static inline void qdisc_run_end(struct Qdisc *qdisc) 159 { 160 write_seqcount_end(&qdisc->running); 161 if (qdisc->flags & TCQ_F_NOLOCK) 162 spin_unlock(&qdisc->seqlock); 163 } 164 165 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 166 { 167 return qdisc->flags & TCQ_F_ONETXQUEUE; 168 } 169 170 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 171 { 172 #ifdef CONFIG_BQL 173 /* Non-BQL migrated drivers will return 0, too. */ 174 return dql_avail(&txq->dql); 175 #else 176 return 0; 177 #endif 178 } 179 180 struct Qdisc_class_ops { 181 /* Child qdisc manipulation */ 182 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 183 int (*graft)(struct Qdisc *, unsigned long cl, 184 struct Qdisc *, struct Qdisc **, 185 struct netlink_ext_ack *extack); 186 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 187 void (*qlen_notify)(struct Qdisc *, unsigned long); 188 189 /* Class manipulation routines */ 190 unsigned long (*find)(struct Qdisc *, u32 classid); 191 int (*change)(struct Qdisc *, u32, u32, 192 struct nlattr **, unsigned long *, 193 struct netlink_ext_ack *); 194 int (*delete)(struct Qdisc *, unsigned long); 195 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 196 197 /* Filter manipulation */ 198 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 199 unsigned long arg, 200 struct netlink_ext_ack *extack); 201 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 202 u32 classid); 203 void (*unbind_tcf)(struct Qdisc *, unsigned long); 204 205 /* rtnetlink specific */ 206 int (*dump)(struct Qdisc *, unsigned long, 207 struct sk_buff *skb, struct tcmsg*); 208 int (*dump_stats)(struct Qdisc *, unsigned long, 209 struct gnet_dump *); 210 }; 211 212 struct Qdisc_ops { 213 struct Qdisc_ops *next; 214 const struct Qdisc_class_ops *cl_ops; 215 char id[IFNAMSIZ]; 216 int priv_size; 217 unsigned int static_flags; 218 219 int (*enqueue)(struct sk_buff *skb, 220 struct Qdisc *sch, 221 struct sk_buff **to_free); 222 struct sk_buff * (*dequeue)(struct Qdisc *); 223 struct sk_buff * (*peek)(struct Qdisc *); 224 225 int (*init)(struct Qdisc *sch, struct nlattr *arg, 226 struct netlink_ext_ack *extack); 227 void (*reset)(struct Qdisc *); 228 void (*destroy)(struct Qdisc *); 229 int (*change)(struct Qdisc *sch, 230 struct nlattr *arg, 231 struct netlink_ext_ack *extack); 232 void (*attach)(struct Qdisc *sch); 233 int (*change_tx_queue_len)(struct Qdisc *, unsigned int); 234 235 int (*dump)(struct Qdisc *, struct sk_buff *); 236 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 237 238 void (*ingress_block_set)(struct Qdisc *sch, 239 u32 block_index); 240 void (*egress_block_set)(struct Qdisc *sch, 241 u32 block_index); 242 u32 (*ingress_block_get)(struct Qdisc *sch); 243 u32 (*egress_block_get)(struct Qdisc *sch); 244 245 struct module *owner; 246 }; 247 248 249 struct tcf_result { 250 union { 251 struct { 252 unsigned long class; 253 u32 classid; 254 }; 255 const struct tcf_proto *goto_tp; 256 257 /* used by the TC_ACT_REINSERT action */ 258 struct { 259 bool ingress; 260 struct gnet_stats_queue *qstats; 261 }; 262 }; 263 }; 264 265 struct tcf_chain; 266 267 struct tcf_proto_ops { 268 struct list_head head; 269 char kind[IFNAMSIZ]; 270 271 int (*classify)(struct sk_buff *, 272 const struct tcf_proto *, 273 struct tcf_result *); 274 int (*init)(struct tcf_proto*); 275 void (*destroy)(struct tcf_proto *tp, 276 struct netlink_ext_ack *extack); 277 278 void* (*get)(struct tcf_proto*, u32 handle); 279 int (*change)(struct net *net, struct sk_buff *, 280 struct tcf_proto*, unsigned long, 281 u32 handle, struct nlattr **, 282 void **, bool, 283 struct netlink_ext_ack *); 284 int (*delete)(struct tcf_proto *tp, void *arg, 285 bool *last, 286 struct netlink_ext_ack *); 287 void (*walk)(struct tcf_proto*, struct tcf_walker *arg); 288 int (*reoffload)(struct tcf_proto *tp, bool add, 289 tc_setup_cb_t *cb, void *cb_priv, 290 struct netlink_ext_ack *extack); 291 void (*bind_class)(void *, u32, unsigned long); 292 void * (*tmplt_create)(struct net *net, 293 struct tcf_chain *chain, 294 struct nlattr **tca, 295 struct netlink_ext_ack *extack); 296 void (*tmplt_destroy)(void *tmplt_priv); 297 298 /* rtnetlink specific */ 299 int (*dump)(struct net*, struct tcf_proto*, void *, 300 struct sk_buff *skb, struct tcmsg*); 301 int (*tmplt_dump)(struct sk_buff *skb, 302 struct net *net, 303 void *tmplt_priv); 304 305 struct module *owner; 306 }; 307 308 struct tcf_proto { 309 /* Fast access part */ 310 struct tcf_proto __rcu *next; 311 void __rcu *root; 312 313 /* called under RCU BH lock*/ 314 int (*classify)(struct sk_buff *, 315 const struct tcf_proto *, 316 struct tcf_result *); 317 __be16 protocol; 318 319 /* All the rest */ 320 u32 prio; 321 void *data; 322 const struct tcf_proto_ops *ops; 323 struct tcf_chain *chain; 324 struct rcu_head rcu; 325 }; 326 327 struct qdisc_skb_cb { 328 union { 329 struct { 330 unsigned int pkt_len; 331 u16 slave_dev_queue_mapping; 332 u16 tc_classid; 333 }; 334 struct bpf_flow_keys *flow_keys; 335 }; 336 #define QDISC_CB_PRIV_LEN 20 337 unsigned char data[QDISC_CB_PRIV_LEN]; 338 }; 339 340 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 341 342 struct tcf_chain { 343 struct tcf_proto __rcu *filter_chain; 344 struct list_head list; 345 struct tcf_block *block; 346 u32 index; /* chain index */ 347 unsigned int refcnt; 348 unsigned int action_refcnt; 349 bool explicitly_created; 350 const struct tcf_proto_ops *tmplt_ops; 351 void *tmplt_priv; 352 }; 353 354 struct tcf_block { 355 struct list_head chain_list; 356 u32 index; /* block index for shared blocks */ 357 refcount_t refcnt; 358 struct net *net; 359 struct Qdisc *q; 360 struct list_head cb_list; 361 struct list_head owner_list; 362 bool keep_dst; 363 unsigned int offloadcnt; /* Number of oddloaded filters */ 364 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ 365 struct { 366 struct tcf_chain *chain; 367 struct list_head filter_chain_list; 368 } chain0; 369 struct rcu_head rcu; 370 }; 371 372 static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) 373 { 374 if (*flags & TCA_CLS_FLAGS_IN_HW) 375 return; 376 *flags |= TCA_CLS_FLAGS_IN_HW; 377 block->offloadcnt++; 378 } 379 380 static inline void tcf_block_offload_dec(struct tcf_block *block, u32 *flags) 381 { 382 if (!(*flags & TCA_CLS_FLAGS_IN_HW)) 383 return; 384 *flags &= ~TCA_CLS_FLAGS_IN_HW; 385 block->offloadcnt--; 386 } 387 388 static inline void 389 tc_cls_offload_cnt_update(struct tcf_block *block, u32 *cnt, 390 u32 *flags, bool add) 391 { 392 if (add) { 393 if (!*cnt) 394 tcf_block_offload_inc(block, flags); 395 (*cnt)++; 396 } else { 397 (*cnt)--; 398 if (!*cnt) 399 tcf_block_offload_dec(block, flags); 400 } 401 } 402 403 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 404 { 405 struct qdisc_skb_cb *qcb; 406 407 BUILD_BUG_ON(sizeof(skb->cb) < offsetof(struct qdisc_skb_cb, data) + sz); 408 BUILD_BUG_ON(sizeof(qcb->data) < sz); 409 } 410 411 static inline int qdisc_qlen_cpu(const struct Qdisc *q) 412 { 413 return this_cpu_ptr(q->cpu_qstats)->qlen; 414 } 415 416 static inline int qdisc_qlen(const struct Qdisc *q) 417 { 418 return q->q.qlen; 419 } 420 421 static inline int qdisc_qlen_sum(const struct Qdisc *q) 422 { 423 __u32 qlen = q->qstats.qlen; 424 int i; 425 426 if (q->flags & TCQ_F_NOLOCK) { 427 for_each_possible_cpu(i) 428 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 429 } else { 430 qlen += q->q.qlen; 431 } 432 433 return qlen; 434 } 435 436 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 437 { 438 return (struct qdisc_skb_cb *)skb->cb; 439 } 440 441 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 442 { 443 return &qdisc->q.lock; 444 } 445 446 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 447 { 448 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 449 450 return q; 451 } 452 453 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 454 { 455 return qdisc->dev_queue->qdisc_sleeping; 456 } 457 458 /* The qdisc root lock is a mechanism by which to top level 459 * of a qdisc tree can be locked from any qdisc node in the 460 * forest. This allows changing the configuration of some 461 * aspect of the qdisc tree while blocking out asynchronous 462 * qdisc access in the packet processing paths. 463 * 464 * It is only legal to do this when the root will not change 465 * on us. Otherwise we'll potentially lock the wrong qdisc 466 * root. This is enforced by holding the RTNL semaphore, which 467 * all users of this lock accessor must do. 468 */ 469 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) 470 { 471 struct Qdisc *root = qdisc_root(qdisc); 472 473 ASSERT_RTNL(); 474 return qdisc_lock(root); 475 } 476 477 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 478 { 479 struct Qdisc *root = qdisc_root_sleeping(qdisc); 480 481 ASSERT_RTNL(); 482 return qdisc_lock(root); 483 } 484 485 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc) 486 { 487 struct Qdisc *root = qdisc_root_sleeping(qdisc); 488 489 ASSERT_RTNL(); 490 return &root->running; 491 } 492 493 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 494 { 495 return qdisc->dev_queue->dev; 496 } 497 498 static inline void sch_tree_lock(const struct Qdisc *q) 499 { 500 spin_lock_bh(qdisc_root_sleeping_lock(q)); 501 } 502 503 static inline void sch_tree_unlock(const struct Qdisc *q) 504 { 505 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 506 } 507 508 extern struct Qdisc noop_qdisc; 509 extern struct Qdisc_ops noop_qdisc_ops; 510 extern struct Qdisc_ops pfifo_fast_ops; 511 extern struct Qdisc_ops mq_qdisc_ops; 512 extern struct Qdisc_ops noqueue_qdisc_ops; 513 extern const struct Qdisc_ops *default_qdisc_ops; 514 static inline const struct Qdisc_ops * 515 get_default_qdisc_ops(const struct net_device *dev, int ntx) 516 { 517 return ntx < dev->real_num_tx_queues ? 518 default_qdisc_ops : &pfifo_fast_ops; 519 } 520 521 struct Qdisc_class_common { 522 u32 classid; 523 struct hlist_node hnode; 524 }; 525 526 struct Qdisc_class_hash { 527 struct hlist_head *hash; 528 unsigned int hashsize; 529 unsigned int hashmask; 530 unsigned int hashelems; 531 }; 532 533 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 534 { 535 id ^= id >> 8; 536 id ^= id >> 4; 537 return id & mask; 538 } 539 540 static inline struct Qdisc_class_common * 541 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 542 { 543 struct Qdisc_class_common *cl; 544 unsigned int h; 545 546 if (!id) 547 return NULL; 548 549 h = qdisc_class_hash(id, hash->hashmask); 550 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 551 if (cl->classid == id) 552 return cl; 553 } 554 return NULL; 555 } 556 557 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 558 { 559 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 560 561 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 562 } 563 564 int qdisc_class_hash_init(struct Qdisc_class_hash *); 565 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 566 struct Qdisc_class_common *); 567 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 568 struct Qdisc_class_common *); 569 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 570 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 571 572 int dev_qdisc_change_tx_queue_len(struct net_device *dev); 573 void dev_init_scheduler(struct net_device *dev); 574 void dev_shutdown(struct net_device *dev); 575 void dev_activate(struct net_device *dev); 576 void dev_deactivate(struct net_device *dev); 577 void dev_deactivate_many(struct list_head *head); 578 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 579 struct Qdisc *qdisc); 580 void qdisc_reset(struct Qdisc *qdisc); 581 void qdisc_put(struct Qdisc *qdisc); 582 void qdisc_put_unlocked(struct Qdisc *qdisc); 583 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n, 584 unsigned int len); 585 #ifdef CONFIG_NET_SCHED 586 int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 587 void *type_data); 588 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 589 struct Qdisc *new, struct Qdisc *old, 590 enum tc_setup_type type, void *type_data, 591 struct netlink_ext_ack *extack); 592 #else 593 static inline int 594 qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 595 void *type_data) 596 { 597 q->flags &= ~TCQ_F_OFFLOADED; 598 return 0; 599 } 600 601 static inline void 602 qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 603 struct Qdisc *new, struct Qdisc *old, 604 enum tc_setup_type type, void *type_data, 605 struct netlink_ext_ack *extack) 606 { 607 } 608 #endif 609 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 610 const struct Qdisc_ops *ops, 611 struct netlink_ext_ack *extack); 612 void qdisc_free(struct Qdisc *qdisc); 613 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 614 const struct Qdisc_ops *ops, u32 parentid, 615 struct netlink_ext_ack *extack); 616 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 617 const struct qdisc_size_table *stab); 618 int skb_do_redirect(struct sk_buff *); 619 620 static inline void skb_reset_tc(struct sk_buff *skb) 621 { 622 #ifdef CONFIG_NET_CLS_ACT 623 skb->tc_redirected = 0; 624 #endif 625 } 626 627 static inline bool skb_is_tc_redirected(const struct sk_buff *skb) 628 { 629 #ifdef CONFIG_NET_CLS_ACT 630 return skb->tc_redirected; 631 #else 632 return false; 633 #endif 634 } 635 636 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 637 { 638 #ifdef CONFIG_NET_CLS_ACT 639 return skb->tc_at_ingress; 640 #else 641 return false; 642 #endif 643 } 644 645 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 646 { 647 #ifdef CONFIG_NET_CLS_ACT 648 if (skb->tc_skip_classify) { 649 skb->tc_skip_classify = 0; 650 return true; 651 } 652 #endif 653 return false; 654 } 655 656 /* Reset all TX qdiscs greater than index of a device. */ 657 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 658 { 659 struct Qdisc *qdisc; 660 661 for (; i < dev->num_tx_queues; i++) { 662 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 663 if (qdisc) { 664 spin_lock_bh(qdisc_lock(qdisc)); 665 qdisc_reset(qdisc); 666 spin_unlock_bh(qdisc_lock(qdisc)); 667 } 668 } 669 } 670 671 static inline void qdisc_reset_all_tx(struct net_device *dev) 672 { 673 qdisc_reset_all_tx_gt(dev, 0); 674 } 675 676 /* Are all TX queues of the device empty? */ 677 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 678 { 679 unsigned int i; 680 681 rcu_read_lock(); 682 for (i = 0; i < dev->num_tx_queues; i++) { 683 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 684 const struct Qdisc *q = rcu_dereference(txq->qdisc); 685 686 if (q->q.qlen) { 687 rcu_read_unlock(); 688 return false; 689 } 690 } 691 rcu_read_unlock(); 692 return true; 693 } 694 695 /* Are any of the TX qdiscs changing? */ 696 static inline bool qdisc_tx_changing(const struct net_device *dev) 697 { 698 unsigned int i; 699 700 for (i = 0; i < dev->num_tx_queues; i++) { 701 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 702 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping) 703 return true; 704 } 705 return false; 706 } 707 708 /* Is the device using the noop qdisc on all queues? */ 709 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 710 { 711 unsigned int i; 712 713 for (i = 0; i < dev->num_tx_queues; i++) { 714 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 715 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 716 return false; 717 } 718 return true; 719 } 720 721 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 722 { 723 return qdisc_skb_cb(skb)->pkt_len; 724 } 725 726 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 727 enum net_xmit_qdisc_t { 728 __NET_XMIT_STOLEN = 0x00010000, 729 __NET_XMIT_BYPASS = 0x00020000, 730 }; 731 732 #ifdef CONFIG_NET_CLS_ACT 733 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 734 #else 735 #define net_xmit_drop_count(e) (1) 736 #endif 737 738 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 739 const struct Qdisc *sch) 740 { 741 #ifdef CONFIG_NET_SCHED 742 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 743 744 if (stab) 745 __qdisc_calculate_pkt_len(skb, stab); 746 #endif 747 } 748 749 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 750 struct sk_buff **to_free) 751 { 752 qdisc_calculate_pkt_len(skb, sch); 753 return sch->enqueue(skb, sch, to_free); 754 } 755 756 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 757 { 758 return q->flags & TCQ_F_CPUSTATS; 759 } 760 761 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats, 762 __u64 bytes, __u32 packets) 763 { 764 bstats->bytes += bytes; 765 bstats->packets += packets; 766 } 767 768 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 769 const struct sk_buff *skb) 770 { 771 _bstats_update(bstats, 772 qdisc_pkt_len(skb), 773 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 774 } 775 776 static inline void _bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 777 __u64 bytes, __u32 packets) 778 { 779 u64_stats_update_begin(&bstats->syncp); 780 _bstats_update(&bstats->bstats, bytes, packets); 781 u64_stats_update_end(&bstats->syncp); 782 } 783 784 static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 785 const struct sk_buff *skb) 786 { 787 u64_stats_update_begin(&bstats->syncp); 788 bstats_update(&bstats->bstats, skb); 789 u64_stats_update_end(&bstats->syncp); 790 } 791 792 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 793 const struct sk_buff *skb) 794 { 795 bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb); 796 } 797 798 static inline void qdisc_bstats_update(struct Qdisc *sch, 799 const struct sk_buff *skb) 800 { 801 bstats_update(&sch->bstats, skb); 802 } 803 804 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 805 const struct sk_buff *skb) 806 { 807 sch->qstats.backlog -= qdisc_pkt_len(skb); 808 } 809 810 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 811 const struct sk_buff *skb) 812 { 813 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 814 } 815 816 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 817 const struct sk_buff *skb) 818 { 819 sch->qstats.backlog += qdisc_pkt_len(skb); 820 } 821 822 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 823 const struct sk_buff *skb) 824 { 825 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 826 } 827 828 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 829 { 830 this_cpu_inc(sch->cpu_qstats->qlen); 831 } 832 833 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 834 { 835 this_cpu_dec(sch->cpu_qstats->qlen); 836 } 837 838 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 839 { 840 this_cpu_inc(sch->cpu_qstats->requeues); 841 } 842 843 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 844 { 845 sch->qstats.drops += count; 846 } 847 848 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 849 { 850 qstats->drops++; 851 } 852 853 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 854 { 855 qstats->overlimits++; 856 } 857 858 static inline void qdisc_qstats_drop(struct Qdisc *sch) 859 { 860 qstats_drop_inc(&sch->qstats); 861 } 862 863 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 864 { 865 this_cpu_inc(sch->cpu_qstats->drops); 866 } 867 868 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 869 { 870 sch->qstats.overlimits++; 871 } 872 873 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) 874 { 875 qh->head = NULL; 876 qh->tail = NULL; 877 qh->qlen = 0; 878 } 879 880 static inline void __qdisc_enqueue_tail(struct sk_buff *skb, 881 struct qdisc_skb_head *qh) 882 { 883 struct sk_buff *last = qh->tail; 884 885 if (last) { 886 skb->next = NULL; 887 last->next = skb; 888 qh->tail = skb; 889 } else { 890 qh->tail = skb; 891 qh->head = skb; 892 } 893 qh->qlen++; 894 } 895 896 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 897 { 898 __qdisc_enqueue_tail(skb, &sch->q); 899 qdisc_qstats_backlog_inc(sch, skb); 900 return NET_XMIT_SUCCESS; 901 } 902 903 static inline void __qdisc_enqueue_head(struct sk_buff *skb, 904 struct qdisc_skb_head *qh) 905 { 906 skb->next = qh->head; 907 908 if (!qh->head) 909 qh->tail = skb; 910 qh->head = skb; 911 qh->qlen++; 912 } 913 914 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 915 { 916 struct sk_buff *skb = qh->head; 917 918 if (likely(skb != NULL)) { 919 qh->head = skb->next; 920 qh->qlen--; 921 if (qh->head == NULL) 922 qh->tail = NULL; 923 skb->next = NULL; 924 } 925 926 return skb; 927 } 928 929 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 930 { 931 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 932 933 if (likely(skb != NULL)) { 934 qdisc_qstats_backlog_dec(sch, skb); 935 qdisc_bstats_update(sch, skb); 936 } 937 938 return skb; 939 } 940 941 /* Instead of calling kfree_skb() while root qdisc lock is held, 942 * queue the skb for future freeing at end of __dev_xmit_skb() 943 */ 944 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 945 { 946 skb->next = *to_free; 947 *to_free = skb; 948 } 949 950 static inline void __qdisc_drop_all(struct sk_buff *skb, 951 struct sk_buff **to_free) 952 { 953 if (skb->prev) 954 skb->prev->next = *to_free; 955 else 956 skb->next = *to_free; 957 *to_free = skb; 958 } 959 960 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 961 struct qdisc_skb_head *qh, 962 struct sk_buff **to_free) 963 { 964 struct sk_buff *skb = __qdisc_dequeue_head(qh); 965 966 if (likely(skb != NULL)) { 967 unsigned int len = qdisc_pkt_len(skb); 968 969 qdisc_qstats_backlog_dec(sch, skb); 970 __qdisc_drop(skb, to_free); 971 return len; 972 } 973 974 return 0; 975 } 976 977 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch, 978 struct sk_buff **to_free) 979 { 980 return __qdisc_queue_drop_head(sch, &sch->q, to_free); 981 } 982 983 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 984 { 985 const struct qdisc_skb_head *qh = &sch->q; 986 987 return qh->head; 988 } 989 990 /* generic pseudo peek method for non-work-conserving qdisc */ 991 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 992 { 993 struct sk_buff *skb = skb_peek(&sch->gso_skb); 994 995 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 996 if (!skb) { 997 skb = sch->dequeue(sch); 998 999 if (skb) { 1000 __skb_queue_head(&sch->gso_skb, skb); 1001 /* it's still part of the queue */ 1002 qdisc_qstats_backlog_inc(sch, skb); 1003 sch->q.qlen++; 1004 } 1005 } 1006 1007 return skb; 1008 } 1009 1010 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 1011 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 1012 { 1013 struct sk_buff *skb = skb_peek(&sch->gso_skb); 1014 1015 if (skb) { 1016 skb = __skb_dequeue(&sch->gso_skb); 1017 qdisc_qstats_backlog_dec(sch, skb); 1018 sch->q.qlen--; 1019 } else { 1020 skb = sch->dequeue(sch); 1021 } 1022 1023 return skb; 1024 } 1025 1026 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 1027 { 1028 /* 1029 * We do not know the backlog in bytes of this list, it 1030 * is up to the caller to correct it 1031 */ 1032 ASSERT_RTNL(); 1033 if (qh->qlen) { 1034 rtnl_kfree_skbs(qh->head, qh->tail); 1035 1036 qh->head = NULL; 1037 qh->tail = NULL; 1038 qh->qlen = 0; 1039 } 1040 } 1041 1042 static inline void qdisc_reset_queue(struct Qdisc *sch) 1043 { 1044 __qdisc_reset_queue(&sch->q); 1045 sch->qstats.backlog = 0; 1046 } 1047 1048 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 1049 struct Qdisc **pold) 1050 { 1051 struct Qdisc *old; 1052 1053 sch_tree_lock(sch); 1054 old = *pold; 1055 *pold = new; 1056 if (old != NULL) { 1057 unsigned int qlen = old->q.qlen; 1058 unsigned int backlog = old->qstats.backlog; 1059 1060 qdisc_reset(old); 1061 qdisc_tree_reduce_backlog(old, qlen, backlog); 1062 } 1063 sch_tree_unlock(sch); 1064 1065 return old; 1066 } 1067 1068 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 1069 { 1070 rtnl_kfree_skbs(skb, skb); 1071 qdisc_qstats_drop(sch); 1072 } 1073 1074 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 1075 struct sk_buff **to_free) 1076 { 1077 __qdisc_drop(skb, to_free); 1078 qdisc_qstats_cpu_drop(sch); 1079 1080 return NET_XMIT_DROP; 1081 } 1082 1083 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 1084 struct sk_buff **to_free) 1085 { 1086 __qdisc_drop(skb, to_free); 1087 qdisc_qstats_drop(sch); 1088 1089 return NET_XMIT_DROP; 1090 } 1091 1092 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, 1093 struct sk_buff **to_free) 1094 { 1095 __qdisc_drop_all(skb, to_free); 1096 qdisc_qstats_drop(sch); 1097 1098 return NET_XMIT_DROP; 1099 } 1100 1101 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 1102 long it will take to send a packet given its size. 1103 */ 1104 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 1105 { 1106 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 1107 if (slot < 0) 1108 slot = 0; 1109 slot >>= rtab->rate.cell_log; 1110 if (slot > 255) 1111 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 1112 return rtab->data[slot]; 1113 } 1114 1115 struct psched_ratecfg { 1116 u64 rate_bytes_ps; /* bytes per second */ 1117 u32 mult; 1118 u16 overhead; 1119 u8 linklayer; 1120 u8 shift; 1121 }; 1122 1123 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 1124 unsigned int len) 1125 { 1126 len += r->overhead; 1127 1128 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 1129 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 1130 1131 return ((u64)len * r->mult) >> r->shift; 1132 } 1133 1134 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1135 const struct tc_ratespec *conf, 1136 u64 rate64); 1137 1138 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 1139 const struct psched_ratecfg *r) 1140 { 1141 memset(res, 0, sizeof(*res)); 1142 1143 /* legacy struct tc_ratespec has a 32bit @rate field 1144 * Qdisc using 64bit rate should add new attributes 1145 * in order to maintain compatibility. 1146 */ 1147 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 1148 1149 res->overhead = r->overhead; 1150 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 1151 } 1152 1153 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 1154 * The fast path only needs to access filter list and to update stats 1155 */ 1156 struct mini_Qdisc { 1157 struct tcf_proto *filter_list; 1158 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 1159 struct gnet_stats_queue __percpu *cpu_qstats; 1160 struct rcu_head rcu; 1161 }; 1162 1163 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 1164 const struct sk_buff *skb) 1165 { 1166 bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb); 1167 } 1168 1169 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 1170 { 1171 this_cpu_inc(miniq->cpu_qstats->drops); 1172 } 1173 1174 struct mini_Qdisc_pair { 1175 struct mini_Qdisc miniq1; 1176 struct mini_Qdisc miniq2; 1177 struct mini_Qdisc __rcu **p_miniq; 1178 }; 1179 1180 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1181 struct tcf_proto *tp_head); 1182 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1183 struct mini_Qdisc __rcu **p_miniq); 1184 1185 static inline void skb_tc_reinsert(struct sk_buff *skb, struct tcf_result *res) 1186 { 1187 struct gnet_stats_queue *stats = res->qstats; 1188 int ret; 1189 1190 if (res->ingress) 1191 ret = netif_receive_skb(skb); 1192 else 1193 ret = dev_queue_xmit(skb); 1194 if (ret && stats) 1195 qstats_overlimit_inc(res->qstats); 1196 } 1197 1198 #endif 1199