1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <net/gen_stats.h> 16 #include <net/rtnetlink.h> 17 18 struct Qdisc_ops; 19 struct qdisc_walker; 20 struct tcf_walker; 21 struct module; 22 23 struct qdisc_rate_table { 24 struct tc_ratespec rate; 25 u32 data[256]; 26 struct qdisc_rate_table *next; 27 int refcnt; 28 }; 29 30 enum qdisc_state_t { 31 __QDISC_STATE_SCHED, 32 __QDISC_STATE_DEACTIVATED, 33 }; 34 35 struct qdisc_size_table { 36 struct rcu_head rcu; 37 struct list_head list; 38 struct tc_sizespec szopts; 39 int refcnt; 40 u16 data[]; 41 }; 42 43 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 44 struct qdisc_skb_head { 45 struct sk_buff *head; 46 struct sk_buff *tail; 47 __u32 qlen; 48 spinlock_t lock; 49 }; 50 51 struct Qdisc { 52 int (*enqueue)(struct sk_buff *skb, 53 struct Qdisc *sch, 54 struct sk_buff **to_free); 55 struct sk_buff * (*dequeue)(struct Qdisc *sch); 56 unsigned int flags; 57 #define TCQ_F_BUILTIN 1 58 #define TCQ_F_INGRESS 2 59 #define TCQ_F_CAN_BYPASS 4 60 #define TCQ_F_MQROOT 8 61 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 62 * q->dev_queue : It can test 63 * netif_xmit_frozen_or_stopped() before 64 * dequeueing next packet. 65 * Its true for MQ/MQPRIO slaves, or non 66 * multiqueue device. 67 */ 68 #define TCQ_F_WARN_NONWC (1 << 16) 69 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 70 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 71 * qdisc_tree_decrease_qlen() should stop. 72 */ 73 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 74 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 75 u32 limit; 76 const struct Qdisc_ops *ops; 77 struct qdisc_size_table __rcu *stab; 78 struct hlist_node hash; 79 u32 handle; 80 u32 parent; 81 82 struct netdev_queue *dev_queue; 83 84 struct net_rate_estimator __rcu *rate_est; 85 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 86 struct gnet_stats_queue __percpu *cpu_qstats; 87 88 /* 89 * For performance sake on SMP, we put highly modified fields at the end 90 */ 91 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 92 struct qdisc_skb_head q; 93 struct gnet_stats_basic_packed bstats; 94 seqcount_t running; 95 struct gnet_stats_queue qstats; 96 unsigned long state; 97 struct Qdisc *next_sched; 98 struct sk_buff_head skb_bad_txq; 99 int padded; 100 refcount_t refcnt; 101 102 spinlock_t busylock ____cacheline_aligned_in_smp; 103 }; 104 105 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 106 { 107 if (qdisc->flags & TCQ_F_BUILTIN) 108 return; 109 refcount_inc(&qdisc->refcnt); 110 } 111 112 static inline bool qdisc_is_running(const struct Qdisc *qdisc) 113 { 114 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; 115 } 116 117 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 118 { 119 if (qdisc_is_running(qdisc)) 120 return false; 121 /* Variant of write_seqcount_begin() telling lockdep a trylock 122 * was attempted. 123 */ 124 raw_write_seqcount_begin(&qdisc->running); 125 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_); 126 return true; 127 } 128 129 static inline void qdisc_run_end(struct Qdisc *qdisc) 130 { 131 write_seqcount_end(&qdisc->running); 132 } 133 134 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 135 { 136 return qdisc->flags & TCQ_F_ONETXQUEUE; 137 } 138 139 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 140 { 141 #ifdef CONFIG_BQL 142 /* Non-BQL migrated drivers will return 0, too. */ 143 return dql_avail(&txq->dql); 144 #else 145 return 0; 146 #endif 147 } 148 149 struct Qdisc_class_ops { 150 /* Child qdisc manipulation */ 151 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 152 int (*graft)(struct Qdisc *, unsigned long cl, 153 struct Qdisc *, struct Qdisc **); 154 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 155 void (*qlen_notify)(struct Qdisc *, unsigned long); 156 157 /* Class manipulation routines */ 158 unsigned long (*find)(struct Qdisc *, u32 classid); 159 int (*change)(struct Qdisc *, u32, u32, 160 struct nlattr **, unsigned long *); 161 int (*delete)(struct Qdisc *, unsigned long); 162 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 163 164 /* Filter manipulation */ 165 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 166 unsigned long arg); 167 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 168 u32 classid); 169 void (*unbind_tcf)(struct Qdisc *, unsigned long); 170 171 /* rtnetlink specific */ 172 int (*dump)(struct Qdisc *, unsigned long, 173 struct sk_buff *skb, struct tcmsg*); 174 int (*dump_stats)(struct Qdisc *, unsigned long, 175 struct gnet_dump *); 176 }; 177 178 struct Qdisc_ops { 179 struct Qdisc_ops *next; 180 const struct Qdisc_class_ops *cl_ops; 181 char id[IFNAMSIZ]; 182 int priv_size; 183 unsigned int static_flags; 184 185 int (*enqueue)(struct sk_buff *skb, 186 struct Qdisc *sch, 187 struct sk_buff **to_free); 188 struct sk_buff * (*dequeue)(struct Qdisc *); 189 struct sk_buff * (*peek)(struct Qdisc *); 190 191 int (*init)(struct Qdisc *sch, struct nlattr *arg); 192 void (*reset)(struct Qdisc *); 193 void (*destroy)(struct Qdisc *); 194 int (*change)(struct Qdisc *sch, 195 struct nlattr *arg); 196 void (*attach)(struct Qdisc *sch); 197 198 int (*dump)(struct Qdisc *, struct sk_buff *); 199 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 200 201 struct module *owner; 202 }; 203 204 205 struct tcf_result { 206 union { 207 struct { 208 unsigned long class; 209 u32 classid; 210 }; 211 const struct tcf_proto *goto_tp; 212 }; 213 }; 214 215 struct tcf_proto_ops { 216 struct list_head head; 217 char kind[IFNAMSIZ]; 218 219 int (*classify)(struct sk_buff *, 220 const struct tcf_proto *, 221 struct tcf_result *); 222 int (*init)(struct tcf_proto*); 223 void (*destroy)(struct tcf_proto*); 224 225 void* (*get)(struct tcf_proto*, u32 handle); 226 int (*change)(struct net *net, struct sk_buff *, 227 struct tcf_proto*, unsigned long, 228 u32 handle, struct nlattr **, 229 void **, bool); 230 int (*delete)(struct tcf_proto*, void *, bool*); 231 void (*walk)(struct tcf_proto*, struct tcf_walker *arg); 232 void (*bind_class)(void *, u32, unsigned long); 233 234 /* rtnetlink specific */ 235 int (*dump)(struct net*, struct tcf_proto*, void *, 236 struct sk_buff *skb, struct tcmsg*); 237 238 struct module *owner; 239 }; 240 241 struct tcf_proto { 242 /* Fast access part */ 243 struct tcf_proto __rcu *next; 244 void __rcu *root; 245 int (*classify)(struct sk_buff *, 246 const struct tcf_proto *, 247 struct tcf_result *); 248 __be16 protocol; 249 250 /* All the rest */ 251 u32 prio; 252 u32 classid; 253 struct Qdisc *q; 254 void *data; 255 const struct tcf_proto_ops *ops; 256 struct tcf_chain *chain; 257 struct rcu_head rcu; 258 }; 259 260 struct qdisc_skb_cb { 261 unsigned int pkt_len; 262 u16 slave_dev_queue_mapping; 263 u16 tc_classid; 264 #define QDISC_CB_PRIV_LEN 20 265 unsigned char data[QDISC_CB_PRIV_LEN]; 266 }; 267 268 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 269 270 struct tcf_chain { 271 struct tcf_proto __rcu *filter_chain; 272 tcf_chain_head_change_t *chain_head_change; 273 void *chain_head_change_priv; 274 struct list_head list; 275 struct tcf_block *block; 276 u32 index; /* chain index */ 277 unsigned int refcnt; 278 }; 279 280 struct tcf_block { 281 struct list_head chain_list; 282 struct net *net; 283 struct Qdisc *q; 284 struct list_head cb_list; 285 }; 286 287 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 288 { 289 struct qdisc_skb_cb *qcb; 290 291 BUILD_BUG_ON(sizeof(skb->cb) < offsetof(struct qdisc_skb_cb, data) + sz); 292 BUILD_BUG_ON(sizeof(qcb->data) < sz); 293 } 294 295 static inline int qdisc_qlen_cpu(const struct Qdisc *q) 296 { 297 return this_cpu_ptr(q->cpu_qstats)->qlen; 298 } 299 300 static inline int qdisc_qlen(const struct Qdisc *q) 301 { 302 return q->q.qlen; 303 } 304 305 static inline int qdisc_qlen_sum(const struct Qdisc *q) 306 { 307 __u32 qlen = 0; 308 int i; 309 310 if (q->flags & TCQ_F_NOLOCK) { 311 for_each_possible_cpu(i) 312 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 313 } else { 314 qlen = q->q.qlen; 315 } 316 317 return qlen; 318 } 319 320 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 321 { 322 return (struct qdisc_skb_cb *)skb->cb; 323 } 324 325 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 326 { 327 return &qdisc->q.lock; 328 } 329 330 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 331 { 332 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 333 334 return q; 335 } 336 337 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 338 { 339 return qdisc->dev_queue->qdisc_sleeping; 340 } 341 342 /* The qdisc root lock is a mechanism by which to top level 343 * of a qdisc tree can be locked from any qdisc node in the 344 * forest. This allows changing the configuration of some 345 * aspect of the qdisc tree while blocking out asynchronous 346 * qdisc access in the packet processing paths. 347 * 348 * It is only legal to do this when the root will not change 349 * on us. Otherwise we'll potentially lock the wrong qdisc 350 * root. This is enforced by holding the RTNL semaphore, which 351 * all users of this lock accessor must do. 352 */ 353 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) 354 { 355 struct Qdisc *root = qdisc_root(qdisc); 356 357 ASSERT_RTNL(); 358 return qdisc_lock(root); 359 } 360 361 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 362 { 363 struct Qdisc *root = qdisc_root_sleeping(qdisc); 364 365 ASSERT_RTNL(); 366 return qdisc_lock(root); 367 } 368 369 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc) 370 { 371 struct Qdisc *root = qdisc_root_sleeping(qdisc); 372 373 ASSERT_RTNL(); 374 return &root->running; 375 } 376 377 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 378 { 379 return qdisc->dev_queue->dev; 380 } 381 382 static inline void sch_tree_lock(const struct Qdisc *q) 383 { 384 spin_lock_bh(qdisc_root_sleeping_lock(q)); 385 } 386 387 static inline void sch_tree_unlock(const struct Qdisc *q) 388 { 389 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 390 } 391 392 extern struct Qdisc noop_qdisc; 393 extern struct Qdisc_ops noop_qdisc_ops; 394 extern struct Qdisc_ops pfifo_fast_ops; 395 extern struct Qdisc_ops mq_qdisc_ops; 396 extern struct Qdisc_ops noqueue_qdisc_ops; 397 extern const struct Qdisc_ops *default_qdisc_ops; 398 static inline const struct Qdisc_ops * 399 get_default_qdisc_ops(const struct net_device *dev, int ntx) 400 { 401 return ntx < dev->real_num_tx_queues ? 402 default_qdisc_ops : &pfifo_fast_ops; 403 } 404 405 struct Qdisc_class_common { 406 u32 classid; 407 struct hlist_node hnode; 408 }; 409 410 struct Qdisc_class_hash { 411 struct hlist_head *hash; 412 unsigned int hashsize; 413 unsigned int hashmask; 414 unsigned int hashelems; 415 }; 416 417 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 418 { 419 id ^= id >> 8; 420 id ^= id >> 4; 421 return id & mask; 422 } 423 424 static inline struct Qdisc_class_common * 425 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 426 { 427 struct Qdisc_class_common *cl; 428 unsigned int h; 429 430 if (!id) 431 return NULL; 432 433 h = qdisc_class_hash(id, hash->hashmask); 434 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 435 if (cl->classid == id) 436 return cl; 437 } 438 return NULL; 439 } 440 441 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 442 { 443 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 444 445 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 446 } 447 448 int qdisc_class_hash_init(struct Qdisc_class_hash *); 449 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 450 struct Qdisc_class_common *); 451 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 452 struct Qdisc_class_common *); 453 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 454 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 455 456 void dev_init_scheduler(struct net_device *dev); 457 void dev_shutdown(struct net_device *dev); 458 void dev_activate(struct net_device *dev); 459 void dev_deactivate(struct net_device *dev); 460 void dev_deactivate_many(struct list_head *head); 461 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 462 struct Qdisc *qdisc); 463 void qdisc_reset(struct Qdisc *qdisc); 464 void qdisc_destroy(struct Qdisc *qdisc); 465 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n, 466 unsigned int len); 467 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 468 const struct Qdisc_ops *ops); 469 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 470 const struct Qdisc_ops *ops, u32 parentid); 471 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 472 const struct qdisc_size_table *stab); 473 int skb_do_redirect(struct sk_buff *); 474 475 static inline void skb_reset_tc(struct sk_buff *skb) 476 { 477 #ifdef CONFIG_NET_CLS_ACT 478 skb->tc_redirected = 0; 479 #endif 480 } 481 482 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 483 { 484 #ifdef CONFIG_NET_CLS_ACT 485 return skb->tc_at_ingress; 486 #else 487 return false; 488 #endif 489 } 490 491 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 492 { 493 #ifdef CONFIG_NET_CLS_ACT 494 if (skb->tc_skip_classify) { 495 skb->tc_skip_classify = 0; 496 return true; 497 } 498 #endif 499 return false; 500 } 501 502 /* Reset all TX qdiscs greater then index of a device. */ 503 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 504 { 505 struct Qdisc *qdisc; 506 507 for (; i < dev->num_tx_queues; i++) { 508 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 509 if (qdisc) { 510 spin_lock_bh(qdisc_lock(qdisc)); 511 qdisc_reset(qdisc); 512 spin_unlock_bh(qdisc_lock(qdisc)); 513 } 514 } 515 } 516 517 static inline void qdisc_reset_all_tx(struct net_device *dev) 518 { 519 qdisc_reset_all_tx_gt(dev, 0); 520 } 521 522 /* Are all TX queues of the device empty? */ 523 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 524 { 525 unsigned int i; 526 527 rcu_read_lock(); 528 for (i = 0; i < dev->num_tx_queues; i++) { 529 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 530 const struct Qdisc *q = rcu_dereference(txq->qdisc); 531 532 if (q->q.qlen) { 533 rcu_read_unlock(); 534 return false; 535 } 536 } 537 rcu_read_unlock(); 538 return true; 539 } 540 541 /* Are any of the TX qdiscs changing? */ 542 static inline bool qdisc_tx_changing(const struct net_device *dev) 543 { 544 unsigned int i; 545 546 for (i = 0; i < dev->num_tx_queues; i++) { 547 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 548 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping) 549 return true; 550 } 551 return false; 552 } 553 554 /* Is the device using the noop qdisc on all queues? */ 555 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 556 { 557 unsigned int i; 558 559 for (i = 0; i < dev->num_tx_queues; i++) { 560 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 561 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 562 return false; 563 } 564 return true; 565 } 566 567 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 568 { 569 return qdisc_skb_cb(skb)->pkt_len; 570 } 571 572 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 573 enum net_xmit_qdisc_t { 574 __NET_XMIT_STOLEN = 0x00010000, 575 __NET_XMIT_BYPASS = 0x00020000, 576 }; 577 578 #ifdef CONFIG_NET_CLS_ACT 579 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 580 #else 581 #define net_xmit_drop_count(e) (1) 582 #endif 583 584 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 585 const struct Qdisc *sch) 586 { 587 #ifdef CONFIG_NET_SCHED 588 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 589 590 if (stab) 591 __qdisc_calculate_pkt_len(skb, stab); 592 #endif 593 } 594 595 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 596 struct sk_buff **to_free) 597 { 598 qdisc_calculate_pkt_len(skb, sch); 599 return sch->enqueue(skb, sch, to_free); 600 } 601 602 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 603 { 604 return q->flags & TCQ_F_CPUSTATS; 605 } 606 607 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats, 608 __u64 bytes, __u32 packets) 609 { 610 bstats->bytes += bytes; 611 bstats->packets += packets; 612 } 613 614 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 615 const struct sk_buff *skb) 616 { 617 _bstats_update(bstats, 618 qdisc_pkt_len(skb), 619 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 620 } 621 622 static inline void _bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 623 __u64 bytes, __u32 packets) 624 { 625 u64_stats_update_begin(&bstats->syncp); 626 _bstats_update(&bstats->bstats, bytes, packets); 627 u64_stats_update_end(&bstats->syncp); 628 } 629 630 static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 631 const struct sk_buff *skb) 632 { 633 u64_stats_update_begin(&bstats->syncp); 634 bstats_update(&bstats->bstats, skb); 635 u64_stats_update_end(&bstats->syncp); 636 } 637 638 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 639 const struct sk_buff *skb) 640 { 641 bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb); 642 } 643 644 static inline void qdisc_bstats_update(struct Qdisc *sch, 645 const struct sk_buff *skb) 646 { 647 bstats_update(&sch->bstats, skb); 648 } 649 650 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 651 const struct sk_buff *skb) 652 { 653 sch->qstats.backlog -= qdisc_pkt_len(skb); 654 } 655 656 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 657 const struct sk_buff *skb) 658 { 659 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 660 } 661 662 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 663 const struct sk_buff *skb) 664 { 665 sch->qstats.backlog += qdisc_pkt_len(skb); 666 } 667 668 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 669 const struct sk_buff *skb) 670 { 671 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 672 } 673 674 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 675 { 676 this_cpu_inc(sch->cpu_qstats->qlen); 677 } 678 679 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 680 { 681 this_cpu_dec(sch->cpu_qstats->qlen); 682 } 683 684 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 685 { 686 this_cpu_inc(sch->cpu_qstats->requeues); 687 } 688 689 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 690 { 691 sch->qstats.drops += count; 692 } 693 694 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 695 { 696 qstats->drops++; 697 } 698 699 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 700 { 701 qstats->overlimits++; 702 } 703 704 static inline void qdisc_qstats_drop(struct Qdisc *sch) 705 { 706 qstats_drop_inc(&sch->qstats); 707 } 708 709 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 710 { 711 this_cpu_inc(sch->cpu_qstats->drops); 712 } 713 714 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 715 { 716 sch->qstats.overlimits++; 717 } 718 719 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) 720 { 721 qh->head = NULL; 722 qh->tail = NULL; 723 qh->qlen = 0; 724 } 725 726 static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, 727 struct qdisc_skb_head *qh) 728 { 729 struct sk_buff *last = qh->tail; 730 731 if (last) { 732 skb->next = NULL; 733 last->next = skb; 734 qh->tail = skb; 735 } else { 736 qh->tail = skb; 737 qh->head = skb; 738 } 739 qh->qlen++; 740 qdisc_qstats_backlog_inc(sch, skb); 741 742 return NET_XMIT_SUCCESS; 743 } 744 745 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 746 { 747 return __qdisc_enqueue_tail(skb, sch, &sch->q); 748 } 749 750 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 751 { 752 struct sk_buff *skb = qh->head; 753 754 if (likely(skb != NULL)) { 755 qh->head = skb->next; 756 qh->qlen--; 757 if (qh->head == NULL) 758 qh->tail = NULL; 759 skb->next = NULL; 760 } 761 762 return skb; 763 } 764 765 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 766 { 767 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 768 769 if (likely(skb != NULL)) { 770 qdisc_qstats_backlog_dec(sch, skb); 771 qdisc_bstats_update(sch, skb); 772 } 773 774 return skb; 775 } 776 777 /* Instead of calling kfree_skb() while root qdisc lock is held, 778 * queue the skb for future freeing at end of __dev_xmit_skb() 779 */ 780 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 781 { 782 skb->next = *to_free; 783 *to_free = skb; 784 } 785 786 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 787 struct qdisc_skb_head *qh, 788 struct sk_buff **to_free) 789 { 790 struct sk_buff *skb = __qdisc_dequeue_head(qh); 791 792 if (likely(skb != NULL)) { 793 unsigned int len = qdisc_pkt_len(skb); 794 795 qdisc_qstats_backlog_dec(sch, skb); 796 __qdisc_drop(skb, to_free); 797 return len; 798 } 799 800 return 0; 801 } 802 803 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch, 804 struct sk_buff **to_free) 805 { 806 return __qdisc_queue_drop_head(sch, &sch->q, to_free); 807 } 808 809 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 810 { 811 const struct qdisc_skb_head *qh = &sch->q; 812 813 return qh->head; 814 } 815 816 /* generic pseudo peek method for non-work-conserving qdisc */ 817 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 818 { 819 struct sk_buff *skb = skb_peek(&sch->gso_skb); 820 821 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 822 if (!skb) { 823 skb = sch->dequeue(sch); 824 825 if (skb) { 826 __skb_queue_head(&sch->gso_skb, skb); 827 /* it's still part of the queue */ 828 qdisc_qstats_backlog_inc(sch, skb); 829 sch->q.qlen++; 830 } 831 } 832 833 return skb; 834 } 835 836 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 837 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 838 { 839 struct sk_buff *skb = skb_peek(&sch->gso_skb); 840 841 if (skb) { 842 skb = __skb_dequeue(&sch->gso_skb); 843 qdisc_qstats_backlog_dec(sch, skb); 844 sch->q.qlen--; 845 } else { 846 skb = sch->dequeue(sch); 847 } 848 849 return skb; 850 } 851 852 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 853 { 854 /* 855 * We do not know the backlog in bytes of this list, it 856 * is up to the caller to correct it 857 */ 858 ASSERT_RTNL(); 859 if (qh->qlen) { 860 rtnl_kfree_skbs(qh->head, qh->tail); 861 862 qh->head = NULL; 863 qh->tail = NULL; 864 qh->qlen = 0; 865 } 866 } 867 868 static inline void qdisc_reset_queue(struct Qdisc *sch) 869 { 870 __qdisc_reset_queue(&sch->q); 871 sch->qstats.backlog = 0; 872 } 873 874 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 875 struct Qdisc **pold) 876 { 877 struct Qdisc *old; 878 879 sch_tree_lock(sch); 880 old = *pold; 881 *pold = new; 882 if (old != NULL) { 883 unsigned int qlen = old->q.qlen; 884 unsigned int backlog = old->qstats.backlog; 885 886 qdisc_reset(old); 887 qdisc_tree_reduce_backlog(old, qlen, backlog); 888 } 889 sch_tree_unlock(sch); 890 891 return old; 892 } 893 894 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 895 { 896 rtnl_kfree_skbs(skb, skb); 897 qdisc_qstats_drop(sch); 898 } 899 900 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 901 struct sk_buff **to_free) 902 { 903 __qdisc_drop(skb, to_free); 904 qdisc_qstats_cpu_drop(sch); 905 906 return NET_XMIT_DROP; 907 } 908 909 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 910 struct sk_buff **to_free) 911 { 912 __qdisc_drop(skb, to_free); 913 qdisc_qstats_drop(sch); 914 915 return NET_XMIT_DROP; 916 } 917 918 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 919 long it will take to send a packet given its size. 920 */ 921 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 922 { 923 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 924 if (slot < 0) 925 slot = 0; 926 slot >>= rtab->rate.cell_log; 927 if (slot > 255) 928 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 929 return rtab->data[slot]; 930 } 931 932 struct psched_ratecfg { 933 u64 rate_bytes_ps; /* bytes per second */ 934 u32 mult; 935 u16 overhead; 936 u8 linklayer; 937 u8 shift; 938 }; 939 940 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 941 unsigned int len) 942 { 943 len += r->overhead; 944 945 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 946 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 947 948 return ((u64)len * r->mult) >> r->shift; 949 } 950 951 void psched_ratecfg_precompute(struct psched_ratecfg *r, 952 const struct tc_ratespec *conf, 953 u64 rate64); 954 955 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 956 const struct psched_ratecfg *r) 957 { 958 memset(res, 0, sizeof(*res)); 959 960 /* legacy struct tc_ratespec has a 32bit @rate field 961 * Qdisc using 64bit rate should add new attributes 962 * in order to maintain compatibility. 963 */ 964 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 965 966 res->overhead = r->overhead; 967 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 968 } 969 970 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 971 * The fast path only needs to access filter list and to update stats 972 */ 973 struct mini_Qdisc { 974 struct tcf_proto *filter_list; 975 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 976 struct gnet_stats_queue __percpu *cpu_qstats; 977 struct rcu_head rcu; 978 }; 979 980 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 981 const struct sk_buff *skb) 982 { 983 bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb); 984 } 985 986 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 987 { 988 this_cpu_inc(miniq->cpu_qstats->drops); 989 } 990 991 struct mini_Qdisc_pair { 992 struct mini_Qdisc miniq1; 993 struct mini_Qdisc miniq2; 994 struct mini_Qdisc __rcu **p_miniq; 995 }; 996 997 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 998 struct tcf_proto *tp_head); 999 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1000 struct mini_Qdisc __rcu **p_miniq); 1001 1002 #endif 1003