1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <net/gen_stats.h> 16 #include <net/rtnetlink.h> 17 18 struct Qdisc_ops; 19 struct qdisc_walker; 20 struct tcf_walker; 21 struct module; 22 23 typedef int tc_setup_cb_t(enum tc_setup_type type, 24 void *type_data, void *cb_priv); 25 26 struct qdisc_rate_table { 27 struct tc_ratespec rate; 28 u32 data[256]; 29 struct qdisc_rate_table *next; 30 int refcnt; 31 }; 32 33 enum qdisc_state_t { 34 __QDISC_STATE_SCHED, 35 __QDISC_STATE_DEACTIVATED, 36 }; 37 38 struct qdisc_size_table { 39 struct rcu_head rcu; 40 struct list_head list; 41 struct tc_sizespec szopts; 42 int refcnt; 43 u16 data[]; 44 }; 45 46 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 47 struct qdisc_skb_head { 48 struct sk_buff *head; 49 struct sk_buff *tail; 50 __u32 qlen; 51 spinlock_t lock; 52 }; 53 54 struct Qdisc { 55 int (*enqueue)(struct sk_buff *skb, 56 struct Qdisc *sch, 57 struct sk_buff **to_free); 58 struct sk_buff * (*dequeue)(struct Qdisc *sch); 59 unsigned int flags; 60 #define TCQ_F_BUILTIN 1 61 #define TCQ_F_INGRESS 2 62 #define TCQ_F_CAN_BYPASS 4 63 #define TCQ_F_MQROOT 8 64 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 65 * q->dev_queue : It can test 66 * netif_xmit_frozen_or_stopped() before 67 * dequeueing next packet. 68 * Its true for MQ/MQPRIO slaves, or non 69 * multiqueue device. 70 */ 71 #define TCQ_F_WARN_NONWC (1 << 16) 72 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 73 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 74 * qdisc_tree_decrease_qlen() should stop. 75 */ 76 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 77 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 78 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */ 79 u32 limit; 80 const struct Qdisc_ops *ops; 81 struct qdisc_size_table __rcu *stab; 82 struct hlist_node hash; 83 u32 handle; 84 u32 parent; 85 86 struct netdev_queue *dev_queue; 87 88 struct net_rate_estimator __rcu *rate_est; 89 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 90 struct gnet_stats_queue __percpu *cpu_qstats; 91 int padded; 92 refcount_t refcnt; 93 94 /* 95 * For performance sake on SMP, we put highly modified fields at the end 96 */ 97 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 98 struct qdisc_skb_head q; 99 struct gnet_stats_basic_packed bstats; 100 seqcount_t running; 101 struct gnet_stats_queue qstats; 102 unsigned long state; 103 struct Qdisc *next_sched; 104 struct sk_buff_head skb_bad_txq; 105 106 spinlock_t busylock ____cacheline_aligned_in_smp; 107 spinlock_t seqlock; 108 }; 109 110 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 111 { 112 if (qdisc->flags & TCQ_F_BUILTIN) 113 return; 114 refcount_inc(&qdisc->refcnt); 115 } 116 117 static inline bool qdisc_is_running(struct Qdisc *qdisc) 118 { 119 if (qdisc->flags & TCQ_F_NOLOCK) 120 return spin_is_locked(&qdisc->seqlock); 121 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; 122 } 123 124 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 125 { 126 if (qdisc->flags & TCQ_F_NOLOCK) { 127 if (!spin_trylock(&qdisc->seqlock)) 128 return false; 129 } else if (qdisc_is_running(qdisc)) { 130 return false; 131 } 132 /* Variant of write_seqcount_begin() telling lockdep a trylock 133 * was attempted. 134 */ 135 raw_write_seqcount_begin(&qdisc->running); 136 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_); 137 return true; 138 } 139 140 static inline void qdisc_run_end(struct Qdisc *qdisc) 141 { 142 write_seqcount_end(&qdisc->running); 143 if (qdisc->flags & TCQ_F_NOLOCK) 144 spin_unlock(&qdisc->seqlock); 145 } 146 147 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 148 { 149 return qdisc->flags & TCQ_F_ONETXQUEUE; 150 } 151 152 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 153 { 154 #ifdef CONFIG_BQL 155 /* Non-BQL migrated drivers will return 0, too. */ 156 return dql_avail(&txq->dql); 157 #else 158 return 0; 159 #endif 160 } 161 162 struct Qdisc_class_ops { 163 /* Child qdisc manipulation */ 164 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 165 int (*graft)(struct Qdisc *, unsigned long cl, 166 struct Qdisc *, struct Qdisc **, 167 struct netlink_ext_ack *extack); 168 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 169 void (*qlen_notify)(struct Qdisc *, unsigned long); 170 171 /* Class manipulation routines */ 172 unsigned long (*find)(struct Qdisc *, u32 classid); 173 int (*change)(struct Qdisc *, u32, u32, 174 struct nlattr **, unsigned long *, 175 struct netlink_ext_ack *); 176 int (*delete)(struct Qdisc *, unsigned long); 177 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 178 179 /* Filter manipulation */ 180 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 181 unsigned long arg, 182 struct netlink_ext_ack *extack); 183 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 184 u32 classid); 185 void (*unbind_tcf)(struct Qdisc *, unsigned long); 186 187 /* rtnetlink specific */ 188 int (*dump)(struct Qdisc *, unsigned long, 189 struct sk_buff *skb, struct tcmsg*); 190 int (*dump_stats)(struct Qdisc *, unsigned long, 191 struct gnet_dump *); 192 }; 193 194 struct Qdisc_ops { 195 struct Qdisc_ops *next; 196 const struct Qdisc_class_ops *cl_ops; 197 char id[IFNAMSIZ]; 198 int priv_size; 199 unsigned int static_flags; 200 201 int (*enqueue)(struct sk_buff *skb, 202 struct Qdisc *sch, 203 struct sk_buff **to_free); 204 struct sk_buff * (*dequeue)(struct Qdisc *); 205 struct sk_buff * (*peek)(struct Qdisc *); 206 207 int (*init)(struct Qdisc *sch, struct nlattr *arg, 208 struct netlink_ext_ack *extack); 209 void (*reset)(struct Qdisc *); 210 void (*destroy)(struct Qdisc *); 211 int (*change)(struct Qdisc *sch, 212 struct nlattr *arg, 213 struct netlink_ext_ack *extack); 214 void (*attach)(struct Qdisc *sch); 215 int (*change_tx_queue_len)(struct Qdisc *, unsigned int); 216 217 int (*dump)(struct Qdisc *, struct sk_buff *); 218 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 219 220 void (*ingress_block_set)(struct Qdisc *sch, 221 u32 block_index); 222 void (*egress_block_set)(struct Qdisc *sch, 223 u32 block_index); 224 u32 (*ingress_block_get)(struct Qdisc *sch); 225 u32 (*egress_block_get)(struct Qdisc *sch); 226 227 struct module *owner; 228 }; 229 230 231 struct tcf_result { 232 union { 233 struct { 234 unsigned long class; 235 u32 classid; 236 }; 237 const struct tcf_proto *goto_tp; 238 }; 239 }; 240 241 struct tcf_proto_ops { 242 struct list_head head; 243 char kind[IFNAMSIZ]; 244 245 int (*classify)(struct sk_buff *, 246 const struct tcf_proto *, 247 struct tcf_result *); 248 int (*init)(struct tcf_proto*); 249 void (*destroy)(struct tcf_proto *tp, 250 struct netlink_ext_ack *extack); 251 252 void* (*get)(struct tcf_proto*, u32 handle); 253 int (*change)(struct net *net, struct sk_buff *, 254 struct tcf_proto*, unsigned long, 255 u32 handle, struct nlattr **, 256 void **, bool, 257 struct netlink_ext_ack *); 258 int (*delete)(struct tcf_proto *tp, void *arg, 259 bool *last, 260 struct netlink_ext_ack *); 261 void (*walk)(struct tcf_proto*, struct tcf_walker *arg); 262 int (*reoffload)(struct tcf_proto *tp, bool add, 263 tc_setup_cb_t *cb, void *cb_priv, 264 struct netlink_ext_ack *extack); 265 void (*bind_class)(void *, u32, unsigned long); 266 267 /* rtnetlink specific */ 268 int (*dump)(struct net*, struct tcf_proto*, void *, 269 struct sk_buff *skb, struct tcmsg*); 270 271 struct module *owner; 272 }; 273 274 struct tcf_proto { 275 /* Fast access part */ 276 struct tcf_proto __rcu *next; 277 void __rcu *root; 278 int (*classify)(struct sk_buff *, 279 const struct tcf_proto *, 280 struct tcf_result *); 281 __be16 protocol; 282 283 /* All the rest */ 284 u32 prio; 285 void *data; 286 const struct tcf_proto_ops *ops; 287 struct tcf_chain *chain; 288 struct rcu_head rcu; 289 }; 290 291 struct qdisc_skb_cb { 292 unsigned int pkt_len; 293 u16 slave_dev_queue_mapping; 294 u16 tc_classid; 295 #define QDISC_CB_PRIV_LEN 20 296 unsigned char data[QDISC_CB_PRIV_LEN]; 297 }; 298 299 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 300 301 struct tcf_chain { 302 struct tcf_proto __rcu *filter_chain; 303 struct list_head filter_chain_list; 304 struct list_head list; 305 struct tcf_block *block; 306 u32 index; /* chain index */ 307 unsigned int refcnt; 308 }; 309 310 struct tcf_block { 311 struct list_head chain_list; 312 u32 index; /* block index for shared blocks */ 313 unsigned int refcnt; 314 struct net *net; 315 struct Qdisc *q; 316 struct list_head cb_list; 317 struct list_head owner_list; 318 bool keep_dst; 319 unsigned int offloadcnt; /* Number of oddloaded filters */ 320 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ 321 }; 322 323 static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) 324 { 325 if (*flags & TCA_CLS_FLAGS_IN_HW) 326 return; 327 *flags |= TCA_CLS_FLAGS_IN_HW; 328 block->offloadcnt++; 329 } 330 331 static inline void tcf_block_offload_dec(struct tcf_block *block, u32 *flags) 332 { 333 if (!(*flags & TCA_CLS_FLAGS_IN_HW)) 334 return; 335 *flags &= ~TCA_CLS_FLAGS_IN_HW; 336 block->offloadcnt--; 337 } 338 339 static inline void 340 tc_cls_offload_cnt_update(struct tcf_block *block, unsigned int *cnt, 341 u32 *flags, bool add) 342 { 343 if (add) { 344 if (!*cnt) 345 tcf_block_offload_inc(block, flags); 346 (*cnt)++; 347 } else { 348 (*cnt)--; 349 if (!*cnt) 350 tcf_block_offload_dec(block, flags); 351 } 352 } 353 354 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 355 { 356 struct qdisc_skb_cb *qcb; 357 358 BUILD_BUG_ON(sizeof(skb->cb) < offsetof(struct qdisc_skb_cb, data) + sz); 359 BUILD_BUG_ON(sizeof(qcb->data) < sz); 360 } 361 362 static inline int qdisc_qlen_cpu(const struct Qdisc *q) 363 { 364 return this_cpu_ptr(q->cpu_qstats)->qlen; 365 } 366 367 static inline int qdisc_qlen(const struct Qdisc *q) 368 { 369 return q->q.qlen; 370 } 371 372 static inline int qdisc_qlen_sum(const struct Qdisc *q) 373 { 374 __u32 qlen = q->qstats.qlen; 375 int i; 376 377 if (q->flags & TCQ_F_NOLOCK) { 378 for_each_possible_cpu(i) 379 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 380 } else { 381 qlen += q->q.qlen; 382 } 383 384 return qlen; 385 } 386 387 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 388 { 389 return (struct qdisc_skb_cb *)skb->cb; 390 } 391 392 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 393 { 394 return &qdisc->q.lock; 395 } 396 397 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 398 { 399 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 400 401 return q; 402 } 403 404 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 405 { 406 return qdisc->dev_queue->qdisc_sleeping; 407 } 408 409 /* The qdisc root lock is a mechanism by which to top level 410 * of a qdisc tree can be locked from any qdisc node in the 411 * forest. This allows changing the configuration of some 412 * aspect of the qdisc tree while blocking out asynchronous 413 * qdisc access in the packet processing paths. 414 * 415 * It is only legal to do this when the root will not change 416 * on us. Otherwise we'll potentially lock the wrong qdisc 417 * root. This is enforced by holding the RTNL semaphore, which 418 * all users of this lock accessor must do. 419 */ 420 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) 421 { 422 struct Qdisc *root = qdisc_root(qdisc); 423 424 ASSERT_RTNL(); 425 return qdisc_lock(root); 426 } 427 428 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 429 { 430 struct Qdisc *root = qdisc_root_sleeping(qdisc); 431 432 ASSERT_RTNL(); 433 return qdisc_lock(root); 434 } 435 436 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc) 437 { 438 struct Qdisc *root = qdisc_root_sleeping(qdisc); 439 440 ASSERT_RTNL(); 441 return &root->running; 442 } 443 444 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 445 { 446 return qdisc->dev_queue->dev; 447 } 448 449 static inline void sch_tree_lock(const struct Qdisc *q) 450 { 451 spin_lock_bh(qdisc_root_sleeping_lock(q)); 452 } 453 454 static inline void sch_tree_unlock(const struct Qdisc *q) 455 { 456 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 457 } 458 459 extern struct Qdisc noop_qdisc; 460 extern struct Qdisc_ops noop_qdisc_ops; 461 extern struct Qdisc_ops pfifo_fast_ops; 462 extern struct Qdisc_ops mq_qdisc_ops; 463 extern struct Qdisc_ops noqueue_qdisc_ops; 464 extern const struct Qdisc_ops *default_qdisc_ops; 465 static inline const struct Qdisc_ops * 466 get_default_qdisc_ops(const struct net_device *dev, int ntx) 467 { 468 return ntx < dev->real_num_tx_queues ? 469 default_qdisc_ops : &pfifo_fast_ops; 470 } 471 472 struct Qdisc_class_common { 473 u32 classid; 474 struct hlist_node hnode; 475 }; 476 477 struct Qdisc_class_hash { 478 struct hlist_head *hash; 479 unsigned int hashsize; 480 unsigned int hashmask; 481 unsigned int hashelems; 482 }; 483 484 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 485 { 486 id ^= id >> 8; 487 id ^= id >> 4; 488 return id & mask; 489 } 490 491 static inline struct Qdisc_class_common * 492 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 493 { 494 struct Qdisc_class_common *cl; 495 unsigned int h; 496 497 if (!id) 498 return NULL; 499 500 h = qdisc_class_hash(id, hash->hashmask); 501 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 502 if (cl->classid == id) 503 return cl; 504 } 505 return NULL; 506 } 507 508 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 509 { 510 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 511 512 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 513 } 514 515 int qdisc_class_hash_init(struct Qdisc_class_hash *); 516 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 517 struct Qdisc_class_common *); 518 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 519 struct Qdisc_class_common *); 520 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 521 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 522 523 int dev_qdisc_change_tx_queue_len(struct net_device *dev); 524 void dev_init_scheduler(struct net_device *dev); 525 void dev_shutdown(struct net_device *dev); 526 void dev_activate(struct net_device *dev); 527 void dev_deactivate(struct net_device *dev); 528 void dev_deactivate_many(struct list_head *head); 529 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 530 struct Qdisc *qdisc); 531 void qdisc_reset(struct Qdisc *qdisc); 532 void qdisc_destroy(struct Qdisc *qdisc); 533 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n, 534 unsigned int len); 535 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 536 const struct Qdisc_ops *ops, 537 struct netlink_ext_ack *extack); 538 void qdisc_free(struct Qdisc *qdisc); 539 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 540 const struct Qdisc_ops *ops, u32 parentid, 541 struct netlink_ext_ack *extack); 542 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 543 const struct qdisc_size_table *stab); 544 int skb_do_redirect(struct sk_buff *); 545 546 static inline void skb_reset_tc(struct sk_buff *skb) 547 { 548 #ifdef CONFIG_NET_CLS_ACT 549 skb->tc_redirected = 0; 550 #endif 551 } 552 553 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 554 { 555 #ifdef CONFIG_NET_CLS_ACT 556 return skb->tc_at_ingress; 557 #else 558 return false; 559 #endif 560 } 561 562 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 563 { 564 #ifdef CONFIG_NET_CLS_ACT 565 if (skb->tc_skip_classify) { 566 skb->tc_skip_classify = 0; 567 return true; 568 } 569 #endif 570 return false; 571 } 572 573 /* Reset all TX qdiscs greater than index of a device. */ 574 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 575 { 576 struct Qdisc *qdisc; 577 578 for (; i < dev->num_tx_queues; i++) { 579 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 580 if (qdisc) { 581 spin_lock_bh(qdisc_lock(qdisc)); 582 qdisc_reset(qdisc); 583 spin_unlock_bh(qdisc_lock(qdisc)); 584 } 585 } 586 } 587 588 static inline void qdisc_reset_all_tx(struct net_device *dev) 589 { 590 qdisc_reset_all_tx_gt(dev, 0); 591 } 592 593 /* Are all TX queues of the device empty? */ 594 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 595 { 596 unsigned int i; 597 598 rcu_read_lock(); 599 for (i = 0; i < dev->num_tx_queues; i++) { 600 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 601 const struct Qdisc *q = rcu_dereference(txq->qdisc); 602 603 if (q->q.qlen) { 604 rcu_read_unlock(); 605 return false; 606 } 607 } 608 rcu_read_unlock(); 609 return true; 610 } 611 612 /* Are any of the TX qdiscs changing? */ 613 static inline bool qdisc_tx_changing(const struct net_device *dev) 614 { 615 unsigned int i; 616 617 for (i = 0; i < dev->num_tx_queues; i++) { 618 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 619 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping) 620 return true; 621 } 622 return false; 623 } 624 625 /* Is the device using the noop qdisc on all queues? */ 626 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 627 { 628 unsigned int i; 629 630 for (i = 0; i < dev->num_tx_queues; i++) { 631 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 632 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 633 return false; 634 } 635 return true; 636 } 637 638 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 639 { 640 return qdisc_skb_cb(skb)->pkt_len; 641 } 642 643 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 644 enum net_xmit_qdisc_t { 645 __NET_XMIT_STOLEN = 0x00010000, 646 __NET_XMIT_BYPASS = 0x00020000, 647 }; 648 649 #ifdef CONFIG_NET_CLS_ACT 650 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 651 #else 652 #define net_xmit_drop_count(e) (1) 653 #endif 654 655 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 656 const struct Qdisc *sch) 657 { 658 #ifdef CONFIG_NET_SCHED 659 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 660 661 if (stab) 662 __qdisc_calculate_pkt_len(skb, stab); 663 #endif 664 } 665 666 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 667 struct sk_buff **to_free) 668 { 669 qdisc_calculate_pkt_len(skb, sch); 670 return sch->enqueue(skb, sch, to_free); 671 } 672 673 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 674 { 675 return q->flags & TCQ_F_CPUSTATS; 676 } 677 678 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats, 679 __u64 bytes, __u32 packets) 680 { 681 bstats->bytes += bytes; 682 bstats->packets += packets; 683 } 684 685 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 686 const struct sk_buff *skb) 687 { 688 _bstats_update(bstats, 689 qdisc_pkt_len(skb), 690 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 691 } 692 693 static inline void _bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 694 __u64 bytes, __u32 packets) 695 { 696 u64_stats_update_begin(&bstats->syncp); 697 _bstats_update(&bstats->bstats, bytes, packets); 698 u64_stats_update_end(&bstats->syncp); 699 } 700 701 static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 702 const struct sk_buff *skb) 703 { 704 u64_stats_update_begin(&bstats->syncp); 705 bstats_update(&bstats->bstats, skb); 706 u64_stats_update_end(&bstats->syncp); 707 } 708 709 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 710 const struct sk_buff *skb) 711 { 712 bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb); 713 } 714 715 static inline void qdisc_bstats_update(struct Qdisc *sch, 716 const struct sk_buff *skb) 717 { 718 bstats_update(&sch->bstats, skb); 719 } 720 721 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 722 const struct sk_buff *skb) 723 { 724 sch->qstats.backlog -= qdisc_pkt_len(skb); 725 } 726 727 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 728 const struct sk_buff *skb) 729 { 730 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 731 } 732 733 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 734 const struct sk_buff *skb) 735 { 736 sch->qstats.backlog += qdisc_pkt_len(skb); 737 } 738 739 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 740 const struct sk_buff *skb) 741 { 742 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 743 } 744 745 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 746 { 747 this_cpu_inc(sch->cpu_qstats->qlen); 748 } 749 750 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 751 { 752 this_cpu_dec(sch->cpu_qstats->qlen); 753 } 754 755 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 756 { 757 this_cpu_inc(sch->cpu_qstats->requeues); 758 } 759 760 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 761 { 762 sch->qstats.drops += count; 763 } 764 765 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 766 { 767 qstats->drops++; 768 } 769 770 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 771 { 772 qstats->overlimits++; 773 } 774 775 static inline void qdisc_qstats_drop(struct Qdisc *sch) 776 { 777 qstats_drop_inc(&sch->qstats); 778 } 779 780 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 781 { 782 this_cpu_inc(sch->cpu_qstats->drops); 783 } 784 785 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 786 { 787 sch->qstats.overlimits++; 788 } 789 790 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) 791 { 792 qh->head = NULL; 793 qh->tail = NULL; 794 qh->qlen = 0; 795 } 796 797 static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, 798 struct qdisc_skb_head *qh) 799 { 800 struct sk_buff *last = qh->tail; 801 802 if (last) { 803 skb->next = NULL; 804 last->next = skb; 805 qh->tail = skb; 806 } else { 807 qh->tail = skb; 808 qh->head = skb; 809 } 810 qh->qlen++; 811 qdisc_qstats_backlog_inc(sch, skb); 812 813 return NET_XMIT_SUCCESS; 814 } 815 816 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 817 { 818 return __qdisc_enqueue_tail(skb, sch, &sch->q); 819 } 820 821 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 822 { 823 struct sk_buff *skb = qh->head; 824 825 if (likely(skb != NULL)) { 826 qh->head = skb->next; 827 qh->qlen--; 828 if (qh->head == NULL) 829 qh->tail = NULL; 830 skb->next = NULL; 831 } 832 833 return skb; 834 } 835 836 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 837 { 838 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 839 840 if (likely(skb != NULL)) { 841 qdisc_qstats_backlog_dec(sch, skb); 842 qdisc_bstats_update(sch, skb); 843 } 844 845 return skb; 846 } 847 848 /* Instead of calling kfree_skb() while root qdisc lock is held, 849 * queue the skb for future freeing at end of __dev_xmit_skb() 850 */ 851 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 852 { 853 skb->next = *to_free; 854 *to_free = skb; 855 } 856 857 static inline void __qdisc_drop_all(struct sk_buff *skb, 858 struct sk_buff **to_free) 859 { 860 if (skb->prev) 861 skb->prev->next = *to_free; 862 else 863 skb->next = *to_free; 864 *to_free = skb; 865 } 866 867 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 868 struct qdisc_skb_head *qh, 869 struct sk_buff **to_free) 870 { 871 struct sk_buff *skb = __qdisc_dequeue_head(qh); 872 873 if (likely(skb != NULL)) { 874 unsigned int len = qdisc_pkt_len(skb); 875 876 qdisc_qstats_backlog_dec(sch, skb); 877 __qdisc_drop(skb, to_free); 878 return len; 879 } 880 881 return 0; 882 } 883 884 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch, 885 struct sk_buff **to_free) 886 { 887 return __qdisc_queue_drop_head(sch, &sch->q, to_free); 888 } 889 890 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 891 { 892 const struct qdisc_skb_head *qh = &sch->q; 893 894 return qh->head; 895 } 896 897 /* generic pseudo peek method for non-work-conserving qdisc */ 898 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 899 { 900 struct sk_buff *skb = skb_peek(&sch->gso_skb); 901 902 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 903 if (!skb) { 904 skb = sch->dequeue(sch); 905 906 if (skb) { 907 __skb_queue_head(&sch->gso_skb, skb); 908 /* it's still part of the queue */ 909 qdisc_qstats_backlog_inc(sch, skb); 910 sch->q.qlen++; 911 } 912 } 913 914 return skb; 915 } 916 917 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 918 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 919 { 920 struct sk_buff *skb = skb_peek(&sch->gso_skb); 921 922 if (skb) { 923 skb = __skb_dequeue(&sch->gso_skb); 924 qdisc_qstats_backlog_dec(sch, skb); 925 sch->q.qlen--; 926 } else { 927 skb = sch->dequeue(sch); 928 } 929 930 return skb; 931 } 932 933 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 934 { 935 /* 936 * We do not know the backlog in bytes of this list, it 937 * is up to the caller to correct it 938 */ 939 ASSERT_RTNL(); 940 if (qh->qlen) { 941 rtnl_kfree_skbs(qh->head, qh->tail); 942 943 qh->head = NULL; 944 qh->tail = NULL; 945 qh->qlen = 0; 946 } 947 } 948 949 static inline void qdisc_reset_queue(struct Qdisc *sch) 950 { 951 __qdisc_reset_queue(&sch->q); 952 sch->qstats.backlog = 0; 953 } 954 955 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 956 struct Qdisc **pold) 957 { 958 struct Qdisc *old; 959 960 sch_tree_lock(sch); 961 old = *pold; 962 *pold = new; 963 if (old != NULL) { 964 unsigned int qlen = old->q.qlen; 965 unsigned int backlog = old->qstats.backlog; 966 967 qdisc_reset(old); 968 qdisc_tree_reduce_backlog(old, qlen, backlog); 969 } 970 sch_tree_unlock(sch); 971 972 return old; 973 } 974 975 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 976 { 977 rtnl_kfree_skbs(skb, skb); 978 qdisc_qstats_drop(sch); 979 } 980 981 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 982 struct sk_buff **to_free) 983 { 984 __qdisc_drop(skb, to_free); 985 qdisc_qstats_cpu_drop(sch); 986 987 return NET_XMIT_DROP; 988 } 989 990 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 991 struct sk_buff **to_free) 992 { 993 __qdisc_drop(skb, to_free); 994 qdisc_qstats_drop(sch); 995 996 return NET_XMIT_DROP; 997 } 998 999 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, 1000 struct sk_buff **to_free) 1001 { 1002 __qdisc_drop_all(skb, to_free); 1003 qdisc_qstats_drop(sch); 1004 1005 return NET_XMIT_DROP; 1006 } 1007 1008 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 1009 long it will take to send a packet given its size. 1010 */ 1011 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 1012 { 1013 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 1014 if (slot < 0) 1015 slot = 0; 1016 slot >>= rtab->rate.cell_log; 1017 if (slot > 255) 1018 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 1019 return rtab->data[slot]; 1020 } 1021 1022 struct psched_ratecfg { 1023 u64 rate_bytes_ps; /* bytes per second */ 1024 u32 mult; 1025 u16 overhead; 1026 u8 linklayer; 1027 u8 shift; 1028 }; 1029 1030 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 1031 unsigned int len) 1032 { 1033 len += r->overhead; 1034 1035 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 1036 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 1037 1038 return ((u64)len * r->mult) >> r->shift; 1039 } 1040 1041 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1042 const struct tc_ratespec *conf, 1043 u64 rate64); 1044 1045 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 1046 const struct psched_ratecfg *r) 1047 { 1048 memset(res, 0, sizeof(*res)); 1049 1050 /* legacy struct tc_ratespec has a 32bit @rate field 1051 * Qdisc using 64bit rate should add new attributes 1052 * in order to maintain compatibility. 1053 */ 1054 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 1055 1056 res->overhead = r->overhead; 1057 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 1058 } 1059 1060 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 1061 * The fast path only needs to access filter list and to update stats 1062 */ 1063 struct mini_Qdisc { 1064 struct tcf_proto *filter_list; 1065 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 1066 struct gnet_stats_queue __percpu *cpu_qstats; 1067 struct rcu_head rcu; 1068 }; 1069 1070 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 1071 const struct sk_buff *skb) 1072 { 1073 bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb); 1074 } 1075 1076 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 1077 { 1078 this_cpu_inc(miniq->cpu_qstats->drops); 1079 } 1080 1081 struct mini_Qdisc_pair { 1082 struct mini_Qdisc miniq1; 1083 struct mini_Qdisc miniq2; 1084 struct mini_Qdisc __rcu **p_miniq; 1085 }; 1086 1087 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1088 struct tcf_proto *tp_head); 1089 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1090 struct mini_Qdisc __rcu **p_miniq); 1091 1092 #endif 1093