1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <net/gen_stats.h> 16 #include <net/rtnetlink.h> 17 18 struct Qdisc_ops; 19 struct qdisc_walker; 20 struct tcf_walker; 21 struct module; 22 23 struct qdisc_rate_table { 24 struct tc_ratespec rate; 25 u32 data[256]; 26 struct qdisc_rate_table *next; 27 int refcnt; 28 }; 29 30 enum qdisc_state_t { 31 __QDISC_STATE_SCHED, 32 __QDISC_STATE_DEACTIVATED, 33 }; 34 35 struct qdisc_size_table { 36 struct rcu_head rcu; 37 struct list_head list; 38 struct tc_sizespec szopts; 39 int refcnt; 40 u16 data[]; 41 }; 42 43 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 44 struct qdisc_skb_head { 45 struct sk_buff *head; 46 struct sk_buff *tail; 47 __u32 qlen; 48 spinlock_t lock; 49 }; 50 51 struct Qdisc { 52 int (*enqueue)(struct sk_buff *skb, 53 struct Qdisc *sch, 54 struct sk_buff **to_free); 55 struct sk_buff * (*dequeue)(struct Qdisc *sch); 56 unsigned int flags; 57 #define TCQ_F_BUILTIN 1 58 #define TCQ_F_INGRESS 2 59 #define TCQ_F_CAN_BYPASS 4 60 #define TCQ_F_MQROOT 8 61 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 62 * q->dev_queue : It can test 63 * netif_xmit_frozen_or_stopped() before 64 * dequeueing next packet. 65 * Its true for MQ/MQPRIO slaves, or non 66 * multiqueue device. 67 */ 68 #define TCQ_F_WARN_NONWC (1 << 16) 69 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 70 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 71 * qdisc_tree_decrease_qlen() should stop. 72 */ 73 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 74 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 75 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */ 76 u32 limit; 77 const struct Qdisc_ops *ops; 78 struct qdisc_size_table __rcu *stab; 79 struct hlist_node hash; 80 u32 handle; 81 u32 parent; 82 83 struct netdev_queue *dev_queue; 84 85 struct net_rate_estimator __rcu *rate_est; 86 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 87 struct gnet_stats_queue __percpu *cpu_qstats; 88 int padded; 89 refcount_t refcnt; 90 91 /* 92 * For performance sake on SMP, we put highly modified fields at the end 93 */ 94 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 95 struct qdisc_skb_head q; 96 struct gnet_stats_basic_packed bstats; 97 seqcount_t running; 98 struct gnet_stats_queue qstats; 99 unsigned long state; 100 struct Qdisc *next_sched; 101 struct sk_buff_head skb_bad_txq; 102 103 spinlock_t busylock ____cacheline_aligned_in_smp; 104 spinlock_t seqlock; 105 }; 106 107 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 108 { 109 if (qdisc->flags & TCQ_F_BUILTIN) 110 return; 111 refcount_inc(&qdisc->refcnt); 112 } 113 114 static inline bool qdisc_is_running(struct Qdisc *qdisc) 115 { 116 if (qdisc->flags & TCQ_F_NOLOCK) 117 return spin_is_locked(&qdisc->seqlock); 118 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; 119 } 120 121 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 122 { 123 if (qdisc->flags & TCQ_F_NOLOCK) { 124 if (!spin_trylock(&qdisc->seqlock)) 125 return false; 126 } else if (qdisc_is_running(qdisc)) { 127 return false; 128 } 129 /* Variant of write_seqcount_begin() telling lockdep a trylock 130 * was attempted. 131 */ 132 raw_write_seqcount_begin(&qdisc->running); 133 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_); 134 return true; 135 } 136 137 static inline void qdisc_run_end(struct Qdisc *qdisc) 138 { 139 write_seqcount_end(&qdisc->running); 140 if (qdisc->flags & TCQ_F_NOLOCK) 141 spin_unlock(&qdisc->seqlock); 142 } 143 144 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 145 { 146 return qdisc->flags & TCQ_F_ONETXQUEUE; 147 } 148 149 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 150 { 151 #ifdef CONFIG_BQL 152 /* Non-BQL migrated drivers will return 0, too. */ 153 return dql_avail(&txq->dql); 154 #else 155 return 0; 156 #endif 157 } 158 159 struct Qdisc_class_ops { 160 /* Child qdisc manipulation */ 161 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 162 int (*graft)(struct Qdisc *, unsigned long cl, 163 struct Qdisc *, struct Qdisc **, 164 struct netlink_ext_ack *extack); 165 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 166 void (*qlen_notify)(struct Qdisc *, unsigned long); 167 168 /* Class manipulation routines */ 169 unsigned long (*find)(struct Qdisc *, u32 classid); 170 int (*change)(struct Qdisc *, u32, u32, 171 struct nlattr **, unsigned long *, 172 struct netlink_ext_ack *); 173 int (*delete)(struct Qdisc *, unsigned long); 174 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 175 176 /* Filter manipulation */ 177 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 178 unsigned long arg, 179 struct netlink_ext_ack *extack); 180 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 181 u32 classid); 182 void (*unbind_tcf)(struct Qdisc *, unsigned long); 183 184 /* rtnetlink specific */ 185 int (*dump)(struct Qdisc *, unsigned long, 186 struct sk_buff *skb, struct tcmsg*); 187 int (*dump_stats)(struct Qdisc *, unsigned long, 188 struct gnet_dump *); 189 }; 190 191 struct Qdisc_ops { 192 struct Qdisc_ops *next; 193 const struct Qdisc_class_ops *cl_ops; 194 char id[IFNAMSIZ]; 195 int priv_size; 196 unsigned int static_flags; 197 198 int (*enqueue)(struct sk_buff *skb, 199 struct Qdisc *sch, 200 struct sk_buff **to_free); 201 struct sk_buff * (*dequeue)(struct Qdisc *); 202 struct sk_buff * (*peek)(struct Qdisc *); 203 204 int (*init)(struct Qdisc *sch, struct nlattr *arg, 205 struct netlink_ext_ack *extack); 206 void (*reset)(struct Qdisc *); 207 void (*destroy)(struct Qdisc *); 208 int (*change)(struct Qdisc *sch, 209 struct nlattr *arg, 210 struct netlink_ext_ack *extack); 211 void (*attach)(struct Qdisc *sch); 212 int (*change_tx_queue_len)(struct Qdisc *, unsigned int); 213 214 int (*dump)(struct Qdisc *, struct sk_buff *); 215 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 216 217 void (*ingress_block_set)(struct Qdisc *sch, 218 u32 block_index); 219 void (*egress_block_set)(struct Qdisc *sch, 220 u32 block_index); 221 u32 (*ingress_block_get)(struct Qdisc *sch); 222 u32 (*egress_block_get)(struct Qdisc *sch); 223 224 struct module *owner; 225 }; 226 227 228 struct tcf_result { 229 union { 230 struct { 231 unsigned long class; 232 u32 classid; 233 }; 234 const struct tcf_proto *goto_tp; 235 }; 236 }; 237 238 struct tcf_proto_ops { 239 struct list_head head; 240 char kind[IFNAMSIZ]; 241 242 int (*classify)(struct sk_buff *, 243 const struct tcf_proto *, 244 struct tcf_result *); 245 int (*init)(struct tcf_proto*); 246 void (*destroy)(struct tcf_proto *tp, 247 struct netlink_ext_ack *extack); 248 249 void* (*get)(struct tcf_proto*, u32 handle); 250 int (*change)(struct net *net, struct sk_buff *, 251 struct tcf_proto*, unsigned long, 252 u32 handle, struct nlattr **, 253 void **, bool, 254 struct netlink_ext_ack *); 255 int (*delete)(struct tcf_proto *tp, void *arg, 256 bool *last, 257 struct netlink_ext_ack *); 258 void (*walk)(struct tcf_proto*, struct tcf_walker *arg); 259 void (*bind_class)(void *, u32, unsigned long); 260 261 /* rtnetlink specific */ 262 int (*dump)(struct net*, struct tcf_proto*, void *, 263 struct sk_buff *skb, struct tcmsg*); 264 265 struct module *owner; 266 }; 267 268 struct tcf_proto { 269 /* Fast access part */ 270 struct tcf_proto __rcu *next; 271 void __rcu *root; 272 int (*classify)(struct sk_buff *, 273 const struct tcf_proto *, 274 struct tcf_result *); 275 __be16 protocol; 276 277 /* All the rest */ 278 u32 prio; 279 void *data; 280 const struct tcf_proto_ops *ops; 281 struct tcf_chain *chain; 282 struct rcu_head rcu; 283 }; 284 285 struct qdisc_skb_cb { 286 unsigned int pkt_len; 287 u16 slave_dev_queue_mapping; 288 u16 tc_classid; 289 #define QDISC_CB_PRIV_LEN 20 290 unsigned char data[QDISC_CB_PRIV_LEN]; 291 }; 292 293 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 294 295 struct tcf_chain { 296 struct tcf_proto __rcu *filter_chain; 297 struct list_head filter_chain_list; 298 struct list_head list; 299 struct tcf_block *block; 300 u32 index; /* chain index */ 301 unsigned int refcnt; 302 }; 303 304 struct tcf_block { 305 struct list_head chain_list; 306 u32 index; /* block index for shared blocks */ 307 unsigned int refcnt; 308 struct net *net; 309 struct Qdisc *q; 310 struct list_head cb_list; 311 struct list_head owner_list; 312 bool keep_dst; 313 unsigned int offloadcnt; /* Number of oddloaded filters */ 314 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ 315 }; 316 317 static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) 318 { 319 if (*flags & TCA_CLS_FLAGS_IN_HW) 320 return; 321 *flags |= TCA_CLS_FLAGS_IN_HW; 322 block->offloadcnt++; 323 } 324 325 static inline void tcf_block_offload_dec(struct tcf_block *block, u32 *flags) 326 { 327 if (!(*flags & TCA_CLS_FLAGS_IN_HW)) 328 return; 329 *flags &= ~TCA_CLS_FLAGS_IN_HW; 330 block->offloadcnt--; 331 } 332 333 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 334 { 335 struct qdisc_skb_cb *qcb; 336 337 BUILD_BUG_ON(sizeof(skb->cb) < offsetof(struct qdisc_skb_cb, data) + sz); 338 BUILD_BUG_ON(sizeof(qcb->data) < sz); 339 } 340 341 static inline int qdisc_qlen_cpu(const struct Qdisc *q) 342 { 343 return this_cpu_ptr(q->cpu_qstats)->qlen; 344 } 345 346 static inline int qdisc_qlen(const struct Qdisc *q) 347 { 348 return q->q.qlen; 349 } 350 351 static inline int qdisc_qlen_sum(const struct Qdisc *q) 352 { 353 __u32 qlen = q->qstats.qlen; 354 int i; 355 356 if (q->flags & TCQ_F_NOLOCK) { 357 for_each_possible_cpu(i) 358 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 359 } else { 360 qlen += q->q.qlen; 361 } 362 363 return qlen; 364 } 365 366 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 367 { 368 return (struct qdisc_skb_cb *)skb->cb; 369 } 370 371 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 372 { 373 return &qdisc->q.lock; 374 } 375 376 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 377 { 378 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 379 380 return q; 381 } 382 383 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 384 { 385 return qdisc->dev_queue->qdisc_sleeping; 386 } 387 388 /* The qdisc root lock is a mechanism by which to top level 389 * of a qdisc tree can be locked from any qdisc node in the 390 * forest. This allows changing the configuration of some 391 * aspect of the qdisc tree while blocking out asynchronous 392 * qdisc access in the packet processing paths. 393 * 394 * It is only legal to do this when the root will not change 395 * on us. Otherwise we'll potentially lock the wrong qdisc 396 * root. This is enforced by holding the RTNL semaphore, which 397 * all users of this lock accessor must do. 398 */ 399 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) 400 { 401 struct Qdisc *root = qdisc_root(qdisc); 402 403 ASSERT_RTNL(); 404 return qdisc_lock(root); 405 } 406 407 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 408 { 409 struct Qdisc *root = qdisc_root_sleeping(qdisc); 410 411 ASSERT_RTNL(); 412 return qdisc_lock(root); 413 } 414 415 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc) 416 { 417 struct Qdisc *root = qdisc_root_sleeping(qdisc); 418 419 ASSERT_RTNL(); 420 return &root->running; 421 } 422 423 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 424 { 425 return qdisc->dev_queue->dev; 426 } 427 428 static inline void sch_tree_lock(const struct Qdisc *q) 429 { 430 spin_lock_bh(qdisc_root_sleeping_lock(q)); 431 } 432 433 static inline void sch_tree_unlock(const struct Qdisc *q) 434 { 435 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 436 } 437 438 extern struct Qdisc noop_qdisc; 439 extern struct Qdisc_ops noop_qdisc_ops; 440 extern struct Qdisc_ops pfifo_fast_ops; 441 extern struct Qdisc_ops mq_qdisc_ops; 442 extern struct Qdisc_ops noqueue_qdisc_ops; 443 extern const struct Qdisc_ops *default_qdisc_ops; 444 static inline const struct Qdisc_ops * 445 get_default_qdisc_ops(const struct net_device *dev, int ntx) 446 { 447 return ntx < dev->real_num_tx_queues ? 448 default_qdisc_ops : &pfifo_fast_ops; 449 } 450 451 struct Qdisc_class_common { 452 u32 classid; 453 struct hlist_node hnode; 454 }; 455 456 struct Qdisc_class_hash { 457 struct hlist_head *hash; 458 unsigned int hashsize; 459 unsigned int hashmask; 460 unsigned int hashelems; 461 }; 462 463 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 464 { 465 id ^= id >> 8; 466 id ^= id >> 4; 467 return id & mask; 468 } 469 470 static inline struct Qdisc_class_common * 471 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 472 { 473 struct Qdisc_class_common *cl; 474 unsigned int h; 475 476 if (!id) 477 return NULL; 478 479 h = qdisc_class_hash(id, hash->hashmask); 480 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 481 if (cl->classid == id) 482 return cl; 483 } 484 return NULL; 485 } 486 487 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 488 { 489 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 490 491 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 492 } 493 494 int qdisc_class_hash_init(struct Qdisc_class_hash *); 495 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 496 struct Qdisc_class_common *); 497 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 498 struct Qdisc_class_common *); 499 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 500 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 501 502 int dev_qdisc_change_tx_queue_len(struct net_device *dev); 503 void dev_init_scheduler(struct net_device *dev); 504 void dev_shutdown(struct net_device *dev); 505 void dev_activate(struct net_device *dev); 506 void dev_deactivate(struct net_device *dev); 507 void dev_deactivate_many(struct list_head *head); 508 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 509 struct Qdisc *qdisc); 510 void qdisc_reset(struct Qdisc *qdisc); 511 void qdisc_destroy(struct Qdisc *qdisc); 512 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n, 513 unsigned int len); 514 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 515 const struct Qdisc_ops *ops, 516 struct netlink_ext_ack *extack); 517 void qdisc_free(struct Qdisc *qdisc); 518 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 519 const struct Qdisc_ops *ops, u32 parentid, 520 struct netlink_ext_ack *extack); 521 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 522 const struct qdisc_size_table *stab); 523 int skb_do_redirect(struct sk_buff *); 524 525 static inline void skb_reset_tc(struct sk_buff *skb) 526 { 527 #ifdef CONFIG_NET_CLS_ACT 528 skb->tc_redirected = 0; 529 #endif 530 } 531 532 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 533 { 534 #ifdef CONFIG_NET_CLS_ACT 535 return skb->tc_at_ingress; 536 #else 537 return false; 538 #endif 539 } 540 541 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 542 { 543 #ifdef CONFIG_NET_CLS_ACT 544 if (skb->tc_skip_classify) { 545 skb->tc_skip_classify = 0; 546 return true; 547 } 548 #endif 549 return false; 550 } 551 552 /* Reset all TX qdiscs greater than index of a device. */ 553 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 554 { 555 struct Qdisc *qdisc; 556 557 for (; i < dev->num_tx_queues; i++) { 558 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 559 if (qdisc) { 560 spin_lock_bh(qdisc_lock(qdisc)); 561 qdisc_reset(qdisc); 562 spin_unlock_bh(qdisc_lock(qdisc)); 563 } 564 } 565 } 566 567 static inline void qdisc_reset_all_tx(struct net_device *dev) 568 { 569 qdisc_reset_all_tx_gt(dev, 0); 570 } 571 572 /* Are all TX queues of the device empty? */ 573 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 574 { 575 unsigned int i; 576 577 rcu_read_lock(); 578 for (i = 0; i < dev->num_tx_queues; i++) { 579 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 580 const struct Qdisc *q = rcu_dereference(txq->qdisc); 581 582 if (q->q.qlen) { 583 rcu_read_unlock(); 584 return false; 585 } 586 } 587 rcu_read_unlock(); 588 return true; 589 } 590 591 /* Are any of the TX qdiscs changing? */ 592 static inline bool qdisc_tx_changing(const struct net_device *dev) 593 { 594 unsigned int i; 595 596 for (i = 0; i < dev->num_tx_queues; i++) { 597 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 598 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping) 599 return true; 600 } 601 return false; 602 } 603 604 /* Is the device using the noop qdisc on all queues? */ 605 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 606 { 607 unsigned int i; 608 609 for (i = 0; i < dev->num_tx_queues; i++) { 610 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 611 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 612 return false; 613 } 614 return true; 615 } 616 617 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 618 { 619 return qdisc_skb_cb(skb)->pkt_len; 620 } 621 622 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 623 enum net_xmit_qdisc_t { 624 __NET_XMIT_STOLEN = 0x00010000, 625 __NET_XMIT_BYPASS = 0x00020000, 626 }; 627 628 #ifdef CONFIG_NET_CLS_ACT 629 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 630 #else 631 #define net_xmit_drop_count(e) (1) 632 #endif 633 634 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 635 const struct Qdisc *sch) 636 { 637 #ifdef CONFIG_NET_SCHED 638 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 639 640 if (stab) 641 __qdisc_calculate_pkt_len(skb, stab); 642 #endif 643 } 644 645 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 646 struct sk_buff **to_free) 647 { 648 qdisc_calculate_pkt_len(skb, sch); 649 return sch->enqueue(skb, sch, to_free); 650 } 651 652 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 653 { 654 return q->flags & TCQ_F_CPUSTATS; 655 } 656 657 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats, 658 __u64 bytes, __u32 packets) 659 { 660 bstats->bytes += bytes; 661 bstats->packets += packets; 662 } 663 664 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 665 const struct sk_buff *skb) 666 { 667 _bstats_update(bstats, 668 qdisc_pkt_len(skb), 669 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 670 } 671 672 static inline void _bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 673 __u64 bytes, __u32 packets) 674 { 675 u64_stats_update_begin(&bstats->syncp); 676 _bstats_update(&bstats->bstats, bytes, packets); 677 u64_stats_update_end(&bstats->syncp); 678 } 679 680 static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 681 const struct sk_buff *skb) 682 { 683 u64_stats_update_begin(&bstats->syncp); 684 bstats_update(&bstats->bstats, skb); 685 u64_stats_update_end(&bstats->syncp); 686 } 687 688 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 689 const struct sk_buff *skb) 690 { 691 bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb); 692 } 693 694 static inline void qdisc_bstats_update(struct Qdisc *sch, 695 const struct sk_buff *skb) 696 { 697 bstats_update(&sch->bstats, skb); 698 } 699 700 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 701 const struct sk_buff *skb) 702 { 703 sch->qstats.backlog -= qdisc_pkt_len(skb); 704 } 705 706 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 707 const struct sk_buff *skb) 708 { 709 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 710 } 711 712 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 713 const struct sk_buff *skb) 714 { 715 sch->qstats.backlog += qdisc_pkt_len(skb); 716 } 717 718 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 719 const struct sk_buff *skb) 720 { 721 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 722 } 723 724 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 725 { 726 this_cpu_inc(sch->cpu_qstats->qlen); 727 } 728 729 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 730 { 731 this_cpu_dec(sch->cpu_qstats->qlen); 732 } 733 734 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 735 { 736 this_cpu_inc(sch->cpu_qstats->requeues); 737 } 738 739 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 740 { 741 sch->qstats.drops += count; 742 } 743 744 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 745 { 746 qstats->drops++; 747 } 748 749 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 750 { 751 qstats->overlimits++; 752 } 753 754 static inline void qdisc_qstats_drop(struct Qdisc *sch) 755 { 756 qstats_drop_inc(&sch->qstats); 757 } 758 759 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 760 { 761 this_cpu_inc(sch->cpu_qstats->drops); 762 } 763 764 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 765 { 766 sch->qstats.overlimits++; 767 } 768 769 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) 770 { 771 qh->head = NULL; 772 qh->tail = NULL; 773 qh->qlen = 0; 774 } 775 776 static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, 777 struct qdisc_skb_head *qh) 778 { 779 struct sk_buff *last = qh->tail; 780 781 if (last) { 782 skb->next = NULL; 783 last->next = skb; 784 qh->tail = skb; 785 } else { 786 qh->tail = skb; 787 qh->head = skb; 788 } 789 qh->qlen++; 790 qdisc_qstats_backlog_inc(sch, skb); 791 792 return NET_XMIT_SUCCESS; 793 } 794 795 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 796 { 797 return __qdisc_enqueue_tail(skb, sch, &sch->q); 798 } 799 800 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 801 { 802 struct sk_buff *skb = qh->head; 803 804 if (likely(skb != NULL)) { 805 qh->head = skb->next; 806 qh->qlen--; 807 if (qh->head == NULL) 808 qh->tail = NULL; 809 skb->next = NULL; 810 } 811 812 return skb; 813 } 814 815 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 816 { 817 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 818 819 if (likely(skb != NULL)) { 820 qdisc_qstats_backlog_dec(sch, skb); 821 qdisc_bstats_update(sch, skb); 822 } 823 824 return skb; 825 } 826 827 /* Instead of calling kfree_skb() while root qdisc lock is held, 828 * queue the skb for future freeing at end of __dev_xmit_skb() 829 */ 830 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 831 { 832 skb->next = *to_free; 833 *to_free = skb; 834 } 835 836 static inline void __qdisc_drop_all(struct sk_buff *skb, 837 struct sk_buff **to_free) 838 { 839 if (skb->prev) 840 skb->prev->next = *to_free; 841 else 842 skb->next = *to_free; 843 *to_free = skb; 844 } 845 846 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 847 struct qdisc_skb_head *qh, 848 struct sk_buff **to_free) 849 { 850 struct sk_buff *skb = __qdisc_dequeue_head(qh); 851 852 if (likely(skb != NULL)) { 853 unsigned int len = qdisc_pkt_len(skb); 854 855 qdisc_qstats_backlog_dec(sch, skb); 856 __qdisc_drop(skb, to_free); 857 return len; 858 } 859 860 return 0; 861 } 862 863 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch, 864 struct sk_buff **to_free) 865 { 866 return __qdisc_queue_drop_head(sch, &sch->q, to_free); 867 } 868 869 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 870 { 871 const struct qdisc_skb_head *qh = &sch->q; 872 873 return qh->head; 874 } 875 876 /* generic pseudo peek method for non-work-conserving qdisc */ 877 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 878 { 879 struct sk_buff *skb = skb_peek(&sch->gso_skb); 880 881 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 882 if (!skb) { 883 skb = sch->dequeue(sch); 884 885 if (skb) { 886 __skb_queue_head(&sch->gso_skb, skb); 887 /* it's still part of the queue */ 888 qdisc_qstats_backlog_inc(sch, skb); 889 sch->q.qlen++; 890 } 891 } 892 893 return skb; 894 } 895 896 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 897 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 898 { 899 struct sk_buff *skb = skb_peek(&sch->gso_skb); 900 901 if (skb) { 902 skb = __skb_dequeue(&sch->gso_skb); 903 qdisc_qstats_backlog_dec(sch, skb); 904 sch->q.qlen--; 905 } else { 906 skb = sch->dequeue(sch); 907 } 908 909 return skb; 910 } 911 912 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 913 { 914 /* 915 * We do not know the backlog in bytes of this list, it 916 * is up to the caller to correct it 917 */ 918 ASSERT_RTNL(); 919 if (qh->qlen) { 920 rtnl_kfree_skbs(qh->head, qh->tail); 921 922 qh->head = NULL; 923 qh->tail = NULL; 924 qh->qlen = 0; 925 } 926 } 927 928 static inline void qdisc_reset_queue(struct Qdisc *sch) 929 { 930 __qdisc_reset_queue(&sch->q); 931 sch->qstats.backlog = 0; 932 } 933 934 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 935 struct Qdisc **pold) 936 { 937 struct Qdisc *old; 938 939 sch_tree_lock(sch); 940 old = *pold; 941 *pold = new; 942 if (old != NULL) { 943 unsigned int qlen = old->q.qlen; 944 unsigned int backlog = old->qstats.backlog; 945 946 qdisc_reset(old); 947 qdisc_tree_reduce_backlog(old, qlen, backlog); 948 } 949 sch_tree_unlock(sch); 950 951 return old; 952 } 953 954 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 955 { 956 rtnl_kfree_skbs(skb, skb); 957 qdisc_qstats_drop(sch); 958 } 959 960 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 961 struct sk_buff **to_free) 962 { 963 __qdisc_drop(skb, to_free); 964 qdisc_qstats_cpu_drop(sch); 965 966 return NET_XMIT_DROP; 967 } 968 969 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 970 struct sk_buff **to_free) 971 { 972 __qdisc_drop(skb, to_free); 973 qdisc_qstats_drop(sch); 974 975 return NET_XMIT_DROP; 976 } 977 978 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, 979 struct sk_buff **to_free) 980 { 981 __qdisc_drop_all(skb, to_free); 982 qdisc_qstats_drop(sch); 983 984 return NET_XMIT_DROP; 985 } 986 987 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 988 long it will take to send a packet given its size. 989 */ 990 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 991 { 992 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 993 if (slot < 0) 994 slot = 0; 995 slot >>= rtab->rate.cell_log; 996 if (slot > 255) 997 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 998 return rtab->data[slot]; 999 } 1000 1001 struct psched_ratecfg { 1002 u64 rate_bytes_ps; /* bytes per second */ 1003 u32 mult; 1004 u16 overhead; 1005 u8 linklayer; 1006 u8 shift; 1007 }; 1008 1009 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 1010 unsigned int len) 1011 { 1012 len += r->overhead; 1013 1014 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 1015 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 1016 1017 return ((u64)len * r->mult) >> r->shift; 1018 } 1019 1020 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1021 const struct tc_ratespec *conf, 1022 u64 rate64); 1023 1024 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 1025 const struct psched_ratecfg *r) 1026 { 1027 memset(res, 0, sizeof(*res)); 1028 1029 /* legacy struct tc_ratespec has a 32bit @rate field 1030 * Qdisc using 64bit rate should add new attributes 1031 * in order to maintain compatibility. 1032 */ 1033 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 1034 1035 res->overhead = r->overhead; 1036 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 1037 } 1038 1039 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 1040 * The fast path only needs to access filter list and to update stats 1041 */ 1042 struct mini_Qdisc { 1043 struct tcf_proto *filter_list; 1044 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 1045 struct gnet_stats_queue __percpu *cpu_qstats; 1046 struct rcu_head rcu; 1047 }; 1048 1049 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 1050 const struct sk_buff *skb) 1051 { 1052 bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb); 1053 } 1054 1055 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 1056 { 1057 this_cpu_inc(miniq->cpu_qstats->drops); 1058 } 1059 1060 struct mini_Qdisc_pair { 1061 struct mini_Qdisc miniq1; 1062 struct mini_Qdisc miniq2; 1063 struct mini_Qdisc __rcu **p_miniq; 1064 }; 1065 1066 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1067 struct tcf_proto *tp_head); 1068 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1069 struct mini_Qdisc __rcu **p_miniq); 1070 1071 #endif 1072