1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <net/gen_stats.h> 16 #include <net/rtnetlink.h> 17 18 struct Qdisc_ops; 19 struct qdisc_walker; 20 struct tcf_walker; 21 struct module; 22 23 typedef int tc_setup_cb_t(enum tc_setup_type type, 24 void *type_data, void *cb_priv); 25 26 struct qdisc_rate_table { 27 struct tc_ratespec rate; 28 u32 data[256]; 29 struct qdisc_rate_table *next; 30 int refcnt; 31 }; 32 33 enum qdisc_state_t { 34 __QDISC_STATE_SCHED, 35 __QDISC_STATE_DEACTIVATED, 36 }; 37 38 struct qdisc_size_table { 39 struct rcu_head rcu; 40 struct list_head list; 41 struct tc_sizespec szopts; 42 int refcnt; 43 u16 data[]; 44 }; 45 46 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 47 struct qdisc_skb_head { 48 struct sk_buff *head; 49 struct sk_buff *tail; 50 __u32 qlen; 51 spinlock_t lock; 52 }; 53 54 struct Qdisc { 55 int (*enqueue)(struct sk_buff *skb, 56 struct Qdisc *sch, 57 struct sk_buff **to_free); 58 struct sk_buff * (*dequeue)(struct Qdisc *sch); 59 unsigned int flags; 60 #define TCQ_F_BUILTIN 1 61 #define TCQ_F_INGRESS 2 62 #define TCQ_F_CAN_BYPASS 4 63 #define TCQ_F_MQROOT 8 64 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 65 * q->dev_queue : It can test 66 * netif_xmit_frozen_or_stopped() before 67 * dequeueing next packet. 68 * Its true for MQ/MQPRIO slaves, or non 69 * multiqueue device. 70 */ 71 #define TCQ_F_WARN_NONWC (1 << 16) 72 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 73 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 74 * qdisc_tree_decrease_qlen() should stop. 75 */ 76 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 77 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 78 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */ 79 u32 limit; 80 const struct Qdisc_ops *ops; 81 struct qdisc_size_table __rcu *stab; 82 struct hlist_node hash; 83 u32 handle; 84 u32 parent; 85 86 struct netdev_queue *dev_queue; 87 88 struct net_rate_estimator __rcu *rate_est; 89 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 90 struct gnet_stats_queue __percpu *cpu_qstats; 91 int padded; 92 refcount_t refcnt; 93 94 /* 95 * For performance sake on SMP, we put highly modified fields at the end 96 */ 97 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 98 struct qdisc_skb_head q; 99 struct gnet_stats_basic_packed bstats; 100 seqcount_t running; 101 struct gnet_stats_queue qstats; 102 unsigned long state; 103 struct Qdisc *next_sched; 104 struct sk_buff_head skb_bad_txq; 105 106 spinlock_t busylock ____cacheline_aligned_in_smp; 107 spinlock_t seqlock; 108 }; 109 110 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 111 { 112 if (qdisc->flags & TCQ_F_BUILTIN) 113 return; 114 refcount_inc(&qdisc->refcnt); 115 } 116 117 static inline bool qdisc_is_running(struct Qdisc *qdisc) 118 { 119 if (qdisc->flags & TCQ_F_NOLOCK) 120 return spin_is_locked(&qdisc->seqlock); 121 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; 122 } 123 124 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 125 { 126 if (qdisc->flags & TCQ_F_NOLOCK) { 127 if (!spin_trylock(&qdisc->seqlock)) 128 return false; 129 } else if (qdisc_is_running(qdisc)) { 130 return false; 131 } 132 /* Variant of write_seqcount_begin() telling lockdep a trylock 133 * was attempted. 134 */ 135 raw_write_seqcount_begin(&qdisc->running); 136 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_); 137 return true; 138 } 139 140 static inline void qdisc_run_end(struct Qdisc *qdisc) 141 { 142 write_seqcount_end(&qdisc->running); 143 if (qdisc->flags & TCQ_F_NOLOCK) 144 spin_unlock(&qdisc->seqlock); 145 } 146 147 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 148 { 149 return qdisc->flags & TCQ_F_ONETXQUEUE; 150 } 151 152 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 153 { 154 #ifdef CONFIG_BQL 155 /* Non-BQL migrated drivers will return 0, too. */ 156 return dql_avail(&txq->dql); 157 #else 158 return 0; 159 #endif 160 } 161 162 struct Qdisc_class_ops { 163 /* Child qdisc manipulation */ 164 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 165 int (*graft)(struct Qdisc *, unsigned long cl, 166 struct Qdisc *, struct Qdisc **, 167 struct netlink_ext_ack *extack); 168 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 169 void (*qlen_notify)(struct Qdisc *, unsigned long); 170 171 /* Class manipulation routines */ 172 unsigned long (*find)(struct Qdisc *, u32 classid); 173 int (*change)(struct Qdisc *, u32, u32, 174 struct nlattr **, unsigned long *, 175 struct netlink_ext_ack *); 176 int (*delete)(struct Qdisc *, unsigned long); 177 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 178 179 /* Filter manipulation */ 180 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 181 unsigned long arg, 182 struct netlink_ext_ack *extack); 183 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 184 u32 classid); 185 void (*unbind_tcf)(struct Qdisc *, unsigned long); 186 187 /* rtnetlink specific */ 188 int (*dump)(struct Qdisc *, unsigned long, 189 struct sk_buff *skb, struct tcmsg*); 190 int (*dump_stats)(struct Qdisc *, unsigned long, 191 struct gnet_dump *); 192 }; 193 194 struct Qdisc_ops { 195 struct Qdisc_ops *next; 196 const struct Qdisc_class_ops *cl_ops; 197 char id[IFNAMSIZ]; 198 int priv_size; 199 unsigned int static_flags; 200 201 int (*enqueue)(struct sk_buff *skb, 202 struct Qdisc *sch, 203 struct sk_buff **to_free); 204 struct sk_buff * (*dequeue)(struct Qdisc *); 205 struct sk_buff * (*peek)(struct Qdisc *); 206 207 int (*init)(struct Qdisc *sch, struct nlattr *arg, 208 struct netlink_ext_ack *extack); 209 void (*reset)(struct Qdisc *); 210 void (*destroy)(struct Qdisc *); 211 int (*change)(struct Qdisc *sch, 212 struct nlattr *arg, 213 struct netlink_ext_ack *extack); 214 void (*attach)(struct Qdisc *sch); 215 int (*change_tx_queue_len)(struct Qdisc *, unsigned int); 216 217 int (*dump)(struct Qdisc *, struct sk_buff *); 218 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 219 220 void (*ingress_block_set)(struct Qdisc *sch, 221 u32 block_index); 222 void (*egress_block_set)(struct Qdisc *sch, 223 u32 block_index); 224 u32 (*ingress_block_get)(struct Qdisc *sch); 225 u32 (*egress_block_get)(struct Qdisc *sch); 226 227 struct module *owner; 228 }; 229 230 231 struct tcf_result { 232 union { 233 struct { 234 unsigned long class; 235 u32 classid; 236 }; 237 const struct tcf_proto *goto_tp; 238 239 /* used by the TC_ACT_REINSERT action */ 240 struct { 241 bool ingress; 242 struct gnet_stats_queue *qstats; 243 }; 244 }; 245 }; 246 247 struct tcf_chain; 248 249 struct tcf_proto_ops { 250 struct list_head head; 251 char kind[IFNAMSIZ]; 252 253 int (*classify)(struct sk_buff *, 254 const struct tcf_proto *, 255 struct tcf_result *); 256 int (*init)(struct tcf_proto*); 257 void (*destroy)(struct tcf_proto *tp, 258 struct netlink_ext_ack *extack); 259 260 void* (*get)(struct tcf_proto*, u32 handle); 261 int (*change)(struct net *net, struct sk_buff *, 262 struct tcf_proto*, unsigned long, 263 u32 handle, struct nlattr **, 264 void **, bool, 265 struct netlink_ext_ack *); 266 int (*delete)(struct tcf_proto *tp, void *arg, 267 bool *last, 268 struct netlink_ext_ack *); 269 void (*walk)(struct tcf_proto*, struct tcf_walker *arg); 270 int (*reoffload)(struct tcf_proto *tp, bool add, 271 tc_setup_cb_t *cb, void *cb_priv, 272 struct netlink_ext_ack *extack); 273 void (*bind_class)(void *, u32, unsigned long); 274 void * (*tmplt_create)(struct net *net, 275 struct tcf_chain *chain, 276 struct nlattr **tca, 277 struct netlink_ext_ack *extack); 278 void (*tmplt_destroy)(void *tmplt_priv); 279 280 /* rtnetlink specific */ 281 int (*dump)(struct net*, struct tcf_proto*, void *, 282 struct sk_buff *skb, struct tcmsg*); 283 int (*tmplt_dump)(struct sk_buff *skb, 284 struct net *net, 285 void *tmplt_priv); 286 287 struct module *owner; 288 }; 289 290 struct tcf_proto { 291 /* Fast access part */ 292 struct tcf_proto __rcu *next; 293 void __rcu *root; 294 295 /* called under RCU BH lock*/ 296 int (*classify)(struct sk_buff *, 297 const struct tcf_proto *, 298 struct tcf_result *); 299 __be16 protocol; 300 301 /* All the rest */ 302 u32 prio; 303 void *data; 304 const struct tcf_proto_ops *ops; 305 struct tcf_chain *chain; 306 struct rcu_head rcu; 307 }; 308 309 struct qdisc_skb_cb { 310 unsigned int pkt_len; 311 u16 slave_dev_queue_mapping; 312 u16 tc_classid; 313 #define QDISC_CB_PRIV_LEN 20 314 unsigned char data[QDISC_CB_PRIV_LEN]; 315 }; 316 317 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 318 319 struct tcf_chain { 320 struct tcf_proto __rcu *filter_chain; 321 struct list_head list; 322 struct tcf_block *block; 323 u32 index; /* chain index */ 324 unsigned int refcnt; 325 unsigned int action_refcnt; 326 bool explicitly_created; 327 const struct tcf_proto_ops *tmplt_ops; 328 void *tmplt_priv; 329 }; 330 331 struct tcf_block { 332 struct list_head chain_list; 333 u32 index; /* block index for shared blocks */ 334 unsigned int refcnt; 335 struct net *net; 336 struct Qdisc *q; 337 struct list_head cb_list; 338 struct list_head owner_list; 339 bool keep_dst; 340 unsigned int offloadcnt; /* Number of oddloaded filters */ 341 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ 342 struct { 343 struct tcf_chain *chain; 344 struct list_head filter_chain_list; 345 } chain0; 346 }; 347 348 static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) 349 { 350 if (*flags & TCA_CLS_FLAGS_IN_HW) 351 return; 352 *flags |= TCA_CLS_FLAGS_IN_HW; 353 block->offloadcnt++; 354 } 355 356 static inline void tcf_block_offload_dec(struct tcf_block *block, u32 *flags) 357 { 358 if (!(*flags & TCA_CLS_FLAGS_IN_HW)) 359 return; 360 *flags &= ~TCA_CLS_FLAGS_IN_HW; 361 block->offloadcnt--; 362 } 363 364 static inline void 365 tc_cls_offload_cnt_update(struct tcf_block *block, u32 *cnt, 366 u32 *flags, bool add) 367 { 368 if (add) { 369 if (!*cnt) 370 tcf_block_offload_inc(block, flags); 371 (*cnt)++; 372 } else { 373 (*cnt)--; 374 if (!*cnt) 375 tcf_block_offload_dec(block, flags); 376 } 377 } 378 379 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 380 { 381 struct qdisc_skb_cb *qcb; 382 383 BUILD_BUG_ON(sizeof(skb->cb) < offsetof(struct qdisc_skb_cb, data) + sz); 384 BUILD_BUG_ON(sizeof(qcb->data) < sz); 385 } 386 387 static inline int qdisc_qlen_cpu(const struct Qdisc *q) 388 { 389 return this_cpu_ptr(q->cpu_qstats)->qlen; 390 } 391 392 static inline int qdisc_qlen(const struct Qdisc *q) 393 { 394 return q->q.qlen; 395 } 396 397 static inline int qdisc_qlen_sum(const struct Qdisc *q) 398 { 399 __u32 qlen = q->qstats.qlen; 400 int i; 401 402 if (q->flags & TCQ_F_NOLOCK) { 403 for_each_possible_cpu(i) 404 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 405 } else { 406 qlen += q->q.qlen; 407 } 408 409 return qlen; 410 } 411 412 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 413 { 414 return (struct qdisc_skb_cb *)skb->cb; 415 } 416 417 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 418 { 419 return &qdisc->q.lock; 420 } 421 422 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 423 { 424 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 425 426 return q; 427 } 428 429 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 430 { 431 return qdisc->dev_queue->qdisc_sleeping; 432 } 433 434 /* The qdisc root lock is a mechanism by which to top level 435 * of a qdisc tree can be locked from any qdisc node in the 436 * forest. This allows changing the configuration of some 437 * aspect of the qdisc tree while blocking out asynchronous 438 * qdisc access in the packet processing paths. 439 * 440 * It is only legal to do this when the root will not change 441 * on us. Otherwise we'll potentially lock the wrong qdisc 442 * root. This is enforced by holding the RTNL semaphore, which 443 * all users of this lock accessor must do. 444 */ 445 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) 446 { 447 struct Qdisc *root = qdisc_root(qdisc); 448 449 ASSERT_RTNL(); 450 return qdisc_lock(root); 451 } 452 453 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 454 { 455 struct Qdisc *root = qdisc_root_sleeping(qdisc); 456 457 ASSERT_RTNL(); 458 return qdisc_lock(root); 459 } 460 461 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc) 462 { 463 struct Qdisc *root = qdisc_root_sleeping(qdisc); 464 465 ASSERT_RTNL(); 466 return &root->running; 467 } 468 469 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 470 { 471 return qdisc->dev_queue->dev; 472 } 473 474 static inline void sch_tree_lock(const struct Qdisc *q) 475 { 476 spin_lock_bh(qdisc_root_sleeping_lock(q)); 477 } 478 479 static inline void sch_tree_unlock(const struct Qdisc *q) 480 { 481 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 482 } 483 484 extern struct Qdisc noop_qdisc; 485 extern struct Qdisc_ops noop_qdisc_ops; 486 extern struct Qdisc_ops pfifo_fast_ops; 487 extern struct Qdisc_ops mq_qdisc_ops; 488 extern struct Qdisc_ops noqueue_qdisc_ops; 489 extern const struct Qdisc_ops *default_qdisc_ops; 490 static inline const struct Qdisc_ops * 491 get_default_qdisc_ops(const struct net_device *dev, int ntx) 492 { 493 return ntx < dev->real_num_tx_queues ? 494 default_qdisc_ops : &pfifo_fast_ops; 495 } 496 497 struct Qdisc_class_common { 498 u32 classid; 499 struct hlist_node hnode; 500 }; 501 502 struct Qdisc_class_hash { 503 struct hlist_head *hash; 504 unsigned int hashsize; 505 unsigned int hashmask; 506 unsigned int hashelems; 507 }; 508 509 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 510 { 511 id ^= id >> 8; 512 id ^= id >> 4; 513 return id & mask; 514 } 515 516 static inline struct Qdisc_class_common * 517 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 518 { 519 struct Qdisc_class_common *cl; 520 unsigned int h; 521 522 if (!id) 523 return NULL; 524 525 h = qdisc_class_hash(id, hash->hashmask); 526 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 527 if (cl->classid == id) 528 return cl; 529 } 530 return NULL; 531 } 532 533 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 534 { 535 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 536 537 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 538 } 539 540 int qdisc_class_hash_init(struct Qdisc_class_hash *); 541 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 542 struct Qdisc_class_common *); 543 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 544 struct Qdisc_class_common *); 545 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 546 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 547 548 int dev_qdisc_change_tx_queue_len(struct net_device *dev); 549 void dev_init_scheduler(struct net_device *dev); 550 void dev_shutdown(struct net_device *dev); 551 void dev_activate(struct net_device *dev); 552 void dev_deactivate(struct net_device *dev); 553 void dev_deactivate_many(struct list_head *head); 554 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 555 struct Qdisc *qdisc); 556 void qdisc_reset(struct Qdisc *qdisc); 557 void qdisc_destroy(struct Qdisc *qdisc); 558 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n, 559 unsigned int len); 560 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 561 const struct Qdisc_ops *ops, 562 struct netlink_ext_ack *extack); 563 void qdisc_free(struct Qdisc *qdisc); 564 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 565 const struct Qdisc_ops *ops, u32 parentid, 566 struct netlink_ext_ack *extack); 567 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 568 const struct qdisc_size_table *stab); 569 int skb_do_redirect(struct sk_buff *); 570 571 static inline void skb_reset_tc(struct sk_buff *skb) 572 { 573 #ifdef CONFIG_NET_CLS_ACT 574 skb->tc_redirected = 0; 575 #endif 576 } 577 578 static inline bool skb_is_tc_redirected(const struct sk_buff *skb) 579 { 580 #ifdef CONFIG_NET_CLS_ACT 581 return skb->tc_redirected; 582 #else 583 return false; 584 #endif 585 } 586 587 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 588 { 589 #ifdef CONFIG_NET_CLS_ACT 590 return skb->tc_at_ingress; 591 #else 592 return false; 593 #endif 594 } 595 596 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 597 { 598 #ifdef CONFIG_NET_CLS_ACT 599 if (skb->tc_skip_classify) { 600 skb->tc_skip_classify = 0; 601 return true; 602 } 603 #endif 604 return false; 605 } 606 607 /* Reset all TX qdiscs greater than index of a device. */ 608 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 609 { 610 struct Qdisc *qdisc; 611 612 for (; i < dev->num_tx_queues; i++) { 613 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 614 if (qdisc) { 615 spin_lock_bh(qdisc_lock(qdisc)); 616 qdisc_reset(qdisc); 617 spin_unlock_bh(qdisc_lock(qdisc)); 618 } 619 } 620 } 621 622 static inline void qdisc_reset_all_tx(struct net_device *dev) 623 { 624 qdisc_reset_all_tx_gt(dev, 0); 625 } 626 627 /* Are all TX queues of the device empty? */ 628 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 629 { 630 unsigned int i; 631 632 rcu_read_lock(); 633 for (i = 0; i < dev->num_tx_queues; i++) { 634 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 635 const struct Qdisc *q = rcu_dereference(txq->qdisc); 636 637 if (q->q.qlen) { 638 rcu_read_unlock(); 639 return false; 640 } 641 } 642 rcu_read_unlock(); 643 return true; 644 } 645 646 /* Are any of the TX qdiscs changing? */ 647 static inline bool qdisc_tx_changing(const struct net_device *dev) 648 { 649 unsigned int i; 650 651 for (i = 0; i < dev->num_tx_queues; i++) { 652 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 653 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping) 654 return true; 655 } 656 return false; 657 } 658 659 /* Is the device using the noop qdisc on all queues? */ 660 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 661 { 662 unsigned int i; 663 664 for (i = 0; i < dev->num_tx_queues; i++) { 665 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 666 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 667 return false; 668 } 669 return true; 670 } 671 672 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 673 { 674 return qdisc_skb_cb(skb)->pkt_len; 675 } 676 677 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 678 enum net_xmit_qdisc_t { 679 __NET_XMIT_STOLEN = 0x00010000, 680 __NET_XMIT_BYPASS = 0x00020000, 681 }; 682 683 #ifdef CONFIG_NET_CLS_ACT 684 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 685 #else 686 #define net_xmit_drop_count(e) (1) 687 #endif 688 689 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 690 const struct Qdisc *sch) 691 { 692 #ifdef CONFIG_NET_SCHED 693 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 694 695 if (stab) 696 __qdisc_calculate_pkt_len(skb, stab); 697 #endif 698 } 699 700 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 701 struct sk_buff **to_free) 702 { 703 qdisc_calculate_pkt_len(skb, sch); 704 return sch->enqueue(skb, sch, to_free); 705 } 706 707 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 708 { 709 return q->flags & TCQ_F_CPUSTATS; 710 } 711 712 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats, 713 __u64 bytes, __u32 packets) 714 { 715 bstats->bytes += bytes; 716 bstats->packets += packets; 717 } 718 719 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 720 const struct sk_buff *skb) 721 { 722 _bstats_update(bstats, 723 qdisc_pkt_len(skb), 724 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 725 } 726 727 static inline void _bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 728 __u64 bytes, __u32 packets) 729 { 730 u64_stats_update_begin(&bstats->syncp); 731 _bstats_update(&bstats->bstats, bytes, packets); 732 u64_stats_update_end(&bstats->syncp); 733 } 734 735 static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 736 const struct sk_buff *skb) 737 { 738 u64_stats_update_begin(&bstats->syncp); 739 bstats_update(&bstats->bstats, skb); 740 u64_stats_update_end(&bstats->syncp); 741 } 742 743 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 744 const struct sk_buff *skb) 745 { 746 bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb); 747 } 748 749 static inline void qdisc_bstats_update(struct Qdisc *sch, 750 const struct sk_buff *skb) 751 { 752 bstats_update(&sch->bstats, skb); 753 } 754 755 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 756 const struct sk_buff *skb) 757 { 758 sch->qstats.backlog -= qdisc_pkt_len(skb); 759 } 760 761 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 762 const struct sk_buff *skb) 763 { 764 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 765 } 766 767 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 768 const struct sk_buff *skb) 769 { 770 sch->qstats.backlog += qdisc_pkt_len(skb); 771 } 772 773 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 774 const struct sk_buff *skb) 775 { 776 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 777 } 778 779 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 780 { 781 this_cpu_inc(sch->cpu_qstats->qlen); 782 } 783 784 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 785 { 786 this_cpu_dec(sch->cpu_qstats->qlen); 787 } 788 789 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 790 { 791 this_cpu_inc(sch->cpu_qstats->requeues); 792 } 793 794 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 795 { 796 sch->qstats.drops += count; 797 } 798 799 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 800 { 801 qstats->drops++; 802 } 803 804 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 805 { 806 qstats->overlimits++; 807 } 808 809 static inline void qdisc_qstats_drop(struct Qdisc *sch) 810 { 811 qstats_drop_inc(&sch->qstats); 812 } 813 814 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 815 { 816 this_cpu_inc(sch->cpu_qstats->drops); 817 } 818 819 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 820 { 821 sch->qstats.overlimits++; 822 } 823 824 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) 825 { 826 qh->head = NULL; 827 qh->tail = NULL; 828 qh->qlen = 0; 829 } 830 831 static inline void __qdisc_enqueue_tail(struct sk_buff *skb, 832 struct qdisc_skb_head *qh) 833 { 834 struct sk_buff *last = qh->tail; 835 836 if (last) { 837 skb->next = NULL; 838 last->next = skb; 839 qh->tail = skb; 840 } else { 841 qh->tail = skb; 842 qh->head = skb; 843 } 844 qh->qlen++; 845 } 846 847 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 848 { 849 __qdisc_enqueue_tail(skb, &sch->q); 850 qdisc_qstats_backlog_inc(sch, skb); 851 return NET_XMIT_SUCCESS; 852 } 853 854 static inline void __qdisc_enqueue_head(struct sk_buff *skb, 855 struct qdisc_skb_head *qh) 856 { 857 skb->next = qh->head; 858 859 if (!qh->head) 860 qh->tail = skb; 861 qh->head = skb; 862 qh->qlen++; 863 } 864 865 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 866 { 867 struct sk_buff *skb = qh->head; 868 869 if (likely(skb != NULL)) { 870 qh->head = skb->next; 871 qh->qlen--; 872 if (qh->head == NULL) 873 qh->tail = NULL; 874 skb->next = NULL; 875 } 876 877 return skb; 878 } 879 880 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 881 { 882 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 883 884 if (likely(skb != NULL)) { 885 qdisc_qstats_backlog_dec(sch, skb); 886 qdisc_bstats_update(sch, skb); 887 } 888 889 return skb; 890 } 891 892 /* Instead of calling kfree_skb() while root qdisc lock is held, 893 * queue the skb for future freeing at end of __dev_xmit_skb() 894 */ 895 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 896 { 897 skb->next = *to_free; 898 *to_free = skb; 899 } 900 901 static inline void __qdisc_drop_all(struct sk_buff *skb, 902 struct sk_buff **to_free) 903 { 904 if (skb->prev) 905 skb->prev->next = *to_free; 906 else 907 skb->next = *to_free; 908 *to_free = skb; 909 } 910 911 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 912 struct qdisc_skb_head *qh, 913 struct sk_buff **to_free) 914 { 915 struct sk_buff *skb = __qdisc_dequeue_head(qh); 916 917 if (likely(skb != NULL)) { 918 unsigned int len = qdisc_pkt_len(skb); 919 920 qdisc_qstats_backlog_dec(sch, skb); 921 __qdisc_drop(skb, to_free); 922 return len; 923 } 924 925 return 0; 926 } 927 928 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch, 929 struct sk_buff **to_free) 930 { 931 return __qdisc_queue_drop_head(sch, &sch->q, to_free); 932 } 933 934 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 935 { 936 const struct qdisc_skb_head *qh = &sch->q; 937 938 return qh->head; 939 } 940 941 /* generic pseudo peek method for non-work-conserving qdisc */ 942 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 943 { 944 struct sk_buff *skb = skb_peek(&sch->gso_skb); 945 946 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 947 if (!skb) { 948 skb = sch->dequeue(sch); 949 950 if (skb) { 951 __skb_queue_head(&sch->gso_skb, skb); 952 /* it's still part of the queue */ 953 qdisc_qstats_backlog_inc(sch, skb); 954 sch->q.qlen++; 955 } 956 } 957 958 return skb; 959 } 960 961 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 962 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 963 { 964 struct sk_buff *skb = skb_peek(&sch->gso_skb); 965 966 if (skb) { 967 skb = __skb_dequeue(&sch->gso_skb); 968 qdisc_qstats_backlog_dec(sch, skb); 969 sch->q.qlen--; 970 } else { 971 skb = sch->dequeue(sch); 972 } 973 974 return skb; 975 } 976 977 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 978 { 979 /* 980 * We do not know the backlog in bytes of this list, it 981 * is up to the caller to correct it 982 */ 983 ASSERT_RTNL(); 984 if (qh->qlen) { 985 rtnl_kfree_skbs(qh->head, qh->tail); 986 987 qh->head = NULL; 988 qh->tail = NULL; 989 qh->qlen = 0; 990 } 991 } 992 993 static inline void qdisc_reset_queue(struct Qdisc *sch) 994 { 995 __qdisc_reset_queue(&sch->q); 996 sch->qstats.backlog = 0; 997 } 998 999 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 1000 struct Qdisc **pold) 1001 { 1002 struct Qdisc *old; 1003 1004 sch_tree_lock(sch); 1005 old = *pold; 1006 *pold = new; 1007 if (old != NULL) { 1008 unsigned int qlen = old->q.qlen; 1009 unsigned int backlog = old->qstats.backlog; 1010 1011 qdisc_reset(old); 1012 qdisc_tree_reduce_backlog(old, qlen, backlog); 1013 } 1014 sch_tree_unlock(sch); 1015 1016 return old; 1017 } 1018 1019 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 1020 { 1021 rtnl_kfree_skbs(skb, skb); 1022 qdisc_qstats_drop(sch); 1023 } 1024 1025 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 1026 struct sk_buff **to_free) 1027 { 1028 __qdisc_drop(skb, to_free); 1029 qdisc_qstats_cpu_drop(sch); 1030 1031 return NET_XMIT_DROP; 1032 } 1033 1034 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 1035 struct sk_buff **to_free) 1036 { 1037 __qdisc_drop(skb, to_free); 1038 qdisc_qstats_drop(sch); 1039 1040 return NET_XMIT_DROP; 1041 } 1042 1043 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, 1044 struct sk_buff **to_free) 1045 { 1046 __qdisc_drop_all(skb, to_free); 1047 qdisc_qstats_drop(sch); 1048 1049 return NET_XMIT_DROP; 1050 } 1051 1052 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 1053 long it will take to send a packet given its size. 1054 */ 1055 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 1056 { 1057 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 1058 if (slot < 0) 1059 slot = 0; 1060 slot >>= rtab->rate.cell_log; 1061 if (slot > 255) 1062 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 1063 return rtab->data[slot]; 1064 } 1065 1066 struct psched_ratecfg { 1067 u64 rate_bytes_ps; /* bytes per second */ 1068 u32 mult; 1069 u16 overhead; 1070 u8 linklayer; 1071 u8 shift; 1072 }; 1073 1074 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 1075 unsigned int len) 1076 { 1077 len += r->overhead; 1078 1079 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 1080 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 1081 1082 return ((u64)len * r->mult) >> r->shift; 1083 } 1084 1085 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1086 const struct tc_ratespec *conf, 1087 u64 rate64); 1088 1089 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 1090 const struct psched_ratecfg *r) 1091 { 1092 memset(res, 0, sizeof(*res)); 1093 1094 /* legacy struct tc_ratespec has a 32bit @rate field 1095 * Qdisc using 64bit rate should add new attributes 1096 * in order to maintain compatibility. 1097 */ 1098 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 1099 1100 res->overhead = r->overhead; 1101 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 1102 } 1103 1104 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 1105 * The fast path only needs to access filter list and to update stats 1106 */ 1107 struct mini_Qdisc { 1108 struct tcf_proto *filter_list; 1109 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 1110 struct gnet_stats_queue __percpu *cpu_qstats; 1111 struct rcu_head rcu; 1112 }; 1113 1114 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 1115 const struct sk_buff *skb) 1116 { 1117 bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb); 1118 } 1119 1120 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 1121 { 1122 this_cpu_inc(miniq->cpu_qstats->drops); 1123 } 1124 1125 struct mini_Qdisc_pair { 1126 struct mini_Qdisc miniq1; 1127 struct mini_Qdisc miniq2; 1128 struct mini_Qdisc __rcu **p_miniq; 1129 }; 1130 1131 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1132 struct tcf_proto *tp_head); 1133 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1134 struct mini_Qdisc __rcu **p_miniq); 1135 1136 static inline void skb_tc_reinsert(struct sk_buff *skb, struct tcf_result *res) 1137 { 1138 struct gnet_stats_queue *stats = res->qstats; 1139 int ret; 1140 1141 if (res->ingress) 1142 ret = netif_receive_skb(skb); 1143 else 1144 ret = dev_queue_xmit(skb); 1145 if (ret && stats) 1146 qstats_overlimit_inc(res->qstats); 1147 } 1148 1149 #endif 1150