1 // SPDX-License-Identifier: GPL-2.0 2 3 /* net/sched/sch_taprio.c Time Aware Priority Scheduler 4 * 5 * Authors: Vinicius Costa Gomes <vinicius.gomes@intel.com> 6 * 7 */ 8 9 #include <linux/ethtool.h> 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/kernel.h> 13 #include <linux/string.h> 14 #include <linux/list.h> 15 #include <linux/errno.h> 16 #include <linux/skbuff.h> 17 #include <linux/math64.h> 18 #include <linux/module.h> 19 #include <linux/spinlock.h> 20 #include <linux/rcupdate.h> 21 #include <linux/time.h> 22 #include <net/netlink.h> 23 #include <net/pkt_sched.h> 24 #include <net/pkt_cls.h> 25 #include <net/sch_generic.h> 26 #include <net/sock.h> 27 #include <net/tcp.h> 28 29 static LIST_HEAD(taprio_list); 30 static DEFINE_SPINLOCK(taprio_list_lock); 31 32 #define TAPRIO_ALL_GATES_OPEN -1 33 34 #define TXTIME_ASSIST_IS_ENABLED(flags) ((flags) & TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST) 35 #define FULL_OFFLOAD_IS_ENABLED(flags) ((flags) & TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD) 36 #define TAPRIO_FLAGS_INVALID U32_MAX 37 38 struct sched_entry { 39 struct list_head list; 40 41 /* The instant that this entry "closes" and the next one 42 * should open, the qdisc will make some effort so that no 43 * packet leaves after this time. 44 */ 45 ktime_t close_time; 46 ktime_t next_txtime; 47 atomic_t budget; 48 int index; 49 u32 gate_mask; 50 u32 interval; 51 u8 command; 52 }; 53 54 struct sched_gate_list { 55 struct rcu_head rcu; 56 struct list_head entries; 57 size_t num_entries; 58 ktime_t cycle_close_time; 59 s64 cycle_time; 60 s64 cycle_time_extension; 61 s64 base_time; 62 }; 63 64 struct taprio_sched { 65 struct Qdisc **qdiscs; 66 struct Qdisc *root; 67 u32 flags; 68 enum tk_offsets tk_offset; 69 int clockid; 70 atomic64_t picos_per_byte; /* Using picoseconds because for 10Gbps+ 71 * speeds it's sub-nanoseconds per byte 72 */ 73 74 /* Protects the update side of the RCU protected current_entry */ 75 spinlock_t current_entry_lock; 76 struct sched_entry __rcu *current_entry; 77 struct sched_gate_list __rcu *oper_sched; 78 struct sched_gate_list __rcu *admin_sched; 79 struct hrtimer advance_timer; 80 struct list_head taprio_list; 81 struct sk_buff *(*dequeue)(struct Qdisc *sch); 82 struct sk_buff *(*peek)(struct Qdisc *sch); 83 u32 txtime_delay; 84 }; 85 86 struct __tc_taprio_qopt_offload { 87 refcount_t users; 88 struct tc_taprio_qopt_offload offload; 89 }; 90 91 static ktime_t sched_base_time(const struct sched_gate_list *sched) 92 { 93 if (!sched) 94 return KTIME_MAX; 95 96 return ns_to_ktime(sched->base_time); 97 } 98 99 static ktime_t taprio_mono_to_any(const struct taprio_sched *q, ktime_t mono) 100 { 101 /* This pairs with WRITE_ONCE() in taprio_parse_clockid() */ 102 enum tk_offsets tk_offset = READ_ONCE(q->tk_offset); 103 104 switch (tk_offset) { 105 case TK_OFFS_MAX: 106 return mono; 107 default: 108 return ktime_mono_to_any(mono, tk_offset); 109 } 110 } 111 112 static ktime_t taprio_get_time(const struct taprio_sched *q) 113 { 114 return taprio_mono_to_any(q, ktime_get()); 115 } 116 117 static void taprio_free_sched_cb(struct rcu_head *head) 118 { 119 struct sched_gate_list *sched = container_of(head, struct sched_gate_list, rcu); 120 struct sched_entry *entry, *n; 121 122 list_for_each_entry_safe(entry, n, &sched->entries, list) { 123 list_del(&entry->list); 124 kfree(entry); 125 } 126 127 kfree(sched); 128 } 129 130 static void switch_schedules(struct taprio_sched *q, 131 struct sched_gate_list **admin, 132 struct sched_gate_list **oper) 133 { 134 rcu_assign_pointer(q->oper_sched, *admin); 135 rcu_assign_pointer(q->admin_sched, NULL); 136 137 if (*oper) 138 call_rcu(&(*oper)->rcu, taprio_free_sched_cb); 139 140 *oper = *admin; 141 *admin = NULL; 142 } 143 144 /* Get how much time has been already elapsed in the current cycle. */ 145 static s32 get_cycle_time_elapsed(struct sched_gate_list *sched, ktime_t time) 146 { 147 ktime_t time_since_sched_start; 148 s32 time_elapsed; 149 150 time_since_sched_start = ktime_sub(time, sched->base_time); 151 div_s64_rem(time_since_sched_start, sched->cycle_time, &time_elapsed); 152 153 return time_elapsed; 154 } 155 156 static ktime_t get_interval_end_time(struct sched_gate_list *sched, 157 struct sched_gate_list *admin, 158 struct sched_entry *entry, 159 ktime_t intv_start) 160 { 161 s32 cycle_elapsed = get_cycle_time_elapsed(sched, intv_start); 162 ktime_t intv_end, cycle_ext_end, cycle_end; 163 164 cycle_end = ktime_add_ns(intv_start, sched->cycle_time - cycle_elapsed); 165 intv_end = ktime_add_ns(intv_start, entry->interval); 166 cycle_ext_end = ktime_add(cycle_end, sched->cycle_time_extension); 167 168 if (ktime_before(intv_end, cycle_end)) 169 return intv_end; 170 else if (admin && admin != sched && 171 ktime_after(admin->base_time, cycle_end) && 172 ktime_before(admin->base_time, cycle_ext_end)) 173 return admin->base_time; 174 else 175 return cycle_end; 176 } 177 178 static int length_to_duration(struct taprio_sched *q, int len) 179 { 180 return div_u64(len * atomic64_read(&q->picos_per_byte), PSEC_PER_NSEC); 181 } 182 183 /* Returns the entry corresponding to next available interval. If 184 * validate_interval is set, it only validates whether the timestamp occurs 185 * when the gate corresponding to the skb's traffic class is open. 186 */ 187 static struct sched_entry *find_entry_to_transmit(struct sk_buff *skb, 188 struct Qdisc *sch, 189 struct sched_gate_list *sched, 190 struct sched_gate_list *admin, 191 ktime_t time, 192 ktime_t *interval_start, 193 ktime_t *interval_end, 194 bool validate_interval) 195 { 196 ktime_t curr_intv_start, curr_intv_end, cycle_end, packet_transmit_time; 197 ktime_t earliest_txtime = KTIME_MAX, txtime, cycle, transmit_end_time; 198 struct sched_entry *entry = NULL, *entry_found = NULL; 199 struct taprio_sched *q = qdisc_priv(sch); 200 struct net_device *dev = qdisc_dev(sch); 201 bool entry_available = false; 202 s32 cycle_elapsed; 203 int tc, n; 204 205 tc = netdev_get_prio_tc_map(dev, skb->priority); 206 packet_transmit_time = length_to_duration(q, qdisc_pkt_len(skb)); 207 208 *interval_start = 0; 209 *interval_end = 0; 210 211 if (!sched) 212 return NULL; 213 214 cycle = sched->cycle_time; 215 cycle_elapsed = get_cycle_time_elapsed(sched, time); 216 curr_intv_end = ktime_sub_ns(time, cycle_elapsed); 217 cycle_end = ktime_add_ns(curr_intv_end, cycle); 218 219 list_for_each_entry(entry, &sched->entries, list) { 220 curr_intv_start = curr_intv_end; 221 curr_intv_end = get_interval_end_time(sched, admin, entry, 222 curr_intv_start); 223 224 if (ktime_after(curr_intv_start, cycle_end)) 225 break; 226 227 if (!(entry->gate_mask & BIT(tc)) || 228 packet_transmit_time > entry->interval) 229 continue; 230 231 txtime = entry->next_txtime; 232 233 if (ktime_before(txtime, time) || validate_interval) { 234 transmit_end_time = ktime_add_ns(time, packet_transmit_time); 235 if ((ktime_before(curr_intv_start, time) && 236 ktime_before(transmit_end_time, curr_intv_end)) || 237 (ktime_after(curr_intv_start, time) && !validate_interval)) { 238 entry_found = entry; 239 *interval_start = curr_intv_start; 240 *interval_end = curr_intv_end; 241 break; 242 } else if (!entry_available && !validate_interval) { 243 /* Here, we are just trying to find out the 244 * first available interval in the next cycle. 245 */ 246 entry_available = true; 247 entry_found = entry; 248 *interval_start = ktime_add_ns(curr_intv_start, cycle); 249 *interval_end = ktime_add_ns(curr_intv_end, cycle); 250 } 251 } else if (ktime_before(txtime, earliest_txtime) && 252 !entry_available) { 253 earliest_txtime = txtime; 254 entry_found = entry; 255 n = div_s64(ktime_sub(txtime, curr_intv_start), cycle); 256 *interval_start = ktime_add(curr_intv_start, n * cycle); 257 *interval_end = ktime_add(curr_intv_end, n * cycle); 258 } 259 } 260 261 return entry_found; 262 } 263 264 static bool is_valid_interval(struct sk_buff *skb, struct Qdisc *sch) 265 { 266 struct taprio_sched *q = qdisc_priv(sch); 267 struct sched_gate_list *sched, *admin; 268 ktime_t interval_start, interval_end; 269 struct sched_entry *entry; 270 271 rcu_read_lock(); 272 sched = rcu_dereference(q->oper_sched); 273 admin = rcu_dereference(q->admin_sched); 274 275 entry = find_entry_to_transmit(skb, sch, sched, admin, skb->tstamp, 276 &interval_start, &interval_end, true); 277 rcu_read_unlock(); 278 279 return entry; 280 } 281 282 static bool taprio_flags_valid(u32 flags) 283 { 284 /* Make sure no other flag bits are set. */ 285 if (flags & ~(TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST | 286 TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD)) 287 return false; 288 /* txtime-assist and full offload are mutually exclusive */ 289 if ((flags & TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST) && 290 (flags & TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD)) 291 return false; 292 return true; 293 } 294 295 /* This returns the tstamp value set by TCP in terms of the set clock. */ 296 static ktime_t get_tcp_tstamp(struct taprio_sched *q, struct sk_buff *skb) 297 { 298 unsigned int offset = skb_network_offset(skb); 299 const struct ipv6hdr *ipv6h; 300 const struct iphdr *iph; 301 struct ipv6hdr _ipv6h; 302 303 ipv6h = skb_header_pointer(skb, offset, sizeof(_ipv6h), &_ipv6h); 304 if (!ipv6h) 305 return 0; 306 307 if (ipv6h->version == 4) { 308 iph = (struct iphdr *)ipv6h; 309 offset += iph->ihl * 4; 310 311 /* special-case 6in4 tunnelling, as that is a common way to get 312 * v6 connectivity in the home 313 */ 314 if (iph->protocol == IPPROTO_IPV6) { 315 ipv6h = skb_header_pointer(skb, offset, 316 sizeof(_ipv6h), &_ipv6h); 317 318 if (!ipv6h || ipv6h->nexthdr != IPPROTO_TCP) 319 return 0; 320 } else if (iph->protocol != IPPROTO_TCP) { 321 return 0; 322 } 323 } else if (ipv6h->version == 6 && ipv6h->nexthdr != IPPROTO_TCP) { 324 return 0; 325 } 326 327 return taprio_mono_to_any(q, skb->skb_mstamp_ns); 328 } 329 330 /* There are a few scenarios where we will have to modify the txtime from 331 * what is read from next_txtime in sched_entry. They are: 332 * 1. If txtime is in the past, 333 * a. The gate for the traffic class is currently open and packet can be 334 * transmitted before it closes, schedule the packet right away. 335 * b. If the gate corresponding to the traffic class is going to open later 336 * in the cycle, set the txtime of packet to the interval start. 337 * 2. If txtime is in the future, there are packets corresponding to the 338 * current traffic class waiting to be transmitted. So, the following 339 * possibilities exist: 340 * a. We can transmit the packet before the window containing the txtime 341 * closes. 342 * b. The window might close before the transmission can be completed 343 * successfully. So, schedule the packet in the next open window. 344 */ 345 static long get_packet_txtime(struct sk_buff *skb, struct Qdisc *sch) 346 { 347 ktime_t transmit_end_time, interval_end, interval_start, tcp_tstamp; 348 struct taprio_sched *q = qdisc_priv(sch); 349 struct sched_gate_list *sched, *admin; 350 ktime_t minimum_time, now, txtime; 351 int len, packet_transmit_time; 352 struct sched_entry *entry; 353 bool sched_changed; 354 355 now = taprio_get_time(q); 356 minimum_time = ktime_add_ns(now, q->txtime_delay); 357 358 tcp_tstamp = get_tcp_tstamp(q, skb); 359 minimum_time = max_t(ktime_t, minimum_time, tcp_tstamp); 360 361 rcu_read_lock(); 362 admin = rcu_dereference(q->admin_sched); 363 sched = rcu_dereference(q->oper_sched); 364 if (admin && ktime_after(minimum_time, admin->base_time)) 365 switch_schedules(q, &admin, &sched); 366 367 /* Until the schedule starts, all the queues are open */ 368 if (!sched || ktime_before(minimum_time, sched->base_time)) { 369 txtime = minimum_time; 370 goto done; 371 } 372 373 len = qdisc_pkt_len(skb); 374 packet_transmit_time = length_to_duration(q, len); 375 376 do { 377 sched_changed = false; 378 379 entry = find_entry_to_transmit(skb, sch, sched, admin, 380 minimum_time, 381 &interval_start, &interval_end, 382 false); 383 if (!entry) { 384 txtime = 0; 385 goto done; 386 } 387 388 txtime = entry->next_txtime; 389 txtime = max_t(ktime_t, txtime, minimum_time); 390 txtime = max_t(ktime_t, txtime, interval_start); 391 392 if (admin && admin != sched && 393 ktime_after(txtime, admin->base_time)) { 394 sched = admin; 395 sched_changed = true; 396 continue; 397 } 398 399 transmit_end_time = ktime_add(txtime, packet_transmit_time); 400 minimum_time = transmit_end_time; 401 402 /* Update the txtime of current entry to the next time it's 403 * interval starts. 404 */ 405 if (ktime_after(transmit_end_time, interval_end)) 406 entry->next_txtime = ktime_add(interval_start, sched->cycle_time); 407 } while (sched_changed || ktime_after(transmit_end_time, interval_end)); 408 409 entry->next_txtime = transmit_end_time; 410 411 done: 412 rcu_read_unlock(); 413 return txtime; 414 } 415 416 static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch, 417 struct Qdisc *child, struct sk_buff **to_free) 418 { 419 struct taprio_sched *q = qdisc_priv(sch); 420 421 /* sk_flags are only safe to use on full sockets. */ 422 if (skb->sk && sk_fullsock(skb->sk) && sock_flag(skb->sk, SOCK_TXTIME)) { 423 if (!is_valid_interval(skb, sch)) 424 return qdisc_drop(skb, sch, to_free); 425 } else if (TXTIME_ASSIST_IS_ENABLED(q->flags)) { 426 skb->tstamp = get_packet_txtime(skb, sch); 427 if (!skb->tstamp) 428 return qdisc_drop(skb, sch, to_free); 429 } 430 431 qdisc_qstats_backlog_inc(sch, skb); 432 sch->q.qlen++; 433 434 return qdisc_enqueue(skb, child, to_free); 435 } 436 437 static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, 438 struct sk_buff **to_free) 439 { 440 struct taprio_sched *q = qdisc_priv(sch); 441 struct Qdisc *child; 442 int queue; 443 444 if (unlikely(FULL_OFFLOAD_IS_ENABLED(q->flags))) { 445 WARN_ONCE(1, "Trying to enqueue skb into the root of a taprio qdisc configured with full offload\n"); 446 return qdisc_drop(skb, sch, to_free); 447 } 448 449 queue = skb_get_queue_mapping(skb); 450 451 child = q->qdiscs[queue]; 452 if (unlikely(!child)) 453 return qdisc_drop(skb, sch, to_free); 454 455 /* Large packets might not be transmitted when the transmission duration 456 * exceeds any configured interval. Therefore, segment the skb into 457 * smaller chunks. Skip it for the full offload case, as the driver 458 * and/or the hardware is expected to handle this. 459 */ 460 if (skb_is_gso(skb) && !FULL_OFFLOAD_IS_ENABLED(q->flags)) { 461 unsigned int slen = 0, numsegs = 0, len = qdisc_pkt_len(skb); 462 netdev_features_t features = netif_skb_features(skb); 463 struct sk_buff *segs, *nskb; 464 int ret; 465 466 segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); 467 if (IS_ERR_OR_NULL(segs)) 468 return qdisc_drop(skb, sch, to_free); 469 470 skb_list_walk_safe(segs, segs, nskb) { 471 skb_mark_not_on_list(segs); 472 qdisc_skb_cb(segs)->pkt_len = segs->len; 473 slen += segs->len; 474 475 ret = taprio_enqueue_one(segs, sch, child, to_free); 476 if (ret != NET_XMIT_SUCCESS) { 477 if (net_xmit_drop_count(ret)) 478 qdisc_qstats_drop(sch); 479 } else { 480 numsegs++; 481 } 482 } 483 484 if (numsegs > 1) 485 qdisc_tree_reduce_backlog(sch, 1 - numsegs, len - slen); 486 consume_skb(skb); 487 488 return numsegs > 0 ? NET_XMIT_SUCCESS : NET_XMIT_DROP; 489 } 490 491 return taprio_enqueue_one(skb, sch, child, to_free); 492 } 493 494 static struct sk_buff *taprio_peek_soft(struct Qdisc *sch) 495 { 496 struct taprio_sched *q = qdisc_priv(sch); 497 struct net_device *dev = qdisc_dev(sch); 498 struct sched_entry *entry; 499 struct sk_buff *skb; 500 u32 gate_mask; 501 int i; 502 503 rcu_read_lock(); 504 entry = rcu_dereference(q->current_entry); 505 gate_mask = entry ? entry->gate_mask : TAPRIO_ALL_GATES_OPEN; 506 rcu_read_unlock(); 507 508 if (!gate_mask) 509 return NULL; 510 511 for (i = 0; i < dev->num_tx_queues; i++) { 512 struct Qdisc *child = q->qdiscs[i]; 513 int prio; 514 u8 tc; 515 516 if (unlikely(!child)) 517 continue; 518 519 skb = child->ops->peek(child); 520 if (!skb) 521 continue; 522 523 if (TXTIME_ASSIST_IS_ENABLED(q->flags)) 524 return skb; 525 526 prio = skb->priority; 527 tc = netdev_get_prio_tc_map(dev, prio); 528 529 if (!(gate_mask & BIT(tc))) 530 continue; 531 532 return skb; 533 } 534 535 return NULL; 536 } 537 538 static struct sk_buff *taprio_peek_offload(struct Qdisc *sch) 539 { 540 WARN_ONCE(1, "Trying to peek into the root of a taprio qdisc configured with full offload\n"); 541 542 return NULL; 543 } 544 545 static struct sk_buff *taprio_peek(struct Qdisc *sch) 546 { 547 struct taprio_sched *q = qdisc_priv(sch); 548 549 return q->peek(sch); 550 } 551 552 static void taprio_set_budget(struct taprio_sched *q, struct sched_entry *entry) 553 { 554 atomic_set(&entry->budget, 555 div64_u64((u64)entry->interval * PSEC_PER_NSEC, 556 atomic64_read(&q->picos_per_byte))); 557 } 558 559 static struct sk_buff *taprio_dequeue_soft(struct Qdisc *sch) 560 { 561 struct taprio_sched *q = qdisc_priv(sch); 562 struct net_device *dev = qdisc_dev(sch); 563 struct sk_buff *skb = NULL; 564 struct sched_entry *entry; 565 u32 gate_mask; 566 int i; 567 568 rcu_read_lock(); 569 entry = rcu_dereference(q->current_entry); 570 /* if there's no entry, it means that the schedule didn't 571 * start yet, so force all gates to be open, this is in 572 * accordance to IEEE 802.1Qbv-2015 Section 8.6.9.4.5 573 * "AdminGateStates" 574 */ 575 gate_mask = entry ? entry->gate_mask : TAPRIO_ALL_GATES_OPEN; 576 577 if (!gate_mask) 578 goto done; 579 580 for (i = 0; i < dev->num_tx_queues; i++) { 581 struct Qdisc *child = q->qdiscs[i]; 582 ktime_t guard; 583 int prio; 584 int len; 585 u8 tc; 586 587 if (unlikely(!child)) 588 continue; 589 590 if (TXTIME_ASSIST_IS_ENABLED(q->flags)) { 591 skb = child->ops->dequeue(child); 592 if (!skb) 593 continue; 594 goto skb_found; 595 } 596 597 skb = child->ops->peek(child); 598 if (!skb) 599 continue; 600 601 prio = skb->priority; 602 tc = netdev_get_prio_tc_map(dev, prio); 603 604 if (!(gate_mask & BIT(tc))) { 605 skb = NULL; 606 continue; 607 } 608 609 len = qdisc_pkt_len(skb); 610 guard = ktime_add_ns(taprio_get_time(q), 611 length_to_duration(q, len)); 612 613 /* In the case that there's no gate entry, there's no 614 * guard band ... 615 */ 616 if (gate_mask != TAPRIO_ALL_GATES_OPEN && 617 ktime_after(guard, entry->close_time)) { 618 skb = NULL; 619 continue; 620 } 621 622 /* ... and no budget. */ 623 if (gate_mask != TAPRIO_ALL_GATES_OPEN && 624 atomic_sub_return(len, &entry->budget) < 0) { 625 skb = NULL; 626 continue; 627 } 628 629 skb = child->ops->dequeue(child); 630 if (unlikely(!skb)) 631 goto done; 632 633 skb_found: 634 qdisc_bstats_update(sch, skb); 635 qdisc_qstats_backlog_dec(sch, skb); 636 sch->q.qlen--; 637 638 goto done; 639 } 640 641 done: 642 rcu_read_unlock(); 643 644 return skb; 645 } 646 647 static struct sk_buff *taprio_dequeue_offload(struct Qdisc *sch) 648 { 649 WARN_ONCE(1, "Trying to dequeue from the root of a taprio qdisc configured with full offload\n"); 650 651 return NULL; 652 } 653 654 static struct sk_buff *taprio_dequeue(struct Qdisc *sch) 655 { 656 struct taprio_sched *q = qdisc_priv(sch); 657 658 return q->dequeue(sch); 659 } 660 661 static bool should_restart_cycle(const struct sched_gate_list *oper, 662 const struct sched_entry *entry) 663 { 664 if (list_is_last(&entry->list, &oper->entries)) 665 return true; 666 667 if (ktime_compare(entry->close_time, oper->cycle_close_time) == 0) 668 return true; 669 670 return false; 671 } 672 673 static bool should_change_schedules(const struct sched_gate_list *admin, 674 const struct sched_gate_list *oper, 675 ktime_t close_time) 676 { 677 ktime_t next_base_time, extension_time; 678 679 if (!admin) 680 return false; 681 682 next_base_time = sched_base_time(admin); 683 684 /* This is the simple case, the close_time would fall after 685 * the next schedule base_time. 686 */ 687 if (ktime_compare(next_base_time, close_time) <= 0) 688 return true; 689 690 /* This is the cycle_time_extension case, if the close_time 691 * plus the amount that can be extended would fall after the 692 * next schedule base_time, we can extend the current schedule 693 * for that amount. 694 */ 695 extension_time = ktime_add_ns(close_time, oper->cycle_time_extension); 696 697 /* FIXME: the IEEE 802.1Q-2018 Specification isn't clear about 698 * how precisely the extension should be made. So after 699 * conformance testing, this logic may change. 700 */ 701 if (ktime_compare(next_base_time, extension_time) <= 0) 702 return true; 703 704 return false; 705 } 706 707 static enum hrtimer_restart advance_sched(struct hrtimer *timer) 708 { 709 struct taprio_sched *q = container_of(timer, struct taprio_sched, 710 advance_timer); 711 struct sched_gate_list *oper, *admin; 712 struct sched_entry *entry, *next; 713 struct Qdisc *sch = q->root; 714 ktime_t close_time; 715 716 spin_lock(&q->current_entry_lock); 717 entry = rcu_dereference_protected(q->current_entry, 718 lockdep_is_held(&q->current_entry_lock)); 719 oper = rcu_dereference_protected(q->oper_sched, 720 lockdep_is_held(&q->current_entry_lock)); 721 admin = rcu_dereference_protected(q->admin_sched, 722 lockdep_is_held(&q->current_entry_lock)); 723 724 if (!oper) 725 switch_schedules(q, &admin, &oper); 726 727 /* This can happen in two cases: 1. this is the very first run 728 * of this function (i.e. we weren't running any schedule 729 * previously); 2. The previous schedule just ended. The first 730 * entry of all schedules are pre-calculated during the 731 * schedule initialization. 732 */ 733 if (unlikely(!entry || entry->close_time == oper->base_time)) { 734 next = list_first_entry(&oper->entries, struct sched_entry, 735 list); 736 close_time = next->close_time; 737 goto first_run; 738 } 739 740 if (should_restart_cycle(oper, entry)) { 741 next = list_first_entry(&oper->entries, struct sched_entry, 742 list); 743 oper->cycle_close_time = ktime_add_ns(oper->cycle_close_time, 744 oper->cycle_time); 745 } else { 746 next = list_next_entry(entry, list); 747 } 748 749 close_time = ktime_add_ns(entry->close_time, next->interval); 750 close_time = min_t(ktime_t, close_time, oper->cycle_close_time); 751 752 if (should_change_schedules(admin, oper, close_time)) { 753 /* Set things so the next time this runs, the new 754 * schedule runs. 755 */ 756 close_time = sched_base_time(admin); 757 switch_schedules(q, &admin, &oper); 758 } 759 760 next->close_time = close_time; 761 taprio_set_budget(q, next); 762 763 first_run: 764 rcu_assign_pointer(q->current_entry, next); 765 spin_unlock(&q->current_entry_lock); 766 767 hrtimer_set_expires(&q->advance_timer, close_time); 768 769 rcu_read_lock(); 770 __netif_schedule(sch); 771 rcu_read_unlock(); 772 773 return HRTIMER_RESTART; 774 } 775 776 static const struct nla_policy entry_policy[TCA_TAPRIO_SCHED_ENTRY_MAX + 1] = { 777 [TCA_TAPRIO_SCHED_ENTRY_INDEX] = { .type = NLA_U32 }, 778 [TCA_TAPRIO_SCHED_ENTRY_CMD] = { .type = NLA_U8 }, 779 [TCA_TAPRIO_SCHED_ENTRY_GATE_MASK] = { .type = NLA_U32 }, 780 [TCA_TAPRIO_SCHED_ENTRY_INTERVAL] = { .type = NLA_U32 }, 781 }; 782 783 static const struct nla_policy taprio_policy[TCA_TAPRIO_ATTR_MAX + 1] = { 784 [TCA_TAPRIO_ATTR_PRIOMAP] = { 785 .len = sizeof(struct tc_mqprio_qopt) 786 }, 787 [TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST] = { .type = NLA_NESTED }, 788 [TCA_TAPRIO_ATTR_SCHED_BASE_TIME] = { .type = NLA_S64 }, 789 [TCA_TAPRIO_ATTR_SCHED_SINGLE_ENTRY] = { .type = NLA_NESTED }, 790 [TCA_TAPRIO_ATTR_SCHED_CLOCKID] = { .type = NLA_S32 }, 791 [TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME] = { .type = NLA_S64 }, 792 [TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION] = { .type = NLA_S64 }, 793 [TCA_TAPRIO_ATTR_FLAGS] = { .type = NLA_U32 }, 794 [TCA_TAPRIO_ATTR_TXTIME_DELAY] = { .type = NLA_U32 }, 795 }; 796 797 static int fill_sched_entry(struct taprio_sched *q, struct nlattr **tb, 798 struct sched_entry *entry, 799 struct netlink_ext_ack *extack) 800 { 801 int min_duration = length_to_duration(q, ETH_ZLEN); 802 u32 interval = 0; 803 804 if (tb[TCA_TAPRIO_SCHED_ENTRY_CMD]) 805 entry->command = nla_get_u8( 806 tb[TCA_TAPRIO_SCHED_ENTRY_CMD]); 807 808 if (tb[TCA_TAPRIO_SCHED_ENTRY_GATE_MASK]) 809 entry->gate_mask = nla_get_u32( 810 tb[TCA_TAPRIO_SCHED_ENTRY_GATE_MASK]); 811 812 if (tb[TCA_TAPRIO_SCHED_ENTRY_INTERVAL]) 813 interval = nla_get_u32( 814 tb[TCA_TAPRIO_SCHED_ENTRY_INTERVAL]); 815 816 /* The interval should allow at least the minimum ethernet 817 * frame to go out. 818 */ 819 if (interval < min_duration) { 820 NL_SET_ERR_MSG(extack, "Invalid interval for schedule entry"); 821 return -EINVAL; 822 } 823 824 entry->interval = interval; 825 826 return 0; 827 } 828 829 static int parse_sched_entry(struct taprio_sched *q, struct nlattr *n, 830 struct sched_entry *entry, int index, 831 struct netlink_ext_ack *extack) 832 { 833 struct nlattr *tb[TCA_TAPRIO_SCHED_ENTRY_MAX + 1] = { }; 834 int err; 835 836 err = nla_parse_nested_deprecated(tb, TCA_TAPRIO_SCHED_ENTRY_MAX, n, 837 entry_policy, NULL); 838 if (err < 0) { 839 NL_SET_ERR_MSG(extack, "Could not parse nested entry"); 840 return -EINVAL; 841 } 842 843 entry->index = index; 844 845 return fill_sched_entry(q, tb, entry, extack); 846 } 847 848 static int parse_sched_list(struct taprio_sched *q, struct nlattr *list, 849 struct sched_gate_list *sched, 850 struct netlink_ext_ack *extack) 851 { 852 struct nlattr *n; 853 int err, rem; 854 int i = 0; 855 856 if (!list) 857 return -EINVAL; 858 859 nla_for_each_nested(n, list, rem) { 860 struct sched_entry *entry; 861 862 if (nla_type(n) != TCA_TAPRIO_SCHED_ENTRY) { 863 NL_SET_ERR_MSG(extack, "Attribute is not of type 'entry'"); 864 continue; 865 } 866 867 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 868 if (!entry) { 869 NL_SET_ERR_MSG(extack, "Not enough memory for entry"); 870 return -ENOMEM; 871 } 872 873 err = parse_sched_entry(q, n, entry, i, extack); 874 if (err < 0) { 875 kfree(entry); 876 return err; 877 } 878 879 list_add_tail(&entry->list, &sched->entries); 880 i++; 881 } 882 883 sched->num_entries = i; 884 885 return i; 886 } 887 888 static int parse_taprio_schedule(struct taprio_sched *q, struct nlattr **tb, 889 struct sched_gate_list *new, 890 struct netlink_ext_ack *extack) 891 { 892 int err = 0; 893 894 if (tb[TCA_TAPRIO_ATTR_SCHED_SINGLE_ENTRY]) { 895 NL_SET_ERR_MSG(extack, "Adding a single entry is not supported"); 896 return -ENOTSUPP; 897 } 898 899 if (tb[TCA_TAPRIO_ATTR_SCHED_BASE_TIME]) 900 new->base_time = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_BASE_TIME]); 901 902 if (tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION]) 903 new->cycle_time_extension = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION]); 904 905 if (tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME]) 906 new->cycle_time = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME]); 907 908 if (tb[TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST]) 909 err = parse_sched_list(q, tb[TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST], 910 new, extack); 911 if (err < 0) 912 return err; 913 914 if (!new->cycle_time) { 915 struct sched_entry *entry; 916 ktime_t cycle = 0; 917 918 list_for_each_entry(entry, &new->entries, list) 919 cycle = ktime_add_ns(cycle, entry->interval); 920 921 if (!cycle) { 922 NL_SET_ERR_MSG(extack, "'cycle_time' can never be 0"); 923 return -EINVAL; 924 } 925 926 new->cycle_time = cycle; 927 } 928 929 return 0; 930 } 931 932 static int taprio_parse_mqprio_opt(struct net_device *dev, 933 struct tc_mqprio_qopt *qopt, 934 struct netlink_ext_ack *extack, 935 u32 taprio_flags) 936 { 937 int i, j; 938 939 if (!qopt && !dev->num_tc) { 940 NL_SET_ERR_MSG(extack, "'mqprio' configuration is necessary"); 941 return -EINVAL; 942 } 943 944 /* If num_tc is already set, it means that the user already 945 * configured the mqprio part 946 */ 947 if (dev->num_tc) 948 return 0; 949 950 /* Verify num_tc is not out of max range */ 951 if (qopt->num_tc > TC_MAX_QUEUE) { 952 NL_SET_ERR_MSG(extack, "Number of traffic classes is outside valid range"); 953 return -EINVAL; 954 } 955 956 /* taprio imposes that traffic classes map 1:n to tx queues */ 957 if (qopt->num_tc > dev->num_tx_queues) { 958 NL_SET_ERR_MSG(extack, "Number of traffic classes is greater than number of HW queues"); 959 return -EINVAL; 960 } 961 962 /* Verify priority mapping uses valid tcs */ 963 for (i = 0; i <= TC_BITMASK; i++) { 964 if (qopt->prio_tc_map[i] >= qopt->num_tc) { 965 NL_SET_ERR_MSG(extack, "Invalid traffic class in priority to traffic class mapping"); 966 return -EINVAL; 967 } 968 } 969 970 for (i = 0; i < qopt->num_tc; i++) { 971 unsigned int last = qopt->offset[i] + qopt->count[i]; 972 973 /* Verify the queue count is in tx range being equal to the 974 * real_num_tx_queues indicates the last queue is in use. 975 */ 976 if (qopt->offset[i] >= dev->num_tx_queues || 977 !qopt->count[i] || 978 last > dev->real_num_tx_queues) { 979 NL_SET_ERR_MSG(extack, "Invalid queue in traffic class to queue mapping"); 980 return -EINVAL; 981 } 982 983 if (TXTIME_ASSIST_IS_ENABLED(taprio_flags)) 984 continue; 985 986 /* Verify that the offset and counts do not overlap */ 987 for (j = i + 1; j < qopt->num_tc; j++) { 988 if (last > qopt->offset[j]) { 989 NL_SET_ERR_MSG(extack, "Detected overlap in the traffic class to queue mapping"); 990 return -EINVAL; 991 } 992 } 993 } 994 995 return 0; 996 } 997 998 static int taprio_get_start_time(struct Qdisc *sch, 999 struct sched_gate_list *sched, 1000 ktime_t *start) 1001 { 1002 struct taprio_sched *q = qdisc_priv(sch); 1003 ktime_t now, base, cycle; 1004 s64 n; 1005 1006 base = sched_base_time(sched); 1007 now = taprio_get_time(q); 1008 1009 if (ktime_after(base, now)) { 1010 *start = base; 1011 return 0; 1012 } 1013 1014 cycle = sched->cycle_time; 1015 1016 /* The qdisc is expected to have at least one sched_entry. Moreover, 1017 * any entry must have 'interval' > 0. Thus if the cycle time is zero, 1018 * something went really wrong. In that case, we should warn about this 1019 * inconsistent state and return error. 1020 */ 1021 if (WARN_ON(!cycle)) 1022 return -EFAULT; 1023 1024 /* Schedule the start time for the beginning of the next 1025 * cycle. 1026 */ 1027 n = div64_s64(ktime_sub_ns(now, base), cycle); 1028 *start = ktime_add_ns(base, (n + 1) * cycle); 1029 return 0; 1030 } 1031 1032 static void setup_first_close_time(struct taprio_sched *q, 1033 struct sched_gate_list *sched, ktime_t base) 1034 { 1035 struct sched_entry *first; 1036 ktime_t cycle; 1037 1038 first = list_first_entry(&sched->entries, 1039 struct sched_entry, list); 1040 1041 cycle = sched->cycle_time; 1042 1043 /* FIXME: find a better place to do this */ 1044 sched->cycle_close_time = ktime_add_ns(base, cycle); 1045 1046 first->close_time = ktime_add_ns(base, first->interval); 1047 taprio_set_budget(q, first); 1048 rcu_assign_pointer(q->current_entry, NULL); 1049 } 1050 1051 static void taprio_start_sched(struct Qdisc *sch, 1052 ktime_t start, struct sched_gate_list *new) 1053 { 1054 struct taprio_sched *q = qdisc_priv(sch); 1055 ktime_t expires; 1056 1057 if (FULL_OFFLOAD_IS_ENABLED(q->flags)) 1058 return; 1059 1060 expires = hrtimer_get_expires(&q->advance_timer); 1061 if (expires == 0) 1062 expires = KTIME_MAX; 1063 1064 /* If the new schedule starts before the next expiration, we 1065 * reprogram it to the earliest one, so we change the admin 1066 * schedule to the operational one at the right time. 1067 */ 1068 start = min_t(ktime_t, start, expires); 1069 1070 hrtimer_start(&q->advance_timer, start, HRTIMER_MODE_ABS); 1071 } 1072 1073 static void taprio_set_picos_per_byte(struct net_device *dev, 1074 struct taprio_sched *q) 1075 { 1076 struct ethtool_link_ksettings ecmd; 1077 int speed = SPEED_10; 1078 int picos_per_byte; 1079 int err; 1080 1081 err = __ethtool_get_link_ksettings(dev, &ecmd); 1082 if (err < 0) 1083 goto skip; 1084 1085 if (ecmd.base.speed && ecmd.base.speed != SPEED_UNKNOWN) 1086 speed = ecmd.base.speed; 1087 1088 skip: 1089 picos_per_byte = (USEC_PER_SEC * 8) / speed; 1090 1091 atomic64_set(&q->picos_per_byte, picos_per_byte); 1092 netdev_dbg(dev, "taprio: set %s's picos_per_byte to: %lld, linkspeed: %d\n", 1093 dev->name, (long long)atomic64_read(&q->picos_per_byte), 1094 ecmd.base.speed); 1095 } 1096 1097 static int taprio_dev_notifier(struct notifier_block *nb, unsigned long event, 1098 void *ptr) 1099 { 1100 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 1101 struct net_device *qdev; 1102 struct taprio_sched *q; 1103 bool found = false; 1104 1105 ASSERT_RTNL(); 1106 1107 if (event != NETDEV_UP && event != NETDEV_CHANGE) 1108 return NOTIFY_DONE; 1109 1110 spin_lock(&taprio_list_lock); 1111 list_for_each_entry(q, &taprio_list, taprio_list) { 1112 qdev = qdisc_dev(q->root); 1113 if (qdev == dev) { 1114 found = true; 1115 break; 1116 } 1117 } 1118 spin_unlock(&taprio_list_lock); 1119 1120 if (found) 1121 taprio_set_picos_per_byte(dev, q); 1122 1123 return NOTIFY_DONE; 1124 } 1125 1126 static void setup_txtime(struct taprio_sched *q, 1127 struct sched_gate_list *sched, ktime_t base) 1128 { 1129 struct sched_entry *entry; 1130 u32 interval = 0; 1131 1132 list_for_each_entry(entry, &sched->entries, list) { 1133 entry->next_txtime = ktime_add_ns(base, interval); 1134 interval += entry->interval; 1135 } 1136 } 1137 1138 static struct tc_taprio_qopt_offload *taprio_offload_alloc(int num_entries) 1139 { 1140 struct __tc_taprio_qopt_offload *__offload; 1141 1142 __offload = kzalloc(struct_size(__offload, offload.entries, num_entries), 1143 GFP_KERNEL); 1144 if (!__offload) 1145 return NULL; 1146 1147 refcount_set(&__offload->users, 1); 1148 1149 return &__offload->offload; 1150 } 1151 1152 struct tc_taprio_qopt_offload *taprio_offload_get(struct tc_taprio_qopt_offload 1153 *offload) 1154 { 1155 struct __tc_taprio_qopt_offload *__offload; 1156 1157 __offload = container_of(offload, struct __tc_taprio_qopt_offload, 1158 offload); 1159 1160 refcount_inc(&__offload->users); 1161 1162 return offload; 1163 } 1164 EXPORT_SYMBOL_GPL(taprio_offload_get); 1165 1166 void taprio_offload_free(struct tc_taprio_qopt_offload *offload) 1167 { 1168 struct __tc_taprio_qopt_offload *__offload; 1169 1170 __offload = container_of(offload, struct __tc_taprio_qopt_offload, 1171 offload); 1172 1173 if (!refcount_dec_and_test(&__offload->users)) 1174 return; 1175 1176 kfree(__offload); 1177 } 1178 EXPORT_SYMBOL_GPL(taprio_offload_free); 1179 1180 /* The function will only serve to keep the pointers to the "oper" and "admin" 1181 * schedules valid in relation to their base times, so when calling dump() the 1182 * users looks at the right schedules. 1183 * When using full offload, the admin configuration is promoted to oper at the 1184 * base_time in the PHC time domain. But because the system time is not 1185 * necessarily in sync with that, we can't just trigger a hrtimer to call 1186 * switch_schedules at the right hardware time. 1187 * At the moment we call this by hand right away from taprio, but in the future 1188 * it will be useful to create a mechanism for drivers to notify taprio of the 1189 * offload state (PENDING, ACTIVE, INACTIVE) so it can be visible in dump(). 1190 * This is left as TODO. 1191 */ 1192 static void taprio_offload_config_changed(struct taprio_sched *q) 1193 { 1194 struct sched_gate_list *oper, *admin; 1195 1196 spin_lock(&q->current_entry_lock); 1197 1198 oper = rcu_dereference_protected(q->oper_sched, 1199 lockdep_is_held(&q->current_entry_lock)); 1200 admin = rcu_dereference_protected(q->admin_sched, 1201 lockdep_is_held(&q->current_entry_lock)); 1202 1203 switch_schedules(q, &admin, &oper); 1204 1205 spin_unlock(&q->current_entry_lock); 1206 } 1207 1208 static u32 tc_map_to_queue_mask(struct net_device *dev, u32 tc_mask) 1209 { 1210 u32 i, queue_mask = 0; 1211 1212 for (i = 0; i < dev->num_tc; i++) { 1213 u32 offset, count; 1214 1215 if (!(tc_mask & BIT(i))) 1216 continue; 1217 1218 offset = dev->tc_to_txq[i].offset; 1219 count = dev->tc_to_txq[i].count; 1220 1221 queue_mask |= GENMASK(offset + count - 1, offset); 1222 } 1223 1224 return queue_mask; 1225 } 1226 1227 static void taprio_sched_to_offload(struct net_device *dev, 1228 struct sched_gate_list *sched, 1229 struct tc_taprio_qopt_offload *offload) 1230 { 1231 struct sched_entry *entry; 1232 int i = 0; 1233 1234 offload->base_time = sched->base_time; 1235 offload->cycle_time = sched->cycle_time; 1236 offload->cycle_time_extension = sched->cycle_time_extension; 1237 1238 list_for_each_entry(entry, &sched->entries, list) { 1239 struct tc_taprio_sched_entry *e = &offload->entries[i]; 1240 1241 e->command = entry->command; 1242 e->interval = entry->interval; 1243 e->gate_mask = tc_map_to_queue_mask(dev, entry->gate_mask); 1244 1245 i++; 1246 } 1247 1248 offload->num_entries = i; 1249 } 1250 1251 static int taprio_enable_offload(struct net_device *dev, 1252 struct taprio_sched *q, 1253 struct sched_gate_list *sched, 1254 struct netlink_ext_ack *extack) 1255 { 1256 const struct net_device_ops *ops = dev->netdev_ops; 1257 struct tc_taprio_qopt_offload *offload; 1258 int err = 0; 1259 1260 if (!ops->ndo_setup_tc) { 1261 NL_SET_ERR_MSG(extack, 1262 "Device does not support taprio offload"); 1263 return -EOPNOTSUPP; 1264 } 1265 1266 offload = taprio_offload_alloc(sched->num_entries); 1267 if (!offload) { 1268 NL_SET_ERR_MSG(extack, 1269 "Not enough memory for enabling offload mode"); 1270 return -ENOMEM; 1271 } 1272 offload->enable = 1; 1273 taprio_sched_to_offload(dev, sched, offload); 1274 1275 err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload); 1276 if (err < 0) { 1277 NL_SET_ERR_MSG(extack, 1278 "Device failed to setup taprio offload"); 1279 goto done; 1280 } 1281 1282 done: 1283 taprio_offload_free(offload); 1284 1285 return err; 1286 } 1287 1288 static int taprio_disable_offload(struct net_device *dev, 1289 struct taprio_sched *q, 1290 struct netlink_ext_ack *extack) 1291 { 1292 const struct net_device_ops *ops = dev->netdev_ops; 1293 struct tc_taprio_qopt_offload *offload; 1294 int err; 1295 1296 if (!FULL_OFFLOAD_IS_ENABLED(q->flags)) 1297 return 0; 1298 1299 if (!ops->ndo_setup_tc) 1300 return -EOPNOTSUPP; 1301 1302 offload = taprio_offload_alloc(0); 1303 if (!offload) { 1304 NL_SET_ERR_MSG(extack, 1305 "Not enough memory to disable offload mode"); 1306 return -ENOMEM; 1307 } 1308 offload->enable = 0; 1309 1310 err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload); 1311 if (err < 0) { 1312 NL_SET_ERR_MSG(extack, 1313 "Device failed to disable offload"); 1314 goto out; 1315 } 1316 1317 out: 1318 taprio_offload_free(offload); 1319 1320 return err; 1321 } 1322 1323 /* If full offload is enabled, the only possible clockid is the net device's 1324 * PHC. For that reason, specifying a clockid through netlink is incorrect. 1325 * For txtime-assist, it is implicitly assumed that the device's PHC is kept 1326 * in sync with the specified clockid via a user space daemon such as phc2sys. 1327 * For both software taprio and txtime-assist, the clockid is used for the 1328 * hrtimer that advances the schedule and hence mandatory. 1329 */ 1330 static int taprio_parse_clockid(struct Qdisc *sch, struct nlattr **tb, 1331 struct netlink_ext_ack *extack) 1332 { 1333 struct taprio_sched *q = qdisc_priv(sch); 1334 struct net_device *dev = qdisc_dev(sch); 1335 int err = -EINVAL; 1336 1337 if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { 1338 const struct ethtool_ops *ops = dev->ethtool_ops; 1339 struct ethtool_ts_info info = { 1340 .cmd = ETHTOOL_GET_TS_INFO, 1341 .phc_index = -1, 1342 }; 1343 1344 if (tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]) { 1345 NL_SET_ERR_MSG(extack, 1346 "The 'clockid' cannot be specified for full offload"); 1347 goto out; 1348 } 1349 1350 if (ops && ops->get_ts_info) 1351 err = ops->get_ts_info(dev, &info); 1352 1353 if (err || info.phc_index < 0) { 1354 NL_SET_ERR_MSG(extack, 1355 "Device does not have a PTP clock"); 1356 err = -ENOTSUPP; 1357 goto out; 1358 } 1359 } else if (tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]) { 1360 int clockid = nla_get_s32(tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]); 1361 enum tk_offsets tk_offset; 1362 1363 /* We only support static clockids and we don't allow 1364 * for it to be modified after the first init. 1365 */ 1366 if (clockid < 0 || 1367 (q->clockid != -1 && q->clockid != clockid)) { 1368 NL_SET_ERR_MSG(extack, 1369 "Changing the 'clockid' of a running schedule is not supported"); 1370 err = -ENOTSUPP; 1371 goto out; 1372 } 1373 1374 switch (clockid) { 1375 case CLOCK_REALTIME: 1376 tk_offset = TK_OFFS_REAL; 1377 break; 1378 case CLOCK_MONOTONIC: 1379 tk_offset = TK_OFFS_MAX; 1380 break; 1381 case CLOCK_BOOTTIME: 1382 tk_offset = TK_OFFS_BOOT; 1383 break; 1384 case CLOCK_TAI: 1385 tk_offset = TK_OFFS_TAI; 1386 break; 1387 default: 1388 NL_SET_ERR_MSG(extack, "Invalid 'clockid'"); 1389 err = -EINVAL; 1390 goto out; 1391 } 1392 /* This pairs with READ_ONCE() in taprio_mono_to_any */ 1393 WRITE_ONCE(q->tk_offset, tk_offset); 1394 1395 q->clockid = clockid; 1396 } else { 1397 NL_SET_ERR_MSG(extack, "Specifying a 'clockid' is mandatory"); 1398 goto out; 1399 } 1400 1401 /* Everything went ok, return success. */ 1402 err = 0; 1403 1404 out: 1405 return err; 1406 } 1407 1408 static int taprio_mqprio_cmp(const struct net_device *dev, 1409 const struct tc_mqprio_qopt *mqprio) 1410 { 1411 int i; 1412 1413 if (!mqprio || mqprio->num_tc != dev->num_tc) 1414 return -1; 1415 1416 for (i = 0; i < mqprio->num_tc; i++) 1417 if (dev->tc_to_txq[i].count != mqprio->count[i] || 1418 dev->tc_to_txq[i].offset != mqprio->offset[i]) 1419 return -1; 1420 1421 for (i = 0; i <= TC_BITMASK; i++) 1422 if (dev->prio_tc_map[i] != mqprio->prio_tc_map[i]) 1423 return -1; 1424 1425 return 0; 1426 } 1427 1428 /* The semantics of the 'flags' argument in relation to 'change()' 1429 * requests, are interpreted following two rules (which are applied in 1430 * this order): (1) an omitted 'flags' argument is interpreted as 1431 * zero; (2) the 'flags' of a "running" taprio instance cannot be 1432 * changed. 1433 */ 1434 static int taprio_new_flags(const struct nlattr *attr, u32 old, 1435 struct netlink_ext_ack *extack) 1436 { 1437 u32 new = 0; 1438 1439 if (attr) 1440 new = nla_get_u32(attr); 1441 1442 if (old != TAPRIO_FLAGS_INVALID && old != new) { 1443 NL_SET_ERR_MSG_MOD(extack, "Changing 'flags' of a running schedule is not supported"); 1444 return -EOPNOTSUPP; 1445 } 1446 1447 if (!taprio_flags_valid(new)) { 1448 NL_SET_ERR_MSG_MOD(extack, "Specified 'flags' are not valid"); 1449 return -EINVAL; 1450 } 1451 1452 return new; 1453 } 1454 1455 static int taprio_change(struct Qdisc *sch, struct nlattr *opt, 1456 struct netlink_ext_ack *extack) 1457 { 1458 struct nlattr *tb[TCA_TAPRIO_ATTR_MAX + 1] = { }; 1459 struct sched_gate_list *oper, *admin, *new_admin; 1460 struct taprio_sched *q = qdisc_priv(sch); 1461 struct net_device *dev = qdisc_dev(sch); 1462 struct tc_mqprio_qopt *mqprio = NULL; 1463 unsigned long flags; 1464 ktime_t start; 1465 int i, err; 1466 1467 err = nla_parse_nested_deprecated(tb, TCA_TAPRIO_ATTR_MAX, opt, 1468 taprio_policy, extack); 1469 if (err < 0) 1470 return err; 1471 1472 if (tb[TCA_TAPRIO_ATTR_PRIOMAP]) 1473 mqprio = nla_data(tb[TCA_TAPRIO_ATTR_PRIOMAP]); 1474 1475 err = taprio_new_flags(tb[TCA_TAPRIO_ATTR_FLAGS], 1476 q->flags, extack); 1477 if (err < 0) 1478 return err; 1479 1480 q->flags = err; 1481 1482 err = taprio_parse_mqprio_opt(dev, mqprio, extack, q->flags); 1483 if (err < 0) 1484 return err; 1485 1486 new_admin = kzalloc(sizeof(*new_admin), GFP_KERNEL); 1487 if (!new_admin) { 1488 NL_SET_ERR_MSG(extack, "Not enough memory for a new schedule"); 1489 return -ENOMEM; 1490 } 1491 INIT_LIST_HEAD(&new_admin->entries); 1492 1493 rcu_read_lock(); 1494 oper = rcu_dereference(q->oper_sched); 1495 admin = rcu_dereference(q->admin_sched); 1496 rcu_read_unlock(); 1497 1498 /* no changes - no new mqprio settings */ 1499 if (!taprio_mqprio_cmp(dev, mqprio)) 1500 mqprio = NULL; 1501 1502 if (mqprio && (oper || admin)) { 1503 NL_SET_ERR_MSG(extack, "Changing the traffic mapping of a running schedule is not supported"); 1504 err = -ENOTSUPP; 1505 goto free_sched; 1506 } 1507 1508 err = parse_taprio_schedule(q, tb, new_admin, extack); 1509 if (err < 0) 1510 goto free_sched; 1511 1512 if (new_admin->num_entries == 0) { 1513 NL_SET_ERR_MSG(extack, "There should be at least one entry in the schedule"); 1514 err = -EINVAL; 1515 goto free_sched; 1516 } 1517 1518 err = taprio_parse_clockid(sch, tb, extack); 1519 if (err < 0) 1520 goto free_sched; 1521 1522 taprio_set_picos_per_byte(dev, q); 1523 1524 if (mqprio) { 1525 err = netdev_set_num_tc(dev, mqprio->num_tc); 1526 if (err) 1527 goto free_sched; 1528 for (i = 0; i < mqprio->num_tc; i++) 1529 netdev_set_tc_queue(dev, i, 1530 mqprio->count[i], 1531 mqprio->offset[i]); 1532 1533 /* Always use supplied priority mappings */ 1534 for (i = 0; i <= TC_BITMASK; i++) 1535 netdev_set_prio_tc_map(dev, i, 1536 mqprio->prio_tc_map[i]); 1537 } 1538 1539 if (FULL_OFFLOAD_IS_ENABLED(q->flags)) 1540 err = taprio_enable_offload(dev, q, new_admin, extack); 1541 else 1542 err = taprio_disable_offload(dev, q, extack); 1543 if (err) 1544 goto free_sched; 1545 1546 /* Protects against enqueue()/dequeue() */ 1547 spin_lock_bh(qdisc_lock(sch)); 1548 1549 if (tb[TCA_TAPRIO_ATTR_TXTIME_DELAY]) { 1550 if (!TXTIME_ASSIST_IS_ENABLED(q->flags)) { 1551 NL_SET_ERR_MSG_MOD(extack, "txtime-delay can only be set when txtime-assist mode is enabled"); 1552 err = -EINVAL; 1553 goto unlock; 1554 } 1555 1556 q->txtime_delay = nla_get_u32(tb[TCA_TAPRIO_ATTR_TXTIME_DELAY]); 1557 } 1558 1559 if (!TXTIME_ASSIST_IS_ENABLED(q->flags) && 1560 !FULL_OFFLOAD_IS_ENABLED(q->flags) && 1561 !hrtimer_active(&q->advance_timer)) { 1562 hrtimer_init(&q->advance_timer, q->clockid, HRTIMER_MODE_ABS); 1563 q->advance_timer.function = advance_sched; 1564 } 1565 1566 if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { 1567 q->dequeue = taprio_dequeue_offload; 1568 q->peek = taprio_peek_offload; 1569 } else { 1570 /* Be sure to always keep the function pointers 1571 * in a consistent state. 1572 */ 1573 q->dequeue = taprio_dequeue_soft; 1574 q->peek = taprio_peek_soft; 1575 } 1576 1577 err = taprio_get_start_time(sch, new_admin, &start); 1578 if (err < 0) { 1579 NL_SET_ERR_MSG(extack, "Internal error: failed get start time"); 1580 goto unlock; 1581 } 1582 1583 setup_txtime(q, new_admin, start); 1584 1585 if (TXTIME_ASSIST_IS_ENABLED(q->flags)) { 1586 if (!oper) { 1587 rcu_assign_pointer(q->oper_sched, new_admin); 1588 err = 0; 1589 new_admin = NULL; 1590 goto unlock; 1591 } 1592 1593 rcu_assign_pointer(q->admin_sched, new_admin); 1594 if (admin) 1595 call_rcu(&admin->rcu, taprio_free_sched_cb); 1596 } else { 1597 setup_first_close_time(q, new_admin, start); 1598 1599 /* Protects against advance_sched() */ 1600 spin_lock_irqsave(&q->current_entry_lock, flags); 1601 1602 taprio_start_sched(sch, start, new_admin); 1603 1604 rcu_assign_pointer(q->admin_sched, new_admin); 1605 if (admin) 1606 call_rcu(&admin->rcu, taprio_free_sched_cb); 1607 1608 spin_unlock_irqrestore(&q->current_entry_lock, flags); 1609 1610 if (FULL_OFFLOAD_IS_ENABLED(q->flags)) 1611 taprio_offload_config_changed(q); 1612 } 1613 1614 new_admin = NULL; 1615 err = 0; 1616 1617 unlock: 1618 spin_unlock_bh(qdisc_lock(sch)); 1619 1620 free_sched: 1621 if (new_admin) 1622 call_rcu(&new_admin->rcu, taprio_free_sched_cb); 1623 1624 return err; 1625 } 1626 1627 static void taprio_reset(struct Qdisc *sch) 1628 { 1629 struct taprio_sched *q = qdisc_priv(sch); 1630 struct net_device *dev = qdisc_dev(sch); 1631 int i; 1632 1633 hrtimer_cancel(&q->advance_timer); 1634 if (q->qdiscs) { 1635 for (i = 0; i < dev->num_tx_queues; i++) 1636 if (q->qdiscs[i]) 1637 qdisc_reset(q->qdiscs[i]); 1638 } 1639 } 1640 1641 static void taprio_destroy(struct Qdisc *sch) 1642 { 1643 struct taprio_sched *q = qdisc_priv(sch); 1644 struct net_device *dev = qdisc_dev(sch); 1645 unsigned int i; 1646 1647 spin_lock(&taprio_list_lock); 1648 list_del(&q->taprio_list); 1649 spin_unlock(&taprio_list_lock); 1650 1651 /* Note that taprio_reset() might not be called if an error 1652 * happens in qdisc_create(), after taprio_init() has been called. 1653 */ 1654 hrtimer_cancel(&q->advance_timer); 1655 1656 taprio_disable_offload(dev, q, NULL); 1657 1658 if (q->qdiscs) { 1659 for (i = 0; i < dev->num_tx_queues; i++) 1660 qdisc_put(q->qdiscs[i]); 1661 1662 kfree(q->qdiscs); 1663 } 1664 q->qdiscs = NULL; 1665 1666 netdev_reset_tc(dev); 1667 1668 if (q->oper_sched) 1669 call_rcu(&q->oper_sched->rcu, taprio_free_sched_cb); 1670 1671 if (q->admin_sched) 1672 call_rcu(&q->admin_sched->rcu, taprio_free_sched_cb); 1673 } 1674 1675 static int taprio_init(struct Qdisc *sch, struct nlattr *opt, 1676 struct netlink_ext_ack *extack) 1677 { 1678 struct taprio_sched *q = qdisc_priv(sch); 1679 struct net_device *dev = qdisc_dev(sch); 1680 int i; 1681 1682 spin_lock_init(&q->current_entry_lock); 1683 1684 hrtimer_init(&q->advance_timer, CLOCK_TAI, HRTIMER_MODE_ABS); 1685 q->advance_timer.function = advance_sched; 1686 1687 q->dequeue = taprio_dequeue_soft; 1688 q->peek = taprio_peek_soft; 1689 1690 q->root = sch; 1691 1692 /* We only support static clockids. Use an invalid value as default 1693 * and get the valid one on taprio_change(). 1694 */ 1695 q->clockid = -1; 1696 q->flags = TAPRIO_FLAGS_INVALID; 1697 1698 spin_lock(&taprio_list_lock); 1699 list_add(&q->taprio_list, &taprio_list); 1700 spin_unlock(&taprio_list_lock); 1701 1702 if (sch->parent != TC_H_ROOT) 1703 return -EOPNOTSUPP; 1704 1705 if (!netif_is_multiqueue(dev)) 1706 return -EOPNOTSUPP; 1707 1708 /* pre-allocate qdisc, attachment can't fail */ 1709 q->qdiscs = kcalloc(dev->num_tx_queues, 1710 sizeof(q->qdiscs[0]), 1711 GFP_KERNEL); 1712 1713 if (!q->qdiscs) 1714 return -ENOMEM; 1715 1716 if (!opt) 1717 return -EINVAL; 1718 1719 for (i = 0; i < dev->num_tx_queues; i++) { 1720 struct netdev_queue *dev_queue; 1721 struct Qdisc *qdisc; 1722 1723 dev_queue = netdev_get_tx_queue(dev, i); 1724 qdisc = qdisc_create_dflt(dev_queue, 1725 &pfifo_qdisc_ops, 1726 TC_H_MAKE(TC_H_MAJ(sch->handle), 1727 TC_H_MIN(i + 1)), 1728 extack); 1729 if (!qdisc) 1730 return -ENOMEM; 1731 1732 if (i < dev->real_num_tx_queues) 1733 qdisc_hash_add(qdisc, false); 1734 1735 q->qdiscs[i] = qdisc; 1736 } 1737 1738 return taprio_change(sch, opt, extack); 1739 } 1740 1741 static void taprio_attach(struct Qdisc *sch) 1742 { 1743 struct taprio_sched *q = qdisc_priv(sch); 1744 struct net_device *dev = qdisc_dev(sch); 1745 unsigned int ntx; 1746 1747 /* Attach underlying qdisc */ 1748 for (ntx = 0; ntx < dev->num_tx_queues; ntx++) { 1749 struct Qdisc *qdisc = q->qdiscs[ntx]; 1750 struct Qdisc *old; 1751 1752 if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { 1753 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; 1754 old = dev_graft_qdisc(qdisc->dev_queue, qdisc); 1755 } else { 1756 old = dev_graft_qdisc(qdisc->dev_queue, sch); 1757 qdisc_refcount_inc(sch); 1758 } 1759 if (old) 1760 qdisc_put(old); 1761 } 1762 1763 /* access to the child qdiscs is not needed in offload mode */ 1764 if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { 1765 kfree(q->qdiscs); 1766 q->qdiscs = NULL; 1767 } 1768 } 1769 1770 static struct netdev_queue *taprio_queue_get(struct Qdisc *sch, 1771 unsigned long cl) 1772 { 1773 struct net_device *dev = qdisc_dev(sch); 1774 unsigned long ntx = cl - 1; 1775 1776 if (ntx >= dev->num_tx_queues) 1777 return NULL; 1778 1779 return netdev_get_tx_queue(dev, ntx); 1780 } 1781 1782 static int taprio_graft(struct Qdisc *sch, unsigned long cl, 1783 struct Qdisc *new, struct Qdisc **old, 1784 struct netlink_ext_ack *extack) 1785 { 1786 struct taprio_sched *q = qdisc_priv(sch); 1787 struct net_device *dev = qdisc_dev(sch); 1788 struct netdev_queue *dev_queue = taprio_queue_get(sch, cl); 1789 1790 if (!dev_queue) 1791 return -EINVAL; 1792 1793 if (dev->flags & IFF_UP) 1794 dev_deactivate(dev); 1795 1796 if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { 1797 *old = dev_graft_qdisc(dev_queue, new); 1798 } else { 1799 *old = q->qdiscs[cl - 1]; 1800 q->qdiscs[cl - 1] = new; 1801 } 1802 1803 if (new) 1804 new->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; 1805 1806 if (dev->flags & IFF_UP) 1807 dev_activate(dev); 1808 1809 return 0; 1810 } 1811 1812 static int dump_entry(struct sk_buff *msg, 1813 const struct sched_entry *entry) 1814 { 1815 struct nlattr *item; 1816 1817 item = nla_nest_start_noflag(msg, TCA_TAPRIO_SCHED_ENTRY); 1818 if (!item) 1819 return -ENOSPC; 1820 1821 if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_INDEX, entry->index)) 1822 goto nla_put_failure; 1823 1824 if (nla_put_u8(msg, TCA_TAPRIO_SCHED_ENTRY_CMD, entry->command)) 1825 goto nla_put_failure; 1826 1827 if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_GATE_MASK, 1828 entry->gate_mask)) 1829 goto nla_put_failure; 1830 1831 if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_INTERVAL, 1832 entry->interval)) 1833 goto nla_put_failure; 1834 1835 return nla_nest_end(msg, item); 1836 1837 nla_put_failure: 1838 nla_nest_cancel(msg, item); 1839 return -1; 1840 } 1841 1842 static int dump_schedule(struct sk_buff *msg, 1843 const struct sched_gate_list *root) 1844 { 1845 struct nlattr *entry_list; 1846 struct sched_entry *entry; 1847 1848 if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_BASE_TIME, 1849 root->base_time, TCA_TAPRIO_PAD)) 1850 return -1; 1851 1852 if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME, 1853 root->cycle_time, TCA_TAPRIO_PAD)) 1854 return -1; 1855 1856 if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION, 1857 root->cycle_time_extension, TCA_TAPRIO_PAD)) 1858 return -1; 1859 1860 entry_list = nla_nest_start_noflag(msg, 1861 TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST); 1862 if (!entry_list) 1863 goto error_nest; 1864 1865 list_for_each_entry(entry, &root->entries, list) { 1866 if (dump_entry(msg, entry) < 0) 1867 goto error_nest; 1868 } 1869 1870 nla_nest_end(msg, entry_list); 1871 return 0; 1872 1873 error_nest: 1874 nla_nest_cancel(msg, entry_list); 1875 return -1; 1876 } 1877 1878 static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb) 1879 { 1880 struct taprio_sched *q = qdisc_priv(sch); 1881 struct net_device *dev = qdisc_dev(sch); 1882 struct sched_gate_list *oper, *admin; 1883 struct tc_mqprio_qopt opt = { 0 }; 1884 struct nlattr *nest, *sched_nest; 1885 unsigned int i; 1886 1887 rcu_read_lock(); 1888 oper = rcu_dereference(q->oper_sched); 1889 admin = rcu_dereference(q->admin_sched); 1890 1891 opt.num_tc = netdev_get_num_tc(dev); 1892 memcpy(opt.prio_tc_map, dev->prio_tc_map, sizeof(opt.prio_tc_map)); 1893 1894 for (i = 0; i < netdev_get_num_tc(dev); i++) { 1895 opt.count[i] = dev->tc_to_txq[i].count; 1896 opt.offset[i] = dev->tc_to_txq[i].offset; 1897 } 1898 1899 nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 1900 if (!nest) 1901 goto start_error; 1902 1903 if (nla_put(skb, TCA_TAPRIO_ATTR_PRIOMAP, sizeof(opt), &opt)) 1904 goto options_error; 1905 1906 if (!FULL_OFFLOAD_IS_ENABLED(q->flags) && 1907 nla_put_s32(skb, TCA_TAPRIO_ATTR_SCHED_CLOCKID, q->clockid)) 1908 goto options_error; 1909 1910 if (q->flags && nla_put_u32(skb, TCA_TAPRIO_ATTR_FLAGS, q->flags)) 1911 goto options_error; 1912 1913 if (q->txtime_delay && 1914 nla_put_u32(skb, TCA_TAPRIO_ATTR_TXTIME_DELAY, q->txtime_delay)) 1915 goto options_error; 1916 1917 if (oper && dump_schedule(skb, oper)) 1918 goto options_error; 1919 1920 if (!admin) 1921 goto done; 1922 1923 sched_nest = nla_nest_start_noflag(skb, TCA_TAPRIO_ATTR_ADMIN_SCHED); 1924 if (!sched_nest) 1925 goto options_error; 1926 1927 if (dump_schedule(skb, admin)) 1928 goto admin_error; 1929 1930 nla_nest_end(skb, sched_nest); 1931 1932 done: 1933 rcu_read_unlock(); 1934 1935 return nla_nest_end(skb, nest); 1936 1937 admin_error: 1938 nla_nest_cancel(skb, sched_nest); 1939 1940 options_error: 1941 nla_nest_cancel(skb, nest); 1942 1943 start_error: 1944 rcu_read_unlock(); 1945 return -ENOSPC; 1946 } 1947 1948 static struct Qdisc *taprio_leaf(struct Qdisc *sch, unsigned long cl) 1949 { 1950 struct netdev_queue *dev_queue = taprio_queue_get(sch, cl); 1951 1952 if (!dev_queue) 1953 return NULL; 1954 1955 return dev_queue->qdisc_sleeping; 1956 } 1957 1958 static unsigned long taprio_find(struct Qdisc *sch, u32 classid) 1959 { 1960 unsigned int ntx = TC_H_MIN(classid); 1961 1962 if (!taprio_queue_get(sch, ntx)) 1963 return 0; 1964 return ntx; 1965 } 1966 1967 static int taprio_dump_class(struct Qdisc *sch, unsigned long cl, 1968 struct sk_buff *skb, struct tcmsg *tcm) 1969 { 1970 struct netdev_queue *dev_queue = taprio_queue_get(sch, cl); 1971 1972 tcm->tcm_parent = TC_H_ROOT; 1973 tcm->tcm_handle |= TC_H_MIN(cl); 1974 tcm->tcm_info = dev_queue->qdisc_sleeping->handle; 1975 1976 return 0; 1977 } 1978 1979 static int taprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, 1980 struct gnet_dump *d) 1981 __releases(d->lock) 1982 __acquires(d->lock) 1983 { 1984 struct netdev_queue *dev_queue = taprio_queue_get(sch, cl); 1985 1986 sch = dev_queue->qdisc_sleeping; 1987 if (gnet_stats_copy_basic(d, NULL, &sch->bstats, true) < 0 || 1988 qdisc_qstats_copy(d, sch) < 0) 1989 return -1; 1990 return 0; 1991 } 1992 1993 static void taprio_walk(struct Qdisc *sch, struct qdisc_walker *arg) 1994 { 1995 struct net_device *dev = qdisc_dev(sch); 1996 unsigned long ntx; 1997 1998 if (arg->stop) 1999 return; 2000 2001 arg->count = arg->skip; 2002 for (ntx = arg->skip; ntx < dev->num_tx_queues; ntx++) { 2003 if (arg->fn(sch, ntx + 1, arg) < 0) { 2004 arg->stop = 1; 2005 break; 2006 } 2007 arg->count++; 2008 } 2009 } 2010 2011 static struct netdev_queue *taprio_select_queue(struct Qdisc *sch, 2012 struct tcmsg *tcm) 2013 { 2014 return taprio_queue_get(sch, TC_H_MIN(tcm->tcm_parent)); 2015 } 2016 2017 static const struct Qdisc_class_ops taprio_class_ops = { 2018 .graft = taprio_graft, 2019 .leaf = taprio_leaf, 2020 .find = taprio_find, 2021 .walk = taprio_walk, 2022 .dump = taprio_dump_class, 2023 .dump_stats = taprio_dump_class_stats, 2024 .select_queue = taprio_select_queue, 2025 }; 2026 2027 static struct Qdisc_ops taprio_qdisc_ops __read_mostly = { 2028 .cl_ops = &taprio_class_ops, 2029 .id = "taprio", 2030 .priv_size = sizeof(struct taprio_sched), 2031 .init = taprio_init, 2032 .change = taprio_change, 2033 .destroy = taprio_destroy, 2034 .reset = taprio_reset, 2035 .attach = taprio_attach, 2036 .peek = taprio_peek, 2037 .dequeue = taprio_dequeue, 2038 .enqueue = taprio_enqueue, 2039 .dump = taprio_dump, 2040 .owner = THIS_MODULE, 2041 }; 2042 2043 static struct notifier_block taprio_device_notifier = { 2044 .notifier_call = taprio_dev_notifier, 2045 }; 2046 2047 static int __init taprio_module_init(void) 2048 { 2049 int err = register_netdevice_notifier(&taprio_device_notifier); 2050 2051 if (err) 2052 return err; 2053 2054 return register_qdisc(&taprio_qdisc_ops); 2055 } 2056 2057 static void __exit taprio_module_exit(void) 2058 { 2059 unregister_qdisc(&taprio_qdisc_ops); 2060 unregister_netdevice_notifier(&taprio_device_notifier); 2061 } 2062 2063 module_init(taprio_module_init); 2064 module_exit(taprio_module_exit); 2065 MODULE_LICENSE("GPL"); 2066