15a781ccbSVinicius Costa Gomes // SPDX-License-Identifier: GPL-2.0 25a781ccbSVinicius Costa Gomes 35a781ccbSVinicius Costa Gomes /* net/sched/sch_taprio.c Time Aware Priority Scheduler 45a781ccbSVinicius Costa Gomes * 55a781ccbSVinicius Costa Gomes * Authors: Vinicius Costa Gomes <vinicius.gomes@intel.com> 65a781ccbSVinicius Costa Gomes * 75a781ccbSVinicius Costa Gomes */ 85a781ccbSVinicius Costa Gomes 9cc69837fSJakub Kicinski #include <linux/ethtool.h> 105a781ccbSVinicius Costa Gomes #include <linux/types.h> 115a781ccbSVinicius Costa Gomes #include <linux/slab.h> 125a781ccbSVinicius Costa Gomes #include <linux/kernel.h> 135a781ccbSVinicius Costa Gomes #include <linux/string.h> 145a781ccbSVinicius Costa Gomes #include <linux/list.h> 155a781ccbSVinicius Costa Gomes #include <linux/errno.h> 165a781ccbSVinicius Costa Gomes #include <linux/skbuff.h> 1723bddf69SJakub Kicinski #include <linux/math64.h> 185a781ccbSVinicius Costa Gomes #include <linux/module.h> 195a781ccbSVinicius Costa Gomes #include <linux/spinlock.h> 20a3d43c0dSVinicius Costa Gomes #include <linux/rcupdate.h> 215a781ccbSVinicius Costa Gomes #include <net/netlink.h> 225a781ccbSVinicius Costa Gomes #include <net/pkt_sched.h> 235a781ccbSVinicius Costa Gomes #include <net/pkt_cls.h> 245a781ccbSVinicius Costa Gomes #include <net/sch_generic.h> 254cfd5779SVedang Patel #include <net/sock.h> 2654002066SVedang Patel #include <net/tcp.h> 275a781ccbSVinicius Costa Gomes 287b9eba7bSLeandro Dorileo static LIST_HEAD(taprio_list); 297b9eba7bSLeandro Dorileo static DEFINE_SPINLOCK(taprio_list_lock); 307b9eba7bSLeandro Dorileo 315a781ccbSVinicius Costa Gomes #define TAPRIO_ALL_GATES_OPEN -1 325a781ccbSVinicius Costa Gomes 334cfd5779SVedang Patel #define TXTIME_ASSIST_IS_ENABLED(flags) ((flags) & TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST) 349c66d156SVinicius Costa Gomes #define FULL_OFFLOAD_IS_ENABLED(flags) ((flags) & TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD) 35a9d62274SVinicius Costa Gomes #define TAPRIO_FLAGS_INVALID U32_MAX 364cfd5779SVedang Patel 375a781ccbSVinicius Costa Gomes struct sched_entry { 385a781ccbSVinicius Costa Gomes struct list_head list; 395a781ccbSVinicius Costa Gomes 405a781ccbSVinicius Costa Gomes /* The instant that this entry "closes" and the next one 415a781ccbSVinicius Costa Gomes * should open, the qdisc will make some effort so that no 425a781ccbSVinicius Costa Gomes * packet leaves after this time. 435a781ccbSVinicius Costa Gomes */ 445a781ccbSVinicius Costa Gomes ktime_t close_time; 454cfd5779SVedang Patel ktime_t next_txtime; 465a781ccbSVinicius Costa Gomes atomic_t budget; 475a781ccbSVinicius Costa Gomes int index; 485a781ccbSVinicius Costa Gomes u32 gate_mask; 495a781ccbSVinicius Costa Gomes u32 interval; 505a781ccbSVinicius Costa Gomes u8 command; 515a781ccbSVinicius Costa Gomes }; 525a781ccbSVinicius Costa Gomes 53a3d43c0dSVinicius Costa Gomes struct sched_gate_list { 54a3d43c0dSVinicius Costa Gomes struct rcu_head rcu; 55a3d43c0dSVinicius Costa Gomes struct list_head entries; 56a3d43c0dSVinicius Costa Gomes size_t num_entries; 576ca6a665SVinicius Costa Gomes ktime_t cycle_close_time; 586ca6a665SVinicius Costa Gomes s64 cycle_time; 59c25031e9SVinicius Costa Gomes s64 cycle_time_extension; 60a3d43c0dSVinicius Costa Gomes s64 base_time; 61a3d43c0dSVinicius Costa Gomes }; 62a3d43c0dSVinicius Costa Gomes 635a781ccbSVinicius Costa Gomes struct taprio_sched { 645a781ccbSVinicius Costa Gomes struct Qdisc **qdiscs; 655a781ccbSVinicius Costa Gomes struct Qdisc *root; 664cfd5779SVedang Patel u32 flags; 677ede7b03SVedang Patel enum tk_offsets tk_offset; 685a781ccbSVinicius Costa Gomes int clockid; 697b9eba7bSLeandro Dorileo atomic64_t picos_per_byte; /* Using picoseconds because for 10Gbps+ 705a781ccbSVinicius Costa Gomes * speeds it's sub-nanoseconds per byte 715a781ccbSVinicius Costa Gomes */ 725a781ccbSVinicius Costa Gomes 735a781ccbSVinicius Costa Gomes /* Protects the update side of the RCU protected current_entry */ 745a781ccbSVinicius Costa Gomes spinlock_t current_entry_lock; 755a781ccbSVinicius Costa Gomes struct sched_entry __rcu *current_entry; 76a3d43c0dSVinicius Costa Gomes struct sched_gate_list __rcu *oper_sched; 77a3d43c0dSVinicius Costa Gomes struct sched_gate_list __rcu *admin_sched; 785a781ccbSVinicius Costa Gomes struct hrtimer advance_timer; 797b9eba7bSLeandro Dorileo struct list_head taprio_list; 809c66d156SVinicius Costa Gomes struct sk_buff *(*dequeue)(struct Qdisc *sch); 819c66d156SVinicius Costa Gomes struct sk_buff *(*peek)(struct Qdisc *sch); 82a5b64700SVedang Patel u32 txtime_delay; 835a781ccbSVinicius Costa Gomes }; 845a781ccbSVinicius Costa Gomes 859c66d156SVinicius Costa Gomes struct __tc_taprio_qopt_offload { 869c66d156SVinicius Costa Gomes refcount_t users; 879c66d156SVinicius Costa Gomes struct tc_taprio_qopt_offload offload; 889c66d156SVinicius Costa Gomes }; 899c66d156SVinicius Costa Gomes 90a3d43c0dSVinicius Costa Gomes static ktime_t sched_base_time(const struct sched_gate_list *sched) 91a3d43c0dSVinicius Costa Gomes { 92a3d43c0dSVinicius Costa Gomes if (!sched) 93a3d43c0dSVinicius Costa Gomes return KTIME_MAX; 94a3d43c0dSVinicius Costa Gomes 95a3d43c0dSVinicius Costa Gomes return ns_to_ktime(sched->base_time); 96a3d43c0dSVinicius Costa Gomes } 97a3d43c0dSVinicius Costa Gomes 987ede7b03SVedang Patel static ktime_t taprio_get_time(struct taprio_sched *q) 997ede7b03SVedang Patel { 1007ede7b03SVedang Patel ktime_t mono = ktime_get(); 1017ede7b03SVedang Patel 1027ede7b03SVedang Patel switch (q->tk_offset) { 1037ede7b03SVedang Patel case TK_OFFS_MAX: 1047ede7b03SVedang Patel return mono; 1057ede7b03SVedang Patel default: 1067ede7b03SVedang Patel return ktime_mono_to_any(mono, q->tk_offset); 1077ede7b03SVedang Patel } 1087ede7b03SVedang Patel 1097ede7b03SVedang Patel return KTIME_MAX; 1107ede7b03SVedang Patel } 1117ede7b03SVedang Patel 112a3d43c0dSVinicius Costa Gomes static void taprio_free_sched_cb(struct rcu_head *head) 113a3d43c0dSVinicius Costa Gomes { 114a3d43c0dSVinicius Costa Gomes struct sched_gate_list *sched = container_of(head, struct sched_gate_list, rcu); 115a3d43c0dSVinicius Costa Gomes struct sched_entry *entry, *n; 116a3d43c0dSVinicius Costa Gomes 117a3d43c0dSVinicius Costa Gomes if (!sched) 118a3d43c0dSVinicius Costa Gomes return; 119a3d43c0dSVinicius Costa Gomes 120a3d43c0dSVinicius Costa Gomes list_for_each_entry_safe(entry, n, &sched->entries, list) { 121a3d43c0dSVinicius Costa Gomes list_del(&entry->list); 122a3d43c0dSVinicius Costa Gomes kfree(entry); 123a3d43c0dSVinicius Costa Gomes } 124a3d43c0dSVinicius Costa Gomes 125a3d43c0dSVinicius Costa Gomes kfree(sched); 126a3d43c0dSVinicius Costa Gomes } 127a3d43c0dSVinicius Costa Gomes 128a3d43c0dSVinicius Costa Gomes static void switch_schedules(struct taprio_sched *q, 129a3d43c0dSVinicius Costa Gomes struct sched_gate_list **admin, 130a3d43c0dSVinicius Costa Gomes struct sched_gate_list **oper) 131a3d43c0dSVinicius Costa Gomes { 132a3d43c0dSVinicius Costa Gomes rcu_assign_pointer(q->oper_sched, *admin); 133a3d43c0dSVinicius Costa Gomes rcu_assign_pointer(q->admin_sched, NULL); 134a3d43c0dSVinicius Costa Gomes 135a3d43c0dSVinicius Costa Gomes if (*oper) 136a3d43c0dSVinicius Costa Gomes call_rcu(&(*oper)->rcu, taprio_free_sched_cb); 137a3d43c0dSVinicius Costa Gomes 138a3d43c0dSVinicius Costa Gomes *oper = *admin; 139a3d43c0dSVinicius Costa Gomes *admin = NULL; 140a3d43c0dSVinicius Costa Gomes } 141a3d43c0dSVinicius Costa Gomes 1424cfd5779SVedang Patel /* Get how much time has been already elapsed in the current cycle. */ 1434cfd5779SVedang Patel static s32 get_cycle_time_elapsed(struct sched_gate_list *sched, ktime_t time) 1444cfd5779SVedang Patel { 1454cfd5779SVedang Patel ktime_t time_since_sched_start; 1464cfd5779SVedang Patel s32 time_elapsed; 1474cfd5779SVedang Patel 1484cfd5779SVedang Patel time_since_sched_start = ktime_sub(time, sched->base_time); 1494cfd5779SVedang Patel div_s64_rem(time_since_sched_start, sched->cycle_time, &time_elapsed); 1504cfd5779SVedang Patel 1514cfd5779SVedang Patel return time_elapsed; 1524cfd5779SVedang Patel } 1534cfd5779SVedang Patel 1544cfd5779SVedang Patel static ktime_t get_interval_end_time(struct sched_gate_list *sched, 1554cfd5779SVedang Patel struct sched_gate_list *admin, 1564cfd5779SVedang Patel struct sched_entry *entry, 1574cfd5779SVedang Patel ktime_t intv_start) 1584cfd5779SVedang Patel { 1594cfd5779SVedang Patel s32 cycle_elapsed = get_cycle_time_elapsed(sched, intv_start); 1604cfd5779SVedang Patel ktime_t intv_end, cycle_ext_end, cycle_end; 1614cfd5779SVedang Patel 1624cfd5779SVedang Patel cycle_end = ktime_add_ns(intv_start, sched->cycle_time - cycle_elapsed); 1634cfd5779SVedang Patel intv_end = ktime_add_ns(intv_start, entry->interval); 1644cfd5779SVedang Patel cycle_ext_end = ktime_add(cycle_end, sched->cycle_time_extension); 1654cfd5779SVedang Patel 1664cfd5779SVedang Patel if (ktime_before(intv_end, cycle_end)) 1674cfd5779SVedang Patel return intv_end; 1684cfd5779SVedang Patel else if (admin && admin != sched && 1694cfd5779SVedang Patel ktime_after(admin->base_time, cycle_end) && 1704cfd5779SVedang Patel ktime_before(admin->base_time, cycle_ext_end)) 1714cfd5779SVedang Patel return admin->base_time; 1724cfd5779SVedang Patel else 1734cfd5779SVedang Patel return cycle_end; 1744cfd5779SVedang Patel } 1754cfd5779SVedang Patel 1764cfd5779SVedang Patel static int length_to_duration(struct taprio_sched *q, int len) 1774cfd5779SVedang Patel { 1784cfd5779SVedang Patel return div_u64(len * atomic64_read(&q->picos_per_byte), 1000); 1794cfd5779SVedang Patel } 1804cfd5779SVedang Patel 1814cfd5779SVedang Patel /* Returns the entry corresponding to next available interval. If 1824cfd5779SVedang Patel * validate_interval is set, it only validates whether the timestamp occurs 1834cfd5779SVedang Patel * when the gate corresponding to the skb's traffic class is open. 1844cfd5779SVedang Patel */ 1854cfd5779SVedang Patel static struct sched_entry *find_entry_to_transmit(struct sk_buff *skb, 1864cfd5779SVedang Patel struct Qdisc *sch, 1874cfd5779SVedang Patel struct sched_gate_list *sched, 1884cfd5779SVedang Patel struct sched_gate_list *admin, 1894cfd5779SVedang Patel ktime_t time, 1904cfd5779SVedang Patel ktime_t *interval_start, 1914cfd5779SVedang Patel ktime_t *interval_end, 1924cfd5779SVedang Patel bool validate_interval) 1934cfd5779SVedang Patel { 1944cfd5779SVedang Patel ktime_t curr_intv_start, curr_intv_end, cycle_end, packet_transmit_time; 1954cfd5779SVedang Patel ktime_t earliest_txtime = KTIME_MAX, txtime, cycle, transmit_end_time; 1964cfd5779SVedang Patel struct sched_entry *entry = NULL, *entry_found = NULL; 1974cfd5779SVedang Patel struct taprio_sched *q = qdisc_priv(sch); 1984cfd5779SVedang Patel struct net_device *dev = qdisc_dev(sch); 1994cfd5779SVedang Patel bool entry_available = false; 2004cfd5779SVedang Patel s32 cycle_elapsed; 2014cfd5779SVedang Patel int tc, n; 2024cfd5779SVedang Patel 2034cfd5779SVedang Patel tc = netdev_get_prio_tc_map(dev, skb->priority); 2044cfd5779SVedang Patel packet_transmit_time = length_to_duration(q, qdisc_pkt_len(skb)); 2054cfd5779SVedang Patel 2064cfd5779SVedang Patel *interval_start = 0; 2074cfd5779SVedang Patel *interval_end = 0; 2084cfd5779SVedang Patel 2094cfd5779SVedang Patel if (!sched) 2104cfd5779SVedang Patel return NULL; 2114cfd5779SVedang Patel 2124cfd5779SVedang Patel cycle = sched->cycle_time; 2134cfd5779SVedang Patel cycle_elapsed = get_cycle_time_elapsed(sched, time); 2144cfd5779SVedang Patel curr_intv_end = ktime_sub_ns(time, cycle_elapsed); 2154cfd5779SVedang Patel cycle_end = ktime_add_ns(curr_intv_end, cycle); 2164cfd5779SVedang Patel 2174cfd5779SVedang Patel list_for_each_entry(entry, &sched->entries, list) { 2184cfd5779SVedang Patel curr_intv_start = curr_intv_end; 2194cfd5779SVedang Patel curr_intv_end = get_interval_end_time(sched, admin, entry, 2204cfd5779SVedang Patel curr_intv_start); 2214cfd5779SVedang Patel 2224cfd5779SVedang Patel if (ktime_after(curr_intv_start, cycle_end)) 2234cfd5779SVedang Patel break; 2244cfd5779SVedang Patel 2254cfd5779SVedang Patel if (!(entry->gate_mask & BIT(tc)) || 2264cfd5779SVedang Patel packet_transmit_time > entry->interval) 2274cfd5779SVedang Patel continue; 2284cfd5779SVedang Patel 2294cfd5779SVedang Patel txtime = entry->next_txtime; 2304cfd5779SVedang Patel 2314cfd5779SVedang Patel if (ktime_before(txtime, time) || validate_interval) { 2324cfd5779SVedang Patel transmit_end_time = ktime_add_ns(time, packet_transmit_time); 2334cfd5779SVedang Patel if ((ktime_before(curr_intv_start, time) && 2344cfd5779SVedang Patel ktime_before(transmit_end_time, curr_intv_end)) || 2354cfd5779SVedang Patel (ktime_after(curr_intv_start, time) && !validate_interval)) { 2364cfd5779SVedang Patel entry_found = entry; 2374cfd5779SVedang Patel *interval_start = curr_intv_start; 2384cfd5779SVedang Patel *interval_end = curr_intv_end; 2394cfd5779SVedang Patel break; 2404cfd5779SVedang Patel } else if (!entry_available && !validate_interval) { 2414cfd5779SVedang Patel /* Here, we are just trying to find out the 2424cfd5779SVedang Patel * first available interval in the next cycle. 2434cfd5779SVedang Patel */ 2440deee7aaSJiapeng Zhong entry_available = true; 2454cfd5779SVedang Patel entry_found = entry; 2464cfd5779SVedang Patel *interval_start = ktime_add_ns(curr_intv_start, cycle); 2474cfd5779SVedang Patel *interval_end = ktime_add_ns(curr_intv_end, cycle); 2484cfd5779SVedang Patel } 2494cfd5779SVedang Patel } else if (ktime_before(txtime, earliest_txtime) && 2504cfd5779SVedang Patel !entry_available) { 2514cfd5779SVedang Patel earliest_txtime = txtime; 2524cfd5779SVedang Patel entry_found = entry; 2534cfd5779SVedang Patel n = div_s64(ktime_sub(txtime, curr_intv_start), cycle); 2544cfd5779SVedang Patel *interval_start = ktime_add(curr_intv_start, n * cycle); 2554cfd5779SVedang Patel *interval_end = ktime_add(curr_intv_end, n * cycle); 2564cfd5779SVedang Patel } 2574cfd5779SVedang Patel } 2584cfd5779SVedang Patel 2594cfd5779SVedang Patel return entry_found; 2604cfd5779SVedang Patel } 2614cfd5779SVedang Patel 2624cfd5779SVedang Patel static bool is_valid_interval(struct sk_buff *skb, struct Qdisc *sch) 2634cfd5779SVedang Patel { 2644cfd5779SVedang Patel struct taprio_sched *q = qdisc_priv(sch); 2654cfd5779SVedang Patel struct sched_gate_list *sched, *admin; 2664cfd5779SVedang Patel ktime_t interval_start, interval_end; 2674cfd5779SVedang Patel struct sched_entry *entry; 2684cfd5779SVedang Patel 2694cfd5779SVedang Patel rcu_read_lock(); 2704cfd5779SVedang Patel sched = rcu_dereference(q->oper_sched); 2714cfd5779SVedang Patel admin = rcu_dereference(q->admin_sched); 2724cfd5779SVedang Patel 2734cfd5779SVedang Patel entry = find_entry_to_transmit(skb, sch, sched, admin, skb->tstamp, 2744cfd5779SVedang Patel &interval_start, &interval_end, true); 2754cfd5779SVedang Patel rcu_read_unlock(); 2764cfd5779SVedang Patel 2774cfd5779SVedang Patel return entry; 2784cfd5779SVedang Patel } 2794cfd5779SVedang Patel 2809c66d156SVinicius Costa Gomes static bool taprio_flags_valid(u32 flags) 2819c66d156SVinicius Costa Gomes { 2829c66d156SVinicius Costa Gomes /* Make sure no other flag bits are set. */ 2839c66d156SVinicius Costa Gomes if (flags & ~(TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST | 2849c66d156SVinicius Costa Gomes TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD)) 2859c66d156SVinicius Costa Gomes return false; 2869c66d156SVinicius Costa Gomes /* txtime-assist and full offload are mutually exclusive */ 2879c66d156SVinicius Costa Gomes if ((flags & TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST) && 2889c66d156SVinicius Costa Gomes (flags & TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD)) 2899c66d156SVinicius Costa Gomes return false; 2909c66d156SVinicius Costa Gomes return true; 2919c66d156SVinicius Costa Gomes } 2929c66d156SVinicius Costa Gomes 29354002066SVedang Patel /* This returns the tstamp value set by TCP in terms of the set clock. */ 29454002066SVedang Patel static ktime_t get_tcp_tstamp(struct taprio_sched *q, struct sk_buff *skb) 29554002066SVedang Patel { 29654002066SVedang Patel unsigned int offset = skb_network_offset(skb); 29754002066SVedang Patel const struct ipv6hdr *ipv6h; 29854002066SVedang Patel const struct iphdr *iph; 29954002066SVedang Patel struct ipv6hdr _ipv6h; 30054002066SVedang Patel 30154002066SVedang Patel ipv6h = skb_header_pointer(skb, offset, sizeof(_ipv6h), &_ipv6h); 30254002066SVedang Patel if (!ipv6h) 30354002066SVedang Patel return 0; 30454002066SVedang Patel 30554002066SVedang Patel if (ipv6h->version == 4) { 30654002066SVedang Patel iph = (struct iphdr *)ipv6h; 30754002066SVedang Patel offset += iph->ihl * 4; 30854002066SVedang Patel 30954002066SVedang Patel /* special-case 6in4 tunnelling, as that is a common way to get 31054002066SVedang Patel * v6 connectivity in the home 31154002066SVedang Patel */ 31254002066SVedang Patel if (iph->protocol == IPPROTO_IPV6) { 31354002066SVedang Patel ipv6h = skb_header_pointer(skb, offset, 31454002066SVedang Patel sizeof(_ipv6h), &_ipv6h); 31554002066SVedang Patel 31654002066SVedang Patel if (!ipv6h || ipv6h->nexthdr != IPPROTO_TCP) 31754002066SVedang Patel return 0; 31854002066SVedang Patel } else if (iph->protocol != IPPROTO_TCP) { 31954002066SVedang Patel return 0; 32054002066SVedang Patel } 32154002066SVedang Patel } else if (ipv6h->version == 6 && ipv6h->nexthdr != IPPROTO_TCP) { 32254002066SVedang Patel return 0; 32354002066SVedang Patel } 32454002066SVedang Patel 32554002066SVedang Patel return ktime_mono_to_any(skb->skb_mstamp_ns, q->tk_offset); 32654002066SVedang Patel } 32754002066SVedang Patel 3284cfd5779SVedang Patel /* There are a few scenarios where we will have to modify the txtime from 3294cfd5779SVedang Patel * what is read from next_txtime in sched_entry. They are: 3304cfd5779SVedang Patel * 1. If txtime is in the past, 3314cfd5779SVedang Patel * a. The gate for the traffic class is currently open and packet can be 3324cfd5779SVedang Patel * transmitted before it closes, schedule the packet right away. 3334cfd5779SVedang Patel * b. If the gate corresponding to the traffic class is going to open later 3344cfd5779SVedang Patel * in the cycle, set the txtime of packet to the interval start. 3354cfd5779SVedang Patel * 2. If txtime is in the future, there are packets corresponding to the 3364cfd5779SVedang Patel * current traffic class waiting to be transmitted. So, the following 3374cfd5779SVedang Patel * possibilities exist: 3384cfd5779SVedang Patel * a. We can transmit the packet before the window containing the txtime 3394cfd5779SVedang Patel * closes. 3404cfd5779SVedang Patel * b. The window might close before the transmission can be completed 3414cfd5779SVedang Patel * successfully. So, schedule the packet in the next open window. 3424cfd5779SVedang Patel */ 3434cfd5779SVedang Patel static long get_packet_txtime(struct sk_buff *skb, struct Qdisc *sch) 3444cfd5779SVedang Patel { 34554002066SVedang Patel ktime_t transmit_end_time, interval_end, interval_start, tcp_tstamp; 3464cfd5779SVedang Patel struct taprio_sched *q = qdisc_priv(sch); 3474cfd5779SVedang Patel struct sched_gate_list *sched, *admin; 3484cfd5779SVedang Patel ktime_t minimum_time, now, txtime; 3494cfd5779SVedang Patel int len, packet_transmit_time; 3504cfd5779SVedang Patel struct sched_entry *entry; 3514cfd5779SVedang Patel bool sched_changed; 3524cfd5779SVedang Patel 3537ede7b03SVedang Patel now = taprio_get_time(q); 3544cfd5779SVedang Patel minimum_time = ktime_add_ns(now, q->txtime_delay); 3554cfd5779SVedang Patel 35654002066SVedang Patel tcp_tstamp = get_tcp_tstamp(q, skb); 35754002066SVedang Patel minimum_time = max_t(ktime_t, minimum_time, tcp_tstamp); 35854002066SVedang Patel 3594cfd5779SVedang Patel rcu_read_lock(); 3604cfd5779SVedang Patel admin = rcu_dereference(q->admin_sched); 3614cfd5779SVedang Patel sched = rcu_dereference(q->oper_sched); 3624cfd5779SVedang Patel if (admin && ktime_after(minimum_time, admin->base_time)) 3634cfd5779SVedang Patel switch_schedules(q, &admin, &sched); 3644cfd5779SVedang Patel 3654cfd5779SVedang Patel /* Until the schedule starts, all the queues are open */ 3664cfd5779SVedang Patel if (!sched || ktime_before(minimum_time, sched->base_time)) { 3674cfd5779SVedang Patel txtime = minimum_time; 3684cfd5779SVedang Patel goto done; 3694cfd5779SVedang Patel } 3704cfd5779SVedang Patel 3714cfd5779SVedang Patel len = qdisc_pkt_len(skb); 3724cfd5779SVedang Patel packet_transmit_time = length_to_duration(q, len); 3734cfd5779SVedang Patel 3744cfd5779SVedang Patel do { 3750deee7aaSJiapeng Zhong sched_changed = false; 3764cfd5779SVedang Patel 3774cfd5779SVedang Patel entry = find_entry_to_transmit(skb, sch, sched, admin, 3784cfd5779SVedang Patel minimum_time, 3794cfd5779SVedang Patel &interval_start, &interval_end, 3804cfd5779SVedang Patel false); 3814cfd5779SVedang Patel if (!entry) { 3824cfd5779SVedang Patel txtime = 0; 3834cfd5779SVedang Patel goto done; 3844cfd5779SVedang Patel } 3854cfd5779SVedang Patel 3864cfd5779SVedang Patel txtime = entry->next_txtime; 3874cfd5779SVedang Patel txtime = max_t(ktime_t, txtime, minimum_time); 3884cfd5779SVedang Patel txtime = max_t(ktime_t, txtime, interval_start); 3894cfd5779SVedang Patel 3904cfd5779SVedang Patel if (admin && admin != sched && 3914cfd5779SVedang Patel ktime_after(txtime, admin->base_time)) { 3924cfd5779SVedang Patel sched = admin; 3930deee7aaSJiapeng Zhong sched_changed = true; 3944cfd5779SVedang Patel continue; 3954cfd5779SVedang Patel } 3964cfd5779SVedang Patel 3974cfd5779SVedang Patel transmit_end_time = ktime_add(txtime, packet_transmit_time); 3984cfd5779SVedang Patel minimum_time = transmit_end_time; 3994cfd5779SVedang Patel 4004cfd5779SVedang Patel /* Update the txtime of current entry to the next time it's 4014cfd5779SVedang Patel * interval starts. 4024cfd5779SVedang Patel */ 4034cfd5779SVedang Patel if (ktime_after(transmit_end_time, interval_end)) 4044cfd5779SVedang Patel entry->next_txtime = ktime_add(interval_start, sched->cycle_time); 4054cfd5779SVedang Patel } while (sched_changed || ktime_after(transmit_end_time, interval_end)); 4064cfd5779SVedang Patel 4074cfd5779SVedang Patel entry->next_txtime = transmit_end_time; 4084cfd5779SVedang Patel 4094cfd5779SVedang Patel done: 4104cfd5779SVedang Patel rcu_read_unlock(); 4114cfd5779SVedang Patel return txtime; 4124cfd5779SVedang Patel } 4134cfd5779SVedang Patel 414*497cc002SKurt Kanzenbach static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch, 415*497cc002SKurt Kanzenbach struct Qdisc *child, struct sk_buff **to_free) 4165a781ccbSVinicius Costa Gomes { 4175a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 4185a781ccbSVinicius Costa Gomes 4194cfd5779SVedang Patel if (skb->sk && sock_flag(skb->sk, SOCK_TXTIME)) { 4204cfd5779SVedang Patel if (!is_valid_interval(skb, sch)) 4214cfd5779SVedang Patel return qdisc_drop(skb, sch, to_free); 4224cfd5779SVedang Patel } else if (TXTIME_ASSIST_IS_ENABLED(q->flags)) { 4234cfd5779SVedang Patel skb->tstamp = get_packet_txtime(skb, sch); 4244cfd5779SVedang Patel if (!skb->tstamp) 4254cfd5779SVedang Patel return qdisc_drop(skb, sch, to_free); 4264cfd5779SVedang Patel } 4274cfd5779SVedang Patel 4285a781ccbSVinicius Costa Gomes qdisc_qstats_backlog_inc(sch, skb); 4295a781ccbSVinicius Costa Gomes sch->q.qlen++; 4305a781ccbSVinicius Costa Gomes 431ac5c66f2SPetr Machata return qdisc_enqueue(skb, child, to_free); 4325a781ccbSVinicius Costa Gomes } 4335a781ccbSVinicius Costa Gomes 434*497cc002SKurt Kanzenbach static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, 435*497cc002SKurt Kanzenbach struct sk_buff **to_free) 436*497cc002SKurt Kanzenbach { 437*497cc002SKurt Kanzenbach struct taprio_sched *q = qdisc_priv(sch); 438*497cc002SKurt Kanzenbach struct Qdisc *child; 439*497cc002SKurt Kanzenbach int queue; 440*497cc002SKurt Kanzenbach 441*497cc002SKurt Kanzenbach queue = skb_get_queue_mapping(skb); 442*497cc002SKurt Kanzenbach 443*497cc002SKurt Kanzenbach child = q->qdiscs[queue]; 444*497cc002SKurt Kanzenbach if (unlikely(!child)) 445*497cc002SKurt Kanzenbach return qdisc_drop(skb, sch, to_free); 446*497cc002SKurt Kanzenbach 447*497cc002SKurt Kanzenbach /* Large packets might not be transmitted when the transmission duration 448*497cc002SKurt Kanzenbach * exceeds any configured interval. Therefore, segment the skb into 449*497cc002SKurt Kanzenbach * smaller chunks. Skip it for the full offload case, as the driver 450*497cc002SKurt Kanzenbach * and/or the hardware is expected to handle this. 451*497cc002SKurt Kanzenbach */ 452*497cc002SKurt Kanzenbach if (skb_is_gso(skb) && !FULL_OFFLOAD_IS_ENABLED(q->flags)) { 453*497cc002SKurt Kanzenbach unsigned int slen = 0, numsegs = 0, len = qdisc_pkt_len(skb); 454*497cc002SKurt Kanzenbach netdev_features_t features = netif_skb_features(skb); 455*497cc002SKurt Kanzenbach struct sk_buff *segs, *nskb; 456*497cc002SKurt Kanzenbach int ret; 457*497cc002SKurt Kanzenbach 458*497cc002SKurt Kanzenbach segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); 459*497cc002SKurt Kanzenbach if (IS_ERR_OR_NULL(segs)) 460*497cc002SKurt Kanzenbach return qdisc_drop(skb, sch, to_free); 461*497cc002SKurt Kanzenbach 462*497cc002SKurt Kanzenbach skb_list_walk_safe(segs, segs, nskb) { 463*497cc002SKurt Kanzenbach skb_mark_not_on_list(segs); 464*497cc002SKurt Kanzenbach qdisc_skb_cb(segs)->pkt_len = segs->len; 465*497cc002SKurt Kanzenbach slen += segs->len; 466*497cc002SKurt Kanzenbach 467*497cc002SKurt Kanzenbach ret = taprio_enqueue_one(segs, sch, child, to_free); 468*497cc002SKurt Kanzenbach if (ret != NET_XMIT_SUCCESS) { 469*497cc002SKurt Kanzenbach if (net_xmit_drop_count(ret)) 470*497cc002SKurt Kanzenbach qdisc_qstats_drop(sch); 471*497cc002SKurt Kanzenbach } else { 472*497cc002SKurt Kanzenbach numsegs++; 473*497cc002SKurt Kanzenbach } 474*497cc002SKurt Kanzenbach } 475*497cc002SKurt Kanzenbach 476*497cc002SKurt Kanzenbach if (numsegs > 1) 477*497cc002SKurt Kanzenbach qdisc_tree_reduce_backlog(sch, 1 - numsegs, len - slen); 478*497cc002SKurt Kanzenbach consume_skb(skb); 479*497cc002SKurt Kanzenbach 480*497cc002SKurt Kanzenbach return numsegs > 0 ? NET_XMIT_SUCCESS : NET_XMIT_DROP; 481*497cc002SKurt Kanzenbach } 482*497cc002SKurt Kanzenbach 483*497cc002SKurt Kanzenbach return taprio_enqueue_one(skb, sch, child, to_free); 484*497cc002SKurt Kanzenbach } 485*497cc002SKurt Kanzenbach 4869c66d156SVinicius Costa Gomes static struct sk_buff *taprio_peek_soft(struct Qdisc *sch) 4875a781ccbSVinicius Costa Gomes { 4885a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 4895a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 4905a781ccbSVinicius Costa Gomes struct sched_entry *entry; 4915a781ccbSVinicius Costa Gomes struct sk_buff *skb; 4925a781ccbSVinicius Costa Gomes u32 gate_mask; 4935a781ccbSVinicius Costa Gomes int i; 4945a781ccbSVinicius Costa Gomes 4955a781ccbSVinicius Costa Gomes rcu_read_lock(); 4965a781ccbSVinicius Costa Gomes entry = rcu_dereference(q->current_entry); 4972684d1b7SAndre Guedes gate_mask = entry ? entry->gate_mask : TAPRIO_ALL_GATES_OPEN; 4985a781ccbSVinicius Costa Gomes rcu_read_unlock(); 4995a781ccbSVinicius Costa Gomes 5005a781ccbSVinicius Costa Gomes if (!gate_mask) 5015a781ccbSVinicius Costa Gomes return NULL; 5025a781ccbSVinicius Costa Gomes 5035a781ccbSVinicius Costa Gomes for (i = 0; i < dev->num_tx_queues; i++) { 5045a781ccbSVinicius Costa Gomes struct Qdisc *child = q->qdiscs[i]; 5055a781ccbSVinicius Costa Gomes int prio; 5065a781ccbSVinicius Costa Gomes u8 tc; 5075a781ccbSVinicius Costa Gomes 5085a781ccbSVinicius Costa Gomes if (unlikely(!child)) 5095a781ccbSVinicius Costa Gomes continue; 5105a781ccbSVinicius Costa Gomes 5115a781ccbSVinicius Costa Gomes skb = child->ops->peek(child); 5125a781ccbSVinicius Costa Gomes if (!skb) 5135a781ccbSVinicius Costa Gomes continue; 5145a781ccbSVinicius Costa Gomes 5154cfd5779SVedang Patel if (TXTIME_ASSIST_IS_ENABLED(q->flags)) 5164cfd5779SVedang Patel return skb; 5174cfd5779SVedang Patel 5185a781ccbSVinicius Costa Gomes prio = skb->priority; 5195a781ccbSVinicius Costa Gomes tc = netdev_get_prio_tc_map(dev, prio); 5205a781ccbSVinicius Costa Gomes 5215a781ccbSVinicius Costa Gomes if (!(gate_mask & BIT(tc))) 5222684d1b7SAndre Guedes continue; 5235a781ccbSVinicius Costa Gomes 5245a781ccbSVinicius Costa Gomes return skb; 5255a781ccbSVinicius Costa Gomes } 5265a781ccbSVinicius Costa Gomes 5275a781ccbSVinicius Costa Gomes return NULL; 5285a781ccbSVinicius Costa Gomes } 5295a781ccbSVinicius Costa Gomes 5309c66d156SVinicius Costa Gomes static struct sk_buff *taprio_peek_offload(struct Qdisc *sch) 5319c66d156SVinicius Costa Gomes { 5329c66d156SVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 5339c66d156SVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 5349c66d156SVinicius Costa Gomes struct sk_buff *skb; 5359c66d156SVinicius Costa Gomes int i; 5369c66d156SVinicius Costa Gomes 5379c66d156SVinicius Costa Gomes for (i = 0; i < dev->num_tx_queues; i++) { 5389c66d156SVinicius Costa Gomes struct Qdisc *child = q->qdiscs[i]; 5399c66d156SVinicius Costa Gomes 5409c66d156SVinicius Costa Gomes if (unlikely(!child)) 5419c66d156SVinicius Costa Gomes continue; 5429c66d156SVinicius Costa Gomes 5439c66d156SVinicius Costa Gomes skb = child->ops->peek(child); 5449c66d156SVinicius Costa Gomes if (!skb) 5459c66d156SVinicius Costa Gomes continue; 5469c66d156SVinicius Costa Gomes 5479c66d156SVinicius Costa Gomes return skb; 5489c66d156SVinicius Costa Gomes } 5499c66d156SVinicius Costa Gomes 5509c66d156SVinicius Costa Gomes return NULL; 5519c66d156SVinicius Costa Gomes } 5529c66d156SVinicius Costa Gomes 5539c66d156SVinicius Costa Gomes static struct sk_buff *taprio_peek(struct Qdisc *sch) 5549c66d156SVinicius Costa Gomes { 5559c66d156SVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 5569c66d156SVinicius Costa Gomes 5579c66d156SVinicius Costa Gomes return q->peek(sch); 5589c66d156SVinicius Costa Gomes } 5599c66d156SVinicius Costa Gomes 56023bddf69SJakub Kicinski static void taprio_set_budget(struct taprio_sched *q, struct sched_entry *entry) 56123bddf69SJakub Kicinski { 56223bddf69SJakub Kicinski atomic_set(&entry->budget, 56323bddf69SJakub Kicinski div64_u64((u64)entry->interval * 1000, 56423bddf69SJakub Kicinski atomic64_read(&q->picos_per_byte))); 5655a781ccbSVinicius Costa Gomes } 5665a781ccbSVinicius Costa Gomes 5679c66d156SVinicius Costa Gomes static struct sk_buff *taprio_dequeue_soft(struct Qdisc *sch) 5685a781ccbSVinicius Costa Gomes { 5695a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 5705a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 5718c79f0eaSVinicius Costa Gomes struct sk_buff *skb = NULL; 5725a781ccbSVinicius Costa Gomes struct sched_entry *entry; 5735a781ccbSVinicius Costa Gomes u32 gate_mask; 5745a781ccbSVinicius Costa Gomes int i; 5755a781ccbSVinicius Costa Gomes 5765a781ccbSVinicius Costa Gomes rcu_read_lock(); 5775a781ccbSVinicius Costa Gomes entry = rcu_dereference(q->current_entry); 5785a781ccbSVinicius Costa Gomes /* if there's no entry, it means that the schedule didn't 5795a781ccbSVinicius Costa Gomes * start yet, so force all gates to be open, this is in 5805a781ccbSVinicius Costa Gomes * accordance to IEEE 802.1Qbv-2015 Section 8.6.9.4.5 5815a781ccbSVinicius Costa Gomes * "AdminGateSates" 5825a781ccbSVinicius Costa Gomes */ 5835a781ccbSVinicius Costa Gomes gate_mask = entry ? entry->gate_mask : TAPRIO_ALL_GATES_OPEN; 5845a781ccbSVinicius Costa Gomes 5855a781ccbSVinicius Costa Gomes if (!gate_mask) 5868c79f0eaSVinicius Costa Gomes goto done; 5875a781ccbSVinicius Costa Gomes 5885a781ccbSVinicius Costa Gomes for (i = 0; i < dev->num_tx_queues; i++) { 5895a781ccbSVinicius Costa Gomes struct Qdisc *child = q->qdiscs[i]; 5905a781ccbSVinicius Costa Gomes ktime_t guard; 5915a781ccbSVinicius Costa Gomes int prio; 5925a781ccbSVinicius Costa Gomes int len; 5935a781ccbSVinicius Costa Gomes u8 tc; 5945a781ccbSVinicius Costa Gomes 5955a781ccbSVinicius Costa Gomes if (unlikely(!child)) 5965a781ccbSVinicius Costa Gomes continue; 5975a781ccbSVinicius Costa Gomes 5984cfd5779SVedang Patel if (TXTIME_ASSIST_IS_ENABLED(q->flags)) { 5994cfd5779SVedang Patel skb = child->ops->dequeue(child); 6004cfd5779SVedang Patel if (!skb) 6014cfd5779SVedang Patel continue; 6024cfd5779SVedang Patel goto skb_found; 6034cfd5779SVedang Patel } 6044cfd5779SVedang Patel 6055a781ccbSVinicius Costa Gomes skb = child->ops->peek(child); 6065a781ccbSVinicius Costa Gomes if (!skb) 6075a781ccbSVinicius Costa Gomes continue; 6085a781ccbSVinicius Costa Gomes 6095a781ccbSVinicius Costa Gomes prio = skb->priority; 6105a781ccbSVinicius Costa Gomes tc = netdev_get_prio_tc_map(dev, prio); 6115a781ccbSVinicius Costa Gomes 612b09fe70eSVinicius Costa Gomes if (!(gate_mask & BIT(tc))) { 613b09fe70eSVinicius Costa Gomes skb = NULL; 6145a781ccbSVinicius Costa Gomes continue; 615b09fe70eSVinicius Costa Gomes } 6165a781ccbSVinicius Costa Gomes 6175a781ccbSVinicius Costa Gomes len = qdisc_pkt_len(skb); 6187ede7b03SVedang Patel guard = ktime_add_ns(taprio_get_time(q), 6195a781ccbSVinicius Costa Gomes length_to_duration(q, len)); 6205a781ccbSVinicius Costa Gomes 6215a781ccbSVinicius Costa Gomes /* In the case that there's no gate entry, there's no 6225a781ccbSVinicius Costa Gomes * guard band ... 6235a781ccbSVinicius Costa Gomes */ 6245a781ccbSVinicius Costa Gomes if (gate_mask != TAPRIO_ALL_GATES_OPEN && 625b09fe70eSVinicius Costa Gomes ktime_after(guard, entry->close_time)) { 626b09fe70eSVinicius Costa Gomes skb = NULL; 6276e734c82SAndre Guedes continue; 628b09fe70eSVinicius Costa Gomes } 6295a781ccbSVinicius Costa Gomes 6305a781ccbSVinicius Costa Gomes /* ... and no budget. */ 6315a781ccbSVinicius Costa Gomes if (gate_mask != TAPRIO_ALL_GATES_OPEN && 632b09fe70eSVinicius Costa Gomes atomic_sub_return(len, &entry->budget) < 0) { 633b09fe70eSVinicius Costa Gomes skb = NULL; 6346e734c82SAndre Guedes continue; 635b09fe70eSVinicius Costa Gomes } 6365a781ccbSVinicius Costa Gomes 6375a781ccbSVinicius Costa Gomes skb = child->ops->dequeue(child); 6385a781ccbSVinicius Costa Gomes if (unlikely(!skb)) 6398c79f0eaSVinicius Costa Gomes goto done; 6405a781ccbSVinicius Costa Gomes 6414cfd5779SVedang Patel skb_found: 6425a781ccbSVinicius Costa Gomes qdisc_bstats_update(sch, skb); 6435a781ccbSVinicius Costa Gomes qdisc_qstats_backlog_dec(sch, skb); 6445a781ccbSVinicius Costa Gomes sch->q.qlen--; 6455a781ccbSVinicius Costa Gomes 6468c79f0eaSVinicius Costa Gomes goto done; 6475a781ccbSVinicius Costa Gomes } 6485a781ccbSVinicius Costa Gomes 6498c79f0eaSVinicius Costa Gomes done: 6508c79f0eaSVinicius Costa Gomes rcu_read_unlock(); 6518c79f0eaSVinicius Costa Gomes 6528c79f0eaSVinicius Costa Gomes return skb; 6535a781ccbSVinicius Costa Gomes } 6545a781ccbSVinicius Costa Gomes 6559c66d156SVinicius Costa Gomes static struct sk_buff *taprio_dequeue_offload(struct Qdisc *sch) 6569c66d156SVinicius Costa Gomes { 6579c66d156SVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 6589c66d156SVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 6599c66d156SVinicius Costa Gomes struct sk_buff *skb; 6609c66d156SVinicius Costa Gomes int i; 6619c66d156SVinicius Costa Gomes 6629c66d156SVinicius Costa Gomes for (i = 0; i < dev->num_tx_queues; i++) { 6639c66d156SVinicius Costa Gomes struct Qdisc *child = q->qdiscs[i]; 6649c66d156SVinicius Costa Gomes 6659c66d156SVinicius Costa Gomes if (unlikely(!child)) 6669c66d156SVinicius Costa Gomes continue; 6679c66d156SVinicius Costa Gomes 6689c66d156SVinicius Costa Gomes skb = child->ops->dequeue(child); 6699c66d156SVinicius Costa Gomes if (unlikely(!skb)) 6709c66d156SVinicius Costa Gomes continue; 6719c66d156SVinicius Costa Gomes 6729c66d156SVinicius Costa Gomes qdisc_bstats_update(sch, skb); 6739c66d156SVinicius Costa Gomes qdisc_qstats_backlog_dec(sch, skb); 6749c66d156SVinicius Costa Gomes sch->q.qlen--; 6759c66d156SVinicius Costa Gomes 6769c66d156SVinicius Costa Gomes return skb; 6779c66d156SVinicius Costa Gomes } 6789c66d156SVinicius Costa Gomes 6799c66d156SVinicius Costa Gomes return NULL; 6809c66d156SVinicius Costa Gomes } 6819c66d156SVinicius Costa Gomes 6829c66d156SVinicius Costa Gomes static struct sk_buff *taprio_dequeue(struct Qdisc *sch) 6839c66d156SVinicius Costa Gomes { 6849c66d156SVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 6859c66d156SVinicius Costa Gomes 6869c66d156SVinicius Costa Gomes return q->dequeue(sch); 6879c66d156SVinicius Costa Gomes } 6889c66d156SVinicius Costa Gomes 6896ca6a665SVinicius Costa Gomes static bool should_restart_cycle(const struct sched_gate_list *oper, 6906ca6a665SVinicius Costa Gomes const struct sched_entry *entry) 6916ca6a665SVinicius Costa Gomes { 6926ca6a665SVinicius Costa Gomes if (list_is_last(&entry->list, &oper->entries)) 6936ca6a665SVinicius Costa Gomes return true; 6946ca6a665SVinicius Costa Gomes 6956ca6a665SVinicius Costa Gomes if (ktime_compare(entry->close_time, oper->cycle_close_time) == 0) 6966ca6a665SVinicius Costa Gomes return true; 6976ca6a665SVinicius Costa Gomes 6986ca6a665SVinicius Costa Gomes return false; 6996ca6a665SVinicius Costa Gomes } 7006ca6a665SVinicius Costa Gomes 701a3d43c0dSVinicius Costa Gomes static bool should_change_schedules(const struct sched_gate_list *admin, 702a3d43c0dSVinicius Costa Gomes const struct sched_gate_list *oper, 703a3d43c0dSVinicius Costa Gomes ktime_t close_time) 704a3d43c0dSVinicius Costa Gomes { 705c25031e9SVinicius Costa Gomes ktime_t next_base_time, extension_time; 706a3d43c0dSVinicius Costa Gomes 707a3d43c0dSVinicius Costa Gomes if (!admin) 708a3d43c0dSVinicius Costa Gomes return false; 709a3d43c0dSVinicius Costa Gomes 710a3d43c0dSVinicius Costa Gomes next_base_time = sched_base_time(admin); 711a3d43c0dSVinicius Costa Gomes 712a3d43c0dSVinicius Costa Gomes /* This is the simple case, the close_time would fall after 713a3d43c0dSVinicius Costa Gomes * the next schedule base_time. 714a3d43c0dSVinicius Costa Gomes */ 715a3d43c0dSVinicius Costa Gomes if (ktime_compare(next_base_time, close_time) <= 0) 716a3d43c0dSVinicius Costa Gomes return true; 717a3d43c0dSVinicius Costa Gomes 718c25031e9SVinicius Costa Gomes /* This is the cycle_time_extension case, if the close_time 719c25031e9SVinicius Costa Gomes * plus the amount that can be extended would fall after the 720c25031e9SVinicius Costa Gomes * next schedule base_time, we can extend the current schedule 721c25031e9SVinicius Costa Gomes * for that amount. 722c25031e9SVinicius Costa Gomes */ 723c25031e9SVinicius Costa Gomes extension_time = ktime_add_ns(close_time, oper->cycle_time_extension); 724c25031e9SVinicius Costa Gomes 725c25031e9SVinicius Costa Gomes /* FIXME: the IEEE 802.1Q-2018 Specification isn't clear about 726c25031e9SVinicius Costa Gomes * how precisely the extension should be made. So after 727c25031e9SVinicius Costa Gomes * conformance testing, this logic may change. 728c25031e9SVinicius Costa Gomes */ 729c25031e9SVinicius Costa Gomes if (ktime_compare(next_base_time, extension_time) <= 0) 730c25031e9SVinicius Costa Gomes return true; 731c25031e9SVinicius Costa Gomes 732a3d43c0dSVinicius Costa Gomes return false; 733a3d43c0dSVinicius Costa Gomes } 734a3d43c0dSVinicius Costa Gomes 7355a781ccbSVinicius Costa Gomes static enum hrtimer_restart advance_sched(struct hrtimer *timer) 7365a781ccbSVinicius Costa Gomes { 7375a781ccbSVinicius Costa Gomes struct taprio_sched *q = container_of(timer, struct taprio_sched, 7385a781ccbSVinicius Costa Gomes advance_timer); 739a3d43c0dSVinicius Costa Gomes struct sched_gate_list *oper, *admin; 7405a781ccbSVinicius Costa Gomes struct sched_entry *entry, *next; 7415a781ccbSVinicius Costa Gomes struct Qdisc *sch = q->root; 7425a781ccbSVinicius Costa Gomes ktime_t close_time; 7435a781ccbSVinicius Costa Gomes 7445a781ccbSVinicius Costa Gomes spin_lock(&q->current_entry_lock); 7455a781ccbSVinicius Costa Gomes entry = rcu_dereference_protected(q->current_entry, 7465a781ccbSVinicius Costa Gomes lockdep_is_held(&q->current_entry_lock)); 747a3d43c0dSVinicius Costa Gomes oper = rcu_dereference_protected(q->oper_sched, 748a3d43c0dSVinicius Costa Gomes lockdep_is_held(&q->current_entry_lock)); 749a3d43c0dSVinicius Costa Gomes admin = rcu_dereference_protected(q->admin_sched, 750a3d43c0dSVinicius Costa Gomes lockdep_is_held(&q->current_entry_lock)); 7515a781ccbSVinicius Costa Gomes 752a3d43c0dSVinicius Costa Gomes if (!oper) 753a3d43c0dSVinicius Costa Gomes switch_schedules(q, &admin, &oper); 754a3d43c0dSVinicius Costa Gomes 755a3d43c0dSVinicius Costa Gomes /* This can happen in two cases: 1. this is the very first run 756a3d43c0dSVinicius Costa Gomes * of this function (i.e. we weren't running any schedule 757a3d43c0dSVinicius Costa Gomes * previously); 2. The previous schedule just ended. The first 758a3d43c0dSVinicius Costa Gomes * entry of all schedules are pre-calculated during the 759a3d43c0dSVinicius Costa Gomes * schedule initialization. 7605a781ccbSVinicius Costa Gomes */ 761a3d43c0dSVinicius Costa Gomes if (unlikely(!entry || entry->close_time == oper->base_time)) { 762a3d43c0dSVinicius Costa Gomes next = list_first_entry(&oper->entries, struct sched_entry, 7635a781ccbSVinicius Costa Gomes list); 7645a781ccbSVinicius Costa Gomes close_time = next->close_time; 7655a781ccbSVinicius Costa Gomes goto first_run; 7665a781ccbSVinicius Costa Gomes } 7675a781ccbSVinicius Costa Gomes 7686ca6a665SVinicius Costa Gomes if (should_restart_cycle(oper, entry)) { 769a3d43c0dSVinicius Costa Gomes next = list_first_entry(&oper->entries, struct sched_entry, 7705a781ccbSVinicius Costa Gomes list); 7716ca6a665SVinicius Costa Gomes oper->cycle_close_time = ktime_add_ns(oper->cycle_close_time, 7726ca6a665SVinicius Costa Gomes oper->cycle_time); 7736ca6a665SVinicius Costa Gomes } else { 7745a781ccbSVinicius Costa Gomes next = list_next_entry(entry, list); 7756ca6a665SVinicius Costa Gomes } 7765a781ccbSVinicius Costa Gomes 7775a781ccbSVinicius Costa Gomes close_time = ktime_add_ns(entry->close_time, next->interval); 7786ca6a665SVinicius Costa Gomes close_time = min_t(ktime_t, close_time, oper->cycle_close_time); 7795a781ccbSVinicius Costa Gomes 780a3d43c0dSVinicius Costa Gomes if (should_change_schedules(admin, oper, close_time)) { 781a3d43c0dSVinicius Costa Gomes /* Set things so the next time this runs, the new 782a3d43c0dSVinicius Costa Gomes * schedule runs. 783a3d43c0dSVinicius Costa Gomes */ 784a3d43c0dSVinicius Costa Gomes close_time = sched_base_time(admin); 785a3d43c0dSVinicius Costa Gomes switch_schedules(q, &admin, &oper); 786a3d43c0dSVinicius Costa Gomes } 787a3d43c0dSVinicius Costa Gomes 7885a781ccbSVinicius Costa Gomes next->close_time = close_time; 78923bddf69SJakub Kicinski taprio_set_budget(q, next); 7905a781ccbSVinicius Costa Gomes 7915a781ccbSVinicius Costa Gomes first_run: 7925a781ccbSVinicius Costa Gomes rcu_assign_pointer(q->current_entry, next); 7935a781ccbSVinicius Costa Gomes spin_unlock(&q->current_entry_lock); 7945a781ccbSVinicius Costa Gomes 7955a781ccbSVinicius Costa Gomes hrtimer_set_expires(&q->advance_timer, close_time); 7965a781ccbSVinicius Costa Gomes 7975a781ccbSVinicius Costa Gomes rcu_read_lock(); 7985a781ccbSVinicius Costa Gomes __netif_schedule(sch); 7995a781ccbSVinicius Costa Gomes rcu_read_unlock(); 8005a781ccbSVinicius Costa Gomes 8015a781ccbSVinicius Costa Gomes return HRTIMER_RESTART; 8025a781ccbSVinicius Costa Gomes } 8035a781ccbSVinicius Costa Gomes 8045a781ccbSVinicius Costa Gomes static const struct nla_policy entry_policy[TCA_TAPRIO_SCHED_ENTRY_MAX + 1] = { 8055a781ccbSVinicius Costa Gomes [TCA_TAPRIO_SCHED_ENTRY_INDEX] = { .type = NLA_U32 }, 8065a781ccbSVinicius Costa Gomes [TCA_TAPRIO_SCHED_ENTRY_CMD] = { .type = NLA_U8 }, 8075a781ccbSVinicius Costa Gomes [TCA_TAPRIO_SCHED_ENTRY_GATE_MASK] = { .type = NLA_U32 }, 8085a781ccbSVinicius Costa Gomes [TCA_TAPRIO_SCHED_ENTRY_INTERVAL] = { .type = NLA_U32 }, 8095a781ccbSVinicius Costa Gomes }; 8105a781ccbSVinicius Costa Gomes 8115a781ccbSVinicius Costa Gomes static const struct nla_policy taprio_policy[TCA_TAPRIO_ATTR_MAX + 1] = { 8125a781ccbSVinicius Costa Gomes [TCA_TAPRIO_ATTR_PRIOMAP] = { 8135a781ccbSVinicius Costa Gomes .len = sizeof(struct tc_mqprio_qopt) 8145a781ccbSVinicius Costa Gomes }, 8155a781ccbSVinicius Costa Gomes [TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST] = { .type = NLA_NESTED }, 8165a781ccbSVinicius Costa Gomes [TCA_TAPRIO_ATTR_SCHED_BASE_TIME] = { .type = NLA_S64 }, 8175a781ccbSVinicius Costa Gomes [TCA_TAPRIO_ATTR_SCHED_SINGLE_ENTRY] = { .type = NLA_NESTED }, 8185a781ccbSVinicius Costa Gomes [TCA_TAPRIO_ATTR_SCHED_CLOCKID] = { .type = NLA_S32 }, 8196ca6a665SVinicius Costa Gomes [TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME] = { .type = NLA_S64 }, 820c25031e9SVinicius Costa Gomes [TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION] = { .type = NLA_S64 }, 82149c684d7SVinicius Costa Gomes [TCA_TAPRIO_ATTR_FLAGS] = { .type = NLA_U32 }, 822e13aaa06SJakub Kicinski [TCA_TAPRIO_ATTR_TXTIME_DELAY] = { .type = NLA_U32 }, 8235a781ccbSVinicius Costa Gomes }; 8245a781ccbSVinicius Costa Gomes 825b5b73b26SVinicius Costa Gomes static int fill_sched_entry(struct taprio_sched *q, struct nlattr **tb, 826b5b73b26SVinicius Costa Gomes struct sched_entry *entry, 8275a781ccbSVinicius Costa Gomes struct netlink_ext_ack *extack) 8285a781ccbSVinicius Costa Gomes { 829b5b73b26SVinicius Costa Gomes int min_duration = length_to_duration(q, ETH_ZLEN); 8305a781ccbSVinicius Costa Gomes u32 interval = 0; 8315a781ccbSVinicius Costa Gomes 8325a781ccbSVinicius Costa Gomes if (tb[TCA_TAPRIO_SCHED_ENTRY_CMD]) 8335a781ccbSVinicius Costa Gomes entry->command = nla_get_u8( 8345a781ccbSVinicius Costa Gomes tb[TCA_TAPRIO_SCHED_ENTRY_CMD]); 8355a781ccbSVinicius Costa Gomes 8365a781ccbSVinicius Costa Gomes if (tb[TCA_TAPRIO_SCHED_ENTRY_GATE_MASK]) 8375a781ccbSVinicius Costa Gomes entry->gate_mask = nla_get_u32( 8385a781ccbSVinicius Costa Gomes tb[TCA_TAPRIO_SCHED_ENTRY_GATE_MASK]); 8395a781ccbSVinicius Costa Gomes 8405a781ccbSVinicius Costa Gomes if (tb[TCA_TAPRIO_SCHED_ENTRY_INTERVAL]) 8415a781ccbSVinicius Costa Gomes interval = nla_get_u32( 8425a781ccbSVinicius Costa Gomes tb[TCA_TAPRIO_SCHED_ENTRY_INTERVAL]); 8435a781ccbSVinicius Costa Gomes 844b5b73b26SVinicius Costa Gomes /* The interval should allow at least the minimum ethernet 845b5b73b26SVinicius Costa Gomes * frame to go out. 846b5b73b26SVinicius Costa Gomes */ 847b5b73b26SVinicius Costa Gomes if (interval < min_duration) { 8485a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Invalid interval for schedule entry"); 8495a781ccbSVinicius Costa Gomes return -EINVAL; 8505a781ccbSVinicius Costa Gomes } 8515a781ccbSVinicius Costa Gomes 8525a781ccbSVinicius Costa Gomes entry->interval = interval; 8535a781ccbSVinicius Costa Gomes 8545a781ccbSVinicius Costa Gomes return 0; 8555a781ccbSVinicius Costa Gomes } 8565a781ccbSVinicius Costa Gomes 857b5b73b26SVinicius Costa Gomes static int parse_sched_entry(struct taprio_sched *q, struct nlattr *n, 858b5b73b26SVinicius Costa Gomes struct sched_entry *entry, int index, 859b5b73b26SVinicius Costa Gomes struct netlink_ext_ack *extack) 8605a781ccbSVinicius Costa Gomes { 8615a781ccbSVinicius Costa Gomes struct nlattr *tb[TCA_TAPRIO_SCHED_ENTRY_MAX + 1] = { }; 8625a781ccbSVinicius Costa Gomes int err; 8635a781ccbSVinicius Costa Gomes 8648cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_TAPRIO_SCHED_ENTRY_MAX, n, 8655a781ccbSVinicius Costa Gomes entry_policy, NULL); 8665a781ccbSVinicius Costa Gomes if (err < 0) { 8675a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Could not parse nested entry"); 8685a781ccbSVinicius Costa Gomes return -EINVAL; 8695a781ccbSVinicius Costa Gomes } 8705a781ccbSVinicius Costa Gomes 8715a781ccbSVinicius Costa Gomes entry->index = index; 8725a781ccbSVinicius Costa Gomes 873b5b73b26SVinicius Costa Gomes return fill_sched_entry(q, tb, entry, extack); 8745a781ccbSVinicius Costa Gomes } 8755a781ccbSVinicius Costa Gomes 876b5b73b26SVinicius Costa Gomes static int parse_sched_list(struct taprio_sched *q, struct nlattr *list, 877a3d43c0dSVinicius Costa Gomes struct sched_gate_list *sched, 8785a781ccbSVinicius Costa Gomes struct netlink_ext_ack *extack) 8795a781ccbSVinicius Costa Gomes { 8805a781ccbSVinicius Costa Gomes struct nlattr *n; 8815a781ccbSVinicius Costa Gomes int err, rem; 8825a781ccbSVinicius Costa Gomes int i = 0; 8835a781ccbSVinicius Costa Gomes 8845a781ccbSVinicius Costa Gomes if (!list) 8855a781ccbSVinicius Costa Gomes return -EINVAL; 8865a781ccbSVinicius Costa Gomes 8875a781ccbSVinicius Costa Gomes nla_for_each_nested(n, list, rem) { 8885a781ccbSVinicius Costa Gomes struct sched_entry *entry; 8895a781ccbSVinicius Costa Gomes 8905a781ccbSVinicius Costa Gomes if (nla_type(n) != TCA_TAPRIO_SCHED_ENTRY) { 8915a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Attribute is not of type 'entry'"); 8925a781ccbSVinicius Costa Gomes continue; 8935a781ccbSVinicius Costa Gomes } 8945a781ccbSVinicius Costa Gomes 8955a781ccbSVinicius Costa Gomes entry = kzalloc(sizeof(*entry), GFP_KERNEL); 8965a781ccbSVinicius Costa Gomes if (!entry) { 8975a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Not enough memory for entry"); 8985a781ccbSVinicius Costa Gomes return -ENOMEM; 8995a781ccbSVinicius Costa Gomes } 9005a781ccbSVinicius Costa Gomes 901b5b73b26SVinicius Costa Gomes err = parse_sched_entry(q, n, entry, i, extack); 9025a781ccbSVinicius Costa Gomes if (err < 0) { 9035a781ccbSVinicius Costa Gomes kfree(entry); 9045a781ccbSVinicius Costa Gomes return err; 9055a781ccbSVinicius Costa Gomes } 9065a781ccbSVinicius Costa Gomes 907a3d43c0dSVinicius Costa Gomes list_add_tail(&entry->list, &sched->entries); 9085a781ccbSVinicius Costa Gomes i++; 9095a781ccbSVinicius Costa Gomes } 9105a781ccbSVinicius Costa Gomes 911a3d43c0dSVinicius Costa Gomes sched->num_entries = i; 9125a781ccbSVinicius Costa Gomes 9135a781ccbSVinicius Costa Gomes return i; 9145a781ccbSVinicius Costa Gomes } 9155a781ccbSVinicius Costa Gomes 916b5b73b26SVinicius Costa Gomes static int parse_taprio_schedule(struct taprio_sched *q, struct nlattr **tb, 917a3d43c0dSVinicius Costa Gomes struct sched_gate_list *new, 9185a781ccbSVinicius Costa Gomes struct netlink_ext_ack *extack) 9195a781ccbSVinicius Costa Gomes { 9205a781ccbSVinicius Costa Gomes int err = 0; 9215a781ccbSVinicius Costa Gomes 922a3d43c0dSVinicius Costa Gomes if (tb[TCA_TAPRIO_ATTR_SCHED_SINGLE_ENTRY]) { 923a3d43c0dSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Adding a single entry is not supported"); 924a3d43c0dSVinicius Costa Gomes return -ENOTSUPP; 925a3d43c0dSVinicius Costa Gomes } 9265a781ccbSVinicius Costa Gomes 9275a781ccbSVinicius Costa Gomes if (tb[TCA_TAPRIO_ATTR_SCHED_BASE_TIME]) 928a3d43c0dSVinicius Costa Gomes new->base_time = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_BASE_TIME]); 9295a781ccbSVinicius Costa Gomes 930c25031e9SVinicius Costa Gomes if (tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION]) 931c25031e9SVinicius Costa Gomes new->cycle_time_extension = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION]); 932c25031e9SVinicius Costa Gomes 9336ca6a665SVinicius Costa Gomes if (tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME]) 9346ca6a665SVinicius Costa Gomes new->cycle_time = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME]); 9356ca6a665SVinicius Costa Gomes 9365a781ccbSVinicius Costa Gomes if (tb[TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST]) 937b5b73b26SVinicius Costa Gomes err = parse_sched_list(q, tb[TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST], 938b5b73b26SVinicius Costa Gomes new, extack); 939a3d43c0dSVinicius Costa Gomes if (err < 0) 9405a781ccbSVinicius Costa Gomes return err; 941a3d43c0dSVinicius Costa Gomes 942037be037SVedang Patel if (!new->cycle_time) { 943037be037SVedang Patel struct sched_entry *entry; 944037be037SVedang Patel ktime_t cycle = 0; 945037be037SVedang Patel 946037be037SVedang Patel list_for_each_entry(entry, &new->entries, list) 947037be037SVedang Patel cycle = ktime_add_ns(cycle, entry->interval); 948037be037SVedang Patel new->cycle_time = cycle; 949037be037SVedang Patel } 950037be037SVedang Patel 951a3d43c0dSVinicius Costa Gomes return 0; 9525a781ccbSVinicius Costa Gomes } 9535a781ccbSVinicius Costa Gomes 9545a781ccbSVinicius Costa Gomes static int taprio_parse_mqprio_opt(struct net_device *dev, 9555a781ccbSVinicius Costa Gomes struct tc_mqprio_qopt *qopt, 9564cfd5779SVedang Patel struct netlink_ext_ack *extack, 9574cfd5779SVedang Patel u32 taprio_flags) 9585a781ccbSVinicius Costa Gomes { 9595a781ccbSVinicius Costa Gomes int i, j; 9605a781ccbSVinicius Costa Gomes 961a3d43c0dSVinicius Costa Gomes if (!qopt && !dev->num_tc) { 9625a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "'mqprio' configuration is necessary"); 9635a781ccbSVinicius Costa Gomes return -EINVAL; 9645a781ccbSVinicius Costa Gomes } 9655a781ccbSVinicius Costa Gomes 966a3d43c0dSVinicius Costa Gomes /* If num_tc is already set, it means that the user already 967a3d43c0dSVinicius Costa Gomes * configured the mqprio part 968a3d43c0dSVinicius Costa Gomes */ 969a3d43c0dSVinicius Costa Gomes if (dev->num_tc) 970a3d43c0dSVinicius Costa Gomes return 0; 971a3d43c0dSVinicius Costa Gomes 9725a781ccbSVinicius Costa Gomes /* Verify num_tc is not out of max range */ 9735a781ccbSVinicius Costa Gomes if (qopt->num_tc > TC_MAX_QUEUE) { 9745a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Number of traffic classes is outside valid range"); 9755a781ccbSVinicius Costa Gomes return -EINVAL; 9765a781ccbSVinicius Costa Gomes } 9775a781ccbSVinicius Costa Gomes 9785a781ccbSVinicius Costa Gomes /* taprio imposes that traffic classes map 1:n to tx queues */ 9795a781ccbSVinicius Costa Gomes if (qopt->num_tc > dev->num_tx_queues) { 9805a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Number of traffic classes is greater than number of HW queues"); 9815a781ccbSVinicius Costa Gomes return -EINVAL; 9825a781ccbSVinicius Costa Gomes } 9835a781ccbSVinicius Costa Gomes 9845a781ccbSVinicius Costa Gomes /* Verify priority mapping uses valid tcs */ 985b5a0faa3SIvan Khoronzhuk for (i = 0; i <= TC_BITMASK; i++) { 9865a781ccbSVinicius Costa Gomes if (qopt->prio_tc_map[i] >= qopt->num_tc) { 9875a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Invalid traffic class in priority to traffic class mapping"); 9885a781ccbSVinicius Costa Gomes return -EINVAL; 9895a781ccbSVinicius Costa Gomes } 9905a781ccbSVinicius Costa Gomes } 9915a781ccbSVinicius Costa Gomes 9925a781ccbSVinicius Costa Gomes for (i = 0; i < qopt->num_tc; i++) { 9935a781ccbSVinicius Costa Gomes unsigned int last = qopt->offset[i] + qopt->count[i]; 9945a781ccbSVinicius Costa Gomes 9955a781ccbSVinicius Costa Gomes /* Verify the queue count is in tx range being equal to the 9965a781ccbSVinicius Costa Gomes * real_num_tx_queues indicates the last queue is in use. 9975a781ccbSVinicius Costa Gomes */ 9985a781ccbSVinicius Costa Gomes if (qopt->offset[i] >= dev->num_tx_queues || 9995a781ccbSVinicius Costa Gomes !qopt->count[i] || 10005a781ccbSVinicius Costa Gomes last > dev->real_num_tx_queues) { 10015a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Invalid queue in traffic class to queue mapping"); 10025a781ccbSVinicius Costa Gomes return -EINVAL; 10035a781ccbSVinicius Costa Gomes } 10045a781ccbSVinicius Costa Gomes 10054cfd5779SVedang Patel if (TXTIME_ASSIST_IS_ENABLED(taprio_flags)) 10064cfd5779SVedang Patel continue; 10074cfd5779SVedang Patel 10085a781ccbSVinicius Costa Gomes /* Verify that the offset and counts do not overlap */ 10095a781ccbSVinicius Costa Gomes for (j = i + 1; j < qopt->num_tc; j++) { 10105a781ccbSVinicius Costa Gomes if (last > qopt->offset[j]) { 10115a781ccbSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Detected overlap in the traffic class to queue mapping"); 10125a781ccbSVinicius Costa Gomes return -EINVAL; 10135a781ccbSVinicius Costa Gomes } 10145a781ccbSVinicius Costa Gomes } 10155a781ccbSVinicius Costa Gomes } 10165a781ccbSVinicius Costa Gomes 10175a781ccbSVinicius Costa Gomes return 0; 10185a781ccbSVinicius Costa Gomes } 10195a781ccbSVinicius Costa Gomes 1020a3d43c0dSVinicius Costa Gomes static int taprio_get_start_time(struct Qdisc *sch, 1021a3d43c0dSVinicius Costa Gomes struct sched_gate_list *sched, 1022a3d43c0dSVinicius Costa Gomes ktime_t *start) 10235a781ccbSVinicius Costa Gomes { 10245a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 10255a781ccbSVinicius Costa Gomes ktime_t now, base, cycle; 10265a781ccbSVinicius Costa Gomes s64 n; 10275a781ccbSVinicius Costa Gomes 1028a3d43c0dSVinicius Costa Gomes base = sched_base_time(sched); 10297ede7b03SVedang Patel now = taprio_get_time(q); 10308599099fSAndre Guedes 10318599099fSAndre Guedes if (ktime_after(base, now)) { 10328599099fSAndre Guedes *start = base; 10338599099fSAndre Guedes return 0; 10348599099fSAndre Guedes } 10355a781ccbSVinicius Costa Gomes 1036037be037SVedang Patel cycle = sched->cycle_time; 10375a781ccbSVinicius Costa Gomes 10388599099fSAndre Guedes /* The qdisc is expected to have at least one sched_entry. Moreover, 10398599099fSAndre Guedes * any entry must have 'interval' > 0. Thus if the cycle time is zero, 10408599099fSAndre Guedes * something went really wrong. In that case, we should warn about this 10418599099fSAndre Guedes * inconsistent state and return error. 10428599099fSAndre Guedes */ 10438599099fSAndre Guedes if (WARN_ON(!cycle)) 10448599099fSAndre Guedes return -EFAULT; 10455a781ccbSVinicius Costa Gomes 10465a781ccbSVinicius Costa Gomes /* Schedule the start time for the beginning of the next 10475a781ccbSVinicius Costa Gomes * cycle. 10485a781ccbSVinicius Costa Gomes */ 10495a781ccbSVinicius Costa Gomes n = div64_s64(ktime_sub_ns(now, base), cycle); 10508599099fSAndre Guedes *start = ktime_add_ns(base, (n + 1) * cycle); 10518599099fSAndre Guedes return 0; 10525a781ccbSVinicius Costa Gomes } 10535a781ccbSVinicius Costa Gomes 1054a3d43c0dSVinicius Costa Gomes static void setup_first_close_time(struct taprio_sched *q, 1055a3d43c0dSVinicius Costa Gomes struct sched_gate_list *sched, ktime_t base) 10565a781ccbSVinicius Costa Gomes { 10575a781ccbSVinicius Costa Gomes struct sched_entry *first; 10586ca6a665SVinicius Costa Gomes ktime_t cycle; 10595a781ccbSVinicius Costa Gomes 1060a3d43c0dSVinicius Costa Gomes first = list_first_entry(&sched->entries, 1061a3d43c0dSVinicius Costa Gomes struct sched_entry, list); 10625a781ccbSVinicius Costa Gomes 1063037be037SVedang Patel cycle = sched->cycle_time; 10646ca6a665SVinicius Costa Gomes 10656ca6a665SVinicius Costa Gomes /* FIXME: find a better place to do this */ 10666ca6a665SVinicius Costa Gomes sched->cycle_close_time = ktime_add_ns(base, cycle); 10676ca6a665SVinicius Costa Gomes 1068a3d43c0dSVinicius Costa Gomes first->close_time = ktime_add_ns(base, first->interval); 106923bddf69SJakub Kicinski taprio_set_budget(q, first); 10705a781ccbSVinicius Costa Gomes rcu_assign_pointer(q->current_entry, NULL); 1071a3d43c0dSVinicius Costa Gomes } 10725a781ccbSVinicius Costa Gomes 1073a3d43c0dSVinicius Costa Gomes static void taprio_start_sched(struct Qdisc *sch, 1074a3d43c0dSVinicius Costa Gomes ktime_t start, struct sched_gate_list *new) 1075a3d43c0dSVinicius Costa Gomes { 1076a3d43c0dSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 1077a3d43c0dSVinicius Costa Gomes ktime_t expires; 1078a3d43c0dSVinicius Costa Gomes 10799c66d156SVinicius Costa Gomes if (FULL_OFFLOAD_IS_ENABLED(q->flags)) 10809c66d156SVinicius Costa Gomes return; 10819c66d156SVinicius Costa Gomes 1082a3d43c0dSVinicius Costa Gomes expires = hrtimer_get_expires(&q->advance_timer); 1083a3d43c0dSVinicius Costa Gomes if (expires == 0) 1084a3d43c0dSVinicius Costa Gomes expires = KTIME_MAX; 1085a3d43c0dSVinicius Costa Gomes 1086a3d43c0dSVinicius Costa Gomes /* If the new schedule starts before the next expiration, we 1087a3d43c0dSVinicius Costa Gomes * reprogram it to the earliest one, so we change the admin 1088a3d43c0dSVinicius Costa Gomes * schedule to the operational one at the right time. 1089a3d43c0dSVinicius Costa Gomes */ 1090a3d43c0dSVinicius Costa Gomes start = min_t(ktime_t, start, expires); 10915a781ccbSVinicius Costa Gomes 10925a781ccbSVinicius Costa Gomes hrtimer_start(&q->advance_timer, start, HRTIMER_MODE_ABS); 10935a781ccbSVinicius Costa Gomes } 10945a781ccbSVinicius Costa Gomes 10957b9eba7bSLeandro Dorileo static void taprio_set_picos_per_byte(struct net_device *dev, 10967b9eba7bSLeandro Dorileo struct taprio_sched *q) 10977b9eba7bSLeandro Dorileo { 10987b9eba7bSLeandro Dorileo struct ethtool_link_ksettings ecmd; 1099f04b514cSVladimir Oltean int speed = SPEED_10; 1100f04b514cSVladimir Oltean int picos_per_byte; 1101f04b514cSVladimir Oltean int err; 11027b9eba7bSLeandro Dorileo 1103f04b514cSVladimir Oltean err = __ethtool_get_link_ksettings(dev, &ecmd); 1104f04b514cSVladimir Oltean if (err < 0) 1105f04b514cSVladimir Oltean goto skip; 1106f04b514cSVladimir Oltean 11079a9251a3SVladimir Oltean if (ecmd.base.speed && ecmd.base.speed != SPEED_UNKNOWN) 1108f04b514cSVladimir Oltean speed = ecmd.base.speed; 1109f04b514cSVladimir Oltean 1110f04b514cSVladimir Oltean skip: 111168ce6688SVladimir Oltean picos_per_byte = (USEC_PER_SEC * 8) / speed; 11127b9eba7bSLeandro Dorileo 11137b9eba7bSLeandro Dorileo atomic64_set(&q->picos_per_byte, picos_per_byte); 11147b9eba7bSLeandro Dorileo netdev_dbg(dev, "taprio: set %s's picos_per_byte to: %lld, linkspeed: %d\n", 11157b9eba7bSLeandro Dorileo dev->name, (long long)atomic64_read(&q->picos_per_byte), 11167b9eba7bSLeandro Dorileo ecmd.base.speed); 11177b9eba7bSLeandro Dorileo } 11187b9eba7bSLeandro Dorileo 11197b9eba7bSLeandro Dorileo static int taprio_dev_notifier(struct notifier_block *nb, unsigned long event, 11207b9eba7bSLeandro Dorileo void *ptr) 11217b9eba7bSLeandro Dorileo { 11227b9eba7bSLeandro Dorileo struct net_device *dev = netdev_notifier_info_to_dev(ptr); 11237b9eba7bSLeandro Dorileo struct net_device *qdev; 11247b9eba7bSLeandro Dorileo struct taprio_sched *q; 11257b9eba7bSLeandro Dorileo bool found = false; 11267b9eba7bSLeandro Dorileo 11277b9eba7bSLeandro Dorileo ASSERT_RTNL(); 11287b9eba7bSLeandro Dorileo 11297b9eba7bSLeandro Dorileo if (event != NETDEV_UP && event != NETDEV_CHANGE) 11307b9eba7bSLeandro Dorileo return NOTIFY_DONE; 11317b9eba7bSLeandro Dorileo 11327b9eba7bSLeandro Dorileo spin_lock(&taprio_list_lock); 11337b9eba7bSLeandro Dorileo list_for_each_entry(q, &taprio_list, taprio_list) { 11347b9eba7bSLeandro Dorileo qdev = qdisc_dev(q->root); 11357b9eba7bSLeandro Dorileo if (qdev == dev) { 11367b9eba7bSLeandro Dorileo found = true; 11377b9eba7bSLeandro Dorileo break; 11387b9eba7bSLeandro Dorileo } 11397b9eba7bSLeandro Dorileo } 11407b9eba7bSLeandro Dorileo spin_unlock(&taprio_list_lock); 11417b9eba7bSLeandro Dorileo 11427b9eba7bSLeandro Dorileo if (found) 11437b9eba7bSLeandro Dorileo taprio_set_picos_per_byte(dev, q); 11447b9eba7bSLeandro Dorileo 11457b9eba7bSLeandro Dorileo return NOTIFY_DONE; 11467b9eba7bSLeandro Dorileo } 11477b9eba7bSLeandro Dorileo 11484cfd5779SVedang Patel static void setup_txtime(struct taprio_sched *q, 11494cfd5779SVedang Patel struct sched_gate_list *sched, ktime_t base) 11504cfd5779SVedang Patel { 11514cfd5779SVedang Patel struct sched_entry *entry; 11524cfd5779SVedang Patel u32 interval = 0; 11534cfd5779SVedang Patel 11544cfd5779SVedang Patel list_for_each_entry(entry, &sched->entries, list) { 11554cfd5779SVedang Patel entry->next_txtime = ktime_add_ns(base, interval); 11564cfd5779SVedang Patel interval += entry->interval; 11574cfd5779SVedang Patel } 11584cfd5779SVedang Patel } 11594cfd5779SVedang Patel 11609c66d156SVinicius Costa Gomes static struct tc_taprio_qopt_offload *taprio_offload_alloc(int num_entries) 11619c66d156SVinicius Costa Gomes { 11629c66d156SVinicius Costa Gomes struct __tc_taprio_qopt_offload *__offload; 11639c66d156SVinicius Costa Gomes 116411a33de2SGustavo A. R. Silva __offload = kzalloc(struct_size(__offload, offload.entries, num_entries), 116511a33de2SGustavo A. R. Silva GFP_KERNEL); 11669c66d156SVinicius Costa Gomes if (!__offload) 11679c66d156SVinicius Costa Gomes return NULL; 11689c66d156SVinicius Costa Gomes 11699c66d156SVinicius Costa Gomes refcount_set(&__offload->users, 1); 11709c66d156SVinicius Costa Gomes 11719c66d156SVinicius Costa Gomes return &__offload->offload; 11729c66d156SVinicius Costa Gomes } 11739c66d156SVinicius Costa Gomes 11749c66d156SVinicius Costa Gomes struct tc_taprio_qopt_offload *taprio_offload_get(struct tc_taprio_qopt_offload 11759c66d156SVinicius Costa Gomes *offload) 11769c66d156SVinicius Costa Gomes { 11779c66d156SVinicius Costa Gomes struct __tc_taprio_qopt_offload *__offload; 11789c66d156SVinicius Costa Gomes 11799c66d156SVinicius Costa Gomes __offload = container_of(offload, struct __tc_taprio_qopt_offload, 11809c66d156SVinicius Costa Gomes offload); 11819c66d156SVinicius Costa Gomes 11829c66d156SVinicius Costa Gomes refcount_inc(&__offload->users); 11839c66d156SVinicius Costa Gomes 11849c66d156SVinicius Costa Gomes return offload; 11859c66d156SVinicius Costa Gomes } 11869c66d156SVinicius Costa Gomes EXPORT_SYMBOL_GPL(taprio_offload_get); 11879c66d156SVinicius Costa Gomes 11889c66d156SVinicius Costa Gomes void taprio_offload_free(struct tc_taprio_qopt_offload *offload) 11899c66d156SVinicius Costa Gomes { 11909c66d156SVinicius Costa Gomes struct __tc_taprio_qopt_offload *__offload; 11919c66d156SVinicius Costa Gomes 11929c66d156SVinicius Costa Gomes __offload = container_of(offload, struct __tc_taprio_qopt_offload, 11939c66d156SVinicius Costa Gomes offload); 11949c66d156SVinicius Costa Gomes 11959c66d156SVinicius Costa Gomes if (!refcount_dec_and_test(&__offload->users)) 11969c66d156SVinicius Costa Gomes return; 11979c66d156SVinicius Costa Gomes 11989c66d156SVinicius Costa Gomes kfree(__offload); 11999c66d156SVinicius Costa Gomes } 12009c66d156SVinicius Costa Gomes EXPORT_SYMBOL_GPL(taprio_offload_free); 12019c66d156SVinicius Costa Gomes 12029c66d156SVinicius Costa Gomes /* The function will only serve to keep the pointers to the "oper" and "admin" 12039c66d156SVinicius Costa Gomes * schedules valid in relation to their base times, so when calling dump() the 12049c66d156SVinicius Costa Gomes * users looks at the right schedules. 12059c66d156SVinicius Costa Gomes * When using full offload, the admin configuration is promoted to oper at the 12069c66d156SVinicius Costa Gomes * base_time in the PHC time domain. But because the system time is not 12079c66d156SVinicius Costa Gomes * necessarily in sync with that, we can't just trigger a hrtimer to call 12089c66d156SVinicius Costa Gomes * switch_schedules at the right hardware time. 12099c66d156SVinicius Costa Gomes * At the moment we call this by hand right away from taprio, but in the future 12109c66d156SVinicius Costa Gomes * it will be useful to create a mechanism for drivers to notify taprio of the 12119c66d156SVinicius Costa Gomes * offload state (PENDING, ACTIVE, INACTIVE) so it can be visible in dump(). 12129c66d156SVinicius Costa Gomes * This is left as TODO. 12139c66d156SVinicius Costa Gomes */ 1214d665c128SYi Wang static void taprio_offload_config_changed(struct taprio_sched *q) 12159c66d156SVinicius Costa Gomes { 12169c66d156SVinicius Costa Gomes struct sched_gate_list *oper, *admin; 12179c66d156SVinicius Costa Gomes 12189c66d156SVinicius Costa Gomes spin_lock(&q->current_entry_lock); 12199c66d156SVinicius Costa Gomes 12209c66d156SVinicius Costa Gomes oper = rcu_dereference_protected(q->oper_sched, 12219c66d156SVinicius Costa Gomes lockdep_is_held(&q->current_entry_lock)); 12229c66d156SVinicius Costa Gomes admin = rcu_dereference_protected(q->admin_sched, 12239c66d156SVinicius Costa Gomes lockdep_is_held(&q->current_entry_lock)); 12249c66d156SVinicius Costa Gomes 12259c66d156SVinicius Costa Gomes switch_schedules(q, &admin, &oper); 12269c66d156SVinicius Costa Gomes 12279c66d156SVinicius Costa Gomes spin_unlock(&q->current_entry_lock); 12289c66d156SVinicius Costa Gomes } 12299c66d156SVinicius Costa Gomes 123009e31cf0SVinicius Costa Gomes static u32 tc_map_to_queue_mask(struct net_device *dev, u32 tc_mask) 123109e31cf0SVinicius Costa Gomes { 123209e31cf0SVinicius Costa Gomes u32 i, queue_mask = 0; 123309e31cf0SVinicius Costa Gomes 123409e31cf0SVinicius Costa Gomes for (i = 0; i < dev->num_tc; i++) { 123509e31cf0SVinicius Costa Gomes u32 offset, count; 123609e31cf0SVinicius Costa Gomes 123709e31cf0SVinicius Costa Gomes if (!(tc_mask & BIT(i))) 123809e31cf0SVinicius Costa Gomes continue; 123909e31cf0SVinicius Costa Gomes 124009e31cf0SVinicius Costa Gomes offset = dev->tc_to_txq[i].offset; 124109e31cf0SVinicius Costa Gomes count = dev->tc_to_txq[i].count; 124209e31cf0SVinicius Costa Gomes 124309e31cf0SVinicius Costa Gomes queue_mask |= GENMASK(offset + count - 1, offset); 124409e31cf0SVinicius Costa Gomes } 124509e31cf0SVinicius Costa Gomes 124609e31cf0SVinicius Costa Gomes return queue_mask; 124709e31cf0SVinicius Costa Gomes } 124809e31cf0SVinicius Costa Gomes 124909e31cf0SVinicius Costa Gomes static void taprio_sched_to_offload(struct net_device *dev, 12509c66d156SVinicius Costa Gomes struct sched_gate_list *sched, 12519c66d156SVinicius Costa Gomes struct tc_taprio_qopt_offload *offload) 12529c66d156SVinicius Costa Gomes { 12539c66d156SVinicius Costa Gomes struct sched_entry *entry; 12549c66d156SVinicius Costa Gomes int i = 0; 12559c66d156SVinicius Costa Gomes 12569c66d156SVinicius Costa Gomes offload->base_time = sched->base_time; 12579c66d156SVinicius Costa Gomes offload->cycle_time = sched->cycle_time; 12589c66d156SVinicius Costa Gomes offload->cycle_time_extension = sched->cycle_time_extension; 12599c66d156SVinicius Costa Gomes 12609c66d156SVinicius Costa Gomes list_for_each_entry(entry, &sched->entries, list) { 12619c66d156SVinicius Costa Gomes struct tc_taprio_sched_entry *e = &offload->entries[i]; 12629c66d156SVinicius Costa Gomes 12639c66d156SVinicius Costa Gomes e->command = entry->command; 12649c66d156SVinicius Costa Gomes e->interval = entry->interval; 126509e31cf0SVinicius Costa Gomes e->gate_mask = tc_map_to_queue_mask(dev, entry->gate_mask); 126609e31cf0SVinicius Costa Gomes 12679c66d156SVinicius Costa Gomes i++; 12689c66d156SVinicius Costa Gomes } 12699c66d156SVinicius Costa Gomes 12709c66d156SVinicius Costa Gomes offload->num_entries = i; 12719c66d156SVinicius Costa Gomes } 12729c66d156SVinicius Costa Gomes 12739c66d156SVinicius Costa Gomes static int taprio_enable_offload(struct net_device *dev, 12749c66d156SVinicius Costa Gomes struct taprio_sched *q, 12759c66d156SVinicius Costa Gomes struct sched_gate_list *sched, 12769c66d156SVinicius Costa Gomes struct netlink_ext_ack *extack) 12779c66d156SVinicius Costa Gomes { 12789c66d156SVinicius Costa Gomes const struct net_device_ops *ops = dev->netdev_ops; 12799c66d156SVinicius Costa Gomes struct tc_taprio_qopt_offload *offload; 12809c66d156SVinicius Costa Gomes int err = 0; 12819c66d156SVinicius Costa Gomes 12829c66d156SVinicius Costa Gomes if (!ops->ndo_setup_tc) { 12839c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, 12849c66d156SVinicius Costa Gomes "Device does not support taprio offload"); 12859c66d156SVinicius Costa Gomes return -EOPNOTSUPP; 12869c66d156SVinicius Costa Gomes } 12879c66d156SVinicius Costa Gomes 12889c66d156SVinicius Costa Gomes offload = taprio_offload_alloc(sched->num_entries); 12899c66d156SVinicius Costa Gomes if (!offload) { 12909c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, 12919c66d156SVinicius Costa Gomes "Not enough memory for enabling offload mode"); 12929c66d156SVinicius Costa Gomes return -ENOMEM; 12939c66d156SVinicius Costa Gomes } 12949c66d156SVinicius Costa Gomes offload->enable = 1; 129509e31cf0SVinicius Costa Gomes taprio_sched_to_offload(dev, sched, offload); 12969c66d156SVinicius Costa Gomes 12979c66d156SVinicius Costa Gomes err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload); 12989c66d156SVinicius Costa Gomes if (err < 0) { 12999c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, 13009c66d156SVinicius Costa Gomes "Device failed to setup taprio offload"); 13019c66d156SVinicius Costa Gomes goto done; 13029c66d156SVinicius Costa Gomes } 13039c66d156SVinicius Costa Gomes 13049c66d156SVinicius Costa Gomes done: 13059c66d156SVinicius Costa Gomes taprio_offload_free(offload); 13069c66d156SVinicius Costa Gomes 13079c66d156SVinicius Costa Gomes return err; 13089c66d156SVinicius Costa Gomes } 13099c66d156SVinicius Costa Gomes 13109c66d156SVinicius Costa Gomes static int taprio_disable_offload(struct net_device *dev, 13119c66d156SVinicius Costa Gomes struct taprio_sched *q, 13129c66d156SVinicius Costa Gomes struct netlink_ext_ack *extack) 13139c66d156SVinicius Costa Gomes { 13149c66d156SVinicius Costa Gomes const struct net_device_ops *ops = dev->netdev_ops; 13159c66d156SVinicius Costa Gomes struct tc_taprio_qopt_offload *offload; 13169c66d156SVinicius Costa Gomes int err; 13179c66d156SVinicius Costa Gomes 13189c66d156SVinicius Costa Gomes if (!FULL_OFFLOAD_IS_ENABLED(q->flags)) 13199c66d156SVinicius Costa Gomes return 0; 13209c66d156SVinicius Costa Gomes 13219c66d156SVinicius Costa Gomes if (!ops->ndo_setup_tc) 13229c66d156SVinicius Costa Gomes return -EOPNOTSUPP; 13239c66d156SVinicius Costa Gomes 13249c66d156SVinicius Costa Gomes offload = taprio_offload_alloc(0); 13259c66d156SVinicius Costa Gomes if (!offload) { 13269c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, 13279c66d156SVinicius Costa Gomes "Not enough memory to disable offload mode"); 13289c66d156SVinicius Costa Gomes return -ENOMEM; 13299c66d156SVinicius Costa Gomes } 13309c66d156SVinicius Costa Gomes offload->enable = 0; 13319c66d156SVinicius Costa Gomes 13329c66d156SVinicius Costa Gomes err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload); 13339c66d156SVinicius Costa Gomes if (err < 0) { 13349c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, 13359c66d156SVinicius Costa Gomes "Device failed to disable offload"); 13369c66d156SVinicius Costa Gomes goto out; 13379c66d156SVinicius Costa Gomes } 13389c66d156SVinicius Costa Gomes 13399c66d156SVinicius Costa Gomes out: 13409c66d156SVinicius Costa Gomes taprio_offload_free(offload); 13419c66d156SVinicius Costa Gomes 13429c66d156SVinicius Costa Gomes return err; 13439c66d156SVinicius Costa Gomes } 13449c66d156SVinicius Costa Gomes 13459c66d156SVinicius Costa Gomes /* If full offload is enabled, the only possible clockid is the net device's 13469c66d156SVinicius Costa Gomes * PHC. For that reason, specifying a clockid through netlink is incorrect. 13479c66d156SVinicius Costa Gomes * For txtime-assist, it is implicitly assumed that the device's PHC is kept 13489c66d156SVinicius Costa Gomes * in sync with the specified clockid via a user space daemon such as phc2sys. 13499c66d156SVinicius Costa Gomes * For both software taprio and txtime-assist, the clockid is used for the 13509c66d156SVinicius Costa Gomes * hrtimer that advances the schedule and hence mandatory. 13519c66d156SVinicius Costa Gomes */ 13529c66d156SVinicius Costa Gomes static int taprio_parse_clockid(struct Qdisc *sch, struct nlattr **tb, 13539c66d156SVinicius Costa Gomes struct netlink_ext_ack *extack) 13549c66d156SVinicius Costa Gomes { 13559c66d156SVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 13569c66d156SVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 13579c66d156SVinicius Costa Gomes int err = -EINVAL; 13589c66d156SVinicius Costa Gomes 13599c66d156SVinicius Costa Gomes if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { 13609c66d156SVinicius Costa Gomes const struct ethtool_ops *ops = dev->ethtool_ops; 13619c66d156SVinicius Costa Gomes struct ethtool_ts_info info = { 13629c66d156SVinicius Costa Gomes .cmd = ETHTOOL_GET_TS_INFO, 13639c66d156SVinicius Costa Gomes .phc_index = -1, 13649c66d156SVinicius Costa Gomes }; 13659c66d156SVinicius Costa Gomes 13669c66d156SVinicius Costa Gomes if (tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]) { 13679c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, 13689c66d156SVinicius Costa Gomes "The 'clockid' cannot be specified for full offload"); 13699c66d156SVinicius Costa Gomes goto out; 13709c66d156SVinicius Costa Gomes } 13719c66d156SVinicius Costa Gomes 13729c66d156SVinicius Costa Gomes if (ops && ops->get_ts_info) 13739c66d156SVinicius Costa Gomes err = ops->get_ts_info(dev, &info); 13749c66d156SVinicius Costa Gomes 13759c66d156SVinicius Costa Gomes if (err || info.phc_index < 0) { 13769c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, 13779c66d156SVinicius Costa Gomes "Device does not have a PTP clock"); 13789c66d156SVinicius Costa Gomes err = -ENOTSUPP; 13799c66d156SVinicius Costa Gomes goto out; 13809c66d156SVinicius Costa Gomes } 13819c66d156SVinicius Costa Gomes } else if (tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]) { 13829c66d156SVinicius Costa Gomes int clockid = nla_get_s32(tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]); 13839c66d156SVinicius Costa Gomes 13849c66d156SVinicius Costa Gomes /* We only support static clockids and we don't allow 13859c66d156SVinicius Costa Gomes * for it to be modified after the first init. 13869c66d156SVinicius Costa Gomes */ 13879c66d156SVinicius Costa Gomes if (clockid < 0 || 13889c66d156SVinicius Costa Gomes (q->clockid != -1 && q->clockid != clockid)) { 13899c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, 13909c66d156SVinicius Costa Gomes "Changing the 'clockid' of a running schedule is not supported"); 13919c66d156SVinicius Costa Gomes err = -ENOTSUPP; 13929c66d156SVinicius Costa Gomes goto out; 13939c66d156SVinicius Costa Gomes } 13949c66d156SVinicius Costa Gomes 13959c66d156SVinicius Costa Gomes switch (clockid) { 13969c66d156SVinicius Costa Gomes case CLOCK_REALTIME: 13979c66d156SVinicius Costa Gomes q->tk_offset = TK_OFFS_REAL; 13989c66d156SVinicius Costa Gomes break; 13999c66d156SVinicius Costa Gomes case CLOCK_MONOTONIC: 14009c66d156SVinicius Costa Gomes q->tk_offset = TK_OFFS_MAX; 14019c66d156SVinicius Costa Gomes break; 14029c66d156SVinicius Costa Gomes case CLOCK_BOOTTIME: 14039c66d156SVinicius Costa Gomes q->tk_offset = TK_OFFS_BOOT; 14049c66d156SVinicius Costa Gomes break; 14059c66d156SVinicius Costa Gomes case CLOCK_TAI: 14069c66d156SVinicius Costa Gomes q->tk_offset = TK_OFFS_TAI; 14079c66d156SVinicius Costa Gomes break; 14089c66d156SVinicius Costa Gomes default: 14099c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Invalid 'clockid'"); 14109c66d156SVinicius Costa Gomes err = -EINVAL; 14119c66d156SVinicius Costa Gomes goto out; 14129c66d156SVinicius Costa Gomes } 14139c66d156SVinicius Costa Gomes 14149c66d156SVinicius Costa Gomes q->clockid = clockid; 14159c66d156SVinicius Costa Gomes } else { 14169c66d156SVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Specifying a 'clockid' is mandatory"); 14179c66d156SVinicius Costa Gomes goto out; 14189c66d156SVinicius Costa Gomes } 1419a954380aSVinicius Costa Gomes 1420a954380aSVinicius Costa Gomes /* Everything went ok, return success. */ 1421a954380aSVinicius Costa Gomes err = 0; 1422a954380aSVinicius Costa Gomes 14239c66d156SVinicius Costa Gomes out: 14249c66d156SVinicius Costa Gomes return err; 14259c66d156SVinicius Costa Gomes } 14269c66d156SVinicius Costa Gomes 1427b5a0faa3SIvan Khoronzhuk static int taprio_mqprio_cmp(const struct net_device *dev, 1428b5a0faa3SIvan Khoronzhuk const struct tc_mqprio_qopt *mqprio) 1429b5a0faa3SIvan Khoronzhuk { 1430b5a0faa3SIvan Khoronzhuk int i; 1431b5a0faa3SIvan Khoronzhuk 1432b5a0faa3SIvan Khoronzhuk if (!mqprio || mqprio->num_tc != dev->num_tc) 1433b5a0faa3SIvan Khoronzhuk return -1; 1434b5a0faa3SIvan Khoronzhuk 1435b5a0faa3SIvan Khoronzhuk for (i = 0; i < mqprio->num_tc; i++) 1436b5a0faa3SIvan Khoronzhuk if (dev->tc_to_txq[i].count != mqprio->count[i] || 1437b5a0faa3SIvan Khoronzhuk dev->tc_to_txq[i].offset != mqprio->offset[i]) 1438b5a0faa3SIvan Khoronzhuk return -1; 1439b5a0faa3SIvan Khoronzhuk 1440b5a0faa3SIvan Khoronzhuk for (i = 0; i <= TC_BITMASK; i++) 1441b5a0faa3SIvan Khoronzhuk if (dev->prio_tc_map[i] != mqprio->prio_tc_map[i]) 1442b5a0faa3SIvan Khoronzhuk return -1; 1443b5a0faa3SIvan Khoronzhuk 1444b5a0faa3SIvan Khoronzhuk return 0; 1445b5a0faa3SIvan Khoronzhuk } 1446b5a0faa3SIvan Khoronzhuk 1447a9d62274SVinicius Costa Gomes /* The semantics of the 'flags' argument in relation to 'change()' 1448a9d62274SVinicius Costa Gomes * requests, are interpreted following two rules (which are applied in 1449a9d62274SVinicius Costa Gomes * this order): (1) an omitted 'flags' argument is interpreted as 1450a9d62274SVinicius Costa Gomes * zero; (2) the 'flags' of a "running" taprio instance cannot be 1451a9d62274SVinicius Costa Gomes * changed. 1452a9d62274SVinicius Costa Gomes */ 1453a9d62274SVinicius Costa Gomes static int taprio_new_flags(const struct nlattr *attr, u32 old, 1454a9d62274SVinicius Costa Gomes struct netlink_ext_ack *extack) 1455a9d62274SVinicius Costa Gomes { 1456a9d62274SVinicius Costa Gomes u32 new = 0; 1457a9d62274SVinicius Costa Gomes 1458a9d62274SVinicius Costa Gomes if (attr) 1459a9d62274SVinicius Costa Gomes new = nla_get_u32(attr); 1460a9d62274SVinicius Costa Gomes 1461a9d62274SVinicius Costa Gomes if (old != TAPRIO_FLAGS_INVALID && old != new) { 1462a9d62274SVinicius Costa Gomes NL_SET_ERR_MSG_MOD(extack, "Changing 'flags' of a running schedule is not supported"); 1463a9d62274SVinicius Costa Gomes return -EOPNOTSUPP; 1464a9d62274SVinicius Costa Gomes } 1465a9d62274SVinicius Costa Gomes 1466a9d62274SVinicius Costa Gomes if (!taprio_flags_valid(new)) { 1467a9d62274SVinicius Costa Gomes NL_SET_ERR_MSG_MOD(extack, "Specified 'flags' are not valid"); 1468a9d62274SVinicius Costa Gomes return -EINVAL; 1469a9d62274SVinicius Costa Gomes } 1470a9d62274SVinicius Costa Gomes 1471a9d62274SVinicius Costa Gomes return new; 1472a9d62274SVinicius Costa Gomes } 1473a9d62274SVinicius Costa Gomes 14745a781ccbSVinicius Costa Gomes static int taprio_change(struct Qdisc *sch, struct nlattr *opt, 14755a781ccbSVinicius Costa Gomes struct netlink_ext_ack *extack) 14765a781ccbSVinicius Costa Gomes { 14775a781ccbSVinicius Costa Gomes struct nlattr *tb[TCA_TAPRIO_ATTR_MAX + 1] = { }; 1478a3d43c0dSVinicius Costa Gomes struct sched_gate_list *oper, *admin, *new_admin; 14795a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 14805a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 14815a781ccbSVinicius Costa Gomes struct tc_mqprio_qopt *mqprio = NULL; 1482a3d43c0dSVinicius Costa Gomes unsigned long flags; 14835a781ccbSVinicius Costa Gomes ktime_t start; 14849c66d156SVinicius Costa Gomes int i, err; 14855a781ccbSVinicius Costa Gomes 14868cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_TAPRIO_ATTR_MAX, opt, 14875a781ccbSVinicius Costa Gomes taprio_policy, extack); 14885a781ccbSVinicius Costa Gomes if (err < 0) 14895a781ccbSVinicius Costa Gomes return err; 14905a781ccbSVinicius Costa Gomes 14915a781ccbSVinicius Costa Gomes if (tb[TCA_TAPRIO_ATTR_PRIOMAP]) 14925a781ccbSVinicius Costa Gomes mqprio = nla_data(tb[TCA_TAPRIO_ATTR_PRIOMAP]); 14935a781ccbSVinicius Costa Gomes 1494a9d62274SVinicius Costa Gomes err = taprio_new_flags(tb[TCA_TAPRIO_ATTR_FLAGS], 1495a9d62274SVinicius Costa Gomes q->flags, extack); 1496a9d62274SVinicius Costa Gomes if (err < 0) 1497a9d62274SVinicius Costa Gomes return err; 14984cfd5779SVedang Patel 1499a9d62274SVinicius Costa Gomes q->flags = err; 15004cfd5779SVedang Patel 1501a9d62274SVinicius Costa Gomes err = taprio_parse_mqprio_opt(dev, mqprio, extack, q->flags); 15025a781ccbSVinicius Costa Gomes if (err < 0) 15035a781ccbSVinicius Costa Gomes return err; 15045a781ccbSVinicius Costa Gomes 1505a3d43c0dSVinicius Costa Gomes new_admin = kzalloc(sizeof(*new_admin), GFP_KERNEL); 1506a3d43c0dSVinicius Costa Gomes if (!new_admin) { 1507a3d43c0dSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Not enough memory for a new schedule"); 1508a3d43c0dSVinicius Costa Gomes return -ENOMEM; 1509a3d43c0dSVinicius Costa Gomes } 1510a3d43c0dSVinicius Costa Gomes INIT_LIST_HEAD(&new_admin->entries); 15115a781ccbSVinicius Costa Gomes 1512a3d43c0dSVinicius Costa Gomes rcu_read_lock(); 1513a3d43c0dSVinicius Costa Gomes oper = rcu_dereference(q->oper_sched); 1514a3d43c0dSVinicius Costa Gomes admin = rcu_dereference(q->admin_sched); 1515a3d43c0dSVinicius Costa Gomes rcu_read_unlock(); 15165a781ccbSVinicius Costa Gomes 1517b5a0faa3SIvan Khoronzhuk /* no changes - no new mqprio settings */ 1518b5a0faa3SIvan Khoronzhuk if (!taprio_mqprio_cmp(dev, mqprio)) 1519b5a0faa3SIvan Khoronzhuk mqprio = NULL; 1520b5a0faa3SIvan Khoronzhuk 1521a3d43c0dSVinicius Costa Gomes if (mqprio && (oper || admin)) { 1522a3d43c0dSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Changing the traffic mapping of a running schedule is not supported"); 1523a3d43c0dSVinicius Costa Gomes err = -ENOTSUPP; 1524a3d43c0dSVinicius Costa Gomes goto free_sched; 15255a781ccbSVinicius Costa Gomes } 15265a781ccbSVinicius Costa Gomes 1527b5b73b26SVinicius Costa Gomes err = parse_taprio_schedule(q, tb, new_admin, extack); 1528a3d43c0dSVinicius Costa Gomes if (err < 0) 1529a3d43c0dSVinicius Costa Gomes goto free_sched; 15305a781ccbSVinicius Costa Gomes 1531a3d43c0dSVinicius Costa Gomes if (new_admin->num_entries == 0) { 1532a3d43c0dSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "There should be at least one entry in the schedule"); 1533a3d43c0dSVinicius Costa Gomes err = -EINVAL; 1534a3d43c0dSVinicius Costa Gomes goto free_sched; 1535a3d43c0dSVinicius Costa Gomes } 15365a781ccbSVinicius Costa Gomes 15379c66d156SVinicius Costa Gomes err = taprio_parse_clockid(sch, tb, extack); 15389c66d156SVinicius Costa Gomes if (err < 0) 1539a3d43c0dSVinicius Costa Gomes goto free_sched; 1540a3d43c0dSVinicius Costa Gomes 1541a3d43c0dSVinicius Costa Gomes taprio_set_picos_per_byte(dev, q); 1542a3d43c0dSVinicius Costa Gomes 15435652e63dSVinicius Costa Gomes if (mqprio) { 15445652e63dSVinicius Costa Gomes netdev_set_num_tc(dev, mqprio->num_tc); 15455652e63dSVinicius Costa Gomes for (i = 0; i < mqprio->num_tc; i++) 15465652e63dSVinicius Costa Gomes netdev_set_tc_queue(dev, i, 15475652e63dSVinicius Costa Gomes mqprio->count[i], 15485652e63dSVinicius Costa Gomes mqprio->offset[i]); 15495652e63dSVinicius Costa Gomes 15505652e63dSVinicius Costa Gomes /* Always use supplied priority mappings */ 15515652e63dSVinicius Costa Gomes for (i = 0; i <= TC_BITMASK; i++) 15525652e63dSVinicius Costa Gomes netdev_set_prio_tc_map(dev, i, 15535652e63dSVinicius Costa Gomes mqprio->prio_tc_map[i]); 15545652e63dSVinicius Costa Gomes } 15555652e63dSVinicius Costa Gomes 1556a9d62274SVinicius Costa Gomes if (FULL_OFFLOAD_IS_ENABLED(q->flags)) 155709e31cf0SVinicius Costa Gomes err = taprio_enable_offload(dev, q, new_admin, extack); 15589c66d156SVinicius Costa Gomes else 15599c66d156SVinicius Costa Gomes err = taprio_disable_offload(dev, q, extack); 15609c66d156SVinicius Costa Gomes if (err) 15619c66d156SVinicius Costa Gomes goto free_sched; 15629c66d156SVinicius Costa Gomes 1563a3d43c0dSVinicius Costa Gomes /* Protects against enqueue()/dequeue() */ 1564a3d43c0dSVinicius Costa Gomes spin_lock_bh(qdisc_lock(sch)); 1565a3d43c0dSVinicius Costa Gomes 15664cfd5779SVedang Patel if (tb[TCA_TAPRIO_ATTR_TXTIME_DELAY]) { 15674cfd5779SVedang Patel if (!TXTIME_ASSIST_IS_ENABLED(q->flags)) { 15684cfd5779SVedang Patel NL_SET_ERR_MSG_MOD(extack, "txtime-delay can only be set when txtime-assist mode is enabled"); 15694cfd5779SVedang Patel err = -EINVAL; 15704cfd5779SVedang Patel goto unlock; 15714cfd5779SVedang Patel } 15724cfd5779SVedang Patel 1573a5b64700SVedang Patel q->txtime_delay = nla_get_u32(tb[TCA_TAPRIO_ATTR_TXTIME_DELAY]); 15744cfd5779SVedang Patel } 15754cfd5779SVedang Patel 1576a9d62274SVinicius Costa Gomes if (!TXTIME_ASSIST_IS_ENABLED(q->flags) && 1577a9d62274SVinicius Costa Gomes !FULL_OFFLOAD_IS_ENABLED(q->flags) && 15784cfd5779SVedang Patel !hrtimer_active(&q->advance_timer)) { 1579a3d43c0dSVinicius Costa Gomes hrtimer_init(&q->advance_timer, q->clockid, HRTIMER_MODE_ABS); 1580a3d43c0dSVinicius Costa Gomes q->advance_timer.function = advance_sched; 15815a781ccbSVinicius Costa Gomes } 15825a781ccbSVinicius Costa Gomes 1583a9d62274SVinicius Costa Gomes if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { 15849c66d156SVinicius Costa Gomes q->dequeue = taprio_dequeue_offload; 15859c66d156SVinicius Costa Gomes q->peek = taprio_peek_offload; 15869c66d156SVinicius Costa Gomes } else { 15879c66d156SVinicius Costa Gomes /* Be sure to always keep the function pointers 15889c66d156SVinicius Costa Gomes * in a consistent state. 15899c66d156SVinicius Costa Gomes */ 15909c66d156SVinicius Costa Gomes q->dequeue = taprio_dequeue_soft; 15919c66d156SVinicius Costa Gomes q->peek = taprio_peek_soft; 15928599099fSAndre Guedes } 15935a781ccbSVinicius Costa Gomes 1594a3d43c0dSVinicius Costa Gomes err = taprio_get_start_time(sch, new_admin, &start); 1595a3d43c0dSVinicius Costa Gomes if (err < 0) { 1596a3d43c0dSVinicius Costa Gomes NL_SET_ERR_MSG(extack, "Internal error: failed get start time"); 1597a3d43c0dSVinicius Costa Gomes goto unlock; 1598a3d43c0dSVinicius Costa Gomes } 15995a781ccbSVinicius Costa Gomes 16004cfd5779SVedang Patel setup_txtime(q, new_admin, start); 16014cfd5779SVedang Patel 1602bfabd41dSVinicius Costa Gomes if (TXTIME_ASSIST_IS_ENABLED(q->flags)) { 16034cfd5779SVedang Patel if (!oper) { 16044cfd5779SVedang Patel rcu_assign_pointer(q->oper_sched, new_admin); 16054cfd5779SVedang Patel err = 0; 16064cfd5779SVedang Patel new_admin = NULL; 16074cfd5779SVedang Patel goto unlock; 16084cfd5779SVedang Patel } 16094cfd5779SVedang Patel 16104cfd5779SVedang Patel rcu_assign_pointer(q->admin_sched, new_admin); 16114cfd5779SVedang Patel if (admin) 16124cfd5779SVedang Patel call_rcu(&admin->rcu, taprio_free_sched_cb); 16134cfd5779SVedang Patel } else { 1614a3d43c0dSVinicius Costa Gomes setup_first_close_time(q, new_admin, start); 1615a3d43c0dSVinicius Costa Gomes 1616a3d43c0dSVinicius Costa Gomes /* Protects against advance_sched() */ 1617a3d43c0dSVinicius Costa Gomes spin_lock_irqsave(&q->current_entry_lock, flags); 1618a3d43c0dSVinicius Costa Gomes 1619a3d43c0dSVinicius Costa Gomes taprio_start_sched(sch, start, new_admin); 1620a3d43c0dSVinicius Costa Gomes 1621a3d43c0dSVinicius Costa Gomes rcu_assign_pointer(q->admin_sched, new_admin); 1622a3d43c0dSVinicius Costa Gomes if (admin) 1623a3d43c0dSVinicius Costa Gomes call_rcu(&admin->rcu, taprio_free_sched_cb); 1624a3d43c0dSVinicius Costa Gomes 1625a3d43c0dSVinicius Costa Gomes spin_unlock_irqrestore(&q->current_entry_lock, flags); 16260763b3e8SIvan Khoronzhuk 1627a9d62274SVinicius Costa Gomes if (FULL_OFFLOAD_IS_ENABLED(q->flags)) 16280763b3e8SIvan Khoronzhuk taprio_offload_config_changed(q); 16294cfd5779SVedang Patel } 1630a3d43c0dSVinicius Costa Gomes 16314cfd5779SVedang Patel new_admin = NULL; 1632a3d43c0dSVinicius Costa Gomes err = 0; 1633a3d43c0dSVinicius Costa Gomes 1634a3d43c0dSVinicius Costa Gomes unlock: 1635a3d43c0dSVinicius Costa Gomes spin_unlock_bh(qdisc_lock(sch)); 1636a3d43c0dSVinicius Costa Gomes 1637a3d43c0dSVinicius Costa Gomes free_sched: 163851650d33SIvan Khoronzhuk if (new_admin) 163951650d33SIvan Khoronzhuk call_rcu(&new_admin->rcu, taprio_free_sched_cb); 1640a3d43c0dSVinicius Costa Gomes 1641a3d43c0dSVinicius Costa Gomes return err; 16425a781ccbSVinicius Costa Gomes } 16435a781ccbSVinicius Costa Gomes 164444d4775cSDavide Caratti static void taprio_reset(struct Qdisc *sch) 164544d4775cSDavide Caratti { 164644d4775cSDavide Caratti struct taprio_sched *q = qdisc_priv(sch); 164744d4775cSDavide Caratti struct net_device *dev = qdisc_dev(sch); 164844d4775cSDavide Caratti int i; 164944d4775cSDavide Caratti 165044d4775cSDavide Caratti hrtimer_cancel(&q->advance_timer); 165144d4775cSDavide Caratti if (q->qdiscs) { 1652698285daSDavide Caratti for (i = 0; i < dev->num_tx_queues; i++) 1653698285daSDavide Caratti if (q->qdiscs[i]) 165444d4775cSDavide Caratti qdisc_reset(q->qdiscs[i]); 165544d4775cSDavide Caratti } 165644d4775cSDavide Caratti sch->qstats.backlog = 0; 165744d4775cSDavide Caratti sch->q.qlen = 0; 165844d4775cSDavide Caratti } 165944d4775cSDavide Caratti 16605a781ccbSVinicius Costa Gomes static void taprio_destroy(struct Qdisc *sch) 16615a781ccbSVinicius Costa Gomes { 16625a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 16635a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 16645a781ccbSVinicius Costa Gomes unsigned int i; 16655a781ccbSVinicius Costa Gomes 16667b9eba7bSLeandro Dorileo spin_lock(&taprio_list_lock); 16677b9eba7bSLeandro Dorileo list_del(&q->taprio_list); 16687b9eba7bSLeandro Dorileo spin_unlock(&taprio_list_lock); 16697b9eba7bSLeandro Dorileo 16705a781ccbSVinicius Costa Gomes 16719c66d156SVinicius Costa Gomes taprio_disable_offload(dev, q, NULL); 16729c66d156SVinicius Costa Gomes 16735a781ccbSVinicius Costa Gomes if (q->qdiscs) { 1674698285daSDavide Caratti for (i = 0; i < dev->num_tx_queues; i++) 16755a781ccbSVinicius Costa Gomes qdisc_put(q->qdiscs[i]); 16765a781ccbSVinicius Costa Gomes 16775a781ccbSVinicius Costa Gomes kfree(q->qdiscs); 16785a781ccbSVinicius Costa Gomes } 16795a781ccbSVinicius Costa Gomes q->qdiscs = NULL; 16805a781ccbSVinicius Costa Gomes 16817c16680aSVinicius Costa Gomes netdev_reset_tc(dev); 16825a781ccbSVinicius Costa Gomes 1683a3d43c0dSVinicius Costa Gomes if (q->oper_sched) 1684a3d43c0dSVinicius Costa Gomes call_rcu(&q->oper_sched->rcu, taprio_free_sched_cb); 1685a3d43c0dSVinicius Costa Gomes 1686a3d43c0dSVinicius Costa Gomes if (q->admin_sched) 1687a3d43c0dSVinicius Costa Gomes call_rcu(&q->admin_sched->rcu, taprio_free_sched_cb); 16885a781ccbSVinicius Costa Gomes } 16895a781ccbSVinicius Costa Gomes 16905a781ccbSVinicius Costa Gomes static int taprio_init(struct Qdisc *sch, struct nlattr *opt, 16915a781ccbSVinicius Costa Gomes struct netlink_ext_ack *extack) 16925a781ccbSVinicius Costa Gomes { 16935a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 16945a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 1695a3d43c0dSVinicius Costa Gomes int i; 16965a781ccbSVinicius Costa Gomes 16975a781ccbSVinicius Costa Gomes spin_lock_init(&q->current_entry_lock); 16985a781ccbSVinicius Costa Gomes 16995a781ccbSVinicius Costa Gomes hrtimer_init(&q->advance_timer, CLOCK_TAI, HRTIMER_MODE_ABS); 1700a3d43c0dSVinicius Costa Gomes q->advance_timer.function = advance_sched; 17015a781ccbSVinicius Costa Gomes 17029c66d156SVinicius Costa Gomes q->dequeue = taprio_dequeue_soft; 17039c66d156SVinicius Costa Gomes q->peek = taprio_peek_soft; 17049c66d156SVinicius Costa Gomes 17055a781ccbSVinicius Costa Gomes q->root = sch; 17065a781ccbSVinicius Costa Gomes 17075a781ccbSVinicius Costa Gomes /* We only support static clockids. Use an invalid value as default 17085a781ccbSVinicius Costa Gomes * and get the valid one on taprio_change(). 17095a781ccbSVinicius Costa Gomes */ 17105a781ccbSVinicius Costa Gomes q->clockid = -1; 1711a9d62274SVinicius Costa Gomes q->flags = TAPRIO_FLAGS_INVALID; 17125a781ccbSVinicius Costa Gomes 1713efb55222SVladimir Oltean spin_lock(&taprio_list_lock); 1714efb55222SVladimir Oltean list_add(&q->taprio_list, &taprio_list); 1715efb55222SVladimir Oltean spin_unlock(&taprio_list_lock); 1716efb55222SVladimir Oltean 17175a781ccbSVinicius Costa Gomes if (sch->parent != TC_H_ROOT) 17185a781ccbSVinicius Costa Gomes return -EOPNOTSUPP; 17195a781ccbSVinicius Costa Gomes 17205a781ccbSVinicius Costa Gomes if (!netif_is_multiqueue(dev)) 17215a781ccbSVinicius Costa Gomes return -EOPNOTSUPP; 17225a781ccbSVinicius Costa Gomes 17235a781ccbSVinicius Costa Gomes /* pre-allocate qdisc, attachment can't fail */ 17245a781ccbSVinicius Costa Gomes q->qdiscs = kcalloc(dev->num_tx_queues, 17255a781ccbSVinicius Costa Gomes sizeof(q->qdiscs[0]), 17265a781ccbSVinicius Costa Gomes GFP_KERNEL); 17275a781ccbSVinicius Costa Gomes 17285a781ccbSVinicius Costa Gomes if (!q->qdiscs) 17295a781ccbSVinicius Costa Gomes return -ENOMEM; 17305a781ccbSVinicius Costa Gomes 17315a781ccbSVinicius Costa Gomes if (!opt) 17325a781ccbSVinicius Costa Gomes return -EINVAL; 17335a781ccbSVinicius Costa Gomes 1734a3d43c0dSVinicius Costa Gomes for (i = 0; i < dev->num_tx_queues; i++) { 1735a3d43c0dSVinicius Costa Gomes struct netdev_queue *dev_queue; 1736a3d43c0dSVinicius Costa Gomes struct Qdisc *qdisc; 1737a3d43c0dSVinicius Costa Gomes 1738a3d43c0dSVinicius Costa Gomes dev_queue = netdev_get_tx_queue(dev, i); 1739a3d43c0dSVinicius Costa Gomes qdisc = qdisc_create_dflt(dev_queue, 1740a3d43c0dSVinicius Costa Gomes &pfifo_qdisc_ops, 1741a3d43c0dSVinicius Costa Gomes TC_H_MAKE(TC_H_MAJ(sch->handle), 1742a3d43c0dSVinicius Costa Gomes TC_H_MIN(i + 1)), 1743a3d43c0dSVinicius Costa Gomes extack); 1744a3d43c0dSVinicius Costa Gomes if (!qdisc) 1745a3d43c0dSVinicius Costa Gomes return -ENOMEM; 1746a3d43c0dSVinicius Costa Gomes 1747a3d43c0dSVinicius Costa Gomes if (i < dev->real_num_tx_queues) 1748a3d43c0dSVinicius Costa Gomes qdisc_hash_add(qdisc, false); 1749a3d43c0dSVinicius Costa Gomes 1750a3d43c0dSVinicius Costa Gomes q->qdiscs[i] = qdisc; 1751a3d43c0dSVinicius Costa Gomes } 1752a3d43c0dSVinicius Costa Gomes 17535a781ccbSVinicius Costa Gomes return taprio_change(sch, opt, extack); 17545a781ccbSVinicius Costa Gomes } 17555a781ccbSVinicius Costa Gomes 17565a781ccbSVinicius Costa Gomes static struct netdev_queue *taprio_queue_get(struct Qdisc *sch, 17575a781ccbSVinicius Costa Gomes unsigned long cl) 17585a781ccbSVinicius Costa Gomes { 17595a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 17605a781ccbSVinicius Costa Gomes unsigned long ntx = cl - 1; 17615a781ccbSVinicius Costa Gomes 17625a781ccbSVinicius Costa Gomes if (ntx >= dev->num_tx_queues) 17635a781ccbSVinicius Costa Gomes return NULL; 17645a781ccbSVinicius Costa Gomes 17655a781ccbSVinicius Costa Gomes return netdev_get_tx_queue(dev, ntx); 17665a781ccbSVinicius Costa Gomes } 17675a781ccbSVinicius Costa Gomes 17685a781ccbSVinicius Costa Gomes static int taprio_graft(struct Qdisc *sch, unsigned long cl, 17695a781ccbSVinicius Costa Gomes struct Qdisc *new, struct Qdisc **old, 17705a781ccbSVinicius Costa Gomes struct netlink_ext_ack *extack) 17715a781ccbSVinicius Costa Gomes { 17725a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 17735a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 17745a781ccbSVinicius Costa Gomes struct netdev_queue *dev_queue = taprio_queue_get(sch, cl); 17755a781ccbSVinicius Costa Gomes 17765a781ccbSVinicius Costa Gomes if (!dev_queue) 17775a781ccbSVinicius Costa Gomes return -EINVAL; 17785a781ccbSVinicius Costa Gomes 17795a781ccbSVinicius Costa Gomes if (dev->flags & IFF_UP) 17805a781ccbSVinicius Costa Gomes dev_deactivate(dev); 17815a781ccbSVinicius Costa Gomes 17825a781ccbSVinicius Costa Gomes *old = q->qdiscs[cl - 1]; 17835a781ccbSVinicius Costa Gomes q->qdiscs[cl - 1] = new; 17845a781ccbSVinicius Costa Gomes 17855a781ccbSVinicius Costa Gomes if (new) 17865a781ccbSVinicius Costa Gomes new->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; 17875a781ccbSVinicius Costa Gomes 17885a781ccbSVinicius Costa Gomes if (dev->flags & IFF_UP) 17895a781ccbSVinicius Costa Gomes dev_activate(dev); 17905a781ccbSVinicius Costa Gomes 17915a781ccbSVinicius Costa Gomes return 0; 17925a781ccbSVinicius Costa Gomes } 17935a781ccbSVinicius Costa Gomes 17945a781ccbSVinicius Costa Gomes static int dump_entry(struct sk_buff *msg, 17955a781ccbSVinicius Costa Gomes const struct sched_entry *entry) 17965a781ccbSVinicius Costa Gomes { 17975a781ccbSVinicius Costa Gomes struct nlattr *item; 17985a781ccbSVinicius Costa Gomes 1799ae0be8deSMichal Kubecek item = nla_nest_start_noflag(msg, TCA_TAPRIO_SCHED_ENTRY); 18005a781ccbSVinicius Costa Gomes if (!item) 18015a781ccbSVinicius Costa Gomes return -ENOSPC; 18025a781ccbSVinicius Costa Gomes 18035a781ccbSVinicius Costa Gomes if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_INDEX, entry->index)) 18045a781ccbSVinicius Costa Gomes goto nla_put_failure; 18055a781ccbSVinicius Costa Gomes 18065a781ccbSVinicius Costa Gomes if (nla_put_u8(msg, TCA_TAPRIO_SCHED_ENTRY_CMD, entry->command)) 18075a781ccbSVinicius Costa Gomes goto nla_put_failure; 18085a781ccbSVinicius Costa Gomes 18095a781ccbSVinicius Costa Gomes if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_GATE_MASK, 18105a781ccbSVinicius Costa Gomes entry->gate_mask)) 18115a781ccbSVinicius Costa Gomes goto nla_put_failure; 18125a781ccbSVinicius Costa Gomes 18135a781ccbSVinicius Costa Gomes if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_INTERVAL, 18145a781ccbSVinicius Costa Gomes entry->interval)) 18155a781ccbSVinicius Costa Gomes goto nla_put_failure; 18165a781ccbSVinicius Costa Gomes 18175a781ccbSVinicius Costa Gomes return nla_nest_end(msg, item); 18185a781ccbSVinicius Costa Gomes 18195a781ccbSVinicius Costa Gomes nla_put_failure: 18205a781ccbSVinicius Costa Gomes nla_nest_cancel(msg, item); 18215a781ccbSVinicius Costa Gomes return -1; 18225a781ccbSVinicius Costa Gomes } 18235a781ccbSVinicius Costa Gomes 1824a3d43c0dSVinicius Costa Gomes static int dump_schedule(struct sk_buff *msg, 1825a3d43c0dSVinicius Costa Gomes const struct sched_gate_list *root) 1826a3d43c0dSVinicius Costa Gomes { 1827a3d43c0dSVinicius Costa Gomes struct nlattr *entry_list; 1828a3d43c0dSVinicius Costa Gomes struct sched_entry *entry; 1829a3d43c0dSVinicius Costa Gomes 1830a3d43c0dSVinicius Costa Gomes if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_BASE_TIME, 1831a3d43c0dSVinicius Costa Gomes root->base_time, TCA_TAPRIO_PAD)) 1832a3d43c0dSVinicius Costa Gomes return -1; 1833a3d43c0dSVinicius Costa Gomes 18346ca6a665SVinicius Costa Gomes if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME, 18356ca6a665SVinicius Costa Gomes root->cycle_time, TCA_TAPRIO_PAD)) 18366ca6a665SVinicius Costa Gomes return -1; 18376ca6a665SVinicius Costa Gomes 1838c25031e9SVinicius Costa Gomes if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION, 1839c25031e9SVinicius Costa Gomes root->cycle_time_extension, TCA_TAPRIO_PAD)) 1840c25031e9SVinicius Costa Gomes return -1; 1841c25031e9SVinicius Costa Gomes 1842a3d43c0dSVinicius Costa Gomes entry_list = nla_nest_start_noflag(msg, 1843a3d43c0dSVinicius Costa Gomes TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST); 1844a3d43c0dSVinicius Costa Gomes if (!entry_list) 1845a3d43c0dSVinicius Costa Gomes goto error_nest; 1846a3d43c0dSVinicius Costa Gomes 1847a3d43c0dSVinicius Costa Gomes list_for_each_entry(entry, &root->entries, list) { 1848a3d43c0dSVinicius Costa Gomes if (dump_entry(msg, entry) < 0) 1849a3d43c0dSVinicius Costa Gomes goto error_nest; 1850a3d43c0dSVinicius Costa Gomes } 1851a3d43c0dSVinicius Costa Gomes 1852a3d43c0dSVinicius Costa Gomes nla_nest_end(msg, entry_list); 1853a3d43c0dSVinicius Costa Gomes return 0; 1854a3d43c0dSVinicius Costa Gomes 1855a3d43c0dSVinicius Costa Gomes error_nest: 1856a3d43c0dSVinicius Costa Gomes nla_nest_cancel(msg, entry_list); 1857a3d43c0dSVinicius Costa Gomes return -1; 1858a3d43c0dSVinicius Costa Gomes } 1859a3d43c0dSVinicius Costa Gomes 18605a781ccbSVinicius Costa Gomes static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb) 18615a781ccbSVinicius Costa Gomes { 18625a781ccbSVinicius Costa Gomes struct taprio_sched *q = qdisc_priv(sch); 18635a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 1864a3d43c0dSVinicius Costa Gomes struct sched_gate_list *oper, *admin; 18655a781ccbSVinicius Costa Gomes struct tc_mqprio_qopt opt = { 0 }; 1866a3d43c0dSVinicius Costa Gomes struct nlattr *nest, *sched_nest; 18675a781ccbSVinicius Costa Gomes unsigned int i; 18685a781ccbSVinicius Costa Gomes 1869a3d43c0dSVinicius Costa Gomes rcu_read_lock(); 1870a3d43c0dSVinicius Costa Gomes oper = rcu_dereference(q->oper_sched); 1871a3d43c0dSVinicius Costa Gomes admin = rcu_dereference(q->admin_sched); 1872a3d43c0dSVinicius Costa Gomes 18735a781ccbSVinicius Costa Gomes opt.num_tc = netdev_get_num_tc(dev); 18745a781ccbSVinicius Costa Gomes memcpy(opt.prio_tc_map, dev->prio_tc_map, sizeof(opt.prio_tc_map)); 18755a781ccbSVinicius Costa Gomes 18765a781ccbSVinicius Costa Gomes for (i = 0; i < netdev_get_num_tc(dev); i++) { 18775a781ccbSVinicius Costa Gomes opt.count[i] = dev->tc_to_txq[i].count; 18785a781ccbSVinicius Costa Gomes opt.offset[i] = dev->tc_to_txq[i].offset; 18795a781ccbSVinicius Costa Gomes } 18805a781ccbSVinicius Costa Gomes 1881ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 18825a781ccbSVinicius Costa Gomes if (!nest) 1883a3d43c0dSVinicius Costa Gomes goto start_error; 18845a781ccbSVinicius Costa Gomes 18855a781ccbSVinicius Costa Gomes if (nla_put(skb, TCA_TAPRIO_ATTR_PRIOMAP, sizeof(opt), &opt)) 18865a781ccbSVinicius Costa Gomes goto options_error; 18875a781ccbSVinicius Costa Gomes 18889c66d156SVinicius Costa Gomes if (!FULL_OFFLOAD_IS_ENABLED(q->flags) && 18899c66d156SVinicius Costa Gomes nla_put_s32(skb, TCA_TAPRIO_ATTR_SCHED_CLOCKID, q->clockid)) 18905a781ccbSVinicius Costa Gomes goto options_error; 18915a781ccbSVinicius Costa Gomes 18924cfd5779SVedang Patel if (q->flags && nla_put_u32(skb, TCA_TAPRIO_ATTR_FLAGS, q->flags)) 18934cfd5779SVedang Patel goto options_error; 18944cfd5779SVedang Patel 18954cfd5779SVedang Patel if (q->txtime_delay && 1896a5b64700SVedang Patel nla_put_u32(skb, TCA_TAPRIO_ATTR_TXTIME_DELAY, q->txtime_delay)) 18974cfd5779SVedang Patel goto options_error; 18984cfd5779SVedang Patel 1899a3d43c0dSVinicius Costa Gomes if (oper && dump_schedule(skb, oper)) 19005a781ccbSVinicius Costa Gomes goto options_error; 19015a781ccbSVinicius Costa Gomes 1902a3d43c0dSVinicius Costa Gomes if (!admin) 1903a3d43c0dSVinicius Costa Gomes goto done; 19045a781ccbSVinicius Costa Gomes 1905a3d43c0dSVinicius Costa Gomes sched_nest = nla_nest_start_noflag(skb, TCA_TAPRIO_ATTR_ADMIN_SCHED); 1906e4acf427SColin Ian King if (!sched_nest) 1907e4acf427SColin Ian King goto options_error; 1908a3d43c0dSVinicius Costa Gomes 1909a3d43c0dSVinicius Costa Gomes if (dump_schedule(skb, admin)) 1910a3d43c0dSVinicius Costa Gomes goto admin_error; 1911a3d43c0dSVinicius Costa Gomes 1912a3d43c0dSVinicius Costa Gomes nla_nest_end(skb, sched_nest); 1913a3d43c0dSVinicius Costa Gomes 1914a3d43c0dSVinicius Costa Gomes done: 1915a3d43c0dSVinicius Costa Gomes rcu_read_unlock(); 19165a781ccbSVinicius Costa Gomes 19175a781ccbSVinicius Costa Gomes return nla_nest_end(skb, nest); 19185a781ccbSVinicius Costa Gomes 1919a3d43c0dSVinicius Costa Gomes admin_error: 1920a3d43c0dSVinicius Costa Gomes nla_nest_cancel(skb, sched_nest); 1921a3d43c0dSVinicius Costa Gomes 19225a781ccbSVinicius Costa Gomes options_error: 19235a781ccbSVinicius Costa Gomes nla_nest_cancel(skb, nest); 1924a3d43c0dSVinicius Costa Gomes 1925a3d43c0dSVinicius Costa Gomes start_error: 1926a3d43c0dSVinicius Costa Gomes rcu_read_unlock(); 1927a3d43c0dSVinicius Costa Gomes return -ENOSPC; 19285a781ccbSVinicius Costa Gomes } 19295a781ccbSVinicius Costa Gomes 19305a781ccbSVinicius Costa Gomes static struct Qdisc *taprio_leaf(struct Qdisc *sch, unsigned long cl) 19315a781ccbSVinicius Costa Gomes { 19325a781ccbSVinicius Costa Gomes struct netdev_queue *dev_queue = taprio_queue_get(sch, cl); 19335a781ccbSVinicius Costa Gomes 19345a781ccbSVinicius Costa Gomes if (!dev_queue) 19355a781ccbSVinicius Costa Gomes return NULL; 19365a781ccbSVinicius Costa Gomes 19375a781ccbSVinicius Costa Gomes return dev_queue->qdisc_sleeping; 19385a781ccbSVinicius Costa Gomes } 19395a781ccbSVinicius Costa Gomes 19405a781ccbSVinicius Costa Gomes static unsigned long taprio_find(struct Qdisc *sch, u32 classid) 19415a781ccbSVinicius Costa Gomes { 19425a781ccbSVinicius Costa Gomes unsigned int ntx = TC_H_MIN(classid); 19435a781ccbSVinicius Costa Gomes 19445a781ccbSVinicius Costa Gomes if (!taprio_queue_get(sch, ntx)) 19455a781ccbSVinicius Costa Gomes return 0; 19465a781ccbSVinicius Costa Gomes return ntx; 19475a781ccbSVinicius Costa Gomes } 19485a781ccbSVinicius Costa Gomes 19495a781ccbSVinicius Costa Gomes static int taprio_dump_class(struct Qdisc *sch, unsigned long cl, 19505a781ccbSVinicius Costa Gomes struct sk_buff *skb, struct tcmsg *tcm) 19515a781ccbSVinicius Costa Gomes { 19525a781ccbSVinicius Costa Gomes struct netdev_queue *dev_queue = taprio_queue_get(sch, cl); 19535a781ccbSVinicius Costa Gomes 19545a781ccbSVinicius Costa Gomes tcm->tcm_parent = TC_H_ROOT; 19555a781ccbSVinicius Costa Gomes tcm->tcm_handle |= TC_H_MIN(cl); 19565a781ccbSVinicius Costa Gomes tcm->tcm_info = dev_queue->qdisc_sleeping->handle; 19575a781ccbSVinicius Costa Gomes 19585a781ccbSVinicius Costa Gomes return 0; 19595a781ccbSVinicius Costa Gomes } 19605a781ccbSVinicius Costa Gomes 19615a781ccbSVinicius Costa Gomes static int taprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, 19625a781ccbSVinicius Costa Gomes struct gnet_dump *d) 19635a781ccbSVinicius Costa Gomes __releases(d->lock) 19645a781ccbSVinicius Costa Gomes __acquires(d->lock) 19655a781ccbSVinicius Costa Gomes { 19665a781ccbSVinicius Costa Gomes struct netdev_queue *dev_queue = taprio_queue_get(sch, cl); 19675a781ccbSVinicius Costa Gomes 19685a781ccbSVinicius Costa Gomes sch = dev_queue->qdisc_sleeping; 19695a781ccbSVinicius Costa Gomes if (gnet_stats_copy_basic(&sch->running, d, NULL, &sch->bstats) < 0 || 19705dd431b6SPaolo Abeni qdisc_qstats_copy(d, sch) < 0) 19715a781ccbSVinicius Costa Gomes return -1; 19725a781ccbSVinicius Costa Gomes return 0; 19735a781ccbSVinicius Costa Gomes } 19745a781ccbSVinicius Costa Gomes 19755a781ccbSVinicius Costa Gomes static void taprio_walk(struct Qdisc *sch, struct qdisc_walker *arg) 19765a781ccbSVinicius Costa Gomes { 19775a781ccbSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 19785a781ccbSVinicius Costa Gomes unsigned long ntx; 19795a781ccbSVinicius Costa Gomes 19805a781ccbSVinicius Costa Gomes if (arg->stop) 19815a781ccbSVinicius Costa Gomes return; 19825a781ccbSVinicius Costa Gomes 19835a781ccbSVinicius Costa Gomes arg->count = arg->skip; 19845a781ccbSVinicius Costa Gomes for (ntx = arg->skip; ntx < dev->num_tx_queues; ntx++) { 19855a781ccbSVinicius Costa Gomes if (arg->fn(sch, ntx + 1, arg) < 0) { 19865a781ccbSVinicius Costa Gomes arg->stop = 1; 19875a781ccbSVinicius Costa Gomes break; 19885a781ccbSVinicius Costa Gomes } 19895a781ccbSVinicius Costa Gomes arg->count++; 19905a781ccbSVinicius Costa Gomes } 19915a781ccbSVinicius Costa Gomes } 19925a781ccbSVinicius Costa Gomes 19935a781ccbSVinicius Costa Gomes static struct netdev_queue *taprio_select_queue(struct Qdisc *sch, 19945a781ccbSVinicius Costa Gomes struct tcmsg *tcm) 19955a781ccbSVinicius Costa Gomes { 19965a781ccbSVinicius Costa Gomes return taprio_queue_get(sch, TC_H_MIN(tcm->tcm_parent)); 19975a781ccbSVinicius Costa Gomes } 19985a781ccbSVinicius Costa Gomes 19995a781ccbSVinicius Costa Gomes static const struct Qdisc_class_ops taprio_class_ops = { 20005a781ccbSVinicius Costa Gomes .graft = taprio_graft, 20015a781ccbSVinicius Costa Gomes .leaf = taprio_leaf, 20025a781ccbSVinicius Costa Gomes .find = taprio_find, 20035a781ccbSVinicius Costa Gomes .walk = taprio_walk, 20045a781ccbSVinicius Costa Gomes .dump = taprio_dump_class, 20055a781ccbSVinicius Costa Gomes .dump_stats = taprio_dump_class_stats, 20065a781ccbSVinicius Costa Gomes .select_queue = taprio_select_queue, 20075a781ccbSVinicius Costa Gomes }; 20085a781ccbSVinicius Costa Gomes 20095a781ccbSVinicius Costa Gomes static struct Qdisc_ops taprio_qdisc_ops __read_mostly = { 20105a781ccbSVinicius Costa Gomes .cl_ops = &taprio_class_ops, 20115a781ccbSVinicius Costa Gomes .id = "taprio", 20125a781ccbSVinicius Costa Gomes .priv_size = sizeof(struct taprio_sched), 20135a781ccbSVinicius Costa Gomes .init = taprio_init, 2014a3d43c0dSVinicius Costa Gomes .change = taprio_change, 20155a781ccbSVinicius Costa Gomes .destroy = taprio_destroy, 201644d4775cSDavide Caratti .reset = taprio_reset, 20175a781ccbSVinicius Costa Gomes .peek = taprio_peek, 20185a781ccbSVinicius Costa Gomes .dequeue = taprio_dequeue, 20195a781ccbSVinicius Costa Gomes .enqueue = taprio_enqueue, 20205a781ccbSVinicius Costa Gomes .dump = taprio_dump, 20215a781ccbSVinicius Costa Gomes .owner = THIS_MODULE, 20225a781ccbSVinicius Costa Gomes }; 20235a781ccbSVinicius Costa Gomes 20247b9eba7bSLeandro Dorileo static struct notifier_block taprio_device_notifier = { 20257b9eba7bSLeandro Dorileo .notifier_call = taprio_dev_notifier, 20267b9eba7bSLeandro Dorileo }; 20277b9eba7bSLeandro Dorileo 20285a781ccbSVinicius Costa Gomes static int __init taprio_module_init(void) 20295a781ccbSVinicius Costa Gomes { 20307b9eba7bSLeandro Dorileo int err = register_netdevice_notifier(&taprio_device_notifier); 20317b9eba7bSLeandro Dorileo 20327b9eba7bSLeandro Dorileo if (err) 20337b9eba7bSLeandro Dorileo return err; 20347b9eba7bSLeandro Dorileo 20355a781ccbSVinicius Costa Gomes return register_qdisc(&taprio_qdisc_ops); 20365a781ccbSVinicius Costa Gomes } 20375a781ccbSVinicius Costa Gomes 20385a781ccbSVinicius Costa Gomes static void __exit taprio_module_exit(void) 20395a781ccbSVinicius Costa Gomes { 20405a781ccbSVinicius Costa Gomes unregister_qdisc(&taprio_qdisc_ops); 20417b9eba7bSLeandro Dorileo unregister_netdevice_notifier(&taprio_device_notifier); 20425a781ccbSVinicius Costa Gomes } 20435a781ccbSVinicius Costa Gomes 20445a781ccbSVinicius Costa Gomes module_init(taprio_module_init); 20455a781ccbSVinicius Costa Gomes module_exit(taprio_module_exit); 20465a781ccbSVinicius Costa Gomes MODULE_LICENSE("GPL"); 2047