12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2585d763aSVinicius Costa Gomes /* 3585d763aSVinicius Costa Gomes * net/sched/sch_cbs.c Credit Based Shaper 4585d763aSVinicius Costa Gomes * 5585d763aSVinicius Costa Gomes * Authors: Vinicius Costa Gomes <vinicius.gomes@intel.com> 6585d763aSVinicius Costa Gomes */ 7585d763aSVinicius Costa Gomes 8585d763aSVinicius Costa Gomes /* Credit Based Shaper (CBS) 9585d763aSVinicius Costa Gomes * ========================= 10585d763aSVinicius Costa Gomes * 11585d763aSVinicius Costa Gomes * This is a simple rate-limiting shaper aimed at TSN applications on 12585d763aSVinicius Costa Gomes * systems with known traffic workloads. 13585d763aSVinicius Costa Gomes * 14585d763aSVinicius Costa Gomes * Its algorithm is defined by the IEEE 802.1Q-2014 Specification, 15585d763aSVinicius Costa Gomes * Section 8.6.8.2, and explained in more detail in the Annex L of the 16585d763aSVinicius Costa Gomes * same specification. 17585d763aSVinicius Costa Gomes * 18585d763aSVinicius Costa Gomes * There are four tunables to be considered: 19585d763aSVinicius Costa Gomes * 20585d763aSVinicius Costa Gomes * 'idleslope': Idleslope is the rate of credits that is 21585d763aSVinicius Costa Gomes * accumulated (in kilobits per second) when there is at least 22585d763aSVinicius Costa Gomes * one packet waiting for transmission. Packets are transmitted 23585d763aSVinicius Costa Gomes * when the current value of credits is equal or greater than 24585d763aSVinicius Costa Gomes * zero. When there is no packet to be transmitted the amount of 25585d763aSVinicius Costa Gomes * credits is set to zero. This is the main tunable of the CBS 26585d763aSVinicius Costa Gomes * algorithm. 27585d763aSVinicius Costa Gomes * 28585d763aSVinicius Costa Gomes * 'sendslope': 29585d763aSVinicius Costa Gomes * Sendslope is the rate of credits that is depleted (it should be a 30585d763aSVinicius Costa Gomes * negative number of kilobits per second) when a transmission is 31585d763aSVinicius Costa Gomes * ocurring. It can be calculated as follows, (IEEE 802.1Q-2014 Section 32585d763aSVinicius Costa Gomes * 8.6.8.2 item g): 33585d763aSVinicius Costa Gomes * 34585d763aSVinicius Costa Gomes * sendslope = idleslope - port_transmit_rate 35585d763aSVinicius Costa Gomes * 36585d763aSVinicius Costa Gomes * 'hicredit': Hicredit defines the maximum amount of credits (in 37585d763aSVinicius Costa Gomes * bytes) that can be accumulated. Hicredit depends on the 38585d763aSVinicius Costa Gomes * characteristics of interfering traffic, 39585d763aSVinicius Costa Gomes * 'max_interference_size' is the maximum size of any burst of 40585d763aSVinicius Costa Gomes * traffic that can delay the transmission of a frame that is 41585d763aSVinicius Costa Gomes * available for transmission for this traffic class, (IEEE 42585d763aSVinicius Costa Gomes * 802.1Q-2014 Annex L, Equation L-3): 43585d763aSVinicius Costa Gomes * 44585d763aSVinicius Costa Gomes * hicredit = max_interference_size * (idleslope / port_transmit_rate) 45585d763aSVinicius Costa Gomes * 46585d763aSVinicius Costa Gomes * 'locredit': Locredit is the minimum amount of credits that can 47585d763aSVinicius Costa Gomes * be reached. It is a function of the traffic flowing through 48585d763aSVinicius Costa Gomes * this qdisc (IEEE 802.1Q-2014 Annex L, Equation L-2): 49585d763aSVinicius Costa Gomes * 50585d763aSVinicius Costa Gomes * locredit = max_frame_size * (sendslope / port_transmit_rate) 51585d763aSVinicius Costa Gomes */ 52585d763aSVinicius Costa Gomes 53585d763aSVinicius Costa Gomes #include <linux/module.h> 54585d763aSVinicius Costa Gomes #include <linux/types.h> 55585d763aSVinicius Costa Gomes #include <linux/kernel.h> 56585d763aSVinicius Costa Gomes #include <linux/string.h> 57585d763aSVinicius Costa Gomes #include <linux/errno.h> 58585d763aSVinicius Costa Gomes #include <linux/skbuff.h> 59e0a7683dSLeandro Dorileo #include <net/netevent.h> 60585d763aSVinicius Costa Gomes #include <net/netlink.h> 61585d763aSVinicius Costa Gomes #include <net/sch_generic.h> 62585d763aSVinicius Costa Gomes #include <net/pkt_sched.h> 63585d763aSVinicius Costa Gomes 64e0a7683dSLeandro Dorileo static LIST_HEAD(cbs_list); 65e0a7683dSLeandro Dorileo static DEFINE_SPINLOCK(cbs_list_lock); 66e0a7683dSLeandro Dorileo 67585d763aSVinicius Costa Gomes #define BYTES_PER_KBIT (1000LL / 8) 68585d763aSVinicius Costa Gomes 69585d763aSVinicius Costa Gomes struct cbs_sched_data { 703d0bd028SVinicius Costa Gomes bool offload; 713d0bd028SVinicius Costa Gomes int queue; 72e0a7683dSLeandro Dorileo atomic64_t port_rate; /* in bytes/s */ 73585d763aSVinicius Costa Gomes s64 last; /* timestamp in ns */ 74585d763aSVinicius Costa Gomes s64 credits; /* in bytes */ 75585d763aSVinicius Costa Gomes s32 locredit; /* in bytes */ 76585d763aSVinicius Costa Gomes s32 hicredit; /* in bytes */ 77585d763aSVinicius Costa Gomes s64 sendslope; /* in bytes/s */ 78585d763aSVinicius Costa Gomes s64 idleslope; /* in bytes/s */ 79585d763aSVinicius Costa Gomes struct qdisc_watchdog watchdog; 80990e35ecSVinicius Costa Gomes int (*enqueue)(struct sk_buff *skb, struct Qdisc *sch, 81990e35ecSVinicius Costa Gomes struct sk_buff **to_free); 82585d763aSVinicius Costa Gomes struct sk_buff *(*dequeue)(struct Qdisc *sch); 83990e35ecSVinicius Costa Gomes struct Qdisc *qdisc; 84e0a7683dSLeandro Dorileo struct list_head cbs_list; 85585d763aSVinicius Costa Gomes }; 86585d763aSVinicius Costa Gomes 87990e35ecSVinicius Costa Gomes static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch, 88990e35ecSVinicius Costa Gomes struct Qdisc *child, 89990e35ecSVinicius Costa Gomes struct sk_buff **to_free) 903d0bd028SVinicius Costa Gomes { 91f6bab199SToke Høiland-Jørgensen unsigned int len = qdisc_pkt_len(skb); 92990e35ecSVinicius Costa Gomes int err; 93990e35ecSVinicius Costa Gomes 94990e35ecSVinicius Costa Gomes err = child->ops->enqueue(skb, child, to_free); 95990e35ecSVinicius Costa Gomes if (err != NET_XMIT_SUCCESS) 96990e35ecSVinicius Costa Gomes return err; 97990e35ecSVinicius Costa Gomes 98f6bab199SToke Høiland-Jørgensen sch->qstats.backlog += len; 99990e35ecSVinicius Costa Gomes sch->q.qlen++; 100990e35ecSVinicius Costa Gomes 101990e35ecSVinicius Costa Gomes return NET_XMIT_SUCCESS; 1023d0bd028SVinicius Costa Gomes } 1033d0bd028SVinicius Costa Gomes 104990e35ecSVinicius Costa Gomes static int cbs_enqueue_offload(struct sk_buff *skb, struct Qdisc *sch, 105990e35ecSVinicius Costa Gomes struct sk_buff **to_free) 106585d763aSVinicius Costa Gomes { 107585d763aSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 108990e35ecSVinicius Costa Gomes struct Qdisc *qdisc = q->qdisc; 109990e35ecSVinicius Costa Gomes 110990e35ecSVinicius Costa Gomes return cbs_child_enqueue(skb, sch, qdisc, to_free); 111990e35ecSVinicius Costa Gomes } 112990e35ecSVinicius Costa Gomes 113990e35ecSVinicius Costa Gomes static int cbs_enqueue_soft(struct sk_buff *skb, struct Qdisc *sch, 114990e35ecSVinicius Costa Gomes struct sk_buff **to_free) 115990e35ecSVinicius Costa Gomes { 116990e35ecSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 117990e35ecSVinicius Costa Gomes struct Qdisc *qdisc = q->qdisc; 118585d763aSVinicius Costa Gomes 119585d763aSVinicius Costa Gomes if (sch->q.qlen == 0 && q->credits > 0) { 120585d763aSVinicius Costa Gomes /* We need to stop accumulating credits when there's 121585d763aSVinicius Costa Gomes * no enqueued packets and q->credits is positive. 122585d763aSVinicius Costa Gomes */ 123585d763aSVinicius Costa Gomes q->credits = 0; 124585d763aSVinicius Costa Gomes q->last = ktime_get_ns(); 125585d763aSVinicius Costa Gomes } 126585d763aSVinicius Costa Gomes 127990e35ecSVinicius Costa Gomes return cbs_child_enqueue(skb, sch, qdisc, to_free); 128585d763aSVinicius Costa Gomes } 129585d763aSVinicius Costa Gomes 130585d763aSVinicius Costa Gomes static int cbs_enqueue(struct sk_buff *skb, struct Qdisc *sch, 131585d763aSVinicius Costa Gomes struct sk_buff **to_free) 132585d763aSVinicius Costa Gomes { 133585d763aSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 134585d763aSVinicius Costa Gomes 135990e35ecSVinicius Costa Gomes return q->enqueue(skb, sch, to_free); 136585d763aSVinicius Costa Gomes } 137585d763aSVinicius Costa Gomes 138585d763aSVinicius Costa Gomes /* timediff is in ns, slope is in bytes/s */ 139585d763aSVinicius Costa Gomes static s64 timediff_to_credits(s64 timediff, s64 slope) 140585d763aSVinicius Costa Gomes { 141585d763aSVinicius Costa Gomes return div64_s64(timediff * slope, NSEC_PER_SEC); 142585d763aSVinicius Costa Gomes } 143585d763aSVinicius Costa Gomes 144585d763aSVinicius Costa Gomes static s64 delay_from_credits(s64 credits, s64 slope) 145585d763aSVinicius Costa Gomes { 146585d763aSVinicius Costa Gomes if (unlikely(slope == 0)) 147585d763aSVinicius Costa Gomes return S64_MAX; 148585d763aSVinicius Costa Gomes 149585d763aSVinicius Costa Gomes return div64_s64(-credits * NSEC_PER_SEC, slope); 150585d763aSVinicius Costa Gomes } 151585d763aSVinicius Costa Gomes 152585d763aSVinicius Costa Gomes static s64 credits_from_len(unsigned int len, s64 slope, s64 port_rate) 153585d763aSVinicius Costa Gomes { 154585d763aSVinicius Costa Gomes if (unlikely(port_rate == 0)) 155585d763aSVinicius Costa Gomes return S64_MAX; 156585d763aSVinicius Costa Gomes 157585d763aSVinicius Costa Gomes return div64_s64(len * slope, port_rate); 158585d763aSVinicius Costa Gomes } 159585d763aSVinicius Costa Gomes 160990e35ecSVinicius Costa Gomes static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child) 161990e35ecSVinicius Costa Gomes { 162990e35ecSVinicius Costa Gomes struct sk_buff *skb; 163990e35ecSVinicius Costa Gomes 164990e35ecSVinicius Costa Gomes skb = child->ops->dequeue(child); 165990e35ecSVinicius Costa Gomes if (!skb) 166990e35ecSVinicius Costa Gomes return NULL; 167990e35ecSVinicius Costa Gomes 168990e35ecSVinicius Costa Gomes qdisc_qstats_backlog_dec(sch, skb); 169990e35ecSVinicius Costa Gomes qdisc_bstats_update(sch, skb); 170990e35ecSVinicius Costa Gomes sch->q.qlen--; 171990e35ecSVinicius Costa Gomes 172990e35ecSVinicius Costa Gomes return skb; 173990e35ecSVinicius Costa Gomes } 174990e35ecSVinicius Costa Gomes 175585d763aSVinicius Costa Gomes static struct sk_buff *cbs_dequeue_soft(struct Qdisc *sch) 176585d763aSVinicius Costa Gomes { 177585d763aSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 178990e35ecSVinicius Costa Gomes struct Qdisc *qdisc = q->qdisc; 179585d763aSVinicius Costa Gomes s64 now = ktime_get_ns(); 180585d763aSVinicius Costa Gomes struct sk_buff *skb; 181585d763aSVinicius Costa Gomes s64 credits; 182585d763aSVinicius Costa Gomes int len; 183585d763aSVinicius Costa Gomes 184961d0e5bSZh-yuan Ye /* The previous packet is still being sent */ 185961d0e5bSZh-yuan Ye if (now < q->last) { 186961d0e5bSZh-yuan Ye qdisc_watchdog_schedule_ns(&q->watchdog, q->last); 187961d0e5bSZh-yuan Ye return NULL; 188961d0e5bSZh-yuan Ye } 189585d763aSVinicius Costa Gomes if (q->credits < 0) { 190585d763aSVinicius Costa Gomes credits = timediff_to_credits(now - q->last, q->idleslope); 191585d763aSVinicius Costa Gomes 192585d763aSVinicius Costa Gomes credits = q->credits + credits; 193585d763aSVinicius Costa Gomes q->credits = min_t(s64, credits, q->hicredit); 194585d763aSVinicius Costa Gomes 195585d763aSVinicius Costa Gomes if (q->credits < 0) { 196585d763aSVinicius Costa Gomes s64 delay; 197585d763aSVinicius Costa Gomes 198585d763aSVinicius Costa Gomes delay = delay_from_credits(q->credits, q->idleslope); 199585d763aSVinicius Costa Gomes qdisc_watchdog_schedule_ns(&q->watchdog, now + delay); 200585d763aSVinicius Costa Gomes 201585d763aSVinicius Costa Gomes q->last = now; 202585d763aSVinicius Costa Gomes 203585d763aSVinicius Costa Gomes return NULL; 204585d763aSVinicius Costa Gomes } 205585d763aSVinicius Costa Gomes } 206990e35ecSVinicius Costa Gomes skb = cbs_child_dequeue(sch, qdisc); 207585d763aSVinicius Costa Gomes if (!skb) 208585d763aSVinicius Costa Gomes return NULL; 209585d763aSVinicius Costa Gomes 210585d763aSVinicius Costa Gomes len = qdisc_pkt_len(skb); 211585d763aSVinicius Costa Gomes 212585d763aSVinicius Costa Gomes /* As sendslope is a negative number, this will decrease the 213585d763aSVinicius Costa Gomes * amount of q->credits. 214585d763aSVinicius Costa Gomes */ 215e0a7683dSLeandro Dorileo credits = credits_from_len(len, q->sendslope, 216e0a7683dSLeandro Dorileo atomic64_read(&q->port_rate)); 217585d763aSVinicius Costa Gomes credits += q->credits; 218585d763aSVinicius Costa Gomes 219585d763aSVinicius Costa Gomes q->credits = max_t(s64, credits, q->locredit); 220961d0e5bSZh-yuan Ye /* Estimate of the transmission of the last byte of the packet in ns */ 221961d0e5bSZh-yuan Ye if (unlikely(atomic64_read(&q->port_rate) == 0)) 222585d763aSVinicius Costa Gomes q->last = now; 223961d0e5bSZh-yuan Ye else 224961d0e5bSZh-yuan Ye q->last = now + div64_s64(len * NSEC_PER_SEC, 225961d0e5bSZh-yuan Ye atomic64_read(&q->port_rate)); 226585d763aSVinicius Costa Gomes 227585d763aSVinicius Costa Gomes return skb; 228585d763aSVinicius Costa Gomes } 229585d763aSVinicius Costa Gomes 2303d0bd028SVinicius Costa Gomes static struct sk_buff *cbs_dequeue_offload(struct Qdisc *sch) 2313d0bd028SVinicius Costa Gomes { 232990e35ecSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 233990e35ecSVinicius Costa Gomes struct Qdisc *qdisc = q->qdisc; 234990e35ecSVinicius Costa Gomes 235990e35ecSVinicius Costa Gomes return cbs_child_dequeue(sch, qdisc); 2363d0bd028SVinicius Costa Gomes } 2373d0bd028SVinicius Costa Gomes 238585d763aSVinicius Costa Gomes static struct sk_buff *cbs_dequeue(struct Qdisc *sch) 239585d763aSVinicius Costa Gomes { 240585d763aSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 241585d763aSVinicius Costa Gomes 242585d763aSVinicius Costa Gomes return q->dequeue(sch); 243585d763aSVinicius Costa Gomes } 244585d763aSVinicius Costa Gomes 245585d763aSVinicius Costa Gomes static const struct nla_policy cbs_policy[TCA_CBS_MAX + 1] = { 246585d763aSVinicius Costa Gomes [TCA_CBS_PARMS] = { .len = sizeof(struct tc_cbs_qopt) }, 247585d763aSVinicius Costa Gomes }; 248585d763aSVinicius Costa Gomes 2493d0bd028SVinicius Costa Gomes static void cbs_disable_offload(struct net_device *dev, 2503d0bd028SVinicius Costa Gomes struct cbs_sched_data *q) 2513d0bd028SVinicius Costa Gomes { 2523d0bd028SVinicius Costa Gomes struct tc_cbs_qopt_offload cbs = { }; 2533d0bd028SVinicius Costa Gomes const struct net_device_ops *ops; 2543d0bd028SVinicius Costa Gomes int err; 2553d0bd028SVinicius Costa Gomes 2563d0bd028SVinicius Costa Gomes if (!q->offload) 2573d0bd028SVinicius Costa Gomes return; 2583d0bd028SVinicius Costa Gomes 2593d0bd028SVinicius Costa Gomes q->enqueue = cbs_enqueue_soft; 2603d0bd028SVinicius Costa Gomes q->dequeue = cbs_dequeue_soft; 2613d0bd028SVinicius Costa Gomes 2623d0bd028SVinicius Costa Gomes ops = dev->netdev_ops; 2633d0bd028SVinicius Costa Gomes if (!ops->ndo_setup_tc) 2643d0bd028SVinicius Costa Gomes return; 2653d0bd028SVinicius Costa Gomes 2663d0bd028SVinicius Costa Gomes cbs.queue = q->queue; 2673d0bd028SVinicius Costa Gomes cbs.enable = 0; 2683d0bd028SVinicius Costa Gomes 2698521db4cSNogah Frankel err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_CBS, &cbs); 2703d0bd028SVinicius Costa Gomes if (err < 0) 2713d0bd028SVinicius Costa Gomes pr_warn("Couldn't disable CBS offload for queue %d\n", 2723d0bd028SVinicius Costa Gomes cbs.queue); 2733d0bd028SVinicius Costa Gomes } 2743d0bd028SVinicius Costa Gomes 2753d0bd028SVinicius Costa Gomes static int cbs_enable_offload(struct net_device *dev, struct cbs_sched_data *q, 276710fb396SAlexander Aring const struct tc_cbs_qopt *opt, 277710fb396SAlexander Aring struct netlink_ext_ack *extack) 2783d0bd028SVinicius Costa Gomes { 2793d0bd028SVinicius Costa Gomes const struct net_device_ops *ops = dev->netdev_ops; 2803d0bd028SVinicius Costa Gomes struct tc_cbs_qopt_offload cbs = { }; 2813d0bd028SVinicius Costa Gomes int err; 2823d0bd028SVinicius Costa Gomes 283710fb396SAlexander Aring if (!ops->ndo_setup_tc) { 284710fb396SAlexander Aring NL_SET_ERR_MSG(extack, "Specified device does not support cbs offload"); 2853d0bd028SVinicius Costa Gomes return -EOPNOTSUPP; 286710fb396SAlexander Aring } 2873d0bd028SVinicius Costa Gomes 2883d0bd028SVinicius Costa Gomes cbs.queue = q->queue; 2893d0bd028SVinicius Costa Gomes 2903d0bd028SVinicius Costa Gomes cbs.enable = 1; 2913d0bd028SVinicius Costa Gomes cbs.hicredit = opt->hicredit; 2923d0bd028SVinicius Costa Gomes cbs.locredit = opt->locredit; 2933d0bd028SVinicius Costa Gomes cbs.idleslope = opt->idleslope; 2943d0bd028SVinicius Costa Gomes cbs.sendslope = opt->sendslope; 2953d0bd028SVinicius Costa Gomes 2968521db4cSNogah Frankel err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_CBS, &cbs); 297710fb396SAlexander Aring if (err < 0) { 298710fb396SAlexander Aring NL_SET_ERR_MSG(extack, "Specified device failed to setup cbs hardware offload"); 2993d0bd028SVinicius Costa Gomes return err; 300710fb396SAlexander Aring } 3013d0bd028SVinicius Costa Gomes 3023d0bd028SVinicius Costa Gomes q->enqueue = cbs_enqueue_offload; 3033d0bd028SVinicius Costa Gomes q->dequeue = cbs_dequeue_offload; 3043d0bd028SVinicius Costa Gomes 3053d0bd028SVinicius Costa Gomes return 0; 3063d0bd028SVinicius Costa Gomes } 3073d0bd028SVinicius Costa Gomes 308e0a7683dSLeandro Dorileo static void cbs_set_port_rate(struct net_device *dev, struct cbs_sched_data *q) 309e0a7683dSLeandro Dorileo { 310e0a7683dSLeandro Dorileo struct ethtool_link_ksettings ecmd; 3111c6c09a0SVladimir Oltean int speed = SPEED_10; 3129367fa08SColin Ian King int port_rate; 3131c6c09a0SVladimir Oltean int err; 314e0a7683dSLeandro Dorileo 3151c6c09a0SVladimir Oltean err = __ethtool_get_link_ksettings(dev, &ecmd); 3161c6c09a0SVladimir Oltean if (err < 0) 3171c6c09a0SVladimir Oltean goto skip; 3181c6c09a0SVladimir Oltean 31983c8c3cfSVladimir Oltean if (ecmd.base.speed && ecmd.base.speed != SPEED_UNKNOWN) 3201c6c09a0SVladimir Oltean speed = ecmd.base.speed; 3211c6c09a0SVladimir Oltean 3221c6c09a0SVladimir Oltean skip: 3231c6c09a0SVladimir Oltean port_rate = speed * 1000 * BYTES_PER_KBIT; 324e0a7683dSLeandro Dorileo 325e0a7683dSLeandro Dorileo atomic64_set(&q->port_rate, port_rate); 326e0a7683dSLeandro Dorileo netdev_dbg(dev, "cbs: set %s's port_rate to: %lld, linkspeed: %d\n", 327e0a7683dSLeandro Dorileo dev->name, (long long)atomic64_read(&q->port_rate), 328e0a7683dSLeandro Dorileo ecmd.base.speed); 329e0a7683dSLeandro Dorileo } 330e0a7683dSLeandro Dorileo 331e0a7683dSLeandro Dorileo static int cbs_dev_notifier(struct notifier_block *nb, unsigned long event, 332e0a7683dSLeandro Dorileo void *ptr) 333e0a7683dSLeandro Dorileo { 334e0a7683dSLeandro Dorileo struct net_device *dev = netdev_notifier_info_to_dev(ptr); 335e0a7683dSLeandro Dorileo struct cbs_sched_data *q; 336e0a7683dSLeandro Dorileo struct net_device *qdev; 337e0a7683dSLeandro Dorileo bool found = false; 338e0a7683dSLeandro Dorileo 339e0a7683dSLeandro Dorileo ASSERT_RTNL(); 340e0a7683dSLeandro Dorileo 341e0a7683dSLeandro Dorileo if (event != NETDEV_UP && event != NETDEV_CHANGE) 342e0a7683dSLeandro Dorileo return NOTIFY_DONE; 343e0a7683dSLeandro Dorileo 344e0a7683dSLeandro Dorileo spin_lock(&cbs_list_lock); 345e0a7683dSLeandro Dorileo list_for_each_entry(q, &cbs_list, cbs_list) { 346e0a7683dSLeandro Dorileo qdev = qdisc_dev(q->qdisc); 347e0a7683dSLeandro Dorileo if (qdev == dev) { 348e0a7683dSLeandro Dorileo found = true; 349e0a7683dSLeandro Dorileo break; 350e0a7683dSLeandro Dorileo } 351e0a7683dSLeandro Dorileo } 352e0a7683dSLeandro Dorileo spin_unlock(&cbs_list_lock); 353e0a7683dSLeandro Dorileo 354e0a7683dSLeandro Dorileo if (found) 355e0a7683dSLeandro Dorileo cbs_set_port_rate(dev, q); 356e0a7683dSLeandro Dorileo 357e0a7683dSLeandro Dorileo return NOTIFY_DONE; 358e0a7683dSLeandro Dorileo } 359e0a7683dSLeandro Dorileo 3602030721cSAlexander Aring static int cbs_change(struct Qdisc *sch, struct nlattr *opt, 3612030721cSAlexander Aring struct netlink_ext_ack *extack) 362585d763aSVinicius Costa Gomes { 363585d763aSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 364585d763aSVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 365585d763aSVinicius Costa Gomes struct nlattr *tb[TCA_CBS_MAX + 1]; 366585d763aSVinicius Costa Gomes struct tc_cbs_qopt *qopt; 367585d763aSVinicius Costa Gomes int err; 368585d763aSVinicius Costa Gomes 3698cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_CBS_MAX, opt, cbs_policy, 3708cb08174SJohannes Berg extack); 371585d763aSVinicius Costa Gomes if (err < 0) 372585d763aSVinicius Costa Gomes return err; 373585d763aSVinicius Costa Gomes 374710fb396SAlexander Aring if (!tb[TCA_CBS_PARMS]) { 375710fb396SAlexander Aring NL_SET_ERR_MSG(extack, "Missing CBS parameter which are mandatory"); 376585d763aSVinicius Costa Gomes return -EINVAL; 377710fb396SAlexander Aring } 378585d763aSVinicius Costa Gomes 379585d763aSVinicius Costa Gomes qopt = nla_data(tb[TCA_CBS_PARMS]); 380585d763aSVinicius Costa Gomes 3813d0bd028SVinicius Costa Gomes if (!qopt->offload) { 382e0a7683dSLeandro Dorileo cbs_set_port_rate(dev, q); 3833d0bd028SVinicius Costa Gomes cbs_disable_offload(dev, q); 3843d0bd028SVinicius Costa Gomes } else { 385710fb396SAlexander Aring err = cbs_enable_offload(dev, q, qopt, extack); 3863d0bd028SVinicius Costa Gomes if (err < 0) 3873d0bd028SVinicius Costa Gomes return err; 3883d0bd028SVinicius Costa Gomes } 389585d763aSVinicius Costa Gomes 3903d0bd028SVinicius Costa Gomes /* Everything went OK, save the parameters used. */ 391585d763aSVinicius Costa Gomes q->hicredit = qopt->hicredit; 392585d763aSVinicius Costa Gomes q->locredit = qopt->locredit; 393585d763aSVinicius Costa Gomes q->idleslope = qopt->idleslope * BYTES_PER_KBIT; 394585d763aSVinicius Costa Gomes q->sendslope = qopt->sendslope * BYTES_PER_KBIT; 3953d0bd028SVinicius Costa Gomes q->offload = qopt->offload; 396585d763aSVinicius Costa Gomes 397585d763aSVinicius Costa Gomes return 0; 398585d763aSVinicius Costa Gomes } 399585d763aSVinicius Costa Gomes 400e63d7dfdSAlexander Aring static int cbs_init(struct Qdisc *sch, struct nlattr *opt, 401e63d7dfdSAlexander Aring struct netlink_ext_ack *extack) 402585d763aSVinicius Costa Gomes { 403585d763aSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 4043d0bd028SVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 405585d763aSVinicius Costa Gomes 406710fb396SAlexander Aring if (!opt) { 407710fb396SAlexander Aring NL_SET_ERR_MSG(extack, "Missing CBS qdisc options which are mandatory"); 408585d763aSVinicius Costa Gomes return -EINVAL; 409710fb396SAlexander Aring } 410585d763aSVinicius Costa Gomes 411990e35ecSVinicius Costa Gomes q->qdisc = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, 412990e35ecSVinicius Costa Gomes sch->handle, extack); 413990e35ecSVinicius Costa Gomes if (!q->qdisc) 414990e35ecSVinicius Costa Gomes return -ENOMEM; 415990e35ecSVinicius Costa Gomes 4163e8b9bfaSVinicius Costa Gomes spin_lock(&cbs_list_lock); 4173e8b9bfaSVinicius Costa Gomes list_add(&q->cbs_list, &cbs_list); 4183e8b9bfaSVinicius Costa Gomes spin_unlock(&cbs_list_lock); 4193e8b9bfaSVinicius Costa Gomes 420990e35ecSVinicius Costa Gomes qdisc_hash_add(q->qdisc, false); 421990e35ecSVinicius Costa Gomes 4223d0bd028SVinicius Costa Gomes q->queue = sch->dev_queue - netdev_get_tx_queue(dev, 0); 4233d0bd028SVinicius Costa Gomes 4243d0bd028SVinicius Costa Gomes q->enqueue = cbs_enqueue_soft; 4253d0bd028SVinicius Costa Gomes q->dequeue = cbs_dequeue_soft; 4263d0bd028SVinicius Costa Gomes 427585d763aSVinicius Costa Gomes qdisc_watchdog_init(&q->watchdog, sch); 428585d763aSVinicius Costa Gomes 4293e8b9bfaSVinicius Costa Gomes return cbs_change(sch, opt, extack); 430585d763aSVinicius Costa Gomes } 431585d763aSVinicius Costa Gomes 432585d763aSVinicius Costa Gomes static void cbs_destroy(struct Qdisc *sch) 433585d763aSVinicius Costa Gomes { 434585d763aSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 4353d0bd028SVinicius Costa Gomes struct net_device *dev = qdisc_dev(sch); 436585d763aSVinicius Costa Gomes 4373e8b9bfaSVinicius Costa Gomes /* Nothing to do if we couldn't create the underlying qdisc */ 4383e8b9bfaSVinicius Costa Gomes if (!q->qdisc) 4393e8b9bfaSVinicius Costa Gomes return; 4403d0bd028SVinicius Costa Gomes 441e0a7683dSLeandro Dorileo qdisc_watchdog_cancel(&q->watchdog); 4423d0bd028SVinicius Costa Gomes cbs_disable_offload(dev, q); 443990e35ecSVinicius Costa Gomes 4443e8b9bfaSVinicius Costa Gomes spin_lock(&cbs_list_lock); 4453e8b9bfaSVinicius Costa Gomes list_del(&q->cbs_list); 4463e8b9bfaSVinicius Costa Gomes spin_unlock(&cbs_list_lock); 4473e8b9bfaSVinicius Costa Gomes 44886bd446bSVlad Buslov qdisc_put(q->qdisc); 449585d763aSVinicius Costa Gomes } 450585d763aSVinicius Costa Gomes 451585d763aSVinicius Costa Gomes static int cbs_dump(struct Qdisc *sch, struct sk_buff *skb) 452585d763aSVinicius Costa Gomes { 453585d763aSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 454585d763aSVinicius Costa Gomes struct tc_cbs_qopt opt = { }; 455585d763aSVinicius Costa Gomes struct nlattr *nest; 456585d763aSVinicius Costa Gomes 457ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 458585d763aSVinicius Costa Gomes if (!nest) 459585d763aSVinicius Costa Gomes goto nla_put_failure; 460585d763aSVinicius Costa Gomes 461585d763aSVinicius Costa Gomes opt.hicredit = q->hicredit; 462585d763aSVinicius Costa Gomes opt.locredit = q->locredit; 463585d763aSVinicius Costa Gomes opt.sendslope = div64_s64(q->sendslope, BYTES_PER_KBIT); 464585d763aSVinicius Costa Gomes opt.idleslope = div64_s64(q->idleslope, BYTES_PER_KBIT); 4653d0bd028SVinicius Costa Gomes opt.offload = q->offload; 466585d763aSVinicius Costa Gomes 467585d763aSVinicius Costa Gomes if (nla_put(skb, TCA_CBS_PARMS, sizeof(opt), &opt)) 468585d763aSVinicius Costa Gomes goto nla_put_failure; 469585d763aSVinicius Costa Gomes 470585d763aSVinicius Costa Gomes return nla_nest_end(skb, nest); 471585d763aSVinicius Costa Gomes 472585d763aSVinicius Costa Gomes nla_put_failure: 473585d763aSVinicius Costa Gomes nla_nest_cancel(skb, nest); 474585d763aSVinicius Costa Gomes return -1; 475585d763aSVinicius Costa Gomes } 476585d763aSVinicius Costa Gomes 477990e35ecSVinicius Costa Gomes static int cbs_dump_class(struct Qdisc *sch, unsigned long cl, 478990e35ecSVinicius Costa Gomes struct sk_buff *skb, struct tcmsg *tcm) 479990e35ecSVinicius Costa Gomes { 480990e35ecSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 481990e35ecSVinicius Costa Gomes 482990e35ecSVinicius Costa Gomes if (cl != 1 || !q->qdisc) /* only one class */ 483990e35ecSVinicius Costa Gomes return -ENOENT; 484990e35ecSVinicius Costa Gomes 485990e35ecSVinicius Costa Gomes tcm->tcm_handle |= TC_H_MIN(1); 486990e35ecSVinicius Costa Gomes tcm->tcm_info = q->qdisc->handle; 487990e35ecSVinicius Costa Gomes 488990e35ecSVinicius Costa Gomes return 0; 489990e35ecSVinicius Costa Gomes } 490990e35ecSVinicius Costa Gomes 491990e35ecSVinicius Costa Gomes static int cbs_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, 492990e35ecSVinicius Costa Gomes struct Qdisc **old, struct netlink_ext_ack *extack) 493990e35ecSVinicius Costa Gomes { 494990e35ecSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 495990e35ecSVinicius Costa Gomes 496990e35ecSVinicius Costa Gomes if (!new) { 497990e35ecSVinicius Costa Gomes new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, 498990e35ecSVinicius Costa Gomes sch->handle, NULL); 499990e35ecSVinicius Costa Gomes if (!new) 500990e35ecSVinicius Costa Gomes new = &noop_qdisc; 501990e35ecSVinicius Costa Gomes } 502990e35ecSVinicius Costa Gomes 503990e35ecSVinicius Costa Gomes *old = qdisc_replace(sch, new, &q->qdisc); 504990e35ecSVinicius Costa Gomes return 0; 505990e35ecSVinicius Costa Gomes } 506990e35ecSVinicius Costa Gomes 507990e35ecSVinicius Costa Gomes static struct Qdisc *cbs_leaf(struct Qdisc *sch, unsigned long arg) 508990e35ecSVinicius Costa Gomes { 509990e35ecSVinicius Costa Gomes struct cbs_sched_data *q = qdisc_priv(sch); 510990e35ecSVinicius Costa Gomes 511990e35ecSVinicius Costa Gomes return q->qdisc; 512990e35ecSVinicius Costa Gomes } 513990e35ecSVinicius Costa Gomes 514990e35ecSVinicius Costa Gomes static unsigned long cbs_find(struct Qdisc *sch, u32 classid) 515990e35ecSVinicius Costa Gomes { 516990e35ecSVinicius Costa Gomes return 1; 517990e35ecSVinicius Costa Gomes } 518990e35ecSVinicius Costa Gomes 519990e35ecSVinicius Costa Gomes static void cbs_walk(struct Qdisc *sch, struct qdisc_walker *walker) 520990e35ecSVinicius Costa Gomes { 521990e35ecSVinicius Costa Gomes if (!walker->stop) { 522990e35ecSVinicius Costa Gomes if (walker->count >= walker->skip) { 523990e35ecSVinicius Costa Gomes if (walker->fn(sch, 1, walker) < 0) { 524990e35ecSVinicius Costa Gomes walker->stop = 1; 525990e35ecSVinicius Costa Gomes return; 526990e35ecSVinicius Costa Gomes } 527990e35ecSVinicius Costa Gomes } 528990e35ecSVinicius Costa Gomes walker->count++; 529990e35ecSVinicius Costa Gomes } 530990e35ecSVinicius Costa Gomes } 531990e35ecSVinicius Costa Gomes 532990e35ecSVinicius Costa Gomes static const struct Qdisc_class_ops cbs_class_ops = { 533990e35ecSVinicius Costa Gomes .graft = cbs_graft, 534990e35ecSVinicius Costa Gomes .leaf = cbs_leaf, 535990e35ecSVinicius Costa Gomes .find = cbs_find, 536990e35ecSVinicius Costa Gomes .walk = cbs_walk, 537990e35ecSVinicius Costa Gomes .dump = cbs_dump_class, 538990e35ecSVinicius Costa Gomes }; 539990e35ecSVinicius Costa Gomes 540585d763aSVinicius Costa Gomes static struct Qdisc_ops cbs_qdisc_ops __read_mostly = { 541585d763aSVinicius Costa Gomes .id = "cbs", 542990e35ecSVinicius Costa Gomes .cl_ops = &cbs_class_ops, 543585d763aSVinicius Costa Gomes .priv_size = sizeof(struct cbs_sched_data), 544585d763aSVinicius Costa Gomes .enqueue = cbs_enqueue, 545585d763aSVinicius Costa Gomes .dequeue = cbs_dequeue, 546585d763aSVinicius Costa Gomes .peek = qdisc_peek_dequeued, 547585d763aSVinicius Costa Gomes .init = cbs_init, 548585d763aSVinicius Costa Gomes .reset = qdisc_reset_queue, 549585d763aSVinicius Costa Gomes .destroy = cbs_destroy, 550585d763aSVinicius Costa Gomes .change = cbs_change, 551585d763aSVinicius Costa Gomes .dump = cbs_dump, 552585d763aSVinicius Costa Gomes .owner = THIS_MODULE, 553585d763aSVinicius Costa Gomes }; 554585d763aSVinicius Costa Gomes 555e0a7683dSLeandro Dorileo static struct notifier_block cbs_device_notifier = { 556e0a7683dSLeandro Dorileo .notifier_call = cbs_dev_notifier, 557e0a7683dSLeandro Dorileo }; 558e0a7683dSLeandro Dorileo 559585d763aSVinicius Costa Gomes static int __init cbs_module_init(void) 560585d763aSVinicius Costa Gomes { 56145d5cb13SYueHaibing int err; 562e0a7683dSLeandro Dorileo 56345d5cb13SYueHaibing err = register_netdevice_notifier(&cbs_device_notifier); 564e0a7683dSLeandro Dorileo if (err) 565e0a7683dSLeandro Dorileo return err; 566e0a7683dSLeandro Dorileo 56745d5cb13SYueHaibing err = register_qdisc(&cbs_qdisc_ops); 56845d5cb13SYueHaibing if (err) 56945d5cb13SYueHaibing unregister_netdevice_notifier(&cbs_device_notifier); 57045d5cb13SYueHaibing 57145d5cb13SYueHaibing return err; 572585d763aSVinicius Costa Gomes } 573585d763aSVinicius Costa Gomes 574585d763aSVinicius Costa Gomes static void __exit cbs_module_exit(void) 575585d763aSVinicius Costa Gomes { 576585d763aSVinicius Costa Gomes unregister_qdisc(&cbs_qdisc_ops); 577e0a7683dSLeandro Dorileo unregister_netdevice_notifier(&cbs_device_notifier); 578585d763aSVinicius Costa Gomes } 579585d763aSVinicius Costa Gomes module_init(cbs_module_init) 580585d763aSVinicius Costa Gomes module_exit(cbs_module_exit) 581585d763aSVinicius Costa Gomes MODULE_LICENSE("GPL"); 582