100 	sch->q.qlen++;  in cbs_child_enqueue()
108 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_enqueue_offload()  local
109 	struct Qdisc *qdisc = q->qdisc;  in cbs_enqueue_offload()
117 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_enqueue_soft()  local
118 	struct Qdisc *qdisc = q->qdisc;  in cbs_enqueue_soft()
120 	if (sch->q.qlen == 0 && q->credits > 0) {  in cbs_enqueue_soft()
124 		q->credits = 0;  in cbs_enqueue_soft()
125 		q->last = ktime_get_ns();  in cbs_enqueue_soft()
134 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_enqueue()  local
136 	return q->enqueue(skb, sch, to_free);  in cbs_enqueue()
171 	sch->q.qlen--;  in cbs_child_dequeue()
178 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_dequeue_soft()  local
179 	struct Qdisc *qdisc = q->qdisc;  in cbs_dequeue_soft()
186 	if (now < q->last) {  in cbs_dequeue_soft()
187 		qdisc_watchdog_schedule_ns(&q->watchdog, q->last);  in cbs_dequeue_soft()
190 	if (q->credits < 0) {  in cbs_dequeue_soft()
191 		credits = timediff_to_credits(now - q->last, q->idleslope);  in cbs_dequeue_soft()
193 		credits = q->credits + credits;  in cbs_dequeue_soft()
194 		q->credits = min_t(s64, credits, q->hicredit);  in cbs_dequeue_soft()
196 		if (q->credits < 0) {  in cbs_dequeue_soft()
199 			delay = delay_from_credits(q->credits, q->idleslope);  in cbs_dequeue_soft()
200 			qdisc_watchdog_schedule_ns(&q->watchdog, now + delay);  in cbs_dequeue_soft()
202 			q->last = now;  in cbs_dequeue_soft()
216 	credits = credits_from_len(len, q->sendslope,  in cbs_dequeue_soft()
217 				   atomic64_read(&q->port_rate));  in cbs_dequeue_soft()
218 	credits += q->credits;  in cbs_dequeue_soft()
220 	q->credits = max_t(s64, credits, q->locredit);  in cbs_dequeue_soft()
222 	if (unlikely(atomic64_read(&q->port_rate) == 0))  in cbs_dequeue_soft()
223 		q->last = now;  in cbs_dequeue_soft()
225 		q->last = now + div64_s64(len * NSEC_PER_SEC,  in cbs_dequeue_soft()
226 					  atomic64_read(&q->port_rate));  in cbs_dequeue_soft()
233 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_dequeue_offload()  local
234 	struct Qdisc *qdisc = q->qdisc;  in cbs_dequeue_offload()
241 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_dequeue()  local
243 	return q->dequeue(sch);  in cbs_dequeue()
251 				struct cbs_sched_data *q)  in cbs_disable_offload()  argument
257 	if (!q->offload)  in cbs_disable_offload()
260 	q->enqueue = cbs_enqueue_soft;  in cbs_disable_offload()
261 	q->dequeue = cbs_dequeue_soft;  in cbs_disable_offload()
267 	cbs.queue = q->queue;  in cbs_disable_offload()
276 static int cbs_enable_offload(struct net_device *dev, struct cbs_sched_data *q,  in cbs_enable_offload()  argument
289 	cbs.queue = q->queue;  in cbs_enable_offload()
303 	q->enqueue = cbs_enqueue_offload;  in cbs_enable_offload()
304 	q->dequeue = cbs_dequeue_offload;  in cbs_enable_offload()
309 static void cbs_set_port_rate(struct net_device *dev, struct cbs_sched_data *q)  in cbs_set_port_rate()  argument
326 	atomic64_set(&q->port_rate, port_rate);  in cbs_set_port_rate()
328 		   dev->name, (long long)atomic64_read(&q->port_rate),  in cbs_set_port_rate()
336 	struct cbs_sched_data *q;  in cbs_dev_notifier()  local
346 	list_for_each_entry(q, &cbs_list, cbs_list) {  in cbs_dev_notifier()
347 		qdev = qdisc_dev(q->qdisc);  in cbs_dev_notifier()
356 		cbs_set_port_rate(dev, q);  in cbs_dev_notifier()
364 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_change()  local
383 		cbs_set_port_rate(dev, q);  in cbs_change()
384 		cbs_disable_offload(dev, q);  in cbs_change()
386 		err = cbs_enable_offload(dev, q, qopt, extack);  in cbs_change()
392 	q->hicredit = qopt->hicredit;  in cbs_change()
393 	q->locredit = qopt->locredit;  in cbs_change()
394 	q->idleslope = qopt->idleslope * BYTES_PER_KBIT;  in cbs_change()
395 	q->sendslope = qopt->sendslope * BYTES_PER_KBIT;  in cbs_change()
396 	q->offload = qopt->offload;  in cbs_change()
404 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_init()  local
412 	q->qdisc = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,  in cbs_init()
414 	if (!q->qdisc)  in cbs_init()
418 	list_add(&q->cbs_list, &cbs_list);  in cbs_init()
421 	qdisc_hash_add(q->qdisc, false);  in cbs_init()
423 	q->queue = sch->dev_queue - netdev_get_tx_queue(dev, 0);  in cbs_init()
425 	q->enqueue = cbs_enqueue_soft;  in cbs_init()
426 	q->dequeue = cbs_dequeue_soft;  in cbs_init()
428 	qdisc_watchdog_init(&q->watchdog, sch);  in cbs_init()
435 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_destroy()  local
439 	if (!q->qdisc)  in cbs_destroy()
442 	qdisc_watchdog_cancel(&q->watchdog);  in cbs_destroy()
443 	cbs_disable_offload(dev, q);  in cbs_destroy()
446 	list_del(&q->cbs_list);  in cbs_destroy()
449 	qdisc_put(q->qdisc);  in cbs_destroy()
454 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_dump()  local
462 	opt.hicredit = q->hicredit;  in cbs_dump()
463 	opt.locredit = q->locredit;  in cbs_dump()
464 	opt.sendslope = div64_s64(q->sendslope, BYTES_PER_KBIT);  in cbs_dump()
465 	opt.idleslope = div64_s64(q->idleslope, BYTES_PER_KBIT);  in cbs_dump()
466 	opt.offload = q->offload;  in cbs_dump()
481 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_dump_class()  local
483 	if (cl != 1 || !q->qdisc)	/* only one class */  in cbs_dump_class()
487 	tcm->tcm_info = q->qdisc->handle;  in cbs_dump_class()
495 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_graft()  local
504 	*old = qdisc_replace(sch, new, &q->qdisc);  in cbs_graft()
510 	struct cbs_sched_data *q = qdisc_priv(sch);  in cbs_leaf()  local
512 	return q->qdisc;  in cbs_leaf()