144 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_offload_change()  local
154 	qopt.replace_params.rate = q->rate;  in tbf_offload_change()
155 	qopt.replace_params.max_size = q->max_size;  in tbf_offload_change()
208 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_segment()  local
224 		ret = qdisc_enqueue(segs, q->qdisc, to_free);  in tbf_segment()
232 	sch->q.qlen += nb;  in tbf_segment()
242 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_enqueue()  local
246 	if (qdisc_pkt_len(skb) > q->max_size) {  in tbf_enqueue()
248 		    skb_gso_validate_mac_len(skb, q->max_size))  in tbf_enqueue()
252 	ret = qdisc_enqueue(skb, q->qdisc, to_free);  in tbf_enqueue()
260 	sch->q.qlen++;  in tbf_enqueue()
264 static bool tbf_peak_present(const struct tbf_sched_data *q)  in tbf_peak_present()  argument
266 	return q->peak.rate_bytes_ps;  in tbf_peak_present()
271 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dequeue()  local
274 	skb = q->qdisc->ops->peek(q->qdisc);  in tbf_dequeue()
283 		toks = min_t(s64, now - q->t_c, q->buffer);  in tbf_dequeue()
285 		if (tbf_peak_present(q)) {  in tbf_dequeue()
286 			ptoks = toks + q->ptokens;  in tbf_dequeue()
287 			if (ptoks > q->mtu)  in tbf_dequeue()
288 				ptoks = q->mtu;  in tbf_dequeue()
289 			ptoks -= (s64) psched_l2t_ns(&q->peak, len);  in tbf_dequeue()
291 		toks += q->tokens;  in tbf_dequeue()
292 		if (toks > q->buffer)  in tbf_dequeue()
293 			toks = q->buffer;  in tbf_dequeue()
294 		toks -= (s64) psched_l2t_ns(&q->rate, len);  in tbf_dequeue()
297 			skb = qdisc_dequeue_peeked(q->qdisc);  in tbf_dequeue()
301 			q->t_c = now;  in tbf_dequeue()
302 			q->tokens = toks;  in tbf_dequeue()
303 			q->ptokens = ptoks;  in tbf_dequeue()
305 			sch->q.qlen--;  in tbf_dequeue()
310 		qdisc_watchdog_schedule_ns(&q->watchdog,  in tbf_dequeue()
331 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_reset()  local
333 	qdisc_reset(q->qdisc);  in tbf_reset()
334 	q->t_c = ktime_get_ns();  in tbf_reset()
335 	q->tokens = q->buffer;  in tbf_reset()
336 	q->ptokens = q->mtu;  in tbf_reset()
337 	qdisc_watchdog_cancel(&q->watchdog);  in tbf_reset()
354 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_change()  local
431 	if (q->qdisc != &noop_qdisc) {  in tbf_change()
432 		err = fifo_set_limit(q->qdisc, qopt->limit);  in tbf_change()
449 		qdisc_tree_flush_backlog(q->qdisc);  in tbf_change()
450 		old = q->qdisc;  in tbf_change()
451 		q->qdisc = child;  in tbf_change()
453 	q->limit = qopt->limit;  in tbf_change()
455 		q->mtu = mtu;  in tbf_change()
457 		q->mtu = PSCHED_TICKS2NS(qopt->mtu);  in tbf_change()
458 	q->max_size = max_size;  in tbf_change()
460 		q->buffer = buffer;  in tbf_change()
462 		q->buffer = PSCHED_TICKS2NS(qopt->buffer);  in tbf_change()
463 	q->tokens = q->buffer;  in tbf_change()
464 	q->ptokens = q->mtu;  in tbf_change()
466 	memcpy(&q->rate, &rate, sizeof(struct psched_ratecfg));  in tbf_change()
467 	memcpy(&q->peak, &peak, sizeof(struct psched_ratecfg));  in tbf_change()
481 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_init()  local
483 	qdisc_watchdog_init(&q->watchdog, sch);  in tbf_init()
484 	q->qdisc = &noop_qdisc;  in tbf_init()
489 	q->t_c = ktime_get_ns();  in tbf_init()
496 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_destroy()  local
498 	qdisc_watchdog_cancel(&q->watchdog);  in tbf_destroy()
500 	qdisc_put(q->qdisc);  in tbf_destroy()
505 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dump()  local
518 	opt.limit = q->limit;  in tbf_dump()
519 	psched_ratecfg_getrate(&opt.rate, &q->rate);  in tbf_dump()
520 	if (tbf_peak_present(q))  in tbf_dump()
521 		psched_ratecfg_getrate(&opt.peakrate, &q->peak);  in tbf_dump()
524 	opt.mtu = PSCHED_NS2TICKS(q->mtu);  in tbf_dump()
525 	opt.buffer = PSCHED_NS2TICKS(q->buffer);  in tbf_dump()
528 	if (q->rate.rate_bytes_ps >= (1ULL << 32) &&  in tbf_dump()
529 	    nla_put_u64_64bit(skb, TCA_TBF_RATE64, q->rate.rate_bytes_ps,  in tbf_dump()
532 	if (tbf_peak_present(q) &&  in tbf_dump()
533 	    q->peak.rate_bytes_ps >= (1ULL << 32) &&  in tbf_dump()
534 	    nla_put_u64_64bit(skb, TCA_TBF_PRATE64, q->peak.rate_bytes_ps,  in tbf_dump()
548 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dump_class()  local
551 	tcm->tcm_info = q->qdisc->handle;  in tbf_dump_class()
559 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_graft()  local
564 	*old = qdisc_replace(sch, new, &q->qdisc);  in tbf_graft()
572 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_leaf()  local
573 	return q->qdisc;  in tbf_leaf()