74 static unsigned int fq_pie_hash(const struct fq_pie_sched_data *q,  in fq_pie_hash()  argument
77 	return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);  in fq_pie_hash()
83 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_classify()  local
90 	    TC_H_MIN(skb->priority) <= q->flows_cnt)  in fq_pie_classify()
93 	filter = rcu_dereference_bh(q->filter_list);  in fq_pie_classify()
95 		return fq_pie_hash(q, skb) + 1;  in fq_pie_classify()
111 		if (TC_H_MIN(res.classid) <= q->flows_cnt)  in fq_pie_classify()
132 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_qdisc_enqueue()  local
150 	sel_flow = &q->flows[idx];  in fq_pie_qdisc_enqueue()
153 	memory_limited = q->memory_usage > q->memory_limit + skb->truesize;  in fq_pie_qdisc_enqueue()
157 		q->stats.overlimit++;  in fq_pie_qdisc_enqueue()
160 		q->overmemory++;  in fq_pie_qdisc_enqueue()
163 	if (!pie_drop_early(sch, &q->p_params, &sel_flow->vars,  in fq_pie_qdisc_enqueue()
166 	} else if (q->p_params.ecn &&  in fq_pie_qdisc_enqueue()
167 		   sel_flow->vars.prob <= (MAX_PROB / 100) * q->ecn_prob &&  in fq_pie_qdisc_enqueue()
172 		q->stats.ecn_mark++;  in fq_pie_qdisc_enqueue()
177 		if (!q->p_params.dq_rate_estimator)  in fq_pie_qdisc_enqueue()
181 		q->stats.packets_in++;  in fq_pie_qdisc_enqueue()
182 		q->memory_usage += skb->truesize;  in fq_pie_qdisc_enqueue()
184 		sch->q.qlen++;  in fq_pie_qdisc_enqueue()
187 			list_add_tail(&sel_flow->flowchain, &q->new_flows);  in fq_pie_qdisc_enqueue()
188 			q->new_flow_count++;  in fq_pie_qdisc_enqueue()
189 			sel_flow->deficit = q->quantum;  in fq_pie_qdisc_enqueue()
198 	q->stats.dropped++;  in fq_pie_qdisc_enqueue()
237 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_qdisc_dequeue()  local
244 	head = &q->new_flows;  in fq_pie_qdisc_dequeue()
246 		head = &q->old_flows;  in fq_pie_qdisc_dequeue()
254 		flow->deficit += q->quantum;  in fq_pie_qdisc_dequeue()
255 		list_move_tail(&flow->flowchain, &q->old_flows);  in fq_pie_qdisc_dequeue()
263 		sch->q.qlen--;  in fq_pie_qdisc_dequeue()
269 		if (head == &q->new_flows && !list_empty(&q->old_flows))  in fq_pie_qdisc_dequeue()
270 			list_move_tail(&flow->flowchain, &q->old_flows);  in fq_pie_qdisc_dequeue()
279 	q->memory_usage -= get_pie_cb(skb)->mem_usage;  in fq_pie_qdisc_dequeue()
280 	pie_process_dequeue(skb, &q->p_params, &flow->vars, flow->backlog);  in fq_pie_qdisc_dequeue()
287 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_change()  local
301 		q->p_params.limit = limit;  in fq_pie_change()
305 		if (q->flows) {  in fq_pie_change()
310 		q->flows_cnt = nla_get_u32(tb[TCA_FQ_PIE_FLOWS]);  in fq_pie_change()
311 		if (!q->flows_cnt || q->flows_cnt > 65536) {  in fq_pie_change()
324 		q->p_params.target =  in fq_pie_change()
330 		q->p_params.tupdate =  in fq_pie_change()
334 		q->p_params.alpha = nla_get_u32(tb[TCA_FQ_PIE_ALPHA]);  in fq_pie_change()
337 		q->p_params.beta = nla_get_u32(tb[TCA_FQ_PIE_BETA]);  in fq_pie_change()
340 		q->quantum = nla_get_u32(tb[TCA_FQ_PIE_QUANTUM]);  in fq_pie_change()
343 		q->memory_limit = nla_get_u32(tb[TCA_FQ_PIE_MEMORY_LIMIT]);  in fq_pie_change()
346 		q->ecn_prob = nla_get_u32(tb[TCA_FQ_PIE_ECN_PROB]);  in fq_pie_change()
349 		q->p_params.ecn = nla_get_u32(tb[TCA_FQ_PIE_ECN]);  in fq_pie_change()
352 		q->p_params.bytemode = nla_get_u32(tb[TCA_FQ_PIE_BYTEMODE]);  in fq_pie_change()
355 		q->p_params.dq_rate_estimator =  in fq_pie_change()
359 	while (sch->q.qlen > sch->limit) {  in fq_pie_change()
378 	struct fq_pie_sched_data *q = from_timer(q, t, adapt_timer);  in fq_pie_timer()  local
380 	struct Qdisc *sch = q->sch;  in fq_pie_timer()
389 	max_cnt = min_t(int, q->flows_cnt - q->flows_cursor, 2048);  in fq_pie_timer()
391 		pie_calculate_probability(&q->p_params,  in fq_pie_timer()
392 					  &q->flows[q->flows_cursor].vars,  in fq_pie_timer()
393 					  q->flows[q->flows_cursor].backlog);  in fq_pie_timer()
394 		q->flows_cursor++;  in fq_pie_timer()
397 	tupdate = q->p_params.tupdate;  in fq_pie_timer()
399 	if (q->flows_cursor >= q->flows_cnt) {  in fq_pie_timer()
400 		q->flows_cursor = 0;  in fq_pie_timer()
404 		mod_timer(&q->adapt_timer, jiffies + next);  in fq_pie_timer()
412 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_init()  local
416 	pie_params_init(&q->p_params);  in fq_pie_init()
418 	q->p_params.limit = sch->limit;  in fq_pie_init()
419 	q->quantum = psched_mtu(qdisc_dev(sch));  in fq_pie_init()
420 	q->sch = sch;  in fq_pie_init()
421 	q->ecn_prob = 10;  in fq_pie_init()
422 	q->flows_cnt = 1024;  in fq_pie_init()
423 	q->memory_limit = SZ_32M;  in fq_pie_init()
425 	INIT_LIST_HEAD(&q->new_flows);  in fq_pie_init()
426 	INIT_LIST_HEAD(&q->old_flows);  in fq_pie_init()
427 	timer_setup(&q->adapt_timer, fq_pie_timer, 0);  in fq_pie_init()
436 	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);  in fq_pie_init()
440 	q->flows = kvcalloc(q->flows_cnt, sizeof(struct fq_pie_flow),  in fq_pie_init()
442 	if (!q->flows) {  in fq_pie_init()
446 	for (idx = 0; idx < q->flows_cnt; idx++) {  in fq_pie_init()
447 		struct fq_pie_flow *flow = q->flows + idx;  in fq_pie_init()
453 	mod_timer(&q->adapt_timer, jiffies + HZ / 2);  in fq_pie_init()
458 	q->flows_cnt = 0;  in fq_pie_init()
465 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_dump()  local
474 	    nla_put_u32(skb, TCA_FQ_PIE_FLOWS, q->flows_cnt) ||  in fq_pie_dump()
476 			((u32)PSCHED_TICKS2NS(q->p_params.target)) /  in fq_pie_dump()
479 			jiffies_to_usecs(q->p_params.tupdate)) ||  in fq_pie_dump()
480 	    nla_put_u32(skb, TCA_FQ_PIE_ALPHA, q->p_params.alpha) ||  in fq_pie_dump()
481 	    nla_put_u32(skb, TCA_FQ_PIE_BETA, q->p_params.beta) ||  in fq_pie_dump()
482 	    nla_put_u32(skb, TCA_FQ_PIE_QUANTUM, q->quantum) ||  in fq_pie_dump()
483 	    nla_put_u32(skb, TCA_FQ_PIE_MEMORY_LIMIT, q->memory_limit) ||  in fq_pie_dump()
484 	    nla_put_u32(skb, TCA_FQ_PIE_ECN_PROB, q->ecn_prob) ||  in fq_pie_dump()
485 	    nla_put_u32(skb, TCA_FQ_PIE_ECN, q->p_params.ecn) ||  in fq_pie_dump()
486 	    nla_put_u32(skb, TCA_FQ_PIE_BYTEMODE, q->p_params.bytemode) ||  in fq_pie_dump()
488 			q->p_params.dq_rate_estimator))  in fq_pie_dump()
500 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_dump_stats()  local
502 		.packets_in	= q->stats.packets_in,  in fq_pie_dump_stats()
503 		.overlimit	= q->stats.overlimit,  in fq_pie_dump_stats()
504 		.overmemory	= q->overmemory,  in fq_pie_dump_stats()
505 		.dropped	= q->stats.dropped,  in fq_pie_dump_stats()
506 		.ecn_mark	= q->stats.ecn_mark,  in fq_pie_dump_stats()
507 		.new_flow_count = q->new_flow_count,  in fq_pie_dump_stats()
508 		.memory_usage   = q->memory_usage,  in fq_pie_dump_stats()
513 	list_for_each(pos, &q->new_flows)  in fq_pie_dump_stats()
516 	list_for_each(pos, &q->old_flows)  in fq_pie_dump_stats()
525 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_reset()  local
528 	INIT_LIST_HEAD(&q->new_flows);  in fq_pie_reset()
529 	INIT_LIST_HEAD(&q->old_flows);  in fq_pie_reset()
530 	for (idx = 0; idx < q->flows_cnt; idx++) {  in fq_pie_reset()
531 		struct fq_pie_flow *flow = q->flows + idx;  in fq_pie_reset()
544 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_destroy()  local
546 	tcf_block_put(q->block);  in fq_pie_destroy()
547 	q->p_params.tupdate = 0;  in fq_pie_destroy()
548 	del_timer_sync(&q->adapt_timer);  in fq_pie_destroy()
549 	kvfree(q->flows);  in fq_pie_destroy()