270 static void q_init(struct queue *q, struct entry_space *es, unsigned int nr_levels)  in q_init()  argument
274 	q->es = es;  in q_init()
275 	q->nr_elts = 0;  in q_init()
276 	q->nr_levels = nr_levels;  in q_init()
278 	for (i = 0; i < q->nr_levels; i++) {  in q_init()
279 		l_init(q->qs + i);  in q_init()
280 		q->target_count[i] = 0u;  in q_init()
283 	q->last_target_nr_elts = 0u;  in q_init()
284 	q->nr_top_levels = 0u;  in q_init()
285 	q->nr_in_top_levels = 0u;  in q_init()
288 static unsigned int q_size(struct queue *q)  in q_size()  argument
290 	return q->nr_elts;  in q_size()
296 static void q_push(struct queue *q, struct entry *e)  in q_push()  argument
301 		q->nr_elts++;  in q_push()
303 	l_add_tail(q->es, q->qs + e->level, e);  in q_push()
306 static void q_push_front(struct queue *q, struct entry *e)  in q_push_front()  argument
311 		q->nr_elts++;  in q_push_front()
313 	l_add_head(q->es, q->qs + e->level, e);  in q_push_front()
316 static void q_push_before(struct queue *q, struct entry *old, struct entry *e)  in q_push_before()  argument
321 		q->nr_elts++;  in q_push_before()
323 	l_add_before(q->es, q->qs + e->level, old, e);  in q_push_before()
326 static void q_del(struct queue *q, struct entry *e)  in q_del()  argument
328 	l_del(q->es, q->qs + e->level, e);  in q_del()
330 		q->nr_elts--;  in q_del()
336 static struct entry *q_peek(struct queue *q, unsigned int max_level, bool can_cross_sentinel)  in q_peek()  argument
341 	max_level = min(max_level, q->nr_levels);  in q_peek()
344 		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e)) {  in q_peek()
358 static struct entry *q_pop(struct queue *q)  in q_pop()  argument
360 	struct entry *e = q_peek(q, q->nr_levels, true);  in q_pop()
363 		q_del(q, e);  in q_pop()
373 static struct entry *__redist_pop_from(struct queue *q, unsigned int level)  in __redist_pop_from()  argument
377 	for (; level < q->nr_levels; level++)  in __redist_pop_from()
378 		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e))  in __redist_pop_from()
380 				l_del(q->es, q->qs + e->level, e);  in __redist_pop_from()
387 static void q_set_targets_subrange_(struct queue *q, unsigned int nr_elts,  in q_set_targets_subrange_()  argument
393 	BUG_ON(lend > q->nr_levels);  in q_set_targets_subrange_()
399 		q->target_count[level] =  in q_set_targets_subrange_()
407 static void q_set_targets(struct queue *q)  in q_set_targets()  argument
409 	if (q->last_target_nr_elts == q->nr_elts)  in q_set_targets()
412 	q->last_target_nr_elts = q->nr_elts;  in q_set_targets()
414 	if (q->nr_top_levels > q->nr_levels)  in q_set_targets()
415 		q_set_targets_subrange_(q, q->nr_elts, 0, q->nr_levels);  in q_set_targets()
418 		q_set_targets_subrange_(q, q->nr_in_top_levels,  in q_set_targets()
419 					q->nr_levels - q->nr_top_levels, q->nr_levels);  in q_set_targets()
421 		if (q->nr_in_top_levels < q->nr_elts)  in q_set_targets()
422 			q_set_targets_subrange_(q, q->nr_elts - q->nr_in_top_levels,  in q_set_targets()
423 						0, q->nr_levels - q->nr_top_levels);  in q_set_targets()
425 			q_set_targets_subrange_(q, 0, 0, q->nr_levels - q->nr_top_levels);  in q_set_targets()
429 static void q_redistribute(struct queue *q)  in q_redistribute()  argument
435 	q_set_targets(q);  in q_redistribute()
437 	for (level = 0u; level < q->nr_levels - 1u; level++) {  in q_redistribute()
438 		l = q->qs + level;  in q_redistribute()
439 		target = q->target_count[level];  in q_redistribute()
445 			e = __redist_pop_from(q, level + 1u);  in q_redistribute()
452 			l_add_tail(q->es, l, e);  in q_redistribute()
458 		l_above = q->qs + level + 1u;  in q_redistribute()
460 			e = l_pop_tail(q->es, l);  in q_redistribute()
467 			l_add_tail(q->es, l_above, e);  in q_redistribute()
472 static void q_requeue(struct queue *q, struct entry *e, unsigned int extra_levels,  in q_requeue()  argument
477 	unsigned int new_level = min(q->nr_levels - 1u, e->level + extra_levels);  in q_requeue()
480 	if (extra_levels && (e->level < q->nr_levels - 1u)) {  in q_requeue()
481 		for (de = l_head(q->es, q->qs + new_level); de && de->sentinel; de = l_next(q->es, de))  in q_requeue()
485 			q_del(q, de);  in q_requeue()
490 					q_push_before(q, s1, de);  in q_requeue()
494 					q_push_before(q, s2, de);  in q_requeue()
498 					q_push(q, de);  in q_requeue()
501 				q_push(q, de);  in q_requeue()
505 	q_del(q, e);  in q_requeue()
507 	q_push(q, e);  in q_requeue()
889 	struct queue *q = &mq->dirty;  in __update_writeback_sentinels()  local
892 	for (level = 0; level < q->nr_levels; level++) {  in __update_writeback_sentinels()
894 		q_del(q, sentinel);  in __update_writeback_sentinels()
895 		q_push(q, sentinel);  in __update_writeback_sentinels()
902 	struct queue *q = &mq->clean;  in __update_demote_sentinels()  local
905 	for (level = 0; level < q->nr_levels; level++) {  in __update_demote_sentinels()
907 		q_del(q, sentinel);  in __update_demote_sentinels()
908 		q_push(q, sentinel);  in __update_demote_sentinels()