87 	struct dd_per_prio per_prio[DD_PRIO_COUNT];  member
117 deadline_rb_root(struct dd_per_prio *per_prio, struct request *rq)  in deadline_rb_root()  argument
119 	return &per_prio->sort_list[rq_data_dir(rq)];  in deadline_rb_root()
163 static inline struct request *deadline_from_pos(struct dd_per_prio *per_prio,  in deadline_from_pos()  argument
166 	struct rb_node *node = per_prio->sort_list[data_dir].rb_node;  in deadline_from_pos()
194 deadline_add_rq_rb(struct dd_per_prio *per_prio, struct request *rq)  in deadline_add_rq_rb()  argument
196 	struct rb_root *root = deadline_rb_root(per_prio, rq);  in deadline_add_rq_rb()
202 deadline_del_rq_rb(struct dd_per_prio *per_prio, struct request *rq)  in deadline_del_rq_rb()  argument
204 	elv_rb_del(deadline_rb_root(per_prio, rq), rq);  in deadline_del_rq_rb()
211 				    struct dd_per_prio *per_prio,  in deadline_remove_request()  argument
220 		deadline_del_rq_rb(per_prio, rq);  in deadline_remove_request()
233 	struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_request_merged()  local
239 		elv_rb_del(deadline_rb_root(per_prio, req), req);  in dd_request_merged()
240 		deadline_add_rq_rb(per_prio, req);  in dd_request_merged()
256 	dd->per_prio[prio].stats.merged++;  in dd_merged_requests()
273 	deadline_remove_request(q, &dd->per_prio[prio], next);  in dd_merged_requests()
280 deadline_move_request(struct deadline_data *dd, struct dd_per_prio *per_prio,  in deadline_move_request()  argument
286 	deadline_remove_request(rq->q, per_prio, rq);  in deadline_move_request()
292 	const struct io_stats_per_prio *stats = &dd->per_prio[prio].stats;  in dd_queued()
303 static inline bool deadline_check_fifo(struct dd_per_prio *per_prio,  in deadline_check_fifo()  argument
306 	struct request *rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);  in deadline_check_fifo()
346 deadline_fifo_request(struct deadline_data *dd, struct dd_per_prio *per_prio,  in deadline_fifo_request()  argument
352 	if (list_empty(&per_prio->fifo_list[data_dir]))  in deadline_fifo_request()
355 	rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);  in deadline_fifo_request()
367 	list_for_each_entry_safe(rq, next, &per_prio->fifo_list[DD_WRITE],  in deadline_fifo_request()
370 		rb_rq = deadline_from_pos(per_prio, data_dir, blk_rq_pos(rq));  in deadline_fifo_request()
390 deadline_next_request(struct deadline_data *dd, struct dd_per_prio *per_prio,  in deadline_next_request()  argument
396 	rq = deadline_from_pos(per_prio, data_dir,  in deadline_next_request()
397 			       per_prio->latest_pos[data_dir]);  in deadline_next_request()
444 					     struct dd_per_prio *per_prio,  in __dd_dispatch_request()  argument
454 	if (!list_empty(&per_prio->dispatch)) {  in __dd_dispatch_request()
455 		rq = list_first_entry(&per_prio->dispatch, struct request,  in __dd_dispatch_request()
467 	rq = deadline_next_request(dd, per_prio, dd->last_dir);  in __dd_dispatch_request()
479 	if (!list_empty(&per_prio->fifo_list[DD_READ])) {  in __dd_dispatch_request()
480 		BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_READ]));  in __dd_dispatch_request()
482 		if (deadline_fifo_request(dd, per_prio, DD_WRITE) &&  in __dd_dispatch_request()
495 	if (!list_empty(&per_prio->fifo_list[DD_WRITE])) {  in __dd_dispatch_request()
497 		BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_WRITE]));  in __dd_dispatch_request()
512 	next_rq = deadline_next_request(dd, per_prio, data_dir);  in __dd_dispatch_request()
513 	if (deadline_check_fifo(per_prio, data_dir) || !next_rq) {  in __dd_dispatch_request()
519 		rq = deadline_fifo_request(dd, per_prio, data_dir);  in __dd_dispatch_request()
546 	deadline_move_request(dd, per_prio, rq);  in __dd_dispatch_request()
550 	dd->per_prio[prio].latest_pos[data_dir] = blk_rq_pos(rq);  in __dd_dispatch_request()
551 	dd->per_prio[prio].stats.dispatched++;  in __dd_dispatch_request()
579 		rq = __dd_dispatch_request(dd, &dd->per_prio[prio],  in dd_dispatch_prio_aged_requests()
613 		rq = __dd_dispatch_request(dd, &dd->per_prio[prio], now);  in dd_dispatch_request()
682 		struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_exit_sched()  local
683 		const struct io_stats_per_prio *stats = &per_prio->stats;  in dd_exit_sched()
686 		WARN_ON_ONCE(!list_empty(&per_prio->fifo_list[DD_READ]));  in dd_exit_sched()
687 		WARN_ON_ONCE(!list_empty(&per_prio->fifo_list[DD_WRITE]));  in dd_exit_sched()
723 		struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_init_sched()  local
725 		INIT_LIST_HEAD(&per_prio->dispatch);  in dd_init_sched()
726 		INIT_LIST_HEAD(&per_prio->fifo_list[DD_READ]);  in dd_init_sched()
727 		INIT_LIST_HEAD(&per_prio->fifo_list[DD_WRITE]);  in dd_init_sched()
728 		per_prio->sort_list[DD_READ] = RB_ROOT;  in dd_init_sched()
729 		per_prio->sort_list[DD_WRITE] = RB_ROOT;  in dd_init_sched()
762 	struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_request_merge()  local
769 	__rq = elv_rb_find(&per_prio->sort_list[bio_data_dir(bio)], sector);  in dd_request_merge()
816 	struct dd_per_prio *per_prio;  in dd_insert_request()  local
828 	per_prio = &dd->per_prio[prio];  in dd_insert_request()
830 		per_prio->stats.inserted++;  in dd_insert_request()
840 		list_add(&rq->queuelist, &per_prio->dispatch);  in dd_insert_request()
845 		deadline_add_rq_rb(per_prio, rq);  in dd_insert_request()
857 		insert_before = &per_prio->fifo_list[data_dir];  in dd_insert_request()
910 		if (!list_empty_careful(&dd->per_prio[p].fifo_list[DD_WRITE]))  in dd_has_write_work()
938 	struct dd_per_prio *per_prio = &dd->per_prio[prio];  in dd_finish_request()  local
948 	atomic_inc(&per_prio->stats.completed);  in dd_finish_request()
962 static bool dd_has_work_for_prio(struct dd_per_prio *per_prio)  in dd_has_work_for_prio()  argument
964 	return !list_empty_careful(&per_prio->dispatch) ||  in dd_has_work_for_prio()
965 		!list_empty_careful(&per_prio->fifo_list[DD_READ]) ||  in dd_has_work_for_prio()
966 		!list_empty_careful(&per_prio->fifo_list[DD_WRITE]);  in dd_has_work_for_prio()
975 		if (dd_has_work_for_prio(&dd->per_prio[prio]))  in dd_has_work()
1055 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
1058 	return seq_list_start(&per_prio->fifo_list[data_dir], *pos);	\
1066 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
1068 	return seq_list_next(v, &per_prio->fifo_list[data_dir], pos);	\
1092 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
1095 	rq = deadline_from_pos(per_prio, data_dir,			\
1096 			       per_prio->latest_pos[data_dir]);		\
1157 	const struct io_stats_per_prio *stats = &dd->per_prio[prio].stats;  in dd_owned_by_driver()
1189 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
1192 	return seq_list_start(&per_prio->dispatch, *pos);		\
1200 	struct dd_per_prio *per_prio = &dd->per_prio[prio];		\
1202 	return seq_list_next(v, &per_prio->dispatch, pos);		\