32 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_class_idx()  local
34 	return bfqq ? bfqq->ioprio_class - 1 :  in bfq_class_idx()
272 	struct bfq_queue *bfqq = NULL;  in bfq_entity_to_bfqq()  local
275 		bfqq = container_of(entity, struct bfq_queue, entity);  in bfq_entity_to_bfqq()
277 	return bfqq;  in bfq_entity_to_bfqq()
298 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_calc_finish()  local
303 	if (bfqq) {  in bfq_calc_finish()
304 		bfq_log_bfqq(bfqq->bfqd, bfqq,  in bfq_calc_finish()
307 		bfq_log_bfqq(bfqq->bfqd, bfqq,  in bfq_calc_finish()
352 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_idle_extract()  local
367 	if (bfqq)  in bfq_idle_extract()
368 		list_del(&bfqq->bfqq_list);  in bfq_idle_extract()
483 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_active_insert()  local
495 	if (bfqq)  in bfq_active_insert()
496 		list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list[bfqq->actuator_idx]);  in bfq_active_insert()
526 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_get_entity()  local
528 	if (bfqq) {  in bfq_get_entity()
529 		bfqq->ref++;  in bfq_get_entity()
530 		bfq_log_bfqq(bfqq->bfqd, bfqq, "get_entity: %p %d",  in bfq_get_entity()
531 			     bfqq, bfqq->ref);  in bfq_get_entity()
573 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_active_extract()  local
581 	if (bfqq)  in bfq_active_extract()
582 		list_del(&bfqq->bfqq_list);  in bfq_active_extract()
595 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_idle_insert()  local
606 	if (bfqq)  in bfq_idle_insert()
607 		list_add(&bfqq->bfqq_list, &bfqq->bfqd->idle_list);  in bfq_idle_insert()
629 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_forget_entity()  local
633 	if (bfqq && !is_in_service)  in bfq_forget_entity()
634 		bfq_put_queue(bfqq);  in bfq_forget_entity()
708 		struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in __bfq_entity_update_weight_prio()  local
726 			if (bfqq)  in __bfq_entity_update_weight_prio()
727 				bfqq->ioprio =  in __bfq_entity_update_weight_prio()
731 		if (bfqq && update_class_too)  in __bfq_entity_update_weight_prio()
732 			bfqq->ioprio_class = bfqq->new_ioprio_class;  in __bfq_entity_update_weight_prio()
738 		if (!bfqq || bfqq->ioprio_class == bfqq->new_ioprio_class)  in __bfq_entity_update_weight_prio()
752 			     (bfqq ? bfqq->wr_coeff : 1);  in __bfq_entity_update_weight_prio()
758 		if (prev_weight != new_weight && bfqq)  in __bfq_entity_update_weight_prio()
759 			bfq_weights_tree_remove(bfqq);  in __bfq_entity_update_weight_prio()
765 		if (prev_weight != new_weight && bfqq && bfqq->wr_coeff == 1)  in __bfq_entity_update_weight_prio()
766 			bfq_weights_tree_add(bfqq);  in __bfq_entity_update_weight_prio()
787 void bfq_bfqq_served(struct bfq_queue *bfqq, int served)  in bfq_bfqq_served()  argument
789 	struct bfq_entity *entity = &bfqq->entity;  in bfq_bfqq_served()
792 	if (!bfqq->service_from_backlogged)  in bfq_bfqq_served()
793 		bfqq->first_IO_time = jiffies;  in bfq_bfqq_served()
795 	if (bfqq->wr_coeff > 1)  in bfq_bfqq_served()
796 		bfqq->service_from_wr += served;  in bfq_bfqq_served()
798 	bfqq->service_from_backlogged += served;  in bfq_bfqq_served()
807 	bfq_log_bfqq(bfqq->bfqd, bfqq, "bfqq_served %d secs", served);  in bfq_bfqq_served()
837 void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_bfqq_charge_time()  argument
840 	struct bfq_entity *entity = &bfqq->entity;  in bfq_bfqq_charge_time()
851 	bfq_bfqq_served(bfqq,  in bfq_bfqq_charge_time()
859 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);  in bfq_update_fin_time_enqueue()  local
905 		if (bfqq)  in bfq_update_fin_time_enqueue()
906 			delta /= bfqq->wr_coeff;  in bfq_update_fin_time_enqueue()
1458 	struct bfq_queue *bfqq;  in bfq_get_next_queue()  local
1530 	bfqq = bfq_entity_to_bfqq(entity);  in bfq_get_next_queue()
1543 	return bfqq;  in bfq_get_next_queue()
1587 void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_deactivate_bfqq()  argument
1590 	struct bfq_entity *entity = &bfqq->entity;  in bfq_deactivate_bfqq()
1595 void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_activate_bfqq()  argument
1597 	struct bfq_entity *entity = &bfqq->entity;  in bfq_activate_bfqq()
1599 	bfq_activate_requeue_entity(entity, bfq_bfqq_non_blocking_wait_rq(bfqq),  in bfq_activate_bfqq()
1601 	bfq_clear_bfqq_non_blocking_wait_rq(bfqq);  in bfq_activate_bfqq()
1604 void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_requeue_bfqq()  argument
1607 	struct bfq_entity *entity = &bfqq->entity;  in bfq_requeue_bfqq()
1610 				    bfqq == bfqd->in_service_queue, expiration);  in bfq_requeue_bfqq()
1613 void bfq_add_bfqq_in_groups_with_pending_reqs(struct bfq_queue *bfqq)  in bfq_add_bfqq_in_groups_with_pending_reqs()  argument
1616 	struct bfq_entity *entity = &bfqq->entity;  in bfq_add_bfqq_in_groups_with_pending_reqs()
1620 		if (!(bfqq_group(bfqq)->num_queues_with_pending_reqs++))  in bfq_add_bfqq_in_groups_with_pending_reqs()
1621 			bfqq->bfqd->num_groups_with_pending_reqs++;  in bfq_add_bfqq_in_groups_with_pending_reqs()
1626 void bfq_del_bfqq_in_groups_with_pending_reqs(struct bfq_queue *bfqq)  in bfq_del_bfqq_in_groups_with_pending_reqs()  argument
1629 	struct bfq_entity *entity = &bfqq->entity;  in bfq_del_bfqq_in_groups_with_pending_reqs()
1633 		if (!(--bfqq_group(bfqq)->num_queues_with_pending_reqs))  in bfq_del_bfqq_in_groups_with_pending_reqs()
1634 			bfqq->bfqd->num_groups_with_pending_reqs--;  in bfq_del_bfqq_in_groups_with_pending_reqs()
1644 void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration)  in bfq_del_bfqq_busy()  argument
1646 	struct bfq_data *bfqd = bfqq->bfqd;  in bfq_del_bfqq_busy()
1648 	bfq_log_bfqq(bfqd, bfqq, "del from busy");  in bfq_del_bfqq_busy()
1650 	bfq_clear_bfqq_busy(bfqq);  in bfq_del_bfqq_busy()
1652 	bfqd->busy_queues[bfqq->ioprio_class - 1]--;  in bfq_del_bfqq_busy()
1654 	if (bfqq->wr_coeff > 1)  in bfq_del_bfqq_busy()
1657 	bfqg_stats_update_dequeue(bfqq_group(bfqq));  in bfq_del_bfqq_busy()
1659 	bfq_deactivate_bfqq(bfqd, bfqq, true, expiration);  in bfq_del_bfqq_busy()
1661 	if (!bfqq->dispatched) {  in bfq_del_bfqq_busy()
1662 		bfq_del_bfqq_in_groups_with_pending_reqs(bfqq);  in bfq_del_bfqq_busy()
1667 		bfq_weights_tree_remove(bfqq);  in bfq_del_bfqq_busy()
1674 void bfq_add_bfqq_busy(struct bfq_queue *bfqq)  in bfq_add_bfqq_busy()  argument
1676 	struct bfq_data *bfqd = bfqq->bfqd;  in bfq_add_bfqq_busy()
1678 	bfq_log_bfqq(bfqd, bfqq, "add to busy");  in bfq_add_bfqq_busy()
1680 	bfq_activate_bfqq(bfqd, bfqq);  in bfq_add_bfqq_busy()
1682 	bfq_mark_bfqq_busy(bfqq);  in bfq_add_bfqq_busy()
1683 	bfqd->busy_queues[bfqq->ioprio_class - 1]++;  in bfq_add_bfqq_busy()
1685 	if (!bfqq->dispatched) {  in bfq_add_bfqq_busy()
1686 		bfq_add_bfqq_in_groups_with_pending_reqs(bfqq);  in bfq_add_bfqq_busy()
1687 		if (bfqq->wr_coeff == 1)  in bfq_add_bfqq_busy()
1688 			bfq_weights_tree_add(bfqq);  in bfq_add_bfqq_busy()
1691 	if (bfqq->wr_coeff > 1)  in bfq_add_bfqq_busy()
1695 	if (!hlist_unhashed(&bfqq->woken_list_node) &&  in bfq_add_bfqq_busy()
1696 	    &bfqq->woken_list_node != bfqq->waker_bfqq->woken_list.first) {  in bfq_add_bfqq_busy()
1697 		hlist_del_init(&bfqq->woken_list_node);  in bfq_add_bfqq_busy()
1698 		hlist_add_head(&bfqq->woken_list_node,  in bfq_add_bfqq_busy()
1699 			       &bfqq->waker_bfqq->woken_list);  in bfq_add_bfqq_busy()