95 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_disable_timer()  argument
101 	if (list_empty(&queue->timer_list.list))  in __rpc_disable_timer()
102 		cancel_delayed_work(&queue->timer_list.dwork);  in __rpc_disable_timer()
106 rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)  in rpc_set_queue_timer()  argument
109 	queue->timer_list.expires = expires;  in rpc_set_queue_timer()
114 	mod_delayed_work(rpciod_workqueue, &queue->timer_list.dwork, expires);  in rpc_set_queue_timer()
121 __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task,  in __rpc_add_timer()  argument
125 	if (list_empty(&queue->timer_list.list) || time_before(timeout, queue->timer_list.expires))  in __rpc_add_timer()
126 		rpc_set_queue_timer(queue, timeout);  in __rpc_add_timer()
127 	list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);  in __rpc_add_timer()
130 static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)  in rpc_set_waitqueue_priority()  argument
132 	if (queue->priority != priority) {  in rpc_set_waitqueue_priority()
133 		queue->priority = priority;  in rpc_set_waitqueue_priority()
134 		queue->nr = 1U << priority;  in rpc_set_waitqueue_priority()
138 static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)  in rpc_reset_waitqueue_priority()  argument
140 	rpc_set_waitqueue_priority(queue, queue->maxpriority);  in rpc_reset_waitqueue_priority()
193 static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,  in __rpc_add_wait_queue_priority()  argument
197 	if (unlikely(queue_priority > queue->maxpriority))  in __rpc_add_wait_queue_priority()
198 		queue_priority = queue->maxpriority;  in __rpc_add_wait_queue_priority()
199 	__rpc_list_enqueue_task(&queue->tasks[queue_priority], task);  in __rpc_add_wait_queue_priority()
205 static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,  in __rpc_add_wait_queue()  argument
210 	if (RPC_IS_PRIORITY(queue))  in __rpc_add_wait_queue()
211 		__rpc_add_wait_queue_priority(queue, task, queue_priority);  in __rpc_add_wait_queue()
213 		list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
214 	task->tk_waitqueue = queue;  in __rpc_add_wait_queue()
215 	queue->qlen++;  in __rpc_add_wait_queue()
233 static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_remove_wait_queue()  argument
235 	__rpc_disable_timer(queue, task);  in __rpc_remove_wait_queue()
236 	if (RPC_IS_PRIORITY(queue))  in __rpc_remove_wait_queue()
240 	queue->qlen--;  in __rpc_remove_wait_queue()
243 static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigne…  in __rpc_init_priority_wait_queue()  argument
247 	spin_lock_init(&queue->lock);  in __rpc_init_priority_wait_queue()
248 	for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)  in __rpc_init_priority_wait_queue()
249 		INIT_LIST_HEAD(&queue->tasks[i]);  in __rpc_init_priority_wait_queue()
250 	queue->maxpriority = nr_queues - 1;  in __rpc_init_priority_wait_queue()
251 	rpc_reset_waitqueue_priority(queue);  in __rpc_init_priority_wait_queue()
252 	queue->qlen = 0;  in __rpc_init_priority_wait_queue()
253 	queue->timer_list.expires = 0;  in __rpc_init_priority_wait_queue()
254 	INIT_DELAYED_WORK(&queue->timer_list.dwork, __rpc_queue_timer_fn);  in __rpc_init_priority_wait_queue()
255 	INIT_LIST_HEAD(&queue->timer_list.list);  in __rpc_init_priority_wait_queue()
256 	rpc_assign_waitqueue_name(queue, qname);  in __rpc_init_priority_wait_queue()
259 void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)  in rpc_init_priority_wait_queue()  argument
261 	__rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);  in rpc_init_priority_wait_queue()
265 void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)  in rpc_init_wait_queue()  argument
267 	__rpc_init_priority_wait_queue(queue, qname, 1);  in rpc_init_wait_queue()
271 void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)  in rpc_destroy_wait_queue()  argument
273 	cancel_delayed_work_sync(&queue->timer_list.dwork);  in rpc_destroy_wait_queue()
509 		struct rpc_wait_queue *queue,  in __rpc_do_wake_up_task_on_wq()  argument
518 	trace_rpc_task_wakeup(task, queue);  in __rpc_do_wake_up_task_on_wq()
520 	__rpc_remove_wait_queue(queue, task);  in __rpc_do_wake_up_task_on_wq()
530 		struct rpc_wait_queue *queue, struct rpc_task *task,  in rpc_wake_up_task_on_wq_queue_action_locked()  argument
535 		if (task->tk_waitqueue == queue) {  in rpc_wake_up_task_on_wq_queue_action_locked()
537 				__rpc_do_wake_up_task_on_wq(wq, queue, task);  in rpc_wake_up_task_on_wq_queue_action_locked()
548 static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue,  in rpc_wake_up_task_queue_locked()  argument
551 	rpc_wake_up_task_on_wq_queue_action_locked(rpciod_workqueue, queue,  in rpc_wake_up_task_queue_locked()
558 void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_queued_task()  argument
562 	spin_lock(&queue->lock);  in rpc_wake_up_queued_task()
563 	rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_queued_task()
564 	spin_unlock(&queue->lock);  in rpc_wake_up_queued_task()
575 rpc_wake_up_task_queue_set_status_locked(struct rpc_wait_queue *queue,  in rpc_wake_up_task_queue_set_status_locked()  argument
578 	rpc_wake_up_task_on_wq_queue_action_locked(rpciod_workqueue, queue,  in rpc_wake_up_task_queue_set_status_locked()
592 rpc_wake_up_queued_task_set_status(struct rpc_wait_queue *queue,  in rpc_wake_up_queued_task_set_status()  argument
597 	spin_lock(&queue->lock);  in rpc_wake_up_queued_task_set_status()
598 	rpc_wake_up_task_queue_set_status_locked(queue, task, status);  in rpc_wake_up_queued_task_set_status()
599 	spin_unlock(&queue->lock);  in rpc_wake_up_queued_task_set_status()
605 static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue)  in __rpc_find_next_queued_priority()  argument
613 	q = &queue->tasks[RPC_NR_PRIORITY - 1];  in __rpc_find_next_queued_priority()
614 	if (queue->maxpriority > RPC_PRIORITY_PRIVILEGED && !list_empty(q)) {  in __rpc_find_next_queued_priority()
622 	q = &queue->tasks[queue->priority];  in __rpc_find_next_queued_priority()
623 	if (!list_empty(q) && queue->nr) {  in __rpc_find_next_queued_priority()
624 		queue->nr--;  in __rpc_find_next_queued_priority()
633 		if (q == &queue->tasks[0])  in __rpc_find_next_queued_priority()
634 			q = &queue->tasks[queue->maxpriority];  in __rpc_find_next_queued_priority()
641 	} while (q != &queue->tasks[queue->priority]);  in __rpc_find_next_queued_priority()
643 	rpc_reset_waitqueue_priority(queue);  in __rpc_find_next_queued_priority()
647 	rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));  in __rpc_find_next_queued_priority()
652 static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue)  in __rpc_find_next_queued()  argument
654 	if (RPC_IS_PRIORITY(queue))  in __rpc_find_next_queued()
655 		return __rpc_find_next_queued_priority(queue);  in __rpc_find_next_queued()
656 	if (!list_empty(&queue->tasks[0]))  in __rpc_find_next_queued()
657 		return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued()
665 		struct rpc_wait_queue *queue,  in rpc_wake_up_first_on_wq()  argument
670 	spin_lock(&queue->lock);  in rpc_wake_up_first_on_wq()
671 	task = __rpc_find_next_queued(queue);  in rpc_wake_up_first_on_wq()
673 		task = rpc_wake_up_task_on_wq_queue_action_locked(wq, queue,  in rpc_wake_up_first_on_wq()
675 	spin_unlock(&queue->lock);  in rpc_wake_up_first_on_wq()
683 struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue,  in rpc_wake_up_first()  argument
686 	return rpc_wake_up_first_on_wq(rpciod_workqueue, queue, func, data);  in rpc_wake_up_first()
698 struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue)  in rpc_wake_up_next()  argument
700 	return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL);  in rpc_wake_up_next()
709 static void rpc_wake_up_locked(struct rpc_wait_queue *queue)  in rpc_wake_up_locked()  argument
714 		task = __rpc_find_next_queued(queue);  in rpc_wake_up_locked()
717 		rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_locked()
727 void rpc_wake_up(struct rpc_wait_queue *queue)  in rpc_wake_up()  argument
729 	spin_lock(&queue->lock);  in rpc_wake_up()
730 	rpc_wake_up_locked(queue);  in rpc_wake_up()
731 	spin_unlock(&queue->lock);  in rpc_wake_up()
740 static void rpc_wake_up_status_locked(struct rpc_wait_queue *queue, int status)  in rpc_wake_up_status_locked()  argument
745 		task = __rpc_find_next_queued(queue);  in rpc_wake_up_status_locked()
748 		rpc_wake_up_task_queue_set_status_locked(queue, task, status);  in rpc_wake_up_status_locked()
759 void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)  in rpc_wake_up_status()  argument
761 	spin_lock(&queue->lock);  in rpc_wake_up_status()
762 	rpc_wake_up_status_locked(queue, status);  in rpc_wake_up_status()
763 	spin_unlock(&queue->lock);  in rpc_wake_up_status()
769 	struct rpc_wait_queue *queue = container_of(work,  in __rpc_queue_timer_fn()  local
775 	spin_lock(&queue->lock);  in __rpc_queue_timer_fn()
777 	list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {  in __rpc_queue_timer_fn()
782 			rpc_wake_up_task_queue_locked(queue, task);  in __rpc_queue_timer_fn()
788 	if (!list_empty(&queue->timer_list.list))  in __rpc_queue_timer_fn()
789 		rpc_set_queue_timer(queue, expires);  in __rpc_queue_timer_fn()
790 	spin_unlock(&queue->lock);  in __rpc_queue_timer_fn()
859 	struct rpc_wait_queue *queue;  in rpc_signal_task()  local
869 	queue = READ_ONCE(task->tk_waitqueue);  in rpc_signal_task()
870 	if (queue)  in rpc_signal_task()
871 		rpc_wake_up_queued_task(queue, task);  in rpc_signal_task()
876 	struct rpc_wait_queue *queue;  in rpc_task_try_cancel()  local
880 	queue = READ_ONCE(task->tk_waitqueue);  in rpc_task_try_cancel()
881 	if (queue)  in rpc_task_try_cancel()
882 		rpc_wake_up_queued_task(queue, task);  in rpc_task_try_cancel()
913 	struct rpc_wait_queue *queue;  in __rpc_execute()  local
968 		queue = task->tk_waitqueue;  in __rpc_execute()
969 		spin_lock(&queue->lock);  in __rpc_execute()
971 			spin_unlock(&queue->lock);  in __rpc_execute()
976 			rpc_wake_up_task_queue_locked(queue, task);  in __rpc_execute()
977 			spin_unlock(&queue->lock);  in __rpc_execute()
981 		spin_unlock(&queue->lock);  in __rpc_execute()