16 static inline void closure_put_after_sub(struct closure *cl, int flags)  in closure_put_after_sub()  argument
24 		if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) {  in closure_put_after_sub()
25 			atomic_set(&cl->remaining,  in closure_put_after_sub()
27 			closure_queue(cl);  in closure_put_after_sub()
29 			struct closure *parent = cl->parent;  in closure_put_after_sub()
30 			closure_fn *destructor = cl->fn;  in closure_put_after_sub()
32 			closure_debug_destroy(cl);  in closure_put_after_sub()
35 				destructor(cl);  in closure_put_after_sub()
44 void closure_sub(struct closure *cl, int v)  in closure_sub()  argument
46 	closure_put_after_sub(cl, atomic_sub_return(v, &cl->remaining));  in closure_sub()
52 void closure_put(struct closure *cl)  in closure_put()  argument
54 	closure_put_after_sub(cl, atomic_dec_return(&cl->remaining));  in closure_put()
63 	struct closure *cl, *t;  in __closure_wake_up()  local
72 	llist_for_each_entry_safe(cl, t, reverse, list) {  in __closure_wake_up()
73 		closure_set_waiting(cl, 0);  in __closure_wake_up()
74 		closure_sub(cl, CLOSURE_WAITING + 1);  in __closure_wake_up()
85 bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl)  in closure_wait()  argument
87 	if (atomic_read(&cl->remaining) & CLOSURE_WAITING)  in closure_wait()
90 	closure_set_waiting(cl, _RET_IP_);  in closure_wait()
91 	atomic_add(CLOSURE_WAITING + 1, &cl->remaining);  in closure_wait()
92 	llist_add(&cl->list, &waitlist->list);  in closure_wait()
102 static void closure_sync_fn(struct closure *cl)  in closure_sync_fn()  argument
104 	struct closure_syncer *s = cl->s;  in closure_sync_fn()
114 void __sched __closure_sync(struct closure *cl)  in __closure_sync()  argument
118 	cl->s = &s;  in __closure_sync()
119 	continue_at(cl, closure_sync_fn, NULL);  in __closure_sync()
136 void closure_debug_create(struct closure *cl)  in closure_debug_create()  argument
140 	BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE);  in closure_debug_create()
141 	cl->magic = CLOSURE_MAGIC_ALIVE;  in closure_debug_create()
144 	list_add(&cl->all, &closure_list);  in closure_debug_create()
148 void closure_debug_destroy(struct closure *cl)  in closure_debug_destroy()  argument
152 	BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE);  in closure_debug_destroy()
153 	cl->magic = CLOSURE_MAGIC_DEAD;  in closure_debug_destroy()
156 	list_del(&cl->all);  in closure_debug_destroy()
164 	struct closure *cl;  in debug_show()  local
168 	list_for_each_entry(cl, &closure_list, all) {  in debug_show()
169 		int r = atomic_read(&cl->remaining);  in debug_show()
172 			   cl, (void *) cl->ip, cl->fn, cl->parent,  in debug_show()
177 				    work_data_bits(&cl->work)) ? "Q" : "",  in debug_show()
182 				   (void *) cl->waiting_on);  in debug_show()