466 	struct kyber_hctx_data *khd;  in kyber_init_hctx()  local
469 	khd = kmalloc_node(sizeof(*khd), GFP_KERNEL, hctx->numa_node);  in kyber_init_hctx()
470 	if (!khd)  in kyber_init_hctx()
473 	khd->kcqs = kmalloc_array_node(hctx->nr_ctx,  in kyber_init_hctx()
476 	if (!khd->kcqs)  in kyber_init_hctx()
480 		kyber_ctx_queue_init(&khd->kcqs[i]);  in kyber_init_hctx()
483 		if (sbitmap_init_node(&khd->kcq_map[i], hctx->nr_ctx,  in kyber_init_hctx()
487 				sbitmap_free(&khd->kcq_map[i]);  in kyber_init_hctx()
492 	spin_lock_init(&khd->lock);  in kyber_init_hctx()
495 		INIT_LIST_HEAD(&khd->rqs[i]);  in kyber_init_hctx()
496 		khd->domain_wait[i].sbq = NULL;  in kyber_init_hctx()
497 		init_waitqueue_func_entry(&khd->domain_wait[i].wait,  in kyber_init_hctx()
499 		khd->domain_wait[i].wait.private = hctx;  in kyber_init_hctx()
500 		INIT_LIST_HEAD(&khd->domain_wait[i].wait.entry);  in kyber_init_hctx()
501 		atomic_set(&khd->wait_index[i], 0);  in kyber_init_hctx()
504 	khd->cur_domain = 0;  in kyber_init_hctx()
505 	khd->batching = 0;  in kyber_init_hctx()
507 	hctx->sched_data = khd;  in kyber_init_hctx()
513 	kfree(khd->kcqs);  in kyber_init_hctx()
515 	kfree(khd);  in kyber_init_hctx()
521 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_exit_hctx()  local
525 		sbitmap_free(&khd->kcq_map[i]);  in kyber_exit_hctx()
526 	kfree(khd->kcqs);  in kyber_exit_hctx()
572 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_bio_merge()  local
573 	struct kyber_ctx_queue *kcq = &khd->kcqs[ctx->index_hw[hctx->type]];  in kyber_bio_merge()
594 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_insert_requests()  local
599 		struct kyber_ctx_queue *kcq = &khd->kcqs[rq->mq_ctx->index_hw[hctx->type]];  in kyber_insert_requests()
608 		sbitmap_set_bit(&khd->kcq_map[sched_domain],  in kyber_insert_requests()
662 	struct kyber_hctx_data *khd;  member
670 	struct kyber_ctx_queue *kcq = &flush_data->khd->kcqs[bitnr];  in flush_busy_kcq()
681 static void kyber_flush_busy_kcqs(struct kyber_hctx_data *khd,  in kyber_flush_busy_kcqs()  argument
686 		.khd = khd,  in kyber_flush_busy_kcqs()
691 	sbitmap_for_each_set(&khd->kcq_map[sched_domain],  in kyber_flush_busy_kcqs()
707 				  struct kyber_hctx_data *khd,  in kyber_get_domain_token()  argument
710 	unsigned int sched_domain = khd->cur_domain;  in kyber_get_domain_token()
712 	struct sbq_wait *wait = &khd->domain_wait[sched_domain];  in kyber_get_domain_token()
725 				  &khd->wait_index[sched_domain]);  in kyber_get_domain_token()
726 		khd->domain_ws[sched_domain] = ws;  in kyber_get_domain_token()
744 		ws = khd->domain_ws[sched_domain];  in kyber_get_domain_token()
755 			  struct kyber_hctx_data *khd,  in kyber_dispatch_cur_domain()  argument
762 	rqs = &khd->rqs[khd->cur_domain];  in kyber_dispatch_cur_domain()
774 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
776 			khd->batching++;  in kyber_dispatch_cur_domain()
782 					      kyber_domain_names[khd->cur_domain]);  in kyber_dispatch_cur_domain()
784 	} else if (sbitmap_any_bit_set(&khd->kcq_map[khd->cur_domain])) {  in kyber_dispatch_cur_domain()
785 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
787 			kyber_flush_busy_kcqs(khd, khd->cur_domain, rqs);  in kyber_dispatch_cur_domain()
789 			khd->batching++;  in kyber_dispatch_cur_domain()
795 					      kyber_domain_names[khd->cur_domain]);  in kyber_dispatch_cur_domain()
806 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_dispatch_request()  local
810 	spin_lock(&khd->lock);  in kyber_dispatch_request()
816 	if (khd->batching < kyber_batch_size[khd->cur_domain]) {  in kyber_dispatch_request()
817 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
831 	khd->batching = 0;  in kyber_dispatch_request()
833 		if (khd->cur_domain == KYBER_NUM_DOMAINS - 1)  in kyber_dispatch_request()
834 			khd->cur_domain = 0;  in kyber_dispatch_request()
836 			khd->cur_domain++;  in kyber_dispatch_request()
838 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
845 	spin_unlock(&khd->lock);  in kyber_dispatch_request()
851 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_has_work()  local
855 		if (!list_empty_careful(&khd->rqs[i]) ||  in kyber_has_work()
856 		    sbitmap_any_bit_set(&khd->kcq_map[i]))  in kyber_has_work()
911 	__acquires(&khd->lock)						\
914 	struct kyber_hctx_data *khd = hctx->sched_data;			\
916 	spin_lock(&khd->lock);						\
917 	return seq_list_start(&khd->rqs[domain], *pos);			\
924 	struct kyber_hctx_data *khd = hctx->sched_data;			\
926 	return seq_list_next(v, &khd->rqs[domain], pos);		\
930 	__releases(&khd->lock)						\
933 	struct kyber_hctx_data *khd = hctx->sched_data;			\
935 	spin_unlock(&khd->lock);					\
948 	struct kyber_hctx_data *khd = hctx->sched_data;			\
949 	wait_queue_entry_t *wait = &khd->domain_wait[domain].wait;	\
972 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_cur_domain_show()  local
974 	seq_printf(m, "%s\n", kyber_domain_names[khd->cur_domain]);  in kyber_cur_domain_show()
981 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_batching_show()  local
983 	seq_printf(m, "%u\n", khd->batching);  in kyber_batching_show()