63 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_bio_merge()  local
65 	if (e->type->ops.allow_merge)  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
87 		const struct elevator_type *e)  in elv_support_features()  argument
89 	return (q->required_elevator_features & e->elevator_features) ==  in elv_support_features()
100 static bool elevator_match(const struct elevator_type *e, const char *name)  in elevator_match()  argument
102 	return !strcmp(e->elevator_name, name) ||  in elevator_match()
103 		(e->elevator_alias && !strcmp(e->elevator_alias, name));  in elevator_match()
108 	struct elevator_type *e;  in __elevator_find()  local
110 	list_for_each_entry(e, &elv_list, list)  in __elevator_find()
111 		if (elevator_match(e, name))  in __elevator_find()
112 			return e;  in __elevator_find()
119 	struct elevator_type *e;  in elevator_find_get()  local
122 	e = __elevator_find(name);  in elevator_find_get()
123 	if (e && (!elv_support_features(q, e) || !elevator_tryget(e)))  in elevator_find_get()
124 		e = NULL;  in elevator_find_get()
126 	return e;  in elevator_find_get()
132 				  struct elevator_type *e)  in elevator_alloc()  argument
140 	__elevator_get(e);  in elevator_alloc()
141 	eq->type = e;  in elevator_alloc()
152 	struct elevator_queue *e;  in elevator_release()  local
154 	e = container_of(kobj, struct elevator_queue, kobj);  in elevator_release()
155 	elevator_put(e->type);  in elevator_release()
156 	kfree(e);  in elevator_release()
161 	struct elevator_queue *e = q->elevator;  in elevator_exit()  local
166 	mutex_lock(&e->sysfs_lock);  in elevator_exit()
167 	blk_mq_exit_sched(q, e);  in elevator_exit()
168 	mutex_unlock(&e->sysfs_lock);  in elevator_exit()
170 	kobject_put(&e->kobj);  in elevator_exit()
188 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()  local
191 	hash_add(e->hash, &rq->hash, rq_hash_key(rq));  in elv_rqhash_add()
204 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()  local
208 	hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {  in elv_rqhash_find()
279 	struct elevator_queue *e = q->elevator;  in elv_merge()  local
318 	if (e->type->ops.request_merge)  in elv_merge()
319 		return e->type->ops.request_merge(q, req, bio);  in elv_merge()
373 	struct elevator_queue *e = q->elevator;  in elv_merged_request()  local
375 	if (e->type->ops.request_merged)  in elv_merged_request()
376 		e->type->ops.request_merged(q, rq, type);  in elv_merged_request()
387 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()  local
389 	if (e->type->ops.requests_merged)  in elv_merge_requests()
390 		e->type->ops.requests_merged(q, rq, next);  in elv_merge_requests()
398 	struct elevator_queue *e = q->elevator;  in elv_latter_request()  local
400 	if (e->type->ops.next_request)  in elv_latter_request()
401 		return e->type->ops.next_request(q, rq);  in elv_latter_request()
408 	struct elevator_queue *e = q->elevator;  in elv_former_request()  local
410 	if (e->type->ops.former_request)  in elv_former_request()
411 		return e->type->ops.former_request(q, rq);  in elv_former_request()
422 	struct elevator_queue *e;  in elv_attr_show()  local
428 	e = container_of(kobj, struct elevator_queue, kobj);  in elv_attr_show()
429 	mutex_lock(&e->sysfs_lock);  in elv_attr_show()
430 	error = e->type ? entry->show(e, page) : -ENOENT;  in elv_attr_show()
431 	mutex_unlock(&e->sysfs_lock);  in elv_attr_show()
440 	struct elevator_queue *e;  in elv_attr_store()  local
446 	e = container_of(kobj, struct elevator_queue, kobj);  in elv_attr_store()
447 	mutex_lock(&e->sysfs_lock);  in elv_attr_store()
448 	error = e->type ? entry->store(e, page, length) : -ENOENT;  in elv_attr_store()
449 	mutex_unlock(&e->sysfs_lock);  in elv_attr_store()
465 	struct elevator_queue *e = q->elevator;  in elv_register_queue()  local
470 	error = kobject_add(&e->kobj, &q->disk->queue_kobj, "iosched");  in elv_register_queue()
472 		struct elv_fs_entry *attr = e->type->elevator_attrs;  in elv_register_queue()
475 				if (sysfs_create_file(&e->kobj, &attr->attr))  in elv_register_queue()
481 			kobject_uevent(&e->kobj, KOBJ_ADD);  in elv_register_queue()
483 		set_bit(ELEVATOR_FLAG_REGISTERED, &e->flags);  in elv_register_queue()
490 	struct elevator_queue *e = q->elevator;  in elv_unregister_queue()  local
494 	if (e && test_and_clear_bit(ELEVATOR_FLAG_REGISTERED, &e->flags)) {  in elv_unregister_queue()
495 		kobject_uevent(&e->kobj, KOBJ_REMOVE);  in elv_unregister_queue()
496 		kobject_del(&e->kobj);  in elv_unregister_queue()
500 int elv_register(struct elevator_type *e)  in elv_register()  argument
503 	if (WARN_ON_ONCE(!e->ops.finish_request))  in elv_register()
506 	if (WARN_ON_ONCE(!e->ops.insert_requests || !e->ops.dispatch_request))  in elv_register()
510 	if (e->icq_size) {  in elv_register()
511 		if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||  in elv_register()
512 		    WARN_ON(e->icq_align < __alignof__(struct io_cq)))  in elv_register()
515 		snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),  in elv_register()
516 			 "%s_io_cq", e->elevator_name);  in elv_register()
517 		e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,  in elv_register()
518 						 e->icq_align, 0, NULL);  in elv_register()
519 		if (!e->icq_cache)  in elv_register()
525 	if (__elevator_find(e->elevator_name)) {  in elv_register()
527 		kmem_cache_destroy(e->icq_cache);  in elv_register()
530 	list_add_tail(&e->list, &elv_list);  in elv_register()
533 	printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);  in elv_register()
539 void elv_unregister(struct elevator_type *e)  in elv_unregister()  argument
543 	list_del_init(&e->list);  in elv_unregister()
550 	if (e->icq_cache) {  in elv_unregister()
552 		kmem_cache_destroy(e->icq_cache);  in elv_unregister()
553 		e->icq_cache = NULL;  in elv_unregister()
588 	struct elevator_type *e, *found = NULL;  in elevator_get_by_features()  local
592 	list_for_each_entry(e, &elv_list, list) {  in elevator_get_by_features()
593 		if (elv_support_features(q, e)) {  in elevator_get_by_features()
594 			found = e;  in elevator_get_by_features()
614 	struct elevator_type *e;  in elevator_init_mq()  local
626 		e = elevator_get_default(q);  in elevator_init_mq()
628 		e = elevator_get_by_features(q);  in elevator_init_mq()
629 	if (!e)  in elevator_init_mq()
642 	err = blk_mq_init_sched(q, e);  in elevator_init_mq()
648 			"falling back to \"none\"\n", e->elevator_name);  in elevator_init_mq()
651 	elevator_put(e);  in elevator_init_mq()
720 	struct elevator_type *e;  in elevator_change()  local
736 	e = elevator_find_get(q, elevator_name);  in elevator_change()
737 	if (!e) {  in elevator_change()
739 		e = elevator_find_get(q, elevator_name);  in elevator_change()
740 		if (!e)  in elevator_change()
743 	ret = elevator_switch(q, e);  in elevator_change()
744 	elevator_put(e);  in elevator_change()
767 	struct elevator_type *cur = NULL, *e;  in elv_iosched_show()  local
781 	list_for_each_entry(e, &elv_list, list) {  in elv_iosched_show()
782 		if (e == cur)  in elv_iosched_show()
783 			len += sprintf(name+len, "[%s] ", e->elevator_name);  in elv_iosched_show()
784 		else if (elv_support_features(q, e))  in elv_iosched_show()
785 			len += sprintf(name+len, "%s ", e->elevator_name);  in elv_iosched_show()