1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 286db1e29SJens Axboe /* 386db1e29SJens Axboe * Functions related to io context handling 486db1e29SJens Axboe */ 586db1e29SJens Axboe #include <linux/kernel.h> 686db1e29SJens Axboe #include <linux/module.h> 786db1e29SJens Axboe #include <linux/init.h> 886db1e29SJens Axboe #include <linux/bio.h> 986db1e29SJens Axboe #include <linux/blkdev.h> 105a0e3ad6STejun Heo #include <linux/slab.h> 11f719ff9bSIngo Molnar #include <linux/sched/task.h> 1286db1e29SJens Axboe 1386db1e29SJens Axboe #include "blk.h" 142aa7745bSChristoph Hellwig #include "blk-mq-sched.h" 1586db1e29SJens Axboe 1686db1e29SJens Axboe /* 1786db1e29SJens Axboe * For io context allocations 1886db1e29SJens Axboe */ 1986db1e29SJens Axboe static struct kmem_cache *iocontext_cachep; 2086db1e29SJens Axboe 216e736be7STejun Heo /** 226e736be7STejun Heo * get_io_context - increment reference count to io_context 236e736be7STejun Heo * @ioc: io_context to get 246e736be7STejun Heo * 256e736be7STejun Heo * Increment reference count to @ioc. 266e736be7STejun Heo */ 2787dd1d63SChristoph Hellwig static void get_io_context(struct io_context *ioc) 286e736be7STejun Heo { 296e736be7STejun Heo BUG_ON(atomic_long_read(&ioc->refcount) <= 0); 306e736be7STejun Heo atomic_long_inc(&ioc->refcount); 316e736be7STejun Heo } 326e736be7STejun Heo 337e5a8794STejun Heo static void icq_free_icq_rcu(struct rcu_head *head) 347e5a8794STejun Heo { 357e5a8794STejun Heo struct io_cq *icq = container_of(head, struct io_cq, __rcu_head); 367e5a8794STejun Heo 377e5a8794STejun Heo kmem_cache_free(icq->__rcu_icq_cache, icq); 387e5a8794STejun Heo } 397e5a8794STejun Heo 403d492c2eSOmar Sandoval /* 417b36a718SJens Axboe * Exit an icq. Called with ioc locked for blk-mq, and with both ioc 427b36a718SJens Axboe * and queue locked for legacy. 433d492c2eSOmar Sandoval */ 447e5a8794STejun Heo static void ioc_exit_icq(struct io_cq *icq) 457e5a8794STejun Heo { 46621032adSTejun Heo struct elevator_type *et = icq->q->elevator->type; 47621032adSTejun Heo 48621032adSTejun Heo if (icq->flags & ICQ_EXITED) 49621032adSTejun Heo return; 50621032adSTejun Heo 51f9cd4bfeSJens Axboe if (et->ops.exit_icq) 52f9cd4bfeSJens Axboe et->ops.exit_icq(icq); 53621032adSTejun Heo 54621032adSTejun Heo icq->flags |= ICQ_EXITED; 55621032adSTejun Heo } 56621032adSTejun Heo 574be8a2eaSChristoph Hellwig static void ioc_exit_icqs(struct io_context *ioc) 584be8a2eaSChristoph Hellwig { 594be8a2eaSChristoph Hellwig struct io_cq *icq; 604be8a2eaSChristoph Hellwig 614be8a2eaSChristoph Hellwig spin_lock_irq(&ioc->lock); 624be8a2eaSChristoph Hellwig hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) 634be8a2eaSChristoph Hellwig ioc_exit_icq(icq); 644be8a2eaSChristoph Hellwig spin_unlock_irq(&ioc->lock); 654be8a2eaSChristoph Hellwig } 664be8a2eaSChristoph Hellwig 677b36a718SJens Axboe /* 687b36a718SJens Axboe * Release an icq. Called with ioc locked for blk-mq, and with both ioc 697b36a718SJens Axboe * and queue locked for legacy. 707b36a718SJens Axboe */ 71621032adSTejun Heo static void ioc_destroy_icq(struct io_cq *icq) 72621032adSTejun Heo { 737e5a8794STejun Heo struct io_context *ioc = icq->ioc; 747e5a8794STejun Heo struct request_queue *q = icq->q; 757e5a8794STejun Heo struct elevator_type *et = q->elevator->type; 767e5a8794STejun Heo 777e5a8794STejun Heo lockdep_assert_held(&ioc->lock); 787e5a8794STejun Heo 797e5a8794STejun Heo radix_tree_delete(&ioc->icq_tree, icq->q->id); 807e5a8794STejun Heo hlist_del_init(&icq->ioc_node); 817e5a8794STejun Heo list_del_init(&icq->q_node); 827e5a8794STejun Heo 837e5a8794STejun Heo /* 847e5a8794STejun Heo * Both setting lookup hint to and clearing it from @icq are done 857e5a8794STejun Heo * under queue_lock. If it's not pointing to @icq now, it never 867e5a8794STejun Heo * will. Hint assignment itself can race safely. 877e5a8794STejun Heo */ 88ec6c676aSPaul E. McKenney if (rcu_access_pointer(ioc->icq_hint) == icq) 897e5a8794STejun Heo rcu_assign_pointer(ioc->icq_hint, NULL); 907e5a8794STejun Heo 91621032adSTejun Heo ioc_exit_icq(icq); 927e5a8794STejun Heo 937e5a8794STejun Heo /* 947e5a8794STejun Heo * @icq->q might have gone away by the time RCU callback runs 957e5a8794STejun Heo * making it impossible to determine icq_cache. Record it in @icq. 967e5a8794STejun Heo */ 977e5a8794STejun Heo icq->__rcu_icq_cache = et->icq_cache; 9830a2da7bSSahitya Tummala icq->flags |= ICQ_DESTROYED; 997e5a8794STejun Heo call_rcu(&icq->__rcu_head, icq_free_icq_rcu); 1007e5a8794STejun Heo } 1017e5a8794STejun Heo 102b2efa052STejun Heo /* 103b2efa052STejun Heo * Slow path for ioc release in put_io_context(). Performs double-lock 104c5869807STejun Heo * dancing to unlink all icq's and then frees ioc. 105b2efa052STejun Heo */ 106b2efa052STejun Heo static void ioc_release_fn(struct work_struct *work) 107b2efa052STejun Heo { 108b2efa052STejun Heo struct io_context *ioc = container_of(work, struct io_context, 109b2efa052STejun Heo release_work); 110a43f085fSJohn Ogness spin_lock_irq(&ioc->lock); 111b2efa052STejun Heo 112c5869807STejun Heo while (!hlist_empty(&ioc->icq_list)) { 113c5869807STejun Heo struct io_cq *icq = hlist_entry(ioc->icq_list.first, 114c5869807STejun Heo struct io_cq, ioc_node); 1152274b029STejun Heo struct request_queue *q = icq->q; 116b2efa052STejun Heo 1170d945c1fSChristoph Hellwig if (spin_trylock(&q->queue_lock)) { 118621032adSTejun Heo ioc_destroy_icq(icq); 1190d945c1fSChristoph Hellwig spin_unlock(&q->queue_lock); 120b2efa052STejun Heo } else { 121ab96bbabSJohn Ogness /* Make sure q and icq cannot be freed. */ 122ab96bbabSJohn Ogness rcu_read_lock(); 123ab96bbabSJohn Ogness 124ab96bbabSJohn Ogness /* Re-acquire the locks in the correct order. */ 125ab96bbabSJohn Ogness spin_unlock(&ioc->lock); 126ab96bbabSJohn Ogness spin_lock(&q->queue_lock); 127ab96bbabSJohn Ogness spin_lock(&ioc->lock); 128ab96bbabSJohn Ogness 129ab96bbabSJohn Ogness /* 130ab96bbabSJohn Ogness * The icq may have been destroyed when the ioc lock 131ab96bbabSJohn Ogness * was released. 132ab96bbabSJohn Ogness */ 133ab96bbabSJohn Ogness if (!(icq->flags & ICQ_DESTROYED)) 134ab96bbabSJohn Ogness ioc_destroy_icq(icq); 135ab96bbabSJohn Ogness 136ab96bbabSJohn Ogness spin_unlock(&q->queue_lock); 137ab96bbabSJohn Ogness rcu_read_unlock(); 138b2efa052STejun Heo } 1392274b029STejun Heo } 1402274b029STejun Heo 141a43f085fSJohn Ogness spin_unlock_irq(&ioc->lock); 142b2efa052STejun Heo 143b2efa052STejun Heo kmem_cache_free(iocontext_cachep, ioc); 14486db1e29SJens Axboe } 14586db1e29SJens Axboe 146*edf70ff5SChristoph Hellwig /* 147*edf70ff5SChristoph Hellwig * Releasing icqs requires reverse order double locking and we may already be 148*edf70ff5SChristoph Hellwig * holding a queue_lock. Do it asynchronously from a workqueue. 149*edf70ff5SChristoph Hellwig */ 150*edf70ff5SChristoph Hellwig static bool ioc_delay_free(struct io_context *ioc) 151*edf70ff5SChristoph Hellwig { 152*edf70ff5SChristoph Hellwig unsigned long flags; 153*edf70ff5SChristoph Hellwig 154*edf70ff5SChristoph Hellwig spin_lock_irqsave(&ioc->lock, flags); 155*edf70ff5SChristoph Hellwig if (!hlist_empty(&ioc->icq_list)) { 156*edf70ff5SChristoph Hellwig queue_work(system_power_efficient_wq, &ioc->release_work); 157*edf70ff5SChristoph Hellwig spin_unlock_irqrestore(&ioc->lock, flags); 158*edf70ff5SChristoph Hellwig return true; 159*edf70ff5SChristoph Hellwig } 160*edf70ff5SChristoph Hellwig spin_unlock_irqrestore(&ioc->lock, flags); 161*edf70ff5SChristoph Hellwig return false; 162*edf70ff5SChristoph Hellwig } 163*edf70ff5SChristoph Hellwig 16442ec57a8STejun Heo /** 16542ec57a8STejun Heo * put_io_context - put a reference of io_context 16642ec57a8STejun Heo * @ioc: io_context to put 16742ec57a8STejun Heo * 16842ec57a8STejun Heo * Decrement reference count of @ioc and release it if the count reaches 16911a3122fSTejun Heo * zero. 17086db1e29SJens Axboe */ 17111a3122fSTejun Heo void put_io_context(struct io_context *ioc) 17286db1e29SJens Axboe { 17342ec57a8STejun Heo BUG_ON(atomic_long_read(&ioc->refcount) <= 0); 174*edf70ff5SChristoph Hellwig if (atomic_long_dec_and_test(&ioc->refcount) && !ioc_delay_free(ioc)) 175ff8c1474SXiaotian Feng kmem_cache_free(iocontext_cachep, ioc); 17686db1e29SJens Axboe } 177222ee581SChristoph Hellwig EXPORT_SYMBOL_GPL(put_io_context); 17886db1e29SJens Axboe 179f6e8d01bSTejun Heo /* Called by the exiting task */ 180f6e8d01bSTejun Heo void exit_io_context(struct task_struct *task) 181f6e8d01bSTejun Heo { 182f6e8d01bSTejun Heo struct io_context *ioc; 183f6e8d01bSTejun Heo 184f6e8d01bSTejun Heo task_lock(task); 185f6e8d01bSTejun Heo ioc = task->io_context; 186f6e8d01bSTejun Heo task->io_context = NULL; 187f6e8d01bSTejun Heo task_unlock(task); 188f6e8d01bSTejun Heo 1894be8a2eaSChristoph Hellwig if (atomic_dec_and_test(&ioc->active_ref)) { 1904be8a2eaSChristoph Hellwig ioc_exit_icqs(ioc); 1914be8a2eaSChristoph Hellwig put_io_context(ioc); 1924be8a2eaSChristoph Hellwig } 193f6e8d01bSTejun Heo } 194f6e8d01bSTejun Heo 1957b36a718SJens Axboe static void __ioc_clear_queue(struct list_head *icq_list) 1967b36a718SJens Axboe { 1977b36a718SJens Axboe unsigned long flags; 1987b36a718SJens Axboe 19930a2da7bSSahitya Tummala rcu_read_lock(); 2007b36a718SJens Axboe while (!list_empty(icq_list)) { 2017b36a718SJens Axboe struct io_cq *icq = list_entry(icq_list->next, 2027b36a718SJens Axboe struct io_cq, q_node); 2037b36a718SJens Axboe struct io_context *ioc = icq->ioc; 2047b36a718SJens Axboe 2057b36a718SJens Axboe spin_lock_irqsave(&ioc->lock, flags); 20630a2da7bSSahitya Tummala if (icq->flags & ICQ_DESTROYED) { 20730a2da7bSSahitya Tummala spin_unlock_irqrestore(&ioc->lock, flags); 20830a2da7bSSahitya Tummala continue; 20930a2da7bSSahitya Tummala } 2107b36a718SJens Axboe ioc_destroy_icq(icq); 2117b36a718SJens Axboe spin_unlock_irqrestore(&ioc->lock, flags); 2127b36a718SJens Axboe } 21330a2da7bSSahitya Tummala rcu_read_unlock(); 2147b36a718SJens Axboe } 2157b36a718SJens Axboe 2167e5a8794STejun Heo /** 2177e5a8794STejun Heo * ioc_clear_queue - break any ioc association with the specified queue 2187e5a8794STejun Heo * @q: request_queue being cleared 2197e5a8794STejun Heo * 2207b36a718SJens Axboe * Walk @q->icq_list and exit all io_cq's. 2217e5a8794STejun Heo */ 2227e5a8794STejun Heo void ioc_clear_queue(struct request_queue *q) 2237e5a8794STejun Heo { 2247b36a718SJens Axboe LIST_HEAD(icq_list); 2257e5a8794STejun Heo 2260d945c1fSChristoph Hellwig spin_lock_irq(&q->queue_lock); 2277b36a718SJens Axboe list_splice_init(&q->icq_list, &icq_list); 2280d945c1fSChristoph Hellwig spin_unlock_irq(&q->queue_lock); 2297e5a8794STejun Heo 2307b36a718SJens Axboe __ioc_clear_queue(&icq_list); 2317e5a8794STejun Heo } 2327e5a8794STejun Heo 233a0f14d8bSChristoph Hellwig static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) 23486db1e29SJens Axboe { 235df415656SPaul Bolle struct io_context *ioc; 23686db1e29SJens Axboe 23742ec57a8STejun Heo ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO, 23842ec57a8STejun Heo node); 23942ec57a8STejun Heo if (unlikely(!ioc)) 240a0f14d8bSChristoph Hellwig return NULL; 24142ec57a8STejun Heo 242df415656SPaul Bolle atomic_long_set(&ioc->refcount, 1); 243f6e8d01bSTejun Heo atomic_set(&ioc->active_ref, 1); 244df415656SPaul Bolle spin_lock_init(&ioc->lock); 245c137969bSShakeel Butt INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC); 246c5869807STejun Heo INIT_HLIST_HEAD(&ioc->icq_list); 247b2efa052STejun Heo INIT_WORK(&ioc->release_work, ioc_release_fn); 248a0f14d8bSChristoph Hellwig return ioc; 249a0f14d8bSChristoph Hellwig } 250a0f14d8bSChristoph Hellwig 251d538ea4cSChristoph Hellwig static struct io_context *create_task_io_context(struct task_struct *task, 252d538ea4cSChristoph Hellwig gfp_t gfp_flags, int node) 253a0f14d8bSChristoph Hellwig { 254a0f14d8bSChristoph Hellwig struct io_context *ioc; 255a0f14d8bSChristoph Hellwig 256a0f14d8bSChristoph Hellwig ioc = alloc_io_context(gfp_flags, node); 257a0f14d8bSChristoph Hellwig if (!ioc) 258d538ea4cSChristoph Hellwig return NULL; 25986db1e29SJens Axboe 260fd638368STejun Heo /* 261fd638368STejun Heo * Try to install. ioc shouldn't be installed if someone else 262fd638368STejun Heo * already did or @task, which isn't %current, is exiting. Note 263fd638368STejun Heo * that we need to allow ioc creation on exiting %current as exit 264fd638368STejun Heo * path may issue IOs from e.g. exit_files(). The exit path is 265fd638368STejun Heo * responsible for not issuing IO after exit_io_context(). 266fd638368STejun Heo */ 2676e736be7STejun Heo task_lock(task); 268fd638368STejun Heo if (!task->io_context && 269fd638368STejun Heo (task == current || !(task->flags & PF_EXITING))) 2706e736be7STejun Heo task->io_context = ioc; 271f2dbd76aSTejun Heo else 2726e736be7STejun Heo kmem_cache_free(iocontext_cachep, ioc); 2733c9c708cSEric Dumazet 274d538ea4cSChristoph Hellwig ioc = task->io_context; 275d538ea4cSChristoph Hellwig if (ioc) 276d538ea4cSChristoph Hellwig get_io_context(ioc); 2776e736be7STejun Heo task_unlock(task); 278d538ea4cSChristoph Hellwig return ioc; 27986db1e29SJens Axboe } 28086db1e29SJens Axboe 2816e736be7STejun Heo /** 2826e736be7STejun Heo * get_task_io_context - get io_context of a task 2836e736be7STejun Heo * @task: task of interest 2846e736be7STejun Heo * @gfp_flags: allocation flags, used if allocation is necessary 2856e736be7STejun Heo * @node: allocation node, used if allocation is necessary 28686db1e29SJens Axboe * 2876e736be7STejun Heo * Return io_context of @task. If it doesn't exist, it is created with 2886e736be7STejun Heo * @gfp_flags and @node. The returned io_context has its reference count 2896e736be7STejun Heo * incremented. 2906e736be7STejun Heo * 2916e736be7STejun Heo * This function always goes through task_lock() and it's better to use 292f2dbd76aSTejun Heo * %current->io_context + get_io_context() for %current. 29386db1e29SJens Axboe */ 2946e736be7STejun Heo struct io_context *get_task_io_context(struct task_struct *task, 2956e736be7STejun Heo gfp_t gfp_flags, int node) 29686db1e29SJens Axboe { 2976e736be7STejun Heo struct io_context *ioc; 29886db1e29SJens Axboe 299d0164adcSMel Gorman might_sleep_if(gfpflags_allow_blocking(gfp_flags)); 30086db1e29SJens Axboe 3016e736be7STejun Heo task_lock(task); 3026e736be7STejun Heo ioc = task->io_context; 303d538ea4cSChristoph Hellwig if (unlikely(!ioc)) { 304d538ea4cSChristoph Hellwig task_unlock(task); 305d538ea4cSChristoph Hellwig return create_task_io_context(task, gfp_flags, node); 306d538ea4cSChristoph Hellwig } 3076e736be7STejun Heo get_io_context(ioc); 3086e736be7STejun Heo task_unlock(task); 309df415656SPaul Bolle return ioc; 31086db1e29SJens Axboe } 31186db1e29SJens Axboe 31288c9a2ceSChristoph Hellwig int __copy_io(unsigned long clone_flags, struct task_struct *tsk) 31388c9a2ceSChristoph Hellwig { 31488c9a2ceSChristoph Hellwig struct io_context *ioc = current->io_context; 31588c9a2ceSChristoph Hellwig 31688c9a2ceSChristoph Hellwig /* 31788c9a2ceSChristoph Hellwig * Share io context with parent, if CLONE_IO is set 31888c9a2ceSChristoph Hellwig */ 31988c9a2ceSChristoph Hellwig if (clone_flags & CLONE_IO) { 32050569c24SChristoph Hellwig atomic_inc(&ioc->active_ref); 32188c9a2ceSChristoph Hellwig tsk->io_context = ioc; 32288c9a2ceSChristoph Hellwig } else if (ioprio_valid(ioc->ioprio)) { 3238ffc1368SChristoph Hellwig tsk->io_context = alloc_io_context(GFP_KERNEL, NUMA_NO_NODE); 3248ffc1368SChristoph Hellwig if (!tsk->io_context) 32588c9a2ceSChristoph Hellwig return -ENOMEM; 3268ffc1368SChristoph Hellwig tsk->io_context->ioprio = ioc->ioprio; 32788c9a2ceSChristoph Hellwig } 32888c9a2ceSChristoph Hellwig 32988c9a2ceSChristoph Hellwig return 0; 33088c9a2ceSChristoph Hellwig } 33188c9a2ceSChristoph Hellwig 33247fdd4caSTejun Heo /** 33347fdd4caSTejun Heo * ioc_lookup_icq - lookup io_cq from ioc 33447fdd4caSTejun Heo * @q: the associated request_queue 33547fdd4caSTejun Heo * 33647fdd4caSTejun Heo * Look up io_cq associated with @ioc - @q pair from @ioc. Must be called 33747fdd4caSTejun Heo * with @q->queue_lock held. 33847fdd4caSTejun Heo */ 339eca5892aSChristoph Hellwig struct io_cq *ioc_lookup_icq(struct request_queue *q) 34047fdd4caSTejun Heo { 341eca5892aSChristoph Hellwig struct io_context *ioc = current->io_context; 34247fdd4caSTejun Heo struct io_cq *icq; 34347fdd4caSTejun Heo 3440d945c1fSChristoph Hellwig lockdep_assert_held(&q->queue_lock); 34547fdd4caSTejun Heo 34647fdd4caSTejun Heo /* 34747fdd4caSTejun Heo * icq's are indexed from @ioc using radix tree and hint pointer, 34847fdd4caSTejun Heo * both of which are protected with RCU. All removals are done 34947fdd4caSTejun Heo * holding both q and ioc locks, and we're holding q lock - if we 35047fdd4caSTejun Heo * find a icq which points to us, it's guaranteed to be valid. 35147fdd4caSTejun Heo */ 35247fdd4caSTejun Heo rcu_read_lock(); 35347fdd4caSTejun Heo icq = rcu_dereference(ioc->icq_hint); 35447fdd4caSTejun Heo if (icq && icq->q == q) 35547fdd4caSTejun Heo goto out; 35647fdd4caSTejun Heo 35747fdd4caSTejun Heo icq = radix_tree_lookup(&ioc->icq_tree, q->id); 35847fdd4caSTejun Heo if (icq && icq->q == q) 35947fdd4caSTejun Heo rcu_assign_pointer(ioc->icq_hint, icq); /* allowed to race */ 36047fdd4caSTejun Heo else 36147fdd4caSTejun Heo icq = NULL; 36247fdd4caSTejun Heo out: 36347fdd4caSTejun Heo rcu_read_unlock(); 36447fdd4caSTejun Heo return icq; 36547fdd4caSTejun Heo } 36647fdd4caSTejun Heo EXPORT_SYMBOL(ioc_lookup_icq); 36747fdd4caSTejun Heo 368f1f8cc94STejun Heo /** 369f1f8cc94STejun Heo * ioc_create_icq - create and link io_cq 370f1f8cc94STejun Heo * @q: request_queue of interest 371f1f8cc94STejun Heo * 37224acfc34STejun Heo * Make sure io_cq linking @ioc and @q exists. If icq doesn't exist, they 37324acfc34STejun Heo * will be created using @gfp_mask. 374f1f8cc94STejun Heo * 375f1f8cc94STejun Heo * The caller is responsible for ensuring @ioc won't go away and @q is 376f1f8cc94STejun Heo * alive and will stay alive until this function returns. 377f1f8cc94STejun Heo */ 37818b74c4dSChristoph Hellwig static struct io_cq *ioc_create_icq(struct request_queue *q) 379f1f8cc94STejun Heo { 38018b74c4dSChristoph Hellwig struct io_context *ioc = current->io_context; 381f1f8cc94STejun Heo struct elevator_type *et = q->elevator->type; 382f1f8cc94STejun Heo struct io_cq *icq; 383f1f8cc94STejun Heo 384f1f8cc94STejun Heo /* allocate stuff */ 38518b74c4dSChristoph Hellwig icq = kmem_cache_alloc_node(et->icq_cache, GFP_ATOMIC | __GFP_ZERO, 386f1f8cc94STejun Heo q->node); 387f1f8cc94STejun Heo if (!icq) 388f1f8cc94STejun Heo return NULL; 389f1f8cc94STejun Heo 39018b74c4dSChristoph Hellwig if (radix_tree_maybe_preload(GFP_ATOMIC) < 0) { 391f1f8cc94STejun Heo kmem_cache_free(et->icq_cache, icq); 392f1f8cc94STejun Heo return NULL; 393f1f8cc94STejun Heo } 394f1f8cc94STejun Heo 395f1f8cc94STejun Heo icq->ioc = ioc; 396f1f8cc94STejun Heo icq->q = q; 397f1f8cc94STejun Heo INIT_LIST_HEAD(&icq->q_node); 398f1f8cc94STejun Heo INIT_HLIST_NODE(&icq->ioc_node); 399f1f8cc94STejun Heo 400f1f8cc94STejun Heo /* lock both q and ioc and try to link @icq */ 4010d945c1fSChristoph Hellwig spin_lock_irq(&q->queue_lock); 402f1f8cc94STejun Heo spin_lock(&ioc->lock); 403f1f8cc94STejun Heo 404f1f8cc94STejun Heo if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) { 405f1f8cc94STejun Heo hlist_add_head(&icq->ioc_node, &ioc->icq_list); 406f1f8cc94STejun Heo list_add(&icq->q_node, &q->icq_list); 407f9cd4bfeSJens Axboe if (et->ops.init_icq) 408f9cd4bfeSJens Axboe et->ops.init_icq(icq); 409f1f8cc94STejun Heo } else { 410f1f8cc94STejun Heo kmem_cache_free(et->icq_cache, icq); 411eca5892aSChristoph Hellwig icq = ioc_lookup_icq(q); 412f1f8cc94STejun Heo if (!icq) 413f1f8cc94STejun Heo printk(KERN_ERR "cfq: icq link failed!\n"); 414f1f8cc94STejun Heo } 415f1f8cc94STejun Heo 416f1f8cc94STejun Heo spin_unlock(&ioc->lock); 4170d945c1fSChristoph Hellwig spin_unlock_irq(&q->queue_lock); 418f1f8cc94STejun Heo radix_tree_preload_end(); 419f1f8cc94STejun Heo return icq; 420f1f8cc94STejun Heo } 421f1f8cc94STejun Heo 42287dd1d63SChristoph Hellwig struct io_cq *ioc_find_get_icq(struct request_queue *q) 42387dd1d63SChristoph Hellwig { 424d538ea4cSChristoph Hellwig struct io_context *ioc = current->io_context; 425d538ea4cSChristoph Hellwig struct io_cq *icq = NULL; 42687dd1d63SChristoph Hellwig 427d538ea4cSChristoph Hellwig if (unlikely(!ioc)) { 428d538ea4cSChristoph Hellwig ioc = create_task_io_context(current, GFP_ATOMIC, q->node); 42987dd1d63SChristoph Hellwig if (!ioc) 43087dd1d63SChristoph Hellwig return NULL; 431d538ea4cSChristoph Hellwig } else { 432d538ea4cSChristoph Hellwig get_io_context(ioc); 43387dd1d63SChristoph Hellwig 43487dd1d63SChristoph Hellwig spin_lock_irq(&q->queue_lock); 435eca5892aSChristoph Hellwig icq = ioc_lookup_icq(q); 43687dd1d63SChristoph Hellwig spin_unlock_irq(&q->queue_lock); 437d538ea4cSChristoph Hellwig } 43887dd1d63SChristoph Hellwig 43987dd1d63SChristoph Hellwig if (!icq) { 44018b74c4dSChristoph Hellwig icq = ioc_create_icq(q); 441d538ea4cSChristoph Hellwig if (!icq) { 442d538ea4cSChristoph Hellwig put_io_context(ioc); 44387dd1d63SChristoph Hellwig return NULL; 44487dd1d63SChristoph Hellwig } 445d538ea4cSChristoph Hellwig } 44687dd1d63SChristoph Hellwig return icq; 44787dd1d63SChristoph Hellwig } 44887dd1d63SChristoph Hellwig EXPORT_SYMBOL_GPL(ioc_find_get_icq); 44987dd1d63SChristoph Hellwig 45013341598SAdrian Bunk static int __init blk_ioc_init(void) 45186db1e29SJens Axboe { 45286db1e29SJens Axboe iocontext_cachep = kmem_cache_create("blkdev_ioc", 45386db1e29SJens Axboe sizeof(struct io_context), 0, SLAB_PANIC, NULL); 45486db1e29SJens Axboe return 0; 45586db1e29SJens Axboe } 45686db1e29SJens Axboe subsys_initcall(blk_ioc_init); 457