1 /* 2 * Functions related to io context handling 3 */ 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/init.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/bootmem.h> /* for max_pfn/max_low_pfn */ 10 11 #include "blk.h" 12 13 /* 14 * For io context allocations 15 */ 16 static struct kmem_cache *iocontext_cachep; 17 18 static void cfq_dtor(struct io_context *ioc) 19 { 20 struct cfq_io_context *cic[1]; 21 int r; 22 23 /* 24 * We don't have a specific key to lookup with, so use the gang 25 * lookup to just retrieve the first item stored. The cfq exit 26 * function will iterate the full tree, so any member will do. 27 */ 28 r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1); 29 if (r > 0) 30 cic[0]->dtor(ioc); 31 } 32 33 /* 34 * IO Context helper functions. put_io_context() returns 1 if there are no 35 * more users of this io context, 0 otherwise. 36 */ 37 int put_io_context(struct io_context *ioc) 38 { 39 if (ioc == NULL) 40 return 1; 41 42 BUG_ON(atomic_read(&ioc->refcount) == 0); 43 44 if (atomic_dec_and_test(&ioc->refcount)) { 45 rcu_read_lock(); 46 if (ioc->aic && ioc->aic->dtor) 47 ioc->aic->dtor(ioc->aic); 48 rcu_read_unlock(); 49 cfq_dtor(ioc); 50 51 kmem_cache_free(iocontext_cachep, ioc); 52 return 1; 53 } 54 return 0; 55 } 56 EXPORT_SYMBOL(put_io_context); 57 58 static void cfq_exit(struct io_context *ioc) 59 { 60 struct cfq_io_context *cic[1]; 61 int r; 62 63 rcu_read_lock(); 64 /* 65 * See comment for cfq_dtor() 66 */ 67 r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1); 68 rcu_read_unlock(); 69 70 if (r > 0) 71 cic[0]->exit(ioc); 72 } 73 74 /* Called by the exitting task */ 75 void exit_io_context(void) 76 { 77 struct io_context *ioc; 78 79 task_lock(current); 80 ioc = current->io_context; 81 current->io_context = NULL; 82 task_unlock(current); 83 84 if (atomic_dec_and_test(&ioc->nr_tasks)) { 85 if (ioc->aic && ioc->aic->exit) 86 ioc->aic->exit(ioc->aic); 87 cfq_exit(ioc); 88 89 put_io_context(ioc); 90 } 91 } 92 93 struct io_context *alloc_io_context(gfp_t gfp_flags, int node) 94 { 95 struct io_context *ret; 96 97 ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node); 98 if (ret) { 99 atomic_set(&ret->refcount, 1); 100 atomic_set(&ret->nr_tasks, 1); 101 spin_lock_init(&ret->lock); 102 ret->ioprio_changed = 0; 103 ret->ioprio = 0; 104 ret->last_waited = jiffies; /* doesn't matter... */ 105 ret->nr_batch_requests = 0; /* because this is 0 */ 106 ret->aic = NULL; 107 INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH); 108 ret->ioc_data = NULL; 109 } 110 111 return ret; 112 } 113 114 /* 115 * If the current task has no IO context then create one and initialise it. 116 * Otherwise, return its existing IO context. 117 * 118 * This returned IO context doesn't have a specifically elevated refcount, 119 * but since the current task itself holds a reference, the context can be 120 * used in general code, so long as it stays within `current` context. 121 */ 122 struct io_context *current_io_context(gfp_t gfp_flags, int node) 123 { 124 struct task_struct *tsk = current; 125 struct io_context *ret; 126 127 ret = tsk->io_context; 128 if (likely(ret)) 129 return ret; 130 131 ret = alloc_io_context(gfp_flags, node); 132 if (ret) { 133 /* make sure set_task_ioprio() sees the settings above */ 134 smp_wmb(); 135 tsk->io_context = ret; 136 } 137 138 return ret; 139 } 140 141 /* 142 * If the current task has no IO context then create one and initialise it. 143 * If it does have a context, take a ref on it. 144 * 145 * This is always called in the context of the task which submitted the I/O. 146 */ 147 struct io_context *get_io_context(gfp_t gfp_flags, int node) 148 { 149 struct io_context *ret = NULL; 150 151 /* 152 * Check for unlikely race with exiting task. ioc ref count is 153 * zero when ioc is being detached. 154 */ 155 do { 156 ret = current_io_context(gfp_flags, node); 157 if (unlikely(!ret)) 158 break; 159 } while (!atomic_inc_not_zero(&ret->refcount)); 160 161 return ret; 162 } 163 EXPORT_SYMBOL(get_io_context); 164 165 void copy_io_context(struct io_context **pdst, struct io_context **psrc) 166 { 167 struct io_context *src = *psrc; 168 struct io_context *dst = *pdst; 169 170 if (src) { 171 BUG_ON(atomic_read(&src->refcount) == 0); 172 atomic_inc(&src->refcount); 173 put_io_context(dst); 174 *pdst = src; 175 } 176 } 177 EXPORT_SYMBOL(copy_io_context); 178 179 int __init blk_ioc_init(void) 180 { 181 iocontext_cachep = kmem_cache_create("blkdev_ioc", 182 sizeof(struct io_context), 0, SLAB_PANIC, NULL); 183 return 0; 184 } 185 subsys_initcall(blk_ioc_init); 186