1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/module.h> 4 #include <linux/backing-dev.h> 5 #include <linux/bio.h> 6 #include <linux/blkdev.h> 7 #include <linux/mm.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/workqueue.h> 11 #include <linux/smp.h> 12 13 #include <linux/blk-mq.h> 14 #include "blk.h" 15 #include "blk-mq.h" 16 #include "blk-mq-tag.h" 17 18 static void blk_mq_sysfs_release(struct kobject *kobj) 19 { 20 struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj); 21 22 free_percpu(ctxs->queue_ctx); 23 kfree(ctxs); 24 } 25 26 static void blk_mq_ctx_sysfs_release(struct kobject *kobj) 27 { 28 struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj); 29 30 /* ctx->ctxs won't be released until all ctx are freed */ 31 kobject_put(&ctx->ctxs->kobj); 32 } 33 34 static void blk_mq_hw_sysfs_release(struct kobject *kobj) 35 { 36 struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx, 37 kobj); 38 39 blk_free_flush_queue(hctx->fq); 40 sbitmap_free(&hctx->ctx_map); 41 free_cpumask_var(hctx->cpumask); 42 kfree(hctx->ctxs); 43 kfree(hctx); 44 } 45 46 struct blk_mq_hw_ctx_sysfs_entry { 47 struct attribute attr; 48 ssize_t (*show)(struct blk_mq_hw_ctx *, char *); 49 }; 50 51 static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj, 52 struct attribute *attr, char *page) 53 { 54 struct blk_mq_hw_ctx_sysfs_entry *entry; 55 struct blk_mq_hw_ctx *hctx; 56 struct request_queue *q; 57 ssize_t res; 58 59 entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); 60 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); 61 q = hctx->queue; 62 63 if (!entry->show) 64 return -EIO; 65 66 mutex_lock(&q->sysfs_lock); 67 res = entry->show(hctx, page); 68 mutex_unlock(&q->sysfs_lock); 69 return res; 70 } 71 72 static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx, 73 char *page) 74 { 75 return sprintf(page, "%u\n", hctx->tags->nr_tags); 76 } 77 78 static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx, 79 char *page) 80 { 81 return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags); 82 } 83 84 static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page) 85 { 86 const size_t size = PAGE_SIZE - 1; 87 unsigned int i, first = 1; 88 int ret = 0, pos = 0; 89 90 for_each_cpu(i, hctx->cpumask) { 91 if (first) 92 ret = snprintf(pos + page, size - pos, "%u", i); 93 else 94 ret = snprintf(pos + page, size - pos, ", %u", i); 95 96 if (ret >= size - pos) 97 break; 98 99 first = 0; 100 pos += ret; 101 } 102 103 ret = snprintf(pos + page, size + 1 - pos, "\n"); 104 return pos + ret; 105 } 106 107 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = { 108 .attr = {.name = "nr_tags", .mode = 0444 }, 109 .show = blk_mq_hw_sysfs_nr_tags_show, 110 }; 111 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = { 112 .attr = {.name = "nr_reserved_tags", .mode = 0444 }, 113 .show = blk_mq_hw_sysfs_nr_reserved_tags_show, 114 }; 115 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { 116 .attr = {.name = "cpu_list", .mode = 0444 }, 117 .show = blk_mq_hw_sysfs_cpus_show, 118 }; 119 120 static struct attribute *default_hw_ctx_attrs[] = { 121 &blk_mq_hw_sysfs_nr_tags.attr, 122 &blk_mq_hw_sysfs_nr_reserved_tags.attr, 123 &blk_mq_hw_sysfs_cpus.attr, 124 NULL, 125 }; 126 ATTRIBUTE_GROUPS(default_hw_ctx); 127 128 static const struct sysfs_ops blk_mq_hw_sysfs_ops = { 129 .show = blk_mq_hw_sysfs_show, 130 }; 131 132 static const struct kobj_type blk_mq_ktype = { 133 .release = blk_mq_sysfs_release, 134 }; 135 136 static const struct kobj_type blk_mq_ctx_ktype = { 137 .release = blk_mq_ctx_sysfs_release, 138 }; 139 140 static const struct kobj_type blk_mq_hw_ktype = { 141 .sysfs_ops = &blk_mq_hw_sysfs_ops, 142 .default_groups = default_hw_ctx_groups, 143 .release = blk_mq_hw_sysfs_release, 144 }; 145 146 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx) 147 { 148 struct blk_mq_ctx *ctx; 149 int i; 150 151 if (!hctx->nr_ctx) 152 return; 153 154 hctx_for_each_ctx(hctx, ctx, i) 155 kobject_del(&ctx->kobj); 156 157 kobject_del(&hctx->kobj); 158 } 159 160 static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) 161 { 162 struct request_queue *q = hctx->queue; 163 struct blk_mq_ctx *ctx; 164 int i, j, ret; 165 166 if (!hctx->nr_ctx) 167 return 0; 168 169 ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num); 170 if (ret) 171 return ret; 172 173 hctx_for_each_ctx(hctx, ctx, i) { 174 ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu); 175 if (ret) 176 goto out; 177 } 178 179 return 0; 180 out: 181 hctx_for_each_ctx(hctx, ctx, j) { 182 if (j < i) 183 kobject_del(&ctx->kobj); 184 } 185 kobject_del(&hctx->kobj); 186 return ret; 187 } 188 189 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx) 190 { 191 kobject_init(&hctx->kobj, &blk_mq_hw_ktype); 192 } 193 194 void blk_mq_sysfs_deinit(struct request_queue *q) 195 { 196 struct blk_mq_ctx *ctx; 197 int cpu; 198 199 for_each_possible_cpu(cpu) { 200 ctx = per_cpu_ptr(q->queue_ctx, cpu); 201 kobject_put(&ctx->kobj); 202 } 203 kobject_put(q->mq_kobj); 204 } 205 206 void blk_mq_sysfs_init(struct request_queue *q) 207 { 208 struct blk_mq_ctx *ctx; 209 int cpu; 210 211 kobject_init(q->mq_kobj, &blk_mq_ktype); 212 213 for_each_possible_cpu(cpu) { 214 ctx = per_cpu_ptr(q->queue_ctx, cpu); 215 216 kobject_get(q->mq_kobj); 217 kobject_init(&ctx->kobj, &blk_mq_ctx_ktype); 218 } 219 } 220 221 int blk_mq_sysfs_register(struct gendisk *disk) 222 { 223 struct request_queue *q = disk->queue; 224 struct blk_mq_hw_ctx *hctx; 225 unsigned long i, j; 226 int ret; 227 228 lockdep_assert_held(&q->sysfs_dir_lock); 229 230 ret = kobject_add(q->mq_kobj, &disk_to_dev(disk)->kobj, "mq"); 231 if (ret < 0) 232 goto out; 233 234 kobject_uevent(q->mq_kobj, KOBJ_ADD); 235 236 queue_for_each_hw_ctx(q, hctx, i) { 237 ret = blk_mq_register_hctx(hctx); 238 if (ret) 239 goto unreg; 240 } 241 242 q->mq_sysfs_init_done = true; 243 244 out: 245 return ret; 246 247 unreg: 248 queue_for_each_hw_ctx(q, hctx, j) { 249 if (j < i) 250 blk_mq_unregister_hctx(hctx); 251 } 252 253 kobject_uevent(q->mq_kobj, KOBJ_REMOVE); 254 kobject_del(q->mq_kobj); 255 return ret; 256 } 257 258 void blk_mq_sysfs_unregister(struct gendisk *disk) 259 { 260 struct request_queue *q = disk->queue; 261 struct blk_mq_hw_ctx *hctx; 262 unsigned long i; 263 264 lockdep_assert_held(&q->sysfs_dir_lock); 265 266 queue_for_each_hw_ctx(q, hctx, i) 267 blk_mq_unregister_hctx(hctx); 268 269 kobject_uevent(q->mq_kobj, KOBJ_REMOVE); 270 kobject_del(q->mq_kobj); 271 272 q->mq_sysfs_init_done = false; 273 } 274 275 void blk_mq_sysfs_unregister_hctxs(struct request_queue *q) 276 { 277 struct blk_mq_hw_ctx *hctx; 278 unsigned long i; 279 280 mutex_lock(&q->sysfs_dir_lock); 281 if (!q->mq_sysfs_init_done) 282 goto unlock; 283 284 queue_for_each_hw_ctx(q, hctx, i) 285 blk_mq_unregister_hctx(hctx); 286 287 unlock: 288 mutex_unlock(&q->sysfs_dir_lock); 289 } 290 291 int blk_mq_sysfs_register_hctxs(struct request_queue *q) 292 { 293 struct blk_mq_hw_ctx *hctx; 294 unsigned long i; 295 int ret = 0; 296 297 mutex_lock(&q->sysfs_dir_lock); 298 if (!q->mq_sysfs_init_done) 299 goto unlock; 300 301 queue_for_each_hw_ctx(q, hctx, i) { 302 ret = blk_mq_register_hctx(hctx); 303 if (ret) 304 break; 305 } 306 307 unlock: 308 mutex_unlock(&q->sysfs_dir_lock); 309 310 return ret; 311 } 312