1 #include <linux/kernel.h> 2 #include <linux/module.h> 3 #include <linux/backing-dev.h> 4 #include <linux/bio.h> 5 #include <linux/blkdev.h> 6 #include <linux/mm.h> 7 #include <linux/init.h> 8 #include <linux/slab.h> 9 #include <linux/workqueue.h> 10 #include <linux/smp.h> 11 12 #include <linux/blk-mq.h> 13 #include "blk-mq.h" 14 #include "blk-mq-tag.h" 15 16 static void blk_mq_sysfs_release(struct kobject *kobj) 17 { 18 struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj); 19 20 free_percpu(ctxs->queue_ctx); 21 kfree(ctxs); 22 } 23 24 static void blk_mq_ctx_sysfs_release(struct kobject *kobj) 25 { 26 struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj); 27 28 /* ctx->ctxs won't be released until all ctx are freed */ 29 kobject_put(&ctx->ctxs->kobj); 30 } 31 32 static void blk_mq_hw_sysfs_release(struct kobject *kobj) 33 { 34 struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx, 35 kobj); 36 free_cpumask_var(hctx->cpumask); 37 kfree(hctx->ctxs); 38 kfree(hctx); 39 } 40 41 struct blk_mq_ctx_sysfs_entry { 42 struct attribute attr; 43 ssize_t (*show)(struct blk_mq_ctx *, char *); 44 ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t); 45 }; 46 47 struct blk_mq_hw_ctx_sysfs_entry { 48 struct attribute attr; 49 ssize_t (*show)(struct blk_mq_hw_ctx *, char *); 50 ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t); 51 }; 52 53 static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr, 54 char *page) 55 { 56 struct blk_mq_ctx_sysfs_entry *entry; 57 struct blk_mq_ctx *ctx; 58 struct request_queue *q; 59 ssize_t res; 60 61 entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr); 62 ctx = container_of(kobj, struct blk_mq_ctx, kobj); 63 q = ctx->queue; 64 65 if (!entry->show) 66 return -EIO; 67 68 res = -ENOENT; 69 mutex_lock(&q->sysfs_lock); 70 if (!blk_queue_dying(q)) 71 res = entry->show(ctx, page); 72 mutex_unlock(&q->sysfs_lock); 73 return res; 74 } 75 76 static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr, 77 const char *page, size_t length) 78 { 79 struct blk_mq_ctx_sysfs_entry *entry; 80 struct blk_mq_ctx *ctx; 81 struct request_queue *q; 82 ssize_t res; 83 84 entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr); 85 ctx = container_of(kobj, struct blk_mq_ctx, kobj); 86 q = ctx->queue; 87 88 if (!entry->store) 89 return -EIO; 90 91 res = -ENOENT; 92 mutex_lock(&q->sysfs_lock); 93 if (!blk_queue_dying(q)) 94 res = entry->store(ctx, page, length); 95 mutex_unlock(&q->sysfs_lock); 96 return res; 97 } 98 99 static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj, 100 struct attribute *attr, char *page) 101 { 102 struct blk_mq_hw_ctx_sysfs_entry *entry; 103 struct blk_mq_hw_ctx *hctx; 104 struct request_queue *q; 105 ssize_t res; 106 107 entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); 108 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); 109 q = hctx->queue; 110 111 if (!entry->show) 112 return -EIO; 113 114 res = -ENOENT; 115 mutex_lock(&q->sysfs_lock); 116 if (!blk_queue_dying(q)) 117 res = entry->show(hctx, page); 118 mutex_unlock(&q->sysfs_lock); 119 return res; 120 } 121 122 static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj, 123 struct attribute *attr, const char *page, 124 size_t length) 125 { 126 struct blk_mq_hw_ctx_sysfs_entry *entry; 127 struct blk_mq_hw_ctx *hctx; 128 struct request_queue *q; 129 ssize_t res; 130 131 entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); 132 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); 133 q = hctx->queue; 134 135 if (!entry->store) 136 return -EIO; 137 138 res = -ENOENT; 139 mutex_lock(&q->sysfs_lock); 140 if (!blk_queue_dying(q)) 141 res = entry->store(hctx, page, length); 142 mutex_unlock(&q->sysfs_lock); 143 return res; 144 } 145 146 static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx, 147 char *page) 148 { 149 return sprintf(page, "%u\n", hctx->tags->nr_tags); 150 } 151 152 static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx, 153 char *page) 154 { 155 return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags); 156 } 157 158 static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page) 159 { 160 unsigned int i, first = 1; 161 ssize_t ret = 0; 162 163 for_each_cpu(i, hctx->cpumask) { 164 if (first) 165 ret += sprintf(ret + page, "%u", i); 166 else 167 ret += sprintf(ret + page, ", %u", i); 168 169 first = 0; 170 } 171 172 ret += sprintf(ret + page, "\n"); 173 return ret; 174 } 175 176 static struct attribute *default_ctx_attrs[] = { 177 NULL, 178 }; 179 180 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = { 181 .attr = {.name = "nr_tags", .mode = 0444 }, 182 .show = blk_mq_hw_sysfs_nr_tags_show, 183 }; 184 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = { 185 .attr = {.name = "nr_reserved_tags", .mode = 0444 }, 186 .show = blk_mq_hw_sysfs_nr_reserved_tags_show, 187 }; 188 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { 189 .attr = {.name = "cpu_list", .mode = 0444 }, 190 .show = blk_mq_hw_sysfs_cpus_show, 191 }; 192 193 static struct attribute *default_hw_ctx_attrs[] = { 194 &blk_mq_hw_sysfs_nr_tags.attr, 195 &blk_mq_hw_sysfs_nr_reserved_tags.attr, 196 &blk_mq_hw_sysfs_cpus.attr, 197 NULL, 198 }; 199 200 static const struct sysfs_ops blk_mq_sysfs_ops = { 201 .show = blk_mq_sysfs_show, 202 .store = blk_mq_sysfs_store, 203 }; 204 205 static const struct sysfs_ops blk_mq_hw_sysfs_ops = { 206 .show = blk_mq_hw_sysfs_show, 207 .store = blk_mq_hw_sysfs_store, 208 }; 209 210 static struct kobj_type blk_mq_ktype = { 211 .sysfs_ops = &blk_mq_sysfs_ops, 212 .release = blk_mq_sysfs_release, 213 }; 214 215 static struct kobj_type blk_mq_ctx_ktype = { 216 .sysfs_ops = &blk_mq_sysfs_ops, 217 .default_attrs = default_ctx_attrs, 218 .release = blk_mq_ctx_sysfs_release, 219 }; 220 221 static struct kobj_type blk_mq_hw_ktype = { 222 .sysfs_ops = &blk_mq_hw_sysfs_ops, 223 .default_attrs = default_hw_ctx_attrs, 224 .release = blk_mq_hw_sysfs_release, 225 }; 226 227 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx) 228 { 229 struct blk_mq_ctx *ctx; 230 int i; 231 232 if (!hctx->nr_ctx) 233 return; 234 235 hctx_for_each_ctx(hctx, ctx, i) 236 kobject_del(&ctx->kobj); 237 238 kobject_del(&hctx->kobj); 239 } 240 241 static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) 242 { 243 struct request_queue *q = hctx->queue; 244 struct blk_mq_ctx *ctx; 245 int i, ret; 246 247 if (!hctx->nr_ctx) 248 return 0; 249 250 ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num); 251 if (ret) 252 return ret; 253 254 hctx_for_each_ctx(hctx, ctx, i) { 255 ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu); 256 if (ret) 257 break; 258 } 259 260 return ret; 261 } 262 263 void blk_mq_unregister_dev(struct device *dev, struct request_queue *q) 264 { 265 struct blk_mq_hw_ctx *hctx; 266 int i; 267 268 lockdep_assert_held(&q->sysfs_lock); 269 270 queue_for_each_hw_ctx(q, hctx, i) 271 blk_mq_unregister_hctx(hctx); 272 273 kobject_uevent(q->mq_kobj, KOBJ_REMOVE); 274 kobject_del(q->mq_kobj); 275 kobject_put(&dev->kobj); 276 277 q->mq_sysfs_init_done = false; 278 } 279 280 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx) 281 { 282 kobject_init(&hctx->kobj, &blk_mq_hw_ktype); 283 } 284 285 void blk_mq_sysfs_deinit(struct request_queue *q) 286 { 287 struct blk_mq_ctx *ctx; 288 int cpu; 289 290 for_each_possible_cpu(cpu) { 291 ctx = per_cpu_ptr(q->queue_ctx, cpu); 292 kobject_put(&ctx->kobj); 293 } 294 kobject_put(q->mq_kobj); 295 } 296 297 void blk_mq_sysfs_init(struct request_queue *q) 298 { 299 struct blk_mq_ctx *ctx; 300 int cpu; 301 302 kobject_init(q->mq_kobj, &blk_mq_ktype); 303 304 for_each_possible_cpu(cpu) { 305 ctx = per_cpu_ptr(q->queue_ctx, cpu); 306 307 kobject_get(q->mq_kobj); 308 kobject_init(&ctx->kobj, &blk_mq_ctx_ktype); 309 } 310 } 311 312 int __blk_mq_register_dev(struct device *dev, struct request_queue *q) 313 { 314 struct blk_mq_hw_ctx *hctx; 315 int ret, i; 316 317 WARN_ON_ONCE(!q->kobj.parent); 318 lockdep_assert_held(&q->sysfs_lock); 319 320 ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq"); 321 if (ret < 0) 322 goto out; 323 324 kobject_uevent(q->mq_kobj, KOBJ_ADD); 325 326 queue_for_each_hw_ctx(q, hctx, i) { 327 ret = blk_mq_register_hctx(hctx); 328 if (ret) 329 goto unreg; 330 } 331 332 q->mq_sysfs_init_done = true; 333 334 out: 335 return ret; 336 337 unreg: 338 while (--i >= 0) 339 blk_mq_unregister_hctx(q->queue_hw_ctx[i]); 340 341 kobject_uevent(q->mq_kobj, KOBJ_REMOVE); 342 kobject_del(q->mq_kobj); 343 kobject_put(&dev->kobj); 344 return ret; 345 } 346 347 int blk_mq_register_dev(struct device *dev, struct request_queue *q) 348 { 349 int ret; 350 351 mutex_lock(&q->sysfs_lock); 352 ret = __blk_mq_register_dev(dev, q); 353 mutex_unlock(&q->sysfs_lock); 354 355 return ret; 356 } 357 358 void blk_mq_sysfs_unregister(struct request_queue *q) 359 { 360 struct blk_mq_hw_ctx *hctx; 361 int i; 362 363 mutex_lock(&q->sysfs_lock); 364 if (!q->mq_sysfs_init_done) 365 goto unlock; 366 367 queue_for_each_hw_ctx(q, hctx, i) 368 blk_mq_unregister_hctx(hctx); 369 370 unlock: 371 mutex_unlock(&q->sysfs_lock); 372 } 373 374 int blk_mq_sysfs_register(struct request_queue *q) 375 { 376 struct blk_mq_hw_ctx *hctx; 377 int i, ret = 0; 378 379 mutex_lock(&q->sysfs_lock); 380 if (!q->mq_sysfs_init_done) 381 goto unlock; 382 383 queue_for_each_hw_ctx(q, hctx, i) { 384 ret = blk_mq_register_hctx(hctx); 385 if (ret) 386 break; 387 } 388 389 unlock: 390 mutex_unlock(&q->sysfs_lock); 391 392 return ret; 393 } 394