1 #include <linux/kernel.h> 2 #include <linux/module.h> 3 #include <linux/backing-dev.h> 4 #include <linux/bio.h> 5 #include <linux/blkdev.h> 6 #include <linux/mm.h> 7 #include <linux/init.h> 8 #include <linux/slab.h> 9 #include <linux/workqueue.h> 10 #include <linux/smp.h> 11 12 #include <linux/blk-mq.h> 13 #include "blk-mq.h" 14 #include "blk-mq-tag.h" 15 16 static void blk_mq_sysfs_release(struct kobject *kobj) 17 { 18 } 19 20 struct blk_mq_ctx_sysfs_entry { 21 struct attribute attr; 22 ssize_t (*show)(struct blk_mq_ctx *, char *); 23 ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t); 24 }; 25 26 struct blk_mq_hw_ctx_sysfs_entry { 27 struct attribute attr; 28 ssize_t (*show)(struct blk_mq_hw_ctx *, char *); 29 ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t); 30 }; 31 32 static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr, 33 char *page) 34 { 35 struct blk_mq_ctx_sysfs_entry *entry; 36 struct blk_mq_ctx *ctx; 37 struct request_queue *q; 38 ssize_t res; 39 40 entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr); 41 ctx = container_of(kobj, struct blk_mq_ctx, kobj); 42 q = ctx->queue; 43 44 if (!entry->show) 45 return -EIO; 46 47 res = -ENOENT; 48 mutex_lock(&q->sysfs_lock); 49 if (!blk_queue_dying(q)) 50 res = entry->show(ctx, page); 51 mutex_unlock(&q->sysfs_lock); 52 return res; 53 } 54 55 static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr, 56 const char *page, size_t length) 57 { 58 struct blk_mq_ctx_sysfs_entry *entry; 59 struct blk_mq_ctx *ctx; 60 struct request_queue *q; 61 ssize_t res; 62 63 entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr); 64 ctx = container_of(kobj, struct blk_mq_ctx, kobj); 65 q = ctx->queue; 66 67 if (!entry->store) 68 return -EIO; 69 70 res = -ENOENT; 71 mutex_lock(&q->sysfs_lock); 72 if (!blk_queue_dying(q)) 73 res = entry->store(ctx, page, length); 74 mutex_unlock(&q->sysfs_lock); 75 return res; 76 } 77 78 static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj, 79 struct attribute *attr, char *page) 80 { 81 struct blk_mq_hw_ctx_sysfs_entry *entry; 82 struct blk_mq_hw_ctx *hctx; 83 struct request_queue *q; 84 ssize_t res; 85 86 entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); 87 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); 88 q = hctx->queue; 89 90 if (!entry->show) 91 return -EIO; 92 93 res = -ENOENT; 94 mutex_lock(&q->sysfs_lock); 95 if (!blk_queue_dying(q)) 96 res = entry->show(hctx, page); 97 mutex_unlock(&q->sysfs_lock); 98 return res; 99 } 100 101 static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj, 102 struct attribute *attr, const char *page, 103 size_t length) 104 { 105 struct blk_mq_hw_ctx_sysfs_entry *entry; 106 struct blk_mq_hw_ctx *hctx; 107 struct request_queue *q; 108 ssize_t res; 109 110 entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); 111 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); 112 q = hctx->queue; 113 114 if (!entry->store) 115 return -EIO; 116 117 res = -ENOENT; 118 mutex_lock(&q->sysfs_lock); 119 if (!blk_queue_dying(q)) 120 res = entry->store(hctx, page, length); 121 mutex_unlock(&q->sysfs_lock); 122 return res; 123 } 124 125 static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx, 126 char *page) 127 { 128 return sprintf(page, "%u\n", hctx->tags->nr_tags); 129 } 130 131 static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx, 132 char *page) 133 { 134 return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags); 135 } 136 137 static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page) 138 { 139 unsigned int i, first = 1; 140 ssize_t ret = 0; 141 142 for_each_cpu(i, hctx->cpumask) { 143 if (first) 144 ret += sprintf(ret + page, "%u", i); 145 else 146 ret += sprintf(ret + page, ", %u", i); 147 148 first = 0; 149 } 150 151 ret += sprintf(ret + page, "\n"); 152 return ret; 153 } 154 155 static struct attribute *default_ctx_attrs[] = { 156 NULL, 157 }; 158 159 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = { 160 .attr = {.name = "nr_tags", .mode = S_IRUGO }, 161 .show = blk_mq_hw_sysfs_nr_tags_show, 162 }; 163 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = { 164 .attr = {.name = "nr_reserved_tags", .mode = S_IRUGO }, 165 .show = blk_mq_hw_sysfs_nr_reserved_tags_show, 166 }; 167 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { 168 .attr = {.name = "cpu_list", .mode = S_IRUGO }, 169 .show = blk_mq_hw_sysfs_cpus_show, 170 }; 171 172 static struct attribute *default_hw_ctx_attrs[] = { 173 &blk_mq_hw_sysfs_nr_tags.attr, 174 &blk_mq_hw_sysfs_nr_reserved_tags.attr, 175 &blk_mq_hw_sysfs_cpus.attr, 176 NULL, 177 }; 178 179 static const struct sysfs_ops blk_mq_sysfs_ops = { 180 .show = blk_mq_sysfs_show, 181 .store = blk_mq_sysfs_store, 182 }; 183 184 static const struct sysfs_ops blk_mq_hw_sysfs_ops = { 185 .show = blk_mq_hw_sysfs_show, 186 .store = blk_mq_hw_sysfs_store, 187 }; 188 189 static struct kobj_type blk_mq_ktype = { 190 .sysfs_ops = &blk_mq_sysfs_ops, 191 .release = blk_mq_sysfs_release, 192 }; 193 194 static struct kobj_type blk_mq_ctx_ktype = { 195 .sysfs_ops = &blk_mq_sysfs_ops, 196 .default_attrs = default_ctx_attrs, 197 .release = blk_mq_sysfs_release, 198 }; 199 200 static struct kobj_type blk_mq_hw_ktype = { 201 .sysfs_ops = &blk_mq_hw_sysfs_ops, 202 .default_attrs = default_hw_ctx_attrs, 203 .release = blk_mq_sysfs_release, 204 }; 205 206 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx) 207 { 208 struct blk_mq_ctx *ctx; 209 int i; 210 211 if (!hctx->nr_ctx) 212 return; 213 214 hctx_for_each_ctx(hctx, ctx, i) 215 kobject_del(&ctx->kobj); 216 217 kobject_del(&hctx->kobj); 218 } 219 220 static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) 221 { 222 struct request_queue *q = hctx->queue; 223 struct blk_mq_ctx *ctx; 224 int i, ret; 225 226 if (!hctx->nr_ctx) 227 return 0; 228 229 ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num); 230 if (ret) 231 return ret; 232 233 hctx_for_each_ctx(hctx, ctx, i) { 234 ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu); 235 if (ret) 236 break; 237 } 238 239 return ret; 240 } 241 242 static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q) 243 { 244 struct blk_mq_hw_ctx *hctx; 245 struct blk_mq_ctx *ctx; 246 int i, j; 247 248 queue_for_each_hw_ctx(q, hctx, i) { 249 blk_mq_unregister_hctx(hctx); 250 251 hctx_for_each_ctx(hctx, ctx, j) 252 kobject_put(&ctx->kobj); 253 254 kobject_put(&hctx->kobj); 255 } 256 257 blk_mq_debugfs_unregister_hctxs(q); 258 259 kobject_uevent(&q->mq_kobj, KOBJ_REMOVE); 260 kobject_del(&q->mq_kobj); 261 kobject_put(&q->mq_kobj); 262 263 kobject_put(&dev->kobj); 264 265 q->mq_sysfs_init_done = false; 266 } 267 268 void blk_mq_unregister_dev(struct device *dev, struct request_queue *q) 269 { 270 blk_mq_disable_hotplug(); 271 __blk_mq_unregister_dev(dev, q); 272 blk_mq_enable_hotplug(); 273 } 274 275 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx) 276 { 277 kobject_init(&hctx->kobj, &blk_mq_hw_ktype); 278 } 279 280 static void blk_mq_sysfs_init(struct request_queue *q) 281 { 282 struct blk_mq_ctx *ctx; 283 int cpu; 284 285 kobject_init(&q->mq_kobj, &blk_mq_ktype); 286 287 for_each_possible_cpu(cpu) { 288 ctx = per_cpu_ptr(q->queue_ctx, cpu); 289 kobject_init(&ctx->kobj, &blk_mq_ctx_ktype); 290 } 291 } 292 293 int blk_mq_register_dev(struct device *dev, struct request_queue *q) 294 { 295 struct blk_mq_hw_ctx *hctx; 296 int ret, i; 297 298 blk_mq_disable_hotplug(); 299 300 blk_mq_sysfs_init(q); 301 302 ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq"); 303 if (ret < 0) 304 goto out; 305 306 kobject_uevent(&q->mq_kobj, KOBJ_ADD); 307 308 blk_mq_debugfs_register(q, kobject_name(&dev->kobj)); 309 310 queue_for_each_hw_ctx(q, hctx, i) { 311 ret = blk_mq_register_hctx(hctx); 312 if (ret) 313 break; 314 } 315 316 if (ret) 317 __blk_mq_unregister_dev(dev, q); 318 else 319 q->mq_sysfs_init_done = true; 320 out: 321 blk_mq_enable_hotplug(); 322 323 return ret; 324 } 325 EXPORT_SYMBOL_GPL(blk_mq_register_dev); 326 327 void blk_mq_sysfs_unregister(struct request_queue *q) 328 { 329 struct blk_mq_hw_ctx *hctx; 330 int i; 331 332 if (!q->mq_sysfs_init_done) 333 return; 334 335 blk_mq_debugfs_unregister_hctxs(q); 336 337 queue_for_each_hw_ctx(q, hctx, i) 338 blk_mq_unregister_hctx(hctx); 339 } 340 341 int blk_mq_sysfs_register(struct request_queue *q) 342 { 343 struct blk_mq_hw_ctx *hctx; 344 int i, ret = 0; 345 346 if (!q->mq_sysfs_init_done) 347 return ret; 348 349 blk_mq_debugfs_register_hctxs(q); 350 351 queue_for_each_hw_ctx(q, hctx, i) { 352 ret = blk_mq_register_hctx(hctx); 353 if (ret) 354 break; 355 } 356 357 return ret; 358 } 359