1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/module.h> 4 #include <linux/backing-dev.h> 5 #include <linux/bio.h> 6 #include <linux/blkdev.h> 7 #include <linux/mm.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/workqueue.h> 11 #include <linux/smp.h> 12 13 #include <linux/blk-mq.h> 14 #include "blk.h" 15 #include "blk-mq.h" 16 #include "blk-mq-tag.h" 17 18 static void blk_mq_sysfs_release(struct kobject *kobj) 19 { 20 struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj); 21 22 free_percpu(ctxs->queue_ctx); 23 kfree(ctxs); 24 } 25 26 static void blk_mq_ctx_sysfs_release(struct kobject *kobj) 27 { 28 struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj); 29 30 /* ctx->ctxs won't be released until all ctx are freed */ 31 kobject_put(&ctx->ctxs->kobj); 32 } 33 34 static void blk_mq_hw_sysfs_release(struct kobject *kobj) 35 { 36 struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx, 37 kobj); 38 39 blk_free_flush_queue(hctx->fq); 40 sbitmap_free(&hctx->ctx_map); 41 free_cpumask_var(hctx->cpumask); 42 kfree(hctx->ctxs); 43 kfree(hctx); 44 } 45 46 struct blk_mq_hw_ctx_sysfs_entry { 47 struct attribute attr; 48 ssize_t (*show)(struct blk_mq_hw_ctx *, char *); 49 ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t); 50 }; 51 52 static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj, 53 struct attribute *attr, char *page) 54 { 55 struct blk_mq_hw_ctx_sysfs_entry *entry; 56 struct blk_mq_hw_ctx *hctx; 57 struct request_queue *q; 58 ssize_t res; 59 60 entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); 61 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); 62 q = hctx->queue; 63 64 if (!entry->show) 65 return -EIO; 66 67 mutex_lock(&q->sysfs_lock); 68 res = entry->show(hctx, page); 69 mutex_unlock(&q->sysfs_lock); 70 return res; 71 } 72 73 static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj, 74 struct attribute *attr, const char *page, 75 size_t length) 76 { 77 struct blk_mq_hw_ctx_sysfs_entry *entry; 78 struct blk_mq_hw_ctx *hctx; 79 struct request_queue *q; 80 ssize_t res; 81 82 entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); 83 hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); 84 q = hctx->queue; 85 86 if (!entry->store) 87 return -EIO; 88 89 mutex_lock(&q->sysfs_lock); 90 res = entry->store(hctx, page, length); 91 mutex_unlock(&q->sysfs_lock); 92 return res; 93 } 94 95 static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx, 96 char *page) 97 { 98 return sprintf(page, "%u\n", hctx->tags->nr_tags); 99 } 100 101 static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx, 102 char *page) 103 { 104 return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags); 105 } 106 107 static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page) 108 { 109 const size_t size = PAGE_SIZE - 1; 110 unsigned int i, first = 1; 111 int ret = 0, pos = 0; 112 113 for_each_cpu(i, hctx->cpumask) { 114 if (first) 115 ret = snprintf(pos + page, size - pos, "%u", i); 116 else 117 ret = snprintf(pos + page, size - pos, ", %u", i); 118 119 if (ret >= size - pos) 120 break; 121 122 first = 0; 123 pos += ret; 124 } 125 126 ret = snprintf(pos + page, size + 1 - pos, "\n"); 127 return pos + ret; 128 } 129 130 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = { 131 .attr = {.name = "nr_tags", .mode = 0444 }, 132 .show = blk_mq_hw_sysfs_nr_tags_show, 133 }; 134 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = { 135 .attr = {.name = "nr_reserved_tags", .mode = 0444 }, 136 .show = blk_mq_hw_sysfs_nr_reserved_tags_show, 137 }; 138 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { 139 .attr = {.name = "cpu_list", .mode = 0444 }, 140 .show = blk_mq_hw_sysfs_cpus_show, 141 }; 142 143 static struct attribute *default_hw_ctx_attrs[] = { 144 &blk_mq_hw_sysfs_nr_tags.attr, 145 &blk_mq_hw_sysfs_nr_reserved_tags.attr, 146 &blk_mq_hw_sysfs_cpus.attr, 147 NULL, 148 }; 149 ATTRIBUTE_GROUPS(default_hw_ctx); 150 151 static const struct sysfs_ops blk_mq_hw_sysfs_ops = { 152 .show = blk_mq_hw_sysfs_show, 153 .store = blk_mq_hw_sysfs_store, 154 }; 155 156 static struct kobj_type blk_mq_ktype = { 157 .release = blk_mq_sysfs_release, 158 }; 159 160 static struct kobj_type blk_mq_ctx_ktype = { 161 .release = blk_mq_ctx_sysfs_release, 162 }; 163 164 static struct kobj_type blk_mq_hw_ktype = { 165 .sysfs_ops = &blk_mq_hw_sysfs_ops, 166 .default_groups = default_hw_ctx_groups, 167 .release = blk_mq_hw_sysfs_release, 168 }; 169 170 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx) 171 { 172 struct blk_mq_ctx *ctx; 173 int i; 174 175 if (!hctx->nr_ctx) 176 return; 177 178 hctx_for_each_ctx(hctx, ctx, i) 179 kobject_del(&ctx->kobj); 180 181 kobject_del(&hctx->kobj); 182 } 183 184 static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) 185 { 186 struct request_queue *q = hctx->queue; 187 struct blk_mq_ctx *ctx; 188 int i, j, ret; 189 190 if (!hctx->nr_ctx) 191 return 0; 192 193 ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num); 194 if (ret) 195 return ret; 196 197 hctx_for_each_ctx(hctx, ctx, i) { 198 ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu); 199 if (ret) 200 goto out; 201 } 202 203 return 0; 204 out: 205 hctx_for_each_ctx(hctx, ctx, j) { 206 if (j < i) 207 kobject_del(&ctx->kobj); 208 } 209 kobject_del(&hctx->kobj); 210 return ret; 211 } 212 213 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx) 214 { 215 kobject_init(&hctx->kobj, &blk_mq_hw_ktype); 216 } 217 218 void blk_mq_sysfs_deinit(struct request_queue *q) 219 { 220 struct blk_mq_ctx *ctx; 221 int cpu; 222 223 for_each_possible_cpu(cpu) { 224 ctx = per_cpu_ptr(q->queue_ctx, cpu); 225 kobject_put(&ctx->kobj); 226 } 227 kobject_put(q->mq_kobj); 228 } 229 230 void blk_mq_sysfs_init(struct request_queue *q) 231 { 232 struct blk_mq_ctx *ctx; 233 int cpu; 234 235 kobject_init(q->mq_kobj, &blk_mq_ktype); 236 237 for_each_possible_cpu(cpu) { 238 ctx = per_cpu_ptr(q->queue_ctx, cpu); 239 240 kobject_get(q->mq_kobj); 241 kobject_init(&ctx->kobj, &blk_mq_ctx_ktype); 242 } 243 } 244 245 int blk_mq_sysfs_register(struct gendisk *disk) 246 { 247 struct request_queue *q = disk->queue; 248 struct blk_mq_hw_ctx *hctx; 249 unsigned long i, j; 250 int ret; 251 252 lockdep_assert_held(&q->sysfs_dir_lock); 253 254 ret = kobject_add(q->mq_kobj, &disk_to_dev(disk)->kobj, "mq"); 255 if (ret < 0) 256 goto out; 257 258 kobject_uevent(q->mq_kobj, KOBJ_ADD); 259 260 queue_for_each_hw_ctx(q, hctx, i) { 261 ret = blk_mq_register_hctx(hctx); 262 if (ret) 263 goto unreg; 264 } 265 266 q->mq_sysfs_init_done = true; 267 268 out: 269 return ret; 270 271 unreg: 272 queue_for_each_hw_ctx(q, hctx, j) { 273 if (j < i) 274 blk_mq_unregister_hctx(hctx); 275 } 276 277 kobject_uevent(q->mq_kobj, KOBJ_REMOVE); 278 kobject_del(q->mq_kobj); 279 return ret; 280 } 281 282 void blk_mq_sysfs_unregister(struct gendisk *disk) 283 { 284 struct request_queue *q = disk->queue; 285 struct blk_mq_hw_ctx *hctx; 286 unsigned long i; 287 288 lockdep_assert_held(&q->sysfs_dir_lock); 289 290 queue_for_each_hw_ctx(q, hctx, i) 291 blk_mq_unregister_hctx(hctx); 292 293 kobject_uevent(q->mq_kobj, KOBJ_REMOVE); 294 kobject_del(q->mq_kobj); 295 296 q->mq_sysfs_init_done = false; 297 } 298 299 void blk_mq_sysfs_unregister_hctxs(struct request_queue *q) 300 { 301 struct blk_mq_hw_ctx *hctx; 302 unsigned long i; 303 304 mutex_lock(&q->sysfs_dir_lock); 305 if (!q->mq_sysfs_init_done) 306 goto unlock; 307 308 queue_for_each_hw_ctx(q, hctx, i) 309 blk_mq_unregister_hctx(hctx); 310 311 unlock: 312 mutex_unlock(&q->sysfs_dir_lock); 313 } 314 315 int blk_mq_sysfs_register_hctxs(struct request_queue *q) 316 { 317 struct blk_mq_hw_ctx *hctx; 318 unsigned long i; 319 int ret = 0; 320 321 mutex_lock(&q->sysfs_dir_lock); 322 if (!q->mq_sysfs_init_done) 323 goto unlock; 324 325 queue_for_each_hw_ctx(q, hctx, i) { 326 ret = blk_mq_register_hctx(hctx); 327 if (ret) 328 break; 329 } 330 331 unlock: 332 mutex_unlock(&q->sysfs_dir_lock); 333 334 return ret; 335 } 336