blk-mq.c (a944a1fb9a8286fb9e4bbd0180ebd53353c7f412) blk-mq.c (93f221ae08381e994ac9f9ff6aa743e612e49718)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Block multiqueue core code
4 *
5 * Copyright (C) 2013-2014 Jens Axboe
6 * Copyright (C) 2013-2014 Christoph Hellwig
7 */
8#include <linux/kernel.h>

--- 505 unchanged lines hidden (view full) ---

514 if (rq->elv.icq) {
515 put_io_context(rq->elv.icq->ioc);
516 rq->elv.icq = NULL;
517 }
518 }
519
520 ctx->rq_completed[rq_is_sync(rq)]++;
521 if (rq->rq_flags & RQF_MQ_INFLIGHT)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Block multiqueue core code
4 *
5 * Copyright (C) 2013-2014 Jens Axboe
6 * Copyright (C) 2013-2014 Christoph Hellwig
7 */
8#include <linux/kernel.h>

--- 505 unchanged lines hidden (view full) ---

514 if (rq->elv.icq) {
515 put_io_context(rq->elv.icq->ioc);
516 rq->elv.icq = NULL;
517 }
518 }
519
520 ctx->rq_completed[rq_is_sync(rq)]++;
521 if (rq->rq_flags & RQF_MQ_INFLIGHT)
522 atomic_dec(&hctx->nr_active);
522 __blk_mq_dec_active_requests(hctx);
523
524 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
525 laptop_io_completion(q->backing_dev_info);
526
527 rq_qos_done(q, rq);
528
529 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
530 if (refcount_dec_and_test(&rq->ref))

--- 560 unchanged lines hidden (view full) ---

1091 if (!queued)
1092 return 0;
1093
1094 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
1095}
1096
1097static bool __blk_mq_get_driver_tag(struct request *rq)
1098{
523
524 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
525 laptop_io_completion(q->backing_dev_info);
526
527 rq_qos_done(q, rq);
528
529 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
530 if (refcount_dec_and_test(&rq->ref))

--- 560 unchanged lines hidden (view full) ---

1091 if (!queued)
1092 return 0;
1093
1094 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
1095}
1096
1097static bool __blk_mq_get_driver_tag(struct request *rq)
1098{
1099 struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
1099 struct sbitmap_queue *bt = rq->mq_hctx->tags->bitmap_tags;
1100 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
1101 int tag;
1102
1103 blk_mq_tag_busy(rq->mq_hctx);
1104
1105 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1100 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
1101 int tag;
1102
1103 blk_mq_tag_busy(rq->mq_hctx);
1104
1105 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1106 bt = &rq->mq_hctx->tags->breserved_tags;
1106 bt = rq->mq_hctx->tags->breserved_tags;
1107 tag_offset = 0;
1107 tag_offset = 0;
1108 } else {
1109 if (!hctx_may_queue(rq->mq_hctx, bt))
1110 return false;
1108 }
1109
1111 }
1112
1110 if (!hctx_may_queue(rq->mq_hctx, bt))
1111 return false;
1112 tag = __sbitmap_queue_get(bt);
1113 if (tag == BLK_MQ_NO_TAG)
1114 return false;
1115
1116 rq->tag = tag + tag_offset;
1117 return true;
1118}
1119
1120static bool blk_mq_get_driver_tag(struct request *rq)
1121{
1122 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1123
1124 if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
1125 return false;
1126
1113 tag = __sbitmap_queue_get(bt);
1114 if (tag == BLK_MQ_NO_TAG)
1115 return false;
1116
1117 rq->tag = tag + tag_offset;
1118 return true;
1119}
1120
1121static bool blk_mq_get_driver_tag(struct request *rq)
1122{
1123 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1124
1125 if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
1126 return false;
1127
1127 if ((hctx->flags & BLK_MQ_F_TAG_SHARED) &&
1128 if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1128 !(rq->rq_flags & RQF_MQ_INFLIGHT)) {
1129 rq->rq_flags |= RQF_MQ_INFLIGHT;
1129 !(rq->rq_flags & RQF_MQ_INFLIGHT)) {
1130 rq->rq_flags |= RQF_MQ_INFLIGHT;
1130 atomic_inc(&hctx->nr_active);
1131 __blk_mq_inc_active_requests(hctx);
1131 }
1132 hctx->tags->rqs[rq->tag] = rq;
1133 return true;
1134}
1135
1136static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1137 int flags, void *key)
1138{
1139 struct blk_mq_hw_ctx *hctx;
1140
1141 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1142
1143 spin_lock(&hctx->dispatch_wait_lock);
1144 if (!list_empty(&wait->entry)) {
1145 struct sbitmap_queue *sbq;
1146
1147 list_del_init(&wait->entry);
1132 }
1133 hctx->tags->rqs[rq->tag] = rq;
1134 return true;
1135}
1136
1137static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1138 int flags, void *key)
1139{
1140 struct blk_mq_hw_ctx *hctx;
1141
1142 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1143
1144 spin_lock(&hctx->dispatch_wait_lock);
1145 if (!list_empty(&wait->entry)) {
1146 struct sbitmap_queue *sbq;
1147
1148 list_del_init(&wait->entry);
1148 sbq = &hctx->tags->bitmap_tags;
1149 sbq = hctx->tags->bitmap_tags;
1149 atomic_dec(&sbq->ws_active);
1150 }
1151 spin_unlock(&hctx->dispatch_wait_lock);
1152
1153 blk_mq_run_hw_queue(hctx, true);
1154 return 1;
1155}
1156
1157/*
1158 * Mark us waiting for a tag. For shared tags, this involves hooking us into
1159 * the tag wakeups. For non-shared tags, we can simply mark us needing a
1160 * restart. For both cases, take care to check the condition again after
1161 * marking us as waiting.
1162 */
1163static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1164 struct request *rq)
1165{
1150 atomic_dec(&sbq->ws_active);
1151 }
1152 spin_unlock(&hctx->dispatch_wait_lock);
1153
1154 blk_mq_run_hw_queue(hctx, true);
1155 return 1;
1156}
1157
1158/*
1159 * Mark us waiting for a tag. For shared tags, this involves hooking us into
1160 * the tag wakeups. For non-shared tags, we can simply mark us needing a
1161 * restart. For both cases, take care to check the condition again after
1162 * marking us as waiting.
1163 */
1164static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1165 struct request *rq)
1166{
1166 struct sbitmap_queue *sbq = &hctx->tags->bitmap_tags;
1167 struct sbitmap_queue *sbq = hctx->tags->bitmap_tags;
1167 struct wait_queue_head *wq;
1168 wait_queue_entry_t *wait;
1169 bool ret;
1170
1168 struct wait_queue_head *wq;
1169 wait_queue_entry_t *wait;
1170 bool ret;
1171
1171 if (!(hctx->flags & BLK_MQ_F_TAG_SHARED)) {
1172 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
1172 blk_mq_sched_mark_restart_hctx(hctx);
1173
1174 /*
1175 * It's possible that a tag was freed in the window between the
1176 * allocation failure and adding the hardware queue to the wait
1177 * queue.
1178 *
1179 * Don't clear RESTART here, someone else could have set it.

--- 227 unchanged lines hidden (view full) ---

1407 }
1408 } while (!list_empty(list));
1409out:
1410 if (!list_empty(&zone_list))
1411 list_splice_tail_init(&zone_list, list);
1412
1413 hctx->dispatched[queued_to_index(queued)]++;
1414
1173 blk_mq_sched_mark_restart_hctx(hctx);
1174
1175 /*
1176 * It's possible that a tag was freed in the window between the
1177 * allocation failure and adding the hardware queue to the wait
1178 * queue.
1179 *
1180 * Don't clear RESTART here, someone else could have set it.

--- 227 unchanged lines hidden (view full) ---

1408 }
1409 } while (!list_empty(list));
1410out:
1411 if (!list_empty(&zone_list))
1412 list_splice_tail_init(&zone_list, list);
1413
1414 hctx->dispatched[queued_to_index(queued)]++;
1415
1415 /* If we didn't flush the entire list, we could have told the driver
1416 * there was more coming, but that turned out to be a lie.
1417 */
1418 if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued)
1419 q->mq_ops->commit_rqs(hctx);
1420 /*
1421 * Any items that need requeuing? Stuff them into hctx->dispatch,
1422 * that is where we will continue on next queue run.
1423 */
1424 if (!list_empty(list)) {
1425 bool needs_restart;
1426 /* For non-shared tags, the RESTART check will suffice */
1427 bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
1416 /*
1417 * Any items that need requeuing? Stuff them into hctx->dispatch,
1418 * that is where we will continue on next queue run.
1419 */
1420 if (!list_empty(list)) {
1421 bool needs_restart;
1422 /* For non-shared tags, the RESTART check will suffice */
1423 bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
1428 (hctx->flags & BLK_MQ_F_TAG_SHARED);
1424 (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
1429 bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
1430
1431 blk_mq_release_budgets(q, nr_budgets);
1432
1425 bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
1426
1427 blk_mq_release_budgets(q, nr_budgets);
1428
1429 /*
1430 * If we didn't flush the entire list, we could have told
1431 * the driver there was more coming, but that turned out to
1432 * be a lie.
1433 */
1434 if (q->mq_ops->commit_rqs && queued)
1435 q->mq_ops->commit_rqs(hctx);
1436
1433 spin_lock(&hctx->lock);
1434 list_splice_tail_init(list, &hctx->dispatch);
1435 spin_unlock(&hctx->lock);
1436
1437 /*
1438 * Order adding requests to hctx->dispatch and checking
1439 * SCHED_RESTART flag. The pair of this smp_mb() is the one
1440 * in blk_mq_sched_restart(). Avoid restart code path to

--- 490 unchanged lines hidden (view full) ---

1931 blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list,
1932 from_schedule);
1933 } while(!list_empty(&list));
1934}
1935
1936static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
1937 unsigned int nr_segs)
1938{
1437 spin_lock(&hctx->lock);
1438 list_splice_tail_init(list, &hctx->dispatch);
1439 spin_unlock(&hctx->lock);
1440
1441 /*
1442 * Order adding requests to hctx->dispatch and checking
1443 * SCHED_RESTART flag. The pair of this smp_mb() is the one
1444 * in blk_mq_sched_restart(). Avoid restart code path to

--- 490 unchanged lines hidden (view full) ---

1935 blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list,
1936 from_schedule);
1937 } while(!list_empty(&list));
1938}
1939
1940static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
1941 unsigned int nr_segs)
1942{
1943 int err;
1944
1939 if (bio->bi_opf & REQ_RAHEAD)
1940 rq->cmd_flags |= REQ_FAILFAST_MASK;
1941
1942 rq->__sector = bio->bi_iter.bi_sector;
1943 rq->write_hint = bio->bi_write_hint;
1944 blk_rq_bio_prep(rq, bio, nr_segs);
1945 if (bio->bi_opf & REQ_RAHEAD)
1946 rq->cmd_flags |= REQ_FAILFAST_MASK;
1947
1948 rq->__sector = bio->bi_iter.bi_sector;
1949 rq->write_hint = bio->bi_write_hint;
1950 blk_rq_bio_prep(rq, bio, nr_segs);
1945 blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
1946
1951
1952 /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
1953 err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
1954 WARN_ON_ONCE(err);
1955
1947 blk_account_io_start(rq);
1948}
1949
1950static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
1951 struct request *rq,
1952 blk_qc_t *cookie, bool last)
1953{
1954 struct request_queue *q = rq->q;

--- 116 unchanged lines hidden (view full) ---

2071
2072 return ret;
2073}
2074
2075void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2076 struct list_head *list)
2077{
2078 int queued = 0;
1956 blk_account_io_start(rq);
1957}
1958
1959static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
1960 struct request *rq,
1961 blk_qc_t *cookie, bool last)
1962{
1963 struct request_queue *q = rq->q;

--- 116 unchanged lines hidden (view full) ---

2080
2081 return ret;
2082}
2083
2084void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2085 struct list_head *list)
2086{
2087 int queued = 0;
2079 int errors = 0;
2080
2081 while (!list_empty(list)) {
2082 blk_status_t ret;
2083 struct request *rq = list_first_entry(list, struct request,
2084 queuelist);
2085
2086 list_del_init(&rq->queuelist);
2087 ret = blk_mq_request_issue_directly(rq, list_empty(list));
2088 if (ret != BLK_STS_OK) {
2089 if (ret == BLK_STS_RESOURCE ||
2090 ret == BLK_STS_DEV_RESOURCE) {
2091 blk_mq_request_bypass_insert(rq, false,
2092 list_empty(list));
2093 break;
2094 }
2095 blk_mq_end_request(rq, ret);
2088
2089 while (!list_empty(list)) {
2090 blk_status_t ret;
2091 struct request *rq = list_first_entry(list, struct request,
2092 queuelist);
2093
2094 list_del_init(&rq->queuelist);
2095 ret = blk_mq_request_issue_directly(rq, list_empty(list));
2096 if (ret != BLK_STS_OK) {
2097 if (ret == BLK_STS_RESOURCE ||
2098 ret == BLK_STS_DEV_RESOURCE) {
2099 blk_mq_request_bypass_insert(rq, false,
2100 list_empty(list));
2101 break;
2102 }
2103 blk_mq_end_request(rq, ret);
2096 errors++;
2097 } else
2098 queued++;
2099 }
2100
2101 /*
2102 * If we didn't flush the entire list, we could have told
2103 * the driver there was more coming, but that turned out to
2104 * be a lie.
2105 */
2104 } else
2105 queued++;
2106 }
2107
2108 /*
2109 * If we didn't flush the entire list, we could have told
2110 * the driver there was more coming, but that turned out to
2111 * be a lie.
2112 */
2106 if ((!list_empty(list) || errors) &&
2107 hctx->queue->mq_ops->commit_rqs && queued)
2113 if (!list_empty(list) && hctx->queue->mq_ops->commit_rqs && queued)
2108 hctx->queue->mq_ops->commit_rqs(hctx);
2109}
2110
2111static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
2112{
2113 list_add_tail(&rq->queuelist, &plug->mq_list);
2114 plug->rq_count++;
2115 if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) {

--- 175 unchanged lines hidden (view full) ---

2291 * Remove kmemleak object previously allocated in
2292 * blk_mq_alloc_rqs().
2293 */
2294 kmemleak_free(page_address(page));
2295 __free_pages(page, page->private);
2296 }
2297}
2298
2114 hctx->queue->mq_ops->commit_rqs(hctx);
2115}
2116
2117static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
2118{
2119 list_add_tail(&rq->queuelist, &plug->mq_list);
2120 plug->rq_count++;
2121 if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) {

--- 175 unchanged lines hidden (view full) ---

2297 * Remove kmemleak object previously allocated in
2298 * blk_mq_alloc_rqs().
2299 */
2300 kmemleak_free(page_address(page));
2301 __free_pages(page, page->private);
2302 }
2303}
2304
2299void blk_mq_free_rq_map(struct blk_mq_tags *tags)
2305void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags)
2300{
2301 kfree(tags->rqs);
2302 tags->rqs = NULL;
2303 kfree(tags->static_rqs);
2304 tags->static_rqs = NULL;
2305
2306{
2307 kfree(tags->rqs);
2308 tags->rqs = NULL;
2309 kfree(tags->static_rqs);
2310 tags->static_rqs = NULL;
2311
2306 blk_mq_free_tags(tags);
2312 blk_mq_free_tags(tags, flags);
2307}
2308
2309struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
2310 unsigned int hctx_idx,
2311 unsigned int nr_tags,
2313}
2314
2315struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
2316 unsigned int hctx_idx,
2317 unsigned int nr_tags,
2312 unsigned int reserved_tags)
2318 unsigned int reserved_tags,
2319 unsigned int flags)
2313{
2314 struct blk_mq_tags *tags;
2315 int node;
2316
2317 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
2318 if (node == NUMA_NO_NODE)
2319 node = set->numa_node;
2320
2320{
2321 struct blk_mq_tags *tags;
2322 int node;
2323
2324 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
2325 if (node == NUMA_NO_NODE)
2326 node = set->numa_node;
2327
2321 tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
2322 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
2328 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags);
2323 if (!tags)
2324 return NULL;
2325
2326 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2327 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2328 node);
2329 if (!tags->rqs) {
2329 if (!tags)
2330 return NULL;
2331
2332 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2333 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2334 node);
2335 if (!tags->rqs) {
2330 blk_mq_free_tags(tags);
2336 blk_mq_free_tags(tags, flags);
2331 return NULL;
2332 }
2333
2334 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2335 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2336 node);
2337 if (!tags->static_rqs) {
2338 kfree(tags->rqs);
2337 return NULL;
2338 }
2339
2340 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2341 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2342 node);
2343 if (!tags->static_rqs) {
2344 kfree(tags->rqs);
2339 blk_mq_free_tags(tags);
2345 blk_mq_free_tags(tags, flags);
2340 return NULL;
2341 }
2342
2343 return tags;
2344}
2345
2346static size_t order_to_size(unsigned int order)
2347{

--- 307 unchanged lines hidden (view full) ---

2655 hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node);
2656 if (!hctx)
2657 goto fail_alloc_hctx;
2658
2659 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
2660 goto free_hctx;
2661
2662 atomic_set(&hctx->nr_active, 0);
2346 return NULL;
2347 }
2348
2349 return tags;
2350}
2351
2352static size_t order_to_size(unsigned int order)
2353{

--- 307 unchanged lines hidden (view full) ---

2661 hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node);
2662 if (!hctx)
2663 goto fail_alloc_hctx;
2664
2665 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
2666 goto free_hctx;
2667
2668 atomic_set(&hctx->nr_active, 0);
2669 atomic_set(&hctx->elevator_queued, 0);
2663 if (node == NUMA_NO_NODE)
2664 node = set->numa_node;
2665 hctx->numa_node = node;
2666
2667 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
2668 spin_lock_init(&hctx->lock);
2669 INIT_LIST_HEAD(&hctx->dispatch);
2670 hctx->queue = q;
2670 if (node == NUMA_NO_NODE)
2671 node = set->numa_node;
2672 hctx->numa_node = node;
2673
2674 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
2675 spin_lock_init(&hctx->lock);
2676 INIT_LIST_HEAD(&hctx->dispatch);
2677 hctx->queue = q;
2671 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
2678 hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
2672
2673 INIT_LIST_HEAD(&hctx->hctx_list);
2674
2675 /*
2676 * Allocate space for all possible cpus to avoid allocation at
2677 * runtime
2678 */
2679 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),

--- 60 unchanged lines hidden (view full) ---

2740 hctx->numa_node = local_memory_node(cpu_to_node(i));
2741 }
2742 }
2743}
2744
2745static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set,
2746 int hctx_idx)
2747{
2679
2680 INIT_LIST_HEAD(&hctx->hctx_list);
2681
2682 /*
2683 * Allocate space for all possible cpus to avoid allocation at
2684 * runtime
2685 */
2686 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),

--- 60 unchanged lines hidden (view full) ---

2747 hctx->numa_node = local_memory_node(cpu_to_node(i));
2748 }
2749 }
2750}
2751
2752static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set,
2753 int hctx_idx)
2754{
2755 unsigned int flags = set->flags;
2748 int ret = 0;
2749
2750 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
2756 int ret = 0;
2757
2758 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
2751 set->queue_depth, set->reserved_tags);
2759 set->queue_depth, set->reserved_tags, flags);
2752 if (!set->tags[hctx_idx])
2753 return false;
2754
2755 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
2756 set->queue_depth);
2757 if (!ret)
2758 return true;
2759
2760 if (!set->tags[hctx_idx])
2761 return false;
2762
2763 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
2764 set->queue_depth);
2765 if (!ret)
2766 return true;
2767
2760 blk_mq_free_rq_map(set->tags[hctx_idx]);
2768 blk_mq_free_rq_map(set->tags[hctx_idx], flags);
2761 set->tags[hctx_idx] = NULL;
2762 return false;
2763}
2764
2765static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
2766 unsigned int hctx_idx)
2767{
2769 set->tags[hctx_idx] = NULL;
2770 return false;
2771}
2772
2773static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
2774 unsigned int hctx_idx)
2775{
2776 unsigned int flags = set->flags;
2777
2768 if (set->tags && set->tags[hctx_idx]) {
2769 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2778 if (set->tags && set->tags[hctx_idx]) {
2779 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2770 blk_mq_free_rq_map(set->tags[hctx_idx]);
2780 blk_mq_free_rq_map(set->tags[hctx_idx], flags);
2771 set->tags[hctx_idx] = NULL;
2772 }
2773}
2774
2775static void blk_mq_map_swqueue(struct request_queue *q)
2776{
2777 unsigned int i, j, hctx_idx;
2778 struct blk_mq_hw_ctx *hctx;

--- 101 unchanged lines hidden (view full) ---

2880 */
2881static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2882{
2883 struct blk_mq_hw_ctx *hctx;
2884 int i;
2885
2886 queue_for_each_hw_ctx(q, hctx, i) {
2887 if (shared)
2781 set->tags[hctx_idx] = NULL;
2782 }
2783}
2784
2785static void blk_mq_map_swqueue(struct request_queue *q)
2786{
2787 unsigned int i, j, hctx_idx;
2788 struct blk_mq_hw_ctx *hctx;

--- 101 unchanged lines hidden (view full) ---

2890 */
2891static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2892{
2893 struct blk_mq_hw_ctx *hctx;
2894 int i;
2895
2896 queue_for_each_hw_ctx(q, hctx, i) {
2897 if (shared)
2888 hctx->flags |= BLK_MQ_F_TAG_SHARED;
2898 hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
2889 else
2899 else
2890 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
2900 hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
2891 }
2892}
2893
2901 }
2902}
2903
2894static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set,
2895 bool shared)
2904static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
2905 bool shared)
2896{
2897 struct request_queue *q;
2898
2899 lockdep_assert_held(&set->tag_list_lock);
2900
2901 list_for_each_entry(q, &set->tag_list, tag_set_list) {
2902 blk_mq_freeze_queue(q);
2903 queue_set_hctx_shared(q, shared);

--- 4 unchanged lines hidden (view full) ---

2908static void blk_mq_del_queue_tag_set(struct request_queue *q)
2909{
2910 struct blk_mq_tag_set *set = q->tag_set;
2911
2912 mutex_lock(&set->tag_list_lock);
2913 list_del(&q->tag_set_list);
2914 if (list_is_singular(&set->tag_list)) {
2915 /* just transitioned to unshared */
2906{
2907 struct request_queue *q;
2908
2909 lockdep_assert_held(&set->tag_list_lock);
2910
2911 list_for_each_entry(q, &set->tag_list, tag_set_list) {
2912 blk_mq_freeze_queue(q);
2913 queue_set_hctx_shared(q, shared);

--- 4 unchanged lines hidden (view full) ---

2918static void blk_mq_del_queue_tag_set(struct request_queue *q)
2919{
2920 struct blk_mq_tag_set *set = q->tag_set;
2921
2922 mutex_lock(&set->tag_list_lock);
2923 list_del(&q->tag_set_list);
2924 if (list_is_singular(&set->tag_list)) {
2925 /* just transitioned to unshared */
2916 set->flags &= ~BLK_MQ_F_TAG_SHARED;
2926 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
2917 /* update existing queue */
2927 /* update existing queue */
2918 blk_mq_update_tag_set_depth(set, false);
2928 blk_mq_update_tag_set_shared(set, false);
2919 }
2920 mutex_unlock(&set->tag_list_lock);
2921 INIT_LIST_HEAD(&q->tag_set_list);
2922}
2923
2924static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
2925 struct request_queue *q)
2926{
2927 mutex_lock(&set->tag_list_lock);
2928
2929 /*
2930 * Check to see if we're transitioning to shared (from 1 to 2 queues).
2931 */
2932 if (!list_empty(&set->tag_list) &&
2929 }
2930 mutex_unlock(&set->tag_list_lock);
2931 INIT_LIST_HEAD(&q->tag_set_list);
2932}
2933
2934static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
2935 struct request_queue *q)
2936{
2937 mutex_lock(&set->tag_list_lock);
2938
2939 /*
2940 * Check to see if we're transitioning to shared (from 1 to 2 queues).
2941 */
2942 if (!list_empty(&set->tag_list) &&
2933 !(set->flags & BLK_MQ_F_TAG_SHARED)) {
2934 set->flags |= BLK_MQ_F_TAG_SHARED;
2943 !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
2944 set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
2935 /* update existing queue */
2945 /* update existing queue */
2936 blk_mq_update_tag_set_depth(set, true);
2946 blk_mq_update_tag_set_shared(set, true);
2937 }
2947 }
2938 if (set->flags & BLK_MQ_F_TAG_SHARED)
2948 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
2939 queue_set_hctx_shared(q, true);
2940 list_add_tail(&q->tag_set_list, &set->tag_list);
2941
2942 mutex_unlock(&set->tag_list_lock);
2943}
2944
2945/* All allocations will be freed in release handler of q->mq_kobj */
2946static int blk_mq_alloc_ctxs(struct request_queue *q)

--- 304 unchanged lines hidden (view full) ---

3251 blk_mq_del_queue_tag_set(q);
3252 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
3253}
3254
3255static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
3256{
3257 int i;
3258
2949 queue_set_hctx_shared(q, true);
2950 list_add_tail(&q->tag_set_list, &set->tag_list);
2951
2952 mutex_unlock(&set->tag_list_lock);
2953}
2954
2955/* All allocations will be freed in release handler of q->mq_kobj */
2956static int blk_mq_alloc_ctxs(struct request_queue *q)

--- 304 unchanged lines hidden (view full) ---

3261 blk_mq_del_queue_tag_set(q);
3262 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
3263}
3264
3265static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
3266{
3267 int i;
3268
3259 for (i = 0; i < set->nr_hw_queues; i++)
3269 for (i = 0; i < set->nr_hw_queues; i++) {
3260 if (!__blk_mq_alloc_map_and_request(set, i))
3261 goto out_unwind;
3270 if (!__blk_mq_alloc_map_and_request(set, i))
3271 goto out_unwind;
3272 cond_resched();
3273 }
3262
3263 return 0;
3264
3265out_unwind:
3266 while (--i >= 0)
3267 blk_mq_free_map_and_requests(set, i);
3268
3269 return -ENOMEM;

--- 163 unchanged lines hidden (view full) ---

3433 ret = blk_mq_update_queue_map(set);
3434 if (ret)
3435 goto out_free_mq_map;
3436
3437 ret = blk_mq_alloc_map_and_requests(set);
3438 if (ret)
3439 goto out_free_mq_map;
3440
3274
3275 return 0;
3276
3277out_unwind:
3278 while (--i >= 0)
3279 blk_mq_free_map_and_requests(set, i);
3280
3281 return -ENOMEM;

--- 163 unchanged lines hidden (view full) ---

3445 ret = blk_mq_update_queue_map(set);
3446 if (ret)
3447 goto out_free_mq_map;
3448
3449 ret = blk_mq_alloc_map_and_requests(set);
3450 if (ret)
3451 goto out_free_mq_map;
3452
3453 if (blk_mq_is_sbitmap_shared(set->flags)) {
3454 atomic_set(&set->active_queues_shared_sbitmap, 0);
3455
3456 if (blk_mq_init_shared_sbitmap(set, set->flags)) {
3457 ret = -ENOMEM;
3458 goto out_free_mq_rq_maps;
3459 }
3460 }
3461
3441 mutex_init(&set->tag_list_lock);
3442 INIT_LIST_HEAD(&set->tag_list);
3443
3444 return 0;
3445
3462 mutex_init(&set->tag_list_lock);
3463 INIT_LIST_HEAD(&set->tag_list);
3464
3465 return 0;
3466
3467out_free_mq_rq_maps:
3468 for (i = 0; i < set->nr_hw_queues; i++)
3469 blk_mq_free_map_and_requests(set, i);
3446out_free_mq_map:
3447 for (i = 0; i < set->nr_maps; i++) {
3448 kfree(set->map[i].mq_map);
3449 set->map[i].mq_map = NULL;
3450 }
3451 kfree(set->tags);
3452 set->tags = NULL;
3453 return ret;
3454}
3455EXPORT_SYMBOL(blk_mq_alloc_tag_set);
3456
3457void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
3458{
3459 int i, j;
3460
3461 for (i = 0; i < set->nr_hw_queues; i++)
3462 blk_mq_free_map_and_requests(set, i);
3463
3470out_free_mq_map:
3471 for (i = 0; i < set->nr_maps; i++) {
3472 kfree(set->map[i].mq_map);
3473 set->map[i].mq_map = NULL;
3474 }
3475 kfree(set->tags);
3476 set->tags = NULL;
3477 return ret;
3478}
3479EXPORT_SYMBOL(blk_mq_alloc_tag_set);
3480
3481void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
3482{
3483 int i, j;
3484
3485 for (i = 0; i < set->nr_hw_queues; i++)
3486 blk_mq_free_map_and_requests(set, i);
3487
3488 if (blk_mq_is_sbitmap_shared(set->flags))
3489 blk_mq_exit_shared_sbitmap(set);
3490
3464 for (j = 0; j < set->nr_maps; j++) {
3465 kfree(set->map[j].mq_map);
3466 set->map[j].mq_map = NULL;
3467 }
3468
3469 kfree(set->tags);
3470 set->tags = NULL;
3471}

--- 20 unchanged lines hidden (view full) ---

3492 continue;
3493 /*
3494 * If we're using an MQ scheduler, just update the scheduler
3495 * queue depth. This is similar to what the old code would do.
3496 */
3497 if (!hctx->sched_tags) {
3498 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
3499 false);
3491 for (j = 0; j < set->nr_maps; j++) {
3492 kfree(set->map[j].mq_map);
3493 set->map[j].mq_map = NULL;
3494 }
3495
3496 kfree(set->tags);
3497 set->tags = NULL;
3498}

--- 20 unchanged lines hidden (view full) ---

3519 continue;
3520 /*
3521 * If we're using an MQ scheduler, just update the scheduler
3522 * queue depth. This is similar to what the old code would do.
3523 */
3524 if (!hctx->sched_tags) {
3525 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
3526 false);
3527 if (!ret && blk_mq_is_sbitmap_shared(set->flags))
3528 blk_mq_tag_resize_shared_sbitmap(set, nr);
3500 } else {
3501 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
3502 nr, true);
3503 }
3504 if (ret)
3505 break;
3506 if (q->elevator && q->elevator->type->ops.depth_updated)
3507 q->elevator->type->ops.depth_updated(hctx);

--- 390 unchanged lines hidden --- 		3529 } else {
3530 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
3531 nr, true);
3532 }
3533 if (ret)
3534 break;
3535 if (q->elevator && q->elevator->type->ops.depth_updated)
3536 q->elevator->type->ops.depth_updated(hctx);

--- 390 unchanged lines hidden ---