1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Tag allocation using scalable bitmaps. Uses active queue tracking to support 4 * fairer distribution of tags between multiple submitters when a shared tag map 5 * is used. 6 * 7 * Copyright (C) 2013-2014 Jens Axboe 8 */ 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 12 #include <linux/delay.h> 13 #include "blk.h" 14 #include "blk-mq.h" 15 #include "blk-mq-sched.h" 16 17 /* 18 * Recalculate wakeup batch when tag is shared by hctx. 19 */ 20 static void blk_mq_update_wake_batch(struct blk_mq_tags *tags, 21 unsigned int users) 22 { 23 if (!users) 24 return; 25 26 sbitmap_queue_recalculate_wake_batch(&tags->bitmap_tags, 27 users); 28 sbitmap_queue_recalculate_wake_batch(&tags->breserved_tags, 29 users); 30 } 31 32 /* 33 * If a previously inactive queue goes active, bump the active user count. 34 * We need to do this before try to allocate driver tag, then even if fail 35 * to get tag when first time, the other shared-tag users could reserve 36 * budget for it. 37 */ 38 void __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) 39 { 40 unsigned int users; 41 struct blk_mq_tags *tags = hctx->tags; 42 43 if (blk_mq_is_shared_tags(hctx->flags)) { 44 struct request_queue *q = hctx->queue; 45 46 if (test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags)) 47 return; 48 set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags); 49 } else { 50 if (test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) 51 return; 52 set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state); 53 } 54 55 spin_lock_irq(&tags->lock); 56 users = tags->active_queues + 1; 57 WRITE_ONCE(tags->active_queues, users); 58 blk_mq_update_wake_batch(tags, users); 59 spin_unlock_irq(&tags->lock); 60 } 61 62 /* 63 * Wakeup all potentially sleeping on tags 64 */ 65 void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve) 66 { 67 sbitmap_queue_wake_all(&tags->bitmap_tags); 68 if (include_reserve) 69 sbitmap_queue_wake_all(&tags->breserved_tags); 70 } 71 72 /* 73 * If a previously busy queue goes inactive, potential waiters could now 74 * be allowed to queue. Wake them up and check. 75 */ 76 void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) 77 { 78 struct blk_mq_tags *tags = hctx->tags; 79 unsigned int users; 80 81 if (blk_mq_is_shared_tags(hctx->flags)) { 82 struct request_queue *q = hctx->queue; 83 84 if (!test_and_clear_bit(QUEUE_FLAG_HCTX_ACTIVE, 85 &q->queue_flags)) 86 return; 87 } else { 88 if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) 89 return; 90 } 91 92 spin_lock_irq(&tags->lock); 93 users = tags->active_queues - 1; 94 WRITE_ONCE(tags->active_queues, users); 95 blk_mq_update_wake_batch(tags, users); 96 spin_unlock_irq(&tags->lock); 97 98 blk_mq_tag_wakeup_all(tags, false); 99 } 100 101 static int __blk_mq_get_tag(struct blk_mq_alloc_data *data, 102 struct sbitmap_queue *bt) 103 { 104 if (!data->q->elevator && !(data->flags & BLK_MQ_REQ_RESERVED) && 105 !hctx_may_queue(data->hctx, bt)) 106 return BLK_MQ_NO_TAG; 107 108 if (data->shallow_depth) 109 return sbitmap_queue_get_shallow(bt, data->shallow_depth); 110 else 111 return __sbitmap_queue_get(bt); 112 } 113 114 unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags, 115 unsigned int *offset) 116 { 117 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 118 struct sbitmap_queue *bt = &tags->bitmap_tags; 119 unsigned long ret; 120 121 if (data->shallow_depth ||data->flags & BLK_MQ_REQ_RESERVED || 122 data->hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) 123 return 0; 124 ret = __sbitmap_queue_get_batch(bt, nr_tags, offset); 125 *offset += tags->nr_reserved_tags; 126 return ret; 127 } 128 129 unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) 130 { 131 struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 132 struct sbitmap_queue *bt; 133 struct sbq_wait_state *ws; 134 DEFINE_SBQ_WAIT(wait); 135 unsigned int tag_offset; 136 int tag; 137 138 if (data->flags & BLK_MQ_REQ_RESERVED) { 139 if (unlikely(!tags->nr_reserved_tags)) { 140 WARN_ON_ONCE(1); 141 return BLK_MQ_NO_TAG; 142 } 143 bt = &tags->breserved_tags; 144 tag_offset = 0; 145 } else { 146 bt = &tags->bitmap_tags; 147 tag_offset = tags->nr_reserved_tags; 148 } 149 150 tag = __blk_mq_get_tag(data, bt); 151 if (tag != BLK_MQ_NO_TAG) 152 goto found_tag; 153 154 if (data->flags & BLK_MQ_REQ_NOWAIT) 155 return BLK_MQ_NO_TAG; 156 157 ws = bt_wait_ptr(bt, data->hctx); 158 do { 159 struct sbitmap_queue *bt_prev; 160 161 /* 162 * We're out of tags on this hardware queue, kick any 163 * pending IO submits before going to sleep waiting for 164 * some to complete. 165 */ 166 blk_mq_run_hw_queue(data->hctx, false); 167 168 /* 169 * Retry tag allocation after running the hardware queue, 170 * as running the queue may also have found completions. 171 */ 172 tag = __blk_mq_get_tag(data, bt); 173 if (tag != BLK_MQ_NO_TAG) 174 break; 175 176 sbitmap_prepare_to_wait(bt, ws, &wait, TASK_UNINTERRUPTIBLE); 177 178 tag = __blk_mq_get_tag(data, bt); 179 if (tag != BLK_MQ_NO_TAG) 180 break; 181 182 bt_prev = bt; 183 io_schedule(); 184 185 sbitmap_finish_wait(bt, ws, &wait); 186 187 data->ctx = blk_mq_get_ctx(data->q); 188 data->hctx = blk_mq_map_queue(data->q, data->cmd_flags, 189 data->ctx); 190 tags = blk_mq_tags_from_data(data); 191 if (data->flags & BLK_MQ_REQ_RESERVED) 192 bt = &tags->breserved_tags; 193 else 194 bt = &tags->bitmap_tags; 195 196 /* 197 * If destination hw queue is changed, fake wake up on 198 * previous queue for compensating the wake up miss, so 199 * other allocations on previous queue won't be starved. 200 */ 201 if (bt != bt_prev) 202 sbitmap_queue_wake_up(bt_prev, 1); 203 204 ws = bt_wait_ptr(bt, data->hctx); 205 } while (1); 206 207 sbitmap_finish_wait(bt, ws, &wait); 208 209 found_tag: 210 /* 211 * Give up this allocation if the hctx is inactive. The caller will 212 * retry on an active hctx. 213 */ 214 if (unlikely(test_bit(BLK_MQ_S_INACTIVE, &data->hctx->state))) { 215 blk_mq_put_tag(tags, data->ctx, tag + tag_offset); 216 return BLK_MQ_NO_TAG; 217 } 218 return tag + tag_offset; 219 } 220 221 void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, 222 unsigned int tag) 223 { 224 if (!blk_mq_tag_is_reserved(tags, tag)) { 225 const int real_tag = tag - tags->nr_reserved_tags; 226 227 BUG_ON(real_tag >= tags->nr_tags); 228 sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu); 229 } else { 230 sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu); 231 } 232 } 233 234 void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags) 235 { 236 sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags, 237 tag_array, nr_tags); 238 } 239 240 struct bt_iter_data { 241 struct blk_mq_hw_ctx *hctx; 242 struct request_queue *q; 243 busy_tag_iter_fn *fn; 244 void *data; 245 bool reserved; 246 }; 247 248 static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags, 249 unsigned int bitnr) 250 { 251 struct request *rq; 252 unsigned long flags; 253 254 spin_lock_irqsave(&tags->lock, flags); 255 rq = tags->rqs[bitnr]; 256 if (!rq || rq->tag != bitnr || !req_ref_inc_not_zero(rq)) 257 rq = NULL; 258 spin_unlock_irqrestore(&tags->lock, flags); 259 return rq; 260 } 261 262 static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) 263 { 264 struct bt_iter_data *iter_data = data; 265 struct blk_mq_hw_ctx *hctx = iter_data->hctx; 266 struct request_queue *q = iter_data->q; 267 struct blk_mq_tag_set *set = q->tag_set; 268 struct blk_mq_tags *tags; 269 struct request *rq; 270 bool ret = true; 271 272 if (blk_mq_is_shared_tags(set->flags)) 273 tags = set->shared_tags; 274 else 275 tags = hctx->tags; 276 277 if (!iter_data->reserved) 278 bitnr += tags->nr_reserved_tags; 279 /* 280 * We can hit rq == NULL here, because the tagging functions 281 * test and set the bit before assigning ->rqs[]. 282 */ 283 rq = blk_mq_find_and_get_req(tags, bitnr); 284 if (!rq) 285 return true; 286 287 if (rq->q == q && (!hctx || rq->mq_hctx == hctx)) 288 ret = iter_data->fn(rq, iter_data->data); 289 blk_mq_put_rq_ref(rq); 290 return ret; 291 } 292 293 /** 294 * bt_for_each - iterate over the requests associated with a hardware queue 295 * @hctx: Hardware queue to examine. 296 * @q: Request queue to examine. 297 * @bt: sbitmap to examine. This is either the breserved_tags member 298 * or the bitmap_tags member of struct blk_mq_tags. 299 * @fn: Pointer to the function that will be called for each request 300 * associated with @hctx that has been assigned a driver tag. 301 * @fn will be called as follows: @fn(@hctx, rq, @data, @reserved) 302 * where rq is a pointer to a request. Return true to continue 303 * iterating tags, false to stop. 304 * @data: Will be passed as third argument to @fn. 305 * @reserved: Indicates whether @bt is the breserved_tags member or the 306 * bitmap_tags member of struct blk_mq_tags. 307 */ 308 static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct request_queue *q, 309 struct sbitmap_queue *bt, busy_tag_iter_fn *fn, 310 void *data, bool reserved) 311 { 312 struct bt_iter_data iter_data = { 313 .hctx = hctx, 314 .fn = fn, 315 .data = data, 316 .reserved = reserved, 317 .q = q, 318 }; 319 320 sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data); 321 } 322 323 struct bt_tags_iter_data { 324 struct blk_mq_tags *tags; 325 busy_tag_iter_fn *fn; 326 void *data; 327 unsigned int flags; 328 }; 329 330 #define BT_TAG_ITER_RESERVED (1 << 0) 331 #define BT_TAG_ITER_STARTED (1 << 1) 332 #define BT_TAG_ITER_STATIC_RQS (1 << 2) 333 334 static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) 335 { 336 struct bt_tags_iter_data *iter_data = data; 337 struct blk_mq_tags *tags = iter_data->tags; 338 struct request *rq; 339 bool ret = true; 340 bool iter_static_rqs = !!(iter_data->flags & BT_TAG_ITER_STATIC_RQS); 341 342 if (!(iter_data->flags & BT_TAG_ITER_RESERVED)) 343 bitnr += tags->nr_reserved_tags; 344 345 /* 346 * We can hit rq == NULL here, because the tagging functions 347 * test and set the bit before assigning ->rqs[]. 348 */ 349 if (iter_static_rqs) 350 rq = tags->static_rqs[bitnr]; 351 else 352 rq = blk_mq_find_and_get_req(tags, bitnr); 353 if (!rq) 354 return true; 355 356 if (!(iter_data->flags & BT_TAG_ITER_STARTED) || 357 blk_mq_request_started(rq)) 358 ret = iter_data->fn(rq, iter_data->data); 359 if (!iter_static_rqs) 360 blk_mq_put_rq_ref(rq); 361 return ret; 362 } 363 364 /** 365 * bt_tags_for_each - iterate over the requests in a tag map 366 * @tags: Tag map to iterate over. 367 * @bt: sbitmap to examine. This is either the breserved_tags member 368 * or the bitmap_tags member of struct blk_mq_tags. 369 * @fn: Pointer to the function that will be called for each started 370 * request. @fn will be called as follows: @fn(rq, @data, 371 * @reserved) where rq is a pointer to a request. Return true 372 * to continue iterating tags, false to stop. 373 * @data: Will be passed as second argument to @fn. 374 * @flags: BT_TAG_ITER_* 375 */ 376 static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt, 377 busy_tag_iter_fn *fn, void *data, unsigned int flags) 378 { 379 struct bt_tags_iter_data iter_data = { 380 .tags = tags, 381 .fn = fn, 382 .data = data, 383 .flags = flags, 384 }; 385 386 if (tags->rqs) 387 sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data); 388 } 389 390 static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags, 391 busy_tag_iter_fn *fn, void *priv, unsigned int flags) 392 { 393 WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED); 394 395 if (tags->nr_reserved_tags) 396 bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, 397 flags | BT_TAG_ITER_RESERVED); 398 bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, flags); 399 } 400 401 /** 402 * blk_mq_all_tag_iter - iterate over all requests in a tag map 403 * @tags: Tag map to iterate over. 404 * @fn: Pointer to the function that will be called for each 405 * request. @fn will be called as follows: @fn(rq, @priv, 406 * reserved) where rq is a pointer to a request. 'reserved' 407 * indicates whether or not @rq is a reserved request. Return 408 * true to continue iterating tags, false to stop. 409 * @priv: Will be passed as second argument to @fn. 410 * 411 * Caller has to pass the tag map from which requests are allocated. 412 */ 413 void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, 414 void *priv) 415 { 416 __blk_mq_all_tag_iter(tags, fn, priv, BT_TAG_ITER_STATIC_RQS); 417 } 418 419 /** 420 * blk_mq_tagset_busy_iter - iterate over all started requests in a tag set 421 * @tagset: Tag set to iterate over. 422 * @fn: Pointer to the function that will be called for each started 423 * request. @fn will be called as follows: @fn(rq, @priv, 424 * reserved) where rq is a pointer to a request. 'reserved' 425 * indicates whether or not @rq is a reserved request. Return 426 * true to continue iterating tags, false to stop. 427 * @priv: Will be passed as second argument to @fn. 428 * 429 * We grab one request reference before calling @fn and release it after 430 * @fn returns. 431 */ 432 void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, 433 busy_tag_iter_fn *fn, void *priv) 434 { 435 unsigned int flags = tagset->flags; 436 int i, nr_tags; 437 438 nr_tags = blk_mq_is_shared_tags(flags) ? 1 : tagset->nr_hw_queues; 439 440 for (i = 0; i < nr_tags; i++) { 441 if (tagset->tags && tagset->tags[i]) 442 __blk_mq_all_tag_iter(tagset->tags[i], fn, priv, 443 BT_TAG_ITER_STARTED); 444 } 445 } 446 EXPORT_SYMBOL(blk_mq_tagset_busy_iter); 447 448 static bool blk_mq_tagset_count_completed_rqs(struct request *rq, void *data) 449 { 450 unsigned *count = data; 451 452 if (blk_mq_request_completed(rq)) 453 (*count)++; 454 return true; 455 } 456 457 /** 458 * blk_mq_tagset_wait_completed_request - Wait until all scheduled request 459 * completions have finished. 460 * @tagset: Tag set to drain completed request 461 * 462 * Note: This function has to be run after all IO queues are shutdown 463 */ 464 void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset) 465 { 466 while (true) { 467 unsigned count = 0; 468 469 blk_mq_tagset_busy_iter(tagset, 470 blk_mq_tagset_count_completed_rqs, &count); 471 if (!count) 472 break; 473 msleep(5); 474 } 475 } 476 EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request); 477 478 /** 479 * blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag 480 * @q: Request queue to examine. 481 * @fn: Pointer to the function that will be called for each request 482 * on @q. @fn will be called as follows: @fn(hctx, rq, @priv, 483 * reserved) where rq is a pointer to a request and hctx points 484 * to the hardware queue associated with the request. 'reserved' 485 * indicates whether or not @rq is a reserved request. 486 * @priv: Will be passed as third argument to @fn. 487 * 488 * Note: if @q->tag_set is shared with other request queues then @fn will be 489 * called for all requests on all queues that share that tag set and not only 490 * for requests associated with @q. 491 */ 492 void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, 493 void *priv) 494 { 495 /* 496 * __blk_mq_update_nr_hw_queues() updates nr_hw_queues and hctx_table 497 * while the queue is frozen. So we can use q_usage_counter to avoid 498 * racing with it. 499 */ 500 if (!percpu_ref_tryget(&q->q_usage_counter)) 501 return; 502 503 if (blk_mq_is_shared_tags(q->tag_set->flags)) { 504 struct blk_mq_tags *tags = q->tag_set->shared_tags; 505 struct sbitmap_queue *bresv = &tags->breserved_tags; 506 struct sbitmap_queue *btags = &tags->bitmap_tags; 507 508 if (tags->nr_reserved_tags) 509 bt_for_each(NULL, q, bresv, fn, priv, true); 510 bt_for_each(NULL, q, btags, fn, priv, false); 511 } else { 512 struct blk_mq_hw_ctx *hctx; 513 unsigned long i; 514 515 queue_for_each_hw_ctx(q, hctx, i) { 516 struct blk_mq_tags *tags = hctx->tags; 517 struct sbitmap_queue *bresv = &tags->breserved_tags; 518 struct sbitmap_queue *btags = &tags->bitmap_tags; 519 520 /* 521 * If no software queues are currently mapped to this 522 * hardware queue, there's nothing to check 523 */ 524 if (!blk_mq_hw_queue_mapped(hctx)) 525 continue; 526 527 if (tags->nr_reserved_tags) 528 bt_for_each(hctx, q, bresv, fn, priv, true); 529 bt_for_each(hctx, q, btags, fn, priv, false); 530 } 531 } 532 blk_queue_exit(q); 533 } 534 535 static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth, 536 bool round_robin, int node) 537 { 538 return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL, 539 node); 540 } 541 542 int blk_mq_init_bitmaps(struct sbitmap_queue *bitmap_tags, 543 struct sbitmap_queue *breserved_tags, 544 unsigned int queue_depth, unsigned int reserved, 545 int node, int alloc_policy) 546 { 547 unsigned int depth = queue_depth - reserved; 548 bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR; 549 550 if (bt_alloc(bitmap_tags, depth, round_robin, node)) 551 return -ENOMEM; 552 if (bt_alloc(breserved_tags, reserved, round_robin, node)) 553 goto free_bitmap_tags; 554 555 return 0; 556 557 free_bitmap_tags: 558 sbitmap_queue_free(bitmap_tags); 559 return -ENOMEM; 560 } 561 562 struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, 563 unsigned int reserved_tags, 564 int node, int alloc_policy) 565 { 566 struct blk_mq_tags *tags; 567 568 if (total_tags > BLK_MQ_TAG_MAX) { 569 pr_err("blk-mq: tag depth too large\n"); 570 return NULL; 571 } 572 573 tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node); 574 if (!tags) 575 return NULL; 576 577 tags->nr_tags = total_tags; 578 tags->nr_reserved_tags = reserved_tags; 579 spin_lock_init(&tags->lock); 580 581 if (blk_mq_init_bitmaps(&tags->bitmap_tags, &tags->breserved_tags, 582 total_tags, reserved_tags, node, 583 alloc_policy) < 0) { 584 kfree(tags); 585 return NULL; 586 } 587 return tags; 588 } 589 590 void blk_mq_free_tags(struct blk_mq_tags *tags) 591 { 592 sbitmap_queue_free(&tags->bitmap_tags); 593 sbitmap_queue_free(&tags->breserved_tags); 594 kfree(tags); 595 } 596 597 int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, 598 struct blk_mq_tags **tagsptr, unsigned int tdepth, 599 bool can_grow) 600 { 601 struct blk_mq_tags *tags = *tagsptr; 602 603 if (tdepth <= tags->nr_reserved_tags) 604 return -EINVAL; 605 606 /* 607 * If we are allowed to grow beyond the original size, allocate 608 * a new set of tags before freeing the old one. 609 */ 610 if (tdepth > tags->nr_tags) { 611 struct blk_mq_tag_set *set = hctx->queue->tag_set; 612 struct blk_mq_tags *new; 613 614 if (!can_grow) 615 return -EINVAL; 616 617 /* 618 * We need some sort of upper limit, set it high enough that 619 * no valid use cases should require more. 620 */ 621 if (tdepth > MAX_SCHED_RQ) 622 return -EINVAL; 623 624 /* 625 * Only the sbitmap needs resizing since we allocated the max 626 * initially. 627 */ 628 if (blk_mq_is_shared_tags(set->flags)) 629 return 0; 630 631 new = blk_mq_alloc_map_and_rqs(set, hctx->queue_num, tdepth); 632 if (!new) 633 return -ENOMEM; 634 635 blk_mq_free_map_and_rqs(set, *tagsptr, hctx->queue_num); 636 *tagsptr = new; 637 } else { 638 /* 639 * Don't need (or can't) update reserved tags here, they 640 * remain static and should never need resizing. 641 */ 642 sbitmap_queue_resize(&tags->bitmap_tags, 643 tdepth - tags->nr_reserved_tags); 644 } 645 646 return 0; 647 } 648 649 void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size) 650 { 651 struct blk_mq_tags *tags = set->shared_tags; 652 653 sbitmap_queue_resize(&tags->bitmap_tags, size - set->reserved_tags); 654 } 655 656 void blk_mq_tag_update_sched_shared_tags(struct request_queue *q) 657 { 658 sbitmap_queue_resize(&q->sched_shared_tags->bitmap_tags, 659 q->nr_requests - q->tag_set->reserved_tags); 660 } 661 662 /** 663 * blk_mq_unique_tag() - return a tag that is unique queue-wide 664 * @rq: request for which to compute a unique tag 665 * 666 * The tag field in struct request is unique per hardware queue but not over 667 * all hardware queues. Hence this function that returns a tag with the 668 * hardware context index in the upper bits and the per hardware queue tag in 669 * the lower bits. 670 * 671 * Note: When called for a request that is queued on a non-multiqueue request 672 * queue, the hardware context index is set to zero. 673 */ 674 u32 blk_mq_unique_tag(struct request *rq) 675 { 676 return (rq->mq_hctx->queue_num << BLK_MQ_UNIQUE_TAG_BITS) | 677 (rq->tag & BLK_MQ_UNIQUE_TAG_MASK); 678 } 679 EXPORT_SYMBOL(blk_mq_unique_tag); 680