1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * buffered writeback throttling. loosely based on CoDel. We can't drop 4 * packets for IO scheduling, so the logic is something like this: 5 * 6 * - Monitor latencies in a defined window of time. 7 * - If the minimum latency in the above window exceeds some target, increment 8 * scaling step and scale down queue depth by a factor of 2x. The monitoring 9 * window is then shrunk to 100 / sqrt(scaling step + 1). 10 * - For any window where we don't have solid data on what the latencies 11 * look like, retain status quo. 12 * - If latencies look good, decrement scaling step. 13 * - If we're only doing writes, allow the scaling step to go negative. This 14 * will temporarily boost write performance, snapping back to a stable 15 * scaling step of 0 if reads show up or the heavy writers finish. Unlike 16 * positive scaling steps where we shrink the monitoring window, a negative 17 * scaling step retains the default step==0 window size. 18 * 19 * Copyright (C) 2016 Jens Axboe 20 * 21 */ 22 #include <linux/kernel.h> 23 #include <linux/blk_types.h> 24 #include <linux/slab.h> 25 #include <linux/backing-dev.h> 26 #include <linux/swap.h> 27 28 #include "blk-stat.h" 29 #include "blk-wbt.h" 30 #include "blk-rq-qos.h" 31 #include "elevator.h" 32 33 #define CREATE_TRACE_POINTS 34 #include <trace/events/wbt.h> 35 36 enum wbt_flags { 37 WBT_TRACKED = 1, /* write, tracked for throttling */ 38 WBT_READ = 2, /* read */ 39 WBT_KSWAPD = 4, /* write, from kswapd */ 40 WBT_DISCARD = 8, /* discard */ 41 42 WBT_NR_BITS = 4, /* number of bits */ 43 }; 44 45 enum { 46 WBT_RWQ_BG = 0, 47 WBT_RWQ_KSWAPD, 48 WBT_RWQ_DISCARD, 49 WBT_NUM_RWQ, 50 }; 51 52 /* 53 * If current state is WBT_STATE_ON/OFF_DEFAULT, it can be covered to any other 54 * state, if current state is WBT_STATE_ON/OFF_MANUAL, it can only be covered 55 * to WBT_STATE_OFF/ON_MANUAL. 56 */ 57 enum { 58 WBT_STATE_ON_DEFAULT = 1, /* on by default */ 59 WBT_STATE_ON_MANUAL = 2, /* on manually by sysfs */ 60 WBT_STATE_OFF_DEFAULT = 3, /* off by default */ 61 WBT_STATE_OFF_MANUAL = 4, /* off manually by sysfs */ 62 }; 63 64 struct rq_wb { 65 /* 66 * Settings that govern how we throttle 67 */ 68 unsigned int wb_background; /* background writeback */ 69 unsigned int wb_normal; /* normal writeback */ 70 71 short enable_state; /* WBT_STATE_* */ 72 73 /* 74 * Number of consecutive periods where we don't have enough 75 * information to make a firm scale up/down decision. 76 */ 77 unsigned int unknown_cnt; 78 79 u64 win_nsec; /* default window size */ 80 u64 cur_win_nsec; /* current window size */ 81 82 struct blk_stat_callback *cb; 83 84 u64 sync_issue; 85 void *sync_cookie; 86 87 unsigned int wc; 88 89 unsigned long last_issue; /* last non-throttled issue */ 90 unsigned long last_comp; /* last non-throttled comp */ 91 unsigned long min_lat_nsec; 92 struct rq_qos rqos; 93 struct rq_wait rq_wait[WBT_NUM_RWQ]; 94 struct rq_depth rq_depth; 95 }; 96 97 static inline struct rq_wb *RQWB(struct rq_qos *rqos) 98 { 99 return container_of(rqos, struct rq_wb, rqos); 100 } 101 102 static inline void wbt_clear_state(struct request *rq) 103 { 104 rq->wbt_flags = 0; 105 } 106 107 static inline enum wbt_flags wbt_flags(struct request *rq) 108 { 109 return rq->wbt_flags; 110 } 111 112 static inline bool wbt_is_tracked(struct request *rq) 113 { 114 return rq->wbt_flags & WBT_TRACKED; 115 } 116 117 static inline bool wbt_is_read(struct request *rq) 118 { 119 return rq->wbt_flags & WBT_READ; 120 } 121 122 enum { 123 /* 124 * Default setting, we'll scale up (to 75% of QD max) or down (min 1) 125 * from here depending on device stats 126 */ 127 RWB_DEF_DEPTH = 16, 128 129 /* 130 * 100msec window 131 */ 132 RWB_WINDOW_NSEC = 100 * 1000 * 1000ULL, 133 134 /* 135 * Disregard stats, if we don't meet this minimum 136 */ 137 RWB_MIN_WRITE_SAMPLES = 3, 138 139 /* 140 * If we have this number of consecutive windows with not enough 141 * information to scale up or down, scale up. 142 */ 143 RWB_UNKNOWN_BUMP = 5, 144 }; 145 146 static inline bool rwb_enabled(struct rq_wb *rwb) 147 { 148 return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT && 149 rwb->wb_normal != 0; 150 } 151 152 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var) 153 { 154 if (rwb_enabled(rwb)) { 155 const unsigned long cur = jiffies; 156 157 if (cur != *var) 158 *var = cur; 159 } 160 } 161 162 /* 163 * If a task was rate throttled in balance_dirty_pages() within the last 164 * second or so, use that to indicate a higher cleaning rate. 165 */ 166 static bool wb_recent_wait(struct rq_wb *rwb) 167 { 168 struct bdi_writeback *wb = &rwb->rqos.q->disk->bdi->wb; 169 170 return time_before(jiffies, wb->dirty_sleep + HZ); 171 } 172 173 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, 174 enum wbt_flags wb_acct) 175 { 176 if (wb_acct & WBT_KSWAPD) 177 return &rwb->rq_wait[WBT_RWQ_KSWAPD]; 178 else if (wb_acct & WBT_DISCARD) 179 return &rwb->rq_wait[WBT_RWQ_DISCARD]; 180 181 return &rwb->rq_wait[WBT_RWQ_BG]; 182 } 183 184 static void rwb_wake_all(struct rq_wb *rwb) 185 { 186 int i; 187 188 for (i = 0; i < WBT_NUM_RWQ; i++) { 189 struct rq_wait *rqw = &rwb->rq_wait[i]; 190 191 if (wq_has_sleeper(&rqw->wait)) 192 wake_up_all(&rqw->wait); 193 } 194 } 195 196 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw, 197 enum wbt_flags wb_acct) 198 { 199 int inflight, limit; 200 201 inflight = atomic_dec_return(&rqw->inflight); 202 203 /* 204 * wbt got disabled with IO in flight. Wake up any potential 205 * waiters, we don't have to do more than that. 206 */ 207 if (unlikely(!rwb_enabled(rwb))) { 208 rwb_wake_all(rwb); 209 return; 210 } 211 212 /* 213 * For discards, our limit is always the background. For writes, if 214 * the device does write back caching, drop further down before we 215 * wake people up. 216 */ 217 if (wb_acct & WBT_DISCARD) 218 limit = rwb->wb_background; 219 else if (rwb->wc && !wb_recent_wait(rwb)) 220 limit = 0; 221 else 222 limit = rwb->wb_normal; 223 224 /* 225 * Don't wake anyone up if we are above the normal limit. 226 */ 227 if (inflight && inflight >= limit) 228 return; 229 230 if (wq_has_sleeper(&rqw->wait)) { 231 int diff = limit - inflight; 232 233 if (!inflight || diff >= rwb->wb_background / 2) 234 wake_up_all(&rqw->wait); 235 } 236 } 237 238 static void __wbt_done(struct rq_qos *rqos, enum wbt_flags wb_acct) 239 { 240 struct rq_wb *rwb = RQWB(rqos); 241 struct rq_wait *rqw; 242 243 if (!(wb_acct & WBT_TRACKED)) 244 return; 245 246 rqw = get_rq_wait(rwb, wb_acct); 247 wbt_rqw_done(rwb, rqw, wb_acct); 248 } 249 250 /* 251 * Called on completion of a request. Note that it's also called when 252 * a request is merged, when the request gets freed. 253 */ 254 static void wbt_done(struct rq_qos *rqos, struct request *rq) 255 { 256 struct rq_wb *rwb = RQWB(rqos); 257 258 if (!wbt_is_tracked(rq)) { 259 if (rwb->sync_cookie == rq) { 260 rwb->sync_issue = 0; 261 rwb->sync_cookie = NULL; 262 } 263 264 if (wbt_is_read(rq)) 265 wb_timestamp(rwb, &rwb->last_comp); 266 } else { 267 WARN_ON_ONCE(rq == rwb->sync_cookie); 268 __wbt_done(rqos, wbt_flags(rq)); 269 } 270 wbt_clear_state(rq); 271 } 272 273 static inline bool stat_sample_valid(struct blk_rq_stat *stat) 274 { 275 /* 276 * We need at least one read sample, and a minimum of 277 * RWB_MIN_WRITE_SAMPLES. We require some write samples to know 278 * that it's writes impacting us, and not just some sole read on 279 * a device that is in a lower power state. 280 */ 281 return (stat[READ].nr_samples >= 1 && 282 stat[WRITE].nr_samples >= RWB_MIN_WRITE_SAMPLES); 283 } 284 285 static u64 rwb_sync_issue_lat(struct rq_wb *rwb) 286 { 287 u64 now, issue = READ_ONCE(rwb->sync_issue); 288 289 if (!issue || !rwb->sync_cookie) 290 return 0; 291 292 now = ktime_to_ns(ktime_get()); 293 return now - issue; 294 } 295 296 static inline unsigned int wbt_inflight(struct rq_wb *rwb) 297 { 298 unsigned int i, ret = 0; 299 300 for (i = 0; i < WBT_NUM_RWQ; i++) 301 ret += atomic_read(&rwb->rq_wait[i].inflight); 302 303 return ret; 304 } 305 306 enum { 307 LAT_OK = 1, 308 LAT_UNKNOWN, 309 LAT_UNKNOWN_WRITES, 310 LAT_EXCEEDED, 311 }; 312 313 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat) 314 { 315 struct backing_dev_info *bdi = rwb->rqos.q->disk->bdi; 316 struct rq_depth *rqd = &rwb->rq_depth; 317 u64 thislat; 318 319 /* 320 * If our stored sync issue exceeds the window size, or it 321 * exceeds our min target AND we haven't logged any entries, 322 * flag the latency as exceeded. wbt works off completion latencies, 323 * but for a flooded device, a single sync IO can take a long time 324 * to complete after being issued. If this time exceeds our 325 * monitoring window AND we didn't see any other completions in that 326 * window, then count that sync IO as a violation of the latency. 327 */ 328 thislat = rwb_sync_issue_lat(rwb); 329 if (thislat > rwb->cur_win_nsec || 330 (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) { 331 trace_wbt_lat(bdi, thislat); 332 return LAT_EXCEEDED; 333 } 334 335 /* 336 * No read/write mix, if stat isn't valid 337 */ 338 if (!stat_sample_valid(stat)) { 339 /* 340 * If we had writes in this stat window and the window is 341 * current, we're only doing writes. If a task recently 342 * waited or still has writes in flights, consider us doing 343 * just writes as well. 344 */ 345 if (stat[WRITE].nr_samples || wb_recent_wait(rwb) || 346 wbt_inflight(rwb)) 347 return LAT_UNKNOWN_WRITES; 348 return LAT_UNKNOWN; 349 } 350 351 /* 352 * If the 'min' latency exceeds our target, step down. 353 */ 354 if (stat[READ].min > rwb->min_lat_nsec) { 355 trace_wbt_lat(bdi, stat[READ].min); 356 trace_wbt_stat(bdi, stat); 357 return LAT_EXCEEDED; 358 } 359 360 if (rqd->scale_step) 361 trace_wbt_stat(bdi, stat); 362 363 return LAT_OK; 364 } 365 366 static void rwb_trace_step(struct rq_wb *rwb, const char *msg) 367 { 368 struct backing_dev_info *bdi = rwb->rqos.q->disk->bdi; 369 struct rq_depth *rqd = &rwb->rq_depth; 370 371 trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec, 372 rwb->wb_background, rwb->wb_normal, rqd->max_depth); 373 } 374 375 static void calc_wb_limits(struct rq_wb *rwb) 376 { 377 if (rwb->min_lat_nsec == 0) { 378 rwb->wb_normal = rwb->wb_background = 0; 379 } else if (rwb->rq_depth.max_depth <= 2) { 380 rwb->wb_normal = rwb->rq_depth.max_depth; 381 rwb->wb_background = 1; 382 } else { 383 rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2; 384 rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4; 385 } 386 } 387 388 static void scale_up(struct rq_wb *rwb) 389 { 390 if (!rq_depth_scale_up(&rwb->rq_depth)) 391 return; 392 calc_wb_limits(rwb); 393 rwb->unknown_cnt = 0; 394 rwb_wake_all(rwb); 395 rwb_trace_step(rwb, tracepoint_string("scale up")); 396 } 397 398 static void scale_down(struct rq_wb *rwb, bool hard_throttle) 399 { 400 if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle)) 401 return; 402 calc_wb_limits(rwb); 403 rwb->unknown_cnt = 0; 404 rwb_trace_step(rwb, tracepoint_string("scale down")); 405 } 406 407 static void rwb_arm_timer(struct rq_wb *rwb) 408 { 409 struct rq_depth *rqd = &rwb->rq_depth; 410 411 if (rqd->scale_step > 0) { 412 /* 413 * We should speed this up, using some variant of a fast 414 * integer inverse square root calculation. Since we only do 415 * this for every window expiration, it's not a huge deal, 416 * though. 417 */ 418 rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4, 419 int_sqrt((rqd->scale_step + 1) << 8)); 420 } else { 421 /* 422 * For step < 0, we don't want to increase/decrease the 423 * window size. 424 */ 425 rwb->cur_win_nsec = rwb->win_nsec; 426 } 427 428 blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec); 429 } 430 431 static void wb_timer_fn(struct blk_stat_callback *cb) 432 { 433 struct rq_wb *rwb = cb->data; 434 struct rq_depth *rqd = &rwb->rq_depth; 435 unsigned int inflight = wbt_inflight(rwb); 436 int status; 437 438 if (!rwb->rqos.q->disk) 439 return; 440 441 status = latency_exceeded(rwb, cb->stat); 442 443 trace_wbt_timer(rwb->rqos.q->disk->bdi, status, rqd->scale_step, 444 inflight); 445 446 /* 447 * If we exceeded the latency target, step down. If we did not, 448 * step one level up. If we don't know enough to say either exceeded 449 * or ok, then don't do anything. 450 */ 451 switch (status) { 452 case LAT_EXCEEDED: 453 scale_down(rwb, true); 454 break; 455 case LAT_OK: 456 scale_up(rwb); 457 break; 458 case LAT_UNKNOWN_WRITES: 459 /* 460 * We started a the center step, but don't have a valid 461 * read/write sample, but we do have writes going on. 462 * Allow step to go negative, to increase write perf. 463 */ 464 scale_up(rwb); 465 break; 466 case LAT_UNKNOWN: 467 if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP) 468 break; 469 /* 470 * We get here when previously scaled reduced depth, and we 471 * currently don't have a valid read/write sample. For that 472 * case, slowly return to center state (step == 0). 473 */ 474 if (rqd->scale_step > 0) 475 scale_up(rwb); 476 else if (rqd->scale_step < 0) 477 scale_down(rwb, false); 478 break; 479 default: 480 break; 481 } 482 483 /* 484 * Re-arm timer, if we have IO in flight 485 */ 486 if (rqd->scale_step || inflight) 487 rwb_arm_timer(rwb); 488 } 489 490 static void wbt_update_limits(struct rq_wb *rwb) 491 { 492 struct rq_depth *rqd = &rwb->rq_depth; 493 494 rqd->scale_step = 0; 495 rqd->scaled_max = false; 496 497 rq_depth_calc_max_depth(rqd); 498 calc_wb_limits(rwb); 499 500 rwb_wake_all(rwb); 501 } 502 503 bool wbt_disabled(struct request_queue *q) 504 { 505 struct rq_qos *rqos = wbt_rq_qos(q); 506 507 return !rqos || RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT || 508 RQWB(rqos)->enable_state == WBT_STATE_OFF_MANUAL; 509 } 510 511 u64 wbt_get_min_lat(struct request_queue *q) 512 { 513 struct rq_qos *rqos = wbt_rq_qos(q); 514 if (!rqos) 515 return 0; 516 return RQWB(rqos)->min_lat_nsec; 517 } 518 519 void wbt_set_min_lat(struct request_queue *q, u64 val) 520 { 521 struct rq_qos *rqos = wbt_rq_qos(q); 522 if (!rqos) 523 return; 524 525 RQWB(rqos)->min_lat_nsec = val; 526 if (val) 527 RQWB(rqos)->enable_state = WBT_STATE_ON_MANUAL; 528 else 529 RQWB(rqos)->enable_state = WBT_STATE_OFF_MANUAL; 530 531 wbt_update_limits(RQWB(rqos)); 532 } 533 534 535 static bool close_io(struct rq_wb *rwb) 536 { 537 const unsigned long now = jiffies; 538 539 return time_before(now, rwb->last_issue + HZ / 10) || 540 time_before(now, rwb->last_comp + HZ / 10); 541 } 542 543 #define REQ_HIPRIO (REQ_SYNC | REQ_META | REQ_PRIO) 544 545 static inline unsigned int get_limit(struct rq_wb *rwb, blk_opf_t opf) 546 { 547 unsigned int limit; 548 549 /* 550 * If we got disabled, just return UINT_MAX. This ensures that 551 * we'll properly inc a new IO, and dec+wakeup at the end. 552 */ 553 if (!rwb_enabled(rwb)) 554 return UINT_MAX; 555 556 if ((opf & REQ_OP_MASK) == REQ_OP_DISCARD) 557 return rwb->wb_background; 558 559 /* 560 * At this point we know it's a buffered write. If this is 561 * kswapd trying to free memory, or REQ_SYNC is set, then 562 * it's WB_SYNC_ALL writeback, and we'll use the max limit for 563 * that. If the write is marked as a background write, then use 564 * the idle limit, or go to normal if we haven't had competing 565 * IO for a bit. 566 */ 567 if ((opf & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd()) 568 limit = rwb->rq_depth.max_depth; 569 else if ((opf & REQ_BACKGROUND) || close_io(rwb)) { 570 /* 571 * If less than 100ms since we completed unrelated IO, 572 * limit us to half the depth for background writeback. 573 */ 574 limit = rwb->wb_background; 575 } else 576 limit = rwb->wb_normal; 577 578 return limit; 579 } 580 581 struct wbt_wait_data { 582 struct rq_wb *rwb; 583 enum wbt_flags wb_acct; 584 blk_opf_t opf; 585 }; 586 587 static bool wbt_inflight_cb(struct rq_wait *rqw, void *private_data) 588 { 589 struct wbt_wait_data *data = private_data; 590 return rq_wait_inc_below(rqw, get_limit(data->rwb, data->opf)); 591 } 592 593 static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data) 594 { 595 struct wbt_wait_data *data = private_data; 596 wbt_rqw_done(data->rwb, rqw, data->wb_acct); 597 } 598 599 /* 600 * Block if we will exceed our limit, or if we are currently waiting for 601 * the timer to kick off queuing again. 602 */ 603 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, 604 blk_opf_t opf) 605 { 606 struct rq_wait *rqw = get_rq_wait(rwb, wb_acct); 607 struct wbt_wait_data data = { 608 .rwb = rwb, 609 .wb_acct = wb_acct, 610 .opf = opf, 611 }; 612 613 rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb); 614 } 615 616 static inline bool wbt_should_throttle(struct bio *bio) 617 { 618 switch (bio_op(bio)) { 619 case REQ_OP_WRITE: 620 /* 621 * Don't throttle WRITE_ODIRECT 622 */ 623 if ((bio->bi_opf & (REQ_SYNC | REQ_IDLE)) == 624 (REQ_SYNC | REQ_IDLE)) 625 return false; 626 fallthrough; 627 case REQ_OP_DISCARD: 628 return true; 629 default: 630 return false; 631 } 632 } 633 634 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio) 635 { 636 enum wbt_flags flags = 0; 637 638 if (!rwb_enabled(rwb)) 639 return 0; 640 641 if (bio_op(bio) == REQ_OP_READ) { 642 flags = WBT_READ; 643 } else if (wbt_should_throttle(bio)) { 644 if (current_is_kswapd()) 645 flags |= WBT_KSWAPD; 646 if (bio_op(bio) == REQ_OP_DISCARD) 647 flags |= WBT_DISCARD; 648 flags |= WBT_TRACKED; 649 } 650 return flags; 651 } 652 653 static void wbt_cleanup(struct rq_qos *rqos, struct bio *bio) 654 { 655 struct rq_wb *rwb = RQWB(rqos); 656 enum wbt_flags flags = bio_to_wbt_flags(rwb, bio); 657 __wbt_done(rqos, flags); 658 } 659 660 /* 661 * May sleep, if we have exceeded the writeback limits. Caller can pass 662 * in an irq held spinlock, if it holds one when calling this function. 663 * If we do sleep, we'll release and re-grab it. 664 */ 665 static void wbt_wait(struct rq_qos *rqos, struct bio *bio) 666 { 667 struct rq_wb *rwb = RQWB(rqos); 668 enum wbt_flags flags; 669 670 flags = bio_to_wbt_flags(rwb, bio); 671 if (!(flags & WBT_TRACKED)) { 672 if (flags & WBT_READ) 673 wb_timestamp(rwb, &rwb->last_issue); 674 return; 675 } 676 677 __wbt_wait(rwb, flags, bio->bi_opf); 678 679 if (!blk_stat_is_active(rwb->cb)) 680 rwb_arm_timer(rwb); 681 } 682 683 static void wbt_track(struct rq_qos *rqos, struct request *rq, struct bio *bio) 684 { 685 struct rq_wb *rwb = RQWB(rqos); 686 rq->wbt_flags |= bio_to_wbt_flags(rwb, bio); 687 } 688 689 static void wbt_issue(struct rq_qos *rqos, struct request *rq) 690 { 691 struct rq_wb *rwb = RQWB(rqos); 692 693 if (!rwb_enabled(rwb)) 694 return; 695 696 /* 697 * Track sync issue, in case it takes a long time to complete. Allows us 698 * to react quicker, if a sync IO takes a long time to complete. Note 699 * that this is just a hint. The request can go away when it completes, 700 * so it's important we never dereference it. We only use the address to 701 * compare with, which is why we store the sync_issue time locally. 702 */ 703 if (wbt_is_read(rq) && !rwb->sync_issue) { 704 rwb->sync_cookie = rq; 705 rwb->sync_issue = rq->io_start_time_ns; 706 } 707 } 708 709 static void wbt_requeue(struct rq_qos *rqos, struct request *rq) 710 { 711 struct rq_wb *rwb = RQWB(rqos); 712 if (!rwb_enabled(rwb)) 713 return; 714 if (rq == rwb->sync_cookie) { 715 rwb->sync_issue = 0; 716 rwb->sync_cookie = NULL; 717 } 718 } 719 720 void wbt_set_write_cache(struct request_queue *q, bool write_cache_on) 721 { 722 struct rq_qos *rqos = wbt_rq_qos(q); 723 if (rqos) 724 RQWB(rqos)->wc = write_cache_on; 725 } 726 727 /* 728 * Enable wbt if defaults are configured that way 729 */ 730 void wbt_enable_default(struct gendisk *disk) 731 { 732 struct request_queue *q = disk->queue; 733 struct rq_qos *rqos; 734 bool disable_flag = q->elevator && 735 test_bit(ELEVATOR_FLAG_DISABLE_WBT, &q->elevator->flags); 736 737 /* Throttling already enabled? */ 738 rqos = wbt_rq_qos(q); 739 if (rqos) { 740 if (!disable_flag && 741 RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT) 742 RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT; 743 return; 744 } 745 746 /* Queue not registered? Maybe shutting down... */ 747 if (!blk_queue_registered(q)) 748 return; 749 750 if (queue_is_mq(q) && !disable_flag) 751 wbt_init(disk); 752 } 753 EXPORT_SYMBOL_GPL(wbt_enable_default); 754 755 u64 wbt_default_latency_nsec(struct request_queue *q) 756 { 757 /* 758 * We default to 2msec for non-rotational storage, and 75msec 759 * for rotational storage. 760 */ 761 if (blk_queue_nonrot(q)) 762 return 2000000ULL; 763 else 764 return 75000000ULL; 765 } 766 767 static int wbt_data_dir(const struct request *rq) 768 { 769 const enum req_op op = req_op(rq); 770 771 if (op == REQ_OP_READ) 772 return READ; 773 else if (op_is_write(op)) 774 return WRITE; 775 776 /* don't account */ 777 return -1; 778 } 779 780 static void wbt_queue_depth_changed(struct rq_qos *rqos) 781 { 782 RQWB(rqos)->rq_depth.queue_depth = blk_queue_depth(rqos->q); 783 wbt_update_limits(RQWB(rqos)); 784 } 785 786 static void wbt_exit(struct rq_qos *rqos) 787 { 788 struct rq_wb *rwb = RQWB(rqos); 789 struct request_queue *q = rqos->q; 790 791 blk_stat_remove_callback(q, rwb->cb); 792 blk_stat_free_callback(rwb->cb); 793 kfree(rwb); 794 } 795 796 /* 797 * Disable wbt, if enabled by default. 798 */ 799 void wbt_disable_default(struct gendisk *disk) 800 { 801 struct rq_qos *rqos = wbt_rq_qos(disk->queue); 802 struct rq_wb *rwb; 803 if (!rqos) 804 return; 805 rwb = RQWB(rqos); 806 if (rwb->enable_state == WBT_STATE_ON_DEFAULT) { 807 blk_stat_deactivate(rwb->cb); 808 rwb->enable_state = WBT_STATE_OFF_DEFAULT; 809 } 810 } 811 EXPORT_SYMBOL_GPL(wbt_disable_default); 812 813 #ifdef CONFIG_BLK_DEBUG_FS 814 static int wbt_curr_win_nsec_show(void *data, struct seq_file *m) 815 { 816 struct rq_qos *rqos = data; 817 struct rq_wb *rwb = RQWB(rqos); 818 819 seq_printf(m, "%llu\n", rwb->cur_win_nsec); 820 return 0; 821 } 822 823 static int wbt_enabled_show(void *data, struct seq_file *m) 824 { 825 struct rq_qos *rqos = data; 826 struct rq_wb *rwb = RQWB(rqos); 827 828 seq_printf(m, "%d\n", rwb->enable_state); 829 return 0; 830 } 831 832 static int wbt_id_show(void *data, struct seq_file *m) 833 { 834 struct rq_qos *rqos = data; 835 836 seq_printf(m, "%u\n", rqos->id); 837 return 0; 838 } 839 840 static int wbt_inflight_show(void *data, struct seq_file *m) 841 { 842 struct rq_qos *rqos = data; 843 struct rq_wb *rwb = RQWB(rqos); 844 int i; 845 846 for (i = 0; i < WBT_NUM_RWQ; i++) 847 seq_printf(m, "%d: inflight %d\n", i, 848 atomic_read(&rwb->rq_wait[i].inflight)); 849 return 0; 850 } 851 852 static int wbt_min_lat_nsec_show(void *data, struct seq_file *m) 853 { 854 struct rq_qos *rqos = data; 855 struct rq_wb *rwb = RQWB(rqos); 856 857 seq_printf(m, "%lu\n", rwb->min_lat_nsec); 858 return 0; 859 } 860 861 static int wbt_unknown_cnt_show(void *data, struct seq_file *m) 862 { 863 struct rq_qos *rqos = data; 864 struct rq_wb *rwb = RQWB(rqos); 865 866 seq_printf(m, "%u\n", rwb->unknown_cnt); 867 return 0; 868 } 869 870 static int wbt_normal_show(void *data, struct seq_file *m) 871 { 872 struct rq_qos *rqos = data; 873 struct rq_wb *rwb = RQWB(rqos); 874 875 seq_printf(m, "%u\n", rwb->wb_normal); 876 return 0; 877 } 878 879 static int wbt_background_show(void *data, struct seq_file *m) 880 { 881 struct rq_qos *rqos = data; 882 struct rq_wb *rwb = RQWB(rqos); 883 884 seq_printf(m, "%u\n", rwb->wb_background); 885 return 0; 886 } 887 888 static const struct blk_mq_debugfs_attr wbt_debugfs_attrs[] = { 889 {"curr_win_nsec", 0400, wbt_curr_win_nsec_show}, 890 {"enabled", 0400, wbt_enabled_show}, 891 {"id", 0400, wbt_id_show}, 892 {"inflight", 0400, wbt_inflight_show}, 893 {"min_lat_nsec", 0400, wbt_min_lat_nsec_show}, 894 {"unknown_cnt", 0400, wbt_unknown_cnt_show}, 895 {"wb_normal", 0400, wbt_normal_show}, 896 {"wb_background", 0400, wbt_background_show}, 897 {}, 898 }; 899 #endif 900 901 static struct rq_qos_ops wbt_rqos_ops = { 902 .throttle = wbt_wait, 903 .issue = wbt_issue, 904 .track = wbt_track, 905 .requeue = wbt_requeue, 906 .done = wbt_done, 907 .cleanup = wbt_cleanup, 908 .queue_depth_changed = wbt_queue_depth_changed, 909 .exit = wbt_exit, 910 #ifdef CONFIG_BLK_DEBUG_FS 911 .debugfs_attrs = wbt_debugfs_attrs, 912 #endif 913 }; 914 915 int wbt_init(struct gendisk *disk) 916 { 917 struct request_queue *q = disk->queue; 918 struct rq_wb *rwb; 919 int i; 920 int ret; 921 922 rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); 923 if (!rwb) 924 return -ENOMEM; 925 926 rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); 927 if (!rwb->cb) { 928 kfree(rwb); 929 return -ENOMEM; 930 } 931 932 for (i = 0; i < WBT_NUM_RWQ; i++) 933 rq_wait_init(&rwb->rq_wait[i]); 934 935 rwb->last_comp = rwb->last_issue = jiffies; 936 rwb->win_nsec = RWB_WINDOW_NSEC; 937 rwb->enable_state = WBT_STATE_ON_DEFAULT; 938 rwb->wc = test_bit(QUEUE_FLAG_WC, &q->queue_flags); 939 rwb->rq_depth.default_depth = RWB_DEF_DEPTH; 940 rwb->min_lat_nsec = wbt_default_latency_nsec(q); 941 rwb->rq_depth.queue_depth = blk_queue_depth(q); 942 wbt_update_limits(rwb); 943 944 /* 945 * Assign rwb and add the stats callback. 946 */ 947 ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops); 948 if (ret) 949 goto err_free; 950 951 blk_stat_add_callback(q, rwb->cb); 952 953 return 0; 954 955 err_free: 956 blk_stat_free_callback(rwb->cb); 957 kfree(rwb); 958 return ret; 959 960 } 961