1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common Block IO controller cgroup interface 4 * 5 * Based on ideas and code from CFQ, CFS and BFQ: 6 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk> 7 * 8 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it> 9 * Paolo Valente <paolo.valente@unimore.it> 10 * 11 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com> 12 * Nauman Rafique <nauman@google.com> 13 * 14 * For policy-specific per-blkcg data: 15 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it> 16 * Arianna Avanzini <avanzini.arianna@gmail.com> 17 */ 18 #include <linux/ioprio.h> 19 #include <linux/kdev_t.h> 20 #include <linux/module.h> 21 #include <linux/sched/signal.h> 22 #include <linux/err.h> 23 #include <linux/blkdev.h> 24 #include <linux/backing-dev.h> 25 #include <linux/slab.h> 26 #include <linux/genhd.h> 27 #include <linux/delay.h> 28 #include <linux/atomic.h> 29 #include <linux/ctype.h> 30 #include <linux/blk-cgroup.h> 31 #include <linux/tracehook.h> 32 #include <linux/psi.h> 33 #include "blk.h" 34 35 #define MAX_KEY_LEN 100 36 37 /* 38 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation. 39 * blkcg_pol_register_mutex nests outside of it and synchronizes entire 40 * policy [un]register operations including cgroup file additions / 41 * removals. Putting cgroup file registration outside blkcg_pol_mutex 42 * allows grabbing it from cgroup callbacks. 43 */ 44 static DEFINE_MUTEX(blkcg_pol_register_mutex); 45 static DEFINE_MUTEX(blkcg_pol_mutex); 46 47 struct blkcg blkcg_root; 48 EXPORT_SYMBOL_GPL(blkcg_root); 49 50 struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css; 51 EXPORT_SYMBOL_GPL(blkcg_root_css); 52 53 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS]; 54 55 static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */ 56 57 bool blkcg_debug_stats = false; 58 static struct workqueue_struct *blkcg_punt_bio_wq; 59 60 static bool blkcg_policy_enabled(struct request_queue *q, 61 const struct blkcg_policy *pol) 62 { 63 return pol && test_bit(pol->plid, q->blkcg_pols); 64 } 65 66 /** 67 * blkg_free - free a blkg 68 * @blkg: blkg to free 69 * 70 * Free @blkg which may be partially allocated. 71 */ 72 static void blkg_free(struct blkcg_gq *blkg) 73 { 74 int i; 75 76 if (!blkg) 77 return; 78 79 for (i = 0; i < BLKCG_MAX_POLS; i++) 80 if (blkg->pd[i]) 81 blkcg_policy[i]->pd_free_fn(blkg->pd[i]); 82 83 free_percpu(blkg->iostat_cpu); 84 percpu_ref_exit(&blkg->refcnt); 85 kfree(blkg); 86 } 87 88 static void __blkg_release(struct rcu_head *rcu) 89 { 90 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); 91 92 WARN_ON(!bio_list_empty(&blkg->async_bios)); 93 94 /* release the blkcg and parent blkg refs this blkg has been holding */ 95 css_put(&blkg->blkcg->css); 96 if (blkg->parent) 97 blkg_put(blkg->parent); 98 blkg_free(blkg); 99 } 100 101 /* 102 * A group is RCU protected, but having an rcu lock does not mean that one 103 * can access all the fields of blkg and assume these are valid. For 104 * example, don't try to follow throtl_data and request queue links. 105 * 106 * Having a reference to blkg under an rcu allows accesses to only values 107 * local to groups like group stats and group rate limits. 108 */ 109 static void blkg_release(struct percpu_ref *ref) 110 { 111 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt); 112 113 call_rcu(&blkg->rcu_head, __blkg_release); 114 } 115 116 static void blkg_async_bio_workfn(struct work_struct *work) 117 { 118 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, 119 async_bio_work); 120 struct bio_list bios = BIO_EMPTY_LIST; 121 struct bio *bio; 122 struct blk_plug plug; 123 bool need_plug = false; 124 125 /* as long as there are pending bios, @blkg can't go away */ 126 spin_lock_bh(&blkg->async_bio_lock); 127 bio_list_merge(&bios, &blkg->async_bios); 128 bio_list_init(&blkg->async_bios); 129 spin_unlock_bh(&blkg->async_bio_lock); 130 131 /* start plug only when bio_list contains at least 2 bios */ 132 if (bios.head && bios.head->bi_next) { 133 need_plug = true; 134 blk_start_plug(&plug); 135 } 136 while ((bio = bio_list_pop(&bios))) 137 submit_bio(bio); 138 if (need_plug) 139 blk_finish_plug(&plug); 140 } 141 142 /** 143 * blkg_alloc - allocate a blkg 144 * @blkcg: block cgroup the new blkg is associated with 145 * @q: request_queue the new blkg is associated with 146 * @gfp_mask: allocation mask to use 147 * 148 * Allocate a new blkg assocating @blkcg and @q. 149 */ 150 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, 151 gfp_t gfp_mask) 152 { 153 struct blkcg_gq *blkg; 154 int i, cpu; 155 156 /* alloc and init base part */ 157 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node); 158 if (!blkg) 159 return NULL; 160 161 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask)) 162 goto err_free; 163 164 blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask); 165 if (!blkg->iostat_cpu) 166 goto err_free; 167 168 blkg->q = q; 169 INIT_LIST_HEAD(&blkg->q_node); 170 spin_lock_init(&blkg->async_bio_lock); 171 bio_list_init(&blkg->async_bios); 172 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn); 173 blkg->blkcg = blkcg; 174 175 u64_stats_init(&blkg->iostat.sync); 176 for_each_possible_cpu(cpu) 177 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync); 178 179 for (i = 0; i < BLKCG_MAX_POLS; i++) { 180 struct blkcg_policy *pol = blkcg_policy[i]; 181 struct blkg_policy_data *pd; 182 183 if (!blkcg_policy_enabled(q, pol)) 184 continue; 185 186 /* alloc per-policy data and attach it to blkg */ 187 pd = pol->pd_alloc_fn(gfp_mask, q, blkcg); 188 if (!pd) 189 goto err_free; 190 191 blkg->pd[i] = pd; 192 pd->blkg = blkg; 193 pd->plid = i; 194 } 195 196 return blkg; 197 198 err_free: 199 blkg_free(blkg); 200 return NULL; 201 } 202 203 struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg, 204 struct request_queue *q, bool update_hint) 205 { 206 struct blkcg_gq *blkg; 207 208 /* 209 * Hint didn't match. Look up from the radix tree. Note that the 210 * hint can only be updated under queue_lock as otherwise @blkg 211 * could have already been removed from blkg_tree. The caller is 212 * responsible for grabbing queue_lock if @update_hint. 213 */ 214 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); 215 if (blkg && blkg->q == q) { 216 if (update_hint) { 217 lockdep_assert_held(&q->queue_lock); 218 rcu_assign_pointer(blkcg->blkg_hint, blkg); 219 } 220 return blkg; 221 } 222 223 return NULL; 224 } 225 EXPORT_SYMBOL_GPL(blkg_lookup_slowpath); 226 227 /* 228 * If @new_blkg is %NULL, this function tries to allocate a new one as 229 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return. 230 */ 231 static struct blkcg_gq *blkg_create(struct blkcg *blkcg, 232 struct request_queue *q, 233 struct blkcg_gq *new_blkg) 234 { 235 struct blkcg_gq *blkg; 236 int i, ret; 237 238 WARN_ON_ONCE(!rcu_read_lock_held()); 239 lockdep_assert_held(&q->queue_lock); 240 241 /* request_queue is dying, do not create/recreate a blkg */ 242 if (blk_queue_dying(q)) { 243 ret = -ENODEV; 244 goto err_free_blkg; 245 } 246 247 /* blkg holds a reference to blkcg */ 248 if (!css_tryget_online(&blkcg->css)) { 249 ret = -ENODEV; 250 goto err_free_blkg; 251 } 252 253 /* allocate */ 254 if (!new_blkg) { 255 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN); 256 if (unlikely(!new_blkg)) { 257 ret = -ENOMEM; 258 goto err_put_css; 259 } 260 } 261 blkg = new_blkg; 262 263 /* link parent */ 264 if (blkcg_parent(blkcg)) { 265 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); 266 if (WARN_ON_ONCE(!blkg->parent)) { 267 ret = -ENODEV; 268 goto err_put_css; 269 } 270 blkg_get(blkg->parent); 271 } 272 273 /* invoke per-policy init */ 274 for (i = 0; i < BLKCG_MAX_POLS; i++) { 275 struct blkcg_policy *pol = blkcg_policy[i]; 276 277 if (blkg->pd[i] && pol->pd_init_fn) 278 pol->pd_init_fn(blkg->pd[i]); 279 } 280 281 /* insert */ 282 spin_lock(&blkcg->lock); 283 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); 284 if (likely(!ret)) { 285 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); 286 list_add(&blkg->q_node, &q->blkg_list); 287 288 for (i = 0; i < BLKCG_MAX_POLS; i++) { 289 struct blkcg_policy *pol = blkcg_policy[i]; 290 291 if (blkg->pd[i] && pol->pd_online_fn) 292 pol->pd_online_fn(blkg->pd[i]); 293 } 294 } 295 blkg->online = true; 296 spin_unlock(&blkcg->lock); 297 298 if (!ret) 299 return blkg; 300 301 /* @blkg failed fully initialized, use the usual release path */ 302 blkg_put(blkg); 303 return ERR_PTR(ret); 304 305 err_put_css: 306 css_put(&blkcg->css); 307 err_free_blkg: 308 blkg_free(new_blkg); 309 return ERR_PTR(ret); 310 } 311 312 /** 313 * blkg_lookup_create - lookup blkg, try to create one if not there 314 * @blkcg: blkcg of interest 315 * @q: request_queue of interest 316 * 317 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to 318 * create one. blkg creation is performed recursively from blkcg_root such 319 * that all non-root blkg's have access to the parent blkg. This function 320 * should be called under RCU read lock and takes @q->queue_lock. 321 * 322 * Returns the blkg or the closest blkg if blkg_create() fails as it walks 323 * down from root. 324 */ 325 static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, 326 struct request_queue *q) 327 { 328 struct blkcg_gq *blkg; 329 unsigned long flags; 330 331 WARN_ON_ONCE(!rcu_read_lock_held()); 332 333 blkg = blkg_lookup(blkcg, q); 334 if (blkg) 335 return blkg; 336 337 spin_lock_irqsave(&q->queue_lock, flags); 338 blkg = __blkg_lookup(blkcg, q, true); 339 if (blkg) 340 goto found; 341 342 /* 343 * Create blkgs walking down from blkcg_root to @blkcg, so that all 344 * non-root blkgs have access to their parents. Returns the closest 345 * blkg to the intended blkg should blkg_create() fail. 346 */ 347 while (true) { 348 struct blkcg *pos = blkcg; 349 struct blkcg *parent = blkcg_parent(blkcg); 350 struct blkcg_gq *ret_blkg = q->root_blkg; 351 352 while (parent) { 353 blkg = __blkg_lookup(parent, q, false); 354 if (blkg) { 355 /* remember closest blkg */ 356 ret_blkg = blkg; 357 break; 358 } 359 pos = parent; 360 parent = blkcg_parent(parent); 361 } 362 363 blkg = blkg_create(pos, q, NULL); 364 if (IS_ERR(blkg)) { 365 blkg = ret_blkg; 366 break; 367 } 368 if (pos == blkcg) 369 break; 370 } 371 372 found: 373 spin_unlock_irqrestore(&q->queue_lock, flags); 374 return blkg; 375 } 376 377 static void blkg_destroy(struct blkcg_gq *blkg) 378 { 379 struct blkcg *blkcg = blkg->blkcg; 380 int i; 381 382 lockdep_assert_held(&blkg->q->queue_lock); 383 lockdep_assert_held(&blkcg->lock); 384 385 /* Something wrong if we are trying to remove same group twice */ 386 WARN_ON_ONCE(list_empty(&blkg->q_node)); 387 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); 388 389 for (i = 0; i < BLKCG_MAX_POLS; i++) { 390 struct blkcg_policy *pol = blkcg_policy[i]; 391 392 if (blkg->pd[i] && pol->pd_offline_fn) 393 pol->pd_offline_fn(blkg->pd[i]); 394 } 395 396 blkg->online = false; 397 398 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); 399 list_del_init(&blkg->q_node); 400 hlist_del_init_rcu(&blkg->blkcg_node); 401 402 /* 403 * Both setting lookup hint to and clearing it from @blkg are done 404 * under queue_lock. If it's not pointing to @blkg now, it never 405 * will. Hint assignment itself can race safely. 406 */ 407 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) 408 rcu_assign_pointer(blkcg->blkg_hint, NULL); 409 410 /* 411 * Put the reference taken at the time of creation so that when all 412 * queues are gone, group can be destroyed. 413 */ 414 percpu_ref_kill(&blkg->refcnt); 415 } 416 417 /** 418 * blkg_destroy_all - destroy all blkgs associated with a request_queue 419 * @q: request_queue of interest 420 * 421 * Destroy all blkgs associated with @q. 422 */ 423 static void blkg_destroy_all(struct request_queue *q) 424 { 425 struct blkcg_gq *blkg, *n; 426 427 spin_lock_irq(&q->queue_lock); 428 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { 429 struct blkcg *blkcg = blkg->blkcg; 430 431 spin_lock(&blkcg->lock); 432 blkg_destroy(blkg); 433 spin_unlock(&blkcg->lock); 434 } 435 436 q->root_blkg = NULL; 437 spin_unlock_irq(&q->queue_lock); 438 } 439 440 static int blkcg_reset_stats(struct cgroup_subsys_state *css, 441 struct cftype *cftype, u64 val) 442 { 443 struct blkcg *blkcg = css_to_blkcg(css); 444 struct blkcg_gq *blkg; 445 int i, cpu; 446 447 mutex_lock(&blkcg_pol_mutex); 448 spin_lock_irq(&blkcg->lock); 449 450 /* 451 * Note that stat reset is racy - it doesn't synchronize against 452 * stat updates. This is a debug feature which shouldn't exist 453 * anyway. If you get hit by a race, retry. 454 */ 455 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { 456 for_each_possible_cpu(cpu) { 457 struct blkg_iostat_set *bis = 458 per_cpu_ptr(blkg->iostat_cpu, cpu); 459 memset(bis, 0, sizeof(*bis)); 460 } 461 memset(&blkg->iostat, 0, sizeof(blkg->iostat)); 462 463 for (i = 0; i < BLKCG_MAX_POLS; i++) { 464 struct blkcg_policy *pol = blkcg_policy[i]; 465 466 if (blkg->pd[i] && pol->pd_reset_stats_fn) 467 pol->pd_reset_stats_fn(blkg->pd[i]); 468 } 469 } 470 471 spin_unlock_irq(&blkcg->lock); 472 mutex_unlock(&blkcg_pol_mutex); 473 return 0; 474 } 475 476 const char *blkg_dev_name(struct blkcg_gq *blkg) 477 { 478 /* some drivers (floppy) instantiate a queue w/o disk registered */ 479 if (blkg->q->backing_dev_info->dev) 480 return bdi_dev_name(blkg->q->backing_dev_info); 481 return NULL; 482 } 483 484 /** 485 * blkcg_print_blkgs - helper for printing per-blkg data 486 * @sf: seq_file to print to 487 * @blkcg: blkcg of interest 488 * @prfill: fill function to print out a blkg 489 * @pol: policy in question 490 * @data: data to be passed to @prfill 491 * @show_total: to print out sum of prfill return values or not 492 * 493 * This function invokes @prfill on each blkg of @blkcg if pd for the 494 * policy specified by @pol exists. @prfill is invoked with @sf, the 495 * policy data and @data and the matching queue lock held. If @show_total 496 * is %true, the sum of the return values from @prfill is printed with 497 * "Total" label at the end. 498 * 499 * This is to be used to construct print functions for 500 * cftype->read_seq_string method. 501 */ 502 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, 503 u64 (*prfill)(struct seq_file *, 504 struct blkg_policy_data *, int), 505 const struct blkcg_policy *pol, int data, 506 bool show_total) 507 { 508 struct blkcg_gq *blkg; 509 u64 total = 0; 510 511 rcu_read_lock(); 512 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { 513 spin_lock_irq(&blkg->q->queue_lock); 514 if (blkcg_policy_enabled(blkg->q, pol)) 515 total += prfill(sf, blkg->pd[pol->plid], data); 516 spin_unlock_irq(&blkg->q->queue_lock); 517 } 518 rcu_read_unlock(); 519 520 if (show_total) 521 seq_printf(sf, "Total %llu\n", (unsigned long long)total); 522 } 523 EXPORT_SYMBOL_GPL(blkcg_print_blkgs); 524 525 /** 526 * __blkg_prfill_u64 - prfill helper for a single u64 value 527 * @sf: seq_file to print to 528 * @pd: policy private data of interest 529 * @v: value to print 530 * 531 * Print @v to @sf for the device assocaited with @pd. 532 */ 533 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v) 534 { 535 const char *dname = blkg_dev_name(pd->blkg); 536 537 if (!dname) 538 return 0; 539 540 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v); 541 return v; 542 } 543 EXPORT_SYMBOL_GPL(__blkg_prfill_u64); 544 545 /* Performs queue bypass and policy enabled checks then looks up blkg. */ 546 static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg, 547 const struct blkcg_policy *pol, 548 struct request_queue *q) 549 { 550 WARN_ON_ONCE(!rcu_read_lock_held()); 551 lockdep_assert_held(&q->queue_lock); 552 553 if (!blkcg_policy_enabled(q, pol)) 554 return ERR_PTR(-EOPNOTSUPP); 555 return __blkg_lookup(blkcg, q, true /* update_hint */); 556 } 557 558 /** 559 * blkg_conf_prep - parse and prepare for per-blkg config update 560 * @inputp: input string pointer 561 * 562 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update 563 * from @input and get and return the matching gendisk. *@inputp is 564 * updated to point past the device node prefix. Returns an ERR_PTR() 565 * value on error. 566 * 567 * Use this function iff blkg_conf_prep() can't be used for some reason. 568 */ 569 struct gendisk *blkcg_conf_get_disk(char **inputp) 570 { 571 char *input = *inputp; 572 unsigned int major, minor; 573 struct gendisk *disk; 574 int key_len, part; 575 576 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2) 577 return ERR_PTR(-EINVAL); 578 579 input += key_len; 580 if (!isspace(*input)) 581 return ERR_PTR(-EINVAL); 582 input = skip_spaces(input); 583 584 disk = get_gendisk(MKDEV(major, minor), &part); 585 if (!disk) 586 return ERR_PTR(-ENODEV); 587 if (part) { 588 put_disk_and_module(disk); 589 return ERR_PTR(-ENODEV); 590 } 591 592 *inputp = input; 593 return disk; 594 } 595 596 /** 597 * blkg_conf_prep - parse and prepare for per-blkg config update 598 * @blkcg: target block cgroup 599 * @pol: target policy 600 * @input: input string 601 * @ctx: blkg_conf_ctx to be filled 602 * 603 * Parse per-blkg config update from @input and initialize @ctx with the 604 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the 605 * part of @input following MAJ:MIN. This function returns with RCU read 606 * lock and queue lock held and must be paired with blkg_conf_finish(). 607 */ 608 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, 609 char *input, struct blkg_conf_ctx *ctx) 610 __acquires(rcu) __acquires(&disk->queue->queue_lock) 611 { 612 struct gendisk *disk; 613 struct request_queue *q; 614 struct blkcg_gq *blkg; 615 int ret; 616 617 disk = blkcg_conf_get_disk(&input); 618 if (IS_ERR(disk)) 619 return PTR_ERR(disk); 620 621 q = disk->queue; 622 623 rcu_read_lock(); 624 spin_lock_irq(&q->queue_lock); 625 626 blkg = blkg_lookup_check(blkcg, pol, q); 627 if (IS_ERR(blkg)) { 628 ret = PTR_ERR(blkg); 629 goto fail_unlock; 630 } 631 632 if (blkg) 633 goto success; 634 635 /* 636 * Create blkgs walking down from blkcg_root to @blkcg, so that all 637 * non-root blkgs have access to their parents. 638 */ 639 while (true) { 640 struct blkcg *pos = blkcg; 641 struct blkcg *parent; 642 struct blkcg_gq *new_blkg; 643 644 parent = blkcg_parent(blkcg); 645 while (parent && !__blkg_lookup(parent, q, false)) { 646 pos = parent; 647 parent = blkcg_parent(parent); 648 } 649 650 /* Drop locks to do new blkg allocation with GFP_KERNEL. */ 651 spin_unlock_irq(&q->queue_lock); 652 rcu_read_unlock(); 653 654 new_blkg = blkg_alloc(pos, q, GFP_KERNEL); 655 if (unlikely(!new_blkg)) { 656 ret = -ENOMEM; 657 goto fail; 658 } 659 660 if (radix_tree_preload(GFP_KERNEL)) { 661 blkg_free(new_blkg); 662 ret = -ENOMEM; 663 goto fail; 664 } 665 666 rcu_read_lock(); 667 spin_lock_irq(&q->queue_lock); 668 669 blkg = blkg_lookup_check(pos, pol, q); 670 if (IS_ERR(blkg)) { 671 ret = PTR_ERR(blkg); 672 blkg_free(new_blkg); 673 goto fail_preloaded; 674 } 675 676 if (blkg) { 677 blkg_free(new_blkg); 678 } else { 679 blkg = blkg_create(pos, q, new_blkg); 680 if (IS_ERR(blkg)) { 681 ret = PTR_ERR(blkg); 682 goto fail_preloaded; 683 } 684 } 685 686 radix_tree_preload_end(); 687 688 if (pos == blkcg) 689 goto success; 690 } 691 success: 692 ctx->disk = disk; 693 ctx->blkg = blkg; 694 ctx->body = input; 695 return 0; 696 697 fail_preloaded: 698 radix_tree_preload_end(); 699 fail_unlock: 700 spin_unlock_irq(&q->queue_lock); 701 rcu_read_unlock(); 702 fail: 703 put_disk_and_module(disk); 704 /* 705 * If queue was bypassing, we should retry. Do so after a 706 * short msleep(). It isn't strictly necessary but queue 707 * can be bypassing for some time and it's always nice to 708 * avoid busy looping. 709 */ 710 if (ret == -EBUSY) { 711 msleep(10); 712 ret = restart_syscall(); 713 } 714 return ret; 715 } 716 EXPORT_SYMBOL_GPL(blkg_conf_prep); 717 718 /** 719 * blkg_conf_finish - finish up per-blkg config update 720 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep() 721 * 722 * Finish up after per-blkg config update. This function must be paired 723 * with blkg_conf_prep(). 724 */ 725 void blkg_conf_finish(struct blkg_conf_ctx *ctx) 726 __releases(&ctx->disk->queue->queue_lock) __releases(rcu) 727 { 728 spin_unlock_irq(&ctx->disk->queue->queue_lock); 729 rcu_read_unlock(); 730 put_disk_and_module(ctx->disk); 731 } 732 EXPORT_SYMBOL_GPL(blkg_conf_finish); 733 734 static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src) 735 { 736 int i; 737 738 for (i = 0; i < BLKG_IOSTAT_NR; i++) { 739 dst->bytes[i] = src->bytes[i]; 740 dst->ios[i] = src->ios[i]; 741 } 742 } 743 744 static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src) 745 { 746 int i; 747 748 for (i = 0; i < BLKG_IOSTAT_NR; i++) { 749 dst->bytes[i] += src->bytes[i]; 750 dst->ios[i] += src->ios[i]; 751 } 752 } 753 754 static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src) 755 { 756 int i; 757 758 for (i = 0; i < BLKG_IOSTAT_NR; i++) { 759 dst->bytes[i] -= src->bytes[i]; 760 dst->ios[i] -= src->ios[i]; 761 } 762 } 763 764 static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) 765 { 766 struct blkcg *blkcg = css_to_blkcg(css); 767 struct blkcg_gq *blkg; 768 769 rcu_read_lock(); 770 771 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { 772 struct blkcg_gq *parent = blkg->parent; 773 struct blkg_iostat_set *bisc = per_cpu_ptr(blkg->iostat_cpu, cpu); 774 struct blkg_iostat cur, delta; 775 unsigned int seq; 776 777 /* fetch the current per-cpu values */ 778 do { 779 seq = u64_stats_fetch_begin(&bisc->sync); 780 blkg_iostat_set(&cur, &bisc->cur); 781 } while (u64_stats_fetch_retry(&bisc->sync, seq)); 782 783 /* propagate percpu delta to global */ 784 u64_stats_update_begin(&blkg->iostat.sync); 785 blkg_iostat_set(&delta, &cur); 786 blkg_iostat_sub(&delta, &bisc->last); 787 blkg_iostat_add(&blkg->iostat.cur, &delta); 788 blkg_iostat_add(&bisc->last, &delta); 789 u64_stats_update_end(&blkg->iostat.sync); 790 791 /* propagate global delta to parent */ 792 if (parent) { 793 u64_stats_update_begin(&parent->iostat.sync); 794 blkg_iostat_set(&delta, &blkg->iostat.cur); 795 blkg_iostat_sub(&delta, &blkg->iostat.last); 796 blkg_iostat_add(&parent->iostat.cur, &delta); 797 blkg_iostat_add(&blkg->iostat.last, &delta); 798 u64_stats_update_end(&parent->iostat.sync); 799 } 800 } 801 802 rcu_read_unlock(); 803 } 804 805 /* 806 * The rstat algorithms intentionally don't handle the root cgroup to avoid 807 * incurring overhead when no cgroups are defined. For that reason, 808 * cgroup_rstat_flush in blkcg_print_stat does not actually fill out the 809 * iostat in the root cgroup's blkcg_gq. 810 * 811 * However, we would like to re-use the printing code between the root and 812 * non-root cgroups to the extent possible. For that reason, we simulate 813 * flushing the root cgroup's stats by explicitly filling in the iostat 814 * with disk level statistics. 815 */ 816 static void blkcg_fill_root_iostats(void) 817 { 818 struct class_dev_iter iter; 819 struct device *dev; 820 821 class_dev_iter_init(&iter, &block_class, NULL, &disk_type); 822 while ((dev = class_dev_iter_next(&iter))) { 823 struct gendisk *disk = dev_to_disk(dev); 824 struct hd_struct *part = disk_get_part(disk, 0); 825 struct blkcg_gq *blkg = blk_queue_root_blkg(disk->queue); 826 struct blkg_iostat tmp; 827 int cpu; 828 829 memset(&tmp, 0, sizeof(tmp)); 830 for_each_possible_cpu(cpu) { 831 struct disk_stats *cpu_dkstats; 832 833 cpu_dkstats = per_cpu_ptr(part->dkstats, cpu); 834 tmp.ios[BLKG_IOSTAT_READ] += 835 cpu_dkstats->ios[STAT_READ]; 836 tmp.ios[BLKG_IOSTAT_WRITE] += 837 cpu_dkstats->ios[STAT_WRITE]; 838 tmp.ios[BLKG_IOSTAT_DISCARD] += 839 cpu_dkstats->ios[STAT_DISCARD]; 840 // convert sectors to bytes 841 tmp.bytes[BLKG_IOSTAT_READ] += 842 cpu_dkstats->sectors[STAT_READ] << 9; 843 tmp.bytes[BLKG_IOSTAT_WRITE] += 844 cpu_dkstats->sectors[STAT_WRITE] << 9; 845 tmp.bytes[BLKG_IOSTAT_DISCARD] += 846 cpu_dkstats->sectors[STAT_DISCARD] << 9; 847 848 u64_stats_update_begin(&blkg->iostat.sync); 849 blkg_iostat_set(&blkg->iostat.cur, &tmp); 850 u64_stats_update_end(&blkg->iostat.sync); 851 } 852 } 853 } 854 855 static int blkcg_print_stat(struct seq_file *sf, void *v) 856 { 857 struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); 858 struct blkcg_gq *blkg; 859 860 if (!seq_css(sf)->parent) 861 blkcg_fill_root_iostats(); 862 else 863 cgroup_rstat_flush(blkcg->css.cgroup); 864 865 rcu_read_lock(); 866 867 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { 868 struct blkg_iostat_set *bis = &blkg->iostat; 869 const char *dname; 870 char *buf; 871 u64 rbytes, wbytes, rios, wios, dbytes, dios; 872 size_t size = seq_get_buf(sf, &buf), off = 0; 873 int i; 874 bool has_stats = false; 875 unsigned seq; 876 877 spin_lock_irq(&blkg->q->queue_lock); 878 879 if (!blkg->online) 880 goto skip; 881 882 dname = blkg_dev_name(blkg); 883 if (!dname) 884 goto skip; 885 886 /* 887 * Hooray string manipulation, count is the size written NOT 888 * INCLUDING THE \0, so size is now count+1 less than what we 889 * had before, but we want to start writing the next bit from 890 * the \0 so we only add count to buf. 891 */ 892 off += scnprintf(buf+off, size-off, "%s ", dname); 893 894 do { 895 seq = u64_stats_fetch_begin(&bis->sync); 896 897 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ]; 898 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE]; 899 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD]; 900 rios = bis->cur.ios[BLKG_IOSTAT_READ]; 901 wios = bis->cur.ios[BLKG_IOSTAT_WRITE]; 902 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD]; 903 } while (u64_stats_fetch_retry(&bis->sync, seq)); 904 905 if (rbytes || wbytes || rios || wios) { 906 has_stats = true; 907 off += scnprintf(buf+off, size-off, 908 "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu", 909 rbytes, wbytes, rios, wios, 910 dbytes, dios); 911 } 912 913 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) { 914 has_stats = true; 915 off += scnprintf(buf+off, size-off, 916 " use_delay=%d delay_nsec=%llu", 917 atomic_read(&blkg->use_delay), 918 (unsigned long long)atomic64_read(&blkg->delay_nsec)); 919 } 920 921 for (i = 0; i < BLKCG_MAX_POLS; i++) { 922 struct blkcg_policy *pol = blkcg_policy[i]; 923 size_t written; 924 925 if (!blkg->pd[i] || !pol->pd_stat_fn) 926 continue; 927 928 written = pol->pd_stat_fn(blkg->pd[i], buf+off, size-off); 929 if (written) 930 has_stats = true; 931 off += written; 932 } 933 934 if (has_stats) { 935 if (off < size - 1) { 936 off += scnprintf(buf+off, size-off, "\n"); 937 seq_commit(sf, off); 938 } else { 939 seq_commit(sf, -1); 940 } 941 } 942 skip: 943 spin_unlock_irq(&blkg->q->queue_lock); 944 } 945 946 rcu_read_unlock(); 947 return 0; 948 } 949 950 static struct cftype blkcg_files[] = { 951 { 952 .name = "stat", 953 .seq_show = blkcg_print_stat, 954 }, 955 { } /* terminate */ 956 }; 957 958 static struct cftype blkcg_legacy_files[] = { 959 { 960 .name = "reset_stats", 961 .write_u64 = blkcg_reset_stats, 962 }, 963 { } /* terminate */ 964 }; 965 966 /* 967 * blkcg destruction is a three-stage process. 968 * 969 * 1. Destruction starts. The blkcg_css_offline() callback is invoked 970 * which offlines writeback. Here we tie the next stage of blkg destruction 971 * to the completion of writeback associated with the blkcg. This lets us 972 * avoid punting potentially large amounts of outstanding writeback to root 973 * while maintaining any ongoing policies. The next stage is triggered when 974 * the nr_cgwbs count goes to zero. 975 * 976 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called 977 * and handles the destruction of blkgs. Here the css reference held by 978 * the blkg is put back eventually allowing blkcg_css_free() to be called. 979 * This work may occur in cgwb_release_workfn() on the cgwb_release 980 * workqueue. Any submitted ios that fail to get the blkg ref will be 981 * punted to the root_blkg. 982 * 983 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called. 984 * This finally frees the blkcg. 985 */ 986 987 /** 988 * blkcg_css_offline - cgroup css_offline callback 989 * @css: css of interest 990 * 991 * This function is called when @css is about to go away. Here the cgwbs are 992 * offlined first and only once writeback associated with the blkcg has 993 * finished do we start step 2 (see above). 994 */ 995 static void blkcg_css_offline(struct cgroup_subsys_state *css) 996 { 997 struct blkcg *blkcg = css_to_blkcg(css); 998 999 /* this prevents anyone from attaching or migrating to this blkcg */ 1000 wb_blkcg_offline(blkcg); 1001 1002 /* put the base online pin allowing step 2 to be triggered */ 1003 blkcg_unpin_online(blkcg); 1004 } 1005 1006 /** 1007 * blkcg_destroy_blkgs - responsible for shooting down blkgs 1008 * @blkcg: blkcg of interest 1009 * 1010 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock 1011 * is nested inside q lock, this function performs reverse double lock dancing. 1012 * Destroying the blkgs releases the reference held on the blkcg's css allowing 1013 * blkcg_css_free to eventually be called. 1014 * 1015 * This is the blkcg counterpart of ioc_release_fn(). 1016 */ 1017 void blkcg_destroy_blkgs(struct blkcg *blkcg) 1018 { 1019 spin_lock_irq(&blkcg->lock); 1020 1021 while (!hlist_empty(&blkcg->blkg_list)) { 1022 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, 1023 struct blkcg_gq, blkcg_node); 1024 struct request_queue *q = blkg->q; 1025 1026 if (spin_trylock(&q->queue_lock)) { 1027 blkg_destroy(blkg); 1028 spin_unlock(&q->queue_lock); 1029 } else { 1030 spin_unlock_irq(&blkcg->lock); 1031 cpu_relax(); 1032 spin_lock_irq(&blkcg->lock); 1033 } 1034 } 1035 1036 spin_unlock_irq(&blkcg->lock); 1037 } 1038 1039 static void blkcg_css_free(struct cgroup_subsys_state *css) 1040 { 1041 struct blkcg *blkcg = css_to_blkcg(css); 1042 int i; 1043 1044 mutex_lock(&blkcg_pol_mutex); 1045 1046 list_del(&blkcg->all_blkcgs_node); 1047 1048 for (i = 0; i < BLKCG_MAX_POLS; i++) 1049 if (blkcg->cpd[i]) 1050 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); 1051 1052 mutex_unlock(&blkcg_pol_mutex); 1053 1054 kfree(blkcg); 1055 } 1056 1057 static struct cgroup_subsys_state * 1058 blkcg_css_alloc(struct cgroup_subsys_state *parent_css) 1059 { 1060 struct blkcg *blkcg; 1061 struct cgroup_subsys_state *ret; 1062 int i; 1063 1064 mutex_lock(&blkcg_pol_mutex); 1065 1066 if (!parent_css) { 1067 blkcg = &blkcg_root; 1068 } else { 1069 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); 1070 if (!blkcg) { 1071 ret = ERR_PTR(-ENOMEM); 1072 goto unlock; 1073 } 1074 } 1075 1076 for (i = 0; i < BLKCG_MAX_POLS ; i++) { 1077 struct blkcg_policy *pol = blkcg_policy[i]; 1078 struct blkcg_policy_data *cpd; 1079 1080 /* 1081 * If the policy hasn't been attached yet, wait for it 1082 * to be attached before doing anything else. Otherwise, 1083 * check if the policy requires any specific per-cgroup 1084 * data: if it does, allocate and initialize it. 1085 */ 1086 if (!pol || !pol->cpd_alloc_fn) 1087 continue; 1088 1089 cpd = pol->cpd_alloc_fn(GFP_KERNEL); 1090 if (!cpd) { 1091 ret = ERR_PTR(-ENOMEM); 1092 goto free_pd_blkcg; 1093 } 1094 blkcg->cpd[i] = cpd; 1095 cpd->blkcg = blkcg; 1096 cpd->plid = i; 1097 if (pol->cpd_init_fn) 1098 pol->cpd_init_fn(cpd); 1099 } 1100 1101 spin_lock_init(&blkcg->lock); 1102 refcount_set(&blkcg->online_pin, 1); 1103 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN); 1104 INIT_HLIST_HEAD(&blkcg->blkg_list); 1105 #ifdef CONFIG_CGROUP_WRITEBACK 1106 INIT_LIST_HEAD(&blkcg->cgwb_list); 1107 #endif 1108 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs); 1109 1110 mutex_unlock(&blkcg_pol_mutex); 1111 return &blkcg->css; 1112 1113 free_pd_blkcg: 1114 for (i--; i >= 0; i--) 1115 if (blkcg->cpd[i]) 1116 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); 1117 1118 if (blkcg != &blkcg_root) 1119 kfree(blkcg); 1120 unlock: 1121 mutex_unlock(&blkcg_pol_mutex); 1122 return ret; 1123 } 1124 1125 static int blkcg_css_online(struct cgroup_subsys_state *css) 1126 { 1127 struct blkcg *blkcg = css_to_blkcg(css); 1128 struct blkcg *parent = blkcg_parent(blkcg); 1129 1130 /* 1131 * blkcg_pin_online() is used to delay blkcg offline so that blkgs 1132 * don't go offline while cgwbs are still active on them. Pin the 1133 * parent so that offline always happens towards the root. 1134 */ 1135 if (parent) 1136 blkcg_pin_online(parent); 1137 return 0; 1138 } 1139 1140 /** 1141 * blkcg_init_queue - initialize blkcg part of request queue 1142 * @q: request_queue to initialize 1143 * 1144 * Called from blk_alloc_queue(). Responsible for initializing blkcg 1145 * part of new request_queue @q. 1146 * 1147 * RETURNS: 1148 * 0 on success, -errno on failure. 1149 */ 1150 int blkcg_init_queue(struct request_queue *q) 1151 { 1152 struct blkcg_gq *new_blkg, *blkg; 1153 bool preloaded; 1154 int ret; 1155 1156 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL); 1157 if (!new_blkg) 1158 return -ENOMEM; 1159 1160 preloaded = !radix_tree_preload(GFP_KERNEL); 1161 1162 /* Make sure the root blkg exists. */ 1163 rcu_read_lock(); 1164 spin_lock_irq(&q->queue_lock); 1165 blkg = blkg_create(&blkcg_root, q, new_blkg); 1166 if (IS_ERR(blkg)) 1167 goto err_unlock; 1168 q->root_blkg = blkg; 1169 spin_unlock_irq(&q->queue_lock); 1170 rcu_read_unlock(); 1171 1172 if (preloaded) 1173 radix_tree_preload_end(); 1174 1175 ret = blk_throtl_init(q); 1176 if (ret) 1177 goto err_destroy_all; 1178 1179 ret = blk_iolatency_init(q); 1180 if (ret) { 1181 blk_throtl_exit(q); 1182 goto err_destroy_all; 1183 } 1184 return 0; 1185 1186 err_destroy_all: 1187 blkg_destroy_all(q); 1188 return ret; 1189 err_unlock: 1190 spin_unlock_irq(&q->queue_lock); 1191 rcu_read_unlock(); 1192 if (preloaded) 1193 radix_tree_preload_end(); 1194 return PTR_ERR(blkg); 1195 } 1196 1197 /** 1198 * blkcg_exit_queue - exit and release blkcg part of request_queue 1199 * @q: request_queue being released 1200 * 1201 * Called from blk_exit_queue(). Responsible for exiting blkcg part. 1202 */ 1203 void blkcg_exit_queue(struct request_queue *q) 1204 { 1205 blkg_destroy_all(q); 1206 blk_throtl_exit(q); 1207 } 1208 1209 /* 1210 * We cannot support shared io contexts, as we have no mean to support 1211 * two tasks with the same ioc in two different groups without major rework 1212 * of the main cic data structures. For now we allow a task to change 1213 * its cgroup only if it's the only owner of its ioc. 1214 */ 1215 static int blkcg_can_attach(struct cgroup_taskset *tset) 1216 { 1217 struct task_struct *task; 1218 struct cgroup_subsys_state *dst_css; 1219 struct io_context *ioc; 1220 int ret = 0; 1221 1222 /* task_lock() is needed to avoid races with exit_io_context() */ 1223 cgroup_taskset_for_each(task, dst_css, tset) { 1224 task_lock(task); 1225 ioc = task->io_context; 1226 if (ioc && atomic_read(&ioc->nr_tasks) > 1) 1227 ret = -EINVAL; 1228 task_unlock(task); 1229 if (ret) 1230 break; 1231 } 1232 return ret; 1233 } 1234 1235 static void blkcg_bind(struct cgroup_subsys_state *root_css) 1236 { 1237 int i; 1238 1239 mutex_lock(&blkcg_pol_mutex); 1240 1241 for (i = 0; i < BLKCG_MAX_POLS; i++) { 1242 struct blkcg_policy *pol = blkcg_policy[i]; 1243 struct blkcg *blkcg; 1244 1245 if (!pol || !pol->cpd_bind_fn) 1246 continue; 1247 1248 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) 1249 if (blkcg->cpd[pol->plid]) 1250 pol->cpd_bind_fn(blkcg->cpd[pol->plid]); 1251 } 1252 mutex_unlock(&blkcg_pol_mutex); 1253 } 1254 1255 static void blkcg_exit(struct task_struct *tsk) 1256 { 1257 if (tsk->throttle_queue) 1258 blk_put_queue(tsk->throttle_queue); 1259 tsk->throttle_queue = NULL; 1260 } 1261 1262 struct cgroup_subsys io_cgrp_subsys = { 1263 .css_alloc = blkcg_css_alloc, 1264 .css_online = blkcg_css_online, 1265 .css_offline = blkcg_css_offline, 1266 .css_free = blkcg_css_free, 1267 .can_attach = blkcg_can_attach, 1268 .css_rstat_flush = blkcg_rstat_flush, 1269 .bind = blkcg_bind, 1270 .dfl_cftypes = blkcg_files, 1271 .legacy_cftypes = blkcg_legacy_files, 1272 .legacy_name = "blkio", 1273 .exit = blkcg_exit, 1274 #ifdef CONFIG_MEMCG 1275 /* 1276 * This ensures that, if available, memcg is automatically enabled 1277 * together on the default hierarchy so that the owner cgroup can 1278 * be retrieved from writeback pages. 1279 */ 1280 .depends_on = 1 << memory_cgrp_id, 1281 #endif 1282 }; 1283 EXPORT_SYMBOL_GPL(io_cgrp_subsys); 1284 1285 /** 1286 * blkcg_activate_policy - activate a blkcg policy on a request_queue 1287 * @q: request_queue of interest 1288 * @pol: blkcg policy to activate 1289 * 1290 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through 1291 * bypass mode to populate its blkgs with policy_data for @pol. 1292 * 1293 * Activation happens with @q bypassed, so nobody would be accessing blkgs 1294 * from IO path. Update of each blkg is protected by both queue and blkcg 1295 * locks so that holding either lock and testing blkcg_policy_enabled() is 1296 * always enough for dereferencing policy data. 1297 * 1298 * The caller is responsible for synchronizing [de]activations and policy 1299 * [un]registerations. Returns 0 on success, -errno on failure. 1300 */ 1301 int blkcg_activate_policy(struct request_queue *q, 1302 const struct blkcg_policy *pol) 1303 { 1304 struct blkg_policy_data *pd_prealloc = NULL; 1305 struct blkcg_gq *blkg, *pinned_blkg = NULL; 1306 int ret; 1307 1308 if (blkcg_policy_enabled(q, pol)) 1309 return 0; 1310 1311 if (queue_is_mq(q)) 1312 blk_mq_freeze_queue(q); 1313 retry: 1314 spin_lock_irq(&q->queue_lock); 1315 1316 /* blkg_list is pushed at the head, reverse walk to allocate parents first */ 1317 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { 1318 struct blkg_policy_data *pd; 1319 1320 if (blkg->pd[pol->plid]) 1321 continue; 1322 1323 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */ 1324 if (blkg == pinned_blkg) { 1325 pd = pd_prealloc; 1326 pd_prealloc = NULL; 1327 } else { 1328 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q, 1329 blkg->blkcg); 1330 } 1331 1332 if (!pd) { 1333 /* 1334 * GFP_NOWAIT failed. Free the existing one and 1335 * prealloc for @blkg w/ GFP_KERNEL. 1336 */ 1337 if (pinned_blkg) 1338 blkg_put(pinned_blkg); 1339 blkg_get(blkg); 1340 pinned_blkg = blkg; 1341 1342 spin_unlock_irq(&q->queue_lock); 1343 1344 if (pd_prealloc) 1345 pol->pd_free_fn(pd_prealloc); 1346 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q, 1347 blkg->blkcg); 1348 if (pd_prealloc) 1349 goto retry; 1350 else 1351 goto enomem; 1352 } 1353 1354 blkg->pd[pol->plid] = pd; 1355 pd->blkg = blkg; 1356 pd->plid = pol->plid; 1357 } 1358 1359 /* all allocated, init in the same order */ 1360 if (pol->pd_init_fn) 1361 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) 1362 pol->pd_init_fn(blkg->pd[pol->plid]); 1363 1364 __set_bit(pol->plid, q->blkcg_pols); 1365 ret = 0; 1366 1367 spin_unlock_irq(&q->queue_lock); 1368 out: 1369 if (queue_is_mq(q)) 1370 blk_mq_unfreeze_queue(q); 1371 if (pinned_blkg) 1372 blkg_put(pinned_blkg); 1373 if (pd_prealloc) 1374 pol->pd_free_fn(pd_prealloc); 1375 return ret; 1376 1377 enomem: 1378 /* alloc failed, nothing's initialized yet, free everything */ 1379 spin_lock_irq(&q->queue_lock); 1380 list_for_each_entry(blkg, &q->blkg_list, q_node) { 1381 if (blkg->pd[pol->plid]) { 1382 pol->pd_free_fn(blkg->pd[pol->plid]); 1383 blkg->pd[pol->plid] = NULL; 1384 } 1385 } 1386 spin_unlock_irq(&q->queue_lock); 1387 ret = -ENOMEM; 1388 goto out; 1389 } 1390 EXPORT_SYMBOL_GPL(blkcg_activate_policy); 1391 1392 /** 1393 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue 1394 * @q: request_queue of interest 1395 * @pol: blkcg policy to deactivate 1396 * 1397 * Deactivate @pol on @q. Follows the same synchronization rules as 1398 * blkcg_activate_policy(). 1399 */ 1400 void blkcg_deactivate_policy(struct request_queue *q, 1401 const struct blkcg_policy *pol) 1402 { 1403 struct blkcg_gq *blkg; 1404 1405 if (!blkcg_policy_enabled(q, pol)) 1406 return; 1407 1408 if (queue_is_mq(q)) 1409 blk_mq_freeze_queue(q); 1410 1411 spin_lock_irq(&q->queue_lock); 1412 1413 __clear_bit(pol->plid, q->blkcg_pols); 1414 1415 list_for_each_entry(blkg, &q->blkg_list, q_node) { 1416 if (blkg->pd[pol->plid]) { 1417 if (pol->pd_offline_fn) 1418 pol->pd_offline_fn(blkg->pd[pol->plid]); 1419 pol->pd_free_fn(blkg->pd[pol->plid]); 1420 blkg->pd[pol->plid] = NULL; 1421 } 1422 } 1423 1424 spin_unlock_irq(&q->queue_lock); 1425 1426 if (queue_is_mq(q)) 1427 blk_mq_unfreeze_queue(q); 1428 } 1429 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy); 1430 1431 /** 1432 * blkcg_policy_register - register a blkcg policy 1433 * @pol: blkcg policy to register 1434 * 1435 * Register @pol with blkcg core. Might sleep and @pol may be modified on 1436 * successful registration. Returns 0 on success and -errno on failure. 1437 */ 1438 int blkcg_policy_register(struct blkcg_policy *pol) 1439 { 1440 struct blkcg *blkcg; 1441 int i, ret; 1442 1443 mutex_lock(&blkcg_pol_register_mutex); 1444 mutex_lock(&blkcg_pol_mutex); 1445 1446 /* find an empty slot */ 1447 ret = -ENOSPC; 1448 for (i = 0; i < BLKCG_MAX_POLS; i++) 1449 if (!blkcg_policy[i]) 1450 break; 1451 if (i >= BLKCG_MAX_POLS) { 1452 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n"); 1453 goto err_unlock; 1454 } 1455 1456 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */ 1457 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) || 1458 (!pol->pd_alloc_fn ^ !pol->pd_free_fn)) 1459 goto err_unlock; 1460 1461 /* register @pol */ 1462 pol->plid = i; 1463 blkcg_policy[pol->plid] = pol; 1464 1465 /* allocate and install cpd's */ 1466 if (pol->cpd_alloc_fn) { 1467 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { 1468 struct blkcg_policy_data *cpd; 1469 1470 cpd = pol->cpd_alloc_fn(GFP_KERNEL); 1471 if (!cpd) 1472 goto err_free_cpds; 1473 1474 blkcg->cpd[pol->plid] = cpd; 1475 cpd->blkcg = blkcg; 1476 cpd->plid = pol->plid; 1477 if (pol->cpd_init_fn) 1478 pol->cpd_init_fn(cpd); 1479 } 1480 } 1481 1482 mutex_unlock(&blkcg_pol_mutex); 1483 1484 /* everything is in place, add intf files for the new policy */ 1485 if (pol->dfl_cftypes) 1486 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys, 1487 pol->dfl_cftypes)); 1488 if (pol->legacy_cftypes) 1489 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys, 1490 pol->legacy_cftypes)); 1491 mutex_unlock(&blkcg_pol_register_mutex); 1492 return 0; 1493 1494 err_free_cpds: 1495 if (pol->cpd_free_fn) { 1496 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { 1497 if (blkcg->cpd[pol->plid]) { 1498 pol->cpd_free_fn(blkcg->cpd[pol->plid]); 1499 blkcg->cpd[pol->plid] = NULL; 1500 } 1501 } 1502 } 1503 blkcg_policy[pol->plid] = NULL; 1504 err_unlock: 1505 mutex_unlock(&blkcg_pol_mutex); 1506 mutex_unlock(&blkcg_pol_register_mutex); 1507 return ret; 1508 } 1509 EXPORT_SYMBOL_GPL(blkcg_policy_register); 1510 1511 /** 1512 * blkcg_policy_unregister - unregister a blkcg policy 1513 * @pol: blkcg policy to unregister 1514 * 1515 * Undo blkcg_policy_register(@pol). Might sleep. 1516 */ 1517 void blkcg_policy_unregister(struct blkcg_policy *pol) 1518 { 1519 struct blkcg *blkcg; 1520 1521 mutex_lock(&blkcg_pol_register_mutex); 1522 1523 if (WARN_ON(blkcg_policy[pol->plid] != pol)) 1524 goto out_unlock; 1525 1526 /* kill the intf files first */ 1527 if (pol->dfl_cftypes) 1528 cgroup_rm_cftypes(pol->dfl_cftypes); 1529 if (pol->legacy_cftypes) 1530 cgroup_rm_cftypes(pol->legacy_cftypes); 1531 1532 /* remove cpds and unregister */ 1533 mutex_lock(&blkcg_pol_mutex); 1534 1535 if (pol->cpd_free_fn) { 1536 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { 1537 if (blkcg->cpd[pol->plid]) { 1538 pol->cpd_free_fn(blkcg->cpd[pol->plid]); 1539 blkcg->cpd[pol->plid] = NULL; 1540 } 1541 } 1542 } 1543 blkcg_policy[pol->plid] = NULL; 1544 1545 mutex_unlock(&blkcg_pol_mutex); 1546 out_unlock: 1547 mutex_unlock(&blkcg_pol_register_mutex); 1548 } 1549 EXPORT_SYMBOL_GPL(blkcg_policy_unregister); 1550 1551 bool __blkcg_punt_bio_submit(struct bio *bio) 1552 { 1553 struct blkcg_gq *blkg = bio->bi_blkg; 1554 1555 /* consume the flag first */ 1556 bio->bi_opf &= ~REQ_CGROUP_PUNT; 1557 1558 /* never bounce for the root cgroup */ 1559 if (!blkg->parent) 1560 return false; 1561 1562 spin_lock_bh(&blkg->async_bio_lock); 1563 bio_list_add(&blkg->async_bios, bio); 1564 spin_unlock_bh(&blkg->async_bio_lock); 1565 1566 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work); 1567 return true; 1568 } 1569 1570 /* 1571 * Scale the accumulated delay based on how long it has been since we updated 1572 * the delay. We only call this when we are adding delay, in case it's been a 1573 * while since we added delay, and when we are checking to see if we need to 1574 * delay a task, to account for any delays that may have occurred. 1575 */ 1576 static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now) 1577 { 1578 u64 old = atomic64_read(&blkg->delay_start); 1579 1580 /* negative use_delay means no scaling, see blkcg_set_delay() */ 1581 if (atomic_read(&blkg->use_delay) < 0) 1582 return; 1583 1584 /* 1585 * We only want to scale down every second. The idea here is that we 1586 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain 1587 * time window. We only want to throttle tasks for recent delay that 1588 * has occurred, in 1 second time windows since that's the maximum 1589 * things can be throttled. We save the current delay window in 1590 * blkg->last_delay so we know what amount is still left to be charged 1591 * to the blkg from this point onward. blkg->last_use keeps track of 1592 * the use_delay counter. The idea is if we're unthrottling the blkg we 1593 * are ok with whatever is happening now, and we can take away more of 1594 * the accumulated delay as we've already throttled enough that 1595 * everybody is happy with their IO latencies. 1596 */ 1597 if (time_before64(old + NSEC_PER_SEC, now) && 1598 atomic64_cmpxchg(&blkg->delay_start, old, now) == old) { 1599 u64 cur = atomic64_read(&blkg->delay_nsec); 1600 u64 sub = min_t(u64, blkg->last_delay, now - old); 1601 int cur_use = atomic_read(&blkg->use_delay); 1602 1603 /* 1604 * We've been unthrottled, subtract a larger chunk of our 1605 * accumulated delay. 1606 */ 1607 if (cur_use < blkg->last_use) 1608 sub = max_t(u64, sub, blkg->last_delay >> 1); 1609 1610 /* 1611 * This shouldn't happen, but handle it anyway. Our delay_nsec 1612 * should only ever be growing except here where we subtract out 1613 * min(last_delay, 1 second), but lord knows bugs happen and I'd 1614 * rather not end up with negative numbers. 1615 */ 1616 if (unlikely(cur < sub)) { 1617 atomic64_set(&blkg->delay_nsec, 0); 1618 blkg->last_delay = 0; 1619 } else { 1620 atomic64_sub(sub, &blkg->delay_nsec); 1621 blkg->last_delay = cur - sub; 1622 } 1623 blkg->last_use = cur_use; 1624 } 1625 } 1626 1627 /* 1628 * This is called when we want to actually walk up the hierarchy and check to 1629 * see if we need to throttle, and then actually throttle if there is some 1630 * accumulated delay. This should only be called upon return to user space so 1631 * we're not holding some lock that would induce a priority inversion. 1632 */ 1633 static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay) 1634 { 1635 unsigned long pflags; 1636 bool clamp; 1637 u64 now = ktime_to_ns(ktime_get()); 1638 u64 exp; 1639 u64 delay_nsec = 0; 1640 int tok; 1641 1642 while (blkg->parent) { 1643 int use_delay = atomic_read(&blkg->use_delay); 1644 1645 if (use_delay) { 1646 u64 this_delay; 1647 1648 blkcg_scale_delay(blkg, now); 1649 this_delay = atomic64_read(&blkg->delay_nsec); 1650 if (this_delay > delay_nsec) { 1651 delay_nsec = this_delay; 1652 clamp = use_delay > 0; 1653 } 1654 } 1655 blkg = blkg->parent; 1656 } 1657 1658 if (!delay_nsec) 1659 return; 1660 1661 /* 1662 * Let's not sleep for all eternity if we've amassed a huge delay. 1663 * Swapping or metadata IO can accumulate 10's of seconds worth of 1664 * delay, and we want userspace to be able to do _something_ so cap the 1665 * delays at 0.25s. If there's 10's of seconds worth of delay then the 1666 * tasks will be delayed for 0.25 second for every syscall. If 1667 * blkcg_set_delay() was used as indicated by negative use_delay, the 1668 * caller is responsible for regulating the range. 1669 */ 1670 if (clamp) 1671 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC); 1672 1673 if (use_memdelay) 1674 psi_memstall_enter(&pflags); 1675 1676 exp = ktime_add_ns(now, delay_nsec); 1677 tok = io_schedule_prepare(); 1678 do { 1679 __set_current_state(TASK_KILLABLE); 1680 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS)) 1681 break; 1682 } while (!fatal_signal_pending(current)); 1683 io_schedule_finish(tok); 1684 1685 if (use_memdelay) 1686 psi_memstall_leave(&pflags); 1687 } 1688 1689 /** 1690 * blkcg_maybe_throttle_current - throttle the current task if it has been marked 1691 * 1692 * This is only called if we've been marked with set_notify_resume(). Obviously 1693 * we can be set_notify_resume() for reasons other than blkcg throttling, so we 1694 * check to see if current->throttle_queue is set and if not this doesn't do 1695 * anything. This should only ever be called by the resume code, it's not meant 1696 * to be called by people willy-nilly as it will actually do the work to 1697 * throttle the task if it is setup for throttling. 1698 */ 1699 void blkcg_maybe_throttle_current(void) 1700 { 1701 struct request_queue *q = current->throttle_queue; 1702 struct cgroup_subsys_state *css; 1703 struct blkcg *blkcg; 1704 struct blkcg_gq *blkg; 1705 bool use_memdelay = current->use_memdelay; 1706 1707 if (!q) 1708 return; 1709 1710 current->throttle_queue = NULL; 1711 current->use_memdelay = false; 1712 1713 rcu_read_lock(); 1714 css = kthread_blkcg(); 1715 if (css) 1716 blkcg = css_to_blkcg(css); 1717 else 1718 blkcg = css_to_blkcg(task_css(current, io_cgrp_id)); 1719 1720 if (!blkcg) 1721 goto out; 1722 blkg = blkg_lookup(blkcg, q); 1723 if (!blkg) 1724 goto out; 1725 if (!blkg_tryget(blkg)) 1726 goto out; 1727 rcu_read_unlock(); 1728 1729 blkcg_maybe_throttle_blkg(blkg, use_memdelay); 1730 blkg_put(blkg); 1731 blk_put_queue(q); 1732 return; 1733 out: 1734 rcu_read_unlock(); 1735 blk_put_queue(q); 1736 } 1737 1738 /** 1739 * blkcg_schedule_throttle - this task needs to check for throttling 1740 * @q: the request queue IO was submitted on 1741 * @use_memdelay: do we charge this to memory delay for PSI 1742 * 1743 * This is called by the IO controller when we know there's delay accumulated 1744 * for the blkg for this task. We do not pass the blkg because there are places 1745 * we call this that may not have that information, the swapping code for 1746 * instance will only have a request_queue at that point. This set's the 1747 * notify_resume for the task to check and see if it requires throttling before 1748 * returning to user space. 1749 * 1750 * We will only schedule once per syscall. You can call this over and over 1751 * again and it will only do the check once upon return to user space, and only 1752 * throttle once. If the task needs to be throttled again it'll need to be 1753 * re-set at the next time we see the task. 1754 */ 1755 void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay) 1756 { 1757 if (unlikely(current->flags & PF_KTHREAD)) 1758 return; 1759 1760 if (!blk_get_queue(q)) 1761 return; 1762 1763 if (current->throttle_queue) 1764 blk_put_queue(current->throttle_queue); 1765 current->throttle_queue = q; 1766 if (use_memdelay) 1767 current->use_memdelay = use_memdelay; 1768 set_notify_resume(current); 1769 } 1770 1771 /** 1772 * blkcg_add_delay - add delay to this blkg 1773 * @blkg: blkg of interest 1774 * @now: the current time in nanoseconds 1775 * @delta: how many nanoseconds of delay to add 1776 * 1777 * Charge @delta to the blkg's current delay accumulation. This is used to 1778 * throttle tasks if an IO controller thinks we need more throttling. 1779 */ 1780 void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) 1781 { 1782 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0)) 1783 return; 1784 blkcg_scale_delay(blkg, now); 1785 atomic64_add(delta, &blkg->delay_nsec); 1786 } 1787 1788 /** 1789 * blkg_tryget_closest - try and get a blkg ref on the closet blkg 1790 * @bio: target bio 1791 * @css: target css 1792 * 1793 * As the failure mode here is to walk up the blkg tree, this ensure that the 1794 * blkg->parent pointers are always valid. This returns the blkg that it ended 1795 * up taking a reference on or %NULL if no reference was taken. 1796 */ 1797 static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, 1798 struct cgroup_subsys_state *css) 1799 { 1800 struct blkcg_gq *blkg, *ret_blkg = NULL; 1801 1802 rcu_read_lock(); 1803 blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_disk->queue); 1804 while (blkg) { 1805 if (blkg_tryget(blkg)) { 1806 ret_blkg = blkg; 1807 break; 1808 } 1809 blkg = blkg->parent; 1810 } 1811 rcu_read_unlock(); 1812 1813 return ret_blkg; 1814 } 1815 1816 /** 1817 * bio_associate_blkg_from_css - associate a bio with a specified css 1818 * @bio: target bio 1819 * @css: target css 1820 * 1821 * Associate @bio with the blkg found by combining the css's blkg and the 1822 * request_queue of the @bio. An association failure is handled by walking up 1823 * the blkg tree. Therefore, the blkg associated can be anything between @blkg 1824 * and q->root_blkg. This situation only happens when a cgroup is dying and 1825 * then the remaining bios will spill to the closest alive blkg. 1826 * 1827 * A reference will be taken on the blkg and will be released when @bio is 1828 * freed. 1829 */ 1830 void bio_associate_blkg_from_css(struct bio *bio, 1831 struct cgroup_subsys_state *css) 1832 { 1833 if (bio->bi_blkg) 1834 blkg_put(bio->bi_blkg); 1835 1836 if (css && css->parent) { 1837 bio->bi_blkg = blkg_tryget_closest(bio, css); 1838 } else { 1839 blkg_get(bio->bi_disk->queue->root_blkg); 1840 bio->bi_blkg = bio->bi_disk->queue->root_blkg; 1841 } 1842 } 1843 EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); 1844 1845 /** 1846 * bio_associate_blkg - associate a bio with a blkg 1847 * @bio: target bio 1848 * 1849 * Associate @bio with the blkg found from the bio's css and request_queue. 1850 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is 1851 * already associated, the css is reused and association redone as the 1852 * request_queue may have changed. 1853 */ 1854 void bio_associate_blkg(struct bio *bio) 1855 { 1856 struct cgroup_subsys_state *css; 1857 1858 rcu_read_lock(); 1859 1860 if (bio->bi_blkg) 1861 css = &bio_blkcg(bio)->css; 1862 else 1863 css = blkcg_css(); 1864 1865 bio_associate_blkg_from_css(bio, css); 1866 1867 rcu_read_unlock(); 1868 } 1869 EXPORT_SYMBOL_GPL(bio_associate_blkg); 1870 1871 /** 1872 * bio_clone_blkg_association - clone blkg association from src to dst bio 1873 * @dst: destination bio 1874 * @src: source bio 1875 */ 1876 void bio_clone_blkg_association(struct bio *dst, struct bio *src) 1877 { 1878 if (src->bi_blkg) { 1879 if (dst->bi_blkg) 1880 blkg_put(dst->bi_blkg); 1881 blkg_get(src->bi_blkg); 1882 dst->bi_blkg = src->bi_blkg; 1883 } 1884 } 1885 EXPORT_SYMBOL_GPL(bio_clone_blkg_association); 1886 1887 static int blk_cgroup_io_type(struct bio *bio) 1888 { 1889 if (op_is_discard(bio->bi_opf)) 1890 return BLKG_IOSTAT_DISCARD; 1891 if (op_is_write(bio->bi_opf)) 1892 return BLKG_IOSTAT_WRITE; 1893 return BLKG_IOSTAT_READ; 1894 } 1895 1896 void blk_cgroup_bio_start(struct bio *bio) 1897 { 1898 int rwd = blk_cgroup_io_type(bio), cpu; 1899 struct blkg_iostat_set *bis; 1900 1901 cpu = get_cpu(); 1902 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu); 1903 u64_stats_update_begin(&bis->sync); 1904 1905 /* 1906 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split 1907 * bio and we would have already accounted for the size of the bio. 1908 */ 1909 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) { 1910 bio_set_flag(bio, BIO_CGROUP_ACCT); 1911 bis->cur.bytes[rwd] += bio->bi_iter.bi_size; 1912 } 1913 bis->cur.ios[rwd]++; 1914 1915 u64_stats_update_end(&bis->sync); 1916 if (cgroup_subsys_on_dfl(io_cgrp_subsys)) 1917 cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu); 1918 put_cpu(); 1919 } 1920 1921 static int __init blkcg_init(void) 1922 { 1923 blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio", 1924 WQ_MEM_RECLAIM | WQ_FREEZABLE | 1925 WQ_UNBOUND | WQ_SYSFS, 0); 1926 if (!blkcg_punt_bio_wq) 1927 return -ENOMEM; 1928 return 0; 1929 } 1930 subsys_initcall(blkcg_init); 1931 1932 module_param(blkcg_debug_stats, bool, 0644); 1933 MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not"); 1934