1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common Block IO controller cgroup interface 4 * 5 * Based on ideas and code from CFQ, CFS and BFQ: 6 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk> 7 * 8 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it> 9 * Paolo Valente <paolo.valente@unimore.it> 10 * 11 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com> 12 * Nauman Rafique <nauman@google.com> 13 * 14 * For policy-specific per-blkcg data: 15 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it> 16 * Arianna Avanzini <avanzini.arianna@gmail.com> 17 */ 18 #include <linux/ioprio.h> 19 #include <linux/kdev_t.h> 20 #include <linux/module.h> 21 #include <linux/sched/signal.h> 22 #include <linux/err.h> 23 #include <linux/blkdev.h> 24 #include <linux/backing-dev.h> 25 #include <linux/slab.h> 26 #include <linux/delay.h> 27 #include <linux/atomic.h> 28 #include <linux/ctype.h> 29 #include <linux/resume_user_mode.h> 30 #include <linux/psi.h> 31 #include <linux/part_stat.h> 32 #include "blk.h" 33 #include "blk-cgroup.h" 34 #include "blk-ioprio.h" 35 #include "blk-throttle.h" 36 37 /* 38 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation. 39 * blkcg_pol_register_mutex nests outside of it and synchronizes entire 40 * policy [un]register operations including cgroup file additions / 41 * removals. Putting cgroup file registration outside blkcg_pol_mutex 42 * allows grabbing it from cgroup callbacks. 43 */ 44 static DEFINE_MUTEX(blkcg_pol_register_mutex); 45 static DEFINE_MUTEX(blkcg_pol_mutex); 46 47 struct blkcg blkcg_root; 48 EXPORT_SYMBOL_GPL(blkcg_root); 49 50 struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css; 51 EXPORT_SYMBOL_GPL(blkcg_root_css); 52 53 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS]; 54 55 static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */ 56 57 bool blkcg_debug_stats = false; 58 static struct workqueue_struct *blkcg_punt_bio_wq; 59 60 #define BLKG_DESTROY_BATCH_SIZE 64 61 62 /* 63 * Lockless lists for tracking IO stats update 64 * 65 * New IO stats are stored in the percpu iostat_cpu within blkcg_gq (blkg). 66 * There are multiple blkg's (one for each block device) attached to each 67 * blkcg. The rstat code keeps track of which cpu has IO stats updated, 68 * but it doesn't know which blkg has the updated stats. If there are many 69 * block devices in a system, the cost of iterating all the blkg's to flush 70 * out the IO stats can be high. To reduce such overhead, a set of percpu 71 * lockless lists (lhead) per blkcg are used to track the set of recently 72 * updated iostat_cpu's since the last flush. An iostat_cpu will be put 73 * onto the lockless list on the update side [blk_cgroup_bio_start()] if 74 * not there yet and then removed when being flushed [blkcg_rstat_flush()]. 75 * References to blkg are gotten and then put back in the process to 76 * protect against blkg removal. 77 * 78 * Return: 0 if successful or -ENOMEM if allocation fails. 79 */ 80 static int init_blkcg_llists(struct blkcg *blkcg) 81 { 82 int cpu; 83 84 blkcg->lhead = alloc_percpu_gfp(struct llist_head, GFP_KERNEL); 85 if (!blkcg->lhead) 86 return -ENOMEM; 87 88 for_each_possible_cpu(cpu) 89 init_llist_head(per_cpu_ptr(blkcg->lhead, cpu)); 90 return 0; 91 } 92 93 /** 94 * blkcg_css - find the current css 95 * 96 * Find the css associated with either the kthread or the current task. 97 * This may return a dying css, so it is up to the caller to use tryget logic 98 * to confirm it is alive and well. 99 */ 100 static struct cgroup_subsys_state *blkcg_css(void) 101 { 102 struct cgroup_subsys_state *css; 103 104 css = kthread_blkcg(); 105 if (css) 106 return css; 107 return task_css(current, io_cgrp_id); 108 } 109 110 static bool blkcg_policy_enabled(struct request_queue *q, 111 const struct blkcg_policy *pol) 112 { 113 return pol && test_bit(pol->plid, q->blkcg_pols); 114 } 115 116 static void blkg_free_workfn(struct work_struct *work) 117 { 118 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, 119 free_work); 120 int i; 121 122 for (i = 0; i < BLKCG_MAX_POLS; i++) 123 if (blkg->pd[i]) 124 blkcg_policy[i]->pd_free_fn(blkg->pd[i]); 125 126 if (blkg->q) 127 blk_put_queue(blkg->q); 128 free_percpu(blkg->iostat_cpu); 129 percpu_ref_exit(&blkg->refcnt); 130 kfree(blkg); 131 } 132 133 /** 134 * blkg_free - free a blkg 135 * @blkg: blkg to free 136 * 137 * Free @blkg which may be partially allocated. 138 */ 139 static void blkg_free(struct blkcg_gq *blkg) 140 { 141 if (!blkg) 142 return; 143 144 /* 145 * Both ->pd_free_fn() and request queue's release handler may 146 * sleep, so free us by scheduling one work func 147 */ 148 INIT_WORK(&blkg->free_work, blkg_free_workfn); 149 schedule_work(&blkg->free_work); 150 } 151 152 static void __blkg_release(struct rcu_head *rcu) 153 { 154 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); 155 156 WARN_ON(!bio_list_empty(&blkg->async_bios)); 157 158 /* release the blkcg and parent blkg refs this blkg has been holding */ 159 css_put(&blkg->blkcg->css); 160 if (blkg->parent) 161 blkg_put(blkg->parent); 162 blkg_free(blkg); 163 } 164 165 /* 166 * A group is RCU protected, but having an rcu lock does not mean that one 167 * can access all the fields of blkg and assume these are valid. For 168 * example, don't try to follow throtl_data and request queue links. 169 * 170 * Having a reference to blkg under an rcu allows accesses to only values 171 * local to groups like group stats and group rate limits. 172 */ 173 static void blkg_release(struct percpu_ref *ref) 174 { 175 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt); 176 177 call_rcu(&blkg->rcu_head, __blkg_release); 178 } 179 180 static void blkg_async_bio_workfn(struct work_struct *work) 181 { 182 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, 183 async_bio_work); 184 struct bio_list bios = BIO_EMPTY_LIST; 185 struct bio *bio; 186 struct blk_plug plug; 187 bool need_plug = false; 188 189 /* as long as there are pending bios, @blkg can't go away */ 190 spin_lock_bh(&blkg->async_bio_lock); 191 bio_list_merge(&bios, &blkg->async_bios); 192 bio_list_init(&blkg->async_bios); 193 spin_unlock_bh(&blkg->async_bio_lock); 194 195 /* start plug only when bio_list contains at least 2 bios */ 196 if (bios.head && bios.head->bi_next) { 197 need_plug = true; 198 blk_start_plug(&plug); 199 } 200 while ((bio = bio_list_pop(&bios))) 201 submit_bio(bio); 202 if (need_plug) 203 blk_finish_plug(&plug); 204 } 205 206 /** 207 * bio_blkcg_css - return the blkcg CSS associated with a bio 208 * @bio: target bio 209 * 210 * This returns the CSS for the blkcg associated with a bio, or %NULL if not 211 * associated. Callers are expected to either handle %NULL or know association 212 * has been done prior to calling this. 213 */ 214 struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio) 215 { 216 if (!bio || !bio->bi_blkg) 217 return NULL; 218 return &bio->bi_blkg->blkcg->css; 219 } 220 EXPORT_SYMBOL_GPL(bio_blkcg_css); 221 222 /** 223 * blkcg_parent - get the parent of a blkcg 224 * @blkcg: blkcg of interest 225 * 226 * Return the parent blkcg of @blkcg. Can be called anytime. 227 */ 228 static inline struct blkcg *blkcg_parent(struct blkcg *blkcg) 229 { 230 return css_to_blkcg(blkcg->css.parent); 231 } 232 233 /** 234 * blkg_alloc - allocate a blkg 235 * @blkcg: block cgroup the new blkg is associated with 236 * @disk: gendisk the new blkg is associated with 237 * @gfp_mask: allocation mask to use 238 * 239 * Allocate a new blkg assocating @blkcg and @q. 240 */ 241 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, 242 gfp_t gfp_mask) 243 { 244 struct blkcg_gq *blkg; 245 int i, cpu; 246 247 /* alloc and init base part */ 248 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, disk->queue->node); 249 if (!blkg) 250 return NULL; 251 252 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask)) 253 goto err_free; 254 255 blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask); 256 if (!blkg->iostat_cpu) 257 goto err_free; 258 259 if (!blk_get_queue(disk->queue)) 260 goto err_free; 261 262 blkg->q = disk->queue; 263 INIT_LIST_HEAD(&blkg->q_node); 264 spin_lock_init(&blkg->async_bio_lock); 265 bio_list_init(&blkg->async_bios); 266 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn); 267 blkg->blkcg = blkcg; 268 269 u64_stats_init(&blkg->iostat.sync); 270 for_each_possible_cpu(cpu) { 271 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync); 272 per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg; 273 } 274 275 for (i = 0; i < BLKCG_MAX_POLS; i++) { 276 struct blkcg_policy *pol = blkcg_policy[i]; 277 struct blkg_policy_data *pd; 278 279 if (!blkcg_policy_enabled(disk->queue, pol)) 280 continue; 281 282 /* alloc per-policy data and attach it to blkg */ 283 pd = pol->pd_alloc_fn(gfp_mask, disk->queue, blkcg); 284 if (!pd) 285 goto err_free; 286 287 blkg->pd[i] = pd; 288 pd->blkg = blkg; 289 pd->plid = i; 290 } 291 292 return blkg; 293 294 err_free: 295 blkg_free(blkg); 296 return NULL; 297 } 298 299 /* 300 * If @new_blkg is %NULL, this function tries to allocate a new one as 301 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return. 302 */ 303 static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, 304 struct blkcg_gq *new_blkg) 305 { 306 struct blkcg_gq *blkg; 307 int i, ret; 308 309 lockdep_assert_held(&disk->queue->queue_lock); 310 311 /* request_queue is dying, do not create/recreate a blkg */ 312 if (blk_queue_dying(disk->queue)) { 313 ret = -ENODEV; 314 goto err_free_blkg; 315 } 316 317 /* blkg holds a reference to blkcg */ 318 if (!css_tryget_online(&blkcg->css)) { 319 ret = -ENODEV; 320 goto err_free_blkg; 321 } 322 323 /* allocate */ 324 if (!new_blkg) { 325 new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT | __GFP_NOWARN); 326 if (unlikely(!new_blkg)) { 327 ret = -ENOMEM; 328 goto err_put_css; 329 } 330 } 331 blkg = new_blkg; 332 333 /* link parent */ 334 if (blkcg_parent(blkcg)) { 335 blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue); 336 if (WARN_ON_ONCE(!blkg->parent)) { 337 ret = -ENODEV; 338 goto err_put_css; 339 } 340 blkg_get(blkg->parent); 341 } 342 343 /* invoke per-policy init */ 344 for (i = 0; i < BLKCG_MAX_POLS; i++) { 345 struct blkcg_policy *pol = blkcg_policy[i]; 346 347 if (blkg->pd[i] && pol->pd_init_fn) 348 pol->pd_init_fn(blkg->pd[i]); 349 } 350 351 /* insert */ 352 spin_lock(&blkcg->lock); 353 ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg); 354 if (likely(!ret)) { 355 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); 356 list_add(&blkg->q_node, &disk->queue->blkg_list); 357 358 for (i = 0; i < BLKCG_MAX_POLS; i++) { 359 struct blkcg_policy *pol = blkcg_policy[i]; 360 361 if (blkg->pd[i] && pol->pd_online_fn) 362 pol->pd_online_fn(blkg->pd[i]); 363 } 364 } 365 blkg->online = true; 366 spin_unlock(&blkcg->lock); 367 368 if (!ret) 369 return blkg; 370 371 /* @blkg failed fully initialized, use the usual release path */ 372 blkg_put(blkg); 373 return ERR_PTR(ret); 374 375 err_put_css: 376 css_put(&blkcg->css); 377 err_free_blkg: 378 blkg_free(new_blkg); 379 return ERR_PTR(ret); 380 } 381 382 /** 383 * blkg_lookup_create - lookup blkg, try to create one if not there 384 * @blkcg: blkcg of interest 385 * @disk: gendisk of interest 386 * 387 * Lookup blkg for the @blkcg - @disk pair. If it doesn't exist, try to 388 * create one. blkg creation is performed recursively from blkcg_root such 389 * that all non-root blkg's have access to the parent blkg. This function 390 * should be called under RCU read lock and takes @disk->queue->queue_lock. 391 * 392 * Returns the blkg or the closest blkg if blkg_create() fails as it walks 393 * down from root. 394 */ 395 static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, 396 struct gendisk *disk) 397 { 398 struct request_queue *q = disk->queue; 399 struct blkcg_gq *blkg; 400 unsigned long flags; 401 402 WARN_ON_ONCE(!rcu_read_lock_held()); 403 404 blkg = blkg_lookup(blkcg, q); 405 if (blkg) 406 return blkg; 407 408 spin_lock_irqsave(&q->queue_lock, flags); 409 blkg = blkg_lookup(blkcg, q); 410 if (blkg) { 411 if (blkcg != &blkcg_root && 412 blkg != rcu_dereference(blkcg->blkg_hint)) 413 rcu_assign_pointer(blkcg->blkg_hint, blkg); 414 goto found; 415 } 416 417 /* 418 * Create blkgs walking down from blkcg_root to @blkcg, so that all 419 * non-root blkgs have access to their parents. Returns the closest 420 * blkg to the intended blkg should blkg_create() fail. 421 */ 422 while (true) { 423 struct blkcg *pos = blkcg; 424 struct blkcg *parent = blkcg_parent(blkcg); 425 struct blkcg_gq *ret_blkg = q->root_blkg; 426 427 while (parent) { 428 blkg = blkg_lookup(parent, q); 429 if (blkg) { 430 /* remember closest blkg */ 431 ret_blkg = blkg; 432 break; 433 } 434 pos = parent; 435 parent = blkcg_parent(parent); 436 } 437 438 blkg = blkg_create(pos, disk, NULL); 439 if (IS_ERR(blkg)) { 440 blkg = ret_blkg; 441 break; 442 } 443 if (pos == blkcg) 444 break; 445 } 446 447 found: 448 spin_unlock_irqrestore(&q->queue_lock, flags); 449 return blkg; 450 } 451 452 static void blkg_destroy(struct blkcg_gq *blkg) 453 { 454 struct blkcg *blkcg = blkg->blkcg; 455 int i; 456 457 lockdep_assert_held(&blkg->q->queue_lock); 458 lockdep_assert_held(&blkcg->lock); 459 460 /* Something wrong if we are trying to remove same group twice */ 461 WARN_ON_ONCE(list_empty(&blkg->q_node)); 462 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); 463 464 for (i = 0; i < BLKCG_MAX_POLS; i++) { 465 struct blkcg_policy *pol = blkcg_policy[i]; 466 467 if (blkg->pd[i] && pol->pd_offline_fn) 468 pol->pd_offline_fn(blkg->pd[i]); 469 } 470 471 blkg->online = false; 472 473 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); 474 list_del_init(&blkg->q_node); 475 hlist_del_init_rcu(&blkg->blkcg_node); 476 477 /* 478 * Both setting lookup hint to and clearing it from @blkg are done 479 * under queue_lock. If it's not pointing to @blkg now, it never 480 * will. Hint assignment itself can race safely. 481 */ 482 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) 483 rcu_assign_pointer(blkcg->blkg_hint, NULL); 484 485 /* 486 * Put the reference taken at the time of creation so that when all 487 * queues are gone, group can be destroyed. 488 */ 489 percpu_ref_kill(&blkg->refcnt); 490 } 491 492 static void blkg_destroy_all(struct gendisk *disk) 493 { 494 struct request_queue *q = disk->queue; 495 struct blkcg_gq *blkg, *n; 496 int count = BLKG_DESTROY_BATCH_SIZE; 497 498 restart: 499 spin_lock_irq(&q->queue_lock); 500 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { 501 struct blkcg *blkcg = blkg->blkcg; 502 503 spin_lock(&blkcg->lock); 504 blkg_destroy(blkg); 505 spin_unlock(&blkcg->lock); 506 507 /* 508 * in order to avoid holding the spin lock for too long, release 509 * it when a batch of blkgs are destroyed. 510 */ 511 if (!(--count)) { 512 count = BLKG_DESTROY_BATCH_SIZE; 513 spin_unlock_irq(&q->queue_lock); 514 cond_resched(); 515 goto restart; 516 } 517 } 518 519 q->root_blkg = NULL; 520 spin_unlock_irq(&q->queue_lock); 521 } 522 523 static int blkcg_reset_stats(struct cgroup_subsys_state *css, 524 struct cftype *cftype, u64 val) 525 { 526 struct blkcg *blkcg = css_to_blkcg(css); 527 struct blkcg_gq *blkg; 528 int i, cpu; 529 530 mutex_lock(&blkcg_pol_mutex); 531 spin_lock_irq(&blkcg->lock); 532 533 /* 534 * Note that stat reset is racy - it doesn't synchronize against 535 * stat updates. This is a debug feature which shouldn't exist 536 * anyway. If you get hit by a race, retry. 537 */ 538 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { 539 for_each_possible_cpu(cpu) { 540 struct blkg_iostat_set *bis = 541 per_cpu_ptr(blkg->iostat_cpu, cpu); 542 memset(bis, 0, sizeof(*bis)); 543 } 544 memset(&blkg->iostat, 0, sizeof(blkg->iostat)); 545 546 for (i = 0; i < BLKCG_MAX_POLS; i++) { 547 struct blkcg_policy *pol = blkcg_policy[i]; 548 549 if (blkg->pd[i] && pol->pd_reset_stats_fn) 550 pol->pd_reset_stats_fn(blkg->pd[i]); 551 } 552 } 553 554 spin_unlock_irq(&blkcg->lock); 555 mutex_unlock(&blkcg_pol_mutex); 556 return 0; 557 } 558 559 const char *blkg_dev_name(struct blkcg_gq *blkg) 560 { 561 if (!blkg->q->disk || !blkg->q->disk->bdi->dev) 562 return NULL; 563 return bdi_dev_name(blkg->q->disk->bdi); 564 } 565 566 /** 567 * blkcg_print_blkgs - helper for printing per-blkg data 568 * @sf: seq_file to print to 569 * @blkcg: blkcg of interest 570 * @prfill: fill function to print out a blkg 571 * @pol: policy in question 572 * @data: data to be passed to @prfill 573 * @show_total: to print out sum of prfill return values or not 574 * 575 * This function invokes @prfill on each blkg of @blkcg if pd for the 576 * policy specified by @pol exists. @prfill is invoked with @sf, the 577 * policy data and @data and the matching queue lock held. If @show_total 578 * is %true, the sum of the return values from @prfill is printed with 579 * "Total" label at the end. 580 * 581 * This is to be used to construct print functions for 582 * cftype->read_seq_string method. 583 */ 584 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, 585 u64 (*prfill)(struct seq_file *, 586 struct blkg_policy_data *, int), 587 const struct blkcg_policy *pol, int data, 588 bool show_total) 589 { 590 struct blkcg_gq *blkg; 591 u64 total = 0; 592 593 rcu_read_lock(); 594 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { 595 spin_lock_irq(&blkg->q->queue_lock); 596 if (blkcg_policy_enabled(blkg->q, pol)) 597 total += prfill(sf, blkg->pd[pol->plid], data); 598 spin_unlock_irq(&blkg->q->queue_lock); 599 } 600 rcu_read_unlock(); 601 602 if (show_total) 603 seq_printf(sf, "Total %llu\n", (unsigned long long)total); 604 } 605 EXPORT_SYMBOL_GPL(blkcg_print_blkgs); 606 607 /** 608 * __blkg_prfill_u64 - prfill helper for a single u64 value 609 * @sf: seq_file to print to 610 * @pd: policy private data of interest 611 * @v: value to print 612 * 613 * Print @v to @sf for the device associated with @pd. 614 */ 615 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v) 616 { 617 const char *dname = blkg_dev_name(pd->blkg); 618 619 if (!dname) 620 return 0; 621 622 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v); 623 return v; 624 } 625 EXPORT_SYMBOL_GPL(__blkg_prfill_u64); 626 627 /** 628 * blkcg_conf_open_bdev - parse and open bdev for per-blkg config update 629 * @inputp: input string pointer 630 * 631 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update 632 * from @input and get and return the matching bdev. *@inputp is 633 * updated to point past the device node prefix. Returns an ERR_PTR() 634 * value on error. 635 * 636 * Use this function iff blkg_conf_prep() can't be used for some reason. 637 */ 638 struct block_device *blkcg_conf_open_bdev(char **inputp) 639 { 640 char *input = *inputp; 641 unsigned int major, minor; 642 struct block_device *bdev; 643 int key_len; 644 645 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2) 646 return ERR_PTR(-EINVAL); 647 648 input += key_len; 649 if (!isspace(*input)) 650 return ERR_PTR(-EINVAL); 651 input = skip_spaces(input); 652 653 bdev = blkdev_get_no_open(MKDEV(major, minor)); 654 if (!bdev) 655 return ERR_PTR(-ENODEV); 656 if (bdev_is_partition(bdev)) { 657 blkdev_put_no_open(bdev); 658 return ERR_PTR(-ENODEV); 659 } 660 661 *inputp = input; 662 return bdev; 663 } 664 665 /** 666 * blkg_conf_prep - parse and prepare for per-blkg config update 667 * @blkcg: target block cgroup 668 * @pol: target policy 669 * @input: input string 670 * @ctx: blkg_conf_ctx to be filled 671 * 672 * Parse per-blkg config update from @input and initialize @ctx with the 673 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the 674 * part of @input following MAJ:MIN. This function returns with RCU read 675 * lock and queue lock held and must be paired with blkg_conf_finish(). 676 */ 677 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, 678 char *input, struct blkg_conf_ctx *ctx) 679 __acquires(rcu) __acquires(&bdev->bd_queue->queue_lock) 680 { 681 struct block_device *bdev; 682 struct gendisk *disk; 683 struct request_queue *q; 684 struct blkcg_gq *blkg; 685 int ret; 686 687 bdev = blkcg_conf_open_bdev(&input); 688 if (IS_ERR(bdev)) 689 return PTR_ERR(bdev); 690 disk = bdev->bd_disk; 691 q = disk->queue; 692 693 /* 694 * blkcg_deactivate_policy() requires queue to be frozen, we can grab 695 * q_usage_counter to prevent concurrent with blkcg_deactivate_policy(). 696 */ 697 ret = blk_queue_enter(q, 0); 698 if (ret) 699 goto fail; 700 701 rcu_read_lock(); 702 spin_lock_irq(&q->queue_lock); 703 704 if (!blkcg_policy_enabled(q, pol)) { 705 ret = -EOPNOTSUPP; 706 goto fail_unlock; 707 } 708 709 blkg = blkg_lookup(blkcg, q); 710 if (blkg) 711 goto success; 712 713 /* 714 * Create blkgs walking down from blkcg_root to @blkcg, so that all 715 * non-root blkgs have access to their parents. 716 */ 717 while (true) { 718 struct blkcg *pos = blkcg; 719 struct blkcg *parent; 720 struct blkcg_gq *new_blkg; 721 722 parent = blkcg_parent(blkcg); 723 while (parent && !blkg_lookup(parent, q)) { 724 pos = parent; 725 parent = blkcg_parent(parent); 726 } 727 728 /* Drop locks to do new blkg allocation with GFP_KERNEL. */ 729 spin_unlock_irq(&q->queue_lock); 730 rcu_read_unlock(); 731 732 new_blkg = blkg_alloc(pos, disk, GFP_KERNEL); 733 if (unlikely(!new_blkg)) { 734 ret = -ENOMEM; 735 goto fail_exit_queue; 736 } 737 738 if (radix_tree_preload(GFP_KERNEL)) { 739 blkg_free(new_blkg); 740 ret = -ENOMEM; 741 goto fail_exit_queue; 742 } 743 744 rcu_read_lock(); 745 spin_lock_irq(&q->queue_lock); 746 747 if (!blkcg_policy_enabled(q, pol)) { 748 blkg_free(new_blkg); 749 ret = -EOPNOTSUPP; 750 goto fail_preloaded; 751 } 752 753 blkg = blkg_lookup(pos, q); 754 if (blkg) { 755 blkg_free(new_blkg); 756 } else { 757 blkg = blkg_create(pos, disk, new_blkg); 758 if (IS_ERR(blkg)) { 759 ret = PTR_ERR(blkg); 760 goto fail_preloaded; 761 } 762 } 763 764 radix_tree_preload_end(); 765 766 if (pos == blkcg) 767 goto success; 768 } 769 success: 770 blk_queue_exit(q); 771 ctx->bdev = bdev; 772 ctx->blkg = blkg; 773 ctx->body = input; 774 return 0; 775 776 fail_preloaded: 777 radix_tree_preload_end(); 778 fail_unlock: 779 spin_unlock_irq(&q->queue_lock); 780 rcu_read_unlock(); 781 fail_exit_queue: 782 blk_queue_exit(q); 783 fail: 784 blkdev_put_no_open(bdev); 785 /* 786 * If queue was bypassing, we should retry. Do so after a 787 * short msleep(). It isn't strictly necessary but queue 788 * can be bypassing for some time and it's always nice to 789 * avoid busy looping. 790 */ 791 if (ret == -EBUSY) { 792 msleep(10); 793 ret = restart_syscall(); 794 } 795 return ret; 796 } 797 EXPORT_SYMBOL_GPL(blkg_conf_prep); 798 799 /** 800 * blkg_conf_finish - finish up per-blkg config update 801 * @ctx: blkg_conf_ctx initialized by blkg_conf_prep() 802 * 803 * Finish up after per-blkg config update. This function must be paired 804 * with blkg_conf_prep(). 805 */ 806 void blkg_conf_finish(struct blkg_conf_ctx *ctx) 807 __releases(&ctx->bdev->bd_queue->queue_lock) __releases(rcu) 808 { 809 spin_unlock_irq(&bdev_get_queue(ctx->bdev)->queue_lock); 810 rcu_read_unlock(); 811 blkdev_put_no_open(ctx->bdev); 812 } 813 EXPORT_SYMBOL_GPL(blkg_conf_finish); 814 815 static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src) 816 { 817 int i; 818 819 for (i = 0; i < BLKG_IOSTAT_NR; i++) { 820 dst->bytes[i] = src->bytes[i]; 821 dst->ios[i] = src->ios[i]; 822 } 823 } 824 825 static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src) 826 { 827 int i; 828 829 for (i = 0; i < BLKG_IOSTAT_NR; i++) { 830 dst->bytes[i] += src->bytes[i]; 831 dst->ios[i] += src->ios[i]; 832 } 833 } 834 835 static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src) 836 { 837 int i; 838 839 for (i = 0; i < BLKG_IOSTAT_NR; i++) { 840 dst->bytes[i] -= src->bytes[i]; 841 dst->ios[i] -= src->ios[i]; 842 } 843 } 844 845 static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, 846 struct blkg_iostat *last) 847 { 848 struct blkg_iostat delta; 849 unsigned long flags; 850 851 /* propagate percpu delta to global */ 852 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync); 853 blkg_iostat_set(&delta, cur); 854 blkg_iostat_sub(&delta, last); 855 blkg_iostat_add(&blkg->iostat.cur, &delta); 856 blkg_iostat_add(last, &delta); 857 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags); 858 } 859 860 static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) 861 { 862 struct blkcg *blkcg = css_to_blkcg(css); 863 struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu); 864 struct llist_node *lnode; 865 struct blkg_iostat_set *bisc, *next_bisc; 866 867 /* Root-level stats are sourced from system-wide IO stats */ 868 if (!cgroup_parent(css->cgroup)) 869 return; 870 871 rcu_read_lock(); 872 873 lnode = llist_del_all(lhead); 874 if (!lnode) 875 goto out; 876 877 /* 878 * Iterate only the iostat_cpu's queued in the lockless list. 879 */ 880 llist_for_each_entry_safe(bisc, next_bisc, lnode, lnode) { 881 struct blkcg_gq *blkg = bisc->blkg; 882 struct blkcg_gq *parent = blkg->parent; 883 struct blkg_iostat cur; 884 unsigned int seq; 885 886 WRITE_ONCE(bisc->lqueued, false); 887 888 /* fetch the current per-cpu values */ 889 do { 890 seq = u64_stats_fetch_begin(&bisc->sync); 891 blkg_iostat_set(&cur, &bisc->cur); 892 } while (u64_stats_fetch_retry(&bisc->sync, seq)); 893 894 blkcg_iostat_update(blkg, &cur, &bisc->last); 895 896 /* propagate global delta to parent (unless that's root) */ 897 if (parent && parent->parent) 898 blkcg_iostat_update(parent, &blkg->iostat.cur, 899 &blkg->iostat.last); 900 percpu_ref_put(&blkg->refcnt); 901 } 902 903 out: 904 rcu_read_unlock(); 905 } 906 907 /* 908 * We source root cgroup stats from the system-wide stats to avoid 909 * tracking the same information twice and incurring overhead when no 910 * cgroups are defined. For that reason, cgroup_rstat_flush in 911 * blkcg_print_stat does not actually fill out the iostat in the root 912 * cgroup's blkcg_gq. 913 * 914 * However, we would like to re-use the printing code between the root and 915 * non-root cgroups to the extent possible. For that reason, we simulate 916 * flushing the root cgroup's stats by explicitly filling in the iostat 917 * with disk level statistics. 918 */ 919 static void blkcg_fill_root_iostats(void) 920 { 921 struct class_dev_iter iter; 922 struct device *dev; 923 924 class_dev_iter_init(&iter, &block_class, NULL, &disk_type); 925 while ((dev = class_dev_iter_next(&iter))) { 926 struct block_device *bdev = dev_to_bdev(dev); 927 struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg; 928 struct blkg_iostat tmp; 929 int cpu; 930 unsigned long flags; 931 932 memset(&tmp, 0, sizeof(tmp)); 933 for_each_possible_cpu(cpu) { 934 struct disk_stats *cpu_dkstats; 935 936 cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu); 937 tmp.ios[BLKG_IOSTAT_READ] += 938 cpu_dkstats->ios[STAT_READ]; 939 tmp.ios[BLKG_IOSTAT_WRITE] += 940 cpu_dkstats->ios[STAT_WRITE]; 941 tmp.ios[BLKG_IOSTAT_DISCARD] += 942 cpu_dkstats->ios[STAT_DISCARD]; 943 // convert sectors to bytes 944 tmp.bytes[BLKG_IOSTAT_READ] += 945 cpu_dkstats->sectors[STAT_READ] << 9; 946 tmp.bytes[BLKG_IOSTAT_WRITE] += 947 cpu_dkstats->sectors[STAT_WRITE] << 9; 948 tmp.bytes[BLKG_IOSTAT_DISCARD] += 949 cpu_dkstats->sectors[STAT_DISCARD] << 9; 950 } 951 952 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync); 953 blkg_iostat_set(&blkg->iostat.cur, &tmp); 954 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags); 955 } 956 } 957 958 static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) 959 { 960 struct blkg_iostat_set *bis = &blkg->iostat; 961 u64 rbytes, wbytes, rios, wios, dbytes, dios; 962 const char *dname; 963 unsigned seq; 964 int i; 965 966 if (!blkg->online) 967 return; 968 969 dname = blkg_dev_name(blkg); 970 if (!dname) 971 return; 972 973 seq_printf(s, "%s ", dname); 974 975 do { 976 seq = u64_stats_fetch_begin(&bis->sync); 977 978 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ]; 979 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE]; 980 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD]; 981 rios = bis->cur.ios[BLKG_IOSTAT_READ]; 982 wios = bis->cur.ios[BLKG_IOSTAT_WRITE]; 983 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD]; 984 } while (u64_stats_fetch_retry(&bis->sync, seq)); 985 986 if (rbytes || wbytes || rios || wios) { 987 seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu", 988 rbytes, wbytes, rios, wios, 989 dbytes, dios); 990 } 991 992 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) { 993 seq_printf(s, " use_delay=%d delay_nsec=%llu", 994 atomic_read(&blkg->use_delay), 995 atomic64_read(&blkg->delay_nsec)); 996 } 997 998 for (i = 0; i < BLKCG_MAX_POLS; i++) { 999 struct blkcg_policy *pol = blkcg_policy[i]; 1000 1001 if (!blkg->pd[i] || !pol->pd_stat_fn) 1002 continue; 1003 1004 pol->pd_stat_fn(blkg->pd[i], s); 1005 } 1006 1007 seq_puts(s, "\n"); 1008 } 1009 1010 static int blkcg_print_stat(struct seq_file *sf, void *v) 1011 { 1012 struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); 1013 struct blkcg_gq *blkg; 1014 1015 if (!seq_css(sf)->parent) 1016 blkcg_fill_root_iostats(); 1017 else 1018 cgroup_rstat_flush(blkcg->css.cgroup); 1019 1020 rcu_read_lock(); 1021 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { 1022 spin_lock_irq(&blkg->q->queue_lock); 1023 blkcg_print_one_stat(blkg, sf); 1024 spin_unlock_irq(&blkg->q->queue_lock); 1025 } 1026 rcu_read_unlock(); 1027 return 0; 1028 } 1029 1030 static struct cftype blkcg_files[] = { 1031 { 1032 .name = "stat", 1033 .seq_show = blkcg_print_stat, 1034 }, 1035 { } /* terminate */ 1036 }; 1037 1038 static struct cftype blkcg_legacy_files[] = { 1039 { 1040 .name = "reset_stats", 1041 .write_u64 = blkcg_reset_stats, 1042 }, 1043 { } /* terminate */ 1044 }; 1045 1046 #ifdef CONFIG_CGROUP_WRITEBACK 1047 struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css) 1048 { 1049 return &css_to_blkcg(css)->cgwb_list; 1050 } 1051 #endif 1052 1053 /* 1054 * blkcg destruction is a three-stage process. 1055 * 1056 * 1. Destruction starts. The blkcg_css_offline() callback is invoked 1057 * which offlines writeback. Here we tie the next stage of blkg destruction 1058 * to the completion of writeback associated with the blkcg. This lets us 1059 * avoid punting potentially large amounts of outstanding writeback to root 1060 * while maintaining any ongoing policies. The next stage is triggered when 1061 * the nr_cgwbs count goes to zero. 1062 * 1063 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called 1064 * and handles the destruction of blkgs. Here the css reference held by 1065 * the blkg is put back eventually allowing blkcg_css_free() to be called. 1066 * This work may occur in cgwb_release_workfn() on the cgwb_release 1067 * workqueue. Any submitted ios that fail to get the blkg ref will be 1068 * punted to the root_blkg. 1069 * 1070 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called. 1071 * This finally frees the blkcg. 1072 */ 1073 1074 /** 1075 * blkcg_destroy_blkgs - responsible for shooting down blkgs 1076 * @blkcg: blkcg of interest 1077 * 1078 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock 1079 * is nested inside q lock, this function performs reverse double lock dancing. 1080 * Destroying the blkgs releases the reference held on the blkcg's css allowing 1081 * blkcg_css_free to eventually be called. 1082 * 1083 * This is the blkcg counterpart of ioc_release_fn(). 1084 */ 1085 static void blkcg_destroy_blkgs(struct blkcg *blkcg) 1086 { 1087 might_sleep(); 1088 1089 spin_lock_irq(&blkcg->lock); 1090 1091 while (!hlist_empty(&blkcg->blkg_list)) { 1092 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, 1093 struct blkcg_gq, blkcg_node); 1094 struct request_queue *q = blkg->q; 1095 1096 if (need_resched() || !spin_trylock(&q->queue_lock)) { 1097 /* 1098 * Given that the system can accumulate a huge number 1099 * of blkgs in pathological cases, check to see if we 1100 * need to rescheduling to avoid softlockup. 1101 */ 1102 spin_unlock_irq(&blkcg->lock); 1103 cond_resched(); 1104 spin_lock_irq(&blkcg->lock); 1105 continue; 1106 } 1107 1108 blkg_destroy(blkg); 1109 spin_unlock(&q->queue_lock); 1110 } 1111 1112 spin_unlock_irq(&blkcg->lock); 1113 } 1114 1115 /** 1116 * blkcg_pin_online - pin online state 1117 * @blkcg_css: blkcg of interest 1118 * 1119 * While pinned, a blkcg is kept online. This is primarily used to 1120 * impedance-match blkg and cgwb lifetimes so that blkg doesn't go offline 1121 * while an associated cgwb is still active. 1122 */ 1123 void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css) 1124 { 1125 refcount_inc(&css_to_blkcg(blkcg_css)->online_pin); 1126 } 1127 1128 /** 1129 * blkcg_unpin_online - unpin online state 1130 * @blkcg_css: blkcg of interest 1131 * 1132 * This is primarily used to impedance-match blkg and cgwb lifetimes so 1133 * that blkg doesn't go offline while an associated cgwb is still active. 1134 * When this count goes to zero, all active cgwbs have finished so the 1135 * blkcg can continue destruction by calling blkcg_destroy_blkgs(). 1136 */ 1137 void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css) 1138 { 1139 struct blkcg *blkcg = css_to_blkcg(blkcg_css); 1140 1141 do { 1142 if (!refcount_dec_and_test(&blkcg->online_pin)) 1143 break; 1144 blkcg_destroy_blkgs(blkcg); 1145 blkcg = blkcg_parent(blkcg); 1146 } while (blkcg); 1147 } 1148 1149 /** 1150 * blkcg_css_offline - cgroup css_offline callback 1151 * @css: css of interest 1152 * 1153 * This function is called when @css is about to go away. Here the cgwbs are 1154 * offlined first and only once writeback associated with the blkcg has 1155 * finished do we start step 2 (see above). 1156 */ 1157 static void blkcg_css_offline(struct cgroup_subsys_state *css) 1158 { 1159 /* this prevents anyone from attaching or migrating to this blkcg */ 1160 wb_blkcg_offline(css); 1161 1162 /* put the base online pin allowing step 2 to be triggered */ 1163 blkcg_unpin_online(css); 1164 } 1165 1166 static void blkcg_css_free(struct cgroup_subsys_state *css) 1167 { 1168 struct blkcg *blkcg = css_to_blkcg(css); 1169 int i; 1170 1171 mutex_lock(&blkcg_pol_mutex); 1172 1173 list_del(&blkcg->all_blkcgs_node); 1174 1175 for (i = 0; i < BLKCG_MAX_POLS; i++) 1176 if (blkcg->cpd[i]) 1177 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); 1178 1179 mutex_unlock(&blkcg_pol_mutex); 1180 1181 free_percpu(blkcg->lhead); 1182 kfree(blkcg); 1183 } 1184 1185 static struct cgroup_subsys_state * 1186 blkcg_css_alloc(struct cgroup_subsys_state *parent_css) 1187 { 1188 struct blkcg *blkcg; 1189 int i; 1190 1191 mutex_lock(&blkcg_pol_mutex); 1192 1193 if (!parent_css) { 1194 blkcg = &blkcg_root; 1195 } else { 1196 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); 1197 if (!blkcg) 1198 goto unlock; 1199 } 1200 1201 if (init_blkcg_llists(blkcg)) 1202 goto free_blkcg; 1203 1204 for (i = 0; i < BLKCG_MAX_POLS ; i++) { 1205 struct blkcg_policy *pol = blkcg_policy[i]; 1206 struct blkcg_policy_data *cpd; 1207 1208 /* 1209 * If the policy hasn't been attached yet, wait for it 1210 * to be attached before doing anything else. Otherwise, 1211 * check if the policy requires any specific per-cgroup 1212 * data: if it does, allocate and initialize it. 1213 */ 1214 if (!pol || !pol->cpd_alloc_fn) 1215 continue; 1216 1217 cpd = pol->cpd_alloc_fn(GFP_KERNEL); 1218 if (!cpd) 1219 goto free_pd_blkcg; 1220 1221 blkcg->cpd[i] = cpd; 1222 cpd->blkcg = blkcg; 1223 cpd->plid = i; 1224 if (pol->cpd_init_fn) 1225 pol->cpd_init_fn(cpd); 1226 } 1227 1228 spin_lock_init(&blkcg->lock); 1229 refcount_set(&blkcg->online_pin, 1); 1230 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN); 1231 INIT_HLIST_HEAD(&blkcg->blkg_list); 1232 #ifdef CONFIG_CGROUP_WRITEBACK 1233 INIT_LIST_HEAD(&blkcg->cgwb_list); 1234 #endif 1235 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs); 1236 1237 mutex_unlock(&blkcg_pol_mutex); 1238 return &blkcg->css; 1239 1240 free_pd_blkcg: 1241 for (i--; i >= 0; i--) 1242 if (blkcg->cpd[i]) 1243 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); 1244 free_percpu(blkcg->lhead); 1245 free_blkcg: 1246 if (blkcg != &blkcg_root) 1247 kfree(blkcg); 1248 unlock: 1249 mutex_unlock(&blkcg_pol_mutex); 1250 return ERR_PTR(-ENOMEM); 1251 } 1252 1253 static int blkcg_css_online(struct cgroup_subsys_state *css) 1254 { 1255 struct blkcg *parent = blkcg_parent(css_to_blkcg(css)); 1256 1257 /* 1258 * blkcg_pin_online() is used to delay blkcg offline so that blkgs 1259 * don't go offline while cgwbs are still active on them. Pin the 1260 * parent so that offline always happens towards the root. 1261 */ 1262 if (parent) 1263 blkcg_pin_online(&parent->css); 1264 return 0; 1265 } 1266 1267 int blkcg_init_disk(struct gendisk *disk) 1268 { 1269 struct request_queue *q = disk->queue; 1270 struct blkcg_gq *new_blkg, *blkg; 1271 bool preloaded; 1272 int ret; 1273 1274 INIT_LIST_HEAD(&q->blkg_list); 1275 1276 new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); 1277 if (!new_blkg) 1278 return -ENOMEM; 1279 1280 preloaded = !radix_tree_preload(GFP_KERNEL); 1281 1282 /* Make sure the root blkg exists. */ 1283 /* spin_lock_irq can serve as RCU read-side critical section. */ 1284 spin_lock_irq(&q->queue_lock); 1285 blkg = blkg_create(&blkcg_root, disk, new_blkg); 1286 if (IS_ERR(blkg)) 1287 goto err_unlock; 1288 q->root_blkg = blkg; 1289 spin_unlock_irq(&q->queue_lock); 1290 1291 if (preloaded) 1292 radix_tree_preload_end(); 1293 1294 ret = blk_ioprio_init(disk); 1295 if (ret) 1296 goto err_destroy_all; 1297 1298 ret = blk_throtl_init(disk); 1299 if (ret) 1300 goto err_ioprio_exit; 1301 1302 ret = blk_iolatency_init(disk); 1303 if (ret) 1304 goto err_throtl_exit; 1305 1306 return 0; 1307 1308 err_throtl_exit: 1309 blk_throtl_exit(disk); 1310 err_ioprio_exit: 1311 blk_ioprio_exit(disk); 1312 err_destroy_all: 1313 blkg_destroy_all(disk); 1314 return ret; 1315 err_unlock: 1316 spin_unlock_irq(&q->queue_lock); 1317 if (preloaded) 1318 radix_tree_preload_end(); 1319 return PTR_ERR(blkg); 1320 } 1321 1322 void blkcg_exit_disk(struct gendisk *disk) 1323 { 1324 blkg_destroy_all(disk); 1325 blk_throtl_exit(disk); 1326 } 1327 1328 static void blkcg_bind(struct cgroup_subsys_state *root_css) 1329 { 1330 int i; 1331 1332 mutex_lock(&blkcg_pol_mutex); 1333 1334 for (i = 0; i < BLKCG_MAX_POLS; i++) { 1335 struct blkcg_policy *pol = blkcg_policy[i]; 1336 struct blkcg *blkcg; 1337 1338 if (!pol || !pol->cpd_bind_fn) 1339 continue; 1340 1341 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) 1342 if (blkcg->cpd[pol->plid]) 1343 pol->cpd_bind_fn(blkcg->cpd[pol->plid]); 1344 } 1345 mutex_unlock(&blkcg_pol_mutex); 1346 } 1347 1348 static void blkcg_exit(struct task_struct *tsk) 1349 { 1350 if (tsk->throttle_queue) 1351 blk_put_queue(tsk->throttle_queue); 1352 tsk->throttle_queue = NULL; 1353 } 1354 1355 struct cgroup_subsys io_cgrp_subsys = { 1356 .css_alloc = blkcg_css_alloc, 1357 .css_online = blkcg_css_online, 1358 .css_offline = blkcg_css_offline, 1359 .css_free = blkcg_css_free, 1360 .css_rstat_flush = blkcg_rstat_flush, 1361 .bind = blkcg_bind, 1362 .dfl_cftypes = blkcg_files, 1363 .legacy_cftypes = blkcg_legacy_files, 1364 .legacy_name = "blkio", 1365 .exit = blkcg_exit, 1366 #ifdef CONFIG_MEMCG 1367 /* 1368 * This ensures that, if available, memcg is automatically enabled 1369 * together on the default hierarchy so that the owner cgroup can 1370 * be retrieved from writeback pages. 1371 */ 1372 .depends_on = 1 << memory_cgrp_id, 1373 #endif 1374 }; 1375 EXPORT_SYMBOL_GPL(io_cgrp_subsys); 1376 1377 /** 1378 * blkcg_activate_policy - activate a blkcg policy on a request_queue 1379 * @q: request_queue of interest 1380 * @pol: blkcg policy to activate 1381 * 1382 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through 1383 * bypass mode to populate its blkgs with policy_data for @pol. 1384 * 1385 * Activation happens with @q bypassed, so nobody would be accessing blkgs 1386 * from IO path. Update of each blkg is protected by both queue and blkcg 1387 * locks so that holding either lock and testing blkcg_policy_enabled() is 1388 * always enough for dereferencing policy data. 1389 * 1390 * The caller is responsible for synchronizing [de]activations and policy 1391 * [un]registerations. Returns 0 on success, -errno on failure. 1392 */ 1393 int blkcg_activate_policy(struct request_queue *q, 1394 const struct blkcg_policy *pol) 1395 { 1396 struct blkg_policy_data *pd_prealloc = NULL; 1397 struct blkcg_gq *blkg, *pinned_blkg = NULL; 1398 int ret; 1399 1400 if (blkcg_policy_enabled(q, pol)) 1401 return 0; 1402 1403 if (queue_is_mq(q)) 1404 blk_mq_freeze_queue(q); 1405 retry: 1406 spin_lock_irq(&q->queue_lock); 1407 1408 /* blkg_list is pushed at the head, reverse walk to allocate parents first */ 1409 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { 1410 struct blkg_policy_data *pd; 1411 1412 if (blkg->pd[pol->plid]) 1413 continue; 1414 1415 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */ 1416 if (blkg == pinned_blkg) { 1417 pd = pd_prealloc; 1418 pd_prealloc = NULL; 1419 } else { 1420 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q, 1421 blkg->blkcg); 1422 } 1423 1424 if (!pd) { 1425 /* 1426 * GFP_NOWAIT failed. Free the existing one and 1427 * prealloc for @blkg w/ GFP_KERNEL. 1428 */ 1429 if (pinned_blkg) 1430 blkg_put(pinned_blkg); 1431 blkg_get(blkg); 1432 pinned_blkg = blkg; 1433 1434 spin_unlock_irq(&q->queue_lock); 1435 1436 if (pd_prealloc) 1437 pol->pd_free_fn(pd_prealloc); 1438 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q, 1439 blkg->blkcg); 1440 if (pd_prealloc) 1441 goto retry; 1442 else 1443 goto enomem; 1444 } 1445 1446 blkg->pd[pol->plid] = pd; 1447 pd->blkg = blkg; 1448 pd->plid = pol->plid; 1449 } 1450 1451 /* all allocated, init in the same order */ 1452 if (pol->pd_init_fn) 1453 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) 1454 pol->pd_init_fn(blkg->pd[pol->plid]); 1455 1456 __set_bit(pol->plid, q->blkcg_pols); 1457 ret = 0; 1458 1459 spin_unlock_irq(&q->queue_lock); 1460 out: 1461 if (queue_is_mq(q)) 1462 blk_mq_unfreeze_queue(q); 1463 if (pinned_blkg) 1464 blkg_put(pinned_blkg); 1465 if (pd_prealloc) 1466 pol->pd_free_fn(pd_prealloc); 1467 return ret; 1468 1469 enomem: 1470 /* alloc failed, nothing's initialized yet, free everything */ 1471 spin_lock_irq(&q->queue_lock); 1472 list_for_each_entry(blkg, &q->blkg_list, q_node) { 1473 struct blkcg *blkcg = blkg->blkcg; 1474 1475 spin_lock(&blkcg->lock); 1476 if (blkg->pd[pol->plid]) { 1477 pol->pd_free_fn(blkg->pd[pol->plid]); 1478 blkg->pd[pol->plid] = NULL; 1479 } 1480 spin_unlock(&blkcg->lock); 1481 } 1482 spin_unlock_irq(&q->queue_lock); 1483 ret = -ENOMEM; 1484 goto out; 1485 } 1486 EXPORT_SYMBOL_GPL(blkcg_activate_policy); 1487 1488 /** 1489 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue 1490 * @q: request_queue of interest 1491 * @pol: blkcg policy to deactivate 1492 * 1493 * Deactivate @pol on @q. Follows the same synchronization rules as 1494 * blkcg_activate_policy(). 1495 */ 1496 void blkcg_deactivate_policy(struct request_queue *q, 1497 const struct blkcg_policy *pol) 1498 { 1499 struct blkcg_gq *blkg; 1500 1501 if (!blkcg_policy_enabled(q, pol)) 1502 return; 1503 1504 if (queue_is_mq(q)) 1505 blk_mq_freeze_queue(q); 1506 1507 spin_lock_irq(&q->queue_lock); 1508 1509 __clear_bit(pol->plid, q->blkcg_pols); 1510 1511 list_for_each_entry(blkg, &q->blkg_list, q_node) { 1512 struct blkcg *blkcg = blkg->blkcg; 1513 1514 spin_lock(&blkcg->lock); 1515 if (blkg->pd[pol->plid]) { 1516 if (pol->pd_offline_fn) 1517 pol->pd_offline_fn(blkg->pd[pol->plid]); 1518 pol->pd_free_fn(blkg->pd[pol->plid]); 1519 blkg->pd[pol->plid] = NULL; 1520 } 1521 spin_unlock(&blkcg->lock); 1522 } 1523 1524 spin_unlock_irq(&q->queue_lock); 1525 1526 if (queue_is_mq(q)) 1527 blk_mq_unfreeze_queue(q); 1528 } 1529 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy); 1530 1531 static void blkcg_free_all_cpd(struct blkcg_policy *pol) 1532 { 1533 struct blkcg *blkcg; 1534 1535 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { 1536 if (blkcg->cpd[pol->plid]) { 1537 pol->cpd_free_fn(blkcg->cpd[pol->plid]); 1538 blkcg->cpd[pol->plid] = NULL; 1539 } 1540 } 1541 } 1542 1543 /** 1544 * blkcg_policy_register - register a blkcg policy 1545 * @pol: blkcg policy to register 1546 * 1547 * Register @pol with blkcg core. Might sleep and @pol may be modified on 1548 * successful registration. Returns 0 on success and -errno on failure. 1549 */ 1550 int blkcg_policy_register(struct blkcg_policy *pol) 1551 { 1552 struct blkcg *blkcg; 1553 int i, ret; 1554 1555 mutex_lock(&blkcg_pol_register_mutex); 1556 mutex_lock(&blkcg_pol_mutex); 1557 1558 /* find an empty slot */ 1559 ret = -ENOSPC; 1560 for (i = 0; i < BLKCG_MAX_POLS; i++) 1561 if (!blkcg_policy[i]) 1562 break; 1563 if (i >= BLKCG_MAX_POLS) { 1564 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n"); 1565 goto err_unlock; 1566 } 1567 1568 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */ 1569 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) || 1570 (!pol->pd_alloc_fn ^ !pol->pd_free_fn)) 1571 goto err_unlock; 1572 1573 /* register @pol */ 1574 pol->plid = i; 1575 blkcg_policy[pol->plid] = pol; 1576 1577 /* allocate and install cpd's */ 1578 if (pol->cpd_alloc_fn) { 1579 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { 1580 struct blkcg_policy_data *cpd; 1581 1582 cpd = pol->cpd_alloc_fn(GFP_KERNEL); 1583 if (!cpd) 1584 goto err_free_cpds; 1585 1586 blkcg->cpd[pol->plid] = cpd; 1587 cpd->blkcg = blkcg; 1588 cpd->plid = pol->plid; 1589 if (pol->cpd_init_fn) 1590 pol->cpd_init_fn(cpd); 1591 } 1592 } 1593 1594 mutex_unlock(&blkcg_pol_mutex); 1595 1596 /* everything is in place, add intf files for the new policy */ 1597 if (pol->dfl_cftypes) 1598 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys, 1599 pol->dfl_cftypes)); 1600 if (pol->legacy_cftypes) 1601 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys, 1602 pol->legacy_cftypes)); 1603 mutex_unlock(&blkcg_pol_register_mutex); 1604 return 0; 1605 1606 err_free_cpds: 1607 if (pol->cpd_free_fn) 1608 blkcg_free_all_cpd(pol); 1609 1610 blkcg_policy[pol->plid] = NULL; 1611 err_unlock: 1612 mutex_unlock(&blkcg_pol_mutex); 1613 mutex_unlock(&blkcg_pol_register_mutex); 1614 return ret; 1615 } 1616 EXPORT_SYMBOL_GPL(blkcg_policy_register); 1617 1618 /** 1619 * blkcg_policy_unregister - unregister a blkcg policy 1620 * @pol: blkcg policy to unregister 1621 * 1622 * Undo blkcg_policy_register(@pol). Might sleep. 1623 */ 1624 void blkcg_policy_unregister(struct blkcg_policy *pol) 1625 { 1626 mutex_lock(&blkcg_pol_register_mutex); 1627 1628 if (WARN_ON(blkcg_policy[pol->plid] != pol)) 1629 goto out_unlock; 1630 1631 /* kill the intf files first */ 1632 if (pol->dfl_cftypes) 1633 cgroup_rm_cftypes(pol->dfl_cftypes); 1634 if (pol->legacy_cftypes) 1635 cgroup_rm_cftypes(pol->legacy_cftypes); 1636 1637 /* remove cpds and unregister */ 1638 mutex_lock(&blkcg_pol_mutex); 1639 1640 if (pol->cpd_free_fn) 1641 blkcg_free_all_cpd(pol); 1642 1643 blkcg_policy[pol->plid] = NULL; 1644 1645 mutex_unlock(&blkcg_pol_mutex); 1646 out_unlock: 1647 mutex_unlock(&blkcg_pol_register_mutex); 1648 } 1649 EXPORT_SYMBOL_GPL(blkcg_policy_unregister); 1650 1651 bool __blkcg_punt_bio_submit(struct bio *bio) 1652 { 1653 struct blkcg_gq *blkg = bio->bi_blkg; 1654 1655 /* consume the flag first */ 1656 bio->bi_opf &= ~REQ_CGROUP_PUNT; 1657 1658 /* never bounce for the root cgroup */ 1659 if (!blkg->parent) 1660 return false; 1661 1662 spin_lock_bh(&blkg->async_bio_lock); 1663 bio_list_add(&blkg->async_bios, bio); 1664 spin_unlock_bh(&blkg->async_bio_lock); 1665 1666 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work); 1667 return true; 1668 } 1669 1670 /* 1671 * Scale the accumulated delay based on how long it has been since we updated 1672 * the delay. We only call this when we are adding delay, in case it's been a 1673 * while since we added delay, and when we are checking to see if we need to 1674 * delay a task, to account for any delays that may have occurred. 1675 */ 1676 static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now) 1677 { 1678 u64 old = atomic64_read(&blkg->delay_start); 1679 1680 /* negative use_delay means no scaling, see blkcg_set_delay() */ 1681 if (atomic_read(&blkg->use_delay) < 0) 1682 return; 1683 1684 /* 1685 * We only want to scale down every second. The idea here is that we 1686 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain 1687 * time window. We only want to throttle tasks for recent delay that 1688 * has occurred, in 1 second time windows since that's the maximum 1689 * things can be throttled. We save the current delay window in 1690 * blkg->last_delay so we know what amount is still left to be charged 1691 * to the blkg from this point onward. blkg->last_use keeps track of 1692 * the use_delay counter. The idea is if we're unthrottling the blkg we 1693 * are ok with whatever is happening now, and we can take away more of 1694 * the accumulated delay as we've already throttled enough that 1695 * everybody is happy with their IO latencies. 1696 */ 1697 if (time_before64(old + NSEC_PER_SEC, now) && 1698 atomic64_try_cmpxchg(&blkg->delay_start, &old, now)) { 1699 u64 cur = atomic64_read(&blkg->delay_nsec); 1700 u64 sub = min_t(u64, blkg->last_delay, now - old); 1701 int cur_use = atomic_read(&blkg->use_delay); 1702 1703 /* 1704 * We've been unthrottled, subtract a larger chunk of our 1705 * accumulated delay. 1706 */ 1707 if (cur_use < blkg->last_use) 1708 sub = max_t(u64, sub, blkg->last_delay >> 1); 1709 1710 /* 1711 * This shouldn't happen, but handle it anyway. Our delay_nsec 1712 * should only ever be growing except here where we subtract out 1713 * min(last_delay, 1 second), but lord knows bugs happen and I'd 1714 * rather not end up with negative numbers. 1715 */ 1716 if (unlikely(cur < sub)) { 1717 atomic64_set(&blkg->delay_nsec, 0); 1718 blkg->last_delay = 0; 1719 } else { 1720 atomic64_sub(sub, &blkg->delay_nsec); 1721 blkg->last_delay = cur - sub; 1722 } 1723 blkg->last_use = cur_use; 1724 } 1725 } 1726 1727 /* 1728 * This is called when we want to actually walk up the hierarchy and check to 1729 * see if we need to throttle, and then actually throttle if there is some 1730 * accumulated delay. This should only be called upon return to user space so 1731 * we're not holding some lock that would induce a priority inversion. 1732 */ 1733 static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay) 1734 { 1735 unsigned long pflags; 1736 bool clamp; 1737 u64 now = ktime_to_ns(ktime_get()); 1738 u64 exp; 1739 u64 delay_nsec = 0; 1740 int tok; 1741 1742 while (blkg->parent) { 1743 int use_delay = atomic_read(&blkg->use_delay); 1744 1745 if (use_delay) { 1746 u64 this_delay; 1747 1748 blkcg_scale_delay(blkg, now); 1749 this_delay = atomic64_read(&blkg->delay_nsec); 1750 if (this_delay > delay_nsec) { 1751 delay_nsec = this_delay; 1752 clamp = use_delay > 0; 1753 } 1754 } 1755 blkg = blkg->parent; 1756 } 1757 1758 if (!delay_nsec) 1759 return; 1760 1761 /* 1762 * Let's not sleep for all eternity if we've amassed a huge delay. 1763 * Swapping or metadata IO can accumulate 10's of seconds worth of 1764 * delay, and we want userspace to be able to do _something_ so cap the 1765 * delays at 0.25s. If there's 10's of seconds worth of delay then the 1766 * tasks will be delayed for 0.25 second for every syscall. If 1767 * blkcg_set_delay() was used as indicated by negative use_delay, the 1768 * caller is responsible for regulating the range. 1769 */ 1770 if (clamp) 1771 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC); 1772 1773 if (use_memdelay) 1774 psi_memstall_enter(&pflags); 1775 1776 exp = ktime_add_ns(now, delay_nsec); 1777 tok = io_schedule_prepare(); 1778 do { 1779 __set_current_state(TASK_KILLABLE); 1780 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS)) 1781 break; 1782 } while (!fatal_signal_pending(current)); 1783 io_schedule_finish(tok); 1784 1785 if (use_memdelay) 1786 psi_memstall_leave(&pflags); 1787 } 1788 1789 /** 1790 * blkcg_maybe_throttle_current - throttle the current task if it has been marked 1791 * 1792 * This is only called if we've been marked with set_notify_resume(). Obviously 1793 * we can be set_notify_resume() for reasons other than blkcg throttling, so we 1794 * check to see if current->throttle_queue is set and if not this doesn't do 1795 * anything. This should only ever be called by the resume code, it's not meant 1796 * to be called by people willy-nilly as it will actually do the work to 1797 * throttle the task if it is setup for throttling. 1798 */ 1799 void blkcg_maybe_throttle_current(void) 1800 { 1801 struct request_queue *q = current->throttle_queue; 1802 struct blkcg *blkcg; 1803 struct blkcg_gq *blkg; 1804 bool use_memdelay = current->use_memdelay; 1805 1806 if (!q) 1807 return; 1808 1809 current->throttle_queue = NULL; 1810 current->use_memdelay = false; 1811 1812 rcu_read_lock(); 1813 blkcg = css_to_blkcg(blkcg_css()); 1814 if (!blkcg) 1815 goto out; 1816 blkg = blkg_lookup(blkcg, q); 1817 if (!blkg) 1818 goto out; 1819 if (!blkg_tryget(blkg)) 1820 goto out; 1821 rcu_read_unlock(); 1822 1823 blkcg_maybe_throttle_blkg(blkg, use_memdelay); 1824 blkg_put(blkg); 1825 blk_put_queue(q); 1826 return; 1827 out: 1828 rcu_read_unlock(); 1829 blk_put_queue(q); 1830 } 1831 1832 /** 1833 * blkcg_schedule_throttle - this task needs to check for throttling 1834 * @disk: disk to throttle 1835 * @use_memdelay: do we charge this to memory delay for PSI 1836 * 1837 * This is called by the IO controller when we know there's delay accumulated 1838 * for the blkg for this task. We do not pass the blkg because there are places 1839 * we call this that may not have that information, the swapping code for 1840 * instance will only have a block_device at that point. This set's the 1841 * notify_resume for the task to check and see if it requires throttling before 1842 * returning to user space. 1843 * 1844 * We will only schedule once per syscall. You can call this over and over 1845 * again and it will only do the check once upon return to user space, and only 1846 * throttle once. If the task needs to be throttled again it'll need to be 1847 * re-set at the next time we see the task. 1848 */ 1849 void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay) 1850 { 1851 struct request_queue *q = disk->queue; 1852 1853 if (unlikely(current->flags & PF_KTHREAD)) 1854 return; 1855 1856 if (current->throttle_queue != q) { 1857 if (!blk_get_queue(q)) 1858 return; 1859 1860 if (current->throttle_queue) 1861 blk_put_queue(current->throttle_queue); 1862 current->throttle_queue = q; 1863 } 1864 1865 if (use_memdelay) 1866 current->use_memdelay = use_memdelay; 1867 set_notify_resume(current); 1868 } 1869 1870 /** 1871 * blkcg_add_delay - add delay to this blkg 1872 * @blkg: blkg of interest 1873 * @now: the current time in nanoseconds 1874 * @delta: how many nanoseconds of delay to add 1875 * 1876 * Charge @delta to the blkg's current delay accumulation. This is used to 1877 * throttle tasks if an IO controller thinks we need more throttling. 1878 */ 1879 void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) 1880 { 1881 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0)) 1882 return; 1883 blkcg_scale_delay(blkg, now); 1884 atomic64_add(delta, &blkg->delay_nsec); 1885 } 1886 1887 /** 1888 * blkg_tryget_closest - try and get a blkg ref on the closet blkg 1889 * @bio: target bio 1890 * @css: target css 1891 * 1892 * As the failure mode here is to walk up the blkg tree, this ensure that the 1893 * blkg->parent pointers are always valid. This returns the blkg that it ended 1894 * up taking a reference on or %NULL if no reference was taken. 1895 */ 1896 static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, 1897 struct cgroup_subsys_state *css) 1898 { 1899 struct blkcg_gq *blkg, *ret_blkg = NULL; 1900 1901 rcu_read_lock(); 1902 blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk); 1903 while (blkg) { 1904 if (blkg_tryget(blkg)) { 1905 ret_blkg = blkg; 1906 break; 1907 } 1908 blkg = blkg->parent; 1909 } 1910 rcu_read_unlock(); 1911 1912 return ret_blkg; 1913 } 1914 1915 /** 1916 * bio_associate_blkg_from_css - associate a bio with a specified css 1917 * @bio: target bio 1918 * @css: target css 1919 * 1920 * Associate @bio with the blkg found by combining the css's blkg and the 1921 * request_queue of the @bio. An association failure is handled by walking up 1922 * the blkg tree. Therefore, the blkg associated can be anything between @blkg 1923 * and q->root_blkg. This situation only happens when a cgroup is dying and 1924 * then the remaining bios will spill to the closest alive blkg. 1925 * 1926 * A reference will be taken on the blkg and will be released when @bio is 1927 * freed. 1928 */ 1929 void bio_associate_blkg_from_css(struct bio *bio, 1930 struct cgroup_subsys_state *css) 1931 { 1932 if (bio->bi_blkg) 1933 blkg_put(bio->bi_blkg); 1934 1935 if (css && css->parent) { 1936 bio->bi_blkg = blkg_tryget_closest(bio, css); 1937 } else { 1938 blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg); 1939 bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg; 1940 } 1941 } 1942 EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); 1943 1944 /** 1945 * bio_associate_blkg - associate a bio with a blkg 1946 * @bio: target bio 1947 * 1948 * Associate @bio with the blkg found from the bio's css and request_queue. 1949 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is 1950 * already associated, the css is reused and association redone as the 1951 * request_queue may have changed. 1952 */ 1953 void bio_associate_blkg(struct bio *bio) 1954 { 1955 struct cgroup_subsys_state *css; 1956 1957 rcu_read_lock(); 1958 1959 if (bio->bi_blkg) 1960 css = bio_blkcg_css(bio); 1961 else 1962 css = blkcg_css(); 1963 1964 bio_associate_blkg_from_css(bio, css); 1965 1966 rcu_read_unlock(); 1967 } 1968 EXPORT_SYMBOL_GPL(bio_associate_blkg); 1969 1970 /** 1971 * bio_clone_blkg_association - clone blkg association from src to dst bio 1972 * @dst: destination bio 1973 * @src: source bio 1974 */ 1975 void bio_clone_blkg_association(struct bio *dst, struct bio *src) 1976 { 1977 if (src->bi_blkg) 1978 bio_associate_blkg_from_css(dst, bio_blkcg_css(src)); 1979 } 1980 EXPORT_SYMBOL_GPL(bio_clone_blkg_association); 1981 1982 static int blk_cgroup_io_type(struct bio *bio) 1983 { 1984 if (op_is_discard(bio->bi_opf)) 1985 return BLKG_IOSTAT_DISCARD; 1986 if (op_is_write(bio->bi_opf)) 1987 return BLKG_IOSTAT_WRITE; 1988 return BLKG_IOSTAT_READ; 1989 } 1990 1991 void blk_cgroup_bio_start(struct bio *bio) 1992 { 1993 struct blkcg *blkcg = bio->bi_blkg->blkcg; 1994 int rwd = blk_cgroup_io_type(bio), cpu; 1995 struct blkg_iostat_set *bis; 1996 unsigned long flags; 1997 1998 cpu = get_cpu(); 1999 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu); 2000 flags = u64_stats_update_begin_irqsave(&bis->sync); 2001 2002 /* 2003 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split 2004 * bio and we would have already accounted for the size of the bio. 2005 */ 2006 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) { 2007 bio_set_flag(bio, BIO_CGROUP_ACCT); 2008 bis->cur.bytes[rwd] += bio->bi_iter.bi_size; 2009 } 2010 bis->cur.ios[rwd]++; 2011 2012 /* 2013 * If the iostat_cpu isn't in a lockless list, put it into the 2014 * list to indicate that a stat update is pending. 2015 */ 2016 if (!READ_ONCE(bis->lqueued)) { 2017 struct llist_head *lhead = this_cpu_ptr(blkcg->lhead); 2018 2019 llist_add(&bis->lnode, lhead); 2020 WRITE_ONCE(bis->lqueued, true); 2021 percpu_ref_get(&bis->blkg->refcnt); 2022 } 2023 2024 u64_stats_update_end_irqrestore(&bis->sync, flags); 2025 if (cgroup_subsys_on_dfl(io_cgrp_subsys)) 2026 cgroup_rstat_updated(blkcg->css.cgroup, cpu); 2027 put_cpu(); 2028 } 2029 2030 bool blk_cgroup_congested(void) 2031 { 2032 struct cgroup_subsys_state *css; 2033 bool ret = false; 2034 2035 rcu_read_lock(); 2036 for (css = blkcg_css(); css; css = css->parent) { 2037 if (atomic_read(&css->cgroup->congestion_count)) { 2038 ret = true; 2039 break; 2040 } 2041 } 2042 rcu_read_unlock(); 2043 return ret; 2044 } 2045 2046 static int __init blkcg_init(void) 2047 { 2048 blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio", 2049 WQ_MEM_RECLAIM | WQ_FREEZABLE | 2050 WQ_UNBOUND | WQ_SYSFS, 0); 2051 if (!blkcg_punt_bio_wq) 2052 return -ENOMEM; 2053 return 0; 2054 } 2055 subsys_initcall(blkcg_init); 2056 2057 module_param(blkcg_debug_stats, bool, 0644); 2058 MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not"); 2059