1 /* 2 * Common Block IO controller cgroup interface 3 * 4 * Based on ideas and code from CFQ, CFS and BFQ: 5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk> 6 * 7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it> 8 * Paolo Valente <paolo.valente@unimore.it> 9 * 10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com> 11 * Nauman Rafique <nauman@google.com> 12 * 13 * For policy-specific per-blkcg data: 14 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it> 15 * Arianna Avanzini <avanzini.arianna@gmail.com> 16 */ 17 #include <linux/ioprio.h> 18 #include <linux/kdev_t.h> 19 #include <linux/module.h> 20 #include <linux/err.h> 21 #include <linux/blkdev.h> 22 #include <linux/backing-dev.h> 23 #include <linux/slab.h> 24 #include <linux/genhd.h> 25 #include <linux/delay.h> 26 #include <linux/atomic.h> 27 #include <linux/blk-cgroup.h> 28 #include "blk.h" 29 30 #define MAX_KEY_LEN 100 31 32 static DEFINE_MUTEX(blkcg_pol_mutex); 33 34 struct blkcg blkcg_root; 35 EXPORT_SYMBOL_GPL(blkcg_root); 36 37 struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css; 38 39 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS]; 40 41 static bool blkcg_policy_enabled(struct request_queue *q, 42 const struct blkcg_policy *pol) 43 { 44 return pol && test_bit(pol->plid, q->blkcg_pols); 45 } 46 47 /** 48 * blkg_free - free a blkg 49 * @blkg: blkg to free 50 * 51 * Free @blkg which may be partially allocated. 52 */ 53 static void blkg_free(struct blkcg_gq *blkg) 54 { 55 int i; 56 57 if (!blkg) 58 return; 59 60 for (i = 0; i < BLKCG_MAX_POLS; i++) 61 kfree(blkg->pd[i]); 62 63 blk_exit_rl(&blkg->rl); 64 kfree(blkg); 65 } 66 67 /** 68 * blkg_alloc - allocate a blkg 69 * @blkcg: block cgroup the new blkg is associated with 70 * @q: request_queue the new blkg is associated with 71 * @gfp_mask: allocation mask to use 72 * 73 * Allocate a new blkg assocating @blkcg and @q. 74 */ 75 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, 76 gfp_t gfp_mask) 77 { 78 struct blkcg_gq *blkg; 79 int i; 80 81 /* alloc and init base part */ 82 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node); 83 if (!blkg) 84 return NULL; 85 86 blkg->q = q; 87 INIT_LIST_HEAD(&blkg->q_node); 88 blkg->blkcg = blkcg; 89 atomic_set(&blkg->refcnt, 1); 90 91 /* root blkg uses @q->root_rl, init rl only for !root blkgs */ 92 if (blkcg != &blkcg_root) { 93 if (blk_init_rl(&blkg->rl, q, gfp_mask)) 94 goto err_free; 95 blkg->rl.blkg = blkg; 96 } 97 98 for (i = 0; i < BLKCG_MAX_POLS; i++) { 99 struct blkcg_policy *pol = blkcg_policy[i]; 100 struct blkg_policy_data *pd; 101 102 if (!blkcg_policy_enabled(q, pol)) 103 continue; 104 105 /* alloc per-policy data and attach it to blkg */ 106 pd = kzalloc_node(pol->pd_size, gfp_mask, q->node); 107 if (!pd) 108 goto err_free; 109 110 blkg->pd[i] = pd; 111 pd->blkg = blkg; 112 pd->plid = i; 113 } 114 115 return blkg; 116 117 err_free: 118 blkg_free(blkg); 119 return NULL; 120 } 121 122 /** 123 * __blkg_lookup - internal version of blkg_lookup() 124 * @blkcg: blkcg of interest 125 * @q: request_queue of interest 126 * @update_hint: whether to update lookup hint with the result or not 127 * 128 * This is internal version and shouldn't be used by policy 129 * implementations. Looks up blkgs for the @blkcg - @q pair regardless of 130 * @q's bypass state. If @update_hint is %true, the caller should be 131 * holding @q->queue_lock and lookup hint is updated on success. 132 */ 133 struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, struct request_queue *q, 134 bool update_hint) 135 { 136 struct blkcg_gq *blkg; 137 138 blkg = rcu_dereference(blkcg->blkg_hint); 139 if (blkg && blkg->q == q) 140 return blkg; 141 142 /* 143 * Hint didn't match. Look up from the radix tree. Note that the 144 * hint can only be updated under queue_lock as otherwise @blkg 145 * could have already been removed from blkg_tree. The caller is 146 * responsible for grabbing queue_lock if @update_hint. 147 */ 148 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); 149 if (blkg && blkg->q == q) { 150 if (update_hint) { 151 lockdep_assert_held(q->queue_lock); 152 rcu_assign_pointer(blkcg->blkg_hint, blkg); 153 } 154 return blkg; 155 } 156 157 return NULL; 158 } 159 160 /** 161 * blkg_lookup - lookup blkg for the specified blkcg - q pair 162 * @blkcg: blkcg of interest 163 * @q: request_queue of interest 164 * 165 * Lookup blkg for the @blkcg - @q pair. This function should be called 166 * under RCU read lock and is guaranteed to return %NULL if @q is bypassing 167 * - see blk_queue_bypass_start() for details. 168 */ 169 struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q) 170 { 171 WARN_ON_ONCE(!rcu_read_lock_held()); 172 173 if (unlikely(blk_queue_bypass(q))) 174 return NULL; 175 return __blkg_lookup(blkcg, q, false); 176 } 177 EXPORT_SYMBOL_GPL(blkg_lookup); 178 179 /* 180 * If @new_blkg is %NULL, this function tries to allocate a new one as 181 * necessary using %GFP_ATOMIC. @new_blkg is always consumed on return. 182 */ 183 static struct blkcg_gq *blkg_create(struct blkcg *blkcg, 184 struct request_queue *q, 185 struct blkcg_gq *new_blkg) 186 { 187 struct blkcg_gq *blkg; 188 struct bdi_writeback_congested *wb_congested; 189 int i, ret; 190 191 WARN_ON_ONCE(!rcu_read_lock_held()); 192 lockdep_assert_held(q->queue_lock); 193 194 /* blkg holds a reference to blkcg */ 195 if (!css_tryget_online(&blkcg->css)) { 196 ret = -EINVAL; 197 goto err_free_blkg; 198 } 199 200 wb_congested = wb_congested_get_create(&q->backing_dev_info, 201 blkcg->css.id, GFP_ATOMIC); 202 if (!wb_congested) { 203 ret = -ENOMEM; 204 goto err_put_css; 205 } 206 207 /* allocate */ 208 if (!new_blkg) { 209 new_blkg = blkg_alloc(blkcg, q, GFP_ATOMIC); 210 if (unlikely(!new_blkg)) { 211 ret = -ENOMEM; 212 goto err_put_congested; 213 } 214 } 215 blkg = new_blkg; 216 blkg->wb_congested = wb_congested; 217 218 /* link parent */ 219 if (blkcg_parent(blkcg)) { 220 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); 221 if (WARN_ON_ONCE(!blkg->parent)) { 222 ret = -EINVAL; 223 goto err_put_congested; 224 } 225 blkg_get(blkg->parent); 226 } 227 228 /* invoke per-policy init */ 229 for (i = 0; i < BLKCG_MAX_POLS; i++) { 230 struct blkcg_policy *pol = blkcg_policy[i]; 231 232 if (blkg->pd[i] && pol->pd_init_fn) 233 pol->pd_init_fn(blkg); 234 } 235 236 /* insert */ 237 spin_lock(&blkcg->lock); 238 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); 239 if (likely(!ret)) { 240 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); 241 list_add(&blkg->q_node, &q->blkg_list); 242 243 for (i = 0; i < BLKCG_MAX_POLS; i++) { 244 struct blkcg_policy *pol = blkcg_policy[i]; 245 246 if (blkg->pd[i] && pol->pd_online_fn) 247 pol->pd_online_fn(blkg); 248 } 249 } 250 blkg->online = true; 251 spin_unlock(&blkcg->lock); 252 253 if (!ret) 254 return blkg; 255 256 /* @blkg failed fully initialized, use the usual release path */ 257 blkg_put(blkg); 258 return ERR_PTR(ret); 259 260 err_put_congested: 261 wb_congested_put(wb_congested); 262 err_put_css: 263 css_put(&blkcg->css); 264 err_free_blkg: 265 blkg_free(new_blkg); 266 return ERR_PTR(ret); 267 } 268 269 /** 270 * blkg_lookup_create - lookup blkg, try to create one if not there 271 * @blkcg: blkcg of interest 272 * @q: request_queue of interest 273 * 274 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to 275 * create one. blkg creation is performed recursively from blkcg_root such 276 * that all non-root blkg's have access to the parent blkg. This function 277 * should be called under RCU read lock and @q->queue_lock. 278 * 279 * Returns pointer to the looked up or created blkg on success, ERR_PTR() 280 * value on error. If @q is dead, returns ERR_PTR(-EINVAL). If @q is not 281 * dead and bypassing, returns ERR_PTR(-EBUSY). 282 */ 283 struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, 284 struct request_queue *q) 285 { 286 struct blkcg_gq *blkg; 287 288 WARN_ON_ONCE(!rcu_read_lock_held()); 289 lockdep_assert_held(q->queue_lock); 290 291 /* 292 * This could be the first entry point of blkcg implementation and 293 * we shouldn't allow anything to go through for a bypassing queue. 294 */ 295 if (unlikely(blk_queue_bypass(q))) 296 return ERR_PTR(blk_queue_dying(q) ? -EINVAL : -EBUSY); 297 298 blkg = __blkg_lookup(blkcg, q, true); 299 if (blkg) 300 return blkg; 301 302 /* 303 * Create blkgs walking down from blkcg_root to @blkcg, so that all 304 * non-root blkgs have access to their parents. 305 */ 306 while (true) { 307 struct blkcg *pos = blkcg; 308 struct blkcg *parent = blkcg_parent(blkcg); 309 310 while (parent && !__blkg_lookup(parent, q, false)) { 311 pos = parent; 312 parent = blkcg_parent(parent); 313 } 314 315 blkg = blkg_create(pos, q, NULL); 316 if (pos == blkcg || IS_ERR(blkg)) 317 return blkg; 318 } 319 } 320 EXPORT_SYMBOL_GPL(blkg_lookup_create); 321 322 static void blkg_destroy(struct blkcg_gq *blkg) 323 { 324 struct blkcg *blkcg = blkg->blkcg; 325 int i; 326 327 lockdep_assert_held(blkg->q->queue_lock); 328 lockdep_assert_held(&blkcg->lock); 329 330 /* Something wrong if we are trying to remove same group twice */ 331 WARN_ON_ONCE(list_empty(&blkg->q_node)); 332 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); 333 334 for (i = 0; i < BLKCG_MAX_POLS; i++) { 335 struct blkcg_policy *pol = blkcg_policy[i]; 336 337 if (blkg->pd[i] && pol->pd_offline_fn) 338 pol->pd_offline_fn(blkg); 339 } 340 blkg->online = false; 341 342 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); 343 list_del_init(&blkg->q_node); 344 hlist_del_init_rcu(&blkg->blkcg_node); 345 346 /* 347 * Both setting lookup hint to and clearing it from @blkg are done 348 * under queue_lock. If it's not pointing to @blkg now, it never 349 * will. Hint assignment itself can race safely. 350 */ 351 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) 352 rcu_assign_pointer(blkcg->blkg_hint, NULL); 353 354 /* 355 * Put the reference taken at the time of creation so that when all 356 * queues are gone, group can be destroyed. 357 */ 358 blkg_put(blkg); 359 } 360 361 /** 362 * blkg_destroy_all - destroy all blkgs associated with a request_queue 363 * @q: request_queue of interest 364 * 365 * Destroy all blkgs associated with @q. 366 */ 367 static void blkg_destroy_all(struct request_queue *q) 368 { 369 struct blkcg_gq *blkg, *n; 370 371 lockdep_assert_held(q->queue_lock); 372 373 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { 374 struct blkcg *blkcg = blkg->blkcg; 375 376 spin_lock(&blkcg->lock); 377 blkg_destroy(blkg); 378 spin_unlock(&blkcg->lock); 379 } 380 } 381 382 /* 383 * A group is RCU protected, but having an rcu lock does not mean that one 384 * can access all the fields of blkg and assume these are valid. For 385 * example, don't try to follow throtl_data and request queue links. 386 * 387 * Having a reference to blkg under an rcu allows accesses to only values 388 * local to groups like group stats and group rate limits. 389 */ 390 void __blkg_release_rcu(struct rcu_head *rcu_head) 391 { 392 struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head); 393 int i; 394 395 /* tell policies that this one is being freed */ 396 for (i = 0; i < BLKCG_MAX_POLS; i++) { 397 struct blkcg_policy *pol = blkcg_policy[i]; 398 399 if (blkg->pd[i] && pol->pd_exit_fn) 400 pol->pd_exit_fn(blkg); 401 } 402 403 /* release the blkcg and parent blkg refs this blkg has been holding */ 404 css_put(&blkg->blkcg->css); 405 if (blkg->parent) 406 blkg_put(blkg->parent); 407 408 wb_congested_put(blkg->wb_congested); 409 410 blkg_free(blkg); 411 } 412 EXPORT_SYMBOL_GPL(__blkg_release_rcu); 413 414 /* 415 * The next function used by blk_queue_for_each_rl(). It's a bit tricky 416 * because the root blkg uses @q->root_rl instead of its own rl. 417 */ 418 struct request_list *__blk_queue_next_rl(struct request_list *rl, 419 struct request_queue *q) 420 { 421 struct list_head *ent; 422 struct blkcg_gq *blkg; 423 424 /* 425 * Determine the current blkg list_head. The first entry is 426 * root_rl which is off @q->blkg_list and mapped to the head. 427 */ 428 if (rl == &q->root_rl) { 429 ent = &q->blkg_list; 430 /* There are no more block groups, hence no request lists */ 431 if (list_empty(ent)) 432 return NULL; 433 } else { 434 blkg = container_of(rl, struct blkcg_gq, rl); 435 ent = &blkg->q_node; 436 } 437 438 /* walk to the next list_head, skip root blkcg */ 439 ent = ent->next; 440 if (ent == &q->root_blkg->q_node) 441 ent = ent->next; 442 if (ent == &q->blkg_list) 443 return NULL; 444 445 blkg = container_of(ent, struct blkcg_gq, q_node); 446 return &blkg->rl; 447 } 448 449 static int blkcg_reset_stats(struct cgroup_subsys_state *css, 450 struct cftype *cftype, u64 val) 451 { 452 struct blkcg *blkcg = css_to_blkcg(css); 453 struct blkcg_gq *blkg; 454 int i; 455 456 /* 457 * XXX: We invoke cgroup_add/rm_cftypes() under blkcg_pol_mutex 458 * which ends up putting cgroup's internal cgroup_tree_mutex under 459 * it; however, cgroup_tree_mutex is nested above cgroup file 460 * active protection and grabbing blkcg_pol_mutex from a cgroup 461 * file operation creates a possible circular dependency. cgroup 462 * internal locking is planned to go through further simplification 463 * and this issue should go away soon. For now, let's trylock 464 * blkcg_pol_mutex and restart the write on failure. 465 * 466 * http://lkml.kernel.org/g/5363C04B.4010400@oracle.com 467 */ 468 if (!mutex_trylock(&blkcg_pol_mutex)) 469 return restart_syscall(); 470 spin_lock_irq(&blkcg->lock); 471 472 /* 473 * Note that stat reset is racy - it doesn't synchronize against 474 * stat updates. This is a debug feature which shouldn't exist 475 * anyway. If you get hit by a race, retry. 476 */ 477 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { 478 for (i = 0; i < BLKCG_MAX_POLS; i++) { 479 struct blkcg_policy *pol = blkcg_policy[i]; 480 481 if (blkcg_policy_enabled(blkg->q, pol) && 482 pol->pd_reset_stats_fn) 483 pol->pd_reset_stats_fn(blkg); 484 } 485 } 486 487 spin_unlock_irq(&blkcg->lock); 488 mutex_unlock(&blkcg_pol_mutex); 489 return 0; 490 } 491 492 static const char *blkg_dev_name(struct blkcg_gq *blkg) 493 { 494 /* some drivers (floppy) instantiate a queue w/o disk registered */ 495 if (blkg->q->backing_dev_info.dev) 496 return dev_name(blkg->q->backing_dev_info.dev); 497 return NULL; 498 } 499 500 /** 501 * blkcg_print_blkgs - helper for printing per-blkg data 502 * @sf: seq_file to print to 503 * @blkcg: blkcg of interest 504 * @prfill: fill function to print out a blkg 505 * @pol: policy in question 506 * @data: data to be passed to @prfill 507 * @show_total: to print out sum of prfill return values or not 508 * 509 * This function invokes @prfill on each blkg of @blkcg if pd for the 510 * policy specified by @pol exists. @prfill is invoked with @sf, the 511 * policy data and @data and the matching queue lock held. If @show_total 512 * is %true, the sum of the return values from @prfill is printed with 513 * "Total" label at the end. 514 * 515 * This is to be used to construct print functions for 516 * cftype->read_seq_string method. 517 */ 518 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, 519 u64 (*prfill)(struct seq_file *, 520 struct blkg_policy_data *, int), 521 const struct blkcg_policy *pol, int data, 522 bool show_total) 523 { 524 struct blkcg_gq *blkg; 525 u64 total = 0; 526 527 rcu_read_lock(); 528 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { 529 spin_lock_irq(blkg->q->queue_lock); 530 if (blkcg_policy_enabled(blkg->q, pol)) 531 total += prfill(sf, blkg->pd[pol->plid], data); 532 spin_unlock_irq(blkg->q->queue_lock); 533 } 534 rcu_read_unlock(); 535 536 if (show_total) 537 seq_printf(sf, "Total %llu\n", (unsigned long long)total); 538 } 539 EXPORT_SYMBOL_GPL(blkcg_print_blkgs); 540 541 /** 542 * __blkg_prfill_u64 - prfill helper for a single u64 value 543 * @sf: seq_file to print to 544 * @pd: policy private data of interest 545 * @v: value to print 546 * 547 * Print @v to @sf for the device assocaited with @pd. 548 */ 549 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v) 550 { 551 const char *dname = blkg_dev_name(pd->blkg); 552 553 if (!dname) 554 return 0; 555 556 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v); 557 return v; 558 } 559 EXPORT_SYMBOL_GPL(__blkg_prfill_u64); 560 561 /** 562 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat 563 * @sf: seq_file to print to 564 * @pd: policy private data of interest 565 * @rwstat: rwstat to print 566 * 567 * Print @rwstat to @sf for the device assocaited with @pd. 568 */ 569 u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, 570 const struct blkg_rwstat *rwstat) 571 { 572 static const char *rwstr[] = { 573 [BLKG_RWSTAT_READ] = "Read", 574 [BLKG_RWSTAT_WRITE] = "Write", 575 [BLKG_RWSTAT_SYNC] = "Sync", 576 [BLKG_RWSTAT_ASYNC] = "Async", 577 }; 578 const char *dname = blkg_dev_name(pd->blkg); 579 u64 v; 580 int i; 581 582 if (!dname) 583 return 0; 584 585 for (i = 0; i < BLKG_RWSTAT_NR; i++) 586 seq_printf(sf, "%s %s %llu\n", dname, rwstr[i], 587 (unsigned long long)rwstat->cnt[i]); 588 589 v = rwstat->cnt[BLKG_RWSTAT_READ] + rwstat->cnt[BLKG_RWSTAT_WRITE]; 590 seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v); 591 return v; 592 } 593 EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat); 594 595 /** 596 * blkg_prfill_stat - prfill callback for blkg_stat 597 * @sf: seq_file to print to 598 * @pd: policy private data of interest 599 * @off: offset to the blkg_stat in @pd 600 * 601 * prfill callback for printing a blkg_stat. 602 */ 603 u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off) 604 { 605 return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off)); 606 } 607 EXPORT_SYMBOL_GPL(blkg_prfill_stat); 608 609 /** 610 * blkg_prfill_rwstat - prfill callback for blkg_rwstat 611 * @sf: seq_file to print to 612 * @pd: policy private data of interest 613 * @off: offset to the blkg_rwstat in @pd 614 * 615 * prfill callback for printing a blkg_rwstat. 616 */ 617 u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, 618 int off) 619 { 620 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off); 621 622 return __blkg_prfill_rwstat(sf, pd, &rwstat); 623 } 624 EXPORT_SYMBOL_GPL(blkg_prfill_rwstat); 625 626 /** 627 * blkg_stat_recursive_sum - collect hierarchical blkg_stat 628 * @pd: policy private data of interest 629 * @off: offset to the blkg_stat in @pd 630 * 631 * Collect the blkg_stat specified by @off from @pd and all its online 632 * descendants and return the sum. The caller must be holding the queue 633 * lock for online tests. 634 */ 635 u64 blkg_stat_recursive_sum(struct blkg_policy_data *pd, int off) 636 { 637 struct blkcg_policy *pol = blkcg_policy[pd->plid]; 638 struct blkcg_gq *pos_blkg; 639 struct cgroup_subsys_state *pos_css; 640 u64 sum = 0; 641 642 lockdep_assert_held(pd->blkg->q->queue_lock); 643 644 rcu_read_lock(); 645 blkg_for_each_descendant_pre(pos_blkg, pos_css, pd_to_blkg(pd)) { 646 struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol); 647 struct blkg_stat *stat = (void *)pos_pd + off; 648 649 if (pos_blkg->online) 650 sum += blkg_stat_read(stat); 651 } 652 rcu_read_unlock(); 653 654 return sum; 655 } 656 EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum); 657 658 /** 659 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat 660 * @pd: policy private data of interest 661 * @off: offset to the blkg_stat in @pd 662 * 663 * Collect the blkg_rwstat specified by @off from @pd and all its online 664 * descendants and return the sum. The caller must be holding the queue 665 * lock for online tests. 666 */ 667 struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkg_policy_data *pd, 668 int off) 669 { 670 struct blkcg_policy *pol = blkcg_policy[pd->plid]; 671 struct blkcg_gq *pos_blkg; 672 struct cgroup_subsys_state *pos_css; 673 struct blkg_rwstat sum = { }; 674 int i; 675 676 lockdep_assert_held(pd->blkg->q->queue_lock); 677 678 rcu_read_lock(); 679 blkg_for_each_descendant_pre(pos_blkg, pos_css, pd_to_blkg(pd)) { 680 struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol); 681 struct blkg_rwstat *rwstat = (void *)pos_pd + off; 682 struct blkg_rwstat tmp; 683 684 if (!pos_blkg->online) 685 continue; 686 687 tmp = blkg_rwstat_read(rwstat); 688 689 for (i = 0; i < BLKG_RWSTAT_NR; i++) 690 sum.cnt[i] += tmp.cnt[i]; 691 } 692 rcu_read_unlock(); 693 694 return sum; 695 } 696 EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum); 697 698 /** 699 * blkg_conf_prep - parse and prepare for per-blkg config update 700 * @blkcg: target block cgroup 701 * @pol: target policy 702 * @input: input string 703 * @ctx: blkg_conf_ctx to be filled 704 * 705 * Parse per-blkg config update from @input and initialize @ctx with the 706 * result. @ctx->blkg points to the blkg to be updated and @ctx->v the new 707 * value. This function returns with RCU read lock and queue lock held and 708 * must be paired with blkg_conf_finish(). 709 */ 710 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, 711 const char *input, struct blkg_conf_ctx *ctx) 712 __acquires(rcu) __acquires(disk->queue->queue_lock) 713 { 714 struct gendisk *disk; 715 struct blkcg_gq *blkg; 716 unsigned int major, minor; 717 unsigned long long v; 718 int part, ret; 719 720 if (sscanf(input, "%u:%u %llu", &major, &minor, &v) != 3) 721 return -EINVAL; 722 723 disk = get_gendisk(MKDEV(major, minor), &part); 724 if (!disk || part) 725 return -EINVAL; 726 727 rcu_read_lock(); 728 spin_lock_irq(disk->queue->queue_lock); 729 730 if (blkcg_policy_enabled(disk->queue, pol)) 731 blkg = blkg_lookup_create(blkcg, disk->queue); 732 else 733 blkg = ERR_PTR(-EINVAL); 734 735 if (IS_ERR(blkg)) { 736 ret = PTR_ERR(blkg); 737 rcu_read_unlock(); 738 spin_unlock_irq(disk->queue->queue_lock); 739 put_disk(disk); 740 /* 741 * If queue was bypassing, we should retry. Do so after a 742 * short msleep(). It isn't strictly necessary but queue 743 * can be bypassing for some time and it's always nice to 744 * avoid busy looping. 745 */ 746 if (ret == -EBUSY) { 747 msleep(10); 748 ret = restart_syscall(); 749 } 750 return ret; 751 } 752 753 ctx->disk = disk; 754 ctx->blkg = blkg; 755 ctx->v = v; 756 return 0; 757 } 758 EXPORT_SYMBOL_GPL(blkg_conf_prep); 759 760 /** 761 * blkg_conf_finish - finish up per-blkg config update 762 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep() 763 * 764 * Finish up after per-blkg config update. This function must be paired 765 * with blkg_conf_prep(). 766 */ 767 void blkg_conf_finish(struct blkg_conf_ctx *ctx) 768 __releases(ctx->disk->queue->queue_lock) __releases(rcu) 769 { 770 spin_unlock_irq(ctx->disk->queue->queue_lock); 771 rcu_read_unlock(); 772 put_disk(ctx->disk); 773 } 774 EXPORT_SYMBOL_GPL(blkg_conf_finish); 775 776 struct cftype blkcg_files[] = { 777 { 778 .name = "reset_stats", 779 .write_u64 = blkcg_reset_stats, 780 }, 781 { } /* terminate */ 782 }; 783 784 /** 785 * blkcg_css_offline - cgroup css_offline callback 786 * @css: css of interest 787 * 788 * This function is called when @css is about to go away and responsible 789 * for shooting down all blkgs associated with @css. blkgs should be 790 * removed while holding both q and blkcg locks. As blkcg lock is nested 791 * inside q lock, this function performs reverse double lock dancing. 792 * 793 * This is the blkcg counterpart of ioc_release_fn(). 794 */ 795 static void blkcg_css_offline(struct cgroup_subsys_state *css) 796 { 797 struct blkcg *blkcg = css_to_blkcg(css); 798 799 spin_lock_irq(&blkcg->lock); 800 801 while (!hlist_empty(&blkcg->blkg_list)) { 802 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, 803 struct blkcg_gq, blkcg_node); 804 struct request_queue *q = blkg->q; 805 806 if (spin_trylock(q->queue_lock)) { 807 blkg_destroy(blkg); 808 spin_unlock(q->queue_lock); 809 } else { 810 spin_unlock_irq(&blkcg->lock); 811 cpu_relax(); 812 spin_lock_irq(&blkcg->lock); 813 } 814 } 815 816 spin_unlock_irq(&blkcg->lock); 817 818 wb_blkcg_offline(blkcg); 819 } 820 821 static void blkcg_css_free(struct cgroup_subsys_state *css) 822 { 823 struct blkcg *blkcg = css_to_blkcg(css); 824 825 if (blkcg != &blkcg_root) 826 kfree(blkcg); 827 } 828 829 static struct cgroup_subsys_state * 830 blkcg_css_alloc(struct cgroup_subsys_state *parent_css) 831 { 832 struct blkcg *blkcg; 833 struct cgroup_subsys_state *ret; 834 int i; 835 836 if (!parent_css) { 837 blkcg = &blkcg_root; 838 goto done; 839 } 840 841 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); 842 if (!blkcg) { 843 ret = ERR_PTR(-ENOMEM); 844 goto free_blkcg; 845 } 846 847 for (i = 0; i < BLKCG_MAX_POLS ; i++) { 848 struct blkcg_policy *pol = blkcg_policy[i]; 849 struct blkcg_policy_data *cpd; 850 851 /* 852 * If the policy hasn't been attached yet, wait for it 853 * to be attached before doing anything else. Otherwise, 854 * check if the policy requires any specific per-cgroup 855 * data: if it does, allocate and initialize it. 856 */ 857 if (!pol || !pol->cpd_size) 858 continue; 859 860 BUG_ON(blkcg->pd[i]); 861 cpd = kzalloc(pol->cpd_size, GFP_KERNEL); 862 if (!cpd) { 863 ret = ERR_PTR(-ENOMEM); 864 goto free_pd_blkcg; 865 } 866 blkcg->pd[i] = cpd; 867 cpd->plid = i; 868 pol->cpd_init_fn(blkcg); 869 } 870 871 done: 872 spin_lock_init(&blkcg->lock); 873 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC); 874 INIT_HLIST_HEAD(&blkcg->blkg_list); 875 #ifdef CONFIG_CGROUP_WRITEBACK 876 INIT_LIST_HEAD(&blkcg->cgwb_list); 877 #endif 878 return &blkcg->css; 879 880 free_pd_blkcg: 881 for (i--; i >= 0; i--) 882 kfree(blkcg->pd[i]); 883 884 free_blkcg: 885 kfree(blkcg); 886 return ret; 887 } 888 889 /** 890 * blkcg_init_queue - initialize blkcg part of request queue 891 * @q: request_queue to initialize 892 * 893 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg 894 * part of new request_queue @q. 895 * 896 * RETURNS: 897 * 0 on success, -errno on failure. 898 */ 899 int blkcg_init_queue(struct request_queue *q) 900 { 901 struct blkcg_gq *new_blkg, *blkg; 902 bool preloaded; 903 int ret; 904 905 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL); 906 if (!new_blkg) 907 return -ENOMEM; 908 909 preloaded = !radix_tree_preload(GFP_KERNEL); 910 911 /* 912 * Make sure the root blkg exists and count the existing blkgs. As 913 * @q is bypassing at this point, blkg_lookup_create() can't be 914 * used. Open code insertion. 915 */ 916 rcu_read_lock(); 917 spin_lock_irq(q->queue_lock); 918 blkg = blkg_create(&blkcg_root, q, new_blkg); 919 spin_unlock_irq(q->queue_lock); 920 rcu_read_unlock(); 921 922 if (preloaded) 923 radix_tree_preload_end(); 924 925 if (IS_ERR(blkg)) { 926 kfree(new_blkg); 927 return PTR_ERR(blkg); 928 } 929 930 q->root_blkg = blkg; 931 q->root_rl.blkg = blkg; 932 933 ret = blk_throtl_init(q); 934 if (ret) { 935 spin_lock_irq(q->queue_lock); 936 blkg_destroy_all(q); 937 spin_unlock_irq(q->queue_lock); 938 } 939 return ret; 940 } 941 942 /** 943 * blkcg_drain_queue - drain blkcg part of request_queue 944 * @q: request_queue to drain 945 * 946 * Called from blk_drain_queue(). Responsible for draining blkcg part. 947 */ 948 void blkcg_drain_queue(struct request_queue *q) 949 { 950 lockdep_assert_held(q->queue_lock); 951 952 /* 953 * @q could be exiting and already have destroyed all blkgs as 954 * indicated by NULL root_blkg. If so, don't confuse policies. 955 */ 956 if (!q->root_blkg) 957 return; 958 959 blk_throtl_drain(q); 960 } 961 962 /** 963 * blkcg_exit_queue - exit and release blkcg part of request_queue 964 * @q: request_queue being released 965 * 966 * Called from blk_release_queue(). Responsible for exiting blkcg part. 967 */ 968 void blkcg_exit_queue(struct request_queue *q) 969 { 970 spin_lock_irq(q->queue_lock); 971 blkg_destroy_all(q); 972 spin_unlock_irq(q->queue_lock); 973 974 blk_throtl_exit(q); 975 } 976 977 /* 978 * We cannot support shared io contexts, as we have no mean to support 979 * two tasks with the same ioc in two different groups without major rework 980 * of the main cic data structures. For now we allow a task to change 981 * its cgroup only if it's the only owner of its ioc. 982 */ 983 static int blkcg_can_attach(struct cgroup_subsys_state *css, 984 struct cgroup_taskset *tset) 985 { 986 struct task_struct *task; 987 struct io_context *ioc; 988 int ret = 0; 989 990 /* task_lock() is needed to avoid races with exit_io_context() */ 991 cgroup_taskset_for_each(task, tset) { 992 task_lock(task); 993 ioc = task->io_context; 994 if (ioc && atomic_read(&ioc->nr_tasks) > 1) 995 ret = -EINVAL; 996 task_unlock(task); 997 if (ret) 998 break; 999 } 1000 return ret; 1001 } 1002 1003 struct cgroup_subsys blkio_cgrp_subsys = { 1004 .css_alloc = blkcg_css_alloc, 1005 .css_offline = blkcg_css_offline, 1006 .css_free = blkcg_css_free, 1007 .can_attach = blkcg_can_attach, 1008 .legacy_cftypes = blkcg_files, 1009 #ifdef CONFIG_MEMCG 1010 /* 1011 * This ensures that, if available, memcg is automatically enabled 1012 * together on the default hierarchy so that the owner cgroup can 1013 * be retrieved from writeback pages. 1014 */ 1015 .depends_on = 1 << memory_cgrp_id, 1016 #endif 1017 }; 1018 EXPORT_SYMBOL_GPL(blkio_cgrp_subsys); 1019 1020 /** 1021 * blkcg_activate_policy - activate a blkcg policy on a request_queue 1022 * @q: request_queue of interest 1023 * @pol: blkcg policy to activate 1024 * 1025 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through 1026 * bypass mode to populate its blkgs with policy_data for @pol. 1027 * 1028 * Activation happens with @q bypassed, so nobody would be accessing blkgs 1029 * from IO path. Update of each blkg is protected by both queue and blkcg 1030 * locks so that holding either lock and testing blkcg_policy_enabled() is 1031 * always enough for dereferencing policy data. 1032 * 1033 * The caller is responsible for synchronizing [de]activations and policy 1034 * [un]registerations. Returns 0 on success, -errno on failure. 1035 */ 1036 int blkcg_activate_policy(struct request_queue *q, 1037 const struct blkcg_policy *pol) 1038 { 1039 LIST_HEAD(pds); 1040 LIST_HEAD(cpds); 1041 struct blkcg_gq *blkg; 1042 struct blkg_policy_data *pd, *nd; 1043 struct blkcg_policy_data *cpd, *cnd; 1044 int cnt = 0, ret; 1045 1046 if (blkcg_policy_enabled(q, pol)) 1047 return 0; 1048 1049 /* count and allocate policy_data for all existing blkgs */ 1050 blk_queue_bypass_start(q); 1051 spin_lock_irq(q->queue_lock); 1052 list_for_each_entry(blkg, &q->blkg_list, q_node) 1053 cnt++; 1054 spin_unlock_irq(q->queue_lock); 1055 1056 /* 1057 * Allocate per-blkg and per-blkcg policy data 1058 * for all existing blkgs. 1059 */ 1060 while (cnt--) { 1061 pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node); 1062 if (!pd) { 1063 ret = -ENOMEM; 1064 goto out_free; 1065 } 1066 list_add_tail(&pd->alloc_node, &pds); 1067 1068 if (!pol->cpd_size) 1069 continue; 1070 cpd = kzalloc_node(pol->cpd_size, GFP_KERNEL, q->node); 1071 if (!cpd) { 1072 ret = -ENOMEM; 1073 goto out_free; 1074 } 1075 list_add_tail(&cpd->alloc_node, &cpds); 1076 } 1077 1078 /* 1079 * Install the allocated pds and cpds. With @q bypassing, no new blkg 1080 * should have been created while the queue lock was dropped. 1081 */ 1082 spin_lock_irq(q->queue_lock); 1083 1084 list_for_each_entry(blkg, &q->blkg_list, q_node) { 1085 if (WARN_ON(list_empty(&pds)) || 1086 WARN_ON(pol->cpd_size && list_empty(&cpds))) { 1087 /* umm... this shouldn't happen, just abort */ 1088 ret = -ENOMEM; 1089 goto out_unlock; 1090 } 1091 cpd = list_first_entry(&cpds, struct blkcg_policy_data, 1092 alloc_node); 1093 list_del_init(&cpd->alloc_node); 1094 pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node); 1095 list_del_init(&pd->alloc_node); 1096 1097 /* grab blkcg lock too while installing @pd on @blkg */ 1098 spin_lock(&blkg->blkcg->lock); 1099 1100 if (!pol->cpd_size) 1101 goto no_cpd; 1102 if (!blkg->blkcg->pd[pol->plid]) { 1103 /* Per-policy per-blkcg data */ 1104 blkg->blkcg->pd[pol->plid] = cpd; 1105 cpd->plid = pol->plid; 1106 pol->cpd_init_fn(blkg->blkcg); 1107 } else { /* must free it as it has already been extracted */ 1108 kfree(cpd); 1109 } 1110 no_cpd: 1111 blkg->pd[pol->plid] = pd; 1112 pd->blkg = blkg; 1113 pd->plid = pol->plid; 1114 pol->pd_init_fn(blkg); 1115 1116 spin_unlock(&blkg->blkcg->lock); 1117 } 1118 1119 __set_bit(pol->plid, q->blkcg_pols); 1120 ret = 0; 1121 out_unlock: 1122 spin_unlock_irq(q->queue_lock); 1123 out_free: 1124 blk_queue_bypass_end(q); 1125 list_for_each_entry_safe(pd, nd, &pds, alloc_node) 1126 kfree(pd); 1127 list_for_each_entry_safe(cpd, cnd, &cpds, alloc_node) 1128 kfree(cpd); 1129 return ret; 1130 } 1131 EXPORT_SYMBOL_GPL(blkcg_activate_policy); 1132 1133 /** 1134 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue 1135 * @q: request_queue of interest 1136 * @pol: blkcg policy to deactivate 1137 * 1138 * Deactivate @pol on @q. Follows the same synchronization rules as 1139 * blkcg_activate_policy(). 1140 */ 1141 void blkcg_deactivate_policy(struct request_queue *q, 1142 const struct blkcg_policy *pol) 1143 { 1144 struct blkcg_gq *blkg; 1145 1146 if (!blkcg_policy_enabled(q, pol)) 1147 return; 1148 1149 blk_queue_bypass_start(q); 1150 spin_lock_irq(q->queue_lock); 1151 1152 __clear_bit(pol->plid, q->blkcg_pols); 1153 1154 list_for_each_entry(blkg, &q->blkg_list, q_node) { 1155 /* grab blkcg lock too while removing @pd from @blkg */ 1156 spin_lock(&blkg->blkcg->lock); 1157 1158 if (pol->pd_offline_fn) 1159 pol->pd_offline_fn(blkg); 1160 if (pol->pd_exit_fn) 1161 pol->pd_exit_fn(blkg); 1162 1163 kfree(blkg->pd[pol->plid]); 1164 blkg->pd[pol->plid] = NULL; 1165 kfree(blkg->blkcg->pd[pol->plid]); 1166 blkg->blkcg->pd[pol->plid] = NULL; 1167 1168 spin_unlock(&blkg->blkcg->lock); 1169 } 1170 1171 spin_unlock_irq(q->queue_lock); 1172 blk_queue_bypass_end(q); 1173 } 1174 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy); 1175 1176 /** 1177 * blkcg_policy_register - register a blkcg policy 1178 * @pol: blkcg policy to register 1179 * 1180 * Register @pol with blkcg core. Might sleep and @pol may be modified on 1181 * successful registration. Returns 0 on success and -errno on failure. 1182 */ 1183 int blkcg_policy_register(struct blkcg_policy *pol) 1184 { 1185 int i, ret; 1186 1187 if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data))) 1188 return -EINVAL; 1189 1190 mutex_lock(&blkcg_pol_mutex); 1191 1192 /* find an empty slot */ 1193 ret = -ENOSPC; 1194 for (i = 0; i < BLKCG_MAX_POLS; i++) 1195 if (!blkcg_policy[i]) 1196 break; 1197 if (i >= BLKCG_MAX_POLS) 1198 goto out_unlock; 1199 1200 /* register and update blkgs */ 1201 pol->plid = i; 1202 blkcg_policy[i] = pol; 1203 1204 /* everything is in place, add intf files for the new policy */ 1205 if (pol->cftypes) 1206 WARN_ON(cgroup_add_legacy_cftypes(&blkio_cgrp_subsys, 1207 pol->cftypes)); 1208 ret = 0; 1209 out_unlock: 1210 mutex_unlock(&blkcg_pol_mutex); 1211 return ret; 1212 } 1213 EXPORT_SYMBOL_GPL(blkcg_policy_register); 1214 1215 /** 1216 * blkcg_policy_unregister - unregister a blkcg policy 1217 * @pol: blkcg policy to unregister 1218 * 1219 * Undo blkcg_policy_register(@pol). Might sleep. 1220 */ 1221 void blkcg_policy_unregister(struct blkcg_policy *pol) 1222 { 1223 mutex_lock(&blkcg_pol_mutex); 1224 1225 if (WARN_ON(blkcg_policy[pol->plid] != pol)) 1226 goto out_unlock; 1227 1228 /* kill the intf files first */ 1229 if (pol->cftypes) 1230 cgroup_rm_cftypes(pol->cftypes); 1231 1232 /* unregister and update blkgs */ 1233 blkcg_policy[pol->plid] = NULL; 1234 out_unlock: 1235 mutex_unlock(&blkcg_pol_mutex); 1236 } 1237 EXPORT_SYMBOL_GPL(blkcg_policy_unregister); 1238