xref: /openbmc/linux/block/blk-cgroup.c (revision 2bdd5238)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Common Block IO controller cgroup interface
4  *
5  * Based on ideas and code from CFQ, CFS and BFQ:
6  * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
7  *
8  * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
9  *		      Paolo Valente <paolo.valente@unimore.it>
10  *
11  * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
12  * 	              Nauman Rafique <nauman@google.com>
13  *
14  * For policy-specific per-blkcg data:
15  * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
16  *                    Arianna Avanzini <avanzini.arianna@gmail.com>
17  */
18 #include <linux/ioprio.h>
19 #include <linux/kdev_t.h>
20 #include <linux/module.h>
21 #include <linux/sched/signal.h>
22 #include <linux/err.h>
23 #include <linux/blkdev.h>
24 #include <linux/backing-dev.h>
25 #include <linux/slab.h>
26 #include <linux/genhd.h>
27 #include <linux/delay.h>
28 #include <linux/atomic.h>
29 #include <linux/ctype.h>
30 #include <linux/blk-cgroup.h>
31 #include <linux/tracehook.h>
32 #include <linux/psi.h>
33 #include "blk.h"
34 #include "blk-ioprio.h"
35 
36 /*
37  * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
38  * blkcg_pol_register_mutex nests outside of it and synchronizes entire
39  * policy [un]register operations including cgroup file additions /
40  * removals.  Putting cgroup file registration outside blkcg_pol_mutex
41  * allows grabbing it from cgroup callbacks.
42  */
43 static DEFINE_MUTEX(blkcg_pol_register_mutex);
44 static DEFINE_MUTEX(blkcg_pol_mutex);
45 
46 struct blkcg blkcg_root;
47 EXPORT_SYMBOL_GPL(blkcg_root);
48 
49 struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
50 EXPORT_SYMBOL_GPL(blkcg_root_css);
51 
52 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
53 
54 static LIST_HEAD(all_blkcgs);		/* protected by blkcg_pol_mutex */
55 
56 bool blkcg_debug_stats = false;
57 static struct workqueue_struct *blkcg_punt_bio_wq;
58 
59 #define BLKG_DESTROY_BATCH_SIZE  64
60 
61 static bool blkcg_policy_enabled(struct request_queue *q,
62 				 const struct blkcg_policy *pol)
63 {
64 	return pol && test_bit(pol->plid, q->blkcg_pols);
65 }
66 
67 /**
68  * blkg_free - free a blkg
69  * @blkg: blkg to free
70  *
71  * Free @blkg which may be partially allocated.
72  */
73 static void blkg_free(struct blkcg_gq *blkg)
74 {
75 	int i;
76 
77 	if (!blkg)
78 		return;
79 
80 	for (i = 0; i < BLKCG_MAX_POLS; i++)
81 		if (blkg->pd[i])
82 			blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
83 
84 	free_percpu(blkg->iostat_cpu);
85 	percpu_ref_exit(&blkg->refcnt);
86 	kfree(blkg);
87 }
88 
89 static void __blkg_release(struct rcu_head *rcu)
90 {
91 	struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
92 
93 	WARN_ON(!bio_list_empty(&blkg->async_bios));
94 
95 	/* release the blkcg and parent blkg refs this blkg has been holding */
96 	css_put(&blkg->blkcg->css);
97 	if (blkg->parent)
98 		blkg_put(blkg->parent);
99 	blkg_free(blkg);
100 }
101 
102 /*
103  * A group is RCU protected, but having an rcu lock does not mean that one
104  * can access all the fields of blkg and assume these are valid.  For
105  * example, don't try to follow throtl_data and request queue links.
106  *
107  * Having a reference to blkg under an rcu allows accesses to only values
108  * local to groups like group stats and group rate limits.
109  */
110 static void blkg_release(struct percpu_ref *ref)
111 {
112 	struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
113 
114 	call_rcu(&blkg->rcu_head, __blkg_release);
115 }
116 
117 static void blkg_async_bio_workfn(struct work_struct *work)
118 {
119 	struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
120 					     async_bio_work);
121 	struct bio_list bios = BIO_EMPTY_LIST;
122 	struct bio *bio;
123 	struct blk_plug plug;
124 	bool need_plug = false;
125 
126 	/* as long as there are pending bios, @blkg can't go away */
127 	spin_lock_bh(&blkg->async_bio_lock);
128 	bio_list_merge(&bios, &blkg->async_bios);
129 	bio_list_init(&blkg->async_bios);
130 	spin_unlock_bh(&blkg->async_bio_lock);
131 
132 	/* start plug only when bio_list contains at least 2 bios */
133 	if (bios.head && bios.head->bi_next) {
134 		need_plug = true;
135 		blk_start_plug(&plug);
136 	}
137 	while ((bio = bio_list_pop(&bios)))
138 		submit_bio(bio);
139 	if (need_plug)
140 		blk_finish_plug(&plug);
141 }
142 
143 /**
144  * blkg_alloc - allocate a blkg
145  * @blkcg: block cgroup the new blkg is associated with
146  * @q: request_queue the new blkg is associated with
147  * @gfp_mask: allocation mask to use
148  *
149  * Allocate a new blkg assocating @blkcg and @q.
150  */
151 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
152 				   gfp_t gfp_mask)
153 {
154 	struct blkcg_gq *blkg;
155 	int i, cpu;
156 
157 	/* alloc and init base part */
158 	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
159 	if (!blkg)
160 		return NULL;
161 
162 	if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
163 		goto err_free;
164 
165 	blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
166 	if (!blkg->iostat_cpu)
167 		goto err_free;
168 
169 	blkg->q = q;
170 	INIT_LIST_HEAD(&blkg->q_node);
171 	spin_lock_init(&blkg->async_bio_lock);
172 	bio_list_init(&blkg->async_bios);
173 	INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
174 	blkg->blkcg = blkcg;
175 
176 	u64_stats_init(&blkg->iostat.sync);
177 	for_each_possible_cpu(cpu)
178 		u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
179 
180 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
181 		struct blkcg_policy *pol = blkcg_policy[i];
182 		struct blkg_policy_data *pd;
183 
184 		if (!blkcg_policy_enabled(q, pol))
185 			continue;
186 
187 		/* alloc per-policy data and attach it to blkg */
188 		pd = pol->pd_alloc_fn(gfp_mask, q, blkcg);
189 		if (!pd)
190 			goto err_free;
191 
192 		blkg->pd[i] = pd;
193 		pd->blkg = blkg;
194 		pd->plid = i;
195 	}
196 
197 	return blkg;
198 
199 err_free:
200 	blkg_free(blkg);
201 	return NULL;
202 }
203 
204 struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
205 				      struct request_queue *q, bool update_hint)
206 {
207 	struct blkcg_gq *blkg;
208 
209 	/*
210 	 * Hint didn't match.  Look up from the radix tree.  Note that the
211 	 * hint can only be updated under queue_lock as otherwise @blkg
212 	 * could have already been removed from blkg_tree.  The caller is
213 	 * responsible for grabbing queue_lock if @update_hint.
214 	 */
215 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
216 	if (blkg && blkg->q == q) {
217 		if (update_hint) {
218 			lockdep_assert_held(&q->queue_lock);
219 			rcu_assign_pointer(blkcg->blkg_hint, blkg);
220 		}
221 		return blkg;
222 	}
223 
224 	return NULL;
225 }
226 EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
227 
228 /*
229  * If @new_blkg is %NULL, this function tries to allocate a new one as
230  * necessary using %GFP_NOWAIT.  @new_blkg is always consumed on return.
231  */
232 static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
233 				    struct request_queue *q,
234 				    struct blkcg_gq *new_blkg)
235 {
236 	struct blkcg_gq *blkg;
237 	int i, ret;
238 
239 	WARN_ON_ONCE(!rcu_read_lock_held());
240 	lockdep_assert_held(&q->queue_lock);
241 
242 	/* request_queue is dying, do not create/recreate a blkg */
243 	if (blk_queue_dying(q)) {
244 		ret = -ENODEV;
245 		goto err_free_blkg;
246 	}
247 
248 	/* blkg holds a reference to blkcg */
249 	if (!css_tryget_online(&blkcg->css)) {
250 		ret = -ENODEV;
251 		goto err_free_blkg;
252 	}
253 
254 	/* allocate */
255 	if (!new_blkg) {
256 		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
257 		if (unlikely(!new_blkg)) {
258 			ret = -ENOMEM;
259 			goto err_put_css;
260 		}
261 	}
262 	blkg = new_blkg;
263 
264 	/* link parent */
265 	if (blkcg_parent(blkcg)) {
266 		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
267 		if (WARN_ON_ONCE(!blkg->parent)) {
268 			ret = -ENODEV;
269 			goto err_put_css;
270 		}
271 		blkg_get(blkg->parent);
272 	}
273 
274 	/* invoke per-policy init */
275 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
276 		struct blkcg_policy *pol = blkcg_policy[i];
277 
278 		if (blkg->pd[i] && pol->pd_init_fn)
279 			pol->pd_init_fn(blkg->pd[i]);
280 	}
281 
282 	/* insert */
283 	spin_lock(&blkcg->lock);
284 	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
285 	if (likely(!ret)) {
286 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
287 		list_add(&blkg->q_node, &q->blkg_list);
288 
289 		for (i = 0; i < BLKCG_MAX_POLS; i++) {
290 			struct blkcg_policy *pol = blkcg_policy[i];
291 
292 			if (blkg->pd[i] && pol->pd_online_fn)
293 				pol->pd_online_fn(blkg->pd[i]);
294 		}
295 	}
296 	blkg->online = true;
297 	spin_unlock(&blkcg->lock);
298 
299 	if (!ret)
300 		return blkg;
301 
302 	/* @blkg failed fully initialized, use the usual release path */
303 	blkg_put(blkg);
304 	return ERR_PTR(ret);
305 
306 err_put_css:
307 	css_put(&blkcg->css);
308 err_free_blkg:
309 	blkg_free(new_blkg);
310 	return ERR_PTR(ret);
311 }
312 
313 /**
314  * blkg_lookup_create - lookup blkg, try to create one if not there
315  * @blkcg: blkcg of interest
316  * @q: request_queue of interest
317  *
318  * Lookup blkg for the @blkcg - @q pair.  If it doesn't exist, try to
319  * create one.  blkg creation is performed recursively from blkcg_root such
320  * that all non-root blkg's have access to the parent blkg.  This function
321  * should be called under RCU read lock and takes @q->queue_lock.
322  *
323  * Returns the blkg or the closest blkg if blkg_create() fails as it walks
324  * down from root.
325  */
326 static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
327 		struct request_queue *q)
328 {
329 	struct blkcg_gq *blkg;
330 	unsigned long flags;
331 
332 	WARN_ON_ONCE(!rcu_read_lock_held());
333 
334 	blkg = blkg_lookup(blkcg, q);
335 	if (blkg)
336 		return blkg;
337 
338 	spin_lock_irqsave(&q->queue_lock, flags);
339 	blkg = __blkg_lookup(blkcg, q, true);
340 	if (blkg)
341 		goto found;
342 
343 	/*
344 	 * Create blkgs walking down from blkcg_root to @blkcg, so that all
345 	 * non-root blkgs have access to their parents.  Returns the closest
346 	 * blkg to the intended blkg should blkg_create() fail.
347 	 */
348 	while (true) {
349 		struct blkcg *pos = blkcg;
350 		struct blkcg *parent = blkcg_parent(blkcg);
351 		struct blkcg_gq *ret_blkg = q->root_blkg;
352 
353 		while (parent) {
354 			blkg = __blkg_lookup(parent, q, false);
355 			if (blkg) {
356 				/* remember closest blkg */
357 				ret_blkg = blkg;
358 				break;
359 			}
360 			pos = parent;
361 			parent = blkcg_parent(parent);
362 		}
363 
364 		blkg = blkg_create(pos, q, NULL);
365 		if (IS_ERR(blkg)) {
366 			blkg = ret_blkg;
367 			break;
368 		}
369 		if (pos == blkcg)
370 			break;
371 	}
372 
373 found:
374 	spin_unlock_irqrestore(&q->queue_lock, flags);
375 	return blkg;
376 }
377 
378 static void blkg_destroy(struct blkcg_gq *blkg)
379 {
380 	struct blkcg *blkcg = blkg->blkcg;
381 	int i;
382 
383 	lockdep_assert_held(&blkg->q->queue_lock);
384 	lockdep_assert_held(&blkcg->lock);
385 
386 	/* Something wrong if we are trying to remove same group twice */
387 	WARN_ON_ONCE(list_empty(&blkg->q_node));
388 	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
389 
390 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
391 		struct blkcg_policy *pol = blkcg_policy[i];
392 
393 		if (blkg->pd[i] && pol->pd_offline_fn)
394 			pol->pd_offline_fn(blkg->pd[i]);
395 	}
396 
397 	blkg->online = false;
398 
399 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
400 	list_del_init(&blkg->q_node);
401 	hlist_del_init_rcu(&blkg->blkcg_node);
402 
403 	/*
404 	 * Both setting lookup hint to and clearing it from @blkg are done
405 	 * under queue_lock.  If it's not pointing to @blkg now, it never
406 	 * will.  Hint assignment itself can race safely.
407 	 */
408 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
409 		rcu_assign_pointer(blkcg->blkg_hint, NULL);
410 
411 	/*
412 	 * Put the reference taken at the time of creation so that when all
413 	 * queues are gone, group can be destroyed.
414 	 */
415 	percpu_ref_kill(&blkg->refcnt);
416 }
417 
418 /**
419  * blkg_destroy_all - destroy all blkgs associated with a request_queue
420  * @q: request_queue of interest
421  *
422  * Destroy all blkgs associated with @q.
423  */
424 static void blkg_destroy_all(struct request_queue *q)
425 {
426 	struct blkcg_gq *blkg, *n;
427 	int count = BLKG_DESTROY_BATCH_SIZE;
428 
429 restart:
430 	spin_lock_irq(&q->queue_lock);
431 	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
432 		struct blkcg *blkcg = blkg->blkcg;
433 
434 		spin_lock(&blkcg->lock);
435 		blkg_destroy(blkg);
436 		spin_unlock(&blkcg->lock);
437 
438 		/*
439 		 * in order to avoid holding the spin lock for too long, release
440 		 * it when a batch of blkgs are destroyed.
441 		 */
442 		if (!(--count)) {
443 			count = BLKG_DESTROY_BATCH_SIZE;
444 			spin_unlock_irq(&q->queue_lock);
445 			cond_resched();
446 			goto restart;
447 		}
448 	}
449 
450 	q->root_blkg = NULL;
451 	spin_unlock_irq(&q->queue_lock);
452 }
453 
454 static int blkcg_reset_stats(struct cgroup_subsys_state *css,
455 			     struct cftype *cftype, u64 val)
456 {
457 	struct blkcg *blkcg = css_to_blkcg(css);
458 	struct blkcg_gq *blkg;
459 	int i, cpu;
460 
461 	mutex_lock(&blkcg_pol_mutex);
462 	spin_lock_irq(&blkcg->lock);
463 
464 	/*
465 	 * Note that stat reset is racy - it doesn't synchronize against
466 	 * stat updates.  This is a debug feature which shouldn't exist
467 	 * anyway.  If you get hit by a race, retry.
468 	 */
469 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
470 		for_each_possible_cpu(cpu) {
471 			struct blkg_iostat_set *bis =
472 				per_cpu_ptr(blkg->iostat_cpu, cpu);
473 			memset(bis, 0, sizeof(*bis));
474 		}
475 		memset(&blkg->iostat, 0, sizeof(blkg->iostat));
476 
477 		for (i = 0; i < BLKCG_MAX_POLS; i++) {
478 			struct blkcg_policy *pol = blkcg_policy[i];
479 
480 			if (blkg->pd[i] && pol->pd_reset_stats_fn)
481 				pol->pd_reset_stats_fn(blkg->pd[i]);
482 		}
483 	}
484 
485 	spin_unlock_irq(&blkcg->lock);
486 	mutex_unlock(&blkcg_pol_mutex);
487 	return 0;
488 }
489 
490 const char *blkg_dev_name(struct blkcg_gq *blkg)
491 {
492 	if (!blkg->q->disk || !blkg->q->disk->bdi->dev)
493 		return NULL;
494 	return bdi_dev_name(blkg->q->disk->bdi);
495 }
496 
497 /**
498  * blkcg_print_blkgs - helper for printing per-blkg data
499  * @sf: seq_file to print to
500  * @blkcg: blkcg of interest
501  * @prfill: fill function to print out a blkg
502  * @pol: policy in question
503  * @data: data to be passed to @prfill
504  * @show_total: to print out sum of prfill return values or not
505  *
506  * This function invokes @prfill on each blkg of @blkcg if pd for the
507  * policy specified by @pol exists.  @prfill is invoked with @sf, the
508  * policy data and @data and the matching queue lock held.  If @show_total
509  * is %true, the sum of the return values from @prfill is printed with
510  * "Total" label at the end.
511  *
512  * This is to be used to construct print functions for
513  * cftype->read_seq_string method.
514  */
515 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
516 		       u64 (*prfill)(struct seq_file *,
517 				     struct blkg_policy_data *, int),
518 		       const struct blkcg_policy *pol, int data,
519 		       bool show_total)
520 {
521 	struct blkcg_gq *blkg;
522 	u64 total = 0;
523 
524 	rcu_read_lock();
525 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
526 		spin_lock_irq(&blkg->q->queue_lock);
527 		if (blkcg_policy_enabled(blkg->q, pol))
528 			total += prfill(sf, blkg->pd[pol->plid], data);
529 		spin_unlock_irq(&blkg->q->queue_lock);
530 	}
531 	rcu_read_unlock();
532 
533 	if (show_total)
534 		seq_printf(sf, "Total %llu\n", (unsigned long long)total);
535 }
536 EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
537 
538 /**
539  * __blkg_prfill_u64 - prfill helper for a single u64 value
540  * @sf: seq_file to print to
541  * @pd: policy private data of interest
542  * @v: value to print
543  *
544  * Print @v to @sf for the device assocaited with @pd.
545  */
546 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
547 {
548 	const char *dname = blkg_dev_name(pd->blkg);
549 
550 	if (!dname)
551 		return 0;
552 
553 	seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
554 	return v;
555 }
556 EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
557 
558 /* Performs queue bypass and policy enabled checks then looks up blkg. */
559 static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
560 					  const struct blkcg_policy *pol,
561 					  struct request_queue *q)
562 {
563 	WARN_ON_ONCE(!rcu_read_lock_held());
564 	lockdep_assert_held(&q->queue_lock);
565 
566 	if (!blkcg_policy_enabled(q, pol))
567 		return ERR_PTR(-EOPNOTSUPP);
568 	return __blkg_lookup(blkcg, q, true /* update_hint */);
569 }
570 
571 /**
572  * blkcg_conf_open_bdev - parse and open bdev for per-blkg config update
573  * @inputp: input string pointer
574  *
575  * Parse the device node prefix part, MAJ:MIN, of per-blkg config update
576  * from @input and get and return the matching bdev.  *@inputp is
577  * updated to point past the device node prefix.  Returns an ERR_PTR()
578  * value on error.
579  *
580  * Use this function iff blkg_conf_prep() can't be used for some reason.
581  */
582 struct block_device *blkcg_conf_open_bdev(char **inputp)
583 {
584 	char *input = *inputp;
585 	unsigned int major, minor;
586 	struct block_device *bdev;
587 	int key_len;
588 
589 	if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
590 		return ERR_PTR(-EINVAL);
591 
592 	input += key_len;
593 	if (!isspace(*input))
594 		return ERR_PTR(-EINVAL);
595 	input = skip_spaces(input);
596 
597 	bdev = blkdev_get_no_open(MKDEV(major, minor));
598 	if (!bdev)
599 		return ERR_PTR(-ENODEV);
600 	if (bdev_is_partition(bdev)) {
601 		blkdev_put_no_open(bdev);
602 		return ERR_PTR(-ENODEV);
603 	}
604 
605 	*inputp = input;
606 	return bdev;
607 }
608 
609 /**
610  * blkg_conf_prep - parse and prepare for per-blkg config update
611  * @blkcg: target block cgroup
612  * @pol: target policy
613  * @input: input string
614  * @ctx: blkg_conf_ctx to be filled
615  *
616  * Parse per-blkg config update from @input and initialize @ctx with the
617  * result.  @ctx->blkg points to the blkg to be updated and @ctx->body the
618  * part of @input following MAJ:MIN.  This function returns with RCU read
619  * lock and queue lock held and must be paired with blkg_conf_finish().
620  */
621 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
622 		   char *input, struct blkg_conf_ctx *ctx)
623 	__acquires(rcu) __acquires(&bdev->bd_disk->queue->queue_lock)
624 {
625 	struct block_device *bdev;
626 	struct request_queue *q;
627 	struct blkcg_gq *blkg;
628 	int ret;
629 
630 	bdev = blkcg_conf_open_bdev(&input);
631 	if (IS_ERR(bdev))
632 		return PTR_ERR(bdev);
633 
634 	q = bdev->bd_disk->queue;
635 
636 	rcu_read_lock();
637 	spin_lock_irq(&q->queue_lock);
638 
639 	blkg = blkg_lookup_check(blkcg, pol, q);
640 	if (IS_ERR(blkg)) {
641 		ret = PTR_ERR(blkg);
642 		goto fail_unlock;
643 	}
644 
645 	if (blkg)
646 		goto success;
647 
648 	/*
649 	 * Create blkgs walking down from blkcg_root to @blkcg, so that all
650 	 * non-root blkgs have access to their parents.
651 	 */
652 	while (true) {
653 		struct blkcg *pos = blkcg;
654 		struct blkcg *parent;
655 		struct blkcg_gq *new_blkg;
656 
657 		parent = blkcg_parent(blkcg);
658 		while (parent && !__blkg_lookup(parent, q, false)) {
659 			pos = parent;
660 			parent = blkcg_parent(parent);
661 		}
662 
663 		/* Drop locks to do new blkg allocation with GFP_KERNEL. */
664 		spin_unlock_irq(&q->queue_lock);
665 		rcu_read_unlock();
666 
667 		new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
668 		if (unlikely(!new_blkg)) {
669 			ret = -ENOMEM;
670 			goto fail;
671 		}
672 
673 		if (radix_tree_preload(GFP_KERNEL)) {
674 			blkg_free(new_blkg);
675 			ret = -ENOMEM;
676 			goto fail;
677 		}
678 
679 		rcu_read_lock();
680 		spin_lock_irq(&q->queue_lock);
681 
682 		blkg = blkg_lookup_check(pos, pol, q);
683 		if (IS_ERR(blkg)) {
684 			ret = PTR_ERR(blkg);
685 			blkg_free(new_blkg);
686 			goto fail_preloaded;
687 		}
688 
689 		if (blkg) {
690 			blkg_free(new_blkg);
691 		} else {
692 			blkg = blkg_create(pos, q, new_blkg);
693 			if (IS_ERR(blkg)) {
694 				ret = PTR_ERR(blkg);
695 				goto fail_preloaded;
696 			}
697 		}
698 
699 		radix_tree_preload_end();
700 
701 		if (pos == blkcg)
702 			goto success;
703 	}
704 success:
705 	ctx->bdev = bdev;
706 	ctx->blkg = blkg;
707 	ctx->body = input;
708 	return 0;
709 
710 fail_preloaded:
711 	radix_tree_preload_end();
712 fail_unlock:
713 	spin_unlock_irq(&q->queue_lock);
714 	rcu_read_unlock();
715 fail:
716 	blkdev_put_no_open(bdev);
717 	/*
718 	 * If queue was bypassing, we should retry.  Do so after a
719 	 * short msleep().  It isn't strictly necessary but queue
720 	 * can be bypassing for some time and it's always nice to
721 	 * avoid busy looping.
722 	 */
723 	if (ret == -EBUSY) {
724 		msleep(10);
725 		ret = restart_syscall();
726 	}
727 	return ret;
728 }
729 EXPORT_SYMBOL_GPL(blkg_conf_prep);
730 
731 /**
732  * blkg_conf_finish - finish up per-blkg config update
733  * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
734  *
735  * Finish up after per-blkg config update.  This function must be paired
736  * with blkg_conf_prep().
737  */
738 void blkg_conf_finish(struct blkg_conf_ctx *ctx)
739 	__releases(&ctx->bdev->bd_disk->queue->queue_lock) __releases(rcu)
740 {
741 	spin_unlock_irq(&ctx->bdev->bd_disk->queue->queue_lock);
742 	rcu_read_unlock();
743 	blkdev_put_no_open(ctx->bdev);
744 }
745 EXPORT_SYMBOL_GPL(blkg_conf_finish);
746 
747 static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
748 {
749 	int i;
750 
751 	for (i = 0; i < BLKG_IOSTAT_NR; i++) {
752 		dst->bytes[i] = src->bytes[i];
753 		dst->ios[i] = src->ios[i];
754 	}
755 }
756 
757 static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src)
758 {
759 	int i;
760 
761 	for (i = 0; i < BLKG_IOSTAT_NR; i++) {
762 		dst->bytes[i] += src->bytes[i];
763 		dst->ios[i] += src->ios[i];
764 	}
765 }
766 
767 static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src)
768 {
769 	int i;
770 
771 	for (i = 0; i < BLKG_IOSTAT_NR; i++) {
772 		dst->bytes[i] -= src->bytes[i];
773 		dst->ios[i] -= src->ios[i];
774 	}
775 }
776 
777 static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
778 {
779 	struct blkcg *blkcg = css_to_blkcg(css);
780 	struct blkcg_gq *blkg;
781 
782 	/* Root-level stats are sourced from system-wide IO stats */
783 	if (!cgroup_parent(css->cgroup))
784 		return;
785 
786 	rcu_read_lock();
787 
788 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
789 		struct blkcg_gq *parent = blkg->parent;
790 		struct blkg_iostat_set *bisc = per_cpu_ptr(blkg->iostat_cpu, cpu);
791 		struct blkg_iostat cur, delta;
792 		unsigned long flags;
793 		unsigned int seq;
794 
795 		/* fetch the current per-cpu values */
796 		do {
797 			seq = u64_stats_fetch_begin(&bisc->sync);
798 			blkg_iostat_set(&cur, &bisc->cur);
799 		} while (u64_stats_fetch_retry(&bisc->sync, seq));
800 
801 		/* propagate percpu delta to global */
802 		flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
803 		blkg_iostat_set(&delta, &cur);
804 		blkg_iostat_sub(&delta, &bisc->last);
805 		blkg_iostat_add(&blkg->iostat.cur, &delta);
806 		blkg_iostat_add(&bisc->last, &delta);
807 		u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
808 
809 		/* propagate global delta to parent (unless that's root) */
810 		if (parent && parent->parent) {
811 			flags = u64_stats_update_begin_irqsave(&parent->iostat.sync);
812 			blkg_iostat_set(&delta, &blkg->iostat.cur);
813 			blkg_iostat_sub(&delta, &blkg->iostat.last);
814 			blkg_iostat_add(&parent->iostat.cur, &delta);
815 			blkg_iostat_add(&blkg->iostat.last, &delta);
816 			u64_stats_update_end_irqrestore(&parent->iostat.sync, flags);
817 		}
818 	}
819 
820 	rcu_read_unlock();
821 }
822 
823 /*
824  * We source root cgroup stats from the system-wide stats to avoid
825  * tracking the same information twice and incurring overhead when no
826  * cgroups are defined. For that reason, cgroup_rstat_flush in
827  * blkcg_print_stat does not actually fill out the iostat in the root
828  * cgroup's blkcg_gq.
829  *
830  * However, we would like to re-use the printing code between the root and
831  * non-root cgroups to the extent possible. For that reason, we simulate
832  * flushing the root cgroup's stats by explicitly filling in the iostat
833  * with disk level statistics.
834  */
835 static void blkcg_fill_root_iostats(void)
836 {
837 	struct class_dev_iter iter;
838 	struct device *dev;
839 
840 	class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
841 	while ((dev = class_dev_iter_next(&iter))) {
842 		struct block_device *bdev = dev_to_bdev(dev);
843 		struct blkcg_gq *blkg =
844 			blk_queue_root_blkg(bdev->bd_disk->queue);
845 		struct blkg_iostat tmp;
846 		int cpu;
847 
848 		memset(&tmp, 0, sizeof(tmp));
849 		for_each_possible_cpu(cpu) {
850 			struct disk_stats *cpu_dkstats;
851 			unsigned long flags;
852 
853 			cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu);
854 			tmp.ios[BLKG_IOSTAT_READ] +=
855 				cpu_dkstats->ios[STAT_READ];
856 			tmp.ios[BLKG_IOSTAT_WRITE] +=
857 				cpu_dkstats->ios[STAT_WRITE];
858 			tmp.ios[BLKG_IOSTAT_DISCARD] +=
859 				cpu_dkstats->ios[STAT_DISCARD];
860 			// convert sectors to bytes
861 			tmp.bytes[BLKG_IOSTAT_READ] +=
862 				cpu_dkstats->sectors[STAT_READ] << 9;
863 			tmp.bytes[BLKG_IOSTAT_WRITE] +=
864 				cpu_dkstats->sectors[STAT_WRITE] << 9;
865 			tmp.bytes[BLKG_IOSTAT_DISCARD] +=
866 				cpu_dkstats->sectors[STAT_DISCARD] << 9;
867 
868 			flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
869 			blkg_iostat_set(&blkg->iostat.cur, &tmp);
870 			u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
871 		}
872 	}
873 }
874 
875 static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
876 {
877 	struct blkg_iostat_set *bis = &blkg->iostat;
878 	u64 rbytes, wbytes, rios, wios, dbytes, dios;
879 	bool has_stats = false;
880 	const char *dname;
881 	unsigned seq;
882 	int i;
883 
884 	if (!blkg->online)
885 		return;
886 
887 	dname = blkg_dev_name(blkg);
888 	if (!dname)
889 		return;
890 
891 	seq_printf(s, "%s ", dname);
892 
893 	do {
894 		seq = u64_stats_fetch_begin(&bis->sync);
895 
896 		rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
897 		wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
898 		dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
899 		rios = bis->cur.ios[BLKG_IOSTAT_READ];
900 		wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
901 		dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
902 	} while (u64_stats_fetch_retry(&bis->sync, seq));
903 
904 	if (rbytes || wbytes || rios || wios) {
905 		has_stats = true;
906 		seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
907 			rbytes, wbytes, rios, wios,
908 			dbytes, dios);
909 	}
910 
911 	if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {
912 		has_stats = true;
913 		seq_printf(s, " use_delay=%d delay_nsec=%llu",
914 			atomic_read(&blkg->use_delay),
915 			atomic64_read(&blkg->delay_nsec));
916 	}
917 
918 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
919 		struct blkcg_policy *pol = blkcg_policy[i];
920 
921 		if (!blkg->pd[i] || !pol->pd_stat_fn)
922 			continue;
923 
924 		if (pol->pd_stat_fn(blkg->pd[i], s))
925 			has_stats = true;
926 	}
927 
928 	if (has_stats)
929 		seq_printf(s, "\n");
930 }
931 
932 static int blkcg_print_stat(struct seq_file *sf, void *v)
933 {
934 	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
935 	struct blkcg_gq *blkg;
936 
937 	if (!seq_css(sf)->parent)
938 		blkcg_fill_root_iostats();
939 	else
940 		cgroup_rstat_flush(blkcg->css.cgroup);
941 
942 	rcu_read_lock();
943 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
944 		spin_lock_irq(&blkg->q->queue_lock);
945 		blkcg_print_one_stat(blkg, sf);
946 		spin_unlock_irq(&blkg->q->queue_lock);
947 	}
948 	rcu_read_unlock();
949 	return 0;
950 }
951 
952 static struct cftype blkcg_files[] = {
953 	{
954 		.name = "stat",
955 		.seq_show = blkcg_print_stat,
956 	},
957 	{ }	/* terminate */
958 };
959 
960 static struct cftype blkcg_legacy_files[] = {
961 	{
962 		.name = "reset_stats",
963 		.write_u64 = blkcg_reset_stats,
964 	},
965 	{ }	/* terminate */
966 };
967 
968 /*
969  * blkcg destruction is a three-stage process.
970  *
971  * 1. Destruction starts.  The blkcg_css_offline() callback is invoked
972  *    which offlines writeback.  Here we tie the next stage of blkg destruction
973  *    to the completion of writeback associated with the blkcg.  This lets us
974  *    avoid punting potentially large amounts of outstanding writeback to root
975  *    while maintaining any ongoing policies.  The next stage is triggered when
976  *    the nr_cgwbs count goes to zero.
977  *
978  * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
979  *    and handles the destruction of blkgs.  Here the css reference held by
980  *    the blkg is put back eventually allowing blkcg_css_free() to be called.
981  *    This work may occur in cgwb_release_workfn() on the cgwb_release
982  *    workqueue.  Any submitted ios that fail to get the blkg ref will be
983  *    punted to the root_blkg.
984  *
985  * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
986  *    This finally frees the blkcg.
987  */
988 
989 /**
990  * blkcg_css_offline - cgroup css_offline callback
991  * @css: css of interest
992  *
993  * This function is called when @css is about to go away.  Here the cgwbs are
994  * offlined first and only once writeback associated with the blkcg has
995  * finished do we start step 2 (see above).
996  */
997 static void blkcg_css_offline(struct cgroup_subsys_state *css)
998 {
999 	struct blkcg *blkcg = css_to_blkcg(css);
1000 
1001 	/* this prevents anyone from attaching or migrating to this blkcg */
1002 	wb_blkcg_offline(blkcg);
1003 
1004 	/* put the base online pin allowing step 2 to be triggered */
1005 	blkcg_unpin_online(blkcg);
1006 }
1007 
1008 /**
1009  * blkcg_destroy_blkgs - responsible for shooting down blkgs
1010  * @blkcg: blkcg of interest
1011  *
1012  * blkgs should be removed while holding both q and blkcg locks.  As blkcg lock
1013  * is nested inside q lock, this function performs reverse double lock dancing.
1014  * Destroying the blkgs releases the reference held on the blkcg's css allowing
1015  * blkcg_css_free to eventually be called.
1016  *
1017  * This is the blkcg counterpart of ioc_release_fn().
1018  */
1019 void blkcg_destroy_blkgs(struct blkcg *blkcg)
1020 {
1021 	might_sleep();
1022 
1023 	spin_lock_irq(&blkcg->lock);
1024 
1025 	while (!hlist_empty(&blkcg->blkg_list)) {
1026 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1027 						struct blkcg_gq, blkcg_node);
1028 		struct request_queue *q = blkg->q;
1029 
1030 		if (need_resched() || !spin_trylock(&q->queue_lock)) {
1031 			/*
1032 			 * Given that the system can accumulate a huge number
1033 			 * of blkgs in pathological cases, check to see if we
1034 			 * need to rescheduling to avoid softlockup.
1035 			 */
1036 			spin_unlock_irq(&blkcg->lock);
1037 			cond_resched();
1038 			spin_lock_irq(&blkcg->lock);
1039 			continue;
1040 		}
1041 
1042 		blkg_destroy(blkg);
1043 		spin_unlock(&q->queue_lock);
1044 	}
1045 
1046 	spin_unlock_irq(&blkcg->lock);
1047 }
1048 
1049 static void blkcg_css_free(struct cgroup_subsys_state *css)
1050 {
1051 	struct blkcg *blkcg = css_to_blkcg(css);
1052 	int i;
1053 
1054 	mutex_lock(&blkcg_pol_mutex);
1055 
1056 	list_del(&blkcg->all_blkcgs_node);
1057 
1058 	for (i = 0; i < BLKCG_MAX_POLS; i++)
1059 		if (blkcg->cpd[i])
1060 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1061 
1062 	mutex_unlock(&blkcg_pol_mutex);
1063 
1064 	kfree(blkcg);
1065 }
1066 
1067 static struct cgroup_subsys_state *
1068 blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
1069 {
1070 	struct blkcg *blkcg;
1071 	struct cgroup_subsys_state *ret;
1072 	int i;
1073 
1074 	mutex_lock(&blkcg_pol_mutex);
1075 
1076 	if (!parent_css) {
1077 		blkcg = &blkcg_root;
1078 	} else {
1079 		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1080 		if (!blkcg) {
1081 			ret = ERR_PTR(-ENOMEM);
1082 			goto unlock;
1083 		}
1084 	}
1085 
1086 	for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1087 		struct blkcg_policy *pol = blkcg_policy[i];
1088 		struct blkcg_policy_data *cpd;
1089 
1090 		/*
1091 		 * If the policy hasn't been attached yet, wait for it
1092 		 * to be attached before doing anything else. Otherwise,
1093 		 * check if the policy requires any specific per-cgroup
1094 		 * data: if it does, allocate and initialize it.
1095 		 */
1096 		if (!pol || !pol->cpd_alloc_fn)
1097 			continue;
1098 
1099 		cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1100 		if (!cpd) {
1101 			ret = ERR_PTR(-ENOMEM);
1102 			goto free_pd_blkcg;
1103 		}
1104 		blkcg->cpd[i] = cpd;
1105 		cpd->blkcg = blkcg;
1106 		cpd->plid = i;
1107 		if (pol->cpd_init_fn)
1108 			pol->cpd_init_fn(cpd);
1109 	}
1110 
1111 	spin_lock_init(&blkcg->lock);
1112 	refcount_set(&blkcg->online_pin, 1);
1113 	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
1114 	INIT_HLIST_HEAD(&blkcg->blkg_list);
1115 #ifdef CONFIG_CGROUP_WRITEBACK
1116 	INIT_LIST_HEAD(&blkcg->cgwb_list);
1117 #endif
1118 	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1119 
1120 	mutex_unlock(&blkcg_pol_mutex);
1121 	return &blkcg->css;
1122 
1123 free_pd_blkcg:
1124 	for (i--; i >= 0; i--)
1125 		if (blkcg->cpd[i])
1126 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1127 
1128 	if (blkcg != &blkcg_root)
1129 		kfree(blkcg);
1130 unlock:
1131 	mutex_unlock(&blkcg_pol_mutex);
1132 	return ret;
1133 }
1134 
1135 static int blkcg_css_online(struct cgroup_subsys_state *css)
1136 {
1137 	struct blkcg *blkcg = css_to_blkcg(css);
1138 	struct blkcg *parent = blkcg_parent(blkcg);
1139 
1140 	/*
1141 	 * blkcg_pin_online() is used to delay blkcg offline so that blkgs
1142 	 * don't go offline while cgwbs are still active on them.  Pin the
1143 	 * parent so that offline always happens towards the root.
1144 	 */
1145 	if (parent)
1146 		blkcg_pin_online(parent);
1147 	return 0;
1148 }
1149 
1150 /**
1151  * blkcg_init_queue - initialize blkcg part of request queue
1152  * @q: request_queue to initialize
1153  *
1154  * Called from blk_alloc_queue(). Responsible for initializing blkcg
1155  * part of new request_queue @q.
1156  *
1157  * RETURNS:
1158  * 0 on success, -errno on failure.
1159  */
1160 int blkcg_init_queue(struct request_queue *q)
1161 {
1162 	struct blkcg_gq *new_blkg, *blkg;
1163 	bool preloaded;
1164 	int ret;
1165 
1166 	new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1167 	if (!new_blkg)
1168 		return -ENOMEM;
1169 
1170 	preloaded = !radix_tree_preload(GFP_KERNEL);
1171 
1172 	/* Make sure the root blkg exists. */
1173 	rcu_read_lock();
1174 	spin_lock_irq(&q->queue_lock);
1175 	blkg = blkg_create(&blkcg_root, q, new_blkg);
1176 	if (IS_ERR(blkg))
1177 		goto err_unlock;
1178 	q->root_blkg = blkg;
1179 	spin_unlock_irq(&q->queue_lock);
1180 	rcu_read_unlock();
1181 
1182 	if (preloaded)
1183 		radix_tree_preload_end();
1184 
1185 	ret = blk_ioprio_init(q);
1186 	if (ret)
1187 		goto err_destroy_all;
1188 
1189 	ret = blk_throtl_init(q);
1190 	if (ret)
1191 		goto err_destroy_all;
1192 
1193 	ret = blk_iolatency_init(q);
1194 	if (ret) {
1195 		blk_throtl_exit(q);
1196 		goto err_destroy_all;
1197 	}
1198 
1199 	return 0;
1200 
1201 err_destroy_all:
1202 	blkg_destroy_all(q);
1203 	return ret;
1204 err_unlock:
1205 	spin_unlock_irq(&q->queue_lock);
1206 	rcu_read_unlock();
1207 	if (preloaded)
1208 		radix_tree_preload_end();
1209 	return PTR_ERR(blkg);
1210 }
1211 
1212 /**
1213  * blkcg_exit_queue - exit and release blkcg part of request_queue
1214  * @q: request_queue being released
1215  *
1216  * Called from blk_exit_queue().  Responsible for exiting blkcg part.
1217  */
1218 void blkcg_exit_queue(struct request_queue *q)
1219 {
1220 	blkg_destroy_all(q);
1221 	blk_throtl_exit(q);
1222 }
1223 
1224 static void blkcg_bind(struct cgroup_subsys_state *root_css)
1225 {
1226 	int i;
1227 
1228 	mutex_lock(&blkcg_pol_mutex);
1229 
1230 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
1231 		struct blkcg_policy *pol = blkcg_policy[i];
1232 		struct blkcg *blkcg;
1233 
1234 		if (!pol || !pol->cpd_bind_fn)
1235 			continue;
1236 
1237 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1238 			if (blkcg->cpd[pol->plid])
1239 				pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1240 	}
1241 	mutex_unlock(&blkcg_pol_mutex);
1242 }
1243 
1244 static void blkcg_exit(struct task_struct *tsk)
1245 {
1246 	if (tsk->throttle_queue)
1247 		blk_put_queue(tsk->throttle_queue);
1248 	tsk->throttle_queue = NULL;
1249 }
1250 
1251 struct cgroup_subsys io_cgrp_subsys = {
1252 	.css_alloc = blkcg_css_alloc,
1253 	.css_online = blkcg_css_online,
1254 	.css_offline = blkcg_css_offline,
1255 	.css_free = blkcg_css_free,
1256 	.css_rstat_flush = blkcg_rstat_flush,
1257 	.bind = blkcg_bind,
1258 	.dfl_cftypes = blkcg_files,
1259 	.legacy_cftypes = blkcg_legacy_files,
1260 	.legacy_name = "blkio",
1261 	.exit = blkcg_exit,
1262 #ifdef CONFIG_MEMCG
1263 	/*
1264 	 * This ensures that, if available, memcg is automatically enabled
1265 	 * together on the default hierarchy so that the owner cgroup can
1266 	 * be retrieved from writeback pages.
1267 	 */
1268 	.depends_on = 1 << memory_cgrp_id,
1269 #endif
1270 };
1271 EXPORT_SYMBOL_GPL(io_cgrp_subsys);
1272 
1273 /**
1274  * blkcg_activate_policy - activate a blkcg policy on a request_queue
1275  * @q: request_queue of interest
1276  * @pol: blkcg policy to activate
1277  *
1278  * Activate @pol on @q.  Requires %GFP_KERNEL context.  @q goes through
1279  * bypass mode to populate its blkgs with policy_data for @pol.
1280  *
1281  * Activation happens with @q bypassed, so nobody would be accessing blkgs
1282  * from IO path.  Update of each blkg is protected by both queue and blkcg
1283  * locks so that holding either lock and testing blkcg_policy_enabled() is
1284  * always enough for dereferencing policy data.
1285  *
1286  * The caller is responsible for synchronizing [de]activations and policy
1287  * [un]registerations.  Returns 0 on success, -errno on failure.
1288  */
1289 int blkcg_activate_policy(struct request_queue *q,
1290 			  const struct blkcg_policy *pol)
1291 {
1292 	struct blkg_policy_data *pd_prealloc = NULL;
1293 	struct blkcg_gq *blkg, *pinned_blkg = NULL;
1294 	int ret;
1295 
1296 	if (blkcg_policy_enabled(q, pol))
1297 		return 0;
1298 
1299 	if (queue_is_mq(q))
1300 		blk_mq_freeze_queue(q);
1301 retry:
1302 	spin_lock_irq(&q->queue_lock);
1303 
1304 	/* blkg_list is pushed at the head, reverse walk to allocate parents first */
1305 	list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
1306 		struct blkg_policy_data *pd;
1307 
1308 		if (blkg->pd[pol->plid])
1309 			continue;
1310 
1311 		/* If prealloc matches, use it; otherwise try GFP_NOWAIT */
1312 		if (blkg == pinned_blkg) {
1313 			pd = pd_prealloc;
1314 			pd_prealloc = NULL;
1315 		} else {
1316 			pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q,
1317 					      blkg->blkcg);
1318 		}
1319 
1320 		if (!pd) {
1321 			/*
1322 			 * GFP_NOWAIT failed.  Free the existing one and
1323 			 * prealloc for @blkg w/ GFP_KERNEL.
1324 			 */
1325 			if (pinned_blkg)
1326 				blkg_put(pinned_blkg);
1327 			blkg_get(blkg);
1328 			pinned_blkg = blkg;
1329 
1330 			spin_unlock_irq(&q->queue_lock);
1331 
1332 			if (pd_prealloc)
1333 				pol->pd_free_fn(pd_prealloc);
1334 			pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q,
1335 						       blkg->blkcg);
1336 			if (pd_prealloc)
1337 				goto retry;
1338 			else
1339 				goto enomem;
1340 		}
1341 
1342 		blkg->pd[pol->plid] = pd;
1343 		pd->blkg = blkg;
1344 		pd->plid = pol->plid;
1345 	}
1346 
1347 	/* all allocated, init in the same order */
1348 	if (pol->pd_init_fn)
1349 		list_for_each_entry_reverse(blkg, &q->blkg_list, q_node)
1350 			pol->pd_init_fn(blkg->pd[pol->plid]);
1351 
1352 	__set_bit(pol->plid, q->blkcg_pols);
1353 	ret = 0;
1354 
1355 	spin_unlock_irq(&q->queue_lock);
1356 out:
1357 	if (queue_is_mq(q))
1358 		blk_mq_unfreeze_queue(q);
1359 	if (pinned_blkg)
1360 		blkg_put(pinned_blkg);
1361 	if (pd_prealloc)
1362 		pol->pd_free_fn(pd_prealloc);
1363 	return ret;
1364 
1365 enomem:
1366 	/* alloc failed, nothing's initialized yet, free everything */
1367 	spin_lock_irq(&q->queue_lock);
1368 	list_for_each_entry(blkg, &q->blkg_list, q_node) {
1369 		struct blkcg *blkcg = blkg->blkcg;
1370 
1371 		spin_lock(&blkcg->lock);
1372 		if (blkg->pd[pol->plid]) {
1373 			pol->pd_free_fn(blkg->pd[pol->plid]);
1374 			blkg->pd[pol->plid] = NULL;
1375 		}
1376 		spin_unlock(&blkcg->lock);
1377 	}
1378 	spin_unlock_irq(&q->queue_lock);
1379 	ret = -ENOMEM;
1380 	goto out;
1381 }
1382 EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1383 
1384 /**
1385  * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1386  * @q: request_queue of interest
1387  * @pol: blkcg policy to deactivate
1388  *
1389  * Deactivate @pol on @q.  Follows the same synchronization rules as
1390  * blkcg_activate_policy().
1391  */
1392 void blkcg_deactivate_policy(struct request_queue *q,
1393 			     const struct blkcg_policy *pol)
1394 {
1395 	struct blkcg_gq *blkg;
1396 
1397 	if (!blkcg_policy_enabled(q, pol))
1398 		return;
1399 
1400 	if (queue_is_mq(q))
1401 		blk_mq_freeze_queue(q);
1402 
1403 	spin_lock_irq(&q->queue_lock);
1404 
1405 	__clear_bit(pol->plid, q->blkcg_pols);
1406 
1407 	list_for_each_entry(blkg, &q->blkg_list, q_node) {
1408 		struct blkcg *blkcg = blkg->blkcg;
1409 
1410 		spin_lock(&blkcg->lock);
1411 		if (blkg->pd[pol->plid]) {
1412 			if (pol->pd_offline_fn)
1413 				pol->pd_offline_fn(blkg->pd[pol->plid]);
1414 			pol->pd_free_fn(blkg->pd[pol->plid]);
1415 			blkg->pd[pol->plid] = NULL;
1416 		}
1417 		spin_unlock(&blkcg->lock);
1418 	}
1419 
1420 	spin_unlock_irq(&q->queue_lock);
1421 
1422 	if (queue_is_mq(q))
1423 		blk_mq_unfreeze_queue(q);
1424 }
1425 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1426 
1427 /**
1428  * blkcg_policy_register - register a blkcg policy
1429  * @pol: blkcg policy to register
1430  *
1431  * Register @pol with blkcg core.  Might sleep and @pol may be modified on
1432  * successful registration.  Returns 0 on success and -errno on failure.
1433  */
1434 int blkcg_policy_register(struct blkcg_policy *pol)
1435 {
1436 	struct blkcg *blkcg;
1437 	int i, ret;
1438 
1439 	mutex_lock(&blkcg_pol_register_mutex);
1440 	mutex_lock(&blkcg_pol_mutex);
1441 
1442 	/* find an empty slot */
1443 	ret = -ENOSPC;
1444 	for (i = 0; i < BLKCG_MAX_POLS; i++)
1445 		if (!blkcg_policy[i])
1446 			break;
1447 	if (i >= BLKCG_MAX_POLS) {
1448 		pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
1449 		goto err_unlock;
1450 	}
1451 
1452 	/* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1453 	if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1454 		(!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1455 		goto err_unlock;
1456 
1457 	/* register @pol */
1458 	pol->plid = i;
1459 	blkcg_policy[pol->plid] = pol;
1460 
1461 	/* allocate and install cpd's */
1462 	if (pol->cpd_alloc_fn) {
1463 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1464 			struct blkcg_policy_data *cpd;
1465 
1466 			cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1467 			if (!cpd)
1468 				goto err_free_cpds;
1469 
1470 			blkcg->cpd[pol->plid] = cpd;
1471 			cpd->blkcg = blkcg;
1472 			cpd->plid = pol->plid;
1473 			if (pol->cpd_init_fn)
1474 				pol->cpd_init_fn(cpd);
1475 		}
1476 	}
1477 
1478 	mutex_unlock(&blkcg_pol_mutex);
1479 
1480 	/* everything is in place, add intf files for the new policy */
1481 	if (pol->dfl_cftypes)
1482 		WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1483 					       pol->dfl_cftypes));
1484 	if (pol->legacy_cftypes)
1485 		WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
1486 						  pol->legacy_cftypes));
1487 	mutex_unlock(&blkcg_pol_register_mutex);
1488 	return 0;
1489 
1490 err_free_cpds:
1491 	if (pol->cpd_free_fn) {
1492 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1493 			if (blkcg->cpd[pol->plid]) {
1494 				pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1495 				blkcg->cpd[pol->plid] = NULL;
1496 			}
1497 		}
1498 	}
1499 	blkcg_policy[pol->plid] = NULL;
1500 err_unlock:
1501 	mutex_unlock(&blkcg_pol_mutex);
1502 	mutex_unlock(&blkcg_pol_register_mutex);
1503 	return ret;
1504 }
1505 EXPORT_SYMBOL_GPL(blkcg_policy_register);
1506 
1507 /**
1508  * blkcg_policy_unregister - unregister a blkcg policy
1509  * @pol: blkcg policy to unregister
1510  *
1511  * Undo blkcg_policy_register(@pol).  Might sleep.
1512  */
1513 void blkcg_policy_unregister(struct blkcg_policy *pol)
1514 {
1515 	struct blkcg *blkcg;
1516 
1517 	mutex_lock(&blkcg_pol_register_mutex);
1518 
1519 	if (WARN_ON(blkcg_policy[pol->plid] != pol))
1520 		goto out_unlock;
1521 
1522 	/* kill the intf files first */
1523 	if (pol->dfl_cftypes)
1524 		cgroup_rm_cftypes(pol->dfl_cftypes);
1525 	if (pol->legacy_cftypes)
1526 		cgroup_rm_cftypes(pol->legacy_cftypes);
1527 
1528 	/* remove cpds and unregister */
1529 	mutex_lock(&blkcg_pol_mutex);
1530 
1531 	if (pol->cpd_free_fn) {
1532 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1533 			if (blkcg->cpd[pol->plid]) {
1534 				pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1535 				blkcg->cpd[pol->plid] = NULL;
1536 			}
1537 		}
1538 	}
1539 	blkcg_policy[pol->plid] = NULL;
1540 
1541 	mutex_unlock(&blkcg_pol_mutex);
1542 out_unlock:
1543 	mutex_unlock(&blkcg_pol_register_mutex);
1544 }
1545 EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
1546 
1547 bool __blkcg_punt_bio_submit(struct bio *bio)
1548 {
1549 	struct blkcg_gq *blkg = bio->bi_blkg;
1550 
1551 	/* consume the flag first */
1552 	bio->bi_opf &= ~REQ_CGROUP_PUNT;
1553 
1554 	/* never bounce for the root cgroup */
1555 	if (!blkg->parent)
1556 		return false;
1557 
1558 	spin_lock_bh(&blkg->async_bio_lock);
1559 	bio_list_add(&blkg->async_bios, bio);
1560 	spin_unlock_bh(&blkg->async_bio_lock);
1561 
1562 	queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
1563 	return true;
1564 }
1565 
1566 /*
1567  * Scale the accumulated delay based on how long it has been since we updated
1568  * the delay.  We only call this when we are adding delay, in case it's been a
1569  * while since we added delay, and when we are checking to see if we need to
1570  * delay a task, to account for any delays that may have occurred.
1571  */
1572 static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
1573 {
1574 	u64 old = atomic64_read(&blkg->delay_start);
1575 
1576 	/* negative use_delay means no scaling, see blkcg_set_delay() */
1577 	if (atomic_read(&blkg->use_delay) < 0)
1578 		return;
1579 
1580 	/*
1581 	 * We only want to scale down every second.  The idea here is that we
1582 	 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
1583 	 * time window.  We only want to throttle tasks for recent delay that
1584 	 * has occurred, in 1 second time windows since that's the maximum
1585 	 * things can be throttled.  We save the current delay window in
1586 	 * blkg->last_delay so we know what amount is still left to be charged
1587 	 * to the blkg from this point onward.  blkg->last_use keeps track of
1588 	 * the use_delay counter.  The idea is if we're unthrottling the blkg we
1589 	 * are ok with whatever is happening now, and we can take away more of
1590 	 * the accumulated delay as we've already throttled enough that
1591 	 * everybody is happy with their IO latencies.
1592 	 */
1593 	if (time_before64(old + NSEC_PER_SEC, now) &&
1594 	    atomic64_cmpxchg(&blkg->delay_start, old, now) == old) {
1595 		u64 cur = atomic64_read(&blkg->delay_nsec);
1596 		u64 sub = min_t(u64, blkg->last_delay, now - old);
1597 		int cur_use = atomic_read(&blkg->use_delay);
1598 
1599 		/*
1600 		 * We've been unthrottled, subtract a larger chunk of our
1601 		 * accumulated delay.
1602 		 */
1603 		if (cur_use < blkg->last_use)
1604 			sub = max_t(u64, sub, blkg->last_delay >> 1);
1605 
1606 		/*
1607 		 * This shouldn't happen, but handle it anyway.  Our delay_nsec
1608 		 * should only ever be growing except here where we subtract out
1609 		 * min(last_delay, 1 second), but lord knows bugs happen and I'd
1610 		 * rather not end up with negative numbers.
1611 		 */
1612 		if (unlikely(cur < sub)) {
1613 			atomic64_set(&blkg->delay_nsec, 0);
1614 			blkg->last_delay = 0;
1615 		} else {
1616 			atomic64_sub(sub, &blkg->delay_nsec);
1617 			blkg->last_delay = cur - sub;
1618 		}
1619 		blkg->last_use = cur_use;
1620 	}
1621 }
1622 
1623 /*
1624  * This is called when we want to actually walk up the hierarchy and check to
1625  * see if we need to throttle, and then actually throttle if there is some
1626  * accumulated delay.  This should only be called upon return to user space so
1627  * we're not holding some lock that would induce a priority inversion.
1628  */
1629 static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
1630 {
1631 	unsigned long pflags;
1632 	bool clamp;
1633 	u64 now = ktime_to_ns(ktime_get());
1634 	u64 exp;
1635 	u64 delay_nsec = 0;
1636 	int tok;
1637 
1638 	while (blkg->parent) {
1639 		int use_delay = atomic_read(&blkg->use_delay);
1640 
1641 		if (use_delay) {
1642 			u64 this_delay;
1643 
1644 			blkcg_scale_delay(blkg, now);
1645 			this_delay = atomic64_read(&blkg->delay_nsec);
1646 			if (this_delay > delay_nsec) {
1647 				delay_nsec = this_delay;
1648 				clamp = use_delay > 0;
1649 			}
1650 		}
1651 		blkg = blkg->parent;
1652 	}
1653 
1654 	if (!delay_nsec)
1655 		return;
1656 
1657 	/*
1658 	 * Let's not sleep for all eternity if we've amassed a huge delay.
1659 	 * Swapping or metadata IO can accumulate 10's of seconds worth of
1660 	 * delay, and we want userspace to be able to do _something_ so cap the
1661 	 * delays at 0.25s. If there's 10's of seconds worth of delay then the
1662 	 * tasks will be delayed for 0.25 second for every syscall. If
1663 	 * blkcg_set_delay() was used as indicated by negative use_delay, the
1664 	 * caller is responsible for regulating the range.
1665 	 */
1666 	if (clamp)
1667 		delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
1668 
1669 	if (use_memdelay)
1670 		psi_memstall_enter(&pflags);
1671 
1672 	exp = ktime_add_ns(now, delay_nsec);
1673 	tok = io_schedule_prepare();
1674 	do {
1675 		__set_current_state(TASK_KILLABLE);
1676 		if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
1677 			break;
1678 	} while (!fatal_signal_pending(current));
1679 	io_schedule_finish(tok);
1680 
1681 	if (use_memdelay)
1682 		psi_memstall_leave(&pflags);
1683 }
1684 
1685 /**
1686  * blkcg_maybe_throttle_current - throttle the current task if it has been marked
1687  *
1688  * This is only called if we've been marked with set_notify_resume().  Obviously
1689  * we can be set_notify_resume() for reasons other than blkcg throttling, so we
1690  * check to see if current->throttle_queue is set and if not this doesn't do
1691  * anything.  This should only ever be called by the resume code, it's not meant
1692  * to be called by people willy-nilly as it will actually do the work to
1693  * throttle the task if it is setup for throttling.
1694  */
1695 void blkcg_maybe_throttle_current(void)
1696 {
1697 	struct request_queue *q = current->throttle_queue;
1698 	struct cgroup_subsys_state *css;
1699 	struct blkcg *blkcg;
1700 	struct blkcg_gq *blkg;
1701 	bool use_memdelay = current->use_memdelay;
1702 
1703 	if (!q)
1704 		return;
1705 
1706 	current->throttle_queue = NULL;
1707 	current->use_memdelay = false;
1708 
1709 	rcu_read_lock();
1710 	css = kthread_blkcg();
1711 	if (css)
1712 		blkcg = css_to_blkcg(css);
1713 	else
1714 		blkcg = css_to_blkcg(task_css(current, io_cgrp_id));
1715 
1716 	if (!blkcg)
1717 		goto out;
1718 	blkg = blkg_lookup(blkcg, q);
1719 	if (!blkg)
1720 		goto out;
1721 	if (!blkg_tryget(blkg))
1722 		goto out;
1723 	rcu_read_unlock();
1724 
1725 	blkcg_maybe_throttle_blkg(blkg, use_memdelay);
1726 	blkg_put(blkg);
1727 	blk_put_queue(q);
1728 	return;
1729 out:
1730 	rcu_read_unlock();
1731 	blk_put_queue(q);
1732 }
1733 
1734 /**
1735  * blkcg_schedule_throttle - this task needs to check for throttling
1736  * @q: the request queue IO was submitted on
1737  * @use_memdelay: do we charge this to memory delay for PSI
1738  *
1739  * This is called by the IO controller when we know there's delay accumulated
1740  * for the blkg for this task.  We do not pass the blkg because there are places
1741  * we call this that may not have that information, the swapping code for
1742  * instance will only have a request_queue at that point.  This set's the
1743  * notify_resume for the task to check and see if it requires throttling before
1744  * returning to user space.
1745  *
1746  * We will only schedule once per syscall.  You can call this over and over
1747  * again and it will only do the check once upon return to user space, and only
1748  * throttle once.  If the task needs to be throttled again it'll need to be
1749  * re-set at the next time we see the task.
1750  */
1751 void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay)
1752 {
1753 	if (unlikely(current->flags & PF_KTHREAD))
1754 		return;
1755 
1756 	if (current->throttle_queue != q) {
1757 		if (!blk_get_queue(q))
1758 			return;
1759 
1760 		if (current->throttle_queue)
1761 			blk_put_queue(current->throttle_queue);
1762 		current->throttle_queue = q;
1763 	}
1764 
1765 	if (use_memdelay)
1766 		current->use_memdelay = use_memdelay;
1767 	set_notify_resume(current);
1768 }
1769 
1770 /**
1771  * blkcg_add_delay - add delay to this blkg
1772  * @blkg: blkg of interest
1773  * @now: the current time in nanoseconds
1774  * @delta: how many nanoseconds of delay to add
1775  *
1776  * Charge @delta to the blkg's current delay accumulation.  This is used to
1777  * throttle tasks if an IO controller thinks we need more throttling.
1778  */
1779 void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
1780 {
1781 	if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
1782 		return;
1783 	blkcg_scale_delay(blkg, now);
1784 	atomic64_add(delta, &blkg->delay_nsec);
1785 }
1786 
1787 /**
1788  * blkg_tryget_closest - try and get a blkg ref on the closet blkg
1789  * @bio: target bio
1790  * @css: target css
1791  *
1792  * As the failure mode here is to walk up the blkg tree, this ensure that the
1793  * blkg->parent pointers are always valid.  This returns the blkg that it ended
1794  * up taking a reference on or %NULL if no reference was taken.
1795  */
1796 static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
1797 		struct cgroup_subsys_state *css)
1798 {
1799 	struct blkcg_gq *blkg, *ret_blkg = NULL;
1800 
1801 	rcu_read_lock();
1802 	blkg = blkg_lookup_create(css_to_blkcg(css),
1803 				  bio->bi_bdev->bd_disk->queue);
1804 	while (blkg) {
1805 		if (blkg_tryget(blkg)) {
1806 			ret_blkg = blkg;
1807 			break;
1808 		}
1809 		blkg = blkg->parent;
1810 	}
1811 	rcu_read_unlock();
1812 
1813 	return ret_blkg;
1814 }
1815 
1816 /**
1817  * bio_associate_blkg_from_css - associate a bio with a specified css
1818  * @bio: target bio
1819  * @css: target css
1820  *
1821  * Associate @bio with the blkg found by combining the css's blkg and the
1822  * request_queue of the @bio.  An association failure is handled by walking up
1823  * the blkg tree.  Therefore, the blkg associated can be anything between @blkg
1824  * and q->root_blkg.  This situation only happens when a cgroup is dying and
1825  * then the remaining bios will spill to the closest alive blkg.
1826  *
1827  * A reference will be taken on the blkg and will be released when @bio is
1828  * freed.
1829  */
1830 void bio_associate_blkg_from_css(struct bio *bio,
1831 				 struct cgroup_subsys_state *css)
1832 {
1833 	if (bio->bi_blkg)
1834 		blkg_put(bio->bi_blkg);
1835 
1836 	if (css && css->parent) {
1837 		bio->bi_blkg = blkg_tryget_closest(bio, css);
1838 	} else {
1839 		blkg_get(bio->bi_bdev->bd_disk->queue->root_blkg);
1840 		bio->bi_blkg = bio->bi_bdev->bd_disk->queue->root_blkg;
1841 	}
1842 }
1843 EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
1844 
1845 /**
1846  * bio_associate_blkg - associate a bio with a blkg
1847  * @bio: target bio
1848  *
1849  * Associate @bio with the blkg found from the bio's css and request_queue.
1850  * If one is not found, bio_lookup_blkg() creates the blkg.  If a blkg is
1851  * already associated, the css is reused and association redone as the
1852  * request_queue may have changed.
1853  */
1854 void bio_associate_blkg(struct bio *bio)
1855 {
1856 	struct cgroup_subsys_state *css;
1857 
1858 	rcu_read_lock();
1859 
1860 	if (bio->bi_blkg)
1861 		css = &bio_blkcg(bio)->css;
1862 	else
1863 		css = blkcg_css();
1864 
1865 	bio_associate_blkg_from_css(bio, css);
1866 
1867 	rcu_read_unlock();
1868 }
1869 EXPORT_SYMBOL_GPL(bio_associate_blkg);
1870 
1871 /**
1872  * bio_clone_blkg_association - clone blkg association from src to dst bio
1873  * @dst: destination bio
1874  * @src: source bio
1875  */
1876 void bio_clone_blkg_association(struct bio *dst, struct bio *src)
1877 {
1878 	if (src->bi_blkg) {
1879 		if (dst->bi_blkg)
1880 			blkg_put(dst->bi_blkg);
1881 		blkg_get(src->bi_blkg);
1882 		dst->bi_blkg = src->bi_blkg;
1883 	}
1884 }
1885 EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
1886 
1887 static int blk_cgroup_io_type(struct bio *bio)
1888 {
1889 	if (op_is_discard(bio->bi_opf))
1890 		return BLKG_IOSTAT_DISCARD;
1891 	if (op_is_write(bio->bi_opf))
1892 		return BLKG_IOSTAT_WRITE;
1893 	return BLKG_IOSTAT_READ;
1894 }
1895 
1896 void blk_cgroup_bio_start(struct bio *bio)
1897 {
1898 	int rwd = blk_cgroup_io_type(bio), cpu;
1899 	struct blkg_iostat_set *bis;
1900 
1901 	cpu = get_cpu();
1902 	bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
1903 	u64_stats_update_begin(&bis->sync);
1904 
1905 	/*
1906 	 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
1907 	 * bio and we would have already accounted for the size of the bio.
1908 	 */
1909 	if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
1910 		bio_set_flag(bio, BIO_CGROUP_ACCT);
1911 		bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
1912 	}
1913 	bis->cur.ios[rwd]++;
1914 
1915 	u64_stats_update_end(&bis->sync);
1916 	if (cgroup_subsys_on_dfl(io_cgrp_subsys))
1917 		cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu);
1918 	put_cpu();
1919 }
1920 
1921 static int __init blkcg_init(void)
1922 {
1923 	blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
1924 					    WQ_MEM_RECLAIM | WQ_FREEZABLE |
1925 					    WQ_UNBOUND | WQ_SYSFS, 0);
1926 	if (!blkcg_punt_bio_wq)
1927 		return -ENOMEM;
1928 	return 0;
1929 }
1930 subsys_initcall(blkcg_init);
1931 
1932 module_param(blkcg_debug_stats, bool, 0644);
1933 MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");
1934