xref: /openbmc/linux/block/blk-cgroup.c (revision ddc141e5)
1 /*
2  * Common Block IO controller cgroup interface
3  *
4  * Based on ideas and code from CFQ, CFS and BFQ:
5  * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
6  *
7  * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
8  *		      Paolo Valente <paolo.valente@unimore.it>
9  *
10  * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
11  * 	              Nauman Rafique <nauman@google.com>
12  *
13  * For policy-specific per-blkcg data:
14  * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
15  *                    Arianna Avanzini <avanzini.arianna@gmail.com>
16  */
17 #include <linux/ioprio.h>
18 #include <linux/kdev_t.h>
19 #include <linux/module.h>
20 #include <linux/sched/signal.h>
21 #include <linux/err.h>
22 #include <linux/blkdev.h>
23 #include <linux/backing-dev.h>
24 #include <linux/slab.h>
25 #include <linux/genhd.h>
26 #include <linux/delay.h>
27 #include <linux/atomic.h>
28 #include <linux/ctype.h>
29 #include <linux/blk-cgroup.h>
30 #include "blk.h"
31 
32 #define MAX_KEY_LEN 100
33 
34 /*
35  * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
36  * blkcg_pol_register_mutex nests outside of it and synchronizes entire
37  * policy [un]register operations including cgroup file additions /
38  * removals.  Putting cgroup file registration outside blkcg_pol_mutex
39  * allows grabbing it from cgroup callbacks.
40  */
41 static DEFINE_MUTEX(blkcg_pol_register_mutex);
42 static DEFINE_MUTEX(blkcg_pol_mutex);
43 
44 struct blkcg blkcg_root;
45 EXPORT_SYMBOL_GPL(blkcg_root);
46 
47 struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
48 
49 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
50 
51 static LIST_HEAD(all_blkcgs);		/* protected by blkcg_pol_mutex */
52 
53 static bool blkcg_policy_enabled(struct request_queue *q,
54 				 const struct blkcg_policy *pol)
55 {
56 	return pol && test_bit(pol->plid, q->blkcg_pols);
57 }
58 
59 /**
60  * blkg_free - free a blkg
61  * @blkg: blkg to free
62  *
63  * Free @blkg which may be partially allocated.
64  */
65 static void blkg_free(struct blkcg_gq *blkg)
66 {
67 	int i;
68 
69 	if (!blkg)
70 		return;
71 
72 	for (i = 0; i < BLKCG_MAX_POLS; i++)
73 		if (blkg->pd[i])
74 			blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
75 
76 	if (blkg->blkcg != &blkcg_root)
77 		blk_exit_rl(blkg->q, &blkg->rl);
78 
79 	blkg_rwstat_exit(&blkg->stat_ios);
80 	blkg_rwstat_exit(&blkg->stat_bytes);
81 	kfree(blkg);
82 }
83 
84 /**
85  * blkg_alloc - allocate a blkg
86  * @blkcg: block cgroup the new blkg is associated with
87  * @q: request_queue the new blkg is associated with
88  * @gfp_mask: allocation mask to use
89  *
90  * Allocate a new blkg assocating @blkcg and @q.
91  */
92 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
93 				   gfp_t gfp_mask)
94 {
95 	struct blkcg_gq *blkg;
96 	int i;
97 
98 	/* alloc and init base part */
99 	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
100 	if (!blkg)
101 		return NULL;
102 
103 	if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
104 	    blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
105 		goto err_free;
106 
107 	blkg->q = q;
108 	INIT_LIST_HEAD(&blkg->q_node);
109 	blkg->blkcg = blkcg;
110 	atomic_set(&blkg->refcnt, 1);
111 
112 	/* root blkg uses @q->root_rl, init rl only for !root blkgs */
113 	if (blkcg != &blkcg_root) {
114 		if (blk_init_rl(&blkg->rl, q, gfp_mask))
115 			goto err_free;
116 		blkg->rl.blkg = blkg;
117 	}
118 
119 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
120 		struct blkcg_policy *pol = blkcg_policy[i];
121 		struct blkg_policy_data *pd;
122 
123 		if (!blkcg_policy_enabled(q, pol))
124 			continue;
125 
126 		/* alloc per-policy data and attach it to blkg */
127 		pd = pol->pd_alloc_fn(gfp_mask, q->node);
128 		if (!pd)
129 			goto err_free;
130 
131 		blkg->pd[i] = pd;
132 		pd->blkg = blkg;
133 		pd->plid = i;
134 	}
135 
136 	return blkg;
137 
138 err_free:
139 	blkg_free(blkg);
140 	return NULL;
141 }
142 
143 struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
144 				      struct request_queue *q, bool update_hint)
145 {
146 	struct blkcg_gq *blkg;
147 
148 	/*
149 	 * Hint didn't match.  Look up from the radix tree.  Note that the
150 	 * hint can only be updated under queue_lock as otherwise @blkg
151 	 * could have already been removed from blkg_tree.  The caller is
152 	 * responsible for grabbing queue_lock if @update_hint.
153 	 */
154 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
155 	if (blkg && blkg->q == q) {
156 		if (update_hint) {
157 			lockdep_assert_held(q->queue_lock);
158 			rcu_assign_pointer(blkcg->blkg_hint, blkg);
159 		}
160 		return blkg;
161 	}
162 
163 	return NULL;
164 }
165 EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
166 
167 /*
168  * If @new_blkg is %NULL, this function tries to allocate a new one as
169  * necessary using %GFP_NOWAIT.  @new_blkg is always consumed on return.
170  */
171 static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
172 				    struct request_queue *q,
173 				    struct blkcg_gq *new_blkg)
174 {
175 	struct blkcg_gq *blkg;
176 	struct bdi_writeback_congested *wb_congested;
177 	int i, ret;
178 
179 	WARN_ON_ONCE(!rcu_read_lock_held());
180 	lockdep_assert_held(q->queue_lock);
181 
182 	/* blkg holds a reference to blkcg */
183 	if (!css_tryget_online(&blkcg->css)) {
184 		ret = -ENODEV;
185 		goto err_free_blkg;
186 	}
187 
188 	wb_congested = wb_congested_get_create(q->backing_dev_info,
189 					       blkcg->css.id,
190 					       GFP_NOWAIT | __GFP_NOWARN);
191 	if (!wb_congested) {
192 		ret = -ENOMEM;
193 		goto err_put_css;
194 	}
195 
196 	/* allocate */
197 	if (!new_blkg) {
198 		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
199 		if (unlikely(!new_blkg)) {
200 			ret = -ENOMEM;
201 			goto err_put_congested;
202 		}
203 	}
204 	blkg = new_blkg;
205 	blkg->wb_congested = wb_congested;
206 
207 	/* link parent */
208 	if (blkcg_parent(blkcg)) {
209 		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
210 		if (WARN_ON_ONCE(!blkg->parent)) {
211 			ret = -ENODEV;
212 			goto err_put_congested;
213 		}
214 		blkg_get(blkg->parent);
215 	}
216 
217 	/* invoke per-policy init */
218 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
219 		struct blkcg_policy *pol = blkcg_policy[i];
220 
221 		if (blkg->pd[i] && pol->pd_init_fn)
222 			pol->pd_init_fn(blkg->pd[i]);
223 	}
224 
225 	/* insert */
226 	spin_lock(&blkcg->lock);
227 	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
228 	if (likely(!ret)) {
229 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
230 		list_add(&blkg->q_node, &q->blkg_list);
231 
232 		for (i = 0; i < BLKCG_MAX_POLS; i++) {
233 			struct blkcg_policy *pol = blkcg_policy[i];
234 
235 			if (blkg->pd[i] && pol->pd_online_fn)
236 				pol->pd_online_fn(blkg->pd[i]);
237 		}
238 	}
239 	blkg->online = true;
240 	spin_unlock(&blkcg->lock);
241 
242 	if (!ret)
243 		return blkg;
244 
245 	/* @blkg failed fully initialized, use the usual release path */
246 	blkg_put(blkg);
247 	return ERR_PTR(ret);
248 
249 err_put_congested:
250 	wb_congested_put(wb_congested);
251 err_put_css:
252 	css_put(&blkcg->css);
253 err_free_blkg:
254 	blkg_free(new_blkg);
255 	return ERR_PTR(ret);
256 }
257 
258 /**
259  * blkg_lookup_create - lookup blkg, try to create one if not there
260  * @blkcg: blkcg of interest
261  * @q: request_queue of interest
262  *
263  * Lookup blkg for the @blkcg - @q pair.  If it doesn't exist, try to
264  * create one.  blkg creation is performed recursively from blkcg_root such
265  * that all non-root blkg's have access to the parent blkg.  This function
266  * should be called under RCU read lock and @q->queue_lock.
267  *
268  * Returns pointer to the looked up or created blkg on success, ERR_PTR()
269  * value on error.  If @q is dead, returns ERR_PTR(-EINVAL).  If @q is not
270  * dead and bypassing, returns ERR_PTR(-EBUSY).
271  */
272 struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
273 				    struct request_queue *q)
274 {
275 	struct blkcg_gq *blkg;
276 
277 	WARN_ON_ONCE(!rcu_read_lock_held());
278 	lockdep_assert_held(q->queue_lock);
279 
280 	/*
281 	 * This could be the first entry point of blkcg implementation and
282 	 * we shouldn't allow anything to go through for a bypassing queue.
283 	 */
284 	if (unlikely(blk_queue_bypass(q)))
285 		return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);
286 
287 	blkg = __blkg_lookup(blkcg, q, true);
288 	if (blkg)
289 		return blkg;
290 
291 	/*
292 	 * Create blkgs walking down from blkcg_root to @blkcg, so that all
293 	 * non-root blkgs have access to their parents.
294 	 */
295 	while (true) {
296 		struct blkcg *pos = blkcg;
297 		struct blkcg *parent = blkcg_parent(blkcg);
298 
299 		while (parent && !__blkg_lookup(parent, q, false)) {
300 			pos = parent;
301 			parent = blkcg_parent(parent);
302 		}
303 
304 		blkg = blkg_create(pos, q, NULL);
305 		if (pos == blkcg || IS_ERR(blkg))
306 			return blkg;
307 	}
308 }
309 
310 static void blkg_destroy(struct blkcg_gq *blkg)
311 {
312 	struct blkcg *blkcg = blkg->blkcg;
313 	struct blkcg_gq *parent = blkg->parent;
314 	int i;
315 
316 	lockdep_assert_held(blkg->q->queue_lock);
317 	lockdep_assert_held(&blkcg->lock);
318 
319 	/* Something wrong if we are trying to remove same group twice */
320 	WARN_ON_ONCE(list_empty(&blkg->q_node));
321 	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
322 
323 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
324 		struct blkcg_policy *pol = blkcg_policy[i];
325 
326 		if (blkg->pd[i] && pol->pd_offline_fn)
327 			pol->pd_offline_fn(blkg->pd[i]);
328 	}
329 
330 	if (parent) {
331 		blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
332 		blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
333 	}
334 
335 	blkg->online = false;
336 
337 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
338 	list_del_init(&blkg->q_node);
339 	hlist_del_init_rcu(&blkg->blkcg_node);
340 
341 	/*
342 	 * Both setting lookup hint to and clearing it from @blkg are done
343 	 * under queue_lock.  If it's not pointing to @blkg now, it never
344 	 * will.  Hint assignment itself can race safely.
345 	 */
346 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
347 		rcu_assign_pointer(blkcg->blkg_hint, NULL);
348 
349 	/*
350 	 * Put the reference taken at the time of creation so that when all
351 	 * queues are gone, group can be destroyed.
352 	 */
353 	blkg_put(blkg);
354 }
355 
356 /**
357  * blkg_destroy_all - destroy all blkgs associated with a request_queue
358  * @q: request_queue of interest
359  *
360  * Destroy all blkgs associated with @q.
361  */
362 static void blkg_destroy_all(struct request_queue *q)
363 {
364 	struct blkcg_gq *blkg, *n;
365 
366 	lockdep_assert_held(q->queue_lock);
367 
368 	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
369 		struct blkcg *blkcg = blkg->blkcg;
370 
371 		spin_lock(&blkcg->lock);
372 		blkg_destroy(blkg);
373 		spin_unlock(&blkcg->lock);
374 	}
375 
376 	q->root_blkg = NULL;
377 	q->root_rl.blkg = NULL;
378 }
379 
380 /*
381  * A group is RCU protected, but having an rcu lock does not mean that one
382  * can access all the fields of blkg and assume these are valid.  For
383  * example, don't try to follow throtl_data and request queue links.
384  *
385  * Having a reference to blkg under an rcu allows accesses to only values
386  * local to groups like group stats and group rate limits.
387  */
388 void __blkg_release_rcu(struct rcu_head *rcu_head)
389 {
390 	struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);
391 
392 	/* release the blkcg and parent blkg refs this blkg has been holding */
393 	css_put(&blkg->blkcg->css);
394 	if (blkg->parent)
395 		blkg_put(blkg->parent);
396 
397 	wb_congested_put(blkg->wb_congested);
398 
399 	blkg_free(blkg);
400 }
401 EXPORT_SYMBOL_GPL(__blkg_release_rcu);
402 
403 /*
404  * The next function used by blk_queue_for_each_rl().  It's a bit tricky
405  * because the root blkg uses @q->root_rl instead of its own rl.
406  */
407 struct request_list *__blk_queue_next_rl(struct request_list *rl,
408 					 struct request_queue *q)
409 {
410 	struct list_head *ent;
411 	struct blkcg_gq *blkg;
412 
413 	/*
414 	 * Determine the current blkg list_head.  The first entry is
415 	 * root_rl which is off @q->blkg_list and mapped to the head.
416 	 */
417 	if (rl == &q->root_rl) {
418 		ent = &q->blkg_list;
419 		/* There are no more block groups, hence no request lists */
420 		if (list_empty(ent))
421 			return NULL;
422 	} else {
423 		blkg = container_of(rl, struct blkcg_gq, rl);
424 		ent = &blkg->q_node;
425 	}
426 
427 	/* walk to the next list_head, skip root blkcg */
428 	ent = ent->next;
429 	if (ent == &q->root_blkg->q_node)
430 		ent = ent->next;
431 	if (ent == &q->blkg_list)
432 		return NULL;
433 
434 	blkg = container_of(ent, struct blkcg_gq, q_node);
435 	return &blkg->rl;
436 }
437 
438 static int blkcg_reset_stats(struct cgroup_subsys_state *css,
439 			     struct cftype *cftype, u64 val)
440 {
441 	struct blkcg *blkcg = css_to_blkcg(css);
442 	struct blkcg_gq *blkg;
443 	int i;
444 
445 	mutex_lock(&blkcg_pol_mutex);
446 	spin_lock_irq(&blkcg->lock);
447 
448 	/*
449 	 * Note that stat reset is racy - it doesn't synchronize against
450 	 * stat updates.  This is a debug feature which shouldn't exist
451 	 * anyway.  If you get hit by a race, retry.
452 	 */
453 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
454 		blkg_rwstat_reset(&blkg->stat_bytes);
455 		blkg_rwstat_reset(&blkg->stat_ios);
456 
457 		for (i = 0; i < BLKCG_MAX_POLS; i++) {
458 			struct blkcg_policy *pol = blkcg_policy[i];
459 
460 			if (blkg->pd[i] && pol->pd_reset_stats_fn)
461 				pol->pd_reset_stats_fn(blkg->pd[i]);
462 		}
463 	}
464 
465 	spin_unlock_irq(&blkcg->lock);
466 	mutex_unlock(&blkcg_pol_mutex);
467 	return 0;
468 }
469 
470 const char *blkg_dev_name(struct blkcg_gq *blkg)
471 {
472 	/* some drivers (floppy) instantiate a queue w/o disk registered */
473 	if (blkg->q->backing_dev_info->dev)
474 		return dev_name(blkg->q->backing_dev_info->dev);
475 	return NULL;
476 }
477 EXPORT_SYMBOL_GPL(blkg_dev_name);
478 
479 /**
480  * blkcg_print_blkgs - helper for printing per-blkg data
481  * @sf: seq_file to print to
482  * @blkcg: blkcg of interest
483  * @prfill: fill function to print out a blkg
484  * @pol: policy in question
485  * @data: data to be passed to @prfill
486  * @show_total: to print out sum of prfill return values or not
487  *
488  * This function invokes @prfill on each blkg of @blkcg if pd for the
489  * policy specified by @pol exists.  @prfill is invoked with @sf, the
490  * policy data and @data and the matching queue lock held.  If @show_total
491  * is %true, the sum of the return values from @prfill is printed with
492  * "Total" label at the end.
493  *
494  * This is to be used to construct print functions for
495  * cftype->read_seq_string method.
496  */
497 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
498 		       u64 (*prfill)(struct seq_file *,
499 				     struct blkg_policy_data *, int),
500 		       const struct blkcg_policy *pol, int data,
501 		       bool show_total)
502 {
503 	struct blkcg_gq *blkg;
504 	u64 total = 0;
505 
506 	rcu_read_lock();
507 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
508 		spin_lock_irq(blkg->q->queue_lock);
509 		if (blkcg_policy_enabled(blkg->q, pol))
510 			total += prfill(sf, blkg->pd[pol->plid], data);
511 		spin_unlock_irq(blkg->q->queue_lock);
512 	}
513 	rcu_read_unlock();
514 
515 	if (show_total)
516 		seq_printf(sf, "Total %llu\n", (unsigned long long)total);
517 }
518 EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
519 
520 /**
521  * __blkg_prfill_u64 - prfill helper for a single u64 value
522  * @sf: seq_file to print to
523  * @pd: policy private data of interest
524  * @v: value to print
525  *
526  * Print @v to @sf for the device assocaited with @pd.
527  */
528 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
529 {
530 	const char *dname = blkg_dev_name(pd->blkg);
531 
532 	if (!dname)
533 		return 0;
534 
535 	seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
536 	return v;
537 }
538 EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
539 
540 /**
541  * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
542  * @sf: seq_file to print to
543  * @pd: policy private data of interest
544  * @rwstat: rwstat to print
545  *
546  * Print @rwstat to @sf for the device assocaited with @pd.
547  */
548 u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
549 			 const struct blkg_rwstat *rwstat)
550 {
551 	static const char *rwstr[] = {
552 		[BLKG_RWSTAT_READ]	= "Read",
553 		[BLKG_RWSTAT_WRITE]	= "Write",
554 		[BLKG_RWSTAT_SYNC]	= "Sync",
555 		[BLKG_RWSTAT_ASYNC]	= "Async",
556 	};
557 	const char *dname = blkg_dev_name(pd->blkg);
558 	u64 v;
559 	int i;
560 
561 	if (!dname)
562 		return 0;
563 
564 	for (i = 0; i < BLKG_RWSTAT_NR; i++)
565 		seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
566 			   (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
567 
568 	v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
569 		atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]);
570 	seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
571 	return v;
572 }
573 EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
574 
575 /**
576  * blkg_prfill_stat - prfill callback for blkg_stat
577  * @sf: seq_file to print to
578  * @pd: policy private data of interest
579  * @off: offset to the blkg_stat in @pd
580  *
581  * prfill callback for printing a blkg_stat.
582  */
583 u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
584 {
585 	return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
586 }
587 EXPORT_SYMBOL_GPL(blkg_prfill_stat);
588 
589 /**
590  * blkg_prfill_rwstat - prfill callback for blkg_rwstat
591  * @sf: seq_file to print to
592  * @pd: policy private data of interest
593  * @off: offset to the blkg_rwstat in @pd
594  *
595  * prfill callback for printing a blkg_rwstat.
596  */
597 u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
598 		       int off)
599 {
600 	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
601 
602 	return __blkg_prfill_rwstat(sf, pd, &rwstat);
603 }
604 EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
605 
606 static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
607 				    struct blkg_policy_data *pd, int off)
608 {
609 	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);
610 
611 	return __blkg_prfill_rwstat(sf, pd, &rwstat);
612 }
613 
614 /**
615  * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
616  * @sf: seq_file to print to
617  * @v: unused
618  *
619  * To be used as cftype->seq_show to print blkg->stat_bytes.
620  * cftype->private must be set to the blkcg_policy.
621  */
622 int blkg_print_stat_bytes(struct seq_file *sf, void *v)
623 {
624 	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
625 			  blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
626 			  offsetof(struct blkcg_gq, stat_bytes), true);
627 	return 0;
628 }
629 EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);
630 
631 /**
632  * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
633  * @sf: seq_file to print to
634  * @v: unused
635  *
636  * To be used as cftype->seq_show to print blkg->stat_ios.  cftype->private
637  * must be set to the blkcg_policy.
638  */
639 int blkg_print_stat_ios(struct seq_file *sf, void *v)
640 {
641 	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
642 			  blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
643 			  offsetof(struct blkcg_gq, stat_ios), true);
644 	return 0;
645 }
646 EXPORT_SYMBOL_GPL(blkg_print_stat_ios);
647 
648 static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
649 					      struct blkg_policy_data *pd,
650 					      int off)
651 {
652 	struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
653 							      NULL, off);
654 	return __blkg_prfill_rwstat(sf, pd, &rwstat);
655 }
656 
657 /**
658  * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
659  * @sf: seq_file to print to
660  * @v: unused
661  */
662 int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
663 {
664 	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
665 			  blkg_prfill_rwstat_field_recursive,
666 			  (void *)seq_cft(sf)->private,
667 			  offsetof(struct blkcg_gq, stat_bytes), true);
668 	return 0;
669 }
670 EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);
671 
672 /**
673  * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
674  * @sf: seq_file to print to
675  * @v: unused
676  */
677 int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
678 {
679 	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
680 			  blkg_prfill_rwstat_field_recursive,
681 			  (void *)seq_cft(sf)->private,
682 			  offsetof(struct blkcg_gq, stat_ios), true);
683 	return 0;
684 }
685 EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);
686 
687 /**
688  * blkg_stat_recursive_sum - collect hierarchical blkg_stat
689  * @blkg: blkg of interest
690  * @pol: blkcg_policy which contains the blkg_stat
691  * @off: offset to the blkg_stat in blkg_policy_data or @blkg
692  *
693  * Collect the blkg_stat specified by @blkg, @pol and @off and all its
694  * online descendants and their aux counts.  The caller must be holding the
695  * queue lock for online tests.
696  *
697  * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
698  * at @off bytes into @blkg's blkg_policy_data of the policy.
699  */
700 u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
701 			    struct blkcg_policy *pol, int off)
702 {
703 	struct blkcg_gq *pos_blkg;
704 	struct cgroup_subsys_state *pos_css;
705 	u64 sum = 0;
706 
707 	lockdep_assert_held(blkg->q->queue_lock);
708 
709 	rcu_read_lock();
710 	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
711 		struct blkg_stat *stat;
712 
713 		if (!pos_blkg->online)
714 			continue;
715 
716 		if (pol)
717 			stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
718 		else
719 			stat = (void *)blkg + off;
720 
721 		sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
722 	}
723 	rcu_read_unlock();
724 
725 	return sum;
726 }
727 EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
728 
729 /**
730  * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
731  * @blkg: blkg of interest
732  * @pol: blkcg_policy which contains the blkg_rwstat
733  * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
734  *
735  * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
736  * online descendants and their aux counts.  The caller must be holding the
737  * queue lock for online tests.
738  *
739  * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
740  * is at @off bytes into @blkg's blkg_policy_data of the policy.
741  */
742 struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
743 					     struct blkcg_policy *pol, int off)
744 {
745 	struct blkcg_gq *pos_blkg;
746 	struct cgroup_subsys_state *pos_css;
747 	struct blkg_rwstat sum = { };
748 	int i;
749 
750 	lockdep_assert_held(blkg->q->queue_lock);
751 
752 	rcu_read_lock();
753 	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
754 		struct blkg_rwstat *rwstat;
755 
756 		if (!pos_blkg->online)
757 			continue;
758 
759 		if (pol)
760 			rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
761 		else
762 			rwstat = (void *)pos_blkg + off;
763 
764 		for (i = 0; i < BLKG_RWSTAT_NR; i++)
765 			atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
766 				percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
767 				&sum.aux_cnt[i]);
768 	}
769 	rcu_read_unlock();
770 
771 	return sum;
772 }
773 EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
774 
775 /* Performs queue bypass and policy enabled checks then looks up blkg. */
776 static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
777 					  const struct blkcg_policy *pol,
778 					  struct request_queue *q)
779 {
780 	WARN_ON_ONCE(!rcu_read_lock_held());
781 	lockdep_assert_held(q->queue_lock);
782 
783 	if (!blkcg_policy_enabled(q, pol))
784 		return ERR_PTR(-EOPNOTSUPP);
785 
786 	/*
787 	 * This could be the first entry point of blkcg implementation and
788 	 * we shouldn't allow anything to go through for a bypassing queue.
789 	 */
790 	if (unlikely(blk_queue_bypass(q)))
791 		return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);
792 
793 	return __blkg_lookup(blkcg, q, true /* update_hint */);
794 }
795 
796 /**
797  * blkg_conf_prep - parse and prepare for per-blkg config update
798  * @blkcg: target block cgroup
799  * @pol: target policy
800  * @input: input string
801  * @ctx: blkg_conf_ctx to be filled
802  *
803  * Parse per-blkg config update from @input and initialize @ctx with the
804  * result.  @ctx->blkg points to the blkg to be updated and @ctx->body the
805  * part of @input following MAJ:MIN.  This function returns with RCU read
806  * lock and queue lock held and must be paired with blkg_conf_finish().
807  */
808 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
809 		   char *input, struct blkg_conf_ctx *ctx)
810 	__acquires(rcu) __acquires(disk->queue->queue_lock)
811 {
812 	struct gendisk *disk;
813 	struct request_queue *q;
814 	struct blkcg_gq *blkg;
815 	unsigned int major, minor;
816 	int key_len, part, ret;
817 	char *body;
818 
819 	if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
820 		return -EINVAL;
821 
822 	body = input + key_len;
823 	if (!isspace(*body))
824 		return -EINVAL;
825 	body = skip_spaces(body);
826 
827 	disk = get_gendisk(MKDEV(major, minor), &part);
828 	if (!disk)
829 		return -ENODEV;
830 	if (part) {
831 		ret = -ENODEV;
832 		goto fail;
833 	}
834 
835 	q = disk->queue;
836 
837 	rcu_read_lock();
838 	spin_lock_irq(q->queue_lock);
839 
840 	blkg = blkg_lookup_check(blkcg, pol, q);
841 	if (IS_ERR(blkg)) {
842 		ret = PTR_ERR(blkg);
843 		goto fail_unlock;
844 	}
845 
846 	if (blkg)
847 		goto success;
848 
849 	/*
850 	 * Create blkgs walking down from blkcg_root to @blkcg, so that all
851 	 * non-root blkgs have access to their parents.
852 	 */
853 	while (true) {
854 		struct blkcg *pos = blkcg;
855 		struct blkcg *parent;
856 		struct blkcg_gq *new_blkg;
857 
858 		parent = blkcg_parent(blkcg);
859 		while (parent && !__blkg_lookup(parent, q, false)) {
860 			pos = parent;
861 			parent = blkcg_parent(parent);
862 		}
863 
864 		/* Drop locks to do new blkg allocation with GFP_KERNEL. */
865 		spin_unlock_irq(q->queue_lock);
866 		rcu_read_unlock();
867 
868 		new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
869 		if (unlikely(!new_blkg)) {
870 			ret = -ENOMEM;
871 			goto fail;
872 		}
873 
874 		rcu_read_lock();
875 		spin_lock_irq(q->queue_lock);
876 
877 		blkg = blkg_lookup_check(pos, pol, q);
878 		if (IS_ERR(blkg)) {
879 			ret = PTR_ERR(blkg);
880 			goto fail_unlock;
881 		}
882 
883 		if (blkg) {
884 			blkg_free(new_blkg);
885 		} else {
886 			blkg = blkg_create(pos, q, new_blkg);
887 			if (unlikely(IS_ERR(blkg))) {
888 				ret = PTR_ERR(blkg);
889 				goto fail_unlock;
890 			}
891 		}
892 
893 		if (pos == blkcg)
894 			goto success;
895 	}
896 success:
897 	ctx->disk = disk;
898 	ctx->blkg = blkg;
899 	ctx->body = body;
900 	return 0;
901 
902 fail_unlock:
903 	spin_unlock_irq(q->queue_lock);
904 	rcu_read_unlock();
905 fail:
906 	put_disk_and_module(disk);
907 	/*
908 	 * If queue was bypassing, we should retry.  Do so after a
909 	 * short msleep().  It isn't strictly necessary but queue
910 	 * can be bypassing for some time and it's always nice to
911 	 * avoid busy looping.
912 	 */
913 	if (ret == -EBUSY) {
914 		msleep(10);
915 		ret = restart_syscall();
916 	}
917 	return ret;
918 }
919 EXPORT_SYMBOL_GPL(blkg_conf_prep);
920 
921 /**
922  * blkg_conf_finish - finish up per-blkg config update
923  * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
924  *
925  * Finish up after per-blkg config update.  This function must be paired
926  * with blkg_conf_prep().
927  */
928 void blkg_conf_finish(struct blkg_conf_ctx *ctx)
929 	__releases(ctx->disk->queue->queue_lock) __releases(rcu)
930 {
931 	spin_unlock_irq(ctx->disk->queue->queue_lock);
932 	rcu_read_unlock();
933 	put_disk_and_module(ctx->disk);
934 }
935 EXPORT_SYMBOL_GPL(blkg_conf_finish);
936 
937 static int blkcg_print_stat(struct seq_file *sf, void *v)
938 {
939 	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
940 	struct blkcg_gq *blkg;
941 
942 	rcu_read_lock();
943 
944 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
945 		const char *dname;
946 		struct blkg_rwstat rwstat;
947 		u64 rbytes, wbytes, rios, wios;
948 
949 		dname = blkg_dev_name(blkg);
950 		if (!dname)
951 			continue;
952 
953 		spin_lock_irq(blkg->q->queue_lock);
954 
955 		rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
956 					offsetof(struct blkcg_gq, stat_bytes));
957 		rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
958 		wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
959 
960 		rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
961 					offsetof(struct blkcg_gq, stat_ios));
962 		rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
963 		wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
964 
965 		spin_unlock_irq(blkg->q->queue_lock);
966 
967 		if (rbytes || wbytes || rios || wios)
968 			seq_printf(sf, "%s rbytes=%llu wbytes=%llu rios=%llu wios=%llu\n",
969 				   dname, rbytes, wbytes, rios, wios);
970 	}
971 
972 	rcu_read_unlock();
973 	return 0;
974 }
975 
976 static struct cftype blkcg_files[] = {
977 	{
978 		.name = "stat",
979 		.flags = CFTYPE_NOT_ON_ROOT,
980 		.seq_show = blkcg_print_stat,
981 	},
982 	{ }	/* terminate */
983 };
984 
985 static struct cftype blkcg_legacy_files[] = {
986 	{
987 		.name = "reset_stats",
988 		.write_u64 = blkcg_reset_stats,
989 	},
990 	{ }	/* terminate */
991 };
992 
993 /**
994  * blkcg_css_offline - cgroup css_offline callback
995  * @css: css of interest
996  *
997  * This function is called when @css is about to go away and responsible
998  * for shooting down all blkgs associated with @css.  blkgs should be
999  * removed while holding both q and blkcg locks.  As blkcg lock is nested
1000  * inside q lock, this function performs reverse double lock dancing.
1001  *
1002  * This is the blkcg counterpart of ioc_release_fn().
1003  */
1004 static void blkcg_css_offline(struct cgroup_subsys_state *css)
1005 {
1006 	struct blkcg *blkcg = css_to_blkcg(css);
1007 
1008 	spin_lock_irq(&blkcg->lock);
1009 
1010 	while (!hlist_empty(&blkcg->blkg_list)) {
1011 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1012 						struct blkcg_gq, blkcg_node);
1013 		struct request_queue *q = blkg->q;
1014 
1015 		if (spin_trylock(q->queue_lock)) {
1016 			blkg_destroy(blkg);
1017 			spin_unlock(q->queue_lock);
1018 		} else {
1019 			spin_unlock_irq(&blkcg->lock);
1020 			cpu_relax();
1021 			spin_lock_irq(&blkcg->lock);
1022 		}
1023 	}
1024 
1025 	spin_unlock_irq(&blkcg->lock);
1026 
1027 	wb_blkcg_offline(blkcg);
1028 }
1029 
1030 static void blkcg_css_free(struct cgroup_subsys_state *css)
1031 {
1032 	struct blkcg *blkcg = css_to_blkcg(css);
1033 	int i;
1034 
1035 	mutex_lock(&blkcg_pol_mutex);
1036 
1037 	list_del(&blkcg->all_blkcgs_node);
1038 
1039 	for (i = 0; i < BLKCG_MAX_POLS; i++)
1040 		if (blkcg->cpd[i])
1041 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1042 
1043 	mutex_unlock(&blkcg_pol_mutex);
1044 
1045 	kfree(blkcg);
1046 }
1047 
1048 static struct cgroup_subsys_state *
1049 blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
1050 {
1051 	struct blkcg *blkcg;
1052 	struct cgroup_subsys_state *ret;
1053 	int i;
1054 
1055 	mutex_lock(&blkcg_pol_mutex);
1056 
1057 	if (!parent_css) {
1058 		blkcg = &blkcg_root;
1059 	} else {
1060 		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1061 		if (!blkcg) {
1062 			ret = ERR_PTR(-ENOMEM);
1063 			goto unlock;
1064 		}
1065 	}
1066 
1067 	for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1068 		struct blkcg_policy *pol = blkcg_policy[i];
1069 		struct blkcg_policy_data *cpd;
1070 
1071 		/*
1072 		 * If the policy hasn't been attached yet, wait for it
1073 		 * to be attached before doing anything else. Otherwise,
1074 		 * check if the policy requires any specific per-cgroup
1075 		 * data: if it does, allocate and initialize it.
1076 		 */
1077 		if (!pol || !pol->cpd_alloc_fn)
1078 			continue;
1079 
1080 		cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1081 		if (!cpd) {
1082 			ret = ERR_PTR(-ENOMEM);
1083 			goto free_pd_blkcg;
1084 		}
1085 		blkcg->cpd[i] = cpd;
1086 		cpd->blkcg = blkcg;
1087 		cpd->plid = i;
1088 		if (pol->cpd_init_fn)
1089 			pol->cpd_init_fn(cpd);
1090 	}
1091 
1092 	spin_lock_init(&blkcg->lock);
1093 	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
1094 	INIT_HLIST_HEAD(&blkcg->blkg_list);
1095 #ifdef CONFIG_CGROUP_WRITEBACK
1096 	INIT_LIST_HEAD(&blkcg->cgwb_list);
1097 #endif
1098 	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1099 
1100 	mutex_unlock(&blkcg_pol_mutex);
1101 	return &blkcg->css;
1102 
1103 free_pd_blkcg:
1104 	for (i--; i >= 0; i--)
1105 		if (blkcg->cpd[i])
1106 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1107 
1108 	if (blkcg != &blkcg_root)
1109 		kfree(blkcg);
1110 unlock:
1111 	mutex_unlock(&blkcg_pol_mutex);
1112 	return ret;
1113 }
1114 
1115 /**
1116  * blkcg_init_queue - initialize blkcg part of request queue
1117  * @q: request_queue to initialize
1118  *
1119  * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
1120  * part of new request_queue @q.
1121  *
1122  * RETURNS:
1123  * 0 on success, -errno on failure.
1124  */
1125 int blkcg_init_queue(struct request_queue *q)
1126 {
1127 	struct blkcg_gq *new_blkg, *blkg;
1128 	bool preloaded;
1129 	int ret;
1130 
1131 	new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1132 	if (!new_blkg)
1133 		return -ENOMEM;
1134 
1135 	preloaded = !radix_tree_preload(GFP_KERNEL);
1136 
1137 	/*
1138 	 * Make sure the root blkg exists and count the existing blkgs.  As
1139 	 * @q is bypassing at this point, blkg_lookup_create() can't be
1140 	 * used.  Open code insertion.
1141 	 */
1142 	rcu_read_lock();
1143 	spin_lock_irq(q->queue_lock);
1144 	blkg = blkg_create(&blkcg_root, q, new_blkg);
1145 	spin_unlock_irq(q->queue_lock);
1146 	rcu_read_unlock();
1147 
1148 	if (preloaded)
1149 		radix_tree_preload_end();
1150 
1151 	if (IS_ERR(blkg))
1152 		return PTR_ERR(blkg);
1153 
1154 	q->root_blkg = blkg;
1155 	q->root_rl.blkg = blkg;
1156 
1157 	ret = blk_throtl_init(q);
1158 	if (ret) {
1159 		spin_lock_irq(q->queue_lock);
1160 		blkg_destroy_all(q);
1161 		spin_unlock_irq(q->queue_lock);
1162 	}
1163 	return ret;
1164 }
1165 
1166 /**
1167  * blkcg_drain_queue - drain blkcg part of request_queue
1168  * @q: request_queue to drain
1169  *
1170  * Called from blk_drain_queue().  Responsible for draining blkcg part.
1171  */
1172 void blkcg_drain_queue(struct request_queue *q)
1173 {
1174 	lockdep_assert_held(q->queue_lock);
1175 
1176 	/*
1177 	 * @q could be exiting and already have destroyed all blkgs as
1178 	 * indicated by NULL root_blkg.  If so, don't confuse policies.
1179 	 */
1180 	if (!q->root_blkg)
1181 		return;
1182 
1183 	blk_throtl_drain(q);
1184 }
1185 
1186 /**
1187  * blkcg_exit_queue - exit and release blkcg part of request_queue
1188  * @q: request_queue being released
1189  *
1190  * Called from blk_release_queue().  Responsible for exiting blkcg part.
1191  */
1192 void blkcg_exit_queue(struct request_queue *q)
1193 {
1194 	spin_lock_irq(q->queue_lock);
1195 	blkg_destroy_all(q);
1196 	spin_unlock_irq(q->queue_lock);
1197 
1198 	blk_throtl_exit(q);
1199 }
1200 
1201 /*
1202  * We cannot support shared io contexts, as we have no mean to support
1203  * two tasks with the same ioc in two different groups without major rework
1204  * of the main cic data structures.  For now we allow a task to change
1205  * its cgroup only if it's the only owner of its ioc.
1206  */
1207 static int blkcg_can_attach(struct cgroup_taskset *tset)
1208 {
1209 	struct task_struct *task;
1210 	struct cgroup_subsys_state *dst_css;
1211 	struct io_context *ioc;
1212 	int ret = 0;
1213 
1214 	/* task_lock() is needed to avoid races with exit_io_context() */
1215 	cgroup_taskset_for_each(task, dst_css, tset) {
1216 		task_lock(task);
1217 		ioc = task->io_context;
1218 		if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1219 			ret = -EINVAL;
1220 		task_unlock(task);
1221 		if (ret)
1222 			break;
1223 	}
1224 	return ret;
1225 }
1226 
1227 static void blkcg_bind(struct cgroup_subsys_state *root_css)
1228 {
1229 	int i;
1230 
1231 	mutex_lock(&blkcg_pol_mutex);
1232 
1233 	for (i = 0; i < BLKCG_MAX_POLS; i++) {
1234 		struct blkcg_policy *pol = blkcg_policy[i];
1235 		struct blkcg *blkcg;
1236 
1237 		if (!pol || !pol->cpd_bind_fn)
1238 			continue;
1239 
1240 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1241 			if (blkcg->cpd[pol->plid])
1242 				pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1243 	}
1244 	mutex_unlock(&blkcg_pol_mutex);
1245 }
1246 
1247 struct cgroup_subsys io_cgrp_subsys = {
1248 	.css_alloc = blkcg_css_alloc,
1249 	.css_offline = blkcg_css_offline,
1250 	.css_free = blkcg_css_free,
1251 	.can_attach = blkcg_can_attach,
1252 	.bind = blkcg_bind,
1253 	.dfl_cftypes = blkcg_files,
1254 	.legacy_cftypes = blkcg_legacy_files,
1255 	.legacy_name = "blkio",
1256 #ifdef CONFIG_MEMCG
1257 	/*
1258 	 * This ensures that, if available, memcg is automatically enabled
1259 	 * together on the default hierarchy so that the owner cgroup can
1260 	 * be retrieved from writeback pages.
1261 	 */
1262 	.depends_on = 1 << memory_cgrp_id,
1263 #endif
1264 };
1265 EXPORT_SYMBOL_GPL(io_cgrp_subsys);
1266 
1267 /**
1268  * blkcg_activate_policy - activate a blkcg policy on a request_queue
1269  * @q: request_queue of interest
1270  * @pol: blkcg policy to activate
1271  *
1272  * Activate @pol on @q.  Requires %GFP_KERNEL context.  @q goes through
1273  * bypass mode to populate its blkgs with policy_data for @pol.
1274  *
1275  * Activation happens with @q bypassed, so nobody would be accessing blkgs
1276  * from IO path.  Update of each blkg is protected by both queue and blkcg
1277  * locks so that holding either lock and testing blkcg_policy_enabled() is
1278  * always enough for dereferencing policy data.
1279  *
1280  * The caller is responsible for synchronizing [de]activations and policy
1281  * [un]registerations.  Returns 0 on success, -errno on failure.
1282  */
1283 int blkcg_activate_policy(struct request_queue *q,
1284 			  const struct blkcg_policy *pol)
1285 {
1286 	struct blkg_policy_data *pd_prealloc = NULL;
1287 	struct blkcg_gq *blkg;
1288 	int ret;
1289 
1290 	if (blkcg_policy_enabled(q, pol))
1291 		return 0;
1292 
1293 	if (q->mq_ops)
1294 		blk_mq_freeze_queue(q);
1295 	else
1296 		blk_queue_bypass_start(q);
1297 pd_prealloc:
1298 	if (!pd_prealloc) {
1299 		pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
1300 		if (!pd_prealloc) {
1301 			ret = -ENOMEM;
1302 			goto out_bypass_end;
1303 		}
1304 	}
1305 
1306 	spin_lock_irq(q->queue_lock);
1307 
1308 	list_for_each_entry(blkg, &q->blkg_list, q_node) {
1309 		struct blkg_policy_data *pd;
1310 
1311 		if (blkg->pd[pol->plid])
1312 			continue;
1313 
1314 		pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q->node);
1315 		if (!pd)
1316 			swap(pd, pd_prealloc);
1317 		if (!pd) {
1318 			spin_unlock_irq(q->queue_lock);
1319 			goto pd_prealloc;
1320 		}
1321 
1322 		blkg->pd[pol->plid] = pd;
1323 		pd->blkg = blkg;
1324 		pd->plid = pol->plid;
1325 		if (pol->pd_init_fn)
1326 			pol->pd_init_fn(pd);
1327 	}
1328 
1329 	__set_bit(pol->plid, q->blkcg_pols);
1330 	ret = 0;
1331 
1332 	spin_unlock_irq(q->queue_lock);
1333 out_bypass_end:
1334 	if (q->mq_ops)
1335 		blk_mq_unfreeze_queue(q);
1336 	else
1337 		blk_queue_bypass_end(q);
1338 	if (pd_prealloc)
1339 		pol->pd_free_fn(pd_prealloc);
1340 	return ret;
1341 }
1342 EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1343 
1344 /**
1345  * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1346  * @q: request_queue of interest
1347  * @pol: blkcg policy to deactivate
1348  *
1349  * Deactivate @pol on @q.  Follows the same synchronization rules as
1350  * blkcg_activate_policy().
1351  */
1352 void blkcg_deactivate_policy(struct request_queue *q,
1353 			     const struct blkcg_policy *pol)
1354 {
1355 	struct blkcg_gq *blkg;
1356 
1357 	if (!blkcg_policy_enabled(q, pol))
1358 		return;
1359 
1360 	if (q->mq_ops)
1361 		blk_mq_freeze_queue(q);
1362 	else
1363 		blk_queue_bypass_start(q);
1364 
1365 	spin_lock_irq(q->queue_lock);
1366 
1367 	__clear_bit(pol->plid, q->blkcg_pols);
1368 
1369 	list_for_each_entry(blkg, &q->blkg_list, q_node) {
1370 		/* grab blkcg lock too while removing @pd from @blkg */
1371 		spin_lock(&blkg->blkcg->lock);
1372 
1373 		if (blkg->pd[pol->plid]) {
1374 			if (pol->pd_offline_fn)
1375 				pol->pd_offline_fn(blkg->pd[pol->plid]);
1376 			pol->pd_free_fn(blkg->pd[pol->plid]);
1377 			blkg->pd[pol->plid] = NULL;
1378 		}
1379 
1380 		spin_unlock(&blkg->blkcg->lock);
1381 	}
1382 
1383 	spin_unlock_irq(q->queue_lock);
1384 
1385 	if (q->mq_ops)
1386 		blk_mq_unfreeze_queue(q);
1387 	else
1388 		blk_queue_bypass_end(q);
1389 }
1390 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1391 
1392 /**
1393  * blkcg_policy_register - register a blkcg policy
1394  * @pol: blkcg policy to register
1395  *
1396  * Register @pol with blkcg core.  Might sleep and @pol may be modified on
1397  * successful registration.  Returns 0 on success and -errno on failure.
1398  */
1399 int blkcg_policy_register(struct blkcg_policy *pol)
1400 {
1401 	struct blkcg *blkcg;
1402 	int i, ret;
1403 
1404 	mutex_lock(&blkcg_pol_register_mutex);
1405 	mutex_lock(&blkcg_pol_mutex);
1406 
1407 	/* find an empty slot */
1408 	ret = -ENOSPC;
1409 	for (i = 0; i < BLKCG_MAX_POLS; i++)
1410 		if (!blkcg_policy[i])
1411 			break;
1412 	if (i >= BLKCG_MAX_POLS)
1413 		goto err_unlock;
1414 
1415 	/* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1416 	if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1417 		(!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1418 		goto err_unlock;
1419 
1420 	/* register @pol */
1421 	pol->plid = i;
1422 	blkcg_policy[pol->plid] = pol;
1423 
1424 	/* allocate and install cpd's */
1425 	if (pol->cpd_alloc_fn) {
1426 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1427 			struct blkcg_policy_data *cpd;
1428 
1429 			cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1430 			if (!cpd)
1431 				goto err_free_cpds;
1432 
1433 			blkcg->cpd[pol->plid] = cpd;
1434 			cpd->blkcg = blkcg;
1435 			cpd->plid = pol->plid;
1436 			pol->cpd_init_fn(cpd);
1437 		}
1438 	}
1439 
1440 	mutex_unlock(&blkcg_pol_mutex);
1441 
1442 	/* everything is in place, add intf files for the new policy */
1443 	if (pol->dfl_cftypes)
1444 		WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1445 					       pol->dfl_cftypes));
1446 	if (pol->legacy_cftypes)
1447 		WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
1448 						  pol->legacy_cftypes));
1449 	mutex_unlock(&blkcg_pol_register_mutex);
1450 	return 0;
1451 
1452 err_free_cpds:
1453 	if (pol->cpd_free_fn) {
1454 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1455 			if (blkcg->cpd[pol->plid]) {
1456 				pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1457 				blkcg->cpd[pol->plid] = NULL;
1458 			}
1459 		}
1460 	}
1461 	blkcg_policy[pol->plid] = NULL;
1462 err_unlock:
1463 	mutex_unlock(&blkcg_pol_mutex);
1464 	mutex_unlock(&blkcg_pol_register_mutex);
1465 	return ret;
1466 }
1467 EXPORT_SYMBOL_GPL(blkcg_policy_register);
1468 
1469 /**
1470  * blkcg_policy_unregister - unregister a blkcg policy
1471  * @pol: blkcg policy to unregister
1472  *
1473  * Undo blkcg_policy_register(@pol).  Might sleep.
1474  */
1475 void blkcg_policy_unregister(struct blkcg_policy *pol)
1476 {
1477 	struct blkcg *blkcg;
1478 
1479 	mutex_lock(&blkcg_pol_register_mutex);
1480 
1481 	if (WARN_ON(blkcg_policy[pol->plid] != pol))
1482 		goto out_unlock;
1483 
1484 	/* kill the intf files first */
1485 	if (pol->dfl_cftypes)
1486 		cgroup_rm_cftypes(pol->dfl_cftypes);
1487 	if (pol->legacy_cftypes)
1488 		cgroup_rm_cftypes(pol->legacy_cftypes);
1489 
1490 	/* remove cpds and unregister */
1491 	mutex_lock(&blkcg_pol_mutex);
1492 
1493 	if (pol->cpd_free_fn) {
1494 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1495 			if (blkcg->cpd[pol->plid]) {
1496 				pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1497 				blkcg->cpd[pol->plid] = NULL;
1498 			}
1499 		}
1500 	}
1501 	blkcg_policy[pol->plid] = NULL;
1502 
1503 	mutex_unlock(&blkcg_pol_mutex);
1504 out_unlock:
1505 	mutex_unlock(&blkcg_pol_register_mutex);
1506 }
1507 EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
1508