1 /* SPDX-License-Identifier: GPL-2.0 */
17 #include <linux/blk-cgroup.h>
20 #include <linux/blk-mq.h>
27 /* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
62 	/* all non-root blkcg_gq's are guaranteed to have access to parent */
74 	struct blkg_policy_data		*pd[BLKCG_MAX_POLS];  member
127  * information per blkcg - q pair.
136 	/* the blkg and policy id this per-policy data belongs to */
143  * Policies that need to keep per-blkcg data which is independent from any
147  * cpd_init() is invoked to let each policy handle per-blkcg data.
150 	/* the blkcg and policy id this per-policy data belongs to */
161 typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
162 typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
163 typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
164 typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
165 typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
166 typedef void (blkcg_pol_stat_pd_fn)(struct blkg_policy_data *pd,
208 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
224  * bio_issue_as_root_blkg - see if this bio needs to be issued as root blkg
236 	return (bio->bi_opf & (REQ_META | REQ_SWAP)) != 0;  in bio_issue_as_root_blkg()
240  * blkg_lookup - lookup blkg for the specified blkcg - q pair
244  * Lookup blkg for the @blkcg - @q pair.
254 		return q->root_blkg;  in blkg_lookup()
256 	blkg = rcu_dereference_check(blkcg->blkg_hint,  in blkg_lookup()
257 			lockdep_is_held(&q->queue_lock));  in blkg_lookup()
258 	if (blkg && blkg->q == q)  in blkg_lookup()
261 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);  in blkg_lookup()
262 	if (blkg && blkg->q != q)  in blkg_lookup()
268  * blkg_to_pdata - get policy private data
272  * Return pointer to private data associated with the @blkg-@pol pair.
277 	return blkg ? blkg->pd[pol->plid] : NULL;  in blkg_to_pd()
283 	return blkcg ? blkcg->cpd[pol->plid] : NULL;  in blkcg_to_cpd()
287  * pdata_to_blkg - get blkg associated with policy private data
288  * @pd: policy private data of interest
290  * @pd is policy private data.  Determine the blkg it's associated with.
292 static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)  in pd_to_blkg()  argument
294 	return pd ? pd->blkg : NULL;  in pd_to_blkg()
299 	return cpd ? cpd->blkcg : NULL;  in cpd_to_blkcg()
303  * blkg_path - format cgroup path of blkg
312 	return cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);  in blkg_path()
316  * blkg_get - get a blkg reference
323 	percpu_ref_get(&blkg->refcnt);  in blkg_get()
327  * blkg_tryget - try and get a blkg reference
335 	return blkg && percpu_ref_tryget(&blkg->refcnt);  in blkg_tryget()
339  * blkg_put - put a blkg reference
344 	percpu_ref_put(&blkg->refcnt);  in blkg_put()
348  * blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants
360 	css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css)	\
362 					    (p_blkg)->q)))
365  * blkg_for_each_descendant_post - post-order walk of a blkg's descendants
370  * Similar to blkg_for_each_descendant_pre() but performs post-order
375 	css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css)	\
377 					    (p_blkg)->q)))
381 	bio_issue_init(&bio->bi_issue, bio_sectors(bio));  in blkcg_bio_issue_init()
386 	if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))  in blkcg_use_delay()
388 	if (atomic_add_return(1, &blkg->use_delay) == 1)  in blkcg_use_delay()
389 		atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);  in blkcg_use_delay()
394 	int old = atomic_read(&blkg->use_delay);  in blkcg_unuse_delay()
408 	while (old && !atomic_try_cmpxchg(&blkg->use_delay, &old, old - 1))  in blkcg_unuse_delay()
414 		atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);  in blkcg_unuse_delay()
419  * blkcg_set_delay - Enable allocator delay mechanism with the specified delay amount
429 	int old = atomic_read(&blkg->use_delay);  in blkcg_set_delay()
432 	if (!old && atomic_try_cmpxchg(&blkg->use_delay, &old, -1))  in blkcg_set_delay()
433 		atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);  in blkcg_set_delay()
435 	atomic64_set(&blkg->delay_nsec, delay);  in blkcg_set_delay()
439  * blkcg_clear_delay - Disable allocator delay mechanism
442  * Disable use_delay mechanism. See blkcg_set_delay().
446 	int old = atomic_read(&blkg->use_delay);  in blkcg_clear_delay()
449 	if (old && atomic_try_cmpxchg(&blkg->use_delay, &old, 0))  in blkcg_clear_delay()
450 		atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);  in blkcg_clear_delay()
454  * blk_cgroup_mergeable - Determine whether to allow or disallow merges
464 	return rq->bio->bi_blkg == bio->bi_blkg &&  in blk_cgroup_mergeable()
465 		bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio);  in blk_cgroup_mergeable()
497 static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }  in pd_to_blkg()  argument
506 	for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)