xref: /openbmc/linux/mm/backing-dev.c (revision 9b9c2cd4)
1 
2 #include <linux/wait.h>
3 #include <linux/backing-dev.h>
4 #include <linux/kthread.h>
5 #include <linux/freezer.h>
6 #include <linux/fs.h>
7 #include <linux/pagemap.h>
8 #include <linux/mm.h>
9 #include <linux/sched.h>
10 #include <linux/module.h>
11 #include <linux/writeback.h>
12 #include <linux/device.h>
13 #include <trace/events/writeback.h>
14 
15 static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
16 
17 struct backing_dev_info noop_backing_dev_info = {
18 	.name		= "noop",
19 	.capabilities	= BDI_CAP_NO_ACCT_AND_WRITEBACK,
20 };
21 EXPORT_SYMBOL_GPL(noop_backing_dev_info);
22 
23 static struct class *bdi_class;
24 
25 /*
26  * bdi_lock protects updates to bdi_list. bdi_list has RCU reader side
27  * locking.
28  */
29 DEFINE_SPINLOCK(bdi_lock);
30 LIST_HEAD(bdi_list);
31 
32 /* bdi_wq serves all asynchronous writeback tasks */
33 struct workqueue_struct *bdi_wq;
34 
35 #ifdef CONFIG_DEBUG_FS
36 #include <linux/debugfs.h>
37 #include <linux/seq_file.h>
38 
39 static struct dentry *bdi_debug_root;
40 
41 static void bdi_debug_init(void)
42 {
43 	bdi_debug_root = debugfs_create_dir("bdi", NULL);
44 }
45 
46 static int bdi_debug_stats_show(struct seq_file *m, void *v)
47 {
48 	struct backing_dev_info *bdi = m->private;
49 	struct bdi_writeback *wb = &bdi->wb;
50 	unsigned long background_thresh;
51 	unsigned long dirty_thresh;
52 	unsigned long wb_thresh;
53 	unsigned long nr_dirty, nr_io, nr_more_io, nr_dirty_time;
54 	struct inode *inode;
55 
56 	nr_dirty = nr_io = nr_more_io = nr_dirty_time = 0;
57 	spin_lock(&wb->list_lock);
58 	list_for_each_entry(inode, &wb->b_dirty, i_io_list)
59 		nr_dirty++;
60 	list_for_each_entry(inode, &wb->b_io, i_io_list)
61 		nr_io++;
62 	list_for_each_entry(inode, &wb->b_more_io, i_io_list)
63 		nr_more_io++;
64 	list_for_each_entry(inode, &wb->b_dirty_time, i_io_list)
65 		if (inode->i_state & I_DIRTY_TIME)
66 			nr_dirty_time++;
67 	spin_unlock(&wb->list_lock);
68 
69 	global_dirty_limits(&background_thresh, &dirty_thresh);
70 	wb_thresh = wb_calc_thresh(wb, dirty_thresh);
71 
72 #define K(x) ((x) << (PAGE_SHIFT - 10))
73 	seq_printf(m,
74 		   "BdiWriteback:       %10lu kB\n"
75 		   "BdiReclaimable:     %10lu kB\n"
76 		   "BdiDirtyThresh:     %10lu kB\n"
77 		   "DirtyThresh:        %10lu kB\n"
78 		   "BackgroundThresh:   %10lu kB\n"
79 		   "BdiDirtied:         %10lu kB\n"
80 		   "BdiWritten:         %10lu kB\n"
81 		   "BdiWriteBandwidth:  %10lu kBps\n"
82 		   "b_dirty:            %10lu\n"
83 		   "b_io:               %10lu\n"
84 		   "b_more_io:          %10lu\n"
85 		   "b_dirty_time:       %10lu\n"
86 		   "bdi_list:           %10u\n"
87 		   "state:              %10lx\n",
88 		   (unsigned long) K(wb_stat(wb, WB_WRITEBACK)),
89 		   (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),
90 		   K(wb_thresh),
91 		   K(dirty_thresh),
92 		   K(background_thresh),
93 		   (unsigned long) K(wb_stat(wb, WB_DIRTIED)),
94 		   (unsigned long) K(wb_stat(wb, WB_WRITTEN)),
95 		   (unsigned long) K(wb->write_bandwidth),
96 		   nr_dirty,
97 		   nr_io,
98 		   nr_more_io,
99 		   nr_dirty_time,
100 		   !list_empty(&bdi->bdi_list), bdi->wb.state);
101 #undef K
102 
103 	return 0;
104 }
105 
106 static int bdi_debug_stats_open(struct inode *inode, struct file *file)
107 {
108 	return single_open(file, bdi_debug_stats_show, inode->i_private);
109 }
110 
111 static const struct file_operations bdi_debug_stats_fops = {
112 	.open		= bdi_debug_stats_open,
113 	.read		= seq_read,
114 	.llseek		= seq_lseek,
115 	.release	= single_release,
116 };
117 
118 static void bdi_debug_register(struct backing_dev_info *bdi, const char *name)
119 {
120 	bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root);
121 	bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir,
122 					       bdi, &bdi_debug_stats_fops);
123 }
124 
125 static void bdi_debug_unregister(struct backing_dev_info *bdi)
126 {
127 	debugfs_remove(bdi->debug_stats);
128 	debugfs_remove(bdi->debug_dir);
129 }
130 #else
131 static inline void bdi_debug_init(void)
132 {
133 }
134 static inline void bdi_debug_register(struct backing_dev_info *bdi,
135 				      const char *name)
136 {
137 }
138 static inline void bdi_debug_unregister(struct backing_dev_info *bdi)
139 {
140 }
141 #endif
142 
143 static ssize_t read_ahead_kb_store(struct device *dev,
144 				  struct device_attribute *attr,
145 				  const char *buf, size_t count)
146 {
147 	struct backing_dev_info *bdi = dev_get_drvdata(dev);
148 	unsigned long read_ahead_kb;
149 	ssize_t ret;
150 
151 	ret = kstrtoul(buf, 10, &read_ahead_kb);
152 	if (ret < 0)
153 		return ret;
154 
155 	bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10);
156 
157 	return count;
158 }
159 
160 #define K(pages) ((pages) << (PAGE_SHIFT - 10))
161 
162 #define BDI_SHOW(name, expr)						\
163 static ssize_t name##_show(struct device *dev,				\
164 			   struct device_attribute *attr, char *page)	\
165 {									\
166 	struct backing_dev_info *bdi = dev_get_drvdata(dev);		\
167 									\
168 	return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr);	\
169 }									\
170 static DEVICE_ATTR_RW(name);
171 
172 BDI_SHOW(read_ahead_kb, K(bdi->ra_pages))
173 
174 static ssize_t min_ratio_store(struct device *dev,
175 		struct device_attribute *attr, const char *buf, size_t count)
176 {
177 	struct backing_dev_info *bdi = dev_get_drvdata(dev);
178 	unsigned int ratio;
179 	ssize_t ret;
180 
181 	ret = kstrtouint(buf, 10, &ratio);
182 	if (ret < 0)
183 		return ret;
184 
185 	ret = bdi_set_min_ratio(bdi, ratio);
186 	if (!ret)
187 		ret = count;
188 
189 	return ret;
190 }
191 BDI_SHOW(min_ratio, bdi->min_ratio)
192 
193 static ssize_t max_ratio_store(struct device *dev,
194 		struct device_attribute *attr, const char *buf, size_t count)
195 {
196 	struct backing_dev_info *bdi = dev_get_drvdata(dev);
197 	unsigned int ratio;
198 	ssize_t ret;
199 
200 	ret = kstrtouint(buf, 10, &ratio);
201 	if (ret < 0)
202 		return ret;
203 
204 	ret = bdi_set_max_ratio(bdi, ratio);
205 	if (!ret)
206 		ret = count;
207 
208 	return ret;
209 }
210 BDI_SHOW(max_ratio, bdi->max_ratio)
211 
212 static ssize_t stable_pages_required_show(struct device *dev,
213 					  struct device_attribute *attr,
214 					  char *page)
215 {
216 	struct backing_dev_info *bdi = dev_get_drvdata(dev);
217 
218 	return snprintf(page, PAGE_SIZE-1, "%d\n",
219 			bdi_cap_stable_pages_required(bdi) ? 1 : 0);
220 }
221 static DEVICE_ATTR_RO(stable_pages_required);
222 
223 static struct attribute *bdi_dev_attrs[] = {
224 	&dev_attr_read_ahead_kb.attr,
225 	&dev_attr_min_ratio.attr,
226 	&dev_attr_max_ratio.attr,
227 	&dev_attr_stable_pages_required.attr,
228 	NULL,
229 };
230 ATTRIBUTE_GROUPS(bdi_dev);
231 
232 static __init int bdi_class_init(void)
233 {
234 	bdi_class = class_create(THIS_MODULE, "bdi");
235 	if (IS_ERR(bdi_class))
236 		return PTR_ERR(bdi_class);
237 
238 	bdi_class->dev_groups = bdi_dev_groups;
239 	bdi_debug_init();
240 	return 0;
241 }
242 postcore_initcall(bdi_class_init);
243 
244 static int __init default_bdi_init(void)
245 {
246 	int err;
247 
248 	bdi_wq = alloc_workqueue("writeback", WQ_MEM_RECLAIM | WQ_FREEZABLE |
249 					      WQ_UNBOUND | WQ_SYSFS, 0);
250 	if (!bdi_wq)
251 		return -ENOMEM;
252 
253 	err = bdi_init(&noop_backing_dev_info);
254 
255 	return err;
256 }
257 subsys_initcall(default_bdi_init);
258 
259 /*
260  * This function is used when the first inode for this wb is marked dirty. It
261  * wakes-up the corresponding bdi thread which should then take care of the
262  * periodic background write-out of dirty inodes. Since the write-out would
263  * starts only 'dirty_writeback_interval' centisecs from now anyway, we just
264  * set up a timer which wakes the bdi thread up later.
265  *
266  * Note, we wouldn't bother setting up the timer, but this function is on the
267  * fast-path (used by '__mark_inode_dirty()'), so we save few context switches
268  * by delaying the wake-up.
269  *
270  * We have to be careful not to postpone flush work if it is scheduled for
271  * earlier. Thus we use queue_delayed_work().
272  */
273 void wb_wakeup_delayed(struct bdi_writeback *wb)
274 {
275 	unsigned long timeout;
276 
277 	timeout = msecs_to_jiffies(dirty_writeback_interval * 10);
278 	spin_lock_bh(&wb->work_lock);
279 	if (test_bit(WB_registered, &wb->state))
280 		queue_delayed_work(bdi_wq, &wb->dwork, timeout);
281 	spin_unlock_bh(&wb->work_lock);
282 }
283 
284 /*
285  * Initial write bandwidth: 100 MB/s
286  */
287 #define INIT_BW		(100 << (20 - PAGE_SHIFT))
288 
289 static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,
290 		   int blkcg_id, gfp_t gfp)
291 {
292 	int i, err;
293 
294 	memset(wb, 0, sizeof(*wb));
295 
296 	wb->bdi = bdi;
297 	wb->last_old_flush = jiffies;
298 	INIT_LIST_HEAD(&wb->b_dirty);
299 	INIT_LIST_HEAD(&wb->b_io);
300 	INIT_LIST_HEAD(&wb->b_more_io);
301 	INIT_LIST_HEAD(&wb->b_dirty_time);
302 	spin_lock_init(&wb->list_lock);
303 
304 	wb->bw_time_stamp = jiffies;
305 	wb->balanced_dirty_ratelimit = INIT_BW;
306 	wb->dirty_ratelimit = INIT_BW;
307 	wb->write_bandwidth = INIT_BW;
308 	wb->avg_write_bandwidth = INIT_BW;
309 
310 	spin_lock_init(&wb->work_lock);
311 	INIT_LIST_HEAD(&wb->work_list);
312 	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);
313 
314 	wb->congested = wb_congested_get_create(bdi, blkcg_id, gfp);
315 	if (!wb->congested)
316 		return -ENOMEM;
317 
318 	err = fprop_local_init_percpu(&wb->completions, gfp);
319 	if (err)
320 		goto out_put_cong;
321 
322 	for (i = 0; i < NR_WB_STAT_ITEMS; i++) {
323 		err = percpu_counter_init(&wb->stat[i], 0, gfp);
324 		if (err)
325 			goto out_destroy_stat;
326 	}
327 
328 	return 0;
329 
330 out_destroy_stat:
331 	while (--i)
332 		percpu_counter_destroy(&wb->stat[i]);
333 	fprop_local_destroy_percpu(&wb->completions);
334 out_put_cong:
335 	wb_congested_put(wb->congested);
336 	return err;
337 }
338 
339 /*
340  * Remove bdi from the global list and shutdown any threads we have running
341  */
342 static void wb_shutdown(struct bdi_writeback *wb)
343 {
344 	/* Make sure nobody queues further work */
345 	spin_lock_bh(&wb->work_lock);
346 	if (!test_and_clear_bit(WB_registered, &wb->state)) {
347 		spin_unlock_bh(&wb->work_lock);
348 		return;
349 	}
350 	spin_unlock_bh(&wb->work_lock);
351 
352 	/*
353 	 * Drain work list and shutdown the delayed_work.  !WB_registered
354 	 * tells wb_workfn() that @wb is dying and its work_list needs to
355 	 * be drained no matter what.
356 	 */
357 	mod_delayed_work(bdi_wq, &wb->dwork, 0);
358 	flush_delayed_work(&wb->dwork);
359 	WARN_ON(!list_empty(&wb->work_list));
360 }
361 
362 static void wb_exit(struct bdi_writeback *wb)
363 {
364 	int i;
365 
366 	WARN_ON(delayed_work_pending(&wb->dwork));
367 
368 	for (i = 0; i < NR_WB_STAT_ITEMS; i++)
369 		percpu_counter_destroy(&wb->stat[i]);
370 
371 	fprop_local_destroy_percpu(&wb->completions);
372 	wb_congested_put(wb->congested);
373 }
374 
375 #ifdef CONFIG_CGROUP_WRITEBACK
376 
377 #include <linux/memcontrol.h>
378 
379 /*
380  * cgwb_lock protects bdi->cgwb_tree, bdi->cgwb_congested_tree,
381  * blkcg->cgwb_list, and memcg->cgwb_list.  bdi->cgwb_tree is also RCU
382  * protected.  cgwb_release_wait is used to wait for the completion of cgwb
383  * releases from bdi destruction path.
384  */
385 static DEFINE_SPINLOCK(cgwb_lock);
386 static DECLARE_WAIT_QUEUE_HEAD(cgwb_release_wait);
387 
388 /**
389  * wb_congested_get_create - get or create a wb_congested
390  * @bdi: associated bdi
391  * @blkcg_id: ID of the associated blkcg
392  * @gfp: allocation mask
393  *
394  * Look up the wb_congested for @blkcg_id on @bdi.  If missing, create one.
395  * The returned wb_congested has its reference count incremented.  Returns
396  * NULL on failure.
397  */
398 struct bdi_writeback_congested *
399 wb_congested_get_create(struct backing_dev_info *bdi, int blkcg_id, gfp_t gfp)
400 {
401 	struct bdi_writeback_congested *new_congested = NULL, *congested;
402 	struct rb_node **node, *parent;
403 	unsigned long flags;
404 retry:
405 	spin_lock_irqsave(&cgwb_lock, flags);
406 
407 	node = &bdi->cgwb_congested_tree.rb_node;
408 	parent = NULL;
409 
410 	while (*node != NULL) {
411 		parent = *node;
412 		congested = container_of(parent, struct bdi_writeback_congested,
413 					 rb_node);
414 		if (congested->blkcg_id < blkcg_id)
415 			node = &parent->rb_left;
416 		else if (congested->blkcg_id > blkcg_id)
417 			node = &parent->rb_right;
418 		else
419 			goto found;
420 	}
421 
422 	if (new_congested) {
423 		/* !found and storage for new one already allocated, insert */
424 		congested = new_congested;
425 		new_congested = NULL;
426 		rb_link_node(&congested->rb_node, parent, node);
427 		rb_insert_color(&congested->rb_node, &bdi->cgwb_congested_tree);
428 		goto found;
429 	}
430 
431 	spin_unlock_irqrestore(&cgwb_lock, flags);
432 
433 	/* allocate storage for new one and retry */
434 	new_congested = kzalloc(sizeof(*new_congested), gfp);
435 	if (!new_congested)
436 		return NULL;
437 
438 	atomic_set(&new_congested->refcnt, 0);
439 	new_congested->bdi = bdi;
440 	new_congested->blkcg_id = blkcg_id;
441 	goto retry;
442 
443 found:
444 	atomic_inc(&congested->refcnt);
445 	spin_unlock_irqrestore(&cgwb_lock, flags);
446 	kfree(new_congested);
447 	return congested;
448 }
449 
450 /**
451  * wb_congested_put - put a wb_congested
452  * @congested: wb_congested to put
453  *
454  * Put @congested and destroy it if the refcnt reaches zero.
455  */
456 void wb_congested_put(struct bdi_writeback_congested *congested)
457 {
458 	unsigned long flags;
459 
460 	local_irq_save(flags);
461 	if (!atomic_dec_and_lock(&congested->refcnt, &cgwb_lock)) {
462 		local_irq_restore(flags);
463 		return;
464 	}
465 
466 	/* bdi might already have been destroyed leaving @congested unlinked */
467 	if (congested->bdi) {
468 		rb_erase(&congested->rb_node,
469 			 &congested->bdi->cgwb_congested_tree);
470 		congested->bdi = NULL;
471 	}
472 
473 	spin_unlock_irqrestore(&cgwb_lock, flags);
474 	kfree(congested);
475 }
476 
477 static void cgwb_release_workfn(struct work_struct *work)
478 {
479 	struct bdi_writeback *wb = container_of(work, struct bdi_writeback,
480 						release_work);
481 	struct backing_dev_info *bdi = wb->bdi;
482 
483 	spin_lock_irq(&cgwb_lock);
484 	list_del_rcu(&wb->bdi_node);
485 	spin_unlock_irq(&cgwb_lock);
486 
487 	wb_shutdown(wb);
488 
489 	css_put(wb->memcg_css);
490 	css_put(wb->blkcg_css);
491 
492 	fprop_local_destroy_percpu(&wb->memcg_completions);
493 	percpu_ref_exit(&wb->refcnt);
494 	wb_exit(wb);
495 	kfree_rcu(wb, rcu);
496 
497 	if (atomic_dec_and_test(&bdi->usage_cnt))
498 		wake_up_all(&cgwb_release_wait);
499 }
500 
501 static void cgwb_release(struct percpu_ref *refcnt)
502 {
503 	struct bdi_writeback *wb = container_of(refcnt, struct bdi_writeback,
504 						refcnt);
505 	schedule_work(&wb->release_work);
506 }
507 
508 static void cgwb_kill(struct bdi_writeback *wb)
509 {
510 	lockdep_assert_held(&cgwb_lock);
511 
512 	WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id));
513 	list_del(&wb->memcg_node);
514 	list_del(&wb->blkcg_node);
515 	percpu_ref_kill(&wb->refcnt);
516 }
517 
518 static int cgwb_create(struct backing_dev_info *bdi,
519 		       struct cgroup_subsys_state *memcg_css, gfp_t gfp)
520 {
521 	struct mem_cgroup *memcg;
522 	struct cgroup_subsys_state *blkcg_css;
523 	struct blkcg *blkcg;
524 	struct list_head *memcg_cgwb_list, *blkcg_cgwb_list;
525 	struct bdi_writeback *wb;
526 	unsigned long flags;
527 	int ret = 0;
528 
529 	memcg = mem_cgroup_from_css(memcg_css);
530 	blkcg_css = cgroup_get_e_css(memcg_css->cgroup, &io_cgrp_subsys);
531 	blkcg = css_to_blkcg(blkcg_css);
532 	memcg_cgwb_list = mem_cgroup_cgwb_list(memcg);
533 	blkcg_cgwb_list = &blkcg->cgwb_list;
534 
535 	/* look up again under lock and discard on blkcg mismatch */
536 	spin_lock_irqsave(&cgwb_lock, flags);
537 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
538 	if (wb && wb->blkcg_css != blkcg_css) {
539 		cgwb_kill(wb);
540 		wb = NULL;
541 	}
542 	spin_unlock_irqrestore(&cgwb_lock, flags);
543 	if (wb)
544 		goto out_put;
545 
546 	/* need to create a new one */
547 	wb = kmalloc(sizeof(*wb), gfp);
548 	if (!wb)
549 		return -ENOMEM;
550 
551 	ret = wb_init(wb, bdi, blkcg_css->id, gfp);
552 	if (ret)
553 		goto err_free;
554 
555 	ret = percpu_ref_init(&wb->refcnt, cgwb_release, 0, gfp);
556 	if (ret)
557 		goto err_wb_exit;
558 
559 	ret = fprop_local_init_percpu(&wb->memcg_completions, gfp);
560 	if (ret)
561 		goto err_ref_exit;
562 
563 	wb->memcg_css = memcg_css;
564 	wb->blkcg_css = blkcg_css;
565 	INIT_WORK(&wb->release_work, cgwb_release_workfn);
566 	set_bit(WB_registered, &wb->state);
567 
568 	/*
569 	 * The root wb determines the registered state of the whole bdi and
570 	 * memcg_cgwb_list and blkcg_cgwb_list's next pointers indicate
571 	 * whether they're still online.  Don't link @wb if any is dead.
572 	 * See wb_memcg_offline() and wb_blkcg_offline().
573 	 */
574 	ret = -ENODEV;
575 	spin_lock_irqsave(&cgwb_lock, flags);
576 	if (test_bit(WB_registered, &bdi->wb.state) &&
577 	    blkcg_cgwb_list->next && memcg_cgwb_list->next) {
578 		/* we might have raced another instance of this function */
579 		ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb);
580 		if (!ret) {
581 			atomic_inc(&bdi->usage_cnt);
582 			list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list);
583 			list_add(&wb->memcg_node, memcg_cgwb_list);
584 			list_add(&wb->blkcg_node, blkcg_cgwb_list);
585 			css_get(memcg_css);
586 			css_get(blkcg_css);
587 		}
588 	}
589 	spin_unlock_irqrestore(&cgwb_lock, flags);
590 	if (ret) {
591 		if (ret == -EEXIST)
592 			ret = 0;
593 		goto err_fprop_exit;
594 	}
595 	goto out_put;
596 
597 err_fprop_exit:
598 	fprop_local_destroy_percpu(&wb->memcg_completions);
599 err_ref_exit:
600 	percpu_ref_exit(&wb->refcnt);
601 err_wb_exit:
602 	wb_exit(wb);
603 err_free:
604 	kfree(wb);
605 out_put:
606 	css_put(blkcg_css);
607 	return ret;
608 }
609 
610 /**
611  * wb_get_create - get wb for a given memcg, create if necessary
612  * @bdi: target bdi
613  * @memcg_css: cgroup_subsys_state of the target memcg (must have positive ref)
614  * @gfp: allocation mask to use
615  *
616  * Try to get the wb for @memcg_css on @bdi.  If it doesn't exist, try to
617  * create one.  The returned wb has its refcount incremented.
618  *
619  * This function uses css_get() on @memcg_css and thus expects its refcnt
620  * to be positive on invocation.  IOW, rcu_read_lock() protection on
621  * @memcg_css isn't enough.  try_get it before calling this function.
622  *
623  * A wb is keyed by its associated memcg.  As blkcg implicitly enables
624  * memcg on the default hierarchy, memcg association is guaranteed to be
625  * more specific (equal or descendant to the associated blkcg) and thus can
626  * identify both the memcg and blkcg associations.
627  *
628  * Because the blkcg associated with a memcg may change as blkcg is enabled
629  * and disabled closer to root in the hierarchy, each wb keeps track of
630  * both the memcg and blkcg associated with it and verifies the blkcg on
631  * each lookup.  On mismatch, the existing wb is discarded and a new one is
632  * created.
633  */
634 struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi,
635 				    struct cgroup_subsys_state *memcg_css,
636 				    gfp_t gfp)
637 {
638 	struct bdi_writeback *wb;
639 
640 	might_sleep_if(gfpflags_allow_blocking(gfp));
641 
642 	if (!memcg_css->parent)
643 		return &bdi->wb;
644 
645 	do {
646 		rcu_read_lock();
647 		wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
648 		if (wb) {
649 			struct cgroup_subsys_state *blkcg_css;
650 
651 			/* see whether the blkcg association has changed */
652 			blkcg_css = cgroup_get_e_css(memcg_css->cgroup,
653 						     &io_cgrp_subsys);
654 			if (unlikely(wb->blkcg_css != blkcg_css ||
655 				     !wb_tryget(wb)))
656 				wb = NULL;
657 			css_put(blkcg_css);
658 		}
659 		rcu_read_unlock();
660 	} while (!wb && !cgwb_create(bdi, memcg_css, gfp));
661 
662 	return wb;
663 }
664 
665 static int cgwb_bdi_init(struct backing_dev_info *bdi)
666 {
667 	int ret;
668 
669 	INIT_RADIX_TREE(&bdi->cgwb_tree, GFP_ATOMIC);
670 	bdi->cgwb_congested_tree = RB_ROOT;
671 	atomic_set(&bdi->usage_cnt, 1);
672 
673 	ret = wb_init(&bdi->wb, bdi, 1, GFP_KERNEL);
674 	if (!ret) {
675 		bdi->wb.memcg_css = mem_cgroup_root_css;
676 		bdi->wb.blkcg_css = blkcg_root_css;
677 	}
678 	return ret;
679 }
680 
681 static void cgwb_bdi_destroy(struct backing_dev_info *bdi)
682 {
683 	struct radix_tree_iter iter;
684 	struct rb_node *rbn;
685 	void **slot;
686 
687 	WARN_ON(test_bit(WB_registered, &bdi->wb.state));
688 
689 	spin_lock_irq(&cgwb_lock);
690 
691 	radix_tree_for_each_slot(slot, &bdi->cgwb_tree, &iter, 0)
692 		cgwb_kill(*slot);
693 
694 	while ((rbn = rb_first(&bdi->cgwb_congested_tree))) {
695 		struct bdi_writeback_congested *congested =
696 			rb_entry(rbn, struct bdi_writeback_congested, rb_node);
697 
698 		rb_erase(rbn, &bdi->cgwb_congested_tree);
699 		congested->bdi = NULL;	/* mark @congested unlinked */
700 	}
701 
702 	spin_unlock_irq(&cgwb_lock);
703 
704 	/*
705 	 * All cgwb's and their congested states must be shutdown and
706 	 * released before returning.  Drain the usage counter to wait for
707 	 * all cgwb's and cgwb_congested's ever created on @bdi.
708 	 */
709 	atomic_dec(&bdi->usage_cnt);
710 	wait_event(cgwb_release_wait, !atomic_read(&bdi->usage_cnt));
711 }
712 
713 /**
714  * wb_memcg_offline - kill all wb's associated with a memcg being offlined
715  * @memcg: memcg being offlined
716  *
717  * Also prevents creation of any new wb's associated with @memcg.
718  */
719 void wb_memcg_offline(struct mem_cgroup *memcg)
720 {
721 	LIST_HEAD(to_destroy);
722 	struct list_head *memcg_cgwb_list = mem_cgroup_cgwb_list(memcg);
723 	struct bdi_writeback *wb, *next;
724 
725 	spin_lock_irq(&cgwb_lock);
726 	list_for_each_entry_safe(wb, next, memcg_cgwb_list, memcg_node)
727 		cgwb_kill(wb);
728 	memcg_cgwb_list->next = NULL;	/* prevent new wb's */
729 	spin_unlock_irq(&cgwb_lock);
730 }
731 
732 /**
733  * wb_blkcg_offline - kill all wb's associated with a blkcg being offlined
734  * @blkcg: blkcg being offlined
735  *
736  * Also prevents creation of any new wb's associated with @blkcg.
737  */
738 void wb_blkcg_offline(struct blkcg *blkcg)
739 {
740 	LIST_HEAD(to_destroy);
741 	struct bdi_writeback *wb, *next;
742 
743 	spin_lock_irq(&cgwb_lock);
744 	list_for_each_entry_safe(wb, next, &blkcg->cgwb_list, blkcg_node)
745 		cgwb_kill(wb);
746 	blkcg->cgwb_list.next = NULL;	/* prevent new wb's */
747 	spin_unlock_irq(&cgwb_lock);
748 }
749 
750 #else	/* CONFIG_CGROUP_WRITEBACK */
751 
752 static int cgwb_bdi_init(struct backing_dev_info *bdi)
753 {
754 	int err;
755 
756 	bdi->wb_congested = kzalloc(sizeof(*bdi->wb_congested), GFP_KERNEL);
757 	if (!bdi->wb_congested)
758 		return -ENOMEM;
759 
760 	err = wb_init(&bdi->wb, bdi, 1, GFP_KERNEL);
761 	if (err) {
762 		kfree(bdi->wb_congested);
763 		return err;
764 	}
765 	return 0;
766 }
767 
768 static void cgwb_bdi_destroy(struct backing_dev_info *bdi) { }
769 
770 #endif	/* CONFIG_CGROUP_WRITEBACK */
771 
772 int bdi_init(struct backing_dev_info *bdi)
773 {
774 	int ret;
775 
776 	bdi->dev = NULL;
777 
778 	bdi->min_ratio = 0;
779 	bdi->max_ratio = 100;
780 	bdi->max_prop_frac = FPROP_FRAC_BASE;
781 	INIT_LIST_HEAD(&bdi->bdi_list);
782 	INIT_LIST_HEAD(&bdi->wb_list);
783 	init_waitqueue_head(&bdi->wb_waitq);
784 
785 	ret = cgwb_bdi_init(bdi);
786 
787 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);
788 
789 	return ret;
790 }
791 EXPORT_SYMBOL(bdi_init);
792 
793 int bdi_register(struct backing_dev_info *bdi, struct device *parent,
794 		const char *fmt, ...)
795 {
796 	va_list args;
797 	struct device *dev;
798 
799 	if (bdi->dev)	/* The driver needs to use separate queues per device */
800 		return 0;
801 
802 	va_start(args, fmt);
803 	dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args);
804 	va_end(args);
805 	if (IS_ERR(dev))
806 		return PTR_ERR(dev);
807 
808 	bdi->dev = dev;
809 
810 	bdi_debug_register(bdi, dev_name(dev));
811 	set_bit(WB_registered, &bdi->wb.state);
812 
813 	spin_lock_bh(&bdi_lock);
814 	list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
815 	spin_unlock_bh(&bdi_lock);
816 
817 	trace_writeback_bdi_register(bdi);
818 	return 0;
819 }
820 EXPORT_SYMBOL(bdi_register);
821 
822 int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
823 {
824 	return bdi_register(bdi, NULL, "%u:%u", MAJOR(dev), MINOR(dev));
825 }
826 EXPORT_SYMBOL(bdi_register_dev);
827 
828 /*
829  * Remove bdi from bdi_list, and ensure that it is no longer visible
830  */
831 static void bdi_remove_from_list(struct backing_dev_info *bdi)
832 {
833 	spin_lock_bh(&bdi_lock);
834 	list_del_rcu(&bdi->bdi_list);
835 	spin_unlock_bh(&bdi_lock);
836 
837 	synchronize_rcu_expedited();
838 }
839 
840 void bdi_unregister(struct backing_dev_info *bdi)
841 {
842 	/* make sure nobody finds us on the bdi_list anymore */
843 	bdi_remove_from_list(bdi);
844 	wb_shutdown(&bdi->wb);
845 	cgwb_bdi_destroy(bdi);
846 
847 	if (bdi->dev) {
848 		bdi_debug_unregister(bdi);
849 		device_unregister(bdi->dev);
850 		bdi->dev = NULL;
851 	}
852 }
853 
854 void bdi_exit(struct backing_dev_info *bdi)
855 {
856 	WARN_ON_ONCE(bdi->dev);
857 	wb_exit(&bdi->wb);
858 }
859 
860 void bdi_destroy(struct backing_dev_info *bdi)
861 {
862 	bdi_unregister(bdi);
863 	bdi_exit(bdi);
864 }
865 EXPORT_SYMBOL(bdi_destroy);
866 
867 /*
868  * For use from filesystems to quickly init and register a bdi associated
869  * with dirty writeback
870  */
871 int bdi_setup_and_register(struct backing_dev_info *bdi, char *name)
872 {
873 	int err;
874 
875 	bdi->name = name;
876 	bdi->capabilities = 0;
877 	err = bdi_init(bdi);
878 	if (err)
879 		return err;
880 
881 	err = bdi_register(bdi, NULL, "%.28s-%ld", name,
882 			   atomic_long_inc_return(&bdi_seq));
883 	if (err) {
884 		bdi_destroy(bdi);
885 		return err;
886 	}
887 
888 	return 0;
889 }
890 EXPORT_SYMBOL(bdi_setup_and_register);
891 
892 static wait_queue_head_t congestion_wqh[2] = {
893 		__WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
894 		__WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
895 	};
896 static atomic_t nr_wb_congested[2];
897 
898 void clear_wb_congested(struct bdi_writeback_congested *congested, int sync)
899 {
900 	wait_queue_head_t *wqh = &congestion_wqh[sync];
901 	enum wb_state bit;
902 
903 	bit = sync ? WB_sync_congested : WB_async_congested;
904 	if (test_and_clear_bit(bit, &congested->state))
905 		atomic_dec(&nr_wb_congested[sync]);
906 	smp_mb__after_atomic();
907 	if (waitqueue_active(wqh))
908 		wake_up(wqh);
909 }
910 EXPORT_SYMBOL(clear_wb_congested);
911 
912 void set_wb_congested(struct bdi_writeback_congested *congested, int sync)
913 {
914 	enum wb_state bit;
915 
916 	bit = sync ? WB_sync_congested : WB_async_congested;
917 	if (!test_and_set_bit(bit, &congested->state))
918 		atomic_inc(&nr_wb_congested[sync]);
919 }
920 EXPORT_SYMBOL(set_wb_congested);
921 
922 /**
923  * congestion_wait - wait for a backing_dev to become uncongested
924  * @sync: SYNC or ASYNC IO
925  * @timeout: timeout in jiffies
926  *
927  * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
928  * write congestion.  If no backing_devs are congested then just wait for the
929  * next write to be completed.
930  */
931 long congestion_wait(int sync, long timeout)
932 {
933 	long ret;
934 	unsigned long start = jiffies;
935 	DEFINE_WAIT(wait);
936 	wait_queue_head_t *wqh = &congestion_wqh[sync];
937 
938 	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
939 	ret = io_schedule_timeout(timeout);
940 	finish_wait(wqh, &wait);
941 
942 	trace_writeback_congestion_wait(jiffies_to_usecs(timeout),
943 					jiffies_to_usecs(jiffies - start));
944 
945 	return ret;
946 }
947 EXPORT_SYMBOL(congestion_wait);
948 
949 /**
950  * wait_iff_congested - Conditionally wait for a backing_dev to become uncongested or a zone to complete writes
951  * @zone: A zone to check if it is heavily congested
952  * @sync: SYNC or ASYNC IO
953  * @timeout: timeout in jiffies
954  *
955  * In the event of a congested backing_dev (any backing_dev) and the given
956  * @zone has experienced recent congestion, this waits for up to @timeout
957  * jiffies for either a BDI to exit congestion of the given @sync queue
958  * or a write to complete.
959  *
960  * In the absence of zone congestion, a short sleep or a cond_resched is
961  * performed to yield the processor and to allow other subsystems to make
962  * a forward progress.
963  *
964  * The return value is 0 if the sleep is for the full timeout. Otherwise,
965  * it is the number of jiffies that were still remaining when the function
966  * returned. return_value == timeout implies the function did not sleep.
967  */
968 long wait_iff_congested(struct zone *zone, int sync, long timeout)
969 {
970 	long ret;
971 	unsigned long start = jiffies;
972 	DEFINE_WAIT(wait);
973 	wait_queue_head_t *wqh = &congestion_wqh[sync];
974 
975 	/*
976 	 * If there is no congestion, or heavy congestion is not being
977 	 * encountered in the current zone, yield if necessary instead
978 	 * of sleeping on the congestion queue
979 	 */
980 	if (atomic_read(&nr_wb_congested[sync]) == 0 ||
981 	    !test_bit(ZONE_CONGESTED, &zone->flags)) {
982 
983 		/*
984 		 * Memory allocation/reclaim might be called from a WQ
985 		 * context and the current implementation of the WQ
986 		 * concurrency control doesn't recognize that a particular
987 		 * WQ is congested if the worker thread is looping without
988 		 * ever sleeping. Therefore we have to do a short sleep
989 		 * here rather than calling cond_resched().
990 		 */
991 		if (current->flags & PF_WQ_WORKER)
992 			schedule_timeout(1);
993 		else
994 			cond_resched();
995 
996 		/* In case we scheduled, work out time remaining */
997 		ret = timeout - (jiffies - start);
998 		if (ret < 0)
999 			ret = 0;
1000 
1001 		goto out;
1002 	}
1003 
1004 	/* Sleep until uncongested or a write happens */
1005 	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
1006 	ret = io_schedule_timeout(timeout);
1007 	finish_wait(wqh, &wait);
1008 
1009 out:
1010 	trace_writeback_wait_iff_congested(jiffies_to_usecs(timeout),
1011 					jiffies_to_usecs(jiffies - start));
1012 
1013 	return ret;
1014 }
1015 EXPORT_SYMBOL(wait_iff_congested);
1016 
1017 int pdflush_proc_obsolete(struct ctl_table *table, int write,
1018 			void __user *buffer, size_t *lenp, loff_t *ppos)
1019 {
1020 	char kbuf[] = "0\n";
1021 
1022 	if (*ppos || *lenp < sizeof(kbuf)) {
1023 		*lenp = 0;
1024 		return 0;
1025 	}
1026 
1027 	if (copy_to_user(buffer, kbuf, sizeof(kbuf)))
1028 		return -EFAULT;
1029 	printk_once(KERN_WARNING "%s exported in /proc is scheduled for removal\n",
1030 			table->procname);
1031 
1032 	*lenp = 2;
1033 	*ppos += *lenp;
1034 	return 2;
1035 }
1036