52 	struct bdi_writeback *wb = &bdi->wb;  in bdi_debug_stats_show()  local
60 	spin_lock(&wb->list_lock);  in bdi_debug_stats_show()
61 	list_for_each_entry(inode, &wb->b_dirty, i_io_list)  in bdi_debug_stats_show()
63 	list_for_each_entry(inode, &wb->b_io, i_io_list)  in bdi_debug_stats_show()
65 	list_for_each_entry(inode, &wb->b_more_io, i_io_list)  in bdi_debug_stats_show()
67 	list_for_each_entry(inode, &wb->b_dirty_time, i_io_list)  in bdi_debug_stats_show()
70 	spin_unlock(&wb->list_lock);  in bdi_debug_stats_show()
73 	wb_thresh = wb_calc_thresh(wb, dirty_thresh);  in bdi_debug_stats_show()
90 		   (unsigned long) K(wb_stat(wb, WB_WRITEBACK)),  in bdi_debug_stats_show()
91 		   (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),  in bdi_debug_stats_show()
95 		   (unsigned long) K(wb_stat(wb, WB_DIRTIED)),  in bdi_debug_stats_show()
96 		   (unsigned long) K(wb_stat(wb, WB_WRITTEN)),  in bdi_debug_stats_show()
97 		   (unsigned long) K(wb->write_bandwidth),  in bdi_debug_stats_show()
102 		   !list_empty(&bdi->bdi_list), bdi->wb.state);  in bdi_debug_stats_show()
389 void wb_wakeup_delayed(struct bdi_writeback *wb)  in wb_wakeup_delayed()  argument
394 	spin_lock_irq(&wb->work_lock);  in wb_wakeup_delayed()
395 	if (test_bit(WB_registered, &wb->state))  in wb_wakeup_delayed()
396 		queue_delayed_work(bdi_wq, &wb->dwork, timeout);  in wb_wakeup_delayed()
397 	spin_unlock_irq(&wb->work_lock);  in wb_wakeup_delayed()
402 	struct bdi_writeback *wb = container_of(to_delayed_work(work),  in wb_update_bandwidth_workfn()  local
405 	wb_update_bandwidth(wb);  in wb_update_bandwidth_workfn()
413 static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,  in wb_init()  argument
418 	memset(wb, 0, sizeof(*wb));  in wb_init()
420 	wb->bdi = bdi;  in wb_init()
421 	wb->last_old_flush = jiffies;  in wb_init()
422 	INIT_LIST_HEAD(&wb->b_dirty);  in wb_init()
423 	INIT_LIST_HEAD(&wb->b_io);  in wb_init()
424 	INIT_LIST_HEAD(&wb->b_more_io);  in wb_init()
425 	INIT_LIST_HEAD(&wb->b_dirty_time);  in wb_init()
426 	spin_lock_init(&wb->list_lock);  in wb_init()
428 	atomic_set(&wb->writeback_inodes, 0);  in wb_init()
429 	wb->bw_time_stamp = jiffies;  in wb_init()
430 	wb->balanced_dirty_ratelimit = INIT_BW;  in wb_init()
431 	wb->dirty_ratelimit = INIT_BW;  in wb_init()
432 	wb->write_bandwidth = INIT_BW;  in wb_init()
433 	wb->avg_write_bandwidth = INIT_BW;  in wb_init()
435 	spin_lock_init(&wb->work_lock);  in wb_init()
436 	INIT_LIST_HEAD(&wb->work_list);  in wb_init()
437 	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);  in wb_init()
438 	INIT_DELAYED_WORK(&wb->bw_dwork, wb_update_bandwidth_workfn);  in wb_init()
440 	err = fprop_local_init_percpu(&wb->completions, gfp);  in wb_init()
445 		err = percpu_counter_init(&wb->stat[i], 0, gfp);  in wb_init()
454 		percpu_counter_destroy(&wb->stat[i]);  in wb_init()
455 	fprop_local_destroy_percpu(&wb->completions);  in wb_init()
459 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb);
464 static void wb_shutdown(struct bdi_writeback *wb)  in wb_shutdown()  argument
467 	spin_lock_irq(&wb->work_lock);  in wb_shutdown()
468 	if (!test_and_clear_bit(WB_registered, &wb->state)) {  in wb_shutdown()
469 		spin_unlock_irq(&wb->work_lock);  in wb_shutdown()
472 	spin_unlock_irq(&wb->work_lock);  in wb_shutdown()
474 	cgwb_remove_from_bdi_list(wb);  in wb_shutdown()
480 	mod_delayed_work(bdi_wq, &wb->dwork, 0);  in wb_shutdown()
481 	flush_delayed_work(&wb->dwork);  in wb_shutdown()
482 	WARN_ON(!list_empty(&wb->work_list));  in wb_shutdown()
483 	flush_delayed_work(&wb->bw_dwork);  in wb_shutdown()
486 static void wb_exit(struct bdi_writeback *wb)  in wb_exit()  argument
490 	WARN_ON(delayed_work_pending(&wb->dwork));  in wb_exit()
493 		percpu_counter_destroy(&wb->stat[i]);  in wb_exit()
495 	fprop_local_destroy_percpu(&wb->completions);  in wb_exit()
515 	struct bdi_writeback *wb = container_of(rcu_head,  in cgwb_free_rcu()  local
518 	percpu_ref_exit(&wb->refcnt);  in cgwb_free_rcu()
519 	kfree(wb);  in cgwb_free_rcu()
524 	struct bdi_writeback *wb = container_of(work, struct bdi_writeback,  in cgwb_release_workfn()  local
526 	struct backing_dev_info *bdi = wb->bdi;  in cgwb_release_workfn()
528 	mutex_lock(&wb->bdi->cgwb_release_mutex);  in cgwb_release_workfn()
529 	wb_shutdown(wb);  in cgwb_release_workfn()
531 	css_put(wb->memcg_css);  in cgwb_release_workfn()
532 	css_put(wb->blkcg_css);  in cgwb_release_workfn()
533 	mutex_unlock(&wb->bdi->cgwb_release_mutex);  in cgwb_release_workfn()
536 	blkcg_unpin_online(wb->blkcg_css);  in cgwb_release_workfn()
538 	fprop_local_destroy_percpu(&wb->memcg_completions);  in cgwb_release_workfn()
541 	list_del(&wb->offline_node);  in cgwb_release_workfn()
544 	wb_exit(wb);  in cgwb_release_workfn()
546 	WARN_ON_ONCE(!list_empty(&wb->b_attached));  in cgwb_release_workfn()
547 	call_rcu(&wb->rcu, cgwb_free_rcu);  in cgwb_release_workfn()
552 	struct bdi_writeback *wb = container_of(refcnt, struct bdi_writeback,  in cgwb_release()  local
554 	queue_work(cgwb_release_wq, &wb->release_work);  in cgwb_release()
557 static void cgwb_kill(struct bdi_writeback *wb)  in cgwb_kill()  argument
561 	WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id));  in cgwb_kill()
562 	list_del(&wb->memcg_node);  in cgwb_kill()
563 	list_del(&wb->blkcg_node);  in cgwb_kill()
564 	list_add(&wb->offline_node, &offline_cgwbs);  in cgwb_kill()
565 	percpu_ref_kill(&wb->refcnt);  in cgwb_kill()
568 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)  in cgwb_remove_from_bdi_list()  argument
571 	list_del_rcu(&wb->bdi_node);  in cgwb_remove_from_bdi_list()
581 	struct bdi_writeback *wb;  in cgwb_create()  local
592 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);  in cgwb_create()
593 	if (wb && wb->blkcg_css != blkcg_css) {  in cgwb_create()
594 		cgwb_kill(wb);  in cgwb_create()
595 		wb = NULL;  in cgwb_create()
598 	if (wb)  in cgwb_create()
602 	wb = kmalloc(sizeof(*wb), gfp);  in cgwb_create()
603 	if (!wb) {  in cgwb_create()
608 	ret = wb_init(wb, bdi, gfp);  in cgwb_create()
612 	ret = percpu_ref_init(&wb->refcnt, cgwb_release, 0, gfp);  in cgwb_create()
616 	ret = fprop_local_init_percpu(&wb->memcg_completions, gfp);  in cgwb_create()
620 	wb->memcg_css = memcg_css;  in cgwb_create()
621 	wb->blkcg_css = blkcg_css;  in cgwb_create()
622 	INIT_LIST_HEAD(&wb->b_attached);  in cgwb_create()
623 	INIT_WORK(&wb->release_work, cgwb_release_workfn);  in cgwb_create()
624 	set_bit(WB_registered, &wb->state);  in cgwb_create()
635 	if (test_bit(WB_registered, &bdi->wb.state) &&  in cgwb_create()
638 		ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb);  in cgwb_create()
640 			list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list);  in cgwb_create()
641 			list_add(&wb->memcg_node, memcg_cgwb_list);  in cgwb_create()
642 			list_add(&wb->blkcg_node, blkcg_cgwb_list);  in cgwb_create()
658 	fprop_local_destroy_percpu(&wb->memcg_completions);  in cgwb_create()
660 	percpu_ref_exit(&wb->refcnt);  in cgwb_create()
662 	wb_exit(wb);  in cgwb_create()
664 	kfree(wb);  in cgwb_create()
696 	struct bdi_writeback *wb;  in wb_get_lookup()  local
699 		return &bdi->wb;  in wb_get_lookup()
702 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);  in wb_get_lookup()
703 	if (wb) {  in wb_get_lookup()
708 		if (unlikely(wb->blkcg_css != blkcg_css || !wb_tryget(wb)))  in wb_get_lookup()
709 			wb = NULL;  in wb_get_lookup()
714 	return wb;  in wb_get_lookup()
730 	struct bdi_writeback *wb;  in wb_get_create()  local
735 		wb = wb_get_lookup(bdi, memcg_css);  in wb_get_create()
736 	} while (!wb && !cgwb_create(bdi, memcg_css, gfp));  in wb_get_create()
738 	return wb;  in wb_get_create()
749 	ret = wb_init(&bdi->wb, bdi, GFP_KERNEL);  in cgwb_bdi_init()
751 		bdi->wb.memcg_css = &root_mem_cgroup->css;  in cgwb_bdi_init()
752 		bdi->wb.blkcg_css = blkcg_root_css;  in cgwb_bdi_init()
761 	struct bdi_writeback *wb;  in cgwb_bdi_unregister()  local
763 	WARN_ON(test_bit(WB_registered, &bdi->wb.state));  in cgwb_bdi_unregister()
773 		wb = list_first_entry(&bdi->wb_list, struct bdi_writeback,  in cgwb_bdi_unregister()
776 		wb_shutdown(wb);  in cgwb_bdi_unregister()
792 	struct bdi_writeback *wb;  in cleanup_offline_cgwbs_workfn()  local
798 		wb = list_first_entry(&offline_cgwbs, struct bdi_writeback,  in cleanup_offline_cgwbs_workfn()
800 		list_move(&wb->offline_node, &processed);  in cleanup_offline_cgwbs_workfn()
811 		if (wb_has_dirty_io(wb))  in cleanup_offline_cgwbs_workfn()
814 		if (!wb_tryget(wb))  in cleanup_offline_cgwbs_workfn()
818 		while (cleanup_offline_cgwb(wb))  in cleanup_offline_cgwbs_workfn()
822 		wb_put(wb);  in cleanup_offline_cgwbs_workfn()
840 	struct bdi_writeback *wb, *next;  in wb_memcg_offline()  local
843 	list_for_each_entry_safe(wb, next, memcg_cgwb_list, memcg_node)  in wb_memcg_offline()
844 		cgwb_kill(wb);  in wb_memcg_offline()
859 	struct bdi_writeback *wb, *next;  in wb_blkcg_offline()  local
863 	list_for_each_entry_safe(wb, next, list, blkcg_node)  in wb_blkcg_offline()
864 		cgwb_kill(wb);  in wb_blkcg_offline()
872 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);  in cgwb_bdi_register()
895 	return wb_init(&bdi->wb, bdi, GFP_KERNEL);  in cgwb_bdi_init()
902 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);  in cgwb_bdi_register()
905 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)  in cgwb_remove_from_bdi_list()  argument
907 	list_del_rcu(&wb->bdi_node);  in cgwb_remove_from_bdi_list()
1013 	set_bit(WB_registered, &bdi->wb.state);  in bdi_register_va()
1069 	wb_shutdown(&bdi->wb);  in bdi_unregister()
1097 	WARN_ON_ONCE(test_bit(WB_registered, &bdi->wb.state));  in release_bdi()
1099 	wb_exit(&bdi->wb);  in release_bdi()