Lines Matching refs:wb
85 static bool wb_io_lists_populated(struct bdi_writeback *wb) in wb_io_lists_populated() argument
87 if (wb_has_dirty_io(wb)) { in wb_io_lists_populated()
90 set_bit(WB_has_dirty_io, &wb->state); in wb_io_lists_populated()
91 WARN_ON_ONCE(!wb->avg_write_bandwidth); in wb_io_lists_populated()
92 atomic_long_add(wb->avg_write_bandwidth, in wb_io_lists_populated()
93 &wb->bdi->tot_write_bandwidth); in wb_io_lists_populated()
98 static void wb_io_lists_depopulated(struct bdi_writeback *wb) in wb_io_lists_depopulated() argument
100 if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) && in wb_io_lists_depopulated()
101 list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) { in wb_io_lists_depopulated()
102 clear_bit(WB_has_dirty_io, &wb->state); in wb_io_lists_depopulated()
103 WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth, in wb_io_lists_depopulated()
104 &wb->bdi->tot_write_bandwidth) < 0); in wb_io_lists_depopulated()
119 struct bdi_writeback *wb, in inode_io_list_move_locked() argument
122 assert_spin_locked(&wb->list_lock); in inode_io_list_move_locked()
129 if (head != &wb->b_dirty_time) in inode_io_list_move_locked()
130 return wb_io_lists_populated(wb); in inode_io_list_move_locked()
132 wb_io_lists_depopulated(wb); in inode_io_list_move_locked()
136 static void wb_wakeup(struct bdi_writeback *wb) in wb_wakeup() argument
138 spin_lock_irq(&wb->work_lock); in wb_wakeup()
139 if (test_bit(WB_registered, &wb->state)) in wb_wakeup()
140 mod_delayed_work(bdi_wq, &wb->dwork, 0); in wb_wakeup()
141 spin_unlock_irq(&wb->work_lock); in wb_wakeup()
144 static void finish_writeback_work(struct bdi_writeback *wb, in finish_writeback_work() argument
160 static void wb_queue_work(struct bdi_writeback *wb, in wb_queue_work() argument
163 trace_writeback_queue(wb, work); in wb_queue_work()
168 spin_lock_irq(&wb->work_lock); in wb_queue_work()
170 if (test_bit(WB_registered, &wb->state)) { in wb_queue_work()
171 list_add_tail(&work->list, &wb->work_list); in wb_queue_work()
172 mod_delayed_work(bdi_wq, &wb->dwork, 0); in wb_queue_work()
174 finish_writeback_work(wb, work); in wb_queue_work()
176 spin_unlock_irq(&wb->work_lock); in wb_queue_work()
243 struct bdi_writeback *wb = NULL; in __inode_attach_wb() local
250 wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); in __inode_attach_wb()
254 wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); in __inode_attach_wb()
259 if (!wb) in __inode_attach_wb()
260 wb = &bdi->wb; in __inode_attach_wb()
266 if (unlikely(cmpxchg(&inode->i_wb, NULL, wb))) in __inode_attach_wb()
267 wb_put(wb); in __inode_attach_wb()
280 struct bdi_writeback *wb) in inode_cgwb_move_to_attached() argument
282 assert_spin_locked(&wb->list_lock); in inode_cgwb_move_to_attached()
287 if (wb != &wb->bdi->wb) in inode_cgwb_move_to_attached()
288 list_move(&inode->i_io_list, &wb->b_attached); in inode_cgwb_move_to_attached()
291 wb_io_lists_depopulated(wb); in inode_cgwb_move_to_attached()
305 __acquires(&wb->list_lock) in locked_inode_to_wb_and_lock_list()
308 struct bdi_writeback *wb = inode_to_wb(inode); in locked_inode_to_wb_and_lock_list() local
316 wb_get(wb); in locked_inode_to_wb_and_lock_list()
318 spin_lock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
321 if (likely(wb == inode->i_wb)) { in locked_inode_to_wb_and_lock_list()
322 wb_put(wb); /* @inode already has ref */ in locked_inode_to_wb_and_lock_list()
323 return wb; in locked_inode_to_wb_and_lock_list()
326 spin_unlock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
327 wb_put(wb); in locked_inode_to_wb_and_lock_list()
341 __acquires(&wb->list_lock) in inode_to_wb_and_lock_list()
642 bool cleanup_offline_cgwb(struct bdi_writeback *wb) in cleanup_offline_cgwb() argument
656 for (memcg_css = wb->memcg_css->parent; memcg_css; in cleanup_offline_cgwb()
658 isw->new_wb = wb_get_create(wb->bdi, memcg_css, GFP_KERNEL); in cleanup_offline_cgwb()
663 isw->new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */ in cleanup_offline_cgwb()
666 spin_lock(&wb->list_lock); in cleanup_offline_cgwb()
675 restart = isw_prepare_wbs_switch(isw, &wb->b_attached, &nr); in cleanup_offline_cgwb()
677 restart = isw_prepare_wbs_switch(isw, &wb->b_dirty_time, &nr); in cleanup_offline_cgwb()
678 spin_unlock(&wb->list_lock); in cleanup_offline_cgwb()
718 wbc->wb = inode_to_wb(inode); in wbc_attach_and_unlock_inode()
721 wbc->wb_id = wbc->wb->memcg_css->id; in wbc_attach_and_unlock_inode()
728 wb_get(wbc->wb); in wbc_attach_and_unlock_inode()
738 if (unlikely(wb_dying(wbc->wb) && !css_is_dying(wbc->wb->memcg_css))) in wbc_attach_and_unlock_inode()
782 struct bdi_writeback *wb = wbc->wb; in wbc_detach_inode() local
788 if (!wb) in wbc_detach_inode()
815 wb->avg_write_bandwidth); in wbc_detach_inode()
861 wb_put(wbc->wb); in wbc_detach_inode()
862 wbc->wb = NULL; in wbc_detach_inode()
889 if (!wbc->wb || wbc->no_cgroup_owner) in wbc_account_cgroup_owner()
927 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages) in wb_split_bdi_pages() argument
929 unsigned long this_bw = wb->avg_write_bandwidth; in wb_split_bdi_pages()
930 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); in wb_split_bdi_pages()
962 struct bdi_writeback *wb = list_entry(&bdi->wb_list, in bdi_split_work_to_wbs() local
968 list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) { in bdi_split_work_to_wbs()
980 if (!wb_has_dirty_io(wb) && in bdi_split_work_to_wbs()
982 list_empty(&wb->b_dirty_time))) in bdi_split_work_to_wbs()
984 if (skip_if_busy && writeback_in_progress(wb)) in bdi_split_work_to_wbs()
987 nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages); in bdi_split_work_to_wbs()
994 wb_queue_work(wb, work); in bdi_split_work_to_wbs()
1005 if (!wb_tryget(wb)) in bdi_split_work_to_wbs()
1015 wb_queue_work(wb, work); in bdi_split_work_to_wbs()
1016 last_wb = wb; in bdi_split_work_to_wbs()
1043 struct bdi_writeback *wb; in cgroup_writeback_by_id() local
1067 wb = wb_get_lookup(bdi, memcg_css); in cgroup_writeback_by_id()
1068 if (!wb) { in cgroup_writeback_by_id()
1095 wb_queue_work(wb, work); in cgroup_writeback_by_id()
1101 wb_put(wb); in cgroup_writeback_by_id()
1152 struct bdi_writeback *wb) in inode_cgwb_move_to_attached() argument
1154 assert_spin_locked(&wb->list_lock); in inode_cgwb_move_to_attached()
1160 wb_io_lists_depopulated(wb); in inode_cgwb_move_to_attached()
1166 __acquires(&wb->list_lock) in locked_inode_to_wb_and_lock_list()
1168 struct bdi_writeback *wb = inode_to_wb(inode); in locked_inode_to_wb_and_lock_list() local
1171 spin_lock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
1172 return wb; in locked_inode_to_wb_and_lock_list()
1176 __acquires(&wb->list_lock) in inode_to_wb_and_lock_list()
1178 struct bdi_writeback *wb = inode_to_wb(inode); in inode_to_wb_and_lock_list() local
1180 spin_lock(&wb->list_lock); in inode_to_wb_and_lock_list()
1181 return wb; in inode_to_wb_and_lock_list()
1184 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages) in wb_split_bdi_pages() argument
1195 if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) { in bdi_split_work_to_wbs()
1197 wb_queue_work(&bdi->wb, base_work); in bdi_split_work_to_wbs()
1213 static void wb_start_writeback(struct bdi_writeback *wb, enum wb_reason reason) in wb_start_writeback() argument
1215 if (!wb_has_dirty_io(wb)) in wb_start_writeback()
1226 if (test_bit(WB_start_all, &wb->state) || in wb_start_writeback()
1227 test_and_set_bit(WB_start_all, &wb->state)) in wb_start_writeback()
1230 wb->start_all_reason = reason; in wb_start_writeback()
1231 wb_wakeup(wb); in wb_start_writeback()
1244 void wb_start_background_writeback(struct bdi_writeback *wb) in wb_start_background_writeback() argument
1250 trace_writeback_wake_background(wb); in wb_start_background_writeback()
1251 wb_wakeup(wb); in wb_start_background_writeback()
1259 struct bdi_writeback *wb; in inode_io_list_del() local
1261 wb = inode_to_wb_and_lock_list(inode); in inode_io_list_del()
1266 wb_io_lists_depopulated(wb); in inode_io_list_del()
1269 spin_unlock(&wb->list_lock); in inode_io_list_del()
1318 static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb) in redirty_tail_locked() argument
1330 wb_io_lists_depopulated(wb); in redirty_tail_locked()
1333 if (!list_empty(&wb->b_dirty)) { in redirty_tail_locked()
1336 tail = wb_inode(wb->b_dirty.next); in redirty_tail_locked()
1340 inode_io_list_move_locked(inode, wb, &wb->b_dirty); in redirty_tail_locked()
1343 static void redirty_tail(struct inode *inode, struct bdi_writeback *wb) in redirty_tail() argument
1346 redirty_tail_locked(inode, wb); in redirty_tail()
1353 static void requeue_io(struct inode *inode, struct bdi_writeback *wb) in requeue_io() argument
1355 inode_io_list_move_locked(inode, wb, &wb->b_more_io); in requeue_io()
1449 static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work, in queue_io() argument
1455 assert_spin_locked(&wb->list_lock); in queue_io()
1456 list_splice_init(&wb->b_more_io, &wb->b_io); in queue_io()
1457 moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before); in queue_io()
1460 moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io, in queue_io()
1463 wb_io_lists_populated(wb); in queue_io()
1464 trace_writeback_queue_io(wb, work, dirtied_before, moved); in queue_io()
1538 static void requeue_inode(struct inode *inode, struct bdi_writeback *wb, in requeue_inode() argument
1561 redirty_tail_locked(inode, wb); in requeue_inode()
1563 inode_cgwb_move_to_attached(inode, wb); in requeue_inode()
1574 requeue_io(inode, wb); in requeue_inode()
1583 redirty_tail_locked(inode, wb); in requeue_inode()
1591 redirty_tail_locked(inode, wb); in requeue_inode()
1594 inode_io_list_move_locked(inode, wb, &wb->b_dirty_time); in requeue_inode()
1598 inode_cgwb_move_to_attached(inode, wb); in requeue_inode()
1711 struct bdi_writeback *wb; in writeback_single_inode() local
1750 wb = inode_to_wb_and_lock_list(inode); in writeback_single_inode()
1763 inode_cgwb_move_to_attached(inode, wb); in writeback_single_inode()
1766 redirty_tail_locked(inode, wb); in writeback_single_inode()
1770 wb, in writeback_single_inode()
1771 &wb->b_dirty_time); in writeback_single_inode()
1776 spin_unlock(&wb->list_lock); in writeback_single_inode()
1783 static long writeback_chunk_size(struct bdi_writeback *wb, in writeback_chunk_size() argument
1804 pages = min(wb->avg_write_bandwidth / 2, in writeback_chunk_size()
1824 struct bdi_writeback *wb, in writeback_sb_inodes() argument
1841 while (!list_empty(&wb->b_io)) { in writeback_sb_inodes()
1842 struct inode *inode = wb_inode(wb->b_io.prev); in writeback_sb_inodes()
1853 redirty_tail(inode, wb); in writeback_sb_inodes()
1872 redirty_tail_locked(inode, wb); in writeback_sb_inodes()
1886 requeue_io(inode, wb); in writeback_sb_inodes()
1891 spin_unlock(&wb->list_lock); in writeback_sb_inodes()
1902 spin_lock(&wb->list_lock); in writeback_sb_inodes()
1908 write_chunk = writeback_chunk_size(wb, work); in writeback_sb_inodes()
1949 if (unlikely(tmp_wb != wb)) { in writeback_sb_inodes()
1951 spin_lock(&wb->list_lock); in writeback_sb_inodes()
1968 static long __writeback_inodes_wb(struct bdi_writeback *wb, in __writeback_inodes_wb() argument
1974 while (!list_empty(&wb->b_io)) { in __writeback_inodes_wb()
1975 struct inode *inode = wb_inode(wb->b_io.prev); in __writeback_inodes_wb()
1984 redirty_tail(inode, wb); in __writeback_inodes_wb()
1987 wrote += writeback_sb_inodes(sb, wb, work); in __writeback_inodes_wb()
2002 static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages, in writeback_inodes_wb() argument
2014 spin_lock(&wb->list_lock); in writeback_inodes_wb()
2015 if (list_empty(&wb->b_io)) in writeback_inodes_wb()
2016 queue_io(wb, &work, jiffies); in writeback_inodes_wb()
2017 __writeback_inodes_wb(wb, &work); in writeback_inodes_wb()
2018 spin_unlock(&wb->list_lock); in writeback_inodes_wb()
2039 static long wb_writeback(struct bdi_writeback *wb, in wb_writeback() argument
2064 !list_empty(&wb->work_list)) in wb_writeback()
2071 if (work->for_background && !wb_over_bg_thresh(wb)) in wb_writeback()
2075 spin_lock(&wb->list_lock); in wb_writeback()
2089 trace_writeback_start(wb, work); in wb_writeback()
2090 if (list_empty(&wb->b_io)) { in wb_writeback()
2091 queue_io(wb, work, dirtied_before); in wb_writeback()
2095 progress = writeback_sb_inodes(work->sb, wb, work); in wb_writeback()
2097 progress = __writeback_inodes_wb(wb, work); in wb_writeback()
2098 trace_writeback_written(wb, work); in wb_writeback()
2109 spin_unlock(&wb->list_lock); in wb_writeback()
2116 if (list_empty(&wb->b_more_io)) { in wb_writeback()
2117 spin_unlock(&wb->list_lock); in wb_writeback()
2126 trace_writeback_wait(wb, work); in wb_writeback()
2127 inode = wb_inode(wb->b_more_io.prev); in wb_writeback()
2129 spin_unlock(&wb->list_lock); in wb_writeback()
2141 static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb) in get_next_work_item() argument
2145 spin_lock_irq(&wb->work_lock); in get_next_work_item()
2146 if (!list_empty(&wb->work_list)) { in get_next_work_item()
2147 work = list_entry(wb->work_list.next, in get_next_work_item()
2151 spin_unlock_irq(&wb->work_lock); in get_next_work_item()
2155 static long wb_check_background_flush(struct bdi_writeback *wb) in wb_check_background_flush() argument
2157 if (wb_over_bg_thresh(wb)) { in wb_check_background_flush()
2167 return wb_writeback(wb, &work); in wb_check_background_flush()
2173 static long wb_check_old_data_flush(struct bdi_writeback *wb) in wb_check_old_data_flush() argument
2184 expired = wb->last_old_flush + in wb_check_old_data_flush()
2189 wb->last_old_flush = jiffies; in wb_check_old_data_flush()
2201 return wb_writeback(wb, &work); in wb_check_old_data_flush()
2207 static long wb_check_start_all(struct bdi_writeback *wb) in wb_check_start_all() argument
2211 if (!test_bit(WB_start_all, &wb->state)) in wb_check_start_all()
2217 .nr_pages = wb_split_bdi_pages(wb, nr_pages), in wb_check_start_all()
2220 .reason = wb->start_all_reason, in wb_check_start_all()
2223 nr_pages = wb_writeback(wb, &work); in wb_check_start_all()
2226 clear_bit(WB_start_all, &wb->state); in wb_check_start_all()
2234 static long wb_do_writeback(struct bdi_writeback *wb) in wb_do_writeback() argument
2239 set_bit(WB_writeback_running, &wb->state); in wb_do_writeback()
2240 while ((work = get_next_work_item(wb)) != NULL) { in wb_do_writeback()
2241 trace_writeback_exec(wb, work); in wb_do_writeback()
2242 wrote += wb_writeback(wb, work); in wb_do_writeback()
2243 finish_writeback_work(wb, work); in wb_do_writeback()
2249 wrote += wb_check_start_all(wb); in wb_do_writeback()
2254 wrote += wb_check_old_data_flush(wb); in wb_do_writeback()
2255 wrote += wb_check_background_flush(wb); in wb_do_writeback()
2256 clear_bit(WB_writeback_running, &wb->state); in wb_do_writeback()
2267 struct bdi_writeback *wb = container_of(to_delayed_work(work), in wb_workfn() local
2271 set_worker_desc("flush-%s", bdi_dev_name(wb->bdi)); in wb_workfn()
2274 !test_bit(WB_registered, &wb->state))) { in wb_workfn()
2282 pages_written = wb_do_writeback(wb); in wb_workfn()
2284 } while (!list_empty(&wb->work_list)); in wb_workfn()
2291 pages_written = writeback_inodes_wb(wb, 1024, in wb_workfn()
2296 if (!list_empty(&wb->work_list)) in wb_workfn()
2297 wb_wakeup(wb); in wb_workfn()
2298 else if (wb_has_dirty_io(wb) && dirty_writeback_interval) in wb_workfn()
2299 wb_wakeup_delayed(wb); in wb_workfn()
2309 struct bdi_writeback *wb; in __wakeup_flusher_threads_bdi() local
2314 list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) in __wakeup_flusher_threads_bdi()
2315 wb_start_writeback(wb, reason); in __wakeup_flusher_threads_bdi()
2368 struct bdi_writeback *wb; in wakeup_dirtytime_writeback() local
2370 list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) in wakeup_dirtytime_writeback()
2371 if (!list_empty(&wb->b_dirty_time)) in wakeup_dirtytime_writeback()
2372 wb_wakeup(wb); in wakeup_dirtytime_writeback()
2427 struct bdi_writeback *wb = NULL; in __mark_inode_dirty() local
2495 wb = locked_inode_to_wb_and_lock_list(inode); in __mark_inode_dirty()
2532 dirty_list = &wb->b_dirty; in __mark_inode_dirty()
2534 dirty_list = &wb->b_dirty_time; in __mark_inode_dirty()
2536 wakeup_bdi = inode_io_list_move_locked(inode, wb, in __mark_inode_dirty()
2539 spin_unlock(&wb->list_lock); in __mark_inode_dirty()
2550 (wb->bdi->capabilities & BDI_CAP_WRITEBACK)) in __mark_inode_dirty()
2551 wb_wakeup_delayed(wb); in __mark_inode_dirty()
2556 if (wb) in __mark_inode_dirty()
2557 spin_unlock(&wb->list_lock); in __mark_inode_dirty()