1 /* 2 * fs/fs-writeback.c 3 * 4 * Copyright (C) 2002, Linus Torvalds. 5 * 6 * Contains all the functions related to writing back and waiting 7 * upon dirty inodes against superblocks, and writing back dirty 8 * pages against inodes. ie: data writeback. Writeout of the 9 * inode itself is not handled here. 10 * 11 * 10Apr2002 Andrew Morton 12 * Split out of fs/inode.c 13 * Additions for address_space-based writeback 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/export.h> 18 #include <linux/spinlock.h> 19 #include <linux/slab.h> 20 #include <linux/sched.h> 21 #include <linux/fs.h> 22 #include <linux/mm.h> 23 #include <linux/pagemap.h> 24 #include <linux/kthread.h> 25 #include <linux/writeback.h> 26 #include <linux/blkdev.h> 27 #include <linux/backing-dev.h> 28 #include <linux/tracepoint.h> 29 #include <linux/device.h> 30 #include <linux/memcontrol.h> 31 #include "internal.h" 32 33 /* 34 * 4MB minimal write chunk size 35 */ 36 #define MIN_WRITEBACK_PAGES (4096UL >> (PAGE_CACHE_SHIFT - 10)) 37 38 struct wb_completion { 39 atomic_t cnt; 40 }; 41 42 /* 43 * Passed into wb_writeback(), essentially a subset of writeback_control 44 */ 45 struct wb_writeback_work { 46 long nr_pages; 47 struct super_block *sb; 48 unsigned long *older_than_this; 49 enum writeback_sync_modes sync_mode; 50 unsigned int tagged_writepages:1; 51 unsigned int for_kupdate:1; 52 unsigned int range_cyclic:1; 53 unsigned int for_background:1; 54 unsigned int for_sync:1; /* sync(2) WB_SYNC_ALL writeback */ 55 unsigned int auto_free:1; /* free on completion */ 56 enum wb_reason reason; /* why was writeback initiated? */ 57 58 struct list_head list; /* pending work list */ 59 struct wb_completion *done; /* set if the caller waits */ 60 }; 61 62 /* 63 * If one wants to wait for one or more wb_writeback_works, each work's 64 * ->done should be set to a wb_completion defined using the following 65 * macro. Once all work items are issued with wb_queue_work(), the caller 66 * can wait for the completion of all using wb_wait_for_completion(). Work 67 * items which are waited upon aren't freed automatically on completion. 68 */ 69 #define DEFINE_WB_COMPLETION_ONSTACK(cmpl) \ 70 struct wb_completion cmpl = { \ 71 .cnt = ATOMIC_INIT(1), \ 72 } 73 74 75 /* 76 * If an inode is constantly having its pages dirtied, but then the 77 * updates stop dirtytime_expire_interval seconds in the past, it's 78 * possible for the worst case time between when an inode has its 79 * timestamps updated and when they finally get written out to be two 80 * dirtytime_expire_intervals. We set the default to 12 hours (in 81 * seconds), which means most of the time inodes will have their 82 * timestamps written to disk after 12 hours, but in the worst case a 83 * few inodes might not their timestamps updated for 24 hours. 84 */ 85 unsigned int dirtytime_expire_interval = 12 * 60 * 60; 86 87 static inline struct inode *wb_inode(struct list_head *head) 88 { 89 return list_entry(head, struct inode, i_io_list); 90 } 91 92 /* 93 * Include the creation of the trace points after defining the 94 * wb_writeback_work structure and inline functions so that the definition 95 * remains local to this file. 96 */ 97 #define CREATE_TRACE_POINTS 98 #include <trace/events/writeback.h> 99 100 EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage); 101 102 static bool wb_io_lists_populated(struct bdi_writeback *wb) 103 { 104 if (wb_has_dirty_io(wb)) { 105 return false; 106 } else { 107 set_bit(WB_has_dirty_io, &wb->state); 108 WARN_ON_ONCE(!wb->avg_write_bandwidth); 109 atomic_long_add(wb->avg_write_bandwidth, 110 &wb->bdi->tot_write_bandwidth); 111 return true; 112 } 113 } 114 115 static void wb_io_lists_depopulated(struct bdi_writeback *wb) 116 { 117 if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) && 118 list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) { 119 clear_bit(WB_has_dirty_io, &wb->state); 120 WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth, 121 &wb->bdi->tot_write_bandwidth) < 0); 122 } 123 } 124 125 /** 126 * inode_io_list_move_locked - move an inode onto a bdi_writeback IO list 127 * @inode: inode to be moved 128 * @wb: target bdi_writeback 129 * @head: one of @wb->b_{dirty|io|more_io} 130 * 131 * Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io. 132 * Returns %true if @inode is the first occupant of the !dirty_time IO 133 * lists; otherwise, %false. 134 */ 135 static bool inode_io_list_move_locked(struct inode *inode, 136 struct bdi_writeback *wb, 137 struct list_head *head) 138 { 139 assert_spin_locked(&wb->list_lock); 140 141 list_move(&inode->i_io_list, head); 142 143 /* dirty_time doesn't count as dirty_io until expiration */ 144 if (head != &wb->b_dirty_time) 145 return wb_io_lists_populated(wb); 146 147 wb_io_lists_depopulated(wb); 148 return false; 149 } 150 151 /** 152 * inode_io_list_del_locked - remove an inode from its bdi_writeback IO list 153 * @inode: inode to be removed 154 * @wb: bdi_writeback @inode is being removed from 155 * 156 * Remove @inode which may be on one of @wb->b_{dirty|io|more_io} lists and 157 * clear %WB_has_dirty_io if all are empty afterwards. 158 */ 159 static void inode_io_list_del_locked(struct inode *inode, 160 struct bdi_writeback *wb) 161 { 162 assert_spin_locked(&wb->list_lock); 163 164 list_del_init(&inode->i_io_list); 165 wb_io_lists_depopulated(wb); 166 } 167 168 static void wb_wakeup(struct bdi_writeback *wb) 169 { 170 spin_lock_bh(&wb->work_lock); 171 if (test_bit(WB_registered, &wb->state)) 172 mod_delayed_work(bdi_wq, &wb->dwork, 0); 173 spin_unlock_bh(&wb->work_lock); 174 } 175 176 static void wb_queue_work(struct bdi_writeback *wb, 177 struct wb_writeback_work *work) 178 { 179 trace_writeback_queue(wb, work); 180 181 spin_lock_bh(&wb->work_lock); 182 if (!test_bit(WB_registered, &wb->state)) 183 goto out_unlock; 184 if (work->done) 185 atomic_inc(&work->done->cnt); 186 list_add_tail(&work->list, &wb->work_list); 187 mod_delayed_work(bdi_wq, &wb->dwork, 0); 188 out_unlock: 189 spin_unlock_bh(&wb->work_lock); 190 } 191 192 /** 193 * wb_wait_for_completion - wait for completion of bdi_writeback_works 194 * @bdi: bdi work items were issued to 195 * @done: target wb_completion 196 * 197 * Wait for one or more work items issued to @bdi with their ->done field 198 * set to @done, which should have been defined with 199 * DEFINE_WB_COMPLETION_ONSTACK(). This function returns after all such 200 * work items are completed. Work items which are waited upon aren't freed 201 * automatically on completion. 202 */ 203 static void wb_wait_for_completion(struct backing_dev_info *bdi, 204 struct wb_completion *done) 205 { 206 atomic_dec(&done->cnt); /* put down the initial count */ 207 wait_event(bdi->wb_waitq, !atomic_read(&done->cnt)); 208 } 209 210 #ifdef CONFIG_CGROUP_WRITEBACK 211 212 /* parameters for foreign inode detection, see wb_detach_inode() */ 213 #define WB_FRN_TIME_SHIFT 13 /* 1s = 2^13, upto 8 secs w/ 16bit */ 214 #define WB_FRN_TIME_AVG_SHIFT 3 /* avg = avg * 7/8 + new * 1/8 */ 215 #define WB_FRN_TIME_CUT_DIV 2 /* ignore rounds < avg / 2 */ 216 #define WB_FRN_TIME_PERIOD (2 * (1 << WB_FRN_TIME_SHIFT)) /* 2s */ 217 218 #define WB_FRN_HIST_SLOTS 16 /* inode->i_wb_frn_history is 16bit */ 219 #define WB_FRN_HIST_UNIT (WB_FRN_TIME_PERIOD / WB_FRN_HIST_SLOTS) 220 /* each slot's duration is 2s / 16 */ 221 #define WB_FRN_HIST_THR_SLOTS (WB_FRN_HIST_SLOTS / 2) 222 /* if foreign slots >= 8, switch */ 223 #define WB_FRN_HIST_MAX_SLOTS (WB_FRN_HIST_THR_SLOTS / 2 + 1) 224 /* one round can affect upto 5 slots */ 225 226 void __inode_attach_wb(struct inode *inode, struct page *page) 227 { 228 struct backing_dev_info *bdi = inode_to_bdi(inode); 229 struct bdi_writeback *wb = NULL; 230 231 if (inode_cgwb_enabled(inode)) { 232 struct cgroup_subsys_state *memcg_css; 233 234 if (page) { 235 memcg_css = mem_cgroup_css_from_page(page); 236 wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); 237 } else { 238 /* must pin memcg_css, see wb_get_create() */ 239 memcg_css = task_get_css(current, memory_cgrp_id); 240 wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); 241 css_put(memcg_css); 242 } 243 } 244 245 if (!wb) 246 wb = &bdi->wb; 247 248 /* 249 * There may be multiple instances of this function racing to 250 * update the same inode. Use cmpxchg() to tell the winner. 251 */ 252 if (unlikely(cmpxchg(&inode->i_wb, NULL, wb))) 253 wb_put(wb); 254 } 255 256 /** 257 * locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it 258 * @inode: inode of interest with i_lock held 259 * 260 * Returns @inode's wb with its list_lock held. @inode->i_lock must be 261 * held on entry and is released on return. The returned wb is guaranteed 262 * to stay @inode's associated wb until its list_lock is released. 263 */ 264 static struct bdi_writeback * 265 locked_inode_to_wb_and_lock_list(struct inode *inode) 266 __releases(&inode->i_lock) 267 __acquires(&wb->list_lock) 268 { 269 while (true) { 270 struct bdi_writeback *wb = inode_to_wb(inode); 271 272 /* 273 * inode_to_wb() association is protected by both 274 * @inode->i_lock and @wb->list_lock but list_lock nests 275 * outside i_lock. Drop i_lock and verify that the 276 * association hasn't changed after acquiring list_lock. 277 */ 278 wb_get(wb); 279 spin_unlock(&inode->i_lock); 280 spin_lock(&wb->list_lock); 281 wb_put(wb); /* not gonna deref it anymore */ 282 283 /* i_wb may have changed inbetween, can't use inode_to_wb() */ 284 if (likely(wb == inode->i_wb)) 285 return wb; /* @inode already has ref */ 286 287 spin_unlock(&wb->list_lock); 288 cpu_relax(); 289 spin_lock(&inode->i_lock); 290 } 291 } 292 293 /** 294 * inode_to_wb_and_lock_list - determine an inode's wb and lock it 295 * @inode: inode of interest 296 * 297 * Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held 298 * on entry. 299 */ 300 static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode) 301 __acquires(&wb->list_lock) 302 { 303 spin_lock(&inode->i_lock); 304 return locked_inode_to_wb_and_lock_list(inode); 305 } 306 307 struct inode_switch_wbs_context { 308 struct inode *inode; 309 struct bdi_writeback *new_wb; 310 311 struct rcu_head rcu_head; 312 struct work_struct work; 313 }; 314 315 static void inode_switch_wbs_work_fn(struct work_struct *work) 316 { 317 struct inode_switch_wbs_context *isw = 318 container_of(work, struct inode_switch_wbs_context, work); 319 struct inode *inode = isw->inode; 320 struct address_space *mapping = inode->i_mapping; 321 struct bdi_writeback *old_wb = inode->i_wb; 322 struct bdi_writeback *new_wb = isw->new_wb; 323 struct radix_tree_iter iter; 324 bool switched = false; 325 void **slot; 326 327 /* 328 * By the time control reaches here, RCU grace period has passed 329 * since I_WB_SWITCH assertion and all wb stat update transactions 330 * between unlocked_inode_to_wb_begin/end() are guaranteed to be 331 * synchronizing against mapping->tree_lock. 332 * 333 * Grabbing old_wb->list_lock, inode->i_lock and mapping->tree_lock 334 * gives us exclusion against all wb related operations on @inode 335 * including IO list manipulations and stat updates. 336 */ 337 if (old_wb < new_wb) { 338 spin_lock(&old_wb->list_lock); 339 spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING); 340 } else { 341 spin_lock(&new_wb->list_lock); 342 spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); 343 } 344 spin_lock(&inode->i_lock); 345 spin_lock_irq(&mapping->tree_lock); 346 347 /* 348 * Once I_FREEING is visible under i_lock, the eviction path owns 349 * the inode and we shouldn't modify ->i_io_list. 350 */ 351 if (unlikely(inode->i_state & I_FREEING)) 352 goto skip_switch; 353 354 /* 355 * Count and transfer stats. Note that PAGECACHE_TAG_DIRTY points 356 * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to 357 * pages actually under underwriteback. 358 */ 359 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0, 360 PAGECACHE_TAG_DIRTY) { 361 struct page *page = radix_tree_deref_slot_protected(slot, 362 &mapping->tree_lock); 363 if (likely(page) && PageDirty(page)) { 364 __dec_wb_stat(old_wb, WB_RECLAIMABLE); 365 __inc_wb_stat(new_wb, WB_RECLAIMABLE); 366 } 367 } 368 369 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0, 370 PAGECACHE_TAG_WRITEBACK) { 371 struct page *page = radix_tree_deref_slot_protected(slot, 372 &mapping->tree_lock); 373 if (likely(page)) { 374 WARN_ON_ONCE(!PageWriteback(page)); 375 __dec_wb_stat(old_wb, WB_WRITEBACK); 376 __inc_wb_stat(new_wb, WB_WRITEBACK); 377 } 378 } 379 380 wb_get(new_wb); 381 382 /* 383 * Transfer to @new_wb's IO list if necessary. The specific list 384 * @inode was on is ignored and the inode is put on ->b_dirty which 385 * is always correct including from ->b_dirty_time. The transfer 386 * preserves @inode->dirtied_when ordering. 387 */ 388 if (!list_empty(&inode->i_io_list)) { 389 struct inode *pos; 390 391 inode_io_list_del_locked(inode, old_wb); 392 inode->i_wb = new_wb; 393 list_for_each_entry(pos, &new_wb->b_dirty, i_io_list) 394 if (time_after_eq(inode->dirtied_when, 395 pos->dirtied_when)) 396 break; 397 inode_io_list_move_locked(inode, new_wb, pos->i_io_list.prev); 398 } else { 399 inode->i_wb = new_wb; 400 } 401 402 /* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */ 403 inode->i_wb_frn_winner = 0; 404 inode->i_wb_frn_avg_time = 0; 405 inode->i_wb_frn_history = 0; 406 switched = true; 407 skip_switch: 408 /* 409 * Paired with load_acquire in unlocked_inode_to_wb_begin() and 410 * ensures that the new wb is visible if they see !I_WB_SWITCH. 411 */ 412 smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH); 413 414 spin_unlock_irq(&mapping->tree_lock); 415 spin_unlock(&inode->i_lock); 416 spin_unlock(&new_wb->list_lock); 417 spin_unlock(&old_wb->list_lock); 418 419 if (switched) { 420 wb_wakeup(new_wb); 421 wb_put(old_wb); 422 } 423 wb_put(new_wb); 424 425 iput(inode); 426 kfree(isw); 427 } 428 429 static void inode_switch_wbs_rcu_fn(struct rcu_head *rcu_head) 430 { 431 struct inode_switch_wbs_context *isw = container_of(rcu_head, 432 struct inode_switch_wbs_context, rcu_head); 433 434 /* needs to grab bh-unsafe locks, bounce to work item */ 435 INIT_WORK(&isw->work, inode_switch_wbs_work_fn); 436 schedule_work(&isw->work); 437 } 438 439 /** 440 * inode_switch_wbs - change the wb association of an inode 441 * @inode: target inode 442 * @new_wb_id: ID of the new wb 443 * 444 * Switch @inode's wb association to the wb identified by @new_wb_id. The 445 * switching is performed asynchronously and may fail silently. 446 */ 447 static void inode_switch_wbs(struct inode *inode, int new_wb_id) 448 { 449 struct backing_dev_info *bdi = inode_to_bdi(inode); 450 struct cgroup_subsys_state *memcg_css; 451 struct inode_switch_wbs_context *isw; 452 453 /* noop if seems to be already in progress */ 454 if (inode->i_state & I_WB_SWITCH) 455 return; 456 457 isw = kzalloc(sizeof(*isw), GFP_ATOMIC); 458 if (!isw) 459 return; 460 461 /* find and pin the new wb */ 462 rcu_read_lock(); 463 memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys); 464 if (memcg_css) 465 isw->new_wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); 466 rcu_read_unlock(); 467 if (!isw->new_wb) 468 goto out_free; 469 470 /* while holding I_WB_SWITCH, no one else can update the association */ 471 spin_lock(&inode->i_lock); 472 if (inode->i_state & (I_WB_SWITCH | I_FREEING) || 473 inode_to_wb(inode) == isw->new_wb) { 474 spin_unlock(&inode->i_lock); 475 goto out_free; 476 } 477 inode->i_state |= I_WB_SWITCH; 478 spin_unlock(&inode->i_lock); 479 480 ihold(inode); 481 isw->inode = inode; 482 483 /* 484 * In addition to synchronizing among switchers, I_WB_SWITCH tells 485 * the RCU protected stat update paths to grab the mapping's 486 * tree_lock so that stat transfer can synchronize against them. 487 * Let's continue after I_WB_SWITCH is guaranteed to be visible. 488 */ 489 call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn); 490 return; 491 492 out_free: 493 if (isw->new_wb) 494 wb_put(isw->new_wb); 495 kfree(isw); 496 } 497 498 /** 499 * wbc_attach_and_unlock_inode - associate wbc with target inode and unlock it 500 * @wbc: writeback_control of interest 501 * @inode: target inode 502 * 503 * @inode is locked and about to be written back under the control of @wbc. 504 * Record @inode's writeback context into @wbc and unlock the i_lock. On 505 * writeback completion, wbc_detach_inode() should be called. This is used 506 * to track the cgroup writeback context. 507 */ 508 void wbc_attach_and_unlock_inode(struct writeback_control *wbc, 509 struct inode *inode) 510 { 511 if (!inode_cgwb_enabled(inode)) { 512 spin_unlock(&inode->i_lock); 513 return; 514 } 515 516 wbc->wb = inode_to_wb(inode); 517 wbc->inode = inode; 518 519 wbc->wb_id = wbc->wb->memcg_css->id; 520 wbc->wb_lcand_id = inode->i_wb_frn_winner; 521 wbc->wb_tcand_id = 0; 522 wbc->wb_bytes = 0; 523 wbc->wb_lcand_bytes = 0; 524 wbc->wb_tcand_bytes = 0; 525 526 wb_get(wbc->wb); 527 spin_unlock(&inode->i_lock); 528 529 /* 530 * A dying wb indicates that the memcg-blkcg mapping has changed 531 * and a new wb is already serving the memcg. Switch immediately. 532 */ 533 if (unlikely(wb_dying(wbc->wb))) 534 inode_switch_wbs(inode, wbc->wb_id); 535 } 536 537 /** 538 * wbc_detach_inode - disassociate wbc from inode and perform foreign detection 539 * @wbc: writeback_control of the just finished writeback 540 * 541 * To be called after a writeback attempt of an inode finishes and undoes 542 * wbc_attach_and_unlock_inode(). Can be called under any context. 543 * 544 * As concurrent write sharing of an inode is expected to be very rare and 545 * memcg only tracks page ownership on first-use basis severely confining 546 * the usefulness of such sharing, cgroup writeback tracks ownership 547 * per-inode. While the support for concurrent write sharing of an inode 548 * is deemed unnecessary, an inode being written to by different cgroups at 549 * different points in time is a lot more common, and, more importantly, 550 * charging only by first-use can too readily lead to grossly incorrect 551 * behaviors (single foreign page can lead to gigabytes of writeback to be 552 * incorrectly attributed). 553 * 554 * To resolve this issue, cgroup writeback detects the majority dirtier of 555 * an inode and transfers the ownership to it. To avoid unnnecessary 556 * oscillation, the detection mechanism keeps track of history and gives 557 * out the switch verdict only if the foreign usage pattern is stable over 558 * a certain amount of time and/or writeback attempts. 559 * 560 * On each writeback attempt, @wbc tries to detect the majority writer 561 * using Boyer-Moore majority vote algorithm. In addition to the byte 562 * count from the majority voting, it also counts the bytes written for the 563 * current wb and the last round's winner wb (max of last round's current 564 * wb, the winner from two rounds ago, and the last round's majority 565 * candidate). Keeping track of the historical winner helps the algorithm 566 * to semi-reliably detect the most active writer even when it's not the 567 * absolute majority. 568 * 569 * Once the winner of the round is determined, whether the winner is 570 * foreign or not and how much IO time the round consumed is recorded in 571 * inode->i_wb_frn_history. If the amount of recorded foreign IO time is 572 * over a certain threshold, the switch verdict is given. 573 */ 574 void wbc_detach_inode(struct writeback_control *wbc) 575 { 576 struct bdi_writeback *wb = wbc->wb; 577 struct inode *inode = wbc->inode; 578 unsigned long avg_time, max_bytes, max_time; 579 u16 history; 580 int max_id; 581 582 if (!wb) 583 return; 584 585 history = inode->i_wb_frn_history; 586 avg_time = inode->i_wb_frn_avg_time; 587 588 /* pick the winner of this round */ 589 if (wbc->wb_bytes >= wbc->wb_lcand_bytes && 590 wbc->wb_bytes >= wbc->wb_tcand_bytes) { 591 max_id = wbc->wb_id; 592 max_bytes = wbc->wb_bytes; 593 } else if (wbc->wb_lcand_bytes >= wbc->wb_tcand_bytes) { 594 max_id = wbc->wb_lcand_id; 595 max_bytes = wbc->wb_lcand_bytes; 596 } else { 597 max_id = wbc->wb_tcand_id; 598 max_bytes = wbc->wb_tcand_bytes; 599 } 600 601 /* 602 * Calculate the amount of IO time the winner consumed and fold it 603 * into the running average kept per inode. If the consumed IO 604 * time is lower than avag / WB_FRN_TIME_CUT_DIV, ignore it for 605 * deciding whether to switch or not. This is to prevent one-off 606 * small dirtiers from skewing the verdict. 607 */ 608 max_time = DIV_ROUND_UP((max_bytes >> PAGE_SHIFT) << WB_FRN_TIME_SHIFT, 609 wb->avg_write_bandwidth); 610 if (avg_time) 611 avg_time += (max_time >> WB_FRN_TIME_AVG_SHIFT) - 612 (avg_time >> WB_FRN_TIME_AVG_SHIFT); 613 else 614 avg_time = max_time; /* immediate catch up on first run */ 615 616 if (max_time >= avg_time / WB_FRN_TIME_CUT_DIV) { 617 int slots; 618 619 /* 620 * The switch verdict is reached if foreign wb's consume 621 * more than a certain proportion of IO time in a 622 * WB_FRN_TIME_PERIOD. This is loosely tracked by 16 slot 623 * history mask where each bit represents one sixteenth of 624 * the period. Determine the number of slots to shift into 625 * history from @max_time. 626 */ 627 slots = min(DIV_ROUND_UP(max_time, WB_FRN_HIST_UNIT), 628 (unsigned long)WB_FRN_HIST_MAX_SLOTS); 629 history <<= slots; 630 if (wbc->wb_id != max_id) 631 history |= (1U << slots) - 1; 632 633 /* 634 * Switch if the current wb isn't the consistent winner. 635 * If there are multiple closely competing dirtiers, the 636 * inode may switch across them repeatedly over time, which 637 * is okay. The main goal is avoiding keeping an inode on 638 * the wrong wb for an extended period of time. 639 */ 640 if (hweight32(history) > WB_FRN_HIST_THR_SLOTS) 641 inode_switch_wbs(inode, max_id); 642 } 643 644 /* 645 * Multiple instances of this function may race to update the 646 * following fields but we don't mind occassional inaccuracies. 647 */ 648 inode->i_wb_frn_winner = max_id; 649 inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX); 650 inode->i_wb_frn_history = history; 651 652 wb_put(wbc->wb); 653 wbc->wb = NULL; 654 } 655 656 /** 657 * wbc_account_io - account IO issued during writeback 658 * @wbc: writeback_control of the writeback in progress 659 * @page: page being written out 660 * @bytes: number of bytes being written out 661 * 662 * @bytes from @page are about to written out during the writeback 663 * controlled by @wbc. Keep the book for foreign inode detection. See 664 * wbc_detach_inode(). 665 */ 666 void wbc_account_io(struct writeback_control *wbc, struct page *page, 667 size_t bytes) 668 { 669 int id; 670 671 /* 672 * pageout() path doesn't attach @wbc to the inode being written 673 * out. This is intentional as we don't want the function to block 674 * behind a slow cgroup. Ultimately, we want pageout() to kick off 675 * regular writeback instead of writing things out itself. 676 */ 677 if (!wbc->wb) 678 return; 679 680 id = mem_cgroup_css_from_page(page)->id; 681 682 if (id == wbc->wb_id) { 683 wbc->wb_bytes += bytes; 684 return; 685 } 686 687 if (id == wbc->wb_lcand_id) 688 wbc->wb_lcand_bytes += bytes; 689 690 /* Boyer-Moore majority vote algorithm */ 691 if (!wbc->wb_tcand_bytes) 692 wbc->wb_tcand_id = id; 693 if (id == wbc->wb_tcand_id) 694 wbc->wb_tcand_bytes += bytes; 695 else 696 wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes); 697 } 698 EXPORT_SYMBOL_GPL(wbc_account_io); 699 700 /** 701 * inode_congested - test whether an inode is congested 702 * @inode: inode to test for congestion (may be NULL) 703 * @cong_bits: mask of WB_[a]sync_congested bits to test 704 * 705 * Tests whether @inode is congested. @cong_bits is the mask of congestion 706 * bits to test and the return value is the mask of set bits. 707 * 708 * If cgroup writeback is enabled for @inode, the congestion state is 709 * determined by whether the cgwb (cgroup bdi_writeback) for the blkcg 710 * associated with @inode is congested; otherwise, the root wb's congestion 711 * state is used. 712 * 713 * @inode is allowed to be NULL as this function is often called on 714 * mapping->host which is NULL for the swapper space. 715 */ 716 int inode_congested(struct inode *inode, int cong_bits) 717 { 718 /* 719 * Once set, ->i_wb never becomes NULL while the inode is alive. 720 * Start transaction iff ->i_wb is visible. 721 */ 722 if (inode && inode_to_wb_is_valid(inode)) { 723 struct bdi_writeback *wb; 724 bool locked, congested; 725 726 wb = unlocked_inode_to_wb_begin(inode, &locked); 727 congested = wb_congested(wb, cong_bits); 728 unlocked_inode_to_wb_end(inode, locked); 729 return congested; 730 } 731 732 return wb_congested(&inode_to_bdi(inode)->wb, cong_bits); 733 } 734 EXPORT_SYMBOL_GPL(inode_congested); 735 736 /** 737 * wb_split_bdi_pages - split nr_pages to write according to bandwidth 738 * @wb: target bdi_writeback to split @nr_pages to 739 * @nr_pages: number of pages to write for the whole bdi 740 * 741 * Split @wb's portion of @nr_pages according to @wb's write bandwidth in 742 * relation to the total write bandwidth of all wb's w/ dirty inodes on 743 * @wb->bdi. 744 */ 745 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages) 746 { 747 unsigned long this_bw = wb->avg_write_bandwidth; 748 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); 749 750 if (nr_pages == LONG_MAX) 751 return LONG_MAX; 752 753 /* 754 * This may be called on clean wb's and proportional distribution 755 * may not make sense, just use the original @nr_pages in those 756 * cases. In general, we wanna err on the side of writing more. 757 */ 758 if (!tot_bw || this_bw >= tot_bw) 759 return nr_pages; 760 else 761 return DIV_ROUND_UP_ULL((u64)nr_pages * this_bw, tot_bw); 762 } 763 764 /** 765 * bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi 766 * @bdi: target backing_dev_info 767 * @base_work: wb_writeback_work to issue 768 * @skip_if_busy: skip wb's which already have writeback in progress 769 * 770 * Split and issue @base_work to all wb's (bdi_writeback's) of @bdi which 771 * have dirty inodes. If @base_work->nr_page isn't %LONG_MAX, it's 772 * distributed to the busy wbs according to each wb's proportion in the 773 * total active write bandwidth of @bdi. 774 */ 775 static void bdi_split_work_to_wbs(struct backing_dev_info *bdi, 776 struct wb_writeback_work *base_work, 777 bool skip_if_busy) 778 { 779 struct bdi_writeback *last_wb = NULL; 780 struct bdi_writeback *wb = list_entry(&bdi->wb_list, 781 struct bdi_writeback, bdi_node); 782 783 might_sleep(); 784 restart: 785 rcu_read_lock(); 786 list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) { 787 DEFINE_WB_COMPLETION_ONSTACK(fallback_work_done); 788 struct wb_writeback_work fallback_work; 789 struct wb_writeback_work *work; 790 long nr_pages; 791 792 if (last_wb) { 793 wb_put(last_wb); 794 last_wb = NULL; 795 } 796 797 /* SYNC_ALL writes out I_DIRTY_TIME too */ 798 if (!wb_has_dirty_io(wb) && 799 (base_work->sync_mode == WB_SYNC_NONE || 800 list_empty(&wb->b_dirty_time))) 801 continue; 802 if (skip_if_busy && writeback_in_progress(wb)) 803 continue; 804 805 nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages); 806 807 work = kmalloc(sizeof(*work), GFP_ATOMIC); 808 if (work) { 809 *work = *base_work; 810 work->nr_pages = nr_pages; 811 work->auto_free = 1; 812 wb_queue_work(wb, work); 813 continue; 814 } 815 816 /* alloc failed, execute synchronously using on-stack fallback */ 817 work = &fallback_work; 818 *work = *base_work; 819 work->nr_pages = nr_pages; 820 work->auto_free = 0; 821 work->done = &fallback_work_done; 822 823 wb_queue_work(wb, work); 824 825 /* 826 * Pin @wb so that it stays on @bdi->wb_list. This allows 827 * continuing iteration from @wb after dropping and 828 * regrabbing rcu read lock. 829 */ 830 wb_get(wb); 831 last_wb = wb; 832 833 rcu_read_unlock(); 834 wb_wait_for_completion(bdi, &fallback_work_done); 835 goto restart; 836 } 837 rcu_read_unlock(); 838 839 if (last_wb) 840 wb_put(last_wb); 841 } 842 843 #else /* CONFIG_CGROUP_WRITEBACK */ 844 845 static struct bdi_writeback * 846 locked_inode_to_wb_and_lock_list(struct inode *inode) 847 __releases(&inode->i_lock) 848 __acquires(&wb->list_lock) 849 { 850 struct bdi_writeback *wb = inode_to_wb(inode); 851 852 spin_unlock(&inode->i_lock); 853 spin_lock(&wb->list_lock); 854 return wb; 855 } 856 857 static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode) 858 __acquires(&wb->list_lock) 859 { 860 struct bdi_writeback *wb = inode_to_wb(inode); 861 862 spin_lock(&wb->list_lock); 863 return wb; 864 } 865 866 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages) 867 { 868 return nr_pages; 869 } 870 871 static void bdi_split_work_to_wbs(struct backing_dev_info *bdi, 872 struct wb_writeback_work *base_work, 873 bool skip_if_busy) 874 { 875 might_sleep(); 876 877 if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) { 878 base_work->auto_free = 0; 879 wb_queue_work(&bdi->wb, base_work); 880 } 881 } 882 883 #endif /* CONFIG_CGROUP_WRITEBACK */ 884 885 void wb_start_writeback(struct bdi_writeback *wb, long nr_pages, 886 bool range_cyclic, enum wb_reason reason) 887 { 888 struct wb_writeback_work *work; 889 890 if (!wb_has_dirty_io(wb)) 891 return; 892 893 /* 894 * This is WB_SYNC_NONE writeback, so if allocation fails just 895 * wakeup the thread for old dirty data writeback 896 */ 897 work = kzalloc(sizeof(*work), GFP_ATOMIC); 898 if (!work) { 899 trace_writeback_nowork(wb); 900 wb_wakeup(wb); 901 return; 902 } 903 904 work->sync_mode = WB_SYNC_NONE; 905 work->nr_pages = nr_pages; 906 work->range_cyclic = range_cyclic; 907 work->reason = reason; 908 work->auto_free = 1; 909 910 wb_queue_work(wb, work); 911 } 912 913 /** 914 * wb_start_background_writeback - start background writeback 915 * @wb: bdi_writback to write from 916 * 917 * Description: 918 * This makes sure WB_SYNC_NONE background writeback happens. When 919 * this function returns, it is only guaranteed that for given wb 920 * some IO is happening if we are over background dirty threshold. 921 * Caller need not hold sb s_umount semaphore. 922 */ 923 void wb_start_background_writeback(struct bdi_writeback *wb) 924 { 925 /* 926 * We just wake up the flusher thread. It will perform background 927 * writeback as soon as there is no other work to do. 928 */ 929 trace_writeback_wake_background(wb); 930 wb_wakeup(wb); 931 } 932 933 /* 934 * Remove the inode from the writeback list it is on. 935 */ 936 void inode_io_list_del(struct inode *inode) 937 { 938 struct bdi_writeback *wb; 939 940 wb = inode_to_wb_and_lock_list(inode); 941 inode_io_list_del_locked(inode, wb); 942 spin_unlock(&wb->list_lock); 943 } 944 945 /* 946 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the 947 * furthest end of its superblock's dirty-inode list. 948 * 949 * Before stamping the inode's ->dirtied_when, we check to see whether it is 950 * already the most-recently-dirtied inode on the b_dirty list. If that is 951 * the case then the inode must have been redirtied while it was being written 952 * out and we don't reset its dirtied_when. 953 */ 954 static void redirty_tail(struct inode *inode, struct bdi_writeback *wb) 955 { 956 if (!list_empty(&wb->b_dirty)) { 957 struct inode *tail; 958 959 tail = wb_inode(wb->b_dirty.next); 960 if (time_before(inode->dirtied_when, tail->dirtied_when)) 961 inode->dirtied_when = jiffies; 962 } 963 inode_io_list_move_locked(inode, wb, &wb->b_dirty); 964 } 965 966 /* 967 * requeue inode for re-scanning after bdi->b_io list is exhausted. 968 */ 969 static void requeue_io(struct inode *inode, struct bdi_writeback *wb) 970 { 971 inode_io_list_move_locked(inode, wb, &wb->b_more_io); 972 } 973 974 static void inode_sync_complete(struct inode *inode) 975 { 976 inode->i_state &= ~I_SYNC; 977 /* If inode is clean an unused, put it into LRU now... */ 978 inode_add_lru(inode); 979 /* Waiters must see I_SYNC cleared before being woken up */ 980 smp_mb(); 981 wake_up_bit(&inode->i_state, __I_SYNC); 982 } 983 984 static bool inode_dirtied_after(struct inode *inode, unsigned long t) 985 { 986 bool ret = time_after(inode->dirtied_when, t); 987 #ifndef CONFIG_64BIT 988 /* 989 * For inodes being constantly redirtied, dirtied_when can get stuck. 990 * It _appears_ to be in the future, but is actually in distant past. 991 * This test is necessary to prevent such wrapped-around relative times 992 * from permanently stopping the whole bdi writeback. 993 */ 994 ret = ret && time_before_eq(inode->dirtied_when, jiffies); 995 #endif 996 return ret; 997 } 998 999 #define EXPIRE_DIRTY_ATIME 0x0001 1000 1001 /* 1002 * Move expired (dirtied before work->older_than_this) dirty inodes from 1003 * @delaying_queue to @dispatch_queue. 1004 */ 1005 static int move_expired_inodes(struct list_head *delaying_queue, 1006 struct list_head *dispatch_queue, 1007 int flags, 1008 struct wb_writeback_work *work) 1009 { 1010 unsigned long *older_than_this = NULL; 1011 unsigned long expire_time; 1012 LIST_HEAD(tmp); 1013 struct list_head *pos, *node; 1014 struct super_block *sb = NULL; 1015 struct inode *inode; 1016 int do_sb_sort = 0; 1017 int moved = 0; 1018 1019 if ((flags & EXPIRE_DIRTY_ATIME) == 0) 1020 older_than_this = work->older_than_this; 1021 else if (!work->for_sync) { 1022 expire_time = jiffies - (dirtytime_expire_interval * HZ); 1023 older_than_this = &expire_time; 1024 } 1025 while (!list_empty(delaying_queue)) { 1026 inode = wb_inode(delaying_queue->prev); 1027 if (older_than_this && 1028 inode_dirtied_after(inode, *older_than_this)) 1029 break; 1030 list_move(&inode->i_io_list, &tmp); 1031 moved++; 1032 if (flags & EXPIRE_DIRTY_ATIME) 1033 set_bit(__I_DIRTY_TIME_EXPIRED, &inode->i_state); 1034 if (sb_is_blkdev_sb(inode->i_sb)) 1035 continue; 1036 if (sb && sb != inode->i_sb) 1037 do_sb_sort = 1; 1038 sb = inode->i_sb; 1039 } 1040 1041 /* just one sb in list, splice to dispatch_queue and we're done */ 1042 if (!do_sb_sort) { 1043 list_splice(&tmp, dispatch_queue); 1044 goto out; 1045 } 1046 1047 /* Move inodes from one superblock together */ 1048 while (!list_empty(&tmp)) { 1049 sb = wb_inode(tmp.prev)->i_sb; 1050 list_for_each_prev_safe(pos, node, &tmp) { 1051 inode = wb_inode(pos); 1052 if (inode->i_sb == sb) 1053 list_move(&inode->i_io_list, dispatch_queue); 1054 } 1055 } 1056 out: 1057 return moved; 1058 } 1059 1060 /* 1061 * Queue all expired dirty inodes for io, eldest first. 1062 * Before 1063 * newly dirtied b_dirty b_io b_more_io 1064 * =============> gf edc BA 1065 * After 1066 * newly dirtied b_dirty b_io b_more_io 1067 * =============> g fBAedc 1068 * | 1069 * +--> dequeue for IO 1070 */ 1071 static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work) 1072 { 1073 int moved; 1074 1075 assert_spin_locked(&wb->list_lock); 1076 list_splice_init(&wb->b_more_io, &wb->b_io); 1077 moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, 0, work); 1078 moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io, 1079 EXPIRE_DIRTY_ATIME, work); 1080 if (moved) 1081 wb_io_lists_populated(wb); 1082 trace_writeback_queue_io(wb, work, moved); 1083 } 1084 1085 static int write_inode(struct inode *inode, struct writeback_control *wbc) 1086 { 1087 int ret; 1088 1089 if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) { 1090 trace_writeback_write_inode_start(inode, wbc); 1091 ret = inode->i_sb->s_op->write_inode(inode, wbc); 1092 trace_writeback_write_inode(inode, wbc); 1093 return ret; 1094 } 1095 return 0; 1096 } 1097 1098 /* 1099 * Wait for writeback on an inode to complete. Called with i_lock held. 1100 * Caller must make sure inode cannot go away when we drop i_lock. 1101 */ 1102 static void __inode_wait_for_writeback(struct inode *inode) 1103 __releases(inode->i_lock) 1104 __acquires(inode->i_lock) 1105 { 1106 DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC); 1107 wait_queue_head_t *wqh; 1108 1109 wqh = bit_waitqueue(&inode->i_state, __I_SYNC); 1110 while (inode->i_state & I_SYNC) { 1111 spin_unlock(&inode->i_lock); 1112 __wait_on_bit(wqh, &wq, bit_wait, 1113 TASK_UNINTERRUPTIBLE); 1114 spin_lock(&inode->i_lock); 1115 } 1116 } 1117 1118 /* 1119 * Wait for writeback on an inode to complete. Caller must have inode pinned. 1120 */ 1121 void inode_wait_for_writeback(struct inode *inode) 1122 { 1123 spin_lock(&inode->i_lock); 1124 __inode_wait_for_writeback(inode); 1125 spin_unlock(&inode->i_lock); 1126 } 1127 1128 /* 1129 * Sleep until I_SYNC is cleared. This function must be called with i_lock 1130 * held and drops it. It is aimed for callers not holding any inode reference 1131 * so once i_lock is dropped, inode can go away. 1132 */ 1133 static void inode_sleep_on_writeback(struct inode *inode) 1134 __releases(inode->i_lock) 1135 { 1136 DEFINE_WAIT(wait); 1137 wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC); 1138 int sleep; 1139 1140 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 1141 sleep = inode->i_state & I_SYNC; 1142 spin_unlock(&inode->i_lock); 1143 if (sleep) 1144 schedule(); 1145 finish_wait(wqh, &wait); 1146 } 1147 1148 /* 1149 * Find proper writeback list for the inode depending on its current state and 1150 * possibly also change of its state while we were doing writeback. Here we 1151 * handle things such as livelock prevention or fairness of writeback among 1152 * inodes. This function can be called only by flusher thread - noone else 1153 * processes all inodes in writeback lists and requeueing inodes behind flusher 1154 * thread's back can have unexpected consequences. 1155 */ 1156 static void requeue_inode(struct inode *inode, struct bdi_writeback *wb, 1157 struct writeback_control *wbc) 1158 { 1159 if (inode->i_state & I_FREEING) 1160 return; 1161 1162 /* 1163 * Sync livelock prevention. Each inode is tagged and synced in one 1164 * shot. If still dirty, it will be redirty_tail()'ed below. Update 1165 * the dirty time to prevent enqueue and sync it again. 1166 */ 1167 if ((inode->i_state & I_DIRTY) && 1168 (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)) 1169 inode->dirtied_when = jiffies; 1170 1171 if (wbc->pages_skipped) { 1172 /* 1173 * writeback is not making progress due to locked 1174 * buffers. Skip this inode for now. 1175 */ 1176 redirty_tail(inode, wb); 1177 return; 1178 } 1179 1180 if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) { 1181 /* 1182 * We didn't write back all the pages. nfs_writepages() 1183 * sometimes bales out without doing anything. 1184 */ 1185 if (wbc->nr_to_write <= 0) { 1186 /* Slice used up. Queue for next turn. */ 1187 requeue_io(inode, wb); 1188 } else { 1189 /* 1190 * Writeback blocked by something other than 1191 * congestion. Delay the inode for some time to 1192 * avoid spinning on the CPU (100% iowait) 1193 * retrying writeback of the dirty page/inode 1194 * that cannot be performed immediately. 1195 */ 1196 redirty_tail(inode, wb); 1197 } 1198 } else if (inode->i_state & I_DIRTY) { 1199 /* 1200 * Filesystems can dirty the inode during writeback operations, 1201 * such as delayed allocation during submission or metadata 1202 * updates after data IO completion. 1203 */ 1204 redirty_tail(inode, wb); 1205 } else if (inode->i_state & I_DIRTY_TIME) { 1206 inode->dirtied_when = jiffies; 1207 inode_io_list_move_locked(inode, wb, &wb->b_dirty_time); 1208 } else { 1209 /* The inode is clean. Remove from writeback lists. */ 1210 inode_io_list_del_locked(inode, wb); 1211 } 1212 } 1213 1214 /* 1215 * Write out an inode and its dirty pages. Do not update the writeback list 1216 * linkage. That is left to the caller. The caller is also responsible for 1217 * setting I_SYNC flag and calling inode_sync_complete() to clear it. 1218 */ 1219 static int 1220 __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) 1221 { 1222 struct address_space *mapping = inode->i_mapping; 1223 long nr_to_write = wbc->nr_to_write; 1224 unsigned dirty; 1225 int ret; 1226 1227 WARN_ON(!(inode->i_state & I_SYNC)); 1228 1229 trace_writeback_single_inode_start(inode, wbc, nr_to_write); 1230 1231 ret = do_writepages(mapping, wbc); 1232 1233 /* 1234 * Make sure to wait on the data before writing out the metadata. 1235 * This is important for filesystems that modify metadata on data 1236 * I/O completion. We don't do it for sync(2) writeback because it has a 1237 * separate, external IO completion path and ->sync_fs for guaranteeing 1238 * inode metadata is written back correctly. 1239 */ 1240 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) { 1241 int err = filemap_fdatawait(mapping); 1242 if (ret == 0) 1243 ret = err; 1244 } 1245 1246 /* 1247 * Some filesystems may redirty the inode during the writeback 1248 * due to delalloc, clear dirty metadata flags right before 1249 * write_inode() 1250 */ 1251 spin_lock(&inode->i_lock); 1252 1253 dirty = inode->i_state & I_DIRTY; 1254 if (inode->i_state & I_DIRTY_TIME) { 1255 if ((dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) || 1256 unlikely(inode->i_state & I_DIRTY_TIME_EXPIRED) || 1257 unlikely(time_after(jiffies, 1258 (inode->dirtied_time_when + 1259 dirtytime_expire_interval * HZ)))) { 1260 dirty |= I_DIRTY_TIME | I_DIRTY_TIME_EXPIRED; 1261 trace_writeback_lazytime(inode); 1262 } 1263 } else 1264 inode->i_state &= ~I_DIRTY_TIME_EXPIRED; 1265 inode->i_state &= ~dirty; 1266 1267 /* 1268 * Paired with smp_mb() in __mark_inode_dirty(). This allows 1269 * __mark_inode_dirty() to test i_state without grabbing i_lock - 1270 * either they see the I_DIRTY bits cleared or we see the dirtied 1271 * inode. 1272 * 1273 * I_DIRTY_PAGES is always cleared together above even if @mapping 1274 * still has dirty pages. The flag is reinstated after smp_mb() if 1275 * necessary. This guarantees that either __mark_inode_dirty() 1276 * sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY. 1277 */ 1278 smp_mb(); 1279 1280 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 1281 inode->i_state |= I_DIRTY_PAGES; 1282 1283 spin_unlock(&inode->i_lock); 1284 1285 if (dirty & I_DIRTY_TIME) 1286 mark_inode_dirty_sync(inode); 1287 /* Don't write the inode if only I_DIRTY_PAGES was set */ 1288 if (dirty & ~I_DIRTY_PAGES) { 1289 int err = write_inode(inode, wbc); 1290 if (ret == 0) 1291 ret = err; 1292 } 1293 trace_writeback_single_inode(inode, wbc, nr_to_write); 1294 return ret; 1295 } 1296 1297 /* 1298 * Write out an inode's dirty pages. Either the caller has an active reference 1299 * on the inode or the inode has I_WILL_FREE set. 1300 * 1301 * This function is designed to be called for writing back one inode which 1302 * we go e.g. from filesystem. Flusher thread uses __writeback_single_inode() 1303 * and does more profound writeback list handling in writeback_sb_inodes(). 1304 */ 1305 static int 1306 writeback_single_inode(struct inode *inode, struct bdi_writeback *wb, 1307 struct writeback_control *wbc) 1308 { 1309 int ret = 0; 1310 1311 spin_lock(&inode->i_lock); 1312 if (!atomic_read(&inode->i_count)) 1313 WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING))); 1314 else 1315 WARN_ON(inode->i_state & I_WILL_FREE); 1316 1317 if (inode->i_state & I_SYNC) { 1318 if (wbc->sync_mode != WB_SYNC_ALL) 1319 goto out; 1320 /* 1321 * It's a data-integrity sync. We must wait. Since callers hold 1322 * inode reference or inode has I_WILL_FREE set, it cannot go 1323 * away under us. 1324 */ 1325 __inode_wait_for_writeback(inode); 1326 } 1327 WARN_ON(inode->i_state & I_SYNC); 1328 /* 1329 * Skip inode if it is clean and we have no outstanding writeback in 1330 * WB_SYNC_ALL mode. We don't want to mess with writeback lists in this 1331 * function since flusher thread may be doing for example sync in 1332 * parallel and if we move the inode, it could get skipped. So here we 1333 * make sure inode is on some writeback list and leave it there unless 1334 * we have completely cleaned the inode. 1335 */ 1336 if (!(inode->i_state & I_DIRTY_ALL) && 1337 (wbc->sync_mode != WB_SYNC_ALL || 1338 !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))) 1339 goto out; 1340 inode->i_state |= I_SYNC; 1341 wbc_attach_and_unlock_inode(wbc, inode); 1342 1343 ret = __writeback_single_inode(inode, wbc); 1344 1345 wbc_detach_inode(wbc); 1346 spin_lock(&wb->list_lock); 1347 spin_lock(&inode->i_lock); 1348 /* 1349 * If inode is clean, remove it from writeback lists. Otherwise don't 1350 * touch it. See comment above for explanation. 1351 */ 1352 if (!(inode->i_state & I_DIRTY_ALL)) 1353 inode_io_list_del_locked(inode, wb); 1354 spin_unlock(&wb->list_lock); 1355 inode_sync_complete(inode); 1356 out: 1357 spin_unlock(&inode->i_lock); 1358 return ret; 1359 } 1360 1361 static long writeback_chunk_size(struct bdi_writeback *wb, 1362 struct wb_writeback_work *work) 1363 { 1364 long pages; 1365 1366 /* 1367 * WB_SYNC_ALL mode does livelock avoidance by syncing dirty 1368 * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX 1369 * here avoids calling into writeback_inodes_wb() more than once. 1370 * 1371 * The intended call sequence for WB_SYNC_ALL writeback is: 1372 * 1373 * wb_writeback() 1374 * writeback_sb_inodes() <== called only once 1375 * write_cache_pages() <== called once for each inode 1376 * (quickly) tag currently dirty pages 1377 * (maybe slowly) sync all tagged pages 1378 */ 1379 if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages) 1380 pages = LONG_MAX; 1381 else { 1382 pages = min(wb->avg_write_bandwidth / 2, 1383 global_wb_domain.dirty_limit / DIRTY_SCOPE); 1384 pages = min(pages, work->nr_pages); 1385 pages = round_down(pages + MIN_WRITEBACK_PAGES, 1386 MIN_WRITEBACK_PAGES); 1387 } 1388 1389 return pages; 1390 } 1391 1392 /* 1393 * Write a portion of b_io inodes which belong to @sb. 1394 * 1395 * Return the number of pages and/or inodes written. 1396 * 1397 * NOTE! This is called with wb->list_lock held, and will 1398 * unlock and relock that for each inode it ends up doing 1399 * IO for. 1400 */ 1401 static long writeback_sb_inodes(struct super_block *sb, 1402 struct bdi_writeback *wb, 1403 struct wb_writeback_work *work) 1404 { 1405 struct writeback_control wbc = { 1406 .sync_mode = work->sync_mode, 1407 .tagged_writepages = work->tagged_writepages, 1408 .for_kupdate = work->for_kupdate, 1409 .for_background = work->for_background, 1410 .for_sync = work->for_sync, 1411 .range_cyclic = work->range_cyclic, 1412 .range_start = 0, 1413 .range_end = LLONG_MAX, 1414 }; 1415 unsigned long start_time = jiffies; 1416 long write_chunk; 1417 long wrote = 0; /* count both pages and inodes */ 1418 1419 while (!list_empty(&wb->b_io)) { 1420 struct inode *inode = wb_inode(wb->b_io.prev); 1421 1422 if (inode->i_sb != sb) { 1423 if (work->sb) { 1424 /* 1425 * We only want to write back data for this 1426 * superblock, move all inodes not belonging 1427 * to it back onto the dirty list. 1428 */ 1429 redirty_tail(inode, wb); 1430 continue; 1431 } 1432 1433 /* 1434 * The inode belongs to a different superblock. 1435 * Bounce back to the caller to unpin this and 1436 * pin the next superblock. 1437 */ 1438 break; 1439 } 1440 1441 /* 1442 * Don't bother with new inodes or inodes being freed, first 1443 * kind does not need periodic writeout yet, and for the latter 1444 * kind writeout is handled by the freer. 1445 */ 1446 spin_lock(&inode->i_lock); 1447 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) { 1448 spin_unlock(&inode->i_lock); 1449 redirty_tail(inode, wb); 1450 continue; 1451 } 1452 if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) { 1453 /* 1454 * If this inode is locked for writeback and we are not 1455 * doing writeback-for-data-integrity, move it to 1456 * b_more_io so that writeback can proceed with the 1457 * other inodes on s_io. 1458 * 1459 * We'll have another go at writing back this inode 1460 * when we completed a full scan of b_io. 1461 */ 1462 spin_unlock(&inode->i_lock); 1463 requeue_io(inode, wb); 1464 trace_writeback_sb_inodes_requeue(inode); 1465 continue; 1466 } 1467 spin_unlock(&wb->list_lock); 1468 1469 /* 1470 * We already requeued the inode if it had I_SYNC set and we 1471 * are doing WB_SYNC_NONE writeback. So this catches only the 1472 * WB_SYNC_ALL case. 1473 */ 1474 if (inode->i_state & I_SYNC) { 1475 /* Wait for I_SYNC. This function drops i_lock... */ 1476 inode_sleep_on_writeback(inode); 1477 /* Inode may be gone, start again */ 1478 spin_lock(&wb->list_lock); 1479 continue; 1480 } 1481 inode->i_state |= I_SYNC; 1482 wbc_attach_and_unlock_inode(&wbc, inode); 1483 1484 write_chunk = writeback_chunk_size(wb, work); 1485 wbc.nr_to_write = write_chunk; 1486 wbc.pages_skipped = 0; 1487 1488 /* 1489 * We use I_SYNC to pin the inode in memory. While it is set 1490 * evict_inode() will wait so the inode cannot be freed. 1491 */ 1492 __writeback_single_inode(inode, &wbc); 1493 1494 wbc_detach_inode(&wbc); 1495 work->nr_pages -= write_chunk - wbc.nr_to_write; 1496 wrote += write_chunk - wbc.nr_to_write; 1497 1498 if (need_resched()) { 1499 /* 1500 * We're trying to balance between building up a nice 1501 * long list of IOs to improve our merge rate, and 1502 * getting those IOs out quickly for anyone throttling 1503 * in balance_dirty_pages(). cond_resched() doesn't 1504 * unplug, so get our IOs out the door before we 1505 * give up the CPU. 1506 */ 1507 blk_flush_plug(current); 1508 cond_resched(); 1509 } 1510 1511 1512 spin_lock(&wb->list_lock); 1513 spin_lock(&inode->i_lock); 1514 if (!(inode->i_state & I_DIRTY_ALL)) 1515 wrote++; 1516 requeue_inode(inode, wb, &wbc); 1517 inode_sync_complete(inode); 1518 spin_unlock(&inode->i_lock); 1519 1520 /* 1521 * bail out to wb_writeback() often enough to check 1522 * background threshold and other termination conditions. 1523 */ 1524 if (wrote) { 1525 if (time_is_before_jiffies(start_time + HZ / 10UL)) 1526 break; 1527 if (work->nr_pages <= 0) 1528 break; 1529 } 1530 } 1531 return wrote; 1532 } 1533 1534 static long __writeback_inodes_wb(struct bdi_writeback *wb, 1535 struct wb_writeback_work *work) 1536 { 1537 unsigned long start_time = jiffies; 1538 long wrote = 0; 1539 1540 while (!list_empty(&wb->b_io)) { 1541 struct inode *inode = wb_inode(wb->b_io.prev); 1542 struct super_block *sb = inode->i_sb; 1543 1544 if (!trylock_super(sb)) { 1545 /* 1546 * trylock_super() may fail consistently due to 1547 * s_umount being grabbed by someone else. Don't use 1548 * requeue_io() to avoid busy retrying the inode/sb. 1549 */ 1550 redirty_tail(inode, wb); 1551 continue; 1552 } 1553 wrote += writeback_sb_inodes(sb, wb, work); 1554 up_read(&sb->s_umount); 1555 1556 /* refer to the same tests at the end of writeback_sb_inodes */ 1557 if (wrote) { 1558 if (time_is_before_jiffies(start_time + HZ / 10UL)) 1559 break; 1560 if (work->nr_pages <= 0) 1561 break; 1562 } 1563 } 1564 /* Leave any unwritten inodes on b_io */ 1565 return wrote; 1566 } 1567 1568 static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages, 1569 enum wb_reason reason) 1570 { 1571 struct wb_writeback_work work = { 1572 .nr_pages = nr_pages, 1573 .sync_mode = WB_SYNC_NONE, 1574 .range_cyclic = 1, 1575 .reason = reason, 1576 }; 1577 struct blk_plug plug; 1578 1579 blk_start_plug(&plug); 1580 spin_lock(&wb->list_lock); 1581 if (list_empty(&wb->b_io)) 1582 queue_io(wb, &work); 1583 __writeback_inodes_wb(wb, &work); 1584 spin_unlock(&wb->list_lock); 1585 blk_finish_plug(&plug); 1586 1587 return nr_pages - work.nr_pages; 1588 } 1589 1590 /* 1591 * Explicit flushing or periodic writeback of "old" data. 1592 * 1593 * Define "old": the first time one of an inode's pages is dirtied, we mark the 1594 * dirtying-time in the inode's address_space. So this periodic writeback code 1595 * just walks the superblock inode list, writing back any inodes which are 1596 * older than a specific point in time. 1597 * 1598 * Try to run once per dirty_writeback_interval. But if a writeback event 1599 * takes longer than a dirty_writeback_interval interval, then leave a 1600 * one-second gap. 1601 * 1602 * older_than_this takes precedence over nr_to_write. So we'll only write back 1603 * all dirty pages if they are all attached to "old" mappings. 1604 */ 1605 static long wb_writeback(struct bdi_writeback *wb, 1606 struct wb_writeback_work *work) 1607 { 1608 unsigned long wb_start = jiffies; 1609 long nr_pages = work->nr_pages; 1610 unsigned long oldest_jif; 1611 struct inode *inode; 1612 long progress; 1613 struct blk_plug plug; 1614 1615 oldest_jif = jiffies; 1616 work->older_than_this = &oldest_jif; 1617 1618 blk_start_plug(&plug); 1619 spin_lock(&wb->list_lock); 1620 for (;;) { 1621 /* 1622 * Stop writeback when nr_pages has been consumed 1623 */ 1624 if (work->nr_pages <= 0) 1625 break; 1626 1627 /* 1628 * Background writeout and kupdate-style writeback may 1629 * run forever. Stop them if there is other work to do 1630 * so that e.g. sync can proceed. They'll be restarted 1631 * after the other works are all done. 1632 */ 1633 if ((work->for_background || work->for_kupdate) && 1634 !list_empty(&wb->work_list)) 1635 break; 1636 1637 /* 1638 * For background writeout, stop when we are below the 1639 * background dirty threshold 1640 */ 1641 if (work->for_background && !wb_over_bg_thresh(wb)) 1642 break; 1643 1644 /* 1645 * Kupdate and background works are special and we want to 1646 * include all inodes that need writing. Livelock avoidance is 1647 * handled by these works yielding to any other work so we are 1648 * safe. 1649 */ 1650 if (work->for_kupdate) { 1651 oldest_jif = jiffies - 1652 msecs_to_jiffies(dirty_expire_interval * 10); 1653 } else if (work->for_background) 1654 oldest_jif = jiffies; 1655 1656 trace_writeback_start(wb, work); 1657 if (list_empty(&wb->b_io)) 1658 queue_io(wb, work); 1659 if (work->sb) 1660 progress = writeback_sb_inodes(work->sb, wb, work); 1661 else 1662 progress = __writeback_inodes_wb(wb, work); 1663 trace_writeback_written(wb, work); 1664 1665 wb_update_bandwidth(wb, wb_start); 1666 1667 /* 1668 * Did we write something? Try for more 1669 * 1670 * Dirty inodes are moved to b_io for writeback in batches. 1671 * The completion of the current batch does not necessarily 1672 * mean the overall work is done. So we keep looping as long 1673 * as made some progress on cleaning pages or inodes. 1674 */ 1675 if (progress) 1676 continue; 1677 /* 1678 * No more inodes for IO, bail 1679 */ 1680 if (list_empty(&wb->b_more_io)) 1681 break; 1682 /* 1683 * Nothing written. Wait for some inode to 1684 * become available for writeback. Otherwise 1685 * we'll just busyloop. 1686 */ 1687 if (!list_empty(&wb->b_more_io)) { 1688 trace_writeback_wait(wb, work); 1689 inode = wb_inode(wb->b_more_io.prev); 1690 spin_lock(&inode->i_lock); 1691 spin_unlock(&wb->list_lock); 1692 /* This function drops i_lock... */ 1693 inode_sleep_on_writeback(inode); 1694 spin_lock(&wb->list_lock); 1695 } 1696 } 1697 spin_unlock(&wb->list_lock); 1698 blk_finish_plug(&plug); 1699 1700 return nr_pages - work->nr_pages; 1701 } 1702 1703 /* 1704 * Return the next wb_writeback_work struct that hasn't been processed yet. 1705 */ 1706 static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb) 1707 { 1708 struct wb_writeback_work *work = NULL; 1709 1710 spin_lock_bh(&wb->work_lock); 1711 if (!list_empty(&wb->work_list)) { 1712 work = list_entry(wb->work_list.next, 1713 struct wb_writeback_work, list); 1714 list_del_init(&work->list); 1715 } 1716 spin_unlock_bh(&wb->work_lock); 1717 return work; 1718 } 1719 1720 /* 1721 * Add in the number of potentially dirty inodes, because each inode 1722 * write can dirty pagecache in the underlying blockdev. 1723 */ 1724 static unsigned long get_nr_dirty_pages(void) 1725 { 1726 return global_page_state(NR_FILE_DIRTY) + 1727 global_page_state(NR_UNSTABLE_NFS) + 1728 get_nr_dirty_inodes(); 1729 } 1730 1731 static long wb_check_background_flush(struct bdi_writeback *wb) 1732 { 1733 if (wb_over_bg_thresh(wb)) { 1734 1735 struct wb_writeback_work work = { 1736 .nr_pages = LONG_MAX, 1737 .sync_mode = WB_SYNC_NONE, 1738 .for_background = 1, 1739 .range_cyclic = 1, 1740 .reason = WB_REASON_BACKGROUND, 1741 }; 1742 1743 return wb_writeback(wb, &work); 1744 } 1745 1746 return 0; 1747 } 1748 1749 static long wb_check_old_data_flush(struct bdi_writeback *wb) 1750 { 1751 unsigned long expired; 1752 long nr_pages; 1753 1754 /* 1755 * When set to zero, disable periodic writeback 1756 */ 1757 if (!dirty_writeback_interval) 1758 return 0; 1759 1760 expired = wb->last_old_flush + 1761 msecs_to_jiffies(dirty_writeback_interval * 10); 1762 if (time_before(jiffies, expired)) 1763 return 0; 1764 1765 wb->last_old_flush = jiffies; 1766 nr_pages = get_nr_dirty_pages(); 1767 1768 if (nr_pages) { 1769 struct wb_writeback_work work = { 1770 .nr_pages = nr_pages, 1771 .sync_mode = WB_SYNC_NONE, 1772 .for_kupdate = 1, 1773 .range_cyclic = 1, 1774 .reason = WB_REASON_PERIODIC, 1775 }; 1776 1777 return wb_writeback(wb, &work); 1778 } 1779 1780 return 0; 1781 } 1782 1783 /* 1784 * Retrieve work items and do the writeback they describe 1785 */ 1786 static long wb_do_writeback(struct bdi_writeback *wb) 1787 { 1788 struct wb_writeback_work *work; 1789 long wrote = 0; 1790 1791 set_bit(WB_writeback_running, &wb->state); 1792 while ((work = get_next_work_item(wb)) != NULL) { 1793 struct wb_completion *done = work->done; 1794 1795 trace_writeback_exec(wb, work); 1796 1797 wrote += wb_writeback(wb, work); 1798 1799 if (work->auto_free) 1800 kfree(work); 1801 if (done && atomic_dec_and_test(&done->cnt)) 1802 wake_up_all(&wb->bdi->wb_waitq); 1803 } 1804 1805 /* 1806 * Check for periodic writeback, kupdated() style 1807 */ 1808 wrote += wb_check_old_data_flush(wb); 1809 wrote += wb_check_background_flush(wb); 1810 clear_bit(WB_writeback_running, &wb->state); 1811 1812 return wrote; 1813 } 1814 1815 /* 1816 * Handle writeback of dirty data for the device backed by this bdi. Also 1817 * reschedules periodically and does kupdated style flushing. 1818 */ 1819 void wb_workfn(struct work_struct *work) 1820 { 1821 struct bdi_writeback *wb = container_of(to_delayed_work(work), 1822 struct bdi_writeback, dwork); 1823 long pages_written; 1824 1825 set_worker_desc("flush-%s", dev_name(wb->bdi->dev)); 1826 current->flags |= PF_SWAPWRITE; 1827 1828 if (likely(!current_is_workqueue_rescuer() || 1829 !test_bit(WB_registered, &wb->state))) { 1830 /* 1831 * The normal path. Keep writing back @wb until its 1832 * work_list is empty. Note that this path is also taken 1833 * if @wb is shutting down even when we're running off the 1834 * rescuer as work_list needs to be drained. 1835 */ 1836 do { 1837 pages_written = wb_do_writeback(wb); 1838 trace_writeback_pages_written(pages_written); 1839 } while (!list_empty(&wb->work_list)); 1840 } else { 1841 /* 1842 * bdi_wq can't get enough workers and we're running off 1843 * the emergency worker. Don't hog it. Hopefully, 1024 is 1844 * enough for efficient IO. 1845 */ 1846 pages_written = writeback_inodes_wb(wb, 1024, 1847 WB_REASON_FORKER_THREAD); 1848 trace_writeback_pages_written(pages_written); 1849 } 1850 1851 if (!list_empty(&wb->work_list)) 1852 mod_delayed_work(bdi_wq, &wb->dwork, 0); 1853 else if (wb_has_dirty_io(wb) && dirty_writeback_interval) 1854 wb_wakeup_delayed(wb); 1855 1856 current->flags &= ~PF_SWAPWRITE; 1857 } 1858 1859 /* 1860 * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back 1861 * the whole world. 1862 */ 1863 void wakeup_flusher_threads(long nr_pages, enum wb_reason reason) 1864 { 1865 struct backing_dev_info *bdi; 1866 1867 if (!nr_pages) 1868 nr_pages = get_nr_dirty_pages(); 1869 1870 rcu_read_lock(); 1871 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) { 1872 struct bdi_writeback *wb; 1873 1874 if (!bdi_has_dirty_io(bdi)) 1875 continue; 1876 1877 list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) 1878 wb_start_writeback(wb, wb_split_bdi_pages(wb, nr_pages), 1879 false, reason); 1880 } 1881 rcu_read_unlock(); 1882 } 1883 1884 /* 1885 * Wake up bdi's periodically to make sure dirtytime inodes gets 1886 * written back periodically. We deliberately do *not* check the 1887 * b_dirtytime list in wb_has_dirty_io(), since this would cause the 1888 * kernel to be constantly waking up once there are any dirtytime 1889 * inodes on the system. So instead we define a separate delayed work 1890 * function which gets called much more rarely. (By default, only 1891 * once every 12 hours.) 1892 * 1893 * If there is any other write activity going on in the file system, 1894 * this function won't be necessary. But if the only thing that has 1895 * happened on the file system is a dirtytime inode caused by an atime 1896 * update, we need this infrastructure below to make sure that inode 1897 * eventually gets pushed out to disk. 1898 */ 1899 static void wakeup_dirtytime_writeback(struct work_struct *w); 1900 static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback); 1901 1902 static void wakeup_dirtytime_writeback(struct work_struct *w) 1903 { 1904 struct backing_dev_info *bdi; 1905 1906 rcu_read_lock(); 1907 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) { 1908 struct bdi_writeback *wb; 1909 1910 list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) 1911 if (!list_empty(&wb->b_dirty_time)) 1912 wb_wakeup(wb); 1913 } 1914 rcu_read_unlock(); 1915 schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ); 1916 } 1917 1918 static int __init start_dirtytime_writeback(void) 1919 { 1920 schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ); 1921 return 0; 1922 } 1923 __initcall(start_dirtytime_writeback); 1924 1925 int dirtytime_interval_handler(struct ctl_table *table, int write, 1926 void __user *buffer, size_t *lenp, loff_t *ppos) 1927 { 1928 int ret; 1929 1930 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 1931 if (ret == 0 && write) 1932 mod_delayed_work(system_wq, &dirtytime_work, 0); 1933 return ret; 1934 } 1935 1936 static noinline void block_dump___mark_inode_dirty(struct inode *inode) 1937 { 1938 if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) { 1939 struct dentry *dentry; 1940 const char *name = "?"; 1941 1942 dentry = d_find_alias(inode); 1943 if (dentry) { 1944 spin_lock(&dentry->d_lock); 1945 name = (const char *) dentry->d_name.name; 1946 } 1947 printk(KERN_DEBUG 1948 "%s(%d): dirtied inode %lu (%s) on %s\n", 1949 current->comm, task_pid_nr(current), inode->i_ino, 1950 name, inode->i_sb->s_id); 1951 if (dentry) { 1952 spin_unlock(&dentry->d_lock); 1953 dput(dentry); 1954 } 1955 } 1956 } 1957 1958 /** 1959 * __mark_inode_dirty - internal function 1960 * @inode: inode to mark 1961 * @flags: what kind of dirty (i.e. I_DIRTY_SYNC) 1962 * Mark an inode as dirty. Callers should use mark_inode_dirty or 1963 * mark_inode_dirty_sync. 1964 * 1965 * Put the inode on the super block's dirty list. 1966 * 1967 * CAREFUL! We mark it dirty unconditionally, but move it onto the 1968 * dirty list only if it is hashed or if it refers to a blockdev. 1969 * If it was not hashed, it will never be added to the dirty list 1970 * even if it is later hashed, as it will have been marked dirty already. 1971 * 1972 * In short, make sure you hash any inodes _before_ you start marking 1973 * them dirty. 1974 * 1975 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of 1976 * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of 1977 * the kernel-internal blockdev inode represents the dirtying time of the 1978 * blockdev's pages. This is why for I_DIRTY_PAGES we always use 1979 * page->mapping->host, so the page-dirtying time is recorded in the internal 1980 * blockdev inode. 1981 */ 1982 void __mark_inode_dirty(struct inode *inode, int flags) 1983 { 1984 #define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC) 1985 struct super_block *sb = inode->i_sb; 1986 int dirtytime; 1987 1988 trace_writeback_mark_inode_dirty(inode, flags); 1989 1990 /* 1991 * Don't do this for I_DIRTY_PAGES - that doesn't actually 1992 * dirty the inode itself 1993 */ 1994 if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC | I_DIRTY_TIME)) { 1995 trace_writeback_dirty_inode_start(inode, flags); 1996 1997 if (sb->s_op->dirty_inode) 1998 sb->s_op->dirty_inode(inode, flags); 1999 2000 trace_writeback_dirty_inode(inode, flags); 2001 } 2002 if (flags & I_DIRTY_INODE) 2003 flags &= ~I_DIRTY_TIME; 2004 dirtytime = flags & I_DIRTY_TIME; 2005 2006 /* 2007 * Paired with smp_mb() in __writeback_single_inode() for the 2008 * following lockless i_state test. See there for details. 2009 */ 2010 smp_mb(); 2011 2012 if (((inode->i_state & flags) == flags) || 2013 (dirtytime && (inode->i_state & I_DIRTY_INODE))) 2014 return; 2015 2016 if (unlikely(block_dump)) 2017 block_dump___mark_inode_dirty(inode); 2018 2019 spin_lock(&inode->i_lock); 2020 if (dirtytime && (inode->i_state & I_DIRTY_INODE)) 2021 goto out_unlock_inode; 2022 if ((inode->i_state & flags) != flags) { 2023 const int was_dirty = inode->i_state & I_DIRTY; 2024 2025 inode_attach_wb(inode, NULL); 2026 2027 if (flags & I_DIRTY_INODE) 2028 inode->i_state &= ~I_DIRTY_TIME; 2029 inode->i_state |= flags; 2030 2031 /* 2032 * If the inode is being synced, just update its dirty state. 2033 * The unlocker will place the inode on the appropriate 2034 * superblock list, based upon its state. 2035 */ 2036 if (inode->i_state & I_SYNC) 2037 goto out_unlock_inode; 2038 2039 /* 2040 * Only add valid (hashed) inodes to the superblock's 2041 * dirty list. Add blockdev inodes as well. 2042 */ 2043 if (!S_ISBLK(inode->i_mode)) { 2044 if (inode_unhashed(inode)) 2045 goto out_unlock_inode; 2046 } 2047 if (inode->i_state & I_FREEING) 2048 goto out_unlock_inode; 2049 2050 /* 2051 * If the inode was already on b_dirty/b_io/b_more_io, don't 2052 * reposition it (that would break b_dirty time-ordering). 2053 */ 2054 if (!was_dirty) { 2055 struct bdi_writeback *wb; 2056 struct list_head *dirty_list; 2057 bool wakeup_bdi = false; 2058 2059 wb = locked_inode_to_wb_and_lock_list(inode); 2060 2061 WARN(bdi_cap_writeback_dirty(wb->bdi) && 2062 !test_bit(WB_registered, &wb->state), 2063 "bdi-%s not registered\n", wb->bdi->name); 2064 2065 inode->dirtied_when = jiffies; 2066 if (dirtytime) 2067 inode->dirtied_time_when = jiffies; 2068 2069 if (inode->i_state & (I_DIRTY_INODE | I_DIRTY_PAGES)) 2070 dirty_list = &wb->b_dirty; 2071 else 2072 dirty_list = &wb->b_dirty_time; 2073 2074 wakeup_bdi = inode_io_list_move_locked(inode, wb, 2075 dirty_list); 2076 2077 spin_unlock(&wb->list_lock); 2078 trace_writeback_dirty_inode_enqueue(inode); 2079 2080 /* 2081 * If this is the first dirty inode for this bdi, 2082 * we have to wake-up the corresponding bdi thread 2083 * to make sure background write-back happens 2084 * later. 2085 */ 2086 if (bdi_cap_writeback_dirty(wb->bdi) && wakeup_bdi) 2087 wb_wakeup_delayed(wb); 2088 return; 2089 } 2090 } 2091 out_unlock_inode: 2092 spin_unlock(&inode->i_lock); 2093 2094 #undef I_DIRTY_INODE 2095 } 2096 EXPORT_SYMBOL(__mark_inode_dirty); 2097 2098 /* 2099 * The @s_sync_lock is used to serialise concurrent sync operations 2100 * to avoid lock contention problems with concurrent wait_sb_inodes() calls. 2101 * Concurrent callers will block on the s_sync_lock rather than doing contending 2102 * walks. The queueing maintains sync(2) required behaviour as all the IO that 2103 * has been issued up to the time this function is enter is guaranteed to be 2104 * completed by the time we have gained the lock and waited for all IO that is 2105 * in progress regardless of the order callers are granted the lock. 2106 */ 2107 static void wait_sb_inodes(struct super_block *sb) 2108 { 2109 struct inode *inode, *old_inode = NULL; 2110 2111 /* 2112 * We need to be protected against the filesystem going from 2113 * r/o to r/w or vice versa. 2114 */ 2115 WARN_ON(!rwsem_is_locked(&sb->s_umount)); 2116 2117 mutex_lock(&sb->s_sync_lock); 2118 spin_lock(&sb->s_inode_list_lock); 2119 2120 /* 2121 * Data integrity sync. Must wait for all pages under writeback, 2122 * because there may have been pages dirtied before our sync 2123 * call, but which had writeout started before we write it out. 2124 * In which case, the inode may not be on the dirty list, but 2125 * we still have to wait for that writeout. 2126 */ 2127 list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { 2128 struct address_space *mapping = inode->i_mapping; 2129 2130 spin_lock(&inode->i_lock); 2131 if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) || 2132 (mapping->nrpages == 0)) { 2133 spin_unlock(&inode->i_lock); 2134 continue; 2135 } 2136 __iget(inode); 2137 spin_unlock(&inode->i_lock); 2138 spin_unlock(&sb->s_inode_list_lock); 2139 2140 /* 2141 * We hold a reference to 'inode' so it couldn't have been 2142 * removed from s_inodes list while we dropped the 2143 * s_inode_list_lock. We cannot iput the inode now as we can 2144 * be holding the last reference and we cannot iput it under 2145 * s_inode_list_lock. So we keep the reference and iput it 2146 * later. 2147 */ 2148 iput(old_inode); 2149 old_inode = inode; 2150 2151 /* 2152 * We keep the error status of individual mapping so that 2153 * applications can catch the writeback error using fsync(2). 2154 * See filemap_fdatawait_keep_errors() for details. 2155 */ 2156 filemap_fdatawait_keep_errors(mapping); 2157 2158 cond_resched(); 2159 2160 spin_lock(&sb->s_inode_list_lock); 2161 } 2162 spin_unlock(&sb->s_inode_list_lock); 2163 iput(old_inode); 2164 mutex_unlock(&sb->s_sync_lock); 2165 } 2166 2167 static void __writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr, 2168 enum wb_reason reason, bool skip_if_busy) 2169 { 2170 DEFINE_WB_COMPLETION_ONSTACK(done); 2171 struct wb_writeback_work work = { 2172 .sb = sb, 2173 .sync_mode = WB_SYNC_NONE, 2174 .tagged_writepages = 1, 2175 .done = &done, 2176 .nr_pages = nr, 2177 .reason = reason, 2178 }; 2179 struct backing_dev_info *bdi = sb->s_bdi; 2180 2181 if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info) 2182 return; 2183 WARN_ON(!rwsem_is_locked(&sb->s_umount)); 2184 2185 bdi_split_work_to_wbs(sb->s_bdi, &work, skip_if_busy); 2186 wb_wait_for_completion(bdi, &done); 2187 } 2188 2189 /** 2190 * writeback_inodes_sb_nr - writeback dirty inodes from given super_block 2191 * @sb: the superblock 2192 * @nr: the number of pages to write 2193 * @reason: reason why some writeback work initiated 2194 * 2195 * Start writeback on some inodes on this super_block. No guarantees are made 2196 * on how many (if any) will be written, and this function does not wait 2197 * for IO completion of submitted IO. 2198 */ 2199 void writeback_inodes_sb_nr(struct super_block *sb, 2200 unsigned long nr, 2201 enum wb_reason reason) 2202 { 2203 __writeback_inodes_sb_nr(sb, nr, reason, false); 2204 } 2205 EXPORT_SYMBOL(writeback_inodes_sb_nr); 2206 2207 /** 2208 * writeback_inodes_sb - writeback dirty inodes from given super_block 2209 * @sb: the superblock 2210 * @reason: reason why some writeback work was initiated 2211 * 2212 * Start writeback on some inodes on this super_block. No guarantees are made 2213 * on how many (if any) will be written, and this function does not wait 2214 * for IO completion of submitted IO. 2215 */ 2216 void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason) 2217 { 2218 return writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason); 2219 } 2220 EXPORT_SYMBOL(writeback_inodes_sb); 2221 2222 /** 2223 * try_to_writeback_inodes_sb_nr - try to start writeback if none underway 2224 * @sb: the superblock 2225 * @nr: the number of pages to write 2226 * @reason: the reason of writeback 2227 * 2228 * Invoke writeback_inodes_sb_nr if no writeback is currently underway. 2229 * Returns 1 if writeback was started, 0 if not. 2230 */ 2231 bool try_to_writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr, 2232 enum wb_reason reason) 2233 { 2234 if (!down_read_trylock(&sb->s_umount)) 2235 return false; 2236 2237 __writeback_inodes_sb_nr(sb, nr, reason, true); 2238 up_read(&sb->s_umount); 2239 return true; 2240 } 2241 EXPORT_SYMBOL(try_to_writeback_inodes_sb_nr); 2242 2243 /** 2244 * try_to_writeback_inodes_sb - try to start writeback if none underway 2245 * @sb: the superblock 2246 * @reason: reason why some writeback work was initiated 2247 * 2248 * Implement by try_to_writeback_inodes_sb_nr() 2249 * Returns 1 if writeback was started, 0 if not. 2250 */ 2251 bool try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason) 2252 { 2253 return try_to_writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason); 2254 } 2255 EXPORT_SYMBOL(try_to_writeback_inodes_sb); 2256 2257 /** 2258 * sync_inodes_sb - sync sb inode pages 2259 * @sb: the superblock 2260 * 2261 * This function writes and waits on any dirty inode belonging to this 2262 * super_block. 2263 */ 2264 void sync_inodes_sb(struct super_block *sb) 2265 { 2266 DEFINE_WB_COMPLETION_ONSTACK(done); 2267 struct wb_writeback_work work = { 2268 .sb = sb, 2269 .sync_mode = WB_SYNC_ALL, 2270 .nr_pages = LONG_MAX, 2271 .range_cyclic = 0, 2272 .done = &done, 2273 .reason = WB_REASON_SYNC, 2274 .for_sync = 1, 2275 }; 2276 struct backing_dev_info *bdi = sb->s_bdi; 2277 2278 /* 2279 * Can't skip on !bdi_has_dirty() because we should wait for !dirty 2280 * inodes under writeback and I_DIRTY_TIME inodes ignored by 2281 * bdi_has_dirty() need to be written out too. 2282 */ 2283 if (bdi == &noop_backing_dev_info) 2284 return; 2285 WARN_ON(!rwsem_is_locked(&sb->s_umount)); 2286 2287 bdi_split_work_to_wbs(bdi, &work, false); 2288 wb_wait_for_completion(bdi, &done); 2289 2290 wait_sb_inodes(sb); 2291 } 2292 EXPORT_SYMBOL(sync_inodes_sb); 2293 2294 /** 2295 * write_inode_now - write an inode to disk 2296 * @inode: inode to write to disk 2297 * @sync: whether the write should be synchronous or not 2298 * 2299 * This function commits an inode to disk immediately if it is dirty. This is 2300 * primarily needed by knfsd. 2301 * 2302 * The caller must either have a ref on the inode or must have set I_WILL_FREE. 2303 */ 2304 int write_inode_now(struct inode *inode, int sync) 2305 { 2306 struct bdi_writeback *wb = &inode_to_bdi(inode)->wb; 2307 struct writeback_control wbc = { 2308 .nr_to_write = LONG_MAX, 2309 .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE, 2310 .range_start = 0, 2311 .range_end = LLONG_MAX, 2312 }; 2313 2314 if (!mapping_cap_writeback_dirty(inode->i_mapping)) 2315 wbc.nr_to_write = 0; 2316 2317 might_sleep(); 2318 return writeback_single_inode(inode, wb, &wbc); 2319 } 2320 EXPORT_SYMBOL(write_inode_now); 2321 2322 /** 2323 * sync_inode - write an inode and its pages to disk. 2324 * @inode: the inode to sync 2325 * @wbc: controls the writeback mode 2326 * 2327 * sync_inode() will write an inode and its pages to disk. It will also 2328 * correctly update the inode on its superblock's dirty inode lists and will 2329 * update inode->i_state. 2330 * 2331 * The caller must have a ref on the inode. 2332 */ 2333 int sync_inode(struct inode *inode, struct writeback_control *wbc) 2334 { 2335 return writeback_single_inode(inode, &inode_to_bdi(inode)->wb, wbc); 2336 } 2337 EXPORT_SYMBOL(sync_inode); 2338 2339 /** 2340 * sync_inode_metadata - write an inode to disk 2341 * @inode: the inode to sync 2342 * @wait: wait for I/O to complete. 2343 * 2344 * Write an inode to disk and adjust its dirty state after completion. 2345 * 2346 * Note: only writes the actual inode, no associated data or other metadata. 2347 */ 2348 int sync_inode_metadata(struct inode *inode, int wait) 2349 { 2350 struct writeback_control wbc = { 2351 .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE, 2352 .nr_to_write = 0, /* metadata-only */ 2353 }; 2354 2355 return sync_inode(inode, &wbc); 2356 } 2357 EXPORT_SYMBOL(sync_inode_metadata); 2358