1 /* 2 * mm/page-writeback.c 3 * 4 * Copyright (C) 2002, Linus Torvalds. 5 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 6 * 7 * Contains functions related to writing back dirty pages at the 8 * address_space level. 9 * 10 * 10Apr2002 Andrew Morton 11 * Initial version 12 */ 13 14 #include <linux/kernel.h> 15 #include <linux/export.h> 16 #include <linux/spinlock.h> 17 #include <linux/fs.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 #include <linux/slab.h> 21 #include <linux/pagemap.h> 22 #include <linux/writeback.h> 23 #include <linux/init.h> 24 #include <linux/backing-dev.h> 25 #include <linux/task_io_accounting_ops.h> 26 #include <linux/blkdev.h> 27 #include <linux/mpage.h> 28 #include <linux/rmap.h> 29 #include <linux/percpu.h> 30 #include <linux/notifier.h> 31 #include <linux/smp.h> 32 #include <linux/sysctl.h> 33 #include <linux/cpu.h> 34 #include <linux/syscalls.h> 35 #include <linux/buffer_head.h> /* __set_page_dirty_buffers */ 36 #include <linux/pagevec.h> 37 #include <linux/timer.h> 38 #include <linux/sched/rt.h> 39 #include <linux/mm_inline.h> 40 #include <trace/events/writeback.h> 41 42 #include "internal.h" 43 44 /* 45 * Sleep at most 200ms at a time in balance_dirty_pages(). 46 */ 47 #define MAX_PAUSE max(HZ/5, 1) 48 49 /* 50 * Try to keep balance_dirty_pages() call intervals higher than this many pages 51 * by raising pause time to max_pause when falls below it. 52 */ 53 #define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10)) 54 55 /* 56 * Estimate write bandwidth at 200ms intervals. 57 */ 58 #define BANDWIDTH_INTERVAL max(HZ/5, 1) 59 60 #define RATELIMIT_CALC_SHIFT 10 61 62 /* 63 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited 64 * will look to see if it needs to force writeback or throttling. 65 */ 66 static long ratelimit_pages = 32; 67 68 /* The following parameters are exported via /proc/sys/vm */ 69 70 /* 71 * Start background writeback (via writeback threads) at this percentage 72 */ 73 int dirty_background_ratio = 10; 74 75 /* 76 * dirty_background_bytes starts at 0 (disabled) so that it is a function of 77 * dirty_background_ratio * the amount of dirtyable memory 78 */ 79 unsigned long dirty_background_bytes; 80 81 /* 82 * free highmem will not be subtracted from the total free memory 83 * for calculating free ratios if vm_highmem_is_dirtyable is true 84 */ 85 int vm_highmem_is_dirtyable; 86 87 /* 88 * The generator of dirty data starts writeback at this percentage 89 */ 90 int vm_dirty_ratio = 20; 91 92 /* 93 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of 94 * vm_dirty_ratio * the amount of dirtyable memory 95 */ 96 unsigned long vm_dirty_bytes; 97 98 /* 99 * The interval between `kupdate'-style writebacks 100 */ 101 unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ 102 103 EXPORT_SYMBOL_GPL(dirty_writeback_interval); 104 105 /* 106 * The longest time for which data is allowed to remain dirty 107 */ 108 unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */ 109 110 /* 111 * Flag that makes the machine dump writes/reads and block dirtyings. 112 */ 113 int block_dump; 114 115 /* 116 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies: 117 * a full sync is triggered after this time elapses without any disk activity. 118 */ 119 int laptop_mode; 120 121 EXPORT_SYMBOL(laptop_mode); 122 123 /* End of sysctl-exported parameters */ 124 125 struct wb_domain global_wb_domain; 126 127 /* consolidated parameters for balance_dirty_pages() and its subroutines */ 128 struct dirty_throttle_control { 129 #ifdef CONFIG_CGROUP_WRITEBACK 130 struct wb_domain *dom; 131 struct dirty_throttle_control *gdtc; /* only set in memcg dtc's */ 132 #endif 133 struct bdi_writeback *wb; 134 struct fprop_local_percpu *wb_completions; 135 136 unsigned long avail; /* dirtyable */ 137 unsigned long dirty; /* file_dirty + write + nfs */ 138 unsigned long thresh; /* dirty threshold */ 139 unsigned long bg_thresh; /* dirty background threshold */ 140 141 unsigned long wb_dirty; /* per-wb counterparts */ 142 unsigned long wb_thresh; 143 unsigned long wb_bg_thresh; 144 145 unsigned long pos_ratio; 146 }; 147 148 /* 149 * Length of period for aging writeout fractions of bdis. This is an 150 * arbitrarily chosen number. The longer the period, the slower fractions will 151 * reflect changes in current writeout rate. 152 */ 153 #define VM_COMPLETIONS_PERIOD_LEN (3*HZ) 154 155 #ifdef CONFIG_CGROUP_WRITEBACK 156 157 #define GDTC_INIT(__wb) .wb = (__wb), \ 158 .dom = &global_wb_domain, \ 159 .wb_completions = &(__wb)->completions 160 161 #define GDTC_INIT_NO_WB .dom = &global_wb_domain 162 163 #define MDTC_INIT(__wb, __gdtc) .wb = (__wb), \ 164 .dom = mem_cgroup_wb_domain(__wb), \ 165 .wb_completions = &(__wb)->memcg_completions, \ 166 .gdtc = __gdtc 167 168 static bool mdtc_valid(struct dirty_throttle_control *dtc) 169 { 170 return dtc->dom; 171 } 172 173 static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc) 174 { 175 return dtc->dom; 176 } 177 178 static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc) 179 { 180 return mdtc->gdtc; 181 } 182 183 static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb) 184 { 185 return &wb->memcg_completions; 186 } 187 188 static void wb_min_max_ratio(struct bdi_writeback *wb, 189 unsigned long *minp, unsigned long *maxp) 190 { 191 unsigned long this_bw = wb->avg_write_bandwidth; 192 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); 193 unsigned long long min = wb->bdi->min_ratio; 194 unsigned long long max = wb->bdi->max_ratio; 195 196 /* 197 * @wb may already be clean by the time control reaches here and 198 * the total may not include its bw. 199 */ 200 if (this_bw < tot_bw) { 201 if (min) { 202 min *= this_bw; 203 do_div(min, tot_bw); 204 } 205 if (max < 100) { 206 max *= this_bw; 207 do_div(max, tot_bw); 208 } 209 } 210 211 *minp = min; 212 *maxp = max; 213 } 214 215 #else /* CONFIG_CGROUP_WRITEBACK */ 216 217 #define GDTC_INIT(__wb) .wb = (__wb), \ 218 .wb_completions = &(__wb)->completions 219 #define GDTC_INIT_NO_WB 220 #define MDTC_INIT(__wb, __gdtc) 221 222 static bool mdtc_valid(struct dirty_throttle_control *dtc) 223 { 224 return false; 225 } 226 227 static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc) 228 { 229 return &global_wb_domain; 230 } 231 232 static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc) 233 { 234 return NULL; 235 } 236 237 static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb) 238 { 239 return NULL; 240 } 241 242 static void wb_min_max_ratio(struct bdi_writeback *wb, 243 unsigned long *minp, unsigned long *maxp) 244 { 245 *minp = wb->bdi->min_ratio; 246 *maxp = wb->bdi->max_ratio; 247 } 248 249 #endif /* CONFIG_CGROUP_WRITEBACK */ 250 251 /* 252 * In a memory zone, there is a certain amount of pages we consider 253 * available for the page cache, which is essentially the number of 254 * free and reclaimable pages, minus some zone reserves to protect 255 * lowmem and the ability to uphold the zone's watermarks without 256 * requiring writeback. 257 * 258 * This number of dirtyable pages is the base value of which the 259 * user-configurable dirty ratio is the effictive number of pages that 260 * are allowed to be actually dirtied. Per individual zone, or 261 * globally by using the sum of dirtyable pages over all zones. 262 * 263 * Because the user is allowed to specify the dirty limit globally as 264 * absolute number of bytes, calculating the per-zone dirty limit can 265 * require translating the configured limit into a percentage of 266 * global dirtyable memory first. 267 */ 268 269 /** 270 * zone_dirtyable_memory - number of dirtyable pages in a zone 271 * @zone: the zone 272 * 273 * Returns the zone's number of pages potentially available for dirty 274 * page cache. This is the base value for the per-zone dirty limits. 275 */ 276 static unsigned long zone_dirtyable_memory(struct zone *zone) 277 { 278 unsigned long nr_pages; 279 280 nr_pages = zone_page_state(zone, NR_FREE_PAGES); 281 /* 282 * Pages reserved for the kernel should not be considered 283 * dirtyable, to prevent a situation where reclaim has to 284 * clean pages in order to balance the zones. 285 */ 286 nr_pages -= min(nr_pages, zone->totalreserve_pages); 287 288 nr_pages += zone_page_state(zone, NR_INACTIVE_FILE); 289 nr_pages += zone_page_state(zone, NR_ACTIVE_FILE); 290 291 return nr_pages; 292 } 293 294 static unsigned long highmem_dirtyable_memory(unsigned long total) 295 { 296 #ifdef CONFIG_HIGHMEM 297 int node; 298 unsigned long x = 0; 299 int i; 300 301 for_each_node_state(node, N_HIGH_MEMORY) { 302 for (i = 0; i < MAX_NR_ZONES; i++) { 303 struct zone *z = &NODE_DATA(node)->node_zones[i]; 304 305 if (is_highmem(z)) 306 x += zone_dirtyable_memory(z); 307 } 308 } 309 /* 310 * Unreclaimable memory (kernel memory or anonymous memory 311 * without swap) can bring down the dirtyable pages below 312 * the zone's dirty balance reserve and the above calculation 313 * will underflow. However we still want to add in nodes 314 * which are below threshold (negative values) to get a more 315 * accurate calculation but make sure that the total never 316 * underflows. 317 */ 318 if ((long)x < 0) 319 x = 0; 320 321 /* 322 * Make sure that the number of highmem pages is never larger 323 * than the number of the total dirtyable memory. This can only 324 * occur in very strange VM situations but we want to make sure 325 * that this does not occur. 326 */ 327 return min(x, total); 328 #else 329 return 0; 330 #endif 331 } 332 333 /** 334 * global_dirtyable_memory - number of globally dirtyable pages 335 * 336 * Returns the global number of pages potentially available for dirty 337 * page cache. This is the base value for the global dirty limits. 338 */ 339 static unsigned long global_dirtyable_memory(void) 340 { 341 unsigned long x; 342 343 x = global_page_state(NR_FREE_PAGES); 344 /* 345 * Pages reserved for the kernel should not be considered 346 * dirtyable, to prevent a situation where reclaim has to 347 * clean pages in order to balance the zones. 348 */ 349 x -= min(x, totalreserve_pages); 350 351 x += global_page_state(NR_INACTIVE_FILE); 352 x += global_page_state(NR_ACTIVE_FILE); 353 354 if (!vm_highmem_is_dirtyable) 355 x -= highmem_dirtyable_memory(x); 356 357 return x + 1; /* Ensure that we never return 0 */ 358 } 359 360 /** 361 * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain 362 * @dtc: dirty_throttle_control of interest 363 * 364 * Calculate @dtc->thresh and ->bg_thresh considering 365 * vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}. The caller 366 * must ensure that @dtc->avail is set before calling this function. The 367 * dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and 368 * real-time tasks. 369 */ 370 static void domain_dirty_limits(struct dirty_throttle_control *dtc) 371 { 372 const unsigned long available_memory = dtc->avail; 373 struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc); 374 unsigned long bytes = vm_dirty_bytes; 375 unsigned long bg_bytes = dirty_background_bytes; 376 unsigned long ratio = vm_dirty_ratio; 377 unsigned long bg_ratio = dirty_background_ratio; 378 unsigned long thresh; 379 unsigned long bg_thresh; 380 struct task_struct *tsk; 381 382 /* gdtc is !NULL iff @dtc is for memcg domain */ 383 if (gdtc) { 384 unsigned long global_avail = gdtc->avail; 385 386 /* 387 * The byte settings can't be applied directly to memcg 388 * domains. Convert them to ratios by scaling against 389 * globally available memory. 390 */ 391 if (bytes) 392 ratio = min(DIV_ROUND_UP(bytes, PAGE_SIZE) * 100 / 393 global_avail, 100UL); 394 if (bg_bytes) 395 bg_ratio = min(DIV_ROUND_UP(bg_bytes, PAGE_SIZE) * 100 / 396 global_avail, 100UL); 397 bytes = bg_bytes = 0; 398 } 399 400 if (bytes) 401 thresh = DIV_ROUND_UP(bytes, PAGE_SIZE); 402 else 403 thresh = (ratio * available_memory) / 100; 404 405 if (bg_bytes) 406 bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE); 407 else 408 bg_thresh = (bg_ratio * available_memory) / 100; 409 410 if (bg_thresh >= thresh) 411 bg_thresh = thresh / 2; 412 tsk = current; 413 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) { 414 bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32; 415 thresh += thresh / 4 + global_wb_domain.dirty_limit / 32; 416 } 417 dtc->thresh = thresh; 418 dtc->bg_thresh = bg_thresh; 419 420 /* we should eventually report the domain in the TP */ 421 if (!gdtc) 422 trace_global_dirty_state(bg_thresh, thresh); 423 } 424 425 /** 426 * global_dirty_limits - background-writeback and dirty-throttling thresholds 427 * @pbackground: out parameter for bg_thresh 428 * @pdirty: out parameter for thresh 429 * 430 * Calculate bg_thresh and thresh for global_wb_domain. See 431 * domain_dirty_limits() for details. 432 */ 433 void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty) 434 { 435 struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB }; 436 437 gdtc.avail = global_dirtyable_memory(); 438 domain_dirty_limits(&gdtc); 439 440 *pbackground = gdtc.bg_thresh; 441 *pdirty = gdtc.thresh; 442 } 443 444 /** 445 * zone_dirty_limit - maximum number of dirty pages allowed in a zone 446 * @zone: the zone 447 * 448 * Returns the maximum number of dirty pages allowed in a zone, based 449 * on the zone's dirtyable memory. 450 */ 451 static unsigned long zone_dirty_limit(struct zone *zone) 452 { 453 unsigned long zone_memory = zone_dirtyable_memory(zone); 454 struct task_struct *tsk = current; 455 unsigned long dirty; 456 457 if (vm_dirty_bytes) 458 dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) * 459 zone_memory / global_dirtyable_memory(); 460 else 461 dirty = vm_dirty_ratio * zone_memory / 100; 462 463 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) 464 dirty += dirty / 4; 465 466 return dirty; 467 } 468 469 /** 470 * zone_dirty_ok - tells whether a zone is within its dirty limits 471 * @zone: the zone to check 472 * 473 * Returns %true when the dirty pages in @zone are within the zone's 474 * dirty limit, %false if the limit is exceeded. 475 */ 476 bool zone_dirty_ok(struct zone *zone) 477 { 478 unsigned long limit = zone_dirty_limit(zone); 479 480 return zone_page_state(zone, NR_FILE_DIRTY) + 481 zone_page_state(zone, NR_UNSTABLE_NFS) + 482 zone_page_state(zone, NR_WRITEBACK) <= limit; 483 } 484 485 int dirty_background_ratio_handler(struct ctl_table *table, int write, 486 void __user *buffer, size_t *lenp, 487 loff_t *ppos) 488 { 489 int ret; 490 491 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 492 if (ret == 0 && write) 493 dirty_background_bytes = 0; 494 return ret; 495 } 496 497 int dirty_background_bytes_handler(struct ctl_table *table, int write, 498 void __user *buffer, size_t *lenp, 499 loff_t *ppos) 500 { 501 int ret; 502 503 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 504 if (ret == 0 && write) 505 dirty_background_ratio = 0; 506 return ret; 507 } 508 509 int dirty_ratio_handler(struct ctl_table *table, int write, 510 void __user *buffer, size_t *lenp, 511 loff_t *ppos) 512 { 513 int old_ratio = vm_dirty_ratio; 514 int ret; 515 516 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 517 if (ret == 0 && write && vm_dirty_ratio != old_ratio) { 518 writeback_set_ratelimit(); 519 vm_dirty_bytes = 0; 520 } 521 return ret; 522 } 523 524 int dirty_bytes_handler(struct ctl_table *table, int write, 525 void __user *buffer, size_t *lenp, 526 loff_t *ppos) 527 { 528 unsigned long old_bytes = vm_dirty_bytes; 529 int ret; 530 531 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 532 if (ret == 0 && write && vm_dirty_bytes != old_bytes) { 533 writeback_set_ratelimit(); 534 vm_dirty_ratio = 0; 535 } 536 return ret; 537 } 538 539 static unsigned long wp_next_time(unsigned long cur_time) 540 { 541 cur_time += VM_COMPLETIONS_PERIOD_LEN; 542 /* 0 has a special meaning... */ 543 if (!cur_time) 544 return 1; 545 return cur_time; 546 } 547 548 static void wb_domain_writeout_inc(struct wb_domain *dom, 549 struct fprop_local_percpu *completions, 550 unsigned int max_prop_frac) 551 { 552 __fprop_inc_percpu_max(&dom->completions, completions, 553 max_prop_frac); 554 /* First event after period switching was turned off? */ 555 if (!unlikely(dom->period_time)) { 556 /* 557 * We can race with other __bdi_writeout_inc calls here but 558 * it does not cause any harm since the resulting time when 559 * timer will fire and what is in writeout_period_time will be 560 * roughly the same. 561 */ 562 dom->period_time = wp_next_time(jiffies); 563 mod_timer(&dom->period_timer, dom->period_time); 564 } 565 } 566 567 /* 568 * Increment @wb's writeout completion count and the global writeout 569 * completion count. Called from test_clear_page_writeback(). 570 */ 571 static inline void __wb_writeout_inc(struct bdi_writeback *wb) 572 { 573 struct wb_domain *cgdom; 574 575 __inc_wb_stat(wb, WB_WRITTEN); 576 wb_domain_writeout_inc(&global_wb_domain, &wb->completions, 577 wb->bdi->max_prop_frac); 578 579 cgdom = mem_cgroup_wb_domain(wb); 580 if (cgdom) 581 wb_domain_writeout_inc(cgdom, wb_memcg_completions(wb), 582 wb->bdi->max_prop_frac); 583 } 584 585 void wb_writeout_inc(struct bdi_writeback *wb) 586 { 587 unsigned long flags; 588 589 local_irq_save(flags); 590 __wb_writeout_inc(wb); 591 local_irq_restore(flags); 592 } 593 EXPORT_SYMBOL_GPL(wb_writeout_inc); 594 595 /* 596 * On idle system, we can be called long after we scheduled because we use 597 * deferred timers so count with missed periods. 598 */ 599 static void writeout_period(unsigned long t) 600 { 601 struct wb_domain *dom = (void *)t; 602 int miss_periods = (jiffies - dom->period_time) / 603 VM_COMPLETIONS_PERIOD_LEN; 604 605 if (fprop_new_period(&dom->completions, miss_periods + 1)) { 606 dom->period_time = wp_next_time(dom->period_time + 607 miss_periods * VM_COMPLETIONS_PERIOD_LEN); 608 mod_timer(&dom->period_timer, dom->period_time); 609 } else { 610 /* 611 * Aging has zeroed all fractions. Stop wasting CPU on period 612 * updates. 613 */ 614 dom->period_time = 0; 615 } 616 } 617 618 int wb_domain_init(struct wb_domain *dom, gfp_t gfp) 619 { 620 memset(dom, 0, sizeof(*dom)); 621 622 spin_lock_init(&dom->lock); 623 624 init_timer_deferrable(&dom->period_timer); 625 dom->period_timer.function = writeout_period; 626 dom->period_timer.data = (unsigned long)dom; 627 628 dom->dirty_limit_tstamp = jiffies; 629 630 return fprop_global_init(&dom->completions, gfp); 631 } 632 633 #ifdef CONFIG_CGROUP_WRITEBACK 634 void wb_domain_exit(struct wb_domain *dom) 635 { 636 del_timer_sync(&dom->period_timer); 637 fprop_global_destroy(&dom->completions); 638 } 639 #endif 640 641 /* 642 * bdi_min_ratio keeps the sum of the minimum dirty shares of all 643 * registered backing devices, which, for obvious reasons, can not 644 * exceed 100%. 645 */ 646 static unsigned int bdi_min_ratio; 647 648 int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio) 649 { 650 int ret = 0; 651 652 spin_lock_bh(&bdi_lock); 653 if (min_ratio > bdi->max_ratio) { 654 ret = -EINVAL; 655 } else { 656 min_ratio -= bdi->min_ratio; 657 if (bdi_min_ratio + min_ratio < 100) { 658 bdi_min_ratio += min_ratio; 659 bdi->min_ratio += min_ratio; 660 } else { 661 ret = -EINVAL; 662 } 663 } 664 spin_unlock_bh(&bdi_lock); 665 666 return ret; 667 } 668 669 int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio) 670 { 671 int ret = 0; 672 673 if (max_ratio > 100) 674 return -EINVAL; 675 676 spin_lock_bh(&bdi_lock); 677 if (bdi->min_ratio > max_ratio) { 678 ret = -EINVAL; 679 } else { 680 bdi->max_ratio = max_ratio; 681 bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100; 682 } 683 spin_unlock_bh(&bdi_lock); 684 685 return ret; 686 } 687 EXPORT_SYMBOL(bdi_set_max_ratio); 688 689 static unsigned long dirty_freerun_ceiling(unsigned long thresh, 690 unsigned long bg_thresh) 691 { 692 return (thresh + bg_thresh) / 2; 693 } 694 695 static unsigned long hard_dirty_limit(struct wb_domain *dom, 696 unsigned long thresh) 697 { 698 return max(thresh, dom->dirty_limit); 699 } 700 701 /* 702 * Memory which can be further allocated to a memcg domain is capped by 703 * system-wide clean memory excluding the amount being used in the domain. 704 */ 705 static void mdtc_calc_avail(struct dirty_throttle_control *mdtc, 706 unsigned long filepages, unsigned long headroom) 707 { 708 struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc); 709 unsigned long clean = filepages - min(filepages, mdtc->dirty); 710 unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty); 711 unsigned long other_clean = global_clean - min(global_clean, clean); 712 713 mdtc->avail = filepages + min(headroom, other_clean); 714 } 715 716 /** 717 * __wb_calc_thresh - @wb's share of dirty throttling threshold 718 * @dtc: dirty_throttle_context of interest 719 * 720 * Returns @wb's dirty limit in pages. The term "dirty" in the context of 721 * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages. 722 * 723 * Note that balance_dirty_pages() will only seriously take it as a hard limit 724 * when sleeping max_pause per page is not enough to keep the dirty pages under 725 * control. For example, when the device is completely stalled due to some error 726 * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key. 727 * In the other normal situations, it acts more gently by throttling the tasks 728 * more (rather than completely block them) when the wb dirty pages go high. 729 * 730 * It allocates high/low dirty limits to fast/slow devices, in order to prevent 731 * - starving fast devices 732 * - piling up dirty pages (that will take long time to sync) on slow devices 733 * 734 * The wb's share of dirty limit will be adapting to its throughput and 735 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set. 736 */ 737 static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc) 738 { 739 struct wb_domain *dom = dtc_dom(dtc); 740 unsigned long thresh = dtc->thresh; 741 u64 wb_thresh; 742 long numerator, denominator; 743 unsigned long wb_min_ratio, wb_max_ratio; 744 745 /* 746 * Calculate this BDI's share of the thresh ratio. 747 */ 748 fprop_fraction_percpu(&dom->completions, dtc->wb_completions, 749 &numerator, &denominator); 750 751 wb_thresh = (thresh * (100 - bdi_min_ratio)) / 100; 752 wb_thresh *= numerator; 753 do_div(wb_thresh, denominator); 754 755 wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio); 756 757 wb_thresh += (thresh * wb_min_ratio) / 100; 758 if (wb_thresh > (thresh * wb_max_ratio) / 100) 759 wb_thresh = thresh * wb_max_ratio / 100; 760 761 return wb_thresh; 762 } 763 764 unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh) 765 { 766 struct dirty_throttle_control gdtc = { GDTC_INIT(wb), 767 .thresh = thresh }; 768 return __wb_calc_thresh(&gdtc); 769 } 770 771 /* 772 * setpoint - dirty 3 773 * f(dirty) := 1.0 + (----------------) 774 * limit - setpoint 775 * 776 * it's a 3rd order polynomial that subjects to 777 * 778 * (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast 779 * (2) f(setpoint) = 1.0 => the balance point 780 * (3) f(limit) = 0 => the hard limit 781 * (4) df/dx <= 0 => negative feedback control 782 * (5) the closer to setpoint, the smaller |df/dx| (and the reverse) 783 * => fast response on large errors; small oscillation near setpoint 784 */ 785 static long long pos_ratio_polynom(unsigned long setpoint, 786 unsigned long dirty, 787 unsigned long limit) 788 { 789 long long pos_ratio; 790 long x; 791 792 x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT, 793 (limit - setpoint) | 1); 794 pos_ratio = x; 795 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; 796 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; 797 pos_ratio += 1 << RATELIMIT_CALC_SHIFT; 798 799 return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT); 800 } 801 802 /* 803 * Dirty position control. 804 * 805 * (o) global/bdi setpoints 806 * 807 * We want the dirty pages be balanced around the global/wb setpoints. 808 * When the number of dirty pages is higher/lower than the setpoint, the 809 * dirty position control ratio (and hence task dirty ratelimit) will be 810 * decreased/increased to bring the dirty pages back to the setpoint. 811 * 812 * pos_ratio = 1 << RATELIMIT_CALC_SHIFT 813 * 814 * if (dirty < setpoint) scale up pos_ratio 815 * if (dirty > setpoint) scale down pos_ratio 816 * 817 * if (wb_dirty < wb_setpoint) scale up pos_ratio 818 * if (wb_dirty > wb_setpoint) scale down pos_ratio 819 * 820 * task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT 821 * 822 * (o) global control line 823 * 824 * ^ pos_ratio 825 * | 826 * | |<===== global dirty control scope ======>| 827 * 2.0 .............* 828 * | .* 829 * | . * 830 * | . * 831 * | . * 832 * | . * 833 * | . * 834 * 1.0 ................................* 835 * | . . * 836 * | . . * 837 * | . . * 838 * | . . * 839 * | . . * 840 * 0 +------------.------------------.----------------------*-------------> 841 * freerun^ setpoint^ limit^ dirty pages 842 * 843 * (o) wb control line 844 * 845 * ^ pos_ratio 846 * | 847 * | * 848 * | * 849 * | * 850 * | * 851 * | * |<=========== span ============>| 852 * 1.0 .......................* 853 * | . * 854 * | . * 855 * | . * 856 * | . * 857 * | . * 858 * | . * 859 * | . * 860 * | . * 861 * | . * 862 * | . * 863 * | . * 864 * 1/4 ...............................................* * * * * * * * * * * * 865 * | . . 866 * | . . 867 * | . . 868 * 0 +----------------------.-------------------------------.-------------> 869 * wb_setpoint^ x_intercept^ 870 * 871 * The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can 872 * be smoothly throttled down to normal if it starts high in situations like 873 * - start writing to a slow SD card and a fast disk at the same time. The SD 874 * card's wb_dirty may rush to many times higher than wb_setpoint. 875 * - the wb dirty thresh drops quickly due to change of JBOD workload 876 */ 877 static void wb_position_ratio(struct dirty_throttle_control *dtc) 878 { 879 struct bdi_writeback *wb = dtc->wb; 880 unsigned long write_bw = wb->avg_write_bandwidth; 881 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); 882 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); 883 unsigned long wb_thresh = dtc->wb_thresh; 884 unsigned long x_intercept; 885 unsigned long setpoint; /* dirty pages' target balance point */ 886 unsigned long wb_setpoint; 887 unsigned long span; 888 long long pos_ratio; /* for scaling up/down the rate limit */ 889 long x; 890 891 dtc->pos_ratio = 0; 892 893 if (unlikely(dtc->dirty >= limit)) 894 return; 895 896 /* 897 * global setpoint 898 * 899 * See comment for pos_ratio_polynom(). 900 */ 901 setpoint = (freerun + limit) / 2; 902 pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit); 903 904 /* 905 * The strictlimit feature is a tool preventing mistrusted filesystems 906 * from growing a large number of dirty pages before throttling. For 907 * such filesystems balance_dirty_pages always checks wb counters 908 * against wb limits. Even if global "nr_dirty" is under "freerun". 909 * This is especially important for fuse which sets bdi->max_ratio to 910 * 1% by default. Without strictlimit feature, fuse writeback may 911 * consume arbitrary amount of RAM because it is accounted in 912 * NR_WRITEBACK_TEMP which is not involved in calculating "nr_dirty". 913 * 914 * Here, in wb_position_ratio(), we calculate pos_ratio based on 915 * two values: wb_dirty and wb_thresh. Let's consider an example: 916 * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global 917 * limits are set by default to 10% and 20% (background and throttle). 918 * Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages. 919 * wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is 920 * about ~6K pages (as the average of background and throttle wb 921 * limits). The 3rd order polynomial will provide positive feedback if 922 * wb_dirty is under wb_setpoint and vice versa. 923 * 924 * Note, that we cannot use global counters in these calculations 925 * because we want to throttle process writing to a strictlimit wb 926 * much earlier than global "freerun" is reached (~23MB vs. ~2.3GB 927 * in the example above). 928 */ 929 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { 930 long long wb_pos_ratio; 931 932 if (dtc->wb_dirty < 8) { 933 dtc->pos_ratio = min_t(long long, pos_ratio * 2, 934 2 << RATELIMIT_CALC_SHIFT); 935 return; 936 } 937 938 if (dtc->wb_dirty >= wb_thresh) 939 return; 940 941 wb_setpoint = dirty_freerun_ceiling(wb_thresh, 942 dtc->wb_bg_thresh); 943 944 if (wb_setpoint == 0 || wb_setpoint == wb_thresh) 945 return; 946 947 wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty, 948 wb_thresh); 949 950 /* 951 * Typically, for strictlimit case, wb_setpoint << setpoint 952 * and pos_ratio >> wb_pos_ratio. In the other words global 953 * state ("dirty") is not limiting factor and we have to 954 * make decision based on wb counters. But there is an 955 * important case when global pos_ratio should get precedence: 956 * global limits are exceeded (e.g. due to activities on other 957 * wb's) while given strictlimit wb is below limit. 958 * 959 * "pos_ratio * wb_pos_ratio" would work for the case above, 960 * but it would look too non-natural for the case of all 961 * activity in the system coming from a single strictlimit wb 962 * with bdi->max_ratio == 100%. 963 * 964 * Note that min() below somewhat changes the dynamics of the 965 * control system. Normally, pos_ratio value can be well over 3 966 * (when globally we are at freerun and wb is well below wb 967 * setpoint). Now the maximum pos_ratio in the same situation 968 * is 2. We might want to tweak this if we observe the control 969 * system is too slow to adapt. 970 */ 971 dtc->pos_ratio = min(pos_ratio, wb_pos_ratio); 972 return; 973 } 974 975 /* 976 * We have computed basic pos_ratio above based on global situation. If 977 * the wb is over/under its share of dirty pages, we want to scale 978 * pos_ratio further down/up. That is done by the following mechanism. 979 */ 980 981 /* 982 * wb setpoint 983 * 984 * f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint) 985 * 986 * x_intercept - wb_dirty 987 * := -------------------------- 988 * x_intercept - wb_setpoint 989 * 990 * The main wb control line is a linear function that subjects to 991 * 992 * (1) f(wb_setpoint) = 1.0 993 * (2) k = - 1 / (8 * write_bw) (in single wb case) 994 * or equally: x_intercept = wb_setpoint + 8 * write_bw 995 * 996 * For single wb case, the dirty pages are observed to fluctuate 997 * regularly within range 998 * [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2] 999 * for various filesystems, where (2) can yield in a reasonable 12.5% 1000 * fluctuation range for pos_ratio. 1001 * 1002 * For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its 1003 * own size, so move the slope over accordingly and choose a slope that 1004 * yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh. 1005 */ 1006 if (unlikely(wb_thresh > dtc->thresh)) 1007 wb_thresh = dtc->thresh; 1008 /* 1009 * It's very possible that wb_thresh is close to 0 not because the 1010 * device is slow, but that it has remained inactive for long time. 1011 * Honour such devices a reasonable good (hopefully IO efficient) 1012 * threshold, so that the occasional writes won't be blocked and active 1013 * writes can rampup the threshold quickly. 1014 */ 1015 wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8); 1016 /* 1017 * scale global setpoint to wb's: 1018 * wb_setpoint = setpoint * wb_thresh / thresh 1019 */ 1020 x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1); 1021 wb_setpoint = setpoint * (u64)x >> 16; 1022 /* 1023 * Use span=(8*write_bw) in single wb case as indicated by 1024 * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case. 1025 * 1026 * wb_thresh thresh - wb_thresh 1027 * span = --------- * (8 * write_bw) + ------------------ * wb_thresh 1028 * thresh thresh 1029 */ 1030 span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16; 1031 x_intercept = wb_setpoint + span; 1032 1033 if (dtc->wb_dirty < x_intercept - span / 4) { 1034 pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty), 1035 (x_intercept - wb_setpoint) | 1); 1036 } else 1037 pos_ratio /= 4; 1038 1039 /* 1040 * wb reserve area, safeguard against dirty pool underrun and disk idle 1041 * It may push the desired control point of global dirty pages higher 1042 * than setpoint. 1043 */ 1044 x_intercept = wb_thresh / 2; 1045 if (dtc->wb_dirty < x_intercept) { 1046 if (dtc->wb_dirty > x_intercept / 8) 1047 pos_ratio = div_u64(pos_ratio * x_intercept, 1048 dtc->wb_dirty); 1049 else 1050 pos_ratio *= 8; 1051 } 1052 1053 dtc->pos_ratio = pos_ratio; 1054 } 1055 1056 static void wb_update_write_bandwidth(struct bdi_writeback *wb, 1057 unsigned long elapsed, 1058 unsigned long written) 1059 { 1060 const unsigned long period = roundup_pow_of_two(3 * HZ); 1061 unsigned long avg = wb->avg_write_bandwidth; 1062 unsigned long old = wb->write_bandwidth; 1063 u64 bw; 1064 1065 /* 1066 * bw = written * HZ / elapsed 1067 * 1068 * bw * elapsed + write_bandwidth * (period - elapsed) 1069 * write_bandwidth = --------------------------------------------------- 1070 * period 1071 * 1072 * @written may have decreased due to account_page_redirty(). 1073 * Avoid underflowing @bw calculation. 1074 */ 1075 bw = written - min(written, wb->written_stamp); 1076 bw *= HZ; 1077 if (unlikely(elapsed > period)) { 1078 do_div(bw, elapsed); 1079 avg = bw; 1080 goto out; 1081 } 1082 bw += (u64)wb->write_bandwidth * (period - elapsed); 1083 bw >>= ilog2(period); 1084 1085 /* 1086 * one more level of smoothing, for filtering out sudden spikes 1087 */ 1088 if (avg > old && old >= (unsigned long)bw) 1089 avg -= (avg - old) >> 3; 1090 1091 if (avg < old && old <= (unsigned long)bw) 1092 avg += (old - avg) >> 3; 1093 1094 out: 1095 /* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */ 1096 avg = max(avg, 1LU); 1097 if (wb_has_dirty_io(wb)) { 1098 long delta = avg - wb->avg_write_bandwidth; 1099 WARN_ON_ONCE(atomic_long_add_return(delta, 1100 &wb->bdi->tot_write_bandwidth) <= 0); 1101 } 1102 wb->write_bandwidth = bw; 1103 wb->avg_write_bandwidth = avg; 1104 } 1105 1106 static void update_dirty_limit(struct dirty_throttle_control *dtc) 1107 { 1108 struct wb_domain *dom = dtc_dom(dtc); 1109 unsigned long thresh = dtc->thresh; 1110 unsigned long limit = dom->dirty_limit; 1111 1112 /* 1113 * Follow up in one step. 1114 */ 1115 if (limit < thresh) { 1116 limit = thresh; 1117 goto update; 1118 } 1119 1120 /* 1121 * Follow down slowly. Use the higher one as the target, because thresh 1122 * may drop below dirty. This is exactly the reason to introduce 1123 * dom->dirty_limit which is guaranteed to lie above the dirty pages. 1124 */ 1125 thresh = max(thresh, dtc->dirty); 1126 if (limit > thresh) { 1127 limit -= (limit - thresh) >> 5; 1128 goto update; 1129 } 1130 return; 1131 update: 1132 dom->dirty_limit = limit; 1133 } 1134 1135 static void domain_update_bandwidth(struct dirty_throttle_control *dtc, 1136 unsigned long now) 1137 { 1138 struct wb_domain *dom = dtc_dom(dtc); 1139 1140 /* 1141 * check locklessly first to optimize away locking for the most time 1142 */ 1143 if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) 1144 return; 1145 1146 spin_lock(&dom->lock); 1147 if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) { 1148 update_dirty_limit(dtc); 1149 dom->dirty_limit_tstamp = now; 1150 } 1151 spin_unlock(&dom->lock); 1152 } 1153 1154 /* 1155 * Maintain wb->dirty_ratelimit, the base dirty throttle rate. 1156 * 1157 * Normal wb tasks will be curbed at or below it in long term. 1158 * Obviously it should be around (write_bw / N) when there are N dd tasks. 1159 */ 1160 static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc, 1161 unsigned long dirtied, 1162 unsigned long elapsed) 1163 { 1164 struct bdi_writeback *wb = dtc->wb; 1165 unsigned long dirty = dtc->dirty; 1166 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); 1167 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); 1168 unsigned long setpoint = (freerun + limit) / 2; 1169 unsigned long write_bw = wb->avg_write_bandwidth; 1170 unsigned long dirty_ratelimit = wb->dirty_ratelimit; 1171 unsigned long dirty_rate; 1172 unsigned long task_ratelimit; 1173 unsigned long balanced_dirty_ratelimit; 1174 unsigned long step; 1175 unsigned long x; 1176 unsigned long shift; 1177 1178 /* 1179 * The dirty rate will match the writeout rate in long term, except 1180 * when dirty pages are truncated by userspace or re-dirtied by FS. 1181 */ 1182 dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed; 1183 1184 /* 1185 * task_ratelimit reflects each dd's dirty rate for the past 200ms. 1186 */ 1187 task_ratelimit = (u64)dirty_ratelimit * 1188 dtc->pos_ratio >> RATELIMIT_CALC_SHIFT; 1189 task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */ 1190 1191 /* 1192 * A linear estimation of the "balanced" throttle rate. The theory is, 1193 * if there are N dd tasks, each throttled at task_ratelimit, the wb's 1194 * dirty_rate will be measured to be (N * task_ratelimit). So the below 1195 * formula will yield the balanced rate limit (write_bw / N). 1196 * 1197 * Note that the expanded form is not a pure rate feedback: 1198 * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1) 1199 * but also takes pos_ratio into account: 1200 * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2) 1201 * 1202 * (1) is not realistic because pos_ratio also takes part in balancing 1203 * the dirty rate. Consider the state 1204 * pos_ratio = 0.5 (3) 1205 * rate = 2 * (write_bw / N) (4) 1206 * If (1) is used, it will stuck in that state! Because each dd will 1207 * be throttled at 1208 * task_ratelimit = pos_ratio * rate = (write_bw / N) (5) 1209 * yielding 1210 * dirty_rate = N * task_ratelimit = write_bw (6) 1211 * put (6) into (1) we get 1212 * rate_(i+1) = rate_(i) (7) 1213 * 1214 * So we end up using (2) to always keep 1215 * rate_(i+1) ~= (write_bw / N) (8) 1216 * regardless of the value of pos_ratio. As long as (8) is satisfied, 1217 * pos_ratio is able to drive itself to 1.0, which is not only where 1218 * the dirty count meet the setpoint, but also where the slope of 1219 * pos_ratio is most flat and hence task_ratelimit is least fluctuated. 1220 */ 1221 balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw, 1222 dirty_rate | 1); 1223 /* 1224 * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw 1225 */ 1226 if (unlikely(balanced_dirty_ratelimit > write_bw)) 1227 balanced_dirty_ratelimit = write_bw; 1228 1229 /* 1230 * We could safely do this and return immediately: 1231 * 1232 * wb->dirty_ratelimit = balanced_dirty_ratelimit; 1233 * 1234 * However to get a more stable dirty_ratelimit, the below elaborated 1235 * code makes use of task_ratelimit to filter out singular points and 1236 * limit the step size. 1237 * 1238 * The below code essentially only uses the relative value of 1239 * 1240 * task_ratelimit - dirty_ratelimit 1241 * = (pos_ratio - 1) * dirty_ratelimit 1242 * 1243 * which reflects the direction and size of dirty position error. 1244 */ 1245 1246 /* 1247 * dirty_ratelimit will follow balanced_dirty_ratelimit iff 1248 * task_ratelimit is on the same side of dirty_ratelimit, too. 1249 * For example, when 1250 * - dirty_ratelimit > balanced_dirty_ratelimit 1251 * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint) 1252 * lowering dirty_ratelimit will help meet both the position and rate 1253 * control targets. Otherwise, don't update dirty_ratelimit if it will 1254 * only help meet the rate target. After all, what the users ultimately 1255 * feel and care are stable dirty rate and small position error. 1256 * 1257 * |task_ratelimit - dirty_ratelimit| is used to limit the step size 1258 * and filter out the singular points of balanced_dirty_ratelimit. Which 1259 * keeps jumping around randomly and can even leap far away at times 1260 * due to the small 200ms estimation period of dirty_rate (we want to 1261 * keep that period small to reduce time lags). 1262 */ 1263 step = 0; 1264 1265 /* 1266 * For strictlimit case, calculations above were based on wb counters 1267 * and limits (starting from pos_ratio = wb_position_ratio() and up to 1268 * balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate). 1269 * Hence, to calculate "step" properly, we have to use wb_dirty as 1270 * "dirty" and wb_setpoint as "setpoint". 1271 * 1272 * We rampup dirty_ratelimit forcibly if wb_dirty is low because 1273 * it's possible that wb_thresh is close to zero due to inactivity 1274 * of backing device. 1275 */ 1276 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { 1277 dirty = dtc->wb_dirty; 1278 if (dtc->wb_dirty < 8) 1279 setpoint = dtc->wb_dirty + 1; 1280 else 1281 setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2; 1282 } 1283 1284 if (dirty < setpoint) { 1285 x = min3(wb->balanced_dirty_ratelimit, 1286 balanced_dirty_ratelimit, task_ratelimit); 1287 if (dirty_ratelimit < x) 1288 step = x - dirty_ratelimit; 1289 } else { 1290 x = max3(wb->balanced_dirty_ratelimit, 1291 balanced_dirty_ratelimit, task_ratelimit); 1292 if (dirty_ratelimit > x) 1293 step = dirty_ratelimit - x; 1294 } 1295 1296 /* 1297 * Don't pursue 100% rate matching. It's impossible since the balanced 1298 * rate itself is constantly fluctuating. So decrease the track speed 1299 * when it gets close to the target. Helps eliminate pointless tremors. 1300 */ 1301 shift = dirty_ratelimit / (2 * step + 1); 1302 if (shift < BITS_PER_LONG) 1303 step = DIV_ROUND_UP(step >> shift, 8); 1304 else 1305 step = 0; 1306 1307 if (dirty_ratelimit < balanced_dirty_ratelimit) 1308 dirty_ratelimit += step; 1309 else 1310 dirty_ratelimit -= step; 1311 1312 wb->dirty_ratelimit = max(dirty_ratelimit, 1UL); 1313 wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit; 1314 1315 trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit); 1316 } 1317 1318 static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc, 1319 struct dirty_throttle_control *mdtc, 1320 unsigned long start_time, 1321 bool update_ratelimit) 1322 { 1323 struct bdi_writeback *wb = gdtc->wb; 1324 unsigned long now = jiffies; 1325 unsigned long elapsed = now - wb->bw_time_stamp; 1326 unsigned long dirtied; 1327 unsigned long written; 1328 1329 lockdep_assert_held(&wb->list_lock); 1330 1331 /* 1332 * rate-limit, only update once every 200ms. 1333 */ 1334 if (elapsed < BANDWIDTH_INTERVAL) 1335 return; 1336 1337 dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]); 1338 written = percpu_counter_read(&wb->stat[WB_WRITTEN]); 1339 1340 /* 1341 * Skip quiet periods when disk bandwidth is under-utilized. 1342 * (at least 1s idle time between two flusher runs) 1343 */ 1344 if (elapsed > HZ && time_before(wb->bw_time_stamp, start_time)) 1345 goto snapshot; 1346 1347 if (update_ratelimit) { 1348 domain_update_bandwidth(gdtc, now); 1349 wb_update_dirty_ratelimit(gdtc, dirtied, elapsed); 1350 1351 /* 1352 * @mdtc is always NULL if !CGROUP_WRITEBACK but the 1353 * compiler has no way to figure that out. Help it. 1354 */ 1355 if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) { 1356 domain_update_bandwidth(mdtc, now); 1357 wb_update_dirty_ratelimit(mdtc, dirtied, elapsed); 1358 } 1359 } 1360 wb_update_write_bandwidth(wb, elapsed, written); 1361 1362 snapshot: 1363 wb->dirtied_stamp = dirtied; 1364 wb->written_stamp = written; 1365 wb->bw_time_stamp = now; 1366 } 1367 1368 void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time) 1369 { 1370 struct dirty_throttle_control gdtc = { GDTC_INIT(wb) }; 1371 1372 __wb_update_bandwidth(&gdtc, NULL, start_time, false); 1373 } 1374 1375 /* 1376 * After a task dirtied this many pages, balance_dirty_pages_ratelimited() 1377 * will look to see if it needs to start dirty throttling. 1378 * 1379 * If dirty_poll_interval is too low, big NUMA machines will call the expensive 1380 * global_page_state() too often. So scale it near-sqrt to the safety margin 1381 * (the number of pages we may dirty without exceeding the dirty limits). 1382 */ 1383 static unsigned long dirty_poll_interval(unsigned long dirty, 1384 unsigned long thresh) 1385 { 1386 if (thresh > dirty) 1387 return 1UL << (ilog2(thresh - dirty) >> 1); 1388 1389 return 1; 1390 } 1391 1392 static unsigned long wb_max_pause(struct bdi_writeback *wb, 1393 unsigned long wb_dirty) 1394 { 1395 unsigned long bw = wb->avg_write_bandwidth; 1396 unsigned long t; 1397 1398 /* 1399 * Limit pause time for small memory systems. If sleeping for too long 1400 * time, a small pool of dirty/writeback pages may go empty and disk go 1401 * idle. 1402 * 1403 * 8 serves as the safety ratio. 1404 */ 1405 t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8)); 1406 t++; 1407 1408 return min_t(unsigned long, t, MAX_PAUSE); 1409 } 1410 1411 static long wb_min_pause(struct bdi_writeback *wb, 1412 long max_pause, 1413 unsigned long task_ratelimit, 1414 unsigned long dirty_ratelimit, 1415 int *nr_dirtied_pause) 1416 { 1417 long hi = ilog2(wb->avg_write_bandwidth); 1418 long lo = ilog2(wb->dirty_ratelimit); 1419 long t; /* target pause */ 1420 long pause; /* estimated next pause */ 1421 int pages; /* target nr_dirtied_pause */ 1422 1423 /* target for 10ms pause on 1-dd case */ 1424 t = max(1, HZ / 100); 1425 1426 /* 1427 * Scale up pause time for concurrent dirtiers in order to reduce CPU 1428 * overheads. 1429 * 1430 * (N * 10ms) on 2^N concurrent tasks. 1431 */ 1432 if (hi > lo) 1433 t += (hi - lo) * (10 * HZ) / 1024; 1434 1435 /* 1436 * This is a bit convoluted. We try to base the next nr_dirtied_pause 1437 * on the much more stable dirty_ratelimit. However the next pause time 1438 * will be computed based on task_ratelimit and the two rate limits may 1439 * depart considerably at some time. Especially if task_ratelimit goes 1440 * below dirty_ratelimit/2 and the target pause is max_pause, the next 1441 * pause time will be max_pause*2 _trimmed down_ to max_pause. As a 1442 * result task_ratelimit won't be executed faithfully, which could 1443 * eventually bring down dirty_ratelimit. 1444 * 1445 * We apply two rules to fix it up: 1446 * 1) try to estimate the next pause time and if necessary, use a lower 1447 * nr_dirtied_pause so as not to exceed max_pause. When this happens, 1448 * nr_dirtied_pause will be "dancing" with task_ratelimit. 1449 * 2) limit the target pause time to max_pause/2, so that the normal 1450 * small fluctuations of task_ratelimit won't trigger rule (1) and 1451 * nr_dirtied_pause will remain as stable as dirty_ratelimit. 1452 */ 1453 t = min(t, 1 + max_pause / 2); 1454 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ); 1455 1456 /* 1457 * Tiny nr_dirtied_pause is found to hurt I/O performance in the test 1458 * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}. 1459 * When the 16 consecutive reads are often interrupted by some dirty 1460 * throttling pause during the async writes, cfq will go into idles 1461 * (deadline is fine). So push nr_dirtied_pause as high as possible 1462 * until reaches DIRTY_POLL_THRESH=32 pages. 1463 */ 1464 if (pages < DIRTY_POLL_THRESH) { 1465 t = max_pause; 1466 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ); 1467 if (pages > DIRTY_POLL_THRESH) { 1468 pages = DIRTY_POLL_THRESH; 1469 t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit; 1470 } 1471 } 1472 1473 pause = HZ * pages / (task_ratelimit + 1); 1474 if (pause > max_pause) { 1475 t = max_pause; 1476 pages = task_ratelimit * t / roundup_pow_of_two(HZ); 1477 } 1478 1479 *nr_dirtied_pause = pages; 1480 /* 1481 * The minimal pause time will normally be half the target pause time. 1482 */ 1483 return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t; 1484 } 1485 1486 static inline void wb_dirty_limits(struct dirty_throttle_control *dtc) 1487 { 1488 struct bdi_writeback *wb = dtc->wb; 1489 unsigned long wb_reclaimable; 1490 1491 /* 1492 * wb_thresh is not treated as some limiting factor as 1493 * dirty_thresh, due to reasons 1494 * - in JBOD setup, wb_thresh can fluctuate a lot 1495 * - in a system with HDD and USB key, the USB key may somehow 1496 * go into state (wb_dirty >> wb_thresh) either because 1497 * wb_dirty starts high, or because wb_thresh drops low. 1498 * In this case we don't want to hard throttle the USB key 1499 * dirtiers for 100 seconds until wb_dirty drops under 1500 * wb_thresh. Instead the auxiliary wb control line in 1501 * wb_position_ratio() will let the dirtier task progress 1502 * at some rate <= (write_bw / 2) for bringing down wb_dirty. 1503 */ 1504 dtc->wb_thresh = __wb_calc_thresh(dtc); 1505 dtc->wb_bg_thresh = dtc->thresh ? 1506 div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0; 1507 1508 /* 1509 * In order to avoid the stacked BDI deadlock we need 1510 * to ensure we accurately count the 'dirty' pages when 1511 * the threshold is low. 1512 * 1513 * Otherwise it would be possible to get thresh+n pages 1514 * reported dirty, even though there are thresh-m pages 1515 * actually dirty; with m+n sitting in the percpu 1516 * deltas. 1517 */ 1518 if (dtc->wb_thresh < 2 * wb_stat_error(wb)) { 1519 wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE); 1520 dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK); 1521 } else { 1522 wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE); 1523 dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK); 1524 } 1525 } 1526 1527 /* 1528 * balance_dirty_pages() must be called by processes which are generating dirty 1529 * data. It looks at the number of dirty pages in the machine and will force 1530 * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2. 1531 * If we're over `background_thresh' then the writeback threads are woken to 1532 * perform some writeout. 1533 */ 1534 static void balance_dirty_pages(struct address_space *mapping, 1535 struct bdi_writeback *wb, 1536 unsigned long pages_dirtied) 1537 { 1538 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) }; 1539 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) }; 1540 struct dirty_throttle_control * const gdtc = &gdtc_stor; 1541 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ? 1542 &mdtc_stor : NULL; 1543 struct dirty_throttle_control *sdtc; 1544 unsigned long nr_reclaimable; /* = file_dirty + unstable_nfs */ 1545 long period; 1546 long pause; 1547 long max_pause; 1548 long min_pause; 1549 int nr_dirtied_pause; 1550 bool dirty_exceeded = false; 1551 unsigned long task_ratelimit; 1552 unsigned long dirty_ratelimit; 1553 struct backing_dev_info *bdi = wb->bdi; 1554 bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT; 1555 unsigned long start_time = jiffies; 1556 1557 for (;;) { 1558 unsigned long now = jiffies; 1559 unsigned long dirty, thresh, bg_thresh; 1560 unsigned long m_dirty = 0; /* stop bogus uninit warnings */ 1561 unsigned long m_thresh = 0; 1562 unsigned long m_bg_thresh = 0; 1563 1564 /* 1565 * Unstable writes are a feature of certain networked 1566 * filesystems (i.e. NFS) in which data may have been 1567 * written to the server's write cache, but has not yet 1568 * been flushed to permanent storage. 1569 */ 1570 nr_reclaimable = global_page_state(NR_FILE_DIRTY) + 1571 global_page_state(NR_UNSTABLE_NFS); 1572 gdtc->avail = global_dirtyable_memory(); 1573 gdtc->dirty = nr_reclaimable + global_page_state(NR_WRITEBACK); 1574 1575 domain_dirty_limits(gdtc); 1576 1577 if (unlikely(strictlimit)) { 1578 wb_dirty_limits(gdtc); 1579 1580 dirty = gdtc->wb_dirty; 1581 thresh = gdtc->wb_thresh; 1582 bg_thresh = gdtc->wb_bg_thresh; 1583 } else { 1584 dirty = gdtc->dirty; 1585 thresh = gdtc->thresh; 1586 bg_thresh = gdtc->bg_thresh; 1587 } 1588 1589 if (mdtc) { 1590 unsigned long filepages, headroom, writeback; 1591 1592 /* 1593 * If @wb belongs to !root memcg, repeat the same 1594 * basic calculations for the memcg domain. 1595 */ 1596 mem_cgroup_wb_stats(wb, &filepages, &headroom, 1597 &mdtc->dirty, &writeback); 1598 mdtc->dirty += writeback; 1599 mdtc_calc_avail(mdtc, filepages, headroom); 1600 1601 domain_dirty_limits(mdtc); 1602 1603 if (unlikely(strictlimit)) { 1604 wb_dirty_limits(mdtc); 1605 m_dirty = mdtc->wb_dirty; 1606 m_thresh = mdtc->wb_thresh; 1607 m_bg_thresh = mdtc->wb_bg_thresh; 1608 } else { 1609 m_dirty = mdtc->dirty; 1610 m_thresh = mdtc->thresh; 1611 m_bg_thresh = mdtc->bg_thresh; 1612 } 1613 } 1614 1615 /* 1616 * Throttle it only when the background writeback cannot 1617 * catch-up. This avoids (excessively) small writeouts 1618 * when the wb limits are ramping up in case of !strictlimit. 1619 * 1620 * In strictlimit case make decision based on the wb counters 1621 * and limits. Small writeouts when the wb limits are ramping 1622 * up are the price we consciously pay for strictlimit-ing. 1623 * 1624 * If memcg domain is in effect, @dirty should be under 1625 * both global and memcg freerun ceilings. 1626 */ 1627 if (dirty <= dirty_freerun_ceiling(thresh, bg_thresh) && 1628 (!mdtc || 1629 m_dirty <= dirty_freerun_ceiling(m_thresh, m_bg_thresh))) { 1630 unsigned long intv = dirty_poll_interval(dirty, thresh); 1631 unsigned long m_intv = ULONG_MAX; 1632 1633 current->dirty_paused_when = now; 1634 current->nr_dirtied = 0; 1635 if (mdtc) 1636 m_intv = dirty_poll_interval(m_dirty, m_thresh); 1637 current->nr_dirtied_pause = min(intv, m_intv); 1638 break; 1639 } 1640 1641 if (unlikely(!writeback_in_progress(wb))) 1642 wb_start_background_writeback(wb); 1643 1644 /* 1645 * Calculate global domain's pos_ratio and select the 1646 * global dtc by default. 1647 */ 1648 if (!strictlimit) 1649 wb_dirty_limits(gdtc); 1650 1651 dirty_exceeded = (gdtc->wb_dirty > gdtc->wb_thresh) && 1652 ((gdtc->dirty > gdtc->thresh) || strictlimit); 1653 1654 wb_position_ratio(gdtc); 1655 sdtc = gdtc; 1656 1657 if (mdtc) { 1658 /* 1659 * If memcg domain is in effect, calculate its 1660 * pos_ratio. @wb should satisfy constraints from 1661 * both global and memcg domains. Choose the one 1662 * w/ lower pos_ratio. 1663 */ 1664 if (!strictlimit) 1665 wb_dirty_limits(mdtc); 1666 1667 dirty_exceeded |= (mdtc->wb_dirty > mdtc->wb_thresh) && 1668 ((mdtc->dirty > mdtc->thresh) || strictlimit); 1669 1670 wb_position_ratio(mdtc); 1671 if (mdtc->pos_ratio < gdtc->pos_ratio) 1672 sdtc = mdtc; 1673 } 1674 1675 if (dirty_exceeded && !wb->dirty_exceeded) 1676 wb->dirty_exceeded = 1; 1677 1678 if (time_is_before_jiffies(wb->bw_time_stamp + 1679 BANDWIDTH_INTERVAL)) { 1680 spin_lock(&wb->list_lock); 1681 __wb_update_bandwidth(gdtc, mdtc, start_time, true); 1682 spin_unlock(&wb->list_lock); 1683 } 1684 1685 /* throttle according to the chosen dtc */ 1686 dirty_ratelimit = wb->dirty_ratelimit; 1687 task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >> 1688 RATELIMIT_CALC_SHIFT; 1689 max_pause = wb_max_pause(wb, sdtc->wb_dirty); 1690 min_pause = wb_min_pause(wb, max_pause, 1691 task_ratelimit, dirty_ratelimit, 1692 &nr_dirtied_pause); 1693 1694 if (unlikely(task_ratelimit == 0)) { 1695 period = max_pause; 1696 pause = max_pause; 1697 goto pause; 1698 } 1699 period = HZ * pages_dirtied / task_ratelimit; 1700 pause = period; 1701 if (current->dirty_paused_when) 1702 pause -= now - current->dirty_paused_when; 1703 /* 1704 * For less than 1s think time (ext3/4 may block the dirtier 1705 * for up to 800ms from time to time on 1-HDD; so does xfs, 1706 * however at much less frequency), try to compensate it in 1707 * future periods by updating the virtual time; otherwise just 1708 * do a reset, as it may be a light dirtier. 1709 */ 1710 if (pause < min_pause) { 1711 trace_balance_dirty_pages(wb, 1712 sdtc->thresh, 1713 sdtc->bg_thresh, 1714 sdtc->dirty, 1715 sdtc->wb_thresh, 1716 sdtc->wb_dirty, 1717 dirty_ratelimit, 1718 task_ratelimit, 1719 pages_dirtied, 1720 period, 1721 min(pause, 0L), 1722 start_time); 1723 if (pause < -HZ) { 1724 current->dirty_paused_when = now; 1725 current->nr_dirtied = 0; 1726 } else if (period) { 1727 current->dirty_paused_when += period; 1728 current->nr_dirtied = 0; 1729 } else if (current->nr_dirtied_pause <= pages_dirtied) 1730 current->nr_dirtied_pause += pages_dirtied; 1731 break; 1732 } 1733 if (unlikely(pause > max_pause)) { 1734 /* for occasional dropped task_ratelimit */ 1735 now += min(pause - max_pause, max_pause); 1736 pause = max_pause; 1737 } 1738 1739 pause: 1740 trace_balance_dirty_pages(wb, 1741 sdtc->thresh, 1742 sdtc->bg_thresh, 1743 sdtc->dirty, 1744 sdtc->wb_thresh, 1745 sdtc->wb_dirty, 1746 dirty_ratelimit, 1747 task_ratelimit, 1748 pages_dirtied, 1749 period, 1750 pause, 1751 start_time); 1752 __set_current_state(TASK_KILLABLE); 1753 io_schedule_timeout(pause); 1754 1755 current->dirty_paused_when = now + pause; 1756 current->nr_dirtied = 0; 1757 current->nr_dirtied_pause = nr_dirtied_pause; 1758 1759 /* 1760 * This is typically equal to (dirty < thresh) and can also 1761 * keep "1000+ dd on a slow USB stick" under control. 1762 */ 1763 if (task_ratelimit) 1764 break; 1765 1766 /* 1767 * In the case of an unresponding NFS server and the NFS dirty 1768 * pages exceeds dirty_thresh, give the other good wb's a pipe 1769 * to go through, so that tasks on them still remain responsive. 1770 * 1771 * In theory 1 page is enough to keep the comsumer-producer 1772 * pipe going: the flusher cleans 1 page => the task dirties 1 1773 * more page. However wb_dirty has accounting errors. So use 1774 * the larger and more IO friendly wb_stat_error. 1775 */ 1776 if (sdtc->wb_dirty <= wb_stat_error(wb)) 1777 break; 1778 1779 if (fatal_signal_pending(current)) 1780 break; 1781 } 1782 1783 if (!dirty_exceeded && wb->dirty_exceeded) 1784 wb->dirty_exceeded = 0; 1785 1786 if (writeback_in_progress(wb)) 1787 return; 1788 1789 /* 1790 * In laptop mode, we wait until hitting the higher threshold before 1791 * starting background writeout, and then write out all the way down 1792 * to the lower threshold. So slow writers cause minimal disk activity. 1793 * 1794 * In normal mode, we start background writeout at the lower 1795 * background_thresh, to keep the amount of dirty memory low. 1796 */ 1797 if (laptop_mode) 1798 return; 1799 1800 if (nr_reclaimable > gdtc->bg_thresh) 1801 wb_start_background_writeback(wb); 1802 } 1803 1804 static DEFINE_PER_CPU(int, bdp_ratelimits); 1805 1806 /* 1807 * Normal tasks are throttled by 1808 * loop { 1809 * dirty tsk->nr_dirtied_pause pages; 1810 * take a snap in balance_dirty_pages(); 1811 * } 1812 * However there is a worst case. If every task exit immediately when dirtied 1813 * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be 1814 * called to throttle the page dirties. The solution is to save the not yet 1815 * throttled page dirties in dirty_throttle_leaks on task exit and charge them 1816 * randomly into the running tasks. This works well for the above worst case, 1817 * as the new task will pick up and accumulate the old task's leaked dirty 1818 * count and eventually get throttled. 1819 */ 1820 DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0; 1821 1822 /** 1823 * balance_dirty_pages_ratelimited - balance dirty memory state 1824 * @mapping: address_space which was dirtied 1825 * 1826 * Processes which are dirtying memory should call in here once for each page 1827 * which was newly dirtied. The function will periodically check the system's 1828 * dirty state and will initiate writeback if needed. 1829 * 1830 * On really big machines, get_writeback_state is expensive, so try to avoid 1831 * calling it too often (ratelimiting). But once we're over the dirty memory 1832 * limit we decrease the ratelimiting by a lot, to prevent individual processes 1833 * from overshooting the limit by (ratelimit_pages) each. 1834 */ 1835 void balance_dirty_pages_ratelimited(struct address_space *mapping) 1836 { 1837 struct inode *inode = mapping->host; 1838 struct backing_dev_info *bdi = inode_to_bdi(inode); 1839 struct bdi_writeback *wb = NULL; 1840 int ratelimit; 1841 int *p; 1842 1843 if (!bdi_cap_account_dirty(bdi)) 1844 return; 1845 1846 if (inode_cgwb_enabled(inode)) 1847 wb = wb_get_create_current(bdi, GFP_KERNEL); 1848 if (!wb) 1849 wb = &bdi->wb; 1850 1851 ratelimit = current->nr_dirtied_pause; 1852 if (wb->dirty_exceeded) 1853 ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10)); 1854 1855 preempt_disable(); 1856 /* 1857 * This prevents one CPU to accumulate too many dirtied pages without 1858 * calling into balance_dirty_pages(), which can happen when there are 1859 * 1000+ tasks, all of them start dirtying pages at exactly the same 1860 * time, hence all honoured too large initial task->nr_dirtied_pause. 1861 */ 1862 p = this_cpu_ptr(&bdp_ratelimits); 1863 if (unlikely(current->nr_dirtied >= ratelimit)) 1864 *p = 0; 1865 else if (unlikely(*p >= ratelimit_pages)) { 1866 *p = 0; 1867 ratelimit = 0; 1868 } 1869 /* 1870 * Pick up the dirtied pages by the exited tasks. This avoids lots of 1871 * short-lived tasks (eg. gcc invocations in a kernel build) escaping 1872 * the dirty throttling and livelock other long-run dirtiers. 1873 */ 1874 p = this_cpu_ptr(&dirty_throttle_leaks); 1875 if (*p > 0 && current->nr_dirtied < ratelimit) { 1876 unsigned long nr_pages_dirtied; 1877 nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied); 1878 *p -= nr_pages_dirtied; 1879 current->nr_dirtied += nr_pages_dirtied; 1880 } 1881 preempt_enable(); 1882 1883 if (unlikely(current->nr_dirtied >= ratelimit)) 1884 balance_dirty_pages(mapping, wb, current->nr_dirtied); 1885 1886 wb_put(wb); 1887 } 1888 EXPORT_SYMBOL(balance_dirty_pages_ratelimited); 1889 1890 /** 1891 * wb_over_bg_thresh - does @wb need to be written back? 1892 * @wb: bdi_writeback of interest 1893 * 1894 * Determines whether background writeback should keep writing @wb or it's 1895 * clean enough. Returns %true if writeback should continue. 1896 */ 1897 bool wb_over_bg_thresh(struct bdi_writeback *wb) 1898 { 1899 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) }; 1900 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) }; 1901 struct dirty_throttle_control * const gdtc = &gdtc_stor; 1902 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ? 1903 &mdtc_stor : NULL; 1904 1905 /* 1906 * Similar to balance_dirty_pages() but ignores pages being written 1907 * as we're trying to decide whether to put more under writeback. 1908 */ 1909 gdtc->avail = global_dirtyable_memory(); 1910 gdtc->dirty = global_page_state(NR_FILE_DIRTY) + 1911 global_page_state(NR_UNSTABLE_NFS); 1912 domain_dirty_limits(gdtc); 1913 1914 if (gdtc->dirty > gdtc->bg_thresh) 1915 return true; 1916 1917 if (wb_stat(wb, WB_RECLAIMABLE) > 1918 wb_calc_thresh(gdtc->wb, gdtc->bg_thresh)) 1919 return true; 1920 1921 if (mdtc) { 1922 unsigned long filepages, headroom, writeback; 1923 1924 mem_cgroup_wb_stats(wb, &filepages, &headroom, &mdtc->dirty, 1925 &writeback); 1926 mdtc_calc_avail(mdtc, filepages, headroom); 1927 domain_dirty_limits(mdtc); /* ditto, ignore writeback */ 1928 1929 if (mdtc->dirty > mdtc->bg_thresh) 1930 return true; 1931 1932 if (wb_stat(wb, WB_RECLAIMABLE) > 1933 wb_calc_thresh(mdtc->wb, mdtc->bg_thresh)) 1934 return true; 1935 } 1936 1937 return false; 1938 } 1939 1940 void throttle_vm_writeout(gfp_t gfp_mask) 1941 { 1942 unsigned long background_thresh; 1943 unsigned long dirty_thresh; 1944 1945 for ( ; ; ) { 1946 global_dirty_limits(&background_thresh, &dirty_thresh); 1947 dirty_thresh = hard_dirty_limit(&global_wb_domain, dirty_thresh); 1948 1949 /* 1950 * Boost the allowable dirty threshold a bit for page 1951 * allocators so they don't get DoS'ed by heavy writers 1952 */ 1953 dirty_thresh += dirty_thresh / 10; /* wheeee... */ 1954 1955 if (global_page_state(NR_UNSTABLE_NFS) + 1956 global_page_state(NR_WRITEBACK) <= dirty_thresh) 1957 break; 1958 congestion_wait(BLK_RW_ASYNC, HZ/10); 1959 1960 /* 1961 * The caller might hold locks which can prevent IO completion 1962 * or progress in the filesystem. So we cannot just sit here 1963 * waiting for IO to complete. 1964 */ 1965 if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO)) 1966 break; 1967 } 1968 } 1969 1970 /* 1971 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs 1972 */ 1973 int dirty_writeback_centisecs_handler(struct ctl_table *table, int write, 1974 void __user *buffer, size_t *length, loff_t *ppos) 1975 { 1976 proc_dointvec(table, write, buffer, length, ppos); 1977 return 0; 1978 } 1979 1980 #ifdef CONFIG_BLOCK 1981 void laptop_mode_timer_fn(unsigned long data) 1982 { 1983 struct request_queue *q = (struct request_queue *)data; 1984 int nr_pages = global_page_state(NR_FILE_DIRTY) + 1985 global_page_state(NR_UNSTABLE_NFS); 1986 struct bdi_writeback *wb; 1987 1988 /* 1989 * We want to write everything out, not just down to the dirty 1990 * threshold 1991 */ 1992 if (!bdi_has_dirty_io(&q->backing_dev_info)) 1993 return; 1994 1995 rcu_read_lock(); 1996 list_for_each_entry_rcu(wb, &q->backing_dev_info.wb_list, bdi_node) 1997 if (wb_has_dirty_io(wb)) 1998 wb_start_writeback(wb, nr_pages, true, 1999 WB_REASON_LAPTOP_TIMER); 2000 rcu_read_unlock(); 2001 } 2002 2003 /* 2004 * We've spun up the disk and we're in laptop mode: schedule writeback 2005 * of all dirty data a few seconds from now. If the flush is already scheduled 2006 * then push it back - the user is still using the disk. 2007 */ 2008 void laptop_io_completion(struct backing_dev_info *info) 2009 { 2010 mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode); 2011 } 2012 2013 /* 2014 * We're in laptop mode and we've just synced. The sync's writes will have 2015 * caused another writeback to be scheduled by laptop_io_completion. 2016 * Nothing needs to be written back anymore, so we unschedule the writeback. 2017 */ 2018 void laptop_sync_completion(void) 2019 { 2020 struct backing_dev_info *bdi; 2021 2022 rcu_read_lock(); 2023 2024 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) 2025 del_timer(&bdi->laptop_mode_wb_timer); 2026 2027 rcu_read_unlock(); 2028 } 2029 #endif 2030 2031 /* 2032 * If ratelimit_pages is too high then we can get into dirty-data overload 2033 * if a large number of processes all perform writes at the same time. 2034 * If it is too low then SMP machines will call the (expensive) 2035 * get_writeback_state too often. 2036 * 2037 * Here we set ratelimit_pages to a level which ensures that when all CPUs are 2038 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory 2039 * thresholds. 2040 */ 2041 2042 void writeback_set_ratelimit(void) 2043 { 2044 struct wb_domain *dom = &global_wb_domain; 2045 unsigned long background_thresh; 2046 unsigned long dirty_thresh; 2047 2048 global_dirty_limits(&background_thresh, &dirty_thresh); 2049 dom->dirty_limit = dirty_thresh; 2050 ratelimit_pages = dirty_thresh / (num_online_cpus() * 32); 2051 if (ratelimit_pages < 16) 2052 ratelimit_pages = 16; 2053 } 2054 2055 static int 2056 ratelimit_handler(struct notifier_block *self, unsigned long action, 2057 void *hcpu) 2058 { 2059 2060 switch (action & ~CPU_TASKS_FROZEN) { 2061 case CPU_ONLINE: 2062 case CPU_DEAD: 2063 writeback_set_ratelimit(); 2064 return NOTIFY_OK; 2065 default: 2066 return NOTIFY_DONE; 2067 } 2068 } 2069 2070 static struct notifier_block ratelimit_nb = { 2071 .notifier_call = ratelimit_handler, 2072 .next = NULL, 2073 }; 2074 2075 /* 2076 * Called early on to tune the page writeback dirty limits. 2077 * 2078 * We used to scale dirty pages according to how total memory 2079 * related to pages that could be allocated for buffers (by 2080 * comparing nr_free_buffer_pages() to vm_total_pages. 2081 * 2082 * However, that was when we used "dirty_ratio" to scale with 2083 * all memory, and we don't do that any more. "dirty_ratio" 2084 * is now applied to total non-HIGHPAGE memory (by subtracting 2085 * totalhigh_pages from vm_total_pages), and as such we can't 2086 * get into the old insane situation any more where we had 2087 * large amounts of dirty pages compared to a small amount of 2088 * non-HIGHMEM memory. 2089 * 2090 * But we might still want to scale the dirty_ratio by how 2091 * much memory the box has.. 2092 */ 2093 void __init page_writeback_init(void) 2094 { 2095 BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL)); 2096 2097 writeback_set_ratelimit(); 2098 register_cpu_notifier(&ratelimit_nb); 2099 } 2100 2101 /** 2102 * tag_pages_for_writeback - tag pages to be written by write_cache_pages 2103 * @mapping: address space structure to write 2104 * @start: starting page index 2105 * @end: ending page index (inclusive) 2106 * 2107 * This function scans the page range from @start to @end (inclusive) and tags 2108 * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is 2109 * that write_cache_pages (or whoever calls this function) will then use 2110 * TOWRITE tag to identify pages eligible for writeback. This mechanism is 2111 * used to avoid livelocking of writeback by a process steadily creating new 2112 * dirty pages in the file (thus it is important for this function to be quick 2113 * so that it can tag pages faster than a dirtying process can create them). 2114 */ 2115 /* 2116 * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency. 2117 */ 2118 void tag_pages_for_writeback(struct address_space *mapping, 2119 pgoff_t start, pgoff_t end) 2120 { 2121 #define WRITEBACK_TAG_BATCH 4096 2122 unsigned long tagged; 2123 2124 do { 2125 spin_lock_irq(&mapping->tree_lock); 2126 tagged = radix_tree_range_tag_if_tagged(&mapping->page_tree, 2127 &start, end, WRITEBACK_TAG_BATCH, 2128 PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE); 2129 spin_unlock_irq(&mapping->tree_lock); 2130 WARN_ON_ONCE(tagged > WRITEBACK_TAG_BATCH); 2131 cond_resched(); 2132 /* We check 'start' to handle wrapping when end == ~0UL */ 2133 } while (tagged >= WRITEBACK_TAG_BATCH && start); 2134 } 2135 EXPORT_SYMBOL(tag_pages_for_writeback); 2136 2137 /** 2138 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. 2139 * @mapping: address space structure to write 2140 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 2141 * @writepage: function called for each page 2142 * @data: data passed to writepage function 2143 * 2144 * If a page is already under I/O, write_cache_pages() skips it, even 2145 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 2146 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 2147 * and msync() need to guarantee that all the data which was dirty at the time 2148 * the call was made get new I/O started against them. If wbc->sync_mode is 2149 * WB_SYNC_ALL then we were called for data integrity and we must wait for 2150 * existing IO to complete. 2151 * 2152 * To avoid livelocks (when other process dirties new pages), we first tag 2153 * pages which should be written back with TOWRITE tag and only then start 2154 * writing them. For data-integrity sync we have to be careful so that we do 2155 * not miss some pages (e.g., because some other process has cleared TOWRITE 2156 * tag we set). The rule we follow is that TOWRITE tag can be cleared only 2157 * by the process clearing the DIRTY tag (and submitting the page for IO). 2158 */ 2159 int write_cache_pages(struct address_space *mapping, 2160 struct writeback_control *wbc, writepage_t writepage, 2161 void *data) 2162 { 2163 int ret = 0; 2164 int done = 0; 2165 struct pagevec pvec; 2166 int nr_pages; 2167 pgoff_t uninitialized_var(writeback_index); 2168 pgoff_t index; 2169 pgoff_t end; /* Inclusive */ 2170 pgoff_t done_index; 2171 int cycled; 2172 int range_whole = 0; 2173 int tag; 2174 2175 pagevec_init(&pvec, 0); 2176 if (wbc->range_cyclic) { 2177 writeback_index = mapping->writeback_index; /* prev offset */ 2178 index = writeback_index; 2179 if (index == 0) 2180 cycled = 1; 2181 else 2182 cycled = 0; 2183 end = -1; 2184 } else { 2185 index = wbc->range_start >> PAGE_SHIFT; 2186 end = wbc->range_end >> PAGE_SHIFT; 2187 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2188 range_whole = 1; 2189 cycled = 1; /* ignore range_cyclic tests */ 2190 } 2191 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2192 tag = PAGECACHE_TAG_TOWRITE; 2193 else 2194 tag = PAGECACHE_TAG_DIRTY; 2195 retry: 2196 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2197 tag_pages_for_writeback(mapping, index, end); 2198 done_index = index; 2199 while (!done && (index <= end)) { 2200 int i; 2201 2202 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 2203 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); 2204 if (nr_pages == 0) 2205 break; 2206 2207 for (i = 0; i < nr_pages; i++) { 2208 struct page *page = pvec.pages[i]; 2209 2210 /* 2211 * At this point, the page may be truncated or 2212 * invalidated (changing page->mapping to NULL), or 2213 * even swizzled back from swapper_space to tmpfs file 2214 * mapping. However, page->index will not change 2215 * because we have a reference on the page. 2216 */ 2217 if (page->index > end) { 2218 /* 2219 * can't be range_cyclic (1st pass) because 2220 * end == -1 in that case. 2221 */ 2222 done = 1; 2223 break; 2224 } 2225 2226 done_index = page->index; 2227 2228 lock_page(page); 2229 2230 /* 2231 * Page truncated or invalidated. We can freely skip it 2232 * then, even for data integrity operations: the page 2233 * has disappeared concurrently, so there could be no 2234 * real expectation of this data interity operation 2235 * even if there is now a new, dirty page at the same 2236 * pagecache address. 2237 */ 2238 if (unlikely(page->mapping != mapping)) { 2239 continue_unlock: 2240 unlock_page(page); 2241 continue; 2242 } 2243 2244 if (!PageDirty(page)) { 2245 /* someone wrote it for us */ 2246 goto continue_unlock; 2247 } 2248 2249 if (PageWriteback(page)) { 2250 if (wbc->sync_mode != WB_SYNC_NONE) 2251 wait_on_page_writeback(page); 2252 else 2253 goto continue_unlock; 2254 } 2255 2256 BUG_ON(PageWriteback(page)); 2257 if (!clear_page_dirty_for_io(page)) 2258 goto continue_unlock; 2259 2260 trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); 2261 ret = (*writepage)(page, wbc, data); 2262 if (unlikely(ret)) { 2263 if (ret == AOP_WRITEPAGE_ACTIVATE) { 2264 unlock_page(page); 2265 ret = 0; 2266 } else { 2267 /* 2268 * done_index is set past this page, 2269 * so media errors will not choke 2270 * background writeout for the entire 2271 * file. This has consequences for 2272 * range_cyclic semantics (ie. it may 2273 * not be suitable for data integrity 2274 * writeout). 2275 */ 2276 done_index = page->index + 1; 2277 done = 1; 2278 break; 2279 } 2280 } 2281 2282 /* 2283 * We stop writing back only if we are not doing 2284 * integrity sync. In case of integrity sync we have to 2285 * keep going until we have written all the pages 2286 * we tagged for writeback prior to entering this loop. 2287 */ 2288 if (--wbc->nr_to_write <= 0 && 2289 wbc->sync_mode == WB_SYNC_NONE) { 2290 done = 1; 2291 break; 2292 } 2293 } 2294 pagevec_release(&pvec); 2295 cond_resched(); 2296 } 2297 if (!cycled && !done) { 2298 /* 2299 * range_cyclic: 2300 * We hit the last page and there is more work to be done: wrap 2301 * back to the start of the file 2302 */ 2303 cycled = 1; 2304 index = 0; 2305 end = writeback_index - 1; 2306 goto retry; 2307 } 2308 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 2309 mapping->writeback_index = done_index; 2310 2311 return ret; 2312 } 2313 EXPORT_SYMBOL(write_cache_pages); 2314 2315 /* 2316 * Function used by generic_writepages to call the real writepage 2317 * function and set the mapping flags on error 2318 */ 2319 static int __writepage(struct page *page, struct writeback_control *wbc, 2320 void *data) 2321 { 2322 struct address_space *mapping = data; 2323 int ret = mapping->a_ops->writepage(page, wbc); 2324 mapping_set_error(mapping, ret); 2325 return ret; 2326 } 2327 2328 /** 2329 * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them. 2330 * @mapping: address space structure to write 2331 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 2332 * 2333 * This is a library function, which implements the writepages() 2334 * address_space_operation. 2335 */ 2336 int generic_writepages(struct address_space *mapping, 2337 struct writeback_control *wbc) 2338 { 2339 struct blk_plug plug; 2340 int ret; 2341 2342 /* deal with chardevs and other special file */ 2343 if (!mapping->a_ops->writepage) 2344 return 0; 2345 2346 blk_start_plug(&plug); 2347 ret = write_cache_pages(mapping, wbc, __writepage, mapping); 2348 blk_finish_plug(&plug); 2349 return ret; 2350 } 2351 2352 EXPORT_SYMBOL(generic_writepages); 2353 2354 int do_writepages(struct address_space *mapping, struct writeback_control *wbc) 2355 { 2356 int ret; 2357 2358 if (wbc->nr_to_write <= 0) 2359 return 0; 2360 if (mapping->a_ops->writepages) 2361 ret = mapping->a_ops->writepages(mapping, wbc); 2362 else 2363 ret = generic_writepages(mapping, wbc); 2364 return ret; 2365 } 2366 2367 /** 2368 * write_one_page - write out a single page and optionally wait on I/O 2369 * @page: the page to write 2370 * @wait: if true, wait on writeout 2371 * 2372 * The page must be locked by the caller and will be unlocked upon return. 2373 * 2374 * write_one_page() returns a negative error code if I/O failed. 2375 */ 2376 int write_one_page(struct page *page, int wait) 2377 { 2378 struct address_space *mapping = page->mapping; 2379 int ret = 0; 2380 struct writeback_control wbc = { 2381 .sync_mode = WB_SYNC_ALL, 2382 .nr_to_write = 1, 2383 }; 2384 2385 BUG_ON(!PageLocked(page)); 2386 2387 if (wait) 2388 wait_on_page_writeback(page); 2389 2390 if (clear_page_dirty_for_io(page)) { 2391 get_page(page); 2392 ret = mapping->a_ops->writepage(page, &wbc); 2393 if (ret == 0 && wait) { 2394 wait_on_page_writeback(page); 2395 if (PageError(page)) 2396 ret = -EIO; 2397 } 2398 put_page(page); 2399 } else { 2400 unlock_page(page); 2401 } 2402 return ret; 2403 } 2404 EXPORT_SYMBOL(write_one_page); 2405 2406 /* 2407 * For address_spaces which do not use buffers nor write back. 2408 */ 2409 int __set_page_dirty_no_writeback(struct page *page) 2410 { 2411 if (!PageDirty(page)) 2412 return !TestSetPageDirty(page); 2413 return 0; 2414 } 2415 2416 /* 2417 * Helper function for set_page_dirty family. 2418 * 2419 * Caller must hold lock_page_memcg(). 2420 * 2421 * NOTE: This relies on being atomic wrt interrupts. 2422 */ 2423 void account_page_dirtied(struct page *page, struct address_space *mapping) 2424 { 2425 struct inode *inode = mapping->host; 2426 2427 trace_writeback_dirty_page(page, mapping); 2428 2429 if (mapping_cap_account_dirty(mapping)) { 2430 struct bdi_writeback *wb; 2431 2432 inode_attach_wb(inode, page); 2433 wb = inode_to_wb(inode); 2434 2435 mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_DIRTY); 2436 __inc_zone_page_state(page, NR_FILE_DIRTY); 2437 __inc_zone_page_state(page, NR_DIRTIED); 2438 __inc_wb_stat(wb, WB_RECLAIMABLE); 2439 __inc_wb_stat(wb, WB_DIRTIED); 2440 task_io_account_write(PAGE_SIZE); 2441 current->nr_dirtied++; 2442 this_cpu_inc(bdp_ratelimits); 2443 } 2444 } 2445 EXPORT_SYMBOL(account_page_dirtied); 2446 2447 /* 2448 * Helper function for deaccounting dirty page without writeback. 2449 * 2450 * Caller must hold lock_page_memcg(). 2451 */ 2452 void account_page_cleaned(struct page *page, struct address_space *mapping, 2453 struct bdi_writeback *wb) 2454 { 2455 if (mapping_cap_account_dirty(mapping)) { 2456 mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_DIRTY); 2457 dec_zone_page_state(page, NR_FILE_DIRTY); 2458 dec_wb_stat(wb, WB_RECLAIMABLE); 2459 task_io_account_cancelled_write(PAGE_SIZE); 2460 } 2461 } 2462 2463 /* 2464 * For address_spaces which do not use buffers. Just tag the page as dirty in 2465 * its radix tree. 2466 * 2467 * This is also used when a single buffer is being dirtied: we want to set the 2468 * page dirty in that case, but not all the buffers. This is a "bottom-up" 2469 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. 2470 * 2471 * The caller must ensure this doesn't race with truncation. Most will simply 2472 * hold the page lock, but e.g. zap_pte_range() calls with the page mapped and 2473 * the pte lock held, which also locks out truncation. 2474 */ 2475 int __set_page_dirty_nobuffers(struct page *page) 2476 { 2477 lock_page_memcg(page); 2478 if (!TestSetPageDirty(page)) { 2479 struct address_space *mapping = page_mapping(page); 2480 unsigned long flags; 2481 2482 if (!mapping) { 2483 unlock_page_memcg(page); 2484 return 1; 2485 } 2486 2487 spin_lock_irqsave(&mapping->tree_lock, flags); 2488 BUG_ON(page_mapping(page) != mapping); 2489 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); 2490 account_page_dirtied(page, mapping); 2491 radix_tree_tag_set(&mapping->page_tree, page_index(page), 2492 PAGECACHE_TAG_DIRTY); 2493 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2494 unlock_page_memcg(page); 2495 2496 if (mapping->host) { 2497 /* !PageAnon && !swapper_space */ 2498 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 2499 } 2500 return 1; 2501 } 2502 unlock_page_memcg(page); 2503 return 0; 2504 } 2505 EXPORT_SYMBOL(__set_page_dirty_nobuffers); 2506 2507 /* 2508 * Call this whenever redirtying a page, to de-account the dirty counters 2509 * (NR_DIRTIED, BDI_DIRTIED, tsk->nr_dirtied), so that they match the written 2510 * counters (NR_WRITTEN, BDI_WRITTEN) in long term. The mismatches will lead to 2511 * systematic errors in balanced_dirty_ratelimit and the dirty pages position 2512 * control. 2513 */ 2514 void account_page_redirty(struct page *page) 2515 { 2516 struct address_space *mapping = page->mapping; 2517 2518 if (mapping && mapping_cap_account_dirty(mapping)) { 2519 struct inode *inode = mapping->host; 2520 struct bdi_writeback *wb; 2521 bool locked; 2522 2523 wb = unlocked_inode_to_wb_begin(inode, &locked); 2524 current->nr_dirtied--; 2525 dec_zone_page_state(page, NR_DIRTIED); 2526 dec_wb_stat(wb, WB_DIRTIED); 2527 unlocked_inode_to_wb_end(inode, locked); 2528 } 2529 } 2530 EXPORT_SYMBOL(account_page_redirty); 2531 2532 /* 2533 * When a writepage implementation decides that it doesn't want to write this 2534 * page for some reason, it should redirty the locked page via 2535 * redirty_page_for_writepage() and it should then unlock the page and return 0 2536 */ 2537 int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page) 2538 { 2539 int ret; 2540 2541 wbc->pages_skipped++; 2542 ret = __set_page_dirty_nobuffers(page); 2543 account_page_redirty(page); 2544 return ret; 2545 } 2546 EXPORT_SYMBOL(redirty_page_for_writepage); 2547 2548 /* 2549 * Dirty a page. 2550 * 2551 * For pages with a mapping this should be done under the page lock 2552 * for the benefit of asynchronous memory errors who prefer a consistent 2553 * dirty state. This rule can be broken in some special cases, 2554 * but should be better not to. 2555 * 2556 * If the mapping doesn't provide a set_page_dirty a_op, then 2557 * just fall through and assume that it wants buffer_heads. 2558 */ 2559 int set_page_dirty(struct page *page) 2560 { 2561 struct address_space *mapping = page_mapping(page); 2562 2563 if (likely(mapping)) { 2564 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty; 2565 /* 2566 * readahead/lru_deactivate_page could remain 2567 * PG_readahead/PG_reclaim due to race with end_page_writeback 2568 * About readahead, if the page is written, the flags would be 2569 * reset. So no problem. 2570 * About lru_deactivate_page, if the page is redirty, the flag 2571 * will be reset. So no problem. but if the page is used by readahead 2572 * it will confuse readahead and make it restart the size rampup 2573 * process. But it's a trivial problem. 2574 */ 2575 if (PageReclaim(page)) 2576 ClearPageReclaim(page); 2577 #ifdef CONFIG_BLOCK 2578 if (!spd) 2579 spd = __set_page_dirty_buffers; 2580 #endif 2581 return (*spd)(page); 2582 } 2583 if (!PageDirty(page)) { 2584 if (!TestSetPageDirty(page)) 2585 return 1; 2586 } 2587 return 0; 2588 } 2589 EXPORT_SYMBOL(set_page_dirty); 2590 2591 /* 2592 * set_page_dirty() is racy if the caller has no reference against 2593 * page->mapping->host, and if the page is unlocked. This is because another 2594 * CPU could truncate the page off the mapping and then free the mapping. 2595 * 2596 * Usually, the page _is_ locked, or the caller is a user-space process which 2597 * holds a reference on the inode by having an open file. 2598 * 2599 * In other cases, the page should be locked before running set_page_dirty(). 2600 */ 2601 int set_page_dirty_lock(struct page *page) 2602 { 2603 int ret; 2604 2605 lock_page(page); 2606 ret = set_page_dirty(page); 2607 unlock_page(page); 2608 return ret; 2609 } 2610 EXPORT_SYMBOL(set_page_dirty_lock); 2611 2612 /* 2613 * This cancels just the dirty bit on the kernel page itself, it does NOT 2614 * actually remove dirty bits on any mmap's that may be around. It also 2615 * leaves the page tagged dirty, so any sync activity will still find it on 2616 * the dirty lists, and in particular, clear_page_dirty_for_io() will still 2617 * look at the dirty bits in the VM. 2618 * 2619 * Doing this should *normally* only ever be done when a page is truncated, 2620 * and is not actually mapped anywhere at all. However, fs/buffer.c does 2621 * this when it notices that somebody has cleaned out all the buffers on a 2622 * page without actually doing it through the VM. Can you say "ext3 is 2623 * horribly ugly"? Thought you could. 2624 */ 2625 void cancel_dirty_page(struct page *page) 2626 { 2627 struct address_space *mapping = page_mapping(page); 2628 2629 if (mapping_cap_account_dirty(mapping)) { 2630 struct inode *inode = mapping->host; 2631 struct bdi_writeback *wb; 2632 bool locked; 2633 2634 lock_page_memcg(page); 2635 wb = unlocked_inode_to_wb_begin(inode, &locked); 2636 2637 if (TestClearPageDirty(page)) 2638 account_page_cleaned(page, mapping, wb); 2639 2640 unlocked_inode_to_wb_end(inode, locked); 2641 unlock_page_memcg(page); 2642 } else { 2643 ClearPageDirty(page); 2644 } 2645 } 2646 EXPORT_SYMBOL(cancel_dirty_page); 2647 2648 /* 2649 * Clear a page's dirty flag, while caring for dirty memory accounting. 2650 * Returns true if the page was previously dirty. 2651 * 2652 * This is for preparing to put the page under writeout. We leave the page 2653 * tagged as dirty in the radix tree so that a concurrent write-for-sync 2654 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage 2655 * implementation will run either set_page_writeback() or set_page_dirty(), 2656 * at which stage we bring the page's dirty flag and radix-tree dirty tag 2657 * back into sync. 2658 * 2659 * This incoherency between the page's dirty flag and radix-tree tag is 2660 * unfortunate, but it only exists while the page is locked. 2661 */ 2662 int clear_page_dirty_for_io(struct page *page) 2663 { 2664 struct address_space *mapping = page_mapping(page); 2665 int ret = 0; 2666 2667 BUG_ON(!PageLocked(page)); 2668 2669 if (mapping && mapping_cap_account_dirty(mapping)) { 2670 struct inode *inode = mapping->host; 2671 struct bdi_writeback *wb; 2672 bool locked; 2673 2674 /* 2675 * Yes, Virginia, this is indeed insane. 2676 * 2677 * We use this sequence to make sure that 2678 * (a) we account for dirty stats properly 2679 * (b) we tell the low-level filesystem to 2680 * mark the whole page dirty if it was 2681 * dirty in a pagetable. Only to then 2682 * (c) clean the page again and return 1 to 2683 * cause the writeback. 2684 * 2685 * This way we avoid all nasty races with the 2686 * dirty bit in multiple places and clearing 2687 * them concurrently from different threads. 2688 * 2689 * Note! Normally the "set_page_dirty(page)" 2690 * has no effect on the actual dirty bit - since 2691 * that will already usually be set. But we 2692 * need the side effects, and it can help us 2693 * avoid races. 2694 * 2695 * We basically use the page "master dirty bit" 2696 * as a serialization point for all the different 2697 * threads doing their things. 2698 */ 2699 if (page_mkclean(page)) 2700 set_page_dirty(page); 2701 /* 2702 * We carefully synchronise fault handlers against 2703 * installing a dirty pte and marking the page dirty 2704 * at this point. We do this by having them hold the 2705 * page lock while dirtying the page, and pages are 2706 * always locked coming in here, so we get the desired 2707 * exclusion. 2708 */ 2709 wb = unlocked_inode_to_wb_begin(inode, &locked); 2710 if (TestClearPageDirty(page)) { 2711 mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_DIRTY); 2712 dec_zone_page_state(page, NR_FILE_DIRTY); 2713 dec_wb_stat(wb, WB_RECLAIMABLE); 2714 ret = 1; 2715 } 2716 unlocked_inode_to_wb_end(inode, locked); 2717 return ret; 2718 } 2719 return TestClearPageDirty(page); 2720 } 2721 EXPORT_SYMBOL(clear_page_dirty_for_io); 2722 2723 int test_clear_page_writeback(struct page *page) 2724 { 2725 struct address_space *mapping = page_mapping(page); 2726 int ret; 2727 2728 lock_page_memcg(page); 2729 if (mapping) { 2730 struct inode *inode = mapping->host; 2731 struct backing_dev_info *bdi = inode_to_bdi(inode); 2732 unsigned long flags; 2733 2734 spin_lock_irqsave(&mapping->tree_lock, flags); 2735 ret = TestClearPageWriteback(page); 2736 if (ret) { 2737 radix_tree_tag_clear(&mapping->page_tree, 2738 page_index(page), 2739 PAGECACHE_TAG_WRITEBACK); 2740 if (bdi_cap_account_writeback(bdi)) { 2741 struct bdi_writeback *wb = inode_to_wb(inode); 2742 2743 __dec_wb_stat(wb, WB_WRITEBACK); 2744 __wb_writeout_inc(wb); 2745 } 2746 } 2747 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2748 } else { 2749 ret = TestClearPageWriteback(page); 2750 } 2751 if (ret) { 2752 mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_WRITEBACK); 2753 dec_zone_page_state(page, NR_WRITEBACK); 2754 inc_zone_page_state(page, NR_WRITTEN); 2755 } 2756 unlock_page_memcg(page); 2757 return ret; 2758 } 2759 2760 int __test_set_page_writeback(struct page *page, bool keep_write) 2761 { 2762 struct address_space *mapping = page_mapping(page); 2763 int ret; 2764 2765 lock_page_memcg(page); 2766 if (mapping) { 2767 struct inode *inode = mapping->host; 2768 struct backing_dev_info *bdi = inode_to_bdi(inode); 2769 unsigned long flags; 2770 2771 spin_lock_irqsave(&mapping->tree_lock, flags); 2772 ret = TestSetPageWriteback(page); 2773 if (!ret) { 2774 radix_tree_tag_set(&mapping->page_tree, 2775 page_index(page), 2776 PAGECACHE_TAG_WRITEBACK); 2777 if (bdi_cap_account_writeback(bdi)) 2778 __inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); 2779 } 2780 if (!PageDirty(page)) 2781 radix_tree_tag_clear(&mapping->page_tree, 2782 page_index(page), 2783 PAGECACHE_TAG_DIRTY); 2784 if (!keep_write) 2785 radix_tree_tag_clear(&mapping->page_tree, 2786 page_index(page), 2787 PAGECACHE_TAG_TOWRITE); 2788 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2789 } else { 2790 ret = TestSetPageWriteback(page); 2791 } 2792 if (!ret) { 2793 mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_WRITEBACK); 2794 inc_zone_page_state(page, NR_WRITEBACK); 2795 } 2796 unlock_page_memcg(page); 2797 return ret; 2798 2799 } 2800 EXPORT_SYMBOL(__test_set_page_writeback); 2801 2802 /* 2803 * Return true if any of the pages in the mapping are marked with the 2804 * passed tag. 2805 */ 2806 int mapping_tagged(struct address_space *mapping, int tag) 2807 { 2808 return radix_tree_tagged(&mapping->page_tree, tag); 2809 } 2810 EXPORT_SYMBOL(mapping_tagged); 2811 2812 /** 2813 * wait_for_stable_page() - wait for writeback to finish, if necessary. 2814 * @page: The page to wait on. 2815 * 2816 * This function determines if the given page is related to a backing device 2817 * that requires page contents to be held stable during writeback. If so, then 2818 * it will wait for any pending writeback to complete. 2819 */ 2820 void wait_for_stable_page(struct page *page) 2821 { 2822 if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host))) 2823 wait_on_page_writeback(page); 2824 } 2825 EXPORT_SYMBOL_GPL(wait_for_stable_page); 2826