1 /* 2 * mm/page-writeback.c 3 * 4 * Copyright (C) 2002, Linus Torvalds. 5 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 6 * 7 * Contains functions related to writing back dirty pages at the 8 * address_space level. 9 * 10 * 10Apr2002 Andrew Morton 11 * Initial version 12 */ 13 14 #include <linux/kernel.h> 15 #include <linux/export.h> 16 #include <linux/spinlock.h> 17 #include <linux/fs.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 #include <linux/slab.h> 21 #include <linux/pagemap.h> 22 #include <linux/writeback.h> 23 #include <linux/init.h> 24 #include <linux/backing-dev.h> 25 #include <linux/task_io_accounting_ops.h> 26 #include <linux/blkdev.h> 27 #include <linux/mpage.h> 28 #include <linux/rmap.h> 29 #include <linux/percpu.h> 30 #include <linux/smp.h> 31 #include <linux/sysctl.h> 32 #include <linux/cpu.h> 33 #include <linux/syscalls.h> 34 #include <linux/buffer_head.h> /* __set_page_dirty_buffers */ 35 #include <linux/pagevec.h> 36 #include <linux/timer.h> 37 #include <linux/sched/rt.h> 38 #include <linux/sched/signal.h> 39 #include <linux/mm_inline.h> 40 #include <trace/events/writeback.h> 41 42 #include "internal.h" 43 44 /* 45 * Sleep at most 200ms at a time in balance_dirty_pages(). 46 */ 47 #define MAX_PAUSE max(HZ/5, 1) 48 49 /* 50 * Try to keep balance_dirty_pages() call intervals higher than this many pages 51 * by raising pause time to max_pause when falls below it. 52 */ 53 #define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10)) 54 55 /* 56 * Estimate write bandwidth at 200ms intervals. 57 */ 58 #define BANDWIDTH_INTERVAL max(HZ/5, 1) 59 60 #define RATELIMIT_CALC_SHIFT 10 61 62 /* 63 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited 64 * will look to see if it needs to force writeback or throttling. 65 */ 66 static long ratelimit_pages = 32; 67 68 /* The following parameters are exported via /proc/sys/vm */ 69 70 /* 71 * Start background writeback (via writeback threads) at this percentage 72 */ 73 int dirty_background_ratio = 10; 74 75 /* 76 * dirty_background_bytes starts at 0 (disabled) so that it is a function of 77 * dirty_background_ratio * the amount of dirtyable memory 78 */ 79 unsigned long dirty_background_bytes; 80 81 /* 82 * free highmem will not be subtracted from the total free memory 83 * for calculating free ratios if vm_highmem_is_dirtyable is true 84 */ 85 int vm_highmem_is_dirtyable; 86 87 /* 88 * The generator of dirty data starts writeback at this percentage 89 */ 90 int vm_dirty_ratio = 20; 91 92 /* 93 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of 94 * vm_dirty_ratio * the amount of dirtyable memory 95 */ 96 unsigned long vm_dirty_bytes; 97 98 /* 99 * The interval between `kupdate'-style writebacks 100 */ 101 unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ 102 103 EXPORT_SYMBOL_GPL(dirty_writeback_interval); 104 105 /* 106 * The longest time for which data is allowed to remain dirty 107 */ 108 unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */ 109 110 /* 111 * Flag that makes the machine dump writes/reads and block dirtyings. 112 */ 113 int block_dump; 114 115 /* 116 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies: 117 * a full sync is triggered after this time elapses without any disk activity. 118 */ 119 int laptop_mode; 120 121 EXPORT_SYMBOL(laptop_mode); 122 123 /* End of sysctl-exported parameters */ 124 125 struct wb_domain global_wb_domain; 126 127 /* consolidated parameters for balance_dirty_pages() and its subroutines */ 128 struct dirty_throttle_control { 129 #ifdef CONFIG_CGROUP_WRITEBACK 130 struct wb_domain *dom; 131 struct dirty_throttle_control *gdtc; /* only set in memcg dtc's */ 132 #endif 133 struct bdi_writeback *wb; 134 struct fprop_local_percpu *wb_completions; 135 136 unsigned long avail; /* dirtyable */ 137 unsigned long dirty; /* file_dirty + write + nfs */ 138 unsigned long thresh; /* dirty threshold */ 139 unsigned long bg_thresh; /* dirty background threshold */ 140 141 unsigned long wb_dirty; /* per-wb counterparts */ 142 unsigned long wb_thresh; 143 unsigned long wb_bg_thresh; 144 145 unsigned long pos_ratio; 146 }; 147 148 /* 149 * Length of period for aging writeout fractions of bdis. This is an 150 * arbitrarily chosen number. The longer the period, the slower fractions will 151 * reflect changes in current writeout rate. 152 */ 153 #define VM_COMPLETIONS_PERIOD_LEN (3*HZ) 154 155 #ifdef CONFIG_CGROUP_WRITEBACK 156 157 #define GDTC_INIT(__wb) .wb = (__wb), \ 158 .dom = &global_wb_domain, \ 159 .wb_completions = &(__wb)->completions 160 161 #define GDTC_INIT_NO_WB .dom = &global_wb_domain 162 163 #define MDTC_INIT(__wb, __gdtc) .wb = (__wb), \ 164 .dom = mem_cgroup_wb_domain(__wb), \ 165 .wb_completions = &(__wb)->memcg_completions, \ 166 .gdtc = __gdtc 167 168 static bool mdtc_valid(struct dirty_throttle_control *dtc) 169 { 170 return dtc->dom; 171 } 172 173 static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc) 174 { 175 return dtc->dom; 176 } 177 178 static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc) 179 { 180 return mdtc->gdtc; 181 } 182 183 static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb) 184 { 185 return &wb->memcg_completions; 186 } 187 188 static void wb_min_max_ratio(struct bdi_writeback *wb, 189 unsigned long *minp, unsigned long *maxp) 190 { 191 unsigned long this_bw = wb->avg_write_bandwidth; 192 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); 193 unsigned long long min = wb->bdi->min_ratio; 194 unsigned long long max = wb->bdi->max_ratio; 195 196 /* 197 * @wb may already be clean by the time control reaches here and 198 * the total may not include its bw. 199 */ 200 if (this_bw < tot_bw) { 201 if (min) { 202 min *= this_bw; 203 do_div(min, tot_bw); 204 } 205 if (max < 100) { 206 max *= this_bw; 207 do_div(max, tot_bw); 208 } 209 } 210 211 *minp = min; 212 *maxp = max; 213 } 214 215 #else /* CONFIG_CGROUP_WRITEBACK */ 216 217 #define GDTC_INIT(__wb) .wb = (__wb), \ 218 .wb_completions = &(__wb)->completions 219 #define GDTC_INIT_NO_WB 220 #define MDTC_INIT(__wb, __gdtc) 221 222 static bool mdtc_valid(struct dirty_throttle_control *dtc) 223 { 224 return false; 225 } 226 227 static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc) 228 { 229 return &global_wb_domain; 230 } 231 232 static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc) 233 { 234 return NULL; 235 } 236 237 static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb) 238 { 239 return NULL; 240 } 241 242 static void wb_min_max_ratio(struct bdi_writeback *wb, 243 unsigned long *minp, unsigned long *maxp) 244 { 245 *minp = wb->bdi->min_ratio; 246 *maxp = wb->bdi->max_ratio; 247 } 248 249 #endif /* CONFIG_CGROUP_WRITEBACK */ 250 251 /* 252 * In a memory zone, there is a certain amount of pages we consider 253 * available for the page cache, which is essentially the number of 254 * free and reclaimable pages, minus some zone reserves to protect 255 * lowmem and the ability to uphold the zone's watermarks without 256 * requiring writeback. 257 * 258 * This number of dirtyable pages is the base value of which the 259 * user-configurable dirty ratio is the effictive number of pages that 260 * are allowed to be actually dirtied. Per individual zone, or 261 * globally by using the sum of dirtyable pages over all zones. 262 * 263 * Because the user is allowed to specify the dirty limit globally as 264 * absolute number of bytes, calculating the per-zone dirty limit can 265 * require translating the configured limit into a percentage of 266 * global dirtyable memory first. 267 */ 268 269 /** 270 * node_dirtyable_memory - number of dirtyable pages in a node 271 * @pgdat: the node 272 * 273 * Returns the node's number of pages potentially available for dirty 274 * page cache. This is the base value for the per-node dirty limits. 275 */ 276 static unsigned long node_dirtyable_memory(struct pglist_data *pgdat) 277 { 278 unsigned long nr_pages = 0; 279 int z; 280 281 for (z = 0; z < MAX_NR_ZONES; z++) { 282 struct zone *zone = pgdat->node_zones + z; 283 284 if (!populated_zone(zone)) 285 continue; 286 287 nr_pages += zone_page_state(zone, NR_FREE_PAGES); 288 } 289 290 /* 291 * Pages reserved for the kernel should not be considered 292 * dirtyable, to prevent a situation where reclaim has to 293 * clean pages in order to balance the zones. 294 */ 295 nr_pages -= min(nr_pages, pgdat->totalreserve_pages); 296 297 nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE); 298 nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE); 299 300 return nr_pages; 301 } 302 303 static unsigned long highmem_dirtyable_memory(unsigned long total) 304 { 305 #ifdef CONFIG_HIGHMEM 306 int node; 307 unsigned long x = 0; 308 int i; 309 310 for_each_node_state(node, N_HIGH_MEMORY) { 311 for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) { 312 struct zone *z; 313 unsigned long nr_pages; 314 315 if (!is_highmem_idx(i)) 316 continue; 317 318 z = &NODE_DATA(node)->node_zones[i]; 319 if (!populated_zone(z)) 320 continue; 321 322 nr_pages = zone_page_state(z, NR_FREE_PAGES); 323 /* watch for underflows */ 324 nr_pages -= min(nr_pages, high_wmark_pages(z)); 325 nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE); 326 nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE); 327 x += nr_pages; 328 } 329 } 330 331 /* 332 * Unreclaimable memory (kernel memory or anonymous memory 333 * without swap) can bring down the dirtyable pages below 334 * the zone's dirty balance reserve and the above calculation 335 * will underflow. However we still want to add in nodes 336 * which are below threshold (negative values) to get a more 337 * accurate calculation but make sure that the total never 338 * underflows. 339 */ 340 if ((long)x < 0) 341 x = 0; 342 343 /* 344 * Make sure that the number of highmem pages is never larger 345 * than the number of the total dirtyable memory. This can only 346 * occur in very strange VM situations but we want to make sure 347 * that this does not occur. 348 */ 349 return min(x, total); 350 #else 351 return 0; 352 #endif 353 } 354 355 /** 356 * global_dirtyable_memory - number of globally dirtyable pages 357 * 358 * Returns the global number of pages potentially available for dirty 359 * page cache. This is the base value for the global dirty limits. 360 */ 361 static unsigned long global_dirtyable_memory(void) 362 { 363 unsigned long x; 364 365 x = global_zone_page_state(NR_FREE_PAGES); 366 /* 367 * Pages reserved for the kernel should not be considered 368 * dirtyable, to prevent a situation where reclaim has to 369 * clean pages in order to balance the zones. 370 */ 371 x -= min(x, totalreserve_pages); 372 373 x += global_node_page_state(NR_INACTIVE_FILE); 374 x += global_node_page_state(NR_ACTIVE_FILE); 375 376 if (!vm_highmem_is_dirtyable) 377 x -= highmem_dirtyable_memory(x); 378 379 return x + 1; /* Ensure that we never return 0 */ 380 } 381 382 /** 383 * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain 384 * @dtc: dirty_throttle_control of interest 385 * 386 * Calculate @dtc->thresh and ->bg_thresh considering 387 * vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}. The caller 388 * must ensure that @dtc->avail is set before calling this function. The 389 * dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and 390 * real-time tasks. 391 */ 392 static void domain_dirty_limits(struct dirty_throttle_control *dtc) 393 { 394 const unsigned long available_memory = dtc->avail; 395 struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc); 396 unsigned long bytes = vm_dirty_bytes; 397 unsigned long bg_bytes = dirty_background_bytes; 398 /* convert ratios to per-PAGE_SIZE for higher precision */ 399 unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100; 400 unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100; 401 unsigned long thresh; 402 unsigned long bg_thresh; 403 struct task_struct *tsk; 404 405 /* gdtc is !NULL iff @dtc is for memcg domain */ 406 if (gdtc) { 407 unsigned long global_avail = gdtc->avail; 408 409 /* 410 * The byte settings can't be applied directly to memcg 411 * domains. Convert them to ratios by scaling against 412 * globally available memory. As the ratios are in 413 * per-PAGE_SIZE, they can be obtained by dividing bytes by 414 * number of pages. 415 */ 416 if (bytes) 417 ratio = min(DIV_ROUND_UP(bytes, global_avail), 418 PAGE_SIZE); 419 if (bg_bytes) 420 bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail), 421 PAGE_SIZE); 422 bytes = bg_bytes = 0; 423 } 424 425 if (bytes) 426 thresh = DIV_ROUND_UP(bytes, PAGE_SIZE); 427 else 428 thresh = (ratio * available_memory) / PAGE_SIZE; 429 430 if (bg_bytes) 431 bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE); 432 else 433 bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE; 434 435 if (bg_thresh >= thresh) 436 bg_thresh = thresh / 2; 437 tsk = current; 438 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) { 439 bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32; 440 thresh += thresh / 4 + global_wb_domain.dirty_limit / 32; 441 } 442 dtc->thresh = thresh; 443 dtc->bg_thresh = bg_thresh; 444 445 /* we should eventually report the domain in the TP */ 446 if (!gdtc) 447 trace_global_dirty_state(bg_thresh, thresh); 448 } 449 450 /** 451 * global_dirty_limits - background-writeback and dirty-throttling thresholds 452 * @pbackground: out parameter for bg_thresh 453 * @pdirty: out parameter for thresh 454 * 455 * Calculate bg_thresh and thresh for global_wb_domain. See 456 * domain_dirty_limits() for details. 457 */ 458 void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty) 459 { 460 struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB }; 461 462 gdtc.avail = global_dirtyable_memory(); 463 domain_dirty_limits(&gdtc); 464 465 *pbackground = gdtc.bg_thresh; 466 *pdirty = gdtc.thresh; 467 } 468 469 /** 470 * node_dirty_limit - maximum number of dirty pages allowed in a node 471 * @pgdat: the node 472 * 473 * Returns the maximum number of dirty pages allowed in a node, based 474 * on the node's dirtyable memory. 475 */ 476 static unsigned long node_dirty_limit(struct pglist_data *pgdat) 477 { 478 unsigned long node_memory = node_dirtyable_memory(pgdat); 479 struct task_struct *tsk = current; 480 unsigned long dirty; 481 482 if (vm_dirty_bytes) 483 dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) * 484 node_memory / global_dirtyable_memory(); 485 else 486 dirty = vm_dirty_ratio * node_memory / 100; 487 488 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) 489 dirty += dirty / 4; 490 491 return dirty; 492 } 493 494 /** 495 * node_dirty_ok - tells whether a node is within its dirty limits 496 * @pgdat: the node to check 497 * 498 * Returns %true when the dirty pages in @pgdat are within the node's 499 * dirty limit, %false if the limit is exceeded. 500 */ 501 bool node_dirty_ok(struct pglist_data *pgdat) 502 { 503 unsigned long limit = node_dirty_limit(pgdat); 504 unsigned long nr_pages = 0; 505 506 nr_pages += node_page_state(pgdat, NR_FILE_DIRTY); 507 nr_pages += node_page_state(pgdat, NR_UNSTABLE_NFS); 508 nr_pages += node_page_state(pgdat, NR_WRITEBACK); 509 510 return nr_pages <= limit; 511 } 512 513 int dirty_background_ratio_handler(struct ctl_table *table, int write, 514 void __user *buffer, size_t *lenp, 515 loff_t *ppos) 516 { 517 int ret; 518 519 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 520 if (ret == 0 && write) 521 dirty_background_bytes = 0; 522 return ret; 523 } 524 525 int dirty_background_bytes_handler(struct ctl_table *table, int write, 526 void __user *buffer, size_t *lenp, 527 loff_t *ppos) 528 { 529 int ret; 530 531 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 532 if (ret == 0 && write) 533 dirty_background_ratio = 0; 534 return ret; 535 } 536 537 int dirty_ratio_handler(struct ctl_table *table, int write, 538 void __user *buffer, size_t *lenp, 539 loff_t *ppos) 540 { 541 int old_ratio = vm_dirty_ratio; 542 int ret; 543 544 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 545 if (ret == 0 && write && vm_dirty_ratio != old_ratio) { 546 writeback_set_ratelimit(); 547 vm_dirty_bytes = 0; 548 } 549 return ret; 550 } 551 552 int dirty_bytes_handler(struct ctl_table *table, int write, 553 void __user *buffer, size_t *lenp, 554 loff_t *ppos) 555 { 556 unsigned long old_bytes = vm_dirty_bytes; 557 int ret; 558 559 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 560 if (ret == 0 && write && vm_dirty_bytes != old_bytes) { 561 writeback_set_ratelimit(); 562 vm_dirty_ratio = 0; 563 } 564 return ret; 565 } 566 567 static unsigned long wp_next_time(unsigned long cur_time) 568 { 569 cur_time += VM_COMPLETIONS_PERIOD_LEN; 570 /* 0 has a special meaning... */ 571 if (!cur_time) 572 return 1; 573 return cur_time; 574 } 575 576 static void wb_domain_writeout_inc(struct wb_domain *dom, 577 struct fprop_local_percpu *completions, 578 unsigned int max_prop_frac) 579 { 580 __fprop_inc_percpu_max(&dom->completions, completions, 581 max_prop_frac); 582 /* First event after period switching was turned off? */ 583 if (unlikely(!dom->period_time)) { 584 /* 585 * We can race with other __bdi_writeout_inc calls here but 586 * it does not cause any harm since the resulting time when 587 * timer will fire and what is in writeout_period_time will be 588 * roughly the same. 589 */ 590 dom->period_time = wp_next_time(jiffies); 591 mod_timer(&dom->period_timer, dom->period_time); 592 } 593 } 594 595 /* 596 * Increment @wb's writeout completion count and the global writeout 597 * completion count. Called from test_clear_page_writeback(). 598 */ 599 static inline void __wb_writeout_inc(struct bdi_writeback *wb) 600 { 601 struct wb_domain *cgdom; 602 603 inc_wb_stat(wb, WB_WRITTEN); 604 wb_domain_writeout_inc(&global_wb_domain, &wb->completions, 605 wb->bdi->max_prop_frac); 606 607 cgdom = mem_cgroup_wb_domain(wb); 608 if (cgdom) 609 wb_domain_writeout_inc(cgdom, wb_memcg_completions(wb), 610 wb->bdi->max_prop_frac); 611 } 612 613 void wb_writeout_inc(struct bdi_writeback *wb) 614 { 615 unsigned long flags; 616 617 local_irq_save(flags); 618 __wb_writeout_inc(wb); 619 local_irq_restore(flags); 620 } 621 EXPORT_SYMBOL_GPL(wb_writeout_inc); 622 623 /* 624 * On idle system, we can be called long after we scheduled because we use 625 * deferred timers so count with missed periods. 626 */ 627 static void writeout_period(struct timer_list *t) 628 { 629 struct wb_domain *dom = from_timer(dom, t, period_timer); 630 int miss_periods = (jiffies - dom->period_time) / 631 VM_COMPLETIONS_PERIOD_LEN; 632 633 if (fprop_new_period(&dom->completions, miss_periods + 1)) { 634 dom->period_time = wp_next_time(dom->period_time + 635 miss_periods * VM_COMPLETIONS_PERIOD_LEN); 636 mod_timer(&dom->period_timer, dom->period_time); 637 } else { 638 /* 639 * Aging has zeroed all fractions. Stop wasting CPU on period 640 * updates. 641 */ 642 dom->period_time = 0; 643 } 644 } 645 646 int wb_domain_init(struct wb_domain *dom, gfp_t gfp) 647 { 648 memset(dom, 0, sizeof(*dom)); 649 650 spin_lock_init(&dom->lock); 651 652 timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE); 653 654 dom->dirty_limit_tstamp = jiffies; 655 656 return fprop_global_init(&dom->completions, gfp); 657 } 658 659 #ifdef CONFIG_CGROUP_WRITEBACK 660 void wb_domain_exit(struct wb_domain *dom) 661 { 662 del_timer_sync(&dom->period_timer); 663 fprop_global_destroy(&dom->completions); 664 } 665 #endif 666 667 /* 668 * bdi_min_ratio keeps the sum of the minimum dirty shares of all 669 * registered backing devices, which, for obvious reasons, can not 670 * exceed 100%. 671 */ 672 static unsigned int bdi_min_ratio; 673 674 int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio) 675 { 676 int ret = 0; 677 678 spin_lock_bh(&bdi_lock); 679 if (min_ratio > bdi->max_ratio) { 680 ret = -EINVAL; 681 } else { 682 min_ratio -= bdi->min_ratio; 683 if (bdi_min_ratio + min_ratio < 100) { 684 bdi_min_ratio += min_ratio; 685 bdi->min_ratio += min_ratio; 686 } else { 687 ret = -EINVAL; 688 } 689 } 690 spin_unlock_bh(&bdi_lock); 691 692 return ret; 693 } 694 695 int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio) 696 { 697 int ret = 0; 698 699 if (max_ratio > 100) 700 return -EINVAL; 701 702 spin_lock_bh(&bdi_lock); 703 if (bdi->min_ratio > max_ratio) { 704 ret = -EINVAL; 705 } else { 706 bdi->max_ratio = max_ratio; 707 bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100; 708 } 709 spin_unlock_bh(&bdi_lock); 710 711 return ret; 712 } 713 EXPORT_SYMBOL(bdi_set_max_ratio); 714 715 static unsigned long dirty_freerun_ceiling(unsigned long thresh, 716 unsigned long bg_thresh) 717 { 718 return (thresh + bg_thresh) / 2; 719 } 720 721 static unsigned long hard_dirty_limit(struct wb_domain *dom, 722 unsigned long thresh) 723 { 724 return max(thresh, dom->dirty_limit); 725 } 726 727 /* 728 * Memory which can be further allocated to a memcg domain is capped by 729 * system-wide clean memory excluding the amount being used in the domain. 730 */ 731 static void mdtc_calc_avail(struct dirty_throttle_control *mdtc, 732 unsigned long filepages, unsigned long headroom) 733 { 734 struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc); 735 unsigned long clean = filepages - min(filepages, mdtc->dirty); 736 unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty); 737 unsigned long other_clean = global_clean - min(global_clean, clean); 738 739 mdtc->avail = filepages + min(headroom, other_clean); 740 } 741 742 /** 743 * __wb_calc_thresh - @wb's share of dirty throttling threshold 744 * @dtc: dirty_throttle_context of interest 745 * 746 * Returns @wb's dirty limit in pages. The term "dirty" in the context of 747 * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages. 748 * 749 * Note that balance_dirty_pages() will only seriously take it as a hard limit 750 * when sleeping max_pause per page is not enough to keep the dirty pages under 751 * control. For example, when the device is completely stalled due to some error 752 * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key. 753 * In the other normal situations, it acts more gently by throttling the tasks 754 * more (rather than completely block them) when the wb dirty pages go high. 755 * 756 * It allocates high/low dirty limits to fast/slow devices, in order to prevent 757 * - starving fast devices 758 * - piling up dirty pages (that will take long time to sync) on slow devices 759 * 760 * The wb's share of dirty limit will be adapting to its throughput and 761 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set. 762 */ 763 static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc) 764 { 765 struct wb_domain *dom = dtc_dom(dtc); 766 unsigned long thresh = dtc->thresh; 767 u64 wb_thresh; 768 long numerator, denominator; 769 unsigned long wb_min_ratio, wb_max_ratio; 770 771 /* 772 * Calculate this BDI's share of the thresh ratio. 773 */ 774 fprop_fraction_percpu(&dom->completions, dtc->wb_completions, 775 &numerator, &denominator); 776 777 wb_thresh = (thresh * (100 - bdi_min_ratio)) / 100; 778 wb_thresh *= numerator; 779 do_div(wb_thresh, denominator); 780 781 wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio); 782 783 wb_thresh += (thresh * wb_min_ratio) / 100; 784 if (wb_thresh > (thresh * wb_max_ratio) / 100) 785 wb_thresh = thresh * wb_max_ratio / 100; 786 787 return wb_thresh; 788 } 789 790 unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh) 791 { 792 struct dirty_throttle_control gdtc = { GDTC_INIT(wb), 793 .thresh = thresh }; 794 return __wb_calc_thresh(&gdtc); 795 } 796 797 /* 798 * setpoint - dirty 3 799 * f(dirty) := 1.0 + (----------------) 800 * limit - setpoint 801 * 802 * it's a 3rd order polynomial that subjects to 803 * 804 * (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast 805 * (2) f(setpoint) = 1.0 => the balance point 806 * (3) f(limit) = 0 => the hard limit 807 * (4) df/dx <= 0 => negative feedback control 808 * (5) the closer to setpoint, the smaller |df/dx| (and the reverse) 809 * => fast response on large errors; small oscillation near setpoint 810 */ 811 static long long pos_ratio_polynom(unsigned long setpoint, 812 unsigned long dirty, 813 unsigned long limit) 814 { 815 long long pos_ratio; 816 long x; 817 818 x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT, 819 (limit - setpoint) | 1); 820 pos_ratio = x; 821 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; 822 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; 823 pos_ratio += 1 << RATELIMIT_CALC_SHIFT; 824 825 return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT); 826 } 827 828 /* 829 * Dirty position control. 830 * 831 * (o) global/bdi setpoints 832 * 833 * We want the dirty pages be balanced around the global/wb setpoints. 834 * When the number of dirty pages is higher/lower than the setpoint, the 835 * dirty position control ratio (and hence task dirty ratelimit) will be 836 * decreased/increased to bring the dirty pages back to the setpoint. 837 * 838 * pos_ratio = 1 << RATELIMIT_CALC_SHIFT 839 * 840 * if (dirty < setpoint) scale up pos_ratio 841 * if (dirty > setpoint) scale down pos_ratio 842 * 843 * if (wb_dirty < wb_setpoint) scale up pos_ratio 844 * if (wb_dirty > wb_setpoint) scale down pos_ratio 845 * 846 * task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT 847 * 848 * (o) global control line 849 * 850 * ^ pos_ratio 851 * | 852 * | |<===== global dirty control scope ======>| 853 * 2.0 .............* 854 * | .* 855 * | . * 856 * | . * 857 * | . * 858 * | . * 859 * | . * 860 * 1.0 ................................* 861 * | . . * 862 * | . . * 863 * | . . * 864 * | . . * 865 * | . . * 866 * 0 +------------.------------------.----------------------*-------------> 867 * freerun^ setpoint^ limit^ dirty pages 868 * 869 * (o) wb control line 870 * 871 * ^ pos_ratio 872 * | 873 * | * 874 * | * 875 * | * 876 * | * 877 * | * |<=========== span ============>| 878 * 1.0 .......................* 879 * | . * 880 * | . * 881 * | . * 882 * | . * 883 * | . * 884 * | . * 885 * | . * 886 * | . * 887 * | . * 888 * | . * 889 * | . * 890 * 1/4 ...............................................* * * * * * * * * * * * 891 * | . . 892 * | . . 893 * | . . 894 * 0 +----------------------.-------------------------------.-------------> 895 * wb_setpoint^ x_intercept^ 896 * 897 * The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can 898 * be smoothly throttled down to normal if it starts high in situations like 899 * - start writing to a slow SD card and a fast disk at the same time. The SD 900 * card's wb_dirty may rush to many times higher than wb_setpoint. 901 * - the wb dirty thresh drops quickly due to change of JBOD workload 902 */ 903 static void wb_position_ratio(struct dirty_throttle_control *dtc) 904 { 905 struct bdi_writeback *wb = dtc->wb; 906 unsigned long write_bw = wb->avg_write_bandwidth; 907 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); 908 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); 909 unsigned long wb_thresh = dtc->wb_thresh; 910 unsigned long x_intercept; 911 unsigned long setpoint; /* dirty pages' target balance point */ 912 unsigned long wb_setpoint; 913 unsigned long span; 914 long long pos_ratio; /* for scaling up/down the rate limit */ 915 long x; 916 917 dtc->pos_ratio = 0; 918 919 if (unlikely(dtc->dirty >= limit)) 920 return; 921 922 /* 923 * global setpoint 924 * 925 * See comment for pos_ratio_polynom(). 926 */ 927 setpoint = (freerun + limit) / 2; 928 pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit); 929 930 /* 931 * The strictlimit feature is a tool preventing mistrusted filesystems 932 * from growing a large number of dirty pages before throttling. For 933 * such filesystems balance_dirty_pages always checks wb counters 934 * against wb limits. Even if global "nr_dirty" is under "freerun". 935 * This is especially important for fuse which sets bdi->max_ratio to 936 * 1% by default. Without strictlimit feature, fuse writeback may 937 * consume arbitrary amount of RAM because it is accounted in 938 * NR_WRITEBACK_TEMP which is not involved in calculating "nr_dirty". 939 * 940 * Here, in wb_position_ratio(), we calculate pos_ratio based on 941 * two values: wb_dirty and wb_thresh. Let's consider an example: 942 * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global 943 * limits are set by default to 10% and 20% (background and throttle). 944 * Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages. 945 * wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is 946 * about ~6K pages (as the average of background and throttle wb 947 * limits). The 3rd order polynomial will provide positive feedback if 948 * wb_dirty is under wb_setpoint and vice versa. 949 * 950 * Note, that we cannot use global counters in these calculations 951 * because we want to throttle process writing to a strictlimit wb 952 * much earlier than global "freerun" is reached (~23MB vs. ~2.3GB 953 * in the example above). 954 */ 955 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { 956 long long wb_pos_ratio; 957 958 if (dtc->wb_dirty < 8) { 959 dtc->pos_ratio = min_t(long long, pos_ratio * 2, 960 2 << RATELIMIT_CALC_SHIFT); 961 return; 962 } 963 964 if (dtc->wb_dirty >= wb_thresh) 965 return; 966 967 wb_setpoint = dirty_freerun_ceiling(wb_thresh, 968 dtc->wb_bg_thresh); 969 970 if (wb_setpoint == 0 || wb_setpoint == wb_thresh) 971 return; 972 973 wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty, 974 wb_thresh); 975 976 /* 977 * Typically, for strictlimit case, wb_setpoint << setpoint 978 * and pos_ratio >> wb_pos_ratio. In the other words global 979 * state ("dirty") is not limiting factor and we have to 980 * make decision based on wb counters. But there is an 981 * important case when global pos_ratio should get precedence: 982 * global limits are exceeded (e.g. due to activities on other 983 * wb's) while given strictlimit wb is below limit. 984 * 985 * "pos_ratio * wb_pos_ratio" would work for the case above, 986 * but it would look too non-natural for the case of all 987 * activity in the system coming from a single strictlimit wb 988 * with bdi->max_ratio == 100%. 989 * 990 * Note that min() below somewhat changes the dynamics of the 991 * control system. Normally, pos_ratio value can be well over 3 992 * (when globally we are at freerun and wb is well below wb 993 * setpoint). Now the maximum pos_ratio in the same situation 994 * is 2. We might want to tweak this if we observe the control 995 * system is too slow to adapt. 996 */ 997 dtc->pos_ratio = min(pos_ratio, wb_pos_ratio); 998 return; 999 } 1000 1001 /* 1002 * We have computed basic pos_ratio above based on global situation. If 1003 * the wb is over/under its share of dirty pages, we want to scale 1004 * pos_ratio further down/up. That is done by the following mechanism. 1005 */ 1006 1007 /* 1008 * wb setpoint 1009 * 1010 * f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint) 1011 * 1012 * x_intercept - wb_dirty 1013 * := -------------------------- 1014 * x_intercept - wb_setpoint 1015 * 1016 * The main wb control line is a linear function that subjects to 1017 * 1018 * (1) f(wb_setpoint) = 1.0 1019 * (2) k = - 1 / (8 * write_bw) (in single wb case) 1020 * or equally: x_intercept = wb_setpoint + 8 * write_bw 1021 * 1022 * For single wb case, the dirty pages are observed to fluctuate 1023 * regularly within range 1024 * [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2] 1025 * for various filesystems, where (2) can yield in a reasonable 12.5% 1026 * fluctuation range for pos_ratio. 1027 * 1028 * For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its 1029 * own size, so move the slope over accordingly and choose a slope that 1030 * yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh. 1031 */ 1032 if (unlikely(wb_thresh > dtc->thresh)) 1033 wb_thresh = dtc->thresh; 1034 /* 1035 * It's very possible that wb_thresh is close to 0 not because the 1036 * device is slow, but that it has remained inactive for long time. 1037 * Honour such devices a reasonable good (hopefully IO efficient) 1038 * threshold, so that the occasional writes won't be blocked and active 1039 * writes can rampup the threshold quickly. 1040 */ 1041 wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8); 1042 /* 1043 * scale global setpoint to wb's: 1044 * wb_setpoint = setpoint * wb_thresh / thresh 1045 */ 1046 x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1); 1047 wb_setpoint = setpoint * (u64)x >> 16; 1048 /* 1049 * Use span=(8*write_bw) in single wb case as indicated by 1050 * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case. 1051 * 1052 * wb_thresh thresh - wb_thresh 1053 * span = --------- * (8 * write_bw) + ------------------ * wb_thresh 1054 * thresh thresh 1055 */ 1056 span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16; 1057 x_intercept = wb_setpoint + span; 1058 1059 if (dtc->wb_dirty < x_intercept - span / 4) { 1060 pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty), 1061 (x_intercept - wb_setpoint) | 1); 1062 } else 1063 pos_ratio /= 4; 1064 1065 /* 1066 * wb reserve area, safeguard against dirty pool underrun and disk idle 1067 * It may push the desired control point of global dirty pages higher 1068 * than setpoint. 1069 */ 1070 x_intercept = wb_thresh / 2; 1071 if (dtc->wb_dirty < x_intercept) { 1072 if (dtc->wb_dirty > x_intercept / 8) 1073 pos_ratio = div_u64(pos_ratio * x_intercept, 1074 dtc->wb_dirty); 1075 else 1076 pos_ratio *= 8; 1077 } 1078 1079 dtc->pos_ratio = pos_ratio; 1080 } 1081 1082 static void wb_update_write_bandwidth(struct bdi_writeback *wb, 1083 unsigned long elapsed, 1084 unsigned long written) 1085 { 1086 const unsigned long period = roundup_pow_of_two(3 * HZ); 1087 unsigned long avg = wb->avg_write_bandwidth; 1088 unsigned long old = wb->write_bandwidth; 1089 u64 bw; 1090 1091 /* 1092 * bw = written * HZ / elapsed 1093 * 1094 * bw * elapsed + write_bandwidth * (period - elapsed) 1095 * write_bandwidth = --------------------------------------------------- 1096 * period 1097 * 1098 * @written may have decreased due to account_page_redirty(). 1099 * Avoid underflowing @bw calculation. 1100 */ 1101 bw = written - min(written, wb->written_stamp); 1102 bw *= HZ; 1103 if (unlikely(elapsed > period)) { 1104 do_div(bw, elapsed); 1105 avg = bw; 1106 goto out; 1107 } 1108 bw += (u64)wb->write_bandwidth * (period - elapsed); 1109 bw >>= ilog2(period); 1110 1111 /* 1112 * one more level of smoothing, for filtering out sudden spikes 1113 */ 1114 if (avg > old && old >= (unsigned long)bw) 1115 avg -= (avg - old) >> 3; 1116 1117 if (avg < old && old <= (unsigned long)bw) 1118 avg += (old - avg) >> 3; 1119 1120 out: 1121 /* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */ 1122 avg = max(avg, 1LU); 1123 if (wb_has_dirty_io(wb)) { 1124 long delta = avg - wb->avg_write_bandwidth; 1125 WARN_ON_ONCE(atomic_long_add_return(delta, 1126 &wb->bdi->tot_write_bandwidth) <= 0); 1127 } 1128 wb->write_bandwidth = bw; 1129 wb->avg_write_bandwidth = avg; 1130 } 1131 1132 static void update_dirty_limit(struct dirty_throttle_control *dtc) 1133 { 1134 struct wb_domain *dom = dtc_dom(dtc); 1135 unsigned long thresh = dtc->thresh; 1136 unsigned long limit = dom->dirty_limit; 1137 1138 /* 1139 * Follow up in one step. 1140 */ 1141 if (limit < thresh) { 1142 limit = thresh; 1143 goto update; 1144 } 1145 1146 /* 1147 * Follow down slowly. Use the higher one as the target, because thresh 1148 * may drop below dirty. This is exactly the reason to introduce 1149 * dom->dirty_limit which is guaranteed to lie above the dirty pages. 1150 */ 1151 thresh = max(thresh, dtc->dirty); 1152 if (limit > thresh) { 1153 limit -= (limit - thresh) >> 5; 1154 goto update; 1155 } 1156 return; 1157 update: 1158 dom->dirty_limit = limit; 1159 } 1160 1161 static void domain_update_bandwidth(struct dirty_throttle_control *dtc, 1162 unsigned long now) 1163 { 1164 struct wb_domain *dom = dtc_dom(dtc); 1165 1166 /* 1167 * check locklessly first to optimize away locking for the most time 1168 */ 1169 if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) 1170 return; 1171 1172 spin_lock(&dom->lock); 1173 if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) { 1174 update_dirty_limit(dtc); 1175 dom->dirty_limit_tstamp = now; 1176 } 1177 spin_unlock(&dom->lock); 1178 } 1179 1180 /* 1181 * Maintain wb->dirty_ratelimit, the base dirty throttle rate. 1182 * 1183 * Normal wb tasks will be curbed at or below it in long term. 1184 * Obviously it should be around (write_bw / N) when there are N dd tasks. 1185 */ 1186 static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc, 1187 unsigned long dirtied, 1188 unsigned long elapsed) 1189 { 1190 struct bdi_writeback *wb = dtc->wb; 1191 unsigned long dirty = dtc->dirty; 1192 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); 1193 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); 1194 unsigned long setpoint = (freerun + limit) / 2; 1195 unsigned long write_bw = wb->avg_write_bandwidth; 1196 unsigned long dirty_ratelimit = wb->dirty_ratelimit; 1197 unsigned long dirty_rate; 1198 unsigned long task_ratelimit; 1199 unsigned long balanced_dirty_ratelimit; 1200 unsigned long step; 1201 unsigned long x; 1202 unsigned long shift; 1203 1204 /* 1205 * The dirty rate will match the writeout rate in long term, except 1206 * when dirty pages are truncated by userspace or re-dirtied by FS. 1207 */ 1208 dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed; 1209 1210 /* 1211 * task_ratelimit reflects each dd's dirty rate for the past 200ms. 1212 */ 1213 task_ratelimit = (u64)dirty_ratelimit * 1214 dtc->pos_ratio >> RATELIMIT_CALC_SHIFT; 1215 task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */ 1216 1217 /* 1218 * A linear estimation of the "balanced" throttle rate. The theory is, 1219 * if there are N dd tasks, each throttled at task_ratelimit, the wb's 1220 * dirty_rate will be measured to be (N * task_ratelimit). So the below 1221 * formula will yield the balanced rate limit (write_bw / N). 1222 * 1223 * Note that the expanded form is not a pure rate feedback: 1224 * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1) 1225 * but also takes pos_ratio into account: 1226 * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2) 1227 * 1228 * (1) is not realistic because pos_ratio also takes part in balancing 1229 * the dirty rate. Consider the state 1230 * pos_ratio = 0.5 (3) 1231 * rate = 2 * (write_bw / N) (4) 1232 * If (1) is used, it will stuck in that state! Because each dd will 1233 * be throttled at 1234 * task_ratelimit = pos_ratio * rate = (write_bw / N) (5) 1235 * yielding 1236 * dirty_rate = N * task_ratelimit = write_bw (6) 1237 * put (6) into (1) we get 1238 * rate_(i+1) = rate_(i) (7) 1239 * 1240 * So we end up using (2) to always keep 1241 * rate_(i+1) ~= (write_bw / N) (8) 1242 * regardless of the value of pos_ratio. As long as (8) is satisfied, 1243 * pos_ratio is able to drive itself to 1.0, which is not only where 1244 * the dirty count meet the setpoint, but also where the slope of 1245 * pos_ratio is most flat and hence task_ratelimit is least fluctuated. 1246 */ 1247 balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw, 1248 dirty_rate | 1); 1249 /* 1250 * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw 1251 */ 1252 if (unlikely(balanced_dirty_ratelimit > write_bw)) 1253 balanced_dirty_ratelimit = write_bw; 1254 1255 /* 1256 * We could safely do this and return immediately: 1257 * 1258 * wb->dirty_ratelimit = balanced_dirty_ratelimit; 1259 * 1260 * However to get a more stable dirty_ratelimit, the below elaborated 1261 * code makes use of task_ratelimit to filter out singular points and 1262 * limit the step size. 1263 * 1264 * The below code essentially only uses the relative value of 1265 * 1266 * task_ratelimit - dirty_ratelimit 1267 * = (pos_ratio - 1) * dirty_ratelimit 1268 * 1269 * which reflects the direction and size of dirty position error. 1270 */ 1271 1272 /* 1273 * dirty_ratelimit will follow balanced_dirty_ratelimit iff 1274 * task_ratelimit is on the same side of dirty_ratelimit, too. 1275 * For example, when 1276 * - dirty_ratelimit > balanced_dirty_ratelimit 1277 * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint) 1278 * lowering dirty_ratelimit will help meet both the position and rate 1279 * control targets. Otherwise, don't update dirty_ratelimit if it will 1280 * only help meet the rate target. After all, what the users ultimately 1281 * feel and care are stable dirty rate and small position error. 1282 * 1283 * |task_ratelimit - dirty_ratelimit| is used to limit the step size 1284 * and filter out the singular points of balanced_dirty_ratelimit. Which 1285 * keeps jumping around randomly and can even leap far away at times 1286 * due to the small 200ms estimation period of dirty_rate (we want to 1287 * keep that period small to reduce time lags). 1288 */ 1289 step = 0; 1290 1291 /* 1292 * For strictlimit case, calculations above were based on wb counters 1293 * and limits (starting from pos_ratio = wb_position_ratio() and up to 1294 * balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate). 1295 * Hence, to calculate "step" properly, we have to use wb_dirty as 1296 * "dirty" and wb_setpoint as "setpoint". 1297 * 1298 * We rampup dirty_ratelimit forcibly if wb_dirty is low because 1299 * it's possible that wb_thresh is close to zero due to inactivity 1300 * of backing device. 1301 */ 1302 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { 1303 dirty = dtc->wb_dirty; 1304 if (dtc->wb_dirty < 8) 1305 setpoint = dtc->wb_dirty + 1; 1306 else 1307 setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2; 1308 } 1309 1310 if (dirty < setpoint) { 1311 x = min3(wb->balanced_dirty_ratelimit, 1312 balanced_dirty_ratelimit, task_ratelimit); 1313 if (dirty_ratelimit < x) 1314 step = x - dirty_ratelimit; 1315 } else { 1316 x = max3(wb->balanced_dirty_ratelimit, 1317 balanced_dirty_ratelimit, task_ratelimit); 1318 if (dirty_ratelimit > x) 1319 step = dirty_ratelimit - x; 1320 } 1321 1322 /* 1323 * Don't pursue 100% rate matching. It's impossible since the balanced 1324 * rate itself is constantly fluctuating. So decrease the track speed 1325 * when it gets close to the target. Helps eliminate pointless tremors. 1326 */ 1327 shift = dirty_ratelimit / (2 * step + 1); 1328 if (shift < BITS_PER_LONG) 1329 step = DIV_ROUND_UP(step >> shift, 8); 1330 else 1331 step = 0; 1332 1333 if (dirty_ratelimit < balanced_dirty_ratelimit) 1334 dirty_ratelimit += step; 1335 else 1336 dirty_ratelimit -= step; 1337 1338 wb->dirty_ratelimit = max(dirty_ratelimit, 1UL); 1339 wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit; 1340 1341 trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit); 1342 } 1343 1344 static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc, 1345 struct dirty_throttle_control *mdtc, 1346 unsigned long start_time, 1347 bool update_ratelimit) 1348 { 1349 struct bdi_writeback *wb = gdtc->wb; 1350 unsigned long now = jiffies; 1351 unsigned long elapsed = now - wb->bw_time_stamp; 1352 unsigned long dirtied; 1353 unsigned long written; 1354 1355 lockdep_assert_held(&wb->list_lock); 1356 1357 /* 1358 * rate-limit, only update once every 200ms. 1359 */ 1360 if (elapsed < BANDWIDTH_INTERVAL) 1361 return; 1362 1363 dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]); 1364 written = percpu_counter_read(&wb->stat[WB_WRITTEN]); 1365 1366 /* 1367 * Skip quiet periods when disk bandwidth is under-utilized. 1368 * (at least 1s idle time between two flusher runs) 1369 */ 1370 if (elapsed > HZ && time_before(wb->bw_time_stamp, start_time)) 1371 goto snapshot; 1372 1373 if (update_ratelimit) { 1374 domain_update_bandwidth(gdtc, now); 1375 wb_update_dirty_ratelimit(gdtc, dirtied, elapsed); 1376 1377 /* 1378 * @mdtc is always NULL if !CGROUP_WRITEBACK but the 1379 * compiler has no way to figure that out. Help it. 1380 */ 1381 if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) { 1382 domain_update_bandwidth(mdtc, now); 1383 wb_update_dirty_ratelimit(mdtc, dirtied, elapsed); 1384 } 1385 } 1386 wb_update_write_bandwidth(wb, elapsed, written); 1387 1388 snapshot: 1389 wb->dirtied_stamp = dirtied; 1390 wb->written_stamp = written; 1391 wb->bw_time_stamp = now; 1392 } 1393 1394 void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time) 1395 { 1396 struct dirty_throttle_control gdtc = { GDTC_INIT(wb) }; 1397 1398 __wb_update_bandwidth(&gdtc, NULL, start_time, false); 1399 } 1400 1401 /* 1402 * After a task dirtied this many pages, balance_dirty_pages_ratelimited() 1403 * will look to see if it needs to start dirty throttling. 1404 * 1405 * If dirty_poll_interval is too low, big NUMA machines will call the expensive 1406 * global_zone_page_state() too often. So scale it near-sqrt to the safety margin 1407 * (the number of pages we may dirty without exceeding the dirty limits). 1408 */ 1409 static unsigned long dirty_poll_interval(unsigned long dirty, 1410 unsigned long thresh) 1411 { 1412 if (thresh > dirty) 1413 return 1UL << (ilog2(thresh - dirty) >> 1); 1414 1415 return 1; 1416 } 1417 1418 static unsigned long wb_max_pause(struct bdi_writeback *wb, 1419 unsigned long wb_dirty) 1420 { 1421 unsigned long bw = wb->avg_write_bandwidth; 1422 unsigned long t; 1423 1424 /* 1425 * Limit pause time for small memory systems. If sleeping for too long 1426 * time, a small pool of dirty/writeback pages may go empty and disk go 1427 * idle. 1428 * 1429 * 8 serves as the safety ratio. 1430 */ 1431 t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8)); 1432 t++; 1433 1434 return min_t(unsigned long, t, MAX_PAUSE); 1435 } 1436 1437 static long wb_min_pause(struct bdi_writeback *wb, 1438 long max_pause, 1439 unsigned long task_ratelimit, 1440 unsigned long dirty_ratelimit, 1441 int *nr_dirtied_pause) 1442 { 1443 long hi = ilog2(wb->avg_write_bandwidth); 1444 long lo = ilog2(wb->dirty_ratelimit); 1445 long t; /* target pause */ 1446 long pause; /* estimated next pause */ 1447 int pages; /* target nr_dirtied_pause */ 1448 1449 /* target for 10ms pause on 1-dd case */ 1450 t = max(1, HZ / 100); 1451 1452 /* 1453 * Scale up pause time for concurrent dirtiers in order to reduce CPU 1454 * overheads. 1455 * 1456 * (N * 10ms) on 2^N concurrent tasks. 1457 */ 1458 if (hi > lo) 1459 t += (hi - lo) * (10 * HZ) / 1024; 1460 1461 /* 1462 * This is a bit convoluted. We try to base the next nr_dirtied_pause 1463 * on the much more stable dirty_ratelimit. However the next pause time 1464 * will be computed based on task_ratelimit and the two rate limits may 1465 * depart considerably at some time. Especially if task_ratelimit goes 1466 * below dirty_ratelimit/2 and the target pause is max_pause, the next 1467 * pause time will be max_pause*2 _trimmed down_ to max_pause. As a 1468 * result task_ratelimit won't be executed faithfully, which could 1469 * eventually bring down dirty_ratelimit. 1470 * 1471 * We apply two rules to fix it up: 1472 * 1) try to estimate the next pause time and if necessary, use a lower 1473 * nr_dirtied_pause so as not to exceed max_pause. When this happens, 1474 * nr_dirtied_pause will be "dancing" with task_ratelimit. 1475 * 2) limit the target pause time to max_pause/2, so that the normal 1476 * small fluctuations of task_ratelimit won't trigger rule (1) and 1477 * nr_dirtied_pause will remain as stable as dirty_ratelimit. 1478 */ 1479 t = min(t, 1 + max_pause / 2); 1480 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ); 1481 1482 /* 1483 * Tiny nr_dirtied_pause is found to hurt I/O performance in the test 1484 * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}. 1485 * When the 16 consecutive reads are often interrupted by some dirty 1486 * throttling pause during the async writes, cfq will go into idles 1487 * (deadline is fine). So push nr_dirtied_pause as high as possible 1488 * until reaches DIRTY_POLL_THRESH=32 pages. 1489 */ 1490 if (pages < DIRTY_POLL_THRESH) { 1491 t = max_pause; 1492 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ); 1493 if (pages > DIRTY_POLL_THRESH) { 1494 pages = DIRTY_POLL_THRESH; 1495 t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit; 1496 } 1497 } 1498 1499 pause = HZ * pages / (task_ratelimit + 1); 1500 if (pause > max_pause) { 1501 t = max_pause; 1502 pages = task_ratelimit * t / roundup_pow_of_two(HZ); 1503 } 1504 1505 *nr_dirtied_pause = pages; 1506 /* 1507 * The minimal pause time will normally be half the target pause time. 1508 */ 1509 return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t; 1510 } 1511 1512 static inline void wb_dirty_limits(struct dirty_throttle_control *dtc) 1513 { 1514 struct bdi_writeback *wb = dtc->wb; 1515 unsigned long wb_reclaimable; 1516 1517 /* 1518 * wb_thresh is not treated as some limiting factor as 1519 * dirty_thresh, due to reasons 1520 * - in JBOD setup, wb_thresh can fluctuate a lot 1521 * - in a system with HDD and USB key, the USB key may somehow 1522 * go into state (wb_dirty >> wb_thresh) either because 1523 * wb_dirty starts high, or because wb_thresh drops low. 1524 * In this case we don't want to hard throttle the USB key 1525 * dirtiers for 100 seconds until wb_dirty drops under 1526 * wb_thresh. Instead the auxiliary wb control line in 1527 * wb_position_ratio() will let the dirtier task progress 1528 * at some rate <= (write_bw / 2) for bringing down wb_dirty. 1529 */ 1530 dtc->wb_thresh = __wb_calc_thresh(dtc); 1531 dtc->wb_bg_thresh = dtc->thresh ? 1532 div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0; 1533 1534 /* 1535 * In order to avoid the stacked BDI deadlock we need 1536 * to ensure we accurately count the 'dirty' pages when 1537 * the threshold is low. 1538 * 1539 * Otherwise it would be possible to get thresh+n pages 1540 * reported dirty, even though there are thresh-m pages 1541 * actually dirty; with m+n sitting in the percpu 1542 * deltas. 1543 */ 1544 if (dtc->wb_thresh < 2 * wb_stat_error()) { 1545 wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE); 1546 dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK); 1547 } else { 1548 wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE); 1549 dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK); 1550 } 1551 } 1552 1553 /* 1554 * balance_dirty_pages() must be called by processes which are generating dirty 1555 * data. It looks at the number of dirty pages in the machine and will force 1556 * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2. 1557 * If we're over `background_thresh' then the writeback threads are woken to 1558 * perform some writeout. 1559 */ 1560 static void balance_dirty_pages(struct bdi_writeback *wb, 1561 unsigned long pages_dirtied) 1562 { 1563 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) }; 1564 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) }; 1565 struct dirty_throttle_control * const gdtc = &gdtc_stor; 1566 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ? 1567 &mdtc_stor : NULL; 1568 struct dirty_throttle_control *sdtc; 1569 unsigned long nr_reclaimable; /* = file_dirty + unstable_nfs */ 1570 long period; 1571 long pause; 1572 long max_pause; 1573 long min_pause; 1574 int nr_dirtied_pause; 1575 bool dirty_exceeded = false; 1576 unsigned long task_ratelimit; 1577 unsigned long dirty_ratelimit; 1578 struct backing_dev_info *bdi = wb->bdi; 1579 bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT; 1580 unsigned long start_time = jiffies; 1581 1582 for (;;) { 1583 unsigned long now = jiffies; 1584 unsigned long dirty, thresh, bg_thresh; 1585 unsigned long m_dirty = 0; /* stop bogus uninit warnings */ 1586 unsigned long m_thresh = 0; 1587 unsigned long m_bg_thresh = 0; 1588 1589 /* 1590 * Unstable writes are a feature of certain networked 1591 * filesystems (i.e. NFS) in which data may have been 1592 * written to the server's write cache, but has not yet 1593 * been flushed to permanent storage. 1594 */ 1595 nr_reclaimable = global_node_page_state(NR_FILE_DIRTY) + 1596 global_node_page_state(NR_UNSTABLE_NFS); 1597 gdtc->avail = global_dirtyable_memory(); 1598 gdtc->dirty = nr_reclaimable + global_node_page_state(NR_WRITEBACK); 1599 1600 domain_dirty_limits(gdtc); 1601 1602 if (unlikely(strictlimit)) { 1603 wb_dirty_limits(gdtc); 1604 1605 dirty = gdtc->wb_dirty; 1606 thresh = gdtc->wb_thresh; 1607 bg_thresh = gdtc->wb_bg_thresh; 1608 } else { 1609 dirty = gdtc->dirty; 1610 thresh = gdtc->thresh; 1611 bg_thresh = gdtc->bg_thresh; 1612 } 1613 1614 if (mdtc) { 1615 unsigned long filepages, headroom, writeback; 1616 1617 /* 1618 * If @wb belongs to !root memcg, repeat the same 1619 * basic calculations for the memcg domain. 1620 */ 1621 mem_cgroup_wb_stats(wb, &filepages, &headroom, 1622 &mdtc->dirty, &writeback); 1623 mdtc->dirty += writeback; 1624 mdtc_calc_avail(mdtc, filepages, headroom); 1625 1626 domain_dirty_limits(mdtc); 1627 1628 if (unlikely(strictlimit)) { 1629 wb_dirty_limits(mdtc); 1630 m_dirty = mdtc->wb_dirty; 1631 m_thresh = mdtc->wb_thresh; 1632 m_bg_thresh = mdtc->wb_bg_thresh; 1633 } else { 1634 m_dirty = mdtc->dirty; 1635 m_thresh = mdtc->thresh; 1636 m_bg_thresh = mdtc->bg_thresh; 1637 } 1638 } 1639 1640 /* 1641 * Throttle it only when the background writeback cannot 1642 * catch-up. This avoids (excessively) small writeouts 1643 * when the wb limits are ramping up in case of !strictlimit. 1644 * 1645 * In strictlimit case make decision based on the wb counters 1646 * and limits. Small writeouts when the wb limits are ramping 1647 * up are the price we consciously pay for strictlimit-ing. 1648 * 1649 * If memcg domain is in effect, @dirty should be under 1650 * both global and memcg freerun ceilings. 1651 */ 1652 if (dirty <= dirty_freerun_ceiling(thresh, bg_thresh) && 1653 (!mdtc || 1654 m_dirty <= dirty_freerun_ceiling(m_thresh, m_bg_thresh))) { 1655 unsigned long intv = dirty_poll_interval(dirty, thresh); 1656 unsigned long m_intv = ULONG_MAX; 1657 1658 current->dirty_paused_when = now; 1659 current->nr_dirtied = 0; 1660 if (mdtc) 1661 m_intv = dirty_poll_interval(m_dirty, m_thresh); 1662 current->nr_dirtied_pause = min(intv, m_intv); 1663 break; 1664 } 1665 1666 if (unlikely(!writeback_in_progress(wb))) 1667 wb_start_background_writeback(wb); 1668 1669 /* 1670 * Calculate global domain's pos_ratio and select the 1671 * global dtc by default. 1672 */ 1673 if (!strictlimit) 1674 wb_dirty_limits(gdtc); 1675 1676 dirty_exceeded = (gdtc->wb_dirty > gdtc->wb_thresh) && 1677 ((gdtc->dirty > gdtc->thresh) || strictlimit); 1678 1679 wb_position_ratio(gdtc); 1680 sdtc = gdtc; 1681 1682 if (mdtc) { 1683 /* 1684 * If memcg domain is in effect, calculate its 1685 * pos_ratio. @wb should satisfy constraints from 1686 * both global and memcg domains. Choose the one 1687 * w/ lower pos_ratio. 1688 */ 1689 if (!strictlimit) 1690 wb_dirty_limits(mdtc); 1691 1692 dirty_exceeded |= (mdtc->wb_dirty > mdtc->wb_thresh) && 1693 ((mdtc->dirty > mdtc->thresh) || strictlimit); 1694 1695 wb_position_ratio(mdtc); 1696 if (mdtc->pos_ratio < gdtc->pos_ratio) 1697 sdtc = mdtc; 1698 } 1699 1700 if (dirty_exceeded && !wb->dirty_exceeded) 1701 wb->dirty_exceeded = 1; 1702 1703 if (time_is_before_jiffies(wb->bw_time_stamp + 1704 BANDWIDTH_INTERVAL)) { 1705 spin_lock(&wb->list_lock); 1706 __wb_update_bandwidth(gdtc, mdtc, start_time, true); 1707 spin_unlock(&wb->list_lock); 1708 } 1709 1710 /* throttle according to the chosen dtc */ 1711 dirty_ratelimit = wb->dirty_ratelimit; 1712 task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >> 1713 RATELIMIT_CALC_SHIFT; 1714 max_pause = wb_max_pause(wb, sdtc->wb_dirty); 1715 min_pause = wb_min_pause(wb, max_pause, 1716 task_ratelimit, dirty_ratelimit, 1717 &nr_dirtied_pause); 1718 1719 if (unlikely(task_ratelimit == 0)) { 1720 period = max_pause; 1721 pause = max_pause; 1722 goto pause; 1723 } 1724 period = HZ * pages_dirtied / task_ratelimit; 1725 pause = period; 1726 if (current->dirty_paused_when) 1727 pause -= now - current->dirty_paused_when; 1728 /* 1729 * For less than 1s think time (ext3/4 may block the dirtier 1730 * for up to 800ms from time to time on 1-HDD; so does xfs, 1731 * however at much less frequency), try to compensate it in 1732 * future periods by updating the virtual time; otherwise just 1733 * do a reset, as it may be a light dirtier. 1734 */ 1735 if (pause < min_pause) { 1736 trace_balance_dirty_pages(wb, 1737 sdtc->thresh, 1738 sdtc->bg_thresh, 1739 sdtc->dirty, 1740 sdtc->wb_thresh, 1741 sdtc->wb_dirty, 1742 dirty_ratelimit, 1743 task_ratelimit, 1744 pages_dirtied, 1745 period, 1746 min(pause, 0L), 1747 start_time); 1748 if (pause < -HZ) { 1749 current->dirty_paused_when = now; 1750 current->nr_dirtied = 0; 1751 } else if (period) { 1752 current->dirty_paused_when += period; 1753 current->nr_dirtied = 0; 1754 } else if (current->nr_dirtied_pause <= pages_dirtied) 1755 current->nr_dirtied_pause += pages_dirtied; 1756 break; 1757 } 1758 if (unlikely(pause > max_pause)) { 1759 /* for occasional dropped task_ratelimit */ 1760 now += min(pause - max_pause, max_pause); 1761 pause = max_pause; 1762 } 1763 1764 pause: 1765 trace_balance_dirty_pages(wb, 1766 sdtc->thresh, 1767 sdtc->bg_thresh, 1768 sdtc->dirty, 1769 sdtc->wb_thresh, 1770 sdtc->wb_dirty, 1771 dirty_ratelimit, 1772 task_ratelimit, 1773 pages_dirtied, 1774 period, 1775 pause, 1776 start_time); 1777 __set_current_state(TASK_KILLABLE); 1778 wb->dirty_sleep = now; 1779 io_schedule_timeout(pause); 1780 1781 current->dirty_paused_when = now + pause; 1782 current->nr_dirtied = 0; 1783 current->nr_dirtied_pause = nr_dirtied_pause; 1784 1785 /* 1786 * This is typically equal to (dirty < thresh) and can also 1787 * keep "1000+ dd on a slow USB stick" under control. 1788 */ 1789 if (task_ratelimit) 1790 break; 1791 1792 /* 1793 * In the case of an unresponding NFS server and the NFS dirty 1794 * pages exceeds dirty_thresh, give the other good wb's a pipe 1795 * to go through, so that tasks on them still remain responsive. 1796 * 1797 * In theory 1 page is enough to keep the consumer-producer 1798 * pipe going: the flusher cleans 1 page => the task dirties 1 1799 * more page. However wb_dirty has accounting errors. So use 1800 * the larger and more IO friendly wb_stat_error. 1801 */ 1802 if (sdtc->wb_dirty <= wb_stat_error()) 1803 break; 1804 1805 if (fatal_signal_pending(current)) 1806 break; 1807 } 1808 1809 if (!dirty_exceeded && wb->dirty_exceeded) 1810 wb->dirty_exceeded = 0; 1811 1812 if (writeback_in_progress(wb)) 1813 return; 1814 1815 /* 1816 * In laptop mode, we wait until hitting the higher threshold before 1817 * starting background writeout, and then write out all the way down 1818 * to the lower threshold. So slow writers cause minimal disk activity. 1819 * 1820 * In normal mode, we start background writeout at the lower 1821 * background_thresh, to keep the amount of dirty memory low. 1822 */ 1823 if (laptop_mode) 1824 return; 1825 1826 if (nr_reclaimable > gdtc->bg_thresh) 1827 wb_start_background_writeback(wb); 1828 } 1829 1830 static DEFINE_PER_CPU(int, bdp_ratelimits); 1831 1832 /* 1833 * Normal tasks are throttled by 1834 * loop { 1835 * dirty tsk->nr_dirtied_pause pages; 1836 * take a snap in balance_dirty_pages(); 1837 * } 1838 * However there is a worst case. If every task exit immediately when dirtied 1839 * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be 1840 * called to throttle the page dirties. The solution is to save the not yet 1841 * throttled page dirties in dirty_throttle_leaks on task exit and charge them 1842 * randomly into the running tasks. This works well for the above worst case, 1843 * as the new task will pick up and accumulate the old task's leaked dirty 1844 * count and eventually get throttled. 1845 */ 1846 DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0; 1847 1848 /** 1849 * balance_dirty_pages_ratelimited - balance dirty memory state 1850 * @mapping: address_space which was dirtied 1851 * 1852 * Processes which are dirtying memory should call in here once for each page 1853 * which was newly dirtied. The function will periodically check the system's 1854 * dirty state and will initiate writeback if needed. 1855 * 1856 * On really big machines, get_writeback_state is expensive, so try to avoid 1857 * calling it too often (ratelimiting). But once we're over the dirty memory 1858 * limit we decrease the ratelimiting by a lot, to prevent individual processes 1859 * from overshooting the limit by (ratelimit_pages) each. 1860 */ 1861 void balance_dirty_pages_ratelimited(struct address_space *mapping) 1862 { 1863 struct inode *inode = mapping->host; 1864 struct backing_dev_info *bdi = inode_to_bdi(inode); 1865 struct bdi_writeback *wb = NULL; 1866 int ratelimit; 1867 int *p; 1868 1869 if (!bdi_cap_account_dirty(bdi)) 1870 return; 1871 1872 if (inode_cgwb_enabled(inode)) 1873 wb = wb_get_create_current(bdi, GFP_KERNEL); 1874 if (!wb) 1875 wb = &bdi->wb; 1876 1877 ratelimit = current->nr_dirtied_pause; 1878 if (wb->dirty_exceeded) 1879 ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10)); 1880 1881 preempt_disable(); 1882 /* 1883 * This prevents one CPU to accumulate too many dirtied pages without 1884 * calling into balance_dirty_pages(), which can happen when there are 1885 * 1000+ tasks, all of them start dirtying pages at exactly the same 1886 * time, hence all honoured too large initial task->nr_dirtied_pause. 1887 */ 1888 p = this_cpu_ptr(&bdp_ratelimits); 1889 if (unlikely(current->nr_dirtied >= ratelimit)) 1890 *p = 0; 1891 else if (unlikely(*p >= ratelimit_pages)) { 1892 *p = 0; 1893 ratelimit = 0; 1894 } 1895 /* 1896 * Pick up the dirtied pages by the exited tasks. This avoids lots of 1897 * short-lived tasks (eg. gcc invocations in a kernel build) escaping 1898 * the dirty throttling and livelock other long-run dirtiers. 1899 */ 1900 p = this_cpu_ptr(&dirty_throttle_leaks); 1901 if (*p > 0 && current->nr_dirtied < ratelimit) { 1902 unsigned long nr_pages_dirtied; 1903 nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied); 1904 *p -= nr_pages_dirtied; 1905 current->nr_dirtied += nr_pages_dirtied; 1906 } 1907 preempt_enable(); 1908 1909 if (unlikely(current->nr_dirtied >= ratelimit)) 1910 balance_dirty_pages(wb, current->nr_dirtied); 1911 1912 wb_put(wb); 1913 } 1914 EXPORT_SYMBOL(balance_dirty_pages_ratelimited); 1915 1916 /** 1917 * wb_over_bg_thresh - does @wb need to be written back? 1918 * @wb: bdi_writeback of interest 1919 * 1920 * Determines whether background writeback should keep writing @wb or it's 1921 * clean enough. Returns %true if writeback should continue. 1922 */ 1923 bool wb_over_bg_thresh(struct bdi_writeback *wb) 1924 { 1925 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) }; 1926 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) }; 1927 struct dirty_throttle_control * const gdtc = &gdtc_stor; 1928 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ? 1929 &mdtc_stor : NULL; 1930 1931 /* 1932 * Similar to balance_dirty_pages() but ignores pages being written 1933 * as we're trying to decide whether to put more under writeback. 1934 */ 1935 gdtc->avail = global_dirtyable_memory(); 1936 gdtc->dirty = global_node_page_state(NR_FILE_DIRTY) + 1937 global_node_page_state(NR_UNSTABLE_NFS); 1938 domain_dirty_limits(gdtc); 1939 1940 if (gdtc->dirty > gdtc->bg_thresh) 1941 return true; 1942 1943 if (wb_stat(wb, WB_RECLAIMABLE) > 1944 wb_calc_thresh(gdtc->wb, gdtc->bg_thresh)) 1945 return true; 1946 1947 if (mdtc) { 1948 unsigned long filepages, headroom, writeback; 1949 1950 mem_cgroup_wb_stats(wb, &filepages, &headroom, &mdtc->dirty, 1951 &writeback); 1952 mdtc_calc_avail(mdtc, filepages, headroom); 1953 domain_dirty_limits(mdtc); /* ditto, ignore writeback */ 1954 1955 if (mdtc->dirty > mdtc->bg_thresh) 1956 return true; 1957 1958 if (wb_stat(wb, WB_RECLAIMABLE) > 1959 wb_calc_thresh(mdtc->wb, mdtc->bg_thresh)) 1960 return true; 1961 } 1962 1963 return false; 1964 } 1965 1966 /* 1967 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs 1968 */ 1969 int dirty_writeback_centisecs_handler(struct ctl_table *table, int write, 1970 void __user *buffer, size_t *length, loff_t *ppos) 1971 { 1972 unsigned int old_interval = dirty_writeback_interval; 1973 int ret; 1974 1975 ret = proc_dointvec(table, write, buffer, length, ppos); 1976 1977 /* 1978 * Writing 0 to dirty_writeback_interval will disable periodic writeback 1979 * and a different non-zero value will wakeup the writeback threads. 1980 * wb_wakeup_delayed() would be more appropriate, but it's a pain to 1981 * iterate over all bdis and wbs. 1982 * The reason we do this is to make the change take effect immediately. 1983 */ 1984 if (!ret && write && dirty_writeback_interval && 1985 dirty_writeback_interval != old_interval) 1986 wakeup_flusher_threads(WB_REASON_PERIODIC); 1987 1988 return ret; 1989 } 1990 1991 #ifdef CONFIG_BLOCK 1992 void laptop_mode_timer_fn(struct timer_list *t) 1993 { 1994 struct backing_dev_info *backing_dev_info = 1995 from_timer(backing_dev_info, t, laptop_mode_wb_timer); 1996 1997 wakeup_flusher_threads_bdi(backing_dev_info, WB_REASON_LAPTOP_TIMER); 1998 } 1999 2000 /* 2001 * We've spun up the disk and we're in laptop mode: schedule writeback 2002 * of all dirty data a few seconds from now. If the flush is already scheduled 2003 * then push it back - the user is still using the disk. 2004 */ 2005 void laptop_io_completion(struct backing_dev_info *info) 2006 { 2007 mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode); 2008 } 2009 2010 /* 2011 * We're in laptop mode and we've just synced. The sync's writes will have 2012 * caused another writeback to be scheduled by laptop_io_completion. 2013 * Nothing needs to be written back anymore, so we unschedule the writeback. 2014 */ 2015 void laptop_sync_completion(void) 2016 { 2017 struct backing_dev_info *bdi; 2018 2019 rcu_read_lock(); 2020 2021 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) 2022 del_timer(&bdi->laptop_mode_wb_timer); 2023 2024 rcu_read_unlock(); 2025 } 2026 #endif 2027 2028 /* 2029 * If ratelimit_pages is too high then we can get into dirty-data overload 2030 * if a large number of processes all perform writes at the same time. 2031 * If it is too low then SMP machines will call the (expensive) 2032 * get_writeback_state too often. 2033 * 2034 * Here we set ratelimit_pages to a level which ensures that when all CPUs are 2035 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory 2036 * thresholds. 2037 */ 2038 2039 void writeback_set_ratelimit(void) 2040 { 2041 struct wb_domain *dom = &global_wb_domain; 2042 unsigned long background_thresh; 2043 unsigned long dirty_thresh; 2044 2045 global_dirty_limits(&background_thresh, &dirty_thresh); 2046 dom->dirty_limit = dirty_thresh; 2047 ratelimit_pages = dirty_thresh / (num_online_cpus() * 32); 2048 if (ratelimit_pages < 16) 2049 ratelimit_pages = 16; 2050 } 2051 2052 static int page_writeback_cpu_online(unsigned int cpu) 2053 { 2054 writeback_set_ratelimit(); 2055 return 0; 2056 } 2057 2058 /* 2059 * Called early on to tune the page writeback dirty limits. 2060 * 2061 * We used to scale dirty pages according to how total memory 2062 * related to pages that could be allocated for buffers (by 2063 * comparing nr_free_buffer_pages() to vm_total_pages. 2064 * 2065 * However, that was when we used "dirty_ratio" to scale with 2066 * all memory, and we don't do that any more. "dirty_ratio" 2067 * is now applied to total non-HIGHPAGE memory (by subtracting 2068 * totalhigh_pages from vm_total_pages), and as such we can't 2069 * get into the old insane situation any more where we had 2070 * large amounts of dirty pages compared to a small amount of 2071 * non-HIGHMEM memory. 2072 * 2073 * But we might still want to scale the dirty_ratio by how 2074 * much memory the box has.. 2075 */ 2076 void __init page_writeback_init(void) 2077 { 2078 BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL)); 2079 2080 cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mm/writeback:online", 2081 page_writeback_cpu_online, NULL); 2082 cpuhp_setup_state(CPUHP_MM_WRITEBACK_DEAD, "mm/writeback:dead", NULL, 2083 page_writeback_cpu_online); 2084 } 2085 2086 /** 2087 * tag_pages_for_writeback - tag pages to be written by write_cache_pages 2088 * @mapping: address space structure to write 2089 * @start: starting page index 2090 * @end: ending page index (inclusive) 2091 * 2092 * This function scans the page range from @start to @end (inclusive) and tags 2093 * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is 2094 * that write_cache_pages (or whoever calls this function) will then use 2095 * TOWRITE tag to identify pages eligible for writeback. This mechanism is 2096 * used to avoid livelocking of writeback by a process steadily creating new 2097 * dirty pages in the file (thus it is important for this function to be quick 2098 * so that it can tag pages faster than a dirtying process can create them). 2099 */ 2100 void tag_pages_for_writeback(struct address_space *mapping, 2101 pgoff_t start, pgoff_t end) 2102 { 2103 XA_STATE(xas, &mapping->i_pages, start); 2104 unsigned int tagged = 0; 2105 void *page; 2106 2107 xas_lock_irq(&xas); 2108 xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) { 2109 xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE); 2110 if (++tagged % XA_CHECK_SCHED) 2111 continue; 2112 2113 xas_pause(&xas); 2114 xas_unlock_irq(&xas); 2115 cond_resched(); 2116 xas_lock_irq(&xas); 2117 } 2118 xas_unlock_irq(&xas); 2119 } 2120 EXPORT_SYMBOL(tag_pages_for_writeback); 2121 2122 /** 2123 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. 2124 * @mapping: address space structure to write 2125 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 2126 * @writepage: function called for each page 2127 * @data: data passed to writepage function 2128 * 2129 * If a page is already under I/O, write_cache_pages() skips it, even 2130 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 2131 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 2132 * and msync() need to guarantee that all the data which was dirty at the time 2133 * the call was made get new I/O started against them. If wbc->sync_mode is 2134 * WB_SYNC_ALL then we were called for data integrity and we must wait for 2135 * existing IO to complete. 2136 * 2137 * To avoid livelocks (when other process dirties new pages), we first tag 2138 * pages which should be written back with TOWRITE tag and only then start 2139 * writing them. For data-integrity sync we have to be careful so that we do 2140 * not miss some pages (e.g., because some other process has cleared TOWRITE 2141 * tag we set). The rule we follow is that TOWRITE tag can be cleared only 2142 * by the process clearing the DIRTY tag (and submitting the page for IO). 2143 * 2144 * To avoid deadlocks between range_cyclic writeback and callers that hold 2145 * pages in PageWriteback to aggregate IO until write_cache_pages() returns, 2146 * we do not loop back to the start of the file. Doing so causes a page 2147 * lock/page writeback access order inversion - we should only ever lock 2148 * multiple pages in ascending page->index order, and looping back to the start 2149 * of the file violates that rule and causes deadlocks. 2150 */ 2151 int write_cache_pages(struct address_space *mapping, 2152 struct writeback_control *wbc, writepage_t writepage, 2153 void *data) 2154 { 2155 int ret = 0; 2156 int done = 0; 2157 int error; 2158 struct pagevec pvec; 2159 int nr_pages; 2160 pgoff_t uninitialized_var(writeback_index); 2161 pgoff_t index; 2162 pgoff_t end; /* Inclusive */ 2163 pgoff_t done_index; 2164 int range_whole = 0; 2165 xa_mark_t tag; 2166 2167 pagevec_init(&pvec); 2168 if (wbc->range_cyclic) { 2169 writeback_index = mapping->writeback_index; /* prev offset */ 2170 index = writeback_index; 2171 end = -1; 2172 } else { 2173 index = wbc->range_start >> PAGE_SHIFT; 2174 end = wbc->range_end >> PAGE_SHIFT; 2175 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2176 range_whole = 1; 2177 } 2178 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2179 tag = PAGECACHE_TAG_TOWRITE; 2180 else 2181 tag = PAGECACHE_TAG_DIRTY; 2182 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2183 tag_pages_for_writeback(mapping, index, end); 2184 done_index = index; 2185 while (!done && (index <= end)) { 2186 int i; 2187 2188 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, 2189 tag); 2190 if (nr_pages == 0) 2191 break; 2192 2193 for (i = 0; i < nr_pages; i++) { 2194 struct page *page = pvec.pages[i]; 2195 2196 done_index = page->index; 2197 2198 lock_page(page); 2199 2200 /* 2201 * Page truncated or invalidated. We can freely skip it 2202 * then, even for data integrity operations: the page 2203 * has disappeared concurrently, so there could be no 2204 * real expectation of this data interity operation 2205 * even if there is now a new, dirty page at the same 2206 * pagecache address. 2207 */ 2208 if (unlikely(page->mapping != mapping)) { 2209 continue_unlock: 2210 unlock_page(page); 2211 continue; 2212 } 2213 2214 if (!PageDirty(page)) { 2215 /* someone wrote it for us */ 2216 goto continue_unlock; 2217 } 2218 2219 if (PageWriteback(page)) { 2220 if (wbc->sync_mode != WB_SYNC_NONE) 2221 wait_on_page_writeback(page); 2222 else 2223 goto continue_unlock; 2224 } 2225 2226 BUG_ON(PageWriteback(page)); 2227 if (!clear_page_dirty_for_io(page)) 2228 goto continue_unlock; 2229 2230 trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); 2231 error = (*writepage)(page, wbc, data); 2232 if (unlikely(error)) { 2233 /* 2234 * Handle errors according to the type of 2235 * writeback. There's no need to continue for 2236 * background writeback. Just push done_index 2237 * past this page so media errors won't choke 2238 * writeout for the entire file. For integrity 2239 * writeback, we must process the entire dirty 2240 * set regardless of errors because the fs may 2241 * still have state to clear for each page. In 2242 * that case we continue processing and return 2243 * the first error. 2244 */ 2245 if (error == AOP_WRITEPAGE_ACTIVATE) { 2246 unlock_page(page); 2247 error = 0; 2248 } else if (wbc->sync_mode != WB_SYNC_ALL) { 2249 ret = error; 2250 done_index = page->index + 1; 2251 done = 1; 2252 break; 2253 } 2254 if (!ret) 2255 ret = error; 2256 } 2257 2258 /* 2259 * We stop writing back only if we are not doing 2260 * integrity sync. In case of integrity sync we have to 2261 * keep going until we have written all the pages 2262 * we tagged for writeback prior to entering this loop. 2263 */ 2264 if (--wbc->nr_to_write <= 0 && 2265 wbc->sync_mode == WB_SYNC_NONE) { 2266 done = 1; 2267 break; 2268 } 2269 } 2270 pagevec_release(&pvec); 2271 cond_resched(); 2272 } 2273 2274 /* 2275 * If we hit the last page and there is more work to be done: wrap 2276 * back the index back to the start of the file for the next 2277 * time we are called. 2278 */ 2279 if (wbc->range_cyclic && !done) 2280 done_index = 0; 2281 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 2282 mapping->writeback_index = done_index; 2283 2284 return ret; 2285 } 2286 EXPORT_SYMBOL(write_cache_pages); 2287 2288 /* 2289 * Function used by generic_writepages to call the real writepage 2290 * function and set the mapping flags on error 2291 */ 2292 static int __writepage(struct page *page, struct writeback_control *wbc, 2293 void *data) 2294 { 2295 struct address_space *mapping = data; 2296 int ret = mapping->a_ops->writepage(page, wbc); 2297 mapping_set_error(mapping, ret); 2298 return ret; 2299 } 2300 2301 /** 2302 * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them. 2303 * @mapping: address space structure to write 2304 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 2305 * 2306 * This is a library function, which implements the writepages() 2307 * address_space_operation. 2308 */ 2309 int generic_writepages(struct address_space *mapping, 2310 struct writeback_control *wbc) 2311 { 2312 struct blk_plug plug; 2313 int ret; 2314 2315 /* deal with chardevs and other special file */ 2316 if (!mapping->a_ops->writepage) 2317 return 0; 2318 2319 blk_start_plug(&plug); 2320 ret = write_cache_pages(mapping, wbc, __writepage, mapping); 2321 blk_finish_plug(&plug); 2322 return ret; 2323 } 2324 2325 EXPORT_SYMBOL(generic_writepages); 2326 2327 int do_writepages(struct address_space *mapping, struct writeback_control *wbc) 2328 { 2329 int ret; 2330 2331 if (wbc->nr_to_write <= 0) 2332 return 0; 2333 while (1) { 2334 if (mapping->a_ops->writepages) 2335 ret = mapping->a_ops->writepages(mapping, wbc); 2336 else 2337 ret = generic_writepages(mapping, wbc); 2338 if ((ret != -ENOMEM) || (wbc->sync_mode != WB_SYNC_ALL)) 2339 break; 2340 cond_resched(); 2341 congestion_wait(BLK_RW_ASYNC, HZ/50); 2342 } 2343 return ret; 2344 } 2345 2346 /** 2347 * write_one_page - write out a single page and wait on I/O 2348 * @page: the page to write 2349 * 2350 * The page must be locked by the caller and will be unlocked upon return. 2351 * 2352 * Note that the mapping's AS_EIO/AS_ENOSPC flags will be cleared when this 2353 * function returns. 2354 */ 2355 int write_one_page(struct page *page) 2356 { 2357 struct address_space *mapping = page->mapping; 2358 int ret = 0; 2359 struct writeback_control wbc = { 2360 .sync_mode = WB_SYNC_ALL, 2361 .nr_to_write = 1, 2362 }; 2363 2364 BUG_ON(!PageLocked(page)); 2365 2366 wait_on_page_writeback(page); 2367 2368 if (clear_page_dirty_for_io(page)) { 2369 get_page(page); 2370 ret = mapping->a_ops->writepage(page, &wbc); 2371 if (ret == 0) 2372 wait_on_page_writeback(page); 2373 put_page(page); 2374 } else { 2375 unlock_page(page); 2376 } 2377 2378 if (!ret) 2379 ret = filemap_check_errors(mapping); 2380 return ret; 2381 } 2382 EXPORT_SYMBOL(write_one_page); 2383 2384 /* 2385 * For address_spaces which do not use buffers nor write back. 2386 */ 2387 int __set_page_dirty_no_writeback(struct page *page) 2388 { 2389 if (!PageDirty(page)) 2390 return !TestSetPageDirty(page); 2391 return 0; 2392 } 2393 2394 /* 2395 * Helper function for set_page_dirty family. 2396 * 2397 * Caller must hold lock_page_memcg(). 2398 * 2399 * NOTE: This relies on being atomic wrt interrupts. 2400 */ 2401 void account_page_dirtied(struct page *page, struct address_space *mapping) 2402 { 2403 struct inode *inode = mapping->host; 2404 2405 trace_writeback_dirty_page(page, mapping); 2406 2407 if (mapping_cap_account_dirty(mapping)) { 2408 struct bdi_writeback *wb; 2409 2410 inode_attach_wb(inode, page); 2411 wb = inode_to_wb(inode); 2412 2413 __inc_lruvec_page_state(page, NR_FILE_DIRTY); 2414 __inc_zone_page_state(page, NR_ZONE_WRITE_PENDING); 2415 __inc_node_page_state(page, NR_DIRTIED); 2416 inc_wb_stat(wb, WB_RECLAIMABLE); 2417 inc_wb_stat(wb, WB_DIRTIED); 2418 task_io_account_write(PAGE_SIZE); 2419 current->nr_dirtied++; 2420 this_cpu_inc(bdp_ratelimits); 2421 } 2422 } 2423 EXPORT_SYMBOL(account_page_dirtied); 2424 2425 /* 2426 * Helper function for deaccounting dirty page without writeback. 2427 * 2428 * Caller must hold lock_page_memcg(). 2429 */ 2430 void account_page_cleaned(struct page *page, struct address_space *mapping, 2431 struct bdi_writeback *wb) 2432 { 2433 if (mapping_cap_account_dirty(mapping)) { 2434 dec_lruvec_page_state(page, NR_FILE_DIRTY); 2435 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING); 2436 dec_wb_stat(wb, WB_RECLAIMABLE); 2437 task_io_account_cancelled_write(PAGE_SIZE); 2438 } 2439 } 2440 2441 /* 2442 * For address_spaces which do not use buffers. Just tag the page as dirty in 2443 * the xarray. 2444 * 2445 * This is also used when a single buffer is being dirtied: we want to set the 2446 * page dirty in that case, but not all the buffers. This is a "bottom-up" 2447 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. 2448 * 2449 * The caller must ensure this doesn't race with truncation. Most will simply 2450 * hold the page lock, but e.g. zap_pte_range() calls with the page mapped and 2451 * the pte lock held, which also locks out truncation. 2452 */ 2453 int __set_page_dirty_nobuffers(struct page *page) 2454 { 2455 lock_page_memcg(page); 2456 if (!TestSetPageDirty(page)) { 2457 struct address_space *mapping = page_mapping(page); 2458 unsigned long flags; 2459 2460 if (!mapping) { 2461 unlock_page_memcg(page); 2462 return 1; 2463 } 2464 2465 xa_lock_irqsave(&mapping->i_pages, flags); 2466 BUG_ON(page_mapping(page) != mapping); 2467 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); 2468 account_page_dirtied(page, mapping); 2469 __xa_set_mark(&mapping->i_pages, page_index(page), 2470 PAGECACHE_TAG_DIRTY); 2471 xa_unlock_irqrestore(&mapping->i_pages, flags); 2472 unlock_page_memcg(page); 2473 2474 if (mapping->host) { 2475 /* !PageAnon && !swapper_space */ 2476 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 2477 } 2478 return 1; 2479 } 2480 unlock_page_memcg(page); 2481 return 0; 2482 } 2483 EXPORT_SYMBOL(__set_page_dirty_nobuffers); 2484 2485 /* 2486 * Call this whenever redirtying a page, to de-account the dirty counters 2487 * (NR_DIRTIED, WB_DIRTIED, tsk->nr_dirtied), so that they match the written 2488 * counters (NR_WRITTEN, WB_WRITTEN) in long term. The mismatches will lead to 2489 * systematic errors in balanced_dirty_ratelimit and the dirty pages position 2490 * control. 2491 */ 2492 void account_page_redirty(struct page *page) 2493 { 2494 struct address_space *mapping = page->mapping; 2495 2496 if (mapping && mapping_cap_account_dirty(mapping)) { 2497 struct inode *inode = mapping->host; 2498 struct bdi_writeback *wb; 2499 struct wb_lock_cookie cookie = {}; 2500 2501 wb = unlocked_inode_to_wb_begin(inode, &cookie); 2502 current->nr_dirtied--; 2503 dec_node_page_state(page, NR_DIRTIED); 2504 dec_wb_stat(wb, WB_DIRTIED); 2505 unlocked_inode_to_wb_end(inode, &cookie); 2506 } 2507 } 2508 EXPORT_SYMBOL(account_page_redirty); 2509 2510 /* 2511 * When a writepage implementation decides that it doesn't want to write this 2512 * page for some reason, it should redirty the locked page via 2513 * redirty_page_for_writepage() and it should then unlock the page and return 0 2514 */ 2515 int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page) 2516 { 2517 int ret; 2518 2519 wbc->pages_skipped++; 2520 ret = __set_page_dirty_nobuffers(page); 2521 account_page_redirty(page); 2522 return ret; 2523 } 2524 EXPORT_SYMBOL(redirty_page_for_writepage); 2525 2526 /* 2527 * Dirty a page. 2528 * 2529 * For pages with a mapping this should be done under the page lock 2530 * for the benefit of asynchronous memory errors who prefer a consistent 2531 * dirty state. This rule can be broken in some special cases, 2532 * but should be better not to. 2533 * 2534 * If the mapping doesn't provide a set_page_dirty a_op, then 2535 * just fall through and assume that it wants buffer_heads. 2536 */ 2537 int set_page_dirty(struct page *page) 2538 { 2539 struct address_space *mapping = page_mapping(page); 2540 2541 page = compound_head(page); 2542 if (likely(mapping)) { 2543 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty; 2544 /* 2545 * readahead/lru_deactivate_page could remain 2546 * PG_readahead/PG_reclaim due to race with end_page_writeback 2547 * About readahead, if the page is written, the flags would be 2548 * reset. So no problem. 2549 * About lru_deactivate_page, if the page is redirty, the flag 2550 * will be reset. So no problem. but if the page is used by readahead 2551 * it will confuse readahead and make it restart the size rampup 2552 * process. But it's a trivial problem. 2553 */ 2554 if (PageReclaim(page)) 2555 ClearPageReclaim(page); 2556 #ifdef CONFIG_BLOCK 2557 if (!spd) 2558 spd = __set_page_dirty_buffers; 2559 #endif 2560 return (*spd)(page); 2561 } 2562 if (!PageDirty(page)) { 2563 if (!TestSetPageDirty(page)) 2564 return 1; 2565 } 2566 return 0; 2567 } 2568 EXPORT_SYMBOL(set_page_dirty); 2569 2570 /* 2571 * set_page_dirty() is racy if the caller has no reference against 2572 * page->mapping->host, and if the page is unlocked. This is because another 2573 * CPU could truncate the page off the mapping and then free the mapping. 2574 * 2575 * Usually, the page _is_ locked, or the caller is a user-space process which 2576 * holds a reference on the inode by having an open file. 2577 * 2578 * In other cases, the page should be locked before running set_page_dirty(). 2579 */ 2580 int set_page_dirty_lock(struct page *page) 2581 { 2582 int ret; 2583 2584 lock_page(page); 2585 ret = set_page_dirty(page); 2586 unlock_page(page); 2587 return ret; 2588 } 2589 EXPORT_SYMBOL(set_page_dirty_lock); 2590 2591 /* 2592 * This cancels just the dirty bit on the kernel page itself, it does NOT 2593 * actually remove dirty bits on any mmap's that may be around. It also 2594 * leaves the page tagged dirty, so any sync activity will still find it on 2595 * the dirty lists, and in particular, clear_page_dirty_for_io() will still 2596 * look at the dirty bits in the VM. 2597 * 2598 * Doing this should *normally* only ever be done when a page is truncated, 2599 * and is not actually mapped anywhere at all. However, fs/buffer.c does 2600 * this when it notices that somebody has cleaned out all the buffers on a 2601 * page without actually doing it through the VM. Can you say "ext3 is 2602 * horribly ugly"? Thought you could. 2603 */ 2604 void __cancel_dirty_page(struct page *page) 2605 { 2606 struct address_space *mapping = page_mapping(page); 2607 2608 if (mapping_cap_account_dirty(mapping)) { 2609 struct inode *inode = mapping->host; 2610 struct bdi_writeback *wb; 2611 struct wb_lock_cookie cookie = {}; 2612 2613 lock_page_memcg(page); 2614 wb = unlocked_inode_to_wb_begin(inode, &cookie); 2615 2616 if (TestClearPageDirty(page)) 2617 account_page_cleaned(page, mapping, wb); 2618 2619 unlocked_inode_to_wb_end(inode, &cookie); 2620 unlock_page_memcg(page); 2621 } else { 2622 ClearPageDirty(page); 2623 } 2624 } 2625 EXPORT_SYMBOL(__cancel_dirty_page); 2626 2627 /* 2628 * Clear a page's dirty flag, while caring for dirty memory accounting. 2629 * Returns true if the page was previously dirty. 2630 * 2631 * This is for preparing to put the page under writeout. We leave the page 2632 * tagged as dirty in the xarray so that a concurrent write-for-sync 2633 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage 2634 * implementation will run either set_page_writeback() or set_page_dirty(), 2635 * at which stage we bring the page's dirty flag and xarray dirty tag 2636 * back into sync. 2637 * 2638 * This incoherency between the page's dirty flag and xarray tag is 2639 * unfortunate, but it only exists while the page is locked. 2640 */ 2641 int clear_page_dirty_for_io(struct page *page) 2642 { 2643 struct address_space *mapping = page_mapping(page); 2644 int ret = 0; 2645 2646 BUG_ON(!PageLocked(page)); 2647 2648 if (mapping && mapping_cap_account_dirty(mapping)) { 2649 struct inode *inode = mapping->host; 2650 struct bdi_writeback *wb; 2651 struct wb_lock_cookie cookie = {}; 2652 2653 /* 2654 * Yes, Virginia, this is indeed insane. 2655 * 2656 * We use this sequence to make sure that 2657 * (a) we account for dirty stats properly 2658 * (b) we tell the low-level filesystem to 2659 * mark the whole page dirty if it was 2660 * dirty in a pagetable. Only to then 2661 * (c) clean the page again and return 1 to 2662 * cause the writeback. 2663 * 2664 * This way we avoid all nasty races with the 2665 * dirty bit in multiple places and clearing 2666 * them concurrently from different threads. 2667 * 2668 * Note! Normally the "set_page_dirty(page)" 2669 * has no effect on the actual dirty bit - since 2670 * that will already usually be set. But we 2671 * need the side effects, and it can help us 2672 * avoid races. 2673 * 2674 * We basically use the page "master dirty bit" 2675 * as a serialization point for all the different 2676 * threads doing their things. 2677 */ 2678 if (page_mkclean(page)) 2679 set_page_dirty(page); 2680 /* 2681 * We carefully synchronise fault handlers against 2682 * installing a dirty pte and marking the page dirty 2683 * at this point. We do this by having them hold the 2684 * page lock while dirtying the page, and pages are 2685 * always locked coming in here, so we get the desired 2686 * exclusion. 2687 */ 2688 wb = unlocked_inode_to_wb_begin(inode, &cookie); 2689 if (TestClearPageDirty(page)) { 2690 dec_lruvec_page_state(page, NR_FILE_DIRTY); 2691 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING); 2692 dec_wb_stat(wb, WB_RECLAIMABLE); 2693 ret = 1; 2694 } 2695 unlocked_inode_to_wb_end(inode, &cookie); 2696 return ret; 2697 } 2698 return TestClearPageDirty(page); 2699 } 2700 EXPORT_SYMBOL(clear_page_dirty_for_io); 2701 2702 int test_clear_page_writeback(struct page *page) 2703 { 2704 struct address_space *mapping = page_mapping(page); 2705 struct mem_cgroup *memcg; 2706 struct lruvec *lruvec; 2707 int ret; 2708 2709 memcg = lock_page_memcg(page); 2710 lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page)); 2711 if (mapping && mapping_use_writeback_tags(mapping)) { 2712 struct inode *inode = mapping->host; 2713 struct backing_dev_info *bdi = inode_to_bdi(inode); 2714 unsigned long flags; 2715 2716 xa_lock_irqsave(&mapping->i_pages, flags); 2717 ret = TestClearPageWriteback(page); 2718 if (ret) { 2719 __xa_clear_mark(&mapping->i_pages, page_index(page), 2720 PAGECACHE_TAG_WRITEBACK); 2721 if (bdi_cap_account_writeback(bdi)) { 2722 struct bdi_writeback *wb = inode_to_wb(inode); 2723 2724 dec_wb_stat(wb, WB_WRITEBACK); 2725 __wb_writeout_inc(wb); 2726 } 2727 } 2728 2729 if (mapping->host && !mapping_tagged(mapping, 2730 PAGECACHE_TAG_WRITEBACK)) 2731 sb_clear_inode_writeback(mapping->host); 2732 2733 xa_unlock_irqrestore(&mapping->i_pages, flags); 2734 } else { 2735 ret = TestClearPageWriteback(page); 2736 } 2737 /* 2738 * NOTE: Page might be free now! Writeback doesn't hold a page 2739 * reference on its own, it relies on truncation to wait for 2740 * the clearing of PG_writeback. The below can only access 2741 * page state that is static across allocation cycles. 2742 */ 2743 if (ret) { 2744 dec_lruvec_state(lruvec, NR_WRITEBACK); 2745 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING); 2746 inc_node_page_state(page, NR_WRITTEN); 2747 } 2748 __unlock_page_memcg(memcg); 2749 return ret; 2750 } 2751 2752 int __test_set_page_writeback(struct page *page, bool keep_write) 2753 { 2754 struct address_space *mapping = page_mapping(page); 2755 int ret; 2756 2757 lock_page_memcg(page); 2758 if (mapping && mapping_use_writeback_tags(mapping)) { 2759 XA_STATE(xas, &mapping->i_pages, page_index(page)); 2760 struct inode *inode = mapping->host; 2761 struct backing_dev_info *bdi = inode_to_bdi(inode); 2762 unsigned long flags; 2763 2764 xas_lock_irqsave(&xas, flags); 2765 xas_load(&xas); 2766 ret = TestSetPageWriteback(page); 2767 if (!ret) { 2768 bool on_wblist; 2769 2770 on_wblist = mapping_tagged(mapping, 2771 PAGECACHE_TAG_WRITEBACK); 2772 2773 xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK); 2774 if (bdi_cap_account_writeback(bdi)) 2775 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); 2776 2777 /* 2778 * We can come through here when swapping anonymous 2779 * pages, so we don't necessarily have an inode to track 2780 * for sync. 2781 */ 2782 if (mapping->host && !on_wblist) 2783 sb_mark_inode_writeback(mapping->host); 2784 } 2785 if (!PageDirty(page)) 2786 xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY); 2787 if (!keep_write) 2788 xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE); 2789 xas_unlock_irqrestore(&xas, flags); 2790 } else { 2791 ret = TestSetPageWriteback(page); 2792 } 2793 if (!ret) { 2794 inc_lruvec_page_state(page, NR_WRITEBACK); 2795 inc_zone_page_state(page, NR_ZONE_WRITE_PENDING); 2796 } 2797 unlock_page_memcg(page); 2798 return ret; 2799 2800 } 2801 EXPORT_SYMBOL(__test_set_page_writeback); 2802 2803 /** 2804 * wait_for_stable_page() - wait for writeback to finish, if necessary. 2805 * @page: The page to wait on. 2806 * 2807 * This function determines if the given page is related to a backing device 2808 * that requires page contents to be held stable during writeback. If so, then 2809 * it will wait for any pending writeback to complete. 2810 */ 2811 void wait_for_stable_page(struct page *page) 2812 { 2813 if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host))) 2814 wait_on_page_writeback(page); 2815 } 2816 EXPORT_SYMBOL_GPL(wait_for_stable_page); 2817