xref: /openbmc/linux/mm/page-writeback.c (revision b6bec26c)
1 /*
2  * mm/page-writeback.c
3  *
4  * Copyright (C) 2002, Linus Torvalds.
5  * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
6  *
7  * Contains functions related to writing back dirty pages at the
8  * address_space level.
9  *
10  * 10Apr2002	Andrew Morton
11  *		Initial version
12  */
13 
14 #include <linux/kernel.h>
15 #include <linux/export.h>
16 #include <linux/spinlock.h>
17 #include <linux/fs.h>
18 #include <linux/mm.h>
19 #include <linux/swap.h>
20 #include <linux/slab.h>
21 #include <linux/pagemap.h>
22 #include <linux/writeback.h>
23 #include <linux/init.h>
24 #include <linux/backing-dev.h>
25 #include <linux/task_io_accounting_ops.h>
26 #include <linux/blkdev.h>
27 #include <linux/mpage.h>
28 #include <linux/rmap.h>
29 #include <linux/percpu.h>
30 #include <linux/notifier.h>
31 #include <linux/smp.h>
32 #include <linux/sysctl.h>
33 #include <linux/cpu.h>
34 #include <linux/syscalls.h>
35 #include <linux/buffer_head.h> /* __set_page_dirty_buffers */
36 #include <linux/pagevec.h>
37 #include <linux/timer.h>
38 #include <trace/events/writeback.h>
39 
40 /*
41  * Sleep at most 200ms at a time in balance_dirty_pages().
42  */
43 #define MAX_PAUSE		max(HZ/5, 1)
44 
45 /*
46  * Try to keep balance_dirty_pages() call intervals higher than this many pages
47  * by raising pause time to max_pause when falls below it.
48  */
49 #define DIRTY_POLL_THRESH	(128 >> (PAGE_SHIFT - 10))
50 
51 /*
52  * Estimate write bandwidth at 200ms intervals.
53  */
54 #define BANDWIDTH_INTERVAL	max(HZ/5, 1)
55 
56 #define RATELIMIT_CALC_SHIFT	10
57 
58 /*
59  * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
60  * will look to see if it needs to force writeback or throttling.
61  */
62 static long ratelimit_pages = 32;
63 
64 /* The following parameters are exported via /proc/sys/vm */
65 
66 /*
67  * Start background writeback (via writeback threads) at this percentage
68  */
69 int dirty_background_ratio = 10;
70 
71 /*
72  * dirty_background_bytes starts at 0 (disabled) so that it is a function of
73  * dirty_background_ratio * the amount of dirtyable memory
74  */
75 unsigned long dirty_background_bytes;
76 
77 /*
78  * free highmem will not be subtracted from the total free memory
79  * for calculating free ratios if vm_highmem_is_dirtyable is true
80  */
81 int vm_highmem_is_dirtyable;
82 
83 /*
84  * The generator of dirty data starts writeback at this percentage
85  */
86 int vm_dirty_ratio = 20;
87 
88 /*
89  * vm_dirty_bytes starts at 0 (disabled) so that it is a function of
90  * vm_dirty_ratio * the amount of dirtyable memory
91  */
92 unsigned long vm_dirty_bytes;
93 
94 /*
95  * The interval between `kupdate'-style writebacks
96  */
97 unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */
98 
99 EXPORT_SYMBOL_GPL(dirty_writeback_interval);
100 
101 /*
102  * The longest time for which data is allowed to remain dirty
103  */
104 unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */
105 
106 /*
107  * Flag that makes the machine dump writes/reads and block dirtyings.
108  */
109 int block_dump;
110 
111 /*
112  * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
113  * a full sync is triggered after this time elapses without any disk activity.
114  */
115 int laptop_mode;
116 
117 EXPORT_SYMBOL(laptop_mode);
118 
119 /* End of sysctl-exported parameters */
120 
121 unsigned long global_dirty_limit;
122 
123 /*
124  * Scale the writeback cache size proportional to the relative writeout speeds.
125  *
126  * We do this by keeping a floating proportion between BDIs, based on page
127  * writeback completions [end_page_writeback()]. Those devices that write out
128  * pages fastest will get the larger share, while the slower will get a smaller
129  * share.
130  *
131  * We use page writeout completions because we are interested in getting rid of
132  * dirty pages. Having them written out is the primary goal.
133  *
134  * We introduce a concept of time, a period over which we measure these events,
135  * because demand can/will vary over time. The length of this period itself is
136  * measured in page writeback completions.
137  *
138  */
139 static struct fprop_global writeout_completions;
140 
141 static void writeout_period(unsigned long t);
142 /* Timer for aging of writeout_completions */
143 static struct timer_list writeout_period_timer =
144 		TIMER_DEFERRED_INITIALIZER(writeout_period, 0, 0);
145 static unsigned long writeout_period_time = 0;
146 
147 /*
148  * Length of period for aging writeout fractions of bdis. This is an
149  * arbitrarily chosen number. The longer the period, the slower fractions will
150  * reflect changes in current writeout rate.
151  */
152 #define VM_COMPLETIONS_PERIOD_LEN (3*HZ)
153 
154 /*
155  * Work out the current dirty-memory clamping and background writeout
156  * thresholds.
157  *
158  * The main aim here is to lower them aggressively if there is a lot of mapped
159  * memory around.  To avoid stressing page reclaim with lots of unreclaimable
160  * pages.  It is better to clamp down on writers than to start swapping, and
161  * performing lots of scanning.
162  *
163  * We only allow 1/2 of the currently-unmapped memory to be dirtied.
164  *
165  * We don't permit the clamping level to fall below 5% - that is getting rather
166  * excessive.
167  *
168  * We make sure that the background writeout level is below the adjusted
169  * clamping level.
170  */
171 
172 /*
173  * In a memory zone, there is a certain amount of pages we consider
174  * available for the page cache, which is essentially the number of
175  * free and reclaimable pages, minus some zone reserves to protect
176  * lowmem and the ability to uphold the zone's watermarks without
177  * requiring writeback.
178  *
179  * This number of dirtyable pages is the base value of which the
180  * user-configurable dirty ratio is the effictive number of pages that
181  * are allowed to be actually dirtied.  Per individual zone, or
182  * globally by using the sum of dirtyable pages over all zones.
183  *
184  * Because the user is allowed to specify the dirty limit globally as
185  * absolute number of bytes, calculating the per-zone dirty limit can
186  * require translating the configured limit into a percentage of
187  * global dirtyable memory first.
188  */
189 
190 static unsigned long highmem_dirtyable_memory(unsigned long total)
191 {
192 #ifdef CONFIG_HIGHMEM
193 	int node;
194 	unsigned long x = 0;
195 
196 	for_each_node_state(node, N_HIGH_MEMORY) {
197 		struct zone *z =
198 			&NODE_DATA(node)->node_zones[ZONE_HIGHMEM];
199 
200 		x += zone_page_state(z, NR_FREE_PAGES) +
201 		     zone_reclaimable_pages(z) - z->dirty_balance_reserve;
202 	}
203 	/*
204 	 * Unreclaimable memory (kernel memory or anonymous memory
205 	 * without swap) can bring down the dirtyable pages below
206 	 * the zone's dirty balance reserve and the above calculation
207 	 * will underflow.  However we still want to add in nodes
208 	 * which are below threshold (negative values) to get a more
209 	 * accurate calculation but make sure that the total never
210 	 * underflows.
211 	 */
212 	if ((long)x < 0)
213 		x = 0;
214 
215 	/*
216 	 * Make sure that the number of highmem pages is never larger
217 	 * than the number of the total dirtyable memory. This can only
218 	 * occur in very strange VM situations but we want to make sure
219 	 * that this does not occur.
220 	 */
221 	return min(x, total);
222 #else
223 	return 0;
224 #endif
225 }
226 
227 /**
228  * global_dirtyable_memory - number of globally dirtyable pages
229  *
230  * Returns the global number of pages potentially available for dirty
231  * page cache.  This is the base value for the global dirty limits.
232  */
233 static unsigned long global_dirtyable_memory(void)
234 {
235 	unsigned long x;
236 
237 	x = global_page_state(NR_FREE_PAGES) + global_reclaimable_pages();
238 	x -= min(x, dirty_balance_reserve);
239 
240 	if (!vm_highmem_is_dirtyable)
241 		x -= highmem_dirtyable_memory(x);
242 
243 	return x + 1;	/* Ensure that we never return 0 */
244 }
245 
246 /*
247  * global_dirty_limits - background-writeback and dirty-throttling thresholds
248  *
249  * Calculate the dirty thresholds based on sysctl parameters
250  * - vm.dirty_background_ratio  or  vm.dirty_background_bytes
251  * - vm.dirty_ratio             or  vm.dirty_bytes
252  * The dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and
253  * real-time tasks.
254  */
255 void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
256 {
257 	unsigned long background;
258 	unsigned long dirty;
259 	unsigned long uninitialized_var(available_memory);
260 	struct task_struct *tsk;
261 
262 	if (!vm_dirty_bytes || !dirty_background_bytes)
263 		available_memory = global_dirtyable_memory();
264 
265 	if (vm_dirty_bytes)
266 		dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE);
267 	else
268 		dirty = (vm_dirty_ratio * available_memory) / 100;
269 
270 	if (dirty_background_bytes)
271 		background = DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE);
272 	else
273 		background = (dirty_background_ratio * available_memory) / 100;
274 
275 	if (background >= dirty)
276 		background = dirty / 2;
277 	tsk = current;
278 	if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {
279 		background += background / 4;
280 		dirty += dirty / 4;
281 	}
282 	*pbackground = background;
283 	*pdirty = dirty;
284 	trace_global_dirty_state(background, dirty);
285 }
286 
287 /**
288  * zone_dirtyable_memory - number of dirtyable pages in a zone
289  * @zone: the zone
290  *
291  * Returns the zone's number of pages potentially available for dirty
292  * page cache.  This is the base value for the per-zone dirty limits.
293  */
294 static unsigned long zone_dirtyable_memory(struct zone *zone)
295 {
296 	/*
297 	 * The effective global number of dirtyable pages may exclude
298 	 * highmem as a big-picture measure to keep the ratio between
299 	 * dirty memory and lowmem reasonable.
300 	 *
301 	 * But this function is purely about the individual zone and a
302 	 * highmem zone can hold its share of dirty pages, so we don't
303 	 * care about vm_highmem_is_dirtyable here.
304 	 */
305 	unsigned long nr_pages = zone_page_state(zone, NR_FREE_PAGES) +
306 		zone_reclaimable_pages(zone);
307 
308 	/* don't allow this to underflow */
309 	nr_pages -= min(nr_pages, zone->dirty_balance_reserve);
310 	return nr_pages;
311 }
312 
313 /**
314  * zone_dirty_limit - maximum number of dirty pages allowed in a zone
315  * @zone: the zone
316  *
317  * Returns the maximum number of dirty pages allowed in a zone, based
318  * on the zone's dirtyable memory.
319  */
320 static unsigned long zone_dirty_limit(struct zone *zone)
321 {
322 	unsigned long zone_memory = zone_dirtyable_memory(zone);
323 	struct task_struct *tsk = current;
324 	unsigned long dirty;
325 
326 	if (vm_dirty_bytes)
327 		dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *
328 			zone_memory / global_dirtyable_memory();
329 	else
330 		dirty = vm_dirty_ratio * zone_memory / 100;
331 
332 	if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk))
333 		dirty += dirty / 4;
334 
335 	return dirty;
336 }
337 
338 /**
339  * zone_dirty_ok - tells whether a zone is within its dirty limits
340  * @zone: the zone to check
341  *
342  * Returns %true when the dirty pages in @zone are within the zone's
343  * dirty limit, %false if the limit is exceeded.
344  */
345 bool zone_dirty_ok(struct zone *zone)
346 {
347 	unsigned long limit = zone_dirty_limit(zone);
348 
349 	return zone_page_state(zone, NR_FILE_DIRTY) +
350 	       zone_page_state(zone, NR_UNSTABLE_NFS) +
351 	       zone_page_state(zone, NR_WRITEBACK) <= limit;
352 }
353 
354 int dirty_background_ratio_handler(struct ctl_table *table, int write,
355 		void __user *buffer, size_t *lenp,
356 		loff_t *ppos)
357 {
358 	int ret;
359 
360 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
361 	if (ret == 0 && write)
362 		dirty_background_bytes = 0;
363 	return ret;
364 }
365 
366 int dirty_background_bytes_handler(struct ctl_table *table, int write,
367 		void __user *buffer, size_t *lenp,
368 		loff_t *ppos)
369 {
370 	int ret;
371 
372 	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
373 	if (ret == 0 && write)
374 		dirty_background_ratio = 0;
375 	return ret;
376 }
377 
378 int dirty_ratio_handler(struct ctl_table *table, int write,
379 		void __user *buffer, size_t *lenp,
380 		loff_t *ppos)
381 {
382 	int old_ratio = vm_dirty_ratio;
383 	int ret;
384 
385 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
386 	if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
387 		writeback_set_ratelimit();
388 		vm_dirty_bytes = 0;
389 	}
390 	return ret;
391 }
392 
393 int dirty_bytes_handler(struct ctl_table *table, int write,
394 		void __user *buffer, size_t *lenp,
395 		loff_t *ppos)
396 {
397 	unsigned long old_bytes = vm_dirty_bytes;
398 	int ret;
399 
400 	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
401 	if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
402 		writeback_set_ratelimit();
403 		vm_dirty_ratio = 0;
404 	}
405 	return ret;
406 }
407 
408 static unsigned long wp_next_time(unsigned long cur_time)
409 {
410 	cur_time += VM_COMPLETIONS_PERIOD_LEN;
411 	/* 0 has a special meaning... */
412 	if (!cur_time)
413 		return 1;
414 	return cur_time;
415 }
416 
417 /*
418  * Increment the BDI's writeout completion count and the global writeout
419  * completion count. Called from test_clear_page_writeback().
420  */
421 static inline void __bdi_writeout_inc(struct backing_dev_info *bdi)
422 {
423 	__inc_bdi_stat(bdi, BDI_WRITTEN);
424 	__fprop_inc_percpu_max(&writeout_completions, &bdi->completions,
425 			       bdi->max_prop_frac);
426 	/* First event after period switching was turned off? */
427 	if (!unlikely(writeout_period_time)) {
428 		/*
429 		 * We can race with other __bdi_writeout_inc calls here but
430 		 * it does not cause any harm since the resulting time when
431 		 * timer will fire and what is in writeout_period_time will be
432 		 * roughly the same.
433 		 */
434 		writeout_period_time = wp_next_time(jiffies);
435 		mod_timer(&writeout_period_timer, writeout_period_time);
436 	}
437 }
438 
439 void bdi_writeout_inc(struct backing_dev_info *bdi)
440 {
441 	unsigned long flags;
442 
443 	local_irq_save(flags);
444 	__bdi_writeout_inc(bdi);
445 	local_irq_restore(flags);
446 }
447 EXPORT_SYMBOL_GPL(bdi_writeout_inc);
448 
449 /*
450  * Obtain an accurate fraction of the BDI's portion.
451  */
452 static void bdi_writeout_fraction(struct backing_dev_info *bdi,
453 		long *numerator, long *denominator)
454 {
455 	fprop_fraction_percpu(&writeout_completions, &bdi->completions,
456 				numerator, denominator);
457 }
458 
459 /*
460  * On idle system, we can be called long after we scheduled because we use
461  * deferred timers so count with missed periods.
462  */
463 static void writeout_period(unsigned long t)
464 {
465 	int miss_periods = (jiffies - writeout_period_time) /
466 						 VM_COMPLETIONS_PERIOD_LEN;
467 
468 	if (fprop_new_period(&writeout_completions, miss_periods + 1)) {
469 		writeout_period_time = wp_next_time(writeout_period_time +
470 				miss_periods * VM_COMPLETIONS_PERIOD_LEN);
471 		mod_timer(&writeout_period_timer, writeout_period_time);
472 	} else {
473 		/*
474 		 * Aging has zeroed all fractions. Stop wasting CPU on period
475 		 * updates.
476 		 */
477 		writeout_period_time = 0;
478 	}
479 }
480 
481 /*
482  * bdi_min_ratio keeps the sum of the minimum dirty shares of all
483  * registered backing devices, which, for obvious reasons, can not
484  * exceed 100%.
485  */
486 static unsigned int bdi_min_ratio;
487 
488 int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
489 {
490 	int ret = 0;
491 
492 	spin_lock_bh(&bdi_lock);
493 	if (min_ratio > bdi->max_ratio) {
494 		ret = -EINVAL;
495 	} else {
496 		min_ratio -= bdi->min_ratio;
497 		if (bdi_min_ratio + min_ratio < 100) {
498 			bdi_min_ratio += min_ratio;
499 			bdi->min_ratio += min_ratio;
500 		} else {
501 			ret = -EINVAL;
502 		}
503 	}
504 	spin_unlock_bh(&bdi_lock);
505 
506 	return ret;
507 }
508 
509 int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
510 {
511 	int ret = 0;
512 
513 	if (max_ratio > 100)
514 		return -EINVAL;
515 
516 	spin_lock_bh(&bdi_lock);
517 	if (bdi->min_ratio > max_ratio) {
518 		ret = -EINVAL;
519 	} else {
520 		bdi->max_ratio = max_ratio;
521 		bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100;
522 	}
523 	spin_unlock_bh(&bdi_lock);
524 
525 	return ret;
526 }
527 EXPORT_SYMBOL(bdi_set_max_ratio);
528 
529 static unsigned long dirty_freerun_ceiling(unsigned long thresh,
530 					   unsigned long bg_thresh)
531 {
532 	return (thresh + bg_thresh) / 2;
533 }
534 
535 static unsigned long hard_dirty_limit(unsigned long thresh)
536 {
537 	return max(thresh, global_dirty_limit);
538 }
539 
540 /**
541  * bdi_dirty_limit - @bdi's share of dirty throttling threshold
542  * @bdi: the backing_dev_info to query
543  * @dirty: global dirty limit in pages
544  *
545  * Returns @bdi's dirty limit in pages. The term "dirty" in the context of
546  * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages.
547  *
548  * Note that balance_dirty_pages() will only seriously take it as a hard limit
549  * when sleeping max_pause per page is not enough to keep the dirty pages under
550  * control. For example, when the device is completely stalled due to some error
551  * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key.
552  * In the other normal situations, it acts more gently by throttling the tasks
553  * more (rather than completely block them) when the bdi dirty pages go high.
554  *
555  * It allocates high/low dirty limits to fast/slow devices, in order to prevent
556  * - starving fast devices
557  * - piling up dirty pages (that will take long time to sync) on slow devices
558  *
559  * The bdi's share of dirty limit will be adapting to its throughput and
560  * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
561  */
562 unsigned long bdi_dirty_limit(struct backing_dev_info *bdi, unsigned long dirty)
563 {
564 	u64 bdi_dirty;
565 	long numerator, denominator;
566 
567 	/*
568 	 * Calculate this BDI's share of the dirty ratio.
569 	 */
570 	bdi_writeout_fraction(bdi, &numerator, &denominator);
571 
572 	bdi_dirty = (dirty * (100 - bdi_min_ratio)) / 100;
573 	bdi_dirty *= numerator;
574 	do_div(bdi_dirty, denominator);
575 
576 	bdi_dirty += (dirty * bdi->min_ratio) / 100;
577 	if (bdi_dirty > (dirty * bdi->max_ratio) / 100)
578 		bdi_dirty = dirty * bdi->max_ratio / 100;
579 
580 	return bdi_dirty;
581 }
582 
583 /*
584  * Dirty position control.
585  *
586  * (o) global/bdi setpoints
587  *
588  * We want the dirty pages be balanced around the global/bdi setpoints.
589  * When the number of dirty pages is higher/lower than the setpoint, the
590  * dirty position control ratio (and hence task dirty ratelimit) will be
591  * decreased/increased to bring the dirty pages back to the setpoint.
592  *
593  *     pos_ratio = 1 << RATELIMIT_CALC_SHIFT
594  *
595  *     if (dirty < setpoint) scale up   pos_ratio
596  *     if (dirty > setpoint) scale down pos_ratio
597  *
598  *     if (bdi_dirty < bdi_setpoint) scale up   pos_ratio
599  *     if (bdi_dirty > bdi_setpoint) scale down pos_ratio
600  *
601  *     task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
602  *
603  * (o) global control line
604  *
605  *     ^ pos_ratio
606  *     |
607  *     |            |<===== global dirty control scope ======>|
608  * 2.0 .............*
609  *     |            .*
610  *     |            . *
611  *     |            .   *
612  *     |            .     *
613  *     |            .        *
614  *     |            .            *
615  * 1.0 ................................*
616  *     |            .                  .     *
617  *     |            .                  .          *
618  *     |            .                  .              *
619  *     |            .                  .                 *
620  *     |            .                  .                    *
621  *   0 +------------.------------------.----------------------*------------->
622  *           freerun^          setpoint^                 limit^   dirty pages
623  *
624  * (o) bdi control line
625  *
626  *     ^ pos_ratio
627  *     |
628  *     |            *
629  *     |              *
630  *     |                *
631  *     |                  *
632  *     |                    * |<=========== span ============>|
633  * 1.0 .......................*
634  *     |                      . *
635  *     |                      .   *
636  *     |                      .     *
637  *     |                      .       *
638  *     |                      .         *
639  *     |                      .           *
640  *     |                      .             *
641  *     |                      .               *
642  *     |                      .                 *
643  *     |                      .                   *
644  *     |                      .                     *
645  * 1/4 ...............................................* * * * * * * * * * * *
646  *     |                      .                         .
647  *     |                      .                           .
648  *     |                      .                             .
649  *   0 +----------------------.-------------------------------.------------->
650  *                bdi_setpoint^                    x_intercept^
651  *
652  * The bdi control line won't drop below pos_ratio=1/4, so that bdi_dirty can
653  * be smoothly throttled down to normal if it starts high in situations like
654  * - start writing to a slow SD card and a fast disk at the same time. The SD
655  *   card's bdi_dirty may rush to many times higher than bdi_setpoint.
656  * - the bdi dirty thresh drops quickly due to change of JBOD workload
657  */
658 static unsigned long bdi_position_ratio(struct backing_dev_info *bdi,
659 					unsigned long thresh,
660 					unsigned long bg_thresh,
661 					unsigned long dirty,
662 					unsigned long bdi_thresh,
663 					unsigned long bdi_dirty)
664 {
665 	unsigned long write_bw = bdi->avg_write_bandwidth;
666 	unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);
667 	unsigned long limit = hard_dirty_limit(thresh);
668 	unsigned long x_intercept;
669 	unsigned long setpoint;		/* dirty pages' target balance point */
670 	unsigned long bdi_setpoint;
671 	unsigned long span;
672 	long long pos_ratio;		/* for scaling up/down the rate limit */
673 	long x;
674 
675 	if (unlikely(dirty >= limit))
676 		return 0;
677 
678 	/*
679 	 * global setpoint
680 	 *
681 	 *                           setpoint - dirty 3
682 	 *        f(dirty) := 1.0 + (----------------)
683 	 *                           limit - setpoint
684 	 *
685 	 * it's a 3rd order polynomial that subjects to
686 	 *
687 	 * (1) f(freerun)  = 2.0 => rampup dirty_ratelimit reasonably fast
688 	 * (2) f(setpoint) = 1.0 => the balance point
689 	 * (3) f(limit)    = 0   => the hard limit
690 	 * (4) df/dx      <= 0	 => negative feedback control
691 	 * (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
692 	 *     => fast response on large errors; small oscillation near setpoint
693 	 */
694 	setpoint = (freerun + limit) / 2;
695 	x = div_s64((setpoint - dirty) << RATELIMIT_CALC_SHIFT,
696 		    limit - setpoint + 1);
697 	pos_ratio = x;
698 	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
699 	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
700 	pos_ratio += 1 << RATELIMIT_CALC_SHIFT;
701 
702 	/*
703 	 * We have computed basic pos_ratio above based on global situation. If
704 	 * the bdi is over/under its share of dirty pages, we want to scale
705 	 * pos_ratio further down/up. That is done by the following mechanism.
706 	 */
707 
708 	/*
709 	 * bdi setpoint
710 	 *
711 	 *        f(bdi_dirty) := 1.0 + k * (bdi_dirty - bdi_setpoint)
712 	 *
713 	 *                        x_intercept - bdi_dirty
714 	 *                     := --------------------------
715 	 *                        x_intercept - bdi_setpoint
716 	 *
717 	 * The main bdi control line is a linear function that subjects to
718 	 *
719 	 * (1) f(bdi_setpoint) = 1.0
720 	 * (2) k = - 1 / (8 * write_bw)  (in single bdi case)
721 	 *     or equally: x_intercept = bdi_setpoint + 8 * write_bw
722 	 *
723 	 * For single bdi case, the dirty pages are observed to fluctuate
724 	 * regularly within range
725 	 *        [bdi_setpoint - write_bw/2, bdi_setpoint + write_bw/2]
726 	 * for various filesystems, where (2) can yield in a reasonable 12.5%
727 	 * fluctuation range for pos_ratio.
728 	 *
729 	 * For JBOD case, bdi_thresh (not bdi_dirty!) could fluctuate up to its
730 	 * own size, so move the slope over accordingly and choose a slope that
731 	 * yields 100% pos_ratio fluctuation on suddenly doubled bdi_thresh.
732 	 */
733 	if (unlikely(bdi_thresh > thresh))
734 		bdi_thresh = thresh;
735 	/*
736 	 * It's very possible that bdi_thresh is close to 0 not because the
737 	 * device is slow, but that it has remained inactive for long time.
738 	 * Honour such devices a reasonable good (hopefully IO efficient)
739 	 * threshold, so that the occasional writes won't be blocked and active
740 	 * writes can rampup the threshold quickly.
741 	 */
742 	bdi_thresh = max(bdi_thresh, (limit - dirty) / 8);
743 	/*
744 	 * scale global setpoint to bdi's:
745 	 *	bdi_setpoint = setpoint * bdi_thresh / thresh
746 	 */
747 	x = div_u64((u64)bdi_thresh << 16, thresh + 1);
748 	bdi_setpoint = setpoint * (u64)x >> 16;
749 	/*
750 	 * Use span=(8*write_bw) in single bdi case as indicated by
751 	 * (thresh - bdi_thresh ~= 0) and transit to bdi_thresh in JBOD case.
752 	 *
753 	 *        bdi_thresh                    thresh - bdi_thresh
754 	 * span = ---------- * (8 * write_bw) + ------------------- * bdi_thresh
755 	 *          thresh                            thresh
756 	 */
757 	span = (thresh - bdi_thresh + 8 * write_bw) * (u64)x >> 16;
758 	x_intercept = bdi_setpoint + span;
759 
760 	if (bdi_dirty < x_intercept - span / 4) {
761 		pos_ratio = div_u64(pos_ratio * (x_intercept - bdi_dirty),
762 				    x_intercept - bdi_setpoint + 1);
763 	} else
764 		pos_ratio /= 4;
765 
766 	/*
767 	 * bdi reserve area, safeguard against dirty pool underrun and disk idle
768 	 * It may push the desired control point of global dirty pages higher
769 	 * than setpoint.
770 	 */
771 	x_intercept = bdi_thresh / 2;
772 	if (bdi_dirty < x_intercept) {
773 		if (bdi_dirty > x_intercept / 8)
774 			pos_ratio = div_u64(pos_ratio * x_intercept, bdi_dirty);
775 		else
776 			pos_ratio *= 8;
777 	}
778 
779 	return pos_ratio;
780 }
781 
782 static void bdi_update_write_bandwidth(struct backing_dev_info *bdi,
783 				       unsigned long elapsed,
784 				       unsigned long written)
785 {
786 	const unsigned long period = roundup_pow_of_two(3 * HZ);
787 	unsigned long avg = bdi->avg_write_bandwidth;
788 	unsigned long old = bdi->write_bandwidth;
789 	u64 bw;
790 
791 	/*
792 	 * bw = written * HZ / elapsed
793 	 *
794 	 *                   bw * elapsed + write_bandwidth * (period - elapsed)
795 	 * write_bandwidth = ---------------------------------------------------
796 	 *                                          period
797 	 */
798 	bw = written - bdi->written_stamp;
799 	bw *= HZ;
800 	if (unlikely(elapsed > period)) {
801 		do_div(bw, elapsed);
802 		avg = bw;
803 		goto out;
804 	}
805 	bw += (u64)bdi->write_bandwidth * (period - elapsed);
806 	bw >>= ilog2(period);
807 
808 	/*
809 	 * one more level of smoothing, for filtering out sudden spikes
810 	 */
811 	if (avg > old && old >= (unsigned long)bw)
812 		avg -= (avg - old) >> 3;
813 
814 	if (avg < old && old <= (unsigned long)bw)
815 		avg += (old - avg) >> 3;
816 
817 out:
818 	bdi->write_bandwidth = bw;
819 	bdi->avg_write_bandwidth = avg;
820 }
821 
822 /*
823  * The global dirtyable memory and dirty threshold could be suddenly knocked
824  * down by a large amount (eg. on the startup of KVM in a swapless system).
825  * This may throw the system into deep dirty exceeded state and throttle
826  * heavy/light dirtiers alike. To retain good responsiveness, maintain
827  * global_dirty_limit for tracking slowly down to the knocked down dirty
828  * threshold.
829  */
830 static void update_dirty_limit(unsigned long thresh, unsigned long dirty)
831 {
832 	unsigned long limit = global_dirty_limit;
833 
834 	/*
835 	 * Follow up in one step.
836 	 */
837 	if (limit < thresh) {
838 		limit = thresh;
839 		goto update;
840 	}
841 
842 	/*
843 	 * Follow down slowly. Use the higher one as the target, because thresh
844 	 * may drop below dirty. This is exactly the reason to introduce
845 	 * global_dirty_limit which is guaranteed to lie above the dirty pages.
846 	 */
847 	thresh = max(thresh, dirty);
848 	if (limit > thresh) {
849 		limit -= (limit - thresh) >> 5;
850 		goto update;
851 	}
852 	return;
853 update:
854 	global_dirty_limit = limit;
855 }
856 
857 static void global_update_bandwidth(unsigned long thresh,
858 				    unsigned long dirty,
859 				    unsigned long now)
860 {
861 	static DEFINE_SPINLOCK(dirty_lock);
862 	static unsigned long update_time;
863 
864 	/*
865 	 * check locklessly first to optimize away locking for the most time
866 	 */
867 	if (time_before(now, update_time + BANDWIDTH_INTERVAL))
868 		return;
869 
870 	spin_lock(&dirty_lock);
871 	if (time_after_eq(now, update_time + BANDWIDTH_INTERVAL)) {
872 		update_dirty_limit(thresh, dirty);
873 		update_time = now;
874 	}
875 	spin_unlock(&dirty_lock);
876 }
877 
878 /*
879  * Maintain bdi->dirty_ratelimit, the base dirty throttle rate.
880  *
881  * Normal bdi tasks will be curbed at or below it in long term.
882  * Obviously it should be around (write_bw / N) when there are N dd tasks.
883  */
884 static void bdi_update_dirty_ratelimit(struct backing_dev_info *bdi,
885 				       unsigned long thresh,
886 				       unsigned long bg_thresh,
887 				       unsigned long dirty,
888 				       unsigned long bdi_thresh,
889 				       unsigned long bdi_dirty,
890 				       unsigned long dirtied,
891 				       unsigned long elapsed)
892 {
893 	unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);
894 	unsigned long limit = hard_dirty_limit(thresh);
895 	unsigned long setpoint = (freerun + limit) / 2;
896 	unsigned long write_bw = bdi->avg_write_bandwidth;
897 	unsigned long dirty_ratelimit = bdi->dirty_ratelimit;
898 	unsigned long dirty_rate;
899 	unsigned long task_ratelimit;
900 	unsigned long balanced_dirty_ratelimit;
901 	unsigned long pos_ratio;
902 	unsigned long step;
903 	unsigned long x;
904 
905 	/*
906 	 * The dirty rate will match the writeout rate in long term, except
907 	 * when dirty pages are truncated by userspace or re-dirtied by FS.
908 	 */
909 	dirty_rate = (dirtied - bdi->dirtied_stamp) * HZ / elapsed;
910 
911 	pos_ratio = bdi_position_ratio(bdi, thresh, bg_thresh, dirty,
912 				       bdi_thresh, bdi_dirty);
913 	/*
914 	 * task_ratelimit reflects each dd's dirty rate for the past 200ms.
915 	 */
916 	task_ratelimit = (u64)dirty_ratelimit *
917 					pos_ratio >> RATELIMIT_CALC_SHIFT;
918 	task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */
919 
920 	/*
921 	 * A linear estimation of the "balanced" throttle rate. The theory is,
922 	 * if there are N dd tasks, each throttled at task_ratelimit, the bdi's
923 	 * dirty_rate will be measured to be (N * task_ratelimit). So the below
924 	 * formula will yield the balanced rate limit (write_bw / N).
925 	 *
926 	 * Note that the expanded form is not a pure rate feedback:
927 	 *	rate_(i+1) = rate_(i) * (write_bw / dirty_rate)		     (1)
928 	 * but also takes pos_ratio into account:
929 	 *	rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio  (2)
930 	 *
931 	 * (1) is not realistic because pos_ratio also takes part in balancing
932 	 * the dirty rate.  Consider the state
933 	 *	pos_ratio = 0.5						     (3)
934 	 *	rate = 2 * (write_bw / N)				     (4)
935 	 * If (1) is used, it will stuck in that state! Because each dd will
936 	 * be throttled at
937 	 *	task_ratelimit = pos_ratio * rate = (write_bw / N)	     (5)
938 	 * yielding
939 	 *	dirty_rate = N * task_ratelimit = write_bw		     (6)
940 	 * put (6) into (1) we get
941 	 *	rate_(i+1) = rate_(i)					     (7)
942 	 *
943 	 * So we end up using (2) to always keep
944 	 *	rate_(i+1) ~= (write_bw / N)				     (8)
945 	 * regardless of the value of pos_ratio. As long as (8) is satisfied,
946 	 * pos_ratio is able to drive itself to 1.0, which is not only where
947 	 * the dirty count meet the setpoint, but also where the slope of
948 	 * pos_ratio is most flat and hence task_ratelimit is least fluctuated.
949 	 */
950 	balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,
951 					   dirty_rate | 1);
952 	/*
953 	 * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw
954 	 */
955 	if (unlikely(balanced_dirty_ratelimit > write_bw))
956 		balanced_dirty_ratelimit = write_bw;
957 
958 	/*
959 	 * We could safely do this and return immediately:
960 	 *
961 	 *	bdi->dirty_ratelimit = balanced_dirty_ratelimit;
962 	 *
963 	 * However to get a more stable dirty_ratelimit, the below elaborated
964 	 * code makes use of task_ratelimit to filter out singular points and
965 	 * limit the step size.
966 	 *
967 	 * The below code essentially only uses the relative value of
968 	 *
969 	 *	task_ratelimit - dirty_ratelimit
970 	 *	= (pos_ratio - 1) * dirty_ratelimit
971 	 *
972 	 * which reflects the direction and size of dirty position error.
973 	 */
974 
975 	/*
976 	 * dirty_ratelimit will follow balanced_dirty_ratelimit iff
977 	 * task_ratelimit is on the same side of dirty_ratelimit, too.
978 	 * For example, when
979 	 * - dirty_ratelimit > balanced_dirty_ratelimit
980 	 * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint)
981 	 * lowering dirty_ratelimit will help meet both the position and rate
982 	 * control targets. Otherwise, don't update dirty_ratelimit if it will
983 	 * only help meet the rate target. After all, what the users ultimately
984 	 * feel and care are stable dirty rate and small position error.
985 	 *
986 	 * |task_ratelimit - dirty_ratelimit| is used to limit the step size
987 	 * and filter out the singular points of balanced_dirty_ratelimit. Which
988 	 * keeps jumping around randomly and can even leap far away at times
989 	 * due to the small 200ms estimation period of dirty_rate (we want to
990 	 * keep that period small to reduce time lags).
991 	 */
992 	step = 0;
993 	if (dirty < setpoint) {
994 		x = min(bdi->balanced_dirty_ratelimit,
995 			 min(balanced_dirty_ratelimit, task_ratelimit));
996 		if (dirty_ratelimit < x)
997 			step = x - dirty_ratelimit;
998 	} else {
999 		x = max(bdi->balanced_dirty_ratelimit,
1000 			 max(balanced_dirty_ratelimit, task_ratelimit));
1001 		if (dirty_ratelimit > x)
1002 			step = dirty_ratelimit - x;
1003 	}
1004 
1005 	/*
1006 	 * Don't pursue 100% rate matching. It's impossible since the balanced
1007 	 * rate itself is constantly fluctuating. So decrease the track speed
1008 	 * when it gets close to the target. Helps eliminate pointless tremors.
1009 	 */
1010 	step >>= dirty_ratelimit / (2 * step + 1);
1011 	/*
1012 	 * Limit the tracking speed to avoid overshooting.
1013 	 */
1014 	step = (step + 7) / 8;
1015 
1016 	if (dirty_ratelimit < balanced_dirty_ratelimit)
1017 		dirty_ratelimit += step;
1018 	else
1019 		dirty_ratelimit -= step;
1020 
1021 	bdi->dirty_ratelimit = max(dirty_ratelimit, 1UL);
1022 	bdi->balanced_dirty_ratelimit = balanced_dirty_ratelimit;
1023 
1024 	trace_bdi_dirty_ratelimit(bdi, dirty_rate, task_ratelimit);
1025 }
1026 
1027 void __bdi_update_bandwidth(struct backing_dev_info *bdi,
1028 			    unsigned long thresh,
1029 			    unsigned long bg_thresh,
1030 			    unsigned long dirty,
1031 			    unsigned long bdi_thresh,
1032 			    unsigned long bdi_dirty,
1033 			    unsigned long start_time)
1034 {
1035 	unsigned long now = jiffies;
1036 	unsigned long elapsed = now - bdi->bw_time_stamp;
1037 	unsigned long dirtied;
1038 	unsigned long written;
1039 
1040 	/*
1041 	 * rate-limit, only update once every 200ms.
1042 	 */
1043 	if (elapsed < BANDWIDTH_INTERVAL)
1044 		return;
1045 
1046 	dirtied = percpu_counter_read(&bdi->bdi_stat[BDI_DIRTIED]);
1047 	written = percpu_counter_read(&bdi->bdi_stat[BDI_WRITTEN]);
1048 
1049 	/*
1050 	 * Skip quiet periods when disk bandwidth is under-utilized.
1051 	 * (at least 1s idle time between two flusher runs)
1052 	 */
1053 	if (elapsed > HZ && time_before(bdi->bw_time_stamp, start_time))
1054 		goto snapshot;
1055 
1056 	if (thresh) {
1057 		global_update_bandwidth(thresh, dirty, now);
1058 		bdi_update_dirty_ratelimit(bdi, thresh, bg_thresh, dirty,
1059 					   bdi_thresh, bdi_dirty,
1060 					   dirtied, elapsed);
1061 	}
1062 	bdi_update_write_bandwidth(bdi, elapsed, written);
1063 
1064 snapshot:
1065 	bdi->dirtied_stamp = dirtied;
1066 	bdi->written_stamp = written;
1067 	bdi->bw_time_stamp = now;
1068 }
1069 
1070 static void bdi_update_bandwidth(struct backing_dev_info *bdi,
1071 				 unsigned long thresh,
1072 				 unsigned long bg_thresh,
1073 				 unsigned long dirty,
1074 				 unsigned long bdi_thresh,
1075 				 unsigned long bdi_dirty,
1076 				 unsigned long start_time)
1077 {
1078 	if (time_is_after_eq_jiffies(bdi->bw_time_stamp + BANDWIDTH_INTERVAL))
1079 		return;
1080 	spin_lock(&bdi->wb.list_lock);
1081 	__bdi_update_bandwidth(bdi, thresh, bg_thresh, dirty,
1082 			       bdi_thresh, bdi_dirty, start_time);
1083 	spin_unlock(&bdi->wb.list_lock);
1084 }
1085 
1086 /*
1087  * After a task dirtied this many pages, balance_dirty_pages_ratelimited()
1088  * will look to see if it needs to start dirty throttling.
1089  *
1090  * If dirty_poll_interval is too low, big NUMA machines will call the expensive
1091  * global_page_state() too often. So scale it near-sqrt to the safety margin
1092  * (the number of pages we may dirty without exceeding the dirty limits).
1093  */
1094 static unsigned long dirty_poll_interval(unsigned long dirty,
1095 					 unsigned long thresh)
1096 {
1097 	if (thresh > dirty)
1098 		return 1UL << (ilog2(thresh - dirty) >> 1);
1099 
1100 	return 1;
1101 }
1102 
1103 static long bdi_max_pause(struct backing_dev_info *bdi,
1104 			  unsigned long bdi_dirty)
1105 {
1106 	long bw = bdi->avg_write_bandwidth;
1107 	long t;
1108 
1109 	/*
1110 	 * Limit pause time for small memory systems. If sleeping for too long
1111 	 * time, a small pool of dirty/writeback pages may go empty and disk go
1112 	 * idle.
1113 	 *
1114 	 * 8 serves as the safety ratio.
1115 	 */
1116 	t = bdi_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8));
1117 	t++;
1118 
1119 	return min_t(long, t, MAX_PAUSE);
1120 }
1121 
1122 static long bdi_min_pause(struct backing_dev_info *bdi,
1123 			  long max_pause,
1124 			  unsigned long task_ratelimit,
1125 			  unsigned long dirty_ratelimit,
1126 			  int *nr_dirtied_pause)
1127 {
1128 	long hi = ilog2(bdi->avg_write_bandwidth);
1129 	long lo = ilog2(bdi->dirty_ratelimit);
1130 	long t;		/* target pause */
1131 	long pause;	/* estimated next pause */
1132 	int pages;	/* target nr_dirtied_pause */
1133 
1134 	/* target for 10ms pause on 1-dd case */
1135 	t = max(1, HZ / 100);
1136 
1137 	/*
1138 	 * Scale up pause time for concurrent dirtiers in order to reduce CPU
1139 	 * overheads.
1140 	 *
1141 	 * (N * 10ms) on 2^N concurrent tasks.
1142 	 */
1143 	if (hi > lo)
1144 		t += (hi - lo) * (10 * HZ) / 1024;
1145 
1146 	/*
1147 	 * This is a bit convoluted. We try to base the next nr_dirtied_pause
1148 	 * on the much more stable dirty_ratelimit. However the next pause time
1149 	 * will be computed based on task_ratelimit and the two rate limits may
1150 	 * depart considerably at some time. Especially if task_ratelimit goes
1151 	 * below dirty_ratelimit/2 and the target pause is max_pause, the next
1152 	 * pause time will be max_pause*2 _trimmed down_ to max_pause.  As a
1153 	 * result task_ratelimit won't be executed faithfully, which could
1154 	 * eventually bring down dirty_ratelimit.
1155 	 *
1156 	 * We apply two rules to fix it up:
1157 	 * 1) try to estimate the next pause time and if necessary, use a lower
1158 	 *    nr_dirtied_pause so as not to exceed max_pause. When this happens,
1159 	 *    nr_dirtied_pause will be "dancing" with task_ratelimit.
1160 	 * 2) limit the target pause time to max_pause/2, so that the normal
1161 	 *    small fluctuations of task_ratelimit won't trigger rule (1) and
1162 	 *    nr_dirtied_pause will remain as stable as dirty_ratelimit.
1163 	 */
1164 	t = min(t, 1 + max_pause / 2);
1165 	pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
1166 
1167 	/*
1168 	 * Tiny nr_dirtied_pause is found to hurt I/O performance in the test
1169 	 * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}.
1170 	 * When the 16 consecutive reads are often interrupted by some dirty
1171 	 * throttling pause during the async writes, cfq will go into idles
1172 	 * (deadline is fine). So push nr_dirtied_pause as high as possible
1173 	 * until reaches DIRTY_POLL_THRESH=32 pages.
1174 	 */
1175 	if (pages < DIRTY_POLL_THRESH) {
1176 		t = max_pause;
1177 		pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
1178 		if (pages > DIRTY_POLL_THRESH) {
1179 			pages = DIRTY_POLL_THRESH;
1180 			t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit;
1181 		}
1182 	}
1183 
1184 	pause = HZ * pages / (task_ratelimit + 1);
1185 	if (pause > max_pause) {
1186 		t = max_pause;
1187 		pages = task_ratelimit * t / roundup_pow_of_two(HZ);
1188 	}
1189 
1190 	*nr_dirtied_pause = pages;
1191 	/*
1192 	 * The minimal pause time will normally be half the target pause time.
1193 	 */
1194 	return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t;
1195 }
1196 
1197 /*
1198  * balance_dirty_pages() must be called by processes which are generating dirty
1199  * data.  It looks at the number of dirty pages in the machine and will force
1200  * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2.
1201  * If we're over `background_thresh' then the writeback threads are woken to
1202  * perform some writeout.
1203  */
1204 static void balance_dirty_pages(struct address_space *mapping,
1205 				unsigned long pages_dirtied)
1206 {
1207 	unsigned long nr_reclaimable;	/* = file_dirty + unstable_nfs */
1208 	unsigned long bdi_reclaimable;
1209 	unsigned long nr_dirty;  /* = file_dirty + writeback + unstable_nfs */
1210 	unsigned long bdi_dirty;
1211 	unsigned long freerun;
1212 	unsigned long background_thresh;
1213 	unsigned long dirty_thresh;
1214 	unsigned long bdi_thresh;
1215 	long period;
1216 	long pause;
1217 	long max_pause;
1218 	long min_pause;
1219 	int nr_dirtied_pause;
1220 	bool dirty_exceeded = false;
1221 	unsigned long task_ratelimit;
1222 	unsigned long dirty_ratelimit;
1223 	unsigned long pos_ratio;
1224 	struct backing_dev_info *bdi = mapping->backing_dev_info;
1225 	unsigned long start_time = jiffies;
1226 
1227 	for (;;) {
1228 		unsigned long now = jiffies;
1229 
1230 		/*
1231 		 * Unstable writes are a feature of certain networked
1232 		 * filesystems (i.e. NFS) in which data may have been
1233 		 * written to the server's write cache, but has not yet
1234 		 * been flushed to permanent storage.
1235 		 */
1236 		nr_reclaimable = global_page_state(NR_FILE_DIRTY) +
1237 					global_page_state(NR_UNSTABLE_NFS);
1238 		nr_dirty = nr_reclaimable + global_page_state(NR_WRITEBACK);
1239 
1240 		global_dirty_limits(&background_thresh, &dirty_thresh);
1241 
1242 		/*
1243 		 * Throttle it only when the background writeback cannot
1244 		 * catch-up. This avoids (excessively) small writeouts
1245 		 * when the bdi limits are ramping up.
1246 		 */
1247 		freerun = dirty_freerun_ceiling(dirty_thresh,
1248 						background_thresh);
1249 		if (nr_dirty <= freerun) {
1250 			current->dirty_paused_when = now;
1251 			current->nr_dirtied = 0;
1252 			current->nr_dirtied_pause =
1253 				dirty_poll_interval(nr_dirty, dirty_thresh);
1254 			break;
1255 		}
1256 
1257 		if (unlikely(!writeback_in_progress(bdi)))
1258 			bdi_start_background_writeback(bdi);
1259 
1260 		/*
1261 		 * bdi_thresh is not treated as some limiting factor as
1262 		 * dirty_thresh, due to reasons
1263 		 * - in JBOD setup, bdi_thresh can fluctuate a lot
1264 		 * - in a system with HDD and USB key, the USB key may somehow
1265 		 *   go into state (bdi_dirty >> bdi_thresh) either because
1266 		 *   bdi_dirty starts high, or because bdi_thresh drops low.
1267 		 *   In this case we don't want to hard throttle the USB key
1268 		 *   dirtiers for 100 seconds until bdi_dirty drops under
1269 		 *   bdi_thresh. Instead the auxiliary bdi control line in
1270 		 *   bdi_position_ratio() will let the dirtier task progress
1271 		 *   at some rate <= (write_bw / 2) for bringing down bdi_dirty.
1272 		 */
1273 		bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh);
1274 
1275 		/*
1276 		 * In order to avoid the stacked BDI deadlock we need
1277 		 * to ensure we accurately count the 'dirty' pages when
1278 		 * the threshold is low.
1279 		 *
1280 		 * Otherwise it would be possible to get thresh+n pages
1281 		 * reported dirty, even though there are thresh-m pages
1282 		 * actually dirty; with m+n sitting in the percpu
1283 		 * deltas.
1284 		 */
1285 		if (bdi_thresh < 2 * bdi_stat_error(bdi)) {
1286 			bdi_reclaimable = bdi_stat_sum(bdi, BDI_RECLAIMABLE);
1287 			bdi_dirty = bdi_reclaimable +
1288 				    bdi_stat_sum(bdi, BDI_WRITEBACK);
1289 		} else {
1290 			bdi_reclaimable = bdi_stat(bdi, BDI_RECLAIMABLE);
1291 			bdi_dirty = bdi_reclaimable +
1292 				    bdi_stat(bdi, BDI_WRITEBACK);
1293 		}
1294 
1295 		dirty_exceeded = (bdi_dirty > bdi_thresh) &&
1296 				  (nr_dirty > dirty_thresh);
1297 		if (dirty_exceeded && !bdi->dirty_exceeded)
1298 			bdi->dirty_exceeded = 1;
1299 
1300 		bdi_update_bandwidth(bdi, dirty_thresh, background_thresh,
1301 				     nr_dirty, bdi_thresh, bdi_dirty,
1302 				     start_time);
1303 
1304 		dirty_ratelimit = bdi->dirty_ratelimit;
1305 		pos_ratio = bdi_position_ratio(bdi, dirty_thresh,
1306 					       background_thresh, nr_dirty,
1307 					       bdi_thresh, bdi_dirty);
1308 		task_ratelimit = ((u64)dirty_ratelimit * pos_ratio) >>
1309 							RATELIMIT_CALC_SHIFT;
1310 		max_pause = bdi_max_pause(bdi, bdi_dirty);
1311 		min_pause = bdi_min_pause(bdi, max_pause,
1312 					  task_ratelimit, dirty_ratelimit,
1313 					  &nr_dirtied_pause);
1314 
1315 		if (unlikely(task_ratelimit == 0)) {
1316 			period = max_pause;
1317 			pause = max_pause;
1318 			goto pause;
1319 		}
1320 		period = HZ * pages_dirtied / task_ratelimit;
1321 		pause = period;
1322 		if (current->dirty_paused_when)
1323 			pause -= now - current->dirty_paused_when;
1324 		/*
1325 		 * For less than 1s think time (ext3/4 may block the dirtier
1326 		 * for up to 800ms from time to time on 1-HDD; so does xfs,
1327 		 * however at much less frequency), try to compensate it in
1328 		 * future periods by updating the virtual time; otherwise just
1329 		 * do a reset, as it may be a light dirtier.
1330 		 */
1331 		if (pause < min_pause) {
1332 			trace_balance_dirty_pages(bdi,
1333 						  dirty_thresh,
1334 						  background_thresh,
1335 						  nr_dirty,
1336 						  bdi_thresh,
1337 						  bdi_dirty,
1338 						  dirty_ratelimit,
1339 						  task_ratelimit,
1340 						  pages_dirtied,
1341 						  period,
1342 						  min(pause, 0L),
1343 						  start_time);
1344 			if (pause < -HZ) {
1345 				current->dirty_paused_when = now;
1346 				current->nr_dirtied = 0;
1347 			} else if (period) {
1348 				current->dirty_paused_when += period;
1349 				current->nr_dirtied = 0;
1350 			} else if (current->nr_dirtied_pause <= pages_dirtied)
1351 				current->nr_dirtied_pause += pages_dirtied;
1352 			break;
1353 		}
1354 		if (unlikely(pause > max_pause)) {
1355 			/* for occasional dropped task_ratelimit */
1356 			now += min(pause - max_pause, max_pause);
1357 			pause = max_pause;
1358 		}
1359 
1360 pause:
1361 		trace_balance_dirty_pages(bdi,
1362 					  dirty_thresh,
1363 					  background_thresh,
1364 					  nr_dirty,
1365 					  bdi_thresh,
1366 					  bdi_dirty,
1367 					  dirty_ratelimit,
1368 					  task_ratelimit,
1369 					  pages_dirtied,
1370 					  period,
1371 					  pause,
1372 					  start_time);
1373 		__set_current_state(TASK_KILLABLE);
1374 		io_schedule_timeout(pause);
1375 
1376 		current->dirty_paused_when = now + pause;
1377 		current->nr_dirtied = 0;
1378 		current->nr_dirtied_pause = nr_dirtied_pause;
1379 
1380 		/*
1381 		 * This is typically equal to (nr_dirty < dirty_thresh) and can
1382 		 * also keep "1000+ dd on a slow USB stick" under control.
1383 		 */
1384 		if (task_ratelimit)
1385 			break;
1386 
1387 		/*
1388 		 * In the case of an unresponding NFS server and the NFS dirty
1389 		 * pages exceeds dirty_thresh, give the other good bdi's a pipe
1390 		 * to go through, so that tasks on them still remain responsive.
1391 		 *
1392 		 * In theory 1 page is enough to keep the comsumer-producer
1393 		 * pipe going: the flusher cleans 1 page => the task dirties 1
1394 		 * more page. However bdi_dirty has accounting errors.  So use
1395 		 * the larger and more IO friendly bdi_stat_error.
1396 		 */
1397 		if (bdi_dirty <= bdi_stat_error(bdi))
1398 			break;
1399 
1400 		if (fatal_signal_pending(current))
1401 			break;
1402 	}
1403 
1404 	if (!dirty_exceeded && bdi->dirty_exceeded)
1405 		bdi->dirty_exceeded = 0;
1406 
1407 	if (writeback_in_progress(bdi))
1408 		return;
1409 
1410 	/*
1411 	 * In laptop mode, we wait until hitting the higher threshold before
1412 	 * starting background writeout, and then write out all the way down
1413 	 * to the lower threshold.  So slow writers cause minimal disk activity.
1414 	 *
1415 	 * In normal mode, we start background writeout at the lower
1416 	 * background_thresh, to keep the amount of dirty memory low.
1417 	 */
1418 	if (laptop_mode)
1419 		return;
1420 
1421 	if (nr_reclaimable > background_thresh)
1422 		bdi_start_background_writeback(bdi);
1423 }
1424 
1425 void set_page_dirty_balance(struct page *page, int page_mkwrite)
1426 {
1427 	if (set_page_dirty(page) || page_mkwrite) {
1428 		struct address_space *mapping = page_mapping(page);
1429 
1430 		if (mapping)
1431 			balance_dirty_pages_ratelimited(mapping);
1432 	}
1433 }
1434 
1435 static DEFINE_PER_CPU(int, bdp_ratelimits);
1436 
1437 /*
1438  * Normal tasks are throttled by
1439  *	loop {
1440  *		dirty tsk->nr_dirtied_pause pages;
1441  *		take a snap in balance_dirty_pages();
1442  *	}
1443  * However there is a worst case. If every task exit immediately when dirtied
1444  * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be
1445  * called to throttle the page dirties. The solution is to save the not yet
1446  * throttled page dirties in dirty_throttle_leaks on task exit and charge them
1447  * randomly into the running tasks. This works well for the above worst case,
1448  * as the new task will pick up and accumulate the old task's leaked dirty
1449  * count and eventually get throttled.
1450  */
1451 DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0;
1452 
1453 /**
1454  * balance_dirty_pages_ratelimited - balance dirty memory state
1455  * @mapping: address_space which was dirtied
1456  *
1457  * Processes which are dirtying memory should call in here once for each page
1458  * which was newly dirtied.  The function will periodically check the system's
1459  * dirty state and will initiate writeback if needed.
1460  *
1461  * On really big machines, get_writeback_state is expensive, so try to avoid
1462  * calling it too often (ratelimiting).  But once we're over the dirty memory
1463  * limit we decrease the ratelimiting by a lot, to prevent individual processes
1464  * from overshooting the limit by (ratelimit_pages) each.
1465  */
1466 void balance_dirty_pages_ratelimited(struct address_space *mapping)
1467 {
1468 	struct backing_dev_info *bdi = mapping->backing_dev_info;
1469 	int ratelimit;
1470 	int *p;
1471 
1472 	if (!bdi_cap_account_dirty(bdi))
1473 		return;
1474 
1475 	ratelimit = current->nr_dirtied_pause;
1476 	if (bdi->dirty_exceeded)
1477 		ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10));
1478 
1479 	preempt_disable();
1480 	/*
1481 	 * This prevents one CPU to accumulate too many dirtied pages without
1482 	 * calling into balance_dirty_pages(), which can happen when there are
1483 	 * 1000+ tasks, all of them start dirtying pages at exactly the same
1484 	 * time, hence all honoured too large initial task->nr_dirtied_pause.
1485 	 */
1486 	p =  &__get_cpu_var(bdp_ratelimits);
1487 	if (unlikely(current->nr_dirtied >= ratelimit))
1488 		*p = 0;
1489 	else if (unlikely(*p >= ratelimit_pages)) {
1490 		*p = 0;
1491 		ratelimit = 0;
1492 	}
1493 	/*
1494 	 * Pick up the dirtied pages by the exited tasks. This avoids lots of
1495 	 * short-lived tasks (eg. gcc invocations in a kernel build) escaping
1496 	 * the dirty throttling and livelock other long-run dirtiers.
1497 	 */
1498 	p = &__get_cpu_var(dirty_throttle_leaks);
1499 	if (*p > 0 && current->nr_dirtied < ratelimit) {
1500 		unsigned long nr_pages_dirtied;
1501 		nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied);
1502 		*p -= nr_pages_dirtied;
1503 		current->nr_dirtied += nr_pages_dirtied;
1504 	}
1505 	preempt_enable();
1506 
1507 	if (unlikely(current->nr_dirtied >= ratelimit))
1508 		balance_dirty_pages(mapping, current->nr_dirtied);
1509 }
1510 EXPORT_SYMBOL(balance_dirty_pages_ratelimited);
1511 
1512 void throttle_vm_writeout(gfp_t gfp_mask)
1513 {
1514 	unsigned long background_thresh;
1515 	unsigned long dirty_thresh;
1516 
1517         for ( ; ; ) {
1518 		global_dirty_limits(&background_thresh, &dirty_thresh);
1519 		dirty_thresh = hard_dirty_limit(dirty_thresh);
1520 
1521                 /*
1522                  * Boost the allowable dirty threshold a bit for page
1523                  * allocators so they don't get DoS'ed by heavy writers
1524                  */
1525                 dirty_thresh += dirty_thresh / 10;      /* wheeee... */
1526 
1527                 if (global_page_state(NR_UNSTABLE_NFS) +
1528 			global_page_state(NR_WRITEBACK) <= dirty_thresh)
1529                         	break;
1530                 congestion_wait(BLK_RW_ASYNC, HZ/10);
1531 
1532 		/*
1533 		 * The caller might hold locks which can prevent IO completion
1534 		 * or progress in the filesystem.  So we cannot just sit here
1535 		 * waiting for IO to complete.
1536 		 */
1537 		if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO))
1538 			break;
1539         }
1540 }
1541 
1542 /*
1543  * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
1544  */
1545 int dirty_writeback_centisecs_handler(ctl_table *table, int write,
1546 	void __user *buffer, size_t *length, loff_t *ppos)
1547 {
1548 	proc_dointvec(table, write, buffer, length, ppos);
1549 	return 0;
1550 }
1551 
1552 #ifdef CONFIG_BLOCK
1553 void laptop_mode_timer_fn(unsigned long data)
1554 {
1555 	struct request_queue *q = (struct request_queue *)data;
1556 	int nr_pages = global_page_state(NR_FILE_DIRTY) +
1557 		global_page_state(NR_UNSTABLE_NFS);
1558 
1559 	/*
1560 	 * We want to write everything out, not just down to the dirty
1561 	 * threshold
1562 	 */
1563 	if (bdi_has_dirty_io(&q->backing_dev_info))
1564 		bdi_start_writeback(&q->backing_dev_info, nr_pages,
1565 					WB_REASON_LAPTOP_TIMER);
1566 }
1567 
1568 /*
1569  * We've spun up the disk and we're in laptop mode: schedule writeback
1570  * of all dirty data a few seconds from now.  If the flush is already scheduled
1571  * then push it back - the user is still using the disk.
1572  */
1573 void laptop_io_completion(struct backing_dev_info *info)
1574 {
1575 	mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode);
1576 }
1577 
1578 /*
1579  * We're in laptop mode and we've just synced. The sync's writes will have
1580  * caused another writeback to be scheduled by laptop_io_completion.
1581  * Nothing needs to be written back anymore, so we unschedule the writeback.
1582  */
1583 void laptop_sync_completion(void)
1584 {
1585 	struct backing_dev_info *bdi;
1586 
1587 	rcu_read_lock();
1588 
1589 	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
1590 		del_timer(&bdi->laptop_mode_wb_timer);
1591 
1592 	rcu_read_unlock();
1593 }
1594 #endif
1595 
1596 /*
1597  * If ratelimit_pages is too high then we can get into dirty-data overload
1598  * if a large number of processes all perform writes at the same time.
1599  * If it is too low then SMP machines will call the (expensive)
1600  * get_writeback_state too often.
1601  *
1602  * Here we set ratelimit_pages to a level which ensures that when all CPUs are
1603  * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory
1604  * thresholds.
1605  */
1606 
1607 void writeback_set_ratelimit(void)
1608 {
1609 	unsigned long background_thresh;
1610 	unsigned long dirty_thresh;
1611 	global_dirty_limits(&background_thresh, &dirty_thresh);
1612 	global_dirty_limit = dirty_thresh;
1613 	ratelimit_pages = dirty_thresh / (num_online_cpus() * 32);
1614 	if (ratelimit_pages < 16)
1615 		ratelimit_pages = 16;
1616 }
1617 
1618 static int __cpuinit
1619 ratelimit_handler(struct notifier_block *self, unsigned long action,
1620 		  void *hcpu)
1621 {
1622 
1623 	switch (action & ~CPU_TASKS_FROZEN) {
1624 	case CPU_ONLINE:
1625 	case CPU_DEAD:
1626 		writeback_set_ratelimit();
1627 		return NOTIFY_OK;
1628 	default:
1629 		return NOTIFY_DONE;
1630 	}
1631 }
1632 
1633 static struct notifier_block __cpuinitdata ratelimit_nb = {
1634 	.notifier_call	= ratelimit_handler,
1635 	.next		= NULL,
1636 };
1637 
1638 /*
1639  * Called early on to tune the page writeback dirty limits.
1640  *
1641  * We used to scale dirty pages according to how total memory
1642  * related to pages that could be allocated for buffers (by
1643  * comparing nr_free_buffer_pages() to vm_total_pages.
1644  *
1645  * However, that was when we used "dirty_ratio" to scale with
1646  * all memory, and we don't do that any more. "dirty_ratio"
1647  * is now applied to total non-HIGHPAGE memory (by subtracting
1648  * totalhigh_pages from vm_total_pages), and as such we can't
1649  * get into the old insane situation any more where we had
1650  * large amounts of dirty pages compared to a small amount of
1651  * non-HIGHMEM memory.
1652  *
1653  * But we might still want to scale the dirty_ratio by how
1654  * much memory the box has..
1655  */
1656 void __init page_writeback_init(void)
1657 {
1658 	writeback_set_ratelimit();
1659 	register_cpu_notifier(&ratelimit_nb);
1660 
1661 	fprop_global_init(&writeout_completions);
1662 }
1663 
1664 /**
1665  * tag_pages_for_writeback - tag pages to be written by write_cache_pages
1666  * @mapping: address space structure to write
1667  * @start: starting page index
1668  * @end: ending page index (inclusive)
1669  *
1670  * This function scans the page range from @start to @end (inclusive) and tags
1671  * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is
1672  * that write_cache_pages (or whoever calls this function) will then use
1673  * TOWRITE tag to identify pages eligible for writeback.  This mechanism is
1674  * used to avoid livelocking of writeback by a process steadily creating new
1675  * dirty pages in the file (thus it is important for this function to be quick
1676  * so that it can tag pages faster than a dirtying process can create them).
1677  */
1678 /*
1679  * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency.
1680  */
1681 void tag_pages_for_writeback(struct address_space *mapping,
1682 			     pgoff_t start, pgoff_t end)
1683 {
1684 #define WRITEBACK_TAG_BATCH 4096
1685 	unsigned long tagged;
1686 
1687 	do {
1688 		spin_lock_irq(&mapping->tree_lock);
1689 		tagged = radix_tree_range_tag_if_tagged(&mapping->page_tree,
1690 				&start, end, WRITEBACK_TAG_BATCH,
1691 				PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
1692 		spin_unlock_irq(&mapping->tree_lock);
1693 		WARN_ON_ONCE(tagged > WRITEBACK_TAG_BATCH);
1694 		cond_resched();
1695 		/* We check 'start' to handle wrapping when end == ~0UL */
1696 	} while (tagged >= WRITEBACK_TAG_BATCH && start);
1697 }
1698 EXPORT_SYMBOL(tag_pages_for_writeback);
1699 
1700 /**
1701  * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
1702  * @mapping: address space structure to write
1703  * @wbc: subtract the number of written pages from *@wbc->nr_to_write
1704  * @writepage: function called for each page
1705  * @data: data passed to writepage function
1706  *
1707  * If a page is already under I/O, write_cache_pages() skips it, even
1708  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
1709  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
1710  * and msync() need to guarantee that all the data which was dirty at the time
1711  * the call was made get new I/O started against them.  If wbc->sync_mode is
1712  * WB_SYNC_ALL then we were called for data integrity and we must wait for
1713  * existing IO to complete.
1714  *
1715  * To avoid livelocks (when other process dirties new pages), we first tag
1716  * pages which should be written back with TOWRITE tag and only then start
1717  * writing them. For data-integrity sync we have to be careful so that we do
1718  * not miss some pages (e.g., because some other process has cleared TOWRITE
1719  * tag we set). The rule we follow is that TOWRITE tag can be cleared only
1720  * by the process clearing the DIRTY tag (and submitting the page for IO).
1721  */
1722 int write_cache_pages(struct address_space *mapping,
1723 		      struct writeback_control *wbc, writepage_t writepage,
1724 		      void *data)
1725 {
1726 	int ret = 0;
1727 	int done = 0;
1728 	struct pagevec pvec;
1729 	int nr_pages;
1730 	pgoff_t uninitialized_var(writeback_index);
1731 	pgoff_t index;
1732 	pgoff_t end;		/* Inclusive */
1733 	pgoff_t done_index;
1734 	int cycled;
1735 	int range_whole = 0;
1736 	int tag;
1737 
1738 	pagevec_init(&pvec, 0);
1739 	if (wbc->range_cyclic) {
1740 		writeback_index = mapping->writeback_index; /* prev offset */
1741 		index = writeback_index;
1742 		if (index == 0)
1743 			cycled = 1;
1744 		else
1745 			cycled = 0;
1746 		end = -1;
1747 	} else {
1748 		index = wbc->range_start >> PAGE_CACHE_SHIFT;
1749 		end = wbc->range_end >> PAGE_CACHE_SHIFT;
1750 		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1751 			range_whole = 1;
1752 		cycled = 1; /* ignore range_cyclic tests */
1753 	}
1754 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1755 		tag = PAGECACHE_TAG_TOWRITE;
1756 	else
1757 		tag = PAGECACHE_TAG_DIRTY;
1758 retry:
1759 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1760 		tag_pages_for_writeback(mapping, index, end);
1761 	done_index = index;
1762 	while (!done && (index <= end)) {
1763 		int i;
1764 
1765 		nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
1766 			      min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
1767 		if (nr_pages == 0)
1768 			break;
1769 
1770 		for (i = 0; i < nr_pages; i++) {
1771 			struct page *page = pvec.pages[i];
1772 
1773 			/*
1774 			 * At this point, the page may be truncated or
1775 			 * invalidated (changing page->mapping to NULL), or
1776 			 * even swizzled back from swapper_space to tmpfs file
1777 			 * mapping. However, page->index will not change
1778 			 * because we have a reference on the page.
1779 			 */
1780 			if (page->index > end) {
1781 				/*
1782 				 * can't be range_cyclic (1st pass) because
1783 				 * end == -1 in that case.
1784 				 */
1785 				done = 1;
1786 				break;
1787 			}
1788 
1789 			done_index = page->index;
1790 
1791 			lock_page(page);
1792 
1793 			/*
1794 			 * Page truncated or invalidated. We can freely skip it
1795 			 * then, even for data integrity operations: the page
1796 			 * has disappeared concurrently, so there could be no
1797 			 * real expectation of this data interity operation
1798 			 * even if there is now a new, dirty page at the same
1799 			 * pagecache address.
1800 			 */
1801 			if (unlikely(page->mapping != mapping)) {
1802 continue_unlock:
1803 				unlock_page(page);
1804 				continue;
1805 			}
1806 
1807 			if (!PageDirty(page)) {
1808 				/* someone wrote it for us */
1809 				goto continue_unlock;
1810 			}
1811 
1812 			if (PageWriteback(page)) {
1813 				if (wbc->sync_mode != WB_SYNC_NONE)
1814 					wait_on_page_writeback(page);
1815 				else
1816 					goto continue_unlock;
1817 			}
1818 
1819 			BUG_ON(PageWriteback(page));
1820 			if (!clear_page_dirty_for_io(page))
1821 				goto continue_unlock;
1822 
1823 			trace_wbc_writepage(wbc, mapping->backing_dev_info);
1824 			ret = (*writepage)(page, wbc, data);
1825 			if (unlikely(ret)) {
1826 				if (ret == AOP_WRITEPAGE_ACTIVATE) {
1827 					unlock_page(page);
1828 					ret = 0;
1829 				} else {
1830 					/*
1831 					 * done_index is set past this page,
1832 					 * so media errors will not choke
1833 					 * background writeout for the entire
1834 					 * file. This has consequences for
1835 					 * range_cyclic semantics (ie. it may
1836 					 * not be suitable for data integrity
1837 					 * writeout).
1838 					 */
1839 					done_index = page->index + 1;
1840 					done = 1;
1841 					break;
1842 				}
1843 			}
1844 
1845 			/*
1846 			 * We stop writing back only if we are not doing
1847 			 * integrity sync. In case of integrity sync we have to
1848 			 * keep going until we have written all the pages
1849 			 * we tagged for writeback prior to entering this loop.
1850 			 */
1851 			if (--wbc->nr_to_write <= 0 &&
1852 			    wbc->sync_mode == WB_SYNC_NONE) {
1853 				done = 1;
1854 				break;
1855 			}
1856 		}
1857 		pagevec_release(&pvec);
1858 		cond_resched();
1859 	}
1860 	if (!cycled && !done) {
1861 		/*
1862 		 * range_cyclic:
1863 		 * We hit the last page and there is more work to be done: wrap
1864 		 * back to the start of the file
1865 		 */
1866 		cycled = 1;
1867 		index = 0;
1868 		end = writeback_index - 1;
1869 		goto retry;
1870 	}
1871 	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1872 		mapping->writeback_index = done_index;
1873 
1874 	return ret;
1875 }
1876 EXPORT_SYMBOL(write_cache_pages);
1877 
1878 /*
1879  * Function used by generic_writepages to call the real writepage
1880  * function and set the mapping flags on error
1881  */
1882 static int __writepage(struct page *page, struct writeback_control *wbc,
1883 		       void *data)
1884 {
1885 	struct address_space *mapping = data;
1886 	int ret = mapping->a_ops->writepage(page, wbc);
1887 	mapping_set_error(mapping, ret);
1888 	return ret;
1889 }
1890 
1891 /**
1892  * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them.
1893  * @mapping: address space structure to write
1894  * @wbc: subtract the number of written pages from *@wbc->nr_to_write
1895  *
1896  * This is a library function, which implements the writepages()
1897  * address_space_operation.
1898  */
1899 int generic_writepages(struct address_space *mapping,
1900 		       struct writeback_control *wbc)
1901 {
1902 	struct blk_plug plug;
1903 	int ret;
1904 
1905 	/* deal with chardevs and other special file */
1906 	if (!mapping->a_ops->writepage)
1907 		return 0;
1908 
1909 	blk_start_plug(&plug);
1910 	ret = write_cache_pages(mapping, wbc, __writepage, mapping);
1911 	blk_finish_plug(&plug);
1912 	return ret;
1913 }
1914 
1915 EXPORT_SYMBOL(generic_writepages);
1916 
1917 int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
1918 {
1919 	int ret;
1920 
1921 	if (wbc->nr_to_write <= 0)
1922 		return 0;
1923 	if (mapping->a_ops->writepages)
1924 		ret = mapping->a_ops->writepages(mapping, wbc);
1925 	else
1926 		ret = generic_writepages(mapping, wbc);
1927 	return ret;
1928 }
1929 
1930 /**
1931  * write_one_page - write out a single page and optionally wait on I/O
1932  * @page: the page to write
1933  * @wait: if true, wait on writeout
1934  *
1935  * The page must be locked by the caller and will be unlocked upon return.
1936  *
1937  * write_one_page() returns a negative error code if I/O failed.
1938  */
1939 int write_one_page(struct page *page, int wait)
1940 {
1941 	struct address_space *mapping = page->mapping;
1942 	int ret = 0;
1943 	struct writeback_control wbc = {
1944 		.sync_mode = WB_SYNC_ALL,
1945 		.nr_to_write = 1,
1946 	};
1947 
1948 	BUG_ON(!PageLocked(page));
1949 
1950 	if (wait)
1951 		wait_on_page_writeback(page);
1952 
1953 	if (clear_page_dirty_for_io(page)) {
1954 		page_cache_get(page);
1955 		ret = mapping->a_ops->writepage(page, &wbc);
1956 		if (ret == 0 && wait) {
1957 			wait_on_page_writeback(page);
1958 			if (PageError(page))
1959 				ret = -EIO;
1960 		}
1961 		page_cache_release(page);
1962 	} else {
1963 		unlock_page(page);
1964 	}
1965 	return ret;
1966 }
1967 EXPORT_SYMBOL(write_one_page);
1968 
1969 /*
1970  * For address_spaces which do not use buffers nor write back.
1971  */
1972 int __set_page_dirty_no_writeback(struct page *page)
1973 {
1974 	if (!PageDirty(page))
1975 		return !TestSetPageDirty(page);
1976 	return 0;
1977 }
1978 
1979 /*
1980  * Helper function for set_page_dirty family.
1981  * NOTE: This relies on being atomic wrt interrupts.
1982  */
1983 void account_page_dirtied(struct page *page, struct address_space *mapping)
1984 {
1985 	if (mapping_cap_account_dirty(mapping)) {
1986 		__inc_zone_page_state(page, NR_FILE_DIRTY);
1987 		__inc_zone_page_state(page, NR_DIRTIED);
1988 		__inc_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
1989 		__inc_bdi_stat(mapping->backing_dev_info, BDI_DIRTIED);
1990 		task_io_account_write(PAGE_CACHE_SIZE);
1991 		current->nr_dirtied++;
1992 		this_cpu_inc(bdp_ratelimits);
1993 	}
1994 }
1995 EXPORT_SYMBOL(account_page_dirtied);
1996 
1997 /*
1998  * Helper function for set_page_writeback family.
1999  * NOTE: Unlike account_page_dirtied this does not rely on being atomic
2000  * wrt interrupts.
2001  */
2002 void account_page_writeback(struct page *page)
2003 {
2004 	inc_zone_page_state(page, NR_WRITEBACK);
2005 }
2006 EXPORT_SYMBOL(account_page_writeback);
2007 
2008 /*
2009  * For address_spaces which do not use buffers.  Just tag the page as dirty in
2010  * its radix tree.
2011  *
2012  * This is also used when a single buffer is being dirtied: we want to set the
2013  * page dirty in that case, but not all the buffers.  This is a "bottom-up"
2014  * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying.
2015  *
2016  * Most callers have locked the page, which pins the address_space in memory.
2017  * But zap_pte_range() does not lock the page, however in that case the
2018  * mapping is pinned by the vma's ->vm_file reference.
2019  *
2020  * We take care to handle the case where the page was truncated from the
2021  * mapping by re-checking page_mapping() inside tree_lock.
2022  */
2023 int __set_page_dirty_nobuffers(struct page *page)
2024 {
2025 	if (!TestSetPageDirty(page)) {
2026 		struct address_space *mapping = page_mapping(page);
2027 		struct address_space *mapping2;
2028 
2029 		if (!mapping)
2030 			return 1;
2031 
2032 		spin_lock_irq(&mapping->tree_lock);
2033 		mapping2 = page_mapping(page);
2034 		if (mapping2) { /* Race with truncate? */
2035 			BUG_ON(mapping2 != mapping);
2036 			WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
2037 			account_page_dirtied(page, mapping);
2038 			radix_tree_tag_set(&mapping->page_tree,
2039 				page_index(page), PAGECACHE_TAG_DIRTY);
2040 		}
2041 		spin_unlock_irq(&mapping->tree_lock);
2042 		if (mapping->host) {
2043 			/* !PageAnon && !swapper_space */
2044 			__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2045 		}
2046 		return 1;
2047 	}
2048 	return 0;
2049 }
2050 EXPORT_SYMBOL(__set_page_dirty_nobuffers);
2051 
2052 /*
2053  * Call this whenever redirtying a page, to de-account the dirty counters
2054  * (NR_DIRTIED, BDI_DIRTIED, tsk->nr_dirtied), so that they match the written
2055  * counters (NR_WRITTEN, BDI_WRITTEN) in long term. The mismatches will lead to
2056  * systematic errors in balanced_dirty_ratelimit and the dirty pages position
2057  * control.
2058  */
2059 void account_page_redirty(struct page *page)
2060 {
2061 	struct address_space *mapping = page->mapping;
2062 	if (mapping && mapping_cap_account_dirty(mapping)) {
2063 		current->nr_dirtied--;
2064 		dec_zone_page_state(page, NR_DIRTIED);
2065 		dec_bdi_stat(mapping->backing_dev_info, BDI_DIRTIED);
2066 	}
2067 }
2068 EXPORT_SYMBOL(account_page_redirty);
2069 
2070 /*
2071  * When a writepage implementation decides that it doesn't want to write this
2072  * page for some reason, it should redirty the locked page via
2073  * redirty_page_for_writepage() and it should then unlock the page and return 0
2074  */
2075 int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page)
2076 {
2077 	wbc->pages_skipped++;
2078 	account_page_redirty(page);
2079 	return __set_page_dirty_nobuffers(page);
2080 }
2081 EXPORT_SYMBOL(redirty_page_for_writepage);
2082 
2083 /*
2084  * Dirty a page.
2085  *
2086  * For pages with a mapping this should be done under the page lock
2087  * for the benefit of asynchronous memory errors who prefer a consistent
2088  * dirty state. This rule can be broken in some special cases,
2089  * but should be better not to.
2090  *
2091  * If the mapping doesn't provide a set_page_dirty a_op, then
2092  * just fall through and assume that it wants buffer_heads.
2093  */
2094 int set_page_dirty(struct page *page)
2095 {
2096 	struct address_space *mapping = page_mapping(page);
2097 
2098 	if (likely(mapping)) {
2099 		int (*spd)(struct page *) = mapping->a_ops->set_page_dirty;
2100 		/*
2101 		 * readahead/lru_deactivate_page could remain
2102 		 * PG_readahead/PG_reclaim due to race with end_page_writeback
2103 		 * About readahead, if the page is written, the flags would be
2104 		 * reset. So no problem.
2105 		 * About lru_deactivate_page, if the page is redirty, the flag
2106 		 * will be reset. So no problem. but if the page is used by readahead
2107 		 * it will confuse readahead and make it restart the size rampup
2108 		 * process. But it's a trivial problem.
2109 		 */
2110 		ClearPageReclaim(page);
2111 #ifdef CONFIG_BLOCK
2112 		if (!spd)
2113 			spd = __set_page_dirty_buffers;
2114 #endif
2115 		return (*spd)(page);
2116 	}
2117 	if (!PageDirty(page)) {
2118 		if (!TestSetPageDirty(page))
2119 			return 1;
2120 	}
2121 	return 0;
2122 }
2123 EXPORT_SYMBOL(set_page_dirty);
2124 
2125 /*
2126  * set_page_dirty() is racy if the caller has no reference against
2127  * page->mapping->host, and if the page is unlocked.  This is because another
2128  * CPU could truncate the page off the mapping and then free the mapping.
2129  *
2130  * Usually, the page _is_ locked, or the caller is a user-space process which
2131  * holds a reference on the inode by having an open file.
2132  *
2133  * In other cases, the page should be locked before running set_page_dirty().
2134  */
2135 int set_page_dirty_lock(struct page *page)
2136 {
2137 	int ret;
2138 
2139 	lock_page(page);
2140 	ret = set_page_dirty(page);
2141 	unlock_page(page);
2142 	return ret;
2143 }
2144 EXPORT_SYMBOL(set_page_dirty_lock);
2145 
2146 /*
2147  * Clear a page's dirty flag, while caring for dirty memory accounting.
2148  * Returns true if the page was previously dirty.
2149  *
2150  * This is for preparing to put the page under writeout.  We leave the page
2151  * tagged as dirty in the radix tree so that a concurrent write-for-sync
2152  * can discover it via a PAGECACHE_TAG_DIRTY walk.  The ->writepage
2153  * implementation will run either set_page_writeback() or set_page_dirty(),
2154  * at which stage we bring the page's dirty flag and radix-tree dirty tag
2155  * back into sync.
2156  *
2157  * This incoherency between the page's dirty flag and radix-tree tag is
2158  * unfortunate, but it only exists while the page is locked.
2159  */
2160 int clear_page_dirty_for_io(struct page *page)
2161 {
2162 	struct address_space *mapping = page_mapping(page);
2163 
2164 	BUG_ON(!PageLocked(page));
2165 
2166 	if (mapping && mapping_cap_account_dirty(mapping)) {
2167 		/*
2168 		 * Yes, Virginia, this is indeed insane.
2169 		 *
2170 		 * We use this sequence to make sure that
2171 		 *  (a) we account for dirty stats properly
2172 		 *  (b) we tell the low-level filesystem to
2173 		 *      mark the whole page dirty if it was
2174 		 *      dirty in a pagetable. Only to then
2175 		 *  (c) clean the page again and return 1 to
2176 		 *      cause the writeback.
2177 		 *
2178 		 * This way we avoid all nasty races with the
2179 		 * dirty bit in multiple places and clearing
2180 		 * them concurrently from different threads.
2181 		 *
2182 		 * Note! Normally the "set_page_dirty(page)"
2183 		 * has no effect on the actual dirty bit - since
2184 		 * that will already usually be set. But we
2185 		 * need the side effects, and it can help us
2186 		 * avoid races.
2187 		 *
2188 		 * We basically use the page "master dirty bit"
2189 		 * as a serialization point for all the different
2190 		 * threads doing their things.
2191 		 */
2192 		if (page_mkclean(page))
2193 			set_page_dirty(page);
2194 		/*
2195 		 * We carefully synchronise fault handlers against
2196 		 * installing a dirty pte and marking the page dirty
2197 		 * at this point. We do this by having them hold the
2198 		 * page lock at some point after installing their
2199 		 * pte, but before marking the page dirty.
2200 		 * Pages are always locked coming in here, so we get
2201 		 * the desired exclusion. See mm/memory.c:do_wp_page()
2202 		 * for more comments.
2203 		 */
2204 		if (TestClearPageDirty(page)) {
2205 			dec_zone_page_state(page, NR_FILE_DIRTY);
2206 			dec_bdi_stat(mapping->backing_dev_info,
2207 					BDI_RECLAIMABLE);
2208 			return 1;
2209 		}
2210 		return 0;
2211 	}
2212 	return TestClearPageDirty(page);
2213 }
2214 EXPORT_SYMBOL(clear_page_dirty_for_io);
2215 
2216 int test_clear_page_writeback(struct page *page)
2217 {
2218 	struct address_space *mapping = page_mapping(page);
2219 	int ret;
2220 
2221 	if (mapping) {
2222 		struct backing_dev_info *bdi = mapping->backing_dev_info;
2223 		unsigned long flags;
2224 
2225 		spin_lock_irqsave(&mapping->tree_lock, flags);
2226 		ret = TestClearPageWriteback(page);
2227 		if (ret) {
2228 			radix_tree_tag_clear(&mapping->page_tree,
2229 						page_index(page),
2230 						PAGECACHE_TAG_WRITEBACK);
2231 			if (bdi_cap_account_writeback(bdi)) {
2232 				__dec_bdi_stat(bdi, BDI_WRITEBACK);
2233 				__bdi_writeout_inc(bdi);
2234 			}
2235 		}
2236 		spin_unlock_irqrestore(&mapping->tree_lock, flags);
2237 	} else {
2238 		ret = TestClearPageWriteback(page);
2239 	}
2240 	if (ret) {
2241 		dec_zone_page_state(page, NR_WRITEBACK);
2242 		inc_zone_page_state(page, NR_WRITTEN);
2243 	}
2244 	return ret;
2245 }
2246 
2247 int test_set_page_writeback(struct page *page)
2248 {
2249 	struct address_space *mapping = page_mapping(page);
2250 	int ret;
2251 
2252 	if (mapping) {
2253 		struct backing_dev_info *bdi = mapping->backing_dev_info;
2254 		unsigned long flags;
2255 
2256 		spin_lock_irqsave(&mapping->tree_lock, flags);
2257 		ret = TestSetPageWriteback(page);
2258 		if (!ret) {
2259 			radix_tree_tag_set(&mapping->page_tree,
2260 						page_index(page),
2261 						PAGECACHE_TAG_WRITEBACK);
2262 			if (bdi_cap_account_writeback(bdi))
2263 				__inc_bdi_stat(bdi, BDI_WRITEBACK);
2264 		}
2265 		if (!PageDirty(page))
2266 			radix_tree_tag_clear(&mapping->page_tree,
2267 						page_index(page),
2268 						PAGECACHE_TAG_DIRTY);
2269 		radix_tree_tag_clear(&mapping->page_tree,
2270 				     page_index(page),
2271 				     PAGECACHE_TAG_TOWRITE);
2272 		spin_unlock_irqrestore(&mapping->tree_lock, flags);
2273 	} else {
2274 		ret = TestSetPageWriteback(page);
2275 	}
2276 	if (!ret)
2277 		account_page_writeback(page);
2278 	return ret;
2279 
2280 }
2281 EXPORT_SYMBOL(test_set_page_writeback);
2282 
2283 /*
2284  * Return true if any of the pages in the mapping are marked with the
2285  * passed tag.
2286  */
2287 int mapping_tagged(struct address_space *mapping, int tag)
2288 {
2289 	return radix_tree_tagged(&mapping->page_tree, tag);
2290 }
2291 EXPORT_SYMBOL(mapping_tagged);
2292