1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * include/linux/writeback.h 4 */ 5 #ifndef WRITEBACK_H 6 #define WRITEBACK_H 7 8 #include <linux/sched.h> 9 #include <linux/workqueue.h> 10 #include <linux/fs.h> 11 #include <linux/flex_proportions.h> 12 #include <linux/backing-dev-defs.h> 13 #include <linux/blk_types.h> 14 15 struct bio; 16 17 DECLARE_PER_CPU(int, dirty_throttle_leaks); 18 19 /* 20 * The 1/4 region under the global dirty thresh is for smooth dirty throttling: 21 * 22 * (thresh - thresh/DIRTY_FULL_SCOPE, thresh) 23 * 24 * Further beyond, all dirtier tasks will enter a loop waiting (possibly long 25 * time) for the dirty pages to drop, unless written enough pages. 26 * 27 * The global dirty threshold is normally equal to the global dirty limit, 28 * except when the system suddenly allocates a lot of anonymous memory and 29 * knocks down the global dirty threshold quickly, in which case the global 30 * dirty limit will follow down slowly to prevent livelocking all dirtier tasks. 31 */ 32 #define DIRTY_SCOPE 8 33 #define DIRTY_FULL_SCOPE (DIRTY_SCOPE / 2) 34 35 struct backing_dev_info; 36 37 /* 38 * fs/fs-writeback.c 39 */ 40 enum writeback_sync_modes { 41 WB_SYNC_NONE, /* Don't wait on anything */ 42 WB_SYNC_ALL, /* Wait on every mapping */ 43 }; 44 45 /* 46 * A control structure which tells the writeback code what to do. These are 47 * always on the stack, and hence need no locking. They are always initialised 48 * in a manner such that unspecified fields are set to zero. 49 */ 50 struct writeback_control { 51 long nr_to_write; /* Write this many pages, and decrement 52 this for each page written */ 53 long pages_skipped; /* Pages which were not written */ 54 55 /* 56 * For a_ops->writepages(): if start or end are non-zero then this is 57 * a hint that the filesystem need only write out the pages inside that 58 * byterange. The byte at `end' is included in the writeout request. 59 */ 60 loff_t range_start; 61 loff_t range_end; 62 63 enum writeback_sync_modes sync_mode; 64 65 unsigned for_kupdate:1; /* A kupdate writeback */ 66 unsigned for_background:1; /* A background writeback */ 67 unsigned tagged_writepages:1; /* tag-and-write to avoid livelock */ 68 unsigned for_reclaim:1; /* Invoked from the page allocator */ 69 unsigned range_cyclic:1; /* range_start is cyclic */ 70 unsigned for_sync:1; /* sync(2) WB_SYNC_ALL writeback */ 71 #ifdef CONFIG_CGROUP_WRITEBACK 72 struct bdi_writeback *wb; /* wb this writeback is issued under */ 73 struct inode *inode; /* inode being written out */ 74 75 /* foreign inode detection, see wbc_detach_inode() */ 76 int wb_id; /* current wb id */ 77 int wb_lcand_id; /* last foreign candidate wb id */ 78 int wb_tcand_id; /* this foreign candidate wb id */ 79 size_t wb_bytes; /* bytes written by current wb */ 80 size_t wb_lcand_bytes; /* bytes written by last candidate */ 81 size_t wb_tcand_bytes; /* bytes written by this candidate */ 82 #endif 83 }; 84 85 static inline int wbc_to_write_flags(struct writeback_control *wbc) 86 { 87 if (wbc->sync_mode == WB_SYNC_ALL) 88 return REQ_SYNC; 89 else if (wbc->for_kupdate || wbc->for_background) 90 return REQ_BACKGROUND; 91 92 return 0; 93 } 94 95 /* 96 * A wb_domain represents a domain that wb's (bdi_writeback's) belong to 97 * and are measured against each other in. There always is one global 98 * domain, global_wb_domain, that every wb in the system is a member of. 99 * This allows measuring the relative bandwidth of each wb to distribute 100 * dirtyable memory accordingly. 101 */ 102 struct wb_domain { 103 spinlock_t lock; 104 105 /* 106 * Scale the writeback cache size proportional to the relative 107 * writeout speed. 108 * 109 * We do this by keeping a floating proportion between BDIs, based 110 * on page writeback completions [end_page_writeback()]. Those 111 * devices that write out pages fastest will get the larger share, 112 * while the slower will get a smaller share. 113 * 114 * We use page writeout completions because we are interested in 115 * getting rid of dirty pages. Having them written out is the 116 * primary goal. 117 * 118 * We introduce a concept of time, a period over which we measure 119 * these events, because demand can/will vary over time. The length 120 * of this period itself is measured in page writeback completions. 121 */ 122 struct fprop_global completions; 123 struct timer_list period_timer; /* timer for aging of completions */ 124 unsigned long period_time; 125 126 /* 127 * The dirtyable memory and dirty threshold could be suddenly 128 * knocked down by a large amount (eg. on the startup of KVM in a 129 * swapless system). This may throw the system into deep dirty 130 * exceeded state and throttle heavy/light dirtiers alike. To 131 * retain good responsiveness, maintain global_dirty_limit for 132 * tracking slowly down to the knocked down dirty threshold. 133 * 134 * Both fields are protected by ->lock. 135 */ 136 unsigned long dirty_limit_tstamp; 137 unsigned long dirty_limit; 138 }; 139 140 /** 141 * wb_domain_size_changed - memory available to a wb_domain has changed 142 * @dom: wb_domain of interest 143 * 144 * This function should be called when the amount of memory available to 145 * @dom has changed. It resets @dom's dirty limit parameters to prevent 146 * the past values which don't match the current configuration from skewing 147 * dirty throttling. Without this, when memory size of a wb_domain is 148 * greatly reduced, the dirty throttling logic may allow too many pages to 149 * be dirtied leading to consecutive unnecessary OOMs and may get stuck in 150 * that situation. 151 */ 152 static inline void wb_domain_size_changed(struct wb_domain *dom) 153 { 154 spin_lock(&dom->lock); 155 dom->dirty_limit_tstamp = jiffies; 156 dom->dirty_limit = 0; 157 spin_unlock(&dom->lock); 158 } 159 160 /* 161 * fs/fs-writeback.c 162 */ 163 struct bdi_writeback; 164 void writeback_inodes_sb(struct super_block *, enum wb_reason reason); 165 void writeback_inodes_sb_nr(struct super_block *, unsigned long nr, 166 enum wb_reason reason); 167 void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason); 168 void sync_inodes_sb(struct super_block *); 169 void wakeup_flusher_threads(enum wb_reason reason); 170 void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi, 171 enum wb_reason reason); 172 void inode_wait_for_writeback(struct inode *inode); 173 174 /* writeback.h requires fs.h; it, too, is not included from here. */ 175 static inline void wait_on_inode(struct inode *inode) 176 { 177 might_sleep(); 178 wait_on_bit(&inode->i_state, __I_NEW, TASK_UNINTERRUPTIBLE); 179 } 180 181 #ifdef CONFIG_CGROUP_WRITEBACK 182 183 #include <linux/cgroup.h> 184 #include <linux/bio.h> 185 186 void __inode_attach_wb(struct inode *inode, struct page *page); 187 void wbc_attach_and_unlock_inode(struct writeback_control *wbc, 188 struct inode *inode) 189 __releases(&inode->i_lock); 190 void wbc_detach_inode(struct writeback_control *wbc); 191 void wbc_account_io(struct writeback_control *wbc, struct page *page, 192 size_t bytes); 193 void cgroup_writeback_umount(void); 194 195 /** 196 * inode_attach_wb - associate an inode with its wb 197 * @inode: inode of interest 198 * @page: page being dirtied (may be NULL) 199 * 200 * If @inode doesn't have its wb, associate it with the wb matching the 201 * memcg of @page or, if @page is NULL, %current. May be called w/ or w/o 202 * @inode->i_lock. 203 */ 204 static inline void inode_attach_wb(struct inode *inode, struct page *page) 205 { 206 if (!inode->i_wb) 207 __inode_attach_wb(inode, page); 208 } 209 210 /** 211 * inode_detach_wb - disassociate an inode from its wb 212 * @inode: inode of interest 213 * 214 * @inode is being freed. Detach from its wb. 215 */ 216 static inline void inode_detach_wb(struct inode *inode) 217 { 218 if (inode->i_wb) { 219 WARN_ON_ONCE(!(inode->i_state & I_CLEAR)); 220 wb_put(inode->i_wb); 221 inode->i_wb = NULL; 222 } 223 } 224 225 /** 226 * wbc_attach_fdatawrite_inode - associate wbc and inode for fdatawrite 227 * @wbc: writeback_control of interest 228 * @inode: target inode 229 * 230 * This function is to be used by __filemap_fdatawrite_range(), which is an 231 * alternative entry point into writeback code, and first ensures @inode is 232 * associated with a bdi_writeback and attaches it to @wbc. 233 */ 234 static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, 235 struct inode *inode) 236 { 237 spin_lock(&inode->i_lock); 238 inode_attach_wb(inode, NULL); 239 wbc_attach_and_unlock_inode(wbc, inode); 240 } 241 242 /** 243 * wbc_init_bio - writeback specific initializtion of bio 244 * @wbc: writeback_control for the writeback in progress 245 * @bio: bio to be initialized 246 * 247 * @bio is a part of the writeback in progress controlled by @wbc. Perform 248 * writeback specific initialization. This is used to apply the cgroup 249 * writeback context. Must be called after the bio has been associated with 250 * a device. 251 */ 252 static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) 253 { 254 /* 255 * pageout() path doesn't attach @wbc to the inode being written 256 * out. This is intentional as we don't want the function to block 257 * behind a slow cgroup. Ultimately, we want pageout() to kick off 258 * regular writeback instead of writing things out itself. 259 */ 260 if (wbc->wb) 261 bio_associate_blkg_from_css(bio, wbc->wb->blkcg_css); 262 } 263 264 #else /* CONFIG_CGROUP_WRITEBACK */ 265 266 static inline void inode_attach_wb(struct inode *inode, struct page *page) 267 { 268 } 269 270 static inline void inode_detach_wb(struct inode *inode) 271 { 272 } 273 274 static inline void wbc_attach_and_unlock_inode(struct writeback_control *wbc, 275 struct inode *inode) 276 __releases(&inode->i_lock) 277 { 278 spin_unlock(&inode->i_lock); 279 } 280 281 static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, 282 struct inode *inode) 283 { 284 } 285 286 static inline void wbc_detach_inode(struct writeback_control *wbc) 287 { 288 } 289 290 static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) 291 { 292 } 293 294 static inline void wbc_account_io(struct writeback_control *wbc, 295 struct page *page, size_t bytes) 296 { 297 } 298 299 static inline void cgroup_writeback_umount(void) 300 { 301 } 302 303 #endif /* CONFIG_CGROUP_WRITEBACK */ 304 305 /* 306 * mm/page-writeback.c 307 */ 308 #ifdef CONFIG_BLOCK 309 void laptop_io_completion(struct backing_dev_info *info); 310 void laptop_sync_completion(void); 311 void laptop_mode_sync(struct work_struct *work); 312 void laptop_mode_timer_fn(struct timer_list *t); 313 #else 314 static inline void laptop_sync_completion(void) { } 315 #endif 316 bool node_dirty_ok(struct pglist_data *pgdat); 317 int wb_domain_init(struct wb_domain *dom, gfp_t gfp); 318 #ifdef CONFIG_CGROUP_WRITEBACK 319 void wb_domain_exit(struct wb_domain *dom); 320 #endif 321 322 extern struct wb_domain global_wb_domain; 323 324 /* These are exported to sysctl. */ 325 extern int dirty_background_ratio; 326 extern unsigned long dirty_background_bytes; 327 extern int vm_dirty_ratio; 328 extern unsigned long vm_dirty_bytes; 329 extern unsigned int dirty_writeback_interval; 330 extern unsigned int dirty_expire_interval; 331 extern unsigned int dirtytime_expire_interval; 332 extern int vm_highmem_is_dirtyable; 333 extern int block_dump; 334 extern int laptop_mode; 335 336 extern int dirty_background_ratio_handler(struct ctl_table *table, int write, 337 void __user *buffer, size_t *lenp, 338 loff_t *ppos); 339 extern int dirty_background_bytes_handler(struct ctl_table *table, int write, 340 void __user *buffer, size_t *lenp, 341 loff_t *ppos); 342 extern int dirty_ratio_handler(struct ctl_table *table, int write, 343 void __user *buffer, size_t *lenp, 344 loff_t *ppos); 345 extern int dirty_bytes_handler(struct ctl_table *table, int write, 346 void __user *buffer, size_t *lenp, 347 loff_t *ppos); 348 int dirtytime_interval_handler(struct ctl_table *table, int write, 349 void __user *buffer, size_t *lenp, loff_t *ppos); 350 351 struct ctl_table; 352 int dirty_writeback_centisecs_handler(struct ctl_table *, int, 353 void __user *, size_t *, loff_t *); 354 355 void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty); 356 unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh); 357 358 void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time); 359 void balance_dirty_pages_ratelimited(struct address_space *mapping); 360 bool wb_over_bg_thresh(struct bdi_writeback *wb); 361 362 typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc, 363 void *data); 364 365 int generic_writepages(struct address_space *mapping, 366 struct writeback_control *wbc); 367 void tag_pages_for_writeback(struct address_space *mapping, 368 pgoff_t start, pgoff_t end); 369 int write_cache_pages(struct address_space *mapping, 370 struct writeback_control *wbc, writepage_t writepage, 371 void *data); 372 int do_writepages(struct address_space *mapping, struct writeback_control *wbc); 373 void writeback_set_ratelimit(void); 374 void tag_pages_for_writeback(struct address_space *mapping, 375 pgoff_t start, pgoff_t end); 376 377 void account_page_redirty(struct page *page); 378 379 void sb_mark_inode_writeback(struct inode *inode); 380 void sb_clear_inode_writeback(struct inode *inode); 381 382 #endif /* WRITEBACK_H */ 383