1 /* 2 * mm/truncate.c - code for taking down pages from address_spaces 3 * 4 * Copyright (C) 2002, Linus Torvalds 5 * 6 * 10Sep2002 akpm@zip.com.au 7 * Initial version. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/mm.h> 12 #include <linux/module.h> 13 #include <linux/pagemap.h> 14 #include <linux/pagevec.h> 15 #include <linux/buffer_head.h> /* grr. try_to_release_page, 16 do_invalidatepage */ 17 18 19 static inline void truncate_partial_page(struct page *page, unsigned partial) 20 { 21 memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial); 22 if (PagePrivate(page)) 23 do_invalidatepage(page, partial); 24 } 25 26 /* 27 * If truncate cannot remove the fs-private metadata from the page, the page 28 * becomes anonymous. It will be left on the LRU and may even be mapped into 29 * user pagetables if we're racing with filemap_nopage(). 30 * 31 * We need to bale out if page->mapping is no longer equal to the original 32 * mapping. This happens a) when the VM reclaimed the page while we waited on 33 * its lock, b) when a concurrent invalidate_inode_pages got there first and 34 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space. 35 */ 36 static void 37 truncate_complete_page(struct address_space *mapping, struct page *page) 38 { 39 if (page->mapping != mapping) 40 return; 41 42 if (PagePrivate(page)) 43 do_invalidatepage(page, 0); 44 45 clear_page_dirty(page); 46 ClearPageUptodate(page); 47 ClearPageMappedToDisk(page); 48 remove_from_page_cache(page); 49 page_cache_release(page); /* pagecache ref */ 50 } 51 52 /* 53 * This is for invalidate_inode_pages(). That function can be called at 54 * any time, and is not supposed to throw away dirty pages. But pages can 55 * be marked dirty at any time too. So we re-check the dirtiness inside 56 * ->tree_lock. That provides exclusion against the __set_page_dirty 57 * functions. 58 * 59 * Returns non-zero if the page was successfully invalidated. 60 */ 61 static int 62 invalidate_complete_page(struct address_space *mapping, struct page *page) 63 { 64 if (page->mapping != mapping) 65 return 0; 66 67 if (PagePrivate(page) && !try_to_release_page(page, 0)) 68 return 0; 69 70 write_lock_irq(&mapping->tree_lock); 71 if (PageDirty(page)) { 72 write_unlock_irq(&mapping->tree_lock); 73 return 0; 74 } 75 76 BUG_ON(PagePrivate(page)); 77 __remove_from_page_cache(page); 78 write_unlock_irq(&mapping->tree_lock); 79 ClearPageUptodate(page); 80 page_cache_release(page); /* pagecache ref */ 81 return 1; 82 } 83 84 /** 85 * truncate_inode_pages - truncate *all* the pages from an offset 86 * @mapping: mapping to truncate 87 * @lstart: offset from which to truncate 88 * 89 * Truncate the page cache at a set offset, removing the pages that are beyond 90 * that offset (and zeroing out partial pages). 91 * 92 * Truncate takes two passes - the first pass is nonblocking. It will not 93 * block on page locks and it will not block on writeback. The second pass 94 * will wait. This is to prevent as much IO as possible in the affected region. 95 * The first pass will remove most pages, so the search cost of the second pass 96 * is low. 97 * 98 * When looking at page->index outside the page lock we need to be careful to 99 * copy it into a local to avoid races (it could change at any time). 100 * 101 * We pass down the cache-hot hint to the page freeing code. Even if the 102 * mapping is large, it is probably the case that the final pages are the most 103 * recently touched, and freeing happens in ascending file offset order. 104 * 105 * Called under (and serialised by) inode->i_sem. 106 */ 107 void truncate_inode_pages(struct address_space *mapping, loff_t lstart) 108 { 109 const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT; 110 const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1); 111 struct pagevec pvec; 112 pgoff_t next; 113 int i; 114 115 if (mapping->nrpages == 0) 116 return; 117 118 pagevec_init(&pvec, 0); 119 next = start; 120 while (pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { 121 for (i = 0; i < pagevec_count(&pvec); i++) { 122 struct page *page = pvec.pages[i]; 123 pgoff_t page_index = page->index; 124 125 if (page_index > next) 126 next = page_index; 127 next++; 128 if (TestSetPageLocked(page)) 129 continue; 130 if (PageWriteback(page)) { 131 unlock_page(page); 132 continue; 133 } 134 truncate_complete_page(mapping, page); 135 unlock_page(page); 136 } 137 pagevec_release(&pvec); 138 cond_resched(); 139 } 140 141 if (partial) { 142 struct page *page = find_lock_page(mapping, start - 1); 143 if (page) { 144 wait_on_page_writeback(page); 145 truncate_partial_page(page, partial); 146 unlock_page(page); 147 page_cache_release(page); 148 } 149 } 150 151 next = start; 152 for ( ; ; ) { 153 cond_resched(); 154 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { 155 if (next == start) 156 break; 157 next = start; 158 continue; 159 } 160 for (i = 0; i < pagevec_count(&pvec); i++) { 161 struct page *page = pvec.pages[i]; 162 163 lock_page(page); 164 wait_on_page_writeback(page); 165 if (page->index > next) 166 next = page->index; 167 next++; 168 truncate_complete_page(mapping, page); 169 unlock_page(page); 170 } 171 pagevec_release(&pvec); 172 } 173 } 174 175 EXPORT_SYMBOL(truncate_inode_pages); 176 177 /** 178 * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode 179 * @mapping: the address_space which holds the pages to invalidate 180 * @start: the offset 'from' which to invalidate 181 * @end: the offset 'to' which to invalidate (inclusive) 182 * 183 * This function only removes the unlocked pages, if you want to 184 * remove all the pages of one inode, you must call truncate_inode_pages. 185 * 186 * invalidate_mapping_pages() will not block on IO activity. It will not 187 * invalidate pages which are dirty, locked, under writeback or mapped into 188 * pagetables. 189 */ 190 unsigned long invalidate_mapping_pages(struct address_space *mapping, 191 pgoff_t start, pgoff_t end) 192 { 193 struct pagevec pvec; 194 pgoff_t next = start; 195 unsigned long ret = 0; 196 int i; 197 198 pagevec_init(&pvec, 0); 199 while (next <= end && 200 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { 201 for (i = 0; i < pagevec_count(&pvec); i++) { 202 struct page *page = pvec.pages[i]; 203 204 if (TestSetPageLocked(page)) { 205 next++; 206 continue; 207 } 208 if (page->index > next) 209 next = page->index; 210 next++; 211 if (PageDirty(page) || PageWriteback(page)) 212 goto unlock; 213 if (page_mapped(page)) 214 goto unlock; 215 ret += invalidate_complete_page(mapping, page); 216 unlock: 217 unlock_page(page); 218 if (next > end) 219 break; 220 } 221 pagevec_release(&pvec); 222 cond_resched(); 223 } 224 return ret; 225 } 226 227 unsigned long invalidate_inode_pages(struct address_space *mapping) 228 { 229 return invalidate_mapping_pages(mapping, 0, ~0UL); 230 } 231 232 EXPORT_SYMBOL(invalidate_inode_pages); 233 234 /** 235 * invalidate_inode_pages2_range - remove range of pages from an address_space 236 * @mapping: the address_space 237 * @start: the page offset 'from' which to invalidate 238 * @end: the page offset 'to' which to invalidate (inclusive) 239 * 240 * Any pages which are found to be mapped into pagetables are unmapped prior to 241 * invalidation. 242 * 243 * Returns -EIO if any pages could not be invalidated. 244 */ 245 int invalidate_inode_pages2_range(struct address_space *mapping, 246 pgoff_t start, pgoff_t end) 247 { 248 struct pagevec pvec; 249 pgoff_t next; 250 int i; 251 int ret = 0; 252 int did_range_unmap = 0; 253 int wrapped = 0; 254 255 pagevec_init(&pvec, 0); 256 next = start; 257 while (next <= end && !ret && !wrapped && 258 pagevec_lookup(&pvec, mapping, next, 259 min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) { 260 for (i = 0; !ret && i < pagevec_count(&pvec); i++) { 261 struct page *page = pvec.pages[i]; 262 pgoff_t page_index; 263 int was_dirty; 264 265 lock_page(page); 266 if (page->mapping != mapping) { 267 unlock_page(page); 268 continue; 269 } 270 page_index = page->index; 271 next = page_index + 1; 272 if (next == 0) 273 wrapped = 1; 274 if (page_index > end) { 275 unlock_page(page); 276 break; 277 } 278 wait_on_page_writeback(page); 279 while (page_mapped(page)) { 280 if (!did_range_unmap) { 281 /* 282 * Zap the rest of the file in one hit. 283 */ 284 unmap_mapping_range(mapping, 285 page_index << PAGE_CACHE_SHIFT, 286 (end - page_index + 1) 287 << PAGE_CACHE_SHIFT, 288 0); 289 did_range_unmap = 1; 290 } else { 291 /* 292 * Just zap this page 293 */ 294 unmap_mapping_range(mapping, 295 page_index << PAGE_CACHE_SHIFT, 296 PAGE_CACHE_SIZE, 0); 297 } 298 } 299 was_dirty = test_clear_page_dirty(page); 300 if (!invalidate_complete_page(mapping, page)) { 301 if (was_dirty) 302 set_page_dirty(page); 303 ret = -EIO; 304 } 305 unlock_page(page); 306 } 307 pagevec_release(&pvec); 308 cond_resched(); 309 } 310 return ret; 311 } 312 EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range); 313 314 /** 315 * invalidate_inode_pages2 - remove all pages from an address_space 316 * @mapping: the address_space 317 * 318 * Any pages which are found to be mapped into pagetables are unmapped prior to 319 * invalidation. 320 * 321 * Returns -EIO if any pages could not be invalidated. 322 */ 323 int invalidate_inode_pages2(struct address_space *mapping) 324 { 325 return invalidate_inode_pages2_range(mapping, 0, -1); 326 } 327 EXPORT_SYMBOL_GPL(invalidate_inode_pages2); 328