1 /* 2 * mm/balloon_compaction.c 3 * 4 * Common interface for making balloon pages movable by compaction. 5 * 6 * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com> 7 */ 8 #include <linux/mm.h> 9 #include <linux/slab.h> 10 #include <linux/export.h> 11 #include <linux/balloon_compaction.h> 12 13 /* 14 * balloon_devinfo_alloc - allocates a balloon device information descriptor. 15 * @balloon_dev_descriptor: pointer to reference the balloon device which 16 * this struct balloon_dev_info will be servicing. 17 * 18 * Driver must call it to properly allocate and initialize an instance of 19 * struct balloon_dev_info which will be used to reference a balloon device 20 * as well as to keep track of the balloon device page list. 21 */ 22 struct balloon_dev_info *balloon_devinfo_alloc(void *balloon_dev_descriptor) 23 { 24 struct balloon_dev_info *b_dev_info; 25 b_dev_info = kmalloc(sizeof(*b_dev_info), GFP_KERNEL); 26 if (!b_dev_info) 27 return ERR_PTR(-ENOMEM); 28 29 b_dev_info->balloon_device = balloon_dev_descriptor; 30 b_dev_info->mapping = NULL; 31 b_dev_info->isolated_pages = 0; 32 spin_lock_init(&b_dev_info->pages_lock); 33 INIT_LIST_HEAD(&b_dev_info->pages); 34 35 return b_dev_info; 36 } 37 EXPORT_SYMBOL_GPL(balloon_devinfo_alloc); 38 39 /* 40 * balloon_page_enqueue - allocates a new page and inserts it into the balloon 41 * page list. 42 * @b_dev_info: balloon device decriptor where we will insert a new page to 43 * 44 * Driver must call it to properly allocate a new enlisted balloon page 45 * before definetively removing it from the guest system. 46 * This function returns the page address for the recently enqueued page or 47 * NULL in the case we fail to allocate a new page this turn. 48 */ 49 struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info) 50 { 51 unsigned long flags; 52 struct page *page = alloc_page(balloon_mapping_gfp_mask() | 53 __GFP_NOMEMALLOC | __GFP_NORETRY); 54 if (!page) 55 return NULL; 56 57 /* 58 * Block others from accessing the 'page' when we get around to 59 * establishing additional references. We should be the only one 60 * holding a reference to the 'page' at this point. 61 */ 62 BUG_ON(!trylock_page(page)); 63 spin_lock_irqsave(&b_dev_info->pages_lock, flags); 64 balloon_page_insert(page, b_dev_info->mapping, &b_dev_info->pages); 65 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); 66 unlock_page(page); 67 return page; 68 } 69 EXPORT_SYMBOL_GPL(balloon_page_enqueue); 70 71 /* 72 * balloon_page_dequeue - removes a page from balloon's page list and returns 73 * the its address to allow the driver release the page. 74 * @b_dev_info: balloon device decriptor where we will grab a page from. 75 * 76 * Driver must call it to properly de-allocate a previous enlisted balloon page 77 * before definetively releasing it back to the guest system. 78 * This function returns the page address for the recently dequeued page or 79 * NULL in the case we find balloon's page list temporarily empty due to 80 * compaction isolated pages. 81 */ 82 struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info) 83 { 84 struct page *page, *tmp; 85 unsigned long flags; 86 bool dequeued_page; 87 88 dequeued_page = false; 89 list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) { 90 /* 91 * Block others from accessing the 'page' while we get around 92 * establishing additional references and preparing the 'page' 93 * to be released by the balloon driver. 94 */ 95 if (trylock_page(page)) { 96 spin_lock_irqsave(&b_dev_info->pages_lock, flags); 97 /* 98 * Raise the page refcount here to prevent any wrong 99 * attempt to isolate this page, in case of coliding 100 * with balloon_page_isolate() just after we release 101 * the page lock. 102 * 103 * balloon_page_free() will take care of dropping 104 * this extra refcount later. 105 */ 106 get_page(page); 107 balloon_page_delete(page); 108 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); 109 unlock_page(page); 110 dequeued_page = true; 111 break; 112 } 113 } 114 115 if (!dequeued_page) { 116 /* 117 * If we are unable to dequeue a balloon page because the page 118 * list is empty and there is no isolated pages, then something 119 * went out of track and some balloon pages are lost. 120 * BUG() here, otherwise the balloon driver may get stuck into 121 * an infinite loop while attempting to release all its pages. 122 */ 123 spin_lock_irqsave(&b_dev_info->pages_lock, flags); 124 if (unlikely(list_empty(&b_dev_info->pages) && 125 !b_dev_info->isolated_pages)) 126 BUG(); 127 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); 128 page = NULL; 129 } 130 return page; 131 } 132 EXPORT_SYMBOL_GPL(balloon_page_dequeue); 133 134 #ifdef CONFIG_BALLOON_COMPACTION 135 /* 136 * balloon_mapping_alloc - allocates a special ->mapping for ballooned pages. 137 * @b_dev_info: holds the balloon device information descriptor. 138 * @a_ops: balloon_mapping address_space_operations descriptor. 139 * 140 * Driver must call it to properly allocate and initialize an instance of 141 * struct address_space which will be used as the special page->mapping for 142 * balloon device enlisted page instances. 143 */ 144 struct address_space *balloon_mapping_alloc(struct balloon_dev_info *b_dev_info, 145 const struct address_space_operations *a_ops) 146 { 147 struct address_space *mapping; 148 149 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 150 if (!mapping) 151 return ERR_PTR(-ENOMEM); 152 153 /* 154 * Give a clean 'zeroed' status to all elements of this special 155 * balloon page->mapping struct address_space instance. 156 */ 157 address_space_init_once(mapping); 158 159 /* 160 * Set mapping->flags appropriately, to allow balloon pages 161 * ->mapping identification. 162 */ 163 mapping_set_balloon(mapping); 164 mapping_set_gfp_mask(mapping, balloon_mapping_gfp_mask()); 165 166 /* balloon's page->mapping->a_ops callback descriptor */ 167 mapping->a_ops = a_ops; 168 169 /* 170 * Establish a pointer reference back to the balloon device descriptor 171 * this particular page->mapping will be servicing. 172 * This is used by compaction / migration procedures to identify and 173 * access the balloon device pageset while isolating / migrating pages. 174 * 175 * As some balloon drivers can register multiple balloon devices 176 * for a single guest, this also helps compaction / migration to 177 * properly deal with multiple balloon pagesets, when required. 178 */ 179 mapping->private_data = b_dev_info; 180 b_dev_info->mapping = mapping; 181 182 return mapping; 183 } 184 EXPORT_SYMBOL_GPL(balloon_mapping_alloc); 185 186 static inline void __isolate_balloon_page(struct page *page) 187 { 188 struct balloon_dev_info *b_dev_info = page->mapping->private_data; 189 unsigned long flags; 190 spin_lock_irqsave(&b_dev_info->pages_lock, flags); 191 list_del(&page->lru); 192 b_dev_info->isolated_pages++; 193 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); 194 } 195 196 static inline void __putback_balloon_page(struct page *page) 197 { 198 struct balloon_dev_info *b_dev_info = page->mapping->private_data; 199 unsigned long flags; 200 spin_lock_irqsave(&b_dev_info->pages_lock, flags); 201 list_add(&page->lru, &b_dev_info->pages); 202 b_dev_info->isolated_pages--; 203 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); 204 } 205 206 static inline int __migrate_balloon_page(struct address_space *mapping, 207 struct page *newpage, struct page *page, enum migrate_mode mode) 208 { 209 return page->mapping->a_ops->migratepage(mapping, newpage, page, mode); 210 } 211 212 /* __isolate_lru_page() counterpart for a ballooned page */ 213 bool balloon_page_isolate(struct page *page) 214 { 215 /* 216 * Avoid burning cycles with pages that are yet under __free_pages(), 217 * or just got freed under us. 218 * 219 * In case we 'win' a race for a balloon page being freed under us and 220 * raise its refcount preventing __free_pages() from doing its job 221 * the put_page() at the end of this block will take care of 222 * release this page, thus avoiding a nasty leakage. 223 */ 224 if (likely(get_page_unless_zero(page))) { 225 /* 226 * As balloon pages are not isolated from LRU lists, concurrent 227 * compaction threads can race against page migration functions 228 * as well as race against the balloon driver releasing a page. 229 * 230 * In order to avoid having an already isolated balloon page 231 * being (wrongly) re-isolated while it is under migration, 232 * or to avoid attempting to isolate pages being released by 233 * the balloon driver, lets be sure we have the page lock 234 * before proceeding with the balloon page isolation steps. 235 */ 236 if (likely(trylock_page(page))) { 237 /* 238 * A ballooned page, by default, has just one refcount. 239 * Prevent concurrent compaction threads from isolating 240 * an already isolated balloon page by refcount check. 241 */ 242 if (__is_movable_balloon_page(page) && 243 page_count(page) == 2) { 244 __isolate_balloon_page(page); 245 unlock_page(page); 246 return true; 247 } 248 unlock_page(page); 249 } 250 put_page(page); 251 } 252 return false; 253 } 254 255 /* putback_lru_page() counterpart for a ballooned page */ 256 void balloon_page_putback(struct page *page) 257 { 258 /* 259 * 'lock_page()' stabilizes the page and prevents races against 260 * concurrent isolation threads attempting to re-isolate it. 261 */ 262 lock_page(page); 263 264 if (__is_movable_balloon_page(page)) { 265 __putback_balloon_page(page); 266 /* drop the extra ref count taken for page isolation */ 267 put_page(page); 268 } else { 269 WARN_ON(1); 270 dump_page(page, "not movable balloon page"); 271 } 272 unlock_page(page); 273 } 274 275 /* move_to_new_page() counterpart for a ballooned page */ 276 int balloon_page_migrate(struct page *newpage, 277 struct page *page, enum migrate_mode mode) 278 { 279 struct address_space *mapping; 280 int rc = -EAGAIN; 281 282 /* 283 * Block others from accessing the 'newpage' when we get around to 284 * establishing additional references. We should be the only one 285 * holding a reference to the 'newpage' at this point. 286 */ 287 BUG_ON(!trylock_page(newpage)); 288 289 if (WARN_ON(!__is_movable_balloon_page(page))) { 290 dump_page(page, "not movable balloon page"); 291 unlock_page(newpage); 292 return rc; 293 } 294 295 mapping = page->mapping; 296 if (mapping) 297 rc = __migrate_balloon_page(mapping, newpage, page, mode); 298 299 unlock_page(newpage); 300 return rc; 301 } 302 #endif /* CONFIG_BALLOON_COMPACTION */ 303