1 /* 2 * linux/kernel/power/snapshot.c 3 * 4 * This file provides system snapshot/restore functionality for swsusp. 5 * 6 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz> 7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 8 * 9 * This file is released under the GPLv2. 10 * 11 */ 12 13 #include <linux/version.h> 14 #include <linux/module.h> 15 #include <linux/mm.h> 16 #include <linux/suspend.h> 17 #include <linux/delay.h> 18 #include <linux/bitops.h> 19 #include <linux/spinlock.h> 20 #include <linux/kernel.h> 21 #include <linux/pm.h> 22 #include <linux/device.h> 23 #include <linux/init.h> 24 #include <linux/bootmem.h> 25 #include <linux/syscalls.h> 26 #include <linux/console.h> 27 #include <linux/highmem.h> 28 #include <linux/list.h> 29 #include <linux/slab.h> 30 31 #include <asm/uaccess.h> 32 #include <asm/mmu_context.h> 33 #include <asm/pgtable.h> 34 #include <asm/tlbflush.h> 35 #include <asm/io.h> 36 37 #include "power.h" 38 39 static int swsusp_page_is_free(struct page *); 40 static void swsusp_set_page_forbidden(struct page *); 41 static void swsusp_unset_page_forbidden(struct page *); 42 43 /* 44 * Number of bytes to reserve for memory allocations made by device drivers 45 * from their ->freeze() and ->freeze_noirq() callbacks so that they don't 46 * cause image creation to fail (tunable via /sys/power/reserved_size). 47 */ 48 unsigned long reserved_size; 49 50 void __init hibernate_reserved_size_init(void) 51 { 52 reserved_size = SPARE_PAGES * PAGE_SIZE; 53 } 54 55 /* 56 * Preferred image size in bytes (tunable via /sys/power/image_size). 57 * When it is set to N, swsusp will do its best to ensure the image 58 * size will not exceed N bytes, but if that is impossible, it will 59 * try to create the smallest image possible. 60 */ 61 unsigned long image_size; 62 63 void __init hibernate_image_size_init(void) 64 { 65 image_size = ((totalram_pages * 2) / 5) * PAGE_SIZE; 66 } 67 68 /* List of PBEs needed for restoring the pages that were allocated before 69 * the suspend and included in the suspend image, but have also been 70 * allocated by the "resume" kernel, so their contents cannot be written 71 * directly to their "original" page frames. 72 */ 73 struct pbe *restore_pblist; 74 75 /* Pointer to an auxiliary buffer (1 page) */ 76 static void *buffer; 77 78 /** 79 * @safe_needed - on resume, for storing the PBE list and the image, 80 * we can only use memory pages that do not conflict with the pages 81 * used before suspend. The unsafe pages have PageNosaveFree set 82 * and we count them using unsafe_pages. 83 * 84 * Each allocated image page is marked as PageNosave and PageNosaveFree 85 * so that swsusp_free() can release it. 86 */ 87 88 #define PG_ANY 0 89 #define PG_SAFE 1 90 #define PG_UNSAFE_CLEAR 1 91 #define PG_UNSAFE_KEEP 0 92 93 static unsigned int allocated_unsafe_pages; 94 95 static void *get_image_page(gfp_t gfp_mask, int safe_needed) 96 { 97 void *res; 98 99 res = (void *)get_zeroed_page(gfp_mask); 100 if (safe_needed) 101 while (res && swsusp_page_is_free(virt_to_page(res))) { 102 /* The page is unsafe, mark it for swsusp_free() */ 103 swsusp_set_page_forbidden(virt_to_page(res)); 104 allocated_unsafe_pages++; 105 res = (void *)get_zeroed_page(gfp_mask); 106 } 107 if (res) { 108 swsusp_set_page_forbidden(virt_to_page(res)); 109 swsusp_set_page_free(virt_to_page(res)); 110 } 111 return res; 112 } 113 114 unsigned long get_safe_page(gfp_t gfp_mask) 115 { 116 return (unsigned long)get_image_page(gfp_mask, PG_SAFE); 117 } 118 119 static struct page *alloc_image_page(gfp_t gfp_mask) 120 { 121 struct page *page; 122 123 page = alloc_page(gfp_mask); 124 if (page) { 125 swsusp_set_page_forbidden(page); 126 swsusp_set_page_free(page); 127 } 128 return page; 129 } 130 131 /** 132 * free_image_page - free page represented by @addr, allocated with 133 * get_image_page (page flags set by it must be cleared) 134 */ 135 136 static inline void free_image_page(void *addr, int clear_nosave_free) 137 { 138 struct page *page; 139 140 BUG_ON(!virt_addr_valid(addr)); 141 142 page = virt_to_page(addr); 143 144 swsusp_unset_page_forbidden(page); 145 if (clear_nosave_free) 146 swsusp_unset_page_free(page); 147 148 __free_page(page); 149 } 150 151 /* struct linked_page is used to build chains of pages */ 152 153 #define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *)) 154 155 struct linked_page { 156 struct linked_page *next; 157 char data[LINKED_PAGE_DATA_SIZE]; 158 } __attribute__((packed)); 159 160 static inline void 161 free_list_of_pages(struct linked_page *list, int clear_page_nosave) 162 { 163 while (list) { 164 struct linked_page *lp = list->next; 165 166 free_image_page(list, clear_page_nosave); 167 list = lp; 168 } 169 } 170 171 /** 172 * struct chain_allocator is used for allocating small objects out of 173 * a linked list of pages called 'the chain'. 174 * 175 * The chain grows each time when there is no room for a new object in 176 * the current page. The allocated objects cannot be freed individually. 177 * It is only possible to free them all at once, by freeing the entire 178 * chain. 179 * 180 * NOTE: The chain allocator may be inefficient if the allocated objects 181 * are not much smaller than PAGE_SIZE. 182 */ 183 184 struct chain_allocator { 185 struct linked_page *chain; /* the chain */ 186 unsigned int used_space; /* total size of objects allocated out 187 * of the current page 188 */ 189 gfp_t gfp_mask; /* mask for allocating pages */ 190 int safe_needed; /* if set, only "safe" pages are allocated */ 191 }; 192 193 static void 194 chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed) 195 { 196 ca->chain = NULL; 197 ca->used_space = LINKED_PAGE_DATA_SIZE; 198 ca->gfp_mask = gfp_mask; 199 ca->safe_needed = safe_needed; 200 } 201 202 static void *chain_alloc(struct chain_allocator *ca, unsigned int size) 203 { 204 void *ret; 205 206 if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) { 207 struct linked_page *lp; 208 209 lp = get_image_page(ca->gfp_mask, ca->safe_needed); 210 if (!lp) 211 return NULL; 212 213 lp->next = ca->chain; 214 ca->chain = lp; 215 ca->used_space = 0; 216 } 217 ret = ca->chain->data + ca->used_space; 218 ca->used_space += size; 219 return ret; 220 } 221 222 /** 223 * Data types related to memory bitmaps. 224 * 225 * Memory bitmap is a structure consiting of many linked lists of 226 * objects. The main list's elements are of type struct zone_bitmap 227 * and each of them corresonds to one zone. For each zone bitmap 228 * object there is a list of objects of type struct bm_block that 229 * represent each blocks of bitmap in which information is stored. 230 * 231 * struct memory_bitmap contains a pointer to the main list of zone 232 * bitmap objects, a struct bm_position used for browsing the bitmap, 233 * and a pointer to the list of pages used for allocating all of the 234 * zone bitmap objects and bitmap block objects. 235 * 236 * NOTE: It has to be possible to lay out the bitmap in memory 237 * using only allocations of order 0. Additionally, the bitmap is 238 * designed to work with arbitrary number of zones (this is over the 239 * top for now, but let's avoid making unnecessary assumptions ;-). 240 * 241 * struct zone_bitmap contains a pointer to a list of bitmap block 242 * objects and a pointer to the bitmap block object that has been 243 * most recently used for setting bits. Additionally, it contains the 244 * pfns that correspond to the start and end of the represented zone. 245 * 246 * struct bm_block contains a pointer to the memory page in which 247 * information is stored (in the form of a block of bitmap) 248 * It also contains the pfns that correspond to the start and end of 249 * the represented memory area. 250 */ 251 252 #define BM_END_OF_MAP (~0UL) 253 254 #define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE) 255 256 struct bm_block { 257 struct list_head hook; /* hook into a list of bitmap blocks */ 258 unsigned long start_pfn; /* pfn represented by the first bit */ 259 unsigned long end_pfn; /* pfn represented by the last bit plus 1 */ 260 unsigned long *data; /* bitmap representing pages */ 261 }; 262 263 static inline unsigned long bm_block_bits(struct bm_block *bb) 264 { 265 return bb->end_pfn - bb->start_pfn; 266 } 267 268 /* strcut bm_position is used for browsing memory bitmaps */ 269 270 struct bm_position { 271 struct bm_block *block; 272 int bit; 273 }; 274 275 struct memory_bitmap { 276 struct list_head blocks; /* list of bitmap blocks */ 277 struct linked_page *p_list; /* list of pages used to store zone 278 * bitmap objects and bitmap block 279 * objects 280 */ 281 struct bm_position cur; /* most recently used bit position */ 282 }; 283 284 /* Functions that operate on memory bitmaps */ 285 286 static void memory_bm_position_reset(struct memory_bitmap *bm) 287 { 288 bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook); 289 bm->cur.bit = 0; 290 } 291 292 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free); 293 294 /** 295 * create_bm_block_list - create a list of block bitmap objects 296 * @pages - number of pages to track 297 * @list - list to put the allocated blocks into 298 * @ca - chain allocator to be used for allocating memory 299 */ 300 static int create_bm_block_list(unsigned long pages, 301 struct list_head *list, 302 struct chain_allocator *ca) 303 { 304 unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK); 305 306 while (nr_blocks-- > 0) { 307 struct bm_block *bb; 308 309 bb = chain_alloc(ca, sizeof(struct bm_block)); 310 if (!bb) 311 return -ENOMEM; 312 list_add(&bb->hook, list); 313 } 314 315 return 0; 316 } 317 318 struct mem_extent { 319 struct list_head hook; 320 unsigned long start; 321 unsigned long end; 322 }; 323 324 /** 325 * free_mem_extents - free a list of memory extents 326 * @list - list of extents to empty 327 */ 328 static void free_mem_extents(struct list_head *list) 329 { 330 struct mem_extent *ext, *aux; 331 332 list_for_each_entry_safe(ext, aux, list, hook) { 333 list_del(&ext->hook); 334 kfree(ext); 335 } 336 } 337 338 /** 339 * create_mem_extents - create a list of memory extents representing 340 * contiguous ranges of PFNs 341 * @list - list to put the extents into 342 * @gfp_mask - mask to use for memory allocations 343 */ 344 static int create_mem_extents(struct list_head *list, gfp_t gfp_mask) 345 { 346 struct zone *zone; 347 348 INIT_LIST_HEAD(list); 349 350 for_each_populated_zone(zone) { 351 unsigned long zone_start, zone_end; 352 struct mem_extent *ext, *cur, *aux; 353 354 zone_start = zone->zone_start_pfn; 355 zone_end = zone_end_pfn(zone); 356 357 list_for_each_entry(ext, list, hook) 358 if (zone_start <= ext->end) 359 break; 360 361 if (&ext->hook == list || zone_end < ext->start) { 362 /* New extent is necessary */ 363 struct mem_extent *new_ext; 364 365 new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask); 366 if (!new_ext) { 367 free_mem_extents(list); 368 return -ENOMEM; 369 } 370 new_ext->start = zone_start; 371 new_ext->end = zone_end; 372 list_add_tail(&new_ext->hook, &ext->hook); 373 continue; 374 } 375 376 /* Merge this zone's range of PFNs with the existing one */ 377 if (zone_start < ext->start) 378 ext->start = zone_start; 379 if (zone_end > ext->end) 380 ext->end = zone_end; 381 382 /* More merging may be possible */ 383 cur = ext; 384 list_for_each_entry_safe_continue(cur, aux, list, hook) { 385 if (zone_end < cur->start) 386 break; 387 if (zone_end < cur->end) 388 ext->end = cur->end; 389 list_del(&cur->hook); 390 kfree(cur); 391 } 392 } 393 394 return 0; 395 } 396 397 /** 398 * memory_bm_create - allocate memory for a memory bitmap 399 */ 400 static int 401 memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed) 402 { 403 struct chain_allocator ca; 404 struct list_head mem_extents; 405 struct mem_extent *ext; 406 int error; 407 408 chain_init(&ca, gfp_mask, safe_needed); 409 INIT_LIST_HEAD(&bm->blocks); 410 411 error = create_mem_extents(&mem_extents, gfp_mask); 412 if (error) 413 return error; 414 415 list_for_each_entry(ext, &mem_extents, hook) { 416 struct bm_block *bb; 417 unsigned long pfn = ext->start; 418 unsigned long pages = ext->end - ext->start; 419 420 bb = list_entry(bm->blocks.prev, struct bm_block, hook); 421 422 error = create_bm_block_list(pages, bm->blocks.prev, &ca); 423 if (error) 424 goto Error; 425 426 list_for_each_entry_continue(bb, &bm->blocks, hook) { 427 bb->data = get_image_page(gfp_mask, safe_needed); 428 if (!bb->data) { 429 error = -ENOMEM; 430 goto Error; 431 } 432 433 bb->start_pfn = pfn; 434 if (pages >= BM_BITS_PER_BLOCK) { 435 pfn += BM_BITS_PER_BLOCK; 436 pages -= BM_BITS_PER_BLOCK; 437 } else { 438 /* This is executed only once in the loop */ 439 pfn += pages; 440 } 441 bb->end_pfn = pfn; 442 } 443 } 444 445 bm->p_list = ca.chain; 446 memory_bm_position_reset(bm); 447 Exit: 448 free_mem_extents(&mem_extents); 449 return error; 450 451 Error: 452 bm->p_list = ca.chain; 453 memory_bm_free(bm, PG_UNSAFE_CLEAR); 454 goto Exit; 455 } 456 457 /** 458 * memory_bm_free - free memory occupied by the memory bitmap @bm 459 */ 460 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) 461 { 462 struct bm_block *bb; 463 464 list_for_each_entry(bb, &bm->blocks, hook) 465 if (bb->data) 466 free_image_page(bb->data, clear_nosave_free); 467 468 free_list_of_pages(bm->p_list, clear_nosave_free); 469 470 INIT_LIST_HEAD(&bm->blocks); 471 } 472 473 /** 474 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds 475 * to given pfn. The cur_zone_bm member of @bm and the cur_block member 476 * of @bm->cur_zone_bm are updated. 477 */ 478 static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn, 479 void **addr, unsigned int *bit_nr) 480 { 481 struct bm_block *bb; 482 483 /* 484 * Check if the pfn corresponds to the current bitmap block and find 485 * the block where it fits if this is not the case. 486 */ 487 bb = bm->cur.block; 488 if (pfn < bb->start_pfn) 489 list_for_each_entry_continue_reverse(bb, &bm->blocks, hook) 490 if (pfn >= bb->start_pfn) 491 break; 492 493 if (pfn >= bb->end_pfn) 494 list_for_each_entry_continue(bb, &bm->blocks, hook) 495 if (pfn >= bb->start_pfn && pfn < bb->end_pfn) 496 break; 497 498 if (&bb->hook == &bm->blocks) 499 return -EFAULT; 500 501 /* The block has been found */ 502 bm->cur.block = bb; 503 pfn -= bb->start_pfn; 504 bm->cur.bit = pfn + 1; 505 *bit_nr = pfn; 506 *addr = bb->data; 507 return 0; 508 } 509 510 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn) 511 { 512 void *addr; 513 unsigned int bit; 514 int error; 515 516 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 517 BUG_ON(error); 518 set_bit(bit, addr); 519 } 520 521 static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn) 522 { 523 void *addr; 524 unsigned int bit; 525 int error; 526 527 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 528 if (!error) 529 set_bit(bit, addr); 530 return error; 531 } 532 533 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn) 534 { 535 void *addr; 536 unsigned int bit; 537 int error; 538 539 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 540 BUG_ON(error); 541 clear_bit(bit, addr); 542 } 543 544 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn) 545 { 546 void *addr; 547 unsigned int bit; 548 int error; 549 550 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 551 BUG_ON(error); 552 return test_bit(bit, addr); 553 } 554 555 static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn) 556 { 557 void *addr; 558 unsigned int bit; 559 560 return !memory_bm_find_bit(bm, pfn, &addr, &bit); 561 } 562 563 /** 564 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit 565 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is 566 * returned. 567 * 568 * It is required to run memory_bm_position_reset() before the first call to 569 * this function. 570 */ 571 572 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm) 573 { 574 struct bm_block *bb; 575 int bit; 576 577 bb = bm->cur.block; 578 do { 579 bit = bm->cur.bit; 580 bit = find_next_bit(bb->data, bm_block_bits(bb), bit); 581 if (bit < bm_block_bits(bb)) 582 goto Return_pfn; 583 584 bb = list_entry(bb->hook.next, struct bm_block, hook); 585 bm->cur.block = bb; 586 bm->cur.bit = 0; 587 } while (&bb->hook != &bm->blocks); 588 589 memory_bm_position_reset(bm); 590 return BM_END_OF_MAP; 591 592 Return_pfn: 593 bm->cur.bit = bit + 1; 594 return bb->start_pfn + bit; 595 } 596 597 /** 598 * This structure represents a range of page frames the contents of which 599 * should not be saved during the suspend. 600 */ 601 602 struct nosave_region { 603 struct list_head list; 604 unsigned long start_pfn; 605 unsigned long end_pfn; 606 }; 607 608 static LIST_HEAD(nosave_regions); 609 610 /** 611 * register_nosave_region - register a range of page frames the contents 612 * of which should not be saved during the suspend (to be used in the early 613 * initialization code) 614 */ 615 616 void __init 617 __register_nosave_region(unsigned long start_pfn, unsigned long end_pfn, 618 int use_kmalloc) 619 { 620 struct nosave_region *region; 621 622 if (start_pfn >= end_pfn) 623 return; 624 625 if (!list_empty(&nosave_regions)) { 626 /* Try to extend the previous region (they should be sorted) */ 627 region = list_entry(nosave_regions.prev, 628 struct nosave_region, list); 629 if (region->end_pfn == start_pfn) { 630 region->end_pfn = end_pfn; 631 goto Report; 632 } 633 } 634 if (use_kmalloc) { 635 /* during init, this shouldn't fail */ 636 region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL); 637 BUG_ON(!region); 638 } else 639 /* This allocation cannot fail */ 640 region = alloc_bootmem(sizeof(struct nosave_region)); 641 region->start_pfn = start_pfn; 642 region->end_pfn = end_pfn; 643 list_add_tail(®ion->list, &nosave_regions); 644 Report: 645 printk(KERN_INFO "PM: Registered nosave memory: [mem %#010llx-%#010llx]\n", 646 (unsigned long long) start_pfn << PAGE_SHIFT, 647 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1); 648 } 649 650 /* 651 * Set bits in this map correspond to the page frames the contents of which 652 * should not be saved during the suspend. 653 */ 654 static struct memory_bitmap *forbidden_pages_map; 655 656 /* Set bits in this map correspond to free page frames. */ 657 static struct memory_bitmap *free_pages_map; 658 659 /* 660 * Each page frame allocated for creating the image is marked by setting the 661 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously 662 */ 663 664 void swsusp_set_page_free(struct page *page) 665 { 666 if (free_pages_map) 667 memory_bm_set_bit(free_pages_map, page_to_pfn(page)); 668 } 669 670 static int swsusp_page_is_free(struct page *page) 671 { 672 return free_pages_map ? 673 memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0; 674 } 675 676 void swsusp_unset_page_free(struct page *page) 677 { 678 if (free_pages_map) 679 memory_bm_clear_bit(free_pages_map, page_to_pfn(page)); 680 } 681 682 static void swsusp_set_page_forbidden(struct page *page) 683 { 684 if (forbidden_pages_map) 685 memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page)); 686 } 687 688 int swsusp_page_is_forbidden(struct page *page) 689 { 690 return forbidden_pages_map ? 691 memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0; 692 } 693 694 static void swsusp_unset_page_forbidden(struct page *page) 695 { 696 if (forbidden_pages_map) 697 memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page)); 698 } 699 700 /** 701 * mark_nosave_pages - set bits corresponding to the page frames the 702 * contents of which should not be saved in a given bitmap. 703 */ 704 705 static void mark_nosave_pages(struct memory_bitmap *bm) 706 { 707 struct nosave_region *region; 708 709 if (list_empty(&nosave_regions)) 710 return; 711 712 list_for_each_entry(region, &nosave_regions, list) { 713 unsigned long pfn; 714 715 pr_debug("PM: Marking nosave pages: [mem %#010llx-%#010llx]\n", 716 (unsigned long long) region->start_pfn << PAGE_SHIFT, 717 ((unsigned long long) region->end_pfn << PAGE_SHIFT) 718 - 1); 719 720 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++) 721 if (pfn_valid(pfn)) { 722 /* 723 * It is safe to ignore the result of 724 * mem_bm_set_bit_check() here, since we won't 725 * touch the PFNs for which the error is 726 * returned anyway. 727 */ 728 mem_bm_set_bit_check(bm, pfn); 729 } 730 } 731 } 732 733 /** 734 * create_basic_memory_bitmaps - create bitmaps needed for marking page 735 * frames that should not be saved and free page frames. The pointers 736 * forbidden_pages_map and free_pages_map are only modified if everything 737 * goes well, because we don't want the bits to be used before both bitmaps 738 * are set up. 739 */ 740 741 int create_basic_memory_bitmaps(void) 742 { 743 struct memory_bitmap *bm1, *bm2; 744 int error = 0; 745 746 if (forbidden_pages_map && free_pages_map) 747 return 0; 748 else 749 BUG_ON(forbidden_pages_map || free_pages_map); 750 751 bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL); 752 if (!bm1) 753 return -ENOMEM; 754 755 error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY); 756 if (error) 757 goto Free_first_object; 758 759 bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL); 760 if (!bm2) 761 goto Free_first_bitmap; 762 763 error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY); 764 if (error) 765 goto Free_second_object; 766 767 forbidden_pages_map = bm1; 768 free_pages_map = bm2; 769 mark_nosave_pages(forbidden_pages_map); 770 771 pr_debug("PM: Basic memory bitmaps created\n"); 772 773 return 0; 774 775 Free_second_object: 776 kfree(bm2); 777 Free_first_bitmap: 778 memory_bm_free(bm1, PG_UNSAFE_CLEAR); 779 Free_first_object: 780 kfree(bm1); 781 return -ENOMEM; 782 } 783 784 /** 785 * free_basic_memory_bitmaps - free memory bitmaps allocated by 786 * create_basic_memory_bitmaps(). The auxiliary pointers are necessary 787 * so that the bitmaps themselves are not referred to while they are being 788 * freed. 789 */ 790 791 void free_basic_memory_bitmaps(void) 792 { 793 struct memory_bitmap *bm1, *bm2; 794 795 BUG_ON(!(forbidden_pages_map && free_pages_map)); 796 797 bm1 = forbidden_pages_map; 798 bm2 = free_pages_map; 799 forbidden_pages_map = NULL; 800 free_pages_map = NULL; 801 memory_bm_free(bm1, PG_UNSAFE_CLEAR); 802 kfree(bm1); 803 memory_bm_free(bm2, PG_UNSAFE_CLEAR); 804 kfree(bm2); 805 806 pr_debug("PM: Basic memory bitmaps freed\n"); 807 } 808 809 /** 810 * snapshot_additional_pages - estimate the number of additional pages 811 * be needed for setting up the suspend image data structures for given 812 * zone (usually the returned value is greater than the exact number) 813 */ 814 815 unsigned int snapshot_additional_pages(struct zone *zone) 816 { 817 unsigned int res; 818 819 res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK); 820 res += DIV_ROUND_UP(res * sizeof(struct bm_block), 821 LINKED_PAGE_DATA_SIZE); 822 return 2 * res; 823 } 824 825 #ifdef CONFIG_HIGHMEM 826 /** 827 * count_free_highmem_pages - compute the total number of free highmem 828 * pages, system-wide. 829 */ 830 831 static unsigned int count_free_highmem_pages(void) 832 { 833 struct zone *zone; 834 unsigned int cnt = 0; 835 836 for_each_populated_zone(zone) 837 if (is_highmem(zone)) 838 cnt += zone_page_state(zone, NR_FREE_PAGES); 839 840 return cnt; 841 } 842 843 /** 844 * saveable_highmem_page - Determine whether a highmem page should be 845 * included in the suspend image. 846 * 847 * We should save the page if it isn't Nosave or NosaveFree, or Reserved, 848 * and it isn't a part of a free chunk of pages. 849 */ 850 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn) 851 { 852 struct page *page; 853 854 if (!pfn_valid(pfn)) 855 return NULL; 856 857 page = pfn_to_page(pfn); 858 if (page_zone(page) != zone) 859 return NULL; 860 861 BUG_ON(!PageHighMem(page)); 862 863 if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page) || 864 PageReserved(page)) 865 return NULL; 866 867 if (page_is_guard(page)) 868 return NULL; 869 870 return page; 871 } 872 873 /** 874 * count_highmem_pages - compute the total number of saveable highmem 875 * pages. 876 */ 877 878 static unsigned int count_highmem_pages(void) 879 { 880 struct zone *zone; 881 unsigned int n = 0; 882 883 for_each_populated_zone(zone) { 884 unsigned long pfn, max_zone_pfn; 885 886 if (!is_highmem(zone)) 887 continue; 888 889 mark_free_pages(zone); 890 max_zone_pfn = zone_end_pfn(zone); 891 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 892 if (saveable_highmem_page(zone, pfn)) 893 n++; 894 } 895 return n; 896 } 897 #else 898 static inline void *saveable_highmem_page(struct zone *z, unsigned long p) 899 { 900 return NULL; 901 } 902 #endif /* CONFIG_HIGHMEM */ 903 904 /** 905 * saveable_page - Determine whether a non-highmem page should be included 906 * in the suspend image. 907 * 908 * We should save the page if it isn't Nosave, and is not in the range 909 * of pages statically defined as 'unsaveable', and it isn't a part of 910 * a free chunk of pages. 911 */ 912 static struct page *saveable_page(struct zone *zone, unsigned long pfn) 913 { 914 struct page *page; 915 916 if (!pfn_valid(pfn)) 917 return NULL; 918 919 page = pfn_to_page(pfn); 920 if (page_zone(page) != zone) 921 return NULL; 922 923 BUG_ON(PageHighMem(page)); 924 925 if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page)) 926 return NULL; 927 928 if (PageReserved(page) 929 && (!kernel_page_present(page) || pfn_is_nosave(pfn))) 930 return NULL; 931 932 if (page_is_guard(page)) 933 return NULL; 934 935 return page; 936 } 937 938 /** 939 * count_data_pages - compute the total number of saveable non-highmem 940 * pages. 941 */ 942 943 static unsigned int count_data_pages(void) 944 { 945 struct zone *zone; 946 unsigned long pfn, max_zone_pfn; 947 unsigned int n = 0; 948 949 for_each_populated_zone(zone) { 950 if (is_highmem(zone)) 951 continue; 952 953 mark_free_pages(zone); 954 max_zone_pfn = zone_end_pfn(zone); 955 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 956 if (saveable_page(zone, pfn)) 957 n++; 958 } 959 return n; 960 } 961 962 /* This is needed, because copy_page and memcpy are not usable for copying 963 * task structs. 964 */ 965 static inline void do_copy_page(long *dst, long *src) 966 { 967 int n; 968 969 for (n = PAGE_SIZE / sizeof(long); n; n--) 970 *dst++ = *src++; 971 } 972 973 974 /** 975 * safe_copy_page - check if the page we are going to copy is marked as 976 * present in the kernel page tables (this always is the case if 977 * CONFIG_DEBUG_PAGEALLOC is not set and in that case 978 * kernel_page_present() always returns 'true'). 979 */ 980 static void safe_copy_page(void *dst, struct page *s_page) 981 { 982 if (kernel_page_present(s_page)) { 983 do_copy_page(dst, page_address(s_page)); 984 } else { 985 kernel_map_pages(s_page, 1, 1); 986 do_copy_page(dst, page_address(s_page)); 987 kernel_map_pages(s_page, 1, 0); 988 } 989 } 990 991 992 #ifdef CONFIG_HIGHMEM 993 static inline struct page * 994 page_is_saveable(struct zone *zone, unsigned long pfn) 995 { 996 return is_highmem(zone) ? 997 saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn); 998 } 999 1000 static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) 1001 { 1002 struct page *s_page, *d_page; 1003 void *src, *dst; 1004 1005 s_page = pfn_to_page(src_pfn); 1006 d_page = pfn_to_page(dst_pfn); 1007 if (PageHighMem(s_page)) { 1008 src = kmap_atomic(s_page); 1009 dst = kmap_atomic(d_page); 1010 do_copy_page(dst, src); 1011 kunmap_atomic(dst); 1012 kunmap_atomic(src); 1013 } else { 1014 if (PageHighMem(d_page)) { 1015 /* Page pointed to by src may contain some kernel 1016 * data modified by kmap_atomic() 1017 */ 1018 safe_copy_page(buffer, s_page); 1019 dst = kmap_atomic(d_page); 1020 copy_page(dst, buffer); 1021 kunmap_atomic(dst); 1022 } else { 1023 safe_copy_page(page_address(d_page), s_page); 1024 } 1025 } 1026 } 1027 #else 1028 #define page_is_saveable(zone, pfn) saveable_page(zone, pfn) 1029 1030 static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) 1031 { 1032 safe_copy_page(page_address(pfn_to_page(dst_pfn)), 1033 pfn_to_page(src_pfn)); 1034 } 1035 #endif /* CONFIG_HIGHMEM */ 1036 1037 static void 1038 copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm) 1039 { 1040 struct zone *zone; 1041 unsigned long pfn; 1042 1043 for_each_populated_zone(zone) { 1044 unsigned long max_zone_pfn; 1045 1046 mark_free_pages(zone); 1047 max_zone_pfn = zone_end_pfn(zone); 1048 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1049 if (page_is_saveable(zone, pfn)) 1050 memory_bm_set_bit(orig_bm, pfn); 1051 } 1052 memory_bm_position_reset(orig_bm); 1053 memory_bm_position_reset(copy_bm); 1054 for(;;) { 1055 pfn = memory_bm_next_pfn(orig_bm); 1056 if (unlikely(pfn == BM_END_OF_MAP)) 1057 break; 1058 copy_data_page(memory_bm_next_pfn(copy_bm), pfn); 1059 } 1060 } 1061 1062 /* Total number of image pages */ 1063 static unsigned int nr_copy_pages; 1064 /* Number of pages needed for saving the original pfns of the image pages */ 1065 static unsigned int nr_meta_pages; 1066 /* 1067 * Numbers of normal and highmem page frames allocated for hibernation image 1068 * before suspending devices. 1069 */ 1070 unsigned int alloc_normal, alloc_highmem; 1071 /* 1072 * Memory bitmap used for marking saveable pages (during hibernation) or 1073 * hibernation image pages (during restore) 1074 */ 1075 static struct memory_bitmap orig_bm; 1076 /* 1077 * Memory bitmap used during hibernation for marking allocated page frames that 1078 * will contain copies of saveable pages. During restore it is initially used 1079 * for marking hibernation image pages, but then the set bits from it are 1080 * duplicated in @orig_bm and it is released. On highmem systems it is next 1081 * used for marking "safe" highmem pages, but it has to be reinitialized for 1082 * this purpose. 1083 */ 1084 static struct memory_bitmap copy_bm; 1085 1086 /** 1087 * swsusp_free - free pages allocated for the suspend. 1088 * 1089 * Suspend pages are alocated before the atomic copy is made, so we 1090 * need to release them after the resume. 1091 */ 1092 1093 void swsusp_free(void) 1094 { 1095 struct zone *zone; 1096 unsigned long pfn, max_zone_pfn; 1097 1098 for_each_populated_zone(zone) { 1099 max_zone_pfn = zone_end_pfn(zone); 1100 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1101 if (pfn_valid(pfn)) { 1102 struct page *page = pfn_to_page(pfn); 1103 1104 if (swsusp_page_is_forbidden(page) && 1105 swsusp_page_is_free(page)) { 1106 swsusp_unset_page_forbidden(page); 1107 swsusp_unset_page_free(page); 1108 __free_page(page); 1109 } 1110 } 1111 } 1112 nr_copy_pages = 0; 1113 nr_meta_pages = 0; 1114 restore_pblist = NULL; 1115 buffer = NULL; 1116 alloc_normal = 0; 1117 alloc_highmem = 0; 1118 } 1119 1120 /* Helper functions used for the shrinking of memory. */ 1121 1122 #define GFP_IMAGE (GFP_KERNEL | __GFP_NOWARN) 1123 1124 /** 1125 * preallocate_image_pages - Allocate a number of pages for hibernation image 1126 * @nr_pages: Number of page frames to allocate. 1127 * @mask: GFP flags to use for the allocation. 1128 * 1129 * Return value: Number of page frames actually allocated 1130 */ 1131 static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask) 1132 { 1133 unsigned long nr_alloc = 0; 1134 1135 while (nr_pages > 0) { 1136 struct page *page; 1137 1138 page = alloc_image_page(mask); 1139 if (!page) 1140 break; 1141 memory_bm_set_bit(©_bm, page_to_pfn(page)); 1142 if (PageHighMem(page)) 1143 alloc_highmem++; 1144 else 1145 alloc_normal++; 1146 nr_pages--; 1147 nr_alloc++; 1148 } 1149 1150 return nr_alloc; 1151 } 1152 1153 static unsigned long preallocate_image_memory(unsigned long nr_pages, 1154 unsigned long avail_normal) 1155 { 1156 unsigned long alloc; 1157 1158 if (avail_normal <= alloc_normal) 1159 return 0; 1160 1161 alloc = avail_normal - alloc_normal; 1162 if (nr_pages < alloc) 1163 alloc = nr_pages; 1164 1165 return preallocate_image_pages(alloc, GFP_IMAGE); 1166 } 1167 1168 #ifdef CONFIG_HIGHMEM 1169 static unsigned long preallocate_image_highmem(unsigned long nr_pages) 1170 { 1171 return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM); 1172 } 1173 1174 /** 1175 * __fraction - Compute (an approximation of) x * (multiplier / base) 1176 */ 1177 static unsigned long __fraction(u64 x, u64 multiplier, u64 base) 1178 { 1179 x *= multiplier; 1180 do_div(x, base); 1181 return (unsigned long)x; 1182 } 1183 1184 static unsigned long preallocate_highmem_fraction(unsigned long nr_pages, 1185 unsigned long highmem, 1186 unsigned long total) 1187 { 1188 unsigned long alloc = __fraction(nr_pages, highmem, total); 1189 1190 return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM); 1191 } 1192 #else /* CONFIG_HIGHMEM */ 1193 static inline unsigned long preallocate_image_highmem(unsigned long nr_pages) 1194 { 1195 return 0; 1196 } 1197 1198 static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages, 1199 unsigned long highmem, 1200 unsigned long total) 1201 { 1202 return 0; 1203 } 1204 #endif /* CONFIG_HIGHMEM */ 1205 1206 /** 1207 * free_unnecessary_pages - Release preallocated pages not needed for the image 1208 */ 1209 static void free_unnecessary_pages(void) 1210 { 1211 unsigned long save, to_free_normal, to_free_highmem; 1212 1213 save = count_data_pages(); 1214 if (alloc_normal >= save) { 1215 to_free_normal = alloc_normal - save; 1216 save = 0; 1217 } else { 1218 to_free_normal = 0; 1219 save -= alloc_normal; 1220 } 1221 save += count_highmem_pages(); 1222 if (alloc_highmem >= save) { 1223 to_free_highmem = alloc_highmem - save; 1224 } else { 1225 to_free_highmem = 0; 1226 save -= alloc_highmem; 1227 if (to_free_normal > save) 1228 to_free_normal -= save; 1229 else 1230 to_free_normal = 0; 1231 } 1232 1233 memory_bm_position_reset(©_bm); 1234 1235 while (to_free_normal > 0 || to_free_highmem > 0) { 1236 unsigned long pfn = memory_bm_next_pfn(©_bm); 1237 struct page *page = pfn_to_page(pfn); 1238 1239 if (PageHighMem(page)) { 1240 if (!to_free_highmem) 1241 continue; 1242 to_free_highmem--; 1243 alloc_highmem--; 1244 } else { 1245 if (!to_free_normal) 1246 continue; 1247 to_free_normal--; 1248 alloc_normal--; 1249 } 1250 memory_bm_clear_bit(©_bm, pfn); 1251 swsusp_unset_page_forbidden(page); 1252 swsusp_unset_page_free(page); 1253 __free_page(page); 1254 } 1255 } 1256 1257 /** 1258 * minimum_image_size - Estimate the minimum acceptable size of an image 1259 * @saveable: Number of saveable pages in the system. 1260 * 1261 * We want to avoid attempting to free too much memory too hard, so estimate the 1262 * minimum acceptable size of a hibernation image to use as the lower limit for 1263 * preallocating memory. 1264 * 1265 * We assume that the minimum image size should be proportional to 1266 * 1267 * [number of saveable pages] - [number of pages that can be freed in theory] 1268 * 1269 * where the second term is the sum of (1) reclaimable slab pages, (2) active 1270 * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages, 1271 * minus mapped file pages. 1272 */ 1273 static unsigned long minimum_image_size(unsigned long saveable) 1274 { 1275 unsigned long size; 1276 1277 size = global_page_state(NR_SLAB_RECLAIMABLE) 1278 + global_page_state(NR_ACTIVE_ANON) 1279 + global_page_state(NR_INACTIVE_ANON) 1280 + global_page_state(NR_ACTIVE_FILE) 1281 + global_page_state(NR_INACTIVE_FILE) 1282 - global_page_state(NR_FILE_MAPPED); 1283 1284 return saveable <= size ? 0 : saveable - size; 1285 } 1286 1287 /** 1288 * hibernate_preallocate_memory - Preallocate memory for hibernation image 1289 * 1290 * To create a hibernation image it is necessary to make a copy of every page 1291 * frame in use. We also need a number of page frames to be free during 1292 * hibernation for allocations made while saving the image and for device 1293 * drivers, in case they need to allocate memory from their hibernation 1294 * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough 1295 * estimate) and reserverd_size divided by PAGE_SIZE (which is tunable through 1296 * /sys/power/reserved_size, respectively). To make this happen, we compute the 1297 * total number of available page frames and allocate at least 1298 * 1299 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2 1300 * + 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE) 1301 * 1302 * of them, which corresponds to the maximum size of a hibernation image. 1303 * 1304 * If image_size is set below the number following from the above formula, 1305 * the preallocation of memory is continued until the total number of saveable 1306 * pages in the system is below the requested image size or the minimum 1307 * acceptable image size returned by minimum_image_size(), whichever is greater. 1308 */ 1309 int hibernate_preallocate_memory(void) 1310 { 1311 struct zone *zone; 1312 unsigned long saveable, size, max_size, count, highmem, pages = 0; 1313 unsigned long alloc, save_highmem, pages_highmem, avail_normal; 1314 struct timeval start, stop; 1315 int error; 1316 1317 printk(KERN_INFO "PM: Preallocating image memory... "); 1318 do_gettimeofday(&start); 1319 1320 error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY); 1321 if (error) 1322 goto err_out; 1323 1324 error = memory_bm_create(©_bm, GFP_IMAGE, PG_ANY); 1325 if (error) 1326 goto err_out; 1327 1328 alloc_normal = 0; 1329 alloc_highmem = 0; 1330 1331 /* Count the number of saveable data pages. */ 1332 save_highmem = count_highmem_pages(); 1333 saveable = count_data_pages(); 1334 1335 /* 1336 * Compute the total number of page frames we can use (count) and the 1337 * number of pages needed for image metadata (size). 1338 */ 1339 count = saveable; 1340 saveable += save_highmem; 1341 highmem = save_highmem; 1342 size = 0; 1343 for_each_populated_zone(zone) { 1344 size += snapshot_additional_pages(zone); 1345 if (is_highmem(zone)) 1346 highmem += zone_page_state(zone, NR_FREE_PAGES); 1347 else 1348 count += zone_page_state(zone, NR_FREE_PAGES); 1349 } 1350 avail_normal = count; 1351 count += highmem; 1352 count -= totalreserve_pages; 1353 1354 /* Add number of pages required for page keys (s390 only). */ 1355 size += page_key_additional_pages(saveable); 1356 1357 /* Compute the maximum number of saveable pages to leave in memory. */ 1358 max_size = (count - (size + PAGES_FOR_IO)) / 2 1359 - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE); 1360 /* Compute the desired number of image pages specified by image_size. */ 1361 size = DIV_ROUND_UP(image_size, PAGE_SIZE); 1362 if (size > max_size) 1363 size = max_size; 1364 /* 1365 * If the desired number of image pages is at least as large as the 1366 * current number of saveable pages in memory, allocate page frames for 1367 * the image and we're done. 1368 */ 1369 if (size >= saveable) { 1370 pages = preallocate_image_highmem(save_highmem); 1371 pages += preallocate_image_memory(saveable - pages, avail_normal); 1372 goto out; 1373 } 1374 1375 /* Estimate the minimum size of the image. */ 1376 pages = minimum_image_size(saveable); 1377 /* 1378 * To avoid excessive pressure on the normal zone, leave room in it to 1379 * accommodate an image of the minimum size (unless it's already too 1380 * small, in which case don't preallocate pages from it at all). 1381 */ 1382 if (avail_normal > pages) 1383 avail_normal -= pages; 1384 else 1385 avail_normal = 0; 1386 if (size < pages) 1387 size = min_t(unsigned long, pages, max_size); 1388 1389 /* 1390 * Let the memory management subsystem know that we're going to need a 1391 * large number of page frames to allocate and make it free some memory. 1392 * NOTE: If this is not done, performance will be hurt badly in some 1393 * test cases. 1394 */ 1395 shrink_all_memory(saveable - size); 1396 1397 /* 1398 * The number of saveable pages in memory was too high, so apply some 1399 * pressure to decrease it. First, make room for the largest possible 1400 * image and fail if that doesn't work. Next, try to decrease the size 1401 * of the image as much as indicated by 'size' using allocations from 1402 * highmem and non-highmem zones separately. 1403 */ 1404 pages_highmem = preallocate_image_highmem(highmem / 2); 1405 alloc = count - max_size; 1406 if (alloc > pages_highmem) 1407 alloc -= pages_highmem; 1408 else 1409 alloc = 0; 1410 pages = preallocate_image_memory(alloc, avail_normal); 1411 if (pages < alloc) { 1412 /* We have exhausted non-highmem pages, try highmem. */ 1413 alloc -= pages; 1414 pages += pages_highmem; 1415 pages_highmem = preallocate_image_highmem(alloc); 1416 if (pages_highmem < alloc) 1417 goto err_out; 1418 pages += pages_highmem; 1419 /* 1420 * size is the desired number of saveable pages to leave in 1421 * memory, so try to preallocate (all memory - size) pages. 1422 */ 1423 alloc = (count - pages) - size; 1424 pages += preallocate_image_highmem(alloc); 1425 } else { 1426 /* 1427 * There are approximately max_size saveable pages at this point 1428 * and we want to reduce this number down to size. 1429 */ 1430 alloc = max_size - size; 1431 size = preallocate_highmem_fraction(alloc, highmem, count); 1432 pages_highmem += size; 1433 alloc -= size; 1434 size = preallocate_image_memory(alloc, avail_normal); 1435 pages_highmem += preallocate_image_highmem(alloc - size); 1436 pages += pages_highmem + size; 1437 } 1438 1439 /* 1440 * We only need as many page frames for the image as there are saveable 1441 * pages in memory, but we have allocated more. Release the excessive 1442 * ones now. 1443 */ 1444 free_unnecessary_pages(); 1445 1446 out: 1447 do_gettimeofday(&stop); 1448 printk(KERN_CONT "done (allocated %lu pages)\n", pages); 1449 swsusp_show_speed(&start, &stop, pages, "Allocated"); 1450 1451 return 0; 1452 1453 err_out: 1454 printk(KERN_CONT "\n"); 1455 swsusp_free(); 1456 return -ENOMEM; 1457 } 1458 1459 #ifdef CONFIG_HIGHMEM 1460 /** 1461 * count_pages_for_highmem - compute the number of non-highmem pages 1462 * that will be necessary for creating copies of highmem pages. 1463 */ 1464 1465 static unsigned int count_pages_for_highmem(unsigned int nr_highmem) 1466 { 1467 unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem; 1468 1469 if (free_highmem >= nr_highmem) 1470 nr_highmem = 0; 1471 else 1472 nr_highmem -= free_highmem; 1473 1474 return nr_highmem; 1475 } 1476 #else 1477 static unsigned int 1478 count_pages_for_highmem(unsigned int nr_highmem) { return 0; } 1479 #endif /* CONFIG_HIGHMEM */ 1480 1481 /** 1482 * enough_free_mem - Make sure we have enough free memory for the 1483 * snapshot image. 1484 */ 1485 1486 static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem) 1487 { 1488 struct zone *zone; 1489 unsigned int free = alloc_normal; 1490 1491 for_each_populated_zone(zone) 1492 if (!is_highmem(zone)) 1493 free += zone_page_state(zone, NR_FREE_PAGES); 1494 1495 nr_pages += count_pages_for_highmem(nr_highmem); 1496 pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n", 1497 nr_pages, PAGES_FOR_IO, free); 1498 1499 return free > nr_pages + PAGES_FOR_IO; 1500 } 1501 1502 #ifdef CONFIG_HIGHMEM 1503 /** 1504 * get_highmem_buffer - if there are some highmem pages in the suspend 1505 * image, we may need the buffer to copy them and/or load their data. 1506 */ 1507 1508 static inline int get_highmem_buffer(int safe_needed) 1509 { 1510 buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed); 1511 return buffer ? 0 : -ENOMEM; 1512 } 1513 1514 /** 1515 * alloc_highmem_image_pages - allocate some highmem pages for the image. 1516 * Try to allocate as many pages as needed, but if the number of free 1517 * highmem pages is lesser than that, allocate them all. 1518 */ 1519 1520 static inline unsigned int 1521 alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem) 1522 { 1523 unsigned int to_alloc = count_free_highmem_pages(); 1524 1525 if (to_alloc > nr_highmem) 1526 to_alloc = nr_highmem; 1527 1528 nr_highmem -= to_alloc; 1529 while (to_alloc-- > 0) { 1530 struct page *page; 1531 1532 page = alloc_image_page(__GFP_HIGHMEM); 1533 memory_bm_set_bit(bm, page_to_pfn(page)); 1534 } 1535 return nr_highmem; 1536 } 1537 #else 1538 static inline int get_highmem_buffer(int safe_needed) { return 0; } 1539 1540 static inline unsigned int 1541 alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; } 1542 #endif /* CONFIG_HIGHMEM */ 1543 1544 /** 1545 * swsusp_alloc - allocate memory for the suspend image 1546 * 1547 * We first try to allocate as many highmem pages as there are 1548 * saveable highmem pages in the system. If that fails, we allocate 1549 * non-highmem pages for the copies of the remaining highmem ones. 1550 * 1551 * In this approach it is likely that the copies of highmem pages will 1552 * also be located in the high memory, because of the way in which 1553 * copy_data_pages() works. 1554 */ 1555 1556 static int 1557 swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm, 1558 unsigned int nr_pages, unsigned int nr_highmem) 1559 { 1560 if (nr_highmem > 0) { 1561 if (get_highmem_buffer(PG_ANY)) 1562 goto err_out; 1563 if (nr_highmem > alloc_highmem) { 1564 nr_highmem -= alloc_highmem; 1565 nr_pages += alloc_highmem_pages(copy_bm, nr_highmem); 1566 } 1567 } 1568 if (nr_pages > alloc_normal) { 1569 nr_pages -= alloc_normal; 1570 while (nr_pages-- > 0) { 1571 struct page *page; 1572 1573 page = alloc_image_page(GFP_ATOMIC | __GFP_COLD); 1574 if (!page) 1575 goto err_out; 1576 memory_bm_set_bit(copy_bm, page_to_pfn(page)); 1577 } 1578 } 1579 1580 return 0; 1581 1582 err_out: 1583 swsusp_free(); 1584 return -ENOMEM; 1585 } 1586 1587 asmlinkage int swsusp_save(void) 1588 { 1589 unsigned int nr_pages, nr_highmem; 1590 1591 printk(KERN_INFO "PM: Creating hibernation image:\n"); 1592 1593 drain_local_pages(NULL); 1594 nr_pages = count_data_pages(); 1595 nr_highmem = count_highmem_pages(); 1596 printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem); 1597 1598 if (!enough_free_mem(nr_pages, nr_highmem)) { 1599 printk(KERN_ERR "PM: Not enough free memory\n"); 1600 return -ENOMEM; 1601 } 1602 1603 if (swsusp_alloc(&orig_bm, ©_bm, nr_pages, nr_highmem)) { 1604 printk(KERN_ERR "PM: Memory allocation failed\n"); 1605 return -ENOMEM; 1606 } 1607 1608 /* During allocating of suspend pagedir, new cold pages may appear. 1609 * Kill them. 1610 */ 1611 drain_local_pages(NULL); 1612 copy_data_pages(©_bm, &orig_bm); 1613 1614 /* 1615 * End of critical section. From now on, we can write to memory, 1616 * but we should not touch disk. This specially means we must _not_ 1617 * touch swap space! Except we must write out our image of course. 1618 */ 1619 1620 nr_pages += nr_highmem; 1621 nr_copy_pages = nr_pages; 1622 nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE); 1623 1624 printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n", 1625 nr_pages); 1626 1627 return 0; 1628 } 1629 1630 #ifndef CONFIG_ARCH_HIBERNATION_HEADER 1631 static int init_header_complete(struct swsusp_info *info) 1632 { 1633 memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname)); 1634 info->version_code = LINUX_VERSION_CODE; 1635 return 0; 1636 } 1637 1638 static char *check_image_kernel(struct swsusp_info *info) 1639 { 1640 if (info->version_code != LINUX_VERSION_CODE) 1641 return "kernel version"; 1642 if (strcmp(info->uts.sysname,init_utsname()->sysname)) 1643 return "system type"; 1644 if (strcmp(info->uts.release,init_utsname()->release)) 1645 return "kernel release"; 1646 if (strcmp(info->uts.version,init_utsname()->version)) 1647 return "version"; 1648 if (strcmp(info->uts.machine,init_utsname()->machine)) 1649 return "machine"; 1650 return NULL; 1651 } 1652 #endif /* CONFIG_ARCH_HIBERNATION_HEADER */ 1653 1654 unsigned long snapshot_get_image_size(void) 1655 { 1656 return nr_copy_pages + nr_meta_pages + 1; 1657 } 1658 1659 static int init_header(struct swsusp_info *info) 1660 { 1661 memset(info, 0, sizeof(struct swsusp_info)); 1662 info->num_physpages = get_num_physpages(); 1663 info->image_pages = nr_copy_pages; 1664 info->pages = snapshot_get_image_size(); 1665 info->size = info->pages; 1666 info->size <<= PAGE_SHIFT; 1667 return init_header_complete(info); 1668 } 1669 1670 /** 1671 * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm 1672 * are stored in the array @buf[] (1 page at a time) 1673 */ 1674 1675 static inline void 1676 pack_pfns(unsigned long *buf, struct memory_bitmap *bm) 1677 { 1678 int j; 1679 1680 for (j = 0; j < PAGE_SIZE / sizeof(long); j++) { 1681 buf[j] = memory_bm_next_pfn(bm); 1682 if (unlikely(buf[j] == BM_END_OF_MAP)) 1683 break; 1684 /* Save page key for data page (s390 only). */ 1685 page_key_read(buf + j); 1686 } 1687 } 1688 1689 /** 1690 * snapshot_read_next - used for reading the system memory snapshot. 1691 * 1692 * On the first call to it @handle should point to a zeroed 1693 * snapshot_handle structure. The structure gets updated and a pointer 1694 * to it should be passed to this function every next time. 1695 * 1696 * On success the function returns a positive number. Then, the caller 1697 * is allowed to read up to the returned number of bytes from the memory 1698 * location computed by the data_of() macro. 1699 * 1700 * The function returns 0 to indicate the end of data stream condition, 1701 * and a negative number is returned on error. In such cases the 1702 * structure pointed to by @handle is not updated and should not be used 1703 * any more. 1704 */ 1705 1706 int snapshot_read_next(struct snapshot_handle *handle) 1707 { 1708 if (handle->cur > nr_meta_pages + nr_copy_pages) 1709 return 0; 1710 1711 if (!buffer) { 1712 /* This makes the buffer be freed by swsusp_free() */ 1713 buffer = get_image_page(GFP_ATOMIC, PG_ANY); 1714 if (!buffer) 1715 return -ENOMEM; 1716 } 1717 if (!handle->cur) { 1718 int error; 1719 1720 error = init_header((struct swsusp_info *)buffer); 1721 if (error) 1722 return error; 1723 handle->buffer = buffer; 1724 memory_bm_position_reset(&orig_bm); 1725 memory_bm_position_reset(©_bm); 1726 } else if (handle->cur <= nr_meta_pages) { 1727 clear_page(buffer); 1728 pack_pfns(buffer, &orig_bm); 1729 } else { 1730 struct page *page; 1731 1732 page = pfn_to_page(memory_bm_next_pfn(©_bm)); 1733 if (PageHighMem(page)) { 1734 /* Highmem pages are copied to the buffer, 1735 * because we can't return with a kmapped 1736 * highmem page (we may not be called again). 1737 */ 1738 void *kaddr; 1739 1740 kaddr = kmap_atomic(page); 1741 copy_page(buffer, kaddr); 1742 kunmap_atomic(kaddr); 1743 handle->buffer = buffer; 1744 } else { 1745 handle->buffer = page_address(page); 1746 } 1747 } 1748 handle->cur++; 1749 return PAGE_SIZE; 1750 } 1751 1752 /** 1753 * mark_unsafe_pages - mark the pages that cannot be used for storing 1754 * the image during resume, because they conflict with the pages that 1755 * had been used before suspend 1756 */ 1757 1758 static int mark_unsafe_pages(struct memory_bitmap *bm) 1759 { 1760 struct zone *zone; 1761 unsigned long pfn, max_zone_pfn; 1762 1763 /* Clear page flags */ 1764 for_each_populated_zone(zone) { 1765 max_zone_pfn = zone_end_pfn(zone); 1766 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1767 if (pfn_valid(pfn)) 1768 swsusp_unset_page_free(pfn_to_page(pfn)); 1769 } 1770 1771 /* Mark pages that correspond to the "original" pfns as "unsafe" */ 1772 memory_bm_position_reset(bm); 1773 do { 1774 pfn = memory_bm_next_pfn(bm); 1775 if (likely(pfn != BM_END_OF_MAP)) { 1776 if (likely(pfn_valid(pfn))) 1777 swsusp_set_page_free(pfn_to_page(pfn)); 1778 else 1779 return -EFAULT; 1780 } 1781 } while (pfn != BM_END_OF_MAP); 1782 1783 allocated_unsafe_pages = 0; 1784 1785 return 0; 1786 } 1787 1788 static void 1789 duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src) 1790 { 1791 unsigned long pfn; 1792 1793 memory_bm_position_reset(src); 1794 pfn = memory_bm_next_pfn(src); 1795 while (pfn != BM_END_OF_MAP) { 1796 memory_bm_set_bit(dst, pfn); 1797 pfn = memory_bm_next_pfn(src); 1798 } 1799 } 1800 1801 static int check_header(struct swsusp_info *info) 1802 { 1803 char *reason; 1804 1805 reason = check_image_kernel(info); 1806 if (!reason && info->num_physpages != get_num_physpages()) 1807 reason = "memory size"; 1808 if (reason) { 1809 printk(KERN_ERR "PM: Image mismatch: %s\n", reason); 1810 return -EPERM; 1811 } 1812 return 0; 1813 } 1814 1815 /** 1816 * load header - check the image header and copy data from it 1817 */ 1818 1819 static int 1820 load_header(struct swsusp_info *info) 1821 { 1822 int error; 1823 1824 restore_pblist = NULL; 1825 error = check_header(info); 1826 if (!error) { 1827 nr_copy_pages = info->image_pages; 1828 nr_meta_pages = info->pages - info->image_pages - 1; 1829 } 1830 return error; 1831 } 1832 1833 /** 1834 * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set 1835 * the corresponding bit in the memory bitmap @bm 1836 */ 1837 static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm) 1838 { 1839 int j; 1840 1841 for (j = 0; j < PAGE_SIZE / sizeof(long); j++) { 1842 if (unlikely(buf[j] == BM_END_OF_MAP)) 1843 break; 1844 1845 /* Extract and buffer page key for data page (s390 only). */ 1846 page_key_memorize(buf + j); 1847 1848 if (memory_bm_pfn_present(bm, buf[j])) 1849 memory_bm_set_bit(bm, buf[j]); 1850 else 1851 return -EFAULT; 1852 } 1853 1854 return 0; 1855 } 1856 1857 /* List of "safe" pages that may be used to store data loaded from the suspend 1858 * image 1859 */ 1860 static struct linked_page *safe_pages_list; 1861 1862 #ifdef CONFIG_HIGHMEM 1863 /* struct highmem_pbe is used for creating the list of highmem pages that 1864 * should be restored atomically during the resume from disk, because the page 1865 * frames they have occupied before the suspend are in use. 1866 */ 1867 struct highmem_pbe { 1868 struct page *copy_page; /* data is here now */ 1869 struct page *orig_page; /* data was here before the suspend */ 1870 struct highmem_pbe *next; 1871 }; 1872 1873 /* List of highmem PBEs needed for restoring the highmem pages that were 1874 * allocated before the suspend and included in the suspend image, but have 1875 * also been allocated by the "resume" kernel, so their contents cannot be 1876 * written directly to their "original" page frames. 1877 */ 1878 static struct highmem_pbe *highmem_pblist; 1879 1880 /** 1881 * count_highmem_image_pages - compute the number of highmem pages in the 1882 * suspend image. The bits in the memory bitmap @bm that correspond to the 1883 * image pages are assumed to be set. 1884 */ 1885 1886 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) 1887 { 1888 unsigned long pfn; 1889 unsigned int cnt = 0; 1890 1891 memory_bm_position_reset(bm); 1892 pfn = memory_bm_next_pfn(bm); 1893 while (pfn != BM_END_OF_MAP) { 1894 if (PageHighMem(pfn_to_page(pfn))) 1895 cnt++; 1896 1897 pfn = memory_bm_next_pfn(bm); 1898 } 1899 return cnt; 1900 } 1901 1902 /** 1903 * prepare_highmem_image - try to allocate as many highmem pages as 1904 * there are highmem image pages (@nr_highmem_p points to the variable 1905 * containing the number of highmem image pages). The pages that are 1906 * "safe" (ie. will not be overwritten when the suspend image is 1907 * restored) have the corresponding bits set in @bm (it must be 1908 * unitialized). 1909 * 1910 * NOTE: This function should not be called if there are no highmem 1911 * image pages. 1912 */ 1913 1914 static unsigned int safe_highmem_pages; 1915 1916 static struct memory_bitmap *safe_highmem_bm; 1917 1918 static int 1919 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p) 1920 { 1921 unsigned int to_alloc; 1922 1923 if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE)) 1924 return -ENOMEM; 1925 1926 if (get_highmem_buffer(PG_SAFE)) 1927 return -ENOMEM; 1928 1929 to_alloc = count_free_highmem_pages(); 1930 if (to_alloc > *nr_highmem_p) 1931 to_alloc = *nr_highmem_p; 1932 else 1933 *nr_highmem_p = to_alloc; 1934 1935 safe_highmem_pages = 0; 1936 while (to_alloc-- > 0) { 1937 struct page *page; 1938 1939 page = alloc_page(__GFP_HIGHMEM); 1940 if (!swsusp_page_is_free(page)) { 1941 /* The page is "safe", set its bit the bitmap */ 1942 memory_bm_set_bit(bm, page_to_pfn(page)); 1943 safe_highmem_pages++; 1944 } 1945 /* Mark the page as allocated */ 1946 swsusp_set_page_forbidden(page); 1947 swsusp_set_page_free(page); 1948 } 1949 memory_bm_position_reset(bm); 1950 safe_highmem_bm = bm; 1951 return 0; 1952 } 1953 1954 /** 1955 * get_highmem_page_buffer - for given highmem image page find the buffer 1956 * that suspend_write_next() should set for its caller to write to. 1957 * 1958 * If the page is to be saved to its "original" page frame or a copy of 1959 * the page is to be made in the highmem, @buffer is returned. Otherwise, 1960 * the copy of the page is to be made in normal memory, so the address of 1961 * the copy is returned. 1962 * 1963 * If @buffer is returned, the caller of suspend_write_next() will write 1964 * the page's contents to @buffer, so they will have to be copied to the 1965 * right location on the next call to suspend_write_next() and it is done 1966 * with the help of copy_last_highmem_page(). For this purpose, if 1967 * @buffer is returned, @last_highmem page is set to the page to which 1968 * the data will have to be copied from @buffer. 1969 */ 1970 1971 static struct page *last_highmem_page; 1972 1973 static void * 1974 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca) 1975 { 1976 struct highmem_pbe *pbe; 1977 void *kaddr; 1978 1979 if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) { 1980 /* We have allocated the "original" page frame and we can 1981 * use it directly to store the loaded page. 1982 */ 1983 last_highmem_page = page; 1984 return buffer; 1985 } 1986 /* The "original" page frame has not been allocated and we have to 1987 * use a "safe" page frame to store the loaded page. 1988 */ 1989 pbe = chain_alloc(ca, sizeof(struct highmem_pbe)); 1990 if (!pbe) { 1991 swsusp_free(); 1992 return ERR_PTR(-ENOMEM); 1993 } 1994 pbe->orig_page = page; 1995 if (safe_highmem_pages > 0) { 1996 struct page *tmp; 1997 1998 /* Copy of the page will be stored in high memory */ 1999 kaddr = buffer; 2000 tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm)); 2001 safe_highmem_pages--; 2002 last_highmem_page = tmp; 2003 pbe->copy_page = tmp; 2004 } else { 2005 /* Copy of the page will be stored in normal memory */ 2006 kaddr = safe_pages_list; 2007 safe_pages_list = safe_pages_list->next; 2008 pbe->copy_page = virt_to_page(kaddr); 2009 } 2010 pbe->next = highmem_pblist; 2011 highmem_pblist = pbe; 2012 return kaddr; 2013 } 2014 2015 /** 2016 * copy_last_highmem_page - copy the contents of a highmem image from 2017 * @buffer, where the caller of snapshot_write_next() has place them, 2018 * to the right location represented by @last_highmem_page . 2019 */ 2020 2021 static void copy_last_highmem_page(void) 2022 { 2023 if (last_highmem_page) { 2024 void *dst; 2025 2026 dst = kmap_atomic(last_highmem_page); 2027 copy_page(dst, buffer); 2028 kunmap_atomic(dst); 2029 last_highmem_page = NULL; 2030 } 2031 } 2032 2033 static inline int last_highmem_page_copied(void) 2034 { 2035 return !last_highmem_page; 2036 } 2037 2038 static inline void free_highmem_data(void) 2039 { 2040 if (safe_highmem_bm) 2041 memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR); 2042 2043 if (buffer) 2044 free_image_page(buffer, PG_UNSAFE_CLEAR); 2045 } 2046 #else 2047 static inline int get_safe_write_buffer(void) { return 0; } 2048 2049 static unsigned int 2050 count_highmem_image_pages(struct memory_bitmap *bm) { return 0; } 2051 2052 static inline int 2053 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p) 2054 { 2055 return 0; 2056 } 2057 2058 static inline void * 2059 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca) 2060 { 2061 return ERR_PTR(-EINVAL); 2062 } 2063 2064 static inline void copy_last_highmem_page(void) {} 2065 static inline int last_highmem_page_copied(void) { return 1; } 2066 static inline void free_highmem_data(void) {} 2067 #endif /* CONFIG_HIGHMEM */ 2068 2069 /** 2070 * prepare_image - use the memory bitmap @bm to mark the pages that will 2071 * be overwritten in the process of restoring the system memory state 2072 * from the suspend image ("unsafe" pages) and allocate memory for the 2073 * image. 2074 * 2075 * The idea is to allocate a new memory bitmap first and then allocate 2076 * as many pages as needed for the image data, but not to assign these 2077 * pages to specific tasks initially. Instead, we just mark them as 2078 * allocated and create a lists of "safe" pages that will be used 2079 * later. On systems with high memory a list of "safe" highmem pages is 2080 * also created. 2081 */ 2082 2083 #define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe)) 2084 2085 static int 2086 prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm) 2087 { 2088 unsigned int nr_pages, nr_highmem; 2089 struct linked_page *sp_list, *lp; 2090 int error; 2091 2092 /* If there is no highmem, the buffer will not be necessary */ 2093 free_image_page(buffer, PG_UNSAFE_CLEAR); 2094 buffer = NULL; 2095 2096 nr_highmem = count_highmem_image_pages(bm); 2097 error = mark_unsafe_pages(bm); 2098 if (error) 2099 goto Free; 2100 2101 error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE); 2102 if (error) 2103 goto Free; 2104 2105 duplicate_memory_bitmap(new_bm, bm); 2106 memory_bm_free(bm, PG_UNSAFE_KEEP); 2107 if (nr_highmem > 0) { 2108 error = prepare_highmem_image(bm, &nr_highmem); 2109 if (error) 2110 goto Free; 2111 } 2112 /* Reserve some safe pages for potential later use. 2113 * 2114 * NOTE: This way we make sure there will be enough safe pages for the 2115 * chain_alloc() in get_buffer(). It is a bit wasteful, but 2116 * nr_copy_pages cannot be greater than 50% of the memory anyway. 2117 */ 2118 sp_list = NULL; 2119 /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */ 2120 nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages; 2121 nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE); 2122 while (nr_pages > 0) { 2123 lp = get_image_page(GFP_ATOMIC, PG_SAFE); 2124 if (!lp) { 2125 error = -ENOMEM; 2126 goto Free; 2127 } 2128 lp->next = sp_list; 2129 sp_list = lp; 2130 nr_pages--; 2131 } 2132 /* Preallocate memory for the image */ 2133 safe_pages_list = NULL; 2134 nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages; 2135 while (nr_pages > 0) { 2136 lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC); 2137 if (!lp) { 2138 error = -ENOMEM; 2139 goto Free; 2140 } 2141 if (!swsusp_page_is_free(virt_to_page(lp))) { 2142 /* The page is "safe", add it to the list */ 2143 lp->next = safe_pages_list; 2144 safe_pages_list = lp; 2145 } 2146 /* Mark the page as allocated */ 2147 swsusp_set_page_forbidden(virt_to_page(lp)); 2148 swsusp_set_page_free(virt_to_page(lp)); 2149 nr_pages--; 2150 } 2151 /* Free the reserved safe pages so that chain_alloc() can use them */ 2152 while (sp_list) { 2153 lp = sp_list->next; 2154 free_image_page(sp_list, PG_UNSAFE_CLEAR); 2155 sp_list = lp; 2156 } 2157 return 0; 2158 2159 Free: 2160 swsusp_free(); 2161 return error; 2162 } 2163 2164 /** 2165 * get_buffer - compute the address that snapshot_write_next() should 2166 * set for its caller to write to. 2167 */ 2168 2169 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca) 2170 { 2171 struct pbe *pbe; 2172 struct page *page; 2173 unsigned long pfn = memory_bm_next_pfn(bm); 2174 2175 if (pfn == BM_END_OF_MAP) 2176 return ERR_PTR(-EFAULT); 2177 2178 page = pfn_to_page(pfn); 2179 if (PageHighMem(page)) 2180 return get_highmem_page_buffer(page, ca); 2181 2182 if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) 2183 /* We have allocated the "original" page frame and we can 2184 * use it directly to store the loaded page. 2185 */ 2186 return page_address(page); 2187 2188 /* The "original" page frame has not been allocated and we have to 2189 * use a "safe" page frame to store the loaded page. 2190 */ 2191 pbe = chain_alloc(ca, sizeof(struct pbe)); 2192 if (!pbe) { 2193 swsusp_free(); 2194 return ERR_PTR(-ENOMEM); 2195 } 2196 pbe->orig_address = page_address(page); 2197 pbe->address = safe_pages_list; 2198 safe_pages_list = safe_pages_list->next; 2199 pbe->next = restore_pblist; 2200 restore_pblist = pbe; 2201 return pbe->address; 2202 } 2203 2204 /** 2205 * snapshot_write_next - used for writing the system memory snapshot. 2206 * 2207 * On the first call to it @handle should point to a zeroed 2208 * snapshot_handle structure. The structure gets updated and a pointer 2209 * to it should be passed to this function every next time. 2210 * 2211 * On success the function returns a positive number. Then, the caller 2212 * is allowed to write up to the returned number of bytes to the memory 2213 * location computed by the data_of() macro. 2214 * 2215 * The function returns 0 to indicate the "end of file" condition, 2216 * and a negative number is returned on error. In such cases the 2217 * structure pointed to by @handle is not updated and should not be used 2218 * any more. 2219 */ 2220 2221 int snapshot_write_next(struct snapshot_handle *handle) 2222 { 2223 static struct chain_allocator ca; 2224 int error = 0; 2225 2226 /* Check if we have already loaded the entire image */ 2227 if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) 2228 return 0; 2229 2230 handle->sync_read = 1; 2231 2232 if (!handle->cur) { 2233 if (!buffer) 2234 /* This makes the buffer be freed by swsusp_free() */ 2235 buffer = get_image_page(GFP_ATOMIC, PG_ANY); 2236 2237 if (!buffer) 2238 return -ENOMEM; 2239 2240 handle->buffer = buffer; 2241 } else if (handle->cur == 1) { 2242 error = load_header(buffer); 2243 if (error) 2244 return error; 2245 2246 error = memory_bm_create(©_bm, GFP_ATOMIC, PG_ANY); 2247 if (error) 2248 return error; 2249 2250 /* Allocate buffer for page keys. */ 2251 error = page_key_alloc(nr_copy_pages); 2252 if (error) 2253 return error; 2254 2255 } else if (handle->cur <= nr_meta_pages + 1) { 2256 error = unpack_orig_pfns(buffer, ©_bm); 2257 if (error) 2258 return error; 2259 2260 if (handle->cur == nr_meta_pages + 1) { 2261 error = prepare_image(&orig_bm, ©_bm); 2262 if (error) 2263 return error; 2264 2265 chain_init(&ca, GFP_ATOMIC, PG_SAFE); 2266 memory_bm_position_reset(&orig_bm); 2267 restore_pblist = NULL; 2268 handle->buffer = get_buffer(&orig_bm, &ca); 2269 handle->sync_read = 0; 2270 if (IS_ERR(handle->buffer)) 2271 return PTR_ERR(handle->buffer); 2272 } 2273 } else { 2274 copy_last_highmem_page(); 2275 /* Restore page key for data page (s390 only). */ 2276 page_key_write(handle->buffer); 2277 handle->buffer = get_buffer(&orig_bm, &ca); 2278 if (IS_ERR(handle->buffer)) 2279 return PTR_ERR(handle->buffer); 2280 if (handle->buffer != buffer) 2281 handle->sync_read = 0; 2282 } 2283 handle->cur++; 2284 return PAGE_SIZE; 2285 } 2286 2287 /** 2288 * snapshot_write_finalize - must be called after the last call to 2289 * snapshot_write_next() in case the last page in the image happens 2290 * to be a highmem page and its contents should be stored in the 2291 * highmem. Additionally, it releases the memory that will not be 2292 * used any more. 2293 */ 2294 2295 void snapshot_write_finalize(struct snapshot_handle *handle) 2296 { 2297 copy_last_highmem_page(); 2298 /* Restore page key for data page (s390 only). */ 2299 page_key_write(handle->buffer); 2300 page_key_free(); 2301 /* Free only if we have loaded the image entirely */ 2302 if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) { 2303 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR); 2304 free_highmem_data(); 2305 } 2306 } 2307 2308 int snapshot_image_loaded(struct snapshot_handle *handle) 2309 { 2310 return !(!nr_copy_pages || !last_highmem_page_copied() || 2311 handle->cur <= nr_meta_pages + nr_copy_pages); 2312 } 2313 2314 #ifdef CONFIG_HIGHMEM 2315 /* Assumes that @buf is ready and points to a "safe" page */ 2316 static inline void 2317 swap_two_pages_data(struct page *p1, struct page *p2, void *buf) 2318 { 2319 void *kaddr1, *kaddr2; 2320 2321 kaddr1 = kmap_atomic(p1); 2322 kaddr2 = kmap_atomic(p2); 2323 copy_page(buf, kaddr1); 2324 copy_page(kaddr1, kaddr2); 2325 copy_page(kaddr2, buf); 2326 kunmap_atomic(kaddr2); 2327 kunmap_atomic(kaddr1); 2328 } 2329 2330 /** 2331 * restore_highmem - for each highmem page that was allocated before 2332 * the suspend and included in the suspend image, and also has been 2333 * allocated by the "resume" kernel swap its current (ie. "before 2334 * resume") contents with the previous (ie. "before suspend") one. 2335 * 2336 * If the resume eventually fails, we can call this function once 2337 * again and restore the "before resume" highmem state. 2338 */ 2339 2340 int restore_highmem(void) 2341 { 2342 struct highmem_pbe *pbe = highmem_pblist; 2343 void *buf; 2344 2345 if (!pbe) 2346 return 0; 2347 2348 buf = get_image_page(GFP_ATOMIC, PG_SAFE); 2349 if (!buf) 2350 return -ENOMEM; 2351 2352 while (pbe) { 2353 swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf); 2354 pbe = pbe->next; 2355 } 2356 free_image_page(buf, PG_UNSAFE_CLEAR); 2357 return 0; 2358 } 2359 #endif /* CONFIG_HIGHMEM */ 2360