1 /* 2 * linux/kernel/power/snapshot.c 3 * 4 * This file provides system snapshot/restore functionality for swsusp. 5 * 6 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz> 7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 8 * 9 * This file is released under the GPLv2. 10 * 11 */ 12 13 #include <linux/version.h> 14 #include <linux/module.h> 15 #include <linux/mm.h> 16 #include <linux/suspend.h> 17 #include <linux/delay.h> 18 #include <linux/bitops.h> 19 #include <linux/spinlock.h> 20 #include <linux/kernel.h> 21 #include <linux/pm.h> 22 #include <linux/device.h> 23 #include <linux/init.h> 24 #include <linux/bootmem.h> 25 #include <linux/syscalls.h> 26 #include <linux/console.h> 27 #include <linux/highmem.h> 28 #include <linux/list.h> 29 #include <linux/slab.h> 30 31 #include <asm/uaccess.h> 32 #include <asm/mmu_context.h> 33 #include <asm/pgtable.h> 34 #include <asm/tlbflush.h> 35 #include <asm/io.h> 36 37 #include "power.h" 38 39 static int swsusp_page_is_free(struct page *); 40 static void swsusp_set_page_forbidden(struct page *); 41 static void swsusp_unset_page_forbidden(struct page *); 42 43 /* 44 * Number of bytes to reserve for memory allocations made by device drivers 45 * from their ->freeze() and ->freeze_noirq() callbacks so that they don't 46 * cause image creation to fail (tunable via /sys/power/reserved_size). 47 */ 48 unsigned long reserved_size; 49 50 void __init hibernate_reserved_size_init(void) 51 { 52 reserved_size = SPARE_PAGES * PAGE_SIZE; 53 } 54 55 /* 56 * Preferred image size in bytes (tunable via /sys/power/image_size). 57 * When it is set to N, swsusp will do its best to ensure the image 58 * size will not exceed N bytes, but if that is impossible, it will 59 * try to create the smallest image possible. 60 */ 61 unsigned long image_size; 62 63 void __init hibernate_image_size_init(void) 64 { 65 image_size = ((totalram_pages * 2) / 5) * PAGE_SIZE; 66 } 67 68 /* List of PBEs needed for restoring the pages that were allocated before 69 * the suspend and included in the suspend image, but have also been 70 * allocated by the "resume" kernel, so their contents cannot be written 71 * directly to their "original" page frames. 72 */ 73 struct pbe *restore_pblist; 74 75 /* Pointer to an auxiliary buffer (1 page) */ 76 static void *buffer; 77 78 /** 79 * @safe_needed - on resume, for storing the PBE list and the image, 80 * we can only use memory pages that do not conflict with the pages 81 * used before suspend. The unsafe pages have PageNosaveFree set 82 * and we count them using unsafe_pages. 83 * 84 * Each allocated image page is marked as PageNosave and PageNosaveFree 85 * so that swsusp_free() can release it. 86 */ 87 88 #define PG_ANY 0 89 #define PG_SAFE 1 90 #define PG_UNSAFE_CLEAR 1 91 #define PG_UNSAFE_KEEP 0 92 93 static unsigned int allocated_unsafe_pages; 94 95 static void *get_image_page(gfp_t gfp_mask, int safe_needed) 96 { 97 void *res; 98 99 res = (void *)get_zeroed_page(gfp_mask); 100 if (safe_needed) 101 while (res && swsusp_page_is_free(virt_to_page(res))) { 102 /* The page is unsafe, mark it for swsusp_free() */ 103 swsusp_set_page_forbidden(virt_to_page(res)); 104 allocated_unsafe_pages++; 105 res = (void *)get_zeroed_page(gfp_mask); 106 } 107 if (res) { 108 swsusp_set_page_forbidden(virt_to_page(res)); 109 swsusp_set_page_free(virt_to_page(res)); 110 } 111 return res; 112 } 113 114 unsigned long get_safe_page(gfp_t gfp_mask) 115 { 116 return (unsigned long)get_image_page(gfp_mask, PG_SAFE); 117 } 118 119 static struct page *alloc_image_page(gfp_t gfp_mask) 120 { 121 struct page *page; 122 123 page = alloc_page(gfp_mask); 124 if (page) { 125 swsusp_set_page_forbidden(page); 126 swsusp_set_page_free(page); 127 } 128 return page; 129 } 130 131 /** 132 * free_image_page - free page represented by @addr, allocated with 133 * get_image_page (page flags set by it must be cleared) 134 */ 135 136 static inline void free_image_page(void *addr, int clear_nosave_free) 137 { 138 struct page *page; 139 140 BUG_ON(!virt_addr_valid(addr)); 141 142 page = virt_to_page(addr); 143 144 swsusp_unset_page_forbidden(page); 145 if (clear_nosave_free) 146 swsusp_unset_page_free(page); 147 148 __free_page(page); 149 } 150 151 /* struct linked_page is used to build chains of pages */ 152 153 #define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *)) 154 155 struct linked_page { 156 struct linked_page *next; 157 char data[LINKED_PAGE_DATA_SIZE]; 158 } __attribute__((packed)); 159 160 static inline void 161 free_list_of_pages(struct linked_page *list, int clear_page_nosave) 162 { 163 while (list) { 164 struct linked_page *lp = list->next; 165 166 free_image_page(list, clear_page_nosave); 167 list = lp; 168 } 169 } 170 171 /** 172 * struct chain_allocator is used for allocating small objects out of 173 * a linked list of pages called 'the chain'. 174 * 175 * The chain grows each time when there is no room for a new object in 176 * the current page. The allocated objects cannot be freed individually. 177 * It is only possible to free them all at once, by freeing the entire 178 * chain. 179 * 180 * NOTE: The chain allocator may be inefficient if the allocated objects 181 * are not much smaller than PAGE_SIZE. 182 */ 183 184 struct chain_allocator { 185 struct linked_page *chain; /* the chain */ 186 unsigned int used_space; /* total size of objects allocated out 187 * of the current page 188 */ 189 gfp_t gfp_mask; /* mask for allocating pages */ 190 int safe_needed; /* if set, only "safe" pages are allocated */ 191 }; 192 193 static void 194 chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed) 195 { 196 ca->chain = NULL; 197 ca->used_space = LINKED_PAGE_DATA_SIZE; 198 ca->gfp_mask = gfp_mask; 199 ca->safe_needed = safe_needed; 200 } 201 202 static void *chain_alloc(struct chain_allocator *ca, unsigned int size) 203 { 204 void *ret; 205 206 if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) { 207 struct linked_page *lp; 208 209 lp = get_image_page(ca->gfp_mask, ca->safe_needed); 210 if (!lp) 211 return NULL; 212 213 lp->next = ca->chain; 214 ca->chain = lp; 215 ca->used_space = 0; 216 } 217 ret = ca->chain->data + ca->used_space; 218 ca->used_space += size; 219 return ret; 220 } 221 222 /** 223 * Data types related to memory bitmaps. 224 * 225 * Memory bitmap is a structure consiting of many linked lists of 226 * objects. The main list's elements are of type struct zone_bitmap 227 * and each of them corresonds to one zone. For each zone bitmap 228 * object there is a list of objects of type struct bm_block that 229 * represent each blocks of bitmap in which information is stored. 230 * 231 * struct memory_bitmap contains a pointer to the main list of zone 232 * bitmap objects, a struct bm_position used for browsing the bitmap, 233 * and a pointer to the list of pages used for allocating all of the 234 * zone bitmap objects and bitmap block objects. 235 * 236 * NOTE: It has to be possible to lay out the bitmap in memory 237 * using only allocations of order 0. Additionally, the bitmap is 238 * designed to work with arbitrary number of zones (this is over the 239 * top for now, but let's avoid making unnecessary assumptions ;-). 240 * 241 * struct zone_bitmap contains a pointer to a list of bitmap block 242 * objects and a pointer to the bitmap block object that has been 243 * most recently used for setting bits. Additionally, it contains the 244 * pfns that correspond to the start and end of the represented zone. 245 * 246 * struct bm_block contains a pointer to the memory page in which 247 * information is stored (in the form of a block of bitmap) 248 * It also contains the pfns that correspond to the start and end of 249 * the represented memory area. 250 */ 251 252 #define BM_END_OF_MAP (~0UL) 253 254 #define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE) 255 256 struct bm_block { 257 struct list_head hook; /* hook into a list of bitmap blocks */ 258 unsigned long start_pfn; /* pfn represented by the first bit */ 259 unsigned long end_pfn; /* pfn represented by the last bit plus 1 */ 260 unsigned long *data; /* bitmap representing pages */ 261 }; 262 263 static inline unsigned long bm_block_bits(struct bm_block *bb) 264 { 265 return bb->end_pfn - bb->start_pfn; 266 } 267 268 /* strcut bm_position is used for browsing memory bitmaps */ 269 270 struct bm_position { 271 struct bm_block *block; 272 int bit; 273 }; 274 275 struct memory_bitmap { 276 struct list_head blocks; /* list of bitmap blocks */ 277 struct linked_page *p_list; /* list of pages used to store zone 278 * bitmap objects and bitmap block 279 * objects 280 */ 281 struct bm_position cur; /* most recently used bit position */ 282 }; 283 284 /* Functions that operate on memory bitmaps */ 285 286 static void memory_bm_position_reset(struct memory_bitmap *bm) 287 { 288 bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook); 289 bm->cur.bit = 0; 290 } 291 292 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free); 293 294 /** 295 * create_bm_block_list - create a list of block bitmap objects 296 * @pages - number of pages to track 297 * @list - list to put the allocated blocks into 298 * @ca - chain allocator to be used for allocating memory 299 */ 300 static int create_bm_block_list(unsigned long pages, 301 struct list_head *list, 302 struct chain_allocator *ca) 303 { 304 unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK); 305 306 while (nr_blocks-- > 0) { 307 struct bm_block *bb; 308 309 bb = chain_alloc(ca, sizeof(struct bm_block)); 310 if (!bb) 311 return -ENOMEM; 312 list_add(&bb->hook, list); 313 } 314 315 return 0; 316 } 317 318 struct mem_extent { 319 struct list_head hook; 320 unsigned long start; 321 unsigned long end; 322 }; 323 324 /** 325 * free_mem_extents - free a list of memory extents 326 * @list - list of extents to empty 327 */ 328 static void free_mem_extents(struct list_head *list) 329 { 330 struct mem_extent *ext, *aux; 331 332 list_for_each_entry_safe(ext, aux, list, hook) { 333 list_del(&ext->hook); 334 kfree(ext); 335 } 336 } 337 338 /** 339 * create_mem_extents - create a list of memory extents representing 340 * contiguous ranges of PFNs 341 * @list - list to put the extents into 342 * @gfp_mask - mask to use for memory allocations 343 */ 344 static int create_mem_extents(struct list_head *list, gfp_t gfp_mask) 345 { 346 struct zone *zone; 347 348 INIT_LIST_HEAD(list); 349 350 for_each_populated_zone(zone) { 351 unsigned long zone_start, zone_end; 352 struct mem_extent *ext, *cur, *aux; 353 354 zone_start = zone->zone_start_pfn; 355 zone_end = zone->zone_start_pfn + zone->spanned_pages; 356 357 list_for_each_entry(ext, list, hook) 358 if (zone_start <= ext->end) 359 break; 360 361 if (&ext->hook == list || zone_end < ext->start) { 362 /* New extent is necessary */ 363 struct mem_extent *new_ext; 364 365 new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask); 366 if (!new_ext) { 367 free_mem_extents(list); 368 return -ENOMEM; 369 } 370 new_ext->start = zone_start; 371 new_ext->end = zone_end; 372 list_add_tail(&new_ext->hook, &ext->hook); 373 continue; 374 } 375 376 /* Merge this zone's range of PFNs with the existing one */ 377 if (zone_start < ext->start) 378 ext->start = zone_start; 379 if (zone_end > ext->end) 380 ext->end = zone_end; 381 382 /* More merging may be possible */ 383 cur = ext; 384 list_for_each_entry_safe_continue(cur, aux, list, hook) { 385 if (zone_end < cur->start) 386 break; 387 if (zone_end < cur->end) 388 ext->end = cur->end; 389 list_del(&cur->hook); 390 kfree(cur); 391 } 392 } 393 394 return 0; 395 } 396 397 /** 398 * memory_bm_create - allocate memory for a memory bitmap 399 */ 400 static int 401 memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed) 402 { 403 struct chain_allocator ca; 404 struct list_head mem_extents; 405 struct mem_extent *ext; 406 int error; 407 408 chain_init(&ca, gfp_mask, safe_needed); 409 INIT_LIST_HEAD(&bm->blocks); 410 411 error = create_mem_extents(&mem_extents, gfp_mask); 412 if (error) 413 return error; 414 415 list_for_each_entry(ext, &mem_extents, hook) { 416 struct bm_block *bb; 417 unsigned long pfn = ext->start; 418 unsigned long pages = ext->end - ext->start; 419 420 bb = list_entry(bm->blocks.prev, struct bm_block, hook); 421 422 error = create_bm_block_list(pages, bm->blocks.prev, &ca); 423 if (error) 424 goto Error; 425 426 list_for_each_entry_continue(bb, &bm->blocks, hook) { 427 bb->data = get_image_page(gfp_mask, safe_needed); 428 if (!bb->data) { 429 error = -ENOMEM; 430 goto Error; 431 } 432 433 bb->start_pfn = pfn; 434 if (pages >= BM_BITS_PER_BLOCK) { 435 pfn += BM_BITS_PER_BLOCK; 436 pages -= BM_BITS_PER_BLOCK; 437 } else { 438 /* This is executed only once in the loop */ 439 pfn += pages; 440 } 441 bb->end_pfn = pfn; 442 } 443 } 444 445 bm->p_list = ca.chain; 446 memory_bm_position_reset(bm); 447 Exit: 448 free_mem_extents(&mem_extents); 449 return error; 450 451 Error: 452 bm->p_list = ca.chain; 453 memory_bm_free(bm, PG_UNSAFE_CLEAR); 454 goto Exit; 455 } 456 457 /** 458 * memory_bm_free - free memory occupied by the memory bitmap @bm 459 */ 460 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) 461 { 462 struct bm_block *bb; 463 464 list_for_each_entry(bb, &bm->blocks, hook) 465 if (bb->data) 466 free_image_page(bb->data, clear_nosave_free); 467 468 free_list_of_pages(bm->p_list, clear_nosave_free); 469 470 INIT_LIST_HEAD(&bm->blocks); 471 } 472 473 /** 474 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds 475 * to given pfn. The cur_zone_bm member of @bm and the cur_block member 476 * of @bm->cur_zone_bm are updated. 477 */ 478 static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn, 479 void **addr, unsigned int *bit_nr) 480 { 481 struct bm_block *bb; 482 483 /* 484 * Check if the pfn corresponds to the current bitmap block and find 485 * the block where it fits if this is not the case. 486 */ 487 bb = bm->cur.block; 488 if (pfn < bb->start_pfn) 489 list_for_each_entry_continue_reverse(bb, &bm->blocks, hook) 490 if (pfn >= bb->start_pfn) 491 break; 492 493 if (pfn >= bb->end_pfn) 494 list_for_each_entry_continue(bb, &bm->blocks, hook) 495 if (pfn >= bb->start_pfn && pfn < bb->end_pfn) 496 break; 497 498 if (&bb->hook == &bm->blocks) 499 return -EFAULT; 500 501 /* The block has been found */ 502 bm->cur.block = bb; 503 pfn -= bb->start_pfn; 504 bm->cur.bit = pfn + 1; 505 *bit_nr = pfn; 506 *addr = bb->data; 507 return 0; 508 } 509 510 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn) 511 { 512 void *addr; 513 unsigned int bit; 514 int error; 515 516 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 517 BUG_ON(error); 518 set_bit(bit, addr); 519 } 520 521 static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn) 522 { 523 void *addr; 524 unsigned int bit; 525 int error; 526 527 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 528 if (!error) 529 set_bit(bit, addr); 530 return error; 531 } 532 533 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn) 534 { 535 void *addr; 536 unsigned int bit; 537 int error; 538 539 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 540 BUG_ON(error); 541 clear_bit(bit, addr); 542 } 543 544 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn) 545 { 546 void *addr; 547 unsigned int bit; 548 int error; 549 550 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 551 BUG_ON(error); 552 return test_bit(bit, addr); 553 } 554 555 static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn) 556 { 557 void *addr; 558 unsigned int bit; 559 560 return !memory_bm_find_bit(bm, pfn, &addr, &bit); 561 } 562 563 /** 564 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit 565 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is 566 * returned. 567 * 568 * It is required to run memory_bm_position_reset() before the first call to 569 * this function. 570 */ 571 572 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm) 573 { 574 struct bm_block *bb; 575 int bit; 576 577 bb = bm->cur.block; 578 do { 579 bit = bm->cur.bit; 580 bit = find_next_bit(bb->data, bm_block_bits(bb), bit); 581 if (bit < bm_block_bits(bb)) 582 goto Return_pfn; 583 584 bb = list_entry(bb->hook.next, struct bm_block, hook); 585 bm->cur.block = bb; 586 bm->cur.bit = 0; 587 } while (&bb->hook != &bm->blocks); 588 589 memory_bm_position_reset(bm); 590 return BM_END_OF_MAP; 591 592 Return_pfn: 593 bm->cur.bit = bit + 1; 594 return bb->start_pfn + bit; 595 } 596 597 /** 598 * This structure represents a range of page frames the contents of which 599 * should not be saved during the suspend. 600 */ 601 602 struct nosave_region { 603 struct list_head list; 604 unsigned long start_pfn; 605 unsigned long end_pfn; 606 }; 607 608 static LIST_HEAD(nosave_regions); 609 610 /** 611 * register_nosave_region - register a range of page frames the contents 612 * of which should not be saved during the suspend (to be used in the early 613 * initialization code) 614 */ 615 616 void __init 617 __register_nosave_region(unsigned long start_pfn, unsigned long end_pfn, 618 int use_kmalloc) 619 { 620 struct nosave_region *region; 621 622 if (start_pfn >= end_pfn) 623 return; 624 625 if (!list_empty(&nosave_regions)) { 626 /* Try to extend the previous region (they should be sorted) */ 627 region = list_entry(nosave_regions.prev, 628 struct nosave_region, list); 629 if (region->end_pfn == start_pfn) { 630 region->end_pfn = end_pfn; 631 goto Report; 632 } 633 } 634 if (use_kmalloc) { 635 /* during init, this shouldn't fail */ 636 region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL); 637 BUG_ON(!region); 638 } else 639 /* This allocation cannot fail */ 640 region = alloc_bootmem(sizeof(struct nosave_region)); 641 region->start_pfn = start_pfn; 642 region->end_pfn = end_pfn; 643 list_add_tail(®ion->list, &nosave_regions); 644 Report: 645 printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n", 646 start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT); 647 } 648 649 /* 650 * Set bits in this map correspond to the page frames the contents of which 651 * should not be saved during the suspend. 652 */ 653 static struct memory_bitmap *forbidden_pages_map; 654 655 /* Set bits in this map correspond to free page frames. */ 656 static struct memory_bitmap *free_pages_map; 657 658 /* 659 * Each page frame allocated for creating the image is marked by setting the 660 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously 661 */ 662 663 void swsusp_set_page_free(struct page *page) 664 { 665 if (free_pages_map) 666 memory_bm_set_bit(free_pages_map, page_to_pfn(page)); 667 } 668 669 static int swsusp_page_is_free(struct page *page) 670 { 671 return free_pages_map ? 672 memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0; 673 } 674 675 void swsusp_unset_page_free(struct page *page) 676 { 677 if (free_pages_map) 678 memory_bm_clear_bit(free_pages_map, page_to_pfn(page)); 679 } 680 681 static void swsusp_set_page_forbidden(struct page *page) 682 { 683 if (forbidden_pages_map) 684 memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page)); 685 } 686 687 int swsusp_page_is_forbidden(struct page *page) 688 { 689 return forbidden_pages_map ? 690 memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0; 691 } 692 693 static void swsusp_unset_page_forbidden(struct page *page) 694 { 695 if (forbidden_pages_map) 696 memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page)); 697 } 698 699 /** 700 * mark_nosave_pages - set bits corresponding to the page frames the 701 * contents of which should not be saved in a given bitmap. 702 */ 703 704 static void mark_nosave_pages(struct memory_bitmap *bm) 705 { 706 struct nosave_region *region; 707 708 if (list_empty(&nosave_regions)) 709 return; 710 711 list_for_each_entry(region, &nosave_regions, list) { 712 unsigned long pfn; 713 714 pr_debug("PM: Marking nosave pages: %016lx - %016lx\n", 715 region->start_pfn << PAGE_SHIFT, 716 region->end_pfn << PAGE_SHIFT); 717 718 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++) 719 if (pfn_valid(pfn)) { 720 /* 721 * It is safe to ignore the result of 722 * mem_bm_set_bit_check() here, since we won't 723 * touch the PFNs for which the error is 724 * returned anyway. 725 */ 726 mem_bm_set_bit_check(bm, pfn); 727 } 728 } 729 } 730 731 /** 732 * create_basic_memory_bitmaps - create bitmaps needed for marking page 733 * frames that should not be saved and free page frames. The pointers 734 * forbidden_pages_map and free_pages_map are only modified if everything 735 * goes well, because we don't want the bits to be used before both bitmaps 736 * are set up. 737 */ 738 739 int create_basic_memory_bitmaps(void) 740 { 741 struct memory_bitmap *bm1, *bm2; 742 int error = 0; 743 744 BUG_ON(forbidden_pages_map || free_pages_map); 745 746 bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL); 747 if (!bm1) 748 return -ENOMEM; 749 750 error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY); 751 if (error) 752 goto Free_first_object; 753 754 bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL); 755 if (!bm2) 756 goto Free_first_bitmap; 757 758 error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY); 759 if (error) 760 goto Free_second_object; 761 762 forbidden_pages_map = bm1; 763 free_pages_map = bm2; 764 mark_nosave_pages(forbidden_pages_map); 765 766 pr_debug("PM: Basic memory bitmaps created\n"); 767 768 return 0; 769 770 Free_second_object: 771 kfree(bm2); 772 Free_first_bitmap: 773 memory_bm_free(bm1, PG_UNSAFE_CLEAR); 774 Free_first_object: 775 kfree(bm1); 776 return -ENOMEM; 777 } 778 779 /** 780 * free_basic_memory_bitmaps - free memory bitmaps allocated by 781 * create_basic_memory_bitmaps(). The auxiliary pointers are necessary 782 * so that the bitmaps themselves are not referred to while they are being 783 * freed. 784 */ 785 786 void free_basic_memory_bitmaps(void) 787 { 788 struct memory_bitmap *bm1, *bm2; 789 790 BUG_ON(!(forbidden_pages_map && free_pages_map)); 791 792 bm1 = forbidden_pages_map; 793 bm2 = free_pages_map; 794 forbidden_pages_map = NULL; 795 free_pages_map = NULL; 796 memory_bm_free(bm1, PG_UNSAFE_CLEAR); 797 kfree(bm1); 798 memory_bm_free(bm2, PG_UNSAFE_CLEAR); 799 kfree(bm2); 800 801 pr_debug("PM: Basic memory bitmaps freed\n"); 802 } 803 804 /** 805 * snapshot_additional_pages - estimate the number of additional pages 806 * be needed for setting up the suspend image data structures for given 807 * zone (usually the returned value is greater than the exact number) 808 */ 809 810 unsigned int snapshot_additional_pages(struct zone *zone) 811 { 812 unsigned int res; 813 814 res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK); 815 res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE); 816 return 2 * res; 817 } 818 819 #ifdef CONFIG_HIGHMEM 820 /** 821 * count_free_highmem_pages - compute the total number of free highmem 822 * pages, system-wide. 823 */ 824 825 static unsigned int count_free_highmem_pages(void) 826 { 827 struct zone *zone; 828 unsigned int cnt = 0; 829 830 for_each_populated_zone(zone) 831 if (is_highmem(zone)) 832 cnt += zone_page_state(zone, NR_FREE_PAGES); 833 834 return cnt; 835 } 836 837 /** 838 * saveable_highmem_page - Determine whether a highmem page should be 839 * included in the suspend image. 840 * 841 * We should save the page if it isn't Nosave or NosaveFree, or Reserved, 842 * and it isn't a part of a free chunk of pages. 843 */ 844 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn) 845 { 846 struct page *page; 847 848 if (!pfn_valid(pfn)) 849 return NULL; 850 851 page = pfn_to_page(pfn); 852 if (page_zone(page) != zone) 853 return NULL; 854 855 BUG_ON(!PageHighMem(page)); 856 857 if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page) || 858 PageReserved(page)) 859 return NULL; 860 861 return page; 862 } 863 864 /** 865 * count_highmem_pages - compute the total number of saveable highmem 866 * pages. 867 */ 868 869 static unsigned int count_highmem_pages(void) 870 { 871 struct zone *zone; 872 unsigned int n = 0; 873 874 for_each_populated_zone(zone) { 875 unsigned long pfn, max_zone_pfn; 876 877 if (!is_highmem(zone)) 878 continue; 879 880 mark_free_pages(zone); 881 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 882 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 883 if (saveable_highmem_page(zone, pfn)) 884 n++; 885 } 886 return n; 887 } 888 #else 889 static inline void *saveable_highmem_page(struct zone *z, unsigned long p) 890 { 891 return NULL; 892 } 893 #endif /* CONFIG_HIGHMEM */ 894 895 /** 896 * saveable_page - Determine whether a non-highmem page should be included 897 * in the suspend image. 898 * 899 * We should save the page if it isn't Nosave, and is not in the range 900 * of pages statically defined as 'unsaveable', and it isn't a part of 901 * a free chunk of pages. 902 */ 903 static struct page *saveable_page(struct zone *zone, unsigned long pfn) 904 { 905 struct page *page; 906 907 if (!pfn_valid(pfn)) 908 return NULL; 909 910 page = pfn_to_page(pfn); 911 if (page_zone(page) != zone) 912 return NULL; 913 914 BUG_ON(PageHighMem(page)); 915 916 if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page)) 917 return NULL; 918 919 if (PageReserved(page) 920 && (!kernel_page_present(page) || pfn_is_nosave(pfn))) 921 return NULL; 922 923 return page; 924 } 925 926 /** 927 * count_data_pages - compute the total number of saveable non-highmem 928 * pages. 929 */ 930 931 static unsigned int count_data_pages(void) 932 { 933 struct zone *zone; 934 unsigned long pfn, max_zone_pfn; 935 unsigned int n = 0; 936 937 for_each_populated_zone(zone) { 938 if (is_highmem(zone)) 939 continue; 940 941 mark_free_pages(zone); 942 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 943 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 944 if (saveable_page(zone, pfn)) 945 n++; 946 } 947 return n; 948 } 949 950 /* This is needed, because copy_page and memcpy are not usable for copying 951 * task structs. 952 */ 953 static inline void do_copy_page(long *dst, long *src) 954 { 955 int n; 956 957 for (n = PAGE_SIZE / sizeof(long); n; n--) 958 *dst++ = *src++; 959 } 960 961 962 /** 963 * safe_copy_page - check if the page we are going to copy is marked as 964 * present in the kernel page tables (this always is the case if 965 * CONFIG_DEBUG_PAGEALLOC is not set and in that case 966 * kernel_page_present() always returns 'true'). 967 */ 968 static void safe_copy_page(void *dst, struct page *s_page) 969 { 970 if (kernel_page_present(s_page)) { 971 do_copy_page(dst, page_address(s_page)); 972 } else { 973 kernel_map_pages(s_page, 1, 1); 974 do_copy_page(dst, page_address(s_page)); 975 kernel_map_pages(s_page, 1, 0); 976 } 977 } 978 979 980 #ifdef CONFIG_HIGHMEM 981 static inline struct page * 982 page_is_saveable(struct zone *zone, unsigned long pfn) 983 { 984 return is_highmem(zone) ? 985 saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn); 986 } 987 988 static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) 989 { 990 struct page *s_page, *d_page; 991 void *src, *dst; 992 993 s_page = pfn_to_page(src_pfn); 994 d_page = pfn_to_page(dst_pfn); 995 if (PageHighMem(s_page)) { 996 src = kmap_atomic(s_page, KM_USER0); 997 dst = kmap_atomic(d_page, KM_USER1); 998 do_copy_page(dst, src); 999 kunmap_atomic(dst, KM_USER1); 1000 kunmap_atomic(src, KM_USER0); 1001 } else { 1002 if (PageHighMem(d_page)) { 1003 /* Page pointed to by src may contain some kernel 1004 * data modified by kmap_atomic() 1005 */ 1006 safe_copy_page(buffer, s_page); 1007 dst = kmap_atomic(d_page, KM_USER0); 1008 copy_page(dst, buffer); 1009 kunmap_atomic(dst, KM_USER0); 1010 } else { 1011 safe_copy_page(page_address(d_page), s_page); 1012 } 1013 } 1014 } 1015 #else 1016 #define page_is_saveable(zone, pfn) saveable_page(zone, pfn) 1017 1018 static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) 1019 { 1020 safe_copy_page(page_address(pfn_to_page(dst_pfn)), 1021 pfn_to_page(src_pfn)); 1022 } 1023 #endif /* CONFIG_HIGHMEM */ 1024 1025 static void 1026 copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm) 1027 { 1028 struct zone *zone; 1029 unsigned long pfn; 1030 1031 for_each_populated_zone(zone) { 1032 unsigned long max_zone_pfn; 1033 1034 mark_free_pages(zone); 1035 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 1036 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1037 if (page_is_saveable(zone, pfn)) 1038 memory_bm_set_bit(orig_bm, pfn); 1039 } 1040 memory_bm_position_reset(orig_bm); 1041 memory_bm_position_reset(copy_bm); 1042 for(;;) { 1043 pfn = memory_bm_next_pfn(orig_bm); 1044 if (unlikely(pfn == BM_END_OF_MAP)) 1045 break; 1046 copy_data_page(memory_bm_next_pfn(copy_bm), pfn); 1047 } 1048 } 1049 1050 /* Total number of image pages */ 1051 static unsigned int nr_copy_pages; 1052 /* Number of pages needed for saving the original pfns of the image pages */ 1053 static unsigned int nr_meta_pages; 1054 /* 1055 * Numbers of normal and highmem page frames allocated for hibernation image 1056 * before suspending devices. 1057 */ 1058 unsigned int alloc_normal, alloc_highmem; 1059 /* 1060 * Memory bitmap used for marking saveable pages (during hibernation) or 1061 * hibernation image pages (during restore) 1062 */ 1063 static struct memory_bitmap orig_bm; 1064 /* 1065 * Memory bitmap used during hibernation for marking allocated page frames that 1066 * will contain copies of saveable pages. During restore it is initially used 1067 * for marking hibernation image pages, but then the set bits from it are 1068 * duplicated in @orig_bm and it is released. On highmem systems it is next 1069 * used for marking "safe" highmem pages, but it has to be reinitialized for 1070 * this purpose. 1071 */ 1072 static struct memory_bitmap copy_bm; 1073 1074 /** 1075 * swsusp_free - free pages allocated for the suspend. 1076 * 1077 * Suspend pages are alocated before the atomic copy is made, so we 1078 * need to release them after the resume. 1079 */ 1080 1081 void swsusp_free(void) 1082 { 1083 struct zone *zone; 1084 unsigned long pfn, max_zone_pfn; 1085 1086 for_each_populated_zone(zone) { 1087 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 1088 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1089 if (pfn_valid(pfn)) { 1090 struct page *page = pfn_to_page(pfn); 1091 1092 if (swsusp_page_is_forbidden(page) && 1093 swsusp_page_is_free(page)) { 1094 swsusp_unset_page_forbidden(page); 1095 swsusp_unset_page_free(page); 1096 __free_page(page); 1097 } 1098 } 1099 } 1100 nr_copy_pages = 0; 1101 nr_meta_pages = 0; 1102 restore_pblist = NULL; 1103 buffer = NULL; 1104 alloc_normal = 0; 1105 alloc_highmem = 0; 1106 } 1107 1108 /* Helper functions used for the shrinking of memory. */ 1109 1110 #define GFP_IMAGE (GFP_KERNEL | __GFP_NOWARN) 1111 1112 /** 1113 * preallocate_image_pages - Allocate a number of pages for hibernation image 1114 * @nr_pages: Number of page frames to allocate. 1115 * @mask: GFP flags to use for the allocation. 1116 * 1117 * Return value: Number of page frames actually allocated 1118 */ 1119 static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask) 1120 { 1121 unsigned long nr_alloc = 0; 1122 1123 while (nr_pages > 0) { 1124 struct page *page; 1125 1126 page = alloc_image_page(mask); 1127 if (!page) 1128 break; 1129 memory_bm_set_bit(©_bm, page_to_pfn(page)); 1130 if (PageHighMem(page)) 1131 alloc_highmem++; 1132 else 1133 alloc_normal++; 1134 nr_pages--; 1135 nr_alloc++; 1136 } 1137 1138 return nr_alloc; 1139 } 1140 1141 static unsigned long preallocate_image_memory(unsigned long nr_pages, 1142 unsigned long avail_normal) 1143 { 1144 unsigned long alloc; 1145 1146 if (avail_normal <= alloc_normal) 1147 return 0; 1148 1149 alloc = avail_normal - alloc_normal; 1150 if (nr_pages < alloc) 1151 alloc = nr_pages; 1152 1153 return preallocate_image_pages(alloc, GFP_IMAGE); 1154 } 1155 1156 #ifdef CONFIG_HIGHMEM 1157 static unsigned long preallocate_image_highmem(unsigned long nr_pages) 1158 { 1159 return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM); 1160 } 1161 1162 /** 1163 * __fraction - Compute (an approximation of) x * (multiplier / base) 1164 */ 1165 static unsigned long __fraction(u64 x, u64 multiplier, u64 base) 1166 { 1167 x *= multiplier; 1168 do_div(x, base); 1169 return (unsigned long)x; 1170 } 1171 1172 static unsigned long preallocate_highmem_fraction(unsigned long nr_pages, 1173 unsigned long highmem, 1174 unsigned long total) 1175 { 1176 unsigned long alloc = __fraction(nr_pages, highmem, total); 1177 1178 return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM); 1179 } 1180 #else /* CONFIG_HIGHMEM */ 1181 static inline unsigned long preallocate_image_highmem(unsigned long nr_pages) 1182 { 1183 return 0; 1184 } 1185 1186 static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages, 1187 unsigned long highmem, 1188 unsigned long total) 1189 { 1190 return 0; 1191 } 1192 #endif /* CONFIG_HIGHMEM */ 1193 1194 /** 1195 * free_unnecessary_pages - Release preallocated pages not needed for the image 1196 */ 1197 static void free_unnecessary_pages(void) 1198 { 1199 unsigned long save, to_free_normal, to_free_highmem; 1200 1201 save = count_data_pages(); 1202 if (alloc_normal >= save) { 1203 to_free_normal = alloc_normal - save; 1204 save = 0; 1205 } else { 1206 to_free_normal = 0; 1207 save -= alloc_normal; 1208 } 1209 save += count_highmem_pages(); 1210 if (alloc_highmem >= save) { 1211 to_free_highmem = alloc_highmem - save; 1212 } else { 1213 to_free_highmem = 0; 1214 save -= alloc_highmem; 1215 if (to_free_normal > save) 1216 to_free_normal -= save; 1217 else 1218 to_free_normal = 0; 1219 } 1220 1221 memory_bm_position_reset(©_bm); 1222 1223 while (to_free_normal > 0 || to_free_highmem > 0) { 1224 unsigned long pfn = memory_bm_next_pfn(©_bm); 1225 struct page *page = pfn_to_page(pfn); 1226 1227 if (PageHighMem(page)) { 1228 if (!to_free_highmem) 1229 continue; 1230 to_free_highmem--; 1231 alloc_highmem--; 1232 } else { 1233 if (!to_free_normal) 1234 continue; 1235 to_free_normal--; 1236 alloc_normal--; 1237 } 1238 memory_bm_clear_bit(©_bm, pfn); 1239 swsusp_unset_page_forbidden(page); 1240 swsusp_unset_page_free(page); 1241 __free_page(page); 1242 } 1243 } 1244 1245 /** 1246 * minimum_image_size - Estimate the minimum acceptable size of an image 1247 * @saveable: Number of saveable pages in the system. 1248 * 1249 * We want to avoid attempting to free too much memory too hard, so estimate the 1250 * minimum acceptable size of a hibernation image to use as the lower limit for 1251 * preallocating memory. 1252 * 1253 * We assume that the minimum image size should be proportional to 1254 * 1255 * [number of saveable pages] - [number of pages that can be freed in theory] 1256 * 1257 * where the second term is the sum of (1) reclaimable slab pages, (2) active 1258 * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages, 1259 * minus mapped file pages. 1260 */ 1261 static unsigned long minimum_image_size(unsigned long saveable) 1262 { 1263 unsigned long size; 1264 1265 size = global_page_state(NR_SLAB_RECLAIMABLE) 1266 + global_page_state(NR_ACTIVE_ANON) 1267 + global_page_state(NR_INACTIVE_ANON) 1268 + global_page_state(NR_ACTIVE_FILE) 1269 + global_page_state(NR_INACTIVE_FILE) 1270 - global_page_state(NR_FILE_MAPPED); 1271 1272 return saveable <= size ? 0 : saveable - size; 1273 } 1274 1275 /** 1276 * hibernate_preallocate_memory - Preallocate memory for hibernation image 1277 * 1278 * To create a hibernation image it is necessary to make a copy of every page 1279 * frame in use. We also need a number of page frames to be free during 1280 * hibernation for allocations made while saving the image and for device 1281 * drivers, in case they need to allocate memory from their hibernation 1282 * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough 1283 * estimate) and reserverd_size divided by PAGE_SIZE (which is tunable through 1284 * /sys/power/reserved_size, respectively). To make this happen, we compute the 1285 * total number of available page frames and allocate at least 1286 * 1287 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2 1288 * + 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE) 1289 * 1290 * of them, which corresponds to the maximum size of a hibernation image. 1291 * 1292 * If image_size is set below the number following from the above formula, 1293 * the preallocation of memory is continued until the total number of saveable 1294 * pages in the system is below the requested image size or the minimum 1295 * acceptable image size returned by minimum_image_size(), whichever is greater. 1296 */ 1297 int hibernate_preallocate_memory(void) 1298 { 1299 struct zone *zone; 1300 unsigned long saveable, size, max_size, count, highmem, pages = 0; 1301 unsigned long alloc, save_highmem, pages_highmem, avail_normal; 1302 struct timeval start, stop; 1303 int error; 1304 1305 printk(KERN_INFO "PM: Preallocating image memory... "); 1306 do_gettimeofday(&start); 1307 1308 error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY); 1309 if (error) 1310 goto err_out; 1311 1312 error = memory_bm_create(©_bm, GFP_IMAGE, PG_ANY); 1313 if (error) 1314 goto err_out; 1315 1316 alloc_normal = 0; 1317 alloc_highmem = 0; 1318 1319 /* Count the number of saveable data pages. */ 1320 save_highmem = count_highmem_pages(); 1321 saveable = count_data_pages(); 1322 1323 /* 1324 * Compute the total number of page frames we can use (count) and the 1325 * number of pages needed for image metadata (size). 1326 */ 1327 count = saveable; 1328 saveable += save_highmem; 1329 highmem = save_highmem; 1330 size = 0; 1331 for_each_populated_zone(zone) { 1332 size += snapshot_additional_pages(zone); 1333 if (is_highmem(zone)) 1334 highmem += zone_page_state(zone, NR_FREE_PAGES); 1335 else 1336 count += zone_page_state(zone, NR_FREE_PAGES); 1337 } 1338 avail_normal = count; 1339 count += highmem; 1340 count -= totalreserve_pages; 1341 1342 /* Compute the maximum number of saveable pages to leave in memory. */ 1343 max_size = (count - (size + PAGES_FOR_IO)) / 2 1344 - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE); 1345 /* Compute the desired number of image pages specified by image_size. */ 1346 size = DIV_ROUND_UP(image_size, PAGE_SIZE); 1347 if (size > max_size) 1348 size = max_size; 1349 /* 1350 * If the desired number of image pages is at least as large as the 1351 * current number of saveable pages in memory, allocate page frames for 1352 * the image and we're done. 1353 */ 1354 if (size >= saveable) { 1355 pages = preallocate_image_highmem(save_highmem); 1356 pages += preallocate_image_memory(saveable - pages, avail_normal); 1357 goto out; 1358 } 1359 1360 /* Estimate the minimum size of the image. */ 1361 pages = minimum_image_size(saveable); 1362 /* 1363 * To avoid excessive pressure on the normal zone, leave room in it to 1364 * accommodate an image of the minimum size (unless it's already too 1365 * small, in which case don't preallocate pages from it at all). 1366 */ 1367 if (avail_normal > pages) 1368 avail_normal -= pages; 1369 else 1370 avail_normal = 0; 1371 if (size < pages) 1372 size = min_t(unsigned long, pages, max_size); 1373 1374 /* 1375 * Let the memory management subsystem know that we're going to need a 1376 * large number of page frames to allocate and make it free some memory. 1377 * NOTE: If this is not done, performance will be hurt badly in some 1378 * test cases. 1379 */ 1380 shrink_all_memory(saveable - size); 1381 1382 /* 1383 * The number of saveable pages in memory was too high, so apply some 1384 * pressure to decrease it. First, make room for the largest possible 1385 * image and fail if that doesn't work. Next, try to decrease the size 1386 * of the image as much as indicated by 'size' using allocations from 1387 * highmem and non-highmem zones separately. 1388 */ 1389 pages_highmem = preallocate_image_highmem(highmem / 2); 1390 alloc = (count - max_size) - pages_highmem; 1391 pages = preallocate_image_memory(alloc, avail_normal); 1392 if (pages < alloc) { 1393 /* We have exhausted non-highmem pages, try highmem. */ 1394 alloc -= pages; 1395 pages += pages_highmem; 1396 pages_highmem = preallocate_image_highmem(alloc); 1397 if (pages_highmem < alloc) 1398 goto err_out; 1399 pages += pages_highmem; 1400 /* 1401 * size is the desired number of saveable pages to leave in 1402 * memory, so try to preallocate (all memory - size) pages. 1403 */ 1404 alloc = (count - pages) - size; 1405 pages += preallocate_image_highmem(alloc); 1406 } else { 1407 /* 1408 * There are approximately max_size saveable pages at this point 1409 * and we want to reduce this number down to size. 1410 */ 1411 alloc = max_size - size; 1412 size = preallocate_highmem_fraction(alloc, highmem, count); 1413 pages_highmem += size; 1414 alloc -= size; 1415 size = preallocate_image_memory(alloc, avail_normal); 1416 pages_highmem += preallocate_image_highmem(alloc - size); 1417 pages += pages_highmem + size; 1418 } 1419 1420 /* 1421 * We only need as many page frames for the image as there are saveable 1422 * pages in memory, but we have allocated more. Release the excessive 1423 * ones now. 1424 */ 1425 free_unnecessary_pages(); 1426 1427 out: 1428 do_gettimeofday(&stop); 1429 printk(KERN_CONT "done (allocated %lu pages)\n", pages); 1430 swsusp_show_speed(&start, &stop, pages, "Allocated"); 1431 1432 return 0; 1433 1434 err_out: 1435 printk(KERN_CONT "\n"); 1436 swsusp_free(); 1437 return -ENOMEM; 1438 } 1439 1440 #ifdef CONFIG_HIGHMEM 1441 /** 1442 * count_pages_for_highmem - compute the number of non-highmem pages 1443 * that will be necessary for creating copies of highmem pages. 1444 */ 1445 1446 static unsigned int count_pages_for_highmem(unsigned int nr_highmem) 1447 { 1448 unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem; 1449 1450 if (free_highmem >= nr_highmem) 1451 nr_highmem = 0; 1452 else 1453 nr_highmem -= free_highmem; 1454 1455 return nr_highmem; 1456 } 1457 #else 1458 static unsigned int 1459 count_pages_for_highmem(unsigned int nr_highmem) { return 0; } 1460 #endif /* CONFIG_HIGHMEM */ 1461 1462 /** 1463 * enough_free_mem - Make sure we have enough free memory for the 1464 * snapshot image. 1465 */ 1466 1467 static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem) 1468 { 1469 struct zone *zone; 1470 unsigned int free = alloc_normal; 1471 1472 for_each_populated_zone(zone) 1473 if (!is_highmem(zone)) 1474 free += zone_page_state(zone, NR_FREE_PAGES); 1475 1476 nr_pages += count_pages_for_highmem(nr_highmem); 1477 pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n", 1478 nr_pages, PAGES_FOR_IO, free); 1479 1480 return free > nr_pages + PAGES_FOR_IO; 1481 } 1482 1483 #ifdef CONFIG_HIGHMEM 1484 /** 1485 * get_highmem_buffer - if there are some highmem pages in the suspend 1486 * image, we may need the buffer to copy them and/or load their data. 1487 */ 1488 1489 static inline int get_highmem_buffer(int safe_needed) 1490 { 1491 buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed); 1492 return buffer ? 0 : -ENOMEM; 1493 } 1494 1495 /** 1496 * alloc_highmem_image_pages - allocate some highmem pages for the image. 1497 * Try to allocate as many pages as needed, but if the number of free 1498 * highmem pages is lesser than that, allocate them all. 1499 */ 1500 1501 static inline unsigned int 1502 alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem) 1503 { 1504 unsigned int to_alloc = count_free_highmem_pages(); 1505 1506 if (to_alloc > nr_highmem) 1507 to_alloc = nr_highmem; 1508 1509 nr_highmem -= to_alloc; 1510 while (to_alloc-- > 0) { 1511 struct page *page; 1512 1513 page = alloc_image_page(__GFP_HIGHMEM); 1514 memory_bm_set_bit(bm, page_to_pfn(page)); 1515 } 1516 return nr_highmem; 1517 } 1518 #else 1519 static inline int get_highmem_buffer(int safe_needed) { return 0; } 1520 1521 static inline unsigned int 1522 alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; } 1523 #endif /* CONFIG_HIGHMEM */ 1524 1525 /** 1526 * swsusp_alloc - allocate memory for the suspend image 1527 * 1528 * We first try to allocate as many highmem pages as there are 1529 * saveable highmem pages in the system. If that fails, we allocate 1530 * non-highmem pages for the copies of the remaining highmem ones. 1531 * 1532 * In this approach it is likely that the copies of highmem pages will 1533 * also be located in the high memory, because of the way in which 1534 * copy_data_pages() works. 1535 */ 1536 1537 static int 1538 swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm, 1539 unsigned int nr_pages, unsigned int nr_highmem) 1540 { 1541 if (nr_highmem > 0) { 1542 if (get_highmem_buffer(PG_ANY)) 1543 goto err_out; 1544 if (nr_highmem > alloc_highmem) { 1545 nr_highmem -= alloc_highmem; 1546 nr_pages += alloc_highmem_pages(copy_bm, nr_highmem); 1547 } 1548 } 1549 if (nr_pages > alloc_normal) { 1550 nr_pages -= alloc_normal; 1551 while (nr_pages-- > 0) { 1552 struct page *page; 1553 1554 page = alloc_image_page(GFP_ATOMIC | __GFP_COLD); 1555 if (!page) 1556 goto err_out; 1557 memory_bm_set_bit(copy_bm, page_to_pfn(page)); 1558 } 1559 } 1560 1561 return 0; 1562 1563 err_out: 1564 swsusp_free(); 1565 return -ENOMEM; 1566 } 1567 1568 asmlinkage int swsusp_save(void) 1569 { 1570 unsigned int nr_pages, nr_highmem; 1571 1572 printk(KERN_INFO "PM: Creating hibernation image:\n"); 1573 1574 drain_local_pages(NULL); 1575 nr_pages = count_data_pages(); 1576 nr_highmem = count_highmem_pages(); 1577 printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem); 1578 1579 if (!enough_free_mem(nr_pages, nr_highmem)) { 1580 printk(KERN_ERR "PM: Not enough free memory\n"); 1581 return -ENOMEM; 1582 } 1583 1584 if (swsusp_alloc(&orig_bm, ©_bm, nr_pages, nr_highmem)) { 1585 printk(KERN_ERR "PM: Memory allocation failed\n"); 1586 return -ENOMEM; 1587 } 1588 1589 /* During allocating of suspend pagedir, new cold pages may appear. 1590 * Kill them. 1591 */ 1592 drain_local_pages(NULL); 1593 copy_data_pages(©_bm, &orig_bm); 1594 1595 /* 1596 * End of critical section. From now on, we can write to memory, 1597 * but we should not touch disk. This specially means we must _not_ 1598 * touch swap space! Except we must write out our image of course. 1599 */ 1600 1601 nr_pages += nr_highmem; 1602 nr_copy_pages = nr_pages; 1603 nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE); 1604 1605 printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n", 1606 nr_pages); 1607 1608 return 0; 1609 } 1610 1611 #ifndef CONFIG_ARCH_HIBERNATION_HEADER 1612 static int init_header_complete(struct swsusp_info *info) 1613 { 1614 memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname)); 1615 info->version_code = LINUX_VERSION_CODE; 1616 return 0; 1617 } 1618 1619 static char *check_image_kernel(struct swsusp_info *info) 1620 { 1621 if (info->version_code != LINUX_VERSION_CODE) 1622 return "kernel version"; 1623 if (strcmp(info->uts.sysname,init_utsname()->sysname)) 1624 return "system type"; 1625 if (strcmp(info->uts.release,init_utsname()->release)) 1626 return "kernel release"; 1627 if (strcmp(info->uts.version,init_utsname()->version)) 1628 return "version"; 1629 if (strcmp(info->uts.machine,init_utsname()->machine)) 1630 return "machine"; 1631 return NULL; 1632 } 1633 #endif /* CONFIG_ARCH_HIBERNATION_HEADER */ 1634 1635 unsigned long snapshot_get_image_size(void) 1636 { 1637 return nr_copy_pages + nr_meta_pages + 1; 1638 } 1639 1640 static int init_header(struct swsusp_info *info) 1641 { 1642 memset(info, 0, sizeof(struct swsusp_info)); 1643 info->num_physpages = num_physpages; 1644 info->image_pages = nr_copy_pages; 1645 info->pages = snapshot_get_image_size(); 1646 info->size = info->pages; 1647 info->size <<= PAGE_SHIFT; 1648 return init_header_complete(info); 1649 } 1650 1651 /** 1652 * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm 1653 * are stored in the array @buf[] (1 page at a time) 1654 */ 1655 1656 static inline void 1657 pack_pfns(unsigned long *buf, struct memory_bitmap *bm) 1658 { 1659 int j; 1660 1661 for (j = 0; j < PAGE_SIZE / sizeof(long); j++) { 1662 buf[j] = memory_bm_next_pfn(bm); 1663 if (unlikely(buf[j] == BM_END_OF_MAP)) 1664 break; 1665 } 1666 } 1667 1668 /** 1669 * snapshot_read_next - used for reading the system memory snapshot. 1670 * 1671 * On the first call to it @handle should point to a zeroed 1672 * snapshot_handle structure. The structure gets updated and a pointer 1673 * to it should be passed to this function every next time. 1674 * 1675 * On success the function returns a positive number. Then, the caller 1676 * is allowed to read up to the returned number of bytes from the memory 1677 * location computed by the data_of() macro. 1678 * 1679 * The function returns 0 to indicate the end of data stream condition, 1680 * and a negative number is returned on error. In such cases the 1681 * structure pointed to by @handle is not updated and should not be used 1682 * any more. 1683 */ 1684 1685 int snapshot_read_next(struct snapshot_handle *handle) 1686 { 1687 if (handle->cur > nr_meta_pages + nr_copy_pages) 1688 return 0; 1689 1690 if (!buffer) { 1691 /* This makes the buffer be freed by swsusp_free() */ 1692 buffer = get_image_page(GFP_ATOMIC, PG_ANY); 1693 if (!buffer) 1694 return -ENOMEM; 1695 } 1696 if (!handle->cur) { 1697 int error; 1698 1699 error = init_header((struct swsusp_info *)buffer); 1700 if (error) 1701 return error; 1702 handle->buffer = buffer; 1703 memory_bm_position_reset(&orig_bm); 1704 memory_bm_position_reset(©_bm); 1705 } else if (handle->cur <= nr_meta_pages) { 1706 clear_page(buffer); 1707 pack_pfns(buffer, &orig_bm); 1708 } else { 1709 struct page *page; 1710 1711 page = pfn_to_page(memory_bm_next_pfn(©_bm)); 1712 if (PageHighMem(page)) { 1713 /* Highmem pages are copied to the buffer, 1714 * because we can't return with a kmapped 1715 * highmem page (we may not be called again). 1716 */ 1717 void *kaddr; 1718 1719 kaddr = kmap_atomic(page, KM_USER0); 1720 copy_page(buffer, kaddr); 1721 kunmap_atomic(kaddr, KM_USER0); 1722 handle->buffer = buffer; 1723 } else { 1724 handle->buffer = page_address(page); 1725 } 1726 } 1727 handle->cur++; 1728 return PAGE_SIZE; 1729 } 1730 1731 /** 1732 * mark_unsafe_pages - mark the pages that cannot be used for storing 1733 * the image during resume, because they conflict with the pages that 1734 * had been used before suspend 1735 */ 1736 1737 static int mark_unsafe_pages(struct memory_bitmap *bm) 1738 { 1739 struct zone *zone; 1740 unsigned long pfn, max_zone_pfn; 1741 1742 /* Clear page flags */ 1743 for_each_populated_zone(zone) { 1744 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 1745 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1746 if (pfn_valid(pfn)) 1747 swsusp_unset_page_free(pfn_to_page(pfn)); 1748 } 1749 1750 /* Mark pages that correspond to the "original" pfns as "unsafe" */ 1751 memory_bm_position_reset(bm); 1752 do { 1753 pfn = memory_bm_next_pfn(bm); 1754 if (likely(pfn != BM_END_OF_MAP)) { 1755 if (likely(pfn_valid(pfn))) 1756 swsusp_set_page_free(pfn_to_page(pfn)); 1757 else 1758 return -EFAULT; 1759 } 1760 } while (pfn != BM_END_OF_MAP); 1761 1762 allocated_unsafe_pages = 0; 1763 1764 return 0; 1765 } 1766 1767 static void 1768 duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src) 1769 { 1770 unsigned long pfn; 1771 1772 memory_bm_position_reset(src); 1773 pfn = memory_bm_next_pfn(src); 1774 while (pfn != BM_END_OF_MAP) { 1775 memory_bm_set_bit(dst, pfn); 1776 pfn = memory_bm_next_pfn(src); 1777 } 1778 } 1779 1780 static int check_header(struct swsusp_info *info) 1781 { 1782 char *reason; 1783 1784 reason = check_image_kernel(info); 1785 if (!reason && info->num_physpages != num_physpages) 1786 reason = "memory size"; 1787 if (reason) { 1788 printk(KERN_ERR "PM: Image mismatch: %s\n", reason); 1789 return -EPERM; 1790 } 1791 return 0; 1792 } 1793 1794 /** 1795 * load header - check the image header and copy data from it 1796 */ 1797 1798 static int 1799 load_header(struct swsusp_info *info) 1800 { 1801 int error; 1802 1803 restore_pblist = NULL; 1804 error = check_header(info); 1805 if (!error) { 1806 nr_copy_pages = info->image_pages; 1807 nr_meta_pages = info->pages - info->image_pages - 1; 1808 } 1809 return error; 1810 } 1811 1812 /** 1813 * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set 1814 * the corresponding bit in the memory bitmap @bm 1815 */ 1816 static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm) 1817 { 1818 int j; 1819 1820 for (j = 0; j < PAGE_SIZE / sizeof(long); j++) { 1821 if (unlikely(buf[j] == BM_END_OF_MAP)) 1822 break; 1823 1824 if (memory_bm_pfn_present(bm, buf[j])) 1825 memory_bm_set_bit(bm, buf[j]); 1826 else 1827 return -EFAULT; 1828 } 1829 1830 return 0; 1831 } 1832 1833 /* List of "safe" pages that may be used to store data loaded from the suspend 1834 * image 1835 */ 1836 static struct linked_page *safe_pages_list; 1837 1838 #ifdef CONFIG_HIGHMEM 1839 /* struct highmem_pbe is used for creating the list of highmem pages that 1840 * should be restored atomically during the resume from disk, because the page 1841 * frames they have occupied before the suspend are in use. 1842 */ 1843 struct highmem_pbe { 1844 struct page *copy_page; /* data is here now */ 1845 struct page *orig_page; /* data was here before the suspend */ 1846 struct highmem_pbe *next; 1847 }; 1848 1849 /* List of highmem PBEs needed for restoring the highmem pages that were 1850 * allocated before the suspend and included in the suspend image, but have 1851 * also been allocated by the "resume" kernel, so their contents cannot be 1852 * written directly to their "original" page frames. 1853 */ 1854 static struct highmem_pbe *highmem_pblist; 1855 1856 /** 1857 * count_highmem_image_pages - compute the number of highmem pages in the 1858 * suspend image. The bits in the memory bitmap @bm that correspond to the 1859 * image pages are assumed to be set. 1860 */ 1861 1862 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) 1863 { 1864 unsigned long pfn; 1865 unsigned int cnt = 0; 1866 1867 memory_bm_position_reset(bm); 1868 pfn = memory_bm_next_pfn(bm); 1869 while (pfn != BM_END_OF_MAP) { 1870 if (PageHighMem(pfn_to_page(pfn))) 1871 cnt++; 1872 1873 pfn = memory_bm_next_pfn(bm); 1874 } 1875 return cnt; 1876 } 1877 1878 /** 1879 * prepare_highmem_image - try to allocate as many highmem pages as 1880 * there are highmem image pages (@nr_highmem_p points to the variable 1881 * containing the number of highmem image pages). The pages that are 1882 * "safe" (ie. will not be overwritten when the suspend image is 1883 * restored) have the corresponding bits set in @bm (it must be 1884 * unitialized). 1885 * 1886 * NOTE: This function should not be called if there are no highmem 1887 * image pages. 1888 */ 1889 1890 static unsigned int safe_highmem_pages; 1891 1892 static struct memory_bitmap *safe_highmem_bm; 1893 1894 static int 1895 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p) 1896 { 1897 unsigned int to_alloc; 1898 1899 if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE)) 1900 return -ENOMEM; 1901 1902 if (get_highmem_buffer(PG_SAFE)) 1903 return -ENOMEM; 1904 1905 to_alloc = count_free_highmem_pages(); 1906 if (to_alloc > *nr_highmem_p) 1907 to_alloc = *nr_highmem_p; 1908 else 1909 *nr_highmem_p = to_alloc; 1910 1911 safe_highmem_pages = 0; 1912 while (to_alloc-- > 0) { 1913 struct page *page; 1914 1915 page = alloc_page(__GFP_HIGHMEM); 1916 if (!swsusp_page_is_free(page)) { 1917 /* The page is "safe", set its bit the bitmap */ 1918 memory_bm_set_bit(bm, page_to_pfn(page)); 1919 safe_highmem_pages++; 1920 } 1921 /* Mark the page as allocated */ 1922 swsusp_set_page_forbidden(page); 1923 swsusp_set_page_free(page); 1924 } 1925 memory_bm_position_reset(bm); 1926 safe_highmem_bm = bm; 1927 return 0; 1928 } 1929 1930 /** 1931 * get_highmem_page_buffer - for given highmem image page find the buffer 1932 * that suspend_write_next() should set for its caller to write to. 1933 * 1934 * If the page is to be saved to its "original" page frame or a copy of 1935 * the page is to be made in the highmem, @buffer is returned. Otherwise, 1936 * the copy of the page is to be made in normal memory, so the address of 1937 * the copy is returned. 1938 * 1939 * If @buffer is returned, the caller of suspend_write_next() will write 1940 * the page's contents to @buffer, so they will have to be copied to the 1941 * right location on the next call to suspend_write_next() and it is done 1942 * with the help of copy_last_highmem_page(). For this purpose, if 1943 * @buffer is returned, @last_highmem page is set to the page to which 1944 * the data will have to be copied from @buffer. 1945 */ 1946 1947 static struct page *last_highmem_page; 1948 1949 static void * 1950 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca) 1951 { 1952 struct highmem_pbe *pbe; 1953 void *kaddr; 1954 1955 if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) { 1956 /* We have allocated the "original" page frame and we can 1957 * use it directly to store the loaded page. 1958 */ 1959 last_highmem_page = page; 1960 return buffer; 1961 } 1962 /* The "original" page frame has not been allocated and we have to 1963 * use a "safe" page frame to store the loaded page. 1964 */ 1965 pbe = chain_alloc(ca, sizeof(struct highmem_pbe)); 1966 if (!pbe) { 1967 swsusp_free(); 1968 return ERR_PTR(-ENOMEM); 1969 } 1970 pbe->orig_page = page; 1971 if (safe_highmem_pages > 0) { 1972 struct page *tmp; 1973 1974 /* Copy of the page will be stored in high memory */ 1975 kaddr = buffer; 1976 tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm)); 1977 safe_highmem_pages--; 1978 last_highmem_page = tmp; 1979 pbe->copy_page = tmp; 1980 } else { 1981 /* Copy of the page will be stored in normal memory */ 1982 kaddr = safe_pages_list; 1983 safe_pages_list = safe_pages_list->next; 1984 pbe->copy_page = virt_to_page(kaddr); 1985 } 1986 pbe->next = highmem_pblist; 1987 highmem_pblist = pbe; 1988 return kaddr; 1989 } 1990 1991 /** 1992 * copy_last_highmem_page - copy the contents of a highmem image from 1993 * @buffer, where the caller of snapshot_write_next() has place them, 1994 * to the right location represented by @last_highmem_page . 1995 */ 1996 1997 static void copy_last_highmem_page(void) 1998 { 1999 if (last_highmem_page) { 2000 void *dst; 2001 2002 dst = kmap_atomic(last_highmem_page, KM_USER0); 2003 copy_page(dst, buffer); 2004 kunmap_atomic(dst, KM_USER0); 2005 last_highmem_page = NULL; 2006 } 2007 } 2008 2009 static inline int last_highmem_page_copied(void) 2010 { 2011 return !last_highmem_page; 2012 } 2013 2014 static inline void free_highmem_data(void) 2015 { 2016 if (safe_highmem_bm) 2017 memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR); 2018 2019 if (buffer) 2020 free_image_page(buffer, PG_UNSAFE_CLEAR); 2021 } 2022 #else 2023 static inline int get_safe_write_buffer(void) { return 0; } 2024 2025 static unsigned int 2026 count_highmem_image_pages(struct memory_bitmap *bm) { return 0; } 2027 2028 static inline int 2029 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p) 2030 { 2031 return 0; 2032 } 2033 2034 static inline void * 2035 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca) 2036 { 2037 return ERR_PTR(-EINVAL); 2038 } 2039 2040 static inline void copy_last_highmem_page(void) {} 2041 static inline int last_highmem_page_copied(void) { return 1; } 2042 static inline void free_highmem_data(void) {} 2043 #endif /* CONFIG_HIGHMEM */ 2044 2045 /** 2046 * prepare_image - use the memory bitmap @bm to mark the pages that will 2047 * be overwritten in the process of restoring the system memory state 2048 * from the suspend image ("unsafe" pages) and allocate memory for the 2049 * image. 2050 * 2051 * The idea is to allocate a new memory bitmap first and then allocate 2052 * as many pages as needed for the image data, but not to assign these 2053 * pages to specific tasks initially. Instead, we just mark them as 2054 * allocated and create a lists of "safe" pages that will be used 2055 * later. On systems with high memory a list of "safe" highmem pages is 2056 * also created. 2057 */ 2058 2059 #define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe)) 2060 2061 static int 2062 prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm) 2063 { 2064 unsigned int nr_pages, nr_highmem; 2065 struct linked_page *sp_list, *lp; 2066 int error; 2067 2068 /* If there is no highmem, the buffer will not be necessary */ 2069 free_image_page(buffer, PG_UNSAFE_CLEAR); 2070 buffer = NULL; 2071 2072 nr_highmem = count_highmem_image_pages(bm); 2073 error = mark_unsafe_pages(bm); 2074 if (error) 2075 goto Free; 2076 2077 error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE); 2078 if (error) 2079 goto Free; 2080 2081 duplicate_memory_bitmap(new_bm, bm); 2082 memory_bm_free(bm, PG_UNSAFE_KEEP); 2083 if (nr_highmem > 0) { 2084 error = prepare_highmem_image(bm, &nr_highmem); 2085 if (error) 2086 goto Free; 2087 } 2088 /* Reserve some safe pages for potential later use. 2089 * 2090 * NOTE: This way we make sure there will be enough safe pages for the 2091 * chain_alloc() in get_buffer(). It is a bit wasteful, but 2092 * nr_copy_pages cannot be greater than 50% of the memory anyway. 2093 */ 2094 sp_list = NULL; 2095 /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */ 2096 nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages; 2097 nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE); 2098 while (nr_pages > 0) { 2099 lp = get_image_page(GFP_ATOMIC, PG_SAFE); 2100 if (!lp) { 2101 error = -ENOMEM; 2102 goto Free; 2103 } 2104 lp->next = sp_list; 2105 sp_list = lp; 2106 nr_pages--; 2107 } 2108 /* Preallocate memory for the image */ 2109 safe_pages_list = NULL; 2110 nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages; 2111 while (nr_pages > 0) { 2112 lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC); 2113 if (!lp) { 2114 error = -ENOMEM; 2115 goto Free; 2116 } 2117 if (!swsusp_page_is_free(virt_to_page(lp))) { 2118 /* The page is "safe", add it to the list */ 2119 lp->next = safe_pages_list; 2120 safe_pages_list = lp; 2121 } 2122 /* Mark the page as allocated */ 2123 swsusp_set_page_forbidden(virt_to_page(lp)); 2124 swsusp_set_page_free(virt_to_page(lp)); 2125 nr_pages--; 2126 } 2127 /* Free the reserved safe pages so that chain_alloc() can use them */ 2128 while (sp_list) { 2129 lp = sp_list->next; 2130 free_image_page(sp_list, PG_UNSAFE_CLEAR); 2131 sp_list = lp; 2132 } 2133 return 0; 2134 2135 Free: 2136 swsusp_free(); 2137 return error; 2138 } 2139 2140 /** 2141 * get_buffer - compute the address that snapshot_write_next() should 2142 * set for its caller to write to. 2143 */ 2144 2145 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca) 2146 { 2147 struct pbe *pbe; 2148 struct page *page; 2149 unsigned long pfn = memory_bm_next_pfn(bm); 2150 2151 if (pfn == BM_END_OF_MAP) 2152 return ERR_PTR(-EFAULT); 2153 2154 page = pfn_to_page(pfn); 2155 if (PageHighMem(page)) 2156 return get_highmem_page_buffer(page, ca); 2157 2158 if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) 2159 /* We have allocated the "original" page frame and we can 2160 * use it directly to store the loaded page. 2161 */ 2162 return page_address(page); 2163 2164 /* The "original" page frame has not been allocated and we have to 2165 * use a "safe" page frame to store the loaded page. 2166 */ 2167 pbe = chain_alloc(ca, sizeof(struct pbe)); 2168 if (!pbe) { 2169 swsusp_free(); 2170 return ERR_PTR(-ENOMEM); 2171 } 2172 pbe->orig_address = page_address(page); 2173 pbe->address = safe_pages_list; 2174 safe_pages_list = safe_pages_list->next; 2175 pbe->next = restore_pblist; 2176 restore_pblist = pbe; 2177 return pbe->address; 2178 } 2179 2180 /** 2181 * snapshot_write_next - used for writing the system memory snapshot. 2182 * 2183 * On the first call to it @handle should point to a zeroed 2184 * snapshot_handle structure. The structure gets updated and a pointer 2185 * to it should be passed to this function every next time. 2186 * 2187 * On success the function returns a positive number. Then, the caller 2188 * is allowed to write up to the returned number of bytes to the memory 2189 * location computed by the data_of() macro. 2190 * 2191 * The function returns 0 to indicate the "end of file" condition, 2192 * and a negative number is returned on error. In such cases the 2193 * structure pointed to by @handle is not updated and should not be used 2194 * any more. 2195 */ 2196 2197 int snapshot_write_next(struct snapshot_handle *handle) 2198 { 2199 static struct chain_allocator ca; 2200 int error = 0; 2201 2202 /* Check if we have already loaded the entire image */ 2203 if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) 2204 return 0; 2205 2206 handle->sync_read = 1; 2207 2208 if (!handle->cur) { 2209 if (!buffer) 2210 /* This makes the buffer be freed by swsusp_free() */ 2211 buffer = get_image_page(GFP_ATOMIC, PG_ANY); 2212 2213 if (!buffer) 2214 return -ENOMEM; 2215 2216 handle->buffer = buffer; 2217 } else if (handle->cur == 1) { 2218 error = load_header(buffer); 2219 if (error) 2220 return error; 2221 2222 error = memory_bm_create(©_bm, GFP_ATOMIC, PG_ANY); 2223 if (error) 2224 return error; 2225 2226 } else if (handle->cur <= nr_meta_pages + 1) { 2227 error = unpack_orig_pfns(buffer, ©_bm); 2228 if (error) 2229 return error; 2230 2231 if (handle->cur == nr_meta_pages + 1) { 2232 error = prepare_image(&orig_bm, ©_bm); 2233 if (error) 2234 return error; 2235 2236 chain_init(&ca, GFP_ATOMIC, PG_SAFE); 2237 memory_bm_position_reset(&orig_bm); 2238 restore_pblist = NULL; 2239 handle->buffer = get_buffer(&orig_bm, &ca); 2240 handle->sync_read = 0; 2241 if (IS_ERR(handle->buffer)) 2242 return PTR_ERR(handle->buffer); 2243 } 2244 } else { 2245 copy_last_highmem_page(); 2246 handle->buffer = get_buffer(&orig_bm, &ca); 2247 if (IS_ERR(handle->buffer)) 2248 return PTR_ERR(handle->buffer); 2249 if (handle->buffer != buffer) 2250 handle->sync_read = 0; 2251 } 2252 handle->cur++; 2253 return PAGE_SIZE; 2254 } 2255 2256 /** 2257 * snapshot_write_finalize - must be called after the last call to 2258 * snapshot_write_next() in case the last page in the image happens 2259 * to be a highmem page and its contents should be stored in the 2260 * highmem. Additionally, it releases the memory that will not be 2261 * used any more. 2262 */ 2263 2264 void snapshot_write_finalize(struct snapshot_handle *handle) 2265 { 2266 copy_last_highmem_page(); 2267 /* Free only if we have loaded the image entirely */ 2268 if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) { 2269 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR); 2270 free_highmem_data(); 2271 } 2272 } 2273 2274 int snapshot_image_loaded(struct snapshot_handle *handle) 2275 { 2276 return !(!nr_copy_pages || !last_highmem_page_copied() || 2277 handle->cur <= nr_meta_pages + nr_copy_pages); 2278 } 2279 2280 #ifdef CONFIG_HIGHMEM 2281 /* Assumes that @buf is ready and points to a "safe" page */ 2282 static inline void 2283 swap_two_pages_data(struct page *p1, struct page *p2, void *buf) 2284 { 2285 void *kaddr1, *kaddr2; 2286 2287 kaddr1 = kmap_atomic(p1, KM_USER0); 2288 kaddr2 = kmap_atomic(p2, KM_USER1); 2289 copy_page(buf, kaddr1); 2290 copy_page(kaddr1, kaddr2); 2291 copy_page(kaddr2, buf); 2292 kunmap_atomic(kaddr2, KM_USER1); 2293 kunmap_atomic(kaddr1, KM_USER0); 2294 } 2295 2296 /** 2297 * restore_highmem - for each highmem page that was allocated before 2298 * the suspend and included in the suspend image, and also has been 2299 * allocated by the "resume" kernel swap its current (ie. "before 2300 * resume") contents with the previous (ie. "before suspend") one. 2301 * 2302 * If the resume eventually fails, we can call this function once 2303 * again and restore the "before resume" highmem state. 2304 */ 2305 2306 int restore_highmem(void) 2307 { 2308 struct highmem_pbe *pbe = highmem_pblist; 2309 void *buf; 2310 2311 if (!pbe) 2312 return 0; 2313 2314 buf = get_image_page(GFP_ATOMIC, PG_SAFE); 2315 if (!buf) 2316 return -ENOMEM; 2317 2318 while (pbe) { 2319 swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf); 2320 pbe = pbe->next; 2321 } 2322 free_image_page(buf, PG_UNSAFE_CLEAR); 2323 return 0; 2324 } 2325 #endif /* CONFIG_HIGHMEM */ 2326