1 /* 2 * Hibernation support for x86-64 3 * 4 * Distribute under GPLv2 5 * 6 * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl> 7 * Copyright (c) 2002 Pavel Machek <pavel@ucw.cz> 8 * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org> 9 */ 10 11 #include <linux/gfp.h> 12 #include <linux/smp.h> 13 #include <linux/suspend.h> 14 #include <linux/scatterlist.h> 15 #include <linux/kdebug.h> 16 17 #include <crypto/hash.h> 18 19 #include <asm/e820/api.h> 20 #include <asm/init.h> 21 #include <asm/proto.h> 22 #include <asm/page.h> 23 #include <asm/pgtable.h> 24 #include <asm/mtrr.h> 25 #include <asm/sections.h> 26 #include <asm/suspend.h> 27 #include <asm/tlbflush.h> 28 29 /* Defined in hibernate_asm_64.S */ 30 extern asmlinkage __visible int restore_image(void); 31 32 /* 33 * Address to jump to in the last phase of restore in order to get to the image 34 * kernel's text (this value is passed in the image header). 35 */ 36 unsigned long restore_jump_address __visible; 37 unsigned long jump_address_phys; 38 39 /* 40 * Value of the cr3 register from before the hibernation (this value is passed 41 * in the image header). 42 */ 43 unsigned long restore_cr3 __visible; 44 45 unsigned long temp_level4_pgt __visible; 46 47 unsigned long relocated_restore_code __visible; 48 49 static int set_up_temporary_text_mapping(pgd_t *pgd) 50 { 51 pmd_t *pmd; 52 pud_t *pud; 53 p4d_t *p4d; 54 55 /* 56 * The new mapping only has to cover the page containing the image 57 * kernel's entry point (jump_address_phys), because the switch over to 58 * it is carried out by relocated code running from a page allocated 59 * specifically for this purpose and covered by the identity mapping, so 60 * the temporary kernel text mapping is only needed for the final jump. 61 * Moreover, in that mapping the virtual address of the image kernel's 62 * entry point must be the same as its virtual address in the image 63 * kernel (restore_jump_address), so the image kernel's 64 * restore_registers() code doesn't find itself in a different area of 65 * the virtual address space after switching over to the original page 66 * tables used by the image kernel. 67 */ 68 69 if (IS_ENABLED(CONFIG_X86_5LEVEL)) { 70 p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); 71 if (!p4d) 72 return -ENOMEM; 73 } 74 75 pud = (pud_t *)get_safe_page(GFP_ATOMIC); 76 if (!pud) 77 return -ENOMEM; 78 79 pmd = (pmd_t *)get_safe_page(GFP_ATOMIC); 80 if (!pmd) 81 return -ENOMEM; 82 83 set_pmd(pmd + pmd_index(restore_jump_address), 84 __pmd((jump_address_phys & PMD_MASK) | __PAGE_KERNEL_LARGE_EXEC)); 85 set_pud(pud + pud_index(restore_jump_address), 86 __pud(__pa(pmd) | _KERNPG_TABLE)); 87 if (IS_ENABLED(CONFIG_X86_5LEVEL)) { 88 set_p4d(p4d + p4d_index(restore_jump_address), __p4d(__pa(pud) | _KERNPG_TABLE)); 89 set_pgd(pgd + pgd_index(restore_jump_address), __pgd(__pa(p4d) | _KERNPG_TABLE)); 90 } else { 91 /* No p4d for 4-level paging: point the pgd to the pud page table */ 92 set_pgd(pgd + pgd_index(restore_jump_address), __pgd(__pa(pud) | _KERNPG_TABLE)); 93 } 94 95 return 0; 96 } 97 98 static void *alloc_pgt_page(void *context) 99 { 100 return (void *)get_safe_page(GFP_ATOMIC); 101 } 102 103 static int set_up_temporary_mappings(void) 104 { 105 struct x86_mapping_info info = { 106 .alloc_pgt_page = alloc_pgt_page, 107 .page_flag = __PAGE_KERNEL_LARGE_EXEC, 108 .offset = __PAGE_OFFSET, 109 }; 110 unsigned long mstart, mend; 111 pgd_t *pgd; 112 int result; 113 int i; 114 115 pgd = (pgd_t *)get_safe_page(GFP_ATOMIC); 116 if (!pgd) 117 return -ENOMEM; 118 119 /* Prepare a temporary mapping for the kernel text */ 120 result = set_up_temporary_text_mapping(pgd); 121 if (result) 122 return result; 123 124 /* Set up the direct mapping from scratch */ 125 for (i = 0; i < nr_pfn_mapped; i++) { 126 mstart = pfn_mapped[i].start << PAGE_SHIFT; 127 mend = pfn_mapped[i].end << PAGE_SHIFT; 128 129 result = kernel_ident_mapping_init(&info, pgd, mstart, mend); 130 if (result) 131 return result; 132 } 133 134 temp_level4_pgt = __pa(pgd); 135 return 0; 136 } 137 138 static int relocate_restore_code(void) 139 { 140 pgd_t *pgd; 141 p4d_t *p4d; 142 pud_t *pud; 143 pmd_t *pmd; 144 pte_t *pte; 145 146 relocated_restore_code = get_safe_page(GFP_ATOMIC); 147 if (!relocated_restore_code) 148 return -ENOMEM; 149 150 memcpy((void *)relocated_restore_code, &core_restore_code, PAGE_SIZE); 151 152 /* Make the page containing the relocated code executable */ 153 pgd = (pgd_t *)__va(read_cr3()) + pgd_index(relocated_restore_code); 154 p4d = p4d_offset(pgd, relocated_restore_code); 155 if (p4d_large(*p4d)) { 156 set_p4d(p4d, __p4d(p4d_val(*p4d) & ~_PAGE_NX)); 157 goto out; 158 } 159 pud = pud_offset(p4d, relocated_restore_code); 160 if (pud_large(*pud)) { 161 set_pud(pud, __pud(pud_val(*pud) & ~_PAGE_NX)); 162 goto out; 163 } 164 pmd = pmd_offset(pud, relocated_restore_code); 165 if (pmd_large(*pmd)) { 166 set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_NX)); 167 goto out; 168 } 169 pte = pte_offset_kernel(pmd, relocated_restore_code); 170 set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_NX)); 171 out: 172 __flush_tlb_all(); 173 return 0; 174 } 175 176 int swsusp_arch_resume(void) 177 { 178 int error; 179 180 /* We have got enough memory and from now on we cannot recover */ 181 error = set_up_temporary_mappings(); 182 if (error) 183 return error; 184 185 error = relocate_restore_code(); 186 if (error) 187 return error; 188 189 restore_image(); 190 return 0; 191 } 192 193 /* 194 * pfn_is_nosave - check if given pfn is in the 'nosave' section 195 */ 196 197 int pfn_is_nosave(unsigned long pfn) 198 { 199 unsigned long nosave_begin_pfn = __pa_symbol(&__nosave_begin) >> PAGE_SHIFT; 200 unsigned long nosave_end_pfn = PAGE_ALIGN(__pa_symbol(&__nosave_end)) >> PAGE_SHIFT; 201 return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn); 202 } 203 204 #define MD5_DIGEST_SIZE 16 205 206 struct restore_data_record { 207 unsigned long jump_address; 208 unsigned long jump_address_phys; 209 unsigned long cr3; 210 unsigned long magic; 211 u8 e820_digest[MD5_DIGEST_SIZE]; 212 }; 213 214 #define RESTORE_MAGIC 0x23456789ABCDEF01UL 215 216 #if IS_BUILTIN(CONFIG_CRYPTO_MD5) 217 /** 218 * get_e820_md5 - calculate md5 according to given e820 table 219 * 220 * @table: the e820 table to be calculated 221 * @buf: the md5 result to be stored to 222 */ 223 static int get_e820_md5(struct e820_table *table, void *buf) 224 { 225 struct scatterlist sg; 226 struct crypto_ahash *tfm; 227 int size; 228 int ret = 0; 229 230 tfm = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); 231 if (IS_ERR(tfm)) 232 return -ENOMEM; 233 234 { 235 AHASH_REQUEST_ON_STACK(req, tfm); 236 size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry) * table->nr_entries; 237 ahash_request_set_tfm(req, tfm); 238 sg_init_one(&sg, (u8 *)table, size); 239 ahash_request_set_callback(req, 0, NULL, NULL); 240 ahash_request_set_crypt(req, &sg, buf, size); 241 242 if (crypto_ahash_digest(req)) 243 ret = -EINVAL; 244 ahash_request_zero(req); 245 } 246 crypto_free_ahash(tfm); 247 248 return ret; 249 } 250 251 static void hibernation_e820_save(void *buf) 252 { 253 get_e820_md5(e820_table_firmware, buf); 254 } 255 256 static bool hibernation_e820_mismatch(void *buf) 257 { 258 int ret; 259 u8 result[MD5_DIGEST_SIZE]; 260 261 memset(result, 0, MD5_DIGEST_SIZE); 262 /* If there is no digest in suspend kernel, let it go. */ 263 if (!memcmp(result, buf, MD5_DIGEST_SIZE)) 264 return false; 265 266 ret = get_e820_md5(e820_table_firmware, result); 267 if (ret) 268 return true; 269 270 return memcmp(result, buf, MD5_DIGEST_SIZE) ? true : false; 271 } 272 #else 273 static void hibernation_e820_save(void *buf) 274 { 275 } 276 277 static bool hibernation_e820_mismatch(void *buf) 278 { 279 /* If md5 is not builtin for restore kernel, let it go. */ 280 return false; 281 } 282 #endif 283 284 /** 285 * arch_hibernation_header_save - populate the architecture specific part 286 * of a hibernation image header 287 * @addr: address to save the data at 288 */ 289 int arch_hibernation_header_save(void *addr, unsigned int max_size) 290 { 291 struct restore_data_record *rdr = addr; 292 293 if (max_size < sizeof(struct restore_data_record)) 294 return -EOVERFLOW; 295 rdr->jump_address = (unsigned long)&restore_registers; 296 rdr->jump_address_phys = __pa_symbol(&restore_registers); 297 rdr->cr3 = restore_cr3; 298 rdr->magic = RESTORE_MAGIC; 299 300 hibernation_e820_save(rdr->e820_digest); 301 302 return 0; 303 } 304 305 /** 306 * arch_hibernation_header_restore - read the architecture specific data 307 * from the hibernation image header 308 * @addr: address to read the data from 309 */ 310 int arch_hibernation_header_restore(void *addr) 311 { 312 struct restore_data_record *rdr = addr; 313 314 restore_jump_address = rdr->jump_address; 315 jump_address_phys = rdr->jump_address_phys; 316 restore_cr3 = rdr->cr3; 317 318 if (rdr->magic != RESTORE_MAGIC) { 319 pr_crit("Unrecognized hibernate image header format!\n"); 320 return -EINVAL; 321 } 322 323 if (hibernation_e820_mismatch(rdr->e820_digest)) { 324 pr_crit("Hibernate inconsistent memory map detected!\n"); 325 return -ENODEV; 326 } 327 328 return 0; 329 } 330