xref: /openbmc/linux/arch/x86/power/cpu.c (revision f3a8b664)
1 /*
2  * Suspend support specific for i386/x86-64.
3  *
4  * Distribute under GPLv2
5  *
6  * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
7  * Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
8  * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
9  */
10 
11 #include <linux/suspend.h>
12 #include <linux/export.h>
13 #include <linux/smp.h>
14 #include <linux/perf_event.h>
15 #include <linux/tboot.h>
16 
17 #include <asm/pgtable.h>
18 #include <asm/proto.h>
19 #include <asm/mtrr.h>
20 #include <asm/page.h>
21 #include <asm/mce.h>
22 #include <asm/suspend.h>
23 #include <asm/fpu/internal.h>
24 #include <asm/debugreg.h>
25 #include <asm/cpu.h>
26 #include <asm/mmu_context.h>
27 #include <linux/dmi.h>
28 
29 #ifdef CONFIG_X86_32
30 __visible unsigned long saved_context_ebx;
31 __visible unsigned long saved_context_esp, saved_context_ebp;
32 __visible unsigned long saved_context_esi, saved_context_edi;
33 __visible unsigned long saved_context_eflags;
34 #endif
35 struct saved_context saved_context;
36 
37 static void msr_save_context(struct saved_context *ctxt)
38 {
39 	struct saved_msr *msr = ctxt->saved_msrs.array;
40 	struct saved_msr *end = msr + ctxt->saved_msrs.num;
41 
42 	while (msr < end) {
43 		msr->valid = !rdmsrl_safe(msr->info.msr_no, &msr->info.reg.q);
44 		msr++;
45 	}
46 }
47 
48 static void msr_restore_context(struct saved_context *ctxt)
49 {
50 	struct saved_msr *msr = ctxt->saved_msrs.array;
51 	struct saved_msr *end = msr + ctxt->saved_msrs.num;
52 
53 	while (msr < end) {
54 		if (msr->valid)
55 			wrmsrl(msr->info.msr_no, msr->info.reg.q);
56 		msr++;
57 	}
58 }
59 
60 /**
61  *	__save_processor_state - save CPU registers before creating a
62  *		hibernation image and before restoring the memory state from it
63  *	@ctxt - structure to store the registers contents in
64  *
65  *	NOTE: If there is a CPU register the modification of which by the
66  *	boot kernel (ie. the kernel used for loading the hibernation image)
67  *	might affect the operations of the restored target kernel (ie. the one
68  *	saved in the hibernation image), then its contents must be saved by this
69  *	function.  In other words, if kernel A is hibernated and different
70  *	kernel B is used for loading the hibernation image into memory, the
71  *	kernel A's __save_processor_state() function must save all registers
72  *	needed by kernel A, so that it can operate correctly after the resume
73  *	regardless of what kernel B does in the meantime.
74  */
75 static void __save_processor_state(struct saved_context *ctxt)
76 {
77 #ifdef CONFIG_X86_32
78 	mtrr_save_fixed_ranges(NULL);
79 #endif
80 	kernel_fpu_begin();
81 
82 	/*
83 	 * descriptor tables
84 	 */
85 #ifdef CONFIG_X86_32
86 	store_idt(&ctxt->idt);
87 #else
88 /* CONFIG_X86_64 */
89 	store_idt((struct desc_ptr *)&ctxt->idt_limit);
90 #endif
91 	/*
92 	 * We save it here, but restore it only in the hibernate case.
93 	 * For ACPI S3 resume, this is loaded via 'early_gdt_desc' in 64-bit
94 	 * mode in "secondary_startup_64". In 32-bit mode it is done via
95 	 * 'pmode_gdt' in wakeup_start.
96 	 */
97 	ctxt->gdt_desc.size = GDT_SIZE - 1;
98 	ctxt->gdt_desc.address = (unsigned long)get_cpu_gdt_table(smp_processor_id());
99 
100 	store_tr(ctxt->tr);
101 
102 	/* XMM0..XMM15 should be handled by kernel_fpu_begin(). */
103 	/*
104 	 * segment registers
105 	 */
106 #ifdef CONFIG_X86_32
107 	savesegment(es, ctxt->es);
108 	savesegment(fs, ctxt->fs);
109 	savesegment(gs, ctxt->gs);
110 	savesegment(ss, ctxt->ss);
111 #else
112 /* CONFIG_X86_64 */
113 	asm volatile ("movw %%ds, %0" : "=m" (ctxt->ds));
114 	asm volatile ("movw %%es, %0" : "=m" (ctxt->es));
115 	asm volatile ("movw %%fs, %0" : "=m" (ctxt->fs));
116 	asm volatile ("movw %%gs, %0" : "=m" (ctxt->gs));
117 	asm volatile ("movw %%ss, %0" : "=m" (ctxt->ss));
118 
119 	rdmsrl(MSR_FS_BASE, ctxt->fs_base);
120 	rdmsrl(MSR_GS_BASE, ctxt->gs_base);
121 	rdmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
122 	mtrr_save_fixed_ranges(NULL);
123 
124 	rdmsrl(MSR_EFER, ctxt->efer);
125 #endif
126 
127 	/*
128 	 * control registers
129 	 */
130 	ctxt->cr0 = read_cr0();
131 	ctxt->cr2 = read_cr2();
132 	ctxt->cr3 = read_cr3();
133 	ctxt->cr4 = __read_cr4();
134 #ifdef CONFIG_X86_64
135 	ctxt->cr8 = read_cr8();
136 #endif
137 	ctxt->misc_enable_saved = !rdmsrl_safe(MSR_IA32_MISC_ENABLE,
138 					       &ctxt->misc_enable);
139 	msr_save_context(ctxt);
140 }
141 
142 /* Needed by apm.c */
143 void save_processor_state(void)
144 {
145 	__save_processor_state(&saved_context);
146 	x86_platform.save_sched_clock_state();
147 }
148 #ifdef CONFIG_X86_32
149 EXPORT_SYMBOL(save_processor_state);
150 #endif
151 
152 static void do_fpu_end(void)
153 {
154 	/*
155 	 * Restore FPU regs if necessary.
156 	 */
157 	kernel_fpu_end();
158 }
159 
160 static void fix_processor_context(void)
161 {
162 	int cpu = smp_processor_id();
163 	struct tss_struct *t = &per_cpu(cpu_tss, cpu);
164 #ifdef CONFIG_X86_64
165 	struct desc_struct *desc = get_cpu_gdt_table(cpu);
166 	tss_desc tss;
167 #endif
168 	set_tss_desc(cpu, t);	/*
169 				 * This just modifies memory; should not be
170 				 * necessary. But... This is necessary, because
171 				 * 386 hardware has concept of busy TSS or some
172 				 * similar stupidity.
173 				 */
174 
175 #ifdef CONFIG_X86_64
176 	memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
177 	tss.type = 0x9; /* The available 64-bit TSS (see AMD vol 2, pg 91 */
178 	write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
179 
180 	syscall_init();				/* This sets MSR_*STAR and related */
181 #endif
182 	load_TR_desc();				/* This does ltr */
183 	load_mm_ldt(current->active_mm);	/* This does lldt */
184 
185 	fpu__resume_cpu();
186 }
187 
188 /**
189  *	__restore_processor_state - restore the contents of CPU registers saved
190  *		by __save_processor_state()
191  *	@ctxt - structure to load the registers contents from
192  */
193 static void notrace __restore_processor_state(struct saved_context *ctxt)
194 {
195 	if (ctxt->misc_enable_saved)
196 		wrmsrl(MSR_IA32_MISC_ENABLE, ctxt->misc_enable);
197 	/*
198 	 * control registers
199 	 */
200 	/* cr4 was introduced in the Pentium CPU */
201 #ifdef CONFIG_X86_32
202 	if (ctxt->cr4)
203 		__write_cr4(ctxt->cr4);
204 #else
205 /* CONFIG X86_64 */
206 	wrmsrl(MSR_EFER, ctxt->efer);
207 	write_cr8(ctxt->cr8);
208 	__write_cr4(ctxt->cr4);
209 #endif
210 	write_cr3(ctxt->cr3);
211 	write_cr2(ctxt->cr2);
212 	write_cr0(ctxt->cr0);
213 
214 	/*
215 	 * now restore the descriptor tables to their proper values
216 	 * ltr is done i fix_processor_context().
217 	 */
218 #ifdef CONFIG_X86_32
219 	load_idt(&ctxt->idt);
220 #else
221 /* CONFIG_X86_64 */
222 	load_idt((const struct desc_ptr *)&ctxt->idt_limit);
223 #endif
224 
225 	/*
226 	 * segment registers
227 	 */
228 #ifdef CONFIG_X86_32
229 	loadsegment(es, ctxt->es);
230 	loadsegment(fs, ctxt->fs);
231 	loadsegment(gs, ctxt->gs);
232 	loadsegment(ss, ctxt->ss);
233 
234 	/*
235 	 * sysenter MSRs
236 	 */
237 	if (boot_cpu_has(X86_FEATURE_SEP))
238 		enable_sep_cpu();
239 #else
240 /* CONFIG_X86_64 */
241 	asm volatile ("movw %0, %%ds" :: "r" (ctxt->ds));
242 	asm volatile ("movw %0, %%es" :: "r" (ctxt->es));
243 	asm volatile ("movw %0, %%fs" :: "r" (ctxt->fs));
244 	load_gs_index(ctxt->gs);
245 	asm volatile ("movw %0, %%ss" :: "r" (ctxt->ss));
246 
247 	wrmsrl(MSR_FS_BASE, ctxt->fs_base);
248 	wrmsrl(MSR_GS_BASE, ctxt->gs_base);
249 	wrmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
250 #endif
251 
252 	fix_processor_context();
253 
254 	do_fpu_end();
255 	x86_platform.restore_sched_clock_state();
256 	mtrr_bp_restore();
257 	perf_restore_debug_store();
258 	msr_restore_context(ctxt);
259 }
260 
261 /* Needed by apm.c */
262 void notrace restore_processor_state(void)
263 {
264 	__restore_processor_state(&saved_context);
265 }
266 #ifdef CONFIG_X86_32
267 EXPORT_SYMBOL(restore_processor_state);
268 #endif
269 
270 #if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU)
271 static void resume_play_dead(void)
272 {
273 	play_dead_common();
274 	tboot_shutdown(TB_SHUTDOWN_WFS);
275 	hlt_play_dead();
276 }
277 
278 int hibernate_resume_nonboot_cpu_disable(void)
279 {
280 	void (*play_dead)(void) = smp_ops.play_dead;
281 	int ret;
282 
283 	/*
284 	 * Ensure that MONITOR/MWAIT will not be used in the "play dead" loop
285 	 * during hibernate image restoration, because it is likely that the
286 	 * monitored address will be actually written to at that time and then
287 	 * the "dead" CPU will attempt to execute instructions again, but the
288 	 * address in its instruction pointer may not be possible to resolve
289 	 * any more at that point (the page tables used by it previously may
290 	 * have been overwritten by hibernate image data).
291 	 */
292 	smp_ops.play_dead = resume_play_dead;
293 	ret = disable_nonboot_cpus();
294 	smp_ops.play_dead = play_dead;
295 	return ret;
296 }
297 #endif
298 
299 /*
300  * When bsp_check() is called in hibernate and suspend, cpu hotplug
301  * is disabled already. So it's unnessary to handle race condition between
302  * cpumask query and cpu hotplug.
303  */
304 static int bsp_check(void)
305 {
306 	if (cpumask_first(cpu_online_mask) != 0) {
307 		pr_warn("CPU0 is offline.\n");
308 		return -ENODEV;
309 	}
310 
311 	return 0;
312 }
313 
314 static int bsp_pm_callback(struct notifier_block *nb, unsigned long action,
315 			   void *ptr)
316 {
317 	int ret = 0;
318 
319 	switch (action) {
320 	case PM_SUSPEND_PREPARE:
321 	case PM_HIBERNATION_PREPARE:
322 		ret = bsp_check();
323 		break;
324 #ifdef CONFIG_DEBUG_HOTPLUG_CPU0
325 	case PM_RESTORE_PREPARE:
326 		/*
327 		 * When system resumes from hibernation, online CPU0 because
328 		 * 1. it's required for resume and
329 		 * 2. the CPU was online before hibernation
330 		 */
331 		if (!cpu_online(0))
332 			_debug_hotplug_cpu(0, 1);
333 		break;
334 	case PM_POST_RESTORE:
335 		/*
336 		 * When a resume really happens, this code won't be called.
337 		 *
338 		 * This code is called only when user space hibernation software
339 		 * prepares for snapshot device during boot time. So we just
340 		 * call _debug_hotplug_cpu() to restore to CPU0's state prior to
341 		 * preparing the snapshot device.
342 		 *
343 		 * This works for normal boot case in our CPU0 hotplug debug
344 		 * mode, i.e. CPU0 is offline and user mode hibernation
345 		 * software initializes during boot time.
346 		 *
347 		 * If CPU0 is online and user application accesses snapshot
348 		 * device after boot time, this will offline CPU0 and user may
349 		 * see different CPU0 state before and after accessing
350 		 * the snapshot device. But hopefully this is not a case when
351 		 * user debugging CPU0 hotplug. Even if users hit this case,
352 		 * they can easily online CPU0 back.
353 		 *
354 		 * To simplify this debug code, we only consider normal boot
355 		 * case. Otherwise we need to remember CPU0's state and restore
356 		 * to that state and resolve racy conditions etc.
357 		 */
358 		_debug_hotplug_cpu(0, 0);
359 		break;
360 #endif
361 	default:
362 		break;
363 	}
364 	return notifier_from_errno(ret);
365 }
366 
367 static int __init bsp_pm_check_init(void)
368 {
369 	/*
370 	 * Set this bsp_pm_callback as lower priority than
371 	 * cpu_hotplug_pm_callback. So cpu_hotplug_pm_callback will be called
372 	 * earlier to disable cpu hotplug before bsp online check.
373 	 */
374 	pm_notifier(bsp_pm_callback, -INT_MAX);
375 	return 0;
376 }
377 
378 core_initcall(bsp_pm_check_init);
379 
380 static int msr_init_context(const u32 *msr_id, const int total_num)
381 {
382 	int i = 0;
383 	struct saved_msr *msr_array;
384 
385 	if (saved_context.saved_msrs.array || saved_context.saved_msrs.num > 0) {
386 		pr_err("x86/pm: MSR quirk already applied, please check your DMI match table.\n");
387 		return -EINVAL;
388 	}
389 
390 	msr_array = kmalloc_array(total_num, sizeof(struct saved_msr), GFP_KERNEL);
391 	if (!msr_array) {
392 		pr_err("x86/pm: Can not allocate memory to save/restore MSRs during suspend.\n");
393 		return -ENOMEM;
394 	}
395 
396 	for (i = 0; i < total_num; i++) {
397 		msr_array[i].info.msr_no	= msr_id[i];
398 		msr_array[i].valid		= false;
399 		msr_array[i].info.reg.q		= 0;
400 	}
401 	saved_context.saved_msrs.num	= total_num;
402 	saved_context.saved_msrs.array	= msr_array;
403 
404 	return 0;
405 }
406 
407 /*
408  * The following section is a quirk framework for problematic BIOSen:
409  * Sometimes MSRs are modified by the BIOSen after suspended to
410  * RAM, this might cause unexpected behavior after wakeup.
411  * Thus we save/restore these specified MSRs across suspend/resume
412  * in order to work around it.
413  *
414  * For any further problematic BIOSen/platforms,
415  * please add your own function similar to msr_initialize_bdw.
416  */
417 static int msr_initialize_bdw(const struct dmi_system_id *d)
418 {
419 	/* Add any extra MSR ids into this array. */
420 	u32 bdw_msr_id[] = { MSR_IA32_THERM_CONTROL };
421 
422 	pr_info("x86/pm: %s detected, MSR saving is needed during suspending.\n", d->ident);
423 	return msr_init_context(bdw_msr_id, ARRAY_SIZE(bdw_msr_id));
424 }
425 
426 static struct dmi_system_id msr_save_dmi_table[] = {
427 	{
428 	 .callback = msr_initialize_bdw,
429 	 .ident = "BROADWELL BDX_EP",
430 	 .matches = {
431 		DMI_MATCH(DMI_PRODUCT_NAME, "GRANTLEY"),
432 		DMI_MATCH(DMI_PRODUCT_VERSION, "E63448-400"),
433 		},
434 	},
435 	{}
436 };
437 
438 static int pm_check_save_msr(void)
439 {
440 	dmi_check_system(msr_save_dmi_table);
441 	return 0;
442 }
443 
444 device_initcall(pm_check_save_msr);
445