1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/power/suspend.c - Suspend to RAM and standby functionality. 4 * 5 * Copyright (c) 2003 Patrick Mochel 6 * Copyright (c) 2003 Open Source Development Lab 7 * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. 8 */ 9 10 #define pr_fmt(fmt) "PM: " fmt 11 12 #include <linux/string.h> 13 #include <linux/delay.h> 14 #include <linux/errno.h> 15 #include <linux/init.h> 16 #include <linux/console.h> 17 #include <linux/cpu.h> 18 #include <linux/cpuidle.h> 19 #include <linux/gfp.h> 20 #include <linux/io.h> 21 #include <linux/kernel.h> 22 #include <linux/list.h> 23 #include <linux/mm.h> 24 #include <linux/slab.h> 25 #include <linux/export.h> 26 #include <linux/suspend.h> 27 #include <linux/syscore_ops.h> 28 #include <linux/swait.h> 29 #include <linux/ftrace.h> 30 #include <trace/events/power.h> 31 #include <linux/compiler.h> 32 #include <linux/moduleparam.h> 33 34 #include "power.h" 35 36 const char * const pm_labels[] = { 37 [PM_SUSPEND_TO_IDLE] = "freeze", 38 [PM_SUSPEND_STANDBY] = "standby", 39 [PM_SUSPEND_MEM] = "mem", 40 }; 41 const char *pm_states[PM_SUSPEND_MAX]; 42 static const char * const mem_sleep_labels[] = { 43 [PM_SUSPEND_TO_IDLE] = "s2idle", 44 [PM_SUSPEND_STANDBY] = "shallow", 45 [PM_SUSPEND_MEM] = "deep", 46 }; 47 const char *mem_sleep_states[PM_SUSPEND_MAX]; 48 49 suspend_state_t mem_sleep_current = PM_SUSPEND_TO_IDLE; 50 suspend_state_t mem_sleep_default = PM_SUSPEND_MAX; 51 suspend_state_t pm_suspend_target_state; 52 EXPORT_SYMBOL_GPL(pm_suspend_target_state); 53 54 unsigned int pm_suspend_global_flags; 55 EXPORT_SYMBOL_GPL(pm_suspend_global_flags); 56 57 static const struct platform_suspend_ops *suspend_ops; 58 static const struct platform_s2idle_ops *s2idle_ops; 59 static DECLARE_SWAIT_QUEUE_HEAD(s2idle_wait_head); 60 61 enum s2idle_states __read_mostly s2idle_state; 62 static DEFINE_RAW_SPINLOCK(s2idle_lock); 63 64 /** 65 * pm_suspend_default_s2idle - Check if suspend-to-idle is the default suspend. 66 * 67 * Return 'true' if suspend-to-idle has been selected as the default system 68 * suspend method. 69 */ 70 bool pm_suspend_default_s2idle(void) 71 { 72 return mem_sleep_current == PM_SUSPEND_TO_IDLE; 73 } 74 EXPORT_SYMBOL_GPL(pm_suspend_default_s2idle); 75 76 void s2idle_set_ops(const struct platform_s2idle_ops *ops) 77 { 78 lock_system_sleep(); 79 s2idle_ops = ops; 80 unlock_system_sleep(); 81 } 82 83 static void s2idle_begin(void) 84 { 85 s2idle_state = S2IDLE_STATE_NONE; 86 } 87 88 static void s2idle_enter(void) 89 { 90 trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, true); 91 92 raw_spin_lock_irq(&s2idle_lock); 93 if (pm_wakeup_pending()) 94 goto out; 95 96 s2idle_state = S2IDLE_STATE_ENTER; 97 raw_spin_unlock_irq(&s2idle_lock); 98 99 get_online_cpus(); 100 cpuidle_resume(); 101 102 /* Push all the CPUs into the idle loop. */ 103 wake_up_all_idle_cpus(); 104 /* Make the current CPU wait so it can enter the idle loop too. */ 105 swait_event_exclusive(s2idle_wait_head, 106 s2idle_state == S2IDLE_STATE_WAKE); 107 108 cpuidle_pause(); 109 put_online_cpus(); 110 111 raw_spin_lock_irq(&s2idle_lock); 112 113 out: 114 s2idle_state = S2IDLE_STATE_NONE; 115 raw_spin_unlock_irq(&s2idle_lock); 116 117 trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, false); 118 } 119 120 static void s2idle_loop(void) 121 { 122 pm_pr_dbg("suspend-to-idle\n"); 123 124 /* 125 * Suspend-to-idle equals: 126 * frozen processes + suspended devices + idle processors. 127 * Thus s2idle_enter() should be called right after all devices have 128 * been suspended. 129 * 130 * Wakeups during the noirq suspend of devices may be spurious, so try 131 * to avoid them upfront. 132 */ 133 for (;;) { 134 if (s2idle_ops && s2idle_ops->wake) 135 s2idle_ops->wake(); 136 137 if (pm_wakeup_pending()) 138 break; 139 140 pm_wakeup_clear(false); 141 142 s2idle_enter(); 143 } 144 145 pm_pr_dbg("resume from suspend-to-idle\n"); 146 } 147 148 void s2idle_wake(void) 149 { 150 unsigned long flags; 151 152 raw_spin_lock_irqsave(&s2idle_lock, flags); 153 if (s2idle_state > S2IDLE_STATE_NONE) { 154 s2idle_state = S2IDLE_STATE_WAKE; 155 swake_up_one(&s2idle_wait_head); 156 } 157 raw_spin_unlock_irqrestore(&s2idle_lock, flags); 158 } 159 EXPORT_SYMBOL_GPL(s2idle_wake); 160 161 static bool valid_state(suspend_state_t state) 162 { 163 /* 164 * PM_SUSPEND_STANDBY and PM_SUSPEND_MEM states need low level 165 * support and need to be valid to the low level 166 * implementation, no valid callback implies that none are valid. 167 */ 168 return suspend_ops && suspend_ops->valid && suspend_ops->valid(state); 169 } 170 171 void __init pm_states_init(void) 172 { 173 /* "mem" and "freeze" are always present in /sys/power/state. */ 174 pm_states[PM_SUSPEND_MEM] = pm_labels[PM_SUSPEND_MEM]; 175 pm_states[PM_SUSPEND_TO_IDLE] = pm_labels[PM_SUSPEND_TO_IDLE]; 176 /* 177 * Suspend-to-idle should be supported even without any suspend_ops, 178 * initialize mem_sleep_states[] accordingly here. 179 */ 180 mem_sleep_states[PM_SUSPEND_TO_IDLE] = mem_sleep_labels[PM_SUSPEND_TO_IDLE]; 181 } 182 183 static int __init mem_sleep_default_setup(char *str) 184 { 185 suspend_state_t state; 186 187 for (state = PM_SUSPEND_TO_IDLE; state <= PM_SUSPEND_MEM; state++) 188 if (mem_sleep_labels[state] && 189 !strcmp(str, mem_sleep_labels[state])) { 190 mem_sleep_default = state; 191 break; 192 } 193 194 return 1; 195 } 196 __setup("mem_sleep_default=", mem_sleep_default_setup); 197 198 /** 199 * suspend_set_ops - Set the global suspend method table. 200 * @ops: Suspend operations to use. 201 */ 202 void suspend_set_ops(const struct platform_suspend_ops *ops) 203 { 204 lock_system_sleep(); 205 206 suspend_ops = ops; 207 208 if (valid_state(PM_SUSPEND_STANDBY)) { 209 mem_sleep_states[PM_SUSPEND_STANDBY] = mem_sleep_labels[PM_SUSPEND_STANDBY]; 210 pm_states[PM_SUSPEND_STANDBY] = pm_labels[PM_SUSPEND_STANDBY]; 211 if (mem_sleep_default == PM_SUSPEND_STANDBY) 212 mem_sleep_current = PM_SUSPEND_STANDBY; 213 } 214 if (valid_state(PM_SUSPEND_MEM)) { 215 mem_sleep_states[PM_SUSPEND_MEM] = mem_sleep_labels[PM_SUSPEND_MEM]; 216 if (mem_sleep_default >= PM_SUSPEND_MEM) 217 mem_sleep_current = PM_SUSPEND_MEM; 218 } 219 220 unlock_system_sleep(); 221 } 222 EXPORT_SYMBOL_GPL(suspend_set_ops); 223 224 /** 225 * suspend_valid_only_mem - Generic memory-only valid callback. 226 * 227 * Platform drivers that implement mem suspend only and only need to check for 228 * that in their .valid() callback can use this instead of rolling their own 229 * .valid() callback. 230 */ 231 int suspend_valid_only_mem(suspend_state_t state) 232 { 233 return state == PM_SUSPEND_MEM; 234 } 235 EXPORT_SYMBOL_GPL(suspend_valid_only_mem); 236 237 static bool sleep_state_supported(suspend_state_t state) 238 { 239 return state == PM_SUSPEND_TO_IDLE || (suspend_ops && suspend_ops->enter); 240 } 241 242 static int platform_suspend_prepare(suspend_state_t state) 243 { 244 return state != PM_SUSPEND_TO_IDLE && suspend_ops->prepare ? 245 suspend_ops->prepare() : 0; 246 } 247 248 static int platform_suspend_prepare_late(suspend_state_t state) 249 { 250 return state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->prepare ? 251 s2idle_ops->prepare() : 0; 252 } 253 254 static int platform_suspend_prepare_noirq(suspend_state_t state) 255 { 256 if (state == PM_SUSPEND_TO_IDLE) 257 return s2idle_ops && s2idle_ops->prepare_late ? 258 s2idle_ops->prepare_late() : 0; 259 260 return suspend_ops->prepare_late ? suspend_ops->prepare_late() : 0; 261 } 262 263 static void platform_resume_noirq(suspend_state_t state) 264 { 265 if (state == PM_SUSPEND_TO_IDLE) { 266 if (s2idle_ops && s2idle_ops->restore_early) 267 s2idle_ops->restore_early(); 268 } else if (suspend_ops->wake) { 269 suspend_ops->wake(); 270 } 271 } 272 273 static void platform_resume_early(suspend_state_t state) 274 { 275 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->restore) 276 s2idle_ops->restore(); 277 } 278 279 static void platform_resume_finish(suspend_state_t state) 280 { 281 if (state != PM_SUSPEND_TO_IDLE && suspend_ops->finish) 282 suspend_ops->finish(); 283 } 284 285 static int platform_suspend_begin(suspend_state_t state) 286 { 287 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->begin) 288 return s2idle_ops->begin(); 289 else if (suspend_ops && suspend_ops->begin) 290 return suspend_ops->begin(state); 291 else 292 return 0; 293 } 294 295 static void platform_resume_end(suspend_state_t state) 296 { 297 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->end) 298 s2idle_ops->end(); 299 else if (suspend_ops && suspend_ops->end) 300 suspend_ops->end(); 301 } 302 303 static void platform_recover(suspend_state_t state) 304 { 305 if (state != PM_SUSPEND_TO_IDLE && suspend_ops->recover) 306 suspend_ops->recover(); 307 } 308 309 static bool platform_suspend_again(suspend_state_t state) 310 { 311 return state != PM_SUSPEND_TO_IDLE && suspend_ops->suspend_again ? 312 suspend_ops->suspend_again() : false; 313 } 314 315 #ifdef CONFIG_PM_DEBUG 316 static unsigned int pm_test_delay = 5; 317 module_param(pm_test_delay, uint, 0644); 318 MODULE_PARM_DESC(pm_test_delay, 319 "Number of seconds to wait before resuming from suspend test"); 320 #endif 321 322 static int suspend_test(int level) 323 { 324 #ifdef CONFIG_PM_DEBUG 325 if (pm_test_level == level) { 326 pr_info("suspend debug: Waiting for %d second(s).\n", 327 pm_test_delay); 328 mdelay(pm_test_delay * 1000); 329 return 1; 330 } 331 #endif /* !CONFIG_PM_DEBUG */ 332 return 0; 333 } 334 335 /** 336 * suspend_prepare - Prepare for entering system sleep state. 337 * 338 * Common code run for every system sleep state that can be entered (except for 339 * hibernation). Run suspend notifiers, allocate the "suspend" console and 340 * freeze processes. 341 */ 342 static int suspend_prepare(suspend_state_t state) 343 { 344 int error, nr_calls = 0; 345 346 if (!sleep_state_supported(state)) 347 return -EPERM; 348 349 pm_prepare_console(); 350 351 error = __pm_notifier_call_chain(PM_SUSPEND_PREPARE, -1, &nr_calls); 352 if (error) { 353 nr_calls--; 354 goto Finish; 355 } 356 357 trace_suspend_resume(TPS("freeze_processes"), 0, true); 358 error = suspend_freeze_processes(); 359 trace_suspend_resume(TPS("freeze_processes"), 0, false); 360 if (!error) 361 return 0; 362 363 suspend_stats.failed_freeze++; 364 dpm_save_failed_step(SUSPEND_FREEZE); 365 Finish: 366 __pm_notifier_call_chain(PM_POST_SUSPEND, nr_calls, NULL); 367 pm_restore_console(); 368 return error; 369 } 370 371 /* default implementation */ 372 void __weak arch_suspend_disable_irqs(void) 373 { 374 local_irq_disable(); 375 } 376 377 /* default implementation */ 378 void __weak arch_suspend_enable_irqs(void) 379 { 380 local_irq_enable(); 381 } 382 383 /** 384 * suspend_enter - Make the system enter the given sleep state. 385 * @state: System sleep state to enter. 386 * @wakeup: Returns information that the sleep state should not be re-entered. 387 * 388 * This function should be called after devices have been suspended. 389 */ 390 static int suspend_enter(suspend_state_t state, bool *wakeup) 391 { 392 int error; 393 394 error = platform_suspend_prepare(state); 395 if (error) 396 goto Platform_finish; 397 398 error = dpm_suspend_late(PMSG_SUSPEND); 399 if (error) { 400 pr_err("late suspend of devices failed\n"); 401 goto Platform_finish; 402 } 403 error = platform_suspend_prepare_late(state); 404 if (error) 405 goto Devices_early_resume; 406 407 error = dpm_suspend_noirq(PMSG_SUSPEND); 408 if (error) { 409 pr_err("noirq suspend of devices failed\n"); 410 goto Platform_early_resume; 411 } 412 error = platform_suspend_prepare_noirq(state); 413 if (error) 414 goto Platform_wake; 415 416 if (suspend_test(TEST_PLATFORM)) 417 goto Platform_wake; 418 419 if (state == PM_SUSPEND_TO_IDLE) { 420 s2idle_loop(); 421 goto Platform_wake; 422 } 423 424 error = suspend_disable_secondary_cpus(); 425 if (error || suspend_test(TEST_CPUS)) 426 goto Enable_cpus; 427 428 arch_suspend_disable_irqs(); 429 BUG_ON(!irqs_disabled()); 430 431 system_state = SYSTEM_SUSPEND; 432 433 error = syscore_suspend(); 434 if (!error) { 435 *wakeup = pm_wakeup_pending(); 436 if (!(suspend_test(TEST_CORE) || *wakeup)) { 437 trace_suspend_resume(TPS("machine_suspend"), 438 state, true); 439 error = suspend_ops->enter(state); 440 trace_suspend_resume(TPS("machine_suspend"), 441 state, false); 442 } else if (*wakeup) { 443 error = -EBUSY; 444 } 445 syscore_resume(); 446 } 447 448 system_state = SYSTEM_RUNNING; 449 450 arch_suspend_enable_irqs(); 451 BUG_ON(irqs_disabled()); 452 453 Enable_cpus: 454 suspend_enable_secondary_cpus(); 455 456 Platform_wake: 457 platform_resume_noirq(state); 458 dpm_resume_noirq(PMSG_RESUME); 459 460 Platform_early_resume: 461 platform_resume_early(state); 462 463 Devices_early_resume: 464 dpm_resume_early(PMSG_RESUME); 465 466 Platform_finish: 467 platform_resume_finish(state); 468 return error; 469 } 470 471 /** 472 * suspend_devices_and_enter - Suspend devices and enter system sleep state. 473 * @state: System sleep state to enter. 474 */ 475 int suspend_devices_and_enter(suspend_state_t state) 476 { 477 int error; 478 bool wakeup = false; 479 480 if (!sleep_state_supported(state)) 481 return -ENOSYS; 482 483 pm_suspend_target_state = state; 484 485 if (state == PM_SUSPEND_TO_IDLE) 486 pm_set_suspend_no_platform(); 487 488 error = platform_suspend_begin(state); 489 if (error) 490 goto Close; 491 492 suspend_console(); 493 suspend_test_start(); 494 error = dpm_suspend_start(PMSG_SUSPEND); 495 if (error) { 496 pr_err("Some devices failed to suspend, or early wake event detected\n"); 497 goto Recover_platform; 498 } 499 suspend_test_finish("suspend devices"); 500 if (suspend_test(TEST_DEVICES)) 501 goto Recover_platform; 502 503 do { 504 error = suspend_enter(state, &wakeup); 505 } while (!error && !wakeup && platform_suspend_again(state)); 506 507 Resume_devices: 508 suspend_test_start(); 509 dpm_resume_end(PMSG_RESUME); 510 suspend_test_finish("resume devices"); 511 trace_suspend_resume(TPS("resume_console"), state, true); 512 resume_console(); 513 trace_suspend_resume(TPS("resume_console"), state, false); 514 515 Close: 516 platform_resume_end(state); 517 pm_suspend_target_state = PM_SUSPEND_ON; 518 return error; 519 520 Recover_platform: 521 platform_recover(state); 522 goto Resume_devices; 523 } 524 525 /** 526 * suspend_finish - Clean up before finishing the suspend sequence. 527 * 528 * Call platform code to clean up, restart processes, and free the console that 529 * we've allocated. This routine is not called for hibernation. 530 */ 531 static void suspend_finish(void) 532 { 533 suspend_thaw_processes(); 534 pm_notifier_call_chain(PM_POST_SUSPEND); 535 pm_restore_console(); 536 } 537 538 /** 539 * enter_state - Do common work needed to enter system sleep state. 540 * @state: System sleep state to enter. 541 * 542 * Make sure that no one else is trying to put the system into a sleep state. 543 * Fail if that's not the case. Otherwise, prepare for system suspend, make the 544 * system enter the given sleep state and clean up after wakeup. 545 */ 546 static int enter_state(suspend_state_t state) 547 { 548 int error; 549 550 trace_suspend_resume(TPS("suspend_enter"), state, true); 551 if (state == PM_SUSPEND_TO_IDLE) { 552 #ifdef CONFIG_PM_DEBUG 553 if (pm_test_level != TEST_NONE && pm_test_level <= TEST_CPUS) { 554 pr_warn("Unsupported test mode for suspend to idle, please choose none/freezer/devices/platform.\n"); 555 return -EAGAIN; 556 } 557 #endif 558 } else if (!valid_state(state)) { 559 return -EINVAL; 560 } 561 if (!mutex_trylock(&system_transition_mutex)) 562 return -EBUSY; 563 564 if (state == PM_SUSPEND_TO_IDLE) 565 s2idle_begin(); 566 567 if (sync_on_suspend_enabled) { 568 trace_suspend_resume(TPS("sync_filesystems"), 0, true); 569 ksys_sync_helper(); 570 trace_suspend_resume(TPS("sync_filesystems"), 0, false); 571 } 572 573 pm_pr_dbg("Preparing system for sleep (%s)\n", mem_sleep_labels[state]); 574 pm_suspend_clear_flags(); 575 error = suspend_prepare(state); 576 if (error) 577 goto Unlock; 578 579 if (suspend_test(TEST_FREEZER)) 580 goto Finish; 581 582 trace_suspend_resume(TPS("suspend_enter"), state, false); 583 pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]); 584 pm_restrict_gfp_mask(); 585 error = suspend_devices_and_enter(state); 586 pm_restore_gfp_mask(); 587 588 Finish: 589 events_check_enabled = false; 590 pm_pr_dbg("Finishing wakeup.\n"); 591 suspend_finish(); 592 Unlock: 593 mutex_unlock(&system_transition_mutex); 594 return error; 595 } 596 597 /** 598 * pm_suspend - Externally visible function for suspending the system. 599 * @state: System sleep state to enter. 600 * 601 * Check if the value of @state represents one of the supported states, 602 * execute enter_state() and update system suspend statistics. 603 */ 604 int pm_suspend(suspend_state_t state) 605 { 606 int error; 607 608 if (state <= PM_SUSPEND_ON || state >= PM_SUSPEND_MAX) 609 return -EINVAL; 610 611 pr_info("suspend entry (%s)\n", mem_sleep_labels[state]); 612 error = enter_state(state); 613 if (error) { 614 suspend_stats.fail++; 615 dpm_save_failed_errno(error); 616 } else { 617 suspend_stats.success++; 618 } 619 pr_info("suspend exit\n"); 620 return error; 621 } 622 EXPORT_SYMBOL(pm_suspend); 623