1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License version 2 as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright (C) 2013 ARM Limited 12 * 13 * Author: Will Deacon <will.deacon@arm.com> 14 */ 15 16 #define pr_fmt(fmt) "psci: " fmt 17 18 #include <linux/init.h> 19 #include <linux/of.h> 20 #include <linux/smp.h> 21 #include <linux/reboot.h> 22 #include <linux/pm.h> 23 #include <linux/delay.h> 24 #include <linux/slab.h> 25 #include <uapi/linux/psci.h> 26 27 #include <asm/compiler.h> 28 #include <asm/cputype.h> 29 #include <asm/cpu_ops.h> 30 #include <asm/errno.h> 31 #include <asm/psci.h> 32 #include <asm/smp_plat.h> 33 #include <asm/suspend.h> 34 #include <asm/system_misc.h> 35 36 #define PSCI_POWER_STATE_TYPE_STANDBY 0 37 #define PSCI_POWER_STATE_TYPE_POWER_DOWN 1 38 39 static bool psci_power_state_loses_context(u32 state) 40 { 41 return state & PSCI_0_2_POWER_STATE_TYPE_MASK; 42 } 43 44 static bool psci_power_state_is_valid(u32 state) 45 { 46 const u32 valid_mask = PSCI_0_2_POWER_STATE_ID_MASK | 47 PSCI_0_2_POWER_STATE_TYPE_MASK | 48 PSCI_0_2_POWER_STATE_AFFL_MASK; 49 50 return !(state & ~valid_mask); 51 } 52 53 /* 54 * The CPU any Trusted OS is resident on. The trusted OS may reject CPU_OFF 55 * calls to its resident CPU, so we must avoid issuing those. We never migrate 56 * a Trusted OS even if it claims to be capable of migration -- doing so will 57 * require cooperation with a Trusted OS driver. 58 */ 59 static int resident_cpu = -1; 60 61 struct psci_operations { 62 int (*cpu_suspend)(u32 state, unsigned long entry_point); 63 int (*cpu_off)(u32 state); 64 int (*cpu_on)(unsigned long cpuid, unsigned long entry_point); 65 int (*migrate)(unsigned long cpuid); 66 int (*affinity_info)(unsigned long target_affinity, 67 unsigned long lowest_affinity_level); 68 int (*migrate_info_type)(void); 69 }; 70 71 static struct psci_operations psci_ops; 72 73 typedef unsigned long (psci_fn)(unsigned long, unsigned long, 74 unsigned long, unsigned long); 75 asmlinkage psci_fn __invoke_psci_fn_hvc; 76 asmlinkage psci_fn __invoke_psci_fn_smc; 77 static psci_fn *invoke_psci_fn; 78 79 enum psci_function { 80 PSCI_FN_CPU_SUSPEND, 81 PSCI_FN_CPU_ON, 82 PSCI_FN_CPU_OFF, 83 PSCI_FN_MIGRATE, 84 PSCI_FN_MAX, 85 }; 86 87 static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state); 88 89 static u32 psci_function_id[PSCI_FN_MAX]; 90 91 static int psci_to_linux_errno(int errno) 92 { 93 switch (errno) { 94 case PSCI_RET_SUCCESS: 95 return 0; 96 case PSCI_RET_NOT_SUPPORTED: 97 return -EOPNOTSUPP; 98 case PSCI_RET_INVALID_PARAMS: 99 return -EINVAL; 100 case PSCI_RET_DENIED: 101 return -EPERM; 102 }; 103 104 return -EINVAL; 105 } 106 107 static u32 psci_get_version(void) 108 { 109 return invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); 110 } 111 112 static int psci_cpu_suspend(u32 state, unsigned long entry_point) 113 { 114 int err; 115 u32 fn; 116 117 fn = psci_function_id[PSCI_FN_CPU_SUSPEND]; 118 err = invoke_psci_fn(fn, state, entry_point, 0); 119 return psci_to_linux_errno(err); 120 } 121 122 static int psci_cpu_off(u32 state) 123 { 124 int err; 125 u32 fn; 126 127 fn = psci_function_id[PSCI_FN_CPU_OFF]; 128 err = invoke_psci_fn(fn, state, 0, 0); 129 return psci_to_linux_errno(err); 130 } 131 132 static int psci_cpu_on(unsigned long cpuid, unsigned long entry_point) 133 { 134 int err; 135 u32 fn; 136 137 fn = psci_function_id[PSCI_FN_CPU_ON]; 138 err = invoke_psci_fn(fn, cpuid, entry_point, 0); 139 return psci_to_linux_errno(err); 140 } 141 142 static int psci_migrate(unsigned long cpuid) 143 { 144 int err; 145 u32 fn; 146 147 fn = psci_function_id[PSCI_FN_MIGRATE]; 148 err = invoke_psci_fn(fn, cpuid, 0, 0); 149 return psci_to_linux_errno(err); 150 } 151 152 static int psci_affinity_info(unsigned long target_affinity, 153 unsigned long lowest_affinity_level) 154 { 155 return invoke_psci_fn(PSCI_0_2_FN64_AFFINITY_INFO, target_affinity, 156 lowest_affinity_level, 0); 157 } 158 159 static int psci_migrate_info_type(void) 160 { 161 return invoke_psci_fn(PSCI_0_2_FN_MIGRATE_INFO_TYPE, 0, 0, 0); 162 } 163 164 static unsigned long psci_migrate_info_up_cpu(void) 165 { 166 return invoke_psci_fn(PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU, 0, 0, 0); 167 } 168 169 static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) 170 { 171 int i, ret, count = 0; 172 u32 *psci_states; 173 struct device_node *state_node, *cpu_node; 174 175 cpu_node = of_get_cpu_node(cpu, NULL); 176 if (!cpu_node) 177 return -ENODEV; 178 179 /* 180 * If the PSCI cpu_suspend function hook has not been initialized 181 * idle states must not be enabled, so bail out 182 */ 183 if (!psci_ops.cpu_suspend) 184 return -EOPNOTSUPP; 185 186 /* Count idle states */ 187 while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states", 188 count))) { 189 count++; 190 of_node_put(state_node); 191 } 192 193 if (!count) 194 return -ENODEV; 195 196 psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL); 197 if (!psci_states) 198 return -ENOMEM; 199 200 for (i = 0; i < count; i++) { 201 u32 state; 202 203 state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i); 204 205 ret = of_property_read_u32(state_node, 206 "arm,psci-suspend-param", 207 &state); 208 if (ret) { 209 pr_warn(" * %s missing arm,psci-suspend-param property\n", 210 state_node->full_name); 211 of_node_put(state_node); 212 goto free_mem; 213 } 214 215 of_node_put(state_node); 216 pr_debug("psci-power-state %#x index %d\n", state, i); 217 if (!psci_power_state_is_valid(state)) { 218 pr_warn("Invalid PSCI power state %#x\n", state); 219 ret = -EINVAL; 220 goto free_mem; 221 } 222 psci_states[i] = state; 223 } 224 /* Idle states parsed correctly, initialize per-cpu pointer */ 225 per_cpu(psci_power_state, cpu) = psci_states; 226 return 0; 227 228 free_mem: 229 kfree(psci_states); 230 return ret; 231 } 232 233 static int get_set_conduit_method(struct device_node *np) 234 { 235 const char *method; 236 237 pr_info("probing for conduit method from DT.\n"); 238 239 if (of_property_read_string(np, "method", &method)) { 240 pr_warn("missing \"method\" property\n"); 241 return -ENXIO; 242 } 243 244 if (!strcmp("hvc", method)) { 245 invoke_psci_fn = __invoke_psci_fn_hvc; 246 } else if (!strcmp("smc", method)) { 247 invoke_psci_fn = __invoke_psci_fn_smc; 248 } else { 249 pr_warn("invalid \"method\" property: %s\n", method); 250 return -EINVAL; 251 } 252 return 0; 253 } 254 255 static void psci_sys_reset(enum reboot_mode reboot_mode, const char *cmd) 256 { 257 invoke_psci_fn(PSCI_0_2_FN_SYSTEM_RESET, 0, 0, 0); 258 } 259 260 static void psci_sys_poweroff(void) 261 { 262 invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0); 263 } 264 265 /* 266 * Detect the presence of a resident Trusted OS which may cause CPU_OFF to 267 * return DENIED (which would be fatal). 268 */ 269 static void __init psci_init_migrate(void) 270 { 271 unsigned long cpuid; 272 int type, cpu; 273 274 type = psci_ops.migrate_info_type(); 275 276 if (type == PSCI_0_2_TOS_MP) { 277 pr_info("Trusted OS migration not required\n"); 278 return; 279 } 280 281 if (type == PSCI_RET_NOT_SUPPORTED) { 282 pr_info("MIGRATE_INFO_TYPE not supported.\n"); 283 return; 284 } 285 286 if (type != PSCI_0_2_TOS_UP_MIGRATE && 287 type != PSCI_0_2_TOS_UP_NO_MIGRATE) { 288 pr_err("MIGRATE_INFO_TYPE returned unknown type (%d)\n", type); 289 return; 290 } 291 292 cpuid = psci_migrate_info_up_cpu(); 293 if (cpuid & ~MPIDR_HWID_BITMASK) { 294 pr_warn("MIGRATE_INFO_UP_CPU reported invalid physical ID (0x%lx)\n", 295 cpuid); 296 return; 297 } 298 299 cpu = get_logical_index(cpuid); 300 resident_cpu = cpu >= 0 ? cpu : -1; 301 302 pr_info("Trusted OS resident on physical CPU 0x%lx\n", cpuid); 303 } 304 305 static void __init psci_0_2_set_functions(void) 306 { 307 pr_info("Using standard PSCI v0.2 function IDs\n"); 308 psci_function_id[PSCI_FN_CPU_SUSPEND] = PSCI_0_2_FN64_CPU_SUSPEND; 309 psci_ops.cpu_suspend = psci_cpu_suspend; 310 311 psci_function_id[PSCI_FN_CPU_OFF] = PSCI_0_2_FN_CPU_OFF; 312 psci_ops.cpu_off = psci_cpu_off; 313 314 psci_function_id[PSCI_FN_CPU_ON] = PSCI_0_2_FN64_CPU_ON; 315 psci_ops.cpu_on = psci_cpu_on; 316 317 psci_function_id[PSCI_FN_MIGRATE] = PSCI_0_2_FN64_MIGRATE; 318 psci_ops.migrate = psci_migrate; 319 320 psci_ops.affinity_info = psci_affinity_info; 321 322 psci_ops.migrate_info_type = psci_migrate_info_type; 323 324 arm_pm_restart = psci_sys_reset; 325 326 pm_power_off = psci_sys_poweroff; 327 } 328 329 /* 330 * Probe function for PSCI firmware versions >= 0.2 331 */ 332 static int __init psci_probe(void) 333 { 334 u32 ver = psci_get_version(); 335 336 pr_info("PSCIv%d.%d detected in firmware.\n", 337 PSCI_VERSION_MAJOR(ver), 338 PSCI_VERSION_MINOR(ver)); 339 340 if (PSCI_VERSION_MAJOR(ver) == 0 && PSCI_VERSION_MINOR(ver) < 2) { 341 pr_err("Conflicting PSCI version detected.\n"); 342 return -EINVAL; 343 } 344 345 psci_0_2_set_functions(); 346 347 psci_init_migrate(); 348 349 return 0; 350 } 351 352 typedef int (*psci_initcall_t)(const struct device_node *); 353 354 /* 355 * PSCI init function for PSCI versions >=0.2 356 * 357 * Probe based on PSCI PSCI_VERSION function 358 */ 359 static int __init psci_0_2_init(struct device_node *np) 360 { 361 int err; 362 363 err = get_set_conduit_method(np); 364 365 if (err) 366 goto out_put_node; 367 /* 368 * Starting with v0.2, the PSCI specification introduced a call 369 * (PSCI_VERSION) that allows probing the firmware version, so 370 * that PSCI function IDs and version specific initialization 371 * can be carried out according to the specific version reported 372 * by firmware 373 */ 374 err = psci_probe(); 375 376 out_put_node: 377 of_node_put(np); 378 return err; 379 } 380 381 /* 382 * PSCI < v0.2 get PSCI Function IDs via DT. 383 */ 384 static int __init psci_0_1_init(struct device_node *np) 385 { 386 u32 id; 387 int err; 388 389 err = get_set_conduit_method(np); 390 391 if (err) 392 goto out_put_node; 393 394 pr_info("Using PSCI v0.1 Function IDs from DT\n"); 395 396 if (!of_property_read_u32(np, "cpu_suspend", &id)) { 397 psci_function_id[PSCI_FN_CPU_SUSPEND] = id; 398 psci_ops.cpu_suspend = psci_cpu_suspend; 399 } 400 401 if (!of_property_read_u32(np, "cpu_off", &id)) { 402 psci_function_id[PSCI_FN_CPU_OFF] = id; 403 psci_ops.cpu_off = psci_cpu_off; 404 } 405 406 if (!of_property_read_u32(np, "cpu_on", &id)) { 407 psci_function_id[PSCI_FN_CPU_ON] = id; 408 psci_ops.cpu_on = psci_cpu_on; 409 } 410 411 if (!of_property_read_u32(np, "migrate", &id)) { 412 psci_function_id[PSCI_FN_MIGRATE] = id; 413 psci_ops.migrate = psci_migrate; 414 } 415 416 out_put_node: 417 of_node_put(np); 418 return err; 419 } 420 421 static const struct of_device_id psci_of_match[] __initconst = { 422 { .compatible = "arm,psci", .data = psci_0_1_init}, 423 { .compatible = "arm,psci-0.2", .data = psci_0_2_init}, 424 {}, 425 }; 426 427 int __init psci_dt_init(void) 428 { 429 struct device_node *np; 430 const struct of_device_id *matched_np; 431 psci_initcall_t init_fn; 432 433 np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np); 434 435 if (!np) 436 return -ENODEV; 437 438 init_fn = (psci_initcall_t)matched_np->data; 439 return init_fn(np); 440 } 441 442 #ifdef CONFIG_ACPI 443 /* 444 * We use PSCI 0.2+ when ACPI is deployed on ARM64 and it's 445 * explicitly clarified in SBBR 446 */ 447 int __init psci_acpi_init(void) 448 { 449 if (!acpi_psci_present()) { 450 pr_info("is not implemented in ACPI.\n"); 451 return -EOPNOTSUPP; 452 } 453 454 pr_info("probing for conduit method from ACPI.\n"); 455 456 if (acpi_psci_use_hvc()) 457 invoke_psci_fn = __invoke_psci_fn_hvc; 458 else 459 invoke_psci_fn = __invoke_psci_fn_smc; 460 461 return psci_probe(); 462 } 463 #endif 464 465 #ifdef CONFIG_SMP 466 467 static int __init cpu_psci_cpu_init(unsigned int cpu) 468 { 469 return 0; 470 } 471 472 static int __init cpu_psci_cpu_prepare(unsigned int cpu) 473 { 474 if (!psci_ops.cpu_on) { 475 pr_err("no cpu_on method, not booting CPU%d\n", cpu); 476 return -ENODEV; 477 } 478 479 return 0; 480 } 481 482 static int cpu_psci_cpu_boot(unsigned int cpu) 483 { 484 int err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa(secondary_entry)); 485 if (err) 486 pr_err("failed to boot CPU%d (%d)\n", cpu, err); 487 488 return err; 489 } 490 491 #ifdef CONFIG_HOTPLUG_CPU 492 static bool psci_tos_resident_on(int cpu) 493 { 494 return cpu == resident_cpu; 495 } 496 497 static int cpu_psci_cpu_disable(unsigned int cpu) 498 { 499 /* Fail early if we don't have CPU_OFF support */ 500 if (!psci_ops.cpu_off) 501 return -EOPNOTSUPP; 502 503 /* Trusted OS will deny CPU_OFF */ 504 if (psci_tos_resident_on(cpu)) 505 return -EPERM; 506 507 return 0; 508 } 509 510 static void cpu_psci_cpu_die(unsigned int cpu) 511 { 512 int ret; 513 /* 514 * There are no known implementations of PSCI actually using the 515 * power state field, pass a sensible default for now. 516 */ 517 u32 state = PSCI_POWER_STATE_TYPE_POWER_DOWN << 518 PSCI_0_2_POWER_STATE_TYPE_SHIFT; 519 520 ret = psci_ops.cpu_off(state); 521 522 pr_crit("unable to power off CPU%u (%d)\n", cpu, ret); 523 } 524 525 static int cpu_psci_cpu_kill(unsigned int cpu) 526 { 527 int err, i; 528 529 if (!psci_ops.affinity_info) 530 return 0; 531 /* 532 * cpu_kill could race with cpu_die and we can 533 * potentially end up declaring this cpu undead 534 * while it is dying. So, try again a few times. 535 */ 536 537 for (i = 0; i < 10; i++) { 538 err = psci_ops.affinity_info(cpu_logical_map(cpu), 0); 539 if (err == PSCI_0_2_AFFINITY_LEVEL_OFF) { 540 pr_info("CPU%d killed.\n", cpu); 541 return 0; 542 } 543 544 msleep(10); 545 pr_info("Retrying again to check for CPU kill\n"); 546 } 547 548 pr_warn("CPU%d may not have shut down cleanly (AFFINITY_INFO reports %d)\n", 549 cpu, err); 550 return -ETIMEDOUT; 551 } 552 #endif 553 #endif 554 555 static int psci_suspend_finisher(unsigned long index) 556 { 557 u32 *state = __this_cpu_read(psci_power_state); 558 559 return psci_ops.cpu_suspend(state[index - 1], 560 virt_to_phys(cpu_resume)); 561 } 562 563 static int __maybe_unused cpu_psci_cpu_suspend(unsigned long index) 564 { 565 int ret; 566 u32 *state = __this_cpu_read(psci_power_state); 567 /* 568 * idle state index 0 corresponds to wfi, should never be called 569 * from the cpu_suspend operations 570 */ 571 if (WARN_ON_ONCE(!index)) 572 return -EINVAL; 573 574 if (!psci_power_state_loses_context(state[index - 1])) 575 ret = psci_ops.cpu_suspend(state[index - 1], 0); 576 else 577 ret = cpu_suspend(index, psci_suspend_finisher); 578 579 return ret; 580 } 581 582 const struct cpu_operations cpu_psci_ops = { 583 .name = "psci", 584 #ifdef CONFIG_CPU_IDLE 585 .cpu_init_idle = cpu_psci_cpu_init_idle, 586 .cpu_suspend = cpu_psci_cpu_suspend, 587 #endif 588 #ifdef CONFIG_SMP 589 .cpu_init = cpu_psci_cpu_init, 590 .cpu_prepare = cpu_psci_cpu_prepare, 591 .cpu_boot = cpu_psci_cpu_boot, 592 #ifdef CONFIG_HOTPLUG_CPU 593 .cpu_disable = cpu_psci_cpu_disable, 594 .cpu_die = cpu_psci_cpu_die, 595 .cpu_kill = cpu_psci_cpu_kill, 596 #endif 597 #endif 598 }; 599 600