1 /* 2 * SMP initialisation and IPI support 3 * Based on arch/arm/kernel/smp.c 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include <linux/acpi.h> 21 #include <linux/delay.h> 22 #include <linux/init.h> 23 #include <linux/spinlock.h> 24 #include <linux/sched.h> 25 #include <linux/interrupt.h> 26 #include <linux/cache.h> 27 #include <linux/profile.h> 28 #include <linux/errno.h> 29 #include <linux/mm.h> 30 #include <linux/err.h> 31 #include <linux/cpu.h> 32 #include <linux/smp.h> 33 #include <linux/seq_file.h> 34 #include <linux/irq.h> 35 #include <linux/percpu.h> 36 #include <linux/clockchips.h> 37 #include <linux/completion.h> 38 #include <linux/of.h> 39 #include <linux/irq_work.h> 40 41 #include <asm/alternative.h> 42 #include <asm/atomic.h> 43 #include <asm/cacheflush.h> 44 #include <asm/cpu.h> 45 #include <asm/cputype.h> 46 #include <asm/cpu_ops.h> 47 #include <asm/mmu_context.h> 48 #include <asm/numa.h> 49 #include <asm/pgtable.h> 50 #include <asm/pgalloc.h> 51 #include <asm/processor.h> 52 #include <asm/smp_plat.h> 53 #include <asm/sections.h> 54 #include <asm/tlbflush.h> 55 #include <asm/ptrace.h> 56 #include <asm/virt.h> 57 58 #define CREATE_TRACE_POINTS 59 #include <trace/events/ipi.h> 60 61 /* 62 * as from 2.5, kernels no longer have an init_tasks structure 63 * so we need some other way of telling a new secondary core 64 * where to place its SVC stack 65 */ 66 struct secondary_data secondary_data; 67 /* Number of CPUs which aren't online, but looping in kernel text. */ 68 int cpus_stuck_in_kernel; 69 70 enum ipi_msg_type { 71 IPI_RESCHEDULE, 72 IPI_CALL_FUNC, 73 IPI_CPU_STOP, 74 IPI_TIMER, 75 IPI_IRQ_WORK, 76 IPI_WAKEUP 77 }; 78 79 #ifdef CONFIG_ARM64_VHE 80 81 /* Whether the boot CPU is running in HYP mode or not*/ 82 static bool boot_cpu_hyp_mode; 83 84 static inline void save_boot_cpu_run_el(void) 85 { 86 boot_cpu_hyp_mode = is_kernel_in_hyp_mode(); 87 } 88 89 static inline bool is_boot_cpu_in_hyp_mode(void) 90 { 91 return boot_cpu_hyp_mode; 92 } 93 94 /* 95 * Verify that a secondary CPU is running the kernel at the same 96 * EL as that of the boot CPU. 97 */ 98 void verify_cpu_run_el(void) 99 { 100 bool in_el2 = is_kernel_in_hyp_mode(); 101 bool boot_cpu_el2 = is_boot_cpu_in_hyp_mode(); 102 103 if (in_el2 ^ boot_cpu_el2) { 104 pr_crit("CPU%d: mismatched Exception Level(EL%d) with boot CPU(EL%d)\n", 105 smp_processor_id(), 106 in_el2 ? 2 : 1, 107 boot_cpu_el2 ? 2 : 1); 108 cpu_panic_kernel(); 109 } 110 } 111 112 #else 113 static inline void save_boot_cpu_run_el(void) {} 114 #endif 115 116 #ifdef CONFIG_HOTPLUG_CPU 117 static int op_cpu_kill(unsigned int cpu); 118 #else 119 static inline int op_cpu_kill(unsigned int cpu) 120 { 121 return -ENOSYS; 122 } 123 #endif 124 125 126 /* 127 * Boot a secondary CPU, and assign it the specified idle task. 128 * This also gives us the initial stack to use for this CPU. 129 */ 130 static int boot_secondary(unsigned int cpu, struct task_struct *idle) 131 { 132 if (cpu_ops[cpu]->cpu_boot) 133 return cpu_ops[cpu]->cpu_boot(cpu); 134 135 return -EOPNOTSUPP; 136 } 137 138 static DECLARE_COMPLETION(cpu_running); 139 140 int __cpu_up(unsigned int cpu, struct task_struct *idle) 141 { 142 int ret; 143 long status; 144 145 /* 146 * We need to tell the secondary core where to find its stack and the 147 * page tables. 148 */ 149 secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; 150 update_cpu_boot_status(CPU_MMU_OFF); 151 __flush_dcache_area(&secondary_data, sizeof(secondary_data)); 152 153 /* 154 * Now bring the CPU into our world. 155 */ 156 ret = boot_secondary(cpu, idle); 157 if (ret == 0) { 158 /* 159 * CPU was successfully started, wait for it to come online or 160 * time out. 161 */ 162 wait_for_completion_timeout(&cpu_running, 163 msecs_to_jiffies(1000)); 164 165 if (!cpu_online(cpu)) { 166 pr_crit("CPU%u: failed to come online\n", cpu); 167 ret = -EIO; 168 } 169 } else { 170 pr_err("CPU%u: failed to boot: %d\n", cpu, ret); 171 } 172 173 secondary_data.stack = NULL; 174 status = READ_ONCE(secondary_data.status); 175 if (ret && status) { 176 177 if (status == CPU_MMU_OFF) 178 status = READ_ONCE(__early_cpu_boot_status); 179 180 switch (status) { 181 default: 182 pr_err("CPU%u: failed in unknown state : 0x%lx\n", 183 cpu, status); 184 break; 185 case CPU_KILL_ME: 186 if (!op_cpu_kill(cpu)) { 187 pr_crit("CPU%u: died during early boot\n", cpu); 188 break; 189 } 190 /* Fall through */ 191 pr_crit("CPU%u: may not have shut down cleanly\n", cpu); 192 case CPU_STUCK_IN_KERNEL: 193 pr_crit("CPU%u: is stuck in kernel\n", cpu); 194 cpus_stuck_in_kernel++; 195 break; 196 case CPU_PANIC_KERNEL: 197 panic("CPU%u detected unsupported configuration\n", cpu); 198 } 199 } 200 201 return ret; 202 } 203 204 static void smp_store_cpu_info(unsigned int cpuid) 205 { 206 store_cpu_topology(cpuid); 207 numa_store_cpu_info(cpuid); 208 } 209 210 /* 211 * This is the secondary CPU boot entry. We're using this CPUs 212 * idle thread stack, but a set of temporary page tables. 213 */ 214 asmlinkage void secondary_start_kernel(void) 215 { 216 struct mm_struct *mm = &init_mm; 217 unsigned int cpu = smp_processor_id(); 218 219 /* 220 * All kernel threads share the same mm context; grab a 221 * reference and switch to it. 222 */ 223 atomic_inc(&mm->mm_count); 224 current->active_mm = mm; 225 226 set_my_cpu_offset(per_cpu_offset(smp_processor_id())); 227 228 /* 229 * TTBR0 is only used for the identity mapping at this stage. Make it 230 * point to zero page to avoid speculatively fetching new entries. 231 */ 232 cpu_uninstall_idmap(); 233 234 preempt_disable(); 235 trace_hardirqs_off(); 236 237 /* 238 * If the system has established the capabilities, make sure 239 * this CPU ticks all of those. If it doesn't, the CPU will 240 * fail to come online. 241 */ 242 verify_local_cpu_capabilities(); 243 244 if (cpu_ops[cpu]->cpu_postboot) 245 cpu_ops[cpu]->cpu_postboot(); 246 247 /* 248 * Log the CPU info before it is marked online and might get read. 249 */ 250 cpuinfo_store_cpu(); 251 252 /* 253 * Enable GIC and timers. 254 */ 255 notify_cpu_starting(cpu); 256 257 smp_store_cpu_info(cpu); 258 259 /* 260 * OK, now it's safe to let the boot CPU continue. Wait for 261 * the CPU migration code to notice that the CPU is online 262 * before we continue. 263 */ 264 pr_info("CPU%u: Booted secondary processor [%08x]\n", 265 cpu, read_cpuid_id()); 266 update_cpu_boot_status(CPU_BOOT_SUCCESS); 267 set_cpu_online(cpu, true); 268 complete(&cpu_running); 269 270 local_dbg_enable(); 271 local_irq_enable(); 272 local_async_enable(); 273 274 /* 275 * OK, it's off to the idle thread for us 276 */ 277 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 278 } 279 280 #ifdef CONFIG_HOTPLUG_CPU 281 static int op_cpu_disable(unsigned int cpu) 282 { 283 /* 284 * If we don't have a cpu_die method, abort before we reach the point 285 * of no return. CPU0 may not have an cpu_ops, so test for it. 286 */ 287 if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_die) 288 return -EOPNOTSUPP; 289 290 /* 291 * We may need to abort a hot unplug for some other mechanism-specific 292 * reason. 293 */ 294 if (cpu_ops[cpu]->cpu_disable) 295 return cpu_ops[cpu]->cpu_disable(cpu); 296 297 return 0; 298 } 299 300 /* 301 * __cpu_disable runs on the processor to be shutdown. 302 */ 303 int __cpu_disable(void) 304 { 305 unsigned int cpu = smp_processor_id(); 306 int ret; 307 308 ret = op_cpu_disable(cpu); 309 if (ret) 310 return ret; 311 312 /* 313 * Take this CPU offline. Once we clear this, we can't return, 314 * and we must not schedule until we're ready to give up the cpu. 315 */ 316 set_cpu_online(cpu, false); 317 318 /* 319 * OK - migrate IRQs away from this CPU 320 */ 321 irq_migrate_all_off_this_cpu(); 322 323 return 0; 324 } 325 326 static int op_cpu_kill(unsigned int cpu) 327 { 328 /* 329 * If we have no means of synchronising with the dying CPU, then assume 330 * that it is really dead. We can only wait for an arbitrary length of 331 * time and hope that it's dead, so let's skip the wait and just hope. 332 */ 333 if (!cpu_ops[cpu]->cpu_kill) 334 return 0; 335 336 return cpu_ops[cpu]->cpu_kill(cpu); 337 } 338 339 /* 340 * called on the thread which is asking for a CPU to be shutdown - 341 * waits until shutdown has completed, or it is timed out. 342 */ 343 void __cpu_die(unsigned int cpu) 344 { 345 int err; 346 347 if (!cpu_wait_death(cpu, 5)) { 348 pr_crit("CPU%u: cpu didn't die\n", cpu); 349 return; 350 } 351 pr_notice("CPU%u: shutdown\n", cpu); 352 353 /* 354 * Now that the dying CPU is beyond the point of no return w.r.t. 355 * in-kernel synchronisation, try to get the firwmare to help us to 356 * verify that it has really left the kernel before we consider 357 * clobbering anything it might still be using. 358 */ 359 err = op_cpu_kill(cpu); 360 if (err) 361 pr_warn("CPU%d may not have shut down cleanly: %d\n", 362 cpu, err); 363 } 364 365 /* 366 * Called from the idle thread for the CPU which has been shutdown. 367 * 368 * Note that we disable IRQs here, but do not re-enable them 369 * before returning to the caller. This is also the behaviour 370 * of the other hotplug-cpu capable cores, so presumably coming 371 * out of idle fixes this. 372 */ 373 void cpu_die(void) 374 { 375 unsigned int cpu = smp_processor_id(); 376 377 idle_task_exit(); 378 379 local_irq_disable(); 380 381 /* Tell __cpu_die() that this CPU is now safe to dispose of */ 382 (void)cpu_report_death(); 383 384 /* 385 * Actually shutdown the CPU. This must never fail. The specific hotplug 386 * mechanism must perform all required cache maintenance to ensure that 387 * no dirty lines are lost in the process of shutting down the CPU. 388 */ 389 cpu_ops[cpu]->cpu_die(cpu); 390 391 BUG(); 392 } 393 #endif 394 395 /* 396 * Kill the calling secondary CPU, early in bringup before it is turned 397 * online. 398 */ 399 void cpu_die_early(void) 400 { 401 int cpu = smp_processor_id(); 402 403 pr_crit("CPU%d: will not boot\n", cpu); 404 405 /* Mark this CPU absent */ 406 set_cpu_present(cpu, 0); 407 408 #ifdef CONFIG_HOTPLUG_CPU 409 update_cpu_boot_status(CPU_KILL_ME); 410 /* Check if we can park ourselves */ 411 if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_die) 412 cpu_ops[cpu]->cpu_die(cpu); 413 #endif 414 update_cpu_boot_status(CPU_STUCK_IN_KERNEL); 415 416 cpu_park_loop(); 417 } 418 419 static void __init hyp_mode_check(void) 420 { 421 if (is_hyp_mode_available()) 422 pr_info("CPU: All CPU(s) started at EL2\n"); 423 else if (is_hyp_mode_mismatched()) 424 WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC, 425 "CPU: CPUs started in inconsistent modes"); 426 else 427 pr_info("CPU: All CPU(s) started at EL1\n"); 428 } 429 430 void __init smp_cpus_done(unsigned int max_cpus) 431 { 432 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); 433 setup_cpu_features(); 434 hyp_mode_check(); 435 apply_alternatives_all(); 436 } 437 438 void __init smp_prepare_boot_cpu(void) 439 { 440 cpuinfo_store_boot_cpu(); 441 save_boot_cpu_run_el(); 442 set_my_cpu_offset(per_cpu_offset(smp_processor_id())); 443 } 444 445 static u64 __init of_get_cpu_mpidr(struct device_node *dn) 446 { 447 const __be32 *cell; 448 u64 hwid; 449 450 /* 451 * A cpu node with missing "reg" property is 452 * considered invalid to build a cpu_logical_map 453 * entry. 454 */ 455 cell = of_get_property(dn, "reg", NULL); 456 if (!cell) { 457 pr_err("%s: missing reg property\n", dn->full_name); 458 return INVALID_HWID; 459 } 460 461 hwid = of_read_number(cell, of_n_addr_cells(dn)); 462 /* 463 * Non affinity bits must be set to 0 in the DT 464 */ 465 if (hwid & ~MPIDR_HWID_BITMASK) { 466 pr_err("%s: invalid reg property\n", dn->full_name); 467 return INVALID_HWID; 468 } 469 return hwid; 470 } 471 472 /* 473 * Duplicate MPIDRs are a recipe for disaster. Scan all initialized 474 * entries and check for duplicates. If any is found just ignore the 475 * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid 476 * matching valid MPIDR values. 477 */ 478 static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid) 479 { 480 unsigned int i; 481 482 for (i = 1; (i < cpu) && (i < NR_CPUS); i++) 483 if (cpu_logical_map(i) == hwid) 484 return true; 485 return false; 486 } 487 488 /* 489 * Initialize cpu operations for a logical cpu and 490 * set it in the possible mask on success 491 */ 492 static int __init smp_cpu_setup(int cpu) 493 { 494 if (cpu_read_ops(cpu)) 495 return -ENODEV; 496 497 if (cpu_ops[cpu]->cpu_init(cpu)) 498 return -ENODEV; 499 500 set_cpu_possible(cpu, true); 501 502 return 0; 503 } 504 505 static bool bootcpu_valid __initdata; 506 static unsigned int cpu_count = 1; 507 508 #ifdef CONFIG_ACPI 509 /* 510 * acpi_map_gic_cpu_interface - parse processor MADT entry 511 * 512 * Carry out sanity checks on MADT processor entry and initialize 513 * cpu_logical_map on success 514 */ 515 static void __init 516 acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) 517 { 518 u64 hwid = processor->arm_mpidr; 519 520 if (!(processor->flags & ACPI_MADT_ENABLED)) { 521 pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid); 522 return; 523 } 524 525 if (hwid & ~MPIDR_HWID_BITMASK || hwid == INVALID_HWID) { 526 pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid); 527 return; 528 } 529 530 if (is_mpidr_duplicate(cpu_count, hwid)) { 531 pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid); 532 return; 533 } 534 535 /* Check if GICC structure of boot CPU is available in the MADT */ 536 if (cpu_logical_map(0) == hwid) { 537 if (bootcpu_valid) { 538 pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n", 539 hwid); 540 return; 541 } 542 bootcpu_valid = true; 543 return; 544 } 545 546 if (cpu_count >= NR_CPUS) 547 return; 548 549 /* map the logical cpu id to cpu MPIDR */ 550 cpu_logical_map(cpu_count) = hwid; 551 552 /* 553 * Set-up the ACPI parking protocol cpu entries 554 * while initializing the cpu_logical_map to 555 * avoid parsing MADT entries multiple times for 556 * nothing (ie a valid cpu_logical_map entry should 557 * contain a valid parking protocol data set to 558 * initialize the cpu if the parking protocol is 559 * the only available enable method). 560 */ 561 acpi_set_mailbox_entry(cpu_count, processor); 562 563 cpu_count++; 564 } 565 566 static int __init 567 acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header, 568 const unsigned long end) 569 { 570 struct acpi_madt_generic_interrupt *processor; 571 572 processor = (struct acpi_madt_generic_interrupt *)header; 573 if (BAD_MADT_GICC_ENTRY(processor, end)) 574 return -EINVAL; 575 576 acpi_table_print_madt_entry(header); 577 578 acpi_map_gic_cpu_interface(processor); 579 580 return 0; 581 } 582 #else 583 #define acpi_table_parse_madt(...) do { } while (0) 584 #endif 585 586 /* 587 * Enumerate the possible CPU set from the device tree and build the 588 * cpu logical map array containing MPIDR values related to logical 589 * cpus. Assumes that cpu_logical_map(0) has already been initialized. 590 */ 591 static void __init of_parse_and_init_cpus(void) 592 { 593 struct device_node *dn = NULL; 594 595 while ((dn = of_find_node_by_type(dn, "cpu"))) { 596 u64 hwid = of_get_cpu_mpidr(dn); 597 598 if (hwid == INVALID_HWID) 599 goto next; 600 601 if (is_mpidr_duplicate(cpu_count, hwid)) { 602 pr_err("%s: duplicate cpu reg properties in the DT\n", 603 dn->full_name); 604 goto next; 605 } 606 607 /* 608 * The numbering scheme requires that the boot CPU 609 * must be assigned logical id 0. Record it so that 610 * the logical map built from DT is validated and can 611 * be used. 612 */ 613 if (hwid == cpu_logical_map(0)) { 614 if (bootcpu_valid) { 615 pr_err("%s: duplicate boot cpu reg property in DT\n", 616 dn->full_name); 617 goto next; 618 } 619 620 bootcpu_valid = true; 621 622 /* 623 * cpu_logical_map has already been 624 * initialized and the boot cpu doesn't need 625 * the enable-method so continue without 626 * incrementing cpu. 627 */ 628 continue; 629 } 630 631 if (cpu_count >= NR_CPUS) 632 goto next; 633 634 pr_debug("cpu logical map 0x%llx\n", hwid); 635 cpu_logical_map(cpu_count) = hwid; 636 637 early_map_cpu_to_node(cpu_count, of_node_to_nid(dn)); 638 next: 639 cpu_count++; 640 } 641 } 642 643 /* 644 * Enumerate the possible CPU set from the device tree or ACPI and build the 645 * cpu logical map array containing MPIDR values related to logical 646 * cpus. Assumes that cpu_logical_map(0) has already been initialized. 647 */ 648 void __init smp_init_cpus(void) 649 { 650 int i; 651 652 if (acpi_disabled) 653 of_parse_and_init_cpus(); 654 else 655 /* 656 * do a walk of MADT to determine how many CPUs 657 * we have including disabled CPUs, and get information 658 * we need for SMP init 659 */ 660 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 661 acpi_parse_gic_cpu_interface, 0); 662 663 if (cpu_count > NR_CPUS) 664 pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n", 665 cpu_count, NR_CPUS); 666 667 if (!bootcpu_valid) { 668 pr_err("missing boot CPU MPIDR, not enabling secondaries\n"); 669 return; 670 } 671 672 /* 673 * We need to set the cpu_logical_map entries before enabling 674 * the cpus so that cpu processor description entries (DT cpu nodes 675 * and ACPI MADT entries) can be retrieved by matching the cpu hwid 676 * with entries in cpu_logical_map while initializing the cpus. 677 * If the cpu set-up fails, invalidate the cpu_logical_map entry. 678 */ 679 for (i = 1; i < NR_CPUS; i++) { 680 if (cpu_logical_map(i) != INVALID_HWID) { 681 if (smp_cpu_setup(i)) 682 cpu_logical_map(i) = INVALID_HWID; 683 } 684 } 685 } 686 687 void __init smp_prepare_cpus(unsigned int max_cpus) 688 { 689 int err; 690 unsigned int cpu; 691 692 init_cpu_topology(); 693 694 smp_store_cpu_info(smp_processor_id()); 695 696 /* 697 * Initialise the present map (which describes the set of CPUs 698 * actually populated at the present time) and release the 699 * secondaries from the bootloader. 700 */ 701 for_each_possible_cpu(cpu) { 702 703 if (cpu == smp_processor_id()) 704 continue; 705 706 if (!cpu_ops[cpu]) 707 continue; 708 709 err = cpu_ops[cpu]->cpu_prepare(cpu); 710 if (err) 711 continue; 712 713 set_cpu_present(cpu, true); 714 } 715 } 716 717 void (*__smp_cross_call)(const struct cpumask *, unsigned int); 718 719 void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) 720 { 721 __smp_cross_call = fn; 722 } 723 724 static const char *ipi_types[NR_IPI] __tracepoint_string = { 725 #define S(x,s) [x] = s 726 S(IPI_RESCHEDULE, "Rescheduling interrupts"), 727 S(IPI_CALL_FUNC, "Function call interrupts"), 728 S(IPI_CPU_STOP, "CPU stop interrupts"), 729 S(IPI_TIMER, "Timer broadcast interrupts"), 730 S(IPI_IRQ_WORK, "IRQ work interrupts"), 731 S(IPI_WAKEUP, "CPU wake-up interrupts"), 732 }; 733 734 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) 735 { 736 trace_ipi_raise(target, ipi_types[ipinr]); 737 __smp_cross_call(target, ipinr); 738 } 739 740 void show_ipi_list(struct seq_file *p, int prec) 741 { 742 unsigned int cpu, i; 743 744 for (i = 0; i < NR_IPI; i++) { 745 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, 746 prec >= 4 ? " " : ""); 747 for_each_online_cpu(cpu) 748 seq_printf(p, "%10u ", 749 __get_irq_stat(cpu, ipi_irqs[i])); 750 seq_printf(p, " %s\n", ipi_types[i]); 751 } 752 } 753 754 u64 smp_irq_stat_cpu(unsigned int cpu) 755 { 756 u64 sum = 0; 757 int i; 758 759 for (i = 0; i < NR_IPI; i++) 760 sum += __get_irq_stat(cpu, ipi_irqs[i]); 761 762 return sum; 763 } 764 765 void arch_send_call_function_ipi_mask(const struct cpumask *mask) 766 { 767 smp_cross_call(mask, IPI_CALL_FUNC); 768 } 769 770 void arch_send_call_function_single_ipi(int cpu) 771 { 772 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC); 773 } 774 775 #ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL 776 void arch_send_wakeup_ipi_mask(const struct cpumask *mask) 777 { 778 smp_cross_call(mask, IPI_WAKEUP); 779 } 780 #endif 781 782 #ifdef CONFIG_IRQ_WORK 783 void arch_irq_work_raise(void) 784 { 785 if (__smp_cross_call) 786 smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); 787 } 788 #endif 789 790 /* 791 * ipi_cpu_stop - handle IPI from smp_send_stop() 792 */ 793 static void ipi_cpu_stop(unsigned int cpu) 794 { 795 set_cpu_online(cpu, false); 796 797 local_irq_disable(); 798 799 while (1) 800 cpu_relax(); 801 } 802 803 /* 804 * Main handler for inter-processor interrupts 805 */ 806 void handle_IPI(int ipinr, struct pt_regs *regs) 807 { 808 unsigned int cpu = smp_processor_id(); 809 struct pt_regs *old_regs = set_irq_regs(regs); 810 811 if ((unsigned)ipinr < NR_IPI) { 812 trace_ipi_entry_rcuidle(ipi_types[ipinr]); 813 __inc_irq_stat(cpu, ipi_irqs[ipinr]); 814 } 815 816 switch (ipinr) { 817 case IPI_RESCHEDULE: 818 scheduler_ipi(); 819 break; 820 821 case IPI_CALL_FUNC: 822 irq_enter(); 823 generic_smp_call_function_interrupt(); 824 irq_exit(); 825 break; 826 827 case IPI_CPU_STOP: 828 irq_enter(); 829 ipi_cpu_stop(cpu); 830 irq_exit(); 831 break; 832 833 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 834 case IPI_TIMER: 835 irq_enter(); 836 tick_receive_broadcast(); 837 irq_exit(); 838 break; 839 #endif 840 841 #ifdef CONFIG_IRQ_WORK 842 case IPI_IRQ_WORK: 843 irq_enter(); 844 irq_work_run(); 845 irq_exit(); 846 break; 847 #endif 848 849 #ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL 850 case IPI_WAKEUP: 851 WARN_ONCE(!acpi_parking_protocol_valid(cpu), 852 "CPU%u: Wake-up IPI outside the ACPI parking protocol\n", 853 cpu); 854 break; 855 #endif 856 857 default: 858 pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr); 859 break; 860 } 861 862 if ((unsigned)ipinr < NR_IPI) 863 trace_ipi_exit_rcuidle(ipi_types[ipinr]); 864 set_irq_regs(old_regs); 865 } 866 867 void smp_send_reschedule(int cpu) 868 { 869 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); 870 } 871 872 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 873 void tick_broadcast(const struct cpumask *mask) 874 { 875 smp_cross_call(mask, IPI_TIMER); 876 } 877 #endif 878 879 void smp_send_stop(void) 880 { 881 unsigned long timeout; 882 883 if (num_online_cpus() > 1) { 884 cpumask_t mask; 885 886 cpumask_copy(&mask, cpu_online_mask); 887 cpumask_clear_cpu(smp_processor_id(), &mask); 888 889 if (system_state == SYSTEM_BOOTING || 890 system_state == SYSTEM_RUNNING) 891 pr_crit("SMP: stopping secondary CPUs\n"); 892 smp_cross_call(&mask, IPI_CPU_STOP); 893 } 894 895 /* Wait up to one second for other CPUs to stop */ 896 timeout = USEC_PER_SEC; 897 while (num_online_cpus() > 1 && timeout--) 898 udelay(1); 899 900 if (num_online_cpus() > 1) 901 pr_warning("SMP: failed to stop secondary CPUs %*pbl\n", 902 cpumask_pr_args(cpu_online_mask)); 903 } 904 905 /* 906 * not supported here 907 */ 908 int setup_profiling_timer(unsigned int multiplier) 909 { 910 return -EINVAL; 911 } 912 913 static bool have_cpu_die(void) 914 { 915 #ifdef CONFIG_HOTPLUG_CPU 916 int any_cpu = raw_smp_processor_id(); 917 918 if (cpu_ops[any_cpu]->cpu_die) 919 return true; 920 #endif 921 return false; 922 } 923 924 bool cpus_are_stuck_in_kernel(void) 925 { 926 bool smp_spin_tables = (num_possible_cpus() > 1 && !have_cpu_die()); 927 928 return !!cpus_stuck_in_kernel || smp_spin_tables; 929 } 930