1 /* 2 * SMP support for ppc. 3 * 4 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great 5 * deal of code from the sparc and intel versions. 6 * 7 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> 8 * 9 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and 10 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com 11 * 12 * This program is free software; you can redistribute it and/or 13 * modify it under the terms of the GNU General Public License 14 * as published by the Free Software Foundation; either version 15 * 2 of the License, or (at your option) any later version. 16 */ 17 18 #undef DEBUG 19 20 #include <linux/kernel.h> 21 #include <linux/module.h> 22 #include <linux/sched.h> 23 #include <linux/smp.h> 24 #include <linux/interrupt.h> 25 #include <linux/delay.h> 26 #include <linux/init.h> 27 #include <linux/spinlock.h> 28 #include <linux/cache.h> 29 #include <linux/err.h> 30 #include <linux/sysdev.h> 31 #include <linux/cpu.h> 32 #include <linux/notifier.h> 33 #include <linux/topology.h> 34 35 #include <asm/ptrace.h> 36 #include <asm/atomic.h> 37 #include <asm/irq.h> 38 #include <asm/page.h> 39 #include <asm/pgtable.h> 40 #include <asm/prom.h> 41 #include <asm/smp.h> 42 #include <asm/time.h> 43 #include <asm/machdep.h> 44 #include <asm/cputhreads.h> 45 #include <asm/cputable.h> 46 #include <asm/system.h> 47 #include <asm/mpic.h> 48 #include <asm/vdso_datapage.h> 49 #ifdef CONFIG_PPC64 50 #include <asm/paca.h> 51 #endif 52 53 #ifdef DEBUG 54 #include <asm/udbg.h> 55 #define DBG(fmt...) udbg_printf(fmt) 56 #else 57 #define DBG(fmt...) 58 #endif 59 60 struct thread_info *secondary_ti; 61 62 DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE; 63 DEFINE_PER_CPU(cpumask_t, cpu_core_map) = CPU_MASK_NONE; 64 65 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); 66 EXPORT_PER_CPU_SYMBOL(cpu_core_map); 67 68 /* SMP operations for this machine */ 69 struct smp_ops_t *smp_ops; 70 71 static volatile unsigned int cpu_callin_map[NR_CPUS]; 72 73 int smt_enabled_at_boot = 1; 74 75 static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL; 76 77 #ifdef CONFIG_PPC64 78 void __devinit smp_generic_kick_cpu(int nr) 79 { 80 BUG_ON(nr < 0 || nr >= NR_CPUS); 81 82 /* 83 * The processor is currently spinning, waiting for the 84 * cpu_start field to become non-zero After we set cpu_start, 85 * the processor will continue on to secondary_start 86 */ 87 paca[nr].cpu_start = 1; 88 smp_mb(); 89 } 90 #endif 91 92 void smp_message_recv(int msg) 93 { 94 switch(msg) { 95 case PPC_MSG_CALL_FUNCTION: 96 generic_smp_call_function_interrupt(); 97 break; 98 case PPC_MSG_RESCHEDULE: 99 /* we notice need_resched on exit */ 100 break; 101 case PPC_MSG_CALL_FUNC_SINGLE: 102 generic_smp_call_function_single_interrupt(); 103 break; 104 case PPC_MSG_DEBUGGER_BREAK: 105 if (crash_ipi_function_ptr) { 106 crash_ipi_function_ptr(get_irq_regs()); 107 break; 108 } 109 #ifdef CONFIG_DEBUGGER 110 debugger_ipi(get_irq_regs()); 111 break; 112 #endif /* CONFIG_DEBUGGER */ 113 /* FALLTHROUGH */ 114 default: 115 printk("SMP %d: smp_message_recv(): unknown msg %d\n", 116 smp_processor_id(), msg); 117 break; 118 } 119 } 120 121 static irqreturn_t call_function_action(int irq, void *data) 122 { 123 generic_smp_call_function_interrupt(); 124 return IRQ_HANDLED; 125 } 126 127 static irqreturn_t reschedule_action(int irq, void *data) 128 { 129 /* we just need the return path side effect of checking need_resched */ 130 return IRQ_HANDLED; 131 } 132 133 static irqreturn_t call_function_single_action(int irq, void *data) 134 { 135 generic_smp_call_function_single_interrupt(); 136 return IRQ_HANDLED; 137 } 138 139 static irqreturn_t debug_ipi_action(int irq, void *data) 140 { 141 smp_message_recv(PPC_MSG_DEBUGGER_BREAK); 142 return IRQ_HANDLED; 143 } 144 145 static irq_handler_t smp_ipi_action[] = { 146 [PPC_MSG_CALL_FUNCTION] = call_function_action, 147 [PPC_MSG_RESCHEDULE] = reschedule_action, 148 [PPC_MSG_CALL_FUNC_SINGLE] = call_function_single_action, 149 [PPC_MSG_DEBUGGER_BREAK] = debug_ipi_action, 150 }; 151 152 const char *smp_ipi_name[] = { 153 [PPC_MSG_CALL_FUNCTION] = "ipi call function", 154 [PPC_MSG_RESCHEDULE] = "ipi reschedule", 155 [PPC_MSG_CALL_FUNC_SINGLE] = "ipi call function single", 156 [PPC_MSG_DEBUGGER_BREAK] = "ipi debugger", 157 }; 158 159 /* optional function to request ipi, for controllers with >= 4 ipis */ 160 int smp_request_message_ipi(int virq, int msg) 161 { 162 int err; 163 164 if (msg < 0 || msg > PPC_MSG_DEBUGGER_BREAK) { 165 return -EINVAL; 166 } 167 #if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC) 168 if (msg == PPC_MSG_DEBUGGER_BREAK) { 169 return 1; 170 } 171 #endif 172 err = request_irq(virq, smp_ipi_action[msg], IRQF_DISABLED|IRQF_PERCPU, 173 smp_ipi_name[msg], 0); 174 WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n", 175 virq, smp_ipi_name[msg], err); 176 177 return err; 178 } 179 180 void smp_send_reschedule(int cpu) 181 { 182 if (likely(smp_ops)) 183 smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE); 184 } 185 186 void arch_send_call_function_single_ipi(int cpu) 187 { 188 smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNC_SINGLE); 189 } 190 191 void arch_send_call_function_ipi(cpumask_t mask) 192 { 193 unsigned int cpu; 194 195 for_each_cpu_mask(cpu, mask) 196 smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION); 197 } 198 199 #ifdef CONFIG_DEBUGGER 200 void smp_send_debugger_break(int cpu) 201 { 202 if (likely(smp_ops)) 203 smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK); 204 } 205 #endif 206 207 #ifdef CONFIG_KEXEC 208 void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) 209 { 210 crash_ipi_function_ptr = crash_ipi_callback; 211 if (crash_ipi_callback && smp_ops) { 212 mb(); 213 smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_DEBUGGER_BREAK); 214 } 215 } 216 #endif 217 218 static void stop_this_cpu(void *dummy) 219 { 220 local_irq_disable(); 221 while (1) 222 ; 223 } 224 225 void smp_send_stop(void) 226 { 227 smp_call_function(stop_this_cpu, NULL, 0); 228 } 229 230 struct thread_info *current_set[NR_CPUS]; 231 232 static void __devinit smp_store_cpu_info(int id) 233 { 234 per_cpu(pvr, id) = mfspr(SPRN_PVR); 235 } 236 237 static void __init smp_create_idle(unsigned int cpu) 238 { 239 struct task_struct *p; 240 241 /* create a process for the processor */ 242 p = fork_idle(cpu); 243 if (IS_ERR(p)) 244 panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p)); 245 #ifdef CONFIG_PPC64 246 paca[cpu].__current = p; 247 paca[cpu].kstack = (unsigned long) task_thread_info(p) 248 + THREAD_SIZE - STACK_FRAME_OVERHEAD; 249 #endif 250 current_set[cpu] = task_thread_info(p); 251 task_thread_info(p)->cpu = cpu; 252 } 253 254 void __init smp_prepare_cpus(unsigned int max_cpus) 255 { 256 unsigned int cpu; 257 258 DBG("smp_prepare_cpus\n"); 259 260 /* 261 * setup_cpu may need to be called on the boot cpu. We havent 262 * spun any cpus up but lets be paranoid. 263 */ 264 BUG_ON(boot_cpuid != smp_processor_id()); 265 266 /* Fixup boot cpu */ 267 smp_store_cpu_info(boot_cpuid); 268 cpu_callin_map[boot_cpuid] = 1; 269 270 if (smp_ops) 271 max_cpus = smp_ops->probe(); 272 else 273 max_cpus = 1; 274 275 smp_space_timers(max_cpus); 276 277 for_each_possible_cpu(cpu) 278 if (cpu != boot_cpuid) 279 smp_create_idle(cpu); 280 } 281 282 void __devinit smp_prepare_boot_cpu(void) 283 { 284 BUG_ON(smp_processor_id() != boot_cpuid); 285 286 cpu_set(boot_cpuid, cpu_online_map); 287 cpu_set(boot_cpuid, per_cpu(cpu_sibling_map, boot_cpuid)); 288 cpu_set(boot_cpuid, per_cpu(cpu_core_map, boot_cpuid)); 289 #ifdef CONFIG_PPC64 290 paca[boot_cpuid].__current = current; 291 #endif 292 current_set[boot_cpuid] = task_thread_info(current); 293 } 294 295 #ifdef CONFIG_HOTPLUG_CPU 296 /* State of each CPU during hotplug phases */ 297 DEFINE_PER_CPU(int, cpu_state) = { 0 }; 298 299 int generic_cpu_disable(void) 300 { 301 unsigned int cpu = smp_processor_id(); 302 303 if (cpu == boot_cpuid) 304 return -EBUSY; 305 306 cpu_clear(cpu, cpu_online_map); 307 #ifdef CONFIG_PPC64 308 vdso_data->processorCount--; 309 fixup_irqs(cpu_online_map); 310 #endif 311 return 0; 312 } 313 314 int generic_cpu_enable(unsigned int cpu) 315 { 316 /* Do the normal bootup if we haven't 317 * already bootstrapped. */ 318 if (system_state != SYSTEM_RUNNING) 319 return -ENOSYS; 320 321 /* get the target out of it's holding state */ 322 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE; 323 smp_wmb(); 324 325 while (!cpu_online(cpu)) 326 cpu_relax(); 327 328 #ifdef CONFIG_PPC64 329 fixup_irqs(cpu_online_map); 330 /* counter the irq disable in fixup_irqs */ 331 local_irq_enable(); 332 #endif 333 return 0; 334 } 335 336 void generic_cpu_die(unsigned int cpu) 337 { 338 int i; 339 340 for (i = 0; i < 100; i++) { 341 smp_rmb(); 342 if (per_cpu(cpu_state, cpu) == CPU_DEAD) 343 return; 344 msleep(100); 345 } 346 printk(KERN_ERR "CPU%d didn't die...\n", cpu); 347 } 348 349 void generic_mach_cpu_die(void) 350 { 351 unsigned int cpu; 352 353 local_irq_disable(); 354 cpu = smp_processor_id(); 355 printk(KERN_DEBUG "CPU%d offline\n", cpu); 356 __get_cpu_var(cpu_state) = CPU_DEAD; 357 smp_wmb(); 358 while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE) 359 cpu_relax(); 360 cpu_set(cpu, cpu_online_map); 361 local_irq_enable(); 362 } 363 #endif 364 365 static int __devinit cpu_enable(unsigned int cpu) 366 { 367 if (smp_ops && smp_ops->cpu_enable) 368 return smp_ops->cpu_enable(cpu); 369 370 return -ENOSYS; 371 } 372 373 int __cpuinit __cpu_up(unsigned int cpu) 374 { 375 int c; 376 377 secondary_ti = current_set[cpu]; 378 if (!cpu_enable(cpu)) 379 return 0; 380 381 if (smp_ops == NULL || 382 (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu))) 383 return -EINVAL; 384 385 /* Make sure callin-map entry is 0 (can be leftover a CPU 386 * hotplug 387 */ 388 cpu_callin_map[cpu] = 0; 389 390 /* The information for processor bringup must 391 * be written out to main store before we release 392 * the processor. 393 */ 394 smp_mb(); 395 396 /* wake up cpus */ 397 DBG("smp: kicking cpu %d\n", cpu); 398 smp_ops->kick_cpu(cpu); 399 400 /* 401 * wait to see if the cpu made a callin (is actually up). 402 * use this value that I found through experimentation. 403 * -- Cort 404 */ 405 if (system_state < SYSTEM_RUNNING) 406 for (c = 50000; c && !cpu_callin_map[cpu]; c--) 407 udelay(100); 408 #ifdef CONFIG_HOTPLUG_CPU 409 else 410 /* 411 * CPUs can take much longer to come up in the 412 * hotplug case. Wait five seconds. 413 */ 414 for (c = 25; c && !cpu_callin_map[cpu]; c--) { 415 msleep(200); 416 } 417 #endif 418 419 if (!cpu_callin_map[cpu]) { 420 printk("Processor %u is stuck.\n", cpu); 421 return -ENOENT; 422 } 423 424 printk("Processor %u found.\n", cpu); 425 426 if (smp_ops->give_timebase) 427 smp_ops->give_timebase(); 428 429 /* Wait until cpu puts itself in the online map */ 430 while (!cpu_online(cpu)) 431 cpu_relax(); 432 433 return 0; 434 } 435 436 /* Return the value of the reg property corresponding to the given 437 * logical cpu. 438 */ 439 int cpu_to_core_id(int cpu) 440 { 441 struct device_node *np; 442 const int *reg; 443 int id = -1; 444 445 np = of_get_cpu_node(cpu, NULL); 446 if (!np) 447 goto out; 448 449 reg = of_get_property(np, "reg", NULL); 450 if (!reg) 451 goto out; 452 453 id = *reg; 454 out: 455 of_node_put(np); 456 return id; 457 } 458 459 /* Must be called when no change can occur to cpu_present_map, 460 * i.e. during cpu online or offline. 461 */ 462 static struct device_node *cpu_to_l2cache(int cpu) 463 { 464 struct device_node *np; 465 struct device_node *cache; 466 467 if (!cpu_present(cpu)) 468 return NULL; 469 470 np = of_get_cpu_node(cpu, NULL); 471 if (np == NULL) 472 return NULL; 473 474 cache = of_find_next_cache_node(np); 475 476 of_node_put(np); 477 478 return cache; 479 } 480 481 /* Activate a secondary processor. */ 482 int __devinit start_secondary(void *unused) 483 { 484 unsigned int cpu = smp_processor_id(); 485 struct device_node *l2_cache; 486 int i, base; 487 488 atomic_inc(&init_mm.mm_count); 489 current->active_mm = &init_mm; 490 491 smp_store_cpu_info(cpu); 492 set_dec(tb_ticks_per_jiffy); 493 preempt_disable(); 494 cpu_callin_map[cpu] = 1; 495 496 smp_ops->setup_cpu(cpu); 497 if (smp_ops->take_timebase) 498 smp_ops->take_timebase(); 499 500 if (system_state > SYSTEM_BOOTING) 501 snapshot_timebase(); 502 503 secondary_cpu_time_init(); 504 505 ipi_call_lock(); 506 notify_cpu_starting(cpu); 507 cpu_set(cpu, cpu_online_map); 508 /* Update sibling maps */ 509 base = cpu_first_thread_in_core(cpu); 510 for (i = 0; i < threads_per_core; i++) { 511 if (cpu_is_offline(base + i)) 512 continue; 513 cpu_set(cpu, per_cpu(cpu_sibling_map, base + i)); 514 cpu_set(base + i, per_cpu(cpu_sibling_map, cpu)); 515 516 /* cpu_core_map should be a superset of 517 * cpu_sibling_map even if we don't have cache 518 * information, so update the former here, too. 519 */ 520 cpu_set(cpu, per_cpu(cpu_core_map, base +i)); 521 cpu_set(base + i, per_cpu(cpu_core_map, cpu)); 522 } 523 l2_cache = cpu_to_l2cache(cpu); 524 for_each_online_cpu(i) { 525 struct device_node *np = cpu_to_l2cache(i); 526 if (!np) 527 continue; 528 if (np == l2_cache) { 529 cpu_set(cpu, per_cpu(cpu_core_map, i)); 530 cpu_set(i, per_cpu(cpu_core_map, cpu)); 531 } 532 of_node_put(np); 533 } 534 of_node_put(l2_cache); 535 ipi_call_unlock(); 536 537 local_irq_enable(); 538 539 cpu_idle(); 540 return 0; 541 } 542 543 int setup_profiling_timer(unsigned int multiplier) 544 { 545 return 0; 546 } 547 548 void __init smp_cpus_done(unsigned int max_cpus) 549 { 550 cpumask_t old_mask; 551 552 /* We want the setup_cpu() here to be called from CPU 0, but our 553 * init thread may have been "borrowed" by another CPU in the meantime 554 * se we pin us down to CPU 0 for a short while 555 */ 556 old_mask = current->cpus_allowed; 557 set_cpus_allowed(current, cpumask_of_cpu(boot_cpuid)); 558 559 if (smp_ops) 560 smp_ops->setup_cpu(boot_cpuid); 561 562 set_cpus_allowed(current, old_mask); 563 564 snapshot_timebases(); 565 566 dump_numa_cpu_topology(); 567 } 568 569 #ifdef CONFIG_HOTPLUG_CPU 570 int __cpu_disable(void) 571 { 572 struct device_node *l2_cache; 573 int cpu = smp_processor_id(); 574 int base, i; 575 int err; 576 577 if (!smp_ops->cpu_disable) 578 return -ENOSYS; 579 580 err = smp_ops->cpu_disable(); 581 if (err) 582 return err; 583 584 /* Update sibling maps */ 585 base = cpu_first_thread_in_core(cpu); 586 for (i = 0; i < threads_per_core; i++) { 587 cpu_clear(cpu, per_cpu(cpu_sibling_map, base + i)); 588 cpu_clear(base + i, per_cpu(cpu_sibling_map, cpu)); 589 cpu_clear(cpu, per_cpu(cpu_core_map, base +i)); 590 cpu_clear(base + i, per_cpu(cpu_core_map, cpu)); 591 } 592 593 l2_cache = cpu_to_l2cache(cpu); 594 for_each_present_cpu(i) { 595 struct device_node *np = cpu_to_l2cache(i); 596 if (!np) 597 continue; 598 if (np == l2_cache) { 599 cpu_clear(cpu, per_cpu(cpu_core_map, i)); 600 cpu_clear(i, per_cpu(cpu_core_map, cpu)); 601 } 602 of_node_put(np); 603 } 604 of_node_put(l2_cache); 605 606 607 return 0; 608 } 609 610 void __cpu_die(unsigned int cpu) 611 { 612 if (smp_ops->cpu_die) 613 smp_ops->cpu_die(cpu); 614 } 615 #endif 616