1 /* 2 * Author: Andy Fleming <afleming@freescale.com> 3 * Kumar Gala <galak@kernel.crashing.org> 4 * 5 * Copyright 2006-2008, 2011-2012 Freescale Semiconductor Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License as published by the 9 * Free Software Foundation; either version 2 of the License, or (at your 10 * option) any later version. 11 */ 12 13 #include <linux/stddef.h> 14 #include <linux/kernel.h> 15 #include <linux/init.h> 16 #include <linux/delay.h> 17 #include <linux/of.h> 18 #include <linux/of_address.h> 19 #include <linux/kexec.h> 20 #include <linux/highmem.h> 21 #include <linux/cpu.h> 22 #include <linux/fsl/guts.h> 23 24 #include <asm/machdep.h> 25 #include <asm/pgtable.h> 26 #include <asm/page.h> 27 #include <asm/mpic.h> 28 #include <asm/cacheflush.h> 29 #include <asm/dbell.h> 30 #include <asm/code-patching.h> 31 #include <asm/cputhreads.h> 32 33 #include <sysdev/fsl_soc.h> 34 #include <sysdev/mpic.h> 35 #include "smp.h" 36 37 struct epapr_spin_table { 38 u32 addr_h; 39 u32 addr_l; 40 u32 r3_h; 41 u32 r3_l; 42 u32 reserved; 43 u32 pir; 44 }; 45 46 static struct ccsr_guts __iomem *guts; 47 static u64 timebase; 48 static int tb_req; 49 static int tb_valid; 50 51 static void mpc85xx_timebase_freeze(int freeze) 52 { 53 uint32_t mask; 54 55 mask = CCSR_GUTS_DEVDISR_TB0 | CCSR_GUTS_DEVDISR_TB1; 56 if (freeze) 57 setbits32(&guts->devdisr, mask); 58 else 59 clrbits32(&guts->devdisr, mask); 60 61 in_be32(&guts->devdisr); 62 } 63 64 static void mpc85xx_give_timebase(void) 65 { 66 unsigned long flags; 67 68 local_irq_save(flags); 69 70 while (!tb_req) 71 barrier(); 72 tb_req = 0; 73 74 mpc85xx_timebase_freeze(1); 75 #ifdef CONFIG_PPC64 76 /* 77 * e5500/e6500 have a workaround for erratum A-006958 in place 78 * that will reread the timebase until TBL is non-zero. 79 * That would be a bad thing when the timebase is frozen. 80 * 81 * Thus, we read it manually, and instead of checking that 82 * TBL is non-zero, we ensure that TB does not change. We don't 83 * do that for the main mftb implementation, because it requires 84 * a scratch register 85 */ 86 { 87 u64 prev; 88 89 asm volatile("mfspr %0, %1" : "=r" (timebase) : 90 "i" (SPRN_TBRL)); 91 92 do { 93 prev = timebase; 94 asm volatile("mfspr %0, %1" : "=r" (timebase) : 95 "i" (SPRN_TBRL)); 96 } while (prev != timebase); 97 } 98 #else 99 timebase = get_tb(); 100 #endif 101 mb(); 102 tb_valid = 1; 103 104 while (tb_valid) 105 barrier(); 106 107 mpc85xx_timebase_freeze(0); 108 109 local_irq_restore(flags); 110 } 111 112 static void mpc85xx_take_timebase(void) 113 { 114 unsigned long flags; 115 116 local_irq_save(flags); 117 118 tb_req = 1; 119 while (!tb_valid) 120 barrier(); 121 122 set_tb(timebase >> 32, timebase & 0xffffffff); 123 isync(); 124 tb_valid = 0; 125 126 local_irq_restore(flags); 127 } 128 129 #ifdef CONFIG_HOTPLUG_CPU 130 static void smp_85xx_mach_cpu_die(void) 131 { 132 unsigned int cpu = smp_processor_id(); 133 u32 tmp; 134 135 local_irq_disable(); 136 idle_task_exit(); 137 generic_set_cpu_dead(cpu); 138 mb(); 139 140 mtspr(SPRN_TCR, 0); 141 142 __flush_disable_L1(); 143 tmp = (mfspr(SPRN_HID0) & ~(HID0_DOZE|HID0_SLEEP)) | HID0_NAP; 144 mtspr(SPRN_HID0, tmp); 145 isync(); 146 147 /* Enter NAP mode. */ 148 tmp = mfmsr(); 149 tmp |= MSR_WE; 150 mb(); 151 mtmsr(tmp); 152 isync(); 153 154 while (1) 155 ; 156 } 157 #endif 158 159 static inline void flush_spin_table(void *spin_table) 160 { 161 flush_dcache_range((ulong)spin_table, 162 (ulong)spin_table + sizeof(struct epapr_spin_table)); 163 } 164 165 static inline u32 read_spin_table_addr_l(void *spin_table) 166 { 167 flush_dcache_range((ulong)spin_table, 168 (ulong)spin_table + sizeof(struct epapr_spin_table)); 169 return in_be32(&((struct epapr_spin_table *)spin_table)->addr_l); 170 } 171 172 #ifdef CONFIG_PPC64 173 static void wake_hw_thread(void *info) 174 { 175 void fsl_secondary_thread_init(void); 176 unsigned long imsr, inia; 177 int nr = *(const int *)info; 178 179 imsr = MSR_KERNEL; 180 inia = *(unsigned long *)fsl_secondary_thread_init; 181 182 if (cpu_thread_in_core(nr) == 0) { 183 /* For when we boot on a secondary thread with kdump */ 184 mttmr(TMRN_IMSR0, imsr); 185 mttmr(TMRN_INIA0, inia); 186 mtspr(SPRN_TENS, TEN_THREAD(0)); 187 } else { 188 mttmr(TMRN_IMSR1, imsr); 189 mttmr(TMRN_INIA1, inia); 190 mtspr(SPRN_TENS, TEN_THREAD(1)); 191 } 192 193 smp_generic_kick_cpu(nr); 194 } 195 #endif 196 197 static int smp_85xx_kick_cpu(int nr) 198 { 199 unsigned long flags; 200 const u64 *cpu_rel_addr; 201 __iomem struct epapr_spin_table *spin_table; 202 struct device_node *np; 203 int hw_cpu = get_hard_smp_processor_id(nr); 204 int ioremappable; 205 int ret = 0; 206 207 WARN_ON(nr < 0 || nr >= NR_CPUS); 208 WARN_ON(hw_cpu < 0 || hw_cpu >= NR_CPUS); 209 210 pr_debug("smp_85xx_kick_cpu: kick CPU #%d\n", nr); 211 212 #ifdef CONFIG_PPC64 213 /* Threads don't use the spin table */ 214 if (cpu_thread_in_core(nr) != 0) { 215 int primary = cpu_first_thread_sibling(nr); 216 217 if (WARN_ON_ONCE(!cpu_has_feature(CPU_FTR_SMT))) 218 return -ENOENT; 219 220 if (cpu_thread_in_core(nr) != 1) { 221 pr_err("%s: cpu %d: invalid hw thread %d\n", 222 __func__, nr, cpu_thread_in_core(nr)); 223 return -ENOENT; 224 } 225 226 if (!cpu_online(primary)) { 227 pr_err("%s: cpu %d: primary %d not online\n", 228 __func__, nr, primary); 229 return -ENOENT; 230 } 231 232 smp_call_function_single(primary, wake_hw_thread, &nr, 0); 233 return 0; 234 } else if (cpu_thread_in_core(boot_cpuid) != 0 && 235 cpu_first_thread_sibling(boot_cpuid) == nr) { 236 if (WARN_ON_ONCE(!cpu_has_feature(CPU_FTR_SMT))) 237 return -ENOENT; 238 239 smp_call_function_single(boot_cpuid, wake_hw_thread, &nr, 0); 240 } 241 #endif 242 243 np = of_get_cpu_node(nr, NULL); 244 cpu_rel_addr = of_get_property(np, "cpu-release-addr", NULL); 245 246 if (cpu_rel_addr == NULL) { 247 printk(KERN_ERR "No cpu-release-addr for cpu %d\n", nr); 248 return -ENOENT; 249 } 250 251 /* 252 * A secondary core could be in a spinloop in the bootpage 253 * (0xfffff000), somewhere in highmem, or somewhere in lowmem. 254 * The bootpage and highmem can be accessed via ioremap(), but 255 * we need to directly access the spinloop if its in lowmem. 256 */ 257 ioremappable = *cpu_rel_addr > virt_to_phys(high_memory); 258 259 /* Map the spin table */ 260 if (ioremappable) 261 spin_table = ioremap_prot(*cpu_rel_addr, 262 sizeof(struct epapr_spin_table), _PAGE_COHERENT); 263 else 264 spin_table = phys_to_virt(*cpu_rel_addr); 265 266 local_irq_save(flags); 267 #ifdef CONFIG_PPC32 268 #ifdef CONFIG_HOTPLUG_CPU 269 /* Corresponding to generic_set_cpu_dead() */ 270 generic_set_cpu_up(nr); 271 272 if (system_state == SYSTEM_RUNNING) { 273 /* 274 * To keep it compatible with old boot program which uses 275 * cache-inhibit spin table, we need to flush the cache 276 * before accessing spin table to invalidate any staled data. 277 * We also need to flush the cache after writing to spin 278 * table to push data out. 279 */ 280 flush_spin_table(spin_table); 281 out_be32(&spin_table->addr_l, 0); 282 flush_spin_table(spin_table); 283 284 /* 285 * We don't set the BPTR register here since it already points 286 * to the boot page properly. 287 */ 288 mpic_reset_core(nr); 289 290 /* 291 * wait until core is ready... 292 * We need to invalidate the stale data, in case the boot 293 * loader uses a cache-inhibited spin table. 294 */ 295 if (!spin_event_timeout( 296 read_spin_table_addr_l(spin_table) == 1, 297 10000, 100)) { 298 pr_err("%s: timeout waiting for core %d to reset\n", 299 __func__, hw_cpu); 300 ret = -ENOENT; 301 goto out; 302 } 303 304 /* clear the acknowledge status */ 305 __secondary_hold_acknowledge = -1; 306 } 307 #endif 308 flush_spin_table(spin_table); 309 out_be32(&spin_table->pir, hw_cpu); 310 out_be32(&spin_table->addr_l, __pa(__early_start)); 311 flush_spin_table(spin_table); 312 313 /* Wait a bit for the CPU to ack. */ 314 if (!spin_event_timeout(__secondary_hold_acknowledge == hw_cpu, 315 10000, 100)) { 316 pr_err("%s: timeout waiting for core %d to ack\n", 317 __func__, hw_cpu); 318 ret = -ENOENT; 319 goto out; 320 } 321 out: 322 #else 323 smp_generic_kick_cpu(nr); 324 325 flush_spin_table(spin_table); 326 out_be32(&spin_table->pir, hw_cpu); 327 out_be64((u64 *)(&spin_table->addr_h), 328 __pa(ppc_function_entry(generic_secondary_smp_init))); 329 flush_spin_table(spin_table); 330 #endif 331 332 local_irq_restore(flags); 333 334 if (ioremappable) 335 iounmap(spin_table); 336 337 return ret; 338 } 339 340 struct smp_ops_t smp_85xx_ops = { 341 .kick_cpu = smp_85xx_kick_cpu, 342 .cpu_bootable = smp_generic_cpu_bootable, 343 #ifdef CONFIG_HOTPLUG_CPU 344 .cpu_disable = generic_cpu_disable, 345 .cpu_die = generic_cpu_die, 346 #endif 347 #if defined(CONFIG_KEXEC) && !defined(CONFIG_PPC64) 348 .give_timebase = smp_generic_give_timebase, 349 .take_timebase = smp_generic_take_timebase, 350 #endif 351 }; 352 353 #ifdef CONFIG_KEXEC 354 #ifdef CONFIG_PPC32 355 atomic_t kexec_down_cpus = ATOMIC_INIT(0); 356 357 void mpc85xx_smp_kexec_cpu_down(int crash_shutdown, int secondary) 358 { 359 local_irq_disable(); 360 361 if (secondary) { 362 __flush_disable_L1(); 363 atomic_inc(&kexec_down_cpus); 364 /* loop forever */ 365 while (1); 366 } 367 } 368 369 static void mpc85xx_smp_kexec_down(void *arg) 370 { 371 if (ppc_md.kexec_cpu_down) 372 ppc_md.kexec_cpu_down(0,1); 373 } 374 #else 375 void mpc85xx_smp_kexec_cpu_down(int crash_shutdown, int secondary) 376 { 377 int cpu = smp_processor_id(); 378 int sibling = cpu_last_thread_sibling(cpu); 379 bool notified = false; 380 int disable_cpu; 381 int disable_threadbit = 0; 382 long start = mftb(); 383 long now; 384 385 local_irq_disable(); 386 hard_irq_disable(); 387 mpic_teardown_this_cpu(secondary); 388 389 if (cpu == crashing_cpu && cpu_thread_in_core(cpu) != 0) { 390 /* 391 * We enter the crash kernel on whatever cpu crashed, 392 * even if it's a secondary thread. If that's the case, 393 * disable the corresponding primary thread. 394 */ 395 disable_threadbit = 1; 396 disable_cpu = cpu_first_thread_sibling(cpu); 397 } else if (sibling != crashing_cpu && 398 cpu_thread_in_core(cpu) == 0 && 399 cpu_thread_in_core(sibling) != 0) { 400 disable_threadbit = 2; 401 disable_cpu = sibling; 402 } 403 404 if (disable_threadbit) { 405 while (paca[disable_cpu].kexec_state < KEXEC_STATE_REAL_MODE) { 406 barrier(); 407 now = mftb(); 408 if (!notified && now - start > 1000000) { 409 pr_info("%s/%d: waiting for cpu %d to enter KEXEC_STATE_REAL_MODE (%d)\n", 410 __func__, smp_processor_id(), 411 disable_cpu, 412 paca[disable_cpu].kexec_state); 413 notified = true; 414 } 415 } 416 417 if (notified) { 418 pr_info("%s: cpu %d done waiting\n", 419 __func__, disable_cpu); 420 } 421 422 mtspr(SPRN_TENC, disable_threadbit); 423 while (mfspr(SPRN_TENSR) & disable_threadbit) 424 cpu_relax(); 425 } 426 } 427 #endif 428 429 static void mpc85xx_smp_machine_kexec(struct kimage *image) 430 { 431 #ifdef CONFIG_PPC32 432 int timeout = INT_MAX; 433 int i, num_cpus = num_present_cpus(); 434 435 if (image->type == KEXEC_TYPE_DEFAULT) 436 smp_call_function(mpc85xx_smp_kexec_down, NULL, 0); 437 438 while ( (atomic_read(&kexec_down_cpus) != (num_cpus - 1)) && 439 ( timeout > 0 ) ) 440 { 441 timeout--; 442 } 443 444 if ( !timeout ) 445 printk(KERN_ERR "Unable to bring down secondary cpu(s)"); 446 447 for_each_online_cpu(i) 448 { 449 if ( i == smp_processor_id() ) continue; 450 mpic_reset_core(i); 451 } 452 #endif 453 454 default_machine_kexec(image); 455 } 456 #endif /* CONFIG_KEXEC */ 457 458 static void smp_85xx_basic_setup(int cpu_nr) 459 { 460 if (cpu_has_feature(CPU_FTR_DBELL)) 461 doorbell_setup_this_cpu(); 462 } 463 464 static void smp_85xx_setup_cpu(int cpu_nr) 465 { 466 mpic_setup_this_cpu(); 467 smp_85xx_basic_setup(cpu_nr); 468 } 469 470 static const struct of_device_id mpc85xx_smp_guts_ids[] = { 471 { .compatible = "fsl,mpc8572-guts", }, 472 { .compatible = "fsl,p1020-guts", }, 473 { .compatible = "fsl,p1021-guts", }, 474 { .compatible = "fsl,p1022-guts", }, 475 { .compatible = "fsl,p1023-guts", }, 476 { .compatible = "fsl,p2020-guts", }, 477 {}, 478 }; 479 480 void __init mpc85xx_smp_init(void) 481 { 482 struct device_node *np; 483 484 485 np = of_find_node_by_type(NULL, "open-pic"); 486 if (np) { 487 smp_85xx_ops.probe = smp_mpic_probe; 488 smp_85xx_ops.setup_cpu = smp_85xx_setup_cpu; 489 smp_85xx_ops.message_pass = smp_mpic_message_pass; 490 } else 491 smp_85xx_ops.setup_cpu = smp_85xx_basic_setup; 492 493 if (cpu_has_feature(CPU_FTR_DBELL)) { 494 /* 495 * If left NULL, .message_pass defaults to 496 * smp_muxed_ipi_message_pass 497 */ 498 smp_85xx_ops.message_pass = NULL; 499 smp_85xx_ops.cause_ipi = doorbell_cause_ipi; 500 smp_85xx_ops.probe = NULL; 501 } 502 503 np = of_find_matching_node(NULL, mpc85xx_smp_guts_ids); 504 if (np) { 505 guts = of_iomap(np, 0); 506 of_node_put(np); 507 if (!guts) { 508 pr_err("%s: Could not map guts node address\n", 509 __func__); 510 return; 511 } 512 smp_85xx_ops.give_timebase = mpc85xx_give_timebase; 513 smp_85xx_ops.take_timebase = mpc85xx_take_timebase; 514 #ifdef CONFIG_HOTPLUG_CPU 515 ppc_md.cpu_die = smp_85xx_mach_cpu_die; 516 #endif 517 } 518 519 smp_ops = &smp_85xx_ops; 520 521 #ifdef CONFIG_KEXEC 522 ppc_md.kexec_cpu_down = mpc85xx_smp_kexec_cpu_down; 523 ppc_md.machine_kexec = mpc85xx_smp_machine_kexec; 524 #endif 525 } 526