1 /* 2 * Xtensa SMP support functions. 3 * 4 * This file is subject to the terms and conditions of the GNU General Public 5 * License. See the file "COPYING" in the main directory of this archive 6 * for more details. 7 * 8 * Copyright (C) 2008 - 2013 Tensilica Inc. 9 * 10 * Chris Zankel <chris@zankel.net> 11 * Joe Taylor <joe@tensilica.com> 12 * Pete Delaney <piet@tensilica.com 13 */ 14 15 #include <linux/cpu.h> 16 #include <linux/cpumask.h> 17 #include <linux/delay.h> 18 #include <linux/init.h> 19 #include <linux/interrupt.h> 20 #include <linux/irqdomain.h> 21 #include <linux/irq.h> 22 #include <linux/kdebug.h> 23 #include <linux/module.h> 24 #include <linux/reboot.h> 25 #include <linux/seq_file.h> 26 #include <linux/smp.h> 27 #include <linux/thread_info.h> 28 29 #include <asm/cacheflush.h> 30 #include <asm/kdebug.h> 31 #include <asm/mmu_context.h> 32 #include <asm/mxregs.h> 33 #include <asm/platform.h> 34 #include <asm/tlbflush.h> 35 #include <asm/traps.h> 36 37 #ifdef CONFIG_SMP 38 # if XCHAL_HAVE_S32C1I == 0 39 # error "The S32C1I option is required for SMP." 40 # endif 41 #endif 42 43 static void system_invalidate_dcache_range(unsigned long start, 44 unsigned long size); 45 static void system_flush_invalidate_dcache_range(unsigned long start, 46 unsigned long size); 47 48 /* IPI (Inter Process Interrupt) */ 49 50 #define IPI_IRQ 0 51 52 static irqreturn_t ipi_interrupt(int irq, void *dev_id); 53 static struct irqaction ipi_irqaction = { 54 .handler = ipi_interrupt, 55 .flags = IRQF_PERCPU, 56 .name = "ipi", 57 }; 58 59 void ipi_init(void) 60 { 61 unsigned irq = irq_create_mapping(NULL, IPI_IRQ); 62 setup_irq(irq, &ipi_irqaction); 63 } 64 65 static inline unsigned int get_core_count(void) 66 { 67 /* Bits 18..21 of SYSCFGID contain the core count minus 1. */ 68 unsigned int syscfgid = get_er(SYSCFGID); 69 return ((syscfgid >> 18) & 0xf) + 1; 70 } 71 72 static inline int get_core_id(void) 73 { 74 /* Bits 0...18 of SYSCFGID contain the core id */ 75 unsigned int core_id = get_er(SYSCFGID); 76 return core_id & 0x3fff; 77 } 78 79 void __init smp_prepare_cpus(unsigned int max_cpus) 80 { 81 unsigned i; 82 83 for (i = 0; i < max_cpus; ++i) 84 set_cpu_present(i, true); 85 } 86 87 void __init smp_init_cpus(void) 88 { 89 unsigned i; 90 unsigned int ncpus = get_core_count(); 91 unsigned int core_id = get_core_id(); 92 93 pr_info("%s: Core Count = %d\n", __func__, ncpus); 94 pr_info("%s: Core Id = %d\n", __func__, core_id); 95 96 for (i = 0; i < ncpus; ++i) 97 set_cpu_possible(i, true); 98 } 99 100 void __init smp_prepare_boot_cpu(void) 101 { 102 unsigned int cpu = smp_processor_id(); 103 BUG_ON(cpu != 0); 104 cpu_asid_cache(cpu) = ASID_USER_FIRST; 105 } 106 107 void __init smp_cpus_done(unsigned int max_cpus) 108 { 109 } 110 111 static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */ 112 static DECLARE_COMPLETION(cpu_running); 113 114 void secondary_start_kernel(void) 115 { 116 struct mm_struct *mm = &init_mm; 117 unsigned int cpu = smp_processor_id(); 118 119 init_mmu(); 120 121 #ifdef CONFIG_DEBUG_KERNEL 122 if (boot_secondary_processors == 0) { 123 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n", 124 __func__, boot_secondary_processors, cpu); 125 for (;;) 126 __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL)); 127 } 128 129 pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n", 130 __func__, boot_secondary_processors, cpu); 131 #endif 132 /* Init EXCSAVE1 */ 133 134 secondary_trap_init(); 135 136 /* All kernel threads share the same mm context. */ 137 138 atomic_inc(&mm->mm_users); 139 atomic_inc(&mm->mm_count); 140 current->active_mm = mm; 141 cpumask_set_cpu(cpu, mm_cpumask(mm)); 142 enter_lazy_tlb(mm, current); 143 144 preempt_disable(); 145 trace_hardirqs_off(); 146 147 calibrate_delay(); 148 149 notify_cpu_starting(cpu); 150 151 secondary_init_irq(); 152 local_timer_setup(cpu); 153 154 set_cpu_online(cpu, true); 155 156 local_irq_enable(); 157 158 complete(&cpu_running); 159 160 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 161 } 162 163 static void mx_cpu_start(void *p) 164 { 165 unsigned cpu = (unsigned)p; 166 unsigned long run_stall_mask = get_er(MPSCORE); 167 168 set_er(run_stall_mask & ~(1u << cpu), MPSCORE); 169 pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n", 170 __func__, cpu, run_stall_mask, get_er(MPSCORE)); 171 } 172 173 static void mx_cpu_stop(void *p) 174 { 175 unsigned cpu = (unsigned)p; 176 unsigned long run_stall_mask = get_er(MPSCORE); 177 178 set_er(run_stall_mask | (1u << cpu), MPSCORE); 179 pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n", 180 __func__, cpu, run_stall_mask, get_er(MPSCORE)); 181 } 182 183 #ifdef CONFIG_HOTPLUG_CPU 184 unsigned long cpu_start_id __cacheline_aligned; 185 #endif 186 unsigned long cpu_start_ccount; 187 188 static int boot_secondary(unsigned int cpu, struct task_struct *ts) 189 { 190 unsigned long timeout = jiffies + msecs_to_jiffies(1000); 191 unsigned long ccount; 192 int i; 193 194 #ifdef CONFIG_HOTPLUG_CPU 195 cpu_start_id = cpu; 196 system_flush_invalidate_dcache_range( 197 (unsigned long)&cpu_start_id, sizeof(cpu_start_id)); 198 #endif 199 smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1); 200 201 for (i = 0; i < 2; ++i) { 202 do 203 ccount = get_ccount(); 204 while (!ccount); 205 206 cpu_start_ccount = ccount; 207 208 while (time_before(jiffies, timeout)) { 209 mb(); 210 if (!cpu_start_ccount) 211 break; 212 } 213 214 if (cpu_start_ccount) { 215 smp_call_function_single(0, mx_cpu_stop, 216 (void *)cpu, 1); 217 cpu_start_ccount = 0; 218 return -EIO; 219 } 220 } 221 return 0; 222 } 223 224 int __cpu_up(unsigned int cpu, struct task_struct *idle) 225 { 226 int ret = 0; 227 228 if (cpu_asid_cache(cpu) == 0) 229 cpu_asid_cache(cpu) = ASID_USER_FIRST; 230 231 start_info.stack = (unsigned long)task_pt_regs(idle); 232 wmb(); 233 234 pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n", 235 __func__, cpu, idle, start_info.stack); 236 237 ret = boot_secondary(cpu, idle); 238 if (ret == 0) { 239 wait_for_completion_timeout(&cpu_running, 240 msecs_to_jiffies(1000)); 241 if (!cpu_online(cpu)) 242 ret = -EIO; 243 } 244 245 if (ret) 246 pr_err("CPU %u failed to boot\n", cpu); 247 248 return ret; 249 } 250 251 #ifdef CONFIG_HOTPLUG_CPU 252 253 /* 254 * __cpu_disable runs on the processor to be shutdown. 255 */ 256 int __cpu_disable(void) 257 { 258 unsigned int cpu = smp_processor_id(); 259 260 /* 261 * Take this CPU offline. Once we clear this, we can't return, 262 * and we must not schedule until we're ready to give up the cpu. 263 */ 264 set_cpu_online(cpu, false); 265 266 /* 267 * OK - migrate IRQs away from this CPU 268 */ 269 migrate_irqs(); 270 271 /* 272 * Flush user cache and TLB mappings, and then remove this CPU 273 * from the vm mask set of all processes. 274 */ 275 local_flush_cache_all(); 276 local_flush_tlb_all(); 277 invalidate_page_directory(); 278 279 clear_tasks_mm_cpumask(cpu); 280 281 return 0; 282 } 283 284 static void platform_cpu_kill(unsigned int cpu) 285 { 286 smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true); 287 } 288 289 /* 290 * called on the thread which is asking for a CPU to be shutdown - 291 * waits until shutdown has completed, or it is timed out. 292 */ 293 void __cpu_die(unsigned int cpu) 294 { 295 unsigned long timeout = jiffies + msecs_to_jiffies(1000); 296 while (time_before(jiffies, timeout)) { 297 system_invalidate_dcache_range((unsigned long)&cpu_start_id, 298 sizeof(cpu_start_id)); 299 if (cpu_start_id == -cpu) { 300 platform_cpu_kill(cpu); 301 return; 302 } 303 } 304 pr_err("CPU%u: unable to kill\n", cpu); 305 } 306 307 void arch_cpu_idle_dead(void) 308 { 309 cpu_die(); 310 } 311 /* 312 * Called from the idle thread for the CPU which has been shutdown. 313 * 314 * Note that we disable IRQs here, but do not re-enable them 315 * before returning to the caller. This is also the behaviour 316 * of the other hotplug-cpu capable cores, so presumably coming 317 * out of idle fixes this. 318 */ 319 void __ref cpu_die(void) 320 { 321 idle_task_exit(); 322 local_irq_disable(); 323 __asm__ __volatile__( 324 " movi a2, cpu_restart\n" 325 " jx a2\n"); 326 } 327 328 #endif /* CONFIG_HOTPLUG_CPU */ 329 330 enum ipi_msg_type { 331 IPI_RESCHEDULE = 0, 332 IPI_CALL_FUNC, 333 IPI_CPU_STOP, 334 IPI_MAX 335 }; 336 337 static const struct { 338 const char *short_text; 339 const char *long_text; 340 } ipi_text[] = { 341 { .short_text = "RES", .long_text = "Rescheduling interrupts" }, 342 { .short_text = "CAL", .long_text = "Function call interrupts" }, 343 { .short_text = "DIE", .long_text = "CPU shutdown interrupts" }, 344 }; 345 346 struct ipi_data { 347 unsigned long ipi_count[IPI_MAX]; 348 }; 349 350 static DEFINE_PER_CPU(struct ipi_data, ipi_data); 351 352 static void send_ipi_message(const struct cpumask *callmask, 353 enum ipi_msg_type msg_id) 354 { 355 int index; 356 unsigned long mask = 0; 357 358 for_each_cpu(index, callmask) 359 if (index != smp_processor_id()) 360 mask |= 1 << index; 361 362 set_er(mask, MIPISET(msg_id)); 363 } 364 365 void arch_send_call_function_ipi_mask(const struct cpumask *mask) 366 { 367 send_ipi_message(mask, IPI_CALL_FUNC); 368 } 369 370 void arch_send_call_function_single_ipi(int cpu) 371 { 372 send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); 373 } 374 375 void smp_send_reschedule(int cpu) 376 { 377 send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); 378 } 379 380 void smp_send_stop(void) 381 { 382 struct cpumask targets; 383 384 cpumask_copy(&targets, cpu_online_mask); 385 cpumask_clear_cpu(smp_processor_id(), &targets); 386 send_ipi_message(&targets, IPI_CPU_STOP); 387 } 388 389 static void ipi_cpu_stop(unsigned int cpu) 390 { 391 set_cpu_online(cpu, false); 392 machine_halt(); 393 } 394 395 irqreturn_t ipi_interrupt(int irq, void *dev_id) 396 { 397 unsigned int cpu = smp_processor_id(); 398 struct ipi_data *ipi = &per_cpu(ipi_data, cpu); 399 unsigned int msg; 400 unsigned i; 401 402 msg = get_er(MIPICAUSE(cpu)); 403 for (i = 0; i < IPI_MAX; i++) 404 if (msg & (1 << i)) { 405 set_er(1 << i, MIPICAUSE(cpu)); 406 ++ipi->ipi_count[i]; 407 } 408 409 if (msg & (1 << IPI_RESCHEDULE)) 410 scheduler_ipi(); 411 if (msg & (1 << IPI_CALL_FUNC)) 412 generic_smp_call_function_interrupt(); 413 if (msg & (1 << IPI_CPU_STOP)) 414 ipi_cpu_stop(cpu); 415 416 return IRQ_HANDLED; 417 } 418 419 void show_ipi_list(struct seq_file *p, int prec) 420 { 421 unsigned int cpu; 422 unsigned i; 423 424 for (i = 0; i < IPI_MAX; ++i) { 425 seq_printf(p, "%*s:", prec, ipi_text[i].short_text); 426 for_each_online_cpu(cpu) 427 seq_printf(p, " %10lu", 428 per_cpu(ipi_data, cpu).ipi_count[i]); 429 seq_printf(p, " %s\n", ipi_text[i].long_text); 430 } 431 } 432 433 int setup_profiling_timer(unsigned int multiplier) 434 { 435 pr_debug("setup_profiling_timer %d\n", multiplier); 436 return 0; 437 } 438 439 /* TLB flush functions */ 440 441 struct flush_data { 442 struct vm_area_struct *vma; 443 unsigned long addr1; 444 unsigned long addr2; 445 }; 446 447 static void ipi_flush_tlb_all(void *arg) 448 { 449 local_flush_tlb_all(); 450 } 451 452 void flush_tlb_all(void) 453 { 454 on_each_cpu(ipi_flush_tlb_all, NULL, 1); 455 } 456 457 static void ipi_flush_tlb_mm(void *arg) 458 { 459 local_flush_tlb_mm(arg); 460 } 461 462 void flush_tlb_mm(struct mm_struct *mm) 463 { 464 on_each_cpu(ipi_flush_tlb_mm, mm, 1); 465 } 466 467 static void ipi_flush_tlb_page(void *arg) 468 { 469 struct flush_data *fd = arg; 470 local_flush_tlb_page(fd->vma, fd->addr1); 471 } 472 473 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) 474 { 475 struct flush_data fd = { 476 .vma = vma, 477 .addr1 = addr, 478 }; 479 on_each_cpu(ipi_flush_tlb_page, &fd, 1); 480 } 481 482 static void ipi_flush_tlb_range(void *arg) 483 { 484 struct flush_data *fd = arg; 485 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); 486 } 487 488 void flush_tlb_range(struct vm_area_struct *vma, 489 unsigned long start, unsigned long end) 490 { 491 struct flush_data fd = { 492 .vma = vma, 493 .addr1 = start, 494 .addr2 = end, 495 }; 496 on_each_cpu(ipi_flush_tlb_range, &fd, 1); 497 } 498 499 static void ipi_flush_tlb_kernel_range(void *arg) 500 { 501 struct flush_data *fd = arg; 502 local_flush_tlb_kernel_range(fd->addr1, fd->addr2); 503 } 504 505 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 506 { 507 struct flush_data fd = { 508 .addr1 = start, 509 .addr2 = end, 510 }; 511 on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1); 512 } 513 514 /* Cache flush functions */ 515 516 static void ipi_flush_cache_all(void *arg) 517 { 518 local_flush_cache_all(); 519 } 520 521 void flush_cache_all(void) 522 { 523 on_each_cpu(ipi_flush_cache_all, NULL, 1); 524 } 525 526 static void ipi_flush_cache_page(void *arg) 527 { 528 struct flush_data *fd = arg; 529 local_flush_cache_page(fd->vma, fd->addr1, fd->addr2); 530 } 531 532 void flush_cache_page(struct vm_area_struct *vma, 533 unsigned long address, unsigned long pfn) 534 { 535 struct flush_data fd = { 536 .vma = vma, 537 .addr1 = address, 538 .addr2 = pfn, 539 }; 540 on_each_cpu(ipi_flush_cache_page, &fd, 1); 541 } 542 543 static void ipi_flush_cache_range(void *arg) 544 { 545 struct flush_data *fd = arg; 546 local_flush_cache_range(fd->vma, fd->addr1, fd->addr2); 547 } 548 549 void flush_cache_range(struct vm_area_struct *vma, 550 unsigned long start, unsigned long end) 551 { 552 struct flush_data fd = { 553 .vma = vma, 554 .addr1 = start, 555 .addr2 = end, 556 }; 557 on_each_cpu(ipi_flush_cache_range, &fd, 1); 558 } 559 560 static void ipi_flush_icache_range(void *arg) 561 { 562 struct flush_data *fd = arg; 563 local_flush_icache_range(fd->addr1, fd->addr2); 564 } 565 566 void flush_icache_range(unsigned long start, unsigned long end) 567 { 568 struct flush_data fd = { 569 .addr1 = start, 570 .addr2 = end, 571 }; 572 on_each_cpu(ipi_flush_icache_range, &fd, 1); 573 } 574 EXPORT_SYMBOL(flush_icache_range); 575 576 /* ------------------------------------------------------------------------- */ 577 578 static void ipi_invalidate_dcache_range(void *arg) 579 { 580 struct flush_data *fd = arg; 581 __invalidate_dcache_range(fd->addr1, fd->addr2); 582 } 583 584 static void system_invalidate_dcache_range(unsigned long start, 585 unsigned long size) 586 { 587 struct flush_data fd = { 588 .addr1 = start, 589 .addr2 = size, 590 }; 591 on_each_cpu(ipi_invalidate_dcache_range, &fd, 1); 592 } 593 594 static void ipi_flush_invalidate_dcache_range(void *arg) 595 { 596 struct flush_data *fd = arg; 597 __flush_invalidate_dcache_range(fd->addr1, fd->addr2); 598 } 599 600 static void system_flush_invalidate_dcache_range(unsigned long start, 601 unsigned long size) 602 { 603 struct flush_data fd = { 604 .addr1 = start, 605 .addr2 = size, 606 }; 607 on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1); 608 } 609