1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 2000, 2001 Kanoj Sarcar 7 * Copyright (C) 2000, 2001 Ralf Baechle 8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc. 9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation 10 */ 11 #include <linux/acpi.h> 12 #include <linux/cpu.h> 13 #include <linux/cpumask.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/profile.h> 17 #include <linux/seq_file.h> 18 #include <linux/smp.h> 19 #include <linux/threads.h> 20 #include <linux/export.h> 21 #include <linux/syscore_ops.h> 22 #include <linux/time.h> 23 #include <linux/tracepoint.h> 24 #include <linux/sched/hotplug.h> 25 #include <linux/sched/task_stack.h> 26 27 #include <asm/cpu.h> 28 #include <asm/idle.h> 29 #include <asm/loongson.h> 30 #include <asm/mmu_context.h> 31 #include <asm/numa.h> 32 #include <asm/processor.h> 33 #include <asm/setup.h> 34 #include <asm/time.h> 35 36 int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ 37 EXPORT_SYMBOL(__cpu_number_map); 38 39 int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ 40 EXPORT_SYMBOL(__cpu_logical_map); 41 42 /* Representing the threads (siblings) of each logical CPU */ 43 cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; 44 EXPORT_SYMBOL(cpu_sibling_map); 45 46 /* Representing the core map of multi-core chips of each logical CPU */ 47 cpumask_t cpu_core_map[NR_CPUS] __read_mostly; 48 EXPORT_SYMBOL(cpu_core_map); 49 50 static DECLARE_COMPLETION(cpu_starting); 51 static DECLARE_COMPLETION(cpu_running); 52 53 /* 54 * A logcal cpu mask containing only one VPE per core to 55 * reduce the number of IPIs on large MT systems. 56 */ 57 cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; 58 EXPORT_SYMBOL(cpu_foreign_map); 59 60 /* representing cpus for which sibling maps can be computed */ 61 static cpumask_t cpu_sibling_setup_map; 62 63 /* representing cpus for which core maps can be computed */ 64 static cpumask_t cpu_core_setup_map; 65 66 struct secondary_data cpuboot_data; 67 static DEFINE_PER_CPU(int, cpu_state); 68 69 enum ipi_msg_type { 70 IPI_RESCHEDULE, 71 IPI_CALL_FUNCTION, 72 }; 73 74 static const char *ipi_types[NR_IPI] __tracepoint_string = { 75 [IPI_RESCHEDULE] = "Rescheduling interrupts", 76 [IPI_CALL_FUNCTION] = "Function call interrupts", 77 }; 78 79 void show_ipi_list(struct seq_file *p, int prec) 80 { 81 unsigned int cpu, i; 82 83 for (i = 0; i < NR_IPI; i++) { 84 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); 85 for_each_online_cpu(cpu) 86 seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).ipi_irqs[i]); 87 seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); 88 } 89 } 90 91 static inline void set_cpu_core_map(int cpu) 92 { 93 int i; 94 95 cpumask_set_cpu(cpu, &cpu_core_setup_map); 96 97 for_each_cpu(i, &cpu_core_setup_map) { 98 if (cpu_data[cpu].package == cpu_data[i].package) { 99 cpumask_set_cpu(i, &cpu_core_map[cpu]); 100 cpumask_set_cpu(cpu, &cpu_core_map[i]); 101 } 102 } 103 } 104 105 static inline void set_cpu_sibling_map(int cpu) 106 { 107 int i; 108 109 cpumask_set_cpu(cpu, &cpu_sibling_setup_map); 110 111 for_each_cpu(i, &cpu_sibling_setup_map) { 112 if (cpus_are_siblings(cpu, i)) { 113 cpumask_set_cpu(i, &cpu_sibling_map[cpu]); 114 cpumask_set_cpu(cpu, &cpu_sibling_map[i]); 115 } 116 } 117 } 118 119 static inline void clear_cpu_sibling_map(int cpu) 120 { 121 int i; 122 123 for_each_cpu(i, &cpu_sibling_setup_map) { 124 if (cpus_are_siblings(cpu, i)) { 125 cpumask_clear_cpu(i, &cpu_sibling_map[cpu]); 126 cpumask_clear_cpu(cpu, &cpu_sibling_map[i]); 127 } 128 } 129 130 cpumask_clear_cpu(cpu, &cpu_sibling_setup_map); 131 } 132 133 /* 134 * Calculate a new cpu_foreign_map mask whenever a 135 * new cpu appears or disappears. 136 */ 137 void calculate_cpu_foreign_map(void) 138 { 139 int i, k, core_present; 140 cpumask_t temp_foreign_map; 141 142 /* Re-calculate the mask */ 143 cpumask_clear(&temp_foreign_map); 144 for_each_online_cpu(i) { 145 core_present = 0; 146 for_each_cpu(k, &temp_foreign_map) 147 if (cpus_are_siblings(i, k)) 148 core_present = 1; 149 if (!core_present) 150 cpumask_set_cpu(i, &temp_foreign_map); 151 } 152 153 for_each_online_cpu(i) 154 cpumask_andnot(&cpu_foreign_map[i], 155 &temp_foreign_map, &cpu_sibling_map[i]); 156 } 157 158 /* Send mailbox buffer via Mail_Send */ 159 static void csr_mail_send(uint64_t data, int cpu, int mailbox) 160 { 161 uint64_t val; 162 163 /* Send high 32 bits */ 164 val = IOCSR_MBUF_SEND_BLOCKING; 165 val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 166 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 167 val |= (data & IOCSR_MBUF_SEND_H32_MASK); 168 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 169 170 /* Send low 32 bits */ 171 val = IOCSR_MBUF_SEND_BLOCKING; 172 val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 173 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 174 val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); 175 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 176 }; 177 178 static u32 ipi_read_clear(int cpu) 179 { 180 u32 action; 181 182 /* Load the ipi register to figure out what we're supposed to do */ 183 action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); 184 /* Clear the ipi register to clear the interrupt */ 185 iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); 186 wbflush(); 187 188 return action; 189 } 190 191 static void ipi_write_action(int cpu, u32 action) 192 { 193 unsigned int irq = 0; 194 195 while ((irq = ffs(action))) { 196 uint32_t val = IOCSR_IPI_SEND_BLOCKING; 197 198 val |= (irq - 1); 199 val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); 200 iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); 201 action &= ~BIT(irq - 1); 202 } 203 } 204 205 void loongson_send_ipi_single(int cpu, unsigned int action) 206 { 207 ipi_write_action(cpu_logical_map(cpu), (u32)action); 208 } 209 210 void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) 211 { 212 unsigned int i; 213 214 for_each_cpu(i, mask) 215 ipi_write_action(cpu_logical_map(i), (u32)action); 216 } 217 218 /* 219 * This function sends a 'reschedule' IPI to another CPU. 220 * it goes straight through and wastes no time serializing 221 * anything. Worst case is that we lose a reschedule ... 222 */ 223 void arch_smp_send_reschedule(int cpu) 224 { 225 loongson_send_ipi_single(cpu, SMP_RESCHEDULE); 226 } 227 EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); 228 229 irqreturn_t loongson_ipi_interrupt(int irq, void *dev) 230 { 231 unsigned int action; 232 unsigned int cpu = smp_processor_id(); 233 234 action = ipi_read_clear(cpu_logical_map(cpu)); 235 236 if (action & SMP_RESCHEDULE) { 237 scheduler_ipi(); 238 per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; 239 } 240 241 if (action & SMP_CALL_FUNCTION) { 242 generic_smp_call_function_interrupt(); 243 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; 244 } 245 246 return IRQ_HANDLED; 247 } 248 249 static void __init fdt_smp_setup(void) 250 { 251 #ifdef CONFIG_OF 252 unsigned int cpu, cpuid; 253 struct device_node *node = NULL; 254 255 for_each_of_cpu_node(node) { 256 if (!of_device_is_available(node)) 257 continue; 258 259 cpuid = of_get_cpu_hwid(node, 0); 260 if (cpuid >= nr_cpu_ids) 261 continue; 262 263 if (cpuid == loongson_sysconf.boot_cpu_id) { 264 cpu = 0; 265 numa_add_cpu(cpu); 266 } else { 267 cpu = cpumask_next_zero(-1, cpu_present_mask); 268 } 269 270 num_processors++; 271 set_cpu_possible(cpu, true); 272 set_cpu_present(cpu, true); 273 __cpu_number_map[cpuid] = cpu; 274 __cpu_logical_map[cpu] = cpuid; 275 } 276 277 loongson_sysconf.nr_cpus = num_processors; 278 set_bit(0, &(loongson_sysconf.cores_io_master)); 279 #endif 280 } 281 282 void __init loongson_smp_setup(void) 283 { 284 fdt_smp_setup(); 285 286 cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; 287 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 288 289 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 290 pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); 291 } 292 293 void __init loongson_prepare_cpus(unsigned int max_cpus) 294 { 295 int i = 0; 296 297 parse_acpi_topology(); 298 299 for (i = 0; i < loongson_sysconf.nr_cpus; i++) { 300 set_cpu_present(i, true); 301 csr_mail_send(0, __cpu_logical_map[i], 0); 302 cpu_data[i].global_id = __cpu_logical_map[i]; 303 } 304 305 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 306 } 307 308 /* 309 * Setup the PC, SP, and TP of a secondary processor and start it running! 310 */ 311 void loongson_boot_secondary(int cpu, struct task_struct *idle) 312 { 313 unsigned long entry; 314 315 pr_info("Booting CPU#%d...\n", cpu); 316 317 entry = __pa_symbol((unsigned long)&smpboot_entry); 318 cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle); 319 cpuboot_data.thread_info = (unsigned long)task_thread_info(idle); 320 321 csr_mail_send(entry, cpu_logical_map(cpu), 0); 322 323 loongson_send_ipi_single(cpu, SMP_BOOT_CPU); 324 } 325 326 /* 327 * SMP init and finish on secondary CPUs 328 */ 329 void loongson_init_secondary(void) 330 { 331 unsigned int cpu = smp_processor_id(); 332 unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | 333 ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER; 334 335 change_csr_ecfg(ECFG0_IM, imask); 336 337 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 338 339 #ifdef CONFIG_NUMA 340 numa_add_cpu(cpu); 341 #endif 342 per_cpu(cpu_state, cpu) = CPU_ONLINE; 343 cpu_data[cpu].package = 344 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 345 cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : 346 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; 347 } 348 349 void loongson_smp_finish(void) 350 { 351 local_irq_enable(); 352 iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); 353 pr_info("CPU#%d finished\n", smp_processor_id()); 354 } 355 356 #ifdef CONFIG_HOTPLUG_CPU 357 358 int loongson_cpu_disable(void) 359 { 360 unsigned long flags; 361 unsigned int cpu = smp_processor_id(); 362 363 if (io_master(cpu)) 364 return -EBUSY; 365 366 #ifdef CONFIG_NUMA 367 numa_remove_cpu(cpu); 368 #endif 369 set_cpu_online(cpu, false); 370 clear_cpu_sibling_map(cpu); 371 calculate_cpu_foreign_map(); 372 local_irq_save(flags); 373 irq_migrate_all_off_this_cpu(); 374 clear_csr_ecfg(ECFG0_IM); 375 local_irq_restore(flags); 376 local_flush_tlb_all(); 377 378 return 0; 379 } 380 381 void loongson_cpu_die(unsigned int cpu) 382 { 383 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 384 cpu_relax(); 385 386 mb(); 387 } 388 389 void __noreturn arch_cpu_idle_dead(void) 390 { 391 register uint64_t addr; 392 register void (*init_fn)(void); 393 394 idle_task_exit(); 395 local_irq_enable(); 396 set_csr_ecfg(ECFGF_IPI); 397 __this_cpu_write(cpu_state, CPU_DEAD); 398 399 __smp_mb(); 400 do { 401 __asm__ __volatile__("idle 0\n\t"); 402 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0); 403 } while (addr == 0); 404 405 local_irq_disable(); 406 init_fn = (void *)TO_CACHE(addr); 407 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); 408 409 init_fn(); 410 BUG(); 411 } 412 413 #endif 414 415 /* 416 * Power management 417 */ 418 #ifdef CONFIG_PM 419 420 static int loongson_ipi_suspend(void) 421 { 422 return 0; 423 } 424 425 static void loongson_ipi_resume(void) 426 { 427 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 428 } 429 430 static struct syscore_ops loongson_ipi_syscore_ops = { 431 .resume = loongson_ipi_resume, 432 .suspend = loongson_ipi_suspend, 433 }; 434 435 /* 436 * Enable boot cpu ipi before enabling nonboot cpus 437 * during syscore_resume. 438 */ 439 static int __init ipi_pm_init(void) 440 { 441 register_syscore_ops(&loongson_ipi_syscore_ops); 442 return 0; 443 } 444 445 core_initcall(ipi_pm_init); 446 #endif 447 448 /* Preload SMP state for boot cpu */ 449 void smp_prepare_boot_cpu(void) 450 { 451 unsigned int cpu, node, rr_node; 452 453 set_cpu_possible(0, true); 454 set_cpu_online(0, true); 455 set_my_cpu_offset(per_cpu_offset(0)); 456 457 rr_node = first_node(node_online_map); 458 for_each_possible_cpu(cpu) { 459 node = early_cpu_to_node(cpu); 460 461 /* 462 * The mapping between present cpus and nodes has been 463 * built during MADT and SRAT parsing. 464 * 465 * If possible cpus = present cpus here, early_cpu_to_node 466 * will return valid node. 467 * 468 * If possible cpus > present cpus here (e.g. some possible 469 * cpus will be added by cpu-hotplug later), for possible but 470 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE, 471 * and we just map them to online nodes in round-robin way. 472 * Once hotplugged, new correct mapping will be built for them. 473 */ 474 if (node != NUMA_NO_NODE) 475 set_cpu_numa_node(cpu, node); 476 else { 477 set_cpu_numa_node(cpu, rr_node); 478 rr_node = next_node_in(rr_node, node_online_map); 479 } 480 } 481 } 482 483 /* called from main before smp_init() */ 484 void __init smp_prepare_cpus(unsigned int max_cpus) 485 { 486 init_new_context(current, &init_mm); 487 current_thread_info()->cpu = 0; 488 loongson_prepare_cpus(max_cpus); 489 set_cpu_sibling_map(0); 490 set_cpu_core_map(0); 491 calculate_cpu_foreign_map(); 492 #ifndef CONFIG_HOTPLUG_CPU 493 init_cpu_present(cpu_possible_mask); 494 #endif 495 } 496 497 int __cpu_up(unsigned int cpu, struct task_struct *tidle) 498 { 499 loongson_boot_secondary(cpu, tidle); 500 501 /* Wait for CPU to start and be ready to sync counters */ 502 if (!wait_for_completion_timeout(&cpu_starting, 503 msecs_to_jiffies(5000))) { 504 pr_crit("CPU%u: failed to start\n", cpu); 505 return -EIO; 506 } 507 508 /* Wait for CPU to finish startup & mark itself online before return */ 509 wait_for_completion(&cpu_running); 510 511 return 0; 512 } 513 514 /* 515 * First C code run on the secondary CPUs after being started up by 516 * the master. 517 */ 518 asmlinkage void start_secondary(void) 519 { 520 unsigned int cpu; 521 522 sync_counter(); 523 cpu = raw_smp_processor_id(); 524 set_my_cpu_offset(per_cpu_offset(cpu)); 525 526 cpu_probe(); 527 constant_clockevent_init(); 528 loongson_init_secondary(); 529 530 set_cpu_sibling_map(cpu); 531 set_cpu_core_map(cpu); 532 533 notify_cpu_starting(cpu); 534 535 /* Notify boot CPU that we're starting */ 536 complete(&cpu_starting); 537 538 /* The CPU is running, now mark it online */ 539 set_cpu_online(cpu, true); 540 541 calculate_cpu_foreign_map(); 542 543 /* 544 * Notify boot CPU that we're up & online and it can safely return 545 * from __cpu_up() 546 */ 547 complete(&cpu_running); 548 549 /* 550 * irq will be enabled in loongson_smp_finish(), enabling it too 551 * early is dangerous. 552 */ 553 WARN_ON_ONCE(!irqs_disabled()); 554 loongson_smp_finish(); 555 556 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 557 } 558 559 void __init smp_cpus_done(unsigned int max_cpus) 560 { 561 } 562 563 static void stop_this_cpu(void *dummy) 564 { 565 set_cpu_online(smp_processor_id(), false); 566 calculate_cpu_foreign_map(); 567 local_irq_disable(); 568 while (true); 569 } 570 571 void smp_send_stop(void) 572 { 573 smp_call_function(stop_this_cpu, NULL, 0); 574 } 575 576 #ifdef CONFIG_PROFILING 577 int setup_profiling_timer(unsigned int multiplier) 578 { 579 return 0; 580 } 581 #endif 582 583 static void flush_tlb_all_ipi(void *info) 584 { 585 local_flush_tlb_all(); 586 } 587 588 void flush_tlb_all(void) 589 { 590 on_each_cpu(flush_tlb_all_ipi, NULL, 1); 591 } 592 593 static void flush_tlb_mm_ipi(void *mm) 594 { 595 local_flush_tlb_mm((struct mm_struct *)mm); 596 } 597 598 void flush_tlb_mm(struct mm_struct *mm) 599 { 600 if (atomic_read(&mm->mm_users) == 0) 601 return; /* happens as a result of exit_mmap() */ 602 603 preempt_disable(); 604 605 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 606 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); 607 } else { 608 unsigned int cpu; 609 610 for_each_online_cpu(cpu) { 611 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 612 cpu_context(cpu, mm) = 0; 613 } 614 local_flush_tlb_mm(mm); 615 } 616 617 preempt_enable(); 618 } 619 620 struct flush_tlb_data { 621 struct vm_area_struct *vma; 622 unsigned long addr1; 623 unsigned long addr2; 624 }; 625 626 static void flush_tlb_range_ipi(void *info) 627 { 628 struct flush_tlb_data *fd = info; 629 630 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); 631 } 632 633 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 634 { 635 struct mm_struct *mm = vma->vm_mm; 636 637 preempt_disable(); 638 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 639 struct flush_tlb_data fd = { 640 .vma = vma, 641 .addr1 = start, 642 .addr2 = end, 643 }; 644 645 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); 646 } else { 647 unsigned int cpu; 648 649 for_each_online_cpu(cpu) { 650 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 651 cpu_context(cpu, mm) = 0; 652 } 653 local_flush_tlb_range(vma, start, end); 654 } 655 preempt_enable(); 656 } 657 658 static void flush_tlb_kernel_range_ipi(void *info) 659 { 660 struct flush_tlb_data *fd = info; 661 662 local_flush_tlb_kernel_range(fd->addr1, fd->addr2); 663 } 664 665 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 666 { 667 struct flush_tlb_data fd = { 668 .addr1 = start, 669 .addr2 = end, 670 }; 671 672 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); 673 } 674 675 static void flush_tlb_page_ipi(void *info) 676 { 677 struct flush_tlb_data *fd = info; 678 679 local_flush_tlb_page(fd->vma, fd->addr1); 680 } 681 682 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 683 { 684 preempt_disable(); 685 if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { 686 struct flush_tlb_data fd = { 687 .vma = vma, 688 .addr1 = page, 689 }; 690 691 on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); 692 } else { 693 unsigned int cpu; 694 695 for_each_online_cpu(cpu) { 696 if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) 697 cpu_context(cpu, vma->vm_mm) = 0; 698 } 699 local_flush_tlb_page(vma, page); 700 } 701 preempt_enable(); 702 } 703 EXPORT_SYMBOL(flush_tlb_page); 704 705 static void flush_tlb_one_ipi(void *info) 706 { 707 unsigned long vaddr = (unsigned long) info; 708 709 local_flush_tlb_one(vaddr); 710 } 711 712 void flush_tlb_one(unsigned long vaddr) 713 { 714 on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1); 715 } 716 EXPORT_SYMBOL(flush_tlb_one); 717