1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/arch/alpha/kernel/smp.c 4 * 5 * 2001-07-09 Phil Ezolt (Phillip.Ezolt@compaq.com) 6 * Renamed modified smp_call_function to smp_call_function_on_cpu() 7 * Created an function that conforms to the old calling convention 8 * of smp_call_function(). 9 * 10 * This is helpful for DCPI. 11 * 12 */ 13 14 #include <linux/errno.h> 15 #include <linux/kernel.h> 16 #include <linux/kernel_stat.h> 17 #include <linux/module.h> 18 #include <linux/sched/mm.h> 19 #include <linux/mm.h> 20 #include <linux/err.h> 21 #include <linux/threads.h> 22 #include <linux/smp.h> 23 #include <linux/interrupt.h> 24 #include <linux/init.h> 25 #include <linux/delay.h> 26 #include <linux/spinlock.h> 27 #include <linux/irq.h> 28 #include <linux/cache.h> 29 #include <linux/profile.h> 30 #include <linux/bitops.h> 31 #include <linux/cpu.h> 32 33 #include <asm/hwrpb.h> 34 #include <asm/ptrace.h> 35 #include <linux/atomic.h> 36 37 #include <asm/io.h> 38 #include <asm/irq.h> 39 #include <asm/mmu_context.h> 40 #include <asm/tlbflush.h> 41 42 #include "proto.h" 43 #include "irq_impl.h" 44 45 46 #define DEBUG_SMP 0 47 #if DEBUG_SMP 48 #define DBGS(args) printk args 49 #else 50 #define DBGS(args) 51 #endif 52 53 /* A collection of per-processor data. */ 54 struct cpuinfo_alpha cpu_data[NR_CPUS]; 55 EXPORT_SYMBOL(cpu_data); 56 57 /* A collection of single bit ipi messages. */ 58 static struct { 59 unsigned long bits ____cacheline_aligned; 60 } ipi_data[NR_CPUS] __cacheline_aligned; 61 62 enum ipi_message_type { 63 IPI_RESCHEDULE, 64 IPI_CALL_FUNC, 65 IPI_CPU_STOP, 66 }; 67 68 /* Set to a secondary's cpuid when it comes online. */ 69 static int smp_secondary_alive = 0; 70 71 int smp_num_probed; /* Internal processor count */ 72 int smp_num_cpus = 1; /* Number that came online. */ 73 EXPORT_SYMBOL(smp_num_cpus); 74 75 /* 76 * Called by both boot and secondaries to move global data into 77 * per-processor storage. 78 */ 79 static inline void __init 80 smp_store_cpu_info(int cpuid) 81 { 82 cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy; 83 cpu_data[cpuid].last_asn = ASN_FIRST_VERSION; 84 cpu_data[cpuid].need_new_asn = 0; 85 cpu_data[cpuid].asn_lock = 0; 86 } 87 88 /* 89 * Ideally sets up per-cpu profiling hooks. Doesn't do much now... 90 */ 91 static inline void __init 92 smp_setup_percpu_timer(int cpuid) 93 { 94 cpu_data[cpuid].prof_counter = 1; 95 cpu_data[cpuid].prof_multiplier = 1; 96 } 97 98 static void __init 99 wait_boot_cpu_to_stop(int cpuid) 100 { 101 unsigned long stop = jiffies + 10*HZ; 102 103 while (time_before(jiffies, stop)) { 104 if (!smp_secondary_alive) 105 return; 106 barrier(); 107 } 108 109 printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid); 110 for (;;) 111 barrier(); 112 } 113 114 /* 115 * Where secondaries begin a life of C. 116 */ 117 void __init 118 smp_callin(void) 119 { 120 int cpuid = hard_smp_processor_id(); 121 122 if (cpu_online(cpuid)) { 123 printk("??, cpu 0x%x already present??\n", cpuid); 124 BUG(); 125 } 126 set_cpu_online(cpuid, true); 127 128 /* Turn on machine checks. */ 129 wrmces(7); 130 131 /* Set trap vectors. */ 132 trap_init(); 133 134 /* Set interrupt vector. */ 135 wrent(entInt, 0); 136 137 /* Get our local ticker going. */ 138 smp_setup_percpu_timer(cpuid); 139 init_clockevent(); 140 141 /* Call platform-specific callin, if specified */ 142 if (alpha_mv.smp_callin) 143 alpha_mv.smp_callin(); 144 145 /* All kernel threads share the same mm context. */ 146 mmgrab(&init_mm); 147 current->active_mm = &init_mm; 148 149 /* inform the notifiers about the new cpu */ 150 notify_cpu_starting(cpuid); 151 152 /* Must have completely accurate bogos. */ 153 local_irq_enable(); 154 155 /* Wait boot CPU to stop with irq enabled before running 156 calibrate_delay. */ 157 wait_boot_cpu_to_stop(cpuid); 158 mb(); 159 calibrate_delay(); 160 161 smp_store_cpu_info(cpuid); 162 /* Allow master to continue only after we written loops_per_jiffy. */ 163 wmb(); 164 smp_secondary_alive = 1; 165 166 DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n", 167 cpuid, current, current->active_mm)); 168 169 preempt_disable(); 170 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 171 } 172 173 /* Wait until hwrpb->txrdy is clear for cpu. Return -1 on timeout. */ 174 static int 175 wait_for_txrdy (unsigned long cpumask) 176 { 177 unsigned long timeout; 178 179 if (!(hwrpb->txrdy & cpumask)) 180 return 0; 181 182 timeout = jiffies + 10*HZ; 183 while (time_before(jiffies, timeout)) { 184 if (!(hwrpb->txrdy & cpumask)) 185 return 0; 186 udelay(10); 187 barrier(); 188 } 189 190 return -1; 191 } 192 193 /* 194 * Send a message to a secondary's console. "START" is one such 195 * interesting message. ;-) 196 */ 197 static void 198 send_secondary_console_msg(char *str, int cpuid) 199 { 200 struct percpu_struct *cpu; 201 register char *cp1, *cp2; 202 unsigned long cpumask; 203 size_t len; 204 205 cpu = (struct percpu_struct *) 206 ((char*)hwrpb 207 + hwrpb->processor_offset 208 + cpuid * hwrpb->processor_size); 209 210 cpumask = (1UL << cpuid); 211 if (wait_for_txrdy(cpumask)) 212 goto timeout; 213 214 cp2 = str; 215 len = strlen(cp2); 216 *(unsigned int *)&cpu->ipc_buffer[0] = len; 217 cp1 = (char *) &cpu->ipc_buffer[1]; 218 memcpy(cp1, cp2, len); 219 220 /* atomic test and set */ 221 wmb(); 222 set_bit(cpuid, &hwrpb->rxrdy); 223 224 if (wait_for_txrdy(cpumask)) 225 goto timeout; 226 return; 227 228 timeout: 229 printk("Processor %x not ready\n", cpuid); 230 } 231 232 /* 233 * A secondary console wants to send a message. Receive it. 234 */ 235 static void 236 recv_secondary_console_msg(void) 237 { 238 int mycpu, i, cnt; 239 unsigned long txrdy = hwrpb->txrdy; 240 char *cp1, *cp2, buf[80]; 241 struct percpu_struct *cpu; 242 243 DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy)); 244 245 mycpu = hard_smp_processor_id(); 246 247 for (i = 0; i < NR_CPUS; i++) { 248 if (!(txrdy & (1UL << i))) 249 continue; 250 251 DBGS(("recv_secondary_console_msg: " 252 "TXRDY contains CPU %d.\n", i)); 253 254 cpu = (struct percpu_struct *) 255 ((char*)hwrpb 256 + hwrpb->processor_offset 257 + i * hwrpb->processor_size); 258 259 DBGS(("recv_secondary_console_msg: on %d from %d" 260 " HALT_REASON 0x%lx FLAGS 0x%lx\n", 261 mycpu, i, cpu->halt_reason, cpu->flags)); 262 263 cnt = cpu->ipc_buffer[0] >> 32; 264 if (cnt <= 0 || cnt >= 80) 265 strcpy(buf, "<<< BOGUS MSG >>>"); 266 else { 267 cp1 = (char *) &cpu->ipc_buffer[1]; 268 cp2 = buf; 269 memcpy(cp2, cp1, cnt); 270 cp2[cnt] = '\0'; 271 272 while ((cp2 = strchr(cp2, '\r')) != 0) { 273 *cp2 = ' '; 274 if (cp2[1] == '\n') 275 cp2[1] = ' '; 276 } 277 } 278 279 DBGS((KERN_INFO "recv_secondary_console_msg: on %d " 280 "message is '%s'\n", mycpu, buf)); 281 } 282 283 hwrpb->txrdy = 0; 284 } 285 286 /* 287 * Convince the console to have a secondary cpu begin execution. 288 */ 289 static int 290 secondary_cpu_start(int cpuid, struct task_struct *idle) 291 { 292 struct percpu_struct *cpu; 293 struct pcb_struct *hwpcb, *ipcb; 294 unsigned long timeout; 295 296 cpu = (struct percpu_struct *) 297 ((char*)hwrpb 298 + hwrpb->processor_offset 299 + cpuid * hwrpb->processor_size); 300 hwpcb = (struct pcb_struct *) cpu->hwpcb; 301 ipcb = &task_thread_info(idle)->pcb; 302 303 /* Initialize the CPU's HWPCB to something just good enough for 304 us to get started. Immediately after starting, we'll swpctx 305 to the target idle task's pcb. Reuse the stack in the mean 306 time. Precalculate the target PCBB. */ 307 hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16; 308 hwpcb->usp = 0; 309 hwpcb->ptbr = ipcb->ptbr; 310 hwpcb->pcc = 0; 311 hwpcb->asn = 0; 312 hwpcb->unique = virt_to_phys(ipcb); 313 hwpcb->flags = ipcb->flags; 314 hwpcb->res1 = hwpcb->res2 = 0; 315 316 #if 0 317 DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n", 318 hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique)); 319 #endif 320 DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n", 321 cpuid, idle->state, ipcb->flags)); 322 323 /* Setup HWRPB fields that SRM uses to activate secondary CPU */ 324 hwrpb->CPU_restart = __smp_callin; 325 hwrpb->CPU_restart_data = (unsigned long) __smp_callin; 326 327 /* Recalculate and update the HWRPB checksum */ 328 hwrpb_update_checksum(hwrpb); 329 330 /* 331 * Send a "start" command to the specified processor. 332 */ 333 334 /* SRM III 3.4.1.3 */ 335 cpu->flags |= 0x22; /* turn on Context Valid and Restart Capable */ 336 cpu->flags &= ~1; /* turn off Bootstrap In Progress */ 337 wmb(); 338 339 send_secondary_console_msg("START\r\n", cpuid); 340 341 /* Wait 10 seconds for an ACK from the console. */ 342 timeout = jiffies + 10*HZ; 343 while (time_before(jiffies, timeout)) { 344 if (cpu->flags & 1) 345 goto started; 346 udelay(10); 347 barrier(); 348 } 349 printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid); 350 return -1; 351 352 started: 353 DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid)); 354 return 0; 355 } 356 357 /* 358 * Bring one cpu online. 359 */ 360 static int 361 smp_boot_one_cpu(int cpuid, struct task_struct *idle) 362 { 363 unsigned long timeout; 364 365 /* Signal the secondary to wait a moment. */ 366 smp_secondary_alive = -1; 367 368 /* Whirrr, whirrr, whirrrrrrrrr... */ 369 if (secondary_cpu_start(cpuid, idle)) 370 return -1; 371 372 /* Notify the secondary CPU it can run calibrate_delay. */ 373 mb(); 374 smp_secondary_alive = 0; 375 376 /* We've been acked by the console; wait one second for 377 the task to start up for real. */ 378 timeout = jiffies + 1*HZ; 379 while (time_before(jiffies, timeout)) { 380 if (smp_secondary_alive == 1) 381 goto alive; 382 udelay(10); 383 barrier(); 384 } 385 386 /* We failed to boot the CPU. */ 387 388 printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid); 389 return -1; 390 391 alive: 392 /* Another "Red Snapper". */ 393 return 0; 394 } 395 396 /* 397 * Called from setup_arch. Detect an SMP system and which processors 398 * are present. 399 */ 400 void __init 401 setup_smp(void) 402 { 403 struct percpu_struct *cpubase, *cpu; 404 unsigned long i; 405 406 if (boot_cpuid != 0) { 407 printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n", 408 boot_cpuid); 409 } 410 411 if (hwrpb->nr_processors > 1) { 412 int boot_cpu_palrev; 413 414 DBGS(("setup_smp: nr_processors %ld\n", 415 hwrpb->nr_processors)); 416 417 cpubase = (struct percpu_struct *) 418 ((char*)hwrpb + hwrpb->processor_offset); 419 boot_cpu_palrev = cpubase->pal_revision; 420 421 for (i = 0; i < hwrpb->nr_processors; i++) { 422 cpu = (struct percpu_struct *) 423 ((char *)cpubase + i*hwrpb->processor_size); 424 if ((cpu->flags & 0x1cc) == 0x1cc) { 425 smp_num_probed++; 426 set_cpu_possible(i, true); 427 set_cpu_present(i, true); 428 cpu->pal_revision = boot_cpu_palrev; 429 } 430 431 DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n", 432 i, cpu->flags, cpu->type)); 433 DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n", 434 i, cpu->pal_revision)); 435 } 436 } else { 437 smp_num_probed = 1; 438 } 439 440 printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n", 441 smp_num_probed, cpumask_bits(cpu_present_mask)[0]); 442 } 443 444 /* 445 * Called by smp_init prepare the secondaries 446 */ 447 void __init 448 smp_prepare_cpus(unsigned int max_cpus) 449 { 450 /* Take care of some initial bookkeeping. */ 451 memset(ipi_data, 0, sizeof(ipi_data)); 452 453 current_thread_info()->cpu = boot_cpuid; 454 455 smp_store_cpu_info(boot_cpuid); 456 smp_setup_percpu_timer(boot_cpuid); 457 458 /* Nothing to do on a UP box, or when told not to. */ 459 if (smp_num_probed == 1 || max_cpus == 0) { 460 init_cpu_possible(cpumask_of(boot_cpuid)); 461 init_cpu_present(cpumask_of(boot_cpuid)); 462 printk(KERN_INFO "SMP mode deactivated.\n"); 463 return; 464 } 465 466 printk(KERN_INFO "SMP starting up secondaries.\n"); 467 468 smp_num_cpus = smp_num_probed; 469 } 470 471 void 472 smp_prepare_boot_cpu(void) 473 { 474 } 475 476 int 477 __cpu_up(unsigned int cpu, struct task_struct *tidle) 478 { 479 smp_boot_one_cpu(cpu, tidle); 480 481 return cpu_online(cpu) ? 0 : -ENOSYS; 482 } 483 484 void __init 485 smp_cpus_done(unsigned int max_cpus) 486 { 487 int cpu; 488 unsigned long bogosum = 0; 489 490 for(cpu = 0; cpu < NR_CPUS; cpu++) 491 if (cpu_online(cpu)) 492 bogosum += cpu_data[cpu].loops_per_jiffy; 493 494 printk(KERN_INFO "SMP: Total of %d processors activated " 495 "(%lu.%02lu BogoMIPS).\n", 496 num_online_cpus(), 497 (bogosum + 2500) / (500000/HZ), 498 ((bogosum + 2500) / (5000/HZ)) % 100); 499 } 500 501 int 502 setup_profiling_timer(unsigned int multiplier) 503 { 504 return -EINVAL; 505 } 506 507 static void 508 send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation) 509 { 510 int i; 511 512 mb(); 513 for_each_cpu(i, to_whom) 514 set_bit(operation, &ipi_data[i].bits); 515 516 mb(); 517 for_each_cpu(i, to_whom) 518 wripir(i); 519 } 520 521 void 522 handle_ipi(struct pt_regs *regs) 523 { 524 int this_cpu = smp_processor_id(); 525 unsigned long *pending_ipis = &ipi_data[this_cpu].bits; 526 unsigned long ops; 527 528 #if 0 529 DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n", 530 this_cpu, *pending_ipis, regs->pc)); 531 #endif 532 533 mb(); /* Order interrupt and bit testing. */ 534 while ((ops = xchg(pending_ipis, 0)) != 0) { 535 mb(); /* Order bit clearing and data access. */ 536 do { 537 unsigned long which; 538 539 which = ops & -ops; 540 ops &= ~which; 541 which = __ffs(which); 542 543 switch (which) { 544 case IPI_RESCHEDULE: 545 scheduler_ipi(); 546 break; 547 548 case IPI_CALL_FUNC: 549 generic_smp_call_function_interrupt(); 550 break; 551 552 case IPI_CPU_STOP: 553 halt(); 554 555 default: 556 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", 557 this_cpu, which); 558 break; 559 } 560 } while (ops); 561 562 mb(); /* Order data access and bit testing. */ 563 } 564 565 cpu_data[this_cpu].ipi_count++; 566 567 if (hwrpb->txrdy) 568 recv_secondary_console_msg(); 569 } 570 571 void 572 smp_send_reschedule(int cpu) 573 { 574 #ifdef DEBUG_IPI_MSG 575 if (cpu == hard_smp_processor_id()) 576 printk(KERN_WARNING 577 "smp_send_reschedule: Sending IPI to self.\n"); 578 #endif 579 send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); 580 } 581 582 void 583 smp_send_stop(void) 584 { 585 cpumask_t to_whom; 586 cpumask_copy(&to_whom, cpu_possible_mask); 587 cpumask_clear_cpu(smp_processor_id(), &to_whom); 588 #ifdef DEBUG_IPI_MSG 589 if (hard_smp_processor_id() != boot_cpu_id) 590 printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n"); 591 #endif 592 send_ipi_message(&to_whom, IPI_CPU_STOP); 593 } 594 595 void arch_send_call_function_ipi_mask(const struct cpumask *mask) 596 { 597 send_ipi_message(mask, IPI_CALL_FUNC); 598 } 599 600 void arch_send_call_function_single_ipi(int cpu) 601 { 602 send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); 603 } 604 605 static void 606 ipi_imb(void *ignored) 607 { 608 imb(); 609 } 610 611 void 612 smp_imb(void) 613 { 614 /* Must wait other processors to flush their icache before continue. */ 615 on_each_cpu(ipi_imb, NULL, 1); 616 } 617 EXPORT_SYMBOL(smp_imb); 618 619 static void 620 ipi_flush_tlb_all(void *ignored) 621 { 622 tbia(); 623 } 624 625 void 626 flush_tlb_all(void) 627 { 628 /* Although we don't have any data to pass, we do want to 629 synchronize with the other processors. */ 630 on_each_cpu(ipi_flush_tlb_all, NULL, 1); 631 } 632 633 #define asn_locked() (cpu_data[smp_processor_id()].asn_lock) 634 635 static void 636 ipi_flush_tlb_mm(void *x) 637 { 638 struct mm_struct *mm = (struct mm_struct *) x; 639 if (mm == current->active_mm && !asn_locked()) 640 flush_tlb_current(mm); 641 else 642 flush_tlb_other(mm); 643 } 644 645 void 646 flush_tlb_mm(struct mm_struct *mm) 647 { 648 preempt_disable(); 649 650 if (mm == current->active_mm) { 651 flush_tlb_current(mm); 652 if (atomic_read(&mm->mm_users) <= 1) { 653 int cpu, this_cpu = smp_processor_id(); 654 for (cpu = 0; cpu < NR_CPUS; cpu++) { 655 if (!cpu_online(cpu) || cpu == this_cpu) 656 continue; 657 if (mm->context[cpu]) 658 mm->context[cpu] = 0; 659 } 660 preempt_enable(); 661 return; 662 } 663 } 664 665 smp_call_function(ipi_flush_tlb_mm, mm, 1); 666 667 preempt_enable(); 668 } 669 EXPORT_SYMBOL(flush_tlb_mm); 670 671 struct flush_tlb_page_struct { 672 struct vm_area_struct *vma; 673 struct mm_struct *mm; 674 unsigned long addr; 675 }; 676 677 static void 678 ipi_flush_tlb_page(void *x) 679 { 680 struct flush_tlb_page_struct *data = (struct flush_tlb_page_struct *)x; 681 struct mm_struct * mm = data->mm; 682 683 if (mm == current->active_mm && !asn_locked()) 684 flush_tlb_current_page(mm, data->vma, data->addr); 685 else 686 flush_tlb_other(mm); 687 } 688 689 void 690 flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) 691 { 692 struct flush_tlb_page_struct data; 693 struct mm_struct *mm = vma->vm_mm; 694 695 preempt_disable(); 696 697 if (mm == current->active_mm) { 698 flush_tlb_current_page(mm, vma, addr); 699 if (atomic_read(&mm->mm_users) <= 1) { 700 int cpu, this_cpu = smp_processor_id(); 701 for (cpu = 0; cpu < NR_CPUS; cpu++) { 702 if (!cpu_online(cpu) || cpu == this_cpu) 703 continue; 704 if (mm->context[cpu]) 705 mm->context[cpu] = 0; 706 } 707 preempt_enable(); 708 return; 709 } 710 } 711 712 data.vma = vma; 713 data.mm = mm; 714 data.addr = addr; 715 716 smp_call_function(ipi_flush_tlb_page, &data, 1); 717 718 preempt_enable(); 719 } 720 EXPORT_SYMBOL(flush_tlb_page); 721 722 void 723 flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 724 { 725 /* On the Alpha we always flush the whole user tlb. */ 726 flush_tlb_mm(vma->vm_mm); 727 } 728 EXPORT_SYMBOL(flush_tlb_range); 729 730 static void 731 ipi_flush_icache_page(void *x) 732 { 733 struct mm_struct *mm = (struct mm_struct *) x; 734 if (mm == current->active_mm && !asn_locked()) 735 __load_new_mm_context(mm); 736 else 737 flush_tlb_other(mm); 738 } 739 740 void 741 flush_icache_user_page(struct vm_area_struct *vma, struct page *page, 742 unsigned long addr, int len) 743 { 744 struct mm_struct *mm = vma->vm_mm; 745 746 if ((vma->vm_flags & VM_EXEC) == 0) 747 return; 748 749 preempt_disable(); 750 751 if (mm == current->active_mm) { 752 __load_new_mm_context(mm); 753 if (atomic_read(&mm->mm_users) <= 1) { 754 int cpu, this_cpu = smp_processor_id(); 755 for (cpu = 0; cpu < NR_CPUS; cpu++) { 756 if (!cpu_online(cpu) || cpu == this_cpu) 757 continue; 758 if (mm->context[cpu]) 759 mm->context[cpu] = 0; 760 } 761 preempt_enable(); 762 return; 763 } 764 } 765 766 smp_call_function(ipi_flush_icache_page, mm, 1); 767 768 preempt_enable(); 769 } 770