1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2010, 2011, 2012, Lemote, Inc. 4 * Author: Chen Huacai, chenhc@lemote.com 5 */ 6 7 #include <linux/init.h> 8 #include <linux/cpu.h> 9 #include <linux/sched.h> 10 #include <linux/sched/hotplug.h> 11 #include <linux/sched/task_stack.h> 12 #include <linux/smp.h> 13 #include <linux/cpufreq.h> 14 #include <linux/kexec.h> 15 #include <asm/processor.h> 16 #include <asm/time.h> 17 #include <asm/clock.h> 18 #include <asm/tlbflush.h> 19 #include <asm/cacheflush.h> 20 #include <loongson.h> 21 #include <loongson_regs.h> 22 #include <workarounds.h> 23 24 #include "smp.h" 25 26 DEFINE_PER_CPU(int, cpu_state); 27 28 static void *ipi_set0_regs[16]; 29 static void *ipi_clear0_regs[16]; 30 static void *ipi_status0_regs[16]; 31 static void *ipi_en0_regs[16]; 32 static void *ipi_mailbox_buf[16]; 33 static uint32_t core0_c0count[NR_CPUS]; 34 35 /* read a 32bit value from ipi register */ 36 #define loongson3_ipi_read32(addr) readl(addr) 37 /* read a 64bit value from ipi register */ 38 #define loongson3_ipi_read64(addr) readq(addr) 39 /* write a 32bit value to ipi register */ 40 #define loongson3_ipi_write32(action, addr) \ 41 do { \ 42 writel(action, addr); \ 43 __wbflush(); \ 44 } while (0) 45 /* write a 64bit value to ipi register */ 46 #define loongson3_ipi_write64(action, addr) \ 47 do { \ 48 writeq(action, addr); \ 49 __wbflush(); \ 50 } while (0) 51 52 u32 (*ipi_read_clear)(int cpu); 53 void (*ipi_write_action)(int cpu, u32 action); 54 55 static u32 csr_ipi_read_clear(int cpu) 56 { 57 u32 action; 58 59 /* Load the ipi register to figure out what we're supposed to do */ 60 action = csr_readl(LOONGSON_CSR_IPI_STATUS); 61 /* Clear the ipi register to clear the interrupt */ 62 csr_writel(action, LOONGSON_CSR_IPI_CLEAR); 63 64 return action; 65 } 66 67 static void csr_ipi_write_action(int cpu, u32 action) 68 { 69 unsigned int irq = 0; 70 71 while ((irq = ffs(action))) { 72 uint32_t val = CSR_IPI_SEND_BLOCK; 73 val |= (irq - 1); 74 val |= (cpu << CSR_IPI_SEND_CPU_SHIFT); 75 csr_writel(val, LOONGSON_CSR_IPI_SEND); 76 action &= ~BIT(irq - 1); 77 } 78 } 79 80 static u32 legacy_ipi_read_clear(int cpu) 81 { 82 u32 action; 83 84 /* Load the ipi register to figure out what we're supposed to do */ 85 action = loongson3_ipi_read32(ipi_status0_regs[cpu_logical_map(cpu)]); 86 /* Clear the ipi register to clear the interrupt */ 87 loongson3_ipi_write32(action, ipi_clear0_regs[cpu_logical_map(cpu)]); 88 89 return action; 90 } 91 92 static void legacy_ipi_write_action(int cpu, u32 action) 93 { 94 loongson3_ipi_write32((u32)action, ipi_set0_regs[cpu]); 95 } 96 97 static void csr_ipi_probe(void) 98 { 99 if (cpu_has_csr() && csr_readl(LOONGSON_CSR_FEATURES) & LOONGSON_CSRF_IPI) { 100 ipi_read_clear = csr_ipi_read_clear; 101 ipi_write_action = csr_ipi_write_action; 102 } else { 103 ipi_read_clear = legacy_ipi_read_clear; 104 ipi_write_action = legacy_ipi_write_action; 105 } 106 } 107 108 static void ipi_set0_regs_init(void) 109 { 110 ipi_set0_regs[0] = (void *) 111 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + SET0); 112 ipi_set0_regs[1] = (void *) 113 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + SET0); 114 ipi_set0_regs[2] = (void *) 115 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + SET0); 116 ipi_set0_regs[3] = (void *) 117 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + SET0); 118 ipi_set0_regs[4] = (void *) 119 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + SET0); 120 ipi_set0_regs[5] = (void *) 121 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + SET0); 122 ipi_set0_regs[6] = (void *) 123 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + SET0); 124 ipi_set0_regs[7] = (void *) 125 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + SET0); 126 ipi_set0_regs[8] = (void *) 127 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + SET0); 128 ipi_set0_regs[9] = (void *) 129 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + SET0); 130 ipi_set0_regs[10] = (void *) 131 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + SET0); 132 ipi_set0_regs[11] = (void *) 133 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + SET0); 134 ipi_set0_regs[12] = (void *) 135 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + SET0); 136 ipi_set0_regs[13] = (void *) 137 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + SET0); 138 ipi_set0_regs[14] = (void *) 139 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + SET0); 140 ipi_set0_regs[15] = (void *) 141 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + SET0); 142 } 143 144 static void ipi_clear0_regs_init(void) 145 { 146 ipi_clear0_regs[0] = (void *) 147 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + CLEAR0); 148 ipi_clear0_regs[1] = (void *) 149 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + CLEAR0); 150 ipi_clear0_regs[2] = (void *) 151 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + CLEAR0); 152 ipi_clear0_regs[3] = (void *) 153 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + CLEAR0); 154 ipi_clear0_regs[4] = (void *) 155 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + CLEAR0); 156 ipi_clear0_regs[5] = (void *) 157 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + CLEAR0); 158 ipi_clear0_regs[6] = (void *) 159 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + CLEAR0); 160 ipi_clear0_regs[7] = (void *) 161 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + CLEAR0); 162 ipi_clear0_regs[8] = (void *) 163 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + CLEAR0); 164 ipi_clear0_regs[9] = (void *) 165 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + CLEAR0); 166 ipi_clear0_regs[10] = (void *) 167 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + CLEAR0); 168 ipi_clear0_regs[11] = (void *) 169 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + CLEAR0); 170 ipi_clear0_regs[12] = (void *) 171 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + CLEAR0); 172 ipi_clear0_regs[13] = (void *) 173 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + CLEAR0); 174 ipi_clear0_regs[14] = (void *) 175 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + CLEAR0); 176 ipi_clear0_regs[15] = (void *) 177 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + CLEAR0); 178 } 179 180 static void ipi_status0_regs_init(void) 181 { 182 ipi_status0_regs[0] = (void *) 183 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + STATUS0); 184 ipi_status0_regs[1] = (void *) 185 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + STATUS0); 186 ipi_status0_regs[2] = (void *) 187 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + STATUS0); 188 ipi_status0_regs[3] = (void *) 189 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + STATUS0); 190 ipi_status0_regs[4] = (void *) 191 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + STATUS0); 192 ipi_status0_regs[5] = (void *) 193 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + STATUS0); 194 ipi_status0_regs[6] = (void *) 195 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + STATUS0); 196 ipi_status0_regs[7] = (void *) 197 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + STATUS0); 198 ipi_status0_regs[8] = (void *) 199 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + STATUS0); 200 ipi_status0_regs[9] = (void *) 201 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + STATUS0); 202 ipi_status0_regs[10] = (void *) 203 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + STATUS0); 204 ipi_status0_regs[11] = (void *) 205 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + STATUS0); 206 ipi_status0_regs[12] = (void *) 207 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + STATUS0); 208 ipi_status0_regs[13] = (void *) 209 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + STATUS0); 210 ipi_status0_regs[14] = (void *) 211 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + STATUS0); 212 ipi_status0_regs[15] = (void *) 213 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + STATUS0); 214 } 215 216 static void ipi_en0_regs_init(void) 217 { 218 ipi_en0_regs[0] = (void *) 219 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + EN0); 220 ipi_en0_regs[1] = (void *) 221 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + EN0); 222 ipi_en0_regs[2] = (void *) 223 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + EN0); 224 ipi_en0_regs[3] = (void *) 225 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + EN0); 226 ipi_en0_regs[4] = (void *) 227 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + EN0); 228 ipi_en0_regs[5] = (void *) 229 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + EN0); 230 ipi_en0_regs[6] = (void *) 231 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + EN0); 232 ipi_en0_regs[7] = (void *) 233 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + EN0); 234 ipi_en0_regs[8] = (void *) 235 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + EN0); 236 ipi_en0_regs[9] = (void *) 237 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + EN0); 238 ipi_en0_regs[10] = (void *) 239 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + EN0); 240 ipi_en0_regs[11] = (void *) 241 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + EN0); 242 ipi_en0_regs[12] = (void *) 243 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + EN0); 244 ipi_en0_regs[13] = (void *) 245 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + EN0); 246 ipi_en0_regs[14] = (void *) 247 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + EN0); 248 ipi_en0_regs[15] = (void *) 249 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + EN0); 250 } 251 252 static void ipi_mailbox_buf_init(void) 253 { 254 ipi_mailbox_buf[0] = (void *) 255 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + BUF); 256 ipi_mailbox_buf[1] = (void *) 257 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + BUF); 258 ipi_mailbox_buf[2] = (void *) 259 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + BUF); 260 ipi_mailbox_buf[3] = (void *) 261 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + BUF); 262 ipi_mailbox_buf[4] = (void *) 263 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + BUF); 264 ipi_mailbox_buf[5] = (void *) 265 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + BUF); 266 ipi_mailbox_buf[6] = (void *) 267 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + BUF); 268 ipi_mailbox_buf[7] = (void *) 269 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + BUF); 270 ipi_mailbox_buf[8] = (void *) 271 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + BUF); 272 ipi_mailbox_buf[9] = (void *) 273 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + BUF); 274 ipi_mailbox_buf[10] = (void *) 275 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + BUF); 276 ipi_mailbox_buf[11] = (void *) 277 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + BUF); 278 ipi_mailbox_buf[12] = (void *) 279 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + BUF); 280 ipi_mailbox_buf[13] = (void *) 281 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + BUF); 282 ipi_mailbox_buf[14] = (void *) 283 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + BUF); 284 ipi_mailbox_buf[15] = (void *) 285 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + BUF); 286 } 287 288 /* 289 * Simple enough, just poke the appropriate ipi register 290 */ 291 static void loongson3_send_ipi_single(int cpu, unsigned int action) 292 { 293 ipi_write_action(cpu_logical_map(cpu), (u32)action); 294 } 295 296 static void 297 loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action) 298 { 299 unsigned int i; 300 301 for_each_cpu(i, mask) 302 ipi_write_action(cpu_logical_map(i), (u32)action); 303 } 304 305 #define IPI_IRQ_OFFSET 6 306 307 void loongson3_send_irq_by_ipi(int cpu, int irqs) 308 { 309 ipi_write_action(cpu_logical_map(cpu), irqs << IPI_IRQ_OFFSET); 310 } 311 312 void loongson3_ipi_interrupt(struct pt_regs *regs) 313 { 314 int i, cpu = smp_processor_id(); 315 unsigned int action, c0count, irqs; 316 317 action = ipi_read_clear(cpu); 318 irqs = action >> IPI_IRQ_OFFSET; 319 320 if (action & SMP_RESCHEDULE_YOURSELF) 321 scheduler_ipi(); 322 323 if (action & SMP_CALL_FUNCTION) { 324 irq_enter(); 325 generic_smp_call_function_interrupt(); 326 irq_exit(); 327 } 328 329 if (action & SMP_ASK_C0COUNT) { 330 BUG_ON(cpu != 0); 331 c0count = read_c0_count(); 332 c0count = c0count ? c0count : 1; 333 for (i = 1; i < nr_cpu_ids; i++) 334 core0_c0count[i] = c0count; 335 __wbflush(); /* Let others see the result ASAP */ 336 } 337 338 if (irqs) { 339 int irq; 340 while ((irq = ffs(irqs))) { 341 do_IRQ(irq-1); 342 irqs &= ~(1<<(irq-1)); 343 } 344 } 345 } 346 347 #define MAX_LOOPS 800 348 /* 349 * SMP init and finish on secondary CPUs 350 */ 351 static void loongson3_init_secondary(void) 352 { 353 int i; 354 uint32_t initcount; 355 unsigned int cpu = smp_processor_id(); 356 unsigned int imask = STATUSF_IP7 | STATUSF_IP6 | 357 STATUSF_IP3 | STATUSF_IP2; 358 359 /* Set interrupt mask, but don't enable */ 360 change_c0_status(ST0_IM, imask); 361 362 for (i = 0; i < num_possible_cpus(); i++) 363 loongson3_ipi_write32(0xffffffff, ipi_en0_regs[cpu_logical_map(i)]); 364 365 per_cpu(cpu_state, cpu) = CPU_ONLINE; 366 cpu_set_core(&cpu_data[cpu], 367 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package); 368 cpu_data[cpu].package = 369 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 370 371 i = 0; 372 core0_c0count[cpu] = 0; 373 loongson3_send_ipi_single(0, SMP_ASK_C0COUNT); 374 while (!core0_c0count[cpu]) { 375 i++; 376 cpu_relax(); 377 } 378 379 if (i > MAX_LOOPS) 380 i = MAX_LOOPS; 381 if (cpu_data[cpu].package) 382 initcount = core0_c0count[cpu] + i; 383 else /* Local access is faster for loops */ 384 initcount = core0_c0count[cpu] + i/2; 385 386 write_c0_count(initcount); 387 } 388 389 static void loongson3_smp_finish(void) 390 { 391 int cpu = smp_processor_id(); 392 393 write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ); 394 local_irq_enable(); 395 loongson3_ipi_write64(0, 396 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x0); 397 pr_info("CPU#%d finished, CP0_ST=%x\n", 398 smp_processor_id(), read_c0_status()); 399 } 400 401 static void __init loongson3_smp_setup(void) 402 { 403 int i = 0, num = 0; /* i: physical id, num: logical id */ 404 405 init_cpu_possible(cpu_none_mask); 406 407 /* For unified kernel, NR_CPUS is the maximum possible value, 408 * loongson_sysconf.nr_cpus is the really present value */ 409 while (i < loongson_sysconf.nr_cpus) { 410 if (loongson_sysconf.reserved_cpus_mask & (1<<i)) { 411 /* Reserved physical CPU cores */ 412 __cpu_number_map[i] = -1; 413 } else { 414 __cpu_number_map[i] = num; 415 __cpu_logical_map[num] = i; 416 set_cpu_possible(num, true); 417 num++; 418 } 419 i++; 420 } 421 pr_info("Detected %i available CPU(s)\n", num); 422 423 while (num < loongson_sysconf.nr_cpus) { 424 __cpu_logical_map[num] = -1; 425 num++; 426 } 427 428 csr_ipi_probe(); 429 ipi_set0_regs_init(); 430 ipi_clear0_regs_init(); 431 ipi_status0_regs_init(); 432 ipi_en0_regs_init(); 433 ipi_mailbox_buf_init(); 434 cpu_set_core(&cpu_data[0], 435 cpu_logical_map(0) % loongson_sysconf.cores_per_package); 436 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 437 } 438 439 static void __init loongson3_prepare_cpus(unsigned int max_cpus) 440 { 441 init_cpu_present(cpu_possible_mask); 442 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 443 } 444 445 /* 446 * Setup the PC, SP, and GP of a secondary processor and start it runing! 447 */ 448 static int loongson3_boot_secondary(int cpu, struct task_struct *idle) 449 { 450 unsigned long startargs[4]; 451 452 pr_info("Booting CPU#%d...\n", cpu); 453 454 /* startargs[] are initial PC, SP and GP for secondary CPU */ 455 startargs[0] = (unsigned long)&smp_bootstrap; 456 startargs[1] = (unsigned long)__KSTK_TOS(idle); 457 startargs[2] = (unsigned long)task_thread_info(idle); 458 startargs[3] = 0; 459 460 pr_debug("CPU#%d, func_pc=%lx, sp=%lx, gp=%lx\n", 461 cpu, startargs[0], startargs[1], startargs[2]); 462 463 loongson3_ipi_write64(startargs[3], 464 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x18); 465 loongson3_ipi_write64(startargs[2], 466 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x10); 467 loongson3_ipi_write64(startargs[1], 468 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x8); 469 loongson3_ipi_write64(startargs[0], 470 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x0); 471 return 0; 472 } 473 474 #ifdef CONFIG_HOTPLUG_CPU 475 476 static int loongson3_cpu_disable(void) 477 { 478 unsigned long flags; 479 unsigned int cpu = smp_processor_id(); 480 481 if (cpu == 0) 482 return -EBUSY; 483 484 set_cpu_online(cpu, false); 485 calculate_cpu_foreign_map(); 486 local_irq_save(flags); 487 fixup_irqs(); 488 local_irq_restore(flags); 489 local_flush_tlb_all(); 490 491 return 0; 492 } 493 494 495 static void loongson3_cpu_die(unsigned int cpu) 496 { 497 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 498 cpu_relax(); 499 500 mb(); 501 } 502 503 /* To shutdown a core in Loongson 3, the target core should go to CKSEG1 and 504 * flush all L1 entries at first. Then, another core (usually Core 0) can 505 * safely disable the clock of the target core. loongson3_play_dead() is 506 * called via CKSEG1 (uncached and unmmaped) */ 507 static void loongson3_type1_play_dead(int *state_addr) 508 { 509 register int val; 510 register long cpuid, core, node, count; 511 register void *addr, *base, *initfunc; 512 513 __asm__ __volatile__( 514 " .set push \n" 515 " .set noreorder \n" 516 " li %[addr], 0x80000000 \n" /* KSEG0 */ 517 "1: cache 0, 0(%[addr]) \n" /* flush L1 ICache */ 518 " cache 0, 1(%[addr]) \n" 519 " cache 0, 2(%[addr]) \n" 520 " cache 0, 3(%[addr]) \n" 521 " cache 1, 0(%[addr]) \n" /* flush L1 DCache */ 522 " cache 1, 1(%[addr]) \n" 523 " cache 1, 2(%[addr]) \n" 524 " cache 1, 3(%[addr]) \n" 525 " addiu %[sets], %[sets], -1 \n" 526 " bnez %[sets], 1b \n" 527 " addiu %[addr], %[addr], 0x20 \n" 528 " li %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ 529 " sw %[val], (%[state_addr]) \n" 530 " sync \n" 531 " cache 21, (%[state_addr]) \n" /* flush entry of *state_addr */ 532 " .set pop \n" 533 : [addr] "=&r" (addr), [val] "=&r" (val) 534 : [state_addr] "r" (state_addr), 535 [sets] "r" (cpu_data[smp_processor_id()].dcache.sets)); 536 537 __asm__ __volatile__( 538 " .set push \n" 539 " .set noreorder \n" 540 " .set mips64 \n" 541 " mfc0 %[cpuid], $15, 1 \n" 542 " andi %[cpuid], 0x3ff \n" 543 " dli %[base], 0x900000003ff01000 \n" 544 " andi %[core], %[cpuid], 0x3 \n" 545 " sll %[core], 8 \n" /* get core id */ 546 " or %[base], %[base], %[core] \n" 547 " andi %[node], %[cpuid], 0xc \n" 548 " dsll %[node], 42 \n" /* get node id */ 549 " or %[base], %[base], %[node] \n" 550 "1: li %[count], 0x100 \n" /* wait for init loop */ 551 "2: bnez %[count], 2b \n" /* limit mailbox access */ 552 " addiu %[count], -1 \n" 553 " ld %[initfunc], 0x20(%[base]) \n" /* get PC via mailbox */ 554 " beqz %[initfunc], 1b \n" 555 " nop \n" 556 " ld $sp, 0x28(%[base]) \n" /* get SP via mailbox */ 557 " ld $gp, 0x30(%[base]) \n" /* get GP via mailbox */ 558 " ld $a1, 0x38(%[base]) \n" 559 " jr %[initfunc] \n" /* jump to initial PC */ 560 " nop \n" 561 " .set pop \n" 562 : [core] "=&r" (core), [node] "=&r" (node), 563 [base] "=&r" (base), [cpuid] "=&r" (cpuid), 564 [count] "=&r" (count), [initfunc] "=&r" (initfunc) 565 : /* No Input */ 566 : "a1"); 567 } 568 569 static void loongson3_type2_play_dead(int *state_addr) 570 { 571 register int val; 572 register long cpuid, core, node, count; 573 register void *addr, *base, *initfunc; 574 575 __asm__ __volatile__( 576 " .set push \n" 577 " .set noreorder \n" 578 " li %[addr], 0x80000000 \n" /* KSEG0 */ 579 "1: cache 0, 0(%[addr]) \n" /* flush L1 ICache */ 580 " cache 0, 1(%[addr]) \n" 581 " cache 0, 2(%[addr]) \n" 582 " cache 0, 3(%[addr]) \n" 583 " cache 1, 0(%[addr]) \n" /* flush L1 DCache */ 584 " cache 1, 1(%[addr]) \n" 585 " cache 1, 2(%[addr]) \n" 586 " cache 1, 3(%[addr]) \n" 587 " addiu %[sets], %[sets], -1 \n" 588 " bnez %[sets], 1b \n" 589 " addiu %[addr], %[addr], 0x20 \n" 590 " li %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ 591 " sw %[val], (%[state_addr]) \n" 592 " sync \n" 593 " cache 21, (%[state_addr]) \n" /* flush entry of *state_addr */ 594 " .set pop \n" 595 : [addr] "=&r" (addr), [val] "=&r" (val) 596 : [state_addr] "r" (state_addr), 597 [sets] "r" (cpu_data[smp_processor_id()].dcache.sets)); 598 599 __asm__ __volatile__( 600 " .set push \n" 601 " .set noreorder \n" 602 " .set mips64 \n" 603 " mfc0 %[cpuid], $15, 1 \n" 604 " andi %[cpuid], 0x3ff \n" 605 " dli %[base], 0x900000003ff01000 \n" 606 " andi %[core], %[cpuid], 0x3 \n" 607 " sll %[core], 8 \n" /* get core id */ 608 " or %[base], %[base], %[core] \n" 609 " andi %[node], %[cpuid], 0xc \n" 610 " dsll %[node], 42 \n" /* get node id */ 611 " or %[base], %[base], %[node] \n" 612 " dsrl %[node], 30 \n" /* 15:14 */ 613 " or %[base], %[base], %[node] \n" 614 "1: li %[count], 0x100 \n" /* wait for init loop */ 615 "2: bnez %[count], 2b \n" /* limit mailbox access */ 616 " addiu %[count], -1 \n" 617 " ld %[initfunc], 0x20(%[base]) \n" /* get PC via mailbox */ 618 " beqz %[initfunc], 1b \n" 619 " nop \n" 620 " ld $sp, 0x28(%[base]) \n" /* get SP via mailbox */ 621 " ld $gp, 0x30(%[base]) \n" /* get GP via mailbox */ 622 " ld $a1, 0x38(%[base]) \n" 623 " jr %[initfunc] \n" /* jump to initial PC */ 624 " nop \n" 625 " .set pop \n" 626 : [core] "=&r" (core), [node] "=&r" (node), 627 [base] "=&r" (base), [cpuid] "=&r" (cpuid), 628 [count] "=&r" (count), [initfunc] "=&r" (initfunc) 629 : /* No Input */ 630 : "a1"); 631 } 632 633 static void loongson3_type3_play_dead(int *state_addr) 634 { 635 register int val; 636 register long cpuid, core, node, count; 637 register void *addr, *base, *initfunc; 638 639 __asm__ __volatile__( 640 " .set push \n" 641 " .set noreorder \n" 642 " li %[addr], 0x80000000 \n" /* KSEG0 */ 643 "1: cache 0, 0(%[addr]) \n" /* flush L1 ICache */ 644 " cache 0, 1(%[addr]) \n" 645 " cache 0, 2(%[addr]) \n" 646 " cache 0, 3(%[addr]) \n" 647 " cache 1, 0(%[addr]) \n" /* flush L1 DCache */ 648 " cache 1, 1(%[addr]) \n" 649 " cache 1, 2(%[addr]) \n" 650 " cache 1, 3(%[addr]) \n" 651 " addiu %[sets], %[sets], -1 \n" 652 " bnez %[sets], 1b \n" 653 " addiu %[addr], %[addr], 0x40 \n" 654 " li %[addr], 0x80000000 \n" /* KSEG0 */ 655 "2: cache 2, 0(%[addr]) \n" /* flush L1 VCache */ 656 " cache 2, 1(%[addr]) \n" 657 " cache 2, 2(%[addr]) \n" 658 " cache 2, 3(%[addr]) \n" 659 " cache 2, 4(%[addr]) \n" 660 " cache 2, 5(%[addr]) \n" 661 " cache 2, 6(%[addr]) \n" 662 " cache 2, 7(%[addr]) \n" 663 " cache 2, 8(%[addr]) \n" 664 " cache 2, 9(%[addr]) \n" 665 " cache 2, 10(%[addr]) \n" 666 " cache 2, 11(%[addr]) \n" 667 " cache 2, 12(%[addr]) \n" 668 " cache 2, 13(%[addr]) \n" 669 " cache 2, 14(%[addr]) \n" 670 " cache 2, 15(%[addr]) \n" 671 " addiu %[vsets], %[vsets], -1 \n" 672 " bnez %[vsets], 2b \n" 673 " addiu %[addr], %[addr], 0x40 \n" 674 " li %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ 675 " sw %[val], (%[state_addr]) \n" 676 " sync \n" 677 " cache 21, (%[state_addr]) \n" /* flush entry of *state_addr */ 678 " .set pop \n" 679 : [addr] "=&r" (addr), [val] "=&r" (val) 680 : [state_addr] "r" (state_addr), 681 [sets] "r" (cpu_data[smp_processor_id()].dcache.sets), 682 [vsets] "r" (cpu_data[smp_processor_id()].vcache.sets)); 683 684 __asm__ __volatile__( 685 " .set push \n" 686 " .set noreorder \n" 687 " .set mips64 \n" 688 " mfc0 %[cpuid], $15, 1 \n" 689 " andi %[cpuid], 0x3ff \n" 690 " dli %[base], 0x900000003ff01000 \n" 691 " andi %[core], %[cpuid], 0x3 \n" 692 " sll %[core], 8 \n" /* get core id */ 693 " or %[base], %[base], %[core] \n" 694 " andi %[node], %[cpuid], 0xc \n" 695 " dsll %[node], 42 \n" /* get node id */ 696 " or %[base], %[base], %[node] \n" 697 "1: li %[count], 0x100 \n" /* wait for init loop */ 698 "2: bnez %[count], 2b \n" /* limit mailbox access */ 699 " addiu %[count], -1 \n" 700 " ld %[initfunc], 0x20(%[base]) \n" /* get PC via mailbox */ 701 " beqz %[initfunc], 1b \n" 702 " nop \n" 703 " ld $sp, 0x28(%[base]) \n" /* get SP via mailbox */ 704 " ld $gp, 0x30(%[base]) \n" /* get GP via mailbox */ 705 " ld $a1, 0x38(%[base]) \n" 706 " jr %[initfunc] \n" /* jump to initial PC */ 707 " nop \n" 708 " .set pop \n" 709 : [core] "=&r" (core), [node] "=&r" (node), 710 [base] "=&r" (base), [cpuid] "=&r" (cpuid), 711 [count] "=&r" (count), [initfunc] "=&r" (initfunc) 712 : /* No Input */ 713 : "a1"); 714 } 715 716 void play_dead(void) 717 { 718 int prid_imp, prid_rev, *state_addr; 719 unsigned int cpu = smp_processor_id(); 720 void (*play_dead_at_ckseg1)(int *); 721 722 idle_task_exit(); 723 724 prid_imp = read_c0_prid() & PRID_IMP_MASK; 725 prid_rev = read_c0_prid() & PRID_REV_MASK; 726 727 if (prid_imp == PRID_IMP_LOONGSON_64G) { 728 play_dead_at_ckseg1 = 729 (void *)CKSEG1ADDR((unsigned long)loongson3_type3_play_dead); 730 goto out; 731 } 732 733 switch (prid_rev) { 734 case PRID_REV_LOONGSON3A_R1: 735 default: 736 play_dead_at_ckseg1 = 737 (void *)CKSEG1ADDR((unsigned long)loongson3_type1_play_dead); 738 break; 739 case PRID_REV_LOONGSON3B_R1: 740 case PRID_REV_LOONGSON3B_R2: 741 play_dead_at_ckseg1 = 742 (void *)CKSEG1ADDR((unsigned long)loongson3_type2_play_dead); 743 break; 744 case PRID_REV_LOONGSON3A_R2_0: 745 case PRID_REV_LOONGSON3A_R2_1: 746 case PRID_REV_LOONGSON3A_R3_0: 747 case PRID_REV_LOONGSON3A_R3_1: 748 play_dead_at_ckseg1 = 749 (void *)CKSEG1ADDR((unsigned long)loongson3_type3_play_dead); 750 break; 751 } 752 753 out: 754 state_addr = &per_cpu(cpu_state, cpu); 755 mb(); 756 play_dead_at_ckseg1(state_addr); 757 } 758 759 static int loongson3_disable_clock(unsigned int cpu) 760 { 761 uint64_t core_id = cpu_core(&cpu_data[cpu]); 762 uint64_t package_id = cpu_data[cpu].package; 763 764 if ((read_c0_prid() & PRID_REV_MASK) == PRID_REV_LOONGSON3A_R1) { 765 LOONGSON_CHIPCFG(package_id) &= ~(1 << (12 + core_id)); 766 } else { 767 if (!(loongson_sysconf.workarounds & WORKAROUND_CPUHOTPLUG)) 768 LOONGSON_FREQCTRL(package_id) &= ~(1 << (core_id * 4 + 3)); 769 } 770 return 0; 771 } 772 773 static int loongson3_enable_clock(unsigned int cpu) 774 { 775 uint64_t core_id = cpu_core(&cpu_data[cpu]); 776 uint64_t package_id = cpu_data[cpu].package; 777 778 if ((read_c0_prid() & PRID_REV_MASK) == PRID_REV_LOONGSON3A_R1) { 779 LOONGSON_CHIPCFG(package_id) |= 1 << (12 + core_id); 780 } else { 781 if (!(loongson_sysconf.workarounds & WORKAROUND_CPUHOTPLUG)) 782 LOONGSON_FREQCTRL(package_id) |= 1 << (core_id * 4 + 3); 783 } 784 return 0; 785 } 786 787 static int register_loongson3_notifier(void) 788 { 789 return cpuhp_setup_state_nocalls(CPUHP_MIPS_SOC_PREPARE, 790 "mips/loongson:prepare", 791 loongson3_enable_clock, 792 loongson3_disable_clock); 793 } 794 early_initcall(register_loongson3_notifier); 795 796 #endif 797 798 const struct plat_smp_ops loongson3_smp_ops = { 799 .send_ipi_single = loongson3_send_ipi_single, 800 .send_ipi_mask = loongson3_send_ipi_mask, 801 .init_secondary = loongson3_init_secondary, 802 .smp_finish = loongson3_smp_finish, 803 .boot_secondary = loongson3_boot_secondary, 804 .smp_setup = loongson3_smp_setup, 805 .prepare_cpus = loongson3_prepare_cpus, 806 #ifdef CONFIG_HOTPLUG_CPU 807 .cpu_disable = loongson3_cpu_disable, 808 .cpu_die = loongson3_cpu_die, 809 #endif 810 #ifdef CONFIG_KEXEC 811 .kexec_nonboot_cpu = kexec_nonboot_cpu_jump, 812 #endif 813 }; 814