1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2010, 2011, 2012, Lemote, Inc. 4 * Author: Chen Huacai, chenhc@lemote.com 5 */ 6 7 #include <irq.h> 8 #include <linux/init.h> 9 #include <linux/cpu.h> 10 #include <linux/sched.h> 11 #include <linux/sched/hotplug.h> 12 #include <linux/sched/task_stack.h> 13 #include <linux/smp.h> 14 #include <linux/cpufreq.h> 15 #include <linux/kexec.h> 16 #include <asm/processor.h> 17 #include <asm/time.h> 18 #include <asm/tlbflush.h> 19 #include <asm/cacheflush.h> 20 #include <loongson.h> 21 #include <loongson_regs.h> 22 #include <workarounds.h> 23 24 #include "smp.h" 25 26 DEFINE_PER_CPU(int, cpu_state); 27 28 #define LS_IPI_IRQ (MIPS_CPU_IRQ_BASE + 6) 29 30 static void *ipi_set0_regs[16]; 31 static void *ipi_clear0_regs[16]; 32 static void *ipi_status0_regs[16]; 33 static void *ipi_en0_regs[16]; 34 static void *ipi_mailbox_buf[16]; 35 static uint32_t core0_c0count[NR_CPUS]; 36 37 /* read a 32bit value from ipi register */ 38 #define loongson3_ipi_read32(addr) readl(addr) 39 /* read a 64bit value from ipi register */ 40 #define loongson3_ipi_read64(addr) readq(addr) 41 /* write a 32bit value to ipi register */ 42 #define loongson3_ipi_write32(action, addr) \ 43 do { \ 44 writel(action, addr); \ 45 __wbflush(); \ 46 } while (0) 47 /* write a 64bit value to ipi register */ 48 #define loongson3_ipi_write64(action, addr) \ 49 do { \ 50 writeq(action, addr); \ 51 __wbflush(); \ 52 } while (0) 53 54 u32 (*ipi_read_clear)(int cpu); 55 void (*ipi_write_action)(int cpu, u32 action); 56 57 static u32 csr_ipi_read_clear(int cpu) 58 { 59 u32 action; 60 61 /* Load the ipi register to figure out what we're supposed to do */ 62 action = csr_readl(LOONGSON_CSR_IPI_STATUS); 63 /* Clear the ipi register to clear the interrupt */ 64 csr_writel(action, LOONGSON_CSR_IPI_CLEAR); 65 66 return action; 67 } 68 69 static void csr_ipi_write_action(int cpu, u32 action) 70 { 71 unsigned int irq = 0; 72 73 while ((irq = ffs(action))) { 74 uint32_t val = CSR_IPI_SEND_BLOCK; 75 val |= (irq - 1); 76 val |= (cpu << CSR_IPI_SEND_CPU_SHIFT); 77 csr_writel(val, LOONGSON_CSR_IPI_SEND); 78 action &= ~BIT(irq - 1); 79 } 80 } 81 82 static u32 legacy_ipi_read_clear(int cpu) 83 { 84 u32 action; 85 86 /* Load the ipi register to figure out what we're supposed to do */ 87 action = loongson3_ipi_read32(ipi_status0_regs[cpu_logical_map(cpu)]); 88 /* Clear the ipi register to clear the interrupt */ 89 loongson3_ipi_write32(action, ipi_clear0_regs[cpu_logical_map(cpu)]); 90 91 return action; 92 } 93 94 static void legacy_ipi_write_action(int cpu, u32 action) 95 { 96 loongson3_ipi_write32((u32)action, ipi_set0_regs[cpu]); 97 } 98 99 static void csr_ipi_probe(void) 100 { 101 if (cpu_has_csr() && csr_readl(LOONGSON_CSR_FEATURES) & LOONGSON_CSRF_IPI) { 102 ipi_read_clear = csr_ipi_read_clear; 103 ipi_write_action = csr_ipi_write_action; 104 } else { 105 ipi_read_clear = legacy_ipi_read_clear; 106 ipi_write_action = legacy_ipi_write_action; 107 } 108 } 109 110 static void ipi_set0_regs_init(void) 111 { 112 ipi_set0_regs[0] = (void *) 113 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + SET0); 114 ipi_set0_regs[1] = (void *) 115 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + SET0); 116 ipi_set0_regs[2] = (void *) 117 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + SET0); 118 ipi_set0_regs[3] = (void *) 119 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + SET0); 120 ipi_set0_regs[4] = (void *) 121 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + SET0); 122 ipi_set0_regs[5] = (void *) 123 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + SET0); 124 ipi_set0_regs[6] = (void *) 125 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + SET0); 126 ipi_set0_regs[7] = (void *) 127 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + SET0); 128 ipi_set0_regs[8] = (void *) 129 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + SET0); 130 ipi_set0_regs[9] = (void *) 131 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + SET0); 132 ipi_set0_regs[10] = (void *) 133 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + SET0); 134 ipi_set0_regs[11] = (void *) 135 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + SET0); 136 ipi_set0_regs[12] = (void *) 137 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + SET0); 138 ipi_set0_regs[13] = (void *) 139 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + SET0); 140 ipi_set0_regs[14] = (void *) 141 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + SET0); 142 ipi_set0_regs[15] = (void *) 143 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + SET0); 144 } 145 146 static void ipi_clear0_regs_init(void) 147 { 148 ipi_clear0_regs[0] = (void *) 149 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + CLEAR0); 150 ipi_clear0_regs[1] = (void *) 151 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + CLEAR0); 152 ipi_clear0_regs[2] = (void *) 153 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + CLEAR0); 154 ipi_clear0_regs[3] = (void *) 155 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + CLEAR0); 156 ipi_clear0_regs[4] = (void *) 157 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + CLEAR0); 158 ipi_clear0_regs[5] = (void *) 159 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + CLEAR0); 160 ipi_clear0_regs[6] = (void *) 161 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + CLEAR0); 162 ipi_clear0_regs[7] = (void *) 163 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + CLEAR0); 164 ipi_clear0_regs[8] = (void *) 165 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + CLEAR0); 166 ipi_clear0_regs[9] = (void *) 167 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + CLEAR0); 168 ipi_clear0_regs[10] = (void *) 169 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + CLEAR0); 170 ipi_clear0_regs[11] = (void *) 171 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + CLEAR0); 172 ipi_clear0_regs[12] = (void *) 173 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + CLEAR0); 174 ipi_clear0_regs[13] = (void *) 175 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + CLEAR0); 176 ipi_clear0_regs[14] = (void *) 177 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + CLEAR0); 178 ipi_clear0_regs[15] = (void *) 179 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + CLEAR0); 180 } 181 182 static void ipi_status0_regs_init(void) 183 { 184 ipi_status0_regs[0] = (void *) 185 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + STATUS0); 186 ipi_status0_regs[1] = (void *) 187 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + STATUS0); 188 ipi_status0_regs[2] = (void *) 189 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + STATUS0); 190 ipi_status0_regs[3] = (void *) 191 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + STATUS0); 192 ipi_status0_regs[4] = (void *) 193 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + STATUS0); 194 ipi_status0_regs[5] = (void *) 195 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + STATUS0); 196 ipi_status0_regs[6] = (void *) 197 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + STATUS0); 198 ipi_status0_regs[7] = (void *) 199 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + STATUS0); 200 ipi_status0_regs[8] = (void *) 201 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + STATUS0); 202 ipi_status0_regs[9] = (void *) 203 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + STATUS0); 204 ipi_status0_regs[10] = (void *) 205 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + STATUS0); 206 ipi_status0_regs[11] = (void *) 207 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + STATUS0); 208 ipi_status0_regs[12] = (void *) 209 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + STATUS0); 210 ipi_status0_regs[13] = (void *) 211 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + STATUS0); 212 ipi_status0_regs[14] = (void *) 213 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + STATUS0); 214 ipi_status0_regs[15] = (void *) 215 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + STATUS0); 216 } 217 218 static void ipi_en0_regs_init(void) 219 { 220 ipi_en0_regs[0] = (void *) 221 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + EN0); 222 ipi_en0_regs[1] = (void *) 223 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + EN0); 224 ipi_en0_regs[2] = (void *) 225 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + EN0); 226 ipi_en0_regs[3] = (void *) 227 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + EN0); 228 ipi_en0_regs[4] = (void *) 229 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + EN0); 230 ipi_en0_regs[5] = (void *) 231 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + EN0); 232 ipi_en0_regs[6] = (void *) 233 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + EN0); 234 ipi_en0_regs[7] = (void *) 235 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + EN0); 236 ipi_en0_regs[8] = (void *) 237 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + EN0); 238 ipi_en0_regs[9] = (void *) 239 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + EN0); 240 ipi_en0_regs[10] = (void *) 241 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + EN0); 242 ipi_en0_regs[11] = (void *) 243 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + EN0); 244 ipi_en0_regs[12] = (void *) 245 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + EN0); 246 ipi_en0_regs[13] = (void *) 247 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + EN0); 248 ipi_en0_regs[14] = (void *) 249 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + EN0); 250 ipi_en0_regs[15] = (void *) 251 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + EN0); 252 } 253 254 static void ipi_mailbox_buf_init(void) 255 { 256 ipi_mailbox_buf[0] = (void *) 257 (SMP_CORE_GROUP0_BASE + SMP_CORE0_OFFSET + BUF); 258 ipi_mailbox_buf[1] = (void *) 259 (SMP_CORE_GROUP0_BASE + SMP_CORE1_OFFSET + BUF); 260 ipi_mailbox_buf[2] = (void *) 261 (SMP_CORE_GROUP0_BASE + SMP_CORE2_OFFSET + BUF); 262 ipi_mailbox_buf[3] = (void *) 263 (SMP_CORE_GROUP0_BASE + SMP_CORE3_OFFSET + BUF); 264 ipi_mailbox_buf[4] = (void *) 265 (SMP_CORE_GROUP1_BASE + SMP_CORE0_OFFSET + BUF); 266 ipi_mailbox_buf[5] = (void *) 267 (SMP_CORE_GROUP1_BASE + SMP_CORE1_OFFSET + BUF); 268 ipi_mailbox_buf[6] = (void *) 269 (SMP_CORE_GROUP1_BASE + SMP_CORE2_OFFSET + BUF); 270 ipi_mailbox_buf[7] = (void *) 271 (SMP_CORE_GROUP1_BASE + SMP_CORE3_OFFSET + BUF); 272 ipi_mailbox_buf[8] = (void *) 273 (SMP_CORE_GROUP2_BASE + SMP_CORE0_OFFSET + BUF); 274 ipi_mailbox_buf[9] = (void *) 275 (SMP_CORE_GROUP2_BASE + SMP_CORE1_OFFSET + BUF); 276 ipi_mailbox_buf[10] = (void *) 277 (SMP_CORE_GROUP2_BASE + SMP_CORE2_OFFSET + BUF); 278 ipi_mailbox_buf[11] = (void *) 279 (SMP_CORE_GROUP2_BASE + SMP_CORE3_OFFSET + BUF); 280 ipi_mailbox_buf[12] = (void *) 281 (SMP_CORE_GROUP3_BASE + SMP_CORE0_OFFSET + BUF); 282 ipi_mailbox_buf[13] = (void *) 283 (SMP_CORE_GROUP3_BASE + SMP_CORE1_OFFSET + BUF); 284 ipi_mailbox_buf[14] = (void *) 285 (SMP_CORE_GROUP3_BASE + SMP_CORE2_OFFSET + BUF); 286 ipi_mailbox_buf[15] = (void *) 287 (SMP_CORE_GROUP3_BASE + SMP_CORE3_OFFSET + BUF); 288 } 289 290 /* 291 * Simple enough, just poke the appropriate ipi register 292 */ 293 static void loongson3_send_ipi_single(int cpu, unsigned int action) 294 { 295 ipi_write_action(cpu_logical_map(cpu), (u32)action); 296 } 297 298 static void 299 loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action) 300 { 301 unsigned int i; 302 303 for_each_cpu(i, mask) 304 ipi_write_action(cpu_logical_map(i), (u32)action); 305 } 306 307 308 static irqreturn_t loongson3_ipi_interrupt(int irq, void *dev_id) 309 { 310 int i, cpu = smp_processor_id(); 311 unsigned int action, c0count; 312 313 action = ipi_read_clear(cpu); 314 315 if (action & SMP_RESCHEDULE_YOURSELF) 316 scheduler_ipi(); 317 318 if (action & SMP_CALL_FUNCTION) { 319 irq_enter(); 320 generic_smp_call_function_interrupt(); 321 irq_exit(); 322 } 323 324 if (action & SMP_ASK_C0COUNT) { 325 BUG_ON(cpu != 0); 326 c0count = read_c0_count(); 327 c0count = c0count ? c0count : 1; 328 for (i = 1; i < nr_cpu_ids; i++) 329 core0_c0count[i] = c0count; 330 __wbflush(); /* Let others see the result ASAP */ 331 } 332 333 return IRQ_HANDLED; 334 } 335 336 #define MAX_LOOPS 800 337 /* 338 * SMP init and finish on secondary CPUs 339 */ 340 static void loongson3_init_secondary(void) 341 { 342 int i; 343 uint32_t initcount; 344 unsigned int cpu = smp_processor_id(); 345 unsigned int imask = STATUSF_IP7 | STATUSF_IP6 | 346 STATUSF_IP3 | STATUSF_IP2; 347 348 /* Set interrupt mask, but don't enable */ 349 change_c0_status(ST0_IM, imask); 350 351 for (i = 0; i < num_possible_cpus(); i++) 352 loongson3_ipi_write32(0xffffffff, ipi_en0_regs[cpu_logical_map(i)]); 353 354 per_cpu(cpu_state, cpu) = CPU_ONLINE; 355 cpu_set_core(&cpu_data[cpu], 356 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package); 357 cpu_data[cpu].package = 358 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 359 360 i = 0; 361 core0_c0count[cpu] = 0; 362 loongson3_send_ipi_single(0, SMP_ASK_C0COUNT); 363 while (!core0_c0count[cpu]) { 364 i++; 365 cpu_relax(); 366 } 367 368 if (i > MAX_LOOPS) 369 i = MAX_LOOPS; 370 if (cpu_data[cpu].package) 371 initcount = core0_c0count[cpu] + i; 372 else /* Local access is faster for loops */ 373 initcount = core0_c0count[cpu] + i/2; 374 375 write_c0_count(initcount); 376 } 377 378 static void loongson3_smp_finish(void) 379 { 380 int cpu = smp_processor_id(); 381 382 write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ); 383 local_irq_enable(); 384 loongson3_ipi_write64(0, 385 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x0); 386 pr_info("CPU#%d finished, CP0_ST=%x\n", 387 smp_processor_id(), read_c0_status()); 388 } 389 390 static void __init loongson3_smp_setup(void) 391 { 392 int i = 0, num = 0; /* i: physical id, num: logical id */ 393 394 init_cpu_possible(cpu_none_mask); 395 396 /* For unified kernel, NR_CPUS is the maximum possible value, 397 * loongson_sysconf.nr_cpus is the really present value */ 398 while (i < loongson_sysconf.nr_cpus) { 399 if (loongson_sysconf.reserved_cpus_mask & (1<<i)) { 400 /* Reserved physical CPU cores */ 401 __cpu_number_map[i] = -1; 402 } else { 403 __cpu_number_map[i] = num; 404 __cpu_logical_map[num] = i; 405 set_cpu_possible(num, true); 406 num++; 407 } 408 i++; 409 } 410 pr_info("Detected %i available CPU(s)\n", num); 411 412 while (num < loongson_sysconf.nr_cpus) { 413 __cpu_logical_map[num] = -1; 414 num++; 415 } 416 417 csr_ipi_probe(); 418 ipi_set0_regs_init(); 419 ipi_clear0_regs_init(); 420 ipi_status0_regs_init(); 421 ipi_en0_regs_init(); 422 ipi_mailbox_buf_init(); 423 cpu_set_core(&cpu_data[0], 424 cpu_logical_map(0) % loongson_sysconf.cores_per_package); 425 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 426 } 427 428 static void __init loongson3_prepare_cpus(unsigned int max_cpus) 429 { 430 if (request_irq(LS_IPI_IRQ, loongson3_ipi_interrupt, 431 IRQF_PERCPU | IRQF_NO_SUSPEND, "SMP_IPI", NULL)) 432 pr_err("Failed to request IPI IRQ\n"); 433 init_cpu_present(cpu_possible_mask); 434 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 435 } 436 437 /* 438 * Setup the PC, SP, and GP of a secondary processor and start it runing! 439 */ 440 static int loongson3_boot_secondary(int cpu, struct task_struct *idle) 441 { 442 unsigned long startargs[4]; 443 444 pr_info("Booting CPU#%d...\n", cpu); 445 446 /* startargs[] are initial PC, SP and GP for secondary CPU */ 447 startargs[0] = (unsigned long)&smp_bootstrap; 448 startargs[1] = (unsigned long)__KSTK_TOS(idle); 449 startargs[2] = (unsigned long)task_thread_info(idle); 450 startargs[3] = 0; 451 452 pr_debug("CPU#%d, func_pc=%lx, sp=%lx, gp=%lx\n", 453 cpu, startargs[0], startargs[1], startargs[2]); 454 455 loongson3_ipi_write64(startargs[3], 456 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x18); 457 loongson3_ipi_write64(startargs[2], 458 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x10); 459 loongson3_ipi_write64(startargs[1], 460 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x8); 461 loongson3_ipi_write64(startargs[0], 462 ipi_mailbox_buf[cpu_logical_map(cpu)] + 0x0); 463 return 0; 464 } 465 466 #ifdef CONFIG_HOTPLUG_CPU 467 468 static int loongson3_cpu_disable(void) 469 { 470 unsigned long flags; 471 unsigned int cpu = smp_processor_id(); 472 473 if (cpu == 0) 474 return -EBUSY; 475 476 set_cpu_online(cpu, false); 477 calculate_cpu_foreign_map(); 478 local_irq_save(flags); 479 irq_cpu_offline(); 480 clear_c0_status(ST0_IM); 481 local_irq_restore(flags); 482 local_flush_tlb_all(); 483 484 return 0; 485 } 486 487 488 static void loongson3_cpu_die(unsigned int cpu) 489 { 490 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 491 cpu_relax(); 492 493 mb(); 494 } 495 496 /* To shutdown a core in Loongson 3, the target core should go to CKSEG1 and 497 * flush all L1 entries at first. Then, another core (usually Core 0) can 498 * safely disable the clock of the target core. loongson3_play_dead() is 499 * called via CKSEG1 (uncached and unmmaped) */ 500 static void loongson3_type1_play_dead(int *state_addr) 501 { 502 register int val; 503 register long cpuid, core, node, count; 504 register void *addr, *base, *initfunc; 505 506 __asm__ __volatile__( 507 " .set push \n" 508 " .set noreorder \n" 509 " li %[addr], 0x80000000 \n" /* KSEG0 */ 510 "1: cache 0, 0(%[addr]) \n" /* flush L1 ICache */ 511 " cache 0, 1(%[addr]) \n" 512 " cache 0, 2(%[addr]) \n" 513 " cache 0, 3(%[addr]) \n" 514 " cache 1, 0(%[addr]) \n" /* flush L1 DCache */ 515 " cache 1, 1(%[addr]) \n" 516 " cache 1, 2(%[addr]) \n" 517 " cache 1, 3(%[addr]) \n" 518 " addiu %[sets], %[sets], -1 \n" 519 " bnez %[sets], 1b \n" 520 " addiu %[addr], %[addr], 0x20 \n" 521 " li %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ 522 " sw %[val], (%[state_addr]) \n" 523 " sync \n" 524 " cache 21, (%[state_addr]) \n" /* flush entry of *state_addr */ 525 " .set pop \n" 526 : [addr] "=&r" (addr), [val] "=&r" (val) 527 : [state_addr] "r" (state_addr), 528 [sets] "r" (cpu_data[smp_processor_id()].dcache.sets)); 529 530 __asm__ __volatile__( 531 " .set push \n" 532 " .set noreorder \n" 533 " .set mips64 \n" 534 " mfc0 %[cpuid], $15, 1 \n" 535 " andi %[cpuid], 0x3ff \n" 536 " dli %[base], 0x900000003ff01000 \n" 537 " andi %[core], %[cpuid], 0x3 \n" 538 " sll %[core], 8 \n" /* get core id */ 539 " or %[base], %[base], %[core] \n" 540 " andi %[node], %[cpuid], 0xc \n" 541 " dsll %[node], 42 \n" /* get node id */ 542 " or %[base], %[base], %[node] \n" 543 "1: li %[count], 0x100 \n" /* wait for init loop */ 544 "2: bnez %[count], 2b \n" /* limit mailbox access */ 545 " addiu %[count], -1 \n" 546 " ld %[initfunc], 0x20(%[base]) \n" /* get PC via mailbox */ 547 " beqz %[initfunc], 1b \n" 548 " nop \n" 549 " ld $sp, 0x28(%[base]) \n" /* get SP via mailbox */ 550 " ld $gp, 0x30(%[base]) \n" /* get GP via mailbox */ 551 " ld $a1, 0x38(%[base]) \n" 552 " jr %[initfunc] \n" /* jump to initial PC */ 553 " nop \n" 554 " .set pop \n" 555 : [core] "=&r" (core), [node] "=&r" (node), 556 [base] "=&r" (base), [cpuid] "=&r" (cpuid), 557 [count] "=&r" (count), [initfunc] "=&r" (initfunc) 558 : /* No Input */ 559 : "a1"); 560 } 561 562 static void loongson3_type2_play_dead(int *state_addr) 563 { 564 register int val; 565 register long cpuid, core, node, count; 566 register void *addr, *base, *initfunc; 567 568 __asm__ __volatile__( 569 " .set push \n" 570 " .set noreorder \n" 571 " li %[addr], 0x80000000 \n" /* KSEG0 */ 572 "1: cache 0, 0(%[addr]) \n" /* flush L1 ICache */ 573 " cache 0, 1(%[addr]) \n" 574 " cache 0, 2(%[addr]) \n" 575 " cache 0, 3(%[addr]) \n" 576 " cache 1, 0(%[addr]) \n" /* flush L1 DCache */ 577 " cache 1, 1(%[addr]) \n" 578 " cache 1, 2(%[addr]) \n" 579 " cache 1, 3(%[addr]) \n" 580 " addiu %[sets], %[sets], -1 \n" 581 " bnez %[sets], 1b \n" 582 " addiu %[addr], %[addr], 0x20 \n" 583 " li %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ 584 " sw %[val], (%[state_addr]) \n" 585 " sync \n" 586 " cache 21, (%[state_addr]) \n" /* flush entry of *state_addr */ 587 " .set pop \n" 588 : [addr] "=&r" (addr), [val] "=&r" (val) 589 : [state_addr] "r" (state_addr), 590 [sets] "r" (cpu_data[smp_processor_id()].dcache.sets)); 591 592 __asm__ __volatile__( 593 " .set push \n" 594 " .set noreorder \n" 595 " .set mips64 \n" 596 " mfc0 %[cpuid], $15, 1 \n" 597 " andi %[cpuid], 0x3ff \n" 598 " dli %[base], 0x900000003ff01000 \n" 599 " andi %[core], %[cpuid], 0x3 \n" 600 " sll %[core], 8 \n" /* get core id */ 601 " or %[base], %[base], %[core] \n" 602 " andi %[node], %[cpuid], 0xc \n" 603 " dsll %[node], 42 \n" /* get node id */ 604 " or %[base], %[base], %[node] \n" 605 " dsrl %[node], 30 \n" /* 15:14 */ 606 " or %[base], %[base], %[node] \n" 607 "1: li %[count], 0x100 \n" /* wait for init loop */ 608 "2: bnez %[count], 2b \n" /* limit mailbox access */ 609 " addiu %[count], -1 \n" 610 " ld %[initfunc], 0x20(%[base]) \n" /* get PC via mailbox */ 611 " beqz %[initfunc], 1b \n" 612 " nop \n" 613 " ld $sp, 0x28(%[base]) \n" /* get SP via mailbox */ 614 " ld $gp, 0x30(%[base]) \n" /* get GP via mailbox */ 615 " ld $a1, 0x38(%[base]) \n" 616 " jr %[initfunc] \n" /* jump to initial PC */ 617 " nop \n" 618 " .set pop \n" 619 : [core] "=&r" (core), [node] "=&r" (node), 620 [base] "=&r" (base), [cpuid] "=&r" (cpuid), 621 [count] "=&r" (count), [initfunc] "=&r" (initfunc) 622 : /* No Input */ 623 : "a1"); 624 } 625 626 static void loongson3_type3_play_dead(int *state_addr) 627 { 628 register int val; 629 register long cpuid, core, node, count; 630 register void *addr, *base, *initfunc; 631 632 __asm__ __volatile__( 633 " .set push \n" 634 " .set noreorder \n" 635 " li %[addr], 0x80000000 \n" /* KSEG0 */ 636 "1: cache 0, 0(%[addr]) \n" /* flush L1 ICache */ 637 " cache 0, 1(%[addr]) \n" 638 " cache 0, 2(%[addr]) \n" 639 " cache 0, 3(%[addr]) \n" 640 " cache 1, 0(%[addr]) \n" /* flush L1 DCache */ 641 " cache 1, 1(%[addr]) \n" 642 " cache 1, 2(%[addr]) \n" 643 " cache 1, 3(%[addr]) \n" 644 " addiu %[sets], %[sets], -1 \n" 645 " bnez %[sets], 1b \n" 646 " addiu %[addr], %[addr], 0x40 \n" 647 " li %[addr], 0x80000000 \n" /* KSEG0 */ 648 "2: cache 2, 0(%[addr]) \n" /* flush L1 VCache */ 649 " cache 2, 1(%[addr]) \n" 650 " cache 2, 2(%[addr]) \n" 651 " cache 2, 3(%[addr]) \n" 652 " cache 2, 4(%[addr]) \n" 653 " cache 2, 5(%[addr]) \n" 654 " cache 2, 6(%[addr]) \n" 655 " cache 2, 7(%[addr]) \n" 656 " cache 2, 8(%[addr]) \n" 657 " cache 2, 9(%[addr]) \n" 658 " cache 2, 10(%[addr]) \n" 659 " cache 2, 11(%[addr]) \n" 660 " cache 2, 12(%[addr]) \n" 661 " cache 2, 13(%[addr]) \n" 662 " cache 2, 14(%[addr]) \n" 663 " cache 2, 15(%[addr]) \n" 664 " addiu %[vsets], %[vsets], -1 \n" 665 " bnez %[vsets], 2b \n" 666 " addiu %[addr], %[addr], 0x40 \n" 667 " li %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ 668 " sw %[val], (%[state_addr]) \n" 669 " sync \n" 670 " cache 21, (%[state_addr]) \n" /* flush entry of *state_addr */ 671 " .set pop \n" 672 : [addr] "=&r" (addr), [val] "=&r" (val) 673 : [state_addr] "r" (state_addr), 674 [sets] "r" (cpu_data[smp_processor_id()].dcache.sets), 675 [vsets] "r" (cpu_data[smp_processor_id()].vcache.sets)); 676 677 __asm__ __volatile__( 678 " .set push \n" 679 " .set noreorder \n" 680 " .set mips64 \n" 681 " mfc0 %[cpuid], $15, 1 \n" 682 " andi %[cpuid], 0x3ff \n" 683 " dli %[base], 0x900000003ff01000 \n" 684 " andi %[core], %[cpuid], 0x3 \n" 685 " sll %[core], 8 \n" /* get core id */ 686 " or %[base], %[base], %[core] \n" 687 " andi %[node], %[cpuid], 0xc \n" 688 " dsll %[node], 42 \n" /* get node id */ 689 " or %[base], %[base], %[node] \n" 690 "1: li %[count], 0x100 \n" /* wait for init loop */ 691 "2: bnez %[count], 2b \n" /* limit mailbox access */ 692 " addiu %[count], -1 \n" 693 " ld %[initfunc], 0x20(%[base]) \n" /* get PC via mailbox */ 694 " beqz %[initfunc], 1b \n" 695 " nop \n" 696 " ld $sp, 0x28(%[base]) \n" /* get SP via mailbox */ 697 " ld $gp, 0x30(%[base]) \n" /* get GP via mailbox */ 698 " ld $a1, 0x38(%[base]) \n" 699 " jr %[initfunc] \n" /* jump to initial PC */ 700 " nop \n" 701 " .set pop \n" 702 : [core] "=&r" (core), [node] "=&r" (node), 703 [base] "=&r" (base), [cpuid] "=&r" (cpuid), 704 [count] "=&r" (count), [initfunc] "=&r" (initfunc) 705 : /* No Input */ 706 : "a1"); 707 } 708 709 void play_dead(void) 710 { 711 int prid_imp, prid_rev, *state_addr; 712 unsigned int cpu = smp_processor_id(); 713 void (*play_dead_at_ckseg1)(int *); 714 715 idle_task_exit(); 716 717 prid_imp = read_c0_prid() & PRID_IMP_MASK; 718 prid_rev = read_c0_prid() & PRID_REV_MASK; 719 720 if (prid_imp == PRID_IMP_LOONGSON_64G) { 721 play_dead_at_ckseg1 = 722 (void *)CKSEG1ADDR((unsigned long)loongson3_type3_play_dead); 723 goto out; 724 } 725 726 switch (prid_rev) { 727 case PRID_REV_LOONGSON3A_R1: 728 default: 729 play_dead_at_ckseg1 = 730 (void *)CKSEG1ADDR((unsigned long)loongson3_type1_play_dead); 731 break; 732 case PRID_REV_LOONGSON3B_R1: 733 case PRID_REV_LOONGSON3B_R2: 734 play_dead_at_ckseg1 = 735 (void *)CKSEG1ADDR((unsigned long)loongson3_type2_play_dead); 736 break; 737 case PRID_REV_LOONGSON3A_R2_0: 738 case PRID_REV_LOONGSON3A_R2_1: 739 case PRID_REV_LOONGSON3A_R3_0: 740 case PRID_REV_LOONGSON3A_R3_1: 741 play_dead_at_ckseg1 = 742 (void *)CKSEG1ADDR((unsigned long)loongson3_type3_play_dead); 743 break; 744 } 745 746 out: 747 state_addr = &per_cpu(cpu_state, cpu); 748 mb(); 749 play_dead_at_ckseg1(state_addr); 750 } 751 752 static int loongson3_disable_clock(unsigned int cpu) 753 { 754 uint64_t core_id = cpu_core(&cpu_data[cpu]); 755 uint64_t package_id = cpu_data[cpu].package; 756 757 if ((read_c0_prid() & PRID_REV_MASK) == PRID_REV_LOONGSON3A_R1) { 758 LOONGSON_CHIPCFG(package_id) &= ~(1 << (12 + core_id)); 759 } else { 760 if (!(loongson_sysconf.workarounds & WORKAROUND_CPUHOTPLUG)) 761 LOONGSON_FREQCTRL(package_id) &= ~(1 << (core_id * 4 + 3)); 762 } 763 return 0; 764 } 765 766 static int loongson3_enable_clock(unsigned int cpu) 767 { 768 uint64_t core_id = cpu_core(&cpu_data[cpu]); 769 uint64_t package_id = cpu_data[cpu].package; 770 771 if ((read_c0_prid() & PRID_REV_MASK) == PRID_REV_LOONGSON3A_R1) { 772 LOONGSON_CHIPCFG(package_id) |= 1 << (12 + core_id); 773 } else { 774 if (!(loongson_sysconf.workarounds & WORKAROUND_CPUHOTPLUG)) 775 LOONGSON_FREQCTRL(package_id) |= 1 << (core_id * 4 + 3); 776 } 777 return 0; 778 } 779 780 static int register_loongson3_notifier(void) 781 { 782 return cpuhp_setup_state_nocalls(CPUHP_MIPS_SOC_PREPARE, 783 "mips/loongson:prepare", 784 loongson3_enable_clock, 785 loongson3_disable_clock); 786 } 787 early_initcall(register_loongson3_notifier); 788 789 #endif 790 791 const struct plat_smp_ops loongson3_smp_ops = { 792 .send_ipi_single = loongson3_send_ipi_single, 793 .send_ipi_mask = loongson3_send_ipi_mask, 794 .init_secondary = loongson3_init_secondary, 795 .smp_finish = loongson3_smp_finish, 796 .boot_secondary = loongson3_boot_secondary, 797 .smp_setup = loongson3_smp_setup, 798 .prepare_cpus = loongson3_prepare_cpus, 799 #ifdef CONFIG_HOTPLUG_CPU 800 .cpu_disable = loongson3_cpu_disable, 801 .cpu_die = loongson3_cpu_die, 802 #endif 803 #ifdef CONFIG_KEXEC 804 .kexec_nonboot_cpu = kexec_nonboot_cpu_jump, 805 #endif 806 }; 807