1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018 Chen-Yu Tsai 4 * 5 * Chen-Yu Tsai <wens@csie.org> 6 * 7 * arch/arm/mach-sunxi/mc_smp.c 8 * 9 * Based on Allwinner code, arch/arm/mach-exynos/mcpm-exynos.c, and 10 * arch/arm/mach-hisi/platmcpm.c 11 * Cluster cache enable trampoline code adapted from MCPM framework 12 */ 13 14 #include <linux/arm-cci.h> 15 #include <linux/cpu_pm.h> 16 #include <linux/delay.h> 17 #include <linux/io.h> 18 #include <linux/iopoll.h> 19 #include <linux/irqchip/arm-gic.h> 20 #include <linux/of.h> 21 #include <linux/of_address.h> 22 #include <linux/of_device.h> 23 #include <linux/smp.h> 24 25 #include <asm/cacheflush.h> 26 #include <asm/cp15.h> 27 #include <asm/cputype.h> 28 #include <asm/idmap.h> 29 #include <asm/smp_plat.h> 30 #include <asm/suspend.h> 31 32 #define SUNXI_CPUS_PER_CLUSTER 4 33 #define SUNXI_NR_CLUSTERS 2 34 35 #define POLL_USEC 100 36 #define TIMEOUT_USEC 100000 37 38 #define CPUCFG_CX_CTRL_REG0(c) (0x10 * (c)) 39 #define CPUCFG_CX_CTRL_REG0_L1_RST_DISABLE(n) BIT(n) 40 #define CPUCFG_CX_CTRL_REG0_L1_RST_DISABLE_ALL 0xf 41 #define CPUCFG_CX_CTRL_REG0_L2_RST_DISABLE_A7 BIT(4) 42 #define CPUCFG_CX_CTRL_REG0_L2_RST_DISABLE_A15 BIT(0) 43 #define CPUCFG_CX_CTRL_REG1(c) (0x10 * (c) + 0x4) 44 #define CPUCFG_CX_CTRL_REG1_ACINACTM BIT(0) 45 #define CPUCFG_CX_STATUS(c) (0x30 + 0x4 * (c)) 46 #define CPUCFG_CX_STATUS_STANDBYWFI(n) BIT(16 + (n)) 47 #define CPUCFG_CX_STATUS_STANDBYWFIL2 BIT(0) 48 #define CPUCFG_CX_RST_CTRL(c) (0x80 + 0x4 * (c)) 49 #define CPUCFG_CX_RST_CTRL_DBG_SOC_RST BIT(24) 50 #define CPUCFG_CX_RST_CTRL_ETM_RST(n) BIT(20 + (n)) 51 #define CPUCFG_CX_RST_CTRL_ETM_RST_ALL (0xf << 20) 52 #define CPUCFG_CX_RST_CTRL_DBG_RST(n) BIT(16 + (n)) 53 #define CPUCFG_CX_RST_CTRL_DBG_RST_ALL (0xf << 16) 54 #define CPUCFG_CX_RST_CTRL_H_RST BIT(12) 55 #define CPUCFG_CX_RST_CTRL_L2_RST BIT(8) 56 #define CPUCFG_CX_RST_CTRL_CX_RST(n) BIT(4 + (n)) 57 #define CPUCFG_CX_RST_CTRL_CORE_RST(n) BIT(n) 58 59 #define PRCM_CPU_PO_RST_CTRL(c) (0x4 + 0x4 * (c)) 60 #define PRCM_CPU_PO_RST_CTRL_CORE(n) BIT(n) 61 #define PRCM_CPU_PO_RST_CTRL_CORE_ALL 0xf 62 #define PRCM_PWROFF_GATING_REG(c) (0x100 + 0x4 * (c)) 63 #define PRCM_PWROFF_GATING_REG_CLUSTER BIT(4) 64 #define PRCM_PWROFF_GATING_REG_CORE(n) BIT(n) 65 #define PRCM_PWR_SWITCH_REG(c, cpu) (0x140 + 0x10 * (c) + 0x4 * (cpu)) 66 #define PRCM_CPU_SOFT_ENTRY_REG 0x164 67 68 #define CPU0_SUPPORT_HOTPLUG_MAGIC0 0xFA50392F 69 #define CPU0_SUPPORT_HOTPLUG_MAGIC1 0x790DCA3A 70 71 static void __iomem *cpucfg_base; 72 static void __iomem *prcm_base; 73 static void __iomem *sram_b_smp_base; 74 75 static bool sunxi_core_is_cortex_a15(unsigned int core, unsigned int cluster) 76 { 77 struct device_node *node; 78 int cpu = cluster * SUNXI_CPUS_PER_CLUSTER + core; 79 80 node = of_cpu_device_node_get(cpu); 81 82 /* In case of_cpu_device_node_get fails */ 83 if (!node) 84 node = of_get_cpu_node(cpu, NULL); 85 86 if (!node) { 87 /* 88 * There's no point in returning an error, since we 89 * would be mid way in a core or cluster power sequence. 90 */ 91 pr_err("%s: Couldn't get CPU cluster %u core %u device node\n", 92 __func__, cluster, core); 93 94 return false; 95 } 96 97 return of_device_is_compatible(node, "arm,cortex-a15"); 98 } 99 100 static int sunxi_cpu_power_switch_set(unsigned int cpu, unsigned int cluster, 101 bool enable) 102 { 103 u32 reg; 104 105 /* control sequence from Allwinner A80 user manual v1.2 PRCM section */ 106 reg = readl(prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); 107 if (enable) { 108 if (reg == 0x00) { 109 pr_debug("power clamp for cluster %u cpu %u already open\n", 110 cluster, cpu); 111 return 0; 112 } 113 114 writel(0xff, prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); 115 udelay(10); 116 writel(0xfe, prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); 117 udelay(10); 118 writel(0xf8, prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); 119 udelay(10); 120 writel(0xf0, prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); 121 udelay(10); 122 writel(0x00, prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); 123 udelay(10); 124 } else { 125 writel(0xff, prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); 126 udelay(10); 127 } 128 129 return 0; 130 } 131 132 static void sunxi_cpu0_hotplug_support_set(bool enable) 133 { 134 if (enable) { 135 writel(CPU0_SUPPORT_HOTPLUG_MAGIC0, sram_b_smp_base); 136 writel(CPU0_SUPPORT_HOTPLUG_MAGIC1, sram_b_smp_base + 0x4); 137 } else { 138 writel(0x0, sram_b_smp_base); 139 writel(0x0, sram_b_smp_base + 0x4); 140 } 141 } 142 143 static int sunxi_cpu_powerup(unsigned int cpu, unsigned int cluster) 144 { 145 u32 reg; 146 147 pr_debug("%s: cluster %u cpu %u\n", __func__, cluster, cpu); 148 if (cpu >= SUNXI_CPUS_PER_CLUSTER || cluster >= SUNXI_NR_CLUSTERS) 149 return -EINVAL; 150 151 /* Set hotplug support magic flags for cpu0 */ 152 if (cluster == 0 && cpu == 0) 153 sunxi_cpu0_hotplug_support_set(true); 154 155 /* assert processor power-on reset */ 156 reg = readl(prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); 157 reg &= ~PRCM_CPU_PO_RST_CTRL_CORE(cpu); 158 writel(reg, prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); 159 160 /* Cortex-A7: hold L1 reset disable signal low */ 161 if (!sunxi_core_is_cortex_a15(cpu, cluster)) { 162 reg = readl(cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster)); 163 reg &= ~CPUCFG_CX_CTRL_REG0_L1_RST_DISABLE(cpu); 164 writel(reg, cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster)); 165 } 166 167 /* assert processor related resets */ 168 reg = readl(cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 169 reg &= ~CPUCFG_CX_RST_CTRL_DBG_RST(cpu); 170 171 /* 172 * Allwinner code also asserts resets for NEON on A15. According 173 * to ARM manuals, asserting power-on reset is sufficient. 174 */ 175 if (!sunxi_core_is_cortex_a15(cpu, cluster)) 176 reg &= ~CPUCFG_CX_RST_CTRL_ETM_RST(cpu); 177 178 writel(reg, cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 179 180 /* open power switch */ 181 sunxi_cpu_power_switch_set(cpu, cluster, true); 182 183 /* clear processor power gate */ 184 reg = readl(prcm_base + PRCM_PWROFF_GATING_REG(cluster)); 185 reg &= ~PRCM_PWROFF_GATING_REG_CORE(cpu); 186 writel(reg, prcm_base + PRCM_PWROFF_GATING_REG(cluster)); 187 udelay(20); 188 189 /* de-assert processor power-on reset */ 190 reg = readl(prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); 191 reg |= PRCM_CPU_PO_RST_CTRL_CORE(cpu); 192 writel(reg, prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); 193 194 /* de-assert all processor resets */ 195 reg = readl(cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 196 reg |= CPUCFG_CX_RST_CTRL_DBG_RST(cpu); 197 reg |= CPUCFG_CX_RST_CTRL_CORE_RST(cpu); 198 if (!sunxi_core_is_cortex_a15(cpu, cluster)) 199 reg |= CPUCFG_CX_RST_CTRL_ETM_RST(cpu); 200 else 201 reg |= CPUCFG_CX_RST_CTRL_CX_RST(cpu); /* NEON */ 202 writel(reg, cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 203 204 return 0; 205 } 206 207 static int sunxi_cluster_powerup(unsigned int cluster) 208 { 209 u32 reg; 210 211 pr_debug("%s: cluster %u\n", __func__, cluster); 212 if (cluster >= SUNXI_NR_CLUSTERS) 213 return -EINVAL; 214 215 /* assert ACINACTM */ 216 reg = readl(cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); 217 reg |= CPUCFG_CX_CTRL_REG1_ACINACTM; 218 writel(reg, cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); 219 220 /* assert cluster processor power-on resets */ 221 reg = readl(prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); 222 reg &= ~PRCM_CPU_PO_RST_CTRL_CORE_ALL; 223 writel(reg, prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); 224 225 /* assert cluster resets */ 226 reg = readl(cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 227 reg &= ~CPUCFG_CX_RST_CTRL_DBG_SOC_RST; 228 reg &= ~CPUCFG_CX_RST_CTRL_DBG_RST_ALL; 229 reg &= ~CPUCFG_CX_RST_CTRL_H_RST; 230 reg &= ~CPUCFG_CX_RST_CTRL_L2_RST; 231 232 /* 233 * Allwinner code also asserts resets for NEON on A15. According 234 * to ARM manuals, asserting power-on reset is sufficient. 235 */ 236 if (!sunxi_core_is_cortex_a15(0, cluster)) 237 reg &= ~CPUCFG_CX_RST_CTRL_ETM_RST_ALL; 238 239 writel(reg, cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 240 241 /* hold L1/L2 reset disable signals low */ 242 reg = readl(cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster)); 243 if (sunxi_core_is_cortex_a15(0, cluster)) { 244 /* Cortex-A15: hold L2RSTDISABLE low */ 245 reg &= ~CPUCFG_CX_CTRL_REG0_L2_RST_DISABLE_A15; 246 } else { 247 /* Cortex-A7: hold L1RSTDISABLE and L2RSTDISABLE low */ 248 reg &= ~CPUCFG_CX_CTRL_REG0_L1_RST_DISABLE_ALL; 249 reg &= ~CPUCFG_CX_CTRL_REG0_L2_RST_DISABLE_A7; 250 } 251 writel(reg, cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster)); 252 253 /* clear cluster power gate */ 254 reg = readl(prcm_base + PRCM_PWROFF_GATING_REG(cluster)); 255 reg &= ~PRCM_PWROFF_GATING_REG_CLUSTER; 256 writel(reg, prcm_base + PRCM_PWROFF_GATING_REG(cluster)); 257 udelay(20); 258 259 /* de-assert cluster resets */ 260 reg = readl(cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 261 reg |= CPUCFG_CX_RST_CTRL_DBG_SOC_RST; 262 reg |= CPUCFG_CX_RST_CTRL_H_RST; 263 reg |= CPUCFG_CX_RST_CTRL_L2_RST; 264 writel(reg, cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 265 266 /* de-assert ACINACTM */ 267 reg = readl(cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); 268 reg &= ~CPUCFG_CX_CTRL_REG1_ACINACTM; 269 writel(reg, cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); 270 271 return 0; 272 } 273 274 /* 275 * This bit is shared between the initial nocache_trampoline call to 276 * enable CCI-400 and proper cluster cache disable before power down. 277 */ 278 static void sunxi_cluster_cache_disable_without_axi(void) 279 { 280 if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) { 281 /* 282 * On the Cortex-A15 we need to disable 283 * L2 prefetching before flushing the cache. 284 */ 285 asm volatile( 286 "mcr p15, 1, %0, c15, c0, 3\n" 287 "isb\n" 288 "dsb" 289 : : "r" (0x400)); 290 } 291 292 /* Flush all cache levels for this cluster. */ 293 v7_exit_coherency_flush(all); 294 295 /* 296 * Disable cluster-level coherency by masking 297 * incoming snoops and DVM messages: 298 */ 299 cci_disable_port_by_cpu(read_cpuid_mpidr()); 300 } 301 302 static int sunxi_mc_smp_cpu_table[SUNXI_NR_CLUSTERS][SUNXI_CPUS_PER_CLUSTER]; 303 static int sunxi_mc_smp_first_comer; 304 305 /* 306 * Enable cluster-level coherency, in preparation for turning on the MMU. 307 * 308 * Also enable regional clock gating and L2 data latency settings for 309 * Cortex-A15. These settings are from the vendor kernel. 310 */ 311 static void __naked sunxi_mc_smp_cluster_cache_enable(void) 312 { 313 asm volatile ( 314 "mrc p15, 0, r1, c0, c0, 0\n" 315 "movw r2, #" __stringify(ARM_CPU_PART_MASK & 0xffff) "\n" 316 "movt r2, #" __stringify(ARM_CPU_PART_MASK >> 16) "\n" 317 "and r1, r1, r2\n" 318 "movw r2, #" __stringify(ARM_CPU_PART_CORTEX_A15 & 0xffff) "\n" 319 "movt r2, #" __stringify(ARM_CPU_PART_CORTEX_A15 >> 16) "\n" 320 "cmp r1, r2\n" 321 "bne not_a15\n" 322 323 /* The following is Cortex-A15 specific */ 324 325 /* ACTLR2: Enable CPU regional clock gates */ 326 "mrc p15, 1, r1, c15, c0, 4\n" 327 "orr r1, r1, #(0x1<<31)\n" 328 "mcr p15, 1, r1, c15, c0, 4\n" 329 330 /* L2ACTLR */ 331 "mrc p15, 1, r1, c15, c0, 0\n" 332 /* Enable L2, GIC, and Timer regional clock gates */ 333 "orr r1, r1, #(0x1<<26)\n" 334 /* Disable clean/evict from being pushed to external */ 335 "orr r1, r1, #(0x1<<3)\n" 336 "mcr p15, 1, r1, c15, c0, 0\n" 337 338 /* L2CTRL: L2 data RAM latency */ 339 "mrc p15, 1, r1, c9, c0, 2\n" 340 "bic r1, r1, #(0x7<<0)\n" 341 "orr r1, r1, #(0x3<<0)\n" 342 "mcr p15, 1, r1, c9, c0, 2\n" 343 344 /* End of Cortex-A15 specific setup */ 345 "not_a15:\n" 346 347 /* Get value of sunxi_mc_smp_first_comer */ 348 "adr r1, first\n" 349 "ldr r0, [r1]\n" 350 "ldr r0, [r1, r0]\n" 351 352 /* Skip cci_enable_port_for_self if not first comer */ 353 "cmp r0, #0\n" 354 "bxeq lr\n" 355 "b cci_enable_port_for_self\n" 356 357 ".align 2\n" 358 "first: .word sunxi_mc_smp_first_comer - .\n" 359 ); 360 } 361 362 static void __naked sunxi_mc_smp_secondary_startup(void) 363 { 364 asm volatile( 365 "bl sunxi_mc_smp_cluster_cache_enable\n" 366 "b secondary_startup" 367 /* Let compiler know about sunxi_mc_smp_cluster_cache_enable */ 368 :: "i" (sunxi_mc_smp_cluster_cache_enable) 369 ); 370 } 371 372 static DEFINE_SPINLOCK(boot_lock); 373 374 static bool sunxi_mc_smp_cluster_is_down(unsigned int cluster) 375 { 376 int i; 377 378 for (i = 0; i < SUNXI_CPUS_PER_CLUSTER; i++) 379 if (sunxi_mc_smp_cpu_table[cluster][i]) 380 return false; 381 return true; 382 } 383 384 static void sunxi_mc_smp_secondary_init(unsigned int cpu) 385 { 386 /* Clear hotplug support magic flags for cpu0 */ 387 if (cpu == 0) 388 sunxi_cpu0_hotplug_support_set(false); 389 } 390 391 static int sunxi_mc_smp_boot_secondary(unsigned int l_cpu, struct task_struct *idle) 392 { 393 unsigned int mpidr, cpu, cluster; 394 395 mpidr = cpu_logical_map(l_cpu); 396 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); 397 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); 398 399 if (!cpucfg_base) 400 return -ENODEV; 401 if (cluster >= SUNXI_NR_CLUSTERS || cpu >= SUNXI_CPUS_PER_CLUSTER) 402 return -EINVAL; 403 404 spin_lock_irq(&boot_lock); 405 406 if (sunxi_mc_smp_cpu_table[cluster][cpu]) 407 goto out; 408 409 if (sunxi_mc_smp_cluster_is_down(cluster)) { 410 sunxi_mc_smp_first_comer = true; 411 sunxi_cluster_powerup(cluster); 412 } else { 413 sunxi_mc_smp_first_comer = false; 414 } 415 416 /* This is read by incoming CPUs with their cache and MMU disabled */ 417 sync_cache_w(&sunxi_mc_smp_first_comer); 418 sunxi_cpu_powerup(cpu, cluster); 419 420 out: 421 sunxi_mc_smp_cpu_table[cluster][cpu]++; 422 spin_unlock_irq(&boot_lock); 423 424 return 0; 425 } 426 427 #ifdef CONFIG_HOTPLUG_CPU 428 static void sunxi_cluster_cache_disable(void) 429 { 430 unsigned int cluster = MPIDR_AFFINITY_LEVEL(read_cpuid_mpidr(), 1); 431 u32 reg; 432 433 pr_debug("%s: cluster %u\n", __func__, cluster); 434 435 sunxi_cluster_cache_disable_without_axi(); 436 437 /* last man standing, assert ACINACTM */ 438 reg = readl(cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); 439 reg |= CPUCFG_CX_CTRL_REG1_ACINACTM; 440 writel(reg, cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); 441 } 442 443 static void sunxi_mc_smp_cpu_die(unsigned int l_cpu) 444 { 445 unsigned int mpidr, cpu, cluster; 446 bool last_man; 447 448 mpidr = cpu_logical_map(l_cpu); 449 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); 450 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); 451 pr_debug("%s: cluster %u cpu %u\n", __func__, cluster, cpu); 452 453 spin_lock(&boot_lock); 454 sunxi_mc_smp_cpu_table[cluster][cpu]--; 455 if (sunxi_mc_smp_cpu_table[cluster][cpu] == 1) { 456 /* A power_up request went ahead of us. */ 457 pr_debug("%s: aborting due to a power up request\n", 458 __func__); 459 spin_unlock(&boot_lock); 460 return; 461 } else if (sunxi_mc_smp_cpu_table[cluster][cpu] > 1) { 462 pr_err("Cluster %d CPU%d boots multiple times\n", 463 cluster, cpu); 464 BUG(); 465 } 466 467 last_man = sunxi_mc_smp_cluster_is_down(cluster); 468 spin_unlock(&boot_lock); 469 470 gic_cpu_if_down(0); 471 if (last_man) 472 sunxi_cluster_cache_disable(); 473 else 474 v7_exit_coherency_flush(louis); 475 476 for (;;) 477 wfi(); 478 } 479 480 static int sunxi_cpu_powerdown(unsigned int cpu, unsigned int cluster) 481 { 482 u32 reg; 483 484 pr_debug("%s: cluster %u cpu %u\n", __func__, cluster, cpu); 485 if (cpu >= SUNXI_CPUS_PER_CLUSTER || cluster >= SUNXI_NR_CLUSTERS) 486 return -EINVAL; 487 488 /* gate processor power */ 489 reg = readl(prcm_base + PRCM_PWROFF_GATING_REG(cluster)); 490 reg |= PRCM_PWROFF_GATING_REG_CORE(cpu); 491 writel(reg, prcm_base + PRCM_PWROFF_GATING_REG(cluster)); 492 udelay(20); 493 494 /* close power switch */ 495 sunxi_cpu_power_switch_set(cpu, cluster, false); 496 497 return 0; 498 } 499 500 static int sunxi_cluster_powerdown(unsigned int cluster) 501 { 502 u32 reg; 503 504 pr_debug("%s: cluster %u\n", __func__, cluster); 505 if (cluster >= SUNXI_NR_CLUSTERS) 506 return -EINVAL; 507 508 /* assert cluster resets or system will hang */ 509 pr_debug("%s: assert cluster reset\n", __func__); 510 reg = readl(cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 511 reg &= ~CPUCFG_CX_RST_CTRL_DBG_SOC_RST; 512 reg &= ~CPUCFG_CX_RST_CTRL_H_RST; 513 reg &= ~CPUCFG_CX_RST_CTRL_L2_RST; 514 writel(reg, cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); 515 516 /* gate cluster power */ 517 pr_debug("%s: gate cluster power\n", __func__); 518 reg = readl(prcm_base + PRCM_PWROFF_GATING_REG(cluster)); 519 reg |= PRCM_PWROFF_GATING_REG_CLUSTER; 520 writel(reg, prcm_base + PRCM_PWROFF_GATING_REG(cluster)); 521 udelay(20); 522 523 return 0; 524 } 525 526 static int sunxi_mc_smp_cpu_kill(unsigned int l_cpu) 527 { 528 unsigned int mpidr, cpu, cluster; 529 unsigned int tries, count; 530 int ret = 0; 531 u32 reg; 532 533 mpidr = cpu_logical_map(l_cpu); 534 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); 535 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); 536 537 /* This should never happen */ 538 if (WARN_ON(cluster >= SUNXI_NR_CLUSTERS || 539 cpu >= SUNXI_CPUS_PER_CLUSTER)) 540 return 0; 541 542 /* wait for CPU core to die and enter WFI */ 543 count = TIMEOUT_USEC / POLL_USEC; 544 spin_lock_irq(&boot_lock); 545 for (tries = 0; tries < count; tries++) { 546 spin_unlock_irq(&boot_lock); 547 usleep_range(POLL_USEC / 2, POLL_USEC); 548 spin_lock_irq(&boot_lock); 549 550 /* 551 * If the user turns off a bunch of cores at the same 552 * time, the kernel might call cpu_kill before some of 553 * them are ready. This is because boot_lock serializes 554 * both cpu_die and cpu_kill callbacks. Either one could 555 * run first. We should wait for cpu_die to complete. 556 */ 557 if (sunxi_mc_smp_cpu_table[cluster][cpu]) 558 continue; 559 560 reg = readl(cpucfg_base + CPUCFG_CX_STATUS(cluster)); 561 if (reg & CPUCFG_CX_STATUS_STANDBYWFI(cpu)) 562 break; 563 } 564 565 if (tries >= count) { 566 ret = ETIMEDOUT; 567 goto out; 568 } 569 570 /* power down CPU core */ 571 sunxi_cpu_powerdown(cpu, cluster); 572 573 if (!sunxi_mc_smp_cluster_is_down(cluster)) 574 goto out; 575 576 /* wait for cluster L2 WFI */ 577 ret = readl_poll_timeout(cpucfg_base + CPUCFG_CX_STATUS(cluster), reg, 578 reg & CPUCFG_CX_STATUS_STANDBYWFIL2, 579 POLL_USEC, TIMEOUT_USEC); 580 if (ret) { 581 /* 582 * Ignore timeout on the cluster. Leaving the cluster on 583 * will not affect system execution, just use a bit more 584 * power. But returning an error here will only confuse 585 * the user as the CPU has already been shutdown. 586 */ 587 ret = 0; 588 goto out; 589 } 590 591 /* Power down cluster */ 592 sunxi_cluster_powerdown(cluster); 593 594 out: 595 spin_unlock_irq(&boot_lock); 596 pr_debug("%s: cluster %u cpu %u powerdown: %d\n", 597 __func__, cluster, cpu, ret); 598 return !ret; 599 } 600 601 static bool sunxi_mc_smp_cpu_can_disable(unsigned int __unused) 602 { 603 return true; 604 } 605 #endif 606 607 static const struct smp_operations sunxi_mc_smp_smp_ops __initconst = { 608 .smp_secondary_init = sunxi_mc_smp_secondary_init, 609 .smp_boot_secondary = sunxi_mc_smp_boot_secondary, 610 #ifdef CONFIG_HOTPLUG_CPU 611 .cpu_die = sunxi_mc_smp_cpu_die, 612 .cpu_kill = sunxi_mc_smp_cpu_kill, 613 .cpu_can_disable = sunxi_mc_smp_cpu_can_disable, 614 #endif 615 }; 616 617 static bool __init sunxi_mc_smp_cpu_table_init(void) 618 { 619 unsigned int mpidr, cpu, cluster; 620 621 mpidr = read_cpuid_mpidr(); 622 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); 623 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); 624 625 if (cluster >= SUNXI_NR_CLUSTERS || cpu >= SUNXI_CPUS_PER_CLUSTER) { 626 pr_err("%s: boot CPU is out of bounds!\n", __func__); 627 return false; 628 } 629 sunxi_mc_smp_cpu_table[cluster][cpu] = 1; 630 return true; 631 } 632 633 /* 634 * Adapted from arch/arm/common/mc_smp_entry.c 635 * 636 * We need the trampoline code to enable CCI-400 on the first cluster 637 */ 638 typedef typeof(cpu_reset) phys_reset_t; 639 640 static void __init __naked sunxi_mc_smp_resume(void) 641 { 642 asm volatile( 643 "bl sunxi_mc_smp_cluster_cache_enable\n" 644 "b cpu_resume" 645 /* Let compiler know about sunxi_mc_smp_cluster_cache_enable */ 646 :: "i" (sunxi_mc_smp_cluster_cache_enable) 647 ); 648 } 649 650 static int __init nocache_trampoline(unsigned long __unused) 651 { 652 phys_reset_t phys_reset; 653 654 setup_mm_for_reboot(); 655 sunxi_cluster_cache_disable_without_axi(); 656 657 phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset); 658 phys_reset(__pa_symbol(sunxi_mc_smp_resume), false); 659 BUG(); 660 } 661 662 static int __init sunxi_mc_smp_loopback(void) 663 { 664 int ret; 665 666 /* 667 * We're going to soft-restart the current CPU through the 668 * low-level MCPM code by leveraging the suspend/resume 669 * infrastructure. Let's play it safe by using cpu_pm_enter() 670 * in case the CPU init code path resets the VFP or similar. 671 */ 672 sunxi_mc_smp_first_comer = true; 673 local_irq_disable(); 674 local_fiq_disable(); 675 ret = cpu_pm_enter(); 676 if (!ret) { 677 ret = cpu_suspend(0, nocache_trampoline); 678 cpu_pm_exit(); 679 } 680 local_fiq_enable(); 681 local_irq_enable(); 682 sunxi_mc_smp_first_comer = false; 683 684 return ret; 685 } 686 687 /* 688 * This holds any device nodes that we requested resources for, 689 * so that we may easily release resources in the error path. 690 */ 691 struct sunxi_mc_smp_nodes { 692 struct device_node *prcm_node; 693 struct device_node *cpucfg_node; 694 struct device_node *sram_node; 695 }; 696 697 /* This structure holds SoC-specific bits tied to an enable-method string. */ 698 struct sunxi_mc_smp_data { 699 const char *enable_method; 700 int (*get_smp_nodes)(struct sunxi_mc_smp_nodes *nodes); 701 }; 702 703 static void __init sunxi_mc_smp_put_nodes(struct sunxi_mc_smp_nodes *nodes) 704 { 705 of_node_put(nodes->prcm_node); 706 of_node_put(nodes->cpucfg_node); 707 of_node_put(nodes->sram_node); 708 memset(nodes, 0, sizeof(*nodes)); 709 } 710 711 static int __init sun9i_a80_get_smp_nodes(struct sunxi_mc_smp_nodes *nodes) 712 { 713 nodes->prcm_node = of_find_compatible_node(NULL, NULL, 714 "allwinner,sun9i-a80-prcm"); 715 if (!nodes->prcm_node) { 716 pr_err("%s: PRCM not available\n", __func__); 717 return -ENODEV; 718 } 719 720 nodes->cpucfg_node = of_find_compatible_node(NULL, NULL, 721 "allwinner,sun9i-a80-cpucfg"); 722 if (!nodes->cpucfg_node) { 723 pr_err("%s: CPUCFG not available\n", __func__); 724 return -ENODEV; 725 } 726 727 nodes->sram_node = of_find_compatible_node(NULL, NULL, 728 "allwinner,sun9i-a80-smp-sram"); 729 if (!nodes->sram_node) { 730 pr_err("%s: Secure SRAM not available\n", __func__); 731 return -ENODEV; 732 } 733 734 return 0; 735 } 736 737 static const struct sunxi_mc_smp_data sunxi_mc_smp_data[] __initconst = { 738 { 739 .enable_method = "allwinner,sun9i-a80-smp", 740 .get_smp_nodes = sun9i_a80_get_smp_nodes, 741 }, 742 }; 743 744 static int __init sunxi_mc_smp_init(void) 745 { 746 struct sunxi_mc_smp_nodes nodes = { 0 }; 747 struct device_node *node; 748 struct resource res; 749 int i, ret; 750 751 /* 752 * Don't bother checking the "cpus" node, as an enable-method 753 * property in that node is undocumented. 754 */ 755 node = of_cpu_device_node_get(0); 756 if (!node) 757 return -ENODEV; 758 759 /* 760 * We can't actually use the enable-method magic in the kernel. 761 * Our loopback / trampoline code uses the CPU suspend framework, 762 * which requires the identity mapping be available. It would not 763 * yet be available if we used the .init_cpus or .prepare_cpus 764 * callbacks in smp_operations, which we would use if we were to 765 * use CPU_METHOD_OF_DECLARE 766 */ 767 for (i = 0; i < ARRAY_SIZE(sunxi_mc_smp_data); i++) { 768 ret = of_property_match_string(node, "enable-method", 769 sunxi_mc_smp_data[i].enable_method); 770 if (!ret) 771 break; 772 } 773 774 of_node_put(node); 775 if (ret) 776 return -ENODEV; 777 778 if (!sunxi_mc_smp_cpu_table_init()) 779 return -EINVAL; 780 781 if (!cci_probed()) { 782 pr_err("%s: CCI-400 not available\n", __func__); 783 return -ENODEV; 784 } 785 786 /* Get needed device tree nodes */ 787 ret = sunxi_mc_smp_data[i].get_smp_nodes(&nodes); 788 if (ret) 789 goto err_put_nodes; 790 791 /* 792 * Unfortunately we can not request the I/O region for the PRCM. 793 * It is shared with the PRCM clock. 794 */ 795 prcm_base = of_iomap(nodes.prcm_node, 0); 796 if (!prcm_base) { 797 pr_err("%s: failed to map PRCM registers\n", __func__); 798 ret = -ENOMEM; 799 goto err_put_nodes; 800 } 801 802 cpucfg_base = of_io_request_and_map(nodes.cpucfg_node, 0, 803 "sunxi-mc-smp"); 804 if (IS_ERR(cpucfg_base)) { 805 ret = PTR_ERR(cpucfg_base); 806 pr_err("%s: failed to map CPUCFG registers: %d\n", 807 __func__, ret); 808 goto err_unmap_prcm; 809 } 810 811 sram_b_smp_base = of_io_request_and_map(nodes.sram_node, 0, 812 "sunxi-mc-smp"); 813 if (IS_ERR(sram_b_smp_base)) { 814 ret = PTR_ERR(sram_b_smp_base); 815 pr_err("%s: failed to map secure SRAM\n", __func__); 816 goto err_unmap_release_cpucfg; 817 } 818 819 /* Configure CCI-400 for boot cluster */ 820 ret = sunxi_mc_smp_loopback(); 821 if (ret) { 822 pr_err("%s: failed to configure boot cluster: %d\n", 823 __func__, ret); 824 goto err_unmap_release_secure_sram; 825 } 826 827 /* We don't need the device nodes anymore */ 828 sunxi_mc_smp_put_nodes(&nodes); 829 830 /* Set the hardware entry point address */ 831 writel(__pa_symbol(sunxi_mc_smp_secondary_startup), 832 prcm_base + PRCM_CPU_SOFT_ENTRY_REG); 833 834 /* Actually enable multi cluster SMP */ 835 smp_set_ops(&sunxi_mc_smp_smp_ops); 836 837 pr_info("sunxi multi cluster SMP support installed\n"); 838 839 return 0; 840 841 err_unmap_release_secure_sram: 842 iounmap(sram_b_smp_base); 843 of_address_to_resource(nodes.sram_node, 0, &res); 844 release_mem_region(res.start, resource_size(&res)); 845 err_unmap_release_cpucfg: 846 iounmap(cpucfg_base); 847 of_address_to_resource(nodes.cpucfg_node, 0, &res); 848 release_mem_region(res.start, resource_size(&res)); 849 err_unmap_prcm: 850 iounmap(prcm_base); 851 err_put_nodes: 852 sunxi_mc_smp_put_nodes(&nodes); 853 return ret; 854 } 855 856 early_initcall(sunxi_mc_smp_init); 857