1 /* 2 * Copyright (c) 2014 Samsung Electronics Co., Ltd. 3 * http://www.samsung.com 4 * 5 * arch/arm/mach-exynos/mcpm-exynos.c 6 * 7 * Based on arch/arm/mach-vexpress/dcscb.c 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/arm-cci.h> 15 #include <linux/delay.h> 16 #include <linux/io.h> 17 #include <linux/of_address.h> 18 #include <linux/syscore_ops.h> 19 20 #include <asm/cputype.h> 21 #include <asm/cp15.h> 22 #include <asm/mcpm.h> 23 24 #include "regs-pmu.h" 25 #include "common.h" 26 27 #define EXYNOS5420_CPUS_PER_CLUSTER 4 28 #define EXYNOS5420_NR_CLUSTERS 2 29 30 #define EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN BIT(9) 31 #define EXYNOS5420_USE_ARM_CORE_DOWN_STATE BIT(29) 32 #define EXYNOS5420_USE_L2_COMMON_UP_STATE BIT(30) 33 34 static void __iomem *ns_sram_base_addr; 35 36 /* 37 * The common v7_exit_coherency_flush API could not be used because of the 38 * Erratum 799270 workaround. This macro is the same as the common one (in 39 * arch/arm/include/asm/cacheflush.h) except for the erratum handling. 40 */ 41 #define exynos_v7_exit_coherency_flush(level) \ 42 asm volatile( \ 43 "stmfd sp!, {fp, ip}\n\t"\ 44 "mrc p15, 0, r0, c1, c0, 0 @ get SCTLR\n\t" \ 45 "bic r0, r0, #"__stringify(CR_C)"\n\t" \ 46 "mcr p15, 0, r0, c1, c0, 0 @ set SCTLR\n\t" \ 47 "isb\n\t"\ 48 "bl v7_flush_dcache_"__stringify(level)"\n\t" \ 49 "mrc p15, 0, r0, c1, c0, 1 @ get ACTLR\n\t" \ 50 "bic r0, r0, #(1 << 6) @ disable local coherency\n\t" \ 51 /* Dummy Load of a device register to avoid Erratum 799270 */ \ 52 "ldr r4, [%0]\n\t" \ 53 "and r4, r4, #0\n\t" \ 54 "orr r0, r0, r4\n\t" \ 55 "mcr p15, 0, r0, c1, c0, 1 @ set ACTLR\n\t" \ 56 "isb\n\t" \ 57 "dsb\n\t" \ 58 "ldmfd sp!, {fp, ip}" \ 59 : \ 60 : "Ir" (pmu_base_addr + S5P_INFORM0) \ 61 : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", \ 62 "r9", "r10", "lr", "memory") 63 64 /* 65 * We can't use regular spinlocks. In the switcher case, it is possible 66 * for an outbound CPU to call power_down() after its inbound counterpart 67 * is already live using the same logical CPU number which trips lockdep 68 * debugging. 69 */ 70 static arch_spinlock_t exynos_mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED; 71 static int 72 cpu_use_count[EXYNOS5420_CPUS_PER_CLUSTER][EXYNOS5420_NR_CLUSTERS]; 73 74 #define exynos_cluster_usecnt(cluster) \ 75 (cpu_use_count[0][cluster] + \ 76 cpu_use_count[1][cluster] + \ 77 cpu_use_count[2][cluster] + \ 78 cpu_use_count[3][cluster]) 79 80 #define exynos_cluster_unused(cluster) !exynos_cluster_usecnt(cluster) 81 82 static int exynos_power_up(unsigned int cpu, unsigned int cluster) 83 { 84 unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); 85 86 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); 87 if (cpu >= EXYNOS5420_CPUS_PER_CLUSTER || 88 cluster >= EXYNOS5420_NR_CLUSTERS) 89 return -EINVAL; 90 91 /* 92 * Since this is called with IRQs enabled, and no arch_spin_lock_irq 93 * variant exists, we need to disable IRQs manually here. 94 */ 95 local_irq_disable(); 96 arch_spin_lock(&exynos_mcpm_lock); 97 98 cpu_use_count[cpu][cluster]++; 99 if (cpu_use_count[cpu][cluster] == 1) { 100 bool was_cluster_down = 101 (exynos_cluster_usecnt(cluster) == 1); 102 103 /* 104 * Turn on the cluster (L2/COMMON) and then power on the 105 * cores. 106 */ 107 if (was_cluster_down) 108 exynos_cluster_power_up(cluster); 109 110 exynos_cpu_power_up(cpunr); 111 } else if (cpu_use_count[cpu][cluster] != 2) { 112 /* 113 * The only possible values are: 114 * 0 = CPU down 115 * 1 = CPU (still) up 116 * 2 = CPU requested to be up before it had a chance 117 * to actually make itself down. 118 * Any other value is a bug. 119 */ 120 BUG(); 121 } 122 123 arch_spin_unlock(&exynos_mcpm_lock); 124 local_irq_enable(); 125 126 return 0; 127 } 128 129 /* 130 * NOTE: This function requires the stack data to be visible through power down 131 * and can only be executed on processors like A15 and A7 that hit the cache 132 * with the C bit clear in the SCTLR register. 133 */ 134 static void exynos_power_down(void) 135 { 136 unsigned int mpidr, cpu, cluster; 137 bool last_man = false, skip_wfi = false; 138 unsigned int cpunr; 139 140 mpidr = read_cpuid_mpidr(); 141 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); 142 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); 143 cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); 144 145 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); 146 BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || 147 cluster >= EXYNOS5420_NR_CLUSTERS); 148 149 __mcpm_cpu_going_down(cpu, cluster); 150 151 arch_spin_lock(&exynos_mcpm_lock); 152 BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP); 153 cpu_use_count[cpu][cluster]--; 154 if (cpu_use_count[cpu][cluster] == 0) { 155 exynos_cpu_power_down(cpunr); 156 157 if (exynos_cluster_unused(cluster)) { 158 exynos_cluster_power_down(cluster); 159 last_man = true; 160 } 161 } else if (cpu_use_count[cpu][cluster] == 1) { 162 /* 163 * A power_up request went ahead of us. 164 * Even if we do not want to shut this CPU down, 165 * the caller expects a certain state as if the WFI 166 * was aborted. So let's continue with cache cleaning. 167 */ 168 skip_wfi = true; 169 } else { 170 BUG(); 171 } 172 173 if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) { 174 arch_spin_unlock(&exynos_mcpm_lock); 175 176 if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) { 177 /* 178 * On the Cortex-A15 we need to disable 179 * L2 prefetching before flushing the cache. 180 */ 181 asm volatile( 182 "mcr p15, 1, %0, c15, c0, 3\n\t" 183 "isb\n\t" 184 "dsb" 185 : : "r" (0x400)); 186 } 187 188 /* Flush all cache levels for this cluster. */ 189 exynos_v7_exit_coherency_flush(all); 190 191 /* 192 * Disable cluster-level coherency by masking 193 * incoming snoops and DVM messages: 194 */ 195 cci_disable_port_by_cpu(mpidr); 196 197 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN); 198 } else { 199 arch_spin_unlock(&exynos_mcpm_lock); 200 201 /* Disable and flush the local CPU cache. */ 202 exynos_v7_exit_coherency_flush(louis); 203 } 204 205 __mcpm_cpu_down(cpu, cluster); 206 207 /* Now we are prepared for power-down, do it: */ 208 if (!skip_wfi) 209 wfi(); 210 211 /* Not dead at this point? Let our caller cope. */ 212 } 213 214 static int exynos_wait_for_powerdown(unsigned int cpu, unsigned int cluster) 215 { 216 unsigned int tries = 100; 217 unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); 218 219 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); 220 BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || 221 cluster >= EXYNOS5420_NR_CLUSTERS); 222 223 /* Wait for the core state to be OFF */ 224 while (tries--) { 225 if (ACCESS_ONCE(cpu_use_count[cpu][cluster]) == 0) { 226 if ((exynos_cpu_power_state(cpunr) == 0)) 227 return 0; /* success: the CPU is halted */ 228 } 229 230 /* Otherwise, wait and retry: */ 231 msleep(1); 232 } 233 234 return -ETIMEDOUT; /* timeout */ 235 } 236 237 static void exynos_powered_up(void) 238 { 239 unsigned int mpidr, cpu, cluster; 240 241 mpidr = read_cpuid_mpidr(); 242 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); 243 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); 244 245 arch_spin_lock(&exynos_mcpm_lock); 246 if (cpu_use_count[cpu][cluster] == 0) 247 cpu_use_count[cpu][cluster] = 1; 248 arch_spin_unlock(&exynos_mcpm_lock); 249 } 250 251 static void exynos_suspend(u64 residency) 252 { 253 unsigned int mpidr, cpunr; 254 255 exynos_power_down(); 256 257 /* 258 * Execution reaches here only if cpu did not power down. 259 * Hence roll back the changes done in exynos_power_down function. 260 * 261 * CAUTION: "This function requires the stack data to be visible through 262 * power down and can only be executed on processors like A15 and A7 263 * that hit the cache with the C bit clear in the SCTLR register." 264 */ 265 mpidr = read_cpuid_mpidr(); 266 cpunr = exynos_pmu_cpunr(mpidr); 267 268 exynos_cpu_power_up(cpunr); 269 } 270 271 static const struct mcpm_platform_ops exynos_power_ops = { 272 .power_up = exynos_power_up, 273 .power_down = exynos_power_down, 274 .wait_for_powerdown = exynos_wait_for_powerdown, 275 .suspend = exynos_suspend, 276 .powered_up = exynos_powered_up, 277 }; 278 279 static void __init exynos_mcpm_usage_count_init(void) 280 { 281 unsigned int mpidr, cpu, cluster; 282 283 mpidr = read_cpuid_mpidr(); 284 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); 285 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); 286 287 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); 288 BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || 289 cluster >= EXYNOS5420_NR_CLUSTERS); 290 291 cpu_use_count[cpu][cluster] = 1; 292 } 293 294 /* 295 * Enable cluster-level coherency, in preparation for turning on the MMU. 296 */ 297 static void __naked exynos_pm_power_up_setup(unsigned int affinity_level) 298 { 299 asm volatile ("\n" 300 "cmp r0, #1\n" 301 "bxne lr\n" 302 "b cci_enable_port_for_self"); 303 } 304 305 static void __init exynos_cache_off(void) 306 { 307 if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) { 308 /* disable L2 prefetching on the Cortex-A15 */ 309 asm volatile( 310 "mcr p15, 1, %0, c15, c0, 3\n\t" 311 "isb\n\t" 312 "dsb" 313 : : "r" (0x400)); 314 } 315 exynos_v7_exit_coherency_flush(all); 316 } 317 318 static const struct of_device_id exynos_dt_mcpm_match[] = { 319 { .compatible = "samsung,exynos5420" }, 320 { .compatible = "samsung,exynos5800" }, 321 {}, 322 }; 323 324 static void exynos_mcpm_setup_entry_point(void) 325 { 326 /* 327 * U-Boot SPL is hardcoded to jump to the start of ns_sram_base_addr 328 * as part of secondary_cpu_start(). Let's redirect it to the 329 * mcpm_entry_point(). This is done during both secondary boot-up as 330 * well as system resume. 331 */ 332 __raw_writel(0xe59f0000, ns_sram_base_addr); /* ldr r0, [pc, #0] */ 333 __raw_writel(0xe12fff10, ns_sram_base_addr + 4); /* bx r0 */ 334 __raw_writel(virt_to_phys(mcpm_entry_point), ns_sram_base_addr + 8); 335 } 336 337 static struct syscore_ops exynos_mcpm_syscore_ops = { 338 .resume = exynos_mcpm_setup_entry_point, 339 }; 340 341 static int __init exynos_mcpm_init(void) 342 { 343 struct device_node *node; 344 unsigned int value, i; 345 int ret; 346 347 node = of_find_matching_node(NULL, exynos_dt_mcpm_match); 348 if (!node) 349 return -ENODEV; 350 of_node_put(node); 351 352 if (!cci_probed()) 353 return -ENODEV; 354 355 node = of_find_compatible_node(NULL, NULL, 356 "samsung,exynos4210-sysram-ns"); 357 if (!node) 358 return -ENODEV; 359 360 ns_sram_base_addr = of_iomap(node, 0); 361 of_node_put(node); 362 if (!ns_sram_base_addr) { 363 pr_err("failed to map non-secure iRAM base address\n"); 364 return -ENOMEM; 365 } 366 367 /* 368 * To increase the stability of KFC reset we need to program 369 * the PMU SPARE3 register 370 */ 371 pmu_raw_writel(EXYNOS5420_SWRESET_KFC_SEL, S5P_PMU_SPARE3); 372 373 exynos_mcpm_usage_count_init(); 374 375 ret = mcpm_platform_register(&exynos_power_ops); 376 if (!ret) 377 ret = mcpm_sync_init(exynos_pm_power_up_setup); 378 if (!ret) 379 ret = mcpm_loopback(exynos_cache_off); /* turn on the CCI */ 380 if (ret) { 381 iounmap(ns_sram_base_addr); 382 return ret; 383 } 384 385 mcpm_smp_set_ops(); 386 387 pr_info("Exynos MCPM support installed\n"); 388 389 /* 390 * On Exynos5420/5800 for the A15 and A7 clusters: 391 * 392 * EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN ensures that all the cores 393 * in a cluster are turned off before turning off the cluster L2. 394 * 395 * EXYNOS5420_USE_ARM_CORE_DOWN_STATE ensures that a cores is powered 396 * off before waking it up. 397 * 398 * EXYNOS5420_USE_L2_COMMON_UP_STATE ensures that cluster L2 will be 399 * turned on before the first man is powered up. 400 */ 401 for (i = 0; i < EXYNOS5420_NR_CLUSTERS; i++) { 402 value = pmu_raw_readl(EXYNOS_COMMON_OPTION(i)); 403 value |= EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN | 404 EXYNOS5420_USE_ARM_CORE_DOWN_STATE | 405 EXYNOS5420_USE_L2_COMMON_UP_STATE; 406 pmu_raw_writel(value, EXYNOS_COMMON_OPTION(i)); 407 } 408 409 exynos_mcpm_setup_entry_point(); 410 411 register_syscore_ops(&exynos_mcpm_syscore_ops); 412 413 return ret; 414 } 415 416 early_initcall(exynos_mcpm_init); 417