/* SPDX-License-Identifier: GPL-2.0-only */ /* * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM * * Created by: Nicolas Pitre, March 2012 * Copyright: (C) 2012-2013 Linaro Limited * * Refer to Documentation/arm/cluster-pm-race-avoidance.rst * for details of the synchronisation algorithms used here. */ #include <linux/linkage.h> #include <asm/mcpm.h> #include <asm/assembler.h> #include "vlock.h" .if MCPM_SYNC_CLUSTER_CPUS .error "cpus must be the first member of struct mcpm_sync_struct" .endif .macro pr_dbg string #if defined(CONFIG_DEBUG_LL) && defined(DEBUG) b 1901f 1902: .asciz "CPU" 1903: .asciz " cluster" 1904: .asciz ": \string" .align 1901: adr r0, 1902b bl printascii mov r0, r9 bl printhex2 adr r0, 1903b bl printascii mov r0, r10 bl printhex2 adr r0, 1904b bl printascii #endif .endm .arm .align ENTRY(mcpm_entry_point) ARM_BE8(setend be) THUMB( badr r12, 1f ) THUMB( bx r12 ) THUMB( .thumb ) 1: mrc p15, 0, r0, c0, c0, 5 @ MPIDR ubfx r9, r0, #0, #8 @ r9 = cpu ubfx r10, r0, #8, #8 @ r10 = cluster mov r3, #MAX_CPUS_PER_CLUSTER mla r4, r3, r10, r9 @ r4 = canonical CPU index cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS) blo 2f /* We didn't expect this CPU. Try to cheaply make it quiet. */ 1: wfi wfe b 1b 2: pr_dbg "kernel mcpm_entry_point\n" /* * MMU is off so we need to get to various variables in a * position independent way. */ adr r5, 3f ldmia r5, {r0, r6, r7, r8, r11} add r0, r5, r0 @ r0 = mcpm_entry_early_pokes add r6, r5, r6 @ r6 = mcpm_entry_vectors ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys add r8, r5, r8 @ r8 = mcpm_sync add r11, r5, r11 @ r11 = first_man_locks @ Perform an early poke, if any add r0, r0, r4, lsl #3 ldmia r0, {r0, r1} teq r0, #0 strne r1, [r0] mov r0, #MCPM_SYNC_CLUSTER_SIZE mla r8, r0, r10, r8 @ r8 = sync cluster base @ Signal that this CPU is coming UP: mov r0, #CPU_COMING_UP mov r5, #MCPM_SYNC_CPU_SIZE mla r5, r9, r5, r8 @ r5 = sync cpu address strb r0, [r5] @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN @ state, because there is at least one active CPU (this CPU). mov r0, #VLOCK_SIZE mla r11, r0, r10, r11 @ r11 = cluster first man lock mov r0, r11 mov r1, r9 @ cpu bl vlock_trylock @ implies DMB cmp r0, #0 @ failed to get the lock? bne mcpm_setup_wait @ wait for cluster setup if so ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] cmp r0, #CLUSTER_UP @ cluster already up? bne mcpm_setup @ if not, set up the cluster @ Otherwise, release the first man lock and skip setup: mov r0, r11 bl vlock_unlock b mcpm_setup_complete mcpm_setup: @ Control dependency implies strb not observable before previous ldrb. @ Signal that the cluster is being brought up: mov r0, #INBOUND_COMING_UP strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] dmb @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this @ point onwards will observe INBOUND_COMING_UP and abort. @ Wait for any previously-pending cluster teardown operations to abort @ or complete: mcpm_teardown_wait: ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] cmp r0, #CLUSTER_GOING_DOWN bne first_man_setup wfe b mcpm_teardown_wait first_man_setup: dmb @ If the outbound gave up before teardown started, skip cluster setup: cmp r0, #CLUSTER_UP beq mcpm_setup_leave @ power_up_setup is now responsible for setting up the cluster: cmp r7, #0 mov r0, #1 @ second (cluster) affinity level blxne r7 @ Call power_up_setup if defined dmb mov r0, #CLUSTER_UP strb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] dmb mcpm_setup_leave: @ Leave the cluster setup critical section: mov r0, #INBOUND_NOT_COMING_UP strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] dsb st sev mov r0, r11 bl vlock_unlock @ implies DMB b mcpm_setup_complete @ In the contended case, non-first men wait here for cluster setup @ to complete: mcpm_setup_wait: ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] cmp r0, #CLUSTER_UP wfene bne mcpm_setup_wait dmb mcpm_setup_complete: @ If a platform-specific CPU setup hook is needed, it is @ called from here. cmp r7, #0 mov r0, #0 @ first (CPU) affinity level blxne r7 @ Call power_up_setup if defined dmb @ Mark the CPU as up: mov r0, #CPU_UP strb r0, [r5] @ Observability order of CPU_UP and opening of the gate does not matter. mcpm_entry_gated: ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector cmp r5, #0 wfeeq beq mcpm_entry_gated dmb pr_dbg "released\n" bx r5 .align 2 3: .word mcpm_entry_early_pokes - . .word mcpm_entry_vectors - 3b .word mcpm_power_up_setup_phys - 3b .word mcpm_sync - 3b .word first_man_locks - 3b ENDPROC(mcpm_entry_point) .bss .align CACHE_WRITEBACK_ORDER .type first_man_locks, #object first_man_locks: .space VLOCK_SIZE * MAX_NR_CLUSTERS .align CACHE_WRITEBACK_ORDER .type mcpm_entry_vectors, #object ENTRY(mcpm_entry_vectors) .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER .type mcpm_entry_early_pokes, #object ENTRY(mcpm_entry_early_pokes) .space 8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER .type mcpm_power_up_setup_phys, #object ENTRY(mcpm_power_up_setup_phys) .space 4 @ set by mcpm_sync_init()