1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2f6f9be1cSFlorian Fainelli /*
3f6f9be1cSFlorian Fainelli * Broadcom Brahma-B15 CPU read-ahead cache management functions
4f6f9be1cSFlorian Fainelli *
5f6f9be1cSFlorian Fainelli * Copyright (C) 2015-2016 Broadcom
6f6f9be1cSFlorian Fainelli */
7f6f9be1cSFlorian Fainelli
8f6f9be1cSFlorian Fainelli #include <linux/err.h>
9f6f9be1cSFlorian Fainelli #include <linux/spinlock.h>
10f6f9be1cSFlorian Fainelli #include <linux/io.h>
11f6f9be1cSFlorian Fainelli #include <linux/bitops.h>
12f6f9be1cSFlorian Fainelli #include <linux/of_address.h>
1355de8877SFlorian Fainelli #include <linux/notifier.h>
1455de8877SFlorian Fainelli #include <linux/cpu.h>
15534f5f36SFlorian Fainelli #include <linux/syscore_ops.h>
16576a0860SFlorian Fainelli #include <linux/reboot.h>
17f6f9be1cSFlorian Fainelli
18f6f9be1cSFlorian Fainelli #include <asm/cacheflush.h>
19f6f9be1cSFlorian Fainelli #include <asm/hardware/cache-b15-rac.h>
20f6f9be1cSFlorian Fainelli
21f6f9be1cSFlorian Fainelli extern void v7_flush_kern_cache_all(void);
22f6f9be1cSFlorian Fainelli
23f6f9be1cSFlorian Fainelli /* RAC register offsets, relative to the HIF_CPU_BIUCTRL register base */
24f6f9be1cSFlorian Fainelli #define RAC_CONFIG0_REG (0x78)
25f6f9be1cSFlorian Fainelli #define RACENPREF_MASK (0x3)
26f6f9be1cSFlorian Fainelli #define RACPREFINST_SHIFT (0)
27f6f9be1cSFlorian Fainelli #define RACENINST_SHIFT (2)
28f6f9be1cSFlorian Fainelli #define RACPREFDATA_SHIFT (4)
29f6f9be1cSFlorian Fainelli #define RACENDATA_SHIFT (6)
30f6f9be1cSFlorian Fainelli #define RAC_CPU_SHIFT (8)
31f6f9be1cSFlorian Fainelli #define RACCFG_MASK (0xff)
32f6f9be1cSFlorian Fainelli #define RAC_CONFIG1_REG (0x7c)
3348e6dd79SFlorian Fainelli /* Brahma-B15 is a quad-core only design */
3448e6dd79SFlorian Fainelli #define B15_RAC_FLUSH_REG (0x80)
3548e6dd79SFlorian Fainelli /* Brahma-B53 is an octo-core design */
3648e6dd79SFlorian Fainelli #define B53_RAC_FLUSH_REG (0x84)
37f6f9be1cSFlorian Fainelli #define FLUSH_RAC (1 << 0)
38f6f9be1cSFlorian Fainelli
39f6f9be1cSFlorian Fainelli /* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
40f6f9be1cSFlorian Fainelli #define RAC_DATA_INST_EN_MASK (1 << RACPREFINST_SHIFT | \
41f6f9be1cSFlorian Fainelli RACENPREF_MASK << RACENINST_SHIFT | \
42f6f9be1cSFlorian Fainelli 1 << RACPREFDATA_SHIFT | \
43f6f9be1cSFlorian Fainelli RACENPREF_MASK << RACENDATA_SHIFT)
44f6f9be1cSFlorian Fainelli
45f6f9be1cSFlorian Fainelli #define RAC_ENABLED 0
46534f5f36SFlorian Fainelli /* Special state where we want to bypass the spinlock and call directly
47534f5f36SFlorian Fainelli * into the v7 cache maintenance operations during suspend/resume
48534f5f36SFlorian Fainelli */
49534f5f36SFlorian Fainelli #define RAC_SUSPENDED 1
50f6f9be1cSFlorian Fainelli
51f6f9be1cSFlorian Fainelli static void __iomem *b15_rac_base;
52f6f9be1cSFlorian Fainelli static DEFINE_SPINLOCK(rac_lock);
53a5281feaSArnd Bergmann
5455de8877SFlorian Fainelli static u32 rac_config0_reg;
5548e6dd79SFlorian Fainelli static u32 rac_flush_offset;
56f6f9be1cSFlorian Fainelli
57f6f9be1cSFlorian Fainelli /* Initialization flag to avoid checking for b15_rac_base, and to prevent
58f6f9be1cSFlorian Fainelli * multi-platform kernels from crashing here as well.
59f6f9be1cSFlorian Fainelli */
60f6f9be1cSFlorian Fainelli static unsigned long b15_rac_flags;
61f6f9be1cSFlorian Fainelli
__b15_rac_disable(void)62f6f9be1cSFlorian Fainelli static inline u32 __b15_rac_disable(void)
63f6f9be1cSFlorian Fainelli {
64f6f9be1cSFlorian Fainelli u32 val = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
65f6f9be1cSFlorian Fainelli __raw_writel(0, b15_rac_base + RAC_CONFIG0_REG);
66f6f9be1cSFlorian Fainelli dmb();
67f6f9be1cSFlorian Fainelli return val;
68f6f9be1cSFlorian Fainelli }
69f6f9be1cSFlorian Fainelli
__b15_rac_flush(void)70f6f9be1cSFlorian Fainelli static inline void __b15_rac_flush(void)
71f6f9be1cSFlorian Fainelli {
72f6f9be1cSFlorian Fainelli u32 reg;
73f6f9be1cSFlorian Fainelli
7448e6dd79SFlorian Fainelli __raw_writel(FLUSH_RAC, b15_rac_base + rac_flush_offset);
75f6f9be1cSFlorian Fainelli do {
76f6f9be1cSFlorian Fainelli /* This dmb() is required to force the Bus Interface Unit
77*05e3a8cbSJulia Lawall * to clean outstanding writes, and forces an idle cycle
78f6f9be1cSFlorian Fainelli * to be inserted.
79f6f9be1cSFlorian Fainelli */
80f6f9be1cSFlorian Fainelli dmb();
8148e6dd79SFlorian Fainelli reg = __raw_readl(b15_rac_base + rac_flush_offset);
82f6f9be1cSFlorian Fainelli } while (reg & FLUSH_RAC);
83f6f9be1cSFlorian Fainelli }
84f6f9be1cSFlorian Fainelli
b15_rac_disable_and_flush(void)85f6f9be1cSFlorian Fainelli static inline u32 b15_rac_disable_and_flush(void)
86f6f9be1cSFlorian Fainelli {
87f6f9be1cSFlorian Fainelli u32 reg;
88f6f9be1cSFlorian Fainelli
89f6f9be1cSFlorian Fainelli reg = __b15_rac_disable();
90f6f9be1cSFlorian Fainelli __b15_rac_flush();
91f6f9be1cSFlorian Fainelli return reg;
92f6f9be1cSFlorian Fainelli }
93f6f9be1cSFlorian Fainelli
__b15_rac_enable(u32 val)94f6f9be1cSFlorian Fainelli static inline void __b15_rac_enable(u32 val)
95f6f9be1cSFlorian Fainelli {
96f6f9be1cSFlorian Fainelli __raw_writel(val, b15_rac_base + RAC_CONFIG0_REG);
97f6f9be1cSFlorian Fainelli /* dsb() is required here to be consistent with __flush_icache_all() */
98f6f9be1cSFlorian Fainelli dsb();
99f6f9be1cSFlorian Fainelli }
100f6f9be1cSFlorian Fainelli
101f6f9be1cSFlorian Fainelli #define BUILD_RAC_CACHE_OP(name, bar) \
102f6f9be1cSFlorian Fainelli void b15_flush_##name(void) \
103f6f9be1cSFlorian Fainelli { \
104f6f9be1cSFlorian Fainelli unsigned int do_flush; \
105f6f9be1cSFlorian Fainelli u32 val = 0; \
106f6f9be1cSFlorian Fainelli \
107534f5f36SFlorian Fainelli if (test_bit(RAC_SUSPENDED, &b15_rac_flags)) { \
108534f5f36SFlorian Fainelli v7_flush_##name(); \
109534f5f36SFlorian Fainelli bar; \
110534f5f36SFlorian Fainelli return; \
111534f5f36SFlorian Fainelli } \
112534f5f36SFlorian Fainelli \
113f6f9be1cSFlorian Fainelli spin_lock(&rac_lock); \
114f6f9be1cSFlorian Fainelli do_flush = test_bit(RAC_ENABLED, &b15_rac_flags); \
115f6f9be1cSFlorian Fainelli if (do_flush) \
116f6f9be1cSFlorian Fainelli val = b15_rac_disable_and_flush(); \
117f6f9be1cSFlorian Fainelli v7_flush_##name(); \
118f6f9be1cSFlorian Fainelli if (!do_flush) \
119f6f9be1cSFlorian Fainelli bar; \
120f6f9be1cSFlorian Fainelli else \
121f6f9be1cSFlorian Fainelli __b15_rac_enable(val); \
122f6f9be1cSFlorian Fainelli spin_unlock(&rac_lock); \
123f6f9be1cSFlorian Fainelli }
124f6f9be1cSFlorian Fainelli
125f6f9be1cSFlorian Fainelli #define nobarrier
126f6f9be1cSFlorian Fainelli
127f6f9be1cSFlorian Fainelli /* The readahead cache present in the Brahma-B15 CPU is a special piece of
128f6f9be1cSFlorian Fainelli * hardware after the integrated L2 cache of the B15 CPU complex whose purpose
129f6f9be1cSFlorian Fainelli * is to prefetch instruction and/or data with a line size of either 64 bytes
130f6f9be1cSFlorian Fainelli * or 256 bytes. The rationale is that the data-bus of the CPU interface is
131f6f9be1cSFlorian Fainelli * optimized for 256-bytes transactions, and enabling the readahead cache
132f6f9be1cSFlorian Fainelli * provides a significant performance boost we want it enabled (typically
133f6f9be1cSFlorian Fainelli * twice the performance for a memcpy benchmark application).
134f6f9be1cSFlorian Fainelli *
135f6f9be1cSFlorian Fainelli * The readahead cache is transparent for Modified Virtual Addresses
136f6f9be1cSFlorian Fainelli * cache maintenance operations: ICIMVAU, DCIMVAC, DCCMVAC, DCCMVAU and
137f6f9be1cSFlorian Fainelli * DCCIMVAC.
138f6f9be1cSFlorian Fainelli *
139f6f9be1cSFlorian Fainelli * It is however not transparent for the following cache maintenance
140f6f9be1cSFlorian Fainelli * operations: DCISW, DCCSW, DCCISW, ICIALLUIS and ICIALLU which is precisely
141f6f9be1cSFlorian Fainelli * what we are patching here with our BUILD_RAC_CACHE_OP here.
142f6f9be1cSFlorian Fainelli */
143f6f9be1cSFlorian Fainelli BUILD_RAC_CACHE_OP(kern_cache_all, nobarrier);
144f6f9be1cSFlorian Fainelli
b15_rac_enable(void)145f6f9be1cSFlorian Fainelli static void b15_rac_enable(void)
146f6f9be1cSFlorian Fainelli {
147f6f9be1cSFlorian Fainelli unsigned int cpu;
148f6f9be1cSFlorian Fainelli u32 enable = 0;
149f6f9be1cSFlorian Fainelli
150f6f9be1cSFlorian Fainelli for_each_possible_cpu(cpu)
151f6f9be1cSFlorian Fainelli enable |= (RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT));
152f6f9be1cSFlorian Fainelli
153f6f9be1cSFlorian Fainelli b15_rac_disable_and_flush();
154f6f9be1cSFlorian Fainelli __b15_rac_enable(enable);
155f6f9be1cSFlorian Fainelli }
156f6f9be1cSFlorian Fainelli
b15_rac_reboot_notifier(struct notifier_block * nb,unsigned long action,void * data)157576a0860SFlorian Fainelli static int b15_rac_reboot_notifier(struct notifier_block *nb,
158576a0860SFlorian Fainelli unsigned long action,
159576a0860SFlorian Fainelli void *data)
160576a0860SFlorian Fainelli {
161576a0860SFlorian Fainelli /* During kexec, we are not yet migrated on the boot CPU, so we need to
162576a0860SFlorian Fainelli * make sure we are SMP safe here. Once the RAC is disabled, flag it as
163576a0860SFlorian Fainelli * suspended such that the hotplug notifier returns early.
164576a0860SFlorian Fainelli */
165576a0860SFlorian Fainelli if (action == SYS_RESTART) {
166576a0860SFlorian Fainelli spin_lock(&rac_lock);
167576a0860SFlorian Fainelli b15_rac_disable_and_flush();
168576a0860SFlorian Fainelli clear_bit(RAC_ENABLED, &b15_rac_flags);
169576a0860SFlorian Fainelli set_bit(RAC_SUSPENDED, &b15_rac_flags);
170576a0860SFlorian Fainelli spin_unlock(&rac_lock);
171576a0860SFlorian Fainelli }
172576a0860SFlorian Fainelli
173576a0860SFlorian Fainelli return NOTIFY_DONE;
174576a0860SFlorian Fainelli }
175576a0860SFlorian Fainelli
176576a0860SFlorian Fainelli static struct notifier_block b15_rac_reboot_nb = {
177576a0860SFlorian Fainelli .notifier_call = b15_rac_reboot_notifier,
178576a0860SFlorian Fainelli };
179576a0860SFlorian Fainelli
18055de8877SFlorian Fainelli /* The CPU hotplug case is the most interesting one, we basically need to make
18155de8877SFlorian Fainelli * sure that the RAC is disabled for the entire system prior to having a CPU
18255de8877SFlorian Fainelli * die, in particular prior to this dying CPU having exited the coherency
18355de8877SFlorian Fainelli * domain.
18455de8877SFlorian Fainelli *
18555de8877SFlorian Fainelli * Once this CPU is marked dead, we can safely re-enable the RAC for the
18655de8877SFlorian Fainelli * remaining CPUs in the system which are still online.
18755de8877SFlorian Fainelli *
18855de8877SFlorian Fainelli * Offlining a CPU is the problematic case, onlining a CPU is not much of an
18955de8877SFlorian Fainelli * issue since the CPU and its cache-level hierarchy will start filling with
19055de8877SFlorian Fainelli * the RAC disabled, so L1 and L2 only.
19155de8877SFlorian Fainelli *
19255de8877SFlorian Fainelli * In this function, we should NOT have to verify any unsafe setting/condition
19355de8877SFlorian Fainelli * b15_rac_base:
19455de8877SFlorian Fainelli *
19555de8877SFlorian Fainelli * It is protected by the RAC_ENABLED flag which is cleared by default, and
19655de8877SFlorian Fainelli * being cleared when initial procedure is done. b15_rac_base had been set at
19755de8877SFlorian Fainelli * that time.
19855de8877SFlorian Fainelli *
19955de8877SFlorian Fainelli * RAC_ENABLED:
20055de8877SFlorian Fainelli * There is a small timing windows, in b15_rac_init(), between
20155de8877SFlorian Fainelli * cpuhp_setup_state_*()
20255de8877SFlorian Fainelli * ...
20355de8877SFlorian Fainelli * set RAC_ENABLED
20455de8877SFlorian Fainelli * However, there is no hotplug activity based on the Linux booting procedure.
20555de8877SFlorian Fainelli *
20655de8877SFlorian Fainelli * Since we have to disable RAC for all cores, we keep RAC on as long as as
20755de8877SFlorian Fainelli * possible (disable it as late as possible) to gain the cache benefit.
20855de8877SFlorian Fainelli *
20955de8877SFlorian Fainelli * Thus, dying/dead states are chosen here
21055de8877SFlorian Fainelli *
21155de8877SFlorian Fainelli * We are choosing not do disable the RAC on a per-CPU basis, here, if we did
21255de8877SFlorian Fainelli * we would want to consider disabling it as early as possible to benefit the
21355de8877SFlorian Fainelli * other active CPUs.
21455de8877SFlorian Fainelli */
21555de8877SFlorian Fainelli
21655de8877SFlorian Fainelli /* Running on the dying CPU */
b15_rac_dying_cpu(unsigned int cpu)21755de8877SFlorian Fainelli static int b15_rac_dying_cpu(unsigned int cpu)
21855de8877SFlorian Fainelli {
219576a0860SFlorian Fainelli /* During kexec/reboot, the RAC is disabled via the reboot notifier
220576a0860SFlorian Fainelli * return early here.
221576a0860SFlorian Fainelli */
222576a0860SFlorian Fainelli if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
223576a0860SFlorian Fainelli return 0;
224576a0860SFlorian Fainelli
22555de8877SFlorian Fainelli spin_lock(&rac_lock);
22655de8877SFlorian Fainelli
22755de8877SFlorian Fainelli /* Indicate that we are starting a hotplug procedure */
22855de8877SFlorian Fainelli __clear_bit(RAC_ENABLED, &b15_rac_flags);
22955de8877SFlorian Fainelli
23055de8877SFlorian Fainelli /* Disable the readahead cache and save its value to a global */
23155de8877SFlorian Fainelli rac_config0_reg = b15_rac_disable_and_flush();
23255de8877SFlorian Fainelli
23355de8877SFlorian Fainelli spin_unlock(&rac_lock);
23455de8877SFlorian Fainelli
23555de8877SFlorian Fainelli return 0;
23655de8877SFlorian Fainelli }
23755de8877SFlorian Fainelli
23855de8877SFlorian Fainelli /* Running on a non-dying CPU */
b15_rac_dead_cpu(unsigned int cpu)23955de8877SFlorian Fainelli static int b15_rac_dead_cpu(unsigned int cpu)
24055de8877SFlorian Fainelli {
241576a0860SFlorian Fainelli /* During kexec/reboot, the RAC is disabled via the reboot notifier
242576a0860SFlorian Fainelli * return early here.
243576a0860SFlorian Fainelli */
244576a0860SFlorian Fainelli if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
245576a0860SFlorian Fainelli return 0;
246576a0860SFlorian Fainelli
24755de8877SFlorian Fainelli spin_lock(&rac_lock);
24855de8877SFlorian Fainelli
24955de8877SFlorian Fainelli /* And enable it */
25055de8877SFlorian Fainelli __b15_rac_enable(rac_config0_reg);
25155de8877SFlorian Fainelli __set_bit(RAC_ENABLED, &b15_rac_flags);
25255de8877SFlorian Fainelli
25355de8877SFlorian Fainelli spin_unlock(&rac_lock);
25455de8877SFlorian Fainelli
25555de8877SFlorian Fainelli return 0;
25655de8877SFlorian Fainelli }
25755de8877SFlorian Fainelli
b15_rac_suspend(void)258534f5f36SFlorian Fainelli static int b15_rac_suspend(void)
259534f5f36SFlorian Fainelli {
260534f5f36SFlorian Fainelli /* Suspend the read-ahead cache oeprations, forcing our cache
261534f5f36SFlorian Fainelli * implementation to fallback to the regular ARMv7 calls.
262534f5f36SFlorian Fainelli *
263534f5f36SFlorian Fainelli * We are guaranteed to be running on the boot CPU at this point and
264534f5f36SFlorian Fainelli * with every other CPU quiesced, so setting RAC_SUSPENDED is not racy
265534f5f36SFlorian Fainelli * here.
266534f5f36SFlorian Fainelli */
267534f5f36SFlorian Fainelli rac_config0_reg = b15_rac_disable_and_flush();
268534f5f36SFlorian Fainelli set_bit(RAC_SUSPENDED, &b15_rac_flags);
269534f5f36SFlorian Fainelli
270534f5f36SFlorian Fainelli return 0;
271534f5f36SFlorian Fainelli }
272534f5f36SFlorian Fainelli
b15_rac_resume(void)273534f5f36SFlorian Fainelli static void b15_rac_resume(void)
274534f5f36SFlorian Fainelli {
275534f5f36SFlorian Fainelli /* Coming out of a S3 suspend/resume cycle, the read-ahead cache
276534f5f36SFlorian Fainelli * register RAC_CONFIG0_REG will be restored to its default value, make
277534f5f36SFlorian Fainelli * sure we re-enable it and set the enable flag, we are also guaranteed
278534f5f36SFlorian Fainelli * to run on the boot CPU, so not racy again.
279534f5f36SFlorian Fainelli */
280534f5f36SFlorian Fainelli __b15_rac_enable(rac_config0_reg);
281534f5f36SFlorian Fainelli clear_bit(RAC_SUSPENDED, &b15_rac_flags);
282534f5f36SFlorian Fainelli }
283534f5f36SFlorian Fainelli
284534f5f36SFlorian Fainelli static struct syscore_ops b15_rac_syscore_ops = {
285534f5f36SFlorian Fainelli .suspend = b15_rac_suspend,
286534f5f36SFlorian Fainelli .resume = b15_rac_resume,
287534f5f36SFlorian Fainelli };
288534f5f36SFlorian Fainelli
b15_rac_init(void)289f6f9be1cSFlorian Fainelli static int __init b15_rac_init(void)
290f6f9be1cSFlorian Fainelli {
29148e6dd79SFlorian Fainelli struct device_node *dn, *cpu_dn;
292f6f9be1cSFlorian Fainelli int ret = 0, cpu;
293f6f9be1cSFlorian Fainelli u32 reg, en_mask = 0;
294f6f9be1cSFlorian Fainelli
295f6f9be1cSFlorian Fainelli dn = of_find_compatible_node(NULL, NULL, "brcm,brcmstb-cpu-biu-ctrl");
296f6f9be1cSFlorian Fainelli if (!dn)
297f6f9be1cSFlorian Fainelli return -ENODEV;
298f6f9be1cSFlorian Fainelli
299f6f9be1cSFlorian Fainelli if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n"))
300f6f9be1cSFlorian Fainelli goto out;
301f6f9be1cSFlorian Fainelli
302f6f9be1cSFlorian Fainelli b15_rac_base = of_iomap(dn, 0);
303f6f9be1cSFlorian Fainelli if (!b15_rac_base) {
304f6f9be1cSFlorian Fainelli pr_err("failed to remap BIU control base\n");
305f6f9be1cSFlorian Fainelli ret = -ENOMEM;
306f6f9be1cSFlorian Fainelli goto out;
307f6f9be1cSFlorian Fainelli }
308f6f9be1cSFlorian Fainelli
30948e6dd79SFlorian Fainelli cpu_dn = of_get_cpu_node(0, NULL);
31048e6dd79SFlorian Fainelli if (!cpu_dn) {
31148e6dd79SFlorian Fainelli ret = -ENODEV;
31248e6dd79SFlorian Fainelli goto out;
31348e6dd79SFlorian Fainelli }
31448e6dd79SFlorian Fainelli
31548e6dd79SFlorian Fainelli if (of_device_is_compatible(cpu_dn, "brcm,brahma-b15"))
31648e6dd79SFlorian Fainelli rac_flush_offset = B15_RAC_FLUSH_REG;
31748e6dd79SFlorian Fainelli else if (of_device_is_compatible(cpu_dn, "brcm,brahma-b53"))
31848e6dd79SFlorian Fainelli rac_flush_offset = B53_RAC_FLUSH_REG;
31948e6dd79SFlorian Fainelli else {
32048e6dd79SFlorian Fainelli pr_err("Unsupported CPU\n");
32148e6dd79SFlorian Fainelli of_node_put(cpu_dn);
32248e6dd79SFlorian Fainelli ret = -EINVAL;
32348e6dd79SFlorian Fainelli goto out;
32448e6dd79SFlorian Fainelli }
32548e6dd79SFlorian Fainelli of_node_put(cpu_dn);
32648e6dd79SFlorian Fainelli
327576a0860SFlorian Fainelli ret = register_reboot_notifier(&b15_rac_reboot_nb);
328576a0860SFlorian Fainelli if (ret) {
329576a0860SFlorian Fainelli pr_err("failed to register reboot notifier\n");
330576a0860SFlorian Fainelli iounmap(b15_rac_base);
331576a0860SFlorian Fainelli goto out;
332576a0860SFlorian Fainelli }
333576a0860SFlorian Fainelli
334a5281feaSArnd Bergmann if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
33555de8877SFlorian Fainelli ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DEAD,
33655de8877SFlorian Fainelli "arm/cache-b15-rac:dead",
33755de8877SFlorian Fainelli NULL, b15_rac_dead_cpu);
33855de8877SFlorian Fainelli if (ret)
33955de8877SFlorian Fainelli goto out_unmap;
34055de8877SFlorian Fainelli
34155de8877SFlorian Fainelli ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING,
34255de8877SFlorian Fainelli "arm/cache-b15-rac:dying",
34355de8877SFlorian Fainelli NULL, b15_rac_dying_cpu);
34455de8877SFlorian Fainelli if (ret)
34555de8877SFlorian Fainelli goto out_cpu_dead;
346a5281feaSArnd Bergmann }
34755de8877SFlorian Fainelli
348a5281feaSArnd Bergmann if (IS_ENABLED(CONFIG_PM_SLEEP))
349534f5f36SFlorian Fainelli register_syscore_ops(&b15_rac_syscore_ops);
350534f5f36SFlorian Fainelli
351f6f9be1cSFlorian Fainelli spin_lock(&rac_lock);
352f6f9be1cSFlorian Fainelli reg = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
353f6f9be1cSFlorian Fainelli for_each_possible_cpu(cpu)
354f6f9be1cSFlorian Fainelli en_mask |= ((1 << RACPREFDATA_SHIFT) << (cpu * RAC_CPU_SHIFT));
355f6f9be1cSFlorian Fainelli WARN(reg & en_mask, "Read-ahead cache not previously disabled\n");
356f6f9be1cSFlorian Fainelli
357f6f9be1cSFlorian Fainelli b15_rac_enable();
358f6f9be1cSFlorian Fainelli set_bit(RAC_ENABLED, &b15_rac_flags);
359f6f9be1cSFlorian Fainelli spin_unlock(&rac_lock);
360f6f9be1cSFlorian Fainelli
361446937a5SFlorian Fainelli pr_info("%pOF: Broadcom Brahma-B15 readahead cache\n", dn);
362f6f9be1cSFlorian Fainelli
36355de8877SFlorian Fainelli goto out;
36455de8877SFlorian Fainelli
36555de8877SFlorian Fainelli out_cpu_dead:
36655de8877SFlorian Fainelli cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING);
36755de8877SFlorian Fainelli out_unmap:
368576a0860SFlorian Fainelli unregister_reboot_notifier(&b15_rac_reboot_nb);
36955de8877SFlorian Fainelli iounmap(b15_rac_base);
370f6f9be1cSFlorian Fainelli out:
371f6f9be1cSFlorian Fainelli of_node_put(dn);
372f6f9be1cSFlorian Fainelli return ret;
373f6f9be1cSFlorian Fainelli }
374f6f9be1cSFlorian Fainelli arch_initcall(b15_rac_init);
375