1*83d290c5STom Rini/* SPDX-License-Identifier: GPL-2.0+ */ 20ae76531SDavid Feng/* 30ae76531SDavid Feng * (C) Copyright 2013 40ae76531SDavid Feng * David Feng <fenghua@phytium.com.cn> 50ae76531SDavid Feng * 60ae76531SDavid Feng * This file is based on sample code from ARMv8 ARM. 70ae76531SDavid Feng */ 80ae76531SDavid Feng 90ae76531SDavid Feng#include <asm-offsets.h> 100ae76531SDavid Feng#include <config.h> 110ae76531SDavid Feng#include <asm/macro.h> 125e2ec773SAlexander Graf#include <asm/system.h> 130ae76531SDavid Feng#include <linux/linkage.h> 140ae76531SDavid Feng 150ae76531SDavid Feng/* 16ba9eb6c7SMasahiro Yamada * void __asm_dcache_level(level) 170ae76531SDavid Feng * 18ba9eb6c7SMasahiro Yamada * flush or invalidate one level cache. 190ae76531SDavid Feng * 200ae76531SDavid Feng * x0: cache level 211a021230SMasahiro Yamada * x1: 0 clean & invalidate, 1 invalidate only 221e6ad55cSYork Sun * x2~x9: clobbered 230ae76531SDavid Feng */ 24e6a05862SPhilipp Tomsich.pushsection .text.__asm_dcache_level, "ax" 25ba9eb6c7SMasahiro YamadaENTRY(__asm_dcache_level) 261e6ad55cSYork Sun lsl x12, x0, #1 271e6ad55cSYork Sun msr csselr_el1, x12 /* select cache level */ 280ae76531SDavid Feng isb /* sync change of cssidr_el1 */ 290ae76531SDavid Feng mrs x6, ccsidr_el1 /* read the new cssidr_el1 */ 300ae76531SDavid Feng and x2, x6, #7 /* x2 <- log2(cache line size)-4 */ 310ae76531SDavid Feng add x2, x2, #4 /* x2 <- log2(cache line size) */ 320ae76531SDavid Feng mov x3, #0x3ff 330ae76531SDavid Feng and x3, x3, x6, lsr #3 /* x3 <- max number of #ways */ 3442ddfad6SLeo Yan clz w5, w3 /* bit position of #ways */ 350ae76531SDavid Feng mov x4, #0x7fff 360ae76531SDavid Feng and x4, x4, x6, lsr #13 /* x4 <- max number of #sets */ 371e6ad55cSYork Sun /* x12 <- cache level << 1 */ 380ae76531SDavid Feng /* x2 <- line length offset */ 390ae76531SDavid Feng /* x3 <- number of cache ways - 1 */ 400ae76531SDavid Feng /* x4 <- number of cache sets - 1 */ 410ae76531SDavid Feng /* x5 <- bit position of #ways */ 420ae76531SDavid Feng 430ae76531SDavid Fengloop_set: 440ae76531SDavid Feng mov x6, x3 /* x6 <- working copy of #ways */ 450ae76531SDavid Fengloop_way: 460ae76531SDavid Feng lsl x7, x6, x5 471e6ad55cSYork Sun orr x9, x12, x7 /* map way and level to cisw value */ 480ae76531SDavid Feng lsl x7, x4, x2 490ae76531SDavid Feng orr x9, x9, x7 /* map set number to cisw value */ 501e6ad55cSYork Sun tbz w1, #0, 1f 511e6ad55cSYork Sun dc isw, x9 521e6ad55cSYork Sun b 2f 531e6ad55cSYork Sun1: dc cisw, x9 /* clean & invalidate by set/way */ 541e6ad55cSYork Sun2: subs x6, x6, #1 /* decrement the way */ 550ae76531SDavid Feng b.ge loop_way 560ae76531SDavid Feng subs x4, x4, #1 /* decrement the set */ 570ae76531SDavid Feng b.ge loop_set 580ae76531SDavid Feng 590ae76531SDavid Feng ret 60ba9eb6c7SMasahiro YamadaENDPROC(__asm_dcache_level) 61e6a05862SPhilipp Tomsich.popsection 620ae76531SDavid Feng 630ae76531SDavid Feng/* 641e6ad55cSYork Sun * void __asm_flush_dcache_all(int invalidate_only) 651e6ad55cSYork Sun * 661a021230SMasahiro Yamada * x0: 0 clean & invalidate, 1 invalidate only 670ae76531SDavid Feng * 68ba9eb6c7SMasahiro Yamada * flush or invalidate all data cache by SET/WAY. 690ae76531SDavid Feng */ 70e6a05862SPhilipp Tomsich.pushsection .text.__asm_dcache_all, "ax" 711e6ad55cSYork SunENTRY(__asm_dcache_all) 721e6ad55cSYork Sun mov x1, x0 730ae76531SDavid Feng dsb sy 740ae76531SDavid Feng mrs x10, clidr_el1 /* read clidr_el1 */ 750ae76531SDavid Feng lsr x11, x10, #24 760ae76531SDavid Feng and x11, x11, #0x7 /* x11 <- loc */ 770ae76531SDavid Feng cbz x11, finished /* if loc is 0, exit */ 780ae76531SDavid Feng mov x15, lr 790ae76531SDavid Feng mov x0, #0 /* start flush at cache level 0 */ 800ae76531SDavid Feng /* x0 <- cache level */ 810ae76531SDavid Feng /* x10 <- clidr_el1 */ 820ae76531SDavid Feng /* x11 <- loc */ 830ae76531SDavid Feng /* x15 <- return address */ 840ae76531SDavid Feng 850ae76531SDavid Fengloop_level: 861e6ad55cSYork Sun lsl x12, x0, #1 871e6ad55cSYork Sun add x12, x12, x0 /* x0 <- tripled cache level */ 881e6ad55cSYork Sun lsr x12, x10, x12 891e6ad55cSYork Sun and x12, x12, #7 /* x12 <- cache type */ 901e6ad55cSYork Sun cmp x12, #2 910ae76531SDavid Feng b.lt skip /* skip if no cache or icache */ 92ba9eb6c7SMasahiro Yamada bl __asm_dcache_level /* x1 = 0 flush, 1 invalidate */ 930ae76531SDavid Fengskip: 940ae76531SDavid Feng add x0, x0, #1 /* increment cache level */ 950ae76531SDavid Feng cmp x11, x0 960ae76531SDavid Feng b.gt loop_level 970ae76531SDavid Feng 980ae76531SDavid Feng mov x0, #0 99f1075aedSMichal Simek msr csselr_el1, x0 /* restore csselr_el1 */ 1000ae76531SDavid Feng dsb sy 1010ae76531SDavid Feng isb 1020ae76531SDavid Feng mov lr, x15 1030ae76531SDavid Feng 1040ae76531SDavid Fengfinished: 1050ae76531SDavid Feng ret 1061e6ad55cSYork SunENDPROC(__asm_dcache_all) 107e6a05862SPhilipp Tomsich.popsection 1081e6ad55cSYork Sun 109e6a05862SPhilipp Tomsich.pushsection .text.__asm_flush_dcache_all, "ax" 1101e6ad55cSYork SunENTRY(__asm_flush_dcache_all) 1111e6ad55cSYork Sun mov x0, #0 11225828588SMasahiro Yamada b __asm_dcache_all 1130ae76531SDavid FengENDPROC(__asm_flush_dcache_all) 114e6a05862SPhilipp Tomsich.popsection 1150ae76531SDavid Feng 116e6a05862SPhilipp Tomsich.pushsection .text.__asm_invalidate_dcache_all, "ax" 1171e6ad55cSYork SunENTRY(__asm_invalidate_dcache_all) 118208bd513SPeng Fan mov x0, #0x1 11925828588SMasahiro Yamada b __asm_dcache_all 1201e6ad55cSYork SunENDPROC(__asm_invalidate_dcache_all) 121e6a05862SPhilipp Tomsich.popsection 1221e6ad55cSYork Sun 1230ae76531SDavid Feng/* 1240ae76531SDavid Feng * void __asm_flush_dcache_range(start, end) 1250ae76531SDavid Feng * 1260ae76531SDavid Feng * clean & invalidate data cache in the range 1270ae76531SDavid Feng * 1280ae76531SDavid Feng * x0: start address 1290ae76531SDavid Feng * x1: end address 1300ae76531SDavid Feng */ 131e6a05862SPhilipp Tomsich.pushsection .text.__asm_flush_dcache_range, "ax" 1320ae76531SDavid FengENTRY(__asm_flush_dcache_range) 1330ae76531SDavid Feng mrs x3, ctr_el0 1340ae76531SDavid Feng lsr x3, x3, #16 1350ae76531SDavid Feng and x3, x3, #0xf 1360ae76531SDavid Feng mov x2, #4 1370ae76531SDavid Feng lsl x2, x2, x3 /* cache line size */ 1380ae76531SDavid Feng 1390ae76531SDavid Feng /* x2 <- minimal cache line size in cache system */ 1400ae76531SDavid Feng sub x3, x2, #1 1410ae76531SDavid Feng bic x0, x0, x3 1420ae76531SDavid Feng1: dc civac, x0 /* clean & invalidate data or unified cache */ 1430ae76531SDavid Feng add x0, x0, x2 1440ae76531SDavid Feng cmp x0, x1 1450ae76531SDavid Feng b.lo 1b 1460ae76531SDavid Feng dsb sy 1470ae76531SDavid Feng ret 1480ae76531SDavid FengENDPROC(__asm_flush_dcache_range) 149e6a05862SPhilipp Tomsich.popsection 1506775a820SSimon Glass/* 1516775a820SSimon Glass * void __asm_invalidate_dcache_range(start, end) 1526775a820SSimon Glass * 1536775a820SSimon Glass * invalidate data cache in the range 1546775a820SSimon Glass * 1556775a820SSimon Glass * x0: start address 1566775a820SSimon Glass * x1: end address 1576775a820SSimon Glass */ 158e6a05862SPhilipp Tomsich.pushsection .text.__asm_invalidate_dcache_range, "ax" 1596775a820SSimon GlassENTRY(__asm_invalidate_dcache_range) 1606775a820SSimon Glass mrs x3, ctr_el0 1616775a820SSimon Glass ubfm x3, x3, #16, #19 1626775a820SSimon Glass mov x2, #4 1636775a820SSimon Glass lsl x2, x2, x3 /* cache line size */ 1646775a820SSimon Glass 1656775a820SSimon Glass /* x2 <- minimal cache line size in cache system */ 1666775a820SSimon Glass sub x3, x2, #1 1676775a820SSimon Glass bic x0, x0, x3 1686775a820SSimon Glass1: dc ivac, x0 /* invalidate data or unified cache */ 1696775a820SSimon Glass add x0, x0, x2 1706775a820SSimon Glass cmp x0, x1 1716775a820SSimon Glass b.lo 1b 1726775a820SSimon Glass dsb sy 1736775a820SSimon Glass ret 1746775a820SSimon GlassENDPROC(__asm_invalidate_dcache_range) 175e6a05862SPhilipp Tomsich.popsection 1760ae76531SDavid Feng 1770ae76531SDavid Feng/* 1780ae76531SDavid Feng * void __asm_invalidate_icache_all(void) 1790ae76531SDavid Feng * 1800ae76531SDavid Feng * invalidate all tlb entries. 1810ae76531SDavid Feng */ 182e6a05862SPhilipp Tomsich.pushsection .text.__asm_invalidate_icache_all, "ax" 1830ae76531SDavid FengENTRY(__asm_invalidate_icache_all) 1840ae76531SDavid Feng ic ialluis 1850ae76531SDavid Feng isb sy 1860ae76531SDavid Feng ret 1870ae76531SDavid FengENDPROC(__asm_invalidate_icache_all) 188e6a05862SPhilipp Tomsich.popsection 189dcd468b8SYork Sun 190e6a05862SPhilipp Tomsich.pushsection .text.__asm_invalidate_l3_dcache, "ax" 1911ab557a0SStephen WarrenENTRY(__asm_invalidate_l3_dcache) 192dcd468b8SYork Sun mov x0, #0 /* return status as success */ 193dcd468b8SYork Sun ret 1941ab557a0SStephen WarrenENDPROC(__asm_invalidate_l3_dcache) 1951ab557a0SStephen Warren .weak __asm_invalidate_l3_dcache 196e6a05862SPhilipp Tomsich.popsection 1971ab557a0SStephen Warren 198e6a05862SPhilipp Tomsich.pushsection .text.__asm_flush_l3_dcache, "ax" 1991ab557a0SStephen WarrenENTRY(__asm_flush_l3_dcache) 2001ab557a0SStephen Warren mov x0, #0 /* return status as success */ 2011ab557a0SStephen Warren ret 2021ab557a0SStephen WarrenENDPROC(__asm_flush_l3_dcache) 2031ab557a0SStephen Warren .weak __asm_flush_l3_dcache 204e6a05862SPhilipp Tomsich.popsection 2051ab557a0SStephen Warren 206e6a05862SPhilipp Tomsich.pushsection .text.__asm_invalidate_l3_icache, "ax" 2071ab557a0SStephen WarrenENTRY(__asm_invalidate_l3_icache) 2081ab557a0SStephen Warren mov x0, #0 /* return status as success */ 2091ab557a0SStephen Warren ret 2101ab557a0SStephen WarrenENDPROC(__asm_invalidate_l3_icache) 2111ab557a0SStephen Warren .weak __asm_invalidate_l3_icache 212e6a05862SPhilipp Tomsich.popsection 2135e2ec773SAlexander Graf 2145e2ec773SAlexander Graf/* 2155e2ec773SAlexander Graf * void __asm_switch_ttbr(ulong new_ttbr) 2165e2ec773SAlexander Graf * 2175e2ec773SAlexander Graf * Safely switches to a new page table. 2185e2ec773SAlexander Graf */ 219e6a05862SPhilipp Tomsich.pushsection .text.__asm_switch_ttbr, "ax" 2205e2ec773SAlexander GrafENTRY(__asm_switch_ttbr) 2215e2ec773SAlexander Graf /* x2 = SCTLR (alive throghout the function) */ 2225e2ec773SAlexander Graf switch_el x4, 3f, 2f, 1f 2235e2ec773SAlexander Graf3: mrs x2, sctlr_el3 2245e2ec773SAlexander Graf b 0f 2255e2ec773SAlexander Graf2: mrs x2, sctlr_el2 2265e2ec773SAlexander Graf b 0f 2275e2ec773SAlexander Graf1: mrs x2, sctlr_el1 2285e2ec773SAlexander Graf0: 2295e2ec773SAlexander Graf 2305e2ec773SAlexander Graf /* Unset CR_M | CR_C | CR_I from SCTLR to disable all caches */ 2315e2ec773SAlexander Graf movn x1, #(CR_M | CR_C | CR_I) 2325e2ec773SAlexander Graf and x1, x2, x1 2335e2ec773SAlexander Graf switch_el x4, 3f, 2f, 1f 2345e2ec773SAlexander Graf3: msr sctlr_el3, x1 2355e2ec773SAlexander Graf b 0f 2365e2ec773SAlexander Graf2: msr sctlr_el2, x1 2375e2ec773SAlexander Graf b 0f 2385e2ec773SAlexander Graf1: msr sctlr_el1, x1 2395e2ec773SAlexander Graf0: isb 2405e2ec773SAlexander Graf 2415e2ec773SAlexander Graf /* This call only clobbers x30 (lr) and x9 (unused) */ 2425e2ec773SAlexander Graf mov x3, x30 2435e2ec773SAlexander Graf bl __asm_invalidate_tlb_all 2445e2ec773SAlexander Graf 2455e2ec773SAlexander Graf /* From here on we're running safely with caches disabled */ 2465e2ec773SAlexander Graf 2475e2ec773SAlexander Graf /* Set TTBR to our first argument */ 2485e2ec773SAlexander Graf switch_el x4, 3f, 2f, 1f 2495e2ec773SAlexander Graf3: msr ttbr0_el3, x0 2505e2ec773SAlexander Graf b 0f 2515e2ec773SAlexander Graf2: msr ttbr0_el2, x0 2525e2ec773SAlexander Graf b 0f 2535e2ec773SAlexander Graf1: msr ttbr0_el1, x0 2545e2ec773SAlexander Graf0: isb 2555e2ec773SAlexander Graf 2565e2ec773SAlexander Graf /* Restore original SCTLR and thus enable caches again */ 2575e2ec773SAlexander Graf switch_el x4, 3f, 2f, 1f 2585e2ec773SAlexander Graf3: msr sctlr_el3, x2 2595e2ec773SAlexander Graf b 0f 2605e2ec773SAlexander Graf2: msr sctlr_el2, x2 2615e2ec773SAlexander Graf b 0f 2625e2ec773SAlexander Graf1: msr sctlr_el1, x2 2635e2ec773SAlexander Graf0: isb 2645e2ec773SAlexander Graf 2655e2ec773SAlexander Graf ret x3 2665e2ec773SAlexander GrafENDPROC(__asm_switch_ttbr) 267e6a05862SPhilipp Tomsich.popsection 268