1d2912cb1SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 21da177e4SLinus Torvalds/* 31da177e4SLinus Torvalds * linux/arch/arm/mm/cache-v6.S 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Copyright (C) 2001 Deep Blue Solutions Ltd. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * This is the "shell" of the ARMv6 processor support. 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds#include <linux/linkage.h> 101da177e4SLinus Torvalds#include <linux/init.h> 111da177e4SLinus Torvalds#include <asm/assembler.h> 12c5102f59SWill Deacon#include <asm/errno.h> 1332cfb1b1SCatalin Marinas#include <asm/unwind.h> 141da177e4SLinus Torvalds 151da177e4SLinus Torvalds#include "proc-macros.S" 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds#define HARVARD_CACHE 181da177e4SLinus Torvalds#define CACHE_LINE_SIZE 32 191da177e4SLinus Torvalds#define D_CACHE_LINE_SIZE 32 20217874feSGen FUKATSU#define BTB_FLUSH_SIZE 8 211da177e4SLinus Torvalds 22*a2faac39SNick Desaulniers.arch armv6 23*a2faac39SNick Desaulniers 249cba3cccSCatalin Marinas/* 2581d11955STony Lindgren * v6_flush_icache_all() 2681d11955STony Lindgren * 2781d11955STony Lindgren * Flush the whole I-cache. 2881d11955STony Lindgren * 2981d11955STony Lindgren * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail. 3081d11955STony Lindgren * This erratum is present in 1136, 1156 and 1176. It does not affect the 3181d11955STony Lindgren * MPCore. 329cba3cccSCatalin Marinas * 339cba3cccSCatalin Marinas * Registers: 349cba3cccSCatalin Marinas * r0 - set to 0 359cba3cccSCatalin Marinas * r1 - corrupted 369cba3cccSCatalin Marinas */ 3781d11955STony LindgrenENTRY(v6_flush_icache_all) 389cba3cccSCatalin Marinas mov r0, #0 3981d11955STony Lindgren#ifdef CONFIG_ARM_ERRATA_411920 409cba3cccSCatalin Marinas mrs r1, cpsr 419cba3cccSCatalin Marinas cpsid ifa @ disable interrupts 429cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 439cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 449cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 459cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 469cba3cccSCatalin Marinas msr cpsr_cx, r1 @ restore interrupts 479cba3cccSCatalin Marinas .rept 11 @ ARM Ltd recommends at least 489cba3cccSCatalin Marinas nop @ 11 NOPs 499cba3cccSCatalin Marinas .endr 5081d11955STony Lindgren#else 5181d11955STony Lindgren mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache 529cba3cccSCatalin Marinas#endif 536ebbf2ceSRussell King ret lr 5481d11955STony LindgrenENDPROC(v6_flush_icache_all) 559cba3cccSCatalin Marinas 561da177e4SLinus Torvalds/* 571da177e4SLinus Torvalds * v6_flush_cache_all() 581da177e4SLinus Torvalds * 591da177e4SLinus Torvalds * Flush the entire cache. 601da177e4SLinus Torvalds * 611da177e4SLinus Torvalds * It is assumed that: 621da177e4SLinus Torvalds */ 631da177e4SLinus TorvaldsENTRY(v6_flush_kern_cache_all) 641da177e4SLinus Torvalds mov r0, #0 651da177e4SLinus Torvalds#ifdef HARVARD_CACHE 661da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate 679cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 681da177e4SLinus Torvalds mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 691da177e4SLinus Torvalds#else 7081d11955STony Lindgren b v6_flush_icache_all 719cba3cccSCatalin Marinas#endif 729cba3cccSCatalin Marinas#else 731da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate 741da177e4SLinus Torvalds#endif 756ebbf2ceSRussell King ret lr 761da177e4SLinus Torvalds 771da177e4SLinus Torvalds/* 781da177e4SLinus Torvalds * v6_flush_cache_all() 791da177e4SLinus Torvalds * 801da177e4SLinus Torvalds * Flush all TLB entries in a particular address space 811da177e4SLinus Torvalds * 821da177e4SLinus Torvalds * - mm - mm_struct describing address space 831da177e4SLinus Torvalds */ 841da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_all) 851da177e4SLinus Torvalds /*FALLTHROUGH*/ 861da177e4SLinus Torvalds 871da177e4SLinus Torvalds/* 881da177e4SLinus Torvalds * v6_flush_cache_range(start, end, flags) 891da177e4SLinus Torvalds * 901da177e4SLinus Torvalds * Flush a range of TLB entries in the specified address space. 911da177e4SLinus Torvalds * 921da177e4SLinus Torvalds * - start - start address (may not be aligned) 931da177e4SLinus Torvalds * - end - end address (exclusive, may not be aligned) 941da177e4SLinus Torvalds * - flags - vm_area_struct flags describing address space 951da177e4SLinus Torvalds * 961da177e4SLinus Torvalds * It is assumed that: 971da177e4SLinus Torvalds * - we have a VIPT cache. 981da177e4SLinus Torvalds */ 991da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_range) 1006ebbf2ceSRussell King ret lr 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds/* 1031da177e4SLinus Torvalds * v6_coherent_kern_range(start,end) 1041da177e4SLinus Torvalds * 1051da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1061da177e4SLinus Torvalds * region. This is typically used when code has been written to 1071da177e4SLinus Torvalds * a memory region, and will be executed. 1081da177e4SLinus Torvalds * 1091da177e4SLinus Torvalds * - start - virtual start address of region 1101da177e4SLinus Torvalds * - end - virtual end address of region 1111da177e4SLinus Torvalds * 1121da177e4SLinus Torvalds * It is assumed that: 1131da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1141da177e4SLinus Torvalds */ 1151da177e4SLinus TorvaldsENTRY(v6_coherent_kern_range) 1161da177e4SLinus Torvalds /* FALLTHROUGH */ 1171da177e4SLinus Torvalds 1181da177e4SLinus Torvalds/* 1191da177e4SLinus Torvalds * v6_coherent_user_range(start,end) 1201da177e4SLinus Torvalds * 1211da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1221da177e4SLinus Torvalds * region. This is typically used when code has been written to 1231da177e4SLinus Torvalds * a memory region, and will be executed. 1241da177e4SLinus Torvalds * 1251da177e4SLinus Torvalds * - start - virtual start address of region 1261da177e4SLinus Torvalds * - end - virtual end address of region 1271da177e4SLinus Torvalds * 1281da177e4SLinus Torvalds * It is assumed that: 1291da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1301da177e4SLinus Torvalds */ 1311da177e4SLinus TorvaldsENTRY(v6_coherent_user_range) 13232cfb1b1SCatalin Marinas UNWIND(.fnstart ) 1331da177e4SLinus Torvalds#ifdef HARVARD_CACHE 13418afea04SNicolas Pitre bic r0, r0, #CACHE_LINE_SIZE - 1 13532cfb1b1SCatalin Marinas1: 13632cfb1b1SCatalin Marinas USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line 13718afea04SNicolas Pitre add r0, r0, #CACHE_LINE_SIZE 1381da177e4SLinus Torvalds cmp r0, r1 1391da177e4SLinus Torvalds blo 1b 14018afea04SNicolas Pitre#endif 1411da177e4SLinus Torvalds mov r0, #0 142141fa40cSCatalin Marinas#ifdef HARVARD_CACHE 1431da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 1449cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 145141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 146141fa40cSCatalin Marinas#else 14781d11955STony Lindgren b v6_flush_icache_all 1489cba3cccSCatalin Marinas#endif 1499cba3cccSCatalin Marinas#else 150141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB 1511da177e4SLinus Torvalds#endif 1526ebbf2ceSRussell King ret lr 1531da177e4SLinus Torvalds 1541da177e4SLinus Torvalds/* 15532cfb1b1SCatalin Marinas * Fault handling for the cache operation above. If the virtual address in r0 156c5102f59SWill Deacon * isn't mapped, fail with -EFAULT. 15732cfb1b1SCatalin Marinas */ 15832cfb1b1SCatalin Marinas9001: 159c5102f59SWill Deacon mov r0, #-EFAULT 1606ebbf2ceSRussell King ret lr 16132cfb1b1SCatalin Marinas UNWIND(.fnend ) 16232cfb1b1SCatalin MarinasENDPROC(v6_coherent_user_range) 16332cfb1b1SCatalin MarinasENDPROC(v6_coherent_kern_range) 16432cfb1b1SCatalin Marinas 16532cfb1b1SCatalin Marinas/* 1662c9b9c84SRussell King * v6_flush_kern_dcache_area(void *addr, size_t size) 1671da177e4SLinus Torvalds * 1681da177e4SLinus Torvalds * Ensure that the data held in the page kaddr is written back 1691da177e4SLinus Torvalds * to the page in question. 1701da177e4SLinus Torvalds * 1712c9b9c84SRussell King * - addr - kernel address 1722c9b9c84SRussell King * - size - region size 1731da177e4SLinus Torvalds */ 1742c9b9c84SRussell KingENTRY(v6_flush_kern_dcache_area) 1752c9b9c84SRussell King add r1, r0, r1 176a248b13bSWill Deacon bic r0, r0, #D_CACHE_LINE_SIZE - 1 1771da177e4SLinus Torvalds1: 1781da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1791da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 1801da177e4SLinus Torvalds#else 1811da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line 1821da177e4SLinus Torvalds#endif 1831da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 1841da177e4SLinus Torvalds cmp r0, r1 1851da177e4SLinus Torvalds blo 1b 1861da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1871da177e4SLinus Torvalds mov r0, #0 1881da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 1891da177e4SLinus Torvalds#endif 1906ebbf2ceSRussell King ret lr 1911da177e4SLinus Torvalds 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds/* 1941da177e4SLinus Torvalds * v6_dma_inv_range(start,end) 1951da177e4SLinus Torvalds * 1961da177e4SLinus Torvalds * Invalidate the data cache within the specified region; we will 1971da177e4SLinus Torvalds * be performing a DMA operation in this region and we want to 1981da177e4SLinus Torvalds * purge old data in the cache. 1991da177e4SLinus Torvalds * 2001da177e4SLinus Torvalds * - start - virtual start address of region 2011da177e4SLinus Torvalds * - end - virtual end address of region 2021da177e4SLinus Torvalds */ 203702b94bfSRussell Kingv6_dma_inv_range: 20485b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 20585b093bcSValentine Barshak ldrb r2, [r0] @ read for ownership 20685b093bcSValentine Barshak strb r2, [r0] @ write for ownership 20785b093bcSValentine Barshak#endif 2081da177e4SLinus Torvalds tst r0, #D_CACHE_LINE_SIZE - 1 2091da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2101da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2111da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c10, 1 @ clean D line 2121da177e4SLinus Torvalds#else 2131da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c11, 1 @ clean unified line 2141da177e4SLinus Torvalds#endif 2151da177e4SLinus Torvalds tst r1, #D_CACHE_LINE_SIZE - 1 21685b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 217e44fc388SStefan Agner ldrbne r2, [r1, #-1] @ read for ownership 218e44fc388SStefan Agner strbne r2, [r1, #-1] @ write for ownership 21985b093bcSValentine Barshak#endif 2201da177e4SLinus Torvalds bic r1, r1, #D_CACHE_LINE_SIZE - 1 2211da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2221da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line 2231da177e4SLinus Torvalds#else 2241da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line 2251da177e4SLinus Torvalds#endif 2261da177e4SLinus Torvalds1: 2271da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2281da177e4SLinus Torvalds mcr p15, 0, r0, c7, c6, 1 @ invalidate D line 2291da177e4SLinus Torvalds#else 2301da177e4SLinus Torvalds mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line 2311da177e4SLinus Torvalds#endif 2321da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2331da177e4SLinus Torvalds cmp r0, r1 23485b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 23585b093bcSValentine Barshak ldrlo r2, [r0] @ read for ownership 23685b093bcSValentine Barshak strlo r2, [r0] @ write for ownership 23785b093bcSValentine Barshak#endif 2381da177e4SLinus Torvalds blo 1b 2391da177e4SLinus Torvalds mov r0, #0 2401da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2416ebbf2ceSRussell King ret lr 2421da177e4SLinus Torvalds 2431da177e4SLinus Torvalds/* 2441da177e4SLinus Torvalds * v6_dma_clean_range(start,end) 2451da177e4SLinus Torvalds * - start - virtual start address of region 2461da177e4SLinus Torvalds * - end - virtual end address of region 2471da177e4SLinus Torvalds */ 248702b94bfSRussell Kingv6_dma_clean_range: 2491da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2501da177e4SLinus Torvalds1: 251ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 252f4d6477fSCatalin Marinas ldr r2, [r0] @ read for ownership 253f4d6477fSCatalin Marinas#endif 2541da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2551da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 1 @ clean D line 2561da177e4SLinus Torvalds#else 2571da177e4SLinus Torvalds mcr p15, 0, r0, c7, c11, 1 @ clean unified line 2581da177e4SLinus Torvalds#endif 2591da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2601da177e4SLinus Torvalds cmp r0, r1 2611da177e4SLinus Torvalds blo 1b 2621da177e4SLinus Torvalds mov r0, #0 2631da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2646ebbf2ceSRussell King ret lr 2651da177e4SLinus Torvalds 2661da177e4SLinus Torvalds/* 2671da177e4SLinus Torvalds * v6_dma_flush_range(start,end) 2681da177e4SLinus Torvalds * - start - virtual start address of region 2691da177e4SLinus Torvalds * - end - virtual end address of region 2701da177e4SLinus Torvalds */ 2711da177e4SLinus TorvaldsENTRY(v6_dma_flush_range) 27285b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 27385b093bcSValentine Barshak ldrb r2, [r0] @ read for ownership 27485b093bcSValentine Barshak strb r2, [r0] @ write for ownership 27585b093bcSValentine Barshak#endif 2761da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2771da177e4SLinus Torvalds1: 2781da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2791da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 2801da177e4SLinus Torvalds#else 2811da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line 2821da177e4SLinus Torvalds#endif 2831da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2841da177e4SLinus Torvalds cmp r0, r1 28585b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 286e44fc388SStefan Agner ldrblo r2, [r0] @ read for ownership 287e44fc388SStefan Agner strblo r2, [r0] @ write for ownership 28885b093bcSValentine Barshak#endif 2891da177e4SLinus Torvalds blo 1b 2901da177e4SLinus Torvalds mov r0, #0 2911da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2926ebbf2ceSRussell King ret lr 2931da177e4SLinus Torvalds 294a9c9147eSRussell King/* 295a9c9147eSRussell King * dma_map_area(start, size, dir) 296a9c9147eSRussell King * - start - kernel virtual start address 297a9c9147eSRussell King * - size - size of region 298a9c9147eSRussell King * - dir - DMA direction 299a9c9147eSRussell King */ 300a9c9147eSRussell KingENTRY(v6_dma_map_area) 301a9c9147eSRussell King add r1, r1, r0 3022ffe2da3SRussell King teq r2, #DMA_FROM_DEVICE 3032ffe2da3SRussell King beq v6_dma_inv_range 304ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 305ad642d9fSCatalin Marinas b v6_dma_clean_range 306ad642d9fSCatalin Marinas#else 307f4d6477fSCatalin Marinas teq r2, #DMA_TO_DEVICE 308f4d6477fSCatalin Marinas beq v6_dma_clean_range 309f4d6477fSCatalin Marinas b v6_dma_flush_range 310ad642d9fSCatalin Marinas#endif 311a9c9147eSRussell KingENDPROC(v6_dma_map_area) 312a9c9147eSRussell King 313a9c9147eSRussell King/* 314a9c9147eSRussell King * dma_unmap_area(start, size, dir) 315a9c9147eSRussell King * - start - kernel virtual start address 316a9c9147eSRussell King * - size - size of region 317a9c9147eSRussell King * - dir - DMA direction 318a9c9147eSRussell King */ 319a9c9147eSRussell KingENTRY(v6_dma_unmap_area) 320ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 321ad642d9fSCatalin Marinas add r1, r1, r0 322ad642d9fSCatalin Marinas teq r2, #DMA_TO_DEVICE 323ad642d9fSCatalin Marinas bne v6_dma_inv_range 324ad642d9fSCatalin Marinas#endif 3256ebbf2ceSRussell King ret lr 326a9c9147eSRussell KingENDPROC(v6_dma_unmap_area) 327a9c9147eSRussell King 328031bd879SLorenzo Pieralisi .globl v6_flush_kern_cache_louis 329031bd879SLorenzo Pieralisi .equ v6_flush_kern_cache_louis, v6_flush_kern_cache_all 330031bd879SLorenzo Pieralisi 3311da177e4SLinus Torvalds __INITDATA 3321da177e4SLinus Torvalds 333641d8233SDave Martin @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) 334641d8233SDave Martin define_cache_functions v6 335