1d2912cb1SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 21da177e4SLinus Torvalds/* 31da177e4SLinus Torvalds * linux/arch/arm/mm/cache-v6.S 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Copyright (C) 2001 Deep Blue Solutions Ltd. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * This is the "shell" of the ARMv6 processor support. 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds#include <linux/linkage.h> 101da177e4SLinus Torvalds#include <linux/init.h> 111da177e4SLinus Torvalds#include <asm/assembler.h> 12c5102f59SWill Deacon#include <asm/errno.h> 1332cfb1b1SCatalin Marinas#include <asm/unwind.h> 141da177e4SLinus Torvalds 151da177e4SLinus Torvalds#include "proc-macros.S" 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds#define HARVARD_CACHE 181da177e4SLinus Torvalds#define CACHE_LINE_SIZE 32 191da177e4SLinus Torvalds#define D_CACHE_LINE_SIZE 32 20217874feSGen FUKATSU#define BTB_FLUSH_SIZE 8 211da177e4SLinus Torvalds 229cba3cccSCatalin Marinas/* 2381d11955STony Lindgren * v6_flush_icache_all() 2481d11955STony Lindgren * 2581d11955STony Lindgren * Flush the whole I-cache. 2681d11955STony Lindgren * 2781d11955STony Lindgren * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail. 2881d11955STony Lindgren * This erratum is present in 1136, 1156 and 1176. It does not affect the 2981d11955STony Lindgren * MPCore. 309cba3cccSCatalin Marinas * 319cba3cccSCatalin Marinas * Registers: 329cba3cccSCatalin Marinas * r0 - set to 0 339cba3cccSCatalin Marinas * r1 - corrupted 349cba3cccSCatalin Marinas */ 3581d11955STony LindgrenENTRY(v6_flush_icache_all) 369cba3cccSCatalin Marinas mov r0, #0 3781d11955STony Lindgren#ifdef CONFIG_ARM_ERRATA_411920 389cba3cccSCatalin Marinas mrs r1, cpsr 399cba3cccSCatalin Marinas cpsid ifa @ disable interrupts 409cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 419cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 429cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 439cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 449cba3cccSCatalin Marinas msr cpsr_cx, r1 @ restore interrupts 459cba3cccSCatalin Marinas .rept 11 @ ARM Ltd recommends at least 469cba3cccSCatalin Marinas nop @ 11 NOPs 479cba3cccSCatalin Marinas .endr 4881d11955STony Lindgren#else 4981d11955STony Lindgren mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache 509cba3cccSCatalin Marinas#endif 516ebbf2ceSRussell King ret lr 5281d11955STony LindgrenENDPROC(v6_flush_icache_all) 539cba3cccSCatalin Marinas 541da177e4SLinus Torvalds/* 551da177e4SLinus Torvalds * v6_flush_cache_all() 561da177e4SLinus Torvalds * 571da177e4SLinus Torvalds * Flush the entire cache. 581da177e4SLinus Torvalds * 591da177e4SLinus Torvalds * It is assumed that: 601da177e4SLinus Torvalds */ 611da177e4SLinus TorvaldsENTRY(v6_flush_kern_cache_all) 621da177e4SLinus Torvalds mov r0, #0 631da177e4SLinus Torvalds#ifdef HARVARD_CACHE 641da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate 659cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 661da177e4SLinus Torvalds mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 671da177e4SLinus Torvalds#else 6881d11955STony Lindgren b v6_flush_icache_all 699cba3cccSCatalin Marinas#endif 709cba3cccSCatalin Marinas#else 711da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate 721da177e4SLinus Torvalds#endif 736ebbf2ceSRussell King ret lr 741da177e4SLinus Torvalds 751da177e4SLinus Torvalds/* 761da177e4SLinus Torvalds * v6_flush_cache_all() 771da177e4SLinus Torvalds * 781da177e4SLinus Torvalds * Flush all TLB entries in a particular address space 791da177e4SLinus Torvalds * 801da177e4SLinus Torvalds * - mm - mm_struct describing address space 811da177e4SLinus Torvalds */ 821da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_all) 831da177e4SLinus Torvalds /*FALLTHROUGH*/ 841da177e4SLinus Torvalds 851da177e4SLinus Torvalds/* 861da177e4SLinus Torvalds * v6_flush_cache_range(start, end, flags) 871da177e4SLinus Torvalds * 881da177e4SLinus Torvalds * Flush a range of TLB entries in the specified address space. 891da177e4SLinus Torvalds * 901da177e4SLinus Torvalds * - start - start address (may not be aligned) 911da177e4SLinus Torvalds * - end - end address (exclusive, may not be aligned) 921da177e4SLinus Torvalds * - flags - vm_area_struct flags describing address space 931da177e4SLinus Torvalds * 941da177e4SLinus Torvalds * It is assumed that: 951da177e4SLinus Torvalds * - we have a VIPT cache. 961da177e4SLinus Torvalds */ 971da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_range) 986ebbf2ceSRussell King ret lr 991da177e4SLinus Torvalds 1001da177e4SLinus Torvalds/* 1011da177e4SLinus Torvalds * v6_coherent_kern_range(start,end) 1021da177e4SLinus Torvalds * 1031da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1041da177e4SLinus Torvalds * region. This is typically used when code has been written to 1051da177e4SLinus Torvalds * a memory region, and will be executed. 1061da177e4SLinus Torvalds * 1071da177e4SLinus Torvalds * - start - virtual start address of region 1081da177e4SLinus Torvalds * - end - virtual end address of region 1091da177e4SLinus Torvalds * 1101da177e4SLinus Torvalds * It is assumed that: 1111da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1121da177e4SLinus Torvalds */ 1131da177e4SLinus TorvaldsENTRY(v6_coherent_kern_range) 1141da177e4SLinus Torvalds /* FALLTHROUGH */ 1151da177e4SLinus Torvalds 1161da177e4SLinus Torvalds/* 1171da177e4SLinus Torvalds * v6_coherent_user_range(start,end) 1181da177e4SLinus Torvalds * 1191da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1201da177e4SLinus Torvalds * region. This is typically used when code has been written to 1211da177e4SLinus Torvalds * a memory region, and will be executed. 1221da177e4SLinus Torvalds * 1231da177e4SLinus Torvalds * - start - virtual start address of region 1241da177e4SLinus Torvalds * - end - virtual end address of region 1251da177e4SLinus Torvalds * 1261da177e4SLinus Torvalds * It is assumed that: 1271da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1281da177e4SLinus Torvalds */ 1291da177e4SLinus TorvaldsENTRY(v6_coherent_user_range) 13032cfb1b1SCatalin Marinas UNWIND(.fnstart ) 1311da177e4SLinus Torvalds#ifdef HARVARD_CACHE 13218afea04SNicolas Pitre bic r0, r0, #CACHE_LINE_SIZE - 1 13332cfb1b1SCatalin Marinas1: 13432cfb1b1SCatalin Marinas USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line 13518afea04SNicolas Pitre add r0, r0, #CACHE_LINE_SIZE 1361da177e4SLinus Torvalds cmp r0, r1 1371da177e4SLinus Torvalds blo 1b 13818afea04SNicolas Pitre#endif 1391da177e4SLinus Torvalds mov r0, #0 140141fa40cSCatalin Marinas#ifdef HARVARD_CACHE 1411da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 1429cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 143141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 144141fa40cSCatalin Marinas#else 14581d11955STony Lindgren b v6_flush_icache_all 1469cba3cccSCatalin Marinas#endif 1479cba3cccSCatalin Marinas#else 148141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB 1491da177e4SLinus Torvalds#endif 1506ebbf2ceSRussell King ret lr 1511da177e4SLinus Torvalds 1521da177e4SLinus Torvalds/* 15332cfb1b1SCatalin Marinas * Fault handling for the cache operation above. If the virtual address in r0 154c5102f59SWill Deacon * isn't mapped, fail with -EFAULT. 15532cfb1b1SCatalin Marinas */ 15632cfb1b1SCatalin Marinas9001: 157c5102f59SWill Deacon mov r0, #-EFAULT 1586ebbf2ceSRussell King ret lr 15932cfb1b1SCatalin Marinas UNWIND(.fnend ) 16032cfb1b1SCatalin MarinasENDPROC(v6_coherent_user_range) 16132cfb1b1SCatalin MarinasENDPROC(v6_coherent_kern_range) 16232cfb1b1SCatalin Marinas 16332cfb1b1SCatalin Marinas/* 1642c9b9c84SRussell King * v6_flush_kern_dcache_area(void *addr, size_t size) 1651da177e4SLinus Torvalds * 1661da177e4SLinus Torvalds * Ensure that the data held in the page kaddr is written back 1671da177e4SLinus Torvalds * to the page in question. 1681da177e4SLinus Torvalds * 1692c9b9c84SRussell King * - addr - kernel address 1702c9b9c84SRussell King * - size - region size 1711da177e4SLinus Torvalds */ 1722c9b9c84SRussell KingENTRY(v6_flush_kern_dcache_area) 1732c9b9c84SRussell King add r1, r0, r1 174a248b13bSWill Deacon bic r0, r0, #D_CACHE_LINE_SIZE - 1 1751da177e4SLinus Torvalds1: 1761da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1771da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 1781da177e4SLinus Torvalds#else 1791da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line 1801da177e4SLinus Torvalds#endif 1811da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 1821da177e4SLinus Torvalds cmp r0, r1 1831da177e4SLinus Torvalds blo 1b 1841da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1851da177e4SLinus Torvalds mov r0, #0 1861da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 1871da177e4SLinus Torvalds#endif 1886ebbf2ceSRussell King ret lr 1891da177e4SLinus Torvalds 1901da177e4SLinus Torvalds 1911da177e4SLinus Torvalds/* 1921da177e4SLinus Torvalds * v6_dma_inv_range(start,end) 1931da177e4SLinus Torvalds * 1941da177e4SLinus Torvalds * Invalidate the data cache within the specified region; we will 1951da177e4SLinus Torvalds * be performing a DMA operation in this region and we want to 1961da177e4SLinus Torvalds * purge old data in the cache. 1971da177e4SLinus Torvalds * 1981da177e4SLinus Torvalds * - start - virtual start address of region 1991da177e4SLinus Torvalds * - end - virtual end address of region 2001da177e4SLinus Torvalds */ 201702b94bfSRussell Kingv6_dma_inv_range: 20285b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 20385b093bcSValentine Barshak ldrb r2, [r0] @ read for ownership 20485b093bcSValentine Barshak strb r2, [r0] @ write for ownership 20585b093bcSValentine Barshak#endif 2061da177e4SLinus Torvalds tst r0, #D_CACHE_LINE_SIZE - 1 2071da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2081da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2091da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c10, 1 @ clean D line 2101da177e4SLinus Torvalds#else 2111da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c11, 1 @ clean unified line 2121da177e4SLinus Torvalds#endif 2131da177e4SLinus Torvalds tst r1, #D_CACHE_LINE_SIZE - 1 21485b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 215e44fc388SStefan Agner ldrbne r2, [r1, #-1] @ read for ownership 216e44fc388SStefan Agner strbne r2, [r1, #-1] @ write for ownership 21785b093bcSValentine Barshak#endif 2181da177e4SLinus Torvalds bic r1, r1, #D_CACHE_LINE_SIZE - 1 2191da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2201da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line 2211da177e4SLinus Torvalds#else 2221da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line 2231da177e4SLinus Torvalds#endif 2241da177e4SLinus Torvalds1: 2251da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2261da177e4SLinus Torvalds mcr p15, 0, r0, c7, c6, 1 @ invalidate D line 2271da177e4SLinus Torvalds#else 2281da177e4SLinus Torvalds mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line 2291da177e4SLinus Torvalds#endif 2301da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2311da177e4SLinus Torvalds cmp r0, r1 23285b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 23385b093bcSValentine Barshak ldrlo r2, [r0] @ read for ownership 23485b093bcSValentine Barshak strlo r2, [r0] @ write for ownership 23585b093bcSValentine Barshak#endif 2361da177e4SLinus Torvalds blo 1b 2371da177e4SLinus Torvalds mov r0, #0 2381da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2396ebbf2ceSRussell King ret lr 2401da177e4SLinus Torvalds 2411da177e4SLinus Torvalds/* 2421da177e4SLinus Torvalds * v6_dma_clean_range(start,end) 2431da177e4SLinus Torvalds * - start - virtual start address of region 2441da177e4SLinus Torvalds * - end - virtual end address of region 2451da177e4SLinus Torvalds */ 246702b94bfSRussell Kingv6_dma_clean_range: 2471da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2481da177e4SLinus Torvalds1: 249ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 250f4d6477fSCatalin Marinas ldr r2, [r0] @ read for ownership 251f4d6477fSCatalin Marinas#endif 2521da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2531da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 1 @ clean D line 2541da177e4SLinus Torvalds#else 2551da177e4SLinus Torvalds mcr p15, 0, r0, c7, c11, 1 @ clean unified line 2561da177e4SLinus Torvalds#endif 2571da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2581da177e4SLinus Torvalds cmp r0, r1 2591da177e4SLinus Torvalds blo 1b 2601da177e4SLinus Torvalds mov r0, #0 2611da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2626ebbf2ceSRussell King ret lr 2631da177e4SLinus Torvalds 2641da177e4SLinus Torvalds/* 2651da177e4SLinus Torvalds * v6_dma_flush_range(start,end) 2661da177e4SLinus Torvalds * - start - virtual start address of region 2671da177e4SLinus Torvalds * - end - virtual end address of region 2681da177e4SLinus Torvalds */ 2691da177e4SLinus TorvaldsENTRY(v6_dma_flush_range) 27085b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 27185b093bcSValentine Barshak ldrb r2, [r0] @ read for ownership 27285b093bcSValentine Barshak strb r2, [r0] @ write for ownership 27385b093bcSValentine Barshak#endif 2741da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2751da177e4SLinus Torvalds1: 2761da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2771da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 2781da177e4SLinus Torvalds#else 2791da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line 2801da177e4SLinus Torvalds#endif 2811da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2821da177e4SLinus Torvalds cmp r0, r1 28385b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 284e44fc388SStefan Agner ldrblo r2, [r0] @ read for ownership 285e44fc388SStefan Agner strblo r2, [r0] @ write for ownership 28685b093bcSValentine Barshak#endif 2871da177e4SLinus Torvalds blo 1b 2881da177e4SLinus Torvalds mov r0, #0 2891da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2906ebbf2ceSRussell King ret lr 2911da177e4SLinus Torvalds 292a9c9147eSRussell King/* 293a9c9147eSRussell King * dma_map_area(start, size, dir) 294a9c9147eSRussell King * - start - kernel virtual start address 295a9c9147eSRussell King * - size - size of region 296a9c9147eSRussell King * - dir - DMA direction 297a9c9147eSRussell King */ 298a9c9147eSRussell KingENTRY(v6_dma_map_area) 299a9c9147eSRussell King add r1, r1, r0 3002ffe2da3SRussell King teq r2, #DMA_FROM_DEVICE 3012ffe2da3SRussell King beq v6_dma_inv_range 302ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 303ad642d9fSCatalin Marinas b v6_dma_clean_range 304ad642d9fSCatalin Marinas#else 305f4d6477fSCatalin Marinas teq r2, #DMA_TO_DEVICE 306f4d6477fSCatalin Marinas beq v6_dma_clean_range 307f4d6477fSCatalin Marinas b v6_dma_flush_range 308ad642d9fSCatalin Marinas#endif 309a9c9147eSRussell KingENDPROC(v6_dma_map_area) 310a9c9147eSRussell King 311a9c9147eSRussell King/* 312a9c9147eSRussell King * dma_unmap_area(start, size, dir) 313a9c9147eSRussell King * - start - kernel virtual start address 314a9c9147eSRussell King * - size - size of region 315a9c9147eSRussell King * - dir - DMA direction 316a9c9147eSRussell King */ 317a9c9147eSRussell KingENTRY(v6_dma_unmap_area) 318ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 319ad642d9fSCatalin Marinas add r1, r1, r0 320ad642d9fSCatalin Marinas teq r2, #DMA_TO_DEVICE 321ad642d9fSCatalin Marinas bne v6_dma_inv_range 322ad642d9fSCatalin Marinas#endif 3236ebbf2ceSRussell King ret lr 324a9c9147eSRussell KingENDPROC(v6_dma_unmap_area) 325a9c9147eSRussell King 326031bd879SLorenzo Pieralisi .globl v6_flush_kern_cache_louis 327031bd879SLorenzo Pieralisi .equ v6_flush_kern_cache_louis, v6_flush_kern_cache_all 328031bd879SLorenzo Pieralisi 3291da177e4SLinus Torvalds __INITDATA 3301da177e4SLinus Torvalds 331641d8233SDave Martin @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) 332641d8233SDave Martin define_cache_functions v6 333