11da177e4SLinus Torvalds/* 21da177e4SLinus Torvalds * linux/arch/arm/mm/cache-v6.S 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 2001 Deep Blue Solutions Ltd. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This program is free software; you can redistribute it and/or modify 71da177e4SLinus Torvalds * it under the terms of the GNU General Public License version 2 as 81da177e4SLinus Torvalds * published by the Free Software Foundation. 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * This is the "shell" of the ARMv6 processor support. 111da177e4SLinus Torvalds */ 121da177e4SLinus Torvalds#include <linux/linkage.h> 131da177e4SLinus Torvalds#include <linux/init.h> 141da177e4SLinus Torvalds#include <asm/assembler.h> 15c5102f59SWill Deacon#include <asm/errno.h> 1632cfb1b1SCatalin Marinas#include <asm/unwind.h> 171da177e4SLinus Torvalds 181da177e4SLinus Torvalds#include "proc-macros.S" 191da177e4SLinus Torvalds 201da177e4SLinus Torvalds#define HARVARD_CACHE 211da177e4SLinus Torvalds#define CACHE_LINE_SIZE 32 221da177e4SLinus Torvalds#define D_CACHE_LINE_SIZE 32 23217874feSGen FUKATSU#define BTB_FLUSH_SIZE 8 241da177e4SLinus Torvalds 259cba3cccSCatalin Marinas/* 2681d11955STony Lindgren * v6_flush_icache_all() 2781d11955STony Lindgren * 2881d11955STony Lindgren * Flush the whole I-cache. 2981d11955STony Lindgren * 3081d11955STony Lindgren * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail. 3181d11955STony Lindgren * This erratum is present in 1136, 1156 and 1176. It does not affect the 3281d11955STony Lindgren * MPCore. 339cba3cccSCatalin Marinas * 349cba3cccSCatalin Marinas * Registers: 359cba3cccSCatalin Marinas * r0 - set to 0 369cba3cccSCatalin Marinas * r1 - corrupted 379cba3cccSCatalin Marinas */ 3881d11955STony LindgrenENTRY(v6_flush_icache_all) 399cba3cccSCatalin Marinas mov r0, #0 4081d11955STony Lindgren#ifdef CONFIG_ARM_ERRATA_411920 419cba3cccSCatalin Marinas mrs r1, cpsr 429cba3cccSCatalin Marinas cpsid ifa @ disable interrupts 439cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 449cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 459cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 469cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 479cba3cccSCatalin Marinas msr cpsr_cx, r1 @ restore interrupts 489cba3cccSCatalin Marinas .rept 11 @ ARM Ltd recommends at least 499cba3cccSCatalin Marinas nop @ 11 NOPs 509cba3cccSCatalin Marinas .endr 5181d11955STony Lindgren#else 5281d11955STony Lindgren mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache 539cba3cccSCatalin Marinas#endif 546ebbf2ceSRussell King ret lr 5581d11955STony LindgrenENDPROC(v6_flush_icache_all) 569cba3cccSCatalin Marinas 571da177e4SLinus Torvalds/* 581da177e4SLinus Torvalds * v6_flush_cache_all() 591da177e4SLinus Torvalds * 601da177e4SLinus Torvalds * Flush the entire cache. 611da177e4SLinus Torvalds * 621da177e4SLinus Torvalds * It is assumed that: 631da177e4SLinus Torvalds */ 641da177e4SLinus TorvaldsENTRY(v6_flush_kern_cache_all) 651da177e4SLinus Torvalds mov r0, #0 661da177e4SLinus Torvalds#ifdef HARVARD_CACHE 671da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate 689cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 691da177e4SLinus Torvalds mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 701da177e4SLinus Torvalds#else 7181d11955STony Lindgren b v6_flush_icache_all 729cba3cccSCatalin Marinas#endif 739cba3cccSCatalin Marinas#else 741da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate 751da177e4SLinus Torvalds#endif 766ebbf2ceSRussell King ret lr 771da177e4SLinus Torvalds 781da177e4SLinus Torvalds/* 791da177e4SLinus Torvalds * v6_flush_cache_all() 801da177e4SLinus Torvalds * 811da177e4SLinus Torvalds * Flush all TLB entries in a particular address space 821da177e4SLinus Torvalds * 831da177e4SLinus Torvalds * - mm - mm_struct describing address space 841da177e4SLinus Torvalds */ 851da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_all) 861da177e4SLinus Torvalds /*FALLTHROUGH*/ 871da177e4SLinus Torvalds 881da177e4SLinus Torvalds/* 891da177e4SLinus Torvalds * v6_flush_cache_range(start, end, flags) 901da177e4SLinus Torvalds * 911da177e4SLinus Torvalds * Flush a range of TLB entries in the specified address space. 921da177e4SLinus Torvalds * 931da177e4SLinus Torvalds * - start - start address (may not be aligned) 941da177e4SLinus Torvalds * - end - end address (exclusive, may not be aligned) 951da177e4SLinus Torvalds * - flags - vm_area_struct flags describing address space 961da177e4SLinus Torvalds * 971da177e4SLinus Torvalds * It is assumed that: 981da177e4SLinus Torvalds * - we have a VIPT cache. 991da177e4SLinus Torvalds */ 1001da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_range) 1016ebbf2ceSRussell King ret lr 1021da177e4SLinus Torvalds 1031da177e4SLinus Torvalds/* 1041da177e4SLinus Torvalds * v6_coherent_kern_range(start,end) 1051da177e4SLinus Torvalds * 1061da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1071da177e4SLinus Torvalds * region. This is typically used when code has been written to 1081da177e4SLinus Torvalds * a memory region, and will be executed. 1091da177e4SLinus Torvalds * 1101da177e4SLinus Torvalds * - start - virtual start address of region 1111da177e4SLinus Torvalds * - end - virtual end address of region 1121da177e4SLinus Torvalds * 1131da177e4SLinus Torvalds * It is assumed that: 1141da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1151da177e4SLinus Torvalds */ 1161da177e4SLinus TorvaldsENTRY(v6_coherent_kern_range) 1171da177e4SLinus Torvalds /* FALLTHROUGH */ 1181da177e4SLinus Torvalds 1191da177e4SLinus Torvalds/* 1201da177e4SLinus Torvalds * v6_coherent_user_range(start,end) 1211da177e4SLinus Torvalds * 1221da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1231da177e4SLinus Torvalds * region. This is typically used when code has been written to 1241da177e4SLinus Torvalds * a memory region, and will be executed. 1251da177e4SLinus Torvalds * 1261da177e4SLinus Torvalds * - start - virtual start address of region 1271da177e4SLinus Torvalds * - end - virtual end address of region 1281da177e4SLinus Torvalds * 1291da177e4SLinus Torvalds * It is assumed that: 1301da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1311da177e4SLinus Torvalds */ 1321da177e4SLinus TorvaldsENTRY(v6_coherent_user_range) 13332cfb1b1SCatalin Marinas UNWIND(.fnstart ) 1341da177e4SLinus Torvalds#ifdef HARVARD_CACHE 13518afea04SNicolas Pitre bic r0, r0, #CACHE_LINE_SIZE - 1 13632cfb1b1SCatalin Marinas1: 13732cfb1b1SCatalin Marinas USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line 13818afea04SNicolas Pitre add r0, r0, #CACHE_LINE_SIZE 1391da177e4SLinus Torvalds cmp r0, r1 1401da177e4SLinus Torvalds blo 1b 14118afea04SNicolas Pitre#endif 1421da177e4SLinus Torvalds mov r0, #0 143141fa40cSCatalin Marinas#ifdef HARVARD_CACHE 1441da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 1459cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 146141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 147141fa40cSCatalin Marinas#else 14881d11955STony Lindgren b v6_flush_icache_all 1499cba3cccSCatalin Marinas#endif 1509cba3cccSCatalin Marinas#else 151141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB 1521da177e4SLinus Torvalds#endif 1536ebbf2ceSRussell King ret lr 1541da177e4SLinus Torvalds 1551da177e4SLinus Torvalds/* 15632cfb1b1SCatalin Marinas * Fault handling for the cache operation above. If the virtual address in r0 157c5102f59SWill Deacon * isn't mapped, fail with -EFAULT. 15832cfb1b1SCatalin Marinas */ 15932cfb1b1SCatalin Marinas9001: 160c5102f59SWill Deacon mov r0, #-EFAULT 1616ebbf2ceSRussell King ret lr 16232cfb1b1SCatalin Marinas UNWIND(.fnend ) 16332cfb1b1SCatalin MarinasENDPROC(v6_coherent_user_range) 16432cfb1b1SCatalin MarinasENDPROC(v6_coherent_kern_range) 16532cfb1b1SCatalin Marinas 16632cfb1b1SCatalin Marinas/* 1672c9b9c84SRussell King * v6_flush_kern_dcache_area(void *addr, size_t size) 1681da177e4SLinus Torvalds * 1691da177e4SLinus Torvalds * Ensure that the data held in the page kaddr is written back 1701da177e4SLinus Torvalds * to the page in question. 1711da177e4SLinus Torvalds * 1722c9b9c84SRussell King * - addr - kernel address 1732c9b9c84SRussell King * - size - region size 1741da177e4SLinus Torvalds */ 1752c9b9c84SRussell KingENTRY(v6_flush_kern_dcache_area) 1762c9b9c84SRussell King add r1, r0, r1 177a248b13bSWill Deacon bic r0, r0, #D_CACHE_LINE_SIZE - 1 1781da177e4SLinus Torvalds1: 1791da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1801da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 1811da177e4SLinus Torvalds#else 1821da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line 1831da177e4SLinus Torvalds#endif 1841da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 1851da177e4SLinus Torvalds cmp r0, r1 1861da177e4SLinus Torvalds blo 1b 1871da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1881da177e4SLinus Torvalds mov r0, #0 1891da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 1901da177e4SLinus Torvalds#endif 1916ebbf2ceSRussell King ret lr 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds 1941da177e4SLinus Torvalds/* 1951da177e4SLinus Torvalds * v6_dma_inv_range(start,end) 1961da177e4SLinus Torvalds * 1971da177e4SLinus Torvalds * Invalidate the data cache within the specified region; we will 1981da177e4SLinus Torvalds * be performing a DMA operation in this region and we want to 1991da177e4SLinus Torvalds * purge old data in the cache. 2001da177e4SLinus Torvalds * 2011da177e4SLinus Torvalds * - start - virtual start address of region 2021da177e4SLinus Torvalds * - end - virtual end address of region 2031da177e4SLinus Torvalds */ 204702b94bfSRussell Kingv6_dma_inv_range: 20585b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 20685b093bcSValentine Barshak ldrb r2, [r0] @ read for ownership 20785b093bcSValentine Barshak strb r2, [r0] @ write for ownership 20885b093bcSValentine Barshak#endif 2091da177e4SLinus Torvalds tst r0, #D_CACHE_LINE_SIZE - 1 2101da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2111da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2121da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c10, 1 @ clean D line 2131da177e4SLinus Torvalds#else 2141da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c11, 1 @ clean unified line 2151da177e4SLinus Torvalds#endif 2161da177e4SLinus Torvalds tst r1, #D_CACHE_LINE_SIZE - 1 21785b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 218e44fc388SStefan Agner ldrbne r2, [r1, #-1] @ read for ownership 219e44fc388SStefan Agner strbne r2, [r1, #-1] @ write for ownership 22085b093bcSValentine Barshak#endif 2211da177e4SLinus Torvalds bic r1, r1, #D_CACHE_LINE_SIZE - 1 2221da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2231da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line 2241da177e4SLinus Torvalds#else 2251da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line 2261da177e4SLinus Torvalds#endif 2271da177e4SLinus Torvalds1: 2281da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2291da177e4SLinus Torvalds mcr p15, 0, r0, c7, c6, 1 @ invalidate D line 2301da177e4SLinus Torvalds#else 2311da177e4SLinus Torvalds mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line 2321da177e4SLinus Torvalds#endif 2331da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2341da177e4SLinus Torvalds cmp r0, r1 23585b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 23685b093bcSValentine Barshak ldrlo r2, [r0] @ read for ownership 23785b093bcSValentine Barshak strlo r2, [r0] @ write for ownership 23885b093bcSValentine Barshak#endif 2391da177e4SLinus Torvalds blo 1b 2401da177e4SLinus Torvalds mov r0, #0 2411da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2426ebbf2ceSRussell King ret lr 2431da177e4SLinus Torvalds 2441da177e4SLinus Torvalds/* 2451da177e4SLinus Torvalds * v6_dma_clean_range(start,end) 2461da177e4SLinus Torvalds * - start - virtual start address of region 2471da177e4SLinus Torvalds * - end - virtual end address of region 2481da177e4SLinus Torvalds */ 249702b94bfSRussell Kingv6_dma_clean_range: 2501da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2511da177e4SLinus Torvalds1: 252ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 253f4d6477fSCatalin Marinas ldr r2, [r0] @ read for ownership 254f4d6477fSCatalin Marinas#endif 2551da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2561da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 1 @ clean D line 2571da177e4SLinus Torvalds#else 2581da177e4SLinus Torvalds mcr p15, 0, r0, c7, c11, 1 @ clean unified line 2591da177e4SLinus Torvalds#endif 2601da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2611da177e4SLinus Torvalds cmp r0, r1 2621da177e4SLinus Torvalds blo 1b 2631da177e4SLinus Torvalds mov r0, #0 2641da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2656ebbf2ceSRussell King ret lr 2661da177e4SLinus Torvalds 2671da177e4SLinus Torvalds/* 2681da177e4SLinus Torvalds * v6_dma_flush_range(start,end) 2691da177e4SLinus Torvalds * - start - virtual start address of region 2701da177e4SLinus Torvalds * - end - virtual end address of region 2711da177e4SLinus Torvalds */ 2721da177e4SLinus TorvaldsENTRY(v6_dma_flush_range) 27385b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 27485b093bcSValentine Barshak ldrb r2, [r0] @ read for ownership 27585b093bcSValentine Barshak strb r2, [r0] @ write for ownership 27685b093bcSValentine Barshak#endif 2771da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2781da177e4SLinus Torvalds1: 2791da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2801da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 2811da177e4SLinus Torvalds#else 2821da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line 2831da177e4SLinus Torvalds#endif 2841da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2851da177e4SLinus Torvalds cmp r0, r1 28685b093bcSValentine Barshak#ifdef CONFIG_DMA_CACHE_RWFO 287e44fc388SStefan Agner ldrblo r2, [r0] @ read for ownership 288e44fc388SStefan Agner strblo r2, [r0] @ write for ownership 28985b093bcSValentine Barshak#endif 2901da177e4SLinus Torvalds blo 1b 2911da177e4SLinus Torvalds mov r0, #0 2921da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2936ebbf2ceSRussell King ret lr 2941da177e4SLinus Torvalds 295a9c9147eSRussell King/* 296a9c9147eSRussell King * dma_map_area(start, size, dir) 297a9c9147eSRussell King * - start - kernel virtual start address 298a9c9147eSRussell King * - size - size of region 299a9c9147eSRussell King * - dir - DMA direction 300a9c9147eSRussell King */ 301a9c9147eSRussell KingENTRY(v6_dma_map_area) 302a9c9147eSRussell King add r1, r1, r0 3032ffe2da3SRussell King teq r2, #DMA_FROM_DEVICE 3042ffe2da3SRussell King beq v6_dma_inv_range 305ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 306ad642d9fSCatalin Marinas b v6_dma_clean_range 307ad642d9fSCatalin Marinas#else 308f4d6477fSCatalin Marinas teq r2, #DMA_TO_DEVICE 309f4d6477fSCatalin Marinas beq v6_dma_clean_range 310f4d6477fSCatalin Marinas b v6_dma_flush_range 311ad642d9fSCatalin Marinas#endif 312a9c9147eSRussell KingENDPROC(v6_dma_map_area) 313a9c9147eSRussell King 314a9c9147eSRussell King/* 315a9c9147eSRussell King * dma_unmap_area(start, size, dir) 316a9c9147eSRussell King * - start - kernel virtual start address 317a9c9147eSRussell King * - size - size of region 318a9c9147eSRussell King * - dir - DMA direction 319a9c9147eSRussell King */ 320a9c9147eSRussell KingENTRY(v6_dma_unmap_area) 321ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 322ad642d9fSCatalin Marinas add r1, r1, r0 323ad642d9fSCatalin Marinas teq r2, #DMA_TO_DEVICE 324ad642d9fSCatalin Marinas bne v6_dma_inv_range 325ad642d9fSCatalin Marinas#endif 3266ebbf2ceSRussell King ret lr 327a9c9147eSRussell KingENDPROC(v6_dma_unmap_area) 328a9c9147eSRussell King 329031bd879SLorenzo Pieralisi .globl v6_flush_kern_cache_louis 330031bd879SLorenzo Pieralisi .equ v6_flush_kern_cache_louis, v6_flush_kern_cache_all 331031bd879SLorenzo Pieralisi 3321da177e4SLinus Torvalds __INITDATA 3331da177e4SLinus Torvalds 334641d8233SDave Martin @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) 335641d8233SDave Martin define_cache_functions v6 336