11da177e4SLinus Torvalds/* 21da177e4SLinus Torvalds * linux/arch/arm/mm/cache-v6.S 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 2001 Deep Blue Solutions Ltd. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This program is free software; you can redistribute it and/or modify 71da177e4SLinus Torvalds * it under the terms of the GNU General Public License version 2 as 81da177e4SLinus Torvalds * published by the Free Software Foundation. 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * This is the "shell" of the ARMv6 processor support. 111da177e4SLinus Torvalds */ 121da177e4SLinus Torvalds#include <linux/linkage.h> 131da177e4SLinus Torvalds#include <linux/init.h> 141da177e4SLinus Torvalds#include <asm/assembler.h> 1532cfb1b1SCatalin Marinas#include <asm/unwind.h> 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds#include "proc-macros.S" 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds#define HARVARD_CACHE 201da177e4SLinus Torvalds#define CACHE_LINE_SIZE 32 211da177e4SLinus Torvalds#define D_CACHE_LINE_SIZE 32 22217874feSGen FUKATSU#define BTB_FLUSH_SIZE 8 231da177e4SLinus Torvalds 249cba3cccSCatalin Marinas/* 2581d11955STony Lindgren * v6_flush_icache_all() 2681d11955STony Lindgren * 2781d11955STony Lindgren * Flush the whole I-cache. 2881d11955STony Lindgren * 2981d11955STony Lindgren * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail. 3081d11955STony Lindgren * This erratum is present in 1136, 1156 and 1176. It does not affect the 3181d11955STony Lindgren * MPCore. 329cba3cccSCatalin Marinas * 339cba3cccSCatalin Marinas * Registers: 349cba3cccSCatalin Marinas * r0 - set to 0 359cba3cccSCatalin Marinas * r1 - corrupted 369cba3cccSCatalin Marinas */ 3781d11955STony LindgrenENTRY(v6_flush_icache_all) 389cba3cccSCatalin Marinas mov r0, #0 3981d11955STony Lindgren#ifdef CONFIG_ARM_ERRATA_411920 409cba3cccSCatalin Marinas mrs r1, cpsr 419cba3cccSCatalin Marinas cpsid ifa @ disable interrupts 429cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 439cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 449cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 459cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 469cba3cccSCatalin Marinas msr cpsr_cx, r1 @ restore interrupts 479cba3cccSCatalin Marinas .rept 11 @ ARM Ltd recommends at least 489cba3cccSCatalin Marinas nop @ 11 NOPs 499cba3cccSCatalin Marinas .endr 5081d11955STony Lindgren#else 5181d11955STony Lindgren mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache 529cba3cccSCatalin Marinas#endif 5381d11955STony Lindgren mov pc, lr 5481d11955STony LindgrenENDPROC(v6_flush_icache_all) 559cba3cccSCatalin Marinas 561da177e4SLinus Torvalds/* 571da177e4SLinus Torvalds * v6_flush_cache_all() 581da177e4SLinus Torvalds * 591da177e4SLinus Torvalds * Flush the entire cache. 601da177e4SLinus Torvalds * 611da177e4SLinus Torvalds * It is assumed that: 621da177e4SLinus Torvalds */ 631da177e4SLinus TorvaldsENTRY(v6_flush_kern_cache_all) 641da177e4SLinus Torvalds mov r0, #0 651da177e4SLinus Torvalds#ifdef HARVARD_CACHE 661da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate 679cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 681da177e4SLinus Torvalds mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 691da177e4SLinus Torvalds#else 7081d11955STony Lindgren b v6_flush_icache_all 719cba3cccSCatalin Marinas#endif 729cba3cccSCatalin Marinas#else 731da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate 741da177e4SLinus Torvalds#endif 751da177e4SLinus Torvalds mov pc, lr 761da177e4SLinus Torvalds 771da177e4SLinus Torvalds/* 781da177e4SLinus Torvalds * v6_flush_cache_all() 791da177e4SLinus Torvalds * 801da177e4SLinus Torvalds * Flush all TLB entries in a particular address space 811da177e4SLinus Torvalds * 821da177e4SLinus Torvalds * - mm - mm_struct describing address space 831da177e4SLinus Torvalds */ 841da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_all) 851da177e4SLinus Torvalds /*FALLTHROUGH*/ 861da177e4SLinus Torvalds 871da177e4SLinus Torvalds/* 881da177e4SLinus Torvalds * v6_flush_cache_range(start, end, flags) 891da177e4SLinus Torvalds * 901da177e4SLinus Torvalds * Flush a range of TLB entries in the specified address space. 911da177e4SLinus Torvalds * 921da177e4SLinus Torvalds * - start - start address (may not be aligned) 931da177e4SLinus Torvalds * - end - end address (exclusive, may not be aligned) 941da177e4SLinus Torvalds * - flags - vm_area_struct flags describing address space 951da177e4SLinus Torvalds * 961da177e4SLinus Torvalds * It is assumed that: 971da177e4SLinus Torvalds * - we have a VIPT cache. 981da177e4SLinus Torvalds */ 991da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_range) 1001da177e4SLinus Torvalds mov pc, lr 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds/* 1031da177e4SLinus Torvalds * v6_coherent_kern_range(start,end) 1041da177e4SLinus Torvalds * 1051da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1061da177e4SLinus Torvalds * region. This is typically used when code has been written to 1071da177e4SLinus Torvalds * a memory region, and will be executed. 1081da177e4SLinus Torvalds * 1091da177e4SLinus Torvalds * - start - virtual start address of region 1101da177e4SLinus Torvalds * - end - virtual end address of region 1111da177e4SLinus Torvalds * 1121da177e4SLinus Torvalds * It is assumed that: 1131da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1141da177e4SLinus Torvalds */ 1151da177e4SLinus TorvaldsENTRY(v6_coherent_kern_range) 1161da177e4SLinus Torvalds /* FALLTHROUGH */ 1171da177e4SLinus Torvalds 1181da177e4SLinus Torvalds/* 1191da177e4SLinus Torvalds * v6_coherent_user_range(start,end) 1201da177e4SLinus Torvalds * 1211da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1221da177e4SLinus Torvalds * region. This is typically used when code has been written to 1231da177e4SLinus Torvalds * a memory region, and will be executed. 1241da177e4SLinus Torvalds * 1251da177e4SLinus Torvalds * - start - virtual start address of region 1261da177e4SLinus Torvalds * - end - virtual end address of region 1271da177e4SLinus Torvalds * 1281da177e4SLinus Torvalds * It is assumed that: 1291da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1301da177e4SLinus Torvalds */ 1311da177e4SLinus TorvaldsENTRY(v6_coherent_user_range) 13232cfb1b1SCatalin Marinas UNWIND(.fnstart ) 1331da177e4SLinus Torvalds#ifdef HARVARD_CACHE 13418afea04SNicolas Pitre bic r0, r0, #CACHE_LINE_SIZE - 1 13532cfb1b1SCatalin Marinas1: 13632cfb1b1SCatalin Marinas USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line 13718afea04SNicolas Pitre add r0, r0, #CACHE_LINE_SIZE 13832cfb1b1SCatalin Marinas2: 1391da177e4SLinus Torvalds cmp r0, r1 1401da177e4SLinus Torvalds blo 1b 14118afea04SNicolas Pitre#endif 1421da177e4SLinus Torvalds mov r0, #0 143141fa40cSCatalin Marinas#ifdef HARVARD_CACHE 1441da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 1459cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 146141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 147141fa40cSCatalin Marinas#else 14881d11955STony Lindgren b v6_flush_icache_all 1499cba3cccSCatalin Marinas#endif 1509cba3cccSCatalin Marinas#else 151141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB 1521da177e4SLinus Torvalds#endif 1531da177e4SLinus Torvalds mov pc, lr 1541da177e4SLinus Torvalds 1551da177e4SLinus Torvalds/* 15632cfb1b1SCatalin Marinas * Fault handling for the cache operation above. If the virtual address in r0 15732cfb1b1SCatalin Marinas * isn't mapped, just try the next page. 15832cfb1b1SCatalin Marinas */ 15932cfb1b1SCatalin Marinas9001: 16032cfb1b1SCatalin Marinas mov r0, r0, lsr #12 16132cfb1b1SCatalin Marinas mov r0, r0, lsl #12 16232cfb1b1SCatalin Marinas add r0, r0, #4096 16332cfb1b1SCatalin Marinas b 2b 16432cfb1b1SCatalin Marinas UNWIND(.fnend ) 16532cfb1b1SCatalin MarinasENDPROC(v6_coherent_user_range) 16632cfb1b1SCatalin MarinasENDPROC(v6_coherent_kern_range) 16732cfb1b1SCatalin Marinas 16832cfb1b1SCatalin Marinas/* 1692c9b9c84SRussell King * v6_flush_kern_dcache_area(void *addr, size_t size) 1701da177e4SLinus Torvalds * 1711da177e4SLinus Torvalds * Ensure that the data held in the page kaddr is written back 1721da177e4SLinus Torvalds * to the page in question. 1731da177e4SLinus Torvalds * 1742c9b9c84SRussell King * - addr - kernel address 1752c9b9c84SRussell King * - size - region size 1761da177e4SLinus Torvalds */ 1772c9b9c84SRussell KingENTRY(v6_flush_kern_dcache_area) 1782c9b9c84SRussell King add r1, r0, r1 1791da177e4SLinus Torvalds1: 1801da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1811da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 1821da177e4SLinus Torvalds#else 1831da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line 1841da177e4SLinus Torvalds#endif 1851da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 1861da177e4SLinus Torvalds cmp r0, r1 1871da177e4SLinus Torvalds blo 1b 1881da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1891da177e4SLinus Torvalds mov r0, #0 1901da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 1911da177e4SLinus Torvalds#endif 1921da177e4SLinus Torvalds mov pc, lr 1931da177e4SLinus Torvalds 1941da177e4SLinus Torvalds 1951da177e4SLinus Torvalds/* 1961da177e4SLinus Torvalds * v6_dma_inv_range(start,end) 1971da177e4SLinus Torvalds * 1981da177e4SLinus Torvalds * Invalidate the data cache within the specified region; we will 1991da177e4SLinus Torvalds * be performing a DMA operation in this region and we want to 2001da177e4SLinus Torvalds * purge old data in the cache. 2011da177e4SLinus Torvalds * 2021da177e4SLinus Torvalds * - start - virtual start address of region 2031da177e4SLinus Torvalds * - end - virtual end address of region 2041da177e4SLinus Torvalds */ 205702b94bfSRussell Kingv6_dma_inv_range: 2061da177e4SLinus Torvalds tst r0, #D_CACHE_LINE_SIZE - 1 2071da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2081da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2091da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c10, 1 @ clean D line 2101da177e4SLinus Torvalds#else 2111da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c11, 1 @ clean unified line 2121da177e4SLinus Torvalds#endif 2131da177e4SLinus Torvalds tst r1, #D_CACHE_LINE_SIZE - 1 2141da177e4SLinus Torvalds bic r1, r1, #D_CACHE_LINE_SIZE - 1 2151da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2161da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line 2171da177e4SLinus Torvalds#else 2181da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line 2191da177e4SLinus Torvalds#endif 2201da177e4SLinus Torvalds1: 221ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 222ca57926dSCatalin Marinas ldr r2, [r0] @ read for ownership 223ca57926dSCatalin Marinas str r2, [r0] @ write for ownership 224f4d6477fSCatalin Marinas#endif 2251da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2261da177e4SLinus Torvalds mcr p15, 0, r0, c7, c6, 1 @ invalidate D line 2271da177e4SLinus Torvalds#else 2281da177e4SLinus Torvalds mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line 2291da177e4SLinus Torvalds#endif 2301da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2311da177e4SLinus Torvalds cmp r0, r1 2321da177e4SLinus Torvalds blo 1b 2331da177e4SLinus Torvalds mov r0, #0 2341da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2351da177e4SLinus Torvalds mov pc, lr 2361da177e4SLinus Torvalds 2371da177e4SLinus Torvalds/* 2381da177e4SLinus Torvalds * v6_dma_clean_range(start,end) 2391da177e4SLinus Torvalds * - start - virtual start address of region 2401da177e4SLinus Torvalds * - end - virtual end address of region 2411da177e4SLinus Torvalds */ 242702b94bfSRussell Kingv6_dma_clean_range: 2431da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2441da177e4SLinus Torvalds1: 245ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 246f4d6477fSCatalin Marinas ldr r2, [r0] @ read for ownership 247f4d6477fSCatalin Marinas#endif 2481da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2491da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 1 @ clean D line 2501da177e4SLinus Torvalds#else 2511da177e4SLinus Torvalds mcr p15, 0, r0, c7, c11, 1 @ clean unified line 2521da177e4SLinus Torvalds#endif 2531da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2541da177e4SLinus Torvalds cmp r0, r1 2551da177e4SLinus Torvalds blo 1b 2561da177e4SLinus Torvalds mov r0, #0 2571da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2581da177e4SLinus Torvalds mov pc, lr 2591da177e4SLinus Torvalds 2601da177e4SLinus Torvalds/* 2611da177e4SLinus Torvalds * v6_dma_flush_range(start,end) 2621da177e4SLinus Torvalds * - start - virtual start address of region 2631da177e4SLinus Torvalds * - end - virtual end address of region 2641da177e4SLinus Torvalds */ 2651da177e4SLinus TorvaldsENTRY(v6_dma_flush_range) 2661da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2671da177e4SLinus Torvalds1: 268ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 269f4d6477fSCatalin Marinas ldr r2, [r0] @ read for ownership 270f4d6477fSCatalin Marinas str r2, [r0] @ write for ownership 271f4d6477fSCatalin Marinas#endif 2721da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2731da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 2741da177e4SLinus Torvalds#else 2751da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line 2761da177e4SLinus Torvalds#endif 2771da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2781da177e4SLinus Torvalds cmp r0, r1 2791da177e4SLinus Torvalds blo 1b 2801da177e4SLinus Torvalds mov r0, #0 2811da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2821da177e4SLinus Torvalds mov pc, lr 2831da177e4SLinus Torvalds 284a9c9147eSRussell King/* 285a9c9147eSRussell King * dma_map_area(start, size, dir) 286a9c9147eSRussell King * - start - kernel virtual start address 287a9c9147eSRussell King * - size - size of region 288a9c9147eSRussell King * - dir - DMA direction 289a9c9147eSRussell King */ 290a9c9147eSRussell KingENTRY(v6_dma_map_area) 291a9c9147eSRussell King add r1, r1, r0 2922ffe2da3SRussell King teq r2, #DMA_FROM_DEVICE 2932ffe2da3SRussell King beq v6_dma_inv_range 294ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 295ad642d9fSCatalin Marinas b v6_dma_clean_range 296ad642d9fSCatalin Marinas#else 297f4d6477fSCatalin Marinas teq r2, #DMA_TO_DEVICE 298f4d6477fSCatalin Marinas beq v6_dma_clean_range 299f4d6477fSCatalin Marinas b v6_dma_flush_range 300ad642d9fSCatalin Marinas#endif 301a9c9147eSRussell KingENDPROC(v6_dma_map_area) 302a9c9147eSRussell King 303a9c9147eSRussell King/* 304a9c9147eSRussell King * dma_unmap_area(start, size, dir) 305a9c9147eSRussell King * - start - kernel virtual start address 306a9c9147eSRussell King * - size - size of region 307a9c9147eSRussell King * - dir - DMA direction 308a9c9147eSRussell King */ 309a9c9147eSRussell KingENTRY(v6_dma_unmap_area) 310ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 311ad642d9fSCatalin Marinas add r1, r1, r0 312ad642d9fSCatalin Marinas teq r2, #DMA_TO_DEVICE 313ad642d9fSCatalin Marinas bne v6_dma_inv_range 314ad642d9fSCatalin Marinas#endif 315a9c9147eSRussell King mov pc, lr 316a9c9147eSRussell KingENDPROC(v6_dma_unmap_area) 317a9c9147eSRussell King 3181da177e4SLinus Torvalds __INITDATA 3191da177e4SLinus Torvalds 3201da177e4SLinus Torvalds .type v6_cache_fns, #object 3211da177e4SLinus TorvaldsENTRY(v6_cache_fns) 32281d11955STony Lindgren .long v6_flush_icache_all 3231da177e4SLinus Torvalds .long v6_flush_kern_cache_all 3241da177e4SLinus Torvalds .long v6_flush_user_cache_all 3251da177e4SLinus Torvalds .long v6_flush_user_cache_range 3261da177e4SLinus Torvalds .long v6_coherent_kern_range 3271da177e4SLinus Torvalds .long v6_coherent_user_range 3282c9b9c84SRussell King .long v6_flush_kern_dcache_area 329a9c9147eSRussell King .long v6_dma_map_area 330a9c9147eSRussell King .long v6_dma_unmap_area 3311da177e4SLinus Torvalds .long v6_dma_flush_range 3321da177e4SLinus Torvalds .size v6_cache_fns, . - v6_cache_fns 333