11da177e4SLinus Torvalds/* 21da177e4SLinus Torvalds * linux/arch/arm/mm/cache-v6.S 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 2001 Deep Blue Solutions Ltd. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This program is free software; you can redistribute it and/or modify 71da177e4SLinus Torvalds * it under the terms of the GNU General Public License version 2 as 81da177e4SLinus Torvalds * published by the Free Software Foundation. 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * This is the "shell" of the ARMv6 processor support. 111da177e4SLinus Torvalds */ 121da177e4SLinus Torvalds#include <linux/linkage.h> 131da177e4SLinus Torvalds#include <linux/init.h> 141da177e4SLinus Torvalds#include <asm/assembler.h> 1532cfb1b1SCatalin Marinas#include <asm/unwind.h> 161da177e4SLinus Torvalds 171da177e4SLinus Torvalds#include "proc-macros.S" 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds#define HARVARD_CACHE 201da177e4SLinus Torvalds#define CACHE_LINE_SIZE 32 211da177e4SLinus Torvalds#define D_CACHE_LINE_SIZE 32 22217874feSGen FUKATSU#define BTB_FLUSH_SIZE 8 231da177e4SLinus Torvalds 249cba3cccSCatalin Marinas#ifdef CONFIG_ARM_ERRATA_411920 259cba3cccSCatalin Marinas/* 269cba3cccSCatalin Marinas * Invalidate the entire I cache (this code is a workaround for the ARM1136 279cba3cccSCatalin Marinas * erratum 411920 - Invalidate Instruction Cache operation can fail. This 289cba3cccSCatalin Marinas * erratum is present in 1136, 1156 and 1176. It does not affect the MPCore. 299cba3cccSCatalin Marinas * 309cba3cccSCatalin Marinas * Registers: 319cba3cccSCatalin Marinas * r0 - set to 0 329cba3cccSCatalin Marinas * r1 - corrupted 339cba3cccSCatalin Marinas */ 349cba3cccSCatalin MarinasENTRY(v6_icache_inval_all) 359cba3cccSCatalin Marinas mov r0, #0 369cba3cccSCatalin Marinas mrs r1, cpsr 379cba3cccSCatalin Marinas cpsid ifa @ disable interrupts 389cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 399cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 409cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 419cba3cccSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache 429cba3cccSCatalin Marinas msr cpsr_cx, r1 @ restore interrupts 439cba3cccSCatalin Marinas .rept 11 @ ARM Ltd recommends at least 449cba3cccSCatalin Marinas nop @ 11 NOPs 459cba3cccSCatalin Marinas .endr 469cba3cccSCatalin Marinas mov pc, lr 479cba3cccSCatalin Marinas#endif 489cba3cccSCatalin Marinas 491da177e4SLinus Torvalds/* 501da177e4SLinus Torvalds * v6_flush_cache_all() 511da177e4SLinus Torvalds * 521da177e4SLinus Torvalds * Flush the entire cache. 531da177e4SLinus Torvalds * 541da177e4SLinus Torvalds * It is assumed that: 551da177e4SLinus Torvalds */ 561da177e4SLinus TorvaldsENTRY(v6_flush_kern_cache_all) 571da177e4SLinus Torvalds mov r0, #0 581da177e4SLinus Torvalds#ifdef HARVARD_CACHE 591da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate 609cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 611da177e4SLinus Torvalds mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 621da177e4SLinus Torvalds#else 639cba3cccSCatalin Marinas b v6_icache_inval_all 649cba3cccSCatalin Marinas#endif 659cba3cccSCatalin Marinas#else 661da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate 671da177e4SLinus Torvalds#endif 681da177e4SLinus Torvalds mov pc, lr 691da177e4SLinus Torvalds 701da177e4SLinus Torvalds/* 711da177e4SLinus Torvalds * v6_flush_cache_all() 721da177e4SLinus Torvalds * 731da177e4SLinus Torvalds * Flush all TLB entries in a particular address space 741da177e4SLinus Torvalds * 751da177e4SLinus Torvalds * - mm - mm_struct describing address space 761da177e4SLinus Torvalds */ 771da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_all) 781da177e4SLinus Torvalds /*FALLTHROUGH*/ 791da177e4SLinus Torvalds 801da177e4SLinus Torvalds/* 811da177e4SLinus Torvalds * v6_flush_cache_range(start, end, flags) 821da177e4SLinus Torvalds * 831da177e4SLinus Torvalds * Flush a range of TLB entries in the specified address space. 841da177e4SLinus Torvalds * 851da177e4SLinus Torvalds * - start - start address (may not be aligned) 861da177e4SLinus Torvalds * - end - end address (exclusive, may not be aligned) 871da177e4SLinus Torvalds * - flags - vm_area_struct flags describing address space 881da177e4SLinus Torvalds * 891da177e4SLinus Torvalds * It is assumed that: 901da177e4SLinus Torvalds * - we have a VIPT cache. 911da177e4SLinus Torvalds */ 921da177e4SLinus TorvaldsENTRY(v6_flush_user_cache_range) 931da177e4SLinus Torvalds mov pc, lr 941da177e4SLinus Torvalds 951da177e4SLinus Torvalds/* 961da177e4SLinus Torvalds * v6_coherent_kern_range(start,end) 971da177e4SLinus Torvalds * 981da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 991da177e4SLinus Torvalds * region. This is typically used when code has been written to 1001da177e4SLinus Torvalds * a memory region, and will be executed. 1011da177e4SLinus Torvalds * 1021da177e4SLinus Torvalds * - start - virtual start address of region 1031da177e4SLinus Torvalds * - end - virtual end address of region 1041da177e4SLinus Torvalds * 1051da177e4SLinus Torvalds * It is assumed that: 1061da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1071da177e4SLinus Torvalds */ 1081da177e4SLinus TorvaldsENTRY(v6_coherent_kern_range) 1091da177e4SLinus Torvalds /* FALLTHROUGH */ 1101da177e4SLinus Torvalds 1111da177e4SLinus Torvalds/* 1121da177e4SLinus Torvalds * v6_coherent_user_range(start,end) 1131da177e4SLinus Torvalds * 1141da177e4SLinus Torvalds * Ensure that the I and D caches are coherent within specified 1151da177e4SLinus Torvalds * region. This is typically used when code has been written to 1161da177e4SLinus Torvalds * a memory region, and will be executed. 1171da177e4SLinus Torvalds * 1181da177e4SLinus Torvalds * - start - virtual start address of region 1191da177e4SLinus Torvalds * - end - virtual end address of region 1201da177e4SLinus Torvalds * 1211da177e4SLinus Torvalds * It is assumed that: 1221da177e4SLinus Torvalds * - the Icache does not read data from the write buffer 1231da177e4SLinus Torvalds */ 1241da177e4SLinus TorvaldsENTRY(v6_coherent_user_range) 12532cfb1b1SCatalin Marinas UNWIND(.fnstart ) 1261da177e4SLinus Torvalds#ifdef HARVARD_CACHE 12718afea04SNicolas Pitre bic r0, r0, #CACHE_LINE_SIZE - 1 12832cfb1b1SCatalin Marinas1: 12932cfb1b1SCatalin Marinas USER( mcr p15, 0, r0, c7, c10, 1 ) @ clean D line 13018afea04SNicolas Pitre add r0, r0, #CACHE_LINE_SIZE 13132cfb1b1SCatalin Marinas2: 1321da177e4SLinus Torvalds cmp r0, r1 1331da177e4SLinus Torvalds blo 1b 13418afea04SNicolas Pitre#endif 1351da177e4SLinus Torvalds mov r0, #0 136141fa40cSCatalin Marinas#ifdef HARVARD_CACHE 1371da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 1389cba3cccSCatalin Marinas#ifndef CONFIG_ARM_ERRATA_411920 139141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 140141fa40cSCatalin Marinas#else 1419cba3cccSCatalin Marinas b v6_icache_inval_all 1429cba3cccSCatalin Marinas#endif 1439cba3cccSCatalin Marinas#else 144141fa40cSCatalin Marinas mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB 1451da177e4SLinus Torvalds#endif 1461da177e4SLinus Torvalds mov pc, lr 1471da177e4SLinus Torvalds 1481da177e4SLinus Torvalds/* 14932cfb1b1SCatalin Marinas * Fault handling for the cache operation above. If the virtual address in r0 15032cfb1b1SCatalin Marinas * isn't mapped, just try the next page. 15132cfb1b1SCatalin Marinas */ 15232cfb1b1SCatalin Marinas9001: 15332cfb1b1SCatalin Marinas mov r0, r0, lsr #12 15432cfb1b1SCatalin Marinas mov r0, r0, lsl #12 15532cfb1b1SCatalin Marinas add r0, r0, #4096 15632cfb1b1SCatalin Marinas b 2b 15732cfb1b1SCatalin Marinas UNWIND(.fnend ) 15832cfb1b1SCatalin MarinasENDPROC(v6_coherent_user_range) 15932cfb1b1SCatalin MarinasENDPROC(v6_coherent_kern_range) 16032cfb1b1SCatalin Marinas 16132cfb1b1SCatalin Marinas/* 1622c9b9c84SRussell King * v6_flush_kern_dcache_area(void *addr, size_t size) 1631da177e4SLinus Torvalds * 1641da177e4SLinus Torvalds * Ensure that the data held in the page kaddr is written back 1651da177e4SLinus Torvalds * to the page in question. 1661da177e4SLinus Torvalds * 1672c9b9c84SRussell King * - addr - kernel address 1682c9b9c84SRussell King * - size - region size 1691da177e4SLinus Torvalds */ 1702c9b9c84SRussell KingENTRY(v6_flush_kern_dcache_area) 1712c9b9c84SRussell King add r1, r0, r1 1721da177e4SLinus Torvalds1: 1731da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1741da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 1751da177e4SLinus Torvalds#else 1761da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate unified line 1771da177e4SLinus Torvalds#endif 1781da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 1791da177e4SLinus Torvalds cmp r0, r1 1801da177e4SLinus Torvalds blo 1b 1811da177e4SLinus Torvalds#ifdef HARVARD_CACHE 1821da177e4SLinus Torvalds mov r0, #0 1831da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 1841da177e4SLinus Torvalds#endif 1851da177e4SLinus Torvalds mov pc, lr 1861da177e4SLinus Torvalds 1871da177e4SLinus Torvalds 1881da177e4SLinus Torvalds/* 1891da177e4SLinus Torvalds * v6_dma_inv_range(start,end) 1901da177e4SLinus Torvalds * 1911da177e4SLinus Torvalds * Invalidate the data cache within the specified region; we will 1921da177e4SLinus Torvalds * be performing a DMA operation in this region and we want to 1931da177e4SLinus Torvalds * purge old data in the cache. 1941da177e4SLinus Torvalds * 1951da177e4SLinus Torvalds * - start - virtual start address of region 1961da177e4SLinus Torvalds * - end - virtual end address of region 1971da177e4SLinus Torvalds */ 198702b94bfSRussell Kingv6_dma_inv_range: 1991da177e4SLinus Torvalds tst r0, #D_CACHE_LINE_SIZE - 1 2001da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2011da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2021da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c10, 1 @ clean D line 2031da177e4SLinus Torvalds#else 2041da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c11, 1 @ clean unified line 2051da177e4SLinus Torvalds#endif 2061da177e4SLinus Torvalds tst r1, #D_CACHE_LINE_SIZE - 1 2071da177e4SLinus Torvalds bic r1, r1, #D_CACHE_LINE_SIZE - 1 2081da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2091da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line 2101da177e4SLinus Torvalds#else 2111da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line 2121da177e4SLinus Torvalds#endif 2131da177e4SLinus Torvalds1: 214ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 215ca57926dSCatalin Marinas ldr r2, [r0] @ read for ownership 216ca57926dSCatalin Marinas str r2, [r0] @ write for ownership 217f4d6477fSCatalin Marinas#endif 2181da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2191da177e4SLinus Torvalds mcr p15, 0, r0, c7, c6, 1 @ invalidate D line 2201da177e4SLinus Torvalds#else 2211da177e4SLinus Torvalds mcr p15, 0, r0, c7, c7, 1 @ invalidate unified line 2221da177e4SLinus Torvalds#endif 2231da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2241da177e4SLinus Torvalds cmp r0, r1 2251da177e4SLinus Torvalds blo 1b 2261da177e4SLinus Torvalds mov r0, #0 2271da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2281da177e4SLinus Torvalds mov pc, lr 2291da177e4SLinus Torvalds 2301da177e4SLinus Torvalds/* 2311da177e4SLinus Torvalds * v6_dma_clean_range(start,end) 2321da177e4SLinus Torvalds * - start - virtual start address of region 2331da177e4SLinus Torvalds * - end - virtual end address of region 2341da177e4SLinus Torvalds */ 235702b94bfSRussell Kingv6_dma_clean_range: 2361da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2371da177e4SLinus Torvalds1: 238ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 239f4d6477fSCatalin Marinas ldr r2, [r0] @ read for ownership 240f4d6477fSCatalin Marinas#endif 2411da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2421da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 1 @ clean D line 2431da177e4SLinus Torvalds#else 2441da177e4SLinus Torvalds mcr p15, 0, r0, c7, c11, 1 @ clean unified line 2451da177e4SLinus Torvalds#endif 2461da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2471da177e4SLinus Torvalds cmp r0, r1 2481da177e4SLinus Torvalds blo 1b 2491da177e4SLinus Torvalds mov r0, #0 2501da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2511da177e4SLinus Torvalds mov pc, lr 2521da177e4SLinus Torvalds 2531da177e4SLinus Torvalds/* 2541da177e4SLinus Torvalds * v6_dma_flush_range(start,end) 2551da177e4SLinus Torvalds * - start - virtual start address of region 2561da177e4SLinus Torvalds * - end - virtual end address of region 2571da177e4SLinus Torvalds */ 2581da177e4SLinus TorvaldsENTRY(v6_dma_flush_range) 2591da177e4SLinus Torvalds bic r0, r0, #D_CACHE_LINE_SIZE - 1 2601da177e4SLinus Torvalds1: 261ad642d9fSCatalin Marinas#ifdef CONFIG_DMA_CACHE_RWFO 262f4d6477fSCatalin Marinas ldr r2, [r0] @ read for ownership 263f4d6477fSCatalin Marinas str r2, [r0] @ write for ownership 264f4d6477fSCatalin Marinas#endif 2651da177e4SLinus Torvalds#ifdef HARVARD_CACHE 2661da177e4SLinus Torvalds mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line 2671da177e4SLinus Torvalds#else 2681da177e4SLinus Torvalds mcr p15, 0, r0, c7, c15, 1 @ clean & invalidate line 2691da177e4SLinus Torvalds#endif 2701da177e4SLinus Torvalds add r0, r0, #D_CACHE_LINE_SIZE 2711da177e4SLinus Torvalds cmp r0, r1 2721da177e4SLinus Torvalds blo 1b 2731da177e4SLinus Torvalds mov r0, #0 2741da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2751da177e4SLinus Torvalds mov pc, lr 2761da177e4SLinus Torvalds 277a9c9147eSRussell King/* 278a9c9147eSRussell King * dma_map_area(start, size, dir) 279a9c9147eSRussell King * - start - kernel virtual start address 280a9c9147eSRussell King * - size - size of region 281a9c9147eSRussell King * - dir - DMA direction 282a9c9147eSRussell King */ 283a9c9147eSRussell KingENTRY(v6_dma_map_area) 284a9c9147eSRussell King add r1, r1, r0 2852ffe2da3SRussell King teq r2, #DMA_FROM_DEVICE 2862ffe2da3SRussell King beq v6_dma_inv_range 287ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 288ad642d9fSCatalin Marinas b v6_dma_clean_range 289ad642d9fSCatalin Marinas#else 290f4d6477fSCatalin Marinas teq r2, #DMA_TO_DEVICE 291f4d6477fSCatalin Marinas beq v6_dma_clean_range 292f4d6477fSCatalin Marinas b v6_dma_flush_range 293ad642d9fSCatalin Marinas#endif 294a9c9147eSRussell KingENDPROC(v6_dma_map_area) 295a9c9147eSRussell King 296a9c9147eSRussell King/* 297a9c9147eSRussell King * dma_unmap_area(start, size, dir) 298a9c9147eSRussell King * - start - kernel virtual start address 299a9c9147eSRussell King * - size - size of region 300a9c9147eSRussell King * - dir - DMA direction 301a9c9147eSRussell King */ 302a9c9147eSRussell KingENTRY(v6_dma_unmap_area) 303ad642d9fSCatalin Marinas#ifndef CONFIG_DMA_CACHE_RWFO 304ad642d9fSCatalin Marinas add r1, r1, r0 305ad642d9fSCatalin Marinas teq r2, #DMA_TO_DEVICE 306ad642d9fSCatalin Marinas bne v6_dma_inv_range 307ad642d9fSCatalin Marinas#endif 308a9c9147eSRussell King mov pc, lr 309a9c9147eSRussell KingENDPROC(v6_dma_unmap_area) 310a9c9147eSRussell King 3111da177e4SLinus Torvalds __INITDATA 3121da177e4SLinus Torvalds 3131da177e4SLinus Torvalds .type v6_cache_fns, #object 3141da177e4SLinus TorvaldsENTRY(v6_cache_fns) 3151da177e4SLinus Torvalds .long v6_flush_kern_cache_all 3161da177e4SLinus Torvalds .long v6_flush_user_cache_all 3171da177e4SLinus Torvalds .long v6_flush_user_cache_range 3181da177e4SLinus Torvalds .long v6_coherent_kern_range 3191da177e4SLinus Torvalds .long v6_coherent_user_range 3202c9b9c84SRussell King .long v6_flush_kern_dcache_area 321a9c9147eSRussell King .long v6_dma_map_area 322a9c9147eSRussell King .long v6_dma_unmap_area 3231da177e4SLinus Torvalds .long v6_dma_flush_range 3241da177e4SLinus Torvalds .size v6_cache_fns, . - v6_cache_fns 325