1bbe88886SCatalin Marinas/* 2bbe88886SCatalin Marinas * linux/arch/arm/mm/cache-v7.S 3bbe88886SCatalin Marinas * 4bbe88886SCatalin Marinas * Copyright (C) 2001 Deep Blue Solutions Ltd. 5bbe88886SCatalin Marinas * Copyright (C) 2005 ARM Ltd. 6bbe88886SCatalin Marinas * 7bbe88886SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8bbe88886SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9bbe88886SCatalin Marinas * published by the Free Software Foundation. 10bbe88886SCatalin Marinas * 11bbe88886SCatalin Marinas * This is the "shell" of the ARMv7 processor support. 12bbe88886SCatalin Marinas */ 13bbe88886SCatalin Marinas#include <linux/linkage.h> 14bbe88886SCatalin Marinas#include <linux/init.h> 15bbe88886SCatalin Marinas#include <asm/assembler.h> 16c5102f59SWill Deacon#include <asm/errno.h> 1732cfb1b1SCatalin Marinas#include <asm/unwind.h> 18bbe88886SCatalin Marinas 19bbe88886SCatalin Marinas#include "proc-macros.S" 20bbe88886SCatalin Marinas 21bbe88886SCatalin Marinas/* 2281d11955STony Lindgren * v7_flush_icache_all() 2381d11955STony Lindgren * 2481d11955STony Lindgren * Flush the whole I-cache. 2581d11955STony Lindgren * 2681d11955STony Lindgren * Registers: 2781d11955STony Lindgren * r0 - set to 0 2881d11955STony Lindgren */ 2981d11955STony LindgrenENTRY(v7_flush_icache_all) 3081d11955STony Lindgren mov r0, #0 3181d11955STony Lindgren ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable 3281d11955STony Lindgren ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate 3381d11955STony Lindgren mov pc, lr 3481d11955STony LindgrenENDPROC(v7_flush_icache_all) 3581d11955STony Lindgren 3681d11955STony Lindgren/* 37bbe88886SCatalin Marinas * v7_flush_dcache_all() 38bbe88886SCatalin Marinas * 39bbe88886SCatalin Marinas * Flush the whole D-cache. 40bbe88886SCatalin Marinas * 41347c8b70SCatalin Marinas * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) 42bbe88886SCatalin Marinas * 43bbe88886SCatalin Marinas * - mm - mm_struct describing address space 44bbe88886SCatalin Marinas */ 45bbe88886SCatalin MarinasENTRY(v7_flush_dcache_all) 46c30c2f99SCatalin Marinas dmb @ ensure ordering with previous memory accesses 47bbe88886SCatalin Marinas mrc p15, 1, r0, c0, c0, 1 @ read clidr 48bbe88886SCatalin Marinas ands r3, r0, #0x7000000 @ extract loc from clidr 49bbe88886SCatalin Marinas mov r3, r3, lsr #23 @ left align loc bit field 50bbe88886SCatalin Marinas beq finished @ if loc is 0, then no need to clean 51bbe88886SCatalin Marinas mov r10, #0 @ start clean at cache level 0 52bbe88886SCatalin Marinasloop1: 53bbe88886SCatalin Marinas add r2, r10, r10, lsr #1 @ work out 3x current cache level 54bbe88886SCatalin Marinas mov r1, r0, lsr r2 @ extract cache type bits from clidr 55bbe88886SCatalin Marinas and r1, r1, #7 @ mask of the bits for current cache only 56bbe88886SCatalin Marinas cmp r1, #2 @ see what cache we have at this level 57bbe88886SCatalin Marinas blt skip @ skip if no cache, or just i-cache 58b46c0f74SStephen Boyd#ifdef CONFIG_PREEMPT 598e43a905SRabin Vincent save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic 60b46c0f74SStephen Boyd#endif 61bbe88886SCatalin Marinas mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 62bbe88886SCatalin Marinas isb @ isb to sych the new cssr&csidr 63bbe88886SCatalin Marinas mrc p15, 1, r1, c0, c0, 0 @ read the new csidr 64b46c0f74SStephen Boyd#ifdef CONFIG_PREEMPT 65b46c0f74SStephen Boyd restore_irqs_notrace r9 66b46c0f74SStephen Boyd#endif 67bbe88886SCatalin Marinas and r2, r1, #7 @ extract the length of the cache lines 68bbe88886SCatalin Marinas add r2, r2, #4 @ add 4 (line length offset) 69bbe88886SCatalin Marinas ldr r4, =0x3ff 70bbe88886SCatalin Marinas ands r4, r4, r1, lsr #3 @ find maximum number on the way size 71bbe88886SCatalin Marinas clz r5, r4 @ find bit position of way size increment 72bbe88886SCatalin Marinas ldr r7, =0x7fff 73bbe88886SCatalin Marinas ands r7, r7, r1, lsr #13 @ extract max number of the index size 74bbe88886SCatalin Marinasloop2: 75bbe88886SCatalin Marinas mov r9, r4 @ create working copy of max way size 76bbe88886SCatalin Marinasloop3: 77347c8b70SCatalin Marinas ARM( orr r11, r10, r9, lsl r5 ) @ factor way and cache number into r11 78347c8b70SCatalin Marinas THUMB( lsl r6, r9, r5 ) 79347c8b70SCatalin Marinas THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11 80347c8b70SCatalin Marinas ARM( orr r11, r11, r7, lsl r2 ) @ factor index number into r11 81347c8b70SCatalin Marinas THUMB( lsl r6, r7, r2 ) 82347c8b70SCatalin Marinas THUMB( orr r11, r11, r6 ) @ factor index number into r11 83bbe88886SCatalin Marinas mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way 84bbe88886SCatalin Marinas subs r9, r9, #1 @ decrement the way 85bbe88886SCatalin Marinas bge loop3 86bbe88886SCatalin Marinas subs r7, r7, #1 @ decrement the index 87bbe88886SCatalin Marinas bge loop2 88bbe88886SCatalin Marinasskip: 89bbe88886SCatalin Marinas add r10, r10, #2 @ increment cache number 90bbe88886SCatalin Marinas cmp r3, r10 91bbe88886SCatalin Marinas bgt loop1 92bbe88886SCatalin Marinasfinished: 93bbe88886SCatalin Marinas mov r10, #0 @ swith back to cache level 0 94bbe88886SCatalin Marinas mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 95c30c2f99SCatalin Marinas dsb 96bbe88886SCatalin Marinas isb 97bbe88886SCatalin Marinas mov pc, lr 9893ed3970SCatalin MarinasENDPROC(v7_flush_dcache_all) 99bbe88886SCatalin Marinas 100bbe88886SCatalin Marinas/* 101bbe88886SCatalin Marinas * v7_flush_cache_all() 102bbe88886SCatalin Marinas * 103bbe88886SCatalin Marinas * Flush the entire cache system. 104bbe88886SCatalin Marinas * The data cache flush is now achieved using atomic clean / invalidates 105bbe88886SCatalin Marinas * working outwards from L1 cache. This is done using Set/Way based cache 10625985edcSLucas De Marchi * maintenance instructions. 107bbe88886SCatalin Marinas * The instruction cache can still be invalidated back to the point of 108bbe88886SCatalin Marinas * unification in a single instruction. 109bbe88886SCatalin Marinas * 110bbe88886SCatalin Marinas */ 111bbe88886SCatalin MarinasENTRY(v7_flush_kern_cache_all) 112347c8b70SCatalin Marinas ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) 113347c8b70SCatalin Marinas THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) 114bbe88886SCatalin Marinas bl v7_flush_dcache_all 115bbe88886SCatalin Marinas mov r0, #0 116f00ec48fSRussell King ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable 117f00ec48fSRussell King ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate 118347c8b70SCatalin Marinas ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) 119347c8b70SCatalin Marinas THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) 120bbe88886SCatalin Marinas mov pc, lr 12193ed3970SCatalin MarinasENDPROC(v7_flush_kern_cache_all) 122bbe88886SCatalin Marinas 123bbe88886SCatalin Marinas/* 124bbe88886SCatalin Marinas * v7_flush_cache_all() 125bbe88886SCatalin Marinas * 126bbe88886SCatalin Marinas * Flush all TLB entries in a particular address space 127bbe88886SCatalin Marinas * 128bbe88886SCatalin Marinas * - mm - mm_struct describing address space 129bbe88886SCatalin Marinas */ 130bbe88886SCatalin MarinasENTRY(v7_flush_user_cache_all) 131bbe88886SCatalin Marinas /*FALLTHROUGH*/ 132bbe88886SCatalin Marinas 133bbe88886SCatalin Marinas/* 134bbe88886SCatalin Marinas * v7_flush_cache_range(start, end, flags) 135bbe88886SCatalin Marinas * 136bbe88886SCatalin Marinas * Flush a range of TLB entries in the specified address space. 137bbe88886SCatalin Marinas * 138bbe88886SCatalin Marinas * - start - start address (may not be aligned) 139bbe88886SCatalin Marinas * - end - end address (exclusive, may not be aligned) 140bbe88886SCatalin Marinas * - flags - vm_area_struct flags describing address space 141bbe88886SCatalin Marinas * 142bbe88886SCatalin Marinas * It is assumed that: 143bbe88886SCatalin Marinas * - we have a VIPT cache. 144bbe88886SCatalin Marinas */ 145bbe88886SCatalin MarinasENTRY(v7_flush_user_cache_range) 146bbe88886SCatalin Marinas mov pc, lr 14793ed3970SCatalin MarinasENDPROC(v7_flush_user_cache_all) 14893ed3970SCatalin MarinasENDPROC(v7_flush_user_cache_range) 149bbe88886SCatalin Marinas 150bbe88886SCatalin Marinas/* 151bbe88886SCatalin Marinas * v7_coherent_kern_range(start,end) 152bbe88886SCatalin Marinas * 153bbe88886SCatalin Marinas * Ensure that the I and D caches are coherent within specified 154bbe88886SCatalin Marinas * region. This is typically used when code has been written to 155bbe88886SCatalin Marinas * a memory region, and will be executed. 156bbe88886SCatalin Marinas * 157bbe88886SCatalin Marinas * - start - virtual start address of region 158bbe88886SCatalin Marinas * - end - virtual end address of region 159bbe88886SCatalin Marinas * 160bbe88886SCatalin Marinas * It is assumed that: 161bbe88886SCatalin Marinas * - the Icache does not read data from the write buffer 162bbe88886SCatalin Marinas */ 163bbe88886SCatalin MarinasENTRY(v7_coherent_kern_range) 164bbe88886SCatalin Marinas /* FALLTHROUGH */ 165bbe88886SCatalin Marinas 166bbe88886SCatalin Marinas/* 167bbe88886SCatalin Marinas * v7_coherent_user_range(start,end) 168bbe88886SCatalin Marinas * 169bbe88886SCatalin Marinas * Ensure that the I and D caches are coherent within specified 170bbe88886SCatalin Marinas * region. This is typically used when code has been written to 171bbe88886SCatalin Marinas * a memory region, and will be executed. 172bbe88886SCatalin Marinas * 173bbe88886SCatalin Marinas * - start - virtual start address of region 174bbe88886SCatalin Marinas * - end - virtual end address of region 175bbe88886SCatalin Marinas * 176bbe88886SCatalin Marinas * It is assumed that: 177bbe88886SCatalin Marinas * - the Icache does not read data from the write buffer 178bbe88886SCatalin Marinas */ 179bbe88886SCatalin MarinasENTRY(v7_coherent_user_range) 18032cfb1b1SCatalin Marinas UNWIND(.fnstart ) 181bbe88886SCatalin Marinas dcache_line_size r2, r3 182bbe88886SCatalin Marinas sub r3, r2, #1 183da30e0acSCatalin Marinas bic r12, r0, r3 184f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 185f630c1bdSWill Deacon ALT_SMP(W(dsb)) 186f630c1bdSWill Deacon ALT_UP(W(nop)) 187f630c1bdSWill Deacon#endif 18832cfb1b1SCatalin Marinas1: 189da30e0acSCatalin Marinas USER( mcr p15, 0, r12, c7, c11, 1 ) @ clean D line to the point of unification 190da30e0acSCatalin Marinas add r12, r12, r2 191da30e0acSCatalin Marinas cmp r12, r1 192bbe88886SCatalin Marinas blo 1b 193da30e0acSCatalin Marinas dsb 194da30e0acSCatalin Marinas icache_line_size r2, r3 195da30e0acSCatalin Marinas sub r3, r2, #1 196da30e0acSCatalin Marinas bic r12, r0, r3 197da30e0acSCatalin Marinas2: 198da30e0acSCatalin Marinas USER( mcr p15, 0, r12, c7, c5, 1 ) @ invalidate I line 199da30e0acSCatalin Marinas add r12, r12, r2 200da30e0acSCatalin Marinas cmp r12, r1 201da30e0acSCatalin Marinas blo 2b 202bbe88886SCatalin Marinas mov r0, #0 203f00ec48fSRussell King ALT_SMP(mcr p15, 0, r0, c7, c1, 6) @ invalidate BTB Inner Shareable 204f00ec48fSRussell King ALT_UP(mcr p15, 0, r0, c7, c5, 6) @ invalidate BTB 205bbe88886SCatalin Marinas dsb 206bbe88886SCatalin Marinas isb 207bbe88886SCatalin Marinas mov pc, lr 20832cfb1b1SCatalin Marinas 20932cfb1b1SCatalin Marinas/* 21032cfb1b1SCatalin Marinas * Fault handling for the cache operation above. If the virtual address in r0 211c5102f59SWill Deacon * isn't mapped, fail with -EFAULT. 21232cfb1b1SCatalin Marinas */ 21332cfb1b1SCatalin Marinas9001: 2147253b85cSSimon Horman#ifdef CONFIG_ARM_ERRATA_775420 2157253b85cSSimon Horman dsb 2167253b85cSSimon Horman#endif 217c5102f59SWill Deacon mov r0, #-EFAULT 218c5102f59SWill Deacon mov pc, lr 21932cfb1b1SCatalin Marinas UNWIND(.fnend ) 22093ed3970SCatalin MarinasENDPROC(v7_coherent_kern_range) 22193ed3970SCatalin MarinasENDPROC(v7_coherent_user_range) 222bbe88886SCatalin Marinas 223bbe88886SCatalin Marinas/* 2242c9b9c84SRussell King * v7_flush_kern_dcache_area(void *addr, size_t size) 225bbe88886SCatalin Marinas * 226bbe88886SCatalin Marinas * Ensure that the data held in the page kaddr is written back 227bbe88886SCatalin Marinas * to the page in question. 228bbe88886SCatalin Marinas * 2292c9b9c84SRussell King * - addr - kernel address 2302c9b9c84SRussell King * - size - region size 231bbe88886SCatalin Marinas */ 2322c9b9c84SRussell KingENTRY(v7_flush_kern_dcache_area) 233bbe88886SCatalin Marinas dcache_line_size r2, r3 2342c9b9c84SRussell King add r1, r0, r1 235a248b13bSWill Deacon sub r3, r2, #1 236a248b13bSWill Deacon bic r0, r0, r3 237f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 238f630c1bdSWill Deacon ALT_SMP(W(dsb)) 239f630c1bdSWill Deacon ALT_UP(W(nop)) 240f630c1bdSWill Deacon#endif 241bbe88886SCatalin Marinas1: 242bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line 243bbe88886SCatalin Marinas add r0, r0, r2 244bbe88886SCatalin Marinas cmp r0, r1 245bbe88886SCatalin Marinas blo 1b 246bbe88886SCatalin Marinas dsb 247bbe88886SCatalin Marinas mov pc, lr 2482c9b9c84SRussell KingENDPROC(v7_flush_kern_dcache_area) 249bbe88886SCatalin Marinas 250bbe88886SCatalin Marinas/* 251bbe88886SCatalin Marinas * v7_dma_inv_range(start,end) 252bbe88886SCatalin Marinas * 253bbe88886SCatalin Marinas * Invalidate the data cache within the specified region; we will 254bbe88886SCatalin Marinas * be performing a DMA operation in this region and we want to 255bbe88886SCatalin Marinas * purge old data in the cache. 256bbe88886SCatalin Marinas * 257bbe88886SCatalin Marinas * - start - virtual start address of region 258bbe88886SCatalin Marinas * - end - virtual end address of region 259bbe88886SCatalin Marinas */ 260702b94bfSRussell Kingv7_dma_inv_range: 261bbe88886SCatalin Marinas dcache_line_size r2, r3 262bbe88886SCatalin Marinas sub r3, r2, #1 263bbe88886SCatalin Marinas tst r0, r3 264bbe88886SCatalin Marinas bic r0, r0, r3 265f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 266f630c1bdSWill Deacon ALT_SMP(W(dsb)) 267f630c1bdSWill Deacon ALT_UP(W(nop)) 268f630c1bdSWill Deacon#endif 269bbe88886SCatalin Marinas mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line 270bbe88886SCatalin Marinas 271bbe88886SCatalin Marinas tst r1, r3 272bbe88886SCatalin Marinas bic r1, r1, r3 273bbe88886SCatalin Marinas mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D / U line 274bbe88886SCatalin Marinas1: 275bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c6, 1 @ invalidate D / U line 276bbe88886SCatalin Marinas add r0, r0, r2 277bbe88886SCatalin Marinas cmp r0, r1 278bbe88886SCatalin Marinas blo 1b 279bbe88886SCatalin Marinas dsb 280bbe88886SCatalin Marinas mov pc, lr 28193ed3970SCatalin MarinasENDPROC(v7_dma_inv_range) 282bbe88886SCatalin Marinas 283bbe88886SCatalin Marinas/* 284bbe88886SCatalin Marinas * v7_dma_clean_range(start,end) 285bbe88886SCatalin Marinas * - start - virtual start address of region 286bbe88886SCatalin Marinas * - end - virtual end address of region 287bbe88886SCatalin Marinas */ 288702b94bfSRussell Kingv7_dma_clean_range: 289bbe88886SCatalin Marinas dcache_line_size r2, r3 290bbe88886SCatalin Marinas sub r3, r2, #1 291bbe88886SCatalin Marinas bic r0, r0, r3 292f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 293f630c1bdSWill Deacon ALT_SMP(W(dsb)) 294f630c1bdSWill Deacon ALT_UP(W(nop)) 295f630c1bdSWill Deacon#endif 296bbe88886SCatalin Marinas1: 297bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c10, 1 @ clean D / U line 298bbe88886SCatalin Marinas add r0, r0, r2 299bbe88886SCatalin Marinas cmp r0, r1 300bbe88886SCatalin Marinas blo 1b 301bbe88886SCatalin Marinas dsb 302bbe88886SCatalin Marinas mov pc, lr 30393ed3970SCatalin MarinasENDPROC(v7_dma_clean_range) 304bbe88886SCatalin Marinas 305bbe88886SCatalin Marinas/* 306bbe88886SCatalin Marinas * v7_dma_flush_range(start,end) 307bbe88886SCatalin Marinas * - start - virtual start address of region 308bbe88886SCatalin Marinas * - end - virtual end address of region 309bbe88886SCatalin Marinas */ 310bbe88886SCatalin MarinasENTRY(v7_dma_flush_range) 311bbe88886SCatalin Marinas dcache_line_size r2, r3 312bbe88886SCatalin Marinas sub r3, r2, #1 313bbe88886SCatalin Marinas bic r0, r0, r3 314f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 315f630c1bdSWill Deacon ALT_SMP(W(dsb)) 316f630c1bdSWill Deacon ALT_UP(W(nop)) 317f630c1bdSWill Deacon#endif 318bbe88886SCatalin Marinas1: 319bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line 320bbe88886SCatalin Marinas add r0, r0, r2 321bbe88886SCatalin Marinas cmp r0, r1 322bbe88886SCatalin Marinas blo 1b 323bbe88886SCatalin Marinas dsb 324bbe88886SCatalin Marinas mov pc, lr 32593ed3970SCatalin MarinasENDPROC(v7_dma_flush_range) 326bbe88886SCatalin Marinas 327a9c9147eSRussell King/* 328a9c9147eSRussell King * dma_map_area(start, size, dir) 329a9c9147eSRussell King * - start - kernel virtual start address 330a9c9147eSRussell King * - size - size of region 331a9c9147eSRussell King * - dir - DMA direction 332a9c9147eSRussell King */ 333a9c9147eSRussell KingENTRY(v7_dma_map_area) 334a9c9147eSRussell King add r1, r1, r0 3352ffe2da3SRussell King teq r2, #DMA_FROM_DEVICE 3362ffe2da3SRussell King beq v7_dma_inv_range 3372ffe2da3SRussell King b v7_dma_clean_range 338a9c9147eSRussell KingENDPROC(v7_dma_map_area) 339a9c9147eSRussell King 340a9c9147eSRussell King/* 341a9c9147eSRussell King * dma_unmap_area(start, size, dir) 342a9c9147eSRussell King * - start - kernel virtual start address 343a9c9147eSRussell King * - size - size of region 344a9c9147eSRussell King * - dir - DMA direction 345a9c9147eSRussell King */ 346a9c9147eSRussell KingENTRY(v7_dma_unmap_area) 3472ffe2da3SRussell King add r1, r1, r0 3482ffe2da3SRussell King teq r2, #DMA_TO_DEVICE 3492ffe2da3SRussell King bne v7_dma_inv_range 350a9c9147eSRussell King mov pc, lr 351a9c9147eSRussell KingENDPROC(v7_dma_unmap_area) 352a9c9147eSRussell King 353bbe88886SCatalin Marinas __INITDATA 354bbe88886SCatalin Marinas 355455a01ecSDave Martin @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) 356455a01ecSDave Martin define_cache_functions v7 357