1bbe88886SCatalin Marinas/* 2bbe88886SCatalin Marinas * linux/arch/arm/mm/cache-v7.S 3bbe88886SCatalin Marinas * 4bbe88886SCatalin Marinas * Copyright (C) 2001 Deep Blue Solutions Ltd. 5bbe88886SCatalin Marinas * Copyright (C) 2005 ARM Ltd. 6bbe88886SCatalin Marinas * 7bbe88886SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8bbe88886SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9bbe88886SCatalin Marinas * published by the Free Software Foundation. 10bbe88886SCatalin Marinas * 11bbe88886SCatalin Marinas * This is the "shell" of the ARMv7 processor support. 12bbe88886SCatalin Marinas */ 13bbe88886SCatalin Marinas#include <linux/linkage.h> 14bbe88886SCatalin Marinas#include <linux/init.h> 15bbe88886SCatalin Marinas#include <asm/assembler.h> 16bbe88886SCatalin Marinas 17bbe88886SCatalin Marinas#include "proc-macros.S" 18bbe88886SCatalin Marinas 19bbe88886SCatalin Marinas/* 20bbe88886SCatalin Marinas * v7_flush_dcache_all() 21bbe88886SCatalin Marinas * 22bbe88886SCatalin Marinas * Flush the whole D-cache. 23bbe88886SCatalin Marinas * 24347c8b70SCatalin Marinas * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) 25bbe88886SCatalin Marinas * 26bbe88886SCatalin Marinas * - mm - mm_struct describing address space 27bbe88886SCatalin Marinas */ 28bbe88886SCatalin MarinasENTRY(v7_flush_dcache_all) 29c30c2f99SCatalin Marinas dmb @ ensure ordering with previous memory accesses 30bbe88886SCatalin Marinas mrc p15, 1, r0, c0, c0, 1 @ read clidr 31bbe88886SCatalin Marinas ands r3, r0, #0x7000000 @ extract loc from clidr 32bbe88886SCatalin Marinas mov r3, r3, lsr #23 @ left align loc bit field 33bbe88886SCatalin Marinas beq finished @ if loc is 0, then no need to clean 34bbe88886SCatalin Marinas mov r10, #0 @ start clean at cache level 0 35bbe88886SCatalin Marinasloop1: 36bbe88886SCatalin Marinas add r2, r10, r10, lsr #1 @ work out 3x current cache level 37bbe88886SCatalin Marinas mov r1, r0, lsr r2 @ extract cache type bits from clidr 38bbe88886SCatalin Marinas and r1, r1, #7 @ mask of the bits for current cache only 39bbe88886SCatalin Marinas cmp r1, #2 @ see what cache we have at this level 40bbe88886SCatalin Marinas blt skip @ skip if no cache, or just i-cache 41bbe88886SCatalin Marinas mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 42bbe88886SCatalin Marinas isb @ isb to sych the new cssr&csidr 43bbe88886SCatalin Marinas mrc p15, 1, r1, c0, c0, 0 @ read the new csidr 44bbe88886SCatalin Marinas and r2, r1, #7 @ extract the length of the cache lines 45bbe88886SCatalin Marinas add r2, r2, #4 @ add 4 (line length offset) 46bbe88886SCatalin Marinas ldr r4, =0x3ff 47bbe88886SCatalin Marinas ands r4, r4, r1, lsr #3 @ find maximum number on the way size 48bbe88886SCatalin Marinas clz r5, r4 @ find bit position of way size increment 49bbe88886SCatalin Marinas ldr r7, =0x7fff 50bbe88886SCatalin Marinas ands r7, r7, r1, lsr #13 @ extract max number of the index size 51bbe88886SCatalin Marinasloop2: 52bbe88886SCatalin Marinas mov r9, r4 @ create working copy of max way size 53bbe88886SCatalin Marinasloop3: 54347c8b70SCatalin Marinas ARM( orr r11, r10, r9, lsl r5 ) @ factor way and cache number into r11 55347c8b70SCatalin Marinas THUMB( lsl r6, r9, r5 ) 56347c8b70SCatalin Marinas THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11 57347c8b70SCatalin Marinas ARM( orr r11, r11, r7, lsl r2 ) @ factor index number into r11 58347c8b70SCatalin Marinas THUMB( lsl r6, r7, r2 ) 59347c8b70SCatalin Marinas THUMB( orr r11, r11, r6 ) @ factor index number into r11 60bbe88886SCatalin Marinas mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way 61bbe88886SCatalin Marinas subs r9, r9, #1 @ decrement the way 62bbe88886SCatalin Marinas bge loop3 63bbe88886SCatalin Marinas subs r7, r7, #1 @ decrement the index 64bbe88886SCatalin Marinas bge loop2 65bbe88886SCatalin Marinasskip: 66bbe88886SCatalin Marinas add r10, r10, #2 @ increment cache number 67bbe88886SCatalin Marinas cmp r3, r10 68bbe88886SCatalin Marinas bgt loop1 69bbe88886SCatalin Marinasfinished: 70bbe88886SCatalin Marinas mov r10, #0 @ swith back to cache level 0 71bbe88886SCatalin Marinas mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 72c30c2f99SCatalin Marinas dsb 73bbe88886SCatalin Marinas isb 74bbe88886SCatalin Marinas mov pc, lr 7593ed3970SCatalin MarinasENDPROC(v7_flush_dcache_all) 76bbe88886SCatalin Marinas 77bbe88886SCatalin Marinas/* 78bbe88886SCatalin Marinas * v7_flush_cache_all() 79bbe88886SCatalin Marinas * 80bbe88886SCatalin Marinas * Flush the entire cache system. 81bbe88886SCatalin Marinas * The data cache flush is now achieved using atomic clean / invalidates 82bbe88886SCatalin Marinas * working outwards from L1 cache. This is done using Set/Way based cache 83bbe88886SCatalin Marinas * maintainance instructions. 84bbe88886SCatalin Marinas * The instruction cache can still be invalidated back to the point of 85bbe88886SCatalin Marinas * unification in a single instruction. 86bbe88886SCatalin Marinas * 87bbe88886SCatalin Marinas */ 88bbe88886SCatalin MarinasENTRY(v7_flush_kern_cache_all) 89347c8b70SCatalin Marinas ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) 90347c8b70SCatalin Marinas THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) 91bbe88886SCatalin Marinas bl v7_flush_dcache_all 92bbe88886SCatalin Marinas mov r0, #0 93bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate 94347c8b70SCatalin Marinas ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) 95347c8b70SCatalin Marinas THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) 96bbe88886SCatalin Marinas mov pc, lr 9793ed3970SCatalin MarinasENDPROC(v7_flush_kern_cache_all) 98bbe88886SCatalin Marinas 99bbe88886SCatalin Marinas/* 100bbe88886SCatalin Marinas * v7_flush_cache_all() 101bbe88886SCatalin Marinas * 102bbe88886SCatalin Marinas * Flush all TLB entries in a particular address space 103bbe88886SCatalin Marinas * 104bbe88886SCatalin Marinas * - mm - mm_struct describing address space 105bbe88886SCatalin Marinas */ 106bbe88886SCatalin MarinasENTRY(v7_flush_user_cache_all) 107bbe88886SCatalin Marinas /*FALLTHROUGH*/ 108bbe88886SCatalin Marinas 109bbe88886SCatalin Marinas/* 110bbe88886SCatalin Marinas * v7_flush_cache_range(start, end, flags) 111bbe88886SCatalin Marinas * 112bbe88886SCatalin Marinas * Flush a range of TLB entries in the specified address space. 113bbe88886SCatalin Marinas * 114bbe88886SCatalin Marinas * - start - start address (may not be aligned) 115bbe88886SCatalin Marinas * - end - end address (exclusive, may not be aligned) 116bbe88886SCatalin Marinas * - flags - vm_area_struct flags describing address space 117bbe88886SCatalin Marinas * 118bbe88886SCatalin Marinas * It is assumed that: 119bbe88886SCatalin Marinas * - we have a VIPT cache. 120bbe88886SCatalin Marinas */ 121bbe88886SCatalin MarinasENTRY(v7_flush_user_cache_range) 122bbe88886SCatalin Marinas mov pc, lr 12393ed3970SCatalin MarinasENDPROC(v7_flush_user_cache_all) 12493ed3970SCatalin MarinasENDPROC(v7_flush_user_cache_range) 125bbe88886SCatalin Marinas 126bbe88886SCatalin Marinas/* 127bbe88886SCatalin Marinas * v7_coherent_kern_range(start,end) 128bbe88886SCatalin Marinas * 129bbe88886SCatalin Marinas * Ensure that the I and D caches are coherent within specified 130bbe88886SCatalin Marinas * region. This is typically used when code has been written to 131bbe88886SCatalin Marinas * a memory region, and will be executed. 132bbe88886SCatalin Marinas * 133bbe88886SCatalin Marinas * - start - virtual start address of region 134bbe88886SCatalin Marinas * - end - virtual end address of region 135bbe88886SCatalin Marinas * 136bbe88886SCatalin Marinas * It is assumed that: 137bbe88886SCatalin Marinas * - the Icache does not read data from the write buffer 138bbe88886SCatalin Marinas */ 139bbe88886SCatalin MarinasENTRY(v7_coherent_kern_range) 140bbe88886SCatalin Marinas /* FALLTHROUGH */ 141bbe88886SCatalin Marinas 142bbe88886SCatalin Marinas/* 143bbe88886SCatalin Marinas * v7_coherent_user_range(start,end) 144bbe88886SCatalin Marinas * 145bbe88886SCatalin Marinas * Ensure that the I and D caches are coherent within specified 146bbe88886SCatalin Marinas * region. This is typically used when code has been written to 147bbe88886SCatalin Marinas * a memory region, and will be executed. 148bbe88886SCatalin Marinas * 149bbe88886SCatalin Marinas * - start - virtual start address of region 150bbe88886SCatalin Marinas * - end - virtual end address of region 151bbe88886SCatalin Marinas * 152bbe88886SCatalin Marinas * It is assumed that: 153bbe88886SCatalin Marinas * - the Icache does not read data from the write buffer 154bbe88886SCatalin Marinas */ 155bbe88886SCatalin MarinasENTRY(v7_coherent_user_range) 156bbe88886SCatalin Marinas dcache_line_size r2, r3 157bbe88886SCatalin Marinas sub r3, r2, #1 158bbe88886SCatalin Marinas bic r0, r0, r3 159bbe88886SCatalin Marinas1: mcr p15, 0, r0, c7, c11, 1 @ clean D line to the point of unification 160bbe88886SCatalin Marinas dsb 161bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c5, 1 @ invalidate I line 162bbe88886SCatalin Marinas add r0, r0, r2 163bbe88886SCatalin Marinas cmp r0, r1 164bbe88886SCatalin Marinas blo 1b 165bbe88886SCatalin Marinas mov r0, #0 166bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB 167bbe88886SCatalin Marinas dsb 168bbe88886SCatalin Marinas isb 169bbe88886SCatalin Marinas mov pc, lr 17093ed3970SCatalin MarinasENDPROC(v7_coherent_kern_range) 17193ed3970SCatalin MarinasENDPROC(v7_coherent_user_range) 172bbe88886SCatalin Marinas 173bbe88886SCatalin Marinas/* 174bbe88886SCatalin Marinas * v7_flush_kern_dcache_page(kaddr) 175bbe88886SCatalin Marinas * 176bbe88886SCatalin Marinas * Ensure that the data held in the page kaddr is written back 177bbe88886SCatalin Marinas * to the page in question. 178bbe88886SCatalin Marinas * 179bbe88886SCatalin Marinas * - kaddr - kernel address (guaranteed to be page aligned) 180bbe88886SCatalin Marinas */ 181bbe88886SCatalin MarinasENTRY(v7_flush_kern_dcache_page) 182bbe88886SCatalin Marinas dcache_line_size r2, r3 183bbe88886SCatalin Marinas add r1, r0, #PAGE_SZ 184bbe88886SCatalin Marinas1: 185bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line 186bbe88886SCatalin Marinas add r0, r0, r2 187bbe88886SCatalin Marinas cmp r0, r1 188bbe88886SCatalin Marinas blo 1b 189bbe88886SCatalin Marinas dsb 190bbe88886SCatalin Marinas mov pc, lr 19193ed3970SCatalin MarinasENDPROC(v7_flush_kern_dcache_page) 192bbe88886SCatalin Marinas 193bbe88886SCatalin Marinas/* 194bbe88886SCatalin Marinas * v7_dma_inv_range(start,end) 195bbe88886SCatalin Marinas * 196bbe88886SCatalin Marinas * Invalidate the data cache within the specified region; we will 197bbe88886SCatalin Marinas * be performing a DMA operation in this region and we want to 198bbe88886SCatalin Marinas * purge old data in the cache. 199bbe88886SCatalin Marinas * 200bbe88886SCatalin Marinas * - start - virtual start address of region 201bbe88886SCatalin Marinas * - end - virtual end address of region 202bbe88886SCatalin Marinas */ 203bbe88886SCatalin MarinasENTRY(v7_dma_inv_range) 204bbe88886SCatalin Marinas dcache_line_size r2, r3 205bbe88886SCatalin Marinas sub r3, r2, #1 206bbe88886SCatalin Marinas tst r0, r3 207bbe88886SCatalin Marinas bic r0, r0, r3 208bbe88886SCatalin Marinas mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line 209bbe88886SCatalin Marinas 210bbe88886SCatalin Marinas tst r1, r3 211bbe88886SCatalin Marinas bic r1, r1, r3 212bbe88886SCatalin Marinas mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D / U line 213bbe88886SCatalin Marinas1: 214bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c6, 1 @ invalidate D / U line 215bbe88886SCatalin Marinas add r0, r0, r2 216bbe88886SCatalin Marinas cmp r0, r1 217bbe88886SCatalin Marinas blo 1b 218bbe88886SCatalin Marinas dsb 219bbe88886SCatalin Marinas mov pc, lr 22093ed3970SCatalin MarinasENDPROC(v7_dma_inv_range) 221bbe88886SCatalin Marinas 222bbe88886SCatalin Marinas/* 223bbe88886SCatalin Marinas * v7_dma_clean_range(start,end) 224bbe88886SCatalin Marinas * - start - virtual start address of region 225bbe88886SCatalin Marinas * - end - virtual end address of region 226bbe88886SCatalin Marinas */ 227bbe88886SCatalin MarinasENTRY(v7_dma_clean_range) 228bbe88886SCatalin Marinas dcache_line_size r2, r3 229bbe88886SCatalin Marinas sub r3, r2, #1 230bbe88886SCatalin Marinas bic r0, r0, r3 231bbe88886SCatalin Marinas1: 232bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c10, 1 @ clean D / U line 233bbe88886SCatalin Marinas add r0, r0, r2 234bbe88886SCatalin Marinas cmp r0, r1 235bbe88886SCatalin Marinas blo 1b 236bbe88886SCatalin Marinas dsb 237bbe88886SCatalin Marinas mov pc, lr 23893ed3970SCatalin MarinasENDPROC(v7_dma_clean_range) 239bbe88886SCatalin Marinas 240bbe88886SCatalin Marinas/* 241bbe88886SCatalin Marinas * v7_dma_flush_range(start,end) 242bbe88886SCatalin Marinas * - start - virtual start address of region 243bbe88886SCatalin Marinas * - end - virtual end address of region 244bbe88886SCatalin Marinas */ 245bbe88886SCatalin MarinasENTRY(v7_dma_flush_range) 246bbe88886SCatalin Marinas dcache_line_size r2, r3 247bbe88886SCatalin Marinas sub r3, r2, #1 248bbe88886SCatalin Marinas bic r0, r0, r3 249bbe88886SCatalin Marinas1: 250bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line 251bbe88886SCatalin Marinas add r0, r0, r2 252bbe88886SCatalin Marinas cmp r0, r1 253bbe88886SCatalin Marinas blo 1b 254bbe88886SCatalin Marinas dsb 255bbe88886SCatalin Marinas mov pc, lr 25693ed3970SCatalin MarinasENDPROC(v7_dma_flush_range) 257bbe88886SCatalin Marinas 258bbe88886SCatalin Marinas __INITDATA 259bbe88886SCatalin Marinas 260bbe88886SCatalin Marinas .type v7_cache_fns, #object 261bbe88886SCatalin MarinasENTRY(v7_cache_fns) 262bbe88886SCatalin Marinas .long v7_flush_kern_cache_all 263bbe88886SCatalin Marinas .long v7_flush_user_cache_all 264bbe88886SCatalin Marinas .long v7_flush_user_cache_range 265bbe88886SCatalin Marinas .long v7_coherent_kern_range 266bbe88886SCatalin Marinas .long v7_coherent_user_range 267bbe88886SCatalin Marinas .long v7_flush_kern_dcache_page 268bbe88886SCatalin Marinas .long v7_dma_inv_range 269bbe88886SCatalin Marinas .long v7_dma_clean_range 270bbe88886SCatalin Marinas .long v7_dma_flush_range 271bbe88886SCatalin Marinas .size v7_cache_fns, . - v7_cache_fns 272