1 /* 2 * (C) Copyright 2010 3 * Texas Instruments, <www.ti.com> 4 * Aneesh V <aneesh@ti.com> 5 * 6 * See file CREDITS for list of people who contributed to this 7 * project. 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License as 11 * published by the Free Software Foundation; either version 2 of 12 * the License, or (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software 21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, 22 * MA 02111-1307 USA 23 */ 24 #include <linux/types.h> 25 #include <common.h> 26 #include <asm/armv7.h> 27 #include <asm/utils.h> 28 29 #define ARMV7_DCACHE_INVAL_ALL 1 30 #define ARMV7_DCACHE_CLEAN_INVAL_ALL 2 31 #define ARMV7_DCACHE_INVAL_RANGE 3 32 #define ARMV7_DCACHE_CLEAN_INVAL_RANGE 4 33 34 #ifndef CONFIG_SYS_DCACHE_OFF 35 /* 36 * Write the level and type you want to Cache Size Selection Register(CSSELR) 37 * to get size details from Current Cache Size ID Register(CCSIDR) 38 */ 39 static void set_csselr(u32 level, u32 type) 40 { u32 csselr = level << 1 | type; 41 42 /* Write to Cache Size Selection Register(CSSELR) */ 43 asm volatile ("mcr p15, 2, %0, c0, c0, 0" : : "r" (csselr)); 44 } 45 46 static u32 get_ccsidr(void) 47 { 48 u32 ccsidr; 49 50 /* Read current CP15 Cache Size ID Register */ 51 asm volatile ("mrc p15, 1, %0, c0, c0, 0" : "=r" (ccsidr)); 52 return ccsidr; 53 } 54 55 static u32 get_clidr(void) 56 { 57 u32 clidr; 58 59 /* Read current CP15 Cache Level ID Register */ 60 asm volatile ("mrc p15,1,%0,c0,c0,1" : "=r" (clidr)); 61 return clidr; 62 } 63 64 static void v7_inval_dcache_level_setway(u32 level, u32 num_sets, 65 u32 num_ways, u32 way_shift, 66 u32 log2_line_len) 67 { 68 int way, set, setway; 69 70 /* 71 * For optimal assembly code: 72 * a. count down 73 * b. have bigger loop inside 74 */ 75 for (way = num_ways - 1; way >= 0 ; way--) { 76 for (set = num_sets - 1; set >= 0; set--) { 77 setway = (level << 1) | (set << log2_line_len) | 78 (way << way_shift); 79 /* Invalidate data/unified cache line by set/way */ 80 asm volatile (" mcr p15, 0, %0, c7, c6, 2" 81 : : "r" (setway)); 82 } 83 } 84 /* DSB to make sure the operation is complete */ 85 CP15DSB; 86 } 87 88 static void v7_clean_inval_dcache_level_setway(u32 level, u32 num_sets, 89 u32 num_ways, u32 way_shift, 90 u32 log2_line_len) 91 { 92 int way, set, setway; 93 94 /* 95 * For optimal assembly code: 96 * a. count down 97 * b. have bigger loop inside 98 */ 99 for (way = num_ways - 1; way >= 0 ; way--) { 100 for (set = num_sets - 1; set >= 0; set--) { 101 setway = (level << 1) | (set << log2_line_len) | 102 (way << way_shift); 103 /* 104 * Clean & Invalidate data/unified 105 * cache line by set/way 106 */ 107 asm volatile (" mcr p15, 0, %0, c7, c14, 2" 108 : : "r" (setway)); 109 } 110 } 111 /* DSB to make sure the operation is complete */ 112 CP15DSB; 113 } 114 115 static void v7_maint_dcache_level_setway(u32 level, u32 operation) 116 { 117 u32 ccsidr; 118 u32 num_sets, num_ways, log2_line_len, log2_num_ways; 119 u32 way_shift; 120 121 set_csselr(level, ARMV7_CSSELR_IND_DATA_UNIFIED); 122 123 ccsidr = get_ccsidr(); 124 125 log2_line_len = ((ccsidr & CCSIDR_LINE_SIZE_MASK) >> 126 CCSIDR_LINE_SIZE_OFFSET) + 2; 127 /* Converting from words to bytes */ 128 log2_line_len += 2; 129 130 num_ways = ((ccsidr & CCSIDR_ASSOCIATIVITY_MASK) >> 131 CCSIDR_ASSOCIATIVITY_OFFSET) + 1; 132 num_sets = ((ccsidr & CCSIDR_NUM_SETS_MASK) >> 133 CCSIDR_NUM_SETS_OFFSET) + 1; 134 /* 135 * According to ARMv7 ARM number of sets and number of ways need 136 * not be a power of 2 137 */ 138 log2_num_ways = log_2_n_round_up(num_ways); 139 140 way_shift = (32 - log2_num_ways); 141 if (operation == ARMV7_DCACHE_INVAL_ALL) { 142 v7_inval_dcache_level_setway(level, num_sets, num_ways, 143 way_shift, log2_line_len); 144 } else if (operation == ARMV7_DCACHE_CLEAN_INVAL_ALL) { 145 v7_clean_inval_dcache_level_setway(level, num_sets, num_ways, 146 way_shift, log2_line_len); 147 } 148 } 149 150 static void v7_maint_dcache_all(u32 operation) 151 { 152 u32 level, cache_type, level_start_bit = 0; 153 154 u32 clidr = get_clidr(); 155 156 for (level = 0; level < 7; level++) { 157 cache_type = (clidr >> level_start_bit) & 0x7; 158 if ((cache_type == ARMV7_CLIDR_CTYPE_DATA_ONLY) || 159 (cache_type == ARMV7_CLIDR_CTYPE_INSTRUCTION_DATA) || 160 (cache_type == ARMV7_CLIDR_CTYPE_UNIFIED)) 161 v7_maint_dcache_level_setway(level, operation); 162 level_start_bit += 3; 163 } 164 } 165 166 static void v7_dcache_clean_inval_range(u32 start, 167 u32 stop, u32 line_len) 168 { 169 u32 mva; 170 171 /* Align start to cache line boundary */ 172 start &= ~(line_len - 1); 173 for (mva = start; mva < stop; mva = mva + line_len) { 174 /* DCCIMVAC - Clean & Invalidate data cache by MVA to PoC */ 175 asm volatile ("mcr p15, 0, %0, c7, c14, 1" : : "r" (mva)); 176 } 177 } 178 179 static void v7_dcache_inval_range(u32 start, u32 stop, u32 line_len) 180 { 181 u32 mva; 182 183 /* 184 * If start address is not aligned to cache-line do not 185 * invalidate the first cache-line 186 */ 187 if (start & (line_len - 1)) { 188 printf("ERROR: %s - start address is not aligned - 0x%08x\n", 189 __func__, start); 190 /* move to next cache line */ 191 start = (start + line_len - 1) & ~(line_len - 1); 192 } 193 194 /* 195 * If stop address is not aligned to cache-line do not 196 * invalidate the last cache-line 197 */ 198 if (stop & (line_len - 1)) { 199 printf("ERROR: %s - stop address is not aligned - 0x%08x\n", 200 __func__, stop); 201 /* align to the beginning of this cache line */ 202 stop &= ~(line_len - 1); 203 } 204 205 for (mva = start; mva < stop; mva = mva + line_len) { 206 /* DCIMVAC - Invalidate data cache by MVA to PoC */ 207 asm volatile ("mcr p15, 0, %0, c7, c6, 1" : : "r" (mva)); 208 } 209 } 210 211 static void v7_dcache_maint_range(u32 start, u32 stop, u32 range_op) 212 { 213 u32 line_len, ccsidr; 214 215 ccsidr = get_ccsidr(); 216 line_len = ((ccsidr & CCSIDR_LINE_SIZE_MASK) >> 217 CCSIDR_LINE_SIZE_OFFSET) + 2; 218 /* Converting from words to bytes */ 219 line_len += 2; 220 /* converting from log2(linelen) to linelen */ 221 line_len = 1 << line_len; 222 223 switch (range_op) { 224 case ARMV7_DCACHE_CLEAN_INVAL_RANGE: 225 v7_dcache_clean_inval_range(start, stop, line_len); 226 break; 227 case ARMV7_DCACHE_INVAL_RANGE: 228 v7_dcache_inval_range(start, stop, line_len); 229 break; 230 } 231 232 /* DSB to make sure the operation is complete */ 233 CP15DSB; 234 } 235 236 /* Invalidate TLB */ 237 static void v7_inval_tlb(void) 238 { 239 /* Invalidate entire unified TLB */ 240 asm volatile ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0)); 241 /* Invalidate entire data TLB */ 242 asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0)); 243 /* Invalidate entire instruction TLB */ 244 asm volatile ("mcr p15, 0, %0, c8, c5, 0" : : "r" (0)); 245 /* Full system DSB - make sure that the invalidation is complete */ 246 CP15DSB; 247 /* Full system ISB - make sure the instruction stream sees it */ 248 CP15ISB; 249 } 250 251 void invalidate_dcache_all(void) 252 { 253 v7_maint_dcache_all(ARMV7_DCACHE_INVAL_ALL); 254 255 v7_outer_cache_inval_all(); 256 } 257 258 /* 259 * Performs a clean & invalidation of the entire data cache 260 * at all levels 261 */ 262 void flush_dcache_all(void) 263 { 264 v7_maint_dcache_all(ARMV7_DCACHE_CLEAN_INVAL_ALL); 265 266 v7_outer_cache_flush_all(); 267 } 268 269 /* 270 * Invalidates range in all levels of D-cache/unified cache used: 271 * Affects the range [start, stop - 1] 272 */ 273 void invalidate_dcache_range(unsigned long start, unsigned long stop) 274 { 275 276 v7_dcache_maint_range(start, stop, ARMV7_DCACHE_INVAL_RANGE); 277 278 v7_outer_cache_inval_range(start, stop); 279 } 280 281 /* 282 * Flush range(clean & invalidate) from all levels of D-cache/unified 283 * cache used: 284 * Affects the range [start, stop - 1] 285 */ 286 void flush_dcache_range(unsigned long start, unsigned long stop) 287 { 288 v7_dcache_maint_range(start, stop, ARMV7_DCACHE_CLEAN_INVAL_RANGE); 289 290 v7_outer_cache_flush_range(start, stop); 291 } 292 293 void arm_init_before_mmu(void) 294 { 295 v7_outer_cache_enable(); 296 invalidate_dcache_all(); 297 v7_inval_tlb(); 298 } 299 300 void mmu_page_table_flush(unsigned long start, unsigned long stop) 301 { 302 flush_dcache_range(start, stop); 303 v7_inval_tlb(); 304 } 305 306 /* 307 * Flush range from all levels of d-cache/unified-cache used: 308 * Affects the range [start, start + size - 1] 309 */ 310 void flush_cache(unsigned long start, unsigned long size) 311 { 312 flush_dcache_range(start, start + size); 313 } 314 #else /* #ifndef CONFIG_SYS_DCACHE_OFF */ 315 void invalidate_dcache_all(void) 316 { 317 } 318 319 void flush_dcache_all(void) 320 { 321 } 322 323 void invalidate_dcache_range(unsigned long start, unsigned long stop) 324 { 325 } 326 327 void flush_dcache_range(unsigned long start, unsigned long stop) 328 { 329 } 330 331 void arm_init_before_mmu(void) 332 { 333 } 334 335 void flush_cache(unsigned long start, unsigned long size) 336 { 337 } 338 339 void mmu_page_table_flush(unsigned long start, unsigned long stop) 340 { 341 } 342 343 void arm_init_domains(void) 344 { 345 } 346 #endif /* #ifndef CONFIG_SYS_DCACHE_OFF */ 347 348 #ifndef CONFIG_SYS_ICACHE_OFF 349 /* Invalidate entire I-cache and branch predictor array */ 350 void invalidate_icache_all(void) 351 { 352 /* 353 * Invalidate all instruction caches to PoU. 354 * Also flushes branch target cache. 355 */ 356 asm volatile ("mcr p15, 0, %0, c7, c5, 0" : : "r" (0)); 357 358 /* Invalidate entire branch predictor array */ 359 asm volatile ("mcr p15, 0, %0, c7, c5, 6" : : "r" (0)); 360 361 /* Full system DSB - make sure that the invalidation is complete */ 362 CP15DSB; 363 364 /* ISB - make sure the instruction stream sees it */ 365 CP15ISB; 366 } 367 #else 368 void invalidate_icache_all(void) 369 { 370 } 371 #endif 372 373 /* 374 * Stub implementations for outer cache operations 375 */ 376 void __v7_outer_cache_enable(void) 377 { 378 } 379 void v7_outer_cache_enable(void) 380 __attribute__((weak, alias("__v7_outer_cache_enable"))); 381 382 void __v7_outer_cache_disable(void) 383 { 384 } 385 void v7_outer_cache_disable(void) 386 __attribute__((weak, alias("__v7_outer_cache_disable"))); 387 388 void __v7_outer_cache_flush_all(void) 389 { 390 } 391 void v7_outer_cache_flush_all(void) 392 __attribute__((weak, alias("__v7_outer_cache_flush_all"))); 393 394 void __v7_outer_cache_inval_all(void) 395 { 396 } 397 void v7_outer_cache_inval_all(void) 398 __attribute__((weak, alias("__v7_outer_cache_inval_all"))); 399 400 void __v7_outer_cache_flush_range(u32 start, u32 end) 401 { 402 } 403 void v7_outer_cache_flush_range(u32 start, u32 end) 404 __attribute__((weak, alias("__v7_outer_cache_flush_range"))); 405 406 void __v7_outer_cache_inval_range(u32 start, u32 end) 407 { 408 } 409 void v7_outer_cache_inval_range(u32 start, u32 end) 410 __attribute__((weak, alias("__v7_outer_cache_inval_range"))); 411