1660d5f0dSAlexey Brodkin /* 2660d5f0dSAlexey Brodkin * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved. 3660d5f0dSAlexey Brodkin * 4660d5f0dSAlexey Brodkin * SPDX-License-Identifier: GPL-2.0+ 5660d5f0dSAlexey Brodkin */ 6660d5f0dSAlexey Brodkin 7660d5f0dSAlexey Brodkin #include <config.h> 8379b3280SAlexey Brodkin #include <common.h> 9ef639e6fSAlexey Brodkin #include <linux/compiler.h> 10ef639e6fSAlexey Brodkin #include <linux/kernel.h> 1197a63144SAlexey Brodkin #include <linux/log2.h> 12660d5f0dSAlexey Brodkin #include <asm/arcregs.h> 13205e7a7bSAlexey Brodkin #include <asm/cache.h> 14660d5f0dSAlexey Brodkin 15*c27814beSEugeniy Paltsev /* 16*c27814beSEugeniy Paltsev * [ NOTE 1 ]: 17*c27814beSEugeniy Paltsev * Data cache (L1 D$ or SL$) entire invalidate operation or data cache disable 18*c27814beSEugeniy Paltsev * operation may result in unexpected behavior and data loss even if we flush 19*c27814beSEugeniy Paltsev * data cache right before invalidation. That may happens if we store any context 20*c27814beSEugeniy Paltsev * on stack (like we store BLINK register on stack before function call). 21*c27814beSEugeniy Paltsev * BLINK register is the register where return address is automatically saved 22*c27814beSEugeniy Paltsev * when we do function call with instructions like 'bl'. 23*c27814beSEugeniy Paltsev * 24*c27814beSEugeniy Paltsev * There is the real example: 25*c27814beSEugeniy Paltsev * We may hang in the next code as we store any BLINK register on stack in 26*c27814beSEugeniy Paltsev * invalidate_dcache_all() function. 27*c27814beSEugeniy Paltsev * 28*c27814beSEugeniy Paltsev * void flush_dcache_all() { 29*c27814beSEugeniy Paltsev * __dc_entire_op(OP_FLUSH); 30*c27814beSEugeniy Paltsev * // Other code // 31*c27814beSEugeniy Paltsev * } 32*c27814beSEugeniy Paltsev * 33*c27814beSEugeniy Paltsev * void invalidate_dcache_all() { 34*c27814beSEugeniy Paltsev * __dc_entire_op(OP_INV); 35*c27814beSEugeniy Paltsev * // Other code // 36*c27814beSEugeniy Paltsev * } 37*c27814beSEugeniy Paltsev * 38*c27814beSEugeniy Paltsev * void foo(void) { 39*c27814beSEugeniy Paltsev * flush_dcache_all(); 40*c27814beSEugeniy Paltsev * invalidate_dcache_all(); 41*c27814beSEugeniy Paltsev * } 42*c27814beSEugeniy Paltsev * 43*c27814beSEugeniy Paltsev * Now let's see what really happens during that code execution: 44*c27814beSEugeniy Paltsev * 45*c27814beSEugeniy Paltsev * foo() 46*c27814beSEugeniy Paltsev * |->> call flush_dcache_all 47*c27814beSEugeniy Paltsev * [return address is saved to BLINK register] 48*c27814beSEugeniy Paltsev * [push BLINK] (save to stack) ![point 1] 49*c27814beSEugeniy Paltsev * |->> call __dc_entire_op(OP_FLUSH) 50*c27814beSEugeniy Paltsev * [return address is saved to BLINK register] 51*c27814beSEugeniy Paltsev * [flush L1 D$] 52*c27814beSEugeniy Paltsev * return [jump to BLINK] 53*c27814beSEugeniy Paltsev * <<------ 54*c27814beSEugeniy Paltsev * [other flush_dcache_all code] 55*c27814beSEugeniy Paltsev * [pop BLINK] (get from stack) 56*c27814beSEugeniy Paltsev * return [jump to BLINK] 57*c27814beSEugeniy Paltsev * <<------ 58*c27814beSEugeniy Paltsev * |->> call invalidate_dcache_all 59*c27814beSEugeniy Paltsev * [return address is saved to BLINK register] 60*c27814beSEugeniy Paltsev * [push BLINK] (save to stack) ![point 2] 61*c27814beSEugeniy Paltsev * |->> call __dc_entire_op(OP_FLUSH) 62*c27814beSEugeniy Paltsev * [return address is saved to BLINK register] 63*c27814beSEugeniy Paltsev * [invalidate L1 D$] ![point 3] 64*c27814beSEugeniy Paltsev * // Oops!!! 65*c27814beSEugeniy Paltsev * // We lose return address from invalidate_dcache_all function: 66*c27814beSEugeniy Paltsev * // we save it to stack and invalidate L1 D$ after that! 67*c27814beSEugeniy Paltsev * return [jump to BLINK] 68*c27814beSEugeniy Paltsev * <<------ 69*c27814beSEugeniy Paltsev * [other invalidate_dcache_all code] 70*c27814beSEugeniy Paltsev * [pop BLINK] (get from stack) 71*c27814beSEugeniy Paltsev * // we don't have this data in L1 dcache as we invalidated it in [point 3] 72*c27814beSEugeniy Paltsev * // so we get it from next memory level (for example DDR memory) 73*c27814beSEugeniy Paltsev * // but in the memory we have value which we save in [point 1], which 74*c27814beSEugeniy Paltsev * // is return address from flush_dcache_all function (instead of 75*c27814beSEugeniy Paltsev * // address from current invalidate_dcache_all function which we 76*c27814beSEugeniy Paltsev * // saved in [point 2] !) 77*c27814beSEugeniy Paltsev * return [jump to BLINK] 78*c27814beSEugeniy Paltsev * <<------ 79*c27814beSEugeniy Paltsev * // As BLINK points to invalidate_dcache_all, we call it again and 80*c27814beSEugeniy Paltsev * // loop forever. 81*c27814beSEugeniy Paltsev * 82*c27814beSEugeniy Paltsev * Fortunately we may fix that by using flush & invalidation of D$ with a single 83*c27814beSEugeniy Paltsev * one instruction (instead of flush and invalidation instructions pair) and 84*c27814beSEugeniy Paltsev * enabling force function inline with '__attribute__((always_inline))' gcc 85*c27814beSEugeniy Paltsev * attribute to avoid any function call (and BLINK store) between cache flush 86*c27814beSEugeniy Paltsev * and disable. 87*c27814beSEugeniy Paltsev */ 88*c27814beSEugeniy Paltsev 89660d5f0dSAlexey Brodkin /* Bit values in IC_CTRL */ 9019b10a42SEugeniy Paltsev #define IC_CTRL_CACHE_DISABLE BIT(0) 91660d5f0dSAlexey Brodkin 92660d5f0dSAlexey Brodkin /* Bit values in DC_CTRL */ 9319b10a42SEugeniy Paltsev #define DC_CTRL_CACHE_DISABLE BIT(0) 9419b10a42SEugeniy Paltsev #define DC_CTRL_INV_MODE_FLUSH BIT(6) 9519b10a42SEugeniy Paltsev #define DC_CTRL_FLUSH_STATUS BIT(8) 96660d5f0dSAlexey Brodkin #define CACHE_VER_NUM_MASK 0xF 97660d5f0dSAlexey Brodkin 985d7a24d6SEugeniy Paltsev #define OP_INV BIT(0) 995d7a24d6SEugeniy Paltsev #define OP_FLUSH BIT(1) 1005d7a24d6SEugeniy Paltsev #define OP_FLUSH_N_INV (OP_FLUSH | OP_INV) 101ef639e6fSAlexey Brodkin 10241cada4dSEugeniy Paltsev /* Bit val in SLC_CONTROL */ 10341cada4dSEugeniy Paltsev #define SLC_CTRL_DIS 0x001 10441cada4dSEugeniy Paltsev #define SLC_CTRL_IM 0x040 10541cada4dSEugeniy Paltsev #define SLC_CTRL_BUSY 0x100 10641cada4dSEugeniy Paltsev #define SLC_CTRL_RGN_OP_INV 0x200 10741cada4dSEugeniy Paltsev 108ef639e6fSAlexey Brodkin /* 109ef639e6fSAlexey Brodkin * By default that variable will fall into .bss section. 110ef639e6fSAlexey Brodkin * But .bss section is not relocated and so it will be initilized before 111ef639e6fSAlexey Brodkin * relocation but will be used after being zeroed. 112ef639e6fSAlexey Brodkin */ 113379b3280SAlexey Brodkin int l1_line_sz __section(".data"); 1143cf23939SEugeniy Paltsev bool dcache_exists __section(".data") = false; 1153cf23939SEugeniy Paltsev bool icache_exists __section(".data") = false; 116379b3280SAlexey Brodkin 117379b3280SAlexey Brodkin #define CACHE_LINE_MASK (~(l1_line_sz - 1)) 118379b3280SAlexey Brodkin 119379b3280SAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 120ef639e6fSAlexey Brodkin int slc_line_sz __section(".data"); 1213cf23939SEugeniy Paltsev bool slc_exists __section(".data") = false; 1223cf23939SEugeniy Paltsev bool ioc_exists __section(".data") = false; 12341cada4dSEugeniy Paltsev bool pae_exists __section(".data") = false; 124ef639e6fSAlexey Brodkin 125b0146f9eSEugeniy Paltsev /* To force enable IOC set ioc_enable to 'true' */ 126b0146f9eSEugeniy Paltsev bool ioc_enable __section(".data") = false; 127b0146f9eSEugeniy Paltsev 12841cada4dSEugeniy Paltsev void read_decode_mmu_bcr(void) 129ef639e6fSAlexey Brodkin { 13041cada4dSEugeniy Paltsev /* TODO: should we compare mmu version from BCR and from CONFIG? */ 13141cada4dSEugeniy Paltsev #if (CONFIG_ARC_MMU_VER >= 4) 13241cada4dSEugeniy Paltsev u32 tmp; 133ef639e6fSAlexey Brodkin 13441cada4dSEugeniy Paltsev tmp = read_aux_reg(ARC_AUX_MMU_BCR); 135ef639e6fSAlexey Brodkin 13641cada4dSEugeniy Paltsev struct bcr_mmu_4 { 13741cada4dSEugeniy Paltsev #ifdef CONFIG_CPU_BIG_ENDIAN 13841cada4dSEugeniy Paltsev unsigned int ver:8, sasid:1, sz1:4, sz0:4, res:2, pae:1, 13941cada4dSEugeniy Paltsev n_ways:2, n_entry:2, n_super:2, u_itlb:3, u_dtlb:3; 140ef639e6fSAlexey Brodkin #else 14141cada4dSEugeniy Paltsev /* DTLB ITLB JES JE JA */ 14241cada4dSEugeniy Paltsev unsigned int u_dtlb:3, u_itlb:3, n_super:2, n_entry:2, n_ways:2, 14341cada4dSEugeniy Paltsev pae:1, res:2, sz0:4, sz1:4, sasid:1, ver:8; 14441cada4dSEugeniy Paltsev #endif /* CONFIG_CPU_BIG_ENDIAN */ 14541cada4dSEugeniy Paltsev } *mmu4; 14641cada4dSEugeniy Paltsev 14741cada4dSEugeniy Paltsev mmu4 = (struct bcr_mmu_4 *)&tmp; 14841cada4dSEugeniy Paltsev 14941cada4dSEugeniy Paltsev pae_exists = !!mmu4->pae; 15041cada4dSEugeniy Paltsev #endif /* (CONFIG_ARC_MMU_VER >= 4) */ 15141cada4dSEugeniy Paltsev } 15241cada4dSEugeniy Paltsev 15341cada4dSEugeniy Paltsev static void __slc_entire_op(const int op) 15441cada4dSEugeniy Paltsev { 15541cada4dSEugeniy Paltsev unsigned int ctrl; 15641cada4dSEugeniy Paltsev 15741cada4dSEugeniy Paltsev ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); 15841cada4dSEugeniy Paltsev 15941cada4dSEugeniy Paltsev if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 16041cada4dSEugeniy Paltsev ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 16141cada4dSEugeniy Paltsev else 16241cada4dSEugeniy Paltsev ctrl |= SLC_CTRL_IM; 16341cada4dSEugeniy Paltsev 16441cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); 16541cada4dSEugeniy Paltsev 16641cada4dSEugeniy Paltsev if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 16741cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_INVALIDATE, 0x1); 16841cada4dSEugeniy Paltsev else 16941cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_FLUSH, 0x1); 17041cada4dSEugeniy Paltsev 17141cada4dSEugeniy Paltsev /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 17241cada4dSEugeniy Paltsev read_aux_reg(ARC_AUX_SLC_CTRL); 17341cada4dSEugeniy Paltsev 17441cada4dSEugeniy Paltsev /* Important to wait for flush to complete */ 17541cada4dSEugeniy Paltsev while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); 17641cada4dSEugeniy Paltsev } 17741cada4dSEugeniy Paltsev 17841cada4dSEugeniy Paltsev static void slc_upper_region_init(void) 17941cada4dSEugeniy Paltsev { 18041cada4dSEugeniy Paltsev /* 18141cada4dSEugeniy Paltsev * ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1 are always == 0 18241cada4dSEugeniy Paltsev * as we don't use PAE40. 18341cada4dSEugeniy Paltsev */ 18441cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_RGN_END1, 0); 18541cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_RGN_START1, 0); 18641cada4dSEugeniy Paltsev } 18741cada4dSEugeniy Paltsev 18841cada4dSEugeniy Paltsev static void __slc_rgn_op(unsigned long paddr, unsigned long sz, const int op) 18941cada4dSEugeniy Paltsev { 19041cada4dSEugeniy Paltsev unsigned int ctrl; 19141cada4dSEugeniy Paltsev unsigned long end; 19241cada4dSEugeniy Paltsev 19341cada4dSEugeniy Paltsev /* 19441cada4dSEugeniy Paltsev * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 19541cada4dSEugeniy Paltsev * - b'000 (default) is Flush, 19641cada4dSEugeniy Paltsev * - b'001 is Invalidate if CTRL.IM == 0 19741cada4dSEugeniy Paltsev * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 19841cada4dSEugeniy Paltsev */ 19941cada4dSEugeniy Paltsev ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); 20041cada4dSEugeniy Paltsev 20141cada4dSEugeniy Paltsev /* Don't rely on default value of IM bit */ 20241cada4dSEugeniy Paltsev if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 20341cada4dSEugeniy Paltsev ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 20441cada4dSEugeniy Paltsev else 20541cada4dSEugeniy Paltsev ctrl |= SLC_CTRL_IM; 20641cada4dSEugeniy Paltsev 20741cada4dSEugeniy Paltsev if (op & OP_INV) 20841cada4dSEugeniy Paltsev ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 20941cada4dSEugeniy Paltsev else 21041cada4dSEugeniy Paltsev ctrl &= ~SLC_CTRL_RGN_OP_INV; 21141cada4dSEugeniy Paltsev 21241cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); 21341cada4dSEugeniy Paltsev 21441cada4dSEugeniy Paltsev /* 21541cada4dSEugeniy Paltsev * Lower bits are ignored, no need to clip 21641cada4dSEugeniy Paltsev * END needs to be setup before START (latter triggers the operation) 21741cada4dSEugeniy Paltsev * END can't be same as START, so add (l2_line_sz - 1) to sz 21841cada4dSEugeniy Paltsev */ 21941cada4dSEugeniy Paltsev end = paddr + sz + slc_line_sz - 1; 22041cada4dSEugeniy Paltsev 22141cada4dSEugeniy Paltsev /* 22241cada4dSEugeniy Paltsev * Upper addresses (ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1) 22341cada4dSEugeniy Paltsev * are always == 0 as we don't use PAE40, so we only setup lower ones 22441cada4dSEugeniy Paltsev * (ARC_AUX_SLC_RGN_END and ARC_AUX_SLC_RGN_START) 22541cada4dSEugeniy Paltsev */ 22641cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_RGN_END, end); 22741cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_RGN_START, paddr); 22841cada4dSEugeniy Paltsev 22941cada4dSEugeniy Paltsev /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 23041cada4dSEugeniy Paltsev read_aux_reg(ARC_AUX_SLC_CTRL); 23141cada4dSEugeniy Paltsev 23241cada4dSEugeniy Paltsev while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); 23341cada4dSEugeniy Paltsev } 23441cada4dSEugeniy Paltsev #endif /* CONFIG_ISA_ARCV2 */ 235ef639e6fSAlexey Brodkin 236379b3280SAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 237379b3280SAlexey Brodkin static void read_decode_cache_bcr_arcv2(void) 238ef639e6fSAlexey Brodkin { 239379b3280SAlexey Brodkin union { 240379b3280SAlexey Brodkin struct { 241379b3280SAlexey Brodkin #ifdef CONFIG_CPU_BIG_ENDIAN 242379b3280SAlexey Brodkin unsigned int pad:24, way:2, lsz:2, sz:4; 243379b3280SAlexey Brodkin #else 244379b3280SAlexey Brodkin unsigned int sz:4, lsz:2, way:2, pad:24; 245379b3280SAlexey Brodkin #endif 246379b3280SAlexey Brodkin } fields; 247379b3280SAlexey Brodkin unsigned int word; 248379b3280SAlexey Brodkin } slc_cfg; 249379b3280SAlexey Brodkin 250379b3280SAlexey Brodkin union { 251379b3280SAlexey Brodkin struct { 252379b3280SAlexey Brodkin #ifdef CONFIG_CPU_BIG_ENDIAN 253379b3280SAlexey Brodkin unsigned int pad:24, ver:8; 254379b3280SAlexey Brodkin #else 255379b3280SAlexey Brodkin unsigned int ver:8, pad:24; 256379b3280SAlexey Brodkin #endif 257379b3280SAlexey Brodkin } fields; 258379b3280SAlexey Brodkin unsigned int word; 259379b3280SAlexey Brodkin } sbcr; 260379b3280SAlexey Brodkin 261379b3280SAlexey Brodkin sbcr.word = read_aux_reg(ARC_BCR_SLC); 262379b3280SAlexey Brodkin if (sbcr.fields.ver) { 263379b3280SAlexey Brodkin slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG); 2643cf23939SEugeniy Paltsev slc_exists = true; 265379b3280SAlexey Brodkin slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64; 266379b3280SAlexey Brodkin } 267db6ce231SAlexey Brodkin 268db6ce231SAlexey Brodkin union { 269db6ce231SAlexey Brodkin struct bcr_clust_cfg { 270db6ce231SAlexey Brodkin #ifdef CONFIG_CPU_BIG_ENDIAN 271db6ce231SAlexey Brodkin unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 272db6ce231SAlexey Brodkin #else 273db6ce231SAlexey Brodkin unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 274db6ce231SAlexey Brodkin #endif 275db6ce231SAlexey Brodkin } fields; 276db6ce231SAlexey Brodkin unsigned int word; 277db6ce231SAlexey Brodkin } cbcr; 278db6ce231SAlexey Brodkin 279db6ce231SAlexey Brodkin cbcr.word = read_aux_reg(ARC_BCR_CLUSTER); 280b0146f9eSEugeniy Paltsev if (cbcr.fields.c && ioc_enable) 2813cf23939SEugeniy Paltsev ioc_exists = true; 282379b3280SAlexey Brodkin } 283379b3280SAlexey Brodkin #endif 284379b3280SAlexey Brodkin 285379b3280SAlexey Brodkin void read_decode_cache_bcr(void) 286379b3280SAlexey Brodkin { 287379b3280SAlexey Brodkin int dc_line_sz = 0, ic_line_sz = 0; 288379b3280SAlexey Brodkin 289379b3280SAlexey Brodkin union { 290379b3280SAlexey Brodkin struct { 291379b3280SAlexey Brodkin #ifdef CONFIG_CPU_BIG_ENDIAN 292379b3280SAlexey Brodkin unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 293379b3280SAlexey Brodkin #else 294379b3280SAlexey Brodkin unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 295379b3280SAlexey Brodkin #endif 296379b3280SAlexey Brodkin } fields; 297379b3280SAlexey Brodkin unsigned int word; 298379b3280SAlexey Brodkin } ibcr, dbcr; 299379b3280SAlexey Brodkin 300379b3280SAlexey Brodkin ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); 301379b3280SAlexey Brodkin if (ibcr.fields.ver) { 3023cf23939SEugeniy Paltsev icache_exists = true; 303379b3280SAlexey Brodkin l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len; 304379b3280SAlexey Brodkin if (!ic_line_sz) 305379b3280SAlexey Brodkin panic("Instruction exists but line length is 0\n"); 306ef639e6fSAlexey Brodkin } 307ef639e6fSAlexey Brodkin 308379b3280SAlexey Brodkin dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); 309379b3280SAlexey Brodkin if (dbcr.fields.ver) { 3103cf23939SEugeniy Paltsev dcache_exists = true; 311379b3280SAlexey Brodkin l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len; 312379b3280SAlexey Brodkin if (!dc_line_sz) 313379b3280SAlexey Brodkin panic("Data cache exists but line length is 0\n"); 314379b3280SAlexey Brodkin } 315379b3280SAlexey Brodkin 316379b3280SAlexey Brodkin if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz)) 317379b3280SAlexey Brodkin panic("Instruction and data cache line lengths differ\n"); 318ef639e6fSAlexey Brodkin } 319ef639e6fSAlexey Brodkin 320ef639e6fSAlexey Brodkin void cache_init(void) 321ef639e6fSAlexey Brodkin { 322379b3280SAlexey Brodkin read_decode_cache_bcr(); 323379b3280SAlexey Brodkin 324ef639e6fSAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 325379b3280SAlexey Brodkin read_decode_cache_bcr_arcv2(); 326db6ce231SAlexey Brodkin 327db6ce231SAlexey Brodkin if (ioc_exists) { 32897a63144SAlexey Brodkin /* IOC Aperture start is equal to DDR start */ 32997a63144SAlexey Brodkin unsigned int ap_base = CONFIG_SYS_SDRAM_BASE; 33097a63144SAlexey Brodkin /* IOC Aperture size is equal to DDR size */ 33197a63144SAlexey Brodkin long ap_size = CONFIG_SYS_SDRAM_SIZE; 33297a63144SAlexey Brodkin 333*c27814beSEugeniy Paltsev flush_n_invalidate_dcache_all(); 334a4a43fcfSAlexey Brodkin 33597a63144SAlexey Brodkin if (!is_power_of_2(ap_size) || ap_size < 4096) 33697a63144SAlexey Brodkin panic("IOC Aperture size must be power of 2 and bigger 4Kib"); 33797a63144SAlexey Brodkin 33897a63144SAlexey Brodkin /* 33997a63144SAlexey Brodkin * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB, 34097a63144SAlexey Brodkin * so setting 0x11 implies 512M, 0x12 implies 1G... 34197a63144SAlexey Brodkin */ 34297a63144SAlexey Brodkin write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, 34397a63144SAlexey Brodkin order_base_2(ap_size / 1024) - 2); 34497a63144SAlexey Brodkin 34597a63144SAlexey Brodkin /* IOC Aperture start must be aligned to the size of the aperture */ 34697a63144SAlexey Brodkin if (ap_base % ap_size != 0) 34797a63144SAlexey Brodkin panic("IOC Aperture start must be aligned to the size of the aperture"); 34897a63144SAlexey Brodkin 34997a63144SAlexey Brodkin write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, ap_base >> 12); 350db6ce231SAlexey Brodkin write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1); 351db6ce231SAlexey Brodkin write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1); 352db6ce231SAlexey Brodkin } 35341cada4dSEugeniy Paltsev 35441cada4dSEugeniy Paltsev read_decode_mmu_bcr(); 35541cada4dSEugeniy Paltsev 35641cada4dSEugeniy Paltsev /* 35741cada4dSEugeniy Paltsev * ARC_AUX_SLC_RGN_START1 and ARC_AUX_SLC_RGN_END1 register exist 35841cada4dSEugeniy Paltsev * only if PAE exists in current HW. So we had to check pae_exist 35941cada4dSEugeniy Paltsev * before using them. 36041cada4dSEugeniy Paltsev */ 36141cada4dSEugeniy Paltsev if (slc_exists && pae_exists) 36241cada4dSEugeniy Paltsev slc_upper_region_init(); 36341cada4dSEugeniy Paltsev #endif /* CONFIG_ISA_ARCV2 */ 364ef639e6fSAlexey Brodkin } 365ef639e6fSAlexey Brodkin 366660d5f0dSAlexey Brodkin int icache_status(void) 367660d5f0dSAlexey Brodkin { 368379b3280SAlexey Brodkin if (!icache_exists) 369660d5f0dSAlexey Brodkin return 0; 370660d5f0dSAlexey Brodkin 371ef639e6fSAlexey Brodkin if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE) 372ef639e6fSAlexey Brodkin return 0; 373ef639e6fSAlexey Brodkin else 374ef639e6fSAlexey Brodkin return 1; 375660d5f0dSAlexey Brodkin } 376660d5f0dSAlexey Brodkin 377660d5f0dSAlexey Brodkin void icache_enable(void) 378660d5f0dSAlexey Brodkin { 379379b3280SAlexey Brodkin if (icache_exists) 380660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) & 381660d5f0dSAlexey Brodkin ~IC_CTRL_CACHE_DISABLE); 382660d5f0dSAlexey Brodkin } 383660d5f0dSAlexey Brodkin 384660d5f0dSAlexey Brodkin void icache_disable(void) 385660d5f0dSAlexey Brodkin { 386379b3280SAlexey Brodkin if (icache_exists) 387660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) | 388660d5f0dSAlexey Brodkin IC_CTRL_CACHE_DISABLE); 389660d5f0dSAlexey Brodkin } 390660d5f0dSAlexey Brodkin 39116aeee81SEugeniy Paltsev /* IC supports only invalidation */ 39216aeee81SEugeniy Paltsev static inline void __ic_entire_invalidate(void) 393660d5f0dSAlexey Brodkin { 39416aeee81SEugeniy Paltsev if (!icache_status()) 39516aeee81SEugeniy Paltsev return; 39616aeee81SEugeniy Paltsev 397660d5f0dSAlexey Brodkin /* Any write to IC_IVIC register triggers invalidation of entire I$ */ 398660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_IC_IVIC, 1); 399f2a22678SAlexey Brodkin /* 400f2a22678SAlexey Brodkin * As per ARC HS databook (see chapter 5.3.3.2) 401f2a22678SAlexey Brodkin * it is required to add 3 NOPs after each write to IC_IVIC. 402f2a22678SAlexey Brodkin */ 403f2a22678SAlexey Brodkin __builtin_arc_nop(); 404f2a22678SAlexey Brodkin __builtin_arc_nop(); 405f2a22678SAlexey Brodkin __builtin_arc_nop(); 406ef639e6fSAlexey Brodkin read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */ 407660d5f0dSAlexey Brodkin } 40841cada4dSEugeniy Paltsev 40916aeee81SEugeniy Paltsev void invalidate_icache_all(void) 41016aeee81SEugeniy Paltsev { 41116aeee81SEugeniy Paltsev __ic_entire_invalidate(); 41216aeee81SEugeniy Paltsev 41341cada4dSEugeniy Paltsev #ifdef CONFIG_ISA_ARCV2 41441cada4dSEugeniy Paltsev if (slc_exists) 41541cada4dSEugeniy Paltsev __slc_entire_op(OP_INV); 416ef639e6fSAlexey Brodkin #endif 41741cada4dSEugeniy Paltsev } 418660d5f0dSAlexey Brodkin 419660d5f0dSAlexey Brodkin int dcache_status(void) 420660d5f0dSAlexey Brodkin { 421379b3280SAlexey Brodkin if (!dcache_exists) 422660d5f0dSAlexey Brodkin return 0; 423660d5f0dSAlexey Brodkin 424ef639e6fSAlexey Brodkin if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE) 425ef639e6fSAlexey Brodkin return 0; 426ef639e6fSAlexey Brodkin else 427ef639e6fSAlexey Brodkin return 1; 428660d5f0dSAlexey Brodkin } 429660d5f0dSAlexey Brodkin 430660d5f0dSAlexey Brodkin void dcache_enable(void) 431660d5f0dSAlexey Brodkin { 432379b3280SAlexey Brodkin if (!dcache_exists) 433660d5f0dSAlexey Brodkin return; 434660d5f0dSAlexey Brodkin 435660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) & 436660d5f0dSAlexey Brodkin ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE)); 437660d5f0dSAlexey Brodkin } 438660d5f0dSAlexey Brodkin 439660d5f0dSAlexey Brodkin void dcache_disable(void) 440660d5f0dSAlexey Brodkin { 441379b3280SAlexey Brodkin if (!dcache_exists) 442660d5f0dSAlexey Brodkin return; 443660d5f0dSAlexey Brodkin 444660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) | 445660d5f0dSAlexey Brodkin DC_CTRL_CACHE_DISABLE); 446660d5f0dSAlexey Brodkin } 447660d5f0dSAlexey Brodkin 448660d5f0dSAlexey Brodkin #ifndef CONFIG_SYS_DCACHE_OFF 449c4ef14d2SEugeniy Paltsev /* Common Helper for Line Operations on D-cache */ 450c4ef14d2SEugeniy Paltsev static inline void __dcache_line_loop(unsigned long paddr, unsigned long sz, 451ef639e6fSAlexey Brodkin const int cacheop) 452660d5f0dSAlexey Brodkin { 453ef639e6fSAlexey Brodkin unsigned int aux_cmd; 454ef639e6fSAlexey Brodkin int num_lines; 455660d5f0dSAlexey Brodkin 456ef639e6fSAlexey Brodkin /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 457ef639e6fSAlexey Brodkin aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL; 458660d5f0dSAlexey Brodkin 459ef639e6fSAlexey Brodkin sz += paddr & ~CACHE_LINE_MASK; 460ef639e6fSAlexey Brodkin paddr &= CACHE_LINE_MASK; 461ef639e6fSAlexey Brodkin 462379b3280SAlexey Brodkin num_lines = DIV_ROUND_UP(sz, l1_line_sz); 463ef639e6fSAlexey Brodkin 464ef639e6fSAlexey Brodkin while (num_lines-- > 0) { 465ef639e6fSAlexey Brodkin #if (CONFIG_ARC_MMU_VER == 3) 466c4ef14d2SEugeniy Paltsev write_aux_reg(ARC_AUX_DC_PTAG, paddr); 467ef639e6fSAlexey Brodkin #endif 468ef639e6fSAlexey Brodkin write_aux_reg(aux_cmd, paddr); 469379b3280SAlexey Brodkin paddr += l1_line_sz; 470ef639e6fSAlexey Brodkin } 471ef639e6fSAlexey Brodkin } 472ef639e6fSAlexey Brodkin 4735d7a24d6SEugeniy Paltsev static void __before_dc_op(const int op) 474ef639e6fSAlexey Brodkin { 4755d7a24d6SEugeniy Paltsev unsigned int ctrl; 476ef639e6fSAlexey Brodkin 4775d7a24d6SEugeniy Paltsev ctrl = read_aux_reg(ARC_AUX_DC_CTRL); 4785d7a24d6SEugeniy Paltsev 4795d7a24d6SEugeniy Paltsev /* IM bit implies flush-n-inv, instead of vanilla inv */ 4805d7a24d6SEugeniy Paltsev if (op == OP_INV) 4815d7a24d6SEugeniy Paltsev ctrl &= ~DC_CTRL_INV_MODE_FLUSH; 4825d7a24d6SEugeniy Paltsev else 4835d7a24d6SEugeniy Paltsev ctrl |= DC_CTRL_INV_MODE_FLUSH; 4845d7a24d6SEugeniy Paltsev 4855d7a24d6SEugeniy Paltsev write_aux_reg(ARC_AUX_DC_CTRL, ctrl); 486ef639e6fSAlexey Brodkin } 487ef639e6fSAlexey Brodkin 4885d7a24d6SEugeniy Paltsev static void __after_dc_op(const int op) 489ef639e6fSAlexey Brodkin { 490ef639e6fSAlexey Brodkin if (op & OP_FLUSH) /* flush / flush-n-inv both wait */ 49119b10a42SEugeniy Paltsev while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS); 492ef639e6fSAlexey Brodkin } 493ef639e6fSAlexey Brodkin 494ef639e6fSAlexey Brodkin static inline void __dc_entire_op(const int cacheop) 495ef639e6fSAlexey Brodkin { 496ef639e6fSAlexey Brodkin int aux; 4975d7a24d6SEugeniy Paltsev 4985d7a24d6SEugeniy Paltsev __before_dc_op(cacheop); 499ef639e6fSAlexey Brodkin 500ef639e6fSAlexey Brodkin if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 501ef639e6fSAlexey Brodkin aux = ARC_AUX_DC_IVDC; 502ef639e6fSAlexey Brodkin else 503ef639e6fSAlexey Brodkin aux = ARC_AUX_DC_FLSH; 504ef639e6fSAlexey Brodkin 505ef639e6fSAlexey Brodkin write_aux_reg(aux, 0x1); 506ef639e6fSAlexey Brodkin 5075d7a24d6SEugeniy Paltsev __after_dc_op(cacheop); 508ef639e6fSAlexey Brodkin } 509ef639e6fSAlexey Brodkin 510ef639e6fSAlexey Brodkin static inline void __dc_line_op(unsigned long paddr, unsigned long sz, 511ef639e6fSAlexey Brodkin const int cacheop) 512ef639e6fSAlexey Brodkin { 5135d7a24d6SEugeniy Paltsev __before_dc_op(cacheop); 514c4ef14d2SEugeniy Paltsev __dcache_line_loop(paddr, sz, cacheop); 5155d7a24d6SEugeniy Paltsev __after_dc_op(cacheop); 516ef639e6fSAlexey Brodkin } 517ef639e6fSAlexey Brodkin #else 518ef639e6fSAlexey Brodkin #define __dc_entire_op(cacheop) 519ef639e6fSAlexey Brodkin #define __dc_line_op(paddr, sz, cacheop) 520ef639e6fSAlexey Brodkin #endif /* !CONFIG_SYS_DCACHE_OFF */ 521ef639e6fSAlexey Brodkin 522660d5f0dSAlexey Brodkin void invalidate_dcache_range(unsigned long start, unsigned long end) 523660d5f0dSAlexey Brodkin { 52441cada4dSEugeniy Paltsev if (start >= end) 52541cada4dSEugeniy Paltsev return; 52641cada4dSEugeniy Paltsev 527ef639e6fSAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 528db6ce231SAlexey Brodkin if (!ioc_exists) 529db6ce231SAlexey Brodkin #endif 530db6ce231SAlexey Brodkin __dc_line_op(start, end - start, OP_INV); 531db6ce231SAlexey Brodkin 532db6ce231SAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 533db6ce231SAlexey Brodkin if (slc_exists && !ioc_exists) 53441cada4dSEugeniy Paltsev __slc_rgn_op(start, end - start, OP_INV); 535660d5f0dSAlexey Brodkin #endif 536660d5f0dSAlexey Brodkin } 537660d5f0dSAlexey Brodkin 538ef639e6fSAlexey Brodkin void flush_dcache_range(unsigned long start, unsigned long end) 539660d5f0dSAlexey Brodkin { 54041cada4dSEugeniy Paltsev if (start >= end) 54141cada4dSEugeniy Paltsev return; 54241cada4dSEugeniy Paltsev 543ef639e6fSAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 544db6ce231SAlexey Brodkin if (!ioc_exists) 545db6ce231SAlexey Brodkin #endif 546db6ce231SAlexey Brodkin __dc_line_op(start, end - start, OP_FLUSH); 547db6ce231SAlexey Brodkin 548db6ce231SAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 549db6ce231SAlexey Brodkin if (slc_exists && !ioc_exists) 55041cada4dSEugeniy Paltsev __slc_rgn_op(start, end - start, OP_FLUSH); 551ef639e6fSAlexey Brodkin #endif 552660d5f0dSAlexey Brodkin } 553660d5f0dSAlexey Brodkin 554660d5f0dSAlexey Brodkin void flush_cache(unsigned long start, unsigned long size) 555660d5f0dSAlexey Brodkin { 556660d5f0dSAlexey Brodkin flush_dcache_range(start, start + size); 557660d5f0dSAlexey Brodkin } 5586eb15e50SAlexey Brodkin 559*c27814beSEugeniy Paltsev /* 560*c27814beSEugeniy Paltsev * As invalidate_dcache_all() is not used in generic U-Boot code and as we 561*c27814beSEugeniy Paltsev * don't need it in arch/arc code alone (invalidate without flush) we implement 562*c27814beSEugeniy Paltsev * flush_n_invalidate_dcache_all (flush and invalidate in 1 operation) because 563*c27814beSEugeniy Paltsev * it's much safer. See [ NOTE 1 ] for more details. 564*c27814beSEugeniy Paltsev */ 565*c27814beSEugeniy Paltsev void flush_n_invalidate_dcache_all(void) 566ef639e6fSAlexey Brodkin { 567*c27814beSEugeniy Paltsev __dc_entire_op(OP_FLUSH_N_INV); 568db6ce231SAlexey Brodkin 569db6ce231SAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 570bd91508bSAlexey Brodkin if (slc_exists) 571*c27814beSEugeniy Paltsev __slc_entire_op(OP_FLUSH_N_INV); 572ef639e6fSAlexey Brodkin #endif 5736eb15e50SAlexey Brodkin } 5746eb15e50SAlexey Brodkin 575ef639e6fSAlexey Brodkin void flush_dcache_all(void) 5766eb15e50SAlexey Brodkin { 577db6ce231SAlexey Brodkin __dc_entire_op(OP_FLUSH); 578db6ce231SAlexey Brodkin 579db6ce231SAlexey Brodkin #ifdef CONFIG_ISA_ARCV2 5802a8382c6SAlexey Brodkin if (slc_exists) 581ef639e6fSAlexey Brodkin __slc_entire_op(OP_FLUSH); 582ef639e6fSAlexey Brodkin #endif 5836eb15e50SAlexey Brodkin } 584