1660d5f0dSAlexey Brodkin /* 2660d5f0dSAlexey Brodkin * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved. 3660d5f0dSAlexey Brodkin * 4660d5f0dSAlexey Brodkin * SPDX-License-Identifier: GPL-2.0+ 5660d5f0dSAlexey Brodkin */ 6660d5f0dSAlexey Brodkin 7660d5f0dSAlexey Brodkin #include <config.h> 8379b3280SAlexey Brodkin #include <common.h> 9ef639e6fSAlexey Brodkin #include <linux/compiler.h> 10ef639e6fSAlexey Brodkin #include <linux/kernel.h> 1197a63144SAlexey Brodkin #include <linux/log2.h> 12660d5f0dSAlexey Brodkin #include <asm/arcregs.h> 1388ae27edSEugeniy Paltsev #include <asm/arc-bcr.h> 14205e7a7bSAlexey Brodkin #include <asm/cache.h> 15660d5f0dSAlexey Brodkin 16c27814beSEugeniy Paltsev /* 17c27814beSEugeniy Paltsev * [ NOTE 1 ]: 18c27814beSEugeniy Paltsev * Data cache (L1 D$ or SL$) entire invalidate operation or data cache disable 19c27814beSEugeniy Paltsev * operation may result in unexpected behavior and data loss even if we flush 20c27814beSEugeniy Paltsev * data cache right before invalidation. That may happens if we store any context 21c27814beSEugeniy Paltsev * on stack (like we store BLINK register on stack before function call). 22c27814beSEugeniy Paltsev * BLINK register is the register where return address is automatically saved 23c27814beSEugeniy Paltsev * when we do function call with instructions like 'bl'. 24c27814beSEugeniy Paltsev * 25c27814beSEugeniy Paltsev * There is the real example: 26c27814beSEugeniy Paltsev * We may hang in the next code as we store any BLINK register on stack in 27c27814beSEugeniy Paltsev * invalidate_dcache_all() function. 28c27814beSEugeniy Paltsev * 29c27814beSEugeniy Paltsev * void flush_dcache_all() { 30c27814beSEugeniy Paltsev * __dc_entire_op(OP_FLUSH); 31c27814beSEugeniy Paltsev * // Other code // 32c27814beSEugeniy Paltsev * } 33c27814beSEugeniy Paltsev * 34c27814beSEugeniy Paltsev * void invalidate_dcache_all() { 35c27814beSEugeniy Paltsev * __dc_entire_op(OP_INV); 36c27814beSEugeniy Paltsev * // Other code // 37c27814beSEugeniy Paltsev * } 38c27814beSEugeniy Paltsev * 39c27814beSEugeniy Paltsev * void foo(void) { 40c27814beSEugeniy Paltsev * flush_dcache_all(); 41c27814beSEugeniy Paltsev * invalidate_dcache_all(); 42c27814beSEugeniy Paltsev * } 43c27814beSEugeniy Paltsev * 44c27814beSEugeniy Paltsev * Now let's see what really happens during that code execution: 45c27814beSEugeniy Paltsev * 46c27814beSEugeniy Paltsev * foo() 47c27814beSEugeniy Paltsev * |->> call flush_dcache_all 48c27814beSEugeniy Paltsev * [return address is saved to BLINK register] 49c27814beSEugeniy Paltsev * [push BLINK] (save to stack) ![point 1] 50c27814beSEugeniy Paltsev * |->> call __dc_entire_op(OP_FLUSH) 51c27814beSEugeniy Paltsev * [return address is saved to BLINK register] 52c27814beSEugeniy Paltsev * [flush L1 D$] 53c27814beSEugeniy Paltsev * return [jump to BLINK] 54c27814beSEugeniy Paltsev * <<------ 55c27814beSEugeniy Paltsev * [other flush_dcache_all code] 56c27814beSEugeniy Paltsev * [pop BLINK] (get from stack) 57c27814beSEugeniy Paltsev * return [jump to BLINK] 58c27814beSEugeniy Paltsev * <<------ 59c27814beSEugeniy Paltsev * |->> call invalidate_dcache_all 60c27814beSEugeniy Paltsev * [return address is saved to BLINK register] 61c27814beSEugeniy Paltsev * [push BLINK] (save to stack) ![point 2] 62c27814beSEugeniy Paltsev * |->> call __dc_entire_op(OP_FLUSH) 63c27814beSEugeniy Paltsev * [return address is saved to BLINK register] 64c27814beSEugeniy Paltsev * [invalidate L1 D$] ![point 3] 65c27814beSEugeniy Paltsev * // Oops!!! 66c27814beSEugeniy Paltsev * // We lose return address from invalidate_dcache_all function: 67c27814beSEugeniy Paltsev * // we save it to stack and invalidate L1 D$ after that! 68c27814beSEugeniy Paltsev * return [jump to BLINK] 69c27814beSEugeniy Paltsev * <<------ 70c27814beSEugeniy Paltsev * [other invalidate_dcache_all code] 71c27814beSEugeniy Paltsev * [pop BLINK] (get from stack) 72c27814beSEugeniy Paltsev * // we don't have this data in L1 dcache as we invalidated it in [point 3] 73c27814beSEugeniy Paltsev * // so we get it from next memory level (for example DDR memory) 74c27814beSEugeniy Paltsev * // but in the memory we have value which we save in [point 1], which 75c27814beSEugeniy Paltsev * // is return address from flush_dcache_all function (instead of 76c27814beSEugeniy Paltsev * // address from current invalidate_dcache_all function which we 77c27814beSEugeniy Paltsev * // saved in [point 2] !) 78c27814beSEugeniy Paltsev * return [jump to BLINK] 79c27814beSEugeniy Paltsev * <<------ 80c27814beSEugeniy Paltsev * // As BLINK points to invalidate_dcache_all, we call it again and 81c27814beSEugeniy Paltsev * // loop forever. 82c27814beSEugeniy Paltsev * 83c27814beSEugeniy Paltsev * Fortunately we may fix that by using flush & invalidation of D$ with a single 84c27814beSEugeniy Paltsev * one instruction (instead of flush and invalidation instructions pair) and 85c27814beSEugeniy Paltsev * enabling force function inline with '__attribute__((always_inline))' gcc 86c27814beSEugeniy Paltsev * attribute to avoid any function call (and BLINK store) between cache flush 87c27814beSEugeniy Paltsev * and disable. 88*7241944aSEugeniy Paltsev * 89*7241944aSEugeniy Paltsev * 90*7241944aSEugeniy Paltsev * [ NOTE 2 ]: 91*7241944aSEugeniy Paltsev * As of today we only support the following cache configurations on ARC. 92*7241944aSEugeniy Paltsev * Other configurations may exist in HW (for example, since version 3.0 HS 93*7241944aSEugeniy Paltsev * supports SL$ (L2 system level cache) disable) but we don't support it in SW. 94*7241944aSEugeniy Paltsev * Configuration 1: 95*7241944aSEugeniy Paltsev * ______________________ 96*7241944aSEugeniy Paltsev * | | 97*7241944aSEugeniy Paltsev * | ARC CPU | 98*7241944aSEugeniy Paltsev * |______________________| 99*7241944aSEugeniy Paltsev * ___|___ ___|___ 100*7241944aSEugeniy Paltsev * | | | | 101*7241944aSEugeniy Paltsev * | L1 I$ | | L1 D$ | 102*7241944aSEugeniy Paltsev * |_______| |_______| 103*7241944aSEugeniy Paltsev * on/off on/off 104*7241944aSEugeniy Paltsev * ___|______________|____ 105*7241944aSEugeniy Paltsev * | | 106*7241944aSEugeniy Paltsev * | main memory | 107*7241944aSEugeniy Paltsev * |______________________| 108*7241944aSEugeniy Paltsev * 109*7241944aSEugeniy Paltsev * Configuration 2: 110*7241944aSEugeniy Paltsev * ______________________ 111*7241944aSEugeniy Paltsev * | | 112*7241944aSEugeniy Paltsev * | ARC CPU | 113*7241944aSEugeniy Paltsev * |______________________| 114*7241944aSEugeniy Paltsev * ___|___ ___|___ 115*7241944aSEugeniy Paltsev * | | | | 116*7241944aSEugeniy Paltsev * | L1 I$ | | L1 D$ | 117*7241944aSEugeniy Paltsev * |_______| |_______| 118*7241944aSEugeniy Paltsev * on/off on/off 119*7241944aSEugeniy Paltsev * ___|______________|____ 120*7241944aSEugeniy Paltsev * | | 121*7241944aSEugeniy Paltsev * | L2 (SL$) | 122*7241944aSEugeniy Paltsev * |______________________| 123*7241944aSEugeniy Paltsev * always must be on 124*7241944aSEugeniy Paltsev * ___|______________|____ 125*7241944aSEugeniy Paltsev * | | 126*7241944aSEugeniy Paltsev * | main memory | 127*7241944aSEugeniy Paltsev * |______________________| 128*7241944aSEugeniy Paltsev * 129*7241944aSEugeniy Paltsev * Configuration 3: 130*7241944aSEugeniy Paltsev * ______________________ 131*7241944aSEugeniy Paltsev * | | 132*7241944aSEugeniy Paltsev * | ARC CPU | 133*7241944aSEugeniy Paltsev * |______________________| 134*7241944aSEugeniy Paltsev * ___|___ ___|___ 135*7241944aSEugeniy Paltsev * | | | | 136*7241944aSEugeniy Paltsev * | L1 I$ | | L1 D$ | 137*7241944aSEugeniy Paltsev * |_______| |_______| 138*7241944aSEugeniy Paltsev * on/off must be on 139*7241944aSEugeniy Paltsev * ___|______________|____ _______ 140*7241944aSEugeniy Paltsev * | | | | 141*7241944aSEugeniy Paltsev * | L2 (SL$) |-----| IOC | 142*7241944aSEugeniy Paltsev * |______________________| |_______| 143*7241944aSEugeniy Paltsev * always must be on on/off 144*7241944aSEugeniy Paltsev * ___|______________|____ 145*7241944aSEugeniy Paltsev * | | 146*7241944aSEugeniy Paltsev * | main memory | 147*7241944aSEugeniy Paltsev * |______________________| 148c27814beSEugeniy Paltsev */ 149c27814beSEugeniy Paltsev 150bf8974edSEugeniy Paltsev DECLARE_GLOBAL_DATA_PTR; 151bf8974edSEugeniy Paltsev 152660d5f0dSAlexey Brodkin /* Bit values in IC_CTRL */ 15319b10a42SEugeniy Paltsev #define IC_CTRL_CACHE_DISABLE BIT(0) 154660d5f0dSAlexey Brodkin 155660d5f0dSAlexey Brodkin /* Bit values in DC_CTRL */ 15619b10a42SEugeniy Paltsev #define DC_CTRL_CACHE_DISABLE BIT(0) 15719b10a42SEugeniy Paltsev #define DC_CTRL_INV_MODE_FLUSH BIT(6) 15819b10a42SEugeniy Paltsev #define DC_CTRL_FLUSH_STATUS BIT(8) 159660d5f0dSAlexey Brodkin 1605d7a24d6SEugeniy Paltsev #define OP_INV BIT(0) 1615d7a24d6SEugeniy Paltsev #define OP_FLUSH BIT(1) 1625d7a24d6SEugeniy Paltsev #define OP_FLUSH_N_INV (OP_FLUSH | OP_INV) 163ef639e6fSAlexey Brodkin 16441cada4dSEugeniy Paltsev /* Bit val in SLC_CONTROL */ 16541cada4dSEugeniy Paltsev #define SLC_CTRL_DIS 0x001 16641cada4dSEugeniy Paltsev #define SLC_CTRL_IM 0x040 16741cada4dSEugeniy Paltsev #define SLC_CTRL_BUSY 0x100 16841cada4dSEugeniy Paltsev #define SLC_CTRL_RGN_OP_INV 0x200 16941cada4dSEugeniy Paltsev 170bf8974edSEugeniy Paltsev #define CACHE_LINE_MASK (~(gd->arch.l1_line_sz - 1)) 171379b3280SAlexey Brodkin 17275790873SEugeniy Paltsev static inline bool pae_exists(void) 173ef639e6fSAlexey Brodkin { 17441cada4dSEugeniy Paltsev /* TODO: should we compare mmu version from BCR and from CONFIG? */ 17541cada4dSEugeniy Paltsev #if (CONFIG_ARC_MMU_VER >= 4) 17688ae27edSEugeniy Paltsev union bcr_mmu_4 mmu4; 177ef639e6fSAlexey Brodkin 17888ae27edSEugeniy Paltsev mmu4.word = read_aux_reg(ARC_AUX_MMU_BCR); 179ef639e6fSAlexey Brodkin 18075790873SEugeniy Paltsev if (mmu4.fields.pae) 18175790873SEugeniy Paltsev return true; 18241cada4dSEugeniy Paltsev #endif /* (CONFIG_ARC_MMU_VER >= 4) */ 18375790873SEugeniy Paltsev 18475790873SEugeniy Paltsev return false; 18575790873SEugeniy Paltsev } 18675790873SEugeniy Paltsev 18775790873SEugeniy Paltsev static inline bool icache_exists(void) 18875790873SEugeniy Paltsev { 18975790873SEugeniy Paltsev union bcr_di_cache ibcr; 19075790873SEugeniy Paltsev 19175790873SEugeniy Paltsev ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); 19275790873SEugeniy Paltsev return !!ibcr.fields.ver; 19375790873SEugeniy Paltsev } 19475790873SEugeniy Paltsev 195c75eeb0bSEugeniy Paltsev static inline bool icache_enabled(void) 196c75eeb0bSEugeniy Paltsev { 197c75eeb0bSEugeniy Paltsev if (!icache_exists()) 198c75eeb0bSEugeniy Paltsev return false; 199c75eeb0bSEugeniy Paltsev 200c75eeb0bSEugeniy Paltsev return !(read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE); 201c75eeb0bSEugeniy Paltsev } 202c75eeb0bSEugeniy Paltsev 20375790873SEugeniy Paltsev static inline bool dcache_exists(void) 20475790873SEugeniy Paltsev { 20575790873SEugeniy Paltsev union bcr_di_cache dbcr; 20675790873SEugeniy Paltsev 20775790873SEugeniy Paltsev dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); 20875790873SEugeniy Paltsev return !!dbcr.fields.ver; 20975790873SEugeniy Paltsev } 21075790873SEugeniy Paltsev 211c75eeb0bSEugeniy Paltsev static inline bool dcache_enabled(void) 212c75eeb0bSEugeniy Paltsev { 213c75eeb0bSEugeniy Paltsev if (!dcache_exists()) 214c75eeb0bSEugeniy Paltsev return false; 215c75eeb0bSEugeniy Paltsev 216c75eeb0bSEugeniy Paltsev return !(read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE); 217c75eeb0bSEugeniy Paltsev } 218c75eeb0bSEugeniy Paltsev 21975790873SEugeniy Paltsev static inline bool slc_exists(void) 22075790873SEugeniy Paltsev { 22175790873SEugeniy Paltsev if (is_isa_arcv2()) { 22275790873SEugeniy Paltsev union bcr_generic sbcr; 22375790873SEugeniy Paltsev 22475790873SEugeniy Paltsev sbcr.word = read_aux_reg(ARC_BCR_SLC); 22575790873SEugeniy Paltsev return !!sbcr.fields.ver; 22675790873SEugeniy Paltsev } 22775790873SEugeniy Paltsev 22875790873SEugeniy Paltsev return false; 22941cada4dSEugeniy Paltsev } 23041cada4dSEugeniy Paltsev 23195336738SEugeniy Paltsev static inline bool slc_data_bypass(void) 23295336738SEugeniy Paltsev { 23395336738SEugeniy Paltsev /* 23495336738SEugeniy Paltsev * If L1 data cache is disabled SL$ is bypassed and all load/store 23595336738SEugeniy Paltsev * requests are sent directly to main memory. 23695336738SEugeniy Paltsev */ 23795336738SEugeniy Paltsev return !dcache_enabled(); 23895336738SEugeniy Paltsev } 23995336738SEugeniy Paltsev 24048b04832SEugeniy Paltsev static inline bool ioc_exists(void) 24148b04832SEugeniy Paltsev { 24248b04832SEugeniy Paltsev if (is_isa_arcv2()) { 24348b04832SEugeniy Paltsev union bcr_clust_cfg cbcr; 24448b04832SEugeniy Paltsev 24548b04832SEugeniy Paltsev cbcr.word = read_aux_reg(ARC_BCR_CLUSTER); 24648b04832SEugeniy Paltsev return cbcr.fields.c; 24748b04832SEugeniy Paltsev } 24848b04832SEugeniy Paltsev 24948b04832SEugeniy Paltsev return false; 25048b04832SEugeniy Paltsev } 25148b04832SEugeniy Paltsev 25248b04832SEugeniy Paltsev static inline bool ioc_enabled(void) 25348b04832SEugeniy Paltsev { 25448b04832SEugeniy Paltsev /* 25548b04832SEugeniy Paltsev * We check only CONFIG option instead of IOC HW state check as IOC 25648b04832SEugeniy Paltsev * must be disabled by default. 25748b04832SEugeniy Paltsev */ 25848b04832SEugeniy Paltsev if (is_ioc_enabled()) 25948b04832SEugeniy Paltsev return ioc_exists(); 26048b04832SEugeniy Paltsev 26148b04832SEugeniy Paltsev return false; 26248b04832SEugeniy Paltsev } 26348b04832SEugeniy Paltsev 26441cada4dSEugeniy Paltsev static void __slc_entire_op(const int op) 26541cada4dSEugeniy Paltsev { 26641cada4dSEugeniy Paltsev unsigned int ctrl; 26741cada4dSEugeniy Paltsev 26875790873SEugeniy Paltsev if (!slc_exists()) 269ea9f6f1eSEugeniy Paltsev return; 270ea9f6f1eSEugeniy Paltsev 27141cada4dSEugeniy Paltsev ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); 27241cada4dSEugeniy Paltsev 27341cada4dSEugeniy Paltsev if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 27441cada4dSEugeniy Paltsev ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 27541cada4dSEugeniy Paltsev else 27641cada4dSEugeniy Paltsev ctrl |= SLC_CTRL_IM; 27741cada4dSEugeniy Paltsev 27841cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); 27941cada4dSEugeniy Paltsev 28041cada4dSEugeniy Paltsev if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 28141cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_INVALIDATE, 0x1); 28241cada4dSEugeniy Paltsev else 28341cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_FLUSH, 0x1); 28441cada4dSEugeniy Paltsev 28541cada4dSEugeniy Paltsev /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 28641cada4dSEugeniy Paltsev read_aux_reg(ARC_AUX_SLC_CTRL); 28741cada4dSEugeniy Paltsev 28841cada4dSEugeniy Paltsev /* Important to wait for flush to complete */ 28941cada4dSEugeniy Paltsev while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); 29041cada4dSEugeniy Paltsev } 29141cada4dSEugeniy Paltsev 29241cada4dSEugeniy Paltsev static void slc_upper_region_init(void) 29341cada4dSEugeniy Paltsev { 29441cada4dSEugeniy Paltsev /* 295246ba284SEugeniy Paltsev * ARC_AUX_SLC_RGN_START1 and ARC_AUX_SLC_RGN_END1 register exist 296246ba284SEugeniy Paltsev * only if PAE exists in current HW. So we had to check pae_exist 297246ba284SEugeniy Paltsev * before using them. 298246ba284SEugeniy Paltsev */ 299246ba284SEugeniy Paltsev if (!pae_exists()) 300246ba284SEugeniy Paltsev return; 301246ba284SEugeniy Paltsev 302246ba284SEugeniy Paltsev /* 30341cada4dSEugeniy Paltsev * ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1 are always == 0 30441cada4dSEugeniy Paltsev * as we don't use PAE40. 30541cada4dSEugeniy Paltsev */ 30641cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_RGN_END1, 0); 30741cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_RGN_START1, 0); 30841cada4dSEugeniy Paltsev } 30941cada4dSEugeniy Paltsev 31041cada4dSEugeniy Paltsev static void __slc_rgn_op(unsigned long paddr, unsigned long sz, const int op) 31141cada4dSEugeniy Paltsev { 31205c6a26aSEugeniy Paltsev #ifdef CONFIG_ISA_ARCV2 31305c6a26aSEugeniy Paltsev 31441cada4dSEugeniy Paltsev unsigned int ctrl; 31541cada4dSEugeniy Paltsev unsigned long end; 31641cada4dSEugeniy Paltsev 31775790873SEugeniy Paltsev if (!slc_exists()) 318ea9f6f1eSEugeniy Paltsev return; 319ea9f6f1eSEugeniy Paltsev 32041cada4dSEugeniy Paltsev /* 32141cada4dSEugeniy Paltsev * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 32241cada4dSEugeniy Paltsev * - b'000 (default) is Flush, 32341cada4dSEugeniy Paltsev * - b'001 is Invalidate if CTRL.IM == 0 32441cada4dSEugeniy Paltsev * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 32541cada4dSEugeniy Paltsev */ 32641cada4dSEugeniy Paltsev ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); 32741cada4dSEugeniy Paltsev 32841cada4dSEugeniy Paltsev /* Don't rely on default value of IM bit */ 32941cada4dSEugeniy Paltsev if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 33041cada4dSEugeniy Paltsev ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 33141cada4dSEugeniy Paltsev else 33241cada4dSEugeniy Paltsev ctrl |= SLC_CTRL_IM; 33341cada4dSEugeniy Paltsev 33441cada4dSEugeniy Paltsev if (op & OP_INV) 33541cada4dSEugeniy Paltsev ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 33641cada4dSEugeniy Paltsev else 33741cada4dSEugeniy Paltsev ctrl &= ~SLC_CTRL_RGN_OP_INV; 33841cada4dSEugeniy Paltsev 33941cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); 34041cada4dSEugeniy Paltsev 34141cada4dSEugeniy Paltsev /* 34241cada4dSEugeniy Paltsev * Lower bits are ignored, no need to clip 34341cada4dSEugeniy Paltsev * END needs to be setup before START (latter triggers the operation) 34441cada4dSEugeniy Paltsev * END can't be same as START, so add (l2_line_sz - 1) to sz 34541cada4dSEugeniy Paltsev */ 346bf8974edSEugeniy Paltsev end = paddr + sz + gd->arch.slc_line_sz - 1; 34741cada4dSEugeniy Paltsev 34841cada4dSEugeniy Paltsev /* 34941cada4dSEugeniy Paltsev * Upper addresses (ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1) 35041cada4dSEugeniy Paltsev * are always == 0 as we don't use PAE40, so we only setup lower ones 35141cada4dSEugeniy Paltsev * (ARC_AUX_SLC_RGN_END and ARC_AUX_SLC_RGN_START) 35241cada4dSEugeniy Paltsev */ 35341cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_RGN_END, end); 35441cada4dSEugeniy Paltsev write_aux_reg(ARC_AUX_SLC_RGN_START, paddr); 35541cada4dSEugeniy Paltsev 35641cada4dSEugeniy Paltsev /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 35741cada4dSEugeniy Paltsev read_aux_reg(ARC_AUX_SLC_CTRL); 35841cada4dSEugeniy Paltsev 35941cada4dSEugeniy Paltsev while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); 36005c6a26aSEugeniy Paltsev 36105c6a26aSEugeniy Paltsev #endif /* CONFIG_ISA_ARCV2 */ 36241cada4dSEugeniy Paltsev } 363a6f557c4SEugeniy Paltsev 364a6f557c4SEugeniy Paltsev static void arc_ioc_setup(void) 365a6f557c4SEugeniy Paltsev { 366a6f557c4SEugeniy Paltsev /* IOC Aperture start is equal to DDR start */ 367a6f557c4SEugeniy Paltsev unsigned int ap_base = CONFIG_SYS_SDRAM_BASE; 368a6f557c4SEugeniy Paltsev /* IOC Aperture size is equal to DDR size */ 369a6f557c4SEugeniy Paltsev long ap_size = CONFIG_SYS_SDRAM_SIZE; 370a6f557c4SEugeniy Paltsev 371*7241944aSEugeniy Paltsev /* Unsupported configuration. See [ NOTE 2 ] for more details. */ 372*7241944aSEugeniy Paltsev if (!slc_exists()) 373*7241944aSEugeniy Paltsev panic("Try to enable IOC but SLC is not present"); 374*7241944aSEugeniy Paltsev 375*7241944aSEugeniy Paltsev /* Unsupported configuration. See [ NOTE 2 ] for more details. */ 376*7241944aSEugeniy Paltsev if (!dcache_enabled()) 377*7241944aSEugeniy Paltsev panic("Try to enable IOC but L1 D$ is disabled"); 378*7241944aSEugeniy Paltsev 379a6f557c4SEugeniy Paltsev flush_n_invalidate_dcache_all(); 380a6f557c4SEugeniy Paltsev 381a6f557c4SEugeniy Paltsev if (!is_power_of_2(ap_size) || ap_size < 4096) 382a6f557c4SEugeniy Paltsev panic("IOC Aperture size must be power of 2 and bigger 4Kib"); 383a6f557c4SEugeniy Paltsev 384a6f557c4SEugeniy Paltsev /* 385a6f557c4SEugeniy Paltsev * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB, 386a6f557c4SEugeniy Paltsev * so setting 0x11 implies 512M, 0x12 implies 1G... 387a6f557c4SEugeniy Paltsev */ 388a6f557c4SEugeniy Paltsev write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, 389a6f557c4SEugeniy Paltsev order_base_2(ap_size / 1024) - 2); 390a6f557c4SEugeniy Paltsev 391a6f557c4SEugeniy Paltsev /* IOC Aperture start must be aligned to the size of the aperture */ 392a6f557c4SEugeniy Paltsev if (ap_base % ap_size != 0) 393a6f557c4SEugeniy Paltsev panic("IOC Aperture start must be aligned to the size of the aperture"); 394a6f557c4SEugeniy Paltsev 395a6f557c4SEugeniy Paltsev write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, ap_base >> 12); 396a6f557c4SEugeniy Paltsev write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1); 397a6f557c4SEugeniy Paltsev write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1); 398a6f557c4SEugeniy Paltsev } 399ef639e6fSAlexey Brodkin 400379b3280SAlexey Brodkin static void read_decode_cache_bcr_arcv2(void) 401ef639e6fSAlexey Brodkin { 40205c6a26aSEugeniy Paltsev #ifdef CONFIG_ISA_ARCV2 40305c6a26aSEugeniy Paltsev 40488ae27edSEugeniy Paltsev union bcr_slc_cfg slc_cfg; 405379b3280SAlexey Brodkin 40675790873SEugeniy Paltsev if (slc_exists()) { 407379b3280SAlexey Brodkin slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG); 408bf8974edSEugeniy Paltsev gd->arch.slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64; 409*7241944aSEugeniy Paltsev 410*7241944aSEugeniy Paltsev /* 411*7241944aSEugeniy Paltsev * We don't support configuration where L1 I$ or L1 D$ is 412*7241944aSEugeniy Paltsev * absent but SL$ exists. See [ NOTE 2 ] for more details. 413*7241944aSEugeniy Paltsev */ 414*7241944aSEugeniy Paltsev if (!icache_exists() || !dcache_exists()) 415*7241944aSEugeniy Paltsev panic("Unsupported cache configuration: SLC exists but one of L1 caches is absent"); 416379b3280SAlexey Brodkin } 417db6ce231SAlexey Brodkin 41805c6a26aSEugeniy Paltsev #endif /* CONFIG_ISA_ARCV2 */ 419379b3280SAlexey Brodkin } 420379b3280SAlexey Brodkin 421379b3280SAlexey Brodkin void read_decode_cache_bcr(void) 422379b3280SAlexey Brodkin { 423379b3280SAlexey Brodkin int dc_line_sz = 0, ic_line_sz = 0; 42488ae27edSEugeniy Paltsev union bcr_di_cache ibcr, dbcr; 425379b3280SAlexey Brodkin 426379b3280SAlexey Brodkin ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); 427379b3280SAlexey Brodkin if (ibcr.fields.ver) { 428bf8974edSEugeniy Paltsev gd->arch.l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len; 429379b3280SAlexey Brodkin if (!ic_line_sz) 430379b3280SAlexey Brodkin panic("Instruction exists but line length is 0\n"); 431ef639e6fSAlexey Brodkin } 432ef639e6fSAlexey Brodkin 433379b3280SAlexey Brodkin dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); 434379b3280SAlexey Brodkin if (dbcr.fields.ver) { 435bf8974edSEugeniy Paltsev gd->arch.l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len; 436379b3280SAlexey Brodkin if (!dc_line_sz) 437379b3280SAlexey Brodkin panic("Data cache exists but line length is 0\n"); 438379b3280SAlexey Brodkin } 439379b3280SAlexey Brodkin 440379b3280SAlexey Brodkin if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz)) 441379b3280SAlexey Brodkin panic("Instruction and data cache line lengths differ\n"); 442ef639e6fSAlexey Brodkin } 443ef639e6fSAlexey Brodkin 444ef639e6fSAlexey Brodkin void cache_init(void) 445ef639e6fSAlexey Brodkin { 446379b3280SAlexey Brodkin read_decode_cache_bcr(); 447379b3280SAlexey Brodkin 44805c6a26aSEugeniy Paltsev if (is_isa_arcv2()) 449379b3280SAlexey Brodkin read_decode_cache_bcr_arcv2(); 450db6ce231SAlexey Brodkin 45148b04832SEugeniy Paltsev if (is_isa_arcv2() && ioc_enabled()) 452a6f557c4SEugeniy Paltsev arc_ioc_setup(); 45341cada4dSEugeniy Paltsev 454246ba284SEugeniy Paltsev if (is_isa_arcv2() && slc_exists()) 45541cada4dSEugeniy Paltsev slc_upper_region_init(); 456ef639e6fSAlexey Brodkin } 457ef639e6fSAlexey Brodkin 458660d5f0dSAlexey Brodkin int icache_status(void) 459660d5f0dSAlexey Brodkin { 460c75eeb0bSEugeniy Paltsev return icache_enabled(); 461660d5f0dSAlexey Brodkin } 462660d5f0dSAlexey Brodkin 463660d5f0dSAlexey Brodkin void icache_enable(void) 464660d5f0dSAlexey Brodkin { 46575790873SEugeniy Paltsev if (icache_exists()) 466660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) & 467660d5f0dSAlexey Brodkin ~IC_CTRL_CACHE_DISABLE); 468660d5f0dSAlexey Brodkin } 469660d5f0dSAlexey Brodkin 470660d5f0dSAlexey Brodkin void icache_disable(void) 471660d5f0dSAlexey Brodkin { 47275790873SEugeniy Paltsev if (icache_exists()) 473660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) | 474660d5f0dSAlexey Brodkin IC_CTRL_CACHE_DISABLE); 475660d5f0dSAlexey Brodkin } 476660d5f0dSAlexey Brodkin 47716aeee81SEugeniy Paltsev /* IC supports only invalidation */ 47816aeee81SEugeniy Paltsev static inline void __ic_entire_invalidate(void) 479660d5f0dSAlexey Brodkin { 480c75eeb0bSEugeniy Paltsev if (!icache_enabled()) 48116aeee81SEugeniy Paltsev return; 48216aeee81SEugeniy Paltsev 483660d5f0dSAlexey Brodkin /* Any write to IC_IVIC register triggers invalidation of entire I$ */ 484660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_IC_IVIC, 1); 485f2a22678SAlexey Brodkin /* 486f2a22678SAlexey Brodkin * As per ARC HS databook (see chapter 5.3.3.2) 487f2a22678SAlexey Brodkin * it is required to add 3 NOPs after each write to IC_IVIC. 488f2a22678SAlexey Brodkin */ 489f2a22678SAlexey Brodkin __builtin_arc_nop(); 490f2a22678SAlexey Brodkin __builtin_arc_nop(); 491f2a22678SAlexey Brodkin __builtin_arc_nop(); 492ef639e6fSAlexey Brodkin read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */ 493660d5f0dSAlexey Brodkin } 49441cada4dSEugeniy Paltsev 49516aeee81SEugeniy Paltsev void invalidate_icache_all(void) 49616aeee81SEugeniy Paltsev { 49716aeee81SEugeniy Paltsev __ic_entire_invalidate(); 49816aeee81SEugeniy Paltsev 49995336738SEugeniy Paltsev /* 50095336738SEugeniy Paltsev * If SL$ is bypassed for data it is used only for instructions, 50195336738SEugeniy Paltsev * so we need to invalidate it too. 50295336738SEugeniy Paltsev * TODO: HS 3.0 supports SLC disable so we need to check slc 50395336738SEugeniy Paltsev * enable/disable status here. 50495336738SEugeniy Paltsev */ 50595336738SEugeniy Paltsev if (is_isa_arcv2() && slc_data_bypass()) 50641cada4dSEugeniy Paltsev __slc_entire_op(OP_INV); 50741cada4dSEugeniy Paltsev } 508660d5f0dSAlexey Brodkin 509660d5f0dSAlexey Brodkin int dcache_status(void) 510660d5f0dSAlexey Brodkin { 511c75eeb0bSEugeniy Paltsev return dcache_enabled(); 512660d5f0dSAlexey Brodkin } 513660d5f0dSAlexey Brodkin 514660d5f0dSAlexey Brodkin void dcache_enable(void) 515660d5f0dSAlexey Brodkin { 51675790873SEugeniy Paltsev if (!dcache_exists()) 517660d5f0dSAlexey Brodkin return; 518660d5f0dSAlexey Brodkin 519660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) & 520660d5f0dSAlexey Brodkin ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE)); 521660d5f0dSAlexey Brodkin } 522660d5f0dSAlexey Brodkin 523660d5f0dSAlexey Brodkin void dcache_disable(void) 524660d5f0dSAlexey Brodkin { 52575790873SEugeniy Paltsev if (!dcache_exists()) 526660d5f0dSAlexey Brodkin return; 527660d5f0dSAlexey Brodkin 528660d5f0dSAlexey Brodkin write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) | 529660d5f0dSAlexey Brodkin DC_CTRL_CACHE_DISABLE); 530660d5f0dSAlexey Brodkin } 531660d5f0dSAlexey Brodkin 532c4ef14d2SEugeniy Paltsev /* Common Helper for Line Operations on D-cache */ 533c4ef14d2SEugeniy Paltsev static inline void __dcache_line_loop(unsigned long paddr, unsigned long sz, 534ef639e6fSAlexey Brodkin const int cacheop) 535660d5f0dSAlexey Brodkin { 536ef639e6fSAlexey Brodkin unsigned int aux_cmd; 537ef639e6fSAlexey Brodkin int num_lines; 538660d5f0dSAlexey Brodkin 539ef639e6fSAlexey Brodkin /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 540ef639e6fSAlexey Brodkin aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL; 541660d5f0dSAlexey Brodkin 542ef639e6fSAlexey Brodkin sz += paddr & ~CACHE_LINE_MASK; 543ef639e6fSAlexey Brodkin paddr &= CACHE_LINE_MASK; 544ef639e6fSAlexey Brodkin 545bf8974edSEugeniy Paltsev num_lines = DIV_ROUND_UP(sz, gd->arch.l1_line_sz); 546ef639e6fSAlexey Brodkin 547ef639e6fSAlexey Brodkin while (num_lines-- > 0) { 548ef639e6fSAlexey Brodkin #if (CONFIG_ARC_MMU_VER == 3) 549c4ef14d2SEugeniy Paltsev write_aux_reg(ARC_AUX_DC_PTAG, paddr); 550ef639e6fSAlexey Brodkin #endif 551ef639e6fSAlexey Brodkin write_aux_reg(aux_cmd, paddr); 552bf8974edSEugeniy Paltsev paddr += gd->arch.l1_line_sz; 553ef639e6fSAlexey Brodkin } 554ef639e6fSAlexey Brodkin } 555ef639e6fSAlexey Brodkin 5565d7a24d6SEugeniy Paltsev static void __before_dc_op(const int op) 557ef639e6fSAlexey Brodkin { 5585d7a24d6SEugeniy Paltsev unsigned int ctrl; 559ef639e6fSAlexey Brodkin 5605d7a24d6SEugeniy Paltsev ctrl = read_aux_reg(ARC_AUX_DC_CTRL); 5615d7a24d6SEugeniy Paltsev 5625d7a24d6SEugeniy Paltsev /* IM bit implies flush-n-inv, instead of vanilla inv */ 5635d7a24d6SEugeniy Paltsev if (op == OP_INV) 5645d7a24d6SEugeniy Paltsev ctrl &= ~DC_CTRL_INV_MODE_FLUSH; 5655d7a24d6SEugeniy Paltsev else 5665d7a24d6SEugeniy Paltsev ctrl |= DC_CTRL_INV_MODE_FLUSH; 5675d7a24d6SEugeniy Paltsev 5685d7a24d6SEugeniy Paltsev write_aux_reg(ARC_AUX_DC_CTRL, ctrl); 569ef639e6fSAlexey Brodkin } 570ef639e6fSAlexey Brodkin 5715d7a24d6SEugeniy Paltsev static void __after_dc_op(const int op) 572ef639e6fSAlexey Brodkin { 573ef639e6fSAlexey Brodkin if (op & OP_FLUSH) /* flush / flush-n-inv both wait */ 57419b10a42SEugeniy Paltsev while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS); 575ef639e6fSAlexey Brodkin } 576ef639e6fSAlexey Brodkin 577ef639e6fSAlexey Brodkin static inline void __dc_entire_op(const int cacheop) 578ef639e6fSAlexey Brodkin { 579ef639e6fSAlexey Brodkin int aux; 5805d7a24d6SEugeniy Paltsev 581c75eeb0bSEugeniy Paltsev if (!dcache_enabled()) 582c877a891SEugeniy Paltsev return; 583c877a891SEugeniy Paltsev 5845d7a24d6SEugeniy Paltsev __before_dc_op(cacheop); 585ef639e6fSAlexey Brodkin 586ef639e6fSAlexey Brodkin if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 587ef639e6fSAlexey Brodkin aux = ARC_AUX_DC_IVDC; 588ef639e6fSAlexey Brodkin else 589ef639e6fSAlexey Brodkin aux = ARC_AUX_DC_FLSH; 590ef639e6fSAlexey Brodkin 591ef639e6fSAlexey Brodkin write_aux_reg(aux, 0x1); 592ef639e6fSAlexey Brodkin 5935d7a24d6SEugeniy Paltsev __after_dc_op(cacheop); 594ef639e6fSAlexey Brodkin } 595ef639e6fSAlexey Brodkin 596ef639e6fSAlexey Brodkin static inline void __dc_line_op(unsigned long paddr, unsigned long sz, 597ef639e6fSAlexey Brodkin const int cacheop) 598ef639e6fSAlexey Brodkin { 599c75eeb0bSEugeniy Paltsev if (!dcache_enabled()) 600c877a891SEugeniy Paltsev return; 601c877a891SEugeniy Paltsev 6025d7a24d6SEugeniy Paltsev __before_dc_op(cacheop); 603c4ef14d2SEugeniy Paltsev __dcache_line_loop(paddr, sz, cacheop); 6045d7a24d6SEugeniy Paltsev __after_dc_op(cacheop); 605ef639e6fSAlexey Brodkin } 606ef639e6fSAlexey Brodkin 607660d5f0dSAlexey Brodkin void invalidate_dcache_range(unsigned long start, unsigned long end) 608660d5f0dSAlexey Brodkin { 60941cada4dSEugeniy Paltsev if (start >= end) 61041cada4dSEugeniy Paltsev return; 61141cada4dSEugeniy Paltsev 61205c6a26aSEugeniy Paltsev /* 61305c6a26aSEugeniy Paltsev * ARCv1 -> call __dc_line_op 61495336738SEugeniy Paltsev * ARCv2 && L1 D$ disabled -> nothing 61595336738SEugeniy Paltsev * ARCv2 && L1 D$ enabled && IOC enabled -> nothing 61695336738SEugeniy Paltsev * ARCv2 && L1 D$ enabled && no IOC -> call __dc_line_op; call __slc_rgn_op 61705c6a26aSEugeniy Paltsev */ 61848b04832SEugeniy Paltsev if (!is_isa_arcv2() || !ioc_enabled()) 619db6ce231SAlexey Brodkin __dc_line_op(start, end - start, OP_INV); 620db6ce231SAlexey Brodkin 62195336738SEugeniy Paltsev if (is_isa_arcv2() && !ioc_enabled() && !slc_data_bypass()) 62241cada4dSEugeniy Paltsev __slc_rgn_op(start, end - start, OP_INV); 623660d5f0dSAlexey Brodkin } 624660d5f0dSAlexey Brodkin 625ef639e6fSAlexey Brodkin void flush_dcache_range(unsigned long start, unsigned long end) 626660d5f0dSAlexey Brodkin { 62741cada4dSEugeniy Paltsev if (start >= end) 62841cada4dSEugeniy Paltsev return; 62941cada4dSEugeniy Paltsev 63005c6a26aSEugeniy Paltsev /* 63105c6a26aSEugeniy Paltsev * ARCv1 -> call __dc_line_op 63295336738SEugeniy Paltsev * ARCv2 && L1 D$ disabled -> nothing 63395336738SEugeniy Paltsev * ARCv2 && L1 D$ enabled && IOC enabled -> nothing 63495336738SEugeniy Paltsev * ARCv2 && L1 D$ enabled && no IOC -> call __dc_line_op; call __slc_rgn_op 63505c6a26aSEugeniy Paltsev */ 63648b04832SEugeniy Paltsev if (!is_isa_arcv2() || !ioc_enabled()) 637db6ce231SAlexey Brodkin __dc_line_op(start, end - start, OP_FLUSH); 638db6ce231SAlexey Brodkin 63995336738SEugeniy Paltsev if (is_isa_arcv2() && !ioc_enabled() && !slc_data_bypass()) 64041cada4dSEugeniy Paltsev __slc_rgn_op(start, end - start, OP_FLUSH); 641660d5f0dSAlexey Brodkin } 642660d5f0dSAlexey Brodkin 643660d5f0dSAlexey Brodkin void flush_cache(unsigned long start, unsigned long size) 644660d5f0dSAlexey Brodkin { 645660d5f0dSAlexey Brodkin flush_dcache_range(start, start + size); 646660d5f0dSAlexey Brodkin } 6476eb15e50SAlexey Brodkin 648c27814beSEugeniy Paltsev /* 649c27814beSEugeniy Paltsev * As invalidate_dcache_all() is not used in generic U-Boot code and as we 650c27814beSEugeniy Paltsev * don't need it in arch/arc code alone (invalidate without flush) we implement 651c27814beSEugeniy Paltsev * flush_n_invalidate_dcache_all (flush and invalidate in 1 operation) because 652c27814beSEugeniy Paltsev * it's much safer. See [ NOTE 1 ] for more details. 653c27814beSEugeniy Paltsev */ 654c27814beSEugeniy Paltsev void flush_n_invalidate_dcache_all(void) 655ef639e6fSAlexey Brodkin { 656c27814beSEugeniy Paltsev __dc_entire_op(OP_FLUSH_N_INV); 657db6ce231SAlexey Brodkin 65895336738SEugeniy Paltsev if (is_isa_arcv2() && !slc_data_bypass()) 659c27814beSEugeniy Paltsev __slc_entire_op(OP_FLUSH_N_INV); 6606eb15e50SAlexey Brodkin } 6616eb15e50SAlexey Brodkin 662ef639e6fSAlexey Brodkin void flush_dcache_all(void) 6636eb15e50SAlexey Brodkin { 664db6ce231SAlexey Brodkin __dc_entire_op(OP_FLUSH); 665db6ce231SAlexey Brodkin 66695336738SEugeniy Paltsev if (is_isa_arcv2() && !slc_data_bypass()) 667ef639e6fSAlexey Brodkin __slc_entire_op(OP_FLUSH); 6686eb15e50SAlexey Brodkin } 669375945baSEugeniy Paltsev 670375945baSEugeniy Paltsev /* 671375945baSEugeniy Paltsev * This is function to cleanup all caches (and therefore sync I/D caches) which 672375945baSEugeniy Paltsev * can be used for cleanup before linux launch or to sync caches during 673375945baSEugeniy Paltsev * relocation. 674375945baSEugeniy Paltsev */ 675375945baSEugeniy Paltsev void sync_n_cleanup_cache_all(void) 676375945baSEugeniy Paltsev { 677375945baSEugeniy Paltsev __dc_entire_op(OP_FLUSH_N_INV); 678375945baSEugeniy Paltsev 679375945baSEugeniy Paltsev /* 680375945baSEugeniy Paltsev * If SL$ is bypassed for data it is used only for instructions, 681375945baSEugeniy Paltsev * and we shouldn't flush it. So invalidate it instead of flush_n_inv. 682375945baSEugeniy Paltsev */ 683375945baSEugeniy Paltsev if (is_isa_arcv2()) { 684375945baSEugeniy Paltsev if (slc_data_bypass()) 685375945baSEugeniy Paltsev __slc_entire_op(OP_INV); 686375945baSEugeniy Paltsev else 687375945baSEugeniy Paltsev __slc_entire_op(OP_FLUSH_N_INV); 688375945baSEugeniy Paltsev } 689375945baSEugeniy Paltsev 690375945baSEugeniy Paltsev __ic_entire_invalidate(); 691375945baSEugeniy Paltsev } 692