1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2008-2009 ST-Ericsson AB 4 * TCM memory handling for ARM systems 5 * 6 * Author: Linus Walleij <linus.walleij@stericsson.com> 7 * Author: Rickard Andersson <rickard.andersson@stericsson.com> 8 */ 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/stddef.h> 13 #include <linux/ioport.h> 14 #include <linux/genalloc.h> 15 #include <linux/string.h> /* memcpy */ 16 #include <asm/cputype.h> 17 #include <asm/mach/map.h> 18 #include <asm/memory.h> 19 #include <asm/system_info.h> 20 #include <asm/traps.h> 21 #include <asm/tcm.h> 22 23 #define TCMTR_FORMAT_MASK 0xe0000000U 24 25 static struct gen_pool *tcm_pool; 26 static bool dtcm_present; 27 static bool itcm_present; 28 29 /* TCM section definitions from the linker */ 30 extern char __itcm_start, __sitcm_text, __eitcm_text; 31 extern char __dtcm_start, __sdtcm_data, __edtcm_data; 32 33 /* These will be increased as we run */ 34 static u32 dtcm_end = DTCM_OFFSET; 35 static u32 itcm_end = ITCM_OFFSET; 36 37 /* 38 * TCM memory resources 39 */ 40 static struct resource dtcm_res = { 41 .name = "DTCM RAM", 42 .start = DTCM_OFFSET, 43 .end = DTCM_OFFSET, 44 .flags = IORESOURCE_MEM 45 }; 46 47 static struct resource itcm_res = { 48 .name = "ITCM RAM", 49 .start = ITCM_OFFSET, 50 .end = ITCM_OFFSET, 51 .flags = IORESOURCE_MEM 52 }; 53 54 static struct map_desc dtcm_iomap[] __initdata = { 55 { 56 .virtual = DTCM_OFFSET, 57 .pfn = __phys_to_pfn(DTCM_OFFSET), 58 .length = 0, 59 .type = MT_MEMORY_RW_DTCM 60 } 61 }; 62 63 static struct map_desc itcm_iomap[] __initdata = { 64 { 65 .virtual = ITCM_OFFSET, 66 .pfn = __phys_to_pfn(ITCM_OFFSET), 67 .length = 0, 68 .type = MT_MEMORY_RWX_ITCM, 69 } 70 }; 71 72 /* 73 * Allocate a chunk of TCM memory 74 */ 75 void *tcm_alloc(size_t len) 76 { 77 unsigned long vaddr; 78 79 if (!tcm_pool) 80 return NULL; 81 82 vaddr = gen_pool_alloc(tcm_pool, len); 83 if (!vaddr) 84 return NULL; 85 86 return (void *) vaddr; 87 } 88 EXPORT_SYMBOL(tcm_alloc); 89 90 /* 91 * Free a chunk of TCM memory 92 */ 93 void tcm_free(void *addr, size_t len) 94 { 95 gen_pool_free(tcm_pool, (unsigned long) addr, len); 96 } 97 EXPORT_SYMBOL(tcm_free); 98 99 bool tcm_dtcm_present(void) 100 { 101 return dtcm_present; 102 } 103 EXPORT_SYMBOL(tcm_dtcm_present); 104 105 bool tcm_itcm_present(void) 106 { 107 return itcm_present; 108 } 109 EXPORT_SYMBOL(tcm_itcm_present); 110 111 static int __init setup_tcm_bank(u8 type, u8 bank, u8 banks, 112 u32 *offset) 113 { 114 const int tcm_sizes[16] = { 0, -1, -1, 4, 8, 16, 32, 64, 128, 115 256, 512, 1024, -1, -1, -1, -1 }; 116 u32 tcm_region; 117 int tcm_size; 118 119 /* 120 * If there are more than one TCM bank of this type, 121 * select the TCM bank to operate on in the TCM selection 122 * register. 123 */ 124 if (banks > 1) 125 asm("mcr p15, 0, %0, c9, c2, 0" 126 : /* No output operands */ 127 : "r" (bank)); 128 129 /* Read the special TCM region register c9, 0 */ 130 if (!type) 131 asm("mrc p15, 0, %0, c9, c1, 0" 132 : "=r" (tcm_region)); 133 else 134 asm("mrc p15, 0, %0, c9, c1, 1" 135 : "=r" (tcm_region)); 136 137 tcm_size = tcm_sizes[(tcm_region >> 2) & 0x0f]; 138 if (tcm_size < 0) { 139 pr_err("CPU: %sTCM%d of unknown size\n", 140 type ? "I" : "D", bank); 141 return -EINVAL; 142 } else if (tcm_size > 32) { 143 pr_err("CPU: %sTCM%d larger than 32k found\n", 144 type ? "I" : "D", bank); 145 return -EINVAL; 146 } else { 147 pr_info("CPU: found %sTCM%d %dk @ %08x, %senabled\n", 148 type ? "I" : "D", 149 bank, 150 tcm_size, 151 (tcm_region & 0xfffff000U), 152 (tcm_region & 1) ? "" : "not "); 153 } 154 155 /* Not much fun you can do with a size 0 bank */ 156 if (tcm_size == 0) 157 return 0; 158 159 /* Force move the TCM bank to where we want it, enable */ 160 tcm_region = *offset | (tcm_region & 0x00000ffeU) | 1; 161 162 if (!type) 163 asm("mcr p15, 0, %0, c9, c1, 0" 164 : /* No output operands */ 165 : "r" (tcm_region)); 166 else 167 asm("mcr p15, 0, %0, c9, c1, 1" 168 : /* No output operands */ 169 : "r" (tcm_region)); 170 171 /* Increase offset */ 172 *offset += (tcm_size << 10); 173 174 pr_info("CPU: moved %sTCM%d %dk to %08x, enabled\n", 175 type ? "I" : "D", 176 bank, 177 tcm_size, 178 (tcm_region & 0xfffff000U)); 179 return 0; 180 } 181 182 /* 183 * When we are running in the non-secure world and the secure world 184 * has not explicitly given us access to the TCM we will get an 185 * undefined error when reading the TCM region register in the 186 * setup_tcm_bank function (above). 187 * 188 * There are two variants of this register read that we need to trap, 189 * the read for the data TCM and the read for the instruction TCM: 190 * c0370628: ee196f11 mrc 15, 0, r6, cr9, cr1, {0} 191 * c0370674: ee196f31 mrc 15, 0, r6, cr9, cr1, {1} 192 * 193 * Our undef hook mask explicitly matches all fields of the encoded 194 * instruction other than the destination register. The mask also 195 * only allows operand 2 to have the values 0 or 1. 196 * 197 * The undefined hook is defined as __init and __initdata, and therefore 198 * must be removed before tcm_init returns. 199 * 200 * In this particular case (MRC with ARM condition code ALways) the 201 * Thumb-2 and ARM instruction encoding are identical, so this hook 202 * will work on a Thumb-2 kernel. 203 * 204 * See A8.8.107, DDI0406C_C ARM Architecture Reference Manual, Encoding 205 * T1/A1 for the bit-by-bit details. 206 * 207 * mrc p15, 0, XX, c9, c1, 0 208 * mrc p15, 0, XX, c9, c1, 1 209 * | | | | | | | +---- opc2 0|1 = 000|001 210 * | | | | | | +------- CRm 0 = 0001 211 * | | | | | +----------- CRn 0 = 1001 212 * | | | | +--------------- Rt ? = ???? 213 * | | | +------------------- opc1 0 = 000 214 * | | +----------------------- coproc 15 = 1111 215 * | +-------------------------- condition ALways = 1110 216 * +----------------------------- instruction MRC = 1110 217 * 218 * Encoding this as per A8.8.107 of DDI0406C, Encoding T1/A1, yields: 219 * 1111 1111 1111 1111 0000 1111 1101 1111 Required Mask 220 * 1110 1110 0001 1001 ???? 1111 0001 0001 mrc p15, 0, XX, c9, c1, 0 221 * 1110 1110 0001 1001 ???? 1111 0011 0001 mrc p15, 0, XX, c9, c1, 1 222 * [ ] [ ] [ ]| [ ] [ ] [ ] [ ]| +--- CRm 223 * | | | | | | | | +----- SBO 224 * | | | | | | | +------- opc2 225 * | | | | | | +----------- coproc 226 * | | | | | +---------------- Rt 227 * | | | | +--------------------- CRn 228 * | | | +------------------------- SBO 229 * | | +--------------------------- opc1 230 * | +------------------------------- instruction 231 * +------------------------------------ condition 232 */ 233 #define TCM_REGION_READ_MASK 0xffff0fdf 234 #define TCM_REGION_READ_INSTR 0xee190f11 235 #define DEST_REG_SHIFT 12 236 #define DEST_REG_MASK 0xf 237 238 static int __init tcm_handler(struct pt_regs *regs, unsigned int instr) 239 { 240 regs->uregs[(instr >> DEST_REG_SHIFT) & DEST_REG_MASK] = 0; 241 regs->ARM_pc += 4; 242 return 0; 243 } 244 245 static struct undef_hook tcm_hook __initdata = { 246 .instr_mask = TCM_REGION_READ_MASK, 247 .instr_val = TCM_REGION_READ_INSTR, 248 .cpsr_mask = MODE_MASK, 249 .cpsr_val = SVC_MODE, 250 .fn = tcm_handler 251 }; 252 253 /* 254 * This initializes the TCM memory 255 */ 256 void __init tcm_init(void) 257 { 258 u32 tcm_status; 259 u8 dtcm_banks; 260 u8 itcm_banks; 261 size_t dtcm_code_sz = &__edtcm_data - &__sdtcm_data; 262 size_t itcm_code_sz = &__eitcm_text - &__sitcm_text; 263 char *start; 264 char *end; 265 char *ram; 266 int ret; 267 int i; 268 269 /* 270 * Prior to ARMv5 there is no TCM, and trying to read the status 271 * register will hang the processor. 272 */ 273 if (cpu_architecture() < CPU_ARCH_ARMv5) { 274 if (dtcm_code_sz || itcm_code_sz) 275 pr_info("CPU TCM: %u bytes of DTCM and %u bytes of " 276 "ITCM code compiled in, but no TCM present " 277 "in pre-v5 CPU\n", dtcm_code_sz, itcm_code_sz); 278 return; 279 } 280 281 tcm_status = read_cpuid_tcmstatus(); 282 283 /* 284 * This code only supports v6-compatible TCMTR implementations. 285 */ 286 if (tcm_status & TCMTR_FORMAT_MASK) 287 return; 288 289 dtcm_banks = (tcm_status >> 16) & 0x03; 290 itcm_banks = (tcm_status & 0x03); 291 292 register_undef_hook(&tcm_hook); 293 294 /* Values greater than 2 for D/ITCM banks are "reserved" */ 295 if (dtcm_banks > 2) 296 dtcm_banks = 0; 297 if (itcm_banks > 2) 298 itcm_banks = 0; 299 300 /* Setup DTCM if present */ 301 if (dtcm_banks > 0) { 302 for (i = 0; i < dtcm_banks; i++) { 303 ret = setup_tcm_bank(0, i, dtcm_banks, &dtcm_end); 304 if (ret) 305 goto unregister; 306 } 307 /* This means you compiled more code than fits into DTCM */ 308 if (dtcm_code_sz > (dtcm_end - DTCM_OFFSET)) { 309 pr_info("CPU DTCM: %u bytes of code compiled to " 310 "DTCM but only %lu bytes of DTCM present\n", 311 dtcm_code_sz, (dtcm_end - DTCM_OFFSET)); 312 goto no_dtcm; 313 } 314 /* 315 * This means that the DTCM sizes were 0 or the DTCM banks 316 * were inaccessible due to TrustZone configuration. 317 */ 318 if (!(dtcm_end - DTCM_OFFSET)) 319 goto no_dtcm; 320 dtcm_res.end = dtcm_end - 1; 321 request_resource(&iomem_resource, &dtcm_res); 322 dtcm_iomap[0].length = dtcm_end - DTCM_OFFSET; 323 iotable_init(dtcm_iomap, 1); 324 /* Copy data from RAM to DTCM */ 325 start = &__sdtcm_data; 326 end = &__edtcm_data; 327 ram = &__dtcm_start; 328 memcpy(start, ram, dtcm_code_sz); 329 pr_debug("CPU DTCM: copied data from %p - %p\n", 330 start, end); 331 dtcm_present = true; 332 } else if (dtcm_code_sz) { 333 pr_info("CPU DTCM: %u bytes of code compiled to DTCM but no " 334 "DTCM banks present in CPU\n", dtcm_code_sz); 335 } 336 337 no_dtcm: 338 /* Setup ITCM if present */ 339 if (itcm_banks > 0) { 340 for (i = 0; i < itcm_banks; i++) { 341 ret = setup_tcm_bank(1, i, itcm_banks, &itcm_end); 342 if (ret) 343 goto unregister; 344 } 345 /* This means you compiled more code than fits into ITCM */ 346 if (itcm_code_sz > (itcm_end - ITCM_OFFSET)) { 347 pr_info("CPU ITCM: %u bytes of code compiled to " 348 "ITCM but only %lu bytes of ITCM present\n", 349 itcm_code_sz, (itcm_end - ITCM_OFFSET)); 350 goto unregister; 351 } 352 /* 353 * This means that the ITCM sizes were 0 or the ITCM banks 354 * were inaccessible due to TrustZone configuration. 355 */ 356 if (!(itcm_end - ITCM_OFFSET)) 357 goto unregister; 358 itcm_res.end = itcm_end - 1; 359 request_resource(&iomem_resource, &itcm_res); 360 itcm_iomap[0].length = itcm_end - ITCM_OFFSET; 361 iotable_init(itcm_iomap, 1); 362 /* Copy code from RAM to ITCM */ 363 start = &__sitcm_text; 364 end = &__eitcm_text; 365 ram = &__itcm_start; 366 memcpy(start, ram, itcm_code_sz); 367 pr_debug("CPU ITCM: copied code from %p - %p\n", 368 start, end); 369 itcm_present = true; 370 } else if (itcm_code_sz) { 371 pr_info("CPU ITCM: %u bytes of code compiled to ITCM but no " 372 "ITCM banks present in CPU\n", itcm_code_sz); 373 } 374 375 unregister: 376 unregister_undef_hook(&tcm_hook); 377 } 378 379 /* 380 * This creates the TCM memory pool and has to be done later, 381 * during the core_initicalls, since the allocator is not yet 382 * up and running when the first initialization runs. 383 */ 384 static int __init setup_tcm_pool(void) 385 { 386 u32 dtcm_pool_start = (u32) &__edtcm_data; 387 u32 itcm_pool_start = (u32) &__eitcm_text; 388 int ret; 389 390 /* 391 * Set up malloc pool, 2^2 = 4 bytes granularity since 392 * the TCM is sometimes just 4 KiB. NB: pages and cache 393 * line alignments does not matter in TCM! 394 */ 395 tcm_pool = gen_pool_create(2, -1); 396 397 pr_debug("Setting up TCM memory pool\n"); 398 399 /* Add the rest of DTCM to the TCM pool */ 400 if (dtcm_present) { 401 if (dtcm_pool_start < dtcm_end) { 402 ret = gen_pool_add(tcm_pool, dtcm_pool_start, 403 dtcm_end - dtcm_pool_start, -1); 404 if (ret) { 405 pr_err("CPU DTCM: could not add DTCM " \ 406 "remainder to pool!\n"); 407 return ret; 408 } 409 pr_debug("CPU DTCM: Added %08x bytes @ %08x to " \ 410 "the TCM memory pool\n", 411 dtcm_end - dtcm_pool_start, 412 dtcm_pool_start); 413 } 414 } 415 416 /* Add the rest of ITCM to the TCM pool */ 417 if (itcm_present) { 418 if (itcm_pool_start < itcm_end) { 419 ret = gen_pool_add(tcm_pool, itcm_pool_start, 420 itcm_end - itcm_pool_start, -1); 421 if (ret) { 422 pr_err("CPU ITCM: could not add ITCM " \ 423 "remainder to pool!\n"); 424 return ret; 425 } 426 pr_debug("CPU ITCM: Added %08x bytes @ %08x to " \ 427 "the TCM memory pool\n", 428 itcm_end - itcm_pool_start, 429 itcm_pool_start); 430 } 431 } 432 return 0; 433 } 434 435 core_initcall(setup_tcm_pool); 436