1 /* 2 * Copyright (C) 2015-2016 Socionext Inc. 3 * Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 */ 15 16 #define pr_fmt(fmt) "uniphier: " fmt 17 18 #include <linux/init.h> 19 #include <linux/io.h> 20 #include <linux/log2.h> 21 #include <linux/of_address.h> 22 #include <linux/slab.h> 23 #include <asm/hardware/cache-uniphier.h> 24 #include <asm/outercache.h> 25 26 /* control registers */ 27 #define UNIPHIER_SSCC 0x0 /* Control Register */ 28 #define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */ 29 #define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */ 30 #define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */ 31 #define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */ 32 #define UNIPHIER_SSCC_ON BIT(0) /* enable cache */ 33 #define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */ 34 #define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */ 35 36 /* revision registers */ 37 #define UNIPHIER_SSCID 0x0 /* ID Register */ 38 39 /* operation registers */ 40 #define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */ 41 #define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */ 42 #define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */ 43 #define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */ 44 #define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */ 45 #define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */ 46 #define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */ 47 #define UNIPHIER_SSCOQM_S_MASK (0x3 << 17) 48 #define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17) 49 #define UNIPHIER_SSCOQM_S_ALL (0x1 << 17) 50 #define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */ 51 #define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */ 52 #define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */ 53 #define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */ 54 #define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */ 55 #define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */ 56 #define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/ 57 #define UNIPHIER_SSCOPPQSEF_FE BIT(1) 58 #define UNIPHIER_SSCOPPQSEF_OE BIT(0) 59 #define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */ 60 #define UNIPHIER_SSCOLPQS_EF BIT(2) 61 #define UNIPHIER_SSCOLPQS_EST BIT(1) 62 #define UNIPHIER_SSCOLPQS_QST BIT(0) 63 64 /* Is the operation region specified by address range? */ 65 #define UNIPHIER_SSCOQM_S_IS_RANGE(op) \ 66 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE) 67 68 /** 69 * uniphier_cache_data - UniPhier outer cache specific data 70 * 71 * @ctrl_base: virtual base address of control registers 72 * @rev_base: virtual base address of revision registers 73 * @op_base: virtual base address of operation registers 74 * @way_present_mask: each bit specifies if the way is present 75 * @way_locked_mask: each bit specifies if the way is locked 76 * @nsets: number of associativity sets 77 * @line_size: line size in bytes 78 * @range_op_max_size: max size that can be handled by a single range operation 79 * @list: list node to include this level in the whole cache hierarchy 80 */ 81 struct uniphier_cache_data { 82 void __iomem *ctrl_base; 83 void __iomem *rev_base; 84 void __iomem *op_base; 85 void __iomem *way_ctrl_base; 86 u32 way_present_mask; 87 u32 way_locked_mask; 88 u32 nsets; 89 u32 line_size; 90 u32 range_op_max_size; 91 struct list_head list; 92 }; 93 94 /* 95 * List of the whole outer cache hierarchy. This list is only modified during 96 * the early boot stage, so no mutex is taken for the access to the list. 97 */ 98 static LIST_HEAD(uniphier_cache_list); 99 100 /** 101 * __uniphier_cache_sync - perform a sync point for a particular cache level 102 * 103 * @data: cache controller specific data 104 */ 105 static void __uniphier_cache_sync(struct uniphier_cache_data *data) 106 { 107 /* This sequence need not be atomic. Do not disable IRQ. */ 108 writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC, 109 data->op_base + UNIPHIER_SSCOPE); 110 /* need a read back to confirm */ 111 readl_relaxed(data->op_base + UNIPHIER_SSCOPE); 112 } 113 114 /** 115 * __uniphier_cache_maint_common - run a queue operation for a particular level 116 * 117 * @data: cache controller specific data 118 * @start: start address of range operation (don't care for "all" operation) 119 * @size: data size of range operation (don't care for "all" operation) 120 * @operation: flags to specify the desired cache operation 121 */ 122 static void __uniphier_cache_maint_common(struct uniphier_cache_data *data, 123 unsigned long start, 124 unsigned long size, 125 u32 operation) 126 { 127 unsigned long flags; 128 129 /* 130 * No spin lock is necessary here because: 131 * 132 * [1] This outer cache controller is able to accept maintenance 133 * operations from multiple CPUs at a time in an SMP system; if a 134 * maintenance operation is under way and another operation is issued, 135 * the new one is stored in the queue. The controller performs one 136 * operation after another. If the queue is full, the status register, 137 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has 138 * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have 139 * different instances for each CPU, i.e. each CPU can track the status 140 * of the maintenance operations triggered by itself. 141 * 142 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ, 143 * SSCOQWN}, are shared between multiple CPUs, but the hardware still 144 * guarantees the registration sequence is atomic; the write access to 145 * them are arbitrated by the hardware. The first accessor to the 146 * register, UNIPHIER_SSCOQM, holds the access right and it is released 147 * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU 148 * is holding the access right, other CPUs fail to register operations. 149 * One CPU should not hold the access right for a long time, so local 150 * IRQs should be disabled while the following sequence. 151 */ 152 local_irq_save(flags); 153 154 /* clear the complete notification flag */ 155 writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS); 156 157 do { 158 /* set cache operation */ 159 writel_relaxed(UNIPHIER_SSCOQM_CE | operation, 160 data->op_base + UNIPHIER_SSCOQM); 161 162 /* set address range if needed */ 163 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) { 164 writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD); 165 writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ); 166 } 167 } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) & 168 (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE))); 169 170 /* wait until the operation is completed */ 171 while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) != 172 UNIPHIER_SSCOLPQS_EF)) 173 cpu_relax(); 174 175 local_irq_restore(flags); 176 } 177 178 static void __uniphier_cache_maint_all(struct uniphier_cache_data *data, 179 u32 operation) 180 { 181 __uniphier_cache_maint_common(data, 0, 0, 182 UNIPHIER_SSCOQM_S_ALL | operation); 183 184 __uniphier_cache_sync(data); 185 } 186 187 static void __uniphier_cache_maint_range(struct uniphier_cache_data *data, 188 unsigned long start, unsigned long end, 189 u32 operation) 190 { 191 unsigned long size; 192 193 /* 194 * If the start address is not aligned, 195 * perform a cache operation for the first cache-line 196 */ 197 start = start & ~(data->line_size - 1); 198 199 size = end - start; 200 201 if (unlikely(size >= (unsigned long)(-data->line_size))) { 202 /* this means cache operation for all range */ 203 __uniphier_cache_maint_all(data, operation); 204 return; 205 } 206 207 /* 208 * If the end address is not aligned, 209 * perform a cache operation for the last cache-line 210 */ 211 size = ALIGN(size, data->line_size); 212 213 while (size) { 214 unsigned long chunk_size = min_t(unsigned long, size, 215 data->range_op_max_size); 216 217 __uniphier_cache_maint_common(data, start, chunk_size, 218 UNIPHIER_SSCOQM_S_RANGE | operation); 219 220 start += chunk_size; 221 size -= chunk_size; 222 } 223 224 __uniphier_cache_sync(data); 225 } 226 227 static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on) 228 { 229 u32 val = 0; 230 231 if (on) 232 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON; 233 234 writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC); 235 } 236 237 static void __init __uniphier_cache_set_locked_ways( 238 struct uniphier_cache_data *data, 239 u32 way_mask) 240 { 241 unsigned int cpu; 242 243 data->way_locked_mask = way_mask & data->way_present_mask; 244 245 for_each_possible_cpu(cpu) 246 writel_relaxed(~data->way_locked_mask & data->way_present_mask, 247 data->way_ctrl_base + 4 * cpu); 248 } 249 250 static void uniphier_cache_maint_range(unsigned long start, unsigned long end, 251 u32 operation) 252 { 253 struct uniphier_cache_data *data; 254 255 list_for_each_entry(data, &uniphier_cache_list, list) 256 __uniphier_cache_maint_range(data, start, end, operation); 257 } 258 259 static void uniphier_cache_maint_all(u32 operation) 260 { 261 struct uniphier_cache_data *data; 262 263 list_for_each_entry(data, &uniphier_cache_list, list) 264 __uniphier_cache_maint_all(data, operation); 265 } 266 267 static void uniphier_cache_inv_range(unsigned long start, unsigned long end) 268 { 269 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV); 270 } 271 272 static void uniphier_cache_clean_range(unsigned long start, unsigned long end) 273 { 274 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN); 275 } 276 277 static void uniphier_cache_flush_range(unsigned long start, unsigned long end) 278 { 279 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH); 280 } 281 282 static void __init uniphier_cache_inv_all(void) 283 { 284 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV); 285 } 286 287 static void uniphier_cache_flush_all(void) 288 { 289 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH); 290 } 291 292 static void uniphier_cache_disable(void) 293 { 294 struct uniphier_cache_data *data; 295 296 list_for_each_entry_reverse(data, &uniphier_cache_list, list) 297 __uniphier_cache_enable(data, false); 298 299 uniphier_cache_flush_all(); 300 } 301 302 static void __init uniphier_cache_enable(void) 303 { 304 struct uniphier_cache_data *data; 305 306 uniphier_cache_inv_all(); 307 308 list_for_each_entry(data, &uniphier_cache_list, list) { 309 __uniphier_cache_enable(data, true); 310 __uniphier_cache_set_locked_ways(data, 0); 311 } 312 } 313 314 static void uniphier_cache_sync(void) 315 { 316 struct uniphier_cache_data *data; 317 318 list_for_each_entry(data, &uniphier_cache_list, list) 319 __uniphier_cache_sync(data); 320 } 321 322 static const struct of_device_id uniphier_cache_match[] __initconst = { 323 { .compatible = "socionext,uniphier-system-cache" }, 324 { /* sentinel */ } 325 }; 326 327 static int __init __uniphier_cache_init(struct device_node *np, 328 unsigned int *cache_level) 329 { 330 struct uniphier_cache_data *data; 331 u32 level, cache_size; 332 struct device_node *next_np; 333 int ret = 0; 334 335 if (!of_match_node(uniphier_cache_match, np)) { 336 pr_err("L%d: not compatible with uniphier cache\n", 337 *cache_level); 338 return -EINVAL; 339 } 340 341 if (of_property_read_u32(np, "cache-level", &level)) { 342 pr_err("L%d: cache-level is not specified\n", *cache_level); 343 return -EINVAL; 344 } 345 346 if (level != *cache_level) { 347 pr_err("L%d: cache-level is unexpected value %d\n", 348 *cache_level, level); 349 return -EINVAL; 350 } 351 352 if (!of_property_read_bool(np, "cache-unified")) { 353 pr_err("L%d: cache-unified is not specified\n", *cache_level); 354 return -EINVAL; 355 } 356 357 data = kzalloc(sizeof(*data), GFP_KERNEL); 358 if (!data) 359 return -ENOMEM; 360 361 if (of_property_read_u32(np, "cache-line-size", &data->line_size) || 362 !is_power_of_2(data->line_size)) { 363 pr_err("L%d: cache-line-size is unspecified or invalid\n", 364 *cache_level); 365 ret = -EINVAL; 366 goto err; 367 } 368 369 if (of_property_read_u32(np, "cache-sets", &data->nsets) || 370 !is_power_of_2(data->nsets)) { 371 pr_err("L%d: cache-sets is unspecified or invalid\n", 372 *cache_level); 373 ret = -EINVAL; 374 goto err; 375 } 376 377 if (of_property_read_u32(np, "cache-size", &cache_size) || 378 cache_size == 0 || cache_size % (data->nsets * data->line_size)) { 379 pr_err("L%d: cache-size is unspecified or invalid\n", 380 *cache_level); 381 ret = -EINVAL; 382 goto err; 383 } 384 385 data->way_present_mask = 386 ((u32)1 << cache_size / data->nsets / data->line_size) - 1; 387 388 data->ctrl_base = of_iomap(np, 0); 389 if (!data->ctrl_base) { 390 pr_err("L%d: failed to map control register\n", *cache_level); 391 ret = -ENOMEM; 392 goto err; 393 } 394 395 data->rev_base = of_iomap(np, 1); 396 if (!data->rev_base) { 397 pr_err("L%d: failed to map revision register\n", *cache_level); 398 ret = -ENOMEM; 399 goto err; 400 } 401 402 data->op_base = of_iomap(np, 2); 403 if (!data->op_base) { 404 pr_err("L%d: failed to map operation register\n", *cache_level); 405 ret = -ENOMEM; 406 goto err; 407 } 408 409 data->way_ctrl_base = data->ctrl_base + 0xc00; 410 411 if (*cache_level == 2) { 412 u32 revision = readl(data->rev_base + UNIPHIER_SSCID); 413 /* 414 * The size of range operation is limited to (1 << 22) or less 415 * for PH-sLD8 or older SoCs. 416 */ 417 if (revision <= 0x16) 418 data->range_op_max_size = (u32)1 << 22; 419 420 /* 421 * Unfortunatly, the offset address of active way control base 422 * varies from SoC to SoC. 423 */ 424 switch (revision) { 425 case 0x11: /* sLD3 */ 426 data->way_ctrl_base = data->ctrl_base + 0x870; 427 break; 428 case 0x12: /* LD4 */ 429 case 0x16: /* sld8 */ 430 data->way_ctrl_base = data->ctrl_base + 0x840; 431 break; 432 default: 433 break; 434 } 435 } 436 437 data->range_op_max_size -= data->line_size; 438 439 INIT_LIST_HEAD(&data->list); 440 list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */ 441 442 /* 443 * OK, this level has been successfully initialized. Look for the next 444 * level cache. Do not roll back even if the initialization of the 445 * next level cache fails because we want to continue with available 446 * cache levels. 447 */ 448 next_np = of_find_next_cache_node(np); 449 if (next_np) { 450 (*cache_level)++; 451 ret = __uniphier_cache_init(next_np, cache_level); 452 } 453 of_node_put(next_np); 454 455 return ret; 456 err: 457 iounmap(data->op_base); 458 iounmap(data->rev_base); 459 iounmap(data->ctrl_base); 460 kfree(data); 461 462 return ret; 463 } 464 465 int __init uniphier_cache_init(void) 466 { 467 struct device_node *np = NULL; 468 unsigned int cache_level; 469 int ret = 0; 470 471 /* look for level 2 cache */ 472 while ((np = of_find_matching_node(np, uniphier_cache_match))) 473 if (!of_property_read_u32(np, "cache-level", &cache_level) && 474 cache_level == 2) 475 break; 476 477 if (!np) 478 return -ENODEV; 479 480 ret = __uniphier_cache_init(np, &cache_level); 481 of_node_put(np); 482 483 if (ret) { 484 /* 485 * Error out iif L2 initialization fails. Continue with any 486 * error on L3 or outer because they are optional. 487 */ 488 if (cache_level == 2) { 489 pr_err("failed to initialize L2 cache\n"); 490 return ret; 491 } 492 493 cache_level--; 494 ret = 0; 495 } 496 497 outer_cache.inv_range = uniphier_cache_inv_range; 498 outer_cache.clean_range = uniphier_cache_clean_range; 499 outer_cache.flush_range = uniphier_cache_flush_range; 500 outer_cache.flush_all = uniphier_cache_flush_all; 501 outer_cache.disable = uniphier_cache_disable; 502 outer_cache.sync = uniphier_cache_sync; 503 504 uniphier_cache_enable(); 505 506 pr_info("enabled outer cache (cache level: %d)\n", cache_level); 507 508 return ret; 509 } 510