1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * You should have received a copy of the GNU General Public License 12 * along with this program; if not, write to the Free Software 13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 14 * 15 * Copyright (C) 2013 Freescale Semiconductor, Inc. 16 * 17 */ 18 19 #define pr_fmt(fmt) "fsl-pamu: %s: " fmt, __func__ 20 21 #include "fsl_pamu.h" 22 23 #include <linux/fsl/guts.h> 24 #include <linux/interrupt.h> 25 #include <linux/genalloc.h> 26 27 #include <asm/mpc85xx.h> 28 29 /* define indexes for each operation mapping scenario */ 30 #define OMI_QMAN 0x00 31 #define OMI_FMAN 0x01 32 #define OMI_QMAN_PRIV 0x02 33 #define OMI_CAAM 0x03 34 35 #define make64(high, low) (((u64)(high) << 32) | (low)) 36 37 struct pamu_isr_data { 38 void __iomem *pamu_reg_base; /* Base address of PAMU regs */ 39 unsigned int count; /* The number of PAMUs */ 40 }; 41 42 static struct paace *ppaact; 43 static struct paace *spaact; 44 45 /* 46 * Table for matching compatible strings, for device tree 47 * guts node, for QorIQ SOCs. 48 * "fsl,qoriq-device-config-2.0" corresponds to T4 & B4 49 * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" 50 * string would be used. 51 */ 52 static const struct of_device_id guts_device_ids[] = { 53 { .compatible = "fsl,qoriq-device-config-1.0", }, 54 { .compatible = "fsl,qoriq-device-config-2.0", }, 55 {} 56 }; 57 58 /* 59 * Table for matching compatible strings, for device tree 60 * L3 cache controller node. 61 * "fsl,t4240-l3-cache-controller" corresponds to T4, 62 * "fsl,b4860-l3-cache-controller" corresponds to B4 & 63 * "fsl,p4080-l3-cache-controller" corresponds to other, 64 * SOCs. 65 */ 66 static const struct of_device_id l3_device_ids[] = { 67 { .compatible = "fsl,t4240-l3-cache-controller", }, 68 { .compatible = "fsl,b4860-l3-cache-controller", }, 69 { .compatible = "fsl,p4080-l3-cache-controller", }, 70 {} 71 }; 72 73 /* maximum subwindows permitted per liodn */ 74 static u32 max_subwindow_count; 75 76 /* Pool for fspi allocation */ 77 static struct gen_pool *spaace_pool; 78 79 /** 80 * pamu_get_max_subwin_cnt() - Return the maximum supported 81 * subwindow count per liodn. 82 * 83 */ 84 u32 pamu_get_max_subwin_cnt(void) 85 { 86 return max_subwindow_count; 87 } 88 89 /** 90 * pamu_get_ppaace() - Return the primary PACCE 91 * @liodn: liodn PAACT index for desired PAACE 92 * 93 * Returns the ppace pointer upon success else return 94 * null. 95 */ 96 static struct paace *pamu_get_ppaace(int liodn) 97 { 98 if (!ppaact || liodn >= PAACE_NUMBER_ENTRIES) { 99 pr_debug("PPAACT doesn't exist\n"); 100 return NULL; 101 } 102 103 return &ppaact[liodn]; 104 } 105 106 /** 107 * pamu_enable_liodn() - Set valid bit of PACCE 108 * @liodn: liodn PAACT index for desired PAACE 109 * 110 * Returns 0 upon success else error code < 0 returned 111 */ 112 int pamu_enable_liodn(int liodn) 113 { 114 struct paace *ppaace; 115 116 ppaace = pamu_get_ppaace(liodn); 117 if (!ppaace) { 118 pr_debug("Invalid primary paace entry\n"); 119 return -ENOENT; 120 } 121 122 if (!get_bf(ppaace->addr_bitfields, PPAACE_AF_WSE)) { 123 pr_debug("liodn %d not configured\n", liodn); 124 return -EINVAL; 125 } 126 127 /* Ensure that all other stores to the ppaace complete first */ 128 mb(); 129 130 set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); 131 mb(); 132 133 return 0; 134 } 135 136 /** 137 * pamu_disable_liodn() - Clears valid bit of PACCE 138 * @liodn: liodn PAACT index for desired PAACE 139 * 140 * Returns 0 upon success else error code < 0 returned 141 */ 142 int pamu_disable_liodn(int liodn) 143 { 144 struct paace *ppaace; 145 146 ppaace = pamu_get_ppaace(liodn); 147 if (!ppaace) { 148 pr_debug("Invalid primary paace entry\n"); 149 return -ENOENT; 150 } 151 152 set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); 153 mb(); 154 155 return 0; 156 } 157 158 /* Derive the window size encoding for a particular PAACE entry */ 159 static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size) 160 { 161 /* Bug if not a power of 2 */ 162 BUG_ON(addrspace_size & (addrspace_size - 1)); 163 164 /* window size is 2^(WSE+1) bytes */ 165 return fls64(addrspace_size) - 2; 166 } 167 168 /* Derive the PAACE window count encoding for the subwindow count */ 169 static unsigned int map_subwindow_cnt_to_wce(u32 subwindow_cnt) 170 { 171 /* window count is 2^(WCE+1) bytes */ 172 return __ffs(subwindow_cnt) - 1; 173 } 174 175 /* 176 * Set the PAACE type as primary and set the coherency required domain 177 * attribute 178 */ 179 static void pamu_init_ppaace(struct paace *ppaace) 180 { 181 set_bf(ppaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_PRIMARY); 182 183 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 184 PAACE_M_COHERENCE_REQ); 185 } 186 187 /* 188 * Set the PAACE type as secondary and set the coherency required domain 189 * attribute. 190 */ 191 static void pamu_init_spaace(struct paace *spaace) 192 { 193 set_bf(spaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_SECONDARY); 194 set_bf(spaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 195 PAACE_M_COHERENCE_REQ); 196 } 197 198 /* 199 * Return the spaace (corresponding to the secondary window index) 200 * for a particular ppaace. 201 */ 202 static struct paace *pamu_get_spaace(struct paace *paace, u32 wnum) 203 { 204 u32 subwin_cnt; 205 struct paace *spaace = NULL; 206 207 subwin_cnt = 1UL << (get_bf(paace->impl_attr, PAACE_IA_WCE) + 1); 208 209 if (wnum < subwin_cnt) 210 spaace = &spaact[paace->fspi + wnum]; 211 else 212 pr_debug("secondary paace out of bounds\n"); 213 214 return spaace; 215 } 216 217 /** 218 * pamu_get_fspi_and_allocate() - Allocates fspi index and reserves subwindows 219 * required for primary PAACE in the secondary 220 * PAACE table. 221 * @subwin_cnt: Number of subwindows to be reserved. 222 * 223 * A PPAACE entry may have a number of associated subwindows. A subwindow 224 * corresponds to a SPAACE entry in the SPAACT table. Each PAACE entry stores 225 * the index (fspi) of the first SPAACE entry in the SPAACT table. This 226 * function returns the index of the first SPAACE entry. The remaining 227 * SPAACE entries are reserved contiguously from that index. 228 * 229 * Returns a valid fspi index in the range of 0 - SPAACE_NUMBER_ENTRIES on success. 230 * If no SPAACE entry is available or the allocator can not reserve the required 231 * number of contiguous entries function returns ULONG_MAX indicating a failure. 232 * 233 */ 234 static unsigned long pamu_get_fspi_and_allocate(u32 subwin_cnt) 235 { 236 unsigned long spaace_addr; 237 238 spaace_addr = gen_pool_alloc(spaace_pool, subwin_cnt * sizeof(struct paace)); 239 if (!spaace_addr) 240 return ULONG_MAX; 241 242 return (spaace_addr - (unsigned long)spaact) / (sizeof(struct paace)); 243 } 244 245 /* Release the subwindows reserved for a particular LIODN */ 246 void pamu_free_subwins(int liodn) 247 { 248 struct paace *ppaace; 249 u32 subwin_cnt, size; 250 251 ppaace = pamu_get_ppaace(liodn); 252 if (!ppaace) { 253 pr_debug("Invalid liodn entry\n"); 254 return; 255 } 256 257 if (get_bf(ppaace->addr_bitfields, PPAACE_AF_MW)) { 258 subwin_cnt = 1UL << (get_bf(ppaace->impl_attr, PAACE_IA_WCE) + 1); 259 size = (subwin_cnt - 1) * sizeof(struct paace); 260 gen_pool_free(spaace_pool, (unsigned long)&spaact[ppaace->fspi], size); 261 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); 262 } 263 } 264 265 /* 266 * Function used for updating stash destination for the coressponding 267 * LIODN. 268 */ 269 int pamu_update_paace_stash(int liodn, u32 subwin, u32 value) 270 { 271 struct paace *paace; 272 273 paace = pamu_get_ppaace(liodn); 274 if (!paace) { 275 pr_debug("Invalid liodn entry\n"); 276 return -ENOENT; 277 } 278 if (subwin) { 279 paace = pamu_get_spaace(paace, subwin - 1); 280 if (!paace) 281 return -ENOENT; 282 } 283 set_bf(paace->impl_attr, PAACE_IA_CID, value); 284 285 mb(); 286 287 return 0; 288 } 289 290 /* Disable a subwindow corresponding to the LIODN */ 291 int pamu_disable_spaace(int liodn, u32 subwin) 292 { 293 struct paace *paace; 294 295 paace = pamu_get_ppaace(liodn); 296 if (!paace) { 297 pr_debug("Invalid liodn entry\n"); 298 return -ENOENT; 299 } 300 if (subwin) { 301 paace = pamu_get_spaace(paace, subwin - 1); 302 if (!paace) 303 return -ENOENT; 304 set_bf(paace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); 305 } else { 306 set_bf(paace->addr_bitfields, PAACE_AF_AP, 307 PAACE_AP_PERMS_DENIED); 308 } 309 310 mb(); 311 312 return 0; 313 } 314 315 /** 316 * pamu_config_paace() - Sets up PPAACE entry for specified liodn 317 * 318 * @liodn: Logical IO device number 319 * @win_addr: starting address of DSA window 320 * @win-size: size of DSA window 321 * @omi: Operation mapping index -- if ~omi == 0 then omi not defined 322 * @rpn: real (true physical) page number 323 * @stashid: cache stash id for associated cpu -- if ~stashid == 0 then 324 * stashid not defined 325 * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then 326 * snoopid not defined 327 * @subwin_cnt: number of sub-windows 328 * @prot: window permissions 329 * 330 * Returns 0 upon success else error code < 0 returned 331 */ 332 int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, 333 u32 omi, unsigned long rpn, u32 snoopid, u32 stashid, 334 u32 subwin_cnt, int prot) 335 { 336 struct paace *ppaace; 337 unsigned long fspi; 338 339 if ((win_size & (win_size - 1)) || win_size < PAMU_PAGE_SIZE) { 340 pr_debug("window size too small or not a power of two %pa\n", 341 &win_size); 342 return -EINVAL; 343 } 344 345 if (win_addr & (win_size - 1)) { 346 pr_debug("window address is not aligned with window size\n"); 347 return -EINVAL; 348 } 349 350 ppaace = pamu_get_ppaace(liodn); 351 if (!ppaace) 352 return -ENOENT; 353 354 /* window size is 2^(WSE+1) bytes */ 355 set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 356 map_addrspace_size_to_wse(win_size)); 357 358 pamu_init_ppaace(ppaace); 359 360 ppaace->wbah = win_addr >> (PAMU_PAGE_SHIFT + 20); 361 set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 362 (win_addr >> PAMU_PAGE_SHIFT)); 363 364 /* set up operation mapping if it's configured */ 365 if (omi < OME_NUMBER_ENTRIES) { 366 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 367 ppaace->op_encode.index_ot.omi = omi; 368 } else if (~omi != 0) { 369 pr_debug("bad operation mapping index: %d\n", omi); 370 return -EINVAL; 371 } 372 373 /* configure stash id */ 374 if (~stashid != 0) 375 set_bf(ppaace->impl_attr, PAACE_IA_CID, stashid); 376 377 /* configure snoop id */ 378 if (~snoopid != 0) 379 ppaace->domain_attr.to_host.snpid = snoopid; 380 381 if (subwin_cnt) { 382 /* The first entry is in the primary PAACE instead */ 383 fspi = pamu_get_fspi_and_allocate(subwin_cnt - 1); 384 if (fspi == ULONG_MAX) { 385 pr_debug("spaace indexes exhausted\n"); 386 return -EINVAL; 387 } 388 389 /* window count is 2^(WCE+1) bytes */ 390 set_bf(ppaace->impl_attr, PAACE_IA_WCE, 391 map_subwindow_cnt_to_wce(subwin_cnt)); 392 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0x1); 393 ppaace->fspi = fspi; 394 } else { 395 set_bf(ppaace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); 396 ppaace->twbah = rpn >> 20; 397 set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, rpn); 398 set_bf(ppaace->addr_bitfields, PAACE_AF_AP, prot); 399 set_bf(ppaace->impl_attr, PAACE_IA_WCE, 0); 400 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); 401 } 402 mb(); 403 404 return 0; 405 } 406 407 /** 408 * pamu_config_spaace() - Sets up SPAACE entry for specified subwindow 409 * 410 * @liodn: Logical IO device number 411 * @subwin_cnt: number of sub-windows associated with dma-window 412 * @subwin: subwindow index 413 * @subwin_size: size of subwindow 414 * @omi: Operation mapping index 415 * @rpn: real (true physical) page number 416 * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then 417 * snoopid not defined 418 * @stashid: cache stash id for associated cpu 419 * @enable: enable/disable subwindow after reconfiguration 420 * @prot: sub window permissions 421 * 422 * Returns 0 upon success else error code < 0 returned 423 */ 424 int pamu_config_spaace(int liodn, u32 subwin_cnt, u32 subwin, 425 phys_addr_t subwin_size, u32 omi, unsigned long rpn, 426 u32 snoopid, u32 stashid, int enable, int prot) 427 { 428 struct paace *paace; 429 430 /* setup sub-windows */ 431 if (!subwin_cnt) { 432 pr_debug("Invalid subwindow count\n"); 433 return -EINVAL; 434 } 435 436 paace = pamu_get_ppaace(liodn); 437 if (subwin > 0 && subwin < subwin_cnt && paace) { 438 paace = pamu_get_spaace(paace, subwin - 1); 439 440 if (paace && !(paace->addr_bitfields & PAACE_V_VALID)) { 441 pamu_init_spaace(paace); 442 set_bf(paace->addr_bitfields, SPAACE_AF_LIODN, liodn); 443 } 444 } 445 446 if (!paace) { 447 pr_debug("Invalid liodn entry\n"); 448 return -ENOENT; 449 } 450 451 if ((subwin_size & (subwin_size - 1)) || subwin_size < PAMU_PAGE_SIZE) { 452 pr_debug("subwindow size out of range, or not a power of 2\n"); 453 return -EINVAL; 454 } 455 456 if (rpn == ULONG_MAX) { 457 pr_debug("real page number out of range\n"); 458 return -EINVAL; 459 } 460 461 /* window size is 2^(WSE+1) bytes */ 462 set_bf(paace->win_bitfields, PAACE_WIN_SWSE, 463 map_addrspace_size_to_wse(subwin_size)); 464 465 set_bf(paace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); 466 paace->twbah = rpn >> 20; 467 set_bf(paace->win_bitfields, PAACE_WIN_TWBAL, rpn); 468 set_bf(paace->addr_bitfields, PAACE_AF_AP, prot); 469 470 /* configure snoop id */ 471 if (~snoopid != 0) 472 paace->domain_attr.to_host.snpid = snoopid; 473 474 /* set up operation mapping if it's configured */ 475 if (omi < OME_NUMBER_ENTRIES) { 476 set_bf(paace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 477 paace->op_encode.index_ot.omi = omi; 478 } else if (~omi != 0) { 479 pr_debug("bad operation mapping index: %d\n", omi); 480 return -EINVAL; 481 } 482 483 if (~stashid != 0) 484 set_bf(paace->impl_attr, PAACE_IA_CID, stashid); 485 486 smp_wmb(); 487 488 if (enable) 489 set_bf(paace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); 490 491 mb(); 492 493 return 0; 494 } 495 496 /** 497 * get_ome_index() - Returns the index in the operation mapping table 498 * for device. 499 * @*omi_index: pointer for storing the index value 500 * 501 */ 502 void get_ome_index(u32 *omi_index, struct device *dev) 503 { 504 if (of_device_is_compatible(dev->of_node, "fsl,qman-portal")) 505 *omi_index = OMI_QMAN; 506 if (of_device_is_compatible(dev->of_node, "fsl,qman")) 507 *omi_index = OMI_QMAN_PRIV; 508 } 509 510 /** 511 * get_stash_id - Returns stash destination id corresponding to a 512 * cache type and vcpu. 513 * @stash_dest_hint: L1, L2 or L3 514 * @vcpu: vpcu target for a particular cache type. 515 * 516 * Returs stash on success or ~(u32)0 on failure. 517 * 518 */ 519 u32 get_stash_id(u32 stash_dest_hint, u32 vcpu) 520 { 521 const u32 *prop; 522 struct device_node *node; 523 u32 cache_level; 524 int len, found = 0; 525 int i; 526 527 /* Fastpath, exit early if L3/CPC cache is target for stashing */ 528 if (stash_dest_hint == PAMU_ATTR_CACHE_L3) { 529 node = of_find_matching_node(NULL, l3_device_ids); 530 if (node) { 531 prop = of_get_property(node, "cache-stash-id", NULL); 532 if (!prop) { 533 pr_debug("missing cache-stash-id at %s\n", 534 node->full_name); 535 of_node_put(node); 536 return ~(u32)0; 537 } 538 of_node_put(node); 539 return be32_to_cpup(prop); 540 } 541 return ~(u32)0; 542 } 543 544 for_each_node_by_type(node, "cpu") { 545 prop = of_get_property(node, "reg", &len); 546 for (i = 0; i < len / sizeof(u32); i++) { 547 if (be32_to_cpup(&prop[i]) == vcpu) { 548 found = 1; 549 goto found_cpu_node; 550 } 551 } 552 } 553 found_cpu_node: 554 555 /* find the hwnode that represents the cache */ 556 for (cache_level = PAMU_ATTR_CACHE_L1; (cache_level < PAMU_ATTR_CACHE_L3) && found; cache_level++) { 557 if (stash_dest_hint == cache_level) { 558 prop = of_get_property(node, "cache-stash-id", NULL); 559 if (!prop) { 560 pr_debug("missing cache-stash-id at %s\n", 561 node->full_name); 562 of_node_put(node); 563 return ~(u32)0; 564 } 565 of_node_put(node); 566 return be32_to_cpup(prop); 567 } 568 569 prop = of_get_property(node, "next-level-cache", NULL); 570 if (!prop) { 571 pr_debug("can't find next-level-cache at %s\n", 572 node->full_name); 573 of_node_put(node); 574 return ~(u32)0; /* can't traverse any further */ 575 } 576 of_node_put(node); 577 578 /* advance to next node in cache hierarchy */ 579 node = of_find_node_by_phandle(*prop); 580 if (!node) { 581 pr_debug("Invalid node for cache hierarchy\n"); 582 return ~(u32)0; 583 } 584 } 585 586 pr_debug("stash dest not found for %d on vcpu %d\n", 587 stash_dest_hint, vcpu); 588 return ~(u32)0; 589 } 590 591 /* Identify if the PAACT table entry belongs to QMAN, BMAN or QMAN Portal */ 592 #define QMAN_PAACE 1 593 #define QMAN_PORTAL_PAACE 2 594 #define BMAN_PAACE 3 595 596 /** 597 * Setup operation mapping and stash destinations for QMAN and QMAN portal. 598 * Memory accesses to QMAN and BMAN private memory need not be coherent, so 599 * clear the PAACE entry coherency attribute for them. 600 */ 601 static void setup_qbman_paace(struct paace *ppaace, int paace_type) 602 { 603 switch (paace_type) { 604 case QMAN_PAACE: 605 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 606 ppaace->op_encode.index_ot.omi = OMI_QMAN_PRIV; 607 /* setup QMAN Private data stashing for the L3 cache */ 608 set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); 609 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 610 0); 611 break; 612 case QMAN_PORTAL_PAACE: 613 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 614 ppaace->op_encode.index_ot.omi = OMI_QMAN; 615 /* Set DQRR and Frame stashing for the L3 cache */ 616 set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); 617 break; 618 case BMAN_PAACE: 619 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 620 0); 621 break; 622 } 623 } 624 625 /** 626 * Setup the operation mapping table for various devices. This is a static 627 * table where each table index corresponds to a particular device. PAMU uses 628 * this table to translate device transaction to appropriate corenet 629 * transaction. 630 */ 631 static void setup_omt(struct ome *omt) 632 { 633 struct ome *ome; 634 635 /* Configure OMI_QMAN */ 636 ome = &omt[OMI_QMAN]; 637 638 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; 639 ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; 640 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 641 ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSAO; 642 643 ome->moe[IOE_DIRECT0_IDX] = EOE_VALID | EOE_LDEC; 644 ome->moe[IOE_DIRECT1_IDX] = EOE_VALID | EOE_LDECPE; 645 646 /* Configure OMI_FMAN */ 647 ome = &omt[OMI_FMAN]; 648 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; 649 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 650 651 /* Configure OMI_QMAN private */ 652 ome = &omt[OMI_QMAN_PRIV]; 653 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; 654 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 655 ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; 656 ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSA; 657 658 /* Configure OMI_CAAM */ 659 ome = &omt[OMI_CAAM]; 660 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; 661 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 662 } 663 664 /* 665 * Get the maximum number of PAACT table entries 666 * and subwindows supported by PAMU 667 */ 668 static void get_pamu_cap_values(unsigned long pamu_reg_base) 669 { 670 u32 pc_val; 671 672 pc_val = in_be32((u32 *)(pamu_reg_base + PAMU_PC3)); 673 /* Maximum number of subwindows per liodn */ 674 max_subwindow_count = 1 << (1 + PAMU_PC3_MWCE(pc_val)); 675 } 676 677 /* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ 678 static int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, 679 phys_addr_t ppaact_phys, phys_addr_t spaact_phys, 680 phys_addr_t omt_phys) 681 { 682 u32 *pc; 683 struct pamu_mmap_regs *pamu_regs; 684 685 pc = (u32 *) (pamu_reg_base + PAMU_PC); 686 pamu_regs = (struct pamu_mmap_regs *) 687 (pamu_reg_base + PAMU_MMAP_REGS_BASE); 688 689 /* set up pointers to corenet control blocks */ 690 691 out_be32(&pamu_regs->ppbah, upper_32_bits(ppaact_phys)); 692 out_be32(&pamu_regs->ppbal, lower_32_bits(ppaact_phys)); 693 ppaact_phys = ppaact_phys + PAACT_SIZE; 694 out_be32(&pamu_regs->pplah, upper_32_bits(ppaact_phys)); 695 out_be32(&pamu_regs->pplal, lower_32_bits(ppaact_phys)); 696 697 out_be32(&pamu_regs->spbah, upper_32_bits(spaact_phys)); 698 out_be32(&pamu_regs->spbal, lower_32_bits(spaact_phys)); 699 spaact_phys = spaact_phys + SPAACT_SIZE; 700 out_be32(&pamu_regs->splah, upper_32_bits(spaact_phys)); 701 out_be32(&pamu_regs->splal, lower_32_bits(spaact_phys)); 702 703 out_be32(&pamu_regs->obah, upper_32_bits(omt_phys)); 704 out_be32(&pamu_regs->obal, lower_32_bits(omt_phys)); 705 omt_phys = omt_phys + OMT_SIZE; 706 out_be32(&pamu_regs->olah, upper_32_bits(omt_phys)); 707 out_be32(&pamu_regs->olal, lower_32_bits(omt_phys)); 708 709 /* 710 * set PAMU enable bit, 711 * allow ppaact & omt to be cached 712 * & enable PAMU access violation interrupts. 713 */ 714 715 out_be32((u32 *)(pamu_reg_base + PAMU_PICS), 716 PAMU_ACCESS_VIOLATION_ENABLE); 717 out_be32(pc, PAMU_PC_PE | PAMU_PC_OCE | PAMU_PC_SPCC | PAMU_PC_PPCC); 718 return 0; 719 } 720 721 /* Enable all device LIODNS */ 722 static void setup_liodns(void) 723 { 724 int i, len; 725 struct paace *ppaace; 726 struct device_node *node = NULL; 727 const u32 *prop; 728 729 for_each_node_with_property(node, "fsl,liodn") { 730 prop = of_get_property(node, "fsl,liodn", &len); 731 for (i = 0; i < len / sizeof(u32); i++) { 732 int liodn; 733 734 liodn = be32_to_cpup(&prop[i]); 735 if (liodn >= PAACE_NUMBER_ENTRIES) { 736 pr_debug("Invalid LIODN value %d\n", liodn); 737 continue; 738 } 739 ppaace = pamu_get_ppaace(liodn); 740 pamu_init_ppaace(ppaace); 741 /* window size is 2^(WSE+1) bytes */ 742 set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 35); 743 ppaace->wbah = 0; 744 set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); 745 set_bf(ppaace->impl_attr, PAACE_IA_ATM, 746 PAACE_ATM_NO_XLATE); 747 set_bf(ppaace->addr_bitfields, PAACE_AF_AP, 748 PAACE_AP_PERMS_ALL); 749 if (of_device_is_compatible(node, "fsl,qman-portal")) 750 setup_qbman_paace(ppaace, QMAN_PORTAL_PAACE); 751 if (of_device_is_compatible(node, "fsl,qman")) 752 setup_qbman_paace(ppaace, QMAN_PAACE); 753 if (of_device_is_compatible(node, "fsl,bman")) 754 setup_qbman_paace(ppaace, BMAN_PAACE); 755 mb(); 756 pamu_enable_liodn(liodn); 757 } 758 } 759 } 760 761 static irqreturn_t pamu_av_isr(int irq, void *arg) 762 { 763 struct pamu_isr_data *data = arg; 764 phys_addr_t phys; 765 unsigned int i, j, ret; 766 767 pr_emerg("access violation interrupt\n"); 768 769 for (i = 0; i < data->count; i++) { 770 void __iomem *p = data->pamu_reg_base + i * PAMU_OFFSET; 771 u32 pics = in_be32(p + PAMU_PICS); 772 773 if (pics & PAMU_ACCESS_VIOLATION_STAT) { 774 u32 avs1 = in_be32(p + PAMU_AVS1); 775 struct paace *paace; 776 777 pr_emerg("POES1=%08x\n", in_be32(p + PAMU_POES1)); 778 pr_emerg("POES2=%08x\n", in_be32(p + PAMU_POES2)); 779 pr_emerg("AVS1=%08x\n", avs1); 780 pr_emerg("AVS2=%08x\n", in_be32(p + PAMU_AVS2)); 781 pr_emerg("AVA=%016llx\n", 782 make64(in_be32(p + PAMU_AVAH), 783 in_be32(p + PAMU_AVAL))); 784 pr_emerg("UDAD=%08x\n", in_be32(p + PAMU_UDAD)); 785 pr_emerg("POEA=%016llx\n", 786 make64(in_be32(p + PAMU_POEAH), 787 in_be32(p + PAMU_POEAL))); 788 789 phys = make64(in_be32(p + PAMU_POEAH), 790 in_be32(p + PAMU_POEAL)); 791 792 /* Assume that POEA points to a PAACE */ 793 if (phys) { 794 u32 *paace = phys_to_virt(phys); 795 796 /* Only the first four words are relevant */ 797 for (j = 0; j < 4; j++) 798 pr_emerg("PAACE[%u]=%08x\n", 799 j, in_be32(paace + j)); 800 } 801 802 /* clear access violation condition */ 803 out_be32(p + PAMU_AVS1, avs1 & PAMU_AV_MASK); 804 paace = pamu_get_ppaace(avs1 >> PAMU_AVS1_LIODN_SHIFT); 805 BUG_ON(!paace); 806 /* check if we got a violation for a disabled LIODN */ 807 if (!get_bf(paace->addr_bitfields, PAACE_AF_V)) { 808 /* 809 * As per hardware erratum A-003638, access 810 * violation can be reported for a disabled 811 * LIODN. If we hit that condition, disable 812 * access violation reporting. 813 */ 814 pics &= ~PAMU_ACCESS_VIOLATION_ENABLE; 815 } else { 816 /* Disable the LIODN */ 817 ret = pamu_disable_liodn(avs1 >> PAMU_AVS1_LIODN_SHIFT); 818 BUG_ON(ret); 819 pr_emerg("Disabling liodn %x\n", 820 avs1 >> PAMU_AVS1_LIODN_SHIFT); 821 } 822 out_be32((p + PAMU_PICS), pics); 823 } 824 } 825 826 return IRQ_HANDLED; 827 } 828 829 #define LAWAR_EN 0x80000000 830 #define LAWAR_TARGET_MASK 0x0FF00000 831 #define LAWAR_TARGET_SHIFT 20 832 #define LAWAR_SIZE_MASK 0x0000003F 833 #define LAWAR_CSDID_MASK 0x000FF000 834 #define LAWAR_CSDID_SHIFT 12 835 836 #define LAW_SIZE_4K 0xb 837 838 struct ccsr_law { 839 u32 lawbarh; /* LAWn base address high */ 840 u32 lawbarl; /* LAWn base address low */ 841 u32 lawar; /* LAWn attributes */ 842 u32 reserved; 843 }; 844 845 /* 846 * Create a coherence subdomain for a given memory block. 847 */ 848 static int create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) 849 { 850 struct device_node *np; 851 const __be32 *iprop; 852 void __iomem *lac = NULL; /* Local Access Control registers */ 853 struct ccsr_law __iomem *law; 854 void __iomem *ccm = NULL; 855 u32 __iomem *csdids; 856 unsigned int i, num_laws, num_csds; 857 u32 law_target = 0; 858 u32 csd_id = 0; 859 int ret = 0; 860 861 np = of_find_compatible_node(NULL, NULL, "fsl,corenet-law"); 862 if (!np) 863 return -ENODEV; 864 865 iprop = of_get_property(np, "fsl,num-laws", NULL); 866 if (!iprop) { 867 ret = -ENODEV; 868 goto error; 869 } 870 871 num_laws = be32_to_cpup(iprop); 872 if (!num_laws) { 873 ret = -ENODEV; 874 goto error; 875 } 876 877 lac = of_iomap(np, 0); 878 if (!lac) { 879 ret = -ENODEV; 880 goto error; 881 } 882 883 /* LAW registers are at offset 0xC00 */ 884 law = lac + 0xC00; 885 886 of_node_put(np); 887 888 np = of_find_compatible_node(NULL, NULL, "fsl,corenet-cf"); 889 if (!np) { 890 ret = -ENODEV; 891 goto error; 892 } 893 894 iprop = of_get_property(np, "fsl,ccf-num-csdids", NULL); 895 if (!iprop) { 896 ret = -ENODEV; 897 goto error; 898 } 899 900 num_csds = be32_to_cpup(iprop); 901 if (!num_csds) { 902 ret = -ENODEV; 903 goto error; 904 } 905 906 ccm = of_iomap(np, 0); 907 if (!ccm) { 908 ret = -ENOMEM; 909 goto error; 910 } 911 912 /* The undocumented CSDID registers are at offset 0x600 */ 913 csdids = ccm + 0x600; 914 915 of_node_put(np); 916 np = NULL; 917 918 /* Find an unused coherence subdomain ID */ 919 for (csd_id = 0; csd_id < num_csds; csd_id++) { 920 if (!csdids[csd_id]) 921 break; 922 } 923 924 /* Store the Port ID in the (undocumented) proper CIDMRxx register */ 925 csdids[csd_id] = csd_port_id; 926 927 /* Find the DDR LAW that maps to our buffer. */ 928 for (i = 0; i < num_laws; i++) { 929 if (law[i].lawar & LAWAR_EN) { 930 phys_addr_t law_start, law_end; 931 932 law_start = make64(law[i].lawbarh, law[i].lawbarl); 933 law_end = law_start + 934 (2ULL << (law[i].lawar & LAWAR_SIZE_MASK)); 935 936 if (law_start <= phys && phys < law_end) { 937 law_target = law[i].lawar & LAWAR_TARGET_MASK; 938 break; 939 } 940 } 941 } 942 943 if (i == 0 || i == num_laws) { 944 /* This should never happen */ 945 ret = -ENOENT; 946 goto error; 947 } 948 949 /* Find a free LAW entry */ 950 while (law[--i].lawar & LAWAR_EN) { 951 if (i == 0) { 952 /* No higher priority LAW slots available */ 953 ret = -ENOENT; 954 goto error; 955 } 956 } 957 958 law[i].lawbarh = upper_32_bits(phys); 959 law[i].lawbarl = lower_32_bits(phys); 960 wmb(); 961 law[i].lawar = LAWAR_EN | law_target | (csd_id << LAWAR_CSDID_SHIFT) | 962 (LAW_SIZE_4K + get_order(size)); 963 wmb(); 964 965 error: 966 if (ccm) 967 iounmap(ccm); 968 969 if (lac) 970 iounmap(lac); 971 972 if (np) 973 of_node_put(np); 974 975 return ret; 976 } 977 978 /* 979 * Table of SVRs and the corresponding PORT_ID values. Port ID corresponds to a 980 * bit map of snoopers for a given range of memory mapped by a LAW. 981 * 982 * All future CoreNet-enabled SOCs will have this erratum(A-004510) fixed, so this 983 * table should never need to be updated. SVRs are guaranteed to be unique, so 984 * there is no worry that a future SOC will inadvertently have one of these 985 * values. 986 */ 987 static const struct { 988 u32 svr; 989 u32 port_id; 990 } port_id_map[] = { 991 {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */ 992 {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */ 993 {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */ 994 {(SVR_P2041 << 8) | 0x11, 0xFF000000}, /* P2041 1.1 */ 995 {(SVR_P3041 << 8) | 0x10, 0xFF000000}, /* P3041 1.0 */ 996 {(SVR_P3041 << 8) | 0x11, 0xFF000000}, /* P3041 1.1 */ 997 {(SVR_P4040 << 8) | 0x20, 0xFFF80000}, /* P4040 2.0 */ 998 {(SVR_P4080 << 8) | 0x20, 0xFFF80000}, /* P4080 2.0 */ 999 {(SVR_P5010 << 8) | 0x10, 0xFC000000}, /* P5010 1.0 */ 1000 {(SVR_P5010 << 8) | 0x20, 0xFC000000}, /* P5010 2.0 */ 1001 {(SVR_P5020 << 8) | 0x10, 0xFC000000}, /* P5020 1.0 */ 1002 {(SVR_P5021 << 8) | 0x10, 0xFF800000}, /* P5021 1.0 */ 1003 {(SVR_P5040 << 8) | 0x10, 0xFF800000}, /* P5040 1.0 */ 1004 }; 1005 1006 #define SVR_SECURITY 0x80000 /* The Security (E) bit */ 1007 1008 static int fsl_pamu_probe(struct platform_device *pdev) 1009 { 1010 struct device *dev = &pdev->dev; 1011 void __iomem *pamu_regs = NULL; 1012 struct ccsr_guts __iomem *guts_regs = NULL; 1013 u32 pamubypenr, pamu_counter; 1014 unsigned long pamu_reg_off; 1015 unsigned long pamu_reg_base; 1016 struct pamu_isr_data *data = NULL; 1017 struct device_node *guts_node; 1018 u64 size; 1019 struct page *p; 1020 int ret = 0; 1021 int irq; 1022 phys_addr_t ppaact_phys; 1023 phys_addr_t spaact_phys; 1024 struct ome *omt; 1025 phys_addr_t omt_phys; 1026 size_t mem_size = 0; 1027 unsigned int order = 0; 1028 u32 csd_port_id = 0; 1029 unsigned i; 1030 /* 1031 * enumerate all PAMUs and allocate and setup PAMU tables 1032 * for each of them, 1033 * NOTE : All PAMUs share the same LIODN tables. 1034 */ 1035 1036 pamu_regs = of_iomap(dev->of_node, 0); 1037 if (!pamu_regs) { 1038 dev_err(dev, "ioremap of PAMU node failed\n"); 1039 return -ENOMEM; 1040 } 1041 of_get_address(dev->of_node, 0, &size, NULL); 1042 1043 irq = irq_of_parse_and_map(dev->of_node, 0); 1044 if (irq == NO_IRQ) { 1045 dev_warn(dev, "no interrupts listed in PAMU node\n"); 1046 goto error; 1047 } 1048 1049 data = kzalloc(sizeof(*data), GFP_KERNEL); 1050 if (!data) { 1051 ret = -ENOMEM; 1052 goto error; 1053 } 1054 data->pamu_reg_base = pamu_regs; 1055 data->count = size / PAMU_OFFSET; 1056 1057 /* The ISR needs access to the regs, so we won't iounmap them */ 1058 ret = request_irq(irq, pamu_av_isr, 0, "pamu", data); 1059 if (ret < 0) { 1060 dev_err(dev, "error %i installing ISR for irq %i\n", ret, irq); 1061 goto error; 1062 } 1063 1064 guts_node = of_find_matching_node(NULL, guts_device_ids); 1065 if (!guts_node) { 1066 dev_err(dev, "could not find GUTS node %s\n", 1067 dev->of_node->full_name); 1068 ret = -ENODEV; 1069 goto error; 1070 } 1071 1072 guts_regs = of_iomap(guts_node, 0); 1073 of_node_put(guts_node); 1074 if (!guts_regs) { 1075 dev_err(dev, "ioremap of GUTS node failed\n"); 1076 ret = -ENODEV; 1077 goto error; 1078 } 1079 1080 /* read in the PAMU capability registers */ 1081 get_pamu_cap_values((unsigned long)pamu_regs); 1082 /* 1083 * To simplify the allocation of a coherency domain, we allocate the 1084 * PAACT and the OMT in the same memory buffer. Unfortunately, this 1085 * wastes more memory compared to allocating the buffers separately. 1086 */ 1087 /* Determine how much memory we need */ 1088 mem_size = (PAGE_SIZE << get_order(PAACT_SIZE)) + 1089 (PAGE_SIZE << get_order(SPAACT_SIZE)) + 1090 (PAGE_SIZE << get_order(OMT_SIZE)); 1091 order = get_order(mem_size); 1092 1093 p = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); 1094 if (!p) { 1095 dev_err(dev, "unable to allocate PAACT/SPAACT/OMT block\n"); 1096 ret = -ENOMEM; 1097 goto error; 1098 } 1099 1100 ppaact = page_address(p); 1101 ppaact_phys = page_to_phys(p); 1102 1103 /* Make sure the memory is naturally aligned */ 1104 if (ppaact_phys & ((PAGE_SIZE << order) - 1)) { 1105 dev_err(dev, "PAACT/OMT block is unaligned\n"); 1106 ret = -ENOMEM; 1107 goto error; 1108 } 1109 1110 spaact = (void *)ppaact + (PAGE_SIZE << get_order(PAACT_SIZE)); 1111 omt = (void *)spaact + (PAGE_SIZE << get_order(SPAACT_SIZE)); 1112 1113 dev_dbg(dev, "ppaact virt=%p phys=%pa\n", ppaact, &ppaact_phys); 1114 1115 /* Check to see if we need to implement the work-around on this SOC */ 1116 1117 /* Determine the Port ID for our coherence subdomain */ 1118 for (i = 0; i < ARRAY_SIZE(port_id_map); i++) { 1119 if (port_id_map[i].svr == (mfspr(SPRN_SVR) & ~SVR_SECURITY)) { 1120 csd_port_id = port_id_map[i].port_id; 1121 dev_dbg(dev, "found matching SVR %08x\n", 1122 port_id_map[i].svr); 1123 break; 1124 } 1125 } 1126 1127 if (csd_port_id) { 1128 dev_dbg(dev, "creating coherency subdomain at address %pa, size %zu, port id 0x%08x", 1129 &ppaact_phys, mem_size, csd_port_id); 1130 1131 ret = create_csd(ppaact_phys, mem_size, csd_port_id); 1132 if (ret) { 1133 dev_err(dev, "could not create coherence subdomain\n"); 1134 return ret; 1135 } 1136 } 1137 1138 spaact_phys = virt_to_phys(spaact); 1139 omt_phys = virt_to_phys(omt); 1140 1141 spaace_pool = gen_pool_create(ilog2(sizeof(struct paace)), -1); 1142 if (!spaace_pool) { 1143 ret = -ENOMEM; 1144 dev_err(dev, "Failed to allocate spaace gen pool\n"); 1145 goto error; 1146 } 1147 1148 ret = gen_pool_add(spaace_pool, (unsigned long)spaact, SPAACT_SIZE, -1); 1149 if (ret) 1150 goto error_genpool; 1151 1152 pamubypenr = in_be32(&guts_regs->pamubypenr); 1153 1154 for (pamu_reg_off = 0, pamu_counter = 0x80000000; pamu_reg_off < size; 1155 pamu_reg_off += PAMU_OFFSET, pamu_counter >>= 1) { 1156 1157 pamu_reg_base = (unsigned long)pamu_regs + pamu_reg_off; 1158 setup_one_pamu(pamu_reg_base, pamu_reg_off, ppaact_phys, 1159 spaact_phys, omt_phys); 1160 /* Disable PAMU bypass for this PAMU */ 1161 pamubypenr &= ~pamu_counter; 1162 } 1163 1164 setup_omt(omt); 1165 1166 /* Enable all relevant PAMU(s) */ 1167 out_be32(&guts_regs->pamubypenr, pamubypenr); 1168 1169 iounmap(guts_regs); 1170 1171 /* Enable DMA for the LIODNs in the device tree */ 1172 1173 setup_liodns(); 1174 1175 return 0; 1176 1177 error_genpool: 1178 gen_pool_destroy(spaace_pool); 1179 1180 error: 1181 if (irq != NO_IRQ) 1182 free_irq(irq, data); 1183 1184 if (data) { 1185 memset(data, 0, sizeof(struct pamu_isr_data)); 1186 kfree(data); 1187 } 1188 1189 if (pamu_regs) 1190 iounmap(pamu_regs); 1191 1192 if (guts_regs) 1193 iounmap(guts_regs); 1194 1195 if (ppaact) 1196 free_pages((unsigned long)ppaact, order); 1197 1198 ppaact = NULL; 1199 1200 return ret; 1201 } 1202 1203 static struct platform_driver fsl_of_pamu_driver = { 1204 .driver = { 1205 .name = "fsl-of-pamu", 1206 }, 1207 .probe = fsl_pamu_probe, 1208 }; 1209 1210 static __init int fsl_pamu_init(void) 1211 { 1212 struct platform_device *pdev = NULL; 1213 struct device_node *np; 1214 int ret; 1215 1216 /* 1217 * The normal OF process calls the probe function at some 1218 * indeterminate later time, after most drivers have loaded. This is 1219 * too late for us, because PAMU clients (like the Qman driver) 1220 * depend on PAMU being initialized early. 1221 * 1222 * So instead, we "manually" call our probe function by creating the 1223 * platform devices ourselves. 1224 */ 1225 1226 /* 1227 * We assume that there is only one PAMU node in the device tree. A 1228 * single PAMU node represents all of the PAMU devices in the SOC 1229 * already. Everything else already makes that assumption, and the 1230 * binding for the PAMU nodes doesn't allow for any parent-child 1231 * relationships anyway. In other words, support for more than one 1232 * PAMU node would require significant changes to a lot of code. 1233 */ 1234 1235 np = of_find_compatible_node(NULL, NULL, "fsl,pamu"); 1236 if (!np) { 1237 pr_err("could not find a PAMU node\n"); 1238 return -ENODEV; 1239 } 1240 1241 ret = platform_driver_register(&fsl_of_pamu_driver); 1242 if (ret) { 1243 pr_err("could not register driver (err=%i)\n", ret); 1244 goto error_driver_register; 1245 } 1246 1247 pdev = platform_device_alloc("fsl-of-pamu", 0); 1248 if (!pdev) { 1249 pr_err("could not allocate device %s\n", 1250 np->full_name); 1251 ret = -ENOMEM; 1252 goto error_device_alloc; 1253 } 1254 pdev->dev.of_node = of_node_get(np); 1255 1256 ret = pamu_domain_init(); 1257 if (ret) 1258 goto error_device_add; 1259 1260 ret = platform_device_add(pdev); 1261 if (ret) { 1262 pr_err("could not add device %s (err=%i)\n", 1263 np->full_name, ret); 1264 goto error_device_add; 1265 } 1266 1267 return 0; 1268 1269 error_device_add: 1270 of_node_put(pdev->dev.of_node); 1271 pdev->dev.of_node = NULL; 1272 1273 platform_device_put(pdev); 1274 1275 error_device_alloc: 1276 platform_driver_unregister(&fsl_of_pamu_driver); 1277 1278 error_driver_register: 1279 of_node_put(np); 1280 1281 return ret; 1282 } 1283 arch_initcall(fsl_pamu_init); 1284