1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * You should have received a copy of the GNU General Public License 12 * along with this program; if not, write to the Free Software 13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 14 * 15 * Copyright (C) 2013 Freescale Semiconductor, Inc. 16 * 17 */ 18 19 #define pr_fmt(fmt) "fsl-pamu: %s: " fmt, __func__ 20 21 #include "fsl_pamu.h" 22 23 #include <linux/fsl/guts.h> 24 #include <linux/interrupt.h> 25 #include <linux/genalloc.h> 26 27 #include <asm/mpc85xx.h> 28 29 /* define indexes for each operation mapping scenario */ 30 #define OMI_QMAN 0x00 31 #define OMI_FMAN 0x01 32 #define OMI_QMAN_PRIV 0x02 33 #define OMI_CAAM 0x03 34 35 #define make64(high, low) (((u64)(high) << 32) | (low)) 36 37 struct pamu_isr_data { 38 void __iomem *pamu_reg_base; /* Base address of PAMU regs */ 39 unsigned int count; /* The number of PAMUs */ 40 }; 41 42 static struct paace *ppaact; 43 static struct paace *spaact; 44 45 static bool probed; /* Has PAMU been probed? */ 46 47 /* 48 * Table for matching compatible strings, for device tree 49 * guts node, for QorIQ SOCs. 50 * "fsl,qoriq-device-config-2.0" corresponds to T4 & B4 51 * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" 52 * string would be used. 53 */ 54 static const struct of_device_id guts_device_ids[] = { 55 { .compatible = "fsl,qoriq-device-config-1.0", }, 56 { .compatible = "fsl,qoriq-device-config-2.0", }, 57 {} 58 }; 59 60 /* 61 * Table for matching compatible strings, for device tree 62 * L3 cache controller node. 63 * "fsl,t4240-l3-cache-controller" corresponds to T4, 64 * "fsl,b4860-l3-cache-controller" corresponds to B4 & 65 * "fsl,p4080-l3-cache-controller" corresponds to other, 66 * SOCs. 67 */ 68 static const struct of_device_id l3_device_ids[] = { 69 { .compatible = "fsl,t4240-l3-cache-controller", }, 70 { .compatible = "fsl,b4860-l3-cache-controller", }, 71 { .compatible = "fsl,p4080-l3-cache-controller", }, 72 {} 73 }; 74 75 /* maximum subwindows permitted per liodn */ 76 static u32 max_subwindow_count; 77 78 /* Pool for fspi allocation */ 79 static struct gen_pool *spaace_pool; 80 81 /** 82 * pamu_get_max_subwin_cnt() - Return the maximum supported 83 * subwindow count per liodn. 84 * 85 */ 86 u32 pamu_get_max_subwin_cnt(void) 87 { 88 return max_subwindow_count; 89 } 90 91 /** 92 * pamu_get_ppaace() - Return the primary PACCE 93 * @liodn: liodn PAACT index for desired PAACE 94 * 95 * Returns the ppace pointer upon success else return 96 * null. 97 */ 98 static struct paace *pamu_get_ppaace(int liodn) 99 { 100 if (!ppaact || liodn >= PAACE_NUMBER_ENTRIES) { 101 pr_debug("PPAACT doesn't exist\n"); 102 return NULL; 103 } 104 105 return &ppaact[liodn]; 106 } 107 108 /** 109 * pamu_enable_liodn() - Set valid bit of PACCE 110 * @liodn: liodn PAACT index for desired PAACE 111 * 112 * Returns 0 upon success else error code < 0 returned 113 */ 114 int pamu_enable_liodn(int liodn) 115 { 116 struct paace *ppaace; 117 118 ppaace = pamu_get_ppaace(liodn); 119 if (!ppaace) { 120 pr_debug("Invalid primary paace entry\n"); 121 return -ENOENT; 122 } 123 124 if (!get_bf(ppaace->addr_bitfields, PPAACE_AF_WSE)) { 125 pr_debug("liodn %d not configured\n", liodn); 126 return -EINVAL; 127 } 128 129 /* Ensure that all other stores to the ppaace complete first */ 130 mb(); 131 132 set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); 133 mb(); 134 135 return 0; 136 } 137 138 /** 139 * pamu_disable_liodn() - Clears valid bit of PACCE 140 * @liodn: liodn PAACT index for desired PAACE 141 * 142 * Returns 0 upon success else error code < 0 returned 143 */ 144 int pamu_disable_liodn(int liodn) 145 { 146 struct paace *ppaace; 147 148 ppaace = pamu_get_ppaace(liodn); 149 if (!ppaace) { 150 pr_debug("Invalid primary paace entry\n"); 151 return -ENOENT; 152 } 153 154 set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); 155 mb(); 156 157 return 0; 158 } 159 160 /* Derive the window size encoding for a particular PAACE entry */ 161 static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size) 162 { 163 /* Bug if not a power of 2 */ 164 BUG_ON(addrspace_size & (addrspace_size - 1)); 165 166 /* window size is 2^(WSE+1) bytes */ 167 return fls64(addrspace_size) - 2; 168 } 169 170 /* Derive the PAACE window count encoding for the subwindow count */ 171 static unsigned int map_subwindow_cnt_to_wce(u32 subwindow_cnt) 172 { 173 /* window count is 2^(WCE+1) bytes */ 174 return __ffs(subwindow_cnt) - 1; 175 } 176 177 /* 178 * Set the PAACE type as primary and set the coherency required domain 179 * attribute 180 */ 181 static void pamu_init_ppaace(struct paace *ppaace) 182 { 183 set_bf(ppaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_PRIMARY); 184 185 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 186 PAACE_M_COHERENCE_REQ); 187 } 188 189 /* 190 * Set the PAACE type as secondary and set the coherency required domain 191 * attribute. 192 */ 193 static void pamu_init_spaace(struct paace *spaace) 194 { 195 set_bf(spaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_SECONDARY); 196 set_bf(spaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 197 PAACE_M_COHERENCE_REQ); 198 } 199 200 /* 201 * Return the spaace (corresponding to the secondary window index) 202 * for a particular ppaace. 203 */ 204 static struct paace *pamu_get_spaace(struct paace *paace, u32 wnum) 205 { 206 u32 subwin_cnt; 207 struct paace *spaace = NULL; 208 209 subwin_cnt = 1UL << (get_bf(paace->impl_attr, PAACE_IA_WCE) + 1); 210 211 if (wnum < subwin_cnt) 212 spaace = &spaact[paace->fspi + wnum]; 213 else 214 pr_debug("secondary paace out of bounds\n"); 215 216 return spaace; 217 } 218 219 /** 220 * pamu_get_fspi_and_allocate() - Allocates fspi index and reserves subwindows 221 * required for primary PAACE in the secondary 222 * PAACE table. 223 * @subwin_cnt: Number of subwindows to be reserved. 224 * 225 * A PPAACE entry may have a number of associated subwindows. A subwindow 226 * corresponds to a SPAACE entry in the SPAACT table. Each PAACE entry stores 227 * the index (fspi) of the first SPAACE entry in the SPAACT table. This 228 * function returns the index of the first SPAACE entry. The remaining 229 * SPAACE entries are reserved contiguously from that index. 230 * 231 * Returns a valid fspi index in the range of 0 - SPAACE_NUMBER_ENTRIES on success. 232 * If no SPAACE entry is available or the allocator can not reserve the required 233 * number of contiguous entries function returns ULONG_MAX indicating a failure. 234 * 235 */ 236 static unsigned long pamu_get_fspi_and_allocate(u32 subwin_cnt) 237 { 238 unsigned long spaace_addr; 239 240 spaace_addr = gen_pool_alloc(spaace_pool, subwin_cnt * sizeof(struct paace)); 241 if (!spaace_addr) 242 return ULONG_MAX; 243 244 return (spaace_addr - (unsigned long)spaact) / (sizeof(struct paace)); 245 } 246 247 /* Release the subwindows reserved for a particular LIODN */ 248 void pamu_free_subwins(int liodn) 249 { 250 struct paace *ppaace; 251 u32 subwin_cnt, size; 252 253 ppaace = pamu_get_ppaace(liodn); 254 if (!ppaace) { 255 pr_debug("Invalid liodn entry\n"); 256 return; 257 } 258 259 if (get_bf(ppaace->addr_bitfields, PPAACE_AF_MW)) { 260 subwin_cnt = 1UL << (get_bf(ppaace->impl_attr, PAACE_IA_WCE) + 1); 261 size = (subwin_cnt - 1) * sizeof(struct paace); 262 gen_pool_free(spaace_pool, (unsigned long)&spaact[ppaace->fspi], size); 263 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); 264 } 265 } 266 267 /* 268 * Function used for updating stash destination for the coressponding 269 * LIODN. 270 */ 271 int pamu_update_paace_stash(int liodn, u32 subwin, u32 value) 272 { 273 struct paace *paace; 274 275 paace = pamu_get_ppaace(liodn); 276 if (!paace) { 277 pr_debug("Invalid liodn entry\n"); 278 return -ENOENT; 279 } 280 if (subwin) { 281 paace = pamu_get_spaace(paace, subwin - 1); 282 if (!paace) 283 return -ENOENT; 284 } 285 set_bf(paace->impl_attr, PAACE_IA_CID, value); 286 287 mb(); 288 289 return 0; 290 } 291 292 /* Disable a subwindow corresponding to the LIODN */ 293 int pamu_disable_spaace(int liodn, u32 subwin) 294 { 295 struct paace *paace; 296 297 paace = pamu_get_ppaace(liodn); 298 if (!paace) { 299 pr_debug("Invalid liodn entry\n"); 300 return -ENOENT; 301 } 302 if (subwin) { 303 paace = pamu_get_spaace(paace, subwin - 1); 304 if (!paace) 305 return -ENOENT; 306 set_bf(paace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); 307 } else { 308 set_bf(paace->addr_bitfields, PAACE_AF_AP, 309 PAACE_AP_PERMS_DENIED); 310 } 311 312 mb(); 313 314 return 0; 315 } 316 317 /** 318 * pamu_config_paace() - Sets up PPAACE entry for specified liodn 319 * 320 * @liodn: Logical IO device number 321 * @win_addr: starting address of DSA window 322 * @win-size: size of DSA window 323 * @omi: Operation mapping index -- if ~omi == 0 then omi not defined 324 * @rpn: real (true physical) page number 325 * @stashid: cache stash id for associated cpu -- if ~stashid == 0 then 326 * stashid not defined 327 * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then 328 * snoopid not defined 329 * @subwin_cnt: number of sub-windows 330 * @prot: window permissions 331 * 332 * Returns 0 upon success else error code < 0 returned 333 */ 334 int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, 335 u32 omi, unsigned long rpn, u32 snoopid, u32 stashid, 336 u32 subwin_cnt, int prot) 337 { 338 struct paace *ppaace; 339 unsigned long fspi; 340 341 if ((win_size & (win_size - 1)) || win_size < PAMU_PAGE_SIZE) { 342 pr_debug("window size too small or not a power of two %pa\n", 343 &win_size); 344 return -EINVAL; 345 } 346 347 if (win_addr & (win_size - 1)) { 348 pr_debug("window address is not aligned with window size\n"); 349 return -EINVAL; 350 } 351 352 ppaace = pamu_get_ppaace(liodn); 353 if (!ppaace) 354 return -ENOENT; 355 356 /* window size is 2^(WSE+1) bytes */ 357 set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 358 map_addrspace_size_to_wse(win_size)); 359 360 pamu_init_ppaace(ppaace); 361 362 ppaace->wbah = win_addr >> (PAMU_PAGE_SHIFT + 20); 363 set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 364 (win_addr >> PAMU_PAGE_SHIFT)); 365 366 /* set up operation mapping if it's configured */ 367 if (omi < OME_NUMBER_ENTRIES) { 368 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 369 ppaace->op_encode.index_ot.omi = omi; 370 } else if (~omi != 0) { 371 pr_debug("bad operation mapping index: %d\n", omi); 372 return -EINVAL; 373 } 374 375 /* configure stash id */ 376 if (~stashid != 0) 377 set_bf(ppaace->impl_attr, PAACE_IA_CID, stashid); 378 379 /* configure snoop id */ 380 if (~snoopid != 0) 381 ppaace->domain_attr.to_host.snpid = snoopid; 382 383 if (subwin_cnt) { 384 /* The first entry is in the primary PAACE instead */ 385 fspi = pamu_get_fspi_and_allocate(subwin_cnt - 1); 386 if (fspi == ULONG_MAX) { 387 pr_debug("spaace indexes exhausted\n"); 388 return -EINVAL; 389 } 390 391 /* window count is 2^(WCE+1) bytes */ 392 set_bf(ppaace->impl_attr, PAACE_IA_WCE, 393 map_subwindow_cnt_to_wce(subwin_cnt)); 394 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0x1); 395 ppaace->fspi = fspi; 396 } else { 397 set_bf(ppaace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); 398 ppaace->twbah = rpn >> 20; 399 set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, rpn); 400 set_bf(ppaace->addr_bitfields, PAACE_AF_AP, prot); 401 set_bf(ppaace->impl_attr, PAACE_IA_WCE, 0); 402 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); 403 } 404 mb(); 405 406 return 0; 407 } 408 409 /** 410 * pamu_config_spaace() - Sets up SPAACE entry for specified subwindow 411 * 412 * @liodn: Logical IO device number 413 * @subwin_cnt: number of sub-windows associated with dma-window 414 * @subwin: subwindow index 415 * @subwin_size: size of subwindow 416 * @omi: Operation mapping index 417 * @rpn: real (true physical) page number 418 * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then 419 * snoopid not defined 420 * @stashid: cache stash id for associated cpu 421 * @enable: enable/disable subwindow after reconfiguration 422 * @prot: sub window permissions 423 * 424 * Returns 0 upon success else error code < 0 returned 425 */ 426 int pamu_config_spaace(int liodn, u32 subwin_cnt, u32 subwin, 427 phys_addr_t subwin_size, u32 omi, unsigned long rpn, 428 u32 snoopid, u32 stashid, int enable, int prot) 429 { 430 struct paace *paace; 431 432 /* setup sub-windows */ 433 if (!subwin_cnt) { 434 pr_debug("Invalid subwindow count\n"); 435 return -EINVAL; 436 } 437 438 paace = pamu_get_ppaace(liodn); 439 if (subwin > 0 && subwin < subwin_cnt && paace) { 440 paace = pamu_get_spaace(paace, subwin - 1); 441 442 if (paace && !(paace->addr_bitfields & PAACE_V_VALID)) { 443 pamu_init_spaace(paace); 444 set_bf(paace->addr_bitfields, SPAACE_AF_LIODN, liodn); 445 } 446 } 447 448 if (!paace) { 449 pr_debug("Invalid liodn entry\n"); 450 return -ENOENT; 451 } 452 453 if ((subwin_size & (subwin_size - 1)) || subwin_size < PAMU_PAGE_SIZE) { 454 pr_debug("subwindow size out of range, or not a power of 2\n"); 455 return -EINVAL; 456 } 457 458 if (rpn == ULONG_MAX) { 459 pr_debug("real page number out of range\n"); 460 return -EINVAL; 461 } 462 463 /* window size is 2^(WSE+1) bytes */ 464 set_bf(paace->win_bitfields, PAACE_WIN_SWSE, 465 map_addrspace_size_to_wse(subwin_size)); 466 467 set_bf(paace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); 468 paace->twbah = rpn >> 20; 469 set_bf(paace->win_bitfields, PAACE_WIN_TWBAL, rpn); 470 set_bf(paace->addr_bitfields, PAACE_AF_AP, prot); 471 472 /* configure snoop id */ 473 if (~snoopid != 0) 474 paace->domain_attr.to_host.snpid = snoopid; 475 476 /* set up operation mapping if it's configured */ 477 if (omi < OME_NUMBER_ENTRIES) { 478 set_bf(paace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 479 paace->op_encode.index_ot.omi = omi; 480 } else if (~omi != 0) { 481 pr_debug("bad operation mapping index: %d\n", omi); 482 return -EINVAL; 483 } 484 485 if (~stashid != 0) 486 set_bf(paace->impl_attr, PAACE_IA_CID, stashid); 487 488 smp_wmb(); 489 490 if (enable) 491 set_bf(paace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); 492 493 mb(); 494 495 return 0; 496 } 497 498 /** 499 * get_ome_index() - Returns the index in the operation mapping table 500 * for device. 501 * @*omi_index: pointer for storing the index value 502 * 503 */ 504 void get_ome_index(u32 *omi_index, struct device *dev) 505 { 506 if (of_device_is_compatible(dev->of_node, "fsl,qman-portal")) 507 *omi_index = OMI_QMAN; 508 if (of_device_is_compatible(dev->of_node, "fsl,qman")) 509 *omi_index = OMI_QMAN_PRIV; 510 } 511 512 /** 513 * get_stash_id - Returns stash destination id corresponding to a 514 * cache type and vcpu. 515 * @stash_dest_hint: L1, L2 or L3 516 * @vcpu: vpcu target for a particular cache type. 517 * 518 * Returs stash on success or ~(u32)0 on failure. 519 * 520 */ 521 u32 get_stash_id(u32 stash_dest_hint, u32 vcpu) 522 { 523 const u32 *prop; 524 struct device_node *node; 525 u32 cache_level; 526 int len, found = 0; 527 int i; 528 529 /* Fastpath, exit early if L3/CPC cache is target for stashing */ 530 if (stash_dest_hint == PAMU_ATTR_CACHE_L3) { 531 node = of_find_matching_node(NULL, l3_device_ids); 532 if (node) { 533 prop = of_get_property(node, "cache-stash-id", NULL); 534 if (!prop) { 535 pr_debug("missing cache-stash-id at %pOF\n", 536 node); 537 of_node_put(node); 538 return ~(u32)0; 539 } 540 of_node_put(node); 541 return be32_to_cpup(prop); 542 } 543 return ~(u32)0; 544 } 545 546 for_each_of_cpu_node(node) { 547 prop = of_get_property(node, "reg", &len); 548 for (i = 0; i < len / sizeof(u32); i++) { 549 if (be32_to_cpup(&prop[i]) == vcpu) { 550 found = 1; 551 goto found_cpu_node; 552 } 553 } 554 } 555 found_cpu_node: 556 557 /* find the hwnode that represents the cache */ 558 for (cache_level = PAMU_ATTR_CACHE_L1; (cache_level < PAMU_ATTR_CACHE_L3) && found; cache_level++) { 559 if (stash_dest_hint == cache_level) { 560 prop = of_get_property(node, "cache-stash-id", NULL); 561 if (!prop) { 562 pr_debug("missing cache-stash-id at %pOF\n", 563 node); 564 of_node_put(node); 565 return ~(u32)0; 566 } 567 of_node_put(node); 568 return be32_to_cpup(prop); 569 } 570 571 prop = of_get_property(node, "next-level-cache", NULL); 572 if (!prop) { 573 pr_debug("can't find next-level-cache at %pOF\n", node); 574 of_node_put(node); 575 return ~(u32)0; /* can't traverse any further */ 576 } 577 of_node_put(node); 578 579 /* advance to next node in cache hierarchy */ 580 node = of_find_node_by_phandle(*prop); 581 if (!node) { 582 pr_debug("Invalid node for cache hierarchy\n"); 583 return ~(u32)0; 584 } 585 } 586 587 pr_debug("stash dest not found for %d on vcpu %d\n", 588 stash_dest_hint, vcpu); 589 return ~(u32)0; 590 } 591 592 /* Identify if the PAACT table entry belongs to QMAN, BMAN or QMAN Portal */ 593 #define QMAN_PAACE 1 594 #define QMAN_PORTAL_PAACE 2 595 #define BMAN_PAACE 3 596 597 /** 598 * Setup operation mapping and stash destinations for QMAN and QMAN portal. 599 * Memory accesses to QMAN and BMAN private memory need not be coherent, so 600 * clear the PAACE entry coherency attribute for them. 601 */ 602 static void setup_qbman_paace(struct paace *ppaace, int paace_type) 603 { 604 switch (paace_type) { 605 case QMAN_PAACE: 606 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 607 ppaace->op_encode.index_ot.omi = OMI_QMAN_PRIV; 608 /* setup QMAN Private data stashing for the L3 cache */ 609 set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); 610 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 611 0); 612 break; 613 case QMAN_PORTAL_PAACE: 614 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 615 ppaace->op_encode.index_ot.omi = OMI_QMAN; 616 /* Set DQRR and Frame stashing for the L3 cache */ 617 set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); 618 break; 619 case BMAN_PAACE: 620 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 621 0); 622 break; 623 } 624 } 625 626 /** 627 * Setup the operation mapping table for various devices. This is a static 628 * table where each table index corresponds to a particular device. PAMU uses 629 * this table to translate device transaction to appropriate corenet 630 * transaction. 631 */ 632 static void setup_omt(struct ome *omt) 633 { 634 struct ome *ome; 635 636 /* Configure OMI_QMAN */ 637 ome = &omt[OMI_QMAN]; 638 639 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; 640 ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; 641 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 642 ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSAO; 643 644 ome->moe[IOE_DIRECT0_IDX] = EOE_VALID | EOE_LDEC; 645 ome->moe[IOE_DIRECT1_IDX] = EOE_VALID | EOE_LDECPE; 646 647 /* Configure OMI_FMAN */ 648 ome = &omt[OMI_FMAN]; 649 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; 650 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 651 652 /* Configure OMI_QMAN private */ 653 ome = &omt[OMI_QMAN_PRIV]; 654 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; 655 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 656 ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; 657 ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSA; 658 659 /* Configure OMI_CAAM */ 660 ome = &omt[OMI_CAAM]; 661 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; 662 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 663 } 664 665 /* 666 * Get the maximum number of PAACT table entries 667 * and subwindows supported by PAMU 668 */ 669 static void get_pamu_cap_values(unsigned long pamu_reg_base) 670 { 671 u32 pc_val; 672 673 pc_val = in_be32((u32 *)(pamu_reg_base + PAMU_PC3)); 674 /* Maximum number of subwindows per liodn */ 675 max_subwindow_count = 1 << (1 + PAMU_PC3_MWCE(pc_val)); 676 } 677 678 /* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ 679 static int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, 680 phys_addr_t ppaact_phys, phys_addr_t spaact_phys, 681 phys_addr_t omt_phys) 682 { 683 u32 *pc; 684 struct pamu_mmap_regs *pamu_regs; 685 686 pc = (u32 *) (pamu_reg_base + PAMU_PC); 687 pamu_regs = (struct pamu_mmap_regs *) 688 (pamu_reg_base + PAMU_MMAP_REGS_BASE); 689 690 /* set up pointers to corenet control blocks */ 691 692 out_be32(&pamu_regs->ppbah, upper_32_bits(ppaact_phys)); 693 out_be32(&pamu_regs->ppbal, lower_32_bits(ppaact_phys)); 694 ppaact_phys = ppaact_phys + PAACT_SIZE; 695 out_be32(&pamu_regs->pplah, upper_32_bits(ppaact_phys)); 696 out_be32(&pamu_regs->pplal, lower_32_bits(ppaact_phys)); 697 698 out_be32(&pamu_regs->spbah, upper_32_bits(spaact_phys)); 699 out_be32(&pamu_regs->spbal, lower_32_bits(spaact_phys)); 700 spaact_phys = spaact_phys + SPAACT_SIZE; 701 out_be32(&pamu_regs->splah, upper_32_bits(spaact_phys)); 702 out_be32(&pamu_regs->splal, lower_32_bits(spaact_phys)); 703 704 out_be32(&pamu_regs->obah, upper_32_bits(omt_phys)); 705 out_be32(&pamu_regs->obal, lower_32_bits(omt_phys)); 706 omt_phys = omt_phys + OMT_SIZE; 707 out_be32(&pamu_regs->olah, upper_32_bits(omt_phys)); 708 out_be32(&pamu_regs->olal, lower_32_bits(omt_phys)); 709 710 /* 711 * set PAMU enable bit, 712 * allow ppaact & omt to be cached 713 * & enable PAMU access violation interrupts. 714 */ 715 716 out_be32((u32 *)(pamu_reg_base + PAMU_PICS), 717 PAMU_ACCESS_VIOLATION_ENABLE); 718 out_be32(pc, PAMU_PC_PE | PAMU_PC_OCE | PAMU_PC_SPCC | PAMU_PC_PPCC); 719 return 0; 720 } 721 722 /* Enable all device LIODNS */ 723 static void setup_liodns(void) 724 { 725 int i, len; 726 struct paace *ppaace; 727 struct device_node *node = NULL; 728 const u32 *prop; 729 730 for_each_node_with_property(node, "fsl,liodn") { 731 prop = of_get_property(node, "fsl,liodn", &len); 732 for (i = 0; i < len / sizeof(u32); i++) { 733 int liodn; 734 735 liodn = be32_to_cpup(&prop[i]); 736 if (liodn >= PAACE_NUMBER_ENTRIES) { 737 pr_debug("Invalid LIODN value %d\n", liodn); 738 continue; 739 } 740 ppaace = pamu_get_ppaace(liodn); 741 pamu_init_ppaace(ppaace); 742 /* window size is 2^(WSE+1) bytes */ 743 set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 35); 744 ppaace->wbah = 0; 745 set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); 746 set_bf(ppaace->impl_attr, PAACE_IA_ATM, 747 PAACE_ATM_NO_XLATE); 748 set_bf(ppaace->addr_bitfields, PAACE_AF_AP, 749 PAACE_AP_PERMS_ALL); 750 if (of_device_is_compatible(node, "fsl,qman-portal")) 751 setup_qbman_paace(ppaace, QMAN_PORTAL_PAACE); 752 if (of_device_is_compatible(node, "fsl,qman")) 753 setup_qbman_paace(ppaace, QMAN_PAACE); 754 if (of_device_is_compatible(node, "fsl,bman")) 755 setup_qbman_paace(ppaace, BMAN_PAACE); 756 mb(); 757 pamu_enable_liodn(liodn); 758 } 759 } 760 } 761 762 static irqreturn_t pamu_av_isr(int irq, void *arg) 763 { 764 struct pamu_isr_data *data = arg; 765 phys_addr_t phys; 766 unsigned int i, j, ret; 767 768 pr_emerg("access violation interrupt\n"); 769 770 for (i = 0; i < data->count; i++) { 771 void __iomem *p = data->pamu_reg_base + i * PAMU_OFFSET; 772 u32 pics = in_be32(p + PAMU_PICS); 773 774 if (pics & PAMU_ACCESS_VIOLATION_STAT) { 775 u32 avs1 = in_be32(p + PAMU_AVS1); 776 struct paace *paace; 777 778 pr_emerg("POES1=%08x\n", in_be32(p + PAMU_POES1)); 779 pr_emerg("POES2=%08x\n", in_be32(p + PAMU_POES2)); 780 pr_emerg("AVS1=%08x\n", avs1); 781 pr_emerg("AVS2=%08x\n", in_be32(p + PAMU_AVS2)); 782 pr_emerg("AVA=%016llx\n", 783 make64(in_be32(p + PAMU_AVAH), 784 in_be32(p + PAMU_AVAL))); 785 pr_emerg("UDAD=%08x\n", in_be32(p + PAMU_UDAD)); 786 pr_emerg("POEA=%016llx\n", 787 make64(in_be32(p + PAMU_POEAH), 788 in_be32(p + PAMU_POEAL))); 789 790 phys = make64(in_be32(p + PAMU_POEAH), 791 in_be32(p + PAMU_POEAL)); 792 793 /* Assume that POEA points to a PAACE */ 794 if (phys) { 795 u32 *paace = phys_to_virt(phys); 796 797 /* Only the first four words are relevant */ 798 for (j = 0; j < 4; j++) 799 pr_emerg("PAACE[%u]=%08x\n", 800 j, in_be32(paace + j)); 801 } 802 803 /* clear access violation condition */ 804 out_be32(p + PAMU_AVS1, avs1 & PAMU_AV_MASK); 805 paace = pamu_get_ppaace(avs1 >> PAMU_AVS1_LIODN_SHIFT); 806 BUG_ON(!paace); 807 /* check if we got a violation for a disabled LIODN */ 808 if (!get_bf(paace->addr_bitfields, PAACE_AF_V)) { 809 /* 810 * As per hardware erratum A-003638, access 811 * violation can be reported for a disabled 812 * LIODN. If we hit that condition, disable 813 * access violation reporting. 814 */ 815 pics &= ~PAMU_ACCESS_VIOLATION_ENABLE; 816 } else { 817 /* Disable the LIODN */ 818 ret = pamu_disable_liodn(avs1 >> PAMU_AVS1_LIODN_SHIFT); 819 BUG_ON(ret); 820 pr_emerg("Disabling liodn %x\n", 821 avs1 >> PAMU_AVS1_LIODN_SHIFT); 822 } 823 out_be32((p + PAMU_PICS), pics); 824 } 825 } 826 827 return IRQ_HANDLED; 828 } 829 830 #define LAWAR_EN 0x80000000 831 #define LAWAR_TARGET_MASK 0x0FF00000 832 #define LAWAR_TARGET_SHIFT 20 833 #define LAWAR_SIZE_MASK 0x0000003F 834 #define LAWAR_CSDID_MASK 0x000FF000 835 #define LAWAR_CSDID_SHIFT 12 836 837 #define LAW_SIZE_4K 0xb 838 839 struct ccsr_law { 840 u32 lawbarh; /* LAWn base address high */ 841 u32 lawbarl; /* LAWn base address low */ 842 u32 lawar; /* LAWn attributes */ 843 u32 reserved; 844 }; 845 846 /* 847 * Create a coherence subdomain for a given memory block. 848 */ 849 static int create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) 850 { 851 struct device_node *np; 852 const __be32 *iprop; 853 void __iomem *lac = NULL; /* Local Access Control registers */ 854 struct ccsr_law __iomem *law; 855 void __iomem *ccm = NULL; 856 u32 __iomem *csdids; 857 unsigned int i, num_laws, num_csds; 858 u32 law_target = 0; 859 u32 csd_id = 0; 860 int ret = 0; 861 862 np = of_find_compatible_node(NULL, NULL, "fsl,corenet-law"); 863 if (!np) 864 return -ENODEV; 865 866 iprop = of_get_property(np, "fsl,num-laws", NULL); 867 if (!iprop) { 868 ret = -ENODEV; 869 goto error; 870 } 871 872 num_laws = be32_to_cpup(iprop); 873 if (!num_laws) { 874 ret = -ENODEV; 875 goto error; 876 } 877 878 lac = of_iomap(np, 0); 879 if (!lac) { 880 ret = -ENODEV; 881 goto error; 882 } 883 884 /* LAW registers are at offset 0xC00 */ 885 law = lac + 0xC00; 886 887 of_node_put(np); 888 889 np = of_find_compatible_node(NULL, NULL, "fsl,corenet-cf"); 890 if (!np) { 891 ret = -ENODEV; 892 goto error; 893 } 894 895 iprop = of_get_property(np, "fsl,ccf-num-csdids", NULL); 896 if (!iprop) { 897 ret = -ENODEV; 898 goto error; 899 } 900 901 num_csds = be32_to_cpup(iprop); 902 if (!num_csds) { 903 ret = -ENODEV; 904 goto error; 905 } 906 907 ccm = of_iomap(np, 0); 908 if (!ccm) { 909 ret = -ENOMEM; 910 goto error; 911 } 912 913 /* The undocumented CSDID registers are at offset 0x600 */ 914 csdids = ccm + 0x600; 915 916 of_node_put(np); 917 np = NULL; 918 919 /* Find an unused coherence subdomain ID */ 920 for (csd_id = 0; csd_id < num_csds; csd_id++) { 921 if (!csdids[csd_id]) 922 break; 923 } 924 925 /* Store the Port ID in the (undocumented) proper CIDMRxx register */ 926 csdids[csd_id] = csd_port_id; 927 928 /* Find the DDR LAW that maps to our buffer. */ 929 for (i = 0; i < num_laws; i++) { 930 if (law[i].lawar & LAWAR_EN) { 931 phys_addr_t law_start, law_end; 932 933 law_start = make64(law[i].lawbarh, law[i].lawbarl); 934 law_end = law_start + 935 (2ULL << (law[i].lawar & LAWAR_SIZE_MASK)); 936 937 if (law_start <= phys && phys < law_end) { 938 law_target = law[i].lawar & LAWAR_TARGET_MASK; 939 break; 940 } 941 } 942 } 943 944 if (i == 0 || i == num_laws) { 945 /* This should never happen */ 946 ret = -ENOENT; 947 goto error; 948 } 949 950 /* Find a free LAW entry */ 951 while (law[--i].lawar & LAWAR_EN) { 952 if (i == 0) { 953 /* No higher priority LAW slots available */ 954 ret = -ENOENT; 955 goto error; 956 } 957 } 958 959 law[i].lawbarh = upper_32_bits(phys); 960 law[i].lawbarl = lower_32_bits(phys); 961 wmb(); 962 law[i].lawar = LAWAR_EN | law_target | (csd_id << LAWAR_CSDID_SHIFT) | 963 (LAW_SIZE_4K + get_order(size)); 964 wmb(); 965 966 error: 967 if (ccm) 968 iounmap(ccm); 969 970 if (lac) 971 iounmap(lac); 972 973 if (np) 974 of_node_put(np); 975 976 return ret; 977 } 978 979 /* 980 * Table of SVRs and the corresponding PORT_ID values. Port ID corresponds to a 981 * bit map of snoopers for a given range of memory mapped by a LAW. 982 * 983 * All future CoreNet-enabled SOCs will have this erratum(A-004510) fixed, so this 984 * table should never need to be updated. SVRs are guaranteed to be unique, so 985 * there is no worry that a future SOC will inadvertently have one of these 986 * values. 987 */ 988 static const struct { 989 u32 svr; 990 u32 port_id; 991 } port_id_map[] = { 992 {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */ 993 {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */ 994 {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */ 995 {(SVR_P2041 << 8) | 0x11, 0xFF000000}, /* P2041 1.1 */ 996 {(SVR_P3041 << 8) | 0x10, 0xFF000000}, /* P3041 1.0 */ 997 {(SVR_P3041 << 8) | 0x11, 0xFF000000}, /* P3041 1.1 */ 998 {(SVR_P4040 << 8) | 0x20, 0xFFF80000}, /* P4040 2.0 */ 999 {(SVR_P4080 << 8) | 0x20, 0xFFF80000}, /* P4080 2.0 */ 1000 {(SVR_P5010 << 8) | 0x10, 0xFC000000}, /* P5010 1.0 */ 1001 {(SVR_P5010 << 8) | 0x20, 0xFC000000}, /* P5010 2.0 */ 1002 {(SVR_P5020 << 8) | 0x10, 0xFC000000}, /* P5020 1.0 */ 1003 {(SVR_P5021 << 8) | 0x10, 0xFF800000}, /* P5021 1.0 */ 1004 {(SVR_P5040 << 8) | 0x10, 0xFF800000}, /* P5040 1.0 */ 1005 }; 1006 1007 #define SVR_SECURITY 0x80000 /* The Security (E) bit */ 1008 1009 static int fsl_pamu_probe(struct platform_device *pdev) 1010 { 1011 struct device *dev = &pdev->dev; 1012 void __iomem *pamu_regs = NULL; 1013 struct ccsr_guts __iomem *guts_regs = NULL; 1014 u32 pamubypenr, pamu_counter; 1015 unsigned long pamu_reg_off; 1016 unsigned long pamu_reg_base; 1017 struct pamu_isr_data *data = NULL; 1018 struct device_node *guts_node; 1019 u64 size; 1020 struct page *p; 1021 int ret = 0; 1022 int irq; 1023 phys_addr_t ppaact_phys; 1024 phys_addr_t spaact_phys; 1025 struct ome *omt; 1026 phys_addr_t omt_phys; 1027 size_t mem_size = 0; 1028 unsigned int order = 0; 1029 u32 csd_port_id = 0; 1030 unsigned i; 1031 /* 1032 * enumerate all PAMUs and allocate and setup PAMU tables 1033 * for each of them, 1034 * NOTE : All PAMUs share the same LIODN tables. 1035 */ 1036 1037 if (WARN_ON(probed)) 1038 return -EBUSY; 1039 1040 pamu_regs = of_iomap(dev->of_node, 0); 1041 if (!pamu_regs) { 1042 dev_err(dev, "ioremap of PAMU node failed\n"); 1043 return -ENOMEM; 1044 } 1045 of_get_address(dev->of_node, 0, &size, NULL); 1046 1047 irq = irq_of_parse_and_map(dev->of_node, 0); 1048 if (irq == NO_IRQ) { 1049 dev_warn(dev, "no interrupts listed in PAMU node\n"); 1050 goto error; 1051 } 1052 1053 data = kzalloc(sizeof(*data), GFP_KERNEL); 1054 if (!data) { 1055 ret = -ENOMEM; 1056 goto error; 1057 } 1058 data->pamu_reg_base = pamu_regs; 1059 data->count = size / PAMU_OFFSET; 1060 1061 /* The ISR needs access to the regs, so we won't iounmap them */ 1062 ret = request_irq(irq, pamu_av_isr, 0, "pamu", data); 1063 if (ret < 0) { 1064 dev_err(dev, "error %i installing ISR for irq %i\n", ret, irq); 1065 goto error; 1066 } 1067 1068 guts_node = of_find_matching_node(NULL, guts_device_ids); 1069 if (!guts_node) { 1070 dev_err(dev, "could not find GUTS node %pOF\n", dev->of_node); 1071 ret = -ENODEV; 1072 goto error; 1073 } 1074 1075 guts_regs = of_iomap(guts_node, 0); 1076 of_node_put(guts_node); 1077 if (!guts_regs) { 1078 dev_err(dev, "ioremap of GUTS node failed\n"); 1079 ret = -ENODEV; 1080 goto error; 1081 } 1082 1083 /* read in the PAMU capability registers */ 1084 get_pamu_cap_values((unsigned long)pamu_regs); 1085 /* 1086 * To simplify the allocation of a coherency domain, we allocate the 1087 * PAACT and the OMT in the same memory buffer. Unfortunately, this 1088 * wastes more memory compared to allocating the buffers separately. 1089 */ 1090 /* Determine how much memory we need */ 1091 mem_size = (PAGE_SIZE << get_order(PAACT_SIZE)) + 1092 (PAGE_SIZE << get_order(SPAACT_SIZE)) + 1093 (PAGE_SIZE << get_order(OMT_SIZE)); 1094 order = get_order(mem_size); 1095 1096 p = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); 1097 if (!p) { 1098 dev_err(dev, "unable to allocate PAACT/SPAACT/OMT block\n"); 1099 ret = -ENOMEM; 1100 goto error; 1101 } 1102 1103 ppaact = page_address(p); 1104 ppaact_phys = page_to_phys(p); 1105 1106 /* Make sure the memory is naturally aligned */ 1107 if (ppaact_phys & ((PAGE_SIZE << order) - 1)) { 1108 dev_err(dev, "PAACT/OMT block is unaligned\n"); 1109 ret = -ENOMEM; 1110 goto error; 1111 } 1112 1113 spaact = (void *)ppaact + (PAGE_SIZE << get_order(PAACT_SIZE)); 1114 omt = (void *)spaact + (PAGE_SIZE << get_order(SPAACT_SIZE)); 1115 1116 dev_dbg(dev, "ppaact virt=%p phys=%pa\n", ppaact, &ppaact_phys); 1117 1118 /* Check to see if we need to implement the work-around on this SOC */ 1119 1120 /* Determine the Port ID for our coherence subdomain */ 1121 for (i = 0; i < ARRAY_SIZE(port_id_map); i++) { 1122 if (port_id_map[i].svr == (mfspr(SPRN_SVR) & ~SVR_SECURITY)) { 1123 csd_port_id = port_id_map[i].port_id; 1124 dev_dbg(dev, "found matching SVR %08x\n", 1125 port_id_map[i].svr); 1126 break; 1127 } 1128 } 1129 1130 if (csd_port_id) { 1131 dev_dbg(dev, "creating coherency subdomain at address %pa, size %zu, port id 0x%08x", 1132 &ppaact_phys, mem_size, csd_port_id); 1133 1134 ret = create_csd(ppaact_phys, mem_size, csd_port_id); 1135 if (ret) { 1136 dev_err(dev, "could not create coherence subdomain\n"); 1137 return ret; 1138 } 1139 } 1140 1141 spaact_phys = virt_to_phys(spaact); 1142 omt_phys = virt_to_phys(omt); 1143 1144 spaace_pool = gen_pool_create(ilog2(sizeof(struct paace)), -1); 1145 if (!spaace_pool) { 1146 ret = -ENOMEM; 1147 dev_err(dev, "Failed to allocate spaace gen pool\n"); 1148 goto error; 1149 } 1150 1151 ret = gen_pool_add(spaace_pool, (unsigned long)spaact, SPAACT_SIZE, -1); 1152 if (ret) 1153 goto error_genpool; 1154 1155 pamubypenr = in_be32(&guts_regs->pamubypenr); 1156 1157 for (pamu_reg_off = 0, pamu_counter = 0x80000000; pamu_reg_off < size; 1158 pamu_reg_off += PAMU_OFFSET, pamu_counter >>= 1) { 1159 1160 pamu_reg_base = (unsigned long)pamu_regs + pamu_reg_off; 1161 setup_one_pamu(pamu_reg_base, pamu_reg_off, ppaact_phys, 1162 spaact_phys, omt_phys); 1163 /* Disable PAMU bypass for this PAMU */ 1164 pamubypenr &= ~pamu_counter; 1165 } 1166 1167 setup_omt(omt); 1168 1169 /* Enable all relevant PAMU(s) */ 1170 out_be32(&guts_regs->pamubypenr, pamubypenr); 1171 1172 iounmap(guts_regs); 1173 1174 /* Enable DMA for the LIODNs in the device tree */ 1175 1176 setup_liodns(); 1177 1178 probed = true; 1179 1180 return 0; 1181 1182 error_genpool: 1183 gen_pool_destroy(spaace_pool); 1184 1185 error: 1186 if (irq != NO_IRQ) 1187 free_irq(irq, data); 1188 1189 if (data) { 1190 memset(data, 0, sizeof(struct pamu_isr_data)); 1191 kfree(data); 1192 } 1193 1194 if (pamu_regs) 1195 iounmap(pamu_regs); 1196 1197 if (guts_regs) 1198 iounmap(guts_regs); 1199 1200 if (ppaact) 1201 free_pages((unsigned long)ppaact, order); 1202 1203 ppaact = NULL; 1204 1205 return ret; 1206 } 1207 1208 static struct platform_driver fsl_of_pamu_driver = { 1209 .driver = { 1210 .name = "fsl-of-pamu", 1211 }, 1212 .probe = fsl_pamu_probe, 1213 }; 1214 1215 static __init int fsl_pamu_init(void) 1216 { 1217 struct platform_device *pdev = NULL; 1218 struct device_node *np; 1219 int ret; 1220 1221 /* 1222 * The normal OF process calls the probe function at some 1223 * indeterminate later time, after most drivers have loaded. This is 1224 * too late for us, because PAMU clients (like the Qman driver) 1225 * depend on PAMU being initialized early. 1226 * 1227 * So instead, we "manually" call our probe function by creating the 1228 * platform devices ourselves. 1229 */ 1230 1231 /* 1232 * We assume that there is only one PAMU node in the device tree. A 1233 * single PAMU node represents all of the PAMU devices in the SOC 1234 * already. Everything else already makes that assumption, and the 1235 * binding for the PAMU nodes doesn't allow for any parent-child 1236 * relationships anyway. In other words, support for more than one 1237 * PAMU node would require significant changes to a lot of code. 1238 */ 1239 1240 np = of_find_compatible_node(NULL, NULL, "fsl,pamu"); 1241 if (!np) { 1242 pr_err("could not find a PAMU node\n"); 1243 return -ENODEV; 1244 } 1245 1246 ret = platform_driver_register(&fsl_of_pamu_driver); 1247 if (ret) { 1248 pr_err("could not register driver (err=%i)\n", ret); 1249 goto error_driver_register; 1250 } 1251 1252 pdev = platform_device_alloc("fsl-of-pamu", 0); 1253 if (!pdev) { 1254 pr_err("could not allocate device %pOF\n", np); 1255 ret = -ENOMEM; 1256 goto error_device_alloc; 1257 } 1258 pdev->dev.of_node = of_node_get(np); 1259 1260 ret = pamu_domain_init(); 1261 if (ret) 1262 goto error_device_add; 1263 1264 ret = platform_device_add(pdev); 1265 if (ret) { 1266 pr_err("could not add device %pOF (err=%i)\n", np, ret); 1267 goto error_device_add; 1268 } 1269 1270 return 0; 1271 1272 error_device_add: 1273 of_node_put(pdev->dev.of_node); 1274 pdev->dev.of_node = NULL; 1275 1276 platform_device_put(pdev); 1277 1278 error_device_alloc: 1279 platform_driver_unregister(&fsl_of_pamu_driver); 1280 1281 error_driver_register: 1282 of_node_put(np); 1283 1284 return ret; 1285 } 1286 arch_initcall(fsl_pamu_init); 1287