1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * You should have received a copy of the GNU General Public License 12 * along with this program; if not, write to the Free Software 13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 14 * 15 * Copyright (C) 2013 Freescale Semiconductor, Inc. 16 * 17 */ 18 19 #define pr_fmt(fmt) "fsl-pamu: %s: " fmt, __func__ 20 21 #include <linux/init.h> 22 #include <linux/iommu.h> 23 #include <linux/slab.h> 24 #include <linux/module.h> 25 #include <linux/types.h> 26 #include <linux/mm.h> 27 #include <linux/interrupt.h> 28 #include <linux/device.h> 29 #include <linux/of_platform.h> 30 #include <linux/bootmem.h> 31 #include <linux/genalloc.h> 32 #include <asm/io.h> 33 #include <asm/bitops.h> 34 #include <asm/fsl_guts.h> 35 36 #include "fsl_pamu.h" 37 38 /* define indexes for each operation mapping scenario */ 39 #define OMI_QMAN 0x00 40 #define OMI_FMAN 0x01 41 #define OMI_QMAN_PRIV 0x02 42 #define OMI_CAAM 0x03 43 44 #define make64(high, low) (((u64)(high) << 32) | (low)) 45 46 struct pamu_isr_data { 47 void __iomem *pamu_reg_base; /* Base address of PAMU regs*/ 48 unsigned int count; /* The number of PAMUs */ 49 }; 50 51 static struct paace *ppaact; 52 static struct paace *spaact; 53 static struct ome *omt; 54 55 /* 56 * Table for matching compatible strings, for device tree 57 * guts node, for QorIQ SOCs. 58 * "fsl,qoriq-device-config-2.0" corresponds to T4 & B4 59 * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" 60 * string would be used. 61 */ 62 static const struct of_device_id guts_device_ids[] = { 63 { .compatible = "fsl,qoriq-device-config-1.0", }, 64 { .compatible = "fsl,qoriq-device-config-2.0", }, 65 {} 66 }; 67 68 69 /* 70 * Table for matching compatible strings, for device tree 71 * L3 cache controller node. 72 * "fsl,t4240-l3-cache-controller" corresponds to T4, 73 * "fsl,b4860-l3-cache-controller" corresponds to B4 & 74 * "fsl,p4080-l3-cache-controller" corresponds to other, 75 * SOCs. 76 */ 77 static const struct of_device_id l3_device_ids[] = { 78 { .compatible = "fsl,t4240-l3-cache-controller", }, 79 { .compatible = "fsl,b4860-l3-cache-controller", }, 80 { .compatible = "fsl,p4080-l3-cache-controller", }, 81 {} 82 }; 83 84 /* maximum subwindows permitted per liodn */ 85 static u32 max_subwindow_count; 86 87 /* Pool for fspi allocation */ 88 struct gen_pool *spaace_pool; 89 90 /** 91 * pamu_get_max_subwin_cnt() - Return the maximum supported 92 * subwindow count per liodn. 93 * 94 */ 95 u32 pamu_get_max_subwin_cnt(void) 96 { 97 return max_subwindow_count; 98 } 99 100 /** 101 * pamu_get_ppaace() - Return the primary PACCE 102 * @liodn: liodn PAACT index for desired PAACE 103 * 104 * Returns the ppace pointer upon success else return 105 * null. 106 */ 107 static struct paace *pamu_get_ppaace(int liodn) 108 { 109 if (!ppaact || liodn >= PAACE_NUMBER_ENTRIES) { 110 pr_debug("PPAACT doesn't exist\n"); 111 return NULL; 112 } 113 114 return &ppaact[liodn]; 115 } 116 117 /** 118 * pamu_enable_liodn() - Set valid bit of PACCE 119 * @liodn: liodn PAACT index for desired PAACE 120 * 121 * Returns 0 upon success else error code < 0 returned 122 */ 123 int pamu_enable_liodn(int liodn) 124 { 125 struct paace *ppaace; 126 127 ppaace = pamu_get_ppaace(liodn); 128 if (!ppaace) { 129 pr_debug("Invalid primary paace entry\n"); 130 return -ENOENT; 131 } 132 133 if (!get_bf(ppaace->addr_bitfields, PPAACE_AF_WSE)) { 134 pr_debug("liodn %d not configured\n", liodn); 135 return -EINVAL; 136 } 137 138 /* Ensure that all other stores to the ppaace complete first */ 139 mb(); 140 141 set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); 142 mb(); 143 144 return 0; 145 } 146 147 /** 148 * pamu_disable_liodn() - Clears valid bit of PACCE 149 * @liodn: liodn PAACT index for desired PAACE 150 * 151 * Returns 0 upon success else error code < 0 returned 152 */ 153 int pamu_disable_liodn(int liodn) 154 { 155 struct paace *ppaace; 156 157 ppaace = pamu_get_ppaace(liodn); 158 if (!ppaace) { 159 pr_debug("Invalid primary paace entry\n"); 160 return -ENOENT; 161 } 162 163 set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); 164 mb(); 165 166 return 0; 167 } 168 169 /* Derive the window size encoding for a particular PAACE entry */ 170 static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size) 171 { 172 /* Bug if not a power of 2 */ 173 BUG_ON((addrspace_size & (addrspace_size - 1))); 174 175 /* window size is 2^(WSE+1) bytes */ 176 return fls64(addrspace_size) - 2; 177 } 178 179 /* Derive the PAACE window count encoding for the subwindow count */ 180 static unsigned int map_subwindow_cnt_to_wce(u32 subwindow_cnt) 181 { 182 /* window count is 2^(WCE+1) bytes */ 183 return __ffs(subwindow_cnt) - 1; 184 } 185 186 /* 187 * Set the PAACE type as primary and set the coherency required domain 188 * attribute 189 */ 190 static void pamu_init_ppaace(struct paace *ppaace) 191 { 192 set_bf(ppaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_PRIMARY); 193 194 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 195 PAACE_M_COHERENCE_REQ); 196 } 197 198 /* 199 * Set the PAACE type as secondary and set the coherency required domain 200 * attribute. 201 */ 202 static void pamu_init_spaace(struct paace *spaace) 203 { 204 set_bf(spaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_SECONDARY); 205 set_bf(spaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 206 PAACE_M_COHERENCE_REQ); 207 } 208 209 /* 210 * Return the spaace (corresponding to the secondary window index) 211 * for a particular ppaace. 212 */ 213 static struct paace *pamu_get_spaace(struct paace *paace, u32 wnum) 214 { 215 u32 subwin_cnt; 216 struct paace *spaace = NULL; 217 218 subwin_cnt = 1UL << (get_bf(paace->impl_attr, PAACE_IA_WCE) + 1); 219 220 if (wnum < subwin_cnt) 221 spaace = &spaact[paace->fspi + wnum]; 222 else 223 pr_debug("secondary paace out of bounds\n"); 224 225 return spaace; 226 } 227 228 /** 229 * pamu_get_fspi_and_allocate() - Allocates fspi index and reserves subwindows 230 * required for primary PAACE in the secondary 231 * PAACE table. 232 * @subwin_cnt: Number of subwindows to be reserved. 233 * 234 * A PPAACE entry may have a number of associated subwindows. A subwindow 235 * corresponds to a SPAACE entry in the SPAACT table. Each PAACE entry stores 236 * the index (fspi) of the first SPAACE entry in the SPAACT table. This 237 * function returns the index of the first SPAACE entry. The remaining 238 * SPAACE entries are reserved contiguously from that index. 239 * 240 * Returns a valid fspi index in the range of 0 - SPAACE_NUMBER_ENTRIES on success. 241 * If no SPAACE entry is available or the allocator can not reserve the required 242 * number of contiguous entries function returns ULONG_MAX indicating a failure. 243 * 244 */ 245 static unsigned long pamu_get_fspi_and_allocate(u32 subwin_cnt) 246 { 247 unsigned long spaace_addr; 248 249 spaace_addr = gen_pool_alloc(spaace_pool, subwin_cnt * sizeof(struct paace)); 250 if (!spaace_addr) 251 return ULONG_MAX; 252 253 return (spaace_addr - (unsigned long)spaact) / (sizeof(struct paace)); 254 } 255 256 /* Release the subwindows reserved for a particular LIODN */ 257 void pamu_free_subwins(int liodn) 258 { 259 struct paace *ppaace; 260 u32 subwin_cnt, size; 261 262 ppaace = pamu_get_ppaace(liodn); 263 if (!ppaace) { 264 pr_debug("Invalid liodn entry\n"); 265 return; 266 } 267 268 if (get_bf(ppaace->addr_bitfields, PPAACE_AF_MW)) { 269 subwin_cnt = 1UL << (get_bf(ppaace->impl_attr, PAACE_IA_WCE) + 1); 270 size = (subwin_cnt - 1) * sizeof(struct paace); 271 gen_pool_free(spaace_pool, (unsigned long)&spaact[ppaace->fspi], size); 272 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); 273 } 274 } 275 276 /* 277 * Function used for updating stash destination for the coressponding 278 * LIODN. 279 */ 280 int pamu_update_paace_stash(int liodn, u32 subwin, u32 value) 281 { 282 struct paace *paace; 283 284 paace = pamu_get_ppaace(liodn); 285 if (!paace) { 286 pr_debug("Invalid liodn entry\n"); 287 return -ENOENT; 288 } 289 if (subwin) { 290 paace = pamu_get_spaace(paace, subwin - 1); 291 if (!paace) { 292 return -ENOENT; 293 } 294 } 295 set_bf(paace->impl_attr, PAACE_IA_CID, value); 296 297 mb(); 298 299 return 0; 300 } 301 302 /* Disable a subwindow corresponding to the LIODN */ 303 int pamu_disable_spaace(int liodn, u32 subwin) 304 { 305 struct paace *paace; 306 307 paace = pamu_get_ppaace(liodn); 308 if (!paace) { 309 pr_debug("Invalid liodn entry\n"); 310 return -ENOENT; 311 } 312 if (subwin) { 313 paace = pamu_get_spaace(paace, subwin - 1); 314 if (!paace) { 315 return -ENOENT; 316 } 317 set_bf(paace->addr_bitfields, PAACE_AF_V, 318 PAACE_V_INVALID); 319 } else { 320 set_bf(paace->addr_bitfields, PAACE_AF_AP, 321 PAACE_AP_PERMS_DENIED); 322 } 323 324 mb(); 325 326 return 0; 327 } 328 329 330 /** 331 * pamu_config_paace() - Sets up PPAACE entry for specified liodn 332 * 333 * @liodn: Logical IO device number 334 * @win_addr: starting address of DSA window 335 * @win-size: size of DSA window 336 * @omi: Operation mapping index -- if ~omi == 0 then omi not defined 337 * @rpn: real (true physical) page number 338 * @stashid: cache stash id for associated cpu -- if ~stashid == 0 then 339 * stashid not defined 340 * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then 341 * snoopid not defined 342 * @subwin_cnt: number of sub-windows 343 * @prot: window permissions 344 * 345 * Returns 0 upon success else error code < 0 returned 346 */ 347 int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, 348 u32 omi, unsigned long rpn, u32 snoopid, u32 stashid, 349 u32 subwin_cnt, int prot) 350 { 351 struct paace *ppaace; 352 unsigned long fspi; 353 354 if ((win_size & (win_size - 1)) || win_size < PAMU_PAGE_SIZE) { 355 pr_debug("window size too small or not a power of two %llx\n", win_size); 356 return -EINVAL; 357 } 358 359 if (win_addr & (win_size - 1)) { 360 pr_debug("window address is not aligned with window size\n"); 361 return -EINVAL; 362 } 363 364 ppaace = pamu_get_ppaace(liodn); 365 if (!ppaace) { 366 return -ENOENT; 367 } 368 369 /* window size is 2^(WSE+1) bytes */ 370 set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 371 map_addrspace_size_to_wse(win_size)); 372 373 pamu_init_ppaace(ppaace); 374 375 ppaace->wbah = win_addr >> (PAMU_PAGE_SHIFT + 20); 376 set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 377 (win_addr >> PAMU_PAGE_SHIFT)); 378 379 /* set up operation mapping if it's configured */ 380 if (omi < OME_NUMBER_ENTRIES) { 381 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 382 ppaace->op_encode.index_ot.omi = omi; 383 } else if (~omi != 0) { 384 pr_debug("bad operation mapping index: %d\n", omi); 385 return -EINVAL; 386 } 387 388 /* configure stash id */ 389 if (~stashid != 0) 390 set_bf(ppaace->impl_attr, PAACE_IA_CID, stashid); 391 392 /* configure snoop id */ 393 if (~snoopid != 0) 394 ppaace->domain_attr.to_host.snpid = snoopid; 395 396 if (subwin_cnt) { 397 /* The first entry is in the primary PAACE instead */ 398 fspi = pamu_get_fspi_and_allocate(subwin_cnt - 1); 399 if (fspi == ULONG_MAX) { 400 pr_debug("spaace indexes exhausted\n"); 401 return -EINVAL; 402 } 403 404 /* window count is 2^(WCE+1) bytes */ 405 set_bf(ppaace->impl_attr, PAACE_IA_WCE, 406 map_subwindow_cnt_to_wce(subwin_cnt)); 407 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0x1); 408 ppaace->fspi = fspi; 409 } else { 410 set_bf(ppaace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); 411 ppaace->twbah = rpn >> 20; 412 set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, rpn); 413 set_bf(ppaace->addr_bitfields, PAACE_AF_AP, prot); 414 set_bf(ppaace->impl_attr, PAACE_IA_WCE, 0); 415 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); 416 } 417 mb(); 418 419 return 0; 420 } 421 422 /** 423 * pamu_config_spaace() - Sets up SPAACE entry for specified subwindow 424 * 425 * @liodn: Logical IO device number 426 * @subwin_cnt: number of sub-windows associated with dma-window 427 * @subwin: subwindow index 428 * @subwin_size: size of subwindow 429 * @omi: Operation mapping index 430 * @rpn: real (true physical) page number 431 * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then 432 * snoopid not defined 433 * @stashid: cache stash id for associated cpu 434 * @enable: enable/disable subwindow after reconfiguration 435 * @prot: sub window permissions 436 * 437 * Returns 0 upon success else error code < 0 returned 438 */ 439 int pamu_config_spaace(int liodn, u32 subwin_cnt, u32 subwin, 440 phys_addr_t subwin_size, u32 omi, unsigned long rpn, 441 u32 snoopid, u32 stashid, int enable, int prot) 442 { 443 struct paace *paace; 444 445 446 /* setup sub-windows */ 447 if (!subwin_cnt) { 448 pr_debug("Invalid subwindow count\n"); 449 return -EINVAL; 450 } 451 452 paace = pamu_get_ppaace(liodn); 453 if (subwin > 0 && subwin < subwin_cnt && paace) { 454 paace = pamu_get_spaace(paace, subwin - 1); 455 456 if (paace && !(paace->addr_bitfields & PAACE_V_VALID)) { 457 pamu_init_spaace(paace); 458 set_bf(paace->addr_bitfields, SPAACE_AF_LIODN, liodn); 459 } 460 } 461 462 if (!paace) { 463 pr_debug("Invalid liodn entry\n"); 464 return -ENOENT; 465 } 466 467 if ((subwin_size & (subwin_size - 1)) || subwin_size < PAMU_PAGE_SIZE) { 468 pr_debug("subwindow size out of range, or not a power of 2\n"); 469 return -EINVAL; 470 } 471 472 if (rpn == ULONG_MAX) { 473 pr_debug("real page number out of range\n"); 474 return -EINVAL; 475 } 476 477 /* window size is 2^(WSE+1) bytes */ 478 set_bf(paace->win_bitfields, PAACE_WIN_SWSE, 479 map_addrspace_size_to_wse(subwin_size)); 480 481 set_bf(paace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); 482 paace->twbah = rpn >> 20; 483 set_bf(paace->win_bitfields, PAACE_WIN_TWBAL, rpn); 484 set_bf(paace->addr_bitfields, PAACE_AF_AP, prot); 485 486 /* configure snoop id */ 487 if (~snoopid != 0) 488 paace->domain_attr.to_host.snpid = snoopid; 489 490 /* set up operation mapping if it's configured */ 491 if (omi < OME_NUMBER_ENTRIES) { 492 set_bf(paace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 493 paace->op_encode.index_ot.omi = omi; 494 } else if (~omi != 0) { 495 pr_debug("bad operation mapping index: %d\n", omi); 496 return -EINVAL; 497 } 498 499 if (~stashid != 0) 500 set_bf(paace->impl_attr, PAACE_IA_CID, stashid); 501 502 smp_wmb(); 503 504 if (enable) 505 set_bf(paace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); 506 507 mb(); 508 509 return 0; 510 } 511 512 /** 513 * get_ome_index() - Returns the index in the operation mapping table 514 * for device. 515 * @*omi_index: pointer for storing the index value 516 * 517 */ 518 void get_ome_index(u32 *omi_index, struct device *dev) 519 { 520 if (of_device_is_compatible(dev->of_node, "fsl,qman-portal")) 521 *omi_index = OMI_QMAN; 522 if (of_device_is_compatible(dev->of_node, "fsl,qman")) 523 *omi_index = OMI_QMAN_PRIV; 524 } 525 526 /** 527 * get_stash_id - Returns stash destination id corresponding to a 528 * cache type and vcpu. 529 * @stash_dest_hint: L1, L2 or L3 530 * @vcpu: vpcu target for a particular cache type. 531 * 532 * Returs stash on success or ~(u32)0 on failure. 533 * 534 */ 535 u32 get_stash_id(u32 stash_dest_hint, u32 vcpu) 536 { 537 const u32 *prop; 538 struct device_node *node; 539 u32 cache_level; 540 int len, found = 0; 541 int i; 542 543 /* Fastpath, exit early if L3/CPC cache is target for stashing */ 544 if (stash_dest_hint == PAMU_ATTR_CACHE_L3) { 545 node = of_find_matching_node(NULL, l3_device_ids); 546 if (node) { 547 prop = of_get_property(node, "cache-stash-id", 0); 548 if (!prop) { 549 pr_debug("missing cache-stash-id at %s\n", node->full_name); 550 of_node_put(node); 551 return ~(u32)0; 552 } 553 of_node_put(node); 554 return be32_to_cpup(prop); 555 } 556 return ~(u32)0; 557 } 558 559 for_each_node_by_type(node, "cpu") { 560 prop = of_get_property(node, "reg", &len); 561 for (i = 0; i < len / sizeof(u32); i++) { 562 if (be32_to_cpup(&prop[i]) == vcpu) { 563 found = 1; 564 goto found_cpu_node; 565 } 566 } 567 } 568 found_cpu_node: 569 570 /* find the hwnode that represents the cache */ 571 for (cache_level = PAMU_ATTR_CACHE_L1; (cache_level < PAMU_ATTR_CACHE_L3) && found; cache_level++) { 572 if (stash_dest_hint == cache_level) { 573 prop = of_get_property(node, "cache-stash-id", 0); 574 if (!prop) { 575 pr_debug("missing cache-stash-id at %s\n", node->full_name); 576 of_node_put(node); 577 return ~(u32)0; 578 } 579 of_node_put(node); 580 return be32_to_cpup(prop); 581 } 582 583 prop = of_get_property(node, "next-level-cache", 0); 584 if (!prop) { 585 pr_debug("can't find next-level-cache at %s\n", 586 node->full_name); 587 of_node_put(node); 588 return ~(u32)0; /* can't traverse any further */ 589 } 590 of_node_put(node); 591 592 /* advance to next node in cache hierarchy */ 593 node = of_find_node_by_phandle(*prop); 594 if (!node) { 595 pr_debug("Invalid node for cache hierarchy\n"); 596 return ~(u32)0; 597 } 598 } 599 600 pr_debug("stash dest not found for %d on vcpu %d\n", 601 stash_dest_hint, vcpu); 602 return ~(u32)0; 603 } 604 605 /* Identify if the PAACT table entry belongs to QMAN, BMAN or QMAN Portal */ 606 #define QMAN_PAACE 1 607 #define QMAN_PORTAL_PAACE 2 608 #define BMAN_PAACE 3 609 610 /** 611 * Setup operation mapping and stash destinations for QMAN and QMAN portal. 612 * Memory accesses to QMAN and BMAN private memory need not be coherent, so 613 * clear the PAACE entry coherency attribute for them. 614 */ 615 static void setup_qbman_paace(struct paace *ppaace, int paace_type) 616 { 617 switch (paace_type) { 618 case QMAN_PAACE: 619 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 620 ppaace->op_encode.index_ot.omi = OMI_QMAN_PRIV; 621 /* setup QMAN Private data stashing for the L3 cache */ 622 set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); 623 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 624 0); 625 break; 626 case QMAN_PORTAL_PAACE: 627 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 628 ppaace->op_encode.index_ot.omi = OMI_QMAN; 629 /*Set DQRR and Frame stashing for the L3 cache */ 630 set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); 631 break; 632 case BMAN_PAACE: 633 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 634 0); 635 break; 636 } 637 } 638 639 /** 640 * Setup the operation mapping table for various devices. This is a static 641 * table where each table index corresponds to a particular device. PAMU uses 642 * this table to translate device transaction to appropriate corenet 643 * transaction. 644 */ 645 static void __init setup_omt(struct ome *omt) 646 { 647 struct ome *ome; 648 649 /* Configure OMI_QMAN */ 650 ome = &omt[OMI_QMAN]; 651 652 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; 653 ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; 654 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 655 ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSAO; 656 657 ome->moe[IOE_DIRECT0_IDX] = EOE_VALID | EOE_LDEC; 658 ome->moe[IOE_DIRECT1_IDX] = EOE_VALID | EOE_LDECPE; 659 660 /* Configure OMI_FMAN */ 661 ome = &omt[OMI_FMAN]; 662 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; 663 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 664 665 /* Configure OMI_QMAN private */ 666 ome = &omt[OMI_QMAN_PRIV]; 667 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; 668 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 669 ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; 670 ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSA; 671 672 /* Configure OMI_CAAM */ 673 ome = &omt[OMI_CAAM]; 674 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; 675 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 676 } 677 678 /* 679 * Get the maximum number of PAACT table entries 680 * and subwindows supported by PAMU 681 */ 682 static void get_pamu_cap_values(unsigned long pamu_reg_base) 683 { 684 u32 pc_val; 685 686 pc_val = in_be32((u32 *)(pamu_reg_base + PAMU_PC3)); 687 /* Maximum number of subwindows per liodn */ 688 max_subwindow_count = 1 << (1 + PAMU_PC3_MWCE(pc_val)); 689 } 690 691 /* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ 692 int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, 693 phys_addr_t ppaact_phys, phys_addr_t spaact_phys, 694 phys_addr_t omt_phys) 695 { 696 u32 *pc; 697 struct pamu_mmap_regs *pamu_regs; 698 699 pc = (u32 *) (pamu_reg_base + PAMU_PC); 700 pamu_regs = (struct pamu_mmap_regs *) 701 (pamu_reg_base + PAMU_MMAP_REGS_BASE); 702 703 /* set up pointers to corenet control blocks */ 704 705 out_be32(&pamu_regs->ppbah, upper_32_bits(ppaact_phys)); 706 out_be32(&pamu_regs->ppbal, lower_32_bits(ppaact_phys)); 707 ppaact_phys = ppaact_phys + PAACT_SIZE; 708 out_be32(&pamu_regs->pplah, upper_32_bits(ppaact_phys)); 709 out_be32(&pamu_regs->pplal, lower_32_bits(ppaact_phys)); 710 711 out_be32(&pamu_regs->spbah, upper_32_bits(spaact_phys)); 712 out_be32(&pamu_regs->spbal, lower_32_bits(spaact_phys)); 713 spaact_phys = spaact_phys + SPAACT_SIZE; 714 out_be32(&pamu_regs->splah, upper_32_bits(spaact_phys)); 715 out_be32(&pamu_regs->splal, lower_32_bits(spaact_phys)); 716 717 out_be32(&pamu_regs->obah, upper_32_bits(omt_phys)); 718 out_be32(&pamu_regs->obal, lower_32_bits(omt_phys)); 719 omt_phys = omt_phys + OMT_SIZE; 720 out_be32(&pamu_regs->olah, upper_32_bits(omt_phys)); 721 out_be32(&pamu_regs->olal, lower_32_bits(omt_phys)); 722 723 /* 724 * set PAMU enable bit, 725 * allow ppaact & omt to be cached 726 * & enable PAMU access violation interrupts. 727 */ 728 729 out_be32((u32 *)(pamu_reg_base + PAMU_PICS), 730 PAMU_ACCESS_VIOLATION_ENABLE); 731 out_be32(pc, PAMU_PC_PE | PAMU_PC_OCE | PAMU_PC_SPCC | PAMU_PC_PPCC); 732 return 0; 733 } 734 735 /* Enable all device LIODNS */ 736 static void __init setup_liodns(void) 737 { 738 int i, len; 739 struct paace *ppaace; 740 struct device_node *node = NULL; 741 const u32 *prop; 742 743 for_each_node_with_property(node, "fsl,liodn") { 744 prop = of_get_property(node, "fsl,liodn", &len); 745 for (i = 0; i < len / sizeof(u32); i++) { 746 int liodn; 747 748 liodn = be32_to_cpup(&prop[i]); 749 if (liodn >= PAACE_NUMBER_ENTRIES) { 750 pr_debug("Invalid LIODN value %d\n", liodn); 751 continue; 752 } 753 ppaace = pamu_get_ppaace(liodn); 754 pamu_init_ppaace(ppaace); 755 /* window size is 2^(WSE+1) bytes */ 756 set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 35); 757 ppaace->wbah = 0; 758 set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); 759 set_bf(ppaace->impl_attr, PAACE_IA_ATM, 760 PAACE_ATM_NO_XLATE); 761 set_bf(ppaace->addr_bitfields, PAACE_AF_AP, 762 PAACE_AP_PERMS_ALL); 763 if (of_device_is_compatible(node, "fsl,qman-portal")) 764 setup_qbman_paace(ppaace, QMAN_PORTAL_PAACE); 765 if (of_device_is_compatible(node, "fsl,qman")) 766 setup_qbman_paace(ppaace, QMAN_PAACE); 767 if (of_device_is_compatible(node, "fsl,bman")) 768 setup_qbman_paace(ppaace, BMAN_PAACE); 769 mb(); 770 pamu_enable_liodn(liodn); 771 } 772 } 773 } 774 775 irqreturn_t pamu_av_isr(int irq, void *arg) 776 { 777 struct pamu_isr_data *data = arg; 778 phys_addr_t phys; 779 unsigned int i, j, ret; 780 781 pr_emerg("access violation interrupt\n"); 782 783 for (i = 0; i < data->count; i++) { 784 void __iomem *p = data->pamu_reg_base + i * PAMU_OFFSET; 785 u32 pics = in_be32(p + PAMU_PICS); 786 787 if (pics & PAMU_ACCESS_VIOLATION_STAT) { 788 u32 avs1 = in_be32(p + PAMU_AVS1); 789 struct paace *paace; 790 791 pr_emerg("POES1=%08x\n", in_be32(p + PAMU_POES1)); 792 pr_emerg("POES2=%08x\n", in_be32(p + PAMU_POES2)); 793 pr_emerg("AVS1=%08x\n", avs1); 794 pr_emerg("AVS2=%08x\n", in_be32(p + PAMU_AVS2)); 795 pr_emerg("AVA=%016llx\n", make64(in_be32(p + PAMU_AVAH), 796 in_be32(p + PAMU_AVAL))); 797 pr_emerg("UDAD=%08x\n", in_be32(p + PAMU_UDAD)); 798 pr_emerg("POEA=%016llx\n", make64(in_be32(p + PAMU_POEAH), 799 in_be32(p + PAMU_POEAL))); 800 801 phys = make64(in_be32(p + PAMU_POEAH), 802 in_be32(p + PAMU_POEAL)); 803 804 /* Assume that POEA points to a PAACE */ 805 if (phys) { 806 u32 *paace = phys_to_virt(phys); 807 808 /* Only the first four words are relevant */ 809 for (j = 0; j < 4; j++) 810 pr_emerg("PAACE[%u]=%08x\n", j, in_be32(paace + j)); 811 } 812 813 /* clear access violation condition */ 814 out_be32((p + PAMU_AVS1), avs1 & PAMU_AV_MASK); 815 paace = pamu_get_ppaace(avs1 >> PAMU_AVS1_LIODN_SHIFT); 816 BUG_ON(!paace); 817 /* check if we got a violation for a disabled LIODN */ 818 if (!get_bf(paace->addr_bitfields, PAACE_AF_V)) { 819 /* 820 * As per hardware erratum A-003638, access 821 * violation can be reported for a disabled 822 * LIODN. If we hit that condition, disable 823 * access violation reporting. 824 */ 825 pics &= ~PAMU_ACCESS_VIOLATION_ENABLE; 826 } else { 827 /* Disable the LIODN */ 828 ret = pamu_disable_liodn(avs1 >> PAMU_AVS1_LIODN_SHIFT); 829 BUG_ON(ret); 830 pr_emerg("Disabling liodn %x\n", avs1 >> PAMU_AVS1_LIODN_SHIFT); 831 } 832 out_be32((p + PAMU_PICS), pics); 833 } 834 } 835 836 837 return IRQ_HANDLED; 838 } 839 840 #define LAWAR_EN 0x80000000 841 #define LAWAR_TARGET_MASK 0x0FF00000 842 #define LAWAR_TARGET_SHIFT 20 843 #define LAWAR_SIZE_MASK 0x0000003F 844 #define LAWAR_CSDID_MASK 0x000FF000 845 #define LAWAR_CSDID_SHIFT 12 846 847 #define LAW_SIZE_4K 0xb 848 849 struct ccsr_law { 850 u32 lawbarh; /* LAWn base address high */ 851 u32 lawbarl; /* LAWn base address low */ 852 u32 lawar; /* LAWn attributes */ 853 u32 reserved; 854 }; 855 856 /* 857 * Create a coherence subdomain for a given memory block. 858 */ 859 static int __init create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) 860 { 861 struct device_node *np; 862 const __be32 *iprop; 863 void __iomem *lac = NULL; /* Local Access Control registers */ 864 struct ccsr_law __iomem *law; 865 void __iomem *ccm = NULL; 866 u32 __iomem *csdids; 867 unsigned int i, num_laws, num_csds; 868 u32 law_target = 0; 869 u32 csd_id = 0; 870 int ret = 0; 871 872 np = of_find_compatible_node(NULL, NULL, "fsl,corenet-law"); 873 if (!np) 874 return -ENODEV; 875 876 iprop = of_get_property(np, "fsl,num-laws", NULL); 877 if (!iprop) { 878 ret = -ENODEV; 879 goto error; 880 } 881 882 num_laws = be32_to_cpup(iprop); 883 if (!num_laws) { 884 ret = -ENODEV; 885 goto error; 886 } 887 888 lac = of_iomap(np, 0); 889 if (!lac) { 890 ret = -ENODEV; 891 goto error; 892 } 893 894 /* LAW registers are at offset 0xC00 */ 895 law = lac + 0xC00; 896 897 of_node_put(np); 898 899 np = of_find_compatible_node(NULL, NULL, "fsl,corenet-cf"); 900 if (!np) { 901 ret = -ENODEV; 902 goto error; 903 } 904 905 iprop = of_get_property(np, "fsl,ccf-num-csdids", NULL); 906 if (!iprop) { 907 ret = -ENODEV; 908 goto error; 909 } 910 911 num_csds = be32_to_cpup(iprop); 912 if (!num_csds) { 913 ret = -ENODEV; 914 goto error; 915 } 916 917 ccm = of_iomap(np, 0); 918 if (!ccm) { 919 ret = -ENOMEM; 920 goto error; 921 } 922 923 /* The undocumented CSDID registers are at offset 0x600 */ 924 csdids = ccm + 0x600; 925 926 of_node_put(np); 927 np = NULL; 928 929 /* Find an unused coherence subdomain ID */ 930 for (csd_id = 0; csd_id < num_csds; csd_id++) { 931 if (!csdids[csd_id]) 932 break; 933 } 934 935 /* Store the Port ID in the (undocumented) proper CIDMRxx register */ 936 csdids[csd_id] = csd_port_id; 937 938 /* Find the DDR LAW that maps to our buffer. */ 939 for (i = 0; i < num_laws; i++) { 940 if (law[i].lawar & LAWAR_EN) { 941 phys_addr_t law_start, law_end; 942 943 law_start = make64(law[i].lawbarh, law[i].lawbarl); 944 law_end = law_start + 945 (2ULL << (law[i].lawar & LAWAR_SIZE_MASK)); 946 947 if (law_start <= phys && phys < law_end) { 948 law_target = law[i].lawar & LAWAR_TARGET_MASK; 949 break; 950 } 951 } 952 } 953 954 if (i == 0 || i == num_laws) { 955 /* This should never happen*/ 956 ret = -ENOENT; 957 goto error; 958 } 959 960 /* Find a free LAW entry */ 961 while (law[--i].lawar & LAWAR_EN) { 962 if (i == 0) { 963 /* No higher priority LAW slots available */ 964 ret = -ENOENT; 965 goto error; 966 } 967 } 968 969 law[i].lawbarh = upper_32_bits(phys); 970 law[i].lawbarl = lower_32_bits(phys); 971 wmb(); 972 law[i].lawar = LAWAR_EN | law_target | (csd_id << LAWAR_CSDID_SHIFT) | 973 (LAW_SIZE_4K + get_order(size)); 974 wmb(); 975 976 error: 977 if (ccm) 978 iounmap(ccm); 979 980 if (lac) 981 iounmap(lac); 982 983 if (np) 984 of_node_put(np); 985 986 return ret; 987 } 988 989 /* 990 * Table of SVRs and the corresponding PORT_ID values. Port ID corresponds to a 991 * bit map of snoopers for a given range of memory mapped by a LAW. 992 * 993 * All future CoreNet-enabled SOCs will have this erratum(A-004510) fixed, so this 994 * table should never need to be updated. SVRs are guaranteed to be unique, so 995 * there is no worry that a future SOC will inadvertently have one of these 996 * values. 997 */ 998 static const struct { 999 u32 svr; 1000 u32 port_id; 1001 } port_id_map[] = { 1002 {0x82100010, 0xFF000000}, /* P2040 1.0 */ 1003 {0x82100011, 0xFF000000}, /* P2040 1.1 */ 1004 {0x82100110, 0xFF000000}, /* P2041 1.0 */ 1005 {0x82100111, 0xFF000000}, /* P2041 1.1 */ 1006 {0x82110310, 0xFF000000}, /* P3041 1.0 */ 1007 {0x82110311, 0xFF000000}, /* P3041 1.1 */ 1008 {0x82010020, 0xFFF80000}, /* P4040 2.0 */ 1009 {0x82000020, 0xFFF80000}, /* P4080 2.0 */ 1010 {0x82210010, 0xFC000000}, /* P5010 1.0 */ 1011 {0x82210020, 0xFC000000}, /* P5010 2.0 */ 1012 {0x82200010, 0xFC000000}, /* P5020 1.0 */ 1013 {0x82050010, 0xFF800000}, /* P5021 1.0 */ 1014 {0x82040010, 0xFF800000}, /* P5040 1.0 */ 1015 }; 1016 1017 #define SVR_SECURITY 0x80000 /* The Security (E) bit */ 1018 1019 static int __init fsl_pamu_probe(struct platform_device *pdev) 1020 { 1021 void __iomem *pamu_regs = NULL; 1022 struct ccsr_guts __iomem *guts_regs = NULL; 1023 u32 pamubypenr, pamu_counter; 1024 unsigned long pamu_reg_off; 1025 unsigned long pamu_reg_base; 1026 struct pamu_isr_data *data = NULL; 1027 struct device_node *guts_node; 1028 u64 size; 1029 struct page *p; 1030 int ret = 0; 1031 int irq; 1032 phys_addr_t ppaact_phys; 1033 phys_addr_t spaact_phys; 1034 phys_addr_t omt_phys; 1035 size_t mem_size = 0; 1036 unsigned int order = 0; 1037 u32 csd_port_id = 0; 1038 unsigned i; 1039 /* 1040 * enumerate all PAMUs and allocate and setup PAMU tables 1041 * for each of them, 1042 * NOTE : All PAMUs share the same LIODN tables. 1043 */ 1044 1045 pamu_regs = of_iomap(pdev->dev.of_node, 0); 1046 if (!pamu_regs) { 1047 dev_err(&pdev->dev, "ioremap of PAMU node failed\n"); 1048 return -ENOMEM; 1049 } 1050 of_get_address(pdev->dev.of_node, 0, &size, NULL); 1051 1052 irq = irq_of_parse_and_map(pdev->dev.of_node, 0); 1053 if (irq == NO_IRQ) { 1054 dev_warn(&pdev->dev, "no interrupts listed in PAMU node\n"); 1055 goto error; 1056 } 1057 1058 data = kzalloc(sizeof(struct pamu_isr_data), GFP_KERNEL); 1059 if (!data) { 1060 dev_err(&pdev->dev, "PAMU isr data memory allocation failed\n"); 1061 ret = -ENOMEM; 1062 goto error; 1063 } 1064 data->pamu_reg_base = pamu_regs; 1065 data->count = size / PAMU_OFFSET; 1066 1067 /* The ISR needs access to the regs, so we won't iounmap them */ 1068 ret = request_irq(irq, pamu_av_isr, 0, "pamu", data); 1069 if (ret < 0) { 1070 dev_err(&pdev->dev, "error %i installing ISR for irq %i\n", 1071 ret, irq); 1072 goto error; 1073 } 1074 1075 guts_node = of_find_matching_node(NULL, guts_device_ids); 1076 if (!guts_node) { 1077 dev_err(&pdev->dev, "could not find GUTS node %s\n", 1078 pdev->dev.of_node->full_name); 1079 ret = -ENODEV; 1080 goto error; 1081 } 1082 1083 guts_regs = of_iomap(guts_node, 0); 1084 of_node_put(guts_node); 1085 if (!guts_regs) { 1086 dev_err(&pdev->dev, "ioremap of GUTS node failed\n"); 1087 ret = -ENODEV; 1088 goto error; 1089 } 1090 1091 /* read in the PAMU capability registers */ 1092 get_pamu_cap_values((unsigned long)pamu_regs); 1093 /* 1094 * To simplify the allocation of a coherency domain, we allocate the 1095 * PAACT and the OMT in the same memory buffer. Unfortunately, this 1096 * wastes more memory compared to allocating the buffers separately. 1097 */ 1098 /* Determine how much memory we need */ 1099 mem_size = (PAGE_SIZE << get_order(PAACT_SIZE)) + 1100 (PAGE_SIZE << get_order(SPAACT_SIZE)) + 1101 (PAGE_SIZE << get_order(OMT_SIZE)); 1102 order = get_order(mem_size); 1103 1104 p = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); 1105 if (!p) { 1106 dev_err(&pdev->dev, "unable to allocate PAACT/SPAACT/OMT block\n"); 1107 ret = -ENOMEM; 1108 goto error; 1109 } 1110 1111 ppaact = page_address(p); 1112 ppaact_phys = page_to_phys(p); 1113 1114 /* Make sure the memory is naturally aligned */ 1115 if (ppaact_phys & ((PAGE_SIZE << order) - 1)) { 1116 dev_err(&pdev->dev, "PAACT/OMT block is unaligned\n"); 1117 ret = -ENOMEM; 1118 goto error; 1119 } 1120 1121 spaact = (void *)ppaact + (PAGE_SIZE << get_order(PAACT_SIZE)); 1122 omt = (void *)spaact + (PAGE_SIZE << get_order(SPAACT_SIZE)); 1123 1124 dev_dbg(&pdev->dev, "ppaact virt=%p phys=0x%llx\n", ppaact, 1125 (unsigned long long) ppaact_phys); 1126 1127 /* Check to see if we need to implement the work-around on this SOC */ 1128 1129 /* Determine the Port ID for our coherence subdomain */ 1130 for (i = 0; i < ARRAY_SIZE(port_id_map); i++) { 1131 if (port_id_map[i].svr == (mfspr(SPRN_SVR) & ~SVR_SECURITY)) { 1132 csd_port_id = port_id_map[i].port_id; 1133 dev_dbg(&pdev->dev, "found matching SVR %08x\n", 1134 port_id_map[i].svr); 1135 break; 1136 } 1137 } 1138 1139 if (csd_port_id) { 1140 dev_dbg(&pdev->dev, "creating coherency subdomain at address " 1141 "0x%llx, size %zu, port id 0x%08x", ppaact_phys, 1142 mem_size, csd_port_id); 1143 1144 ret = create_csd(ppaact_phys, mem_size, csd_port_id); 1145 if (ret) { 1146 dev_err(&pdev->dev, "could not create coherence " 1147 "subdomain\n"); 1148 return ret; 1149 } 1150 } 1151 1152 spaact_phys = virt_to_phys(spaact); 1153 omt_phys = virt_to_phys(omt); 1154 1155 spaace_pool = gen_pool_create(ilog2(sizeof(struct paace)), -1); 1156 if (!spaace_pool) { 1157 ret = -ENOMEM; 1158 dev_err(&pdev->dev, "PAMU : failed to allocate spaace gen pool\n"); 1159 goto error; 1160 } 1161 1162 ret = gen_pool_add(spaace_pool, (unsigned long)spaact, SPAACT_SIZE, -1); 1163 if (ret) 1164 goto error_genpool; 1165 1166 pamubypenr = in_be32(&guts_regs->pamubypenr); 1167 1168 for (pamu_reg_off = 0, pamu_counter = 0x80000000; pamu_reg_off < size; 1169 pamu_reg_off += PAMU_OFFSET, pamu_counter >>= 1) { 1170 1171 pamu_reg_base = (unsigned long) pamu_regs + pamu_reg_off; 1172 setup_one_pamu(pamu_reg_base, pamu_reg_off, ppaact_phys, 1173 spaact_phys, omt_phys); 1174 /* Disable PAMU bypass for this PAMU */ 1175 pamubypenr &= ~pamu_counter; 1176 } 1177 1178 setup_omt(omt); 1179 1180 /* Enable all relevant PAMU(s) */ 1181 out_be32(&guts_regs->pamubypenr, pamubypenr); 1182 1183 iounmap(guts_regs); 1184 1185 /* Enable DMA for the LIODNs in the device tree*/ 1186 1187 setup_liodns(); 1188 1189 return 0; 1190 1191 error_genpool: 1192 gen_pool_destroy(spaace_pool); 1193 1194 error: 1195 if (irq != NO_IRQ) 1196 free_irq(irq, data); 1197 1198 if (data) { 1199 memset(data, 0, sizeof(struct pamu_isr_data)); 1200 kfree(data); 1201 } 1202 1203 if (pamu_regs) 1204 iounmap(pamu_regs); 1205 1206 if (guts_regs) 1207 iounmap(guts_regs); 1208 1209 if (ppaact) 1210 free_pages((unsigned long)ppaact, order); 1211 1212 ppaact = NULL; 1213 1214 return ret; 1215 } 1216 1217 static const struct of_device_id fsl_of_pamu_ids[] = { 1218 { 1219 .compatible = "fsl,p4080-pamu", 1220 }, 1221 { 1222 .compatible = "fsl,pamu", 1223 }, 1224 {}, 1225 }; 1226 1227 static struct platform_driver fsl_of_pamu_driver = { 1228 .driver = { 1229 .name = "fsl-of-pamu", 1230 }, 1231 .probe = fsl_pamu_probe, 1232 }; 1233 1234 static __init int fsl_pamu_init(void) 1235 { 1236 struct platform_device *pdev = NULL; 1237 struct device_node *np; 1238 int ret; 1239 1240 /* 1241 * The normal OF process calls the probe function at some 1242 * indeterminate later time, after most drivers have loaded. This is 1243 * too late for us, because PAMU clients (like the Qman driver) 1244 * depend on PAMU being initialized early. 1245 * 1246 * So instead, we "manually" call our probe function by creating the 1247 * platform devices ourselves. 1248 */ 1249 1250 /* 1251 * We assume that there is only one PAMU node in the device tree. A 1252 * single PAMU node represents all of the PAMU devices in the SOC 1253 * already. Everything else already makes that assumption, and the 1254 * binding for the PAMU nodes doesn't allow for any parent-child 1255 * relationships anyway. In other words, support for more than one 1256 * PAMU node would require significant changes to a lot of code. 1257 */ 1258 1259 np = of_find_compatible_node(NULL, NULL, "fsl,pamu"); 1260 if (!np) { 1261 pr_err("could not find a PAMU node\n"); 1262 return -ENODEV; 1263 } 1264 1265 ret = platform_driver_register(&fsl_of_pamu_driver); 1266 if (ret) { 1267 pr_err("could not register driver (err=%i)\n", ret); 1268 goto error_driver_register; 1269 } 1270 1271 pdev = platform_device_alloc("fsl-of-pamu", 0); 1272 if (!pdev) { 1273 pr_err("could not allocate device %s\n", 1274 np->full_name); 1275 ret = -ENOMEM; 1276 goto error_device_alloc; 1277 } 1278 pdev->dev.of_node = of_node_get(np); 1279 1280 ret = pamu_domain_init(); 1281 if (ret) 1282 goto error_device_add; 1283 1284 ret = platform_device_add(pdev); 1285 if (ret) { 1286 pr_err("could not add device %s (err=%i)\n", 1287 np->full_name, ret); 1288 goto error_device_add; 1289 } 1290 1291 return 0; 1292 1293 error_device_add: 1294 of_node_put(pdev->dev.of_node); 1295 pdev->dev.of_node = NULL; 1296 1297 platform_device_put(pdev); 1298 1299 error_device_alloc: 1300 platform_driver_unregister(&fsl_of_pamu_driver); 1301 1302 error_driver_register: 1303 of_node_put(np); 1304 1305 return ret; 1306 } 1307 arch_initcall(fsl_pamu_init); 1308