1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation 4 * 5 * Rewrite, cleanup: 6 * 7 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation 8 * Copyright (C) 2006 Olof Johansson <olof@lixom.net> 9 * 10 * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR. 11 */ 12 13 #include <linux/init.h> 14 #include <linux/types.h> 15 #include <linux/slab.h> 16 #include <linux/mm.h> 17 #include <linux/memblock.h> 18 #include <linux/spinlock.h> 19 #include <linux/string.h> 20 #include <linux/pci.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/crash_dump.h> 23 #include <linux/memory.h> 24 #include <linux/of.h> 25 #include <linux/iommu.h> 26 #include <linux/rculist.h> 27 #include <asm/io.h> 28 #include <asm/prom.h> 29 #include <asm/rtas.h> 30 #include <asm/iommu.h> 31 #include <asm/pci-bridge.h> 32 #include <asm/machdep.h> 33 #include <asm/firmware.h> 34 #include <asm/tce.h> 35 #include <asm/ppc-pci.h> 36 #include <asm/udbg.h> 37 #include <asm/mmzone.h> 38 #include <asm/plpar_wrappers.h> 39 #include <asm/svm.h> 40 41 #include "pseries.h" 42 43 static struct iommu_table_group *iommu_pseries_alloc_group(int node) 44 { 45 struct iommu_table_group *table_group; 46 struct iommu_table *tbl; 47 48 table_group = kzalloc_node(sizeof(struct iommu_table_group), GFP_KERNEL, 49 node); 50 if (!table_group) 51 return NULL; 52 53 tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, node); 54 if (!tbl) 55 goto free_group; 56 57 INIT_LIST_HEAD_RCU(&tbl->it_group_list); 58 kref_init(&tbl->it_kref); 59 60 table_group->tables[0] = tbl; 61 62 return table_group; 63 64 free_group: 65 kfree(table_group); 66 return NULL; 67 } 68 69 static void iommu_pseries_free_group(struct iommu_table_group *table_group, 70 const char *node_name) 71 { 72 struct iommu_table *tbl; 73 74 if (!table_group) 75 return; 76 77 tbl = table_group->tables[0]; 78 #ifdef CONFIG_IOMMU_API 79 if (table_group->group) { 80 iommu_group_put(table_group->group); 81 BUG_ON(table_group->group); 82 } 83 #endif 84 iommu_tce_table_put(tbl); 85 86 kfree(table_group); 87 } 88 89 static int tce_build_pSeries(struct iommu_table *tbl, long index, 90 long npages, unsigned long uaddr, 91 enum dma_data_direction direction, 92 unsigned long attrs) 93 { 94 u64 proto_tce; 95 __be64 *tcep; 96 u64 rpn; 97 98 proto_tce = TCE_PCI_READ; // Read allowed 99 100 if (direction != DMA_TO_DEVICE) 101 proto_tce |= TCE_PCI_WRITE; 102 103 tcep = ((__be64 *)tbl->it_base) + index; 104 105 while (npages--) { 106 /* can't move this out since we might cross MEMBLOCK boundary */ 107 rpn = __pa(uaddr) >> TCE_SHIFT; 108 *tcep = cpu_to_be64(proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT); 109 110 uaddr += TCE_PAGE_SIZE; 111 tcep++; 112 } 113 return 0; 114 } 115 116 117 static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages) 118 { 119 __be64 *tcep; 120 121 tcep = ((__be64 *)tbl->it_base) + index; 122 123 while (npages--) 124 *(tcep++) = 0; 125 } 126 127 static unsigned long tce_get_pseries(struct iommu_table *tbl, long index) 128 { 129 __be64 *tcep; 130 131 tcep = ((__be64 *)tbl->it_base) + index; 132 133 return be64_to_cpu(*tcep); 134 } 135 136 static void tce_free_pSeriesLP(struct iommu_table*, long, long); 137 static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long); 138 139 static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum, 140 long npages, unsigned long uaddr, 141 enum dma_data_direction direction, 142 unsigned long attrs) 143 { 144 u64 rc = 0; 145 u64 proto_tce, tce; 146 u64 rpn; 147 int ret = 0; 148 long tcenum_start = tcenum, npages_start = npages; 149 150 rpn = __pa(uaddr) >> TCE_SHIFT; 151 proto_tce = TCE_PCI_READ; 152 if (direction != DMA_TO_DEVICE) 153 proto_tce |= TCE_PCI_WRITE; 154 155 while (npages--) { 156 tce = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT; 157 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce); 158 159 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) { 160 ret = (int)rc; 161 tce_free_pSeriesLP(tbl, tcenum_start, 162 (npages_start - (npages + 1))); 163 break; 164 } 165 166 if (rc && printk_ratelimit()) { 167 printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 168 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 169 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 170 printk("\ttce val = 0x%llx\n", tce ); 171 dump_stack(); 172 } 173 174 tcenum++; 175 rpn++; 176 } 177 return ret; 178 } 179 180 static DEFINE_PER_CPU(__be64 *, tce_page); 181 182 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, 183 long npages, unsigned long uaddr, 184 enum dma_data_direction direction, 185 unsigned long attrs) 186 { 187 u64 rc = 0; 188 u64 proto_tce; 189 __be64 *tcep; 190 u64 rpn; 191 long l, limit; 192 long tcenum_start = tcenum, npages_start = npages; 193 int ret = 0; 194 unsigned long flags; 195 196 if ((npages == 1) || !firmware_has_feature(FW_FEATURE_MULTITCE)) { 197 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr, 198 direction, attrs); 199 } 200 201 local_irq_save(flags); /* to protect tcep and the page behind it */ 202 203 tcep = __this_cpu_read(tce_page); 204 205 /* This is safe to do since interrupts are off when we're called 206 * from iommu_alloc{,_sg}() 207 */ 208 if (!tcep) { 209 tcep = (__be64 *)__get_free_page(GFP_ATOMIC); 210 /* If allocation fails, fall back to the loop implementation */ 211 if (!tcep) { 212 local_irq_restore(flags); 213 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr, 214 direction, attrs); 215 } 216 __this_cpu_write(tce_page, tcep); 217 } 218 219 rpn = __pa(uaddr) >> TCE_SHIFT; 220 proto_tce = TCE_PCI_READ; 221 if (direction != DMA_TO_DEVICE) 222 proto_tce |= TCE_PCI_WRITE; 223 224 /* We can map max one pageful of TCEs at a time */ 225 do { 226 /* 227 * Set up the page with TCE data, looping through and setting 228 * the values. 229 */ 230 limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE); 231 232 for (l = 0; l < limit; l++) { 233 tcep[l] = cpu_to_be64(proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT); 234 rpn++; 235 } 236 237 rc = plpar_tce_put_indirect((u64)tbl->it_index, 238 (u64)tcenum << 12, 239 (u64)__pa(tcep), 240 limit); 241 242 npages -= limit; 243 tcenum += limit; 244 } while (npages > 0 && !rc); 245 246 local_irq_restore(flags); 247 248 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) { 249 ret = (int)rc; 250 tce_freemulti_pSeriesLP(tbl, tcenum_start, 251 (npages_start - (npages + limit))); 252 return ret; 253 } 254 255 if (rc && printk_ratelimit()) { 256 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 257 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 258 printk("\tnpages = 0x%llx\n", (u64)npages); 259 printk("\ttce[0] val = 0x%llx\n", tcep[0]); 260 dump_stack(); 261 } 262 return ret; 263 } 264 265 static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages) 266 { 267 u64 rc; 268 269 while (npages--) { 270 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0); 271 272 if (rc && printk_ratelimit()) { 273 printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc); 274 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 275 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 276 dump_stack(); 277 } 278 279 tcenum++; 280 } 281 } 282 283 284 static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages) 285 { 286 u64 rc; 287 288 if (!firmware_has_feature(FW_FEATURE_MULTITCE)) 289 return tce_free_pSeriesLP(tbl, tcenum, npages); 290 291 rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages); 292 293 if (rc && printk_ratelimit()) { 294 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n"); 295 printk("\trc = %lld\n", rc); 296 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 297 printk("\tnpages = 0x%llx\n", (u64)npages); 298 dump_stack(); 299 } 300 } 301 302 static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum) 303 { 304 u64 rc; 305 unsigned long tce_ret; 306 307 rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret); 308 309 if (rc && printk_ratelimit()) { 310 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc); 311 printk("\tindex = 0x%llx\n", (u64)tbl->it_index); 312 printk("\ttcenum = 0x%llx\n", (u64)tcenum); 313 dump_stack(); 314 } 315 316 return tce_ret; 317 } 318 319 /* this is compatible with cells for the device tree property */ 320 struct dynamic_dma_window_prop { 321 __be32 liobn; /* tce table number */ 322 __be64 dma_base; /* address hi,lo */ 323 __be32 tce_shift; /* ilog2(tce_page_size) */ 324 __be32 window_shift; /* ilog2(tce_window_size) */ 325 }; 326 327 struct direct_window { 328 struct device_node *device; 329 const struct dynamic_dma_window_prop *prop; 330 struct list_head list; 331 }; 332 333 /* Dynamic DMA Window support */ 334 struct ddw_query_response { 335 u32 windows_available; 336 u32 largest_available_block; 337 u32 page_size; 338 u32 migration_capable; 339 }; 340 341 struct ddw_create_response { 342 u32 liobn; 343 u32 addr_hi; 344 u32 addr_lo; 345 }; 346 347 static LIST_HEAD(direct_window_list); 348 /* prevents races between memory on/offline and window creation */ 349 static DEFINE_SPINLOCK(direct_window_list_lock); 350 /* protects initializing window twice for same device */ 351 static DEFINE_MUTEX(direct_window_init_mutex); 352 #define DIRECT64_PROPNAME "linux,direct64-ddr-window-info" 353 354 static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn, 355 unsigned long num_pfn, const void *arg) 356 { 357 const struct dynamic_dma_window_prop *maprange = arg; 358 int rc; 359 u64 tce_size, num_tce, dma_offset, next; 360 u32 tce_shift; 361 long limit; 362 363 tce_shift = be32_to_cpu(maprange->tce_shift); 364 tce_size = 1ULL << tce_shift; 365 next = start_pfn << PAGE_SHIFT; 366 num_tce = num_pfn << PAGE_SHIFT; 367 368 /* round back to the beginning of the tce page size */ 369 num_tce += next & (tce_size - 1); 370 next &= ~(tce_size - 1); 371 372 /* covert to number of tces */ 373 num_tce |= tce_size - 1; 374 num_tce >>= tce_shift; 375 376 do { 377 /* 378 * Set up the page with TCE data, looping through and setting 379 * the values. 380 */ 381 limit = min_t(long, num_tce, 512); 382 dma_offset = next + be64_to_cpu(maprange->dma_base); 383 384 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn), 385 dma_offset, 386 0, limit); 387 next += limit * tce_size; 388 num_tce -= limit; 389 } while (num_tce > 0 && !rc); 390 391 return rc; 392 } 393 394 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn, 395 unsigned long num_pfn, const void *arg) 396 { 397 const struct dynamic_dma_window_prop *maprange = arg; 398 u64 tce_size, num_tce, dma_offset, next, proto_tce, liobn; 399 __be64 *tcep; 400 u32 tce_shift; 401 u64 rc = 0; 402 long l, limit; 403 404 local_irq_disable(); /* to protect tcep and the page behind it */ 405 tcep = __this_cpu_read(tce_page); 406 407 if (!tcep) { 408 tcep = (__be64 *)__get_free_page(GFP_ATOMIC); 409 if (!tcep) { 410 local_irq_enable(); 411 return -ENOMEM; 412 } 413 __this_cpu_write(tce_page, tcep); 414 } 415 416 proto_tce = TCE_PCI_READ | TCE_PCI_WRITE; 417 418 liobn = (u64)be32_to_cpu(maprange->liobn); 419 tce_shift = be32_to_cpu(maprange->tce_shift); 420 tce_size = 1ULL << tce_shift; 421 next = start_pfn << PAGE_SHIFT; 422 num_tce = num_pfn << PAGE_SHIFT; 423 424 /* round back to the beginning of the tce page size */ 425 num_tce += next & (tce_size - 1); 426 next &= ~(tce_size - 1); 427 428 /* covert to number of tces */ 429 num_tce |= tce_size - 1; 430 num_tce >>= tce_shift; 431 432 /* We can map max one pageful of TCEs at a time */ 433 do { 434 /* 435 * Set up the page with TCE data, looping through and setting 436 * the values. 437 */ 438 limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE); 439 dma_offset = next + be64_to_cpu(maprange->dma_base); 440 441 for (l = 0; l < limit; l++) { 442 tcep[l] = cpu_to_be64(proto_tce | next); 443 next += tce_size; 444 } 445 446 rc = plpar_tce_put_indirect(liobn, 447 dma_offset, 448 (u64)__pa(tcep), 449 limit); 450 451 num_tce -= limit; 452 } while (num_tce > 0 && !rc); 453 454 /* error cleanup: caller will clear whole range */ 455 456 local_irq_enable(); 457 return rc; 458 } 459 460 static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn, 461 unsigned long num_pfn, void *arg) 462 { 463 return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg); 464 } 465 466 static void iommu_table_setparms(struct pci_controller *phb, 467 struct device_node *dn, 468 struct iommu_table *tbl) 469 { 470 struct device_node *node; 471 const unsigned long *basep; 472 const u32 *sizep; 473 474 node = phb->dn; 475 476 basep = of_get_property(node, "linux,tce-base", NULL); 477 sizep = of_get_property(node, "linux,tce-size", NULL); 478 if (basep == NULL || sizep == NULL) { 479 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %pOF has " 480 "missing tce entries !\n", dn); 481 return; 482 } 483 484 tbl->it_base = (unsigned long)__va(*basep); 485 486 if (!is_kdump_kernel()) 487 memset((void *)tbl->it_base, 0, *sizep); 488 489 tbl->it_busno = phb->bus->number; 490 tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K; 491 492 /* Units of tce entries */ 493 tbl->it_offset = phb->dma_window_base_cur >> tbl->it_page_shift; 494 495 /* Test if we are going over 2GB of DMA space */ 496 if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) { 497 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n"); 498 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n"); 499 } 500 501 phb->dma_window_base_cur += phb->dma_window_size; 502 503 /* Set the tce table size - measured in entries */ 504 tbl->it_size = phb->dma_window_size >> tbl->it_page_shift; 505 506 tbl->it_index = 0; 507 tbl->it_blocksize = 16; 508 tbl->it_type = TCE_PCI; 509 } 510 511 /* 512 * iommu_table_setparms_lpar 513 * 514 * Function: On pSeries LPAR systems, return TCE table info, given a pci bus. 515 */ 516 static void iommu_table_setparms_lpar(struct pci_controller *phb, 517 struct device_node *dn, 518 struct iommu_table *tbl, 519 struct iommu_table_group *table_group, 520 const __be32 *dma_window) 521 { 522 unsigned long offset, size; 523 524 of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size); 525 526 tbl->it_busno = phb->bus->number; 527 tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K; 528 tbl->it_base = 0; 529 tbl->it_blocksize = 16; 530 tbl->it_type = TCE_PCI; 531 tbl->it_offset = offset >> tbl->it_page_shift; 532 tbl->it_size = size >> tbl->it_page_shift; 533 534 table_group->tce32_start = offset; 535 table_group->tce32_size = size; 536 } 537 538 struct iommu_table_ops iommu_table_pseries_ops = { 539 .set = tce_build_pSeries, 540 .clear = tce_free_pSeries, 541 .get = tce_get_pseries 542 }; 543 544 static void pci_dma_bus_setup_pSeries(struct pci_bus *bus) 545 { 546 struct device_node *dn; 547 struct iommu_table *tbl; 548 struct device_node *isa_dn, *isa_dn_orig; 549 struct device_node *tmp; 550 struct pci_dn *pci; 551 int children; 552 553 dn = pci_bus_to_OF_node(bus); 554 555 pr_debug("pci_dma_bus_setup_pSeries: setting up bus %pOF\n", dn); 556 557 if (bus->self) { 558 /* This is not a root bus, any setup will be done for the 559 * device-side of the bridge in iommu_dev_setup_pSeries(). 560 */ 561 return; 562 } 563 pci = PCI_DN(dn); 564 565 /* Check if the ISA bus on the system is under 566 * this PHB. 567 */ 568 isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa"); 569 570 while (isa_dn && isa_dn != dn) 571 isa_dn = isa_dn->parent; 572 573 of_node_put(isa_dn_orig); 574 575 /* Count number of direct PCI children of the PHB. */ 576 for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling) 577 children++; 578 579 pr_debug("Children: %d\n", children); 580 581 /* Calculate amount of DMA window per slot. Each window must be 582 * a power of two (due to pci_alloc_consistent requirements). 583 * 584 * Keep 256MB aside for PHBs with ISA. 585 */ 586 587 if (!isa_dn) { 588 /* No ISA/IDE - just set window size and return */ 589 pci->phb->dma_window_size = 0x80000000ul; /* To be divided */ 590 591 while (pci->phb->dma_window_size * children > 0x80000000ul) 592 pci->phb->dma_window_size >>= 1; 593 pr_debug("No ISA/IDE, window size is 0x%llx\n", 594 pci->phb->dma_window_size); 595 pci->phb->dma_window_base_cur = 0; 596 597 return; 598 } 599 600 /* If we have ISA, then we probably have an IDE 601 * controller too. Allocate a 128MB table but 602 * skip the first 128MB to avoid stepping on ISA 603 * space. 604 */ 605 pci->phb->dma_window_size = 0x8000000ul; 606 pci->phb->dma_window_base_cur = 0x8000000ul; 607 608 pci->table_group = iommu_pseries_alloc_group(pci->phb->node); 609 tbl = pci->table_group->tables[0]; 610 611 iommu_table_setparms(pci->phb, dn, tbl); 612 tbl->it_ops = &iommu_table_pseries_ops; 613 iommu_init_table(tbl, pci->phb->node, 0, 0); 614 615 /* Divide the rest (1.75GB) among the children */ 616 pci->phb->dma_window_size = 0x80000000ul; 617 while (pci->phb->dma_window_size * children > 0x70000000ul) 618 pci->phb->dma_window_size >>= 1; 619 620 pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size); 621 } 622 623 #ifdef CONFIG_IOMMU_API 624 static int tce_exchange_pseries(struct iommu_table *tbl, long index, unsigned 625 long *tce, enum dma_data_direction *direction, 626 bool realmode) 627 { 628 long rc; 629 unsigned long ioba = (unsigned long) index << tbl->it_page_shift; 630 unsigned long flags, oldtce = 0; 631 u64 proto_tce = iommu_direction_to_tce_perm(*direction); 632 unsigned long newtce = *tce | proto_tce; 633 634 spin_lock_irqsave(&tbl->large_pool.lock, flags); 635 636 rc = plpar_tce_get((u64)tbl->it_index, ioba, &oldtce); 637 if (!rc) 638 rc = plpar_tce_put((u64)tbl->it_index, ioba, newtce); 639 640 if (!rc) { 641 *direction = iommu_tce_direction(oldtce); 642 *tce = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE); 643 } 644 645 spin_unlock_irqrestore(&tbl->large_pool.lock, flags); 646 647 return rc; 648 } 649 #endif 650 651 struct iommu_table_ops iommu_table_lpar_multi_ops = { 652 .set = tce_buildmulti_pSeriesLP, 653 #ifdef CONFIG_IOMMU_API 654 .xchg_no_kill = tce_exchange_pseries, 655 #endif 656 .clear = tce_freemulti_pSeriesLP, 657 .get = tce_get_pSeriesLP 658 }; 659 660 static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus) 661 { 662 struct iommu_table *tbl; 663 struct device_node *dn, *pdn; 664 struct pci_dn *ppci; 665 const __be32 *dma_window = NULL; 666 667 dn = pci_bus_to_OF_node(bus); 668 669 pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %pOF\n", 670 dn); 671 672 /* Find nearest ibm,dma-window, walking up the device tree */ 673 for (pdn = dn; pdn != NULL; pdn = pdn->parent) { 674 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 675 if (dma_window != NULL) 676 break; 677 } 678 679 if (dma_window == NULL) { 680 pr_debug(" no ibm,dma-window property !\n"); 681 return; 682 } 683 684 ppci = PCI_DN(pdn); 685 686 pr_debug(" parent is %pOF, iommu_table: 0x%p\n", 687 pdn, ppci->table_group); 688 689 if (!ppci->table_group) { 690 ppci->table_group = iommu_pseries_alloc_group(ppci->phb->node); 691 tbl = ppci->table_group->tables[0]; 692 iommu_table_setparms_lpar(ppci->phb, pdn, tbl, 693 ppci->table_group, dma_window); 694 tbl->it_ops = &iommu_table_lpar_multi_ops; 695 iommu_init_table(tbl, ppci->phb->node, 0, 0); 696 iommu_register_group(ppci->table_group, 697 pci_domain_nr(bus), 0); 698 pr_debug(" created table: %p\n", ppci->table_group); 699 } 700 } 701 702 703 static void pci_dma_dev_setup_pSeries(struct pci_dev *dev) 704 { 705 struct device_node *dn; 706 struct iommu_table *tbl; 707 708 pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev)); 709 710 dn = dev->dev.of_node; 711 712 /* If we're the direct child of a root bus, then we need to allocate 713 * an iommu table ourselves. The bus setup code should have setup 714 * the window sizes already. 715 */ 716 if (!dev->bus->self) { 717 struct pci_controller *phb = PCI_DN(dn)->phb; 718 719 pr_debug(" --> first child, no bridge. Allocating iommu table.\n"); 720 PCI_DN(dn)->table_group = iommu_pseries_alloc_group(phb->node); 721 tbl = PCI_DN(dn)->table_group->tables[0]; 722 iommu_table_setparms(phb, dn, tbl); 723 tbl->it_ops = &iommu_table_pseries_ops; 724 iommu_init_table(tbl, phb->node, 0, 0); 725 set_iommu_table_base(&dev->dev, tbl); 726 return; 727 } 728 729 /* If this device is further down the bus tree, search upwards until 730 * an already allocated iommu table is found and use that. 731 */ 732 733 while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL) 734 dn = dn->parent; 735 736 if (dn && PCI_DN(dn)) 737 set_iommu_table_base(&dev->dev, 738 PCI_DN(dn)->table_group->tables[0]); 739 else 740 printk(KERN_WARNING "iommu: Device %s has no iommu table\n", 741 pci_name(dev)); 742 } 743 744 static int __read_mostly disable_ddw; 745 746 static int __init disable_ddw_setup(char *str) 747 { 748 disable_ddw = 1; 749 printk(KERN_INFO "ppc iommu: disabling ddw.\n"); 750 751 return 0; 752 } 753 754 early_param("disable_ddw", disable_ddw_setup); 755 756 static void remove_ddw(struct device_node *np, bool remove_prop) 757 { 758 struct dynamic_dma_window_prop *dwp; 759 struct property *win64; 760 u32 ddw_avail[3]; 761 u64 liobn; 762 int ret = 0; 763 764 ret = of_property_read_u32_array(np, "ibm,ddw-applicable", 765 &ddw_avail[0], 3); 766 767 win64 = of_find_property(np, DIRECT64_PROPNAME, NULL); 768 if (!win64) 769 return; 770 771 if (ret || win64->length < sizeof(*dwp)) 772 goto delprop; 773 774 dwp = win64->value; 775 liobn = (u64)be32_to_cpu(dwp->liobn); 776 777 /* clear the whole window, note the arg is in kernel pages */ 778 ret = tce_clearrange_multi_pSeriesLP(0, 779 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp); 780 if (ret) 781 pr_warn("%pOF failed to clear tces in window.\n", 782 np); 783 else 784 pr_debug("%pOF successfully cleared tces in window.\n", 785 np); 786 787 ret = rtas_call(ddw_avail[2], 1, 1, NULL, liobn); 788 if (ret) 789 pr_warn("%pOF: failed to remove direct window: rtas returned " 790 "%d to ibm,remove-pe-dma-window(%x) %llx\n", 791 np, ret, ddw_avail[2], liobn); 792 else 793 pr_debug("%pOF: successfully removed direct window: rtas returned " 794 "%d to ibm,remove-pe-dma-window(%x) %llx\n", 795 np, ret, ddw_avail[2], liobn); 796 797 delprop: 798 if (remove_prop) 799 ret = of_remove_property(np, win64); 800 if (ret) 801 pr_warn("%pOF: failed to remove direct window property: %d\n", 802 np, ret); 803 } 804 805 static u64 find_existing_ddw(struct device_node *pdn) 806 { 807 struct direct_window *window; 808 const struct dynamic_dma_window_prop *direct64; 809 u64 dma_addr = 0; 810 811 spin_lock(&direct_window_list_lock); 812 /* check if we already created a window and dupe that config if so */ 813 list_for_each_entry(window, &direct_window_list, list) { 814 if (window->device == pdn) { 815 direct64 = window->prop; 816 dma_addr = be64_to_cpu(direct64->dma_base); 817 break; 818 } 819 } 820 spin_unlock(&direct_window_list_lock); 821 822 return dma_addr; 823 } 824 825 static int find_existing_ddw_windows(void) 826 { 827 int len; 828 struct device_node *pdn; 829 struct direct_window *window; 830 const struct dynamic_dma_window_prop *direct64; 831 832 if (!firmware_has_feature(FW_FEATURE_LPAR)) 833 return 0; 834 835 for_each_node_with_property(pdn, DIRECT64_PROPNAME) { 836 direct64 = of_get_property(pdn, DIRECT64_PROPNAME, &len); 837 if (!direct64) 838 continue; 839 840 window = kzalloc(sizeof(*window), GFP_KERNEL); 841 if (!window || len < sizeof(struct dynamic_dma_window_prop)) { 842 kfree(window); 843 remove_ddw(pdn, true); 844 continue; 845 } 846 847 window->device = pdn; 848 window->prop = direct64; 849 spin_lock(&direct_window_list_lock); 850 list_add(&window->list, &direct_window_list); 851 spin_unlock(&direct_window_list_lock); 852 } 853 854 return 0; 855 } 856 machine_arch_initcall(pseries, find_existing_ddw_windows); 857 858 static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail, 859 struct ddw_query_response *query) 860 { 861 struct device_node *dn; 862 struct pci_dn *pdn; 863 u32 cfg_addr; 864 u64 buid; 865 int ret; 866 867 /* 868 * Get the config address and phb buid of the PE window. 869 * Rely on eeh to retrieve this for us. 870 * Retrieve them from the pci device, not the node with the 871 * dma-window property 872 */ 873 dn = pci_device_to_OF_node(dev); 874 pdn = PCI_DN(dn); 875 buid = pdn->phb->buid; 876 cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8)); 877 878 ret = rtas_call(ddw_avail[0], 3, 5, (u32 *)query, 879 cfg_addr, BUID_HI(buid), BUID_LO(buid)); 880 dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x" 881 " returned %d\n", ddw_avail[0], cfg_addr, BUID_HI(buid), 882 BUID_LO(buid), ret); 883 return ret; 884 } 885 886 static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail, 887 struct ddw_create_response *create, int page_shift, 888 int window_shift) 889 { 890 struct device_node *dn; 891 struct pci_dn *pdn; 892 u32 cfg_addr; 893 u64 buid; 894 int ret; 895 896 /* 897 * Get the config address and phb buid of the PE window. 898 * Rely on eeh to retrieve this for us. 899 * Retrieve them from the pci device, not the node with the 900 * dma-window property 901 */ 902 dn = pci_device_to_OF_node(dev); 903 pdn = PCI_DN(dn); 904 buid = pdn->phb->buid; 905 cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8)); 906 907 do { 908 /* extra outputs are LIOBN and dma-addr (hi, lo) */ 909 ret = rtas_call(ddw_avail[1], 5, 4, (u32 *)create, 910 cfg_addr, BUID_HI(buid), BUID_LO(buid), 911 page_shift, window_shift); 912 } while (rtas_busy_delay(ret)); 913 dev_info(&dev->dev, 914 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d " 915 "(liobn = 0x%x starting addr = %x %x)\n", ddw_avail[1], 916 cfg_addr, BUID_HI(buid), BUID_LO(buid), page_shift, 917 window_shift, ret, create->liobn, create->addr_hi, create->addr_lo); 918 919 return ret; 920 } 921 922 struct failed_ddw_pdn { 923 struct device_node *pdn; 924 struct list_head list; 925 }; 926 927 static LIST_HEAD(failed_ddw_pdn_list); 928 929 static phys_addr_t ddw_memory_hotplug_max(void) 930 { 931 phys_addr_t max_addr = memory_hotplug_max(); 932 struct device_node *memory; 933 934 for_each_node_by_type(memory, "memory") { 935 unsigned long start, size; 936 int n_mem_addr_cells, n_mem_size_cells, len; 937 const __be32 *memcell_buf; 938 939 memcell_buf = of_get_property(memory, "reg", &len); 940 if (!memcell_buf || len <= 0) 941 continue; 942 943 n_mem_addr_cells = of_n_addr_cells(memory); 944 n_mem_size_cells = of_n_size_cells(memory); 945 946 start = of_read_number(memcell_buf, n_mem_addr_cells); 947 memcell_buf += n_mem_addr_cells; 948 size = of_read_number(memcell_buf, n_mem_size_cells); 949 memcell_buf += n_mem_size_cells; 950 951 max_addr = max_t(phys_addr_t, max_addr, start + size); 952 } 953 954 return max_addr; 955 } 956 957 /* 958 * If the PE supports dynamic dma windows, and there is space for a table 959 * that can map all pages in a linear offset, then setup such a table, 960 * and record the dma-offset in the struct device. 961 * 962 * dev: the pci device we are checking 963 * pdn: the parent pe node with the ibm,dma_window property 964 * Future: also check if we can remap the base window for our base page size 965 * 966 * returns the dma offset for use by the direct mapped DMA code. 967 */ 968 static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn) 969 { 970 int len, ret; 971 struct ddw_query_response query; 972 struct ddw_create_response create; 973 int page_shift; 974 u64 dma_addr, max_addr; 975 struct device_node *dn; 976 u32 ddw_avail[3]; 977 struct direct_window *window; 978 struct property *win64; 979 struct dynamic_dma_window_prop *ddwprop; 980 struct failed_ddw_pdn *fpdn; 981 982 mutex_lock(&direct_window_init_mutex); 983 984 dma_addr = find_existing_ddw(pdn); 985 if (dma_addr != 0) 986 goto out_unlock; 987 988 /* 989 * If we already went through this for a previous function of 990 * the same device and failed, we don't want to muck with the 991 * DMA window again, as it will race with in-flight operations 992 * and can lead to EEHs. The above mutex protects access to the 993 * list. 994 */ 995 list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) { 996 if (fpdn->pdn == pdn) 997 goto out_unlock; 998 } 999 1000 /* 1001 * the ibm,ddw-applicable property holds the tokens for: 1002 * ibm,query-pe-dma-window 1003 * ibm,create-pe-dma-window 1004 * ibm,remove-pe-dma-window 1005 * for the given node in that order. 1006 * the property is actually in the parent, not the PE 1007 */ 1008 ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable", 1009 &ddw_avail[0], 3); 1010 if (ret) 1011 goto out_failed; 1012 1013 /* 1014 * Query if there is a second window of size to map the 1015 * whole partition. Query returns number of windows, largest 1016 * block assigned to PE (partition endpoint), and two bitmasks 1017 * of page sizes: supported and supported for migrate-dma. 1018 */ 1019 dn = pci_device_to_OF_node(dev); 1020 ret = query_ddw(dev, ddw_avail, &query); 1021 if (ret != 0) 1022 goto out_failed; 1023 1024 if (query.windows_available == 0) { 1025 /* 1026 * no additional windows are available for this device. 1027 * We might be able to reallocate the existing window, 1028 * trading in for a larger page size. 1029 */ 1030 dev_dbg(&dev->dev, "no free dynamic windows"); 1031 goto out_failed; 1032 } 1033 if (query.page_size & 4) { 1034 page_shift = 24; /* 16MB */ 1035 } else if (query.page_size & 2) { 1036 page_shift = 16; /* 64kB */ 1037 } else if (query.page_size & 1) { 1038 page_shift = 12; /* 4kB */ 1039 } else { 1040 dev_dbg(&dev->dev, "no supported direct page size in mask %x", 1041 query.page_size); 1042 goto out_failed; 1043 } 1044 /* verify the window * number of ptes will map the partition */ 1045 /* check largest block * page size > max memory hotplug addr */ 1046 max_addr = ddw_memory_hotplug_max(); 1047 if (query.largest_available_block < (max_addr >> page_shift)) { 1048 dev_dbg(&dev->dev, "can't map partition max 0x%llx with %u " 1049 "%llu-sized pages\n", max_addr, query.largest_available_block, 1050 1ULL << page_shift); 1051 goto out_failed; 1052 } 1053 len = order_base_2(max_addr); 1054 win64 = kzalloc(sizeof(struct property), GFP_KERNEL); 1055 if (!win64) { 1056 dev_info(&dev->dev, 1057 "couldn't allocate property for 64bit dma window\n"); 1058 goto out_failed; 1059 } 1060 win64->name = kstrdup(DIRECT64_PROPNAME, GFP_KERNEL); 1061 win64->value = ddwprop = kmalloc(sizeof(*ddwprop), GFP_KERNEL); 1062 win64->length = sizeof(*ddwprop); 1063 if (!win64->name || !win64->value) { 1064 dev_info(&dev->dev, 1065 "couldn't allocate property name and value\n"); 1066 goto out_free_prop; 1067 } 1068 1069 ret = create_ddw(dev, ddw_avail, &create, page_shift, len); 1070 if (ret != 0) 1071 goto out_free_prop; 1072 1073 ddwprop->liobn = cpu_to_be32(create.liobn); 1074 ddwprop->dma_base = cpu_to_be64(((u64)create.addr_hi << 32) | 1075 create.addr_lo); 1076 ddwprop->tce_shift = cpu_to_be32(page_shift); 1077 ddwprop->window_shift = cpu_to_be32(len); 1078 1079 dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %pOF\n", 1080 create.liobn, dn); 1081 1082 window = kzalloc(sizeof(*window), GFP_KERNEL); 1083 if (!window) 1084 goto out_clear_window; 1085 1086 ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT, 1087 win64->value, tce_setrange_multi_pSeriesLP_walk); 1088 if (ret) { 1089 dev_info(&dev->dev, "failed to map direct window for %pOF: %d\n", 1090 dn, ret); 1091 goto out_free_window; 1092 } 1093 1094 ret = of_add_property(pdn, win64); 1095 if (ret) { 1096 dev_err(&dev->dev, "unable to add dma window property for %pOF: %d", 1097 pdn, ret); 1098 goto out_free_window; 1099 } 1100 1101 window->device = pdn; 1102 window->prop = ddwprop; 1103 spin_lock(&direct_window_list_lock); 1104 list_add(&window->list, &direct_window_list); 1105 spin_unlock(&direct_window_list_lock); 1106 1107 dma_addr = be64_to_cpu(ddwprop->dma_base); 1108 goto out_unlock; 1109 1110 out_free_window: 1111 kfree(window); 1112 1113 out_clear_window: 1114 remove_ddw(pdn, true); 1115 1116 out_free_prop: 1117 kfree(win64->name); 1118 kfree(win64->value); 1119 kfree(win64); 1120 1121 out_failed: 1122 1123 fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL); 1124 if (!fpdn) 1125 goto out_unlock; 1126 fpdn->pdn = pdn; 1127 list_add(&fpdn->list, &failed_ddw_pdn_list); 1128 1129 out_unlock: 1130 mutex_unlock(&direct_window_init_mutex); 1131 return dma_addr; 1132 } 1133 1134 static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev) 1135 { 1136 struct device_node *pdn, *dn; 1137 struct iommu_table *tbl; 1138 const __be32 *dma_window = NULL; 1139 struct pci_dn *pci; 1140 1141 pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev)); 1142 1143 /* dev setup for LPAR is a little tricky, since the device tree might 1144 * contain the dma-window properties per-device and not necessarily 1145 * for the bus. So we need to search upwards in the tree until we 1146 * either hit a dma-window property, OR find a parent with a table 1147 * already allocated. 1148 */ 1149 dn = pci_device_to_OF_node(dev); 1150 pr_debug(" node is %pOF\n", dn); 1151 1152 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->table_group; 1153 pdn = pdn->parent) { 1154 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 1155 if (dma_window) 1156 break; 1157 } 1158 1159 if (!pdn || !PCI_DN(pdn)) { 1160 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: " 1161 "no DMA window found for pci dev=%s dn=%pOF\n", 1162 pci_name(dev), dn); 1163 return; 1164 } 1165 pr_debug(" parent is %pOF\n", pdn); 1166 1167 pci = PCI_DN(pdn); 1168 if (!pci->table_group) { 1169 pci->table_group = iommu_pseries_alloc_group(pci->phb->node); 1170 tbl = pci->table_group->tables[0]; 1171 iommu_table_setparms_lpar(pci->phb, pdn, tbl, 1172 pci->table_group, dma_window); 1173 tbl->it_ops = &iommu_table_lpar_multi_ops; 1174 iommu_init_table(tbl, pci->phb->node, 0, 0); 1175 iommu_register_group(pci->table_group, 1176 pci_domain_nr(pci->phb->bus), 0); 1177 pr_debug(" created table: %p\n", pci->table_group); 1178 } else { 1179 pr_debug(" found DMA window, table: %p\n", pci->table_group); 1180 } 1181 1182 set_iommu_table_base(&dev->dev, pci->table_group->tables[0]); 1183 iommu_add_device(pci->table_group, &dev->dev); 1184 } 1185 1186 static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask) 1187 { 1188 struct device_node *dn = pci_device_to_OF_node(pdev), *pdn; 1189 const __be32 *dma_window = NULL; 1190 1191 /* only attempt to use a new window if 64-bit DMA is requested */ 1192 if (dma_mask < DMA_BIT_MASK(64)) 1193 return false; 1194 1195 dev_dbg(&pdev->dev, "node is %pOF\n", dn); 1196 1197 /* 1198 * the device tree might contain the dma-window properties 1199 * per-device and not necessarily for the bus. So we need to 1200 * search upwards in the tree until we either hit a dma-window 1201 * property, OR find a parent with a table already allocated. 1202 */ 1203 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->table_group; 1204 pdn = pdn->parent) { 1205 dma_window = of_get_property(pdn, "ibm,dma-window", NULL); 1206 if (dma_window) 1207 break; 1208 } 1209 1210 if (pdn && PCI_DN(pdn)) { 1211 pdev->dev.archdata.dma_offset = enable_ddw(pdev, pdn); 1212 if (pdev->dev.archdata.dma_offset) 1213 return true; 1214 } 1215 1216 return false; 1217 } 1218 1219 static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, 1220 void *data) 1221 { 1222 struct direct_window *window; 1223 struct memory_notify *arg = data; 1224 int ret = 0; 1225 1226 switch (action) { 1227 case MEM_GOING_ONLINE: 1228 spin_lock(&direct_window_list_lock); 1229 list_for_each_entry(window, &direct_window_list, list) { 1230 ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn, 1231 arg->nr_pages, window->prop); 1232 /* XXX log error */ 1233 } 1234 spin_unlock(&direct_window_list_lock); 1235 break; 1236 case MEM_CANCEL_ONLINE: 1237 case MEM_OFFLINE: 1238 spin_lock(&direct_window_list_lock); 1239 list_for_each_entry(window, &direct_window_list, list) { 1240 ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn, 1241 arg->nr_pages, window->prop); 1242 /* XXX log error */ 1243 } 1244 spin_unlock(&direct_window_list_lock); 1245 break; 1246 default: 1247 break; 1248 } 1249 if (ret && action != MEM_CANCEL_ONLINE) 1250 return NOTIFY_BAD; 1251 1252 return NOTIFY_OK; 1253 } 1254 1255 static struct notifier_block iommu_mem_nb = { 1256 .notifier_call = iommu_mem_notifier, 1257 }; 1258 1259 static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data) 1260 { 1261 int err = NOTIFY_OK; 1262 struct of_reconfig_data *rd = data; 1263 struct device_node *np = rd->dn; 1264 struct pci_dn *pci = PCI_DN(np); 1265 struct direct_window *window; 1266 1267 switch (action) { 1268 case OF_RECONFIG_DETACH_NODE: 1269 /* 1270 * Removing the property will invoke the reconfig 1271 * notifier again, which causes dead-lock on the 1272 * read-write semaphore of the notifier chain. So 1273 * we have to remove the property when releasing 1274 * the device node. 1275 */ 1276 remove_ddw(np, false); 1277 if (pci && pci->table_group) 1278 iommu_pseries_free_group(pci->table_group, 1279 np->full_name); 1280 1281 spin_lock(&direct_window_list_lock); 1282 list_for_each_entry(window, &direct_window_list, list) { 1283 if (window->device == np) { 1284 list_del(&window->list); 1285 kfree(window); 1286 break; 1287 } 1288 } 1289 spin_unlock(&direct_window_list_lock); 1290 break; 1291 default: 1292 err = NOTIFY_DONE; 1293 break; 1294 } 1295 return err; 1296 } 1297 1298 static struct notifier_block iommu_reconfig_nb = { 1299 .notifier_call = iommu_reconfig_notifier, 1300 }; 1301 1302 /* These are called very early. */ 1303 void iommu_init_early_pSeries(void) 1304 { 1305 if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL)) 1306 return; 1307 1308 if (firmware_has_feature(FW_FEATURE_LPAR)) { 1309 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP; 1310 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP; 1311 if (!disable_ddw) 1312 pseries_pci_controller_ops.iommu_bypass_supported = 1313 iommu_bypass_supported_pSeriesLP; 1314 } else { 1315 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries; 1316 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries; 1317 } 1318 1319 1320 of_reconfig_notifier_register(&iommu_reconfig_nb); 1321 register_memory_notifier(&iommu_mem_nb); 1322 1323 /* 1324 * Secure guest memory is inacessible to devices so regular DMA isn't 1325 * possible. 1326 * 1327 * In that case keep devices' dma_map_ops as NULL so that the generic 1328 * DMA code path will use SWIOTLB to bounce buffers for DMA. 1329 */ 1330 if (!is_secure_guest()) 1331 set_pci_dma_ops(&dma_iommu_ops); 1332 } 1333 1334 static int __init disable_multitce(char *str) 1335 { 1336 if (strcmp(str, "off") == 0 && 1337 firmware_has_feature(FW_FEATURE_LPAR) && 1338 firmware_has_feature(FW_FEATURE_MULTITCE)) { 1339 printk(KERN_INFO "Disabling MULTITCE firmware feature\n"); 1340 powerpc_firmware_features &= ~FW_FEATURE_MULTITCE; 1341 } 1342 return 1; 1343 } 1344 1345 __setup("multitce=", disable_multitce); 1346 1347 static int tce_iommu_bus_notifier(struct notifier_block *nb, 1348 unsigned long action, void *data) 1349 { 1350 struct device *dev = data; 1351 1352 switch (action) { 1353 case BUS_NOTIFY_DEL_DEVICE: 1354 iommu_del_device(dev); 1355 return 0; 1356 default: 1357 return 0; 1358 } 1359 } 1360 1361 static struct notifier_block tce_iommu_bus_nb = { 1362 .notifier_call = tce_iommu_bus_notifier, 1363 }; 1364 1365 static int __init tce_iommu_bus_notifier_init(void) 1366 { 1367 bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb); 1368 return 0; 1369 } 1370 machine_subsys_initcall_sync(pseries, tce_iommu_bus_notifier_init); 1371