1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * You should have received a copy of the GNU General Public License 12 * along with this program; if not, write to the Free Software 13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 14 * 15 * Copyright (C) 2013 Freescale Semiconductor, Inc. 16 * Author: Varun Sethi <varun.sethi@freescale.com> 17 * 18 */ 19 20 #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__ 21 22 #include <linux/init.h> 23 #include <linux/iommu.h> 24 #include <linux/notifier.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include <linux/types.h> 28 #include <linux/mm.h> 29 #include <linux/interrupt.h> 30 #include <linux/device.h> 31 #include <linux/of_platform.h> 32 #include <linux/bootmem.h> 33 #include <linux/err.h> 34 #include <asm/io.h> 35 #include <asm/bitops.h> 36 37 #include <asm/pci-bridge.h> 38 #include <sysdev/fsl_pci.h> 39 40 #include "fsl_pamu_domain.h" 41 42 /* 43 * Global spinlock that needs to be held while 44 * configuring PAMU. 45 */ 46 static DEFINE_SPINLOCK(iommu_lock); 47 48 static struct kmem_cache *fsl_pamu_domain_cache; 49 static struct kmem_cache *iommu_devinfo_cache; 50 static DEFINE_SPINLOCK(device_domain_lock); 51 52 static int __init iommu_init_mempool(void) 53 { 54 55 fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain", 56 sizeof(struct fsl_dma_domain), 57 0, 58 SLAB_HWCACHE_ALIGN, 59 60 NULL); 61 if (!fsl_pamu_domain_cache) { 62 pr_debug("Couldn't create fsl iommu_domain cache\n"); 63 return -ENOMEM; 64 } 65 66 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo", 67 sizeof(struct device_domain_info), 68 0, 69 SLAB_HWCACHE_ALIGN, 70 NULL); 71 if (!iommu_devinfo_cache) { 72 pr_debug("Couldn't create devinfo cache\n"); 73 kmem_cache_destroy(fsl_pamu_domain_cache); 74 return -ENOMEM; 75 } 76 77 return 0; 78 } 79 80 static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova) 81 { 82 u32 win_cnt = dma_domain->win_cnt; 83 struct dma_window *win_ptr = 84 &dma_domain->win_arr[0]; 85 struct iommu_domain_geometry *geom; 86 87 geom = &dma_domain->iommu_domain->geometry; 88 89 if (!win_cnt || !dma_domain->geom_size) { 90 pr_debug("Number of windows/geometry not configured for the domain\n"); 91 return 0; 92 } 93 94 if (win_cnt > 1) { 95 u64 subwin_size; 96 dma_addr_t subwin_iova; 97 u32 wnd; 98 99 subwin_size = dma_domain->geom_size >> ilog2(win_cnt); 100 subwin_iova = iova & ~(subwin_size - 1); 101 wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size); 102 win_ptr = &dma_domain->win_arr[wnd]; 103 } 104 105 if (win_ptr->valid) 106 return (win_ptr->paddr + (iova & (win_ptr->size - 1))); 107 108 return 0; 109 } 110 111 static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain) 112 { 113 struct dma_window *sub_win_ptr = 114 &dma_domain->win_arr[0]; 115 int i, ret; 116 unsigned long rpn, flags; 117 118 for (i = 0; i < dma_domain->win_cnt; i++) { 119 if (sub_win_ptr[i].valid) { 120 rpn = sub_win_ptr[i].paddr >> 121 PAMU_PAGE_SHIFT; 122 spin_lock_irqsave(&iommu_lock, flags); 123 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i, 124 sub_win_ptr[i].size, 125 ~(u32)0, 126 rpn, 127 dma_domain->snoop_id, 128 dma_domain->stash_id, 129 (i > 0) ? 1 : 0, 130 sub_win_ptr[i].prot); 131 spin_unlock_irqrestore(&iommu_lock, flags); 132 if (ret) { 133 pr_debug("PAMU SPAACE configuration failed for liodn %d\n", 134 liodn); 135 return ret; 136 } 137 } 138 } 139 140 return ret; 141 } 142 143 static int map_win(int liodn, struct fsl_dma_domain *dma_domain) 144 { 145 int ret; 146 struct dma_window *wnd = &dma_domain->win_arr[0]; 147 phys_addr_t wnd_addr = dma_domain->iommu_domain->geometry.aperture_start; 148 unsigned long flags; 149 150 spin_lock_irqsave(&iommu_lock, flags); 151 ret = pamu_config_ppaace(liodn, wnd_addr, 152 wnd->size, 153 ~(u32)0, 154 wnd->paddr >> PAMU_PAGE_SHIFT, 155 dma_domain->snoop_id, dma_domain->stash_id, 156 0, wnd->prot); 157 spin_unlock_irqrestore(&iommu_lock, flags); 158 if (ret) 159 pr_debug("PAMU PAACE configuration failed for liodn %d\n", 160 liodn); 161 162 return ret; 163 } 164 165 /* Map the DMA window corresponding to the LIODN */ 166 static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) 167 { 168 if (dma_domain->win_cnt > 1) 169 return map_subwins(liodn, dma_domain); 170 else 171 return map_win(liodn, dma_domain); 172 173 } 174 175 /* Update window/subwindow mapping for the LIODN */ 176 static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr) 177 { 178 int ret; 179 struct dma_window *wnd = &dma_domain->win_arr[wnd_nr]; 180 unsigned long flags; 181 182 spin_lock_irqsave(&iommu_lock, flags); 183 if (dma_domain->win_cnt > 1) { 184 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr, 185 wnd->size, 186 ~(u32)0, 187 wnd->paddr >> PAMU_PAGE_SHIFT, 188 dma_domain->snoop_id, 189 dma_domain->stash_id, 190 (wnd_nr > 0) ? 1 : 0, 191 wnd->prot); 192 if (ret) 193 pr_debug("Subwindow reconfiguration failed for liodn %d\n", liodn); 194 } else { 195 phys_addr_t wnd_addr; 196 197 wnd_addr = dma_domain->iommu_domain->geometry.aperture_start; 198 199 ret = pamu_config_ppaace(liodn, wnd_addr, 200 wnd->size, 201 ~(u32)0, 202 wnd->paddr >> PAMU_PAGE_SHIFT, 203 dma_domain->snoop_id, dma_domain->stash_id, 204 0, wnd->prot); 205 if (ret) 206 pr_debug("Window reconfiguration failed for liodn %d\n", liodn); 207 } 208 209 spin_unlock_irqrestore(&iommu_lock, flags); 210 211 return ret; 212 } 213 214 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, 215 u32 val) 216 { 217 int ret = 0, i; 218 unsigned long flags; 219 220 spin_lock_irqsave(&iommu_lock, flags); 221 if (!dma_domain->win_arr) { 222 pr_debug("Windows not configured, stash destination update failed for liodn %d\n", liodn); 223 spin_unlock_irqrestore(&iommu_lock, flags); 224 return -EINVAL; 225 } 226 227 for (i = 0; i < dma_domain->win_cnt; i++) { 228 ret = pamu_update_paace_stash(liodn, i, val); 229 if (ret) { 230 pr_debug("Failed to update SPAACE %d field for liodn %d\n ", i, liodn); 231 spin_unlock_irqrestore(&iommu_lock, flags); 232 return ret; 233 } 234 } 235 236 spin_unlock_irqrestore(&iommu_lock, flags); 237 238 return ret; 239 } 240 241 /* Set the geometry parameters for a LIODN */ 242 static int pamu_set_liodn(int liodn, struct device *dev, 243 struct fsl_dma_domain *dma_domain, 244 struct iommu_domain_geometry *geom_attr, 245 u32 win_cnt) 246 { 247 phys_addr_t window_addr, window_size; 248 phys_addr_t subwin_size; 249 int ret = 0, i; 250 u32 omi_index = ~(u32)0; 251 unsigned long flags; 252 253 /* 254 * Configure the omi_index at the geometry setup time. 255 * This is a static value which depends on the type of 256 * device and would not change thereafter. 257 */ 258 get_ome_index(&omi_index, dev); 259 260 window_addr = geom_attr->aperture_start; 261 window_size = dma_domain->geom_size; 262 263 spin_lock_irqsave(&iommu_lock, flags); 264 ret = pamu_disable_liodn(liodn); 265 if (!ret) 266 ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index, 267 0, dma_domain->snoop_id, 268 dma_domain->stash_id, win_cnt, 0); 269 spin_unlock_irqrestore(&iommu_lock, flags); 270 if (ret) { 271 pr_debug("PAMU PAACE configuration failed for liodn %d, win_cnt =%d\n", liodn, win_cnt); 272 return ret; 273 } 274 275 if (win_cnt > 1) { 276 subwin_size = window_size >> ilog2(win_cnt); 277 for (i = 0; i < win_cnt; i++) { 278 spin_lock_irqsave(&iommu_lock, flags); 279 ret = pamu_disable_spaace(liodn, i); 280 if (!ret) 281 ret = pamu_config_spaace(liodn, win_cnt, i, 282 subwin_size, omi_index, 283 0, dma_domain->snoop_id, 284 dma_domain->stash_id, 285 0, 0); 286 spin_unlock_irqrestore(&iommu_lock, flags); 287 if (ret) { 288 pr_debug("PAMU SPAACE configuration failed for liodn %d\n", liodn); 289 return ret; 290 } 291 } 292 } 293 294 return ret; 295 } 296 297 static int check_size(u64 size, dma_addr_t iova) 298 { 299 /* 300 * Size must be a power of two and at least be equal 301 * to PAMU page size. 302 */ 303 if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) { 304 pr_debug("%s: size too small or not a power of two\n", __func__); 305 return -EINVAL; 306 } 307 308 /* iova must be page size aligned*/ 309 if (iova & (size - 1)) { 310 pr_debug("%s: address is not aligned with window size\n", __func__); 311 return -EINVAL; 312 } 313 314 return 0; 315 } 316 317 static struct fsl_dma_domain *iommu_alloc_dma_domain(void) 318 { 319 struct fsl_dma_domain *domain; 320 321 domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL); 322 if (!domain) 323 return NULL; 324 325 domain->stash_id = ~(u32)0; 326 domain->snoop_id = ~(u32)0; 327 domain->win_cnt = pamu_get_max_subwin_cnt(); 328 domain->geom_size = 0; 329 330 INIT_LIST_HEAD(&domain->devices); 331 332 spin_lock_init(&domain->domain_lock); 333 334 return domain; 335 } 336 337 static void remove_device_ref(struct device_domain_info *info, u32 win_cnt) 338 { 339 unsigned long flags; 340 341 list_del(&info->link); 342 spin_lock_irqsave(&iommu_lock, flags); 343 if (win_cnt > 1) 344 pamu_free_subwins(info->liodn); 345 pamu_disable_liodn(info->liodn); 346 spin_unlock_irqrestore(&iommu_lock, flags); 347 spin_lock_irqsave(&device_domain_lock, flags); 348 info->dev->archdata.iommu_domain = NULL; 349 kmem_cache_free(iommu_devinfo_cache, info); 350 spin_unlock_irqrestore(&device_domain_lock, flags); 351 } 352 353 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain) 354 { 355 struct device_domain_info *info, *tmp; 356 unsigned long flags; 357 358 spin_lock_irqsave(&dma_domain->domain_lock, flags); 359 /* Remove the device from the domain device list */ 360 list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) { 361 if (!dev || (info->dev == dev)) 362 remove_device_ref(info, dma_domain->win_cnt); 363 } 364 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 365 } 366 367 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev) 368 { 369 struct device_domain_info *info, *old_domain_info; 370 unsigned long flags; 371 372 spin_lock_irqsave(&device_domain_lock, flags); 373 /* 374 * Check here if the device is already attached to domain or not. 375 * If the device is already attached to a domain detach it. 376 */ 377 old_domain_info = dev->archdata.iommu_domain; 378 if (old_domain_info && old_domain_info->domain != dma_domain) { 379 spin_unlock_irqrestore(&device_domain_lock, flags); 380 detach_device(dev, old_domain_info->domain); 381 spin_lock_irqsave(&device_domain_lock, flags); 382 } 383 384 info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC); 385 386 info->dev = dev; 387 info->liodn = liodn; 388 info->domain = dma_domain; 389 390 list_add(&info->link, &dma_domain->devices); 391 /* 392 * In case of devices with multiple LIODNs just store 393 * the info for the first LIODN as all 394 * LIODNs share the same domain 395 */ 396 if (!dev->archdata.iommu_domain) 397 dev->archdata.iommu_domain = info; 398 spin_unlock_irqrestore(&device_domain_lock, flags); 399 400 } 401 402 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, 403 dma_addr_t iova) 404 { 405 struct fsl_dma_domain *dma_domain = domain->priv; 406 407 if ((iova < domain->geometry.aperture_start) || 408 iova > (domain->geometry.aperture_end)) 409 return 0; 410 411 return get_phys_addr(dma_domain, iova); 412 } 413 414 static bool fsl_pamu_capable(enum iommu_cap cap) 415 { 416 return cap == IOMMU_CAP_CACHE_COHERENCY; 417 } 418 419 static void fsl_pamu_domain_destroy(struct iommu_domain *domain) 420 { 421 struct fsl_dma_domain *dma_domain = domain->priv; 422 423 domain->priv = NULL; 424 425 /* remove all the devices from the device list */ 426 detach_device(NULL, dma_domain); 427 428 dma_domain->enabled = 0; 429 dma_domain->mapped = 0; 430 431 kmem_cache_free(fsl_pamu_domain_cache, dma_domain); 432 } 433 434 static int fsl_pamu_domain_init(struct iommu_domain *domain) 435 { 436 struct fsl_dma_domain *dma_domain; 437 438 dma_domain = iommu_alloc_dma_domain(); 439 if (!dma_domain) { 440 pr_debug("dma_domain allocation failed\n"); 441 return -ENOMEM; 442 } 443 domain->priv = dma_domain; 444 dma_domain->iommu_domain = domain; 445 /* defaul geometry 64 GB i.e. maximum system address */ 446 domain->geometry.aperture_start = 0; 447 domain->geometry.aperture_end = (1ULL << 36) - 1; 448 domain->geometry.force_aperture = true; 449 450 return 0; 451 } 452 453 /* Configure geometry settings for all LIODNs associated with domain */ 454 static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain, 455 struct iommu_domain_geometry *geom_attr, 456 u32 win_cnt) 457 { 458 struct device_domain_info *info; 459 int ret = 0; 460 461 list_for_each_entry(info, &dma_domain->devices, link) { 462 ret = pamu_set_liodn(info->liodn, info->dev, dma_domain, 463 geom_attr, win_cnt); 464 if (ret) 465 break; 466 } 467 468 return ret; 469 } 470 471 /* Update stash destination for all LIODNs associated with the domain */ 472 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val) 473 { 474 struct device_domain_info *info; 475 int ret = 0; 476 477 list_for_each_entry(info, &dma_domain->devices, link) { 478 ret = update_liodn_stash(info->liodn, dma_domain, val); 479 if (ret) 480 break; 481 } 482 483 return ret; 484 } 485 486 /* Update domain mappings for all LIODNs associated with the domain */ 487 static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr) 488 { 489 struct device_domain_info *info; 490 int ret = 0; 491 492 list_for_each_entry(info, &dma_domain->devices, link) { 493 ret = update_liodn(info->liodn, dma_domain, wnd_nr); 494 if (ret) 495 break; 496 } 497 return ret; 498 } 499 500 static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr) 501 { 502 struct device_domain_info *info; 503 int ret = 0; 504 505 list_for_each_entry(info, &dma_domain->devices, link) { 506 if (dma_domain->win_cnt == 1 && dma_domain->enabled) { 507 ret = pamu_disable_liodn(info->liodn); 508 if (!ret) 509 dma_domain->enabled = 0; 510 } else { 511 ret = pamu_disable_spaace(info->liodn, wnd_nr); 512 } 513 } 514 515 return ret; 516 } 517 518 static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr) 519 { 520 struct fsl_dma_domain *dma_domain = domain->priv; 521 unsigned long flags; 522 int ret; 523 524 spin_lock_irqsave(&dma_domain->domain_lock, flags); 525 if (!dma_domain->win_arr) { 526 pr_debug("Number of windows not configured\n"); 527 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 528 return; 529 } 530 531 if (wnd_nr >= dma_domain->win_cnt) { 532 pr_debug("Invalid window index\n"); 533 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 534 return; 535 } 536 537 if (dma_domain->win_arr[wnd_nr].valid) { 538 ret = disable_domain_win(dma_domain, wnd_nr); 539 if (!ret) { 540 dma_domain->win_arr[wnd_nr].valid = 0; 541 dma_domain->mapped--; 542 } 543 } 544 545 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 546 547 } 548 549 static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, 550 phys_addr_t paddr, u64 size, int prot) 551 { 552 struct fsl_dma_domain *dma_domain = domain->priv; 553 struct dma_window *wnd; 554 int pamu_prot = 0; 555 int ret; 556 unsigned long flags; 557 u64 win_size; 558 559 if (prot & IOMMU_READ) 560 pamu_prot |= PAACE_AP_PERMS_QUERY; 561 if (prot & IOMMU_WRITE) 562 pamu_prot |= PAACE_AP_PERMS_UPDATE; 563 564 spin_lock_irqsave(&dma_domain->domain_lock, flags); 565 if (!dma_domain->win_arr) { 566 pr_debug("Number of windows not configured\n"); 567 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 568 return -ENODEV; 569 } 570 571 if (wnd_nr >= dma_domain->win_cnt) { 572 pr_debug("Invalid window index\n"); 573 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 574 return -EINVAL; 575 } 576 577 win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt); 578 if (size > win_size) { 579 pr_debug("Invalid window size \n"); 580 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 581 return -EINVAL; 582 } 583 584 if (dma_domain->win_cnt == 1) { 585 if (dma_domain->enabled) { 586 pr_debug("Disable the window before updating the mapping\n"); 587 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 588 return -EBUSY; 589 } 590 591 ret = check_size(size, domain->geometry.aperture_start); 592 if (ret) { 593 pr_debug("Aperture start not aligned to the size\n"); 594 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 595 return -EINVAL; 596 } 597 } 598 599 wnd = &dma_domain->win_arr[wnd_nr]; 600 if (!wnd->valid) { 601 wnd->paddr = paddr; 602 wnd->size = size; 603 wnd->prot = pamu_prot; 604 605 ret = update_domain_mapping(dma_domain, wnd_nr); 606 if (!ret) { 607 wnd->valid = 1; 608 dma_domain->mapped++; 609 } 610 } else { 611 pr_debug("Disable the window before updating the mapping\n"); 612 ret = -EBUSY; 613 } 614 615 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 616 617 return ret; 618 } 619 620 /* 621 * Attach the LIODN to the DMA domain and configure the geometry 622 * and window mappings. 623 */ 624 static int handle_attach_device(struct fsl_dma_domain *dma_domain, 625 struct device *dev, const u32 *liodn, 626 int num) 627 { 628 unsigned long flags; 629 struct iommu_domain *domain = dma_domain->iommu_domain; 630 int ret = 0; 631 int i; 632 633 spin_lock_irqsave(&dma_domain->domain_lock, flags); 634 for (i = 0; i < num; i++) { 635 636 /* Ensure that LIODN value is valid */ 637 if (liodn[i] >= PAACE_NUMBER_ENTRIES) { 638 pr_debug("Invalid liodn %d, attach device failed for %s\n", 639 liodn[i], dev->of_node->full_name); 640 ret = -EINVAL; 641 break; 642 } 643 644 attach_device(dma_domain, liodn[i], dev); 645 /* 646 * Check if geometry has already been configured 647 * for the domain. If yes, set the geometry for 648 * the LIODN. 649 */ 650 if (dma_domain->win_arr) { 651 u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0; 652 ret = pamu_set_liodn(liodn[i], dev, dma_domain, 653 &domain->geometry, 654 win_cnt); 655 if (ret) 656 break; 657 if (dma_domain->mapped) { 658 /* 659 * Create window/subwindow mapping for 660 * the LIODN. 661 */ 662 ret = map_liodn(liodn[i], dma_domain); 663 if (ret) 664 break; 665 } 666 } 667 } 668 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 669 670 return ret; 671 } 672 673 static int fsl_pamu_attach_device(struct iommu_domain *domain, 674 struct device *dev) 675 { 676 struct fsl_dma_domain *dma_domain = domain->priv; 677 const u32 *liodn; 678 u32 liodn_cnt; 679 int len, ret = 0; 680 struct pci_dev *pdev = NULL; 681 struct pci_controller *pci_ctl; 682 683 /* 684 * Use LIODN of the PCI controller while attaching a 685 * PCI device. 686 */ 687 if (dev_is_pci(dev)) { 688 pdev = to_pci_dev(dev); 689 pci_ctl = pci_bus_to_host(pdev->bus); 690 /* 691 * make dev point to pci controller device 692 * so we can get the LIODN programmed by 693 * u-boot. 694 */ 695 dev = pci_ctl->parent; 696 } 697 698 liodn = of_get_property(dev->of_node, "fsl,liodn", &len); 699 if (liodn) { 700 liodn_cnt = len / sizeof(u32); 701 ret = handle_attach_device(dma_domain, dev, 702 liodn, liodn_cnt); 703 } else { 704 pr_debug("missing fsl,liodn property at %s\n", 705 dev->of_node->full_name); 706 ret = -EINVAL; 707 } 708 709 return ret; 710 } 711 712 static void fsl_pamu_detach_device(struct iommu_domain *domain, 713 struct device *dev) 714 { 715 struct fsl_dma_domain *dma_domain = domain->priv; 716 const u32 *prop; 717 int len; 718 struct pci_dev *pdev = NULL; 719 struct pci_controller *pci_ctl; 720 721 /* 722 * Use LIODN of the PCI controller while detaching a 723 * PCI device. 724 */ 725 if (dev_is_pci(dev)) { 726 pdev = to_pci_dev(dev); 727 pci_ctl = pci_bus_to_host(pdev->bus); 728 /* 729 * make dev point to pci controller device 730 * so we can get the LIODN programmed by 731 * u-boot. 732 */ 733 dev = pci_ctl->parent; 734 } 735 736 prop = of_get_property(dev->of_node, "fsl,liodn", &len); 737 if (prop) 738 detach_device(dev, dma_domain); 739 else 740 pr_debug("missing fsl,liodn property at %s\n", 741 dev->of_node->full_name); 742 } 743 744 static int configure_domain_geometry(struct iommu_domain *domain, void *data) 745 { 746 struct iommu_domain_geometry *geom_attr = data; 747 struct fsl_dma_domain *dma_domain = domain->priv; 748 dma_addr_t geom_size; 749 unsigned long flags; 750 751 geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1; 752 /* 753 * Sanity check the geometry size. Also, we do not support 754 * DMA outside of the geometry. 755 */ 756 if (check_size(geom_size, geom_attr->aperture_start) || 757 !geom_attr->force_aperture) { 758 pr_debug("Invalid PAMU geometry attributes\n"); 759 return -EINVAL; 760 } 761 762 spin_lock_irqsave(&dma_domain->domain_lock, flags); 763 if (dma_domain->enabled) { 764 pr_debug("Can't set geometry attributes as domain is active\n"); 765 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 766 return -EBUSY; 767 } 768 769 /* Copy the domain geometry information */ 770 memcpy(&domain->geometry, geom_attr, 771 sizeof(struct iommu_domain_geometry)); 772 dma_domain->geom_size = geom_size; 773 774 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 775 776 return 0; 777 } 778 779 /* Set the domain stash attribute */ 780 static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data) 781 { 782 struct pamu_stash_attribute *stash_attr = data; 783 unsigned long flags; 784 int ret; 785 786 spin_lock_irqsave(&dma_domain->domain_lock, flags); 787 788 memcpy(&dma_domain->dma_stash, stash_attr, 789 sizeof(struct pamu_stash_attribute)); 790 791 dma_domain->stash_id = get_stash_id(stash_attr->cache, 792 stash_attr->cpu); 793 if (dma_domain->stash_id == ~(u32)0) { 794 pr_debug("Invalid stash attributes\n"); 795 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 796 return -EINVAL; 797 } 798 799 ret = update_domain_stash(dma_domain, dma_domain->stash_id); 800 801 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 802 803 return ret; 804 } 805 806 /* Configure domain dma state i.e. enable/disable DMA*/ 807 static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable) 808 { 809 struct device_domain_info *info; 810 unsigned long flags; 811 int ret; 812 813 spin_lock_irqsave(&dma_domain->domain_lock, flags); 814 815 if (enable && !dma_domain->mapped) { 816 pr_debug("Can't enable DMA domain without valid mapping\n"); 817 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 818 return -ENODEV; 819 } 820 821 dma_domain->enabled = enable; 822 list_for_each_entry(info, &dma_domain->devices, 823 link) { 824 ret = (enable) ? pamu_enable_liodn(info->liodn) : 825 pamu_disable_liodn(info->liodn); 826 if (ret) 827 pr_debug("Unable to set dma state for liodn %d", 828 info->liodn); 829 } 830 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 831 832 return 0; 833 } 834 835 static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, 836 enum iommu_attr attr_type, void *data) 837 { 838 struct fsl_dma_domain *dma_domain = domain->priv; 839 int ret = 0; 840 841 842 switch (attr_type) { 843 case DOMAIN_ATTR_GEOMETRY: 844 ret = configure_domain_geometry(domain, data); 845 break; 846 case DOMAIN_ATTR_FSL_PAMU_STASH: 847 ret = configure_domain_stash(dma_domain, data); 848 break; 849 case DOMAIN_ATTR_FSL_PAMU_ENABLE: 850 ret = configure_domain_dma_state(dma_domain, *(int *)data); 851 break; 852 default: 853 pr_debug("Unsupported attribute type\n"); 854 ret = -EINVAL; 855 break; 856 }; 857 858 return ret; 859 } 860 861 static int fsl_pamu_get_domain_attr(struct iommu_domain *domain, 862 enum iommu_attr attr_type, void *data) 863 { 864 struct fsl_dma_domain *dma_domain = domain->priv; 865 int ret = 0; 866 867 868 switch (attr_type) { 869 case DOMAIN_ATTR_FSL_PAMU_STASH: 870 memcpy((struct pamu_stash_attribute *) data, &dma_domain->dma_stash, 871 sizeof(struct pamu_stash_attribute)); 872 break; 873 case DOMAIN_ATTR_FSL_PAMU_ENABLE: 874 *(int *)data = dma_domain->enabled; 875 break; 876 case DOMAIN_ATTR_FSL_PAMUV1: 877 *(int *)data = DOMAIN_ATTR_FSL_PAMUV1; 878 break; 879 default: 880 pr_debug("Unsupported attribute type\n"); 881 ret = -EINVAL; 882 break; 883 }; 884 885 return ret; 886 } 887 888 static struct iommu_group *get_device_iommu_group(struct device *dev) 889 { 890 struct iommu_group *group; 891 892 group = iommu_group_get(dev); 893 if (!group) 894 group = iommu_group_alloc(); 895 896 return group; 897 } 898 899 static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl) 900 { 901 u32 version; 902 903 /* Check the PCI controller version number by readding BRR1 register */ 904 version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2)); 905 version &= PCI_FSL_BRR1_VER; 906 /* If PCI controller version is >= 0x204 we can partition endpoints*/ 907 if (version >= 0x204) 908 return 1; 909 910 return 0; 911 } 912 913 /* Get iommu group information from peer devices or devices on the parent bus */ 914 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev) 915 { 916 struct pci_dev *tmp; 917 struct iommu_group *group; 918 struct pci_bus *bus = pdev->bus; 919 920 /* 921 * Traverese the pci bus device list to get 922 * the shared iommu group. 923 */ 924 while (bus) { 925 list_for_each_entry(tmp, &bus->devices, bus_list) { 926 if (tmp == pdev) 927 continue; 928 group = iommu_group_get(&tmp->dev); 929 if (group) 930 return group; 931 } 932 933 bus = bus->parent; 934 } 935 936 return NULL; 937 } 938 939 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev) 940 { 941 struct pci_controller *pci_ctl; 942 bool pci_endpt_partioning; 943 struct iommu_group *group = NULL; 944 945 pci_ctl = pci_bus_to_host(pdev->bus); 946 pci_endpt_partioning = check_pci_ctl_endpt_part(pci_ctl); 947 /* We can partition PCIe devices so assign device group to the device */ 948 if (pci_endpt_partioning) { 949 group = iommu_group_get_for_dev(&pdev->dev); 950 951 /* 952 * PCIe controller is not a paritionable entity 953 * free the controller device iommu_group. 954 */ 955 if (pci_ctl->parent->iommu_group) 956 iommu_group_remove_device(pci_ctl->parent); 957 } else { 958 /* 959 * All devices connected to the controller will share the 960 * PCI controllers device group. If this is the first 961 * device to be probed for the pci controller, copy the 962 * device group information from the PCI controller device 963 * node and remove the PCI controller iommu group. 964 * For subsequent devices, the iommu group information can 965 * be obtained from sibling devices (i.e. from the bus_devices 966 * link list). 967 */ 968 if (pci_ctl->parent->iommu_group) { 969 group = get_device_iommu_group(pci_ctl->parent); 970 iommu_group_remove_device(pci_ctl->parent); 971 } else 972 group = get_shared_pci_device_group(pdev); 973 } 974 975 if (!group) 976 group = ERR_PTR(-ENODEV); 977 978 return group; 979 } 980 981 static int fsl_pamu_add_device(struct device *dev) 982 { 983 struct iommu_group *group = ERR_PTR(-ENODEV); 984 struct pci_dev *pdev; 985 const u32 *prop; 986 int ret = 0, len; 987 988 /* 989 * For platform devices we allocate a separate group for 990 * each of the devices. 991 */ 992 if (dev_is_pci(dev)) { 993 pdev = to_pci_dev(dev); 994 /* Don't create device groups for virtual PCI bridges */ 995 if (pdev->subordinate) 996 return 0; 997 998 group = get_pci_device_group(pdev); 999 1000 } else { 1001 prop = of_get_property(dev->of_node, "fsl,liodn", &len); 1002 if (prop) 1003 group = get_device_iommu_group(dev); 1004 } 1005 1006 if (IS_ERR(group)) 1007 return PTR_ERR(group); 1008 1009 /* 1010 * Check if device has already been added to an iommu group. 1011 * Group could have already been created for a PCI device in 1012 * the iommu_group_get_for_dev path. 1013 */ 1014 if (!dev->iommu_group) 1015 ret = iommu_group_add_device(group, dev); 1016 1017 iommu_group_put(group); 1018 return ret; 1019 } 1020 1021 static void fsl_pamu_remove_device(struct device *dev) 1022 { 1023 iommu_group_remove_device(dev); 1024 } 1025 1026 static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count) 1027 { 1028 struct fsl_dma_domain *dma_domain = domain->priv; 1029 unsigned long flags; 1030 int ret; 1031 1032 spin_lock_irqsave(&dma_domain->domain_lock, flags); 1033 /* Ensure domain is inactive i.e. DMA should be disabled for the domain */ 1034 if (dma_domain->enabled) { 1035 pr_debug("Can't set geometry attributes as domain is active\n"); 1036 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1037 return -EBUSY; 1038 } 1039 1040 /* Ensure that the geometry has been set for the domain */ 1041 if (!dma_domain->geom_size) { 1042 pr_debug("Please configure geometry before setting the number of windows\n"); 1043 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1044 return -EINVAL; 1045 } 1046 1047 /* 1048 * Ensure we have valid window count i.e. it should be less than 1049 * maximum permissible limit and should be a power of two. 1050 */ 1051 if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) { 1052 pr_debug("Invalid window count\n"); 1053 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1054 return -EINVAL; 1055 } 1056 1057 ret = pamu_set_domain_geometry(dma_domain, &domain->geometry, 1058 ((w_count > 1) ? w_count : 0)); 1059 if (!ret) { 1060 kfree(dma_domain->win_arr); 1061 dma_domain->win_arr = kzalloc(sizeof(struct dma_window) * 1062 w_count, GFP_ATOMIC); 1063 if (!dma_domain->win_arr) { 1064 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1065 return -ENOMEM; 1066 } 1067 dma_domain->win_cnt = w_count; 1068 } 1069 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1070 1071 return ret; 1072 } 1073 1074 static u32 fsl_pamu_get_windows(struct iommu_domain *domain) 1075 { 1076 struct fsl_dma_domain *dma_domain = domain->priv; 1077 1078 return dma_domain->win_cnt; 1079 } 1080 1081 static const struct iommu_ops fsl_pamu_ops = { 1082 .capable = fsl_pamu_capable, 1083 .domain_init = fsl_pamu_domain_init, 1084 .domain_destroy = fsl_pamu_domain_destroy, 1085 .attach_dev = fsl_pamu_attach_device, 1086 .detach_dev = fsl_pamu_detach_device, 1087 .domain_window_enable = fsl_pamu_window_enable, 1088 .domain_window_disable = fsl_pamu_window_disable, 1089 .domain_get_windows = fsl_pamu_get_windows, 1090 .domain_set_windows = fsl_pamu_set_windows, 1091 .iova_to_phys = fsl_pamu_iova_to_phys, 1092 .domain_set_attr = fsl_pamu_set_domain_attr, 1093 .domain_get_attr = fsl_pamu_get_domain_attr, 1094 .add_device = fsl_pamu_add_device, 1095 .remove_device = fsl_pamu_remove_device, 1096 }; 1097 1098 int pamu_domain_init(void) 1099 { 1100 int ret = 0; 1101 1102 ret = iommu_init_mempool(); 1103 if (ret) 1104 return ret; 1105 1106 bus_set_iommu(&platform_bus_type, &fsl_pamu_ops); 1107 bus_set_iommu(&pci_bus_type, &fsl_pamu_ops); 1108 1109 return ret; 1110 } 1111