1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (C) 2013 Freescale Semiconductor, Inc. 5 * Author: Varun Sethi <varun.sethi@freescale.com> 6 */ 7 8 #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__ 9 10 #include "fsl_pamu_domain.h" 11 12 #include <sysdev/fsl_pci.h> 13 14 /* 15 * Global spinlock that needs to be held while 16 * configuring PAMU. 17 */ 18 static DEFINE_SPINLOCK(iommu_lock); 19 20 static struct kmem_cache *fsl_pamu_domain_cache; 21 static struct kmem_cache *iommu_devinfo_cache; 22 static DEFINE_SPINLOCK(device_domain_lock); 23 24 struct iommu_device pamu_iommu; /* IOMMU core code handle */ 25 26 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom) 27 { 28 return container_of(dom, struct fsl_dma_domain, iommu_domain); 29 } 30 31 static int __init iommu_init_mempool(void) 32 { 33 fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain", 34 sizeof(struct fsl_dma_domain), 35 0, 36 SLAB_HWCACHE_ALIGN, 37 NULL); 38 if (!fsl_pamu_domain_cache) { 39 pr_debug("Couldn't create fsl iommu_domain cache\n"); 40 return -ENOMEM; 41 } 42 43 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo", 44 sizeof(struct device_domain_info), 45 0, 46 SLAB_HWCACHE_ALIGN, 47 NULL); 48 if (!iommu_devinfo_cache) { 49 pr_debug("Couldn't create devinfo cache\n"); 50 kmem_cache_destroy(fsl_pamu_domain_cache); 51 return -ENOMEM; 52 } 53 54 return 0; 55 } 56 57 static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova) 58 { 59 u32 win_cnt = dma_domain->win_cnt; 60 struct dma_window *win_ptr = &dma_domain->win_arr[0]; 61 struct iommu_domain_geometry *geom; 62 63 geom = &dma_domain->iommu_domain.geometry; 64 65 if (!win_cnt || !dma_domain->geom_size) { 66 pr_debug("Number of windows/geometry not configured for the domain\n"); 67 return 0; 68 } 69 70 if (win_cnt > 1) { 71 u64 subwin_size; 72 dma_addr_t subwin_iova; 73 u32 wnd; 74 75 subwin_size = dma_domain->geom_size >> ilog2(win_cnt); 76 subwin_iova = iova & ~(subwin_size - 1); 77 wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size); 78 win_ptr = &dma_domain->win_arr[wnd]; 79 } 80 81 if (win_ptr->valid) 82 return win_ptr->paddr + (iova & (win_ptr->size - 1)); 83 84 return 0; 85 } 86 87 static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain) 88 { 89 struct dma_window *sub_win_ptr = &dma_domain->win_arr[0]; 90 int i, ret; 91 unsigned long rpn, flags; 92 93 for (i = 0; i < dma_domain->win_cnt; i++) { 94 if (sub_win_ptr[i].valid) { 95 rpn = sub_win_ptr[i].paddr >> PAMU_PAGE_SHIFT; 96 spin_lock_irqsave(&iommu_lock, flags); 97 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i, 98 sub_win_ptr[i].size, 99 ~(u32)0, 100 rpn, 101 dma_domain->snoop_id, 102 dma_domain->stash_id, 103 (i > 0) ? 1 : 0, 104 sub_win_ptr[i].prot); 105 spin_unlock_irqrestore(&iommu_lock, flags); 106 if (ret) { 107 pr_debug("SPAACE configuration failed for liodn %d\n", 108 liodn); 109 return ret; 110 } 111 } 112 } 113 114 return ret; 115 } 116 117 static int map_win(int liodn, struct fsl_dma_domain *dma_domain) 118 { 119 int ret; 120 struct dma_window *wnd = &dma_domain->win_arr[0]; 121 phys_addr_t wnd_addr = dma_domain->iommu_domain.geometry.aperture_start; 122 unsigned long flags; 123 124 spin_lock_irqsave(&iommu_lock, flags); 125 ret = pamu_config_ppaace(liodn, wnd_addr, 126 wnd->size, 127 ~(u32)0, 128 wnd->paddr >> PAMU_PAGE_SHIFT, 129 dma_domain->snoop_id, dma_domain->stash_id, 130 0, wnd->prot); 131 spin_unlock_irqrestore(&iommu_lock, flags); 132 if (ret) 133 pr_debug("PAACE configuration failed for liodn %d\n", liodn); 134 135 return ret; 136 } 137 138 /* Map the DMA window corresponding to the LIODN */ 139 static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) 140 { 141 if (dma_domain->win_cnt > 1) 142 return map_subwins(liodn, dma_domain); 143 else 144 return map_win(liodn, dma_domain); 145 } 146 147 /* Update window/subwindow mapping for the LIODN */ 148 static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr) 149 { 150 int ret; 151 struct dma_window *wnd = &dma_domain->win_arr[wnd_nr]; 152 unsigned long flags; 153 154 spin_lock_irqsave(&iommu_lock, flags); 155 if (dma_domain->win_cnt > 1) { 156 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr, 157 wnd->size, 158 ~(u32)0, 159 wnd->paddr >> PAMU_PAGE_SHIFT, 160 dma_domain->snoop_id, 161 dma_domain->stash_id, 162 (wnd_nr > 0) ? 1 : 0, 163 wnd->prot); 164 if (ret) 165 pr_debug("Subwindow reconfiguration failed for liodn %d\n", 166 liodn); 167 } else { 168 phys_addr_t wnd_addr; 169 170 wnd_addr = dma_domain->iommu_domain.geometry.aperture_start; 171 172 ret = pamu_config_ppaace(liodn, wnd_addr, 173 wnd->size, 174 ~(u32)0, 175 wnd->paddr >> PAMU_PAGE_SHIFT, 176 dma_domain->snoop_id, dma_domain->stash_id, 177 0, wnd->prot); 178 if (ret) 179 pr_debug("Window reconfiguration failed for liodn %d\n", 180 liodn); 181 } 182 183 spin_unlock_irqrestore(&iommu_lock, flags); 184 185 return ret; 186 } 187 188 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, 189 u32 val) 190 { 191 int ret = 0, i; 192 unsigned long flags; 193 194 spin_lock_irqsave(&iommu_lock, flags); 195 if (!dma_domain->win_arr) { 196 pr_debug("Windows not configured, stash destination update failed for liodn %d\n", 197 liodn); 198 spin_unlock_irqrestore(&iommu_lock, flags); 199 return -EINVAL; 200 } 201 202 for (i = 0; i < dma_domain->win_cnt; i++) { 203 ret = pamu_update_paace_stash(liodn, i, val); 204 if (ret) { 205 pr_debug("Failed to update SPAACE %d field for liodn %d\n ", 206 i, liodn); 207 spin_unlock_irqrestore(&iommu_lock, flags); 208 return ret; 209 } 210 } 211 212 spin_unlock_irqrestore(&iommu_lock, flags); 213 214 return ret; 215 } 216 217 /* Set the geometry parameters for a LIODN */ 218 static int pamu_set_liodn(int liodn, struct device *dev, 219 struct fsl_dma_domain *dma_domain, 220 struct iommu_domain_geometry *geom_attr, 221 u32 win_cnt) 222 { 223 phys_addr_t window_addr, window_size; 224 phys_addr_t subwin_size; 225 int ret = 0, i; 226 u32 omi_index = ~(u32)0; 227 unsigned long flags; 228 229 /* 230 * Configure the omi_index at the geometry setup time. 231 * This is a static value which depends on the type of 232 * device and would not change thereafter. 233 */ 234 get_ome_index(&omi_index, dev); 235 236 window_addr = geom_attr->aperture_start; 237 window_size = dma_domain->geom_size; 238 239 spin_lock_irqsave(&iommu_lock, flags); 240 ret = pamu_disable_liodn(liodn); 241 if (!ret) 242 ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index, 243 0, dma_domain->snoop_id, 244 dma_domain->stash_id, win_cnt, 0); 245 spin_unlock_irqrestore(&iommu_lock, flags); 246 if (ret) { 247 pr_debug("PAACE configuration failed for liodn %d, win_cnt =%d\n", 248 liodn, win_cnt); 249 return ret; 250 } 251 252 if (win_cnt > 1) { 253 subwin_size = window_size >> ilog2(win_cnt); 254 for (i = 0; i < win_cnt; i++) { 255 spin_lock_irqsave(&iommu_lock, flags); 256 ret = pamu_disable_spaace(liodn, i); 257 if (!ret) 258 ret = pamu_config_spaace(liodn, win_cnt, i, 259 subwin_size, omi_index, 260 0, dma_domain->snoop_id, 261 dma_domain->stash_id, 262 0, 0); 263 spin_unlock_irqrestore(&iommu_lock, flags); 264 if (ret) { 265 pr_debug("SPAACE configuration failed for liodn %d\n", 266 liodn); 267 return ret; 268 } 269 } 270 } 271 272 return ret; 273 } 274 275 static int check_size(u64 size, dma_addr_t iova) 276 { 277 /* 278 * Size must be a power of two and at least be equal 279 * to PAMU page size. 280 */ 281 if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) { 282 pr_debug("Size too small or not a power of two\n"); 283 return -EINVAL; 284 } 285 286 /* iova must be page size aligned */ 287 if (iova & (size - 1)) { 288 pr_debug("Address is not aligned with window size\n"); 289 return -EINVAL; 290 } 291 292 return 0; 293 } 294 295 static struct fsl_dma_domain *iommu_alloc_dma_domain(void) 296 { 297 struct fsl_dma_domain *domain; 298 299 domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL); 300 if (!domain) 301 return NULL; 302 303 domain->stash_id = ~(u32)0; 304 domain->snoop_id = ~(u32)0; 305 domain->win_cnt = pamu_get_max_subwin_cnt(); 306 domain->geom_size = 0; 307 308 INIT_LIST_HEAD(&domain->devices); 309 310 spin_lock_init(&domain->domain_lock); 311 312 return domain; 313 } 314 315 static void remove_device_ref(struct device_domain_info *info, u32 win_cnt) 316 { 317 unsigned long flags; 318 319 list_del(&info->link); 320 spin_lock_irqsave(&iommu_lock, flags); 321 if (win_cnt > 1) 322 pamu_free_subwins(info->liodn); 323 pamu_disable_liodn(info->liodn); 324 spin_unlock_irqrestore(&iommu_lock, flags); 325 spin_lock_irqsave(&device_domain_lock, flags); 326 info->dev->archdata.iommu_domain = NULL; 327 kmem_cache_free(iommu_devinfo_cache, info); 328 spin_unlock_irqrestore(&device_domain_lock, flags); 329 } 330 331 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain) 332 { 333 struct device_domain_info *info, *tmp; 334 unsigned long flags; 335 336 spin_lock_irqsave(&dma_domain->domain_lock, flags); 337 /* Remove the device from the domain device list */ 338 list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) { 339 if (!dev || (info->dev == dev)) 340 remove_device_ref(info, dma_domain->win_cnt); 341 } 342 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 343 } 344 345 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev) 346 { 347 struct device_domain_info *info, *old_domain_info; 348 unsigned long flags; 349 350 spin_lock_irqsave(&device_domain_lock, flags); 351 /* 352 * Check here if the device is already attached to domain or not. 353 * If the device is already attached to a domain detach it. 354 */ 355 old_domain_info = dev->archdata.iommu_domain; 356 if (old_domain_info && old_domain_info->domain != dma_domain) { 357 spin_unlock_irqrestore(&device_domain_lock, flags); 358 detach_device(dev, old_domain_info->domain); 359 spin_lock_irqsave(&device_domain_lock, flags); 360 } 361 362 info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC); 363 364 info->dev = dev; 365 info->liodn = liodn; 366 info->domain = dma_domain; 367 368 list_add(&info->link, &dma_domain->devices); 369 /* 370 * In case of devices with multiple LIODNs just store 371 * the info for the first LIODN as all 372 * LIODNs share the same domain 373 */ 374 if (!dev->archdata.iommu_domain) 375 dev->archdata.iommu_domain = info; 376 spin_unlock_irqrestore(&device_domain_lock, flags); 377 } 378 379 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, 380 dma_addr_t iova) 381 { 382 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 383 384 if (iova < domain->geometry.aperture_start || 385 iova > domain->geometry.aperture_end) 386 return 0; 387 388 return get_phys_addr(dma_domain, iova); 389 } 390 391 static bool fsl_pamu_capable(enum iommu_cap cap) 392 { 393 return cap == IOMMU_CAP_CACHE_COHERENCY; 394 } 395 396 static void fsl_pamu_domain_free(struct iommu_domain *domain) 397 { 398 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 399 400 /* remove all the devices from the device list */ 401 detach_device(NULL, dma_domain); 402 403 dma_domain->enabled = 0; 404 dma_domain->mapped = 0; 405 406 kmem_cache_free(fsl_pamu_domain_cache, dma_domain); 407 } 408 409 static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type) 410 { 411 struct fsl_dma_domain *dma_domain; 412 413 if (type != IOMMU_DOMAIN_UNMANAGED) 414 return NULL; 415 416 dma_domain = iommu_alloc_dma_domain(); 417 if (!dma_domain) { 418 pr_debug("dma_domain allocation failed\n"); 419 return NULL; 420 } 421 /* defaul geometry 64 GB i.e. maximum system address */ 422 dma_domain->iommu_domain. geometry.aperture_start = 0; 423 dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1; 424 dma_domain->iommu_domain.geometry.force_aperture = true; 425 426 return &dma_domain->iommu_domain; 427 } 428 429 /* Configure geometry settings for all LIODNs associated with domain */ 430 static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain, 431 struct iommu_domain_geometry *geom_attr, 432 u32 win_cnt) 433 { 434 struct device_domain_info *info; 435 int ret = 0; 436 437 list_for_each_entry(info, &dma_domain->devices, link) { 438 ret = pamu_set_liodn(info->liodn, info->dev, dma_domain, 439 geom_attr, win_cnt); 440 if (ret) 441 break; 442 } 443 444 return ret; 445 } 446 447 /* Update stash destination for all LIODNs associated with the domain */ 448 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val) 449 { 450 struct device_domain_info *info; 451 int ret = 0; 452 453 list_for_each_entry(info, &dma_domain->devices, link) { 454 ret = update_liodn_stash(info->liodn, dma_domain, val); 455 if (ret) 456 break; 457 } 458 459 return ret; 460 } 461 462 /* Update domain mappings for all LIODNs associated with the domain */ 463 static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr) 464 { 465 struct device_domain_info *info; 466 int ret = 0; 467 468 list_for_each_entry(info, &dma_domain->devices, link) { 469 ret = update_liodn(info->liodn, dma_domain, wnd_nr); 470 if (ret) 471 break; 472 } 473 return ret; 474 } 475 476 static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr) 477 { 478 struct device_domain_info *info; 479 int ret = 0; 480 481 list_for_each_entry(info, &dma_domain->devices, link) { 482 if (dma_domain->win_cnt == 1 && dma_domain->enabled) { 483 ret = pamu_disable_liodn(info->liodn); 484 if (!ret) 485 dma_domain->enabled = 0; 486 } else { 487 ret = pamu_disable_spaace(info->liodn, wnd_nr); 488 } 489 } 490 491 return ret; 492 } 493 494 static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr) 495 { 496 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 497 unsigned long flags; 498 int ret; 499 500 spin_lock_irqsave(&dma_domain->domain_lock, flags); 501 if (!dma_domain->win_arr) { 502 pr_debug("Number of windows not configured\n"); 503 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 504 return; 505 } 506 507 if (wnd_nr >= dma_domain->win_cnt) { 508 pr_debug("Invalid window index\n"); 509 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 510 return; 511 } 512 513 if (dma_domain->win_arr[wnd_nr].valid) { 514 ret = disable_domain_win(dma_domain, wnd_nr); 515 if (!ret) { 516 dma_domain->win_arr[wnd_nr].valid = 0; 517 dma_domain->mapped--; 518 } 519 } 520 521 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 522 } 523 524 static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, 525 phys_addr_t paddr, u64 size, int prot) 526 { 527 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 528 struct dma_window *wnd; 529 int pamu_prot = 0; 530 int ret; 531 unsigned long flags; 532 u64 win_size; 533 534 if (prot & IOMMU_READ) 535 pamu_prot |= PAACE_AP_PERMS_QUERY; 536 if (prot & IOMMU_WRITE) 537 pamu_prot |= PAACE_AP_PERMS_UPDATE; 538 539 spin_lock_irqsave(&dma_domain->domain_lock, flags); 540 if (!dma_domain->win_arr) { 541 pr_debug("Number of windows not configured\n"); 542 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 543 return -ENODEV; 544 } 545 546 if (wnd_nr >= dma_domain->win_cnt) { 547 pr_debug("Invalid window index\n"); 548 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 549 return -EINVAL; 550 } 551 552 win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt); 553 if (size > win_size) { 554 pr_debug("Invalid window size\n"); 555 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 556 return -EINVAL; 557 } 558 559 if (dma_domain->win_cnt == 1) { 560 if (dma_domain->enabled) { 561 pr_debug("Disable the window before updating the mapping\n"); 562 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 563 return -EBUSY; 564 } 565 566 ret = check_size(size, domain->geometry.aperture_start); 567 if (ret) { 568 pr_debug("Aperture start not aligned to the size\n"); 569 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 570 return -EINVAL; 571 } 572 } 573 574 wnd = &dma_domain->win_arr[wnd_nr]; 575 if (!wnd->valid) { 576 wnd->paddr = paddr; 577 wnd->size = size; 578 wnd->prot = pamu_prot; 579 580 ret = update_domain_mapping(dma_domain, wnd_nr); 581 if (!ret) { 582 wnd->valid = 1; 583 dma_domain->mapped++; 584 } 585 } else { 586 pr_debug("Disable the window before updating the mapping\n"); 587 ret = -EBUSY; 588 } 589 590 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 591 592 return ret; 593 } 594 595 /* 596 * Attach the LIODN to the DMA domain and configure the geometry 597 * and window mappings. 598 */ 599 static int handle_attach_device(struct fsl_dma_domain *dma_domain, 600 struct device *dev, const u32 *liodn, 601 int num) 602 { 603 unsigned long flags; 604 struct iommu_domain *domain = &dma_domain->iommu_domain; 605 int ret = 0; 606 int i; 607 608 spin_lock_irqsave(&dma_domain->domain_lock, flags); 609 for (i = 0; i < num; i++) { 610 /* Ensure that LIODN value is valid */ 611 if (liodn[i] >= PAACE_NUMBER_ENTRIES) { 612 pr_debug("Invalid liodn %d, attach device failed for %pOF\n", 613 liodn[i], dev->of_node); 614 ret = -EINVAL; 615 break; 616 } 617 618 attach_device(dma_domain, liodn[i], dev); 619 /* 620 * Check if geometry has already been configured 621 * for the domain. If yes, set the geometry for 622 * the LIODN. 623 */ 624 if (dma_domain->win_arr) { 625 u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0; 626 627 ret = pamu_set_liodn(liodn[i], dev, dma_domain, 628 &domain->geometry, win_cnt); 629 if (ret) 630 break; 631 if (dma_domain->mapped) { 632 /* 633 * Create window/subwindow mapping for 634 * the LIODN. 635 */ 636 ret = map_liodn(liodn[i], dma_domain); 637 if (ret) 638 break; 639 } 640 } 641 } 642 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 643 644 return ret; 645 } 646 647 static int fsl_pamu_attach_device(struct iommu_domain *domain, 648 struct device *dev) 649 { 650 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 651 const u32 *liodn; 652 u32 liodn_cnt; 653 int len, ret = 0; 654 struct pci_dev *pdev = NULL; 655 struct pci_controller *pci_ctl; 656 657 /* 658 * Use LIODN of the PCI controller while attaching a 659 * PCI device. 660 */ 661 if (dev_is_pci(dev)) { 662 pdev = to_pci_dev(dev); 663 pci_ctl = pci_bus_to_host(pdev->bus); 664 /* 665 * make dev point to pci controller device 666 * so we can get the LIODN programmed by 667 * u-boot. 668 */ 669 dev = pci_ctl->parent; 670 } 671 672 liodn = of_get_property(dev->of_node, "fsl,liodn", &len); 673 if (liodn) { 674 liodn_cnt = len / sizeof(u32); 675 ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt); 676 } else { 677 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node); 678 ret = -EINVAL; 679 } 680 681 return ret; 682 } 683 684 static void fsl_pamu_detach_device(struct iommu_domain *domain, 685 struct device *dev) 686 { 687 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 688 const u32 *prop; 689 int len; 690 struct pci_dev *pdev = NULL; 691 struct pci_controller *pci_ctl; 692 693 /* 694 * Use LIODN of the PCI controller while detaching a 695 * PCI device. 696 */ 697 if (dev_is_pci(dev)) { 698 pdev = to_pci_dev(dev); 699 pci_ctl = pci_bus_to_host(pdev->bus); 700 /* 701 * make dev point to pci controller device 702 * so we can get the LIODN programmed by 703 * u-boot. 704 */ 705 dev = pci_ctl->parent; 706 } 707 708 prop = of_get_property(dev->of_node, "fsl,liodn", &len); 709 if (prop) 710 detach_device(dev, dma_domain); 711 else 712 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node); 713 } 714 715 static int configure_domain_geometry(struct iommu_domain *domain, void *data) 716 { 717 struct iommu_domain_geometry *geom_attr = data; 718 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 719 dma_addr_t geom_size; 720 unsigned long flags; 721 722 geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1; 723 /* 724 * Sanity check the geometry size. Also, we do not support 725 * DMA outside of the geometry. 726 */ 727 if (check_size(geom_size, geom_attr->aperture_start) || 728 !geom_attr->force_aperture) { 729 pr_debug("Invalid PAMU geometry attributes\n"); 730 return -EINVAL; 731 } 732 733 spin_lock_irqsave(&dma_domain->domain_lock, flags); 734 if (dma_domain->enabled) { 735 pr_debug("Can't set geometry attributes as domain is active\n"); 736 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 737 return -EBUSY; 738 } 739 740 /* Copy the domain geometry information */ 741 memcpy(&domain->geometry, geom_attr, 742 sizeof(struct iommu_domain_geometry)); 743 dma_domain->geom_size = geom_size; 744 745 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 746 747 return 0; 748 } 749 750 /* Set the domain stash attribute */ 751 static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data) 752 { 753 struct pamu_stash_attribute *stash_attr = data; 754 unsigned long flags; 755 int ret; 756 757 spin_lock_irqsave(&dma_domain->domain_lock, flags); 758 759 memcpy(&dma_domain->dma_stash, stash_attr, 760 sizeof(struct pamu_stash_attribute)); 761 762 dma_domain->stash_id = get_stash_id(stash_attr->cache, 763 stash_attr->cpu); 764 if (dma_domain->stash_id == ~(u32)0) { 765 pr_debug("Invalid stash attributes\n"); 766 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 767 return -EINVAL; 768 } 769 770 ret = update_domain_stash(dma_domain, dma_domain->stash_id); 771 772 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 773 774 return ret; 775 } 776 777 /* Configure domain dma state i.e. enable/disable DMA */ 778 static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable) 779 { 780 struct device_domain_info *info; 781 unsigned long flags; 782 int ret; 783 784 spin_lock_irqsave(&dma_domain->domain_lock, flags); 785 786 if (enable && !dma_domain->mapped) { 787 pr_debug("Can't enable DMA domain without valid mapping\n"); 788 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 789 return -ENODEV; 790 } 791 792 dma_domain->enabled = enable; 793 list_for_each_entry(info, &dma_domain->devices, link) { 794 ret = (enable) ? pamu_enable_liodn(info->liodn) : 795 pamu_disable_liodn(info->liodn); 796 if (ret) 797 pr_debug("Unable to set dma state for liodn %d", 798 info->liodn); 799 } 800 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 801 802 return 0; 803 } 804 805 static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count) 806 { 807 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 808 unsigned long flags; 809 int ret; 810 811 spin_lock_irqsave(&dma_domain->domain_lock, flags); 812 /* Ensure domain is inactive i.e. DMA should be disabled for the domain */ 813 if (dma_domain->enabled) { 814 pr_debug("Can't set geometry attributes as domain is active\n"); 815 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 816 return -EBUSY; 817 } 818 819 /* Ensure that the geometry has been set for the domain */ 820 if (!dma_domain->geom_size) { 821 pr_debug("Please configure geometry before setting the number of windows\n"); 822 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 823 return -EINVAL; 824 } 825 826 /* 827 * Ensure we have valid window count i.e. it should be less than 828 * maximum permissible limit and should be a power of two. 829 */ 830 if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) { 831 pr_debug("Invalid window count\n"); 832 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 833 return -EINVAL; 834 } 835 836 ret = pamu_set_domain_geometry(dma_domain, &domain->geometry, 837 w_count > 1 ? w_count : 0); 838 if (!ret) { 839 kfree(dma_domain->win_arr); 840 dma_domain->win_arr = kcalloc(w_count, 841 sizeof(*dma_domain->win_arr), 842 GFP_ATOMIC); 843 if (!dma_domain->win_arr) { 844 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 845 return -ENOMEM; 846 } 847 dma_domain->win_cnt = w_count; 848 } 849 spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 850 851 return ret; 852 } 853 854 static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, 855 enum iommu_attr attr_type, void *data) 856 { 857 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 858 int ret = 0; 859 860 switch (attr_type) { 861 case DOMAIN_ATTR_GEOMETRY: 862 ret = configure_domain_geometry(domain, data); 863 break; 864 case DOMAIN_ATTR_FSL_PAMU_STASH: 865 ret = configure_domain_stash(dma_domain, data); 866 break; 867 case DOMAIN_ATTR_FSL_PAMU_ENABLE: 868 ret = configure_domain_dma_state(dma_domain, *(int *)data); 869 break; 870 case DOMAIN_ATTR_WINDOWS: 871 ret = fsl_pamu_set_windows(domain, *(u32 *)data); 872 break; 873 default: 874 pr_debug("Unsupported attribute type\n"); 875 ret = -EINVAL; 876 break; 877 } 878 879 return ret; 880 } 881 882 static int fsl_pamu_get_domain_attr(struct iommu_domain *domain, 883 enum iommu_attr attr_type, void *data) 884 { 885 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 886 int ret = 0; 887 888 switch (attr_type) { 889 case DOMAIN_ATTR_FSL_PAMU_STASH: 890 memcpy(data, &dma_domain->dma_stash, 891 sizeof(struct pamu_stash_attribute)); 892 break; 893 case DOMAIN_ATTR_FSL_PAMU_ENABLE: 894 *(int *)data = dma_domain->enabled; 895 break; 896 case DOMAIN_ATTR_FSL_PAMUV1: 897 *(int *)data = DOMAIN_ATTR_FSL_PAMUV1; 898 break; 899 case DOMAIN_ATTR_WINDOWS: 900 *(u32 *)data = dma_domain->win_cnt; 901 break; 902 default: 903 pr_debug("Unsupported attribute type\n"); 904 ret = -EINVAL; 905 break; 906 } 907 908 return ret; 909 } 910 911 static struct iommu_group *get_device_iommu_group(struct device *dev) 912 { 913 struct iommu_group *group; 914 915 group = iommu_group_get(dev); 916 if (!group) 917 group = iommu_group_alloc(); 918 919 return group; 920 } 921 922 static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl) 923 { 924 u32 version; 925 926 /* Check the PCI controller version number by readding BRR1 register */ 927 version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2)); 928 version &= PCI_FSL_BRR1_VER; 929 /* If PCI controller version is >= 0x204 we can partition endpoints */ 930 return version >= 0x204; 931 } 932 933 /* Get iommu group information from peer devices or devices on the parent bus */ 934 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev) 935 { 936 struct pci_dev *tmp; 937 struct iommu_group *group; 938 struct pci_bus *bus = pdev->bus; 939 940 /* 941 * Traverese the pci bus device list to get 942 * the shared iommu group. 943 */ 944 while (bus) { 945 list_for_each_entry(tmp, &bus->devices, bus_list) { 946 if (tmp == pdev) 947 continue; 948 group = iommu_group_get(&tmp->dev); 949 if (group) 950 return group; 951 } 952 953 bus = bus->parent; 954 } 955 956 return NULL; 957 } 958 959 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev) 960 { 961 struct pci_controller *pci_ctl; 962 bool pci_endpt_partitioning; 963 struct iommu_group *group = NULL; 964 965 pci_ctl = pci_bus_to_host(pdev->bus); 966 pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl); 967 /* We can partition PCIe devices so assign device group to the device */ 968 if (pci_endpt_partitioning) { 969 group = pci_device_group(&pdev->dev); 970 971 /* 972 * PCIe controller is not a paritionable entity 973 * free the controller device iommu_group. 974 */ 975 if (pci_ctl->parent->iommu_group) 976 iommu_group_remove_device(pci_ctl->parent); 977 } else { 978 /* 979 * All devices connected to the controller will share the 980 * PCI controllers device group. If this is the first 981 * device to be probed for the pci controller, copy the 982 * device group information from the PCI controller device 983 * node and remove the PCI controller iommu group. 984 * For subsequent devices, the iommu group information can 985 * be obtained from sibling devices (i.e. from the bus_devices 986 * link list). 987 */ 988 if (pci_ctl->parent->iommu_group) { 989 group = get_device_iommu_group(pci_ctl->parent); 990 iommu_group_remove_device(pci_ctl->parent); 991 } else { 992 group = get_shared_pci_device_group(pdev); 993 } 994 } 995 996 if (!group) 997 group = ERR_PTR(-ENODEV); 998 999 return group; 1000 } 1001 1002 static struct iommu_group *fsl_pamu_device_group(struct device *dev) 1003 { 1004 struct iommu_group *group = ERR_PTR(-ENODEV); 1005 int len; 1006 1007 /* 1008 * For platform devices we allocate a separate group for 1009 * each of the devices. 1010 */ 1011 if (dev_is_pci(dev)) 1012 group = get_pci_device_group(to_pci_dev(dev)); 1013 else if (of_get_property(dev->of_node, "fsl,liodn", &len)) 1014 group = get_device_iommu_group(dev); 1015 1016 return group; 1017 } 1018 1019 static int fsl_pamu_add_device(struct device *dev) 1020 { 1021 struct iommu_group *group; 1022 1023 group = iommu_group_get_for_dev(dev); 1024 if (IS_ERR(group)) 1025 return PTR_ERR(group); 1026 1027 iommu_group_put(group); 1028 1029 iommu_device_link(&pamu_iommu, dev); 1030 1031 return 0; 1032 } 1033 1034 static void fsl_pamu_remove_device(struct device *dev) 1035 { 1036 iommu_device_unlink(&pamu_iommu, dev); 1037 iommu_group_remove_device(dev); 1038 } 1039 1040 static const struct iommu_ops fsl_pamu_ops = { 1041 .capable = fsl_pamu_capable, 1042 .domain_alloc = fsl_pamu_domain_alloc, 1043 .domain_free = fsl_pamu_domain_free, 1044 .attach_dev = fsl_pamu_attach_device, 1045 .detach_dev = fsl_pamu_detach_device, 1046 .domain_window_enable = fsl_pamu_window_enable, 1047 .domain_window_disable = fsl_pamu_window_disable, 1048 .iova_to_phys = fsl_pamu_iova_to_phys, 1049 .domain_set_attr = fsl_pamu_set_domain_attr, 1050 .domain_get_attr = fsl_pamu_get_domain_attr, 1051 .add_device = fsl_pamu_add_device, 1052 .remove_device = fsl_pamu_remove_device, 1053 .device_group = fsl_pamu_device_group, 1054 }; 1055 1056 int __init pamu_domain_init(void) 1057 { 1058 int ret = 0; 1059 1060 ret = iommu_init_mempool(); 1061 if (ret) 1062 return ret; 1063 1064 ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0"); 1065 if (ret) 1066 return ret; 1067 1068 iommu_device_set_ops(&pamu_iommu, &fsl_pamu_ops); 1069 1070 ret = iommu_device_register(&pamu_iommu); 1071 if (ret) { 1072 iommu_device_sysfs_remove(&pamu_iommu); 1073 pr_err("Can't register iommu device\n"); 1074 return ret; 1075 } 1076 1077 bus_set_iommu(&platform_bus_type, &fsl_pamu_ops); 1078 bus_set_iommu(&pci_bus_type, &fsl_pamu_ops); 1079 1080 return ret; 1081 } 1082