1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved. 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/debugfs.h> 8 #include <linux/err.h> 9 #include <linux/iommu.h> 10 #include <linux/kernel.h> 11 #include <linux/of.h> 12 #include <linux/of_device.h> 13 #include <linux/platform_device.h> 14 #include <linux/slab.h> 15 #include <linux/dma-mapping.h> 16 17 #include <soc/tegra/ahb.h> 18 #include <soc/tegra/mc.h> 19 20 struct tegra_smmu_group { 21 struct list_head list; 22 const struct tegra_smmu_group_soc *soc; 23 struct iommu_group *group; 24 }; 25 26 struct tegra_smmu { 27 void __iomem *regs; 28 struct device *dev; 29 30 struct tegra_mc *mc; 31 const struct tegra_smmu_soc *soc; 32 33 struct list_head groups; 34 35 unsigned long pfn_mask; 36 unsigned long tlb_mask; 37 38 unsigned long *asids; 39 struct mutex lock; 40 41 struct list_head list; 42 43 struct dentry *debugfs; 44 45 struct iommu_device iommu; /* IOMMU Core code handle */ 46 }; 47 48 struct tegra_smmu_as { 49 struct iommu_domain domain; 50 struct tegra_smmu *smmu; 51 unsigned int use_count; 52 u32 *count; 53 struct page **pts; 54 struct page *pd; 55 dma_addr_t pd_dma; 56 unsigned id; 57 u32 attr; 58 }; 59 60 static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom) 61 { 62 return container_of(dom, struct tegra_smmu_as, domain); 63 } 64 65 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value, 66 unsigned long offset) 67 { 68 writel(value, smmu->regs + offset); 69 } 70 71 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) 72 { 73 return readl(smmu->regs + offset); 74 } 75 76 #define SMMU_CONFIG 0x010 77 #define SMMU_CONFIG_ENABLE (1 << 0) 78 79 #define SMMU_TLB_CONFIG 0x14 80 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29) 81 #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28) 82 #define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \ 83 ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask) 84 85 #define SMMU_PTC_CONFIG 0x18 86 #define SMMU_PTC_CONFIG_ENABLE (1 << 29) 87 #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24) 88 #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f) 89 90 #define SMMU_PTB_ASID 0x01c 91 #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) 92 93 #define SMMU_PTB_DATA 0x020 94 #define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr)) 95 96 #define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr)) 97 98 #define SMMU_TLB_FLUSH 0x030 99 #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) 100 #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0) 101 #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0) 102 #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \ 103 SMMU_TLB_FLUSH_VA_MATCH_SECTION) 104 #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \ 105 SMMU_TLB_FLUSH_VA_MATCH_GROUP) 106 #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31) 107 108 #define SMMU_PTC_FLUSH 0x034 109 #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0) 110 #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0) 111 112 #define SMMU_PTC_FLUSH_HI 0x9b8 113 #define SMMU_PTC_FLUSH_HI_MASK 0x3 114 115 /* per-SWGROUP SMMU_*_ASID register */ 116 #define SMMU_ASID_ENABLE (1 << 31) 117 #define SMMU_ASID_MASK 0x7f 118 #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK) 119 120 /* page table definitions */ 121 #define SMMU_NUM_PDE 1024 122 #define SMMU_NUM_PTE 1024 123 124 #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4) 125 #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4) 126 127 #define SMMU_PDE_SHIFT 22 128 #define SMMU_PTE_SHIFT 12 129 130 #define SMMU_PD_READABLE (1 << 31) 131 #define SMMU_PD_WRITABLE (1 << 30) 132 #define SMMU_PD_NONSECURE (1 << 29) 133 134 #define SMMU_PDE_READABLE (1 << 31) 135 #define SMMU_PDE_WRITABLE (1 << 30) 136 #define SMMU_PDE_NONSECURE (1 << 29) 137 #define SMMU_PDE_NEXT (1 << 28) 138 139 #define SMMU_PTE_READABLE (1 << 31) 140 #define SMMU_PTE_WRITABLE (1 << 30) 141 #define SMMU_PTE_NONSECURE (1 << 29) 142 143 #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \ 144 SMMU_PDE_NONSECURE) 145 146 static unsigned int iova_pd_index(unsigned long iova) 147 { 148 return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1); 149 } 150 151 static unsigned int iova_pt_index(unsigned long iova) 152 { 153 return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); 154 } 155 156 static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr) 157 { 158 addr >>= 12; 159 return (addr & smmu->pfn_mask) == addr; 160 } 161 162 static dma_addr_t smmu_pde_to_dma(u32 pde) 163 { 164 return pde << 12; 165 } 166 167 static void smmu_flush_ptc_all(struct tegra_smmu *smmu) 168 { 169 smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH); 170 } 171 172 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma, 173 unsigned long offset) 174 { 175 u32 value; 176 177 offset &= ~(smmu->mc->soc->atom_size - 1); 178 179 if (smmu->mc->soc->num_address_bits > 32) { 180 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 181 value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK; 182 #else 183 value = 0; 184 #endif 185 smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); 186 } 187 188 value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR; 189 smmu_writel(smmu, value, SMMU_PTC_FLUSH); 190 } 191 192 static inline void smmu_flush_tlb(struct tegra_smmu *smmu) 193 { 194 smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH); 195 } 196 197 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu, 198 unsigned long asid) 199 { 200 u32 value; 201 202 if (smmu->soc->num_asids == 4) 203 value = (asid & 0x3) << 29; 204 else 205 value = (asid & 0x7f) << 24; 206 207 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL; 208 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 209 } 210 211 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu, 212 unsigned long asid, 213 unsigned long iova) 214 { 215 u32 value; 216 217 if (smmu->soc->num_asids == 4) 218 value = (asid & 0x3) << 29; 219 else 220 value = (asid & 0x7f) << 24; 221 222 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova); 223 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 224 } 225 226 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu, 227 unsigned long asid, 228 unsigned long iova) 229 { 230 u32 value; 231 232 if (smmu->soc->num_asids == 4) 233 value = (asid & 0x3) << 29; 234 else 235 value = (asid & 0x7f) << 24; 236 237 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova); 238 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 239 } 240 241 static inline void smmu_flush(struct tegra_smmu *smmu) 242 { 243 smmu_readl(smmu, SMMU_CONFIG); 244 } 245 246 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp) 247 { 248 unsigned long id; 249 250 mutex_lock(&smmu->lock); 251 252 id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids); 253 if (id >= smmu->soc->num_asids) { 254 mutex_unlock(&smmu->lock); 255 return -ENOSPC; 256 } 257 258 set_bit(id, smmu->asids); 259 *idp = id; 260 261 mutex_unlock(&smmu->lock); 262 return 0; 263 } 264 265 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id) 266 { 267 mutex_lock(&smmu->lock); 268 clear_bit(id, smmu->asids); 269 mutex_unlock(&smmu->lock); 270 } 271 272 static bool tegra_smmu_capable(enum iommu_cap cap) 273 { 274 return false; 275 } 276 277 static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) 278 { 279 struct tegra_smmu_as *as; 280 281 if (type != IOMMU_DOMAIN_UNMANAGED) 282 return NULL; 283 284 as = kzalloc(sizeof(*as), GFP_KERNEL); 285 if (!as) 286 return NULL; 287 288 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; 289 290 as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); 291 if (!as->pd) { 292 kfree(as); 293 return NULL; 294 } 295 296 as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); 297 if (!as->count) { 298 __free_page(as->pd); 299 kfree(as); 300 return NULL; 301 } 302 303 as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); 304 if (!as->pts) { 305 kfree(as->count); 306 __free_page(as->pd); 307 kfree(as); 308 return NULL; 309 } 310 311 /* setup aperture */ 312 as->domain.geometry.aperture_start = 0; 313 as->domain.geometry.aperture_end = 0xffffffff; 314 as->domain.geometry.force_aperture = true; 315 316 return &as->domain; 317 } 318 319 static void tegra_smmu_domain_free(struct iommu_domain *domain) 320 { 321 struct tegra_smmu_as *as = to_smmu_as(domain); 322 323 /* TODO: free page directory and page tables */ 324 325 WARN_ON_ONCE(as->use_count); 326 kfree(as->count); 327 kfree(as->pts); 328 kfree(as); 329 } 330 331 static const struct tegra_smmu_swgroup * 332 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup) 333 { 334 const struct tegra_smmu_swgroup *group = NULL; 335 unsigned int i; 336 337 for (i = 0; i < smmu->soc->num_swgroups; i++) { 338 if (smmu->soc->swgroups[i].swgroup == swgroup) { 339 group = &smmu->soc->swgroups[i]; 340 break; 341 } 342 } 343 344 return group; 345 } 346 347 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup, 348 unsigned int asid) 349 { 350 const struct tegra_smmu_swgroup *group; 351 unsigned int i; 352 u32 value; 353 354 for (i = 0; i < smmu->soc->num_clients; i++) { 355 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 356 357 if (client->swgroup != swgroup) 358 continue; 359 360 value = smmu_readl(smmu, client->smmu.reg); 361 value |= BIT(client->smmu.bit); 362 smmu_writel(smmu, value, client->smmu.reg); 363 } 364 365 group = tegra_smmu_find_swgroup(smmu, swgroup); 366 if (group) { 367 value = smmu_readl(smmu, group->reg); 368 value &= ~SMMU_ASID_MASK; 369 value |= SMMU_ASID_VALUE(asid); 370 value |= SMMU_ASID_ENABLE; 371 smmu_writel(smmu, value, group->reg); 372 } 373 } 374 375 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup, 376 unsigned int asid) 377 { 378 const struct tegra_smmu_swgroup *group; 379 unsigned int i; 380 u32 value; 381 382 group = tegra_smmu_find_swgroup(smmu, swgroup); 383 if (group) { 384 value = smmu_readl(smmu, group->reg); 385 value &= ~SMMU_ASID_MASK; 386 value |= SMMU_ASID_VALUE(asid); 387 value &= ~SMMU_ASID_ENABLE; 388 smmu_writel(smmu, value, group->reg); 389 } 390 391 for (i = 0; i < smmu->soc->num_clients; i++) { 392 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 393 394 if (client->swgroup != swgroup) 395 continue; 396 397 value = smmu_readl(smmu, client->smmu.reg); 398 value &= ~BIT(client->smmu.bit); 399 smmu_writel(smmu, value, client->smmu.reg); 400 } 401 } 402 403 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, 404 struct tegra_smmu_as *as) 405 { 406 u32 value; 407 int err; 408 409 if (as->use_count > 0) { 410 as->use_count++; 411 return 0; 412 } 413 414 as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD, 415 DMA_TO_DEVICE); 416 if (dma_mapping_error(smmu->dev, as->pd_dma)) 417 return -ENOMEM; 418 419 /* We can't handle 64-bit DMA addresses */ 420 if (!smmu_dma_addr_valid(smmu, as->pd_dma)) { 421 err = -ENOMEM; 422 goto err_unmap; 423 } 424 425 err = tegra_smmu_alloc_asid(smmu, &as->id); 426 if (err < 0) 427 goto err_unmap; 428 429 smmu_flush_ptc(smmu, as->pd_dma, 0); 430 smmu_flush_tlb_asid(smmu, as->id); 431 432 smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); 433 value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr); 434 smmu_writel(smmu, value, SMMU_PTB_DATA); 435 smmu_flush(smmu); 436 437 as->smmu = smmu; 438 as->use_count++; 439 440 return 0; 441 442 err_unmap: 443 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 444 return err; 445 } 446 447 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, 448 struct tegra_smmu_as *as) 449 { 450 if (--as->use_count > 0) 451 return; 452 453 tegra_smmu_free_asid(smmu, as->id); 454 455 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 456 457 as->smmu = NULL; 458 } 459 460 static int tegra_smmu_attach_dev(struct iommu_domain *domain, 461 struct device *dev) 462 { 463 struct tegra_smmu *smmu = dev->archdata.iommu; 464 struct tegra_smmu_as *as = to_smmu_as(domain); 465 struct device_node *np = dev->of_node; 466 struct of_phandle_args args; 467 unsigned int index = 0; 468 int err = 0; 469 470 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 471 &args)) { 472 unsigned int swgroup = args.args[0]; 473 474 if (args.np != smmu->dev->of_node) { 475 of_node_put(args.np); 476 continue; 477 } 478 479 of_node_put(args.np); 480 481 err = tegra_smmu_as_prepare(smmu, as); 482 if (err < 0) 483 return err; 484 485 tegra_smmu_enable(smmu, swgroup, as->id); 486 index++; 487 } 488 489 if (index == 0) 490 return -ENODEV; 491 492 return 0; 493 } 494 495 static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) 496 { 497 struct tegra_smmu_as *as = to_smmu_as(domain); 498 struct device_node *np = dev->of_node; 499 struct tegra_smmu *smmu = as->smmu; 500 struct of_phandle_args args; 501 unsigned int index = 0; 502 503 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 504 &args)) { 505 unsigned int swgroup = args.args[0]; 506 507 if (args.np != smmu->dev->of_node) { 508 of_node_put(args.np); 509 continue; 510 } 511 512 of_node_put(args.np); 513 514 tegra_smmu_disable(smmu, swgroup, as->id); 515 tegra_smmu_as_unprepare(smmu, as); 516 index++; 517 } 518 } 519 520 static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova, 521 u32 value) 522 { 523 unsigned int pd_index = iova_pd_index(iova); 524 struct tegra_smmu *smmu = as->smmu; 525 u32 *pd = page_address(as->pd); 526 unsigned long offset = pd_index * sizeof(*pd); 527 528 /* Set the page directory entry first */ 529 pd[pd_index] = value; 530 531 /* The flush the page directory entry from caches */ 532 dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, 533 sizeof(*pd), DMA_TO_DEVICE); 534 535 /* And flush the iommu */ 536 smmu_flush_ptc(smmu, as->pd_dma, offset); 537 smmu_flush_tlb_section(smmu, as->id, iova); 538 smmu_flush(smmu); 539 } 540 541 static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) 542 { 543 u32 *pt = page_address(pt_page); 544 545 return pt + iova_pt_index(iova); 546 } 547 548 static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, 549 dma_addr_t *dmap) 550 { 551 unsigned int pd_index = iova_pd_index(iova); 552 struct page *pt_page; 553 u32 *pd; 554 555 pt_page = as->pts[pd_index]; 556 if (!pt_page) 557 return NULL; 558 559 pd = page_address(as->pd); 560 *dmap = smmu_pde_to_dma(pd[pd_index]); 561 562 return tegra_smmu_pte_offset(pt_page, iova); 563 } 564 565 static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, 566 dma_addr_t *dmap) 567 { 568 unsigned int pde = iova_pd_index(iova); 569 struct tegra_smmu *smmu = as->smmu; 570 571 if (!as->pts[pde]) { 572 struct page *page; 573 dma_addr_t dma; 574 575 page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); 576 if (!page) 577 return NULL; 578 579 dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, 580 DMA_TO_DEVICE); 581 if (dma_mapping_error(smmu->dev, dma)) { 582 __free_page(page); 583 return NULL; 584 } 585 586 if (!smmu_dma_addr_valid(smmu, dma)) { 587 dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, 588 DMA_TO_DEVICE); 589 __free_page(page); 590 return NULL; 591 } 592 593 as->pts[pde] = page; 594 595 tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR | 596 SMMU_PDE_NEXT)); 597 598 *dmap = dma; 599 } else { 600 u32 *pd = page_address(as->pd); 601 602 *dmap = smmu_pde_to_dma(pd[pde]); 603 } 604 605 return tegra_smmu_pte_offset(as->pts[pde], iova); 606 } 607 608 static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova) 609 { 610 unsigned int pd_index = iova_pd_index(iova); 611 612 as->count[pd_index]++; 613 } 614 615 static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) 616 { 617 unsigned int pde = iova_pd_index(iova); 618 struct page *page = as->pts[pde]; 619 620 /* 621 * When no entries in this page table are used anymore, return the 622 * memory page to the system. 623 */ 624 if (--as->count[pde] == 0) { 625 struct tegra_smmu *smmu = as->smmu; 626 u32 *pd = page_address(as->pd); 627 dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]); 628 629 tegra_smmu_set_pde(as, iova, 0); 630 631 dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); 632 __free_page(page); 633 as->pts[pde] = NULL; 634 } 635 } 636 637 static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, 638 u32 *pte, dma_addr_t pte_dma, u32 val) 639 { 640 struct tegra_smmu *smmu = as->smmu; 641 unsigned long offset = offset_in_page(pte); 642 643 *pte = val; 644 645 dma_sync_single_range_for_device(smmu->dev, pte_dma, offset, 646 4, DMA_TO_DEVICE); 647 smmu_flush_ptc(smmu, pte_dma, offset); 648 smmu_flush_tlb_group(smmu, as->id, iova); 649 smmu_flush(smmu); 650 } 651 652 static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, 653 phys_addr_t paddr, size_t size, int prot) 654 { 655 struct tegra_smmu_as *as = to_smmu_as(domain); 656 dma_addr_t pte_dma; 657 u32 pte_attrs; 658 u32 *pte; 659 660 pte = as_get_pte(as, iova, &pte_dma); 661 if (!pte) 662 return -ENOMEM; 663 664 /* If we aren't overwriting a pre-existing entry, increment use */ 665 if (*pte == 0) 666 tegra_smmu_pte_get_use(as, iova); 667 668 pte_attrs = SMMU_PTE_NONSECURE; 669 670 if (prot & IOMMU_READ) 671 pte_attrs |= SMMU_PTE_READABLE; 672 673 if (prot & IOMMU_WRITE) 674 pte_attrs |= SMMU_PTE_WRITABLE; 675 676 tegra_smmu_set_pte(as, iova, pte, pte_dma, 677 __phys_to_pfn(paddr) | pte_attrs); 678 679 return 0; 680 } 681 682 static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 683 size_t size) 684 { 685 struct tegra_smmu_as *as = to_smmu_as(domain); 686 dma_addr_t pte_dma; 687 u32 *pte; 688 689 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); 690 if (!pte || !*pte) 691 return 0; 692 693 tegra_smmu_set_pte(as, iova, pte, pte_dma, 0); 694 tegra_smmu_pte_put_use(as, iova); 695 696 return size; 697 } 698 699 static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, 700 dma_addr_t iova) 701 { 702 struct tegra_smmu_as *as = to_smmu_as(domain); 703 unsigned long pfn; 704 dma_addr_t pte_dma; 705 u32 *pte; 706 707 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); 708 if (!pte || !*pte) 709 return 0; 710 711 pfn = *pte & as->smmu->pfn_mask; 712 713 return PFN_PHYS(pfn); 714 } 715 716 static struct tegra_smmu *tegra_smmu_find(struct device_node *np) 717 { 718 struct platform_device *pdev; 719 struct tegra_mc *mc; 720 721 pdev = of_find_device_by_node(np); 722 if (!pdev) 723 return NULL; 724 725 mc = platform_get_drvdata(pdev); 726 if (!mc) 727 return NULL; 728 729 return mc->smmu; 730 } 731 732 static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev, 733 struct of_phandle_args *args) 734 { 735 const struct iommu_ops *ops = smmu->iommu.ops; 736 int err; 737 738 err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops); 739 if (err < 0) { 740 dev_err(dev, "failed to initialize fwspec: %d\n", err); 741 return err; 742 } 743 744 err = ops->of_xlate(dev, args); 745 if (err < 0) { 746 dev_err(dev, "failed to parse SW group ID: %d\n", err); 747 iommu_fwspec_free(dev); 748 return err; 749 } 750 751 return 0; 752 } 753 754 static int tegra_smmu_add_device(struct device *dev) 755 { 756 struct device_node *np = dev->of_node; 757 struct tegra_smmu *smmu = NULL; 758 struct iommu_group *group; 759 struct of_phandle_args args; 760 unsigned int index = 0; 761 int err; 762 763 while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 764 &args) == 0) { 765 smmu = tegra_smmu_find(args.np); 766 if (smmu) { 767 err = tegra_smmu_configure(smmu, dev, &args); 768 of_node_put(args.np); 769 770 if (err < 0) 771 return err; 772 773 /* 774 * Only a single IOMMU master interface is currently 775 * supported by the Linux kernel, so abort after the 776 * first match. 777 */ 778 dev->archdata.iommu = smmu; 779 780 iommu_device_link(&smmu->iommu, dev); 781 782 break; 783 } 784 785 of_node_put(args.np); 786 index++; 787 } 788 789 if (!smmu) 790 return -ENODEV; 791 792 group = iommu_group_get_for_dev(dev); 793 if (IS_ERR(group)) 794 return PTR_ERR(group); 795 796 iommu_group_put(group); 797 798 return 0; 799 } 800 801 static void tegra_smmu_remove_device(struct device *dev) 802 { 803 struct tegra_smmu *smmu = dev->archdata.iommu; 804 805 if (smmu) 806 iommu_device_unlink(&smmu->iommu, dev); 807 808 dev->archdata.iommu = NULL; 809 iommu_group_remove_device(dev); 810 } 811 812 static const struct tegra_smmu_group_soc * 813 tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup) 814 { 815 unsigned int i, j; 816 817 for (i = 0; i < smmu->soc->num_groups; i++) 818 for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++) 819 if (smmu->soc->groups[i].swgroups[j] == swgroup) 820 return &smmu->soc->groups[i]; 821 822 return NULL; 823 } 824 825 static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu, 826 unsigned int swgroup) 827 { 828 const struct tegra_smmu_group_soc *soc; 829 struct tegra_smmu_group *group; 830 831 soc = tegra_smmu_find_group(smmu, swgroup); 832 if (!soc) 833 return NULL; 834 835 mutex_lock(&smmu->lock); 836 837 list_for_each_entry(group, &smmu->groups, list) 838 if (group->soc == soc) { 839 mutex_unlock(&smmu->lock); 840 return group->group; 841 } 842 843 group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL); 844 if (!group) { 845 mutex_unlock(&smmu->lock); 846 return NULL; 847 } 848 849 INIT_LIST_HEAD(&group->list); 850 group->soc = soc; 851 852 group->group = iommu_group_alloc(); 853 if (IS_ERR(group->group)) { 854 devm_kfree(smmu->dev, group); 855 mutex_unlock(&smmu->lock); 856 return NULL; 857 } 858 859 list_add_tail(&group->list, &smmu->groups); 860 mutex_unlock(&smmu->lock); 861 862 return group->group; 863 } 864 865 static struct iommu_group *tegra_smmu_device_group(struct device *dev) 866 { 867 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 868 struct tegra_smmu *smmu = dev->archdata.iommu; 869 struct iommu_group *group; 870 871 group = tegra_smmu_group_get(smmu, fwspec->ids[0]); 872 if (!group) 873 group = generic_device_group(dev); 874 875 return group; 876 } 877 878 static int tegra_smmu_of_xlate(struct device *dev, 879 struct of_phandle_args *args) 880 { 881 u32 id = args->args[0]; 882 883 return iommu_fwspec_add_ids(dev, &id, 1); 884 } 885 886 static const struct iommu_ops tegra_smmu_ops = { 887 .capable = tegra_smmu_capable, 888 .domain_alloc = tegra_smmu_domain_alloc, 889 .domain_free = tegra_smmu_domain_free, 890 .attach_dev = tegra_smmu_attach_dev, 891 .detach_dev = tegra_smmu_detach_dev, 892 .add_device = tegra_smmu_add_device, 893 .remove_device = tegra_smmu_remove_device, 894 .device_group = tegra_smmu_device_group, 895 .map = tegra_smmu_map, 896 .unmap = tegra_smmu_unmap, 897 .iova_to_phys = tegra_smmu_iova_to_phys, 898 .of_xlate = tegra_smmu_of_xlate, 899 .pgsize_bitmap = SZ_4K, 900 }; 901 902 static void tegra_smmu_ahb_enable(void) 903 { 904 static const struct of_device_id ahb_match[] = { 905 { .compatible = "nvidia,tegra30-ahb", }, 906 { } 907 }; 908 struct device_node *ahb; 909 910 ahb = of_find_matching_node(NULL, ahb_match); 911 if (ahb) { 912 tegra_ahb_enable_smmu(ahb); 913 of_node_put(ahb); 914 } 915 } 916 917 static int tegra_smmu_swgroups_show(struct seq_file *s, void *data) 918 { 919 struct tegra_smmu *smmu = s->private; 920 unsigned int i; 921 u32 value; 922 923 seq_printf(s, "swgroup enabled ASID\n"); 924 seq_printf(s, "------------------------\n"); 925 926 for (i = 0; i < smmu->soc->num_swgroups; i++) { 927 const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i]; 928 const char *status; 929 unsigned int asid; 930 931 value = smmu_readl(smmu, group->reg); 932 933 if (value & SMMU_ASID_ENABLE) 934 status = "yes"; 935 else 936 status = "no"; 937 938 asid = value & SMMU_ASID_MASK; 939 940 seq_printf(s, "%-9s %-7s %#04x\n", group->name, status, 941 asid); 942 } 943 944 return 0; 945 } 946 947 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups); 948 949 static int tegra_smmu_clients_show(struct seq_file *s, void *data) 950 { 951 struct tegra_smmu *smmu = s->private; 952 unsigned int i; 953 u32 value; 954 955 seq_printf(s, "client enabled\n"); 956 seq_printf(s, "--------------------\n"); 957 958 for (i = 0; i < smmu->soc->num_clients; i++) { 959 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 960 const char *status; 961 962 value = smmu_readl(smmu, client->smmu.reg); 963 964 if (value & BIT(client->smmu.bit)) 965 status = "yes"; 966 else 967 status = "no"; 968 969 seq_printf(s, "%-12s %s\n", client->name, status); 970 } 971 972 return 0; 973 } 974 975 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients); 976 977 static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu) 978 { 979 smmu->debugfs = debugfs_create_dir("smmu", NULL); 980 if (!smmu->debugfs) 981 return; 982 983 debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu, 984 &tegra_smmu_swgroups_fops); 985 debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu, 986 &tegra_smmu_clients_fops); 987 } 988 989 static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu) 990 { 991 debugfs_remove_recursive(smmu->debugfs); 992 } 993 994 struct tegra_smmu *tegra_smmu_probe(struct device *dev, 995 const struct tegra_smmu_soc *soc, 996 struct tegra_mc *mc) 997 { 998 struct tegra_smmu *smmu; 999 size_t size; 1000 u32 value; 1001 int err; 1002 1003 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 1004 if (!smmu) 1005 return ERR_PTR(-ENOMEM); 1006 1007 /* 1008 * This is a bit of a hack. Ideally we'd want to simply return this 1009 * value. However the IOMMU registration process will attempt to add 1010 * all devices to the IOMMU when bus_set_iommu() is called. In order 1011 * not to rely on global variables to track the IOMMU instance, we 1012 * set it here so that it can be looked up from the .add_device() 1013 * callback via the IOMMU device's .drvdata field. 1014 */ 1015 mc->smmu = smmu; 1016 1017 size = BITS_TO_LONGS(soc->num_asids) * sizeof(long); 1018 1019 smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL); 1020 if (!smmu->asids) 1021 return ERR_PTR(-ENOMEM); 1022 1023 INIT_LIST_HEAD(&smmu->groups); 1024 mutex_init(&smmu->lock); 1025 1026 smmu->regs = mc->regs; 1027 smmu->soc = soc; 1028 smmu->dev = dev; 1029 smmu->mc = mc; 1030 1031 smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1; 1032 dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n", 1033 mc->soc->num_address_bits, smmu->pfn_mask); 1034 smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1; 1035 dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines, 1036 smmu->tlb_mask); 1037 1038 value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f); 1039 1040 if (soc->supports_request_limit) 1041 value |= SMMU_PTC_CONFIG_REQ_LIMIT(8); 1042 1043 smmu_writel(smmu, value, SMMU_PTC_CONFIG); 1044 1045 value = SMMU_TLB_CONFIG_HIT_UNDER_MISS | 1046 SMMU_TLB_CONFIG_ACTIVE_LINES(smmu); 1047 1048 if (soc->supports_round_robin_arbitration) 1049 value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION; 1050 1051 smmu_writel(smmu, value, SMMU_TLB_CONFIG); 1052 1053 smmu_flush_ptc_all(smmu); 1054 smmu_flush_tlb(smmu); 1055 smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); 1056 smmu_flush(smmu); 1057 1058 tegra_smmu_ahb_enable(); 1059 1060 err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev)); 1061 if (err) 1062 return ERR_PTR(err); 1063 1064 iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops); 1065 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); 1066 1067 err = iommu_device_register(&smmu->iommu); 1068 if (err) { 1069 iommu_device_sysfs_remove(&smmu->iommu); 1070 return ERR_PTR(err); 1071 } 1072 1073 err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops); 1074 if (err < 0) { 1075 iommu_device_unregister(&smmu->iommu); 1076 iommu_device_sysfs_remove(&smmu->iommu); 1077 return ERR_PTR(err); 1078 } 1079 1080 if (IS_ENABLED(CONFIG_DEBUG_FS)) 1081 tegra_smmu_debugfs_init(smmu); 1082 1083 return smmu; 1084 } 1085 1086 void tegra_smmu_remove(struct tegra_smmu *smmu) 1087 { 1088 iommu_device_unregister(&smmu->iommu); 1089 iommu_device_sysfs_remove(&smmu->iommu); 1090 1091 if (IS_ENABLED(CONFIG_DEBUG_FS)) 1092 tegra_smmu_debugfs_exit(smmu); 1093 } 1094