1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved. 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/debugfs.h> 8 #include <linux/err.h> 9 #include <linux/iommu.h> 10 #include <linux/kernel.h> 11 #include <linux/of.h> 12 #include <linux/of_device.h> 13 #include <linux/platform_device.h> 14 #include <linux/slab.h> 15 #include <linux/dma-mapping.h> 16 17 #include <soc/tegra/ahb.h> 18 #include <soc/tegra/mc.h> 19 20 struct tegra_smmu_group { 21 struct list_head list; 22 const struct tegra_smmu_group_soc *soc; 23 struct iommu_group *group; 24 }; 25 26 struct tegra_smmu { 27 void __iomem *regs; 28 struct device *dev; 29 30 struct tegra_mc *mc; 31 const struct tegra_smmu_soc *soc; 32 33 struct list_head groups; 34 35 unsigned long pfn_mask; 36 unsigned long tlb_mask; 37 38 unsigned long *asids; 39 struct mutex lock; 40 41 struct list_head list; 42 43 struct dentry *debugfs; 44 45 struct iommu_device iommu; /* IOMMU Core code handle */ 46 }; 47 48 struct tegra_smmu_as { 49 struct iommu_domain domain; 50 struct tegra_smmu *smmu; 51 unsigned int use_count; 52 u32 *count; 53 struct page **pts; 54 struct page *pd; 55 dma_addr_t pd_dma; 56 unsigned id; 57 u32 attr; 58 }; 59 60 static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom) 61 { 62 return container_of(dom, struct tegra_smmu_as, domain); 63 } 64 65 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value, 66 unsigned long offset) 67 { 68 writel(value, smmu->regs + offset); 69 } 70 71 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) 72 { 73 return readl(smmu->regs + offset); 74 } 75 76 #define SMMU_CONFIG 0x010 77 #define SMMU_CONFIG_ENABLE (1 << 0) 78 79 #define SMMU_TLB_CONFIG 0x14 80 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29) 81 #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28) 82 #define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \ 83 ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask) 84 85 #define SMMU_PTC_CONFIG 0x18 86 #define SMMU_PTC_CONFIG_ENABLE (1 << 29) 87 #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24) 88 #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f) 89 90 #define SMMU_PTB_ASID 0x01c 91 #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) 92 93 #define SMMU_PTB_DATA 0x020 94 #define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr)) 95 96 #define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr)) 97 98 #define SMMU_TLB_FLUSH 0x030 99 #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) 100 #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0) 101 #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0) 102 #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \ 103 SMMU_TLB_FLUSH_VA_MATCH_SECTION) 104 #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \ 105 SMMU_TLB_FLUSH_VA_MATCH_GROUP) 106 #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31) 107 108 #define SMMU_PTC_FLUSH 0x034 109 #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0) 110 #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0) 111 112 #define SMMU_PTC_FLUSH_HI 0x9b8 113 #define SMMU_PTC_FLUSH_HI_MASK 0x3 114 115 /* per-SWGROUP SMMU_*_ASID register */ 116 #define SMMU_ASID_ENABLE (1 << 31) 117 #define SMMU_ASID_MASK 0x7f 118 #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK) 119 120 /* page table definitions */ 121 #define SMMU_NUM_PDE 1024 122 #define SMMU_NUM_PTE 1024 123 124 #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4) 125 #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4) 126 127 #define SMMU_PDE_SHIFT 22 128 #define SMMU_PTE_SHIFT 12 129 130 #define SMMU_PD_READABLE (1 << 31) 131 #define SMMU_PD_WRITABLE (1 << 30) 132 #define SMMU_PD_NONSECURE (1 << 29) 133 134 #define SMMU_PDE_READABLE (1 << 31) 135 #define SMMU_PDE_WRITABLE (1 << 30) 136 #define SMMU_PDE_NONSECURE (1 << 29) 137 #define SMMU_PDE_NEXT (1 << 28) 138 139 #define SMMU_PTE_READABLE (1 << 31) 140 #define SMMU_PTE_WRITABLE (1 << 30) 141 #define SMMU_PTE_NONSECURE (1 << 29) 142 143 #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \ 144 SMMU_PDE_NONSECURE) 145 146 static unsigned int iova_pd_index(unsigned long iova) 147 { 148 return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1); 149 } 150 151 static unsigned int iova_pt_index(unsigned long iova) 152 { 153 return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); 154 } 155 156 static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr) 157 { 158 addr >>= 12; 159 return (addr & smmu->pfn_mask) == addr; 160 } 161 162 static dma_addr_t smmu_pde_to_dma(struct tegra_smmu *smmu, u32 pde) 163 { 164 return (dma_addr_t)(pde & smmu->pfn_mask) << 12; 165 } 166 167 static void smmu_flush_ptc_all(struct tegra_smmu *smmu) 168 { 169 smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH); 170 } 171 172 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma, 173 unsigned long offset) 174 { 175 u32 value; 176 177 offset &= ~(smmu->mc->soc->atom_size - 1); 178 179 if (smmu->mc->soc->num_address_bits > 32) { 180 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 181 value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK; 182 #else 183 value = 0; 184 #endif 185 smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); 186 } 187 188 value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR; 189 smmu_writel(smmu, value, SMMU_PTC_FLUSH); 190 } 191 192 static inline void smmu_flush_tlb(struct tegra_smmu *smmu) 193 { 194 smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH); 195 } 196 197 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu, 198 unsigned long asid) 199 { 200 u32 value; 201 202 if (smmu->soc->num_asids == 4) 203 value = (asid & 0x3) << 29; 204 else 205 value = (asid & 0x7f) << 24; 206 207 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL; 208 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 209 } 210 211 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu, 212 unsigned long asid, 213 unsigned long iova) 214 { 215 u32 value; 216 217 if (smmu->soc->num_asids == 4) 218 value = (asid & 0x3) << 29; 219 else 220 value = (asid & 0x7f) << 24; 221 222 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova); 223 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 224 } 225 226 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu, 227 unsigned long asid, 228 unsigned long iova) 229 { 230 u32 value; 231 232 if (smmu->soc->num_asids == 4) 233 value = (asid & 0x3) << 29; 234 else 235 value = (asid & 0x7f) << 24; 236 237 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova); 238 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 239 } 240 241 static inline void smmu_flush(struct tegra_smmu *smmu) 242 { 243 smmu_readl(smmu, SMMU_PTB_ASID); 244 } 245 246 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp) 247 { 248 unsigned long id; 249 250 mutex_lock(&smmu->lock); 251 252 id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids); 253 if (id >= smmu->soc->num_asids) { 254 mutex_unlock(&smmu->lock); 255 return -ENOSPC; 256 } 257 258 set_bit(id, smmu->asids); 259 *idp = id; 260 261 mutex_unlock(&smmu->lock); 262 return 0; 263 } 264 265 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id) 266 { 267 mutex_lock(&smmu->lock); 268 clear_bit(id, smmu->asids); 269 mutex_unlock(&smmu->lock); 270 } 271 272 static bool tegra_smmu_capable(enum iommu_cap cap) 273 { 274 return false; 275 } 276 277 static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) 278 { 279 struct tegra_smmu_as *as; 280 281 if (type != IOMMU_DOMAIN_UNMANAGED) 282 return NULL; 283 284 as = kzalloc(sizeof(*as), GFP_KERNEL); 285 if (!as) 286 return NULL; 287 288 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; 289 290 as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); 291 if (!as->pd) { 292 kfree(as); 293 return NULL; 294 } 295 296 as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); 297 if (!as->count) { 298 __free_page(as->pd); 299 kfree(as); 300 return NULL; 301 } 302 303 as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); 304 if (!as->pts) { 305 kfree(as->count); 306 __free_page(as->pd); 307 kfree(as); 308 return NULL; 309 } 310 311 /* setup aperture */ 312 as->domain.geometry.aperture_start = 0; 313 as->domain.geometry.aperture_end = 0xffffffff; 314 as->domain.geometry.force_aperture = true; 315 316 return &as->domain; 317 } 318 319 static void tegra_smmu_domain_free(struct iommu_domain *domain) 320 { 321 struct tegra_smmu_as *as = to_smmu_as(domain); 322 323 /* TODO: free page directory and page tables */ 324 325 WARN_ON_ONCE(as->use_count); 326 kfree(as->count); 327 kfree(as->pts); 328 kfree(as); 329 } 330 331 static const struct tegra_smmu_swgroup * 332 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup) 333 { 334 const struct tegra_smmu_swgroup *group = NULL; 335 unsigned int i; 336 337 for (i = 0; i < smmu->soc->num_swgroups; i++) { 338 if (smmu->soc->swgroups[i].swgroup == swgroup) { 339 group = &smmu->soc->swgroups[i]; 340 break; 341 } 342 } 343 344 return group; 345 } 346 347 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup, 348 unsigned int asid) 349 { 350 const struct tegra_smmu_swgroup *group; 351 unsigned int i; 352 u32 value; 353 354 group = tegra_smmu_find_swgroup(smmu, swgroup); 355 if (group) { 356 value = smmu_readl(smmu, group->reg); 357 value &= ~SMMU_ASID_MASK; 358 value |= SMMU_ASID_VALUE(asid); 359 value |= SMMU_ASID_ENABLE; 360 smmu_writel(smmu, value, group->reg); 361 } else { 362 pr_warn("%s group from swgroup %u not found\n", __func__, 363 swgroup); 364 /* No point moving ahead if group was not found */ 365 return; 366 } 367 368 for (i = 0; i < smmu->soc->num_clients; i++) { 369 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 370 371 if (client->swgroup != swgroup) 372 continue; 373 374 value = smmu_readl(smmu, client->smmu.reg); 375 value |= BIT(client->smmu.bit); 376 smmu_writel(smmu, value, client->smmu.reg); 377 } 378 } 379 380 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup, 381 unsigned int asid) 382 { 383 const struct tegra_smmu_swgroup *group; 384 unsigned int i; 385 u32 value; 386 387 group = tegra_smmu_find_swgroup(smmu, swgroup); 388 if (group) { 389 value = smmu_readl(smmu, group->reg); 390 value &= ~SMMU_ASID_MASK; 391 value |= SMMU_ASID_VALUE(asid); 392 value &= ~SMMU_ASID_ENABLE; 393 smmu_writel(smmu, value, group->reg); 394 } 395 396 for (i = 0; i < smmu->soc->num_clients; i++) { 397 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 398 399 if (client->swgroup != swgroup) 400 continue; 401 402 value = smmu_readl(smmu, client->smmu.reg); 403 value &= ~BIT(client->smmu.bit); 404 smmu_writel(smmu, value, client->smmu.reg); 405 } 406 } 407 408 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, 409 struct tegra_smmu_as *as) 410 { 411 u32 value; 412 int err; 413 414 if (as->use_count > 0) { 415 as->use_count++; 416 return 0; 417 } 418 419 as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD, 420 DMA_TO_DEVICE); 421 if (dma_mapping_error(smmu->dev, as->pd_dma)) 422 return -ENOMEM; 423 424 /* We can't handle 64-bit DMA addresses */ 425 if (!smmu_dma_addr_valid(smmu, as->pd_dma)) { 426 err = -ENOMEM; 427 goto err_unmap; 428 } 429 430 err = tegra_smmu_alloc_asid(smmu, &as->id); 431 if (err < 0) 432 goto err_unmap; 433 434 smmu_flush_ptc(smmu, as->pd_dma, 0); 435 smmu_flush_tlb_asid(smmu, as->id); 436 437 smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); 438 value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr); 439 smmu_writel(smmu, value, SMMU_PTB_DATA); 440 smmu_flush(smmu); 441 442 as->smmu = smmu; 443 as->use_count++; 444 445 return 0; 446 447 err_unmap: 448 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 449 return err; 450 } 451 452 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, 453 struct tegra_smmu_as *as) 454 { 455 if (--as->use_count > 0) 456 return; 457 458 tegra_smmu_free_asid(smmu, as->id); 459 460 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 461 462 as->smmu = NULL; 463 } 464 465 static int tegra_smmu_attach_dev(struct iommu_domain *domain, 466 struct device *dev) 467 { 468 struct tegra_smmu *smmu = dev_iommu_priv_get(dev); 469 struct tegra_smmu_as *as = to_smmu_as(domain); 470 struct device_node *np = dev->of_node; 471 struct of_phandle_args args; 472 unsigned int index = 0; 473 int err = 0; 474 475 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 476 &args)) { 477 unsigned int swgroup = args.args[0]; 478 479 if (args.np != smmu->dev->of_node) { 480 of_node_put(args.np); 481 continue; 482 } 483 484 of_node_put(args.np); 485 486 err = tegra_smmu_as_prepare(smmu, as); 487 if (err < 0) 488 return err; 489 490 tegra_smmu_enable(smmu, swgroup, as->id); 491 index++; 492 } 493 494 if (index == 0) 495 return -ENODEV; 496 497 return 0; 498 } 499 500 static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) 501 { 502 struct tegra_smmu_as *as = to_smmu_as(domain); 503 struct device_node *np = dev->of_node; 504 struct tegra_smmu *smmu = as->smmu; 505 struct of_phandle_args args; 506 unsigned int index = 0; 507 508 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 509 &args)) { 510 unsigned int swgroup = args.args[0]; 511 512 if (args.np != smmu->dev->of_node) { 513 of_node_put(args.np); 514 continue; 515 } 516 517 of_node_put(args.np); 518 519 tegra_smmu_disable(smmu, swgroup, as->id); 520 tegra_smmu_as_unprepare(smmu, as); 521 index++; 522 } 523 } 524 525 static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova, 526 u32 value) 527 { 528 unsigned int pd_index = iova_pd_index(iova); 529 struct tegra_smmu *smmu = as->smmu; 530 u32 *pd = page_address(as->pd); 531 unsigned long offset = pd_index * sizeof(*pd); 532 533 /* Set the page directory entry first */ 534 pd[pd_index] = value; 535 536 /* The flush the page directory entry from caches */ 537 dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, 538 sizeof(*pd), DMA_TO_DEVICE); 539 540 /* And flush the iommu */ 541 smmu_flush_ptc(smmu, as->pd_dma, offset); 542 smmu_flush_tlb_section(smmu, as->id, iova); 543 smmu_flush(smmu); 544 } 545 546 static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) 547 { 548 u32 *pt = page_address(pt_page); 549 550 return pt + iova_pt_index(iova); 551 } 552 553 static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, 554 dma_addr_t *dmap) 555 { 556 unsigned int pd_index = iova_pd_index(iova); 557 struct tegra_smmu *smmu = as->smmu; 558 struct page *pt_page; 559 u32 *pd; 560 561 pt_page = as->pts[pd_index]; 562 if (!pt_page) 563 return NULL; 564 565 pd = page_address(as->pd); 566 *dmap = smmu_pde_to_dma(smmu, pd[pd_index]); 567 568 return tegra_smmu_pte_offset(pt_page, iova); 569 } 570 571 static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, 572 dma_addr_t *dmap) 573 { 574 unsigned int pde = iova_pd_index(iova); 575 struct tegra_smmu *smmu = as->smmu; 576 577 if (!as->pts[pde]) { 578 struct page *page; 579 dma_addr_t dma; 580 581 page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); 582 if (!page) 583 return NULL; 584 585 dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, 586 DMA_TO_DEVICE); 587 if (dma_mapping_error(smmu->dev, dma)) { 588 __free_page(page); 589 return NULL; 590 } 591 592 if (!smmu_dma_addr_valid(smmu, dma)) { 593 dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, 594 DMA_TO_DEVICE); 595 __free_page(page); 596 return NULL; 597 } 598 599 as->pts[pde] = page; 600 601 tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR | 602 SMMU_PDE_NEXT)); 603 604 *dmap = dma; 605 } else { 606 u32 *pd = page_address(as->pd); 607 608 *dmap = smmu_pde_to_dma(smmu, pd[pde]); 609 } 610 611 return tegra_smmu_pte_offset(as->pts[pde], iova); 612 } 613 614 static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova) 615 { 616 unsigned int pd_index = iova_pd_index(iova); 617 618 as->count[pd_index]++; 619 } 620 621 static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) 622 { 623 unsigned int pde = iova_pd_index(iova); 624 struct page *page = as->pts[pde]; 625 626 /* 627 * When no entries in this page table are used anymore, return the 628 * memory page to the system. 629 */ 630 if (--as->count[pde] == 0) { 631 struct tegra_smmu *smmu = as->smmu; 632 u32 *pd = page_address(as->pd); 633 dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]); 634 635 tegra_smmu_set_pde(as, iova, 0); 636 637 dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); 638 __free_page(page); 639 as->pts[pde] = NULL; 640 } 641 } 642 643 static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, 644 u32 *pte, dma_addr_t pte_dma, u32 val) 645 { 646 struct tegra_smmu *smmu = as->smmu; 647 unsigned long offset = offset_in_page(pte); 648 649 *pte = val; 650 651 dma_sync_single_range_for_device(smmu->dev, pte_dma, offset, 652 4, DMA_TO_DEVICE); 653 smmu_flush_ptc(smmu, pte_dma, offset); 654 smmu_flush_tlb_group(smmu, as->id, iova); 655 smmu_flush(smmu); 656 } 657 658 static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, 659 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 660 { 661 struct tegra_smmu_as *as = to_smmu_as(domain); 662 dma_addr_t pte_dma; 663 u32 pte_attrs; 664 u32 *pte; 665 666 pte = as_get_pte(as, iova, &pte_dma); 667 if (!pte) 668 return -ENOMEM; 669 670 /* If we aren't overwriting a pre-existing entry, increment use */ 671 if (*pte == 0) 672 tegra_smmu_pte_get_use(as, iova); 673 674 pte_attrs = SMMU_PTE_NONSECURE; 675 676 if (prot & IOMMU_READ) 677 pte_attrs |= SMMU_PTE_READABLE; 678 679 if (prot & IOMMU_WRITE) 680 pte_attrs |= SMMU_PTE_WRITABLE; 681 682 tegra_smmu_set_pte(as, iova, pte, pte_dma, 683 __phys_to_pfn(paddr) | pte_attrs); 684 685 return 0; 686 } 687 688 static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 689 size_t size, struct iommu_iotlb_gather *gather) 690 { 691 struct tegra_smmu_as *as = to_smmu_as(domain); 692 dma_addr_t pte_dma; 693 u32 *pte; 694 695 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); 696 if (!pte || !*pte) 697 return 0; 698 699 tegra_smmu_set_pte(as, iova, pte, pte_dma, 0); 700 tegra_smmu_pte_put_use(as, iova); 701 702 return size; 703 } 704 705 static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, 706 dma_addr_t iova) 707 { 708 struct tegra_smmu_as *as = to_smmu_as(domain); 709 unsigned long pfn; 710 dma_addr_t pte_dma; 711 u32 *pte; 712 713 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); 714 if (!pte || !*pte) 715 return 0; 716 717 pfn = *pte & as->smmu->pfn_mask; 718 719 return PFN_PHYS(pfn); 720 } 721 722 static struct tegra_smmu *tegra_smmu_find(struct device_node *np) 723 { 724 struct platform_device *pdev; 725 struct tegra_mc *mc; 726 727 pdev = of_find_device_by_node(np); 728 if (!pdev) 729 return NULL; 730 731 mc = platform_get_drvdata(pdev); 732 if (!mc) 733 return NULL; 734 735 return mc->smmu; 736 } 737 738 static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev, 739 struct of_phandle_args *args) 740 { 741 const struct iommu_ops *ops = smmu->iommu.ops; 742 int err; 743 744 err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops); 745 if (err < 0) { 746 dev_err(dev, "failed to initialize fwspec: %d\n", err); 747 return err; 748 } 749 750 err = ops->of_xlate(dev, args); 751 if (err < 0) { 752 dev_err(dev, "failed to parse SW group ID: %d\n", err); 753 iommu_fwspec_free(dev); 754 return err; 755 } 756 757 return 0; 758 } 759 760 static struct iommu_device *tegra_smmu_probe_device(struct device *dev) 761 { 762 struct device_node *np = dev->of_node; 763 struct tegra_smmu *smmu = NULL; 764 struct of_phandle_args args; 765 unsigned int index = 0; 766 int err; 767 768 while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 769 &args) == 0) { 770 smmu = tegra_smmu_find(args.np); 771 if (smmu) { 772 err = tegra_smmu_configure(smmu, dev, &args); 773 of_node_put(args.np); 774 775 if (err < 0) 776 return ERR_PTR(err); 777 778 /* 779 * Only a single IOMMU master interface is currently 780 * supported by the Linux kernel, so abort after the 781 * first match. 782 */ 783 dev_iommu_priv_set(dev, smmu); 784 785 break; 786 } 787 788 of_node_put(args.np); 789 index++; 790 } 791 792 if (!smmu) 793 return ERR_PTR(-ENODEV); 794 795 return &smmu->iommu; 796 } 797 798 static void tegra_smmu_release_device(struct device *dev) 799 { 800 dev_iommu_priv_set(dev, NULL); 801 } 802 803 static const struct tegra_smmu_group_soc * 804 tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup) 805 { 806 unsigned int i, j; 807 808 for (i = 0; i < smmu->soc->num_groups; i++) 809 for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++) 810 if (smmu->soc->groups[i].swgroups[j] == swgroup) 811 return &smmu->soc->groups[i]; 812 813 return NULL; 814 } 815 816 static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu, 817 unsigned int swgroup) 818 { 819 const struct tegra_smmu_group_soc *soc; 820 struct tegra_smmu_group *group; 821 822 soc = tegra_smmu_find_group(smmu, swgroup); 823 if (!soc) 824 return NULL; 825 826 mutex_lock(&smmu->lock); 827 828 list_for_each_entry(group, &smmu->groups, list) 829 if (group->soc == soc) { 830 mutex_unlock(&smmu->lock); 831 return group->group; 832 } 833 834 group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL); 835 if (!group) { 836 mutex_unlock(&smmu->lock); 837 return NULL; 838 } 839 840 INIT_LIST_HEAD(&group->list); 841 group->soc = soc; 842 843 group->group = iommu_group_alloc(); 844 if (IS_ERR(group->group)) { 845 devm_kfree(smmu->dev, group); 846 mutex_unlock(&smmu->lock); 847 return NULL; 848 } 849 850 list_add_tail(&group->list, &smmu->groups); 851 mutex_unlock(&smmu->lock); 852 853 return group->group; 854 } 855 856 static struct iommu_group *tegra_smmu_device_group(struct device *dev) 857 { 858 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 859 struct tegra_smmu *smmu = dev_iommu_priv_get(dev); 860 struct iommu_group *group; 861 862 group = tegra_smmu_group_get(smmu, fwspec->ids[0]); 863 if (!group) 864 group = generic_device_group(dev); 865 866 return group; 867 } 868 869 static int tegra_smmu_of_xlate(struct device *dev, 870 struct of_phandle_args *args) 871 { 872 u32 id = args->args[0]; 873 874 return iommu_fwspec_add_ids(dev, &id, 1); 875 } 876 877 static const struct iommu_ops tegra_smmu_ops = { 878 .capable = tegra_smmu_capable, 879 .domain_alloc = tegra_smmu_domain_alloc, 880 .domain_free = tegra_smmu_domain_free, 881 .attach_dev = tegra_smmu_attach_dev, 882 .detach_dev = tegra_smmu_detach_dev, 883 .probe_device = tegra_smmu_probe_device, 884 .release_device = tegra_smmu_release_device, 885 .device_group = tegra_smmu_device_group, 886 .map = tegra_smmu_map, 887 .unmap = tegra_smmu_unmap, 888 .iova_to_phys = tegra_smmu_iova_to_phys, 889 .of_xlate = tegra_smmu_of_xlate, 890 .pgsize_bitmap = SZ_4K, 891 }; 892 893 static void tegra_smmu_ahb_enable(void) 894 { 895 static const struct of_device_id ahb_match[] = { 896 { .compatible = "nvidia,tegra30-ahb", }, 897 { } 898 }; 899 struct device_node *ahb; 900 901 ahb = of_find_matching_node(NULL, ahb_match); 902 if (ahb) { 903 tegra_ahb_enable_smmu(ahb); 904 of_node_put(ahb); 905 } 906 } 907 908 static int tegra_smmu_swgroups_show(struct seq_file *s, void *data) 909 { 910 struct tegra_smmu *smmu = s->private; 911 unsigned int i; 912 u32 value; 913 914 seq_printf(s, "swgroup enabled ASID\n"); 915 seq_printf(s, "------------------------\n"); 916 917 for (i = 0; i < smmu->soc->num_swgroups; i++) { 918 const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i]; 919 const char *status; 920 unsigned int asid; 921 922 value = smmu_readl(smmu, group->reg); 923 924 if (value & SMMU_ASID_ENABLE) 925 status = "yes"; 926 else 927 status = "no"; 928 929 asid = value & SMMU_ASID_MASK; 930 931 seq_printf(s, "%-9s %-7s %#04x\n", group->name, status, 932 asid); 933 } 934 935 return 0; 936 } 937 938 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups); 939 940 static int tegra_smmu_clients_show(struct seq_file *s, void *data) 941 { 942 struct tegra_smmu *smmu = s->private; 943 unsigned int i; 944 u32 value; 945 946 seq_printf(s, "client enabled\n"); 947 seq_printf(s, "--------------------\n"); 948 949 for (i = 0; i < smmu->soc->num_clients; i++) { 950 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 951 const char *status; 952 953 value = smmu_readl(smmu, client->smmu.reg); 954 955 if (value & BIT(client->smmu.bit)) 956 status = "yes"; 957 else 958 status = "no"; 959 960 seq_printf(s, "%-12s %s\n", client->name, status); 961 } 962 963 return 0; 964 } 965 966 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients); 967 968 static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu) 969 { 970 smmu->debugfs = debugfs_create_dir("smmu", NULL); 971 if (!smmu->debugfs) 972 return; 973 974 debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu, 975 &tegra_smmu_swgroups_fops); 976 debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu, 977 &tegra_smmu_clients_fops); 978 } 979 980 static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu) 981 { 982 debugfs_remove_recursive(smmu->debugfs); 983 } 984 985 struct tegra_smmu *tegra_smmu_probe(struct device *dev, 986 const struct tegra_smmu_soc *soc, 987 struct tegra_mc *mc) 988 { 989 struct tegra_smmu *smmu; 990 size_t size; 991 u32 value; 992 int err; 993 994 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 995 if (!smmu) 996 return ERR_PTR(-ENOMEM); 997 998 /* 999 * This is a bit of a hack. Ideally we'd want to simply return this 1000 * value. However the IOMMU registration process will attempt to add 1001 * all devices to the IOMMU when bus_set_iommu() is called. In order 1002 * not to rely on global variables to track the IOMMU instance, we 1003 * set it here so that it can be looked up from the .probe_device() 1004 * callback via the IOMMU device's .drvdata field. 1005 */ 1006 mc->smmu = smmu; 1007 1008 size = BITS_TO_LONGS(soc->num_asids) * sizeof(long); 1009 1010 smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL); 1011 if (!smmu->asids) 1012 return ERR_PTR(-ENOMEM); 1013 1014 INIT_LIST_HEAD(&smmu->groups); 1015 mutex_init(&smmu->lock); 1016 1017 smmu->regs = mc->regs; 1018 smmu->soc = soc; 1019 smmu->dev = dev; 1020 smmu->mc = mc; 1021 1022 smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1; 1023 dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n", 1024 mc->soc->num_address_bits, smmu->pfn_mask); 1025 smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1; 1026 dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines, 1027 smmu->tlb_mask); 1028 1029 value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f); 1030 1031 if (soc->supports_request_limit) 1032 value |= SMMU_PTC_CONFIG_REQ_LIMIT(8); 1033 1034 smmu_writel(smmu, value, SMMU_PTC_CONFIG); 1035 1036 value = SMMU_TLB_CONFIG_HIT_UNDER_MISS | 1037 SMMU_TLB_CONFIG_ACTIVE_LINES(smmu); 1038 1039 if (soc->supports_round_robin_arbitration) 1040 value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION; 1041 1042 smmu_writel(smmu, value, SMMU_TLB_CONFIG); 1043 1044 smmu_flush_ptc_all(smmu); 1045 smmu_flush_tlb(smmu); 1046 smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); 1047 smmu_flush(smmu); 1048 1049 tegra_smmu_ahb_enable(); 1050 1051 err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev)); 1052 if (err) 1053 return ERR_PTR(err); 1054 1055 iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops); 1056 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); 1057 1058 err = iommu_device_register(&smmu->iommu); 1059 if (err) { 1060 iommu_device_sysfs_remove(&smmu->iommu); 1061 return ERR_PTR(err); 1062 } 1063 1064 err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops); 1065 if (err < 0) { 1066 iommu_device_unregister(&smmu->iommu); 1067 iommu_device_sysfs_remove(&smmu->iommu); 1068 return ERR_PTR(err); 1069 } 1070 1071 if (IS_ENABLED(CONFIG_DEBUG_FS)) 1072 tegra_smmu_debugfs_init(smmu); 1073 1074 return smmu; 1075 } 1076 1077 void tegra_smmu_remove(struct tegra_smmu *smmu) 1078 { 1079 iommu_device_unregister(&smmu->iommu); 1080 iommu_device_sysfs_remove(&smmu->iommu); 1081 1082 if (IS_ENABLED(CONFIG_DEBUG_FS)) 1083 tegra_smmu_debugfs_exit(smmu); 1084 } 1085