1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved. 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/debugfs.h> 8 #include <linux/err.h> 9 #include <linux/iommu.h> 10 #include <linux/kernel.h> 11 #include <linux/of.h> 12 #include <linux/of_device.h> 13 #include <linux/platform_device.h> 14 #include <linux/slab.h> 15 #include <linux/dma-mapping.h> 16 17 #include <soc/tegra/ahb.h> 18 #include <soc/tegra/mc.h> 19 20 struct tegra_smmu_group { 21 struct list_head list; 22 const struct tegra_smmu_group_soc *soc; 23 struct iommu_group *group; 24 }; 25 26 struct tegra_smmu { 27 void __iomem *regs; 28 struct device *dev; 29 30 struct tegra_mc *mc; 31 const struct tegra_smmu_soc *soc; 32 33 struct list_head groups; 34 35 unsigned long pfn_mask; 36 unsigned long tlb_mask; 37 38 unsigned long *asids; 39 struct mutex lock; 40 41 struct list_head list; 42 43 struct dentry *debugfs; 44 45 struct iommu_device iommu; /* IOMMU Core code handle */ 46 }; 47 48 struct tegra_smmu_as { 49 struct iommu_domain domain; 50 struct tegra_smmu *smmu; 51 unsigned int use_count; 52 u32 *count; 53 struct page **pts; 54 struct page *pd; 55 dma_addr_t pd_dma; 56 unsigned id; 57 u32 attr; 58 }; 59 60 static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom) 61 { 62 return container_of(dom, struct tegra_smmu_as, domain); 63 } 64 65 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value, 66 unsigned long offset) 67 { 68 writel(value, smmu->regs + offset); 69 } 70 71 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) 72 { 73 return readl(smmu->regs + offset); 74 } 75 76 #define SMMU_CONFIG 0x010 77 #define SMMU_CONFIG_ENABLE (1 << 0) 78 79 #define SMMU_TLB_CONFIG 0x14 80 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29) 81 #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28) 82 #define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \ 83 ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask) 84 85 #define SMMU_PTC_CONFIG 0x18 86 #define SMMU_PTC_CONFIG_ENABLE (1 << 29) 87 #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24) 88 #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f) 89 90 #define SMMU_PTB_ASID 0x01c 91 #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) 92 93 #define SMMU_PTB_DATA 0x020 94 #define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr)) 95 96 #define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr)) 97 98 #define SMMU_TLB_FLUSH 0x030 99 #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) 100 #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0) 101 #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0) 102 #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \ 103 SMMU_TLB_FLUSH_VA_MATCH_SECTION) 104 #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \ 105 SMMU_TLB_FLUSH_VA_MATCH_GROUP) 106 #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31) 107 108 #define SMMU_PTC_FLUSH 0x034 109 #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0) 110 #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0) 111 112 #define SMMU_PTC_FLUSH_HI 0x9b8 113 #define SMMU_PTC_FLUSH_HI_MASK 0x3 114 115 /* per-SWGROUP SMMU_*_ASID register */ 116 #define SMMU_ASID_ENABLE (1 << 31) 117 #define SMMU_ASID_MASK 0x7f 118 #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK) 119 120 /* page table definitions */ 121 #define SMMU_NUM_PDE 1024 122 #define SMMU_NUM_PTE 1024 123 124 #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4) 125 #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4) 126 127 #define SMMU_PDE_SHIFT 22 128 #define SMMU_PTE_SHIFT 12 129 130 #define SMMU_PD_READABLE (1 << 31) 131 #define SMMU_PD_WRITABLE (1 << 30) 132 #define SMMU_PD_NONSECURE (1 << 29) 133 134 #define SMMU_PDE_READABLE (1 << 31) 135 #define SMMU_PDE_WRITABLE (1 << 30) 136 #define SMMU_PDE_NONSECURE (1 << 29) 137 #define SMMU_PDE_NEXT (1 << 28) 138 139 #define SMMU_PTE_READABLE (1 << 31) 140 #define SMMU_PTE_WRITABLE (1 << 30) 141 #define SMMU_PTE_NONSECURE (1 << 29) 142 143 #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \ 144 SMMU_PDE_NONSECURE) 145 146 static unsigned int iova_pd_index(unsigned long iova) 147 { 148 return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1); 149 } 150 151 static unsigned int iova_pt_index(unsigned long iova) 152 { 153 return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); 154 } 155 156 static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr) 157 { 158 addr >>= 12; 159 return (addr & smmu->pfn_mask) == addr; 160 } 161 162 static dma_addr_t smmu_pde_to_dma(struct tegra_smmu *smmu, u32 pde) 163 { 164 return (dma_addr_t)(pde & smmu->pfn_mask) << 12; 165 } 166 167 static void smmu_flush_ptc_all(struct tegra_smmu *smmu) 168 { 169 smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH); 170 } 171 172 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma, 173 unsigned long offset) 174 { 175 u32 value; 176 177 offset &= ~(smmu->mc->soc->atom_size - 1); 178 179 if (smmu->mc->soc->num_address_bits > 32) { 180 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 181 value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK; 182 #else 183 value = 0; 184 #endif 185 smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); 186 } 187 188 value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR; 189 smmu_writel(smmu, value, SMMU_PTC_FLUSH); 190 } 191 192 static inline void smmu_flush_tlb(struct tegra_smmu *smmu) 193 { 194 smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH); 195 } 196 197 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu, 198 unsigned long asid) 199 { 200 u32 value; 201 202 if (smmu->soc->num_asids == 4) 203 value = (asid & 0x3) << 29; 204 else 205 value = (asid & 0x7f) << 24; 206 207 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL; 208 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 209 } 210 211 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu, 212 unsigned long asid, 213 unsigned long iova) 214 { 215 u32 value; 216 217 if (smmu->soc->num_asids == 4) 218 value = (asid & 0x3) << 29; 219 else 220 value = (asid & 0x7f) << 24; 221 222 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova); 223 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 224 } 225 226 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu, 227 unsigned long asid, 228 unsigned long iova) 229 { 230 u32 value; 231 232 if (smmu->soc->num_asids == 4) 233 value = (asid & 0x3) << 29; 234 else 235 value = (asid & 0x7f) << 24; 236 237 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova); 238 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 239 } 240 241 static inline void smmu_flush(struct tegra_smmu *smmu) 242 { 243 smmu_readl(smmu, SMMU_PTB_ASID); 244 } 245 246 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp) 247 { 248 unsigned long id; 249 250 mutex_lock(&smmu->lock); 251 252 id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids); 253 if (id >= smmu->soc->num_asids) { 254 mutex_unlock(&smmu->lock); 255 return -ENOSPC; 256 } 257 258 set_bit(id, smmu->asids); 259 *idp = id; 260 261 mutex_unlock(&smmu->lock); 262 return 0; 263 } 264 265 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id) 266 { 267 mutex_lock(&smmu->lock); 268 clear_bit(id, smmu->asids); 269 mutex_unlock(&smmu->lock); 270 } 271 272 static bool tegra_smmu_capable(enum iommu_cap cap) 273 { 274 return false; 275 } 276 277 static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) 278 { 279 struct tegra_smmu_as *as; 280 281 if (type != IOMMU_DOMAIN_UNMANAGED) 282 return NULL; 283 284 as = kzalloc(sizeof(*as), GFP_KERNEL); 285 if (!as) 286 return NULL; 287 288 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; 289 290 as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); 291 if (!as->pd) { 292 kfree(as); 293 return NULL; 294 } 295 296 as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); 297 if (!as->count) { 298 __free_page(as->pd); 299 kfree(as); 300 return NULL; 301 } 302 303 as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); 304 if (!as->pts) { 305 kfree(as->count); 306 __free_page(as->pd); 307 kfree(as); 308 return NULL; 309 } 310 311 /* setup aperture */ 312 as->domain.geometry.aperture_start = 0; 313 as->domain.geometry.aperture_end = 0xffffffff; 314 as->domain.geometry.force_aperture = true; 315 316 return &as->domain; 317 } 318 319 static void tegra_smmu_domain_free(struct iommu_domain *domain) 320 { 321 struct tegra_smmu_as *as = to_smmu_as(domain); 322 323 /* TODO: free page directory and page tables */ 324 325 WARN_ON_ONCE(as->use_count); 326 kfree(as->count); 327 kfree(as->pts); 328 kfree(as); 329 } 330 331 static const struct tegra_smmu_swgroup * 332 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup) 333 { 334 const struct tegra_smmu_swgroup *group = NULL; 335 unsigned int i; 336 337 for (i = 0; i < smmu->soc->num_swgroups; i++) { 338 if (smmu->soc->swgroups[i].swgroup == swgroup) { 339 group = &smmu->soc->swgroups[i]; 340 break; 341 } 342 } 343 344 return group; 345 } 346 347 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup, 348 unsigned int asid) 349 { 350 const struct tegra_smmu_swgroup *group; 351 unsigned int i; 352 u32 value; 353 354 group = tegra_smmu_find_swgroup(smmu, swgroup); 355 if (group) { 356 value = smmu_readl(smmu, group->reg); 357 value &= ~SMMU_ASID_MASK; 358 value |= SMMU_ASID_VALUE(asid); 359 value |= SMMU_ASID_ENABLE; 360 smmu_writel(smmu, value, group->reg); 361 } else { 362 pr_warn("%s group from swgroup %u not found\n", __func__, 363 swgroup); 364 /* No point moving ahead if group was not found */ 365 return; 366 } 367 368 for (i = 0; i < smmu->soc->num_clients; i++) { 369 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 370 371 if (client->swgroup != swgroup) 372 continue; 373 374 value = smmu_readl(smmu, client->smmu.reg); 375 value |= BIT(client->smmu.bit); 376 smmu_writel(smmu, value, client->smmu.reg); 377 } 378 } 379 380 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup, 381 unsigned int asid) 382 { 383 const struct tegra_smmu_swgroup *group; 384 unsigned int i; 385 u32 value; 386 387 group = tegra_smmu_find_swgroup(smmu, swgroup); 388 if (group) { 389 value = smmu_readl(smmu, group->reg); 390 value &= ~SMMU_ASID_MASK; 391 value |= SMMU_ASID_VALUE(asid); 392 value &= ~SMMU_ASID_ENABLE; 393 smmu_writel(smmu, value, group->reg); 394 } 395 396 for (i = 0; i < smmu->soc->num_clients; i++) { 397 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 398 399 if (client->swgroup != swgroup) 400 continue; 401 402 value = smmu_readl(smmu, client->smmu.reg); 403 value &= ~BIT(client->smmu.bit); 404 smmu_writel(smmu, value, client->smmu.reg); 405 } 406 } 407 408 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, 409 struct tegra_smmu_as *as) 410 { 411 u32 value; 412 int err; 413 414 if (as->use_count > 0) { 415 as->use_count++; 416 return 0; 417 } 418 419 as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD, 420 DMA_TO_DEVICE); 421 if (dma_mapping_error(smmu->dev, as->pd_dma)) 422 return -ENOMEM; 423 424 /* We can't handle 64-bit DMA addresses */ 425 if (!smmu_dma_addr_valid(smmu, as->pd_dma)) { 426 err = -ENOMEM; 427 goto err_unmap; 428 } 429 430 err = tegra_smmu_alloc_asid(smmu, &as->id); 431 if (err < 0) 432 goto err_unmap; 433 434 smmu_flush_ptc(smmu, as->pd_dma, 0); 435 smmu_flush_tlb_asid(smmu, as->id); 436 437 smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); 438 value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr); 439 smmu_writel(smmu, value, SMMU_PTB_DATA); 440 smmu_flush(smmu); 441 442 as->smmu = smmu; 443 as->use_count++; 444 445 return 0; 446 447 err_unmap: 448 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 449 return err; 450 } 451 452 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, 453 struct tegra_smmu_as *as) 454 { 455 if (--as->use_count > 0) 456 return; 457 458 tegra_smmu_free_asid(smmu, as->id); 459 460 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 461 462 as->smmu = NULL; 463 } 464 465 static int tegra_smmu_attach_dev(struct iommu_domain *domain, 466 struct device *dev) 467 { 468 struct tegra_smmu *smmu = dev->archdata.iommu; 469 struct tegra_smmu_as *as = to_smmu_as(domain); 470 struct device_node *np = dev->of_node; 471 struct of_phandle_args args; 472 unsigned int index = 0; 473 int err = 0; 474 475 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 476 &args)) { 477 unsigned int swgroup = args.args[0]; 478 479 if (args.np != smmu->dev->of_node) { 480 of_node_put(args.np); 481 continue; 482 } 483 484 of_node_put(args.np); 485 486 err = tegra_smmu_as_prepare(smmu, as); 487 if (err < 0) 488 return err; 489 490 tegra_smmu_enable(smmu, swgroup, as->id); 491 index++; 492 } 493 494 if (index == 0) 495 return -ENODEV; 496 497 return 0; 498 } 499 500 static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) 501 { 502 struct tegra_smmu_as *as = to_smmu_as(domain); 503 struct device_node *np = dev->of_node; 504 struct tegra_smmu *smmu = as->smmu; 505 struct of_phandle_args args; 506 unsigned int index = 0; 507 508 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 509 &args)) { 510 unsigned int swgroup = args.args[0]; 511 512 if (args.np != smmu->dev->of_node) { 513 of_node_put(args.np); 514 continue; 515 } 516 517 of_node_put(args.np); 518 519 tegra_smmu_disable(smmu, swgroup, as->id); 520 tegra_smmu_as_unprepare(smmu, as); 521 index++; 522 } 523 } 524 525 static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova, 526 u32 value) 527 { 528 unsigned int pd_index = iova_pd_index(iova); 529 struct tegra_smmu *smmu = as->smmu; 530 u32 *pd = page_address(as->pd); 531 unsigned long offset = pd_index * sizeof(*pd); 532 533 /* Set the page directory entry first */ 534 pd[pd_index] = value; 535 536 /* The flush the page directory entry from caches */ 537 dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, 538 sizeof(*pd), DMA_TO_DEVICE); 539 540 /* And flush the iommu */ 541 smmu_flush_ptc(smmu, as->pd_dma, offset); 542 smmu_flush_tlb_section(smmu, as->id, iova); 543 smmu_flush(smmu); 544 } 545 546 static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) 547 { 548 u32 *pt = page_address(pt_page); 549 550 return pt + iova_pt_index(iova); 551 } 552 553 static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, 554 dma_addr_t *dmap) 555 { 556 unsigned int pd_index = iova_pd_index(iova); 557 struct tegra_smmu *smmu = as->smmu; 558 struct page *pt_page; 559 u32 *pd; 560 561 pt_page = as->pts[pd_index]; 562 if (!pt_page) 563 return NULL; 564 565 pd = page_address(as->pd); 566 *dmap = smmu_pde_to_dma(smmu, pd[pd_index]); 567 568 return tegra_smmu_pte_offset(pt_page, iova); 569 } 570 571 static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, 572 dma_addr_t *dmap) 573 { 574 unsigned int pde = iova_pd_index(iova); 575 struct tegra_smmu *smmu = as->smmu; 576 577 if (!as->pts[pde]) { 578 struct page *page; 579 dma_addr_t dma; 580 581 page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); 582 if (!page) 583 return NULL; 584 585 dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, 586 DMA_TO_DEVICE); 587 if (dma_mapping_error(smmu->dev, dma)) { 588 __free_page(page); 589 return NULL; 590 } 591 592 if (!smmu_dma_addr_valid(smmu, dma)) { 593 dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, 594 DMA_TO_DEVICE); 595 __free_page(page); 596 return NULL; 597 } 598 599 as->pts[pde] = page; 600 601 tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR | 602 SMMU_PDE_NEXT)); 603 604 *dmap = dma; 605 } else { 606 u32 *pd = page_address(as->pd); 607 608 *dmap = smmu_pde_to_dma(smmu, pd[pde]); 609 } 610 611 return tegra_smmu_pte_offset(as->pts[pde], iova); 612 } 613 614 static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova) 615 { 616 unsigned int pd_index = iova_pd_index(iova); 617 618 as->count[pd_index]++; 619 } 620 621 static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) 622 { 623 unsigned int pde = iova_pd_index(iova); 624 struct page *page = as->pts[pde]; 625 626 /* 627 * When no entries in this page table are used anymore, return the 628 * memory page to the system. 629 */ 630 if (--as->count[pde] == 0) { 631 struct tegra_smmu *smmu = as->smmu; 632 u32 *pd = page_address(as->pd); 633 dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]); 634 635 tegra_smmu_set_pde(as, iova, 0); 636 637 dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); 638 __free_page(page); 639 as->pts[pde] = NULL; 640 } 641 } 642 643 static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, 644 u32 *pte, dma_addr_t pte_dma, u32 val) 645 { 646 struct tegra_smmu *smmu = as->smmu; 647 unsigned long offset = offset_in_page(pte); 648 649 *pte = val; 650 651 dma_sync_single_range_for_device(smmu->dev, pte_dma, offset, 652 4, DMA_TO_DEVICE); 653 smmu_flush_ptc(smmu, pte_dma, offset); 654 smmu_flush_tlb_group(smmu, as->id, iova); 655 smmu_flush(smmu); 656 } 657 658 static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, 659 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 660 { 661 struct tegra_smmu_as *as = to_smmu_as(domain); 662 dma_addr_t pte_dma; 663 u32 pte_attrs; 664 u32 *pte; 665 666 pte = as_get_pte(as, iova, &pte_dma); 667 if (!pte) 668 return -ENOMEM; 669 670 /* If we aren't overwriting a pre-existing entry, increment use */ 671 if (*pte == 0) 672 tegra_smmu_pte_get_use(as, iova); 673 674 pte_attrs = SMMU_PTE_NONSECURE; 675 676 if (prot & IOMMU_READ) 677 pte_attrs |= SMMU_PTE_READABLE; 678 679 if (prot & IOMMU_WRITE) 680 pte_attrs |= SMMU_PTE_WRITABLE; 681 682 tegra_smmu_set_pte(as, iova, pte, pte_dma, 683 __phys_to_pfn(paddr) | pte_attrs); 684 685 return 0; 686 } 687 688 static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 689 size_t size, struct iommu_iotlb_gather *gather) 690 { 691 struct tegra_smmu_as *as = to_smmu_as(domain); 692 dma_addr_t pte_dma; 693 u32 *pte; 694 695 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); 696 if (!pte || !*pte) 697 return 0; 698 699 tegra_smmu_set_pte(as, iova, pte, pte_dma, 0); 700 tegra_smmu_pte_put_use(as, iova); 701 702 return size; 703 } 704 705 static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, 706 dma_addr_t iova) 707 { 708 struct tegra_smmu_as *as = to_smmu_as(domain); 709 unsigned long pfn; 710 dma_addr_t pte_dma; 711 u32 *pte; 712 713 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); 714 if (!pte || !*pte) 715 return 0; 716 717 pfn = *pte & as->smmu->pfn_mask; 718 719 return PFN_PHYS(pfn); 720 } 721 722 static struct tegra_smmu *tegra_smmu_find(struct device_node *np) 723 { 724 struct platform_device *pdev; 725 struct tegra_mc *mc; 726 727 pdev = of_find_device_by_node(np); 728 if (!pdev) 729 return NULL; 730 731 mc = platform_get_drvdata(pdev); 732 if (!mc) 733 return NULL; 734 735 return mc->smmu; 736 } 737 738 static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev, 739 struct of_phandle_args *args) 740 { 741 const struct iommu_ops *ops = smmu->iommu.ops; 742 int err; 743 744 err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops); 745 if (err < 0) { 746 dev_err(dev, "failed to initialize fwspec: %d\n", err); 747 return err; 748 } 749 750 err = ops->of_xlate(dev, args); 751 if (err < 0) { 752 dev_err(dev, "failed to parse SW group ID: %d\n", err); 753 iommu_fwspec_free(dev); 754 return err; 755 } 756 757 return 0; 758 } 759 760 static int tegra_smmu_add_device(struct device *dev) 761 { 762 struct device_node *np = dev->of_node; 763 struct tegra_smmu *smmu = NULL; 764 struct iommu_group *group; 765 struct of_phandle_args args; 766 unsigned int index = 0; 767 int err; 768 769 while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 770 &args) == 0) { 771 smmu = tegra_smmu_find(args.np); 772 if (smmu) { 773 err = tegra_smmu_configure(smmu, dev, &args); 774 of_node_put(args.np); 775 776 if (err < 0) 777 return err; 778 779 /* 780 * Only a single IOMMU master interface is currently 781 * supported by the Linux kernel, so abort after the 782 * first match. 783 */ 784 dev->archdata.iommu = smmu; 785 786 iommu_device_link(&smmu->iommu, dev); 787 788 break; 789 } 790 791 of_node_put(args.np); 792 index++; 793 } 794 795 if (!smmu) 796 return -ENODEV; 797 798 group = iommu_group_get_for_dev(dev); 799 if (IS_ERR(group)) 800 return PTR_ERR(group); 801 802 iommu_group_put(group); 803 804 return 0; 805 } 806 807 static void tegra_smmu_remove_device(struct device *dev) 808 { 809 struct tegra_smmu *smmu = dev->archdata.iommu; 810 811 if (smmu) 812 iommu_device_unlink(&smmu->iommu, dev); 813 814 dev->archdata.iommu = NULL; 815 iommu_group_remove_device(dev); 816 } 817 818 static const struct tegra_smmu_group_soc * 819 tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup) 820 { 821 unsigned int i, j; 822 823 for (i = 0; i < smmu->soc->num_groups; i++) 824 for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++) 825 if (smmu->soc->groups[i].swgroups[j] == swgroup) 826 return &smmu->soc->groups[i]; 827 828 return NULL; 829 } 830 831 static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu, 832 unsigned int swgroup) 833 { 834 const struct tegra_smmu_group_soc *soc; 835 struct tegra_smmu_group *group; 836 837 soc = tegra_smmu_find_group(smmu, swgroup); 838 if (!soc) 839 return NULL; 840 841 mutex_lock(&smmu->lock); 842 843 list_for_each_entry(group, &smmu->groups, list) 844 if (group->soc == soc) { 845 mutex_unlock(&smmu->lock); 846 return group->group; 847 } 848 849 group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL); 850 if (!group) { 851 mutex_unlock(&smmu->lock); 852 return NULL; 853 } 854 855 INIT_LIST_HEAD(&group->list); 856 group->soc = soc; 857 858 group->group = iommu_group_alloc(); 859 if (IS_ERR(group->group)) { 860 devm_kfree(smmu->dev, group); 861 mutex_unlock(&smmu->lock); 862 return NULL; 863 } 864 865 list_add_tail(&group->list, &smmu->groups); 866 mutex_unlock(&smmu->lock); 867 868 return group->group; 869 } 870 871 static struct iommu_group *tegra_smmu_device_group(struct device *dev) 872 { 873 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 874 struct tegra_smmu *smmu = dev->archdata.iommu; 875 struct iommu_group *group; 876 877 group = tegra_smmu_group_get(smmu, fwspec->ids[0]); 878 if (!group) 879 group = generic_device_group(dev); 880 881 return group; 882 } 883 884 static int tegra_smmu_of_xlate(struct device *dev, 885 struct of_phandle_args *args) 886 { 887 u32 id = args->args[0]; 888 889 return iommu_fwspec_add_ids(dev, &id, 1); 890 } 891 892 static const struct iommu_ops tegra_smmu_ops = { 893 .capable = tegra_smmu_capable, 894 .domain_alloc = tegra_smmu_domain_alloc, 895 .domain_free = tegra_smmu_domain_free, 896 .attach_dev = tegra_smmu_attach_dev, 897 .detach_dev = tegra_smmu_detach_dev, 898 .add_device = tegra_smmu_add_device, 899 .remove_device = tegra_smmu_remove_device, 900 .device_group = tegra_smmu_device_group, 901 .map = tegra_smmu_map, 902 .unmap = tegra_smmu_unmap, 903 .iova_to_phys = tegra_smmu_iova_to_phys, 904 .of_xlate = tegra_smmu_of_xlate, 905 .pgsize_bitmap = SZ_4K, 906 }; 907 908 static void tegra_smmu_ahb_enable(void) 909 { 910 static const struct of_device_id ahb_match[] = { 911 { .compatible = "nvidia,tegra30-ahb", }, 912 { } 913 }; 914 struct device_node *ahb; 915 916 ahb = of_find_matching_node(NULL, ahb_match); 917 if (ahb) { 918 tegra_ahb_enable_smmu(ahb); 919 of_node_put(ahb); 920 } 921 } 922 923 static int tegra_smmu_swgroups_show(struct seq_file *s, void *data) 924 { 925 struct tegra_smmu *smmu = s->private; 926 unsigned int i; 927 u32 value; 928 929 seq_printf(s, "swgroup enabled ASID\n"); 930 seq_printf(s, "------------------------\n"); 931 932 for (i = 0; i < smmu->soc->num_swgroups; i++) { 933 const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i]; 934 const char *status; 935 unsigned int asid; 936 937 value = smmu_readl(smmu, group->reg); 938 939 if (value & SMMU_ASID_ENABLE) 940 status = "yes"; 941 else 942 status = "no"; 943 944 asid = value & SMMU_ASID_MASK; 945 946 seq_printf(s, "%-9s %-7s %#04x\n", group->name, status, 947 asid); 948 } 949 950 return 0; 951 } 952 953 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups); 954 955 static int tegra_smmu_clients_show(struct seq_file *s, void *data) 956 { 957 struct tegra_smmu *smmu = s->private; 958 unsigned int i; 959 u32 value; 960 961 seq_printf(s, "client enabled\n"); 962 seq_printf(s, "--------------------\n"); 963 964 for (i = 0; i < smmu->soc->num_clients; i++) { 965 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 966 const char *status; 967 968 value = smmu_readl(smmu, client->smmu.reg); 969 970 if (value & BIT(client->smmu.bit)) 971 status = "yes"; 972 else 973 status = "no"; 974 975 seq_printf(s, "%-12s %s\n", client->name, status); 976 } 977 978 return 0; 979 } 980 981 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients); 982 983 static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu) 984 { 985 smmu->debugfs = debugfs_create_dir("smmu", NULL); 986 if (!smmu->debugfs) 987 return; 988 989 debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu, 990 &tegra_smmu_swgroups_fops); 991 debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu, 992 &tegra_smmu_clients_fops); 993 } 994 995 static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu) 996 { 997 debugfs_remove_recursive(smmu->debugfs); 998 } 999 1000 struct tegra_smmu *tegra_smmu_probe(struct device *dev, 1001 const struct tegra_smmu_soc *soc, 1002 struct tegra_mc *mc) 1003 { 1004 struct tegra_smmu *smmu; 1005 size_t size; 1006 u32 value; 1007 int err; 1008 1009 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 1010 if (!smmu) 1011 return ERR_PTR(-ENOMEM); 1012 1013 /* 1014 * This is a bit of a hack. Ideally we'd want to simply return this 1015 * value. However the IOMMU registration process will attempt to add 1016 * all devices to the IOMMU when bus_set_iommu() is called. In order 1017 * not to rely on global variables to track the IOMMU instance, we 1018 * set it here so that it can be looked up from the .add_device() 1019 * callback via the IOMMU device's .drvdata field. 1020 */ 1021 mc->smmu = smmu; 1022 1023 size = BITS_TO_LONGS(soc->num_asids) * sizeof(long); 1024 1025 smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL); 1026 if (!smmu->asids) 1027 return ERR_PTR(-ENOMEM); 1028 1029 INIT_LIST_HEAD(&smmu->groups); 1030 mutex_init(&smmu->lock); 1031 1032 smmu->regs = mc->regs; 1033 smmu->soc = soc; 1034 smmu->dev = dev; 1035 smmu->mc = mc; 1036 1037 smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1; 1038 dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n", 1039 mc->soc->num_address_bits, smmu->pfn_mask); 1040 smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1; 1041 dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines, 1042 smmu->tlb_mask); 1043 1044 value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f); 1045 1046 if (soc->supports_request_limit) 1047 value |= SMMU_PTC_CONFIG_REQ_LIMIT(8); 1048 1049 smmu_writel(smmu, value, SMMU_PTC_CONFIG); 1050 1051 value = SMMU_TLB_CONFIG_HIT_UNDER_MISS | 1052 SMMU_TLB_CONFIG_ACTIVE_LINES(smmu); 1053 1054 if (soc->supports_round_robin_arbitration) 1055 value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION; 1056 1057 smmu_writel(smmu, value, SMMU_TLB_CONFIG); 1058 1059 smmu_flush_ptc_all(smmu); 1060 smmu_flush_tlb(smmu); 1061 smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); 1062 smmu_flush(smmu); 1063 1064 tegra_smmu_ahb_enable(); 1065 1066 err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev)); 1067 if (err) 1068 return ERR_PTR(err); 1069 1070 iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops); 1071 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); 1072 1073 err = iommu_device_register(&smmu->iommu); 1074 if (err) { 1075 iommu_device_sysfs_remove(&smmu->iommu); 1076 return ERR_PTR(err); 1077 } 1078 1079 err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops); 1080 if (err < 0) { 1081 iommu_device_unregister(&smmu->iommu); 1082 iommu_device_sysfs_remove(&smmu->iommu); 1083 return ERR_PTR(err); 1084 } 1085 1086 if (IS_ENABLED(CONFIG_DEBUG_FS)) 1087 tegra_smmu_debugfs_init(smmu); 1088 1089 return smmu; 1090 } 1091 1092 void tegra_smmu_remove(struct tegra_smmu *smmu) 1093 { 1094 iommu_device_unregister(&smmu->iommu); 1095 iommu_device_sysfs_remove(&smmu->iommu); 1096 1097 if (IS_ENABLED(CONFIG_DEBUG_FS)) 1098 tegra_smmu_debugfs_exit(smmu); 1099 } 1100