1 /* 2 * omap iommu: tlb and pagetable primitives 3 * 4 * Copyright (C) 2008-2010 Nokia Corporation 5 * 6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>, 7 * Paul Mundt and Toshihiro Kobayashi 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #include <linux/err.h> 15 #include <linux/module.h> 16 #include <linux/slab.h> 17 #include <linux/interrupt.h> 18 #include <linux/ioport.h> 19 #include <linux/platform_device.h> 20 #include <linux/iommu.h> 21 #include <linux/omap-iommu.h> 22 #include <linux/mutex.h> 23 #include <linux/spinlock.h> 24 #include <linux/io.h> 25 #include <linux/pm_runtime.h> 26 #include <linux/of.h> 27 #include <linux/of_iommu.h> 28 #include <linux/of_irq.h> 29 30 #include <asm/cacheflush.h> 31 32 #include <linux/platform_data/iommu-omap.h> 33 34 #include "omap-iopgtable.h" 35 #include "omap-iommu.h" 36 37 #define to_iommu(dev) \ 38 ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev))) 39 40 #define for_each_iotlb_cr(obj, n, __i, cr) \ 41 for (__i = 0; \ 42 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ 43 __i++) 44 45 /* bitmap of the page sizes currently supported */ 46 #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) 47 48 /** 49 * struct omap_iommu_domain - omap iommu domain 50 * @pgtable: the page table 51 * @iommu_dev: an omap iommu device attached to this domain. only a single 52 * iommu device can be attached for now. 53 * @dev: Device using this domain. 54 * @lock: domain lock, should be taken when attaching/detaching 55 */ 56 struct omap_iommu_domain { 57 u32 *pgtable; 58 struct omap_iommu *iommu_dev; 59 struct device *dev; 60 spinlock_t lock; 61 }; 62 63 #define MMU_LOCK_BASE_SHIFT 10 64 #define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT) 65 #define MMU_LOCK_BASE(x) \ 66 ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT) 67 68 #define MMU_LOCK_VICT_SHIFT 4 69 #define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT) 70 #define MMU_LOCK_VICT(x) \ 71 ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT) 72 73 struct iotlb_lock { 74 short base; 75 short vict; 76 }; 77 78 /* accommodate the difference between omap1 and omap2/3 */ 79 static const struct iommu_functions *arch_iommu; 80 81 static struct platform_driver omap_iommu_driver; 82 static struct kmem_cache *iopte_cachep; 83 84 /** 85 * omap_install_iommu_arch - Install archtecure specific iommu functions 86 * @ops: a pointer to architecture specific iommu functions 87 * 88 * There are several kind of iommu algorithm(tlb, pagetable) among 89 * omap series. This interface installs such an iommu algorighm. 90 **/ 91 int omap_install_iommu_arch(const struct iommu_functions *ops) 92 { 93 if (arch_iommu) 94 return -EBUSY; 95 96 arch_iommu = ops; 97 return 0; 98 } 99 EXPORT_SYMBOL_GPL(omap_install_iommu_arch); 100 101 /** 102 * omap_uninstall_iommu_arch - Uninstall archtecure specific iommu functions 103 * @ops: a pointer to architecture specific iommu functions 104 * 105 * This interface uninstalls the iommu algorighm installed previously. 106 **/ 107 void omap_uninstall_iommu_arch(const struct iommu_functions *ops) 108 { 109 if (arch_iommu != ops) 110 pr_err("%s: not your arch\n", __func__); 111 112 arch_iommu = NULL; 113 } 114 EXPORT_SYMBOL_GPL(omap_uninstall_iommu_arch); 115 116 /** 117 * omap_iommu_save_ctx - Save registers for pm off-mode support 118 * @dev: client device 119 **/ 120 void omap_iommu_save_ctx(struct device *dev) 121 { 122 struct omap_iommu *obj = dev_to_omap_iommu(dev); 123 124 arch_iommu->save_ctx(obj); 125 } 126 EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); 127 128 /** 129 * omap_iommu_restore_ctx - Restore registers for pm off-mode support 130 * @dev: client device 131 **/ 132 void omap_iommu_restore_ctx(struct device *dev) 133 { 134 struct omap_iommu *obj = dev_to_omap_iommu(dev); 135 136 arch_iommu->restore_ctx(obj); 137 } 138 EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx); 139 140 /** 141 * omap_iommu_arch_version - Return running iommu arch version 142 **/ 143 u32 omap_iommu_arch_version(void) 144 { 145 return arch_iommu->version; 146 } 147 EXPORT_SYMBOL_GPL(omap_iommu_arch_version); 148 149 static int iommu_enable(struct omap_iommu *obj) 150 { 151 int err; 152 struct platform_device *pdev = to_platform_device(obj->dev); 153 struct iommu_platform_data *pdata = pdev->dev.platform_data; 154 155 if (!arch_iommu) 156 return -ENODEV; 157 158 if (pdata && pdata->deassert_reset) { 159 err = pdata->deassert_reset(pdev, pdata->reset_name); 160 if (err) { 161 dev_err(obj->dev, "deassert_reset failed: %d\n", err); 162 return err; 163 } 164 } 165 166 pm_runtime_get_sync(obj->dev); 167 168 err = arch_iommu->enable(obj); 169 170 return err; 171 } 172 173 static void iommu_disable(struct omap_iommu *obj) 174 { 175 struct platform_device *pdev = to_platform_device(obj->dev); 176 struct iommu_platform_data *pdata = pdev->dev.platform_data; 177 178 arch_iommu->disable(obj); 179 180 pm_runtime_put_sync(obj->dev); 181 182 if (pdata && pdata->assert_reset) 183 pdata->assert_reset(pdev, pdata->reset_name); 184 } 185 186 /* 187 * TLB operations 188 */ 189 void omap_iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e) 190 { 191 BUG_ON(!cr || !e); 192 193 arch_iommu->cr_to_e(cr, e); 194 } 195 EXPORT_SYMBOL_GPL(omap_iotlb_cr_to_e); 196 197 static inline int iotlb_cr_valid(struct cr_regs *cr) 198 { 199 if (!cr) 200 return -EINVAL; 201 202 return arch_iommu->cr_valid(cr); 203 } 204 205 static inline struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj, 206 struct iotlb_entry *e) 207 { 208 if (!e) 209 return NULL; 210 211 return arch_iommu->alloc_cr(obj, e); 212 } 213 214 static u32 iotlb_cr_to_virt(struct cr_regs *cr) 215 { 216 return arch_iommu->cr_to_virt(cr); 217 } 218 219 static u32 get_iopte_attr(struct iotlb_entry *e) 220 { 221 return arch_iommu->get_pte_attr(e); 222 } 223 224 static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da) 225 { 226 return arch_iommu->fault_isr(obj, da); 227 } 228 229 static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) 230 { 231 u32 val; 232 233 val = iommu_read_reg(obj, MMU_LOCK); 234 235 l->base = MMU_LOCK_BASE(val); 236 l->vict = MMU_LOCK_VICT(val); 237 238 } 239 240 static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) 241 { 242 u32 val; 243 244 val = (l->base << MMU_LOCK_BASE_SHIFT); 245 val |= (l->vict << MMU_LOCK_VICT_SHIFT); 246 247 iommu_write_reg(obj, val, MMU_LOCK); 248 } 249 250 static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr) 251 { 252 arch_iommu->tlb_read_cr(obj, cr); 253 } 254 255 static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) 256 { 257 arch_iommu->tlb_load_cr(obj, cr); 258 259 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); 260 iommu_write_reg(obj, 1, MMU_LD_TLB); 261 } 262 263 /** 264 * iotlb_dump_cr - Dump an iommu tlb entry into buf 265 * @obj: target iommu 266 * @cr: contents of cam and ram register 267 * @buf: output buffer 268 **/ 269 static inline ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr, 270 char *buf) 271 { 272 BUG_ON(!cr || !buf); 273 274 return arch_iommu->dump_cr(obj, cr, buf); 275 } 276 277 /* only used in iotlb iteration for-loop */ 278 static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) 279 { 280 struct cr_regs cr; 281 struct iotlb_lock l; 282 283 iotlb_lock_get(obj, &l); 284 l.vict = n; 285 iotlb_lock_set(obj, &l); 286 iotlb_read_cr(obj, &cr); 287 288 return cr; 289 } 290 291 /** 292 * load_iotlb_entry - Set an iommu tlb entry 293 * @obj: target iommu 294 * @e: an iommu tlb entry info 295 **/ 296 #ifdef PREFETCH_IOTLB 297 static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) 298 { 299 int err = 0; 300 struct iotlb_lock l; 301 struct cr_regs *cr; 302 303 if (!obj || !obj->nr_tlb_entries || !e) 304 return -EINVAL; 305 306 pm_runtime_get_sync(obj->dev); 307 308 iotlb_lock_get(obj, &l); 309 if (l.base == obj->nr_tlb_entries) { 310 dev_warn(obj->dev, "%s: preserve entries full\n", __func__); 311 err = -EBUSY; 312 goto out; 313 } 314 if (!e->prsvd) { 315 int i; 316 struct cr_regs tmp; 317 318 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp) 319 if (!iotlb_cr_valid(&tmp)) 320 break; 321 322 if (i == obj->nr_tlb_entries) { 323 dev_dbg(obj->dev, "%s: full: no entry\n", __func__); 324 err = -EBUSY; 325 goto out; 326 } 327 328 iotlb_lock_get(obj, &l); 329 } else { 330 l.vict = l.base; 331 iotlb_lock_set(obj, &l); 332 } 333 334 cr = iotlb_alloc_cr(obj, e); 335 if (IS_ERR(cr)) { 336 pm_runtime_put_sync(obj->dev); 337 return PTR_ERR(cr); 338 } 339 340 iotlb_load_cr(obj, cr); 341 kfree(cr); 342 343 if (e->prsvd) 344 l.base++; 345 /* increment victim for next tlb load */ 346 if (++l.vict == obj->nr_tlb_entries) 347 l.vict = l.base; 348 iotlb_lock_set(obj, &l); 349 out: 350 pm_runtime_put_sync(obj->dev); 351 return err; 352 } 353 354 #else /* !PREFETCH_IOTLB */ 355 356 static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) 357 { 358 return 0; 359 } 360 361 #endif /* !PREFETCH_IOTLB */ 362 363 static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) 364 { 365 return load_iotlb_entry(obj, e); 366 } 367 368 /** 369 * flush_iotlb_page - Clear an iommu tlb entry 370 * @obj: target iommu 371 * @da: iommu device virtual address 372 * 373 * Clear an iommu tlb entry which includes 'da' address. 374 **/ 375 static void flush_iotlb_page(struct omap_iommu *obj, u32 da) 376 { 377 int i; 378 struct cr_regs cr; 379 380 pm_runtime_get_sync(obj->dev); 381 382 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) { 383 u32 start; 384 size_t bytes; 385 386 if (!iotlb_cr_valid(&cr)) 387 continue; 388 389 start = iotlb_cr_to_virt(&cr); 390 bytes = iopgsz_to_bytes(cr.cam & 3); 391 392 if ((start <= da) && (da < start + bytes)) { 393 dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n", 394 __func__, start, da, bytes); 395 iotlb_load_cr(obj, &cr); 396 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); 397 break; 398 } 399 } 400 pm_runtime_put_sync(obj->dev); 401 402 if (i == obj->nr_tlb_entries) 403 dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da); 404 } 405 406 /** 407 * flush_iotlb_all - Clear all iommu tlb entries 408 * @obj: target iommu 409 **/ 410 static void flush_iotlb_all(struct omap_iommu *obj) 411 { 412 struct iotlb_lock l; 413 414 pm_runtime_get_sync(obj->dev); 415 416 l.base = 0; 417 l.vict = 0; 418 iotlb_lock_set(obj, &l); 419 420 iommu_write_reg(obj, 1, MMU_GFLUSH); 421 422 pm_runtime_put_sync(obj->dev); 423 } 424 425 #if defined(CONFIG_OMAP_IOMMU_DEBUG) || defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE) 426 427 ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes) 428 { 429 if (!obj || !buf) 430 return -EINVAL; 431 432 pm_runtime_get_sync(obj->dev); 433 434 bytes = arch_iommu->dump_ctx(obj, buf, bytes); 435 436 pm_runtime_put_sync(obj->dev); 437 438 return bytes; 439 } 440 EXPORT_SYMBOL_GPL(omap_iommu_dump_ctx); 441 442 static int 443 __dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num) 444 { 445 int i; 446 struct iotlb_lock saved; 447 struct cr_regs tmp; 448 struct cr_regs *p = crs; 449 450 pm_runtime_get_sync(obj->dev); 451 iotlb_lock_get(obj, &saved); 452 453 for_each_iotlb_cr(obj, num, i, tmp) { 454 if (!iotlb_cr_valid(&tmp)) 455 continue; 456 *p++ = tmp; 457 } 458 459 iotlb_lock_set(obj, &saved); 460 pm_runtime_put_sync(obj->dev); 461 462 return p - crs; 463 } 464 465 /** 466 * omap_dump_tlb_entries - dump cr arrays to given buffer 467 * @obj: target iommu 468 * @buf: output buffer 469 **/ 470 size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes) 471 { 472 int i, num; 473 struct cr_regs *cr; 474 char *p = buf; 475 476 num = bytes / sizeof(*cr); 477 num = min(obj->nr_tlb_entries, num); 478 479 cr = kcalloc(num, sizeof(*cr), GFP_KERNEL); 480 if (!cr) 481 return 0; 482 483 num = __dump_tlb_entries(obj, cr, num); 484 for (i = 0; i < num; i++) 485 p += iotlb_dump_cr(obj, cr + i, p); 486 kfree(cr); 487 488 return p - buf; 489 } 490 EXPORT_SYMBOL_GPL(omap_dump_tlb_entries); 491 492 int omap_foreach_iommu_device(void *data, int (*fn)(struct device *, void *)) 493 { 494 return driver_for_each_device(&omap_iommu_driver.driver, 495 NULL, data, fn); 496 } 497 EXPORT_SYMBOL_GPL(omap_foreach_iommu_device); 498 499 #endif /* CONFIG_OMAP_IOMMU_DEBUG_MODULE */ 500 501 /* 502 * H/W pagetable operations 503 */ 504 static void flush_iopgd_range(u32 *first, u32 *last) 505 { 506 /* FIXME: L2 cache should be taken care of if it exists */ 507 do { 508 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd" 509 : : "r" (first)); 510 first += L1_CACHE_BYTES / sizeof(*first); 511 } while (first <= last); 512 } 513 514 static void flush_iopte_range(u32 *first, u32 *last) 515 { 516 /* FIXME: L2 cache should be taken care of if it exists */ 517 do { 518 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte" 519 : : "r" (first)); 520 first += L1_CACHE_BYTES / sizeof(*first); 521 } while (first <= last); 522 } 523 524 static void iopte_free(u32 *iopte) 525 { 526 /* Note: freed iopte's must be clean ready for re-use */ 527 if (iopte) 528 kmem_cache_free(iopte_cachep, iopte); 529 } 530 531 static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da) 532 { 533 u32 *iopte; 534 535 /* a table has already existed */ 536 if (*iopgd) 537 goto pte_ready; 538 539 /* 540 * do the allocation outside the page table lock 541 */ 542 spin_unlock(&obj->page_table_lock); 543 iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL); 544 spin_lock(&obj->page_table_lock); 545 546 if (!*iopgd) { 547 if (!iopte) 548 return ERR_PTR(-ENOMEM); 549 550 *iopgd = virt_to_phys(iopte) | IOPGD_TABLE; 551 flush_iopgd_range(iopgd, iopgd); 552 553 dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte); 554 } else { 555 /* We raced, free the reduniovant table */ 556 iopte_free(iopte); 557 } 558 559 pte_ready: 560 iopte = iopte_offset(iopgd, da); 561 562 dev_vdbg(obj->dev, 563 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n", 564 __func__, da, iopgd, *iopgd, iopte, *iopte); 565 566 return iopte; 567 } 568 569 static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) 570 { 571 u32 *iopgd = iopgd_offset(obj, da); 572 573 if ((da | pa) & ~IOSECTION_MASK) { 574 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", 575 __func__, da, pa, IOSECTION_SIZE); 576 return -EINVAL; 577 } 578 579 *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION; 580 flush_iopgd_range(iopgd, iopgd); 581 return 0; 582 } 583 584 static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) 585 { 586 u32 *iopgd = iopgd_offset(obj, da); 587 int i; 588 589 if ((da | pa) & ~IOSUPER_MASK) { 590 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", 591 __func__, da, pa, IOSUPER_SIZE); 592 return -EINVAL; 593 } 594 595 for (i = 0; i < 16; i++) 596 *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER; 597 flush_iopgd_range(iopgd, iopgd + 15); 598 return 0; 599 } 600 601 static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) 602 { 603 u32 *iopgd = iopgd_offset(obj, da); 604 u32 *iopte = iopte_alloc(obj, iopgd, da); 605 606 if (IS_ERR(iopte)) 607 return PTR_ERR(iopte); 608 609 *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL; 610 flush_iopte_range(iopte, iopte); 611 612 dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n", 613 __func__, da, pa, iopte, *iopte); 614 615 return 0; 616 } 617 618 static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) 619 { 620 u32 *iopgd = iopgd_offset(obj, da); 621 u32 *iopte = iopte_alloc(obj, iopgd, da); 622 int i; 623 624 if ((da | pa) & ~IOLARGE_MASK) { 625 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", 626 __func__, da, pa, IOLARGE_SIZE); 627 return -EINVAL; 628 } 629 630 if (IS_ERR(iopte)) 631 return PTR_ERR(iopte); 632 633 for (i = 0; i < 16; i++) 634 *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE; 635 flush_iopte_range(iopte, iopte + 15); 636 return 0; 637 } 638 639 static int 640 iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e) 641 { 642 int (*fn)(struct omap_iommu *, u32, u32, u32); 643 u32 prot; 644 int err; 645 646 if (!obj || !e) 647 return -EINVAL; 648 649 switch (e->pgsz) { 650 case MMU_CAM_PGSZ_16M: 651 fn = iopgd_alloc_super; 652 break; 653 case MMU_CAM_PGSZ_1M: 654 fn = iopgd_alloc_section; 655 break; 656 case MMU_CAM_PGSZ_64K: 657 fn = iopte_alloc_large; 658 break; 659 case MMU_CAM_PGSZ_4K: 660 fn = iopte_alloc_page; 661 break; 662 default: 663 fn = NULL; 664 BUG(); 665 break; 666 } 667 668 prot = get_iopte_attr(e); 669 670 spin_lock(&obj->page_table_lock); 671 err = fn(obj, e->da, e->pa, prot); 672 spin_unlock(&obj->page_table_lock); 673 674 return err; 675 } 676 677 /** 678 * omap_iopgtable_store_entry - Make an iommu pte entry 679 * @obj: target iommu 680 * @e: an iommu tlb entry info 681 **/ 682 int omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e) 683 { 684 int err; 685 686 flush_iotlb_page(obj, e->da); 687 err = iopgtable_store_entry_core(obj, e); 688 if (!err) 689 prefetch_iotlb_entry(obj, e); 690 return err; 691 } 692 EXPORT_SYMBOL_GPL(omap_iopgtable_store_entry); 693 694 /** 695 * iopgtable_lookup_entry - Lookup an iommu pte entry 696 * @obj: target iommu 697 * @da: iommu device virtual address 698 * @ppgd: iommu pgd entry pointer to be returned 699 * @ppte: iommu pte entry pointer to be returned 700 **/ 701 static void 702 iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte) 703 { 704 u32 *iopgd, *iopte = NULL; 705 706 iopgd = iopgd_offset(obj, da); 707 if (!*iopgd) 708 goto out; 709 710 if (iopgd_is_table(*iopgd)) 711 iopte = iopte_offset(iopgd, da); 712 out: 713 *ppgd = iopgd; 714 *ppte = iopte; 715 } 716 717 static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da) 718 { 719 size_t bytes; 720 u32 *iopgd = iopgd_offset(obj, da); 721 int nent = 1; 722 723 if (!*iopgd) 724 return 0; 725 726 if (iopgd_is_table(*iopgd)) { 727 int i; 728 u32 *iopte = iopte_offset(iopgd, da); 729 730 bytes = IOPTE_SIZE; 731 if (*iopte & IOPTE_LARGE) { 732 nent *= 16; 733 /* rewind to the 1st entry */ 734 iopte = iopte_offset(iopgd, (da & IOLARGE_MASK)); 735 } 736 bytes *= nent; 737 memset(iopte, 0, nent * sizeof(*iopte)); 738 flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte)); 739 740 /* 741 * do table walk to check if this table is necessary or not 742 */ 743 iopte = iopte_offset(iopgd, 0); 744 for (i = 0; i < PTRS_PER_IOPTE; i++) 745 if (iopte[i]) 746 goto out; 747 748 iopte_free(iopte); 749 nent = 1; /* for the next L1 entry */ 750 } else { 751 bytes = IOPGD_SIZE; 752 if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) { 753 nent *= 16; 754 /* rewind to the 1st entry */ 755 iopgd = iopgd_offset(obj, (da & IOSUPER_MASK)); 756 } 757 bytes *= nent; 758 } 759 memset(iopgd, 0, nent * sizeof(*iopgd)); 760 flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd)); 761 out: 762 return bytes; 763 } 764 765 /** 766 * iopgtable_clear_entry - Remove an iommu pte entry 767 * @obj: target iommu 768 * @da: iommu device virtual address 769 **/ 770 static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da) 771 { 772 size_t bytes; 773 774 spin_lock(&obj->page_table_lock); 775 776 bytes = iopgtable_clear_entry_core(obj, da); 777 flush_iotlb_page(obj, da); 778 779 spin_unlock(&obj->page_table_lock); 780 781 return bytes; 782 } 783 784 static void iopgtable_clear_entry_all(struct omap_iommu *obj) 785 { 786 int i; 787 788 spin_lock(&obj->page_table_lock); 789 790 for (i = 0; i < PTRS_PER_IOPGD; i++) { 791 u32 da; 792 u32 *iopgd; 793 794 da = i << IOPGD_SHIFT; 795 iopgd = iopgd_offset(obj, da); 796 797 if (!*iopgd) 798 continue; 799 800 if (iopgd_is_table(*iopgd)) 801 iopte_free(iopte_offset(iopgd, 0)); 802 803 *iopgd = 0; 804 flush_iopgd_range(iopgd, iopgd); 805 } 806 807 flush_iotlb_all(obj); 808 809 spin_unlock(&obj->page_table_lock); 810 } 811 812 /* 813 * Device IOMMU generic operations 814 */ 815 static irqreturn_t iommu_fault_handler(int irq, void *data) 816 { 817 u32 da, errs; 818 u32 *iopgd, *iopte; 819 struct omap_iommu *obj = data; 820 struct iommu_domain *domain = obj->domain; 821 822 if (!obj->refcount) 823 return IRQ_NONE; 824 825 errs = iommu_report_fault(obj, &da); 826 if (errs == 0) 827 return IRQ_HANDLED; 828 829 /* Fault callback or TLB/PTE Dynamic loading */ 830 if (!report_iommu_fault(domain, obj->dev, da, 0)) 831 return IRQ_HANDLED; 832 833 iommu_disable(obj); 834 835 iopgd = iopgd_offset(obj, da); 836 837 if (!iopgd_is_table(*iopgd)) { 838 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n", 839 obj->name, errs, da, iopgd, *iopgd); 840 return IRQ_NONE; 841 } 842 843 iopte = iopte_offset(iopgd, da); 844 845 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n", 846 obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); 847 848 return IRQ_NONE; 849 } 850 851 static int device_match_by_alias(struct device *dev, void *data) 852 { 853 struct omap_iommu *obj = to_iommu(dev); 854 const char *name = data; 855 856 pr_debug("%s: %s %s\n", __func__, obj->name, name); 857 858 return strcmp(obj->name, name) == 0; 859 } 860 861 /** 862 * omap_iommu_attach() - attach iommu device to an iommu domain 863 * @name: name of target omap iommu device 864 * @iopgd: page table 865 **/ 866 static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd) 867 { 868 int err; 869 struct device *dev; 870 struct omap_iommu *obj; 871 872 dev = driver_find_device(&omap_iommu_driver.driver, NULL, 873 (void *)name, 874 device_match_by_alias); 875 if (!dev) 876 return ERR_PTR(-ENODEV); 877 878 obj = to_iommu(dev); 879 880 spin_lock(&obj->iommu_lock); 881 882 /* an iommu device can only be attached once */ 883 if (++obj->refcount > 1) { 884 dev_err(dev, "%s: already attached!\n", obj->name); 885 err = -EBUSY; 886 goto err_enable; 887 } 888 889 obj->iopgd = iopgd; 890 err = iommu_enable(obj); 891 if (err) 892 goto err_enable; 893 flush_iotlb_all(obj); 894 895 if (!try_module_get(obj->owner)) { 896 err = -ENODEV; 897 goto err_module; 898 } 899 900 spin_unlock(&obj->iommu_lock); 901 902 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); 903 return obj; 904 905 err_module: 906 if (obj->refcount == 1) 907 iommu_disable(obj); 908 err_enable: 909 obj->refcount--; 910 spin_unlock(&obj->iommu_lock); 911 return ERR_PTR(err); 912 } 913 914 /** 915 * omap_iommu_detach - release iommu device 916 * @obj: target iommu 917 **/ 918 static void omap_iommu_detach(struct omap_iommu *obj) 919 { 920 if (!obj || IS_ERR(obj)) 921 return; 922 923 spin_lock(&obj->iommu_lock); 924 925 if (--obj->refcount == 0) 926 iommu_disable(obj); 927 928 module_put(obj->owner); 929 930 obj->iopgd = NULL; 931 932 spin_unlock(&obj->iommu_lock); 933 934 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); 935 } 936 937 /* 938 * OMAP Device MMU(IOMMU) detection 939 */ 940 static int omap_iommu_probe(struct platform_device *pdev) 941 { 942 int err = -ENODEV; 943 int irq; 944 struct omap_iommu *obj; 945 struct resource *res; 946 struct iommu_platform_data *pdata = pdev->dev.platform_data; 947 struct device_node *of = pdev->dev.of_node; 948 949 obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL); 950 if (!obj) 951 return -ENOMEM; 952 953 if (of) { 954 obj->name = dev_name(&pdev->dev); 955 obj->nr_tlb_entries = 32; 956 err = of_property_read_u32(of, "ti,#tlb-entries", 957 &obj->nr_tlb_entries); 958 if (err && err != -EINVAL) 959 return err; 960 if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8) 961 return -EINVAL; 962 if (of_find_property(of, "ti,iommu-bus-err-back", NULL)) 963 obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN; 964 } else { 965 obj->nr_tlb_entries = pdata->nr_tlb_entries; 966 obj->name = pdata->name; 967 } 968 969 obj->dev = &pdev->dev; 970 obj->ctx = (void *)obj + sizeof(*obj); 971 972 spin_lock_init(&obj->iommu_lock); 973 spin_lock_init(&obj->page_table_lock); 974 975 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 976 obj->regbase = devm_ioremap_resource(obj->dev, res); 977 if (IS_ERR(obj->regbase)) 978 return PTR_ERR(obj->regbase); 979 980 irq = platform_get_irq(pdev, 0); 981 if (irq < 0) 982 return -ENODEV; 983 984 err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED, 985 dev_name(obj->dev), obj); 986 if (err < 0) 987 return err; 988 platform_set_drvdata(pdev, obj); 989 990 pm_runtime_irq_safe(obj->dev); 991 pm_runtime_enable(obj->dev); 992 993 dev_info(&pdev->dev, "%s registered\n", obj->name); 994 return 0; 995 } 996 997 static int omap_iommu_remove(struct platform_device *pdev) 998 { 999 struct omap_iommu *obj = platform_get_drvdata(pdev); 1000 1001 iopgtable_clear_entry_all(obj); 1002 1003 pm_runtime_disable(obj->dev); 1004 1005 dev_info(&pdev->dev, "%s removed\n", obj->name); 1006 return 0; 1007 } 1008 1009 static struct of_device_id omap_iommu_of_match[] = { 1010 { .compatible = "ti,omap2-iommu" }, 1011 { .compatible = "ti,omap4-iommu" }, 1012 { .compatible = "ti,dra7-iommu" }, 1013 {}, 1014 }; 1015 MODULE_DEVICE_TABLE(of, omap_iommu_of_match); 1016 1017 static struct platform_driver omap_iommu_driver = { 1018 .probe = omap_iommu_probe, 1019 .remove = omap_iommu_remove, 1020 .driver = { 1021 .name = "omap-iommu", 1022 .of_match_table = of_match_ptr(omap_iommu_of_match), 1023 }, 1024 }; 1025 1026 static void iopte_cachep_ctor(void *iopte) 1027 { 1028 clean_dcache_area(iopte, IOPTE_TABLE_SIZE); 1029 } 1030 1031 static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz) 1032 { 1033 memset(e, 0, sizeof(*e)); 1034 1035 e->da = da; 1036 e->pa = pa; 1037 e->valid = MMU_CAM_V; 1038 /* FIXME: add OMAP1 support */ 1039 e->pgsz = pgsz; 1040 e->endian = MMU_RAM_ENDIAN_LITTLE; 1041 e->elsz = MMU_RAM_ELSZ_8; 1042 e->mixed = 0; 1043 1044 return iopgsz_to_bytes(e->pgsz); 1045 } 1046 1047 static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, 1048 phys_addr_t pa, size_t bytes, int prot) 1049 { 1050 struct omap_iommu_domain *omap_domain = domain->priv; 1051 struct omap_iommu *oiommu = omap_domain->iommu_dev; 1052 struct device *dev = oiommu->dev; 1053 struct iotlb_entry e; 1054 int omap_pgsz; 1055 u32 ret; 1056 1057 omap_pgsz = bytes_to_iopgsz(bytes); 1058 if (omap_pgsz < 0) { 1059 dev_err(dev, "invalid size to map: %d\n", bytes); 1060 return -EINVAL; 1061 } 1062 1063 dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes); 1064 1065 iotlb_init_entry(&e, da, pa, omap_pgsz); 1066 1067 ret = omap_iopgtable_store_entry(oiommu, &e); 1068 if (ret) 1069 dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret); 1070 1071 return ret; 1072 } 1073 1074 static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, 1075 size_t size) 1076 { 1077 struct omap_iommu_domain *omap_domain = domain->priv; 1078 struct omap_iommu *oiommu = omap_domain->iommu_dev; 1079 struct device *dev = oiommu->dev; 1080 1081 dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size); 1082 1083 return iopgtable_clear_entry(oiommu, da); 1084 } 1085 1086 static int 1087 omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) 1088 { 1089 struct omap_iommu_domain *omap_domain = domain->priv; 1090 struct omap_iommu *oiommu; 1091 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; 1092 int ret = 0; 1093 1094 spin_lock(&omap_domain->lock); 1095 1096 /* only a single device is supported per domain for now */ 1097 if (omap_domain->iommu_dev) { 1098 dev_err(dev, "iommu domain is already attached\n"); 1099 ret = -EBUSY; 1100 goto out; 1101 } 1102 1103 /* get a handle to and enable the omap iommu */ 1104 oiommu = omap_iommu_attach(arch_data->name, omap_domain->pgtable); 1105 if (IS_ERR(oiommu)) { 1106 ret = PTR_ERR(oiommu); 1107 dev_err(dev, "can't get omap iommu: %d\n", ret); 1108 goto out; 1109 } 1110 1111 omap_domain->iommu_dev = arch_data->iommu_dev = oiommu; 1112 omap_domain->dev = dev; 1113 oiommu->domain = domain; 1114 1115 out: 1116 spin_unlock(&omap_domain->lock); 1117 return ret; 1118 } 1119 1120 static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, 1121 struct device *dev) 1122 { 1123 struct omap_iommu *oiommu = dev_to_omap_iommu(dev); 1124 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; 1125 1126 /* only a single device is supported per domain for now */ 1127 if (omap_domain->iommu_dev != oiommu) { 1128 dev_err(dev, "invalid iommu device\n"); 1129 return; 1130 } 1131 1132 iopgtable_clear_entry_all(oiommu); 1133 1134 omap_iommu_detach(oiommu); 1135 1136 omap_domain->iommu_dev = arch_data->iommu_dev = NULL; 1137 omap_domain->dev = NULL; 1138 } 1139 1140 static void omap_iommu_detach_dev(struct iommu_domain *domain, 1141 struct device *dev) 1142 { 1143 struct omap_iommu_domain *omap_domain = domain->priv; 1144 1145 spin_lock(&omap_domain->lock); 1146 _omap_iommu_detach_dev(omap_domain, dev); 1147 spin_unlock(&omap_domain->lock); 1148 } 1149 1150 static int omap_iommu_domain_init(struct iommu_domain *domain) 1151 { 1152 struct omap_iommu_domain *omap_domain; 1153 1154 omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); 1155 if (!omap_domain) { 1156 pr_err("kzalloc failed\n"); 1157 goto out; 1158 } 1159 1160 omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL); 1161 if (!omap_domain->pgtable) { 1162 pr_err("kzalloc failed\n"); 1163 goto fail_nomem; 1164 } 1165 1166 /* 1167 * should never fail, but please keep this around to ensure 1168 * we keep the hardware happy 1169 */ 1170 BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)); 1171 1172 clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE); 1173 spin_lock_init(&omap_domain->lock); 1174 1175 domain->priv = omap_domain; 1176 1177 domain->geometry.aperture_start = 0; 1178 domain->geometry.aperture_end = (1ULL << 32) - 1; 1179 domain->geometry.force_aperture = true; 1180 1181 return 0; 1182 1183 fail_nomem: 1184 kfree(omap_domain); 1185 out: 1186 return -ENOMEM; 1187 } 1188 1189 static void omap_iommu_domain_destroy(struct iommu_domain *domain) 1190 { 1191 struct omap_iommu_domain *omap_domain = domain->priv; 1192 1193 domain->priv = NULL; 1194 1195 /* 1196 * An iommu device is still attached 1197 * (currently, only one device can be attached) ? 1198 */ 1199 if (omap_domain->iommu_dev) 1200 _omap_iommu_detach_dev(omap_domain, omap_domain->dev); 1201 1202 kfree(omap_domain->pgtable); 1203 kfree(omap_domain); 1204 } 1205 1206 static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, 1207 dma_addr_t da) 1208 { 1209 struct omap_iommu_domain *omap_domain = domain->priv; 1210 struct omap_iommu *oiommu = omap_domain->iommu_dev; 1211 struct device *dev = oiommu->dev; 1212 u32 *pgd, *pte; 1213 phys_addr_t ret = 0; 1214 1215 iopgtable_lookup_entry(oiommu, da, &pgd, &pte); 1216 1217 if (pte) { 1218 if (iopte_is_small(*pte)) 1219 ret = omap_iommu_translate(*pte, da, IOPTE_MASK); 1220 else if (iopte_is_large(*pte)) 1221 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); 1222 else 1223 dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte, 1224 (unsigned long long)da); 1225 } else { 1226 if (iopgd_is_section(*pgd)) 1227 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); 1228 else if (iopgd_is_super(*pgd)) 1229 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); 1230 else 1231 dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd, 1232 (unsigned long long)da); 1233 } 1234 1235 return ret; 1236 } 1237 1238 static int omap_iommu_add_device(struct device *dev) 1239 { 1240 struct omap_iommu_arch_data *arch_data; 1241 struct device_node *np; 1242 1243 /* 1244 * Allocate the archdata iommu structure for DT-based devices. 1245 * 1246 * TODO: Simplify this when removing non-DT support completely from the 1247 * IOMMU users. 1248 */ 1249 if (!dev->of_node) 1250 return 0; 1251 1252 np = of_parse_phandle(dev->of_node, "iommus", 0); 1253 if (!np) 1254 return 0; 1255 1256 arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL); 1257 if (!arch_data) { 1258 of_node_put(np); 1259 return -ENOMEM; 1260 } 1261 1262 arch_data->name = kstrdup(dev_name(dev), GFP_KERNEL); 1263 dev->archdata.iommu = arch_data; 1264 1265 of_node_put(np); 1266 1267 return 0; 1268 } 1269 1270 static void omap_iommu_remove_device(struct device *dev) 1271 { 1272 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; 1273 1274 if (!dev->of_node || !arch_data) 1275 return; 1276 1277 kfree(arch_data->name); 1278 kfree(arch_data); 1279 } 1280 1281 static const struct iommu_ops omap_iommu_ops = { 1282 .domain_init = omap_iommu_domain_init, 1283 .domain_destroy = omap_iommu_domain_destroy, 1284 .attach_dev = omap_iommu_attach_dev, 1285 .detach_dev = omap_iommu_detach_dev, 1286 .map = omap_iommu_map, 1287 .unmap = omap_iommu_unmap, 1288 .iova_to_phys = omap_iommu_iova_to_phys, 1289 .add_device = omap_iommu_add_device, 1290 .remove_device = omap_iommu_remove_device, 1291 .pgsize_bitmap = OMAP_IOMMU_PGSIZES, 1292 }; 1293 1294 static int __init omap_iommu_init(void) 1295 { 1296 struct kmem_cache *p; 1297 const unsigned long flags = SLAB_HWCACHE_ALIGN; 1298 size_t align = 1 << 10; /* L2 pagetable alignement */ 1299 1300 p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags, 1301 iopte_cachep_ctor); 1302 if (!p) 1303 return -ENOMEM; 1304 iopte_cachep = p; 1305 1306 bus_set_iommu(&platform_bus_type, &omap_iommu_ops); 1307 1308 return platform_driver_register(&omap_iommu_driver); 1309 } 1310 /* must be ready before omap3isp is probed */ 1311 subsys_initcall(omap_iommu_init); 1312 1313 static void __exit omap_iommu_exit(void) 1314 { 1315 kmem_cache_destroy(iopte_cachep); 1316 1317 platform_driver_unregister(&omap_iommu_driver); 1318 } 1319 module_exit(omap_iommu_exit); 1320 1321 MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives"); 1322 MODULE_ALIAS("platform:omap-iommu"); 1323 MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi"); 1324 MODULE_LICENSE("GPL v2"); 1325