1 /* 2 * IPMMU VMSA 3 * 4 * Copyright (C) 2014 Renesas Electronics Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; version 2 of the License. 9 */ 10 11 #include <linux/delay.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/err.h> 14 #include <linux/export.h> 15 #include <linux/interrupt.h> 16 #include <linux/io.h> 17 #include <linux/iommu.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/platform_device.h> 21 #include <linux/sizes.h> 22 #include <linux/slab.h> 23 24 #include <asm/dma-iommu.h> 25 #include <asm/pgalloc.h> 26 27 #include "io-pgtable.h" 28 29 struct ipmmu_vmsa_device { 30 struct device *dev; 31 void __iomem *base; 32 struct list_head list; 33 34 unsigned int num_utlbs; 35 36 struct dma_iommu_mapping *mapping; 37 }; 38 39 struct ipmmu_vmsa_domain { 40 struct ipmmu_vmsa_device *mmu; 41 struct iommu_domain io_domain; 42 43 struct io_pgtable_cfg cfg; 44 struct io_pgtable_ops *iop; 45 46 unsigned int context_id; 47 spinlock_t lock; /* Protects mappings */ 48 }; 49 50 struct ipmmu_vmsa_archdata { 51 struct ipmmu_vmsa_device *mmu; 52 unsigned int *utlbs; 53 unsigned int num_utlbs; 54 }; 55 56 static DEFINE_SPINLOCK(ipmmu_devices_lock); 57 static LIST_HEAD(ipmmu_devices); 58 59 static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) 60 { 61 return container_of(dom, struct ipmmu_vmsa_domain, io_domain); 62 } 63 64 #define TLB_LOOP_TIMEOUT 100 /* 100us */ 65 66 /* ----------------------------------------------------------------------------- 67 * Registers Definition 68 */ 69 70 #define IM_NS_ALIAS_OFFSET 0x800 71 72 #define IM_CTX_SIZE 0x40 73 74 #define IMCTR 0x0000 75 #define IMCTR_TRE (1 << 17) 76 #define IMCTR_AFE (1 << 16) 77 #define IMCTR_RTSEL_MASK (3 << 4) 78 #define IMCTR_RTSEL_SHIFT 4 79 #define IMCTR_TREN (1 << 3) 80 #define IMCTR_INTEN (1 << 2) 81 #define IMCTR_FLUSH (1 << 1) 82 #define IMCTR_MMUEN (1 << 0) 83 84 #define IMCAAR 0x0004 85 86 #define IMTTBCR 0x0008 87 #define IMTTBCR_EAE (1 << 31) 88 #define IMTTBCR_PMB (1 << 30) 89 #define IMTTBCR_SH1_NON_SHAREABLE (0 << 28) 90 #define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28) 91 #define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28) 92 #define IMTTBCR_SH1_MASK (3 << 28) 93 #define IMTTBCR_ORGN1_NC (0 << 26) 94 #define IMTTBCR_ORGN1_WB_WA (1 << 26) 95 #define IMTTBCR_ORGN1_WT (2 << 26) 96 #define IMTTBCR_ORGN1_WB (3 << 26) 97 #define IMTTBCR_ORGN1_MASK (3 << 26) 98 #define IMTTBCR_IRGN1_NC (0 << 24) 99 #define IMTTBCR_IRGN1_WB_WA (1 << 24) 100 #define IMTTBCR_IRGN1_WT (2 << 24) 101 #define IMTTBCR_IRGN1_WB (3 << 24) 102 #define IMTTBCR_IRGN1_MASK (3 << 24) 103 #define IMTTBCR_TSZ1_MASK (7 << 16) 104 #define IMTTBCR_TSZ1_SHIFT 16 105 #define IMTTBCR_SH0_NON_SHAREABLE (0 << 12) 106 #define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12) 107 #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) 108 #define IMTTBCR_SH0_MASK (3 << 12) 109 #define IMTTBCR_ORGN0_NC (0 << 10) 110 #define IMTTBCR_ORGN0_WB_WA (1 << 10) 111 #define IMTTBCR_ORGN0_WT (2 << 10) 112 #define IMTTBCR_ORGN0_WB (3 << 10) 113 #define IMTTBCR_ORGN0_MASK (3 << 10) 114 #define IMTTBCR_IRGN0_NC (0 << 8) 115 #define IMTTBCR_IRGN0_WB_WA (1 << 8) 116 #define IMTTBCR_IRGN0_WT (2 << 8) 117 #define IMTTBCR_IRGN0_WB (3 << 8) 118 #define IMTTBCR_IRGN0_MASK (3 << 8) 119 #define IMTTBCR_SL0_LVL_2 (0 << 4) 120 #define IMTTBCR_SL0_LVL_1 (1 << 4) 121 #define IMTTBCR_TSZ0_MASK (7 << 0) 122 #define IMTTBCR_TSZ0_SHIFT O 123 124 #define IMBUSCR 0x000c 125 #define IMBUSCR_DVM (1 << 2) 126 #define IMBUSCR_BUSSEL_SYS (0 << 0) 127 #define IMBUSCR_BUSSEL_CCI (1 << 0) 128 #define IMBUSCR_BUSSEL_IMCAAR (2 << 0) 129 #define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0) 130 #define IMBUSCR_BUSSEL_MASK (3 << 0) 131 132 #define IMTTLBR0 0x0010 133 #define IMTTUBR0 0x0014 134 #define IMTTLBR1 0x0018 135 #define IMTTUBR1 0x001c 136 137 #define IMSTR 0x0020 138 #define IMSTR_ERRLVL_MASK (3 << 12) 139 #define IMSTR_ERRLVL_SHIFT 12 140 #define IMSTR_ERRCODE_TLB_FORMAT (1 << 8) 141 #define IMSTR_ERRCODE_ACCESS_PERM (4 << 8) 142 #define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8) 143 #define IMSTR_ERRCODE_MASK (7 << 8) 144 #define IMSTR_MHIT (1 << 4) 145 #define IMSTR_ABORT (1 << 2) 146 #define IMSTR_PF (1 << 1) 147 #define IMSTR_TF (1 << 0) 148 149 #define IMMAIR0 0x0028 150 #define IMMAIR1 0x002c 151 #define IMMAIR_ATTR_MASK 0xff 152 #define IMMAIR_ATTR_DEVICE 0x04 153 #define IMMAIR_ATTR_NC 0x44 154 #define IMMAIR_ATTR_WBRWA 0xff 155 #define IMMAIR_ATTR_SHIFT(n) ((n) << 3) 156 #define IMMAIR_ATTR_IDX_NC 0 157 #define IMMAIR_ATTR_IDX_WBRWA 1 158 #define IMMAIR_ATTR_IDX_DEV 2 159 160 #define IMEAR 0x0030 161 162 #define IMPCTR 0x0200 163 #define IMPSTR 0x0208 164 #define IMPEAR 0x020c 165 #define IMPMBA(n) (0x0280 + ((n) * 4)) 166 #define IMPMBD(n) (0x02c0 + ((n) * 4)) 167 168 #define IMUCTR(n) (0x0300 + ((n) * 16)) 169 #define IMUCTR_FIXADDEN (1 << 31) 170 #define IMUCTR_FIXADD_MASK (0xff << 16) 171 #define IMUCTR_FIXADD_SHIFT 16 172 #define IMUCTR_TTSEL_MMU(n) ((n) << 4) 173 #define IMUCTR_TTSEL_PMB (8 << 4) 174 #define IMUCTR_TTSEL_MASK (15 << 4) 175 #define IMUCTR_FLUSH (1 << 1) 176 #define IMUCTR_MMUEN (1 << 0) 177 178 #define IMUASID(n) (0x0308 + ((n) * 16)) 179 #define IMUASID_ASID8_MASK (0xff << 8) 180 #define IMUASID_ASID8_SHIFT 8 181 #define IMUASID_ASID0_MASK (0xff << 0) 182 #define IMUASID_ASID0_SHIFT 0 183 184 /* ----------------------------------------------------------------------------- 185 * Read/Write Access 186 */ 187 188 static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset) 189 { 190 return ioread32(mmu->base + offset); 191 } 192 193 static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, 194 u32 data) 195 { 196 iowrite32(data, mmu->base + offset); 197 } 198 199 static u32 ipmmu_ctx_read(struct ipmmu_vmsa_domain *domain, unsigned int reg) 200 { 201 return ipmmu_read(domain->mmu, domain->context_id * IM_CTX_SIZE + reg); 202 } 203 204 static void ipmmu_ctx_write(struct ipmmu_vmsa_domain *domain, unsigned int reg, 205 u32 data) 206 { 207 ipmmu_write(domain->mmu, domain->context_id * IM_CTX_SIZE + reg, data); 208 } 209 210 /* ----------------------------------------------------------------------------- 211 * TLB and microTLB Management 212 */ 213 214 /* Wait for any pending TLB invalidations to complete */ 215 static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain) 216 { 217 unsigned int count = 0; 218 219 while (ipmmu_ctx_read(domain, IMCTR) & IMCTR_FLUSH) { 220 cpu_relax(); 221 if (++count == TLB_LOOP_TIMEOUT) { 222 dev_err_ratelimited(domain->mmu->dev, 223 "TLB sync timed out -- MMU may be deadlocked\n"); 224 return; 225 } 226 udelay(1); 227 } 228 } 229 230 static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain) 231 { 232 u32 reg; 233 234 reg = ipmmu_ctx_read(domain, IMCTR); 235 reg |= IMCTR_FLUSH; 236 ipmmu_ctx_write(domain, IMCTR, reg); 237 238 ipmmu_tlb_sync(domain); 239 } 240 241 /* 242 * Enable MMU translation for the microTLB. 243 */ 244 static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, 245 unsigned int utlb) 246 { 247 struct ipmmu_vmsa_device *mmu = domain->mmu; 248 249 /* 250 * TODO: Reference-count the microTLB as several bus masters can be 251 * connected to the same microTLB. 252 */ 253 254 /* TODO: What should we set the ASID to ? */ 255 ipmmu_write(mmu, IMUASID(utlb), 0); 256 /* TODO: Do we need to flush the microTLB ? */ 257 ipmmu_write(mmu, IMUCTR(utlb), 258 IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH | 259 IMUCTR_MMUEN); 260 } 261 262 /* 263 * Disable MMU translation for the microTLB. 264 */ 265 static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, 266 unsigned int utlb) 267 { 268 struct ipmmu_vmsa_device *mmu = domain->mmu; 269 270 ipmmu_write(mmu, IMUCTR(utlb), 0); 271 } 272 273 static void ipmmu_tlb_flush_all(void *cookie) 274 { 275 struct ipmmu_vmsa_domain *domain = cookie; 276 277 ipmmu_tlb_invalidate(domain); 278 } 279 280 static void ipmmu_tlb_add_flush(unsigned long iova, size_t size, 281 size_t granule, bool leaf, void *cookie) 282 { 283 /* The hardware doesn't support selective TLB flush. */ 284 } 285 286 static struct iommu_gather_ops ipmmu_gather_ops = { 287 .tlb_flush_all = ipmmu_tlb_flush_all, 288 .tlb_add_flush = ipmmu_tlb_add_flush, 289 .tlb_sync = ipmmu_tlb_flush_all, 290 }; 291 292 /* ----------------------------------------------------------------------------- 293 * Domain/Context Management 294 */ 295 296 static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) 297 { 298 u64 ttbr; 299 300 /* 301 * Allocate the page table operations. 302 * 303 * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory 304 * access, Long-descriptor format" that the NStable bit being set in a 305 * table descriptor will result in the NStable and NS bits of all child 306 * entries being ignored and considered as being set. The IPMMU seems 307 * not to comply with this, as it generates a secure access page fault 308 * if any of the NStable and NS bits isn't set when running in 309 * non-secure mode. 310 */ 311 domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; 312 domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, 313 domain->cfg.ias = 32; 314 domain->cfg.oas = 40; 315 domain->cfg.tlb = &ipmmu_gather_ops; 316 domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); 317 domain->io_domain.geometry.force_aperture = true; 318 /* 319 * TODO: Add support for coherent walk through CCI with DVM and remove 320 * cache handling. For now, delegate it to the io-pgtable code. 321 */ 322 domain->cfg.iommu_dev = domain->mmu->dev; 323 324 domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, 325 domain); 326 if (!domain->iop) 327 return -EINVAL; 328 329 /* 330 * TODO: When adding support for multiple contexts, find an unused 331 * context. 332 */ 333 domain->context_id = 0; 334 335 /* TTBR0 */ 336 ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0]; 337 ipmmu_ctx_write(domain, IMTTLBR0, ttbr); 338 ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32); 339 340 /* 341 * TTBCR 342 * We use long descriptors with inner-shareable WBWA tables and allocate 343 * the whole 32-bit VA space to TTBR0. 344 */ 345 ipmmu_ctx_write(domain, IMTTBCR, IMTTBCR_EAE | 346 IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | 347 IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1); 348 349 /* MAIR0 */ 350 ipmmu_ctx_write(domain, IMMAIR0, domain->cfg.arm_lpae_s1_cfg.mair[0]); 351 352 /* IMBUSCR */ 353 ipmmu_ctx_write(domain, IMBUSCR, 354 ipmmu_ctx_read(domain, IMBUSCR) & 355 ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); 356 357 /* 358 * IMSTR 359 * Clear all interrupt flags. 360 */ 361 ipmmu_ctx_write(domain, IMSTR, ipmmu_ctx_read(domain, IMSTR)); 362 363 /* 364 * IMCTR 365 * Enable the MMU and interrupt generation. The long-descriptor 366 * translation table format doesn't use TEX remapping. Don't enable AF 367 * software management as we have no use for it. Flush the TLB as 368 * required when modifying the context registers. 369 */ 370 ipmmu_ctx_write(domain, IMCTR, IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); 371 372 return 0; 373 } 374 375 static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) 376 { 377 /* 378 * Disable the context. Flush the TLB as required when modifying the 379 * context registers. 380 * 381 * TODO: Is TLB flush really needed ? 382 */ 383 ipmmu_ctx_write(domain, IMCTR, IMCTR_FLUSH); 384 ipmmu_tlb_sync(domain); 385 } 386 387 /* ----------------------------------------------------------------------------- 388 * Fault Handling 389 */ 390 391 static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) 392 { 393 const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF; 394 struct ipmmu_vmsa_device *mmu = domain->mmu; 395 u32 status; 396 u32 iova; 397 398 status = ipmmu_ctx_read(domain, IMSTR); 399 if (!(status & err_mask)) 400 return IRQ_NONE; 401 402 iova = ipmmu_ctx_read(domain, IMEAR); 403 404 /* 405 * Clear the error status flags. Unlike traditional interrupt flag 406 * registers that must be cleared by writing 1, this status register 407 * seems to require 0. The error address register must be read before, 408 * otherwise its value will be 0. 409 */ 410 ipmmu_ctx_write(domain, IMSTR, 0); 411 412 /* Log fatal errors. */ 413 if (status & IMSTR_MHIT) 414 dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%08x\n", 415 iova); 416 if (status & IMSTR_ABORT) 417 dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%08x\n", 418 iova); 419 420 if (!(status & (IMSTR_PF | IMSTR_TF))) 421 return IRQ_NONE; 422 423 /* 424 * Try to handle page faults and translation faults. 425 * 426 * TODO: We need to look up the faulty device based on the I/O VA. Use 427 * the IOMMU device for now. 428 */ 429 if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0)) 430 return IRQ_HANDLED; 431 432 dev_err_ratelimited(mmu->dev, 433 "Unhandled fault: status 0x%08x iova 0x%08x\n", 434 status, iova); 435 436 return IRQ_HANDLED; 437 } 438 439 static irqreturn_t ipmmu_irq(int irq, void *dev) 440 { 441 struct ipmmu_vmsa_device *mmu = dev; 442 struct iommu_domain *io_domain; 443 struct ipmmu_vmsa_domain *domain; 444 445 if (!mmu->mapping) 446 return IRQ_NONE; 447 448 io_domain = mmu->mapping->domain; 449 domain = to_vmsa_domain(io_domain); 450 451 return ipmmu_domain_irq(domain); 452 } 453 454 /* ----------------------------------------------------------------------------- 455 * IOMMU Operations 456 */ 457 458 static struct iommu_domain *ipmmu_domain_alloc(unsigned type) 459 { 460 struct ipmmu_vmsa_domain *domain; 461 462 if (type != IOMMU_DOMAIN_UNMANAGED) 463 return NULL; 464 465 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 466 if (!domain) 467 return NULL; 468 469 spin_lock_init(&domain->lock); 470 471 return &domain->io_domain; 472 } 473 474 static void ipmmu_domain_free(struct iommu_domain *io_domain) 475 { 476 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 477 478 /* 479 * Free the domain resources. We assume that all devices have already 480 * been detached. 481 */ 482 ipmmu_domain_destroy_context(domain); 483 free_io_pgtable_ops(domain->iop); 484 kfree(domain); 485 } 486 487 static int ipmmu_attach_device(struct iommu_domain *io_domain, 488 struct device *dev) 489 { 490 struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; 491 struct ipmmu_vmsa_device *mmu = archdata->mmu; 492 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 493 unsigned long flags; 494 unsigned int i; 495 int ret = 0; 496 497 if (!mmu) { 498 dev_err(dev, "Cannot attach to IPMMU\n"); 499 return -ENXIO; 500 } 501 502 spin_lock_irqsave(&domain->lock, flags); 503 504 if (!domain->mmu) { 505 /* The domain hasn't been used yet, initialize it. */ 506 domain->mmu = mmu; 507 ret = ipmmu_domain_init_context(domain); 508 } else if (domain->mmu != mmu) { 509 /* 510 * Something is wrong, we can't attach two devices using 511 * different IOMMUs to the same domain. 512 */ 513 dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n", 514 dev_name(mmu->dev), dev_name(domain->mmu->dev)); 515 ret = -EINVAL; 516 } 517 518 spin_unlock_irqrestore(&domain->lock, flags); 519 520 if (ret < 0) 521 return ret; 522 523 for (i = 0; i < archdata->num_utlbs; ++i) 524 ipmmu_utlb_enable(domain, archdata->utlbs[i]); 525 526 return 0; 527 } 528 529 static void ipmmu_detach_device(struct iommu_domain *io_domain, 530 struct device *dev) 531 { 532 struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; 533 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 534 unsigned int i; 535 536 for (i = 0; i < archdata->num_utlbs; ++i) 537 ipmmu_utlb_disable(domain, archdata->utlbs[i]); 538 539 /* 540 * TODO: Optimize by disabling the context when no device is attached. 541 */ 542 } 543 544 static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, 545 phys_addr_t paddr, size_t size, int prot) 546 { 547 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 548 549 if (!domain) 550 return -ENODEV; 551 552 return domain->iop->map(domain->iop, iova, paddr, size, prot); 553 } 554 555 static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, 556 size_t size) 557 { 558 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 559 560 return domain->iop->unmap(domain->iop, iova, size); 561 } 562 563 static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, 564 dma_addr_t iova) 565 { 566 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 567 568 /* TODO: Is locking needed ? */ 569 570 return domain->iop->iova_to_phys(domain->iop, iova); 571 } 572 573 static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev, 574 unsigned int *utlbs, unsigned int num_utlbs) 575 { 576 unsigned int i; 577 578 for (i = 0; i < num_utlbs; ++i) { 579 struct of_phandle_args args; 580 int ret; 581 582 ret = of_parse_phandle_with_args(dev->of_node, "iommus", 583 "#iommu-cells", i, &args); 584 if (ret < 0) 585 return ret; 586 587 of_node_put(args.np); 588 589 if (args.np != mmu->dev->of_node || args.args_count != 1) 590 return -EINVAL; 591 592 utlbs[i] = args.args[0]; 593 } 594 595 return 0; 596 } 597 598 static int ipmmu_add_device(struct device *dev) 599 { 600 struct ipmmu_vmsa_archdata *archdata; 601 struct ipmmu_vmsa_device *mmu; 602 struct iommu_group *group = NULL; 603 unsigned int *utlbs; 604 unsigned int i; 605 int num_utlbs; 606 int ret = -ENODEV; 607 608 if (dev->archdata.iommu) { 609 dev_warn(dev, "IOMMU driver already assigned to device %s\n", 610 dev_name(dev)); 611 return -EINVAL; 612 } 613 614 /* Find the master corresponding to the device. */ 615 616 num_utlbs = of_count_phandle_with_args(dev->of_node, "iommus", 617 "#iommu-cells"); 618 if (num_utlbs < 0) 619 return -ENODEV; 620 621 utlbs = kcalloc(num_utlbs, sizeof(*utlbs), GFP_KERNEL); 622 if (!utlbs) 623 return -ENOMEM; 624 625 spin_lock(&ipmmu_devices_lock); 626 627 list_for_each_entry(mmu, &ipmmu_devices, list) { 628 ret = ipmmu_find_utlbs(mmu, dev, utlbs, num_utlbs); 629 if (!ret) { 630 /* 631 * TODO Take a reference to the MMU to protect 632 * against device removal. 633 */ 634 break; 635 } 636 } 637 638 spin_unlock(&ipmmu_devices_lock); 639 640 if (ret < 0) 641 goto error; 642 643 for (i = 0; i < num_utlbs; ++i) { 644 if (utlbs[i] >= mmu->num_utlbs) { 645 ret = -EINVAL; 646 goto error; 647 } 648 } 649 650 /* Create a device group and add the device to it. */ 651 group = iommu_group_alloc(); 652 if (IS_ERR(group)) { 653 dev_err(dev, "Failed to allocate IOMMU group\n"); 654 ret = PTR_ERR(group); 655 goto error; 656 } 657 658 ret = iommu_group_add_device(group, dev); 659 iommu_group_put(group); 660 661 if (ret < 0) { 662 dev_err(dev, "Failed to add device to IPMMU group\n"); 663 group = NULL; 664 goto error; 665 } 666 667 archdata = kzalloc(sizeof(*archdata), GFP_KERNEL); 668 if (!archdata) { 669 ret = -ENOMEM; 670 goto error; 671 } 672 673 archdata->mmu = mmu; 674 archdata->utlbs = utlbs; 675 archdata->num_utlbs = num_utlbs; 676 dev->archdata.iommu = archdata; 677 678 /* 679 * Create the ARM mapping, used by the ARM DMA mapping core to allocate 680 * VAs. This will allocate a corresponding IOMMU domain. 681 * 682 * TODO: 683 * - Create one mapping per context (TLB). 684 * - Make the mapping size configurable ? We currently use a 2GB mapping 685 * at a 1GB offset to ensure that NULL VAs will fault. 686 */ 687 if (!mmu->mapping) { 688 struct dma_iommu_mapping *mapping; 689 690 mapping = arm_iommu_create_mapping(&platform_bus_type, 691 SZ_1G, SZ_2G); 692 if (IS_ERR(mapping)) { 693 dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); 694 ret = PTR_ERR(mapping); 695 goto error; 696 } 697 698 mmu->mapping = mapping; 699 } 700 701 /* Attach the ARM VA mapping to the device. */ 702 ret = arm_iommu_attach_device(dev, mmu->mapping); 703 if (ret < 0) { 704 dev_err(dev, "Failed to attach device to VA mapping\n"); 705 goto error; 706 } 707 708 return 0; 709 710 error: 711 arm_iommu_release_mapping(mmu->mapping); 712 713 kfree(dev->archdata.iommu); 714 kfree(utlbs); 715 716 dev->archdata.iommu = NULL; 717 718 if (!IS_ERR_OR_NULL(group)) 719 iommu_group_remove_device(dev); 720 721 return ret; 722 } 723 724 static void ipmmu_remove_device(struct device *dev) 725 { 726 struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; 727 728 arm_iommu_detach_device(dev); 729 iommu_group_remove_device(dev); 730 731 kfree(archdata->utlbs); 732 kfree(archdata); 733 734 dev->archdata.iommu = NULL; 735 } 736 737 static const struct iommu_ops ipmmu_ops = { 738 .domain_alloc = ipmmu_domain_alloc, 739 .domain_free = ipmmu_domain_free, 740 .attach_dev = ipmmu_attach_device, 741 .detach_dev = ipmmu_detach_device, 742 .map = ipmmu_map, 743 .unmap = ipmmu_unmap, 744 .map_sg = default_iommu_map_sg, 745 .iova_to_phys = ipmmu_iova_to_phys, 746 .add_device = ipmmu_add_device, 747 .remove_device = ipmmu_remove_device, 748 .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, 749 }; 750 751 /* ----------------------------------------------------------------------------- 752 * Probe/remove and init 753 */ 754 755 static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) 756 { 757 unsigned int i; 758 759 /* Disable all contexts. */ 760 for (i = 0; i < 4; ++i) 761 ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0); 762 } 763 764 static int ipmmu_probe(struct platform_device *pdev) 765 { 766 struct ipmmu_vmsa_device *mmu; 767 struct resource *res; 768 int irq; 769 int ret; 770 771 if (!IS_ENABLED(CONFIG_OF) && !pdev->dev.platform_data) { 772 dev_err(&pdev->dev, "missing platform data\n"); 773 return -EINVAL; 774 } 775 776 mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); 777 if (!mmu) { 778 dev_err(&pdev->dev, "cannot allocate device data\n"); 779 return -ENOMEM; 780 } 781 782 mmu->dev = &pdev->dev; 783 mmu->num_utlbs = 32; 784 785 /* Map I/O memory and request IRQ. */ 786 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 787 mmu->base = devm_ioremap_resource(&pdev->dev, res); 788 if (IS_ERR(mmu->base)) 789 return PTR_ERR(mmu->base); 790 791 /* 792 * The IPMMU has two register banks, for secure and non-secure modes. 793 * The bank mapped at the beginning of the IPMMU address space 794 * corresponds to the running mode of the CPU. When running in secure 795 * mode the non-secure register bank is also available at an offset. 796 * 797 * Secure mode operation isn't clearly documented and is thus currently 798 * not implemented in the driver. Furthermore, preliminary tests of 799 * non-secure operation with the main register bank were not successful. 800 * Offset the registers base unconditionally to point to the non-secure 801 * alias space for now. 802 */ 803 mmu->base += IM_NS_ALIAS_OFFSET; 804 805 irq = platform_get_irq(pdev, 0); 806 if (irq < 0) { 807 dev_err(&pdev->dev, "no IRQ found\n"); 808 return irq; 809 } 810 811 ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, 812 dev_name(&pdev->dev), mmu); 813 if (ret < 0) { 814 dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); 815 return ret; 816 } 817 818 ipmmu_device_reset(mmu); 819 820 /* 821 * We can't create the ARM mapping here as it requires the bus to have 822 * an IOMMU, which only happens when bus_set_iommu() is called in 823 * ipmmu_init() after the probe function returns. 824 */ 825 826 spin_lock(&ipmmu_devices_lock); 827 list_add(&mmu->list, &ipmmu_devices); 828 spin_unlock(&ipmmu_devices_lock); 829 830 platform_set_drvdata(pdev, mmu); 831 832 return 0; 833 } 834 835 static int ipmmu_remove(struct platform_device *pdev) 836 { 837 struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev); 838 839 spin_lock(&ipmmu_devices_lock); 840 list_del(&mmu->list); 841 spin_unlock(&ipmmu_devices_lock); 842 843 arm_iommu_release_mapping(mmu->mapping); 844 845 ipmmu_device_reset(mmu); 846 847 return 0; 848 } 849 850 static const struct of_device_id ipmmu_of_ids[] = { 851 { .compatible = "renesas,ipmmu-vmsa", }, 852 { } 853 }; 854 855 static struct platform_driver ipmmu_driver = { 856 .driver = { 857 .name = "ipmmu-vmsa", 858 .of_match_table = of_match_ptr(ipmmu_of_ids), 859 }, 860 .probe = ipmmu_probe, 861 .remove = ipmmu_remove, 862 }; 863 864 static int __init ipmmu_init(void) 865 { 866 int ret; 867 868 ret = platform_driver_register(&ipmmu_driver); 869 if (ret < 0) 870 return ret; 871 872 if (!iommu_present(&platform_bus_type)) 873 bus_set_iommu(&platform_bus_type, &ipmmu_ops); 874 875 return 0; 876 } 877 878 static void __exit ipmmu_exit(void) 879 { 880 return platform_driver_unregister(&ipmmu_driver); 881 } 882 883 subsys_initcall(ipmmu_init); 884 module_exit(ipmmu_exit); 885 886 MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU"); 887 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>"); 888 MODULE_LICENSE("GPL v2"); 889