1 /* 2 * IPMMU VMSA 3 * 4 * Copyright (C) 2014 Renesas Electronics Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; version 2 of the License. 9 */ 10 11 #include <linux/delay.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/err.h> 14 #include <linux/export.h> 15 #include <linux/interrupt.h> 16 #include <linux/io.h> 17 #include <linux/iommu.h> 18 #include <linux/module.h> 19 #include <linux/platform_data/ipmmu-vmsa.h> 20 #include <linux/platform_device.h> 21 #include <linux/sizes.h> 22 #include <linux/slab.h> 23 24 #include <asm/dma-iommu.h> 25 #include <asm/pgalloc.h> 26 27 struct ipmmu_vmsa_device { 28 struct device *dev; 29 void __iomem *base; 30 struct list_head list; 31 32 const struct ipmmu_vmsa_platform_data *pdata; 33 unsigned int num_utlbs; 34 35 struct dma_iommu_mapping *mapping; 36 }; 37 38 struct ipmmu_vmsa_domain { 39 struct ipmmu_vmsa_device *mmu; 40 struct iommu_domain *io_domain; 41 42 unsigned int context_id; 43 spinlock_t lock; /* Protects mappings */ 44 pgd_t *pgd; 45 }; 46 47 struct ipmmu_vmsa_archdata { 48 struct ipmmu_vmsa_device *mmu; 49 unsigned int utlb; 50 }; 51 52 static DEFINE_SPINLOCK(ipmmu_devices_lock); 53 static LIST_HEAD(ipmmu_devices); 54 55 #define TLB_LOOP_TIMEOUT 100 /* 100us */ 56 57 /* ----------------------------------------------------------------------------- 58 * Registers Definition 59 */ 60 61 #define IM_CTX_SIZE 0x40 62 63 #define IMCTR 0x0000 64 #define IMCTR_TRE (1 << 17) 65 #define IMCTR_AFE (1 << 16) 66 #define IMCTR_RTSEL_MASK (3 << 4) 67 #define IMCTR_RTSEL_SHIFT 4 68 #define IMCTR_TREN (1 << 3) 69 #define IMCTR_INTEN (1 << 2) 70 #define IMCTR_FLUSH (1 << 1) 71 #define IMCTR_MMUEN (1 << 0) 72 73 #define IMCAAR 0x0004 74 75 #define IMTTBCR 0x0008 76 #define IMTTBCR_EAE (1 << 31) 77 #define IMTTBCR_PMB (1 << 30) 78 #define IMTTBCR_SH1_NON_SHAREABLE (0 << 28) 79 #define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28) 80 #define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28) 81 #define IMTTBCR_SH1_MASK (3 << 28) 82 #define IMTTBCR_ORGN1_NC (0 << 26) 83 #define IMTTBCR_ORGN1_WB_WA (1 << 26) 84 #define IMTTBCR_ORGN1_WT (2 << 26) 85 #define IMTTBCR_ORGN1_WB (3 << 26) 86 #define IMTTBCR_ORGN1_MASK (3 << 26) 87 #define IMTTBCR_IRGN1_NC (0 << 24) 88 #define IMTTBCR_IRGN1_WB_WA (1 << 24) 89 #define IMTTBCR_IRGN1_WT (2 << 24) 90 #define IMTTBCR_IRGN1_WB (3 << 24) 91 #define IMTTBCR_IRGN1_MASK (3 << 24) 92 #define IMTTBCR_TSZ1_MASK (7 << 16) 93 #define IMTTBCR_TSZ1_SHIFT 16 94 #define IMTTBCR_SH0_NON_SHAREABLE (0 << 12) 95 #define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12) 96 #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) 97 #define IMTTBCR_SH0_MASK (3 << 12) 98 #define IMTTBCR_ORGN0_NC (0 << 10) 99 #define IMTTBCR_ORGN0_WB_WA (1 << 10) 100 #define IMTTBCR_ORGN0_WT (2 << 10) 101 #define IMTTBCR_ORGN0_WB (3 << 10) 102 #define IMTTBCR_ORGN0_MASK (3 << 10) 103 #define IMTTBCR_IRGN0_NC (0 << 8) 104 #define IMTTBCR_IRGN0_WB_WA (1 << 8) 105 #define IMTTBCR_IRGN0_WT (2 << 8) 106 #define IMTTBCR_IRGN0_WB (3 << 8) 107 #define IMTTBCR_IRGN0_MASK (3 << 8) 108 #define IMTTBCR_SL0_LVL_2 (0 << 4) 109 #define IMTTBCR_SL0_LVL_1 (1 << 4) 110 #define IMTTBCR_TSZ0_MASK (7 << 0) 111 #define IMTTBCR_TSZ0_SHIFT O 112 113 #define IMBUSCR 0x000c 114 #define IMBUSCR_DVM (1 << 2) 115 #define IMBUSCR_BUSSEL_SYS (0 << 0) 116 #define IMBUSCR_BUSSEL_CCI (1 << 0) 117 #define IMBUSCR_BUSSEL_IMCAAR (2 << 0) 118 #define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0) 119 #define IMBUSCR_BUSSEL_MASK (3 << 0) 120 121 #define IMTTLBR0 0x0010 122 #define IMTTUBR0 0x0014 123 #define IMTTLBR1 0x0018 124 #define IMTTUBR1 0x001c 125 126 #define IMSTR 0x0020 127 #define IMSTR_ERRLVL_MASK (3 << 12) 128 #define IMSTR_ERRLVL_SHIFT 12 129 #define IMSTR_ERRCODE_TLB_FORMAT (1 << 8) 130 #define IMSTR_ERRCODE_ACCESS_PERM (4 << 8) 131 #define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8) 132 #define IMSTR_ERRCODE_MASK (7 << 8) 133 #define IMSTR_MHIT (1 << 4) 134 #define IMSTR_ABORT (1 << 2) 135 #define IMSTR_PF (1 << 1) 136 #define IMSTR_TF (1 << 0) 137 138 #define IMMAIR0 0x0028 139 #define IMMAIR1 0x002c 140 #define IMMAIR_ATTR_MASK 0xff 141 #define IMMAIR_ATTR_DEVICE 0x04 142 #define IMMAIR_ATTR_NC 0x44 143 #define IMMAIR_ATTR_WBRWA 0xff 144 #define IMMAIR_ATTR_SHIFT(n) ((n) << 3) 145 #define IMMAIR_ATTR_IDX_NC 0 146 #define IMMAIR_ATTR_IDX_WBRWA 1 147 #define IMMAIR_ATTR_IDX_DEV 2 148 149 #define IMEAR 0x0030 150 151 #define IMPCTR 0x0200 152 #define IMPSTR 0x0208 153 #define IMPEAR 0x020c 154 #define IMPMBA(n) (0x0280 + ((n) * 4)) 155 #define IMPMBD(n) (0x02c0 + ((n) * 4)) 156 157 #define IMUCTR(n) (0x0300 + ((n) * 16)) 158 #define IMUCTR_FIXADDEN (1 << 31) 159 #define IMUCTR_FIXADD_MASK (0xff << 16) 160 #define IMUCTR_FIXADD_SHIFT 16 161 #define IMUCTR_TTSEL_MMU(n) ((n) << 4) 162 #define IMUCTR_TTSEL_PMB (8 << 4) 163 #define IMUCTR_TTSEL_MASK (15 << 4) 164 #define IMUCTR_FLUSH (1 << 1) 165 #define IMUCTR_MMUEN (1 << 0) 166 167 #define IMUASID(n) (0x0308 + ((n) * 16)) 168 #define IMUASID_ASID8_MASK (0xff << 8) 169 #define IMUASID_ASID8_SHIFT 8 170 #define IMUASID_ASID0_MASK (0xff << 0) 171 #define IMUASID_ASID0_SHIFT 0 172 173 /* ----------------------------------------------------------------------------- 174 * Page Table Bits 175 */ 176 177 /* 178 * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory access, 179 * Long-descriptor format" that the NStable bit being set in a table descriptor 180 * will result in the NStable and NS bits of all child entries being ignored and 181 * considered as being set. The IPMMU seems not to comply with this, as it 182 * generates a secure access page fault if any of the NStable and NS bits isn't 183 * set when running in non-secure mode. 184 */ 185 #ifndef PMD_NSTABLE 186 #define PMD_NSTABLE (_AT(pmdval_t, 1) << 63) 187 #endif 188 189 #define ARM_VMSA_PTE_XN (((pteval_t)3) << 53) 190 #define ARM_VMSA_PTE_CONT (((pteval_t)1) << 52) 191 #define ARM_VMSA_PTE_AF (((pteval_t)1) << 10) 192 #define ARM_VMSA_PTE_SH_NS (((pteval_t)0) << 8) 193 #define ARM_VMSA_PTE_SH_OS (((pteval_t)2) << 8) 194 #define ARM_VMSA_PTE_SH_IS (((pteval_t)3) << 8) 195 #define ARM_VMSA_PTE_SH_MASK (((pteval_t)3) << 8) 196 #define ARM_VMSA_PTE_NS (((pteval_t)1) << 5) 197 #define ARM_VMSA_PTE_PAGE (((pteval_t)3) << 0) 198 199 /* Stage-1 PTE */ 200 #define ARM_VMSA_PTE_nG (((pteval_t)1) << 11) 201 #define ARM_VMSA_PTE_AP_UNPRIV (((pteval_t)1) << 6) 202 #define ARM_VMSA_PTE_AP_RDONLY (((pteval_t)2) << 6) 203 #define ARM_VMSA_PTE_AP_MASK (((pteval_t)3) << 6) 204 #define ARM_VMSA_PTE_ATTRINDX_MASK (((pteval_t)3) << 2) 205 #define ARM_VMSA_PTE_ATTRINDX_SHIFT 2 206 207 #define ARM_VMSA_PTE_ATTRS_MASK \ 208 (ARM_VMSA_PTE_XN | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_nG | \ 209 ARM_VMSA_PTE_AF | ARM_VMSA_PTE_SH_MASK | ARM_VMSA_PTE_AP_MASK | \ 210 ARM_VMSA_PTE_NS | ARM_VMSA_PTE_ATTRINDX_MASK) 211 212 #define ARM_VMSA_PTE_CONT_ENTRIES 16 213 #define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES) 214 215 #define IPMMU_PTRS_PER_PTE 512 216 #define IPMMU_PTRS_PER_PMD 512 217 #define IPMMU_PTRS_PER_PGD 4 218 219 /* ----------------------------------------------------------------------------- 220 * Read/Write Access 221 */ 222 223 static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset) 224 { 225 return ioread32(mmu->base + offset); 226 } 227 228 static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, 229 u32 data) 230 { 231 iowrite32(data, mmu->base + offset); 232 } 233 234 static u32 ipmmu_ctx_read(struct ipmmu_vmsa_domain *domain, unsigned int reg) 235 { 236 return ipmmu_read(domain->mmu, domain->context_id * IM_CTX_SIZE + reg); 237 } 238 239 static void ipmmu_ctx_write(struct ipmmu_vmsa_domain *domain, unsigned int reg, 240 u32 data) 241 { 242 ipmmu_write(domain->mmu, domain->context_id * IM_CTX_SIZE + reg, data); 243 } 244 245 /* ----------------------------------------------------------------------------- 246 * TLB and microTLB Management 247 */ 248 249 /* Wait for any pending TLB invalidations to complete */ 250 static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain) 251 { 252 unsigned int count = 0; 253 254 while (ipmmu_ctx_read(domain, IMCTR) & IMCTR_FLUSH) { 255 cpu_relax(); 256 if (++count == TLB_LOOP_TIMEOUT) { 257 dev_err_ratelimited(domain->mmu->dev, 258 "TLB sync timed out -- MMU may be deadlocked\n"); 259 return; 260 } 261 udelay(1); 262 } 263 } 264 265 static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain) 266 { 267 u32 reg; 268 269 reg = ipmmu_ctx_read(domain, IMCTR); 270 reg |= IMCTR_FLUSH; 271 ipmmu_ctx_write(domain, IMCTR, reg); 272 273 ipmmu_tlb_sync(domain); 274 } 275 276 /* 277 * Enable MMU translation for the microTLB. 278 */ 279 static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, 280 unsigned int utlb) 281 { 282 struct ipmmu_vmsa_device *mmu = domain->mmu; 283 284 /* 285 * TODO: Reference-count the microTLB as several bus masters can be 286 * connected to the same microTLB. 287 */ 288 289 /* TODO: What should we set the ASID to ? */ 290 ipmmu_write(mmu, IMUASID(utlb), 0); 291 /* TODO: Do we need to flush the microTLB ? */ 292 ipmmu_write(mmu, IMUCTR(utlb), 293 IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH | 294 IMUCTR_MMUEN); 295 } 296 297 /* 298 * Disable MMU translation for the microTLB. 299 */ 300 static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, 301 unsigned int utlb) 302 { 303 struct ipmmu_vmsa_device *mmu = domain->mmu; 304 305 ipmmu_write(mmu, IMUCTR(utlb), 0); 306 } 307 308 static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device *mmu, void *addr, 309 size_t size) 310 { 311 unsigned long offset = (unsigned long)addr & ~PAGE_MASK; 312 313 /* 314 * TODO: Add support for coherent walk through CCI with DVM and remove 315 * cache handling. 316 */ 317 dma_map_page(mmu->dev, virt_to_page(addr), offset, size, DMA_TO_DEVICE); 318 } 319 320 /* ----------------------------------------------------------------------------- 321 * Domain/Context Management 322 */ 323 324 static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) 325 { 326 phys_addr_t ttbr; 327 u32 reg; 328 329 /* 330 * TODO: When adding support for multiple contexts, find an unused 331 * context. 332 */ 333 domain->context_id = 0; 334 335 /* TTBR0 */ 336 ipmmu_flush_pgtable(domain->mmu, domain->pgd, 337 IPMMU_PTRS_PER_PGD * sizeof(*domain->pgd)); 338 ttbr = __pa(domain->pgd); 339 ipmmu_ctx_write(domain, IMTTLBR0, ttbr); 340 ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32); 341 342 /* 343 * TTBCR 344 * We use long descriptors with inner-shareable WBWA tables and allocate 345 * the whole 32-bit VA space to TTBR0. 346 */ 347 ipmmu_ctx_write(domain, IMTTBCR, IMTTBCR_EAE | 348 IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | 349 IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1); 350 351 /* 352 * MAIR0 353 * We need three attributes only, non-cacheable, write-back read/write 354 * allocate and device memory. 355 */ 356 reg = (IMMAIR_ATTR_NC << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_NC)) 357 | (IMMAIR_ATTR_WBRWA << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_WBRWA)) 358 | (IMMAIR_ATTR_DEVICE << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_DEV)); 359 ipmmu_ctx_write(domain, IMMAIR0, reg); 360 361 /* IMBUSCR */ 362 ipmmu_ctx_write(domain, IMBUSCR, 363 ipmmu_ctx_read(domain, IMBUSCR) & 364 ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); 365 366 /* 367 * IMSTR 368 * Clear all interrupt flags. 369 */ 370 ipmmu_ctx_write(domain, IMSTR, ipmmu_ctx_read(domain, IMSTR)); 371 372 /* 373 * IMCTR 374 * Enable the MMU and interrupt generation. The long-descriptor 375 * translation table format doesn't use TEX remapping. Don't enable AF 376 * software management as we have no use for it. Flush the TLB as 377 * required when modifying the context registers. 378 */ 379 ipmmu_ctx_write(domain, IMCTR, IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); 380 381 return 0; 382 } 383 384 static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) 385 { 386 /* 387 * Disable the context. Flush the TLB as required when modifying the 388 * context registers. 389 * 390 * TODO: Is TLB flush really needed ? 391 */ 392 ipmmu_ctx_write(domain, IMCTR, IMCTR_FLUSH); 393 ipmmu_tlb_sync(domain); 394 } 395 396 /* ----------------------------------------------------------------------------- 397 * Fault Handling 398 */ 399 400 static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) 401 { 402 const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF; 403 struct ipmmu_vmsa_device *mmu = domain->mmu; 404 u32 status; 405 u32 iova; 406 407 status = ipmmu_ctx_read(domain, IMSTR); 408 if (!(status & err_mask)) 409 return IRQ_NONE; 410 411 iova = ipmmu_ctx_read(domain, IMEAR); 412 413 /* 414 * Clear the error status flags. Unlike traditional interrupt flag 415 * registers that must be cleared by writing 1, this status register 416 * seems to require 0. The error address register must be read before, 417 * otherwise its value will be 0. 418 */ 419 ipmmu_ctx_write(domain, IMSTR, 0); 420 421 /* Log fatal errors. */ 422 if (status & IMSTR_MHIT) 423 dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%08x\n", 424 iova); 425 if (status & IMSTR_ABORT) 426 dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%08x\n", 427 iova); 428 429 if (!(status & (IMSTR_PF | IMSTR_TF))) 430 return IRQ_NONE; 431 432 /* 433 * Try to handle page faults and translation faults. 434 * 435 * TODO: We need to look up the faulty device based on the I/O VA. Use 436 * the IOMMU device for now. 437 */ 438 if (!report_iommu_fault(domain->io_domain, mmu->dev, iova, 0)) 439 return IRQ_HANDLED; 440 441 dev_err_ratelimited(mmu->dev, 442 "Unhandled fault: status 0x%08x iova 0x%08x\n", 443 status, iova); 444 445 return IRQ_HANDLED; 446 } 447 448 static irqreturn_t ipmmu_irq(int irq, void *dev) 449 { 450 struct ipmmu_vmsa_device *mmu = dev; 451 struct iommu_domain *io_domain; 452 struct ipmmu_vmsa_domain *domain; 453 454 if (!mmu->mapping) 455 return IRQ_NONE; 456 457 io_domain = mmu->mapping->domain; 458 domain = io_domain->priv; 459 460 return ipmmu_domain_irq(domain); 461 } 462 463 /* ----------------------------------------------------------------------------- 464 * Page Table Management 465 */ 466 467 #define pud_pgtable(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK)) 468 469 static void ipmmu_free_ptes(pmd_t *pmd) 470 { 471 pgtable_t table = pmd_pgtable(*pmd); 472 __free_page(table); 473 } 474 475 static void ipmmu_free_pmds(pud_t *pud) 476 { 477 pmd_t *pmd = pmd_offset(pud, 0); 478 pgtable_t table; 479 unsigned int i; 480 481 for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { 482 if (!pmd_table(*pmd)) 483 continue; 484 485 ipmmu_free_ptes(pmd); 486 pmd++; 487 } 488 489 table = pud_pgtable(*pud); 490 __free_page(table); 491 } 492 493 static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain) 494 { 495 pgd_t *pgd, *pgd_base = domain->pgd; 496 unsigned int i; 497 498 /* 499 * Recursively free the page tables for this domain. We don't care about 500 * speculative TLB filling, because the TLB will be nuked next time this 501 * context bank is re-allocated and no devices currently map to these 502 * tables. 503 */ 504 pgd = pgd_base; 505 for (i = 0; i < IPMMU_PTRS_PER_PGD; ++i) { 506 if (pgd_none(*pgd)) 507 continue; 508 ipmmu_free_pmds((pud_t *)pgd); 509 pgd++; 510 } 511 512 kfree(pgd_base); 513 } 514 515 /* 516 * We can't use the (pgd|pud|pmd|pte)_populate or the set_(pgd|pud|pmd|pte) 517 * functions as they would flush the CPU TLB. 518 */ 519 520 static pte_t *ipmmu_alloc_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, 521 unsigned long iova) 522 { 523 pte_t *pte; 524 525 if (!pmd_none(*pmd)) 526 return pte_offset_kernel(pmd, iova); 527 528 pte = (pte_t *)get_zeroed_page(GFP_ATOMIC); 529 if (!pte) 530 return NULL; 531 532 ipmmu_flush_pgtable(mmu, pte, PAGE_SIZE); 533 *pmd = __pmd(__pa(pte) | PMD_NSTABLE | PMD_TYPE_TABLE); 534 ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); 535 536 return pte + pte_index(iova); 537 } 538 539 static pmd_t *ipmmu_alloc_pmd(struct ipmmu_vmsa_device *mmu, pgd_t *pgd, 540 unsigned long iova) 541 { 542 pud_t *pud = (pud_t *)pgd; 543 pmd_t *pmd; 544 545 if (!pud_none(*pud)) 546 return pmd_offset(pud, iova); 547 548 pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC); 549 if (!pmd) 550 return NULL; 551 552 ipmmu_flush_pgtable(mmu, pmd, PAGE_SIZE); 553 *pud = __pud(__pa(pmd) | PMD_NSTABLE | PMD_TYPE_TABLE); 554 ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); 555 556 return pmd + pmd_index(iova); 557 } 558 559 static u64 ipmmu_page_prot(unsigned int prot, u64 type) 560 { 561 u64 pgprot = ARM_VMSA_PTE_XN | ARM_VMSA_PTE_nG | ARM_VMSA_PTE_AF 562 | ARM_VMSA_PTE_SH_IS | ARM_VMSA_PTE_AP_UNPRIV 563 | ARM_VMSA_PTE_NS | type; 564 565 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) 566 pgprot |= ARM_VMSA_PTE_AP_RDONLY; 567 568 if (prot & IOMMU_CACHE) 569 pgprot |= IMMAIR_ATTR_IDX_WBRWA << ARM_VMSA_PTE_ATTRINDX_SHIFT; 570 571 if (prot & IOMMU_EXEC) 572 pgprot &= ~ARM_VMSA_PTE_XN; 573 else if (!(prot & (IOMMU_READ | IOMMU_WRITE))) 574 /* If no access create a faulting entry to avoid TLB fills. */ 575 pgprot &= ~ARM_VMSA_PTE_PAGE; 576 577 return pgprot; 578 } 579 580 static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, 581 unsigned long iova, unsigned long pfn, 582 size_t size, int prot) 583 { 584 pteval_t pteval = ipmmu_page_prot(prot, ARM_VMSA_PTE_PAGE); 585 unsigned int num_ptes = 1; 586 pte_t *pte, *start; 587 unsigned int i; 588 589 pte = ipmmu_alloc_pte(mmu, pmd, iova); 590 if (!pte) 591 return -ENOMEM; 592 593 start = pte; 594 595 /* 596 * Install the page table entries. We can be called both for a single 597 * page or for a block of 16 physically contiguous pages. In the latter 598 * case set the PTE contiguous hint. 599 */ 600 if (size == SZ_64K) { 601 pteval |= ARM_VMSA_PTE_CONT; 602 num_ptes = ARM_VMSA_PTE_CONT_ENTRIES; 603 } 604 605 for (i = num_ptes; i; --i) 606 *pte++ = pfn_pte(pfn++, __pgprot(pteval)); 607 608 ipmmu_flush_pgtable(mmu, start, sizeof(*pte) * num_ptes); 609 610 return 0; 611 } 612 613 static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, 614 unsigned long iova, unsigned long pfn, 615 int prot) 616 { 617 pmdval_t pmdval = ipmmu_page_prot(prot, PMD_TYPE_SECT); 618 619 *pmd = pfn_pmd(pfn, __pgprot(pmdval)); 620 ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); 621 622 return 0; 623 } 624 625 static int ipmmu_create_mapping(struct ipmmu_vmsa_domain *domain, 626 unsigned long iova, phys_addr_t paddr, 627 size_t size, int prot) 628 { 629 struct ipmmu_vmsa_device *mmu = domain->mmu; 630 pgd_t *pgd = domain->pgd; 631 unsigned long flags; 632 unsigned long pfn; 633 pmd_t *pmd; 634 int ret; 635 636 if (!pgd) 637 return -EINVAL; 638 639 if (size & ~PAGE_MASK) 640 return -EINVAL; 641 642 if (paddr & ~((1ULL << 40) - 1)) 643 return -ERANGE; 644 645 pfn = __phys_to_pfn(paddr); 646 pgd += pgd_index(iova); 647 648 /* Update the page tables. */ 649 spin_lock_irqsave(&domain->lock, flags); 650 651 pmd = ipmmu_alloc_pmd(mmu, pgd, iova); 652 if (!pmd) { 653 ret = -ENOMEM; 654 goto done; 655 } 656 657 switch (size) { 658 case SZ_2M: 659 ret = ipmmu_alloc_init_pmd(mmu, pmd, iova, pfn, prot); 660 break; 661 case SZ_64K: 662 case SZ_4K: 663 ret = ipmmu_alloc_init_pte(mmu, pmd, iova, pfn, size, prot); 664 break; 665 default: 666 ret = -EINVAL; 667 break; 668 } 669 670 done: 671 spin_unlock_irqrestore(&domain->lock, flags); 672 673 if (!ret) 674 ipmmu_tlb_invalidate(domain); 675 676 return ret; 677 } 678 679 static void ipmmu_clear_pud(struct ipmmu_vmsa_device *mmu, pud_t *pud) 680 { 681 /* Free the page table. */ 682 pgtable_t table = pud_pgtable(*pud); 683 __free_page(table); 684 685 /* Clear the PUD. */ 686 *pud = __pud(0); 687 ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); 688 } 689 690 static void ipmmu_clear_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, 691 pmd_t *pmd) 692 { 693 unsigned int i; 694 695 /* Free the page table. */ 696 if (pmd_table(*pmd)) { 697 pgtable_t table = pmd_pgtable(*pmd); 698 __free_page(table); 699 } 700 701 /* Clear the PMD. */ 702 *pmd = __pmd(0); 703 ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); 704 705 /* Check whether the PUD is still needed. */ 706 pmd = pmd_offset(pud, 0); 707 for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { 708 if (!pmd_none(pmd[i])) 709 return; 710 } 711 712 /* Clear the parent PUD. */ 713 ipmmu_clear_pud(mmu, pud); 714 } 715 716 static void ipmmu_clear_pte(struct ipmmu_vmsa_device *mmu, pud_t *pud, 717 pmd_t *pmd, pte_t *pte, unsigned int num_ptes) 718 { 719 unsigned int i; 720 721 /* Clear the PTE. */ 722 for (i = num_ptes; i; --i) 723 pte[i-1] = __pte(0); 724 725 ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * num_ptes); 726 727 /* Check whether the PMD is still needed. */ 728 pte = pte_offset_kernel(pmd, 0); 729 for (i = 0; i < IPMMU_PTRS_PER_PTE; ++i) { 730 if (!pte_none(pte[i])) 731 return; 732 } 733 734 /* Clear the parent PMD. */ 735 ipmmu_clear_pmd(mmu, pud, pmd); 736 } 737 738 static int ipmmu_split_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd) 739 { 740 pte_t *pte, *start; 741 pteval_t pteval; 742 unsigned long pfn; 743 unsigned int i; 744 745 pte = (pte_t *)get_zeroed_page(GFP_ATOMIC); 746 if (!pte) 747 return -ENOMEM; 748 749 /* Copy the PMD attributes. */ 750 pteval = (pmd_val(*pmd) & ARM_VMSA_PTE_ATTRS_MASK) 751 | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_PAGE; 752 753 pfn = pmd_pfn(*pmd); 754 start = pte; 755 756 for (i = IPMMU_PTRS_PER_PTE; i; --i) 757 *pte++ = pfn_pte(pfn++, __pgprot(pteval)); 758 759 ipmmu_flush_pgtable(mmu, start, PAGE_SIZE); 760 *pmd = __pmd(__pa(start) | PMD_NSTABLE | PMD_TYPE_TABLE); 761 ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); 762 763 return 0; 764 } 765 766 static void ipmmu_split_pte(struct ipmmu_vmsa_device *mmu, pte_t *pte) 767 { 768 unsigned int i; 769 770 for (i = ARM_VMSA_PTE_CONT_ENTRIES; i; --i) 771 pte[i-1] = __pte(pte_val(*pte) & ~ARM_VMSA_PTE_CONT); 772 773 ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * ARM_VMSA_PTE_CONT_ENTRIES); 774 } 775 776 static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain *domain, 777 unsigned long iova, size_t size) 778 { 779 struct ipmmu_vmsa_device *mmu = domain->mmu; 780 unsigned long flags; 781 pgd_t *pgd = domain->pgd; 782 pud_t *pud; 783 pmd_t *pmd; 784 pte_t *pte; 785 int ret = 0; 786 787 if (!pgd) 788 return -EINVAL; 789 790 if (size & ~PAGE_MASK) 791 return -EINVAL; 792 793 pgd += pgd_index(iova); 794 pud = (pud_t *)pgd; 795 796 spin_lock_irqsave(&domain->lock, flags); 797 798 /* If there's no PUD or PMD we're done. */ 799 if (pud_none(*pud)) 800 goto done; 801 802 pmd = pmd_offset(pud, iova); 803 if (pmd_none(*pmd)) 804 goto done; 805 806 /* 807 * When freeing a 2MB block just clear the PMD. In the unlikely case the 808 * block is mapped as individual pages this will free the corresponding 809 * PTE page table. 810 */ 811 if (size == SZ_2M) { 812 ipmmu_clear_pmd(mmu, pud, pmd); 813 goto done; 814 } 815 816 /* 817 * If the PMD has been mapped as a section remap it as pages to allow 818 * freeing individual pages. 819 */ 820 if (pmd_sect(*pmd)) 821 ipmmu_split_pmd(mmu, pmd); 822 823 pte = pte_offset_kernel(pmd, iova); 824 825 /* 826 * When freeing a 64kB block just clear the PTE entries. We don't have 827 * to care about the contiguous hint of the surrounding entries. 828 */ 829 if (size == SZ_64K) { 830 ipmmu_clear_pte(mmu, pud, pmd, pte, ARM_VMSA_PTE_CONT_ENTRIES); 831 goto done; 832 } 833 834 /* 835 * If the PTE has been mapped with the contiguous hint set remap it and 836 * its surrounding PTEs to allow unmapping a single page. 837 */ 838 if (pte_val(*pte) & ARM_VMSA_PTE_CONT) 839 ipmmu_split_pte(mmu, pte); 840 841 /* Clear the PTE. */ 842 ipmmu_clear_pte(mmu, pud, pmd, pte, 1); 843 844 done: 845 spin_unlock_irqrestore(&domain->lock, flags); 846 847 if (ret) 848 ipmmu_tlb_invalidate(domain); 849 850 return 0; 851 } 852 853 /* ----------------------------------------------------------------------------- 854 * IOMMU Operations 855 */ 856 857 static int ipmmu_domain_init(struct iommu_domain *io_domain) 858 { 859 struct ipmmu_vmsa_domain *domain; 860 861 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 862 if (!domain) 863 return -ENOMEM; 864 865 spin_lock_init(&domain->lock); 866 867 domain->pgd = kzalloc(IPMMU_PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL); 868 if (!domain->pgd) { 869 kfree(domain); 870 return -ENOMEM; 871 } 872 873 io_domain->priv = domain; 874 domain->io_domain = io_domain; 875 876 return 0; 877 } 878 879 static void ipmmu_domain_destroy(struct iommu_domain *io_domain) 880 { 881 struct ipmmu_vmsa_domain *domain = io_domain->priv; 882 883 /* 884 * Free the domain resources. We assume that all devices have already 885 * been detached. 886 */ 887 ipmmu_domain_destroy_context(domain); 888 ipmmu_free_pgtables(domain); 889 kfree(domain); 890 } 891 892 static int ipmmu_attach_device(struct iommu_domain *io_domain, 893 struct device *dev) 894 { 895 struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; 896 struct ipmmu_vmsa_device *mmu = archdata->mmu; 897 struct ipmmu_vmsa_domain *domain = io_domain->priv; 898 unsigned long flags; 899 int ret = 0; 900 901 if (!mmu) { 902 dev_err(dev, "Cannot attach to IPMMU\n"); 903 return -ENXIO; 904 } 905 906 spin_lock_irqsave(&domain->lock, flags); 907 908 if (!domain->mmu) { 909 /* The domain hasn't been used yet, initialize it. */ 910 domain->mmu = mmu; 911 ret = ipmmu_domain_init_context(domain); 912 } else if (domain->mmu != mmu) { 913 /* 914 * Something is wrong, we can't attach two devices using 915 * different IOMMUs to the same domain. 916 */ 917 dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n", 918 dev_name(mmu->dev), dev_name(domain->mmu->dev)); 919 ret = -EINVAL; 920 } 921 922 spin_unlock_irqrestore(&domain->lock, flags); 923 924 if (ret < 0) 925 return ret; 926 927 ipmmu_utlb_enable(domain, archdata->utlb); 928 929 return 0; 930 } 931 932 static void ipmmu_detach_device(struct iommu_domain *io_domain, 933 struct device *dev) 934 { 935 struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; 936 struct ipmmu_vmsa_domain *domain = io_domain->priv; 937 938 ipmmu_utlb_disable(domain, archdata->utlb); 939 940 /* 941 * TODO: Optimize by disabling the context when no device is attached. 942 */ 943 } 944 945 static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, 946 phys_addr_t paddr, size_t size, int prot) 947 { 948 struct ipmmu_vmsa_domain *domain = io_domain->priv; 949 950 if (!domain) 951 return -ENODEV; 952 953 return ipmmu_create_mapping(domain, iova, paddr, size, prot); 954 } 955 956 static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, 957 size_t size) 958 { 959 struct ipmmu_vmsa_domain *domain = io_domain->priv; 960 int ret; 961 962 ret = ipmmu_clear_mapping(domain, iova, size); 963 return ret ? 0 : size; 964 } 965 966 static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, 967 dma_addr_t iova) 968 { 969 struct ipmmu_vmsa_domain *domain = io_domain->priv; 970 pgd_t pgd; 971 pud_t pud; 972 pmd_t pmd; 973 pte_t pte; 974 975 /* TODO: Is locking needed ? */ 976 977 if (!domain->pgd) 978 return 0; 979 980 pgd = *(domain->pgd + pgd_index(iova)); 981 if (pgd_none(pgd)) 982 return 0; 983 984 pud = *pud_offset(&pgd, iova); 985 if (pud_none(pud)) 986 return 0; 987 988 pmd = *pmd_offset(&pud, iova); 989 if (pmd_none(pmd)) 990 return 0; 991 992 if (pmd_sect(pmd)) 993 return __pfn_to_phys(pmd_pfn(pmd)) | (iova & ~PMD_MASK); 994 995 pte = *(pmd_page_vaddr(pmd) + pte_index(iova)); 996 if (pte_none(pte)) 997 return 0; 998 999 return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK); 1000 } 1001 1002 static int ipmmu_find_utlb(struct ipmmu_vmsa_device *mmu, struct device *dev) 1003 { 1004 const struct ipmmu_vmsa_master *master = mmu->pdata->masters; 1005 const char *devname = dev_name(dev); 1006 unsigned int i; 1007 1008 for (i = 0; i < mmu->pdata->num_masters; ++i, ++master) { 1009 if (strcmp(master->name, devname) == 0) 1010 return master->utlb; 1011 } 1012 1013 return -1; 1014 } 1015 1016 static int ipmmu_add_device(struct device *dev) 1017 { 1018 struct ipmmu_vmsa_archdata *archdata; 1019 struct ipmmu_vmsa_device *mmu; 1020 struct iommu_group *group; 1021 int utlb = -1; 1022 int ret; 1023 1024 if (dev->archdata.iommu) { 1025 dev_warn(dev, "IOMMU driver already assigned to device %s\n", 1026 dev_name(dev)); 1027 return -EINVAL; 1028 } 1029 1030 /* Find the master corresponding to the device. */ 1031 spin_lock(&ipmmu_devices_lock); 1032 1033 list_for_each_entry(mmu, &ipmmu_devices, list) { 1034 utlb = ipmmu_find_utlb(mmu, dev); 1035 if (utlb >= 0) { 1036 /* 1037 * TODO Take a reference to the MMU to protect 1038 * against device removal. 1039 */ 1040 break; 1041 } 1042 } 1043 1044 spin_unlock(&ipmmu_devices_lock); 1045 1046 if (utlb < 0) 1047 return -ENODEV; 1048 1049 if (utlb >= mmu->num_utlbs) 1050 return -EINVAL; 1051 1052 /* Create a device group and add the device to it. */ 1053 group = iommu_group_alloc(); 1054 if (IS_ERR(group)) { 1055 dev_err(dev, "Failed to allocate IOMMU group\n"); 1056 return PTR_ERR(group); 1057 } 1058 1059 ret = iommu_group_add_device(group, dev); 1060 iommu_group_put(group); 1061 1062 if (ret < 0) { 1063 dev_err(dev, "Failed to add device to IPMMU group\n"); 1064 return ret; 1065 } 1066 1067 archdata = kzalloc(sizeof(*archdata), GFP_KERNEL); 1068 if (!archdata) { 1069 ret = -ENOMEM; 1070 goto error; 1071 } 1072 1073 archdata->mmu = mmu; 1074 archdata->utlb = utlb; 1075 dev->archdata.iommu = archdata; 1076 1077 /* 1078 * Create the ARM mapping, used by the ARM DMA mapping core to allocate 1079 * VAs. This will allocate a corresponding IOMMU domain. 1080 * 1081 * TODO: 1082 * - Create one mapping per context (TLB). 1083 * - Make the mapping size configurable ? We currently use a 2GB mapping 1084 * at a 1GB offset to ensure that NULL VAs will fault. 1085 */ 1086 if (!mmu->mapping) { 1087 struct dma_iommu_mapping *mapping; 1088 1089 mapping = arm_iommu_create_mapping(&platform_bus_type, 1090 SZ_1G, SZ_2G); 1091 if (IS_ERR(mapping)) { 1092 dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); 1093 return PTR_ERR(mapping); 1094 } 1095 1096 mmu->mapping = mapping; 1097 } 1098 1099 /* Attach the ARM VA mapping to the device. */ 1100 ret = arm_iommu_attach_device(dev, mmu->mapping); 1101 if (ret < 0) { 1102 dev_err(dev, "Failed to attach device to VA mapping\n"); 1103 goto error; 1104 } 1105 1106 return 0; 1107 1108 error: 1109 kfree(dev->archdata.iommu); 1110 dev->archdata.iommu = NULL; 1111 iommu_group_remove_device(dev); 1112 return ret; 1113 } 1114 1115 static void ipmmu_remove_device(struct device *dev) 1116 { 1117 arm_iommu_detach_device(dev); 1118 iommu_group_remove_device(dev); 1119 kfree(dev->archdata.iommu); 1120 dev->archdata.iommu = NULL; 1121 } 1122 1123 static struct iommu_ops ipmmu_ops = { 1124 .domain_init = ipmmu_domain_init, 1125 .domain_destroy = ipmmu_domain_destroy, 1126 .attach_dev = ipmmu_attach_device, 1127 .detach_dev = ipmmu_detach_device, 1128 .map = ipmmu_map, 1129 .unmap = ipmmu_unmap, 1130 .iova_to_phys = ipmmu_iova_to_phys, 1131 .add_device = ipmmu_add_device, 1132 .remove_device = ipmmu_remove_device, 1133 .pgsize_bitmap = SZ_2M | SZ_64K | SZ_4K, 1134 }; 1135 1136 /* ----------------------------------------------------------------------------- 1137 * Probe/remove and init 1138 */ 1139 1140 static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) 1141 { 1142 unsigned int i; 1143 1144 /* Disable all contexts. */ 1145 for (i = 0; i < 4; ++i) 1146 ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0); 1147 } 1148 1149 static int ipmmu_probe(struct platform_device *pdev) 1150 { 1151 struct ipmmu_vmsa_device *mmu; 1152 struct resource *res; 1153 int irq; 1154 int ret; 1155 1156 if (!pdev->dev.platform_data) { 1157 dev_err(&pdev->dev, "missing platform data\n"); 1158 return -EINVAL; 1159 } 1160 1161 mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); 1162 if (!mmu) { 1163 dev_err(&pdev->dev, "cannot allocate device data\n"); 1164 return -ENOMEM; 1165 } 1166 1167 mmu->dev = &pdev->dev; 1168 mmu->pdata = pdev->dev.platform_data; 1169 mmu->num_utlbs = 32; 1170 1171 /* Map I/O memory and request IRQ. */ 1172 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1173 mmu->base = devm_ioremap_resource(&pdev->dev, res); 1174 if (IS_ERR(mmu->base)) 1175 return PTR_ERR(mmu->base); 1176 1177 irq = platform_get_irq(pdev, 0); 1178 if (irq < 0) { 1179 dev_err(&pdev->dev, "no IRQ found\n"); 1180 return irq; 1181 } 1182 1183 ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, 1184 dev_name(&pdev->dev), mmu); 1185 if (ret < 0) { 1186 dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); 1187 return irq; 1188 } 1189 1190 ipmmu_device_reset(mmu); 1191 1192 /* 1193 * We can't create the ARM mapping here as it requires the bus to have 1194 * an IOMMU, which only happens when bus_set_iommu() is called in 1195 * ipmmu_init() after the probe function returns. 1196 */ 1197 1198 spin_lock(&ipmmu_devices_lock); 1199 list_add(&mmu->list, &ipmmu_devices); 1200 spin_unlock(&ipmmu_devices_lock); 1201 1202 platform_set_drvdata(pdev, mmu); 1203 1204 return 0; 1205 } 1206 1207 static int ipmmu_remove(struct platform_device *pdev) 1208 { 1209 struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev); 1210 1211 spin_lock(&ipmmu_devices_lock); 1212 list_del(&mmu->list); 1213 spin_unlock(&ipmmu_devices_lock); 1214 1215 arm_iommu_release_mapping(mmu->mapping); 1216 1217 ipmmu_device_reset(mmu); 1218 1219 return 0; 1220 } 1221 1222 static struct platform_driver ipmmu_driver = { 1223 .driver = { 1224 .owner = THIS_MODULE, 1225 .name = "ipmmu-vmsa", 1226 }, 1227 .probe = ipmmu_probe, 1228 .remove = ipmmu_remove, 1229 }; 1230 1231 static int __init ipmmu_init(void) 1232 { 1233 int ret; 1234 1235 ret = platform_driver_register(&ipmmu_driver); 1236 if (ret < 0) 1237 return ret; 1238 1239 if (!iommu_present(&platform_bus_type)) 1240 bus_set_iommu(&platform_bus_type, &ipmmu_ops); 1241 1242 return 0; 1243 } 1244 1245 static void __exit ipmmu_exit(void) 1246 { 1247 return platform_driver_unregister(&ipmmu_driver); 1248 } 1249 1250 subsys_initcall(ipmmu_init); 1251 module_exit(ipmmu_exit); 1252 1253 MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU"); 1254 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>"); 1255 MODULE_LICENSE("GPL v2"); 1256