1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License version 2 as 4 * published by the Free Software Foundation. 5 */ 6 7 #include <linux/compiler.h> 8 #include <linux/delay.h> 9 #include <linux/device.h> 10 #include <linux/dma-iommu.h> 11 #include <linux/errno.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/iommu.h> 15 #include <linux/jiffies.h> 16 #include <linux/list.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_platform.h> 21 #include <linux/platform_device.h> 22 #include <linux/slab.h> 23 #include <linux/spinlock.h> 24 25 /** MMU register offsets */ 26 #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ 27 #define RK_MMU_STATUS 0x04 28 #define RK_MMU_COMMAND 0x08 29 #define RK_MMU_PAGE_FAULT_ADDR 0x0C /* IOVA of last page fault */ 30 #define RK_MMU_ZAP_ONE_LINE 0x10 /* Shootdown one IOTLB entry */ 31 #define RK_MMU_INT_RAWSTAT 0x14 /* IRQ status ignoring mask */ 32 #define RK_MMU_INT_CLEAR 0x18 /* Acknowledge and re-arm irq */ 33 #define RK_MMU_INT_MASK 0x1C /* IRQ enable */ 34 #define RK_MMU_INT_STATUS 0x20 /* IRQ status after masking */ 35 #define RK_MMU_AUTO_GATING 0x24 36 37 #define DTE_ADDR_DUMMY 0xCAFEBABE 38 #define FORCE_RESET_TIMEOUT 100 /* ms */ 39 40 /* RK_MMU_STATUS fields */ 41 #define RK_MMU_STATUS_PAGING_ENABLED BIT(0) 42 #define RK_MMU_STATUS_PAGE_FAULT_ACTIVE BIT(1) 43 #define RK_MMU_STATUS_STALL_ACTIVE BIT(2) 44 #define RK_MMU_STATUS_IDLE BIT(3) 45 #define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4) 46 #define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5) 47 #define RK_MMU_STATUS_STALL_NOT_ACTIVE BIT(31) 48 49 /* RK_MMU_COMMAND command values */ 50 #define RK_MMU_CMD_ENABLE_PAGING 0 /* Enable memory translation */ 51 #define RK_MMU_CMD_DISABLE_PAGING 1 /* Disable memory translation */ 52 #define RK_MMU_CMD_ENABLE_STALL 2 /* Stall paging to allow other cmds */ 53 #define RK_MMU_CMD_DISABLE_STALL 3 /* Stop stall re-enables paging */ 54 #define RK_MMU_CMD_ZAP_CACHE 4 /* Shoot down entire IOTLB */ 55 #define RK_MMU_CMD_PAGE_FAULT_DONE 5 /* Clear page fault */ 56 #define RK_MMU_CMD_FORCE_RESET 6 /* Reset all registers */ 57 58 /* RK_MMU_INT_* register fields */ 59 #define RK_MMU_IRQ_PAGE_FAULT 0x01 /* page fault */ 60 #define RK_MMU_IRQ_BUS_ERROR 0x02 /* bus read error */ 61 #define RK_MMU_IRQ_MASK (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR) 62 63 #define NUM_DT_ENTRIES 1024 64 #define NUM_PT_ENTRIES 1024 65 66 #define SPAGE_ORDER 12 67 #define SPAGE_SIZE (1 << SPAGE_ORDER) 68 69 /* 70 * Support mapping any size that fits in one page table: 71 * 4 KiB to 4 MiB 72 */ 73 #define RK_IOMMU_PGSIZE_BITMAP 0x007ff000 74 75 #define IOMMU_REG_POLL_COUNT_FAST 1000 76 77 struct rk_iommu_domain { 78 struct list_head iommus; 79 struct platform_device *pdev; 80 u32 *dt; /* page directory table */ 81 dma_addr_t dt_dma; 82 spinlock_t iommus_lock; /* lock for iommus list */ 83 spinlock_t dt_lock; /* lock for modifying page directory table */ 84 85 struct iommu_domain domain; 86 }; 87 88 struct rk_iommu { 89 struct device *dev; 90 void __iomem **bases; 91 int num_mmu; 92 int irq; 93 struct list_head node; /* entry in rk_iommu_domain.iommus */ 94 struct iommu_domain *domain; /* domain to which iommu is attached */ 95 }; 96 97 static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, 98 unsigned int count) 99 { 100 size_t size = count * sizeof(u32); /* count of u32 entry */ 101 102 dma_sync_single_for_device(&dom->pdev->dev, dma, size, DMA_TO_DEVICE); 103 } 104 105 static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) 106 { 107 return container_of(dom, struct rk_iommu_domain, domain); 108 } 109 110 /** 111 * Inspired by _wait_for in intel_drv.h 112 * This is NOT safe for use in interrupt context. 113 * 114 * Note that it's important that we check the condition again after having 115 * timed out, since the timeout could be due to preemption or similar and 116 * we've never had a chance to check the condition before the timeout. 117 */ 118 #define rk_wait_for(COND, MS) ({ \ 119 unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ 120 int ret__ = 0; \ 121 while (!(COND)) { \ 122 if (time_after(jiffies, timeout__)) { \ 123 ret__ = (COND) ? 0 : -ETIMEDOUT; \ 124 break; \ 125 } \ 126 usleep_range(50, 100); \ 127 } \ 128 ret__; \ 129 }) 130 131 /* 132 * The Rockchip rk3288 iommu uses a 2-level page table. 133 * The first level is the "Directory Table" (DT). 134 * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing 135 * to a "Page Table". 136 * The second level is the 1024 Page Tables (PT). 137 * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to 138 * a 4 KB page of physical memory. 139 * 140 * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries). 141 * Each iommu device has a MMU_DTE_ADDR register that contains the physical 142 * address of the start of the DT page. 143 * 144 * The structure of the page table is as follows: 145 * 146 * DT 147 * MMU_DTE_ADDR -> +-----+ 148 * | | 149 * +-----+ PT 150 * | DTE | -> +-----+ 151 * +-----+ | | Memory 152 * | | +-----+ Page 153 * | | | PTE | -> +-----+ 154 * +-----+ +-----+ | | 155 * | | | | 156 * | | | | 157 * +-----+ | | 158 * | | 159 * | | 160 * +-----+ 161 */ 162 163 /* 164 * Each DTE has a PT address and a valid bit: 165 * +---------------------+-----------+-+ 166 * | PT address | Reserved |V| 167 * +---------------------+-----------+-+ 168 * 31:12 - PT address (PTs always starts on a 4 KB boundary) 169 * 11: 1 - Reserved 170 * 0 - 1 if PT @ PT address is valid 171 */ 172 #define RK_DTE_PT_ADDRESS_MASK 0xfffff000 173 #define RK_DTE_PT_VALID BIT(0) 174 175 static inline phys_addr_t rk_dte_pt_address(u32 dte) 176 { 177 return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK; 178 } 179 180 static inline bool rk_dte_is_pt_valid(u32 dte) 181 { 182 return dte & RK_DTE_PT_VALID; 183 } 184 185 static inline u32 rk_mk_dte(dma_addr_t pt_dma) 186 { 187 return (pt_dma & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; 188 } 189 190 /* 191 * Each PTE has a Page address, some flags and a valid bit: 192 * +---------------------+---+-------+-+ 193 * | Page address |Rsv| Flags |V| 194 * +---------------------+---+-------+-+ 195 * 31:12 - Page address (Pages always start on a 4 KB boundary) 196 * 11: 9 - Reserved 197 * 8: 1 - Flags 198 * 8 - Read allocate - allocate cache space on read misses 199 * 7 - Read cache - enable cache & prefetch of data 200 * 6 - Write buffer - enable delaying writes on their way to memory 201 * 5 - Write allocate - allocate cache space on write misses 202 * 4 - Write cache - different writes can be merged together 203 * 3 - Override cache attributes 204 * if 1, bits 4-8 control cache attributes 205 * if 0, the system bus defaults are used 206 * 2 - Writable 207 * 1 - Readable 208 * 0 - 1 if Page @ Page address is valid 209 */ 210 #define RK_PTE_PAGE_ADDRESS_MASK 0xfffff000 211 #define RK_PTE_PAGE_FLAGS_MASK 0x000001fe 212 #define RK_PTE_PAGE_WRITABLE BIT(2) 213 #define RK_PTE_PAGE_READABLE BIT(1) 214 #define RK_PTE_PAGE_VALID BIT(0) 215 216 static inline phys_addr_t rk_pte_page_address(u32 pte) 217 { 218 return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK; 219 } 220 221 static inline bool rk_pte_is_page_valid(u32 pte) 222 { 223 return pte & RK_PTE_PAGE_VALID; 224 } 225 226 /* TODO: set cache flags per prot IOMMU_CACHE */ 227 static u32 rk_mk_pte(phys_addr_t page, int prot) 228 { 229 u32 flags = 0; 230 flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; 231 flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; 232 page &= RK_PTE_PAGE_ADDRESS_MASK; 233 return page | flags | RK_PTE_PAGE_VALID; 234 } 235 236 static u32 rk_mk_pte_invalid(u32 pte) 237 { 238 return pte & ~RK_PTE_PAGE_VALID; 239 } 240 241 /* 242 * rk3288 iova (IOMMU Virtual Address) format 243 * 31 22.21 12.11 0 244 * +-----------+-----------+-------------+ 245 * | DTE index | PTE index | Page offset | 246 * +-----------+-----------+-------------+ 247 * 31:22 - DTE index - index of DTE in DT 248 * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address 249 * 11: 0 - Page offset - offset into page @ PTE.page_address 250 */ 251 #define RK_IOVA_DTE_MASK 0xffc00000 252 #define RK_IOVA_DTE_SHIFT 22 253 #define RK_IOVA_PTE_MASK 0x003ff000 254 #define RK_IOVA_PTE_SHIFT 12 255 #define RK_IOVA_PAGE_MASK 0x00000fff 256 #define RK_IOVA_PAGE_SHIFT 0 257 258 static u32 rk_iova_dte_index(dma_addr_t iova) 259 { 260 return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT; 261 } 262 263 static u32 rk_iova_pte_index(dma_addr_t iova) 264 { 265 return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT; 266 } 267 268 static u32 rk_iova_page_offset(dma_addr_t iova) 269 { 270 return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT; 271 } 272 273 static u32 rk_iommu_read(void __iomem *base, u32 offset) 274 { 275 return readl(base + offset); 276 } 277 278 static void rk_iommu_write(void __iomem *base, u32 offset, u32 value) 279 { 280 writel(value, base + offset); 281 } 282 283 static void rk_iommu_command(struct rk_iommu *iommu, u32 command) 284 { 285 int i; 286 287 for (i = 0; i < iommu->num_mmu; i++) 288 writel(command, iommu->bases[i] + RK_MMU_COMMAND); 289 } 290 291 static void rk_iommu_base_command(void __iomem *base, u32 command) 292 { 293 writel(command, base + RK_MMU_COMMAND); 294 } 295 static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova, 296 size_t size) 297 { 298 int i; 299 300 dma_addr_t iova_end = iova + size; 301 /* 302 * TODO(djkurtz): Figure out when it is more efficient to shootdown the 303 * entire iotlb rather than iterate over individual iovas. 304 */ 305 for (i = 0; i < iommu->num_mmu; i++) 306 for (; iova < iova_end; iova += SPAGE_SIZE) 307 rk_iommu_write(iommu->bases[i], RK_MMU_ZAP_ONE_LINE, iova); 308 } 309 310 static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) 311 { 312 bool active = true; 313 int i; 314 315 for (i = 0; i < iommu->num_mmu; i++) 316 active &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) & 317 RK_MMU_STATUS_STALL_ACTIVE); 318 319 return active; 320 } 321 322 static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) 323 { 324 bool enable = true; 325 int i; 326 327 for (i = 0; i < iommu->num_mmu; i++) 328 enable &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) & 329 RK_MMU_STATUS_PAGING_ENABLED); 330 331 return enable; 332 } 333 334 static int rk_iommu_enable_stall(struct rk_iommu *iommu) 335 { 336 int ret, i; 337 338 if (rk_iommu_is_stall_active(iommu)) 339 return 0; 340 341 /* Stall can only be enabled if paging is enabled */ 342 if (!rk_iommu_is_paging_enabled(iommu)) 343 return 0; 344 345 rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL); 346 347 ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1); 348 if (ret) 349 for (i = 0; i < iommu->num_mmu; i++) 350 dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", 351 rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); 352 353 return ret; 354 } 355 356 static int rk_iommu_disable_stall(struct rk_iommu *iommu) 357 { 358 int ret, i; 359 360 if (!rk_iommu_is_stall_active(iommu)) 361 return 0; 362 363 rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL); 364 365 ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1); 366 if (ret) 367 for (i = 0; i < iommu->num_mmu; i++) 368 dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", 369 rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); 370 371 return ret; 372 } 373 374 static int rk_iommu_enable_paging(struct rk_iommu *iommu) 375 { 376 int ret, i; 377 378 if (rk_iommu_is_paging_enabled(iommu)) 379 return 0; 380 381 rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING); 382 383 ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1); 384 if (ret) 385 for (i = 0; i < iommu->num_mmu; i++) 386 dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", 387 rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); 388 389 return ret; 390 } 391 392 static int rk_iommu_disable_paging(struct rk_iommu *iommu) 393 { 394 int ret, i; 395 396 if (!rk_iommu_is_paging_enabled(iommu)) 397 return 0; 398 399 rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING); 400 401 ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1); 402 if (ret) 403 for (i = 0; i < iommu->num_mmu; i++) 404 dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", 405 rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); 406 407 return ret; 408 } 409 410 static int rk_iommu_force_reset(struct rk_iommu *iommu) 411 { 412 int ret, i; 413 u32 dte_addr; 414 415 /* 416 * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY 417 * and verifying that upper 5 nybbles are read back. 418 */ 419 for (i = 0; i < iommu->num_mmu; i++) { 420 rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY); 421 422 dte_addr = rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR); 423 if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) { 424 dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n"); 425 return -EFAULT; 426 } 427 } 428 429 rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); 430 431 for (i = 0; i < iommu->num_mmu; i++) { 432 ret = rk_wait_for(rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0x00000000, 433 FORCE_RESET_TIMEOUT); 434 if (ret) { 435 dev_err(iommu->dev, "FORCE_RESET command timed out\n"); 436 return ret; 437 } 438 } 439 440 return 0; 441 } 442 443 static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova) 444 { 445 void __iomem *base = iommu->bases[index]; 446 u32 dte_index, pte_index, page_offset; 447 u32 mmu_dte_addr; 448 phys_addr_t mmu_dte_addr_phys, dte_addr_phys; 449 u32 *dte_addr; 450 u32 dte; 451 phys_addr_t pte_addr_phys = 0; 452 u32 *pte_addr = NULL; 453 u32 pte = 0; 454 phys_addr_t page_addr_phys = 0; 455 u32 page_flags = 0; 456 457 dte_index = rk_iova_dte_index(iova); 458 pte_index = rk_iova_pte_index(iova); 459 page_offset = rk_iova_page_offset(iova); 460 461 mmu_dte_addr = rk_iommu_read(base, RK_MMU_DTE_ADDR); 462 mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr; 463 464 dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index); 465 dte_addr = phys_to_virt(dte_addr_phys); 466 dte = *dte_addr; 467 468 if (!rk_dte_is_pt_valid(dte)) 469 goto print_it; 470 471 pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4); 472 pte_addr = phys_to_virt(pte_addr_phys); 473 pte = *pte_addr; 474 475 if (!rk_pte_is_page_valid(pte)) 476 goto print_it; 477 478 page_addr_phys = rk_pte_page_address(pte) + page_offset; 479 page_flags = pte & RK_PTE_PAGE_FLAGS_MASK; 480 481 print_it: 482 dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n", 483 &iova, dte_index, pte_index, page_offset); 484 dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n", 485 &mmu_dte_addr_phys, &dte_addr_phys, dte, 486 rk_dte_is_pt_valid(dte), &pte_addr_phys, pte, 487 rk_pte_is_page_valid(pte), &page_addr_phys, page_flags); 488 } 489 490 static irqreturn_t rk_iommu_irq(int irq, void *dev_id) 491 { 492 struct rk_iommu *iommu = dev_id; 493 u32 status; 494 u32 int_status; 495 dma_addr_t iova; 496 irqreturn_t ret = IRQ_NONE; 497 int i; 498 499 for (i = 0; i < iommu->num_mmu; i++) { 500 int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS); 501 if (int_status == 0) 502 continue; 503 504 ret = IRQ_HANDLED; 505 iova = rk_iommu_read(iommu->bases[i], RK_MMU_PAGE_FAULT_ADDR); 506 507 if (int_status & RK_MMU_IRQ_PAGE_FAULT) { 508 int flags; 509 510 status = rk_iommu_read(iommu->bases[i], RK_MMU_STATUS); 511 flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? 512 IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; 513 514 dev_err(iommu->dev, "Page fault at %pad of type %s\n", 515 &iova, 516 (flags == IOMMU_FAULT_WRITE) ? "write" : "read"); 517 518 log_iova(iommu, i, iova); 519 520 /* 521 * Report page fault to any installed handlers. 522 * Ignore the return code, though, since we always zap cache 523 * and clear the page fault anyway. 524 */ 525 if (iommu->domain) 526 report_iommu_fault(iommu->domain, iommu->dev, iova, 527 flags); 528 else 529 dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n"); 530 531 rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); 532 rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_PAGE_FAULT_DONE); 533 } 534 535 if (int_status & RK_MMU_IRQ_BUS_ERROR) 536 dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova); 537 538 if (int_status & ~RK_MMU_IRQ_MASK) 539 dev_err(iommu->dev, "unexpected int_status: %#08x\n", 540 int_status); 541 542 rk_iommu_write(iommu->bases[i], RK_MMU_INT_CLEAR, int_status); 543 } 544 545 return ret; 546 } 547 548 static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, 549 dma_addr_t iova) 550 { 551 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 552 unsigned long flags; 553 phys_addr_t pt_phys, phys = 0; 554 u32 dte, pte; 555 u32 *page_table; 556 557 spin_lock_irqsave(&rk_domain->dt_lock, flags); 558 559 dte = rk_domain->dt[rk_iova_dte_index(iova)]; 560 if (!rk_dte_is_pt_valid(dte)) 561 goto out; 562 563 pt_phys = rk_dte_pt_address(dte); 564 page_table = (u32 *)phys_to_virt(pt_phys); 565 pte = page_table[rk_iova_pte_index(iova)]; 566 if (!rk_pte_is_page_valid(pte)) 567 goto out; 568 569 phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova); 570 out: 571 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 572 573 return phys; 574 } 575 576 static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, 577 dma_addr_t iova, size_t size) 578 { 579 struct list_head *pos; 580 unsigned long flags; 581 582 /* shootdown these iova from all iommus using this domain */ 583 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 584 list_for_each(pos, &rk_domain->iommus) { 585 struct rk_iommu *iommu; 586 iommu = list_entry(pos, struct rk_iommu, node); 587 rk_iommu_zap_lines(iommu, iova, size); 588 } 589 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 590 } 591 592 static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, 593 dma_addr_t iova, size_t size) 594 { 595 rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); 596 if (size > SPAGE_SIZE) 597 rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE, 598 SPAGE_SIZE); 599 } 600 601 static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, 602 dma_addr_t iova) 603 { 604 struct device *dev = &rk_domain->pdev->dev; 605 u32 *page_table, *dte_addr; 606 u32 dte_index, dte; 607 phys_addr_t pt_phys; 608 dma_addr_t pt_dma; 609 610 assert_spin_locked(&rk_domain->dt_lock); 611 612 dte_index = rk_iova_dte_index(iova); 613 dte_addr = &rk_domain->dt[dte_index]; 614 dte = *dte_addr; 615 if (rk_dte_is_pt_valid(dte)) 616 goto done; 617 618 page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 619 if (!page_table) 620 return ERR_PTR(-ENOMEM); 621 622 pt_dma = dma_map_single(dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); 623 if (dma_mapping_error(dev, pt_dma)) { 624 dev_err(dev, "DMA mapping error while allocating page table\n"); 625 free_page((unsigned long)page_table); 626 return ERR_PTR(-ENOMEM); 627 } 628 629 dte = rk_mk_dte(pt_dma); 630 *dte_addr = dte; 631 632 rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES); 633 rk_table_flush(rk_domain, 634 rk_domain->dt_dma + dte_index * sizeof(u32), 1); 635 done: 636 pt_phys = rk_dte_pt_address(dte); 637 return (u32 *)phys_to_virt(pt_phys); 638 } 639 640 static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, 641 u32 *pte_addr, dma_addr_t pte_dma, 642 size_t size) 643 { 644 unsigned int pte_count; 645 unsigned int pte_total = size / SPAGE_SIZE; 646 647 assert_spin_locked(&rk_domain->dt_lock); 648 649 for (pte_count = 0; pte_count < pte_total; pte_count++) { 650 u32 pte = pte_addr[pte_count]; 651 if (!rk_pte_is_page_valid(pte)) 652 break; 653 654 pte_addr[pte_count] = rk_mk_pte_invalid(pte); 655 } 656 657 rk_table_flush(rk_domain, pte_dma, pte_count); 658 659 return pte_count * SPAGE_SIZE; 660 } 661 662 static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, 663 dma_addr_t pte_dma, dma_addr_t iova, 664 phys_addr_t paddr, size_t size, int prot) 665 { 666 unsigned int pte_count; 667 unsigned int pte_total = size / SPAGE_SIZE; 668 phys_addr_t page_phys; 669 670 assert_spin_locked(&rk_domain->dt_lock); 671 672 for (pte_count = 0; pte_count < pte_total; pte_count++) { 673 u32 pte = pte_addr[pte_count]; 674 675 if (rk_pte_is_page_valid(pte)) 676 goto unwind; 677 678 pte_addr[pte_count] = rk_mk_pte(paddr, prot); 679 680 paddr += SPAGE_SIZE; 681 } 682 683 rk_table_flush(rk_domain, pte_dma, pte_total); 684 685 /* 686 * Zap the first and last iova to evict from iotlb any previously 687 * mapped cachelines holding stale values for its dte and pte. 688 * We only zap the first and last iova, since only they could have 689 * dte or pte shared with an existing mapping. 690 */ 691 rk_iommu_zap_iova_first_last(rk_domain, iova, size); 692 693 return 0; 694 unwind: 695 /* Unmap the range of iovas that we just mapped */ 696 rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, 697 pte_count * SPAGE_SIZE); 698 699 iova += pte_count * SPAGE_SIZE; 700 page_phys = rk_pte_page_address(pte_addr[pte_count]); 701 pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n", 702 &iova, &page_phys, &paddr, prot); 703 704 return -EADDRINUSE; 705 } 706 707 static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, 708 phys_addr_t paddr, size_t size, int prot) 709 { 710 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 711 unsigned long flags; 712 dma_addr_t pte_dma, iova = (dma_addr_t)_iova; 713 u32 *page_table, *pte_addr; 714 u32 dte_index, pte_index; 715 int ret; 716 717 spin_lock_irqsave(&rk_domain->dt_lock, flags); 718 719 /* 720 * pgsize_bitmap specifies iova sizes that fit in one page table 721 * (1024 4-KiB pages = 4 MiB). 722 * So, size will always be 4096 <= size <= 4194304. 723 * Since iommu_map() guarantees that both iova and size will be 724 * aligned, we will always only be mapping from a single dte here. 725 */ 726 page_table = rk_dte_get_page_table(rk_domain, iova); 727 if (IS_ERR(page_table)) { 728 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 729 return PTR_ERR(page_table); 730 } 731 732 dte_index = rk_domain->dt[rk_iova_dte_index(iova)]; 733 pte_index = rk_iova_pte_index(iova); 734 pte_addr = &page_table[pte_index]; 735 pte_dma = rk_dte_pt_address(dte_index) + pte_index * sizeof(u32); 736 ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova, 737 paddr, size, prot); 738 739 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 740 741 return ret; 742 } 743 744 static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, 745 size_t size) 746 { 747 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 748 unsigned long flags; 749 dma_addr_t pte_dma, iova = (dma_addr_t)_iova; 750 phys_addr_t pt_phys; 751 u32 dte; 752 u32 *pte_addr; 753 size_t unmap_size; 754 755 spin_lock_irqsave(&rk_domain->dt_lock, flags); 756 757 /* 758 * pgsize_bitmap specifies iova sizes that fit in one page table 759 * (1024 4-KiB pages = 4 MiB). 760 * So, size will always be 4096 <= size <= 4194304. 761 * Since iommu_unmap() guarantees that both iova and size will be 762 * aligned, we will always only be unmapping from a single dte here. 763 */ 764 dte = rk_domain->dt[rk_iova_dte_index(iova)]; 765 /* Just return 0 if iova is unmapped */ 766 if (!rk_dte_is_pt_valid(dte)) { 767 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 768 return 0; 769 } 770 771 pt_phys = rk_dte_pt_address(dte); 772 pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); 773 pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32); 774 unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size); 775 776 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 777 778 /* Shootdown iotlb entries for iova range that was just unmapped */ 779 rk_iommu_zap_iova(rk_domain, iova, unmap_size); 780 781 return unmap_size; 782 } 783 784 static struct rk_iommu *rk_iommu_from_dev(struct device *dev) 785 { 786 struct iommu_group *group; 787 struct device *iommu_dev; 788 struct rk_iommu *rk_iommu; 789 790 group = iommu_group_get(dev); 791 if (!group) 792 return NULL; 793 iommu_dev = iommu_group_get_iommudata(group); 794 rk_iommu = dev_get_drvdata(iommu_dev); 795 iommu_group_put(group); 796 797 return rk_iommu; 798 } 799 800 static int rk_iommu_attach_device(struct iommu_domain *domain, 801 struct device *dev) 802 { 803 struct rk_iommu *iommu; 804 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 805 unsigned long flags; 806 int ret, i; 807 808 /* 809 * Allow 'virtual devices' (e.g., drm) to attach to domain. 810 * Such a device does not belong to an iommu group. 811 */ 812 iommu = rk_iommu_from_dev(dev); 813 if (!iommu) 814 return 0; 815 816 ret = rk_iommu_enable_stall(iommu); 817 if (ret) 818 return ret; 819 820 ret = rk_iommu_force_reset(iommu); 821 if (ret) 822 return ret; 823 824 iommu->domain = domain; 825 826 ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq, 827 IRQF_SHARED, dev_name(dev), iommu); 828 if (ret) 829 return ret; 830 831 for (i = 0; i < iommu->num_mmu; i++) { 832 rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 833 rk_domain->dt_dma); 834 rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); 835 rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); 836 } 837 838 ret = rk_iommu_enable_paging(iommu); 839 if (ret) 840 return ret; 841 842 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 843 list_add_tail(&iommu->node, &rk_domain->iommus); 844 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 845 846 dev_dbg(dev, "Attached to iommu domain\n"); 847 848 rk_iommu_disable_stall(iommu); 849 850 return 0; 851 } 852 853 static void rk_iommu_detach_device(struct iommu_domain *domain, 854 struct device *dev) 855 { 856 struct rk_iommu *iommu; 857 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 858 unsigned long flags; 859 int i; 860 861 /* Allow 'virtual devices' (eg drm) to detach from domain */ 862 iommu = rk_iommu_from_dev(dev); 863 if (!iommu) 864 return; 865 866 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 867 list_del_init(&iommu->node); 868 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 869 870 /* Ignore error while disabling, just keep going */ 871 rk_iommu_enable_stall(iommu); 872 rk_iommu_disable_paging(iommu); 873 for (i = 0; i < iommu->num_mmu; i++) { 874 rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0); 875 rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0); 876 } 877 rk_iommu_disable_stall(iommu); 878 879 devm_free_irq(iommu->dev, iommu->irq, iommu); 880 881 iommu->domain = NULL; 882 883 dev_dbg(dev, "Detached from iommu domain\n"); 884 } 885 886 static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) 887 { 888 struct rk_iommu_domain *rk_domain; 889 struct platform_device *pdev; 890 struct device *iommu_dev; 891 892 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) 893 return NULL; 894 895 /* Register a pdev per domain, so DMA API can base on this *dev 896 * even some virtual master doesn't have an iommu slave 897 */ 898 pdev = platform_device_register_simple("rk_iommu_domain", 899 PLATFORM_DEVID_AUTO, NULL, 0); 900 if (IS_ERR(pdev)) 901 return NULL; 902 903 rk_domain = devm_kzalloc(&pdev->dev, sizeof(*rk_domain), GFP_KERNEL); 904 if (!rk_domain) 905 goto err_unreg_pdev; 906 907 rk_domain->pdev = pdev; 908 909 if (type == IOMMU_DOMAIN_DMA && 910 iommu_get_dma_cookie(&rk_domain->domain)) 911 goto err_unreg_pdev; 912 913 /* 914 * rk32xx iommus use a 2 level pagetable. 915 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. 916 * Allocate one 4 KiB page for each table. 917 */ 918 rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); 919 if (!rk_domain->dt) 920 goto err_put_cookie; 921 922 iommu_dev = &pdev->dev; 923 rk_domain->dt_dma = dma_map_single(iommu_dev, rk_domain->dt, 924 SPAGE_SIZE, DMA_TO_DEVICE); 925 if (dma_mapping_error(iommu_dev, rk_domain->dt_dma)) { 926 dev_err(iommu_dev, "DMA map error for DT\n"); 927 goto err_free_dt; 928 } 929 930 rk_table_flush(rk_domain, rk_domain->dt_dma, NUM_DT_ENTRIES); 931 932 spin_lock_init(&rk_domain->iommus_lock); 933 spin_lock_init(&rk_domain->dt_lock); 934 INIT_LIST_HEAD(&rk_domain->iommus); 935 936 rk_domain->domain.geometry.aperture_start = 0; 937 rk_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); 938 rk_domain->domain.geometry.force_aperture = true; 939 940 return &rk_domain->domain; 941 942 err_free_dt: 943 free_page((unsigned long)rk_domain->dt); 944 err_put_cookie: 945 if (type == IOMMU_DOMAIN_DMA) 946 iommu_put_dma_cookie(&rk_domain->domain); 947 err_unreg_pdev: 948 platform_device_unregister(pdev); 949 950 return NULL; 951 } 952 953 static void rk_iommu_domain_free(struct iommu_domain *domain) 954 { 955 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 956 int i; 957 958 WARN_ON(!list_empty(&rk_domain->iommus)); 959 960 for (i = 0; i < NUM_DT_ENTRIES; i++) { 961 u32 dte = rk_domain->dt[i]; 962 if (rk_dte_is_pt_valid(dte)) { 963 phys_addr_t pt_phys = rk_dte_pt_address(dte); 964 u32 *page_table = phys_to_virt(pt_phys); 965 dma_unmap_single(&rk_domain->pdev->dev, pt_phys, 966 SPAGE_SIZE, DMA_TO_DEVICE); 967 free_page((unsigned long)page_table); 968 } 969 } 970 971 dma_unmap_single(&rk_domain->pdev->dev, rk_domain->dt_dma, 972 SPAGE_SIZE, DMA_TO_DEVICE); 973 free_page((unsigned long)rk_domain->dt); 974 975 if (domain->type == IOMMU_DOMAIN_DMA) 976 iommu_put_dma_cookie(&rk_domain->domain); 977 978 platform_device_unregister(rk_domain->pdev); 979 } 980 981 static bool rk_iommu_is_dev_iommu_master(struct device *dev) 982 { 983 struct device_node *np = dev->of_node; 984 int ret; 985 986 /* 987 * An iommu master has an iommus property containing a list of phandles 988 * to iommu nodes, each with an #iommu-cells property with value 0. 989 */ 990 ret = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); 991 return (ret > 0); 992 } 993 994 static int rk_iommu_group_set_iommudata(struct iommu_group *group, 995 struct device *dev) 996 { 997 struct device_node *np = dev->of_node; 998 struct platform_device *pd; 999 int ret; 1000 struct of_phandle_args args; 1001 1002 /* 1003 * An iommu master has an iommus property containing a list of phandles 1004 * to iommu nodes, each with an #iommu-cells property with value 0. 1005 */ 1006 ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0, 1007 &args); 1008 if (ret) { 1009 dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n", 1010 np->full_name, ret); 1011 return ret; 1012 } 1013 if (args.args_count != 0) { 1014 dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n", 1015 args.np->full_name, args.args_count); 1016 return -EINVAL; 1017 } 1018 1019 pd = of_find_device_by_node(args.np); 1020 of_node_put(args.np); 1021 if (!pd) { 1022 dev_err(dev, "iommu %s not found\n", args.np->full_name); 1023 return -EPROBE_DEFER; 1024 } 1025 1026 /* TODO(djkurtz): handle multiple slave iommus for a single master */ 1027 iommu_group_set_iommudata(group, &pd->dev, NULL); 1028 1029 return 0; 1030 } 1031 1032 static int rk_iommu_add_device(struct device *dev) 1033 { 1034 struct iommu_group *group; 1035 int ret; 1036 1037 if (!rk_iommu_is_dev_iommu_master(dev)) 1038 return -ENODEV; 1039 1040 group = iommu_group_get(dev); 1041 if (!group) { 1042 group = iommu_group_alloc(); 1043 if (IS_ERR(group)) { 1044 dev_err(dev, "Failed to allocate IOMMU group\n"); 1045 return PTR_ERR(group); 1046 } 1047 } 1048 1049 ret = iommu_group_add_device(group, dev); 1050 if (ret) 1051 goto err_put_group; 1052 1053 ret = rk_iommu_group_set_iommudata(group, dev); 1054 if (ret) 1055 goto err_remove_device; 1056 1057 iommu_group_put(group); 1058 1059 return 0; 1060 1061 err_remove_device: 1062 iommu_group_remove_device(dev); 1063 err_put_group: 1064 iommu_group_put(group); 1065 return ret; 1066 } 1067 1068 static void rk_iommu_remove_device(struct device *dev) 1069 { 1070 if (!rk_iommu_is_dev_iommu_master(dev)) 1071 return; 1072 1073 iommu_group_remove_device(dev); 1074 } 1075 1076 static const struct iommu_ops rk_iommu_ops = { 1077 .domain_alloc = rk_iommu_domain_alloc, 1078 .domain_free = rk_iommu_domain_free, 1079 .attach_dev = rk_iommu_attach_device, 1080 .detach_dev = rk_iommu_detach_device, 1081 .map = rk_iommu_map, 1082 .unmap = rk_iommu_unmap, 1083 .map_sg = default_iommu_map_sg, 1084 .add_device = rk_iommu_add_device, 1085 .remove_device = rk_iommu_remove_device, 1086 .iova_to_phys = rk_iommu_iova_to_phys, 1087 .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, 1088 }; 1089 1090 static int rk_iommu_domain_probe(struct platform_device *pdev) 1091 { 1092 struct device *dev = &pdev->dev; 1093 1094 dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL); 1095 if (!dev->dma_parms) 1096 return -ENOMEM; 1097 1098 /* Set dma_ops for dev, otherwise it would be dummy_dma_ops */ 1099 arch_setup_dma_ops(dev, 0, DMA_BIT_MASK(32), NULL, false); 1100 1101 dma_set_max_seg_size(dev, DMA_BIT_MASK(32)); 1102 dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32)); 1103 1104 return 0; 1105 } 1106 1107 static struct platform_driver rk_iommu_domain_driver = { 1108 .probe = rk_iommu_domain_probe, 1109 .driver = { 1110 .name = "rk_iommu_domain", 1111 }, 1112 }; 1113 1114 static int rk_iommu_probe(struct platform_device *pdev) 1115 { 1116 struct device *dev = &pdev->dev; 1117 struct rk_iommu *iommu; 1118 struct resource *res; 1119 int num_res = pdev->num_resources; 1120 int i; 1121 1122 iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); 1123 if (!iommu) 1124 return -ENOMEM; 1125 1126 platform_set_drvdata(pdev, iommu); 1127 iommu->dev = dev; 1128 iommu->num_mmu = 0; 1129 1130 iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * num_res, 1131 GFP_KERNEL); 1132 if (!iommu->bases) 1133 return -ENOMEM; 1134 1135 for (i = 0; i < num_res; i++) { 1136 res = platform_get_resource(pdev, IORESOURCE_MEM, i); 1137 if (!res) 1138 continue; 1139 iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res); 1140 if (IS_ERR(iommu->bases[i])) 1141 continue; 1142 iommu->num_mmu++; 1143 } 1144 if (iommu->num_mmu == 0) 1145 return PTR_ERR(iommu->bases[0]); 1146 1147 iommu->irq = platform_get_irq(pdev, 0); 1148 if (iommu->irq < 0) { 1149 dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq); 1150 return -ENXIO; 1151 } 1152 1153 return 0; 1154 } 1155 1156 static int rk_iommu_remove(struct platform_device *pdev) 1157 { 1158 return 0; 1159 } 1160 1161 static const struct of_device_id rk_iommu_dt_ids[] = { 1162 { .compatible = "rockchip,iommu" }, 1163 { /* sentinel */ } 1164 }; 1165 MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); 1166 1167 static struct platform_driver rk_iommu_driver = { 1168 .probe = rk_iommu_probe, 1169 .remove = rk_iommu_remove, 1170 .driver = { 1171 .name = "rk_iommu", 1172 .of_match_table = rk_iommu_dt_ids, 1173 }, 1174 }; 1175 1176 static int __init rk_iommu_init(void) 1177 { 1178 struct device_node *np; 1179 int ret; 1180 1181 np = of_find_matching_node(NULL, rk_iommu_dt_ids); 1182 if (!np) 1183 return 0; 1184 1185 of_node_put(np); 1186 1187 ret = bus_set_iommu(&platform_bus_type, &rk_iommu_ops); 1188 if (ret) 1189 return ret; 1190 1191 ret = platform_driver_register(&rk_iommu_domain_driver); 1192 if (ret) 1193 return ret; 1194 1195 ret = platform_driver_register(&rk_iommu_driver); 1196 if (ret) 1197 platform_driver_unregister(&rk_iommu_domain_driver); 1198 return ret; 1199 } 1200 static void __exit rk_iommu_exit(void) 1201 { 1202 platform_driver_unregister(&rk_iommu_driver); 1203 platform_driver_unregister(&rk_iommu_domain_driver); 1204 } 1205 1206 subsys_initcall(rk_iommu_init); 1207 module_exit(rk_iommu_exit); 1208 1209 MODULE_DESCRIPTION("IOMMU API for Rockchip"); 1210 MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com> and Daniel Kurtz <djkurtz@chromium.org>"); 1211 MODULE_ALIAS("platform:rockchip-iommu"); 1212 MODULE_LICENSE("GPL v2"); 1213