1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License version 2 as 4 * published by the Free Software Foundation. 5 */ 6 7 #include <asm/cacheflush.h> 8 #include <asm/pgtable.h> 9 #include <linux/compiler.h> 10 #include <linux/delay.h> 11 #include <linux/device.h> 12 #include <linux/errno.h> 13 #include <linux/interrupt.h> 14 #include <linux/io.h> 15 #include <linux/iommu.h> 16 #include <linux/jiffies.h> 17 #include <linux/list.h> 18 #include <linux/mm.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_platform.h> 22 #include <linux/platform_device.h> 23 #include <linux/slab.h> 24 #include <linux/spinlock.h> 25 26 /** MMU register offsets */ 27 #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ 28 #define RK_MMU_STATUS 0x04 29 #define RK_MMU_COMMAND 0x08 30 #define RK_MMU_PAGE_FAULT_ADDR 0x0C /* IOVA of last page fault */ 31 #define RK_MMU_ZAP_ONE_LINE 0x10 /* Shootdown one IOTLB entry */ 32 #define RK_MMU_INT_RAWSTAT 0x14 /* IRQ status ignoring mask */ 33 #define RK_MMU_INT_CLEAR 0x18 /* Acknowledge and re-arm irq */ 34 #define RK_MMU_INT_MASK 0x1C /* IRQ enable */ 35 #define RK_MMU_INT_STATUS 0x20 /* IRQ status after masking */ 36 #define RK_MMU_AUTO_GATING 0x24 37 38 #define DTE_ADDR_DUMMY 0xCAFEBABE 39 #define FORCE_RESET_TIMEOUT 100 /* ms */ 40 41 /* RK_MMU_STATUS fields */ 42 #define RK_MMU_STATUS_PAGING_ENABLED BIT(0) 43 #define RK_MMU_STATUS_PAGE_FAULT_ACTIVE BIT(1) 44 #define RK_MMU_STATUS_STALL_ACTIVE BIT(2) 45 #define RK_MMU_STATUS_IDLE BIT(3) 46 #define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4) 47 #define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5) 48 #define RK_MMU_STATUS_STALL_NOT_ACTIVE BIT(31) 49 50 /* RK_MMU_COMMAND command values */ 51 #define RK_MMU_CMD_ENABLE_PAGING 0 /* Enable memory translation */ 52 #define RK_MMU_CMD_DISABLE_PAGING 1 /* Disable memory translation */ 53 #define RK_MMU_CMD_ENABLE_STALL 2 /* Stall paging to allow other cmds */ 54 #define RK_MMU_CMD_DISABLE_STALL 3 /* Stop stall re-enables paging */ 55 #define RK_MMU_CMD_ZAP_CACHE 4 /* Shoot down entire IOTLB */ 56 #define RK_MMU_CMD_PAGE_FAULT_DONE 5 /* Clear page fault */ 57 #define RK_MMU_CMD_FORCE_RESET 6 /* Reset all registers */ 58 59 /* RK_MMU_INT_* register fields */ 60 #define RK_MMU_IRQ_PAGE_FAULT 0x01 /* page fault */ 61 #define RK_MMU_IRQ_BUS_ERROR 0x02 /* bus read error */ 62 #define RK_MMU_IRQ_MASK (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR) 63 64 #define NUM_DT_ENTRIES 1024 65 #define NUM_PT_ENTRIES 1024 66 67 #define SPAGE_ORDER 12 68 #define SPAGE_SIZE (1 << SPAGE_ORDER) 69 70 /* 71 * Support mapping any size that fits in one page table: 72 * 4 KiB to 4 MiB 73 */ 74 #define RK_IOMMU_PGSIZE_BITMAP 0x007ff000 75 76 #define IOMMU_REG_POLL_COUNT_FAST 1000 77 78 struct rk_iommu_domain { 79 struct list_head iommus; 80 u32 *dt; /* page directory table */ 81 spinlock_t iommus_lock; /* lock for iommus list */ 82 spinlock_t dt_lock; /* lock for modifying page directory table */ 83 84 struct iommu_domain domain; 85 }; 86 87 struct rk_iommu { 88 struct device *dev; 89 void __iomem *base; 90 int irq; 91 struct list_head node; /* entry in rk_iommu_domain.iommus */ 92 struct iommu_domain *domain; /* domain to which iommu is attached */ 93 }; 94 95 static inline void rk_table_flush(u32 *va, unsigned int count) 96 { 97 phys_addr_t pa_start = virt_to_phys(va); 98 phys_addr_t pa_end = virt_to_phys(va + count); 99 size_t size = pa_end - pa_start; 100 101 __cpuc_flush_dcache_area(va, size); 102 outer_flush_range(pa_start, pa_end); 103 } 104 105 static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) 106 { 107 return container_of(dom, struct rk_iommu_domain, domain); 108 } 109 110 /** 111 * Inspired by _wait_for in intel_drv.h 112 * This is NOT safe for use in interrupt context. 113 * 114 * Note that it's important that we check the condition again after having 115 * timed out, since the timeout could be due to preemption or similar and 116 * we've never had a chance to check the condition before the timeout. 117 */ 118 #define rk_wait_for(COND, MS) ({ \ 119 unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ 120 int ret__ = 0; \ 121 while (!(COND)) { \ 122 if (time_after(jiffies, timeout__)) { \ 123 ret__ = (COND) ? 0 : -ETIMEDOUT; \ 124 break; \ 125 } \ 126 usleep_range(50, 100); \ 127 } \ 128 ret__; \ 129 }) 130 131 /* 132 * The Rockchip rk3288 iommu uses a 2-level page table. 133 * The first level is the "Directory Table" (DT). 134 * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing 135 * to a "Page Table". 136 * The second level is the 1024 Page Tables (PT). 137 * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to 138 * a 4 KB page of physical memory. 139 * 140 * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries). 141 * Each iommu device has a MMU_DTE_ADDR register that contains the physical 142 * address of the start of the DT page. 143 * 144 * The structure of the page table is as follows: 145 * 146 * DT 147 * MMU_DTE_ADDR -> +-----+ 148 * | | 149 * +-----+ PT 150 * | DTE | -> +-----+ 151 * +-----+ | | Memory 152 * | | +-----+ Page 153 * | | | PTE | -> +-----+ 154 * +-----+ +-----+ | | 155 * | | | | 156 * | | | | 157 * +-----+ | | 158 * | | 159 * | | 160 * +-----+ 161 */ 162 163 /* 164 * Each DTE has a PT address and a valid bit: 165 * +---------------------+-----------+-+ 166 * | PT address | Reserved |V| 167 * +---------------------+-----------+-+ 168 * 31:12 - PT address (PTs always starts on a 4 KB boundary) 169 * 11: 1 - Reserved 170 * 0 - 1 if PT @ PT address is valid 171 */ 172 #define RK_DTE_PT_ADDRESS_MASK 0xfffff000 173 #define RK_DTE_PT_VALID BIT(0) 174 175 static inline phys_addr_t rk_dte_pt_address(u32 dte) 176 { 177 return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK; 178 } 179 180 static inline bool rk_dte_is_pt_valid(u32 dte) 181 { 182 return dte & RK_DTE_PT_VALID; 183 } 184 185 static u32 rk_mk_dte(u32 *pt) 186 { 187 phys_addr_t pt_phys = virt_to_phys(pt); 188 return (pt_phys & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; 189 } 190 191 /* 192 * Each PTE has a Page address, some flags and a valid bit: 193 * +---------------------+---+-------+-+ 194 * | Page address |Rsv| Flags |V| 195 * +---------------------+---+-------+-+ 196 * 31:12 - Page address (Pages always start on a 4 KB boundary) 197 * 11: 9 - Reserved 198 * 8: 1 - Flags 199 * 8 - Read allocate - allocate cache space on read misses 200 * 7 - Read cache - enable cache & prefetch of data 201 * 6 - Write buffer - enable delaying writes on their way to memory 202 * 5 - Write allocate - allocate cache space on write misses 203 * 4 - Write cache - different writes can be merged together 204 * 3 - Override cache attributes 205 * if 1, bits 4-8 control cache attributes 206 * if 0, the system bus defaults are used 207 * 2 - Writable 208 * 1 - Readable 209 * 0 - 1 if Page @ Page address is valid 210 */ 211 #define RK_PTE_PAGE_ADDRESS_MASK 0xfffff000 212 #define RK_PTE_PAGE_FLAGS_MASK 0x000001fe 213 #define RK_PTE_PAGE_WRITABLE BIT(2) 214 #define RK_PTE_PAGE_READABLE BIT(1) 215 #define RK_PTE_PAGE_VALID BIT(0) 216 217 static inline phys_addr_t rk_pte_page_address(u32 pte) 218 { 219 return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK; 220 } 221 222 static inline bool rk_pte_is_page_valid(u32 pte) 223 { 224 return pte & RK_PTE_PAGE_VALID; 225 } 226 227 /* TODO: set cache flags per prot IOMMU_CACHE */ 228 static u32 rk_mk_pte(phys_addr_t page, int prot) 229 { 230 u32 flags = 0; 231 flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; 232 flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; 233 page &= RK_PTE_PAGE_ADDRESS_MASK; 234 return page | flags | RK_PTE_PAGE_VALID; 235 } 236 237 static u32 rk_mk_pte_invalid(u32 pte) 238 { 239 return pte & ~RK_PTE_PAGE_VALID; 240 } 241 242 /* 243 * rk3288 iova (IOMMU Virtual Address) format 244 * 31 22.21 12.11 0 245 * +-----------+-----------+-------------+ 246 * | DTE index | PTE index | Page offset | 247 * +-----------+-----------+-------------+ 248 * 31:22 - DTE index - index of DTE in DT 249 * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address 250 * 11: 0 - Page offset - offset into page @ PTE.page_address 251 */ 252 #define RK_IOVA_DTE_MASK 0xffc00000 253 #define RK_IOVA_DTE_SHIFT 22 254 #define RK_IOVA_PTE_MASK 0x003ff000 255 #define RK_IOVA_PTE_SHIFT 12 256 #define RK_IOVA_PAGE_MASK 0x00000fff 257 #define RK_IOVA_PAGE_SHIFT 0 258 259 static u32 rk_iova_dte_index(dma_addr_t iova) 260 { 261 return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT; 262 } 263 264 static u32 rk_iova_pte_index(dma_addr_t iova) 265 { 266 return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT; 267 } 268 269 static u32 rk_iova_page_offset(dma_addr_t iova) 270 { 271 return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT; 272 } 273 274 static u32 rk_iommu_read(struct rk_iommu *iommu, u32 offset) 275 { 276 return readl(iommu->base + offset); 277 } 278 279 static void rk_iommu_write(struct rk_iommu *iommu, u32 offset, u32 value) 280 { 281 writel(value, iommu->base + offset); 282 } 283 284 static void rk_iommu_command(struct rk_iommu *iommu, u32 command) 285 { 286 writel(command, iommu->base + RK_MMU_COMMAND); 287 } 288 289 static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova, 290 size_t size) 291 { 292 dma_addr_t iova_end = iova + size; 293 /* 294 * TODO(djkurtz): Figure out when it is more efficient to shootdown the 295 * entire iotlb rather than iterate over individual iovas. 296 */ 297 for (; iova < iova_end; iova += SPAGE_SIZE) 298 rk_iommu_write(iommu, RK_MMU_ZAP_ONE_LINE, iova); 299 } 300 301 static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) 302 { 303 return rk_iommu_read(iommu, RK_MMU_STATUS) & RK_MMU_STATUS_STALL_ACTIVE; 304 } 305 306 static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) 307 { 308 return rk_iommu_read(iommu, RK_MMU_STATUS) & 309 RK_MMU_STATUS_PAGING_ENABLED; 310 } 311 312 static int rk_iommu_enable_stall(struct rk_iommu *iommu) 313 { 314 int ret; 315 316 if (rk_iommu_is_stall_active(iommu)) 317 return 0; 318 319 /* Stall can only be enabled if paging is enabled */ 320 if (!rk_iommu_is_paging_enabled(iommu)) 321 return 0; 322 323 rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL); 324 325 ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1); 326 if (ret) 327 dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", 328 rk_iommu_read(iommu, RK_MMU_STATUS)); 329 330 return ret; 331 } 332 333 static int rk_iommu_disable_stall(struct rk_iommu *iommu) 334 { 335 int ret; 336 337 if (!rk_iommu_is_stall_active(iommu)) 338 return 0; 339 340 rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL); 341 342 ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1); 343 if (ret) 344 dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", 345 rk_iommu_read(iommu, RK_MMU_STATUS)); 346 347 return ret; 348 } 349 350 static int rk_iommu_enable_paging(struct rk_iommu *iommu) 351 { 352 int ret; 353 354 if (rk_iommu_is_paging_enabled(iommu)) 355 return 0; 356 357 rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING); 358 359 ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1); 360 if (ret) 361 dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", 362 rk_iommu_read(iommu, RK_MMU_STATUS)); 363 364 return ret; 365 } 366 367 static int rk_iommu_disable_paging(struct rk_iommu *iommu) 368 { 369 int ret; 370 371 if (!rk_iommu_is_paging_enabled(iommu)) 372 return 0; 373 374 rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING); 375 376 ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1); 377 if (ret) 378 dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", 379 rk_iommu_read(iommu, RK_MMU_STATUS)); 380 381 return ret; 382 } 383 384 static int rk_iommu_force_reset(struct rk_iommu *iommu) 385 { 386 int ret; 387 u32 dte_addr; 388 389 /* 390 * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY 391 * and verifying that upper 5 nybbles are read back. 392 */ 393 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY); 394 395 dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); 396 if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) { 397 dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n"); 398 return -EFAULT; 399 } 400 401 rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); 402 403 ret = rk_wait_for(rk_iommu_read(iommu, RK_MMU_DTE_ADDR) == 0x00000000, 404 FORCE_RESET_TIMEOUT); 405 if (ret) 406 dev_err(iommu->dev, "FORCE_RESET command timed out\n"); 407 408 return ret; 409 } 410 411 static void log_iova(struct rk_iommu *iommu, dma_addr_t iova) 412 { 413 u32 dte_index, pte_index, page_offset; 414 u32 mmu_dte_addr; 415 phys_addr_t mmu_dte_addr_phys, dte_addr_phys; 416 u32 *dte_addr; 417 u32 dte; 418 phys_addr_t pte_addr_phys = 0; 419 u32 *pte_addr = NULL; 420 u32 pte = 0; 421 phys_addr_t page_addr_phys = 0; 422 u32 page_flags = 0; 423 424 dte_index = rk_iova_dte_index(iova); 425 pte_index = rk_iova_pte_index(iova); 426 page_offset = rk_iova_page_offset(iova); 427 428 mmu_dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); 429 mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr; 430 431 dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index); 432 dte_addr = phys_to_virt(dte_addr_phys); 433 dte = *dte_addr; 434 435 if (!rk_dte_is_pt_valid(dte)) 436 goto print_it; 437 438 pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4); 439 pte_addr = phys_to_virt(pte_addr_phys); 440 pte = *pte_addr; 441 442 if (!rk_pte_is_page_valid(pte)) 443 goto print_it; 444 445 page_addr_phys = rk_pte_page_address(pte) + page_offset; 446 page_flags = pte & RK_PTE_PAGE_FLAGS_MASK; 447 448 print_it: 449 dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n", 450 &iova, dte_index, pte_index, page_offset); 451 dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n", 452 &mmu_dte_addr_phys, &dte_addr_phys, dte, 453 rk_dte_is_pt_valid(dte), &pte_addr_phys, pte, 454 rk_pte_is_page_valid(pte), &page_addr_phys, page_flags); 455 } 456 457 static irqreturn_t rk_iommu_irq(int irq, void *dev_id) 458 { 459 struct rk_iommu *iommu = dev_id; 460 u32 status; 461 u32 int_status; 462 dma_addr_t iova; 463 464 int_status = rk_iommu_read(iommu, RK_MMU_INT_STATUS); 465 if (int_status == 0) 466 return IRQ_NONE; 467 468 iova = rk_iommu_read(iommu, RK_MMU_PAGE_FAULT_ADDR); 469 470 if (int_status & RK_MMU_IRQ_PAGE_FAULT) { 471 int flags; 472 473 status = rk_iommu_read(iommu, RK_MMU_STATUS); 474 flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? 475 IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; 476 477 dev_err(iommu->dev, "Page fault at %pad of type %s\n", 478 &iova, 479 (flags == IOMMU_FAULT_WRITE) ? "write" : "read"); 480 481 log_iova(iommu, iova); 482 483 /* 484 * Report page fault to any installed handlers. 485 * Ignore the return code, though, since we always zap cache 486 * and clear the page fault anyway. 487 */ 488 if (iommu->domain) 489 report_iommu_fault(iommu->domain, iommu->dev, iova, 490 flags); 491 else 492 dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n"); 493 494 rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); 495 rk_iommu_command(iommu, RK_MMU_CMD_PAGE_FAULT_DONE); 496 } 497 498 if (int_status & RK_MMU_IRQ_BUS_ERROR) 499 dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova); 500 501 if (int_status & ~RK_MMU_IRQ_MASK) 502 dev_err(iommu->dev, "unexpected int_status: %#08x\n", 503 int_status); 504 505 rk_iommu_write(iommu, RK_MMU_INT_CLEAR, int_status); 506 507 return IRQ_HANDLED; 508 } 509 510 static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, 511 dma_addr_t iova) 512 { 513 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 514 unsigned long flags; 515 phys_addr_t pt_phys, phys = 0; 516 u32 dte, pte; 517 u32 *page_table; 518 519 spin_lock_irqsave(&rk_domain->dt_lock, flags); 520 521 dte = rk_domain->dt[rk_iova_dte_index(iova)]; 522 if (!rk_dte_is_pt_valid(dte)) 523 goto out; 524 525 pt_phys = rk_dte_pt_address(dte); 526 page_table = (u32 *)phys_to_virt(pt_phys); 527 pte = page_table[rk_iova_pte_index(iova)]; 528 if (!rk_pte_is_page_valid(pte)) 529 goto out; 530 531 phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova); 532 out: 533 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 534 535 return phys; 536 } 537 538 static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, 539 dma_addr_t iova, size_t size) 540 { 541 struct list_head *pos; 542 unsigned long flags; 543 544 /* shootdown these iova from all iommus using this domain */ 545 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 546 list_for_each(pos, &rk_domain->iommus) { 547 struct rk_iommu *iommu; 548 iommu = list_entry(pos, struct rk_iommu, node); 549 rk_iommu_zap_lines(iommu, iova, size); 550 } 551 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 552 } 553 554 static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, 555 dma_addr_t iova) 556 { 557 u32 *page_table, *dte_addr; 558 u32 dte; 559 phys_addr_t pt_phys; 560 561 assert_spin_locked(&rk_domain->dt_lock); 562 563 dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)]; 564 dte = *dte_addr; 565 if (rk_dte_is_pt_valid(dte)) 566 goto done; 567 568 page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 569 if (!page_table) 570 return ERR_PTR(-ENOMEM); 571 572 dte = rk_mk_dte(page_table); 573 *dte_addr = dte; 574 575 rk_table_flush(page_table, NUM_PT_ENTRIES); 576 rk_table_flush(dte_addr, 1); 577 578 /* 579 * Zap the first iova of newly allocated page table so iommu evicts 580 * old cached value of new dte from the iotlb. 581 */ 582 rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); 583 584 done: 585 pt_phys = rk_dte_pt_address(dte); 586 return (u32 *)phys_to_virt(pt_phys); 587 } 588 589 static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, 590 u32 *pte_addr, dma_addr_t iova, size_t size) 591 { 592 unsigned int pte_count; 593 unsigned int pte_total = size / SPAGE_SIZE; 594 595 assert_spin_locked(&rk_domain->dt_lock); 596 597 for (pte_count = 0; pte_count < pte_total; pte_count++) { 598 u32 pte = pte_addr[pte_count]; 599 if (!rk_pte_is_page_valid(pte)) 600 break; 601 602 pte_addr[pte_count] = rk_mk_pte_invalid(pte); 603 } 604 605 rk_table_flush(pte_addr, pte_count); 606 607 return pte_count * SPAGE_SIZE; 608 } 609 610 static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, 611 dma_addr_t iova, phys_addr_t paddr, size_t size, 612 int prot) 613 { 614 unsigned int pte_count; 615 unsigned int pte_total = size / SPAGE_SIZE; 616 phys_addr_t page_phys; 617 618 assert_spin_locked(&rk_domain->dt_lock); 619 620 for (pte_count = 0; pte_count < pte_total; pte_count++) { 621 u32 pte = pte_addr[pte_count]; 622 623 if (rk_pte_is_page_valid(pte)) 624 goto unwind; 625 626 pte_addr[pte_count] = rk_mk_pte(paddr, prot); 627 628 paddr += SPAGE_SIZE; 629 } 630 631 rk_table_flush(pte_addr, pte_count); 632 633 return 0; 634 unwind: 635 /* Unmap the range of iovas that we just mapped */ 636 rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE); 637 638 iova += pte_count * SPAGE_SIZE; 639 page_phys = rk_pte_page_address(pte_addr[pte_count]); 640 pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n", 641 &iova, &page_phys, &paddr, prot); 642 643 return -EADDRINUSE; 644 } 645 646 static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, 647 phys_addr_t paddr, size_t size, int prot) 648 { 649 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 650 unsigned long flags; 651 dma_addr_t iova = (dma_addr_t)_iova; 652 u32 *page_table, *pte_addr; 653 int ret; 654 655 spin_lock_irqsave(&rk_domain->dt_lock, flags); 656 657 /* 658 * pgsize_bitmap specifies iova sizes that fit in one page table 659 * (1024 4-KiB pages = 4 MiB). 660 * So, size will always be 4096 <= size <= 4194304. 661 * Since iommu_map() guarantees that both iova and size will be 662 * aligned, we will always only be mapping from a single dte here. 663 */ 664 page_table = rk_dte_get_page_table(rk_domain, iova); 665 if (IS_ERR(page_table)) { 666 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 667 return PTR_ERR(page_table); 668 } 669 670 pte_addr = &page_table[rk_iova_pte_index(iova)]; 671 ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot); 672 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 673 674 return ret; 675 } 676 677 static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, 678 size_t size) 679 { 680 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 681 unsigned long flags; 682 dma_addr_t iova = (dma_addr_t)_iova; 683 phys_addr_t pt_phys; 684 u32 dte; 685 u32 *pte_addr; 686 size_t unmap_size; 687 688 spin_lock_irqsave(&rk_domain->dt_lock, flags); 689 690 /* 691 * pgsize_bitmap specifies iova sizes that fit in one page table 692 * (1024 4-KiB pages = 4 MiB). 693 * So, size will always be 4096 <= size <= 4194304. 694 * Since iommu_unmap() guarantees that both iova and size will be 695 * aligned, we will always only be unmapping from a single dte here. 696 */ 697 dte = rk_domain->dt[rk_iova_dte_index(iova)]; 698 /* Just return 0 if iova is unmapped */ 699 if (!rk_dte_is_pt_valid(dte)) { 700 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 701 return 0; 702 } 703 704 pt_phys = rk_dte_pt_address(dte); 705 pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); 706 unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size); 707 708 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 709 710 /* Shootdown iotlb entries for iova range that was just unmapped */ 711 rk_iommu_zap_iova(rk_domain, iova, unmap_size); 712 713 return unmap_size; 714 } 715 716 static struct rk_iommu *rk_iommu_from_dev(struct device *dev) 717 { 718 struct iommu_group *group; 719 struct device *iommu_dev; 720 struct rk_iommu *rk_iommu; 721 722 group = iommu_group_get(dev); 723 if (!group) 724 return NULL; 725 iommu_dev = iommu_group_get_iommudata(group); 726 rk_iommu = dev_get_drvdata(iommu_dev); 727 iommu_group_put(group); 728 729 return rk_iommu; 730 } 731 732 static int rk_iommu_attach_device(struct iommu_domain *domain, 733 struct device *dev) 734 { 735 struct rk_iommu *iommu; 736 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 737 unsigned long flags; 738 int ret; 739 phys_addr_t dte_addr; 740 741 /* 742 * Allow 'virtual devices' (e.g., drm) to attach to domain. 743 * Such a device does not belong to an iommu group. 744 */ 745 iommu = rk_iommu_from_dev(dev); 746 if (!iommu) 747 return 0; 748 749 ret = rk_iommu_enable_stall(iommu); 750 if (ret) 751 return ret; 752 753 ret = rk_iommu_force_reset(iommu); 754 if (ret) 755 return ret; 756 757 iommu->domain = domain; 758 759 ret = devm_request_irq(dev, iommu->irq, rk_iommu_irq, 760 IRQF_SHARED, dev_name(dev), iommu); 761 if (ret) 762 return ret; 763 764 dte_addr = virt_to_phys(rk_domain->dt); 765 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, dte_addr); 766 rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); 767 rk_iommu_write(iommu, RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); 768 769 ret = rk_iommu_enable_paging(iommu); 770 if (ret) 771 return ret; 772 773 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 774 list_add_tail(&iommu->node, &rk_domain->iommus); 775 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 776 777 dev_info(dev, "Attached to iommu domain\n"); 778 779 rk_iommu_disable_stall(iommu); 780 781 return 0; 782 } 783 784 static void rk_iommu_detach_device(struct iommu_domain *domain, 785 struct device *dev) 786 { 787 struct rk_iommu *iommu; 788 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 789 unsigned long flags; 790 791 /* Allow 'virtual devices' (eg drm) to detach from domain */ 792 iommu = rk_iommu_from_dev(dev); 793 if (!iommu) 794 return; 795 796 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 797 list_del_init(&iommu->node); 798 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 799 800 /* Ignore error while disabling, just keep going */ 801 rk_iommu_enable_stall(iommu); 802 rk_iommu_disable_paging(iommu); 803 rk_iommu_write(iommu, RK_MMU_INT_MASK, 0); 804 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, 0); 805 rk_iommu_disable_stall(iommu); 806 807 devm_free_irq(dev, iommu->irq, iommu); 808 809 iommu->domain = NULL; 810 811 dev_info(dev, "Detached from iommu domain\n"); 812 } 813 814 static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) 815 { 816 struct rk_iommu_domain *rk_domain; 817 818 if (type != IOMMU_DOMAIN_UNMANAGED) 819 return NULL; 820 821 rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL); 822 if (!rk_domain) 823 return NULL; 824 825 /* 826 * rk32xx iommus use a 2 level pagetable. 827 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. 828 * Allocate one 4 KiB page for each table. 829 */ 830 rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); 831 if (!rk_domain->dt) 832 goto err_dt; 833 834 rk_table_flush(rk_domain->dt, NUM_DT_ENTRIES); 835 836 spin_lock_init(&rk_domain->iommus_lock); 837 spin_lock_init(&rk_domain->dt_lock); 838 INIT_LIST_HEAD(&rk_domain->iommus); 839 840 return &rk_domain->domain; 841 842 err_dt: 843 kfree(rk_domain); 844 return NULL; 845 } 846 847 static void rk_iommu_domain_free(struct iommu_domain *domain) 848 { 849 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 850 int i; 851 852 WARN_ON(!list_empty(&rk_domain->iommus)); 853 854 for (i = 0; i < NUM_DT_ENTRIES; i++) { 855 u32 dte = rk_domain->dt[i]; 856 if (rk_dte_is_pt_valid(dte)) { 857 phys_addr_t pt_phys = rk_dte_pt_address(dte); 858 u32 *page_table = phys_to_virt(pt_phys); 859 free_page((unsigned long)page_table); 860 } 861 } 862 863 free_page((unsigned long)rk_domain->dt); 864 kfree(rk_domain); 865 } 866 867 static bool rk_iommu_is_dev_iommu_master(struct device *dev) 868 { 869 struct device_node *np = dev->of_node; 870 int ret; 871 872 /* 873 * An iommu master has an iommus property containing a list of phandles 874 * to iommu nodes, each with an #iommu-cells property with value 0. 875 */ 876 ret = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); 877 return (ret > 0); 878 } 879 880 static int rk_iommu_group_set_iommudata(struct iommu_group *group, 881 struct device *dev) 882 { 883 struct device_node *np = dev->of_node; 884 struct platform_device *pd; 885 int ret; 886 struct of_phandle_args args; 887 888 /* 889 * An iommu master has an iommus property containing a list of phandles 890 * to iommu nodes, each with an #iommu-cells property with value 0. 891 */ 892 ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0, 893 &args); 894 if (ret) { 895 dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n", 896 np->full_name, ret); 897 return ret; 898 } 899 if (args.args_count != 0) { 900 dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n", 901 args.np->full_name, args.args_count); 902 return -EINVAL; 903 } 904 905 pd = of_find_device_by_node(args.np); 906 of_node_put(args.np); 907 if (!pd) { 908 dev_err(dev, "iommu %s not found\n", args.np->full_name); 909 return -EPROBE_DEFER; 910 } 911 912 /* TODO(djkurtz): handle multiple slave iommus for a single master */ 913 iommu_group_set_iommudata(group, &pd->dev, NULL); 914 915 return 0; 916 } 917 918 static int rk_iommu_add_device(struct device *dev) 919 { 920 struct iommu_group *group; 921 int ret; 922 923 if (!rk_iommu_is_dev_iommu_master(dev)) 924 return -ENODEV; 925 926 group = iommu_group_get(dev); 927 if (!group) { 928 group = iommu_group_alloc(); 929 if (IS_ERR(group)) { 930 dev_err(dev, "Failed to allocate IOMMU group\n"); 931 return PTR_ERR(group); 932 } 933 } 934 935 ret = iommu_group_add_device(group, dev); 936 if (ret) 937 goto err_put_group; 938 939 ret = rk_iommu_group_set_iommudata(group, dev); 940 if (ret) 941 goto err_remove_device; 942 943 iommu_group_put(group); 944 945 return 0; 946 947 err_remove_device: 948 iommu_group_remove_device(dev); 949 err_put_group: 950 iommu_group_put(group); 951 return ret; 952 } 953 954 static void rk_iommu_remove_device(struct device *dev) 955 { 956 if (!rk_iommu_is_dev_iommu_master(dev)) 957 return; 958 959 iommu_group_remove_device(dev); 960 } 961 962 static const struct iommu_ops rk_iommu_ops = { 963 .domain_alloc = rk_iommu_domain_alloc, 964 .domain_free = rk_iommu_domain_free, 965 .attach_dev = rk_iommu_attach_device, 966 .detach_dev = rk_iommu_detach_device, 967 .map = rk_iommu_map, 968 .unmap = rk_iommu_unmap, 969 .add_device = rk_iommu_add_device, 970 .remove_device = rk_iommu_remove_device, 971 .iova_to_phys = rk_iommu_iova_to_phys, 972 .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, 973 }; 974 975 static int rk_iommu_probe(struct platform_device *pdev) 976 { 977 struct device *dev = &pdev->dev; 978 struct rk_iommu *iommu; 979 struct resource *res; 980 981 iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); 982 if (!iommu) 983 return -ENOMEM; 984 985 platform_set_drvdata(pdev, iommu); 986 iommu->dev = dev; 987 988 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 989 iommu->base = devm_ioremap_resource(&pdev->dev, res); 990 if (IS_ERR(iommu->base)) 991 return PTR_ERR(iommu->base); 992 993 iommu->irq = platform_get_irq(pdev, 0); 994 if (iommu->irq < 0) { 995 dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq); 996 return -ENXIO; 997 } 998 999 return 0; 1000 } 1001 1002 static int rk_iommu_remove(struct platform_device *pdev) 1003 { 1004 return 0; 1005 } 1006 1007 #ifdef CONFIG_OF 1008 static const struct of_device_id rk_iommu_dt_ids[] = { 1009 { .compatible = "rockchip,iommu" }, 1010 { /* sentinel */ } 1011 }; 1012 MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); 1013 #endif 1014 1015 static struct platform_driver rk_iommu_driver = { 1016 .probe = rk_iommu_probe, 1017 .remove = rk_iommu_remove, 1018 .driver = { 1019 .name = "rk_iommu", 1020 .of_match_table = of_match_ptr(rk_iommu_dt_ids), 1021 }, 1022 }; 1023 1024 static int __init rk_iommu_init(void) 1025 { 1026 struct device_node *np; 1027 int ret; 1028 1029 np = of_find_matching_node(NULL, rk_iommu_dt_ids); 1030 if (!np) 1031 return 0; 1032 1033 of_node_put(np); 1034 1035 ret = bus_set_iommu(&platform_bus_type, &rk_iommu_ops); 1036 if (ret) 1037 return ret; 1038 1039 return platform_driver_register(&rk_iommu_driver); 1040 } 1041 static void __exit rk_iommu_exit(void) 1042 { 1043 platform_driver_unregister(&rk_iommu_driver); 1044 } 1045 1046 subsys_initcall(rk_iommu_init); 1047 module_exit(rk_iommu_exit); 1048 1049 MODULE_DESCRIPTION("IOMMU API for Rockchip"); 1050 MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com> and Daniel Kurtz <djkurtz@chromium.org>"); 1051 MODULE_ALIAS("platform:rockchip-iommu"); 1052 MODULE_LICENSE("GPL v2"); 1053