1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License version 2 as 4 * published by the Free Software Foundation. 5 */ 6 7 #include <asm/cacheflush.h> 8 #include <asm/pgtable.h> 9 #include <linux/compiler.h> 10 #include <linux/delay.h> 11 #include <linux/device.h> 12 #include <linux/errno.h> 13 #include <linux/interrupt.h> 14 #include <linux/io.h> 15 #include <linux/iommu.h> 16 #include <linux/jiffies.h> 17 #include <linux/list.h> 18 #include <linux/mm.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_platform.h> 22 #include <linux/platform_device.h> 23 #include <linux/slab.h> 24 #include <linux/spinlock.h> 25 26 /** MMU register offsets */ 27 #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ 28 #define RK_MMU_STATUS 0x04 29 #define RK_MMU_COMMAND 0x08 30 #define RK_MMU_PAGE_FAULT_ADDR 0x0C /* IOVA of last page fault */ 31 #define RK_MMU_ZAP_ONE_LINE 0x10 /* Shootdown one IOTLB entry */ 32 #define RK_MMU_INT_RAWSTAT 0x14 /* IRQ status ignoring mask */ 33 #define RK_MMU_INT_CLEAR 0x18 /* Acknowledge and re-arm irq */ 34 #define RK_MMU_INT_MASK 0x1C /* IRQ enable */ 35 #define RK_MMU_INT_STATUS 0x20 /* IRQ status after masking */ 36 #define RK_MMU_AUTO_GATING 0x24 37 38 #define DTE_ADDR_DUMMY 0xCAFEBABE 39 #define FORCE_RESET_TIMEOUT 100 /* ms */ 40 41 /* RK_MMU_STATUS fields */ 42 #define RK_MMU_STATUS_PAGING_ENABLED BIT(0) 43 #define RK_MMU_STATUS_PAGE_FAULT_ACTIVE BIT(1) 44 #define RK_MMU_STATUS_STALL_ACTIVE BIT(2) 45 #define RK_MMU_STATUS_IDLE BIT(3) 46 #define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4) 47 #define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5) 48 #define RK_MMU_STATUS_STALL_NOT_ACTIVE BIT(31) 49 50 /* RK_MMU_COMMAND command values */ 51 #define RK_MMU_CMD_ENABLE_PAGING 0 /* Enable memory translation */ 52 #define RK_MMU_CMD_DISABLE_PAGING 1 /* Disable memory translation */ 53 #define RK_MMU_CMD_ENABLE_STALL 2 /* Stall paging to allow other cmds */ 54 #define RK_MMU_CMD_DISABLE_STALL 3 /* Stop stall re-enables paging */ 55 #define RK_MMU_CMD_ZAP_CACHE 4 /* Shoot down entire IOTLB */ 56 #define RK_MMU_CMD_PAGE_FAULT_DONE 5 /* Clear page fault */ 57 #define RK_MMU_CMD_FORCE_RESET 6 /* Reset all registers */ 58 59 /* RK_MMU_INT_* register fields */ 60 #define RK_MMU_IRQ_PAGE_FAULT 0x01 /* page fault */ 61 #define RK_MMU_IRQ_BUS_ERROR 0x02 /* bus read error */ 62 #define RK_MMU_IRQ_MASK (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR) 63 64 #define NUM_DT_ENTRIES 1024 65 #define NUM_PT_ENTRIES 1024 66 67 #define SPAGE_ORDER 12 68 #define SPAGE_SIZE (1 << SPAGE_ORDER) 69 70 /* 71 * Support mapping any size that fits in one page table: 72 * 4 KiB to 4 MiB 73 */ 74 #define RK_IOMMU_PGSIZE_BITMAP 0x007ff000 75 76 #define IOMMU_REG_POLL_COUNT_FAST 1000 77 78 struct rk_iommu_domain { 79 struct list_head iommus; 80 u32 *dt; /* page directory table */ 81 spinlock_t iommus_lock; /* lock for iommus list */ 82 spinlock_t dt_lock; /* lock for modifying page directory table */ 83 84 struct iommu_domain domain; 85 }; 86 87 struct rk_iommu { 88 struct device *dev; 89 void __iomem *base; 90 int irq; 91 struct list_head node; /* entry in rk_iommu_domain.iommus */ 92 struct iommu_domain *domain; /* domain to which iommu is attached */ 93 }; 94 95 static inline void rk_table_flush(u32 *va, unsigned int count) 96 { 97 phys_addr_t pa_start = virt_to_phys(va); 98 phys_addr_t pa_end = virt_to_phys(va + count); 99 size_t size = pa_end - pa_start; 100 101 __cpuc_flush_dcache_area(va, size); 102 outer_flush_range(pa_start, pa_end); 103 } 104 105 static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) 106 { 107 return container_of(dom, struct rk_iommu_domain, domain); 108 } 109 110 /** 111 * Inspired by _wait_for in intel_drv.h 112 * This is NOT safe for use in interrupt context. 113 * 114 * Note that it's important that we check the condition again after having 115 * timed out, since the timeout could be due to preemption or similar and 116 * we've never had a chance to check the condition before the timeout. 117 */ 118 #define rk_wait_for(COND, MS) ({ \ 119 unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ 120 int ret__ = 0; \ 121 while (!(COND)) { \ 122 if (time_after(jiffies, timeout__)) { \ 123 ret__ = (COND) ? 0 : -ETIMEDOUT; \ 124 break; \ 125 } \ 126 usleep_range(50, 100); \ 127 } \ 128 ret__; \ 129 }) 130 131 /* 132 * The Rockchip rk3288 iommu uses a 2-level page table. 133 * The first level is the "Directory Table" (DT). 134 * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing 135 * to a "Page Table". 136 * The second level is the 1024 Page Tables (PT). 137 * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to 138 * a 4 KB page of physical memory. 139 * 140 * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries). 141 * Each iommu device has a MMU_DTE_ADDR register that contains the physical 142 * address of the start of the DT page. 143 * 144 * The structure of the page table is as follows: 145 * 146 * DT 147 * MMU_DTE_ADDR -> +-----+ 148 * | | 149 * +-----+ PT 150 * | DTE | -> +-----+ 151 * +-----+ | | Memory 152 * | | +-----+ Page 153 * | | | PTE | -> +-----+ 154 * +-----+ +-----+ | | 155 * | | | | 156 * | | | | 157 * +-----+ | | 158 * | | 159 * | | 160 * +-----+ 161 */ 162 163 /* 164 * Each DTE has a PT address and a valid bit: 165 * +---------------------+-----------+-+ 166 * | PT address | Reserved |V| 167 * +---------------------+-----------+-+ 168 * 31:12 - PT address (PTs always starts on a 4 KB boundary) 169 * 11: 1 - Reserved 170 * 0 - 1 if PT @ PT address is valid 171 */ 172 #define RK_DTE_PT_ADDRESS_MASK 0xfffff000 173 #define RK_DTE_PT_VALID BIT(0) 174 175 static inline phys_addr_t rk_dte_pt_address(u32 dte) 176 { 177 return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK; 178 } 179 180 static inline bool rk_dte_is_pt_valid(u32 dte) 181 { 182 return dte & RK_DTE_PT_VALID; 183 } 184 185 static u32 rk_mk_dte(u32 *pt) 186 { 187 phys_addr_t pt_phys = virt_to_phys(pt); 188 return (pt_phys & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; 189 } 190 191 /* 192 * Each PTE has a Page address, some flags and a valid bit: 193 * +---------------------+---+-------+-+ 194 * | Page address |Rsv| Flags |V| 195 * +---------------------+---+-------+-+ 196 * 31:12 - Page address (Pages always start on a 4 KB boundary) 197 * 11: 9 - Reserved 198 * 8: 1 - Flags 199 * 8 - Read allocate - allocate cache space on read misses 200 * 7 - Read cache - enable cache & prefetch of data 201 * 6 - Write buffer - enable delaying writes on their way to memory 202 * 5 - Write allocate - allocate cache space on write misses 203 * 4 - Write cache - different writes can be merged together 204 * 3 - Override cache attributes 205 * if 1, bits 4-8 control cache attributes 206 * if 0, the system bus defaults are used 207 * 2 - Writable 208 * 1 - Readable 209 * 0 - 1 if Page @ Page address is valid 210 */ 211 #define RK_PTE_PAGE_ADDRESS_MASK 0xfffff000 212 #define RK_PTE_PAGE_FLAGS_MASK 0x000001fe 213 #define RK_PTE_PAGE_WRITABLE BIT(2) 214 #define RK_PTE_PAGE_READABLE BIT(1) 215 #define RK_PTE_PAGE_VALID BIT(0) 216 217 static inline phys_addr_t rk_pte_page_address(u32 pte) 218 { 219 return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK; 220 } 221 222 static inline bool rk_pte_is_page_valid(u32 pte) 223 { 224 return pte & RK_PTE_PAGE_VALID; 225 } 226 227 /* TODO: set cache flags per prot IOMMU_CACHE */ 228 static u32 rk_mk_pte(phys_addr_t page, int prot) 229 { 230 u32 flags = 0; 231 flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; 232 flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; 233 page &= RK_PTE_PAGE_ADDRESS_MASK; 234 return page | flags | RK_PTE_PAGE_VALID; 235 } 236 237 static u32 rk_mk_pte_invalid(u32 pte) 238 { 239 return pte & ~RK_PTE_PAGE_VALID; 240 } 241 242 /* 243 * rk3288 iova (IOMMU Virtual Address) format 244 * 31 22.21 12.11 0 245 * +-----------+-----------+-------------+ 246 * | DTE index | PTE index | Page offset | 247 * +-----------+-----------+-------------+ 248 * 31:22 - DTE index - index of DTE in DT 249 * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address 250 * 11: 0 - Page offset - offset into page @ PTE.page_address 251 */ 252 #define RK_IOVA_DTE_MASK 0xffc00000 253 #define RK_IOVA_DTE_SHIFT 22 254 #define RK_IOVA_PTE_MASK 0x003ff000 255 #define RK_IOVA_PTE_SHIFT 12 256 #define RK_IOVA_PAGE_MASK 0x00000fff 257 #define RK_IOVA_PAGE_SHIFT 0 258 259 static u32 rk_iova_dte_index(dma_addr_t iova) 260 { 261 return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT; 262 } 263 264 static u32 rk_iova_pte_index(dma_addr_t iova) 265 { 266 return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT; 267 } 268 269 static u32 rk_iova_page_offset(dma_addr_t iova) 270 { 271 return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT; 272 } 273 274 static u32 rk_iommu_read(struct rk_iommu *iommu, u32 offset) 275 { 276 return readl(iommu->base + offset); 277 } 278 279 static void rk_iommu_write(struct rk_iommu *iommu, u32 offset, u32 value) 280 { 281 writel(value, iommu->base + offset); 282 } 283 284 static void rk_iommu_command(struct rk_iommu *iommu, u32 command) 285 { 286 writel(command, iommu->base + RK_MMU_COMMAND); 287 } 288 289 static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova, 290 size_t size) 291 { 292 dma_addr_t iova_end = iova + size; 293 /* 294 * TODO(djkurtz): Figure out when it is more efficient to shootdown the 295 * entire iotlb rather than iterate over individual iovas. 296 */ 297 for (; iova < iova_end; iova += SPAGE_SIZE) 298 rk_iommu_write(iommu, RK_MMU_ZAP_ONE_LINE, iova); 299 } 300 301 static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) 302 { 303 return rk_iommu_read(iommu, RK_MMU_STATUS) & RK_MMU_STATUS_STALL_ACTIVE; 304 } 305 306 static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) 307 { 308 return rk_iommu_read(iommu, RK_MMU_STATUS) & 309 RK_MMU_STATUS_PAGING_ENABLED; 310 } 311 312 static int rk_iommu_enable_stall(struct rk_iommu *iommu) 313 { 314 int ret; 315 316 if (rk_iommu_is_stall_active(iommu)) 317 return 0; 318 319 /* Stall can only be enabled if paging is enabled */ 320 if (!rk_iommu_is_paging_enabled(iommu)) 321 return 0; 322 323 rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL); 324 325 ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1); 326 if (ret) 327 dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", 328 rk_iommu_read(iommu, RK_MMU_STATUS)); 329 330 return ret; 331 } 332 333 static int rk_iommu_disable_stall(struct rk_iommu *iommu) 334 { 335 int ret; 336 337 if (!rk_iommu_is_stall_active(iommu)) 338 return 0; 339 340 rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL); 341 342 ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1); 343 if (ret) 344 dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", 345 rk_iommu_read(iommu, RK_MMU_STATUS)); 346 347 return ret; 348 } 349 350 static int rk_iommu_enable_paging(struct rk_iommu *iommu) 351 { 352 int ret; 353 354 if (rk_iommu_is_paging_enabled(iommu)) 355 return 0; 356 357 rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING); 358 359 ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1); 360 if (ret) 361 dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", 362 rk_iommu_read(iommu, RK_MMU_STATUS)); 363 364 return ret; 365 } 366 367 static int rk_iommu_disable_paging(struct rk_iommu *iommu) 368 { 369 int ret; 370 371 if (!rk_iommu_is_paging_enabled(iommu)) 372 return 0; 373 374 rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING); 375 376 ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1); 377 if (ret) 378 dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", 379 rk_iommu_read(iommu, RK_MMU_STATUS)); 380 381 return ret; 382 } 383 384 static int rk_iommu_force_reset(struct rk_iommu *iommu) 385 { 386 int ret; 387 u32 dte_addr; 388 389 /* 390 * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY 391 * and verifying that upper 5 nybbles are read back. 392 */ 393 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY); 394 395 dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); 396 if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) { 397 dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n"); 398 return -EFAULT; 399 } 400 401 rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); 402 403 ret = rk_wait_for(rk_iommu_read(iommu, RK_MMU_DTE_ADDR) == 0x00000000, 404 FORCE_RESET_TIMEOUT); 405 if (ret) 406 dev_err(iommu->dev, "FORCE_RESET command timed out\n"); 407 408 return ret; 409 } 410 411 static void log_iova(struct rk_iommu *iommu, dma_addr_t iova) 412 { 413 u32 dte_index, pte_index, page_offset; 414 u32 mmu_dte_addr; 415 phys_addr_t mmu_dte_addr_phys, dte_addr_phys; 416 u32 *dte_addr; 417 u32 dte; 418 phys_addr_t pte_addr_phys = 0; 419 u32 *pte_addr = NULL; 420 u32 pte = 0; 421 phys_addr_t page_addr_phys = 0; 422 u32 page_flags = 0; 423 424 dte_index = rk_iova_dte_index(iova); 425 pte_index = rk_iova_pte_index(iova); 426 page_offset = rk_iova_page_offset(iova); 427 428 mmu_dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); 429 mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr; 430 431 dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index); 432 dte_addr = phys_to_virt(dte_addr_phys); 433 dte = *dte_addr; 434 435 if (!rk_dte_is_pt_valid(dte)) 436 goto print_it; 437 438 pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4); 439 pte_addr = phys_to_virt(pte_addr_phys); 440 pte = *pte_addr; 441 442 if (!rk_pte_is_page_valid(pte)) 443 goto print_it; 444 445 page_addr_phys = rk_pte_page_address(pte) + page_offset; 446 page_flags = pte & RK_PTE_PAGE_FLAGS_MASK; 447 448 print_it: 449 dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n", 450 &iova, dte_index, pte_index, page_offset); 451 dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n", 452 &mmu_dte_addr_phys, &dte_addr_phys, dte, 453 rk_dte_is_pt_valid(dte), &pte_addr_phys, pte, 454 rk_pte_is_page_valid(pte), &page_addr_phys, page_flags); 455 } 456 457 static irqreturn_t rk_iommu_irq(int irq, void *dev_id) 458 { 459 struct rk_iommu *iommu = dev_id; 460 u32 status; 461 u32 int_status; 462 dma_addr_t iova; 463 464 int_status = rk_iommu_read(iommu, RK_MMU_INT_STATUS); 465 if (int_status == 0) 466 return IRQ_NONE; 467 468 iova = rk_iommu_read(iommu, RK_MMU_PAGE_FAULT_ADDR); 469 470 if (int_status & RK_MMU_IRQ_PAGE_FAULT) { 471 int flags; 472 473 status = rk_iommu_read(iommu, RK_MMU_STATUS); 474 flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? 475 IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; 476 477 dev_err(iommu->dev, "Page fault at %pad of type %s\n", 478 &iova, 479 (flags == IOMMU_FAULT_WRITE) ? "write" : "read"); 480 481 log_iova(iommu, iova); 482 483 /* 484 * Report page fault to any installed handlers. 485 * Ignore the return code, though, since we always zap cache 486 * and clear the page fault anyway. 487 */ 488 if (iommu->domain) 489 report_iommu_fault(iommu->domain, iommu->dev, iova, 490 flags); 491 else 492 dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n"); 493 494 rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); 495 rk_iommu_command(iommu, RK_MMU_CMD_PAGE_FAULT_DONE); 496 } 497 498 if (int_status & RK_MMU_IRQ_BUS_ERROR) 499 dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova); 500 501 if (int_status & ~RK_MMU_IRQ_MASK) 502 dev_err(iommu->dev, "unexpected int_status: %#08x\n", 503 int_status); 504 505 rk_iommu_write(iommu, RK_MMU_INT_CLEAR, int_status); 506 507 return IRQ_HANDLED; 508 } 509 510 static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, 511 dma_addr_t iova) 512 { 513 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 514 unsigned long flags; 515 phys_addr_t pt_phys, phys = 0; 516 u32 dte, pte; 517 u32 *page_table; 518 519 spin_lock_irqsave(&rk_domain->dt_lock, flags); 520 521 dte = rk_domain->dt[rk_iova_dte_index(iova)]; 522 if (!rk_dte_is_pt_valid(dte)) 523 goto out; 524 525 pt_phys = rk_dte_pt_address(dte); 526 page_table = (u32 *)phys_to_virt(pt_phys); 527 pte = page_table[rk_iova_pte_index(iova)]; 528 if (!rk_pte_is_page_valid(pte)) 529 goto out; 530 531 phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova); 532 out: 533 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 534 535 return phys; 536 } 537 538 static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, 539 dma_addr_t iova, size_t size) 540 { 541 struct list_head *pos; 542 unsigned long flags; 543 544 /* shootdown these iova from all iommus using this domain */ 545 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 546 list_for_each(pos, &rk_domain->iommus) { 547 struct rk_iommu *iommu; 548 iommu = list_entry(pos, struct rk_iommu, node); 549 rk_iommu_zap_lines(iommu, iova, size); 550 } 551 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 552 } 553 554 static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, 555 dma_addr_t iova, size_t size) 556 { 557 rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); 558 if (size > SPAGE_SIZE) 559 rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE, 560 SPAGE_SIZE); 561 } 562 563 static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, 564 dma_addr_t iova) 565 { 566 u32 *page_table, *dte_addr; 567 u32 dte; 568 phys_addr_t pt_phys; 569 570 assert_spin_locked(&rk_domain->dt_lock); 571 572 dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)]; 573 dte = *dte_addr; 574 if (rk_dte_is_pt_valid(dte)) 575 goto done; 576 577 page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 578 if (!page_table) 579 return ERR_PTR(-ENOMEM); 580 581 dte = rk_mk_dte(page_table); 582 *dte_addr = dte; 583 584 rk_table_flush(page_table, NUM_PT_ENTRIES); 585 rk_table_flush(dte_addr, 1); 586 587 done: 588 pt_phys = rk_dte_pt_address(dte); 589 return (u32 *)phys_to_virt(pt_phys); 590 } 591 592 static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, 593 u32 *pte_addr, dma_addr_t iova, size_t size) 594 { 595 unsigned int pte_count; 596 unsigned int pte_total = size / SPAGE_SIZE; 597 598 assert_spin_locked(&rk_domain->dt_lock); 599 600 for (pte_count = 0; pte_count < pte_total; pte_count++) { 601 u32 pte = pte_addr[pte_count]; 602 if (!rk_pte_is_page_valid(pte)) 603 break; 604 605 pte_addr[pte_count] = rk_mk_pte_invalid(pte); 606 } 607 608 rk_table_flush(pte_addr, pte_count); 609 610 return pte_count * SPAGE_SIZE; 611 } 612 613 static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, 614 dma_addr_t iova, phys_addr_t paddr, size_t size, 615 int prot) 616 { 617 unsigned int pte_count; 618 unsigned int pte_total = size / SPAGE_SIZE; 619 phys_addr_t page_phys; 620 621 assert_spin_locked(&rk_domain->dt_lock); 622 623 for (pte_count = 0; pte_count < pte_total; pte_count++) { 624 u32 pte = pte_addr[pte_count]; 625 626 if (rk_pte_is_page_valid(pte)) 627 goto unwind; 628 629 pte_addr[pte_count] = rk_mk_pte(paddr, prot); 630 631 paddr += SPAGE_SIZE; 632 } 633 634 rk_table_flush(pte_addr, pte_count); 635 636 /* 637 * Zap the first and last iova to evict from iotlb any previously 638 * mapped cachelines holding stale values for its dte and pte. 639 * We only zap the first and last iova, since only they could have 640 * dte or pte shared with an existing mapping. 641 */ 642 rk_iommu_zap_iova_first_last(rk_domain, iova, size); 643 644 return 0; 645 unwind: 646 /* Unmap the range of iovas that we just mapped */ 647 rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE); 648 649 iova += pte_count * SPAGE_SIZE; 650 page_phys = rk_pte_page_address(pte_addr[pte_count]); 651 pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n", 652 &iova, &page_phys, &paddr, prot); 653 654 return -EADDRINUSE; 655 } 656 657 static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, 658 phys_addr_t paddr, size_t size, int prot) 659 { 660 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 661 unsigned long flags; 662 dma_addr_t iova = (dma_addr_t)_iova; 663 u32 *page_table, *pte_addr; 664 int ret; 665 666 spin_lock_irqsave(&rk_domain->dt_lock, flags); 667 668 /* 669 * pgsize_bitmap specifies iova sizes that fit in one page table 670 * (1024 4-KiB pages = 4 MiB). 671 * So, size will always be 4096 <= size <= 4194304. 672 * Since iommu_map() guarantees that both iova and size will be 673 * aligned, we will always only be mapping from a single dte here. 674 */ 675 page_table = rk_dte_get_page_table(rk_domain, iova); 676 if (IS_ERR(page_table)) { 677 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 678 return PTR_ERR(page_table); 679 } 680 681 pte_addr = &page_table[rk_iova_pte_index(iova)]; 682 ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot); 683 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 684 685 return ret; 686 } 687 688 static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, 689 size_t size) 690 { 691 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 692 unsigned long flags; 693 dma_addr_t iova = (dma_addr_t)_iova; 694 phys_addr_t pt_phys; 695 u32 dte; 696 u32 *pte_addr; 697 size_t unmap_size; 698 699 spin_lock_irqsave(&rk_domain->dt_lock, flags); 700 701 /* 702 * pgsize_bitmap specifies iova sizes that fit in one page table 703 * (1024 4-KiB pages = 4 MiB). 704 * So, size will always be 4096 <= size <= 4194304. 705 * Since iommu_unmap() guarantees that both iova and size will be 706 * aligned, we will always only be unmapping from a single dte here. 707 */ 708 dte = rk_domain->dt[rk_iova_dte_index(iova)]; 709 /* Just return 0 if iova is unmapped */ 710 if (!rk_dte_is_pt_valid(dte)) { 711 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 712 return 0; 713 } 714 715 pt_phys = rk_dte_pt_address(dte); 716 pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); 717 unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size); 718 719 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 720 721 /* Shootdown iotlb entries for iova range that was just unmapped */ 722 rk_iommu_zap_iova(rk_domain, iova, unmap_size); 723 724 return unmap_size; 725 } 726 727 static struct rk_iommu *rk_iommu_from_dev(struct device *dev) 728 { 729 struct iommu_group *group; 730 struct device *iommu_dev; 731 struct rk_iommu *rk_iommu; 732 733 group = iommu_group_get(dev); 734 if (!group) 735 return NULL; 736 iommu_dev = iommu_group_get_iommudata(group); 737 rk_iommu = dev_get_drvdata(iommu_dev); 738 iommu_group_put(group); 739 740 return rk_iommu; 741 } 742 743 static int rk_iommu_attach_device(struct iommu_domain *domain, 744 struct device *dev) 745 { 746 struct rk_iommu *iommu; 747 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 748 unsigned long flags; 749 int ret; 750 phys_addr_t dte_addr; 751 752 /* 753 * Allow 'virtual devices' (e.g., drm) to attach to domain. 754 * Such a device does not belong to an iommu group. 755 */ 756 iommu = rk_iommu_from_dev(dev); 757 if (!iommu) 758 return 0; 759 760 ret = rk_iommu_enable_stall(iommu); 761 if (ret) 762 return ret; 763 764 ret = rk_iommu_force_reset(iommu); 765 if (ret) 766 return ret; 767 768 iommu->domain = domain; 769 770 ret = devm_request_irq(dev, iommu->irq, rk_iommu_irq, 771 IRQF_SHARED, dev_name(dev), iommu); 772 if (ret) 773 return ret; 774 775 dte_addr = virt_to_phys(rk_domain->dt); 776 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, dte_addr); 777 rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); 778 rk_iommu_write(iommu, RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); 779 780 ret = rk_iommu_enable_paging(iommu); 781 if (ret) 782 return ret; 783 784 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 785 list_add_tail(&iommu->node, &rk_domain->iommus); 786 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 787 788 dev_dbg(dev, "Attached to iommu domain\n"); 789 790 rk_iommu_disable_stall(iommu); 791 792 return 0; 793 } 794 795 static void rk_iommu_detach_device(struct iommu_domain *domain, 796 struct device *dev) 797 { 798 struct rk_iommu *iommu; 799 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 800 unsigned long flags; 801 802 /* Allow 'virtual devices' (eg drm) to detach from domain */ 803 iommu = rk_iommu_from_dev(dev); 804 if (!iommu) 805 return; 806 807 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 808 list_del_init(&iommu->node); 809 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 810 811 /* Ignore error while disabling, just keep going */ 812 rk_iommu_enable_stall(iommu); 813 rk_iommu_disable_paging(iommu); 814 rk_iommu_write(iommu, RK_MMU_INT_MASK, 0); 815 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, 0); 816 rk_iommu_disable_stall(iommu); 817 818 devm_free_irq(dev, iommu->irq, iommu); 819 820 iommu->domain = NULL; 821 822 dev_dbg(dev, "Detached from iommu domain\n"); 823 } 824 825 static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) 826 { 827 struct rk_iommu_domain *rk_domain; 828 829 if (type != IOMMU_DOMAIN_UNMANAGED) 830 return NULL; 831 832 rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL); 833 if (!rk_domain) 834 return NULL; 835 836 /* 837 * rk32xx iommus use a 2 level pagetable. 838 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. 839 * Allocate one 4 KiB page for each table. 840 */ 841 rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); 842 if (!rk_domain->dt) 843 goto err_dt; 844 845 rk_table_flush(rk_domain->dt, NUM_DT_ENTRIES); 846 847 spin_lock_init(&rk_domain->iommus_lock); 848 spin_lock_init(&rk_domain->dt_lock); 849 INIT_LIST_HEAD(&rk_domain->iommus); 850 851 return &rk_domain->domain; 852 853 err_dt: 854 kfree(rk_domain); 855 return NULL; 856 } 857 858 static void rk_iommu_domain_free(struct iommu_domain *domain) 859 { 860 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 861 int i; 862 863 WARN_ON(!list_empty(&rk_domain->iommus)); 864 865 for (i = 0; i < NUM_DT_ENTRIES; i++) { 866 u32 dte = rk_domain->dt[i]; 867 if (rk_dte_is_pt_valid(dte)) { 868 phys_addr_t pt_phys = rk_dte_pt_address(dte); 869 u32 *page_table = phys_to_virt(pt_phys); 870 free_page((unsigned long)page_table); 871 } 872 } 873 874 free_page((unsigned long)rk_domain->dt); 875 kfree(rk_domain); 876 } 877 878 static bool rk_iommu_is_dev_iommu_master(struct device *dev) 879 { 880 struct device_node *np = dev->of_node; 881 int ret; 882 883 /* 884 * An iommu master has an iommus property containing a list of phandles 885 * to iommu nodes, each with an #iommu-cells property with value 0. 886 */ 887 ret = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); 888 return (ret > 0); 889 } 890 891 static int rk_iommu_group_set_iommudata(struct iommu_group *group, 892 struct device *dev) 893 { 894 struct device_node *np = dev->of_node; 895 struct platform_device *pd; 896 int ret; 897 struct of_phandle_args args; 898 899 /* 900 * An iommu master has an iommus property containing a list of phandles 901 * to iommu nodes, each with an #iommu-cells property with value 0. 902 */ 903 ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0, 904 &args); 905 if (ret) { 906 dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n", 907 np->full_name, ret); 908 return ret; 909 } 910 if (args.args_count != 0) { 911 dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n", 912 args.np->full_name, args.args_count); 913 return -EINVAL; 914 } 915 916 pd = of_find_device_by_node(args.np); 917 of_node_put(args.np); 918 if (!pd) { 919 dev_err(dev, "iommu %s not found\n", args.np->full_name); 920 return -EPROBE_DEFER; 921 } 922 923 /* TODO(djkurtz): handle multiple slave iommus for a single master */ 924 iommu_group_set_iommudata(group, &pd->dev, NULL); 925 926 return 0; 927 } 928 929 static int rk_iommu_add_device(struct device *dev) 930 { 931 struct iommu_group *group; 932 int ret; 933 934 if (!rk_iommu_is_dev_iommu_master(dev)) 935 return -ENODEV; 936 937 group = iommu_group_get(dev); 938 if (!group) { 939 group = iommu_group_alloc(); 940 if (IS_ERR(group)) { 941 dev_err(dev, "Failed to allocate IOMMU group\n"); 942 return PTR_ERR(group); 943 } 944 } 945 946 ret = iommu_group_add_device(group, dev); 947 if (ret) 948 goto err_put_group; 949 950 ret = rk_iommu_group_set_iommudata(group, dev); 951 if (ret) 952 goto err_remove_device; 953 954 iommu_group_put(group); 955 956 return 0; 957 958 err_remove_device: 959 iommu_group_remove_device(dev); 960 err_put_group: 961 iommu_group_put(group); 962 return ret; 963 } 964 965 static void rk_iommu_remove_device(struct device *dev) 966 { 967 if (!rk_iommu_is_dev_iommu_master(dev)) 968 return; 969 970 iommu_group_remove_device(dev); 971 } 972 973 static const struct iommu_ops rk_iommu_ops = { 974 .domain_alloc = rk_iommu_domain_alloc, 975 .domain_free = rk_iommu_domain_free, 976 .attach_dev = rk_iommu_attach_device, 977 .detach_dev = rk_iommu_detach_device, 978 .map = rk_iommu_map, 979 .unmap = rk_iommu_unmap, 980 .add_device = rk_iommu_add_device, 981 .remove_device = rk_iommu_remove_device, 982 .iova_to_phys = rk_iommu_iova_to_phys, 983 .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, 984 }; 985 986 static int rk_iommu_probe(struct platform_device *pdev) 987 { 988 struct device *dev = &pdev->dev; 989 struct rk_iommu *iommu; 990 struct resource *res; 991 992 iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); 993 if (!iommu) 994 return -ENOMEM; 995 996 platform_set_drvdata(pdev, iommu); 997 iommu->dev = dev; 998 999 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1000 iommu->base = devm_ioremap_resource(&pdev->dev, res); 1001 if (IS_ERR(iommu->base)) 1002 return PTR_ERR(iommu->base); 1003 1004 iommu->irq = platform_get_irq(pdev, 0); 1005 if (iommu->irq < 0) { 1006 dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq); 1007 return -ENXIO; 1008 } 1009 1010 return 0; 1011 } 1012 1013 static int rk_iommu_remove(struct platform_device *pdev) 1014 { 1015 return 0; 1016 } 1017 1018 static const struct of_device_id rk_iommu_dt_ids[] = { 1019 { .compatible = "rockchip,iommu" }, 1020 { /* sentinel */ } 1021 }; 1022 MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); 1023 1024 static struct platform_driver rk_iommu_driver = { 1025 .probe = rk_iommu_probe, 1026 .remove = rk_iommu_remove, 1027 .driver = { 1028 .name = "rk_iommu", 1029 .of_match_table = rk_iommu_dt_ids, 1030 }, 1031 }; 1032 1033 static int __init rk_iommu_init(void) 1034 { 1035 struct device_node *np; 1036 int ret; 1037 1038 np = of_find_matching_node(NULL, rk_iommu_dt_ids); 1039 if (!np) 1040 return 0; 1041 1042 of_node_put(np); 1043 1044 ret = bus_set_iommu(&platform_bus_type, &rk_iommu_ops); 1045 if (ret) 1046 return ret; 1047 1048 return platform_driver_register(&rk_iommu_driver); 1049 } 1050 static void __exit rk_iommu_exit(void) 1051 { 1052 platform_driver_unregister(&rk_iommu_driver); 1053 } 1054 1055 subsys_initcall(rk_iommu_init); 1056 module_exit(rk_iommu_exit); 1057 1058 MODULE_DESCRIPTION("IOMMU API for Rockchip"); 1059 MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com> and Daniel Kurtz <djkurtz@chromium.org>"); 1060 MODULE_ALIAS("platform:rockchip-iommu"); 1061 MODULE_LICENSE("GPL v2"); 1062