1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (C) 2016-2018, Allwinner Technology CO., LTD. 3 // Copyright (C) 2019-2020, Cerno 4 5 #include <linux/bitfield.h> 6 #include <linux/bug.h> 7 #include <linux/clk.h> 8 #include <linux/device.h> 9 #include <linux/dma-direction.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/err.h> 12 #include <linux/errno.h> 13 #include <linux/interrupt.h> 14 #include <linux/iommu.h> 15 #include <linux/iopoll.h> 16 #include <linux/ioport.h> 17 #include <linux/log2.h> 18 #include <linux/module.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/reset.h> 24 #include <linux/sizes.h> 25 #include <linux/slab.h> 26 #include <linux/spinlock.h> 27 #include <linux/types.h> 28 29 #define IOMMU_RESET_REG 0x010 30 #define IOMMU_ENABLE_REG 0x020 31 #define IOMMU_ENABLE_ENABLE BIT(0) 32 33 #define IOMMU_BYPASS_REG 0x030 34 #define IOMMU_AUTO_GATING_REG 0x040 35 #define IOMMU_AUTO_GATING_ENABLE BIT(0) 36 37 #define IOMMU_WBUF_CTRL_REG 0x044 38 #define IOMMU_OOO_CTRL_REG 0x048 39 #define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c 40 #define IOMMU_TTB_REG 0x050 41 #define IOMMU_TLB_ENABLE_REG 0x060 42 #define IOMMU_TLB_PREFETCH_REG 0x070 43 #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m) 44 45 #define IOMMU_TLB_FLUSH_REG 0x080 46 #define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17) 47 #define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16) 48 #define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0)) 49 50 #define IOMMU_TLB_IVLD_ADDR_REG 0x090 51 #define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094 52 #define IOMMU_TLB_IVLD_ENABLE_REG 0x098 53 #define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0) 54 55 #define IOMMU_PC_IVLD_ADDR_REG 0x0a0 56 #define IOMMU_PC_IVLD_ENABLE_REG 0x0a8 57 #define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0) 58 59 #define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4) 60 #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2))) 61 #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1)) 62 63 #define IOMMU_DM_AUT_OVWT_REG 0x0d0 64 #define IOMMU_INT_ENABLE_REG 0x100 65 #define IOMMU_INT_CLR_REG 0x104 66 #define IOMMU_INT_STA_REG 0x108 67 #define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4) 68 #define IOMMU_INT_ERR_ADDR_L1_REG 0x130 69 #define IOMMU_INT_ERR_ADDR_L2_REG 0x134 70 #define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4) 71 #define IOMMU_L1PG_INT_REG 0x0180 72 #define IOMMU_L2PG_INT_REG 0x0184 73 74 #define IOMMU_INT_INVALID_L2PG BIT(17) 75 #define IOMMU_INT_INVALID_L1PG BIT(16) 76 #define IOMMU_INT_MASTER_PERMISSION(m) BIT(m) 77 #define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \ 78 IOMMU_INT_MASTER_PERMISSION(1) | \ 79 IOMMU_INT_MASTER_PERMISSION(2) | \ 80 IOMMU_INT_MASTER_PERMISSION(3) | \ 81 IOMMU_INT_MASTER_PERMISSION(4) | \ 82 IOMMU_INT_MASTER_PERMISSION(5)) 83 #define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \ 84 IOMMU_INT_INVALID_L2PG | \ 85 IOMMU_INT_MASTER_MASK) 86 87 #define PT_ENTRY_SIZE sizeof(u32) 88 89 #define NUM_DT_ENTRIES 4096 90 #define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE) 91 92 #define NUM_PT_ENTRIES 256 93 #define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE) 94 95 struct sun50i_iommu { 96 struct iommu_device iommu; 97 98 /* Lock to modify the IOMMU registers */ 99 spinlock_t iommu_lock; 100 101 struct device *dev; 102 void __iomem *base; 103 struct reset_control *reset; 104 struct clk *clk; 105 106 struct iommu_domain *domain; 107 struct iommu_group *group; 108 struct kmem_cache *pt_pool; 109 }; 110 111 struct sun50i_iommu_domain { 112 struct iommu_domain domain; 113 114 /* Number of devices attached to the domain */ 115 refcount_t refcnt; 116 117 /* L1 Page Table */ 118 u32 *dt; 119 dma_addr_t dt_dma; 120 121 struct sun50i_iommu *iommu; 122 }; 123 124 static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain) 125 { 126 return container_of(domain, struct sun50i_iommu_domain, domain); 127 } 128 129 static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev) 130 { 131 return dev_iommu_priv_get(dev); 132 } 133 134 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) 135 { 136 return readl(iommu->base + offset); 137 } 138 139 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) 140 { 141 writel(value, iommu->base + offset); 142 } 143 144 /* 145 * The Allwinner H6 IOMMU uses a 2-level page table. 146 * 147 * The first level is the usual Directory Table (DT), that consists of 148 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page 149 * Table (PT). 150 * 151 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each 152 * pointing to a 4kB page of physical memory. 153 * 154 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG 155 * register that contains its physical address. 156 */ 157 158 #define SUN50I_IOVA_DTE_MASK GENMASK(31, 20) 159 #define SUN50I_IOVA_PTE_MASK GENMASK(19, 12) 160 #define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0) 161 162 static u32 sun50i_iova_get_dte_index(dma_addr_t iova) 163 { 164 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova); 165 } 166 167 static u32 sun50i_iova_get_pte_index(dma_addr_t iova) 168 { 169 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova); 170 } 171 172 static u32 sun50i_iova_get_page_offset(dma_addr_t iova) 173 { 174 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova); 175 } 176 177 /* 178 * Each Directory Table Entry has a Page Table address and a valid 179 * bit: 180 181 * +---------------------+-----------+-+ 182 * | PT address | Reserved |V| 183 * +---------------------+-----------+-+ 184 * 31:10 - Page Table address 185 * 9:2 - Reserved 186 * 1:0 - 1 if the entry is valid 187 */ 188 189 #define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10) 190 #define SUN50I_DTE_PT_ATTRS GENMASK(1, 0) 191 #define SUN50I_DTE_PT_VALID 1 192 193 static phys_addr_t sun50i_dte_get_pt_address(u32 dte) 194 { 195 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK; 196 } 197 198 static bool sun50i_dte_is_pt_valid(u32 dte) 199 { 200 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID; 201 } 202 203 static u32 sun50i_mk_dte(dma_addr_t pt_dma) 204 { 205 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID; 206 } 207 208 /* 209 * Each PTE has a Page address, an authority index and a valid bit: 210 * 211 * +----------------+-----+-----+-----+---+-----+ 212 * | Page address | Rsv | ACI | Rsv | V | Rsv | 213 * +----------------+-----+-----+-----+---+-----+ 214 * 31:12 - Page address 215 * 11:8 - Reserved 216 * 7:4 - Authority Control Index 217 * 3:2 - Reserved 218 * 1 - 1 if the entry is valid 219 * 0 - Reserved 220 * 221 * The way permissions work is that the IOMMU has 16 "domains" that 222 * can be configured to give each masters either read or write 223 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain 224 * 0 seems like the default domain, and its permissions in the 225 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really 226 * useful to enforce any particular permission. 227 * 228 * Each page entry will then have a reference to the domain they are 229 * affected to, so that we can actually enforce them on a per-page 230 * basis. 231 * 232 * In order to make it work with the IOMMU framework, we will be using 233 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE 234 * depending on the permission we want to enforce. Each domain will 235 * have each master setup in the same way, since the IOMMU framework 236 * doesn't seem to restrict page access on a per-device basis. And 237 * then we will use the relevant domain index when generating the page 238 * table entry depending on the permissions we want to be enforced. 239 */ 240 241 enum sun50i_iommu_aci { 242 SUN50I_IOMMU_ACI_DO_NOT_USE = 0, 243 SUN50I_IOMMU_ACI_NONE, 244 SUN50I_IOMMU_ACI_RD, 245 SUN50I_IOMMU_ACI_WR, 246 SUN50I_IOMMU_ACI_RD_WR, 247 }; 248 249 #define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12) 250 #define SUN50I_PTE_ACI_MASK GENMASK(7, 4) 251 #define SUN50I_PTE_PAGE_VALID BIT(1) 252 253 static phys_addr_t sun50i_pte_get_page_address(u32 pte) 254 { 255 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK; 256 } 257 258 static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte) 259 { 260 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte); 261 } 262 263 static bool sun50i_pte_is_page_valid(u32 pte) 264 { 265 return pte & SUN50I_PTE_PAGE_VALID; 266 } 267 268 static u32 sun50i_mk_pte(phys_addr_t page, int prot) 269 { 270 enum sun50i_iommu_aci aci; 271 u32 flags = 0; 272 273 if (prot & (IOMMU_READ | IOMMU_WRITE)) 274 aci = SUN50I_IOMMU_ACI_RD_WR; 275 else if (prot & IOMMU_READ) 276 aci = SUN50I_IOMMU_ACI_RD; 277 else if (prot & IOMMU_WRITE) 278 aci = SUN50I_IOMMU_ACI_WR; 279 else 280 aci = SUN50I_IOMMU_ACI_NONE; 281 282 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci); 283 page &= SUN50I_PTE_PAGE_ADDRESS_MASK; 284 return page | flags | SUN50I_PTE_PAGE_VALID; 285 } 286 287 static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain, 288 void *vaddr, unsigned int count) 289 { 290 struct sun50i_iommu *iommu = sun50i_domain->iommu; 291 dma_addr_t dma = virt_to_phys(vaddr); 292 size_t size = count * PT_ENTRY_SIZE; 293 294 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE); 295 } 296 297 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) 298 { 299 u32 reg; 300 int ret; 301 302 assert_spin_locked(&iommu->iommu_lock); 303 304 iommu_write(iommu, 305 IOMMU_TLB_FLUSH_REG, 306 IOMMU_TLB_FLUSH_PTW_CACHE | 307 IOMMU_TLB_FLUSH_MACRO_TLB | 308 IOMMU_TLB_FLUSH_MICRO_TLB(5) | 309 IOMMU_TLB_FLUSH_MICRO_TLB(4) | 310 IOMMU_TLB_FLUSH_MICRO_TLB(3) | 311 IOMMU_TLB_FLUSH_MICRO_TLB(2) | 312 IOMMU_TLB_FLUSH_MICRO_TLB(1) | 313 IOMMU_TLB_FLUSH_MICRO_TLB(0)); 314 315 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG, 316 reg, !reg, 317 1, 2000); 318 if (ret) 319 dev_warn(iommu->dev, "TLB Flush timed out!\n"); 320 321 return ret; 322 } 323 324 static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain) 325 { 326 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 327 struct sun50i_iommu *iommu = sun50i_domain->iommu; 328 unsigned long flags; 329 330 /* 331 * At boot, we'll have a first call into .flush_iotlb_all right after 332 * .probe_device, and since we link our (single) domain to our iommu in 333 * the .attach_device callback, we don't have that pointer set. 334 * 335 * It shouldn't really be any trouble to ignore it though since we flush 336 * all caches as part of the device powerup. 337 */ 338 if (!iommu) 339 return; 340 341 spin_lock_irqsave(&iommu->iommu_lock, flags); 342 sun50i_iommu_flush_all_tlb(iommu); 343 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 344 } 345 346 static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain, 347 struct iommu_iotlb_gather *gather) 348 { 349 sun50i_iommu_flush_iotlb_all(domain); 350 } 351 352 static int sun50i_iommu_enable(struct sun50i_iommu *iommu) 353 { 354 struct sun50i_iommu_domain *sun50i_domain; 355 unsigned long flags; 356 int ret; 357 358 if (!iommu->domain) 359 return 0; 360 361 sun50i_domain = to_sun50i_domain(iommu->domain); 362 363 ret = reset_control_deassert(iommu->reset); 364 if (ret) 365 return ret; 366 367 ret = clk_prepare_enable(iommu->clk); 368 if (ret) 369 goto err_reset_assert; 370 371 spin_lock_irqsave(&iommu->iommu_lock, flags); 372 373 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma); 374 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, 375 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) | 376 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) | 377 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) | 378 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) | 379 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) | 380 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5)); 381 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); 382 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), 383 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 384 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 385 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 386 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 387 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 388 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 389 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 390 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 391 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 392 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 393 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) | 394 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5)); 395 396 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD), 397 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) | 398 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) | 399 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) | 400 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) | 401 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) | 402 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5)); 403 404 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR), 405 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) | 406 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) | 407 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) | 408 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) | 409 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) | 410 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5)); 411 412 ret = sun50i_iommu_flush_all_tlb(iommu); 413 if (ret) { 414 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 415 goto err_clk_disable; 416 } 417 418 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); 419 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE); 420 421 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 422 423 return 0; 424 425 err_clk_disable: 426 clk_disable_unprepare(iommu->clk); 427 428 err_reset_assert: 429 reset_control_assert(iommu->reset); 430 431 return ret; 432 } 433 434 static void sun50i_iommu_disable(struct sun50i_iommu *iommu) 435 { 436 unsigned long flags; 437 438 spin_lock_irqsave(&iommu->iommu_lock, flags); 439 440 iommu_write(iommu, IOMMU_ENABLE_REG, 0); 441 iommu_write(iommu, IOMMU_TTB_REG, 0); 442 443 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 444 445 clk_disable_unprepare(iommu->clk); 446 reset_control_assert(iommu->reset); 447 } 448 449 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, 450 gfp_t gfp) 451 { 452 dma_addr_t pt_dma; 453 u32 *page_table; 454 455 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp); 456 if (!page_table) 457 return ERR_PTR(-ENOMEM); 458 459 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE); 460 if (dma_mapping_error(iommu->dev, pt_dma)) { 461 dev_err(iommu->dev, "Couldn't map L2 Page Table\n"); 462 kmem_cache_free(iommu->pt_pool, page_table); 463 return ERR_PTR(-ENOMEM); 464 } 465 466 /* We rely on the physical address and DMA address being the same */ 467 WARN_ON(pt_dma != virt_to_phys(page_table)); 468 469 return page_table; 470 } 471 472 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, 473 u32 *page_table) 474 { 475 phys_addr_t pt_phys = virt_to_phys(page_table); 476 477 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE); 478 kmem_cache_free(iommu->pt_pool, page_table); 479 } 480 481 static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain, 482 dma_addr_t iova, gfp_t gfp) 483 { 484 struct sun50i_iommu *iommu = sun50i_domain->iommu; 485 u32 *page_table; 486 u32 *dte_addr; 487 u32 old_dte; 488 u32 dte; 489 490 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 491 dte = *dte_addr; 492 if (sun50i_dte_is_pt_valid(dte)) { 493 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte); 494 return (u32 *)phys_to_virt(pt_phys); 495 } 496 497 page_table = sun50i_iommu_alloc_page_table(iommu, gfp); 498 if (IS_ERR(page_table)) 499 return page_table; 500 501 dte = sun50i_mk_dte(virt_to_phys(page_table)); 502 old_dte = cmpxchg(dte_addr, 0, dte); 503 if (old_dte) { 504 phys_addr_t installed_pt_phys = 505 sun50i_dte_get_pt_address(old_dte); 506 u32 *installed_pt = phys_to_virt(installed_pt_phys); 507 u32 *drop_pt = page_table; 508 509 page_table = installed_pt; 510 dte = old_dte; 511 sun50i_iommu_free_page_table(iommu, drop_pt); 512 } 513 514 sun50i_table_flush(sun50i_domain, page_table, PT_SIZE); 515 sun50i_table_flush(sun50i_domain, dte_addr, 1); 516 517 return page_table; 518 } 519 520 static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova, 521 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 522 { 523 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 524 struct sun50i_iommu *iommu = sun50i_domain->iommu; 525 u32 pte_index; 526 u32 *page_table, *pte_addr; 527 int ret = 0; 528 529 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp); 530 if (IS_ERR(page_table)) { 531 ret = PTR_ERR(page_table); 532 goto out; 533 } 534 535 pte_index = sun50i_iova_get_pte_index(iova); 536 pte_addr = &page_table[pte_index]; 537 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) { 538 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr); 539 dev_err(iommu->dev, 540 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n", 541 &iova, &page_phys, &paddr, prot); 542 ret = -EBUSY; 543 goto out; 544 } 545 546 *pte_addr = sun50i_mk_pte(paddr, prot); 547 sun50i_table_flush(sun50i_domain, pte_addr, 1); 548 549 out: 550 return ret; 551 } 552 553 static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova, 554 size_t size, struct iommu_iotlb_gather *gather) 555 { 556 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 557 phys_addr_t pt_phys; 558 u32 *pte_addr; 559 u32 dte; 560 561 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 562 if (!sun50i_dte_is_pt_valid(dte)) 563 return 0; 564 565 pt_phys = sun50i_dte_get_pt_address(dte); 566 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova); 567 568 if (!sun50i_pte_is_page_valid(*pte_addr)) 569 return 0; 570 571 memset(pte_addr, 0, sizeof(*pte_addr)); 572 sun50i_table_flush(sun50i_domain, pte_addr, 1); 573 574 return SZ_4K; 575 } 576 577 static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain, 578 dma_addr_t iova) 579 { 580 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 581 phys_addr_t pt_phys; 582 u32 *page_table; 583 u32 dte, pte; 584 585 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 586 if (!sun50i_dte_is_pt_valid(dte)) 587 return 0; 588 589 pt_phys = sun50i_dte_get_pt_address(dte); 590 page_table = (u32 *)phys_to_virt(pt_phys); 591 pte = page_table[sun50i_iova_get_pte_index(iova)]; 592 if (!sun50i_pte_is_page_valid(pte)) 593 return 0; 594 595 return sun50i_pte_get_page_address(pte) + 596 sun50i_iova_get_page_offset(iova); 597 } 598 599 static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type) 600 { 601 struct sun50i_iommu_domain *sun50i_domain; 602 603 if (type != IOMMU_DOMAIN_DMA && 604 type != IOMMU_DOMAIN_IDENTITY && 605 type != IOMMU_DOMAIN_UNMANAGED) 606 return NULL; 607 608 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL); 609 if (!sun50i_domain) 610 return NULL; 611 612 sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 613 get_order(DT_SIZE)); 614 if (!sun50i_domain->dt) 615 goto err_free_domain; 616 617 refcount_set(&sun50i_domain->refcnt, 1); 618 619 sun50i_domain->domain.geometry.aperture_start = 0; 620 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); 621 sun50i_domain->domain.geometry.force_aperture = true; 622 623 return &sun50i_domain->domain; 624 625 err_free_domain: 626 kfree(sun50i_domain); 627 628 return NULL; 629 } 630 631 static void sun50i_iommu_domain_free(struct iommu_domain *domain) 632 { 633 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 634 635 free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE)); 636 sun50i_domain->dt = NULL; 637 638 kfree(sun50i_domain); 639 } 640 641 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, 642 struct sun50i_iommu_domain *sun50i_domain) 643 { 644 iommu->domain = &sun50i_domain->domain; 645 sun50i_domain->iommu = iommu; 646 647 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt, 648 DT_SIZE, DMA_TO_DEVICE); 649 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) { 650 dev_err(iommu->dev, "Couldn't map L1 Page Table\n"); 651 return -ENOMEM; 652 } 653 654 return sun50i_iommu_enable(iommu); 655 } 656 657 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, 658 struct sun50i_iommu_domain *sun50i_domain) 659 { 660 unsigned int i; 661 662 for (i = 0; i < NUM_DT_ENTRIES; i++) { 663 phys_addr_t pt_phys; 664 u32 *page_table; 665 u32 *dte_addr; 666 u32 dte; 667 668 dte_addr = &sun50i_domain->dt[i]; 669 dte = *dte_addr; 670 if (!sun50i_dte_is_pt_valid(dte)) 671 continue; 672 673 memset(dte_addr, 0, sizeof(*dte_addr)); 674 sun50i_table_flush(sun50i_domain, dte_addr, 1); 675 676 pt_phys = sun50i_dte_get_pt_address(dte); 677 page_table = phys_to_virt(pt_phys); 678 sun50i_iommu_free_page_table(iommu, page_table); 679 } 680 681 682 sun50i_iommu_disable(iommu); 683 684 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt), 685 DT_SIZE, DMA_TO_DEVICE); 686 687 iommu->domain = NULL; 688 } 689 690 static void sun50i_iommu_detach_device(struct iommu_domain *domain, 691 struct device *dev) 692 { 693 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 694 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev); 695 696 dev_dbg(dev, "Detaching from IOMMU domain\n"); 697 698 if (iommu->domain != domain) 699 return; 700 701 if (refcount_dec_and_test(&sun50i_domain->refcnt)) 702 sun50i_iommu_detach_domain(iommu, sun50i_domain); 703 } 704 705 static int sun50i_iommu_attach_device(struct iommu_domain *domain, 706 struct device *dev) 707 { 708 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 709 struct sun50i_iommu *iommu; 710 711 iommu = sun50i_iommu_from_dev(dev); 712 if (!iommu) 713 return -ENODEV; 714 715 dev_dbg(dev, "Attaching to IOMMU domain\n"); 716 717 refcount_inc(&sun50i_domain->refcnt); 718 719 if (iommu->domain == domain) 720 return 0; 721 722 if (iommu->domain) 723 sun50i_iommu_detach_device(iommu->domain, dev); 724 725 sun50i_iommu_attach_domain(iommu, sun50i_domain); 726 727 return 0; 728 } 729 730 static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) 731 { 732 struct sun50i_iommu *iommu; 733 734 iommu = sun50i_iommu_from_dev(dev); 735 if (!iommu) 736 return ERR_PTR(-ENODEV); 737 738 return &iommu->iommu; 739 } 740 741 static void sun50i_iommu_release_device(struct device *dev) {} 742 743 static struct iommu_group *sun50i_iommu_device_group(struct device *dev) 744 { 745 struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev); 746 747 return iommu_group_ref_get(iommu->group); 748 } 749 750 static int sun50i_iommu_of_xlate(struct device *dev, 751 struct of_phandle_args *args) 752 { 753 struct platform_device *iommu_pdev = of_find_device_by_node(args->np); 754 unsigned id = args->args[0]; 755 756 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev)); 757 758 return iommu_fwspec_add_ids(dev, &id, 1); 759 } 760 761 static const struct iommu_ops sun50i_iommu_ops = { 762 .pgsize_bitmap = SZ_4K, 763 .attach_dev = sun50i_iommu_attach_device, 764 .detach_dev = sun50i_iommu_detach_device, 765 .device_group = sun50i_iommu_device_group, 766 .domain_alloc = sun50i_iommu_domain_alloc, 767 .domain_free = sun50i_iommu_domain_free, 768 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all, 769 .iotlb_sync = sun50i_iommu_iotlb_sync, 770 .iova_to_phys = sun50i_iommu_iova_to_phys, 771 .map = sun50i_iommu_map, 772 .of_xlate = sun50i_iommu_of_xlate, 773 .probe_device = sun50i_iommu_probe_device, 774 .release_device = sun50i_iommu_release_device, 775 .unmap = sun50i_iommu_unmap, 776 }; 777 778 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, 779 unsigned master, phys_addr_t iova, 780 unsigned prot) 781 { 782 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n", 783 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd"); 784 785 if (iommu->domain) 786 report_iommu_fault(iommu->domain, iommu->dev, iova, prot); 787 else 788 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n"); 789 } 790 791 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, 792 unsigned addr_reg, 793 unsigned blame_reg) 794 { 795 phys_addr_t iova; 796 unsigned master; 797 u32 blame; 798 799 assert_spin_locked(&iommu->iommu_lock); 800 801 iova = iommu_read(iommu, addr_reg); 802 blame = iommu_read(iommu, blame_reg); 803 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 804 805 /* 806 * If the address is not in the page table, we can't get what 807 * operation triggered the fault. Assume it's a read 808 * operation. 809 */ 810 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ); 811 812 return iova; 813 } 814 815 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) 816 { 817 enum sun50i_iommu_aci aci; 818 phys_addr_t iova; 819 unsigned master; 820 unsigned dir; 821 u32 blame; 822 823 assert_spin_locked(&iommu->iommu_lock); 824 825 blame = iommu_read(iommu, IOMMU_INT_STA_REG); 826 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 827 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master)); 828 aci = sun50i_get_pte_aci(iommu_read(iommu, 829 IOMMU_INT_ERR_DATA_REG(master))); 830 831 switch (aci) { 832 /* 833 * If we are in the read-only domain, then it means we 834 * tried to write. 835 */ 836 case SUN50I_IOMMU_ACI_RD: 837 dir = IOMMU_FAULT_WRITE; 838 break; 839 840 /* 841 * If we are in the write-only domain, then it means 842 * we tried to read. 843 */ 844 case SUN50I_IOMMU_ACI_WR: 845 846 /* 847 * If we are in the domain without any permission, we 848 * can't really tell. Let's default to a read 849 * operation. 850 */ 851 case SUN50I_IOMMU_ACI_NONE: 852 853 /* WTF? */ 854 case SUN50I_IOMMU_ACI_RD_WR: 855 default: 856 dir = IOMMU_FAULT_READ; 857 break; 858 } 859 860 /* 861 * If the address is not in the page table, we can't get what 862 * operation triggered the fault. Assume it's a read 863 * operation. 864 */ 865 sun50i_iommu_report_fault(iommu, master, iova, dir); 866 867 return iova; 868 } 869 870 static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id) 871 { 872 struct sun50i_iommu *iommu = dev_id; 873 u32 status; 874 875 spin_lock(&iommu->iommu_lock); 876 877 status = iommu_read(iommu, IOMMU_INT_STA_REG); 878 if (!(status & IOMMU_INT_MASK)) { 879 spin_unlock(&iommu->iommu_lock); 880 return IRQ_NONE; 881 } 882 883 if (status & IOMMU_INT_INVALID_L2PG) 884 sun50i_iommu_handle_pt_irq(iommu, 885 IOMMU_INT_ERR_ADDR_L2_REG, 886 IOMMU_L2PG_INT_REG); 887 else if (status & IOMMU_INT_INVALID_L1PG) 888 sun50i_iommu_handle_pt_irq(iommu, 889 IOMMU_INT_ERR_ADDR_L1_REG, 890 IOMMU_L1PG_INT_REG); 891 else 892 sun50i_iommu_handle_perm_irq(iommu); 893 894 iommu_write(iommu, IOMMU_INT_CLR_REG, status); 895 896 iommu_write(iommu, IOMMU_RESET_REG, ~status); 897 iommu_write(iommu, IOMMU_RESET_REG, status); 898 899 spin_unlock(&iommu->iommu_lock); 900 901 return IRQ_HANDLED; 902 } 903 904 static int sun50i_iommu_probe(struct platform_device *pdev) 905 { 906 struct sun50i_iommu *iommu; 907 int ret, irq; 908 909 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); 910 if (!iommu) 911 return -ENOMEM; 912 spin_lock_init(&iommu->iommu_lock); 913 platform_set_drvdata(pdev, iommu); 914 iommu->dev = &pdev->dev; 915 916 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), 917 PT_SIZE, PT_SIZE, 918 SLAB_HWCACHE_ALIGN, 919 NULL); 920 if (!iommu->pt_pool) 921 return -ENOMEM; 922 923 iommu->group = iommu_group_alloc(); 924 if (IS_ERR(iommu->group)) { 925 ret = PTR_ERR(iommu->group); 926 goto err_free_cache; 927 } 928 929 iommu->base = devm_platform_ioremap_resource(pdev, 0); 930 if (IS_ERR(iommu->base)) { 931 ret = PTR_ERR(iommu->base); 932 goto err_free_group; 933 } 934 935 irq = platform_get_irq(pdev, 0); 936 if (irq < 0) { 937 ret = irq; 938 goto err_free_group; 939 } 940 941 iommu->clk = devm_clk_get(&pdev->dev, NULL); 942 if (IS_ERR(iommu->clk)) { 943 dev_err(&pdev->dev, "Couldn't get our clock.\n"); 944 ret = PTR_ERR(iommu->clk); 945 goto err_free_group; 946 } 947 948 iommu->reset = devm_reset_control_get(&pdev->dev, NULL); 949 if (IS_ERR(iommu->reset)) { 950 dev_err(&pdev->dev, "Couldn't get our reset line.\n"); 951 ret = PTR_ERR(iommu->reset); 952 goto err_free_group; 953 } 954 955 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev, 956 NULL, dev_name(&pdev->dev)); 957 if (ret) 958 goto err_free_group; 959 960 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev); 961 if (ret) 962 goto err_remove_sysfs; 963 964 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0, 965 dev_name(&pdev->dev), iommu); 966 if (ret < 0) 967 goto err_unregister; 968 969 bus_set_iommu(&platform_bus_type, &sun50i_iommu_ops); 970 971 return 0; 972 973 err_unregister: 974 iommu_device_unregister(&iommu->iommu); 975 976 err_remove_sysfs: 977 iommu_device_sysfs_remove(&iommu->iommu); 978 979 err_free_group: 980 iommu_group_put(iommu->group); 981 982 err_free_cache: 983 kmem_cache_destroy(iommu->pt_pool); 984 985 return ret; 986 } 987 988 static const struct of_device_id sun50i_iommu_dt[] = { 989 { .compatible = "allwinner,sun50i-h6-iommu", }, 990 { /* sentinel */ }, 991 }; 992 MODULE_DEVICE_TABLE(of, sun50i_iommu_dt); 993 994 static struct platform_driver sun50i_iommu_driver = { 995 .driver = { 996 .name = "sun50i-iommu", 997 .of_match_table = sun50i_iommu_dt, 998 .suppress_bind_attrs = true, 999 } 1000 }; 1001 builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe); 1002 1003 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver"); 1004 MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>"); 1005 MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>"); 1006 MODULE_LICENSE("Dual BSD/GPL"); 1007