1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (C) 2016-2018, Allwinner Technology CO., LTD. 3 // Copyright (C) 2019-2020, Cerno 4 5 #include <linux/bitfield.h> 6 #include <linux/bug.h> 7 #include <linux/clk.h> 8 #include <linux/device.h> 9 #include <linux/dma-direction.h> 10 #include <linux/dma-iommu.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/err.h> 13 #include <linux/errno.h> 14 #include <linux/interrupt.h> 15 #include <linux/iommu.h> 16 #include <linux/iopoll.h> 17 #include <linux/ioport.h> 18 #include <linux/log2.h> 19 #include <linux/module.h> 20 #include <linux/of_platform.h> 21 #include <linux/platform_device.h> 22 #include <linux/pm.h> 23 #include <linux/pm_runtime.h> 24 #include <linux/reset.h> 25 #include <linux/sizes.h> 26 #include <linux/slab.h> 27 #include <linux/spinlock.h> 28 #include <linux/types.h> 29 30 #define IOMMU_RESET_REG 0x010 31 #define IOMMU_ENABLE_REG 0x020 32 #define IOMMU_ENABLE_ENABLE BIT(0) 33 34 #define IOMMU_BYPASS_REG 0x030 35 #define IOMMU_AUTO_GATING_REG 0x040 36 #define IOMMU_AUTO_GATING_ENABLE BIT(0) 37 38 #define IOMMU_WBUF_CTRL_REG 0x044 39 #define IOMMU_OOO_CTRL_REG 0x048 40 #define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c 41 #define IOMMU_TTB_REG 0x050 42 #define IOMMU_TLB_ENABLE_REG 0x060 43 #define IOMMU_TLB_PREFETCH_REG 0x070 44 #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m) 45 46 #define IOMMU_TLB_FLUSH_REG 0x080 47 #define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17) 48 #define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16) 49 #define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0)) 50 51 #define IOMMU_TLB_IVLD_ADDR_REG 0x090 52 #define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094 53 #define IOMMU_TLB_IVLD_ENABLE_REG 0x098 54 #define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0) 55 56 #define IOMMU_PC_IVLD_ADDR_REG 0x0a0 57 #define IOMMU_PC_IVLD_ENABLE_REG 0x0a8 58 #define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0) 59 60 #define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4) 61 #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2))) 62 #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1)) 63 64 #define IOMMU_DM_AUT_OVWT_REG 0x0d0 65 #define IOMMU_INT_ENABLE_REG 0x100 66 #define IOMMU_INT_CLR_REG 0x104 67 #define IOMMU_INT_STA_REG 0x108 68 #define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4) 69 #define IOMMU_INT_ERR_ADDR_L1_REG 0x130 70 #define IOMMU_INT_ERR_ADDR_L2_REG 0x134 71 #define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4) 72 #define IOMMU_L1PG_INT_REG 0x0180 73 #define IOMMU_L2PG_INT_REG 0x0184 74 75 #define IOMMU_INT_INVALID_L2PG BIT(17) 76 #define IOMMU_INT_INVALID_L1PG BIT(16) 77 #define IOMMU_INT_MASTER_PERMISSION(m) BIT(m) 78 #define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \ 79 IOMMU_INT_MASTER_PERMISSION(1) | \ 80 IOMMU_INT_MASTER_PERMISSION(2) | \ 81 IOMMU_INT_MASTER_PERMISSION(3) | \ 82 IOMMU_INT_MASTER_PERMISSION(4) | \ 83 IOMMU_INT_MASTER_PERMISSION(5)) 84 #define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \ 85 IOMMU_INT_INVALID_L2PG | \ 86 IOMMU_INT_MASTER_MASK) 87 88 #define PT_ENTRY_SIZE sizeof(u32) 89 90 #define NUM_DT_ENTRIES 4096 91 #define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE) 92 93 #define NUM_PT_ENTRIES 256 94 #define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE) 95 96 struct sun50i_iommu { 97 struct iommu_device iommu; 98 99 /* Lock to modify the IOMMU registers */ 100 spinlock_t iommu_lock; 101 102 struct device *dev; 103 void __iomem *base; 104 struct reset_control *reset; 105 struct clk *clk; 106 107 struct iommu_domain *domain; 108 struct iommu_group *group; 109 struct kmem_cache *pt_pool; 110 }; 111 112 struct sun50i_iommu_domain { 113 struct iommu_domain domain; 114 115 /* Number of devices attached to the domain */ 116 refcount_t refcnt; 117 118 /* L1 Page Table */ 119 u32 *dt; 120 dma_addr_t dt_dma; 121 122 struct sun50i_iommu *iommu; 123 }; 124 125 static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain) 126 { 127 return container_of(domain, struct sun50i_iommu_domain, domain); 128 } 129 130 static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev) 131 { 132 return dev_iommu_priv_get(dev); 133 } 134 135 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) 136 { 137 return readl(iommu->base + offset); 138 } 139 140 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) 141 { 142 writel(value, iommu->base + offset); 143 } 144 145 /* 146 * The Allwinner H6 IOMMU uses a 2-level page table. 147 * 148 * The first level is the usual Directory Table (DT), that consists of 149 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page 150 * Table (PT). 151 * 152 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each 153 * pointing to a 4kB page of physical memory. 154 * 155 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG 156 * register that contains its physical address. 157 */ 158 159 #define SUN50I_IOVA_DTE_MASK GENMASK(31, 20) 160 #define SUN50I_IOVA_PTE_MASK GENMASK(19, 12) 161 #define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0) 162 163 static u32 sun50i_iova_get_dte_index(dma_addr_t iova) 164 { 165 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova); 166 } 167 168 static u32 sun50i_iova_get_pte_index(dma_addr_t iova) 169 { 170 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova); 171 } 172 173 static u32 sun50i_iova_get_page_offset(dma_addr_t iova) 174 { 175 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova); 176 } 177 178 /* 179 * Each Directory Table Entry has a Page Table address and a valid 180 * bit: 181 182 * +---------------------+-----------+-+ 183 * | PT address | Reserved |V| 184 * +---------------------+-----------+-+ 185 * 31:10 - Page Table address 186 * 9:2 - Reserved 187 * 1:0 - 1 if the entry is valid 188 */ 189 190 #define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10) 191 #define SUN50I_DTE_PT_ATTRS GENMASK(1, 0) 192 #define SUN50I_DTE_PT_VALID 1 193 194 static phys_addr_t sun50i_dte_get_pt_address(u32 dte) 195 { 196 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK; 197 } 198 199 static bool sun50i_dte_is_pt_valid(u32 dte) 200 { 201 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID; 202 } 203 204 static u32 sun50i_mk_dte(dma_addr_t pt_dma) 205 { 206 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID; 207 } 208 209 /* 210 * Each PTE has a Page address, an authority index and a valid bit: 211 * 212 * +----------------+-----+-----+-----+---+-----+ 213 * | Page address | Rsv | ACI | Rsv | V | Rsv | 214 * +----------------+-----+-----+-----+---+-----+ 215 * 31:12 - Page address 216 * 11:8 - Reserved 217 * 7:4 - Authority Control Index 218 * 3:2 - Reserved 219 * 1 - 1 if the entry is valid 220 * 0 - Reserved 221 * 222 * The way permissions work is that the IOMMU has 16 "domains" that 223 * can be configured to give each masters either read or write 224 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain 225 * 0 seems like the default domain, and its permissions in the 226 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really 227 * useful to enforce any particular permission. 228 * 229 * Each page entry will then have a reference to the domain they are 230 * affected to, so that we can actually enforce them on a per-page 231 * basis. 232 * 233 * In order to make it work with the IOMMU framework, we will be using 234 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE 235 * depending on the permission we want to enforce. Each domain will 236 * have each master setup in the same way, since the IOMMU framework 237 * doesn't seem to restrict page access on a per-device basis. And 238 * then we will use the relevant domain index when generating the page 239 * table entry depending on the permissions we want to be enforced. 240 */ 241 242 enum sun50i_iommu_aci { 243 SUN50I_IOMMU_ACI_DO_NOT_USE = 0, 244 SUN50I_IOMMU_ACI_NONE, 245 SUN50I_IOMMU_ACI_RD, 246 SUN50I_IOMMU_ACI_WR, 247 SUN50I_IOMMU_ACI_RD_WR, 248 }; 249 250 #define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12) 251 #define SUN50I_PTE_ACI_MASK GENMASK(7, 4) 252 #define SUN50I_PTE_PAGE_VALID BIT(1) 253 254 static phys_addr_t sun50i_pte_get_page_address(u32 pte) 255 { 256 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK; 257 } 258 259 static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte) 260 { 261 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte); 262 } 263 264 static bool sun50i_pte_is_page_valid(u32 pte) 265 { 266 return pte & SUN50I_PTE_PAGE_VALID; 267 } 268 269 static u32 sun50i_mk_pte(phys_addr_t page, int prot) 270 { 271 enum sun50i_iommu_aci aci; 272 u32 flags = 0; 273 274 if (prot & (IOMMU_READ | IOMMU_WRITE)) 275 aci = SUN50I_IOMMU_ACI_RD_WR; 276 else if (prot & IOMMU_READ) 277 aci = SUN50I_IOMMU_ACI_RD; 278 else if (prot & IOMMU_WRITE) 279 aci = SUN50I_IOMMU_ACI_WR; 280 else 281 aci = SUN50I_IOMMU_ACI_NONE; 282 283 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci); 284 page &= SUN50I_PTE_PAGE_ADDRESS_MASK; 285 return page | flags | SUN50I_PTE_PAGE_VALID; 286 } 287 288 static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain, 289 void *vaddr, unsigned int count) 290 { 291 struct sun50i_iommu *iommu = sun50i_domain->iommu; 292 dma_addr_t dma = virt_to_phys(vaddr); 293 size_t size = count * PT_ENTRY_SIZE; 294 295 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE); 296 } 297 298 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) 299 { 300 u32 reg; 301 int ret; 302 303 assert_spin_locked(&iommu->iommu_lock); 304 305 iommu_write(iommu, 306 IOMMU_TLB_FLUSH_REG, 307 IOMMU_TLB_FLUSH_PTW_CACHE | 308 IOMMU_TLB_FLUSH_MACRO_TLB | 309 IOMMU_TLB_FLUSH_MICRO_TLB(5) | 310 IOMMU_TLB_FLUSH_MICRO_TLB(4) | 311 IOMMU_TLB_FLUSH_MICRO_TLB(3) | 312 IOMMU_TLB_FLUSH_MICRO_TLB(2) | 313 IOMMU_TLB_FLUSH_MICRO_TLB(1) | 314 IOMMU_TLB_FLUSH_MICRO_TLB(0)); 315 316 ret = readl_poll_timeout(iommu->base + IOMMU_TLB_FLUSH_REG, 317 reg, !reg, 318 1, 2000); 319 if (ret) 320 dev_warn(iommu->dev, "TLB Flush timed out!\n"); 321 322 return ret; 323 } 324 325 static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain) 326 { 327 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 328 struct sun50i_iommu *iommu = sun50i_domain->iommu; 329 unsigned long flags; 330 331 /* 332 * At boot, we'll have a first call into .flush_iotlb_all right after 333 * .probe_device, and since we link our (single) domain to our iommu in 334 * the .attach_device callback, we don't have that pointer set. 335 * 336 * It shouldn't really be any trouble to ignore it though since we flush 337 * all caches as part of the device powerup. 338 */ 339 if (!iommu) 340 return; 341 342 spin_lock_irqsave(&iommu->iommu_lock, flags); 343 sun50i_iommu_flush_all_tlb(iommu); 344 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 345 } 346 347 static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain, 348 struct iommu_iotlb_gather *gather) 349 { 350 sun50i_iommu_flush_iotlb_all(domain); 351 } 352 353 static int sun50i_iommu_enable(struct sun50i_iommu *iommu) 354 { 355 struct sun50i_iommu_domain *sun50i_domain; 356 unsigned long flags; 357 int ret; 358 359 if (!iommu->domain) 360 return 0; 361 362 sun50i_domain = to_sun50i_domain(iommu->domain); 363 364 ret = reset_control_deassert(iommu->reset); 365 if (ret) 366 return ret; 367 368 ret = clk_prepare_enable(iommu->clk); 369 if (ret) 370 goto err_reset_assert; 371 372 spin_lock_irqsave(&iommu->iommu_lock, flags); 373 374 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma); 375 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, 376 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) | 377 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) | 378 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) | 379 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) | 380 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) | 381 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5)); 382 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); 383 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), 384 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 385 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 386 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 387 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 388 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 389 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 390 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 391 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 392 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 393 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 394 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) | 395 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5)); 396 397 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD), 398 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) | 399 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) | 400 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) | 401 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) | 402 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) | 403 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5)); 404 405 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR), 406 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) | 407 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) | 408 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) | 409 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) | 410 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) | 411 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5)); 412 413 ret = sun50i_iommu_flush_all_tlb(iommu); 414 if (ret) { 415 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 416 goto err_clk_disable; 417 } 418 419 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); 420 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE); 421 422 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 423 424 return 0; 425 426 err_clk_disable: 427 clk_disable_unprepare(iommu->clk); 428 429 err_reset_assert: 430 reset_control_assert(iommu->reset); 431 432 return ret; 433 } 434 435 static void sun50i_iommu_disable(struct sun50i_iommu *iommu) 436 { 437 unsigned long flags; 438 439 spin_lock_irqsave(&iommu->iommu_lock, flags); 440 441 iommu_write(iommu, IOMMU_ENABLE_REG, 0); 442 iommu_write(iommu, IOMMU_TTB_REG, 0); 443 444 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 445 446 clk_disable_unprepare(iommu->clk); 447 reset_control_assert(iommu->reset); 448 } 449 450 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, 451 gfp_t gfp) 452 { 453 dma_addr_t pt_dma; 454 u32 *page_table; 455 456 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp); 457 if (!page_table) 458 return ERR_PTR(-ENOMEM); 459 460 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE); 461 if (dma_mapping_error(iommu->dev, pt_dma)) { 462 dev_err(iommu->dev, "Couldn't map L2 Page Table\n"); 463 kmem_cache_free(iommu->pt_pool, page_table); 464 return ERR_PTR(-ENOMEM); 465 } 466 467 /* We rely on the physical address and DMA address being the same */ 468 WARN_ON(pt_dma != virt_to_phys(page_table)); 469 470 return page_table; 471 } 472 473 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, 474 u32 *page_table) 475 { 476 phys_addr_t pt_phys = virt_to_phys(page_table); 477 478 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE); 479 kmem_cache_free(iommu->pt_pool, page_table); 480 } 481 482 static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain, 483 dma_addr_t iova, gfp_t gfp) 484 { 485 struct sun50i_iommu *iommu = sun50i_domain->iommu; 486 u32 *page_table; 487 u32 *dte_addr; 488 u32 old_dte; 489 u32 dte; 490 491 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 492 dte = *dte_addr; 493 if (sun50i_dte_is_pt_valid(dte)) { 494 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte); 495 return (u32 *)phys_to_virt(pt_phys); 496 } 497 498 page_table = sun50i_iommu_alloc_page_table(iommu, gfp); 499 if (IS_ERR(page_table)) 500 return page_table; 501 502 dte = sun50i_mk_dte(virt_to_phys(page_table)); 503 old_dte = cmpxchg(dte_addr, 0, dte); 504 if (old_dte) { 505 phys_addr_t installed_pt_phys = 506 sun50i_dte_get_pt_address(old_dte); 507 u32 *installed_pt = phys_to_virt(installed_pt_phys); 508 u32 *drop_pt = page_table; 509 510 page_table = installed_pt; 511 dte = old_dte; 512 sun50i_iommu_free_page_table(iommu, drop_pt); 513 } 514 515 sun50i_table_flush(sun50i_domain, page_table, PT_SIZE); 516 sun50i_table_flush(sun50i_domain, dte_addr, 1); 517 518 return page_table; 519 } 520 521 static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova, 522 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 523 { 524 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 525 struct sun50i_iommu *iommu = sun50i_domain->iommu; 526 u32 pte_index; 527 u32 *page_table, *pte_addr; 528 int ret = 0; 529 530 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp); 531 if (IS_ERR(page_table)) { 532 ret = PTR_ERR(page_table); 533 goto out; 534 } 535 536 pte_index = sun50i_iova_get_pte_index(iova); 537 pte_addr = &page_table[pte_index]; 538 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) { 539 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr); 540 dev_err(iommu->dev, 541 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n", 542 &iova, &page_phys, &paddr, prot); 543 ret = -EBUSY; 544 goto out; 545 } 546 547 *pte_addr = sun50i_mk_pte(paddr, prot); 548 sun50i_table_flush(sun50i_domain, pte_addr, 1); 549 550 out: 551 return ret; 552 } 553 554 static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova, 555 size_t size, struct iommu_iotlb_gather *gather) 556 { 557 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 558 phys_addr_t pt_phys; 559 dma_addr_t pte_dma; 560 u32 *pte_addr; 561 u32 dte; 562 563 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 564 if (!sun50i_dte_is_pt_valid(dte)) 565 return 0; 566 567 pt_phys = sun50i_dte_get_pt_address(dte); 568 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova); 569 pte_dma = pt_phys + sun50i_iova_get_pte_index(iova) * PT_ENTRY_SIZE; 570 571 if (!sun50i_pte_is_page_valid(*pte_addr)) 572 return 0; 573 574 memset(pte_addr, 0, sizeof(*pte_addr)); 575 sun50i_table_flush(sun50i_domain, pte_addr, 1); 576 577 return SZ_4K; 578 } 579 580 static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain, 581 dma_addr_t iova) 582 { 583 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 584 phys_addr_t pt_phys; 585 u32 *page_table; 586 u32 dte, pte; 587 588 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 589 if (!sun50i_dte_is_pt_valid(dte)) 590 return 0; 591 592 pt_phys = sun50i_dte_get_pt_address(dte); 593 page_table = (u32 *)phys_to_virt(pt_phys); 594 pte = page_table[sun50i_iova_get_pte_index(iova)]; 595 if (!sun50i_pte_is_page_valid(pte)) 596 return 0; 597 598 return sun50i_pte_get_page_address(pte) + 599 sun50i_iova_get_page_offset(iova); 600 } 601 602 static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type) 603 { 604 struct sun50i_iommu_domain *sun50i_domain; 605 606 if (type != IOMMU_DOMAIN_DMA && 607 type != IOMMU_DOMAIN_IDENTITY && 608 type != IOMMU_DOMAIN_UNMANAGED) 609 return NULL; 610 611 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL); 612 if (!sun50i_domain) 613 return NULL; 614 615 if (type == IOMMU_DOMAIN_DMA && 616 iommu_get_dma_cookie(&sun50i_domain->domain)) 617 goto err_free_domain; 618 619 sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL, 620 get_order(DT_SIZE)); 621 if (!sun50i_domain->dt) 622 goto err_put_cookie; 623 memset(sun50i_domain->dt, 0, DT_SIZE); 624 625 refcount_set(&sun50i_domain->refcnt, 1); 626 627 sun50i_domain->domain.geometry.aperture_start = 0; 628 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); 629 sun50i_domain->domain.geometry.force_aperture = true; 630 631 return &sun50i_domain->domain; 632 633 err_put_cookie: 634 if (type == IOMMU_DOMAIN_DMA) 635 iommu_put_dma_cookie(&sun50i_domain->domain); 636 637 err_free_domain: 638 kfree(sun50i_domain); 639 640 return NULL; 641 } 642 643 static void sun50i_iommu_domain_free(struct iommu_domain *domain) 644 { 645 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 646 647 free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE)); 648 sun50i_domain->dt = NULL; 649 650 iommu_put_dma_cookie(domain); 651 652 kfree(sun50i_domain); 653 } 654 655 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, 656 struct sun50i_iommu_domain *sun50i_domain) 657 { 658 iommu->domain = &sun50i_domain->domain; 659 sun50i_domain->iommu = iommu; 660 661 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt, 662 DT_SIZE, DMA_TO_DEVICE); 663 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) { 664 dev_err(iommu->dev, "Couldn't map L1 Page Table\n"); 665 return -ENOMEM; 666 } 667 668 return sun50i_iommu_enable(iommu); 669 } 670 671 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, 672 struct sun50i_iommu_domain *sun50i_domain) 673 { 674 unsigned int i; 675 676 for (i = 0; i < NUM_DT_ENTRIES; i++) { 677 phys_addr_t pt_phys; 678 u32 *page_table; 679 u32 *dte_addr; 680 u32 dte; 681 682 dte_addr = &sun50i_domain->dt[i]; 683 dte = *dte_addr; 684 if (!sun50i_dte_is_pt_valid(dte)) 685 continue; 686 687 memset(dte_addr, 0, sizeof(*dte_addr)); 688 sun50i_table_flush(sun50i_domain, dte_addr, 1); 689 690 pt_phys = sun50i_dte_get_pt_address(dte); 691 page_table = phys_to_virt(pt_phys); 692 sun50i_iommu_free_page_table(iommu, page_table); 693 } 694 695 696 sun50i_iommu_disable(iommu); 697 698 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt), 699 DT_SIZE, DMA_TO_DEVICE); 700 701 iommu->domain = NULL; 702 } 703 704 static void sun50i_iommu_detach_device(struct iommu_domain *domain, 705 struct device *dev) 706 { 707 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 708 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev); 709 710 dev_dbg(dev, "Detaching from IOMMU domain\n"); 711 712 if (iommu->domain != domain) 713 return; 714 715 if (refcount_dec_and_test(&sun50i_domain->refcnt)) 716 sun50i_iommu_detach_domain(iommu, sun50i_domain); 717 } 718 719 static int sun50i_iommu_attach_device(struct iommu_domain *domain, 720 struct device *dev) 721 { 722 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 723 struct sun50i_iommu *iommu; 724 725 iommu = sun50i_iommu_from_dev(dev); 726 if (!iommu) 727 return -ENODEV; 728 729 dev_dbg(dev, "Attaching to IOMMU domain\n"); 730 731 refcount_inc(&sun50i_domain->refcnt); 732 733 if (iommu->domain == domain) 734 return 0; 735 736 if (iommu->domain) 737 sun50i_iommu_detach_device(iommu->domain, dev); 738 739 sun50i_iommu_attach_domain(iommu, sun50i_domain); 740 741 return 0; 742 } 743 744 static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) 745 { 746 struct sun50i_iommu *iommu; 747 748 iommu = sun50i_iommu_from_dev(dev); 749 if (!iommu) 750 return ERR_PTR(-ENODEV); 751 752 return &iommu->iommu; 753 } 754 755 static void sun50i_iommu_release_device(struct device *dev) {} 756 757 static struct iommu_group *sun50i_iommu_device_group(struct device *dev) 758 { 759 struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev); 760 761 return iommu_group_ref_get(iommu->group); 762 } 763 764 static int sun50i_iommu_of_xlate(struct device *dev, 765 struct of_phandle_args *args) 766 { 767 struct platform_device *iommu_pdev = of_find_device_by_node(args->np); 768 unsigned id = args->args[0]; 769 770 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev)); 771 772 return iommu_fwspec_add_ids(dev, &id, 1); 773 } 774 775 static struct iommu_ops sun50i_iommu_ops = { 776 .pgsize_bitmap = SZ_4K, 777 .attach_dev = sun50i_iommu_attach_device, 778 .detach_dev = sun50i_iommu_detach_device, 779 .device_group = sun50i_iommu_device_group, 780 .domain_alloc = sun50i_iommu_domain_alloc, 781 .domain_free = sun50i_iommu_domain_free, 782 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all, 783 .iotlb_sync = sun50i_iommu_iotlb_sync, 784 .iova_to_phys = sun50i_iommu_iova_to_phys, 785 .map = sun50i_iommu_map, 786 .of_xlate = sun50i_iommu_of_xlate, 787 .probe_device = sun50i_iommu_probe_device, 788 .release_device = sun50i_iommu_release_device, 789 .unmap = sun50i_iommu_unmap, 790 }; 791 792 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, 793 unsigned master, phys_addr_t iova, 794 unsigned prot) 795 { 796 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n", 797 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd"); 798 799 if (iommu->domain) 800 report_iommu_fault(iommu->domain, iommu->dev, iova, prot); 801 else 802 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n"); 803 } 804 805 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, 806 unsigned addr_reg, 807 unsigned blame_reg) 808 { 809 phys_addr_t iova; 810 unsigned master; 811 u32 blame; 812 813 assert_spin_locked(&iommu->iommu_lock); 814 815 iova = iommu_read(iommu, addr_reg); 816 blame = iommu_read(iommu, blame_reg); 817 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 818 819 /* 820 * If the address is not in the page table, we can't get what 821 * operation triggered the fault. Assume it's a read 822 * operation. 823 */ 824 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ); 825 826 return iova; 827 } 828 829 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) 830 { 831 enum sun50i_iommu_aci aci; 832 phys_addr_t iova; 833 unsigned master; 834 unsigned dir; 835 u32 blame; 836 837 assert_spin_locked(&iommu->iommu_lock); 838 839 blame = iommu_read(iommu, IOMMU_INT_STA_REG); 840 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 841 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master)); 842 aci = sun50i_get_pte_aci(iommu_read(iommu, 843 IOMMU_INT_ERR_DATA_REG(master))); 844 845 switch (aci) { 846 /* 847 * If we are in the read-only domain, then it means we 848 * tried to write. 849 */ 850 case SUN50I_IOMMU_ACI_RD: 851 dir = IOMMU_FAULT_WRITE; 852 break; 853 854 /* 855 * If we are in the write-only domain, then it means 856 * we tried to read. 857 */ 858 case SUN50I_IOMMU_ACI_WR: 859 860 /* 861 * If we are in the domain without any permission, we 862 * can't really tell. Let's default to a read 863 * operation. 864 */ 865 case SUN50I_IOMMU_ACI_NONE: 866 867 /* WTF? */ 868 case SUN50I_IOMMU_ACI_RD_WR: 869 default: 870 dir = IOMMU_FAULT_READ; 871 break; 872 } 873 874 /* 875 * If the address is not in the page table, we can't get what 876 * operation triggered the fault. Assume it's a read 877 * operation. 878 */ 879 sun50i_iommu_report_fault(iommu, master, iova, dir); 880 881 return iova; 882 } 883 884 static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id) 885 { 886 struct sun50i_iommu *iommu = dev_id; 887 phys_addr_t iova; 888 u32 status; 889 890 spin_lock(&iommu->iommu_lock); 891 892 status = iommu_read(iommu, IOMMU_INT_STA_REG); 893 if (!(status & IOMMU_INT_MASK)) { 894 spin_unlock(&iommu->iommu_lock); 895 return IRQ_NONE; 896 } 897 898 if (status & IOMMU_INT_INVALID_L2PG) 899 iova = sun50i_iommu_handle_pt_irq(iommu, 900 IOMMU_INT_ERR_ADDR_L2_REG, 901 IOMMU_L2PG_INT_REG); 902 else if (status & IOMMU_INT_INVALID_L1PG) 903 iova = sun50i_iommu_handle_pt_irq(iommu, 904 IOMMU_INT_ERR_ADDR_L1_REG, 905 IOMMU_L1PG_INT_REG); 906 else 907 iova = sun50i_iommu_handle_perm_irq(iommu); 908 909 iommu_write(iommu, IOMMU_INT_CLR_REG, status); 910 911 iommu_write(iommu, IOMMU_RESET_REG, ~status); 912 iommu_write(iommu, IOMMU_RESET_REG, status); 913 914 spin_unlock(&iommu->iommu_lock); 915 916 return IRQ_HANDLED; 917 } 918 919 static int sun50i_iommu_probe(struct platform_device *pdev) 920 { 921 struct sun50i_iommu *iommu; 922 int ret, irq; 923 924 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); 925 if (!iommu) 926 return -ENOMEM; 927 spin_lock_init(&iommu->iommu_lock); 928 platform_set_drvdata(pdev, iommu); 929 iommu->dev = &pdev->dev; 930 931 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), 932 PT_SIZE, PT_SIZE, 933 SLAB_HWCACHE_ALIGN, 934 NULL); 935 if (!iommu->pt_pool) 936 return -ENOMEM; 937 938 iommu->group = iommu_group_alloc(); 939 if (IS_ERR(iommu->group)) { 940 ret = PTR_ERR(iommu->group); 941 goto err_free_cache; 942 } 943 944 iommu->base = devm_platform_ioremap_resource(pdev, 0); 945 if (!iommu->base) { 946 ret = PTR_ERR(iommu->base); 947 goto err_free_group; 948 } 949 950 irq = platform_get_irq(pdev, 0); 951 if (irq < 0) { 952 ret = irq; 953 goto err_free_group; 954 } 955 956 iommu->clk = devm_clk_get(&pdev->dev, NULL); 957 if (IS_ERR(iommu->clk)) { 958 dev_err(&pdev->dev, "Couldn't get our clock.\n"); 959 ret = PTR_ERR(iommu->clk); 960 goto err_free_group; 961 } 962 963 iommu->reset = devm_reset_control_get(&pdev->dev, NULL); 964 if (IS_ERR(iommu->reset)) { 965 dev_err(&pdev->dev, "Couldn't get our reset line.\n"); 966 ret = PTR_ERR(iommu->reset); 967 goto err_free_group; 968 } 969 970 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev, 971 NULL, dev_name(&pdev->dev)); 972 if (ret) 973 goto err_free_group; 974 975 iommu_device_set_ops(&iommu->iommu, &sun50i_iommu_ops); 976 iommu_device_set_fwnode(&iommu->iommu, &pdev->dev.of_node->fwnode); 977 978 ret = iommu_device_register(&iommu->iommu); 979 if (ret) 980 goto err_remove_sysfs; 981 982 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0, 983 dev_name(&pdev->dev), iommu); 984 if (ret < 0) 985 goto err_unregister; 986 987 bus_set_iommu(&platform_bus_type, &sun50i_iommu_ops); 988 989 return 0; 990 991 err_unregister: 992 iommu_device_unregister(&iommu->iommu); 993 994 err_remove_sysfs: 995 iommu_device_sysfs_remove(&iommu->iommu); 996 997 err_free_group: 998 iommu_group_put(iommu->group); 999 1000 err_free_cache: 1001 kmem_cache_destroy(iommu->pt_pool); 1002 1003 return ret; 1004 } 1005 1006 static const struct of_device_id sun50i_iommu_dt[] = { 1007 { .compatible = "allwinner,sun50i-h6-iommu", }, 1008 { /* sentinel */ }, 1009 }; 1010 MODULE_DEVICE_TABLE(of, sun50i_iommu_dt); 1011 1012 static struct platform_driver sun50i_iommu_driver = { 1013 .driver = { 1014 .name = "sun50i-iommu", 1015 .of_match_table = sun50i_iommu_dt, 1016 .suppress_bind_attrs = true, 1017 } 1018 }; 1019 builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe); 1020 1021 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver"); 1022 MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>"); 1023 MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>"); 1024 MODULE_LICENSE("Dual BSD/GPL"); 1025