1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (C) 2016-2018, Allwinner Technology CO., LTD. 3 // Copyright (C) 2019-2020, Cerno 4 5 #include <linux/bitfield.h> 6 #include <linux/bug.h> 7 #include <linux/clk.h> 8 #include <linux/device.h> 9 #include <linux/dma-direction.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/err.h> 12 #include <linux/errno.h> 13 #include <linux/interrupt.h> 14 #include <linux/iommu.h> 15 #include <linux/iopoll.h> 16 #include <linux/ioport.h> 17 #include <linux/log2.h> 18 #include <linux/module.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/reset.h> 24 #include <linux/sizes.h> 25 #include <linux/slab.h> 26 #include <linux/spinlock.h> 27 #include <linux/types.h> 28 29 #define IOMMU_RESET_REG 0x010 30 #define IOMMU_RESET_RELEASE_ALL 0xffffffff 31 #define IOMMU_ENABLE_REG 0x020 32 #define IOMMU_ENABLE_ENABLE BIT(0) 33 34 #define IOMMU_BYPASS_REG 0x030 35 #define IOMMU_AUTO_GATING_REG 0x040 36 #define IOMMU_AUTO_GATING_ENABLE BIT(0) 37 38 #define IOMMU_WBUF_CTRL_REG 0x044 39 #define IOMMU_OOO_CTRL_REG 0x048 40 #define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c 41 #define IOMMU_TTB_REG 0x050 42 #define IOMMU_TLB_ENABLE_REG 0x060 43 #define IOMMU_TLB_PREFETCH_REG 0x070 44 #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m) 45 46 #define IOMMU_TLB_FLUSH_REG 0x080 47 #define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17) 48 #define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16) 49 #define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0)) 50 51 #define IOMMU_TLB_IVLD_ADDR_REG 0x090 52 #define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094 53 #define IOMMU_TLB_IVLD_ENABLE_REG 0x098 54 #define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0) 55 56 #define IOMMU_PC_IVLD_ADDR_REG 0x0a0 57 #define IOMMU_PC_IVLD_ENABLE_REG 0x0a8 58 #define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0) 59 60 #define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4) 61 #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2))) 62 #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1)) 63 64 #define IOMMU_DM_AUT_OVWT_REG 0x0d0 65 #define IOMMU_INT_ENABLE_REG 0x100 66 #define IOMMU_INT_CLR_REG 0x104 67 #define IOMMU_INT_STA_REG 0x108 68 #define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4) 69 #define IOMMU_INT_ERR_ADDR_L1_REG 0x130 70 #define IOMMU_INT_ERR_ADDR_L2_REG 0x134 71 #define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4) 72 #define IOMMU_L1PG_INT_REG 0x0180 73 #define IOMMU_L2PG_INT_REG 0x0184 74 75 #define IOMMU_INT_INVALID_L2PG BIT(17) 76 #define IOMMU_INT_INVALID_L1PG BIT(16) 77 #define IOMMU_INT_MASTER_PERMISSION(m) BIT(m) 78 #define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \ 79 IOMMU_INT_MASTER_PERMISSION(1) | \ 80 IOMMU_INT_MASTER_PERMISSION(2) | \ 81 IOMMU_INT_MASTER_PERMISSION(3) | \ 82 IOMMU_INT_MASTER_PERMISSION(4) | \ 83 IOMMU_INT_MASTER_PERMISSION(5)) 84 #define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \ 85 IOMMU_INT_INVALID_L2PG | \ 86 IOMMU_INT_MASTER_MASK) 87 88 #define PT_ENTRY_SIZE sizeof(u32) 89 90 #define NUM_DT_ENTRIES 4096 91 #define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE) 92 93 #define NUM_PT_ENTRIES 256 94 #define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE) 95 96 #define SPAGE_SIZE 4096 97 98 struct sun50i_iommu { 99 struct iommu_device iommu; 100 101 /* Lock to modify the IOMMU registers */ 102 spinlock_t iommu_lock; 103 104 struct device *dev; 105 void __iomem *base; 106 struct reset_control *reset; 107 struct clk *clk; 108 109 struct iommu_domain *domain; 110 struct iommu_group *group; 111 struct kmem_cache *pt_pool; 112 }; 113 114 struct sun50i_iommu_domain { 115 struct iommu_domain domain; 116 117 /* Number of devices attached to the domain */ 118 refcount_t refcnt; 119 120 /* L1 Page Table */ 121 u32 *dt; 122 dma_addr_t dt_dma; 123 124 struct sun50i_iommu *iommu; 125 }; 126 127 static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain) 128 { 129 return container_of(domain, struct sun50i_iommu_domain, domain); 130 } 131 132 static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev) 133 { 134 return dev_iommu_priv_get(dev); 135 } 136 137 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) 138 { 139 return readl(iommu->base + offset); 140 } 141 142 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) 143 { 144 writel(value, iommu->base + offset); 145 } 146 147 /* 148 * The Allwinner H6 IOMMU uses a 2-level page table. 149 * 150 * The first level is the usual Directory Table (DT), that consists of 151 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page 152 * Table (PT). 153 * 154 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each 155 * pointing to a 4kB page of physical memory. 156 * 157 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG 158 * register that contains its physical address. 159 */ 160 161 #define SUN50I_IOVA_DTE_MASK GENMASK(31, 20) 162 #define SUN50I_IOVA_PTE_MASK GENMASK(19, 12) 163 #define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0) 164 165 static u32 sun50i_iova_get_dte_index(dma_addr_t iova) 166 { 167 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova); 168 } 169 170 static u32 sun50i_iova_get_pte_index(dma_addr_t iova) 171 { 172 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova); 173 } 174 175 static u32 sun50i_iova_get_page_offset(dma_addr_t iova) 176 { 177 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova); 178 } 179 180 /* 181 * Each Directory Table Entry has a Page Table address and a valid 182 * bit: 183 184 * +---------------------+-----------+-+ 185 * | PT address | Reserved |V| 186 * +---------------------+-----------+-+ 187 * 31:10 - Page Table address 188 * 9:2 - Reserved 189 * 1:0 - 1 if the entry is valid 190 */ 191 192 #define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10) 193 #define SUN50I_DTE_PT_ATTRS GENMASK(1, 0) 194 #define SUN50I_DTE_PT_VALID 1 195 196 static phys_addr_t sun50i_dte_get_pt_address(u32 dte) 197 { 198 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK; 199 } 200 201 static bool sun50i_dte_is_pt_valid(u32 dte) 202 { 203 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID; 204 } 205 206 static u32 sun50i_mk_dte(dma_addr_t pt_dma) 207 { 208 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID; 209 } 210 211 /* 212 * Each PTE has a Page address, an authority index and a valid bit: 213 * 214 * +----------------+-----+-----+-----+---+-----+ 215 * | Page address | Rsv | ACI | Rsv | V | Rsv | 216 * +----------------+-----+-----+-----+---+-----+ 217 * 31:12 - Page address 218 * 11:8 - Reserved 219 * 7:4 - Authority Control Index 220 * 3:2 - Reserved 221 * 1 - 1 if the entry is valid 222 * 0 - Reserved 223 * 224 * The way permissions work is that the IOMMU has 16 "domains" that 225 * can be configured to give each masters either read or write 226 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain 227 * 0 seems like the default domain, and its permissions in the 228 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really 229 * useful to enforce any particular permission. 230 * 231 * Each page entry will then have a reference to the domain they are 232 * affected to, so that we can actually enforce them on a per-page 233 * basis. 234 * 235 * In order to make it work with the IOMMU framework, we will be using 236 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE 237 * depending on the permission we want to enforce. Each domain will 238 * have each master setup in the same way, since the IOMMU framework 239 * doesn't seem to restrict page access on a per-device basis. And 240 * then we will use the relevant domain index when generating the page 241 * table entry depending on the permissions we want to be enforced. 242 */ 243 244 enum sun50i_iommu_aci { 245 SUN50I_IOMMU_ACI_DO_NOT_USE = 0, 246 SUN50I_IOMMU_ACI_NONE, 247 SUN50I_IOMMU_ACI_RD, 248 SUN50I_IOMMU_ACI_WR, 249 SUN50I_IOMMU_ACI_RD_WR, 250 }; 251 252 #define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12) 253 #define SUN50I_PTE_ACI_MASK GENMASK(7, 4) 254 #define SUN50I_PTE_PAGE_VALID BIT(1) 255 256 static phys_addr_t sun50i_pte_get_page_address(u32 pte) 257 { 258 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK; 259 } 260 261 static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte) 262 { 263 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte); 264 } 265 266 static bool sun50i_pte_is_page_valid(u32 pte) 267 { 268 return pte & SUN50I_PTE_PAGE_VALID; 269 } 270 271 static u32 sun50i_mk_pte(phys_addr_t page, int prot) 272 { 273 enum sun50i_iommu_aci aci; 274 u32 flags = 0; 275 276 if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE)) 277 aci = SUN50I_IOMMU_ACI_RD_WR; 278 else if (prot & IOMMU_READ) 279 aci = SUN50I_IOMMU_ACI_RD; 280 else if (prot & IOMMU_WRITE) 281 aci = SUN50I_IOMMU_ACI_WR; 282 else 283 aci = SUN50I_IOMMU_ACI_NONE; 284 285 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci); 286 page &= SUN50I_PTE_PAGE_ADDRESS_MASK; 287 return page | flags | SUN50I_PTE_PAGE_VALID; 288 } 289 290 static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain, 291 void *vaddr, unsigned int count) 292 { 293 struct sun50i_iommu *iommu = sun50i_domain->iommu; 294 dma_addr_t dma = virt_to_phys(vaddr); 295 size_t size = count * PT_ENTRY_SIZE; 296 297 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE); 298 } 299 300 static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu, 301 unsigned long iova) 302 { 303 u32 reg; 304 int ret; 305 306 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova); 307 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12)); 308 iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 309 IOMMU_TLB_IVLD_ENABLE_ENABLE); 310 311 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG, 312 reg, !reg, 1, 2000); 313 if (ret) 314 dev_warn(iommu->dev, "TLB invalidation timed out!\n"); 315 } 316 317 static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu, 318 unsigned long iova) 319 { 320 u32 reg; 321 int ret; 322 323 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova); 324 iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 325 IOMMU_PC_IVLD_ENABLE_ENABLE); 326 327 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG, 328 reg, !reg, 1, 2000); 329 if (ret) 330 dev_warn(iommu->dev, "PTW cache invalidation timed out!\n"); 331 } 332 333 static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu, 334 unsigned long iova, size_t size) 335 { 336 assert_spin_locked(&iommu->iommu_lock); 337 338 iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0); 339 340 sun50i_iommu_zap_iova(iommu, iova); 341 sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE); 342 if (size > SPAGE_SIZE) { 343 sun50i_iommu_zap_iova(iommu, iova + size); 344 sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE); 345 } 346 sun50i_iommu_zap_ptw_cache(iommu, iova); 347 sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M); 348 if (size > SZ_1M) { 349 sun50i_iommu_zap_ptw_cache(iommu, iova + size); 350 sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M); 351 } 352 353 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); 354 } 355 356 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) 357 { 358 u32 reg; 359 int ret; 360 361 assert_spin_locked(&iommu->iommu_lock); 362 363 iommu_write(iommu, 364 IOMMU_TLB_FLUSH_REG, 365 IOMMU_TLB_FLUSH_PTW_CACHE | 366 IOMMU_TLB_FLUSH_MACRO_TLB | 367 IOMMU_TLB_FLUSH_MICRO_TLB(5) | 368 IOMMU_TLB_FLUSH_MICRO_TLB(4) | 369 IOMMU_TLB_FLUSH_MICRO_TLB(3) | 370 IOMMU_TLB_FLUSH_MICRO_TLB(2) | 371 IOMMU_TLB_FLUSH_MICRO_TLB(1) | 372 IOMMU_TLB_FLUSH_MICRO_TLB(0)); 373 374 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG, 375 reg, !reg, 376 1, 2000); 377 if (ret) 378 dev_warn(iommu->dev, "TLB Flush timed out!\n"); 379 380 return ret; 381 } 382 383 static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain) 384 { 385 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 386 struct sun50i_iommu *iommu = sun50i_domain->iommu; 387 unsigned long flags; 388 389 /* 390 * At boot, we'll have a first call into .flush_iotlb_all right after 391 * .probe_device, and since we link our (single) domain to our iommu in 392 * the .attach_device callback, we don't have that pointer set. 393 * 394 * It shouldn't really be any trouble to ignore it though since we flush 395 * all caches as part of the device powerup. 396 */ 397 if (!iommu) 398 return; 399 400 spin_lock_irqsave(&iommu->iommu_lock, flags); 401 sun50i_iommu_flush_all_tlb(iommu); 402 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 403 } 404 405 static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain, 406 unsigned long iova, size_t size) 407 { 408 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 409 struct sun50i_iommu *iommu = sun50i_domain->iommu; 410 unsigned long flags; 411 412 spin_lock_irqsave(&iommu->iommu_lock, flags); 413 sun50i_iommu_zap_range(iommu, iova, size); 414 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 415 } 416 417 static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain, 418 struct iommu_iotlb_gather *gather) 419 { 420 sun50i_iommu_flush_iotlb_all(domain); 421 } 422 423 static int sun50i_iommu_enable(struct sun50i_iommu *iommu) 424 { 425 struct sun50i_iommu_domain *sun50i_domain; 426 unsigned long flags; 427 int ret; 428 429 if (!iommu->domain) 430 return 0; 431 432 sun50i_domain = to_sun50i_domain(iommu->domain); 433 434 ret = reset_control_deassert(iommu->reset); 435 if (ret) 436 return ret; 437 438 ret = clk_prepare_enable(iommu->clk); 439 if (ret) 440 goto err_reset_assert; 441 442 spin_lock_irqsave(&iommu->iommu_lock, flags); 443 444 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma); 445 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, 446 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) | 447 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) | 448 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) | 449 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) | 450 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) | 451 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5)); 452 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); 453 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), 454 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 455 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 456 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 457 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 458 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 459 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 460 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 461 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 462 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 463 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 464 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) | 465 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5)); 466 467 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD), 468 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) | 469 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) | 470 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) | 471 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) | 472 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) | 473 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5)); 474 475 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR), 476 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) | 477 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) | 478 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) | 479 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) | 480 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) | 481 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5)); 482 483 ret = sun50i_iommu_flush_all_tlb(iommu); 484 if (ret) { 485 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 486 goto err_clk_disable; 487 } 488 489 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); 490 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE); 491 492 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 493 494 return 0; 495 496 err_clk_disable: 497 clk_disable_unprepare(iommu->clk); 498 499 err_reset_assert: 500 reset_control_assert(iommu->reset); 501 502 return ret; 503 } 504 505 static void sun50i_iommu_disable(struct sun50i_iommu *iommu) 506 { 507 unsigned long flags; 508 509 spin_lock_irqsave(&iommu->iommu_lock, flags); 510 511 iommu_write(iommu, IOMMU_ENABLE_REG, 0); 512 iommu_write(iommu, IOMMU_TTB_REG, 0); 513 514 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 515 516 clk_disable_unprepare(iommu->clk); 517 reset_control_assert(iommu->reset); 518 } 519 520 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, 521 gfp_t gfp) 522 { 523 dma_addr_t pt_dma; 524 u32 *page_table; 525 526 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp); 527 if (!page_table) 528 return ERR_PTR(-ENOMEM); 529 530 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE); 531 if (dma_mapping_error(iommu->dev, pt_dma)) { 532 dev_err(iommu->dev, "Couldn't map L2 Page Table\n"); 533 kmem_cache_free(iommu->pt_pool, page_table); 534 return ERR_PTR(-ENOMEM); 535 } 536 537 /* We rely on the physical address and DMA address being the same */ 538 WARN_ON(pt_dma != virt_to_phys(page_table)); 539 540 return page_table; 541 } 542 543 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, 544 u32 *page_table) 545 { 546 phys_addr_t pt_phys = virt_to_phys(page_table); 547 548 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE); 549 kmem_cache_free(iommu->pt_pool, page_table); 550 } 551 552 static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain, 553 dma_addr_t iova, gfp_t gfp) 554 { 555 struct sun50i_iommu *iommu = sun50i_domain->iommu; 556 u32 *page_table; 557 u32 *dte_addr; 558 u32 old_dte; 559 u32 dte; 560 561 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 562 dte = *dte_addr; 563 if (sun50i_dte_is_pt_valid(dte)) { 564 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte); 565 return (u32 *)phys_to_virt(pt_phys); 566 } 567 568 page_table = sun50i_iommu_alloc_page_table(iommu, gfp); 569 if (IS_ERR(page_table)) 570 return page_table; 571 572 dte = sun50i_mk_dte(virt_to_phys(page_table)); 573 old_dte = cmpxchg(dte_addr, 0, dte); 574 if (old_dte) { 575 phys_addr_t installed_pt_phys = 576 sun50i_dte_get_pt_address(old_dte); 577 u32 *installed_pt = phys_to_virt(installed_pt_phys); 578 u32 *drop_pt = page_table; 579 580 page_table = installed_pt; 581 dte = old_dte; 582 sun50i_iommu_free_page_table(iommu, drop_pt); 583 } 584 585 sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES); 586 sun50i_table_flush(sun50i_domain, dte_addr, 1); 587 588 return page_table; 589 } 590 591 static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova, 592 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 593 { 594 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 595 struct sun50i_iommu *iommu = sun50i_domain->iommu; 596 u32 pte_index; 597 u32 *page_table, *pte_addr; 598 int ret = 0; 599 600 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp); 601 if (IS_ERR(page_table)) { 602 ret = PTR_ERR(page_table); 603 goto out; 604 } 605 606 pte_index = sun50i_iova_get_pte_index(iova); 607 pte_addr = &page_table[pte_index]; 608 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) { 609 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr); 610 dev_err(iommu->dev, 611 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n", 612 &iova, &page_phys, &paddr, prot); 613 ret = -EBUSY; 614 goto out; 615 } 616 617 *pte_addr = sun50i_mk_pte(paddr, prot); 618 sun50i_table_flush(sun50i_domain, pte_addr, 1); 619 620 out: 621 return ret; 622 } 623 624 static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova, 625 size_t size, struct iommu_iotlb_gather *gather) 626 { 627 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 628 phys_addr_t pt_phys; 629 u32 *pte_addr; 630 u32 dte; 631 632 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 633 if (!sun50i_dte_is_pt_valid(dte)) 634 return 0; 635 636 pt_phys = sun50i_dte_get_pt_address(dte); 637 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova); 638 639 if (!sun50i_pte_is_page_valid(*pte_addr)) 640 return 0; 641 642 memset(pte_addr, 0, sizeof(*pte_addr)); 643 sun50i_table_flush(sun50i_domain, pte_addr, 1); 644 645 return SZ_4K; 646 } 647 648 static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain, 649 dma_addr_t iova) 650 { 651 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 652 phys_addr_t pt_phys; 653 u32 *page_table; 654 u32 dte, pte; 655 656 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 657 if (!sun50i_dte_is_pt_valid(dte)) 658 return 0; 659 660 pt_phys = sun50i_dte_get_pt_address(dte); 661 page_table = (u32 *)phys_to_virt(pt_phys); 662 pte = page_table[sun50i_iova_get_pte_index(iova)]; 663 if (!sun50i_pte_is_page_valid(pte)) 664 return 0; 665 666 return sun50i_pte_get_page_address(pte) + 667 sun50i_iova_get_page_offset(iova); 668 } 669 670 static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type) 671 { 672 struct sun50i_iommu_domain *sun50i_domain; 673 674 if (type != IOMMU_DOMAIN_DMA && 675 type != IOMMU_DOMAIN_UNMANAGED) 676 return NULL; 677 678 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL); 679 if (!sun50i_domain) 680 return NULL; 681 682 sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 683 get_order(DT_SIZE)); 684 if (!sun50i_domain->dt) 685 goto err_free_domain; 686 687 refcount_set(&sun50i_domain->refcnt, 1); 688 689 sun50i_domain->domain.geometry.aperture_start = 0; 690 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); 691 sun50i_domain->domain.geometry.force_aperture = true; 692 693 return &sun50i_domain->domain; 694 695 err_free_domain: 696 kfree(sun50i_domain); 697 698 return NULL; 699 } 700 701 static void sun50i_iommu_domain_free(struct iommu_domain *domain) 702 { 703 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 704 705 free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE)); 706 sun50i_domain->dt = NULL; 707 708 kfree(sun50i_domain); 709 } 710 711 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, 712 struct sun50i_iommu_domain *sun50i_domain) 713 { 714 iommu->domain = &sun50i_domain->domain; 715 sun50i_domain->iommu = iommu; 716 717 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt, 718 DT_SIZE, DMA_TO_DEVICE); 719 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) { 720 dev_err(iommu->dev, "Couldn't map L1 Page Table\n"); 721 return -ENOMEM; 722 } 723 724 return sun50i_iommu_enable(iommu); 725 } 726 727 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, 728 struct sun50i_iommu_domain *sun50i_domain) 729 { 730 unsigned int i; 731 732 for (i = 0; i < NUM_DT_ENTRIES; i++) { 733 phys_addr_t pt_phys; 734 u32 *page_table; 735 u32 *dte_addr; 736 u32 dte; 737 738 dte_addr = &sun50i_domain->dt[i]; 739 dte = *dte_addr; 740 if (!sun50i_dte_is_pt_valid(dte)) 741 continue; 742 743 memset(dte_addr, 0, sizeof(*dte_addr)); 744 sun50i_table_flush(sun50i_domain, dte_addr, 1); 745 746 pt_phys = sun50i_dte_get_pt_address(dte); 747 page_table = phys_to_virt(pt_phys); 748 sun50i_iommu_free_page_table(iommu, page_table); 749 } 750 751 752 sun50i_iommu_disable(iommu); 753 754 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt), 755 DT_SIZE, DMA_TO_DEVICE); 756 757 iommu->domain = NULL; 758 } 759 760 static void sun50i_iommu_detach_device(struct iommu_domain *domain, 761 struct device *dev) 762 { 763 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 764 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev); 765 766 dev_dbg(dev, "Detaching from IOMMU domain\n"); 767 768 if (iommu->domain != domain) 769 return; 770 771 if (refcount_dec_and_test(&sun50i_domain->refcnt)) 772 sun50i_iommu_detach_domain(iommu, sun50i_domain); 773 } 774 775 static int sun50i_iommu_attach_device(struct iommu_domain *domain, 776 struct device *dev) 777 { 778 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 779 struct sun50i_iommu *iommu; 780 781 iommu = sun50i_iommu_from_dev(dev); 782 if (!iommu) 783 return -ENODEV; 784 785 dev_dbg(dev, "Attaching to IOMMU domain\n"); 786 787 refcount_inc(&sun50i_domain->refcnt); 788 789 if (iommu->domain == domain) 790 return 0; 791 792 if (iommu->domain) 793 sun50i_iommu_detach_device(iommu->domain, dev); 794 795 sun50i_iommu_attach_domain(iommu, sun50i_domain); 796 797 return 0; 798 } 799 800 static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) 801 { 802 struct sun50i_iommu *iommu; 803 804 iommu = sun50i_iommu_from_dev(dev); 805 if (!iommu) 806 return ERR_PTR(-ENODEV); 807 808 return &iommu->iommu; 809 } 810 811 static struct iommu_group *sun50i_iommu_device_group(struct device *dev) 812 { 813 struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev); 814 815 return iommu_group_ref_get(iommu->group); 816 } 817 818 static int sun50i_iommu_of_xlate(struct device *dev, 819 struct of_phandle_args *args) 820 { 821 struct platform_device *iommu_pdev = of_find_device_by_node(args->np); 822 unsigned id = args->args[0]; 823 824 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev)); 825 826 return iommu_fwspec_add_ids(dev, &id, 1); 827 } 828 829 static const struct iommu_ops sun50i_iommu_ops = { 830 .pgsize_bitmap = SZ_4K, 831 .device_group = sun50i_iommu_device_group, 832 .domain_alloc = sun50i_iommu_domain_alloc, 833 .of_xlate = sun50i_iommu_of_xlate, 834 .probe_device = sun50i_iommu_probe_device, 835 .default_domain_ops = &(const struct iommu_domain_ops) { 836 .attach_dev = sun50i_iommu_attach_device, 837 .detach_dev = sun50i_iommu_detach_device, 838 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all, 839 .iotlb_sync_map = sun50i_iommu_iotlb_sync_map, 840 .iotlb_sync = sun50i_iommu_iotlb_sync, 841 .iova_to_phys = sun50i_iommu_iova_to_phys, 842 .map = sun50i_iommu_map, 843 .unmap = sun50i_iommu_unmap, 844 .free = sun50i_iommu_domain_free, 845 } 846 }; 847 848 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, 849 unsigned master, phys_addr_t iova, 850 unsigned prot) 851 { 852 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n", 853 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd"); 854 855 if (iommu->domain) 856 report_iommu_fault(iommu->domain, iommu->dev, iova, prot); 857 else 858 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n"); 859 860 sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE); 861 } 862 863 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, 864 unsigned addr_reg, 865 unsigned blame_reg) 866 { 867 phys_addr_t iova; 868 unsigned master; 869 u32 blame; 870 871 assert_spin_locked(&iommu->iommu_lock); 872 873 iova = iommu_read(iommu, addr_reg); 874 blame = iommu_read(iommu, blame_reg); 875 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 876 877 /* 878 * If the address is not in the page table, we can't get what 879 * operation triggered the fault. Assume it's a read 880 * operation. 881 */ 882 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ); 883 884 return iova; 885 } 886 887 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) 888 { 889 enum sun50i_iommu_aci aci; 890 phys_addr_t iova; 891 unsigned master; 892 unsigned dir; 893 u32 blame; 894 895 assert_spin_locked(&iommu->iommu_lock); 896 897 blame = iommu_read(iommu, IOMMU_INT_STA_REG); 898 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 899 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master)); 900 aci = sun50i_get_pte_aci(iommu_read(iommu, 901 IOMMU_INT_ERR_DATA_REG(master))); 902 903 switch (aci) { 904 /* 905 * If we are in the read-only domain, then it means we 906 * tried to write. 907 */ 908 case SUN50I_IOMMU_ACI_RD: 909 dir = IOMMU_FAULT_WRITE; 910 break; 911 912 /* 913 * If we are in the write-only domain, then it means 914 * we tried to read. 915 */ 916 case SUN50I_IOMMU_ACI_WR: 917 918 /* 919 * If we are in the domain without any permission, we 920 * can't really tell. Let's default to a read 921 * operation. 922 */ 923 case SUN50I_IOMMU_ACI_NONE: 924 925 /* WTF? */ 926 case SUN50I_IOMMU_ACI_RD_WR: 927 default: 928 dir = IOMMU_FAULT_READ; 929 break; 930 } 931 932 /* 933 * If the address is not in the page table, we can't get what 934 * operation triggered the fault. Assume it's a read 935 * operation. 936 */ 937 sun50i_iommu_report_fault(iommu, master, iova, dir); 938 939 return iova; 940 } 941 942 static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id) 943 { 944 u32 status, l1_status, l2_status, resets; 945 struct sun50i_iommu *iommu = dev_id; 946 947 spin_lock(&iommu->iommu_lock); 948 949 status = iommu_read(iommu, IOMMU_INT_STA_REG); 950 if (!(status & IOMMU_INT_MASK)) { 951 spin_unlock(&iommu->iommu_lock); 952 return IRQ_NONE; 953 } 954 955 l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG); 956 l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG); 957 958 if (status & IOMMU_INT_INVALID_L2PG) 959 sun50i_iommu_handle_pt_irq(iommu, 960 IOMMU_INT_ERR_ADDR_L2_REG, 961 IOMMU_L2PG_INT_REG); 962 else if (status & IOMMU_INT_INVALID_L1PG) 963 sun50i_iommu_handle_pt_irq(iommu, 964 IOMMU_INT_ERR_ADDR_L1_REG, 965 IOMMU_L1PG_INT_REG); 966 else 967 sun50i_iommu_handle_perm_irq(iommu); 968 969 iommu_write(iommu, IOMMU_INT_CLR_REG, status); 970 971 resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK; 972 iommu_write(iommu, IOMMU_RESET_REG, ~resets); 973 iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL); 974 975 spin_unlock(&iommu->iommu_lock); 976 977 return IRQ_HANDLED; 978 } 979 980 static int sun50i_iommu_probe(struct platform_device *pdev) 981 { 982 struct sun50i_iommu *iommu; 983 int ret, irq; 984 985 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); 986 if (!iommu) 987 return -ENOMEM; 988 spin_lock_init(&iommu->iommu_lock); 989 platform_set_drvdata(pdev, iommu); 990 iommu->dev = &pdev->dev; 991 992 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), 993 PT_SIZE, PT_SIZE, 994 SLAB_HWCACHE_ALIGN, 995 NULL); 996 if (!iommu->pt_pool) 997 return -ENOMEM; 998 999 iommu->group = iommu_group_alloc(); 1000 if (IS_ERR(iommu->group)) { 1001 ret = PTR_ERR(iommu->group); 1002 goto err_free_cache; 1003 } 1004 1005 iommu->base = devm_platform_ioremap_resource(pdev, 0); 1006 if (IS_ERR(iommu->base)) { 1007 ret = PTR_ERR(iommu->base); 1008 goto err_free_group; 1009 } 1010 1011 irq = platform_get_irq(pdev, 0); 1012 if (irq < 0) { 1013 ret = irq; 1014 goto err_free_group; 1015 } 1016 1017 iommu->clk = devm_clk_get(&pdev->dev, NULL); 1018 if (IS_ERR(iommu->clk)) { 1019 dev_err(&pdev->dev, "Couldn't get our clock.\n"); 1020 ret = PTR_ERR(iommu->clk); 1021 goto err_free_group; 1022 } 1023 1024 iommu->reset = devm_reset_control_get(&pdev->dev, NULL); 1025 if (IS_ERR(iommu->reset)) { 1026 dev_err(&pdev->dev, "Couldn't get our reset line.\n"); 1027 ret = PTR_ERR(iommu->reset); 1028 goto err_free_group; 1029 } 1030 1031 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev, 1032 NULL, dev_name(&pdev->dev)); 1033 if (ret) 1034 goto err_free_group; 1035 1036 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev); 1037 if (ret) 1038 goto err_remove_sysfs; 1039 1040 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0, 1041 dev_name(&pdev->dev), iommu); 1042 if (ret < 0) 1043 goto err_unregister; 1044 1045 return 0; 1046 1047 err_unregister: 1048 iommu_device_unregister(&iommu->iommu); 1049 1050 err_remove_sysfs: 1051 iommu_device_sysfs_remove(&iommu->iommu); 1052 1053 err_free_group: 1054 iommu_group_put(iommu->group); 1055 1056 err_free_cache: 1057 kmem_cache_destroy(iommu->pt_pool); 1058 1059 return ret; 1060 } 1061 1062 static const struct of_device_id sun50i_iommu_dt[] = { 1063 { .compatible = "allwinner,sun50i-h6-iommu", }, 1064 { /* sentinel */ }, 1065 }; 1066 MODULE_DEVICE_TABLE(of, sun50i_iommu_dt); 1067 1068 static struct platform_driver sun50i_iommu_driver = { 1069 .driver = { 1070 .name = "sun50i-iommu", 1071 .of_match_table = sun50i_iommu_dt, 1072 .suppress_bind_attrs = true, 1073 } 1074 }; 1075 builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe); 1076 1077 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver"); 1078 MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>"); 1079 MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>"); 1080 MODULE_LICENSE("Dual BSD/GPL"); 1081