1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. 4 * http://www.samsung.com 5 */ 6 7 #ifdef CONFIG_EXYNOS_IOMMU_DEBUG 8 #define DEBUG 9 #endif 10 11 #include <linux/clk.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/err.h> 14 #include <linux/io.h> 15 #include <linux/iommu.h> 16 #include <linux/interrupt.h> 17 #include <linux/kmemleak.h> 18 #include <linux/list.h> 19 #include <linux/of.h> 20 #include <linux/of_platform.h> 21 #include <linux/platform_device.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/slab.h> 24 25 typedef u32 sysmmu_iova_t; 26 typedef u32 sysmmu_pte_t; 27 28 /* We do not consider super section mapping (16MB) */ 29 #define SECT_ORDER 20 30 #define LPAGE_ORDER 16 31 #define SPAGE_ORDER 12 32 33 #define SECT_SIZE (1 << SECT_ORDER) 34 #define LPAGE_SIZE (1 << LPAGE_ORDER) 35 #define SPAGE_SIZE (1 << SPAGE_ORDER) 36 37 #define SECT_MASK (~(SECT_SIZE - 1)) 38 #define LPAGE_MASK (~(LPAGE_SIZE - 1)) 39 #define SPAGE_MASK (~(SPAGE_SIZE - 1)) 40 41 #define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \ 42 ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3)) 43 #define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK) 44 #define lv1ent_page_zero(sent) ((*(sent) & 3) == 1) 45 #define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \ 46 ((*(sent) & 3) == 1)) 47 #define lv1ent_section(sent) ((*(sent) & 3) == 2) 48 49 #define lv2ent_fault(pent) ((*(pent) & 3) == 0) 50 #define lv2ent_small(pent) ((*(pent) & 2) == 2) 51 #define lv2ent_large(pent) ((*(pent) & 3) == 1) 52 53 /* 54 * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces 55 * v5.0 introduced support for 36bit physical address space by shifting 56 * all page entry values by 4 bits. 57 * All SYSMMU controllers in the system support the address spaces of the same 58 * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper 59 * value (0 or 4). 60 */ 61 static short PG_ENT_SHIFT = -1; 62 #define SYSMMU_PG_ENT_SHIFT 0 63 #define SYSMMU_V5_PG_ENT_SHIFT 4 64 65 static const sysmmu_pte_t *LV1_PROT; 66 static const sysmmu_pte_t SYSMMU_LV1_PROT[] = { 67 ((0 << 15) | (0 << 10)), /* no access */ 68 ((1 << 15) | (1 << 10)), /* IOMMU_READ only */ 69 ((0 << 15) | (1 << 10)), /* IOMMU_WRITE not supported, use read/write */ 70 ((0 << 15) | (1 << 10)), /* IOMMU_READ | IOMMU_WRITE */ 71 }; 72 static const sysmmu_pte_t SYSMMU_V5_LV1_PROT[] = { 73 (0 << 4), /* no access */ 74 (1 << 4), /* IOMMU_READ only */ 75 (2 << 4), /* IOMMU_WRITE only */ 76 (3 << 4), /* IOMMU_READ | IOMMU_WRITE */ 77 }; 78 79 static const sysmmu_pte_t *LV2_PROT; 80 static const sysmmu_pte_t SYSMMU_LV2_PROT[] = { 81 ((0 << 9) | (0 << 4)), /* no access */ 82 ((1 << 9) | (1 << 4)), /* IOMMU_READ only */ 83 ((0 << 9) | (1 << 4)), /* IOMMU_WRITE not supported, use read/write */ 84 ((0 << 9) | (1 << 4)), /* IOMMU_READ | IOMMU_WRITE */ 85 }; 86 static const sysmmu_pte_t SYSMMU_V5_LV2_PROT[] = { 87 (0 << 2), /* no access */ 88 (1 << 2), /* IOMMU_READ only */ 89 (2 << 2), /* IOMMU_WRITE only */ 90 (3 << 2), /* IOMMU_READ | IOMMU_WRITE */ 91 }; 92 93 #define SYSMMU_SUPPORTED_PROT_BITS (IOMMU_READ | IOMMU_WRITE) 94 95 #define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT) 96 #define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK) 97 #define section_offs(iova) (iova & (SECT_SIZE - 1)) 98 #define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK) 99 #define lpage_offs(iova) (iova & (LPAGE_SIZE - 1)) 100 #define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK) 101 #define spage_offs(iova) (iova & (SPAGE_SIZE - 1)) 102 103 #define NUM_LV1ENTRIES 4096 104 #define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE) 105 106 static u32 lv1ent_offset(sysmmu_iova_t iova) 107 { 108 return iova >> SECT_ORDER; 109 } 110 111 static u32 lv2ent_offset(sysmmu_iova_t iova) 112 { 113 return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1); 114 } 115 116 #define LV1TABLE_SIZE (NUM_LV1ENTRIES * sizeof(sysmmu_pte_t)) 117 #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t)) 118 119 #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) 120 #define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0)) 121 122 #define mk_lv1ent_sect(pa, prot) ((pa >> PG_ENT_SHIFT) | LV1_PROT[prot] | 2) 123 #define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1) 124 #define mk_lv2ent_lpage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 1) 125 #define mk_lv2ent_spage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 2) 126 127 #define CTRL_ENABLE 0x5 128 #define CTRL_BLOCK 0x7 129 #define CTRL_DISABLE 0x0 130 131 #define CFG_LRU 0x1 132 #define CFG_EAP (1 << 2) 133 #define CFG_QOS(n) ((n & 0xF) << 7) 134 #define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */ 135 #define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */ 136 #define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */ 137 138 #define CTRL_VM_ENABLE BIT(0) 139 #define CTRL_VM_FAULT_MODE_STALL BIT(3) 140 #define CAPA0_CAPA1_EXIST BIT(11) 141 #define CAPA1_VCR_ENABLED BIT(14) 142 143 /* common registers */ 144 #define REG_MMU_CTRL 0x000 145 #define REG_MMU_CFG 0x004 146 #define REG_MMU_STATUS 0x008 147 #define REG_MMU_VERSION 0x034 148 149 #define MMU_MAJ_VER(val) ((val) >> 7) 150 #define MMU_MIN_VER(val) ((val) & 0x7F) 151 #define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ 152 153 #define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) 154 155 /* v1.x - v3.x registers */ 156 #define REG_PAGE_FAULT_ADDR 0x024 157 #define REG_AW_FAULT_ADDR 0x028 158 #define REG_AR_FAULT_ADDR 0x02C 159 #define REG_DEFAULT_SLAVE_ADDR 0x030 160 161 /* v5.x registers */ 162 #define REG_V5_FAULT_AR_VA 0x070 163 #define REG_V5_FAULT_AW_VA 0x080 164 165 /* v7.x registers */ 166 #define REG_V7_CAPA0 0x870 167 #define REG_V7_CAPA1 0x874 168 #define REG_V7_CTRL_VM 0x8000 169 170 #define has_sysmmu(dev) (dev_iommu_priv_get(dev) != NULL) 171 172 static struct device *dma_dev; 173 static struct kmem_cache *lv2table_kmem_cache; 174 static sysmmu_pte_t *zero_lv2_table; 175 #define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table)) 176 177 static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) 178 { 179 return pgtable + lv1ent_offset(iova); 180 } 181 182 static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova) 183 { 184 return (sysmmu_pte_t *)phys_to_virt( 185 lv2table_base(sent)) + lv2ent_offset(iova); 186 } 187 188 /* 189 * IOMMU fault information register 190 */ 191 struct sysmmu_fault_info { 192 unsigned int bit; /* bit number in STATUS register */ 193 unsigned short addr_reg; /* register to read VA fault address */ 194 const char *name; /* human readable fault name */ 195 unsigned int type; /* fault type for report_iommu_fault */ 196 }; 197 198 static const struct sysmmu_fault_info sysmmu_faults[] = { 199 { 0, REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, 200 { 1, REG_AR_FAULT_ADDR, "AR MULTI-HIT", IOMMU_FAULT_READ }, 201 { 2, REG_AW_FAULT_ADDR, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, 202 { 3, REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, 203 { 4, REG_AR_FAULT_ADDR, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, 204 { 5, REG_AR_FAULT_ADDR, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, 205 { 6, REG_AW_FAULT_ADDR, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, 206 { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, 207 }; 208 209 static const struct sysmmu_fault_info sysmmu_v5_faults[] = { 210 { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ }, 211 { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ }, 212 { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ }, 213 { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, 214 { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, 215 { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE }, 216 { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE }, 217 { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, 218 { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, 219 { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, 220 }; 221 222 /* 223 * This structure is attached to dev->iommu->priv of the master device 224 * on device add, contains a list of SYSMMU controllers defined by device tree, 225 * which are bound to given master device. It is usually referenced by 'owner' 226 * pointer. 227 */ 228 struct exynos_iommu_owner { 229 struct list_head controllers; /* list of sysmmu_drvdata.owner_node */ 230 struct iommu_domain *domain; /* domain this device is attached */ 231 struct mutex rpm_lock; /* for runtime pm of all sysmmus */ 232 }; 233 234 /* 235 * This structure exynos specific generalization of struct iommu_domain. 236 * It contains list of SYSMMU controllers from all master devices, which has 237 * been attached to this domain and page tables of IO address space defined by 238 * it. It is usually referenced by 'domain' pointer. 239 */ 240 struct exynos_iommu_domain { 241 struct list_head clients; /* list of sysmmu_drvdata.domain_node */ 242 sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ 243 short *lv2entcnt; /* free lv2 entry counter for each section */ 244 spinlock_t lock; /* lock for modyfying list of clients */ 245 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ 246 struct iommu_domain domain; /* generic domain data structure */ 247 }; 248 249 /* 250 * SysMMU version specific data. Contains offsets for the registers which can 251 * be found in different SysMMU variants, but have different offset values. 252 */ 253 struct sysmmu_variant { 254 u32 pt_base; /* page table base address (physical) */ 255 u32 flush_all; /* invalidate all TLB entries */ 256 u32 flush_entry; /* invalidate specific TLB entry */ 257 u32 flush_range; /* invalidate TLB entries in specified range */ 258 u32 flush_start; /* start address of range invalidation */ 259 u32 flush_end; /* end address of range invalidation */ 260 u32 int_status; /* interrupt status information */ 261 u32 int_clear; /* clear the interrupt */ 262 }; 263 264 /* 265 * This structure hold all data of a single SYSMMU controller, this includes 266 * hw resources like registers and clocks, pointers and list nodes to connect 267 * it to all other structures, internal state and parameters read from device 268 * tree. It is usually referenced by 'data' pointer. 269 */ 270 struct sysmmu_drvdata { 271 struct device *sysmmu; /* SYSMMU controller device */ 272 struct device *master; /* master device (owner) */ 273 struct device_link *link; /* runtime PM link to master */ 274 void __iomem *sfrbase; /* our registers */ 275 struct clk *clk; /* SYSMMU's clock */ 276 struct clk *aclk; /* SYSMMU's aclk clock */ 277 struct clk *pclk; /* SYSMMU's pclk clock */ 278 struct clk *clk_master; /* master's device clock */ 279 spinlock_t lock; /* lock for modyfying state */ 280 bool active; /* current status */ 281 struct exynos_iommu_domain *domain; /* domain we belong to */ 282 struct list_head domain_node; /* node for domain clients list */ 283 struct list_head owner_node; /* node for owner controllers list */ 284 phys_addr_t pgtable; /* assigned page table structure */ 285 unsigned int version; /* our version */ 286 287 struct iommu_device iommu; /* IOMMU core handle */ 288 const struct sysmmu_variant *variant; /* version specific data */ 289 290 /* v7 fields */ 291 bool has_vcr; /* virtual machine control register */ 292 }; 293 294 #define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg) 295 296 /* SysMMU v1..v3 */ 297 static const struct sysmmu_variant sysmmu_v1_variant = { 298 .flush_all = 0x0c, 299 .flush_entry = 0x10, 300 .pt_base = 0x14, 301 .int_status = 0x18, 302 .int_clear = 0x1c, 303 }; 304 305 /* SysMMU v5 and v7 (non-VM capable) */ 306 static const struct sysmmu_variant sysmmu_v5_variant = { 307 .pt_base = 0x0c, 308 .flush_all = 0x10, 309 .flush_entry = 0x14, 310 .flush_range = 0x18, 311 .flush_start = 0x20, 312 .flush_end = 0x24, 313 .int_status = 0x60, 314 .int_clear = 0x64, 315 }; 316 317 /* SysMMU v7: VM capable register set */ 318 static const struct sysmmu_variant sysmmu_v7_vm_variant = { 319 .pt_base = 0x800c, 320 .flush_all = 0x8010, 321 .flush_entry = 0x8014, 322 .flush_range = 0x8018, 323 .flush_start = 0x8020, 324 .flush_end = 0x8024, 325 .int_status = 0x60, 326 .int_clear = 0x64, 327 }; 328 329 static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) 330 { 331 return container_of(dom, struct exynos_iommu_domain, domain); 332 } 333 334 static void sysmmu_unblock(struct sysmmu_drvdata *data) 335 { 336 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); 337 } 338 339 static bool sysmmu_block(struct sysmmu_drvdata *data) 340 { 341 int i = 120; 342 343 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); 344 while ((i > 0) && !(readl(data->sfrbase + REG_MMU_STATUS) & 1)) 345 --i; 346 347 if (!(readl(data->sfrbase + REG_MMU_STATUS) & 1)) { 348 sysmmu_unblock(data); 349 return false; 350 } 351 352 return true; 353 } 354 355 static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) 356 { 357 writel(0x1, SYSMMU_REG(data, flush_all)); 358 } 359 360 static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, 361 sysmmu_iova_t iova, unsigned int num_inv) 362 { 363 unsigned int i; 364 365 if (MMU_MAJ_VER(data->version) < 5 || num_inv == 1) { 366 for (i = 0; i < num_inv; i++) { 367 writel((iova & SPAGE_MASK) | 1, 368 SYSMMU_REG(data, flush_entry)); 369 iova += SPAGE_SIZE; 370 } 371 } else { 372 writel(iova & SPAGE_MASK, SYSMMU_REG(data, flush_start)); 373 writel((iova & SPAGE_MASK) + (num_inv - 1) * SPAGE_SIZE, 374 SYSMMU_REG(data, flush_end)); 375 writel(0x1, SYSMMU_REG(data, flush_range)); 376 } 377 } 378 379 static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) 380 { 381 u32 pt_base; 382 383 if (MMU_MAJ_VER(data->version) < 5) 384 pt_base = pgd; 385 else 386 pt_base = pgd >> SPAGE_ORDER; 387 388 writel(pt_base, SYSMMU_REG(data, pt_base)); 389 __sysmmu_tlb_invalidate(data); 390 } 391 392 static void __sysmmu_enable_clocks(struct sysmmu_drvdata *data) 393 { 394 BUG_ON(clk_prepare_enable(data->clk_master)); 395 BUG_ON(clk_prepare_enable(data->clk)); 396 BUG_ON(clk_prepare_enable(data->pclk)); 397 BUG_ON(clk_prepare_enable(data->aclk)); 398 } 399 400 static void __sysmmu_disable_clocks(struct sysmmu_drvdata *data) 401 { 402 clk_disable_unprepare(data->aclk); 403 clk_disable_unprepare(data->pclk); 404 clk_disable_unprepare(data->clk); 405 clk_disable_unprepare(data->clk_master); 406 } 407 408 static bool __sysmmu_has_capa1(struct sysmmu_drvdata *data) 409 { 410 u32 capa0 = readl(data->sfrbase + REG_V7_CAPA0); 411 412 return capa0 & CAPA0_CAPA1_EXIST; 413 } 414 415 static void __sysmmu_get_vcr(struct sysmmu_drvdata *data) 416 { 417 u32 capa1 = readl(data->sfrbase + REG_V7_CAPA1); 418 419 data->has_vcr = capa1 & CAPA1_VCR_ENABLED; 420 } 421 422 static void __sysmmu_get_version(struct sysmmu_drvdata *data) 423 { 424 u32 ver; 425 426 __sysmmu_enable_clocks(data); 427 428 ver = readl(data->sfrbase + REG_MMU_VERSION); 429 430 /* controllers on some SoCs don't report proper version */ 431 if (ver == 0x80000001u) 432 data->version = MAKE_MMU_VER(1, 0); 433 else 434 data->version = MMU_RAW_VER(ver); 435 436 dev_dbg(data->sysmmu, "hardware version: %d.%d\n", 437 MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); 438 439 if (MMU_MAJ_VER(data->version) < 5) { 440 data->variant = &sysmmu_v1_variant; 441 } else if (MMU_MAJ_VER(data->version) < 7) { 442 data->variant = &sysmmu_v5_variant; 443 } else { 444 if (__sysmmu_has_capa1(data)) 445 __sysmmu_get_vcr(data); 446 if (data->has_vcr) 447 data->variant = &sysmmu_v7_vm_variant; 448 else 449 data->variant = &sysmmu_v5_variant; 450 } 451 452 __sysmmu_disable_clocks(data); 453 } 454 455 static void show_fault_information(struct sysmmu_drvdata *data, 456 const struct sysmmu_fault_info *finfo, 457 sysmmu_iova_t fault_addr) 458 { 459 sysmmu_pte_t *ent; 460 461 dev_err(data->sysmmu, "%s: %s FAULT occurred at %#x\n", 462 dev_name(data->master), finfo->name, fault_addr); 463 dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); 464 ent = section_entry(phys_to_virt(data->pgtable), fault_addr); 465 dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent); 466 if (lv1ent_page(ent)) { 467 ent = page_entry(ent, fault_addr); 468 dev_dbg(data->sysmmu, "\t Lv2 entry: %#x\n", *ent); 469 } 470 } 471 472 static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) 473 { 474 /* SYSMMU is in blocked state when interrupt occurred. */ 475 struct sysmmu_drvdata *data = dev_id; 476 const struct sysmmu_fault_info *finfo; 477 unsigned int i, n, itype; 478 sysmmu_iova_t fault_addr; 479 int ret = -ENOSYS; 480 481 WARN_ON(!data->active); 482 483 if (MMU_MAJ_VER(data->version) < 5) { 484 finfo = sysmmu_faults; 485 n = ARRAY_SIZE(sysmmu_faults); 486 } else { 487 finfo = sysmmu_v5_faults; 488 n = ARRAY_SIZE(sysmmu_v5_faults); 489 } 490 491 spin_lock(&data->lock); 492 493 clk_enable(data->clk_master); 494 495 itype = __ffs(readl(SYSMMU_REG(data, int_status))); 496 for (i = 0; i < n; i++, finfo++) 497 if (finfo->bit == itype) 498 break; 499 /* unknown/unsupported fault */ 500 BUG_ON(i == n); 501 502 /* print debug message */ 503 fault_addr = readl(data->sfrbase + finfo->addr_reg); 504 show_fault_information(data, finfo, fault_addr); 505 506 if (data->domain) 507 ret = report_iommu_fault(&data->domain->domain, 508 data->master, fault_addr, finfo->type); 509 /* fault is not recovered by fault handler */ 510 BUG_ON(ret != 0); 511 512 writel(1 << itype, SYSMMU_REG(data, int_clear)); 513 514 sysmmu_unblock(data); 515 516 clk_disable(data->clk_master); 517 518 spin_unlock(&data->lock); 519 520 return IRQ_HANDLED; 521 } 522 523 static void __sysmmu_disable(struct sysmmu_drvdata *data) 524 { 525 unsigned long flags; 526 527 clk_enable(data->clk_master); 528 529 spin_lock_irqsave(&data->lock, flags); 530 writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); 531 writel(0, data->sfrbase + REG_MMU_CFG); 532 data->active = false; 533 spin_unlock_irqrestore(&data->lock, flags); 534 535 __sysmmu_disable_clocks(data); 536 } 537 538 static void __sysmmu_init_config(struct sysmmu_drvdata *data) 539 { 540 unsigned int cfg; 541 542 if (data->version <= MAKE_MMU_VER(3, 1)) 543 cfg = CFG_LRU | CFG_QOS(15); 544 else if (data->version <= MAKE_MMU_VER(3, 2)) 545 cfg = CFG_LRU | CFG_QOS(15) | CFG_FLPDCACHE | CFG_SYSSEL; 546 else 547 cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN; 548 549 cfg |= CFG_EAP; /* enable access protection bits check */ 550 551 writel(cfg, data->sfrbase + REG_MMU_CFG); 552 } 553 554 static void __sysmmu_enable_vid(struct sysmmu_drvdata *data) 555 { 556 u32 ctrl; 557 558 if (MMU_MAJ_VER(data->version) < 7 || !data->has_vcr) 559 return; 560 561 ctrl = readl(data->sfrbase + REG_V7_CTRL_VM); 562 ctrl |= CTRL_VM_ENABLE | CTRL_VM_FAULT_MODE_STALL; 563 writel(ctrl, data->sfrbase + REG_V7_CTRL_VM); 564 } 565 566 static void __sysmmu_enable(struct sysmmu_drvdata *data) 567 { 568 unsigned long flags; 569 570 __sysmmu_enable_clocks(data); 571 572 spin_lock_irqsave(&data->lock, flags); 573 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); 574 __sysmmu_init_config(data); 575 __sysmmu_set_ptbase(data, data->pgtable); 576 __sysmmu_enable_vid(data); 577 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); 578 data->active = true; 579 spin_unlock_irqrestore(&data->lock, flags); 580 581 /* 582 * SYSMMU driver keeps master's clock enabled only for the short 583 * time, while accessing the registers. For performing address 584 * translation during DMA transaction it relies on the client 585 * driver to enable it. 586 */ 587 clk_disable(data->clk_master); 588 } 589 590 static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, 591 sysmmu_iova_t iova) 592 { 593 unsigned long flags; 594 595 spin_lock_irqsave(&data->lock, flags); 596 if (data->active && data->version >= MAKE_MMU_VER(3, 3)) { 597 clk_enable(data->clk_master); 598 if (sysmmu_block(data)) { 599 if (data->version >= MAKE_MMU_VER(5, 0)) 600 __sysmmu_tlb_invalidate(data); 601 else 602 __sysmmu_tlb_invalidate_entry(data, iova, 1); 603 sysmmu_unblock(data); 604 } 605 clk_disable(data->clk_master); 606 } 607 spin_unlock_irqrestore(&data->lock, flags); 608 } 609 610 static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, 611 sysmmu_iova_t iova, size_t size) 612 { 613 unsigned long flags; 614 615 spin_lock_irqsave(&data->lock, flags); 616 if (data->active) { 617 unsigned int num_inv = 1; 618 619 clk_enable(data->clk_master); 620 621 /* 622 * L2TLB invalidation required 623 * 4KB page: 1 invalidation 624 * 64KB page: 16 invalidations 625 * 1MB page: 64 invalidations 626 * because it is set-associative TLB 627 * with 8-way and 64 sets. 628 * 1MB page can be cached in one of all sets. 629 * 64KB page can be one of 16 consecutive sets. 630 */ 631 if (MMU_MAJ_VER(data->version) == 2) 632 num_inv = min_t(unsigned int, size / SPAGE_SIZE, 64); 633 634 if (sysmmu_block(data)) { 635 __sysmmu_tlb_invalidate_entry(data, iova, num_inv); 636 sysmmu_unblock(data); 637 } 638 clk_disable(data->clk_master); 639 } 640 spin_unlock_irqrestore(&data->lock, flags); 641 } 642 643 static const struct iommu_ops exynos_iommu_ops; 644 645 static int exynos_sysmmu_probe(struct platform_device *pdev) 646 { 647 int irq, ret; 648 struct device *dev = &pdev->dev; 649 struct sysmmu_drvdata *data; 650 struct resource *res; 651 652 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 653 if (!data) 654 return -ENOMEM; 655 656 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 657 data->sfrbase = devm_ioremap_resource(dev, res); 658 if (IS_ERR(data->sfrbase)) 659 return PTR_ERR(data->sfrbase); 660 661 irq = platform_get_irq(pdev, 0); 662 if (irq <= 0) 663 return irq; 664 665 ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0, 666 dev_name(dev), data); 667 if (ret) { 668 dev_err(dev, "Unabled to register handler of irq %d\n", irq); 669 return ret; 670 } 671 672 data->clk = devm_clk_get(dev, "sysmmu"); 673 if (PTR_ERR(data->clk) == -ENOENT) 674 data->clk = NULL; 675 else if (IS_ERR(data->clk)) 676 return PTR_ERR(data->clk); 677 678 data->aclk = devm_clk_get(dev, "aclk"); 679 if (PTR_ERR(data->aclk) == -ENOENT) 680 data->aclk = NULL; 681 else if (IS_ERR(data->aclk)) 682 return PTR_ERR(data->aclk); 683 684 data->pclk = devm_clk_get(dev, "pclk"); 685 if (PTR_ERR(data->pclk) == -ENOENT) 686 data->pclk = NULL; 687 else if (IS_ERR(data->pclk)) 688 return PTR_ERR(data->pclk); 689 690 if (!data->clk && (!data->aclk || !data->pclk)) { 691 dev_err(dev, "Failed to get device clock(s)!\n"); 692 return -ENOSYS; 693 } 694 695 data->clk_master = devm_clk_get(dev, "master"); 696 if (PTR_ERR(data->clk_master) == -ENOENT) 697 data->clk_master = NULL; 698 else if (IS_ERR(data->clk_master)) 699 return PTR_ERR(data->clk_master); 700 701 data->sysmmu = dev; 702 spin_lock_init(&data->lock); 703 704 __sysmmu_get_version(data); 705 706 ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL, 707 dev_name(data->sysmmu)); 708 if (ret) 709 return ret; 710 711 ret = iommu_device_register(&data->iommu, &exynos_iommu_ops, dev); 712 if (ret) 713 goto err_iommu_register; 714 715 platform_set_drvdata(pdev, data); 716 717 if (PG_ENT_SHIFT < 0) { 718 if (MMU_MAJ_VER(data->version) < 5) { 719 PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT; 720 LV1_PROT = SYSMMU_LV1_PROT; 721 LV2_PROT = SYSMMU_LV2_PROT; 722 } else { 723 PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT; 724 LV1_PROT = SYSMMU_V5_LV1_PROT; 725 LV2_PROT = SYSMMU_V5_LV2_PROT; 726 } 727 } 728 729 if (MMU_MAJ_VER(data->version) >= 5) { 730 ret = dma_set_mask(dev, DMA_BIT_MASK(36)); 731 if (ret) { 732 dev_err(dev, "Unable to set DMA mask: %d\n", ret); 733 goto err_dma_set_mask; 734 } 735 } 736 737 /* 738 * use the first registered sysmmu device for performing 739 * dma mapping operations on iommu page tables (cpu cache flush) 740 */ 741 if (!dma_dev) 742 dma_dev = &pdev->dev; 743 744 pm_runtime_enable(dev); 745 746 return 0; 747 748 err_dma_set_mask: 749 iommu_device_unregister(&data->iommu); 750 err_iommu_register: 751 iommu_device_sysfs_remove(&data->iommu); 752 return ret; 753 } 754 755 static int __maybe_unused exynos_sysmmu_suspend(struct device *dev) 756 { 757 struct sysmmu_drvdata *data = dev_get_drvdata(dev); 758 struct device *master = data->master; 759 760 if (master) { 761 struct exynos_iommu_owner *owner = dev_iommu_priv_get(master); 762 763 mutex_lock(&owner->rpm_lock); 764 if (data->domain) { 765 dev_dbg(data->sysmmu, "saving state\n"); 766 __sysmmu_disable(data); 767 } 768 mutex_unlock(&owner->rpm_lock); 769 } 770 return 0; 771 } 772 773 static int __maybe_unused exynos_sysmmu_resume(struct device *dev) 774 { 775 struct sysmmu_drvdata *data = dev_get_drvdata(dev); 776 struct device *master = data->master; 777 778 if (master) { 779 struct exynos_iommu_owner *owner = dev_iommu_priv_get(master); 780 781 mutex_lock(&owner->rpm_lock); 782 if (data->domain) { 783 dev_dbg(data->sysmmu, "restoring state\n"); 784 __sysmmu_enable(data); 785 } 786 mutex_unlock(&owner->rpm_lock); 787 } 788 return 0; 789 } 790 791 static const struct dev_pm_ops sysmmu_pm_ops = { 792 SET_RUNTIME_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume, NULL) 793 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 794 pm_runtime_force_resume) 795 }; 796 797 static const struct of_device_id sysmmu_of_match[] = { 798 { .compatible = "samsung,exynos-sysmmu", }, 799 { }, 800 }; 801 802 static struct platform_driver exynos_sysmmu_driver __refdata = { 803 .probe = exynos_sysmmu_probe, 804 .driver = { 805 .name = "exynos-sysmmu", 806 .of_match_table = sysmmu_of_match, 807 .pm = &sysmmu_pm_ops, 808 .suppress_bind_attrs = true, 809 } 810 }; 811 812 static inline void exynos_iommu_set_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) 813 { 814 dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), 815 DMA_TO_DEVICE); 816 *ent = cpu_to_le32(val); 817 dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent), 818 DMA_TO_DEVICE); 819 } 820 821 static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) 822 { 823 struct exynos_iommu_domain *domain; 824 dma_addr_t handle; 825 int i; 826 827 /* Check if correct PTE offsets are initialized */ 828 BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev); 829 830 if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED) 831 return NULL; 832 833 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 834 if (!domain) 835 return NULL; 836 837 domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); 838 if (!domain->pgtable) 839 goto err_pgtable; 840 841 domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); 842 if (!domain->lv2entcnt) 843 goto err_counter; 844 845 /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */ 846 for (i = 0; i < NUM_LV1ENTRIES; i++) 847 domain->pgtable[i] = ZERO_LV2LINK; 848 849 handle = dma_map_single(dma_dev, domain->pgtable, LV1TABLE_SIZE, 850 DMA_TO_DEVICE); 851 /* For mapping page table entries we rely on dma == phys */ 852 BUG_ON(handle != virt_to_phys(domain->pgtable)); 853 if (dma_mapping_error(dma_dev, handle)) 854 goto err_lv2ent; 855 856 spin_lock_init(&domain->lock); 857 spin_lock_init(&domain->pgtablelock); 858 INIT_LIST_HEAD(&domain->clients); 859 860 domain->domain.geometry.aperture_start = 0; 861 domain->domain.geometry.aperture_end = ~0UL; 862 domain->domain.geometry.force_aperture = true; 863 864 return &domain->domain; 865 866 err_lv2ent: 867 free_pages((unsigned long)domain->lv2entcnt, 1); 868 err_counter: 869 free_pages((unsigned long)domain->pgtable, 2); 870 err_pgtable: 871 kfree(domain); 872 return NULL; 873 } 874 875 static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) 876 { 877 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 878 struct sysmmu_drvdata *data, *next; 879 unsigned long flags; 880 int i; 881 882 WARN_ON(!list_empty(&domain->clients)); 883 884 spin_lock_irqsave(&domain->lock, flags); 885 886 list_for_each_entry_safe(data, next, &domain->clients, domain_node) { 887 spin_lock(&data->lock); 888 __sysmmu_disable(data); 889 data->pgtable = 0; 890 data->domain = NULL; 891 list_del_init(&data->domain_node); 892 spin_unlock(&data->lock); 893 } 894 895 spin_unlock_irqrestore(&domain->lock, flags); 896 897 dma_unmap_single(dma_dev, virt_to_phys(domain->pgtable), LV1TABLE_SIZE, 898 DMA_TO_DEVICE); 899 900 for (i = 0; i < NUM_LV1ENTRIES; i++) 901 if (lv1ent_page(domain->pgtable + i)) { 902 phys_addr_t base = lv2table_base(domain->pgtable + i); 903 904 dma_unmap_single(dma_dev, base, LV2TABLE_SIZE, 905 DMA_TO_DEVICE); 906 kmem_cache_free(lv2table_kmem_cache, 907 phys_to_virt(base)); 908 } 909 910 free_pages((unsigned long)domain->pgtable, 2); 911 free_pages((unsigned long)domain->lv2entcnt, 1); 912 kfree(domain); 913 } 914 915 static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, 916 struct device *dev) 917 { 918 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 919 struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); 920 phys_addr_t pagetable = virt_to_phys(domain->pgtable); 921 struct sysmmu_drvdata *data, *next; 922 unsigned long flags; 923 924 if (!has_sysmmu(dev) || owner->domain != iommu_domain) 925 return; 926 927 mutex_lock(&owner->rpm_lock); 928 929 list_for_each_entry(data, &owner->controllers, owner_node) { 930 pm_runtime_get_noresume(data->sysmmu); 931 if (pm_runtime_active(data->sysmmu)) 932 __sysmmu_disable(data); 933 pm_runtime_put(data->sysmmu); 934 } 935 936 spin_lock_irqsave(&domain->lock, flags); 937 list_for_each_entry_safe(data, next, &domain->clients, domain_node) { 938 spin_lock(&data->lock); 939 data->pgtable = 0; 940 data->domain = NULL; 941 list_del_init(&data->domain_node); 942 spin_unlock(&data->lock); 943 } 944 owner->domain = NULL; 945 spin_unlock_irqrestore(&domain->lock, flags); 946 947 mutex_unlock(&owner->rpm_lock); 948 949 dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", __func__, 950 &pagetable); 951 } 952 953 static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, 954 struct device *dev) 955 { 956 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 957 struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); 958 struct sysmmu_drvdata *data; 959 phys_addr_t pagetable = virt_to_phys(domain->pgtable); 960 unsigned long flags; 961 962 if (!has_sysmmu(dev)) 963 return -ENODEV; 964 965 if (owner->domain) 966 exynos_iommu_detach_device(owner->domain, dev); 967 968 mutex_lock(&owner->rpm_lock); 969 970 spin_lock_irqsave(&domain->lock, flags); 971 list_for_each_entry(data, &owner->controllers, owner_node) { 972 spin_lock(&data->lock); 973 data->pgtable = pagetable; 974 data->domain = domain; 975 list_add_tail(&data->domain_node, &domain->clients); 976 spin_unlock(&data->lock); 977 } 978 owner->domain = iommu_domain; 979 spin_unlock_irqrestore(&domain->lock, flags); 980 981 list_for_each_entry(data, &owner->controllers, owner_node) { 982 pm_runtime_get_noresume(data->sysmmu); 983 if (pm_runtime_active(data->sysmmu)) 984 __sysmmu_enable(data); 985 pm_runtime_put(data->sysmmu); 986 } 987 988 mutex_unlock(&owner->rpm_lock); 989 990 dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa\n", __func__, 991 &pagetable); 992 993 return 0; 994 } 995 996 static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, 997 sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) 998 { 999 if (lv1ent_section(sent)) { 1000 WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova); 1001 return ERR_PTR(-EADDRINUSE); 1002 } 1003 1004 if (lv1ent_fault(sent)) { 1005 dma_addr_t handle; 1006 sysmmu_pte_t *pent; 1007 bool need_flush_flpd_cache = lv1ent_zero(sent); 1008 1009 pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); 1010 BUG_ON((uintptr_t)pent & (LV2TABLE_SIZE - 1)); 1011 if (!pent) 1012 return ERR_PTR(-ENOMEM); 1013 1014 exynos_iommu_set_pte(sent, mk_lv1ent_page(virt_to_phys(pent))); 1015 kmemleak_ignore(pent); 1016 *pgcounter = NUM_LV2ENTRIES; 1017 handle = dma_map_single(dma_dev, pent, LV2TABLE_SIZE, 1018 DMA_TO_DEVICE); 1019 if (dma_mapping_error(dma_dev, handle)) { 1020 kmem_cache_free(lv2table_kmem_cache, pent); 1021 return ERR_PTR(-EADDRINUSE); 1022 } 1023 1024 /* 1025 * If pre-fetched SLPD is a faulty SLPD in zero_l2_table, 1026 * FLPD cache may cache the address of zero_l2_table. This 1027 * function replaces the zero_l2_table with new L2 page table 1028 * to write valid mappings. 1029 * Accessing the valid area may cause page fault since FLPD 1030 * cache may still cache zero_l2_table for the valid area 1031 * instead of new L2 page table that has the mapping 1032 * information of the valid area. 1033 * Thus any replacement of zero_l2_table with other valid L2 1034 * page table must involve FLPD cache invalidation for System 1035 * MMU v3.3. 1036 * FLPD cache invalidation is performed with TLB invalidation 1037 * by VPN without blocking. It is safe to invalidate TLB without 1038 * blocking because the target address of TLB invalidation is 1039 * not currently mapped. 1040 */ 1041 if (need_flush_flpd_cache) { 1042 struct sysmmu_drvdata *data; 1043 1044 spin_lock(&domain->lock); 1045 list_for_each_entry(data, &domain->clients, domain_node) 1046 sysmmu_tlb_invalidate_flpdcache(data, iova); 1047 spin_unlock(&domain->lock); 1048 } 1049 } 1050 1051 return page_entry(sent, iova); 1052 } 1053 1054 static int lv1set_section(struct exynos_iommu_domain *domain, 1055 sysmmu_pte_t *sent, sysmmu_iova_t iova, 1056 phys_addr_t paddr, int prot, short *pgcnt) 1057 { 1058 if (lv1ent_section(sent)) { 1059 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", 1060 iova); 1061 return -EADDRINUSE; 1062 } 1063 1064 if (lv1ent_page(sent)) { 1065 if (*pgcnt != NUM_LV2ENTRIES) { 1066 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", 1067 iova); 1068 return -EADDRINUSE; 1069 } 1070 1071 kmem_cache_free(lv2table_kmem_cache, page_entry(sent, 0)); 1072 *pgcnt = 0; 1073 } 1074 1075 exynos_iommu_set_pte(sent, mk_lv1ent_sect(paddr, prot)); 1076 1077 spin_lock(&domain->lock); 1078 if (lv1ent_page_zero(sent)) { 1079 struct sysmmu_drvdata *data; 1080 /* 1081 * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD 1082 * entry by speculative prefetch of SLPD which has no mapping. 1083 */ 1084 list_for_each_entry(data, &domain->clients, domain_node) 1085 sysmmu_tlb_invalidate_flpdcache(data, iova); 1086 } 1087 spin_unlock(&domain->lock); 1088 1089 return 0; 1090 } 1091 1092 static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, 1093 int prot, short *pgcnt) 1094 { 1095 if (size == SPAGE_SIZE) { 1096 if (WARN_ON(!lv2ent_fault(pent))) 1097 return -EADDRINUSE; 1098 1099 exynos_iommu_set_pte(pent, mk_lv2ent_spage(paddr, prot)); 1100 *pgcnt -= 1; 1101 } else { /* size == LPAGE_SIZE */ 1102 int i; 1103 dma_addr_t pent_base = virt_to_phys(pent); 1104 1105 dma_sync_single_for_cpu(dma_dev, pent_base, 1106 sizeof(*pent) * SPAGES_PER_LPAGE, 1107 DMA_TO_DEVICE); 1108 for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { 1109 if (WARN_ON(!lv2ent_fault(pent))) { 1110 if (i > 0) 1111 memset(pent - i, 0, sizeof(*pent) * i); 1112 return -EADDRINUSE; 1113 } 1114 1115 *pent = mk_lv2ent_lpage(paddr, prot); 1116 } 1117 dma_sync_single_for_device(dma_dev, pent_base, 1118 sizeof(*pent) * SPAGES_PER_LPAGE, 1119 DMA_TO_DEVICE); 1120 *pgcnt -= SPAGES_PER_LPAGE; 1121 } 1122 1123 return 0; 1124 } 1125 1126 /* 1127 * *CAUTION* to the I/O virtual memory managers that support exynos-iommu: 1128 * 1129 * System MMU v3.x has advanced logic to improve address translation 1130 * performance with caching more page table entries by a page table walk. 1131 * However, the logic has a bug that while caching faulty page table entries, 1132 * System MMU reports page fault if the cached fault entry is hit even though 1133 * the fault entry is updated to a valid entry after the entry is cached. 1134 * To prevent caching faulty page table entries which may be updated to valid 1135 * entries later, the virtual memory manager should care about the workaround 1136 * for the problem. The following describes the workaround. 1137 * 1138 * Any two consecutive I/O virtual address regions must have a hole of 128KiB 1139 * at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug). 1140 * 1141 * Precisely, any start address of I/O virtual region must be aligned with 1142 * the following sizes for System MMU v3.1 and v3.2. 1143 * System MMU v3.1: 128KiB 1144 * System MMU v3.2: 256KiB 1145 * 1146 * Because System MMU v3.3 caches page table entries more aggressively, it needs 1147 * more workarounds. 1148 * - Any two consecutive I/O virtual regions must have a hole of size larger 1149 * than or equal to 128KiB. 1150 * - Start address of an I/O virtual region must be aligned by 128KiB. 1151 */ 1152 static int exynos_iommu_map(struct iommu_domain *iommu_domain, 1153 unsigned long l_iova, phys_addr_t paddr, size_t size, 1154 int prot, gfp_t gfp) 1155 { 1156 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 1157 sysmmu_pte_t *entry; 1158 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; 1159 unsigned long flags; 1160 int ret = -ENOMEM; 1161 1162 BUG_ON(domain->pgtable == NULL); 1163 prot &= SYSMMU_SUPPORTED_PROT_BITS; 1164 1165 spin_lock_irqsave(&domain->pgtablelock, flags); 1166 1167 entry = section_entry(domain->pgtable, iova); 1168 1169 if (size == SECT_SIZE) { 1170 ret = lv1set_section(domain, entry, iova, paddr, prot, 1171 &domain->lv2entcnt[lv1ent_offset(iova)]); 1172 } else { 1173 sysmmu_pte_t *pent; 1174 1175 pent = alloc_lv2entry(domain, entry, iova, 1176 &domain->lv2entcnt[lv1ent_offset(iova)]); 1177 1178 if (IS_ERR(pent)) 1179 ret = PTR_ERR(pent); 1180 else 1181 ret = lv2set_page(pent, paddr, size, prot, 1182 &domain->lv2entcnt[lv1ent_offset(iova)]); 1183 } 1184 1185 if (ret) 1186 pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n", 1187 __func__, ret, size, iova); 1188 1189 spin_unlock_irqrestore(&domain->pgtablelock, flags); 1190 1191 return ret; 1192 } 1193 1194 static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain, 1195 sysmmu_iova_t iova, size_t size) 1196 { 1197 struct sysmmu_drvdata *data; 1198 unsigned long flags; 1199 1200 spin_lock_irqsave(&domain->lock, flags); 1201 1202 list_for_each_entry(data, &domain->clients, domain_node) 1203 sysmmu_tlb_invalidate_entry(data, iova, size); 1204 1205 spin_unlock_irqrestore(&domain->lock, flags); 1206 } 1207 1208 static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, 1209 unsigned long l_iova, size_t size, 1210 struct iommu_iotlb_gather *gather) 1211 { 1212 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 1213 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; 1214 sysmmu_pte_t *ent; 1215 size_t err_pgsize; 1216 unsigned long flags; 1217 1218 BUG_ON(domain->pgtable == NULL); 1219 1220 spin_lock_irqsave(&domain->pgtablelock, flags); 1221 1222 ent = section_entry(domain->pgtable, iova); 1223 1224 if (lv1ent_section(ent)) { 1225 if (WARN_ON(size < SECT_SIZE)) { 1226 err_pgsize = SECT_SIZE; 1227 goto err; 1228 } 1229 1230 /* workaround for h/w bug in System MMU v3.3 */ 1231 exynos_iommu_set_pte(ent, ZERO_LV2LINK); 1232 size = SECT_SIZE; 1233 goto done; 1234 } 1235 1236 if (unlikely(lv1ent_fault(ent))) { 1237 if (size > SECT_SIZE) 1238 size = SECT_SIZE; 1239 goto done; 1240 } 1241 1242 /* lv1ent_page(sent) == true here */ 1243 1244 ent = page_entry(ent, iova); 1245 1246 if (unlikely(lv2ent_fault(ent))) { 1247 size = SPAGE_SIZE; 1248 goto done; 1249 } 1250 1251 if (lv2ent_small(ent)) { 1252 exynos_iommu_set_pte(ent, 0); 1253 size = SPAGE_SIZE; 1254 domain->lv2entcnt[lv1ent_offset(iova)] += 1; 1255 goto done; 1256 } 1257 1258 /* lv1ent_large(ent) == true here */ 1259 if (WARN_ON(size < LPAGE_SIZE)) { 1260 err_pgsize = LPAGE_SIZE; 1261 goto err; 1262 } 1263 1264 dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), 1265 sizeof(*ent) * SPAGES_PER_LPAGE, 1266 DMA_TO_DEVICE); 1267 memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); 1268 dma_sync_single_for_device(dma_dev, virt_to_phys(ent), 1269 sizeof(*ent) * SPAGES_PER_LPAGE, 1270 DMA_TO_DEVICE); 1271 size = LPAGE_SIZE; 1272 domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; 1273 done: 1274 spin_unlock_irqrestore(&domain->pgtablelock, flags); 1275 1276 exynos_iommu_tlb_invalidate_entry(domain, iova, size); 1277 1278 return size; 1279 err: 1280 spin_unlock_irqrestore(&domain->pgtablelock, flags); 1281 1282 pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n", 1283 __func__, size, iova, err_pgsize); 1284 1285 return 0; 1286 } 1287 1288 static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, 1289 dma_addr_t iova) 1290 { 1291 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 1292 sysmmu_pte_t *entry; 1293 unsigned long flags; 1294 phys_addr_t phys = 0; 1295 1296 spin_lock_irqsave(&domain->pgtablelock, flags); 1297 1298 entry = section_entry(domain->pgtable, iova); 1299 1300 if (lv1ent_section(entry)) { 1301 phys = section_phys(entry) + section_offs(iova); 1302 } else if (lv1ent_page(entry)) { 1303 entry = page_entry(entry, iova); 1304 1305 if (lv2ent_large(entry)) 1306 phys = lpage_phys(entry) + lpage_offs(iova); 1307 else if (lv2ent_small(entry)) 1308 phys = spage_phys(entry) + spage_offs(iova); 1309 } 1310 1311 spin_unlock_irqrestore(&domain->pgtablelock, flags); 1312 1313 return phys; 1314 } 1315 1316 static struct iommu_device *exynos_iommu_probe_device(struct device *dev) 1317 { 1318 struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); 1319 struct sysmmu_drvdata *data; 1320 1321 if (!has_sysmmu(dev)) 1322 return ERR_PTR(-ENODEV); 1323 1324 list_for_each_entry(data, &owner->controllers, owner_node) { 1325 /* 1326 * SYSMMU will be runtime activated via device link 1327 * (dependency) to its master device, so there are no 1328 * direct calls to pm_runtime_get/put in this driver. 1329 */ 1330 data->link = device_link_add(dev, data->sysmmu, 1331 DL_FLAG_STATELESS | 1332 DL_FLAG_PM_RUNTIME); 1333 } 1334 1335 /* There is always at least one entry, see exynos_iommu_of_xlate() */ 1336 data = list_first_entry(&owner->controllers, 1337 struct sysmmu_drvdata, owner_node); 1338 1339 return &data->iommu; 1340 } 1341 1342 static void exynos_iommu_release_device(struct device *dev) 1343 { 1344 struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); 1345 struct sysmmu_drvdata *data; 1346 1347 if (owner->domain) { 1348 struct iommu_group *group = iommu_group_get(dev); 1349 1350 if (group) { 1351 WARN_ON(owner->domain != 1352 iommu_group_default_domain(group)); 1353 exynos_iommu_detach_device(owner->domain, dev); 1354 iommu_group_put(group); 1355 } 1356 } 1357 1358 list_for_each_entry(data, &owner->controllers, owner_node) 1359 device_link_del(data->link); 1360 } 1361 1362 static int exynos_iommu_of_xlate(struct device *dev, 1363 struct of_phandle_args *spec) 1364 { 1365 struct platform_device *sysmmu = of_find_device_by_node(spec->np); 1366 struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); 1367 struct sysmmu_drvdata *data, *entry; 1368 1369 if (!sysmmu) 1370 return -ENODEV; 1371 1372 data = platform_get_drvdata(sysmmu); 1373 if (!data) { 1374 put_device(&sysmmu->dev); 1375 return -ENODEV; 1376 } 1377 1378 if (!owner) { 1379 owner = kzalloc(sizeof(*owner), GFP_KERNEL); 1380 if (!owner) { 1381 put_device(&sysmmu->dev); 1382 return -ENOMEM; 1383 } 1384 1385 INIT_LIST_HEAD(&owner->controllers); 1386 mutex_init(&owner->rpm_lock); 1387 dev_iommu_priv_set(dev, owner); 1388 } 1389 1390 list_for_each_entry(entry, &owner->controllers, owner_node) 1391 if (entry == data) 1392 return 0; 1393 1394 list_add_tail(&data->owner_node, &owner->controllers); 1395 data->master = dev; 1396 1397 return 0; 1398 } 1399 1400 static const struct iommu_ops exynos_iommu_ops = { 1401 .domain_alloc = exynos_iommu_domain_alloc, 1402 .device_group = generic_device_group, 1403 .probe_device = exynos_iommu_probe_device, 1404 .release_device = exynos_iommu_release_device, 1405 .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, 1406 .of_xlate = exynos_iommu_of_xlate, 1407 .default_domain_ops = &(const struct iommu_domain_ops) { 1408 .attach_dev = exynos_iommu_attach_device, 1409 .detach_dev = exynos_iommu_detach_device, 1410 .map = exynos_iommu_map, 1411 .unmap = exynos_iommu_unmap, 1412 .iova_to_phys = exynos_iommu_iova_to_phys, 1413 .free = exynos_iommu_domain_free, 1414 } 1415 }; 1416 1417 static int __init exynos_iommu_init(void) 1418 { 1419 struct device_node *np; 1420 int ret; 1421 1422 np = of_find_matching_node(NULL, sysmmu_of_match); 1423 if (!np) 1424 return 0; 1425 1426 of_node_put(np); 1427 1428 lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table", 1429 LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL); 1430 if (!lv2table_kmem_cache) { 1431 pr_err("%s: Failed to create kmem cache\n", __func__); 1432 return -ENOMEM; 1433 } 1434 1435 ret = platform_driver_register(&exynos_sysmmu_driver); 1436 if (ret) { 1437 pr_err("%s: Failed to register driver\n", __func__); 1438 goto err_reg_driver; 1439 } 1440 1441 zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL); 1442 if (zero_lv2_table == NULL) { 1443 pr_err("%s: Failed to allocate zero level2 page table\n", 1444 __func__); 1445 ret = -ENOMEM; 1446 goto err_zero_lv2; 1447 } 1448 1449 return 0; 1450 err_zero_lv2: 1451 platform_driver_unregister(&exynos_sysmmu_driver); 1452 err_reg_driver: 1453 kmem_cache_destroy(lv2table_kmem_cache); 1454 return ret; 1455 } 1456 core_initcall(exynos_iommu_init); 1457