1 /* 2 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. 3 * http://www.samsung.com 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 */ 9 10 #ifdef CONFIG_EXYNOS_IOMMU_DEBUG 11 #define DEBUG 12 #endif 13 14 #include <linux/clk.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/err.h> 17 #include <linux/io.h> 18 #include <linux/iommu.h> 19 #include <linux/interrupt.h> 20 #include <linux/kmemleak.h> 21 #include <linux/list.h> 22 #include <linux/of.h> 23 #include <linux/of_iommu.h> 24 #include <linux/of_platform.h> 25 #include <linux/platform_device.h> 26 #include <linux/pm_runtime.h> 27 #include <linux/slab.h> 28 #include <linux/dma-iommu.h> 29 30 typedef u32 sysmmu_iova_t; 31 typedef u32 sysmmu_pte_t; 32 33 /* We do not consider super section mapping (16MB) */ 34 #define SECT_ORDER 20 35 #define LPAGE_ORDER 16 36 #define SPAGE_ORDER 12 37 38 #define SECT_SIZE (1 << SECT_ORDER) 39 #define LPAGE_SIZE (1 << LPAGE_ORDER) 40 #define SPAGE_SIZE (1 << SPAGE_ORDER) 41 42 #define SECT_MASK (~(SECT_SIZE - 1)) 43 #define LPAGE_MASK (~(LPAGE_SIZE - 1)) 44 #define SPAGE_MASK (~(SPAGE_SIZE - 1)) 45 46 #define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \ 47 ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3)) 48 #define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK) 49 #define lv1ent_page_zero(sent) ((*(sent) & 3) == 1) 50 #define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \ 51 ((*(sent) & 3) == 1)) 52 #define lv1ent_section(sent) ((*(sent) & 3) == 2) 53 54 #define lv2ent_fault(pent) ((*(pent) & 3) == 0) 55 #define lv2ent_small(pent) ((*(pent) & 2) == 2) 56 #define lv2ent_large(pent) ((*(pent) & 3) == 1) 57 58 /* 59 * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces 60 * v5.0 introduced support for 36bit physical address space by shifting 61 * all page entry values by 4 bits. 62 * All SYSMMU controllers in the system support the address spaces of the same 63 * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper 64 * value (0 or 4). 65 */ 66 static short PG_ENT_SHIFT = -1; 67 #define SYSMMU_PG_ENT_SHIFT 0 68 #define SYSMMU_V5_PG_ENT_SHIFT 4 69 70 static const sysmmu_pte_t *LV1_PROT; 71 static const sysmmu_pte_t SYSMMU_LV1_PROT[] = { 72 ((0 << 15) | (0 << 10)), /* no access */ 73 ((1 << 15) | (1 << 10)), /* IOMMU_READ only */ 74 ((0 << 15) | (1 << 10)), /* IOMMU_WRITE not supported, use read/write */ 75 ((0 << 15) | (1 << 10)), /* IOMMU_READ | IOMMU_WRITE */ 76 }; 77 static const sysmmu_pte_t SYSMMU_V5_LV1_PROT[] = { 78 (0 << 4), /* no access */ 79 (1 << 4), /* IOMMU_READ only */ 80 (2 << 4), /* IOMMU_WRITE only */ 81 (3 << 4), /* IOMMU_READ | IOMMU_WRITE */ 82 }; 83 84 static const sysmmu_pte_t *LV2_PROT; 85 static const sysmmu_pte_t SYSMMU_LV2_PROT[] = { 86 ((0 << 9) | (0 << 4)), /* no access */ 87 ((1 << 9) | (1 << 4)), /* IOMMU_READ only */ 88 ((0 << 9) | (1 << 4)), /* IOMMU_WRITE not supported, use read/write */ 89 ((0 << 9) | (1 << 4)), /* IOMMU_READ | IOMMU_WRITE */ 90 }; 91 static const sysmmu_pte_t SYSMMU_V5_LV2_PROT[] = { 92 (0 << 2), /* no access */ 93 (1 << 2), /* IOMMU_READ only */ 94 (2 << 2), /* IOMMU_WRITE only */ 95 (3 << 2), /* IOMMU_READ | IOMMU_WRITE */ 96 }; 97 98 #define SYSMMU_SUPPORTED_PROT_BITS (IOMMU_READ | IOMMU_WRITE) 99 100 #define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT) 101 #define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK) 102 #define section_offs(iova) (iova & (SECT_SIZE - 1)) 103 #define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK) 104 #define lpage_offs(iova) (iova & (LPAGE_SIZE - 1)) 105 #define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK) 106 #define spage_offs(iova) (iova & (SPAGE_SIZE - 1)) 107 108 #define NUM_LV1ENTRIES 4096 109 #define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE) 110 111 static u32 lv1ent_offset(sysmmu_iova_t iova) 112 { 113 return iova >> SECT_ORDER; 114 } 115 116 static u32 lv2ent_offset(sysmmu_iova_t iova) 117 { 118 return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1); 119 } 120 121 #define LV1TABLE_SIZE (NUM_LV1ENTRIES * sizeof(sysmmu_pte_t)) 122 #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t)) 123 124 #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) 125 #define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0)) 126 127 #define mk_lv1ent_sect(pa, prot) ((pa >> PG_ENT_SHIFT) | LV1_PROT[prot] | 2) 128 #define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1) 129 #define mk_lv2ent_lpage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 1) 130 #define mk_lv2ent_spage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 2) 131 132 #define CTRL_ENABLE 0x5 133 #define CTRL_BLOCK 0x7 134 #define CTRL_DISABLE 0x0 135 136 #define CFG_LRU 0x1 137 #define CFG_EAP (1 << 2) 138 #define CFG_QOS(n) ((n & 0xF) << 7) 139 #define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */ 140 #define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */ 141 #define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */ 142 143 /* common registers */ 144 #define REG_MMU_CTRL 0x000 145 #define REG_MMU_CFG 0x004 146 #define REG_MMU_STATUS 0x008 147 #define REG_MMU_VERSION 0x034 148 149 #define MMU_MAJ_VER(val) ((val) >> 7) 150 #define MMU_MIN_VER(val) ((val) & 0x7F) 151 #define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ 152 153 #define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) 154 155 /* v1.x - v3.x registers */ 156 #define REG_MMU_FLUSH 0x00C 157 #define REG_MMU_FLUSH_ENTRY 0x010 158 #define REG_PT_BASE_ADDR 0x014 159 #define REG_INT_STATUS 0x018 160 #define REG_INT_CLEAR 0x01C 161 162 #define REG_PAGE_FAULT_ADDR 0x024 163 #define REG_AW_FAULT_ADDR 0x028 164 #define REG_AR_FAULT_ADDR 0x02C 165 #define REG_DEFAULT_SLAVE_ADDR 0x030 166 167 /* v5.x registers */ 168 #define REG_V5_PT_BASE_PFN 0x00C 169 #define REG_V5_MMU_FLUSH_ALL 0x010 170 #define REG_V5_MMU_FLUSH_ENTRY 0x014 171 #define REG_V5_MMU_FLUSH_RANGE 0x018 172 #define REG_V5_MMU_FLUSH_START 0x020 173 #define REG_V5_MMU_FLUSH_END 0x024 174 #define REG_V5_INT_STATUS 0x060 175 #define REG_V5_INT_CLEAR 0x064 176 #define REG_V5_FAULT_AR_VA 0x070 177 #define REG_V5_FAULT_AW_VA 0x080 178 179 #define has_sysmmu(dev) (dev->archdata.iommu != NULL) 180 181 static struct device *dma_dev; 182 static struct kmem_cache *lv2table_kmem_cache; 183 static sysmmu_pte_t *zero_lv2_table; 184 #define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table)) 185 186 static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) 187 { 188 return pgtable + lv1ent_offset(iova); 189 } 190 191 static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova) 192 { 193 return (sysmmu_pte_t *)phys_to_virt( 194 lv2table_base(sent)) + lv2ent_offset(iova); 195 } 196 197 /* 198 * IOMMU fault information register 199 */ 200 struct sysmmu_fault_info { 201 unsigned int bit; /* bit number in STATUS register */ 202 unsigned short addr_reg; /* register to read VA fault address */ 203 const char *name; /* human readable fault name */ 204 unsigned int type; /* fault type for report_iommu_fault */ 205 }; 206 207 static const struct sysmmu_fault_info sysmmu_faults[] = { 208 { 0, REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, 209 { 1, REG_AR_FAULT_ADDR, "AR MULTI-HIT", IOMMU_FAULT_READ }, 210 { 2, REG_AW_FAULT_ADDR, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, 211 { 3, REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, 212 { 4, REG_AR_FAULT_ADDR, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, 213 { 5, REG_AR_FAULT_ADDR, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, 214 { 6, REG_AW_FAULT_ADDR, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, 215 { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, 216 }; 217 218 static const struct sysmmu_fault_info sysmmu_v5_faults[] = { 219 { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ }, 220 { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ }, 221 { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ }, 222 { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, 223 { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, 224 { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE }, 225 { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE }, 226 { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, 227 { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, 228 { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, 229 }; 230 231 /* 232 * This structure is attached to dev.archdata.iommu of the master device 233 * on device add, contains a list of SYSMMU controllers defined by device tree, 234 * which are bound to given master device. It is usually referenced by 'owner' 235 * pointer. 236 */ 237 struct exynos_iommu_owner { 238 struct list_head controllers; /* list of sysmmu_drvdata.owner_node */ 239 struct iommu_domain *domain; /* domain this device is attached */ 240 struct mutex rpm_lock; /* for runtime pm of all sysmmus */ 241 }; 242 243 /* 244 * This structure exynos specific generalization of struct iommu_domain. 245 * It contains list of SYSMMU controllers from all master devices, which has 246 * been attached to this domain and page tables of IO address space defined by 247 * it. It is usually referenced by 'domain' pointer. 248 */ 249 struct exynos_iommu_domain { 250 struct list_head clients; /* list of sysmmu_drvdata.domain_node */ 251 sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ 252 short *lv2entcnt; /* free lv2 entry counter for each section */ 253 spinlock_t lock; /* lock for modyfying list of clients */ 254 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ 255 struct iommu_domain domain; /* generic domain data structure */ 256 }; 257 258 /* 259 * This structure hold all data of a single SYSMMU controller, this includes 260 * hw resources like registers and clocks, pointers and list nodes to connect 261 * it to all other structures, internal state and parameters read from device 262 * tree. It is usually referenced by 'data' pointer. 263 */ 264 struct sysmmu_drvdata { 265 struct device *sysmmu; /* SYSMMU controller device */ 266 struct device *master; /* master device (owner) */ 267 struct device_link *link; /* runtime PM link to master */ 268 void __iomem *sfrbase; /* our registers */ 269 struct clk *clk; /* SYSMMU's clock */ 270 struct clk *aclk; /* SYSMMU's aclk clock */ 271 struct clk *pclk; /* SYSMMU's pclk clock */ 272 struct clk *clk_master; /* master's device clock */ 273 spinlock_t lock; /* lock for modyfying state */ 274 bool active; /* current status */ 275 struct exynos_iommu_domain *domain; /* domain we belong to */ 276 struct list_head domain_node; /* node for domain clients list */ 277 struct list_head owner_node; /* node for owner controllers list */ 278 phys_addr_t pgtable; /* assigned page table structure */ 279 unsigned int version; /* our version */ 280 281 struct iommu_device iommu; /* IOMMU core handle */ 282 }; 283 284 static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) 285 { 286 return container_of(dom, struct exynos_iommu_domain, domain); 287 } 288 289 static void sysmmu_unblock(struct sysmmu_drvdata *data) 290 { 291 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); 292 } 293 294 static bool sysmmu_block(struct sysmmu_drvdata *data) 295 { 296 int i = 120; 297 298 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); 299 while ((i > 0) && !(readl(data->sfrbase + REG_MMU_STATUS) & 1)) 300 --i; 301 302 if (!(readl(data->sfrbase + REG_MMU_STATUS) & 1)) { 303 sysmmu_unblock(data); 304 return false; 305 } 306 307 return true; 308 } 309 310 static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) 311 { 312 if (MMU_MAJ_VER(data->version) < 5) 313 writel(0x1, data->sfrbase + REG_MMU_FLUSH); 314 else 315 writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL); 316 } 317 318 static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, 319 sysmmu_iova_t iova, unsigned int num_inv) 320 { 321 unsigned int i; 322 323 if (MMU_MAJ_VER(data->version) < 5) { 324 for (i = 0; i < num_inv; i++) { 325 writel((iova & SPAGE_MASK) | 1, 326 data->sfrbase + REG_MMU_FLUSH_ENTRY); 327 iova += SPAGE_SIZE; 328 } 329 } else { 330 if (num_inv == 1) { 331 writel((iova & SPAGE_MASK) | 1, 332 data->sfrbase + REG_V5_MMU_FLUSH_ENTRY); 333 } else { 334 writel((iova & SPAGE_MASK), 335 data->sfrbase + REG_V5_MMU_FLUSH_START); 336 writel((iova & SPAGE_MASK) + (num_inv - 1) * SPAGE_SIZE, 337 data->sfrbase + REG_V5_MMU_FLUSH_END); 338 writel(1, data->sfrbase + REG_V5_MMU_FLUSH_RANGE); 339 } 340 } 341 } 342 343 static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) 344 { 345 if (MMU_MAJ_VER(data->version) < 5) 346 writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); 347 else 348 writel(pgd >> PAGE_SHIFT, 349 data->sfrbase + REG_V5_PT_BASE_PFN); 350 351 __sysmmu_tlb_invalidate(data); 352 } 353 354 static void __sysmmu_enable_clocks(struct sysmmu_drvdata *data) 355 { 356 BUG_ON(clk_prepare_enable(data->clk_master)); 357 BUG_ON(clk_prepare_enable(data->clk)); 358 BUG_ON(clk_prepare_enable(data->pclk)); 359 BUG_ON(clk_prepare_enable(data->aclk)); 360 } 361 362 static void __sysmmu_disable_clocks(struct sysmmu_drvdata *data) 363 { 364 clk_disable_unprepare(data->aclk); 365 clk_disable_unprepare(data->pclk); 366 clk_disable_unprepare(data->clk); 367 clk_disable_unprepare(data->clk_master); 368 } 369 370 static void __sysmmu_get_version(struct sysmmu_drvdata *data) 371 { 372 u32 ver; 373 374 __sysmmu_enable_clocks(data); 375 376 ver = readl(data->sfrbase + REG_MMU_VERSION); 377 378 /* controllers on some SoCs don't report proper version */ 379 if (ver == 0x80000001u) 380 data->version = MAKE_MMU_VER(1, 0); 381 else 382 data->version = MMU_RAW_VER(ver); 383 384 dev_dbg(data->sysmmu, "hardware version: %d.%d\n", 385 MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); 386 387 __sysmmu_disable_clocks(data); 388 } 389 390 static void show_fault_information(struct sysmmu_drvdata *data, 391 const struct sysmmu_fault_info *finfo, 392 sysmmu_iova_t fault_addr) 393 { 394 sysmmu_pte_t *ent; 395 396 dev_err(data->sysmmu, "%s: %s FAULT occurred at %#x\n", 397 dev_name(data->master), finfo->name, fault_addr); 398 dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); 399 ent = section_entry(phys_to_virt(data->pgtable), fault_addr); 400 dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent); 401 if (lv1ent_page(ent)) { 402 ent = page_entry(ent, fault_addr); 403 dev_dbg(data->sysmmu, "\t Lv2 entry: %#x\n", *ent); 404 } 405 } 406 407 static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) 408 { 409 /* SYSMMU is in blocked state when interrupt occurred. */ 410 struct sysmmu_drvdata *data = dev_id; 411 const struct sysmmu_fault_info *finfo; 412 unsigned int i, n, itype; 413 sysmmu_iova_t fault_addr = -1; 414 unsigned short reg_status, reg_clear; 415 int ret = -ENOSYS; 416 417 WARN_ON(!data->active); 418 419 if (MMU_MAJ_VER(data->version) < 5) { 420 reg_status = REG_INT_STATUS; 421 reg_clear = REG_INT_CLEAR; 422 finfo = sysmmu_faults; 423 n = ARRAY_SIZE(sysmmu_faults); 424 } else { 425 reg_status = REG_V5_INT_STATUS; 426 reg_clear = REG_V5_INT_CLEAR; 427 finfo = sysmmu_v5_faults; 428 n = ARRAY_SIZE(sysmmu_v5_faults); 429 } 430 431 spin_lock(&data->lock); 432 433 clk_enable(data->clk_master); 434 435 itype = __ffs(readl(data->sfrbase + reg_status)); 436 for (i = 0; i < n; i++, finfo++) 437 if (finfo->bit == itype) 438 break; 439 /* unknown/unsupported fault */ 440 BUG_ON(i == n); 441 442 /* print debug message */ 443 fault_addr = readl(data->sfrbase + finfo->addr_reg); 444 show_fault_information(data, finfo, fault_addr); 445 446 if (data->domain) 447 ret = report_iommu_fault(&data->domain->domain, 448 data->master, fault_addr, finfo->type); 449 /* fault is not recovered by fault handler */ 450 BUG_ON(ret != 0); 451 452 writel(1 << itype, data->sfrbase + reg_clear); 453 454 sysmmu_unblock(data); 455 456 clk_disable(data->clk_master); 457 458 spin_unlock(&data->lock); 459 460 return IRQ_HANDLED; 461 } 462 463 static void __sysmmu_disable(struct sysmmu_drvdata *data) 464 { 465 unsigned long flags; 466 467 clk_enable(data->clk_master); 468 469 spin_lock_irqsave(&data->lock, flags); 470 writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); 471 writel(0, data->sfrbase + REG_MMU_CFG); 472 data->active = false; 473 spin_unlock_irqrestore(&data->lock, flags); 474 475 __sysmmu_disable_clocks(data); 476 } 477 478 static void __sysmmu_init_config(struct sysmmu_drvdata *data) 479 { 480 unsigned int cfg; 481 482 if (data->version <= MAKE_MMU_VER(3, 1)) 483 cfg = CFG_LRU | CFG_QOS(15); 484 else if (data->version <= MAKE_MMU_VER(3, 2)) 485 cfg = CFG_LRU | CFG_QOS(15) | CFG_FLPDCACHE | CFG_SYSSEL; 486 else 487 cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN; 488 489 cfg |= CFG_EAP; /* enable access protection bits check */ 490 491 writel(cfg, data->sfrbase + REG_MMU_CFG); 492 } 493 494 static void __sysmmu_enable(struct sysmmu_drvdata *data) 495 { 496 unsigned long flags; 497 498 __sysmmu_enable_clocks(data); 499 500 spin_lock_irqsave(&data->lock, flags); 501 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); 502 __sysmmu_init_config(data); 503 __sysmmu_set_ptbase(data, data->pgtable); 504 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); 505 data->active = true; 506 spin_unlock_irqrestore(&data->lock, flags); 507 508 /* 509 * SYSMMU driver keeps master's clock enabled only for the short 510 * time, while accessing the registers. For performing address 511 * translation during DMA transaction it relies on the client 512 * driver to enable it. 513 */ 514 clk_disable(data->clk_master); 515 } 516 517 static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, 518 sysmmu_iova_t iova) 519 { 520 unsigned long flags; 521 522 spin_lock_irqsave(&data->lock, flags); 523 if (data->active && data->version >= MAKE_MMU_VER(3, 3)) { 524 clk_enable(data->clk_master); 525 if (sysmmu_block(data)) { 526 if (data->version >= MAKE_MMU_VER(5, 0)) 527 __sysmmu_tlb_invalidate(data); 528 else 529 __sysmmu_tlb_invalidate_entry(data, iova, 1); 530 sysmmu_unblock(data); 531 } 532 clk_disable(data->clk_master); 533 } 534 spin_unlock_irqrestore(&data->lock, flags); 535 } 536 537 static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, 538 sysmmu_iova_t iova, size_t size) 539 { 540 unsigned long flags; 541 542 spin_lock_irqsave(&data->lock, flags); 543 if (data->active) { 544 unsigned int num_inv = 1; 545 546 clk_enable(data->clk_master); 547 548 /* 549 * L2TLB invalidation required 550 * 4KB page: 1 invalidation 551 * 64KB page: 16 invalidations 552 * 1MB page: 64 invalidations 553 * because it is set-associative TLB 554 * with 8-way and 64 sets. 555 * 1MB page can be cached in one of all sets. 556 * 64KB page can be one of 16 consecutive sets. 557 */ 558 if (MMU_MAJ_VER(data->version) == 2) 559 num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); 560 561 if (sysmmu_block(data)) { 562 __sysmmu_tlb_invalidate_entry(data, iova, num_inv); 563 sysmmu_unblock(data); 564 } 565 clk_disable(data->clk_master); 566 } 567 spin_unlock_irqrestore(&data->lock, flags); 568 } 569 570 static const struct iommu_ops exynos_iommu_ops; 571 572 static int __init exynos_sysmmu_probe(struct platform_device *pdev) 573 { 574 int irq, ret; 575 struct device *dev = &pdev->dev; 576 struct sysmmu_drvdata *data; 577 struct resource *res; 578 579 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 580 if (!data) 581 return -ENOMEM; 582 583 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 584 data->sfrbase = devm_ioremap_resource(dev, res); 585 if (IS_ERR(data->sfrbase)) 586 return PTR_ERR(data->sfrbase); 587 588 irq = platform_get_irq(pdev, 0); 589 if (irq <= 0) { 590 dev_err(dev, "Unable to find IRQ resource\n"); 591 return irq; 592 } 593 594 ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0, 595 dev_name(dev), data); 596 if (ret) { 597 dev_err(dev, "Unabled to register handler of irq %d\n", irq); 598 return ret; 599 } 600 601 data->clk = devm_clk_get(dev, "sysmmu"); 602 if (PTR_ERR(data->clk) == -ENOENT) 603 data->clk = NULL; 604 else if (IS_ERR(data->clk)) 605 return PTR_ERR(data->clk); 606 607 data->aclk = devm_clk_get(dev, "aclk"); 608 if (PTR_ERR(data->aclk) == -ENOENT) 609 data->aclk = NULL; 610 else if (IS_ERR(data->aclk)) 611 return PTR_ERR(data->aclk); 612 613 data->pclk = devm_clk_get(dev, "pclk"); 614 if (PTR_ERR(data->pclk) == -ENOENT) 615 data->pclk = NULL; 616 else if (IS_ERR(data->pclk)) 617 return PTR_ERR(data->pclk); 618 619 if (!data->clk && (!data->aclk || !data->pclk)) { 620 dev_err(dev, "Failed to get device clock(s)!\n"); 621 return -ENOSYS; 622 } 623 624 data->clk_master = devm_clk_get(dev, "master"); 625 if (PTR_ERR(data->clk_master) == -ENOENT) 626 data->clk_master = NULL; 627 else if (IS_ERR(data->clk_master)) 628 return PTR_ERR(data->clk_master); 629 630 data->sysmmu = dev; 631 spin_lock_init(&data->lock); 632 633 ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL, 634 dev_name(data->sysmmu)); 635 if (ret) 636 return ret; 637 638 iommu_device_set_ops(&data->iommu, &exynos_iommu_ops); 639 iommu_device_set_fwnode(&data->iommu, &dev->of_node->fwnode); 640 641 ret = iommu_device_register(&data->iommu); 642 if (ret) 643 return ret; 644 645 platform_set_drvdata(pdev, data); 646 647 __sysmmu_get_version(data); 648 if (PG_ENT_SHIFT < 0) { 649 if (MMU_MAJ_VER(data->version) < 5) { 650 PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT; 651 LV1_PROT = SYSMMU_LV1_PROT; 652 LV2_PROT = SYSMMU_LV2_PROT; 653 } else { 654 PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT; 655 LV1_PROT = SYSMMU_V5_LV1_PROT; 656 LV2_PROT = SYSMMU_V5_LV2_PROT; 657 } 658 } 659 660 /* 661 * use the first registered sysmmu device for performing 662 * dma mapping operations on iommu page tables (cpu cache flush) 663 */ 664 if (!dma_dev) 665 dma_dev = &pdev->dev; 666 667 pm_runtime_enable(dev); 668 669 return 0; 670 } 671 672 static int __maybe_unused exynos_sysmmu_suspend(struct device *dev) 673 { 674 struct sysmmu_drvdata *data = dev_get_drvdata(dev); 675 struct device *master = data->master; 676 677 if (master) { 678 struct exynos_iommu_owner *owner = master->archdata.iommu; 679 680 mutex_lock(&owner->rpm_lock); 681 if (data->domain) { 682 dev_dbg(data->sysmmu, "saving state\n"); 683 __sysmmu_disable(data); 684 } 685 mutex_unlock(&owner->rpm_lock); 686 } 687 return 0; 688 } 689 690 static int __maybe_unused exynos_sysmmu_resume(struct device *dev) 691 { 692 struct sysmmu_drvdata *data = dev_get_drvdata(dev); 693 struct device *master = data->master; 694 695 if (master) { 696 struct exynos_iommu_owner *owner = master->archdata.iommu; 697 698 mutex_lock(&owner->rpm_lock); 699 if (data->domain) { 700 dev_dbg(data->sysmmu, "restoring state\n"); 701 __sysmmu_enable(data); 702 } 703 mutex_unlock(&owner->rpm_lock); 704 } 705 return 0; 706 } 707 708 static const struct dev_pm_ops sysmmu_pm_ops = { 709 SET_RUNTIME_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume, NULL) 710 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 711 pm_runtime_force_resume) 712 }; 713 714 static const struct of_device_id sysmmu_of_match[] = { 715 { .compatible = "samsung,exynos-sysmmu", }, 716 { }, 717 }; 718 719 static struct platform_driver exynos_sysmmu_driver __refdata = { 720 .probe = exynos_sysmmu_probe, 721 .driver = { 722 .name = "exynos-sysmmu", 723 .of_match_table = sysmmu_of_match, 724 .pm = &sysmmu_pm_ops, 725 .suppress_bind_attrs = true, 726 } 727 }; 728 729 static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) 730 { 731 dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), 732 DMA_TO_DEVICE); 733 *ent = cpu_to_le32(val); 734 dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent), 735 DMA_TO_DEVICE); 736 } 737 738 static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) 739 { 740 struct exynos_iommu_domain *domain; 741 dma_addr_t handle; 742 int i; 743 744 /* Check if correct PTE offsets are initialized */ 745 BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev); 746 747 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 748 if (!domain) 749 return NULL; 750 751 if (type == IOMMU_DOMAIN_DMA) { 752 if (iommu_get_dma_cookie(&domain->domain) != 0) 753 goto err_pgtable; 754 } else if (type != IOMMU_DOMAIN_UNMANAGED) { 755 goto err_pgtable; 756 } 757 758 domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); 759 if (!domain->pgtable) 760 goto err_dma_cookie; 761 762 domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); 763 if (!domain->lv2entcnt) 764 goto err_counter; 765 766 /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */ 767 for (i = 0; i < NUM_LV1ENTRIES; i++) 768 domain->pgtable[i] = ZERO_LV2LINK; 769 770 handle = dma_map_single(dma_dev, domain->pgtable, LV1TABLE_SIZE, 771 DMA_TO_DEVICE); 772 /* For mapping page table entries we rely on dma == phys */ 773 BUG_ON(handle != virt_to_phys(domain->pgtable)); 774 if (dma_mapping_error(dma_dev, handle)) 775 goto err_lv2ent; 776 777 spin_lock_init(&domain->lock); 778 spin_lock_init(&domain->pgtablelock); 779 INIT_LIST_HEAD(&domain->clients); 780 781 domain->domain.geometry.aperture_start = 0; 782 domain->domain.geometry.aperture_end = ~0UL; 783 domain->domain.geometry.force_aperture = true; 784 785 return &domain->domain; 786 787 err_lv2ent: 788 free_pages((unsigned long)domain->lv2entcnt, 1); 789 err_counter: 790 free_pages((unsigned long)domain->pgtable, 2); 791 err_dma_cookie: 792 if (type == IOMMU_DOMAIN_DMA) 793 iommu_put_dma_cookie(&domain->domain); 794 err_pgtable: 795 kfree(domain); 796 return NULL; 797 } 798 799 static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) 800 { 801 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 802 struct sysmmu_drvdata *data, *next; 803 unsigned long flags; 804 int i; 805 806 WARN_ON(!list_empty(&domain->clients)); 807 808 spin_lock_irqsave(&domain->lock, flags); 809 810 list_for_each_entry_safe(data, next, &domain->clients, domain_node) { 811 spin_lock(&data->lock); 812 __sysmmu_disable(data); 813 data->pgtable = 0; 814 data->domain = NULL; 815 list_del_init(&data->domain_node); 816 spin_unlock(&data->lock); 817 } 818 819 spin_unlock_irqrestore(&domain->lock, flags); 820 821 if (iommu_domain->type == IOMMU_DOMAIN_DMA) 822 iommu_put_dma_cookie(iommu_domain); 823 824 dma_unmap_single(dma_dev, virt_to_phys(domain->pgtable), LV1TABLE_SIZE, 825 DMA_TO_DEVICE); 826 827 for (i = 0; i < NUM_LV1ENTRIES; i++) 828 if (lv1ent_page(domain->pgtable + i)) { 829 phys_addr_t base = lv2table_base(domain->pgtable + i); 830 831 dma_unmap_single(dma_dev, base, LV2TABLE_SIZE, 832 DMA_TO_DEVICE); 833 kmem_cache_free(lv2table_kmem_cache, 834 phys_to_virt(base)); 835 } 836 837 free_pages((unsigned long)domain->pgtable, 2); 838 free_pages((unsigned long)domain->lv2entcnt, 1); 839 kfree(domain); 840 } 841 842 static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, 843 struct device *dev) 844 { 845 struct exynos_iommu_owner *owner = dev->archdata.iommu; 846 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 847 phys_addr_t pagetable = virt_to_phys(domain->pgtable); 848 struct sysmmu_drvdata *data, *next; 849 unsigned long flags; 850 851 if (!has_sysmmu(dev) || owner->domain != iommu_domain) 852 return; 853 854 mutex_lock(&owner->rpm_lock); 855 856 list_for_each_entry(data, &owner->controllers, owner_node) { 857 pm_runtime_get_noresume(data->sysmmu); 858 if (pm_runtime_active(data->sysmmu)) 859 __sysmmu_disable(data); 860 pm_runtime_put(data->sysmmu); 861 } 862 863 spin_lock_irqsave(&domain->lock, flags); 864 list_for_each_entry_safe(data, next, &domain->clients, domain_node) { 865 spin_lock(&data->lock); 866 data->pgtable = 0; 867 data->domain = NULL; 868 list_del_init(&data->domain_node); 869 spin_unlock(&data->lock); 870 } 871 owner->domain = NULL; 872 spin_unlock_irqrestore(&domain->lock, flags); 873 874 mutex_unlock(&owner->rpm_lock); 875 876 dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", __func__, 877 &pagetable); 878 } 879 880 static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, 881 struct device *dev) 882 { 883 struct exynos_iommu_owner *owner = dev->archdata.iommu; 884 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 885 struct sysmmu_drvdata *data; 886 phys_addr_t pagetable = virt_to_phys(domain->pgtable); 887 unsigned long flags; 888 889 if (!has_sysmmu(dev)) 890 return -ENODEV; 891 892 if (owner->domain) 893 exynos_iommu_detach_device(owner->domain, dev); 894 895 mutex_lock(&owner->rpm_lock); 896 897 spin_lock_irqsave(&domain->lock, flags); 898 list_for_each_entry(data, &owner->controllers, owner_node) { 899 spin_lock(&data->lock); 900 data->pgtable = pagetable; 901 data->domain = domain; 902 list_add_tail(&data->domain_node, &domain->clients); 903 spin_unlock(&data->lock); 904 } 905 owner->domain = iommu_domain; 906 spin_unlock_irqrestore(&domain->lock, flags); 907 908 list_for_each_entry(data, &owner->controllers, owner_node) { 909 pm_runtime_get_noresume(data->sysmmu); 910 if (pm_runtime_active(data->sysmmu)) 911 __sysmmu_enable(data); 912 pm_runtime_put(data->sysmmu); 913 } 914 915 mutex_unlock(&owner->rpm_lock); 916 917 dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa\n", __func__, 918 &pagetable); 919 920 return 0; 921 } 922 923 static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, 924 sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) 925 { 926 if (lv1ent_section(sent)) { 927 WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova); 928 return ERR_PTR(-EADDRINUSE); 929 } 930 931 if (lv1ent_fault(sent)) { 932 dma_addr_t handle; 933 sysmmu_pte_t *pent; 934 bool need_flush_flpd_cache = lv1ent_zero(sent); 935 936 pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); 937 BUG_ON((uintptr_t)pent & (LV2TABLE_SIZE - 1)); 938 if (!pent) 939 return ERR_PTR(-ENOMEM); 940 941 update_pte(sent, mk_lv1ent_page(virt_to_phys(pent))); 942 kmemleak_ignore(pent); 943 *pgcounter = NUM_LV2ENTRIES; 944 handle = dma_map_single(dma_dev, pent, LV2TABLE_SIZE, 945 DMA_TO_DEVICE); 946 if (dma_mapping_error(dma_dev, handle)) { 947 kmem_cache_free(lv2table_kmem_cache, pent); 948 return ERR_PTR(-EADDRINUSE); 949 } 950 951 /* 952 * If pre-fetched SLPD is a faulty SLPD in zero_l2_table, 953 * FLPD cache may cache the address of zero_l2_table. This 954 * function replaces the zero_l2_table with new L2 page table 955 * to write valid mappings. 956 * Accessing the valid area may cause page fault since FLPD 957 * cache may still cache zero_l2_table for the valid area 958 * instead of new L2 page table that has the mapping 959 * information of the valid area. 960 * Thus any replacement of zero_l2_table with other valid L2 961 * page table must involve FLPD cache invalidation for System 962 * MMU v3.3. 963 * FLPD cache invalidation is performed with TLB invalidation 964 * by VPN without blocking. It is safe to invalidate TLB without 965 * blocking because the target address of TLB invalidation is 966 * not currently mapped. 967 */ 968 if (need_flush_flpd_cache) { 969 struct sysmmu_drvdata *data; 970 971 spin_lock(&domain->lock); 972 list_for_each_entry(data, &domain->clients, domain_node) 973 sysmmu_tlb_invalidate_flpdcache(data, iova); 974 spin_unlock(&domain->lock); 975 } 976 } 977 978 return page_entry(sent, iova); 979 } 980 981 static int lv1set_section(struct exynos_iommu_domain *domain, 982 sysmmu_pte_t *sent, sysmmu_iova_t iova, 983 phys_addr_t paddr, int prot, short *pgcnt) 984 { 985 if (lv1ent_section(sent)) { 986 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", 987 iova); 988 return -EADDRINUSE; 989 } 990 991 if (lv1ent_page(sent)) { 992 if (*pgcnt != NUM_LV2ENTRIES) { 993 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", 994 iova); 995 return -EADDRINUSE; 996 } 997 998 kmem_cache_free(lv2table_kmem_cache, page_entry(sent, 0)); 999 *pgcnt = 0; 1000 } 1001 1002 update_pte(sent, mk_lv1ent_sect(paddr, prot)); 1003 1004 spin_lock(&domain->lock); 1005 if (lv1ent_page_zero(sent)) { 1006 struct sysmmu_drvdata *data; 1007 /* 1008 * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD 1009 * entry by speculative prefetch of SLPD which has no mapping. 1010 */ 1011 list_for_each_entry(data, &domain->clients, domain_node) 1012 sysmmu_tlb_invalidate_flpdcache(data, iova); 1013 } 1014 spin_unlock(&domain->lock); 1015 1016 return 0; 1017 } 1018 1019 static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, 1020 int prot, short *pgcnt) 1021 { 1022 if (size == SPAGE_SIZE) { 1023 if (WARN_ON(!lv2ent_fault(pent))) 1024 return -EADDRINUSE; 1025 1026 update_pte(pent, mk_lv2ent_spage(paddr, prot)); 1027 *pgcnt -= 1; 1028 } else { /* size == LPAGE_SIZE */ 1029 int i; 1030 dma_addr_t pent_base = virt_to_phys(pent); 1031 1032 dma_sync_single_for_cpu(dma_dev, pent_base, 1033 sizeof(*pent) * SPAGES_PER_LPAGE, 1034 DMA_TO_DEVICE); 1035 for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { 1036 if (WARN_ON(!lv2ent_fault(pent))) { 1037 if (i > 0) 1038 memset(pent - i, 0, sizeof(*pent) * i); 1039 return -EADDRINUSE; 1040 } 1041 1042 *pent = mk_lv2ent_lpage(paddr, prot); 1043 } 1044 dma_sync_single_for_device(dma_dev, pent_base, 1045 sizeof(*pent) * SPAGES_PER_LPAGE, 1046 DMA_TO_DEVICE); 1047 *pgcnt -= SPAGES_PER_LPAGE; 1048 } 1049 1050 return 0; 1051 } 1052 1053 /* 1054 * *CAUTION* to the I/O virtual memory managers that support exynos-iommu: 1055 * 1056 * System MMU v3.x has advanced logic to improve address translation 1057 * performance with caching more page table entries by a page table walk. 1058 * However, the logic has a bug that while caching faulty page table entries, 1059 * System MMU reports page fault if the cached fault entry is hit even though 1060 * the fault entry is updated to a valid entry after the entry is cached. 1061 * To prevent caching faulty page table entries which may be updated to valid 1062 * entries later, the virtual memory manager should care about the workaround 1063 * for the problem. The following describes the workaround. 1064 * 1065 * Any two consecutive I/O virtual address regions must have a hole of 128KiB 1066 * at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug). 1067 * 1068 * Precisely, any start address of I/O virtual region must be aligned with 1069 * the following sizes for System MMU v3.1 and v3.2. 1070 * System MMU v3.1: 128KiB 1071 * System MMU v3.2: 256KiB 1072 * 1073 * Because System MMU v3.3 caches page table entries more aggressively, it needs 1074 * more workarounds. 1075 * - Any two consecutive I/O virtual regions must have a hole of size larger 1076 * than or equal to 128KiB. 1077 * - Start address of an I/O virtual region must be aligned by 128KiB. 1078 */ 1079 static int exynos_iommu_map(struct iommu_domain *iommu_domain, 1080 unsigned long l_iova, phys_addr_t paddr, size_t size, 1081 int prot) 1082 { 1083 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 1084 sysmmu_pte_t *entry; 1085 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; 1086 unsigned long flags; 1087 int ret = -ENOMEM; 1088 1089 BUG_ON(domain->pgtable == NULL); 1090 prot &= SYSMMU_SUPPORTED_PROT_BITS; 1091 1092 spin_lock_irqsave(&domain->pgtablelock, flags); 1093 1094 entry = section_entry(domain->pgtable, iova); 1095 1096 if (size == SECT_SIZE) { 1097 ret = lv1set_section(domain, entry, iova, paddr, prot, 1098 &domain->lv2entcnt[lv1ent_offset(iova)]); 1099 } else { 1100 sysmmu_pte_t *pent; 1101 1102 pent = alloc_lv2entry(domain, entry, iova, 1103 &domain->lv2entcnt[lv1ent_offset(iova)]); 1104 1105 if (IS_ERR(pent)) 1106 ret = PTR_ERR(pent); 1107 else 1108 ret = lv2set_page(pent, paddr, size, prot, 1109 &domain->lv2entcnt[lv1ent_offset(iova)]); 1110 } 1111 1112 if (ret) 1113 pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n", 1114 __func__, ret, size, iova); 1115 1116 spin_unlock_irqrestore(&domain->pgtablelock, flags); 1117 1118 return ret; 1119 } 1120 1121 static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain, 1122 sysmmu_iova_t iova, size_t size) 1123 { 1124 struct sysmmu_drvdata *data; 1125 unsigned long flags; 1126 1127 spin_lock_irqsave(&domain->lock, flags); 1128 1129 list_for_each_entry(data, &domain->clients, domain_node) 1130 sysmmu_tlb_invalidate_entry(data, iova, size); 1131 1132 spin_unlock_irqrestore(&domain->lock, flags); 1133 } 1134 1135 static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, 1136 unsigned long l_iova, size_t size) 1137 { 1138 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 1139 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; 1140 sysmmu_pte_t *ent; 1141 size_t err_pgsize; 1142 unsigned long flags; 1143 1144 BUG_ON(domain->pgtable == NULL); 1145 1146 spin_lock_irqsave(&domain->pgtablelock, flags); 1147 1148 ent = section_entry(domain->pgtable, iova); 1149 1150 if (lv1ent_section(ent)) { 1151 if (WARN_ON(size < SECT_SIZE)) { 1152 err_pgsize = SECT_SIZE; 1153 goto err; 1154 } 1155 1156 /* workaround for h/w bug in System MMU v3.3 */ 1157 update_pte(ent, ZERO_LV2LINK); 1158 size = SECT_SIZE; 1159 goto done; 1160 } 1161 1162 if (unlikely(lv1ent_fault(ent))) { 1163 if (size > SECT_SIZE) 1164 size = SECT_SIZE; 1165 goto done; 1166 } 1167 1168 /* lv1ent_page(sent) == true here */ 1169 1170 ent = page_entry(ent, iova); 1171 1172 if (unlikely(lv2ent_fault(ent))) { 1173 size = SPAGE_SIZE; 1174 goto done; 1175 } 1176 1177 if (lv2ent_small(ent)) { 1178 update_pte(ent, 0); 1179 size = SPAGE_SIZE; 1180 domain->lv2entcnt[lv1ent_offset(iova)] += 1; 1181 goto done; 1182 } 1183 1184 /* lv1ent_large(ent) == true here */ 1185 if (WARN_ON(size < LPAGE_SIZE)) { 1186 err_pgsize = LPAGE_SIZE; 1187 goto err; 1188 } 1189 1190 dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), 1191 sizeof(*ent) * SPAGES_PER_LPAGE, 1192 DMA_TO_DEVICE); 1193 memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); 1194 dma_sync_single_for_device(dma_dev, virt_to_phys(ent), 1195 sizeof(*ent) * SPAGES_PER_LPAGE, 1196 DMA_TO_DEVICE); 1197 size = LPAGE_SIZE; 1198 domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; 1199 done: 1200 spin_unlock_irqrestore(&domain->pgtablelock, flags); 1201 1202 exynos_iommu_tlb_invalidate_entry(domain, iova, size); 1203 1204 return size; 1205 err: 1206 spin_unlock_irqrestore(&domain->pgtablelock, flags); 1207 1208 pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n", 1209 __func__, size, iova, err_pgsize); 1210 1211 return 0; 1212 } 1213 1214 static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, 1215 dma_addr_t iova) 1216 { 1217 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); 1218 sysmmu_pte_t *entry; 1219 unsigned long flags; 1220 phys_addr_t phys = 0; 1221 1222 spin_lock_irqsave(&domain->pgtablelock, flags); 1223 1224 entry = section_entry(domain->pgtable, iova); 1225 1226 if (lv1ent_section(entry)) { 1227 phys = section_phys(entry) + section_offs(iova); 1228 } else if (lv1ent_page(entry)) { 1229 entry = page_entry(entry, iova); 1230 1231 if (lv2ent_large(entry)) 1232 phys = lpage_phys(entry) + lpage_offs(iova); 1233 else if (lv2ent_small(entry)) 1234 phys = spage_phys(entry) + spage_offs(iova); 1235 } 1236 1237 spin_unlock_irqrestore(&domain->pgtablelock, flags); 1238 1239 return phys; 1240 } 1241 1242 static int exynos_iommu_add_device(struct device *dev) 1243 { 1244 struct exynos_iommu_owner *owner = dev->archdata.iommu; 1245 struct sysmmu_drvdata *data; 1246 struct iommu_group *group; 1247 1248 if (!has_sysmmu(dev)) 1249 return -ENODEV; 1250 1251 group = iommu_group_get_for_dev(dev); 1252 1253 if (IS_ERR(group)) 1254 return PTR_ERR(group); 1255 1256 list_for_each_entry(data, &owner->controllers, owner_node) { 1257 /* 1258 * SYSMMU will be runtime activated via device link 1259 * (dependency) to its master device, so there are no 1260 * direct calls to pm_runtime_get/put in this driver. 1261 */ 1262 data->link = device_link_add(dev, data->sysmmu, 1263 DL_FLAG_PM_RUNTIME); 1264 } 1265 iommu_group_put(group); 1266 1267 return 0; 1268 } 1269 1270 static void exynos_iommu_remove_device(struct device *dev) 1271 { 1272 struct exynos_iommu_owner *owner = dev->archdata.iommu; 1273 struct sysmmu_drvdata *data; 1274 1275 if (!has_sysmmu(dev)) 1276 return; 1277 1278 if (owner->domain) { 1279 struct iommu_group *group = iommu_group_get(dev); 1280 1281 if (group) { 1282 WARN_ON(owner->domain != 1283 iommu_group_default_domain(group)); 1284 exynos_iommu_detach_device(owner->domain, dev); 1285 iommu_group_put(group); 1286 } 1287 } 1288 iommu_group_remove_device(dev); 1289 1290 list_for_each_entry(data, &owner->controllers, owner_node) 1291 device_link_del(data->link); 1292 } 1293 1294 static int exynos_iommu_of_xlate(struct device *dev, 1295 struct of_phandle_args *spec) 1296 { 1297 struct exynos_iommu_owner *owner = dev->archdata.iommu; 1298 struct platform_device *sysmmu = of_find_device_by_node(spec->np); 1299 struct sysmmu_drvdata *data, *entry; 1300 1301 if (!sysmmu) 1302 return -ENODEV; 1303 1304 data = platform_get_drvdata(sysmmu); 1305 if (!data) 1306 return -ENODEV; 1307 1308 if (!owner) { 1309 owner = kzalloc(sizeof(*owner), GFP_KERNEL); 1310 if (!owner) 1311 return -ENOMEM; 1312 1313 INIT_LIST_HEAD(&owner->controllers); 1314 mutex_init(&owner->rpm_lock); 1315 dev->archdata.iommu = owner; 1316 } 1317 1318 list_for_each_entry(entry, &owner->controllers, owner_node) 1319 if (entry == data) 1320 return 0; 1321 1322 list_add_tail(&data->owner_node, &owner->controllers); 1323 data->master = dev; 1324 1325 return 0; 1326 } 1327 1328 static const struct iommu_ops exynos_iommu_ops = { 1329 .domain_alloc = exynos_iommu_domain_alloc, 1330 .domain_free = exynos_iommu_domain_free, 1331 .attach_dev = exynos_iommu_attach_device, 1332 .detach_dev = exynos_iommu_detach_device, 1333 .map = exynos_iommu_map, 1334 .unmap = exynos_iommu_unmap, 1335 .map_sg = default_iommu_map_sg, 1336 .iova_to_phys = exynos_iommu_iova_to_phys, 1337 .device_group = generic_device_group, 1338 .add_device = exynos_iommu_add_device, 1339 .remove_device = exynos_iommu_remove_device, 1340 .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, 1341 .of_xlate = exynos_iommu_of_xlate, 1342 }; 1343 1344 static int __init exynos_iommu_init(void) 1345 { 1346 struct device_node *np; 1347 int ret; 1348 1349 np = of_find_matching_node(NULL, sysmmu_of_match); 1350 if (!np) 1351 return 0; 1352 1353 of_node_put(np); 1354 1355 lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table", 1356 LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL); 1357 if (!lv2table_kmem_cache) { 1358 pr_err("%s: Failed to create kmem cache\n", __func__); 1359 return -ENOMEM; 1360 } 1361 1362 ret = platform_driver_register(&exynos_sysmmu_driver); 1363 if (ret) { 1364 pr_err("%s: Failed to register driver\n", __func__); 1365 goto err_reg_driver; 1366 } 1367 1368 zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL); 1369 if (zero_lv2_table == NULL) { 1370 pr_err("%s: Failed to allocate zero level2 page table\n", 1371 __func__); 1372 ret = -ENOMEM; 1373 goto err_zero_lv2; 1374 } 1375 1376 ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); 1377 if (ret) { 1378 pr_err("%s: Failed to register exynos-iommu driver.\n", 1379 __func__); 1380 goto err_set_iommu; 1381 } 1382 1383 return 0; 1384 err_set_iommu: 1385 kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); 1386 err_zero_lv2: 1387 platform_driver_unregister(&exynos_sysmmu_driver); 1388 err_reg_driver: 1389 kmem_cache_destroy(lv2table_kmem_cache); 1390 return ret; 1391 } 1392 core_initcall(exynos_iommu_init); 1393 1394 IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu"); 1395