1 /* 2 * CPU-agnostic ARM page table allocator. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * along with this program. If not, see <http://www.gnu.org/licenses/>. 15 * 16 * Copyright (C) 2014 ARM Limited 17 * 18 * Author: Will Deacon <will.deacon@arm.com> 19 */ 20 21 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt 22 23 #include <linux/atomic.h> 24 #include <linux/bitops.h> 25 #include <linux/iommu.h> 26 #include <linux/kernel.h> 27 #include <linux/sizes.h> 28 #include <linux/slab.h> 29 #include <linux/types.h> 30 #include <linux/dma-mapping.h> 31 32 #include <asm/barrier.h> 33 34 #include "io-pgtable.h" 35 36 #define ARM_LPAE_MAX_ADDR_BITS 52 37 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 38 #define ARM_LPAE_MAX_LEVELS 4 39 40 /* Struct accessors */ 41 #define io_pgtable_to_data(x) \ 42 container_of((x), struct arm_lpae_io_pgtable, iop) 43 44 #define io_pgtable_ops_to_data(x) \ 45 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) 46 47 /* 48 * For consistency with the architecture, we always consider 49 * ARM_LPAE_MAX_LEVELS levels, with the walk starting at level n >=0 50 */ 51 #define ARM_LPAE_START_LVL(d) (ARM_LPAE_MAX_LEVELS - (d)->levels) 52 53 /* 54 * Calculate the right shift amount to get to the portion describing level l 55 * in a virtual address mapped by the pagetable in d. 56 */ 57 #define ARM_LPAE_LVL_SHIFT(l,d) \ 58 ((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1)) \ 59 * (d)->bits_per_level) + (d)->pg_shift) 60 61 #define ARM_LPAE_GRANULE(d) (1UL << (d)->pg_shift) 62 63 #define ARM_LPAE_PAGES_PER_PGD(d) \ 64 DIV_ROUND_UP((d)->pgd_size, ARM_LPAE_GRANULE(d)) 65 66 /* 67 * Calculate the index at level l used to map virtual address a using the 68 * pagetable in d. 69 */ 70 #define ARM_LPAE_PGD_IDX(l,d) \ 71 ((l) == ARM_LPAE_START_LVL(d) ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0) 72 73 #define ARM_LPAE_LVL_IDX(a,l,d) \ 74 (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ 75 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1)) 76 77 /* Calculate the block/page mapping size at level l for pagetable in d. */ 78 #define ARM_LPAE_BLOCK_SIZE(l,d) \ 79 (1ULL << (ilog2(sizeof(arm_lpae_iopte)) + \ 80 ((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level))) 81 82 /* Page table bits */ 83 #define ARM_LPAE_PTE_TYPE_SHIFT 0 84 #define ARM_LPAE_PTE_TYPE_MASK 0x3 85 86 #define ARM_LPAE_PTE_TYPE_BLOCK 1 87 #define ARM_LPAE_PTE_TYPE_TABLE 3 88 #define ARM_LPAE_PTE_TYPE_PAGE 3 89 90 #define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12) 91 92 #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) 93 #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) 94 #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) 95 #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) 96 #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) 97 #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8) 98 #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5) 99 #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0) 100 101 #define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2) 102 /* Ignore the contiguous bit for block splitting */ 103 #define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52) 104 #define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \ 105 ARM_LPAE_PTE_ATTR_HI_MASK) 106 /* Software bit for solving coherency races */ 107 #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) 108 109 /* Stage-1 PTE */ 110 #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) 111 #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6) 112 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 113 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) 114 115 /* Stage-2 PTE */ 116 #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6) 117 #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6) 118 #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6) 119 #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2) 120 #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2) 121 #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) 122 123 /* Register bits */ 124 #define ARM_32_LPAE_TCR_EAE (1 << 31) 125 #define ARM_64_LPAE_S2_TCR_RES1 (1 << 31) 126 127 #define ARM_LPAE_TCR_EPD1 (1 << 23) 128 129 #define ARM_LPAE_TCR_TG0_4K (0 << 14) 130 #define ARM_LPAE_TCR_TG0_64K (1 << 14) 131 #define ARM_LPAE_TCR_TG0_16K (2 << 14) 132 133 #define ARM_LPAE_TCR_SH0_SHIFT 12 134 #define ARM_LPAE_TCR_SH0_MASK 0x3 135 #define ARM_LPAE_TCR_SH_NS 0 136 #define ARM_LPAE_TCR_SH_OS 2 137 #define ARM_LPAE_TCR_SH_IS 3 138 139 #define ARM_LPAE_TCR_ORGN0_SHIFT 10 140 #define ARM_LPAE_TCR_IRGN0_SHIFT 8 141 #define ARM_LPAE_TCR_RGN_MASK 0x3 142 #define ARM_LPAE_TCR_RGN_NC 0 143 #define ARM_LPAE_TCR_RGN_WBWA 1 144 #define ARM_LPAE_TCR_RGN_WT 2 145 #define ARM_LPAE_TCR_RGN_WB 3 146 147 #define ARM_LPAE_TCR_SL0_SHIFT 6 148 #define ARM_LPAE_TCR_SL0_MASK 0x3 149 150 #define ARM_LPAE_TCR_T0SZ_SHIFT 0 151 #define ARM_LPAE_TCR_SZ_MASK 0xf 152 153 #define ARM_LPAE_TCR_PS_SHIFT 16 154 #define ARM_LPAE_TCR_PS_MASK 0x7 155 156 #define ARM_LPAE_TCR_IPS_SHIFT 32 157 #define ARM_LPAE_TCR_IPS_MASK 0x7 158 159 #define ARM_LPAE_TCR_PS_32_BIT 0x0ULL 160 #define ARM_LPAE_TCR_PS_36_BIT 0x1ULL 161 #define ARM_LPAE_TCR_PS_40_BIT 0x2ULL 162 #define ARM_LPAE_TCR_PS_42_BIT 0x3ULL 163 #define ARM_LPAE_TCR_PS_44_BIT 0x4ULL 164 #define ARM_LPAE_TCR_PS_48_BIT 0x5ULL 165 #define ARM_LPAE_TCR_PS_52_BIT 0x6ULL 166 167 #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) 168 #define ARM_LPAE_MAIR_ATTR_MASK 0xff 169 #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 170 #define ARM_LPAE_MAIR_ATTR_NC 0x44 171 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff 172 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0 173 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 174 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 175 176 /* IOPTE accessors */ 177 #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) 178 179 #define iopte_type(pte,l) \ 180 (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) 181 182 #define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK) 183 184 #define iopte_leaf(pte,l) \ 185 (l == (ARM_LPAE_MAX_LEVELS - 1) ? \ 186 (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) : \ 187 (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK)) 188 189 struct arm_lpae_io_pgtable { 190 struct io_pgtable iop; 191 192 int levels; 193 size_t pgd_size; 194 unsigned long pg_shift; 195 unsigned long bits_per_level; 196 197 void *pgd; 198 }; 199 200 typedef u64 arm_lpae_iopte; 201 202 static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, 203 struct arm_lpae_io_pgtable *data) 204 { 205 arm_lpae_iopte pte = paddr; 206 207 /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */ 208 return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; 209 } 210 211 static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte, 212 struct arm_lpae_io_pgtable *data) 213 { 214 u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; 215 216 if (data->pg_shift < 16) 217 return paddr; 218 219 /* Rotate the packed high-order bits back to the top */ 220 return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 221 } 222 223 static bool selftest_running = false; 224 225 static dma_addr_t __arm_lpae_dma_addr(void *pages) 226 { 227 return (dma_addr_t)virt_to_phys(pages); 228 } 229 230 static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, 231 struct io_pgtable_cfg *cfg) 232 { 233 struct device *dev = cfg->iommu_dev; 234 int order = get_order(size); 235 struct page *p; 236 dma_addr_t dma; 237 void *pages; 238 239 VM_BUG_ON((gfp & __GFP_HIGHMEM)); 240 p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); 241 if (!p) 242 return NULL; 243 244 pages = page_address(p); 245 if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { 246 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); 247 if (dma_mapping_error(dev, dma)) 248 goto out_free; 249 /* 250 * We depend on the IOMMU being able to work with any physical 251 * address directly, so if the DMA layer suggests otherwise by 252 * translating or truncating them, that bodes very badly... 253 */ 254 if (dma != virt_to_phys(pages)) 255 goto out_unmap; 256 } 257 258 return pages; 259 260 out_unmap: 261 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); 262 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); 263 out_free: 264 __free_pages(p, order); 265 return NULL; 266 } 267 268 static void __arm_lpae_free_pages(void *pages, size_t size, 269 struct io_pgtable_cfg *cfg) 270 { 271 if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) 272 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), 273 size, DMA_TO_DEVICE); 274 free_pages((unsigned long)pages, get_order(size)); 275 } 276 277 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, 278 struct io_pgtable_cfg *cfg) 279 { 280 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), 281 sizeof(*ptep), DMA_TO_DEVICE); 282 } 283 284 static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, 285 struct io_pgtable_cfg *cfg) 286 { 287 *ptep = pte; 288 289 if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) 290 __arm_lpae_sync_pte(ptep, cfg); 291 } 292 293 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 294 unsigned long iova, size_t size, int lvl, 295 arm_lpae_iopte *ptep); 296 297 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 298 phys_addr_t paddr, arm_lpae_iopte prot, 299 int lvl, arm_lpae_iopte *ptep) 300 { 301 arm_lpae_iopte pte = prot; 302 303 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) 304 pte |= ARM_LPAE_PTE_NS; 305 306 if (lvl == ARM_LPAE_MAX_LEVELS - 1) 307 pte |= ARM_LPAE_PTE_TYPE_PAGE; 308 else 309 pte |= ARM_LPAE_PTE_TYPE_BLOCK; 310 311 pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; 312 pte |= paddr_to_iopte(paddr, data); 313 314 __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); 315 } 316 317 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 318 unsigned long iova, phys_addr_t paddr, 319 arm_lpae_iopte prot, int lvl, 320 arm_lpae_iopte *ptep) 321 { 322 arm_lpae_iopte pte = *ptep; 323 324 if (iopte_leaf(pte, lvl)) { 325 /* We require an unmap first */ 326 WARN_ON(!selftest_running); 327 return -EEXIST; 328 } else if (iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_TABLE) { 329 /* 330 * We need to unmap and free the old table before 331 * overwriting it with a block entry. 332 */ 333 arm_lpae_iopte *tblp; 334 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 335 336 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); 337 if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz)) 338 return -EINVAL; 339 } 340 341 __arm_lpae_init_pte(data, paddr, prot, lvl, ptep); 342 return 0; 343 } 344 345 static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, 346 arm_lpae_iopte *ptep, 347 arm_lpae_iopte curr, 348 struct io_pgtable_cfg *cfg) 349 { 350 arm_lpae_iopte old, new; 351 352 new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE; 353 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) 354 new |= ARM_LPAE_PTE_NSTABLE; 355 356 /* 357 * Ensure the table itself is visible before its PTE can be. 358 * Whilst we could get away with cmpxchg64_release below, this 359 * doesn't have any ordering semantics when !CONFIG_SMP. 360 */ 361 dma_wmb(); 362 363 old = cmpxchg64_relaxed(ptep, curr, new); 364 365 if ((cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) || 366 (old & ARM_LPAE_PTE_SW_SYNC)) 367 return old; 368 369 /* Even if it's not ours, there's no point waiting; just kick it */ 370 __arm_lpae_sync_pte(ptep, cfg); 371 if (old == curr) 372 WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC); 373 374 return old; 375 } 376 377 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, 378 phys_addr_t paddr, size_t size, arm_lpae_iopte prot, 379 int lvl, arm_lpae_iopte *ptep) 380 { 381 arm_lpae_iopte *cptep, pte; 382 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 383 size_t tblsz = ARM_LPAE_GRANULE(data); 384 struct io_pgtable_cfg *cfg = &data->iop.cfg; 385 386 /* Find our entry at the current level */ 387 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 388 389 /* If we can install a leaf entry at this level, then do so */ 390 if (size == block_size && (size & cfg->pgsize_bitmap)) 391 return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); 392 393 /* We can't allocate tables at the final level */ 394 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) 395 return -EINVAL; 396 397 /* Grab a pointer to the next level */ 398 pte = READ_ONCE(*ptep); 399 if (!pte) { 400 cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg); 401 if (!cptep) 402 return -ENOMEM; 403 404 pte = arm_lpae_install_table(cptep, ptep, 0, cfg); 405 if (pte) 406 __arm_lpae_free_pages(cptep, tblsz, cfg); 407 } else if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) && 408 !(pte & ARM_LPAE_PTE_SW_SYNC)) { 409 __arm_lpae_sync_pte(ptep, cfg); 410 } 411 412 if (pte && !iopte_leaf(pte, lvl)) { 413 cptep = iopte_deref(pte, data); 414 } else if (pte) { 415 /* We require an unmap first */ 416 WARN_ON(!selftest_running); 417 return -EEXIST; 418 } 419 420 /* Rinse, repeat */ 421 return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); 422 } 423 424 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, 425 int prot) 426 { 427 arm_lpae_iopte pte; 428 429 if (data->iop.fmt == ARM_64_LPAE_S1 || 430 data->iop.fmt == ARM_32_LPAE_S1) { 431 pte = ARM_LPAE_PTE_nG; 432 433 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) 434 pte |= ARM_LPAE_PTE_AP_RDONLY; 435 436 if (!(prot & IOMMU_PRIV)) 437 pte |= ARM_LPAE_PTE_AP_UNPRIV; 438 439 if (prot & IOMMU_MMIO) 440 pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV 441 << ARM_LPAE_PTE_ATTRINDX_SHIFT); 442 else if (prot & IOMMU_CACHE) 443 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE 444 << ARM_LPAE_PTE_ATTRINDX_SHIFT); 445 } else { 446 pte = ARM_LPAE_PTE_HAP_FAULT; 447 if (prot & IOMMU_READ) 448 pte |= ARM_LPAE_PTE_HAP_READ; 449 if (prot & IOMMU_WRITE) 450 pte |= ARM_LPAE_PTE_HAP_WRITE; 451 if (prot & IOMMU_MMIO) 452 pte |= ARM_LPAE_PTE_MEMATTR_DEV; 453 else if (prot & IOMMU_CACHE) 454 pte |= ARM_LPAE_PTE_MEMATTR_OIWB; 455 else 456 pte |= ARM_LPAE_PTE_MEMATTR_NC; 457 } 458 459 if (prot & IOMMU_NOEXEC) 460 pte |= ARM_LPAE_PTE_XN; 461 462 return pte; 463 } 464 465 static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, 466 phys_addr_t paddr, size_t size, int iommu_prot) 467 { 468 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 469 arm_lpae_iopte *ptep = data->pgd; 470 int ret, lvl = ARM_LPAE_START_LVL(data); 471 arm_lpae_iopte prot; 472 473 /* If no access, then nothing to do */ 474 if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) 475 return 0; 476 477 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || 478 paddr >= (1ULL << data->iop.cfg.oas))) 479 return -ERANGE; 480 481 prot = arm_lpae_prot_to_pte(data, iommu_prot); 482 ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); 483 /* 484 * Synchronise all PTE updates for the new mapping before there's 485 * a chance for anything to kick off a table walk for the new iova. 486 */ 487 wmb(); 488 489 return ret; 490 } 491 492 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, 493 arm_lpae_iopte *ptep) 494 { 495 arm_lpae_iopte *start, *end; 496 unsigned long table_size; 497 498 if (lvl == ARM_LPAE_START_LVL(data)) 499 table_size = data->pgd_size; 500 else 501 table_size = ARM_LPAE_GRANULE(data); 502 503 start = ptep; 504 505 /* Only leaf entries at the last level */ 506 if (lvl == ARM_LPAE_MAX_LEVELS - 1) 507 end = ptep; 508 else 509 end = (void *)ptep + table_size; 510 511 while (ptep != end) { 512 arm_lpae_iopte pte = *ptep++; 513 514 if (!pte || iopte_leaf(pte, lvl)) 515 continue; 516 517 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); 518 } 519 520 __arm_lpae_free_pages(start, table_size, &data->iop.cfg); 521 } 522 523 static void arm_lpae_free_pgtable(struct io_pgtable *iop) 524 { 525 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); 526 527 __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd); 528 kfree(data); 529 } 530 531 static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, 532 unsigned long iova, size_t size, 533 arm_lpae_iopte blk_pte, int lvl, 534 arm_lpae_iopte *ptep) 535 { 536 struct io_pgtable_cfg *cfg = &data->iop.cfg; 537 arm_lpae_iopte pte, *tablep; 538 phys_addr_t blk_paddr; 539 size_t tablesz = ARM_LPAE_GRANULE(data); 540 size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 541 int i, unmap_idx = -1; 542 543 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 544 return 0; 545 546 tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg); 547 if (!tablep) 548 return 0; /* Bytes unmapped */ 549 550 if (size == split_sz) 551 unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data); 552 553 blk_paddr = iopte_to_paddr(blk_pte, data); 554 pte = iopte_prot(blk_pte); 555 556 for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { 557 /* Unmap! */ 558 if (i == unmap_idx) 559 continue; 560 561 __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]); 562 } 563 564 pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg); 565 if (pte != blk_pte) { 566 __arm_lpae_free_pages(tablep, tablesz, cfg); 567 /* 568 * We may race against someone unmapping another part of this 569 * block, but anything else is invalid. We can't misinterpret 570 * a page entry here since we're never at the last level. 571 */ 572 if (iopte_type(pte, lvl - 1) != ARM_LPAE_PTE_TYPE_TABLE) 573 return 0; 574 575 tablep = iopte_deref(pte, data); 576 } 577 578 if (unmap_idx < 0) 579 return __arm_lpae_unmap(data, iova, size, lvl, tablep); 580 581 io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true); 582 return size; 583 } 584 585 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 586 unsigned long iova, size_t size, int lvl, 587 arm_lpae_iopte *ptep) 588 { 589 arm_lpae_iopte pte; 590 struct io_pgtable *iop = &data->iop; 591 592 /* Something went horribly wrong and we ran out of page table */ 593 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 594 return 0; 595 596 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 597 pte = READ_ONCE(*ptep); 598 if (WARN_ON(!pte)) 599 return 0; 600 601 /* If the size matches this level, we're in the right place */ 602 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { 603 __arm_lpae_set_pte(ptep, 0, &iop->cfg); 604 605 if (!iopte_leaf(pte, lvl)) { 606 /* Also flush any partial walks */ 607 io_pgtable_tlb_add_flush(iop, iova, size, 608 ARM_LPAE_GRANULE(data), false); 609 io_pgtable_tlb_sync(iop); 610 ptep = iopte_deref(pte, data); 611 __arm_lpae_free_pgtable(data, lvl + 1, ptep); 612 } else { 613 io_pgtable_tlb_add_flush(iop, iova, size, size, true); 614 } 615 616 return size; 617 } else if (iopte_leaf(pte, lvl)) { 618 /* 619 * Insert a table at the next level to map the old region, 620 * minus the part we want to unmap 621 */ 622 return arm_lpae_split_blk_unmap(data, iova, size, pte, 623 lvl + 1, ptep); 624 } 625 626 /* Keep on walkin' */ 627 ptep = iopte_deref(pte, data); 628 return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep); 629 } 630 631 static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, 632 size_t size) 633 { 634 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 635 arm_lpae_iopte *ptep = data->pgd; 636 int lvl = ARM_LPAE_START_LVL(data); 637 638 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) 639 return 0; 640 641 return __arm_lpae_unmap(data, iova, size, lvl, ptep); 642 } 643 644 static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, 645 unsigned long iova) 646 { 647 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 648 arm_lpae_iopte pte, *ptep = data->pgd; 649 int lvl = ARM_LPAE_START_LVL(data); 650 651 do { 652 /* Valid IOPTE pointer? */ 653 if (!ptep) 654 return 0; 655 656 /* Grab the IOPTE we're interested in */ 657 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 658 pte = READ_ONCE(*ptep); 659 660 /* Valid entry? */ 661 if (!pte) 662 return 0; 663 664 /* Leaf entry? */ 665 if (iopte_leaf(pte,lvl)) 666 goto found_translation; 667 668 /* Take it to the next level */ 669 ptep = iopte_deref(pte, data); 670 } while (++lvl < ARM_LPAE_MAX_LEVELS); 671 672 /* Ran out of page tables to walk */ 673 return 0; 674 675 found_translation: 676 iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); 677 return iopte_to_paddr(pte, data) | iova; 678 } 679 680 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) 681 { 682 unsigned long granule, page_sizes; 683 unsigned int max_addr_bits = 48; 684 685 /* 686 * We need to restrict the supported page sizes to match the 687 * translation regime for a particular granule. Aim to match 688 * the CPU page size if possible, otherwise prefer smaller sizes. 689 * While we're at it, restrict the block sizes to match the 690 * chosen granule. 691 */ 692 if (cfg->pgsize_bitmap & PAGE_SIZE) 693 granule = PAGE_SIZE; 694 else if (cfg->pgsize_bitmap & ~PAGE_MASK) 695 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); 696 else if (cfg->pgsize_bitmap & PAGE_MASK) 697 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); 698 else 699 granule = 0; 700 701 switch (granule) { 702 case SZ_4K: 703 page_sizes = (SZ_4K | SZ_2M | SZ_1G); 704 break; 705 case SZ_16K: 706 page_sizes = (SZ_16K | SZ_32M); 707 break; 708 case SZ_64K: 709 max_addr_bits = 52; 710 page_sizes = (SZ_64K | SZ_512M); 711 if (cfg->oas > 48) 712 page_sizes |= 1ULL << 42; /* 4TB */ 713 break; 714 default: 715 page_sizes = 0; 716 } 717 718 cfg->pgsize_bitmap &= page_sizes; 719 cfg->ias = min(cfg->ias, max_addr_bits); 720 cfg->oas = min(cfg->oas, max_addr_bits); 721 } 722 723 static struct arm_lpae_io_pgtable * 724 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) 725 { 726 unsigned long va_bits, pgd_bits; 727 struct arm_lpae_io_pgtable *data; 728 729 arm_lpae_restrict_pgsizes(cfg); 730 731 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) 732 return NULL; 733 734 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) 735 return NULL; 736 737 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) 738 return NULL; 739 740 if (!selftest_running && cfg->iommu_dev->dma_pfn_offset) { 741 dev_err(cfg->iommu_dev, "Cannot accommodate DMA offset for IOMMU page tables\n"); 742 return NULL; 743 } 744 745 data = kmalloc(sizeof(*data), GFP_KERNEL); 746 if (!data) 747 return NULL; 748 749 data->pg_shift = __ffs(cfg->pgsize_bitmap); 750 data->bits_per_level = data->pg_shift - ilog2(sizeof(arm_lpae_iopte)); 751 752 va_bits = cfg->ias - data->pg_shift; 753 data->levels = DIV_ROUND_UP(va_bits, data->bits_per_level); 754 755 /* Calculate the actual size of our pgd (without concatenation) */ 756 pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1)); 757 data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte))); 758 759 data->iop.ops = (struct io_pgtable_ops) { 760 .map = arm_lpae_map, 761 .unmap = arm_lpae_unmap, 762 .iova_to_phys = arm_lpae_iova_to_phys, 763 }; 764 765 return data; 766 } 767 768 static struct io_pgtable * 769 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 770 { 771 u64 reg; 772 struct arm_lpae_io_pgtable *data; 773 774 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA)) 775 return NULL; 776 777 data = arm_lpae_alloc_pgtable(cfg); 778 if (!data) 779 return NULL; 780 781 /* TCR */ 782 reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | 783 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | 784 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); 785 786 switch (ARM_LPAE_GRANULE(data)) { 787 case SZ_4K: 788 reg |= ARM_LPAE_TCR_TG0_4K; 789 break; 790 case SZ_16K: 791 reg |= ARM_LPAE_TCR_TG0_16K; 792 break; 793 case SZ_64K: 794 reg |= ARM_LPAE_TCR_TG0_64K; 795 break; 796 } 797 798 switch (cfg->oas) { 799 case 32: 800 reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT); 801 break; 802 case 36: 803 reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT); 804 break; 805 case 40: 806 reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT); 807 break; 808 case 42: 809 reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT); 810 break; 811 case 44: 812 reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT); 813 break; 814 case 48: 815 reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); 816 break; 817 case 52: 818 reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_IPS_SHIFT); 819 break; 820 default: 821 goto out_free_data; 822 } 823 824 reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; 825 826 /* Disable speculative walks through TTBR1 */ 827 reg |= ARM_LPAE_TCR_EPD1; 828 cfg->arm_lpae_s1_cfg.tcr = reg; 829 830 /* MAIRs */ 831 reg = (ARM_LPAE_MAIR_ATTR_NC 832 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | 833 (ARM_LPAE_MAIR_ATTR_WBRWA 834 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | 835 (ARM_LPAE_MAIR_ATTR_DEVICE 836 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); 837 838 cfg->arm_lpae_s1_cfg.mair[0] = reg; 839 cfg->arm_lpae_s1_cfg.mair[1] = 0; 840 841 /* Looking good; allocate a pgd */ 842 data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); 843 if (!data->pgd) 844 goto out_free_data; 845 846 /* Ensure the empty pgd is visible before any actual TTBR write */ 847 wmb(); 848 849 /* TTBRs */ 850 cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); 851 cfg->arm_lpae_s1_cfg.ttbr[1] = 0; 852 return &data->iop; 853 854 out_free_data: 855 kfree(data); 856 return NULL; 857 } 858 859 static struct io_pgtable * 860 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 861 { 862 u64 reg, sl; 863 struct arm_lpae_io_pgtable *data; 864 865 /* The NS quirk doesn't apply at stage 2 */ 866 if (cfg->quirks & ~IO_PGTABLE_QUIRK_NO_DMA) 867 return NULL; 868 869 data = arm_lpae_alloc_pgtable(cfg); 870 if (!data) 871 return NULL; 872 873 /* 874 * Concatenate PGDs at level 1 if possible in order to reduce 875 * the depth of the stage-2 walk. 876 */ 877 if (data->levels == ARM_LPAE_MAX_LEVELS) { 878 unsigned long pgd_pages; 879 880 pgd_pages = data->pgd_size >> ilog2(sizeof(arm_lpae_iopte)); 881 if (pgd_pages <= ARM_LPAE_S2_MAX_CONCAT_PAGES) { 882 data->pgd_size = pgd_pages << data->pg_shift; 883 data->levels--; 884 } 885 } 886 887 /* VTCR */ 888 reg = ARM_64_LPAE_S2_TCR_RES1 | 889 (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | 890 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | 891 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); 892 893 sl = ARM_LPAE_START_LVL(data); 894 895 switch (ARM_LPAE_GRANULE(data)) { 896 case SZ_4K: 897 reg |= ARM_LPAE_TCR_TG0_4K; 898 sl++; /* SL0 format is different for 4K granule size */ 899 break; 900 case SZ_16K: 901 reg |= ARM_LPAE_TCR_TG0_16K; 902 break; 903 case SZ_64K: 904 reg |= ARM_LPAE_TCR_TG0_64K; 905 break; 906 } 907 908 switch (cfg->oas) { 909 case 32: 910 reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_PS_SHIFT); 911 break; 912 case 36: 913 reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_PS_SHIFT); 914 break; 915 case 40: 916 reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_PS_SHIFT); 917 break; 918 case 42: 919 reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_PS_SHIFT); 920 break; 921 case 44: 922 reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_PS_SHIFT); 923 break; 924 case 48: 925 reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT); 926 break; 927 case 52: 928 reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_PS_SHIFT); 929 break; 930 default: 931 goto out_free_data; 932 } 933 934 reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; 935 reg |= (~sl & ARM_LPAE_TCR_SL0_MASK) << ARM_LPAE_TCR_SL0_SHIFT; 936 cfg->arm_lpae_s2_cfg.vtcr = reg; 937 938 /* Allocate pgd pages */ 939 data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); 940 if (!data->pgd) 941 goto out_free_data; 942 943 /* Ensure the empty pgd is visible before any actual TTBR write */ 944 wmb(); 945 946 /* VTTBR */ 947 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); 948 return &data->iop; 949 950 out_free_data: 951 kfree(data); 952 return NULL; 953 } 954 955 static struct io_pgtable * 956 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 957 { 958 struct io_pgtable *iop; 959 960 if (cfg->ias > 32 || cfg->oas > 40) 961 return NULL; 962 963 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 964 iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie); 965 if (iop) { 966 cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE; 967 cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff; 968 } 969 970 return iop; 971 } 972 973 static struct io_pgtable * 974 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 975 { 976 struct io_pgtable *iop; 977 978 if (cfg->ias > 40 || cfg->oas > 40) 979 return NULL; 980 981 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 982 iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie); 983 if (iop) 984 cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff; 985 986 return iop; 987 } 988 989 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { 990 .alloc = arm_64_lpae_alloc_pgtable_s1, 991 .free = arm_lpae_free_pgtable, 992 }; 993 994 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = { 995 .alloc = arm_64_lpae_alloc_pgtable_s2, 996 .free = arm_lpae_free_pgtable, 997 }; 998 999 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = { 1000 .alloc = arm_32_lpae_alloc_pgtable_s1, 1001 .free = arm_lpae_free_pgtable, 1002 }; 1003 1004 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = { 1005 .alloc = arm_32_lpae_alloc_pgtable_s2, 1006 .free = arm_lpae_free_pgtable, 1007 }; 1008 1009 #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST 1010 1011 static struct io_pgtable_cfg *cfg_cookie; 1012 1013 static void dummy_tlb_flush_all(void *cookie) 1014 { 1015 WARN_ON(cookie != cfg_cookie); 1016 } 1017 1018 static void dummy_tlb_add_flush(unsigned long iova, size_t size, 1019 size_t granule, bool leaf, void *cookie) 1020 { 1021 WARN_ON(cookie != cfg_cookie); 1022 WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); 1023 } 1024 1025 static void dummy_tlb_sync(void *cookie) 1026 { 1027 WARN_ON(cookie != cfg_cookie); 1028 } 1029 1030 static const struct iommu_gather_ops dummy_tlb_ops __initconst = { 1031 .tlb_flush_all = dummy_tlb_flush_all, 1032 .tlb_add_flush = dummy_tlb_add_flush, 1033 .tlb_sync = dummy_tlb_sync, 1034 }; 1035 1036 static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) 1037 { 1038 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 1039 struct io_pgtable_cfg *cfg = &data->iop.cfg; 1040 1041 pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", 1042 cfg->pgsize_bitmap, cfg->ias); 1043 pr_err("data: %d levels, 0x%zx pgd_size, %lu pg_shift, %lu bits_per_level, pgd @ %p\n", 1044 data->levels, data->pgd_size, data->pg_shift, 1045 data->bits_per_level, data->pgd); 1046 } 1047 1048 #define __FAIL(ops, i) ({ \ 1049 WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \ 1050 arm_lpae_dump_ops(ops); \ 1051 selftest_running = false; \ 1052 -EFAULT; \ 1053 }) 1054 1055 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) 1056 { 1057 static const enum io_pgtable_fmt fmts[] = { 1058 ARM_64_LPAE_S1, 1059 ARM_64_LPAE_S2, 1060 }; 1061 1062 int i, j; 1063 unsigned long iova; 1064 size_t size; 1065 struct io_pgtable_ops *ops; 1066 1067 selftest_running = true; 1068 1069 for (i = 0; i < ARRAY_SIZE(fmts); ++i) { 1070 cfg_cookie = cfg; 1071 ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg); 1072 if (!ops) { 1073 pr_err("selftest: failed to allocate io pgtable ops\n"); 1074 return -ENOMEM; 1075 } 1076 1077 /* 1078 * Initial sanity checks. 1079 * Empty page tables shouldn't provide any translations. 1080 */ 1081 if (ops->iova_to_phys(ops, 42)) 1082 return __FAIL(ops, i); 1083 1084 if (ops->iova_to_phys(ops, SZ_1G + 42)) 1085 return __FAIL(ops, i); 1086 1087 if (ops->iova_to_phys(ops, SZ_2G + 42)) 1088 return __FAIL(ops, i); 1089 1090 /* 1091 * Distinct mappings of different granule sizes. 1092 */ 1093 iova = 0; 1094 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { 1095 size = 1UL << j; 1096 1097 if (ops->map(ops, iova, iova, size, IOMMU_READ | 1098 IOMMU_WRITE | 1099 IOMMU_NOEXEC | 1100 IOMMU_CACHE)) 1101 return __FAIL(ops, i); 1102 1103 /* Overlapping mappings */ 1104 if (!ops->map(ops, iova, iova + size, size, 1105 IOMMU_READ | IOMMU_NOEXEC)) 1106 return __FAIL(ops, i); 1107 1108 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) 1109 return __FAIL(ops, i); 1110 1111 iova += SZ_1G; 1112 } 1113 1114 /* Partial unmap */ 1115 size = 1UL << __ffs(cfg->pgsize_bitmap); 1116 if (ops->unmap(ops, SZ_1G + size, size) != size) 1117 return __FAIL(ops, i); 1118 1119 /* Remap of partial unmap */ 1120 if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ)) 1121 return __FAIL(ops, i); 1122 1123 if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) 1124 return __FAIL(ops, i); 1125 1126 /* Full unmap */ 1127 iova = 0; 1128 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { 1129 size = 1UL << j; 1130 1131 if (ops->unmap(ops, iova, size) != size) 1132 return __FAIL(ops, i); 1133 1134 if (ops->iova_to_phys(ops, iova + 42)) 1135 return __FAIL(ops, i); 1136 1137 /* Remap full block */ 1138 if (ops->map(ops, iova, iova, size, IOMMU_WRITE)) 1139 return __FAIL(ops, i); 1140 1141 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) 1142 return __FAIL(ops, i); 1143 1144 iova += SZ_1G; 1145 } 1146 1147 free_io_pgtable_ops(ops); 1148 } 1149 1150 selftest_running = false; 1151 return 0; 1152 } 1153 1154 static int __init arm_lpae_do_selftests(void) 1155 { 1156 static const unsigned long pgsize[] = { 1157 SZ_4K | SZ_2M | SZ_1G, 1158 SZ_16K | SZ_32M, 1159 SZ_64K | SZ_512M, 1160 }; 1161 1162 static const unsigned int ias[] = { 1163 32, 36, 40, 42, 44, 48, 1164 }; 1165 1166 int i, j, pass = 0, fail = 0; 1167 struct io_pgtable_cfg cfg = { 1168 .tlb = &dummy_tlb_ops, 1169 .oas = 48, 1170 .quirks = IO_PGTABLE_QUIRK_NO_DMA, 1171 }; 1172 1173 for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { 1174 for (j = 0; j < ARRAY_SIZE(ias); ++j) { 1175 cfg.pgsize_bitmap = pgsize[i]; 1176 cfg.ias = ias[j]; 1177 pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n", 1178 pgsize[i], ias[j]); 1179 if (arm_lpae_run_tests(&cfg)) 1180 fail++; 1181 else 1182 pass++; 1183 } 1184 } 1185 1186 pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail); 1187 return fail ? -EFAULT : 0; 1188 } 1189 subsys_initcall(arm_lpae_do_selftests); 1190 #endif 1191