1 /* 2 * CPU-agnostic ARM page table allocator. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * along with this program. If not, see <http://www.gnu.org/licenses/>. 15 * 16 * Copyright (C) 2014 ARM Limited 17 * 18 * Author: Will Deacon <will.deacon@arm.com> 19 */ 20 21 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt 22 23 #include <linux/atomic.h> 24 #include <linux/iommu.h> 25 #include <linux/kernel.h> 26 #include <linux/sizes.h> 27 #include <linux/slab.h> 28 #include <linux/types.h> 29 #include <linux/dma-mapping.h> 30 31 #include <asm/barrier.h> 32 33 #include "io-pgtable.h" 34 35 #define ARM_LPAE_MAX_ADDR_BITS 48 36 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 37 #define ARM_LPAE_MAX_LEVELS 4 38 39 /* Struct accessors */ 40 #define io_pgtable_to_data(x) \ 41 container_of((x), struct arm_lpae_io_pgtable, iop) 42 43 #define io_pgtable_ops_to_data(x) \ 44 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) 45 46 /* 47 * For consistency with the architecture, we always consider 48 * ARM_LPAE_MAX_LEVELS levels, with the walk starting at level n >=0 49 */ 50 #define ARM_LPAE_START_LVL(d) (ARM_LPAE_MAX_LEVELS - (d)->levels) 51 52 /* 53 * Calculate the right shift amount to get to the portion describing level l 54 * in a virtual address mapped by the pagetable in d. 55 */ 56 #define ARM_LPAE_LVL_SHIFT(l,d) \ 57 ((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1)) \ 58 * (d)->bits_per_level) + (d)->pg_shift) 59 60 #define ARM_LPAE_GRANULE(d) (1UL << (d)->pg_shift) 61 62 #define ARM_LPAE_PAGES_PER_PGD(d) \ 63 DIV_ROUND_UP((d)->pgd_size, ARM_LPAE_GRANULE(d)) 64 65 /* 66 * Calculate the index at level l used to map virtual address a using the 67 * pagetable in d. 68 */ 69 #define ARM_LPAE_PGD_IDX(l,d) \ 70 ((l) == ARM_LPAE_START_LVL(d) ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0) 71 72 #define ARM_LPAE_LVL_IDX(a,l,d) \ 73 (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ 74 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1)) 75 76 /* Calculate the block/page mapping size at level l for pagetable in d. */ 77 #define ARM_LPAE_BLOCK_SIZE(l,d) \ 78 (1ULL << (ilog2(sizeof(arm_lpae_iopte)) + \ 79 ((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level))) 80 81 /* Page table bits */ 82 #define ARM_LPAE_PTE_TYPE_SHIFT 0 83 #define ARM_LPAE_PTE_TYPE_MASK 0x3 84 85 #define ARM_LPAE_PTE_TYPE_BLOCK 1 86 #define ARM_LPAE_PTE_TYPE_TABLE 3 87 #define ARM_LPAE_PTE_TYPE_PAGE 3 88 89 #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) 90 #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) 91 #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) 92 #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) 93 #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) 94 #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8) 95 #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5) 96 #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0) 97 98 #define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2) 99 /* Ignore the contiguous bit for block splitting */ 100 #define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52) 101 #define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \ 102 ARM_LPAE_PTE_ATTR_HI_MASK) 103 /* Software bit for solving coherency races */ 104 #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) 105 106 /* Stage-1 PTE */ 107 #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) 108 #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6) 109 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 110 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) 111 112 /* Stage-2 PTE */ 113 #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6) 114 #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6) 115 #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6) 116 #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2) 117 #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2) 118 #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) 119 120 /* Register bits */ 121 #define ARM_32_LPAE_TCR_EAE (1 << 31) 122 #define ARM_64_LPAE_S2_TCR_RES1 (1 << 31) 123 124 #define ARM_LPAE_TCR_EPD1 (1 << 23) 125 126 #define ARM_LPAE_TCR_TG0_4K (0 << 14) 127 #define ARM_LPAE_TCR_TG0_64K (1 << 14) 128 #define ARM_LPAE_TCR_TG0_16K (2 << 14) 129 130 #define ARM_LPAE_TCR_SH0_SHIFT 12 131 #define ARM_LPAE_TCR_SH0_MASK 0x3 132 #define ARM_LPAE_TCR_SH_NS 0 133 #define ARM_LPAE_TCR_SH_OS 2 134 #define ARM_LPAE_TCR_SH_IS 3 135 136 #define ARM_LPAE_TCR_ORGN0_SHIFT 10 137 #define ARM_LPAE_TCR_IRGN0_SHIFT 8 138 #define ARM_LPAE_TCR_RGN_MASK 0x3 139 #define ARM_LPAE_TCR_RGN_NC 0 140 #define ARM_LPAE_TCR_RGN_WBWA 1 141 #define ARM_LPAE_TCR_RGN_WT 2 142 #define ARM_LPAE_TCR_RGN_WB 3 143 144 #define ARM_LPAE_TCR_SL0_SHIFT 6 145 #define ARM_LPAE_TCR_SL0_MASK 0x3 146 147 #define ARM_LPAE_TCR_T0SZ_SHIFT 0 148 #define ARM_LPAE_TCR_SZ_MASK 0xf 149 150 #define ARM_LPAE_TCR_PS_SHIFT 16 151 #define ARM_LPAE_TCR_PS_MASK 0x7 152 153 #define ARM_LPAE_TCR_IPS_SHIFT 32 154 #define ARM_LPAE_TCR_IPS_MASK 0x7 155 156 #define ARM_LPAE_TCR_PS_32_BIT 0x0ULL 157 #define ARM_LPAE_TCR_PS_36_BIT 0x1ULL 158 #define ARM_LPAE_TCR_PS_40_BIT 0x2ULL 159 #define ARM_LPAE_TCR_PS_42_BIT 0x3ULL 160 #define ARM_LPAE_TCR_PS_44_BIT 0x4ULL 161 #define ARM_LPAE_TCR_PS_48_BIT 0x5ULL 162 163 #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) 164 #define ARM_LPAE_MAIR_ATTR_MASK 0xff 165 #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 166 #define ARM_LPAE_MAIR_ATTR_NC 0x44 167 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff 168 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0 169 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 170 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 171 172 /* IOPTE accessors */ 173 #define iopte_deref(pte,d) \ 174 (__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \ 175 & ~(ARM_LPAE_GRANULE(d) - 1ULL))) 176 177 #define iopte_type(pte,l) \ 178 (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) 179 180 #define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK) 181 182 #define iopte_leaf(pte,l) \ 183 (l == (ARM_LPAE_MAX_LEVELS - 1) ? \ 184 (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) : \ 185 (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK)) 186 187 #define iopte_to_pfn(pte,d) \ 188 (((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift) 189 190 #define pfn_to_iopte(pfn,d) \ 191 (((pfn) << (d)->pg_shift) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) 192 193 struct arm_lpae_io_pgtable { 194 struct io_pgtable iop; 195 196 int levels; 197 size_t pgd_size; 198 unsigned long pg_shift; 199 unsigned long bits_per_level; 200 201 void *pgd; 202 }; 203 204 typedef u64 arm_lpae_iopte; 205 206 static bool selftest_running = false; 207 208 static dma_addr_t __arm_lpae_dma_addr(void *pages) 209 { 210 return (dma_addr_t)virt_to_phys(pages); 211 } 212 213 static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, 214 struct io_pgtable_cfg *cfg) 215 { 216 struct device *dev = cfg->iommu_dev; 217 dma_addr_t dma; 218 void *pages = alloc_pages_exact(size, gfp | __GFP_ZERO); 219 220 if (!pages) 221 return NULL; 222 223 if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { 224 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); 225 if (dma_mapping_error(dev, dma)) 226 goto out_free; 227 /* 228 * We depend on the IOMMU being able to work with any physical 229 * address directly, so if the DMA layer suggests otherwise by 230 * translating or truncating them, that bodes very badly... 231 */ 232 if (dma != virt_to_phys(pages)) 233 goto out_unmap; 234 } 235 236 return pages; 237 238 out_unmap: 239 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); 240 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); 241 out_free: 242 free_pages_exact(pages, size); 243 return NULL; 244 } 245 246 static void __arm_lpae_free_pages(void *pages, size_t size, 247 struct io_pgtable_cfg *cfg) 248 { 249 if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) 250 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), 251 size, DMA_TO_DEVICE); 252 free_pages_exact(pages, size); 253 } 254 255 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, 256 struct io_pgtable_cfg *cfg) 257 { 258 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), 259 sizeof(*ptep), DMA_TO_DEVICE); 260 } 261 262 static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, 263 struct io_pgtable_cfg *cfg) 264 { 265 *ptep = pte; 266 267 if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) 268 __arm_lpae_sync_pte(ptep, cfg); 269 } 270 271 static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 272 unsigned long iova, size_t size, int lvl, 273 arm_lpae_iopte *ptep); 274 275 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 276 phys_addr_t paddr, arm_lpae_iopte prot, 277 int lvl, arm_lpae_iopte *ptep) 278 { 279 arm_lpae_iopte pte = prot; 280 281 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) 282 pte |= ARM_LPAE_PTE_NS; 283 284 if (lvl == ARM_LPAE_MAX_LEVELS - 1) 285 pte |= ARM_LPAE_PTE_TYPE_PAGE; 286 else 287 pte |= ARM_LPAE_PTE_TYPE_BLOCK; 288 289 pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; 290 pte |= pfn_to_iopte(paddr >> data->pg_shift, data); 291 292 __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); 293 } 294 295 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 296 unsigned long iova, phys_addr_t paddr, 297 arm_lpae_iopte prot, int lvl, 298 arm_lpae_iopte *ptep) 299 { 300 arm_lpae_iopte pte = *ptep; 301 302 if (iopte_leaf(pte, lvl)) { 303 /* We require an unmap first */ 304 WARN_ON(!selftest_running); 305 return -EEXIST; 306 } else if (iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_TABLE) { 307 /* 308 * We need to unmap and free the old table before 309 * overwriting it with a block entry. 310 */ 311 arm_lpae_iopte *tblp; 312 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 313 314 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); 315 if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz)) 316 return -EINVAL; 317 } 318 319 __arm_lpae_init_pte(data, paddr, prot, lvl, ptep); 320 return 0; 321 } 322 323 static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, 324 arm_lpae_iopte *ptep, 325 arm_lpae_iopte curr, 326 struct io_pgtable_cfg *cfg) 327 { 328 arm_lpae_iopte old, new; 329 330 new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE; 331 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) 332 new |= ARM_LPAE_PTE_NSTABLE; 333 334 /* 335 * Ensure the table itself is visible before its PTE can be. 336 * Whilst we could get away with cmpxchg64_release below, this 337 * doesn't have any ordering semantics when !CONFIG_SMP. 338 */ 339 dma_wmb(); 340 341 old = cmpxchg64_relaxed(ptep, curr, new); 342 343 if ((cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) || 344 (old & ARM_LPAE_PTE_SW_SYNC)) 345 return old; 346 347 /* Even if it's not ours, there's no point waiting; just kick it */ 348 __arm_lpae_sync_pte(ptep, cfg); 349 if (old == curr) 350 WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC); 351 352 return old; 353 } 354 355 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, 356 phys_addr_t paddr, size_t size, arm_lpae_iopte prot, 357 int lvl, arm_lpae_iopte *ptep) 358 { 359 arm_lpae_iopte *cptep, pte; 360 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 361 size_t tblsz = ARM_LPAE_GRANULE(data); 362 struct io_pgtable_cfg *cfg = &data->iop.cfg; 363 364 /* Find our entry at the current level */ 365 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 366 367 /* If we can install a leaf entry at this level, then do so */ 368 if (size == block_size && (size & cfg->pgsize_bitmap)) 369 return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); 370 371 /* We can't allocate tables at the final level */ 372 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) 373 return -EINVAL; 374 375 /* Grab a pointer to the next level */ 376 pte = READ_ONCE(*ptep); 377 if (!pte) { 378 cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg); 379 if (!cptep) 380 return -ENOMEM; 381 382 pte = arm_lpae_install_table(cptep, ptep, 0, cfg); 383 if (pte) 384 __arm_lpae_free_pages(cptep, tblsz, cfg); 385 } else if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) && 386 !(pte & ARM_LPAE_PTE_SW_SYNC)) { 387 __arm_lpae_sync_pte(ptep, cfg); 388 } 389 390 if (pte && !iopte_leaf(pte, lvl)) { 391 cptep = iopte_deref(pte, data); 392 } else if (pte) { 393 /* We require an unmap first */ 394 WARN_ON(!selftest_running); 395 return -EEXIST; 396 } 397 398 /* Rinse, repeat */ 399 return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); 400 } 401 402 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, 403 int prot) 404 { 405 arm_lpae_iopte pte; 406 407 if (data->iop.fmt == ARM_64_LPAE_S1 || 408 data->iop.fmt == ARM_32_LPAE_S1) { 409 pte = ARM_LPAE_PTE_nG; 410 411 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) 412 pte |= ARM_LPAE_PTE_AP_RDONLY; 413 414 if (!(prot & IOMMU_PRIV)) 415 pte |= ARM_LPAE_PTE_AP_UNPRIV; 416 417 if (prot & IOMMU_MMIO) 418 pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV 419 << ARM_LPAE_PTE_ATTRINDX_SHIFT); 420 else if (prot & IOMMU_CACHE) 421 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE 422 << ARM_LPAE_PTE_ATTRINDX_SHIFT); 423 } else { 424 pte = ARM_LPAE_PTE_HAP_FAULT; 425 if (prot & IOMMU_READ) 426 pte |= ARM_LPAE_PTE_HAP_READ; 427 if (prot & IOMMU_WRITE) 428 pte |= ARM_LPAE_PTE_HAP_WRITE; 429 if (prot & IOMMU_MMIO) 430 pte |= ARM_LPAE_PTE_MEMATTR_DEV; 431 else if (prot & IOMMU_CACHE) 432 pte |= ARM_LPAE_PTE_MEMATTR_OIWB; 433 else 434 pte |= ARM_LPAE_PTE_MEMATTR_NC; 435 } 436 437 if (prot & IOMMU_NOEXEC) 438 pte |= ARM_LPAE_PTE_XN; 439 440 return pte; 441 } 442 443 static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, 444 phys_addr_t paddr, size_t size, int iommu_prot) 445 { 446 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 447 arm_lpae_iopte *ptep = data->pgd; 448 int ret, lvl = ARM_LPAE_START_LVL(data); 449 arm_lpae_iopte prot; 450 451 /* If no access, then nothing to do */ 452 if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) 453 return 0; 454 455 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || 456 paddr >= (1ULL << data->iop.cfg.oas))) 457 return -ERANGE; 458 459 prot = arm_lpae_prot_to_pte(data, iommu_prot); 460 ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); 461 /* 462 * Synchronise all PTE updates for the new mapping before there's 463 * a chance for anything to kick off a table walk for the new iova. 464 */ 465 wmb(); 466 467 return ret; 468 } 469 470 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, 471 arm_lpae_iopte *ptep) 472 { 473 arm_lpae_iopte *start, *end; 474 unsigned long table_size; 475 476 if (lvl == ARM_LPAE_START_LVL(data)) 477 table_size = data->pgd_size; 478 else 479 table_size = ARM_LPAE_GRANULE(data); 480 481 start = ptep; 482 483 /* Only leaf entries at the last level */ 484 if (lvl == ARM_LPAE_MAX_LEVELS - 1) 485 end = ptep; 486 else 487 end = (void *)ptep + table_size; 488 489 while (ptep != end) { 490 arm_lpae_iopte pte = *ptep++; 491 492 if (!pte || iopte_leaf(pte, lvl)) 493 continue; 494 495 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); 496 } 497 498 __arm_lpae_free_pages(start, table_size, &data->iop.cfg); 499 } 500 501 static void arm_lpae_free_pgtable(struct io_pgtable *iop) 502 { 503 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); 504 505 __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd); 506 kfree(data); 507 } 508 509 static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, 510 unsigned long iova, size_t size, 511 arm_lpae_iopte blk_pte, int lvl, 512 arm_lpae_iopte *ptep) 513 { 514 struct io_pgtable_cfg *cfg = &data->iop.cfg; 515 arm_lpae_iopte pte, *tablep; 516 phys_addr_t blk_paddr; 517 size_t tablesz = ARM_LPAE_GRANULE(data); 518 size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 519 int i, unmap_idx = -1; 520 521 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 522 return 0; 523 524 tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg); 525 if (!tablep) 526 return 0; /* Bytes unmapped */ 527 528 if (size == split_sz) 529 unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data); 530 531 blk_paddr = iopte_to_pfn(blk_pte, data) << data->pg_shift; 532 pte = iopte_prot(blk_pte); 533 534 for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { 535 /* Unmap! */ 536 if (i == unmap_idx) 537 continue; 538 539 __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]); 540 } 541 542 pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg); 543 if (pte != blk_pte) { 544 __arm_lpae_free_pages(tablep, tablesz, cfg); 545 /* 546 * We may race against someone unmapping another part of this 547 * block, but anything else is invalid. We can't misinterpret 548 * a page entry here since we're never at the last level. 549 */ 550 if (iopte_type(pte, lvl - 1) != ARM_LPAE_PTE_TYPE_TABLE) 551 return 0; 552 553 tablep = iopte_deref(pte, data); 554 } 555 556 if (unmap_idx < 0) 557 return __arm_lpae_unmap(data, iova, size, lvl, tablep); 558 559 io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true); 560 return size; 561 } 562 563 static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 564 unsigned long iova, size_t size, int lvl, 565 arm_lpae_iopte *ptep) 566 { 567 arm_lpae_iopte pte; 568 struct io_pgtable *iop = &data->iop; 569 570 /* Something went horribly wrong and we ran out of page table */ 571 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 572 return 0; 573 574 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 575 pte = READ_ONCE(*ptep); 576 if (WARN_ON(!pte)) 577 return 0; 578 579 /* If the size matches this level, we're in the right place */ 580 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { 581 __arm_lpae_set_pte(ptep, 0, &iop->cfg); 582 583 if (!iopte_leaf(pte, lvl)) { 584 /* Also flush any partial walks */ 585 io_pgtable_tlb_add_flush(iop, iova, size, 586 ARM_LPAE_GRANULE(data), false); 587 io_pgtable_tlb_sync(iop); 588 ptep = iopte_deref(pte, data); 589 __arm_lpae_free_pgtable(data, lvl + 1, ptep); 590 } else { 591 io_pgtable_tlb_add_flush(iop, iova, size, size, true); 592 } 593 594 return size; 595 } else if (iopte_leaf(pte, lvl)) { 596 /* 597 * Insert a table at the next level to map the old region, 598 * minus the part we want to unmap 599 */ 600 return arm_lpae_split_blk_unmap(data, iova, size, pte, 601 lvl + 1, ptep); 602 } 603 604 /* Keep on walkin' */ 605 ptep = iopte_deref(pte, data); 606 return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep); 607 } 608 609 static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, 610 size_t size) 611 { 612 size_t unmapped; 613 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 614 arm_lpae_iopte *ptep = data->pgd; 615 int lvl = ARM_LPAE_START_LVL(data); 616 617 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) 618 return 0; 619 620 unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep); 621 if (unmapped) 622 io_pgtable_tlb_sync(&data->iop); 623 624 return unmapped; 625 } 626 627 static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, 628 unsigned long iova) 629 { 630 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 631 arm_lpae_iopte pte, *ptep = data->pgd; 632 int lvl = ARM_LPAE_START_LVL(data); 633 634 do { 635 /* Valid IOPTE pointer? */ 636 if (!ptep) 637 return 0; 638 639 /* Grab the IOPTE we're interested in */ 640 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 641 pte = READ_ONCE(*ptep); 642 643 /* Valid entry? */ 644 if (!pte) 645 return 0; 646 647 /* Leaf entry? */ 648 if (iopte_leaf(pte,lvl)) 649 goto found_translation; 650 651 /* Take it to the next level */ 652 ptep = iopte_deref(pte, data); 653 } while (++lvl < ARM_LPAE_MAX_LEVELS); 654 655 /* Ran out of page tables to walk */ 656 return 0; 657 658 found_translation: 659 iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); 660 return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; 661 } 662 663 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) 664 { 665 unsigned long granule; 666 667 /* 668 * We need to restrict the supported page sizes to match the 669 * translation regime for a particular granule. Aim to match 670 * the CPU page size if possible, otherwise prefer smaller sizes. 671 * While we're at it, restrict the block sizes to match the 672 * chosen granule. 673 */ 674 if (cfg->pgsize_bitmap & PAGE_SIZE) 675 granule = PAGE_SIZE; 676 else if (cfg->pgsize_bitmap & ~PAGE_MASK) 677 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); 678 else if (cfg->pgsize_bitmap & PAGE_MASK) 679 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); 680 else 681 granule = 0; 682 683 switch (granule) { 684 case SZ_4K: 685 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 686 break; 687 case SZ_16K: 688 cfg->pgsize_bitmap &= (SZ_16K | SZ_32M); 689 break; 690 case SZ_64K: 691 cfg->pgsize_bitmap &= (SZ_64K | SZ_512M); 692 break; 693 default: 694 cfg->pgsize_bitmap = 0; 695 } 696 } 697 698 static struct arm_lpae_io_pgtable * 699 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) 700 { 701 unsigned long va_bits, pgd_bits; 702 struct arm_lpae_io_pgtable *data; 703 704 arm_lpae_restrict_pgsizes(cfg); 705 706 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) 707 return NULL; 708 709 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) 710 return NULL; 711 712 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) 713 return NULL; 714 715 if (!selftest_running && cfg->iommu_dev->dma_pfn_offset) { 716 dev_err(cfg->iommu_dev, "Cannot accommodate DMA offset for IOMMU page tables\n"); 717 return NULL; 718 } 719 720 data = kmalloc(sizeof(*data), GFP_KERNEL); 721 if (!data) 722 return NULL; 723 724 data->pg_shift = __ffs(cfg->pgsize_bitmap); 725 data->bits_per_level = data->pg_shift - ilog2(sizeof(arm_lpae_iopte)); 726 727 va_bits = cfg->ias - data->pg_shift; 728 data->levels = DIV_ROUND_UP(va_bits, data->bits_per_level); 729 730 /* Calculate the actual size of our pgd (without concatenation) */ 731 pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1)); 732 data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte))); 733 734 data->iop.ops = (struct io_pgtable_ops) { 735 .map = arm_lpae_map, 736 .unmap = arm_lpae_unmap, 737 .iova_to_phys = arm_lpae_iova_to_phys, 738 }; 739 740 return data; 741 } 742 743 static struct io_pgtable * 744 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 745 { 746 u64 reg; 747 struct arm_lpae_io_pgtable *data; 748 749 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA)) 750 return NULL; 751 752 data = arm_lpae_alloc_pgtable(cfg); 753 if (!data) 754 return NULL; 755 756 /* TCR */ 757 reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | 758 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | 759 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); 760 761 switch (ARM_LPAE_GRANULE(data)) { 762 case SZ_4K: 763 reg |= ARM_LPAE_TCR_TG0_4K; 764 break; 765 case SZ_16K: 766 reg |= ARM_LPAE_TCR_TG0_16K; 767 break; 768 case SZ_64K: 769 reg |= ARM_LPAE_TCR_TG0_64K; 770 break; 771 } 772 773 switch (cfg->oas) { 774 case 32: 775 reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT); 776 break; 777 case 36: 778 reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT); 779 break; 780 case 40: 781 reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT); 782 break; 783 case 42: 784 reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT); 785 break; 786 case 44: 787 reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT); 788 break; 789 case 48: 790 reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); 791 break; 792 default: 793 goto out_free_data; 794 } 795 796 reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; 797 798 /* Disable speculative walks through TTBR1 */ 799 reg |= ARM_LPAE_TCR_EPD1; 800 cfg->arm_lpae_s1_cfg.tcr = reg; 801 802 /* MAIRs */ 803 reg = (ARM_LPAE_MAIR_ATTR_NC 804 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | 805 (ARM_LPAE_MAIR_ATTR_WBRWA 806 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | 807 (ARM_LPAE_MAIR_ATTR_DEVICE 808 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); 809 810 cfg->arm_lpae_s1_cfg.mair[0] = reg; 811 cfg->arm_lpae_s1_cfg.mair[1] = 0; 812 813 /* Looking good; allocate a pgd */ 814 data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); 815 if (!data->pgd) 816 goto out_free_data; 817 818 /* Ensure the empty pgd is visible before any actual TTBR write */ 819 wmb(); 820 821 /* TTBRs */ 822 cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); 823 cfg->arm_lpae_s1_cfg.ttbr[1] = 0; 824 return &data->iop; 825 826 out_free_data: 827 kfree(data); 828 return NULL; 829 } 830 831 static struct io_pgtable * 832 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 833 { 834 u64 reg, sl; 835 struct arm_lpae_io_pgtable *data; 836 837 /* The NS quirk doesn't apply at stage 2 */ 838 if (cfg->quirks & ~IO_PGTABLE_QUIRK_NO_DMA) 839 return NULL; 840 841 data = arm_lpae_alloc_pgtable(cfg); 842 if (!data) 843 return NULL; 844 845 /* 846 * Concatenate PGDs at level 1 if possible in order to reduce 847 * the depth of the stage-2 walk. 848 */ 849 if (data->levels == ARM_LPAE_MAX_LEVELS) { 850 unsigned long pgd_pages; 851 852 pgd_pages = data->pgd_size >> ilog2(sizeof(arm_lpae_iopte)); 853 if (pgd_pages <= ARM_LPAE_S2_MAX_CONCAT_PAGES) { 854 data->pgd_size = pgd_pages << data->pg_shift; 855 data->levels--; 856 } 857 } 858 859 /* VTCR */ 860 reg = ARM_64_LPAE_S2_TCR_RES1 | 861 (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | 862 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | 863 (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); 864 865 sl = ARM_LPAE_START_LVL(data); 866 867 switch (ARM_LPAE_GRANULE(data)) { 868 case SZ_4K: 869 reg |= ARM_LPAE_TCR_TG0_4K; 870 sl++; /* SL0 format is different for 4K granule size */ 871 break; 872 case SZ_16K: 873 reg |= ARM_LPAE_TCR_TG0_16K; 874 break; 875 case SZ_64K: 876 reg |= ARM_LPAE_TCR_TG0_64K; 877 break; 878 } 879 880 switch (cfg->oas) { 881 case 32: 882 reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_PS_SHIFT); 883 break; 884 case 36: 885 reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_PS_SHIFT); 886 break; 887 case 40: 888 reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_PS_SHIFT); 889 break; 890 case 42: 891 reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_PS_SHIFT); 892 break; 893 case 44: 894 reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_PS_SHIFT); 895 break; 896 case 48: 897 reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT); 898 break; 899 default: 900 goto out_free_data; 901 } 902 903 reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; 904 reg |= (~sl & ARM_LPAE_TCR_SL0_MASK) << ARM_LPAE_TCR_SL0_SHIFT; 905 cfg->arm_lpae_s2_cfg.vtcr = reg; 906 907 /* Allocate pgd pages */ 908 data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); 909 if (!data->pgd) 910 goto out_free_data; 911 912 /* Ensure the empty pgd is visible before any actual TTBR write */ 913 wmb(); 914 915 /* VTTBR */ 916 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); 917 return &data->iop; 918 919 out_free_data: 920 kfree(data); 921 return NULL; 922 } 923 924 static struct io_pgtable * 925 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 926 { 927 struct io_pgtable *iop; 928 929 if (cfg->ias > 32 || cfg->oas > 40) 930 return NULL; 931 932 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 933 iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie); 934 if (iop) { 935 cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE; 936 cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff; 937 } 938 939 return iop; 940 } 941 942 static struct io_pgtable * 943 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 944 { 945 struct io_pgtable *iop; 946 947 if (cfg->ias > 40 || cfg->oas > 40) 948 return NULL; 949 950 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 951 iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie); 952 if (iop) 953 cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff; 954 955 return iop; 956 } 957 958 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { 959 .alloc = arm_64_lpae_alloc_pgtable_s1, 960 .free = arm_lpae_free_pgtable, 961 }; 962 963 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = { 964 .alloc = arm_64_lpae_alloc_pgtable_s2, 965 .free = arm_lpae_free_pgtable, 966 }; 967 968 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = { 969 .alloc = arm_32_lpae_alloc_pgtable_s1, 970 .free = arm_lpae_free_pgtable, 971 }; 972 973 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = { 974 .alloc = arm_32_lpae_alloc_pgtable_s2, 975 .free = arm_lpae_free_pgtable, 976 }; 977 978 #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST 979 980 static struct io_pgtable_cfg *cfg_cookie; 981 982 static void dummy_tlb_flush_all(void *cookie) 983 { 984 WARN_ON(cookie != cfg_cookie); 985 } 986 987 static void dummy_tlb_add_flush(unsigned long iova, size_t size, 988 size_t granule, bool leaf, void *cookie) 989 { 990 WARN_ON(cookie != cfg_cookie); 991 WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); 992 } 993 994 static void dummy_tlb_sync(void *cookie) 995 { 996 WARN_ON(cookie != cfg_cookie); 997 } 998 999 static const struct iommu_gather_ops dummy_tlb_ops __initconst = { 1000 .tlb_flush_all = dummy_tlb_flush_all, 1001 .tlb_add_flush = dummy_tlb_add_flush, 1002 .tlb_sync = dummy_tlb_sync, 1003 }; 1004 1005 static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) 1006 { 1007 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 1008 struct io_pgtable_cfg *cfg = &data->iop.cfg; 1009 1010 pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", 1011 cfg->pgsize_bitmap, cfg->ias); 1012 pr_err("data: %d levels, 0x%zx pgd_size, %lu pg_shift, %lu bits_per_level, pgd @ %p\n", 1013 data->levels, data->pgd_size, data->pg_shift, 1014 data->bits_per_level, data->pgd); 1015 } 1016 1017 #define __FAIL(ops, i) ({ \ 1018 WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \ 1019 arm_lpae_dump_ops(ops); \ 1020 selftest_running = false; \ 1021 -EFAULT; \ 1022 }) 1023 1024 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) 1025 { 1026 static const enum io_pgtable_fmt fmts[] = { 1027 ARM_64_LPAE_S1, 1028 ARM_64_LPAE_S2, 1029 }; 1030 1031 int i, j; 1032 unsigned long iova; 1033 size_t size; 1034 struct io_pgtable_ops *ops; 1035 1036 selftest_running = true; 1037 1038 for (i = 0; i < ARRAY_SIZE(fmts); ++i) { 1039 cfg_cookie = cfg; 1040 ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg); 1041 if (!ops) { 1042 pr_err("selftest: failed to allocate io pgtable ops\n"); 1043 return -ENOMEM; 1044 } 1045 1046 /* 1047 * Initial sanity checks. 1048 * Empty page tables shouldn't provide any translations. 1049 */ 1050 if (ops->iova_to_phys(ops, 42)) 1051 return __FAIL(ops, i); 1052 1053 if (ops->iova_to_phys(ops, SZ_1G + 42)) 1054 return __FAIL(ops, i); 1055 1056 if (ops->iova_to_phys(ops, SZ_2G + 42)) 1057 return __FAIL(ops, i); 1058 1059 /* 1060 * Distinct mappings of different granule sizes. 1061 */ 1062 iova = 0; 1063 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { 1064 size = 1UL << j; 1065 1066 if (ops->map(ops, iova, iova, size, IOMMU_READ | 1067 IOMMU_WRITE | 1068 IOMMU_NOEXEC | 1069 IOMMU_CACHE)) 1070 return __FAIL(ops, i); 1071 1072 /* Overlapping mappings */ 1073 if (!ops->map(ops, iova, iova + size, size, 1074 IOMMU_READ | IOMMU_NOEXEC)) 1075 return __FAIL(ops, i); 1076 1077 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) 1078 return __FAIL(ops, i); 1079 1080 iova += SZ_1G; 1081 } 1082 1083 /* Partial unmap */ 1084 size = 1UL << __ffs(cfg->pgsize_bitmap); 1085 if (ops->unmap(ops, SZ_1G + size, size) != size) 1086 return __FAIL(ops, i); 1087 1088 /* Remap of partial unmap */ 1089 if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ)) 1090 return __FAIL(ops, i); 1091 1092 if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) 1093 return __FAIL(ops, i); 1094 1095 /* Full unmap */ 1096 iova = 0; 1097 j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG); 1098 while (j != BITS_PER_LONG) { 1099 size = 1UL << j; 1100 1101 if (ops->unmap(ops, iova, size) != size) 1102 return __FAIL(ops, i); 1103 1104 if (ops->iova_to_phys(ops, iova + 42)) 1105 return __FAIL(ops, i); 1106 1107 /* Remap full block */ 1108 if (ops->map(ops, iova, iova, size, IOMMU_WRITE)) 1109 return __FAIL(ops, i); 1110 1111 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) 1112 return __FAIL(ops, i); 1113 1114 iova += SZ_1G; 1115 j++; 1116 j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j); 1117 } 1118 1119 free_io_pgtable_ops(ops); 1120 } 1121 1122 selftest_running = false; 1123 return 0; 1124 } 1125 1126 static int __init arm_lpae_do_selftests(void) 1127 { 1128 static const unsigned long pgsize[] = { 1129 SZ_4K | SZ_2M | SZ_1G, 1130 SZ_16K | SZ_32M, 1131 SZ_64K | SZ_512M, 1132 }; 1133 1134 static const unsigned int ias[] = { 1135 32, 36, 40, 42, 44, 48, 1136 }; 1137 1138 int i, j, pass = 0, fail = 0; 1139 struct io_pgtable_cfg cfg = { 1140 .tlb = &dummy_tlb_ops, 1141 .oas = 48, 1142 .quirks = IO_PGTABLE_QUIRK_NO_DMA, 1143 }; 1144 1145 for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { 1146 for (j = 0; j < ARRAY_SIZE(ias); ++j) { 1147 cfg.pgsize_bitmap = pgsize[i]; 1148 cfg.ias = ias[j]; 1149 pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n", 1150 pgsize[i], ias[j]); 1151 if (arm_lpae_run_tests(&cfg)) 1152 fail++; 1153 else 1154 pass++; 1155 } 1156 } 1157 1158 pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail); 1159 return fail ? -EFAULT : 0; 1160 } 1161 subsys_initcall(arm_lpae_do_selftests); 1162 #endif 1163