42 	ilog2(sizeof(arm_lpae_iopte)))
45 	(sizeof(arm_lpae_iopte) << (d)->bits_per_level)
47 	(sizeof(arm_lpae_iopte) << (d)->pgd_bits)
50 	(ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))
76 #define ARM_LPAE_PTE_NSTABLE		(((arm_lpae_iopte)1) << 63)
77 #define ARM_LPAE_PTE_XN			(((arm_lpae_iopte)3) << 53)
78 #define ARM_LPAE_PTE_AF			(((arm_lpae_iopte)1) << 10)
79 #define ARM_LPAE_PTE_SH_NS		(((arm_lpae_iopte)0) << 8)
80 #define ARM_LPAE_PTE_SH_OS		(((arm_lpae_iopte)2) << 8)
81 #define ARM_LPAE_PTE_SH_IS		(((arm_lpae_iopte)3) << 8)
82 #define ARM_LPAE_PTE_NS			(((arm_lpae_iopte)1) << 5)
83 #define ARM_LPAE_PTE_VALID		(((arm_lpae_iopte)1) << 0)
85 #define ARM_LPAE_PTE_ATTR_LO_MASK	(((arm_lpae_iopte)0x3ff) << 2)
87 #define ARM_LPAE_PTE_ATTR_HI_MASK	(((arm_lpae_iopte)6) << 52)
91 #define ARM_LPAE_PTE_SW_SYNC		(((arm_lpae_iopte)1) << 55)
94 #define ARM_LPAE_PTE_AP_UNPRIV		(((arm_lpae_iopte)1) << 6)
95 #define ARM_LPAE_PTE_AP_RDONLY		(((arm_lpae_iopte)2) << 6)
97 #define ARM_LPAE_PTE_nG			(((arm_lpae_iopte)1) << 11)
100 #define ARM_LPAE_PTE_HAP_FAULT		(((arm_lpae_iopte)0) << 6)
101 #define ARM_LPAE_PTE_HAP_READ		(((arm_lpae_iopte)1) << 6)
102 #define ARM_LPAE_PTE_HAP_WRITE		(((arm_lpae_iopte)2) << 6)
103 #define ARM_LPAE_PTE_MEMATTR_OIWB	(((arm_lpae_iopte)0xf) << 2)
104 #define ARM_LPAE_PTE_MEMATTR_NC		(((arm_lpae_iopte)0x5) << 2)
105 #define ARM_LPAE_PTE_MEMATTR_DEV	(((arm_lpae_iopte)0x1) << 2)
151 typedef u64 arm_lpae_iopte;  typedef
153 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,  in iopte_leaf()
162 static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,  in paddr_to_iopte()
165 	arm_lpae_iopte pte = paddr;  in paddr_to_iopte()
171 static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,  in iopte_to_paddr()
237 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,  in __arm_lpae_sync_pte()
244 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg)  in __arm_lpae_clear_pte()
256 			       int lvl, arm_lpae_iopte *ptep);
259 				phys_addr_t paddr, arm_lpae_iopte prot,  in __arm_lpae_init_pte()
260 				int lvl, int num_entries, arm_lpae_iopte *ptep)  in __arm_lpae_init_pte()
262 	arm_lpae_iopte pte = prot;  in __arm_lpae_init_pte()
281 			     arm_lpae_iopte prot, int lvl, int num_entries,  in arm_lpae_init_pte()
282 			     arm_lpae_iopte *ptep)  in arm_lpae_init_pte()
296 			arm_lpae_iopte *tblp;  in arm_lpae_init_pte()
311 static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,  in arm_lpae_install_table()
312 					     arm_lpae_iopte *ptep,  in arm_lpae_install_table()
313 					     arm_lpae_iopte curr,  in arm_lpae_install_table()
316 	arm_lpae_iopte old, new;  in arm_lpae_install_table()
345 			  arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep,  in __arm_lpae_map()
348 	arm_lpae_iopte *cptep, pte;  in __arm_lpae_map()
400 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,  in arm_lpae_prot_to_pte()
403 	arm_lpae_iopte pte;  in arm_lpae_prot_to_pte()
470 	arm_lpae_iopte *ptep = data->pgd;  in arm_lpae_map_pages()
472 	arm_lpae_iopte prot;  in arm_lpae_map_pages()
500 				    arm_lpae_iopte *ptep)  in __arm_lpae_free_pgtable()
502 	arm_lpae_iopte *start, *end;  in __arm_lpae_free_pgtable()
519 		arm_lpae_iopte pte = *ptep++;  in __arm_lpae_free_pgtable()
541 				       arm_lpae_iopte blk_pte, int lvl,  in arm_lpae_split_blk_unmap()
542 				       arm_lpae_iopte *ptep, size_t pgcount)  in arm_lpae_split_blk_unmap()
545 	arm_lpae_iopte pte, *tablep;  in arm_lpae_split_blk_unmap()
601 			       int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_unmap()
603 	arm_lpae_iopte pte;  in __arm_lpae_unmap()
663 	arm_lpae_iopte *ptep = data->pgd;  in arm_lpae_unmap_pages()
682 	arm_lpae_iopte pte, *ptep = data->pgd;  in arm_lpae_iova_to_phys()
779 	data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte));  in arm_lpae_alloc_pgtable()
923 		pgd_pages = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte);  in arm_64_lpae_alloc_pgtable_s2()