1 /* 2 * Copyright (C) 2016 Etnaviv Project 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 as published by 6 * the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16 17 #include <linux/iommu.h> 18 #include <linux/platform_device.h> 19 #include <linux/sizes.h> 20 #include <linux/slab.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/bitops.h> 23 24 #include "etnaviv_cmdbuf.h" 25 #include "etnaviv_gpu.h" 26 #include "etnaviv_mmu.h" 27 #include "etnaviv_iommu.h" 28 #include "state.xml.h" 29 #include "state_hi.xml.h" 30 31 #define MMUv2_PTE_PRESENT BIT(0) 32 #define MMUv2_PTE_EXCEPTION BIT(1) 33 #define MMUv2_PTE_WRITEABLE BIT(2) 34 35 #define MMUv2_MTLB_MASK 0xffc00000 36 #define MMUv2_MTLB_SHIFT 22 37 #define MMUv2_STLB_MASK 0x003ff000 38 #define MMUv2_STLB_SHIFT 12 39 40 #define MMUv2_MAX_STLB_ENTRIES 1024 41 42 struct etnaviv_iommuv2_domain { 43 struct iommu_domain domain; 44 struct device *dev; 45 void *bad_page_cpu; 46 dma_addr_t bad_page_dma; 47 /* M(aster) TLB aka first level pagetable */ 48 u32 *mtlb_cpu; 49 dma_addr_t mtlb_dma; 50 /* S(lave) TLB aka second level pagetable */ 51 u32 *stlb_cpu[1024]; 52 dma_addr_t stlb_dma[1024]; 53 }; 54 55 static struct etnaviv_iommuv2_domain *to_etnaviv_domain(struct iommu_domain *domain) 56 { 57 return container_of(domain, struct etnaviv_iommuv2_domain, domain); 58 } 59 60 static int etnaviv_iommuv2_map(struct iommu_domain *domain, unsigned long iova, 61 phys_addr_t paddr, size_t size, int prot) 62 { 63 struct etnaviv_iommuv2_domain *etnaviv_domain = 64 to_etnaviv_domain(domain); 65 int mtlb_entry, stlb_entry; 66 u32 entry = (u32)paddr | MMUv2_PTE_PRESENT; 67 68 if (size != SZ_4K) 69 return -EINVAL; 70 71 if (prot & IOMMU_WRITE) 72 entry |= MMUv2_PTE_WRITEABLE; 73 74 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 75 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 76 77 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry; 78 79 return 0; 80 } 81 82 static size_t etnaviv_iommuv2_unmap(struct iommu_domain *domain, 83 unsigned long iova, size_t size) 84 { 85 struct etnaviv_iommuv2_domain *etnaviv_domain = 86 to_etnaviv_domain(domain); 87 int mtlb_entry, stlb_entry; 88 89 if (size != SZ_4K) 90 return -EINVAL; 91 92 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 93 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 94 95 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION; 96 97 return SZ_4K; 98 } 99 100 static phys_addr_t etnaviv_iommuv2_iova_to_phys(struct iommu_domain *domain, 101 dma_addr_t iova) 102 { 103 struct etnaviv_iommuv2_domain *etnaviv_domain = 104 to_etnaviv_domain(domain); 105 int mtlb_entry, stlb_entry; 106 107 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 108 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 109 110 return etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] & ~(SZ_4K - 1); 111 } 112 113 static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain) 114 { 115 u32 *p; 116 int ret, i, j; 117 118 /* allocate scratch page */ 119 etnaviv_domain->bad_page_cpu = dma_alloc_coherent(etnaviv_domain->dev, 120 SZ_4K, 121 &etnaviv_domain->bad_page_dma, 122 GFP_KERNEL); 123 if (!etnaviv_domain->bad_page_cpu) { 124 ret = -ENOMEM; 125 goto fail_mem; 126 } 127 p = etnaviv_domain->bad_page_cpu; 128 for (i = 0; i < SZ_4K / 4; i++) 129 *p++ = 0xdead55aa; 130 131 etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->dev, 132 SZ_4K, 133 &etnaviv_domain->mtlb_dma, 134 GFP_KERNEL); 135 if (!etnaviv_domain->mtlb_cpu) { 136 ret = -ENOMEM; 137 goto fail_mem; 138 } 139 140 /* pre-populate STLB pages (may want to switch to on-demand later) */ 141 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 142 etnaviv_domain->stlb_cpu[i] = 143 dma_alloc_coherent(etnaviv_domain->dev, 144 SZ_4K, 145 &etnaviv_domain->stlb_dma[i], 146 GFP_KERNEL); 147 if (!etnaviv_domain->stlb_cpu[i]) { 148 ret = -ENOMEM; 149 goto fail_mem; 150 } 151 p = etnaviv_domain->stlb_cpu[i]; 152 for (j = 0; j < SZ_4K / 4; j++) 153 *p++ = MMUv2_PTE_EXCEPTION; 154 155 etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] | 156 MMUv2_PTE_PRESENT; 157 } 158 159 return 0; 160 161 fail_mem: 162 if (etnaviv_domain->bad_page_cpu) 163 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 164 etnaviv_domain->bad_page_cpu, 165 etnaviv_domain->bad_page_dma); 166 167 if (etnaviv_domain->mtlb_cpu) 168 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 169 etnaviv_domain->mtlb_cpu, 170 etnaviv_domain->mtlb_dma); 171 172 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 173 if (etnaviv_domain->stlb_cpu[i]) 174 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 175 etnaviv_domain->stlb_cpu[i], 176 etnaviv_domain->stlb_dma[i]); 177 } 178 179 return ret; 180 } 181 182 static void etnaviv_iommuv2_domain_free(struct iommu_domain *domain) 183 { 184 struct etnaviv_iommuv2_domain *etnaviv_domain = 185 to_etnaviv_domain(domain); 186 int i; 187 188 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 189 etnaviv_domain->bad_page_cpu, 190 etnaviv_domain->bad_page_dma); 191 192 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 193 etnaviv_domain->mtlb_cpu, 194 etnaviv_domain->mtlb_dma); 195 196 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 197 if (etnaviv_domain->stlb_cpu[i]) 198 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 199 etnaviv_domain->stlb_cpu[i], 200 etnaviv_domain->stlb_dma[i]); 201 } 202 203 vfree(etnaviv_domain); 204 } 205 206 static size_t etnaviv_iommuv2_dump_size(struct iommu_domain *domain) 207 { 208 struct etnaviv_iommuv2_domain *etnaviv_domain = 209 to_etnaviv_domain(domain); 210 size_t dump_size = SZ_4K; 211 int i; 212 213 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) 214 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 215 dump_size += SZ_4K; 216 217 return dump_size; 218 } 219 220 static void etnaviv_iommuv2_dump(struct iommu_domain *domain, void *buf) 221 { 222 struct etnaviv_iommuv2_domain *etnaviv_domain = 223 to_etnaviv_domain(domain); 224 int i; 225 226 memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K); 227 buf += SZ_4K; 228 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K) 229 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 230 memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K); 231 } 232 233 static const struct etnaviv_iommu_ops etnaviv_iommu_ops = { 234 .ops = { 235 .domain_free = etnaviv_iommuv2_domain_free, 236 .map = etnaviv_iommuv2_map, 237 .unmap = etnaviv_iommuv2_unmap, 238 .iova_to_phys = etnaviv_iommuv2_iova_to_phys, 239 .pgsize_bitmap = SZ_4K, 240 }, 241 .dump_size = etnaviv_iommuv2_dump_size, 242 .dump = etnaviv_iommuv2_dump, 243 }; 244 245 void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) 246 { 247 struct etnaviv_iommuv2_domain *etnaviv_domain = 248 to_etnaviv_domain(gpu->mmu->domain); 249 u16 prefetch; 250 251 /* If the MMU is already enabled the state is still there. */ 252 if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE) 253 return; 254 255 prefetch = etnaviv_buffer_config_mmuv2(gpu, 256 (u32)etnaviv_domain->mtlb_dma, 257 (u32)etnaviv_domain->bad_page_dma); 258 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(gpu->buffer), 259 prefetch); 260 etnaviv_gpu_wait_idle(gpu, 100); 261 262 gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); 263 } 264 struct iommu_domain *etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu) 265 { 266 struct etnaviv_iommuv2_domain *etnaviv_domain; 267 int ret; 268 269 etnaviv_domain = vzalloc(sizeof(*etnaviv_domain)); 270 if (!etnaviv_domain) 271 return NULL; 272 273 etnaviv_domain->dev = gpu->dev; 274 275 etnaviv_domain->domain.type = __IOMMU_DOMAIN_PAGING; 276 etnaviv_domain->domain.ops = &etnaviv_iommu_ops.ops; 277 etnaviv_domain->domain.pgsize_bitmap = SZ_4K; 278 etnaviv_domain->domain.geometry.aperture_start = 0; 279 etnaviv_domain->domain.geometry.aperture_end = ~0UL & ~(SZ_4K - 1); 280 281 ret = etnaviv_iommuv2_init(etnaviv_domain); 282 if (ret) 283 goto out_free; 284 285 return &etnaviv_domain->domain; 286 287 out_free: 288 vfree(etnaviv_domain); 289 return NULL; 290 } 291