1 /* 2 * Copyright (C) 2016 Etnaviv Project 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 as published by 6 * the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16 17 #include <linux/iommu.h> 18 #include <linux/platform_device.h> 19 #include <linux/sizes.h> 20 #include <linux/slab.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/bitops.h> 23 24 #include "etnaviv_gpu.h" 25 #include "etnaviv_mmu.h" 26 #include "etnaviv_iommu.h" 27 #include "state.xml.h" 28 #include "state_hi.xml.h" 29 30 #define MMUv2_PTE_PRESENT BIT(0) 31 #define MMUv2_PTE_EXCEPTION BIT(1) 32 #define MMUv2_PTE_WRITEABLE BIT(2) 33 34 #define MMUv2_MTLB_MASK 0xffc00000 35 #define MMUv2_MTLB_SHIFT 22 36 #define MMUv2_STLB_MASK 0x003ff000 37 #define MMUv2_STLB_SHIFT 12 38 39 #define MMUv2_MAX_STLB_ENTRIES 1024 40 41 struct etnaviv_iommuv2_domain { 42 struct iommu_domain domain; 43 struct device *dev; 44 void *bad_page_cpu; 45 dma_addr_t bad_page_dma; 46 /* M(aster) TLB aka first level pagetable */ 47 u32 *mtlb_cpu; 48 dma_addr_t mtlb_dma; 49 /* S(lave) TLB aka second level pagetable */ 50 u32 *stlb_cpu[1024]; 51 dma_addr_t stlb_dma[1024]; 52 }; 53 54 static struct etnaviv_iommuv2_domain *to_etnaviv_domain(struct iommu_domain *domain) 55 { 56 return container_of(domain, struct etnaviv_iommuv2_domain, domain); 57 } 58 59 static int etnaviv_iommuv2_map(struct iommu_domain *domain, unsigned long iova, 60 phys_addr_t paddr, size_t size, int prot) 61 { 62 struct etnaviv_iommuv2_domain *etnaviv_domain = 63 to_etnaviv_domain(domain); 64 int mtlb_entry, stlb_entry; 65 u32 entry = (u32)paddr | MMUv2_PTE_PRESENT; 66 67 if (size != SZ_4K) 68 return -EINVAL; 69 70 if (prot & IOMMU_WRITE) 71 entry |= MMUv2_PTE_WRITEABLE; 72 73 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 74 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 75 76 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry; 77 78 return 0; 79 } 80 81 static size_t etnaviv_iommuv2_unmap(struct iommu_domain *domain, 82 unsigned long iova, size_t size) 83 { 84 struct etnaviv_iommuv2_domain *etnaviv_domain = 85 to_etnaviv_domain(domain); 86 int mtlb_entry, stlb_entry; 87 88 if (size != SZ_4K) 89 return -EINVAL; 90 91 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 92 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 93 94 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION; 95 96 return SZ_4K; 97 } 98 99 static phys_addr_t etnaviv_iommuv2_iova_to_phys(struct iommu_domain *domain, 100 dma_addr_t iova) 101 { 102 struct etnaviv_iommuv2_domain *etnaviv_domain = 103 to_etnaviv_domain(domain); 104 int mtlb_entry, stlb_entry; 105 106 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 107 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 108 109 return etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] & ~(SZ_4K - 1); 110 } 111 112 static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain) 113 { 114 u32 *p; 115 int ret, i, j; 116 117 /* allocate scratch page */ 118 etnaviv_domain->bad_page_cpu = dma_alloc_coherent(etnaviv_domain->dev, 119 SZ_4K, 120 &etnaviv_domain->bad_page_dma, 121 GFP_KERNEL); 122 if (!etnaviv_domain->bad_page_cpu) { 123 ret = -ENOMEM; 124 goto fail_mem; 125 } 126 p = etnaviv_domain->bad_page_cpu; 127 for (i = 0; i < SZ_4K / 4; i++) 128 *p++ = 0xdead55aa; 129 130 etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->dev, 131 SZ_4K, 132 &etnaviv_domain->mtlb_dma, 133 GFP_KERNEL); 134 if (!etnaviv_domain->mtlb_cpu) { 135 ret = -ENOMEM; 136 goto fail_mem; 137 } 138 139 /* pre-populate STLB pages (may want to switch to on-demand later) */ 140 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 141 etnaviv_domain->stlb_cpu[i] = 142 dma_alloc_coherent(etnaviv_domain->dev, 143 SZ_4K, 144 &etnaviv_domain->stlb_dma[i], 145 GFP_KERNEL); 146 if (!etnaviv_domain->stlb_cpu[i]) { 147 ret = -ENOMEM; 148 goto fail_mem; 149 } 150 p = etnaviv_domain->stlb_cpu[i]; 151 for (j = 0; j < SZ_4K / 4; j++) 152 *p++ = MMUv2_PTE_EXCEPTION; 153 154 etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] | 155 MMUv2_PTE_PRESENT; 156 } 157 158 return 0; 159 160 fail_mem: 161 if (etnaviv_domain->bad_page_cpu) 162 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 163 etnaviv_domain->bad_page_cpu, 164 etnaviv_domain->bad_page_dma); 165 166 if (etnaviv_domain->mtlb_cpu) 167 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 168 etnaviv_domain->mtlb_cpu, 169 etnaviv_domain->mtlb_dma); 170 171 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 172 if (etnaviv_domain->stlb_cpu[i]) 173 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 174 etnaviv_domain->stlb_cpu[i], 175 etnaviv_domain->stlb_dma[i]); 176 } 177 178 return ret; 179 } 180 181 static void etnaviv_iommuv2_domain_free(struct iommu_domain *domain) 182 { 183 struct etnaviv_iommuv2_domain *etnaviv_domain = 184 to_etnaviv_domain(domain); 185 int i; 186 187 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 188 etnaviv_domain->bad_page_cpu, 189 etnaviv_domain->bad_page_dma); 190 191 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 192 etnaviv_domain->mtlb_cpu, 193 etnaviv_domain->mtlb_dma); 194 195 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 196 if (etnaviv_domain->stlb_cpu[i]) 197 dma_free_coherent(etnaviv_domain->dev, SZ_4K, 198 etnaviv_domain->stlb_cpu[i], 199 etnaviv_domain->stlb_dma[i]); 200 } 201 202 vfree(etnaviv_domain); 203 } 204 205 static size_t etnaviv_iommuv2_dump_size(struct iommu_domain *domain) 206 { 207 struct etnaviv_iommuv2_domain *etnaviv_domain = 208 to_etnaviv_domain(domain); 209 size_t dump_size = SZ_4K; 210 int i; 211 212 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) 213 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 214 dump_size += SZ_4K; 215 216 return dump_size; 217 } 218 219 static void etnaviv_iommuv2_dump(struct iommu_domain *domain, void *buf) 220 { 221 struct etnaviv_iommuv2_domain *etnaviv_domain = 222 to_etnaviv_domain(domain); 223 int i; 224 225 memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K); 226 buf += SZ_4K; 227 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K) 228 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 229 memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K); 230 } 231 232 static struct etnaviv_iommu_ops etnaviv_iommu_ops = { 233 .ops = { 234 .domain_free = etnaviv_iommuv2_domain_free, 235 .map = etnaviv_iommuv2_map, 236 .unmap = etnaviv_iommuv2_unmap, 237 .iova_to_phys = etnaviv_iommuv2_iova_to_phys, 238 .pgsize_bitmap = SZ_4K, 239 }, 240 .dump_size = etnaviv_iommuv2_dump_size, 241 .dump = etnaviv_iommuv2_dump, 242 }; 243 244 void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) 245 { 246 struct etnaviv_iommuv2_domain *etnaviv_domain = 247 to_etnaviv_domain(gpu->mmu->domain); 248 u16 prefetch; 249 250 /* If the MMU is already enabled the state is still there. */ 251 if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE) 252 return; 253 254 prefetch = etnaviv_buffer_config_mmuv2(gpu, 255 (u32)etnaviv_domain->mtlb_dma, 256 (u32)etnaviv_domain->bad_page_dma); 257 etnaviv_gpu_start_fe(gpu, gpu->buffer->paddr, prefetch); 258 etnaviv_gpu_wait_idle(gpu, 100); 259 260 gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); 261 } 262 struct iommu_domain *etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu) 263 { 264 struct etnaviv_iommuv2_domain *etnaviv_domain; 265 int ret; 266 267 etnaviv_domain = vzalloc(sizeof(*etnaviv_domain)); 268 if (!etnaviv_domain) 269 return NULL; 270 271 etnaviv_domain->dev = gpu->dev; 272 273 etnaviv_domain->domain.type = __IOMMU_DOMAIN_PAGING; 274 etnaviv_domain->domain.ops = &etnaviv_iommu_ops.ops; 275 etnaviv_domain->domain.pgsize_bitmap = SZ_4K; 276 etnaviv_domain->domain.geometry.aperture_start = 0; 277 etnaviv_domain->domain.geometry.aperture_end = ~0UL & ~(SZ_4K - 1); 278 279 ret = etnaviv_iommuv2_init(etnaviv_domain); 280 if (ret) 281 goto out_free; 282 283 return &etnaviv_domain->domain; 284 285 out_free: 286 vfree(etnaviv_domain); 287 return NULL; 288 } 289