1 /* 2 * Copyright (C) 2016 Etnaviv Project 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 as published by 6 * the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16 17 #include <linux/platform_device.h> 18 #include <linux/sizes.h> 19 #include <linux/slab.h> 20 #include <linux/dma-mapping.h> 21 #include <linux/bitops.h> 22 23 #include "etnaviv_cmdbuf.h" 24 #include "etnaviv_gpu.h" 25 #include "etnaviv_mmu.h" 26 #include "etnaviv_iommu.h" 27 #include "state.xml.h" 28 #include "state_hi.xml.h" 29 30 #define MMUv2_PTE_PRESENT BIT(0) 31 #define MMUv2_PTE_EXCEPTION BIT(1) 32 #define MMUv2_PTE_WRITEABLE BIT(2) 33 34 #define MMUv2_MTLB_MASK 0xffc00000 35 #define MMUv2_MTLB_SHIFT 22 36 #define MMUv2_STLB_MASK 0x003ff000 37 #define MMUv2_STLB_SHIFT 12 38 39 #define MMUv2_MAX_STLB_ENTRIES 1024 40 41 struct etnaviv_iommuv2_domain { 42 struct etnaviv_iommu_domain base; 43 /* M(aster) TLB aka first level pagetable */ 44 u32 *mtlb_cpu; 45 dma_addr_t mtlb_dma; 46 /* S(lave) TLB aka second level pagetable */ 47 u32 *stlb_cpu[1024]; 48 dma_addr_t stlb_dma[1024]; 49 }; 50 51 static struct etnaviv_iommuv2_domain * 52 to_etnaviv_domain(struct etnaviv_iommu_domain *domain) 53 { 54 return container_of(domain, struct etnaviv_iommuv2_domain, base); 55 } 56 57 static int etnaviv_iommuv2_map(struct etnaviv_iommu_domain *domain, 58 unsigned long iova, phys_addr_t paddr, 59 size_t size, int prot) 60 { 61 struct etnaviv_iommuv2_domain *etnaviv_domain = 62 to_etnaviv_domain(domain); 63 int mtlb_entry, stlb_entry; 64 u32 entry = (u32)paddr | MMUv2_PTE_PRESENT; 65 66 if (size != SZ_4K) 67 return -EINVAL; 68 69 if (prot & ETNAVIV_PROT_WRITE) 70 entry |= MMUv2_PTE_WRITEABLE; 71 72 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 73 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 74 75 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry; 76 77 return 0; 78 } 79 80 static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain, 81 unsigned long iova, size_t size) 82 { 83 struct etnaviv_iommuv2_domain *etnaviv_domain = 84 to_etnaviv_domain(domain); 85 int mtlb_entry, stlb_entry; 86 87 if (size != SZ_4K) 88 return -EINVAL; 89 90 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 91 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 92 93 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION; 94 95 return SZ_4K; 96 } 97 98 static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain) 99 { 100 u32 *p; 101 int ret, i, j; 102 103 /* allocate scratch page */ 104 etnaviv_domain->base.bad_page_cpu = dma_alloc_coherent( 105 etnaviv_domain->base.dev, 106 SZ_4K, 107 &etnaviv_domain->base.bad_page_dma, 108 GFP_KERNEL); 109 if (!etnaviv_domain->base.bad_page_cpu) { 110 ret = -ENOMEM; 111 goto fail_mem; 112 } 113 p = etnaviv_domain->base.bad_page_cpu; 114 for (i = 0; i < SZ_4K / 4; i++) 115 *p++ = 0xdead55aa; 116 117 etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->base.dev, 118 SZ_4K, 119 &etnaviv_domain->mtlb_dma, 120 GFP_KERNEL); 121 if (!etnaviv_domain->mtlb_cpu) { 122 ret = -ENOMEM; 123 goto fail_mem; 124 } 125 126 /* pre-populate STLB pages (may want to switch to on-demand later) */ 127 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 128 etnaviv_domain->stlb_cpu[i] = 129 dma_alloc_coherent(etnaviv_domain->base.dev, 130 SZ_4K, 131 &etnaviv_domain->stlb_dma[i], 132 GFP_KERNEL); 133 if (!etnaviv_domain->stlb_cpu[i]) { 134 ret = -ENOMEM; 135 goto fail_mem; 136 } 137 p = etnaviv_domain->stlb_cpu[i]; 138 for (j = 0; j < SZ_4K / 4; j++) 139 *p++ = MMUv2_PTE_EXCEPTION; 140 141 etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] | 142 MMUv2_PTE_PRESENT; 143 } 144 145 return 0; 146 147 fail_mem: 148 if (etnaviv_domain->base.bad_page_cpu) 149 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, 150 etnaviv_domain->base.bad_page_cpu, 151 etnaviv_domain->base.bad_page_dma); 152 153 if (etnaviv_domain->mtlb_cpu) 154 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, 155 etnaviv_domain->mtlb_cpu, 156 etnaviv_domain->mtlb_dma); 157 158 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 159 if (etnaviv_domain->stlb_cpu[i]) 160 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, 161 etnaviv_domain->stlb_cpu[i], 162 etnaviv_domain->stlb_dma[i]); 163 } 164 165 return ret; 166 } 167 168 static void etnaviv_iommuv2_domain_free(struct etnaviv_iommu_domain *domain) 169 { 170 struct etnaviv_iommuv2_domain *etnaviv_domain = 171 to_etnaviv_domain(domain); 172 int i; 173 174 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, 175 etnaviv_domain->base.bad_page_cpu, 176 etnaviv_domain->base.bad_page_dma); 177 178 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, 179 etnaviv_domain->mtlb_cpu, 180 etnaviv_domain->mtlb_dma); 181 182 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 183 if (etnaviv_domain->stlb_cpu[i]) 184 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, 185 etnaviv_domain->stlb_cpu[i], 186 etnaviv_domain->stlb_dma[i]); 187 } 188 189 vfree(etnaviv_domain); 190 } 191 192 static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_domain *domain) 193 { 194 struct etnaviv_iommuv2_domain *etnaviv_domain = 195 to_etnaviv_domain(domain); 196 size_t dump_size = SZ_4K; 197 int i; 198 199 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) 200 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 201 dump_size += SZ_4K; 202 203 return dump_size; 204 } 205 206 static void etnaviv_iommuv2_dump(struct etnaviv_iommu_domain *domain, void *buf) 207 { 208 struct etnaviv_iommuv2_domain *etnaviv_domain = 209 to_etnaviv_domain(domain); 210 int i; 211 212 memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K); 213 buf += SZ_4K; 214 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K) 215 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 216 memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K); 217 } 218 219 void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) 220 { 221 struct etnaviv_iommuv2_domain *etnaviv_domain = 222 to_etnaviv_domain(gpu->mmu->domain); 223 u16 prefetch; 224 225 /* If the MMU is already enabled the state is still there. */ 226 if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE) 227 return; 228 229 prefetch = etnaviv_buffer_config_mmuv2(gpu, 230 (u32)etnaviv_domain->mtlb_dma, 231 (u32)etnaviv_domain->base.bad_page_dma); 232 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer), 233 prefetch); 234 etnaviv_gpu_wait_idle(gpu, 100); 235 236 gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); 237 } 238 239 const struct etnaviv_iommu_domain_ops etnaviv_iommuv2_ops = { 240 .free = etnaviv_iommuv2_domain_free, 241 .map = etnaviv_iommuv2_map, 242 .unmap = etnaviv_iommuv2_unmap, 243 .dump_size = etnaviv_iommuv2_dump_size, 244 .dump = etnaviv_iommuv2_dump, 245 }; 246 247 struct etnaviv_iommu_domain * 248 etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu) 249 { 250 struct etnaviv_iommuv2_domain *etnaviv_domain; 251 struct etnaviv_iommu_domain *domain; 252 int ret; 253 254 etnaviv_domain = vzalloc(sizeof(*etnaviv_domain)); 255 if (!etnaviv_domain) 256 return NULL; 257 258 domain = &etnaviv_domain->base; 259 260 domain->dev = gpu->dev; 261 domain->base = 0; 262 domain->size = (u64)SZ_1G * 4; 263 domain->ops = &etnaviv_iommuv2_ops; 264 265 ret = etnaviv_iommuv2_init(etnaviv_domain); 266 if (ret) 267 goto out_free; 268 269 return &etnaviv_domain->base; 270 271 out_free: 272 vfree(etnaviv_domain); 273 return NULL; 274 } 275