1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2016-2018 Etnaviv Project 4 */ 5 6 #include <linux/platform_device.h> 7 #include <linux/sizes.h> 8 #include <linux/slab.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/bitops.h> 11 12 #include "etnaviv_cmdbuf.h" 13 #include "etnaviv_gpu.h" 14 #include "etnaviv_mmu.h" 15 #include "etnaviv_iommu.h" 16 #include "state.xml.h" 17 #include "state_hi.xml.h" 18 19 #define MMUv2_PTE_PRESENT BIT(0) 20 #define MMUv2_PTE_EXCEPTION BIT(1) 21 #define MMUv2_PTE_WRITEABLE BIT(2) 22 23 #define MMUv2_MTLB_MASK 0xffc00000 24 #define MMUv2_MTLB_SHIFT 22 25 #define MMUv2_STLB_MASK 0x003ff000 26 #define MMUv2_STLB_SHIFT 12 27 28 #define MMUv2_MAX_STLB_ENTRIES 1024 29 30 struct etnaviv_iommuv2_domain { 31 struct etnaviv_iommu_domain base; 32 /* P(age) T(able) A(rray) */ 33 u64 *pta_cpu; 34 dma_addr_t pta_dma; 35 /* M(aster) TLB aka first level pagetable */ 36 u32 *mtlb_cpu; 37 dma_addr_t mtlb_dma; 38 /* S(lave) TLB aka second level pagetable */ 39 u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES]; 40 dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES]; 41 }; 42 43 static struct etnaviv_iommuv2_domain * 44 to_etnaviv_domain(struct etnaviv_iommu_domain *domain) 45 { 46 return container_of(domain, struct etnaviv_iommuv2_domain, base); 47 } 48 49 static int 50 etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_domain *etnaviv_domain, 51 int stlb) 52 { 53 if (etnaviv_domain->stlb_cpu[stlb]) 54 return 0; 55 56 etnaviv_domain->stlb_cpu[stlb] = 57 dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K, 58 &etnaviv_domain->stlb_dma[stlb], 59 GFP_KERNEL); 60 61 if (!etnaviv_domain->stlb_cpu[stlb]) 62 return -ENOMEM; 63 64 memset32(etnaviv_domain->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION, 65 SZ_4K / sizeof(u32)); 66 67 etnaviv_domain->mtlb_cpu[stlb] = etnaviv_domain->stlb_dma[stlb] | 68 MMUv2_PTE_PRESENT; 69 return 0; 70 } 71 72 static int etnaviv_iommuv2_map(struct etnaviv_iommu_domain *domain, 73 unsigned long iova, phys_addr_t paddr, 74 size_t size, int prot) 75 { 76 struct etnaviv_iommuv2_domain *etnaviv_domain = 77 to_etnaviv_domain(domain); 78 int mtlb_entry, stlb_entry, ret; 79 u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT; 80 81 if (size != SZ_4K) 82 return -EINVAL; 83 84 if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT)) 85 entry |= (upper_32_bits(paddr) & 0xff) << 4; 86 87 if (prot & ETNAVIV_PROT_WRITE) 88 entry |= MMUv2_PTE_WRITEABLE; 89 90 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 91 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 92 93 ret = etnaviv_iommuv2_ensure_stlb(etnaviv_domain, mtlb_entry); 94 if (ret) 95 return ret; 96 97 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry; 98 99 return 0; 100 } 101 102 static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain, 103 unsigned long iova, size_t size) 104 { 105 struct etnaviv_iommuv2_domain *etnaviv_domain = 106 to_etnaviv_domain(domain); 107 int mtlb_entry, stlb_entry; 108 109 if (size != SZ_4K) 110 return -EINVAL; 111 112 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 113 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 114 115 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION; 116 117 return SZ_4K; 118 } 119 120 static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain) 121 { 122 u32 *p; 123 int ret, i; 124 125 /* allocate scratch page */ 126 etnaviv_domain->base.bad_page_cpu = 127 dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K, 128 &etnaviv_domain->base.bad_page_dma, 129 GFP_KERNEL); 130 if (!etnaviv_domain->base.bad_page_cpu) { 131 ret = -ENOMEM; 132 goto fail_mem; 133 } 134 p = etnaviv_domain->base.bad_page_cpu; 135 for (i = 0; i < SZ_4K / 4; i++) 136 *p++ = 0xdead55aa; 137 138 etnaviv_domain->pta_cpu = dma_alloc_wc(etnaviv_domain->base.dev, 139 SZ_4K, &etnaviv_domain->pta_dma, 140 GFP_KERNEL); 141 if (!etnaviv_domain->pta_cpu) { 142 ret = -ENOMEM; 143 goto fail_mem; 144 } 145 146 etnaviv_domain->mtlb_cpu = dma_alloc_wc(etnaviv_domain->base.dev, 147 SZ_4K, &etnaviv_domain->mtlb_dma, 148 GFP_KERNEL); 149 if (!etnaviv_domain->mtlb_cpu) { 150 ret = -ENOMEM; 151 goto fail_mem; 152 } 153 154 memset32(etnaviv_domain->mtlb_cpu, MMUv2_PTE_EXCEPTION, 155 MMUv2_MAX_STLB_ENTRIES); 156 157 return 0; 158 159 fail_mem: 160 if (etnaviv_domain->base.bad_page_cpu) 161 dma_free_wc(etnaviv_domain->base.dev, SZ_4K, 162 etnaviv_domain->base.bad_page_cpu, 163 etnaviv_domain->base.bad_page_dma); 164 165 if (etnaviv_domain->pta_cpu) 166 dma_free_wc(etnaviv_domain->base.dev, SZ_4K, 167 etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma); 168 169 if (etnaviv_domain->mtlb_cpu) 170 dma_free_wc(etnaviv_domain->base.dev, SZ_4K, 171 etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma); 172 173 return ret; 174 } 175 176 static void etnaviv_iommuv2_domain_free(struct etnaviv_iommu_domain *domain) 177 { 178 struct etnaviv_iommuv2_domain *etnaviv_domain = 179 to_etnaviv_domain(domain); 180 int i; 181 182 dma_free_wc(etnaviv_domain->base.dev, SZ_4K, 183 etnaviv_domain->base.bad_page_cpu, 184 etnaviv_domain->base.bad_page_dma); 185 186 dma_free_wc(etnaviv_domain->base.dev, SZ_4K, 187 etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma); 188 189 dma_free_wc(etnaviv_domain->base.dev, SZ_4K, 190 etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma); 191 192 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 193 if (etnaviv_domain->stlb_cpu[i]) 194 dma_free_wc(etnaviv_domain->base.dev, SZ_4K, 195 etnaviv_domain->stlb_cpu[i], 196 etnaviv_domain->stlb_dma[i]); 197 } 198 199 vfree(etnaviv_domain); 200 } 201 202 static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_domain *domain) 203 { 204 struct etnaviv_iommuv2_domain *etnaviv_domain = 205 to_etnaviv_domain(domain); 206 size_t dump_size = SZ_4K; 207 int i; 208 209 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) 210 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 211 dump_size += SZ_4K; 212 213 return dump_size; 214 } 215 216 static void etnaviv_iommuv2_dump(struct etnaviv_iommu_domain *domain, void *buf) 217 { 218 struct etnaviv_iommuv2_domain *etnaviv_domain = 219 to_etnaviv_domain(domain); 220 int i; 221 222 memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K); 223 buf += SZ_4K; 224 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K) 225 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 226 memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K); 227 } 228 229 static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu) 230 { 231 struct etnaviv_iommuv2_domain *etnaviv_domain = 232 to_etnaviv_domain(gpu->mmu->domain); 233 u16 prefetch; 234 235 /* If the MMU is already enabled the state is still there. */ 236 if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE) 237 return; 238 239 prefetch = etnaviv_buffer_config_mmuv2(gpu, 240 (u32)etnaviv_domain->mtlb_dma, 241 (u32)etnaviv_domain->base.bad_page_dma); 242 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer), 243 prefetch); 244 etnaviv_gpu_wait_idle(gpu, 100); 245 246 gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); 247 } 248 249 static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu) 250 { 251 struct etnaviv_iommuv2_domain *etnaviv_domain = 252 to_etnaviv_domain(gpu->mmu->domain); 253 u16 prefetch; 254 255 /* If the MMU is already enabled the state is still there. */ 256 if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE) 257 return; 258 259 gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW, 260 lower_32_bits(etnaviv_domain->pta_dma)); 261 gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH, 262 upper_32_bits(etnaviv_domain->pta_dma)); 263 gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE); 264 265 gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW, 266 lower_32_bits(etnaviv_domain->base.bad_page_dma)); 267 gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW, 268 lower_32_bits(etnaviv_domain->base.bad_page_dma)); 269 gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG, 270 VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH( 271 upper_32_bits(etnaviv_domain->base.bad_page_dma)) | 272 VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH( 273 upper_32_bits(etnaviv_domain->base.bad_page_dma))); 274 275 etnaviv_domain->pta_cpu[0] = etnaviv_domain->mtlb_dma | 276 VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K; 277 278 /* trigger a PTA load through the FE */ 279 prefetch = etnaviv_buffer_config_pta(gpu); 280 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer), 281 prefetch); 282 etnaviv_gpu_wait_idle(gpu, 100); 283 284 gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE); 285 } 286 287 void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) 288 { 289 switch (gpu->sec_mode) { 290 case ETNA_SEC_NONE: 291 etnaviv_iommuv2_restore_nonsec(gpu); 292 break; 293 case ETNA_SEC_KERNEL: 294 etnaviv_iommuv2_restore_sec(gpu); 295 break; 296 default: 297 WARN(1, "unhandled GPU security mode\n"); 298 break; 299 } 300 } 301 302 static const struct etnaviv_iommu_domain_ops etnaviv_iommuv2_ops = { 303 .free = etnaviv_iommuv2_domain_free, 304 .map = etnaviv_iommuv2_map, 305 .unmap = etnaviv_iommuv2_unmap, 306 .dump_size = etnaviv_iommuv2_dump_size, 307 .dump = etnaviv_iommuv2_dump, 308 }; 309 310 struct etnaviv_iommu_domain * 311 etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu) 312 { 313 struct etnaviv_iommuv2_domain *etnaviv_domain; 314 struct etnaviv_iommu_domain *domain; 315 int ret; 316 317 etnaviv_domain = vzalloc(sizeof(*etnaviv_domain)); 318 if (!etnaviv_domain) 319 return NULL; 320 321 domain = &etnaviv_domain->base; 322 323 domain->dev = gpu->dev; 324 domain->base = 0; 325 domain->size = (u64)SZ_1G * 4; 326 domain->ops = &etnaviv_iommuv2_ops; 327 328 ret = etnaviv_iommuv2_init(etnaviv_domain); 329 if (ret) 330 goto out_free; 331 332 return &etnaviv_domain->base; 333 334 out_free: 335 vfree(etnaviv_domain); 336 return NULL; 337 } 338