1 /* 2 * Copyright 2017 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "vmm.h" 23 24 #include <subdev/fb.h> 25 #include <subdev/ltc.h> 26 #include <subdev/timer.h> 27 28 #include <nvif/if900d.h> 29 #include <nvif/unpack.h> 30 31 static inline void 32 gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 33 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) 34 { 35 u64 base = (addr >> 8) | map->type; 36 u64 data = base; 37 38 if (map->ctag && !(map->next & (1ULL << 44))) { 39 while (ptes--) { 40 data = base | ((map->ctag >> 1) << 44); 41 if (!(map->ctag++ & 1)) 42 data |= BIT_ULL(60); 43 44 VMM_WO064(pt, vmm, ptei++ * 8, data); 45 base += map->next; 46 } 47 } else { 48 map->type += ptes * map->ctag; 49 50 while (ptes--) { 51 VMM_WO064(pt, vmm, ptei++ * 8, data); 52 data += map->next; 53 } 54 } 55 } 56 57 void 58 gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 59 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 60 { 61 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); 62 } 63 64 void 65 gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 66 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 67 { 68 if (map->page->shift == PAGE_SHIFT) { 69 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes); 70 nvkm_kmap(pt->memory); 71 while (ptes--) { 72 const u64 data = (*map->dma++ >> 8) | map->type; 73 VMM_WO064(pt, vmm, ptei++ * 8, data); 74 map->type += map->ctag; 75 } 76 nvkm_done(pt->memory); 77 return; 78 } 79 80 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); 81 } 82 83 void 84 gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 85 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 86 { 87 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); 88 } 89 90 void 91 gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm, 92 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 93 { 94 VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes); 95 } 96 97 const struct nvkm_vmm_desc_func 98 gf100_vmm_pgt = { 99 .unmap = gf100_vmm_pgt_unmap, 100 .mem = gf100_vmm_pgt_mem, 101 .dma = gf100_vmm_pgt_dma, 102 .sgl = gf100_vmm_pgt_sgl, 103 }; 104 105 void 106 gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) 107 { 108 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 109 struct nvkm_mmu_pt *pd = pgd->pt[0]; 110 struct nvkm_mmu_pt *pt; 111 u64 data = 0; 112 113 if ((pt = pgt->pt[0])) { 114 switch (nvkm_memory_target(pt->memory)) { 115 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break; 116 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0; 117 data |= BIT_ULL(35); /* VOL */ 118 break; 119 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break; 120 default: 121 WARN_ON(1); 122 return; 123 } 124 data |= pt->addr >> 8; 125 } 126 127 if ((pt = pgt->pt[1])) { 128 switch (nvkm_memory_target(pt->memory)) { 129 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break; 130 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32; 131 data |= BIT_ULL(34); /* VOL */ 132 break; 133 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break; 134 default: 135 WARN_ON(1); 136 return; 137 } 138 data |= pt->addr << 24; 139 } 140 141 nvkm_kmap(pd->memory); 142 VMM_WO064(pd, vmm, pdei * 8, data); 143 nvkm_done(pd->memory); 144 } 145 146 const struct nvkm_vmm_desc_func 147 gf100_vmm_pgd = { 148 .unmap = gf100_vmm_pgt_unmap, 149 .pde = gf100_vmm_pgd_pde, 150 }; 151 152 static const struct nvkm_vmm_desc 153 gf100_vmm_desc_17_12[] = { 154 { SPT, 15, 8, 0x1000, &gf100_vmm_pgt }, 155 { PGD, 13, 8, 0x1000, &gf100_vmm_pgd }, 156 {} 157 }; 158 159 static const struct nvkm_vmm_desc 160 gf100_vmm_desc_17_17[] = { 161 { LPT, 10, 8, 0x1000, &gf100_vmm_pgt }, 162 { PGD, 13, 8, 0x1000, &gf100_vmm_pgd }, 163 {} 164 }; 165 166 static const struct nvkm_vmm_desc 167 gf100_vmm_desc_16_12[] = { 168 { SPT, 14, 8, 0x1000, &gf100_vmm_pgt }, 169 { PGD, 14, 8, 0x1000, &gf100_vmm_pgd }, 170 {} 171 }; 172 173 static const struct nvkm_vmm_desc 174 gf100_vmm_desc_16_16[] = { 175 { LPT, 10, 8, 0x1000, &gf100_vmm_pgt }, 176 { PGD, 14, 8, 0x1000, &gf100_vmm_pgd }, 177 {} 178 }; 179 180 void 181 gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type) 182 { 183 struct nvkm_subdev *subdev = &vmm->mmu->subdev; 184 struct nvkm_device *device = subdev->device; 185 186 mutex_lock(&subdev->mutex); 187 /* Looks like maybe a "free flush slots" counter, the 188 * faster you write to 0x100cbc to more it decreases. 189 */ 190 nvkm_msec(device, 2000, 191 if (nvkm_rd32(device, 0x100c80) & 0x00ff0000) 192 break; 193 ); 194 195 nvkm_wr32(device, 0x100cb8, vmm->pd->pt[0]->addr >> 8); 196 nvkm_wr32(device, 0x100cbc, 0x80000000 | type); 197 198 /* Wait for flush to be queued? */ 199 nvkm_msec(device, 2000, 200 if (nvkm_rd32(device, 0x100c80) & 0x00008000) 201 break; 202 ); 203 mutex_unlock(&subdev->mutex); 204 } 205 206 void 207 gf100_vmm_flush(struct nvkm_vmm *vmm, int depth) 208 { 209 u32 type = 0x00000001; /* PAGE_ALL */ 210 if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) 211 type |= 0x00000004; /* HUB_ONLY */ 212 gf100_vmm_invalidate(vmm, type); 213 } 214 215 int 216 gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, 217 struct nvkm_vmm_map *map) 218 { 219 const enum nvkm_memory_target target = nvkm_memory_target(map->memory); 220 const struct nvkm_vmm_page *page = map->page; 221 const bool gm20x = page->desc->func->sparse != NULL; 222 union { 223 struct gf100_vmm_map_vn vn; 224 struct gf100_vmm_map_v0 v0; 225 } *args = argv; 226 struct nvkm_device *device = vmm->mmu->subdev.device; 227 struct nvkm_memory *memory = map->memory; 228 u8 kind, priv, ro, vol; 229 int kindn, aper, ret = -ENOSYS; 230 const u8 *kindm; 231 232 map->next = (1 << page->shift) >> 8; 233 map->type = map->ctag = 0; 234 235 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 236 vol = !!args->v0.vol; 237 ro = !!args->v0.ro; 238 priv = !!args->v0.priv; 239 kind = args->v0.kind; 240 } else 241 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { 242 vol = target == NVKM_MEM_TARGET_HOST; 243 ro = 0; 244 priv = 0; 245 kind = 0x00; 246 } else { 247 VMM_DEBUG(vmm, "args"); 248 return ret; 249 } 250 251 aper = vmm->func->aper(target); 252 if (WARN_ON(aper < 0)) 253 return aper; 254 255 kindm = vmm->mmu->func->kind(vmm->mmu, &kindn); 256 if (kind >= kindn || kindm[kind] == 0xff) { 257 VMM_DEBUG(vmm, "kind %02x", kind); 258 return -EINVAL; 259 } 260 261 if (kindm[kind] != kind) { 262 u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17; 263 u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp; 264 if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) { 265 VMM_DEBUG(vmm, "comp %d %02x", aper, page->type); 266 return -EINVAL; 267 } 268 269 ret = nvkm_memory_tags_get(memory, device, tags, 270 nvkm_ltc_tags_clear, 271 &map->tags); 272 if (ret) { 273 VMM_DEBUG(vmm, "comp %d", ret); 274 return ret; 275 } 276 277 if (map->tags->mn) { 278 u64 tags = map->tags->mn->offset + (map->offset >> 17); 279 if (page->shift == 17 || !gm20x) { 280 map->type |= tags << 44; 281 map->ctag |= 1ULL << 44; 282 map->next |= 1ULL << 44; 283 } else { 284 map->ctag |= tags << 1 | 1; 285 } 286 } else { 287 kind = kindm[kind]; 288 } 289 } 290 291 map->type |= BIT(0); 292 map->type |= (u64)priv << 1; 293 map->type |= (u64) ro << 2; 294 map->type |= (u64) vol << 32; 295 map->type |= (u64)aper << 33; 296 map->type |= (u64)kind << 36; 297 return 0; 298 } 299 300 int 301 gf100_vmm_aper(enum nvkm_memory_target target) 302 { 303 switch (target) { 304 case NVKM_MEM_TARGET_VRAM: return 0; 305 case NVKM_MEM_TARGET_HOST: return 2; 306 case NVKM_MEM_TARGET_NCOH: return 3; 307 default: 308 return -EINVAL; 309 } 310 } 311 312 void 313 gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) 314 { 315 nvkm_fo64(inst, 0x0200, 0x00000000, 2); 316 } 317 318 int 319 gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base) 320 { 321 struct nvkm_mmu_pt *pd = vmm->pd->pt[0]; 322 323 switch (nvkm_memory_target(pd->memory)) { 324 case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break; 325 case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0; 326 base |= BIT_ULL(2) /* VOL. */; 327 break; 328 case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break; 329 default: 330 WARN_ON(1); 331 return -EINVAL; 332 } 333 base |= pd->addr; 334 335 nvkm_kmap(inst); 336 nvkm_wo64(inst, 0x0200, base); 337 nvkm_wo64(inst, 0x0208, vmm->limit - 1); 338 nvkm_done(inst); 339 return 0; 340 } 341 342 int 343 gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) 344 { 345 return gf100_vmm_join_(vmm, inst, 0); 346 } 347 348 static const struct nvkm_vmm_func 349 gf100_vmm_17 = { 350 .join = gf100_vmm_join, 351 .part = gf100_vmm_part, 352 .aper = gf100_vmm_aper, 353 .valid = gf100_vmm_valid, 354 .flush = gf100_vmm_flush, 355 .page = { 356 { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, 357 { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, 358 {} 359 } 360 }; 361 362 static const struct nvkm_vmm_func 363 gf100_vmm_16 = { 364 .join = gf100_vmm_join, 365 .part = gf100_vmm_part, 366 .aper = gf100_vmm_aper, 367 .valid = gf100_vmm_valid, 368 .flush = gf100_vmm_flush, 369 .page = { 370 { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, 371 { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, 372 {} 373 } 374 }; 375 376 int 377 gf100_vmm_new_(const struct nvkm_vmm_func *func_16, 378 const struct nvkm_vmm_func *func_17, 379 struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, 380 struct lock_class_key *key, const char *name, 381 struct nvkm_vmm **pvmm) 382 { 383 switch (mmu->subdev.device->fb->page) { 384 case 16: return nv04_vmm_new_(func_16, mmu, 0, addr, size, 385 argv, argc, key, name, pvmm); 386 case 17: return nv04_vmm_new_(func_17, mmu, 0, addr, size, 387 argv, argc, key, name, pvmm); 388 default: 389 WARN_ON(1); 390 return -EINVAL; 391 } 392 } 393 394 int 395 gf100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, 396 struct lock_class_key *key, const char *name, 397 struct nvkm_vmm **pvmm) 398 { 399 return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, addr, 400 size, argv, argc, key, name, pvmm); 401 } 402