1 /* 2 * Copyright 2017 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "vmm.h" 23 24 #include <core/client.h> 25 #include <subdev/fb.h> 26 #include <subdev/ltc.h> 27 #include <subdev/timer.h> 28 #include <engine/gr.h> 29 30 #include <nvif/ifc00d.h> 31 #include <nvif/unpack.h> 32 33 static void 34 gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm, 35 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 36 { 37 struct device *dev = vmm->mmu->subdev.device->dev; 38 dma_addr_t addr; 39 40 nvkm_kmap(pt->memory); 41 while (ptes--) { 42 u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); 43 u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); 44 u64 data = (u64)datahi << 32 | datalo; 45 if ((data & (3ULL << 1)) != 0) { 46 addr = (data >> 8) << 12; 47 dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 48 } 49 ptei++; 50 } 51 nvkm_done(pt->memory); 52 } 53 54 static bool 55 gp100_vmm_pfn_clear(struct nvkm_vmm *vmm, 56 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 57 { 58 bool dma = false; 59 nvkm_kmap(pt->memory); 60 while (ptes--) { 61 u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); 62 u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); 63 u64 data = (u64)datahi << 32 | datalo; 64 if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) { 65 VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0)); 66 dma = true; 67 } 68 ptei++; 69 } 70 nvkm_done(pt->memory); 71 return dma; 72 } 73 74 static void 75 gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 76 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 77 { 78 struct device *dev = vmm->mmu->subdev.device->dev; 79 dma_addr_t addr; 80 81 nvkm_kmap(pt->memory); 82 while (ptes--) { 83 u64 data = 0; 84 if (!(*map->pfn & NVKM_VMM_PFN_W)) 85 data |= BIT_ULL(6); /* RO. */ 86 87 if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) { 88 addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT; 89 addr = dma_map_page(dev, pfn_to_page(addr), 0, 90 PAGE_SIZE, DMA_BIDIRECTIONAL); 91 if (!WARN_ON(dma_mapping_error(dev, addr))) { 92 data |= addr >> 4; 93 data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */ 94 data |= BIT_ULL(3); /* VOL. */ 95 data |= BIT_ULL(0); /* VALID. */ 96 } 97 } else { 98 data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4; 99 data |= BIT_ULL(0); /* VALID. */ 100 } 101 102 VMM_WO064(pt, vmm, ptei++ * 8, data); 103 map->pfn++; 104 } 105 nvkm_done(pt->memory); 106 } 107 108 static inline void 109 gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 110 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) 111 { 112 u64 data = (addr >> 4) | map->type; 113 114 map->type += ptes * map->ctag; 115 116 while (ptes--) { 117 VMM_WO064(pt, vmm, ptei++ * 8, data); 118 data += map->next; 119 } 120 } 121 122 static void 123 gp100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 124 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 125 { 126 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); 127 } 128 129 static void 130 gp100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 131 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 132 { 133 if (map->page->shift == PAGE_SHIFT) { 134 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes); 135 nvkm_kmap(pt->memory); 136 while (ptes--) { 137 const u64 data = (*map->dma++ >> 4) | map->type; 138 VMM_WO064(pt, vmm, ptei++ * 8, data); 139 map->type += map->ctag; 140 } 141 nvkm_done(pt->memory); 142 return; 143 } 144 145 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); 146 } 147 148 static void 149 gp100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 150 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 151 { 152 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); 153 } 154 155 static void 156 gp100_vmm_pgt_sparse(struct nvkm_vmm *vmm, 157 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 158 { 159 /* VALID_FALSE + VOL tells the MMU to treat the PTE as sparse. */ 160 VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(3) /* VOL. */, ptes); 161 } 162 163 static const struct nvkm_vmm_desc_func 164 gp100_vmm_desc_spt = { 165 .unmap = gf100_vmm_pgt_unmap, 166 .sparse = gp100_vmm_pgt_sparse, 167 .mem = gp100_vmm_pgt_mem, 168 .dma = gp100_vmm_pgt_dma, 169 .sgl = gp100_vmm_pgt_sgl, 170 .pfn = gp100_vmm_pgt_pfn, 171 .pfn_clear = gp100_vmm_pfn_clear, 172 .pfn_unmap = gp100_vmm_pfn_unmap, 173 }; 174 175 static void 176 gp100_vmm_lpt_invalid(struct nvkm_vmm *vmm, 177 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 178 { 179 /* VALID_FALSE + PRIV tells the MMU to ignore corresponding SPTEs. */ 180 VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(5) /* PRIV. */, ptes); 181 } 182 183 static const struct nvkm_vmm_desc_func 184 gp100_vmm_desc_lpt = { 185 .invalid = gp100_vmm_lpt_invalid, 186 .unmap = gf100_vmm_pgt_unmap, 187 .sparse = gp100_vmm_pgt_sparse, 188 .mem = gp100_vmm_pgt_mem, 189 }; 190 191 static inline void 192 gp100_vmm_pd0_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 193 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) 194 { 195 u64 data = (addr >> 4) | map->type; 196 197 map->type += ptes * map->ctag; 198 199 while (ptes--) { 200 VMM_WO128(pt, vmm, ptei++ * 0x10, data, 0ULL); 201 data += map->next; 202 } 203 } 204 205 static void 206 gp100_vmm_pd0_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 207 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 208 { 209 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pd0_pte); 210 } 211 212 static inline bool 213 gp100_vmm_pde(struct nvkm_mmu_pt *pt, u64 *data) 214 { 215 switch (nvkm_memory_target(pt->memory)) { 216 case NVKM_MEM_TARGET_VRAM: *data |= 1ULL << 1; break; 217 case NVKM_MEM_TARGET_HOST: *data |= 2ULL << 1; 218 *data |= BIT_ULL(3); /* VOL. */ 219 break; 220 case NVKM_MEM_TARGET_NCOH: *data |= 3ULL << 1; break; 221 default: 222 WARN_ON(1); 223 return false; 224 } 225 *data |= pt->addr >> 4; 226 return true; 227 } 228 229 static void 230 gp100_vmm_pd0_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) 231 { 232 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 233 struct nvkm_mmu_pt *pd = pgd->pt[0]; 234 u64 data[2] = {}; 235 236 if (pgt->pt[0] && !gp100_vmm_pde(pgt->pt[0], &data[0])) 237 return; 238 if (pgt->pt[1] && !gp100_vmm_pde(pgt->pt[1], &data[1])) 239 return; 240 241 nvkm_kmap(pd->memory); 242 VMM_WO128(pd, vmm, pdei * 0x10, data[0], data[1]); 243 nvkm_done(pd->memory); 244 } 245 246 static void 247 gp100_vmm_pd0_sparse(struct nvkm_vmm *vmm, 248 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) 249 { 250 /* VALID_FALSE + VOL_BIG tells the MMU to treat the PDE as sparse. */ 251 VMM_FO128(pt, vmm, pdei * 0x10, BIT_ULL(3) /* VOL_BIG. */, 0ULL, pdes); 252 } 253 254 static void 255 gp100_vmm_pd0_unmap(struct nvkm_vmm *vmm, 256 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) 257 { 258 VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes); 259 } 260 261 static const struct nvkm_vmm_desc_func 262 gp100_vmm_desc_pd0 = { 263 .unmap = gp100_vmm_pd0_unmap, 264 .sparse = gp100_vmm_pd0_sparse, 265 .pde = gp100_vmm_pd0_pde, 266 .mem = gp100_vmm_pd0_mem, 267 }; 268 269 static void 270 gp100_vmm_pd1_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) 271 { 272 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 273 struct nvkm_mmu_pt *pd = pgd->pt[0]; 274 u64 data = 0; 275 276 if (!gp100_vmm_pde(pgt->pt[0], &data)) 277 return; 278 279 nvkm_kmap(pd->memory); 280 VMM_WO064(pd, vmm, pdei * 8, data); 281 nvkm_done(pd->memory); 282 } 283 284 static const struct nvkm_vmm_desc_func 285 gp100_vmm_desc_pd1 = { 286 .unmap = gf100_vmm_pgt_unmap, 287 .sparse = gp100_vmm_pgt_sparse, 288 .pde = gp100_vmm_pd1_pde, 289 }; 290 291 const struct nvkm_vmm_desc 292 gp100_vmm_desc_16[] = { 293 { LPT, 5, 8, 0x0100, &gp100_vmm_desc_lpt }, 294 { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 }, 295 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, 296 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, 297 { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 }, 298 {} 299 }; 300 301 const struct nvkm_vmm_desc 302 gp100_vmm_desc_12[] = { 303 { SPT, 9, 8, 0x1000, &gp100_vmm_desc_spt }, 304 { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 }, 305 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, 306 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, 307 { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 }, 308 {} 309 }; 310 311 int 312 gp100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, 313 struct nvkm_vmm_map *map) 314 { 315 const enum nvkm_memory_target target = nvkm_memory_target(map->memory); 316 const struct nvkm_vmm_page *page = map->page; 317 union { 318 struct gp100_vmm_map_vn vn; 319 struct gp100_vmm_map_v0 v0; 320 } *args = argv; 321 struct nvkm_device *device = vmm->mmu->subdev.device; 322 struct nvkm_memory *memory = map->memory; 323 u8 kind, kind_inv, priv, ro, vol; 324 int kindn, aper, ret = -ENOSYS; 325 const u8 *kindm; 326 327 map->next = (1ULL << page->shift) >> 4; 328 map->type = 0; 329 330 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 331 vol = !!args->v0.vol; 332 ro = !!args->v0.ro; 333 priv = !!args->v0.priv; 334 kind = args->v0.kind; 335 } else 336 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { 337 vol = target == NVKM_MEM_TARGET_HOST; 338 ro = 0; 339 priv = 0; 340 kind = 0x00; 341 } else { 342 VMM_DEBUG(vmm, "args"); 343 return ret; 344 } 345 346 aper = vmm->func->aper(target); 347 if (WARN_ON(aper < 0)) 348 return aper; 349 350 kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv); 351 if (kind >= kindn || kindm[kind] == kind_inv) { 352 VMM_DEBUG(vmm, "kind %02x", kind); 353 return -EINVAL; 354 } 355 356 if (kindm[kind] != kind) { 357 u64 tags = nvkm_memory_size(memory) >> 16; 358 if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) { 359 VMM_DEBUG(vmm, "comp %d %02x", aper, page->type); 360 return -EINVAL; 361 } 362 363 ret = nvkm_memory_tags_get(memory, device, tags, 364 nvkm_ltc_tags_clear, 365 &map->tags); 366 if (ret) { 367 VMM_DEBUG(vmm, "comp %d", ret); 368 return ret; 369 } 370 371 if (map->tags->mn) { 372 tags = map->tags->mn->offset + (map->offset >> 16); 373 map->ctag |= ((1ULL << page->shift) >> 16) << 36; 374 map->type |= tags << 36; 375 map->next |= map->ctag; 376 } else { 377 kind = kindm[kind]; 378 } 379 } 380 381 map->type |= BIT(0); 382 map->type |= (u64)aper << 1; 383 map->type |= (u64) vol << 3; 384 map->type |= (u64)priv << 5; 385 map->type |= (u64) ro << 6; 386 map->type |= (u64)kind << 56; 387 return 0; 388 } 389 390 static int 391 gp100_vmm_fault_cancel(struct nvkm_vmm *vmm, void *argv, u32 argc) 392 { 393 struct nvkm_device *device = vmm->mmu->subdev.device; 394 union { 395 struct gp100_vmm_fault_cancel_v0 v0; 396 } *args = argv; 397 int ret = -ENOSYS; 398 u32 inst, aper; 399 400 if ((ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) 401 return ret; 402 403 /* Translate MaxwellFaultBufferA instance pointer to the same 404 * format as the NV_GR_FECS_CURRENT_CTX register. 405 */ 406 aper = (args->v0.inst >> 8) & 3; 407 args->v0.inst >>= 12; 408 args->v0.inst |= aper << 28; 409 args->v0.inst |= 0x80000000; 410 411 if (!WARN_ON(nvkm_gr_ctxsw_pause(device))) { 412 if ((inst = nvkm_gr_ctxsw_inst(device)) == args->v0.inst) { 413 gf100_vmm_invalidate(vmm, 0x0000001b 414 /* CANCEL_TARGETED. */ | 415 (args->v0.hub << 20) | 416 (args->v0.gpc << 15) | 417 (args->v0.client << 9)); 418 } 419 WARN_ON(nvkm_gr_ctxsw_resume(device)); 420 } 421 422 return 0; 423 } 424 425 static int 426 gp100_vmm_fault_replay(struct nvkm_vmm *vmm, void *argv, u32 argc) 427 { 428 union { 429 struct gp100_vmm_fault_replay_vn vn; 430 } *args = argv; 431 int ret = -ENOSYS; 432 433 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { 434 gf100_vmm_invalidate(vmm, 0x0000000b); /* REPLAY_GLOBAL. */ 435 } 436 437 return ret; 438 } 439 440 int 441 gp100_vmm_mthd(struct nvkm_vmm *vmm, 442 struct nvkm_client *client, u32 mthd, void *argv, u32 argc) 443 { 444 if (client->super) { 445 switch (mthd) { 446 case GP100_VMM_VN_FAULT_REPLAY: 447 return gp100_vmm_fault_replay(vmm, argv, argc); 448 case GP100_VMM_VN_FAULT_CANCEL: 449 return gp100_vmm_fault_cancel(vmm, argv, argc); 450 default: 451 break; 452 } 453 } 454 return -EINVAL; 455 } 456 457 void 458 gp100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) 459 { 460 struct nvkm_device *device = vmm->mmu->subdev.device; 461 nvkm_wr32(device, 0x100cb8, lower_32_bits(addr)); 462 nvkm_wr32(device, 0x100cec, upper_32_bits(addr)); 463 } 464 465 void 466 gp100_vmm_flush(struct nvkm_vmm *vmm, int depth) 467 { 468 u32 type = (5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth) << 24; 469 type = 0; /*XXX: need to confirm stuff works with depth enabled... */ 470 if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) 471 type |= 0x00000004; /* HUB_ONLY */ 472 type |= 0x00000001; /* PAGE_ALL */ 473 gf100_vmm_invalidate(vmm, type); 474 } 475 476 int 477 gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) 478 { 479 u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11) /* 64KiB */; 480 if (vmm->replay) { 481 base |= BIT_ULL(4); /* FAULT_REPLAY_TEX */ 482 base |= BIT_ULL(5); /* FAULT_REPLAY_GCC */ 483 } 484 return gf100_vmm_join_(vmm, inst, base); 485 } 486 487 static const struct nvkm_vmm_func 488 gp100_vmm = { 489 .join = gp100_vmm_join, 490 .part = gf100_vmm_part, 491 .aper = gf100_vmm_aper, 492 .valid = gp100_vmm_valid, 493 .flush = gp100_vmm_flush, 494 .mthd = gp100_vmm_mthd, 495 .invalidate_pdb = gp100_vmm_invalidate_pdb, 496 .page = { 497 { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, 498 { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, 499 { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx }, 500 { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC }, 501 { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC }, 502 { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx }, 503 {} 504 } 505 }; 506 507 int 508 gp100_vmm_new_(const struct nvkm_vmm_func *func, 509 struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, 510 void *argv, u32 argc, struct lock_class_key *key, 511 const char *name, struct nvkm_vmm **pvmm) 512 { 513 union { 514 struct gp100_vmm_vn vn; 515 struct gp100_vmm_v0 v0; 516 } *args = argv; 517 int ret = -ENOSYS; 518 bool replay; 519 520 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 521 replay = args->v0.fault_replay != 0; 522 } else 523 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { 524 replay = false; 525 } else 526 return ret; 527 528 ret = nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm); 529 if (ret) 530 return ret; 531 532 (*pvmm)->replay = replay; 533 return 0; 534 } 535 536 int 537 gp100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, 538 void *argv, u32 argc, struct lock_class_key *key, 539 const char *name, struct nvkm_vmm **pvmm) 540 { 541 return gp100_vmm_new_(&gp100_vmm, mmu, managed, addr, size, 542 argv, argc, key, name, pvmm); 543 } 544