1 /* 2 * Copyright 2017 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #define NVKM_VMM_LEVELS_MAX 5 23 #include "vmm.h" 24 25 #include <subdev/fb.h> 26 27 static void 28 nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt) 29 { 30 struct nvkm_vmm_pt *pgt = *ppgt; 31 if (pgt) { 32 kvfree(pgt->pde); 33 kfree(pgt); 34 *ppgt = NULL; 35 } 36 } 37 38 39 static struct nvkm_vmm_pt * 40 nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse, 41 const struct nvkm_vmm_page *page) 42 { 43 const u32 pten = 1 << desc->bits; 44 struct nvkm_vmm_pt *pgt; 45 u32 lpte = 0; 46 47 if (desc->type > PGT) { 48 if (desc->type == SPT) { 49 const struct nvkm_vmm_desc *pair = page[-1].desc; 50 lpte = pten >> (desc->bits - pair->bits); 51 } else { 52 lpte = pten; 53 } 54 } 55 56 if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL))) 57 return NULL; 58 pgt->page = page ? page->shift : 0; 59 pgt->sparse = sparse; 60 61 if (desc->type == PGD) { 62 pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL); 63 if (!pgt->pde) { 64 kfree(pgt); 65 return NULL; 66 } 67 } 68 69 return pgt; 70 } 71 72 struct nvkm_vmm_iter { 73 const struct nvkm_vmm_page *page; 74 const struct nvkm_vmm_desc *desc; 75 struct nvkm_vmm *vmm; 76 u64 cnt; 77 u16 max, lvl; 78 u32 pte[NVKM_VMM_LEVELS_MAX]; 79 struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX]; 80 int flush; 81 }; 82 83 #ifdef CONFIG_NOUVEAU_DEBUG_MMU 84 static const char * 85 nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc) 86 { 87 switch (desc->type) { 88 case PGD: return "PGD"; 89 case PGT: return "PGT"; 90 case SPT: return "SPT"; 91 case LPT: return "LPT"; 92 default: 93 return "UNKNOWN"; 94 } 95 } 96 97 static void 98 nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf) 99 { 100 int lvl; 101 for (lvl = it->max; lvl >= 0; lvl--) { 102 if (lvl >= it->lvl) 103 buf += sprintf(buf, "%05x:", it->pte[lvl]); 104 else 105 buf += sprintf(buf, "xxxxx:"); 106 } 107 } 108 109 #define TRA(i,f,a...) do { \ 110 char _buf[NVKM_VMM_LEVELS_MAX * 7]; \ 111 struct nvkm_vmm_iter *_it = (i); \ 112 nvkm_vmm_trace(_it, _buf); \ 113 VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \ 114 } while(0) 115 #else 116 #define TRA(i,f,a...) 117 #endif 118 119 static inline void 120 nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it) 121 { 122 it->flush = min(it->flush, it->max - it->lvl); 123 } 124 125 static inline void 126 nvkm_vmm_flush(struct nvkm_vmm_iter *it) 127 { 128 if (it->flush != NVKM_VMM_LEVELS_MAX) { 129 if (it->vmm->func->flush) { 130 TRA(it, "flush: %d", it->flush); 131 it->vmm->func->flush(it->vmm, it->flush); 132 } 133 it->flush = NVKM_VMM_LEVELS_MAX; 134 } 135 } 136 137 static void 138 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it) 139 { 140 const struct nvkm_vmm_desc *desc = it->desc; 141 const int type = desc[it->lvl].type == SPT; 142 struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1]; 143 struct nvkm_vmm_pt *pgt = it->pt[it->lvl]; 144 struct nvkm_mmu_pt *pt = pgt->pt[type]; 145 struct nvkm_vmm *vmm = it->vmm; 146 u32 pdei = it->pte[it->lvl + 1]; 147 148 /* Recurse up the tree, unreferencing/destroying unneeded PDs. */ 149 it->lvl++; 150 if (--pgd->refs[0]) { 151 const struct nvkm_vmm_desc_func *func = desc[it->lvl].func; 152 /* PD has other valid PDEs, so we need a proper update. */ 153 TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1])); 154 pgt->pt[type] = NULL; 155 if (!pgt->refs[!type]) { 156 /* PDE no longer required. */ 157 if (pgd->pt[0]) { 158 if (pgt->sparse) { 159 func->sparse(vmm, pgd->pt[0], pdei, 1); 160 pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE; 161 } else { 162 func->unmap(vmm, pgd->pt[0], pdei, 1); 163 pgd->pde[pdei] = NULL; 164 } 165 } else { 166 /* Special handling for Tesla-class GPUs, 167 * where there's no central PD, but each 168 * instance has its own embedded PD. 169 */ 170 func->pde(vmm, pgd, pdei); 171 pgd->pde[pdei] = NULL; 172 } 173 } else { 174 /* PDE was pointing at dual-PTs and we're removing 175 * one of them, leaving the other in place. 176 */ 177 func->pde(vmm, pgd, pdei); 178 } 179 180 /* GPU may have cached the PTs, flush before freeing. */ 181 nvkm_vmm_flush_mark(it); 182 nvkm_vmm_flush(it); 183 } else { 184 /* PD has no valid PDEs left, so we can just destroy it. */ 185 nvkm_vmm_unref_pdes(it); 186 } 187 188 /* Destroy PD/PT. */ 189 TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1])); 190 nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt); 191 if (!pgt->refs[!type]) 192 nvkm_vmm_pt_del(&pgt); 193 it->lvl--; 194 } 195 196 static void 197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, 198 const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes) 199 { 200 const struct nvkm_vmm_desc *pair = it->page[-1].desc; 201 const u32 sptb = desc->bits - pair->bits; 202 const u32 sptn = 1 << sptb; 203 struct nvkm_vmm *vmm = it->vmm; 204 u32 spti = ptei & (sptn - 1), lpti, pteb; 205 206 /* Determine how many SPTEs are being touched under each LPTE, 207 * and drop reference counts. 208 */ 209 for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) { 210 const u32 pten = min(sptn - spti, ptes); 211 pgt->pte[lpti] -= pten; 212 ptes -= pten; 213 } 214 215 /* We're done here if there's no corresponding LPT. */ 216 if (!pgt->refs[0]) 217 return; 218 219 for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) { 220 /* Skip over any LPTEs that still have valid SPTEs. */ 221 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) { 222 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { 223 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)) 224 break; 225 } 226 continue; 227 } 228 229 /* As there's no more non-UNMAPPED SPTEs left in the range 230 * covered by a number of LPTEs, the LPTEs once again take 231 * control over their address range. 232 * 233 * Determine how many LPTEs need to transition state. 234 */ 235 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID; 236 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { 237 if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES) 238 break; 239 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID; 240 } 241 242 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) { 243 TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes); 244 pair->func->sparse(vmm, pgt->pt[0], pteb, ptes); 245 } else 246 if (pair->func->invalid) { 247 /* If the MMU supports it, restore the LPTE to the 248 * INVALID state to tell the MMU there is no point 249 * trying to fetch the corresponding SPTEs. 250 */ 251 TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes); 252 pair->func->invalid(vmm, pgt->pt[0], pteb, ptes); 253 } 254 } 255 } 256 257 static bool 258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) 259 { 260 const struct nvkm_vmm_desc *desc = it->desc; 261 const int type = desc->type == SPT; 262 struct nvkm_vmm_pt *pgt = it->pt[0]; 263 bool dma; 264 265 if (pfn) { 266 /* Need to clear PTE valid bits before we dma_unmap_page(). */ 267 dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes); 268 if (dma) { 269 /* GPU may have cached the PT, flush before unmap. */ 270 nvkm_vmm_flush_mark(it); 271 nvkm_vmm_flush(it); 272 desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes); 273 } 274 } 275 276 /* Drop PTE references. */ 277 pgt->refs[type] -= ptes; 278 279 /* Dual-PTs need special handling, unless PDE becoming invalid. */ 280 if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1])) 281 nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes); 282 283 /* PT no longer neeed? Destroy it. */ 284 if (!pgt->refs[type]) { 285 it->lvl++; 286 TRA(it, "%s empty", nvkm_vmm_desc_type(desc)); 287 it->lvl--; 288 nvkm_vmm_unref_pdes(it); 289 return false; /* PTE writes for unmap() not necessary. */ 290 } 291 292 return true; 293 } 294 295 static void 296 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, 297 const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes) 298 { 299 const struct nvkm_vmm_desc *pair = it->page[-1].desc; 300 const u32 sptb = desc->bits - pair->bits; 301 const u32 sptn = 1 << sptb; 302 struct nvkm_vmm *vmm = it->vmm; 303 u32 spti = ptei & (sptn - 1), lpti, pteb; 304 305 /* Determine how many SPTEs are being touched under each LPTE, 306 * and increase reference counts. 307 */ 308 for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) { 309 const u32 pten = min(sptn - spti, ptes); 310 pgt->pte[lpti] += pten; 311 ptes -= pten; 312 } 313 314 /* We're done here if there's no corresponding LPT. */ 315 if (!pgt->refs[0]) 316 return; 317 318 for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) { 319 /* Skip over any LPTEs that already have valid SPTEs. */ 320 if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) { 321 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { 322 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID)) 323 break; 324 } 325 continue; 326 } 327 328 /* As there are now non-UNMAPPED SPTEs in the range covered 329 * by a number of LPTEs, we need to transfer control of the 330 * address range to the SPTEs. 331 * 332 * Determine how many LPTEs need to transition state. 333 */ 334 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID; 335 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { 336 if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID) 337 break; 338 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID; 339 } 340 341 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) { 342 const u32 spti = pteb * sptn; 343 const u32 sptc = ptes * sptn; 344 /* The entire LPTE is marked as sparse, we need 345 * to make sure that the SPTEs are too. 346 */ 347 TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc); 348 desc->func->sparse(vmm, pgt->pt[1], spti, sptc); 349 /* Sparse LPTEs prevent SPTEs from being accessed. */ 350 TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes); 351 pair->func->unmap(vmm, pgt->pt[0], pteb, ptes); 352 } else 353 if (pair->func->invalid) { 354 /* MMU supports blocking SPTEs by marking an LPTE 355 * as INVALID. We need to reverse that here. 356 */ 357 TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes); 358 pair->func->unmap(vmm, pgt->pt[0], pteb, ptes); 359 } 360 } 361 } 362 363 static bool 364 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) 365 { 366 const struct nvkm_vmm_desc *desc = it->desc; 367 const int type = desc->type == SPT; 368 struct nvkm_vmm_pt *pgt = it->pt[0]; 369 370 /* Take PTE references. */ 371 pgt->refs[type] += ptes; 372 373 /* Dual-PTs need special handling. */ 374 if (desc->type == SPT) 375 nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes); 376 377 return true; 378 } 379 380 static void 381 nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc, 382 struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes) 383 { 384 if (desc->type == PGD) { 385 while (ptes--) 386 pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE; 387 } else 388 if (desc->type == LPT) { 389 memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes); 390 } 391 } 392 393 static bool 394 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) 395 { 396 struct nvkm_vmm_pt *pt = it->pt[0]; 397 if (it->desc->type == PGD) 398 memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes); 399 else 400 if (it->desc->type == LPT) 401 memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes); 402 return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes); 403 } 404 405 static bool 406 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) 407 { 408 nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes); 409 return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes); 410 } 411 412 static bool 413 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) 414 { 415 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1]; 416 const int type = desc->type == SPT; 417 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 418 const bool zero = !pgt->sparse && !desc->func->invalid; 419 struct nvkm_vmm *vmm = it->vmm; 420 struct nvkm_mmu *mmu = vmm->mmu; 421 struct nvkm_mmu_pt *pt; 422 u32 pten = 1 << desc->bits; 423 u32 pteb, ptei, ptes; 424 u32 size = desc->size * pten; 425 426 pgd->refs[0]++; 427 428 pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero); 429 if (!pgt->pt[type]) { 430 it->lvl--; 431 nvkm_vmm_unref_pdes(it); 432 return false; 433 } 434 435 if (zero) 436 goto done; 437 438 pt = pgt->pt[type]; 439 440 if (desc->type == LPT && pgt->refs[1]) { 441 /* SPT already exists covering the same range as this LPT, 442 * which means we need to be careful that any LPTEs which 443 * overlap valid SPTEs are unmapped as opposed to invalid 444 * or sparse, which would prevent the MMU from looking at 445 * the SPTEs on some GPUs. 446 */ 447 for (ptei = pteb = 0; ptei < pten; pteb = ptei) { 448 bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES; 449 for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) { 450 bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES; 451 if (spte != next) 452 break; 453 } 454 455 if (!spte) { 456 if (pgt->sparse) 457 desc->func->sparse(vmm, pt, pteb, ptes); 458 else 459 desc->func->invalid(vmm, pt, pteb, ptes); 460 memset(&pgt->pte[pteb], 0x00, ptes); 461 } else { 462 desc->func->unmap(vmm, pt, pteb, ptes); 463 while (ptes--) 464 pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID; 465 } 466 } 467 } else { 468 if (pgt->sparse) { 469 nvkm_vmm_sparse_ptes(desc, pgt, 0, pten); 470 desc->func->sparse(vmm, pt, 0, pten); 471 } else { 472 desc->func->invalid(vmm, pt, 0, pten); 473 } 474 } 475 476 done: 477 TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc)); 478 it->desc[it->lvl].func->pde(it->vmm, pgd, pdei); 479 nvkm_vmm_flush_mark(it); 480 return true; 481 } 482 483 static bool 484 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) 485 { 486 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1]; 487 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 488 489 pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page); 490 if (!pgt) { 491 if (!pgd->refs[0]) 492 nvkm_vmm_unref_pdes(it); 493 return false; 494 } 495 496 pgd->pde[pdei] = pgt; 497 return true; 498 } 499 500 static inline u64 501 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 502 u64 addr, u64 size, const char *name, bool ref, bool pfn, 503 bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32), 504 nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map, 505 nvkm_vmm_pxe_func CLR_PTES) 506 { 507 const struct nvkm_vmm_desc *desc = page->desc; 508 struct nvkm_vmm_iter it; 509 u64 bits = addr >> page->shift; 510 511 it.page = page; 512 it.desc = desc; 513 it.vmm = vmm; 514 it.cnt = size >> page->shift; 515 it.flush = NVKM_VMM_LEVELS_MAX; 516 517 /* Deconstruct address into PTE indices for each mapping level. */ 518 for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) { 519 it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1); 520 bits >>= desc[it.lvl].bits; 521 } 522 it.max = --it.lvl; 523 it.pt[it.max] = vmm->pd; 524 525 it.lvl = 0; 526 TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name, 527 addr, size, page->shift, it.cnt); 528 it.lvl = it.max; 529 530 /* Depth-first traversal of page tables. */ 531 while (it.cnt) { 532 struct nvkm_vmm_pt *pgt = it.pt[it.lvl]; 533 const int type = desc->type == SPT; 534 const u32 pten = 1 << desc->bits; 535 const u32 ptei = it.pte[0]; 536 const u32 ptes = min_t(u64, it.cnt, pten - ptei); 537 538 /* Walk down the tree, finding page tables for each level. */ 539 for (; it.lvl; it.lvl--) { 540 const u32 pdei = it.pte[it.lvl]; 541 struct nvkm_vmm_pt *pgd = pgt; 542 543 /* Software PT. */ 544 if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) { 545 if (!nvkm_vmm_ref_swpt(&it, pgd, pdei)) 546 goto fail; 547 } 548 it.pt[it.lvl - 1] = pgt = pgd->pde[pdei]; 549 550 /* Hardware PT. 551 * 552 * This is a separate step from above due to GF100 and 553 * newer having dual page tables at some levels, which 554 * are refcounted independently. 555 */ 556 if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) { 557 if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei)) 558 goto fail; 559 } 560 } 561 562 /* Handle PTE updates. */ 563 if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) { 564 struct nvkm_mmu_pt *pt = pgt->pt[type]; 565 if (MAP_PTES || CLR_PTES) { 566 if (MAP_PTES) 567 MAP_PTES(vmm, pt, ptei, ptes, map); 568 else 569 CLR_PTES(vmm, pt, ptei, ptes); 570 nvkm_vmm_flush_mark(&it); 571 } 572 } 573 574 /* Walk back up the tree to the next position. */ 575 it.pte[it.lvl] += ptes; 576 it.cnt -= ptes; 577 if (it.cnt) { 578 while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) { 579 it.pte[it.lvl++] = 0; 580 it.pte[it.lvl]++; 581 } 582 } 583 } 584 585 nvkm_vmm_flush(&it); 586 return ~0ULL; 587 588 fail: 589 /* Reconstruct the failure address so the caller is able to 590 * reverse any partially completed operations. 591 */ 592 addr = it.pte[it.max--]; 593 do { 594 addr = addr << desc[it.max].bits; 595 addr |= it.pte[it.max]; 596 } while (it.max--); 597 598 return addr << page->shift; 599 } 600 601 static void 602 nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 603 u64 addr, u64 size) 604 { 605 nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false, 606 nvkm_vmm_sparse_unref_ptes, NULL, NULL, 607 page->desc->func->invalid ? 608 page->desc->func->invalid : page->desc->func->unmap); 609 } 610 611 static int 612 nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 613 u64 addr, u64 size) 614 { 615 if ((page->type & NVKM_VMM_PAGE_SPARSE)) { 616 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref", 617 true, false, nvkm_vmm_sparse_ref_ptes, 618 NULL, NULL, page->desc->func->sparse); 619 if (fail != ~0ULL) { 620 if ((size = fail - addr)) 621 nvkm_vmm_ptes_sparse_put(vmm, page, addr, size); 622 return -ENOMEM; 623 } 624 return 0; 625 } 626 return -EINVAL; 627 } 628 629 static int 630 nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref) 631 { 632 const struct nvkm_vmm_page *page = vmm->func->page; 633 int m = 0, i; 634 u64 start = addr; 635 u64 block; 636 637 while (size) { 638 /* Limit maximum page size based on remaining size. */ 639 while (size < (1ULL << page[m].shift)) 640 m++; 641 i = m; 642 643 /* Find largest page size suitable for alignment. */ 644 while (!IS_ALIGNED(addr, 1ULL << page[i].shift)) 645 i++; 646 647 /* Determine number of PTEs at this page size. */ 648 if (i != m) { 649 /* Limited to alignment boundary of next page size. */ 650 u64 next = 1ULL << page[i - 1].shift; 651 u64 part = ALIGN(addr, next) - addr; 652 if (size - part >= next) 653 block = (part >> page[i].shift) << page[i].shift; 654 else 655 block = (size >> page[i].shift) << page[i].shift; 656 } else { 657 block = (size >> page[i].shift) << page[i].shift; 658 } 659 660 /* Perform operation. */ 661 if (ref) { 662 int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block); 663 if (ret) { 664 if ((size = addr - start)) 665 nvkm_vmm_ptes_sparse(vmm, start, size, false); 666 return ret; 667 } 668 } else { 669 nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block); 670 } 671 672 size -= block; 673 addr += block; 674 } 675 676 return 0; 677 } 678 679 static void 680 nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 681 u64 addr, u64 size, bool sparse, bool pfn) 682 { 683 const struct nvkm_vmm_desc_func *func = page->desc->func; 684 nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref", 685 false, pfn, nvkm_vmm_unref_ptes, NULL, NULL, 686 sparse ? func->sparse : func->invalid ? func->invalid : 687 func->unmap); 688 } 689 690 static int 691 nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 692 u64 addr, u64 size, struct nvkm_vmm_map *map, 693 nvkm_vmm_pte_func func) 694 { 695 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true, 696 false, nvkm_vmm_ref_ptes, func, map, NULL); 697 if (fail != ~0ULL) { 698 if ((size = fail - addr)) 699 nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false); 700 return -ENOMEM; 701 } 702 return 0; 703 } 704 705 static void 706 nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 707 u64 addr, u64 size, bool sparse, bool pfn) 708 { 709 const struct nvkm_vmm_desc_func *func = page->desc->func; 710 nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn, 711 NULL, NULL, NULL, 712 sparse ? func->sparse : func->invalid ? func->invalid : 713 func->unmap); 714 } 715 716 static void 717 nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 718 u64 addr, u64 size, struct nvkm_vmm_map *map, 719 nvkm_vmm_pte_func func) 720 { 721 nvkm_vmm_iter(vmm, page, addr, size, "map", false, false, 722 NULL, func, map, NULL); 723 } 724 725 static void 726 nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 727 u64 addr, u64 size) 728 { 729 nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false, 730 nvkm_vmm_unref_ptes, NULL, NULL, NULL); 731 } 732 733 static int 734 nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, 735 u64 addr, u64 size) 736 { 737 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false, 738 nvkm_vmm_ref_ptes, NULL, NULL, NULL); 739 if (fail != ~0ULL) { 740 if (fail != addr) 741 nvkm_vmm_ptes_put(vmm, page, addr, fail - addr); 742 return -ENOMEM; 743 } 744 return 0; 745 } 746 747 static inline struct nvkm_vma * 748 nvkm_vma_new(u64 addr, u64 size) 749 { 750 struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 751 if (vma) { 752 vma->addr = addr; 753 vma->size = size; 754 vma->page = NVKM_VMA_PAGE_NONE; 755 vma->refd = NVKM_VMA_PAGE_NONE; 756 } 757 return vma; 758 } 759 760 struct nvkm_vma * 761 nvkm_vma_tail(struct nvkm_vma *vma, u64 tail) 762 { 763 struct nvkm_vma *new; 764 765 BUG_ON(vma->size == tail); 766 767 if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail))) 768 return NULL; 769 vma->size -= tail; 770 771 new->mapref = vma->mapref; 772 new->sparse = vma->sparse; 773 new->page = vma->page; 774 new->refd = vma->refd; 775 new->used = vma->used; 776 new->part = vma->part; 777 new->user = vma->user; 778 new->busy = vma->busy; 779 new->mapped = vma->mapped; 780 list_add(&new->head, &vma->head); 781 return new; 782 } 783 784 static inline void 785 nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 786 { 787 rb_erase(&vma->tree, &vmm->free); 788 } 789 790 static inline void 791 nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 792 { 793 nvkm_vmm_free_remove(vmm, vma); 794 list_del(&vma->head); 795 kfree(vma); 796 } 797 798 static void 799 nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 800 { 801 struct rb_node **ptr = &vmm->free.rb_node; 802 struct rb_node *parent = NULL; 803 804 while (*ptr) { 805 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree); 806 parent = *ptr; 807 if (vma->size < this->size) 808 ptr = &parent->rb_left; 809 else 810 if (vma->size > this->size) 811 ptr = &parent->rb_right; 812 else 813 if (vma->addr < this->addr) 814 ptr = &parent->rb_left; 815 else 816 if (vma->addr > this->addr) 817 ptr = &parent->rb_right; 818 else 819 BUG(); 820 } 821 822 rb_link_node(&vma->tree, parent, ptr); 823 rb_insert_color(&vma->tree, &vmm->free); 824 } 825 826 static inline void 827 nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 828 { 829 rb_erase(&vma->tree, &vmm->root); 830 } 831 832 static inline void 833 nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 834 { 835 nvkm_vmm_node_remove(vmm, vma); 836 list_del(&vma->head); 837 kfree(vma); 838 } 839 840 static void 841 nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 842 { 843 struct rb_node **ptr = &vmm->root.rb_node; 844 struct rb_node *parent = NULL; 845 846 while (*ptr) { 847 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree); 848 parent = *ptr; 849 if (vma->addr < this->addr) 850 ptr = &parent->rb_left; 851 else 852 if (vma->addr > this->addr) 853 ptr = &parent->rb_right; 854 else 855 BUG(); 856 } 857 858 rb_link_node(&vma->tree, parent, ptr); 859 rb_insert_color(&vma->tree, &vmm->root); 860 } 861 862 struct nvkm_vma * 863 nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr) 864 { 865 struct rb_node *node = vmm->root.rb_node; 866 while (node) { 867 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree); 868 if (addr < vma->addr) 869 node = node->rb_left; 870 else 871 if (addr >= vma->addr + vma->size) 872 node = node->rb_right; 873 else 874 return vma; 875 } 876 return NULL; 877 } 878 879 #define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL : \ 880 list_entry((root)->head.dir, struct nvkm_vma, head)) 881 882 static struct nvkm_vma * 883 nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev, 884 struct nvkm_vma *vma, struct nvkm_vma *next, u64 size) 885 { 886 if (next) { 887 if (vma->size == size) { 888 vma->size += next->size; 889 nvkm_vmm_node_delete(vmm, next); 890 if (prev) { 891 prev->size += vma->size; 892 nvkm_vmm_node_delete(vmm, vma); 893 return prev; 894 } 895 return vma; 896 } 897 BUG_ON(prev); 898 899 nvkm_vmm_node_remove(vmm, next); 900 vma->size -= size; 901 next->addr -= size; 902 next->size += size; 903 nvkm_vmm_node_insert(vmm, next); 904 return next; 905 } 906 907 if (prev) { 908 if (vma->size != size) { 909 nvkm_vmm_node_remove(vmm, vma); 910 prev->size += size; 911 vma->addr += size; 912 vma->size -= size; 913 nvkm_vmm_node_insert(vmm, vma); 914 } else { 915 prev->size += vma->size; 916 nvkm_vmm_node_delete(vmm, vma); 917 } 918 return prev; 919 } 920 921 return vma; 922 } 923 924 struct nvkm_vma * 925 nvkm_vmm_node_split(struct nvkm_vmm *vmm, 926 struct nvkm_vma *vma, u64 addr, u64 size) 927 { 928 struct nvkm_vma *prev = NULL; 929 930 if (vma->addr != addr) { 931 prev = vma; 932 if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) 933 return NULL; 934 vma->part = true; 935 nvkm_vmm_node_insert(vmm, vma); 936 } 937 938 if (vma->size != size) { 939 struct nvkm_vma *tmp; 940 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) { 941 nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size); 942 return NULL; 943 } 944 tmp->part = true; 945 nvkm_vmm_node_insert(vmm, tmp); 946 } 947 948 return vma; 949 } 950 951 static void 952 nvkm_vma_dump(struct nvkm_vma *vma) 953 { 954 printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c%c %p\n", 955 vma->addr, (u64)vma->size, 956 vma->used ? '-' : 'F', 957 vma->mapref ? 'R' : '-', 958 vma->sparse ? 'S' : '-', 959 vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-', 960 vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-', 961 vma->part ? 'P' : '-', 962 vma->user ? 'U' : '-', 963 vma->busy ? 'B' : '-', 964 vma->mapped ? 'M' : '-', 965 vma->memory); 966 } 967 968 static void 969 nvkm_vmm_dump(struct nvkm_vmm *vmm) 970 { 971 struct nvkm_vma *vma; 972 list_for_each_entry(vma, &vmm->list, head) { 973 nvkm_vma_dump(vma); 974 } 975 } 976 977 static void 978 nvkm_vmm_dtor(struct nvkm_vmm *vmm) 979 { 980 struct nvkm_vma *vma; 981 struct rb_node *node; 982 983 if (0) 984 nvkm_vmm_dump(vmm); 985 986 while ((node = rb_first(&vmm->root))) { 987 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree); 988 nvkm_vmm_put(vmm, &vma); 989 } 990 991 if (vmm->bootstrapped) { 992 const struct nvkm_vmm_page *page = vmm->func->page; 993 const u64 limit = vmm->limit - vmm->start; 994 995 while (page[1].shift) 996 page++; 997 998 nvkm_mmu_ptc_dump(vmm->mmu); 999 nvkm_vmm_ptes_put(vmm, page, vmm->start, limit); 1000 } 1001 1002 vma = list_first_entry(&vmm->list, typeof(*vma), head); 1003 list_del(&vma->head); 1004 kfree(vma); 1005 WARN_ON(!list_empty(&vmm->list)); 1006 1007 if (vmm->nullp) { 1008 dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024, 1009 vmm->nullp, vmm->null); 1010 } 1011 1012 if (vmm->pd) { 1013 nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]); 1014 nvkm_vmm_pt_del(&vmm->pd); 1015 } 1016 } 1017 1018 static int 1019 nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size) 1020 { 1021 struct nvkm_vma *vma; 1022 if (!(vma = nvkm_vma_new(addr, size))) 1023 return -ENOMEM; 1024 vma->mapref = true; 1025 vma->sparse = false; 1026 vma->used = true; 1027 vma->user = true; 1028 nvkm_vmm_node_insert(vmm, vma); 1029 list_add_tail(&vma->head, &vmm->list); 1030 return 0; 1031 } 1032 1033 static int 1034 nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, 1035 u32 pd_header, bool managed, u64 addr, u64 size, 1036 struct lock_class_key *key, const char *name, 1037 struct nvkm_vmm *vmm) 1038 { 1039 static struct lock_class_key _key; 1040 const struct nvkm_vmm_page *page = func->page; 1041 const struct nvkm_vmm_desc *desc; 1042 struct nvkm_vma *vma; 1043 int levels, bits = 0, ret; 1044 1045 vmm->func = func; 1046 vmm->mmu = mmu; 1047 vmm->name = name; 1048 vmm->debug = mmu->subdev.debug; 1049 kref_init(&vmm->kref); 1050 1051 __mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key); 1052 1053 /* Locate the smallest page size supported by the backend, it will 1054 * have the the deepest nesting of page tables. 1055 */ 1056 while (page[1].shift) 1057 page++; 1058 1059 /* Locate the structure that describes the layout of the top-level 1060 * page table, and determine the number of valid bits in a virtual 1061 * address. 1062 */ 1063 for (levels = 0, desc = page->desc; desc->bits; desc++, levels++) 1064 bits += desc->bits; 1065 bits += page->shift; 1066 desc--; 1067 1068 if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX)) 1069 return -EINVAL; 1070 1071 /* Allocate top-level page table. */ 1072 vmm->pd = nvkm_vmm_pt_new(desc, false, NULL); 1073 if (!vmm->pd) 1074 return -ENOMEM; 1075 vmm->pd->refs[0] = 1; 1076 INIT_LIST_HEAD(&vmm->join); 1077 1078 /* ... and the GPU storage for it, except on Tesla-class GPUs that 1079 * have the PD embedded in the instance structure. 1080 */ 1081 if (desc->size) { 1082 const u32 size = pd_header + desc->size * (1 << desc->bits); 1083 vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true); 1084 if (!vmm->pd->pt[0]) 1085 return -ENOMEM; 1086 } 1087 1088 /* Initialise address-space MM. */ 1089 INIT_LIST_HEAD(&vmm->list); 1090 vmm->free = RB_ROOT; 1091 vmm->root = RB_ROOT; 1092 1093 if (managed) { 1094 /* Address-space will be managed by the client for the most 1095 * part, except for a specified area where NVKM allocations 1096 * are allowed to be placed. 1097 */ 1098 vmm->start = 0; 1099 vmm->limit = 1ULL << bits; 1100 if (addr + size < addr || addr + size > vmm->limit) 1101 return -EINVAL; 1102 1103 /* Client-managed area before the NVKM-managed area. */ 1104 if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr))) 1105 return ret; 1106 1107 /* NVKM-managed area. */ 1108 if (size) { 1109 if (!(vma = nvkm_vma_new(addr, size))) 1110 return -ENOMEM; 1111 nvkm_vmm_free_insert(vmm, vma); 1112 list_add_tail(&vma->head, &vmm->list); 1113 } 1114 1115 /* Client-managed area after the NVKM-managed area. */ 1116 addr = addr + size; 1117 size = vmm->limit - addr; 1118 if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size))) 1119 return ret; 1120 } else { 1121 /* Address-space fully managed by NVKM, requiring calls to 1122 * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space. 1123 */ 1124 vmm->start = addr; 1125 vmm->limit = size ? (addr + size) : (1ULL << bits); 1126 if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) 1127 return -EINVAL; 1128 1129 if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start))) 1130 return -ENOMEM; 1131 1132 nvkm_vmm_free_insert(vmm, vma); 1133 list_add(&vma->head, &vmm->list); 1134 } 1135 1136 return 0; 1137 } 1138 1139 int 1140 nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, 1141 u32 hdr, bool managed, u64 addr, u64 size, 1142 struct lock_class_key *key, const char *name, 1143 struct nvkm_vmm **pvmm) 1144 { 1145 if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL))) 1146 return -ENOMEM; 1147 return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm); 1148 } 1149 1150 static struct nvkm_vma * 1151 nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma, 1152 u64 addr, u64 size, u8 page, bool map) 1153 { 1154 struct nvkm_vma *prev = NULL; 1155 struct nvkm_vma *next = NULL; 1156 1157 if (vma->addr == addr && vma->part && (prev = node(vma, prev))) { 1158 if (prev->memory || prev->mapped != map) 1159 prev = NULL; 1160 } 1161 1162 if (vma->addr + vma->size == addr + size && (next = node(vma, next))) { 1163 if (!next->part || 1164 next->memory || next->mapped != map) 1165 next = NULL; 1166 } 1167 1168 if (prev || next) 1169 return nvkm_vmm_node_merge(vmm, prev, vma, next, size); 1170 return nvkm_vmm_node_split(vmm, vma, addr, size); 1171 } 1172 1173 int 1174 nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size) 1175 { 1176 struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr); 1177 struct nvkm_vma *next; 1178 u64 limit = addr + size; 1179 u64 start = addr; 1180 1181 if (!vma) 1182 return -EINVAL; 1183 1184 do { 1185 if (!vma->mapped || vma->memory) 1186 continue; 1187 1188 size = min(limit - start, vma->size - (start - vma->addr)); 1189 1190 nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd], 1191 start, size, false, true); 1192 1193 next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false); 1194 if (!WARN_ON(!next)) { 1195 vma = next; 1196 vma->refd = NVKM_VMA_PAGE_NONE; 1197 vma->mapped = false; 1198 } 1199 } while ((vma = node(vma, next)) && (start = vma->addr) < limit); 1200 1201 return 0; 1202 } 1203 1204 /*TODO: 1205 * - Avoid PT readback (for dma_unmap etc), this might end up being dealt 1206 * with inside HMM, which would be a lot nicer for us to deal with. 1207 * - Support for systems without a 4KiB page size. 1208 */ 1209 int 1210 nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn) 1211 { 1212 const struct nvkm_vmm_page *page = vmm->func->page; 1213 struct nvkm_vma *vma, *tmp; 1214 u64 limit = addr + size; 1215 u64 start = addr; 1216 int pm = size >> shift; 1217 int pi = 0; 1218 1219 /* Only support mapping where the page size of the incoming page 1220 * array matches a page size available for direct mapping. 1221 */ 1222 while (page->shift && (page->shift != shift || 1223 page->desc->func->pfn == NULL)) 1224 page++; 1225 1226 if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) || 1227 !IS_ALIGNED(size, 1ULL << shift) || 1228 addr + size < addr || addr + size > vmm->limit) { 1229 VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n", 1230 shift, page->shift, addr, size); 1231 return -EINVAL; 1232 } 1233 1234 if (!(vma = nvkm_vmm_node_search(vmm, addr))) 1235 return -ENOENT; 1236 1237 do { 1238 bool map = !!(pfn[pi] & NVKM_VMM_PFN_V); 1239 bool mapped = vma->mapped; 1240 u64 size = limit - start; 1241 u64 addr = start; 1242 int pn, ret = 0; 1243 1244 /* Narrow the operation window to cover a single action (page 1245 * should be mapped or not) within a single VMA. 1246 */ 1247 for (pn = 0; pi + pn < pm; pn++) { 1248 if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V)) 1249 break; 1250 } 1251 size = min_t(u64, size, pn << page->shift); 1252 size = min_t(u64, size, vma->size + vma->addr - addr); 1253 1254 /* Reject any operation to unmanaged regions, and areas that 1255 * have nvkm_memory objects mapped in them already. 1256 */ 1257 if (!vma->mapref || vma->memory) { 1258 ret = -EINVAL; 1259 goto next; 1260 } 1261 1262 /* In order to both properly refcount GPU page tables, and 1263 * prevent "normal" mappings and these direct mappings from 1264 * interfering with each other, we need to track contiguous 1265 * ranges that have been mapped with this interface. 1266 * 1267 * Here we attempt to either split an existing VMA so we're 1268 * able to flag the region as either unmapped/mapped, or to 1269 * merge with adjacent VMAs that are already compatible. 1270 * 1271 * If the region is already compatible, nothing is required. 1272 */ 1273 if (map != mapped) { 1274 tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size, 1275 page - 1276 vmm->func->page, map); 1277 if (WARN_ON(!tmp)) { 1278 ret = -ENOMEM; 1279 goto next; 1280 } 1281 1282 if ((tmp->mapped = map)) 1283 tmp->refd = page - vmm->func->page; 1284 else 1285 tmp->refd = NVKM_VMA_PAGE_NONE; 1286 vma = tmp; 1287 } 1288 1289 /* Update HW page tables. */ 1290 if (map) { 1291 struct nvkm_vmm_map args; 1292 args.page = page; 1293 args.pfn = &pfn[pi]; 1294 1295 if (!mapped) { 1296 ret = nvkm_vmm_ptes_get_map(vmm, page, addr, 1297 size, &args, page-> 1298 desc->func->pfn); 1299 } else { 1300 nvkm_vmm_ptes_map(vmm, page, addr, size, &args, 1301 page->desc->func->pfn); 1302 } 1303 } else { 1304 if (mapped) { 1305 nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, 1306 false, true); 1307 } 1308 } 1309 1310 next: 1311 /* Iterate to next operation. */ 1312 if (vma->addr + vma->size == addr + size) 1313 vma = node(vma, next); 1314 start += size; 1315 1316 if (ret) { 1317 /* Failure is signalled by clearing the valid bit on 1318 * any PFN that couldn't be modified as requested. 1319 */ 1320 while (size) { 1321 pfn[pi++] = NVKM_VMM_PFN_NONE; 1322 size -= 1 << page->shift; 1323 } 1324 } else { 1325 pi += size >> page->shift; 1326 } 1327 } while (vma && start < limit); 1328 1329 return 0; 1330 } 1331 1332 void 1333 nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 1334 { 1335 struct nvkm_vma *prev = NULL; 1336 struct nvkm_vma *next; 1337 1338 nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); 1339 nvkm_memory_unref(&vma->memory); 1340 vma->mapped = false; 1341 1342 if (vma->part && (prev = node(vma, prev)) && prev->mapped) 1343 prev = NULL; 1344 if ((next = node(vma, next)) && (!next->part || next->mapped)) 1345 next = NULL; 1346 nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size); 1347 } 1348 1349 void 1350 nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn) 1351 { 1352 const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd]; 1353 1354 if (vma->mapref) { 1355 nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn); 1356 vma->refd = NVKM_VMA_PAGE_NONE; 1357 } else { 1358 nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn); 1359 } 1360 1361 nvkm_vmm_unmap_region(vmm, vma); 1362 } 1363 1364 void 1365 nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 1366 { 1367 if (vma->memory) { 1368 mutex_lock(&vmm->mutex); 1369 nvkm_vmm_unmap_locked(vmm, vma, false); 1370 mutex_unlock(&vmm->mutex); 1371 } 1372 } 1373 1374 static int 1375 nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma, 1376 void *argv, u32 argc, struct nvkm_vmm_map *map) 1377 { 1378 switch (nvkm_memory_target(map->memory)) { 1379 case NVKM_MEM_TARGET_VRAM: 1380 if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) { 1381 VMM_DEBUG(vmm, "%d !VRAM", map->page->shift); 1382 return -EINVAL; 1383 } 1384 break; 1385 case NVKM_MEM_TARGET_HOST: 1386 case NVKM_MEM_TARGET_NCOH: 1387 if (!(map->page->type & NVKM_VMM_PAGE_HOST)) { 1388 VMM_DEBUG(vmm, "%d !HOST", map->page->shift); 1389 return -EINVAL; 1390 } 1391 break; 1392 default: 1393 WARN_ON(1); 1394 return -ENOSYS; 1395 } 1396 1397 if (!IS_ALIGNED( vma->addr, 1ULL << map->page->shift) || 1398 !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) || 1399 !IS_ALIGNED( map->offset, 1ULL << map->page->shift) || 1400 nvkm_memory_page(map->memory) < map->page->shift) { 1401 VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d", 1402 vma->addr, (u64)vma->size, map->offset, map->page->shift, 1403 nvkm_memory_page(map->memory)); 1404 return -EINVAL; 1405 } 1406 1407 return vmm->func->valid(vmm, argv, argc, map); 1408 } 1409 1410 static int 1411 nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma, 1412 void *argv, u32 argc, struct nvkm_vmm_map *map) 1413 { 1414 for (map->page = vmm->func->page; map->page->shift; map->page++) { 1415 VMM_DEBUG(vmm, "trying %d", map->page->shift); 1416 if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map)) 1417 return 0; 1418 } 1419 return -EINVAL; 1420 } 1421 1422 static int 1423 nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, 1424 void *argv, u32 argc, struct nvkm_vmm_map *map) 1425 { 1426 nvkm_vmm_pte_func func; 1427 int ret; 1428 1429 /* Make sure we won't overrun the end of the memory object. */ 1430 if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) { 1431 VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx", 1432 nvkm_memory_size(map->memory), 1433 map->offset, (u64)vma->size); 1434 return -EINVAL; 1435 } 1436 1437 /* Check remaining arguments for validity. */ 1438 if (vma->page == NVKM_VMA_PAGE_NONE && 1439 vma->refd == NVKM_VMA_PAGE_NONE) { 1440 /* Find the largest page size we can perform the mapping at. */ 1441 const u32 debug = vmm->debug; 1442 vmm->debug = 0; 1443 ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map); 1444 vmm->debug = debug; 1445 if (ret) { 1446 VMM_DEBUG(vmm, "invalid at any page size"); 1447 nvkm_vmm_map_choose(vmm, vma, argv, argc, map); 1448 return -EINVAL; 1449 } 1450 } else { 1451 /* Page size of the VMA is already pre-determined. */ 1452 if (vma->refd != NVKM_VMA_PAGE_NONE) 1453 map->page = &vmm->func->page[vma->refd]; 1454 else 1455 map->page = &vmm->func->page[vma->page]; 1456 1457 ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map); 1458 if (ret) { 1459 VMM_DEBUG(vmm, "invalid %d\n", ret); 1460 return ret; 1461 } 1462 } 1463 1464 /* Deal with the 'offset' argument, and fetch the backend function. */ 1465 map->off = map->offset; 1466 if (map->mem) { 1467 for (; map->off; map->mem = map->mem->next) { 1468 u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT; 1469 if (size > map->off) 1470 break; 1471 map->off -= size; 1472 } 1473 func = map->page->desc->func->mem; 1474 } else 1475 if (map->sgl) { 1476 for (; map->off; map->sgl = sg_next(map->sgl)) { 1477 u64 size = sg_dma_len(map->sgl); 1478 if (size > map->off) 1479 break; 1480 map->off -= size; 1481 } 1482 func = map->page->desc->func->sgl; 1483 } else { 1484 map->dma += map->offset >> PAGE_SHIFT; 1485 map->off = map->offset & PAGE_MASK; 1486 func = map->page->desc->func->dma; 1487 } 1488 1489 /* Perform the map. */ 1490 if (vma->refd == NVKM_VMA_PAGE_NONE) { 1491 ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func); 1492 if (ret) 1493 return ret; 1494 1495 vma->refd = map->page - vmm->func->page; 1496 } else { 1497 nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func); 1498 } 1499 1500 nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); 1501 nvkm_memory_unref(&vma->memory); 1502 vma->memory = nvkm_memory_ref(map->memory); 1503 vma->mapped = true; 1504 vma->tags = map->tags; 1505 return 0; 1506 } 1507 1508 int 1509 nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc, 1510 struct nvkm_vmm_map *map) 1511 { 1512 int ret; 1513 mutex_lock(&vmm->mutex); 1514 ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map); 1515 vma->busy = false; 1516 mutex_unlock(&vmm->mutex); 1517 return ret; 1518 } 1519 1520 static void 1521 nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 1522 { 1523 struct nvkm_vma *prev, *next; 1524 1525 if ((prev = node(vma, prev)) && !prev->used) { 1526 vma->addr = prev->addr; 1527 vma->size += prev->size; 1528 nvkm_vmm_free_delete(vmm, prev); 1529 } 1530 1531 if ((next = node(vma, next)) && !next->used) { 1532 vma->size += next->size; 1533 nvkm_vmm_free_delete(vmm, next); 1534 } 1535 1536 nvkm_vmm_free_insert(vmm, vma); 1537 } 1538 1539 void 1540 nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) 1541 { 1542 const struct nvkm_vmm_page *page = vmm->func->page; 1543 struct nvkm_vma *next = vma; 1544 1545 BUG_ON(vma->part); 1546 1547 if (vma->mapref || !vma->sparse) { 1548 do { 1549 const bool mem = next->memory != NULL; 1550 const bool map = next->mapped; 1551 const u8 refd = next->refd; 1552 const u64 addr = next->addr; 1553 u64 size = next->size; 1554 1555 /* Merge regions that are in the same state. */ 1556 while ((next = node(next, next)) && next->part && 1557 (next->mapped == map) && 1558 (next->memory != NULL) == mem && 1559 (next->refd == refd)) 1560 size += next->size; 1561 1562 if (map) { 1563 /* Region(s) are mapped, merge the unmap 1564 * and dereference into a single walk of 1565 * the page tree. 1566 */ 1567 nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr, 1568 size, vma->sparse, 1569 !mem); 1570 } else 1571 if (refd != NVKM_VMA_PAGE_NONE) { 1572 /* Drop allocation-time PTE references. */ 1573 nvkm_vmm_ptes_put(vmm, &page[refd], addr, size); 1574 } 1575 } while (next && next->part); 1576 } 1577 1578 /* Merge any mapped regions that were split from the initial 1579 * address-space allocation back into the allocated VMA, and 1580 * release memory/compression resources. 1581 */ 1582 next = vma; 1583 do { 1584 if (next->mapped) 1585 nvkm_vmm_unmap_region(vmm, next); 1586 } while ((next = node(vma, next)) && next->part); 1587 1588 if (vma->sparse && !vma->mapref) { 1589 /* Sparse region that was allocated with a fixed page size, 1590 * meaning all relevant PTEs were referenced once when the 1591 * region was allocated, and remained that way, regardless 1592 * of whether memory was mapped into it afterwards. 1593 * 1594 * The process of unmapping, unsparsing, and dereferencing 1595 * PTEs can be done in a single page tree walk. 1596 */ 1597 nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size); 1598 } else 1599 if (vma->sparse) { 1600 /* Sparse region that wasn't allocated with a fixed page size, 1601 * PTE references were taken both at allocation time (to make 1602 * the GPU see the region as sparse), and when mapping memory 1603 * into the region. 1604 * 1605 * The latter was handled above, and the remaining references 1606 * are dealt with here. 1607 */ 1608 nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false); 1609 } 1610 1611 /* Remove VMA from the list of allocated nodes. */ 1612 nvkm_vmm_node_remove(vmm, vma); 1613 1614 /* Merge VMA back into the free list. */ 1615 vma->page = NVKM_VMA_PAGE_NONE; 1616 vma->refd = NVKM_VMA_PAGE_NONE; 1617 vma->used = false; 1618 vma->user = false; 1619 nvkm_vmm_put_region(vmm, vma); 1620 } 1621 1622 void 1623 nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma) 1624 { 1625 struct nvkm_vma *vma = *pvma; 1626 if (vma) { 1627 mutex_lock(&vmm->mutex); 1628 nvkm_vmm_put_locked(vmm, vma); 1629 mutex_unlock(&vmm->mutex); 1630 *pvma = NULL; 1631 } 1632 } 1633 1634 int 1635 nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse, 1636 u8 shift, u8 align, u64 size, struct nvkm_vma **pvma) 1637 { 1638 const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE]; 1639 struct rb_node *node = NULL, *temp; 1640 struct nvkm_vma *vma = NULL, *tmp; 1641 u64 addr, tail; 1642 int ret; 1643 1644 VMM_TRACE(vmm, "getref %d mapref %d sparse %d " 1645 "shift: %d align: %d size: %016llx", 1646 getref, mapref, sparse, shift, align, size); 1647 1648 /* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */ 1649 if (unlikely(!size || (!getref && !mapref && sparse))) { 1650 VMM_DEBUG(vmm, "args %016llx %d %d %d", 1651 size, getref, mapref, sparse); 1652 return -EINVAL; 1653 } 1654 1655 /* Tesla-class GPUs can only select page size per-PDE, which means 1656 * we're required to know the mapping granularity up-front to find 1657 * a suitable region of address-space. 1658 * 1659 * The same goes if we're requesting up-front allocation of PTES. 1660 */ 1661 if (unlikely((getref || vmm->func->page_block) && !shift)) { 1662 VMM_DEBUG(vmm, "page size required: %d %016llx", 1663 getref, vmm->func->page_block); 1664 return -EINVAL; 1665 } 1666 1667 /* If a specific page size was requested, determine its index and 1668 * make sure the requested size is a multiple of the page size. 1669 */ 1670 if (shift) { 1671 for (page = vmm->func->page; page->shift; page++) { 1672 if (shift == page->shift) 1673 break; 1674 } 1675 1676 if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) { 1677 VMM_DEBUG(vmm, "page %d %016llx", shift, size); 1678 return -EINVAL; 1679 } 1680 align = max_t(u8, align, shift); 1681 } else { 1682 align = max_t(u8, align, 12); 1683 } 1684 1685 /* Locate smallest block that can possibly satisfy the allocation. */ 1686 temp = vmm->free.rb_node; 1687 while (temp) { 1688 struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree); 1689 if (this->size < size) { 1690 temp = temp->rb_right; 1691 } else { 1692 node = temp; 1693 temp = temp->rb_left; 1694 } 1695 } 1696 1697 if (unlikely(!node)) 1698 return -ENOSPC; 1699 1700 /* Take into account alignment restrictions, trying larger blocks 1701 * in turn until we find a suitable free block. 1702 */ 1703 do { 1704 struct nvkm_vma *this = rb_entry(node, typeof(*this), tree); 1705 struct nvkm_vma *prev = node(this, prev); 1706 struct nvkm_vma *next = node(this, next); 1707 const int p = page - vmm->func->page; 1708 1709 addr = this->addr; 1710 if (vmm->func->page_block && prev && prev->page != p) 1711 addr = ALIGN(addr, vmm->func->page_block); 1712 addr = ALIGN(addr, 1ULL << align); 1713 1714 tail = this->addr + this->size; 1715 if (vmm->func->page_block && next && next->page != p) 1716 tail = ALIGN_DOWN(tail, vmm->func->page_block); 1717 1718 if (addr <= tail && tail - addr >= size) { 1719 nvkm_vmm_free_remove(vmm, this); 1720 vma = this; 1721 break; 1722 } 1723 } while ((node = rb_next(node))); 1724 1725 if (unlikely(!vma)) 1726 return -ENOSPC; 1727 1728 /* If the VMA we found isn't already exactly the requested size, 1729 * it needs to be split, and the remaining free blocks returned. 1730 */ 1731 if (addr != vma->addr) { 1732 if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) { 1733 nvkm_vmm_put_region(vmm, vma); 1734 return -ENOMEM; 1735 } 1736 nvkm_vmm_free_insert(vmm, vma); 1737 vma = tmp; 1738 } 1739 1740 if (size != vma->size) { 1741 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) { 1742 nvkm_vmm_put_region(vmm, vma); 1743 return -ENOMEM; 1744 } 1745 nvkm_vmm_free_insert(vmm, tmp); 1746 } 1747 1748 /* Pre-allocate page tables and/or setup sparse mappings. */ 1749 if (sparse && getref) 1750 ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size); 1751 else if (sparse) 1752 ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true); 1753 else if (getref) 1754 ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size); 1755 else 1756 ret = 0; 1757 if (ret) { 1758 nvkm_vmm_put_region(vmm, vma); 1759 return ret; 1760 } 1761 1762 vma->mapref = mapref && !getref; 1763 vma->sparse = sparse; 1764 vma->page = page - vmm->func->page; 1765 vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE; 1766 vma->used = true; 1767 nvkm_vmm_node_insert(vmm, vma); 1768 *pvma = vma; 1769 return 0; 1770 } 1771 1772 int 1773 nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma) 1774 { 1775 int ret; 1776 mutex_lock(&vmm->mutex); 1777 ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma); 1778 mutex_unlock(&vmm->mutex); 1779 return ret; 1780 } 1781 1782 void 1783 nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) 1784 { 1785 if (inst && vmm && vmm->func->part) { 1786 mutex_lock(&vmm->mutex); 1787 vmm->func->part(vmm, inst); 1788 mutex_unlock(&vmm->mutex); 1789 } 1790 } 1791 1792 int 1793 nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) 1794 { 1795 int ret = 0; 1796 if (vmm->func->join) { 1797 mutex_lock(&vmm->mutex); 1798 ret = vmm->func->join(vmm, inst); 1799 mutex_unlock(&vmm->mutex); 1800 } 1801 return ret; 1802 } 1803 1804 static bool 1805 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) 1806 { 1807 const struct nvkm_vmm_desc *desc = it->desc; 1808 const int type = desc->type == SPT; 1809 nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm); 1810 return false; 1811 } 1812 1813 int 1814 nvkm_vmm_boot(struct nvkm_vmm *vmm) 1815 { 1816 const struct nvkm_vmm_page *page = vmm->func->page; 1817 const u64 limit = vmm->limit - vmm->start; 1818 int ret; 1819 1820 while (page[1].shift) 1821 page++; 1822 1823 ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit); 1824 if (ret) 1825 return ret; 1826 1827 nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false, 1828 nvkm_vmm_boot_ptes, NULL, NULL, NULL); 1829 vmm->bootstrapped = true; 1830 return 0; 1831 } 1832 1833 static void 1834 nvkm_vmm_del(struct kref *kref) 1835 { 1836 struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref); 1837 nvkm_vmm_dtor(vmm); 1838 kfree(vmm); 1839 } 1840 1841 void 1842 nvkm_vmm_unref(struct nvkm_vmm **pvmm) 1843 { 1844 struct nvkm_vmm *vmm = *pvmm; 1845 if (vmm) { 1846 kref_put(&vmm->kref, nvkm_vmm_del); 1847 *pvmm = NULL; 1848 } 1849 } 1850 1851 struct nvkm_vmm * 1852 nvkm_vmm_ref(struct nvkm_vmm *vmm) 1853 { 1854 if (vmm) 1855 kref_get(&vmm->kref); 1856 return vmm; 1857 } 1858 1859 int 1860 nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv, 1861 u32 argc, struct lock_class_key *key, const char *name, 1862 struct nvkm_vmm **pvmm) 1863 { 1864 struct nvkm_mmu *mmu = device->mmu; 1865 struct nvkm_vmm *vmm = NULL; 1866 int ret; 1867 ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc, 1868 key, name, &vmm); 1869 if (ret) 1870 nvkm_vmm_unref(&vmm); 1871 *pvmm = vmm; 1872 return ret; 1873 } 1874