1 /*
2  * Copyright 2017 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 #define NVKM_VMM_LEVELS_MAX 5
23 #include "vmm.h"
24 
25 #include <subdev/fb.h>
26 
27 static void
28 nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
29 {
30 	struct nvkm_vmm_pt *pgt = *ppgt;
31 	if (pgt) {
32 		kvfree(pgt->pde);
33 		kfree(pgt);
34 		*ppgt = NULL;
35 	}
36 }
37 
38 
39 static struct nvkm_vmm_pt *
40 nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
41 		const struct nvkm_vmm_page *page)
42 {
43 	const u32 pten = 1 << desc->bits;
44 	struct nvkm_vmm_pt *pgt;
45 	u32 lpte = 0;
46 
47 	if (desc->type > PGT) {
48 		if (desc->type == SPT) {
49 			const struct nvkm_vmm_desc *pair = page[-1].desc;
50 			lpte = pten >> (desc->bits - pair->bits);
51 		} else {
52 			lpte = pten;
53 		}
54 	}
55 
56 	if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
57 		return NULL;
58 	pgt->page = page ? page->shift : 0;
59 	pgt->sparse = sparse;
60 
61 	if (desc->type == PGD) {
62 		pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
63 		if (!pgt->pde) {
64 			kfree(pgt);
65 			return NULL;
66 		}
67 	}
68 
69 	return pgt;
70 }
71 
72 struct nvkm_vmm_iter {
73 	const struct nvkm_vmm_page *page;
74 	const struct nvkm_vmm_desc *desc;
75 	struct nvkm_vmm *vmm;
76 	u64 cnt;
77 	u16 max, lvl;
78 	u32 pte[NVKM_VMM_LEVELS_MAX];
79 	struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
80 	int flush;
81 };
82 
83 #ifdef CONFIG_NOUVEAU_DEBUG_MMU
84 static const char *
85 nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
86 {
87 	switch (desc->type) {
88 	case PGD: return "PGD";
89 	case PGT: return "PGT";
90 	case SPT: return "SPT";
91 	case LPT: return "LPT";
92 	default:
93 		return "UNKNOWN";
94 	}
95 }
96 
97 static void
98 nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
99 {
100 	int lvl;
101 	for (lvl = it->max; lvl >= 0; lvl--) {
102 		if (lvl >= it->lvl)
103 			buf += sprintf(buf,  "%05x:", it->pte[lvl]);
104 		else
105 			buf += sprintf(buf, "xxxxx:");
106 	}
107 }
108 
109 #define TRA(i,f,a...) do {                                                     \
110 	char _buf[NVKM_VMM_LEVELS_MAX * 7];                                    \
111 	struct nvkm_vmm_iter *_it = (i);                                       \
112 	nvkm_vmm_trace(_it, _buf);                                             \
113 	VMM_TRACE(_it->vmm, "%s "f, _buf, ##a);                                \
114 } while(0)
115 #else
116 #define TRA(i,f,a...)
117 #endif
118 
119 static inline void
120 nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
121 {
122 	it->flush = min(it->flush, it->max - it->lvl);
123 }
124 
125 static inline void
126 nvkm_vmm_flush(struct nvkm_vmm_iter *it)
127 {
128 	if (it->flush != NVKM_VMM_LEVELS_MAX) {
129 		if (it->vmm->func->flush) {
130 			TRA(it, "flush: %d", it->flush);
131 			it->vmm->func->flush(it->vmm, it->flush);
132 		}
133 		it->flush = NVKM_VMM_LEVELS_MAX;
134 	}
135 }
136 
137 static void
138 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
139 {
140 	const struct nvkm_vmm_desc *desc = it->desc;
141 	const int type = desc[it->lvl].type == SPT;
142 	struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
143 	struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
144 	struct nvkm_mmu_pt *pt = pgt->pt[type];
145 	struct nvkm_vmm *vmm = it->vmm;
146 	u32 pdei = it->pte[it->lvl + 1];
147 
148 	/* Recurse up the tree, unreferencing/destroying unneeded PDs. */
149 	it->lvl++;
150 	if (--pgd->refs[0]) {
151 		const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
152 		/* PD has other valid PDEs, so we need a proper update. */
153 		TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
154 		pgt->pt[type] = NULL;
155 		if (!pgt->refs[!type]) {
156 			/* PDE no longer required. */
157 			if (pgd->pt[0]) {
158 				if (pgt->sparse) {
159 					func->sparse(vmm, pgd->pt[0], pdei, 1);
160 					pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
161 				} else {
162 					func->unmap(vmm, pgd->pt[0], pdei, 1);
163 					pgd->pde[pdei] = NULL;
164 				}
165 			} else {
166 				/* Special handling for Tesla-class GPUs,
167 				 * where there's no central PD, but each
168 				 * instance has its own embedded PD.
169 				 */
170 				func->pde(vmm, pgd, pdei);
171 				pgd->pde[pdei] = NULL;
172 			}
173 		} else {
174 			/* PDE was pointing at dual-PTs and we're removing
175 			 * one of them, leaving the other in place.
176 			 */
177 			func->pde(vmm, pgd, pdei);
178 		}
179 
180 		/* GPU may have cached the PTs, flush before freeing. */
181 		nvkm_vmm_flush_mark(it);
182 		nvkm_vmm_flush(it);
183 	} else {
184 		/* PD has no valid PDEs left, so we can just destroy it. */
185 		nvkm_vmm_unref_pdes(it);
186 	}
187 
188 	/* Destroy PD/PT. */
189 	TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
190 	nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
191 	if (!pgt->refs[!type])
192 		nvkm_vmm_pt_del(&pgt);
193 	it->lvl--;
194 }
195 
196 static void
197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
198 		     const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
199 {
200 	const struct nvkm_vmm_desc *pair = it->page[-1].desc;
201 	const u32 sptb = desc->bits - pair->bits;
202 	const u32 sptn = 1 << sptb;
203 	struct nvkm_vmm *vmm = it->vmm;
204 	u32 spti = ptei & (sptn - 1), lpti, pteb;
205 
206 	/* Determine how many SPTEs are being touched under each LPTE,
207 	 * and drop reference counts.
208 	 */
209 	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
210 		const u32 pten = min(sptn - spti, ptes);
211 		pgt->pte[lpti] -= pten;
212 		ptes -= pten;
213 	}
214 
215 	/* We're done here if there's no corresponding LPT. */
216 	if (!pgt->refs[0])
217 		return;
218 
219 	for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
220 		/* Skip over any LPTEs that still have valid SPTEs. */
221 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
222 			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
223 				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
224 					break;
225 			}
226 			continue;
227 		}
228 
229 		/* As there's no more non-UNMAPPED SPTEs left in the range
230 		 * covered by a number of LPTEs, the LPTEs once again take
231 		 * control over their address range.
232 		 *
233 		 * Determine how many LPTEs need to transition state.
234 		 */
235 		pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
236 		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
237 			if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
238 				break;
239 			pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
240 		}
241 
242 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
243 			TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
244 			pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
245 		} else
246 		if (pair->func->invalid) {
247 			/* If the MMU supports it, restore the LPTE to the
248 			 * INVALID state to tell the MMU there is no point
249 			 * trying to fetch the corresponding SPTEs.
250 			 */
251 			TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
252 			pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
253 		}
254 	}
255 }
256 
257 static bool
258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
259 {
260 	const struct nvkm_vmm_desc *desc = it->desc;
261 	const int type = desc->type == SPT;
262 	struct nvkm_vmm_pt *pgt = it->pt[0];
263 	bool dma;
264 
265 	if (pfn) {
266 		/* Need to clear PTE valid bits before we dma_unmap_page(). */
267 		dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
268 		if (dma) {
269 			/* GPU may have cached the PT, flush before unmap. */
270 			nvkm_vmm_flush_mark(it);
271 			nvkm_vmm_flush(it);
272 			desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
273 		}
274 	}
275 
276 	/* Drop PTE references. */
277 	pgt->refs[type] -= ptes;
278 
279 	/* Dual-PTs need special handling, unless PDE becoming invalid. */
280 	if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
281 		nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
282 
283 	/* PT no longer needed? Destroy it. */
284 	if (!pgt->refs[type]) {
285 		it->lvl++;
286 		TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
287 		it->lvl--;
288 		nvkm_vmm_unref_pdes(it);
289 		return false; /* PTE writes for unmap() not necessary. */
290 	}
291 
292 	return true;
293 }
294 
295 static void
296 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
297 		   const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
298 {
299 	const struct nvkm_vmm_desc *pair = it->page[-1].desc;
300 	const u32 sptb = desc->bits - pair->bits;
301 	const u32 sptn = 1 << sptb;
302 	struct nvkm_vmm *vmm = it->vmm;
303 	u32 spti = ptei & (sptn - 1), lpti, pteb;
304 
305 	/* Determine how many SPTEs are being touched under each LPTE,
306 	 * and increase reference counts.
307 	 */
308 	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
309 		const u32 pten = min(sptn - spti, ptes);
310 		pgt->pte[lpti] += pten;
311 		ptes -= pten;
312 	}
313 
314 	/* We're done here if there's no corresponding LPT. */
315 	if (!pgt->refs[0])
316 		return;
317 
318 	for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
319 		/* Skip over any LPTEs that already have valid SPTEs. */
320 		if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
321 			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
322 				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
323 					break;
324 			}
325 			continue;
326 		}
327 
328 		/* As there are now non-UNMAPPED SPTEs in the range covered
329 		 * by a number of LPTEs, we need to transfer control of the
330 		 * address range to the SPTEs.
331 		 *
332 		 * Determine how many LPTEs need to transition state.
333 		 */
334 		pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
335 		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
336 			if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
337 				break;
338 			pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
339 		}
340 
341 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
342 			const u32 spti = pteb * sptn;
343 			const u32 sptc = ptes * sptn;
344 			/* The entire LPTE is marked as sparse, we need
345 			 * to make sure that the SPTEs are too.
346 			 */
347 			TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
348 			desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
349 			/* Sparse LPTEs prevent SPTEs from being accessed. */
350 			TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
351 			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
352 		} else
353 		if (pair->func->invalid) {
354 			/* MMU supports blocking SPTEs by marking an LPTE
355 			 * as INVALID.  We need to reverse that here.
356 			 */
357 			TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
358 			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
359 		}
360 	}
361 }
362 
363 static bool
364 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
365 {
366 	const struct nvkm_vmm_desc *desc = it->desc;
367 	const int type = desc->type == SPT;
368 	struct nvkm_vmm_pt *pgt = it->pt[0];
369 
370 	/* Take PTE references. */
371 	pgt->refs[type] += ptes;
372 
373 	/* Dual-PTs need special handling. */
374 	if (desc->type == SPT)
375 		nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
376 
377 	return true;
378 }
379 
380 static void
381 nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
382 		     struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
383 {
384 	if (desc->type == PGD) {
385 		while (ptes--)
386 			pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
387 	} else
388 	if (desc->type == LPT) {
389 		memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
390 	}
391 }
392 
393 static bool
394 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
395 {
396 	struct nvkm_vmm_pt *pt = it->pt[0];
397 	if (it->desc->type == PGD)
398 		memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
399 	else
400 	if (it->desc->type == LPT)
401 		memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
402 	return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
403 }
404 
405 static bool
406 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
407 {
408 	nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
409 	return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
410 }
411 
412 static bool
413 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
414 {
415 	const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
416 	const int type = desc->type == SPT;
417 	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
418 	const bool zero = !pgt->sparse && !desc->func->invalid;
419 	struct nvkm_vmm *vmm = it->vmm;
420 	struct nvkm_mmu *mmu = vmm->mmu;
421 	struct nvkm_mmu_pt *pt;
422 	u32 pten = 1 << desc->bits;
423 	u32 pteb, ptei, ptes;
424 	u32 size = desc->size * pten;
425 
426 	pgd->refs[0]++;
427 
428 	pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
429 	if (!pgt->pt[type]) {
430 		it->lvl--;
431 		nvkm_vmm_unref_pdes(it);
432 		return false;
433 	}
434 
435 	if (zero)
436 		goto done;
437 
438 	pt = pgt->pt[type];
439 
440 	if (desc->type == LPT && pgt->refs[1]) {
441 		/* SPT already exists covering the same range as this LPT,
442 		 * which means we need to be careful that any LPTEs which
443 		 * overlap valid SPTEs are unmapped as opposed to invalid
444 		 * or sparse, which would prevent the MMU from looking at
445 		 * the SPTEs on some GPUs.
446 		 */
447 		for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
448 			bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
449 			for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
450 				bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
451 				if (spte != next)
452 					break;
453 			}
454 
455 			if (!spte) {
456 				if (pgt->sparse)
457 					desc->func->sparse(vmm, pt, pteb, ptes);
458 				else
459 					desc->func->invalid(vmm, pt, pteb, ptes);
460 				memset(&pgt->pte[pteb], 0x00, ptes);
461 			} else {
462 				desc->func->unmap(vmm, pt, pteb, ptes);
463 				while (ptes--)
464 					pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
465 			}
466 		}
467 	} else {
468 		if (pgt->sparse) {
469 			nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
470 			desc->func->sparse(vmm, pt, 0, pten);
471 		} else {
472 			desc->func->invalid(vmm, pt, 0, pten);
473 		}
474 	}
475 
476 done:
477 	TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
478 	it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
479 	nvkm_vmm_flush_mark(it);
480 	return true;
481 }
482 
483 static bool
484 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
485 {
486 	const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
487 	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
488 
489 	pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
490 	if (!pgt) {
491 		if (!pgd->refs[0])
492 			nvkm_vmm_unref_pdes(it);
493 		return false;
494 	}
495 
496 	pgd->pde[pdei] = pgt;
497 	return true;
498 }
499 
500 static inline u64
501 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
502 	      u64 addr, u64 size, const char *name, bool ref, bool pfn,
503 	      bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
504 	      nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
505 	      nvkm_vmm_pxe_func CLR_PTES)
506 {
507 	const struct nvkm_vmm_desc *desc = page->desc;
508 	struct nvkm_vmm_iter it;
509 	u64 bits = addr >> page->shift;
510 
511 	it.page = page;
512 	it.desc = desc;
513 	it.vmm = vmm;
514 	it.cnt = size >> page->shift;
515 	it.flush = NVKM_VMM_LEVELS_MAX;
516 
517 	/* Deconstruct address into PTE indices for each mapping level. */
518 	for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
519 		it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
520 		bits >>= desc[it.lvl].bits;
521 	}
522 	it.max = --it.lvl;
523 	it.pt[it.max] = vmm->pd;
524 
525 	it.lvl = 0;
526 	TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
527 	         addr, size, page->shift, it.cnt);
528 	it.lvl = it.max;
529 
530 	/* Depth-first traversal of page tables. */
531 	while (it.cnt) {
532 		struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
533 		const int type = desc->type == SPT;
534 		const u32 pten = 1 << desc->bits;
535 		const u32 ptei = it.pte[0];
536 		const u32 ptes = min_t(u64, it.cnt, pten - ptei);
537 
538 		/* Walk down the tree, finding page tables for each level. */
539 		for (; it.lvl; it.lvl--) {
540 			const u32 pdei = it.pte[it.lvl];
541 			struct nvkm_vmm_pt *pgd = pgt;
542 
543 			/* Software PT. */
544 			if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
545 				if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
546 					goto fail;
547 			}
548 			it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
549 
550 			/* Hardware PT.
551 			 *
552 			 * This is a separate step from above due to GF100 and
553 			 * newer having dual page tables at some levels, which
554 			 * are refcounted independently.
555 			 */
556 			if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
557 				if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
558 					goto fail;
559 			}
560 		}
561 
562 		/* Handle PTE updates. */
563 		if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
564 			struct nvkm_mmu_pt *pt = pgt->pt[type];
565 			if (MAP_PTES || CLR_PTES) {
566 				if (MAP_PTES)
567 					MAP_PTES(vmm, pt, ptei, ptes, map);
568 				else
569 					CLR_PTES(vmm, pt, ptei, ptes);
570 				nvkm_vmm_flush_mark(&it);
571 			}
572 		}
573 
574 		/* Walk back up the tree to the next position. */
575 		it.pte[it.lvl] += ptes;
576 		it.cnt -= ptes;
577 		if (it.cnt) {
578 			while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
579 				it.pte[it.lvl++] = 0;
580 				it.pte[it.lvl]++;
581 			}
582 		}
583 	}
584 
585 	nvkm_vmm_flush(&it);
586 	return ~0ULL;
587 
588 fail:
589 	/* Reconstruct the failure address so the caller is able to
590 	 * reverse any partially completed operations.
591 	 */
592 	addr = it.pte[it.max--];
593 	do {
594 		addr  = addr << desc[it.max].bits;
595 		addr |= it.pte[it.max];
596 	} while (it.max--);
597 
598 	return addr << page->shift;
599 }
600 
601 static void
602 nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
603 			 u64 addr, u64 size)
604 {
605 	nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
606 		      nvkm_vmm_sparse_unref_ptes, NULL, NULL,
607 		      page->desc->func->invalid ?
608 		      page->desc->func->invalid : page->desc->func->unmap);
609 }
610 
611 static int
612 nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
613 			 u64 addr, u64 size)
614 {
615 	if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
616 		u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
617 					 true, false, nvkm_vmm_sparse_ref_ptes,
618 					 NULL, NULL, page->desc->func->sparse);
619 		if (fail != ~0ULL) {
620 			if ((size = fail - addr))
621 				nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
622 			return -ENOMEM;
623 		}
624 		return 0;
625 	}
626 	return -EINVAL;
627 }
628 
629 static int
630 nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
631 {
632 	const struct nvkm_vmm_page *page = vmm->func->page;
633 	int m = 0, i;
634 	u64 start = addr;
635 	u64 block;
636 
637 	while (size) {
638 		/* Limit maximum page size based on remaining size. */
639 		while (size < (1ULL << page[m].shift))
640 			m++;
641 		i = m;
642 
643 		/* Find largest page size suitable for alignment. */
644 		while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
645 			i++;
646 
647 		/* Determine number of PTEs at this page size. */
648 		if (i != m) {
649 			/* Limited to alignment boundary of next page size. */
650 			u64 next = 1ULL << page[i - 1].shift;
651 			u64 part = ALIGN(addr, next) - addr;
652 			if (size - part >= next)
653 				block = (part >> page[i].shift) << page[i].shift;
654 			else
655 				block = (size >> page[i].shift) << page[i].shift;
656 		} else {
657 			block = (size >> page[i].shift) << page[i].shift;
658 		}
659 
660 		/* Perform operation. */
661 		if (ref) {
662 			int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
663 			if (ret) {
664 				if ((size = addr - start))
665 					nvkm_vmm_ptes_sparse(vmm, start, size, false);
666 				return ret;
667 			}
668 		} else {
669 			nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
670 		}
671 
672 		size -= block;
673 		addr += block;
674 	}
675 
676 	return 0;
677 }
678 
679 static void
680 nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
681 			u64 addr, u64 size, bool sparse, bool pfn)
682 {
683 	const struct nvkm_vmm_desc_func *func = page->desc->func;
684 	nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
685 		      false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
686 		      sparse ? func->sparse : func->invalid ? func->invalid :
687 							      func->unmap);
688 }
689 
690 static int
691 nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
692 		      u64 addr, u64 size, struct nvkm_vmm_map *map,
693 		      nvkm_vmm_pte_func func)
694 {
695 	u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
696 				 false, nvkm_vmm_ref_ptes, func, map, NULL);
697 	if (fail != ~0ULL) {
698 		if ((size = fail - addr))
699 			nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
700 		return -ENOMEM;
701 	}
702 	return 0;
703 }
704 
705 static void
706 nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
707 		    u64 addr, u64 size, bool sparse, bool pfn)
708 {
709 	const struct nvkm_vmm_desc_func *func = page->desc->func;
710 	nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
711 		      NULL, NULL, NULL,
712 		      sparse ? func->sparse : func->invalid ? func->invalid :
713 							      func->unmap);
714 }
715 
716 static void
717 nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
718 		  u64 addr, u64 size, struct nvkm_vmm_map *map,
719 		  nvkm_vmm_pte_func func)
720 {
721 	nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
722 		      NULL, func, map, NULL);
723 }
724 
725 static void
726 nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
727 		  u64 addr, u64 size)
728 {
729 	nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
730 		      nvkm_vmm_unref_ptes, NULL, NULL, NULL);
731 }
732 
733 static int
734 nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
735 		  u64 addr, u64 size)
736 {
737 	u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
738 				 nvkm_vmm_ref_ptes, NULL, NULL, NULL);
739 	if (fail != ~0ULL) {
740 		if (fail != addr)
741 			nvkm_vmm_ptes_put(vmm, page, addr, fail - addr);
742 		return -ENOMEM;
743 	}
744 	return 0;
745 }
746 
747 static inline struct nvkm_vma *
748 nvkm_vma_new(u64 addr, u64 size)
749 {
750 	struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
751 	if (vma) {
752 		vma->addr = addr;
753 		vma->size = size;
754 		vma->page = NVKM_VMA_PAGE_NONE;
755 		vma->refd = NVKM_VMA_PAGE_NONE;
756 	}
757 	return vma;
758 }
759 
760 struct nvkm_vma *
761 nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
762 {
763 	struct nvkm_vma *new;
764 
765 	BUG_ON(vma->size == tail);
766 
767 	if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
768 		return NULL;
769 	vma->size -= tail;
770 
771 	new->mapref = vma->mapref;
772 	new->sparse = vma->sparse;
773 	new->page = vma->page;
774 	new->refd = vma->refd;
775 	new->used = vma->used;
776 	new->part = vma->part;
777 	new->busy = vma->busy;
778 	new->mapped = vma->mapped;
779 	list_add(&new->head, &vma->head);
780 	return new;
781 }
782 
783 static inline void
784 nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
785 {
786 	rb_erase(&vma->tree, &vmm->free);
787 }
788 
789 static inline void
790 nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
791 {
792 	nvkm_vmm_free_remove(vmm, vma);
793 	list_del(&vma->head);
794 	kfree(vma);
795 }
796 
797 static void
798 nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
799 {
800 	struct rb_node **ptr = &vmm->free.rb_node;
801 	struct rb_node *parent = NULL;
802 
803 	while (*ptr) {
804 		struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
805 		parent = *ptr;
806 		if (vma->size < this->size)
807 			ptr = &parent->rb_left;
808 		else
809 		if (vma->size > this->size)
810 			ptr = &parent->rb_right;
811 		else
812 		if (vma->addr < this->addr)
813 			ptr = &parent->rb_left;
814 		else
815 		if (vma->addr > this->addr)
816 			ptr = &parent->rb_right;
817 		else
818 			BUG();
819 	}
820 
821 	rb_link_node(&vma->tree, parent, ptr);
822 	rb_insert_color(&vma->tree, &vmm->free);
823 }
824 
825 static inline void
826 nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
827 {
828 	rb_erase(&vma->tree, &vmm->root);
829 }
830 
831 static inline void
832 nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
833 {
834 	nvkm_vmm_node_remove(vmm, vma);
835 	list_del(&vma->head);
836 	kfree(vma);
837 }
838 
839 static void
840 nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
841 {
842 	struct rb_node **ptr = &vmm->root.rb_node;
843 	struct rb_node *parent = NULL;
844 
845 	while (*ptr) {
846 		struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
847 		parent = *ptr;
848 		if (vma->addr < this->addr)
849 			ptr = &parent->rb_left;
850 		else
851 		if (vma->addr > this->addr)
852 			ptr = &parent->rb_right;
853 		else
854 			BUG();
855 	}
856 
857 	rb_link_node(&vma->tree, parent, ptr);
858 	rb_insert_color(&vma->tree, &vmm->root);
859 }
860 
861 struct nvkm_vma *
862 nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
863 {
864 	struct rb_node *node = vmm->root.rb_node;
865 	while (node) {
866 		struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
867 		if (addr < vma->addr)
868 			node = node->rb_left;
869 		else
870 		if (addr >= vma->addr + vma->size)
871 			node = node->rb_right;
872 		else
873 			return vma;
874 	}
875 	return NULL;
876 }
877 
878 #define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL :             \
879 	list_entry((root)->head.dir, struct nvkm_vma, head))
880 
881 static struct nvkm_vma *
882 nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev,
883 		    struct nvkm_vma *vma, struct nvkm_vma *next, u64 size)
884 {
885 	if (next) {
886 		if (vma->size == size) {
887 			vma->size += next->size;
888 			nvkm_vmm_node_delete(vmm, next);
889 			if (prev) {
890 				prev->size += vma->size;
891 				nvkm_vmm_node_delete(vmm, vma);
892 				return prev;
893 			}
894 			return vma;
895 		}
896 		BUG_ON(prev);
897 
898 		nvkm_vmm_node_remove(vmm, next);
899 		vma->size -= size;
900 		next->addr -= size;
901 		next->size += size;
902 		nvkm_vmm_node_insert(vmm, next);
903 		return next;
904 	}
905 
906 	if (prev) {
907 		if (vma->size != size) {
908 			nvkm_vmm_node_remove(vmm, vma);
909 			prev->size += size;
910 			vma->addr += size;
911 			vma->size -= size;
912 			nvkm_vmm_node_insert(vmm, vma);
913 		} else {
914 			prev->size += vma->size;
915 			nvkm_vmm_node_delete(vmm, vma);
916 		}
917 		return prev;
918 	}
919 
920 	return vma;
921 }
922 
923 struct nvkm_vma *
924 nvkm_vmm_node_split(struct nvkm_vmm *vmm,
925 		    struct nvkm_vma *vma, u64 addr, u64 size)
926 {
927 	struct nvkm_vma *prev = NULL;
928 
929 	if (vma->addr != addr) {
930 		prev = vma;
931 		if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr)))
932 			return NULL;
933 		vma->part = true;
934 		nvkm_vmm_node_insert(vmm, vma);
935 	}
936 
937 	if (vma->size != size) {
938 		struct nvkm_vma *tmp;
939 		if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
940 			nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size);
941 			return NULL;
942 		}
943 		tmp->part = true;
944 		nvkm_vmm_node_insert(vmm, tmp);
945 	}
946 
947 	return vma;
948 }
949 
950 static void
951 nvkm_vma_dump(struct nvkm_vma *vma)
952 {
953 	printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c %p\n",
954 	       vma->addr, (u64)vma->size,
955 	       vma->used ? '-' : 'F',
956 	       vma->mapref ? 'R' : '-',
957 	       vma->sparse ? 'S' : '-',
958 	       vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
959 	       vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
960 	       vma->part ? 'P' : '-',
961 	       vma->busy ? 'B' : '-',
962 	       vma->mapped ? 'M' : '-',
963 	       vma->memory);
964 }
965 
966 static void
967 nvkm_vmm_dump(struct nvkm_vmm *vmm)
968 {
969 	struct nvkm_vma *vma;
970 	list_for_each_entry(vma, &vmm->list, head) {
971 		nvkm_vma_dump(vma);
972 	}
973 }
974 
975 static void
976 nvkm_vmm_dtor(struct nvkm_vmm *vmm)
977 {
978 	struct nvkm_vma *vma;
979 	struct rb_node *node;
980 
981 	if (0)
982 		nvkm_vmm_dump(vmm);
983 
984 	while ((node = rb_first(&vmm->root))) {
985 		struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
986 		nvkm_vmm_put(vmm, &vma);
987 	}
988 
989 	if (vmm->bootstrapped) {
990 		const struct nvkm_vmm_page *page = vmm->func->page;
991 		const u64 limit = vmm->limit - vmm->start;
992 
993 		while (page[1].shift)
994 			page++;
995 
996 		nvkm_mmu_ptc_dump(vmm->mmu);
997 		nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
998 	}
999 
1000 	vma = list_first_entry(&vmm->list, typeof(*vma), head);
1001 	list_del(&vma->head);
1002 	kfree(vma);
1003 	WARN_ON(!list_empty(&vmm->list));
1004 
1005 	if (vmm->nullp) {
1006 		dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
1007 				  vmm->nullp, vmm->null);
1008 	}
1009 
1010 	if (vmm->pd) {
1011 		nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
1012 		nvkm_vmm_pt_del(&vmm->pd);
1013 	}
1014 }
1015 
1016 static int
1017 nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size)
1018 {
1019 	struct nvkm_vma *vma;
1020 	if (!(vma = nvkm_vma_new(addr, size)))
1021 		return -ENOMEM;
1022 	vma->mapref = true;
1023 	vma->sparse = false;
1024 	vma->used = true;
1025 	nvkm_vmm_node_insert(vmm, vma);
1026 	list_add_tail(&vma->head, &vmm->list);
1027 	return 0;
1028 }
1029 
1030 static int
1031 nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1032 	      u32 pd_header, bool managed, u64 addr, u64 size,
1033 	      struct lock_class_key *key, const char *name,
1034 	      struct nvkm_vmm *vmm)
1035 {
1036 	static struct lock_class_key _key;
1037 	const struct nvkm_vmm_page *page = func->page;
1038 	const struct nvkm_vmm_desc *desc;
1039 	struct nvkm_vma *vma;
1040 	int levels, bits = 0, ret;
1041 
1042 	vmm->func = func;
1043 	vmm->mmu = mmu;
1044 	vmm->name = name;
1045 	vmm->debug = mmu->subdev.debug;
1046 	kref_init(&vmm->kref);
1047 
1048 	__mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key);
1049 
1050 	/* Locate the smallest page size supported by the backend, it will
1051 	 * have the deepest nesting of page tables.
1052 	 */
1053 	while (page[1].shift)
1054 		page++;
1055 
1056 	/* Locate the structure that describes the layout of the top-level
1057 	 * page table, and determine the number of valid bits in a virtual
1058 	 * address.
1059 	 */
1060 	for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
1061 		bits += desc->bits;
1062 	bits += page->shift;
1063 	desc--;
1064 
1065 	if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
1066 		return -EINVAL;
1067 
1068 	/* Allocate top-level page table. */
1069 	vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
1070 	if (!vmm->pd)
1071 		return -ENOMEM;
1072 	vmm->pd->refs[0] = 1;
1073 	INIT_LIST_HEAD(&vmm->join);
1074 
1075 	/* ... and the GPU storage for it, except on Tesla-class GPUs that
1076 	 * have the PD embedded in the instance structure.
1077 	 */
1078 	if (desc->size) {
1079 		const u32 size = pd_header + desc->size * (1 << desc->bits);
1080 		vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
1081 		if (!vmm->pd->pt[0])
1082 			return -ENOMEM;
1083 	}
1084 
1085 	/* Initialise address-space MM. */
1086 	INIT_LIST_HEAD(&vmm->list);
1087 	vmm->free = RB_ROOT;
1088 	vmm->root = RB_ROOT;
1089 
1090 	if (managed) {
1091 		/* Address-space will be managed by the client for the most
1092 		 * part, except for a specified area where NVKM allocations
1093 		 * are allowed to be placed.
1094 		 */
1095 		vmm->start = 0;
1096 		vmm->limit = 1ULL << bits;
1097 		if (addr + size < addr || addr + size > vmm->limit)
1098 			return -EINVAL;
1099 
1100 		/* Client-managed area before the NVKM-managed area. */
1101 		if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr)))
1102 			return ret;
1103 
1104 		/* NVKM-managed area. */
1105 		if (size) {
1106 			if (!(vma = nvkm_vma_new(addr, size)))
1107 				return -ENOMEM;
1108 			nvkm_vmm_free_insert(vmm, vma);
1109 			list_add_tail(&vma->head, &vmm->list);
1110 		}
1111 
1112 		/* Client-managed area after the NVKM-managed area. */
1113 		addr = addr + size;
1114 		size = vmm->limit - addr;
1115 		if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size)))
1116 			return ret;
1117 	} else {
1118 		/* Address-space fully managed by NVKM, requiring calls to
1119 		 * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space.
1120 		 */
1121 		vmm->start = addr;
1122 		vmm->limit = size ? (addr + size) : (1ULL << bits);
1123 		if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
1124 			return -EINVAL;
1125 
1126 		if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
1127 			return -ENOMEM;
1128 
1129 		nvkm_vmm_free_insert(vmm, vma);
1130 		list_add(&vma->head, &vmm->list);
1131 	}
1132 
1133 	return 0;
1134 }
1135 
1136 int
1137 nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1138 	      u32 hdr, bool managed, u64 addr, u64 size,
1139 	      struct lock_class_key *key, const char *name,
1140 	      struct nvkm_vmm **pvmm)
1141 {
1142 	if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
1143 		return -ENOMEM;
1144 	return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
1145 }
1146 
1147 static struct nvkm_vma *
1148 nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1149 			 u64 addr, u64 size, u8 page, bool map)
1150 {
1151 	struct nvkm_vma *prev = NULL;
1152 	struct nvkm_vma *next = NULL;
1153 
1154 	if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
1155 		if (prev->memory || prev->mapped != map)
1156 			prev = NULL;
1157 	}
1158 
1159 	if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
1160 		if (!next->part ||
1161 		    next->memory || next->mapped != map)
1162 			next = NULL;
1163 	}
1164 
1165 	if (prev || next)
1166 		return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
1167 	return nvkm_vmm_node_split(vmm, vma, addr, size);
1168 }
1169 
1170 int
1171 nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
1172 {
1173 	struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
1174 	struct nvkm_vma *next;
1175 	u64 limit = addr + size;
1176 	u64 start = addr;
1177 
1178 	if (!vma)
1179 		return -EINVAL;
1180 
1181 	do {
1182 		if (!vma->mapped || vma->memory)
1183 			continue;
1184 
1185 		size = min(limit - start, vma->size - (start - vma->addr));
1186 
1187 		nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
1188 					start, size, false, true);
1189 
1190 		next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
1191 		if (!WARN_ON(!next)) {
1192 			vma = next;
1193 			vma->refd = NVKM_VMA_PAGE_NONE;
1194 			vma->mapped = false;
1195 		}
1196 	} while ((vma = node(vma, next)) && (start = vma->addr) < limit);
1197 
1198 	return 0;
1199 }
1200 
1201 /*TODO:
1202  * - Avoid PT readback (for dma_unmap etc), this might end up being dealt
1203  *   with inside HMM, which would be a lot nicer for us to deal with.
1204  * - Support for systems without a 4KiB page size.
1205  */
1206 int
1207 nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
1208 {
1209 	const struct nvkm_vmm_page *page = vmm->func->page;
1210 	struct nvkm_vma *vma, *tmp;
1211 	u64 limit = addr + size;
1212 	u64 start = addr;
1213 	int pm = size >> shift;
1214 	int pi = 0;
1215 
1216 	/* Only support mapping where the page size of the incoming page
1217 	 * array matches a page size available for direct mapping.
1218 	 */
1219 	while (page->shift && (page->shift != shift ||
1220 	       page->desc->func->pfn == NULL))
1221 		page++;
1222 
1223 	if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
1224 			    !IS_ALIGNED(size, 1ULL << shift) ||
1225 	    addr + size < addr || addr + size > vmm->limit) {
1226 		VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
1227 			  shift, page->shift, addr, size);
1228 		return -EINVAL;
1229 	}
1230 
1231 	if (!(vma = nvkm_vmm_node_search(vmm, addr)))
1232 		return -ENOENT;
1233 
1234 	do {
1235 		bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
1236 		bool mapped = vma->mapped;
1237 		u64 size = limit - start;
1238 		u64 addr = start;
1239 		int pn, ret = 0;
1240 
1241 		/* Narrow the operation window to cover a single action (page
1242 		 * should be mapped or not) within a single VMA.
1243 		 */
1244 		for (pn = 0; pi + pn < pm; pn++) {
1245 			if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
1246 				break;
1247 		}
1248 		size = min_t(u64, size, pn << page->shift);
1249 		size = min_t(u64, size, vma->size + vma->addr - addr);
1250 
1251 		/* Reject any operation to unmanaged regions, and areas that
1252 		 * have nvkm_memory objects mapped in them already.
1253 		 */
1254 		if (!vma->mapref || vma->memory) {
1255 			ret = -EINVAL;
1256 			goto next;
1257 		}
1258 
1259 		/* In order to both properly refcount GPU page tables, and
1260 		 * prevent "normal" mappings and these direct mappings from
1261 		 * interfering with each other, we need to track contiguous
1262 		 * ranges that have been mapped with this interface.
1263 		 *
1264 		 * Here we attempt to either split an existing VMA so we're
1265 		 * able to flag the region as either unmapped/mapped, or to
1266 		 * merge with adjacent VMAs that are already compatible.
1267 		 *
1268 		 * If the region is already compatible, nothing is required.
1269 		 */
1270 		if (map != mapped) {
1271 			tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
1272 						       page -
1273 						       vmm->func->page, map);
1274 			if (WARN_ON(!tmp)) {
1275 				ret = -ENOMEM;
1276 				goto next;
1277 			}
1278 
1279 			if ((tmp->mapped = map))
1280 				tmp->refd = page - vmm->func->page;
1281 			else
1282 				tmp->refd = NVKM_VMA_PAGE_NONE;
1283 			vma = tmp;
1284 		}
1285 
1286 		/* Update HW page tables. */
1287 		if (map) {
1288 			struct nvkm_vmm_map args;
1289 			args.page = page;
1290 			args.pfn = &pfn[pi];
1291 
1292 			if (!mapped) {
1293 				ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
1294 							    size, &args, page->
1295 							    desc->func->pfn);
1296 			} else {
1297 				nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
1298 						  page->desc->func->pfn);
1299 			}
1300 		} else {
1301 			if (mapped) {
1302 				nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
1303 							false, true);
1304 			}
1305 		}
1306 
1307 next:
1308 		/* Iterate to next operation. */
1309 		if (vma->addr + vma->size == addr + size)
1310 			vma = node(vma, next);
1311 		start += size;
1312 
1313 		if (ret) {
1314 			/* Failure is signalled by clearing the valid bit on
1315 			 * any PFN that couldn't be modified as requested.
1316 			 */
1317 			while (size) {
1318 				pfn[pi++] = NVKM_VMM_PFN_NONE;
1319 				size -= 1 << page->shift;
1320 			}
1321 		} else {
1322 			pi += size >> page->shift;
1323 		}
1324 	} while (vma && start < limit);
1325 
1326 	return 0;
1327 }
1328 
1329 void
1330 nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1331 {
1332 	struct nvkm_vma *prev = NULL;
1333 	struct nvkm_vma *next;
1334 
1335 	nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1336 	nvkm_memory_unref(&vma->memory);
1337 	vma->mapped = false;
1338 
1339 	if (vma->part && (prev = node(vma, prev)) && prev->mapped)
1340 		prev = NULL;
1341 	if ((next = node(vma, next)) && (!next->part || next->mapped))
1342 		next = NULL;
1343 	nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
1344 }
1345 
1346 void
1347 nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
1348 {
1349 	const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
1350 
1351 	if (vma->mapref) {
1352 		nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1353 		vma->refd = NVKM_VMA_PAGE_NONE;
1354 	} else {
1355 		nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1356 	}
1357 
1358 	nvkm_vmm_unmap_region(vmm, vma);
1359 }
1360 
1361 void
1362 nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1363 {
1364 	if (vma->memory) {
1365 		mutex_lock(&vmm->mutex);
1366 		nvkm_vmm_unmap_locked(vmm, vma, false);
1367 		mutex_unlock(&vmm->mutex);
1368 	}
1369 }
1370 
1371 static int
1372 nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1373 		   void *argv, u32 argc, struct nvkm_vmm_map *map)
1374 {
1375 	switch (nvkm_memory_target(map->memory)) {
1376 	case NVKM_MEM_TARGET_VRAM:
1377 		if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
1378 			VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
1379 			return -EINVAL;
1380 		}
1381 		break;
1382 	case NVKM_MEM_TARGET_HOST:
1383 	case NVKM_MEM_TARGET_NCOH:
1384 		if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
1385 			VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
1386 			return -EINVAL;
1387 		}
1388 		break;
1389 	default:
1390 		WARN_ON(1);
1391 		return -ENOSYS;
1392 	}
1393 
1394 	if (!IS_ALIGNED(     vma->addr, 1ULL << map->page->shift) ||
1395 	    !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
1396 	    !IS_ALIGNED(   map->offset, 1ULL << map->page->shift) ||
1397 	    nvkm_memory_page(map->memory) < map->page->shift) {
1398 		VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
1399 		    vma->addr, (u64)vma->size, map->offset, map->page->shift,
1400 		    nvkm_memory_page(map->memory));
1401 		return -EINVAL;
1402 	}
1403 
1404 	return vmm->func->valid(vmm, argv, argc, map);
1405 }
1406 
1407 static int
1408 nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1409 		    void *argv, u32 argc, struct nvkm_vmm_map *map)
1410 {
1411 	for (map->page = vmm->func->page; map->page->shift; map->page++) {
1412 		VMM_DEBUG(vmm, "trying %d", map->page->shift);
1413 		if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
1414 			return 0;
1415 	}
1416 	return -EINVAL;
1417 }
1418 
1419 static int
1420 nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1421 		    void *argv, u32 argc, struct nvkm_vmm_map *map)
1422 {
1423 	nvkm_vmm_pte_func func;
1424 	int ret;
1425 
1426 	/* Make sure we won't overrun the end of the memory object. */
1427 	if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
1428 		VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
1429 			  nvkm_memory_size(map->memory),
1430 			  map->offset, (u64)vma->size);
1431 		return -EINVAL;
1432 	}
1433 
1434 	/* Check remaining arguments for validity. */
1435 	if (vma->page == NVKM_VMA_PAGE_NONE &&
1436 	    vma->refd == NVKM_VMA_PAGE_NONE) {
1437 		/* Find the largest page size we can perform the mapping at. */
1438 		const u32 debug = vmm->debug;
1439 		vmm->debug = 0;
1440 		ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1441 		vmm->debug = debug;
1442 		if (ret) {
1443 			VMM_DEBUG(vmm, "invalid at any page size");
1444 			nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1445 			return -EINVAL;
1446 		}
1447 	} else {
1448 		/* Page size of the VMA is already pre-determined. */
1449 		if (vma->refd != NVKM_VMA_PAGE_NONE)
1450 			map->page = &vmm->func->page[vma->refd];
1451 		else
1452 			map->page = &vmm->func->page[vma->page];
1453 
1454 		ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
1455 		if (ret) {
1456 			VMM_DEBUG(vmm, "invalid %d\n", ret);
1457 			return ret;
1458 		}
1459 	}
1460 
1461 	/* Deal with the 'offset' argument, and fetch the backend function. */
1462 	map->off = map->offset;
1463 	if (map->mem) {
1464 		for (; map->off; map->mem = map->mem->next) {
1465 			u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
1466 			if (size > map->off)
1467 				break;
1468 			map->off -= size;
1469 		}
1470 		func = map->page->desc->func->mem;
1471 	} else
1472 	if (map->sgl) {
1473 		for (; map->off; map->sgl = sg_next(map->sgl)) {
1474 			u64 size = sg_dma_len(map->sgl);
1475 			if (size > map->off)
1476 				break;
1477 			map->off -= size;
1478 		}
1479 		func = map->page->desc->func->sgl;
1480 	} else {
1481 		map->dma += map->offset >> PAGE_SHIFT;
1482 		map->off  = map->offset & PAGE_MASK;
1483 		func = map->page->desc->func->dma;
1484 	}
1485 
1486 	/* Perform the map. */
1487 	if (vma->refd == NVKM_VMA_PAGE_NONE) {
1488 		ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
1489 		if (ret)
1490 			return ret;
1491 
1492 		vma->refd = map->page - vmm->func->page;
1493 	} else {
1494 		nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
1495 	}
1496 
1497 	nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1498 	nvkm_memory_unref(&vma->memory);
1499 	vma->memory = nvkm_memory_ref(map->memory);
1500 	vma->mapped = true;
1501 	vma->tags = map->tags;
1502 	return 0;
1503 }
1504 
1505 int
1506 nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
1507 	     struct nvkm_vmm_map *map)
1508 {
1509 	int ret;
1510 	mutex_lock(&vmm->mutex);
1511 	ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1512 	vma->busy = false;
1513 	mutex_unlock(&vmm->mutex);
1514 	return ret;
1515 }
1516 
1517 static void
1518 nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1519 {
1520 	struct nvkm_vma *prev, *next;
1521 
1522 	if ((prev = node(vma, prev)) && !prev->used) {
1523 		vma->addr  = prev->addr;
1524 		vma->size += prev->size;
1525 		nvkm_vmm_free_delete(vmm, prev);
1526 	}
1527 
1528 	if ((next = node(vma, next)) && !next->used) {
1529 		vma->size += next->size;
1530 		nvkm_vmm_free_delete(vmm, next);
1531 	}
1532 
1533 	nvkm_vmm_free_insert(vmm, vma);
1534 }
1535 
1536 void
1537 nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1538 {
1539 	const struct nvkm_vmm_page *page = vmm->func->page;
1540 	struct nvkm_vma *next = vma;
1541 
1542 	BUG_ON(vma->part);
1543 
1544 	if (vma->mapref || !vma->sparse) {
1545 		do {
1546 			const bool mem = next->memory != NULL;
1547 			const bool map = next->mapped;
1548 			const u8  refd = next->refd;
1549 			const u64 addr = next->addr;
1550 			u64 size = next->size;
1551 
1552 			/* Merge regions that are in the same state. */
1553 			while ((next = node(next, next)) && next->part &&
1554 			       (next->mapped == map) &&
1555 			       (next->memory != NULL) == mem &&
1556 			       (next->refd == refd))
1557 				size += next->size;
1558 
1559 			if (map) {
1560 				/* Region(s) are mapped, merge the unmap
1561 				 * and dereference into a single walk of
1562 				 * the page tree.
1563 				 */
1564 				nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
1565 							size, vma->sparse,
1566 							!mem);
1567 			} else
1568 			if (refd != NVKM_VMA_PAGE_NONE) {
1569 				/* Drop allocation-time PTE references. */
1570 				nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1571 			}
1572 		} while (next && next->part);
1573 	}
1574 
1575 	/* Merge any mapped regions that were split from the initial
1576 	 * address-space allocation back into the allocated VMA, and
1577 	 * release memory/compression resources.
1578 	 */
1579 	next = vma;
1580 	do {
1581 		if (next->mapped)
1582 			nvkm_vmm_unmap_region(vmm, next);
1583 	} while ((next = node(vma, next)) && next->part);
1584 
1585 	if (vma->sparse && !vma->mapref) {
1586 		/* Sparse region that was allocated with a fixed page size,
1587 		 * meaning all relevant PTEs were referenced once when the
1588 		 * region was allocated, and remained that way, regardless
1589 		 * of whether memory was mapped into it afterwards.
1590 		 *
1591 		 * The process of unmapping, unsparsing, and dereferencing
1592 		 * PTEs can be done in a single page tree walk.
1593 		 */
1594 		nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
1595 	} else
1596 	if (vma->sparse) {
1597 		/* Sparse region that wasn't allocated with a fixed page size,
1598 		 * PTE references were taken both at allocation time (to make
1599 		 * the GPU see the region as sparse), and when mapping memory
1600 		 * into the region.
1601 		 *
1602 		 * The latter was handled above, and the remaining references
1603 		 * are dealt with here.
1604 		 */
1605 		nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
1606 	}
1607 
1608 	/* Remove VMA from the list of allocated nodes. */
1609 	nvkm_vmm_node_remove(vmm, vma);
1610 
1611 	/* Merge VMA back into the free list. */
1612 	vma->page = NVKM_VMA_PAGE_NONE;
1613 	vma->refd = NVKM_VMA_PAGE_NONE;
1614 	vma->used = false;
1615 	nvkm_vmm_put_region(vmm, vma);
1616 }
1617 
1618 void
1619 nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
1620 {
1621 	struct nvkm_vma *vma = *pvma;
1622 	if (vma) {
1623 		mutex_lock(&vmm->mutex);
1624 		nvkm_vmm_put_locked(vmm, vma);
1625 		mutex_unlock(&vmm->mutex);
1626 		*pvma = NULL;
1627 	}
1628 }
1629 
1630 int
1631 nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
1632 		    u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
1633 {
1634 	const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
1635 	struct rb_node *node = NULL, *temp;
1636 	struct nvkm_vma *vma = NULL, *tmp;
1637 	u64 addr, tail;
1638 	int ret;
1639 
1640 	VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
1641 		       "shift: %d align: %d size: %016llx",
1642 		  getref, mapref, sparse, shift, align, size);
1643 
1644 	/* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
1645 	if (unlikely(!size || (!getref && !mapref && sparse))) {
1646 		VMM_DEBUG(vmm, "args %016llx %d %d %d",
1647 			  size, getref, mapref, sparse);
1648 		return -EINVAL;
1649 	}
1650 
1651 	/* Tesla-class GPUs can only select page size per-PDE, which means
1652 	 * we're required to know the mapping granularity up-front to find
1653 	 * a suitable region of address-space.
1654 	 *
1655 	 * The same goes if we're requesting up-front allocation of PTES.
1656 	 */
1657 	if (unlikely((getref || vmm->func->page_block) && !shift)) {
1658 		VMM_DEBUG(vmm, "page size required: %d %016llx",
1659 			  getref, vmm->func->page_block);
1660 		return -EINVAL;
1661 	}
1662 
1663 	/* If a specific page size was requested, determine its index and
1664 	 * make sure the requested size is a multiple of the page size.
1665 	 */
1666 	if (shift) {
1667 		for (page = vmm->func->page; page->shift; page++) {
1668 			if (shift == page->shift)
1669 				break;
1670 		}
1671 
1672 		if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
1673 			VMM_DEBUG(vmm, "page %d %016llx", shift, size);
1674 			return -EINVAL;
1675 		}
1676 		align = max_t(u8, align, shift);
1677 	} else {
1678 		align = max_t(u8, align, 12);
1679 	}
1680 
1681 	/* Locate smallest block that can possibly satisfy the allocation. */
1682 	temp = vmm->free.rb_node;
1683 	while (temp) {
1684 		struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
1685 		if (this->size < size) {
1686 			temp = temp->rb_right;
1687 		} else {
1688 			node = temp;
1689 			temp = temp->rb_left;
1690 		}
1691 	}
1692 
1693 	if (unlikely(!node))
1694 		return -ENOSPC;
1695 
1696 	/* Take into account alignment restrictions, trying larger blocks
1697 	 * in turn until we find a suitable free block.
1698 	 */
1699 	do {
1700 		struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
1701 		struct nvkm_vma *prev = node(this, prev);
1702 		struct nvkm_vma *next = node(this, next);
1703 		const int p = page - vmm->func->page;
1704 
1705 		addr = this->addr;
1706 		if (vmm->func->page_block && prev && prev->page != p)
1707 			addr = ALIGN(addr, vmm->func->page_block);
1708 		addr = ALIGN(addr, 1ULL << align);
1709 
1710 		tail = this->addr + this->size;
1711 		if (vmm->func->page_block && next && next->page != p)
1712 			tail = ALIGN_DOWN(tail, vmm->func->page_block);
1713 
1714 		if (addr <= tail && tail - addr >= size) {
1715 			nvkm_vmm_free_remove(vmm, this);
1716 			vma = this;
1717 			break;
1718 		}
1719 	} while ((node = rb_next(node)));
1720 
1721 	if (unlikely(!vma))
1722 		return -ENOSPC;
1723 
1724 	/* If the VMA we found isn't already exactly the requested size,
1725 	 * it needs to be split, and the remaining free blocks returned.
1726 	 */
1727 	if (addr != vma->addr) {
1728 		if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
1729 			nvkm_vmm_put_region(vmm, vma);
1730 			return -ENOMEM;
1731 		}
1732 		nvkm_vmm_free_insert(vmm, vma);
1733 		vma = tmp;
1734 	}
1735 
1736 	if (size != vma->size) {
1737 		if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
1738 			nvkm_vmm_put_region(vmm, vma);
1739 			return -ENOMEM;
1740 		}
1741 		nvkm_vmm_free_insert(vmm, tmp);
1742 	}
1743 
1744 	/* Pre-allocate page tables and/or setup sparse mappings. */
1745 	if (sparse && getref)
1746 		ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
1747 	else if (sparse)
1748 		ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
1749 	else if (getref)
1750 		ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
1751 	else
1752 		ret = 0;
1753 	if (ret) {
1754 		nvkm_vmm_put_region(vmm, vma);
1755 		return ret;
1756 	}
1757 
1758 	vma->mapref = mapref && !getref;
1759 	vma->sparse = sparse;
1760 	vma->page = page - vmm->func->page;
1761 	vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
1762 	vma->used = true;
1763 	nvkm_vmm_node_insert(vmm, vma);
1764 	*pvma = vma;
1765 	return 0;
1766 }
1767 
1768 int
1769 nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
1770 {
1771 	int ret;
1772 	mutex_lock(&vmm->mutex);
1773 	ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
1774 	mutex_unlock(&vmm->mutex);
1775 	return ret;
1776 }
1777 
1778 void
1779 nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1780 {
1781 	if (inst && vmm && vmm->func->part) {
1782 		mutex_lock(&vmm->mutex);
1783 		vmm->func->part(vmm, inst);
1784 		mutex_unlock(&vmm->mutex);
1785 	}
1786 }
1787 
1788 int
1789 nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1790 {
1791 	int ret = 0;
1792 	if (vmm->func->join) {
1793 		mutex_lock(&vmm->mutex);
1794 		ret = vmm->func->join(vmm, inst);
1795 		mutex_unlock(&vmm->mutex);
1796 	}
1797 	return ret;
1798 }
1799 
1800 static bool
1801 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
1802 {
1803 	const struct nvkm_vmm_desc *desc = it->desc;
1804 	const int type = desc->type == SPT;
1805 	nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
1806 	return false;
1807 }
1808 
1809 int
1810 nvkm_vmm_boot(struct nvkm_vmm *vmm)
1811 {
1812 	const struct nvkm_vmm_page *page = vmm->func->page;
1813 	const u64 limit = vmm->limit - vmm->start;
1814 	int ret;
1815 
1816 	while (page[1].shift)
1817 		page++;
1818 
1819 	ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
1820 	if (ret)
1821 		return ret;
1822 
1823 	nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
1824 		      nvkm_vmm_boot_ptes, NULL, NULL, NULL);
1825 	vmm->bootstrapped = true;
1826 	return 0;
1827 }
1828 
1829 static void
1830 nvkm_vmm_del(struct kref *kref)
1831 {
1832 	struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
1833 	nvkm_vmm_dtor(vmm);
1834 	kfree(vmm);
1835 }
1836 
1837 void
1838 nvkm_vmm_unref(struct nvkm_vmm **pvmm)
1839 {
1840 	struct nvkm_vmm *vmm = *pvmm;
1841 	if (vmm) {
1842 		kref_put(&vmm->kref, nvkm_vmm_del);
1843 		*pvmm = NULL;
1844 	}
1845 }
1846 
1847 struct nvkm_vmm *
1848 nvkm_vmm_ref(struct nvkm_vmm *vmm)
1849 {
1850 	if (vmm)
1851 		kref_get(&vmm->kref);
1852 	return vmm;
1853 }
1854 
1855 int
1856 nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
1857 	     u32 argc, struct lock_class_key *key, const char *name,
1858 	     struct nvkm_vmm **pvmm)
1859 {
1860 	struct nvkm_mmu *mmu = device->mmu;
1861 	struct nvkm_vmm *vmm = NULL;
1862 	int ret;
1863 	ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc,
1864 				  key, name, &vmm);
1865 	if (ret)
1866 		nvkm_vmm_unref(&vmm);
1867 	*pvmm = vmm;
1868 	return ret;
1869 }
1870