1 /*
2  * Copyright 2017 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 #define NVKM_VMM_LEVELS_MAX 5
23 #include "vmm.h"
24 
25 #include <subdev/fb.h>
26 
27 static void
28 nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
29 {
30 	struct nvkm_vmm_pt *pgt = *ppgt;
31 	if (pgt) {
32 		kvfree(pgt->pde);
33 		kfree(pgt);
34 		*ppgt = NULL;
35 	}
36 }
37 
38 
39 static struct nvkm_vmm_pt *
40 nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
41 		const struct nvkm_vmm_page *page)
42 {
43 	const u32 pten = 1 << desc->bits;
44 	struct nvkm_vmm_pt *pgt;
45 	u32 lpte = 0;
46 
47 	if (desc->type > PGT) {
48 		if (desc->type == SPT) {
49 			const struct nvkm_vmm_desc *pair = page[-1].desc;
50 			lpte = pten >> (desc->bits - pair->bits);
51 		} else {
52 			lpte = pten;
53 		}
54 	}
55 
56 	if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
57 		return NULL;
58 	pgt->page = page ? page->shift : 0;
59 	pgt->sparse = sparse;
60 
61 	if (desc->type == PGD) {
62 		pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
63 		if (!pgt->pde) {
64 			kfree(pgt);
65 			return NULL;
66 		}
67 	}
68 
69 	return pgt;
70 }
71 
72 struct nvkm_vmm_iter {
73 	const struct nvkm_vmm_page *page;
74 	const struct nvkm_vmm_desc *desc;
75 	struct nvkm_vmm *vmm;
76 	u64 cnt;
77 	u16 max, lvl;
78 	u32 pte[NVKM_VMM_LEVELS_MAX];
79 	struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
80 	int flush;
81 };
82 
83 #ifdef CONFIG_NOUVEAU_DEBUG_MMU
84 static const char *
85 nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
86 {
87 	switch (desc->type) {
88 	case PGD: return "PGD";
89 	case PGT: return "PGT";
90 	case SPT: return "SPT";
91 	case LPT: return "LPT";
92 	default:
93 		return "UNKNOWN";
94 	}
95 }
96 
97 static void
98 nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
99 {
100 	int lvl;
101 	for (lvl = it->max; lvl >= 0; lvl--) {
102 		if (lvl >= it->lvl)
103 			buf += sprintf(buf,  "%05x:", it->pte[lvl]);
104 		else
105 			buf += sprintf(buf, "xxxxx:");
106 	}
107 }
108 
109 #define TRA(i,f,a...) do {                                                     \
110 	char _buf[NVKM_VMM_LEVELS_MAX * 7];                                    \
111 	struct nvkm_vmm_iter *_it = (i);                                       \
112 	nvkm_vmm_trace(_it, _buf);                                             \
113 	VMM_TRACE(_it->vmm, "%s "f, _buf, ##a);                                \
114 } while(0)
115 #else
116 #define TRA(i,f,a...)
117 #endif
118 
119 static inline void
120 nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
121 {
122 	it->flush = min(it->flush, it->max - it->lvl);
123 }
124 
125 static inline void
126 nvkm_vmm_flush(struct nvkm_vmm_iter *it)
127 {
128 	if (it->flush != NVKM_VMM_LEVELS_MAX) {
129 		if (it->vmm->func->flush) {
130 			TRA(it, "flush: %d", it->flush);
131 			it->vmm->func->flush(it->vmm, it->flush);
132 		}
133 		it->flush = NVKM_VMM_LEVELS_MAX;
134 	}
135 }
136 
137 static void
138 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
139 {
140 	const struct nvkm_vmm_desc *desc = it->desc;
141 	const int type = desc[it->lvl].type == SPT;
142 	struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
143 	struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
144 	struct nvkm_mmu_pt *pt = pgt->pt[type];
145 	struct nvkm_vmm *vmm = it->vmm;
146 	u32 pdei = it->pte[it->lvl + 1];
147 
148 	/* Recurse up the tree, unreferencing/destroying unneeded PDs. */
149 	it->lvl++;
150 	if (--pgd->refs[0]) {
151 		const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
152 		/* PD has other valid PDEs, so we need a proper update. */
153 		TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
154 		pgt->pt[type] = NULL;
155 		if (!pgt->refs[!type]) {
156 			/* PDE no longer required. */
157 			if (pgd->pt[0]) {
158 				if (pgt->sparse) {
159 					func->sparse(vmm, pgd->pt[0], pdei, 1);
160 					pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
161 				} else {
162 					func->unmap(vmm, pgd->pt[0], pdei, 1);
163 					pgd->pde[pdei] = NULL;
164 				}
165 			} else {
166 				/* Special handling for Tesla-class GPUs,
167 				 * where there's no central PD, but each
168 				 * instance has its own embedded PD.
169 				 */
170 				func->pde(vmm, pgd, pdei);
171 				pgd->pde[pdei] = NULL;
172 			}
173 		} else {
174 			/* PDE was pointing at dual-PTs and we're removing
175 			 * one of them, leaving the other in place.
176 			 */
177 			func->pde(vmm, pgd, pdei);
178 		}
179 
180 		/* GPU may have cached the PTs, flush before freeing. */
181 		nvkm_vmm_flush_mark(it);
182 		nvkm_vmm_flush(it);
183 	} else {
184 		/* PD has no valid PDEs left, so we can just destroy it. */
185 		nvkm_vmm_unref_pdes(it);
186 	}
187 
188 	/* Destroy PD/PT. */
189 	TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
190 	nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
191 	if (!pgt->refs[!type])
192 		nvkm_vmm_pt_del(&pgt);
193 	it->lvl--;
194 }
195 
196 static void
197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
198 		     const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
199 {
200 	const struct nvkm_vmm_desc *pair = it->page[-1].desc;
201 	const u32 sptb = desc->bits - pair->bits;
202 	const u32 sptn = 1 << sptb;
203 	struct nvkm_vmm *vmm = it->vmm;
204 	u32 spti = ptei & (sptn - 1), lpti, pteb;
205 
206 	/* Determine how many SPTEs are being touched under each LPTE,
207 	 * and drop reference counts.
208 	 */
209 	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
210 		const u32 pten = min(sptn - spti, ptes);
211 		pgt->pte[lpti] -= pten;
212 		ptes -= pten;
213 	}
214 
215 	/* We're done here if there's no corresponding LPT. */
216 	if (!pgt->refs[0])
217 		return;
218 
219 	for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
220 		/* Skip over any LPTEs that still have valid SPTEs. */
221 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
222 			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
223 				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
224 					break;
225 			}
226 			continue;
227 		}
228 
229 		/* As there's no more non-UNMAPPED SPTEs left in the range
230 		 * covered by a number of LPTEs, the LPTEs once again take
231 		 * control over their address range.
232 		 *
233 		 * Determine how many LPTEs need to transition state.
234 		 */
235 		pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
236 		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
237 			if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
238 				break;
239 			pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
240 		}
241 
242 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
243 			TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
244 			pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
245 		} else
246 		if (pair->func->invalid) {
247 			/* If the MMU supports it, restore the LPTE to the
248 			 * INVALID state to tell the MMU there is no point
249 			 * trying to fetch the corresponding SPTEs.
250 			 */
251 			TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
252 			pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
253 		}
254 	}
255 }
256 
257 static bool
258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
259 {
260 	const struct nvkm_vmm_desc *desc = it->desc;
261 	const int type = desc->type == SPT;
262 	struct nvkm_vmm_pt *pgt = it->pt[0];
263 	bool dma;
264 
265 	if (pfn) {
266 		/* Need to clear PTE valid bits before we dma_unmap_page(). */
267 		dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
268 		if (dma) {
269 			/* GPU may have cached the PT, flush before unmap. */
270 			nvkm_vmm_flush_mark(it);
271 			nvkm_vmm_flush(it);
272 			desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
273 		}
274 	}
275 
276 	/* Drop PTE references. */
277 	pgt->refs[type] -= ptes;
278 
279 	/* Dual-PTs need special handling, unless PDE becoming invalid. */
280 	if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
281 		nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
282 
283 	/* PT no longer needed? Destroy it. */
284 	if (!pgt->refs[type]) {
285 		it->lvl++;
286 		TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
287 		it->lvl--;
288 		nvkm_vmm_unref_pdes(it);
289 		return false; /* PTE writes for unmap() not necessary. */
290 	}
291 
292 	return true;
293 }
294 
295 static void
296 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
297 		   const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
298 {
299 	const struct nvkm_vmm_desc *pair = it->page[-1].desc;
300 	const u32 sptb = desc->bits - pair->bits;
301 	const u32 sptn = 1 << sptb;
302 	struct nvkm_vmm *vmm = it->vmm;
303 	u32 spti = ptei & (sptn - 1), lpti, pteb;
304 
305 	/* Determine how many SPTEs are being touched under each LPTE,
306 	 * and increase reference counts.
307 	 */
308 	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
309 		const u32 pten = min(sptn - spti, ptes);
310 		pgt->pte[lpti] += pten;
311 		ptes -= pten;
312 	}
313 
314 	/* We're done here if there's no corresponding LPT. */
315 	if (!pgt->refs[0])
316 		return;
317 
318 	for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
319 		/* Skip over any LPTEs that already have valid SPTEs. */
320 		if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
321 			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
322 				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
323 					break;
324 			}
325 			continue;
326 		}
327 
328 		/* As there are now non-UNMAPPED SPTEs in the range covered
329 		 * by a number of LPTEs, we need to transfer control of the
330 		 * address range to the SPTEs.
331 		 *
332 		 * Determine how many LPTEs need to transition state.
333 		 */
334 		pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
335 		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
336 			if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
337 				break;
338 			pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
339 		}
340 
341 		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
342 			const u32 spti = pteb * sptn;
343 			const u32 sptc = ptes * sptn;
344 			/* The entire LPTE is marked as sparse, we need
345 			 * to make sure that the SPTEs are too.
346 			 */
347 			TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
348 			desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
349 			/* Sparse LPTEs prevent SPTEs from being accessed. */
350 			TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
351 			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
352 		} else
353 		if (pair->func->invalid) {
354 			/* MMU supports blocking SPTEs by marking an LPTE
355 			 * as INVALID.  We need to reverse that here.
356 			 */
357 			TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
358 			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
359 		}
360 	}
361 }
362 
363 static bool
364 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
365 {
366 	const struct nvkm_vmm_desc *desc = it->desc;
367 	const int type = desc->type == SPT;
368 	struct nvkm_vmm_pt *pgt = it->pt[0];
369 
370 	/* Take PTE references. */
371 	pgt->refs[type] += ptes;
372 
373 	/* Dual-PTs need special handling. */
374 	if (desc->type == SPT)
375 		nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
376 
377 	return true;
378 }
379 
380 static void
381 nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
382 		     struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
383 {
384 	if (desc->type == PGD) {
385 		while (ptes--)
386 			pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
387 	} else
388 	if (desc->type == LPT) {
389 		memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
390 	}
391 }
392 
393 static bool
394 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
395 {
396 	struct nvkm_vmm_pt *pt = it->pt[0];
397 	if (it->desc->type == PGD)
398 		memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
399 	else
400 	if (it->desc->type == LPT)
401 		memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
402 	return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
403 }
404 
405 static bool
406 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
407 {
408 	nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
409 	return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
410 }
411 
412 static bool
413 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
414 {
415 	const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
416 	const int type = desc->type == SPT;
417 	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
418 	const bool zero = !pgt->sparse && !desc->func->invalid;
419 	struct nvkm_vmm *vmm = it->vmm;
420 	struct nvkm_mmu *mmu = vmm->mmu;
421 	struct nvkm_mmu_pt *pt;
422 	u32 pten = 1 << desc->bits;
423 	u32 pteb, ptei, ptes;
424 	u32 size = desc->size * pten;
425 
426 	pgd->refs[0]++;
427 
428 	pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
429 	if (!pgt->pt[type]) {
430 		it->lvl--;
431 		nvkm_vmm_unref_pdes(it);
432 		return false;
433 	}
434 
435 	if (zero)
436 		goto done;
437 
438 	pt = pgt->pt[type];
439 
440 	if (desc->type == LPT && pgt->refs[1]) {
441 		/* SPT already exists covering the same range as this LPT,
442 		 * which means we need to be careful that any LPTEs which
443 		 * overlap valid SPTEs are unmapped as opposed to invalid
444 		 * or sparse, which would prevent the MMU from looking at
445 		 * the SPTEs on some GPUs.
446 		 */
447 		for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
448 			bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
449 			for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
450 				bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
451 				if (spte != next)
452 					break;
453 			}
454 
455 			if (!spte) {
456 				if (pgt->sparse)
457 					desc->func->sparse(vmm, pt, pteb, ptes);
458 				else
459 					desc->func->invalid(vmm, pt, pteb, ptes);
460 				memset(&pgt->pte[pteb], 0x00, ptes);
461 			} else {
462 				desc->func->unmap(vmm, pt, pteb, ptes);
463 				while (ptes--)
464 					pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
465 			}
466 		}
467 	} else {
468 		if (pgt->sparse) {
469 			nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
470 			desc->func->sparse(vmm, pt, 0, pten);
471 		} else {
472 			desc->func->invalid(vmm, pt, 0, pten);
473 		}
474 	}
475 
476 done:
477 	TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
478 	it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
479 	nvkm_vmm_flush_mark(it);
480 	return true;
481 }
482 
483 static bool
484 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
485 {
486 	const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
487 	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
488 
489 	pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
490 	if (!pgt) {
491 		if (!pgd->refs[0])
492 			nvkm_vmm_unref_pdes(it);
493 		return false;
494 	}
495 
496 	pgd->pde[pdei] = pgt;
497 	return true;
498 }
499 
500 static inline u64
501 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
502 	      u64 addr, u64 size, const char *name, bool ref, bool pfn,
503 	      bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
504 	      nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
505 	      nvkm_vmm_pxe_func CLR_PTES)
506 {
507 	const struct nvkm_vmm_desc *desc = page->desc;
508 	struct nvkm_vmm_iter it;
509 	u64 bits = addr >> page->shift;
510 
511 	it.page = page;
512 	it.desc = desc;
513 	it.vmm = vmm;
514 	it.cnt = size >> page->shift;
515 	it.flush = NVKM_VMM_LEVELS_MAX;
516 
517 	/* Deconstruct address into PTE indices for each mapping level. */
518 	for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
519 		it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
520 		bits >>= desc[it.lvl].bits;
521 	}
522 	it.max = --it.lvl;
523 	it.pt[it.max] = vmm->pd;
524 
525 	it.lvl = 0;
526 	TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
527 	         addr, size, page->shift, it.cnt);
528 	it.lvl = it.max;
529 
530 	/* Depth-first traversal of page tables. */
531 	while (it.cnt) {
532 		struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
533 		const int type = desc->type == SPT;
534 		const u32 pten = 1 << desc->bits;
535 		const u32 ptei = it.pte[0];
536 		const u32 ptes = min_t(u64, it.cnt, pten - ptei);
537 
538 		/* Walk down the tree, finding page tables for each level. */
539 		for (; it.lvl; it.lvl--) {
540 			const u32 pdei = it.pte[it.lvl];
541 			struct nvkm_vmm_pt *pgd = pgt;
542 
543 			/* Software PT. */
544 			if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
545 				if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
546 					goto fail;
547 			}
548 			it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
549 
550 			/* Hardware PT.
551 			 *
552 			 * This is a separate step from above due to GF100 and
553 			 * newer having dual page tables at some levels, which
554 			 * are refcounted independently.
555 			 */
556 			if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
557 				if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
558 					goto fail;
559 			}
560 		}
561 
562 		/* Handle PTE updates. */
563 		if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
564 			struct nvkm_mmu_pt *pt = pgt->pt[type];
565 			if (MAP_PTES || CLR_PTES) {
566 				if (MAP_PTES)
567 					MAP_PTES(vmm, pt, ptei, ptes, map);
568 				else
569 					CLR_PTES(vmm, pt, ptei, ptes);
570 				nvkm_vmm_flush_mark(&it);
571 			}
572 		}
573 
574 		/* Walk back up the tree to the next position. */
575 		it.pte[it.lvl] += ptes;
576 		it.cnt -= ptes;
577 		if (it.cnt) {
578 			while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
579 				it.pte[it.lvl++] = 0;
580 				it.pte[it.lvl]++;
581 			}
582 		}
583 	}
584 
585 	nvkm_vmm_flush(&it);
586 	return ~0ULL;
587 
588 fail:
589 	/* Reconstruct the failure address so the caller is able to
590 	 * reverse any partially completed operations.
591 	 */
592 	addr = it.pte[it.max--];
593 	do {
594 		addr  = addr << desc[it.max].bits;
595 		addr |= it.pte[it.max];
596 	} while (it.max--);
597 
598 	return addr << page->shift;
599 }
600 
601 static void
602 nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
603 			 u64 addr, u64 size)
604 {
605 	nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
606 		      nvkm_vmm_sparse_unref_ptes, NULL, NULL,
607 		      page->desc->func->invalid ?
608 		      page->desc->func->invalid : page->desc->func->unmap);
609 }
610 
611 static int
612 nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
613 			 u64 addr, u64 size)
614 {
615 	if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
616 		u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
617 					 true, false, nvkm_vmm_sparse_ref_ptes,
618 					 NULL, NULL, page->desc->func->sparse);
619 		if (fail != ~0ULL) {
620 			if ((size = fail - addr))
621 				nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
622 			return -ENOMEM;
623 		}
624 		return 0;
625 	}
626 	return -EINVAL;
627 }
628 
629 static int
630 nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
631 {
632 	const struct nvkm_vmm_page *page = vmm->func->page;
633 	int m = 0, i;
634 	u64 start = addr;
635 	u64 block;
636 
637 	while (size) {
638 		/* Limit maximum page size based on remaining size. */
639 		while (size < (1ULL << page[m].shift))
640 			m++;
641 		i = m;
642 
643 		/* Find largest page size suitable for alignment. */
644 		while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
645 			i++;
646 
647 		/* Determine number of PTEs at this page size. */
648 		if (i != m) {
649 			/* Limited to alignment boundary of next page size. */
650 			u64 next = 1ULL << page[i - 1].shift;
651 			u64 part = ALIGN(addr, next) - addr;
652 			if (size - part >= next)
653 				block = (part >> page[i].shift) << page[i].shift;
654 			else
655 				block = (size >> page[i].shift) << page[i].shift;
656 		} else {
657 			block = (size >> page[i].shift) << page[i].shift;
658 		}
659 
660 		/* Perform operation. */
661 		if (ref) {
662 			int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
663 			if (ret) {
664 				if ((size = addr - start))
665 					nvkm_vmm_ptes_sparse(vmm, start, size, false);
666 				return ret;
667 			}
668 		} else {
669 			nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
670 		}
671 
672 		size -= block;
673 		addr += block;
674 	}
675 
676 	return 0;
677 }
678 
679 static void
680 nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
681 		    u64 addr, u64 size, bool sparse, bool pfn)
682 {
683 	const struct nvkm_vmm_desc_func *func = page->desc->func;
684 
685 	mutex_lock(&vmm->mutex.map);
686 	nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
687 		      NULL, NULL, NULL,
688 		      sparse ? func->sparse : func->invalid ? func->invalid :
689 							      func->unmap);
690 	mutex_unlock(&vmm->mutex.map);
691 }
692 
693 static void
694 nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
695 		  u64 addr, u64 size, struct nvkm_vmm_map *map,
696 		  nvkm_vmm_pte_func func)
697 {
698 	mutex_lock(&vmm->mutex.map);
699 	nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
700 		      NULL, func, map, NULL);
701 	mutex_unlock(&vmm->mutex.map);
702 }
703 
704 static void
705 nvkm_vmm_ptes_put_locked(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
706 			 u64 addr, u64 size)
707 {
708 	nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
709 		      nvkm_vmm_unref_ptes, NULL, NULL, NULL);
710 }
711 
712 static void
713 nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
714 		  u64 addr, u64 size)
715 {
716 	mutex_lock(&vmm->mutex.ref);
717 	nvkm_vmm_ptes_put_locked(vmm, page, addr, size);
718 	mutex_unlock(&vmm->mutex.ref);
719 }
720 
721 static int
722 nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
723 		  u64 addr, u64 size)
724 {
725 	u64 fail;
726 
727 	mutex_lock(&vmm->mutex.ref);
728 	fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
729 			     nvkm_vmm_ref_ptes, NULL, NULL, NULL);
730 	if (fail != ~0ULL) {
731 		if (fail != addr)
732 			nvkm_vmm_ptes_put_locked(vmm, page, addr, fail - addr);
733 		mutex_unlock(&vmm->mutex.ref);
734 		return -ENOMEM;
735 	}
736 	mutex_unlock(&vmm->mutex.ref);
737 	return 0;
738 }
739 
740 static void
741 __nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
742 			  u64 addr, u64 size, bool sparse, bool pfn)
743 {
744 	const struct nvkm_vmm_desc_func *func = page->desc->func;
745 
746 	nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
747 		      false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
748 		      sparse ? func->sparse : func->invalid ? func->invalid :
749 							      func->unmap);
750 }
751 
752 static void
753 nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
754 			u64 addr, u64 size, bool sparse, bool pfn)
755 {
756 	if (vmm->managed.raw) {
757 		nvkm_vmm_ptes_unmap(vmm, page, addr, size, sparse, pfn);
758 		nvkm_vmm_ptes_put(vmm, page, addr, size);
759 	} else {
760 		__nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, sparse, pfn);
761 	}
762 }
763 
764 static int
765 __nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
766 			u64 addr, u64 size, struct nvkm_vmm_map *map,
767 			nvkm_vmm_pte_func func)
768 {
769 	u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
770 				 false, nvkm_vmm_ref_ptes, func, map, NULL);
771 	if (fail != ~0ULL) {
772 		if ((size = fail - addr))
773 			nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
774 		return -ENOMEM;
775 	}
776 	return 0;
777 }
778 
779 static int
780 nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
781 		      u64 addr, u64 size, struct nvkm_vmm_map *map,
782 		      nvkm_vmm_pte_func func)
783 {
784 	int ret;
785 
786 	if (vmm->managed.raw) {
787 		ret = nvkm_vmm_ptes_get(vmm, page, addr, size);
788 		if (ret)
789 			return ret;
790 
791 		nvkm_vmm_ptes_map(vmm, page, addr, size, map, func);
792 
793 		return 0;
794 	} else {
795 		return __nvkm_vmm_ptes_get_map(vmm, page, addr, size, map, func);
796 	}
797 }
798 
799 struct nvkm_vma *
800 nvkm_vma_new(u64 addr, u64 size)
801 {
802 	struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
803 	if (vma) {
804 		vma->addr = addr;
805 		vma->size = size;
806 		vma->page = NVKM_VMA_PAGE_NONE;
807 		vma->refd = NVKM_VMA_PAGE_NONE;
808 	}
809 	return vma;
810 }
811 
812 struct nvkm_vma *
813 nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
814 {
815 	struct nvkm_vma *new;
816 
817 	BUG_ON(vma->size == tail);
818 
819 	if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
820 		return NULL;
821 	vma->size -= tail;
822 
823 	new->mapref = vma->mapref;
824 	new->sparse = vma->sparse;
825 	new->page = vma->page;
826 	new->refd = vma->refd;
827 	new->used = vma->used;
828 	new->part = vma->part;
829 	new->busy = vma->busy;
830 	new->mapped = vma->mapped;
831 	list_add(&new->head, &vma->head);
832 	return new;
833 }
834 
835 static inline void
836 nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
837 {
838 	rb_erase(&vma->tree, &vmm->free);
839 }
840 
841 static inline void
842 nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
843 {
844 	nvkm_vmm_free_remove(vmm, vma);
845 	list_del(&vma->head);
846 	kfree(vma);
847 }
848 
849 static void
850 nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
851 {
852 	struct rb_node **ptr = &vmm->free.rb_node;
853 	struct rb_node *parent = NULL;
854 
855 	while (*ptr) {
856 		struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
857 		parent = *ptr;
858 		if (vma->size < this->size)
859 			ptr = &parent->rb_left;
860 		else
861 		if (vma->size > this->size)
862 			ptr = &parent->rb_right;
863 		else
864 		if (vma->addr < this->addr)
865 			ptr = &parent->rb_left;
866 		else
867 		if (vma->addr > this->addr)
868 			ptr = &parent->rb_right;
869 		else
870 			BUG();
871 	}
872 
873 	rb_link_node(&vma->tree, parent, ptr);
874 	rb_insert_color(&vma->tree, &vmm->free);
875 }
876 
877 static inline void
878 nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
879 {
880 	rb_erase(&vma->tree, &vmm->root);
881 }
882 
883 static inline void
884 nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
885 {
886 	nvkm_vmm_node_remove(vmm, vma);
887 	list_del(&vma->head);
888 	kfree(vma);
889 }
890 
891 static void
892 nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
893 {
894 	struct rb_node **ptr = &vmm->root.rb_node;
895 	struct rb_node *parent = NULL;
896 
897 	while (*ptr) {
898 		struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
899 		parent = *ptr;
900 		if (vma->addr < this->addr)
901 			ptr = &parent->rb_left;
902 		else
903 		if (vma->addr > this->addr)
904 			ptr = &parent->rb_right;
905 		else
906 			BUG();
907 	}
908 
909 	rb_link_node(&vma->tree, parent, ptr);
910 	rb_insert_color(&vma->tree, &vmm->root);
911 }
912 
913 struct nvkm_vma *
914 nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
915 {
916 	struct rb_node *node = vmm->root.rb_node;
917 	while (node) {
918 		struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
919 		if (addr < vma->addr)
920 			node = node->rb_left;
921 		else
922 		if (addr >= vma->addr + vma->size)
923 			node = node->rb_right;
924 		else
925 			return vma;
926 	}
927 	return NULL;
928 }
929 
930 #define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL :             \
931 	list_entry((root)->head.dir, struct nvkm_vma, head))
932 
933 static struct nvkm_vma *
934 nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev,
935 		    struct nvkm_vma *vma, struct nvkm_vma *next, u64 size)
936 {
937 	if (next) {
938 		if (vma->size == size) {
939 			vma->size += next->size;
940 			nvkm_vmm_node_delete(vmm, next);
941 			if (prev) {
942 				prev->size += vma->size;
943 				nvkm_vmm_node_delete(vmm, vma);
944 				return prev;
945 			}
946 			return vma;
947 		}
948 		BUG_ON(prev);
949 
950 		nvkm_vmm_node_remove(vmm, next);
951 		vma->size -= size;
952 		next->addr -= size;
953 		next->size += size;
954 		nvkm_vmm_node_insert(vmm, next);
955 		return next;
956 	}
957 
958 	if (prev) {
959 		if (vma->size != size) {
960 			nvkm_vmm_node_remove(vmm, vma);
961 			prev->size += size;
962 			vma->addr += size;
963 			vma->size -= size;
964 			nvkm_vmm_node_insert(vmm, vma);
965 		} else {
966 			prev->size += vma->size;
967 			nvkm_vmm_node_delete(vmm, vma);
968 		}
969 		return prev;
970 	}
971 
972 	return vma;
973 }
974 
975 struct nvkm_vma *
976 nvkm_vmm_node_split(struct nvkm_vmm *vmm,
977 		    struct nvkm_vma *vma, u64 addr, u64 size)
978 {
979 	struct nvkm_vma *prev = NULL;
980 
981 	if (vma->addr != addr) {
982 		prev = vma;
983 		if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr)))
984 			return NULL;
985 		vma->part = true;
986 		nvkm_vmm_node_insert(vmm, vma);
987 	}
988 
989 	if (vma->size != size) {
990 		struct nvkm_vma *tmp;
991 		if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
992 			nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size);
993 			return NULL;
994 		}
995 		tmp->part = true;
996 		nvkm_vmm_node_insert(vmm, tmp);
997 	}
998 
999 	return vma;
1000 }
1001 
1002 static void
1003 nvkm_vma_dump(struct nvkm_vma *vma)
1004 {
1005 	printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c %p\n",
1006 	       vma->addr, (u64)vma->size,
1007 	       vma->used ? '-' : 'F',
1008 	       vma->mapref ? 'R' : '-',
1009 	       vma->sparse ? 'S' : '-',
1010 	       vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
1011 	       vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
1012 	       vma->part ? 'P' : '-',
1013 	       vma->busy ? 'B' : '-',
1014 	       vma->mapped ? 'M' : '-',
1015 	       vma->memory);
1016 }
1017 
1018 static void
1019 nvkm_vmm_dump(struct nvkm_vmm *vmm)
1020 {
1021 	struct nvkm_vma *vma;
1022 	list_for_each_entry(vma, &vmm->list, head) {
1023 		nvkm_vma_dump(vma);
1024 	}
1025 }
1026 
1027 static void
1028 nvkm_vmm_dtor(struct nvkm_vmm *vmm)
1029 {
1030 	struct nvkm_vma *vma;
1031 	struct rb_node *node;
1032 
1033 	if (0)
1034 		nvkm_vmm_dump(vmm);
1035 
1036 	while ((node = rb_first(&vmm->root))) {
1037 		struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
1038 		nvkm_vmm_put(vmm, &vma);
1039 	}
1040 
1041 	if (vmm->bootstrapped) {
1042 		const struct nvkm_vmm_page *page = vmm->func->page;
1043 		const u64 limit = vmm->limit - vmm->start;
1044 
1045 		while (page[1].shift)
1046 			page++;
1047 
1048 		nvkm_mmu_ptc_dump(vmm->mmu);
1049 		nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
1050 	}
1051 
1052 	vma = list_first_entry(&vmm->list, typeof(*vma), head);
1053 	list_del(&vma->head);
1054 	kfree(vma);
1055 	WARN_ON(!list_empty(&vmm->list));
1056 
1057 	if (vmm->nullp) {
1058 		dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
1059 				  vmm->nullp, vmm->null);
1060 	}
1061 
1062 	if (vmm->pd) {
1063 		nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
1064 		nvkm_vmm_pt_del(&vmm->pd);
1065 	}
1066 }
1067 
1068 static int
1069 nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size)
1070 {
1071 	struct nvkm_vma *vma;
1072 	if (!(vma = nvkm_vma_new(addr, size)))
1073 		return -ENOMEM;
1074 	vma->mapref = true;
1075 	vma->sparse = false;
1076 	vma->used = true;
1077 	nvkm_vmm_node_insert(vmm, vma);
1078 	list_add_tail(&vma->head, &vmm->list);
1079 	return 0;
1080 }
1081 
1082 static int
1083 nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1084 	      u32 pd_header, bool managed, u64 addr, u64 size,
1085 	      struct lock_class_key *key, const char *name,
1086 	      struct nvkm_vmm *vmm)
1087 {
1088 	static struct lock_class_key _key;
1089 	const struct nvkm_vmm_page *page = func->page;
1090 	const struct nvkm_vmm_desc *desc;
1091 	struct nvkm_vma *vma;
1092 	int levels, bits = 0, ret;
1093 
1094 	vmm->func = func;
1095 	vmm->mmu = mmu;
1096 	vmm->name = name;
1097 	vmm->debug = mmu->subdev.debug;
1098 	kref_init(&vmm->kref);
1099 
1100 	__mutex_init(&vmm->mutex.vmm, "&vmm->mutex.vmm", key ? key : &_key);
1101 	mutex_init(&vmm->mutex.ref);
1102 	mutex_init(&vmm->mutex.map);
1103 
1104 	/* Locate the smallest page size supported by the backend, it will
1105 	 * have the deepest nesting of page tables.
1106 	 */
1107 	while (page[1].shift)
1108 		page++;
1109 
1110 	/* Locate the structure that describes the layout of the top-level
1111 	 * page table, and determine the number of valid bits in a virtual
1112 	 * address.
1113 	 */
1114 	for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
1115 		bits += desc->bits;
1116 	bits += page->shift;
1117 	desc--;
1118 
1119 	if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
1120 		return -EINVAL;
1121 
1122 	/* Allocate top-level page table. */
1123 	vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
1124 	if (!vmm->pd)
1125 		return -ENOMEM;
1126 	vmm->pd->refs[0] = 1;
1127 	INIT_LIST_HEAD(&vmm->join);
1128 
1129 	/* ... and the GPU storage for it, except on Tesla-class GPUs that
1130 	 * have the PD embedded in the instance structure.
1131 	 */
1132 	if (desc->size) {
1133 		const u32 size = pd_header + desc->size * (1 << desc->bits);
1134 		vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
1135 		if (!vmm->pd->pt[0])
1136 			return -ENOMEM;
1137 	}
1138 
1139 	/* Initialise address-space MM. */
1140 	INIT_LIST_HEAD(&vmm->list);
1141 	vmm->free = RB_ROOT;
1142 	vmm->root = RB_ROOT;
1143 
1144 	if (managed) {
1145 		/* Address-space will be managed by the client for the most
1146 		 * part, except for a specified area where NVKM allocations
1147 		 * are allowed to be placed.
1148 		 */
1149 		vmm->start = 0;
1150 		vmm->limit = 1ULL << bits;
1151 		if (addr + size < addr || addr + size > vmm->limit)
1152 			return -EINVAL;
1153 
1154 		/* Client-managed area before the NVKM-managed area. */
1155 		if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr)))
1156 			return ret;
1157 
1158 		vmm->managed.p.addr = 0;
1159 		vmm->managed.p.size = addr;
1160 
1161 		/* NVKM-managed area. */
1162 		if (size) {
1163 			if (!(vma = nvkm_vma_new(addr, size)))
1164 				return -ENOMEM;
1165 			nvkm_vmm_free_insert(vmm, vma);
1166 			list_add_tail(&vma->head, &vmm->list);
1167 		}
1168 
1169 		/* Client-managed area after the NVKM-managed area. */
1170 		addr = addr + size;
1171 		size = vmm->limit - addr;
1172 		if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size)))
1173 			return ret;
1174 
1175 		vmm->managed.n.addr = addr;
1176 		vmm->managed.n.size = size;
1177 	} else {
1178 		/* Address-space fully managed by NVKM, requiring calls to
1179 		 * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space.
1180 		 */
1181 		vmm->start = addr;
1182 		vmm->limit = size ? (addr + size) : (1ULL << bits);
1183 		if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
1184 			return -EINVAL;
1185 
1186 		if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
1187 			return -ENOMEM;
1188 
1189 		nvkm_vmm_free_insert(vmm, vma);
1190 		list_add(&vma->head, &vmm->list);
1191 	}
1192 
1193 	return 0;
1194 }
1195 
1196 int
1197 nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1198 	      u32 hdr, bool managed, u64 addr, u64 size,
1199 	      struct lock_class_key *key, const char *name,
1200 	      struct nvkm_vmm **pvmm)
1201 {
1202 	if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
1203 		return -ENOMEM;
1204 	return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
1205 }
1206 
1207 static struct nvkm_vma *
1208 nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1209 			 u64 addr, u64 size, u8 page, bool map)
1210 {
1211 	struct nvkm_vma *prev = NULL;
1212 	struct nvkm_vma *next = NULL;
1213 
1214 	if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
1215 		if (prev->memory || prev->mapped != map)
1216 			prev = NULL;
1217 	}
1218 
1219 	if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
1220 		if (!next->part ||
1221 		    next->memory || next->mapped != map)
1222 			next = NULL;
1223 	}
1224 
1225 	if (prev || next)
1226 		return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
1227 	return nvkm_vmm_node_split(vmm, vma, addr, size);
1228 }
1229 
1230 int
1231 nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
1232 {
1233 	struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
1234 	struct nvkm_vma *next;
1235 	u64 limit = addr + size;
1236 	u64 start = addr;
1237 
1238 	if (!vma)
1239 		return -EINVAL;
1240 
1241 	do {
1242 		if (!vma->mapped || vma->memory)
1243 			continue;
1244 
1245 		size = min(limit - start, vma->size - (start - vma->addr));
1246 
1247 		nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
1248 					start, size, false, true);
1249 
1250 		next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
1251 		if (!WARN_ON(!next)) {
1252 			vma = next;
1253 			vma->refd = NVKM_VMA_PAGE_NONE;
1254 			vma->mapped = false;
1255 		}
1256 	} while ((vma = node(vma, next)) && (start = vma->addr) < limit);
1257 
1258 	return 0;
1259 }
1260 
1261 /*TODO:
1262  * - Avoid PT readback (for dma_unmap etc), this might end up being dealt
1263  *   with inside HMM, which would be a lot nicer for us to deal with.
1264  * - Support for systems without a 4KiB page size.
1265  */
1266 int
1267 nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
1268 {
1269 	const struct nvkm_vmm_page *page = vmm->func->page;
1270 	struct nvkm_vma *vma, *tmp;
1271 	u64 limit = addr + size;
1272 	u64 start = addr;
1273 	int pm = size >> shift;
1274 	int pi = 0;
1275 
1276 	/* Only support mapping where the page size of the incoming page
1277 	 * array matches a page size available for direct mapping.
1278 	 */
1279 	while (page->shift && (page->shift != shift ||
1280 	       page->desc->func->pfn == NULL))
1281 		page++;
1282 
1283 	if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
1284 			    !IS_ALIGNED(size, 1ULL << shift) ||
1285 	    addr + size < addr || addr + size > vmm->limit) {
1286 		VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
1287 			  shift, page->shift, addr, size);
1288 		return -EINVAL;
1289 	}
1290 
1291 	if (!(vma = nvkm_vmm_node_search(vmm, addr)))
1292 		return -ENOENT;
1293 
1294 	do {
1295 		bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
1296 		bool mapped = vma->mapped;
1297 		u64 size = limit - start;
1298 		u64 addr = start;
1299 		int pn, ret = 0;
1300 
1301 		/* Narrow the operation window to cover a single action (page
1302 		 * should be mapped or not) within a single VMA.
1303 		 */
1304 		for (pn = 0; pi + pn < pm; pn++) {
1305 			if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
1306 				break;
1307 		}
1308 		size = min_t(u64, size, pn << page->shift);
1309 		size = min_t(u64, size, vma->size + vma->addr - addr);
1310 
1311 		/* Reject any operation to unmanaged regions, and areas that
1312 		 * have nvkm_memory objects mapped in them already.
1313 		 */
1314 		if (!vma->mapref || vma->memory) {
1315 			ret = -EINVAL;
1316 			goto next;
1317 		}
1318 
1319 		/* In order to both properly refcount GPU page tables, and
1320 		 * prevent "normal" mappings and these direct mappings from
1321 		 * interfering with each other, we need to track contiguous
1322 		 * ranges that have been mapped with this interface.
1323 		 *
1324 		 * Here we attempt to either split an existing VMA so we're
1325 		 * able to flag the region as either unmapped/mapped, or to
1326 		 * merge with adjacent VMAs that are already compatible.
1327 		 *
1328 		 * If the region is already compatible, nothing is required.
1329 		 */
1330 		if (map != mapped) {
1331 			tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
1332 						       page -
1333 						       vmm->func->page, map);
1334 			if (WARN_ON(!tmp)) {
1335 				ret = -ENOMEM;
1336 				goto next;
1337 			}
1338 
1339 			if ((tmp->mapped = map))
1340 				tmp->refd = page - vmm->func->page;
1341 			else
1342 				tmp->refd = NVKM_VMA_PAGE_NONE;
1343 			vma = tmp;
1344 		}
1345 
1346 		/* Update HW page tables. */
1347 		if (map) {
1348 			struct nvkm_vmm_map args;
1349 			args.page = page;
1350 			args.pfn = &pfn[pi];
1351 
1352 			if (!mapped) {
1353 				ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
1354 							    size, &args, page->
1355 							    desc->func->pfn);
1356 			} else {
1357 				nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
1358 						  page->desc->func->pfn);
1359 			}
1360 		} else {
1361 			if (mapped) {
1362 				nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
1363 							false, true);
1364 			}
1365 		}
1366 
1367 next:
1368 		/* Iterate to next operation. */
1369 		if (vma->addr + vma->size == addr + size)
1370 			vma = node(vma, next);
1371 		start += size;
1372 
1373 		if (ret) {
1374 			/* Failure is signalled by clearing the valid bit on
1375 			 * any PFN that couldn't be modified as requested.
1376 			 */
1377 			while (size) {
1378 				pfn[pi++] = NVKM_VMM_PFN_NONE;
1379 				size -= 1 << page->shift;
1380 			}
1381 		} else {
1382 			pi += size >> page->shift;
1383 		}
1384 	} while (vma && start < limit);
1385 
1386 	return 0;
1387 }
1388 
1389 void
1390 nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1391 {
1392 	struct nvkm_vma *prev = NULL;
1393 	struct nvkm_vma *next;
1394 
1395 	nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1396 	nvkm_memory_unref(&vma->memory);
1397 	vma->mapped = false;
1398 
1399 	if (vma->part && (prev = node(vma, prev)) && prev->mapped)
1400 		prev = NULL;
1401 	if ((next = node(vma, next)) && (!next->part || next->mapped))
1402 		next = NULL;
1403 	nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
1404 }
1405 
1406 void
1407 nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
1408 {
1409 	const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
1410 
1411 	if (vma->mapref) {
1412 		nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1413 		vma->refd = NVKM_VMA_PAGE_NONE;
1414 	} else {
1415 		nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1416 	}
1417 
1418 	nvkm_vmm_unmap_region(vmm, vma);
1419 }
1420 
1421 void
1422 nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1423 {
1424 	if (vma->memory) {
1425 		mutex_lock(&vmm->mutex.vmm);
1426 		nvkm_vmm_unmap_locked(vmm, vma, false);
1427 		mutex_unlock(&vmm->mutex.vmm);
1428 	}
1429 }
1430 
1431 static int
1432 nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1433 		   void *argv, u32 argc, struct nvkm_vmm_map *map)
1434 {
1435 	switch (nvkm_memory_target(map->memory)) {
1436 	case NVKM_MEM_TARGET_VRAM:
1437 		if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
1438 			VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
1439 			return -EINVAL;
1440 		}
1441 		break;
1442 	case NVKM_MEM_TARGET_HOST:
1443 	case NVKM_MEM_TARGET_NCOH:
1444 		if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
1445 			VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
1446 			return -EINVAL;
1447 		}
1448 		break;
1449 	default:
1450 		WARN_ON(1);
1451 		return -ENOSYS;
1452 	}
1453 
1454 	if (!IS_ALIGNED(     vma->addr, 1ULL << map->page->shift) ||
1455 	    !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
1456 	    !IS_ALIGNED(   map->offset, 1ULL << map->page->shift) ||
1457 	    nvkm_memory_page(map->memory) < map->page->shift) {
1458 		VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
1459 		    vma->addr, (u64)vma->size, map->offset, map->page->shift,
1460 		    nvkm_memory_page(map->memory));
1461 		return -EINVAL;
1462 	}
1463 
1464 	return vmm->func->valid(vmm, argv, argc, map);
1465 }
1466 
1467 static int
1468 nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1469 		    void *argv, u32 argc, struct nvkm_vmm_map *map)
1470 {
1471 	for (map->page = vmm->func->page; map->page->shift; map->page++) {
1472 		VMM_DEBUG(vmm, "trying %d", map->page->shift);
1473 		if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
1474 			return 0;
1475 	}
1476 	return -EINVAL;
1477 }
1478 
1479 static int
1480 nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1481 		    void *argv, u32 argc, struct nvkm_vmm_map *map)
1482 {
1483 	nvkm_vmm_pte_func func;
1484 	int ret;
1485 
1486 	map->no_comp = vma->no_comp;
1487 
1488 	/* Make sure we won't overrun the end of the memory object. */
1489 	if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
1490 		VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
1491 			  nvkm_memory_size(map->memory),
1492 			  map->offset, (u64)vma->size);
1493 		return -EINVAL;
1494 	}
1495 
1496 	/* Check remaining arguments for validity. */
1497 	if (vma->page == NVKM_VMA_PAGE_NONE &&
1498 	    vma->refd == NVKM_VMA_PAGE_NONE) {
1499 		/* Find the largest page size we can perform the mapping at. */
1500 		const u32 debug = vmm->debug;
1501 		vmm->debug = 0;
1502 		ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1503 		vmm->debug = debug;
1504 		if (ret) {
1505 			VMM_DEBUG(vmm, "invalid at any page size");
1506 			nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1507 			return -EINVAL;
1508 		}
1509 	} else {
1510 		/* Page size of the VMA is already pre-determined. */
1511 		if (vma->refd != NVKM_VMA_PAGE_NONE)
1512 			map->page = &vmm->func->page[vma->refd];
1513 		else
1514 			map->page = &vmm->func->page[vma->page];
1515 
1516 		ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
1517 		if (ret) {
1518 			VMM_DEBUG(vmm, "invalid %d\n", ret);
1519 			return ret;
1520 		}
1521 	}
1522 
1523 	/* Deal with the 'offset' argument, and fetch the backend function. */
1524 	map->off = map->offset;
1525 	if (map->mem) {
1526 		for (; map->off; map->mem = map->mem->next) {
1527 			u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
1528 			if (size > map->off)
1529 				break;
1530 			map->off -= size;
1531 		}
1532 		func = map->page->desc->func->mem;
1533 	} else
1534 	if (map->sgl) {
1535 		for (; map->off; map->sgl = sg_next(map->sgl)) {
1536 			u64 size = sg_dma_len(map->sgl);
1537 			if (size > map->off)
1538 				break;
1539 			map->off -= size;
1540 		}
1541 		func = map->page->desc->func->sgl;
1542 	} else {
1543 		map->dma += map->offset >> PAGE_SHIFT;
1544 		map->off  = map->offset & PAGE_MASK;
1545 		func = map->page->desc->func->dma;
1546 	}
1547 
1548 	/* Perform the map. */
1549 	if (vma->refd == NVKM_VMA_PAGE_NONE) {
1550 		ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
1551 		if (ret)
1552 			return ret;
1553 
1554 		vma->refd = map->page - vmm->func->page;
1555 	} else {
1556 		nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
1557 	}
1558 
1559 	nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1560 	nvkm_memory_unref(&vma->memory);
1561 	vma->memory = nvkm_memory_ref(map->memory);
1562 	vma->mapped = true;
1563 	vma->tags = map->tags;
1564 	return 0;
1565 }
1566 
1567 int
1568 nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
1569 	     struct nvkm_vmm_map *map)
1570 {
1571 	int ret;
1572 
1573 	if (nvkm_vmm_in_managed_range(vmm, vma->addr, vma->size) &&
1574 	    vmm->managed.raw)
1575 		return nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1576 
1577 	mutex_lock(&vmm->mutex.vmm);
1578 	ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1579 	vma->busy = false;
1580 	mutex_unlock(&vmm->mutex.vmm);
1581 	return ret;
1582 }
1583 
1584 static void
1585 nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1586 {
1587 	struct nvkm_vma *prev, *next;
1588 
1589 	if ((prev = node(vma, prev)) && !prev->used) {
1590 		vma->addr  = prev->addr;
1591 		vma->size += prev->size;
1592 		nvkm_vmm_free_delete(vmm, prev);
1593 	}
1594 
1595 	if ((next = node(vma, next)) && !next->used) {
1596 		vma->size += next->size;
1597 		nvkm_vmm_free_delete(vmm, next);
1598 	}
1599 
1600 	nvkm_vmm_free_insert(vmm, vma);
1601 }
1602 
1603 void
1604 nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1605 {
1606 	const struct nvkm_vmm_page *page = vmm->func->page;
1607 	struct nvkm_vma *next = vma;
1608 
1609 	BUG_ON(vma->part);
1610 
1611 	if (vma->mapref || !vma->sparse) {
1612 		do {
1613 			const bool mem = next->memory != NULL;
1614 			const bool map = next->mapped;
1615 			const u8  refd = next->refd;
1616 			const u64 addr = next->addr;
1617 			u64 size = next->size;
1618 
1619 			/* Merge regions that are in the same state. */
1620 			while ((next = node(next, next)) && next->part &&
1621 			       (next->mapped == map) &&
1622 			       (next->memory != NULL) == mem &&
1623 			       (next->refd == refd))
1624 				size += next->size;
1625 
1626 			if (map) {
1627 				/* Region(s) are mapped, merge the unmap
1628 				 * and dereference into a single walk of
1629 				 * the page tree.
1630 				 */
1631 				nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
1632 							size, vma->sparse,
1633 							!mem);
1634 			} else
1635 			if (refd != NVKM_VMA_PAGE_NONE) {
1636 				/* Drop allocation-time PTE references. */
1637 				nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1638 			}
1639 		} while (next && next->part);
1640 	}
1641 
1642 	/* Merge any mapped regions that were split from the initial
1643 	 * address-space allocation back into the allocated VMA, and
1644 	 * release memory/compression resources.
1645 	 */
1646 	next = vma;
1647 	do {
1648 		if (next->mapped)
1649 			nvkm_vmm_unmap_region(vmm, next);
1650 	} while ((next = node(vma, next)) && next->part);
1651 
1652 	if (vma->sparse && !vma->mapref) {
1653 		/* Sparse region that was allocated with a fixed page size,
1654 		 * meaning all relevant PTEs were referenced once when the
1655 		 * region was allocated, and remained that way, regardless
1656 		 * of whether memory was mapped into it afterwards.
1657 		 *
1658 		 * The process of unmapping, unsparsing, and dereferencing
1659 		 * PTEs can be done in a single page tree walk.
1660 		 */
1661 		nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
1662 	} else
1663 	if (vma->sparse) {
1664 		/* Sparse region that wasn't allocated with a fixed page size,
1665 		 * PTE references were taken both at allocation time (to make
1666 		 * the GPU see the region as sparse), and when mapping memory
1667 		 * into the region.
1668 		 *
1669 		 * The latter was handled above, and the remaining references
1670 		 * are dealt with here.
1671 		 */
1672 		nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
1673 	}
1674 
1675 	/* Remove VMA from the list of allocated nodes. */
1676 	nvkm_vmm_node_remove(vmm, vma);
1677 
1678 	/* Merge VMA back into the free list. */
1679 	vma->page = NVKM_VMA_PAGE_NONE;
1680 	vma->refd = NVKM_VMA_PAGE_NONE;
1681 	vma->used = false;
1682 	nvkm_vmm_put_region(vmm, vma);
1683 }
1684 
1685 void
1686 nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
1687 {
1688 	struct nvkm_vma *vma = *pvma;
1689 	if (vma) {
1690 		mutex_lock(&vmm->mutex.vmm);
1691 		nvkm_vmm_put_locked(vmm, vma);
1692 		mutex_unlock(&vmm->mutex.vmm);
1693 		*pvma = NULL;
1694 	}
1695 }
1696 
1697 int
1698 nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
1699 		    u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
1700 {
1701 	const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
1702 	struct rb_node *node = NULL, *temp;
1703 	struct nvkm_vma *vma = NULL, *tmp;
1704 	u64 addr, tail;
1705 	int ret;
1706 
1707 	VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
1708 		       "shift: %d align: %d size: %016llx",
1709 		  getref, mapref, sparse, shift, align, size);
1710 
1711 	/* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
1712 	if (unlikely(!size || (!getref && !mapref && sparse))) {
1713 		VMM_DEBUG(vmm, "args %016llx %d %d %d",
1714 			  size, getref, mapref, sparse);
1715 		return -EINVAL;
1716 	}
1717 
1718 	/* Tesla-class GPUs can only select page size per-PDE, which means
1719 	 * we're required to know the mapping granularity up-front to find
1720 	 * a suitable region of address-space.
1721 	 *
1722 	 * The same goes if we're requesting up-front allocation of PTES.
1723 	 */
1724 	if (unlikely((getref || vmm->func->page_block) && !shift)) {
1725 		VMM_DEBUG(vmm, "page size required: %d %016llx",
1726 			  getref, vmm->func->page_block);
1727 		return -EINVAL;
1728 	}
1729 
1730 	/* If a specific page size was requested, determine its index and
1731 	 * make sure the requested size is a multiple of the page size.
1732 	 */
1733 	if (shift) {
1734 		for (page = vmm->func->page; page->shift; page++) {
1735 			if (shift == page->shift)
1736 				break;
1737 		}
1738 
1739 		if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
1740 			VMM_DEBUG(vmm, "page %d %016llx", shift, size);
1741 			return -EINVAL;
1742 		}
1743 		align = max_t(u8, align, shift);
1744 	} else {
1745 		align = max_t(u8, align, 12);
1746 	}
1747 
1748 	/* Locate smallest block that can possibly satisfy the allocation. */
1749 	temp = vmm->free.rb_node;
1750 	while (temp) {
1751 		struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
1752 		if (this->size < size) {
1753 			temp = temp->rb_right;
1754 		} else {
1755 			node = temp;
1756 			temp = temp->rb_left;
1757 		}
1758 	}
1759 
1760 	if (unlikely(!node))
1761 		return -ENOSPC;
1762 
1763 	/* Take into account alignment restrictions, trying larger blocks
1764 	 * in turn until we find a suitable free block.
1765 	 */
1766 	do {
1767 		struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
1768 		struct nvkm_vma *prev = node(this, prev);
1769 		struct nvkm_vma *next = node(this, next);
1770 		const int p = page - vmm->func->page;
1771 
1772 		addr = this->addr;
1773 		if (vmm->func->page_block && prev && prev->page != p)
1774 			addr = ALIGN(addr, vmm->func->page_block);
1775 		addr = ALIGN(addr, 1ULL << align);
1776 
1777 		tail = this->addr + this->size;
1778 		if (vmm->func->page_block && next && next->page != p)
1779 			tail = ALIGN_DOWN(tail, vmm->func->page_block);
1780 
1781 		if (addr <= tail && tail - addr >= size) {
1782 			nvkm_vmm_free_remove(vmm, this);
1783 			vma = this;
1784 			break;
1785 		}
1786 	} while ((node = rb_next(node)));
1787 
1788 	if (unlikely(!vma))
1789 		return -ENOSPC;
1790 
1791 	/* If the VMA we found isn't already exactly the requested size,
1792 	 * it needs to be split, and the remaining free blocks returned.
1793 	 */
1794 	if (addr != vma->addr) {
1795 		if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
1796 			nvkm_vmm_put_region(vmm, vma);
1797 			return -ENOMEM;
1798 		}
1799 		nvkm_vmm_free_insert(vmm, vma);
1800 		vma = tmp;
1801 	}
1802 
1803 	if (size != vma->size) {
1804 		if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
1805 			nvkm_vmm_put_region(vmm, vma);
1806 			return -ENOMEM;
1807 		}
1808 		nvkm_vmm_free_insert(vmm, tmp);
1809 	}
1810 
1811 	/* Pre-allocate page tables and/or setup sparse mappings. */
1812 	if (sparse && getref)
1813 		ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
1814 	else if (sparse)
1815 		ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
1816 	else if (getref)
1817 		ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
1818 	else
1819 		ret = 0;
1820 	if (ret) {
1821 		nvkm_vmm_put_region(vmm, vma);
1822 		return ret;
1823 	}
1824 
1825 	vma->mapref = mapref && !getref;
1826 	vma->sparse = sparse;
1827 	vma->page = page - vmm->func->page;
1828 	vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
1829 	vma->used = true;
1830 	nvkm_vmm_node_insert(vmm, vma);
1831 	*pvma = vma;
1832 	return 0;
1833 }
1834 
1835 int
1836 nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
1837 {
1838 	int ret;
1839 	mutex_lock(&vmm->mutex.vmm);
1840 	ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
1841 	mutex_unlock(&vmm->mutex.vmm);
1842 	return ret;
1843 }
1844 
1845 void
1846 nvkm_vmm_raw_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size,
1847 		   bool sparse, u8 refd)
1848 {
1849 	const struct nvkm_vmm_page *page = &vmm->func->page[refd];
1850 
1851 	nvkm_vmm_ptes_unmap(vmm, page, addr, size, sparse, false);
1852 }
1853 
1854 void
1855 nvkm_vmm_raw_put(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd)
1856 {
1857 	const struct nvkm_vmm_page *page = vmm->func->page;
1858 
1859 	nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1860 }
1861 
1862 int
1863 nvkm_vmm_raw_get(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd)
1864 {
1865 	const struct nvkm_vmm_page *page = vmm->func->page;
1866 
1867 	if (unlikely(!size))
1868 		return -EINVAL;
1869 
1870 	return nvkm_vmm_ptes_get(vmm, &page[refd], addr, size);
1871 }
1872 
1873 int
1874 nvkm_vmm_raw_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
1875 {
1876 	int ret;
1877 
1878 	mutex_lock(&vmm->mutex.ref);
1879 	ret = nvkm_vmm_ptes_sparse(vmm, addr, size, ref);
1880 	mutex_unlock(&vmm->mutex.ref);
1881 
1882 	return ret;
1883 }
1884 
1885 void
1886 nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1887 {
1888 	if (inst && vmm && vmm->func->part) {
1889 		mutex_lock(&vmm->mutex.vmm);
1890 		vmm->func->part(vmm, inst);
1891 		mutex_unlock(&vmm->mutex.vmm);
1892 	}
1893 }
1894 
1895 int
1896 nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1897 {
1898 	int ret = 0;
1899 	if (vmm->func->join) {
1900 		mutex_lock(&vmm->mutex.vmm);
1901 		ret = vmm->func->join(vmm, inst);
1902 		mutex_unlock(&vmm->mutex.vmm);
1903 	}
1904 	return ret;
1905 }
1906 
1907 static bool
1908 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
1909 {
1910 	const struct nvkm_vmm_desc *desc = it->desc;
1911 	const int type = desc->type == SPT;
1912 	nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
1913 	return false;
1914 }
1915 
1916 int
1917 nvkm_vmm_boot(struct nvkm_vmm *vmm)
1918 {
1919 	const struct nvkm_vmm_page *page = vmm->func->page;
1920 	const u64 limit = vmm->limit - vmm->start;
1921 	int ret;
1922 
1923 	while (page[1].shift)
1924 		page++;
1925 
1926 	ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
1927 	if (ret)
1928 		return ret;
1929 
1930 	nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
1931 		      nvkm_vmm_boot_ptes, NULL, NULL, NULL);
1932 	vmm->bootstrapped = true;
1933 	return 0;
1934 }
1935 
1936 static void
1937 nvkm_vmm_del(struct kref *kref)
1938 {
1939 	struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
1940 	nvkm_vmm_dtor(vmm);
1941 	kfree(vmm);
1942 }
1943 
1944 void
1945 nvkm_vmm_unref(struct nvkm_vmm **pvmm)
1946 {
1947 	struct nvkm_vmm *vmm = *pvmm;
1948 	if (vmm) {
1949 		kref_put(&vmm->kref, nvkm_vmm_del);
1950 		*pvmm = NULL;
1951 	}
1952 }
1953 
1954 struct nvkm_vmm *
1955 nvkm_vmm_ref(struct nvkm_vmm *vmm)
1956 {
1957 	if (vmm)
1958 		kref_get(&vmm->kref);
1959 	return vmm;
1960 }
1961 
1962 int
1963 nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
1964 	     u32 argc, struct lock_class_key *key, const char *name,
1965 	     struct nvkm_vmm **pvmm)
1966 {
1967 	struct nvkm_mmu *mmu = device->mmu;
1968 	struct nvkm_vmm *vmm = NULL;
1969 	int ret;
1970 	ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc,
1971 				  key, name, &vmm);
1972 	if (ret)
1973 		nvkm_vmm_unref(&vmm);
1974 	*pvmm = vmm;
1975 	return ret;
1976 }
1977