1806a7335SBen Skeggs /*
2806a7335SBen Skeggs * Copyright 2017 Red Hat Inc.
3806a7335SBen Skeggs *
4806a7335SBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a
5806a7335SBen Skeggs * copy of this software and associated documentation files (the "Software"),
6806a7335SBen Skeggs * to deal in the Software without restriction, including without limitation
7806a7335SBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8806a7335SBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the
9806a7335SBen Skeggs * Software is furnished to do so, subject to the following conditions:
10806a7335SBen Skeggs *
11806a7335SBen Skeggs * The above copyright notice and this permission notice shall be included in
12806a7335SBen Skeggs * all copies or substantial portions of the Software.
13806a7335SBen Skeggs *
14806a7335SBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15806a7335SBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16806a7335SBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17806a7335SBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18806a7335SBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19806a7335SBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20806a7335SBen Skeggs * OTHER DEALINGS IN THE SOFTWARE.
21806a7335SBen Skeggs */
22806a7335SBen Skeggs #define NVKM_VMM_LEVELS_MAX 5
23806a7335SBen Skeggs #include "vmm.h"
24806a7335SBen Skeggs
25f9463a4bSBen Skeggs #include <subdev/fb.h>
26f9463a4bSBen Skeggs
27806a7335SBen Skeggs static void
nvkm_vmm_pt_del(struct nvkm_vmm_pt ** ppgt)28806a7335SBen Skeggs nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
29806a7335SBen Skeggs {
30806a7335SBen Skeggs struct nvkm_vmm_pt *pgt = *ppgt;
31806a7335SBen Skeggs if (pgt) {
32806a7335SBen Skeggs kvfree(pgt->pde);
33806a7335SBen Skeggs kfree(pgt);
34806a7335SBen Skeggs *ppgt = NULL;
35806a7335SBen Skeggs }
36806a7335SBen Skeggs }
37806a7335SBen Skeggs
38806a7335SBen Skeggs
39806a7335SBen Skeggs static struct nvkm_vmm_pt *
nvkm_vmm_pt_new(const struct nvkm_vmm_desc * desc,bool sparse,const struct nvkm_vmm_page * page)40806a7335SBen Skeggs nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
41806a7335SBen Skeggs const struct nvkm_vmm_page *page)
42806a7335SBen Skeggs {
43806a7335SBen Skeggs const u32 pten = 1 << desc->bits;
44806a7335SBen Skeggs struct nvkm_vmm_pt *pgt;
45806a7335SBen Skeggs u32 lpte = 0;
46806a7335SBen Skeggs
47806a7335SBen Skeggs if (desc->type > PGT) {
48806a7335SBen Skeggs if (desc->type == SPT) {
49806a7335SBen Skeggs const struct nvkm_vmm_desc *pair = page[-1].desc;
50806a7335SBen Skeggs lpte = pten >> (desc->bits - pair->bits);
51806a7335SBen Skeggs } else {
52806a7335SBen Skeggs lpte = pten;
53806a7335SBen Skeggs }
54806a7335SBen Skeggs }
55806a7335SBen Skeggs
56806a7335SBen Skeggs if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
57806a7335SBen Skeggs return NULL;
58806a7335SBen Skeggs pgt->page = page ? page->shift : 0;
59806a7335SBen Skeggs pgt->sparse = sparse;
60806a7335SBen Skeggs
61806a7335SBen Skeggs if (desc->type == PGD) {
62778e1cddSKees Cook pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
63806a7335SBen Skeggs if (!pgt->pde) {
64806a7335SBen Skeggs kfree(pgt);
65806a7335SBen Skeggs return NULL;
66806a7335SBen Skeggs }
67806a7335SBen Skeggs }
68806a7335SBen Skeggs
69806a7335SBen Skeggs return pgt;
70806a7335SBen Skeggs }
71806a7335SBen Skeggs
72eb813999SBen Skeggs struct nvkm_vmm_iter {
73eb813999SBen Skeggs const struct nvkm_vmm_page *page;
74eb813999SBen Skeggs const struct nvkm_vmm_desc *desc;
75eb813999SBen Skeggs struct nvkm_vmm *vmm;
76eb813999SBen Skeggs u64 cnt;
77eb813999SBen Skeggs u16 max, lvl;
78eb813999SBen Skeggs u32 pte[NVKM_VMM_LEVELS_MAX];
79eb813999SBen Skeggs struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
80eb813999SBen Skeggs int flush;
81eb813999SBen Skeggs };
82eb813999SBen Skeggs
83eb813999SBen Skeggs #ifdef CONFIG_NOUVEAU_DEBUG_MMU
84eb813999SBen Skeggs static const char *
nvkm_vmm_desc_type(const struct nvkm_vmm_desc * desc)85eb813999SBen Skeggs nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
86eb813999SBen Skeggs {
87eb813999SBen Skeggs switch (desc->type) {
88eb813999SBen Skeggs case PGD: return "PGD";
89eb813999SBen Skeggs case PGT: return "PGT";
90eb813999SBen Skeggs case SPT: return "SPT";
91eb813999SBen Skeggs case LPT: return "LPT";
92eb813999SBen Skeggs default:
93eb813999SBen Skeggs return "UNKNOWN";
94eb813999SBen Skeggs }
95eb813999SBen Skeggs }
96eb813999SBen Skeggs
97eb813999SBen Skeggs static void
nvkm_vmm_trace(struct nvkm_vmm_iter * it,char * buf)98eb813999SBen Skeggs nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
99eb813999SBen Skeggs {
100eb813999SBen Skeggs int lvl;
101eb813999SBen Skeggs for (lvl = it->max; lvl >= 0; lvl--) {
102eb813999SBen Skeggs if (lvl >= it->lvl)
103eb813999SBen Skeggs buf += sprintf(buf, "%05x:", it->pte[lvl]);
104eb813999SBen Skeggs else
105eb813999SBen Skeggs buf += sprintf(buf, "xxxxx:");
106eb813999SBen Skeggs }
107eb813999SBen Skeggs }
108eb813999SBen Skeggs
109eb813999SBen Skeggs #define TRA(i,f,a...) do { \
110eb813999SBen Skeggs char _buf[NVKM_VMM_LEVELS_MAX * 7]; \
111eb813999SBen Skeggs struct nvkm_vmm_iter *_it = (i); \
112eb813999SBen Skeggs nvkm_vmm_trace(_it, _buf); \
113eb813999SBen Skeggs VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \
114eb813999SBen Skeggs } while(0)
115eb813999SBen Skeggs #else
116eb813999SBen Skeggs #define TRA(i,f,a...)
117eb813999SBen Skeggs #endif
118eb813999SBen Skeggs
119eb813999SBen Skeggs static inline void
nvkm_vmm_flush_mark(struct nvkm_vmm_iter * it)120eb813999SBen Skeggs nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
121eb813999SBen Skeggs {
122eb813999SBen Skeggs it->flush = min(it->flush, it->max - it->lvl);
123eb813999SBen Skeggs }
124eb813999SBen Skeggs
125eb813999SBen Skeggs static inline void
nvkm_vmm_flush(struct nvkm_vmm_iter * it)126eb813999SBen Skeggs nvkm_vmm_flush(struct nvkm_vmm_iter *it)
127eb813999SBen Skeggs {
128eb813999SBen Skeggs if (it->flush != NVKM_VMM_LEVELS_MAX) {
129eb813999SBen Skeggs if (it->vmm->func->flush) {
130eb813999SBen Skeggs TRA(it, "flush: %d", it->flush);
131eb813999SBen Skeggs it->vmm->func->flush(it->vmm, it->flush);
132eb813999SBen Skeggs }
133eb813999SBen Skeggs it->flush = NVKM_VMM_LEVELS_MAX;
134eb813999SBen Skeggs }
135eb813999SBen Skeggs }
136eb813999SBen Skeggs
137eb813999SBen Skeggs static void
nvkm_vmm_unref_pdes(struct nvkm_vmm_iter * it)138eb813999SBen Skeggs nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
139eb813999SBen Skeggs {
140eb813999SBen Skeggs const struct nvkm_vmm_desc *desc = it->desc;
141eb813999SBen Skeggs const int type = desc[it->lvl].type == SPT;
142eb813999SBen Skeggs struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
143eb813999SBen Skeggs struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
144eb813999SBen Skeggs struct nvkm_mmu_pt *pt = pgt->pt[type];
145eb813999SBen Skeggs struct nvkm_vmm *vmm = it->vmm;
146eb813999SBen Skeggs u32 pdei = it->pte[it->lvl + 1];
147eb813999SBen Skeggs
148eb813999SBen Skeggs /* Recurse up the tree, unreferencing/destroying unneeded PDs. */
149eb813999SBen Skeggs it->lvl++;
150eb813999SBen Skeggs if (--pgd->refs[0]) {
151eb813999SBen Skeggs const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
152eb813999SBen Skeggs /* PD has other valid PDEs, so we need a proper update. */
153eb813999SBen Skeggs TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
154eb813999SBen Skeggs pgt->pt[type] = NULL;
155eb813999SBen Skeggs if (!pgt->refs[!type]) {
156eb813999SBen Skeggs /* PDE no longer required. */
157eb813999SBen Skeggs if (pgd->pt[0]) {
158eb813999SBen Skeggs if (pgt->sparse) {
159eb813999SBen Skeggs func->sparse(vmm, pgd->pt[0], pdei, 1);
160eb813999SBen Skeggs pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
161eb813999SBen Skeggs } else {
162eb813999SBen Skeggs func->unmap(vmm, pgd->pt[0], pdei, 1);
163eb813999SBen Skeggs pgd->pde[pdei] = NULL;
164eb813999SBen Skeggs }
165eb813999SBen Skeggs } else {
166eb813999SBen Skeggs /* Special handling for Tesla-class GPUs,
167eb813999SBen Skeggs * where there's no central PD, but each
168eb813999SBen Skeggs * instance has its own embedded PD.
169eb813999SBen Skeggs */
170eb813999SBen Skeggs func->pde(vmm, pgd, pdei);
171eb813999SBen Skeggs pgd->pde[pdei] = NULL;
172eb813999SBen Skeggs }
173eb813999SBen Skeggs } else {
174eb813999SBen Skeggs /* PDE was pointing at dual-PTs and we're removing
175eb813999SBen Skeggs * one of them, leaving the other in place.
176eb813999SBen Skeggs */
177eb813999SBen Skeggs func->pde(vmm, pgd, pdei);
178eb813999SBen Skeggs }
179eb813999SBen Skeggs
180eb813999SBen Skeggs /* GPU may have cached the PTs, flush before freeing. */
181eb813999SBen Skeggs nvkm_vmm_flush_mark(it);
182eb813999SBen Skeggs nvkm_vmm_flush(it);
183eb813999SBen Skeggs } else {
184eb813999SBen Skeggs /* PD has no valid PDEs left, so we can just destroy it. */
185eb813999SBen Skeggs nvkm_vmm_unref_pdes(it);
186eb813999SBen Skeggs }
187eb813999SBen Skeggs
188eb813999SBen Skeggs /* Destroy PD/PT. */
189eb813999SBen Skeggs TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
190eb813999SBen Skeggs nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
191eb813999SBen Skeggs if (!pgt->refs[!type])
192eb813999SBen Skeggs nvkm_vmm_pt_del(&pgt);
193eb813999SBen Skeggs it->lvl--;
194eb813999SBen Skeggs }
195eb813999SBen Skeggs
196eb813999SBen Skeggs static void
nvkm_vmm_unref_sptes(struct nvkm_vmm_iter * it,struct nvkm_vmm_pt * pgt,const struct nvkm_vmm_desc * desc,u32 ptei,u32 ptes)197eb813999SBen Skeggs nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
198eb813999SBen Skeggs const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
199eb813999SBen Skeggs {
200eb813999SBen Skeggs const struct nvkm_vmm_desc *pair = it->page[-1].desc;
201eb813999SBen Skeggs const u32 sptb = desc->bits - pair->bits;
202eb813999SBen Skeggs const u32 sptn = 1 << sptb;
203eb813999SBen Skeggs struct nvkm_vmm *vmm = it->vmm;
204eb813999SBen Skeggs u32 spti = ptei & (sptn - 1), lpti, pteb;
205eb813999SBen Skeggs
206eb813999SBen Skeggs /* Determine how many SPTEs are being touched under each LPTE,
207eb813999SBen Skeggs * and drop reference counts.
208eb813999SBen Skeggs */
209eb813999SBen Skeggs for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
210eb813999SBen Skeggs const u32 pten = min(sptn - spti, ptes);
211eb813999SBen Skeggs pgt->pte[lpti] -= pten;
212eb813999SBen Skeggs ptes -= pten;
213eb813999SBen Skeggs }
214eb813999SBen Skeggs
215eb813999SBen Skeggs /* We're done here if there's no corresponding LPT. */
216eb813999SBen Skeggs if (!pgt->refs[0])
217eb813999SBen Skeggs return;
218eb813999SBen Skeggs
219eb813999SBen Skeggs for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
220eb813999SBen Skeggs /* Skip over any LPTEs that still have valid SPTEs. */
221eb813999SBen Skeggs if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
222eb813999SBen Skeggs for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
223eb813999SBen Skeggs if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
224eb813999SBen Skeggs break;
225eb813999SBen Skeggs }
226eb813999SBen Skeggs continue;
227eb813999SBen Skeggs }
228eb813999SBen Skeggs
229eb813999SBen Skeggs /* As there's no more non-UNMAPPED SPTEs left in the range
230eb813999SBen Skeggs * covered by a number of LPTEs, the LPTEs once again take
231eb813999SBen Skeggs * control over their address range.
232eb813999SBen Skeggs *
233eb813999SBen Skeggs * Determine how many LPTEs need to transition state.
234eb813999SBen Skeggs */
235eb813999SBen Skeggs pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
236eb813999SBen Skeggs for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
237eb813999SBen Skeggs if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
238eb813999SBen Skeggs break;
239eb813999SBen Skeggs pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
240eb813999SBen Skeggs }
241eb813999SBen Skeggs
242eb813999SBen Skeggs if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
243eb813999SBen Skeggs TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
244eb813999SBen Skeggs pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
245eb813999SBen Skeggs } else
246eb813999SBen Skeggs if (pair->func->invalid) {
247eb813999SBen Skeggs /* If the MMU supports it, restore the LPTE to the
248eb813999SBen Skeggs * INVALID state to tell the MMU there is no point
249eb813999SBen Skeggs * trying to fetch the corresponding SPTEs.
250eb813999SBen Skeggs */
251eb813999SBen Skeggs TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
252eb813999SBen Skeggs pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
253eb813999SBen Skeggs }
254eb813999SBen Skeggs }
255eb813999SBen Skeggs }
256eb813999SBen Skeggs
257eb813999SBen Skeggs static bool
nvkm_vmm_unref_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)258a5ff307fSBen Skeggs nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
259eb813999SBen Skeggs {
260eb813999SBen Skeggs const struct nvkm_vmm_desc *desc = it->desc;
261eb813999SBen Skeggs const int type = desc->type == SPT;
262eb813999SBen Skeggs struct nvkm_vmm_pt *pgt = it->pt[0];
263a5ff307fSBen Skeggs bool dma;
264a5ff307fSBen Skeggs
265a5ff307fSBen Skeggs if (pfn) {
266a5ff307fSBen Skeggs /* Need to clear PTE valid bits before we dma_unmap_page(). */
267a5ff307fSBen Skeggs dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
268a5ff307fSBen Skeggs if (dma) {
269a5ff307fSBen Skeggs /* GPU may have cached the PT, flush before unmap. */
270a5ff307fSBen Skeggs nvkm_vmm_flush_mark(it);
271a5ff307fSBen Skeggs nvkm_vmm_flush(it);
272a5ff307fSBen Skeggs desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
273a5ff307fSBen Skeggs }
274a5ff307fSBen Skeggs }
275eb813999SBen Skeggs
276eb813999SBen Skeggs /* Drop PTE references. */
277eb813999SBen Skeggs pgt->refs[type] -= ptes;
278eb813999SBen Skeggs
279eb813999SBen Skeggs /* Dual-PTs need special handling, unless PDE becoming invalid. */
280eb813999SBen Skeggs if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
281eb813999SBen Skeggs nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
282eb813999SBen Skeggs
283e4f21d14SJulia Lawall /* PT no longer needed? Destroy it. */
284eb813999SBen Skeggs if (!pgt->refs[type]) {
285eb813999SBen Skeggs it->lvl++;
286eb813999SBen Skeggs TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
287eb813999SBen Skeggs it->lvl--;
288eb813999SBen Skeggs nvkm_vmm_unref_pdes(it);
289eb813999SBen Skeggs return false; /* PTE writes for unmap() not necessary. */
290eb813999SBen Skeggs }
291eb813999SBen Skeggs
292eb813999SBen Skeggs return true;
293eb813999SBen Skeggs }
294eb813999SBen Skeggs
295eb813999SBen Skeggs static void
nvkm_vmm_ref_sptes(struct nvkm_vmm_iter * it,struct nvkm_vmm_pt * pgt,const struct nvkm_vmm_desc * desc,u32 ptei,u32 ptes)296eb813999SBen Skeggs nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
297eb813999SBen Skeggs const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
298eb813999SBen Skeggs {
299eb813999SBen Skeggs const struct nvkm_vmm_desc *pair = it->page[-1].desc;
300eb813999SBen Skeggs const u32 sptb = desc->bits - pair->bits;
301eb813999SBen Skeggs const u32 sptn = 1 << sptb;
302eb813999SBen Skeggs struct nvkm_vmm *vmm = it->vmm;
303eb813999SBen Skeggs u32 spti = ptei & (sptn - 1), lpti, pteb;
304eb813999SBen Skeggs
305eb813999SBen Skeggs /* Determine how many SPTEs are being touched under each LPTE,
306eb813999SBen Skeggs * and increase reference counts.
307eb813999SBen Skeggs */
308eb813999SBen Skeggs for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
309eb813999SBen Skeggs const u32 pten = min(sptn - spti, ptes);
310eb813999SBen Skeggs pgt->pte[lpti] += pten;
311eb813999SBen Skeggs ptes -= pten;
312eb813999SBen Skeggs }
313eb813999SBen Skeggs
314eb813999SBen Skeggs /* We're done here if there's no corresponding LPT. */
315eb813999SBen Skeggs if (!pgt->refs[0])
316eb813999SBen Skeggs return;
317eb813999SBen Skeggs
318eb813999SBen Skeggs for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
319eb813999SBen Skeggs /* Skip over any LPTEs that already have valid SPTEs. */
320eb813999SBen Skeggs if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
321eb813999SBen Skeggs for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
322eb813999SBen Skeggs if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
323eb813999SBen Skeggs break;
324eb813999SBen Skeggs }
325eb813999SBen Skeggs continue;
326eb813999SBen Skeggs }
327eb813999SBen Skeggs
328eb813999SBen Skeggs /* As there are now non-UNMAPPED SPTEs in the range covered
329eb813999SBen Skeggs * by a number of LPTEs, we need to transfer control of the
330eb813999SBen Skeggs * address range to the SPTEs.
331eb813999SBen Skeggs *
332eb813999SBen Skeggs * Determine how many LPTEs need to transition state.
333eb813999SBen Skeggs */
334eb813999SBen Skeggs pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
335eb813999SBen Skeggs for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
336eb813999SBen Skeggs if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
337eb813999SBen Skeggs break;
338eb813999SBen Skeggs pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
339eb813999SBen Skeggs }
340eb813999SBen Skeggs
341eb813999SBen Skeggs if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
342eb813999SBen Skeggs const u32 spti = pteb * sptn;
343eb813999SBen Skeggs const u32 sptc = ptes * sptn;
344eb813999SBen Skeggs /* The entire LPTE is marked as sparse, we need
345eb813999SBen Skeggs * to make sure that the SPTEs are too.
346eb813999SBen Skeggs */
347eb813999SBen Skeggs TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
348eb813999SBen Skeggs desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
349eb813999SBen Skeggs /* Sparse LPTEs prevent SPTEs from being accessed. */
350eb813999SBen Skeggs TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
351eb813999SBen Skeggs pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
352eb813999SBen Skeggs } else
353eb813999SBen Skeggs if (pair->func->invalid) {
354eb813999SBen Skeggs /* MMU supports blocking SPTEs by marking an LPTE
355eb813999SBen Skeggs * as INVALID. We need to reverse that here.
356eb813999SBen Skeggs */
357eb813999SBen Skeggs TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
358eb813999SBen Skeggs pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
359eb813999SBen Skeggs }
360eb813999SBen Skeggs }
361eb813999SBen Skeggs }
362eb813999SBen Skeggs
363eb813999SBen Skeggs static bool
nvkm_vmm_ref_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)364a5ff307fSBen Skeggs nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
365eb813999SBen Skeggs {
366eb813999SBen Skeggs const struct nvkm_vmm_desc *desc = it->desc;
367eb813999SBen Skeggs const int type = desc->type == SPT;
368eb813999SBen Skeggs struct nvkm_vmm_pt *pgt = it->pt[0];
369eb813999SBen Skeggs
370eb813999SBen Skeggs /* Take PTE references. */
371eb813999SBen Skeggs pgt->refs[type] += ptes;
372eb813999SBen Skeggs
373eb813999SBen Skeggs /* Dual-PTs need special handling. */
374eb813999SBen Skeggs if (desc->type == SPT)
375eb813999SBen Skeggs nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
376eb813999SBen Skeggs
377eb813999SBen Skeggs return true;
378eb813999SBen Skeggs }
379eb813999SBen Skeggs
380eb813999SBen Skeggs static void
nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc * desc,struct nvkm_vmm_pt * pgt,u32 ptei,u32 ptes)381eb813999SBen Skeggs nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
382eb813999SBen Skeggs struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
383eb813999SBen Skeggs {
384eb813999SBen Skeggs if (desc->type == PGD) {
385eb813999SBen Skeggs while (ptes--)
386eb813999SBen Skeggs pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
387eb813999SBen Skeggs } else
388eb813999SBen Skeggs if (desc->type == LPT) {
389eb813999SBen Skeggs memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
390eb813999SBen Skeggs }
391eb813999SBen Skeggs }
392eb813999SBen Skeggs
393eb813999SBen Skeggs static bool
nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)394a5ff307fSBen Skeggs nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
395f9463a4bSBen Skeggs {
396f9463a4bSBen Skeggs struct nvkm_vmm_pt *pt = it->pt[0];
397f9463a4bSBen Skeggs if (it->desc->type == PGD)
398f9463a4bSBen Skeggs memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
399f9463a4bSBen Skeggs else
400f9463a4bSBen Skeggs if (it->desc->type == LPT)
401f9463a4bSBen Skeggs memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
402a5ff307fSBen Skeggs return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
403f9463a4bSBen Skeggs }
404f9463a4bSBen Skeggs
405f9463a4bSBen Skeggs static bool
nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)406a5ff307fSBen Skeggs nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
407f9463a4bSBen Skeggs {
408f9463a4bSBen Skeggs nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
409a5ff307fSBen Skeggs return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
410f9463a4bSBen Skeggs }
411f9463a4bSBen Skeggs
412f9463a4bSBen Skeggs static bool
nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter * it,struct nvkm_vmm_pt * pgd,u32 pdei)413eb813999SBen Skeggs nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
414eb813999SBen Skeggs {
415eb813999SBen Skeggs const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
416eb813999SBen Skeggs const int type = desc->type == SPT;
417eb813999SBen Skeggs struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
418eb813999SBen Skeggs const bool zero = !pgt->sparse && !desc->func->invalid;
419eb813999SBen Skeggs struct nvkm_vmm *vmm = it->vmm;
420eb813999SBen Skeggs struct nvkm_mmu *mmu = vmm->mmu;
421eb813999SBen Skeggs struct nvkm_mmu_pt *pt;
422eb813999SBen Skeggs u32 pten = 1 << desc->bits;
423eb813999SBen Skeggs u32 pteb, ptei, ptes;
424eb813999SBen Skeggs u32 size = desc->size * pten;
425eb813999SBen Skeggs
426eb813999SBen Skeggs pgd->refs[0]++;
427eb813999SBen Skeggs
428eb813999SBen Skeggs pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
429eb813999SBen Skeggs if (!pgt->pt[type]) {
430eb813999SBen Skeggs it->lvl--;
431eb813999SBen Skeggs nvkm_vmm_unref_pdes(it);
432eb813999SBen Skeggs return false;
433eb813999SBen Skeggs }
434eb813999SBen Skeggs
435eb813999SBen Skeggs if (zero)
436eb813999SBen Skeggs goto done;
437eb813999SBen Skeggs
438eb813999SBen Skeggs pt = pgt->pt[type];
439eb813999SBen Skeggs
440eb813999SBen Skeggs if (desc->type == LPT && pgt->refs[1]) {
441eb813999SBen Skeggs /* SPT already exists covering the same range as this LPT,
442eb813999SBen Skeggs * which means we need to be careful that any LPTEs which
443eb813999SBen Skeggs * overlap valid SPTEs are unmapped as opposed to invalid
444eb813999SBen Skeggs * or sparse, which would prevent the MMU from looking at
445eb813999SBen Skeggs * the SPTEs on some GPUs.
446eb813999SBen Skeggs */
447eb813999SBen Skeggs for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
448eb813999SBen Skeggs bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
449eb813999SBen Skeggs for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
450eb813999SBen Skeggs bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
451eb813999SBen Skeggs if (spte != next)
452eb813999SBen Skeggs break;
453eb813999SBen Skeggs }
454eb813999SBen Skeggs
455eb813999SBen Skeggs if (!spte) {
456eb813999SBen Skeggs if (pgt->sparse)
457eb813999SBen Skeggs desc->func->sparse(vmm, pt, pteb, ptes);
458eb813999SBen Skeggs else
459eb813999SBen Skeggs desc->func->invalid(vmm, pt, pteb, ptes);
460eb813999SBen Skeggs memset(&pgt->pte[pteb], 0x00, ptes);
461eb813999SBen Skeggs } else {
462eb813999SBen Skeggs desc->func->unmap(vmm, pt, pteb, ptes);
463eb813999SBen Skeggs while (ptes--)
464eb813999SBen Skeggs pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
465eb813999SBen Skeggs }
466eb813999SBen Skeggs }
467eb813999SBen Skeggs } else {
468eb813999SBen Skeggs if (pgt->sparse) {
469eb813999SBen Skeggs nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
470eb813999SBen Skeggs desc->func->sparse(vmm, pt, 0, pten);
471eb813999SBen Skeggs } else {
472eb813999SBen Skeggs desc->func->invalid(vmm, pt, 0, pten);
473eb813999SBen Skeggs }
474eb813999SBen Skeggs }
475eb813999SBen Skeggs
476eb813999SBen Skeggs done:
477eb813999SBen Skeggs TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
478eb813999SBen Skeggs it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
479eb813999SBen Skeggs nvkm_vmm_flush_mark(it);
480eb813999SBen Skeggs return true;
481eb813999SBen Skeggs }
482eb813999SBen Skeggs
483eb813999SBen Skeggs static bool
nvkm_vmm_ref_swpt(struct nvkm_vmm_iter * it,struct nvkm_vmm_pt * pgd,u32 pdei)484eb813999SBen Skeggs nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
485eb813999SBen Skeggs {
486eb813999SBen Skeggs const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
487eb813999SBen Skeggs struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
488eb813999SBen Skeggs
489eb813999SBen Skeggs pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
490eb813999SBen Skeggs if (!pgt) {
491eb813999SBen Skeggs if (!pgd->refs[0])
492eb813999SBen Skeggs nvkm_vmm_unref_pdes(it);
493eb813999SBen Skeggs return false;
494eb813999SBen Skeggs }
495eb813999SBen Skeggs
496eb813999SBen Skeggs pgd->pde[pdei] = pgt;
497eb813999SBen Skeggs return true;
498eb813999SBen Skeggs }
499eb813999SBen Skeggs
500eb813999SBen Skeggs static inline u64
nvkm_vmm_iter(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,const char * name,bool ref,bool pfn,bool (* REF_PTES)(struct nvkm_vmm_iter *,bool pfn,u32,u32),nvkm_vmm_pte_func MAP_PTES,struct nvkm_vmm_map * map,nvkm_vmm_pxe_func CLR_PTES)501eb813999SBen Skeggs nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
502a5ff307fSBen Skeggs u64 addr, u64 size, const char *name, bool ref, bool pfn,
503a5ff307fSBen Skeggs bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
504eb813999SBen Skeggs nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
505eb813999SBen Skeggs nvkm_vmm_pxe_func CLR_PTES)
506eb813999SBen Skeggs {
507eb813999SBen Skeggs const struct nvkm_vmm_desc *desc = page->desc;
508eb813999SBen Skeggs struct nvkm_vmm_iter it;
509eb813999SBen Skeggs u64 bits = addr >> page->shift;
510eb813999SBen Skeggs
511eb813999SBen Skeggs it.page = page;
512eb813999SBen Skeggs it.desc = desc;
513eb813999SBen Skeggs it.vmm = vmm;
514eb813999SBen Skeggs it.cnt = size >> page->shift;
515eb813999SBen Skeggs it.flush = NVKM_VMM_LEVELS_MAX;
516eb813999SBen Skeggs
517eb813999SBen Skeggs /* Deconstruct address into PTE indices for each mapping level. */
518eb813999SBen Skeggs for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
519eb813999SBen Skeggs it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
520eb813999SBen Skeggs bits >>= desc[it.lvl].bits;
521eb813999SBen Skeggs }
522eb813999SBen Skeggs it.max = --it.lvl;
523eb813999SBen Skeggs it.pt[it.max] = vmm->pd;
524eb813999SBen Skeggs
525eb813999SBen Skeggs it.lvl = 0;
526eb813999SBen Skeggs TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
527eb813999SBen Skeggs addr, size, page->shift, it.cnt);
528eb813999SBen Skeggs it.lvl = it.max;
529eb813999SBen Skeggs
530eb813999SBen Skeggs /* Depth-first traversal of page tables. */
531eb813999SBen Skeggs while (it.cnt) {
532eb813999SBen Skeggs struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
533eb813999SBen Skeggs const int type = desc->type == SPT;
534eb813999SBen Skeggs const u32 pten = 1 << desc->bits;
535eb813999SBen Skeggs const u32 ptei = it.pte[0];
536eb813999SBen Skeggs const u32 ptes = min_t(u64, it.cnt, pten - ptei);
537eb813999SBen Skeggs
538eb813999SBen Skeggs /* Walk down the tree, finding page tables for each level. */
539eb813999SBen Skeggs for (; it.lvl; it.lvl--) {
540eb813999SBen Skeggs const u32 pdei = it.pte[it.lvl];
541eb813999SBen Skeggs struct nvkm_vmm_pt *pgd = pgt;
542eb813999SBen Skeggs
543eb813999SBen Skeggs /* Software PT. */
544eb813999SBen Skeggs if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
545eb813999SBen Skeggs if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
546eb813999SBen Skeggs goto fail;
547eb813999SBen Skeggs }
548eb813999SBen Skeggs it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
549eb813999SBen Skeggs
550eb813999SBen Skeggs /* Hardware PT.
551eb813999SBen Skeggs *
552eb813999SBen Skeggs * This is a separate step from above due to GF100 and
553eb813999SBen Skeggs * newer having dual page tables at some levels, which
554eb813999SBen Skeggs * are refcounted independently.
555eb813999SBen Skeggs */
556eb813999SBen Skeggs if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
557eb813999SBen Skeggs if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
558eb813999SBen Skeggs goto fail;
559eb813999SBen Skeggs }
560eb813999SBen Skeggs }
561eb813999SBen Skeggs
562eb813999SBen Skeggs /* Handle PTE updates. */
563a5ff307fSBen Skeggs if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
564eb813999SBen Skeggs struct nvkm_mmu_pt *pt = pgt->pt[type];
565eb813999SBen Skeggs if (MAP_PTES || CLR_PTES) {
566eb813999SBen Skeggs if (MAP_PTES)
567eb813999SBen Skeggs MAP_PTES(vmm, pt, ptei, ptes, map);
568eb813999SBen Skeggs else
569eb813999SBen Skeggs CLR_PTES(vmm, pt, ptei, ptes);
570eb813999SBen Skeggs nvkm_vmm_flush_mark(&it);
571eb813999SBen Skeggs }
572eb813999SBen Skeggs }
573eb813999SBen Skeggs
574eb813999SBen Skeggs /* Walk back up the tree to the next position. */
575eb813999SBen Skeggs it.pte[it.lvl] += ptes;
576eb813999SBen Skeggs it.cnt -= ptes;
577eb813999SBen Skeggs if (it.cnt) {
578eb813999SBen Skeggs while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
579eb813999SBen Skeggs it.pte[it.lvl++] = 0;
580eb813999SBen Skeggs it.pte[it.lvl]++;
581eb813999SBen Skeggs }
582eb813999SBen Skeggs }
58394db9a3bSZheng Bin }
584eb813999SBen Skeggs
585eb813999SBen Skeggs nvkm_vmm_flush(&it);
586eb813999SBen Skeggs return ~0ULL;
587eb813999SBen Skeggs
588eb813999SBen Skeggs fail:
589eb813999SBen Skeggs /* Reconstruct the failure address so the caller is able to
590eb813999SBen Skeggs * reverse any partially completed operations.
591eb813999SBen Skeggs */
592eb813999SBen Skeggs addr = it.pte[it.max--];
593eb813999SBen Skeggs do {
594eb813999SBen Skeggs addr = addr << desc[it.max].bits;
595eb813999SBen Skeggs addr |= it.pte[it.max];
596eb813999SBen Skeggs } while (it.max--);
597eb813999SBen Skeggs
598eb813999SBen Skeggs return addr << page->shift;
599eb813999SBen Skeggs }
600eb813999SBen Skeggs
601f9463a4bSBen Skeggs static void
nvkm_vmm_ptes_sparse_put(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)602f9463a4bSBen Skeggs nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
603f9463a4bSBen Skeggs u64 addr, u64 size)
604f9463a4bSBen Skeggs {
605a5ff307fSBen Skeggs nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
606f9463a4bSBen Skeggs nvkm_vmm_sparse_unref_ptes, NULL, NULL,
607f9463a4bSBen Skeggs page->desc->func->invalid ?
608f9463a4bSBen Skeggs page->desc->func->invalid : page->desc->func->unmap);
609f9463a4bSBen Skeggs }
610f9463a4bSBen Skeggs
611f9463a4bSBen Skeggs static int
nvkm_vmm_ptes_sparse_get(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)612f9463a4bSBen Skeggs nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
613f9463a4bSBen Skeggs u64 addr, u64 size)
614f9463a4bSBen Skeggs {
615f9463a4bSBen Skeggs if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
616f9463a4bSBen Skeggs u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
617a5ff307fSBen Skeggs true, false, nvkm_vmm_sparse_ref_ptes,
618a5ff307fSBen Skeggs NULL, NULL, page->desc->func->sparse);
619f9463a4bSBen Skeggs if (fail != ~0ULL) {
620f9463a4bSBen Skeggs if ((size = fail - addr))
621f9463a4bSBen Skeggs nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
622f9463a4bSBen Skeggs return -ENOMEM;
623f9463a4bSBen Skeggs }
624f9463a4bSBen Skeggs return 0;
625f9463a4bSBen Skeggs }
626f9463a4bSBen Skeggs return -EINVAL;
627f9463a4bSBen Skeggs }
628f9463a4bSBen Skeggs
629f9463a4bSBen Skeggs static int
nvkm_vmm_ptes_sparse(struct nvkm_vmm * vmm,u64 addr,u64 size,bool ref)630f9463a4bSBen Skeggs nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
631f9463a4bSBen Skeggs {
632f9463a4bSBen Skeggs const struct nvkm_vmm_page *page = vmm->func->page;
633f9463a4bSBen Skeggs int m = 0, i;
634f9463a4bSBen Skeggs u64 start = addr;
635f9463a4bSBen Skeggs u64 block;
636f9463a4bSBen Skeggs
637f9463a4bSBen Skeggs while (size) {
638f9463a4bSBen Skeggs /* Limit maximum page size based on remaining size. */
639f9463a4bSBen Skeggs while (size < (1ULL << page[m].shift))
640f9463a4bSBen Skeggs m++;
641f9463a4bSBen Skeggs i = m;
642f9463a4bSBen Skeggs
643f9463a4bSBen Skeggs /* Find largest page size suitable for alignment. */
644f9463a4bSBen Skeggs while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
645f9463a4bSBen Skeggs i++;
646f9463a4bSBen Skeggs
647f9463a4bSBen Skeggs /* Determine number of PTEs at this page size. */
648f9463a4bSBen Skeggs if (i != m) {
649f9463a4bSBen Skeggs /* Limited to alignment boundary of next page size. */
650f9463a4bSBen Skeggs u64 next = 1ULL << page[i - 1].shift;
651f9463a4bSBen Skeggs u64 part = ALIGN(addr, next) - addr;
652f9463a4bSBen Skeggs if (size - part >= next)
653f9463a4bSBen Skeggs block = (part >> page[i].shift) << page[i].shift;
654f9463a4bSBen Skeggs else
655f9463a4bSBen Skeggs block = (size >> page[i].shift) << page[i].shift;
656f9463a4bSBen Skeggs } else {
657e64fe9dbSLuis de Bethencourt block = (size >> page[i].shift) << page[i].shift;
658f9463a4bSBen Skeggs }
659f9463a4bSBen Skeggs
660f9463a4bSBen Skeggs /* Perform operation. */
661f9463a4bSBen Skeggs if (ref) {
662f9463a4bSBen Skeggs int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
663f9463a4bSBen Skeggs if (ret) {
664f9463a4bSBen Skeggs if ((size = addr - start))
665f9463a4bSBen Skeggs nvkm_vmm_ptes_sparse(vmm, start, size, false);
666f9463a4bSBen Skeggs return ret;
667f9463a4bSBen Skeggs }
668f9463a4bSBen Skeggs } else {
669f9463a4bSBen Skeggs nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
670f9463a4bSBen Skeggs }
671f9463a4bSBen Skeggs
672f9463a4bSBen Skeggs size -= block;
673f9463a4bSBen Skeggs addr += block;
674f9463a4bSBen Skeggs }
675f9463a4bSBen Skeggs
676f9463a4bSBen Skeggs return 0;
677f9463a4bSBen Skeggs }
678f9463a4bSBen Skeggs
679f9463a4bSBen Skeggs static void
nvkm_vmm_ptes_unmap(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,bool sparse,bool pfn)680*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
681a5ff307fSBen Skeggs u64 addr, u64 size, bool sparse, bool pfn)
682f9463a4bSBen Skeggs {
683f9463a4bSBen Skeggs const struct nvkm_vmm_desc_func *func = page->desc->func;
684*6b252cf4SDanilo Krummrich
685*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.map);
686*6b252cf4SDanilo Krummrich nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
687*6b252cf4SDanilo Krummrich NULL, NULL, NULL,
688*6b252cf4SDanilo Krummrich sparse ? func->sparse : func->invalid ? func->invalid :
689*6b252cf4SDanilo Krummrich func->unmap);
690*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.map);
691*6b252cf4SDanilo Krummrich }
692*6b252cf4SDanilo Krummrich
693*6b252cf4SDanilo Krummrich static void
nvkm_vmm_ptes_map(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,struct nvkm_vmm_map * map,nvkm_vmm_pte_func func)694*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
695*6b252cf4SDanilo Krummrich u64 addr, u64 size, struct nvkm_vmm_map *map,
696*6b252cf4SDanilo Krummrich nvkm_vmm_pte_func func)
697*6b252cf4SDanilo Krummrich {
698*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.map);
699*6b252cf4SDanilo Krummrich nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
700*6b252cf4SDanilo Krummrich NULL, func, map, NULL);
701*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.map);
702*6b252cf4SDanilo Krummrich }
703*6b252cf4SDanilo Krummrich
704*6b252cf4SDanilo Krummrich static void
nvkm_vmm_ptes_put_locked(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)705*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_put_locked(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
706*6b252cf4SDanilo Krummrich u64 addr, u64 size)
707*6b252cf4SDanilo Krummrich {
708*6b252cf4SDanilo Krummrich nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
709*6b252cf4SDanilo Krummrich nvkm_vmm_unref_ptes, NULL, NULL, NULL);
710*6b252cf4SDanilo Krummrich }
711*6b252cf4SDanilo Krummrich
712*6b252cf4SDanilo Krummrich static void
nvkm_vmm_ptes_put(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)713*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
714*6b252cf4SDanilo Krummrich u64 addr, u64 size)
715*6b252cf4SDanilo Krummrich {
716*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.ref);
717*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_put_locked(vmm, page, addr, size);
718*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.ref);
719*6b252cf4SDanilo Krummrich }
720*6b252cf4SDanilo Krummrich
721*6b252cf4SDanilo Krummrich static int
nvkm_vmm_ptes_get(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size)722*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
723*6b252cf4SDanilo Krummrich u64 addr, u64 size)
724*6b252cf4SDanilo Krummrich {
725*6b252cf4SDanilo Krummrich u64 fail;
726*6b252cf4SDanilo Krummrich
727*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.ref);
728*6b252cf4SDanilo Krummrich fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
729*6b252cf4SDanilo Krummrich nvkm_vmm_ref_ptes, NULL, NULL, NULL);
730*6b252cf4SDanilo Krummrich if (fail != ~0ULL) {
731*6b252cf4SDanilo Krummrich if (fail != addr)
732*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_put_locked(vmm, page, addr, fail - addr);
733*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.ref);
734*6b252cf4SDanilo Krummrich return -ENOMEM;
735*6b252cf4SDanilo Krummrich }
736*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.ref);
737*6b252cf4SDanilo Krummrich return 0;
738*6b252cf4SDanilo Krummrich }
739*6b252cf4SDanilo Krummrich
740*6b252cf4SDanilo Krummrich static void
__nvkm_vmm_ptes_unmap_put(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,bool sparse,bool pfn)741*6b252cf4SDanilo Krummrich __nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
742*6b252cf4SDanilo Krummrich u64 addr, u64 size, bool sparse, bool pfn)
743*6b252cf4SDanilo Krummrich {
744*6b252cf4SDanilo Krummrich const struct nvkm_vmm_desc_func *func = page->desc->func;
745*6b252cf4SDanilo Krummrich
746f9463a4bSBen Skeggs nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
747a5ff307fSBen Skeggs false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
748f9463a4bSBen Skeggs sparse ? func->sparse : func->invalid ? func->invalid :
749f9463a4bSBen Skeggs func->unmap);
750f9463a4bSBen Skeggs }
751f9463a4bSBen Skeggs
752*6b252cf4SDanilo Krummrich static void
nvkm_vmm_ptes_unmap_put(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,bool sparse,bool pfn)753*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
754*6b252cf4SDanilo Krummrich u64 addr, u64 size, bool sparse, bool pfn)
755*6b252cf4SDanilo Krummrich {
756*6b252cf4SDanilo Krummrich if (vmm->managed.raw) {
757*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_unmap(vmm, page, addr, size, sparse, pfn);
758*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_put(vmm, page, addr, size);
759*6b252cf4SDanilo Krummrich } else {
760*6b252cf4SDanilo Krummrich __nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, sparse, pfn);
761*6b252cf4SDanilo Krummrich }
762*6b252cf4SDanilo Krummrich }
763*6b252cf4SDanilo Krummrich
764f9463a4bSBen Skeggs static int
__nvkm_vmm_ptes_get_map(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,struct nvkm_vmm_map * map,nvkm_vmm_pte_func func)765*6b252cf4SDanilo Krummrich __nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
766f9463a4bSBen Skeggs u64 addr, u64 size, struct nvkm_vmm_map *map,
767f9463a4bSBen Skeggs nvkm_vmm_pte_func func)
768f9463a4bSBen Skeggs {
769f9463a4bSBen Skeggs u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
770a5ff307fSBen Skeggs false, nvkm_vmm_ref_ptes, func, map, NULL);
771f9463a4bSBen Skeggs if (fail != ~0ULL) {
772f9463a4bSBen Skeggs if ((size = fail - addr))
773a5ff307fSBen Skeggs nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
774f9463a4bSBen Skeggs return -ENOMEM;
775f9463a4bSBen Skeggs }
776f9463a4bSBen Skeggs return 0;
777f9463a4bSBen Skeggs }
778f9463a4bSBen Skeggs
779*6b252cf4SDanilo Krummrich static int
nvkm_vmm_ptes_get_map(struct nvkm_vmm * vmm,const struct nvkm_vmm_page * page,u64 addr,u64 size,struct nvkm_vmm_map * map,nvkm_vmm_pte_func func)780*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
781eb813999SBen Skeggs u64 addr, u64 size, struct nvkm_vmm_map *map,
782eb813999SBen Skeggs nvkm_vmm_pte_func func)
783eb813999SBen Skeggs {
784*6b252cf4SDanilo Krummrich int ret;
785eb813999SBen Skeggs
786*6b252cf4SDanilo Krummrich if (vmm->managed.raw) {
787*6b252cf4SDanilo Krummrich ret = nvkm_vmm_ptes_get(vmm, page, addr, size);
788*6b252cf4SDanilo Krummrich if (ret)
789*6b252cf4SDanilo Krummrich return ret;
790eb813999SBen Skeggs
791*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_map(vmm, page, addr, size, map, func);
792*6b252cf4SDanilo Krummrich
793eb813999SBen Skeggs return 0;
794*6b252cf4SDanilo Krummrich } else {
795*6b252cf4SDanilo Krummrich return __nvkm_vmm_ptes_get_map(vmm, page, addr, size, map, func);
796*6b252cf4SDanilo Krummrich }
797eb813999SBen Skeggs }
798eb813999SBen Skeggs
799*6b252cf4SDanilo Krummrich struct nvkm_vma *
nvkm_vma_new(u64 addr,u64 size)800f9463a4bSBen Skeggs nvkm_vma_new(u64 addr, u64 size)
801f9463a4bSBen Skeggs {
802f9463a4bSBen Skeggs struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
803f9463a4bSBen Skeggs if (vma) {
804f9463a4bSBen Skeggs vma->addr = addr;
805f9463a4bSBen Skeggs vma->size = size;
806f9463a4bSBen Skeggs vma->page = NVKM_VMA_PAGE_NONE;
807f9463a4bSBen Skeggs vma->refd = NVKM_VMA_PAGE_NONE;
808f9463a4bSBen Skeggs }
809f9463a4bSBen Skeggs return vma;
810f9463a4bSBen Skeggs }
811f9463a4bSBen Skeggs
812f9463a4bSBen Skeggs struct nvkm_vma *
nvkm_vma_tail(struct nvkm_vma * vma,u64 tail)813f9463a4bSBen Skeggs nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
814f9463a4bSBen Skeggs {
815f9463a4bSBen Skeggs struct nvkm_vma *new;
816f9463a4bSBen Skeggs
817f9463a4bSBen Skeggs BUG_ON(vma->size == tail);
818f9463a4bSBen Skeggs
819f9463a4bSBen Skeggs if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
820f9463a4bSBen Skeggs return NULL;
821f9463a4bSBen Skeggs vma->size -= tail;
822f9463a4bSBen Skeggs
823f9463a4bSBen Skeggs new->mapref = vma->mapref;
824f9463a4bSBen Skeggs new->sparse = vma->sparse;
825f9463a4bSBen Skeggs new->page = vma->page;
826f9463a4bSBen Skeggs new->refd = vma->refd;
827f9463a4bSBen Skeggs new->used = vma->used;
828f9463a4bSBen Skeggs new->part = vma->part;
829f9463a4bSBen Skeggs new->busy = vma->busy;
8308e68271dSBen Skeggs new->mapped = vma->mapped;
831f9463a4bSBen Skeggs list_add(&new->head, &vma->head);
832f9463a4bSBen Skeggs return new;
833f9463a4bSBen Skeggs }
834f9463a4bSBen Skeggs
835729eba33SBen Skeggs static inline void
nvkm_vmm_free_remove(struct nvkm_vmm * vmm,struct nvkm_vma * vma)836729eba33SBen Skeggs nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
837729eba33SBen Skeggs {
838729eba33SBen Skeggs rb_erase(&vma->tree, &vmm->free);
839729eba33SBen Skeggs }
840729eba33SBen Skeggs
841729eba33SBen Skeggs static inline void
nvkm_vmm_free_delete(struct nvkm_vmm * vmm,struct nvkm_vma * vma)842729eba33SBen Skeggs nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
843729eba33SBen Skeggs {
844729eba33SBen Skeggs nvkm_vmm_free_remove(vmm, vma);
845729eba33SBen Skeggs list_del(&vma->head);
846729eba33SBen Skeggs kfree(vma);
847729eba33SBen Skeggs }
848729eba33SBen Skeggs
849f9463a4bSBen Skeggs static void
nvkm_vmm_free_insert(struct nvkm_vmm * vmm,struct nvkm_vma * vma)850f9463a4bSBen Skeggs nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
851f9463a4bSBen Skeggs {
852f9463a4bSBen Skeggs struct rb_node **ptr = &vmm->free.rb_node;
853f9463a4bSBen Skeggs struct rb_node *parent = NULL;
854f9463a4bSBen Skeggs
855f9463a4bSBen Skeggs while (*ptr) {
856f9463a4bSBen Skeggs struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
857f9463a4bSBen Skeggs parent = *ptr;
858f9463a4bSBen Skeggs if (vma->size < this->size)
859f9463a4bSBen Skeggs ptr = &parent->rb_left;
860f9463a4bSBen Skeggs else
861f9463a4bSBen Skeggs if (vma->size > this->size)
862f9463a4bSBen Skeggs ptr = &parent->rb_right;
863f9463a4bSBen Skeggs else
864f9463a4bSBen Skeggs if (vma->addr < this->addr)
865f9463a4bSBen Skeggs ptr = &parent->rb_left;
866f9463a4bSBen Skeggs else
867f9463a4bSBen Skeggs if (vma->addr > this->addr)
868f9463a4bSBen Skeggs ptr = &parent->rb_right;
869f9463a4bSBen Skeggs else
870f9463a4bSBen Skeggs BUG();
871f9463a4bSBen Skeggs }
872f9463a4bSBen Skeggs
873f9463a4bSBen Skeggs rb_link_node(&vma->tree, parent, ptr);
874f9463a4bSBen Skeggs rb_insert_color(&vma->tree, &vmm->free);
875f9463a4bSBen Skeggs }
876f9463a4bSBen Skeggs
877729eba33SBen Skeggs static inline void
nvkm_vmm_node_remove(struct nvkm_vmm * vmm,struct nvkm_vma * vma)878729eba33SBen Skeggs nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
879729eba33SBen Skeggs {
880729eba33SBen Skeggs rb_erase(&vma->tree, &vmm->root);
881729eba33SBen Skeggs }
882729eba33SBen Skeggs
883729eba33SBen Skeggs static inline void
nvkm_vmm_node_delete(struct nvkm_vmm * vmm,struct nvkm_vma * vma)884729eba33SBen Skeggs nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
885729eba33SBen Skeggs {
886729eba33SBen Skeggs nvkm_vmm_node_remove(vmm, vma);
887729eba33SBen Skeggs list_del(&vma->head);
888729eba33SBen Skeggs kfree(vma);
889729eba33SBen Skeggs }
890729eba33SBen Skeggs
891729eba33SBen Skeggs static void
nvkm_vmm_node_insert(struct nvkm_vmm * vmm,struct nvkm_vma * vma)892f9463a4bSBen Skeggs nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
893f9463a4bSBen Skeggs {
894f9463a4bSBen Skeggs struct rb_node **ptr = &vmm->root.rb_node;
895f9463a4bSBen Skeggs struct rb_node *parent = NULL;
896f9463a4bSBen Skeggs
897f9463a4bSBen Skeggs while (*ptr) {
898f9463a4bSBen Skeggs struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
899f9463a4bSBen Skeggs parent = *ptr;
900f9463a4bSBen Skeggs if (vma->addr < this->addr)
901f9463a4bSBen Skeggs ptr = &parent->rb_left;
902f9463a4bSBen Skeggs else
903f9463a4bSBen Skeggs if (vma->addr > this->addr)
904f9463a4bSBen Skeggs ptr = &parent->rb_right;
905f9463a4bSBen Skeggs else
906f9463a4bSBen Skeggs BUG();
907f9463a4bSBen Skeggs }
908f9463a4bSBen Skeggs
909f9463a4bSBen Skeggs rb_link_node(&vma->tree, parent, ptr);
910f9463a4bSBen Skeggs rb_insert_color(&vma->tree, &vmm->root);
911f9463a4bSBen Skeggs }
912f9463a4bSBen Skeggs
913f9463a4bSBen Skeggs struct nvkm_vma *
nvkm_vmm_node_search(struct nvkm_vmm * vmm,u64 addr)914f9463a4bSBen Skeggs nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
915f9463a4bSBen Skeggs {
916f9463a4bSBen Skeggs struct rb_node *node = vmm->root.rb_node;
917f9463a4bSBen Skeggs while (node) {
918f9463a4bSBen Skeggs struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
919f9463a4bSBen Skeggs if (addr < vma->addr)
920f9463a4bSBen Skeggs node = node->rb_left;
921f9463a4bSBen Skeggs else
922f9463a4bSBen Skeggs if (addr >= vma->addr + vma->size)
923f9463a4bSBen Skeggs node = node->rb_right;
924f9463a4bSBen Skeggs else
925f9463a4bSBen Skeggs return vma;
926f9463a4bSBen Skeggs }
927f9463a4bSBen Skeggs return NULL;
928f9463a4bSBen Skeggs }
929f9463a4bSBen Skeggs
930729eba33SBen Skeggs #define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL : \
931729eba33SBen Skeggs list_entry((root)->head.dir, struct nvkm_vma, head))
932729eba33SBen Skeggs
933729eba33SBen Skeggs static struct nvkm_vma *
nvkm_vmm_node_merge(struct nvkm_vmm * vmm,struct nvkm_vma * prev,struct nvkm_vma * vma,struct nvkm_vma * next,u64 size)934729eba33SBen Skeggs nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev,
935729eba33SBen Skeggs struct nvkm_vma *vma, struct nvkm_vma *next, u64 size)
936729eba33SBen Skeggs {
937729eba33SBen Skeggs if (next) {
938729eba33SBen Skeggs if (vma->size == size) {
939729eba33SBen Skeggs vma->size += next->size;
940729eba33SBen Skeggs nvkm_vmm_node_delete(vmm, next);
941729eba33SBen Skeggs if (prev) {
942729eba33SBen Skeggs prev->size += vma->size;
943729eba33SBen Skeggs nvkm_vmm_node_delete(vmm, vma);
944729eba33SBen Skeggs return prev;
945729eba33SBen Skeggs }
946729eba33SBen Skeggs return vma;
947729eba33SBen Skeggs }
948729eba33SBen Skeggs BUG_ON(prev);
949729eba33SBen Skeggs
950729eba33SBen Skeggs nvkm_vmm_node_remove(vmm, next);
951729eba33SBen Skeggs vma->size -= size;
952729eba33SBen Skeggs next->addr -= size;
953729eba33SBen Skeggs next->size += size;
954729eba33SBen Skeggs nvkm_vmm_node_insert(vmm, next);
955729eba33SBen Skeggs return next;
956729eba33SBen Skeggs }
957729eba33SBen Skeggs
958729eba33SBen Skeggs if (prev) {
959729eba33SBen Skeggs if (vma->size != size) {
960729eba33SBen Skeggs nvkm_vmm_node_remove(vmm, vma);
961729eba33SBen Skeggs prev->size += size;
962729eba33SBen Skeggs vma->addr += size;
963729eba33SBen Skeggs vma->size -= size;
964729eba33SBen Skeggs nvkm_vmm_node_insert(vmm, vma);
965729eba33SBen Skeggs } else {
966729eba33SBen Skeggs prev->size += vma->size;
967729eba33SBen Skeggs nvkm_vmm_node_delete(vmm, vma);
968729eba33SBen Skeggs }
969729eba33SBen Skeggs return prev;
970729eba33SBen Skeggs }
971729eba33SBen Skeggs
972729eba33SBen Skeggs return vma;
973729eba33SBen Skeggs }
974729eba33SBen Skeggs
975729eba33SBen Skeggs struct nvkm_vma *
nvkm_vmm_node_split(struct nvkm_vmm * vmm,struct nvkm_vma * vma,u64 addr,u64 size)976729eba33SBen Skeggs nvkm_vmm_node_split(struct nvkm_vmm *vmm,
977729eba33SBen Skeggs struct nvkm_vma *vma, u64 addr, u64 size)
978729eba33SBen Skeggs {
979729eba33SBen Skeggs struct nvkm_vma *prev = NULL;
980729eba33SBen Skeggs
981729eba33SBen Skeggs if (vma->addr != addr) {
982729eba33SBen Skeggs prev = vma;
983729eba33SBen Skeggs if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr)))
984729eba33SBen Skeggs return NULL;
985729eba33SBen Skeggs vma->part = true;
986729eba33SBen Skeggs nvkm_vmm_node_insert(vmm, vma);
987729eba33SBen Skeggs }
988729eba33SBen Skeggs
989729eba33SBen Skeggs if (vma->size != size) {
990729eba33SBen Skeggs struct nvkm_vma *tmp;
991729eba33SBen Skeggs if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
992729eba33SBen Skeggs nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size);
993729eba33SBen Skeggs return NULL;
994729eba33SBen Skeggs }
995729eba33SBen Skeggs tmp->part = true;
996729eba33SBen Skeggs nvkm_vmm_node_insert(vmm, tmp);
997729eba33SBen Skeggs }
998729eba33SBen Skeggs
999729eba33SBen Skeggs return vma;
1000729eba33SBen Skeggs }
1001729eba33SBen Skeggs
1002f9463a4bSBen Skeggs static void
nvkm_vma_dump(struct nvkm_vma * vma)1003a5ff307fSBen Skeggs nvkm_vma_dump(struct nvkm_vma *vma)
1004a5ff307fSBen Skeggs {
100559f216cfSBen Skeggs printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c %p\n",
1006a5ff307fSBen Skeggs vma->addr, (u64)vma->size,
1007a5ff307fSBen Skeggs vma->used ? '-' : 'F',
1008a5ff307fSBen Skeggs vma->mapref ? 'R' : '-',
1009a5ff307fSBen Skeggs vma->sparse ? 'S' : '-',
1010a5ff307fSBen Skeggs vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
1011a5ff307fSBen Skeggs vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
1012a5ff307fSBen Skeggs vma->part ? 'P' : '-',
1013a5ff307fSBen Skeggs vma->busy ? 'B' : '-',
1014a5ff307fSBen Skeggs vma->mapped ? 'M' : '-',
1015a5ff307fSBen Skeggs vma->memory);
1016a5ff307fSBen Skeggs }
1017a5ff307fSBen Skeggs
1018a5ff307fSBen Skeggs static void
nvkm_vmm_dump(struct nvkm_vmm * vmm)1019a5ff307fSBen Skeggs nvkm_vmm_dump(struct nvkm_vmm *vmm)
1020a5ff307fSBen Skeggs {
1021a5ff307fSBen Skeggs struct nvkm_vma *vma;
1022a5ff307fSBen Skeggs list_for_each_entry(vma, &vmm->list, head) {
1023a5ff307fSBen Skeggs nvkm_vma_dump(vma);
1024a5ff307fSBen Skeggs }
1025a5ff307fSBen Skeggs }
1026a5ff307fSBen Skeggs
1027a5ff307fSBen Skeggs static void
nvkm_vmm_dtor(struct nvkm_vmm * vmm)1028806a7335SBen Skeggs nvkm_vmm_dtor(struct nvkm_vmm *vmm)
1029806a7335SBen Skeggs {
1030f9463a4bSBen Skeggs struct nvkm_vma *vma;
1031f9463a4bSBen Skeggs struct rb_node *node;
1032f9463a4bSBen Skeggs
1033a5ff307fSBen Skeggs if (0)
1034a5ff307fSBen Skeggs nvkm_vmm_dump(vmm);
1035a5ff307fSBen Skeggs
1036f9463a4bSBen Skeggs while ((node = rb_first(&vmm->root))) {
1037f9463a4bSBen Skeggs struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
1038f9463a4bSBen Skeggs nvkm_vmm_put(vmm, &vma);
1039f9463a4bSBen Skeggs }
1040f9463a4bSBen Skeggs
1041eb813999SBen Skeggs if (vmm->bootstrapped) {
1042eb813999SBen Skeggs const struct nvkm_vmm_page *page = vmm->func->page;
1043eb813999SBen Skeggs const u64 limit = vmm->limit - vmm->start;
1044eb813999SBen Skeggs
1045eb813999SBen Skeggs while (page[1].shift)
1046eb813999SBen Skeggs page++;
1047eb813999SBen Skeggs
1048eb813999SBen Skeggs nvkm_mmu_ptc_dump(vmm->mmu);
1049eb813999SBen Skeggs nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
1050eb813999SBen Skeggs }
1051eb813999SBen Skeggs
1052f9463a4bSBen Skeggs vma = list_first_entry(&vmm->list, typeof(*vma), head);
1053f9463a4bSBen Skeggs list_del(&vma->head);
1054f9463a4bSBen Skeggs kfree(vma);
1055f9463a4bSBen Skeggs WARN_ON(!list_empty(&vmm->list));
1056f9463a4bSBen Skeggs
105703b0ba7bSBen Skeggs if (vmm->nullp) {
105803b0ba7bSBen Skeggs dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
105903b0ba7bSBen Skeggs vmm->nullp, vmm->null);
106003b0ba7bSBen Skeggs }
106103b0ba7bSBen Skeggs
1062806a7335SBen Skeggs if (vmm->pd) {
1063806a7335SBen Skeggs nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
1064806a7335SBen Skeggs nvkm_vmm_pt_del(&vmm->pd);
1065806a7335SBen Skeggs }
1066806a7335SBen Skeggs }
1067806a7335SBen Skeggs
10682606f291SBen Skeggs static int
nvkm_vmm_ctor_managed(struct nvkm_vmm * vmm,u64 addr,u64 size)10692606f291SBen Skeggs nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size)
10702606f291SBen Skeggs {
10712606f291SBen Skeggs struct nvkm_vma *vma;
10722606f291SBen Skeggs if (!(vma = nvkm_vma_new(addr, size)))
10732606f291SBen Skeggs return -ENOMEM;
10742606f291SBen Skeggs vma->mapref = true;
10752606f291SBen Skeggs vma->sparse = false;
10762606f291SBen Skeggs vma->used = true;
10772606f291SBen Skeggs nvkm_vmm_node_insert(vmm, vma);
10782606f291SBen Skeggs list_add_tail(&vma->head, &vmm->list);
10792606f291SBen Skeggs return 0;
10802606f291SBen Skeggs }
10812606f291SBen Skeggs
1082e5c7864fSRalph Campbell static int
nvkm_vmm_ctor(const struct nvkm_vmm_func * func,struct nvkm_mmu * mmu,u32 pd_header,bool managed,u64 addr,u64 size,struct lock_class_key * key,const char * name,struct nvkm_vmm * vmm)1083806a7335SBen Skeggs nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
10842606f291SBen Skeggs u32 pd_header, bool managed, u64 addr, u64 size,
10852606f291SBen Skeggs struct lock_class_key *key, const char *name,
10862606f291SBen Skeggs struct nvkm_vmm *vmm)
1087806a7335SBen Skeggs {
1088806a7335SBen Skeggs static struct lock_class_key _key;
1089806a7335SBen Skeggs const struct nvkm_vmm_page *page = func->page;
1090806a7335SBen Skeggs const struct nvkm_vmm_desc *desc;
1091f9463a4bSBen Skeggs struct nvkm_vma *vma;
10922606f291SBen Skeggs int levels, bits = 0, ret;
1093806a7335SBen Skeggs
1094806a7335SBen Skeggs vmm->func = func;
1095806a7335SBen Skeggs vmm->mmu = mmu;
1096806a7335SBen Skeggs vmm->name = name;
1097eb813999SBen Skeggs vmm->debug = mmu->subdev.debug;
1098806a7335SBen Skeggs kref_init(&vmm->kref);
1099806a7335SBen Skeggs
1100*6b252cf4SDanilo Krummrich __mutex_init(&vmm->mutex.vmm, "&vmm->mutex.vmm", key ? key : &_key);
1101*6b252cf4SDanilo Krummrich mutex_init(&vmm->mutex.ref);
1102*6b252cf4SDanilo Krummrich mutex_init(&vmm->mutex.map);
1103806a7335SBen Skeggs
1104806a7335SBen Skeggs /* Locate the smallest page size supported by the backend, it will
11057c021558SJiang Jian * have the deepest nesting of page tables.
1106806a7335SBen Skeggs */
1107806a7335SBen Skeggs while (page[1].shift)
1108806a7335SBen Skeggs page++;
1109806a7335SBen Skeggs
1110806a7335SBen Skeggs /* Locate the structure that describes the layout of the top-level
1111806a7335SBen Skeggs * page table, and determine the number of valid bits in a virtual
1112806a7335SBen Skeggs * address.
1113806a7335SBen Skeggs */
1114806a7335SBen Skeggs for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
1115806a7335SBen Skeggs bits += desc->bits;
1116806a7335SBen Skeggs bits += page->shift;
1117806a7335SBen Skeggs desc--;
1118806a7335SBen Skeggs
1119806a7335SBen Skeggs if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
1120806a7335SBen Skeggs return -EINVAL;
1121806a7335SBen Skeggs
1122806a7335SBen Skeggs /* Allocate top-level page table. */
1123806a7335SBen Skeggs vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
1124806a7335SBen Skeggs if (!vmm->pd)
1125806a7335SBen Skeggs return -ENOMEM;
1126806a7335SBen Skeggs vmm->pd->refs[0] = 1;
1127806a7335SBen Skeggs INIT_LIST_HEAD(&vmm->join);
1128806a7335SBen Skeggs
1129806a7335SBen Skeggs /* ... and the GPU storage for it, except on Tesla-class GPUs that
1130806a7335SBen Skeggs * have the PD embedded in the instance structure.
1131806a7335SBen Skeggs */
1132d30af7ceSBen Skeggs if (desc->size) {
1133806a7335SBen Skeggs const u32 size = pd_header + desc->size * (1 << desc->bits);
1134806a7335SBen Skeggs vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
1135806a7335SBen Skeggs if (!vmm->pd->pt[0])
1136806a7335SBen Skeggs return -ENOMEM;
1137806a7335SBen Skeggs }
1138806a7335SBen Skeggs
1139f9463a4bSBen Skeggs /* Initialise address-space MM. */
1140f9463a4bSBen Skeggs INIT_LIST_HEAD(&vmm->list);
1141f9463a4bSBen Skeggs vmm->free = RB_ROOT;
1142f9463a4bSBen Skeggs vmm->root = RB_ROOT;
1143f9463a4bSBen Skeggs
11442606f291SBen Skeggs if (managed) {
11452606f291SBen Skeggs /* Address-space will be managed by the client for the most
11462606f291SBen Skeggs * part, except for a specified area where NVKM allocations
11472606f291SBen Skeggs * are allowed to be placed.
11482606f291SBen Skeggs */
11492606f291SBen Skeggs vmm->start = 0;
11502606f291SBen Skeggs vmm->limit = 1ULL << bits;
11512606f291SBen Skeggs if (addr + size < addr || addr + size > vmm->limit)
11522606f291SBen Skeggs return -EINVAL;
11532606f291SBen Skeggs
11542606f291SBen Skeggs /* Client-managed area before the NVKM-managed area. */
11552606f291SBen Skeggs if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr)))
11562606f291SBen Skeggs return ret;
11572606f291SBen Skeggs
1158*6b252cf4SDanilo Krummrich vmm->managed.p.addr = 0;
1159*6b252cf4SDanilo Krummrich vmm->managed.p.size = addr;
1160*6b252cf4SDanilo Krummrich
11612606f291SBen Skeggs /* NVKM-managed area. */
11622606f291SBen Skeggs if (size) {
11632606f291SBen Skeggs if (!(vma = nvkm_vma_new(addr, size)))
11642606f291SBen Skeggs return -ENOMEM;
11652606f291SBen Skeggs nvkm_vmm_free_insert(vmm, vma);
11662606f291SBen Skeggs list_add_tail(&vma->head, &vmm->list);
11672606f291SBen Skeggs }
11682606f291SBen Skeggs
11692606f291SBen Skeggs /* Client-managed area after the NVKM-managed area. */
11702606f291SBen Skeggs addr = addr + size;
11712606f291SBen Skeggs size = vmm->limit - addr;
11722606f291SBen Skeggs if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size)))
11732606f291SBen Skeggs return ret;
1174*6b252cf4SDanilo Krummrich
1175*6b252cf4SDanilo Krummrich vmm->managed.n.addr = addr;
1176*6b252cf4SDanilo Krummrich vmm->managed.n.size = size;
11772606f291SBen Skeggs } else {
11782606f291SBen Skeggs /* Address-space fully managed by NVKM, requiring calls to
11792606f291SBen Skeggs * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space.
11802606f291SBen Skeggs */
11812606f291SBen Skeggs vmm->start = addr;
11822606f291SBen Skeggs vmm->limit = size ? (addr + size) : (1ULL << bits);
11832606f291SBen Skeggs if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
11842606f291SBen Skeggs return -EINVAL;
11852606f291SBen Skeggs
1186f9463a4bSBen Skeggs if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
1187f9463a4bSBen Skeggs return -ENOMEM;
1188f9463a4bSBen Skeggs
1189f9463a4bSBen Skeggs nvkm_vmm_free_insert(vmm, vma);
1190f9463a4bSBen Skeggs list_add(&vma->head, &vmm->list);
11912606f291SBen Skeggs }
11922606f291SBen Skeggs
1193806a7335SBen Skeggs return 0;
1194806a7335SBen Skeggs }
1195806a7335SBen Skeggs
1196806a7335SBen Skeggs int
nvkm_vmm_new_(const struct nvkm_vmm_func * func,struct nvkm_mmu * mmu,u32 hdr,bool managed,u64 addr,u64 size,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)1197806a7335SBen Skeggs nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
11982606f291SBen Skeggs u32 hdr, bool managed, u64 addr, u64 size,
11992606f291SBen Skeggs struct lock_class_key *key, const char *name,
12002606f291SBen Skeggs struct nvkm_vmm **pvmm)
1201806a7335SBen Skeggs {
1202806a7335SBen Skeggs if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
1203806a7335SBen Skeggs return -ENOMEM;
12042606f291SBen Skeggs return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
1205806a7335SBen Skeggs }
1206eb813999SBen Skeggs
1207a5ff307fSBen Skeggs static struct nvkm_vma *
nvkm_vmm_pfn_split_merge(struct nvkm_vmm * vmm,struct nvkm_vma * vma,u64 addr,u64 size,u8 page,bool map)1208a5ff307fSBen Skeggs nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1209a5ff307fSBen Skeggs u64 addr, u64 size, u8 page, bool map)
1210a5ff307fSBen Skeggs {
1211a5ff307fSBen Skeggs struct nvkm_vma *prev = NULL;
1212a5ff307fSBen Skeggs struct nvkm_vma *next = NULL;
1213a5ff307fSBen Skeggs
1214a5ff307fSBen Skeggs if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
1215a5ff307fSBen Skeggs if (prev->memory || prev->mapped != map)
1216a5ff307fSBen Skeggs prev = NULL;
1217a5ff307fSBen Skeggs }
1218a5ff307fSBen Skeggs
1219a5ff307fSBen Skeggs if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
1220a5ff307fSBen Skeggs if (!next->part ||
1221a5ff307fSBen Skeggs next->memory || next->mapped != map)
1222a5ff307fSBen Skeggs next = NULL;
1223a5ff307fSBen Skeggs }
1224a5ff307fSBen Skeggs
1225a5ff307fSBen Skeggs if (prev || next)
1226a5ff307fSBen Skeggs return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
1227a5ff307fSBen Skeggs return nvkm_vmm_node_split(vmm, vma, addr, size);
1228a5ff307fSBen Skeggs }
1229a5ff307fSBen Skeggs
1230a5ff307fSBen Skeggs int
nvkm_vmm_pfn_unmap(struct nvkm_vmm * vmm,u64 addr,u64 size)1231a5ff307fSBen Skeggs nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
1232a5ff307fSBen Skeggs {
1233a5ff307fSBen Skeggs struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
1234a5ff307fSBen Skeggs struct nvkm_vma *next;
1235a5ff307fSBen Skeggs u64 limit = addr + size;
1236a5ff307fSBen Skeggs u64 start = addr;
1237a5ff307fSBen Skeggs
1238a5ff307fSBen Skeggs if (!vma)
1239a5ff307fSBen Skeggs return -EINVAL;
1240a5ff307fSBen Skeggs
1241a5ff307fSBen Skeggs do {
1242a5ff307fSBen Skeggs if (!vma->mapped || vma->memory)
1243a5ff307fSBen Skeggs continue;
1244a5ff307fSBen Skeggs
1245a5ff307fSBen Skeggs size = min(limit - start, vma->size - (start - vma->addr));
1246a5ff307fSBen Skeggs
1247a5ff307fSBen Skeggs nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
1248a5ff307fSBen Skeggs start, size, false, true);
1249a5ff307fSBen Skeggs
1250a5ff307fSBen Skeggs next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
1251a5ff307fSBen Skeggs if (!WARN_ON(!next)) {
1252a5ff307fSBen Skeggs vma = next;
1253a5ff307fSBen Skeggs vma->refd = NVKM_VMA_PAGE_NONE;
1254a5ff307fSBen Skeggs vma->mapped = false;
1255a5ff307fSBen Skeggs }
1256a5ff307fSBen Skeggs } while ((vma = node(vma, next)) && (start = vma->addr) < limit);
1257a5ff307fSBen Skeggs
1258a5ff307fSBen Skeggs return 0;
1259a5ff307fSBen Skeggs }
1260a5ff307fSBen Skeggs
1261a5ff307fSBen Skeggs /*TODO:
1262a5ff307fSBen Skeggs * - Avoid PT readback (for dma_unmap etc), this might end up being dealt
1263a5ff307fSBen Skeggs * with inside HMM, which would be a lot nicer for us to deal with.
1264a5ff307fSBen Skeggs * - Support for systems without a 4KiB page size.
1265a5ff307fSBen Skeggs */
1266a5ff307fSBen Skeggs int
nvkm_vmm_pfn_map(struct nvkm_vmm * vmm,u8 shift,u64 addr,u64 size,u64 * pfn)1267a5ff307fSBen Skeggs nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
1268a5ff307fSBen Skeggs {
1269a5ff307fSBen Skeggs const struct nvkm_vmm_page *page = vmm->func->page;
1270a5ff307fSBen Skeggs struct nvkm_vma *vma, *tmp;
1271a5ff307fSBen Skeggs u64 limit = addr + size;
1272a5ff307fSBen Skeggs u64 start = addr;
1273a5ff307fSBen Skeggs int pm = size >> shift;
1274a5ff307fSBen Skeggs int pi = 0;
1275a5ff307fSBen Skeggs
1276a5ff307fSBen Skeggs /* Only support mapping where the page size of the incoming page
1277a5ff307fSBen Skeggs * array matches a page size available for direct mapping.
1278a5ff307fSBen Skeggs */
12797763d24fSRalph Campbell while (page->shift && (page->shift != shift ||
12807763d24fSRalph Campbell page->desc->func->pfn == NULL))
1281a5ff307fSBen Skeggs page++;
1282a5ff307fSBen Skeggs
1283a5ff307fSBen Skeggs if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
1284a5ff307fSBen Skeggs !IS_ALIGNED(size, 1ULL << shift) ||
1285a5ff307fSBen Skeggs addr + size < addr || addr + size > vmm->limit) {
1286a5ff307fSBen Skeggs VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
1287a5ff307fSBen Skeggs shift, page->shift, addr, size);
1288a5ff307fSBen Skeggs return -EINVAL;
1289a5ff307fSBen Skeggs }
1290a5ff307fSBen Skeggs
1291a5ff307fSBen Skeggs if (!(vma = nvkm_vmm_node_search(vmm, addr)))
1292a5ff307fSBen Skeggs return -ENOENT;
1293a5ff307fSBen Skeggs
1294a5ff307fSBen Skeggs do {
1295a5ff307fSBen Skeggs bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
1296a5ff307fSBen Skeggs bool mapped = vma->mapped;
1297a5ff307fSBen Skeggs u64 size = limit - start;
1298a5ff307fSBen Skeggs u64 addr = start;
1299a5ff307fSBen Skeggs int pn, ret = 0;
1300a5ff307fSBen Skeggs
1301a5ff307fSBen Skeggs /* Narrow the operation window to cover a single action (page
1302a5ff307fSBen Skeggs * should be mapped or not) within a single VMA.
1303a5ff307fSBen Skeggs */
1304a5ff307fSBen Skeggs for (pn = 0; pi + pn < pm; pn++) {
1305a5ff307fSBen Skeggs if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
1306a5ff307fSBen Skeggs break;
1307a5ff307fSBen Skeggs }
1308a5ff307fSBen Skeggs size = min_t(u64, size, pn << page->shift);
1309a5ff307fSBen Skeggs size = min_t(u64, size, vma->size + vma->addr - addr);
1310a5ff307fSBen Skeggs
1311a5ff307fSBen Skeggs /* Reject any operation to unmanaged regions, and areas that
1312a5ff307fSBen Skeggs * have nvkm_memory objects mapped in them already.
1313a5ff307fSBen Skeggs */
1314a5ff307fSBen Skeggs if (!vma->mapref || vma->memory) {
1315a5ff307fSBen Skeggs ret = -EINVAL;
1316a5ff307fSBen Skeggs goto next;
1317a5ff307fSBen Skeggs }
1318a5ff307fSBen Skeggs
1319a5ff307fSBen Skeggs /* In order to both properly refcount GPU page tables, and
1320a5ff307fSBen Skeggs * prevent "normal" mappings and these direct mappings from
1321a5ff307fSBen Skeggs * interfering with each other, we need to track contiguous
1322a5ff307fSBen Skeggs * ranges that have been mapped with this interface.
1323a5ff307fSBen Skeggs *
1324a5ff307fSBen Skeggs * Here we attempt to either split an existing VMA so we're
1325a5ff307fSBen Skeggs * able to flag the region as either unmapped/mapped, or to
1326a5ff307fSBen Skeggs * merge with adjacent VMAs that are already compatible.
1327a5ff307fSBen Skeggs *
1328a5ff307fSBen Skeggs * If the region is already compatible, nothing is required.
1329a5ff307fSBen Skeggs */
1330a5ff307fSBen Skeggs if (map != mapped) {
1331a5ff307fSBen Skeggs tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
1332a5ff307fSBen Skeggs page -
1333a5ff307fSBen Skeggs vmm->func->page, map);
1334a5ff307fSBen Skeggs if (WARN_ON(!tmp)) {
1335a5ff307fSBen Skeggs ret = -ENOMEM;
1336a5ff307fSBen Skeggs goto next;
1337a5ff307fSBen Skeggs }
1338a5ff307fSBen Skeggs
1339a5ff307fSBen Skeggs if ((tmp->mapped = map))
1340a5ff307fSBen Skeggs tmp->refd = page - vmm->func->page;
1341a5ff307fSBen Skeggs else
1342a5ff307fSBen Skeggs tmp->refd = NVKM_VMA_PAGE_NONE;
1343a5ff307fSBen Skeggs vma = tmp;
1344a5ff307fSBen Skeggs }
1345a5ff307fSBen Skeggs
1346a5ff307fSBen Skeggs /* Update HW page tables. */
1347a5ff307fSBen Skeggs if (map) {
1348a5ff307fSBen Skeggs struct nvkm_vmm_map args;
1349a5ff307fSBen Skeggs args.page = page;
1350a5ff307fSBen Skeggs args.pfn = &pfn[pi];
1351a5ff307fSBen Skeggs
1352a5ff307fSBen Skeggs if (!mapped) {
1353a5ff307fSBen Skeggs ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
1354a5ff307fSBen Skeggs size, &args, page->
1355a5ff307fSBen Skeggs desc->func->pfn);
1356a5ff307fSBen Skeggs } else {
1357a5ff307fSBen Skeggs nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
1358a5ff307fSBen Skeggs page->desc->func->pfn);
1359a5ff307fSBen Skeggs }
1360a5ff307fSBen Skeggs } else {
1361a5ff307fSBen Skeggs if (mapped) {
1362a5ff307fSBen Skeggs nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
1363a5ff307fSBen Skeggs false, true);
1364a5ff307fSBen Skeggs }
1365a5ff307fSBen Skeggs }
1366a5ff307fSBen Skeggs
1367a5ff307fSBen Skeggs next:
1368a5ff307fSBen Skeggs /* Iterate to next operation. */
1369a5ff307fSBen Skeggs if (vma->addr + vma->size == addr + size)
1370a5ff307fSBen Skeggs vma = node(vma, next);
1371a5ff307fSBen Skeggs start += size;
1372a5ff307fSBen Skeggs
1373a5ff307fSBen Skeggs if (ret) {
1374a5ff307fSBen Skeggs /* Failure is signalled by clearing the valid bit on
1375a5ff307fSBen Skeggs * any PFN that couldn't be modified as requested.
1376a5ff307fSBen Skeggs */
1377a5ff307fSBen Skeggs while (size) {
1378a5ff307fSBen Skeggs pfn[pi++] = NVKM_VMM_PFN_NONE;
1379a5ff307fSBen Skeggs size -= 1 << page->shift;
1380a5ff307fSBen Skeggs }
1381a5ff307fSBen Skeggs } else {
1382a5ff307fSBen Skeggs pi += size >> page->shift;
1383a5ff307fSBen Skeggs }
1384a5ff307fSBen Skeggs } while (vma && start < limit);
1385a5ff307fSBen Skeggs
1386a5ff307fSBen Skeggs return 0;
1387a5ff307fSBen Skeggs }
1388a5ff307fSBen Skeggs
1389f9463a4bSBen Skeggs void
nvkm_vmm_unmap_region(struct nvkm_vmm * vmm,struct nvkm_vma * vma)1390f9463a4bSBen Skeggs nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1391f9463a4bSBen Skeggs {
1392729eba33SBen Skeggs struct nvkm_vma *prev = NULL;
1393a5ff307fSBen Skeggs struct nvkm_vma *next;
1394f9463a4bSBen Skeggs
1395f9463a4bSBen Skeggs nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1396f9463a4bSBen Skeggs nvkm_memory_unref(&vma->memory);
13978e68271dSBen Skeggs vma->mapped = false;
1398f9463a4bSBen Skeggs
1399a5ff307fSBen Skeggs if (vma->part && (prev = node(vma, prev)) && prev->mapped)
1400729eba33SBen Skeggs prev = NULL;
1401a5ff307fSBen Skeggs if ((next = node(vma, next)) && (!next->part || next->mapped))
1402729eba33SBen Skeggs next = NULL;
1403729eba33SBen Skeggs nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
1404f9463a4bSBen Skeggs }
1405f9463a4bSBen Skeggs
1406f9463a4bSBen Skeggs void
nvkm_vmm_unmap_locked(struct nvkm_vmm * vmm,struct nvkm_vma * vma,bool pfn)1407a5ff307fSBen Skeggs nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
1408f9463a4bSBen Skeggs {
1409f9463a4bSBen Skeggs const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
1410f9463a4bSBen Skeggs
1411f9463a4bSBen Skeggs if (vma->mapref) {
1412a5ff307fSBen Skeggs nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1413f9463a4bSBen Skeggs vma->refd = NVKM_VMA_PAGE_NONE;
1414f9463a4bSBen Skeggs } else {
1415a5ff307fSBen Skeggs nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1416f9463a4bSBen Skeggs }
1417f9463a4bSBen Skeggs
1418f9463a4bSBen Skeggs nvkm_vmm_unmap_region(vmm, vma);
1419f9463a4bSBen Skeggs }
1420f9463a4bSBen Skeggs
1421f9463a4bSBen Skeggs void
nvkm_vmm_unmap(struct nvkm_vmm * vmm,struct nvkm_vma * vma)1422f9463a4bSBen Skeggs nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1423f9463a4bSBen Skeggs {
1424f9463a4bSBen Skeggs if (vma->memory) {
1425*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.vmm);
1426a5ff307fSBen Skeggs nvkm_vmm_unmap_locked(vmm, vma, false);
1427*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.vmm);
1428f9463a4bSBen Skeggs }
1429f9463a4bSBen Skeggs }
1430f9463a4bSBen Skeggs
1431f9463a4bSBen Skeggs static int
nvkm_vmm_map_valid(struct nvkm_vmm * vmm,struct nvkm_vma * vma,void * argv,u32 argc,struct nvkm_vmm_map * map)1432f9463a4bSBen Skeggs nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1433f9463a4bSBen Skeggs void *argv, u32 argc, struct nvkm_vmm_map *map)
1434f9463a4bSBen Skeggs {
1435f9463a4bSBen Skeggs switch (nvkm_memory_target(map->memory)) {
1436f9463a4bSBen Skeggs case NVKM_MEM_TARGET_VRAM:
1437f9463a4bSBen Skeggs if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
1438f9463a4bSBen Skeggs VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
1439f9463a4bSBen Skeggs return -EINVAL;
1440f9463a4bSBen Skeggs }
1441f9463a4bSBen Skeggs break;
1442f9463a4bSBen Skeggs case NVKM_MEM_TARGET_HOST:
1443f9463a4bSBen Skeggs case NVKM_MEM_TARGET_NCOH:
1444f9463a4bSBen Skeggs if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
1445f9463a4bSBen Skeggs VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
1446f9463a4bSBen Skeggs return -EINVAL;
1447f9463a4bSBen Skeggs }
1448f9463a4bSBen Skeggs break;
1449f9463a4bSBen Skeggs default:
1450f9463a4bSBen Skeggs WARN_ON(1);
1451f9463a4bSBen Skeggs return -ENOSYS;
1452f9463a4bSBen Skeggs }
1453f9463a4bSBen Skeggs
1454f9463a4bSBen Skeggs if (!IS_ALIGNED( vma->addr, 1ULL << map->page->shift) ||
1455f9463a4bSBen Skeggs !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
1456f9463a4bSBen Skeggs !IS_ALIGNED( map->offset, 1ULL << map->page->shift) ||
1457f9463a4bSBen Skeggs nvkm_memory_page(map->memory) < map->page->shift) {
1458f9463a4bSBen Skeggs VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
1459f9463a4bSBen Skeggs vma->addr, (u64)vma->size, map->offset, map->page->shift,
1460f9463a4bSBen Skeggs nvkm_memory_page(map->memory));
1461f9463a4bSBen Skeggs return -EINVAL;
1462f9463a4bSBen Skeggs }
1463f9463a4bSBen Skeggs
1464f9463a4bSBen Skeggs return vmm->func->valid(vmm, argv, argc, map);
1465f9463a4bSBen Skeggs }
1466f9463a4bSBen Skeggs
1467f9463a4bSBen Skeggs static int
nvkm_vmm_map_choose(struct nvkm_vmm * vmm,struct nvkm_vma * vma,void * argv,u32 argc,struct nvkm_vmm_map * map)1468f9463a4bSBen Skeggs nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1469f9463a4bSBen Skeggs void *argv, u32 argc, struct nvkm_vmm_map *map)
1470f9463a4bSBen Skeggs {
1471f9463a4bSBen Skeggs for (map->page = vmm->func->page; map->page->shift; map->page++) {
1472f9463a4bSBen Skeggs VMM_DEBUG(vmm, "trying %d", map->page->shift);
1473f9463a4bSBen Skeggs if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
1474f9463a4bSBen Skeggs return 0;
1475f9463a4bSBen Skeggs }
1476f9463a4bSBen Skeggs return -EINVAL;
1477f9463a4bSBen Skeggs }
1478f9463a4bSBen Skeggs
1479f9463a4bSBen Skeggs static int
nvkm_vmm_map_locked(struct nvkm_vmm * vmm,struct nvkm_vma * vma,void * argv,u32 argc,struct nvkm_vmm_map * map)1480f9463a4bSBen Skeggs nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1481f9463a4bSBen Skeggs void *argv, u32 argc, struct nvkm_vmm_map *map)
1482f9463a4bSBen Skeggs {
1483f9463a4bSBen Skeggs nvkm_vmm_pte_func func;
1484f9463a4bSBen Skeggs int ret;
1485f9463a4bSBen Skeggs
1486*6b252cf4SDanilo Krummrich map->no_comp = vma->no_comp;
1487*6b252cf4SDanilo Krummrich
1488f9463a4bSBen Skeggs /* Make sure we won't overrun the end of the memory object. */
1489f9463a4bSBen Skeggs if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
1490f9463a4bSBen Skeggs VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
1491f9463a4bSBen Skeggs nvkm_memory_size(map->memory),
1492f9463a4bSBen Skeggs map->offset, (u64)vma->size);
1493f9463a4bSBen Skeggs return -EINVAL;
1494f9463a4bSBen Skeggs }
1495f9463a4bSBen Skeggs
1496f9463a4bSBen Skeggs /* Check remaining arguments for validity. */
1497f9463a4bSBen Skeggs if (vma->page == NVKM_VMA_PAGE_NONE &&
1498f9463a4bSBen Skeggs vma->refd == NVKM_VMA_PAGE_NONE) {
1499f9463a4bSBen Skeggs /* Find the largest page size we can perform the mapping at. */
1500f9463a4bSBen Skeggs const u32 debug = vmm->debug;
1501f9463a4bSBen Skeggs vmm->debug = 0;
1502f9463a4bSBen Skeggs ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1503f9463a4bSBen Skeggs vmm->debug = debug;
1504f9463a4bSBen Skeggs if (ret) {
1505f9463a4bSBen Skeggs VMM_DEBUG(vmm, "invalid at any page size");
1506f9463a4bSBen Skeggs nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1507f9463a4bSBen Skeggs return -EINVAL;
1508f9463a4bSBen Skeggs }
1509f9463a4bSBen Skeggs } else {
1510f9463a4bSBen Skeggs /* Page size of the VMA is already pre-determined. */
1511f9463a4bSBen Skeggs if (vma->refd != NVKM_VMA_PAGE_NONE)
1512f9463a4bSBen Skeggs map->page = &vmm->func->page[vma->refd];
1513f9463a4bSBen Skeggs else
1514f9463a4bSBen Skeggs map->page = &vmm->func->page[vma->page];
1515f9463a4bSBen Skeggs
1516f9463a4bSBen Skeggs ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
1517f9463a4bSBen Skeggs if (ret) {
1518f9463a4bSBen Skeggs VMM_DEBUG(vmm, "invalid %d\n", ret);
1519f9463a4bSBen Skeggs return ret;
1520f9463a4bSBen Skeggs }
1521f9463a4bSBen Skeggs }
1522f9463a4bSBen Skeggs
1523f9463a4bSBen Skeggs /* Deal with the 'offset' argument, and fetch the backend function. */
1524f9463a4bSBen Skeggs map->off = map->offset;
1525f9463a4bSBen Skeggs if (map->mem) {
1526f9463a4bSBen Skeggs for (; map->off; map->mem = map->mem->next) {
1527f9463a4bSBen Skeggs u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
1528f9463a4bSBen Skeggs if (size > map->off)
1529f9463a4bSBen Skeggs break;
1530f9463a4bSBen Skeggs map->off -= size;
1531f9463a4bSBen Skeggs }
1532f9463a4bSBen Skeggs func = map->page->desc->func->mem;
1533f9463a4bSBen Skeggs } else
1534f9463a4bSBen Skeggs if (map->sgl) {
1535f9463a4bSBen Skeggs for (; map->off; map->sgl = sg_next(map->sgl)) {
1536f9463a4bSBen Skeggs u64 size = sg_dma_len(map->sgl);
1537f9463a4bSBen Skeggs if (size > map->off)
1538f9463a4bSBen Skeggs break;
1539f9463a4bSBen Skeggs map->off -= size;
1540f9463a4bSBen Skeggs }
1541f9463a4bSBen Skeggs func = map->page->desc->func->sgl;
1542f9463a4bSBen Skeggs } else {
1543f9463a4bSBen Skeggs map->dma += map->offset >> PAGE_SHIFT;
1544f9463a4bSBen Skeggs map->off = map->offset & PAGE_MASK;
1545f9463a4bSBen Skeggs func = map->page->desc->func->dma;
1546f9463a4bSBen Skeggs }
1547f9463a4bSBen Skeggs
1548f9463a4bSBen Skeggs /* Perform the map. */
1549f9463a4bSBen Skeggs if (vma->refd == NVKM_VMA_PAGE_NONE) {
1550f9463a4bSBen Skeggs ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
1551f9463a4bSBen Skeggs if (ret)
1552f9463a4bSBen Skeggs return ret;
1553f9463a4bSBen Skeggs
1554f9463a4bSBen Skeggs vma->refd = map->page - vmm->func->page;
1555f9463a4bSBen Skeggs } else {
1556f9463a4bSBen Skeggs nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
1557f9463a4bSBen Skeggs }
1558f9463a4bSBen Skeggs
1559f9463a4bSBen Skeggs nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1560f9463a4bSBen Skeggs nvkm_memory_unref(&vma->memory);
1561f9463a4bSBen Skeggs vma->memory = nvkm_memory_ref(map->memory);
15628e68271dSBen Skeggs vma->mapped = true;
1563f9463a4bSBen Skeggs vma->tags = map->tags;
1564f9463a4bSBen Skeggs return 0;
1565f9463a4bSBen Skeggs }
1566f9463a4bSBen Skeggs
1567f9463a4bSBen Skeggs int
nvkm_vmm_map(struct nvkm_vmm * vmm,struct nvkm_vma * vma,void * argv,u32 argc,struct nvkm_vmm_map * map)1568f9463a4bSBen Skeggs nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
1569f9463a4bSBen Skeggs struct nvkm_vmm_map *map)
1570f9463a4bSBen Skeggs {
1571f9463a4bSBen Skeggs int ret;
1572*6b252cf4SDanilo Krummrich
1573*6b252cf4SDanilo Krummrich if (nvkm_vmm_in_managed_range(vmm, vma->addr, vma->size) &&
1574*6b252cf4SDanilo Krummrich vmm->managed.raw)
1575*6b252cf4SDanilo Krummrich return nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1576*6b252cf4SDanilo Krummrich
1577*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.vmm);
1578f9463a4bSBen Skeggs ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1579f9463a4bSBen Skeggs vma->busy = false;
1580*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.vmm);
1581f9463a4bSBen Skeggs return ret;
1582f9463a4bSBen Skeggs }
1583f9463a4bSBen Skeggs
1584f9463a4bSBen Skeggs static void
nvkm_vmm_put_region(struct nvkm_vmm * vmm,struct nvkm_vma * vma)1585f9463a4bSBen Skeggs nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1586f9463a4bSBen Skeggs {
1587f9463a4bSBen Skeggs struct nvkm_vma *prev, *next;
1588f9463a4bSBen Skeggs
1589f9463a4bSBen Skeggs if ((prev = node(vma, prev)) && !prev->used) {
1590f9463a4bSBen Skeggs vma->addr = prev->addr;
1591f9463a4bSBen Skeggs vma->size += prev->size;
1592729eba33SBen Skeggs nvkm_vmm_free_delete(vmm, prev);
1593f9463a4bSBen Skeggs }
1594f9463a4bSBen Skeggs
1595f9463a4bSBen Skeggs if ((next = node(vma, next)) && !next->used) {
1596f9463a4bSBen Skeggs vma->size += next->size;
1597729eba33SBen Skeggs nvkm_vmm_free_delete(vmm, next);
1598f9463a4bSBen Skeggs }
1599f9463a4bSBen Skeggs
1600f9463a4bSBen Skeggs nvkm_vmm_free_insert(vmm, vma);
1601f9463a4bSBen Skeggs }
1602f9463a4bSBen Skeggs
1603f9463a4bSBen Skeggs void
nvkm_vmm_put_locked(struct nvkm_vmm * vmm,struct nvkm_vma * vma)1604f9463a4bSBen Skeggs nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1605f9463a4bSBen Skeggs {
1606f9463a4bSBen Skeggs const struct nvkm_vmm_page *page = vmm->func->page;
1607f9463a4bSBen Skeggs struct nvkm_vma *next = vma;
1608f9463a4bSBen Skeggs
1609f9463a4bSBen Skeggs BUG_ON(vma->part);
1610f9463a4bSBen Skeggs
1611f9463a4bSBen Skeggs if (vma->mapref || !vma->sparse) {
1612f9463a4bSBen Skeggs do {
16138e68271dSBen Skeggs const bool mem = next->memory != NULL;
16148e68271dSBen Skeggs const bool map = next->mapped;
1615f9463a4bSBen Skeggs const u8 refd = next->refd;
1616f9463a4bSBen Skeggs const u64 addr = next->addr;
1617f9463a4bSBen Skeggs u64 size = next->size;
1618f9463a4bSBen Skeggs
1619f9463a4bSBen Skeggs /* Merge regions that are in the same state. */
1620f9463a4bSBen Skeggs while ((next = node(next, next)) && next->part &&
16218e68271dSBen Skeggs (next->mapped == map) &&
16228e68271dSBen Skeggs (next->memory != NULL) == mem &&
1623f9463a4bSBen Skeggs (next->refd == refd))
1624f9463a4bSBen Skeggs size += next->size;
1625f9463a4bSBen Skeggs
1626f9463a4bSBen Skeggs if (map) {
1627f9463a4bSBen Skeggs /* Region(s) are mapped, merge the unmap
1628f9463a4bSBen Skeggs * and dereference into a single walk of
1629f9463a4bSBen Skeggs * the page tree.
1630f9463a4bSBen Skeggs */
1631f9463a4bSBen Skeggs nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
1632a5ff307fSBen Skeggs size, vma->sparse,
1633a5ff307fSBen Skeggs !mem);
1634f9463a4bSBen Skeggs } else
1635f9463a4bSBen Skeggs if (refd != NVKM_VMA_PAGE_NONE) {
1636f9463a4bSBen Skeggs /* Drop allocation-time PTE references. */
1637f9463a4bSBen Skeggs nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1638f9463a4bSBen Skeggs }
1639f9463a4bSBen Skeggs } while (next && next->part);
1640f9463a4bSBen Skeggs }
1641f9463a4bSBen Skeggs
1642f9463a4bSBen Skeggs /* Merge any mapped regions that were split from the initial
1643f9463a4bSBen Skeggs * address-space allocation back into the allocated VMA, and
1644f9463a4bSBen Skeggs * release memory/compression resources.
1645f9463a4bSBen Skeggs */
1646f9463a4bSBen Skeggs next = vma;
1647f9463a4bSBen Skeggs do {
16488e68271dSBen Skeggs if (next->mapped)
1649f9463a4bSBen Skeggs nvkm_vmm_unmap_region(vmm, next);
1650f9463a4bSBen Skeggs } while ((next = node(vma, next)) && next->part);
1651f9463a4bSBen Skeggs
1652f9463a4bSBen Skeggs if (vma->sparse && !vma->mapref) {
1653f9463a4bSBen Skeggs /* Sparse region that was allocated with a fixed page size,
1654f9463a4bSBen Skeggs * meaning all relevant PTEs were referenced once when the
1655f9463a4bSBen Skeggs * region was allocated, and remained that way, regardless
1656f9463a4bSBen Skeggs * of whether memory was mapped into it afterwards.
1657f9463a4bSBen Skeggs *
1658f9463a4bSBen Skeggs * The process of unmapping, unsparsing, and dereferencing
1659f9463a4bSBen Skeggs * PTEs can be done in a single page tree walk.
1660f9463a4bSBen Skeggs */
1661f9463a4bSBen Skeggs nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
1662f9463a4bSBen Skeggs } else
1663f9463a4bSBen Skeggs if (vma->sparse) {
1664f9463a4bSBen Skeggs /* Sparse region that wasn't allocated with a fixed page size,
1665f9463a4bSBen Skeggs * PTE references were taken both at allocation time (to make
1666f9463a4bSBen Skeggs * the GPU see the region as sparse), and when mapping memory
1667f9463a4bSBen Skeggs * into the region.
1668f9463a4bSBen Skeggs *
1669f9463a4bSBen Skeggs * The latter was handled above, and the remaining references
1670f9463a4bSBen Skeggs * are dealt with here.
1671f9463a4bSBen Skeggs */
1672f9463a4bSBen Skeggs nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
1673f9463a4bSBen Skeggs }
1674f9463a4bSBen Skeggs
1675f9463a4bSBen Skeggs /* Remove VMA from the list of allocated nodes. */
1676729eba33SBen Skeggs nvkm_vmm_node_remove(vmm, vma);
1677f9463a4bSBen Skeggs
1678f9463a4bSBen Skeggs /* Merge VMA back into the free list. */
1679f9463a4bSBen Skeggs vma->page = NVKM_VMA_PAGE_NONE;
1680f9463a4bSBen Skeggs vma->refd = NVKM_VMA_PAGE_NONE;
1681f9463a4bSBen Skeggs vma->used = false;
1682f9463a4bSBen Skeggs nvkm_vmm_put_region(vmm, vma);
1683f9463a4bSBen Skeggs }
1684f9463a4bSBen Skeggs
1685f9463a4bSBen Skeggs void
nvkm_vmm_put(struct nvkm_vmm * vmm,struct nvkm_vma ** pvma)1686f9463a4bSBen Skeggs nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
1687f9463a4bSBen Skeggs {
1688f9463a4bSBen Skeggs struct nvkm_vma *vma = *pvma;
1689f9463a4bSBen Skeggs if (vma) {
1690*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.vmm);
1691f9463a4bSBen Skeggs nvkm_vmm_put_locked(vmm, vma);
1692*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.vmm);
1693f9463a4bSBen Skeggs *pvma = NULL;
1694f9463a4bSBen Skeggs }
1695f9463a4bSBen Skeggs }
1696f9463a4bSBen Skeggs
1697f9463a4bSBen Skeggs int
nvkm_vmm_get_locked(struct nvkm_vmm * vmm,bool getref,bool mapref,bool sparse,u8 shift,u8 align,u64 size,struct nvkm_vma ** pvma)1698f9463a4bSBen Skeggs nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
1699f9463a4bSBen Skeggs u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
1700f9463a4bSBen Skeggs {
1701f9463a4bSBen Skeggs const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
1702f9463a4bSBen Skeggs struct rb_node *node = NULL, *temp;
1703f9463a4bSBen Skeggs struct nvkm_vma *vma = NULL, *tmp;
1704f9463a4bSBen Skeggs u64 addr, tail;
1705f9463a4bSBen Skeggs int ret;
1706f9463a4bSBen Skeggs
1707f9463a4bSBen Skeggs VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
1708f9463a4bSBen Skeggs "shift: %d align: %d size: %016llx",
1709f9463a4bSBen Skeggs getref, mapref, sparse, shift, align, size);
1710f9463a4bSBen Skeggs
1711f9463a4bSBen Skeggs /* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
1712f9463a4bSBen Skeggs if (unlikely(!size || (!getref && !mapref && sparse))) {
1713f9463a4bSBen Skeggs VMM_DEBUG(vmm, "args %016llx %d %d %d",
1714f9463a4bSBen Skeggs size, getref, mapref, sparse);
1715f9463a4bSBen Skeggs return -EINVAL;
1716f9463a4bSBen Skeggs }
1717f9463a4bSBen Skeggs
1718f9463a4bSBen Skeggs /* Tesla-class GPUs can only select page size per-PDE, which means
1719f9463a4bSBen Skeggs * we're required to know the mapping granularity up-front to find
1720f9463a4bSBen Skeggs * a suitable region of address-space.
1721f9463a4bSBen Skeggs *
1722f9463a4bSBen Skeggs * The same goes if we're requesting up-front allocation of PTES.
1723f9463a4bSBen Skeggs */
1724f9463a4bSBen Skeggs if (unlikely((getref || vmm->func->page_block) && !shift)) {
1725f9463a4bSBen Skeggs VMM_DEBUG(vmm, "page size required: %d %016llx",
1726f9463a4bSBen Skeggs getref, vmm->func->page_block);
1727f9463a4bSBen Skeggs return -EINVAL;
1728f9463a4bSBen Skeggs }
1729f9463a4bSBen Skeggs
1730f9463a4bSBen Skeggs /* If a specific page size was requested, determine its index and
1731f9463a4bSBen Skeggs * make sure the requested size is a multiple of the page size.
1732f9463a4bSBen Skeggs */
1733f9463a4bSBen Skeggs if (shift) {
1734f9463a4bSBen Skeggs for (page = vmm->func->page; page->shift; page++) {
1735f9463a4bSBen Skeggs if (shift == page->shift)
1736f9463a4bSBen Skeggs break;
1737f9463a4bSBen Skeggs }
1738f9463a4bSBen Skeggs
1739f9463a4bSBen Skeggs if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
1740f9463a4bSBen Skeggs VMM_DEBUG(vmm, "page %d %016llx", shift, size);
1741f9463a4bSBen Skeggs return -EINVAL;
1742f9463a4bSBen Skeggs }
1743f9463a4bSBen Skeggs align = max_t(u8, align, shift);
1744f9463a4bSBen Skeggs } else {
1745f9463a4bSBen Skeggs align = max_t(u8, align, 12);
1746f9463a4bSBen Skeggs }
1747f9463a4bSBen Skeggs
1748f9463a4bSBen Skeggs /* Locate smallest block that can possibly satisfy the allocation. */
1749f9463a4bSBen Skeggs temp = vmm->free.rb_node;
1750f9463a4bSBen Skeggs while (temp) {
1751f9463a4bSBen Skeggs struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
1752f9463a4bSBen Skeggs if (this->size < size) {
1753f9463a4bSBen Skeggs temp = temp->rb_right;
1754f9463a4bSBen Skeggs } else {
1755f9463a4bSBen Skeggs node = temp;
1756f9463a4bSBen Skeggs temp = temp->rb_left;
1757f9463a4bSBen Skeggs }
1758f9463a4bSBen Skeggs }
1759f9463a4bSBen Skeggs
1760f9463a4bSBen Skeggs if (unlikely(!node))
1761f9463a4bSBen Skeggs return -ENOSPC;
1762f9463a4bSBen Skeggs
1763f9463a4bSBen Skeggs /* Take into account alignment restrictions, trying larger blocks
1764f9463a4bSBen Skeggs * in turn until we find a suitable free block.
1765f9463a4bSBen Skeggs */
1766f9463a4bSBen Skeggs do {
1767f9463a4bSBen Skeggs struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
1768f9463a4bSBen Skeggs struct nvkm_vma *prev = node(this, prev);
1769f9463a4bSBen Skeggs struct nvkm_vma *next = node(this, next);
1770f9463a4bSBen Skeggs const int p = page - vmm->func->page;
1771f9463a4bSBen Skeggs
1772f9463a4bSBen Skeggs addr = this->addr;
1773f9463a4bSBen Skeggs if (vmm->func->page_block && prev && prev->page != p)
17746497c2baSBen Skeggs addr = ALIGN(addr, vmm->func->page_block);
1775f9463a4bSBen Skeggs addr = ALIGN(addr, 1ULL << align);
1776f9463a4bSBen Skeggs
1777f9463a4bSBen Skeggs tail = this->addr + this->size;
1778f9463a4bSBen Skeggs if (vmm->func->page_block && next && next->page != p)
1779da5e45e6SMāris Nartišs tail = ALIGN_DOWN(tail, vmm->func->page_block);
1780f9463a4bSBen Skeggs
1781f9463a4bSBen Skeggs if (addr <= tail && tail - addr >= size) {
1782729eba33SBen Skeggs nvkm_vmm_free_remove(vmm, this);
1783f9463a4bSBen Skeggs vma = this;
1784f9463a4bSBen Skeggs break;
1785f9463a4bSBen Skeggs }
1786f9463a4bSBen Skeggs } while ((node = rb_next(node)));
1787f9463a4bSBen Skeggs
1788f9463a4bSBen Skeggs if (unlikely(!vma))
1789f9463a4bSBen Skeggs return -ENOSPC;
1790f9463a4bSBen Skeggs
1791f9463a4bSBen Skeggs /* If the VMA we found isn't already exactly the requested size,
1792f9463a4bSBen Skeggs * it needs to be split, and the remaining free blocks returned.
1793f9463a4bSBen Skeggs */
1794f9463a4bSBen Skeggs if (addr != vma->addr) {
1795f9463a4bSBen Skeggs if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
1796f9463a4bSBen Skeggs nvkm_vmm_put_region(vmm, vma);
1797f9463a4bSBen Skeggs return -ENOMEM;
1798f9463a4bSBen Skeggs }
1799f9463a4bSBen Skeggs nvkm_vmm_free_insert(vmm, vma);
1800f9463a4bSBen Skeggs vma = tmp;
1801f9463a4bSBen Skeggs }
1802f9463a4bSBen Skeggs
1803f9463a4bSBen Skeggs if (size != vma->size) {
1804f9463a4bSBen Skeggs if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
1805f9463a4bSBen Skeggs nvkm_vmm_put_region(vmm, vma);
1806f9463a4bSBen Skeggs return -ENOMEM;
1807f9463a4bSBen Skeggs }
1808f9463a4bSBen Skeggs nvkm_vmm_free_insert(vmm, tmp);
1809f9463a4bSBen Skeggs }
1810f9463a4bSBen Skeggs
1811f9463a4bSBen Skeggs /* Pre-allocate page tables and/or setup sparse mappings. */
1812f9463a4bSBen Skeggs if (sparse && getref)
1813f9463a4bSBen Skeggs ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
1814f9463a4bSBen Skeggs else if (sparse)
1815f9463a4bSBen Skeggs ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
1816f9463a4bSBen Skeggs else if (getref)
1817f9463a4bSBen Skeggs ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
1818f9463a4bSBen Skeggs else
1819f9463a4bSBen Skeggs ret = 0;
1820f9463a4bSBen Skeggs if (ret) {
1821f9463a4bSBen Skeggs nvkm_vmm_put_region(vmm, vma);
1822f9463a4bSBen Skeggs return ret;
1823f9463a4bSBen Skeggs }
1824f9463a4bSBen Skeggs
1825f9463a4bSBen Skeggs vma->mapref = mapref && !getref;
1826f9463a4bSBen Skeggs vma->sparse = sparse;
1827f9463a4bSBen Skeggs vma->page = page - vmm->func->page;
1828f9463a4bSBen Skeggs vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
1829f9463a4bSBen Skeggs vma->used = true;
1830f9463a4bSBen Skeggs nvkm_vmm_node_insert(vmm, vma);
1831f9463a4bSBen Skeggs *pvma = vma;
1832f9463a4bSBen Skeggs return 0;
1833f9463a4bSBen Skeggs }
1834f9463a4bSBen Skeggs
1835f9463a4bSBen Skeggs int
nvkm_vmm_get(struct nvkm_vmm * vmm,u8 page,u64 size,struct nvkm_vma ** pvma)1836f9463a4bSBen Skeggs nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
1837f9463a4bSBen Skeggs {
1838f9463a4bSBen Skeggs int ret;
1839*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.vmm);
1840f9463a4bSBen Skeggs ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
1841*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.vmm);
1842*6b252cf4SDanilo Krummrich return ret;
1843*6b252cf4SDanilo Krummrich }
1844*6b252cf4SDanilo Krummrich
1845*6b252cf4SDanilo Krummrich void
nvkm_vmm_raw_unmap(struct nvkm_vmm * vmm,u64 addr,u64 size,bool sparse,u8 refd)1846*6b252cf4SDanilo Krummrich nvkm_vmm_raw_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size,
1847*6b252cf4SDanilo Krummrich bool sparse, u8 refd)
1848*6b252cf4SDanilo Krummrich {
1849*6b252cf4SDanilo Krummrich const struct nvkm_vmm_page *page = &vmm->func->page[refd];
1850*6b252cf4SDanilo Krummrich
1851*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_unmap(vmm, page, addr, size, sparse, false);
1852*6b252cf4SDanilo Krummrich }
1853*6b252cf4SDanilo Krummrich
1854*6b252cf4SDanilo Krummrich void
nvkm_vmm_raw_put(struct nvkm_vmm * vmm,u64 addr,u64 size,u8 refd)1855*6b252cf4SDanilo Krummrich nvkm_vmm_raw_put(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd)
1856*6b252cf4SDanilo Krummrich {
1857*6b252cf4SDanilo Krummrich const struct nvkm_vmm_page *page = vmm->func->page;
1858*6b252cf4SDanilo Krummrich
1859*6b252cf4SDanilo Krummrich nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1860*6b252cf4SDanilo Krummrich }
1861*6b252cf4SDanilo Krummrich
1862*6b252cf4SDanilo Krummrich int
nvkm_vmm_raw_get(struct nvkm_vmm * vmm,u64 addr,u64 size,u8 refd)1863*6b252cf4SDanilo Krummrich nvkm_vmm_raw_get(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd)
1864*6b252cf4SDanilo Krummrich {
1865*6b252cf4SDanilo Krummrich const struct nvkm_vmm_page *page = vmm->func->page;
1866*6b252cf4SDanilo Krummrich
1867*6b252cf4SDanilo Krummrich if (unlikely(!size))
1868*6b252cf4SDanilo Krummrich return -EINVAL;
1869*6b252cf4SDanilo Krummrich
1870*6b252cf4SDanilo Krummrich return nvkm_vmm_ptes_get(vmm, &page[refd], addr, size);
1871*6b252cf4SDanilo Krummrich }
1872*6b252cf4SDanilo Krummrich
1873*6b252cf4SDanilo Krummrich int
nvkm_vmm_raw_sparse(struct nvkm_vmm * vmm,u64 addr,u64 size,bool ref)1874*6b252cf4SDanilo Krummrich nvkm_vmm_raw_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
1875*6b252cf4SDanilo Krummrich {
1876*6b252cf4SDanilo Krummrich int ret;
1877*6b252cf4SDanilo Krummrich
1878*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.ref);
1879*6b252cf4SDanilo Krummrich ret = nvkm_vmm_ptes_sparse(vmm, addr, size, ref);
1880*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.ref);
1881*6b252cf4SDanilo Krummrich
1882f9463a4bSBen Skeggs return ret;
1883f9463a4bSBen Skeggs }
1884f9463a4bSBen Skeggs
1885f9463a4bSBen Skeggs void
nvkm_vmm_part(struct nvkm_vmm * vmm,struct nvkm_memory * inst)1886f9463a4bSBen Skeggs nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1887f9463a4bSBen Skeggs {
188815516bf9SJon Derrick if (inst && vmm && vmm->func->part) {
1889*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.vmm);
1890f9463a4bSBen Skeggs vmm->func->part(vmm, inst);
1891*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.vmm);
1892f9463a4bSBen Skeggs }
1893f9463a4bSBen Skeggs }
1894f9463a4bSBen Skeggs
1895f9463a4bSBen Skeggs int
nvkm_vmm_join(struct nvkm_vmm * vmm,struct nvkm_memory * inst)1896f9463a4bSBen Skeggs nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1897f9463a4bSBen Skeggs {
1898f9463a4bSBen Skeggs int ret = 0;
1899f9463a4bSBen Skeggs if (vmm->func->join) {
1900*6b252cf4SDanilo Krummrich mutex_lock(&vmm->mutex.vmm);
1901f9463a4bSBen Skeggs ret = vmm->func->join(vmm, inst);
1902*6b252cf4SDanilo Krummrich mutex_unlock(&vmm->mutex.vmm);
1903f9463a4bSBen Skeggs }
1904f9463a4bSBen Skeggs return ret;
1905f9463a4bSBen Skeggs }
1906f9463a4bSBen Skeggs
1907eb813999SBen Skeggs static bool
nvkm_vmm_boot_ptes(struct nvkm_vmm_iter * it,bool pfn,u32 ptei,u32 ptes)1908a5ff307fSBen Skeggs nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
1909eb813999SBen Skeggs {
1910eb813999SBen Skeggs const struct nvkm_vmm_desc *desc = it->desc;
1911eb813999SBen Skeggs const int type = desc->type == SPT;
1912eb813999SBen Skeggs nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
1913eb813999SBen Skeggs return false;
1914eb813999SBen Skeggs }
1915eb813999SBen Skeggs
1916eb813999SBen Skeggs int
nvkm_vmm_boot(struct nvkm_vmm * vmm)1917eb813999SBen Skeggs nvkm_vmm_boot(struct nvkm_vmm *vmm)
1918eb813999SBen Skeggs {
1919eb813999SBen Skeggs const struct nvkm_vmm_page *page = vmm->func->page;
1920eb813999SBen Skeggs const u64 limit = vmm->limit - vmm->start;
1921eb813999SBen Skeggs int ret;
1922eb813999SBen Skeggs
1923eb813999SBen Skeggs while (page[1].shift)
1924eb813999SBen Skeggs page++;
1925eb813999SBen Skeggs
1926eb813999SBen Skeggs ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
1927eb813999SBen Skeggs if (ret)
1928eb813999SBen Skeggs return ret;
1929eb813999SBen Skeggs
1930a5ff307fSBen Skeggs nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
1931eb813999SBen Skeggs nvkm_vmm_boot_ptes, NULL, NULL, NULL);
1932eb813999SBen Skeggs vmm->bootstrapped = true;
1933eb813999SBen Skeggs return 0;
1934eb813999SBen Skeggs }
1935f9463a4bSBen Skeggs
1936f9463a4bSBen Skeggs static void
nvkm_vmm_del(struct kref * kref)1937f9463a4bSBen Skeggs nvkm_vmm_del(struct kref *kref)
1938f9463a4bSBen Skeggs {
1939f9463a4bSBen Skeggs struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
1940f9463a4bSBen Skeggs nvkm_vmm_dtor(vmm);
1941f9463a4bSBen Skeggs kfree(vmm);
1942f9463a4bSBen Skeggs }
1943f9463a4bSBen Skeggs
1944f9463a4bSBen Skeggs void
nvkm_vmm_unref(struct nvkm_vmm ** pvmm)1945f9463a4bSBen Skeggs nvkm_vmm_unref(struct nvkm_vmm **pvmm)
1946f9463a4bSBen Skeggs {
1947f9463a4bSBen Skeggs struct nvkm_vmm *vmm = *pvmm;
1948f9463a4bSBen Skeggs if (vmm) {
1949f9463a4bSBen Skeggs kref_put(&vmm->kref, nvkm_vmm_del);
1950f9463a4bSBen Skeggs *pvmm = NULL;
1951f9463a4bSBen Skeggs }
1952f9463a4bSBen Skeggs }
1953f9463a4bSBen Skeggs
1954f9463a4bSBen Skeggs struct nvkm_vmm *
nvkm_vmm_ref(struct nvkm_vmm * vmm)1955f9463a4bSBen Skeggs nvkm_vmm_ref(struct nvkm_vmm *vmm)
1956f9463a4bSBen Skeggs {
1957f9463a4bSBen Skeggs if (vmm)
1958f9463a4bSBen Skeggs kref_get(&vmm->kref);
1959f9463a4bSBen Skeggs return vmm;
1960f9463a4bSBen Skeggs }
1961f9463a4bSBen Skeggs
1962f9463a4bSBen Skeggs int
nvkm_vmm_new(struct nvkm_device * device,u64 addr,u64 size,void * argv,u32 argc,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)1963f9463a4bSBen Skeggs nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
1964f9463a4bSBen Skeggs u32 argc, struct lock_class_key *key, const char *name,
1965f9463a4bSBen Skeggs struct nvkm_vmm **pvmm)
1966f9463a4bSBen Skeggs {
1967f9463a4bSBen Skeggs struct nvkm_mmu *mmu = device->mmu;
1968f9463a4bSBen Skeggs struct nvkm_vmm *vmm = NULL;
1969f9463a4bSBen Skeggs int ret;
19702606f291SBen Skeggs ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc,
19712606f291SBen Skeggs key, name, &vmm);
1972f9463a4bSBen Skeggs if (ret)
1973f9463a4bSBen Skeggs nvkm_vmm_unref(&vmm);
1974f9463a4bSBen Skeggs *pvmm = vmm;
1975f9463a4bSBen Skeggs return ret;
1976f9463a4bSBen Skeggs }
1977