1806a7335SBen Skeggs #ifndef __NVKM_VMM_H__
2806a7335SBen Skeggs #define __NVKM_VMM_H__
3806a7335SBen Skeggs #include "priv.h"
4806a7335SBen Skeggs #include <core/memory.h>
5eb813999SBen Skeggs enum nvkm_memory_target;
6806a7335SBen Skeggs
7806a7335SBen Skeggs struct nvkm_vmm_pt {
8806a7335SBen Skeggs /* Some GPUs have a mapping level with a dual page tables to
9806a7335SBen Skeggs * support large and small pages in the same address-range.
10806a7335SBen Skeggs *
11806a7335SBen Skeggs * We track the state of both page tables in one place, which
12806a7335SBen Skeggs * is why there's multiple PT pointers/refcounts here.
13806a7335SBen Skeggs */
14806a7335SBen Skeggs struct nvkm_mmu_pt *pt[2];
15806a7335SBen Skeggs u32 refs[2];
16806a7335SBen Skeggs
17806a7335SBen Skeggs /* Page size handled by this PT.
18806a7335SBen Skeggs *
19806a7335SBen Skeggs * Tesla backend needs to know this when writinge PDEs,
20806a7335SBen Skeggs * otherwise unnecessary.
21806a7335SBen Skeggs */
22806a7335SBen Skeggs u8 page;
23806a7335SBen Skeggs
24806a7335SBen Skeggs /* Entire page table sparse.
25806a7335SBen Skeggs *
26806a7335SBen Skeggs * Used to propagate sparseness to child page tables.
27806a7335SBen Skeggs */
28806a7335SBen Skeggs bool sparse:1;
29806a7335SBen Skeggs
30806a7335SBen Skeggs /* Tracking for page directories.
31806a7335SBen Skeggs *
32806a7335SBen Skeggs * The array is indexed by PDE, and will either point to the
33806a7335SBen Skeggs * child page table, or indicate the PDE is marked as sparse.
34806a7335SBen Skeggs **/
35806a7335SBen Skeggs #define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde)
36806a7335SBen Skeggs #define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde)
37806a7335SBen Skeggs #define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY)
38806a7335SBen Skeggs struct nvkm_vmm_pt **pde;
39806a7335SBen Skeggs
40806a7335SBen Skeggs /* Tracking for dual page tables.
41806a7335SBen Skeggs *
42806a7335SBen Skeggs * There's one entry for each LPTE, keeping track of whether
43806a7335SBen Skeggs * there are valid SPTEs in the same address-range.
44806a7335SBen Skeggs *
45806a7335SBen Skeggs * This information is used to manage LPTE state transitions.
46806a7335SBen Skeggs */
47806a7335SBen Skeggs #define NVKM_VMM_PTE_SPARSE 0x80
48806a7335SBen Skeggs #define NVKM_VMM_PTE_VALID 0x40
49806a7335SBen Skeggs #define NVKM_VMM_PTE_SPTES 0x3f
50806a7335SBen Skeggs u8 pte[];
51806a7335SBen Skeggs };
52806a7335SBen Skeggs
53eb813999SBen Skeggs typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *,
54eb813999SBen Skeggs struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
55eb813999SBen Skeggs typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *,
56eb813999SBen Skeggs struct nvkm_vmm_pt *, u32 pdei);
57eb813999SBen Skeggs typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *,
58eb813999SBen Skeggs u32 ptei, u32 ptes, struct nvkm_vmm_map *);
59eb813999SBen Skeggs
60806a7335SBen Skeggs struct nvkm_vmm_desc_func {
61eb813999SBen Skeggs nvkm_vmm_pxe_func invalid;
62eb813999SBen Skeggs nvkm_vmm_pxe_func unmap;
63eb813999SBen Skeggs nvkm_vmm_pxe_func sparse;
64eb813999SBen Skeggs
65eb813999SBen Skeggs nvkm_vmm_pde_func pde;
66eb813999SBen Skeggs
67eb813999SBen Skeggs nvkm_vmm_pte_func mem;
68eb813999SBen Skeggs nvkm_vmm_pte_func dma;
69eb813999SBen Skeggs nvkm_vmm_pte_func sgl;
70a5ff307fSBen Skeggs
71a5ff307fSBen Skeggs nvkm_vmm_pte_func pfn;
72a5ff307fSBen Skeggs bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
73a5ff307fSBen Skeggs nvkm_vmm_pxe_func pfn_unmap;
74806a7335SBen Skeggs };
75806a7335SBen Skeggs
76540a1ddeSBen Skeggs extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;
77b77791daSBen Skeggs void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32);
78540a1ddeSBen Skeggs extern const struct nvkm_vmm_desc_func gf100_vmm_pgt;
79b77791daSBen Skeggs void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
80b77791daSBen Skeggs void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
81b77791daSBen Skeggs struct nvkm_vmm_map *);
82b77791daSBen Skeggs void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
83b77791daSBen Skeggs struct nvkm_vmm_map *);
84b77791daSBen Skeggs void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
85b77791daSBen Skeggs struct nvkm_vmm_map *);
86540a1ddeSBen Skeggs
87b5977643SBen Skeggs void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
88b5977643SBen Skeggs
89806a7335SBen Skeggs struct nvkm_vmm_desc {
90806a7335SBen Skeggs enum {
91806a7335SBen Skeggs PGD,
92806a7335SBen Skeggs PGT,
93806a7335SBen Skeggs SPT,
94806a7335SBen Skeggs LPT,
95806a7335SBen Skeggs } type;
96806a7335SBen Skeggs u8 bits; /* VMA bits covered by PT. */
97806a7335SBen Skeggs u8 size; /* Bytes-per-PTE. */
98806a7335SBen Skeggs u32 align; /* PT address alignment. */
99806a7335SBen Skeggs const struct nvkm_vmm_desc_func *func;
100806a7335SBen Skeggs };
101806a7335SBen Skeggs
1022ffa64ebSBen Skeggs extern const struct nvkm_vmm_desc nv50_vmm_desc_12[];
1032ffa64ebSBen Skeggs extern const struct nvkm_vmm_desc nv50_vmm_desc_16[];
1042ffa64ebSBen Skeggs
1057de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[];
1067de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[];
1077de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[];
1087de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[];
1097de078aaSBen Skeggs
1105f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[];
1115f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[];
1125f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[];
1135f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[];
1145f300fedSBen Skeggs
1158e39abffSBen Skeggs extern const struct nvkm_vmm_desc gp100_vmm_desc_12[];
1168e39abffSBen Skeggs extern const struct nvkm_vmm_desc gp100_vmm_desc_16[];
1178e39abffSBen Skeggs
118806a7335SBen Skeggs struct nvkm_vmm_page {
119806a7335SBen Skeggs u8 shift;
120806a7335SBen Skeggs const struct nvkm_vmm_desc *desc;
121806a7335SBen Skeggs #define NVKM_VMM_PAGE_SPARSE 0x01
122806a7335SBen Skeggs #define NVKM_VMM_PAGE_VRAM 0x02
123806a7335SBen Skeggs #define NVKM_VMM_PAGE_HOST 0x04
124806a7335SBen Skeggs #define NVKM_VMM_PAGE_COMP 0x08
125806a7335SBen Skeggs #define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE)
126806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM)
127806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM)
128806a7335SBen Skeggs #define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST)
129806a7335SBen Skeggs #define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST)
130806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST)
131806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST)
132806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP)
133806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP)
134806a7335SBen Skeggs #define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP)
135806a7335SBen Skeggs #define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP)
136806a7335SBen Skeggs u8 type;
137806a7335SBen Skeggs };
138806a7335SBen Skeggs
139806a7335SBen Skeggs struct nvkm_vmm_func {
140806a7335SBen Skeggs int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst);
141806a7335SBen Skeggs void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst);
142806a7335SBen Skeggs
143eb813999SBen Skeggs int (*aper)(enum nvkm_memory_target);
144eb813999SBen Skeggs int (*valid)(struct nvkm_vmm *, void *argv, u32 argc,
145eb813999SBen Skeggs struct nvkm_vmm_map *);
146eb813999SBen Skeggs void (*flush)(struct nvkm_vmm *, int depth);
147eb813999SBen Skeggs
14871871aa6SBen Skeggs int (*mthd)(struct nvkm_vmm *, struct nvkm_client *,
14971871aa6SBen Skeggs u32 mthd, void *argv, u32 argc);
15071871aa6SBen Skeggs
151d389fd4fSBen Skeggs void (*invalidate_pdb)(struct nvkm_vmm *, u64 addr);
152d389fd4fSBen Skeggs
153806a7335SBen Skeggs u64 page_block;
154806a7335SBen Skeggs const struct nvkm_vmm_page page[];
155806a7335SBen Skeggs };
156806a7335SBen Skeggs
1579f6219fdSBen Skeggs struct nvkm_vmm_join {
1589f6219fdSBen Skeggs struct nvkm_memory *inst;
1599f6219fdSBen Skeggs struct list_head head;
1609f6219fdSBen Skeggs };
1619f6219fdSBen Skeggs
162806a7335SBen Skeggs int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *,
1632606f291SBen Skeggs u32 pd_header, bool managed, u64 addr, u64 size,
1642606f291SBen Skeggs struct lock_class_key *, const char *name,
1652606f291SBen Skeggs struct nvkm_vmm **);
166*6b252cf4SDanilo Krummrich struct nvkm_vma *nvkm_vma_new(u64 addr, u64 size);
167f9463a4bSBen Skeggs struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr);
168729eba33SBen Skeggs struct nvkm_vma *nvkm_vmm_node_split(struct nvkm_vmm *, struct nvkm_vma *,
169729eba33SBen Skeggs u64 addr, u64 size);
170f9463a4bSBen Skeggs int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
171f9463a4bSBen Skeggs bool sparse, u8 page, u8 align, u64 size,
172f9463a4bSBen Skeggs struct nvkm_vma **pvma);
173f9463a4bSBen Skeggs void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
174a5ff307fSBen Skeggs void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn);
175a5ff307fSBen Skeggs void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *);
176a5ff307fSBen Skeggs
177*6b252cf4SDanilo Krummrich int nvkm_vmm_raw_get(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd);
178*6b252cf4SDanilo Krummrich void nvkm_vmm_raw_put(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd);
179*6b252cf4SDanilo Krummrich void nvkm_vmm_raw_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size,
180*6b252cf4SDanilo Krummrich bool sparse, u8 refd);
181*6b252cf4SDanilo Krummrich int nvkm_vmm_raw_sparse(struct nvkm_vmm *, u64 addr, u64 size, bool ref);
182*6b252cf4SDanilo Krummrich
183*6b252cf4SDanilo Krummrich static inline bool
nvkm_vmm_in_managed_range(struct nvkm_vmm * vmm,u64 start,u64 size)184*6b252cf4SDanilo Krummrich nvkm_vmm_in_managed_range(struct nvkm_vmm *vmm, u64 start, u64 size)
185*6b252cf4SDanilo Krummrich {
186*6b252cf4SDanilo Krummrich u64 p_start = vmm->managed.p.addr;
187*6b252cf4SDanilo Krummrich u64 p_end = p_start + vmm->managed.p.size;
188*6b252cf4SDanilo Krummrich u64 n_start = vmm->managed.n.addr;
189*6b252cf4SDanilo Krummrich u64 n_end = n_start + vmm->managed.n.size;
190*6b252cf4SDanilo Krummrich u64 end = start + size;
191*6b252cf4SDanilo Krummrich
192*6b252cf4SDanilo Krummrich if (start >= p_start && end <= p_end)
193*6b252cf4SDanilo Krummrich return true;
194*6b252cf4SDanilo Krummrich
195*6b252cf4SDanilo Krummrich if (start >= n_start && end <= n_end)
196*6b252cf4SDanilo Krummrich return true;
197*6b252cf4SDanilo Krummrich
198*6b252cf4SDanilo Krummrich return false;
199*6b252cf4SDanilo Krummrich }
200*6b252cf4SDanilo Krummrich
201a5ff307fSBen Skeggs #define NVKM_VMM_PFN_ADDR 0xfffffffffffff000ULL
202a5ff307fSBen Skeggs #define NVKM_VMM_PFN_ADDR_SHIFT 12
203a5ff307fSBen Skeggs #define NVKM_VMM_PFN_APER 0x00000000000000f0ULL
204a5ff307fSBen Skeggs #define NVKM_VMM_PFN_HOST 0x0000000000000000ULL
205a5ff307fSBen Skeggs #define NVKM_VMM_PFN_VRAM 0x0000000000000010ULL
2068f187163SAlistair Popple #define NVKM_VMM_PFN_A 0x0000000000000004ULL
207a5ff307fSBen Skeggs #define NVKM_VMM_PFN_W 0x0000000000000002ULL
208a5ff307fSBen Skeggs #define NVKM_VMM_PFN_V 0x0000000000000001ULL
209a5ff307fSBen Skeggs #define NVKM_VMM_PFN_NONE 0x0000000000000000ULL
210a5ff307fSBen Skeggs
211a5ff307fSBen Skeggs int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn);
212a5ff307fSBen Skeggs int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size);
213f9463a4bSBen Skeggs
214f9463a4bSBen Skeggs struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
215806a7335SBen Skeggs
2165b17f362SBen Skeggs int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32,
2172606f291SBen Skeggs bool, u64, u64, void *, u32, struct lock_class_key *,
2185b17f362SBen Skeggs const char *, struct nvkm_vmm **);
219dd12d158SBen Skeggs int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
22077783435SBen Skeggs
2212ffa64ebSBen Skeggs int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
2222ffa64ebSBen Skeggs void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
2232ffa64ebSBen Skeggs int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
2242ffa64ebSBen Skeggs void nv50_vmm_flush(struct nvkm_vmm *, int);
2252ffa64ebSBen Skeggs
226540a1ddeSBen Skeggs int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
2272606f291SBen Skeggs struct nvkm_mmu *, bool, u64, u64, void *, u32,
228540a1ddeSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
229540a1ddeSBen Skeggs int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
230540a1ddeSBen Skeggs int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
231540a1ddeSBen Skeggs void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
232b77791daSBen Skeggs int gf100_vmm_aper(enum nvkm_memory_target);
233b77791daSBen Skeggs int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
234b77791daSBen Skeggs void gf100_vmm_flush(struct nvkm_vmm *, int);
235d389fd4fSBen Skeggs void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type);
236d389fd4fSBen Skeggs void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr);
237540a1ddeSBen Skeggs
238b5977643SBen Skeggs int gk20a_vmm_aper(enum nvkm_memory_target);
239b5977643SBen Skeggs
2405f300fedSBen Skeggs int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
2412606f291SBen Skeggs struct nvkm_mmu *, bool, u64, u64, void *, u32,
2425f300fedSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2435f300fedSBen Skeggs int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
2445f300fedSBen Skeggs int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
2455f300fedSBen Skeggs
246ab2ee9ffSBen Skeggs int gp100_vmm_new_(const struct nvkm_vmm_func *,
247ab2ee9ffSBen Skeggs struct nvkm_mmu *, bool, u64, u64, void *, u32,
248ab2ee9ffSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2498e39abffSBen Skeggs int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
250f9400afbSBen Skeggs int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
251f9400afbSBen Skeggs void gp100_vmm_flush(struct nvkm_vmm *, int);
25271871aa6SBen Skeggs int gp100_vmm_mthd(struct nvkm_vmm *, struct nvkm_client *, u32, void *, u32);
253d389fd4fSBen Skeggs void gp100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr);
2548e39abffSBen Skeggs
2557986f813SBen Skeggs int gv100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
2567986f813SBen Skeggs
2572606f291SBen Skeggs int nv04_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2585b17f362SBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2592606f291SBen Skeggs int nv41_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
26077783435SBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2612606f291SBen Skeggs int nv44_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
26203b0ba7bSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2632606f291SBen Skeggs int nv50_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2649f6219fdSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2652606f291SBen Skeggs int mcp77_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2662ffa64ebSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2672606f291SBen Skeggs int g84_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2689f6219fdSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2692606f291SBen Skeggs int gf100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
270540a1ddeSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2712606f291SBen Skeggs int gk104_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2727de078aaSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2732606f291SBen Skeggs int gk20a_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2747de078aaSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **);
2752606f291SBen Skeggs int gm200_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2765f300fedSBen Skeggs struct lock_class_key *, const char *,
2775f300fedSBen Skeggs struct nvkm_vmm **);
2782606f291SBen Skeggs int gm200_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2795f300fedSBen Skeggs struct lock_class_key *, const char *,
2805f300fedSBen Skeggs struct nvkm_vmm **);
2812606f291SBen Skeggs int gm20b_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2825f300fedSBen Skeggs struct lock_class_key *, const char *,
2835f300fedSBen Skeggs struct nvkm_vmm **);
2842606f291SBen Skeggs int gm20b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2855f300fedSBen Skeggs struct lock_class_key *, const char *,
2865f300fedSBen Skeggs struct nvkm_vmm **);
2872606f291SBen Skeggs int gp100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2888e39abffSBen Skeggs struct lock_class_key *, const char *,
2898e39abffSBen Skeggs struct nvkm_vmm **);
2902606f291SBen Skeggs int gp10b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2918e39abffSBen Skeggs struct lock_class_key *, const char *,
2928e39abffSBen Skeggs struct nvkm_vmm **);
2932606f291SBen Skeggs int gv100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
294edf50395SBen Skeggs struct lock_class_key *, const char *,
295edf50395SBen Skeggs struct nvkm_vmm **);
2962606f291SBen Skeggs int tu102_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,
2977986f813SBen Skeggs struct lock_class_key *, const char *,
2987986f813SBen Skeggs struct nvkm_vmm **);
299eb813999SBen Skeggs
300eb813999SBen Skeggs #define VMM_PRINT(l,v,p,f,a...) do { \
301eb813999SBen Skeggs struct nvkm_vmm *_vmm = (v); \
302eb813999SBen Skeggs if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \
303eb813999SBen Skeggs nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \
304eb813999SBen Skeggs _vmm->name, ##a); \
305eb813999SBen Skeggs } \
306eb813999SBen Skeggs } while(0)
307eb813999SBen Skeggs #define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a)
308eb813999SBen Skeggs #define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a)
309eb813999SBen Skeggs #define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a)
310eb813999SBen Skeggs
311eb813999SBen Skeggs #define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \
312eb813999SBen Skeggs nvkm_kmap((PT)->memory); \
313eb813999SBen Skeggs while (PTEN) { \
314eb813999SBen Skeggs u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \
315eb813999SBen Skeggs u64 _addr = ((BASE) + MAP->off); \
316eb813999SBen Skeggs \
317eb813999SBen Skeggs if (_ptes > PTEN) { \
318eb813999SBen Skeggs MAP->off += PTEN << MAP->page->shift; \
319eb813999SBen Skeggs _ptes = PTEN; \
320eb813999SBen Skeggs } else { \
321eb813999SBen Skeggs MAP->off = 0; \
322eb813999SBen Skeggs NEXT; \
323eb813999SBen Skeggs } \
324eb813999SBen Skeggs \
325eb813999SBen Skeggs VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \
326eb813999SBen Skeggs \
327eb813999SBen Skeggs FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \
328eb813999SBen Skeggs PTEI += _ptes; \
329eb813999SBen Skeggs PTEN -= _ptes; \
33094db9a3bSZheng Bin } \
331eb813999SBen Skeggs nvkm_done((PT)->memory); \
332eb813999SBen Skeggs } while(0)
333eb813999SBen Skeggs
334eb813999SBen Skeggs #define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \
335eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
336eb813999SBen Skeggs ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \
337eb813999SBen Skeggs ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \
338eb813999SBen Skeggs (MAP->mem = MAP->mem->next))
339eb813999SBen Skeggs #define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \
340eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
341eb813999SBen Skeggs *MAP->dma, PAGE_SIZE, MAP->dma++)
342eb813999SBen Skeggs #define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \
343eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
344eb813999SBen Skeggs sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \
345eb813999SBen Skeggs (MAP->sgl = sg_next(MAP->sgl)))
346eb813999SBen Skeggs
347eb813999SBen Skeggs #define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c))
348eb813999SBen Skeggs #define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d))
349eb813999SBen Skeggs #define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \
350eb813999SBen Skeggs const u32 _pteo = (o); u##b _data = (d); \
351eb813999SBen Skeggs VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \
352eb813999SBen Skeggs VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \
353eb813999SBen Skeggs } while(0)
354eb813999SBen Skeggs
355eb813999SBen Skeggs #define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x")
356eb813999SBen Skeggs #define VMM_FO032(m,v,o,d,c) \
357eb813999SBen Skeggs VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c))
358eb813999SBen Skeggs
359eb813999SBen Skeggs #define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx")
360eb813999SBen Skeggs #define VMM_FO064(m,v,o,d,c) \
361eb813999SBen Skeggs VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c))
362eb813999SBen Skeggs
363eb813999SBen Skeggs #define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \
364eb813999SBen Skeggs u32 _pteo = (o), _ptes = (c); \
365eb813999SBen Skeggs const u64 _addr = (m)->addr + _pteo; \
366eb813999SBen Skeggs VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \
367eb813999SBen Skeggs while (_ptes--) { \
368eb813999SBen Skeggs nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \
369eb813999SBen Skeggs nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \
370eb813999SBen Skeggs _pteo += 0x10; \
371eb813999SBen Skeggs } \
372eb813999SBen Skeggs } while(0)
373eb813999SBen Skeggs
374eb813999SBen Skeggs #define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "")
375eb813999SBen Skeggs #define VMM_FO128(m,v,o,lo,hi,c) do { \
376eb813999SBen Skeggs nvkm_kmap((m)->memory); \
377eb813999SBen Skeggs VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \
378eb813999SBen Skeggs nvkm_done((m)->memory); \
379eb813999SBen Skeggs } while(0)
380806a7335SBen Skeggs #endif
381