1806a7335SBen Skeggs #ifndef __NVKM_VMM_H__ 2806a7335SBen Skeggs #define __NVKM_VMM_H__ 3806a7335SBen Skeggs #include "priv.h" 4806a7335SBen Skeggs #include <core/memory.h> 5eb813999SBen Skeggs enum nvkm_memory_target; 6806a7335SBen Skeggs 7806a7335SBen Skeggs struct nvkm_vmm_pt { 8806a7335SBen Skeggs /* Some GPUs have a mapping level with a dual page tables to 9806a7335SBen Skeggs * support large and small pages in the same address-range. 10806a7335SBen Skeggs * 11806a7335SBen Skeggs * We track the state of both page tables in one place, which 12806a7335SBen Skeggs * is why there's multiple PT pointers/refcounts here. 13806a7335SBen Skeggs */ 14806a7335SBen Skeggs struct nvkm_mmu_pt *pt[2]; 15806a7335SBen Skeggs u32 refs[2]; 16806a7335SBen Skeggs 17806a7335SBen Skeggs /* Page size handled by this PT. 18806a7335SBen Skeggs * 19806a7335SBen Skeggs * Tesla backend needs to know this when writinge PDEs, 20806a7335SBen Skeggs * otherwise unnecessary. 21806a7335SBen Skeggs */ 22806a7335SBen Skeggs u8 page; 23806a7335SBen Skeggs 24806a7335SBen Skeggs /* Entire page table sparse. 25806a7335SBen Skeggs * 26806a7335SBen Skeggs * Used to propagate sparseness to child page tables. 27806a7335SBen Skeggs */ 28806a7335SBen Skeggs bool sparse:1; 29806a7335SBen Skeggs 30806a7335SBen Skeggs /* Tracking for page directories. 31806a7335SBen Skeggs * 32806a7335SBen Skeggs * The array is indexed by PDE, and will either point to the 33806a7335SBen Skeggs * child page table, or indicate the PDE is marked as sparse. 34806a7335SBen Skeggs **/ 35806a7335SBen Skeggs #define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde) 36806a7335SBen Skeggs #define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde) 37806a7335SBen Skeggs #define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY) 38806a7335SBen Skeggs struct nvkm_vmm_pt **pde; 39806a7335SBen Skeggs 40806a7335SBen Skeggs /* Tracking for dual page tables. 41806a7335SBen Skeggs * 42806a7335SBen Skeggs * There's one entry for each LPTE, keeping track of whether 43806a7335SBen Skeggs * there are valid SPTEs in the same address-range. 44806a7335SBen Skeggs * 45806a7335SBen Skeggs * This information is used to manage LPTE state transitions. 46806a7335SBen Skeggs */ 47806a7335SBen Skeggs #define NVKM_VMM_PTE_SPARSE 0x80 48806a7335SBen Skeggs #define NVKM_VMM_PTE_VALID 0x40 49806a7335SBen Skeggs #define NVKM_VMM_PTE_SPTES 0x3f 50806a7335SBen Skeggs u8 pte[]; 51806a7335SBen Skeggs }; 52806a7335SBen Skeggs 53eb813999SBen Skeggs typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *, 54eb813999SBen Skeggs struct nvkm_mmu_pt *, u32 ptei, u32 ptes); 55eb813999SBen Skeggs typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *, 56eb813999SBen Skeggs struct nvkm_vmm_pt *, u32 pdei); 57eb813999SBen Skeggs typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *, 58eb813999SBen Skeggs u32 ptei, u32 ptes, struct nvkm_vmm_map *); 59eb813999SBen Skeggs 60806a7335SBen Skeggs struct nvkm_vmm_desc_func { 61eb813999SBen Skeggs nvkm_vmm_pxe_func invalid; 62eb813999SBen Skeggs nvkm_vmm_pxe_func unmap; 63eb813999SBen Skeggs nvkm_vmm_pxe_func sparse; 64eb813999SBen Skeggs 65eb813999SBen Skeggs nvkm_vmm_pde_func pde; 66eb813999SBen Skeggs 67eb813999SBen Skeggs nvkm_vmm_pte_func mem; 68eb813999SBen Skeggs nvkm_vmm_pte_func dma; 69eb813999SBen Skeggs nvkm_vmm_pte_func sgl; 70806a7335SBen Skeggs }; 71806a7335SBen Skeggs 72540a1ddeSBen Skeggs extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; 73b77791daSBen Skeggs void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32); 74540a1ddeSBen Skeggs extern const struct nvkm_vmm_desc_func gf100_vmm_pgt; 75b77791daSBen Skeggs void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); 76b77791daSBen Skeggs void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 77b77791daSBen Skeggs struct nvkm_vmm_map *); 78b77791daSBen Skeggs void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 79b77791daSBen Skeggs struct nvkm_vmm_map *); 80b77791daSBen Skeggs void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 81b77791daSBen Skeggs struct nvkm_vmm_map *); 82540a1ddeSBen Skeggs 83b5977643SBen Skeggs void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); 84b5977643SBen Skeggs 85806a7335SBen Skeggs struct nvkm_vmm_desc { 86806a7335SBen Skeggs enum { 87806a7335SBen Skeggs PGD, 88806a7335SBen Skeggs PGT, 89806a7335SBen Skeggs SPT, 90806a7335SBen Skeggs LPT, 91806a7335SBen Skeggs } type; 92806a7335SBen Skeggs u8 bits; /* VMA bits covered by PT. */ 93806a7335SBen Skeggs u8 size; /* Bytes-per-PTE. */ 94806a7335SBen Skeggs u32 align; /* PT address alignment. */ 95806a7335SBen Skeggs const struct nvkm_vmm_desc_func *func; 96806a7335SBen Skeggs }; 97806a7335SBen Skeggs 982ffa64ebSBen Skeggs extern const struct nvkm_vmm_desc nv50_vmm_desc_12[]; 992ffa64ebSBen Skeggs extern const struct nvkm_vmm_desc nv50_vmm_desc_16[]; 1002ffa64ebSBen Skeggs 1017de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[]; 1027de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[]; 1037de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[]; 1047de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[]; 1057de078aaSBen Skeggs 1065f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[]; 1075f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[]; 1085f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[]; 1095f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[]; 1105f300fedSBen Skeggs 1118e39abffSBen Skeggs extern const struct nvkm_vmm_desc gp100_vmm_desc_12[]; 1128e39abffSBen Skeggs extern const struct nvkm_vmm_desc gp100_vmm_desc_16[]; 1138e39abffSBen Skeggs 114806a7335SBen Skeggs struct nvkm_vmm_page { 115806a7335SBen Skeggs u8 shift; 116806a7335SBen Skeggs const struct nvkm_vmm_desc *desc; 117806a7335SBen Skeggs #define NVKM_VMM_PAGE_SPARSE 0x01 118806a7335SBen Skeggs #define NVKM_VMM_PAGE_VRAM 0x02 119806a7335SBen Skeggs #define NVKM_VMM_PAGE_HOST 0x04 120806a7335SBen Skeggs #define NVKM_VMM_PAGE_COMP 0x08 121806a7335SBen Skeggs #define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE) 122806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM) 123806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM) 124806a7335SBen Skeggs #define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST) 125806a7335SBen Skeggs #define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST) 126806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST) 127806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST) 128806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP) 129806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP) 130806a7335SBen Skeggs #define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP) 131806a7335SBen Skeggs #define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP) 132806a7335SBen Skeggs u8 type; 133806a7335SBen Skeggs }; 134806a7335SBen Skeggs 135806a7335SBen Skeggs struct nvkm_vmm_func { 136806a7335SBen Skeggs int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst); 137806a7335SBen Skeggs void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst); 138806a7335SBen Skeggs 139eb813999SBen Skeggs int (*aper)(enum nvkm_memory_target); 140eb813999SBen Skeggs int (*valid)(struct nvkm_vmm *, void *argv, u32 argc, 141eb813999SBen Skeggs struct nvkm_vmm_map *); 142eb813999SBen Skeggs void (*flush)(struct nvkm_vmm *, int depth); 143eb813999SBen Skeggs 144806a7335SBen Skeggs u64 page_block; 145806a7335SBen Skeggs const struct nvkm_vmm_page page[]; 146806a7335SBen Skeggs }; 147806a7335SBen Skeggs 1489f6219fdSBen Skeggs struct nvkm_vmm_join { 1499f6219fdSBen Skeggs struct nvkm_memory *inst; 1509f6219fdSBen Skeggs struct list_head head; 1519f6219fdSBen Skeggs }; 1529f6219fdSBen Skeggs 153806a7335SBen Skeggs int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, 154806a7335SBen Skeggs u32 pd_header, u64 addr, u64 size, struct lock_class_key *, 155806a7335SBen Skeggs const char *name, struct nvkm_vmm **); 156806a7335SBen Skeggs int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, 157806a7335SBen Skeggs u32 pd_header, u64 addr, u64 size, struct lock_class_key *, 158806a7335SBen Skeggs const char *name, struct nvkm_vmm *); 159f9463a4bSBen Skeggs struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr); 160f9463a4bSBen Skeggs int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, 161f9463a4bSBen Skeggs bool sparse, u8 page, u8 align, u64 size, 162f9463a4bSBen Skeggs struct nvkm_vma **pvma); 163f9463a4bSBen Skeggs void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); 164f9463a4bSBen Skeggs void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *); 165f9463a4bSBen Skeggs void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma); 166f9463a4bSBen Skeggs 167f9463a4bSBen Skeggs struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); 168f9463a4bSBen Skeggs void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *); 169806a7335SBen Skeggs 1705b17f362SBen Skeggs int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, 1715b17f362SBen Skeggs u64, u64, void *, u32, struct lock_class_key *, 1725b17f362SBen Skeggs const char *, struct nvkm_vmm **); 173dd12d158SBen Skeggs int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 17477783435SBen Skeggs 1752ffa64ebSBen Skeggs int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 1762ffa64ebSBen Skeggs void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 1772ffa64ebSBen Skeggs int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 1782ffa64ebSBen Skeggs void nv50_vmm_flush(struct nvkm_vmm *, int); 1792ffa64ebSBen Skeggs 180540a1ddeSBen Skeggs int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 181540a1ddeSBen Skeggs struct nvkm_mmu *, u64, u64, void *, u32, 182540a1ddeSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 183540a1ddeSBen Skeggs int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 184540a1ddeSBen Skeggs int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 185540a1ddeSBen Skeggs void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 186b77791daSBen Skeggs int gf100_vmm_aper(enum nvkm_memory_target); 187b77791daSBen Skeggs int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 188b77791daSBen Skeggs void gf100_vmm_flush_(struct nvkm_vmm *, int); 189b77791daSBen Skeggs void gf100_vmm_flush(struct nvkm_vmm *, int); 190540a1ddeSBen Skeggs 191b5977643SBen Skeggs int gk20a_vmm_aper(enum nvkm_memory_target); 192b5977643SBen Skeggs 1935f300fedSBen Skeggs int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 1945f300fedSBen Skeggs struct nvkm_mmu *, u64, u64, void *, u32, 1955f300fedSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 1965f300fedSBen Skeggs int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 1975f300fedSBen Skeggs int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 1985f300fedSBen Skeggs 1998e39abffSBen Skeggs int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 200f9400afbSBen Skeggs int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 201f9400afbSBen Skeggs void gp100_vmm_flush(struct nvkm_vmm *, int); 2028e39abffSBen Skeggs 2035b17f362SBen Skeggs int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2045b17f362SBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 20577783435SBen Skeggs int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 20677783435SBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 20703b0ba7bSBen Skeggs int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 20803b0ba7bSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2099f6219fdSBen Skeggs int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2109f6219fdSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2112ffa64ebSBen Skeggs int mcp77_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2122ffa64ebSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2139f6219fdSBen Skeggs int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2149f6219fdSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 215540a1ddeSBen Skeggs int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 216540a1ddeSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2177de078aaSBen Skeggs int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2187de078aaSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2197de078aaSBen Skeggs int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2207de078aaSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2215f300fedSBen Skeggs int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, 2225f300fedSBen Skeggs struct lock_class_key *, const char *, 2235f300fedSBen Skeggs struct nvkm_vmm **); 2245f300fedSBen Skeggs int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2255f300fedSBen Skeggs struct lock_class_key *, const char *, 2265f300fedSBen Skeggs struct nvkm_vmm **); 2275f300fedSBen Skeggs int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, 2285f300fedSBen Skeggs struct lock_class_key *, const char *, 2295f300fedSBen Skeggs struct nvkm_vmm **); 2305f300fedSBen Skeggs int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2315f300fedSBen Skeggs struct lock_class_key *, const char *, 2325f300fedSBen Skeggs struct nvkm_vmm **); 2338e39abffSBen Skeggs int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2348e39abffSBen Skeggs struct lock_class_key *, const char *, 2358e39abffSBen Skeggs struct nvkm_vmm **); 2368e39abffSBen Skeggs int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2378e39abffSBen Skeggs struct lock_class_key *, const char *, 2388e39abffSBen Skeggs struct nvkm_vmm **); 239eb813999SBen Skeggs 240eb813999SBen Skeggs #define VMM_PRINT(l,v,p,f,a...) do { \ 241eb813999SBen Skeggs struct nvkm_vmm *_vmm = (v); \ 242eb813999SBen Skeggs if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \ 243eb813999SBen Skeggs nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \ 244eb813999SBen Skeggs _vmm->name, ##a); \ 245eb813999SBen Skeggs } \ 246eb813999SBen Skeggs } while(0) 247eb813999SBen Skeggs #define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a) 248eb813999SBen Skeggs #define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a) 249eb813999SBen Skeggs #define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a) 250eb813999SBen Skeggs 251eb813999SBen Skeggs #define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \ 252eb813999SBen Skeggs nvkm_kmap((PT)->memory); \ 253eb813999SBen Skeggs while (PTEN) { \ 254eb813999SBen Skeggs u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \ 255eb813999SBen Skeggs u64 _addr = ((BASE) + MAP->off); \ 256eb813999SBen Skeggs \ 257eb813999SBen Skeggs if (_ptes > PTEN) { \ 258eb813999SBen Skeggs MAP->off += PTEN << MAP->page->shift; \ 259eb813999SBen Skeggs _ptes = PTEN; \ 260eb813999SBen Skeggs } else { \ 261eb813999SBen Skeggs MAP->off = 0; \ 262eb813999SBen Skeggs NEXT; \ 263eb813999SBen Skeggs } \ 264eb813999SBen Skeggs \ 265eb813999SBen Skeggs VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \ 266eb813999SBen Skeggs \ 267eb813999SBen Skeggs FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \ 268eb813999SBen Skeggs PTEI += _ptes; \ 269eb813999SBen Skeggs PTEN -= _ptes; \ 270eb813999SBen Skeggs }; \ 271eb813999SBen Skeggs nvkm_done((PT)->memory); \ 272eb813999SBen Skeggs } while(0) 273eb813999SBen Skeggs 274eb813999SBen Skeggs #define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \ 275eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 276eb813999SBen Skeggs ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \ 277eb813999SBen Skeggs ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \ 278eb813999SBen Skeggs (MAP->mem = MAP->mem->next)) 279eb813999SBen Skeggs #define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \ 280eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 281eb813999SBen Skeggs *MAP->dma, PAGE_SIZE, MAP->dma++) 282eb813999SBen Skeggs #define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \ 283eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 284eb813999SBen Skeggs sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \ 285eb813999SBen Skeggs (MAP->sgl = sg_next(MAP->sgl))) 286eb813999SBen Skeggs 287eb813999SBen Skeggs #define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c)) 288eb813999SBen Skeggs #define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d)) 289eb813999SBen Skeggs #define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \ 290eb813999SBen Skeggs const u32 _pteo = (o); u##b _data = (d); \ 291eb813999SBen Skeggs VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \ 292eb813999SBen Skeggs VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \ 293eb813999SBen Skeggs } while(0) 294eb813999SBen Skeggs 295eb813999SBen Skeggs #define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x") 296eb813999SBen Skeggs #define VMM_FO032(m,v,o,d,c) \ 297eb813999SBen Skeggs VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c)) 298eb813999SBen Skeggs 299eb813999SBen Skeggs #define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx") 300eb813999SBen Skeggs #define VMM_FO064(m,v,o,d,c) \ 301eb813999SBen Skeggs VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c)) 302eb813999SBen Skeggs 303eb813999SBen Skeggs #define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \ 304eb813999SBen Skeggs u32 _pteo = (o), _ptes = (c); \ 305eb813999SBen Skeggs const u64 _addr = (m)->addr + _pteo; \ 306eb813999SBen Skeggs VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \ 307eb813999SBen Skeggs while (_ptes--) { \ 308eb813999SBen Skeggs nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \ 309eb813999SBen Skeggs nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \ 310eb813999SBen Skeggs _pteo += 0x10; \ 311eb813999SBen Skeggs } \ 312eb813999SBen Skeggs } while(0) 313eb813999SBen Skeggs 314eb813999SBen Skeggs #define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "") 315eb813999SBen Skeggs #define VMM_FO128(m,v,o,lo,hi,c) do { \ 316eb813999SBen Skeggs nvkm_kmap((m)->memory); \ 317eb813999SBen Skeggs VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \ 318eb813999SBen Skeggs nvkm_done((m)->memory); \ 319eb813999SBen Skeggs } while(0) 320806a7335SBen Skeggs #endif 321