1806a7335SBen Skeggs #ifndef __NVKM_VMM_H__ 2806a7335SBen Skeggs #define __NVKM_VMM_H__ 3806a7335SBen Skeggs #include "priv.h" 4806a7335SBen Skeggs #include <core/memory.h> 5eb813999SBen Skeggs enum nvkm_memory_target; 6806a7335SBen Skeggs 7806a7335SBen Skeggs struct nvkm_vmm_pt { 8806a7335SBen Skeggs /* Some GPUs have a mapping level with a dual page tables to 9806a7335SBen Skeggs * support large and small pages in the same address-range. 10806a7335SBen Skeggs * 11806a7335SBen Skeggs * We track the state of both page tables in one place, which 12806a7335SBen Skeggs * is why there's multiple PT pointers/refcounts here. 13806a7335SBen Skeggs */ 14806a7335SBen Skeggs struct nvkm_mmu_pt *pt[2]; 15806a7335SBen Skeggs u32 refs[2]; 16806a7335SBen Skeggs 17806a7335SBen Skeggs /* Page size handled by this PT. 18806a7335SBen Skeggs * 19806a7335SBen Skeggs * Tesla backend needs to know this when writinge PDEs, 20806a7335SBen Skeggs * otherwise unnecessary. 21806a7335SBen Skeggs */ 22806a7335SBen Skeggs u8 page; 23806a7335SBen Skeggs 24806a7335SBen Skeggs /* Entire page table sparse. 25806a7335SBen Skeggs * 26806a7335SBen Skeggs * Used to propagate sparseness to child page tables. 27806a7335SBen Skeggs */ 28806a7335SBen Skeggs bool sparse:1; 29806a7335SBen Skeggs 30806a7335SBen Skeggs /* Tracking for page directories. 31806a7335SBen Skeggs * 32806a7335SBen Skeggs * The array is indexed by PDE, and will either point to the 33806a7335SBen Skeggs * child page table, or indicate the PDE is marked as sparse. 34806a7335SBen Skeggs **/ 35806a7335SBen Skeggs #define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde) 36806a7335SBen Skeggs #define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde) 37806a7335SBen Skeggs #define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY) 38806a7335SBen Skeggs struct nvkm_vmm_pt **pde; 39806a7335SBen Skeggs 40806a7335SBen Skeggs /* Tracking for dual page tables. 41806a7335SBen Skeggs * 42806a7335SBen Skeggs * There's one entry for each LPTE, keeping track of whether 43806a7335SBen Skeggs * there are valid SPTEs in the same address-range. 44806a7335SBen Skeggs * 45806a7335SBen Skeggs * This information is used to manage LPTE state transitions. 46806a7335SBen Skeggs */ 47806a7335SBen Skeggs #define NVKM_VMM_PTE_SPARSE 0x80 48806a7335SBen Skeggs #define NVKM_VMM_PTE_VALID 0x40 49806a7335SBen Skeggs #define NVKM_VMM_PTE_SPTES 0x3f 50806a7335SBen Skeggs u8 pte[]; 51806a7335SBen Skeggs }; 52806a7335SBen Skeggs 53eb813999SBen Skeggs typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *, 54eb813999SBen Skeggs struct nvkm_mmu_pt *, u32 ptei, u32 ptes); 55eb813999SBen Skeggs typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *, 56eb813999SBen Skeggs struct nvkm_vmm_pt *, u32 pdei); 57eb813999SBen Skeggs typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *, 58eb813999SBen Skeggs u32 ptei, u32 ptes, struct nvkm_vmm_map *); 59eb813999SBen Skeggs 60806a7335SBen Skeggs struct nvkm_vmm_desc_func { 61eb813999SBen Skeggs nvkm_vmm_pxe_func invalid; 62eb813999SBen Skeggs nvkm_vmm_pxe_func unmap; 63eb813999SBen Skeggs nvkm_vmm_pxe_func sparse; 64eb813999SBen Skeggs 65eb813999SBen Skeggs nvkm_vmm_pde_func pde; 66eb813999SBen Skeggs 67eb813999SBen Skeggs nvkm_vmm_pte_func mem; 68eb813999SBen Skeggs nvkm_vmm_pte_func dma; 69eb813999SBen Skeggs nvkm_vmm_pte_func sgl; 70a5ff307fSBen Skeggs 71a5ff307fSBen Skeggs nvkm_vmm_pte_func pfn; 72a5ff307fSBen Skeggs bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes); 73a5ff307fSBen Skeggs nvkm_vmm_pxe_func pfn_unmap; 74806a7335SBen Skeggs }; 75806a7335SBen Skeggs 76540a1ddeSBen Skeggs extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; 77b77791daSBen Skeggs void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32); 78540a1ddeSBen Skeggs extern const struct nvkm_vmm_desc_func gf100_vmm_pgt; 79b77791daSBen Skeggs void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); 80b77791daSBen Skeggs void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 81b77791daSBen Skeggs struct nvkm_vmm_map *); 82b77791daSBen Skeggs void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 83b77791daSBen Skeggs struct nvkm_vmm_map *); 84b77791daSBen Skeggs void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 85b77791daSBen Skeggs struct nvkm_vmm_map *); 86540a1ddeSBen Skeggs 87b5977643SBen Skeggs void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); 88b5977643SBen Skeggs 89806a7335SBen Skeggs struct nvkm_vmm_desc { 90806a7335SBen Skeggs enum { 91806a7335SBen Skeggs PGD, 92806a7335SBen Skeggs PGT, 93806a7335SBen Skeggs SPT, 94806a7335SBen Skeggs LPT, 95806a7335SBen Skeggs } type; 96806a7335SBen Skeggs u8 bits; /* VMA bits covered by PT. */ 97806a7335SBen Skeggs u8 size; /* Bytes-per-PTE. */ 98806a7335SBen Skeggs u32 align; /* PT address alignment. */ 99806a7335SBen Skeggs const struct nvkm_vmm_desc_func *func; 100806a7335SBen Skeggs }; 101806a7335SBen Skeggs 1022ffa64ebSBen Skeggs extern const struct nvkm_vmm_desc nv50_vmm_desc_12[]; 1032ffa64ebSBen Skeggs extern const struct nvkm_vmm_desc nv50_vmm_desc_16[]; 1042ffa64ebSBen Skeggs 1057de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[]; 1067de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[]; 1077de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[]; 1087de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[]; 1097de078aaSBen Skeggs 1105f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[]; 1115f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[]; 1125f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[]; 1135f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[]; 1145f300fedSBen Skeggs 1158e39abffSBen Skeggs extern const struct nvkm_vmm_desc gp100_vmm_desc_12[]; 1168e39abffSBen Skeggs extern const struct nvkm_vmm_desc gp100_vmm_desc_16[]; 1178e39abffSBen Skeggs 118806a7335SBen Skeggs struct nvkm_vmm_page { 119806a7335SBen Skeggs u8 shift; 120806a7335SBen Skeggs const struct nvkm_vmm_desc *desc; 121806a7335SBen Skeggs #define NVKM_VMM_PAGE_SPARSE 0x01 122806a7335SBen Skeggs #define NVKM_VMM_PAGE_VRAM 0x02 123806a7335SBen Skeggs #define NVKM_VMM_PAGE_HOST 0x04 124806a7335SBen Skeggs #define NVKM_VMM_PAGE_COMP 0x08 125806a7335SBen Skeggs #define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE) 126806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM) 127806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM) 128806a7335SBen Skeggs #define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST) 129806a7335SBen Skeggs #define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST) 130806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST) 131806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST) 132806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP) 133806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP) 134806a7335SBen Skeggs #define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP) 135806a7335SBen Skeggs #define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP) 136806a7335SBen Skeggs u8 type; 137806a7335SBen Skeggs }; 138806a7335SBen Skeggs 139806a7335SBen Skeggs struct nvkm_vmm_func { 140806a7335SBen Skeggs int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst); 141806a7335SBen Skeggs void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst); 142806a7335SBen Skeggs 143eb813999SBen Skeggs int (*aper)(enum nvkm_memory_target); 144eb813999SBen Skeggs int (*valid)(struct nvkm_vmm *, void *argv, u32 argc, 145eb813999SBen Skeggs struct nvkm_vmm_map *); 146eb813999SBen Skeggs void (*flush)(struct nvkm_vmm *, int depth); 147eb813999SBen Skeggs 14871871aa6SBen Skeggs int (*mthd)(struct nvkm_vmm *, struct nvkm_client *, 14971871aa6SBen Skeggs u32 mthd, void *argv, u32 argc); 15071871aa6SBen Skeggs 151d389fd4fSBen Skeggs void (*invalidate_pdb)(struct nvkm_vmm *, u64 addr); 152d389fd4fSBen Skeggs 153806a7335SBen Skeggs u64 page_block; 154806a7335SBen Skeggs const struct nvkm_vmm_page page[]; 155806a7335SBen Skeggs }; 156806a7335SBen Skeggs 1579f6219fdSBen Skeggs struct nvkm_vmm_join { 1589f6219fdSBen Skeggs struct nvkm_memory *inst; 1599f6219fdSBen Skeggs struct list_head head; 1609f6219fdSBen Skeggs }; 1619f6219fdSBen Skeggs 162806a7335SBen Skeggs int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, 1632606f291SBen Skeggs u32 pd_header, bool managed, u64 addr, u64 size, 1642606f291SBen Skeggs struct lock_class_key *, const char *name, 1652606f291SBen Skeggs struct nvkm_vmm **); 166806a7335SBen Skeggs int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, 1672606f291SBen Skeggs u32 pd_header, bool managed, u64 addr, u64 size, 1682606f291SBen Skeggs struct lock_class_key *, const char *name, struct nvkm_vmm *); 169f9463a4bSBen Skeggs struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr); 170729eba33SBen Skeggs struct nvkm_vma *nvkm_vmm_node_split(struct nvkm_vmm *, struct nvkm_vma *, 171729eba33SBen Skeggs u64 addr, u64 size); 172f9463a4bSBen Skeggs int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, 173f9463a4bSBen Skeggs bool sparse, u8 page, u8 align, u64 size, 174f9463a4bSBen Skeggs struct nvkm_vma **pvma); 175f9463a4bSBen Skeggs void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); 176a5ff307fSBen Skeggs void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn); 177a5ff307fSBen Skeggs void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *); 178a5ff307fSBen Skeggs 179a5ff307fSBen Skeggs #define NVKM_VMM_PFN_ADDR 0xfffffffffffff000ULL 180a5ff307fSBen Skeggs #define NVKM_VMM_PFN_ADDR_SHIFT 12 181a5ff307fSBen Skeggs #define NVKM_VMM_PFN_APER 0x00000000000000f0ULL 182a5ff307fSBen Skeggs #define NVKM_VMM_PFN_HOST 0x0000000000000000ULL 183a5ff307fSBen Skeggs #define NVKM_VMM_PFN_VRAM 0x0000000000000010ULL 184a5ff307fSBen Skeggs #define NVKM_VMM_PFN_W 0x0000000000000002ULL 185a5ff307fSBen Skeggs #define NVKM_VMM_PFN_V 0x0000000000000001ULL 186a5ff307fSBen Skeggs #define NVKM_VMM_PFN_NONE 0x0000000000000000ULL 187a5ff307fSBen Skeggs 188a5ff307fSBen Skeggs int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn); 189a5ff307fSBen Skeggs int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size); 190f9463a4bSBen Skeggs 191f9463a4bSBen Skeggs struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); 192806a7335SBen Skeggs 1935b17f362SBen Skeggs int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, 1942606f291SBen Skeggs bool, u64, u64, void *, u32, struct lock_class_key *, 1955b17f362SBen Skeggs const char *, struct nvkm_vmm **); 196dd12d158SBen Skeggs int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 19777783435SBen Skeggs 1982ffa64ebSBen Skeggs int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 1992ffa64ebSBen Skeggs void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 2002ffa64ebSBen Skeggs int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 2012ffa64ebSBen Skeggs void nv50_vmm_flush(struct nvkm_vmm *, int); 2022ffa64ebSBen Skeggs 203540a1ddeSBen Skeggs int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 2042606f291SBen Skeggs struct nvkm_mmu *, bool, u64, u64, void *, u32, 205540a1ddeSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 206540a1ddeSBen Skeggs int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 207540a1ddeSBen Skeggs int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 208540a1ddeSBen Skeggs void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 209b77791daSBen Skeggs int gf100_vmm_aper(enum nvkm_memory_target); 210b77791daSBen Skeggs int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 211b77791daSBen Skeggs void gf100_vmm_flush(struct nvkm_vmm *, int); 212d389fd4fSBen Skeggs void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type); 213d389fd4fSBen Skeggs void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); 214540a1ddeSBen Skeggs 215b5977643SBen Skeggs int gk20a_vmm_aper(enum nvkm_memory_target); 216b5977643SBen Skeggs 2175f300fedSBen Skeggs int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 2182606f291SBen Skeggs struct nvkm_mmu *, bool, u64, u64, void *, u32, 2195f300fedSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2205f300fedSBen Skeggs int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 2215f300fedSBen Skeggs int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 2225f300fedSBen Skeggs 223ab2ee9ffSBen Skeggs int gp100_vmm_new_(const struct nvkm_vmm_func *, 224ab2ee9ffSBen Skeggs struct nvkm_mmu *, bool, u64, u64, void *, u32, 225ab2ee9ffSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2268e39abffSBen Skeggs int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 227f9400afbSBen Skeggs int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 228f9400afbSBen Skeggs void gp100_vmm_flush(struct nvkm_vmm *, int); 22971871aa6SBen Skeggs int gp100_vmm_mthd(struct nvkm_vmm *, struct nvkm_client *, u32, void *, u32); 230d389fd4fSBen Skeggs void gp100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); 2318e39abffSBen Skeggs 2327986f813SBen Skeggs int gv100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 2337986f813SBen Skeggs 2342606f291SBen Skeggs int nv04_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2355b17f362SBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2362606f291SBen Skeggs int nv41_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 23777783435SBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2382606f291SBen Skeggs int nv44_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 23903b0ba7bSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2402606f291SBen Skeggs int nv50_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2419f6219fdSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2422606f291SBen Skeggs int mcp77_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2432ffa64ebSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2442606f291SBen Skeggs int g84_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2459f6219fdSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2462606f291SBen Skeggs int gf100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 247540a1ddeSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2482606f291SBen Skeggs int gk104_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2497de078aaSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2502606f291SBen Skeggs int gk20a_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2517de078aaSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 2522606f291SBen Skeggs int gm200_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2535f300fedSBen Skeggs struct lock_class_key *, const char *, 2545f300fedSBen Skeggs struct nvkm_vmm **); 2552606f291SBen Skeggs int gm200_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2565f300fedSBen Skeggs struct lock_class_key *, const char *, 2575f300fedSBen Skeggs struct nvkm_vmm **); 2582606f291SBen Skeggs int gm20b_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2595f300fedSBen Skeggs struct lock_class_key *, const char *, 2605f300fedSBen Skeggs struct nvkm_vmm **); 2612606f291SBen Skeggs int gm20b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2625f300fedSBen Skeggs struct lock_class_key *, const char *, 2635f300fedSBen Skeggs struct nvkm_vmm **); 2642606f291SBen Skeggs int gp100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2658e39abffSBen Skeggs struct lock_class_key *, const char *, 2668e39abffSBen Skeggs struct nvkm_vmm **); 2672606f291SBen Skeggs int gp10b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2688e39abffSBen Skeggs struct lock_class_key *, const char *, 2698e39abffSBen Skeggs struct nvkm_vmm **); 2702606f291SBen Skeggs int gv100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 271edf50395SBen Skeggs struct lock_class_key *, const char *, 272edf50395SBen Skeggs struct nvkm_vmm **); 2732606f291SBen Skeggs int tu102_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 2747986f813SBen Skeggs struct lock_class_key *, const char *, 2757986f813SBen Skeggs struct nvkm_vmm **); 276eb813999SBen Skeggs 277eb813999SBen Skeggs #define VMM_PRINT(l,v,p,f,a...) do { \ 278eb813999SBen Skeggs struct nvkm_vmm *_vmm = (v); \ 279eb813999SBen Skeggs if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \ 280eb813999SBen Skeggs nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \ 281eb813999SBen Skeggs _vmm->name, ##a); \ 282eb813999SBen Skeggs } \ 283eb813999SBen Skeggs } while(0) 284eb813999SBen Skeggs #define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a) 285eb813999SBen Skeggs #define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a) 286eb813999SBen Skeggs #define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a) 287eb813999SBen Skeggs 288eb813999SBen Skeggs #define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \ 289eb813999SBen Skeggs nvkm_kmap((PT)->memory); \ 290eb813999SBen Skeggs while (PTEN) { \ 291eb813999SBen Skeggs u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \ 292eb813999SBen Skeggs u64 _addr = ((BASE) + MAP->off); \ 293eb813999SBen Skeggs \ 294eb813999SBen Skeggs if (_ptes > PTEN) { \ 295eb813999SBen Skeggs MAP->off += PTEN << MAP->page->shift; \ 296eb813999SBen Skeggs _ptes = PTEN; \ 297eb813999SBen Skeggs } else { \ 298eb813999SBen Skeggs MAP->off = 0; \ 299eb813999SBen Skeggs NEXT; \ 300eb813999SBen Skeggs } \ 301eb813999SBen Skeggs \ 302eb813999SBen Skeggs VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \ 303eb813999SBen Skeggs \ 304eb813999SBen Skeggs FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \ 305eb813999SBen Skeggs PTEI += _ptes; \ 306eb813999SBen Skeggs PTEN -= _ptes; \ 30794db9a3bSZheng Bin } \ 308eb813999SBen Skeggs nvkm_done((PT)->memory); \ 309eb813999SBen Skeggs } while(0) 310eb813999SBen Skeggs 311eb813999SBen Skeggs #define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \ 312eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 313eb813999SBen Skeggs ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \ 314eb813999SBen Skeggs ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \ 315eb813999SBen Skeggs (MAP->mem = MAP->mem->next)) 316eb813999SBen Skeggs #define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \ 317eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 318eb813999SBen Skeggs *MAP->dma, PAGE_SIZE, MAP->dma++) 319eb813999SBen Skeggs #define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \ 320eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 321eb813999SBen Skeggs sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \ 322eb813999SBen Skeggs (MAP->sgl = sg_next(MAP->sgl))) 323eb813999SBen Skeggs 324eb813999SBen Skeggs #define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c)) 325eb813999SBen Skeggs #define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d)) 326eb813999SBen Skeggs #define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \ 327eb813999SBen Skeggs const u32 _pteo = (o); u##b _data = (d); \ 328eb813999SBen Skeggs VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \ 329eb813999SBen Skeggs VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \ 330eb813999SBen Skeggs } while(0) 331eb813999SBen Skeggs 332eb813999SBen Skeggs #define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x") 333eb813999SBen Skeggs #define VMM_FO032(m,v,o,d,c) \ 334eb813999SBen Skeggs VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c)) 335eb813999SBen Skeggs 336eb813999SBen Skeggs #define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx") 337eb813999SBen Skeggs #define VMM_FO064(m,v,o,d,c) \ 338eb813999SBen Skeggs VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c)) 339eb813999SBen Skeggs 340eb813999SBen Skeggs #define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \ 341eb813999SBen Skeggs u32 _pteo = (o), _ptes = (c); \ 342eb813999SBen Skeggs const u64 _addr = (m)->addr + _pteo; \ 343eb813999SBen Skeggs VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \ 344eb813999SBen Skeggs while (_ptes--) { \ 345eb813999SBen Skeggs nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \ 346eb813999SBen Skeggs nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \ 347eb813999SBen Skeggs _pteo += 0x10; \ 348eb813999SBen Skeggs } \ 349eb813999SBen Skeggs } while(0) 350eb813999SBen Skeggs 351eb813999SBen Skeggs #define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "") 352eb813999SBen Skeggs #define VMM_FO128(m,v,o,lo,hi,c) do { \ 353eb813999SBen Skeggs nvkm_kmap((m)->memory); \ 354eb813999SBen Skeggs VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \ 355eb813999SBen Skeggs nvkm_done((m)->memory); \ 356eb813999SBen Skeggs } while(0) 357806a7335SBen Skeggs #endif 358