1806a7335SBen Skeggs #ifndef __NVKM_VMM_H__ 2806a7335SBen Skeggs #define __NVKM_VMM_H__ 3806a7335SBen Skeggs #include "priv.h" 4806a7335SBen Skeggs #include <core/memory.h> 5eb813999SBen Skeggs enum nvkm_memory_target; 6806a7335SBen Skeggs 7806a7335SBen Skeggs struct nvkm_vmm_pt { 8806a7335SBen Skeggs /* Some GPUs have a mapping level with a dual page tables to 9806a7335SBen Skeggs * support large and small pages in the same address-range. 10806a7335SBen Skeggs * 11806a7335SBen Skeggs * We track the state of both page tables in one place, which 12806a7335SBen Skeggs * is why there's multiple PT pointers/refcounts here. 13806a7335SBen Skeggs */ 14806a7335SBen Skeggs struct nvkm_mmu_pt *pt[2]; 15806a7335SBen Skeggs u32 refs[2]; 16806a7335SBen Skeggs 17806a7335SBen Skeggs /* Page size handled by this PT. 18806a7335SBen Skeggs * 19806a7335SBen Skeggs * Tesla backend needs to know this when writinge PDEs, 20806a7335SBen Skeggs * otherwise unnecessary. 21806a7335SBen Skeggs */ 22806a7335SBen Skeggs u8 page; 23806a7335SBen Skeggs 24806a7335SBen Skeggs /* Entire page table sparse. 25806a7335SBen Skeggs * 26806a7335SBen Skeggs * Used to propagate sparseness to child page tables. 27806a7335SBen Skeggs */ 28806a7335SBen Skeggs bool sparse:1; 29806a7335SBen Skeggs 30806a7335SBen Skeggs /* Tracking for page directories. 31806a7335SBen Skeggs * 32806a7335SBen Skeggs * The array is indexed by PDE, and will either point to the 33806a7335SBen Skeggs * child page table, or indicate the PDE is marked as sparse. 34806a7335SBen Skeggs **/ 35806a7335SBen Skeggs #define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde) 36806a7335SBen Skeggs #define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde) 37806a7335SBen Skeggs #define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY) 38806a7335SBen Skeggs struct nvkm_vmm_pt **pde; 39806a7335SBen Skeggs 40806a7335SBen Skeggs /* Tracking for dual page tables. 41806a7335SBen Skeggs * 42806a7335SBen Skeggs * There's one entry for each LPTE, keeping track of whether 43806a7335SBen Skeggs * there are valid SPTEs in the same address-range. 44806a7335SBen Skeggs * 45806a7335SBen Skeggs * This information is used to manage LPTE state transitions. 46806a7335SBen Skeggs */ 47806a7335SBen Skeggs #define NVKM_VMM_PTE_SPARSE 0x80 48806a7335SBen Skeggs #define NVKM_VMM_PTE_VALID 0x40 49806a7335SBen Skeggs #define NVKM_VMM_PTE_SPTES 0x3f 50806a7335SBen Skeggs u8 pte[]; 51806a7335SBen Skeggs }; 52806a7335SBen Skeggs 53eb813999SBen Skeggs typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *, 54eb813999SBen Skeggs struct nvkm_mmu_pt *, u32 ptei, u32 ptes); 55eb813999SBen Skeggs typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *, 56eb813999SBen Skeggs struct nvkm_vmm_pt *, u32 pdei); 57eb813999SBen Skeggs typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *, 58eb813999SBen Skeggs u32 ptei, u32 ptes, struct nvkm_vmm_map *); 59eb813999SBen Skeggs 60806a7335SBen Skeggs struct nvkm_vmm_desc_func { 61eb813999SBen Skeggs nvkm_vmm_pxe_func invalid; 62eb813999SBen Skeggs nvkm_vmm_pxe_func unmap; 63eb813999SBen Skeggs nvkm_vmm_pxe_func sparse; 64eb813999SBen Skeggs 65eb813999SBen Skeggs nvkm_vmm_pde_func pde; 66eb813999SBen Skeggs 67eb813999SBen Skeggs nvkm_vmm_pte_func mem; 68eb813999SBen Skeggs nvkm_vmm_pte_func dma; 69eb813999SBen Skeggs nvkm_vmm_pte_func sgl; 70806a7335SBen Skeggs }; 71806a7335SBen Skeggs 72540a1ddeSBen Skeggs extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; 73540a1ddeSBen Skeggs extern const struct nvkm_vmm_desc_func gf100_vmm_pgt; 74540a1ddeSBen Skeggs 75806a7335SBen Skeggs struct nvkm_vmm_desc { 76806a7335SBen Skeggs enum { 77806a7335SBen Skeggs PGD, 78806a7335SBen Skeggs PGT, 79806a7335SBen Skeggs SPT, 80806a7335SBen Skeggs LPT, 81806a7335SBen Skeggs } type; 82806a7335SBen Skeggs u8 bits; /* VMA bits covered by PT. */ 83806a7335SBen Skeggs u8 size; /* Bytes-per-PTE. */ 84806a7335SBen Skeggs u32 align; /* PT address alignment. */ 85806a7335SBen Skeggs const struct nvkm_vmm_desc_func *func; 86806a7335SBen Skeggs }; 87806a7335SBen Skeggs 887de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[]; 897de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[]; 907de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[]; 917de078aaSBen Skeggs extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[]; 927de078aaSBen Skeggs 935f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[]; 945f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[]; 955f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[]; 965f300fedSBen Skeggs extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[]; 975f300fedSBen Skeggs 988e39abffSBen Skeggs extern const struct nvkm_vmm_desc gp100_vmm_desc_12[]; 998e39abffSBen Skeggs extern const struct nvkm_vmm_desc gp100_vmm_desc_16[]; 1008e39abffSBen Skeggs 101806a7335SBen Skeggs struct nvkm_vmm_page { 102806a7335SBen Skeggs u8 shift; 103806a7335SBen Skeggs const struct nvkm_vmm_desc *desc; 104806a7335SBen Skeggs #define NVKM_VMM_PAGE_SPARSE 0x01 105806a7335SBen Skeggs #define NVKM_VMM_PAGE_VRAM 0x02 106806a7335SBen Skeggs #define NVKM_VMM_PAGE_HOST 0x04 107806a7335SBen Skeggs #define NVKM_VMM_PAGE_COMP 0x08 108806a7335SBen Skeggs #define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE) 109806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM) 110806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM) 111806a7335SBen Skeggs #define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST) 112806a7335SBen Skeggs #define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST) 113806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST) 114806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST) 115806a7335SBen Skeggs #define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP) 116806a7335SBen Skeggs #define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP) 117806a7335SBen Skeggs #define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP) 118806a7335SBen Skeggs #define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP) 119806a7335SBen Skeggs u8 type; 120806a7335SBen Skeggs }; 121806a7335SBen Skeggs 122806a7335SBen Skeggs struct nvkm_vmm_func { 123806a7335SBen Skeggs int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst); 124806a7335SBen Skeggs void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst); 125806a7335SBen Skeggs 126eb813999SBen Skeggs int (*aper)(enum nvkm_memory_target); 127eb813999SBen Skeggs int (*valid)(struct nvkm_vmm *, void *argv, u32 argc, 128eb813999SBen Skeggs struct nvkm_vmm_map *); 129eb813999SBen Skeggs void (*flush)(struct nvkm_vmm *, int depth); 130eb813999SBen Skeggs 131806a7335SBen Skeggs u64 page_block; 132806a7335SBen Skeggs const struct nvkm_vmm_page page[]; 133806a7335SBen Skeggs }; 134806a7335SBen Skeggs 1359f6219fdSBen Skeggs struct nvkm_vmm_join { 1369f6219fdSBen Skeggs struct nvkm_memory *inst; 1379f6219fdSBen Skeggs struct list_head head; 1389f6219fdSBen Skeggs }; 1399f6219fdSBen Skeggs 140806a7335SBen Skeggs int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, 141806a7335SBen Skeggs u32 pd_header, u64 addr, u64 size, struct lock_class_key *, 142806a7335SBen Skeggs const char *name, struct nvkm_vmm **); 143806a7335SBen Skeggs int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, 144806a7335SBen Skeggs u32 pd_header, u64 addr, u64 size, struct lock_class_key *, 145806a7335SBen Skeggs const char *name, struct nvkm_vmm *); 146806a7335SBen Skeggs void nvkm_vmm_dtor(struct nvkm_vmm *); 147eb813999SBen Skeggs void nvkm_vmm_ptes_put(struct nvkm_vmm *, const struct nvkm_vmm_page *, 148eb813999SBen Skeggs u64 addr, u64 size); 149eb813999SBen Skeggs int nvkm_vmm_ptes_get(struct nvkm_vmm *, const struct nvkm_vmm_page *, 150eb813999SBen Skeggs u64 addr, u64 size); 151eb813999SBen Skeggs void nvkm_vmm_ptes_map(struct nvkm_vmm *, const struct nvkm_vmm_page *, 152eb813999SBen Skeggs u64 addr, u64 size, struct nvkm_vmm_map *, 153eb813999SBen Skeggs nvkm_vmm_pte_func); 154eb813999SBen Skeggs void nvkm_vmm_ptes_unmap(struct nvkm_vmm *, const struct nvkm_vmm_page *, 155eb813999SBen Skeggs u64 addr, u64 size, bool sparse); 156806a7335SBen Skeggs 1575b17f362SBen Skeggs int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, 1585b17f362SBen Skeggs u64, u64, void *, u32, struct lock_class_key *, 1595b17f362SBen Skeggs const char *, struct nvkm_vmm **); 16077783435SBen Skeggs 161540a1ddeSBen Skeggs int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 162540a1ddeSBen Skeggs struct nvkm_mmu *, u64, u64, void *, u32, 163540a1ddeSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 164540a1ddeSBen Skeggs int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 165540a1ddeSBen Skeggs int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 166540a1ddeSBen Skeggs void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 167540a1ddeSBen Skeggs 1685f300fedSBen Skeggs int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 1695f300fedSBen Skeggs struct nvkm_mmu *, u64, u64, void *, u32, 1705f300fedSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 1715f300fedSBen Skeggs int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 1725f300fedSBen Skeggs int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 1735f300fedSBen Skeggs 1748e39abffSBen Skeggs int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 1758e39abffSBen Skeggs 1765b17f362SBen Skeggs int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 1775b17f362SBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 17877783435SBen Skeggs int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 17977783435SBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 18003b0ba7bSBen Skeggs int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 18103b0ba7bSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 1829f6219fdSBen Skeggs int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 1839f6219fdSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 1849f6219fdSBen Skeggs int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 1859f6219fdSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 186540a1ddeSBen Skeggs int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 187540a1ddeSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 1887de078aaSBen Skeggs int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 1897de078aaSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 1907de078aaSBen Skeggs int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 1917de078aaSBen Skeggs struct lock_class_key *, const char *, struct nvkm_vmm **); 1925f300fedSBen Skeggs int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, 1935f300fedSBen Skeggs struct lock_class_key *, const char *, 1945f300fedSBen Skeggs struct nvkm_vmm **); 1955f300fedSBen Skeggs int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 1965f300fedSBen Skeggs struct lock_class_key *, const char *, 1975f300fedSBen Skeggs struct nvkm_vmm **); 1985f300fedSBen Skeggs int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, 1995f300fedSBen Skeggs struct lock_class_key *, const char *, 2005f300fedSBen Skeggs struct nvkm_vmm **); 2015f300fedSBen Skeggs int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2025f300fedSBen Skeggs struct lock_class_key *, const char *, 2035f300fedSBen Skeggs struct nvkm_vmm **); 2048e39abffSBen Skeggs int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2058e39abffSBen Skeggs struct lock_class_key *, const char *, 2068e39abffSBen Skeggs struct nvkm_vmm **); 2078e39abffSBen Skeggs int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 2088e39abffSBen Skeggs struct lock_class_key *, const char *, 2098e39abffSBen Skeggs struct nvkm_vmm **); 210eb813999SBen Skeggs 211eb813999SBen Skeggs #define VMM_PRINT(l,v,p,f,a...) do { \ 212eb813999SBen Skeggs struct nvkm_vmm *_vmm = (v); \ 213eb813999SBen Skeggs if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \ 214eb813999SBen Skeggs nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \ 215eb813999SBen Skeggs _vmm->name, ##a); \ 216eb813999SBen Skeggs } \ 217eb813999SBen Skeggs } while(0) 218eb813999SBen Skeggs #define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a) 219eb813999SBen Skeggs #define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a) 220eb813999SBen Skeggs #define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a) 221eb813999SBen Skeggs 222eb813999SBen Skeggs #define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \ 223eb813999SBen Skeggs nvkm_kmap((PT)->memory); \ 224eb813999SBen Skeggs while (PTEN) { \ 225eb813999SBen Skeggs u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \ 226eb813999SBen Skeggs u64 _addr = ((BASE) + MAP->off); \ 227eb813999SBen Skeggs \ 228eb813999SBen Skeggs if (_ptes > PTEN) { \ 229eb813999SBen Skeggs MAP->off += PTEN << MAP->page->shift; \ 230eb813999SBen Skeggs _ptes = PTEN; \ 231eb813999SBen Skeggs } else { \ 232eb813999SBen Skeggs MAP->off = 0; \ 233eb813999SBen Skeggs NEXT; \ 234eb813999SBen Skeggs } \ 235eb813999SBen Skeggs \ 236eb813999SBen Skeggs VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \ 237eb813999SBen Skeggs \ 238eb813999SBen Skeggs FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \ 239eb813999SBen Skeggs PTEI += _ptes; \ 240eb813999SBen Skeggs PTEN -= _ptes; \ 241eb813999SBen Skeggs }; \ 242eb813999SBen Skeggs nvkm_done((PT)->memory); \ 243eb813999SBen Skeggs } while(0) 244eb813999SBen Skeggs 245eb813999SBen Skeggs #define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \ 246eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 247eb813999SBen Skeggs ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \ 248eb813999SBen Skeggs ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \ 249eb813999SBen Skeggs (MAP->mem = MAP->mem->next)) 250eb813999SBen Skeggs #define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \ 251eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 252eb813999SBen Skeggs *MAP->dma, PAGE_SIZE, MAP->dma++) 253eb813999SBen Skeggs #define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \ 254eb813999SBen Skeggs VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 255eb813999SBen Skeggs sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \ 256eb813999SBen Skeggs (MAP->sgl = sg_next(MAP->sgl))) 257eb813999SBen Skeggs 258eb813999SBen Skeggs #define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c)) 259eb813999SBen Skeggs #define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d)) 260eb813999SBen Skeggs #define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \ 261eb813999SBen Skeggs const u32 _pteo = (o); u##b _data = (d); \ 262eb813999SBen Skeggs VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \ 263eb813999SBen Skeggs VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \ 264eb813999SBen Skeggs } while(0) 265eb813999SBen Skeggs 266eb813999SBen Skeggs #define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x") 267eb813999SBen Skeggs #define VMM_FO032(m,v,o,d,c) \ 268eb813999SBen Skeggs VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c)) 269eb813999SBen Skeggs 270eb813999SBen Skeggs #define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx") 271eb813999SBen Skeggs #define VMM_FO064(m,v,o,d,c) \ 272eb813999SBen Skeggs VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c)) 273eb813999SBen Skeggs 274eb813999SBen Skeggs #define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \ 275eb813999SBen Skeggs u32 _pteo = (o), _ptes = (c); \ 276eb813999SBen Skeggs const u64 _addr = (m)->addr + _pteo; \ 277eb813999SBen Skeggs VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \ 278eb813999SBen Skeggs while (_ptes--) { \ 279eb813999SBen Skeggs nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \ 280eb813999SBen Skeggs nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \ 281eb813999SBen Skeggs _pteo += 0x10; \ 282eb813999SBen Skeggs } \ 283eb813999SBen Skeggs } while(0) 284eb813999SBen Skeggs 285eb813999SBen Skeggs #define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "") 286eb813999SBen Skeggs #define VMM_FO128(m,v,o,lo,hi,c) do { \ 287eb813999SBen Skeggs nvkm_kmap((m)->memory); \ 288eb813999SBen Skeggs VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \ 289eb813999SBen Skeggs nvkm_done((m)->memory); \ 290eb813999SBen Skeggs } while(0) 291806a7335SBen Skeggs #endif 292