1b7019ac5SIlia Mirkin /* SPDX-License-Identifier: MIT */ 242594600SBen Skeggs #ifndef __NVKM_MMU_H__ 342594600SBen Skeggs #define __NVKM_MMU_H__ 45ce3bf3cSBen Skeggs #include <core/subdev.h> 55ce3bf3cSBen Skeggs 642594600SBen Skeggs struct nvkm_vma { 75ce3bf3cSBen Skeggs struct list_head head; 8f9463a4bSBen Skeggs struct rb_node tree; 99ce523ccSBen Skeggs u64 addr; 10f9463a4bSBen Skeggs u64 size:50; 11f9463a4bSBen Skeggs bool mapref:1; /* PTs (de)referenced on (un)map (vs pre-allocated). */ 12f9463a4bSBen Skeggs bool sparse:1; /* Unmapped PDEs/PTEs will not trigger MMU faults. */ 13f9463a4bSBen Skeggs #define NVKM_VMA_PAGE_NONE 7 14f9463a4bSBen Skeggs u8 page:3; /* Requested page type (index, or NONE for automatic). */ 15f9463a4bSBen Skeggs u8 refd:3; /* Current page type (index, or NONE for unreferenced). */ 16f9463a4bSBen Skeggs bool used:1; /* Region allocated. */ 17f9463a4bSBen Skeggs bool part:1; /* Region was split from an allocated region by map(). */ 18f9463a4bSBen Skeggs bool busy:1; /* Region busy (for temporarily preventing user access). */ 198e68271dSBen Skeggs bool mapped:1; /* Region contains valid pages. */ 20*6b252cf4SDanilo Krummrich bool no_comp:1; /* Force no memory compression. */ 21f9463a4bSBen Skeggs struct nvkm_memory *memory; /* Memory currently mapped into VMA. */ 22f9463a4bSBen Skeggs struct nvkm_tags *tags; /* Compression tag reference. */ 235ce3bf3cSBen Skeggs }; 245ce3bf3cSBen Skeggs 25632b740cSBen Skeggs struct nvkm_vmm { 26806a7335SBen Skeggs const struct nvkm_vmm_func *func; 2742594600SBen Skeggs struct nvkm_mmu *mmu; 28806a7335SBen Skeggs const char *name; 29eb813999SBen Skeggs u32 debug; 30806a7335SBen Skeggs struct kref kref; 31*6b252cf4SDanilo Krummrich 32*6b252cf4SDanilo Krummrich struct { 33*6b252cf4SDanilo Krummrich struct mutex vmm; 34*6b252cf4SDanilo Krummrich struct mutex ref; 35*6b252cf4SDanilo Krummrich struct mutex map; 36*6b252cf4SDanilo Krummrich } mutex; 375ce3bf3cSBen Skeggs 38806a7335SBen Skeggs u64 start; 39806a7335SBen Skeggs u64 limit; 40*6b252cf4SDanilo Krummrich struct { 41*6b252cf4SDanilo Krummrich struct { 42*6b252cf4SDanilo Krummrich u64 addr; 43*6b252cf4SDanilo Krummrich u64 size; 44*6b252cf4SDanilo Krummrich } p; 45*6b252cf4SDanilo Krummrich struct { 46*6b252cf4SDanilo Krummrich u64 addr; 47*6b252cf4SDanilo Krummrich u64 size; 48*6b252cf4SDanilo Krummrich } n; 49*6b252cf4SDanilo Krummrich bool raw; 50*6b252cf4SDanilo Krummrich } managed; 51806a7335SBen Skeggs 52806a7335SBen Skeggs struct nvkm_vmm_pt *pd; 53806a7335SBen Skeggs struct list_head join; 54806a7335SBen Skeggs 55f9463a4bSBen Skeggs struct list_head list; 56f9463a4bSBen Skeggs struct rb_root free; 57f9463a4bSBen Skeggs struct rb_root root; 585e075fdeSBen Skeggs 595e075fdeSBen Skeggs bool bootstrapped; 6068f3f702SBen Skeggs atomic_t engref[NVKM_SUBDEV_NR]; 615ce3bf3cSBen Skeggs 6203b0ba7bSBen Skeggs dma_addr_t null; 6303b0ba7bSBen Skeggs void *nullp; 64ab2ee9ffSBen Skeggs 65ab2ee9ffSBen Skeggs bool replay; 665ce3bf3cSBen Skeggs }; 675ce3bf3cSBen Skeggs 68f9463a4bSBen Skeggs int nvkm_vmm_new(struct nvkm_device *, u64 addr, u64 size, void *argv, u32 argc, 69f9463a4bSBen Skeggs struct lock_class_key *, const char *name, struct nvkm_vmm **); 70f9463a4bSBen Skeggs struct nvkm_vmm *nvkm_vmm_ref(struct nvkm_vmm *); 71f9463a4bSBen Skeggs void nvkm_vmm_unref(struct nvkm_vmm **); 72eb813999SBen Skeggs int nvkm_vmm_boot(struct nvkm_vmm *); 73f9463a4bSBen Skeggs int nvkm_vmm_join(struct nvkm_vmm *, struct nvkm_memory *inst); 74f9463a4bSBen Skeggs void nvkm_vmm_part(struct nvkm_vmm *, struct nvkm_memory *inst); 75f9463a4bSBen Skeggs int nvkm_vmm_get(struct nvkm_vmm *, u8 page, u64 size, struct nvkm_vma **); 76f9463a4bSBen Skeggs void nvkm_vmm_put(struct nvkm_vmm *, struct nvkm_vma **); 77eb813999SBen Skeggs 78eb813999SBen Skeggs struct nvkm_vmm_map { 79eb813999SBen Skeggs struct nvkm_memory *memory; 80eb813999SBen Skeggs u64 offset; 81eb813999SBen Skeggs 82eb813999SBen Skeggs struct nvkm_mm_node *mem; 83eb813999SBen Skeggs struct scatterlist *sgl; 84eb813999SBen Skeggs dma_addr_t *dma; 85a5ff307fSBen Skeggs u64 *pfn; 86eb813999SBen Skeggs u64 off; 87eb813999SBen Skeggs 88eb813999SBen Skeggs const struct nvkm_vmm_page *page; 89eb813999SBen Skeggs 90*6b252cf4SDanilo Krummrich bool no_comp; 91eb813999SBen Skeggs struct nvkm_tags *tags; 92eb813999SBen Skeggs u64 next; 93eb813999SBen Skeggs u64 type; 94eb813999SBen Skeggs u64 ctag; 95eb813999SBen Skeggs }; 96eb813999SBen Skeggs 97f9463a4bSBen Skeggs int nvkm_vmm_map(struct nvkm_vmm *, struct nvkm_vma *, void *argv, u32 argc, 98f9463a4bSBen Skeggs struct nvkm_vmm_map *); 99f9463a4bSBen Skeggs void nvkm_vmm_unmap(struct nvkm_vmm *, struct nvkm_vma *); 100f9463a4bSBen Skeggs 101c83c4097SBen Skeggs struct nvkm_memory *nvkm_umem_search(struct nvkm_client *, u64); 102f9463a4bSBen Skeggs struct nvkm_vmm *nvkm_uvmm_search(struct nvkm_client *, u64 handle); 103c9582455SBen Skeggs 104c9582455SBen Skeggs struct nvkm_mmu { 105c9582455SBen Skeggs const struct nvkm_mmu_func *func; 106c9582455SBen Skeggs struct nvkm_subdev subdev; 107c9582455SBen Skeggs 108c9582455SBen Skeggs u8 dma_bits; 1090b11b30dSBen Skeggs 11051645eb7SBen Skeggs int heap_nr; 11151645eb7SBen Skeggs struct { 11251645eb7SBen Skeggs #define NVKM_MEM_VRAM 0x01 11351645eb7SBen Skeggs #define NVKM_MEM_HOST 0x02 11451645eb7SBen Skeggs #define NVKM_MEM_COMP 0x04 11551645eb7SBen Skeggs #define NVKM_MEM_DISP 0x08 11651645eb7SBen Skeggs u8 type; 11751645eb7SBen Skeggs u64 size; 11851645eb7SBen Skeggs } heap[4]; 11951645eb7SBen Skeggs 12051645eb7SBen Skeggs int type_nr; 12151645eb7SBen Skeggs struct { 12251645eb7SBen Skeggs #define NVKM_MEM_KIND 0x10 12351645eb7SBen Skeggs #define NVKM_MEM_MAPPABLE 0x20 12451645eb7SBen Skeggs #define NVKM_MEM_COHERENT 0x40 12551645eb7SBen Skeggs #define NVKM_MEM_UNCACHED 0x80 12651645eb7SBen Skeggs u8 type; 12751645eb7SBen Skeggs u8 heap; 12851645eb7SBen Skeggs } type[16]; 12951645eb7SBen Skeggs 1300b11b30dSBen Skeggs struct nvkm_vmm *vmm; 1319a45ddaaSBen Skeggs 1329a45ddaaSBen Skeggs struct { 1339a45ddaaSBen Skeggs struct mutex mutex; 1349a45ddaaSBen Skeggs struct list_head list; 135f1280394SBen Skeggs } ptc, ptp; 136eea5cf0fSBen Skeggs 1375ec69c91SBen Skeggs struct mutex mutex; /* serialises mmu invalidations */ 1385ec69c91SBen Skeggs 139eea5cf0fSBen Skeggs struct nvkm_device_oclass user; 140c9582455SBen Skeggs }; 141c9582455SBen Skeggs 1426dd123baSBen Skeggs int nv04_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1436dd123baSBen Skeggs int nv41_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1446dd123baSBen Skeggs int nv44_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1456dd123baSBen Skeggs int nv50_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1466dd123baSBen Skeggs int g84_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1476dd123baSBen Skeggs int mcp77_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1486dd123baSBen Skeggs int gf100_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1496dd123baSBen Skeggs int gk104_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1506dd123baSBen Skeggs int gk20a_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1516dd123baSBen Skeggs int gm200_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1526dd123baSBen Skeggs int gm20b_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1536dd123baSBen Skeggs int gp100_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1546dd123baSBen Skeggs int gp10b_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1556dd123baSBen Skeggs int gv100_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1566dd123baSBen Skeggs int tu102_mmu_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mmu **); 1575ce3bf3cSBen Skeggs #endif 158