1 #ifndef __NVKM_VMM_H__ 2 #define __NVKM_VMM_H__ 3 #include "priv.h" 4 #include <core/memory.h> 5 enum nvkm_memory_target; 6 7 struct nvkm_vmm_pt { 8 /* Some GPUs have a mapping level with a dual page tables to 9 * support large and small pages in the same address-range. 10 * 11 * We track the state of both page tables in one place, which 12 * is why there's multiple PT pointers/refcounts here. 13 */ 14 struct nvkm_mmu_pt *pt[2]; 15 u32 refs[2]; 16 17 /* Page size handled by this PT. 18 * 19 * Tesla backend needs to know this when writinge PDEs, 20 * otherwise unnecessary. 21 */ 22 u8 page; 23 24 /* Entire page table sparse. 25 * 26 * Used to propagate sparseness to child page tables. 27 */ 28 bool sparse:1; 29 30 /* Tracking for page directories. 31 * 32 * The array is indexed by PDE, and will either point to the 33 * child page table, or indicate the PDE is marked as sparse. 34 **/ 35 #define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde) 36 #define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde) 37 #define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY) 38 struct nvkm_vmm_pt **pde; 39 40 /* Tracking for dual page tables. 41 * 42 * There's one entry for each LPTE, keeping track of whether 43 * there are valid SPTEs in the same address-range. 44 * 45 * This information is used to manage LPTE state transitions. 46 */ 47 #define NVKM_VMM_PTE_SPARSE 0x80 48 #define NVKM_VMM_PTE_VALID 0x40 49 #define NVKM_VMM_PTE_SPTES 0x3f 50 u8 pte[]; 51 }; 52 53 typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *, 54 struct nvkm_mmu_pt *, u32 ptei, u32 ptes); 55 typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *, 56 struct nvkm_vmm_pt *, u32 pdei); 57 typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *, 58 u32 ptei, u32 ptes, struct nvkm_vmm_map *); 59 60 struct nvkm_vmm_desc_func { 61 nvkm_vmm_pxe_func invalid; 62 nvkm_vmm_pxe_func unmap; 63 nvkm_vmm_pxe_func sparse; 64 65 nvkm_vmm_pde_func pde; 66 67 nvkm_vmm_pte_func mem; 68 nvkm_vmm_pte_func dma; 69 nvkm_vmm_pte_func sgl; 70 }; 71 72 extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; 73 void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32); 74 extern const struct nvkm_vmm_desc_func gf100_vmm_pgt; 75 void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); 76 void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 77 struct nvkm_vmm_map *); 78 void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 79 struct nvkm_vmm_map *); 80 void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 81 struct nvkm_vmm_map *); 82 83 void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); 84 85 struct nvkm_vmm_desc { 86 enum { 87 PGD, 88 PGT, 89 SPT, 90 LPT, 91 } type; 92 u8 bits; /* VMA bits covered by PT. */ 93 u8 size; /* Bytes-per-PTE. */ 94 u32 align; /* PT address alignment. */ 95 const struct nvkm_vmm_desc_func *func; 96 }; 97 98 extern const struct nvkm_vmm_desc nv50_vmm_desc_12[]; 99 extern const struct nvkm_vmm_desc nv50_vmm_desc_16[]; 100 101 extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[]; 102 extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[]; 103 extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[]; 104 extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[]; 105 106 extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[]; 107 extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[]; 108 extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[]; 109 extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[]; 110 111 extern const struct nvkm_vmm_desc gp100_vmm_desc_12[]; 112 extern const struct nvkm_vmm_desc gp100_vmm_desc_16[]; 113 114 struct nvkm_vmm_page { 115 u8 shift; 116 const struct nvkm_vmm_desc *desc; 117 #define NVKM_VMM_PAGE_SPARSE 0x01 118 #define NVKM_VMM_PAGE_VRAM 0x02 119 #define NVKM_VMM_PAGE_HOST 0x04 120 #define NVKM_VMM_PAGE_COMP 0x08 121 #define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE) 122 #define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM) 123 #define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM) 124 #define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST) 125 #define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST) 126 #define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST) 127 #define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST) 128 #define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP) 129 #define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP) 130 #define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP) 131 #define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP) 132 u8 type; 133 }; 134 135 struct nvkm_vmm_func { 136 int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst); 137 void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst); 138 139 int (*aper)(enum nvkm_memory_target); 140 int (*valid)(struct nvkm_vmm *, void *argv, u32 argc, 141 struct nvkm_vmm_map *); 142 void (*flush)(struct nvkm_vmm *, int depth); 143 144 u64 page_block; 145 const struct nvkm_vmm_page page[]; 146 }; 147 148 struct nvkm_vmm_join { 149 struct nvkm_memory *inst; 150 struct list_head head; 151 }; 152 153 int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, 154 u32 pd_header, u64 addr, u64 size, struct lock_class_key *, 155 const char *name, struct nvkm_vmm **); 156 int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, 157 u32 pd_header, u64 addr, u64 size, struct lock_class_key *, 158 const char *name, struct nvkm_vmm *); 159 struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr); 160 int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, 161 bool sparse, u8 page, u8 align, u64 size, 162 struct nvkm_vma **pvma); 163 void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); 164 void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *); 165 void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma); 166 167 struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); 168 void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *); 169 170 int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, 171 u64, u64, void *, u32, struct lock_class_key *, 172 const char *, struct nvkm_vmm **); 173 int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 174 175 int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 176 void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 177 int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 178 void nv50_vmm_flush(struct nvkm_vmm *, int); 179 180 int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 181 struct nvkm_mmu *, u64, u64, void *, u32, 182 struct lock_class_key *, const char *, struct nvkm_vmm **); 183 int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 184 int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 185 void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 186 int gf100_vmm_aper(enum nvkm_memory_target); 187 int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 188 void gf100_vmm_flush_(struct nvkm_vmm *, int); 189 void gf100_vmm_flush(struct nvkm_vmm *, int); 190 191 int gk20a_vmm_aper(enum nvkm_memory_target); 192 193 int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 194 struct nvkm_mmu *, u64, u64, void *, u32, 195 struct lock_class_key *, const char *, struct nvkm_vmm **); 196 int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 197 int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 198 199 int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 200 int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 201 void gp100_vmm_flush(struct nvkm_vmm *, int); 202 203 int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 204 struct lock_class_key *, const char *, struct nvkm_vmm **); 205 int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 206 struct lock_class_key *, const char *, struct nvkm_vmm **); 207 int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 208 struct lock_class_key *, const char *, struct nvkm_vmm **); 209 int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 210 struct lock_class_key *, const char *, struct nvkm_vmm **); 211 int mcp77_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 212 struct lock_class_key *, const char *, struct nvkm_vmm **); 213 int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 214 struct lock_class_key *, const char *, struct nvkm_vmm **); 215 int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 216 struct lock_class_key *, const char *, struct nvkm_vmm **); 217 int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 218 struct lock_class_key *, const char *, struct nvkm_vmm **); 219 int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 220 struct lock_class_key *, const char *, struct nvkm_vmm **); 221 int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, 222 struct lock_class_key *, const char *, 223 struct nvkm_vmm **); 224 int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 225 struct lock_class_key *, const char *, 226 struct nvkm_vmm **); 227 int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, 228 struct lock_class_key *, const char *, 229 struct nvkm_vmm **); 230 int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 231 struct lock_class_key *, const char *, 232 struct nvkm_vmm **); 233 int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 234 struct lock_class_key *, const char *, 235 struct nvkm_vmm **); 236 int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 237 struct lock_class_key *, const char *, 238 struct nvkm_vmm **); 239 int gv100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, 240 struct lock_class_key *, const char *, 241 struct nvkm_vmm **); 242 243 #define VMM_PRINT(l,v,p,f,a...) do { \ 244 struct nvkm_vmm *_vmm = (v); \ 245 if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \ 246 nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \ 247 _vmm->name, ##a); \ 248 } \ 249 } while(0) 250 #define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a) 251 #define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a) 252 #define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a) 253 254 #define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \ 255 nvkm_kmap((PT)->memory); \ 256 while (PTEN) { \ 257 u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \ 258 u64 _addr = ((BASE) + MAP->off); \ 259 \ 260 if (_ptes > PTEN) { \ 261 MAP->off += PTEN << MAP->page->shift; \ 262 _ptes = PTEN; \ 263 } else { \ 264 MAP->off = 0; \ 265 NEXT; \ 266 } \ 267 \ 268 VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \ 269 \ 270 FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \ 271 PTEI += _ptes; \ 272 PTEN -= _ptes; \ 273 }; \ 274 nvkm_done((PT)->memory); \ 275 } while(0) 276 277 #define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \ 278 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 279 ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \ 280 ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \ 281 (MAP->mem = MAP->mem->next)) 282 #define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \ 283 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 284 *MAP->dma, PAGE_SIZE, MAP->dma++) 285 #define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \ 286 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 287 sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \ 288 (MAP->sgl = sg_next(MAP->sgl))) 289 290 #define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c)) 291 #define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d)) 292 #define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \ 293 const u32 _pteo = (o); u##b _data = (d); \ 294 VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \ 295 VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \ 296 } while(0) 297 298 #define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x") 299 #define VMM_FO032(m,v,o,d,c) \ 300 VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c)) 301 302 #define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx") 303 #define VMM_FO064(m,v,o,d,c) \ 304 VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c)) 305 306 #define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \ 307 u32 _pteo = (o), _ptes = (c); \ 308 const u64 _addr = (m)->addr + _pteo; \ 309 VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \ 310 while (_ptes--) { \ 311 nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \ 312 nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \ 313 _pteo += 0x10; \ 314 } \ 315 } while(0) 316 317 #define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "") 318 #define VMM_FO128(m,v,o,lo,hi,c) do { \ 319 nvkm_kmap((m)->memory); \ 320 VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \ 321 nvkm_done((m)->memory); \ 322 } while(0) 323 #endif 324