142594600SBen Skeggs /* 242594600SBen Skeggs * Copyright 2010 Red Hat Inc. 342594600SBen Skeggs * 442594600SBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a 542594600SBen Skeggs * copy of this software and associated documentation files (the "Software"), 642594600SBen Skeggs * to deal in the Software without restriction, including without limitation 742594600SBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense, 842594600SBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the 942594600SBen Skeggs * Software is furnished to do so, subject to the following conditions: 1042594600SBen Skeggs * 1142594600SBen Skeggs * The above copyright notice and this permission notice shall be included in 1242594600SBen Skeggs * all copies or substantial portions of the Software. 1342594600SBen Skeggs * 1442594600SBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1542594600SBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1642594600SBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1742594600SBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 1842594600SBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 1942594600SBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 2042594600SBen Skeggs * OTHER DEALINGS IN THE SOFTWARE. 2142594600SBen Skeggs * 2242594600SBen Skeggs * Authors: Ben Skeggs 2342594600SBen Skeggs */ 2442594600SBen Skeggs #include <subdev/mmu.h> 2542594600SBen Skeggs #include <subdev/bar.h> 2642594600SBen Skeggs #include <subdev/fb.h> 2742594600SBen Skeggs #include <subdev/ltc.h> 2842594600SBen Skeggs #include <subdev/timer.h> 2942594600SBen Skeggs 3042594600SBen Skeggs #include <core/gpuobj.h> 3142594600SBen Skeggs 3242594600SBen Skeggs struct gf100_mmu_priv { 3342594600SBen Skeggs struct nvkm_mmu base; 3442594600SBen Skeggs }; 3542594600SBen Skeggs 3642594600SBen Skeggs 3742594600SBen Skeggs /* Map from compressed to corresponding uncompressed storage type. 3842594600SBen Skeggs * The value 0xff represents an invalid storage type. 3942594600SBen Skeggs */ 4042594600SBen Skeggs const u8 gf100_pte_storage_type_map[256] = 4142594600SBen Skeggs { 4242594600SBen Skeggs 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0x01, /* 0x00 */ 4342594600SBen Skeggs 0x01, 0x01, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 4442594600SBen Skeggs 0xff, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, /* 0x10 */ 4542594600SBen Skeggs 0x11, 0x11, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 4642594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x27, /* 0x20 */ 4742594600SBen Skeggs 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 4842594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30 */ 4942594600SBen Skeggs 0xff, 0xff, 0x26, 0x27, 0x28, 0x29, 0x26, 0x27, 5042594600SBen Skeggs 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0x46, 0xff, /* 0x40 */ 5142594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 5242594600SBen Skeggs 0xff, 0x46, 0x46, 0x46, 0x46, 0xff, 0xff, 0xff, /* 0x50 */ 5342594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 5442594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60 */ 5542594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 5642594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70 */ 5742594600SBen Skeggs 0xff, 0xff, 0xff, 0x7b, 0xff, 0xff, 0xff, 0xff, 5842594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0x7b, /* 0x80 */ 5942594600SBen Skeggs 0x7b, 0x7b, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xff, 6042594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90 */ 6142594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 6242594600SBen Skeggs 0xff, 0xff, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xa7, /* 0xa0 */ 6342594600SBen Skeggs 0xa8, 0xa9, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 6442594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0 */ 6542594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7, 6642594600SBen Skeggs 0xa8, 0xa9, 0xaa, 0xc3, 0xff, 0xff, 0xff, 0xff, /* 0xc0 */ 6742594600SBen Skeggs 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfe, 0xc3, 0xc3, 6842594600SBen Skeggs 0xc3, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0 */ 6942594600SBen Skeggs 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 7042594600SBen Skeggs 0xfe, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xff, /* 0xe0 */ 7142594600SBen Skeggs 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xfe, 0xff, 7242594600SBen Skeggs 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0 */ 7342594600SBen Skeggs 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff 7442594600SBen Skeggs }; 7542594600SBen Skeggs 7642594600SBen Skeggs 7742594600SBen Skeggs static void 7842594600SBen Skeggs gf100_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 index, struct nvkm_gpuobj *pgt[2]) 7942594600SBen Skeggs { 8042594600SBen Skeggs u32 pde[2] = { 0, 0 }; 8142594600SBen Skeggs 8242594600SBen Skeggs if (pgt[0]) 8342594600SBen Skeggs pde[1] = 0x00000001 | (pgt[0]->addr >> 8); 8442594600SBen Skeggs if (pgt[1]) 8542594600SBen Skeggs pde[0] = 0x00000001 | (pgt[1]->addr >> 8); 8642594600SBen Skeggs 8742594600SBen Skeggs nv_wo32(pgd, (index * 8) + 0, pde[0]); 8842594600SBen Skeggs nv_wo32(pgd, (index * 8) + 4, pde[1]); 8942594600SBen Skeggs } 9042594600SBen Skeggs 9142594600SBen Skeggs static inline u64 9242594600SBen Skeggs gf100_vm_addr(struct nvkm_vma *vma, u64 phys, u32 memtype, u32 target) 9342594600SBen Skeggs { 9442594600SBen Skeggs phys >>= 8; 9542594600SBen Skeggs 9642594600SBen Skeggs phys |= 0x00000001; /* present */ 9742594600SBen Skeggs if (vma->access & NV_MEM_ACCESS_SYS) 9842594600SBen Skeggs phys |= 0x00000002; 9942594600SBen Skeggs 10042594600SBen Skeggs phys |= ((u64)target << 32); 10142594600SBen Skeggs phys |= ((u64)memtype << 36); 10242594600SBen Skeggs return phys; 10342594600SBen Skeggs } 10442594600SBen Skeggs 10542594600SBen Skeggs static void 10642594600SBen Skeggs gf100_vm_map(struct nvkm_vma *vma, struct nvkm_gpuobj *pgt, 10742594600SBen Skeggs struct nvkm_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta) 10842594600SBen Skeggs { 10942594600SBen Skeggs u64 next = 1 << (vma->node->type - 8); 11042594600SBen Skeggs 11142594600SBen Skeggs phys = gf100_vm_addr(vma, phys, mem->memtype, 0); 11242594600SBen Skeggs pte <<= 3; 11342594600SBen Skeggs 11442594600SBen Skeggs if (mem->tag) { 11542594600SBen Skeggs struct nvkm_ltc *ltc = nvkm_ltc(vma->vm->mmu); 11642594600SBen Skeggs u32 tag = mem->tag->offset + (delta >> 17); 11742594600SBen Skeggs phys |= (u64)tag << (32 + 12); 11842594600SBen Skeggs next |= (u64)1 << (32 + 12); 11942594600SBen Skeggs ltc->tags_clear(ltc, tag, cnt); 12042594600SBen Skeggs } 12142594600SBen Skeggs 12242594600SBen Skeggs while (cnt--) { 12342594600SBen Skeggs nv_wo32(pgt, pte + 0, lower_32_bits(phys)); 12442594600SBen Skeggs nv_wo32(pgt, pte + 4, upper_32_bits(phys)); 12542594600SBen Skeggs phys += next; 12642594600SBen Skeggs pte += 8; 12742594600SBen Skeggs } 12842594600SBen Skeggs } 12942594600SBen Skeggs 13042594600SBen Skeggs static void 13142594600SBen Skeggs gf100_vm_map_sg(struct nvkm_vma *vma, struct nvkm_gpuobj *pgt, 13242594600SBen Skeggs struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) 13342594600SBen Skeggs { 13442594600SBen Skeggs u32 target = (vma->access & NV_MEM_ACCESS_NOSNOOP) ? 7 : 5; 13542594600SBen Skeggs /* compressed storage types are invalid for system memory */ 13642594600SBen Skeggs u32 memtype = gf100_pte_storage_type_map[mem->memtype & 0xff]; 13742594600SBen Skeggs 13842594600SBen Skeggs pte <<= 3; 13942594600SBen Skeggs while (cnt--) { 14042594600SBen Skeggs u64 phys = gf100_vm_addr(vma, *list++, memtype, target); 14142594600SBen Skeggs nv_wo32(pgt, pte + 0, lower_32_bits(phys)); 14242594600SBen Skeggs nv_wo32(pgt, pte + 4, upper_32_bits(phys)); 14342594600SBen Skeggs pte += 8; 14442594600SBen Skeggs } 14542594600SBen Skeggs } 14642594600SBen Skeggs 14742594600SBen Skeggs static void 14842594600SBen Skeggs gf100_vm_unmap(struct nvkm_gpuobj *pgt, u32 pte, u32 cnt) 14942594600SBen Skeggs { 15042594600SBen Skeggs pte <<= 3; 15142594600SBen Skeggs while (cnt--) { 15242594600SBen Skeggs nv_wo32(pgt, pte + 0, 0x00000000); 15342594600SBen Skeggs nv_wo32(pgt, pte + 4, 0x00000000); 15442594600SBen Skeggs pte += 8; 15542594600SBen Skeggs } 15642594600SBen Skeggs } 15742594600SBen Skeggs 15842594600SBen Skeggs static void 15942594600SBen Skeggs gf100_vm_flush(struct nvkm_vm *vm) 16042594600SBen Skeggs { 16142594600SBen Skeggs struct gf100_mmu_priv *priv = (void *)vm->mmu; 16242594600SBen Skeggs struct nvkm_bar *bar = nvkm_bar(priv); 16342594600SBen Skeggs struct nvkm_vm_pgd *vpgd; 16442594600SBen Skeggs u32 type; 16542594600SBen Skeggs 16642594600SBen Skeggs bar->flush(bar); 16742594600SBen Skeggs 16842594600SBen Skeggs type = 0x00000001; /* PAGE_ALL */ 16942594600SBen Skeggs if (atomic_read(&vm->engref[NVDEV_SUBDEV_BAR])) 17042594600SBen Skeggs type |= 0x00000004; /* HUB_ONLY */ 17142594600SBen Skeggs 17242594600SBen Skeggs mutex_lock(&nv_subdev(priv)->mutex); 17342594600SBen Skeggs list_for_each_entry(vpgd, &vm->pgd_list, head) { 17442594600SBen Skeggs /* looks like maybe a "free flush slots" counter, the 17542594600SBen Skeggs * faster you write to 0x100cbc to more it decreases 17642594600SBen Skeggs */ 17742594600SBen Skeggs if (!nv_wait_ne(priv, 0x100c80, 0x00ff0000, 0x00000000)) { 17842594600SBen Skeggs nv_error(priv, "vm timeout 0: 0x%08x %d\n", 17942594600SBen Skeggs nv_rd32(priv, 0x100c80), type); 18042594600SBen Skeggs } 18142594600SBen Skeggs 18242594600SBen Skeggs nv_wr32(priv, 0x100cb8, vpgd->obj->addr >> 8); 18342594600SBen Skeggs nv_wr32(priv, 0x100cbc, 0x80000000 | type); 18442594600SBen Skeggs 18542594600SBen Skeggs /* wait for flush to be queued? */ 18642594600SBen Skeggs if (!nv_wait(priv, 0x100c80, 0x00008000, 0x00008000)) { 18742594600SBen Skeggs nv_error(priv, "vm timeout 1: 0x%08x %d\n", 18842594600SBen Skeggs nv_rd32(priv, 0x100c80), type); 18942594600SBen Skeggs } 19042594600SBen Skeggs } 19142594600SBen Skeggs mutex_unlock(&nv_subdev(priv)->mutex); 19242594600SBen Skeggs } 19342594600SBen Skeggs 19442594600SBen Skeggs static int 19542594600SBen Skeggs gf100_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, 19642594600SBen Skeggs struct nvkm_vm **pvm) 19742594600SBen Skeggs { 19842594600SBen Skeggs return nvkm_vm_create(mmu, offset, length, mm_offset, 4096, pvm); 19942594600SBen Skeggs } 20042594600SBen Skeggs 20142594600SBen Skeggs static int 20242594600SBen Skeggs gf100_mmu_ctor(struct nvkm_object *parent, struct nvkm_object *engine, 20342594600SBen Skeggs struct nvkm_oclass *oclass, void *data, u32 size, 20442594600SBen Skeggs struct nvkm_object **pobject) 20542594600SBen Skeggs { 20642594600SBen Skeggs struct gf100_mmu_priv *priv; 20742594600SBen Skeggs int ret; 20842594600SBen Skeggs 20942594600SBen Skeggs ret = nvkm_mmu_create(parent, engine, oclass, "VM", "vm", &priv); 21042594600SBen Skeggs *pobject = nv_object(priv); 21142594600SBen Skeggs if (ret) 21242594600SBen Skeggs return ret; 21342594600SBen Skeggs 21442594600SBen Skeggs priv->base.limit = 1ULL << 40; 21542594600SBen Skeggs priv->base.dma_bits = 40; 21642594600SBen Skeggs priv->base.pgt_bits = 27 - 12; 21742594600SBen Skeggs priv->base.spg_shift = 12; 21842594600SBen Skeggs priv->base.lpg_shift = 17; 21942594600SBen Skeggs priv->base.create = gf100_vm_create; 22042594600SBen Skeggs priv->base.map_pgt = gf100_vm_map_pgt; 22142594600SBen Skeggs priv->base.map = gf100_vm_map; 22242594600SBen Skeggs priv->base.map_sg = gf100_vm_map_sg; 22342594600SBen Skeggs priv->base.unmap = gf100_vm_unmap; 22442594600SBen Skeggs priv->base.flush = gf100_vm_flush; 22542594600SBen Skeggs return 0; 22642594600SBen Skeggs } 22742594600SBen Skeggs 22842594600SBen Skeggs struct nvkm_oclass 22942594600SBen Skeggs gf100_mmu_oclass = { 23042594600SBen Skeggs .handle = NV_SUBDEV(MMU, 0xc0), 23142594600SBen Skeggs .ofuncs = &(struct nvkm_ofuncs) { 23242594600SBen Skeggs .ctor = gf100_mmu_ctor, 23342594600SBen Skeggs .dtor = _nvkm_mmu_dtor, 23442594600SBen Skeggs .init = _nvkm_mmu_init, 23542594600SBen Skeggs .fini = _nvkm_mmu_fini, 23642594600SBen Skeggs }, 23742594600SBen Skeggs }; 238