Lines Matching full:vm
89 struct drm_i915_private *i915 = ppgtt->vm.i915; in gen8_ppgtt_notify_vgt()
90 struct intel_uncore *uncore = ppgtt->vm.gt->uncore; in gen8_ppgtt_notify_vgt()
101 if (i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_notify_vgt()
179 static unsigned int gen8_pd_top_count(const struct i915_address_space *vm) in gen8_pd_top_count() argument
181 unsigned int shift = __gen8_pte_shift(vm->top); in gen8_pd_top_count()
183 return (vm->total + (1ull << shift) - 1) >> shift; in gen8_pd_top_count()
187 gen8_pdp_for_page_index(struct i915_address_space * const vm, const u64 idx) in gen8_pdp_for_page_index() argument
189 struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm); in gen8_pdp_for_page_index()
191 if (vm->top == 2) in gen8_pdp_for_page_index()
194 return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top)); in gen8_pdp_for_page_index()
198 gen8_pdp_for_page_address(struct i915_address_space * const vm, const u64 addr) in gen8_pdp_for_page_address() argument
200 return gen8_pdp_for_page_index(vm, addr >> GEN8_PTE_SHIFT); in gen8_pdp_for_page_address()
203 static void __gen8_ppgtt_cleanup(struct i915_address_space *vm, in __gen8_ppgtt_cleanup() argument
214 __gen8_ppgtt_cleanup(vm, *pde, GEN8_PDES, lvl - 1); in __gen8_ppgtt_cleanup()
218 free_px(vm, &pd->pt, lvl); in __gen8_ppgtt_cleanup()
221 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) in gen8_ppgtt_cleanup() argument
223 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); in gen8_ppgtt_cleanup()
225 if (intel_vgpu_active(vm->i915)) in gen8_ppgtt_cleanup()
229 __gen8_ppgtt_cleanup(vm, ppgtt->pd, in gen8_ppgtt_cleanup()
230 gen8_pd_top_count(vm), vm->top); in gen8_ppgtt_cleanup()
232 free_scratch(vm); in gen8_ppgtt_cleanup()
235 static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, in __gen8_ppgtt_clear() argument
239 const struct drm_i915_gem_object * const scratch = vm->scratch[lvl]; in __gen8_ppgtt_clear()
242 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_clear()
246 __func__, vm, lvl + 1, start, end, in __gen8_ppgtt_clear()
256 __func__, vm, lvl + 1, idx, start, end); in __gen8_ppgtt_clear()
258 __gen8_ppgtt_cleanup(vm, as_pd(pt), I915_PDES, lvl); in __gen8_ppgtt_clear()
264 start = __gen8_ppgtt_clear(vm, as_pd(pt), in __gen8_ppgtt_clear()
274 __func__, vm, lvl, start, end, in __gen8_ppgtt_clear()
289 vm->scratch[0]->encode, in __gen8_ppgtt_clear()
297 free_px(vm, pt, lvl); in __gen8_ppgtt_clear()
303 static void gen8_ppgtt_clear(struct i915_address_space *vm, in gen8_ppgtt_clear() argument
308 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_clear()
314 __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_clear()
315 start, start + length, vm->top); in gen8_ppgtt_clear()
318 static void __gen8_ppgtt_alloc(struct i915_address_space * const vm, in __gen8_ppgtt_alloc() argument
325 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_alloc()
329 __func__, vm, lvl + 1, *start, end, in __gen8_ppgtt_alloc()
342 __func__, vm, lvl + 1, idx); in __gen8_ppgtt_alloc()
347 fill_px(pt, vm->scratch[lvl]->encode); in __gen8_ppgtt_alloc()
363 __gen8_ppgtt_alloc(vm, stash, in __gen8_ppgtt_alloc()
373 __func__, vm, lvl, *start, end, in __gen8_ppgtt_alloc()
386 static void gen8_ppgtt_alloc(struct i915_address_space *vm, in gen8_ppgtt_alloc() argument
392 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_alloc()
398 __gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_alloc()
399 &start, start + length, vm->top); in gen8_ppgtt_alloc()
402 static void __gen8_ppgtt_foreach(struct i915_address_space *vm, in __gen8_ppgtt_foreach() argument
405 void (*fn)(struct i915_address_space *vm, in __gen8_ppgtt_foreach() argument
422 __gen8_ppgtt_foreach(vm, as_pd(pt), start, end, lvl, in __gen8_ppgtt_foreach()
425 fn(vm, pt, data); in __gen8_ppgtt_foreach()
435 static void gen8_ppgtt_foreach(struct i915_address_space *vm, in gen8_ppgtt_foreach() argument
437 void (*fn)(struct i915_address_space *vm, in gen8_ppgtt_foreach() argument
445 __gen8_ppgtt_foreach(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_foreach()
446 &start, start + length, vm->top, in gen8_ppgtt_foreach()
459 const gen8_pte_t pte_encode = ppgtt->vm.pte_encode(0, pat_index, flags); in gen8_ppgtt_insert_pte()
499 xehpsdv_ppgtt_insert_huge(struct i915_address_space *vm, in xehpsdv_ppgtt_insert_huge() argument
505 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); in xehpsdv_ppgtt_insert_huge()
510 GEM_BUG_ON(!i915_vm_is_4lvl(vm)); in xehpsdv_ppgtt_insert_huge()
514 gen8_pdp_for_page_address(vm, start); in xehpsdv_ppgtt_insert_huge()
607 static void gen8_ppgtt_insert_huge(struct i915_address_space *vm, in gen8_ppgtt_insert_huge() argument
613 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); in gen8_ppgtt_insert_huge()
617 GEM_BUG_ON(!i915_vm_is_4lvl(vm)); in gen8_ppgtt_insert_huge()
621 gen8_pdp_for_page_address(vm, start); in gen8_ppgtt_insert_huge()
696 (i915_vm_has_scratch_64K(vm) && in gen8_ppgtt_insert_huge()
714 if (I915_SELFTEST_ONLY(vm->scrub_64K)) { in gen8_ppgtt_insert_huge()
717 encode = vm->scratch[0]->encode; in gen8_ppgtt_insert_huge()
731 static void gen8_ppgtt_insert(struct i915_address_space *vm, in gen8_ppgtt_insert() argument
736 struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm); in gen8_ppgtt_insert()
740 if (GRAPHICS_VER_FULL(vm->i915) >= IP_VER(12, 50)) in gen8_ppgtt_insert()
741 xehpsdv_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags); in gen8_ppgtt_insert()
743 gen8_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags); in gen8_ppgtt_insert()
749 gen8_pdp_for_page_index(vm, idx); in gen8_ppgtt_insert()
759 static void gen8_ppgtt_insert_entry(struct i915_address_space *vm, in gen8_ppgtt_insert_entry() argument
767 gen8_pdp_for_page_index(vm, idx); in gen8_ppgtt_insert_entry()
776 vaddr[gen8_pd_index(idx, 0)] = vm->pte_encode(addr, pat_index, flags); in gen8_ppgtt_insert_entry()
780 static void __xehpsdv_ppgtt_insert_entry_lm(struct i915_address_space *vm, in __xehpsdv_ppgtt_insert_entry_lm() argument
788 gen8_pdp_for_page_index(vm, idx); in __xehpsdv_ppgtt_insert_entry_lm()
806 vaddr[gen8_pd_index(idx, 0) / 16] = vm->pte_encode(addr, pat_index, flags); in __xehpsdv_ppgtt_insert_entry_lm()
809 static void xehpsdv_ppgtt_insert_entry(struct i915_address_space *vm, in xehpsdv_ppgtt_insert_entry() argument
816 return __xehpsdv_ppgtt_insert_entry_lm(vm, addr, offset, in xehpsdv_ppgtt_insert_entry()
819 return gen8_ppgtt_insert_entry(vm, addr, offset, pat_index, flags); in xehpsdv_ppgtt_insert_entry()
822 static int gen8_init_scratch(struct i915_address_space *vm) in gen8_init_scratch() argument
830 * we can reuse it for all vm, keeping contexts and processes separate. in gen8_init_scratch()
832 if (vm->has_read_only && vm->gt->vm && !i915_is_ggtt(vm->gt->vm)) { in gen8_init_scratch()
833 struct i915_address_space *clone = vm->gt->vm; in gen8_init_scratch()
837 vm->scratch_order = clone->scratch_order; in gen8_init_scratch()
838 for (i = 0; i <= vm->top; i++) in gen8_init_scratch()
839 vm->scratch[i] = i915_gem_object_get(clone->scratch[i]); in gen8_init_scratch()
844 ret = setup_scratch_page(vm); in gen8_init_scratch()
848 pte_flags = vm->has_read_only; in gen8_init_scratch()
849 if (i915_gem_object_is_lmem(vm->scratch[0])) in gen8_init_scratch()
852 vm->scratch[0]->encode = in gen8_init_scratch()
853 vm->pte_encode(px_dma(vm->scratch[0]), in gen8_init_scratch()
854 i915_gem_get_pat_index(vm->i915, in gen8_init_scratch()
858 for (i = 1; i <= vm->top; i++) { in gen8_init_scratch()
861 obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_init_scratch()
867 ret = map_pt_dma(vm, obj); in gen8_init_scratch()
873 fill_px(obj, vm->scratch[i - 1]->encode); in gen8_init_scratch()
876 vm->scratch[i] = obj; in gen8_init_scratch()
883 i915_gem_object_put(vm->scratch[i]); in gen8_init_scratch()
884 vm->scratch[0] = NULL; in gen8_init_scratch()
890 struct i915_address_space *vm = &ppgtt->vm; in gen8_preallocate_top_level_pdp() local
894 GEM_BUG_ON(vm->top != 2); in gen8_preallocate_top_level_pdp()
895 GEM_BUG_ON(gen8_pd_top_count(vm) != GEN8_3LVL_PDPES); in gen8_preallocate_top_level_pdp()
901 pde = alloc_pd(vm); in gen8_preallocate_top_level_pdp()
905 err = map_pt_dma(vm, pde->pt.base); in gen8_preallocate_top_level_pdp()
907 free_pd(vm, pde); in gen8_preallocate_top_level_pdp()
911 fill_px(pde, vm->scratch[1]->encode); in gen8_preallocate_top_level_pdp()
921 gen8_alloc_top_pd(struct i915_address_space *vm) in gen8_alloc_top_pd() argument
923 const unsigned int count = gen8_pd_top_count(vm); in gen8_alloc_top_pd()
933 pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_alloc_top_pd()
940 err = map_pt_dma(vm, pd->pt.base); in gen8_alloc_top_pd()
944 fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count); in gen8_alloc_top_pd()
949 free_pd(vm, pd); in gen8_alloc_top_pd()
972 ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2; in gen8_ppgtt_create()
973 ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t)); in gen8_ppgtt_create()
983 ppgtt->vm.has_read_only = !IS_GRAPHICS_VER(gt->i915, 11, 12); in gen8_ppgtt_create()
986 ppgtt->vm.alloc_pt_dma = alloc_pt_lmem; in gen8_ppgtt_create()
988 ppgtt->vm.alloc_pt_dma = alloc_pt_dma; in gen8_ppgtt_create()
996 ppgtt->vm.alloc_scratch_dma = alloc_pt_dma; in gen8_ppgtt_create()
999 ppgtt->vm.pte_encode = gen12_pte_encode; in gen8_ppgtt_create()
1001 ppgtt->vm.pte_encode = gen8_pte_encode; in gen8_ppgtt_create()
1003 ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; in gen8_ppgtt_create()
1004 ppgtt->vm.insert_entries = gen8_ppgtt_insert; in gen8_ppgtt_create()
1006 ppgtt->vm.insert_page = xehpsdv_ppgtt_insert_entry; in gen8_ppgtt_create()
1008 ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; in gen8_ppgtt_create()
1009 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; in gen8_ppgtt_create()
1010 ppgtt->vm.clear_range = gen8_ppgtt_clear; in gen8_ppgtt_create()
1011 ppgtt->vm.foreach = gen8_ppgtt_foreach; in gen8_ppgtt_create()
1012 ppgtt->vm.cleanup = gen8_ppgtt_cleanup; in gen8_ppgtt_create()
1014 err = gen8_init_scratch(&ppgtt->vm); in gen8_ppgtt_create()
1018 pd = gen8_alloc_top_pd(&ppgtt->vm); in gen8_ppgtt_create()
1025 if (!i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_create()
1037 i915_vm_put(&ppgtt->vm); in gen8_ppgtt_create()