12c86e55dSMatthew Auld // SPDX-License-Identifier: MIT 22c86e55dSMatthew Auld /* 32c86e55dSMatthew Auld * Copyright © 2020 Intel Corporation 42c86e55dSMatthew Auld */ 52c86e55dSMatthew Auld 62c86e55dSMatthew Auld #include <asm/set_memory.h> 78801eb48SChen Zhou #include <asm/smp.h> 89ce07d94SLucas De Marchi #include <linux/types.h> 99ce07d94SLucas De Marchi #include <linux/stop_machine.h> 102c86e55dSMatthew Auld 110f857158SAravind Iddamsetty #include <drm/drm_managed.h> 1283d2bdb6SJani Nikula #include <drm/i915_drm.h> 139ce07d94SLucas De Marchi #include <drm/intel-gtt.h> 1483d2bdb6SJani Nikula 15e762bdf5SMatthew Auld #include "gem/i915_gem_lmem.h" 16e762bdf5SMatthew Auld 179ce07d94SLucas De Marchi #include "intel_ggtt_gmch.h" 182c86e55dSMatthew Auld #include "intel_gt.h" 190d6419e9SMatt Roper #include "intel_gt_regs.h" 206bba2b30SPiotr Piórkowski #include "intel_pci_config.h" 212c86e55dSMatthew Auld #include "i915_drv.h" 221bba7323SPiotr Piórkowski #include "i915_pci.h" 232c86e55dSMatthew Auld #include "i915_scatterlist.h" 24a7f46d5bSTvrtko Ursulin #include "i915_utils.h" 252c86e55dSMatthew Auld #include "i915_vgpu.h" 262c86e55dSMatthew Auld 272c86e55dSMatthew Auld #include "intel_gtt.h" 2833e7a975SVille Syrjälä #include "gen8_ppgtt.h" 292c86e55dSMatthew Auld 302c86e55dSMatthew Auld static void i915_ggtt_color_adjust(const struct drm_mm_node *node, 312c86e55dSMatthew Auld unsigned long color, 322c86e55dSMatthew Auld u64 *start, 332c86e55dSMatthew Auld u64 *end) 342c86e55dSMatthew Auld { 352c86e55dSMatthew Auld if (i915_node_color_differs(node, color)) 362c86e55dSMatthew Auld *start += I915_GTT_PAGE_SIZE; 372c86e55dSMatthew Auld 382c86e55dSMatthew Auld /* 392c86e55dSMatthew Auld * Also leave a space between the unallocated reserved node after the 402c86e55dSMatthew Auld * GTT and any objects within the GTT, i.e. we use the color adjustment 412c86e55dSMatthew Auld * to insert a guard page to prevent prefetches crossing over the 422c86e55dSMatthew Auld * GTT boundary. 432c86e55dSMatthew Auld */ 442c86e55dSMatthew Auld node = list_next_entry(node, node_list); 452c86e55dSMatthew Auld if (node->color != color) 462c86e55dSMatthew Auld *end -= I915_GTT_PAGE_SIZE; 472c86e55dSMatthew Auld } 482c86e55dSMatthew Auld 492c86e55dSMatthew Auld static int ggtt_init_hw(struct i915_ggtt *ggtt) 502c86e55dSMatthew Auld { 512c86e55dSMatthew Auld struct drm_i915_private *i915 = ggtt->vm.i915; 522c86e55dSMatthew Auld 532c86e55dSMatthew Auld i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); 542c86e55dSMatthew Auld 552c86e55dSMatthew Auld ggtt->vm.is_ggtt = true; 562c86e55dSMatthew Auld 572c86e55dSMatthew Auld /* Only VLV supports read-only GGTT mappings */ 582c86e55dSMatthew Auld ggtt->vm.has_read_only = IS_VALLEYVIEW(i915); 592c86e55dSMatthew Auld 602c86e55dSMatthew Auld if (!HAS_LLC(i915) && !HAS_PPGTT(i915)) 612c86e55dSMatthew Auld ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust; 622c86e55dSMatthew Auld 632c86e55dSMatthew Auld if (ggtt->mappable_end) { 642c86e55dSMatthew Auld if (!io_mapping_init_wc(&ggtt->iomap, 652c86e55dSMatthew Auld ggtt->gmadr.start, 662c86e55dSMatthew Auld ggtt->mappable_end)) { 672c86e55dSMatthew Auld ggtt->vm.cleanup(&ggtt->vm); 682c86e55dSMatthew Auld return -EIO; 692c86e55dSMatthew Auld } 702c86e55dSMatthew Auld 712c86e55dSMatthew Auld ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, 722c86e55dSMatthew Auld ggtt->mappable_end); 732c86e55dSMatthew Auld } 742c86e55dSMatthew Auld 75f899f786SChris Wilson intel_ggtt_init_fences(ggtt); 762c86e55dSMatthew Auld 772c86e55dSMatthew Auld return 0; 782c86e55dSMatthew Auld } 792c86e55dSMatthew Auld 802c86e55dSMatthew Auld /** 812c86e55dSMatthew Auld * i915_ggtt_init_hw - Initialize GGTT hardware 822c86e55dSMatthew Auld * @i915: i915 device 832c86e55dSMatthew Auld */ 842c86e55dSMatthew Auld int i915_ggtt_init_hw(struct drm_i915_private *i915) 852c86e55dSMatthew Auld { 862c86e55dSMatthew Auld int ret; 872c86e55dSMatthew Auld 882c86e55dSMatthew Auld /* 892c86e55dSMatthew Auld * Note that we use page colouring to enforce a guard page at the 902c86e55dSMatthew Auld * end of the address space. This is required as the CS may prefetch 912c86e55dSMatthew Auld * beyond the end of the batch buffer, across the page boundary, 922c86e55dSMatthew Auld * and beyond the end of the GTT if we do not provide a guard. 932c86e55dSMatthew Auld */ 94848915c3SMichał Winiarski ret = ggtt_init_hw(to_gt(i915)->ggtt); 952c86e55dSMatthew Auld if (ret) 962c86e55dSMatthew Auld return ret; 972c86e55dSMatthew Auld 982c86e55dSMatthew Auld return 0; 992c86e55dSMatthew Auld } 1002c86e55dSMatthew Auld 1018d2f683fSImre Deak /** 1028d2f683fSImre Deak * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM 1038d2f683fSImre Deak * @vm: The VM to suspend the mappings for 1048d2f683fSImre Deak * 1058d2f683fSImre Deak * Suspend the memory mappings for all objects mapped to HW via the GGTT or a 1068d2f683fSImre Deak * DPT page table. 1078d2f683fSImre Deak */ 1088d2f683fSImre Deak void i915_ggtt_suspend_vm(struct i915_address_space *vm) 1092c86e55dSMatthew Auld { 110bffa18ddSChris Wilson struct i915_vma *vma, *vn; 111e1a7ab4fSThomas Hellström int save_skip_rewrite; 112e3793468SChris Wilson 1138d2f683fSImre Deak drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); 1148d2f683fSImre Deak 1150f341974SMaarten Lankhorst retry: 1160f341974SMaarten Lankhorst i915_gem_drain_freed_objects(vm->i915); 1170f341974SMaarten Lankhorst 1188d2f683fSImre Deak mutex_lock(&vm->mutex); 119bffa18ddSChris Wilson 120e1a7ab4fSThomas Hellström /* 121e1a7ab4fSThomas Hellström * Skip rewriting PTE on VMA unbind. 122e1a7ab4fSThomas Hellström * FIXME: Use an argument to i915_vma_unbind() instead? 123e1a7ab4fSThomas Hellström */ 124e1a7ab4fSThomas Hellström save_skip_rewrite = vm->skip_pte_rewrite; 125e1a7ab4fSThomas Hellström vm->skip_pte_rewrite = true; 126bffa18ddSChris Wilson 1278d2f683fSImre Deak list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { 1280f341974SMaarten Lankhorst struct drm_i915_gem_object *obj = vma->obj; 129e3793468SChris Wilson 1300f341974SMaarten Lankhorst GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1310f341974SMaarten Lankhorst 1320f341974SMaarten Lankhorst if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 133bffa18ddSChris Wilson continue; 134bffa18ddSChris Wilson 1350f341974SMaarten Lankhorst /* unlikely to race when GPU is idle, so no worry about slowpath.. */ 1360f341974SMaarten Lankhorst if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) { 1370f341974SMaarten Lankhorst /* 1380f341974SMaarten Lankhorst * No dead objects should appear here, GPU should be 1390f341974SMaarten Lankhorst * completely idle, and userspace suspended 1400f341974SMaarten Lankhorst */ 1410f341974SMaarten Lankhorst i915_gem_object_get(obj); 1420f341974SMaarten Lankhorst 1430f341974SMaarten Lankhorst mutex_unlock(&vm->mutex); 1440f341974SMaarten Lankhorst 1450f341974SMaarten Lankhorst i915_gem_object_lock(obj, NULL); 146e1a7ab4fSThomas Hellström GEM_WARN_ON(i915_vma_unbind(vma)); 1470f341974SMaarten Lankhorst i915_gem_object_unlock(obj); 1480f341974SMaarten Lankhorst i915_gem_object_put(obj); 149e1a7ab4fSThomas Hellström 150e1a7ab4fSThomas Hellström vm->skip_pte_rewrite = save_skip_rewrite; 1510f341974SMaarten Lankhorst goto retry; 1520f341974SMaarten Lankhorst } 1530f341974SMaarten Lankhorst 154bffa18ddSChris Wilson if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) { 1550f341974SMaarten Lankhorst i915_vma_wait_for_bind(vma); 1560f341974SMaarten Lankhorst 1572f6b90daSThomas Hellström __i915_vma_evict(vma, false); 158bffa18ddSChris Wilson drm_mm_remove_node(&vma->node); 159bffa18ddSChris Wilson } 1600f341974SMaarten Lankhorst 1610f341974SMaarten Lankhorst i915_gem_object_unlock(obj); 162bffa18ddSChris Wilson } 163bffa18ddSChris Wilson 1648d2f683fSImre Deak vm->clear_range(vm, 0, vm->total); 165bffa18ddSChris Wilson 166e1a7ab4fSThomas Hellström vm->skip_pte_rewrite = save_skip_rewrite; 1678d2f683fSImre Deak 1688d2f683fSImre Deak mutex_unlock(&vm->mutex); 1698d2f683fSImre Deak } 1708d2f683fSImre Deak 1718d2f683fSImre Deak void i915_ggtt_suspend(struct i915_ggtt *ggtt) 1728d2f683fSImre Deak { 1730f857158SAravind Iddamsetty struct intel_gt *gt; 1740f857158SAravind Iddamsetty 1758d2f683fSImre Deak i915_ggtt_suspend_vm(&ggtt->vm); 1768d2f683fSImre Deak ggtt->invalidate(ggtt); 1772c86e55dSMatthew Auld 1780f857158SAravind Iddamsetty list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 1790f857158SAravind Iddamsetty intel_gt_check_and_clear_faults(gt); 1802c86e55dSMatthew Auld } 1812c86e55dSMatthew Auld 1822c86e55dSMatthew Auld void gen6_ggtt_invalidate(struct i915_ggtt *ggtt) 1832c86e55dSMatthew Auld { 1842c86e55dSMatthew Auld struct intel_uncore *uncore = ggtt->vm.gt->uncore; 1852c86e55dSMatthew Auld 1862c86e55dSMatthew Auld spin_lock_irq(&uncore->lock); 1872c86e55dSMatthew Auld intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 1882c86e55dSMatthew Auld intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6); 1892c86e55dSMatthew Auld spin_unlock_irq(&uncore->lock); 1902c86e55dSMatthew Auld } 1912c86e55dSMatthew Auld 1929ce07d94SLucas De Marchi static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt) 1932c86e55dSMatthew Auld { 1942c86e55dSMatthew Auld struct intel_uncore *uncore = ggtt->vm.gt->uncore; 1952c86e55dSMatthew Auld 1962c86e55dSMatthew Auld /* 1972c86e55dSMatthew Auld * Note that as an uncached mmio write, this will flush the 1982c86e55dSMatthew Auld * WCB of the writes into the GGTT before it triggers the invalidate. 1992c86e55dSMatthew Auld */ 2002c86e55dSMatthew Auld intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2012c86e55dSMatthew Auld } 2022c86e55dSMatthew Auld 2032c86e55dSMatthew Auld static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) 2042c86e55dSMatthew Auld { 2052c86e55dSMatthew Auld struct drm_i915_private *i915 = ggtt->vm.i915; 2062c86e55dSMatthew Auld 2072c86e55dSMatthew Auld gen8_ggtt_invalidate(ggtt); 2082c86e55dSMatthew Auld 2090f857158SAravind Iddamsetty if (GRAPHICS_VER(i915) >= 12) { 2100f857158SAravind Iddamsetty struct intel_gt *gt; 2110f857158SAravind Iddamsetty 2120f857158SAravind Iddamsetty list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 2130f857158SAravind Iddamsetty intel_uncore_write_fw(gt->uncore, 2140f857158SAravind Iddamsetty GEN12_GUC_TLB_INV_CR, 2152c86e55dSMatthew Auld GEN12_GUC_TLB_INV_CR_INVALIDATE); 2160f857158SAravind Iddamsetty } else { 2170f857158SAravind Iddamsetty intel_uncore_write_fw(ggtt->vm.gt->uncore, 2180f857158SAravind Iddamsetty GEN8_GTCR, GEN8_GTCR_INVALIDATE); 2190f857158SAravind Iddamsetty } 2202c86e55dSMatthew Auld } 2212c86e55dSMatthew Auld 22233e7a975SVille Syrjälä u64 gen8_ggtt_pte_encode(dma_addr_t addr, 22369edc390SDaniele Ceraolo Spurio enum i915_cache_level level, 22469edc390SDaniele Ceraolo Spurio u32 flags) 22569edc390SDaniele Ceraolo Spurio { 2265f978167SMichael Cheng gen8_pte_t pte = addr | GEN8_PAGE_PRESENT; 227e762bdf5SMatthew Auld 228e762bdf5SMatthew Auld if (flags & PTE_LM) 229e762bdf5SMatthew Auld pte |= GEN12_GGTT_PTE_LM; 230e762bdf5SMatthew Auld 231e762bdf5SMatthew Auld return pte; 23269edc390SDaniele Ceraolo Spurio } 23369edc390SDaniele Ceraolo Spurio 2349ce07d94SLucas De Marchi static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 2359ce07d94SLucas De Marchi { 2369ce07d94SLucas De Marchi writeq(pte, addr); 2379ce07d94SLucas De Marchi } 2389ce07d94SLucas De Marchi 2399ce07d94SLucas De Marchi static void gen8_ggtt_insert_page(struct i915_address_space *vm, 2409ce07d94SLucas De Marchi dma_addr_t addr, 2419ce07d94SLucas De Marchi u64 offset, 2429ce07d94SLucas De Marchi enum i915_cache_level level, 2439ce07d94SLucas De Marchi u32 flags) 2449ce07d94SLucas De Marchi { 2459ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2469ce07d94SLucas De Marchi gen8_pte_t __iomem *pte = 2479ce07d94SLucas De Marchi (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 2489ce07d94SLucas De Marchi 2499ce07d94SLucas De Marchi gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, flags)); 2509ce07d94SLucas De Marchi 2519ce07d94SLucas De Marchi ggtt->invalidate(ggtt); 2529ce07d94SLucas De Marchi } 2539ce07d94SLucas De Marchi 2549ce07d94SLucas De Marchi static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 2559ce07d94SLucas De Marchi struct i915_vma_resource *vma_res, 2569ce07d94SLucas De Marchi enum i915_cache_level level, 2579ce07d94SLucas De Marchi u32 flags) 2589ce07d94SLucas De Marchi { 2599ce07d94SLucas De Marchi const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, flags); 2609ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 2619ce07d94SLucas De Marchi gen8_pte_t __iomem *gte; 2629ce07d94SLucas De Marchi gen8_pte_t __iomem *end; 2639ce07d94SLucas De Marchi struct sgt_iter iter; 2649ce07d94SLucas De Marchi dma_addr_t addr; 2659ce07d94SLucas De Marchi 2669ce07d94SLucas De Marchi /* 2679ce07d94SLucas De Marchi * Note that we ignore PTE_READ_ONLY here. The caller must be careful 2689ce07d94SLucas De Marchi * not to allow the user to override access to a read only page. 2699ce07d94SLucas De Marchi */ 2709ce07d94SLucas De Marchi 2719ce07d94SLucas De Marchi gte = (gen8_pte_t __iomem *)ggtt->gsm; 27261102251SChris Wilson gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 27361102251SChris Wilson end = gte + vma_res->guard / I915_GTT_PAGE_SIZE; 27461102251SChris Wilson while (gte < end) 27561102251SChris Wilson gen8_set_pte(gte++, vm->scratch[0]->encode); 27661102251SChris Wilson end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 2779ce07d94SLucas De Marchi 2789ce07d94SLucas De Marchi for_each_sgt_daddr(addr, iter, vma_res->bi.pages) 2799ce07d94SLucas De Marchi gen8_set_pte(gte++, pte_encode | addr); 2809ce07d94SLucas De Marchi GEM_BUG_ON(gte > end); 2819ce07d94SLucas De Marchi 2829ce07d94SLucas De Marchi /* Fill the allocated but "unused" space beyond the end of the buffer */ 2839ce07d94SLucas De Marchi while (gte < end) 2849ce07d94SLucas De Marchi gen8_set_pte(gte++, vm->scratch[0]->encode); 2859ce07d94SLucas De Marchi 2869ce07d94SLucas De Marchi /* 2879ce07d94SLucas De Marchi * We want to flush the TLBs only after we're certain all the PTE 2889ce07d94SLucas De Marchi * updates have finished. 2899ce07d94SLucas De Marchi */ 2909ce07d94SLucas De Marchi ggtt->invalidate(ggtt); 2919ce07d94SLucas De Marchi } 2929ce07d94SLucas De Marchi 2939ce07d94SLucas De Marchi static void gen6_ggtt_insert_page(struct i915_address_space *vm, 2949ce07d94SLucas De Marchi dma_addr_t addr, 2959ce07d94SLucas De Marchi u64 offset, 2969ce07d94SLucas De Marchi enum i915_cache_level level, 2979ce07d94SLucas De Marchi u32 flags) 2989ce07d94SLucas De Marchi { 2999ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 3009ce07d94SLucas De Marchi gen6_pte_t __iomem *pte = 3019ce07d94SLucas De Marchi (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 3029ce07d94SLucas De Marchi 3039ce07d94SLucas De Marchi iowrite32(vm->pte_encode(addr, level, flags), pte); 3049ce07d94SLucas De Marchi 3059ce07d94SLucas De Marchi ggtt->invalidate(ggtt); 3069ce07d94SLucas De Marchi } 3079ce07d94SLucas De Marchi 3089ce07d94SLucas De Marchi /* 3099ce07d94SLucas De Marchi * Binds an object into the global gtt with the specified cache level. 3109ce07d94SLucas De Marchi * The object will be accessible to the GPU via commands whose operands 3119ce07d94SLucas De Marchi * reference offsets within the global GTT as well as accessible by the GPU 3129ce07d94SLucas De Marchi * through the GMADR mapped BAR (i915->mm.gtt->gtt). 3139ce07d94SLucas De Marchi */ 3149ce07d94SLucas De Marchi static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 3159ce07d94SLucas De Marchi struct i915_vma_resource *vma_res, 3169ce07d94SLucas De Marchi enum i915_cache_level level, 3179ce07d94SLucas De Marchi u32 flags) 3189ce07d94SLucas De Marchi { 3199ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 3209ce07d94SLucas De Marchi gen6_pte_t __iomem *gte; 3219ce07d94SLucas De Marchi gen6_pte_t __iomem *end; 3229ce07d94SLucas De Marchi struct sgt_iter iter; 3239ce07d94SLucas De Marchi dma_addr_t addr; 3249ce07d94SLucas De Marchi 3259ce07d94SLucas De Marchi gte = (gen6_pte_t __iomem *)ggtt->gsm; 32661102251SChris Wilson gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 3279ce07d94SLucas De Marchi 32861102251SChris Wilson end = gte + vma_res->guard / I915_GTT_PAGE_SIZE; 32961102251SChris Wilson while (gte < end) 33061102251SChris Wilson iowrite32(vm->scratch[0]->encode, gte++); 33161102251SChris Wilson end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 3329ce07d94SLucas De Marchi for_each_sgt_daddr(addr, iter, vma_res->bi.pages) 3339ce07d94SLucas De Marchi iowrite32(vm->pte_encode(addr, level, flags), gte++); 3349ce07d94SLucas De Marchi GEM_BUG_ON(gte > end); 3359ce07d94SLucas De Marchi 3369ce07d94SLucas De Marchi /* Fill the allocated but "unused" space beyond the end of the buffer */ 3379ce07d94SLucas De Marchi while (gte < end) 3389ce07d94SLucas De Marchi iowrite32(vm->scratch[0]->encode, gte++); 3399ce07d94SLucas De Marchi 3409ce07d94SLucas De Marchi /* 3419ce07d94SLucas De Marchi * We want to flush the TLBs only after we're certain all the PTE 3429ce07d94SLucas De Marchi * updates have finished. 3439ce07d94SLucas De Marchi */ 3449ce07d94SLucas De Marchi ggtt->invalidate(ggtt); 3459ce07d94SLucas De Marchi } 3469ce07d94SLucas De Marchi 3479ce07d94SLucas De Marchi static void nop_clear_range(struct i915_address_space *vm, 3489ce07d94SLucas De Marchi u64 start, u64 length) 3499ce07d94SLucas De Marchi { 3509ce07d94SLucas De Marchi } 3519ce07d94SLucas De Marchi 3529ce07d94SLucas De Marchi static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) 3539ce07d94SLucas De Marchi { 3549ce07d94SLucas De Marchi /* 3559ce07d94SLucas De Marchi * Make sure the internal GAM fifo has been cleared of all GTT 3569ce07d94SLucas De Marchi * writes before exiting stop_machine(). This guarantees that 3579ce07d94SLucas De Marchi * any aperture accesses waiting to start in another process 3589ce07d94SLucas De Marchi * cannot back up behind the GTT writes causing a hang. 3599ce07d94SLucas De Marchi * The register can be any arbitrary GAM register. 3609ce07d94SLucas De Marchi */ 3619ce07d94SLucas De Marchi intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6); 3629ce07d94SLucas De Marchi } 3639ce07d94SLucas De Marchi 3649ce07d94SLucas De Marchi struct insert_page { 3659ce07d94SLucas De Marchi struct i915_address_space *vm; 3669ce07d94SLucas De Marchi dma_addr_t addr; 3679ce07d94SLucas De Marchi u64 offset; 3689ce07d94SLucas De Marchi enum i915_cache_level level; 3699ce07d94SLucas De Marchi }; 3709ce07d94SLucas De Marchi 3719ce07d94SLucas De Marchi static int bxt_vtd_ggtt_insert_page__cb(void *_arg) 3729ce07d94SLucas De Marchi { 3739ce07d94SLucas De Marchi struct insert_page *arg = _arg; 3749ce07d94SLucas De Marchi 3759ce07d94SLucas De Marchi gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); 3769ce07d94SLucas De Marchi bxt_vtd_ggtt_wa(arg->vm); 3779ce07d94SLucas De Marchi 3789ce07d94SLucas De Marchi return 0; 3799ce07d94SLucas De Marchi } 3809ce07d94SLucas De Marchi 3819ce07d94SLucas De Marchi static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, 3829ce07d94SLucas De Marchi dma_addr_t addr, 3839ce07d94SLucas De Marchi u64 offset, 3849ce07d94SLucas De Marchi enum i915_cache_level level, 3859ce07d94SLucas De Marchi u32 unused) 3869ce07d94SLucas De Marchi { 3879ce07d94SLucas De Marchi struct insert_page arg = { vm, addr, offset, level }; 3889ce07d94SLucas De Marchi 3899ce07d94SLucas De Marchi stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); 3909ce07d94SLucas De Marchi } 3919ce07d94SLucas De Marchi 3929ce07d94SLucas De Marchi struct insert_entries { 3939ce07d94SLucas De Marchi struct i915_address_space *vm; 3949ce07d94SLucas De Marchi struct i915_vma_resource *vma_res; 3959ce07d94SLucas De Marchi enum i915_cache_level level; 3969ce07d94SLucas De Marchi u32 flags; 3979ce07d94SLucas De Marchi }; 3989ce07d94SLucas De Marchi 3999ce07d94SLucas De Marchi static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) 4009ce07d94SLucas De Marchi { 4019ce07d94SLucas De Marchi struct insert_entries *arg = _arg; 4029ce07d94SLucas De Marchi 4039ce07d94SLucas De Marchi gen8_ggtt_insert_entries(arg->vm, arg->vma_res, arg->level, arg->flags); 4049ce07d94SLucas De Marchi bxt_vtd_ggtt_wa(arg->vm); 4059ce07d94SLucas De Marchi 4069ce07d94SLucas De Marchi return 0; 4079ce07d94SLucas De Marchi } 4089ce07d94SLucas De Marchi 4099ce07d94SLucas De Marchi static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, 4109ce07d94SLucas De Marchi struct i915_vma_resource *vma_res, 4119ce07d94SLucas De Marchi enum i915_cache_level level, 4129ce07d94SLucas De Marchi u32 flags) 4139ce07d94SLucas De Marchi { 4149ce07d94SLucas De Marchi struct insert_entries arg = { vm, vma_res, level, flags }; 4159ce07d94SLucas De Marchi 4169ce07d94SLucas De Marchi stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); 4179ce07d94SLucas De Marchi } 4189ce07d94SLucas De Marchi 4199ce07d94SLucas De Marchi static void gen6_ggtt_clear_range(struct i915_address_space *vm, 4209ce07d94SLucas De Marchi u64 start, u64 length) 4219ce07d94SLucas De Marchi { 4229ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 4239ce07d94SLucas De Marchi unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 4249ce07d94SLucas De Marchi unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 4259ce07d94SLucas De Marchi gen6_pte_t scratch_pte, __iomem *gtt_base = 4269ce07d94SLucas De Marchi (gen6_pte_t __iomem *)ggtt->gsm + first_entry; 4279ce07d94SLucas De Marchi const int max_entries = ggtt_total_entries(ggtt) - first_entry; 4289ce07d94SLucas De Marchi int i; 4299ce07d94SLucas De Marchi 4309ce07d94SLucas De Marchi if (WARN(num_entries > max_entries, 4319ce07d94SLucas De Marchi "First entry = %d; Num entries = %d (max=%d)\n", 4329ce07d94SLucas De Marchi first_entry, num_entries, max_entries)) 4339ce07d94SLucas De Marchi num_entries = max_entries; 4349ce07d94SLucas De Marchi 4359ce07d94SLucas De Marchi scratch_pte = vm->scratch[0]->encode; 4369ce07d94SLucas De Marchi for (i = 0; i < num_entries; i++) 4379ce07d94SLucas De Marchi iowrite32(scratch_pte, >t_base[i]); 4389ce07d94SLucas De Marchi } 4399ce07d94SLucas De Marchi 4407a5c9223SCasey Bowman void intel_ggtt_bind_vma(struct i915_address_space *vm, 441cd0452aaSChris Wilson struct i915_vm_pt_stash *stash, 44239a2bd34SThomas Hellström struct i915_vma_resource *vma_res, 4432c86e55dSMatthew Auld enum i915_cache_level cache_level, 4442c86e55dSMatthew Auld u32 flags) 4452c86e55dSMatthew Auld { 4462c86e55dSMatthew Auld u32 pte_flags; 4472c86e55dSMatthew Auld 44839a2bd34SThomas Hellström if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK)) 449cd0452aaSChris Wilson return; 450bf0840cdSChris Wilson 45139a2bd34SThomas Hellström vma_res->bound_flags |= flags; 45239a2bd34SThomas Hellström 4532c86e55dSMatthew Auld /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 4542c86e55dSMatthew Auld pte_flags = 0; 45539a2bd34SThomas Hellström if (vma_res->bi.readonly) 4562c86e55dSMatthew Auld pte_flags |= PTE_READ_ONLY; 45739a2bd34SThomas Hellström if (vma_res->bi.lmem) 458e762bdf5SMatthew Auld pte_flags |= PTE_LM; 4592c86e55dSMatthew Auld 46039a2bd34SThomas Hellström vm->insert_entries(vm, vma_res, cache_level, pte_flags); 46139a2bd34SThomas Hellström vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 4622c86e55dSMatthew Auld } 4632c86e55dSMatthew Auld 4647a5c9223SCasey Bowman void intel_ggtt_unbind_vma(struct i915_address_space *vm, 46539a2bd34SThomas Hellström struct i915_vma_resource *vma_res) 4662c86e55dSMatthew Auld { 46739a2bd34SThomas Hellström vm->clear_range(vm, vma_res->start, vma_res->vma_size); 4682c86e55dSMatthew Auld } 4692c86e55dSMatthew Auld 4702c86e55dSMatthew Auld static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) 4712c86e55dSMatthew Auld { 4722c86e55dSMatthew Auld u64 size; 4732c86e55dSMatthew Auld int ret; 4742c86e55dSMatthew Auld 47534bbfde6SDaniele Ceraolo Spurio if (!intel_uc_uses_guc(&ggtt->vm.gt->uc)) 4762c86e55dSMatthew Auld return 0; 4772c86e55dSMatthew Auld 4782c86e55dSMatthew Auld GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP); 4792c86e55dSMatthew Auld size = ggtt->vm.total - GUC_GGTT_TOP; 4802c86e55dSMatthew Auld 4817e00897bSMaarten Lankhorst ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw, size, 4822c86e55dSMatthew Auld GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE, 4832c86e55dSMatthew Auld PIN_NOEVICT); 4842c86e55dSMatthew Auld if (ret) 48552ce7074SWambui Karuga drm_dbg(&ggtt->vm.i915->drm, 48652ce7074SWambui Karuga "Failed to reserve top of GGTT for GuC\n"); 4872c86e55dSMatthew Auld 4882c86e55dSMatthew Auld return ret; 4892c86e55dSMatthew Auld } 4902c86e55dSMatthew Auld 4912c86e55dSMatthew Auld static void ggtt_release_guc_top(struct i915_ggtt *ggtt) 4922c86e55dSMatthew Auld { 4932c86e55dSMatthew Auld if (drm_mm_node_allocated(&ggtt->uc_fw)) 4942c86e55dSMatthew Auld drm_mm_remove_node(&ggtt->uc_fw); 4952c86e55dSMatthew Auld } 4962c86e55dSMatthew Auld 4972c86e55dSMatthew Auld static void cleanup_init_ggtt(struct i915_ggtt *ggtt) 4982c86e55dSMatthew Auld { 4992c86e55dSMatthew Auld ggtt_release_guc_top(ggtt); 5002c86e55dSMatthew Auld if (drm_mm_node_allocated(&ggtt->error_capture)) 5012c86e55dSMatthew Auld drm_mm_remove_node(&ggtt->error_capture); 502742379c0SChris Wilson mutex_destroy(&ggtt->error_mutex); 5032c86e55dSMatthew Auld } 5042c86e55dSMatthew Auld 5052c86e55dSMatthew Auld static int init_ggtt(struct i915_ggtt *ggtt) 5062c86e55dSMatthew Auld { 5072c86e55dSMatthew Auld /* 5082c86e55dSMatthew Auld * Let GEM Manage all of the aperture. 5092c86e55dSMatthew Auld * 5102c86e55dSMatthew Auld * However, leave one page at the end still bound to the scratch page. 5112c86e55dSMatthew Auld * There are a number of places where the hardware apparently prefetches 5122c86e55dSMatthew Auld * past the end of the object, and we've seen multiple hangs with the 5132c86e55dSMatthew Auld * GPU head pointer stuck in a batchbuffer bound at the last page of the 5142c86e55dSMatthew Auld * aperture. One page should be enough to keep any prefetching inside 5152c86e55dSMatthew Auld * of the aperture. 5162c86e55dSMatthew Auld */ 5172c86e55dSMatthew Auld unsigned long hole_start, hole_end; 5182c86e55dSMatthew Auld struct drm_mm_node *entry; 5192c86e55dSMatthew Auld int ret; 5202c86e55dSMatthew Auld 5212c86e55dSMatthew Auld /* 5222c86e55dSMatthew Auld * GuC requires all resources that we're sharing with it to be placed in 5232c86e55dSMatthew Auld * non-WOPCM memory. If GuC is not present or not in use we still need a 5242c86e55dSMatthew Auld * small bias as ring wraparound at offset 0 sometimes hangs. No idea 5252c86e55dSMatthew Auld * why. 5262c86e55dSMatthew Auld */ 5272c86e55dSMatthew Auld ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE, 528ee71434eSAravind Iddamsetty intel_wopcm_guc_size(&ggtt->vm.gt->wopcm)); 5292c86e55dSMatthew Auld 5302c86e55dSMatthew Auld ret = intel_vgt_balloon(ggtt); 5312c86e55dSMatthew Auld if (ret) 5322c86e55dSMatthew Auld return ret; 5332c86e55dSMatthew Auld 534742379c0SChris Wilson mutex_init(&ggtt->error_mutex); 5352c86e55dSMatthew Auld if (ggtt->mappable_end) { 536489140b5SChris Wilson /* 537489140b5SChris Wilson * Reserve a mappable slot for our lockless error capture. 538489140b5SChris Wilson * 539489140b5SChris Wilson * We strongly prefer taking address 0x0 in order to protect 540489140b5SChris Wilson * other critical buffers against accidental overwrites, 541489140b5SChris Wilson * as writing to address 0 is a very common mistake. 542489140b5SChris Wilson * 543489140b5SChris Wilson * Since 0 may already be in use by the system (e.g. the BIOS 544489140b5SChris Wilson * framebuffer), we let the reservation fail quietly and hope 545489140b5SChris Wilson * 0 remains reserved always. 546489140b5SChris Wilson * 547489140b5SChris Wilson * If we fail to reserve 0, and then fail to find any space 548489140b5SChris Wilson * for an error-capture, remain silent. We can afford not 549489140b5SChris Wilson * to reserve an error_capture node as we have fallback 550489140b5SChris Wilson * paths, and we trust that 0 will remain reserved. However, 551489140b5SChris Wilson * the only likely reason for failure to insert is a driver 552489140b5SChris Wilson * bug, which we expect to cause other failures... 553489140b5SChris Wilson */ 554489140b5SChris Wilson ggtt->error_capture.size = I915_GTT_PAGE_SIZE; 555489140b5SChris Wilson ggtt->error_capture.color = I915_COLOR_UNEVICTABLE; 556489140b5SChris Wilson if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture)) 557489140b5SChris Wilson drm_mm_insert_node_in_range(&ggtt->vm.mm, 5582c86e55dSMatthew Auld &ggtt->error_capture, 559489140b5SChris Wilson ggtt->error_capture.size, 0, 560489140b5SChris Wilson ggtt->error_capture.color, 5612c86e55dSMatthew Auld 0, ggtt->mappable_end, 5622c86e55dSMatthew Auld DRM_MM_INSERT_LOW); 5632c86e55dSMatthew Auld } 564489140b5SChris Wilson if (drm_mm_node_allocated(&ggtt->error_capture)) 565489140b5SChris Wilson drm_dbg(&ggtt->vm.i915->drm, 566489140b5SChris Wilson "Reserved GGTT:[%llx, %llx] for use by error capture\n", 567489140b5SChris Wilson ggtt->error_capture.start, 568489140b5SChris Wilson ggtt->error_capture.start + ggtt->error_capture.size); 5692c86e55dSMatthew Auld 5702c86e55dSMatthew Auld /* 5712c86e55dSMatthew Auld * The upper portion of the GuC address space has a sizeable hole 5722c86e55dSMatthew Auld * (several MB) that is inaccessible by GuC. Reserve this range within 5732c86e55dSMatthew Auld * GGTT as it can comfortably hold GuC/HuC firmware images. 5742c86e55dSMatthew Auld */ 5752c86e55dSMatthew Auld ret = ggtt_reserve_guc_top(ggtt); 5762c86e55dSMatthew Auld if (ret) 5772c86e55dSMatthew Auld goto err; 5782c86e55dSMatthew Auld 5792c86e55dSMatthew Auld /* Clear any non-preallocated blocks */ 5802c86e55dSMatthew Auld drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) { 581489140b5SChris Wilson drm_dbg(&ggtt->vm.i915->drm, 58252ce7074SWambui Karuga "clearing unused GTT space: [%lx, %lx]\n", 5832c86e55dSMatthew Auld hole_start, hole_end); 5842c86e55dSMatthew Auld ggtt->vm.clear_range(&ggtt->vm, hole_start, 5852c86e55dSMatthew Auld hole_end - hole_start); 5862c86e55dSMatthew Auld } 5872c86e55dSMatthew Auld 5882c86e55dSMatthew Auld /* And finally clear the reserved guard page */ 5892c86e55dSMatthew Auld ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); 5902c86e55dSMatthew Auld 5912c86e55dSMatthew Auld return 0; 5922c86e55dSMatthew Auld 5932c86e55dSMatthew Auld err: 5942c86e55dSMatthew Auld cleanup_init_ggtt(ggtt); 5952c86e55dSMatthew Auld return ret; 5962c86e55dSMatthew Auld } 5972c86e55dSMatthew Auld 598cd0452aaSChris Wilson static void aliasing_gtt_bind_vma(struct i915_address_space *vm, 599cd0452aaSChris Wilson struct i915_vm_pt_stash *stash, 60039a2bd34SThomas Hellström struct i915_vma_resource *vma_res, 6012c86e55dSMatthew Auld enum i915_cache_level cache_level, 6022c86e55dSMatthew Auld u32 flags) 6032c86e55dSMatthew Auld { 6042c86e55dSMatthew Auld u32 pte_flags; 6052c86e55dSMatthew Auld 6062c86e55dSMatthew Auld /* Currently applicable only to VLV */ 6072c86e55dSMatthew Auld pte_flags = 0; 60839a2bd34SThomas Hellström if (vma_res->bi.readonly) 6092c86e55dSMatthew Auld pte_flags |= PTE_READ_ONLY; 6102c86e55dSMatthew Auld 611cd0452aaSChris Wilson if (flags & I915_VMA_LOCAL_BIND) 612cd0452aaSChris Wilson ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, 61339a2bd34SThomas Hellström stash, vma_res, cache_level, flags); 6142c86e55dSMatthew Auld 615c0e60347SChris Wilson if (flags & I915_VMA_GLOBAL_BIND) 61639a2bd34SThomas Hellström vm->insert_entries(vm, vma_res, cache_level, pte_flags); 61739a2bd34SThomas Hellström 61839a2bd34SThomas Hellström vma_res->bound_flags |= flags; 6192c86e55dSMatthew Auld } 6202c86e55dSMatthew Auld 62112b07256SChris Wilson static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, 62239a2bd34SThomas Hellström struct i915_vma_resource *vma_res) 6232c86e55dSMatthew Auld { 62439a2bd34SThomas Hellström if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND) 62539a2bd34SThomas Hellström vm->clear_range(vm, vma_res->start, vma_res->vma_size); 6262c86e55dSMatthew Auld 62739a2bd34SThomas Hellström if (vma_res->bound_flags & I915_VMA_LOCAL_BIND) 62839a2bd34SThomas Hellström ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res); 6292c86e55dSMatthew Auld } 6302c86e55dSMatthew Auld 6312c86e55dSMatthew Auld static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) 6322c86e55dSMatthew Auld { 633cd0452aaSChris Wilson struct i915_vm_pt_stash stash = {}; 6342c86e55dSMatthew Auld struct i915_ppgtt *ppgtt; 6352c86e55dSMatthew Auld int err; 6362c86e55dSMatthew Auld 637a259cc14SThomas Hellström ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0); 6382c86e55dSMatthew Auld if (IS_ERR(ppgtt)) 6392c86e55dSMatthew Auld return PTR_ERR(ppgtt); 6402c86e55dSMatthew Auld 6412c86e55dSMatthew Auld if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) { 6422c86e55dSMatthew Auld err = -ENODEV; 6432c86e55dSMatthew Auld goto err_ppgtt; 6442c86e55dSMatthew Auld } 6452c86e55dSMatthew Auld 646cd0452aaSChris Wilson err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total); 647cd0452aaSChris Wilson if (err) 648cd0452aaSChris Wilson goto err_ppgtt; 649cd0452aaSChris Wilson 65026ad4f8bSMaarten Lankhorst i915_gem_object_lock(ppgtt->vm.scratch[0], NULL); 651529b9ec8SMatthew Auld err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 65226ad4f8bSMaarten Lankhorst i915_gem_object_unlock(ppgtt->vm.scratch[0]); 65389351925SChris Wilson if (err) 65489351925SChris Wilson goto err_stash; 65589351925SChris Wilson 6562c86e55dSMatthew Auld /* 6572c86e55dSMatthew Auld * Note we only pre-allocate as far as the end of the global 6582c86e55dSMatthew Auld * GTT. On 48b / 4-level page-tables, the difference is very, 6592c86e55dSMatthew Auld * very significant! We have to preallocate as GVT/vgpu does 6602c86e55dSMatthew Auld * not like the page directory disappearing. 6612c86e55dSMatthew Auld */ 662cd0452aaSChris Wilson ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total); 6632c86e55dSMatthew Auld 6642c86e55dSMatthew Auld ggtt->alias = ppgtt; 6652c86e55dSMatthew Auld ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags; 6662c86e55dSMatthew Auld 6677a5c9223SCasey Bowman GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma); 6682c86e55dSMatthew Auld ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma; 6692c86e55dSMatthew Auld 6707a5c9223SCasey Bowman GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma); 6712c86e55dSMatthew Auld ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; 6722c86e55dSMatthew Auld 673cd0452aaSChris Wilson i915_vm_free_pt_stash(&ppgtt->vm, &stash); 6742c86e55dSMatthew Auld return 0; 6752c86e55dSMatthew Auld 67689351925SChris Wilson err_stash: 67789351925SChris Wilson i915_vm_free_pt_stash(&ppgtt->vm, &stash); 6782c86e55dSMatthew Auld err_ppgtt: 6792c86e55dSMatthew Auld i915_vm_put(&ppgtt->vm); 6802c86e55dSMatthew Auld return err; 6812c86e55dSMatthew Auld } 6822c86e55dSMatthew Auld 6832c86e55dSMatthew Auld static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt) 6842c86e55dSMatthew Auld { 6852c86e55dSMatthew Auld struct i915_ppgtt *ppgtt; 6862c86e55dSMatthew Auld 6872c86e55dSMatthew Auld ppgtt = fetch_and_zero(&ggtt->alias); 6882c86e55dSMatthew Auld if (!ppgtt) 6892c86e55dSMatthew Auld return; 6902c86e55dSMatthew Auld 6912c86e55dSMatthew Auld i915_vm_put(&ppgtt->vm); 6922c86e55dSMatthew Auld 6937a5c9223SCasey Bowman ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 6947a5c9223SCasey Bowman ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 6952c86e55dSMatthew Auld } 6962c86e55dSMatthew Auld 6972c86e55dSMatthew Auld int i915_init_ggtt(struct drm_i915_private *i915) 6982c86e55dSMatthew Auld { 6992c86e55dSMatthew Auld int ret; 7002c86e55dSMatthew Auld 701848915c3SMichał Winiarski ret = init_ggtt(to_gt(i915)->ggtt); 7022c86e55dSMatthew Auld if (ret) 7032c86e55dSMatthew Auld return ret; 7042c86e55dSMatthew Auld 7052c86e55dSMatthew Auld if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { 706848915c3SMichał Winiarski ret = init_aliasing_ppgtt(to_gt(i915)->ggtt); 7072c86e55dSMatthew Auld if (ret) 708848915c3SMichał Winiarski cleanup_init_ggtt(to_gt(i915)->ggtt); 7092c86e55dSMatthew Auld } 7102c86e55dSMatthew Auld 7112c86e55dSMatthew Auld return 0; 7122c86e55dSMatthew Auld } 7132c86e55dSMatthew Auld 7142c86e55dSMatthew Auld static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) 7152c86e55dSMatthew Auld { 7162c86e55dSMatthew Auld struct i915_vma *vma, *vn; 7172c86e55dSMatthew Auld 7182c86e55dSMatthew Auld flush_workqueue(ggtt->vm.i915->wq); 7190f341974SMaarten Lankhorst i915_gem_drain_freed_objects(ggtt->vm.i915); 7202c86e55dSMatthew Auld 7212c86e55dSMatthew Auld mutex_lock(&ggtt->vm.mutex); 7222c86e55dSMatthew Auld 723e1a7ab4fSThomas Hellström ggtt->vm.skip_pte_rewrite = true; 724e1a7ab4fSThomas Hellström 7250f341974SMaarten Lankhorst list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { 7260f341974SMaarten Lankhorst struct drm_i915_gem_object *obj = vma->obj; 7270f341974SMaarten Lankhorst bool trylock; 7280f341974SMaarten Lankhorst 7290f341974SMaarten Lankhorst trylock = i915_gem_object_trylock(obj, NULL); 7300f341974SMaarten Lankhorst WARN_ON(!trylock); 7310f341974SMaarten Lankhorst 7322c86e55dSMatthew Auld WARN_ON(__i915_vma_unbind(vma)); 7330f341974SMaarten Lankhorst if (trylock) 7340f341974SMaarten Lankhorst i915_gem_object_unlock(obj); 7350f341974SMaarten Lankhorst } 7362c86e55dSMatthew Auld 7372c86e55dSMatthew Auld if (drm_mm_node_allocated(&ggtt->error_capture)) 7382c86e55dSMatthew Auld drm_mm_remove_node(&ggtt->error_capture); 739742379c0SChris Wilson mutex_destroy(&ggtt->error_mutex); 7402c86e55dSMatthew Auld 7412c86e55dSMatthew Auld ggtt_release_guc_top(ggtt); 7422c86e55dSMatthew Auld intel_vgt_deballoon(ggtt); 7432c86e55dSMatthew Auld 7442c86e55dSMatthew Auld ggtt->vm.cleanup(&ggtt->vm); 7452c86e55dSMatthew Auld 7462c86e55dSMatthew Auld mutex_unlock(&ggtt->vm.mutex); 7472c86e55dSMatthew Auld i915_address_space_fini(&ggtt->vm); 7482c86e55dSMatthew Auld 7492c86e55dSMatthew Auld arch_phys_wc_del(ggtt->mtrr); 7502c86e55dSMatthew Auld 7512c86e55dSMatthew Auld if (ggtt->iomap.size) 7522c86e55dSMatthew Auld io_mapping_fini(&ggtt->iomap); 7532c86e55dSMatthew Auld } 7542c86e55dSMatthew Auld 7552c86e55dSMatthew Auld /** 7562c86e55dSMatthew Auld * i915_ggtt_driver_release - Clean up GGTT hardware initialization 7572c86e55dSMatthew Auld * @i915: i915 device 7582c86e55dSMatthew Auld */ 7592c86e55dSMatthew Auld void i915_ggtt_driver_release(struct drm_i915_private *i915) 7602c86e55dSMatthew Auld { 761848915c3SMichał Winiarski struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 7622c86e55dSMatthew Auld 7630b6bc81dSChris Wilson fini_aliasing_ppgtt(ggtt); 7642c86e55dSMatthew Auld 7650b6bc81dSChris Wilson intel_ggtt_fini_fences(ggtt); 7660b6bc81dSChris Wilson ggtt_cleanup_hw(ggtt); 7672c86e55dSMatthew Auld } 7682c86e55dSMatthew Auld 7694d8151aeSThomas Hellström /** 7704d8151aeSThomas Hellström * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after 7714d8151aeSThomas Hellström * all free objects have been drained. 7724d8151aeSThomas Hellström * @i915: i915 device 7734d8151aeSThomas Hellström */ 7744d8151aeSThomas Hellström void i915_ggtt_driver_late_release(struct drm_i915_private *i915) 7754d8151aeSThomas Hellström { 776848915c3SMichał Winiarski struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 7774d8151aeSThomas Hellström 7784d8151aeSThomas Hellström GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1); 7794d8151aeSThomas Hellström dma_resv_fini(&ggtt->vm._resv); 7804d8151aeSThomas Hellström } 7814d8151aeSThomas Hellström 7829ce07d94SLucas De Marchi static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 7839ce07d94SLucas De Marchi { 7849ce07d94SLucas De Marchi snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 7859ce07d94SLucas De Marchi snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 7869ce07d94SLucas De Marchi return snb_gmch_ctl << 20; 7879ce07d94SLucas De Marchi } 7889ce07d94SLucas De Marchi 7899ce07d94SLucas De Marchi static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 7909ce07d94SLucas De Marchi { 7919ce07d94SLucas De Marchi bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 7929ce07d94SLucas De Marchi bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 7939ce07d94SLucas De Marchi if (bdw_gmch_ctl) 7949ce07d94SLucas De Marchi bdw_gmch_ctl = 1 << bdw_gmch_ctl; 7959ce07d94SLucas De Marchi 7969ce07d94SLucas De Marchi #ifdef CONFIG_X86_32 7979ce07d94SLucas De Marchi /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */ 7989ce07d94SLucas De Marchi if (bdw_gmch_ctl > 4) 7999ce07d94SLucas De Marchi bdw_gmch_ctl = 4; 8009ce07d94SLucas De Marchi #endif 8019ce07d94SLucas De Marchi 8029ce07d94SLucas De Marchi return bdw_gmch_ctl << 20; 8039ce07d94SLucas De Marchi } 8049ce07d94SLucas De Marchi 8059ce07d94SLucas De Marchi static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 8069ce07d94SLucas De Marchi { 8079ce07d94SLucas De Marchi gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 8089ce07d94SLucas De Marchi gmch_ctrl &= SNB_GMCH_GGMS_MASK; 8099ce07d94SLucas De Marchi 8109ce07d94SLucas De Marchi if (gmch_ctrl) 8119ce07d94SLucas De Marchi return 1 << (20 + gmch_ctrl); 8129ce07d94SLucas De Marchi 8139ce07d94SLucas De Marchi return 0; 8149ce07d94SLucas De Marchi } 8159ce07d94SLucas De Marchi 8169ce07d94SLucas De Marchi static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915) 8179ce07d94SLucas De Marchi { 8189ce07d94SLucas De Marchi /* 8199ce07d94SLucas De Marchi * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset 8209ce07d94SLucas De Marchi * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset 8219ce07d94SLucas De Marchi */ 8229ce07d94SLucas De Marchi GEM_BUG_ON(GRAPHICS_VER(i915) < 6); 8239ce07d94SLucas De Marchi return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M; 8249ce07d94SLucas De Marchi } 8259ce07d94SLucas De Marchi 8269ce07d94SLucas De Marchi static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915) 8279ce07d94SLucas De Marchi { 8289ce07d94SLucas De Marchi return gen6_gttmmadr_size(i915) / 2; 8299ce07d94SLucas De Marchi } 8309ce07d94SLucas De Marchi 8319ce07d94SLucas De Marchi static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 8329ce07d94SLucas De Marchi { 8339ce07d94SLucas De Marchi struct drm_i915_private *i915 = ggtt->vm.i915; 8349ce07d94SLucas De Marchi struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 8359ce07d94SLucas De Marchi phys_addr_t phys_addr; 8369ce07d94SLucas De Marchi u32 pte_flags; 8379ce07d94SLucas De Marchi int ret; 8389ce07d94SLucas De Marchi 8390492a34cSVille Syrjälä GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915)); 8400492a34cSVille Syrjälä phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915); 8419ce07d94SLucas De Marchi 8429ce07d94SLucas De Marchi /* 8439ce07d94SLucas De Marchi * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range 8449ce07d94SLucas De Marchi * will be dropped. For WC mappings in general we have 64 byte burst 8459ce07d94SLucas De Marchi * writes when the WC buffer is flushed, so we can't use it, but have to 8469ce07d94SLucas De Marchi * resort to an uncached mapping. The WC issue is easily caught by the 8479ce07d94SLucas De Marchi * readback check when writing GTT PTE entries. 8489ce07d94SLucas De Marchi */ 8499ce07d94SLucas De Marchi if (IS_GEN9_LP(i915) || GRAPHICS_VER(i915) >= 11) 8509ce07d94SLucas De Marchi ggtt->gsm = ioremap(phys_addr, size); 8519ce07d94SLucas De Marchi else 8529ce07d94SLucas De Marchi ggtt->gsm = ioremap_wc(phys_addr, size); 8539ce07d94SLucas De Marchi if (!ggtt->gsm) { 8549ce07d94SLucas De Marchi drm_err(&i915->drm, "Failed to map the ggtt page table\n"); 8559ce07d94SLucas De Marchi return -ENOMEM; 8569ce07d94SLucas De Marchi } 8579ce07d94SLucas De Marchi 8589ce07d94SLucas De Marchi kref_init(&ggtt->vm.resv_ref); 8599ce07d94SLucas De Marchi ret = setup_scratch_page(&ggtt->vm); 8609ce07d94SLucas De Marchi if (ret) { 8619ce07d94SLucas De Marchi drm_err(&i915->drm, "Scratch setup failed\n"); 8629ce07d94SLucas De Marchi /* iounmap will also get called at remove, but meh */ 8639ce07d94SLucas De Marchi iounmap(ggtt->gsm); 8649ce07d94SLucas De Marchi return ret; 8659ce07d94SLucas De Marchi } 8669ce07d94SLucas De Marchi 8679ce07d94SLucas De Marchi pte_flags = 0; 8689ce07d94SLucas De Marchi if (i915_gem_object_is_lmem(ggtt->vm.scratch[0])) 8699ce07d94SLucas De Marchi pte_flags |= PTE_LM; 8709ce07d94SLucas De Marchi 8719ce07d94SLucas De Marchi ggtt->vm.scratch[0]->encode = 8729ce07d94SLucas De Marchi ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), 8739ce07d94SLucas De Marchi I915_CACHE_NONE, pte_flags); 8749ce07d94SLucas De Marchi 8759ce07d94SLucas De Marchi return 0; 8769ce07d94SLucas De Marchi } 8779ce07d94SLucas De Marchi 8789ce07d94SLucas De Marchi static void gen6_gmch_remove(struct i915_address_space *vm) 8799ce07d94SLucas De Marchi { 8809ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 8819ce07d94SLucas De Marchi 8829ce07d94SLucas De Marchi iounmap(ggtt->gsm); 8839ce07d94SLucas De Marchi free_scratch(vm); 8849ce07d94SLucas De Marchi } 8859ce07d94SLucas De Marchi 8869ce07d94SLucas De Marchi static struct resource pci_resource(struct pci_dev *pdev, int bar) 8872c86e55dSMatthew Auld { 8882c86e55dSMatthew Auld return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar), 8892c86e55dSMatthew Auld pci_resource_len(pdev, bar)); 8902c86e55dSMatthew Auld } 8912c86e55dSMatthew Auld 8929ce07d94SLucas De Marchi static int gen8_gmch_probe(struct i915_ggtt *ggtt) 8939ce07d94SLucas De Marchi { 8949ce07d94SLucas De Marchi struct drm_i915_private *i915 = ggtt->vm.i915; 8959ce07d94SLucas De Marchi struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 8969ce07d94SLucas De Marchi unsigned int size; 8979ce07d94SLucas De Marchi u16 snb_gmch_ctl; 8989ce07d94SLucas De Marchi 89903eababbSVille Syrjälä if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) { 9000492a34cSVille Syrjälä if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR)) 9011bba7323SPiotr Piórkowski return -ENXIO; 9021bba7323SPiotr Piórkowski 9030492a34cSVille Syrjälä ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR); 9049ce07d94SLucas De Marchi ggtt->mappable_end = resource_size(&ggtt->gmadr); 9059ce07d94SLucas De Marchi } 9069ce07d94SLucas De Marchi 9079ce07d94SLucas De Marchi pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 9089ce07d94SLucas De Marchi if (IS_CHERRYVIEW(i915)) 9099ce07d94SLucas De Marchi size = chv_get_total_gtt_size(snb_gmch_ctl); 9109ce07d94SLucas De Marchi else 9119ce07d94SLucas De Marchi size = gen8_get_total_gtt_size(snb_gmch_ctl); 9129ce07d94SLucas De Marchi 9139ce07d94SLucas De Marchi ggtt->vm.alloc_pt_dma = alloc_pt_dma; 9149ce07d94SLucas De Marchi ggtt->vm.alloc_scratch_dma = alloc_pt_dma; 9159ce07d94SLucas De Marchi ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY; 9169ce07d94SLucas De Marchi 9179ce07d94SLucas De Marchi ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 9189ce07d94SLucas De Marchi ggtt->vm.cleanup = gen6_gmch_remove; 9199ce07d94SLucas De Marchi ggtt->vm.insert_page = gen8_ggtt_insert_page; 9209ce07d94SLucas De Marchi ggtt->vm.clear_range = nop_clear_range; 9219ce07d94SLucas De Marchi 9229ce07d94SLucas De Marchi ggtt->vm.insert_entries = gen8_ggtt_insert_entries; 9239ce07d94SLucas De Marchi 9249ce07d94SLucas De Marchi /* 9259ce07d94SLucas De Marchi * Serialize GTT updates with aperture access on BXT if VT-d is on, 9269ce07d94SLucas De Marchi * and always on CHV. 9279ce07d94SLucas De Marchi */ 9289ce07d94SLucas De Marchi if (intel_vm_no_concurrent_access_wa(i915)) { 9299ce07d94SLucas De Marchi ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; 9309ce07d94SLucas De Marchi ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; 931a0696856SNirmoy Das 932a0696856SNirmoy Das /* 933a0696856SNirmoy Das * Calling stop_machine() version of GGTT update function 934a0696856SNirmoy Das * at error capture/reset path will raise lockdep warning. 935a0696856SNirmoy Das * Allow calling gen8_ggtt_insert_* directly at reset path 936a0696856SNirmoy Das * which is safe from parallel GGTT updates. 937a0696856SNirmoy Das */ 938a0696856SNirmoy Das ggtt->vm.raw_insert_page = gen8_ggtt_insert_page; 939a0696856SNirmoy Das ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries; 940a0696856SNirmoy Das 9419ce07d94SLucas De Marchi ggtt->vm.bind_async_flags = 9429ce07d94SLucas De Marchi I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 9439ce07d94SLucas De Marchi } 9449ce07d94SLucas De Marchi 945f2053d34SDaniele Ceraolo Spurio if (intel_uc_wants_guc(&ggtt->vm.gt->uc)) 946f2053d34SDaniele Ceraolo Spurio ggtt->invalidate = guc_ggtt_invalidate; 947f2053d34SDaniele Ceraolo Spurio else 9489ce07d94SLucas De Marchi ggtt->invalidate = gen8_ggtt_invalidate; 9499ce07d94SLucas De Marchi 9509ce07d94SLucas De Marchi ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 9519ce07d94SLucas De Marchi ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 9529ce07d94SLucas De Marchi 9539ce07d94SLucas De Marchi ggtt->vm.pte_encode = gen8_ggtt_pte_encode; 9549ce07d94SLucas De Marchi 9559ce07d94SLucas De Marchi return ggtt_probe_common(ggtt, size); 9569ce07d94SLucas De Marchi } 9579ce07d94SLucas De Marchi 9589ce07d94SLucas De Marchi static u64 snb_pte_encode(dma_addr_t addr, 9599ce07d94SLucas De Marchi enum i915_cache_level level, 9609ce07d94SLucas De Marchi u32 flags) 9619ce07d94SLucas De Marchi { 9629ce07d94SLucas De Marchi gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 9639ce07d94SLucas De Marchi 9649ce07d94SLucas De Marchi switch (level) { 9659ce07d94SLucas De Marchi case I915_CACHE_L3_LLC: 9669ce07d94SLucas De Marchi case I915_CACHE_LLC: 9679ce07d94SLucas De Marchi pte |= GEN6_PTE_CACHE_LLC; 9689ce07d94SLucas De Marchi break; 9699ce07d94SLucas De Marchi case I915_CACHE_NONE: 9709ce07d94SLucas De Marchi pte |= GEN6_PTE_UNCACHED; 9719ce07d94SLucas De Marchi break; 9729ce07d94SLucas De Marchi default: 9739ce07d94SLucas De Marchi MISSING_CASE(level); 9749ce07d94SLucas De Marchi } 9759ce07d94SLucas De Marchi 9769ce07d94SLucas De Marchi return pte; 9779ce07d94SLucas De Marchi } 9789ce07d94SLucas De Marchi 9799ce07d94SLucas De Marchi static u64 ivb_pte_encode(dma_addr_t addr, 9809ce07d94SLucas De Marchi enum i915_cache_level level, 9819ce07d94SLucas De Marchi u32 flags) 9829ce07d94SLucas De Marchi { 9839ce07d94SLucas De Marchi gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 9849ce07d94SLucas De Marchi 9859ce07d94SLucas De Marchi switch (level) { 9869ce07d94SLucas De Marchi case I915_CACHE_L3_LLC: 9879ce07d94SLucas De Marchi pte |= GEN7_PTE_CACHE_L3_LLC; 9889ce07d94SLucas De Marchi break; 9899ce07d94SLucas De Marchi case I915_CACHE_LLC: 9909ce07d94SLucas De Marchi pte |= GEN6_PTE_CACHE_LLC; 9919ce07d94SLucas De Marchi break; 9929ce07d94SLucas De Marchi case I915_CACHE_NONE: 9939ce07d94SLucas De Marchi pte |= GEN6_PTE_UNCACHED; 9949ce07d94SLucas De Marchi break; 9959ce07d94SLucas De Marchi default: 9969ce07d94SLucas De Marchi MISSING_CASE(level); 9979ce07d94SLucas De Marchi } 9989ce07d94SLucas De Marchi 9999ce07d94SLucas De Marchi return pte; 10009ce07d94SLucas De Marchi } 10019ce07d94SLucas De Marchi 10029ce07d94SLucas De Marchi static u64 byt_pte_encode(dma_addr_t addr, 10039ce07d94SLucas De Marchi enum i915_cache_level level, 10049ce07d94SLucas De Marchi u32 flags) 10059ce07d94SLucas De Marchi { 10069ce07d94SLucas De Marchi gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 10079ce07d94SLucas De Marchi 10089ce07d94SLucas De Marchi if (!(flags & PTE_READ_ONLY)) 10099ce07d94SLucas De Marchi pte |= BYT_PTE_WRITEABLE; 10109ce07d94SLucas De Marchi 10119ce07d94SLucas De Marchi if (level != I915_CACHE_NONE) 10129ce07d94SLucas De Marchi pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 10139ce07d94SLucas De Marchi 10149ce07d94SLucas De Marchi return pte; 10159ce07d94SLucas De Marchi } 10169ce07d94SLucas De Marchi 10179ce07d94SLucas De Marchi static u64 hsw_pte_encode(dma_addr_t addr, 10189ce07d94SLucas De Marchi enum i915_cache_level level, 10199ce07d94SLucas De Marchi u32 flags) 10209ce07d94SLucas De Marchi { 10219ce07d94SLucas De Marchi gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 10229ce07d94SLucas De Marchi 10239ce07d94SLucas De Marchi if (level != I915_CACHE_NONE) 10249ce07d94SLucas De Marchi pte |= HSW_WB_LLC_AGE3; 10259ce07d94SLucas De Marchi 10269ce07d94SLucas De Marchi return pte; 10279ce07d94SLucas De Marchi } 10289ce07d94SLucas De Marchi 10299ce07d94SLucas De Marchi static u64 iris_pte_encode(dma_addr_t addr, 10309ce07d94SLucas De Marchi enum i915_cache_level level, 10319ce07d94SLucas De Marchi u32 flags) 10329ce07d94SLucas De Marchi { 10339ce07d94SLucas De Marchi gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 10349ce07d94SLucas De Marchi 10359ce07d94SLucas De Marchi switch (level) { 10369ce07d94SLucas De Marchi case I915_CACHE_NONE: 10379ce07d94SLucas De Marchi break; 10389ce07d94SLucas De Marchi case I915_CACHE_WT: 10399ce07d94SLucas De Marchi pte |= HSW_WT_ELLC_LLC_AGE3; 10409ce07d94SLucas De Marchi break; 10419ce07d94SLucas De Marchi default: 10429ce07d94SLucas De Marchi pte |= HSW_WB_ELLC_LLC_AGE3; 10439ce07d94SLucas De Marchi break; 10449ce07d94SLucas De Marchi } 10459ce07d94SLucas De Marchi 10469ce07d94SLucas De Marchi return pte; 10479ce07d94SLucas De Marchi } 10489ce07d94SLucas De Marchi 10499ce07d94SLucas De Marchi static int gen6_gmch_probe(struct i915_ggtt *ggtt) 10509ce07d94SLucas De Marchi { 10519ce07d94SLucas De Marchi struct drm_i915_private *i915 = ggtt->vm.i915; 10529ce07d94SLucas De Marchi struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 10539ce07d94SLucas De Marchi unsigned int size; 10549ce07d94SLucas De Marchi u16 snb_gmch_ctl; 10559ce07d94SLucas De Marchi 10560492a34cSVille Syrjälä if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR)) 10571bba7323SPiotr Piórkowski return -ENXIO; 10581bba7323SPiotr Piórkowski 10590492a34cSVille Syrjälä ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR); 10609ce07d94SLucas De Marchi ggtt->mappable_end = resource_size(&ggtt->gmadr); 10619ce07d94SLucas De Marchi 10629ce07d94SLucas De Marchi /* 10639ce07d94SLucas De Marchi * 64/512MB is the current min/max we actually know of, but this is 10649ce07d94SLucas De Marchi * just a coarse sanity check. 10659ce07d94SLucas De Marchi */ 10669ce07d94SLucas De Marchi if (ggtt->mappable_end < (64 << 20) || 10679ce07d94SLucas De Marchi ggtt->mappable_end > (512 << 20)) { 10689ce07d94SLucas De Marchi drm_err(&i915->drm, "Unknown GMADR size (%pa)\n", 10699ce07d94SLucas De Marchi &ggtt->mappable_end); 10709ce07d94SLucas De Marchi return -ENXIO; 10719ce07d94SLucas De Marchi } 10729ce07d94SLucas De Marchi 10739ce07d94SLucas De Marchi pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 10749ce07d94SLucas De Marchi 10759ce07d94SLucas De Marchi size = gen6_get_total_gtt_size(snb_gmch_ctl); 10769ce07d94SLucas De Marchi ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; 10779ce07d94SLucas De Marchi 10789ce07d94SLucas De Marchi ggtt->vm.alloc_pt_dma = alloc_pt_dma; 10799ce07d94SLucas De Marchi ggtt->vm.alloc_scratch_dma = alloc_pt_dma; 10809ce07d94SLucas De Marchi 10819ce07d94SLucas De Marchi ggtt->vm.clear_range = nop_clear_range; 1082eea380adSChris Wilson if (!HAS_FULL_PPGTT(i915)) 10839ce07d94SLucas De Marchi ggtt->vm.clear_range = gen6_ggtt_clear_range; 10849ce07d94SLucas De Marchi ggtt->vm.insert_page = gen6_ggtt_insert_page; 10859ce07d94SLucas De Marchi ggtt->vm.insert_entries = gen6_ggtt_insert_entries; 10869ce07d94SLucas De Marchi ggtt->vm.cleanup = gen6_gmch_remove; 10879ce07d94SLucas De Marchi 10889ce07d94SLucas De Marchi ggtt->invalidate = gen6_ggtt_invalidate; 10899ce07d94SLucas De Marchi 10909ce07d94SLucas De Marchi if (HAS_EDRAM(i915)) 10919ce07d94SLucas De Marchi ggtt->vm.pte_encode = iris_pte_encode; 10929ce07d94SLucas De Marchi else if (IS_HASWELL(i915)) 10939ce07d94SLucas De Marchi ggtt->vm.pte_encode = hsw_pte_encode; 10949ce07d94SLucas De Marchi else if (IS_VALLEYVIEW(i915)) 10959ce07d94SLucas De Marchi ggtt->vm.pte_encode = byt_pte_encode; 10969ce07d94SLucas De Marchi else if (GRAPHICS_VER(i915) >= 7) 10979ce07d94SLucas De Marchi ggtt->vm.pte_encode = ivb_pte_encode; 10989ce07d94SLucas De Marchi else 10999ce07d94SLucas De Marchi ggtt->vm.pte_encode = snb_pte_encode; 11009ce07d94SLucas De Marchi 11019ce07d94SLucas De Marchi ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 11029ce07d94SLucas De Marchi ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 11039ce07d94SLucas De Marchi 11049ce07d94SLucas De Marchi return ggtt_probe_common(ggtt, size); 11059ce07d94SLucas De Marchi } 11069ce07d94SLucas De Marchi 11072c86e55dSMatthew Auld static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) 11082c86e55dSMatthew Auld { 11092c86e55dSMatthew Auld struct drm_i915_private *i915 = gt->i915; 11102c86e55dSMatthew Auld int ret; 11112c86e55dSMatthew Auld 11122c86e55dSMatthew Auld ggtt->vm.gt = gt; 11132c86e55dSMatthew Auld ggtt->vm.i915 = i915; 1114e322551fSThomas Zimmermann ggtt->vm.dma = i915->drm.dev; 11154d8151aeSThomas Hellström dma_resv_init(&ggtt->vm._resv); 11162c86e55dSMatthew Auld 11179ce07d94SLucas De Marchi if (GRAPHICS_VER(i915) >= 8) 11189ce07d94SLucas De Marchi ret = gen8_gmch_probe(ggtt); 11199ce07d94SLucas De Marchi else if (GRAPHICS_VER(i915) >= 6) 11209ce07d94SLucas De Marchi ret = gen6_gmch_probe(ggtt); 11212c86e55dSMatthew Auld else 11229ce07d94SLucas De Marchi ret = intel_ggtt_gmch_probe(ggtt); 11239ce07d94SLucas De Marchi 112426ad4f8bSMaarten Lankhorst if (ret) { 11254d8151aeSThomas Hellström dma_resv_fini(&ggtt->vm._resv); 11262c86e55dSMatthew Auld return ret; 112726ad4f8bSMaarten Lankhorst } 11282c86e55dSMatthew Auld 11292c86e55dSMatthew Auld if ((ggtt->vm.total - 1) >> 32) { 113036034c95SWambui Karuga drm_err(&i915->drm, 113136034c95SWambui Karuga "We never expected a Global GTT with more than 32bits" 11322c86e55dSMatthew Auld " of address space! Found %lldM!\n", 11332c86e55dSMatthew Auld ggtt->vm.total >> 20); 11342c86e55dSMatthew Auld ggtt->vm.total = 1ULL << 32; 11352c86e55dSMatthew Auld ggtt->mappable_end = 11362c86e55dSMatthew Auld min_t(u64, ggtt->mappable_end, ggtt->vm.total); 11372c86e55dSMatthew Auld } 11382c86e55dSMatthew Auld 11392c86e55dSMatthew Auld if (ggtt->mappable_end > ggtt->vm.total) { 114036034c95SWambui Karuga drm_err(&i915->drm, 114136034c95SWambui Karuga "mappable aperture extends past end of GGTT," 11422c86e55dSMatthew Auld " aperture=%pa, total=%llx\n", 11432c86e55dSMatthew Auld &ggtt->mappable_end, ggtt->vm.total); 11442c86e55dSMatthew Auld ggtt->mappable_end = ggtt->vm.total; 11452c86e55dSMatthew Auld } 11462c86e55dSMatthew Auld 11472c86e55dSMatthew Auld /* GMADR is the PCI mmio aperture into the global GTT. */ 114836034c95SWambui Karuga drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20); 114936034c95SWambui Karuga drm_dbg(&i915->drm, "GMADR size = %lluM\n", 115036034c95SWambui Karuga (u64)ggtt->mappable_end >> 20); 115136034c95SWambui Karuga drm_dbg(&i915->drm, "DSM size = %lluM\n", 11522c86e55dSMatthew Auld (u64)resource_size(&intel_graphics_stolen_res) >> 20); 11532c86e55dSMatthew Auld 11542c86e55dSMatthew Auld return 0; 11552c86e55dSMatthew Auld } 11562c86e55dSMatthew Auld 11572c86e55dSMatthew Auld /** 11582c86e55dSMatthew Auld * i915_ggtt_probe_hw - Probe GGTT hardware location 11592c86e55dSMatthew Auld * @i915: i915 device 11602c86e55dSMatthew Auld */ 11612c86e55dSMatthew Auld int i915_ggtt_probe_hw(struct drm_i915_private *i915) 11622c86e55dSMatthew Auld { 11630f857158SAravind Iddamsetty struct intel_gt *gt; 11640f857158SAravind Iddamsetty int ret, i; 11650f857158SAravind Iddamsetty 11660f857158SAravind Iddamsetty for_each_gt(gt, i915, i) { 11670f857158SAravind Iddamsetty ret = intel_gt_assign_ggtt(gt); 11680f857158SAravind Iddamsetty if (ret) 11690f857158SAravind Iddamsetty return ret; 11700f857158SAravind Iddamsetty } 11712c86e55dSMatthew Auld 1172848915c3SMichał Winiarski ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915)); 11732c86e55dSMatthew Auld if (ret) 11742c86e55dSMatthew Auld return ret; 11752c86e55dSMatthew Auld 1176a7f46d5bSTvrtko Ursulin if (i915_vtd_active(i915)) 1177dc483ba5SJani Nikula drm_info(&i915->drm, "VT-d active for gfx access\n"); 11782c86e55dSMatthew Auld 11792c86e55dSMatthew Auld return 0; 11802c86e55dSMatthew Auld } 11812c86e55dSMatthew Auld 11820f857158SAravind Iddamsetty struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915) 11830f857158SAravind Iddamsetty { 11840f857158SAravind Iddamsetty struct i915_ggtt *ggtt; 11850f857158SAravind Iddamsetty 11860f857158SAravind Iddamsetty ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL); 11870f857158SAravind Iddamsetty if (!ggtt) 11880f857158SAravind Iddamsetty return ERR_PTR(-ENOMEM); 11890f857158SAravind Iddamsetty 11900f857158SAravind Iddamsetty INIT_LIST_HEAD(&ggtt->gt_list); 11910f857158SAravind Iddamsetty 11920f857158SAravind Iddamsetty return ggtt; 11930f857158SAravind Iddamsetty } 11940f857158SAravind Iddamsetty 11952c86e55dSMatthew Auld int i915_ggtt_enable_hw(struct drm_i915_private *i915) 11962c86e55dSMatthew Auld { 11979ce07d94SLucas De Marchi if (GRAPHICS_VER(i915) < 6) 11989ce07d94SLucas De Marchi return intel_ggtt_gmch_enable_hw(i915); 11999ce07d94SLucas De Marchi 12009ce07d94SLucas De Marchi return 0; 12012c86e55dSMatthew Auld } 12022c86e55dSMatthew Auld 12038d2f683fSImre Deak /** 12048d2f683fSImre Deak * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM 12058d2f683fSImre Deak * @vm: The VM to restore the mappings for 12068d2f683fSImre Deak * 12078d2f683fSImre Deak * Restore the memory mappings for all objects mapped to HW via the GGTT or a 12088d2f683fSImre Deak * DPT page table. 12098d2f683fSImre Deak * 12108d2f683fSImre Deak * Returns %true if restoring the mapping for any object that was in a write 12118d2f683fSImre Deak * domain before suspend. 12128d2f683fSImre Deak */ 12138d2f683fSImre Deak bool i915_ggtt_resume_vm(struct i915_address_space *vm) 12142c86e55dSMatthew Auld { 121580e5351dSChris Wilson struct i915_vma *vma; 12168d2f683fSImre Deak bool write_domain_objs = false; 12172c86e55dSMatthew Auld 12188d2f683fSImre Deak drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); 12192c86e55dSMatthew Auld 1220*de3a9ab9SAndi Shyti /* First fill our portion of the GTT with scratch pages */ 12218d2f683fSImre Deak vm->clear_range(vm, 0, vm->total); 12222c86e55dSMatthew Auld 12232c86e55dSMatthew Auld /* clflush objects bound into the GGTT and rebind them. */ 12248d2f683fSImre Deak list_for_each_entry(vma, &vm->bound_list, vm_link) { 12252c86e55dSMatthew Auld struct drm_i915_gem_object *obj = vma->obj; 1226cd0452aaSChris Wilson unsigned int was_bound = 1227cd0452aaSChris Wilson atomic_read(&vma->flags) & I915_VMA_BIND_MASK; 12282c86e55dSMatthew Auld 1229cd0452aaSChris Wilson GEM_BUG_ON(!was_bound); 1230*de3a9ab9SAndi Shyti 1231bc247253SThomas Hellström /* 1232bc247253SThomas Hellström * Clear the bound flags of the vma resource to allow 1233bc247253SThomas Hellström * ptes to be repopulated. 1234bc247253SThomas Hellström */ 1235bc247253SThomas Hellström vma->resource->bound_flags = 0; 1236647bfd26STvrtko Ursulin vma->ops->bind_vma(vm, NULL, vma->resource, 12372c86e55dSMatthew Auld obj ? obj->cache_level : 0, 1238cd0452aaSChris Wilson was_bound); 1239*de3a9ab9SAndi Shyti 12402c86e55dSMatthew Auld if (obj) { /* only used during resume => exclusive access */ 12418d2f683fSImre Deak write_domain_objs |= fetch_and_zero(&obj->write_domain); 12422c86e55dSMatthew Auld obj->read_domains |= I915_GEM_DOMAIN_GTT; 12432c86e55dSMatthew Auld } 12442c86e55dSMatthew Auld } 12452c86e55dSMatthew Auld 12468d2f683fSImre Deak return write_domain_objs; 12478d2f683fSImre Deak } 12488d2f683fSImre Deak 12498d2f683fSImre Deak void i915_ggtt_resume(struct i915_ggtt *ggtt) 12508d2f683fSImre Deak { 12510f857158SAravind Iddamsetty struct intel_gt *gt; 12528d2f683fSImre Deak bool flush; 12538d2f683fSImre Deak 12540f857158SAravind Iddamsetty list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 12550f857158SAravind Iddamsetty intel_gt_check_and_clear_faults(gt); 12568d2f683fSImre Deak 12578d2f683fSImre Deak flush = i915_ggtt_resume_vm(&ggtt->vm); 12588d2f683fSImre Deak 12592c86e55dSMatthew Auld ggtt->invalidate(ggtt); 12602c86e55dSMatthew Auld 12612c86e55dSMatthew Auld if (flush) 12622c86e55dSMatthew Auld wbinvd_on_all_cpus(); 12632c86e55dSMatthew Auld 1264dec9cf9eSChris Wilson intel_ggtt_restore_fences(ggtt); 12652c86e55dSMatthew Auld } 1266