12c86e55dSMatthew Auld // SPDX-License-Identifier: MIT
22c86e55dSMatthew Auld /*
32c86e55dSMatthew Auld * Copyright © 2020 Intel Corporation
42c86e55dSMatthew Auld */
52c86e55dSMatthew Auld
62c86e55dSMatthew Auld #include <asm/set_memory.h>
78801eb48SChen Zhou #include <asm/smp.h>
89ce07d94SLucas De Marchi #include <linux/types.h>
99ce07d94SLucas De Marchi #include <linux/stop_machine.h>
102c86e55dSMatthew Auld
110f857158SAravind Iddamsetty #include <drm/drm_managed.h>
1283d2bdb6SJani Nikula #include <drm/i915_drm.h>
139ce07d94SLucas De Marchi #include <drm/intel-gtt.h>
1483d2bdb6SJani Nikula
15acc855d3SJani Nikula #include "display/intel_display.h"
16e762bdf5SMatthew Auld #include "gem/i915_gem_lmem.h"
17e762bdf5SMatthew Auld
189ce07d94SLucas De Marchi #include "intel_ggtt_gmch.h"
192c86e55dSMatthew Auld #include "intel_gt.h"
200d6419e9SMatt Roper #include "intel_gt_regs.h"
216bba2b30SPiotr Piórkowski #include "intel_pci_config.h"
222c86e55dSMatthew Auld #include "i915_drv.h"
231bba7323SPiotr Piórkowski #include "i915_pci.h"
242c86e55dSMatthew Auld #include "i915_scatterlist.h"
25a7f46d5bSTvrtko Ursulin #include "i915_utils.h"
262c86e55dSMatthew Auld #include "i915_vgpu.h"
272c86e55dSMatthew Auld
282c86e55dSMatthew Auld #include "intel_gtt.h"
2933e7a975SVille Syrjälä #include "gen8_ppgtt.h"
302c86e55dSMatthew Auld
i915_ggtt_color_adjust(const struct drm_mm_node * node,unsigned long color,u64 * start,u64 * end)312c86e55dSMatthew Auld static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
322c86e55dSMatthew Auld unsigned long color,
332c86e55dSMatthew Auld u64 *start,
342c86e55dSMatthew Auld u64 *end)
352c86e55dSMatthew Auld {
362c86e55dSMatthew Auld if (i915_node_color_differs(node, color))
372c86e55dSMatthew Auld *start += I915_GTT_PAGE_SIZE;
382c86e55dSMatthew Auld
392c86e55dSMatthew Auld /*
402c86e55dSMatthew Auld * Also leave a space between the unallocated reserved node after the
412c86e55dSMatthew Auld * GTT and any objects within the GTT, i.e. we use the color adjustment
422c86e55dSMatthew Auld * to insert a guard page to prevent prefetches crossing over the
432c86e55dSMatthew Auld * GTT boundary.
442c86e55dSMatthew Auld */
452c86e55dSMatthew Auld node = list_next_entry(node, node_list);
462c86e55dSMatthew Auld if (node->color != color)
472c86e55dSMatthew Auld *end -= I915_GTT_PAGE_SIZE;
482c86e55dSMatthew Auld }
492c86e55dSMatthew Auld
ggtt_init_hw(struct i915_ggtt * ggtt)502c86e55dSMatthew Auld static int ggtt_init_hw(struct i915_ggtt *ggtt)
512c86e55dSMatthew Auld {
522c86e55dSMatthew Auld struct drm_i915_private *i915 = ggtt->vm.i915;
532c86e55dSMatthew Auld
542c86e55dSMatthew Auld i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
552c86e55dSMatthew Auld
562c86e55dSMatthew Auld ggtt->vm.is_ggtt = true;
572c86e55dSMatthew Auld
582c86e55dSMatthew Auld /* Only VLV supports read-only GGTT mappings */
592c86e55dSMatthew Auld ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
602c86e55dSMatthew Auld
612c86e55dSMatthew Auld if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
622c86e55dSMatthew Auld ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
632c86e55dSMatthew Auld
642c86e55dSMatthew Auld if (ggtt->mappable_end) {
652c86e55dSMatthew Auld if (!io_mapping_init_wc(&ggtt->iomap,
662c86e55dSMatthew Auld ggtt->gmadr.start,
672c86e55dSMatthew Auld ggtt->mappable_end)) {
682c86e55dSMatthew Auld ggtt->vm.cleanup(&ggtt->vm);
692c86e55dSMatthew Auld return -EIO;
702c86e55dSMatthew Auld }
712c86e55dSMatthew Auld
722c86e55dSMatthew Auld ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
732c86e55dSMatthew Auld ggtt->mappable_end);
742c86e55dSMatthew Auld }
752c86e55dSMatthew Auld
76f899f786SChris Wilson intel_ggtt_init_fences(ggtt);
772c86e55dSMatthew Auld
782c86e55dSMatthew Auld return 0;
792c86e55dSMatthew Auld }
802c86e55dSMatthew Auld
812c86e55dSMatthew Auld /**
822c86e55dSMatthew Auld * i915_ggtt_init_hw - Initialize GGTT hardware
832c86e55dSMatthew Auld * @i915: i915 device
842c86e55dSMatthew Auld */
i915_ggtt_init_hw(struct drm_i915_private * i915)852c86e55dSMatthew Auld int i915_ggtt_init_hw(struct drm_i915_private *i915)
862c86e55dSMatthew Auld {
872c86e55dSMatthew Auld int ret;
882c86e55dSMatthew Auld
892c86e55dSMatthew Auld /*
902c86e55dSMatthew Auld * Note that we use page colouring to enforce a guard page at the
912c86e55dSMatthew Auld * end of the address space. This is required as the CS may prefetch
922c86e55dSMatthew Auld * beyond the end of the batch buffer, across the page boundary,
932c86e55dSMatthew Auld * and beyond the end of the GTT if we do not provide a guard.
942c86e55dSMatthew Auld */
95848915c3SMichał Winiarski ret = ggtt_init_hw(to_gt(i915)->ggtt);
962c86e55dSMatthew Auld if (ret)
972c86e55dSMatthew Auld return ret;
982c86e55dSMatthew Auld
992c86e55dSMatthew Auld return 0;
1002c86e55dSMatthew Auld }
1012c86e55dSMatthew Auld
1028d2f683fSImre Deak /**
1038d2f683fSImre Deak * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM
1048d2f683fSImre Deak * @vm: The VM to suspend the mappings for
1058d2f683fSImre Deak *
1068d2f683fSImre Deak * Suspend the memory mappings for all objects mapped to HW via the GGTT or a
1078d2f683fSImre Deak * DPT page table.
1088d2f683fSImre Deak */
i915_ggtt_suspend_vm(struct i915_address_space * vm)1098d2f683fSImre Deak void i915_ggtt_suspend_vm(struct i915_address_space *vm)
1102c86e55dSMatthew Auld {
111bffa18ddSChris Wilson struct i915_vma *vma, *vn;
112e1a7ab4fSThomas Hellström int save_skip_rewrite;
113e3793468SChris Wilson
1148d2f683fSImre Deak drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
1158d2f683fSImre Deak
1160f341974SMaarten Lankhorst retry:
1170f341974SMaarten Lankhorst i915_gem_drain_freed_objects(vm->i915);
1180f341974SMaarten Lankhorst
1198d2f683fSImre Deak mutex_lock(&vm->mutex);
120bffa18ddSChris Wilson
121e1a7ab4fSThomas Hellström /*
122e1a7ab4fSThomas Hellström * Skip rewriting PTE on VMA unbind.
123e1a7ab4fSThomas Hellström * FIXME: Use an argument to i915_vma_unbind() instead?
124e1a7ab4fSThomas Hellström */
125e1a7ab4fSThomas Hellström save_skip_rewrite = vm->skip_pte_rewrite;
126e1a7ab4fSThomas Hellström vm->skip_pte_rewrite = true;
127bffa18ddSChris Wilson
1288d2f683fSImre Deak list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
1290f341974SMaarten Lankhorst struct drm_i915_gem_object *obj = vma->obj;
130e3793468SChris Wilson
1310f341974SMaarten Lankhorst GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1320f341974SMaarten Lankhorst
1330f341974SMaarten Lankhorst if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
134bffa18ddSChris Wilson continue;
135bffa18ddSChris Wilson
1360f341974SMaarten Lankhorst /* unlikely to race when GPU is idle, so no worry about slowpath.. */
1370f341974SMaarten Lankhorst if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) {
1380f341974SMaarten Lankhorst /*
1390f341974SMaarten Lankhorst * No dead objects should appear here, GPU should be
1400f341974SMaarten Lankhorst * completely idle, and userspace suspended
1410f341974SMaarten Lankhorst */
1420f341974SMaarten Lankhorst i915_gem_object_get(obj);
1430f341974SMaarten Lankhorst
1440f341974SMaarten Lankhorst mutex_unlock(&vm->mutex);
1450f341974SMaarten Lankhorst
1460f341974SMaarten Lankhorst i915_gem_object_lock(obj, NULL);
147e1a7ab4fSThomas Hellström GEM_WARN_ON(i915_vma_unbind(vma));
1480f341974SMaarten Lankhorst i915_gem_object_unlock(obj);
1490f341974SMaarten Lankhorst i915_gem_object_put(obj);
150e1a7ab4fSThomas Hellström
151e1a7ab4fSThomas Hellström vm->skip_pte_rewrite = save_skip_rewrite;
1520f341974SMaarten Lankhorst goto retry;
1530f341974SMaarten Lankhorst }
1540f341974SMaarten Lankhorst
155bffa18ddSChris Wilson if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
1560f341974SMaarten Lankhorst i915_vma_wait_for_bind(vma);
1570f341974SMaarten Lankhorst
1582f6b90daSThomas Hellström __i915_vma_evict(vma, false);
159bffa18ddSChris Wilson drm_mm_remove_node(&vma->node);
160bffa18ddSChris Wilson }
1610f341974SMaarten Lankhorst
1620f341974SMaarten Lankhorst i915_gem_object_unlock(obj);
163bffa18ddSChris Wilson }
164bffa18ddSChris Wilson
1658d2f683fSImre Deak vm->clear_range(vm, 0, vm->total);
166bffa18ddSChris Wilson
167e1a7ab4fSThomas Hellström vm->skip_pte_rewrite = save_skip_rewrite;
1688d2f683fSImre Deak
1698d2f683fSImre Deak mutex_unlock(&vm->mutex);
1708d2f683fSImre Deak }
1718d2f683fSImre Deak
i915_ggtt_suspend(struct i915_ggtt * ggtt)1728d2f683fSImre Deak void i915_ggtt_suspend(struct i915_ggtt *ggtt)
1738d2f683fSImre Deak {
1740f857158SAravind Iddamsetty struct intel_gt *gt;
1750f857158SAravind Iddamsetty
1768d2f683fSImre Deak i915_ggtt_suspend_vm(&ggtt->vm);
1778d2f683fSImre Deak ggtt->invalidate(ggtt);
1782c86e55dSMatthew Auld
1790f857158SAravind Iddamsetty list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1800f857158SAravind Iddamsetty intel_gt_check_and_clear_faults(gt);
1812c86e55dSMatthew Auld }
1822c86e55dSMatthew Auld
gen6_ggtt_invalidate(struct i915_ggtt * ggtt)1832c86e55dSMatthew Auld void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
1842c86e55dSMatthew Auld {
1852c86e55dSMatthew Auld struct intel_uncore *uncore = ggtt->vm.gt->uncore;
1862c86e55dSMatthew Auld
1872c86e55dSMatthew Auld spin_lock_irq(&uncore->lock);
1882c86e55dSMatthew Auld intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1892c86e55dSMatthew Auld intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
1902c86e55dSMatthew Auld spin_unlock_irq(&uncore->lock);
1912c86e55dSMatthew Auld }
1922c86e55dSMatthew Auld
needs_wc_ggtt_mapping(struct drm_i915_private * i915)193*4a6bb698SNirmoy Das static bool needs_wc_ggtt_mapping(struct drm_i915_private *i915)
194*4a6bb698SNirmoy Das {
195*4a6bb698SNirmoy Das /*
196*4a6bb698SNirmoy Das * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range
197*4a6bb698SNirmoy Das * will be dropped. For WC mappings in general we have 64 byte burst
198*4a6bb698SNirmoy Das * writes when the WC buffer is flushed, so we can't use it, but have to
199*4a6bb698SNirmoy Das * resort to an uncached mapping. The WC issue is easily caught by the
200*4a6bb698SNirmoy Das * readback check when writing GTT PTE entries.
201*4a6bb698SNirmoy Das */
202*4a6bb698SNirmoy Das if (!IS_GEN9_LP(i915) && GRAPHICS_VER(i915) < 11)
203*4a6bb698SNirmoy Das return true;
204*4a6bb698SNirmoy Das
205*4a6bb698SNirmoy Das return false;
206*4a6bb698SNirmoy Das }
207*4a6bb698SNirmoy Das
gen8_ggtt_invalidate(struct i915_ggtt * ggtt)2089ce07d94SLucas De Marchi static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
2092c86e55dSMatthew Auld {
2102c86e55dSMatthew Auld struct intel_uncore *uncore = ggtt->vm.gt->uncore;
2112c86e55dSMatthew Auld
2122c86e55dSMatthew Auld /*
2132c86e55dSMatthew Auld * Note that as an uncached mmio write, this will flush the
2142c86e55dSMatthew Auld * WCB of the writes into the GGTT before it triggers the invalidate.
215*4a6bb698SNirmoy Das *
216*4a6bb698SNirmoy Das * Only perform this when GGTT is mapped as WC, see ggtt_probe_common().
2172c86e55dSMatthew Auld */
218*4a6bb698SNirmoy Das if (needs_wc_ggtt_mapping(ggtt->vm.i915))
219*4a6bb698SNirmoy Das intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6,
220*4a6bb698SNirmoy Das GFX_FLSH_CNTL_EN);
2212c86e55dSMatthew Auld }
2222c86e55dSMatthew Auld
guc_ggtt_invalidate(struct i915_ggtt * ggtt)2232c86e55dSMatthew Auld static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
2242c86e55dSMatthew Auld {
2252c86e55dSMatthew Auld struct drm_i915_private *i915 = ggtt->vm.i915;
2262c86e55dSMatthew Auld
2272c86e55dSMatthew Auld gen8_ggtt_invalidate(ggtt);
2282c86e55dSMatthew Auld
2290f857158SAravind Iddamsetty if (GRAPHICS_VER(i915) >= 12) {
2300f857158SAravind Iddamsetty struct intel_gt *gt;
2310f857158SAravind Iddamsetty
2320f857158SAravind Iddamsetty list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
2330f857158SAravind Iddamsetty intel_uncore_write_fw(gt->uncore,
2340f857158SAravind Iddamsetty GEN12_GUC_TLB_INV_CR,
2352c86e55dSMatthew Auld GEN12_GUC_TLB_INV_CR_INVALIDATE);
2360f857158SAravind Iddamsetty } else {
2370f857158SAravind Iddamsetty intel_uncore_write_fw(ggtt->vm.gt->uncore,
2380f857158SAravind Iddamsetty GEN8_GTCR, GEN8_GTCR_INVALIDATE);
2390f857158SAravind Iddamsetty }
2402c86e55dSMatthew Auld }
2412c86e55dSMatthew Auld
mtl_ggtt_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)242341ad0e8SFei Yang static u64 mtl_ggtt_pte_encode(dma_addr_t addr,
2439275277dSFei Yang unsigned int pat_index,
244341ad0e8SFei Yang u32 flags)
245341ad0e8SFei Yang {
246341ad0e8SFei Yang gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
247341ad0e8SFei Yang
248341ad0e8SFei Yang WARN_ON_ONCE(addr & ~GEN12_GGTT_PTE_ADDR_MASK);
249341ad0e8SFei Yang
250341ad0e8SFei Yang if (flags & PTE_LM)
251341ad0e8SFei Yang pte |= GEN12_GGTT_PTE_LM;
252341ad0e8SFei Yang
2539275277dSFei Yang if (pat_index & BIT(0))
254341ad0e8SFei Yang pte |= MTL_GGTT_PTE_PAT0;
2559275277dSFei Yang
2569275277dSFei Yang if (pat_index & BIT(1))
2579275277dSFei Yang pte |= MTL_GGTT_PTE_PAT1;
258341ad0e8SFei Yang
259341ad0e8SFei Yang return pte;
260341ad0e8SFei Yang }
261341ad0e8SFei Yang
gen8_ggtt_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)26233e7a975SVille Syrjälä u64 gen8_ggtt_pte_encode(dma_addr_t addr,
2639275277dSFei Yang unsigned int pat_index,
26469edc390SDaniele Ceraolo Spurio u32 flags)
26569edc390SDaniele Ceraolo Spurio {
2665f978167SMichael Cheng gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
267e762bdf5SMatthew Auld
268e762bdf5SMatthew Auld if (flags & PTE_LM)
269e762bdf5SMatthew Auld pte |= GEN12_GGTT_PTE_LM;
270e762bdf5SMatthew Auld
271e762bdf5SMatthew Auld return pte;
27269edc390SDaniele Ceraolo Spurio }
27369edc390SDaniele Ceraolo Spurio
gen8_set_pte(void __iomem * addr,gen8_pte_t pte)2749ce07d94SLucas De Marchi static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2759ce07d94SLucas De Marchi {
2769ce07d94SLucas De Marchi writeq(pte, addr);
2779ce07d94SLucas De Marchi }
2789ce07d94SLucas De Marchi
gen8_ggtt_insert_page(struct i915_address_space * vm,dma_addr_t addr,u64 offset,unsigned int pat_index,u32 flags)2799ce07d94SLucas De Marchi static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2809ce07d94SLucas De Marchi dma_addr_t addr,
2819ce07d94SLucas De Marchi u64 offset,
2829275277dSFei Yang unsigned int pat_index,
2839ce07d94SLucas De Marchi u32 flags)
2849ce07d94SLucas De Marchi {
2859ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2869ce07d94SLucas De Marchi gen8_pte_t __iomem *pte =
2879ce07d94SLucas De Marchi (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
2889ce07d94SLucas De Marchi
2899275277dSFei Yang gen8_set_pte(pte, ggtt->vm.pte_encode(addr, pat_index, flags));
2909ce07d94SLucas De Marchi
2919ce07d94SLucas De Marchi ggtt->invalidate(ggtt);
2929ce07d94SLucas De Marchi }
2939ce07d94SLucas De Marchi
gen8_ggtt_insert_entries(struct i915_address_space * vm,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)2949ce07d94SLucas De Marchi static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2959ce07d94SLucas De Marchi struct i915_vma_resource *vma_res,
2969275277dSFei Yang unsigned int pat_index,
2979ce07d94SLucas De Marchi u32 flags)
2989ce07d94SLucas De Marchi {
2999ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3009275277dSFei Yang const gen8_pte_t pte_encode = ggtt->vm.pte_encode(0, pat_index, flags);
3019ce07d94SLucas De Marchi gen8_pte_t __iomem *gte;
3029ce07d94SLucas De Marchi gen8_pte_t __iomem *end;
3039ce07d94SLucas De Marchi struct sgt_iter iter;
3049ce07d94SLucas De Marchi dma_addr_t addr;
3059ce07d94SLucas De Marchi
3069ce07d94SLucas De Marchi /*
3079ce07d94SLucas De Marchi * Note that we ignore PTE_READ_ONLY here. The caller must be careful
3089ce07d94SLucas De Marchi * not to allow the user to override access to a read only page.
3099ce07d94SLucas De Marchi */
3109ce07d94SLucas De Marchi
3119ce07d94SLucas De Marchi gte = (gen8_pte_t __iomem *)ggtt->gsm;
31261102251SChris Wilson gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
31361102251SChris Wilson end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;
31461102251SChris Wilson while (gte < end)
31561102251SChris Wilson gen8_set_pte(gte++, vm->scratch[0]->encode);
31661102251SChris Wilson end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
3179ce07d94SLucas De Marchi
3189ce07d94SLucas De Marchi for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
3199ce07d94SLucas De Marchi gen8_set_pte(gte++, pte_encode | addr);
3209ce07d94SLucas De Marchi GEM_BUG_ON(gte > end);
3219ce07d94SLucas De Marchi
3229ce07d94SLucas De Marchi /* Fill the allocated but "unused" space beyond the end of the buffer */
3239ce07d94SLucas De Marchi while (gte < end)
3249ce07d94SLucas De Marchi gen8_set_pte(gte++, vm->scratch[0]->encode);
3259ce07d94SLucas De Marchi
3269ce07d94SLucas De Marchi /*
3279ce07d94SLucas De Marchi * We want to flush the TLBs only after we're certain all the PTE
3289ce07d94SLucas De Marchi * updates have finished.
3299ce07d94SLucas De Marchi */
3309ce07d94SLucas De Marchi ggtt->invalidate(ggtt);
3319ce07d94SLucas De Marchi }
3329ce07d94SLucas De Marchi
gen8_ggtt_clear_range(struct i915_address_space * vm,u64 start,u64 length)333b288d740SAndrzej Hajda static void gen8_ggtt_clear_range(struct i915_address_space *vm,
334b288d740SAndrzej Hajda u64 start, u64 length)
335b288d740SAndrzej Hajda {
336b288d740SAndrzej Hajda struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
337b288d740SAndrzej Hajda unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
338b288d740SAndrzej Hajda unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
339b288d740SAndrzej Hajda const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
340b288d740SAndrzej Hajda gen8_pte_t __iomem *gtt_base =
341b288d740SAndrzej Hajda (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
342b288d740SAndrzej Hajda const int max_entries = ggtt_total_entries(ggtt) - first_entry;
343b288d740SAndrzej Hajda int i;
344b288d740SAndrzej Hajda
345b288d740SAndrzej Hajda if (WARN(num_entries > max_entries,
346b288d740SAndrzej Hajda "First entry = %d; Num entries = %d (max=%d)\n",
347b288d740SAndrzej Hajda first_entry, num_entries, max_entries))
348b288d740SAndrzej Hajda num_entries = max_entries;
349b288d740SAndrzej Hajda
350b288d740SAndrzej Hajda for (i = 0; i < num_entries; i++)
351b288d740SAndrzej Hajda gen8_set_pte(>t_base[i], scratch_pte);
352b288d740SAndrzej Hajda }
353b288d740SAndrzej Hajda
gen6_ggtt_insert_page(struct i915_address_space * vm,dma_addr_t addr,u64 offset,unsigned int pat_index,u32 flags)3549ce07d94SLucas De Marchi static void gen6_ggtt_insert_page(struct i915_address_space *vm,
3559ce07d94SLucas De Marchi dma_addr_t addr,
3569ce07d94SLucas De Marchi u64 offset,
3579275277dSFei Yang unsigned int pat_index,
3589ce07d94SLucas De Marchi u32 flags)
3599ce07d94SLucas De Marchi {
3609ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3619ce07d94SLucas De Marchi gen6_pte_t __iomem *pte =
3629ce07d94SLucas De Marchi (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
3639ce07d94SLucas De Marchi
3649275277dSFei Yang iowrite32(vm->pte_encode(addr, pat_index, flags), pte);
3659ce07d94SLucas De Marchi
3669ce07d94SLucas De Marchi ggtt->invalidate(ggtt);
3679ce07d94SLucas De Marchi }
3689ce07d94SLucas De Marchi
3699ce07d94SLucas De Marchi /*
3709ce07d94SLucas De Marchi * Binds an object into the global gtt with the specified cache level.
3719ce07d94SLucas De Marchi * The object will be accessible to the GPU via commands whose operands
3729ce07d94SLucas De Marchi * reference offsets within the global GTT as well as accessible by the GPU
3739ce07d94SLucas De Marchi * through the GMADR mapped BAR (i915->mm.gtt->gtt).
3749ce07d94SLucas De Marchi */
gen6_ggtt_insert_entries(struct i915_address_space * vm,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)3759ce07d94SLucas De Marchi static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
3769ce07d94SLucas De Marchi struct i915_vma_resource *vma_res,
3779275277dSFei Yang unsigned int pat_index,
3789ce07d94SLucas De Marchi u32 flags)
3799ce07d94SLucas De Marchi {
3809ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3819ce07d94SLucas De Marchi gen6_pte_t __iomem *gte;
3829ce07d94SLucas De Marchi gen6_pte_t __iomem *end;
3839ce07d94SLucas De Marchi struct sgt_iter iter;
3849ce07d94SLucas De Marchi dma_addr_t addr;
3859ce07d94SLucas De Marchi
3869ce07d94SLucas De Marchi gte = (gen6_pte_t __iomem *)ggtt->gsm;
38761102251SChris Wilson gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
3889ce07d94SLucas De Marchi
38961102251SChris Wilson end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;
39061102251SChris Wilson while (gte < end)
39161102251SChris Wilson iowrite32(vm->scratch[0]->encode, gte++);
39261102251SChris Wilson end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
3939ce07d94SLucas De Marchi for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
3949275277dSFei Yang iowrite32(vm->pte_encode(addr, pat_index, flags), gte++);
3959ce07d94SLucas De Marchi GEM_BUG_ON(gte > end);
3969ce07d94SLucas De Marchi
3979ce07d94SLucas De Marchi /* Fill the allocated but "unused" space beyond the end of the buffer */
3989ce07d94SLucas De Marchi while (gte < end)
3999ce07d94SLucas De Marchi iowrite32(vm->scratch[0]->encode, gte++);
4009ce07d94SLucas De Marchi
4019ce07d94SLucas De Marchi /*
4029ce07d94SLucas De Marchi * We want to flush the TLBs only after we're certain all the PTE
4039ce07d94SLucas De Marchi * updates have finished.
4049ce07d94SLucas De Marchi */
4059ce07d94SLucas De Marchi ggtt->invalidate(ggtt);
4069ce07d94SLucas De Marchi }
4079ce07d94SLucas De Marchi
nop_clear_range(struct i915_address_space * vm,u64 start,u64 length)4089ce07d94SLucas De Marchi static void nop_clear_range(struct i915_address_space *vm,
4099ce07d94SLucas De Marchi u64 start, u64 length)
4109ce07d94SLucas De Marchi {
4119ce07d94SLucas De Marchi }
4129ce07d94SLucas De Marchi
bxt_vtd_ggtt_wa(struct i915_address_space * vm)4139ce07d94SLucas De Marchi static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
4149ce07d94SLucas De Marchi {
4159ce07d94SLucas De Marchi /*
4169ce07d94SLucas De Marchi * Make sure the internal GAM fifo has been cleared of all GTT
4179ce07d94SLucas De Marchi * writes before exiting stop_machine(). This guarantees that
4189ce07d94SLucas De Marchi * any aperture accesses waiting to start in another process
4199ce07d94SLucas De Marchi * cannot back up behind the GTT writes causing a hang.
4209ce07d94SLucas De Marchi * The register can be any arbitrary GAM register.
4219ce07d94SLucas De Marchi */
4229ce07d94SLucas De Marchi intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6);
4239ce07d94SLucas De Marchi }
4249ce07d94SLucas De Marchi
4259ce07d94SLucas De Marchi struct insert_page {
4269ce07d94SLucas De Marchi struct i915_address_space *vm;
4279ce07d94SLucas De Marchi dma_addr_t addr;
4289ce07d94SLucas De Marchi u64 offset;
4299275277dSFei Yang unsigned int pat_index;
4309ce07d94SLucas De Marchi };
4319ce07d94SLucas De Marchi
bxt_vtd_ggtt_insert_page__cb(void * _arg)4329ce07d94SLucas De Marchi static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
4339ce07d94SLucas De Marchi {
4349ce07d94SLucas De Marchi struct insert_page *arg = _arg;
4359ce07d94SLucas De Marchi
4369275277dSFei Yang gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset,
4379275277dSFei Yang arg->pat_index, 0);
4389ce07d94SLucas De Marchi bxt_vtd_ggtt_wa(arg->vm);
4399ce07d94SLucas De Marchi
4409ce07d94SLucas De Marchi return 0;
4419ce07d94SLucas De Marchi }
4429ce07d94SLucas De Marchi
bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space * vm,dma_addr_t addr,u64 offset,unsigned int pat_index,u32 unused)4439ce07d94SLucas De Marchi static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
4449ce07d94SLucas De Marchi dma_addr_t addr,
4459ce07d94SLucas De Marchi u64 offset,
4469275277dSFei Yang unsigned int pat_index,
4479ce07d94SLucas De Marchi u32 unused)
4489ce07d94SLucas De Marchi {
4499275277dSFei Yang struct insert_page arg = { vm, addr, offset, pat_index };
4509ce07d94SLucas De Marchi
4519ce07d94SLucas De Marchi stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
4529ce07d94SLucas De Marchi }
4539ce07d94SLucas De Marchi
4549ce07d94SLucas De Marchi struct insert_entries {
4559ce07d94SLucas De Marchi struct i915_address_space *vm;
4569ce07d94SLucas De Marchi struct i915_vma_resource *vma_res;
4579275277dSFei Yang unsigned int pat_index;
4589ce07d94SLucas De Marchi u32 flags;
4599ce07d94SLucas De Marchi };
4609ce07d94SLucas De Marchi
bxt_vtd_ggtt_insert_entries__cb(void * _arg)4619ce07d94SLucas De Marchi static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
4629ce07d94SLucas De Marchi {
4639ce07d94SLucas De Marchi struct insert_entries *arg = _arg;
4649ce07d94SLucas De Marchi
4659275277dSFei Yang gen8_ggtt_insert_entries(arg->vm, arg->vma_res,
4669275277dSFei Yang arg->pat_index, arg->flags);
4679ce07d94SLucas De Marchi bxt_vtd_ggtt_wa(arg->vm);
4689ce07d94SLucas De Marchi
4699ce07d94SLucas De Marchi return 0;
4709ce07d94SLucas De Marchi }
4719ce07d94SLucas De Marchi
bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space * vm,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)4729ce07d94SLucas De Marchi static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
4739ce07d94SLucas De Marchi struct i915_vma_resource *vma_res,
4749275277dSFei Yang unsigned int pat_index,
4759ce07d94SLucas De Marchi u32 flags)
4769ce07d94SLucas De Marchi {
4779275277dSFei Yang struct insert_entries arg = { vm, vma_res, pat_index, flags };
4789ce07d94SLucas De Marchi
4799ce07d94SLucas De Marchi stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
4809ce07d94SLucas De Marchi }
4819ce07d94SLucas De Marchi
gen6_ggtt_clear_range(struct i915_address_space * vm,u64 start,u64 length)4829ce07d94SLucas De Marchi static void gen6_ggtt_clear_range(struct i915_address_space *vm,
4839ce07d94SLucas De Marchi u64 start, u64 length)
4849ce07d94SLucas De Marchi {
4859ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
4869ce07d94SLucas De Marchi unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
4879ce07d94SLucas De Marchi unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
4889ce07d94SLucas De Marchi gen6_pte_t scratch_pte, __iomem *gtt_base =
4899ce07d94SLucas De Marchi (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
4909ce07d94SLucas De Marchi const int max_entries = ggtt_total_entries(ggtt) - first_entry;
4919ce07d94SLucas De Marchi int i;
4929ce07d94SLucas De Marchi
4939ce07d94SLucas De Marchi if (WARN(num_entries > max_entries,
4949ce07d94SLucas De Marchi "First entry = %d; Num entries = %d (max=%d)\n",
4959ce07d94SLucas De Marchi first_entry, num_entries, max_entries))
4969ce07d94SLucas De Marchi num_entries = max_entries;
4979ce07d94SLucas De Marchi
4989ce07d94SLucas De Marchi scratch_pte = vm->scratch[0]->encode;
4999ce07d94SLucas De Marchi for (i = 0; i < num_entries; i++)
5009ce07d94SLucas De Marchi iowrite32(scratch_pte, >t_base[i]);
5019ce07d94SLucas De Marchi }
5029ce07d94SLucas De Marchi
intel_ggtt_bind_vma(struct i915_address_space * vm,struct i915_vm_pt_stash * stash,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)5037a5c9223SCasey Bowman void intel_ggtt_bind_vma(struct i915_address_space *vm,
504cd0452aaSChris Wilson struct i915_vm_pt_stash *stash,
50539a2bd34SThomas Hellström struct i915_vma_resource *vma_res,
5069275277dSFei Yang unsigned int pat_index,
5072c86e55dSMatthew Auld u32 flags)
5082c86e55dSMatthew Auld {
5092c86e55dSMatthew Auld u32 pte_flags;
5102c86e55dSMatthew Auld
51139a2bd34SThomas Hellström if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK))
512cd0452aaSChris Wilson return;
513bf0840cdSChris Wilson
51439a2bd34SThomas Hellström vma_res->bound_flags |= flags;
51539a2bd34SThomas Hellström
5162c86e55dSMatthew Auld /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
5172c86e55dSMatthew Auld pte_flags = 0;
51839a2bd34SThomas Hellström if (vma_res->bi.readonly)
5192c86e55dSMatthew Auld pte_flags |= PTE_READ_ONLY;
52039a2bd34SThomas Hellström if (vma_res->bi.lmem)
521e762bdf5SMatthew Auld pte_flags |= PTE_LM;
5222c86e55dSMatthew Auld
5239275277dSFei Yang vm->insert_entries(vm, vma_res, pat_index, pte_flags);
52439a2bd34SThomas Hellström vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
5252c86e55dSMatthew Auld }
5262c86e55dSMatthew Auld
intel_ggtt_unbind_vma(struct i915_address_space * vm,struct i915_vma_resource * vma_res)5277a5c9223SCasey Bowman void intel_ggtt_unbind_vma(struct i915_address_space *vm,
52839a2bd34SThomas Hellström struct i915_vma_resource *vma_res)
5292c86e55dSMatthew Auld {
53039a2bd34SThomas Hellström vm->clear_range(vm, vma_res->start, vma_res->vma_size);
5312c86e55dSMatthew Auld }
5322c86e55dSMatthew Auld
533b7599d24SJavier Pello /*
534b7599d24SJavier Pello * Reserve the top of the GuC address space for firmware images. Addresses
535b7599d24SJavier Pello * beyond GUC_GGTT_TOP in the GuC address space are inaccessible by GuC,
536b7599d24SJavier Pello * which makes for a suitable range to hold GuC/HuC firmware images if the
537b7599d24SJavier Pello * size of the GGTT is 4G. However, on a 32-bit platform the size of the GGTT
538b7599d24SJavier Pello * is limited to 2G, which is less than GUC_GGTT_TOP, but we reserve a chunk
539b7599d24SJavier Pello * of the same size anyway, which is far more than needed, to keep the logic
540b7599d24SJavier Pello * in uc_fw_ggtt_offset() simple.
541b7599d24SJavier Pello */
542b7599d24SJavier Pello #define GUC_TOP_RESERVE_SIZE (SZ_4G - GUC_GGTT_TOP)
543b7599d24SJavier Pello
ggtt_reserve_guc_top(struct i915_ggtt * ggtt)5442c86e55dSMatthew Auld static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
5452c86e55dSMatthew Auld {
546b7599d24SJavier Pello u64 offset;
5472c86e55dSMatthew Auld int ret;
5482c86e55dSMatthew Auld
54934bbfde6SDaniele Ceraolo Spurio if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
5502c86e55dSMatthew Auld return 0;
5512c86e55dSMatthew Auld
552b7599d24SJavier Pello GEM_BUG_ON(ggtt->vm.total <= GUC_TOP_RESERVE_SIZE);
553b7599d24SJavier Pello offset = ggtt->vm.total - GUC_TOP_RESERVE_SIZE;
5542c86e55dSMatthew Auld
555b7599d24SJavier Pello ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw,
556b7599d24SJavier Pello GUC_TOP_RESERVE_SIZE, offset,
557b7599d24SJavier Pello I915_COLOR_UNEVICTABLE, PIN_NOEVICT);
5582c86e55dSMatthew Auld if (ret)
55952ce7074SWambui Karuga drm_dbg(&ggtt->vm.i915->drm,
56052ce7074SWambui Karuga "Failed to reserve top of GGTT for GuC\n");
5612c86e55dSMatthew Auld
5622c86e55dSMatthew Auld return ret;
5632c86e55dSMatthew Auld }
5642c86e55dSMatthew Auld
ggtt_release_guc_top(struct i915_ggtt * ggtt)5652c86e55dSMatthew Auld static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
5662c86e55dSMatthew Auld {
5672c86e55dSMatthew Auld if (drm_mm_node_allocated(&ggtt->uc_fw))
5682c86e55dSMatthew Auld drm_mm_remove_node(&ggtt->uc_fw);
5692c86e55dSMatthew Auld }
5702c86e55dSMatthew Auld
cleanup_init_ggtt(struct i915_ggtt * ggtt)5712c86e55dSMatthew Auld static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
5722c86e55dSMatthew Auld {
5732c86e55dSMatthew Auld ggtt_release_guc_top(ggtt);
5742c86e55dSMatthew Auld if (drm_mm_node_allocated(&ggtt->error_capture))
5752c86e55dSMatthew Auld drm_mm_remove_node(&ggtt->error_capture);
576742379c0SChris Wilson mutex_destroy(&ggtt->error_mutex);
5772c86e55dSMatthew Auld }
5782c86e55dSMatthew Auld
init_ggtt(struct i915_ggtt * ggtt)5792c86e55dSMatthew Auld static int init_ggtt(struct i915_ggtt *ggtt)
5802c86e55dSMatthew Auld {
5812c86e55dSMatthew Auld /*
5822c86e55dSMatthew Auld * Let GEM Manage all of the aperture.
5832c86e55dSMatthew Auld *
5842c86e55dSMatthew Auld * However, leave one page at the end still bound to the scratch page.
5852c86e55dSMatthew Auld * There are a number of places where the hardware apparently prefetches
5862c86e55dSMatthew Auld * past the end of the object, and we've seen multiple hangs with the
5872c86e55dSMatthew Auld * GPU head pointer stuck in a batchbuffer bound at the last page of the
5882c86e55dSMatthew Auld * aperture. One page should be enough to keep any prefetching inside
5892c86e55dSMatthew Auld * of the aperture.
5902c86e55dSMatthew Auld */
5912c86e55dSMatthew Auld unsigned long hole_start, hole_end;
5922c86e55dSMatthew Auld struct drm_mm_node *entry;
5932c86e55dSMatthew Auld int ret;
5942c86e55dSMatthew Auld
5952c86e55dSMatthew Auld /*
5962c86e55dSMatthew Auld * GuC requires all resources that we're sharing with it to be placed in
5972c86e55dSMatthew Auld * non-WOPCM memory. If GuC is not present or not in use we still need a
5982c86e55dSMatthew Auld * small bias as ring wraparound at offset 0 sometimes hangs. No idea
5992c86e55dSMatthew Auld * why.
6002c86e55dSMatthew Auld */
6012c86e55dSMatthew Auld ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
602ee71434eSAravind Iddamsetty intel_wopcm_guc_size(&ggtt->vm.gt->wopcm));
6032c86e55dSMatthew Auld
6042c86e55dSMatthew Auld ret = intel_vgt_balloon(ggtt);
6052c86e55dSMatthew Auld if (ret)
6062c86e55dSMatthew Auld return ret;
6072c86e55dSMatthew Auld
608742379c0SChris Wilson mutex_init(&ggtt->error_mutex);
6092c86e55dSMatthew Auld if (ggtt->mappable_end) {
610489140b5SChris Wilson /*
611489140b5SChris Wilson * Reserve a mappable slot for our lockless error capture.
612489140b5SChris Wilson *
613489140b5SChris Wilson * We strongly prefer taking address 0x0 in order to protect
614489140b5SChris Wilson * other critical buffers against accidental overwrites,
615489140b5SChris Wilson * as writing to address 0 is a very common mistake.
616489140b5SChris Wilson *
617489140b5SChris Wilson * Since 0 may already be in use by the system (e.g. the BIOS
618489140b5SChris Wilson * framebuffer), we let the reservation fail quietly and hope
619489140b5SChris Wilson * 0 remains reserved always.
620489140b5SChris Wilson *
621489140b5SChris Wilson * If we fail to reserve 0, and then fail to find any space
622489140b5SChris Wilson * for an error-capture, remain silent. We can afford not
623489140b5SChris Wilson * to reserve an error_capture node as we have fallback
624489140b5SChris Wilson * paths, and we trust that 0 will remain reserved. However,
625489140b5SChris Wilson * the only likely reason for failure to insert is a driver
626489140b5SChris Wilson * bug, which we expect to cause other failures...
62772f6107dSAndrzej Hajda *
62872f6107dSAndrzej Hajda * Since CPU can perform speculative reads on error capture
62972f6107dSAndrzej Hajda * (write-combining allows it) add scratch page after error
63072f6107dSAndrzej Hajda * capture to avoid DMAR errors.
631489140b5SChris Wilson */
63272f6107dSAndrzej Hajda ggtt->error_capture.size = 2 * I915_GTT_PAGE_SIZE;
633489140b5SChris Wilson ggtt->error_capture.color = I915_COLOR_UNEVICTABLE;
634489140b5SChris Wilson if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))
635489140b5SChris Wilson drm_mm_insert_node_in_range(&ggtt->vm.mm,
6362c86e55dSMatthew Auld &ggtt->error_capture,
637489140b5SChris Wilson ggtt->error_capture.size, 0,
638489140b5SChris Wilson ggtt->error_capture.color,
6392c86e55dSMatthew Auld 0, ggtt->mappable_end,
6402c86e55dSMatthew Auld DRM_MM_INSERT_LOW);
6412c86e55dSMatthew Auld }
64272f6107dSAndrzej Hajda if (drm_mm_node_allocated(&ggtt->error_capture)) {
64372f6107dSAndrzej Hajda u64 start = ggtt->error_capture.start;
64472f6107dSAndrzej Hajda u64 size = ggtt->error_capture.size;
64572f6107dSAndrzej Hajda
64672f6107dSAndrzej Hajda ggtt->vm.scratch_range(&ggtt->vm, start, size);
647489140b5SChris Wilson drm_dbg(&ggtt->vm.i915->drm,
648489140b5SChris Wilson "Reserved GGTT:[%llx, %llx] for use by error capture\n",
64972f6107dSAndrzej Hajda start, start + size);
65072f6107dSAndrzej Hajda }
6512c86e55dSMatthew Auld
6522c86e55dSMatthew Auld /*
6532c86e55dSMatthew Auld * The upper portion of the GuC address space has a sizeable hole
6542c86e55dSMatthew Auld * (several MB) that is inaccessible by GuC. Reserve this range within
6552c86e55dSMatthew Auld * GGTT as it can comfortably hold GuC/HuC firmware images.
6562c86e55dSMatthew Auld */
6572c86e55dSMatthew Auld ret = ggtt_reserve_guc_top(ggtt);
6582c86e55dSMatthew Auld if (ret)
6592c86e55dSMatthew Auld goto err;
6602c86e55dSMatthew Auld
6612c86e55dSMatthew Auld /* Clear any non-preallocated blocks */
6622c86e55dSMatthew Auld drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
663489140b5SChris Wilson drm_dbg(&ggtt->vm.i915->drm,
66452ce7074SWambui Karuga "clearing unused GTT space: [%lx, %lx]\n",
6652c86e55dSMatthew Auld hole_start, hole_end);
6662c86e55dSMatthew Auld ggtt->vm.clear_range(&ggtt->vm, hole_start,
6672c86e55dSMatthew Auld hole_end - hole_start);
6682c86e55dSMatthew Auld }
6692c86e55dSMatthew Auld
6702c86e55dSMatthew Auld /* And finally clear the reserved guard page */
6712c86e55dSMatthew Auld ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
6722c86e55dSMatthew Auld
6732c86e55dSMatthew Auld return 0;
6742c86e55dSMatthew Auld
6752c86e55dSMatthew Auld err:
6762c86e55dSMatthew Auld cleanup_init_ggtt(ggtt);
6772c86e55dSMatthew Auld return ret;
6782c86e55dSMatthew Auld }
6792c86e55dSMatthew Auld
aliasing_gtt_bind_vma(struct i915_address_space * vm,struct i915_vm_pt_stash * stash,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)680cd0452aaSChris Wilson static void aliasing_gtt_bind_vma(struct i915_address_space *vm,
681cd0452aaSChris Wilson struct i915_vm_pt_stash *stash,
68239a2bd34SThomas Hellström struct i915_vma_resource *vma_res,
6839275277dSFei Yang unsigned int pat_index,
6842c86e55dSMatthew Auld u32 flags)
6852c86e55dSMatthew Auld {
6862c86e55dSMatthew Auld u32 pte_flags;
6872c86e55dSMatthew Auld
6882c86e55dSMatthew Auld /* Currently applicable only to VLV */
6892c86e55dSMatthew Auld pte_flags = 0;
69039a2bd34SThomas Hellström if (vma_res->bi.readonly)
6912c86e55dSMatthew Auld pte_flags |= PTE_READ_ONLY;
6922c86e55dSMatthew Auld
693cd0452aaSChris Wilson if (flags & I915_VMA_LOCAL_BIND)
694cd0452aaSChris Wilson ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,
6959275277dSFei Yang stash, vma_res, pat_index, flags);
6962c86e55dSMatthew Auld
697c0e60347SChris Wilson if (flags & I915_VMA_GLOBAL_BIND)
6989275277dSFei Yang vm->insert_entries(vm, vma_res, pat_index, pte_flags);
69939a2bd34SThomas Hellström
70039a2bd34SThomas Hellström vma_res->bound_flags |= flags;
7012c86e55dSMatthew Auld }
7022c86e55dSMatthew Auld
aliasing_gtt_unbind_vma(struct i915_address_space * vm,struct i915_vma_resource * vma_res)70312b07256SChris Wilson static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,
70439a2bd34SThomas Hellström struct i915_vma_resource *vma_res)
7052c86e55dSMatthew Auld {
70639a2bd34SThomas Hellström if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND)
70739a2bd34SThomas Hellström vm->clear_range(vm, vma_res->start, vma_res->vma_size);
7082c86e55dSMatthew Auld
70939a2bd34SThomas Hellström if (vma_res->bound_flags & I915_VMA_LOCAL_BIND)
71039a2bd34SThomas Hellström ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res);
7112c86e55dSMatthew Auld }
7122c86e55dSMatthew Auld
init_aliasing_ppgtt(struct i915_ggtt * ggtt)7132c86e55dSMatthew Auld static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
7142c86e55dSMatthew Auld {
715cd0452aaSChris Wilson struct i915_vm_pt_stash stash = {};
7162c86e55dSMatthew Auld struct i915_ppgtt *ppgtt;
7172c86e55dSMatthew Auld int err;
7182c86e55dSMatthew Auld
719a259cc14SThomas Hellström ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0);
7202c86e55dSMatthew Auld if (IS_ERR(ppgtt))
7212c86e55dSMatthew Auld return PTR_ERR(ppgtt);
7222c86e55dSMatthew Auld
7232c86e55dSMatthew Auld if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
7242c86e55dSMatthew Auld err = -ENODEV;
7252c86e55dSMatthew Auld goto err_ppgtt;
7262c86e55dSMatthew Auld }
7272c86e55dSMatthew Auld
728cd0452aaSChris Wilson err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);
729cd0452aaSChris Wilson if (err)
730cd0452aaSChris Wilson goto err_ppgtt;
731cd0452aaSChris Wilson
73226ad4f8bSMaarten Lankhorst i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);
733529b9ec8SMatthew Auld err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
73426ad4f8bSMaarten Lankhorst i915_gem_object_unlock(ppgtt->vm.scratch[0]);
73589351925SChris Wilson if (err)
73689351925SChris Wilson goto err_stash;
73789351925SChris Wilson
7382c86e55dSMatthew Auld /*
7392c86e55dSMatthew Auld * Note we only pre-allocate as far as the end of the global
7402c86e55dSMatthew Auld * GTT. On 48b / 4-level page-tables, the difference is very,
7412c86e55dSMatthew Auld * very significant! We have to preallocate as GVT/vgpu does
7422c86e55dSMatthew Auld * not like the page directory disappearing.
7432c86e55dSMatthew Auld */
744cd0452aaSChris Wilson ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);
7452c86e55dSMatthew Auld
7462c86e55dSMatthew Auld ggtt->alias = ppgtt;
7472c86e55dSMatthew Auld ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
7482c86e55dSMatthew Auld
7497a5c9223SCasey Bowman GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma);
7502c86e55dSMatthew Auld ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
7512c86e55dSMatthew Auld
7527a5c9223SCasey Bowman GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma);
7532c86e55dSMatthew Auld ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
7542c86e55dSMatthew Auld
755cd0452aaSChris Wilson i915_vm_free_pt_stash(&ppgtt->vm, &stash);
7562c86e55dSMatthew Auld return 0;
7572c86e55dSMatthew Auld
75889351925SChris Wilson err_stash:
75989351925SChris Wilson i915_vm_free_pt_stash(&ppgtt->vm, &stash);
7602c86e55dSMatthew Auld err_ppgtt:
7612c86e55dSMatthew Auld i915_vm_put(&ppgtt->vm);
7622c86e55dSMatthew Auld return err;
7632c86e55dSMatthew Auld }
7642c86e55dSMatthew Auld
fini_aliasing_ppgtt(struct i915_ggtt * ggtt)7652c86e55dSMatthew Auld static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
7662c86e55dSMatthew Auld {
7672c86e55dSMatthew Auld struct i915_ppgtt *ppgtt;
7682c86e55dSMatthew Auld
7692c86e55dSMatthew Auld ppgtt = fetch_and_zero(&ggtt->alias);
7702c86e55dSMatthew Auld if (!ppgtt)
7712c86e55dSMatthew Auld return;
7722c86e55dSMatthew Auld
7732c86e55dSMatthew Auld i915_vm_put(&ppgtt->vm);
7742c86e55dSMatthew Auld
7757a5c9223SCasey Bowman ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;
7767a5c9223SCasey Bowman ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
7772c86e55dSMatthew Auld }
7782c86e55dSMatthew Auld
i915_init_ggtt(struct drm_i915_private * i915)7792c86e55dSMatthew Auld int i915_init_ggtt(struct drm_i915_private *i915)
7802c86e55dSMatthew Auld {
7812c86e55dSMatthew Auld int ret;
7822c86e55dSMatthew Auld
783848915c3SMichał Winiarski ret = init_ggtt(to_gt(i915)->ggtt);
7842c86e55dSMatthew Auld if (ret)
7852c86e55dSMatthew Auld return ret;
7862c86e55dSMatthew Auld
7872c86e55dSMatthew Auld if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
788848915c3SMichał Winiarski ret = init_aliasing_ppgtt(to_gt(i915)->ggtt);
7892c86e55dSMatthew Auld if (ret)
790848915c3SMichał Winiarski cleanup_init_ggtt(to_gt(i915)->ggtt);
7912c86e55dSMatthew Auld }
7922c86e55dSMatthew Auld
7932c86e55dSMatthew Auld return 0;
7942c86e55dSMatthew Auld }
7952c86e55dSMatthew Auld
ggtt_cleanup_hw(struct i915_ggtt * ggtt)7962c86e55dSMatthew Auld static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
7972c86e55dSMatthew Auld {
7982c86e55dSMatthew Auld struct i915_vma *vma, *vn;
7992c86e55dSMatthew Auld
8002c86e55dSMatthew Auld flush_workqueue(ggtt->vm.i915->wq);
8010f341974SMaarten Lankhorst i915_gem_drain_freed_objects(ggtt->vm.i915);
8022c86e55dSMatthew Auld
8032c86e55dSMatthew Auld mutex_lock(&ggtt->vm.mutex);
8042c86e55dSMatthew Auld
805e1a7ab4fSThomas Hellström ggtt->vm.skip_pte_rewrite = true;
806e1a7ab4fSThomas Hellström
8070f341974SMaarten Lankhorst list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
8080f341974SMaarten Lankhorst struct drm_i915_gem_object *obj = vma->obj;
8090f341974SMaarten Lankhorst bool trylock;
8100f341974SMaarten Lankhorst
8110f341974SMaarten Lankhorst trylock = i915_gem_object_trylock(obj, NULL);
8120f341974SMaarten Lankhorst WARN_ON(!trylock);
8130f341974SMaarten Lankhorst
8142c86e55dSMatthew Auld WARN_ON(__i915_vma_unbind(vma));
8150f341974SMaarten Lankhorst if (trylock)
8160f341974SMaarten Lankhorst i915_gem_object_unlock(obj);
8170f341974SMaarten Lankhorst }
8182c86e55dSMatthew Auld
8192c86e55dSMatthew Auld if (drm_mm_node_allocated(&ggtt->error_capture))
8202c86e55dSMatthew Auld drm_mm_remove_node(&ggtt->error_capture);
821742379c0SChris Wilson mutex_destroy(&ggtt->error_mutex);
8222c86e55dSMatthew Auld
8232c86e55dSMatthew Auld ggtt_release_guc_top(ggtt);
8242c86e55dSMatthew Auld intel_vgt_deballoon(ggtt);
8252c86e55dSMatthew Auld
8262c86e55dSMatthew Auld ggtt->vm.cleanup(&ggtt->vm);
8272c86e55dSMatthew Auld
8282c86e55dSMatthew Auld mutex_unlock(&ggtt->vm.mutex);
8292c86e55dSMatthew Auld i915_address_space_fini(&ggtt->vm);
8302c86e55dSMatthew Auld
8312c86e55dSMatthew Auld arch_phys_wc_del(ggtt->mtrr);
8322c86e55dSMatthew Auld
8332c86e55dSMatthew Auld if (ggtt->iomap.size)
8342c86e55dSMatthew Auld io_mapping_fini(&ggtt->iomap);
8352c86e55dSMatthew Auld }
8362c86e55dSMatthew Auld
8372c86e55dSMatthew Auld /**
8382c86e55dSMatthew Auld * i915_ggtt_driver_release - Clean up GGTT hardware initialization
8392c86e55dSMatthew Auld * @i915: i915 device
8402c86e55dSMatthew Auld */
i915_ggtt_driver_release(struct drm_i915_private * i915)8412c86e55dSMatthew Auld void i915_ggtt_driver_release(struct drm_i915_private *i915)
8422c86e55dSMatthew Auld {
843848915c3SMichał Winiarski struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
8442c86e55dSMatthew Auld
8450b6bc81dSChris Wilson fini_aliasing_ppgtt(ggtt);
8462c86e55dSMatthew Auld
8470b6bc81dSChris Wilson intel_ggtt_fini_fences(ggtt);
8480b6bc81dSChris Wilson ggtt_cleanup_hw(ggtt);
8492c86e55dSMatthew Auld }
8502c86e55dSMatthew Auld
8514d8151aeSThomas Hellström /**
8524d8151aeSThomas Hellström * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after
8534d8151aeSThomas Hellström * all free objects have been drained.
8544d8151aeSThomas Hellström * @i915: i915 device
8554d8151aeSThomas Hellström */
i915_ggtt_driver_late_release(struct drm_i915_private * i915)8564d8151aeSThomas Hellström void i915_ggtt_driver_late_release(struct drm_i915_private *i915)
8574d8151aeSThomas Hellström {
858848915c3SMichał Winiarski struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
8594d8151aeSThomas Hellström
8604d8151aeSThomas Hellström GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1);
8614d8151aeSThomas Hellström dma_resv_fini(&ggtt->vm._resv);
8624d8151aeSThomas Hellström }
8634d8151aeSThomas Hellström
gen6_get_total_gtt_size(u16 snb_gmch_ctl)8649ce07d94SLucas De Marchi static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
8659ce07d94SLucas De Marchi {
8669ce07d94SLucas De Marchi snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
8679ce07d94SLucas De Marchi snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
8689ce07d94SLucas De Marchi return snb_gmch_ctl << 20;
8699ce07d94SLucas De Marchi }
8709ce07d94SLucas De Marchi
gen8_get_total_gtt_size(u16 bdw_gmch_ctl)8719ce07d94SLucas De Marchi static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
8729ce07d94SLucas De Marchi {
8739ce07d94SLucas De Marchi bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
8749ce07d94SLucas De Marchi bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
8759ce07d94SLucas De Marchi if (bdw_gmch_ctl)
8769ce07d94SLucas De Marchi bdw_gmch_ctl = 1 << bdw_gmch_ctl;
8779ce07d94SLucas De Marchi
8789ce07d94SLucas De Marchi #ifdef CONFIG_X86_32
8799ce07d94SLucas De Marchi /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
8809ce07d94SLucas De Marchi if (bdw_gmch_ctl > 4)
8819ce07d94SLucas De Marchi bdw_gmch_ctl = 4;
8829ce07d94SLucas De Marchi #endif
8839ce07d94SLucas De Marchi
8849ce07d94SLucas De Marchi return bdw_gmch_ctl << 20;
8859ce07d94SLucas De Marchi }
8869ce07d94SLucas De Marchi
chv_get_total_gtt_size(u16 gmch_ctrl)8879ce07d94SLucas De Marchi static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
8889ce07d94SLucas De Marchi {
8899ce07d94SLucas De Marchi gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
8909ce07d94SLucas De Marchi gmch_ctrl &= SNB_GMCH_GGMS_MASK;
8919ce07d94SLucas De Marchi
8929ce07d94SLucas De Marchi if (gmch_ctrl)
8939ce07d94SLucas De Marchi return 1 << (20 + gmch_ctrl);
8949ce07d94SLucas De Marchi
8959ce07d94SLucas De Marchi return 0;
8969ce07d94SLucas De Marchi }
8979ce07d94SLucas De Marchi
gen6_gttmmadr_size(struct drm_i915_private * i915)8989ce07d94SLucas De Marchi static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915)
8999ce07d94SLucas De Marchi {
9009ce07d94SLucas De Marchi /*
9019ce07d94SLucas De Marchi * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset
9029ce07d94SLucas De Marchi * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset
9039ce07d94SLucas De Marchi */
9049ce07d94SLucas De Marchi GEM_BUG_ON(GRAPHICS_VER(i915) < 6);
9059ce07d94SLucas De Marchi return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M;
9069ce07d94SLucas De Marchi }
9079ce07d94SLucas De Marchi
gen6_gttadr_offset(struct drm_i915_private * i915)9089ce07d94SLucas De Marchi static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915)
9099ce07d94SLucas De Marchi {
9109ce07d94SLucas De Marchi return gen6_gttmmadr_size(i915) / 2;
9119ce07d94SLucas De Marchi }
9129ce07d94SLucas De Marchi
ggtt_probe_common(struct i915_ggtt * ggtt,u64 size)9139ce07d94SLucas De Marchi static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
9149ce07d94SLucas De Marchi {
9159ce07d94SLucas De Marchi struct drm_i915_private *i915 = ggtt->vm.i915;
9169ce07d94SLucas De Marchi struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
9179ce07d94SLucas De Marchi phys_addr_t phys_addr;
9189ce07d94SLucas De Marchi u32 pte_flags;
9199ce07d94SLucas De Marchi int ret;
9209ce07d94SLucas De Marchi
9210492a34cSVille Syrjälä GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915));
9220492a34cSVille Syrjälä phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915);
9239ce07d94SLucas De Marchi
924*4a6bb698SNirmoy Das if (needs_wc_ggtt_mapping(i915))
9259ce07d94SLucas De Marchi ggtt->gsm = ioremap_wc(phys_addr, size);
926*4a6bb698SNirmoy Das else
927*4a6bb698SNirmoy Das ggtt->gsm = ioremap(phys_addr, size);
928*4a6bb698SNirmoy Das
9299ce07d94SLucas De Marchi if (!ggtt->gsm) {
9309ce07d94SLucas De Marchi drm_err(&i915->drm, "Failed to map the ggtt page table\n");
9319ce07d94SLucas De Marchi return -ENOMEM;
9329ce07d94SLucas De Marchi }
9339ce07d94SLucas De Marchi
9349ce07d94SLucas De Marchi kref_init(&ggtt->vm.resv_ref);
9359ce07d94SLucas De Marchi ret = setup_scratch_page(&ggtt->vm);
9369ce07d94SLucas De Marchi if (ret) {
9379ce07d94SLucas De Marchi drm_err(&i915->drm, "Scratch setup failed\n");
9389ce07d94SLucas De Marchi /* iounmap will also get called at remove, but meh */
9399ce07d94SLucas De Marchi iounmap(ggtt->gsm);
9409ce07d94SLucas De Marchi return ret;
9419ce07d94SLucas De Marchi }
9429ce07d94SLucas De Marchi
9439ce07d94SLucas De Marchi pte_flags = 0;
9449ce07d94SLucas De Marchi if (i915_gem_object_is_lmem(ggtt->vm.scratch[0]))
9459ce07d94SLucas De Marchi pte_flags |= PTE_LM;
9469ce07d94SLucas De Marchi
9479ce07d94SLucas De Marchi ggtt->vm.scratch[0]->encode =
9489ce07d94SLucas De Marchi ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]),
9499275277dSFei Yang i915_gem_get_pat_index(i915,
9509275277dSFei Yang I915_CACHE_NONE),
9519275277dSFei Yang pte_flags);
9529ce07d94SLucas De Marchi
9539ce07d94SLucas De Marchi return 0;
9549ce07d94SLucas De Marchi }
9559ce07d94SLucas De Marchi
gen6_gmch_remove(struct i915_address_space * vm)9569ce07d94SLucas De Marchi static void gen6_gmch_remove(struct i915_address_space *vm)
9579ce07d94SLucas De Marchi {
9589ce07d94SLucas De Marchi struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
9599ce07d94SLucas De Marchi
9609ce07d94SLucas De Marchi iounmap(ggtt->gsm);
9619ce07d94SLucas De Marchi free_scratch(vm);
9629ce07d94SLucas De Marchi }
9639ce07d94SLucas De Marchi
pci_resource(struct pci_dev * pdev,int bar)9649ce07d94SLucas De Marchi static struct resource pci_resource(struct pci_dev *pdev, int bar)
9652c86e55dSMatthew Auld {
966e5405178SJani Nikula return DEFINE_RES_MEM(pci_resource_start(pdev, bar),
9672c86e55dSMatthew Auld pci_resource_len(pdev, bar));
9682c86e55dSMatthew Auld }
9692c86e55dSMatthew Auld
gen8_gmch_probe(struct i915_ggtt * ggtt)9709ce07d94SLucas De Marchi static int gen8_gmch_probe(struct i915_ggtt *ggtt)
9719ce07d94SLucas De Marchi {
9729ce07d94SLucas De Marchi struct drm_i915_private *i915 = ggtt->vm.i915;
9739ce07d94SLucas De Marchi struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
9749ce07d94SLucas De Marchi unsigned int size;
9759ce07d94SLucas De Marchi u16 snb_gmch_ctl;
9769ce07d94SLucas De Marchi
97703eababbSVille Syrjälä if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) {
9780492a34cSVille Syrjälä if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))
9791bba7323SPiotr Piórkowski return -ENXIO;
9801bba7323SPiotr Piórkowski
9810492a34cSVille Syrjälä ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);
9829ce07d94SLucas De Marchi ggtt->mappable_end = resource_size(&ggtt->gmadr);
9839ce07d94SLucas De Marchi }
9849ce07d94SLucas De Marchi
9859ce07d94SLucas De Marchi pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
9869ce07d94SLucas De Marchi if (IS_CHERRYVIEW(i915))
9879ce07d94SLucas De Marchi size = chv_get_total_gtt_size(snb_gmch_ctl);
9889ce07d94SLucas De Marchi else
9899ce07d94SLucas De Marchi size = gen8_get_total_gtt_size(snb_gmch_ctl);
9909ce07d94SLucas De Marchi
9919ce07d94SLucas De Marchi ggtt->vm.alloc_pt_dma = alloc_pt_dma;
9929ce07d94SLucas De Marchi ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
9939ce07d94SLucas De Marchi ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY;
9949ce07d94SLucas De Marchi
9959ce07d94SLucas De Marchi ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
9969ce07d94SLucas De Marchi ggtt->vm.cleanup = gen6_gmch_remove;
9979ce07d94SLucas De Marchi ggtt->vm.insert_page = gen8_ggtt_insert_page;
9989ce07d94SLucas De Marchi ggtt->vm.clear_range = nop_clear_range;
999b288d740SAndrzej Hajda ggtt->vm.scratch_range = gen8_ggtt_clear_range;
10009ce07d94SLucas De Marchi
10019ce07d94SLucas De Marchi ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
10029ce07d94SLucas De Marchi
10039ce07d94SLucas De Marchi /*
10049ce07d94SLucas De Marchi * Serialize GTT updates with aperture access on BXT if VT-d is on,
10059ce07d94SLucas De Marchi * and always on CHV.
10069ce07d94SLucas De Marchi */
10079ce07d94SLucas De Marchi if (intel_vm_no_concurrent_access_wa(i915)) {
10089ce07d94SLucas De Marchi ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
10099ce07d94SLucas De Marchi ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL;
1010a0696856SNirmoy Das
1011a0696856SNirmoy Das /*
1012a0696856SNirmoy Das * Calling stop_machine() version of GGTT update function
1013a0696856SNirmoy Das * at error capture/reset path will raise lockdep warning.
1014a0696856SNirmoy Das * Allow calling gen8_ggtt_insert_* directly at reset path
1015a0696856SNirmoy Das * which is safe from parallel GGTT updates.
1016a0696856SNirmoy Das */
1017a0696856SNirmoy Das ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;
1018a0696856SNirmoy Das ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries;
1019a0696856SNirmoy Das
10209ce07d94SLucas De Marchi ggtt->vm.bind_async_flags =
10219ce07d94SLucas De Marchi I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
10229ce07d94SLucas De Marchi }
10239ce07d94SLucas De Marchi
1024f2053d34SDaniele Ceraolo Spurio if (intel_uc_wants_guc(&ggtt->vm.gt->uc))
1025f2053d34SDaniele Ceraolo Spurio ggtt->invalidate = guc_ggtt_invalidate;
1026f2053d34SDaniele Ceraolo Spurio else
10279ce07d94SLucas De Marchi ggtt->invalidate = gen8_ggtt_invalidate;
10289ce07d94SLucas De Marchi
10299ce07d94SLucas De Marchi ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;
10309ce07d94SLucas De Marchi ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
10319ce07d94SLucas De Marchi
1032341ad0e8SFei Yang if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1033341ad0e8SFei Yang ggtt->vm.pte_encode = mtl_ggtt_pte_encode;
1034341ad0e8SFei Yang else
10359ce07d94SLucas De Marchi ggtt->vm.pte_encode = gen8_ggtt_pte_encode;
10369ce07d94SLucas De Marchi
10379ce07d94SLucas De Marchi return ggtt_probe_common(ggtt, size);
10389ce07d94SLucas De Marchi }
10399ce07d94SLucas De Marchi
10409275277dSFei Yang /*
10419275277dSFei Yang * For pre-gen8 platforms pat_index is the same as enum i915_cache_level,
10424722e2ebSNathan Chancellor * so the switch-case statements in these PTE encode functions are still valid.
10439275277dSFei Yang * See translation table LEGACY_CACHELEVEL.
10449275277dSFei Yang */
snb_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)10459ce07d94SLucas De Marchi static u64 snb_pte_encode(dma_addr_t addr,
10464722e2ebSNathan Chancellor unsigned int pat_index,
10479ce07d94SLucas De Marchi u32 flags)
10489ce07d94SLucas De Marchi {
10499ce07d94SLucas De Marchi gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
10509ce07d94SLucas De Marchi
10514722e2ebSNathan Chancellor switch (pat_index) {
10529ce07d94SLucas De Marchi case I915_CACHE_L3_LLC:
10539ce07d94SLucas De Marchi case I915_CACHE_LLC:
10549ce07d94SLucas De Marchi pte |= GEN6_PTE_CACHE_LLC;
10559ce07d94SLucas De Marchi break;
10569ce07d94SLucas De Marchi case I915_CACHE_NONE:
10579ce07d94SLucas De Marchi pte |= GEN6_PTE_UNCACHED;
10589ce07d94SLucas De Marchi break;
10599ce07d94SLucas De Marchi default:
10604722e2ebSNathan Chancellor MISSING_CASE(pat_index);
10619ce07d94SLucas De Marchi }
10629ce07d94SLucas De Marchi
10639ce07d94SLucas De Marchi return pte;
10649ce07d94SLucas De Marchi }
10659ce07d94SLucas De Marchi
ivb_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)10669ce07d94SLucas De Marchi static u64 ivb_pte_encode(dma_addr_t addr,
10674722e2ebSNathan Chancellor unsigned int pat_index,
10689ce07d94SLucas De Marchi u32 flags)
10699ce07d94SLucas De Marchi {
10709ce07d94SLucas De Marchi gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
10719ce07d94SLucas De Marchi
10724722e2ebSNathan Chancellor switch (pat_index) {
10739ce07d94SLucas De Marchi case I915_CACHE_L3_LLC:
10749ce07d94SLucas De Marchi pte |= GEN7_PTE_CACHE_L3_LLC;
10759ce07d94SLucas De Marchi break;
10769ce07d94SLucas De Marchi case I915_CACHE_LLC:
10779ce07d94SLucas De Marchi pte |= GEN6_PTE_CACHE_LLC;
10789ce07d94SLucas De Marchi break;
10799ce07d94SLucas De Marchi case I915_CACHE_NONE:
10809ce07d94SLucas De Marchi pte |= GEN6_PTE_UNCACHED;
10819ce07d94SLucas De Marchi break;
10829ce07d94SLucas De Marchi default:
10834722e2ebSNathan Chancellor MISSING_CASE(pat_index);
10849ce07d94SLucas De Marchi }
10859ce07d94SLucas De Marchi
10869ce07d94SLucas De Marchi return pte;
10879ce07d94SLucas De Marchi }
10889ce07d94SLucas De Marchi
byt_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)10899ce07d94SLucas De Marchi static u64 byt_pte_encode(dma_addr_t addr,
10904722e2ebSNathan Chancellor unsigned int pat_index,
10919ce07d94SLucas De Marchi u32 flags)
10929ce07d94SLucas De Marchi {
10939ce07d94SLucas De Marchi gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
10949ce07d94SLucas De Marchi
10959ce07d94SLucas De Marchi if (!(flags & PTE_READ_ONLY))
10969ce07d94SLucas De Marchi pte |= BYT_PTE_WRITEABLE;
10979ce07d94SLucas De Marchi
10984722e2ebSNathan Chancellor if (pat_index != I915_CACHE_NONE)
10999ce07d94SLucas De Marchi pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
11009ce07d94SLucas De Marchi
11019ce07d94SLucas De Marchi return pte;
11029ce07d94SLucas De Marchi }
11039ce07d94SLucas De Marchi
hsw_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)11049ce07d94SLucas De Marchi static u64 hsw_pte_encode(dma_addr_t addr,
11054722e2ebSNathan Chancellor unsigned int pat_index,
11069ce07d94SLucas De Marchi u32 flags)
11079ce07d94SLucas De Marchi {
11089ce07d94SLucas De Marchi gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
11099ce07d94SLucas De Marchi
11104722e2ebSNathan Chancellor if (pat_index != I915_CACHE_NONE)
11119ce07d94SLucas De Marchi pte |= HSW_WB_LLC_AGE3;
11129ce07d94SLucas De Marchi
11139ce07d94SLucas De Marchi return pte;
11149ce07d94SLucas De Marchi }
11159ce07d94SLucas De Marchi
iris_pte_encode(dma_addr_t addr,unsigned int pat_index,u32 flags)11169ce07d94SLucas De Marchi static u64 iris_pte_encode(dma_addr_t addr,
11174722e2ebSNathan Chancellor unsigned int pat_index,
11189ce07d94SLucas De Marchi u32 flags)
11199ce07d94SLucas De Marchi {
11209ce07d94SLucas De Marchi gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
11219ce07d94SLucas De Marchi
11224722e2ebSNathan Chancellor switch (pat_index) {
11239ce07d94SLucas De Marchi case I915_CACHE_NONE:
11249ce07d94SLucas De Marchi break;
11259ce07d94SLucas De Marchi case I915_CACHE_WT:
11269ce07d94SLucas De Marchi pte |= HSW_WT_ELLC_LLC_AGE3;
11279ce07d94SLucas De Marchi break;
11289ce07d94SLucas De Marchi default:
11299ce07d94SLucas De Marchi pte |= HSW_WB_ELLC_LLC_AGE3;
11309ce07d94SLucas De Marchi break;
11319ce07d94SLucas De Marchi }
11329ce07d94SLucas De Marchi
11339ce07d94SLucas De Marchi return pte;
11349ce07d94SLucas De Marchi }
11359ce07d94SLucas De Marchi
gen6_gmch_probe(struct i915_ggtt * ggtt)11369ce07d94SLucas De Marchi static int gen6_gmch_probe(struct i915_ggtt *ggtt)
11379ce07d94SLucas De Marchi {
11389ce07d94SLucas De Marchi struct drm_i915_private *i915 = ggtt->vm.i915;
11399ce07d94SLucas De Marchi struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
11409ce07d94SLucas De Marchi unsigned int size;
11419ce07d94SLucas De Marchi u16 snb_gmch_ctl;
11429ce07d94SLucas De Marchi
11430492a34cSVille Syrjälä if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))
11441bba7323SPiotr Piórkowski return -ENXIO;
11451bba7323SPiotr Piórkowski
11460492a34cSVille Syrjälä ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);
11479ce07d94SLucas De Marchi ggtt->mappable_end = resource_size(&ggtt->gmadr);
11489ce07d94SLucas De Marchi
11499ce07d94SLucas De Marchi /*
11509ce07d94SLucas De Marchi * 64/512MB is the current min/max we actually know of, but this is
11519ce07d94SLucas De Marchi * just a coarse sanity check.
11529ce07d94SLucas De Marchi */
11539ce07d94SLucas De Marchi if (ggtt->mappable_end < (64 << 20) ||
11549ce07d94SLucas De Marchi ggtt->mappable_end > (512 << 20)) {
11559ce07d94SLucas De Marchi drm_err(&i915->drm, "Unknown GMADR size (%pa)\n",
11569ce07d94SLucas De Marchi &ggtt->mappable_end);
11579ce07d94SLucas De Marchi return -ENXIO;
11589ce07d94SLucas De Marchi }
11599ce07d94SLucas De Marchi
11609ce07d94SLucas De Marchi pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
11619ce07d94SLucas De Marchi
11629ce07d94SLucas De Marchi size = gen6_get_total_gtt_size(snb_gmch_ctl);
11639ce07d94SLucas De Marchi ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
11649ce07d94SLucas De Marchi
11659ce07d94SLucas De Marchi ggtt->vm.alloc_pt_dma = alloc_pt_dma;
11669ce07d94SLucas De Marchi ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
11679ce07d94SLucas De Marchi
11689ce07d94SLucas De Marchi ggtt->vm.clear_range = nop_clear_range;
1169eea380adSChris Wilson if (!HAS_FULL_PPGTT(i915))
11709ce07d94SLucas De Marchi ggtt->vm.clear_range = gen6_ggtt_clear_range;
1171b288d740SAndrzej Hajda ggtt->vm.scratch_range = gen6_ggtt_clear_range;
11729ce07d94SLucas De Marchi ggtt->vm.insert_page = gen6_ggtt_insert_page;
11739ce07d94SLucas De Marchi ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
11749ce07d94SLucas De Marchi ggtt->vm.cleanup = gen6_gmch_remove;
11759ce07d94SLucas De Marchi
11769ce07d94SLucas De Marchi ggtt->invalidate = gen6_ggtt_invalidate;
11779ce07d94SLucas De Marchi
11789ce07d94SLucas De Marchi if (HAS_EDRAM(i915))
11799ce07d94SLucas De Marchi ggtt->vm.pte_encode = iris_pte_encode;
11809ce07d94SLucas De Marchi else if (IS_HASWELL(i915))
11819ce07d94SLucas De Marchi ggtt->vm.pte_encode = hsw_pte_encode;
11829ce07d94SLucas De Marchi else if (IS_VALLEYVIEW(i915))
11839ce07d94SLucas De Marchi ggtt->vm.pte_encode = byt_pte_encode;
11849ce07d94SLucas De Marchi else if (GRAPHICS_VER(i915) >= 7)
11859ce07d94SLucas De Marchi ggtt->vm.pte_encode = ivb_pte_encode;
11869ce07d94SLucas De Marchi else
11879ce07d94SLucas De Marchi ggtt->vm.pte_encode = snb_pte_encode;
11889ce07d94SLucas De Marchi
11899ce07d94SLucas De Marchi ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;
11909ce07d94SLucas De Marchi ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
11919ce07d94SLucas De Marchi
11929ce07d94SLucas De Marchi return ggtt_probe_common(ggtt, size);
11939ce07d94SLucas De Marchi }
11949ce07d94SLucas De Marchi
ggtt_probe_hw(struct i915_ggtt * ggtt,struct intel_gt * gt)11952c86e55dSMatthew Auld static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
11962c86e55dSMatthew Auld {
11972c86e55dSMatthew Auld struct drm_i915_private *i915 = gt->i915;
11982c86e55dSMatthew Auld int ret;
11992c86e55dSMatthew Auld
12002c86e55dSMatthew Auld ggtt->vm.gt = gt;
12012c86e55dSMatthew Auld ggtt->vm.i915 = i915;
1202e322551fSThomas Zimmermann ggtt->vm.dma = i915->drm.dev;
12034d8151aeSThomas Hellström dma_resv_init(&ggtt->vm._resv);
12042c86e55dSMatthew Auld
12059ce07d94SLucas De Marchi if (GRAPHICS_VER(i915) >= 8)
12069ce07d94SLucas De Marchi ret = gen8_gmch_probe(ggtt);
12079ce07d94SLucas De Marchi else if (GRAPHICS_VER(i915) >= 6)
12089ce07d94SLucas De Marchi ret = gen6_gmch_probe(ggtt);
12092c86e55dSMatthew Auld else
12109ce07d94SLucas De Marchi ret = intel_ggtt_gmch_probe(ggtt);
12119ce07d94SLucas De Marchi
121226ad4f8bSMaarten Lankhorst if (ret) {
12134d8151aeSThomas Hellström dma_resv_fini(&ggtt->vm._resv);
12142c86e55dSMatthew Auld return ret;
121526ad4f8bSMaarten Lankhorst }
12162c86e55dSMatthew Auld
12172c86e55dSMatthew Auld if ((ggtt->vm.total - 1) >> 32) {
121836034c95SWambui Karuga drm_err(&i915->drm,
121936034c95SWambui Karuga "We never expected a Global GTT with more than 32bits"
12202c86e55dSMatthew Auld " of address space! Found %lldM!\n",
12212c86e55dSMatthew Auld ggtt->vm.total >> 20);
12222c86e55dSMatthew Auld ggtt->vm.total = 1ULL << 32;
12232c86e55dSMatthew Auld ggtt->mappable_end =
12242c86e55dSMatthew Auld min_t(u64, ggtt->mappable_end, ggtt->vm.total);
12252c86e55dSMatthew Auld }
12262c86e55dSMatthew Auld
12272c86e55dSMatthew Auld if (ggtt->mappable_end > ggtt->vm.total) {
122836034c95SWambui Karuga drm_err(&i915->drm,
122936034c95SWambui Karuga "mappable aperture extends past end of GGTT,"
12302c86e55dSMatthew Auld " aperture=%pa, total=%llx\n",
12312c86e55dSMatthew Auld &ggtt->mappable_end, ggtt->vm.total);
12322c86e55dSMatthew Auld ggtt->mappable_end = ggtt->vm.total;
12332c86e55dSMatthew Auld }
12342c86e55dSMatthew Auld
12352c86e55dSMatthew Auld /* GMADR is the PCI mmio aperture into the global GTT. */
123636034c95SWambui Karuga drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20);
123736034c95SWambui Karuga drm_dbg(&i915->drm, "GMADR size = %lluM\n",
123836034c95SWambui Karuga (u64)ggtt->mappable_end >> 20);
123936034c95SWambui Karuga drm_dbg(&i915->drm, "DSM size = %lluM\n",
12402c86e55dSMatthew Auld (u64)resource_size(&intel_graphics_stolen_res) >> 20);
12412c86e55dSMatthew Auld
12422c86e55dSMatthew Auld return 0;
12432c86e55dSMatthew Auld }
12442c86e55dSMatthew Auld
12452c86e55dSMatthew Auld /**
12462c86e55dSMatthew Auld * i915_ggtt_probe_hw - Probe GGTT hardware location
12472c86e55dSMatthew Auld * @i915: i915 device
12482c86e55dSMatthew Auld */
i915_ggtt_probe_hw(struct drm_i915_private * i915)12492c86e55dSMatthew Auld int i915_ggtt_probe_hw(struct drm_i915_private *i915)
12502c86e55dSMatthew Auld {
12510f857158SAravind Iddamsetty struct intel_gt *gt;
12520f857158SAravind Iddamsetty int ret, i;
12530f857158SAravind Iddamsetty
12540f857158SAravind Iddamsetty for_each_gt(gt, i915, i) {
12550f857158SAravind Iddamsetty ret = intel_gt_assign_ggtt(gt);
12560f857158SAravind Iddamsetty if (ret)
12570f857158SAravind Iddamsetty return ret;
12580f857158SAravind Iddamsetty }
12592c86e55dSMatthew Auld
1260848915c3SMichał Winiarski ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));
12612c86e55dSMatthew Auld if (ret)
12622c86e55dSMatthew Auld return ret;
12632c86e55dSMatthew Auld
1264a7f46d5bSTvrtko Ursulin if (i915_vtd_active(i915))
1265dc483ba5SJani Nikula drm_info(&i915->drm, "VT-d active for gfx access\n");
12662c86e55dSMatthew Auld
12672c86e55dSMatthew Auld return 0;
12682c86e55dSMatthew Auld }
12692c86e55dSMatthew Auld
i915_ggtt_create(struct drm_i915_private * i915)12700f857158SAravind Iddamsetty struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915)
12710f857158SAravind Iddamsetty {
12720f857158SAravind Iddamsetty struct i915_ggtt *ggtt;
12730f857158SAravind Iddamsetty
12740f857158SAravind Iddamsetty ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL);
12750f857158SAravind Iddamsetty if (!ggtt)
12760f857158SAravind Iddamsetty return ERR_PTR(-ENOMEM);
12770f857158SAravind Iddamsetty
12780f857158SAravind Iddamsetty INIT_LIST_HEAD(&ggtt->gt_list);
12790f857158SAravind Iddamsetty
12800f857158SAravind Iddamsetty return ggtt;
12810f857158SAravind Iddamsetty }
12820f857158SAravind Iddamsetty
i915_ggtt_enable_hw(struct drm_i915_private * i915)12832c86e55dSMatthew Auld int i915_ggtt_enable_hw(struct drm_i915_private *i915)
12842c86e55dSMatthew Auld {
12859ce07d94SLucas De Marchi if (GRAPHICS_VER(i915) < 6)
12869ce07d94SLucas De Marchi return intel_ggtt_gmch_enable_hw(i915);
12879ce07d94SLucas De Marchi
12889ce07d94SLucas De Marchi return 0;
12892c86e55dSMatthew Auld }
12902c86e55dSMatthew Auld
12918d2f683fSImre Deak /**
12928d2f683fSImre Deak * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM
12938d2f683fSImre Deak * @vm: The VM to restore the mappings for
12948d2f683fSImre Deak *
12958d2f683fSImre Deak * Restore the memory mappings for all objects mapped to HW via the GGTT or a
12968d2f683fSImre Deak * DPT page table.
12978d2f683fSImre Deak *
12988d2f683fSImre Deak * Returns %true if restoring the mapping for any object that was in a write
12998d2f683fSImre Deak * domain before suspend.
13008d2f683fSImre Deak */
i915_ggtt_resume_vm(struct i915_address_space * vm)13018d2f683fSImre Deak bool i915_ggtt_resume_vm(struct i915_address_space *vm)
13022c86e55dSMatthew Auld {
130380e5351dSChris Wilson struct i915_vma *vma;
13048d2f683fSImre Deak bool write_domain_objs = false;
13052c86e55dSMatthew Auld
13068d2f683fSImre Deak drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
13072c86e55dSMatthew Auld
1308de3a9ab9SAndi Shyti /* First fill our portion of the GTT with scratch pages */
13098d2f683fSImre Deak vm->clear_range(vm, 0, vm->total);
13102c86e55dSMatthew Auld
13112c86e55dSMatthew Auld /* clflush objects bound into the GGTT and rebind them. */
13128d2f683fSImre Deak list_for_each_entry(vma, &vm->bound_list, vm_link) {
13132c86e55dSMatthew Auld struct drm_i915_gem_object *obj = vma->obj;
1314cd0452aaSChris Wilson unsigned int was_bound =
1315cd0452aaSChris Wilson atomic_read(&vma->flags) & I915_VMA_BIND_MASK;
13162c86e55dSMatthew Auld
1317cd0452aaSChris Wilson GEM_BUG_ON(!was_bound);
1318de3a9ab9SAndi Shyti
1319bc247253SThomas Hellström /*
1320bc247253SThomas Hellström * Clear the bound flags of the vma resource to allow
1321bc247253SThomas Hellström * ptes to be repopulated.
1322bc247253SThomas Hellström */
1323bc247253SThomas Hellström vma->resource->bound_flags = 0;
1324647bfd26STvrtko Ursulin vma->ops->bind_vma(vm, NULL, vma->resource,
13259275277dSFei Yang obj ? obj->pat_index :
13269275277dSFei Yang i915_gem_get_pat_index(vm->i915,
13279275277dSFei Yang I915_CACHE_NONE),
1328cd0452aaSChris Wilson was_bound);
1329de3a9ab9SAndi Shyti
13302c86e55dSMatthew Auld if (obj) { /* only used during resume => exclusive access */
13318d2f683fSImre Deak write_domain_objs |= fetch_and_zero(&obj->write_domain);
13322c86e55dSMatthew Auld obj->read_domains |= I915_GEM_DOMAIN_GTT;
13332c86e55dSMatthew Auld }
13342c86e55dSMatthew Auld }
13352c86e55dSMatthew Auld
13368d2f683fSImre Deak return write_domain_objs;
13378d2f683fSImre Deak }
13388d2f683fSImre Deak
i915_ggtt_resume(struct i915_ggtt * ggtt)13398d2f683fSImre Deak void i915_ggtt_resume(struct i915_ggtt *ggtt)
13408d2f683fSImre Deak {
13410f857158SAravind Iddamsetty struct intel_gt *gt;
13428d2f683fSImre Deak bool flush;
13438d2f683fSImre Deak
13440f857158SAravind Iddamsetty list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
13450f857158SAravind Iddamsetty intel_gt_check_and_clear_faults(gt);
13468d2f683fSImre Deak
13478d2f683fSImre Deak flush = i915_ggtt_resume_vm(&ggtt->vm);
13488d2f683fSImre Deak
134972f6107dSAndrzej Hajda if (drm_mm_node_allocated(&ggtt->error_capture))
135072f6107dSAndrzej Hajda ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start,
135172f6107dSAndrzej Hajda ggtt->error_capture.size);
135272f6107dSAndrzej Hajda
13533532e75dSDaniele Ceraolo Spurio list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
13543532e75dSDaniele Ceraolo Spurio intel_uc_resume_mappings(>->uc);
13553532e75dSDaniele Ceraolo Spurio
13562c86e55dSMatthew Auld ggtt->invalidate(ggtt);
13572c86e55dSMatthew Auld
13582c86e55dSMatthew Auld if (flush)
13592c86e55dSMatthew Auld wbinvd_on_all_cpus();
13602c86e55dSMatthew Auld
1361dec9cf9eSChris Wilson intel_ggtt_restore_fences(ggtt);
13622c86e55dSMatthew Auld }
1363