10260c420SBen Widawsky /* 20260c420SBen Widawsky * Copyright © 2014 Intel Corporation 30260c420SBen Widawsky * 40260c420SBen Widawsky * Permission is hereby granted, free of charge, to any person obtaining a 50260c420SBen Widawsky * copy of this software and associated documentation files (the "Software"), 60260c420SBen Widawsky * to deal in the Software without restriction, including without limitation 70260c420SBen Widawsky * the rights to use, copy, modify, merge, publish, distribute, sublicense, 80260c420SBen Widawsky * and/or sell copies of the Software, and to permit persons to whom the 90260c420SBen Widawsky * Software is furnished to do so, subject to the following conditions: 100260c420SBen Widawsky * 110260c420SBen Widawsky * The above copyright notice and this permission notice (including the next 120260c420SBen Widawsky * paragraph) shall be included in all copies or substantial portions of the 130260c420SBen Widawsky * Software. 140260c420SBen Widawsky * 150260c420SBen Widawsky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 160260c420SBen Widawsky * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 170260c420SBen Widawsky * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 180260c420SBen Widawsky * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 190260c420SBen Widawsky * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 200260c420SBen Widawsky * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 210260c420SBen Widawsky * IN THE SOFTWARE. 220260c420SBen Widawsky * 230260c420SBen Widawsky * Please try to maintain the following order within this file unless it makes 240260c420SBen Widawsky * sense to do otherwise. From top to bottom: 250260c420SBen Widawsky * 1. typedefs 260260c420SBen Widawsky * 2. #defines, and macros 270260c420SBen Widawsky * 3. structure definitions 280260c420SBen Widawsky * 4. function prototypes 290260c420SBen Widawsky * 300260c420SBen Widawsky * Within each section, please try to order by generation in ascending order, 310260c420SBen Widawsky * from top to bottom (ie. gen6 on the top, gen8 on the bottom). 320260c420SBen Widawsky */ 330260c420SBen Widawsky 340260c420SBen Widawsky #ifndef __I915_GEM_GTT_H__ 350260c420SBen Widawsky #define __I915_GEM_GTT_H__ 360260c420SBen Widawsky 378ef8561fSChris Wilson #include <linux/io-mapping.h> 388ef8561fSChris Wilson 39b0decaf7SChris Wilson #include "i915_gem_request.h" 40b0decaf7SChris Wilson 4149ef5294SChris Wilson #define I915_FENCE_REG_NONE -1 4249ef5294SChris Wilson #define I915_MAX_NUM_FENCES 32 4349ef5294SChris Wilson /* 32 fences + sign bit for FENCE_REG_NONE */ 4449ef5294SChris Wilson #define I915_MAX_NUM_FENCE_BITS 6 4549ef5294SChris Wilson 464d884705SDaniel Vetter struct drm_i915_file_private; 4749ef5294SChris Wilson struct drm_i915_fence_reg; 484d884705SDaniel Vetter 4907749ef3SMichel Thierry typedef uint32_t gen6_pte_t; 5007749ef3SMichel Thierry typedef uint64_t gen8_pte_t; 5107749ef3SMichel Thierry typedef uint64_t gen8_pde_t; 52762d9936SMichel Thierry typedef uint64_t gen8_ppgtt_pdpe_t; 53762d9936SMichel Thierry typedef uint64_t gen8_ppgtt_pml4e_t; 540260c420SBen Widawsky 5572e96d64SJoonas Lahtinen #define ggtt_total_entries(ggtt) ((ggtt)->base.total >> PAGE_SHIFT) 560260c420SBen Widawsky 570260c420SBen Widawsky /* gen6-hsw has bit 11-4 for physical addr bit 39-32 */ 580260c420SBen Widawsky #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) 590260c420SBen Widawsky #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 600260c420SBen Widawsky #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 610260c420SBen Widawsky #define GEN6_PTE_CACHE_LLC (2 << 1) 620260c420SBen Widawsky #define GEN6_PTE_UNCACHED (1 << 1) 630260c420SBen Widawsky #define GEN6_PTE_VALID (1 << 0) 640260c420SBen Widawsky 6507749ef3SMichel Thierry #define I915_PTES(pte_len) (PAGE_SIZE / (pte_len)) 6607749ef3SMichel Thierry #define I915_PTE_MASK(pte_len) (I915_PTES(pte_len) - 1) 6707749ef3SMichel Thierry #define I915_PDES 512 6807749ef3SMichel Thierry #define I915_PDE_MASK (I915_PDES - 1) 69678d96fbSBen Widawsky #define NUM_PTE(pde_shift) (1 << (pde_shift - PAGE_SHIFT)) 7007749ef3SMichel Thierry 7107749ef3SMichel Thierry #define GEN6_PTES I915_PTES(sizeof(gen6_pte_t)) 7207749ef3SMichel Thierry #define GEN6_PD_SIZE (I915_PDES * PAGE_SIZE) 730260c420SBen Widawsky #define GEN6_PD_ALIGN (PAGE_SIZE * 16) 74678d96fbSBen Widawsky #define GEN6_PDE_SHIFT 22 750260c420SBen Widawsky #define GEN6_PDE_VALID (1 << 0) 760260c420SBen Widawsky 770260c420SBen Widawsky #define GEN7_PTE_CACHE_L3_LLC (3 << 1) 780260c420SBen Widawsky 790260c420SBen Widawsky #define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2) 800260c420SBen Widawsky #define BYT_PTE_WRITEABLE (1 << 1) 810260c420SBen Widawsky 820260c420SBen Widawsky /* Cacheability Control is a 4-bit value. The low three bits are stored in bits 830260c420SBen Widawsky * 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. 840260c420SBen Widawsky */ 850260c420SBen Widawsky #define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \ 860260c420SBen Widawsky (((bits) & 0x8) << (11 - 3))) 870260c420SBen Widawsky #define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2) 880260c420SBen Widawsky #define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3) 890260c420SBen Widawsky #define HSW_WB_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x8) 900260c420SBen Widawsky #define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb) 910260c420SBen Widawsky #define HSW_WT_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x7) 920260c420SBen Widawsky #define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6) 930260c420SBen Widawsky #define HSW_PTE_UNCACHED (0) 940260c420SBen Widawsky #define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0)) 950260c420SBen Widawsky #define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr) 960260c420SBen Widawsky 970260c420SBen Widawsky /* GEN8 legacy style address is defined as a 3 level page table: 980260c420SBen Widawsky * 31:30 | 29:21 | 20:12 | 11:0 990260c420SBen Widawsky * PDPE | PDE | PTE | offset 1000260c420SBen Widawsky * The difference as compared to normal x86 3 level page table is the PDPEs are 1010260c420SBen Widawsky * programmed via register. 10281ba8aefSMichel Thierry * 10381ba8aefSMichel Thierry * GEN8 48b legacy style address is defined as a 4 level page table: 10481ba8aefSMichel Thierry * 47:39 | 38:30 | 29:21 | 20:12 | 11:0 10581ba8aefSMichel Thierry * PML4E | PDPE | PDE | PTE | offset 1060260c420SBen Widawsky */ 10781ba8aefSMichel Thierry #define GEN8_PML4ES_PER_PML4 512 10881ba8aefSMichel Thierry #define GEN8_PML4E_SHIFT 39 109762d9936SMichel Thierry #define GEN8_PML4E_MASK (GEN8_PML4ES_PER_PML4 - 1) 1100260c420SBen Widawsky #define GEN8_PDPE_SHIFT 30 11181ba8aefSMichel Thierry /* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page 11281ba8aefSMichel Thierry * tables */ 11381ba8aefSMichel Thierry #define GEN8_PDPE_MASK 0x1ff 1140260c420SBen Widawsky #define GEN8_PDE_SHIFT 21 1150260c420SBen Widawsky #define GEN8_PDE_MASK 0x1ff 1160260c420SBen Widawsky #define GEN8_PTE_SHIFT 12 1170260c420SBen Widawsky #define GEN8_PTE_MASK 0x1ff 11876643600SBen Widawsky #define GEN8_LEGACY_PDPES 4 11907749ef3SMichel Thierry #define GEN8_PTES I915_PTES(sizeof(gen8_pte_t)) 1200260c420SBen Widawsky 12181ba8aefSMichel Thierry #define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\ 12281ba8aefSMichel Thierry GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES) 1236ac18502SMichel Thierry 1240260c420SBen Widawsky #define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD) 1250260c420SBen Widawsky #define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */ 1260260c420SBen Widawsky #define PPAT_CACHED_INDEX _PAGE_PAT /* WB LLCeLLC */ 1270260c420SBen Widawsky #define PPAT_DISPLAY_ELLC_INDEX _PAGE_PCD /* WT eLLC */ 1280260c420SBen Widawsky 129ee0ce478SVille Syrjälä #define CHV_PPAT_SNOOP (1<<6) 1300260c420SBen Widawsky #define GEN8_PPAT_AGE(x) (x<<4) 1310260c420SBen Widawsky #define GEN8_PPAT_LLCeLLC (3<<2) 1320260c420SBen Widawsky #define GEN8_PPAT_LLCELLC (2<<2) 1330260c420SBen Widawsky #define GEN8_PPAT_LLC (1<<2) 1340260c420SBen Widawsky #define GEN8_PPAT_WB (3<<0) 1350260c420SBen Widawsky #define GEN8_PPAT_WT (2<<0) 1360260c420SBen Widawsky #define GEN8_PPAT_WC (1<<0) 1370260c420SBen Widawsky #define GEN8_PPAT_UC (0<<0) 1380260c420SBen Widawsky #define GEN8_PPAT_ELLC_OVERRIDE (0<<2) 1390260c420SBen Widawsky #define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8)) 1400260c420SBen Widawsky 141fe14d5f4STvrtko Ursulin enum i915_ggtt_view_type { 142fe14d5f4STvrtko Ursulin I915_GGTT_VIEW_NORMAL = 0, 1438bd7ef16SJoonas Lahtinen I915_GGTT_VIEW_ROTATED, 1448bd7ef16SJoonas Lahtinen I915_GGTT_VIEW_PARTIAL, 14550470bb0STvrtko Ursulin }; 14650470bb0STvrtko Ursulin 14750470bb0STvrtko Ursulin struct intel_rotation_info { 1481663b9d6SVille Syrjälä struct { 1491663b9d6SVille Syrjälä /* tiles */ 1506687c906SVille Syrjälä unsigned int width, height, stride, offset; 1511663b9d6SVille Syrjälä } plane[2]; 152fe14d5f4STvrtko Ursulin }; 153fe14d5f4STvrtko Ursulin 154fe14d5f4STvrtko Ursulin struct i915_ggtt_view { 155fe14d5f4STvrtko Ursulin enum i915_ggtt_view_type type; 156fe14d5f4STvrtko Ursulin 1578bd7ef16SJoonas Lahtinen union { 1588bd7ef16SJoonas Lahtinen struct { 159088e0df4SMichel Thierry u64 offset; 1608bd7ef16SJoonas Lahtinen unsigned int size; 1618bd7ef16SJoonas Lahtinen } partial; 1627723f47dSVille Syrjälä struct intel_rotation_info rotated; 1638bd7ef16SJoonas Lahtinen } params; 164fe14d5f4STvrtko Ursulin }; 165fe14d5f4STvrtko Ursulin 166fe14d5f4STvrtko Ursulin extern const struct i915_ggtt_view i915_ggtt_view_normal; 1679abc4648SJoonas Lahtinen extern const struct i915_ggtt_view i915_ggtt_view_rotated; 168fe14d5f4STvrtko Ursulin 1690260c420SBen Widawsky enum i915_cache_level; 170fe14d5f4STvrtko Ursulin 1710260c420SBen Widawsky /** 1720260c420SBen Widawsky * A VMA represents a GEM BO that is bound into an address space. Therefore, a 1730260c420SBen Widawsky * VMA's presence cannot be guaranteed before binding, or after unbinding the 1740260c420SBen Widawsky * object into/from the address space. 1750260c420SBen Widawsky * 1760260c420SBen Widawsky * To make things as simple as possible (ie. no refcounting), a VMA's lifetime 1770260c420SBen Widawsky * will always be <= an objects lifetime. So object refcounting should cover us. 1780260c420SBen Widawsky */ 1790260c420SBen Widawsky struct i915_vma { 1800260c420SBen Widawsky struct drm_mm_node node; 1810260c420SBen Widawsky struct drm_i915_gem_object *obj; 1820260c420SBen Widawsky struct i915_address_space *vm; 18349ef5294SChris Wilson struct drm_i915_fence_reg *fence; 184247177ddSChris Wilson struct sg_table *pages; 1858ef8561fSChris Wilson void __iomem *iomap; 186de180033SChris Wilson u64 size; 187d8923dcfSChris Wilson u64 display_alignment; 1880260c420SBen Widawsky 1893272db53SChris Wilson unsigned int flags; 1903272db53SChris Wilson /** 1913272db53SChris Wilson * How many users have pinned this object in GTT space. The following 1923272db53SChris Wilson * users can each hold at most one reference: pwrite/pread, execbuffer 1933272db53SChris Wilson * (objects are not allowed multiple times for the same batchbuffer), 1943272db53SChris Wilson * and the framebuffer code. When switching/pageflipping, the 1953272db53SChris Wilson * framebuffer code has at most two buffers pinned per crtc. 1963272db53SChris Wilson * 1973272db53SChris Wilson * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3 1983272db53SChris Wilson * bits with absolutely no headroom. So use 4 bits. 1993272db53SChris Wilson */ 2003272db53SChris Wilson #define I915_VMA_PIN_MASK 0xf 201305bc234SChris Wilson #define I915_VMA_PIN_OVERFLOW BIT(5) 202b0decaf7SChris Wilson 203aff43766STvrtko Ursulin /** Flags and address space this VMA is bound to */ 204305bc234SChris Wilson #define I915_VMA_GLOBAL_BIND BIT(6) 205305bc234SChris Wilson #define I915_VMA_LOCAL_BIND BIT(7) 206305bc234SChris Wilson #define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW) 2073272db53SChris Wilson 208305bc234SChris Wilson #define I915_VMA_GGTT BIT(8) 20905a20d09SChris Wilson #define I915_VMA_CAN_FENCE BIT(9) 21005a20d09SChris Wilson #define I915_VMA_CLOSED BIT(10) 2113272db53SChris Wilson 2123272db53SChris Wilson unsigned int active; 2133272db53SChris Wilson struct i915_gem_active last_read[I915_NUM_ENGINES]; 21449ef5294SChris Wilson struct i915_gem_active last_fence; 215aff43766STvrtko Ursulin 216fe14d5f4STvrtko Ursulin /** 217fe14d5f4STvrtko Ursulin * Support different GGTT views into the same object. 218fe14d5f4STvrtko Ursulin * This means there can be multiple VMA mappings per object and per VM. 219fe14d5f4STvrtko Ursulin * i915_ggtt_view_type is used to distinguish between those entries. 220fe14d5f4STvrtko Ursulin * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also 221fe14d5f4STvrtko Ursulin * assumed in GEM functions which take no ggtt view parameter. 222fe14d5f4STvrtko Ursulin */ 223fe14d5f4STvrtko Ursulin struct i915_ggtt_view ggtt_view; 224fe14d5f4STvrtko Ursulin 2250260c420SBen Widawsky /** This object's place on the active/inactive lists */ 2261c7f4bcaSChris Wilson struct list_head vm_link; 2270260c420SBen Widawsky 2281c7f4bcaSChris Wilson struct list_head obj_link; /* Link in the object's VMA list */ 2290260c420SBen Widawsky 2300260c420SBen Widawsky /** This vma's place in the batchbuffer or on the eviction list */ 2310260c420SBen Widawsky struct list_head exec_list; 2320260c420SBen Widawsky 2330260c420SBen Widawsky /** 2340260c420SBen Widawsky * Used for performing relocations during execbuffer insertion. 2350260c420SBen Widawsky */ 2360260c420SBen Widawsky struct hlist_node exec_node; 2370260c420SBen Widawsky unsigned long exec_handle; 2380260c420SBen Widawsky struct drm_i915_gem_exec_object2 *exec_entry; 2390260c420SBen Widawsky }; 2400260c420SBen Widawsky 24181a8aa4aSChris Wilson struct i915_vma * 24281a8aa4aSChris Wilson i915_vma_create(struct drm_i915_gem_object *obj, 24381a8aa4aSChris Wilson struct i915_address_space *vm, 24481a8aa4aSChris Wilson const struct i915_ggtt_view *view); 24519880c4aSChris Wilson void i915_vma_unpin_and_release(struct i915_vma **p_vma); 24681a8aa4aSChris Wilson 2473272db53SChris Wilson static inline bool i915_vma_is_ggtt(const struct i915_vma *vma) 2483272db53SChris Wilson { 2493272db53SChris Wilson return vma->flags & I915_VMA_GGTT; 2503272db53SChris Wilson } 2513272db53SChris Wilson 25205a20d09SChris Wilson static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma) 25305a20d09SChris Wilson { 25405a20d09SChris Wilson return vma->flags & I915_VMA_CAN_FENCE; 25505a20d09SChris Wilson } 25605a20d09SChris Wilson 2573272db53SChris Wilson static inline bool i915_vma_is_closed(const struct i915_vma *vma) 2583272db53SChris Wilson { 2593272db53SChris Wilson return vma->flags & I915_VMA_CLOSED; 2603272db53SChris Wilson } 2613272db53SChris Wilson 262b0decaf7SChris Wilson static inline unsigned int i915_vma_get_active(const struct i915_vma *vma) 263b0decaf7SChris Wilson { 264b0decaf7SChris Wilson return vma->active; 265b0decaf7SChris Wilson } 266b0decaf7SChris Wilson 267b0decaf7SChris Wilson static inline bool i915_vma_is_active(const struct i915_vma *vma) 268b0decaf7SChris Wilson { 269b0decaf7SChris Wilson return i915_vma_get_active(vma); 270b0decaf7SChris Wilson } 271b0decaf7SChris Wilson 272b0decaf7SChris Wilson static inline void i915_vma_set_active(struct i915_vma *vma, 273b0decaf7SChris Wilson unsigned int engine) 274b0decaf7SChris Wilson { 275b0decaf7SChris Wilson vma->active |= BIT(engine); 276b0decaf7SChris Wilson } 277b0decaf7SChris Wilson 278b0decaf7SChris Wilson static inline void i915_vma_clear_active(struct i915_vma *vma, 279b0decaf7SChris Wilson unsigned int engine) 280b0decaf7SChris Wilson { 281b0decaf7SChris Wilson vma->active &= ~BIT(engine); 282b0decaf7SChris Wilson } 283b0decaf7SChris Wilson 284b0decaf7SChris Wilson static inline bool i915_vma_has_active_engine(const struct i915_vma *vma, 285b0decaf7SChris Wilson unsigned int engine) 286b0decaf7SChris Wilson { 287b0decaf7SChris Wilson return vma->active & BIT(engine); 288b0decaf7SChris Wilson } 289b0decaf7SChris Wilson 290bde13ebdSChris Wilson static inline u32 i915_ggtt_offset(const struct i915_vma *vma) 291bde13ebdSChris Wilson { 292bde13ebdSChris Wilson GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 293bde13ebdSChris Wilson GEM_BUG_ON(!vma->node.allocated); 294bde13ebdSChris Wilson GEM_BUG_ON(upper_32_bits(vma->node.start)); 295bde13ebdSChris Wilson GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1)); 296bde13ebdSChris Wilson return lower_32_bits(vma->node.start); 297bde13ebdSChris Wilson } 298bde13ebdSChris Wilson 29944159ddbSMika Kuoppala struct i915_page_dma { 300d7b3de91SBen Widawsky struct page *page; 30144159ddbSMika Kuoppala union { 3027324cc04SBen Widawsky dma_addr_t daddr; 303678d96fbSBen Widawsky 30444159ddbSMika Kuoppala /* For gen6/gen7 only. This is the offset in the GGTT 30544159ddbSMika Kuoppala * where the page directory entries for PPGTT begin 30644159ddbSMika Kuoppala */ 30744159ddbSMika Kuoppala uint32_t ggtt_offset; 30844159ddbSMika Kuoppala }; 30944159ddbSMika Kuoppala }; 31044159ddbSMika Kuoppala 311567047beSMika Kuoppala #define px_base(px) (&(px)->base) 312567047beSMika Kuoppala #define px_page(px) (px_base(px)->page) 313567047beSMika Kuoppala #define px_dma(px) (px_base(px)->daddr) 314567047beSMika Kuoppala 31544159ddbSMika Kuoppala struct i915_page_table { 31644159ddbSMika Kuoppala struct i915_page_dma base; 31744159ddbSMika Kuoppala 318678d96fbSBen Widawsky unsigned long *used_ptes; 319d7b3de91SBen Widawsky }; 320d7b3de91SBen Widawsky 321ec565b3cSMichel Thierry struct i915_page_directory { 32244159ddbSMika Kuoppala struct i915_page_dma base; 3237324cc04SBen Widawsky 32433c8819fSMichel Thierry unsigned long *used_pdes; 325ec565b3cSMichel Thierry struct i915_page_table *page_table[I915_PDES]; /* PDEs */ 326d7b3de91SBen Widawsky }; 327d7b3de91SBen Widawsky 328ec565b3cSMichel Thierry struct i915_page_directory_pointer { 3296ac18502SMichel Thierry struct i915_page_dma base; 3306ac18502SMichel Thierry 3316ac18502SMichel Thierry unsigned long *used_pdpes; 3326ac18502SMichel Thierry struct i915_page_directory **page_directory; 333d7b3de91SBen Widawsky }; 334d7b3de91SBen Widawsky 33581ba8aefSMichel Thierry struct i915_pml4 { 33681ba8aefSMichel Thierry struct i915_page_dma base; 33781ba8aefSMichel Thierry 33881ba8aefSMichel Thierry DECLARE_BITMAP(used_pml4es, GEN8_PML4ES_PER_PML4); 33981ba8aefSMichel Thierry struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4]; 34081ba8aefSMichel Thierry }; 34181ba8aefSMichel Thierry 3420260c420SBen Widawsky struct i915_address_space { 3430260c420SBen Widawsky struct drm_mm mm; 3440260c420SBen Widawsky struct drm_device *dev; 3452bfa996eSChris Wilson /* Every address space belongs to a struct file - except for the global 3462bfa996eSChris Wilson * GTT that is owned by the driver (and so @file is set to NULL). In 3472bfa996eSChris Wilson * principle, no information should leak from one context to another 3482bfa996eSChris Wilson * (or between files/processes etc) unless explicitly shared by the 3492bfa996eSChris Wilson * owner. Tracking the owner is important in order to free up per-file 3502bfa996eSChris Wilson * objects along with the file, to aide resource tracking, and to 3512bfa996eSChris Wilson * assign blame. 3522bfa996eSChris Wilson */ 3532bfa996eSChris Wilson struct drm_i915_file_private *file; 3540260c420SBen Widawsky struct list_head global_link; 355c44ef60eSMika Kuoppala u64 start; /* Start offset always 0 for dri2 */ 356c44ef60eSMika Kuoppala u64 total; /* size addr space maps (ex. 2GB for ggtt) */ 3570260c420SBen Widawsky 35850e046b6SChris Wilson bool closed; 35950e046b6SChris Wilson 3608bcdd0f7SChris Wilson struct i915_page_dma scratch_page; 36179ab9370SMika Kuoppala struct i915_page_table *scratch_pt; 36279ab9370SMika Kuoppala struct i915_page_directory *scratch_pd; 36369ab76fdSMichel Thierry struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */ 3640260c420SBen Widawsky 3650260c420SBen Widawsky /** 3660260c420SBen Widawsky * List of objects currently involved in rendering. 3670260c420SBen Widawsky * 3680260c420SBen Widawsky * Includes buffers having the contents of their GPU caches 36997b2a6a1SJohn Harrison * flushed, not necessarily primitives. last_read_req 3700260c420SBen Widawsky * represents when the rendering involved will be completed. 3710260c420SBen Widawsky * 3720260c420SBen Widawsky * A reference is held on the buffer while on this list. 3730260c420SBen Widawsky */ 3740260c420SBen Widawsky struct list_head active_list; 3750260c420SBen Widawsky 3760260c420SBen Widawsky /** 3770260c420SBen Widawsky * LRU list of objects which are not in the ringbuffer and 3780260c420SBen Widawsky * are ready to unbind, but are still in the GTT. 3790260c420SBen Widawsky * 38097b2a6a1SJohn Harrison * last_read_req is NULL while an object is in this list. 3810260c420SBen Widawsky * 3820260c420SBen Widawsky * A reference is not held on the buffer while on this list, 3830260c420SBen Widawsky * as merely being GTT-bound shouldn't prevent its being 3840260c420SBen Widawsky * freed, and we'll pull it off the list in the free path. 3850260c420SBen Widawsky */ 3860260c420SBen Widawsky struct list_head inactive_list; 3870260c420SBen Widawsky 38850e046b6SChris Wilson /** 38950e046b6SChris Wilson * List of vma that have been unbound. 39050e046b6SChris Wilson * 39150e046b6SChris Wilson * A reference is not held on the buffer while on this list. 39250e046b6SChris Wilson */ 39350e046b6SChris Wilson struct list_head unbound_list; 39450e046b6SChris Wilson 3950260c420SBen Widawsky /* FIXME: Need a more generic return type */ 39607749ef3SMichel Thierry gen6_pte_t (*pte_encode)(dma_addr_t addr, 3970260c420SBen Widawsky enum i915_cache_level level, 39824f3a8cfSAkash Goel bool valid, u32 flags); /* Create a valid PTE */ 399f329f5f6SDaniel Vetter /* flags for pte_encode */ 400f329f5f6SDaniel Vetter #define PTE_READ_ONLY (1<<0) 401678d96fbSBen Widawsky int (*allocate_va_range)(struct i915_address_space *vm, 402678d96fbSBen Widawsky uint64_t start, 403678d96fbSBen Widawsky uint64_t length); 4040260c420SBen Widawsky void (*clear_range)(struct i915_address_space *vm, 4050260c420SBen Widawsky uint64_t start, 4060260c420SBen Widawsky uint64_t length, 4070260c420SBen Widawsky bool use_scratch); 408d6473f56SChris Wilson void (*insert_page)(struct i915_address_space *vm, 409d6473f56SChris Wilson dma_addr_t addr, 410d6473f56SChris Wilson uint64_t offset, 411d6473f56SChris Wilson enum i915_cache_level cache_level, 412d6473f56SChris Wilson u32 flags); 4130260c420SBen Widawsky void (*insert_entries)(struct i915_address_space *vm, 4140260c420SBen Widawsky struct sg_table *st, 4150260c420SBen Widawsky uint64_t start, 41624f3a8cfSAkash Goel enum i915_cache_level cache_level, u32 flags); 4170260c420SBen Widawsky void (*cleanup)(struct i915_address_space *vm); 418777dc5bbSDaniel Vetter /** Unmap an object from an address space. This usually consists of 419777dc5bbSDaniel Vetter * setting the valid PTE entries to a reserved scratch page. */ 420777dc5bbSDaniel Vetter void (*unbind_vma)(struct i915_vma *vma); 421777dc5bbSDaniel Vetter /* Map an object into an address space with the given cache flags. */ 42270b9f6f8SDaniel Vetter int (*bind_vma)(struct i915_vma *vma, 423777dc5bbSDaniel Vetter enum i915_cache_level cache_level, 424777dc5bbSDaniel Vetter u32 flags); 4250260c420SBen Widawsky }; 4260260c420SBen Widawsky 4272bfa996eSChris Wilson #define i915_is_ggtt(V) (!(V)->file) 428596c5923SChris Wilson 4290260c420SBen Widawsky /* The Graphics Translation Table is the way in which GEN hardware translates a 4300260c420SBen Widawsky * Graphics Virtual Address into a Physical Address. In addition to the normal 4310260c420SBen Widawsky * collateral associated with any va->pa translations GEN hardware also has a 4320260c420SBen Widawsky * portion of the GTT which can be mapped by the CPU and remain both coherent 4330260c420SBen Widawsky * and correct (in cases like swizzling). That region is referred to as GMADR in 4340260c420SBen Widawsky * the spec. 4350260c420SBen Widawsky */ 43662106b4fSJoonas Lahtinen struct i915_ggtt { 4370260c420SBen Widawsky struct i915_address_space base; 438f7bbe788SChris Wilson struct io_mapping mappable; /* Mapping to our CPU mappable region */ 4390260c420SBen Widawsky 440c44ef60eSMika Kuoppala size_t stolen_size; /* Total size of stolen memory */ 441a9da512bSPaulo Zanoni size_t stolen_usable_size; /* Total size minus BIOS reserved */ 442274008e8SSagar Arun Kamble size_t stolen_reserved_base; 443274008e8SSagar Arun Kamble size_t stolen_reserved_size; 444c44ef60eSMika Kuoppala u64 mappable_end; /* End offset that we can CPU map */ 4450260c420SBen Widawsky phys_addr_t mappable_base; /* PA of our GMADR */ 4460260c420SBen Widawsky 4470260c420SBen Widawsky /** "Graphics Stolen Memory" holds the global PTEs */ 4480260c420SBen Widawsky void __iomem *gsm; 4490260c420SBen Widawsky 4500260c420SBen Widawsky bool do_idle_maps; 4510260c420SBen Widawsky 4520260c420SBen Widawsky int mtrr; 4530260c420SBen Widawsky }; 4540260c420SBen Widawsky 4550260c420SBen Widawsky struct i915_hw_ppgtt { 4560260c420SBen Widawsky struct i915_address_space base; 4570260c420SBen Widawsky struct kref ref; 4580260c420SBen Widawsky struct drm_mm_node node; 459563222a7SBen Widawsky unsigned long pd_dirty_rings; 4600260c420SBen Widawsky union { 46181ba8aefSMichel Thierry struct i915_pml4 pml4; /* GEN8+ & 48b PPGTT */ 46281ba8aefSMichel Thierry struct i915_page_directory_pointer pdp; /* GEN8+ */ 46381ba8aefSMichel Thierry struct i915_page_directory pd; /* GEN6-7 */ 464d7b3de91SBen Widawsky }; 4650260c420SBen Widawsky 466678d96fbSBen Widawsky gen6_pte_t __iomem *pd_addr; 467678d96fbSBen Widawsky 4680260c420SBen Widawsky int (*enable)(struct i915_hw_ppgtt *ppgtt); 4690260c420SBen Widawsky int (*switch_mm)(struct i915_hw_ppgtt *ppgtt, 470e85b26dcSJohn Harrison struct drm_i915_gem_request *req); 4710260c420SBen Widawsky void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m); 4720260c420SBen Widawsky }; 4730260c420SBen Widawsky 474731f74c5SDave Gordon /* 475731f74c5SDave Gordon * gen6_for_each_pde() iterates over every pde from start until start+length. 476731f74c5SDave Gordon * If start and start+length are not perfectly divisible, the macro will round 477731f74c5SDave Gordon * down and up as needed. Start=0 and length=2G effectively iterates over 478731f74c5SDave Gordon * every PDE in the system. The macro modifies ALL its parameters except 'pd', 479731f74c5SDave Gordon * so each of the other parameters should preferably be a simple variable, or 480731f74c5SDave Gordon * at most an lvalue with no side-effects! 481678d96fbSBen Widawsky */ 482731f74c5SDave Gordon #define gen6_for_each_pde(pt, pd, start, length, iter) \ 483fdc454c1SMichel Thierry for (iter = gen6_pde_index(start); \ 484731f74c5SDave Gordon length > 0 && iter < I915_PDES && \ 485731f74c5SDave Gordon (pt = (pd)->page_table[iter], true); \ 486731f74c5SDave Gordon ({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT); \ 487731f74c5SDave Gordon temp = min(temp - start, length); \ 488731f74c5SDave Gordon start += temp, length -= temp; }), ++iter) 489678d96fbSBen Widawsky 490731f74c5SDave Gordon #define gen6_for_all_pdes(pt, pd, iter) \ 49109942c65SMichel Thierry for (iter = 0; \ 492731f74c5SDave Gordon iter < I915_PDES && \ 493731f74c5SDave Gordon (pt = (pd)->page_table[iter], true); \ 494731f74c5SDave Gordon ++iter) 49509942c65SMichel Thierry 496678d96fbSBen Widawsky static inline uint32_t i915_pte_index(uint64_t address, uint32_t pde_shift) 497678d96fbSBen Widawsky { 498678d96fbSBen Widawsky const uint32_t mask = NUM_PTE(pde_shift) - 1; 499678d96fbSBen Widawsky 500678d96fbSBen Widawsky return (address >> PAGE_SHIFT) & mask; 501678d96fbSBen Widawsky } 502678d96fbSBen Widawsky 503678d96fbSBen Widawsky /* Helper to counts the number of PTEs within the given length. This count 504678d96fbSBen Widawsky * does not cross a page table boundary, so the max value would be 505678d96fbSBen Widawsky * GEN6_PTES for GEN6, and GEN8_PTES for GEN8. 506678d96fbSBen Widawsky */ 507678d96fbSBen Widawsky static inline uint32_t i915_pte_count(uint64_t addr, size_t length, 508678d96fbSBen Widawsky uint32_t pde_shift) 509678d96fbSBen Widawsky { 51069603dbbSAlan const uint64_t mask = ~((1ULL << pde_shift) - 1); 511678d96fbSBen Widawsky uint64_t end; 512678d96fbSBen Widawsky 513678d96fbSBen Widawsky WARN_ON(length == 0); 514678d96fbSBen Widawsky WARN_ON(offset_in_page(addr|length)); 515678d96fbSBen Widawsky 516678d96fbSBen Widawsky end = addr + length; 517678d96fbSBen Widawsky 518678d96fbSBen Widawsky if ((addr & mask) != (end & mask)) 519678d96fbSBen Widawsky return NUM_PTE(pde_shift) - i915_pte_index(addr, pde_shift); 520678d96fbSBen Widawsky 521678d96fbSBen Widawsky return i915_pte_index(end, pde_shift) - i915_pte_index(addr, pde_shift); 522678d96fbSBen Widawsky } 523678d96fbSBen Widawsky 524678d96fbSBen Widawsky static inline uint32_t i915_pde_index(uint64_t addr, uint32_t shift) 525678d96fbSBen Widawsky { 526678d96fbSBen Widawsky return (addr >> shift) & I915_PDE_MASK; 527678d96fbSBen Widawsky } 528678d96fbSBen Widawsky 529678d96fbSBen Widawsky static inline uint32_t gen6_pte_index(uint32_t addr) 530678d96fbSBen Widawsky { 531678d96fbSBen Widawsky return i915_pte_index(addr, GEN6_PDE_SHIFT); 532678d96fbSBen Widawsky } 533678d96fbSBen Widawsky 534678d96fbSBen Widawsky static inline size_t gen6_pte_count(uint32_t addr, uint32_t length) 535678d96fbSBen Widawsky { 536678d96fbSBen Widawsky return i915_pte_count(addr, length, GEN6_PDE_SHIFT); 537678d96fbSBen Widawsky } 538678d96fbSBen Widawsky 539678d96fbSBen Widawsky static inline uint32_t gen6_pde_index(uint32_t addr) 540678d96fbSBen Widawsky { 541678d96fbSBen Widawsky return i915_pde_index(addr, GEN6_PDE_SHIFT); 542678d96fbSBen Widawsky } 543678d96fbSBen Widawsky 5449271d959SMichel Thierry /* Equivalent to the gen6 version, For each pde iterates over every pde 5459271d959SMichel Thierry * between from start until start + length. On gen8+ it simply iterates 5469271d959SMichel Thierry * over every page directory entry in a page directory. 5479271d959SMichel Thierry */ 548e8ebd8e2SDave Gordon #define gen8_for_each_pde(pt, pd, start, length, iter) \ 5499271d959SMichel Thierry for (iter = gen8_pde_index(start); \ 550e8ebd8e2SDave Gordon length > 0 && iter < I915_PDES && \ 551e8ebd8e2SDave Gordon (pt = (pd)->page_table[iter], true); \ 552e8ebd8e2SDave Gordon ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT); \ 553e8ebd8e2SDave Gordon temp = min(temp - start, length); \ 554e8ebd8e2SDave Gordon start += temp, length -= temp; }), ++iter) 5559271d959SMichel Thierry 556e8ebd8e2SDave Gordon #define gen8_for_each_pdpe(pd, pdp, start, length, iter) \ 5579271d959SMichel Thierry for (iter = gen8_pdpe_index(start); \ 558e8ebd8e2SDave Gordon length > 0 && iter < I915_PDPES_PER_PDP(dev) && \ 559e8ebd8e2SDave Gordon (pd = (pdp)->page_directory[iter], true); \ 560e8ebd8e2SDave Gordon ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT); \ 561e8ebd8e2SDave Gordon temp = min(temp - start, length); \ 562e8ebd8e2SDave Gordon start += temp, length -= temp; }), ++iter) 5639271d959SMichel Thierry 564e8ebd8e2SDave Gordon #define gen8_for_each_pml4e(pdp, pml4, start, length, iter) \ 565762d9936SMichel Thierry for (iter = gen8_pml4e_index(start); \ 566e8ebd8e2SDave Gordon length > 0 && iter < GEN8_PML4ES_PER_PML4 && \ 567e8ebd8e2SDave Gordon (pdp = (pml4)->pdps[iter], true); \ 568e8ebd8e2SDave Gordon ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT); \ 569e8ebd8e2SDave Gordon temp = min(temp - start, length); \ 570e8ebd8e2SDave Gordon start += temp, length -= temp; }), ++iter) 571762d9936SMichel Thierry 5729271d959SMichel Thierry static inline uint32_t gen8_pte_index(uint64_t address) 5739271d959SMichel Thierry { 5749271d959SMichel Thierry return i915_pte_index(address, GEN8_PDE_SHIFT); 5759271d959SMichel Thierry } 5769271d959SMichel Thierry 5779271d959SMichel Thierry static inline uint32_t gen8_pde_index(uint64_t address) 5789271d959SMichel Thierry { 5799271d959SMichel Thierry return i915_pde_index(address, GEN8_PDE_SHIFT); 5809271d959SMichel Thierry } 5819271d959SMichel Thierry 5829271d959SMichel Thierry static inline uint32_t gen8_pdpe_index(uint64_t address) 5839271d959SMichel Thierry { 5849271d959SMichel Thierry return (address >> GEN8_PDPE_SHIFT) & GEN8_PDPE_MASK; 5859271d959SMichel Thierry } 5869271d959SMichel Thierry 5879271d959SMichel Thierry static inline uint32_t gen8_pml4e_index(uint64_t address) 5889271d959SMichel Thierry { 589762d9936SMichel Thierry return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK; 5909271d959SMichel Thierry } 5919271d959SMichel Thierry 59233c8819fSMichel Thierry static inline size_t gen8_pte_count(uint64_t address, uint64_t length) 59333c8819fSMichel Thierry { 59433c8819fSMichel Thierry return i915_pte_count(address, length, GEN8_PDE_SHIFT); 59533c8819fSMichel Thierry } 59633c8819fSMichel Thierry 597d852c7bfSMika Kuoppala static inline dma_addr_t 598d852c7bfSMika Kuoppala i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n) 599d852c7bfSMika Kuoppala { 600d852c7bfSMika Kuoppala return test_bit(n, ppgtt->pdp.used_pdpes) ? 601567047beSMika Kuoppala px_dma(ppgtt->pdp.page_directory[n]) : 60279ab9370SMika Kuoppala px_dma(ppgtt->base.scratch_pd); 603d852c7bfSMika Kuoppala } 604d852c7bfSMika Kuoppala 60597d6d7abSChris Wilson int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv); 60697d6d7abSChris Wilson int i915_ggtt_init_hw(struct drm_i915_private *dev_priv); 60797d6d7abSChris Wilson int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv); 608f6b9d5caSChris Wilson int i915_gem_init_ggtt(struct drm_i915_private *dev_priv); 60997d6d7abSChris Wilson void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv); 610ee960be7SDaniel Vetter 61182460d97SDaniel Vetter int i915_ppgtt_init_hw(struct drm_device *dev); 612ee960be7SDaniel Vetter void i915_ppgtt_release(struct kref *kref); 6132bfa996eSChris Wilson struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv, 6144d884705SDaniel Vetter struct drm_i915_file_private *fpriv); 615ee960be7SDaniel Vetter static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt) 616ee960be7SDaniel Vetter { 617ee960be7SDaniel Vetter if (ppgtt) 618ee960be7SDaniel Vetter kref_get(&ppgtt->ref); 619ee960be7SDaniel Vetter } 620ee960be7SDaniel Vetter static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt) 621ee960be7SDaniel Vetter { 622ee960be7SDaniel Vetter if (ppgtt) 623ee960be7SDaniel Vetter kref_put(&ppgtt->ref, i915_ppgtt_release); 624ee960be7SDaniel Vetter } 6250260c420SBen Widawsky 626dc97997aSChris Wilson void i915_check_and_clear_faults(struct drm_i915_private *dev_priv); 6270260c420SBen Widawsky void i915_gem_suspend_gtt_mappings(struct drm_device *dev); 6280260c420SBen Widawsky void i915_gem_restore_gtt_mappings(struct drm_device *dev); 6290260c420SBen Widawsky 6300260c420SBen Widawsky int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj); 6310260c420SBen Widawsky void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj); 6320260c420SBen Widawsky 63359bfa124SChris Wilson /* Flags used by pin/bind&friends. */ 634305bc234SChris Wilson #define PIN_NONBLOCK BIT(0) 635305bc234SChris Wilson #define PIN_MAPPABLE BIT(1) 636305bc234SChris Wilson #define PIN_ZONE_4G BIT(2) 63782118877SChris Wilson #define PIN_NONFAULT BIT(3) 638305bc234SChris Wilson 639305bc234SChris Wilson #define PIN_MBZ BIT(5) /* I915_VMA_PIN_OVERFLOW */ 640305bc234SChris Wilson #define PIN_GLOBAL BIT(6) /* I915_VMA_GLOBAL_BIND */ 641305bc234SChris Wilson #define PIN_USER BIT(7) /* I915_VMA_LOCAL_BIND */ 642305bc234SChris Wilson #define PIN_UPDATE BIT(8) 643305bc234SChris Wilson 644305bc234SChris Wilson #define PIN_HIGH BIT(9) 645305bc234SChris Wilson #define PIN_OFFSET_BIAS BIT(10) 646305bc234SChris Wilson #define PIN_OFFSET_FIXED BIT(11) 64759bfa124SChris Wilson #define PIN_OFFSET_MASK (~4095) 64859bfa124SChris Wilson 649305bc234SChris Wilson int __i915_vma_do_pin(struct i915_vma *vma, 650305bc234SChris Wilson u64 size, u64 alignment, u64 flags); 651305bc234SChris Wilson static inline int __must_check 652305bc234SChris Wilson i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) 653305bc234SChris Wilson { 654305bc234SChris Wilson BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW); 655305bc234SChris Wilson BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); 656305bc234SChris Wilson BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); 657305bc234SChris Wilson 658305bc234SChris Wilson /* Pin early to prevent the shrinker/eviction logic from destroying 659305bc234SChris Wilson * our vma as we insert and bind. 660305bc234SChris Wilson */ 661305bc234SChris Wilson if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) 662305bc234SChris Wilson return 0; 663305bc234SChris Wilson 664305bc234SChris Wilson return __i915_vma_do_pin(vma, size, alignment, flags); 665305bc234SChris Wilson } 666305bc234SChris Wilson 66720dfbde4SChris Wilson static inline int i915_vma_pin_count(const struct i915_vma *vma) 66820dfbde4SChris Wilson { 6693272db53SChris Wilson return vma->flags & I915_VMA_PIN_MASK; 67020dfbde4SChris Wilson } 67120dfbde4SChris Wilson 67220dfbde4SChris Wilson static inline bool i915_vma_is_pinned(const struct i915_vma *vma) 67320dfbde4SChris Wilson { 67420dfbde4SChris Wilson return i915_vma_pin_count(vma); 67520dfbde4SChris Wilson } 67620dfbde4SChris Wilson 67720dfbde4SChris Wilson static inline void __i915_vma_pin(struct i915_vma *vma) 67820dfbde4SChris Wilson { 6793272db53SChris Wilson vma->flags++; 680305bc234SChris Wilson GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW); 68120dfbde4SChris Wilson } 68220dfbde4SChris Wilson 68320dfbde4SChris Wilson static inline void __i915_vma_unpin(struct i915_vma *vma) 68420dfbde4SChris Wilson { 68520dfbde4SChris Wilson GEM_BUG_ON(!i915_vma_is_pinned(vma)); 6863272db53SChris Wilson vma->flags--; 68720dfbde4SChris Wilson } 68820dfbde4SChris Wilson 68920dfbde4SChris Wilson static inline void i915_vma_unpin(struct i915_vma *vma) 69020dfbde4SChris Wilson { 69120dfbde4SChris Wilson GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 69220dfbde4SChris Wilson __i915_vma_unpin(vma); 69320dfbde4SChris Wilson } 69420dfbde4SChris Wilson 6958ef8561fSChris Wilson /** 6968ef8561fSChris Wilson * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture 6978ef8561fSChris Wilson * @vma: VMA to iomap 6988ef8561fSChris Wilson * 6998ef8561fSChris Wilson * The passed in VMA has to be pinned in the global GTT mappable region. 7008ef8561fSChris Wilson * An extra pinning of the VMA is acquired for the return iomapping, 7018ef8561fSChris Wilson * the caller must call i915_vma_unpin_iomap to relinquish the pinning 7028ef8561fSChris Wilson * after the iomapping is no longer required. 7038ef8561fSChris Wilson * 7048ef8561fSChris Wilson * Callers must hold the struct_mutex. 7058ef8561fSChris Wilson * 7068ef8561fSChris Wilson * Returns a valid iomapped pointer or ERR_PTR. 7078ef8561fSChris Wilson */ 7088ef8561fSChris Wilson void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); 709406ea8d2SChris Wilson #define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x)) 7108ef8561fSChris Wilson 7118ef8561fSChris Wilson /** 7128ef8561fSChris Wilson * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap 7138ef8561fSChris Wilson * @vma: VMA to unpin 7148ef8561fSChris Wilson * 7158ef8561fSChris Wilson * Unpins the previously iomapped VMA from i915_vma_pin_iomap(). 7168ef8561fSChris Wilson * 7178ef8561fSChris Wilson * Callers must hold the struct_mutex. This function is only valid to be 7188ef8561fSChris Wilson * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap(). 7198ef8561fSChris Wilson */ 7208ef8561fSChris Wilson static inline void i915_vma_unpin_iomap(struct i915_vma *vma) 7218ef8561fSChris Wilson { 7228ef8561fSChris Wilson lockdep_assert_held(&vma->vm->dev->struct_mutex); 7238ef8561fSChris Wilson GEM_BUG_ON(vma->iomap == NULL); 72420dfbde4SChris Wilson i915_vma_unpin(vma); 7258ef8561fSChris Wilson } 7268ef8561fSChris Wilson 7278b797af1SChris Wilson static inline struct page *i915_vma_first_page(struct i915_vma *vma) 7288b797af1SChris Wilson { 7298b797af1SChris Wilson GEM_BUG_ON(!vma->pages); 7308b797af1SChris Wilson return sg_page(vma->pages->sgl); 7318b797af1SChris Wilson } 7328b797af1SChris Wilson 7330260c420SBen Widawsky #endif 734