198932149SChris Wilson /* 298932149SChris Wilson * SPDX-License-Identifier: MIT 398932149SChris Wilson * 498932149SChris Wilson * Copyright © 2016 Intel Corporation 598932149SChris Wilson */ 698932149SChris Wilson 798932149SChris Wilson #ifndef __I915_GEM_OBJECT_H__ 898932149SChris Wilson #define __I915_GEM_OBJECT_H__ 998932149SChris Wilson 1098932149SChris Wilson #include <drm/drm_gem.h> 1198932149SChris Wilson #include <drm/drm_file.h> 1298932149SChris Wilson #include <drm/drm_device.h> 1398932149SChris Wilson 14da42104fSChris Wilson #include "display/intel_frontbuffer.h" 15b3f450d9SMatthew Auld #include "intel_memory_region.h" 1698932149SChris Wilson #include "i915_gem_object_types.h" 17f0e4a063SChris Wilson #include "i915_gem_gtt.h" 185c43ec5dSThomas Hellström #include "i915_gem_ww.h" 192c86e55dSMatthew Auld #include "i915_vma_types.h" 20f0e4a063SChris Wilson 21b6e913e1SThomas Hellström enum intel_region_id; 22b6e913e1SThomas Hellström 23ae2fb480SMatthew Auld /* 24ae2fb480SMatthew Auld * XXX: There is a prevalence of the assumption that we fit the 25ae2fb480SMatthew Auld * object's page count inside a 32bit _signed_ variable. Let's document 26ae2fb480SMatthew Auld * this and catch if we ever need to fix it. In the meantime, if you do 27ae2fb480SMatthew Auld * spot such a local variable, please consider fixing! 28ae2fb480SMatthew Auld * 29ae2fb480SMatthew Auld * Aside from our own locals (for which we have no excuse!): 30ae2fb480SMatthew Auld * - sg_table embeds unsigned int for num_pages 31ae2fb480SMatthew Auld * - get_user_pages*() mixed ints with longs 32ae2fb480SMatthew Auld */ 33ae2fb480SMatthew Auld #define GEM_CHECK_SIZE_OVERFLOW(sz) \ 34ae2fb480SMatthew Auld GEM_WARN_ON((sz) >> PAGE_SHIFT > INT_MAX) 35ae2fb480SMatthew Auld 36ae2fb480SMatthew Auld static inline bool i915_gem_object_size_2big(u64 size) 37ae2fb480SMatthew Auld { 38ae2fb480SMatthew Auld struct drm_i915_gem_object *obj; 39ae2fb480SMatthew Auld 40ae2fb480SMatthew Auld if (GEM_CHECK_SIZE_OVERFLOW(size)) 41ae2fb480SMatthew Auld return true; 42ae2fb480SMatthew Auld 43ae2fb480SMatthew Auld if (overflows_type(size, obj->base.size)) 44ae2fb480SMatthew Auld return true; 45ae2fb480SMatthew Auld 46ae2fb480SMatthew Auld return false; 47ae2fb480SMatthew Auld } 48ae2fb480SMatthew Auld 498475355fSChris Wilson void i915_gem_init__objects(struct drm_i915_private *i915); 508475355fSChris Wilson 51c8ad09afSDaniel Vetter void i915_objects_module_exit(void); 52c8ad09afSDaniel Vetter int i915_objects_module_init(void); 53c8ad09afSDaniel Vetter 5498932149SChris Wilson struct drm_i915_gem_object *i915_gem_object_alloc(void); 5598932149SChris Wilson void i915_gem_object_free(struct drm_i915_gem_object *obj); 5698932149SChris Wilson 578475355fSChris Wilson void i915_gem_object_init(struct drm_i915_gem_object *obj, 587867d709SChris Wilson const struct drm_i915_gem_object_ops *ops, 59c471748dSMaarten Lankhorst struct lock_class_key *key, 60c471748dSMaarten Lankhorst unsigned alloc_flags); 61068396bbSThomas Hellström 62068396bbSThomas Hellström void __i915_gem_object_fini(struct drm_i915_gem_object *obj); 63068396bbSThomas Hellström 648475355fSChris Wilson struct drm_i915_gem_object * 65da1184cdSMatthew Auld i915_gem_object_create_shmem(struct drm_i915_private *i915, 66da1184cdSMatthew Auld resource_size_t size); 678475355fSChris Wilson struct drm_i915_gem_object * 688475355fSChris Wilson i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, 69da1184cdSMatthew Auld const void *data, resource_size_t size); 70bf947c98SJason Ekstrand struct drm_i915_gem_object * 71bf947c98SJason Ekstrand __i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, 72bf947c98SJason Ekstrand struct intel_memory_region **placements, 73bf947c98SJason Ekstrand unsigned int n_placements); 748475355fSChris Wilson 758475355fSChris Wilson extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops; 76ed29c269SMaarten Lankhorst 778475355fSChris Wilson void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 788475355fSChris Wilson struct sg_table *pages, 798475355fSChris Wilson bool needs_clflush); 808475355fSChris Wilson 81a6117097SMaarten Lankhorst int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj, 82a6117097SMaarten Lankhorst const struct drm_i915_gem_pwrite *args); 83a6117097SMaarten Lankhorst int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj, 84a6117097SMaarten Lankhorst const struct drm_i915_gem_pread *args); 85a6117097SMaarten Lankhorst 86f033428dSChris Wilson int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align); 87a85fffe3SMaarten Lankhorst void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, 88a85fffe3SMaarten Lankhorst struct sg_table *pages); 89a6117097SMaarten Lankhorst void i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 90a6117097SMaarten Lankhorst struct sg_table *pages); 91a6117097SMaarten Lankhorst 928475355fSChris Wilson void i915_gem_flush_free_objects(struct drm_i915_private *i915); 938475355fSChris Wilson 94f033428dSChris Wilson struct sg_table * 95f033428dSChris Wilson __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj); 96f033428dSChris Wilson 9798932149SChris Wilson /** 9898932149SChris Wilson * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle 9998932149SChris Wilson * @filp: DRM file private date 10098932149SChris Wilson * @handle: userspace handle 10198932149SChris Wilson * 10298932149SChris Wilson * Returns: 10398932149SChris Wilson * 10498932149SChris Wilson * A pointer to the object named by the handle if such exists on @filp, NULL 10598932149SChris Wilson * otherwise. This object is only valid whilst under the RCU read lock, and 10698932149SChris Wilson * note carefully the object may be in the process of being destroyed. 10798932149SChris Wilson */ 10898932149SChris Wilson static inline struct drm_i915_gem_object * 10998932149SChris Wilson i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) 11098932149SChris Wilson { 11198932149SChris Wilson #ifdef CONFIG_LOCKDEP 11298932149SChris Wilson WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); 11398932149SChris Wilson #endif 11498932149SChris Wilson return idr_find(&file->object_idr, handle); 11598932149SChris Wilson } 11698932149SChris Wilson 11798932149SChris Wilson static inline struct drm_i915_gem_object * 118280d14a6SChris Wilson i915_gem_object_get_rcu(struct drm_i915_gem_object *obj) 119280d14a6SChris Wilson { 120280d14a6SChris Wilson if (obj && !kref_get_unless_zero(&obj->base.refcount)) 121280d14a6SChris Wilson obj = NULL; 122280d14a6SChris Wilson 123280d14a6SChris Wilson return obj; 124280d14a6SChris Wilson } 125280d14a6SChris Wilson 126280d14a6SChris Wilson static inline struct drm_i915_gem_object * 12798932149SChris Wilson i915_gem_object_lookup(struct drm_file *file, u32 handle) 12898932149SChris Wilson { 12998932149SChris Wilson struct drm_i915_gem_object *obj; 13098932149SChris Wilson 13198932149SChris Wilson rcu_read_lock(); 13298932149SChris Wilson obj = i915_gem_object_lookup_rcu(file, handle); 133280d14a6SChris Wilson obj = i915_gem_object_get_rcu(obj); 13498932149SChris Wilson rcu_read_unlock(); 13598932149SChris Wilson 13698932149SChris Wilson return obj; 13798932149SChris Wilson } 13898932149SChris Wilson 13998932149SChris Wilson __deprecated 140b5893ffcSJanusz Krzysztofik struct drm_gem_object * 14198932149SChris Wilson drm_gem_object_lookup(struct drm_file *file, u32 handle); 14298932149SChris Wilson 14398932149SChris Wilson __attribute__((nonnull)) 14498932149SChris Wilson static inline struct drm_i915_gem_object * 14598932149SChris Wilson i915_gem_object_get(struct drm_i915_gem_object *obj) 14698932149SChris Wilson { 14798932149SChris Wilson drm_gem_object_get(&obj->base); 14898932149SChris Wilson return obj; 14998932149SChris Wilson } 15098932149SChris Wilson 15198932149SChris Wilson __attribute__((nonnull)) 15298932149SChris Wilson static inline void 15398932149SChris Wilson i915_gem_object_put(struct drm_i915_gem_object *obj) 15498932149SChris Wilson { 15598932149SChris Wilson __drm_gem_object_put(&obj->base); 15698932149SChris Wilson } 15798932149SChris Wilson 15852791eeeSChristian König #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv) 1596951e589SChris Wilson 160a3258dbdSThomas Hellström /* 161a3258dbdSThomas Hellström * If more than one potential simultaneous locker, assert held. 162a3258dbdSThomas Hellström */ 1630ff37575SThomas Hellström static inline void assert_object_held_shared(const struct drm_i915_gem_object *obj) 164a3258dbdSThomas Hellström { 165a3258dbdSThomas Hellström /* 166a3258dbdSThomas Hellström * Note mm list lookup is protected by 167a3258dbdSThomas Hellström * kref_get_unless_zero(). 168a3258dbdSThomas Hellström */ 169a3258dbdSThomas Hellström if (IS_ENABLED(CONFIG_LOCKDEP) && 170a3258dbdSThomas Hellström kref_read(&obj->base.refcount) > 0) 171cf41a8f1SMaarten Lankhorst assert_object_held(obj); 172a3258dbdSThomas Hellström } 173a3258dbdSThomas Hellström 17480f0b679SMaarten Lankhorst static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj, 17580f0b679SMaarten Lankhorst struct i915_gem_ww_ctx *ww, 17680f0b679SMaarten Lankhorst bool intr) 17798932149SChris Wilson { 17880f0b679SMaarten Lankhorst int ret; 17980f0b679SMaarten Lankhorst 18080f0b679SMaarten Lankhorst if (intr) 18180f0b679SMaarten Lankhorst ret = dma_resv_lock_interruptible(obj->base.resv, ww ? &ww->ctx : NULL); 18280f0b679SMaarten Lankhorst else 18380f0b679SMaarten Lankhorst ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL); 18480f0b679SMaarten Lankhorst 1851c4dbe05SThomas Hellström if (!ret && ww) { 1861c4dbe05SThomas Hellström i915_gem_object_get(obj); 18780f0b679SMaarten Lankhorst list_add_tail(&obj->obj_link, &ww->obj_list); 1881c4dbe05SThomas Hellström } 18980f0b679SMaarten Lankhorst if (ret == -EALREADY) 19080f0b679SMaarten Lankhorst ret = 0; 19180f0b679SMaarten Lankhorst 1921c4dbe05SThomas Hellström if (ret == -EDEADLK) { 1931c4dbe05SThomas Hellström i915_gem_object_get(obj); 19480f0b679SMaarten Lankhorst ww->contended = obj; 1951c4dbe05SThomas Hellström } 19680f0b679SMaarten Lankhorst 19780f0b679SMaarten Lankhorst return ret; 19880f0b679SMaarten Lankhorst } 19980f0b679SMaarten Lankhorst 20080f0b679SMaarten Lankhorst static inline int i915_gem_object_lock(struct drm_i915_gem_object *obj, 20180f0b679SMaarten Lankhorst struct i915_gem_ww_ctx *ww) 20280f0b679SMaarten Lankhorst { 20380f0b679SMaarten Lankhorst return __i915_gem_object_lock(obj, ww, ww && ww->intr); 20480f0b679SMaarten Lankhorst } 20580f0b679SMaarten Lankhorst 20680f0b679SMaarten Lankhorst static inline int i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj, 20780f0b679SMaarten Lankhorst struct i915_gem_ww_ctx *ww) 20880f0b679SMaarten Lankhorst { 20980f0b679SMaarten Lankhorst WARN_ON(ww && !ww->intr); 21080f0b679SMaarten Lankhorst return __i915_gem_object_lock(obj, ww, true); 21198932149SChris Wilson } 21298932149SChris Wilson 213d8be1357SMaarten Lankhorst static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj, 214d8be1357SMaarten Lankhorst struct i915_gem_ww_ctx *ww) 2152850748eSChris Wilson { 216d8be1357SMaarten Lankhorst if (!ww) 2172850748eSChris Wilson return dma_resv_trylock(obj->base.resv); 218d8be1357SMaarten Lankhorst else 219d8be1357SMaarten Lankhorst return ww_mutex_trylock(&obj->base.resv->lock, &ww->ctx); 2202850748eSChris Wilson } 2212850748eSChris Wilson 22298932149SChris Wilson static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) 22398932149SChris Wilson { 224213d5092SThomas Hellström if (obj->ops->adjust_lru) 225213d5092SThomas Hellström obj->ops->adjust_lru(obj); 226213d5092SThomas Hellström 22752791eeeSChristian König dma_resv_unlock(obj->base.resv); 22898932149SChris Wilson } 22998932149SChris Wilson 23098932149SChris Wilson static inline void 23198932149SChris Wilson i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) 23298932149SChris Wilson { 233cc662126SAbdiel Janulgue obj->flags |= I915_BO_READONLY; 23498932149SChris Wilson } 23598932149SChris Wilson 23698932149SChris Wilson static inline bool 23798932149SChris Wilson i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) 23898932149SChris Wilson { 239cc662126SAbdiel Janulgue return obj->flags & I915_BO_READONLY; 24098932149SChris Wilson } 24198932149SChris Wilson 24298932149SChris Wilson static inline bool 2432f0b97caSMatthew Auld i915_gem_object_is_contiguous(const struct drm_i915_gem_object *obj) 2442f0b97caSMatthew Auld { 2452f0b97caSMatthew Auld return obj->flags & I915_BO_ALLOC_CONTIGUOUS; 2462f0b97caSMatthew Auld } 2472f0b97caSMatthew Auld 2482f0b97caSMatthew Auld static inline bool 2497c98501aSMatthew Auld i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj) 2507c98501aSMatthew Auld { 2517c98501aSMatthew Auld return obj->flags & I915_BO_ALLOC_VOLATILE; 2527c98501aSMatthew Auld } 2537c98501aSMatthew Auld 2547c98501aSMatthew Auld static inline void 2557c98501aSMatthew Auld i915_gem_object_set_volatile(struct drm_i915_gem_object *obj) 2567c98501aSMatthew Auld { 2577c98501aSMatthew Auld obj->flags |= I915_BO_ALLOC_VOLATILE; 2587c98501aSMatthew Auld } 2597c98501aSMatthew Auld 2607c98501aSMatthew Auld static inline bool 2610175969eSChris Wilson i915_gem_object_has_tiling_quirk(struct drm_i915_gem_object *obj) 2620175969eSChris Wilson { 2630175969eSChris Wilson return test_bit(I915_TILING_QUIRK_BIT, &obj->flags); 2640175969eSChris Wilson } 2650175969eSChris Wilson 2660175969eSChris Wilson static inline void 2670175969eSChris Wilson i915_gem_object_set_tiling_quirk(struct drm_i915_gem_object *obj) 2680175969eSChris Wilson { 2690175969eSChris Wilson set_bit(I915_TILING_QUIRK_BIT, &obj->flags); 2700175969eSChris Wilson } 2710175969eSChris Wilson 2720175969eSChris Wilson static inline void 2730175969eSChris Wilson i915_gem_object_clear_tiling_quirk(struct drm_i915_gem_object *obj) 2740175969eSChris Wilson { 2750175969eSChris Wilson clear_bit(I915_TILING_QUIRK_BIT, &obj->flags); 2760175969eSChris Wilson } 2770175969eSChris Wilson 2780175969eSChris Wilson static inline bool 279d3ac8d42SDaniele Ceraolo Spurio i915_gem_object_is_protected(const struct drm_i915_gem_object *obj) 280d3ac8d42SDaniele Ceraolo Spurio { 281d3ac8d42SDaniele Ceraolo Spurio return obj->flags & I915_BO_PROTECTED; 282d3ac8d42SDaniele Ceraolo Spurio } 283d3ac8d42SDaniele Ceraolo Spurio 284d3ac8d42SDaniele Ceraolo Spurio static inline bool 2853cbad5d7SChris Wilson i915_gem_object_type_has(const struct drm_i915_gem_object *obj, 2863cbad5d7SChris Wilson unsigned long flags) 2873cbad5d7SChris Wilson { 2883cbad5d7SChris Wilson return obj->ops->flags & flags; 2893cbad5d7SChris Wilson } 2903cbad5d7SChris Wilson 2910ff37575SThomas Hellström bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj); 29298932149SChris Wilson 2930ff37575SThomas Hellström bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj); 2945fbc2c2bSImre Deak 2955fbc2c2bSImre Deak static inline bool 29698932149SChris Wilson i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) 29798932149SChris Wilson { 2983cbad5d7SChris Wilson return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE); 29998932149SChris Wilson } 30098932149SChris Wilson 30198932149SChris Wilson static inline bool 302ebd4a8ecSMatthew Auld i915_gem_object_has_self_managed_shrink_list(const struct drm_i915_gem_object *obj) 303ebd4a8ecSMatthew Auld { 304ebd4a8ecSMatthew Auld return i915_gem_object_type_has(obj, I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST); 305ebd4a8ecSMatthew Auld } 306ebd4a8ecSMatthew Auld 307ebd4a8ecSMatthew Auld static inline bool 30898932149SChris Wilson i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj) 30998932149SChris Wilson { 3103cbad5d7SChris Wilson return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_PROXY); 31198932149SChris Wilson } 31298932149SChris Wilson 31398932149SChris Wilson static inline bool 314f6c26b55SJanusz Krzysztofik i915_gem_object_never_mmap(const struct drm_i915_gem_object *obj) 315a4311745SChris Wilson { 316f6c26b55SJanusz Krzysztofik return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP); 317a4311745SChris Wilson } 318a4311745SChris Wilson 319a4311745SChris Wilson static inline bool 32098932149SChris Wilson i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj) 32198932149SChris Wilson { 3228e7cb179SChris Wilson return READ_ONCE(obj->frontbuffer); 32398932149SChris Wilson } 32498932149SChris Wilson 32598932149SChris Wilson static inline unsigned int 32698932149SChris Wilson i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj) 32798932149SChris Wilson { 32898932149SChris Wilson return obj->tiling_and_stride & TILING_MASK; 32998932149SChris Wilson } 33098932149SChris Wilson 33198932149SChris Wilson static inline bool 33298932149SChris Wilson i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj) 33398932149SChris Wilson { 33498932149SChris Wilson return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; 33598932149SChris Wilson } 33698932149SChris Wilson 33798932149SChris Wilson static inline unsigned int 33898932149SChris Wilson i915_gem_object_get_stride(const struct drm_i915_gem_object *obj) 33998932149SChris Wilson { 34098932149SChris Wilson return obj->tiling_and_stride & STRIDE_MASK; 34198932149SChris Wilson } 34298932149SChris Wilson 34398932149SChris Wilson static inline unsigned int 34498932149SChris Wilson i915_gem_tile_height(unsigned int tiling) 34598932149SChris Wilson { 34698932149SChris Wilson GEM_BUG_ON(!tiling); 34798932149SChris Wilson return tiling == I915_TILING_Y ? 32 : 8; 34898932149SChris Wilson } 34998932149SChris Wilson 35098932149SChris Wilson static inline unsigned int 35198932149SChris Wilson i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj) 35298932149SChris Wilson { 35398932149SChris Wilson return i915_gem_tile_height(i915_gem_object_get_tiling(obj)); 35498932149SChris Wilson } 35598932149SChris Wilson 35698932149SChris Wilson static inline unsigned int 35798932149SChris Wilson i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj) 35898932149SChris Wilson { 35998932149SChris Wilson return (i915_gem_object_get_stride(obj) * 36098932149SChris Wilson i915_gem_object_get_tile_height(obj)); 36198932149SChris Wilson } 36298932149SChris Wilson 36398932149SChris Wilson int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, 36498932149SChris Wilson unsigned int tiling, unsigned int stride); 36598932149SChris Wilson 36698932149SChris Wilson struct scatterlist * 367934941edSTvrtko Ursulin __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 368934941edSTvrtko Ursulin struct i915_gem_object_page_iter *iter, 369934941edSTvrtko Ursulin unsigned int n, 3707d6a276eSJason Ekstrand unsigned int *offset, bool dma); 371934941edSTvrtko Ursulin 372934941edSTvrtko Ursulin static inline struct scatterlist * 37398932149SChris Wilson i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 374934941edSTvrtko Ursulin unsigned int n, 3757d6a276eSJason Ekstrand unsigned int *offset) 376934941edSTvrtko Ursulin { 3777d6a276eSJason Ekstrand return __i915_gem_object_get_sg(obj, &obj->mm.get_page, n, offset, false); 378934941edSTvrtko Ursulin } 379934941edSTvrtko Ursulin 380934941edSTvrtko Ursulin static inline struct scatterlist * 381934941edSTvrtko Ursulin i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj, 382934941edSTvrtko Ursulin unsigned int n, 3837d6a276eSJason Ekstrand unsigned int *offset) 384934941edSTvrtko Ursulin { 3857d6a276eSJason Ekstrand return __i915_gem_object_get_sg(obj, &obj->mm.get_dma_page, n, offset, true); 386934941edSTvrtko Ursulin } 38798932149SChris Wilson 38898932149SChris Wilson struct page * 38998932149SChris Wilson i915_gem_object_get_page(struct drm_i915_gem_object *obj, 39098932149SChris Wilson unsigned int n); 39198932149SChris Wilson 3924993a8a3SDave Airlie struct page * 3934993a8a3SDave Airlie i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 3944993a8a3SDave Airlie unsigned int n); 3954993a8a3SDave Airlie 39698932149SChris Wilson dma_addr_t 39798932149SChris Wilson i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, 39898932149SChris Wilson unsigned long n, 39998932149SChris Wilson unsigned int *len); 40098932149SChris Wilson 40198932149SChris Wilson dma_addr_t 40298932149SChris Wilson i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 40398932149SChris Wilson unsigned long n); 40498932149SChris Wilson 40598932149SChris Wilson void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 40698932149SChris Wilson struct sg_table *pages, 40798932149SChris Wilson unsigned int sg_page_sizes); 408f033428dSChris Wilson 409f033428dSChris Wilson int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 41098932149SChris Wilson int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 41198932149SChris Wilson 41298932149SChris Wilson static inline int __must_check 41398932149SChris Wilson i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 41498932149SChris Wilson { 415cf41a8f1SMaarten Lankhorst assert_object_held(obj); 41698932149SChris Wilson 41798932149SChris Wilson if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) 41898932149SChris Wilson return 0; 41998932149SChris Wilson 42098932149SChris Wilson return __i915_gem_object_get_pages(obj); 42198932149SChris Wilson } 42298932149SChris Wilson 423c858ffa1SMaarten Lankhorst int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj); 424c858ffa1SMaarten Lankhorst 42598932149SChris Wilson static inline bool 42698932149SChris Wilson i915_gem_object_has_pages(struct drm_i915_gem_object *obj) 42798932149SChris Wilson { 42898932149SChris Wilson return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)); 42998932149SChris Wilson } 43098932149SChris Wilson 43198932149SChris Wilson static inline void 43298932149SChris Wilson __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 43398932149SChris Wilson { 43498932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 43598932149SChris Wilson 43698932149SChris Wilson atomic_inc(&obj->mm.pages_pin_count); 43798932149SChris Wilson } 43898932149SChris Wilson 43998932149SChris Wilson static inline bool 44098932149SChris Wilson i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) 44198932149SChris Wilson { 44298932149SChris Wilson return atomic_read(&obj->mm.pages_pin_count); 44398932149SChris Wilson } 44498932149SChris Wilson 44598932149SChris Wilson static inline void 44698932149SChris Wilson __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 44798932149SChris Wilson { 44898932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 44998932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 45098932149SChris Wilson 45198932149SChris Wilson atomic_dec(&obj->mm.pages_pin_count); 45298932149SChris Wilson } 45398932149SChris Wilson 45498932149SChris Wilson static inline void 45598932149SChris Wilson i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 45698932149SChris Wilson { 45798932149SChris Wilson __i915_gem_object_unpin_pages(obj); 45898932149SChris Wilson } 45998932149SChris Wilson 460f86dbacbSDaniel Vetter int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj); 4617ae03459SMatthew Auld int i915_gem_object_truncate(struct drm_i915_gem_object *obj); 46298932149SChris Wilson 46398932149SChris Wilson /** 46498932149SChris Wilson * i915_gem_object_pin_map - return a contiguous mapping of the entire object 46598932149SChris Wilson * @obj: the object to map into kernel address space 46698932149SChris Wilson * @type: the type of mapping, used to select pgprot_t 46798932149SChris Wilson * 46898932149SChris Wilson * Calls i915_gem_object_pin_pages() to prevent reaping of the object's 46998932149SChris Wilson * pages and then returns a contiguous mapping of the backing storage into 47098932149SChris Wilson * the kernel address space. Based on the @type of mapping, the PTE will be 47198932149SChris Wilson * set to either WriteBack or WriteCombine (via pgprot_t). 47298932149SChris Wilson * 47398932149SChris Wilson * The caller is responsible for calling i915_gem_object_unpin_map() when the 47498932149SChris Wilson * mapping is no longer required. 47598932149SChris Wilson * 47698932149SChris Wilson * Returns the pointer through which to access the mapped object, or an 47798932149SChris Wilson * ERR_PTR() on error. 47898932149SChris Wilson */ 47998932149SChris Wilson void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 48098932149SChris Wilson enum i915_map_type type); 48198932149SChris Wilson 48274827b53SMaarten Lankhorst void *__must_check i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, 48374827b53SMaarten Lankhorst enum i915_map_type type); 48474827b53SMaarten Lankhorst 485d09aa852SJani Nikula enum i915_map_type i915_coherent_map_type(struct drm_i915_private *i915, 486d09aa852SJani Nikula struct drm_i915_gem_object *obj, 487d09aa852SJani Nikula bool always_coherent); 488d09aa852SJani Nikula 48998932149SChris Wilson void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 49098932149SChris Wilson unsigned long offset, 49198932149SChris Wilson unsigned long size); 49298932149SChris Wilson static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj) 49398932149SChris Wilson { 49498932149SChris Wilson __i915_gem_object_flush_map(obj, 0, obj->base.size); 49598932149SChris Wilson } 49698932149SChris Wilson 49798932149SChris Wilson /** 49898932149SChris Wilson * i915_gem_object_unpin_map - releases an earlier mapping 49998932149SChris Wilson * @obj: the object to unmap 50098932149SChris Wilson * 50198932149SChris Wilson * After pinning the object and mapping its pages, once you are finished 50298932149SChris Wilson * with your access, call i915_gem_object_unpin_map() to release the pin 50398932149SChris Wilson * upon the mapping. Once the pin count reaches zero, that mapping may be 50498932149SChris Wilson * removed. 50598932149SChris Wilson */ 50698932149SChris Wilson static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) 50798932149SChris Wilson { 50898932149SChris Wilson i915_gem_object_unpin_pages(obj); 50998932149SChris Wilson } 51098932149SChris Wilson 51189d19b2bSChris Wilson void __i915_gem_object_release_map(struct drm_i915_gem_object *obj); 51289d19b2bSChris Wilson 513f0e4a063SChris Wilson int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj, 514f0e4a063SChris Wilson unsigned int *needs_clflush); 515f0e4a063SChris Wilson int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj, 516f0e4a063SChris Wilson unsigned int *needs_clflush); 517f0e4a063SChris Wilson #define CLFLUSH_BEFORE BIT(0) 518f0e4a063SChris Wilson #define CLFLUSH_AFTER BIT(1) 519f0e4a063SChris Wilson #define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER) 520f0e4a063SChris Wilson 521f0e4a063SChris Wilson static inline void 522f0e4a063SChris Wilson i915_gem_object_finish_access(struct drm_i915_gem_object *obj) 523f0e4a063SChris Wilson { 524f0e4a063SChris Wilson i915_gem_object_unpin_pages(obj); 525f0e4a063SChris Wilson } 526f0e4a063SChris Wilson 5271d7f5e6cSChristian König int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, 5281d7f5e6cSChristian König struct dma_fence **fence); 529f6c466b8SMaarten Lankhorst int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, 530f6c466b8SMaarten Lankhorst bool intr); 531bfe53be2SMatthew Auld bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj); 532f6c466b8SMaarten Lankhorst 53398932149SChris Wilson void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, 53498932149SChris Wilson unsigned int cache_level); 53530f1dccdSMatthew Auld bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj); 53698932149SChris Wilson void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); 537c1793ba8SMaarten Lankhorst void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj); 5382ea6ec76SMatthew Auld bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj); 53998932149SChris Wilson 540f0e4a063SChris Wilson int __must_check 541f0e4a063SChris Wilson i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); 542f0e4a063SChris Wilson int __must_check 543f0e4a063SChris Wilson i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); 544f0e4a063SChris Wilson int __must_check 545f0e4a063SChris Wilson i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); 546f0e4a063SChris Wilson struct i915_vma * __must_check 547f0e4a063SChris Wilson i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 5481b321026SMaarten Lankhorst struct i915_gem_ww_ctx *ww, 549f0e4a063SChris Wilson u32 alignment, 5503bb6a442SNiranjana Vishwanathapura const struct i915_gtt_view *view, 551f0e4a063SChris Wilson unsigned int flags); 552f0e4a063SChris Wilson 5531aff1903SChris Wilson void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj); 5541aff1903SChris Wilson void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 555ebd4a8ecSMatthew Auld void __i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 556ebd4a8ecSMatthew Auld void __i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 5571aff1903SChris Wilson void i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 5581aff1903SChris Wilson 5598475355fSChris Wilson static inline void __start_cpu_write(struct drm_i915_gem_object *obj) 5608475355fSChris Wilson { 5618475355fSChris Wilson obj->read_domains = I915_GEM_DOMAIN_CPU; 5628475355fSChris Wilson obj->write_domain = I915_GEM_DOMAIN_CPU; 5632ea6ec76SMatthew Auld if (i915_gem_cpu_write_needs_clflush(obj)) 5648475355fSChris Wilson obj->cache_dirty = true; 5658475355fSChris Wilson } 56698932149SChris Wilson 567f99e67f1SChris Wilson void i915_gem_fence_wait_priority(struct dma_fence *fence, 568f99e67f1SChris Wilson const struct i915_sched_attr *attr); 569f99e67f1SChris Wilson 570d45a1a53SChris Wilson int i915_gem_object_wait(struct drm_i915_gem_object *obj, 571d45a1a53SChris Wilson unsigned int flags, 572d45a1a53SChris Wilson long timeout); 573d45a1a53SChris Wilson int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 574d45a1a53SChris Wilson unsigned int flags, 575d45a1a53SChris Wilson const struct i915_sched_attr *attr); 576d45a1a53SChris Wilson 577da42104fSChris Wilson void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, 578da42104fSChris Wilson enum fb_op_origin origin); 579da42104fSChris Wilson void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, 580da42104fSChris Wilson enum fb_op_origin origin); 581da42104fSChris Wilson 582da42104fSChris Wilson static inline void 583da42104fSChris Wilson i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, 584da42104fSChris Wilson enum fb_op_origin origin) 585da42104fSChris Wilson { 586da42104fSChris Wilson if (unlikely(rcu_access_pointer(obj->frontbuffer))) 587da42104fSChris Wilson __i915_gem_object_flush_frontbuffer(obj, origin); 588da42104fSChris Wilson } 589da42104fSChris Wilson 590da42104fSChris Wilson static inline void 591da42104fSChris Wilson i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, 592da42104fSChris Wilson enum fb_op_origin origin) 593da42104fSChris Wilson { 594da42104fSChris Wilson if (unlikely(rcu_access_pointer(obj->frontbuffer))) 595da42104fSChris Wilson __i915_gem_object_invalidate_frontbuffer(obj, origin); 596da42104fSChris Wilson } 597da42104fSChris Wilson 5985fbc2c2bSImre Deak int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size); 5995fbc2c2bSImre Deak 60041a9c75dSChris Wilson bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj); 60141a9c75dSChris Wilson 602213d5092SThomas Hellström void __i915_gem_free_object_rcu(struct rcu_head *head); 603213d5092SThomas Hellström 604068396bbSThomas Hellström void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj); 605068396bbSThomas Hellström 606213d5092SThomas Hellström void __i915_gem_free_object(struct drm_i915_gem_object *obj); 607213d5092SThomas Hellström 608213d5092SThomas Hellström bool i915_gem_object_evictable(struct drm_i915_gem_object *obj); 609213d5092SThomas Hellström 6102e53d7c1SThomas Hellström bool i915_gem_object_migratable(struct drm_i915_gem_object *obj); 6112e53d7c1SThomas Hellström 612b6e913e1SThomas Hellström int i915_gem_object_migrate(struct drm_i915_gem_object *obj, 613b6e913e1SThomas Hellström struct i915_gem_ww_ctx *ww, 614b6e913e1SThomas Hellström enum intel_region_id id); 615*999f4562SMatthew Auld int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, 616*999f4562SMatthew Auld struct i915_gem_ww_ctx *ww, 617*999f4562SMatthew Auld enum intel_region_id id, 618*999f4562SMatthew Auld unsigned int flags); 619b6e913e1SThomas Hellström 620b6e913e1SThomas Hellström bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, 621b6e913e1SThomas Hellström enum intel_region_id id); 622b6e913e1SThomas Hellström 623b6e913e1SThomas Hellström int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, 624b6e913e1SThomas Hellström unsigned int flags); 625b6e913e1SThomas Hellström 626b3f450d9SMatthew Auld bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, 627b3f450d9SMatthew Auld enum intel_memory_type type); 628b3f450d9SMatthew Auld 629efeb3cafSMatthew Auld bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj); 630efeb3cafSMatthew Auld 631cad7109aSThomas Hellström int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 6327ae03459SMatthew Auld size_t size, struct intel_memory_region *mr, 6337ae03459SMatthew Auld struct address_space *mapping, 6347ae03459SMatthew Auld unsigned int max_segment); 635cad7109aSThomas Hellström void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 6367ae03459SMatthew Auld bool dirty, bool backup); 6377ae03459SMatthew Auld void __shmem_writeback(size_t size, struct address_space *mapping); 6387ae03459SMatthew Auld 639ed29c269SMaarten Lankhorst #ifdef CONFIG_MMU_NOTIFIER 640ed29c269SMaarten Lankhorst static inline bool 641ed29c269SMaarten Lankhorst i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) 642ed29c269SMaarten Lankhorst { 643ed29c269SMaarten Lankhorst return obj->userptr.notifier.mm; 644ed29c269SMaarten Lankhorst } 645ed29c269SMaarten Lankhorst 646ed29c269SMaarten Lankhorst int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj); 647ed29c269SMaarten Lankhorst int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj); 648ed29c269SMaarten Lankhorst int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj); 649ed29c269SMaarten Lankhorst #else 650ed29c269SMaarten Lankhorst static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) { return false; } 651ed29c269SMaarten Lankhorst 652ed29c269SMaarten Lankhorst static inline int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 653ed29c269SMaarten Lankhorst static inline int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 654ed29c269SMaarten Lankhorst static inline int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 655ed29c269SMaarten Lankhorst 656ed29c269SMaarten Lankhorst #endif 657ed29c269SMaarten Lankhorst 65898932149SChris Wilson #endif 659