198932149SChris Wilson /* 298932149SChris Wilson * SPDX-License-Identifier: MIT 398932149SChris Wilson * 498932149SChris Wilson * Copyright © 2016 Intel Corporation 598932149SChris Wilson */ 698932149SChris Wilson 798932149SChris Wilson #ifndef __I915_GEM_OBJECT_H__ 898932149SChris Wilson #define __I915_GEM_OBJECT_H__ 998932149SChris Wilson 1098932149SChris Wilson #include <drm/drm_gem.h> 1198932149SChris Wilson #include <drm/drm_file.h> 1298932149SChris Wilson #include <drm/drm_device.h> 1398932149SChris Wilson 14da42104fSChris Wilson #include "display/intel_frontbuffer.h" 15b3f450d9SMatthew Auld #include "intel_memory_region.h" 1698932149SChris Wilson #include "i915_gem_object_types.h" 17f0e4a063SChris Wilson #include "i915_gem_gtt.h" 185c43ec5dSThomas Hellström #include "i915_gem_ww.h" 192c86e55dSMatthew Auld #include "i915_vma_types.h" 20f0e4a063SChris Wilson 21b6e913e1SThomas Hellström enum intel_region_id; 22b6e913e1SThomas Hellström 23ae2fb480SMatthew Auld /* 24ae2fb480SMatthew Auld * XXX: There is a prevalence of the assumption that we fit the 25ae2fb480SMatthew Auld * object's page count inside a 32bit _signed_ variable. Let's document 26ae2fb480SMatthew Auld * this and catch if we ever need to fix it. In the meantime, if you do 27ae2fb480SMatthew Auld * spot such a local variable, please consider fixing! 28ae2fb480SMatthew Auld * 29ae2fb480SMatthew Auld * Aside from our own locals (for which we have no excuse!): 30*f47e6306SChris Wilson * - sg_table embeds unsigned int for nents 31*f47e6306SChris Wilson * 32*f47e6306SChris Wilson * We can check for invalidly typed locals with typecheck(), see for example 33*f47e6306SChris Wilson * i915_gem_object_get_sg(). 34ae2fb480SMatthew Auld */ 35ae2fb480SMatthew Auld #define GEM_CHECK_SIZE_OVERFLOW(sz) \ 36ae2fb480SMatthew Auld GEM_WARN_ON((sz) >> PAGE_SHIFT > INT_MAX) 37ae2fb480SMatthew Auld 38ae2fb480SMatthew Auld static inline bool i915_gem_object_size_2big(u64 size) 39ae2fb480SMatthew Auld { 40ae2fb480SMatthew Auld struct drm_i915_gem_object *obj; 41ae2fb480SMatthew Auld 42ae2fb480SMatthew Auld if (GEM_CHECK_SIZE_OVERFLOW(size)) 43ae2fb480SMatthew Auld return true; 44ae2fb480SMatthew Auld 45ae2fb480SMatthew Auld if (overflows_type(size, obj->base.size)) 46ae2fb480SMatthew Auld return true; 47ae2fb480SMatthew Auld 48ae2fb480SMatthew Auld return false; 49ae2fb480SMatthew Auld } 50ae2fb480SMatthew Auld 518475355fSChris Wilson void i915_gem_init__objects(struct drm_i915_private *i915); 528475355fSChris Wilson 53c8ad09afSDaniel Vetter void i915_objects_module_exit(void); 54c8ad09afSDaniel Vetter int i915_objects_module_init(void); 55c8ad09afSDaniel Vetter 5698932149SChris Wilson struct drm_i915_gem_object *i915_gem_object_alloc(void); 5798932149SChris Wilson void i915_gem_object_free(struct drm_i915_gem_object *obj); 5898932149SChris Wilson 598475355fSChris Wilson void i915_gem_object_init(struct drm_i915_gem_object *obj, 607867d709SChris Wilson const struct drm_i915_gem_object_ops *ops, 61c471748dSMaarten Lankhorst struct lock_class_key *key, 62c471748dSMaarten Lankhorst unsigned alloc_flags); 63068396bbSThomas Hellström 64068396bbSThomas Hellström void __i915_gem_object_fini(struct drm_i915_gem_object *obj); 65068396bbSThomas Hellström 668475355fSChris Wilson struct drm_i915_gem_object * 67da1184cdSMatthew Auld i915_gem_object_create_shmem(struct drm_i915_private *i915, 68da1184cdSMatthew Auld resource_size_t size); 698475355fSChris Wilson struct drm_i915_gem_object * 708475355fSChris Wilson i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, 71da1184cdSMatthew Auld const void *data, resource_size_t size); 72bf947c98SJason Ekstrand struct drm_i915_gem_object * 73bf947c98SJason Ekstrand __i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, 74bf947c98SJason Ekstrand struct intel_memory_region **placements, 75bf947c98SJason Ekstrand unsigned int n_placements); 768475355fSChris Wilson 778475355fSChris Wilson extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops; 78ed29c269SMaarten Lankhorst 798475355fSChris Wilson void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 808475355fSChris Wilson struct sg_table *pages, 818475355fSChris Wilson bool needs_clflush); 828475355fSChris Wilson 83a6117097SMaarten Lankhorst int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj, 84a6117097SMaarten Lankhorst const struct drm_i915_gem_pwrite *args); 85a6117097SMaarten Lankhorst int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj, 86a6117097SMaarten Lankhorst const struct drm_i915_gem_pread *args); 87a6117097SMaarten Lankhorst 88f033428dSChris Wilson int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align); 89a85fffe3SMaarten Lankhorst void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, 90a85fffe3SMaarten Lankhorst struct sg_table *pages); 91a6117097SMaarten Lankhorst void i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 92a6117097SMaarten Lankhorst struct sg_table *pages); 93a6117097SMaarten Lankhorst 948475355fSChris Wilson void i915_gem_flush_free_objects(struct drm_i915_private *i915); 958475355fSChris Wilson 96f033428dSChris Wilson struct sg_table * 97f033428dSChris Wilson __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj); 98f033428dSChris Wilson 9998932149SChris Wilson /** 10098932149SChris Wilson * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle 10198932149SChris Wilson * @filp: DRM file private date 10298932149SChris Wilson * @handle: userspace handle 10398932149SChris Wilson * 10498932149SChris Wilson * Returns: 10598932149SChris Wilson * 10698932149SChris Wilson * A pointer to the object named by the handle if such exists on @filp, NULL 10798932149SChris Wilson * otherwise. This object is only valid whilst under the RCU read lock, and 10898932149SChris Wilson * note carefully the object may be in the process of being destroyed. 10998932149SChris Wilson */ 11098932149SChris Wilson static inline struct drm_i915_gem_object * 11198932149SChris Wilson i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) 11298932149SChris Wilson { 11398932149SChris Wilson #ifdef CONFIG_LOCKDEP 11498932149SChris Wilson WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); 11598932149SChris Wilson #endif 11698932149SChris Wilson return idr_find(&file->object_idr, handle); 11798932149SChris Wilson } 11898932149SChris Wilson 11998932149SChris Wilson static inline struct drm_i915_gem_object * 120280d14a6SChris Wilson i915_gem_object_get_rcu(struct drm_i915_gem_object *obj) 121280d14a6SChris Wilson { 122280d14a6SChris Wilson if (obj && !kref_get_unless_zero(&obj->base.refcount)) 123280d14a6SChris Wilson obj = NULL; 124280d14a6SChris Wilson 125280d14a6SChris Wilson return obj; 126280d14a6SChris Wilson } 127280d14a6SChris Wilson 128280d14a6SChris Wilson static inline struct drm_i915_gem_object * 12998932149SChris Wilson i915_gem_object_lookup(struct drm_file *file, u32 handle) 13098932149SChris Wilson { 13198932149SChris Wilson struct drm_i915_gem_object *obj; 13298932149SChris Wilson 13398932149SChris Wilson rcu_read_lock(); 13498932149SChris Wilson obj = i915_gem_object_lookup_rcu(file, handle); 135280d14a6SChris Wilson obj = i915_gem_object_get_rcu(obj); 13698932149SChris Wilson rcu_read_unlock(); 13798932149SChris Wilson 13898932149SChris Wilson return obj; 13998932149SChris Wilson } 14098932149SChris Wilson 14198932149SChris Wilson __deprecated 142b5893ffcSJanusz Krzysztofik struct drm_gem_object * 14398932149SChris Wilson drm_gem_object_lookup(struct drm_file *file, u32 handle); 14498932149SChris Wilson 14598932149SChris Wilson __attribute__((nonnull)) 14698932149SChris Wilson static inline struct drm_i915_gem_object * 14798932149SChris Wilson i915_gem_object_get(struct drm_i915_gem_object *obj) 14898932149SChris Wilson { 14998932149SChris Wilson drm_gem_object_get(&obj->base); 15098932149SChris Wilson return obj; 15198932149SChris Wilson } 15298932149SChris Wilson 15398932149SChris Wilson __attribute__((nonnull)) 15498932149SChris Wilson static inline void 15598932149SChris Wilson i915_gem_object_put(struct drm_i915_gem_object *obj) 15698932149SChris Wilson { 15798932149SChris Wilson __drm_gem_object_put(&obj->base); 15898932149SChris Wilson } 15998932149SChris Wilson 16052791eeeSChristian König #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv) 1616951e589SChris Wilson 162a3258dbdSThomas Hellström /* 163a3258dbdSThomas Hellström * If more than one potential simultaneous locker, assert held. 164a3258dbdSThomas Hellström */ 1650ff37575SThomas Hellström static inline void assert_object_held_shared(const struct drm_i915_gem_object *obj) 166a3258dbdSThomas Hellström { 167a3258dbdSThomas Hellström /* 168a3258dbdSThomas Hellström * Note mm list lookup is protected by 169a3258dbdSThomas Hellström * kref_get_unless_zero(). 170a3258dbdSThomas Hellström */ 171a3258dbdSThomas Hellström if (IS_ENABLED(CONFIG_LOCKDEP) && 172a3258dbdSThomas Hellström kref_read(&obj->base.refcount) > 0) 173cf41a8f1SMaarten Lankhorst assert_object_held(obj); 174a3258dbdSThomas Hellström } 175a3258dbdSThomas Hellström 17680f0b679SMaarten Lankhorst static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj, 17780f0b679SMaarten Lankhorst struct i915_gem_ww_ctx *ww, 17880f0b679SMaarten Lankhorst bool intr) 17998932149SChris Wilson { 18080f0b679SMaarten Lankhorst int ret; 18180f0b679SMaarten Lankhorst 18280f0b679SMaarten Lankhorst if (intr) 18380f0b679SMaarten Lankhorst ret = dma_resv_lock_interruptible(obj->base.resv, ww ? &ww->ctx : NULL); 18480f0b679SMaarten Lankhorst else 18580f0b679SMaarten Lankhorst ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL); 18680f0b679SMaarten Lankhorst 1871c4dbe05SThomas Hellström if (!ret && ww) { 1881c4dbe05SThomas Hellström i915_gem_object_get(obj); 18980f0b679SMaarten Lankhorst list_add_tail(&obj->obj_link, &ww->obj_list); 1901c4dbe05SThomas Hellström } 19180f0b679SMaarten Lankhorst if (ret == -EALREADY) 19280f0b679SMaarten Lankhorst ret = 0; 19380f0b679SMaarten Lankhorst 1941c4dbe05SThomas Hellström if (ret == -EDEADLK) { 1951c4dbe05SThomas Hellström i915_gem_object_get(obj); 19680f0b679SMaarten Lankhorst ww->contended = obj; 1971c4dbe05SThomas Hellström } 19880f0b679SMaarten Lankhorst 19980f0b679SMaarten Lankhorst return ret; 20080f0b679SMaarten Lankhorst } 20180f0b679SMaarten Lankhorst 20280f0b679SMaarten Lankhorst static inline int i915_gem_object_lock(struct drm_i915_gem_object *obj, 20380f0b679SMaarten Lankhorst struct i915_gem_ww_ctx *ww) 20480f0b679SMaarten Lankhorst { 20580f0b679SMaarten Lankhorst return __i915_gem_object_lock(obj, ww, ww && ww->intr); 20680f0b679SMaarten Lankhorst } 20780f0b679SMaarten Lankhorst 20880f0b679SMaarten Lankhorst static inline int i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj, 20980f0b679SMaarten Lankhorst struct i915_gem_ww_ctx *ww) 21080f0b679SMaarten Lankhorst { 21180f0b679SMaarten Lankhorst WARN_ON(ww && !ww->intr); 21280f0b679SMaarten Lankhorst return __i915_gem_object_lock(obj, ww, true); 21398932149SChris Wilson } 21498932149SChris Wilson 215d8be1357SMaarten Lankhorst static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj, 216d8be1357SMaarten Lankhorst struct i915_gem_ww_ctx *ww) 2172850748eSChris Wilson { 218d8be1357SMaarten Lankhorst if (!ww) 2192850748eSChris Wilson return dma_resv_trylock(obj->base.resv); 220d8be1357SMaarten Lankhorst else 221d8be1357SMaarten Lankhorst return ww_mutex_trylock(&obj->base.resv->lock, &ww->ctx); 2222850748eSChris Wilson } 2232850748eSChris Wilson 22498932149SChris Wilson static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) 22598932149SChris Wilson { 226213d5092SThomas Hellström if (obj->ops->adjust_lru) 227213d5092SThomas Hellström obj->ops->adjust_lru(obj); 228213d5092SThomas Hellström 22952791eeeSChristian König dma_resv_unlock(obj->base.resv); 23098932149SChris Wilson } 23198932149SChris Wilson 23298932149SChris Wilson static inline void 23398932149SChris Wilson i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) 23498932149SChris Wilson { 235cc662126SAbdiel Janulgue obj->flags |= I915_BO_READONLY; 23698932149SChris Wilson } 23798932149SChris Wilson 23898932149SChris Wilson static inline bool 23998932149SChris Wilson i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) 24098932149SChris Wilson { 241cc662126SAbdiel Janulgue return obj->flags & I915_BO_READONLY; 24298932149SChris Wilson } 24398932149SChris Wilson 24498932149SChris Wilson static inline bool 2452f0b97caSMatthew Auld i915_gem_object_is_contiguous(const struct drm_i915_gem_object *obj) 2462f0b97caSMatthew Auld { 2472f0b97caSMatthew Auld return obj->flags & I915_BO_ALLOC_CONTIGUOUS; 2482f0b97caSMatthew Auld } 2492f0b97caSMatthew Auld 2502f0b97caSMatthew Auld static inline bool 2517c98501aSMatthew Auld i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj) 2527c98501aSMatthew Auld { 2537c98501aSMatthew Auld return obj->flags & I915_BO_ALLOC_VOLATILE; 2547c98501aSMatthew Auld } 2557c98501aSMatthew Auld 2567c98501aSMatthew Auld static inline void 2577c98501aSMatthew Auld i915_gem_object_set_volatile(struct drm_i915_gem_object *obj) 2587c98501aSMatthew Auld { 2597c98501aSMatthew Auld obj->flags |= I915_BO_ALLOC_VOLATILE; 2607c98501aSMatthew Auld } 2617c98501aSMatthew Auld 2627c98501aSMatthew Auld static inline bool 2630175969eSChris Wilson i915_gem_object_has_tiling_quirk(struct drm_i915_gem_object *obj) 2640175969eSChris Wilson { 2650175969eSChris Wilson return test_bit(I915_TILING_QUIRK_BIT, &obj->flags); 2660175969eSChris Wilson } 2670175969eSChris Wilson 2680175969eSChris Wilson static inline void 2690175969eSChris Wilson i915_gem_object_set_tiling_quirk(struct drm_i915_gem_object *obj) 2700175969eSChris Wilson { 2710175969eSChris Wilson set_bit(I915_TILING_QUIRK_BIT, &obj->flags); 2720175969eSChris Wilson } 2730175969eSChris Wilson 2740175969eSChris Wilson static inline void 2750175969eSChris Wilson i915_gem_object_clear_tiling_quirk(struct drm_i915_gem_object *obj) 2760175969eSChris Wilson { 2770175969eSChris Wilson clear_bit(I915_TILING_QUIRK_BIT, &obj->flags); 2780175969eSChris Wilson } 2790175969eSChris Wilson 2800175969eSChris Wilson static inline bool 281d3ac8d42SDaniele Ceraolo Spurio i915_gem_object_is_protected(const struct drm_i915_gem_object *obj) 282d3ac8d42SDaniele Ceraolo Spurio { 283d3ac8d42SDaniele Ceraolo Spurio return obj->flags & I915_BO_PROTECTED; 284d3ac8d42SDaniele Ceraolo Spurio } 285d3ac8d42SDaniele Ceraolo Spurio 286d3ac8d42SDaniele Ceraolo Spurio static inline bool 2873cbad5d7SChris Wilson i915_gem_object_type_has(const struct drm_i915_gem_object *obj, 2883cbad5d7SChris Wilson unsigned long flags) 2893cbad5d7SChris Wilson { 2903cbad5d7SChris Wilson return obj->ops->flags & flags; 2913cbad5d7SChris Wilson } 2923cbad5d7SChris Wilson 2930ff37575SThomas Hellström bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj); 29498932149SChris Wilson 2950ff37575SThomas Hellström bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj); 2965fbc2c2bSImre Deak 2975fbc2c2bSImre Deak static inline bool 29898932149SChris Wilson i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) 29998932149SChris Wilson { 3003cbad5d7SChris Wilson return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE); 30198932149SChris Wilson } 30298932149SChris Wilson 30398932149SChris Wilson static inline bool 304ebd4a8ecSMatthew Auld i915_gem_object_has_self_managed_shrink_list(const struct drm_i915_gem_object *obj) 305ebd4a8ecSMatthew Auld { 306ebd4a8ecSMatthew Auld return i915_gem_object_type_has(obj, I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST); 307ebd4a8ecSMatthew Auld } 308ebd4a8ecSMatthew Auld 309ebd4a8ecSMatthew Auld static inline bool 31098932149SChris Wilson i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj) 31198932149SChris Wilson { 3123cbad5d7SChris Wilson return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_PROXY); 31398932149SChris Wilson } 31498932149SChris Wilson 31598932149SChris Wilson static inline bool 316f6c26b55SJanusz Krzysztofik i915_gem_object_never_mmap(const struct drm_i915_gem_object *obj) 317a4311745SChris Wilson { 318f6c26b55SJanusz Krzysztofik return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP); 319a4311745SChris Wilson } 320a4311745SChris Wilson 321a4311745SChris Wilson static inline bool 32298932149SChris Wilson i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj) 32398932149SChris Wilson { 3248e7cb179SChris Wilson return READ_ONCE(obj->frontbuffer); 32598932149SChris Wilson } 32698932149SChris Wilson 32798932149SChris Wilson static inline unsigned int 32898932149SChris Wilson i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj) 32998932149SChris Wilson { 33098932149SChris Wilson return obj->tiling_and_stride & TILING_MASK; 33198932149SChris Wilson } 33298932149SChris Wilson 33398932149SChris Wilson static inline bool 33498932149SChris Wilson i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj) 33598932149SChris Wilson { 33698932149SChris Wilson return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; 33798932149SChris Wilson } 33898932149SChris Wilson 33998932149SChris Wilson static inline unsigned int 34098932149SChris Wilson i915_gem_object_get_stride(const struct drm_i915_gem_object *obj) 34198932149SChris Wilson { 34298932149SChris Wilson return obj->tiling_and_stride & STRIDE_MASK; 34398932149SChris Wilson } 34498932149SChris Wilson 34598932149SChris Wilson static inline unsigned int 34698932149SChris Wilson i915_gem_tile_height(unsigned int tiling) 34798932149SChris Wilson { 34898932149SChris Wilson GEM_BUG_ON(!tiling); 34998932149SChris Wilson return tiling == I915_TILING_Y ? 32 : 8; 35098932149SChris Wilson } 35198932149SChris Wilson 35298932149SChris Wilson static inline unsigned int 35398932149SChris Wilson i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj) 35498932149SChris Wilson { 35598932149SChris Wilson return i915_gem_tile_height(i915_gem_object_get_tiling(obj)); 35698932149SChris Wilson } 35798932149SChris Wilson 35898932149SChris Wilson static inline unsigned int 35998932149SChris Wilson i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj) 36098932149SChris Wilson { 36198932149SChris Wilson return (i915_gem_object_get_stride(obj) * 36298932149SChris Wilson i915_gem_object_get_tile_height(obj)); 36398932149SChris Wilson } 36498932149SChris Wilson 36598932149SChris Wilson int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, 36698932149SChris Wilson unsigned int tiling, unsigned int stride); 36798932149SChris Wilson 368*f47e6306SChris Wilson /** 369*f47e6306SChris Wilson * __i915_gem_object_page_iter_get_sg - helper to find the target scatterlist 370*f47e6306SChris Wilson * pointer and the target page position using pgoff_t n input argument and 371*f47e6306SChris Wilson * i915_gem_object_page_iter 372*f47e6306SChris Wilson * @obj: i915 GEM buffer object 373*f47e6306SChris Wilson * @iter: i915 GEM buffer object page iterator 374*f47e6306SChris Wilson * @n: page offset 375*f47e6306SChris Wilson * @offset: searched physical offset, 376*f47e6306SChris Wilson * it will be used for returning physical page offset value 377*f47e6306SChris Wilson * 378*f47e6306SChris Wilson * Context: Takes and releases the mutex lock of the i915_gem_object_page_iter. 379*f47e6306SChris Wilson * Takes and releases the RCU lock to search the radix_tree of 380*f47e6306SChris Wilson * i915_gem_object_page_iter. 381*f47e6306SChris Wilson * 382*f47e6306SChris Wilson * Returns: 383*f47e6306SChris Wilson * The target scatterlist pointer and the target page position. 384*f47e6306SChris Wilson * 385*f47e6306SChris Wilson * Recommended to use wrapper macro: i915_gem_object_page_iter_get_sg() 386*f47e6306SChris Wilson */ 38798932149SChris Wilson struct scatterlist * 388*f47e6306SChris Wilson __i915_gem_object_page_iter_get_sg(struct drm_i915_gem_object *obj, 389934941edSTvrtko Ursulin struct i915_gem_object_page_iter *iter, 390*f47e6306SChris Wilson pgoff_t n, 391*f47e6306SChris Wilson unsigned int *offset); 392934941edSTvrtko Ursulin 393*f47e6306SChris Wilson /** 394*f47e6306SChris Wilson * i915_gem_object_page_iter_get_sg - wrapper macro for 395*f47e6306SChris Wilson * __i915_gem_object_page_iter_get_sg() 396*f47e6306SChris Wilson * @obj: i915 GEM buffer object 397*f47e6306SChris Wilson * @it: i915 GEM buffer object page iterator 398*f47e6306SChris Wilson * @n: page offset 399*f47e6306SChris Wilson * @offset: searched physical offset, 400*f47e6306SChris Wilson * it will be used for returning physical page offset value 401*f47e6306SChris Wilson * 402*f47e6306SChris Wilson * Context: Takes and releases the mutex lock of the i915_gem_object_page_iter. 403*f47e6306SChris Wilson * Takes and releases the RCU lock to search the radix_tree of 404*f47e6306SChris Wilson * i915_gem_object_page_iter. 405*f47e6306SChris Wilson * 406*f47e6306SChris Wilson * Returns: 407*f47e6306SChris Wilson * The target scatterlist pointer and the target page position. 408*f47e6306SChris Wilson * 409*f47e6306SChris Wilson * In order to avoid the truncation of the input parameter, it checks the page 410*f47e6306SChris Wilson * offset n's type from the input parameter before calling 411*f47e6306SChris Wilson * __i915_gem_object_page_iter_get_sg(). 412*f47e6306SChris Wilson */ 413*f47e6306SChris Wilson #define i915_gem_object_page_iter_get_sg(obj, it, n, offset) ({ \ 414*f47e6306SChris Wilson static_assert(castable_to_type(n, pgoff_t)); \ 415*f47e6306SChris Wilson __i915_gem_object_page_iter_get_sg(obj, it, n, offset); \ 416*f47e6306SChris Wilson }) 417*f47e6306SChris Wilson 418*f47e6306SChris Wilson /** 419*f47e6306SChris Wilson * __i915_gem_object_get_sg - helper to find the target scatterlist 420*f47e6306SChris Wilson * pointer and the target page position using pgoff_t n input argument and 421*f47e6306SChris Wilson * drm_i915_gem_object. It uses an internal shmem scatterlist lookup function. 422*f47e6306SChris Wilson * @obj: i915 GEM buffer object 423*f47e6306SChris Wilson * @n: page offset 424*f47e6306SChris Wilson * @offset: searched physical offset, 425*f47e6306SChris Wilson * it will be used for returning physical page offset value 426*f47e6306SChris Wilson * 427*f47e6306SChris Wilson * It uses drm_i915_gem_object's internal shmem scatterlist lookup function as 428*f47e6306SChris Wilson * i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg(). 429*f47e6306SChris Wilson * 430*f47e6306SChris Wilson * Returns: 431*f47e6306SChris Wilson * The target scatterlist pointer and the target page position. 432*f47e6306SChris Wilson * 433*f47e6306SChris Wilson * Recommended to use wrapper macro: i915_gem_object_get_sg() 434*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() 435*f47e6306SChris Wilson */ 436934941edSTvrtko Ursulin static inline struct scatterlist * 437*f47e6306SChris Wilson __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, pgoff_t n, 4387d6a276eSJason Ekstrand unsigned int *offset) 439934941edSTvrtko Ursulin { 440*f47e6306SChris Wilson return __i915_gem_object_page_iter_get_sg(obj, &obj->mm.get_page, n, offset); 441934941edSTvrtko Ursulin } 442934941edSTvrtko Ursulin 443*f47e6306SChris Wilson /** 444*f47e6306SChris Wilson * i915_gem_object_get_sg - wrapper macro for __i915_gem_object_get_sg() 445*f47e6306SChris Wilson * @obj: i915 GEM buffer object 446*f47e6306SChris Wilson * @n: page offset 447*f47e6306SChris Wilson * @offset: searched physical offset, 448*f47e6306SChris Wilson * it will be used for returning physical page offset value 449*f47e6306SChris Wilson * 450*f47e6306SChris Wilson * Returns: 451*f47e6306SChris Wilson * The target scatterlist pointer and the target page position. 452*f47e6306SChris Wilson * 453*f47e6306SChris Wilson * In order to avoid the truncation of the input parameter, it checks the page 454*f47e6306SChris Wilson * offset n's type from the input parameter before calling 455*f47e6306SChris Wilson * __i915_gem_object_get_sg(). 456*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() 457*f47e6306SChris Wilson */ 458*f47e6306SChris Wilson #define i915_gem_object_get_sg(obj, n, offset) ({ \ 459*f47e6306SChris Wilson static_assert(castable_to_type(n, pgoff_t)); \ 460*f47e6306SChris Wilson __i915_gem_object_get_sg(obj, n, offset); \ 461*f47e6306SChris Wilson }) 462*f47e6306SChris Wilson 463*f47e6306SChris Wilson /** 464*f47e6306SChris Wilson * __i915_gem_object_get_sg_dma - helper to find the target scatterlist 465*f47e6306SChris Wilson * pointer and the target page position using pgoff_t n input argument and 466*f47e6306SChris Wilson * drm_i915_gem_object. It uses an internal DMA mapped scatterlist lookup function 467*f47e6306SChris Wilson * @obj: i915 GEM buffer object 468*f47e6306SChris Wilson * @n: page offset 469*f47e6306SChris Wilson * @offset: searched physical offset, 470*f47e6306SChris Wilson * it will be used for returning physical page offset value 471*f47e6306SChris Wilson * 472*f47e6306SChris Wilson * It uses drm_i915_gem_object's internal DMA mapped scatterlist lookup function 473*f47e6306SChris Wilson * as i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg(). 474*f47e6306SChris Wilson * 475*f47e6306SChris Wilson * Returns: 476*f47e6306SChris Wilson * The target scatterlist pointer and the target page position. 477*f47e6306SChris Wilson * 478*f47e6306SChris Wilson * Recommended to use wrapper macro: i915_gem_object_get_sg_dma() 479*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() 480*f47e6306SChris Wilson */ 481934941edSTvrtko Ursulin static inline struct scatterlist * 482*f47e6306SChris Wilson __i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj, pgoff_t n, 4837d6a276eSJason Ekstrand unsigned int *offset) 484934941edSTvrtko Ursulin { 485*f47e6306SChris Wilson return __i915_gem_object_page_iter_get_sg(obj, &obj->mm.get_dma_page, n, offset); 486934941edSTvrtko Ursulin } 48798932149SChris Wilson 488*f47e6306SChris Wilson /** 489*f47e6306SChris Wilson * i915_gem_object_get_sg_dma - wrapper macro for __i915_gem_object_get_sg_dma() 490*f47e6306SChris Wilson * @obj: i915 GEM buffer object 491*f47e6306SChris Wilson * @n: page offset 492*f47e6306SChris Wilson * @offset: searched physical offset, 493*f47e6306SChris Wilson * it will be used for returning physical page offset value 494*f47e6306SChris Wilson * 495*f47e6306SChris Wilson * Returns: 496*f47e6306SChris Wilson * The target scatterlist pointer and the target page position. 497*f47e6306SChris Wilson * 498*f47e6306SChris Wilson * In order to avoid the truncation of the input parameter, it checks the page 499*f47e6306SChris Wilson * offset n's type from the input parameter before calling 500*f47e6306SChris Wilson * __i915_gem_object_get_sg_dma(). 501*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() 502*f47e6306SChris Wilson */ 503*f47e6306SChris Wilson #define i915_gem_object_get_sg_dma(obj, n, offset) ({ \ 504*f47e6306SChris Wilson static_assert(castable_to_type(n, pgoff_t)); \ 505*f47e6306SChris Wilson __i915_gem_object_get_sg_dma(obj, n, offset); \ 506*f47e6306SChris Wilson }) 50798932149SChris Wilson 508*f47e6306SChris Wilson /** 509*f47e6306SChris Wilson * __i915_gem_object_get_page - helper to find the target page with a page offset 510*f47e6306SChris Wilson * @obj: i915 GEM buffer object 511*f47e6306SChris Wilson * @n: page offset 512*f47e6306SChris Wilson * 513*f47e6306SChris Wilson * It uses drm_i915_gem_object's internal shmem scatterlist lookup function as 514*f47e6306SChris Wilson * i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg() 515*f47e6306SChris Wilson * internally. 516*f47e6306SChris Wilson * 517*f47e6306SChris Wilson * Returns: 518*f47e6306SChris Wilson * The target page pointer. 519*f47e6306SChris Wilson * 520*f47e6306SChris Wilson * Recommended to use wrapper macro: i915_gem_object_get_page() 521*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() 522*f47e6306SChris Wilson */ 5234993a8a3SDave Airlie struct page * 524*f47e6306SChris Wilson __i915_gem_object_get_page(struct drm_i915_gem_object *obj, pgoff_t n); 5254993a8a3SDave Airlie 526*f47e6306SChris Wilson /** 527*f47e6306SChris Wilson * i915_gem_object_get_page - wrapper macro for __i915_gem_object_get_page 528*f47e6306SChris Wilson * @obj: i915 GEM buffer object 529*f47e6306SChris Wilson * @n: page offset 530*f47e6306SChris Wilson * 531*f47e6306SChris Wilson * Returns: 532*f47e6306SChris Wilson * The target page pointer. 533*f47e6306SChris Wilson * 534*f47e6306SChris Wilson * In order to avoid the truncation of the input parameter, it checks the page 535*f47e6306SChris Wilson * offset n's type from the input parameter before calling 536*f47e6306SChris Wilson * __i915_gem_object_get_page(). 537*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() 538*f47e6306SChris Wilson */ 539*f47e6306SChris Wilson #define i915_gem_object_get_page(obj, n) ({ \ 540*f47e6306SChris Wilson static_assert(castable_to_type(n, pgoff_t)); \ 541*f47e6306SChris Wilson __i915_gem_object_get_page(obj, n); \ 542*f47e6306SChris Wilson }) 543*f47e6306SChris Wilson 544*f47e6306SChris Wilson /** 545*f47e6306SChris Wilson * __i915_gem_object_get_dirty_page - helper to find the target page with a page 546*f47e6306SChris Wilson * offset 547*f47e6306SChris Wilson * @obj: i915 GEM buffer object 548*f47e6306SChris Wilson * @n: page offset 549*f47e6306SChris Wilson * 550*f47e6306SChris Wilson * It works like i915_gem_object_get_page(), but it marks the returned page dirty. 551*f47e6306SChris Wilson * 552*f47e6306SChris Wilson * Returns: 553*f47e6306SChris Wilson * The target page pointer. 554*f47e6306SChris Wilson * 555*f47e6306SChris Wilson * Recommended to use wrapper macro: i915_gem_object_get_dirty_page() 556*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_page() 557*f47e6306SChris Wilson */ 558*f47e6306SChris Wilson struct page * 559*f47e6306SChris Wilson __i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, pgoff_t n); 560*f47e6306SChris Wilson 561*f47e6306SChris Wilson /** 562*f47e6306SChris Wilson * i915_gem_object_get_dirty_page - wrapper macro for __i915_gem_object_get_dirty_page 563*f47e6306SChris Wilson * @obj: i915 GEM buffer object 564*f47e6306SChris Wilson * @n: page offset 565*f47e6306SChris Wilson * 566*f47e6306SChris Wilson * Returns: 567*f47e6306SChris Wilson * The target page pointer. 568*f47e6306SChris Wilson * 569*f47e6306SChris Wilson * In order to avoid the truncation of the input parameter, it checks the page 570*f47e6306SChris Wilson * offset n's type from the input parameter before calling 571*f47e6306SChris Wilson * __i915_gem_object_get_dirty_page(). 572*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_page() 573*f47e6306SChris Wilson */ 574*f47e6306SChris Wilson #define i915_gem_object_get_dirty_page(obj, n) ({ \ 575*f47e6306SChris Wilson static_assert(castable_to_type(n, pgoff_t)); \ 576*f47e6306SChris Wilson __i915_gem_object_get_dirty_page(obj, n); \ 577*f47e6306SChris Wilson }) 578*f47e6306SChris Wilson 579*f47e6306SChris Wilson /** 580*f47e6306SChris Wilson * __i915_gem_object_get_dma_address_len - helper to get bus addresses of 581*f47e6306SChris Wilson * targeted DMA mapped scatterlist from i915 GEM buffer object and it's length 582*f47e6306SChris Wilson * @obj: i915 GEM buffer object 583*f47e6306SChris Wilson * @n: page offset 584*f47e6306SChris Wilson * @len: DMA mapped scatterlist's DMA bus addresses length to return 585*f47e6306SChris Wilson * 586*f47e6306SChris Wilson * Returns: 587*f47e6306SChris Wilson * Bus addresses of targeted DMA mapped scatterlist 588*f47e6306SChris Wilson * 589*f47e6306SChris Wilson * Recommended to use wrapper macro: i915_gem_object_get_dma_address_len() 590*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_sg_dma() 591*f47e6306SChris Wilson */ 59298932149SChris Wilson dma_addr_t 593*f47e6306SChris Wilson __i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, pgoff_t n, 59498932149SChris Wilson unsigned int *len); 59598932149SChris Wilson 596*f47e6306SChris Wilson /** 597*f47e6306SChris Wilson * i915_gem_object_get_dma_address_len - wrapper macro for 598*f47e6306SChris Wilson * __i915_gem_object_get_dma_address_len 599*f47e6306SChris Wilson * @obj: i915 GEM buffer object 600*f47e6306SChris Wilson * @n: page offset 601*f47e6306SChris Wilson * @len: DMA mapped scatterlist's DMA bus addresses length to return 602*f47e6306SChris Wilson * 603*f47e6306SChris Wilson * Returns: 604*f47e6306SChris Wilson * Bus addresses of targeted DMA mapped scatterlist 605*f47e6306SChris Wilson * 606*f47e6306SChris Wilson * In order to avoid the truncation of the input parameter, it checks the page 607*f47e6306SChris Wilson * offset n's type from the input parameter before calling 608*f47e6306SChris Wilson * __i915_gem_object_get_dma_address_len(). 609*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() and 610*f47e6306SChris Wilson * __i915_gem_object_get_dma_address_len() 611*f47e6306SChris Wilson */ 612*f47e6306SChris Wilson #define i915_gem_object_get_dma_address_len(obj, n, len) ({ \ 613*f47e6306SChris Wilson static_assert(castable_to_type(n, pgoff_t)); \ 614*f47e6306SChris Wilson __i915_gem_object_get_dma_address_len(obj, n, len); \ 615*f47e6306SChris Wilson }) 616*f47e6306SChris Wilson 617*f47e6306SChris Wilson /** 618*f47e6306SChris Wilson * __i915_gem_object_get_dma_address - helper to get bus addresses of 619*f47e6306SChris Wilson * targeted DMA mapped scatterlist from i915 GEM buffer object 620*f47e6306SChris Wilson * @obj: i915 GEM buffer object 621*f47e6306SChris Wilson * @n: page offset 622*f47e6306SChris Wilson * 623*f47e6306SChris Wilson * Returns: 624*f47e6306SChris Wilson * Bus addresses of targeted DMA mapped scatterlis 625*f47e6306SChris Wilson * 626*f47e6306SChris Wilson * Recommended to use wrapper macro: i915_gem_object_get_dma_address() 627*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_sg_dma() 628*f47e6306SChris Wilson */ 62998932149SChris Wilson dma_addr_t 630*f47e6306SChris Wilson __i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, pgoff_t n); 631*f47e6306SChris Wilson 632*f47e6306SChris Wilson /** 633*f47e6306SChris Wilson * i915_gem_object_get_dma_address - wrapper macro for 634*f47e6306SChris Wilson * __i915_gem_object_get_dma_address 635*f47e6306SChris Wilson * @obj: i915 GEM buffer object 636*f47e6306SChris Wilson * @n: page offset 637*f47e6306SChris Wilson * 638*f47e6306SChris Wilson * Returns: 639*f47e6306SChris Wilson * Bus addresses of targeted DMA mapped scatterlist 640*f47e6306SChris Wilson * 641*f47e6306SChris Wilson * In order to avoid the truncation of the input parameter, it checks the page 642*f47e6306SChris Wilson * offset n's type from the input parameter before calling 643*f47e6306SChris Wilson * __i915_gem_object_get_dma_address(). 644*f47e6306SChris Wilson * See also __i915_gem_object_page_iter_get_sg() and 645*f47e6306SChris Wilson * __i915_gem_object_get_dma_address() 646*f47e6306SChris Wilson */ 647*f47e6306SChris Wilson #define i915_gem_object_get_dma_address(obj, n) ({ \ 648*f47e6306SChris Wilson static_assert(castable_to_type(n, pgoff_t)); \ 649*f47e6306SChris Wilson __i915_gem_object_get_dma_address(obj, n); \ 650*f47e6306SChris Wilson }) 65198932149SChris Wilson 65298932149SChris Wilson void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 6538c949515SMatthew Auld struct sg_table *pages); 654f033428dSChris Wilson 655f033428dSChris Wilson int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 65698932149SChris Wilson int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 65798932149SChris Wilson 65898932149SChris Wilson static inline int __must_check 65998932149SChris Wilson i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 66098932149SChris Wilson { 661cf41a8f1SMaarten Lankhorst assert_object_held(obj); 66298932149SChris Wilson 66398932149SChris Wilson if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) 66498932149SChris Wilson return 0; 66598932149SChris Wilson 66698932149SChris Wilson return __i915_gem_object_get_pages(obj); 66798932149SChris Wilson } 66898932149SChris Wilson 669c858ffa1SMaarten Lankhorst int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj); 670c858ffa1SMaarten Lankhorst 67198932149SChris Wilson static inline bool 67298932149SChris Wilson i915_gem_object_has_pages(struct drm_i915_gem_object *obj) 67398932149SChris Wilson { 67498932149SChris Wilson return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)); 67598932149SChris Wilson } 67698932149SChris Wilson 67798932149SChris Wilson static inline void 67898932149SChris Wilson __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 67998932149SChris Wilson { 68098932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 68198932149SChris Wilson 68298932149SChris Wilson atomic_inc(&obj->mm.pages_pin_count); 68398932149SChris Wilson } 68498932149SChris Wilson 68598932149SChris Wilson static inline bool 68698932149SChris Wilson i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) 68798932149SChris Wilson { 68898932149SChris Wilson return atomic_read(&obj->mm.pages_pin_count); 68998932149SChris Wilson } 69098932149SChris Wilson 69198932149SChris Wilson static inline void 69298932149SChris Wilson __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 69398932149SChris Wilson { 69498932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 69598932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 69698932149SChris Wilson 69798932149SChris Wilson atomic_dec(&obj->mm.pages_pin_count); 69898932149SChris Wilson } 69998932149SChris Wilson 70098932149SChris Wilson static inline void 70198932149SChris Wilson i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 70298932149SChris Wilson { 70398932149SChris Wilson __i915_gem_object_unpin_pages(obj); 70498932149SChris Wilson } 70598932149SChris Wilson 706f86dbacbSDaniel Vetter int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj); 7077ae03459SMatthew Auld int i915_gem_object_truncate(struct drm_i915_gem_object *obj); 70898932149SChris Wilson 70998932149SChris Wilson /** 71098932149SChris Wilson * i915_gem_object_pin_map - return a contiguous mapping of the entire object 71198932149SChris Wilson * @obj: the object to map into kernel address space 71298932149SChris Wilson * @type: the type of mapping, used to select pgprot_t 71398932149SChris Wilson * 71498932149SChris Wilson * Calls i915_gem_object_pin_pages() to prevent reaping of the object's 71598932149SChris Wilson * pages and then returns a contiguous mapping of the backing storage into 71698932149SChris Wilson * the kernel address space. Based on the @type of mapping, the PTE will be 71798932149SChris Wilson * set to either WriteBack or WriteCombine (via pgprot_t). 71898932149SChris Wilson * 71998932149SChris Wilson * The caller is responsible for calling i915_gem_object_unpin_map() when the 72098932149SChris Wilson * mapping is no longer required. 72198932149SChris Wilson * 72298932149SChris Wilson * Returns the pointer through which to access the mapped object, or an 72398932149SChris Wilson * ERR_PTR() on error. 72498932149SChris Wilson */ 72598932149SChris Wilson void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 72698932149SChris Wilson enum i915_map_type type); 72798932149SChris Wilson 72874827b53SMaarten Lankhorst void *__must_check i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, 72974827b53SMaarten Lankhorst enum i915_map_type type); 73074827b53SMaarten Lankhorst 731d09aa852SJani Nikula enum i915_map_type i915_coherent_map_type(struct drm_i915_private *i915, 732d09aa852SJani Nikula struct drm_i915_gem_object *obj, 733d09aa852SJani Nikula bool always_coherent); 734d09aa852SJani Nikula 73598932149SChris Wilson void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 73698932149SChris Wilson unsigned long offset, 73798932149SChris Wilson unsigned long size); 73898932149SChris Wilson static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj) 73998932149SChris Wilson { 74098932149SChris Wilson __i915_gem_object_flush_map(obj, 0, obj->base.size); 74198932149SChris Wilson } 74298932149SChris Wilson 74398932149SChris Wilson /** 74498932149SChris Wilson * i915_gem_object_unpin_map - releases an earlier mapping 74598932149SChris Wilson * @obj: the object to unmap 74698932149SChris Wilson * 74798932149SChris Wilson * After pinning the object and mapping its pages, once you are finished 74898932149SChris Wilson * with your access, call i915_gem_object_unpin_map() to release the pin 74998932149SChris Wilson * upon the mapping. Once the pin count reaches zero, that mapping may be 75098932149SChris Wilson * removed. 75198932149SChris Wilson */ 75298932149SChris Wilson static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) 75398932149SChris Wilson { 75498932149SChris Wilson i915_gem_object_unpin_pages(obj); 75598932149SChris Wilson } 75698932149SChris Wilson 75789d19b2bSChris Wilson void __i915_gem_object_release_map(struct drm_i915_gem_object *obj); 75889d19b2bSChris Wilson 759f0e4a063SChris Wilson int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj, 760f0e4a063SChris Wilson unsigned int *needs_clflush); 761f0e4a063SChris Wilson int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj, 762f0e4a063SChris Wilson unsigned int *needs_clflush); 763f0e4a063SChris Wilson #define CLFLUSH_BEFORE BIT(0) 764f0e4a063SChris Wilson #define CLFLUSH_AFTER BIT(1) 765f0e4a063SChris Wilson #define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER) 766f0e4a063SChris Wilson 767f0e4a063SChris Wilson static inline void 768f0e4a063SChris Wilson i915_gem_object_finish_access(struct drm_i915_gem_object *obj) 769f0e4a063SChris Wilson { 770f0e4a063SChris Wilson i915_gem_object_unpin_pages(obj); 771f0e4a063SChris Wilson } 772f0e4a063SChris Wilson 7731d7f5e6cSChristian König int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, 7741d7f5e6cSChristian König struct dma_fence **fence); 775f6c466b8SMaarten Lankhorst int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, 776f6c466b8SMaarten Lankhorst bool intr); 777bfe53be2SMatthew Auld bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj); 778f6c466b8SMaarten Lankhorst 77998932149SChris Wilson void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, 78098932149SChris Wilson unsigned int cache_level); 78130f1dccdSMatthew Auld bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj); 78298932149SChris Wilson void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); 783c1793ba8SMaarten Lankhorst void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj); 7842ea6ec76SMatthew Auld bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj); 78598932149SChris Wilson 786f0e4a063SChris Wilson int __must_check 787f0e4a063SChris Wilson i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); 788f0e4a063SChris Wilson int __must_check 789f0e4a063SChris Wilson i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); 790f0e4a063SChris Wilson int __must_check 791f0e4a063SChris Wilson i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); 792f0e4a063SChris Wilson struct i915_vma * __must_check 793f0e4a063SChris Wilson i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 7941b321026SMaarten Lankhorst struct i915_gem_ww_ctx *ww, 795f0e4a063SChris Wilson u32 alignment, 7963bb6a442SNiranjana Vishwanathapura const struct i915_gtt_view *view, 797f0e4a063SChris Wilson unsigned int flags); 798f0e4a063SChris Wilson 7991aff1903SChris Wilson void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj); 8001aff1903SChris Wilson void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 801ebd4a8ecSMatthew Auld void __i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 802ebd4a8ecSMatthew Auld void __i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 8031aff1903SChris Wilson void i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 8041aff1903SChris Wilson 8058475355fSChris Wilson static inline void __start_cpu_write(struct drm_i915_gem_object *obj) 8068475355fSChris Wilson { 8078475355fSChris Wilson obj->read_domains = I915_GEM_DOMAIN_CPU; 8088475355fSChris Wilson obj->write_domain = I915_GEM_DOMAIN_CPU; 8092ea6ec76SMatthew Auld if (i915_gem_cpu_write_needs_clflush(obj)) 8108475355fSChris Wilson obj->cache_dirty = true; 8118475355fSChris Wilson } 81298932149SChris Wilson 813f99e67f1SChris Wilson void i915_gem_fence_wait_priority(struct dma_fence *fence, 814f99e67f1SChris Wilson const struct i915_sched_attr *attr); 815f99e67f1SChris Wilson 816d45a1a53SChris Wilson int i915_gem_object_wait(struct drm_i915_gem_object *obj, 817d45a1a53SChris Wilson unsigned int flags, 818d45a1a53SChris Wilson long timeout); 819d45a1a53SChris Wilson int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 820d45a1a53SChris Wilson unsigned int flags, 821d45a1a53SChris Wilson const struct i915_sched_attr *attr); 822d45a1a53SChris Wilson 823da42104fSChris Wilson void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, 824da42104fSChris Wilson enum fb_op_origin origin); 825da42104fSChris Wilson void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, 826da42104fSChris Wilson enum fb_op_origin origin); 827da42104fSChris Wilson 828da42104fSChris Wilson static inline void 829da42104fSChris Wilson i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, 830da42104fSChris Wilson enum fb_op_origin origin) 831da42104fSChris Wilson { 832da42104fSChris Wilson if (unlikely(rcu_access_pointer(obj->frontbuffer))) 833da42104fSChris Wilson __i915_gem_object_flush_frontbuffer(obj, origin); 834da42104fSChris Wilson } 835da42104fSChris Wilson 836da42104fSChris Wilson static inline void 837da42104fSChris Wilson i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, 838da42104fSChris Wilson enum fb_op_origin origin) 839da42104fSChris Wilson { 840da42104fSChris Wilson if (unlikely(rcu_access_pointer(obj->frontbuffer))) 841da42104fSChris Wilson __i915_gem_object_invalidate_frontbuffer(obj, origin); 842da42104fSChris Wilson } 843da42104fSChris Wilson 8445fbc2c2bSImre Deak int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size); 8455fbc2c2bSImre Deak 84641a9c75dSChris Wilson bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj); 84741a9c75dSChris Wilson 848213d5092SThomas Hellström void __i915_gem_free_object_rcu(struct rcu_head *head); 849213d5092SThomas Hellström 850068396bbSThomas Hellström void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj); 851068396bbSThomas Hellström 852213d5092SThomas Hellström void __i915_gem_free_object(struct drm_i915_gem_object *obj); 853213d5092SThomas Hellström 854213d5092SThomas Hellström bool i915_gem_object_evictable(struct drm_i915_gem_object *obj); 855213d5092SThomas Hellström 8562e53d7c1SThomas Hellström bool i915_gem_object_migratable(struct drm_i915_gem_object *obj); 8572e53d7c1SThomas Hellström 858b6e913e1SThomas Hellström int i915_gem_object_migrate(struct drm_i915_gem_object *obj, 859b6e913e1SThomas Hellström struct i915_gem_ww_ctx *ww, 860b6e913e1SThomas Hellström enum intel_region_id id); 861999f4562SMatthew Auld int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, 862999f4562SMatthew Auld struct i915_gem_ww_ctx *ww, 863999f4562SMatthew Auld enum intel_region_id id, 864999f4562SMatthew Auld unsigned int flags); 865b6e913e1SThomas Hellström 866b6e913e1SThomas Hellström bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, 867b6e913e1SThomas Hellström enum intel_region_id id); 868b6e913e1SThomas Hellström 869b6e913e1SThomas Hellström int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, 870b6e913e1SThomas Hellström unsigned int flags); 871b6e913e1SThomas Hellström 872b3f450d9SMatthew Auld bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, 873b3f450d9SMatthew Auld enum intel_memory_type type); 874b3f450d9SMatthew Auld 875efeb3cafSMatthew Auld bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj); 876efeb3cafSMatthew Auld 877cad7109aSThomas Hellström int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 8787ae03459SMatthew Auld size_t size, struct intel_memory_region *mr, 8797ae03459SMatthew Auld struct address_space *mapping, 8807ae03459SMatthew Auld unsigned int max_segment); 881cad7109aSThomas Hellström void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 8827ae03459SMatthew Auld bool dirty, bool backup); 8837ae03459SMatthew Auld void __shmem_writeback(size_t size, struct address_space *mapping); 8847ae03459SMatthew Auld 885ed29c269SMaarten Lankhorst #ifdef CONFIG_MMU_NOTIFIER 886ed29c269SMaarten Lankhorst static inline bool 887ed29c269SMaarten Lankhorst i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) 888ed29c269SMaarten Lankhorst { 889ed29c269SMaarten Lankhorst return obj->userptr.notifier.mm; 890ed29c269SMaarten Lankhorst } 891ed29c269SMaarten Lankhorst 892ed29c269SMaarten Lankhorst int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj); 893ed29c269SMaarten Lankhorst int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj); 894ed29c269SMaarten Lankhorst int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj); 895ed29c269SMaarten Lankhorst #else 896ed29c269SMaarten Lankhorst static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) { return false; } 897ed29c269SMaarten Lankhorst 898ed29c269SMaarten Lankhorst static inline int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 899ed29c269SMaarten Lankhorst static inline int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 900ed29c269SMaarten Lankhorst static inline int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 901ed29c269SMaarten Lankhorst 902ed29c269SMaarten Lankhorst #endif 903ed29c269SMaarten Lankhorst 90498932149SChris Wilson #endif 905