198932149SChris Wilson /* 298932149SChris Wilson * SPDX-License-Identifier: MIT 398932149SChris Wilson * 498932149SChris Wilson * Copyright © 2016 Intel Corporation 598932149SChris Wilson */ 698932149SChris Wilson 798932149SChris Wilson #ifndef __I915_GEM_OBJECT_H__ 898932149SChris Wilson #define __I915_GEM_OBJECT_H__ 998932149SChris Wilson 1098932149SChris Wilson #include <drm/drm_gem.h> 1198932149SChris Wilson #include <drm/drm_file.h> 1298932149SChris Wilson #include <drm/drm_device.h> 1398932149SChris Wilson 1498932149SChris Wilson #include <drm/i915_drm.h> 1598932149SChris Wilson 1698932149SChris Wilson #include "i915_gem_object_types.h" 1798932149SChris Wilson 188475355fSChris Wilson void i915_gem_init__objects(struct drm_i915_private *i915); 198475355fSChris Wilson 2098932149SChris Wilson struct drm_i915_gem_object *i915_gem_object_alloc(void); 2198932149SChris Wilson void i915_gem_object_free(struct drm_i915_gem_object *obj); 2298932149SChris Wilson 238475355fSChris Wilson void i915_gem_object_init(struct drm_i915_gem_object *obj, 248475355fSChris Wilson const struct drm_i915_gem_object_ops *ops); 258475355fSChris Wilson struct drm_i915_gem_object * 268475355fSChris Wilson i915_gem_object_create_shmem(struct drm_i915_private *i915, u64 size); 278475355fSChris Wilson struct drm_i915_gem_object * 288475355fSChris Wilson i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, 298475355fSChris Wilson const void *data, size_t size); 308475355fSChris Wilson 318475355fSChris Wilson extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops; 328475355fSChris Wilson void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 338475355fSChris Wilson struct sg_table *pages, 348475355fSChris Wilson bool needs_clflush); 358475355fSChris Wilson 36f033428dSChris Wilson int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align); 37f033428dSChris Wilson 388475355fSChris Wilson void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file); 398475355fSChris Wilson void i915_gem_free_object(struct drm_gem_object *obj); 408475355fSChris Wilson 418475355fSChris Wilson void i915_gem_flush_free_objects(struct drm_i915_private *i915); 428475355fSChris Wilson 43f033428dSChris Wilson struct sg_table * 44f033428dSChris Wilson __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj); 45f033428dSChris Wilson void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 46f033428dSChris Wilson 4798932149SChris Wilson /** 4898932149SChris Wilson * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle 4998932149SChris Wilson * @filp: DRM file private date 5098932149SChris Wilson * @handle: userspace handle 5198932149SChris Wilson * 5298932149SChris Wilson * Returns: 5398932149SChris Wilson * 5498932149SChris Wilson * A pointer to the object named by the handle if such exists on @filp, NULL 5598932149SChris Wilson * otherwise. This object is only valid whilst under the RCU read lock, and 5698932149SChris Wilson * note carefully the object may be in the process of being destroyed. 5798932149SChris Wilson */ 5898932149SChris Wilson static inline struct drm_i915_gem_object * 5998932149SChris Wilson i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) 6098932149SChris Wilson { 6198932149SChris Wilson #ifdef CONFIG_LOCKDEP 6298932149SChris Wilson WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); 6398932149SChris Wilson #endif 6498932149SChris Wilson return idr_find(&file->object_idr, handle); 6598932149SChris Wilson } 6698932149SChris Wilson 6798932149SChris Wilson static inline struct drm_i915_gem_object * 6898932149SChris Wilson i915_gem_object_lookup(struct drm_file *file, u32 handle) 6998932149SChris Wilson { 7098932149SChris Wilson struct drm_i915_gem_object *obj; 7198932149SChris Wilson 7298932149SChris Wilson rcu_read_lock(); 7398932149SChris Wilson obj = i915_gem_object_lookup_rcu(file, handle); 7498932149SChris Wilson if (obj && !kref_get_unless_zero(&obj->base.refcount)) 7598932149SChris Wilson obj = NULL; 7698932149SChris Wilson rcu_read_unlock(); 7798932149SChris Wilson 7898932149SChris Wilson return obj; 7998932149SChris Wilson } 8098932149SChris Wilson 8198932149SChris Wilson __deprecated 8298932149SChris Wilson extern struct drm_gem_object * 8398932149SChris Wilson drm_gem_object_lookup(struct drm_file *file, u32 handle); 8498932149SChris Wilson 8598932149SChris Wilson __attribute__((nonnull)) 8698932149SChris Wilson static inline struct drm_i915_gem_object * 8798932149SChris Wilson i915_gem_object_get(struct drm_i915_gem_object *obj) 8898932149SChris Wilson { 8998932149SChris Wilson drm_gem_object_get(&obj->base); 9098932149SChris Wilson return obj; 9198932149SChris Wilson } 9298932149SChris Wilson 9398932149SChris Wilson __attribute__((nonnull)) 9498932149SChris Wilson static inline void 9598932149SChris Wilson i915_gem_object_put(struct drm_i915_gem_object *obj) 9698932149SChris Wilson { 9798932149SChris Wilson __drm_gem_object_put(&obj->base); 9898932149SChris Wilson } 9998932149SChris Wilson 10098932149SChris Wilson static inline void i915_gem_object_lock(struct drm_i915_gem_object *obj) 10198932149SChris Wilson { 10298932149SChris Wilson reservation_object_lock(obj->resv, NULL); 10398932149SChris Wilson } 10498932149SChris Wilson 10598932149SChris Wilson static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) 10698932149SChris Wilson { 10798932149SChris Wilson reservation_object_unlock(obj->resv); 10898932149SChris Wilson } 10998932149SChris Wilson 11098932149SChris Wilson static inline void 11198932149SChris Wilson i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) 11298932149SChris Wilson { 11398932149SChris Wilson obj->base.vma_node.readonly = true; 11498932149SChris Wilson } 11598932149SChris Wilson 11698932149SChris Wilson static inline bool 11798932149SChris Wilson i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) 11898932149SChris Wilson { 11998932149SChris Wilson return obj->base.vma_node.readonly; 12098932149SChris Wilson } 12198932149SChris Wilson 12298932149SChris Wilson static inline bool 12398932149SChris Wilson i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) 12498932149SChris Wilson { 12598932149SChris Wilson return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE; 12698932149SChris Wilson } 12798932149SChris Wilson 12898932149SChris Wilson static inline bool 12998932149SChris Wilson i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) 13098932149SChris Wilson { 13198932149SChris Wilson return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE; 13298932149SChris Wilson } 13398932149SChris Wilson 13498932149SChris Wilson static inline bool 13598932149SChris Wilson i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj) 13698932149SChris Wilson { 13798932149SChris Wilson return obj->ops->flags & I915_GEM_OBJECT_IS_PROXY; 13898932149SChris Wilson } 13998932149SChris Wilson 14098932149SChris Wilson static inline bool 14198932149SChris Wilson i915_gem_object_needs_async_cancel(const struct drm_i915_gem_object *obj) 14298932149SChris Wilson { 14398932149SChris Wilson return obj->ops->flags & I915_GEM_OBJECT_ASYNC_CANCEL; 14498932149SChris Wilson } 14598932149SChris Wilson 14698932149SChris Wilson static inline bool 14798932149SChris Wilson i915_gem_object_is_active(const struct drm_i915_gem_object *obj) 14898932149SChris Wilson { 14998932149SChris Wilson return obj->active_count; 15098932149SChris Wilson } 15198932149SChris Wilson 15298932149SChris Wilson static inline bool 15398932149SChris Wilson i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj) 15498932149SChris Wilson { 15598932149SChris Wilson return test_bit(I915_BO_ACTIVE_REF, &obj->flags); 15698932149SChris Wilson } 15798932149SChris Wilson 15898932149SChris Wilson static inline void 15998932149SChris Wilson i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj) 16098932149SChris Wilson { 16198932149SChris Wilson lockdep_assert_held(&obj->base.dev->struct_mutex); 16298932149SChris Wilson __set_bit(I915_BO_ACTIVE_REF, &obj->flags); 16398932149SChris Wilson } 16498932149SChris Wilson 16598932149SChris Wilson static inline void 16698932149SChris Wilson i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj) 16798932149SChris Wilson { 16898932149SChris Wilson lockdep_assert_held(&obj->base.dev->struct_mutex); 16998932149SChris Wilson __clear_bit(I915_BO_ACTIVE_REF, &obj->flags); 17098932149SChris Wilson } 17198932149SChris Wilson 17298932149SChris Wilson void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj); 17398932149SChris Wilson 17498932149SChris Wilson static inline bool 17598932149SChris Wilson i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj) 17698932149SChris Wilson { 17798932149SChris Wilson return READ_ONCE(obj->framebuffer_references); 17898932149SChris Wilson } 17998932149SChris Wilson 18098932149SChris Wilson static inline unsigned int 18198932149SChris Wilson i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj) 18298932149SChris Wilson { 18398932149SChris Wilson return obj->tiling_and_stride & TILING_MASK; 18498932149SChris Wilson } 18598932149SChris Wilson 18698932149SChris Wilson static inline bool 18798932149SChris Wilson i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj) 18898932149SChris Wilson { 18998932149SChris Wilson return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; 19098932149SChris Wilson } 19198932149SChris Wilson 19298932149SChris Wilson static inline unsigned int 19398932149SChris Wilson i915_gem_object_get_stride(const struct drm_i915_gem_object *obj) 19498932149SChris Wilson { 19598932149SChris Wilson return obj->tiling_and_stride & STRIDE_MASK; 19698932149SChris Wilson } 19798932149SChris Wilson 19898932149SChris Wilson static inline unsigned int 19998932149SChris Wilson i915_gem_tile_height(unsigned int tiling) 20098932149SChris Wilson { 20198932149SChris Wilson GEM_BUG_ON(!tiling); 20298932149SChris Wilson return tiling == I915_TILING_Y ? 32 : 8; 20398932149SChris Wilson } 20498932149SChris Wilson 20598932149SChris Wilson static inline unsigned int 20698932149SChris Wilson i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj) 20798932149SChris Wilson { 20898932149SChris Wilson return i915_gem_tile_height(i915_gem_object_get_tiling(obj)); 20998932149SChris Wilson } 21098932149SChris Wilson 21198932149SChris Wilson static inline unsigned int 21298932149SChris Wilson i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj) 21398932149SChris Wilson { 21498932149SChris Wilson return (i915_gem_object_get_stride(obj) * 21598932149SChris Wilson i915_gem_object_get_tile_height(obj)); 21698932149SChris Wilson } 21798932149SChris Wilson 21898932149SChris Wilson int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, 21998932149SChris Wilson unsigned int tiling, unsigned int stride); 22098932149SChris Wilson 22198932149SChris Wilson struct scatterlist * 22298932149SChris Wilson i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 22398932149SChris Wilson unsigned int n, unsigned int *offset); 22498932149SChris Wilson 22598932149SChris Wilson struct page * 22698932149SChris Wilson i915_gem_object_get_page(struct drm_i915_gem_object *obj, 22798932149SChris Wilson unsigned int n); 22898932149SChris Wilson 22998932149SChris Wilson struct page * 23098932149SChris Wilson i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 23198932149SChris Wilson unsigned int n); 23298932149SChris Wilson 23398932149SChris Wilson dma_addr_t 23498932149SChris Wilson i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, 23598932149SChris Wilson unsigned long n, 23698932149SChris Wilson unsigned int *len); 23798932149SChris Wilson 23898932149SChris Wilson dma_addr_t 23998932149SChris Wilson i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 24098932149SChris Wilson unsigned long n); 24198932149SChris Wilson 24298932149SChris Wilson void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 24398932149SChris Wilson struct sg_table *pages, 24498932149SChris Wilson unsigned int sg_page_sizes); 245f033428dSChris Wilson 246f033428dSChris Wilson int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 24798932149SChris Wilson int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 24898932149SChris Wilson 24998932149SChris Wilson static inline int __must_check 25098932149SChris Wilson i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 25198932149SChris Wilson { 25298932149SChris Wilson might_lock(&obj->mm.lock); 25398932149SChris Wilson 25498932149SChris Wilson if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) 25598932149SChris Wilson return 0; 25698932149SChris Wilson 25798932149SChris Wilson return __i915_gem_object_get_pages(obj); 25898932149SChris Wilson } 25998932149SChris Wilson 26098932149SChris Wilson static inline bool 26198932149SChris Wilson i915_gem_object_has_pages(struct drm_i915_gem_object *obj) 26298932149SChris Wilson { 26398932149SChris Wilson return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)); 26498932149SChris Wilson } 26598932149SChris Wilson 26698932149SChris Wilson static inline void 26798932149SChris Wilson __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 26898932149SChris Wilson { 26998932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 27098932149SChris Wilson 27198932149SChris Wilson atomic_inc(&obj->mm.pages_pin_count); 27298932149SChris Wilson } 27398932149SChris Wilson 27498932149SChris Wilson static inline bool 27598932149SChris Wilson i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) 27698932149SChris Wilson { 27798932149SChris Wilson return atomic_read(&obj->mm.pages_pin_count); 27898932149SChris Wilson } 27998932149SChris Wilson 28098932149SChris Wilson static inline void 28198932149SChris Wilson __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 28298932149SChris Wilson { 28398932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 28498932149SChris Wilson GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 28598932149SChris Wilson 28698932149SChris Wilson atomic_dec(&obj->mm.pages_pin_count); 28798932149SChris Wilson } 28898932149SChris Wilson 28998932149SChris Wilson static inline void 29098932149SChris Wilson i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 29198932149SChris Wilson { 29298932149SChris Wilson __i915_gem_object_unpin_pages(obj); 29398932149SChris Wilson } 29498932149SChris Wilson 29598932149SChris Wilson enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock/struct_mutex */ 29698932149SChris Wilson I915_MM_NORMAL = 0, 29798932149SChris Wilson I915_MM_SHRINKER /* called "recursively" from direct-reclaim-esque */ 29898932149SChris Wilson }; 29998932149SChris Wilson 30098932149SChris Wilson int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, 30198932149SChris Wilson enum i915_mm_subclass subclass); 302f033428dSChris Wilson void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 303f033428dSChris Wilson void i915_gem_object_writeback(struct drm_i915_gem_object *obj); 30498932149SChris Wilson 30598932149SChris Wilson enum i915_map_type { 30698932149SChris Wilson I915_MAP_WB = 0, 30798932149SChris Wilson I915_MAP_WC, 30898932149SChris Wilson #define I915_MAP_OVERRIDE BIT(31) 30998932149SChris Wilson I915_MAP_FORCE_WB = I915_MAP_WB | I915_MAP_OVERRIDE, 31098932149SChris Wilson I915_MAP_FORCE_WC = I915_MAP_WC | I915_MAP_OVERRIDE, 31198932149SChris Wilson }; 31298932149SChris Wilson 31398932149SChris Wilson /** 31498932149SChris Wilson * i915_gem_object_pin_map - return a contiguous mapping of the entire object 31598932149SChris Wilson * @obj: the object to map into kernel address space 31698932149SChris Wilson * @type: the type of mapping, used to select pgprot_t 31798932149SChris Wilson * 31898932149SChris Wilson * Calls i915_gem_object_pin_pages() to prevent reaping of the object's 31998932149SChris Wilson * pages and then returns a contiguous mapping of the backing storage into 32098932149SChris Wilson * the kernel address space. Based on the @type of mapping, the PTE will be 32198932149SChris Wilson * set to either WriteBack or WriteCombine (via pgprot_t). 32298932149SChris Wilson * 32398932149SChris Wilson * The caller is responsible for calling i915_gem_object_unpin_map() when the 32498932149SChris Wilson * mapping is no longer required. 32598932149SChris Wilson * 32698932149SChris Wilson * Returns the pointer through which to access the mapped object, or an 32798932149SChris Wilson * ERR_PTR() on error. 32898932149SChris Wilson */ 32998932149SChris Wilson void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 33098932149SChris Wilson enum i915_map_type type); 33198932149SChris Wilson 33298932149SChris Wilson void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 33398932149SChris Wilson unsigned long offset, 33498932149SChris Wilson unsigned long size); 33598932149SChris Wilson static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj) 33698932149SChris Wilson { 33798932149SChris Wilson __i915_gem_object_flush_map(obj, 0, obj->base.size); 33898932149SChris Wilson } 33998932149SChris Wilson 34098932149SChris Wilson /** 34198932149SChris Wilson * i915_gem_object_unpin_map - releases an earlier mapping 34298932149SChris Wilson * @obj: the object to unmap 34398932149SChris Wilson * 34498932149SChris Wilson * After pinning the object and mapping its pages, once you are finished 34598932149SChris Wilson * with your access, call i915_gem_object_unpin_map() to release the pin 34698932149SChris Wilson * upon the mapping. Once the pin count reaches zero, that mapping may be 34798932149SChris Wilson * removed. 34898932149SChris Wilson */ 34998932149SChris Wilson static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) 35098932149SChris Wilson { 35198932149SChris Wilson i915_gem_object_unpin_pages(obj); 35298932149SChris Wilson } 35398932149SChris Wilson 354b414fcd5SChris Wilson void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj); 355b414fcd5SChris Wilson void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj); 356b414fcd5SChris Wilson 357b414fcd5SChris Wilson void 358b414fcd5SChris Wilson i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj, 359b414fcd5SChris Wilson unsigned int flush_domains); 360b414fcd5SChris Wilson 36198932149SChris Wilson static inline struct intel_engine_cs * 36298932149SChris Wilson i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj) 36398932149SChris Wilson { 36498932149SChris Wilson struct intel_engine_cs *engine = NULL; 36598932149SChris Wilson struct dma_fence *fence; 36698932149SChris Wilson 36798932149SChris Wilson rcu_read_lock(); 36898932149SChris Wilson fence = reservation_object_get_excl_rcu(obj->resv); 36998932149SChris Wilson rcu_read_unlock(); 37098932149SChris Wilson 37198932149SChris Wilson if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence)) 37298932149SChris Wilson engine = to_request(fence)->engine; 37398932149SChris Wilson dma_fence_put(fence); 37498932149SChris Wilson 37598932149SChris Wilson return engine; 37698932149SChris Wilson } 37798932149SChris Wilson 37898932149SChris Wilson void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, 37998932149SChris Wilson unsigned int cache_level); 38098932149SChris Wilson void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); 38198932149SChris Wilson 3828475355fSChris Wilson static inline bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 3838475355fSChris Wilson { 3848475355fSChris Wilson if (obj->cache_dirty) 3858475355fSChris Wilson return false; 3868475355fSChris Wilson 3878475355fSChris Wilson if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)) 3888475355fSChris Wilson return true; 3898475355fSChris Wilson 3908475355fSChris Wilson return obj->pin_global; /* currently in use by HW, keep flushed */ 3918475355fSChris Wilson } 3928475355fSChris Wilson 3938475355fSChris Wilson static inline void __start_cpu_write(struct drm_i915_gem_object *obj) 3948475355fSChris Wilson { 3958475355fSChris Wilson obj->read_domains = I915_GEM_DOMAIN_CPU; 3968475355fSChris Wilson obj->write_domain = I915_GEM_DOMAIN_CPU; 3978475355fSChris Wilson if (cpu_write_needs_clflush(obj)) 3988475355fSChris Wilson obj->cache_dirty = true; 3998475355fSChris Wilson } 40098932149SChris Wilson 40198932149SChris Wilson #endif 402