1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2013 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #ifndef __MSM_GEM_H__ 8 #define __MSM_GEM_H__ 9 10 #include <linux/kref.h> 11 #include <linux/dma-resv.h> 12 #include "drm/gpu_scheduler.h" 13 #include "msm_drv.h" 14 15 /* Make all GEM related WARN_ON()s ratelimited.. when things go wrong they 16 * tend to go wrong 1000s of times in a short timespan. 17 */ 18 #define GEM_WARN_ON(x) WARN_RATELIMIT(x, "%s", __stringify(x)) 19 20 /* Additional internal-use only BO flags: */ 21 #define MSM_BO_STOLEN 0x10000000 /* try to use stolen/splash memory */ 22 #define MSM_BO_MAP_PRIV 0x20000000 /* use IOMMU_PRIV when mapping */ 23 24 struct msm_gem_address_space { 25 const char *name; 26 /* NOTE: mm managed at the page level, size is in # of pages 27 * and position mm_node->start is in # of pages: 28 */ 29 struct drm_mm mm; 30 spinlock_t lock; /* Protects drm_mm node allocation/removal */ 31 struct msm_mmu *mmu; 32 struct kref kref; 33 34 /* For address spaces associated with a specific process, this 35 * will be non-NULL: 36 */ 37 struct pid *pid; 38 }; 39 40 struct msm_gem_vma { 41 struct drm_mm_node node; 42 uint64_t iova; 43 struct msm_gem_address_space *aspace; 44 struct list_head list; /* node in msm_gem_object::vmas */ 45 bool mapped; 46 int inuse; 47 }; 48 49 struct msm_gem_object { 50 struct drm_gem_object base; 51 52 uint32_t flags; 53 54 /** 55 * Advice: are the backing pages purgeable? 56 */ 57 uint8_t madv; 58 59 /** 60 * Is object on inactive_dontneed list (ie. counted in priv->shrinkable_count)? 61 */ 62 bool dontneed : 1; 63 64 /** 65 * Is object evictable (ie. counted in priv->evictable_count)? 66 */ 67 bool evictable : 1; 68 69 /** 70 * count of active vmap'ing 71 */ 72 uint8_t vmap_count; 73 74 /** 75 * Node in list of all objects (mainly for debugfs, protected by 76 * priv->obj_lock 77 */ 78 struct list_head node; 79 80 /** 81 * An object is either: 82 * inactive - on priv->inactive_dontneed or priv->inactive_willneed 83 * (depending on purgeability status) 84 * active - on one one of the gpu's active_list.. well, at 85 * least for now we don't have (I don't think) hw sync between 86 * 2d and 3d one devices which have both, meaning we need to 87 * block on submit if a bo is already on other ring 88 */ 89 struct list_head mm_list; 90 91 struct page **pages; 92 struct sg_table *sgt; 93 void *vaddr; 94 95 struct list_head vmas; /* list of msm_gem_vma */ 96 97 /* For physically contiguous buffers. Used when we don't have 98 * an IOMMU. Also used for stolen/splashscreen buffer. 99 */ 100 struct drm_mm_node *vram_node; 101 102 char name[32]; /* Identifier to print for the debugfs files */ 103 104 int active_count; 105 int pin_count; 106 }; 107 #define to_msm_bo(x) container_of(x, struct msm_gem_object, base) 108 109 uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj); 110 int msm_gem_get_iova(struct drm_gem_object *obj, 111 struct msm_gem_address_space *aspace, uint64_t *iova); 112 int msm_gem_get_and_pin_iova_range(struct drm_gem_object *obj, 113 struct msm_gem_address_space *aspace, uint64_t *iova, 114 u64 range_start, u64 range_end); 115 int msm_gem_get_and_pin_iova_locked(struct drm_gem_object *obj, 116 struct msm_gem_address_space *aspace, uint64_t *iova); 117 int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, 118 struct msm_gem_address_space *aspace, uint64_t *iova); 119 uint64_t msm_gem_iova(struct drm_gem_object *obj, 120 struct msm_gem_address_space *aspace); 121 void msm_gem_unpin_iova_locked(struct drm_gem_object *obj, 122 struct msm_gem_address_space *aspace); 123 void msm_gem_unpin_iova(struct drm_gem_object *obj, 124 struct msm_gem_address_space *aspace); 125 struct page **msm_gem_get_pages(struct drm_gem_object *obj); 126 void msm_gem_put_pages(struct drm_gem_object *obj); 127 int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 128 struct drm_mode_create_dumb *args); 129 int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, 130 uint32_t handle, uint64_t *offset); 131 void *msm_gem_get_vaddr_locked(struct drm_gem_object *obj); 132 void *msm_gem_get_vaddr(struct drm_gem_object *obj); 133 void *msm_gem_get_vaddr_active(struct drm_gem_object *obj); 134 void msm_gem_put_vaddr_locked(struct drm_gem_object *obj); 135 void msm_gem_put_vaddr(struct drm_gem_object *obj); 136 int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv); 137 void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu); 138 void msm_gem_active_put(struct drm_gem_object *obj); 139 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout); 140 int msm_gem_cpu_fini(struct drm_gem_object *obj); 141 void msm_gem_free_object(struct drm_gem_object *obj); 142 int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file, 143 uint32_t size, uint32_t flags, uint32_t *handle, char *name); 144 struct drm_gem_object *msm_gem_new(struct drm_device *dev, 145 uint32_t size, uint32_t flags); 146 void *msm_gem_kernel_new(struct drm_device *dev, uint32_t size, 147 uint32_t flags, struct msm_gem_address_space *aspace, 148 struct drm_gem_object **bo, uint64_t *iova); 149 void msm_gem_kernel_put(struct drm_gem_object *bo, 150 struct msm_gem_address_space *aspace); 151 struct drm_gem_object *msm_gem_import(struct drm_device *dev, 152 struct dma_buf *dmabuf, struct sg_table *sgt); 153 __printf(2, 3) 154 void msm_gem_object_set_name(struct drm_gem_object *bo, const char *fmt, ...); 155 156 #ifdef CONFIG_DEBUG_FS 157 struct msm_gem_stats { 158 struct { 159 unsigned count; 160 size_t size; 161 } all, active, resident, purgeable, purged; 162 }; 163 164 void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, 165 struct msm_gem_stats *stats); 166 void msm_gem_describe_objects(struct list_head *list, struct seq_file *m); 167 #endif 168 169 static inline void 170 msm_gem_lock(struct drm_gem_object *obj) 171 { 172 dma_resv_lock(obj->resv, NULL); 173 } 174 175 static inline bool __must_check 176 msm_gem_trylock(struct drm_gem_object *obj) 177 { 178 return dma_resv_trylock(obj->resv); 179 } 180 181 static inline int 182 msm_gem_lock_interruptible(struct drm_gem_object *obj) 183 { 184 return dma_resv_lock_interruptible(obj->resv, NULL); 185 } 186 187 static inline void 188 msm_gem_unlock(struct drm_gem_object *obj) 189 { 190 dma_resv_unlock(obj->resv); 191 } 192 193 static inline bool 194 msm_gem_is_locked(struct drm_gem_object *obj) 195 { 196 return dma_resv_is_locked(obj->resv); 197 } 198 199 static inline bool is_active(struct msm_gem_object *msm_obj) 200 { 201 GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base)); 202 return msm_obj->active_count; 203 } 204 205 /* imported/exported objects are not purgeable: */ 206 static inline bool is_unpurgeable(struct msm_gem_object *msm_obj) 207 { 208 return msm_obj->base.import_attach || msm_obj->pin_count; 209 } 210 211 static inline bool is_purgeable(struct msm_gem_object *msm_obj) 212 { 213 return (msm_obj->madv == MSM_MADV_DONTNEED) && msm_obj->sgt && 214 !is_unpurgeable(msm_obj); 215 } 216 217 static inline bool is_vunmapable(struct msm_gem_object *msm_obj) 218 { 219 GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base)); 220 return (msm_obj->vmap_count == 0) && msm_obj->vaddr; 221 } 222 223 static inline void mark_purgeable(struct msm_gem_object *msm_obj) 224 { 225 struct msm_drm_private *priv = msm_obj->base.dev->dev_private; 226 227 GEM_WARN_ON(!mutex_is_locked(&priv->mm_lock)); 228 229 if (is_unpurgeable(msm_obj)) 230 return; 231 232 if (GEM_WARN_ON(msm_obj->dontneed)) 233 return; 234 235 priv->shrinkable_count += msm_obj->base.size >> PAGE_SHIFT; 236 msm_obj->dontneed = true; 237 } 238 239 static inline void mark_unpurgeable(struct msm_gem_object *msm_obj) 240 { 241 struct msm_drm_private *priv = msm_obj->base.dev->dev_private; 242 243 GEM_WARN_ON(!mutex_is_locked(&priv->mm_lock)); 244 245 if (is_unpurgeable(msm_obj)) 246 return; 247 248 if (GEM_WARN_ON(!msm_obj->dontneed)) 249 return; 250 251 priv->shrinkable_count -= msm_obj->base.size >> PAGE_SHIFT; 252 GEM_WARN_ON(priv->shrinkable_count < 0); 253 msm_obj->dontneed = false; 254 } 255 256 static inline bool is_unevictable(struct msm_gem_object *msm_obj) 257 { 258 return is_unpurgeable(msm_obj) || msm_obj->vaddr; 259 } 260 261 static inline void mark_evictable(struct msm_gem_object *msm_obj) 262 { 263 struct msm_drm_private *priv = msm_obj->base.dev->dev_private; 264 265 WARN_ON(!mutex_is_locked(&priv->mm_lock)); 266 267 if (is_unevictable(msm_obj)) 268 return; 269 270 if (WARN_ON(msm_obj->evictable)) 271 return; 272 273 priv->evictable_count += msm_obj->base.size >> PAGE_SHIFT; 274 msm_obj->evictable = true; 275 } 276 277 static inline void mark_unevictable(struct msm_gem_object *msm_obj) 278 { 279 struct msm_drm_private *priv = msm_obj->base.dev->dev_private; 280 281 WARN_ON(!mutex_is_locked(&priv->mm_lock)); 282 283 if (is_unevictable(msm_obj)) 284 return; 285 286 if (WARN_ON(!msm_obj->evictable)) 287 return; 288 289 priv->evictable_count -= msm_obj->base.size >> PAGE_SHIFT; 290 WARN_ON(priv->evictable_count < 0); 291 msm_obj->evictable = false; 292 } 293 294 void msm_gem_purge(struct drm_gem_object *obj); 295 void msm_gem_evict(struct drm_gem_object *obj); 296 void msm_gem_vunmap(struct drm_gem_object *obj); 297 298 /* Created per submit-ioctl, to track bo's and cmdstream bufs, etc, 299 * associated with the cmdstream submission for synchronization (and 300 * make it easier to unwind when things go wrong, etc). 301 */ 302 struct msm_gem_submit { 303 struct drm_sched_job base; 304 struct kref ref; 305 struct drm_device *dev; 306 struct msm_gpu *gpu; 307 struct msm_gem_address_space *aspace; 308 struct list_head node; /* node in ring submit list */ 309 struct ww_acquire_ctx ticket; 310 uint32_t seqno; /* Sequence number of the submit on the ring */ 311 312 /* Hw fence, which is created when the scheduler executes the job, and 313 * is signaled when the hw finishes (via seqno write from cmdstream) 314 */ 315 struct dma_fence *hw_fence; 316 317 /* Userspace visible fence, which is signaled by the scheduler after 318 * the hw_fence is signaled. 319 */ 320 struct dma_fence *user_fence; 321 322 int fence_id; /* key into queue->fence_idr */ 323 struct msm_gpu_submitqueue *queue; 324 struct pid *pid; /* submitting process */ 325 bool fault_dumped; /* Limit devcoredump dumping to one per submit */ 326 bool valid; /* true if no cmdstream patching needed */ 327 bool in_rb; /* "sudo" mode, copy cmds into RB */ 328 struct msm_ringbuffer *ring; 329 struct msm_file_private *ctx; 330 unsigned int nr_cmds; 331 unsigned int nr_bos; 332 u32 ident; /* A "identifier" for the submit for logging */ 333 struct { 334 uint32_t type; 335 uint32_t size; /* in dwords */ 336 uint64_t iova; 337 uint32_t offset;/* in dwords */ 338 uint32_t idx; /* cmdstream buffer idx in bos[] */ 339 uint32_t nr_relocs; 340 struct drm_msm_gem_submit_reloc *relocs; 341 } *cmd; /* array of size nr_cmds */ 342 struct { 343 uint32_t flags; 344 union { 345 struct msm_gem_object *obj; 346 uint32_t handle; 347 }; 348 uint64_t iova; 349 } bos[]; 350 }; 351 352 static inline struct msm_gem_submit *to_msm_submit(struct drm_sched_job *job) 353 { 354 return container_of(job, struct msm_gem_submit, base); 355 } 356 357 void __msm_gem_submit_destroy(struct kref *kref); 358 359 static inline void msm_gem_submit_get(struct msm_gem_submit *submit) 360 { 361 kref_get(&submit->ref); 362 } 363 364 static inline void msm_gem_submit_put(struct msm_gem_submit *submit) 365 { 366 kref_put(&submit->ref, __msm_gem_submit_destroy); 367 } 368 369 void msm_submit_retire(struct msm_gem_submit *submit); 370 371 /* helper to determine of a buffer in submit should be dumped, used for both 372 * devcoredump and debugfs cmdstream dumping: 373 */ 374 static inline bool 375 should_dump(struct msm_gem_submit *submit, int idx) 376 { 377 extern bool rd_full; 378 return rd_full || (submit->bos[idx].flags & MSM_SUBMIT_BO_DUMP); 379 } 380 381 #endif /* __MSM_GEM_H__ */ 382