1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #ifndef __I915_VMA_H__ 26 #define __I915_VMA_H__ 27 28 #include <linux/io-mapping.h> 29 #include <linux/rbtree.h> 30 31 #include <drm/drm_mm.h> 32 33 #include "i915_gem_gtt.h" 34 #include "i915_gem_fence_reg.h" 35 #include "gem/i915_gem_object.h" 36 37 #include "i915_active.h" 38 #include "i915_request.h" 39 40 enum i915_cache_level; 41 42 /** 43 * DOC: Virtual Memory Address 44 * 45 * A VMA represents a GEM BO that is bound into an address space. Therefore, a 46 * VMA's presence cannot be guaranteed before binding, or after unbinding the 47 * object into/from the address space. 48 * 49 * To make things as simple as possible (ie. no refcounting), a VMA's lifetime 50 * will always be <= an objects lifetime. So object refcounting should cover us. 51 */ 52 struct i915_vma { 53 struct drm_mm_node node; 54 struct drm_i915_gem_object *obj; 55 struct i915_address_space *vm; 56 const struct i915_vma_ops *ops; 57 struct i915_fence_reg *fence; 58 struct dma_resv *resv; /** Alias of obj->resv */ 59 struct sg_table *pages; 60 void __iomem *iomap; 61 void *private; /* owned by creator */ 62 u64 size; 63 u64 display_alignment; 64 struct i915_page_sizes page_sizes; 65 66 u32 fence_size; 67 u32 fence_alignment; 68 69 /** 70 * Count of the number of times this vma has been opened by different 71 * handles (but same file) for execbuf, i.e. the number of aliases 72 * that exist in the ctx->handle_vmas LUT for this vma. 73 */ 74 atomic_t open_count; 75 unsigned long flags; 76 /** 77 * How many users have pinned this object in GTT space. 78 * 79 * This is a tightly bound, fairly small number of users, so we 80 * stuff inside the flags field so that we can both check for overflow 81 * and detect a no-op i915_vma_pin() in a single check, while also 82 * pinning the vma. 83 * 84 * The worst case display setup would have the same vma pinned for 85 * use on each plane on each crtc, while also building the next atomic 86 * state and holding a pin for the length of the cleanup queue. In the 87 * future, the flip queue may be increased from 1. 88 * Estimated worst case: 3 [qlen] * 4 [max crtcs] * 7 [max planes] = 84 89 * 90 * For GEM, the number of concurrent users for pwrite/pread is 91 * unbounded. For execbuffer, it is currently one but will in future 92 * be extended to allow multiple clients to pin vma concurrently. 93 * 94 * We also use suballocated pages, with each suballocation claiming 95 * its own pin on the shared vma. At present, this is limited to 96 * exclusive cachelines of a single page, so a maximum of 64 possible 97 * users. 98 */ 99 #define I915_VMA_PIN_MASK 0xff 100 #define I915_VMA_PIN_OVERFLOW BIT(8) 101 102 /** Flags and address space this VMA is bound to */ 103 #define I915_VMA_GLOBAL_BIND BIT(9) 104 #define I915_VMA_LOCAL_BIND BIT(10) 105 #define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW) 106 107 #define I915_VMA_GGTT BIT(11) 108 #define I915_VMA_CAN_FENCE BIT(12) 109 #define I915_VMA_USERFAULT_BIT 13 110 #define I915_VMA_USERFAULT BIT(I915_VMA_USERFAULT_BIT) 111 #define I915_VMA_GGTT_WRITE BIT(14) 112 113 struct i915_active active; 114 115 /** 116 * Support different GGTT views into the same object. 117 * This means there can be multiple VMA mappings per object and per VM. 118 * i915_ggtt_view_type is used to distinguish between those entries. 119 * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also 120 * assumed in GEM functions which take no ggtt view parameter. 121 */ 122 struct i915_ggtt_view ggtt_view; 123 124 /** This object's place on the active/inactive lists */ 125 struct list_head vm_link; 126 127 struct list_head obj_link; /* Link in the object's VMA list */ 128 struct rb_node obj_node; 129 struct hlist_node obj_hash; 130 131 /** This vma's place in the execbuf reservation list */ 132 struct list_head exec_link; 133 struct list_head reloc_link; 134 135 /** This vma's place in the eviction list */ 136 struct list_head evict_link; 137 138 struct list_head closed_link; 139 140 /** 141 * Used for performing relocations during execbuffer insertion. 142 */ 143 unsigned int *exec_flags; 144 struct hlist_node exec_node; 145 u32 exec_handle; 146 }; 147 148 struct i915_vma * 149 i915_vma_instance(struct drm_i915_gem_object *obj, 150 struct i915_address_space *vm, 151 const struct i915_ggtt_view *view); 152 153 void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags); 154 #define I915_VMA_RELEASE_MAP BIT(0) 155 156 static inline bool i915_vma_is_active(const struct i915_vma *vma) 157 { 158 return !i915_active_is_idle(&vma->active); 159 } 160 161 int __must_check i915_vma_move_to_active(struct i915_vma *vma, 162 struct i915_request *rq, 163 unsigned int flags); 164 165 static inline bool i915_vma_is_ggtt(const struct i915_vma *vma) 166 { 167 return vma->flags & I915_VMA_GGTT; 168 } 169 170 static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma) 171 { 172 return vma->flags & I915_VMA_GGTT_WRITE; 173 } 174 175 static inline void i915_vma_set_ggtt_write(struct i915_vma *vma) 176 { 177 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 178 vma->flags |= I915_VMA_GGTT_WRITE; 179 } 180 181 static inline void i915_vma_unset_ggtt_write(struct i915_vma *vma) 182 { 183 vma->flags &= ~I915_VMA_GGTT_WRITE; 184 } 185 186 void i915_vma_flush_writes(struct i915_vma *vma); 187 188 static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma) 189 { 190 return vma->flags & I915_VMA_CAN_FENCE; 191 } 192 193 static inline bool i915_vma_set_userfault(struct i915_vma *vma) 194 { 195 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); 196 return __test_and_set_bit(I915_VMA_USERFAULT_BIT, &vma->flags); 197 } 198 199 static inline void i915_vma_unset_userfault(struct i915_vma *vma) 200 { 201 return __clear_bit(I915_VMA_USERFAULT_BIT, &vma->flags); 202 } 203 204 static inline bool i915_vma_has_userfault(const struct i915_vma *vma) 205 { 206 return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags); 207 } 208 209 static inline bool i915_vma_is_closed(const struct i915_vma *vma) 210 { 211 return !list_empty(&vma->closed_link); 212 } 213 214 static inline u32 i915_ggtt_offset(const struct i915_vma *vma) 215 { 216 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 217 GEM_BUG_ON(!vma->node.allocated); 218 GEM_BUG_ON(upper_32_bits(vma->node.start)); 219 GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1)); 220 return lower_32_bits(vma->node.start); 221 } 222 223 static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma) 224 { 225 return i915_vm_to_ggtt(vma->vm)->pin_bias; 226 } 227 228 static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) 229 { 230 i915_gem_object_get(vma->obj); 231 return vma; 232 } 233 234 static inline struct i915_vma *i915_vma_tryget(struct i915_vma *vma) 235 { 236 if (likely(kref_get_unless_zero(&vma->obj->base.refcount))) 237 return vma; 238 239 return NULL; 240 } 241 242 static inline void i915_vma_put(struct i915_vma *vma) 243 { 244 i915_gem_object_put(vma->obj); 245 } 246 247 static __always_inline ptrdiff_t ptrdiff(const void *a, const void *b) 248 { 249 return a - b; 250 } 251 252 static inline long 253 i915_vma_compare(struct i915_vma *vma, 254 struct i915_address_space *vm, 255 const struct i915_ggtt_view *view) 256 { 257 ptrdiff_t cmp; 258 259 GEM_BUG_ON(view && !i915_is_ggtt(vm)); 260 261 cmp = ptrdiff(vma->vm, vm); 262 if (cmp) 263 return cmp; 264 265 BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL != 0); 266 cmp = vma->ggtt_view.type; 267 if (!view) 268 return cmp; 269 270 cmp -= view->type; 271 if (cmp) 272 return cmp; 273 274 assert_i915_gem_gtt_types(); 275 276 /* ggtt_view.type also encodes its size so that we both distinguish 277 * different views using it as a "type" and also use a compact (no 278 * accessing of uninitialised padding bytes) memcmp without storing 279 * an extra parameter or adding more code. 280 * 281 * To ensure that the memcmp is valid for all branches of the union, 282 * even though the code looks like it is just comparing one branch, 283 * we assert above that all branches have the same address, and that 284 * each branch has a unique type/size. 285 */ 286 BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL >= I915_GGTT_VIEW_PARTIAL); 287 BUILD_BUG_ON(I915_GGTT_VIEW_PARTIAL >= I915_GGTT_VIEW_ROTATED); 288 BUILD_BUG_ON(I915_GGTT_VIEW_ROTATED >= I915_GGTT_VIEW_REMAPPED); 289 BUILD_BUG_ON(offsetof(typeof(*view), rotated) != 290 offsetof(typeof(*view), partial)); 291 BUILD_BUG_ON(offsetof(typeof(*view), rotated) != 292 offsetof(typeof(*view), remapped)); 293 return memcmp(&vma->ggtt_view.partial, &view->partial, view->type); 294 } 295 296 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, 297 u32 flags); 298 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level); 299 bool i915_vma_misplaced(const struct i915_vma *vma, 300 u64 size, u64 alignment, u64 flags); 301 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma); 302 void i915_vma_revoke_mmap(struct i915_vma *vma); 303 int __must_check i915_vma_unbind(struct i915_vma *vma); 304 void i915_vma_unlink_ctx(struct i915_vma *vma); 305 void i915_vma_close(struct i915_vma *vma); 306 void i915_vma_reopen(struct i915_vma *vma); 307 void i915_vma_destroy(struct i915_vma *vma); 308 309 #define assert_vma_held(vma) dma_resv_assert_held((vma)->resv) 310 311 static inline void i915_vma_lock(struct i915_vma *vma) 312 { 313 dma_resv_lock(vma->resv, NULL); 314 } 315 316 static inline void i915_vma_unlock(struct i915_vma *vma) 317 { 318 dma_resv_unlock(vma->resv); 319 } 320 321 int __i915_vma_do_pin(struct i915_vma *vma, 322 u64 size, u64 alignment, u64 flags); 323 static inline int __must_check 324 i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) 325 { 326 BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW); 327 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); 328 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); 329 330 /* Pin early to prevent the shrinker/eviction logic from destroying 331 * our vma as we insert and bind. 332 */ 333 if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) { 334 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 335 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); 336 return 0; 337 } 338 339 return __i915_vma_do_pin(vma, size, alignment, flags); 340 } 341 342 static inline int i915_vma_pin_count(const struct i915_vma *vma) 343 { 344 return vma->flags & I915_VMA_PIN_MASK; 345 } 346 347 static inline bool i915_vma_is_pinned(const struct i915_vma *vma) 348 { 349 return i915_vma_pin_count(vma); 350 } 351 352 static inline void __i915_vma_pin(struct i915_vma *vma) 353 { 354 vma->flags++; 355 GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW); 356 } 357 358 static inline void __i915_vma_unpin(struct i915_vma *vma) 359 { 360 vma->flags--; 361 } 362 363 static inline void i915_vma_unpin(struct i915_vma *vma) 364 { 365 GEM_BUG_ON(!i915_vma_is_pinned(vma)); 366 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 367 __i915_vma_unpin(vma); 368 } 369 370 static inline bool i915_vma_is_bound(const struct i915_vma *vma, 371 unsigned int where) 372 { 373 return vma->flags & where; 374 } 375 376 /** 377 * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture 378 * @vma: VMA to iomap 379 * 380 * The passed in VMA has to be pinned in the global GTT mappable region. 381 * An extra pinning of the VMA is acquired for the return iomapping, 382 * the caller must call i915_vma_unpin_iomap to relinquish the pinning 383 * after the iomapping is no longer required. 384 * 385 * Callers must hold the struct_mutex. 386 * 387 * Returns a valid iomapped pointer or ERR_PTR. 388 */ 389 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); 390 #define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x)) 391 392 /** 393 * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap 394 * @vma: VMA to unpin 395 * 396 * Unpins the previously iomapped VMA from i915_vma_pin_iomap(). 397 * 398 * Callers must hold the struct_mutex. This function is only valid to be 399 * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap(). 400 */ 401 void i915_vma_unpin_iomap(struct i915_vma *vma); 402 403 static inline struct page *i915_vma_first_page(struct i915_vma *vma) 404 { 405 GEM_BUG_ON(!vma->pages); 406 return sg_page(vma->pages->sgl); 407 } 408 409 /** 410 * i915_vma_pin_fence - pin fencing state 411 * @vma: vma to pin fencing for 412 * 413 * This pins the fencing state (whether tiled or untiled) to make sure the 414 * vma (and its object) is ready to be used as a scanout target. Fencing 415 * status must be synchronize first by calling i915_vma_get_fence(): 416 * 417 * The resulting fence pin reference must be released again with 418 * i915_vma_unpin_fence(). 419 * 420 * Returns: 421 * 422 * True if the vma has a fence, false otherwise. 423 */ 424 int __must_check i915_vma_pin_fence(struct i915_vma *vma); 425 int __must_check i915_vma_revoke_fence(struct i915_vma *vma); 426 427 static inline void __i915_vma_unpin_fence(struct i915_vma *vma) 428 { 429 GEM_BUG_ON(atomic_read(&vma->fence->pin_count) <= 0); 430 atomic_dec(&vma->fence->pin_count); 431 } 432 433 /** 434 * i915_vma_unpin_fence - unpin fencing state 435 * @vma: vma to unpin fencing for 436 * 437 * This releases the fence pin reference acquired through 438 * i915_vma_pin_fence. It will handle both objects with and without an 439 * attached fence correctly, callers do not need to distinguish this. 440 */ 441 static inline void 442 i915_vma_unpin_fence(struct i915_vma *vma) 443 { 444 /* lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); */ 445 if (vma->fence) 446 __i915_vma_unpin_fence(vma); 447 } 448 449 void i915_vma_parked(struct drm_i915_private *i915); 450 451 #define for_each_until(cond) if (cond) break; else 452 453 /** 454 * for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object. 455 * @V: the #i915_vma iterator 456 * @OBJ: the #drm_i915_gem_object 457 * 458 * GGTT VMA are placed at the being of the object's vma_list, see 459 * vma_create(), so we can stop our walk as soon as we see a ppgtt VMA, 460 * or the list is empty ofc. 461 */ 462 #define for_each_ggtt_vma(V, OBJ) \ 463 list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \ 464 for_each_until(!i915_vma_is_ggtt(V)) 465 466 struct i915_vma *i915_vma_alloc(void); 467 void i915_vma_free(struct i915_vma *vma); 468 469 struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma); 470 void i915_vma_make_shrinkable(struct i915_vma *vma); 471 void i915_vma_make_purgeable(struct i915_vma *vma); 472 473 #endif 474