1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #ifndef __I915_GEM_OBJECT_H__ 8 #define __I915_GEM_OBJECT_H__ 9 10 #include <drm/drm_gem.h> 11 #include <drm/drm_file.h> 12 #include <drm/drm_device.h> 13 14 #include "display/intel_frontbuffer.h" 15 #include "intel_memory_region.h" 16 #include "i915_gem_object_types.h" 17 #include "i915_gem_gtt.h" 18 #include "i915_gem_ww.h" 19 #include "i915_vma_types.h" 20 21 enum intel_region_id; 22 23 static inline bool i915_gem_object_size_2big(u64 size) 24 { 25 struct drm_i915_gem_object *obj; 26 27 if (overflows_type(size, obj->base.size)) 28 return true; 29 30 return false; 31 } 32 33 void i915_gem_init__objects(struct drm_i915_private *i915); 34 35 void i915_objects_module_exit(void); 36 int i915_objects_module_init(void); 37 38 struct drm_i915_gem_object *i915_gem_object_alloc(void); 39 void i915_gem_object_free(struct drm_i915_gem_object *obj); 40 41 void i915_gem_object_init(struct drm_i915_gem_object *obj, 42 const struct drm_i915_gem_object_ops *ops, 43 struct lock_class_key *key, 44 unsigned alloc_flags); 45 46 void __i915_gem_object_fini(struct drm_i915_gem_object *obj); 47 48 struct drm_i915_gem_object * 49 i915_gem_object_create_shmem(struct drm_i915_private *i915, 50 resource_size_t size); 51 struct drm_i915_gem_object * 52 i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, 53 const void *data, resource_size_t size); 54 struct drm_i915_gem_object * 55 __i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, 56 struct intel_memory_region **placements, 57 unsigned int n_placements); 58 59 extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops; 60 61 void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 62 struct sg_table *pages, 63 bool needs_clflush); 64 65 int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj, 66 const struct drm_i915_gem_pwrite *args); 67 int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj, 68 const struct drm_i915_gem_pread *args); 69 70 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align); 71 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, 72 struct sg_table *pages); 73 void i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 74 struct sg_table *pages); 75 76 void i915_gem_flush_free_objects(struct drm_i915_private *i915); 77 78 struct sg_table * 79 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj); 80 81 /** 82 * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle 83 * @filp: DRM file private date 84 * @handle: userspace handle 85 * 86 * Returns: 87 * 88 * A pointer to the object named by the handle if such exists on @filp, NULL 89 * otherwise. This object is only valid whilst under the RCU read lock, and 90 * note carefully the object may be in the process of being destroyed. 91 */ 92 static inline struct drm_i915_gem_object * 93 i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) 94 { 95 #ifdef CONFIG_LOCKDEP 96 WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); 97 #endif 98 return idr_find(&file->object_idr, handle); 99 } 100 101 static inline struct drm_i915_gem_object * 102 i915_gem_object_get_rcu(struct drm_i915_gem_object *obj) 103 { 104 if (obj && !kref_get_unless_zero(&obj->base.refcount)) 105 obj = NULL; 106 107 return obj; 108 } 109 110 static inline struct drm_i915_gem_object * 111 i915_gem_object_lookup(struct drm_file *file, u32 handle) 112 { 113 struct drm_i915_gem_object *obj; 114 115 rcu_read_lock(); 116 obj = i915_gem_object_lookup_rcu(file, handle); 117 obj = i915_gem_object_get_rcu(obj); 118 rcu_read_unlock(); 119 120 return obj; 121 } 122 123 __deprecated 124 struct drm_gem_object * 125 drm_gem_object_lookup(struct drm_file *file, u32 handle); 126 127 __attribute__((nonnull)) 128 static inline struct drm_i915_gem_object * 129 i915_gem_object_get(struct drm_i915_gem_object *obj) 130 { 131 drm_gem_object_get(&obj->base); 132 return obj; 133 } 134 135 __attribute__((nonnull)) 136 static inline void 137 i915_gem_object_put(struct drm_i915_gem_object *obj) 138 { 139 __drm_gem_object_put(&obj->base); 140 } 141 142 #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv) 143 144 /* 145 * If more than one potential simultaneous locker, assert held. 146 */ 147 static inline void assert_object_held_shared(const struct drm_i915_gem_object *obj) 148 { 149 /* 150 * Note mm list lookup is protected by 151 * kref_get_unless_zero(). 152 */ 153 if (IS_ENABLED(CONFIG_LOCKDEP) && 154 kref_read(&obj->base.refcount) > 0) 155 assert_object_held(obj); 156 } 157 158 static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj, 159 struct i915_gem_ww_ctx *ww, 160 bool intr) 161 { 162 int ret; 163 164 if (intr) 165 ret = dma_resv_lock_interruptible(obj->base.resv, ww ? &ww->ctx : NULL); 166 else 167 ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL); 168 169 if (!ret && ww) { 170 i915_gem_object_get(obj); 171 list_add_tail(&obj->obj_link, &ww->obj_list); 172 } 173 if (ret == -EALREADY) 174 ret = 0; 175 176 if (ret == -EDEADLK) { 177 i915_gem_object_get(obj); 178 ww->contended = obj; 179 } 180 181 return ret; 182 } 183 184 static inline int i915_gem_object_lock(struct drm_i915_gem_object *obj, 185 struct i915_gem_ww_ctx *ww) 186 { 187 return __i915_gem_object_lock(obj, ww, ww && ww->intr); 188 } 189 190 static inline int i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj, 191 struct i915_gem_ww_ctx *ww) 192 { 193 WARN_ON(ww && !ww->intr); 194 return __i915_gem_object_lock(obj, ww, true); 195 } 196 197 static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj, 198 struct i915_gem_ww_ctx *ww) 199 { 200 if (!ww) 201 return dma_resv_trylock(obj->base.resv); 202 else 203 return ww_mutex_trylock(&obj->base.resv->lock, &ww->ctx); 204 } 205 206 static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) 207 { 208 if (obj->ops->adjust_lru) 209 obj->ops->adjust_lru(obj); 210 211 dma_resv_unlock(obj->base.resv); 212 } 213 214 static inline void 215 i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) 216 { 217 obj->flags |= I915_BO_READONLY; 218 } 219 220 static inline bool 221 i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) 222 { 223 return obj->flags & I915_BO_READONLY; 224 } 225 226 static inline bool 227 i915_gem_object_is_contiguous(const struct drm_i915_gem_object *obj) 228 { 229 return obj->flags & I915_BO_ALLOC_CONTIGUOUS; 230 } 231 232 static inline bool 233 i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj) 234 { 235 return obj->flags & I915_BO_ALLOC_VOLATILE; 236 } 237 238 static inline void 239 i915_gem_object_set_volatile(struct drm_i915_gem_object *obj) 240 { 241 obj->flags |= I915_BO_ALLOC_VOLATILE; 242 } 243 244 static inline bool 245 i915_gem_object_has_tiling_quirk(struct drm_i915_gem_object *obj) 246 { 247 return test_bit(I915_TILING_QUIRK_BIT, &obj->flags); 248 } 249 250 static inline void 251 i915_gem_object_set_tiling_quirk(struct drm_i915_gem_object *obj) 252 { 253 set_bit(I915_TILING_QUIRK_BIT, &obj->flags); 254 } 255 256 static inline void 257 i915_gem_object_clear_tiling_quirk(struct drm_i915_gem_object *obj) 258 { 259 clear_bit(I915_TILING_QUIRK_BIT, &obj->flags); 260 } 261 262 static inline bool 263 i915_gem_object_is_protected(const struct drm_i915_gem_object *obj) 264 { 265 return obj->flags & I915_BO_PROTECTED; 266 } 267 268 static inline bool 269 i915_gem_object_type_has(const struct drm_i915_gem_object *obj, 270 unsigned long flags) 271 { 272 return obj->ops->flags & flags; 273 } 274 275 bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj); 276 277 bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj); 278 279 static inline bool 280 i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) 281 { 282 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE); 283 } 284 285 static inline bool 286 i915_gem_object_has_self_managed_shrink_list(const struct drm_i915_gem_object *obj) 287 { 288 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST); 289 } 290 291 static inline bool 292 i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj) 293 { 294 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_PROXY); 295 } 296 297 static inline bool 298 i915_gem_object_never_mmap(const struct drm_i915_gem_object *obj) 299 { 300 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP); 301 } 302 303 static inline bool 304 i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj) 305 { 306 return READ_ONCE(obj->frontbuffer); 307 } 308 309 static inline unsigned int 310 i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj) 311 { 312 return obj->tiling_and_stride & TILING_MASK; 313 } 314 315 static inline bool 316 i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj) 317 { 318 return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; 319 } 320 321 static inline unsigned int 322 i915_gem_object_get_stride(const struct drm_i915_gem_object *obj) 323 { 324 return obj->tiling_and_stride & STRIDE_MASK; 325 } 326 327 static inline unsigned int 328 i915_gem_tile_height(unsigned int tiling) 329 { 330 GEM_BUG_ON(!tiling); 331 return tiling == I915_TILING_Y ? 32 : 8; 332 } 333 334 static inline unsigned int 335 i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj) 336 { 337 return i915_gem_tile_height(i915_gem_object_get_tiling(obj)); 338 } 339 340 static inline unsigned int 341 i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj) 342 { 343 return (i915_gem_object_get_stride(obj) * 344 i915_gem_object_get_tile_height(obj)); 345 } 346 347 int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, 348 unsigned int tiling, unsigned int stride); 349 350 /** 351 * __i915_gem_object_page_iter_get_sg - helper to find the target scatterlist 352 * pointer and the target page position using pgoff_t n input argument and 353 * i915_gem_object_page_iter 354 * @obj: i915 GEM buffer object 355 * @iter: i915 GEM buffer object page iterator 356 * @n: page offset 357 * @offset: searched physical offset, 358 * it will be used for returning physical page offset value 359 * 360 * Context: Takes and releases the mutex lock of the i915_gem_object_page_iter. 361 * Takes and releases the RCU lock to search the radix_tree of 362 * i915_gem_object_page_iter. 363 * 364 * Returns: 365 * The target scatterlist pointer and the target page position. 366 * 367 * Recommended to use wrapper macro: i915_gem_object_page_iter_get_sg() 368 */ 369 struct scatterlist * 370 __i915_gem_object_page_iter_get_sg(struct drm_i915_gem_object *obj, 371 struct i915_gem_object_page_iter *iter, 372 pgoff_t n, 373 unsigned int *offset); 374 375 /** 376 * i915_gem_object_page_iter_get_sg - wrapper macro for 377 * __i915_gem_object_page_iter_get_sg() 378 * @obj: i915 GEM buffer object 379 * @it: i915 GEM buffer object page iterator 380 * @n: page offset 381 * @offset: searched physical offset, 382 * it will be used for returning physical page offset value 383 * 384 * Context: Takes and releases the mutex lock of the i915_gem_object_page_iter. 385 * Takes and releases the RCU lock to search the radix_tree of 386 * i915_gem_object_page_iter. 387 * 388 * Returns: 389 * The target scatterlist pointer and the target page position. 390 * 391 * In order to avoid the truncation of the input parameter, it checks the page 392 * offset n's type from the input parameter before calling 393 * __i915_gem_object_page_iter_get_sg(). 394 */ 395 #define i915_gem_object_page_iter_get_sg(obj, it, n, offset) ({ \ 396 static_assert(castable_to_type(n, pgoff_t)); \ 397 __i915_gem_object_page_iter_get_sg(obj, it, n, offset); \ 398 }) 399 400 /** 401 * __i915_gem_object_get_sg - helper to find the target scatterlist 402 * pointer and the target page position using pgoff_t n input argument and 403 * drm_i915_gem_object. It uses an internal shmem scatterlist lookup function. 404 * @obj: i915 GEM buffer object 405 * @n: page offset 406 * @offset: searched physical offset, 407 * it will be used for returning physical page offset value 408 * 409 * It uses drm_i915_gem_object's internal shmem scatterlist lookup function as 410 * i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg(). 411 * 412 * Returns: 413 * The target scatterlist pointer and the target page position. 414 * 415 * Recommended to use wrapper macro: i915_gem_object_get_sg() 416 * See also __i915_gem_object_page_iter_get_sg() 417 */ 418 static inline struct scatterlist * 419 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, pgoff_t n, 420 unsigned int *offset) 421 { 422 return __i915_gem_object_page_iter_get_sg(obj, &obj->mm.get_page, n, offset); 423 } 424 425 /** 426 * i915_gem_object_get_sg - wrapper macro for __i915_gem_object_get_sg() 427 * @obj: i915 GEM buffer object 428 * @n: page offset 429 * @offset: searched physical offset, 430 * it will be used for returning physical page offset value 431 * 432 * Returns: 433 * The target scatterlist pointer and the target page position. 434 * 435 * In order to avoid the truncation of the input parameter, it checks the page 436 * offset n's type from the input parameter before calling 437 * __i915_gem_object_get_sg(). 438 * See also __i915_gem_object_page_iter_get_sg() 439 */ 440 #define i915_gem_object_get_sg(obj, n, offset) ({ \ 441 static_assert(castable_to_type(n, pgoff_t)); \ 442 __i915_gem_object_get_sg(obj, n, offset); \ 443 }) 444 445 /** 446 * __i915_gem_object_get_sg_dma - helper to find the target scatterlist 447 * pointer and the target page position using pgoff_t n input argument and 448 * drm_i915_gem_object. It uses an internal DMA mapped scatterlist lookup function 449 * @obj: i915 GEM buffer object 450 * @n: page offset 451 * @offset: searched physical offset, 452 * it will be used for returning physical page offset value 453 * 454 * It uses drm_i915_gem_object's internal DMA mapped scatterlist lookup function 455 * as i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg(). 456 * 457 * Returns: 458 * The target scatterlist pointer and the target page position. 459 * 460 * Recommended to use wrapper macro: i915_gem_object_get_sg_dma() 461 * See also __i915_gem_object_page_iter_get_sg() 462 */ 463 static inline struct scatterlist * 464 __i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj, pgoff_t n, 465 unsigned int *offset) 466 { 467 return __i915_gem_object_page_iter_get_sg(obj, &obj->mm.get_dma_page, n, offset); 468 } 469 470 /** 471 * i915_gem_object_get_sg_dma - wrapper macro for __i915_gem_object_get_sg_dma() 472 * @obj: i915 GEM buffer object 473 * @n: page offset 474 * @offset: searched physical offset, 475 * it will be used for returning physical page offset value 476 * 477 * Returns: 478 * The target scatterlist pointer and the target page position. 479 * 480 * In order to avoid the truncation of the input parameter, it checks the page 481 * offset n's type from the input parameter before calling 482 * __i915_gem_object_get_sg_dma(). 483 * See also __i915_gem_object_page_iter_get_sg() 484 */ 485 #define i915_gem_object_get_sg_dma(obj, n, offset) ({ \ 486 static_assert(castable_to_type(n, pgoff_t)); \ 487 __i915_gem_object_get_sg_dma(obj, n, offset); \ 488 }) 489 490 /** 491 * __i915_gem_object_get_page - helper to find the target page with a page offset 492 * @obj: i915 GEM buffer object 493 * @n: page offset 494 * 495 * It uses drm_i915_gem_object's internal shmem scatterlist lookup function as 496 * i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg() 497 * internally. 498 * 499 * Returns: 500 * The target page pointer. 501 * 502 * Recommended to use wrapper macro: i915_gem_object_get_page() 503 * See also __i915_gem_object_page_iter_get_sg() 504 */ 505 struct page * 506 __i915_gem_object_get_page(struct drm_i915_gem_object *obj, pgoff_t n); 507 508 /** 509 * i915_gem_object_get_page - wrapper macro for __i915_gem_object_get_page 510 * @obj: i915 GEM buffer object 511 * @n: page offset 512 * 513 * Returns: 514 * The target page pointer. 515 * 516 * In order to avoid the truncation of the input parameter, it checks the page 517 * offset n's type from the input parameter before calling 518 * __i915_gem_object_get_page(). 519 * See also __i915_gem_object_page_iter_get_sg() 520 */ 521 #define i915_gem_object_get_page(obj, n) ({ \ 522 static_assert(castable_to_type(n, pgoff_t)); \ 523 __i915_gem_object_get_page(obj, n); \ 524 }) 525 526 /** 527 * __i915_gem_object_get_dirty_page - helper to find the target page with a page 528 * offset 529 * @obj: i915 GEM buffer object 530 * @n: page offset 531 * 532 * It works like i915_gem_object_get_page(), but it marks the returned page dirty. 533 * 534 * Returns: 535 * The target page pointer. 536 * 537 * Recommended to use wrapper macro: i915_gem_object_get_dirty_page() 538 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_page() 539 */ 540 struct page * 541 __i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, pgoff_t n); 542 543 /** 544 * i915_gem_object_get_dirty_page - wrapper macro for __i915_gem_object_get_dirty_page 545 * @obj: i915 GEM buffer object 546 * @n: page offset 547 * 548 * Returns: 549 * The target page pointer. 550 * 551 * In order to avoid the truncation of the input parameter, it checks the page 552 * offset n's type from the input parameter before calling 553 * __i915_gem_object_get_dirty_page(). 554 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_page() 555 */ 556 #define i915_gem_object_get_dirty_page(obj, n) ({ \ 557 static_assert(castable_to_type(n, pgoff_t)); \ 558 __i915_gem_object_get_dirty_page(obj, n); \ 559 }) 560 561 /** 562 * __i915_gem_object_get_dma_address_len - helper to get bus addresses of 563 * targeted DMA mapped scatterlist from i915 GEM buffer object and it's length 564 * @obj: i915 GEM buffer object 565 * @n: page offset 566 * @len: DMA mapped scatterlist's DMA bus addresses length to return 567 * 568 * Returns: 569 * Bus addresses of targeted DMA mapped scatterlist 570 * 571 * Recommended to use wrapper macro: i915_gem_object_get_dma_address_len() 572 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_sg_dma() 573 */ 574 dma_addr_t 575 __i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, pgoff_t n, 576 unsigned int *len); 577 578 /** 579 * i915_gem_object_get_dma_address_len - wrapper macro for 580 * __i915_gem_object_get_dma_address_len 581 * @obj: i915 GEM buffer object 582 * @n: page offset 583 * @len: DMA mapped scatterlist's DMA bus addresses length to return 584 * 585 * Returns: 586 * Bus addresses of targeted DMA mapped scatterlist 587 * 588 * In order to avoid the truncation of the input parameter, it checks the page 589 * offset n's type from the input parameter before calling 590 * __i915_gem_object_get_dma_address_len(). 591 * See also __i915_gem_object_page_iter_get_sg() and 592 * __i915_gem_object_get_dma_address_len() 593 */ 594 #define i915_gem_object_get_dma_address_len(obj, n, len) ({ \ 595 static_assert(castable_to_type(n, pgoff_t)); \ 596 __i915_gem_object_get_dma_address_len(obj, n, len); \ 597 }) 598 599 /** 600 * __i915_gem_object_get_dma_address - helper to get bus addresses of 601 * targeted DMA mapped scatterlist from i915 GEM buffer object 602 * @obj: i915 GEM buffer object 603 * @n: page offset 604 * 605 * Returns: 606 * Bus addresses of targeted DMA mapped scatterlis 607 * 608 * Recommended to use wrapper macro: i915_gem_object_get_dma_address() 609 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_sg_dma() 610 */ 611 dma_addr_t 612 __i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, pgoff_t n); 613 614 /** 615 * i915_gem_object_get_dma_address - wrapper macro for 616 * __i915_gem_object_get_dma_address 617 * @obj: i915 GEM buffer object 618 * @n: page offset 619 * 620 * Returns: 621 * Bus addresses of targeted DMA mapped scatterlist 622 * 623 * In order to avoid the truncation of the input parameter, it checks the page 624 * offset n's type from the input parameter before calling 625 * __i915_gem_object_get_dma_address(). 626 * See also __i915_gem_object_page_iter_get_sg() and 627 * __i915_gem_object_get_dma_address() 628 */ 629 #define i915_gem_object_get_dma_address(obj, n) ({ \ 630 static_assert(castable_to_type(n, pgoff_t)); \ 631 __i915_gem_object_get_dma_address(obj, n); \ 632 }) 633 634 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 635 struct sg_table *pages); 636 637 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 638 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 639 640 static inline int __must_check 641 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 642 { 643 assert_object_held(obj); 644 645 if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) 646 return 0; 647 648 return __i915_gem_object_get_pages(obj); 649 } 650 651 int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj); 652 653 static inline bool 654 i915_gem_object_has_pages(struct drm_i915_gem_object *obj) 655 { 656 return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)); 657 } 658 659 static inline void 660 __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 661 { 662 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 663 664 atomic_inc(&obj->mm.pages_pin_count); 665 } 666 667 static inline bool 668 i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) 669 { 670 return atomic_read(&obj->mm.pages_pin_count); 671 } 672 673 static inline void 674 __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 675 { 676 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 677 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 678 679 atomic_dec(&obj->mm.pages_pin_count); 680 } 681 682 static inline void 683 i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 684 { 685 __i915_gem_object_unpin_pages(obj); 686 } 687 688 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj); 689 int i915_gem_object_truncate(struct drm_i915_gem_object *obj); 690 691 /** 692 * i915_gem_object_pin_map - return a contiguous mapping of the entire object 693 * @obj: the object to map into kernel address space 694 * @type: the type of mapping, used to select pgprot_t 695 * 696 * Calls i915_gem_object_pin_pages() to prevent reaping of the object's 697 * pages and then returns a contiguous mapping of the backing storage into 698 * the kernel address space. Based on the @type of mapping, the PTE will be 699 * set to either WriteBack or WriteCombine (via pgprot_t). 700 * 701 * The caller is responsible for calling i915_gem_object_unpin_map() when the 702 * mapping is no longer required. 703 * 704 * Returns the pointer through which to access the mapped object, or an 705 * ERR_PTR() on error. 706 */ 707 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 708 enum i915_map_type type); 709 710 void *__must_check i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, 711 enum i915_map_type type); 712 713 enum i915_map_type i915_coherent_map_type(struct drm_i915_private *i915, 714 struct drm_i915_gem_object *obj, 715 bool always_coherent); 716 717 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 718 unsigned long offset, 719 unsigned long size); 720 static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj) 721 { 722 __i915_gem_object_flush_map(obj, 0, obj->base.size); 723 } 724 725 /** 726 * i915_gem_object_unpin_map - releases an earlier mapping 727 * @obj: the object to unmap 728 * 729 * After pinning the object and mapping its pages, once you are finished 730 * with your access, call i915_gem_object_unpin_map() to release the pin 731 * upon the mapping. Once the pin count reaches zero, that mapping may be 732 * removed. 733 */ 734 static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) 735 { 736 i915_gem_object_unpin_pages(obj); 737 } 738 739 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj); 740 741 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj, 742 unsigned int *needs_clflush); 743 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj, 744 unsigned int *needs_clflush); 745 #define CLFLUSH_BEFORE BIT(0) 746 #define CLFLUSH_AFTER BIT(1) 747 #define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER) 748 749 static inline void 750 i915_gem_object_finish_access(struct drm_i915_gem_object *obj) 751 { 752 i915_gem_object_unpin_pages(obj); 753 } 754 755 int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, 756 struct dma_fence **fence); 757 int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, 758 bool intr); 759 bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj); 760 761 void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, 762 unsigned int cache_level); 763 bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj); 764 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); 765 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj); 766 bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj); 767 768 int __must_check 769 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); 770 int __must_check 771 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); 772 int __must_check 773 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); 774 struct i915_vma * __must_check 775 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 776 struct i915_gem_ww_ctx *ww, 777 u32 alignment, 778 const struct i915_gtt_view *view, 779 unsigned int flags); 780 781 void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj); 782 void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 783 void __i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 784 void __i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 785 void i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 786 787 static inline void __start_cpu_write(struct drm_i915_gem_object *obj) 788 { 789 obj->read_domains = I915_GEM_DOMAIN_CPU; 790 obj->write_domain = I915_GEM_DOMAIN_CPU; 791 if (i915_gem_cpu_write_needs_clflush(obj)) 792 obj->cache_dirty = true; 793 } 794 795 void i915_gem_fence_wait_priority(struct dma_fence *fence, 796 const struct i915_sched_attr *attr); 797 798 int i915_gem_object_wait(struct drm_i915_gem_object *obj, 799 unsigned int flags, 800 long timeout); 801 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 802 unsigned int flags, 803 const struct i915_sched_attr *attr); 804 805 void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, 806 enum fb_op_origin origin); 807 void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, 808 enum fb_op_origin origin); 809 810 static inline void 811 i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, 812 enum fb_op_origin origin) 813 { 814 if (unlikely(rcu_access_pointer(obj->frontbuffer))) 815 __i915_gem_object_flush_frontbuffer(obj, origin); 816 } 817 818 static inline void 819 i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, 820 enum fb_op_origin origin) 821 { 822 if (unlikely(rcu_access_pointer(obj->frontbuffer))) 823 __i915_gem_object_invalidate_frontbuffer(obj, origin); 824 } 825 826 int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size); 827 828 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj); 829 830 void __i915_gem_free_object_rcu(struct rcu_head *head); 831 832 void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj); 833 834 void __i915_gem_free_object(struct drm_i915_gem_object *obj); 835 836 bool i915_gem_object_evictable(struct drm_i915_gem_object *obj); 837 838 bool i915_gem_object_migratable(struct drm_i915_gem_object *obj); 839 840 int i915_gem_object_migrate(struct drm_i915_gem_object *obj, 841 struct i915_gem_ww_ctx *ww, 842 enum intel_region_id id); 843 int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, 844 struct i915_gem_ww_ctx *ww, 845 enum intel_region_id id, 846 unsigned int flags); 847 848 bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, 849 enum intel_region_id id); 850 851 int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, 852 unsigned int flags); 853 854 bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, 855 enum intel_memory_type type); 856 857 bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj); 858 859 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 860 size_t size, struct intel_memory_region *mr, 861 struct address_space *mapping, 862 unsigned int max_segment); 863 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 864 bool dirty, bool backup); 865 void __shmem_writeback(size_t size, struct address_space *mapping); 866 867 #ifdef CONFIG_MMU_NOTIFIER 868 static inline bool 869 i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) 870 { 871 return obj->userptr.notifier.mm; 872 } 873 874 int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj); 875 int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj); 876 int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj); 877 #else 878 static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) { return false; } 879 880 static inline int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 881 static inline int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 882 static inline int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 883 884 #endif 885 886 #endif 887