1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include "display/intel_frontbuffer.h" 8 9 #include "i915_drv.h" 10 #include "i915_gem_clflush.h" 11 #include "i915_gem_gtt.h" 12 #include "i915_gem_ioctls.h" 13 #include "i915_gem_object.h" 14 #include "i915_vma.h" 15 #include "i915_gem_lmem.h" 16 #include "i915_gem_mman.h" 17 18 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj) 19 { 20 /* 21 * We manually flush the CPU domain so that we can override and 22 * force the flush for the display, and perform it asyncrhonously. 23 */ 24 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 25 if (obj->cache_dirty) 26 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE); 27 obj->write_domain = 0; 28 } 29 30 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj) 31 { 32 if (!i915_gem_object_is_framebuffer(obj)) 33 return; 34 35 i915_gem_object_lock(obj, NULL); 36 __i915_gem_object_flush_for_display(obj); 37 i915_gem_object_unlock(obj); 38 } 39 40 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj) 41 { 42 if (i915_gem_object_is_framebuffer(obj)) 43 __i915_gem_object_flush_for_display(obj); 44 } 45 46 /** 47 * Moves a single object to the WC read, and possibly write domain. 48 * @obj: object to act on 49 * @write: ask for write access or read only 50 * 51 * This function returns when the move is complete, including waiting on 52 * flushes to occur. 53 */ 54 int 55 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write) 56 { 57 int ret; 58 59 assert_object_held(obj); 60 61 ret = i915_gem_object_wait(obj, 62 I915_WAIT_INTERRUPTIBLE | 63 (write ? I915_WAIT_ALL : 0), 64 MAX_SCHEDULE_TIMEOUT); 65 if (ret) 66 return ret; 67 68 if (obj->write_domain == I915_GEM_DOMAIN_WC) 69 return 0; 70 71 /* Flush and acquire obj->pages so that we are coherent through 72 * direct access in memory with previous cached writes through 73 * shmemfs and that our cache domain tracking remains valid. 74 * For example, if the obj->filp was moved to swap without us 75 * being notified and releasing the pages, we would mistakenly 76 * continue to assume that the obj remained out of the CPU cached 77 * domain. 78 */ 79 ret = i915_gem_object_pin_pages(obj); 80 if (ret) 81 return ret; 82 83 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_WC); 84 85 /* Serialise direct access to this object with the barriers for 86 * coherent writes from the GPU, by effectively invalidating the 87 * WC domain upon first access. 88 */ 89 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0) 90 mb(); 91 92 /* It should now be out of any other write domains, and we can update 93 * the domain values for our changes. 94 */ 95 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0); 96 obj->read_domains |= I915_GEM_DOMAIN_WC; 97 if (write) { 98 obj->read_domains = I915_GEM_DOMAIN_WC; 99 obj->write_domain = I915_GEM_DOMAIN_WC; 100 obj->mm.dirty = true; 101 } 102 103 i915_gem_object_unpin_pages(obj); 104 return 0; 105 } 106 107 /** 108 * Moves a single object to the GTT read, and possibly write domain. 109 * @obj: object to act on 110 * @write: ask for write access or read only 111 * 112 * This function returns when the move is complete, including waiting on 113 * flushes to occur. 114 */ 115 int 116 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 117 { 118 int ret; 119 120 assert_object_held(obj); 121 122 ret = i915_gem_object_wait(obj, 123 I915_WAIT_INTERRUPTIBLE | 124 (write ? I915_WAIT_ALL : 0), 125 MAX_SCHEDULE_TIMEOUT); 126 if (ret) 127 return ret; 128 129 if (obj->write_domain == I915_GEM_DOMAIN_GTT) 130 return 0; 131 132 /* Flush and acquire obj->pages so that we are coherent through 133 * direct access in memory with previous cached writes through 134 * shmemfs and that our cache domain tracking remains valid. 135 * For example, if the obj->filp was moved to swap without us 136 * being notified and releasing the pages, we would mistakenly 137 * continue to assume that the obj remained out of the CPU cached 138 * domain. 139 */ 140 ret = i915_gem_object_pin_pages(obj); 141 if (ret) 142 return ret; 143 144 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT); 145 146 /* Serialise direct access to this object with the barriers for 147 * coherent writes from the GPU, by effectively invalidating the 148 * GTT domain upon first access. 149 */ 150 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0) 151 mb(); 152 153 /* It should now be out of any other write domains, and we can update 154 * the domain values for our changes. 155 */ 156 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 157 obj->read_domains |= I915_GEM_DOMAIN_GTT; 158 if (write) { 159 struct i915_vma *vma; 160 161 obj->read_domains = I915_GEM_DOMAIN_GTT; 162 obj->write_domain = I915_GEM_DOMAIN_GTT; 163 obj->mm.dirty = true; 164 165 spin_lock(&obj->vma.lock); 166 for_each_ggtt_vma(vma, obj) 167 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 168 i915_vma_set_ggtt_write(vma); 169 spin_unlock(&obj->vma.lock); 170 } 171 172 i915_gem_object_unpin_pages(obj); 173 return 0; 174 } 175 176 /** 177 * Changes the cache-level of an object across all VMA. 178 * @obj: object to act on 179 * @cache_level: new cache level to set for the object 180 * 181 * After this function returns, the object will be in the new cache-level 182 * across all GTT and the contents of the backing storage will be coherent, 183 * with respect to the new cache-level. In order to keep the backing storage 184 * coherent for all users, we only allow a single cache level to be set 185 * globally on the object and prevent it from being changed whilst the 186 * hardware is reading from the object. That is if the object is currently 187 * on the scanout it will be set to uncached (or equivalent display 188 * cache coherency) and all non-MOCS GPU access will also be uncached so 189 * that all direct access to the scanout remains coherent. 190 */ 191 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 192 enum i915_cache_level cache_level) 193 { 194 int ret; 195 196 if (obj->cache_level == cache_level) 197 return 0; 198 199 ret = i915_gem_object_wait(obj, 200 I915_WAIT_INTERRUPTIBLE | 201 I915_WAIT_ALL, 202 MAX_SCHEDULE_TIMEOUT); 203 if (ret) 204 return ret; 205 206 /* Always invalidate stale cachelines */ 207 if (obj->cache_level != cache_level) { 208 i915_gem_object_set_cache_coherency(obj, cache_level); 209 obj->cache_dirty = true; 210 } 211 212 /* The cache-level will be applied when each vma is rebound. */ 213 return i915_gem_object_unbind(obj, 214 I915_GEM_OBJECT_UNBIND_ACTIVE | 215 I915_GEM_OBJECT_UNBIND_BARRIER); 216 } 217 218 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 219 struct drm_file *file) 220 { 221 struct drm_i915_gem_caching *args = data; 222 struct drm_i915_gem_object *obj; 223 int err = 0; 224 225 rcu_read_lock(); 226 obj = i915_gem_object_lookup_rcu(file, args->handle); 227 if (!obj) { 228 err = -ENOENT; 229 goto out; 230 } 231 232 switch (obj->cache_level) { 233 case I915_CACHE_LLC: 234 case I915_CACHE_L3_LLC: 235 args->caching = I915_CACHING_CACHED; 236 break; 237 238 case I915_CACHE_WT: 239 args->caching = I915_CACHING_DISPLAY; 240 break; 241 242 default: 243 args->caching = I915_CACHING_NONE; 244 break; 245 } 246 out: 247 rcu_read_unlock(); 248 return err; 249 } 250 251 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 252 struct drm_file *file) 253 { 254 struct drm_i915_private *i915 = to_i915(dev); 255 struct drm_i915_gem_caching *args = data; 256 struct drm_i915_gem_object *obj; 257 enum i915_cache_level level; 258 int ret = 0; 259 260 switch (args->caching) { 261 case I915_CACHING_NONE: 262 level = I915_CACHE_NONE; 263 break; 264 case I915_CACHING_CACHED: 265 /* 266 * Due to a HW issue on BXT A stepping, GPU stores via a 267 * snooped mapping may leave stale data in a corresponding CPU 268 * cacheline, whereas normally such cachelines would get 269 * invalidated. 270 */ 271 if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) 272 return -ENODEV; 273 274 level = I915_CACHE_LLC; 275 break; 276 case I915_CACHING_DISPLAY: 277 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE; 278 break; 279 default: 280 return -EINVAL; 281 } 282 283 obj = i915_gem_object_lookup(file, args->handle); 284 if (!obj) 285 return -ENOENT; 286 287 /* 288 * The caching mode of proxy object is handled by its generator, and 289 * not allowed to be changed by userspace. 290 */ 291 if (i915_gem_object_is_proxy(obj)) { 292 ret = -ENXIO; 293 goto out; 294 } 295 296 ret = i915_gem_object_lock_interruptible(obj, NULL); 297 if (ret) 298 goto out; 299 300 ret = i915_gem_object_set_cache_level(obj, level); 301 i915_gem_object_unlock(obj); 302 303 out: 304 i915_gem_object_put(obj); 305 return ret; 306 } 307 308 /* 309 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from 310 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined 311 * (for pageflips). We only flush the caches while preparing the buffer for 312 * display, the callers are responsible for frontbuffer flush. 313 */ 314 struct i915_vma * 315 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 316 u32 alignment, 317 const struct i915_ggtt_view *view, 318 unsigned int flags) 319 { 320 struct drm_i915_private *i915 = to_i915(obj->base.dev); 321 struct i915_gem_ww_ctx ww; 322 struct i915_vma *vma; 323 int ret; 324 325 /* Frame buffer must be in LMEM (no migration yet) */ 326 if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj)) 327 return ERR_PTR(-EINVAL); 328 329 i915_gem_ww_ctx_init(&ww, true); 330 retry: 331 ret = i915_gem_object_lock(obj, &ww); 332 if (ret) 333 goto err; 334 /* 335 * The display engine is not coherent with the LLC cache on gen6. As 336 * a result, we make sure that the pinning that is about to occur is 337 * done with uncached PTEs. This is lowest common denominator for all 338 * chipsets. 339 * 340 * However for gen6+, we could do better by using the GFDT bit instead 341 * of uncaching, which would allow us to flush all the LLC-cached data 342 * with that bit in the PTE to main memory with just one PIPE_CONTROL. 343 */ 344 ret = i915_gem_object_set_cache_level(obj, 345 HAS_WT(i915) ? 346 I915_CACHE_WT : I915_CACHE_NONE); 347 if (ret) 348 goto err; 349 350 /* 351 * As the user may map the buffer once pinned in the display plane 352 * (e.g. libkms for the bootup splash), we have to ensure that we 353 * always use map_and_fenceable for all scanout buffers. However, 354 * it may simply be too big to fit into mappable, in which case 355 * put it anyway and hope that userspace can cope (but always first 356 * try to preserve the existing ABI). 357 */ 358 vma = ERR_PTR(-ENOSPC); 359 if ((flags & PIN_MAPPABLE) == 0 && 360 (!view || view->type == I915_GGTT_VIEW_NORMAL)) 361 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, alignment, 362 flags | PIN_MAPPABLE | 363 PIN_NONBLOCK); 364 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) 365 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, 366 alignment, flags); 367 if (IS_ERR(vma)) { 368 ret = PTR_ERR(vma); 369 goto err; 370 } 371 372 vma->display_alignment = max_t(u64, vma->display_alignment, alignment); 373 374 i915_gem_object_flush_if_display_locked(obj); 375 376 err: 377 if (ret == -EDEADLK) { 378 ret = i915_gem_ww_ctx_backoff(&ww); 379 if (!ret) 380 goto retry; 381 } 382 i915_gem_ww_ctx_fini(&ww); 383 384 if (ret) 385 return ERR_PTR(ret); 386 387 return vma; 388 } 389 390 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) 391 { 392 struct drm_i915_private *i915 = to_i915(obj->base.dev); 393 struct i915_vma *vma; 394 395 if (list_empty(&obj->vma.list)) 396 return; 397 398 mutex_lock(&i915->ggtt.vm.mutex); 399 spin_lock(&obj->vma.lock); 400 for_each_ggtt_vma(vma, obj) { 401 if (!drm_mm_node_allocated(&vma->node)) 402 continue; 403 404 GEM_BUG_ON(vma->vm != &i915->ggtt.vm); 405 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 406 } 407 spin_unlock(&obj->vma.lock); 408 mutex_unlock(&i915->ggtt.vm.mutex); 409 410 if (i915_gem_object_is_shrinkable(obj)) { 411 unsigned long flags; 412 413 spin_lock_irqsave(&i915->mm.obj_lock, flags); 414 415 if (obj->mm.madv == I915_MADV_WILLNEED && 416 !atomic_read(&obj->mm.shrink_pin)) 417 list_move_tail(&obj->mm.link, &i915->mm.shrink_list); 418 419 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 420 } 421 } 422 423 void 424 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) 425 { 426 /* Bump the LRU to try and avoid premature eviction whilst flipping */ 427 i915_gem_object_bump_inactive_ggtt(vma->obj); 428 429 i915_vma_unpin(vma); 430 } 431 432 /** 433 * Moves a single object to the CPU read, and possibly write domain. 434 * @obj: object to act on 435 * @write: requesting write or read-only access 436 * 437 * This function returns when the move is complete, including waiting on 438 * flushes to occur. 439 */ 440 int 441 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 442 { 443 int ret; 444 445 assert_object_held(obj); 446 447 ret = i915_gem_object_wait(obj, 448 I915_WAIT_INTERRUPTIBLE | 449 (write ? I915_WAIT_ALL : 0), 450 MAX_SCHEDULE_TIMEOUT); 451 if (ret) 452 return ret; 453 454 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 455 456 /* Flush the CPU cache if it's still invalid. */ 457 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 458 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); 459 obj->read_domains |= I915_GEM_DOMAIN_CPU; 460 } 461 462 /* It should now be out of any other write domains, and we can update 463 * the domain values for our changes. 464 */ 465 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU); 466 467 /* If we're writing through the CPU, then the GPU read domains will 468 * need to be invalidated at next use. 469 */ 470 if (write) 471 __start_cpu_write(obj); 472 473 return 0; 474 } 475 476 /** 477 * Called when user space prepares to use an object with the CPU, either 478 * through the mmap ioctl's mapping or a GTT mapping. 479 * @dev: drm device 480 * @data: ioctl data blob 481 * @file: drm file 482 */ 483 int 484 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 485 struct drm_file *file) 486 { 487 struct drm_i915_gem_set_domain *args = data; 488 struct drm_i915_gem_object *obj; 489 u32 read_domains = args->read_domains; 490 u32 write_domain = args->write_domain; 491 int err; 492 493 /* Only handle setting domains to types used by the CPU. */ 494 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS) 495 return -EINVAL; 496 497 /* 498 * Having something in the write domain implies it's in the read 499 * domain, and only that read domain. Enforce that in the request. 500 */ 501 if (write_domain && read_domains != write_domain) 502 return -EINVAL; 503 504 if (!read_domains) 505 return 0; 506 507 obj = i915_gem_object_lookup(file, args->handle); 508 if (!obj) 509 return -ENOENT; 510 511 /* 512 * Already in the desired write domain? Nothing for us to do! 513 * 514 * We apply a little bit of cunning here to catch a broader set of 515 * no-ops. If obj->write_domain is set, we must be in the same 516 * obj->read_domains, and only that domain. Therefore, if that 517 * obj->write_domain matches the request read_domains, we are 518 * already in the same read/write domain and can skip the operation, 519 * without having to further check the requested write_domain. 520 */ 521 if (READ_ONCE(obj->write_domain) == read_domains) { 522 err = 0; 523 goto out; 524 } 525 526 /* 527 * Try to flush the object off the GPU without holding the lock. 528 * We will repeat the flush holding the lock in the normal manner 529 * to catch cases where we are gazumped. 530 */ 531 err = i915_gem_object_wait(obj, 532 I915_WAIT_INTERRUPTIBLE | 533 I915_WAIT_PRIORITY | 534 (write_domain ? I915_WAIT_ALL : 0), 535 MAX_SCHEDULE_TIMEOUT); 536 if (err) 537 goto out; 538 539 /* 540 * Proxy objects do not control access to the backing storage, ergo 541 * they cannot be used as a means to manipulate the cache domain 542 * tracking for that backing storage. The proxy object is always 543 * considered to be outside of any cache domain. 544 */ 545 if (i915_gem_object_is_proxy(obj)) { 546 err = -ENXIO; 547 goto out; 548 } 549 550 /* 551 * Flush and acquire obj->pages so that we are coherent through 552 * direct access in memory with previous cached writes through 553 * shmemfs and that our cache domain tracking remains valid. 554 * For example, if the obj->filp was moved to swap without us 555 * being notified and releasing the pages, we would mistakenly 556 * continue to assume that the obj remained out of the CPU cached 557 * domain. 558 */ 559 err = i915_gem_object_pin_pages(obj); 560 if (err) 561 goto out; 562 563 err = i915_gem_object_lock_interruptible(obj, NULL); 564 if (err) 565 goto out_unpin; 566 567 if (read_domains & I915_GEM_DOMAIN_WC) 568 err = i915_gem_object_set_to_wc_domain(obj, write_domain); 569 else if (read_domains & I915_GEM_DOMAIN_GTT) 570 err = i915_gem_object_set_to_gtt_domain(obj, write_domain); 571 else 572 err = i915_gem_object_set_to_cpu_domain(obj, write_domain); 573 574 /* And bump the LRU for this access */ 575 i915_gem_object_bump_inactive_ggtt(obj); 576 577 i915_gem_object_unlock(obj); 578 579 if (write_domain) 580 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); 581 582 out_unpin: 583 i915_gem_object_unpin_pages(obj); 584 out: 585 i915_gem_object_put(obj); 586 return err; 587 } 588 589 /* 590 * Pins the specified object's pages and synchronizes the object with 591 * GPU accesses. Sets needs_clflush to non-zero if the caller should 592 * flush the object from the CPU cache. 593 */ 594 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj, 595 unsigned int *needs_clflush) 596 { 597 int ret; 598 599 *needs_clflush = 0; 600 if (!i915_gem_object_has_struct_page(obj)) 601 return -ENODEV; 602 603 assert_object_held(obj); 604 605 ret = i915_gem_object_wait(obj, 606 I915_WAIT_INTERRUPTIBLE, 607 MAX_SCHEDULE_TIMEOUT); 608 if (ret) 609 return ret; 610 611 ret = i915_gem_object_pin_pages(obj); 612 if (ret) 613 return ret; 614 615 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ || 616 !static_cpu_has(X86_FEATURE_CLFLUSH)) { 617 ret = i915_gem_object_set_to_cpu_domain(obj, false); 618 if (ret) 619 goto err_unpin; 620 else 621 goto out; 622 } 623 624 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 625 626 /* If we're not in the cpu read domain, set ourself into the gtt 627 * read domain and manually flush cachelines (if required). This 628 * optimizes for the case when the gpu will dirty the data 629 * anyway again before the next pread happens. 630 */ 631 if (!obj->cache_dirty && 632 !(obj->read_domains & I915_GEM_DOMAIN_CPU)) 633 *needs_clflush = CLFLUSH_BEFORE; 634 635 out: 636 /* return with the pages pinned */ 637 return 0; 638 639 err_unpin: 640 i915_gem_object_unpin_pages(obj); 641 return ret; 642 } 643 644 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj, 645 unsigned int *needs_clflush) 646 { 647 int ret; 648 649 *needs_clflush = 0; 650 if (!i915_gem_object_has_struct_page(obj)) 651 return -ENODEV; 652 653 assert_object_held(obj); 654 655 ret = i915_gem_object_wait(obj, 656 I915_WAIT_INTERRUPTIBLE | 657 I915_WAIT_ALL, 658 MAX_SCHEDULE_TIMEOUT); 659 if (ret) 660 return ret; 661 662 ret = i915_gem_object_pin_pages(obj); 663 if (ret) 664 return ret; 665 666 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE || 667 !static_cpu_has(X86_FEATURE_CLFLUSH)) { 668 ret = i915_gem_object_set_to_cpu_domain(obj, true); 669 if (ret) 670 goto err_unpin; 671 else 672 goto out; 673 } 674 675 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 676 677 /* If we're not in the cpu write domain, set ourself into the 678 * gtt write domain and manually flush cachelines (as required). 679 * This optimizes for the case when the gpu will use the data 680 * right away and we therefore have to clflush anyway. 681 */ 682 if (!obj->cache_dirty) { 683 *needs_clflush |= CLFLUSH_AFTER; 684 685 /* 686 * Same trick applies to invalidate partially written 687 * cachelines read before writing. 688 */ 689 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU)) 690 *needs_clflush |= CLFLUSH_BEFORE; 691 } 692 693 out: 694 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); 695 obj->mm.dirty = true; 696 /* return with the pages pinned */ 697 return 0; 698 699 err_unpin: 700 i915_gem_object_unpin_pages(obj); 701 return ret; 702 } 703