1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include "display/intel_frontbuffer.h" 8 9 #include "i915_drv.h" 10 #include "i915_gem_clflush.h" 11 #include "i915_gem_gtt.h" 12 #include "i915_gem_ioctls.h" 13 #include "i915_gem_object.h" 14 #include "i915_vma.h" 15 #include "i915_gem_lmem.h" 16 #include "i915_gem_mman.h" 17 18 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj) 19 { 20 /* 21 * We manually flush the CPU domain so that we can override and 22 * force the flush for the display, and perform it asyncrhonously. 23 */ 24 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 25 if (obj->cache_dirty) 26 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE); 27 obj->write_domain = 0; 28 } 29 30 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj) 31 { 32 if (!i915_gem_object_is_framebuffer(obj)) 33 return; 34 35 i915_gem_object_lock(obj); 36 __i915_gem_object_flush_for_display(obj); 37 i915_gem_object_unlock(obj); 38 } 39 40 /** 41 * Moves a single object to the WC read, and possibly write domain. 42 * @obj: object to act on 43 * @write: ask for write access or read only 44 * 45 * This function returns when the move is complete, including waiting on 46 * flushes to occur. 47 */ 48 int 49 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write) 50 { 51 int ret; 52 53 assert_object_held(obj); 54 55 ret = i915_gem_object_wait(obj, 56 I915_WAIT_INTERRUPTIBLE | 57 (write ? I915_WAIT_ALL : 0), 58 MAX_SCHEDULE_TIMEOUT); 59 if (ret) 60 return ret; 61 62 if (obj->write_domain == I915_GEM_DOMAIN_WC) 63 return 0; 64 65 /* Flush and acquire obj->pages so that we are coherent through 66 * direct access in memory with previous cached writes through 67 * shmemfs and that our cache domain tracking remains valid. 68 * For example, if the obj->filp was moved to swap without us 69 * being notified and releasing the pages, we would mistakenly 70 * continue to assume that the obj remained out of the CPU cached 71 * domain. 72 */ 73 ret = i915_gem_object_pin_pages(obj); 74 if (ret) 75 return ret; 76 77 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_WC); 78 79 /* Serialise direct access to this object with the barriers for 80 * coherent writes from the GPU, by effectively invalidating the 81 * WC domain upon first access. 82 */ 83 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0) 84 mb(); 85 86 /* It should now be out of any other write domains, and we can update 87 * the domain values for our changes. 88 */ 89 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0); 90 obj->read_domains |= I915_GEM_DOMAIN_WC; 91 if (write) { 92 obj->read_domains = I915_GEM_DOMAIN_WC; 93 obj->write_domain = I915_GEM_DOMAIN_WC; 94 obj->mm.dirty = true; 95 } 96 97 i915_gem_object_unpin_pages(obj); 98 return 0; 99 } 100 101 /** 102 * Moves a single object to the GTT read, and possibly write domain. 103 * @obj: object to act on 104 * @write: ask for write access or read only 105 * 106 * This function returns when the move is complete, including waiting on 107 * flushes to occur. 108 */ 109 int 110 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 111 { 112 int ret; 113 114 assert_object_held(obj); 115 116 ret = i915_gem_object_wait(obj, 117 I915_WAIT_INTERRUPTIBLE | 118 (write ? I915_WAIT_ALL : 0), 119 MAX_SCHEDULE_TIMEOUT); 120 if (ret) 121 return ret; 122 123 if (obj->write_domain == I915_GEM_DOMAIN_GTT) 124 return 0; 125 126 /* Flush and acquire obj->pages so that we are coherent through 127 * direct access in memory with previous cached writes through 128 * shmemfs and that our cache domain tracking remains valid. 129 * For example, if the obj->filp was moved to swap without us 130 * being notified and releasing the pages, we would mistakenly 131 * continue to assume that the obj remained out of the CPU cached 132 * domain. 133 */ 134 ret = i915_gem_object_pin_pages(obj); 135 if (ret) 136 return ret; 137 138 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT); 139 140 /* Serialise direct access to this object with the barriers for 141 * coherent writes from the GPU, by effectively invalidating the 142 * GTT domain upon first access. 143 */ 144 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0) 145 mb(); 146 147 /* It should now be out of any other write domains, and we can update 148 * the domain values for our changes. 149 */ 150 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 151 obj->read_domains |= I915_GEM_DOMAIN_GTT; 152 if (write) { 153 struct i915_vma *vma; 154 155 obj->read_domains = I915_GEM_DOMAIN_GTT; 156 obj->write_domain = I915_GEM_DOMAIN_GTT; 157 obj->mm.dirty = true; 158 159 spin_lock(&obj->vma.lock); 160 for_each_ggtt_vma(vma, obj) 161 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 162 i915_vma_set_ggtt_write(vma); 163 spin_unlock(&obj->vma.lock); 164 } 165 166 i915_gem_object_unpin_pages(obj); 167 return 0; 168 } 169 170 /** 171 * Changes the cache-level of an object across all VMA. 172 * @obj: object to act on 173 * @cache_level: new cache level to set for the object 174 * 175 * After this function returns, the object will be in the new cache-level 176 * across all GTT and the contents of the backing storage will be coherent, 177 * with respect to the new cache-level. In order to keep the backing storage 178 * coherent for all users, we only allow a single cache level to be set 179 * globally on the object and prevent it from being changed whilst the 180 * hardware is reading from the object. That is if the object is currently 181 * on the scanout it will be set to uncached (or equivalent display 182 * cache coherency) and all non-MOCS GPU access will also be uncached so 183 * that all direct access to the scanout remains coherent. 184 */ 185 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 186 enum i915_cache_level cache_level) 187 { 188 int ret; 189 190 if (obj->cache_level == cache_level) 191 return 0; 192 193 ret = i915_gem_object_wait(obj, 194 I915_WAIT_INTERRUPTIBLE | 195 I915_WAIT_ALL, 196 MAX_SCHEDULE_TIMEOUT); 197 if (ret) 198 return ret; 199 200 ret = i915_gem_object_lock_interruptible(obj); 201 if (ret) 202 return ret; 203 204 /* Always invalidate stale cachelines */ 205 if (obj->cache_level != cache_level) { 206 i915_gem_object_set_cache_coherency(obj, cache_level); 207 obj->cache_dirty = true; 208 } 209 210 i915_gem_object_unlock(obj); 211 212 /* The cache-level will be applied when each vma is rebound. */ 213 return i915_gem_object_unbind(obj, 214 I915_GEM_OBJECT_UNBIND_ACTIVE | 215 I915_GEM_OBJECT_UNBIND_BARRIER); 216 } 217 218 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 219 struct drm_file *file) 220 { 221 struct drm_i915_gem_caching *args = data; 222 struct drm_i915_gem_object *obj; 223 int err = 0; 224 225 rcu_read_lock(); 226 obj = i915_gem_object_lookup_rcu(file, args->handle); 227 if (!obj) { 228 err = -ENOENT; 229 goto out; 230 } 231 232 switch (obj->cache_level) { 233 case I915_CACHE_LLC: 234 case I915_CACHE_L3_LLC: 235 args->caching = I915_CACHING_CACHED; 236 break; 237 238 case I915_CACHE_WT: 239 args->caching = I915_CACHING_DISPLAY; 240 break; 241 242 default: 243 args->caching = I915_CACHING_NONE; 244 break; 245 } 246 out: 247 rcu_read_unlock(); 248 return err; 249 } 250 251 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 252 struct drm_file *file) 253 { 254 struct drm_i915_private *i915 = to_i915(dev); 255 struct drm_i915_gem_caching *args = data; 256 struct drm_i915_gem_object *obj; 257 enum i915_cache_level level; 258 int ret = 0; 259 260 switch (args->caching) { 261 case I915_CACHING_NONE: 262 level = I915_CACHE_NONE; 263 break; 264 case I915_CACHING_CACHED: 265 /* 266 * Due to a HW issue on BXT A stepping, GPU stores via a 267 * snooped mapping may leave stale data in a corresponding CPU 268 * cacheline, whereas normally such cachelines would get 269 * invalidated. 270 */ 271 if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) 272 return -ENODEV; 273 274 level = I915_CACHE_LLC; 275 break; 276 case I915_CACHING_DISPLAY: 277 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE; 278 break; 279 default: 280 return -EINVAL; 281 } 282 283 obj = i915_gem_object_lookup(file, args->handle); 284 if (!obj) 285 return -ENOENT; 286 287 /* 288 * The caching mode of proxy object is handled by its generator, and 289 * not allowed to be changed by userspace. 290 */ 291 if (i915_gem_object_is_proxy(obj)) { 292 ret = -ENXIO; 293 goto out; 294 } 295 296 ret = i915_gem_object_set_cache_level(obj, level); 297 298 out: 299 i915_gem_object_put(obj); 300 return ret; 301 } 302 303 /* 304 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from 305 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined 306 * (for pageflips). We only flush the caches while preparing the buffer for 307 * display, the callers are responsible for frontbuffer flush. 308 */ 309 struct i915_vma * 310 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 311 u32 alignment, 312 const struct i915_ggtt_view *view, 313 unsigned int flags) 314 { 315 struct drm_i915_private *i915 = to_i915(obj->base.dev); 316 struct i915_vma *vma; 317 int ret; 318 319 /* Frame buffer must be in LMEM (no migration yet) */ 320 if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj)) 321 return ERR_PTR(-EINVAL); 322 323 /* 324 * The display engine is not coherent with the LLC cache on gen6. As 325 * a result, we make sure that the pinning that is about to occur is 326 * done with uncached PTEs. This is lowest common denominator for all 327 * chipsets. 328 * 329 * However for gen6+, we could do better by using the GFDT bit instead 330 * of uncaching, which would allow us to flush all the LLC-cached data 331 * with that bit in the PTE to main memory with just one PIPE_CONTROL. 332 */ 333 ret = i915_gem_object_set_cache_level(obj, 334 HAS_WT(i915) ? 335 I915_CACHE_WT : I915_CACHE_NONE); 336 if (ret) 337 return ERR_PTR(ret); 338 339 /* 340 * As the user may map the buffer once pinned in the display plane 341 * (e.g. libkms for the bootup splash), we have to ensure that we 342 * always use map_and_fenceable for all scanout buffers. However, 343 * it may simply be too big to fit into mappable, in which case 344 * put it anyway and hope that userspace can cope (but always first 345 * try to preserve the existing ABI). 346 */ 347 vma = ERR_PTR(-ENOSPC); 348 if ((flags & PIN_MAPPABLE) == 0 && 349 (!view || view->type == I915_GGTT_VIEW_NORMAL)) 350 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, 351 flags | 352 PIN_MAPPABLE | 353 PIN_NONBLOCK); 354 if (IS_ERR(vma)) 355 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags); 356 if (IS_ERR(vma)) 357 return vma; 358 359 vma->display_alignment = max_t(u64, vma->display_alignment, alignment); 360 361 i915_gem_object_flush_if_display(obj); 362 363 return vma; 364 } 365 366 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) 367 { 368 struct drm_i915_private *i915 = to_i915(obj->base.dev); 369 struct i915_vma *vma; 370 371 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 372 if (!atomic_read(&obj->bind_count)) 373 return; 374 375 mutex_lock(&i915->ggtt.vm.mutex); 376 spin_lock(&obj->vma.lock); 377 for_each_ggtt_vma(vma, obj) { 378 if (!drm_mm_node_allocated(&vma->node)) 379 continue; 380 381 GEM_BUG_ON(vma->vm != &i915->ggtt.vm); 382 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 383 } 384 spin_unlock(&obj->vma.lock); 385 mutex_unlock(&i915->ggtt.vm.mutex); 386 387 if (i915_gem_object_is_shrinkable(obj)) { 388 unsigned long flags; 389 390 spin_lock_irqsave(&i915->mm.obj_lock, flags); 391 392 if (obj->mm.madv == I915_MADV_WILLNEED && 393 !atomic_read(&obj->mm.shrink_pin)) 394 list_move_tail(&obj->mm.link, &i915->mm.shrink_list); 395 396 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 397 } 398 } 399 400 void 401 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) 402 { 403 struct drm_i915_gem_object *obj = vma->obj; 404 405 assert_object_held(obj); 406 407 /* Bump the LRU to try and avoid premature eviction whilst flipping */ 408 i915_gem_object_bump_inactive_ggtt(obj); 409 410 i915_vma_unpin(vma); 411 } 412 413 /** 414 * Moves a single object to the CPU read, and possibly write domain. 415 * @obj: object to act on 416 * @write: requesting write or read-only access 417 * 418 * This function returns when the move is complete, including waiting on 419 * flushes to occur. 420 */ 421 int 422 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 423 { 424 int ret; 425 426 assert_object_held(obj); 427 428 ret = i915_gem_object_wait(obj, 429 I915_WAIT_INTERRUPTIBLE | 430 (write ? I915_WAIT_ALL : 0), 431 MAX_SCHEDULE_TIMEOUT); 432 if (ret) 433 return ret; 434 435 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 436 437 /* Flush the CPU cache if it's still invalid. */ 438 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 439 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); 440 obj->read_domains |= I915_GEM_DOMAIN_CPU; 441 } 442 443 /* It should now be out of any other write domains, and we can update 444 * the domain values for our changes. 445 */ 446 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU); 447 448 /* If we're writing through the CPU, then the GPU read domains will 449 * need to be invalidated at next use. 450 */ 451 if (write) 452 __start_cpu_write(obj); 453 454 return 0; 455 } 456 457 /** 458 * Called when user space prepares to use an object with the CPU, either 459 * through the mmap ioctl's mapping or a GTT mapping. 460 * @dev: drm device 461 * @data: ioctl data blob 462 * @file: drm file 463 */ 464 int 465 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 466 struct drm_file *file) 467 { 468 struct drm_i915_gem_set_domain *args = data; 469 struct drm_i915_gem_object *obj; 470 u32 read_domains = args->read_domains; 471 u32 write_domain = args->write_domain; 472 int err; 473 474 /* Only handle setting domains to types used by the CPU. */ 475 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS) 476 return -EINVAL; 477 478 /* 479 * Having something in the write domain implies it's in the read 480 * domain, and only that read domain. Enforce that in the request. 481 */ 482 if (write_domain && read_domains != write_domain) 483 return -EINVAL; 484 485 if (!read_domains) 486 return 0; 487 488 obj = i915_gem_object_lookup(file, args->handle); 489 if (!obj) 490 return -ENOENT; 491 492 /* 493 * Already in the desired write domain? Nothing for us to do! 494 * 495 * We apply a little bit of cunning here to catch a broader set of 496 * no-ops. If obj->write_domain is set, we must be in the same 497 * obj->read_domains, and only that domain. Therefore, if that 498 * obj->write_domain matches the request read_domains, we are 499 * already in the same read/write domain and can skip the operation, 500 * without having to further check the requested write_domain. 501 */ 502 if (READ_ONCE(obj->write_domain) == read_domains) { 503 err = 0; 504 goto out; 505 } 506 507 /* 508 * Try to flush the object off the GPU without holding the lock. 509 * We will repeat the flush holding the lock in the normal manner 510 * to catch cases where we are gazumped. 511 */ 512 err = i915_gem_object_wait(obj, 513 I915_WAIT_INTERRUPTIBLE | 514 I915_WAIT_PRIORITY | 515 (write_domain ? I915_WAIT_ALL : 0), 516 MAX_SCHEDULE_TIMEOUT); 517 if (err) 518 goto out; 519 520 /* 521 * Proxy objects do not control access to the backing storage, ergo 522 * they cannot be used as a means to manipulate the cache domain 523 * tracking for that backing storage. The proxy object is always 524 * considered to be outside of any cache domain. 525 */ 526 if (i915_gem_object_is_proxy(obj)) { 527 err = -ENXIO; 528 goto out; 529 } 530 531 /* 532 * Flush and acquire obj->pages so that we are coherent through 533 * direct access in memory with previous cached writes through 534 * shmemfs and that our cache domain tracking remains valid. 535 * For example, if the obj->filp was moved to swap without us 536 * being notified and releasing the pages, we would mistakenly 537 * continue to assume that the obj remained out of the CPU cached 538 * domain. 539 */ 540 err = i915_gem_object_pin_pages(obj); 541 if (err) 542 goto out; 543 544 err = i915_gem_object_lock_interruptible(obj); 545 if (err) 546 goto out_unpin; 547 548 if (read_domains & I915_GEM_DOMAIN_WC) 549 err = i915_gem_object_set_to_wc_domain(obj, write_domain); 550 else if (read_domains & I915_GEM_DOMAIN_GTT) 551 err = i915_gem_object_set_to_gtt_domain(obj, write_domain); 552 else 553 err = i915_gem_object_set_to_cpu_domain(obj, write_domain); 554 555 /* And bump the LRU for this access */ 556 i915_gem_object_bump_inactive_ggtt(obj); 557 558 i915_gem_object_unlock(obj); 559 560 if (write_domain) 561 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); 562 563 out_unpin: 564 i915_gem_object_unpin_pages(obj); 565 out: 566 i915_gem_object_put(obj); 567 return err; 568 } 569 570 /* 571 * Pins the specified object's pages and synchronizes the object with 572 * GPU accesses. Sets needs_clflush to non-zero if the caller should 573 * flush the object from the CPU cache. 574 */ 575 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj, 576 unsigned int *needs_clflush) 577 { 578 int ret; 579 580 *needs_clflush = 0; 581 if (!i915_gem_object_has_struct_page(obj)) 582 return -ENODEV; 583 584 ret = i915_gem_object_lock_interruptible(obj); 585 if (ret) 586 return ret; 587 588 ret = i915_gem_object_wait(obj, 589 I915_WAIT_INTERRUPTIBLE, 590 MAX_SCHEDULE_TIMEOUT); 591 if (ret) 592 goto err_unlock; 593 594 ret = i915_gem_object_pin_pages(obj); 595 if (ret) 596 goto err_unlock; 597 598 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ || 599 !static_cpu_has(X86_FEATURE_CLFLUSH)) { 600 ret = i915_gem_object_set_to_cpu_domain(obj, false); 601 if (ret) 602 goto err_unpin; 603 else 604 goto out; 605 } 606 607 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 608 609 /* If we're not in the cpu read domain, set ourself into the gtt 610 * read domain and manually flush cachelines (if required). This 611 * optimizes for the case when the gpu will dirty the data 612 * anyway again before the next pread happens. 613 */ 614 if (!obj->cache_dirty && 615 !(obj->read_domains & I915_GEM_DOMAIN_CPU)) 616 *needs_clflush = CLFLUSH_BEFORE; 617 618 out: 619 /* return with the pages pinned */ 620 return 0; 621 622 err_unpin: 623 i915_gem_object_unpin_pages(obj); 624 err_unlock: 625 i915_gem_object_unlock(obj); 626 return ret; 627 } 628 629 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj, 630 unsigned int *needs_clflush) 631 { 632 int ret; 633 634 *needs_clflush = 0; 635 if (!i915_gem_object_has_struct_page(obj)) 636 return -ENODEV; 637 638 ret = i915_gem_object_lock_interruptible(obj); 639 if (ret) 640 return ret; 641 642 ret = i915_gem_object_wait(obj, 643 I915_WAIT_INTERRUPTIBLE | 644 I915_WAIT_ALL, 645 MAX_SCHEDULE_TIMEOUT); 646 if (ret) 647 goto err_unlock; 648 649 ret = i915_gem_object_pin_pages(obj); 650 if (ret) 651 goto err_unlock; 652 653 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE || 654 !static_cpu_has(X86_FEATURE_CLFLUSH)) { 655 ret = i915_gem_object_set_to_cpu_domain(obj, true); 656 if (ret) 657 goto err_unpin; 658 else 659 goto out; 660 } 661 662 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 663 664 /* If we're not in the cpu write domain, set ourself into the 665 * gtt write domain and manually flush cachelines (as required). 666 * This optimizes for the case when the gpu will use the data 667 * right away and we therefore have to clflush anyway. 668 */ 669 if (!obj->cache_dirty) { 670 *needs_clflush |= CLFLUSH_AFTER; 671 672 /* 673 * Same trick applies to invalidate partially written 674 * cachelines read before writing. 675 */ 676 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU)) 677 *needs_clflush |= CLFLUSH_BEFORE; 678 } 679 680 out: 681 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); 682 obj->mm.dirty = true; 683 /* return with the pages pinned */ 684 return 0; 685 686 err_unpin: 687 i915_gem_object_unpin_pages(obj); 688 err_unlock: 689 i915_gem_object_unlock(obj); 690 return ret; 691 } 692