1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include <drm/drmP.h> 29 #include <drm/drm_vma_manager.h> 30 #include <drm/i915_drm.h> 31 #include "i915_drv.h" 32 #include "i915_gem_clflush.h" 33 #include "i915_vgpu.h" 34 #include "i915_trace.h" 35 #include "intel_drv.h" 36 #include "intel_frontbuffer.h" 37 #include "intel_mocs.h" 38 #include "intel_workarounds.h" 39 #include "i915_gemfs.h" 40 #include <linux/dma-fence-array.h> 41 #include <linux/kthread.h> 42 #include <linux/reservation.h> 43 #include <linux/shmem_fs.h> 44 #include <linux/slab.h> 45 #include <linux/stop_machine.h> 46 #include <linux/swap.h> 47 #include <linux/pci.h> 48 #include <linux/dma-buf.h> 49 50 static void i915_gem_flush_free_objects(struct drm_i915_private *i915); 51 52 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 53 { 54 if (obj->cache_dirty) 55 return false; 56 57 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)) 58 return true; 59 60 return obj->pin_global; /* currently in use by HW, keep flushed */ 61 } 62 63 static int 64 insert_mappable_node(struct i915_ggtt *ggtt, 65 struct drm_mm_node *node, u32 size) 66 { 67 memset(node, 0, sizeof(*node)); 68 return drm_mm_insert_node_in_range(&ggtt->vm.mm, node, 69 size, 0, I915_COLOR_UNEVICTABLE, 70 0, ggtt->mappable_end, 71 DRM_MM_INSERT_LOW); 72 } 73 74 static void 75 remove_mappable_node(struct drm_mm_node *node) 76 { 77 drm_mm_remove_node(node); 78 } 79 80 /* some bookkeeping */ 81 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 82 u64 size) 83 { 84 spin_lock(&dev_priv->mm.object_stat_lock); 85 dev_priv->mm.object_count++; 86 dev_priv->mm.object_memory += size; 87 spin_unlock(&dev_priv->mm.object_stat_lock); 88 } 89 90 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 91 u64 size) 92 { 93 spin_lock(&dev_priv->mm.object_stat_lock); 94 dev_priv->mm.object_count--; 95 dev_priv->mm.object_memory -= size; 96 spin_unlock(&dev_priv->mm.object_stat_lock); 97 } 98 99 static int 100 i915_gem_wait_for_error(struct i915_gpu_error *error) 101 { 102 int ret; 103 104 might_sleep(); 105 106 /* 107 * Only wait 10 seconds for the gpu reset to complete to avoid hanging 108 * userspace. If it takes that long something really bad is going on and 109 * we should simply try to bail out and fail as gracefully as possible. 110 */ 111 ret = wait_event_interruptible_timeout(error->reset_queue, 112 !i915_reset_backoff(error), 113 I915_RESET_TIMEOUT); 114 if (ret == 0) { 115 DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); 116 return -EIO; 117 } else if (ret < 0) { 118 return ret; 119 } else { 120 return 0; 121 } 122 } 123 124 int i915_mutex_lock_interruptible(struct drm_device *dev) 125 { 126 struct drm_i915_private *dev_priv = to_i915(dev); 127 int ret; 128 129 ret = i915_gem_wait_for_error(&dev_priv->gpu_error); 130 if (ret) 131 return ret; 132 133 ret = mutex_lock_interruptible(&dev->struct_mutex); 134 if (ret) 135 return ret; 136 137 return 0; 138 } 139 140 static u32 __i915_gem_park(struct drm_i915_private *i915) 141 { 142 GEM_TRACE("\n"); 143 144 lockdep_assert_held(&i915->drm.struct_mutex); 145 GEM_BUG_ON(i915->gt.active_requests); 146 GEM_BUG_ON(!list_empty(&i915->gt.active_rings)); 147 148 if (!i915->gt.awake) 149 return I915_EPOCH_INVALID; 150 151 GEM_BUG_ON(i915->gt.epoch == I915_EPOCH_INVALID); 152 153 /* 154 * Be paranoid and flush a concurrent interrupt to make sure 155 * we don't reactivate any irq tasklets after parking. 156 * 157 * FIXME: Note that even though we have waited for execlists to be idle, 158 * there may still be an in-flight interrupt even though the CSB 159 * is now empty. synchronize_irq() makes sure that a residual interrupt 160 * is completed before we continue, but it doesn't prevent the HW from 161 * raising a spurious interrupt later. To complete the shield we should 162 * coordinate disabling the CS irq with flushing the interrupts. 163 */ 164 synchronize_irq(i915->drm.irq); 165 166 intel_engines_park(i915); 167 i915_timelines_park(i915); 168 169 i915_pmu_gt_parked(i915); 170 i915_vma_parked(i915); 171 172 i915->gt.awake = false; 173 174 if (INTEL_GEN(i915) >= 6) 175 gen6_rps_idle(i915); 176 177 intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ); 178 179 intel_runtime_pm_put(i915); 180 181 return i915->gt.epoch; 182 } 183 184 void i915_gem_park(struct drm_i915_private *i915) 185 { 186 GEM_TRACE("\n"); 187 188 lockdep_assert_held(&i915->drm.struct_mutex); 189 GEM_BUG_ON(i915->gt.active_requests); 190 191 if (!i915->gt.awake) 192 return; 193 194 /* Defer the actual call to __i915_gem_park() to prevent ping-pongs */ 195 mod_delayed_work(i915->wq, &i915->gt.idle_work, msecs_to_jiffies(100)); 196 } 197 198 void i915_gem_unpark(struct drm_i915_private *i915) 199 { 200 GEM_TRACE("\n"); 201 202 lockdep_assert_held(&i915->drm.struct_mutex); 203 GEM_BUG_ON(!i915->gt.active_requests); 204 205 if (i915->gt.awake) 206 return; 207 208 intel_runtime_pm_get_noresume(i915); 209 210 /* 211 * It seems that the DMC likes to transition between the DC states a lot 212 * when there are no connected displays (no active power domains) during 213 * command submission. 214 * 215 * This activity has negative impact on the performance of the chip with 216 * huge latencies observed in the interrupt handler and elsewhere. 217 * 218 * Work around it by grabbing a GT IRQ power domain whilst there is any 219 * GT activity, preventing any DC state transitions. 220 */ 221 intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); 222 223 i915->gt.awake = true; 224 if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */ 225 i915->gt.epoch = 1; 226 227 intel_enable_gt_powersave(i915); 228 i915_update_gfx_val(i915); 229 if (INTEL_GEN(i915) >= 6) 230 gen6_rps_busy(i915); 231 i915_pmu_gt_unparked(i915); 232 233 intel_engines_unpark(i915); 234 235 i915_queue_hangcheck(i915); 236 237 queue_delayed_work(i915->wq, 238 &i915->gt.retire_work, 239 round_jiffies_up_relative(HZ)); 240 } 241 242 int 243 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 244 struct drm_file *file) 245 { 246 struct drm_i915_private *dev_priv = to_i915(dev); 247 struct i915_ggtt *ggtt = &dev_priv->ggtt; 248 struct drm_i915_gem_get_aperture *args = data; 249 struct i915_vma *vma; 250 u64 pinned; 251 252 pinned = ggtt->vm.reserved; 253 mutex_lock(&dev->struct_mutex); 254 list_for_each_entry(vma, &ggtt->vm.active_list, vm_link) 255 if (i915_vma_is_pinned(vma)) 256 pinned += vma->node.size; 257 list_for_each_entry(vma, &ggtt->vm.inactive_list, vm_link) 258 if (i915_vma_is_pinned(vma)) 259 pinned += vma->node.size; 260 mutex_unlock(&dev->struct_mutex); 261 262 args->aper_size = ggtt->vm.total; 263 args->aper_available_size = args->aper_size - pinned; 264 265 return 0; 266 } 267 268 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 269 { 270 struct address_space *mapping = obj->base.filp->f_mapping; 271 drm_dma_handle_t *phys; 272 struct sg_table *st; 273 struct scatterlist *sg; 274 char *vaddr; 275 int i; 276 int err; 277 278 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 279 return -EINVAL; 280 281 /* Always aligning to the object size, allows a single allocation 282 * to handle all possible callers, and given typical object sizes, 283 * the alignment of the buddy allocation will naturally match. 284 */ 285 phys = drm_pci_alloc(obj->base.dev, 286 roundup_pow_of_two(obj->base.size), 287 roundup_pow_of_two(obj->base.size)); 288 if (!phys) 289 return -ENOMEM; 290 291 vaddr = phys->vaddr; 292 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 293 struct page *page; 294 char *src; 295 296 page = shmem_read_mapping_page(mapping, i); 297 if (IS_ERR(page)) { 298 err = PTR_ERR(page); 299 goto err_phys; 300 } 301 302 src = kmap_atomic(page); 303 memcpy(vaddr, src, PAGE_SIZE); 304 drm_clflush_virt_range(vaddr, PAGE_SIZE); 305 kunmap_atomic(src); 306 307 put_page(page); 308 vaddr += PAGE_SIZE; 309 } 310 311 i915_gem_chipset_flush(to_i915(obj->base.dev)); 312 313 st = kmalloc(sizeof(*st), GFP_KERNEL); 314 if (!st) { 315 err = -ENOMEM; 316 goto err_phys; 317 } 318 319 if (sg_alloc_table(st, 1, GFP_KERNEL)) { 320 kfree(st); 321 err = -ENOMEM; 322 goto err_phys; 323 } 324 325 sg = st->sgl; 326 sg->offset = 0; 327 sg->length = obj->base.size; 328 329 sg_dma_address(sg) = phys->busaddr; 330 sg_dma_len(sg) = obj->base.size; 331 332 obj->phys_handle = phys; 333 334 __i915_gem_object_set_pages(obj, st, sg->length); 335 336 return 0; 337 338 err_phys: 339 drm_pci_free(obj->base.dev, phys); 340 341 return err; 342 } 343 344 static void __start_cpu_write(struct drm_i915_gem_object *obj) 345 { 346 obj->read_domains = I915_GEM_DOMAIN_CPU; 347 obj->write_domain = I915_GEM_DOMAIN_CPU; 348 if (cpu_write_needs_clflush(obj)) 349 obj->cache_dirty = true; 350 } 351 352 static void 353 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 354 struct sg_table *pages, 355 bool needs_clflush) 356 { 357 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 358 359 if (obj->mm.madv == I915_MADV_DONTNEED) 360 obj->mm.dirty = false; 361 362 if (needs_clflush && 363 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 && 364 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 365 drm_clflush_sg(pages); 366 367 __start_cpu_write(obj); 368 } 369 370 static void 371 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 372 struct sg_table *pages) 373 { 374 __i915_gem_object_release_shmem(obj, pages, false); 375 376 if (obj->mm.dirty) { 377 struct address_space *mapping = obj->base.filp->f_mapping; 378 char *vaddr = obj->phys_handle->vaddr; 379 int i; 380 381 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 382 struct page *page; 383 char *dst; 384 385 page = shmem_read_mapping_page(mapping, i); 386 if (IS_ERR(page)) 387 continue; 388 389 dst = kmap_atomic(page); 390 drm_clflush_virt_range(vaddr, PAGE_SIZE); 391 memcpy(dst, vaddr, PAGE_SIZE); 392 kunmap_atomic(dst); 393 394 set_page_dirty(page); 395 if (obj->mm.madv == I915_MADV_WILLNEED) 396 mark_page_accessed(page); 397 put_page(page); 398 vaddr += PAGE_SIZE; 399 } 400 obj->mm.dirty = false; 401 } 402 403 sg_free_table(pages); 404 kfree(pages); 405 406 drm_pci_free(obj->base.dev, obj->phys_handle); 407 } 408 409 static void 410 i915_gem_object_release_phys(struct drm_i915_gem_object *obj) 411 { 412 i915_gem_object_unpin_pages(obj); 413 } 414 415 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { 416 .get_pages = i915_gem_object_get_pages_phys, 417 .put_pages = i915_gem_object_put_pages_phys, 418 .release = i915_gem_object_release_phys, 419 }; 420 421 static const struct drm_i915_gem_object_ops i915_gem_object_ops; 422 423 int i915_gem_object_unbind(struct drm_i915_gem_object *obj) 424 { 425 struct i915_vma *vma; 426 LIST_HEAD(still_in_list); 427 int ret; 428 429 lockdep_assert_held(&obj->base.dev->struct_mutex); 430 431 /* Closed vma are removed from the obj->vma_list - but they may 432 * still have an active binding on the object. To remove those we 433 * must wait for all rendering to complete to the object (as unbinding 434 * must anyway), and retire the requests. 435 */ 436 ret = i915_gem_object_set_to_cpu_domain(obj, false); 437 if (ret) 438 return ret; 439 440 while ((vma = list_first_entry_or_null(&obj->vma_list, 441 struct i915_vma, 442 obj_link))) { 443 list_move_tail(&vma->obj_link, &still_in_list); 444 ret = i915_vma_unbind(vma); 445 if (ret) 446 break; 447 } 448 list_splice(&still_in_list, &obj->vma_list); 449 450 return ret; 451 } 452 453 static long 454 i915_gem_object_wait_fence(struct dma_fence *fence, 455 unsigned int flags, 456 long timeout, 457 struct intel_rps_client *rps_client) 458 { 459 struct i915_request *rq; 460 461 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); 462 463 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 464 return timeout; 465 466 if (!dma_fence_is_i915(fence)) 467 return dma_fence_wait_timeout(fence, 468 flags & I915_WAIT_INTERRUPTIBLE, 469 timeout); 470 471 rq = to_request(fence); 472 if (i915_request_completed(rq)) 473 goto out; 474 475 /* 476 * This client is about to stall waiting for the GPU. In many cases 477 * this is undesirable and limits the throughput of the system, as 478 * many clients cannot continue processing user input/output whilst 479 * blocked. RPS autotuning may take tens of milliseconds to respond 480 * to the GPU load and thus incurs additional latency for the client. 481 * We can circumvent that by promoting the GPU frequency to maximum 482 * before we wait. This makes the GPU throttle up much more quickly 483 * (good for benchmarks and user experience, e.g. window animations), 484 * but at a cost of spending more power processing the workload 485 * (bad for battery). Not all clients even want their results 486 * immediately and for them we should just let the GPU select its own 487 * frequency to maximise efficiency. To prevent a single client from 488 * forcing the clocks too high for the whole system, we only allow 489 * each client to waitboost once in a busy period. 490 */ 491 if (rps_client && !i915_request_started(rq)) { 492 if (INTEL_GEN(rq->i915) >= 6) 493 gen6_rps_boost(rq, rps_client); 494 } 495 496 timeout = i915_request_wait(rq, flags, timeout); 497 498 out: 499 if (flags & I915_WAIT_LOCKED && i915_request_completed(rq)) 500 i915_request_retire_upto(rq); 501 502 return timeout; 503 } 504 505 static long 506 i915_gem_object_wait_reservation(struct reservation_object *resv, 507 unsigned int flags, 508 long timeout, 509 struct intel_rps_client *rps_client) 510 { 511 unsigned int seq = __read_seqcount_begin(&resv->seq); 512 struct dma_fence *excl; 513 bool prune_fences = false; 514 515 if (flags & I915_WAIT_ALL) { 516 struct dma_fence **shared; 517 unsigned int count, i; 518 int ret; 519 520 ret = reservation_object_get_fences_rcu(resv, 521 &excl, &count, &shared); 522 if (ret) 523 return ret; 524 525 for (i = 0; i < count; i++) { 526 timeout = i915_gem_object_wait_fence(shared[i], 527 flags, timeout, 528 rps_client); 529 if (timeout < 0) 530 break; 531 532 dma_fence_put(shared[i]); 533 } 534 535 for (; i < count; i++) 536 dma_fence_put(shared[i]); 537 kfree(shared); 538 539 /* 540 * If both shared fences and an exclusive fence exist, 541 * then by construction the shared fences must be later 542 * than the exclusive fence. If we successfully wait for 543 * all the shared fences, we know that the exclusive fence 544 * must all be signaled. If all the shared fences are 545 * signaled, we can prune the array and recover the 546 * floating references on the fences/requests. 547 */ 548 prune_fences = count && timeout >= 0; 549 } else { 550 excl = reservation_object_get_excl_rcu(resv); 551 } 552 553 if (excl && timeout >= 0) 554 timeout = i915_gem_object_wait_fence(excl, flags, timeout, 555 rps_client); 556 557 dma_fence_put(excl); 558 559 /* 560 * Opportunistically prune the fences iff we know they have *all* been 561 * signaled and that the reservation object has not been changed (i.e. 562 * no new fences have been added). 563 */ 564 if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) { 565 if (reservation_object_trylock(resv)) { 566 if (!__read_seqcount_retry(&resv->seq, seq)) 567 reservation_object_add_excl_fence(resv, NULL); 568 reservation_object_unlock(resv); 569 } 570 } 571 572 return timeout; 573 } 574 575 static void __fence_set_priority(struct dma_fence *fence, 576 const struct i915_sched_attr *attr) 577 { 578 struct i915_request *rq; 579 struct intel_engine_cs *engine; 580 581 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) 582 return; 583 584 rq = to_request(fence); 585 engine = rq->engine; 586 587 local_bh_disable(); 588 rcu_read_lock(); /* RCU serialisation for set-wedged protection */ 589 if (engine->schedule) 590 engine->schedule(rq, attr); 591 rcu_read_unlock(); 592 local_bh_enable(); /* kick the tasklets if queues were reprioritised */ 593 } 594 595 static void fence_set_priority(struct dma_fence *fence, 596 const struct i915_sched_attr *attr) 597 { 598 /* Recurse once into a fence-array */ 599 if (dma_fence_is_array(fence)) { 600 struct dma_fence_array *array = to_dma_fence_array(fence); 601 int i; 602 603 for (i = 0; i < array->num_fences; i++) 604 __fence_set_priority(array->fences[i], attr); 605 } else { 606 __fence_set_priority(fence, attr); 607 } 608 } 609 610 int 611 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 612 unsigned int flags, 613 const struct i915_sched_attr *attr) 614 { 615 struct dma_fence *excl; 616 617 if (flags & I915_WAIT_ALL) { 618 struct dma_fence **shared; 619 unsigned int count, i; 620 int ret; 621 622 ret = reservation_object_get_fences_rcu(obj->resv, 623 &excl, &count, &shared); 624 if (ret) 625 return ret; 626 627 for (i = 0; i < count; i++) { 628 fence_set_priority(shared[i], attr); 629 dma_fence_put(shared[i]); 630 } 631 632 kfree(shared); 633 } else { 634 excl = reservation_object_get_excl_rcu(obj->resv); 635 } 636 637 if (excl) { 638 fence_set_priority(excl, attr); 639 dma_fence_put(excl); 640 } 641 return 0; 642 } 643 644 /** 645 * Waits for rendering to the object to be completed 646 * @obj: i915 gem object 647 * @flags: how to wait (under a lock, for all rendering or just for writes etc) 648 * @timeout: how long to wait 649 * @rps_client: client (user process) to charge for any waitboosting 650 */ 651 int 652 i915_gem_object_wait(struct drm_i915_gem_object *obj, 653 unsigned int flags, 654 long timeout, 655 struct intel_rps_client *rps_client) 656 { 657 might_sleep(); 658 #if IS_ENABLED(CONFIG_LOCKDEP) 659 GEM_BUG_ON(debug_locks && 660 !!lockdep_is_held(&obj->base.dev->struct_mutex) != 661 !!(flags & I915_WAIT_LOCKED)); 662 #endif 663 GEM_BUG_ON(timeout < 0); 664 665 timeout = i915_gem_object_wait_reservation(obj->resv, 666 flags, timeout, 667 rps_client); 668 return timeout < 0 ? timeout : 0; 669 } 670 671 static struct intel_rps_client *to_rps_client(struct drm_file *file) 672 { 673 struct drm_i915_file_private *fpriv = file->driver_priv; 674 675 return &fpriv->rps_client; 676 } 677 678 static int 679 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, 680 struct drm_i915_gem_pwrite *args, 681 struct drm_file *file) 682 { 683 void *vaddr = obj->phys_handle->vaddr + args->offset; 684 char __user *user_data = u64_to_user_ptr(args->data_ptr); 685 686 /* We manually control the domain here and pretend that it 687 * remains coherent i.e. in the GTT domain, like shmem_pwrite. 688 */ 689 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 690 if (copy_from_user(vaddr, user_data, args->size)) 691 return -EFAULT; 692 693 drm_clflush_virt_range(vaddr, args->size); 694 i915_gem_chipset_flush(to_i915(obj->base.dev)); 695 696 intel_fb_obj_flush(obj, ORIGIN_CPU); 697 return 0; 698 } 699 700 void *i915_gem_object_alloc(struct drm_i915_private *dev_priv) 701 { 702 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL); 703 } 704 705 void i915_gem_object_free(struct drm_i915_gem_object *obj) 706 { 707 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 708 kmem_cache_free(dev_priv->objects, obj); 709 } 710 711 static int 712 i915_gem_create(struct drm_file *file, 713 struct drm_i915_private *dev_priv, 714 uint64_t size, 715 uint32_t *handle_p) 716 { 717 struct drm_i915_gem_object *obj; 718 int ret; 719 u32 handle; 720 721 size = roundup(size, PAGE_SIZE); 722 if (size == 0) 723 return -EINVAL; 724 725 /* Allocate the new object */ 726 obj = i915_gem_object_create(dev_priv, size); 727 if (IS_ERR(obj)) 728 return PTR_ERR(obj); 729 730 ret = drm_gem_handle_create(file, &obj->base, &handle); 731 /* drop reference from allocate - handle holds it now */ 732 i915_gem_object_put(obj); 733 if (ret) 734 return ret; 735 736 *handle_p = handle; 737 return 0; 738 } 739 740 int 741 i915_gem_dumb_create(struct drm_file *file, 742 struct drm_device *dev, 743 struct drm_mode_create_dumb *args) 744 { 745 /* have to work out size/pitch and return them */ 746 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64); 747 args->size = args->pitch * args->height; 748 return i915_gem_create(file, to_i915(dev), 749 args->size, &args->handle); 750 } 751 752 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj) 753 { 754 return !(obj->cache_level == I915_CACHE_NONE || 755 obj->cache_level == I915_CACHE_WT); 756 } 757 758 /** 759 * Creates a new mm object and returns a handle to it. 760 * @dev: drm device pointer 761 * @data: ioctl data blob 762 * @file: drm file pointer 763 */ 764 int 765 i915_gem_create_ioctl(struct drm_device *dev, void *data, 766 struct drm_file *file) 767 { 768 struct drm_i915_private *dev_priv = to_i915(dev); 769 struct drm_i915_gem_create *args = data; 770 771 i915_gem_flush_free_objects(dev_priv); 772 773 return i915_gem_create(file, dev_priv, 774 args->size, &args->handle); 775 } 776 777 static inline enum fb_op_origin 778 fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain) 779 { 780 return (domain == I915_GEM_DOMAIN_GTT ? 781 obj->frontbuffer_ggtt_origin : ORIGIN_CPU); 782 } 783 784 void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv) 785 { 786 /* 787 * No actual flushing is required for the GTT write domain for reads 788 * from the GTT domain. Writes to it "immediately" go to main memory 789 * as far as we know, so there's no chipset flush. It also doesn't 790 * land in the GPU render cache. 791 * 792 * However, we do have to enforce the order so that all writes through 793 * the GTT land before any writes to the device, such as updates to 794 * the GATT itself. 795 * 796 * We also have to wait a bit for the writes to land from the GTT. 797 * An uncached read (i.e. mmio) seems to be ideal for the round-trip 798 * timing. This issue has only been observed when switching quickly 799 * between GTT writes and CPU reads from inside the kernel on recent hw, 800 * and it appears to only affect discrete GTT blocks (i.e. on LLC 801 * system agents we cannot reproduce this behaviour, until Cannonlake 802 * that was!). 803 */ 804 805 wmb(); 806 807 intel_runtime_pm_get(dev_priv); 808 spin_lock_irq(&dev_priv->uncore.lock); 809 810 POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE)); 811 812 spin_unlock_irq(&dev_priv->uncore.lock); 813 intel_runtime_pm_put(dev_priv); 814 } 815 816 static void 817 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains) 818 { 819 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 820 struct i915_vma *vma; 821 822 if (!(obj->write_domain & flush_domains)) 823 return; 824 825 switch (obj->write_domain) { 826 case I915_GEM_DOMAIN_GTT: 827 i915_gem_flush_ggtt_writes(dev_priv); 828 829 intel_fb_obj_flush(obj, 830 fb_write_origin(obj, I915_GEM_DOMAIN_GTT)); 831 832 for_each_ggtt_vma(vma, obj) { 833 if (vma->iomap) 834 continue; 835 836 i915_vma_unset_ggtt_write(vma); 837 } 838 break; 839 840 case I915_GEM_DOMAIN_CPU: 841 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); 842 break; 843 844 case I915_GEM_DOMAIN_RENDER: 845 if (gpu_write_needs_clflush(obj)) 846 obj->cache_dirty = true; 847 break; 848 } 849 850 obj->write_domain = 0; 851 } 852 853 static inline int 854 __copy_to_user_swizzled(char __user *cpu_vaddr, 855 const char *gpu_vaddr, int gpu_offset, 856 int length) 857 { 858 int ret, cpu_offset = 0; 859 860 while (length > 0) { 861 int cacheline_end = ALIGN(gpu_offset + 1, 64); 862 int this_length = min(cacheline_end - gpu_offset, length); 863 int swizzled_gpu_offset = gpu_offset ^ 64; 864 865 ret = __copy_to_user(cpu_vaddr + cpu_offset, 866 gpu_vaddr + swizzled_gpu_offset, 867 this_length); 868 if (ret) 869 return ret + length; 870 871 cpu_offset += this_length; 872 gpu_offset += this_length; 873 length -= this_length; 874 } 875 876 return 0; 877 } 878 879 static inline int 880 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 881 const char __user *cpu_vaddr, 882 int length) 883 { 884 int ret, cpu_offset = 0; 885 886 while (length > 0) { 887 int cacheline_end = ALIGN(gpu_offset + 1, 64); 888 int this_length = min(cacheline_end - gpu_offset, length); 889 int swizzled_gpu_offset = gpu_offset ^ 64; 890 891 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 892 cpu_vaddr + cpu_offset, 893 this_length); 894 if (ret) 895 return ret + length; 896 897 cpu_offset += this_length; 898 gpu_offset += this_length; 899 length -= this_length; 900 } 901 902 return 0; 903 } 904 905 /* 906 * Pins the specified object's pages and synchronizes the object with 907 * GPU accesses. Sets needs_clflush to non-zero if the caller should 908 * flush the object from the CPU cache. 909 */ 910 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, 911 unsigned int *needs_clflush) 912 { 913 int ret; 914 915 lockdep_assert_held(&obj->base.dev->struct_mutex); 916 917 *needs_clflush = 0; 918 if (!i915_gem_object_has_struct_page(obj)) 919 return -ENODEV; 920 921 ret = i915_gem_object_wait(obj, 922 I915_WAIT_INTERRUPTIBLE | 923 I915_WAIT_LOCKED, 924 MAX_SCHEDULE_TIMEOUT, 925 NULL); 926 if (ret) 927 return ret; 928 929 ret = i915_gem_object_pin_pages(obj); 930 if (ret) 931 return ret; 932 933 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ || 934 !static_cpu_has(X86_FEATURE_CLFLUSH)) { 935 ret = i915_gem_object_set_to_cpu_domain(obj, false); 936 if (ret) 937 goto err_unpin; 938 else 939 goto out; 940 } 941 942 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 943 944 /* If we're not in the cpu read domain, set ourself into the gtt 945 * read domain and manually flush cachelines (if required). This 946 * optimizes for the case when the gpu will dirty the data 947 * anyway again before the next pread happens. 948 */ 949 if (!obj->cache_dirty && 950 !(obj->read_domains & I915_GEM_DOMAIN_CPU)) 951 *needs_clflush = CLFLUSH_BEFORE; 952 953 out: 954 /* return with the pages pinned */ 955 return 0; 956 957 err_unpin: 958 i915_gem_object_unpin_pages(obj); 959 return ret; 960 } 961 962 int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, 963 unsigned int *needs_clflush) 964 { 965 int ret; 966 967 lockdep_assert_held(&obj->base.dev->struct_mutex); 968 969 *needs_clflush = 0; 970 if (!i915_gem_object_has_struct_page(obj)) 971 return -ENODEV; 972 973 ret = i915_gem_object_wait(obj, 974 I915_WAIT_INTERRUPTIBLE | 975 I915_WAIT_LOCKED | 976 I915_WAIT_ALL, 977 MAX_SCHEDULE_TIMEOUT, 978 NULL); 979 if (ret) 980 return ret; 981 982 ret = i915_gem_object_pin_pages(obj); 983 if (ret) 984 return ret; 985 986 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE || 987 !static_cpu_has(X86_FEATURE_CLFLUSH)) { 988 ret = i915_gem_object_set_to_cpu_domain(obj, true); 989 if (ret) 990 goto err_unpin; 991 else 992 goto out; 993 } 994 995 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 996 997 /* If we're not in the cpu write domain, set ourself into the 998 * gtt write domain and manually flush cachelines (as required). 999 * This optimizes for the case when the gpu will use the data 1000 * right away and we therefore have to clflush anyway. 1001 */ 1002 if (!obj->cache_dirty) { 1003 *needs_clflush |= CLFLUSH_AFTER; 1004 1005 /* 1006 * Same trick applies to invalidate partially written 1007 * cachelines read before writing. 1008 */ 1009 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU)) 1010 *needs_clflush |= CLFLUSH_BEFORE; 1011 } 1012 1013 out: 1014 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 1015 obj->mm.dirty = true; 1016 /* return with the pages pinned */ 1017 return 0; 1018 1019 err_unpin: 1020 i915_gem_object_unpin_pages(obj); 1021 return ret; 1022 } 1023 1024 static void 1025 shmem_clflush_swizzled_range(char *addr, unsigned long length, 1026 bool swizzled) 1027 { 1028 if (unlikely(swizzled)) { 1029 unsigned long start = (unsigned long) addr; 1030 unsigned long end = (unsigned long) addr + length; 1031 1032 /* For swizzling simply ensure that we always flush both 1033 * channels. Lame, but simple and it works. Swizzled 1034 * pwrite/pread is far from a hotpath - current userspace 1035 * doesn't use it at all. */ 1036 start = round_down(start, 128); 1037 end = round_up(end, 128); 1038 1039 drm_clflush_virt_range((void *)start, end - start); 1040 } else { 1041 drm_clflush_virt_range(addr, length); 1042 } 1043 1044 } 1045 1046 /* Only difference to the fast-path function is that this can handle bit17 1047 * and uses non-atomic copy and kmap functions. */ 1048 static int 1049 shmem_pread_slow(struct page *page, int offset, int length, 1050 char __user *user_data, 1051 bool page_do_bit17_swizzling, bool needs_clflush) 1052 { 1053 char *vaddr; 1054 int ret; 1055 1056 vaddr = kmap(page); 1057 if (needs_clflush) 1058 shmem_clflush_swizzled_range(vaddr + offset, length, 1059 page_do_bit17_swizzling); 1060 1061 if (page_do_bit17_swizzling) 1062 ret = __copy_to_user_swizzled(user_data, vaddr, offset, length); 1063 else 1064 ret = __copy_to_user(user_data, vaddr + offset, length); 1065 kunmap(page); 1066 1067 return ret ? - EFAULT : 0; 1068 } 1069 1070 static int 1071 shmem_pread(struct page *page, int offset, int length, char __user *user_data, 1072 bool page_do_bit17_swizzling, bool needs_clflush) 1073 { 1074 int ret; 1075 1076 ret = -ENODEV; 1077 if (!page_do_bit17_swizzling) { 1078 char *vaddr = kmap_atomic(page); 1079 1080 if (needs_clflush) 1081 drm_clflush_virt_range(vaddr + offset, length); 1082 ret = __copy_to_user_inatomic(user_data, vaddr + offset, length); 1083 kunmap_atomic(vaddr); 1084 } 1085 if (ret == 0) 1086 return 0; 1087 1088 return shmem_pread_slow(page, offset, length, user_data, 1089 page_do_bit17_swizzling, needs_clflush); 1090 } 1091 1092 static int 1093 i915_gem_shmem_pread(struct drm_i915_gem_object *obj, 1094 struct drm_i915_gem_pread *args) 1095 { 1096 char __user *user_data; 1097 u64 remain; 1098 unsigned int obj_do_bit17_swizzling; 1099 unsigned int needs_clflush; 1100 unsigned int idx, offset; 1101 int ret; 1102 1103 obj_do_bit17_swizzling = 0; 1104 if (i915_gem_object_needs_bit17_swizzle(obj)) 1105 obj_do_bit17_swizzling = BIT(17); 1106 1107 ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex); 1108 if (ret) 1109 return ret; 1110 1111 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); 1112 mutex_unlock(&obj->base.dev->struct_mutex); 1113 if (ret) 1114 return ret; 1115 1116 remain = args->size; 1117 user_data = u64_to_user_ptr(args->data_ptr); 1118 offset = offset_in_page(args->offset); 1119 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { 1120 struct page *page = i915_gem_object_get_page(obj, idx); 1121 int length; 1122 1123 length = remain; 1124 if (offset + length > PAGE_SIZE) 1125 length = PAGE_SIZE - offset; 1126 1127 ret = shmem_pread(page, offset, length, user_data, 1128 page_to_phys(page) & obj_do_bit17_swizzling, 1129 needs_clflush); 1130 if (ret) 1131 break; 1132 1133 remain -= length; 1134 user_data += length; 1135 offset = 0; 1136 } 1137 1138 i915_gem_obj_finish_shmem_access(obj); 1139 return ret; 1140 } 1141 1142 static inline bool 1143 gtt_user_read(struct io_mapping *mapping, 1144 loff_t base, int offset, 1145 char __user *user_data, int length) 1146 { 1147 void __iomem *vaddr; 1148 unsigned long unwritten; 1149 1150 /* We can use the cpu mem copy function because this is X86. */ 1151 vaddr = io_mapping_map_atomic_wc(mapping, base); 1152 unwritten = __copy_to_user_inatomic(user_data, 1153 (void __force *)vaddr + offset, 1154 length); 1155 io_mapping_unmap_atomic(vaddr); 1156 if (unwritten) { 1157 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE); 1158 unwritten = copy_to_user(user_data, 1159 (void __force *)vaddr + offset, 1160 length); 1161 io_mapping_unmap(vaddr); 1162 } 1163 return unwritten; 1164 } 1165 1166 static int 1167 i915_gem_gtt_pread(struct drm_i915_gem_object *obj, 1168 const struct drm_i915_gem_pread *args) 1169 { 1170 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1171 struct i915_ggtt *ggtt = &i915->ggtt; 1172 struct drm_mm_node node; 1173 struct i915_vma *vma; 1174 void __user *user_data; 1175 u64 remain, offset; 1176 int ret; 1177 1178 ret = mutex_lock_interruptible(&i915->drm.struct_mutex); 1179 if (ret) 1180 return ret; 1181 1182 intel_runtime_pm_get(i915); 1183 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 1184 PIN_MAPPABLE | 1185 PIN_NONFAULT | 1186 PIN_NONBLOCK); 1187 if (!IS_ERR(vma)) { 1188 node.start = i915_ggtt_offset(vma); 1189 node.allocated = false; 1190 ret = i915_vma_put_fence(vma); 1191 if (ret) { 1192 i915_vma_unpin(vma); 1193 vma = ERR_PTR(ret); 1194 } 1195 } 1196 if (IS_ERR(vma)) { 1197 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); 1198 if (ret) 1199 goto out_unlock; 1200 GEM_BUG_ON(!node.allocated); 1201 } 1202 1203 ret = i915_gem_object_set_to_gtt_domain(obj, false); 1204 if (ret) 1205 goto out_unpin; 1206 1207 mutex_unlock(&i915->drm.struct_mutex); 1208 1209 user_data = u64_to_user_ptr(args->data_ptr); 1210 remain = args->size; 1211 offset = args->offset; 1212 1213 while (remain > 0) { 1214 /* Operation in this page 1215 * 1216 * page_base = page offset within aperture 1217 * page_offset = offset within page 1218 * page_length = bytes to copy for this page 1219 */ 1220 u32 page_base = node.start; 1221 unsigned page_offset = offset_in_page(offset); 1222 unsigned page_length = PAGE_SIZE - page_offset; 1223 page_length = remain < page_length ? remain : page_length; 1224 if (node.allocated) { 1225 wmb(); 1226 ggtt->vm.insert_page(&ggtt->vm, 1227 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), 1228 node.start, I915_CACHE_NONE, 0); 1229 wmb(); 1230 } else { 1231 page_base += offset & PAGE_MASK; 1232 } 1233 1234 if (gtt_user_read(&ggtt->iomap, page_base, page_offset, 1235 user_data, page_length)) { 1236 ret = -EFAULT; 1237 break; 1238 } 1239 1240 remain -= page_length; 1241 user_data += page_length; 1242 offset += page_length; 1243 } 1244 1245 mutex_lock(&i915->drm.struct_mutex); 1246 out_unpin: 1247 if (node.allocated) { 1248 wmb(); 1249 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size); 1250 remove_mappable_node(&node); 1251 } else { 1252 i915_vma_unpin(vma); 1253 } 1254 out_unlock: 1255 intel_runtime_pm_put(i915); 1256 mutex_unlock(&i915->drm.struct_mutex); 1257 1258 return ret; 1259 } 1260 1261 /** 1262 * Reads data from the object referenced by handle. 1263 * @dev: drm device pointer 1264 * @data: ioctl data blob 1265 * @file: drm file pointer 1266 * 1267 * On error, the contents of *data are undefined. 1268 */ 1269 int 1270 i915_gem_pread_ioctl(struct drm_device *dev, void *data, 1271 struct drm_file *file) 1272 { 1273 struct drm_i915_gem_pread *args = data; 1274 struct drm_i915_gem_object *obj; 1275 int ret; 1276 1277 if (args->size == 0) 1278 return 0; 1279 1280 if (!access_ok(VERIFY_WRITE, 1281 u64_to_user_ptr(args->data_ptr), 1282 args->size)) 1283 return -EFAULT; 1284 1285 obj = i915_gem_object_lookup(file, args->handle); 1286 if (!obj) 1287 return -ENOENT; 1288 1289 /* Bounds check source. */ 1290 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { 1291 ret = -EINVAL; 1292 goto out; 1293 } 1294 1295 trace_i915_gem_object_pread(obj, args->offset, args->size); 1296 1297 ret = i915_gem_object_wait(obj, 1298 I915_WAIT_INTERRUPTIBLE, 1299 MAX_SCHEDULE_TIMEOUT, 1300 to_rps_client(file)); 1301 if (ret) 1302 goto out; 1303 1304 ret = i915_gem_object_pin_pages(obj); 1305 if (ret) 1306 goto out; 1307 1308 ret = i915_gem_shmem_pread(obj, args); 1309 if (ret == -EFAULT || ret == -ENODEV) 1310 ret = i915_gem_gtt_pread(obj, args); 1311 1312 i915_gem_object_unpin_pages(obj); 1313 out: 1314 i915_gem_object_put(obj); 1315 return ret; 1316 } 1317 1318 /* This is the fast write path which cannot handle 1319 * page faults in the source data 1320 */ 1321 1322 static inline bool 1323 ggtt_write(struct io_mapping *mapping, 1324 loff_t base, int offset, 1325 char __user *user_data, int length) 1326 { 1327 void __iomem *vaddr; 1328 unsigned long unwritten; 1329 1330 /* We can use the cpu mem copy function because this is X86. */ 1331 vaddr = io_mapping_map_atomic_wc(mapping, base); 1332 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, 1333 user_data, length); 1334 io_mapping_unmap_atomic(vaddr); 1335 if (unwritten) { 1336 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE); 1337 unwritten = copy_from_user((void __force *)vaddr + offset, 1338 user_data, length); 1339 io_mapping_unmap(vaddr); 1340 } 1341 1342 return unwritten; 1343 } 1344 1345 /** 1346 * This is the fast pwrite path, where we copy the data directly from the 1347 * user into the GTT, uncached. 1348 * @obj: i915 GEM object 1349 * @args: pwrite arguments structure 1350 */ 1351 static int 1352 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, 1353 const struct drm_i915_gem_pwrite *args) 1354 { 1355 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1356 struct i915_ggtt *ggtt = &i915->ggtt; 1357 struct drm_mm_node node; 1358 struct i915_vma *vma; 1359 u64 remain, offset; 1360 void __user *user_data; 1361 int ret; 1362 1363 ret = mutex_lock_interruptible(&i915->drm.struct_mutex); 1364 if (ret) 1365 return ret; 1366 1367 if (i915_gem_object_has_struct_page(obj)) { 1368 /* 1369 * Avoid waking the device up if we can fallback, as 1370 * waking/resuming is very slow (worst-case 10-100 ms 1371 * depending on PCI sleeps and our own resume time). 1372 * This easily dwarfs any performance advantage from 1373 * using the cache bypass of indirect GGTT access. 1374 */ 1375 if (!intel_runtime_pm_get_if_in_use(i915)) { 1376 ret = -EFAULT; 1377 goto out_unlock; 1378 } 1379 } else { 1380 /* No backing pages, no fallback, we must force GGTT access */ 1381 intel_runtime_pm_get(i915); 1382 } 1383 1384 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 1385 PIN_MAPPABLE | 1386 PIN_NONFAULT | 1387 PIN_NONBLOCK); 1388 if (!IS_ERR(vma)) { 1389 node.start = i915_ggtt_offset(vma); 1390 node.allocated = false; 1391 ret = i915_vma_put_fence(vma); 1392 if (ret) { 1393 i915_vma_unpin(vma); 1394 vma = ERR_PTR(ret); 1395 } 1396 } 1397 if (IS_ERR(vma)) { 1398 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); 1399 if (ret) 1400 goto out_rpm; 1401 GEM_BUG_ON(!node.allocated); 1402 } 1403 1404 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1405 if (ret) 1406 goto out_unpin; 1407 1408 mutex_unlock(&i915->drm.struct_mutex); 1409 1410 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 1411 1412 user_data = u64_to_user_ptr(args->data_ptr); 1413 offset = args->offset; 1414 remain = args->size; 1415 while (remain) { 1416 /* Operation in this page 1417 * 1418 * page_base = page offset within aperture 1419 * page_offset = offset within page 1420 * page_length = bytes to copy for this page 1421 */ 1422 u32 page_base = node.start; 1423 unsigned int page_offset = offset_in_page(offset); 1424 unsigned int page_length = PAGE_SIZE - page_offset; 1425 page_length = remain < page_length ? remain : page_length; 1426 if (node.allocated) { 1427 wmb(); /* flush the write before we modify the GGTT */ 1428 ggtt->vm.insert_page(&ggtt->vm, 1429 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), 1430 node.start, I915_CACHE_NONE, 0); 1431 wmb(); /* flush modifications to the GGTT (insert_page) */ 1432 } else { 1433 page_base += offset & PAGE_MASK; 1434 } 1435 /* If we get a fault while copying data, then (presumably) our 1436 * source page isn't available. Return the error and we'll 1437 * retry in the slow path. 1438 * If the object is non-shmem backed, we retry again with the 1439 * path that handles page fault. 1440 */ 1441 if (ggtt_write(&ggtt->iomap, page_base, page_offset, 1442 user_data, page_length)) { 1443 ret = -EFAULT; 1444 break; 1445 } 1446 1447 remain -= page_length; 1448 user_data += page_length; 1449 offset += page_length; 1450 } 1451 intel_fb_obj_flush(obj, ORIGIN_CPU); 1452 1453 mutex_lock(&i915->drm.struct_mutex); 1454 out_unpin: 1455 if (node.allocated) { 1456 wmb(); 1457 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size); 1458 remove_mappable_node(&node); 1459 } else { 1460 i915_vma_unpin(vma); 1461 } 1462 out_rpm: 1463 intel_runtime_pm_put(i915); 1464 out_unlock: 1465 mutex_unlock(&i915->drm.struct_mutex); 1466 return ret; 1467 } 1468 1469 static int 1470 shmem_pwrite_slow(struct page *page, int offset, int length, 1471 char __user *user_data, 1472 bool page_do_bit17_swizzling, 1473 bool needs_clflush_before, 1474 bool needs_clflush_after) 1475 { 1476 char *vaddr; 1477 int ret; 1478 1479 vaddr = kmap(page); 1480 if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 1481 shmem_clflush_swizzled_range(vaddr + offset, length, 1482 page_do_bit17_swizzling); 1483 if (page_do_bit17_swizzling) 1484 ret = __copy_from_user_swizzled(vaddr, offset, user_data, 1485 length); 1486 else 1487 ret = __copy_from_user(vaddr + offset, user_data, length); 1488 if (needs_clflush_after) 1489 shmem_clflush_swizzled_range(vaddr + offset, length, 1490 page_do_bit17_swizzling); 1491 kunmap(page); 1492 1493 return ret ? -EFAULT : 0; 1494 } 1495 1496 /* Per-page copy function for the shmem pwrite fastpath. 1497 * Flushes invalid cachelines before writing to the target if 1498 * needs_clflush_before is set and flushes out any written cachelines after 1499 * writing if needs_clflush is set. 1500 */ 1501 static int 1502 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, 1503 bool page_do_bit17_swizzling, 1504 bool needs_clflush_before, 1505 bool needs_clflush_after) 1506 { 1507 int ret; 1508 1509 ret = -ENODEV; 1510 if (!page_do_bit17_swizzling) { 1511 char *vaddr = kmap_atomic(page); 1512 1513 if (needs_clflush_before) 1514 drm_clflush_virt_range(vaddr + offset, len); 1515 ret = __copy_from_user_inatomic(vaddr + offset, user_data, len); 1516 if (needs_clflush_after) 1517 drm_clflush_virt_range(vaddr + offset, len); 1518 1519 kunmap_atomic(vaddr); 1520 } 1521 if (ret == 0) 1522 return ret; 1523 1524 return shmem_pwrite_slow(page, offset, len, user_data, 1525 page_do_bit17_swizzling, 1526 needs_clflush_before, 1527 needs_clflush_after); 1528 } 1529 1530 static int 1531 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, 1532 const struct drm_i915_gem_pwrite *args) 1533 { 1534 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1535 void __user *user_data; 1536 u64 remain; 1537 unsigned int obj_do_bit17_swizzling; 1538 unsigned int partial_cacheline_write; 1539 unsigned int needs_clflush; 1540 unsigned int offset, idx; 1541 int ret; 1542 1543 ret = mutex_lock_interruptible(&i915->drm.struct_mutex); 1544 if (ret) 1545 return ret; 1546 1547 ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); 1548 mutex_unlock(&i915->drm.struct_mutex); 1549 if (ret) 1550 return ret; 1551 1552 obj_do_bit17_swizzling = 0; 1553 if (i915_gem_object_needs_bit17_swizzle(obj)) 1554 obj_do_bit17_swizzling = BIT(17); 1555 1556 /* If we don't overwrite a cacheline completely we need to be 1557 * careful to have up-to-date data by first clflushing. Don't 1558 * overcomplicate things and flush the entire patch. 1559 */ 1560 partial_cacheline_write = 0; 1561 if (needs_clflush & CLFLUSH_BEFORE) 1562 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1; 1563 1564 user_data = u64_to_user_ptr(args->data_ptr); 1565 remain = args->size; 1566 offset = offset_in_page(args->offset); 1567 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { 1568 struct page *page = i915_gem_object_get_page(obj, idx); 1569 int length; 1570 1571 length = remain; 1572 if (offset + length > PAGE_SIZE) 1573 length = PAGE_SIZE - offset; 1574 1575 ret = shmem_pwrite(page, offset, length, user_data, 1576 page_to_phys(page) & obj_do_bit17_swizzling, 1577 (offset | length) & partial_cacheline_write, 1578 needs_clflush & CLFLUSH_AFTER); 1579 if (ret) 1580 break; 1581 1582 remain -= length; 1583 user_data += length; 1584 offset = 0; 1585 } 1586 1587 intel_fb_obj_flush(obj, ORIGIN_CPU); 1588 i915_gem_obj_finish_shmem_access(obj); 1589 return ret; 1590 } 1591 1592 /** 1593 * Writes data to the object referenced by handle. 1594 * @dev: drm device 1595 * @data: ioctl data blob 1596 * @file: drm file 1597 * 1598 * On error, the contents of the buffer that were to be modified are undefined. 1599 */ 1600 int 1601 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 1602 struct drm_file *file) 1603 { 1604 struct drm_i915_gem_pwrite *args = data; 1605 struct drm_i915_gem_object *obj; 1606 int ret; 1607 1608 if (args->size == 0) 1609 return 0; 1610 1611 if (!access_ok(VERIFY_READ, 1612 u64_to_user_ptr(args->data_ptr), 1613 args->size)) 1614 return -EFAULT; 1615 1616 obj = i915_gem_object_lookup(file, args->handle); 1617 if (!obj) 1618 return -ENOENT; 1619 1620 /* Bounds check destination. */ 1621 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { 1622 ret = -EINVAL; 1623 goto err; 1624 } 1625 1626 trace_i915_gem_object_pwrite(obj, args->offset, args->size); 1627 1628 ret = -ENODEV; 1629 if (obj->ops->pwrite) 1630 ret = obj->ops->pwrite(obj, args); 1631 if (ret != -ENODEV) 1632 goto err; 1633 1634 ret = i915_gem_object_wait(obj, 1635 I915_WAIT_INTERRUPTIBLE | 1636 I915_WAIT_ALL, 1637 MAX_SCHEDULE_TIMEOUT, 1638 to_rps_client(file)); 1639 if (ret) 1640 goto err; 1641 1642 ret = i915_gem_object_pin_pages(obj); 1643 if (ret) 1644 goto err; 1645 1646 ret = -EFAULT; 1647 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1648 * it would end up going through the fenced access, and we'll get 1649 * different detiling behavior between reading and writing. 1650 * pread/pwrite currently are reading and writing from the CPU 1651 * perspective, requiring manual detiling by the client. 1652 */ 1653 if (!i915_gem_object_has_struct_page(obj) || 1654 cpu_write_needs_clflush(obj)) 1655 /* Note that the gtt paths might fail with non-page-backed user 1656 * pointers (e.g. gtt mappings when moving data between 1657 * textures). Fallback to the shmem path in that case. 1658 */ 1659 ret = i915_gem_gtt_pwrite_fast(obj, args); 1660 1661 if (ret == -EFAULT || ret == -ENOSPC) { 1662 if (obj->phys_handle) 1663 ret = i915_gem_phys_pwrite(obj, args, file); 1664 else 1665 ret = i915_gem_shmem_pwrite(obj, args); 1666 } 1667 1668 i915_gem_object_unpin_pages(obj); 1669 err: 1670 i915_gem_object_put(obj); 1671 return ret; 1672 } 1673 1674 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) 1675 { 1676 struct drm_i915_private *i915; 1677 struct list_head *list; 1678 struct i915_vma *vma; 1679 1680 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 1681 1682 for_each_ggtt_vma(vma, obj) { 1683 if (i915_vma_is_active(vma)) 1684 continue; 1685 1686 if (!drm_mm_node_allocated(&vma->node)) 1687 continue; 1688 1689 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 1690 } 1691 1692 i915 = to_i915(obj->base.dev); 1693 spin_lock(&i915->mm.obj_lock); 1694 list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list; 1695 list_move_tail(&obj->mm.link, list); 1696 spin_unlock(&i915->mm.obj_lock); 1697 } 1698 1699 /** 1700 * Called when user space prepares to use an object with the CPU, either 1701 * through the mmap ioctl's mapping or a GTT mapping. 1702 * @dev: drm device 1703 * @data: ioctl data blob 1704 * @file: drm file 1705 */ 1706 int 1707 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1708 struct drm_file *file) 1709 { 1710 struct drm_i915_gem_set_domain *args = data; 1711 struct drm_i915_gem_object *obj; 1712 uint32_t read_domains = args->read_domains; 1713 uint32_t write_domain = args->write_domain; 1714 int err; 1715 1716 /* Only handle setting domains to types used by the CPU. */ 1717 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS) 1718 return -EINVAL; 1719 1720 /* Having something in the write domain implies it's in the read 1721 * domain, and only that read domain. Enforce that in the request. 1722 */ 1723 if (write_domain != 0 && read_domains != write_domain) 1724 return -EINVAL; 1725 1726 obj = i915_gem_object_lookup(file, args->handle); 1727 if (!obj) 1728 return -ENOENT; 1729 1730 /* Try to flush the object off the GPU without holding the lock. 1731 * We will repeat the flush holding the lock in the normal manner 1732 * to catch cases where we are gazumped. 1733 */ 1734 err = i915_gem_object_wait(obj, 1735 I915_WAIT_INTERRUPTIBLE | 1736 (write_domain ? I915_WAIT_ALL : 0), 1737 MAX_SCHEDULE_TIMEOUT, 1738 to_rps_client(file)); 1739 if (err) 1740 goto out; 1741 1742 /* 1743 * Proxy objects do not control access to the backing storage, ergo 1744 * they cannot be used as a means to manipulate the cache domain 1745 * tracking for that backing storage. The proxy object is always 1746 * considered to be outside of any cache domain. 1747 */ 1748 if (i915_gem_object_is_proxy(obj)) { 1749 err = -ENXIO; 1750 goto out; 1751 } 1752 1753 /* 1754 * Flush and acquire obj->pages so that we are coherent through 1755 * direct access in memory with previous cached writes through 1756 * shmemfs and that our cache domain tracking remains valid. 1757 * For example, if the obj->filp was moved to swap without us 1758 * being notified and releasing the pages, we would mistakenly 1759 * continue to assume that the obj remained out of the CPU cached 1760 * domain. 1761 */ 1762 err = i915_gem_object_pin_pages(obj); 1763 if (err) 1764 goto out; 1765 1766 err = i915_mutex_lock_interruptible(dev); 1767 if (err) 1768 goto out_unpin; 1769 1770 if (read_domains & I915_GEM_DOMAIN_WC) 1771 err = i915_gem_object_set_to_wc_domain(obj, write_domain); 1772 else if (read_domains & I915_GEM_DOMAIN_GTT) 1773 err = i915_gem_object_set_to_gtt_domain(obj, write_domain); 1774 else 1775 err = i915_gem_object_set_to_cpu_domain(obj, write_domain); 1776 1777 /* And bump the LRU for this access */ 1778 i915_gem_object_bump_inactive_ggtt(obj); 1779 1780 mutex_unlock(&dev->struct_mutex); 1781 1782 if (write_domain != 0) 1783 intel_fb_obj_invalidate(obj, 1784 fb_write_origin(obj, write_domain)); 1785 1786 out_unpin: 1787 i915_gem_object_unpin_pages(obj); 1788 out: 1789 i915_gem_object_put(obj); 1790 return err; 1791 } 1792 1793 /** 1794 * Called when user space has done writes to this buffer 1795 * @dev: drm device 1796 * @data: ioctl data blob 1797 * @file: drm file 1798 */ 1799 int 1800 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1801 struct drm_file *file) 1802 { 1803 struct drm_i915_gem_sw_finish *args = data; 1804 struct drm_i915_gem_object *obj; 1805 1806 obj = i915_gem_object_lookup(file, args->handle); 1807 if (!obj) 1808 return -ENOENT; 1809 1810 /* 1811 * Proxy objects are barred from CPU access, so there is no 1812 * need to ban sw_finish as it is a nop. 1813 */ 1814 1815 /* Pinned buffers may be scanout, so flush the cache */ 1816 i915_gem_object_flush_if_display(obj); 1817 i915_gem_object_put(obj); 1818 1819 return 0; 1820 } 1821 1822 /** 1823 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address 1824 * it is mapped to. 1825 * @dev: drm device 1826 * @data: ioctl data blob 1827 * @file: drm file 1828 * 1829 * While the mapping holds a reference on the contents of the object, it doesn't 1830 * imply a ref on the object itself. 1831 * 1832 * IMPORTANT: 1833 * 1834 * DRM driver writers who look a this function as an example for how to do GEM 1835 * mmap support, please don't implement mmap support like here. The modern way 1836 * to implement DRM mmap support is with an mmap offset ioctl (like 1837 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. 1838 * That way debug tooling like valgrind will understand what's going on, hiding 1839 * the mmap call in a driver private ioctl will break that. The i915 driver only 1840 * does cpu mmaps this way because we didn't know better. 1841 */ 1842 int 1843 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1844 struct drm_file *file) 1845 { 1846 struct drm_i915_gem_mmap *args = data; 1847 struct drm_i915_gem_object *obj; 1848 unsigned long addr; 1849 1850 if (args->flags & ~(I915_MMAP_WC)) 1851 return -EINVAL; 1852 1853 if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT)) 1854 return -ENODEV; 1855 1856 obj = i915_gem_object_lookup(file, args->handle); 1857 if (!obj) 1858 return -ENOENT; 1859 1860 /* prime objects have no backing filp to GEM mmap 1861 * pages from. 1862 */ 1863 if (!obj->base.filp) { 1864 i915_gem_object_put(obj); 1865 return -ENXIO; 1866 } 1867 1868 addr = vm_mmap(obj->base.filp, 0, args->size, 1869 PROT_READ | PROT_WRITE, MAP_SHARED, 1870 args->offset); 1871 if (args->flags & I915_MMAP_WC) { 1872 struct mm_struct *mm = current->mm; 1873 struct vm_area_struct *vma; 1874 1875 if (down_write_killable(&mm->mmap_sem)) { 1876 i915_gem_object_put(obj); 1877 return -EINTR; 1878 } 1879 vma = find_vma(mm, addr); 1880 if (vma) 1881 vma->vm_page_prot = 1882 pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 1883 else 1884 addr = -ENOMEM; 1885 up_write(&mm->mmap_sem); 1886 1887 /* This may race, but that's ok, it only gets set */ 1888 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU); 1889 } 1890 i915_gem_object_put(obj); 1891 if (IS_ERR((void *)addr)) 1892 return addr; 1893 1894 args->addr_ptr = (uint64_t) addr; 1895 1896 return 0; 1897 } 1898 1899 static unsigned int tile_row_pages(struct drm_i915_gem_object *obj) 1900 { 1901 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; 1902 } 1903 1904 /** 1905 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps 1906 * 1907 * A history of the GTT mmap interface: 1908 * 1909 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to 1910 * aligned and suitable for fencing, and still fit into the available 1911 * mappable space left by the pinned display objects. A classic problem 1912 * we called the page-fault-of-doom where we would ping-pong between 1913 * two objects that could not fit inside the GTT and so the memcpy 1914 * would page one object in at the expense of the other between every 1915 * single byte. 1916 * 1917 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none 1918 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the 1919 * object is too large for the available space (or simply too large 1920 * for the mappable aperture!), a view is created instead and faulted 1921 * into userspace. (This view is aligned and sized appropriately for 1922 * fenced access.) 1923 * 1924 * 2 - Recognise WC as a separate cache domain so that we can flush the 1925 * delayed writes via GTT before performing direct access via WC. 1926 * 1927 * Restrictions: 1928 * 1929 * * snoopable objects cannot be accessed via the GTT. It can cause machine 1930 * hangs on some architectures, corruption on others. An attempt to service 1931 * a GTT page fault from a snoopable object will generate a SIGBUS. 1932 * 1933 * * the object must be able to fit into RAM (physical memory, though no 1934 * limited to the mappable aperture). 1935 * 1936 * 1937 * Caveats: 1938 * 1939 * * a new GTT page fault will synchronize rendering from the GPU and flush 1940 * all data to system memory. Subsequent access will not be synchronized. 1941 * 1942 * * all mappings are revoked on runtime device suspend. 1943 * 1944 * * there are only 8, 16 or 32 fence registers to share between all users 1945 * (older machines require fence register for display and blitter access 1946 * as well). Contention of the fence registers will cause the previous users 1947 * to be unmapped and any new access will generate new page faults. 1948 * 1949 * * running out of memory while servicing a fault may generate a SIGBUS, 1950 * rather than the expected SIGSEGV. 1951 */ 1952 int i915_gem_mmap_gtt_version(void) 1953 { 1954 return 2; 1955 } 1956 1957 static inline struct i915_ggtt_view 1958 compute_partial_view(struct drm_i915_gem_object *obj, 1959 pgoff_t page_offset, 1960 unsigned int chunk) 1961 { 1962 struct i915_ggtt_view view; 1963 1964 if (i915_gem_object_is_tiled(obj)) 1965 chunk = roundup(chunk, tile_row_pages(obj)); 1966 1967 view.type = I915_GGTT_VIEW_PARTIAL; 1968 view.partial.offset = rounddown(page_offset, chunk); 1969 view.partial.size = 1970 min_t(unsigned int, chunk, 1971 (obj->base.size >> PAGE_SHIFT) - view.partial.offset); 1972 1973 /* If the partial covers the entire object, just create a normal VMA. */ 1974 if (chunk >= obj->base.size >> PAGE_SHIFT) 1975 view.type = I915_GGTT_VIEW_NORMAL; 1976 1977 return view; 1978 } 1979 1980 /** 1981 * i915_gem_fault - fault a page into the GTT 1982 * @vmf: fault info 1983 * 1984 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 1985 * from userspace. The fault handler takes care of binding the object to 1986 * the GTT (if needed), allocating and programming a fence register (again, 1987 * only if needed based on whether the old reg is still valid or the object 1988 * is tiled) and inserting a new PTE into the faulting process. 1989 * 1990 * Note that the faulting process may involve evicting existing objects 1991 * from the GTT and/or fence registers to make room. So performance may 1992 * suffer if the GTT working set is large or there are few fence registers 1993 * left. 1994 * 1995 * The current feature set supported by i915_gem_fault() and thus GTT mmaps 1996 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version). 1997 */ 1998 vm_fault_t i915_gem_fault(struct vm_fault *vmf) 1999 { 2000 #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) 2001 struct vm_area_struct *area = vmf->vma; 2002 struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data); 2003 struct drm_device *dev = obj->base.dev; 2004 struct drm_i915_private *dev_priv = to_i915(dev); 2005 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2006 bool write = !!(vmf->flags & FAULT_FLAG_WRITE); 2007 struct i915_vma *vma; 2008 pgoff_t page_offset; 2009 unsigned int flags; 2010 int ret; 2011 2012 /* We don't use vmf->pgoff since that has the fake offset */ 2013 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; 2014 2015 trace_i915_gem_object_fault(obj, page_offset, true, write); 2016 2017 /* Try to flush the object off the GPU first without holding the lock. 2018 * Upon acquiring the lock, we will perform our sanity checks and then 2019 * repeat the flush holding the lock in the normal manner to catch cases 2020 * where we are gazumped. 2021 */ 2022 ret = i915_gem_object_wait(obj, 2023 I915_WAIT_INTERRUPTIBLE, 2024 MAX_SCHEDULE_TIMEOUT, 2025 NULL); 2026 if (ret) 2027 goto err; 2028 2029 ret = i915_gem_object_pin_pages(obj); 2030 if (ret) 2031 goto err; 2032 2033 intel_runtime_pm_get(dev_priv); 2034 2035 ret = i915_mutex_lock_interruptible(dev); 2036 if (ret) 2037 goto err_rpm; 2038 2039 /* Access to snoopable pages through the GTT is incoherent. */ 2040 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) { 2041 ret = -EFAULT; 2042 goto err_unlock; 2043 } 2044 2045 /* If the object is smaller than a couple of partial vma, it is 2046 * not worth only creating a single partial vma - we may as well 2047 * clear enough space for the full object. 2048 */ 2049 flags = PIN_MAPPABLE; 2050 if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT) 2051 flags |= PIN_NONBLOCK | PIN_NONFAULT; 2052 2053 /* Now pin it into the GTT as needed */ 2054 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags); 2055 if (IS_ERR(vma)) { 2056 /* Use a partial view if it is bigger than available space */ 2057 struct i915_ggtt_view view = 2058 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); 2059 2060 /* Userspace is now writing through an untracked VMA, abandon 2061 * all hope that the hardware is able to track future writes. 2062 */ 2063 obj->frontbuffer_ggtt_origin = ORIGIN_CPU; 2064 2065 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 2066 } 2067 if (IS_ERR(vma)) { 2068 ret = PTR_ERR(vma); 2069 goto err_unlock; 2070 } 2071 2072 ret = i915_gem_object_set_to_gtt_domain(obj, write); 2073 if (ret) 2074 goto err_unpin; 2075 2076 ret = i915_vma_pin_fence(vma); 2077 if (ret) 2078 goto err_unpin; 2079 2080 /* Finally, remap it using the new GTT offset */ 2081 ret = remap_io_mapping(area, 2082 area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT), 2083 (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT, 2084 min_t(u64, vma->size, area->vm_end - area->vm_start), 2085 &ggtt->iomap); 2086 if (ret) 2087 goto err_fence; 2088 2089 /* Mark as being mmapped into userspace for later revocation */ 2090 assert_rpm_wakelock_held(dev_priv); 2091 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) 2092 list_add(&obj->userfault_link, &dev_priv->mm.userfault_list); 2093 GEM_BUG_ON(!obj->userfault_count); 2094 2095 i915_vma_set_ggtt_write(vma); 2096 2097 err_fence: 2098 i915_vma_unpin_fence(vma); 2099 err_unpin: 2100 __i915_vma_unpin(vma); 2101 err_unlock: 2102 mutex_unlock(&dev->struct_mutex); 2103 err_rpm: 2104 intel_runtime_pm_put(dev_priv); 2105 i915_gem_object_unpin_pages(obj); 2106 err: 2107 switch (ret) { 2108 case -EIO: 2109 /* 2110 * We eat errors when the gpu is terminally wedged to avoid 2111 * userspace unduly crashing (gl has no provisions for mmaps to 2112 * fail). But any other -EIO isn't ours (e.g. swap in failure) 2113 * and so needs to be reported. 2114 */ 2115 if (!i915_terminally_wedged(&dev_priv->gpu_error)) 2116 return VM_FAULT_SIGBUS; 2117 case -EAGAIN: 2118 /* 2119 * EAGAIN means the gpu is hung and we'll wait for the error 2120 * handler to reset everything when re-faulting in 2121 * i915_mutex_lock_interruptible. 2122 */ 2123 case 0: 2124 case -ERESTARTSYS: 2125 case -EINTR: 2126 case -EBUSY: 2127 /* 2128 * EBUSY is ok: this just means that another thread 2129 * already did the job. 2130 */ 2131 return VM_FAULT_NOPAGE; 2132 case -ENOMEM: 2133 return VM_FAULT_OOM; 2134 case -ENOSPC: 2135 case -EFAULT: 2136 return VM_FAULT_SIGBUS; 2137 default: 2138 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret); 2139 return VM_FAULT_SIGBUS; 2140 } 2141 } 2142 2143 static void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) 2144 { 2145 struct i915_vma *vma; 2146 2147 GEM_BUG_ON(!obj->userfault_count); 2148 2149 obj->userfault_count = 0; 2150 list_del(&obj->userfault_link); 2151 drm_vma_node_unmap(&obj->base.vma_node, 2152 obj->base.dev->anon_inode->i_mapping); 2153 2154 for_each_ggtt_vma(vma, obj) 2155 i915_vma_unset_userfault(vma); 2156 } 2157 2158 /** 2159 * i915_gem_release_mmap - remove physical page mappings 2160 * @obj: obj in question 2161 * 2162 * Preserve the reservation of the mmapping with the DRM core code, but 2163 * relinquish ownership of the pages back to the system. 2164 * 2165 * It is vital that we remove the page mapping if we have mapped a tiled 2166 * object through the GTT and then lose the fence register due to 2167 * resource pressure. Similarly if the object has been moved out of the 2168 * aperture, than pages mapped into userspace must be revoked. Removing the 2169 * mapping will then trigger a page fault on the next user access, allowing 2170 * fixup by i915_gem_fault(). 2171 */ 2172 void 2173 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2174 { 2175 struct drm_i915_private *i915 = to_i915(obj->base.dev); 2176 2177 /* Serialisation between user GTT access and our code depends upon 2178 * revoking the CPU's PTE whilst the mutex is held. The next user 2179 * pagefault then has to wait until we release the mutex. 2180 * 2181 * Note that RPM complicates somewhat by adding an additional 2182 * requirement that operations to the GGTT be made holding the RPM 2183 * wakeref. 2184 */ 2185 lockdep_assert_held(&i915->drm.struct_mutex); 2186 intel_runtime_pm_get(i915); 2187 2188 if (!obj->userfault_count) 2189 goto out; 2190 2191 __i915_gem_object_release_mmap(obj); 2192 2193 /* Ensure that the CPU's PTE are revoked and there are not outstanding 2194 * memory transactions from userspace before we return. The TLB 2195 * flushing implied above by changing the PTE above *should* be 2196 * sufficient, an extra barrier here just provides us with a bit 2197 * of paranoid documentation about our requirement to serialise 2198 * memory writes before touching registers / GSM. 2199 */ 2200 wmb(); 2201 2202 out: 2203 intel_runtime_pm_put(i915); 2204 } 2205 2206 void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv) 2207 { 2208 struct drm_i915_gem_object *obj, *on; 2209 int i; 2210 2211 /* 2212 * Only called during RPM suspend. All users of the userfault_list 2213 * must be holding an RPM wakeref to ensure that this can not 2214 * run concurrently with themselves (and use the struct_mutex for 2215 * protection between themselves). 2216 */ 2217 2218 list_for_each_entry_safe(obj, on, 2219 &dev_priv->mm.userfault_list, userfault_link) 2220 __i915_gem_object_release_mmap(obj); 2221 2222 /* The fence will be lost when the device powers down. If any were 2223 * in use by hardware (i.e. they are pinned), we should not be powering 2224 * down! All other fences will be reacquired by the user upon waking. 2225 */ 2226 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2227 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2228 2229 /* Ideally we want to assert that the fence register is not 2230 * live at this point (i.e. that no piece of code will be 2231 * trying to write through fence + GTT, as that both violates 2232 * our tracking of activity and associated locking/barriers, 2233 * but also is illegal given that the hw is powered down). 2234 * 2235 * Previously we used reg->pin_count as a "liveness" indicator. 2236 * That is not sufficient, and we need a more fine-grained 2237 * tool if we want to have a sanity check here. 2238 */ 2239 2240 if (!reg->vma) 2241 continue; 2242 2243 GEM_BUG_ON(i915_vma_has_userfault(reg->vma)); 2244 reg->dirty = true; 2245 } 2246 } 2247 2248 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) 2249 { 2250 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 2251 int err; 2252 2253 err = drm_gem_create_mmap_offset(&obj->base); 2254 if (likely(!err)) 2255 return 0; 2256 2257 /* Attempt to reap some mmap space from dead objects */ 2258 do { 2259 err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE); 2260 if (err) 2261 break; 2262 2263 i915_gem_drain_freed_objects(dev_priv); 2264 err = drm_gem_create_mmap_offset(&obj->base); 2265 if (!err) 2266 break; 2267 2268 } while (flush_delayed_work(&dev_priv->gt.retire_work)); 2269 2270 return err; 2271 } 2272 2273 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) 2274 { 2275 drm_gem_free_mmap_offset(&obj->base); 2276 } 2277 2278 int 2279 i915_gem_mmap_gtt(struct drm_file *file, 2280 struct drm_device *dev, 2281 uint32_t handle, 2282 uint64_t *offset) 2283 { 2284 struct drm_i915_gem_object *obj; 2285 int ret; 2286 2287 obj = i915_gem_object_lookup(file, handle); 2288 if (!obj) 2289 return -ENOENT; 2290 2291 ret = i915_gem_object_create_mmap_offset(obj); 2292 if (ret == 0) 2293 *offset = drm_vma_node_offset_addr(&obj->base.vma_node); 2294 2295 i915_gem_object_put(obj); 2296 return ret; 2297 } 2298 2299 /** 2300 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 2301 * @dev: DRM device 2302 * @data: GTT mapping ioctl data 2303 * @file: GEM object info 2304 * 2305 * Simply returns the fake offset to userspace so it can mmap it. 2306 * The mmap call will end up in drm_gem_mmap(), which will set things 2307 * up so we can get faults in the handler above. 2308 * 2309 * The fault handler will take care of binding the object into the GTT 2310 * (since it may have been evicted to make room for something), allocating 2311 * a fence register, and mapping the appropriate aperture address into 2312 * userspace. 2313 */ 2314 int 2315 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 2316 struct drm_file *file) 2317 { 2318 struct drm_i915_gem_mmap_gtt *args = data; 2319 2320 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); 2321 } 2322 2323 /* Immediately discard the backing storage */ 2324 static void 2325 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2326 { 2327 i915_gem_object_free_mmap_offset(obj); 2328 2329 if (obj->base.filp == NULL) 2330 return; 2331 2332 /* Our goal here is to return as much of the memory as 2333 * is possible back to the system as we are called from OOM. 2334 * To do this we must instruct the shmfs to drop all of its 2335 * backing pages, *now*. 2336 */ 2337 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 2338 obj->mm.madv = __I915_MADV_PURGED; 2339 obj->mm.pages = ERR_PTR(-EFAULT); 2340 } 2341 2342 /* Try to discard unwanted pages */ 2343 void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj) 2344 { 2345 struct address_space *mapping; 2346 2347 lockdep_assert_held(&obj->mm.lock); 2348 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 2349 2350 switch (obj->mm.madv) { 2351 case I915_MADV_DONTNEED: 2352 i915_gem_object_truncate(obj); 2353 case __I915_MADV_PURGED: 2354 return; 2355 } 2356 2357 if (obj->base.filp == NULL) 2358 return; 2359 2360 mapping = obj->base.filp->f_mapping, 2361 invalidate_mapping_pages(mapping, 0, (loff_t)-1); 2362 } 2363 2364 static void 2365 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj, 2366 struct sg_table *pages) 2367 { 2368 struct sgt_iter sgt_iter; 2369 struct page *page; 2370 2371 __i915_gem_object_release_shmem(obj, pages, true); 2372 2373 i915_gem_gtt_finish_pages(obj, pages); 2374 2375 if (i915_gem_object_needs_bit17_swizzle(obj)) 2376 i915_gem_object_save_bit_17_swizzle(obj, pages); 2377 2378 for_each_sgt_page(page, sgt_iter, pages) { 2379 if (obj->mm.dirty) 2380 set_page_dirty(page); 2381 2382 if (obj->mm.madv == I915_MADV_WILLNEED) 2383 mark_page_accessed(page); 2384 2385 put_page(page); 2386 } 2387 obj->mm.dirty = false; 2388 2389 sg_free_table(pages); 2390 kfree(pages); 2391 } 2392 2393 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) 2394 { 2395 struct radix_tree_iter iter; 2396 void __rcu **slot; 2397 2398 rcu_read_lock(); 2399 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) 2400 radix_tree_delete(&obj->mm.get_page.radix, iter.index); 2401 rcu_read_unlock(); 2402 } 2403 2404 static struct sg_table * 2405 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) 2406 { 2407 struct drm_i915_private *i915 = to_i915(obj->base.dev); 2408 struct sg_table *pages; 2409 2410 pages = fetch_and_zero(&obj->mm.pages); 2411 if (!pages) 2412 return NULL; 2413 2414 spin_lock(&i915->mm.obj_lock); 2415 list_del(&obj->mm.link); 2416 spin_unlock(&i915->mm.obj_lock); 2417 2418 if (obj->mm.mapping) { 2419 void *ptr; 2420 2421 ptr = page_mask_bits(obj->mm.mapping); 2422 if (is_vmalloc_addr(ptr)) 2423 vunmap(ptr); 2424 else 2425 kunmap(kmap_to_page(ptr)); 2426 2427 obj->mm.mapping = NULL; 2428 } 2429 2430 __i915_gem_object_reset_page_iter(obj); 2431 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; 2432 2433 return pages; 2434 } 2435 2436 void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, 2437 enum i915_mm_subclass subclass) 2438 { 2439 struct sg_table *pages; 2440 2441 if (i915_gem_object_has_pinned_pages(obj)) 2442 return; 2443 2444 GEM_BUG_ON(obj->bind_count); 2445 if (!i915_gem_object_has_pages(obj)) 2446 return; 2447 2448 /* May be called by shrinker from within get_pages() (on another bo) */ 2449 mutex_lock_nested(&obj->mm.lock, subclass); 2450 if (unlikely(atomic_read(&obj->mm.pages_pin_count))) 2451 goto unlock; 2452 2453 /* 2454 * ->put_pages might need to allocate memory for the bit17 swizzle 2455 * array, hence protect them from being reaped by removing them from gtt 2456 * lists early. 2457 */ 2458 pages = __i915_gem_object_unset_pages(obj); 2459 if (!IS_ERR(pages)) 2460 obj->ops->put_pages(obj, pages); 2461 2462 unlock: 2463 mutex_unlock(&obj->mm.lock); 2464 } 2465 2466 static bool i915_sg_trim(struct sg_table *orig_st) 2467 { 2468 struct sg_table new_st; 2469 struct scatterlist *sg, *new_sg; 2470 unsigned int i; 2471 2472 if (orig_st->nents == orig_st->orig_nents) 2473 return false; 2474 2475 if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN)) 2476 return false; 2477 2478 new_sg = new_st.sgl; 2479 for_each_sg(orig_st->sgl, sg, orig_st->nents, i) { 2480 sg_set_page(new_sg, sg_page(sg), sg->length, 0); 2481 /* called before being DMA mapped, no need to copy sg->dma_* */ 2482 new_sg = sg_next(new_sg); 2483 } 2484 GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */ 2485 2486 sg_free_table(orig_st); 2487 2488 *orig_st = new_st; 2489 return true; 2490 } 2491 2492 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) 2493 { 2494 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 2495 const unsigned long page_count = obj->base.size / PAGE_SIZE; 2496 unsigned long i; 2497 struct address_space *mapping; 2498 struct sg_table *st; 2499 struct scatterlist *sg; 2500 struct sgt_iter sgt_iter; 2501 struct page *page; 2502 unsigned long last_pfn = 0; /* suppress gcc warning */ 2503 unsigned int max_segment = i915_sg_segment_size(); 2504 unsigned int sg_page_sizes; 2505 gfp_t noreclaim; 2506 int ret; 2507 2508 /* Assert that the object is not currently in any GPU domain. As it 2509 * wasn't in the GTT, there shouldn't be any way it could have been in 2510 * a GPU cache 2511 */ 2512 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 2513 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 2514 2515 st = kmalloc(sizeof(*st), GFP_KERNEL); 2516 if (st == NULL) 2517 return -ENOMEM; 2518 2519 rebuild_st: 2520 if (sg_alloc_table(st, page_count, GFP_KERNEL)) { 2521 kfree(st); 2522 return -ENOMEM; 2523 } 2524 2525 /* Get the list of pages out of our struct file. They'll be pinned 2526 * at this point until we release them. 2527 * 2528 * Fail silently without starting the shrinker 2529 */ 2530 mapping = obj->base.filp->f_mapping; 2531 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); 2532 noreclaim |= __GFP_NORETRY | __GFP_NOWARN; 2533 2534 sg = st->sgl; 2535 st->nents = 0; 2536 sg_page_sizes = 0; 2537 for (i = 0; i < page_count; i++) { 2538 const unsigned int shrink[] = { 2539 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE, 2540 0, 2541 }, *s = shrink; 2542 gfp_t gfp = noreclaim; 2543 2544 do { 2545 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 2546 if (likely(!IS_ERR(page))) 2547 break; 2548 2549 if (!*s) { 2550 ret = PTR_ERR(page); 2551 goto err_sg; 2552 } 2553 2554 i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++); 2555 cond_resched(); 2556 2557 /* We've tried hard to allocate the memory by reaping 2558 * our own buffer, now let the real VM do its job and 2559 * go down in flames if truly OOM. 2560 * 2561 * However, since graphics tend to be disposable, 2562 * defer the oom here by reporting the ENOMEM back 2563 * to userspace. 2564 */ 2565 if (!*s) { 2566 /* reclaim and warn, but no oom */ 2567 gfp = mapping_gfp_mask(mapping); 2568 2569 /* Our bo are always dirty and so we require 2570 * kswapd to reclaim our pages (direct reclaim 2571 * does not effectively begin pageout of our 2572 * buffers on its own). However, direct reclaim 2573 * only waits for kswapd when under allocation 2574 * congestion. So as a result __GFP_RECLAIM is 2575 * unreliable and fails to actually reclaim our 2576 * dirty pages -- unless you try over and over 2577 * again with !__GFP_NORETRY. However, we still 2578 * want to fail this allocation rather than 2579 * trigger the out-of-memory killer and for 2580 * this we want __GFP_RETRY_MAYFAIL. 2581 */ 2582 gfp |= __GFP_RETRY_MAYFAIL; 2583 } 2584 } while (1); 2585 2586 if (!i || 2587 sg->length >= max_segment || 2588 page_to_pfn(page) != last_pfn + 1) { 2589 if (i) { 2590 sg_page_sizes |= sg->length; 2591 sg = sg_next(sg); 2592 } 2593 st->nents++; 2594 sg_set_page(sg, page, PAGE_SIZE, 0); 2595 } else { 2596 sg->length += PAGE_SIZE; 2597 } 2598 last_pfn = page_to_pfn(page); 2599 2600 /* Check that the i965g/gm workaround works. */ 2601 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL)); 2602 } 2603 if (sg) { /* loop terminated early; short sg table */ 2604 sg_page_sizes |= sg->length; 2605 sg_mark_end(sg); 2606 } 2607 2608 /* Trim unused sg entries to avoid wasting memory. */ 2609 i915_sg_trim(st); 2610 2611 ret = i915_gem_gtt_prepare_pages(obj, st); 2612 if (ret) { 2613 /* DMA remapping failed? One possible cause is that 2614 * it could not reserve enough large entries, asking 2615 * for PAGE_SIZE chunks instead may be helpful. 2616 */ 2617 if (max_segment > PAGE_SIZE) { 2618 for_each_sgt_page(page, sgt_iter, st) 2619 put_page(page); 2620 sg_free_table(st); 2621 2622 max_segment = PAGE_SIZE; 2623 goto rebuild_st; 2624 } else { 2625 dev_warn(&dev_priv->drm.pdev->dev, 2626 "Failed to DMA remap %lu pages\n", 2627 page_count); 2628 goto err_pages; 2629 } 2630 } 2631 2632 if (i915_gem_object_needs_bit17_swizzle(obj)) 2633 i915_gem_object_do_bit_17_swizzle(obj, st); 2634 2635 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 2636 2637 return 0; 2638 2639 err_sg: 2640 sg_mark_end(sg); 2641 err_pages: 2642 for_each_sgt_page(page, sgt_iter, st) 2643 put_page(page); 2644 sg_free_table(st); 2645 kfree(st); 2646 2647 /* shmemfs first checks if there is enough memory to allocate the page 2648 * and reports ENOSPC should there be insufficient, along with the usual 2649 * ENOMEM for a genuine allocation failure. 2650 * 2651 * We use ENOSPC in our driver to mean that we have run out of aperture 2652 * space and so want to translate the error from shmemfs back to our 2653 * usual understanding of ENOMEM. 2654 */ 2655 if (ret == -ENOSPC) 2656 ret = -ENOMEM; 2657 2658 return ret; 2659 } 2660 2661 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 2662 struct sg_table *pages, 2663 unsigned int sg_page_sizes) 2664 { 2665 struct drm_i915_private *i915 = to_i915(obj->base.dev); 2666 unsigned long supported = INTEL_INFO(i915)->page_sizes; 2667 int i; 2668 2669 lockdep_assert_held(&obj->mm.lock); 2670 2671 obj->mm.get_page.sg_pos = pages->sgl; 2672 obj->mm.get_page.sg_idx = 0; 2673 2674 obj->mm.pages = pages; 2675 2676 if (i915_gem_object_is_tiled(obj) && 2677 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 2678 GEM_BUG_ON(obj->mm.quirked); 2679 __i915_gem_object_pin_pages(obj); 2680 obj->mm.quirked = true; 2681 } 2682 2683 GEM_BUG_ON(!sg_page_sizes); 2684 obj->mm.page_sizes.phys = sg_page_sizes; 2685 2686 /* 2687 * Calculate the supported page-sizes which fit into the given 2688 * sg_page_sizes. This will give us the page-sizes which we may be able 2689 * to use opportunistically when later inserting into the GTT. For 2690 * example if phys=2G, then in theory we should be able to use 1G, 2M, 2691 * 64K or 4K pages, although in practice this will depend on a number of 2692 * other factors. 2693 */ 2694 obj->mm.page_sizes.sg = 0; 2695 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 2696 if (obj->mm.page_sizes.phys & ~0u << i) 2697 obj->mm.page_sizes.sg |= BIT(i); 2698 } 2699 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); 2700 2701 spin_lock(&i915->mm.obj_lock); 2702 list_add(&obj->mm.link, &i915->mm.unbound_list); 2703 spin_unlock(&i915->mm.obj_lock); 2704 } 2705 2706 static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2707 { 2708 int err; 2709 2710 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { 2711 DRM_DEBUG("Attempting to obtain a purgeable object\n"); 2712 return -EFAULT; 2713 } 2714 2715 err = obj->ops->get_pages(obj); 2716 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj)); 2717 2718 return err; 2719 } 2720 2721 /* Ensure that the associated pages are gathered from the backing storage 2722 * and pinned into our object. i915_gem_object_pin_pages() may be called 2723 * multiple times before they are released by a single call to 2724 * i915_gem_object_unpin_pages() - once the pages are no longer referenced 2725 * either as a result of memory pressure (reaping pages under the shrinker) 2726 * or as the object is itself released. 2727 */ 2728 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2729 { 2730 int err; 2731 2732 err = mutex_lock_interruptible(&obj->mm.lock); 2733 if (err) 2734 return err; 2735 2736 if (unlikely(!i915_gem_object_has_pages(obj))) { 2737 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 2738 2739 err = ____i915_gem_object_get_pages(obj); 2740 if (err) 2741 goto unlock; 2742 2743 smp_mb__before_atomic(); 2744 } 2745 atomic_inc(&obj->mm.pages_pin_count); 2746 2747 unlock: 2748 mutex_unlock(&obj->mm.lock); 2749 return err; 2750 } 2751 2752 /* The 'mapping' part of i915_gem_object_pin_map() below */ 2753 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, 2754 enum i915_map_type type) 2755 { 2756 unsigned long n_pages = obj->base.size >> PAGE_SHIFT; 2757 struct sg_table *sgt = obj->mm.pages; 2758 struct sgt_iter sgt_iter; 2759 struct page *page; 2760 struct page *stack_pages[32]; 2761 struct page **pages = stack_pages; 2762 unsigned long i = 0; 2763 pgprot_t pgprot; 2764 void *addr; 2765 2766 /* A single page can always be kmapped */ 2767 if (n_pages == 1 && type == I915_MAP_WB) 2768 return kmap(sg_page(sgt->sgl)); 2769 2770 if (n_pages > ARRAY_SIZE(stack_pages)) { 2771 /* Too big for stack -- allocate temporary array instead */ 2772 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); 2773 if (!pages) 2774 return NULL; 2775 } 2776 2777 for_each_sgt_page(page, sgt_iter, sgt) 2778 pages[i++] = page; 2779 2780 /* Check that we have the expected number of pages */ 2781 GEM_BUG_ON(i != n_pages); 2782 2783 switch (type) { 2784 default: 2785 MISSING_CASE(type); 2786 /* fallthrough to use PAGE_KERNEL anyway */ 2787 case I915_MAP_WB: 2788 pgprot = PAGE_KERNEL; 2789 break; 2790 case I915_MAP_WC: 2791 pgprot = pgprot_writecombine(PAGE_KERNEL_IO); 2792 break; 2793 } 2794 addr = vmap(pages, n_pages, 0, pgprot); 2795 2796 if (pages != stack_pages) 2797 kvfree(pages); 2798 2799 return addr; 2800 } 2801 2802 /* get, pin, and map the pages of the object into kernel space */ 2803 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 2804 enum i915_map_type type) 2805 { 2806 enum i915_map_type has_type; 2807 bool pinned; 2808 void *ptr; 2809 int ret; 2810 2811 if (unlikely(!i915_gem_object_has_struct_page(obj))) 2812 return ERR_PTR(-ENXIO); 2813 2814 ret = mutex_lock_interruptible(&obj->mm.lock); 2815 if (ret) 2816 return ERR_PTR(ret); 2817 2818 pinned = !(type & I915_MAP_OVERRIDE); 2819 type &= ~I915_MAP_OVERRIDE; 2820 2821 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { 2822 if (unlikely(!i915_gem_object_has_pages(obj))) { 2823 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); 2824 2825 ret = ____i915_gem_object_get_pages(obj); 2826 if (ret) 2827 goto err_unlock; 2828 2829 smp_mb__before_atomic(); 2830 } 2831 atomic_inc(&obj->mm.pages_pin_count); 2832 pinned = false; 2833 } 2834 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 2835 2836 ptr = page_unpack_bits(obj->mm.mapping, &has_type); 2837 if (ptr && has_type != type) { 2838 if (pinned) { 2839 ret = -EBUSY; 2840 goto err_unpin; 2841 } 2842 2843 if (is_vmalloc_addr(ptr)) 2844 vunmap(ptr); 2845 else 2846 kunmap(kmap_to_page(ptr)); 2847 2848 ptr = obj->mm.mapping = NULL; 2849 } 2850 2851 if (!ptr) { 2852 ptr = i915_gem_object_map(obj, type); 2853 if (!ptr) { 2854 ret = -ENOMEM; 2855 goto err_unpin; 2856 } 2857 2858 obj->mm.mapping = page_pack_bits(ptr, type); 2859 } 2860 2861 out_unlock: 2862 mutex_unlock(&obj->mm.lock); 2863 return ptr; 2864 2865 err_unpin: 2866 atomic_dec(&obj->mm.pages_pin_count); 2867 err_unlock: 2868 ptr = ERR_PTR(ret); 2869 goto out_unlock; 2870 } 2871 2872 static int 2873 i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj, 2874 const struct drm_i915_gem_pwrite *arg) 2875 { 2876 struct address_space *mapping = obj->base.filp->f_mapping; 2877 char __user *user_data = u64_to_user_ptr(arg->data_ptr); 2878 u64 remain, offset; 2879 unsigned int pg; 2880 2881 /* Before we instantiate/pin the backing store for our use, we 2882 * can prepopulate the shmemfs filp efficiently using a write into 2883 * the pagecache. We avoid the penalty of instantiating all the 2884 * pages, important if the user is just writing to a few and never 2885 * uses the object on the GPU, and using a direct write into shmemfs 2886 * allows it to avoid the cost of retrieving a page (either swapin 2887 * or clearing-before-use) before it is overwritten. 2888 */ 2889 if (i915_gem_object_has_pages(obj)) 2890 return -ENODEV; 2891 2892 if (obj->mm.madv != I915_MADV_WILLNEED) 2893 return -EFAULT; 2894 2895 /* Before the pages are instantiated the object is treated as being 2896 * in the CPU domain. The pages will be clflushed as required before 2897 * use, and we can freely write into the pages directly. If userspace 2898 * races pwrite with any other operation; corruption will ensue - 2899 * that is userspace's prerogative! 2900 */ 2901 2902 remain = arg->size; 2903 offset = arg->offset; 2904 pg = offset_in_page(offset); 2905 2906 do { 2907 unsigned int len, unwritten; 2908 struct page *page; 2909 void *data, *vaddr; 2910 int err; 2911 2912 len = PAGE_SIZE - pg; 2913 if (len > remain) 2914 len = remain; 2915 2916 err = pagecache_write_begin(obj->base.filp, mapping, 2917 offset, len, 0, 2918 &page, &data); 2919 if (err < 0) 2920 return err; 2921 2922 vaddr = kmap(page); 2923 unwritten = copy_from_user(vaddr + pg, user_data, len); 2924 kunmap(page); 2925 2926 err = pagecache_write_end(obj->base.filp, mapping, 2927 offset, len, len - unwritten, 2928 page, data); 2929 if (err < 0) 2930 return err; 2931 2932 if (unwritten) 2933 return -EFAULT; 2934 2935 remain -= len; 2936 user_data += len; 2937 offset += len; 2938 pg = 0; 2939 } while (remain); 2940 2941 return 0; 2942 } 2943 2944 static void i915_gem_client_mark_guilty(struct drm_i915_file_private *file_priv, 2945 const struct i915_gem_context *ctx) 2946 { 2947 unsigned int score; 2948 unsigned long prev_hang; 2949 2950 if (i915_gem_context_is_banned(ctx)) 2951 score = I915_CLIENT_SCORE_CONTEXT_BAN; 2952 else 2953 score = 0; 2954 2955 prev_hang = xchg(&file_priv->hang_timestamp, jiffies); 2956 if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES)) 2957 score += I915_CLIENT_SCORE_HANG_FAST; 2958 2959 if (score) { 2960 atomic_add(score, &file_priv->ban_score); 2961 2962 DRM_DEBUG_DRIVER("client %s: gained %u ban score, now %u\n", 2963 ctx->name, score, 2964 atomic_read(&file_priv->ban_score)); 2965 } 2966 } 2967 2968 static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx) 2969 { 2970 unsigned int score; 2971 bool banned, bannable; 2972 2973 atomic_inc(&ctx->guilty_count); 2974 2975 bannable = i915_gem_context_is_bannable(ctx); 2976 score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score); 2977 banned = score >= CONTEXT_SCORE_BAN_THRESHOLD; 2978 2979 /* Cool contexts don't accumulate client ban score */ 2980 if (!bannable) 2981 return; 2982 2983 if (banned) { 2984 DRM_DEBUG_DRIVER("context %s: guilty %d, score %u, banned\n", 2985 ctx->name, atomic_read(&ctx->guilty_count), 2986 score); 2987 i915_gem_context_set_banned(ctx); 2988 } 2989 2990 if (!IS_ERR_OR_NULL(ctx->file_priv)) 2991 i915_gem_client_mark_guilty(ctx->file_priv, ctx); 2992 } 2993 2994 static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx) 2995 { 2996 atomic_inc(&ctx->active_count); 2997 } 2998 2999 struct i915_request * 3000 i915_gem_find_active_request(struct intel_engine_cs *engine) 3001 { 3002 struct i915_request *request, *active = NULL; 3003 unsigned long flags; 3004 3005 /* 3006 * We are called by the error capture, reset and to dump engine 3007 * state at random points in time. In particular, note that neither is 3008 * crucially ordered with an interrupt. After a hang, the GPU is dead 3009 * and we assume that no more writes can happen (we waited long enough 3010 * for all writes that were in transaction to be flushed) - adding an 3011 * extra delay for a recent interrupt is pointless. Hence, we do 3012 * not need an engine->irq_seqno_barrier() before the seqno reads. 3013 * At all other times, we must assume the GPU is still running, but 3014 * we only care about the snapshot of this moment. 3015 */ 3016 spin_lock_irqsave(&engine->timeline.lock, flags); 3017 list_for_each_entry(request, &engine->timeline.requests, link) { 3018 if (__i915_request_completed(request, request->global_seqno)) 3019 continue; 3020 3021 active = request; 3022 break; 3023 } 3024 spin_unlock_irqrestore(&engine->timeline.lock, flags); 3025 3026 return active; 3027 } 3028 3029 /* 3030 * Ensure irq handler finishes, and not run again. 3031 * Also return the active request so that we only search for it once. 3032 */ 3033 struct i915_request * 3034 i915_gem_reset_prepare_engine(struct intel_engine_cs *engine) 3035 { 3036 struct i915_request *request; 3037 3038 /* 3039 * During the reset sequence, we must prevent the engine from 3040 * entering RC6. As the context state is undefined until we restart 3041 * the engine, if it does enter RC6 during the reset, the state 3042 * written to the powercontext is undefined and so we may lose 3043 * GPU state upon resume, i.e. fail to restart after a reset. 3044 */ 3045 intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL); 3046 3047 request = engine->reset.prepare(engine); 3048 if (request && request->fence.error == -EIO) 3049 request = ERR_PTR(-EIO); /* Previous reset failed! */ 3050 3051 return request; 3052 } 3053 3054 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv) 3055 { 3056 struct intel_engine_cs *engine; 3057 struct i915_request *request; 3058 enum intel_engine_id id; 3059 int err = 0; 3060 3061 for_each_engine(engine, dev_priv, id) { 3062 request = i915_gem_reset_prepare_engine(engine); 3063 if (IS_ERR(request)) { 3064 err = PTR_ERR(request); 3065 continue; 3066 } 3067 3068 engine->hangcheck.active_request = request; 3069 } 3070 3071 i915_gem_revoke_fences(dev_priv); 3072 intel_uc_sanitize(dev_priv); 3073 3074 return err; 3075 } 3076 3077 static void skip_request(struct i915_request *request) 3078 { 3079 void *vaddr = request->ring->vaddr; 3080 u32 head; 3081 3082 /* As this request likely depends on state from the lost 3083 * context, clear out all the user operations leaving the 3084 * breadcrumb at the end (so we get the fence notifications). 3085 */ 3086 head = request->head; 3087 if (request->postfix < head) { 3088 memset(vaddr + head, 0, request->ring->size - head); 3089 head = 0; 3090 } 3091 memset(vaddr + head, 0, request->postfix - head); 3092 3093 dma_fence_set_error(&request->fence, -EIO); 3094 } 3095 3096 static void engine_skip_context(struct i915_request *request) 3097 { 3098 struct intel_engine_cs *engine = request->engine; 3099 struct i915_gem_context *hung_ctx = request->gem_context; 3100 struct i915_timeline *timeline = request->timeline; 3101 unsigned long flags; 3102 3103 GEM_BUG_ON(timeline == &engine->timeline); 3104 3105 spin_lock_irqsave(&engine->timeline.lock, flags); 3106 spin_lock_nested(&timeline->lock, SINGLE_DEPTH_NESTING); 3107 3108 list_for_each_entry_continue(request, &engine->timeline.requests, link) 3109 if (request->gem_context == hung_ctx) 3110 skip_request(request); 3111 3112 list_for_each_entry(request, &timeline->requests, link) 3113 skip_request(request); 3114 3115 spin_unlock(&timeline->lock); 3116 spin_unlock_irqrestore(&engine->timeline.lock, flags); 3117 } 3118 3119 /* Returns the request if it was guilty of the hang */ 3120 static struct i915_request * 3121 i915_gem_reset_request(struct intel_engine_cs *engine, 3122 struct i915_request *request, 3123 bool stalled) 3124 { 3125 /* The guilty request will get skipped on a hung engine. 3126 * 3127 * Users of client default contexts do not rely on logical 3128 * state preserved between batches so it is safe to execute 3129 * queued requests following the hang. Non default contexts 3130 * rely on preserved state, so skipping a batch loses the 3131 * evolution of the state and it needs to be considered corrupted. 3132 * Executing more queued batches on top of corrupted state is 3133 * risky. But we take the risk by trying to advance through 3134 * the queued requests in order to make the client behaviour 3135 * more predictable around resets, by not throwing away random 3136 * amount of batches it has prepared for execution. Sophisticated 3137 * clients can use gem_reset_stats_ioctl and dma fence status 3138 * (exported via sync_file info ioctl on explicit fences) to observe 3139 * when it loses the context state and should rebuild accordingly. 3140 * 3141 * The context ban, and ultimately the client ban, mechanism are safety 3142 * valves if client submission ends up resulting in nothing more than 3143 * subsequent hangs. 3144 */ 3145 3146 if (i915_request_completed(request)) { 3147 GEM_TRACE("%s pardoned global=%d (fence %llx:%d), current %d\n", 3148 engine->name, request->global_seqno, 3149 request->fence.context, request->fence.seqno, 3150 intel_engine_get_seqno(engine)); 3151 stalled = false; 3152 } 3153 3154 if (stalled) { 3155 i915_gem_context_mark_guilty(request->gem_context); 3156 skip_request(request); 3157 3158 /* If this context is now banned, skip all pending requests. */ 3159 if (i915_gem_context_is_banned(request->gem_context)) 3160 engine_skip_context(request); 3161 } else { 3162 /* 3163 * Since this is not the hung engine, it may have advanced 3164 * since the hang declaration. Double check by refinding 3165 * the active request at the time of the reset. 3166 */ 3167 request = i915_gem_find_active_request(engine); 3168 if (request) { 3169 unsigned long flags; 3170 3171 i915_gem_context_mark_innocent(request->gem_context); 3172 dma_fence_set_error(&request->fence, -EAGAIN); 3173 3174 /* Rewind the engine to replay the incomplete rq */ 3175 spin_lock_irqsave(&engine->timeline.lock, flags); 3176 request = list_prev_entry(request, link); 3177 if (&request->link == &engine->timeline.requests) 3178 request = NULL; 3179 spin_unlock_irqrestore(&engine->timeline.lock, flags); 3180 } 3181 } 3182 3183 return request; 3184 } 3185 3186 void i915_gem_reset_engine(struct intel_engine_cs *engine, 3187 struct i915_request *request, 3188 bool stalled) 3189 { 3190 /* 3191 * Make sure this write is visible before we re-enable the interrupt 3192 * handlers on another CPU, as tasklet_enable() resolves to just 3193 * a compiler barrier which is insufficient for our purpose here. 3194 */ 3195 smp_store_mb(engine->irq_posted, 0); 3196 3197 if (request) 3198 request = i915_gem_reset_request(engine, request, stalled); 3199 3200 /* Setup the CS to resume from the breadcrumb of the hung request */ 3201 engine->reset.reset(engine, request); 3202 } 3203 3204 void i915_gem_reset(struct drm_i915_private *dev_priv, 3205 unsigned int stalled_mask) 3206 { 3207 struct intel_engine_cs *engine; 3208 enum intel_engine_id id; 3209 3210 lockdep_assert_held(&dev_priv->drm.struct_mutex); 3211 3212 i915_retire_requests(dev_priv); 3213 3214 for_each_engine(engine, dev_priv, id) { 3215 struct intel_context *ce; 3216 3217 i915_gem_reset_engine(engine, 3218 engine->hangcheck.active_request, 3219 stalled_mask & ENGINE_MASK(id)); 3220 ce = fetch_and_zero(&engine->last_retired_context); 3221 if (ce) 3222 intel_context_unpin(ce); 3223 3224 /* 3225 * Ostensibily, we always want a context loaded for powersaving, 3226 * so if the engine is idle after the reset, send a request 3227 * to load our scratch kernel_context. 3228 * 3229 * More mysteriously, if we leave the engine idle after a reset, 3230 * the next userspace batch may hang, with what appears to be 3231 * an incoherent read by the CS (presumably stale TLB). An 3232 * empty request appears sufficient to paper over the glitch. 3233 */ 3234 if (intel_engine_is_idle(engine)) { 3235 struct i915_request *rq; 3236 3237 rq = i915_request_alloc(engine, 3238 dev_priv->kernel_context); 3239 if (!IS_ERR(rq)) 3240 i915_request_add(rq); 3241 } 3242 } 3243 3244 i915_gem_restore_fences(dev_priv); 3245 } 3246 3247 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine) 3248 { 3249 engine->reset.finish(engine); 3250 3251 intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL); 3252 } 3253 3254 void i915_gem_reset_finish(struct drm_i915_private *dev_priv) 3255 { 3256 struct intel_engine_cs *engine; 3257 enum intel_engine_id id; 3258 3259 lockdep_assert_held(&dev_priv->drm.struct_mutex); 3260 3261 for_each_engine(engine, dev_priv, id) { 3262 engine->hangcheck.active_request = NULL; 3263 i915_gem_reset_finish_engine(engine); 3264 } 3265 } 3266 3267 static void nop_submit_request(struct i915_request *request) 3268 { 3269 GEM_TRACE("%s fence %llx:%d -> -EIO\n", 3270 request->engine->name, 3271 request->fence.context, request->fence.seqno); 3272 dma_fence_set_error(&request->fence, -EIO); 3273 3274 i915_request_submit(request); 3275 } 3276 3277 static void nop_complete_submit_request(struct i915_request *request) 3278 { 3279 unsigned long flags; 3280 3281 GEM_TRACE("%s fence %llx:%d -> -EIO\n", 3282 request->engine->name, 3283 request->fence.context, request->fence.seqno); 3284 dma_fence_set_error(&request->fence, -EIO); 3285 3286 spin_lock_irqsave(&request->engine->timeline.lock, flags); 3287 __i915_request_submit(request); 3288 intel_engine_init_global_seqno(request->engine, request->global_seqno); 3289 spin_unlock_irqrestore(&request->engine->timeline.lock, flags); 3290 } 3291 3292 void i915_gem_set_wedged(struct drm_i915_private *i915) 3293 { 3294 struct intel_engine_cs *engine; 3295 enum intel_engine_id id; 3296 3297 GEM_TRACE("start\n"); 3298 3299 if (GEM_SHOW_DEBUG()) { 3300 struct drm_printer p = drm_debug_printer(__func__); 3301 3302 for_each_engine(engine, i915, id) 3303 intel_engine_dump(engine, &p, "%s\n", engine->name); 3304 } 3305 3306 set_bit(I915_WEDGED, &i915->gpu_error.flags); 3307 smp_mb__after_atomic(); 3308 3309 /* 3310 * First, stop submission to hw, but do not yet complete requests by 3311 * rolling the global seqno forward (since this would complete requests 3312 * for which we haven't set the fence error to EIO yet). 3313 */ 3314 for_each_engine(engine, i915, id) { 3315 i915_gem_reset_prepare_engine(engine); 3316 3317 engine->submit_request = nop_submit_request; 3318 engine->schedule = NULL; 3319 } 3320 i915->caps.scheduler = 0; 3321 3322 /* Even if the GPU reset fails, it should still stop the engines */ 3323 intel_gpu_reset(i915, ALL_ENGINES); 3324 3325 /* 3326 * Make sure no one is running the old callback before we proceed with 3327 * cancelling requests and resetting the completion tracking. Otherwise 3328 * we might submit a request to the hardware which never completes. 3329 */ 3330 synchronize_rcu(); 3331 3332 for_each_engine(engine, i915, id) { 3333 /* Mark all executing requests as skipped */ 3334 engine->cancel_requests(engine); 3335 3336 /* 3337 * Only once we've force-cancelled all in-flight requests can we 3338 * start to complete all requests. 3339 */ 3340 engine->submit_request = nop_complete_submit_request; 3341 } 3342 3343 /* 3344 * Make sure no request can slip through without getting completed by 3345 * either this call here to intel_engine_init_global_seqno, or the one 3346 * in nop_complete_submit_request. 3347 */ 3348 synchronize_rcu(); 3349 3350 for_each_engine(engine, i915, id) { 3351 unsigned long flags; 3352 3353 /* 3354 * Mark all pending requests as complete so that any concurrent 3355 * (lockless) lookup doesn't try and wait upon the request as we 3356 * reset it. 3357 */ 3358 spin_lock_irqsave(&engine->timeline.lock, flags); 3359 intel_engine_init_global_seqno(engine, 3360 intel_engine_last_submit(engine)); 3361 spin_unlock_irqrestore(&engine->timeline.lock, flags); 3362 3363 i915_gem_reset_finish_engine(engine); 3364 } 3365 3366 GEM_TRACE("end\n"); 3367 3368 wake_up_all(&i915->gpu_error.reset_queue); 3369 } 3370 3371 bool i915_gem_unset_wedged(struct drm_i915_private *i915) 3372 { 3373 struct i915_timeline *tl; 3374 3375 lockdep_assert_held(&i915->drm.struct_mutex); 3376 if (!test_bit(I915_WEDGED, &i915->gpu_error.flags)) 3377 return true; 3378 3379 GEM_TRACE("start\n"); 3380 3381 /* 3382 * Before unwedging, make sure that all pending operations 3383 * are flushed and errored out - we may have requests waiting upon 3384 * third party fences. We marked all inflight requests as EIO, and 3385 * every execbuf since returned EIO, for consistency we want all 3386 * the currently pending requests to also be marked as EIO, which 3387 * is done inside our nop_submit_request - and so we must wait. 3388 * 3389 * No more can be submitted until we reset the wedged bit. 3390 */ 3391 list_for_each_entry(tl, &i915->gt.timelines, link) { 3392 struct i915_request *rq; 3393 3394 rq = i915_gem_active_peek(&tl->last_request, 3395 &i915->drm.struct_mutex); 3396 if (!rq) 3397 continue; 3398 3399 /* 3400 * We can't use our normal waiter as we want to 3401 * avoid recursively trying to handle the current 3402 * reset. The basic dma_fence_default_wait() installs 3403 * a callback for dma_fence_signal(), which is 3404 * triggered by our nop handler (indirectly, the 3405 * callback enables the signaler thread which is 3406 * woken by the nop_submit_request() advancing the seqno 3407 * and when the seqno passes the fence, the signaler 3408 * then signals the fence waking us up). 3409 */ 3410 if (dma_fence_default_wait(&rq->fence, true, 3411 MAX_SCHEDULE_TIMEOUT) < 0) 3412 return false; 3413 } 3414 i915_retire_requests(i915); 3415 GEM_BUG_ON(i915->gt.active_requests); 3416 3417 /* 3418 * Undo nop_submit_request. We prevent all new i915 requests from 3419 * being queued (by disallowing execbuf whilst wedged) so having 3420 * waited for all active requests above, we know the system is idle 3421 * and do not have to worry about a thread being inside 3422 * engine->submit_request() as we swap over. So unlike installing 3423 * the nop_submit_request on reset, we can do this from normal 3424 * context and do not require stop_machine(). 3425 */ 3426 intel_engines_reset_default_submission(i915); 3427 i915_gem_contexts_lost(i915); 3428 3429 GEM_TRACE("end\n"); 3430 3431 smp_mb__before_atomic(); /* complete takeover before enabling execbuf */ 3432 clear_bit(I915_WEDGED, &i915->gpu_error.flags); 3433 3434 return true; 3435 } 3436 3437 static void 3438 i915_gem_retire_work_handler(struct work_struct *work) 3439 { 3440 struct drm_i915_private *dev_priv = 3441 container_of(work, typeof(*dev_priv), gt.retire_work.work); 3442 struct drm_device *dev = &dev_priv->drm; 3443 3444 /* Come back later if the device is busy... */ 3445 if (mutex_trylock(&dev->struct_mutex)) { 3446 i915_retire_requests(dev_priv); 3447 mutex_unlock(&dev->struct_mutex); 3448 } 3449 3450 /* 3451 * Keep the retire handler running until we are finally idle. 3452 * We do not need to do this test under locking as in the worst-case 3453 * we queue the retire worker once too often. 3454 */ 3455 if (READ_ONCE(dev_priv->gt.awake)) 3456 queue_delayed_work(dev_priv->wq, 3457 &dev_priv->gt.retire_work, 3458 round_jiffies_up_relative(HZ)); 3459 } 3460 3461 static void shrink_caches(struct drm_i915_private *i915) 3462 { 3463 /* 3464 * kmem_cache_shrink() discards empty slabs and reorders partially 3465 * filled slabs to prioritise allocating from the mostly full slabs, 3466 * with the aim of reducing fragmentation. 3467 */ 3468 kmem_cache_shrink(i915->priorities); 3469 kmem_cache_shrink(i915->dependencies); 3470 kmem_cache_shrink(i915->requests); 3471 kmem_cache_shrink(i915->luts); 3472 kmem_cache_shrink(i915->vmas); 3473 kmem_cache_shrink(i915->objects); 3474 } 3475 3476 struct sleep_rcu_work { 3477 union { 3478 struct rcu_head rcu; 3479 struct work_struct work; 3480 }; 3481 struct drm_i915_private *i915; 3482 unsigned int epoch; 3483 }; 3484 3485 static inline bool 3486 same_epoch(struct drm_i915_private *i915, unsigned int epoch) 3487 { 3488 /* 3489 * There is a small chance that the epoch wrapped since we started 3490 * sleeping. If we assume that epoch is at least a u32, then it will 3491 * take at least 2^32 * 100ms for it to wrap, or about 326 years. 3492 */ 3493 return epoch == READ_ONCE(i915->gt.epoch); 3494 } 3495 3496 static void __sleep_work(struct work_struct *work) 3497 { 3498 struct sleep_rcu_work *s = container_of(work, typeof(*s), work); 3499 struct drm_i915_private *i915 = s->i915; 3500 unsigned int epoch = s->epoch; 3501 3502 kfree(s); 3503 if (same_epoch(i915, epoch)) 3504 shrink_caches(i915); 3505 } 3506 3507 static void __sleep_rcu(struct rcu_head *rcu) 3508 { 3509 struct sleep_rcu_work *s = container_of(rcu, typeof(*s), rcu); 3510 struct drm_i915_private *i915 = s->i915; 3511 3512 if (same_epoch(i915, s->epoch)) { 3513 INIT_WORK(&s->work, __sleep_work); 3514 queue_work(i915->wq, &s->work); 3515 } else { 3516 kfree(s); 3517 } 3518 } 3519 3520 static inline bool 3521 new_requests_since_last_retire(const struct drm_i915_private *i915) 3522 { 3523 return (READ_ONCE(i915->gt.active_requests) || 3524 work_pending(&i915->gt.idle_work.work)); 3525 } 3526 3527 static void assert_kernel_context_is_current(struct drm_i915_private *i915) 3528 { 3529 struct intel_engine_cs *engine; 3530 enum intel_engine_id id; 3531 3532 if (i915_terminally_wedged(&i915->gpu_error)) 3533 return; 3534 3535 GEM_BUG_ON(i915->gt.active_requests); 3536 for_each_engine(engine, i915, id) { 3537 GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline.last_request)); 3538 GEM_BUG_ON(engine->last_retired_context != 3539 to_intel_context(i915->kernel_context, engine)); 3540 } 3541 } 3542 3543 static void 3544 i915_gem_idle_work_handler(struct work_struct *work) 3545 { 3546 struct drm_i915_private *dev_priv = 3547 container_of(work, typeof(*dev_priv), gt.idle_work.work); 3548 unsigned int epoch = I915_EPOCH_INVALID; 3549 bool rearm_hangcheck; 3550 3551 if (!READ_ONCE(dev_priv->gt.awake)) 3552 return; 3553 3554 if (READ_ONCE(dev_priv->gt.active_requests)) 3555 return; 3556 3557 /* 3558 * Flush out the last user context, leaving only the pinned 3559 * kernel context resident. When we are idling on the kernel_context, 3560 * no more new requests (with a context switch) are emitted and we 3561 * can finally rest. A consequence is that the idle work handler is 3562 * always called at least twice before idling (and if the system is 3563 * idle that implies a round trip through the retire worker). 3564 */ 3565 mutex_lock(&dev_priv->drm.struct_mutex); 3566 i915_gem_switch_to_kernel_context(dev_priv); 3567 mutex_unlock(&dev_priv->drm.struct_mutex); 3568 3569 GEM_TRACE("active_requests=%d (after switch-to-kernel-context)\n", 3570 READ_ONCE(dev_priv->gt.active_requests)); 3571 3572 /* 3573 * Wait for last execlists context complete, but bail out in case a 3574 * new request is submitted. As we don't trust the hardware, we 3575 * continue on if the wait times out. This is necessary to allow 3576 * the machine to suspend even if the hardware dies, and we will 3577 * try to recover in resume (after depriving the hardware of power, 3578 * it may be in a better mmod). 3579 */ 3580 __wait_for(if (new_requests_since_last_retire(dev_priv)) return, 3581 intel_engines_are_idle(dev_priv), 3582 I915_IDLE_ENGINES_TIMEOUT * 1000, 3583 10, 500); 3584 3585 rearm_hangcheck = 3586 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 3587 3588 if (!mutex_trylock(&dev_priv->drm.struct_mutex)) { 3589 /* Currently busy, come back later */ 3590 mod_delayed_work(dev_priv->wq, 3591 &dev_priv->gt.idle_work, 3592 msecs_to_jiffies(50)); 3593 goto out_rearm; 3594 } 3595 3596 /* 3597 * New request retired after this work handler started, extend active 3598 * period until next instance of the work. 3599 */ 3600 if (new_requests_since_last_retire(dev_priv)) 3601 goto out_unlock; 3602 3603 epoch = __i915_gem_park(dev_priv); 3604 3605 assert_kernel_context_is_current(dev_priv); 3606 3607 rearm_hangcheck = false; 3608 out_unlock: 3609 mutex_unlock(&dev_priv->drm.struct_mutex); 3610 3611 out_rearm: 3612 if (rearm_hangcheck) { 3613 GEM_BUG_ON(!dev_priv->gt.awake); 3614 i915_queue_hangcheck(dev_priv); 3615 } 3616 3617 /* 3618 * When we are idle, it is an opportune time to reap our caches. 3619 * However, we have many objects that utilise RCU and the ordered 3620 * i915->wq that this work is executing on. To try and flush any 3621 * pending frees now we are idle, we first wait for an RCU grace 3622 * period, and then queue a task (that will run last on the wq) to 3623 * shrink and re-optimize the caches. 3624 */ 3625 if (same_epoch(dev_priv, epoch)) { 3626 struct sleep_rcu_work *s = kmalloc(sizeof(*s), GFP_KERNEL); 3627 if (s) { 3628 s->i915 = dev_priv; 3629 s->epoch = epoch; 3630 call_rcu(&s->rcu, __sleep_rcu); 3631 } 3632 } 3633 } 3634 3635 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) 3636 { 3637 struct drm_i915_private *i915 = to_i915(gem->dev); 3638 struct drm_i915_gem_object *obj = to_intel_bo(gem); 3639 struct drm_i915_file_private *fpriv = file->driver_priv; 3640 struct i915_lut_handle *lut, *ln; 3641 3642 mutex_lock(&i915->drm.struct_mutex); 3643 3644 list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) { 3645 struct i915_gem_context *ctx = lut->ctx; 3646 struct i915_vma *vma; 3647 3648 GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF)); 3649 if (ctx->file_priv != fpriv) 3650 continue; 3651 3652 vma = radix_tree_delete(&ctx->handles_vma, lut->handle); 3653 GEM_BUG_ON(vma->obj != obj); 3654 3655 /* We allow the process to have multiple handles to the same 3656 * vma, in the same fd namespace, by virtue of flink/open. 3657 */ 3658 GEM_BUG_ON(!vma->open_count); 3659 if (!--vma->open_count && !i915_vma_is_ggtt(vma)) 3660 i915_vma_close(vma); 3661 3662 list_del(&lut->obj_link); 3663 list_del(&lut->ctx_link); 3664 3665 kmem_cache_free(i915->luts, lut); 3666 __i915_gem_object_release_unless_active(obj); 3667 } 3668 3669 mutex_unlock(&i915->drm.struct_mutex); 3670 } 3671 3672 static unsigned long to_wait_timeout(s64 timeout_ns) 3673 { 3674 if (timeout_ns < 0) 3675 return MAX_SCHEDULE_TIMEOUT; 3676 3677 if (timeout_ns == 0) 3678 return 0; 3679 3680 return nsecs_to_jiffies_timeout(timeout_ns); 3681 } 3682 3683 /** 3684 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 3685 * @dev: drm device pointer 3686 * @data: ioctl data blob 3687 * @file: drm file pointer 3688 * 3689 * Returns 0 if successful, else an error is returned with the remaining time in 3690 * the timeout parameter. 3691 * -ETIME: object is still busy after timeout 3692 * -ERESTARTSYS: signal interrupted the wait 3693 * -ENONENT: object doesn't exist 3694 * Also possible, but rare: 3695 * -EAGAIN: incomplete, restart syscall 3696 * -ENOMEM: damn 3697 * -ENODEV: Internal IRQ fail 3698 * -E?: The add request failed 3699 * 3700 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 3701 * non-zero timeout parameter the wait ioctl will wait for the given number of 3702 * nanoseconds on an object becoming unbusy. Since the wait itself does so 3703 * without holding struct_mutex the object may become re-busied before this 3704 * function completes. A similar but shorter * race condition exists in the busy 3705 * ioctl 3706 */ 3707 int 3708 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 3709 { 3710 struct drm_i915_gem_wait *args = data; 3711 struct drm_i915_gem_object *obj; 3712 ktime_t start; 3713 long ret; 3714 3715 if (args->flags != 0) 3716 return -EINVAL; 3717 3718 obj = i915_gem_object_lookup(file, args->bo_handle); 3719 if (!obj) 3720 return -ENOENT; 3721 3722 start = ktime_get(); 3723 3724 ret = i915_gem_object_wait(obj, 3725 I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL, 3726 to_wait_timeout(args->timeout_ns), 3727 to_rps_client(file)); 3728 3729 if (args->timeout_ns > 0) { 3730 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); 3731 if (args->timeout_ns < 0) 3732 args->timeout_ns = 0; 3733 3734 /* 3735 * Apparently ktime isn't accurate enough and occasionally has a 3736 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 3737 * things up to make the test happy. We allow up to 1 jiffy. 3738 * 3739 * This is a regression from the timespec->ktime conversion. 3740 */ 3741 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) 3742 args->timeout_ns = 0; 3743 3744 /* Asked to wait beyond the jiffie/scheduler precision? */ 3745 if (ret == -ETIME && args->timeout_ns) 3746 ret = -EAGAIN; 3747 } 3748 3749 i915_gem_object_put(obj); 3750 return ret; 3751 } 3752 3753 static int wait_for_timeline(struct i915_timeline *tl, unsigned int flags) 3754 { 3755 struct i915_request *rq; 3756 long ret; 3757 3758 rq = i915_gem_active_get_unlocked(&tl->last_request); 3759 if (!rq) 3760 return 0; 3761 3762 /* 3763 * "Race-to-idle". 3764 * 3765 * Switching to the kernel context is often used a synchronous 3766 * step prior to idling, e.g. in suspend for flushing all 3767 * current operations to memory before sleeping. These we 3768 * want to complete as quickly as possible to avoid prolonged 3769 * stalls, so allow the gpu to boost to maximum clocks. 3770 */ 3771 if (flags & I915_WAIT_FOR_IDLE_BOOST) 3772 gen6_rps_boost(rq, NULL); 3773 3774 ret = i915_request_wait(rq, flags, MAX_SCHEDULE_TIMEOUT); 3775 i915_request_put(rq); 3776 3777 return ret < 0 ? ret : 0; 3778 } 3779 3780 static int wait_for_engines(struct drm_i915_private *i915) 3781 { 3782 if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) { 3783 dev_err(i915->drm.dev, 3784 "Failed to idle engines, declaring wedged!\n"); 3785 GEM_TRACE_DUMP(); 3786 i915_gem_set_wedged(i915); 3787 return -EIO; 3788 } 3789 3790 return 0; 3791 } 3792 3793 int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags) 3794 { 3795 GEM_TRACE("flags=%x (%s)\n", 3796 flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked"); 3797 3798 /* If the device is asleep, we have no requests outstanding */ 3799 if (!READ_ONCE(i915->gt.awake)) 3800 return 0; 3801 3802 if (flags & I915_WAIT_LOCKED) { 3803 struct i915_timeline *tl; 3804 int err; 3805 3806 lockdep_assert_held(&i915->drm.struct_mutex); 3807 3808 list_for_each_entry(tl, &i915->gt.timelines, link) { 3809 err = wait_for_timeline(tl, flags); 3810 if (err) 3811 return err; 3812 } 3813 i915_retire_requests(i915); 3814 GEM_BUG_ON(i915->gt.active_requests); 3815 3816 return wait_for_engines(i915); 3817 } else { 3818 struct intel_engine_cs *engine; 3819 enum intel_engine_id id; 3820 int err; 3821 3822 for_each_engine(engine, i915, id) { 3823 err = wait_for_timeline(&engine->timeline, flags); 3824 if (err) 3825 return err; 3826 } 3827 3828 return 0; 3829 } 3830 } 3831 3832 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj) 3833 { 3834 /* 3835 * We manually flush the CPU domain so that we can override and 3836 * force the flush for the display, and perform it asyncrhonously. 3837 */ 3838 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 3839 if (obj->cache_dirty) 3840 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE); 3841 obj->write_domain = 0; 3842 } 3843 3844 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj) 3845 { 3846 if (!READ_ONCE(obj->pin_global)) 3847 return; 3848 3849 mutex_lock(&obj->base.dev->struct_mutex); 3850 __i915_gem_object_flush_for_display(obj); 3851 mutex_unlock(&obj->base.dev->struct_mutex); 3852 } 3853 3854 /** 3855 * Moves a single object to the WC read, and possibly write domain. 3856 * @obj: object to act on 3857 * @write: ask for write access or read only 3858 * 3859 * This function returns when the move is complete, including waiting on 3860 * flushes to occur. 3861 */ 3862 int 3863 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write) 3864 { 3865 int ret; 3866 3867 lockdep_assert_held(&obj->base.dev->struct_mutex); 3868 3869 ret = i915_gem_object_wait(obj, 3870 I915_WAIT_INTERRUPTIBLE | 3871 I915_WAIT_LOCKED | 3872 (write ? I915_WAIT_ALL : 0), 3873 MAX_SCHEDULE_TIMEOUT, 3874 NULL); 3875 if (ret) 3876 return ret; 3877 3878 if (obj->write_domain == I915_GEM_DOMAIN_WC) 3879 return 0; 3880 3881 /* Flush and acquire obj->pages so that we are coherent through 3882 * direct access in memory with previous cached writes through 3883 * shmemfs and that our cache domain tracking remains valid. 3884 * For example, if the obj->filp was moved to swap without us 3885 * being notified and releasing the pages, we would mistakenly 3886 * continue to assume that the obj remained out of the CPU cached 3887 * domain. 3888 */ 3889 ret = i915_gem_object_pin_pages(obj); 3890 if (ret) 3891 return ret; 3892 3893 flush_write_domain(obj, ~I915_GEM_DOMAIN_WC); 3894 3895 /* Serialise direct access to this object with the barriers for 3896 * coherent writes from the GPU, by effectively invalidating the 3897 * WC domain upon first access. 3898 */ 3899 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0) 3900 mb(); 3901 3902 /* It should now be out of any other write domains, and we can update 3903 * the domain values for our changes. 3904 */ 3905 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0); 3906 obj->read_domains |= I915_GEM_DOMAIN_WC; 3907 if (write) { 3908 obj->read_domains = I915_GEM_DOMAIN_WC; 3909 obj->write_domain = I915_GEM_DOMAIN_WC; 3910 obj->mm.dirty = true; 3911 } 3912 3913 i915_gem_object_unpin_pages(obj); 3914 return 0; 3915 } 3916 3917 /** 3918 * Moves a single object to the GTT read, and possibly write domain. 3919 * @obj: object to act on 3920 * @write: ask for write access or read only 3921 * 3922 * This function returns when the move is complete, including waiting on 3923 * flushes to occur. 3924 */ 3925 int 3926 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 3927 { 3928 int ret; 3929 3930 lockdep_assert_held(&obj->base.dev->struct_mutex); 3931 3932 ret = i915_gem_object_wait(obj, 3933 I915_WAIT_INTERRUPTIBLE | 3934 I915_WAIT_LOCKED | 3935 (write ? I915_WAIT_ALL : 0), 3936 MAX_SCHEDULE_TIMEOUT, 3937 NULL); 3938 if (ret) 3939 return ret; 3940 3941 if (obj->write_domain == I915_GEM_DOMAIN_GTT) 3942 return 0; 3943 3944 /* Flush and acquire obj->pages so that we are coherent through 3945 * direct access in memory with previous cached writes through 3946 * shmemfs and that our cache domain tracking remains valid. 3947 * For example, if the obj->filp was moved to swap without us 3948 * being notified and releasing the pages, we would mistakenly 3949 * continue to assume that the obj remained out of the CPU cached 3950 * domain. 3951 */ 3952 ret = i915_gem_object_pin_pages(obj); 3953 if (ret) 3954 return ret; 3955 3956 flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT); 3957 3958 /* Serialise direct access to this object with the barriers for 3959 * coherent writes from the GPU, by effectively invalidating the 3960 * GTT domain upon first access. 3961 */ 3962 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0) 3963 mb(); 3964 3965 /* It should now be out of any other write domains, and we can update 3966 * the domain values for our changes. 3967 */ 3968 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 3969 obj->read_domains |= I915_GEM_DOMAIN_GTT; 3970 if (write) { 3971 obj->read_domains = I915_GEM_DOMAIN_GTT; 3972 obj->write_domain = I915_GEM_DOMAIN_GTT; 3973 obj->mm.dirty = true; 3974 } 3975 3976 i915_gem_object_unpin_pages(obj); 3977 return 0; 3978 } 3979 3980 /** 3981 * Changes the cache-level of an object across all VMA. 3982 * @obj: object to act on 3983 * @cache_level: new cache level to set for the object 3984 * 3985 * After this function returns, the object will be in the new cache-level 3986 * across all GTT and the contents of the backing storage will be coherent, 3987 * with respect to the new cache-level. In order to keep the backing storage 3988 * coherent for all users, we only allow a single cache level to be set 3989 * globally on the object and prevent it from being changed whilst the 3990 * hardware is reading from the object. That is if the object is currently 3991 * on the scanout it will be set to uncached (or equivalent display 3992 * cache coherency) and all non-MOCS GPU access will also be uncached so 3993 * that all direct access to the scanout remains coherent. 3994 */ 3995 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 3996 enum i915_cache_level cache_level) 3997 { 3998 struct i915_vma *vma; 3999 int ret; 4000 4001 lockdep_assert_held(&obj->base.dev->struct_mutex); 4002 4003 if (obj->cache_level == cache_level) 4004 return 0; 4005 4006 /* Inspect the list of currently bound VMA and unbind any that would 4007 * be invalid given the new cache-level. This is principally to 4008 * catch the issue of the CS prefetch crossing page boundaries and 4009 * reading an invalid PTE on older architectures. 4010 */ 4011 restart: 4012 list_for_each_entry(vma, &obj->vma_list, obj_link) { 4013 if (!drm_mm_node_allocated(&vma->node)) 4014 continue; 4015 4016 if (i915_vma_is_pinned(vma)) { 4017 DRM_DEBUG("can not change the cache level of pinned objects\n"); 4018 return -EBUSY; 4019 } 4020 4021 if (!i915_vma_is_closed(vma) && 4022 i915_gem_valid_gtt_space(vma, cache_level)) 4023 continue; 4024 4025 ret = i915_vma_unbind(vma); 4026 if (ret) 4027 return ret; 4028 4029 /* As unbinding may affect other elements in the 4030 * obj->vma_list (due to side-effects from retiring 4031 * an active vma), play safe and restart the iterator. 4032 */ 4033 goto restart; 4034 } 4035 4036 /* We can reuse the existing drm_mm nodes but need to change the 4037 * cache-level on the PTE. We could simply unbind them all and 4038 * rebind with the correct cache-level on next use. However since 4039 * we already have a valid slot, dma mapping, pages etc, we may as 4040 * rewrite the PTE in the belief that doing so tramples upon less 4041 * state and so involves less work. 4042 */ 4043 if (obj->bind_count) { 4044 /* Before we change the PTE, the GPU must not be accessing it. 4045 * If we wait upon the object, we know that all the bound 4046 * VMA are no longer active. 4047 */ 4048 ret = i915_gem_object_wait(obj, 4049 I915_WAIT_INTERRUPTIBLE | 4050 I915_WAIT_LOCKED | 4051 I915_WAIT_ALL, 4052 MAX_SCHEDULE_TIMEOUT, 4053 NULL); 4054 if (ret) 4055 return ret; 4056 4057 if (!HAS_LLC(to_i915(obj->base.dev)) && 4058 cache_level != I915_CACHE_NONE) { 4059 /* Access to snoopable pages through the GTT is 4060 * incoherent and on some machines causes a hard 4061 * lockup. Relinquish the CPU mmaping to force 4062 * userspace to refault in the pages and we can 4063 * then double check if the GTT mapping is still 4064 * valid for that pointer access. 4065 */ 4066 i915_gem_release_mmap(obj); 4067 4068 /* As we no longer need a fence for GTT access, 4069 * we can relinquish it now (and so prevent having 4070 * to steal a fence from someone else on the next 4071 * fence request). Note GPU activity would have 4072 * dropped the fence as all snoopable access is 4073 * supposed to be linear. 4074 */ 4075 for_each_ggtt_vma(vma, obj) { 4076 ret = i915_vma_put_fence(vma); 4077 if (ret) 4078 return ret; 4079 } 4080 } else { 4081 /* We either have incoherent backing store and 4082 * so no GTT access or the architecture is fully 4083 * coherent. In such cases, existing GTT mmaps 4084 * ignore the cache bit in the PTE and we can 4085 * rewrite it without confusing the GPU or having 4086 * to force userspace to fault back in its mmaps. 4087 */ 4088 } 4089 4090 list_for_each_entry(vma, &obj->vma_list, obj_link) { 4091 if (!drm_mm_node_allocated(&vma->node)) 4092 continue; 4093 4094 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE); 4095 if (ret) 4096 return ret; 4097 } 4098 } 4099 4100 list_for_each_entry(vma, &obj->vma_list, obj_link) 4101 vma->node.color = cache_level; 4102 i915_gem_object_set_cache_coherency(obj, cache_level); 4103 obj->cache_dirty = true; /* Always invalidate stale cachelines */ 4104 4105 return 0; 4106 } 4107 4108 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 4109 struct drm_file *file) 4110 { 4111 struct drm_i915_gem_caching *args = data; 4112 struct drm_i915_gem_object *obj; 4113 int err = 0; 4114 4115 rcu_read_lock(); 4116 obj = i915_gem_object_lookup_rcu(file, args->handle); 4117 if (!obj) { 4118 err = -ENOENT; 4119 goto out; 4120 } 4121 4122 switch (obj->cache_level) { 4123 case I915_CACHE_LLC: 4124 case I915_CACHE_L3_LLC: 4125 args->caching = I915_CACHING_CACHED; 4126 break; 4127 4128 case I915_CACHE_WT: 4129 args->caching = I915_CACHING_DISPLAY; 4130 break; 4131 4132 default: 4133 args->caching = I915_CACHING_NONE; 4134 break; 4135 } 4136 out: 4137 rcu_read_unlock(); 4138 return err; 4139 } 4140 4141 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 4142 struct drm_file *file) 4143 { 4144 struct drm_i915_private *i915 = to_i915(dev); 4145 struct drm_i915_gem_caching *args = data; 4146 struct drm_i915_gem_object *obj; 4147 enum i915_cache_level level; 4148 int ret = 0; 4149 4150 switch (args->caching) { 4151 case I915_CACHING_NONE: 4152 level = I915_CACHE_NONE; 4153 break; 4154 case I915_CACHING_CACHED: 4155 /* 4156 * Due to a HW issue on BXT A stepping, GPU stores via a 4157 * snooped mapping may leave stale data in a corresponding CPU 4158 * cacheline, whereas normally such cachelines would get 4159 * invalidated. 4160 */ 4161 if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) 4162 return -ENODEV; 4163 4164 level = I915_CACHE_LLC; 4165 break; 4166 case I915_CACHING_DISPLAY: 4167 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE; 4168 break; 4169 default: 4170 return -EINVAL; 4171 } 4172 4173 obj = i915_gem_object_lookup(file, args->handle); 4174 if (!obj) 4175 return -ENOENT; 4176 4177 /* 4178 * The caching mode of proxy object is handled by its generator, and 4179 * not allowed to be changed by userspace. 4180 */ 4181 if (i915_gem_object_is_proxy(obj)) { 4182 ret = -ENXIO; 4183 goto out; 4184 } 4185 4186 if (obj->cache_level == level) 4187 goto out; 4188 4189 ret = i915_gem_object_wait(obj, 4190 I915_WAIT_INTERRUPTIBLE, 4191 MAX_SCHEDULE_TIMEOUT, 4192 to_rps_client(file)); 4193 if (ret) 4194 goto out; 4195 4196 ret = i915_mutex_lock_interruptible(dev); 4197 if (ret) 4198 goto out; 4199 4200 ret = i915_gem_object_set_cache_level(obj, level); 4201 mutex_unlock(&dev->struct_mutex); 4202 4203 out: 4204 i915_gem_object_put(obj); 4205 return ret; 4206 } 4207 4208 /* 4209 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from 4210 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined 4211 * (for pageflips). We only flush the caches while preparing the buffer for 4212 * display, the callers are responsible for frontbuffer flush. 4213 */ 4214 struct i915_vma * 4215 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 4216 u32 alignment, 4217 const struct i915_ggtt_view *view, 4218 unsigned int flags) 4219 { 4220 struct i915_vma *vma; 4221 int ret; 4222 4223 lockdep_assert_held(&obj->base.dev->struct_mutex); 4224 4225 /* Mark the global pin early so that we account for the 4226 * display coherency whilst setting up the cache domains. 4227 */ 4228 obj->pin_global++; 4229 4230 /* The display engine is not coherent with the LLC cache on gen6. As 4231 * a result, we make sure that the pinning that is about to occur is 4232 * done with uncached PTEs. This is lowest common denominator for all 4233 * chipsets. 4234 * 4235 * However for gen6+, we could do better by using the GFDT bit instead 4236 * of uncaching, which would allow us to flush all the LLC-cached data 4237 * with that bit in the PTE to main memory with just one PIPE_CONTROL. 4238 */ 4239 ret = i915_gem_object_set_cache_level(obj, 4240 HAS_WT(to_i915(obj->base.dev)) ? 4241 I915_CACHE_WT : I915_CACHE_NONE); 4242 if (ret) { 4243 vma = ERR_PTR(ret); 4244 goto err_unpin_global; 4245 } 4246 4247 /* As the user may map the buffer once pinned in the display plane 4248 * (e.g. libkms for the bootup splash), we have to ensure that we 4249 * always use map_and_fenceable for all scanout buffers. However, 4250 * it may simply be too big to fit into mappable, in which case 4251 * put it anyway and hope that userspace can cope (but always first 4252 * try to preserve the existing ABI). 4253 */ 4254 vma = ERR_PTR(-ENOSPC); 4255 if ((flags & PIN_MAPPABLE) == 0 && 4256 (!view || view->type == I915_GGTT_VIEW_NORMAL)) 4257 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, 4258 flags | 4259 PIN_MAPPABLE | 4260 PIN_NONBLOCK); 4261 if (IS_ERR(vma)) 4262 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags); 4263 if (IS_ERR(vma)) 4264 goto err_unpin_global; 4265 4266 vma->display_alignment = max_t(u64, vma->display_alignment, alignment); 4267 4268 __i915_gem_object_flush_for_display(obj); 4269 4270 /* It should now be out of any other write domains, and we can update 4271 * the domain values for our changes. 4272 */ 4273 obj->read_domains |= I915_GEM_DOMAIN_GTT; 4274 4275 return vma; 4276 4277 err_unpin_global: 4278 obj->pin_global--; 4279 return vma; 4280 } 4281 4282 void 4283 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) 4284 { 4285 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 4286 4287 if (WARN_ON(vma->obj->pin_global == 0)) 4288 return; 4289 4290 if (--vma->obj->pin_global == 0) 4291 vma->display_alignment = I915_GTT_MIN_ALIGNMENT; 4292 4293 /* Bump the LRU to try and avoid premature eviction whilst flipping */ 4294 i915_gem_object_bump_inactive_ggtt(vma->obj); 4295 4296 i915_vma_unpin(vma); 4297 } 4298 4299 /** 4300 * Moves a single object to the CPU read, and possibly write domain. 4301 * @obj: object to act on 4302 * @write: requesting write or read-only access 4303 * 4304 * This function returns when the move is complete, including waiting on 4305 * flushes to occur. 4306 */ 4307 int 4308 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 4309 { 4310 int ret; 4311 4312 lockdep_assert_held(&obj->base.dev->struct_mutex); 4313 4314 ret = i915_gem_object_wait(obj, 4315 I915_WAIT_INTERRUPTIBLE | 4316 I915_WAIT_LOCKED | 4317 (write ? I915_WAIT_ALL : 0), 4318 MAX_SCHEDULE_TIMEOUT, 4319 NULL); 4320 if (ret) 4321 return ret; 4322 4323 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); 4324 4325 /* Flush the CPU cache if it's still invalid. */ 4326 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 4327 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); 4328 obj->read_domains |= I915_GEM_DOMAIN_CPU; 4329 } 4330 4331 /* It should now be out of any other write domains, and we can update 4332 * the domain values for our changes. 4333 */ 4334 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU); 4335 4336 /* If we're writing through the CPU, then the GPU read domains will 4337 * need to be invalidated at next use. 4338 */ 4339 if (write) 4340 __start_cpu_write(obj); 4341 4342 return 0; 4343 } 4344 4345 /* Throttle our rendering by waiting until the ring has completed our requests 4346 * emitted over 20 msec ago. 4347 * 4348 * Note that if we were to use the current jiffies each time around the loop, 4349 * we wouldn't escape the function with any frames outstanding if the time to 4350 * render a frame was over 20ms. 4351 * 4352 * This should get us reasonable parallelism between CPU and GPU but also 4353 * relatively low latency when blocking on a particular request to finish. 4354 */ 4355 static int 4356 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 4357 { 4358 struct drm_i915_private *dev_priv = to_i915(dev); 4359 struct drm_i915_file_private *file_priv = file->driver_priv; 4360 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; 4361 struct i915_request *request, *target = NULL; 4362 long ret; 4363 4364 /* ABI: return -EIO if already wedged */ 4365 if (i915_terminally_wedged(&dev_priv->gpu_error)) 4366 return -EIO; 4367 4368 spin_lock(&file_priv->mm.lock); 4369 list_for_each_entry(request, &file_priv->mm.request_list, client_link) { 4370 if (time_after_eq(request->emitted_jiffies, recent_enough)) 4371 break; 4372 4373 if (target) { 4374 list_del(&target->client_link); 4375 target->file_priv = NULL; 4376 } 4377 4378 target = request; 4379 } 4380 if (target) 4381 i915_request_get(target); 4382 spin_unlock(&file_priv->mm.lock); 4383 4384 if (target == NULL) 4385 return 0; 4386 4387 ret = i915_request_wait(target, 4388 I915_WAIT_INTERRUPTIBLE, 4389 MAX_SCHEDULE_TIMEOUT); 4390 i915_request_put(target); 4391 4392 return ret < 0 ? ret : 0; 4393 } 4394 4395 struct i915_vma * 4396 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, 4397 const struct i915_ggtt_view *view, 4398 u64 size, 4399 u64 alignment, 4400 u64 flags) 4401 { 4402 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 4403 struct i915_address_space *vm = &dev_priv->ggtt.vm; 4404 struct i915_vma *vma; 4405 int ret; 4406 4407 lockdep_assert_held(&obj->base.dev->struct_mutex); 4408 4409 if (flags & PIN_MAPPABLE && 4410 (!view || view->type == I915_GGTT_VIEW_NORMAL)) { 4411 /* If the required space is larger than the available 4412 * aperture, we will not able to find a slot for the 4413 * object and unbinding the object now will be in 4414 * vain. Worse, doing so may cause us to ping-pong 4415 * the object in and out of the Global GTT and 4416 * waste a lot of cycles under the mutex. 4417 */ 4418 if (obj->base.size > dev_priv->ggtt.mappable_end) 4419 return ERR_PTR(-E2BIG); 4420 4421 /* If NONBLOCK is set the caller is optimistically 4422 * trying to cache the full object within the mappable 4423 * aperture, and *must* have a fallback in place for 4424 * situations where we cannot bind the object. We 4425 * can be a little more lax here and use the fallback 4426 * more often to avoid costly migrations of ourselves 4427 * and other objects within the aperture. 4428 * 4429 * Half-the-aperture is used as a simple heuristic. 4430 * More interesting would to do search for a free 4431 * block prior to making the commitment to unbind. 4432 * That caters for the self-harm case, and with a 4433 * little more heuristics (e.g. NOFAULT, NOEVICT) 4434 * we could try to minimise harm to others. 4435 */ 4436 if (flags & PIN_NONBLOCK && 4437 obj->base.size > dev_priv->ggtt.mappable_end / 2) 4438 return ERR_PTR(-ENOSPC); 4439 } 4440 4441 vma = i915_vma_instance(obj, vm, view); 4442 if (unlikely(IS_ERR(vma))) 4443 return vma; 4444 4445 if (i915_vma_misplaced(vma, size, alignment, flags)) { 4446 if (flags & PIN_NONBLOCK) { 4447 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) 4448 return ERR_PTR(-ENOSPC); 4449 4450 if (flags & PIN_MAPPABLE && 4451 vma->fence_size > dev_priv->ggtt.mappable_end / 2) 4452 return ERR_PTR(-ENOSPC); 4453 } 4454 4455 WARN(i915_vma_is_pinned(vma), 4456 "bo is already pinned in ggtt with incorrect alignment:" 4457 " offset=%08x, req.alignment=%llx," 4458 " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n", 4459 i915_ggtt_offset(vma), alignment, 4460 !!(flags & PIN_MAPPABLE), 4461 i915_vma_is_map_and_fenceable(vma)); 4462 ret = i915_vma_unbind(vma); 4463 if (ret) 4464 return ERR_PTR(ret); 4465 } 4466 4467 ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL); 4468 if (ret) 4469 return ERR_PTR(ret); 4470 4471 return vma; 4472 } 4473 4474 static __always_inline unsigned int __busy_read_flag(unsigned int id) 4475 { 4476 /* Note that we could alias engines in the execbuf API, but 4477 * that would be very unwise as it prevents userspace from 4478 * fine control over engine selection. Ahem. 4479 * 4480 * This should be something like EXEC_MAX_ENGINE instead of 4481 * I915_NUM_ENGINES. 4482 */ 4483 BUILD_BUG_ON(I915_NUM_ENGINES > 16); 4484 return 0x10000 << id; 4485 } 4486 4487 static __always_inline unsigned int __busy_write_id(unsigned int id) 4488 { 4489 /* The uABI guarantees an active writer is also amongst the read 4490 * engines. This would be true if we accessed the activity tracking 4491 * under the lock, but as we perform the lookup of the object and 4492 * its activity locklessly we can not guarantee that the last_write 4493 * being active implies that we have set the same engine flag from 4494 * last_read - hence we always set both read and write busy for 4495 * last_write. 4496 */ 4497 return id | __busy_read_flag(id); 4498 } 4499 4500 static __always_inline unsigned int 4501 __busy_set_if_active(const struct dma_fence *fence, 4502 unsigned int (*flag)(unsigned int id)) 4503 { 4504 struct i915_request *rq; 4505 4506 /* We have to check the current hw status of the fence as the uABI 4507 * guarantees forward progress. We could rely on the idle worker 4508 * to eventually flush us, but to minimise latency just ask the 4509 * hardware. 4510 * 4511 * Note we only report on the status of native fences. 4512 */ 4513 if (!dma_fence_is_i915(fence)) 4514 return 0; 4515 4516 /* opencode to_request() in order to avoid const warnings */ 4517 rq = container_of(fence, struct i915_request, fence); 4518 if (i915_request_completed(rq)) 4519 return 0; 4520 4521 return flag(rq->engine->uabi_id); 4522 } 4523 4524 static __always_inline unsigned int 4525 busy_check_reader(const struct dma_fence *fence) 4526 { 4527 return __busy_set_if_active(fence, __busy_read_flag); 4528 } 4529 4530 static __always_inline unsigned int 4531 busy_check_writer(const struct dma_fence *fence) 4532 { 4533 if (!fence) 4534 return 0; 4535 4536 return __busy_set_if_active(fence, __busy_write_id); 4537 } 4538 4539 int 4540 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 4541 struct drm_file *file) 4542 { 4543 struct drm_i915_gem_busy *args = data; 4544 struct drm_i915_gem_object *obj; 4545 struct reservation_object_list *list; 4546 unsigned int seq; 4547 int err; 4548 4549 err = -ENOENT; 4550 rcu_read_lock(); 4551 obj = i915_gem_object_lookup_rcu(file, args->handle); 4552 if (!obj) 4553 goto out; 4554 4555 /* A discrepancy here is that we do not report the status of 4556 * non-i915 fences, i.e. even though we may report the object as idle, 4557 * a call to set-domain may still stall waiting for foreign rendering. 4558 * This also means that wait-ioctl may report an object as busy, 4559 * where busy-ioctl considers it idle. 4560 * 4561 * We trade the ability to warn of foreign fences to report on which 4562 * i915 engines are active for the object. 4563 * 4564 * Alternatively, we can trade that extra information on read/write 4565 * activity with 4566 * args->busy = 4567 * !reservation_object_test_signaled_rcu(obj->resv, true); 4568 * to report the overall busyness. This is what the wait-ioctl does. 4569 * 4570 */ 4571 retry: 4572 seq = raw_read_seqcount(&obj->resv->seq); 4573 4574 /* Translate the exclusive fence to the READ *and* WRITE engine */ 4575 args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl)); 4576 4577 /* Translate shared fences to READ set of engines */ 4578 list = rcu_dereference(obj->resv->fence); 4579 if (list) { 4580 unsigned int shared_count = list->shared_count, i; 4581 4582 for (i = 0; i < shared_count; ++i) { 4583 struct dma_fence *fence = 4584 rcu_dereference(list->shared[i]); 4585 4586 args->busy |= busy_check_reader(fence); 4587 } 4588 } 4589 4590 if (args->busy && read_seqcount_retry(&obj->resv->seq, seq)) 4591 goto retry; 4592 4593 err = 0; 4594 out: 4595 rcu_read_unlock(); 4596 return err; 4597 } 4598 4599 int 4600 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 4601 struct drm_file *file_priv) 4602 { 4603 return i915_gem_ring_throttle(dev, file_priv); 4604 } 4605 4606 int 4607 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 4608 struct drm_file *file_priv) 4609 { 4610 struct drm_i915_private *dev_priv = to_i915(dev); 4611 struct drm_i915_gem_madvise *args = data; 4612 struct drm_i915_gem_object *obj; 4613 int err; 4614 4615 switch (args->madv) { 4616 case I915_MADV_DONTNEED: 4617 case I915_MADV_WILLNEED: 4618 break; 4619 default: 4620 return -EINVAL; 4621 } 4622 4623 obj = i915_gem_object_lookup(file_priv, args->handle); 4624 if (!obj) 4625 return -ENOENT; 4626 4627 err = mutex_lock_interruptible(&obj->mm.lock); 4628 if (err) 4629 goto out; 4630 4631 if (i915_gem_object_has_pages(obj) && 4632 i915_gem_object_is_tiled(obj) && 4633 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 4634 if (obj->mm.madv == I915_MADV_WILLNEED) { 4635 GEM_BUG_ON(!obj->mm.quirked); 4636 __i915_gem_object_unpin_pages(obj); 4637 obj->mm.quirked = false; 4638 } 4639 if (args->madv == I915_MADV_WILLNEED) { 4640 GEM_BUG_ON(obj->mm.quirked); 4641 __i915_gem_object_pin_pages(obj); 4642 obj->mm.quirked = true; 4643 } 4644 } 4645 4646 if (obj->mm.madv != __I915_MADV_PURGED) 4647 obj->mm.madv = args->madv; 4648 4649 /* if the object is no longer attached, discard its backing storage */ 4650 if (obj->mm.madv == I915_MADV_DONTNEED && 4651 !i915_gem_object_has_pages(obj)) 4652 i915_gem_object_truncate(obj); 4653 4654 args->retained = obj->mm.madv != __I915_MADV_PURGED; 4655 mutex_unlock(&obj->mm.lock); 4656 4657 out: 4658 i915_gem_object_put(obj); 4659 return err; 4660 } 4661 4662 static void 4663 frontbuffer_retire(struct i915_gem_active *active, struct i915_request *request) 4664 { 4665 struct drm_i915_gem_object *obj = 4666 container_of(active, typeof(*obj), frontbuffer_write); 4667 4668 intel_fb_obj_flush(obj, ORIGIN_CS); 4669 } 4670 4671 void i915_gem_object_init(struct drm_i915_gem_object *obj, 4672 const struct drm_i915_gem_object_ops *ops) 4673 { 4674 mutex_init(&obj->mm.lock); 4675 4676 INIT_LIST_HEAD(&obj->vma_list); 4677 INIT_LIST_HEAD(&obj->lut_list); 4678 INIT_LIST_HEAD(&obj->batch_pool_link); 4679 4680 obj->ops = ops; 4681 4682 reservation_object_init(&obj->__builtin_resv); 4683 obj->resv = &obj->__builtin_resv; 4684 4685 obj->frontbuffer_ggtt_origin = ORIGIN_GTT; 4686 init_request_active(&obj->frontbuffer_write, frontbuffer_retire); 4687 4688 obj->mm.madv = I915_MADV_WILLNEED; 4689 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); 4690 mutex_init(&obj->mm.get_page.lock); 4691 4692 i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size); 4693 } 4694 4695 static const struct drm_i915_gem_object_ops i915_gem_object_ops = { 4696 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 4697 I915_GEM_OBJECT_IS_SHRINKABLE, 4698 4699 .get_pages = i915_gem_object_get_pages_gtt, 4700 .put_pages = i915_gem_object_put_pages_gtt, 4701 4702 .pwrite = i915_gem_object_pwrite_gtt, 4703 }; 4704 4705 static int i915_gem_object_create_shmem(struct drm_device *dev, 4706 struct drm_gem_object *obj, 4707 size_t size) 4708 { 4709 struct drm_i915_private *i915 = to_i915(dev); 4710 unsigned long flags = VM_NORESERVE; 4711 struct file *filp; 4712 4713 drm_gem_private_object_init(dev, obj, size); 4714 4715 if (i915->mm.gemfs) 4716 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, 4717 flags); 4718 else 4719 filp = shmem_file_setup("i915", size, flags); 4720 4721 if (IS_ERR(filp)) 4722 return PTR_ERR(filp); 4723 4724 obj->filp = filp; 4725 4726 return 0; 4727 } 4728 4729 struct drm_i915_gem_object * 4730 i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size) 4731 { 4732 struct drm_i915_gem_object *obj; 4733 struct address_space *mapping; 4734 unsigned int cache_level; 4735 gfp_t mask; 4736 int ret; 4737 4738 /* There is a prevalence of the assumption that we fit the object's 4739 * page count inside a 32bit _signed_ variable. Let's document this and 4740 * catch if we ever need to fix it. In the meantime, if you do spot 4741 * such a local variable, please consider fixing! 4742 */ 4743 if (size >> PAGE_SHIFT > INT_MAX) 4744 return ERR_PTR(-E2BIG); 4745 4746 if (overflows_type(size, obj->base.size)) 4747 return ERR_PTR(-E2BIG); 4748 4749 obj = i915_gem_object_alloc(dev_priv); 4750 if (obj == NULL) 4751 return ERR_PTR(-ENOMEM); 4752 4753 ret = i915_gem_object_create_shmem(&dev_priv->drm, &obj->base, size); 4754 if (ret) 4755 goto fail; 4756 4757 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 4758 if (IS_I965GM(dev_priv) || IS_I965G(dev_priv)) { 4759 /* 965gm cannot relocate objects above 4GiB. */ 4760 mask &= ~__GFP_HIGHMEM; 4761 mask |= __GFP_DMA32; 4762 } 4763 4764 mapping = obj->base.filp->f_mapping; 4765 mapping_set_gfp_mask(mapping, mask); 4766 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); 4767 4768 i915_gem_object_init(obj, &i915_gem_object_ops); 4769 4770 obj->write_domain = I915_GEM_DOMAIN_CPU; 4771 obj->read_domains = I915_GEM_DOMAIN_CPU; 4772 4773 if (HAS_LLC(dev_priv)) 4774 /* On some devices, we can have the GPU use the LLC (the CPU 4775 * cache) for about a 10% performance improvement 4776 * compared to uncached. Graphics requests other than 4777 * display scanout are coherent with the CPU in 4778 * accessing this cache. This means in this mode we 4779 * don't need to clflush on the CPU side, and on the 4780 * GPU side we only need to flush internal caches to 4781 * get data visible to the CPU. 4782 * 4783 * However, we maintain the display planes as UC, and so 4784 * need to rebind when first used as such. 4785 */ 4786 cache_level = I915_CACHE_LLC; 4787 else 4788 cache_level = I915_CACHE_NONE; 4789 4790 i915_gem_object_set_cache_coherency(obj, cache_level); 4791 4792 trace_i915_gem_object_create(obj); 4793 4794 return obj; 4795 4796 fail: 4797 i915_gem_object_free(obj); 4798 return ERR_PTR(ret); 4799 } 4800 4801 static bool discard_backing_storage(struct drm_i915_gem_object *obj) 4802 { 4803 /* If we are the last user of the backing storage (be it shmemfs 4804 * pages or stolen etc), we know that the pages are going to be 4805 * immediately released. In this case, we can then skip copying 4806 * back the contents from the GPU. 4807 */ 4808 4809 if (obj->mm.madv != I915_MADV_WILLNEED) 4810 return false; 4811 4812 if (obj->base.filp == NULL) 4813 return true; 4814 4815 /* At first glance, this looks racy, but then again so would be 4816 * userspace racing mmap against close. However, the first external 4817 * reference to the filp can only be obtained through the 4818 * i915_gem_mmap_ioctl() which safeguards us against the user 4819 * acquiring such a reference whilst we are in the middle of 4820 * freeing the object. 4821 */ 4822 return atomic_long_read(&obj->base.filp->f_count) == 1; 4823 } 4824 4825 static void __i915_gem_free_objects(struct drm_i915_private *i915, 4826 struct llist_node *freed) 4827 { 4828 struct drm_i915_gem_object *obj, *on; 4829 4830 intel_runtime_pm_get(i915); 4831 llist_for_each_entry_safe(obj, on, freed, freed) { 4832 struct i915_vma *vma, *vn; 4833 4834 trace_i915_gem_object_destroy(obj); 4835 4836 mutex_lock(&i915->drm.struct_mutex); 4837 4838 GEM_BUG_ON(i915_gem_object_is_active(obj)); 4839 list_for_each_entry_safe(vma, vn, 4840 &obj->vma_list, obj_link) { 4841 GEM_BUG_ON(i915_vma_is_active(vma)); 4842 vma->flags &= ~I915_VMA_PIN_MASK; 4843 i915_vma_destroy(vma); 4844 } 4845 GEM_BUG_ON(!list_empty(&obj->vma_list)); 4846 GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree)); 4847 4848 /* This serializes freeing with the shrinker. Since the free 4849 * is delayed, first by RCU then by the workqueue, we want the 4850 * shrinker to be able to free pages of unreferenced objects, 4851 * or else we may oom whilst there are plenty of deferred 4852 * freed objects. 4853 */ 4854 if (i915_gem_object_has_pages(obj)) { 4855 spin_lock(&i915->mm.obj_lock); 4856 list_del_init(&obj->mm.link); 4857 spin_unlock(&i915->mm.obj_lock); 4858 } 4859 4860 mutex_unlock(&i915->drm.struct_mutex); 4861 4862 GEM_BUG_ON(obj->bind_count); 4863 GEM_BUG_ON(obj->userfault_count); 4864 GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits)); 4865 GEM_BUG_ON(!list_empty(&obj->lut_list)); 4866 4867 if (obj->ops->release) 4868 obj->ops->release(obj); 4869 4870 if (WARN_ON(i915_gem_object_has_pinned_pages(obj))) 4871 atomic_set(&obj->mm.pages_pin_count, 0); 4872 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 4873 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 4874 4875 if (obj->base.import_attach) 4876 drm_prime_gem_destroy(&obj->base, NULL); 4877 4878 reservation_object_fini(&obj->__builtin_resv); 4879 drm_gem_object_release(&obj->base); 4880 i915_gem_info_remove_obj(i915, obj->base.size); 4881 4882 kfree(obj->bit_17); 4883 i915_gem_object_free(obj); 4884 4885 GEM_BUG_ON(!atomic_read(&i915->mm.free_count)); 4886 atomic_dec(&i915->mm.free_count); 4887 4888 if (on) 4889 cond_resched(); 4890 } 4891 intel_runtime_pm_put(i915); 4892 } 4893 4894 static void i915_gem_flush_free_objects(struct drm_i915_private *i915) 4895 { 4896 struct llist_node *freed; 4897 4898 /* Free the oldest, most stale object to keep the free_list short */ 4899 freed = NULL; 4900 if (!llist_empty(&i915->mm.free_list)) { /* quick test for hotpath */ 4901 /* Only one consumer of llist_del_first() allowed */ 4902 spin_lock(&i915->mm.free_lock); 4903 freed = llist_del_first(&i915->mm.free_list); 4904 spin_unlock(&i915->mm.free_lock); 4905 } 4906 if (unlikely(freed)) { 4907 freed->next = NULL; 4908 __i915_gem_free_objects(i915, freed); 4909 } 4910 } 4911 4912 static void __i915_gem_free_work(struct work_struct *work) 4913 { 4914 struct drm_i915_private *i915 = 4915 container_of(work, struct drm_i915_private, mm.free_work); 4916 struct llist_node *freed; 4917 4918 /* 4919 * All file-owned VMA should have been released by this point through 4920 * i915_gem_close_object(), or earlier by i915_gem_context_close(). 4921 * However, the object may also be bound into the global GTT (e.g. 4922 * older GPUs without per-process support, or for direct access through 4923 * the GTT either for the user or for scanout). Those VMA still need to 4924 * unbound now. 4925 */ 4926 4927 spin_lock(&i915->mm.free_lock); 4928 while ((freed = llist_del_all(&i915->mm.free_list))) { 4929 spin_unlock(&i915->mm.free_lock); 4930 4931 __i915_gem_free_objects(i915, freed); 4932 if (need_resched()) 4933 return; 4934 4935 spin_lock(&i915->mm.free_lock); 4936 } 4937 spin_unlock(&i915->mm.free_lock); 4938 } 4939 4940 static void __i915_gem_free_object_rcu(struct rcu_head *head) 4941 { 4942 struct drm_i915_gem_object *obj = 4943 container_of(head, typeof(*obj), rcu); 4944 struct drm_i915_private *i915 = to_i915(obj->base.dev); 4945 4946 /* 4947 * Since we require blocking on struct_mutex to unbind the freed 4948 * object from the GPU before releasing resources back to the 4949 * system, we can not do that directly from the RCU callback (which may 4950 * be a softirq context), but must instead then defer that work onto a 4951 * kthread. We use the RCU callback rather than move the freed object 4952 * directly onto the work queue so that we can mix between using the 4953 * worker and performing frees directly from subsequent allocations for 4954 * crude but effective memory throttling. 4955 */ 4956 if (llist_add(&obj->freed, &i915->mm.free_list)) 4957 queue_work(i915->wq, &i915->mm.free_work); 4958 } 4959 4960 void i915_gem_free_object(struct drm_gem_object *gem_obj) 4961 { 4962 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 4963 4964 if (obj->mm.quirked) 4965 __i915_gem_object_unpin_pages(obj); 4966 4967 if (discard_backing_storage(obj)) 4968 obj->mm.madv = I915_MADV_DONTNEED; 4969 4970 /* 4971 * Before we free the object, make sure any pure RCU-only 4972 * read-side critical sections are complete, e.g. 4973 * i915_gem_busy_ioctl(). For the corresponding synchronized 4974 * lookup see i915_gem_object_lookup_rcu(). 4975 */ 4976 atomic_inc(&to_i915(obj->base.dev)->mm.free_count); 4977 call_rcu(&obj->rcu, __i915_gem_free_object_rcu); 4978 } 4979 4980 void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj) 4981 { 4982 lockdep_assert_held(&obj->base.dev->struct_mutex); 4983 4984 if (!i915_gem_object_has_active_reference(obj) && 4985 i915_gem_object_is_active(obj)) 4986 i915_gem_object_set_active_reference(obj); 4987 else 4988 i915_gem_object_put(obj); 4989 } 4990 4991 void i915_gem_sanitize(struct drm_i915_private *i915) 4992 { 4993 int err; 4994 4995 GEM_TRACE("\n"); 4996 4997 mutex_lock(&i915->drm.struct_mutex); 4998 4999 intel_runtime_pm_get(i915); 5000 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL); 5001 5002 /* 5003 * As we have just resumed the machine and woken the device up from 5004 * deep PCI sleep (presumably D3_cold), assume the HW has been reset 5005 * back to defaults, recovering from whatever wedged state we left it 5006 * in and so worth trying to use the device once more. 5007 */ 5008 if (i915_terminally_wedged(&i915->gpu_error)) 5009 i915_gem_unset_wedged(i915); 5010 5011 /* 5012 * If we inherit context state from the BIOS or earlier occupants 5013 * of the GPU, the GPU may be in an inconsistent state when we 5014 * try to take over. The only way to remove the earlier state 5015 * is by resetting. However, resetting on earlier gen is tricky as 5016 * it may impact the display and we are uncertain about the stability 5017 * of the reset, so this could be applied to even earlier gen. 5018 */ 5019 err = -ENODEV; 5020 if (INTEL_GEN(i915) >= 5 && intel_has_gpu_reset(i915)) 5021 err = WARN_ON(intel_gpu_reset(i915, ALL_ENGINES)); 5022 if (!err) 5023 intel_engines_sanitize(i915); 5024 5025 intel_uncore_forcewake_put(i915, FORCEWAKE_ALL); 5026 intel_runtime_pm_put(i915); 5027 5028 i915_gem_contexts_lost(i915); 5029 mutex_unlock(&i915->drm.struct_mutex); 5030 } 5031 5032 int i915_gem_suspend(struct drm_i915_private *dev_priv) 5033 { 5034 struct drm_device *dev = &dev_priv->drm; 5035 int ret; 5036 5037 GEM_TRACE("\n"); 5038 5039 intel_runtime_pm_get(dev_priv); 5040 intel_suspend_gt_powersave(dev_priv); 5041 5042 mutex_lock(&dev->struct_mutex); 5043 5044 /* We have to flush all the executing contexts to main memory so 5045 * that they can saved in the hibernation image. To ensure the last 5046 * context image is coherent, we have to switch away from it. That 5047 * leaves the dev_priv->kernel_context still active when 5048 * we actually suspend, and its image in memory may not match the GPU 5049 * state. Fortunately, the kernel_context is disposable and we do 5050 * not rely on its state. 5051 */ 5052 if (!i915_terminally_wedged(&dev_priv->gpu_error)) { 5053 ret = i915_gem_switch_to_kernel_context(dev_priv); 5054 if (ret) 5055 goto err_unlock; 5056 5057 ret = i915_gem_wait_for_idle(dev_priv, 5058 I915_WAIT_INTERRUPTIBLE | 5059 I915_WAIT_LOCKED | 5060 I915_WAIT_FOR_IDLE_BOOST); 5061 if (ret && ret != -EIO) 5062 goto err_unlock; 5063 5064 assert_kernel_context_is_current(dev_priv); 5065 } 5066 mutex_unlock(&dev->struct_mutex); 5067 5068 intel_uc_suspend(dev_priv); 5069 5070 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 5071 cancel_delayed_work_sync(&dev_priv->gt.retire_work); 5072 5073 /* As the idle_work is rearming if it detects a race, play safe and 5074 * repeat the flush until it is definitely idle. 5075 */ 5076 drain_delayed_work(&dev_priv->gt.idle_work); 5077 5078 /* Assert that we sucessfully flushed all the work and 5079 * reset the GPU back to its idle, low power state. 5080 */ 5081 WARN_ON(dev_priv->gt.awake); 5082 if (WARN_ON(!intel_engines_are_idle(dev_priv))) 5083 i915_gem_set_wedged(dev_priv); /* no hope, discard everything */ 5084 5085 intel_runtime_pm_put(dev_priv); 5086 return 0; 5087 5088 err_unlock: 5089 mutex_unlock(&dev->struct_mutex); 5090 intel_runtime_pm_put(dev_priv); 5091 return ret; 5092 } 5093 5094 void i915_gem_suspend_late(struct drm_i915_private *i915) 5095 { 5096 struct drm_i915_gem_object *obj; 5097 struct list_head *phases[] = { 5098 &i915->mm.unbound_list, 5099 &i915->mm.bound_list, 5100 NULL 5101 }, **phase; 5102 5103 /* 5104 * Neither the BIOS, ourselves or any other kernel 5105 * expects the system to be in execlists mode on startup, 5106 * so we need to reset the GPU back to legacy mode. And the only 5107 * known way to disable logical contexts is through a GPU reset. 5108 * 5109 * So in order to leave the system in a known default configuration, 5110 * always reset the GPU upon unload and suspend. Afterwards we then 5111 * clean up the GEM state tracking, flushing off the requests and 5112 * leaving the system in a known idle state. 5113 * 5114 * Note that is of the upmost importance that the GPU is idle and 5115 * all stray writes are flushed *before* we dismantle the backing 5116 * storage for the pinned objects. 5117 * 5118 * However, since we are uncertain that resetting the GPU on older 5119 * machines is a good idea, we don't - just in case it leaves the 5120 * machine in an unusable condition. 5121 */ 5122 5123 mutex_lock(&i915->drm.struct_mutex); 5124 for (phase = phases; *phase; phase++) { 5125 list_for_each_entry(obj, *phase, mm.link) 5126 WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false)); 5127 } 5128 mutex_unlock(&i915->drm.struct_mutex); 5129 5130 intel_uc_sanitize(i915); 5131 i915_gem_sanitize(i915); 5132 } 5133 5134 void i915_gem_resume(struct drm_i915_private *i915) 5135 { 5136 GEM_TRACE("\n"); 5137 5138 WARN_ON(i915->gt.awake); 5139 5140 mutex_lock(&i915->drm.struct_mutex); 5141 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL); 5142 5143 i915_gem_restore_gtt_mappings(i915); 5144 i915_gem_restore_fences(i915); 5145 5146 /* 5147 * As we didn't flush the kernel context before suspend, we cannot 5148 * guarantee that the context image is complete. So let's just reset 5149 * it and start again. 5150 */ 5151 i915->gt.resume(i915); 5152 5153 if (i915_gem_init_hw(i915)) 5154 goto err_wedged; 5155 5156 intel_uc_resume(i915); 5157 5158 /* Always reload a context for powersaving. */ 5159 if (i915_gem_switch_to_kernel_context(i915)) 5160 goto err_wedged; 5161 5162 out_unlock: 5163 intel_uncore_forcewake_put(i915, FORCEWAKE_ALL); 5164 mutex_unlock(&i915->drm.struct_mutex); 5165 return; 5166 5167 err_wedged: 5168 if (!i915_terminally_wedged(&i915->gpu_error)) { 5169 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); 5170 i915_gem_set_wedged(i915); 5171 } 5172 goto out_unlock; 5173 } 5174 5175 void i915_gem_init_swizzling(struct drm_i915_private *dev_priv) 5176 { 5177 if (INTEL_GEN(dev_priv) < 5 || 5178 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 5179 return; 5180 5181 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 5182 DISP_TILE_SURFACE_SWIZZLING); 5183 5184 if (IS_GEN5(dev_priv)) 5185 return; 5186 5187 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 5188 if (IS_GEN6(dev_priv)) 5189 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 5190 else if (IS_GEN7(dev_priv)) 5191 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 5192 else if (IS_GEN8(dev_priv)) 5193 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); 5194 else 5195 BUG(); 5196 } 5197 5198 static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base) 5199 { 5200 I915_WRITE(RING_CTL(base), 0); 5201 I915_WRITE(RING_HEAD(base), 0); 5202 I915_WRITE(RING_TAIL(base), 0); 5203 I915_WRITE(RING_START(base), 0); 5204 } 5205 5206 static void init_unused_rings(struct drm_i915_private *dev_priv) 5207 { 5208 if (IS_I830(dev_priv)) { 5209 init_unused_ring(dev_priv, PRB1_BASE); 5210 init_unused_ring(dev_priv, SRB0_BASE); 5211 init_unused_ring(dev_priv, SRB1_BASE); 5212 init_unused_ring(dev_priv, SRB2_BASE); 5213 init_unused_ring(dev_priv, SRB3_BASE); 5214 } else if (IS_GEN2(dev_priv)) { 5215 init_unused_ring(dev_priv, SRB0_BASE); 5216 init_unused_ring(dev_priv, SRB1_BASE); 5217 } else if (IS_GEN3(dev_priv)) { 5218 init_unused_ring(dev_priv, PRB1_BASE); 5219 init_unused_ring(dev_priv, PRB2_BASE); 5220 } 5221 } 5222 5223 static int __i915_gem_restart_engines(void *data) 5224 { 5225 struct drm_i915_private *i915 = data; 5226 struct intel_engine_cs *engine; 5227 enum intel_engine_id id; 5228 int err; 5229 5230 for_each_engine(engine, i915, id) { 5231 err = engine->init_hw(engine); 5232 if (err) { 5233 DRM_ERROR("Failed to restart %s (%d)\n", 5234 engine->name, err); 5235 return err; 5236 } 5237 } 5238 5239 return 0; 5240 } 5241 5242 int i915_gem_init_hw(struct drm_i915_private *dev_priv) 5243 { 5244 int ret; 5245 5246 dev_priv->gt.last_init_time = ktime_get(); 5247 5248 /* Double layer security blanket, see i915_gem_init() */ 5249 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5250 5251 if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9) 5252 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); 5253 5254 if (IS_HASWELL(dev_priv)) 5255 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ? 5256 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); 5257 5258 if (HAS_PCH_NOP(dev_priv)) { 5259 if (IS_IVYBRIDGE(dev_priv)) { 5260 u32 temp = I915_READ(GEN7_MSG_CTL); 5261 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); 5262 I915_WRITE(GEN7_MSG_CTL, temp); 5263 } else if (INTEL_GEN(dev_priv) >= 7) { 5264 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT); 5265 temp &= ~RESET_PCH_HANDSHAKE_ENABLE; 5266 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp); 5267 } 5268 } 5269 5270 intel_gt_workarounds_apply(dev_priv); 5271 5272 i915_gem_init_swizzling(dev_priv); 5273 5274 /* 5275 * At least 830 can leave some of the unused rings 5276 * "active" (ie. head != tail) after resume which 5277 * will prevent c3 entry. Makes sure all unused rings 5278 * are totally idle. 5279 */ 5280 init_unused_rings(dev_priv); 5281 5282 BUG_ON(!dev_priv->kernel_context); 5283 if (i915_terminally_wedged(&dev_priv->gpu_error)) { 5284 ret = -EIO; 5285 goto out; 5286 } 5287 5288 ret = i915_ppgtt_init_hw(dev_priv); 5289 if (ret) { 5290 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret); 5291 goto out; 5292 } 5293 5294 ret = intel_wopcm_init_hw(&dev_priv->wopcm); 5295 if (ret) { 5296 DRM_ERROR("Enabling WOPCM failed (%d)\n", ret); 5297 goto out; 5298 } 5299 5300 /* We can't enable contexts until all firmware is loaded */ 5301 ret = intel_uc_init_hw(dev_priv); 5302 if (ret) { 5303 DRM_ERROR("Enabling uc failed (%d)\n", ret); 5304 goto out; 5305 } 5306 5307 intel_mocs_init_l3cc_table(dev_priv); 5308 5309 /* Only when the HW is re-initialised, can we replay the requests */ 5310 ret = __i915_gem_restart_engines(dev_priv); 5311 if (ret) 5312 goto cleanup_uc; 5313 out: 5314 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5315 return ret; 5316 5317 cleanup_uc: 5318 intel_uc_fini_hw(dev_priv); 5319 goto out; 5320 } 5321 5322 static int __intel_engines_record_defaults(struct drm_i915_private *i915) 5323 { 5324 struct i915_gem_context *ctx; 5325 struct intel_engine_cs *engine; 5326 enum intel_engine_id id; 5327 int err; 5328 5329 /* 5330 * As we reset the gpu during very early sanitisation, the current 5331 * register state on the GPU should reflect its defaults values. 5332 * We load a context onto the hw (with restore-inhibit), then switch 5333 * over to a second context to save that default register state. We 5334 * can then prime every new context with that state so they all start 5335 * from the same default HW values. 5336 */ 5337 5338 ctx = i915_gem_context_create_kernel(i915, 0); 5339 if (IS_ERR(ctx)) 5340 return PTR_ERR(ctx); 5341 5342 for_each_engine(engine, i915, id) { 5343 struct i915_request *rq; 5344 5345 rq = i915_request_alloc(engine, ctx); 5346 if (IS_ERR(rq)) { 5347 err = PTR_ERR(rq); 5348 goto out_ctx; 5349 } 5350 5351 err = 0; 5352 if (engine->init_context) 5353 err = engine->init_context(rq); 5354 5355 i915_request_add(rq); 5356 if (err) 5357 goto err_active; 5358 } 5359 5360 err = i915_gem_switch_to_kernel_context(i915); 5361 if (err) 5362 goto err_active; 5363 5364 err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED); 5365 if (err) 5366 goto err_active; 5367 5368 assert_kernel_context_is_current(i915); 5369 5370 for_each_engine(engine, i915, id) { 5371 struct i915_vma *state; 5372 5373 state = to_intel_context(ctx, engine)->state; 5374 if (!state) 5375 continue; 5376 5377 /* 5378 * As we will hold a reference to the logical state, it will 5379 * not be torn down with the context, and importantly the 5380 * object will hold onto its vma (making it possible for a 5381 * stray GTT write to corrupt our defaults). Unmap the vma 5382 * from the GTT to prevent such accidents and reclaim the 5383 * space. 5384 */ 5385 err = i915_vma_unbind(state); 5386 if (err) 5387 goto err_active; 5388 5389 err = i915_gem_object_set_to_cpu_domain(state->obj, false); 5390 if (err) 5391 goto err_active; 5392 5393 engine->default_state = i915_gem_object_get(state->obj); 5394 } 5395 5396 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) { 5397 unsigned int found = intel_engines_has_context_isolation(i915); 5398 5399 /* 5400 * Make sure that classes with multiple engine instances all 5401 * share the same basic configuration. 5402 */ 5403 for_each_engine(engine, i915, id) { 5404 unsigned int bit = BIT(engine->uabi_class); 5405 unsigned int expected = engine->default_state ? bit : 0; 5406 5407 if ((found & bit) != expected) { 5408 DRM_ERROR("mismatching default context state for class %d on engine %s\n", 5409 engine->uabi_class, engine->name); 5410 } 5411 } 5412 } 5413 5414 out_ctx: 5415 i915_gem_context_set_closed(ctx); 5416 i915_gem_context_put(ctx); 5417 return err; 5418 5419 err_active: 5420 /* 5421 * If we have to abandon now, we expect the engines to be idle 5422 * and ready to be torn-down. First try to flush any remaining 5423 * request, ensure we are pointing at the kernel context and 5424 * then remove it. 5425 */ 5426 if (WARN_ON(i915_gem_switch_to_kernel_context(i915))) 5427 goto out_ctx; 5428 5429 if (WARN_ON(i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED))) 5430 goto out_ctx; 5431 5432 i915_gem_contexts_lost(i915); 5433 goto out_ctx; 5434 } 5435 5436 int i915_gem_init(struct drm_i915_private *dev_priv) 5437 { 5438 int ret; 5439 5440 /* We need to fallback to 4K pages if host doesn't support huge gtt. */ 5441 if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv)) 5442 mkwrite_device_info(dev_priv)->page_sizes = 5443 I915_GTT_PAGE_SIZE_4K; 5444 5445 dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1); 5446 5447 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) { 5448 dev_priv->gt.resume = intel_lr_context_resume; 5449 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup; 5450 } else { 5451 dev_priv->gt.resume = intel_legacy_submission_resume; 5452 dev_priv->gt.cleanup_engine = intel_engine_cleanup; 5453 } 5454 5455 ret = i915_gem_init_userptr(dev_priv); 5456 if (ret) 5457 return ret; 5458 5459 ret = intel_wopcm_init(&dev_priv->wopcm); 5460 if (ret) 5461 return ret; 5462 5463 ret = intel_uc_init_misc(dev_priv); 5464 if (ret) 5465 return ret; 5466 5467 /* This is just a security blanket to placate dragons. 5468 * On some systems, we very sporadically observe that the first TLBs 5469 * used by the CS may be stale, despite us poking the TLB reset. If 5470 * we hold the forcewake during initialisation these problems 5471 * just magically go away. 5472 */ 5473 mutex_lock(&dev_priv->drm.struct_mutex); 5474 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5475 5476 ret = i915_gem_init_ggtt(dev_priv); 5477 if (ret) { 5478 GEM_BUG_ON(ret == -EIO); 5479 goto err_unlock; 5480 } 5481 5482 ret = i915_gem_contexts_init(dev_priv); 5483 if (ret) { 5484 GEM_BUG_ON(ret == -EIO); 5485 goto err_ggtt; 5486 } 5487 5488 ret = intel_engines_init(dev_priv); 5489 if (ret) { 5490 GEM_BUG_ON(ret == -EIO); 5491 goto err_context; 5492 } 5493 5494 intel_init_gt_powersave(dev_priv); 5495 5496 ret = intel_uc_init(dev_priv); 5497 if (ret) 5498 goto err_pm; 5499 5500 ret = i915_gem_init_hw(dev_priv); 5501 if (ret) 5502 goto err_uc_init; 5503 5504 /* 5505 * Despite its name intel_init_clock_gating applies both display 5506 * clock gating workarounds; GT mmio workarounds and the occasional 5507 * GT power context workaround. Worse, sometimes it includes a context 5508 * register workaround which we need to apply before we record the 5509 * default HW state for all contexts. 5510 * 5511 * FIXME: break up the workarounds and apply them at the right time! 5512 */ 5513 intel_init_clock_gating(dev_priv); 5514 5515 ret = __intel_engines_record_defaults(dev_priv); 5516 if (ret) 5517 goto err_init_hw; 5518 5519 if (i915_inject_load_failure()) { 5520 ret = -ENODEV; 5521 goto err_init_hw; 5522 } 5523 5524 if (i915_inject_load_failure()) { 5525 ret = -EIO; 5526 goto err_init_hw; 5527 } 5528 5529 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5530 mutex_unlock(&dev_priv->drm.struct_mutex); 5531 5532 return 0; 5533 5534 /* 5535 * Unwinding is complicated by that we want to handle -EIO to mean 5536 * disable GPU submission but keep KMS alive. We want to mark the 5537 * HW as irrevisibly wedged, but keep enough state around that the 5538 * driver doesn't explode during runtime. 5539 */ 5540 err_init_hw: 5541 mutex_unlock(&dev_priv->drm.struct_mutex); 5542 5543 WARN_ON(i915_gem_suspend(dev_priv)); 5544 i915_gem_suspend_late(dev_priv); 5545 5546 mutex_lock(&dev_priv->drm.struct_mutex); 5547 intel_uc_fini_hw(dev_priv); 5548 err_uc_init: 5549 intel_uc_fini(dev_priv); 5550 err_pm: 5551 if (ret != -EIO) { 5552 intel_cleanup_gt_powersave(dev_priv); 5553 i915_gem_cleanup_engines(dev_priv); 5554 } 5555 err_context: 5556 if (ret != -EIO) 5557 i915_gem_contexts_fini(dev_priv); 5558 err_ggtt: 5559 err_unlock: 5560 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5561 mutex_unlock(&dev_priv->drm.struct_mutex); 5562 5563 intel_uc_fini_misc(dev_priv); 5564 5565 if (ret != -EIO) 5566 i915_gem_cleanup_userptr(dev_priv); 5567 5568 if (ret == -EIO) { 5569 /* 5570 * Allow engine initialisation to fail by marking the GPU as 5571 * wedged. But we only want to do this where the GPU is angry, 5572 * for all other failure, such as an allocation failure, bail. 5573 */ 5574 if (!i915_terminally_wedged(&dev_priv->gpu_error)) { 5575 i915_load_error(dev_priv, 5576 "Failed to initialize GPU, declaring it wedged!\n"); 5577 i915_gem_set_wedged(dev_priv); 5578 } 5579 ret = 0; 5580 } 5581 5582 i915_gem_drain_freed_objects(dev_priv); 5583 return ret; 5584 } 5585 5586 void i915_gem_fini(struct drm_i915_private *dev_priv) 5587 { 5588 i915_gem_suspend_late(dev_priv); 5589 5590 /* Flush any outstanding unpin_work. */ 5591 i915_gem_drain_workqueue(dev_priv); 5592 5593 mutex_lock(&dev_priv->drm.struct_mutex); 5594 intel_uc_fini_hw(dev_priv); 5595 intel_uc_fini(dev_priv); 5596 i915_gem_cleanup_engines(dev_priv); 5597 i915_gem_contexts_fini(dev_priv); 5598 mutex_unlock(&dev_priv->drm.struct_mutex); 5599 5600 intel_uc_fini_misc(dev_priv); 5601 i915_gem_cleanup_userptr(dev_priv); 5602 5603 i915_gem_drain_freed_objects(dev_priv); 5604 5605 WARN_ON(!list_empty(&dev_priv->contexts.list)); 5606 } 5607 5608 void i915_gem_init_mmio(struct drm_i915_private *i915) 5609 { 5610 i915_gem_sanitize(i915); 5611 } 5612 5613 void 5614 i915_gem_cleanup_engines(struct drm_i915_private *dev_priv) 5615 { 5616 struct intel_engine_cs *engine; 5617 enum intel_engine_id id; 5618 5619 for_each_engine(engine, dev_priv, id) 5620 dev_priv->gt.cleanup_engine(engine); 5621 } 5622 5623 void 5624 i915_gem_load_init_fences(struct drm_i915_private *dev_priv) 5625 { 5626 int i; 5627 5628 if (INTEL_GEN(dev_priv) >= 7 && !IS_VALLEYVIEW(dev_priv) && 5629 !IS_CHERRYVIEW(dev_priv)) 5630 dev_priv->num_fence_regs = 32; 5631 else if (INTEL_GEN(dev_priv) >= 4 || 5632 IS_I945G(dev_priv) || IS_I945GM(dev_priv) || 5633 IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) 5634 dev_priv->num_fence_regs = 16; 5635 else 5636 dev_priv->num_fence_regs = 8; 5637 5638 if (intel_vgpu_active(dev_priv)) 5639 dev_priv->num_fence_regs = 5640 I915_READ(vgtif_reg(avail_rs.fence_num)); 5641 5642 /* Initialize fence registers to zero */ 5643 for (i = 0; i < dev_priv->num_fence_regs; i++) { 5644 struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i]; 5645 5646 fence->i915 = dev_priv; 5647 fence->id = i; 5648 list_add_tail(&fence->link, &dev_priv->mm.fence_list); 5649 } 5650 i915_gem_restore_fences(dev_priv); 5651 5652 i915_gem_detect_bit_6_swizzle(dev_priv); 5653 } 5654 5655 static void i915_gem_init__mm(struct drm_i915_private *i915) 5656 { 5657 spin_lock_init(&i915->mm.object_stat_lock); 5658 spin_lock_init(&i915->mm.obj_lock); 5659 spin_lock_init(&i915->mm.free_lock); 5660 5661 init_llist_head(&i915->mm.free_list); 5662 5663 INIT_LIST_HEAD(&i915->mm.unbound_list); 5664 INIT_LIST_HEAD(&i915->mm.bound_list); 5665 INIT_LIST_HEAD(&i915->mm.fence_list); 5666 INIT_LIST_HEAD(&i915->mm.userfault_list); 5667 5668 INIT_WORK(&i915->mm.free_work, __i915_gem_free_work); 5669 } 5670 5671 int i915_gem_init_early(struct drm_i915_private *dev_priv) 5672 { 5673 int err = -ENOMEM; 5674 5675 dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); 5676 if (!dev_priv->objects) 5677 goto err_out; 5678 5679 dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); 5680 if (!dev_priv->vmas) 5681 goto err_objects; 5682 5683 dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0); 5684 if (!dev_priv->luts) 5685 goto err_vmas; 5686 5687 dev_priv->requests = KMEM_CACHE(i915_request, 5688 SLAB_HWCACHE_ALIGN | 5689 SLAB_RECLAIM_ACCOUNT | 5690 SLAB_TYPESAFE_BY_RCU); 5691 if (!dev_priv->requests) 5692 goto err_luts; 5693 5694 dev_priv->dependencies = KMEM_CACHE(i915_dependency, 5695 SLAB_HWCACHE_ALIGN | 5696 SLAB_RECLAIM_ACCOUNT); 5697 if (!dev_priv->dependencies) 5698 goto err_requests; 5699 5700 dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN); 5701 if (!dev_priv->priorities) 5702 goto err_dependencies; 5703 5704 INIT_LIST_HEAD(&dev_priv->gt.timelines); 5705 INIT_LIST_HEAD(&dev_priv->gt.active_rings); 5706 INIT_LIST_HEAD(&dev_priv->gt.closed_vma); 5707 5708 i915_gem_init__mm(dev_priv); 5709 5710 INIT_DELAYED_WORK(&dev_priv->gt.retire_work, 5711 i915_gem_retire_work_handler); 5712 INIT_DELAYED_WORK(&dev_priv->gt.idle_work, 5713 i915_gem_idle_work_handler); 5714 init_waitqueue_head(&dev_priv->gpu_error.wait_queue); 5715 init_waitqueue_head(&dev_priv->gpu_error.reset_queue); 5716 5717 atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0); 5718 5719 spin_lock_init(&dev_priv->fb_tracking.lock); 5720 5721 err = i915_gemfs_init(dev_priv); 5722 if (err) 5723 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err); 5724 5725 return 0; 5726 5727 err_dependencies: 5728 kmem_cache_destroy(dev_priv->dependencies); 5729 err_requests: 5730 kmem_cache_destroy(dev_priv->requests); 5731 err_luts: 5732 kmem_cache_destroy(dev_priv->luts); 5733 err_vmas: 5734 kmem_cache_destroy(dev_priv->vmas); 5735 err_objects: 5736 kmem_cache_destroy(dev_priv->objects); 5737 err_out: 5738 return err; 5739 } 5740 5741 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv) 5742 { 5743 i915_gem_drain_freed_objects(dev_priv); 5744 GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list)); 5745 GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count)); 5746 WARN_ON(dev_priv->mm.object_count); 5747 WARN_ON(!list_empty(&dev_priv->gt.timelines)); 5748 5749 kmem_cache_destroy(dev_priv->priorities); 5750 kmem_cache_destroy(dev_priv->dependencies); 5751 kmem_cache_destroy(dev_priv->requests); 5752 kmem_cache_destroy(dev_priv->luts); 5753 kmem_cache_destroy(dev_priv->vmas); 5754 kmem_cache_destroy(dev_priv->objects); 5755 5756 /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */ 5757 rcu_barrier(); 5758 5759 i915_gemfs_fini(dev_priv); 5760 } 5761 5762 int i915_gem_freeze(struct drm_i915_private *dev_priv) 5763 { 5764 /* Discard all purgeable objects, let userspace recover those as 5765 * required after resuming. 5766 */ 5767 i915_gem_shrink_all(dev_priv); 5768 5769 return 0; 5770 } 5771 5772 int i915_gem_freeze_late(struct drm_i915_private *i915) 5773 { 5774 struct drm_i915_gem_object *obj; 5775 struct list_head *phases[] = { 5776 &i915->mm.unbound_list, 5777 &i915->mm.bound_list, 5778 NULL 5779 }, **phase; 5780 5781 /* 5782 * Called just before we write the hibernation image. 5783 * 5784 * We need to update the domain tracking to reflect that the CPU 5785 * will be accessing all the pages to create and restore from the 5786 * hibernation, and so upon restoration those pages will be in the 5787 * CPU domain. 5788 * 5789 * To make sure the hibernation image contains the latest state, 5790 * we update that state just before writing out the image. 5791 * 5792 * To try and reduce the hibernation image, we manually shrink 5793 * the objects as well, see i915_gem_freeze() 5794 */ 5795 5796 i915_gem_shrink(i915, -1UL, NULL, I915_SHRINK_UNBOUND); 5797 i915_gem_drain_freed_objects(i915); 5798 5799 mutex_lock(&i915->drm.struct_mutex); 5800 for (phase = phases; *phase; phase++) { 5801 list_for_each_entry(obj, *phase, mm.link) 5802 WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true)); 5803 } 5804 mutex_unlock(&i915->drm.struct_mutex); 5805 5806 return 0; 5807 } 5808 5809 void i915_gem_release(struct drm_device *dev, struct drm_file *file) 5810 { 5811 struct drm_i915_file_private *file_priv = file->driver_priv; 5812 struct i915_request *request; 5813 5814 /* Clean up our request list when the client is going away, so that 5815 * later retire_requests won't dereference our soon-to-be-gone 5816 * file_priv. 5817 */ 5818 spin_lock(&file_priv->mm.lock); 5819 list_for_each_entry(request, &file_priv->mm.request_list, client_link) 5820 request->file_priv = NULL; 5821 spin_unlock(&file_priv->mm.lock); 5822 } 5823 5824 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) 5825 { 5826 struct drm_i915_file_private *file_priv; 5827 int ret; 5828 5829 DRM_DEBUG("\n"); 5830 5831 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); 5832 if (!file_priv) 5833 return -ENOMEM; 5834 5835 file->driver_priv = file_priv; 5836 file_priv->dev_priv = i915; 5837 file_priv->file = file; 5838 5839 spin_lock_init(&file_priv->mm.lock); 5840 INIT_LIST_HEAD(&file_priv->mm.request_list); 5841 5842 file_priv->bsd_engine = -1; 5843 file_priv->hang_timestamp = jiffies; 5844 5845 ret = i915_gem_context_open(i915, file); 5846 if (ret) 5847 kfree(file_priv); 5848 5849 return ret; 5850 } 5851 5852 /** 5853 * i915_gem_track_fb - update frontbuffer tracking 5854 * @old: current GEM buffer for the frontbuffer slots 5855 * @new: new GEM buffer for the frontbuffer slots 5856 * @frontbuffer_bits: bitmask of frontbuffer slots 5857 * 5858 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them 5859 * from @old and setting them in @new. Both @old and @new can be NULL. 5860 */ 5861 void i915_gem_track_fb(struct drm_i915_gem_object *old, 5862 struct drm_i915_gem_object *new, 5863 unsigned frontbuffer_bits) 5864 { 5865 /* Control of individual bits within the mask are guarded by 5866 * the owning plane->mutex, i.e. we can never see concurrent 5867 * manipulation of individual bits. But since the bitfield as a whole 5868 * is updated using RMW, we need to use atomics in order to update 5869 * the bits. 5870 */ 5871 BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > 5872 sizeof(atomic_t) * BITS_PER_BYTE); 5873 5874 if (old) { 5875 WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits)); 5876 atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits); 5877 } 5878 5879 if (new) { 5880 WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits); 5881 atomic_or(frontbuffer_bits, &new->frontbuffer_bits); 5882 } 5883 } 5884 5885 /* Allocate a new GEM object and fill it with the supplied data */ 5886 struct drm_i915_gem_object * 5887 i915_gem_object_create_from_data(struct drm_i915_private *dev_priv, 5888 const void *data, size_t size) 5889 { 5890 struct drm_i915_gem_object *obj; 5891 struct file *file; 5892 size_t offset; 5893 int err; 5894 5895 obj = i915_gem_object_create(dev_priv, round_up(size, PAGE_SIZE)); 5896 if (IS_ERR(obj)) 5897 return obj; 5898 5899 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); 5900 5901 file = obj->base.filp; 5902 offset = 0; 5903 do { 5904 unsigned int len = min_t(typeof(size), size, PAGE_SIZE); 5905 struct page *page; 5906 void *pgdata, *vaddr; 5907 5908 err = pagecache_write_begin(file, file->f_mapping, 5909 offset, len, 0, 5910 &page, &pgdata); 5911 if (err < 0) 5912 goto fail; 5913 5914 vaddr = kmap(page); 5915 memcpy(vaddr, data, len); 5916 kunmap(page); 5917 5918 err = pagecache_write_end(file, file->f_mapping, 5919 offset, len, len, 5920 page, pgdata); 5921 if (err < 0) 5922 goto fail; 5923 5924 size -= len; 5925 data += len; 5926 offset += len; 5927 } while (size); 5928 5929 return obj; 5930 5931 fail: 5932 i915_gem_object_put(obj); 5933 return ERR_PTR(err); 5934 } 5935 5936 struct scatterlist * 5937 i915_gem_object_get_sg(struct drm_i915_gem_object *obj, 5938 unsigned int n, 5939 unsigned int *offset) 5940 { 5941 struct i915_gem_object_page_iter *iter = &obj->mm.get_page; 5942 struct scatterlist *sg; 5943 unsigned int idx, count; 5944 5945 might_sleep(); 5946 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); 5947 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 5948 5949 /* As we iterate forward through the sg, we record each entry in a 5950 * radixtree for quick repeated (backwards) lookups. If we have seen 5951 * this index previously, we will have an entry for it. 5952 * 5953 * Initial lookup is O(N), but this is amortized to O(1) for 5954 * sequential page access (where each new request is consecutive 5955 * to the previous one). Repeated lookups are O(lg(obj->base.size)), 5956 * i.e. O(1) with a large constant! 5957 */ 5958 if (n < READ_ONCE(iter->sg_idx)) 5959 goto lookup; 5960 5961 mutex_lock(&iter->lock); 5962 5963 /* We prefer to reuse the last sg so that repeated lookup of this 5964 * (or the subsequent) sg are fast - comparing against the last 5965 * sg is faster than going through the radixtree. 5966 */ 5967 5968 sg = iter->sg_pos; 5969 idx = iter->sg_idx; 5970 count = __sg_page_count(sg); 5971 5972 while (idx + count <= n) { 5973 unsigned long exception, i; 5974 int ret; 5975 5976 /* If we cannot allocate and insert this entry, or the 5977 * individual pages from this range, cancel updating the 5978 * sg_idx so that on this lookup we are forced to linearly 5979 * scan onwards, but on future lookups we will try the 5980 * insertion again (in which case we need to be careful of 5981 * the error return reporting that we have already inserted 5982 * this index). 5983 */ 5984 ret = radix_tree_insert(&iter->radix, idx, sg); 5985 if (ret && ret != -EEXIST) 5986 goto scan; 5987 5988 exception = 5989 RADIX_TREE_EXCEPTIONAL_ENTRY | 5990 idx << RADIX_TREE_EXCEPTIONAL_SHIFT; 5991 for (i = 1; i < count; i++) { 5992 ret = radix_tree_insert(&iter->radix, idx + i, 5993 (void *)exception); 5994 if (ret && ret != -EEXIST) 5995 goto scan; 5996 } 5997 5998 idx += count; 5999 sg = ____sg_next(sg); 6000 count = __sg_page_count(sg); 6001 } 6002 6003 scan: 6004 iter->sg_pos = sg; 6005 iter->sg_idx = idx; 6006 6007 mutex_unlock(&iter->lock); 6008 6009 if (unlikely(n < idx)) /* insertion completed by another thread */ 6010 goto lookup; 6011 6012 /* In case we failed to insert the entry into the radixtree, we need 6013 * to look beyond the current sg. 6014 */ 6015 while (idx + count <= n) { 6016 idx += count; 6017 sg = ____sg_next(sg); 6018 count = __sg_page_count(sg); 6019 } 6020 6021 *offset = n - idx; 6022 return sg; 6023 6024 lookup: 6025 rcu_read_lock(); 6026 6027 sg = radix_tree_lookup(&iter->radix, n); 6028 GEM_BUG_ON(!sg); 6029 6030 /* If this index is in the middle of multi-page sg entry, 6031 * the radixtree will contain an exceptional entry that points 6032 * to the start of that range. We will return the pointer to 6033 * the base page and the offset of this page within the 6034 * sg entry's range. 6035 */ 6036 *offset = 0; 6037 if (unlikely(radix_tree_exception(sg))) { 6038 unsigned long base = 6039 (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT; 6040 6041 sg = radix_tree_lookup(&iter->radix, base); 6042 GEM_BUG_ON(!sg); 6043 6044 *offset = n - base; 6045 } 6046 6047 rcu_read_unlock(); 6048 6049 return sg; 6050 } 6051 6052 struct page * 6053 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) 6054 { 6055 struct scatterlist *sg; 6056 unsigned int offset; 6057 6058 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); 6059 6060 sg = i915_gem_object_get_sg(obj, n, &offset); 6061 return nth_page(sg_page(sg), offset); 6062 } 6063 6064 /* Like i915_gem_object_get_page(), but mark the returned page dirty */ 6065 struct page * 6066 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, 6067 unsigned int n) 6068 { 6069 struct page *page; 6070 6071 page = i915_gem_object_get_page(obj, n); 6072 if (!obj->mm.dirty) 6073 set_page_dirty(page); 6074 6075 return page; 6076 } 6077 6078 dma_addr_t 6079 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, 6080 unsigned long n) 6081 { 6082 struct scatterlist *sg; 6083 unsigned int offset; 6084 6085 sg = i915_gem_object_get_sg(obj, n, &offset); 6086 return sg_dma_address(sg) + (offset << PAGE_SHIFT); 6087 } 6088 6089 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) 6090 { 6091 struct sg_table *pages; 6092 int err; 6093 6094 if (align > obj->base.size) 6095 return -EINVAL; 6096 6097 if (obj->ops == &i915_gem_phys_ops) 6098 return 0; 6099 6100 if (obj->ops != &i915_gem_object_ops) 6101 return -EINVAL; 6102 6103 err = i915_gem_object_unbind(obj); 6104 if (err) 6105 return err; 6106 6107 mutex_lock(&obj->mm.lock); 6108 6109 if (obj->mm.madv != I915_MADV_WILLNEED) { 6110 err = -EFAULT; 6111 goto err_unlock; 6112 } 6113 6114 if (obj->mm.quirked) { 6115 err = -EFAULT; 6116 goto err_unlock; 6117 } 6118 6119 if (obj->mm.mapping) { 6120 err = -EBUSY; 6121 goto err_unlock; 6122 } 6123 6124 pages = __i915_gem_object_unset_pages(obj); 6125 6126 obj->ops = &i915_gem_phys_ops; 6127 6128 err = ____i915_gem_object_get_pages(obj); 6129 if (err) 6130 goto err_xfer; 6131 6132 /* Perma-pin (until release) the physical set of pages */ 6133 __i915_gem_object_pin_pages(obj); 6134 6135 if (!IS_ERR_OR_NULL(pages)) 6136 i915_gem_object_ops.put_pages(obj, pages); 6137 mutex_unlock(&obj->mm.lock); 6138 return 0; 6139 6140 err_xfer: 6141 obj->ops = &i915_gem_object_ops; 6142 if (!IS_ERR_OR_NULL(pages)) { 6143 unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl); 6144 6145 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 6146 } 6147 err_unlock: 6148 mutex_unlock(&obj->mm.lock); 6149 return err; 6150 } 6151 6152 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 6153 #include "selftests/scatterlist.c" 6154 #include "selftests/mock_gem_device.c" 6155 #include "selftests/huge_gem_object.c" 6156 #include "selftests/huge_pages.c" 6157 #include "selftests/i915_gem_object.c" 6158 #include "selftests/i915_gem_coherency.c" 6159 #endif 6160