1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include <drm/drm_vma_manager.h> 29 #include <drm/i915_drm.h> 30 #include <linux/dma-fence-array.h> 31 #include <linux/kthread.h> 32 #include <linux/dma-resv.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/slab.h> 35 #include <linux/stop_machine.h> 36 #include <linux/swap.h> 37 #include <linux/pci.h> 38 #include <linux/dma-buf.h> 39 #include <linux/mman.h> 40 41 #include "display/intel_display.h" 42 #include "display/intel_frontbuffer.h" 43 44 #include "gem/i915_gem_clflush.h" 45 #include "gem/i915_gem_context.h" 46 #include "gem/i915_gem_ioctls.h" 47 #include "gem/i915_gem_pm.h" 48 #include "gt/intel_context.h" 49 #include "gt/intel_engine_user.h" 50 #include "gt/intel_gt.h" 51 #include "gt/intel_gt_pm.h" 52 #include "gt/intel_gt_requests.h" 53 #include "gt/intel_mocs.h" 54 #include "gt/intel_reset.h" 55 #include "gt/intel_renderstate.h" 56 #include "gt/intel_rps.h" 57 #include "gt/intel_workarounds.h" 58 59 #include "i915_drv.h" 60 #include "i915_scatterlist.h" 61 #include "i915_trace.h" 62 #include "i915_vgpu.h" 63 64 #include "intel_pm.h" 65 66 static int 67 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size) 68 { 69 int err; 70 71 err = mutex_lock_interruptible(&ggtt->vm.mutex); 72 if (err) 73 return err; 74 75 memset(node, 0, sizeof(*node)); 76 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node, 77 size, 0, I915_COLOR_UNEVICTABLE, 78 0, ggtt->mappable_end, 79 DRM_MM_INSERT_LOW); 80 81 mutex_unlock(&ggtt->vm.mutex); 82 83 return err; 84 } 85 86 static void 87 remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node) 88 { 89 mutex_lock(&ggtt->vm.mutex); 90 drm_mm_remove_node(node); 91 mutex_unlock(&ggtt->vm.mutex); 92 } 93 94 int 95 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 96 struct drm_file *file) 97 { 98 struct i915_ggtt *ggtt = &to_i915(dev)->ggtt; 99 struct drm_i915_gem_get_aperture *args = data; 100 struct i915_vma *vma; 101 u64 pinned; 102 103 if (mutex_lock_interruptible(&ggtt->vm.mutex)) 104 return -EINTR; 105 106 pinned = ggtt->vm.reserved; 107 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) 108 if (i915_vma_is_pinned(vma)) 109 pinned += vma->node.size; 110 111 mutex_unlock(&ggtt->vm.mutex); 112 113 args->aper_size = ggtt->vm.total; 114 args->aper_available_size = args->aper_size - pinned; 115 116 return 0; 117 } 118 119 int i915_gem_object_unbind(struct drm_i915_gem_object *obj, 120 unsigned long flags) 121 { 122 struct i915_vma *vma; 123 LIST_HEAD(still_in_list); 124 int ret = 0; 125 126 spin_lock(&obj->vma.lock); 127 while (!ret && (vma = list_first_entry_or_null(&obj->vma.list, 128 struct i915_vma, 129 obj_link))) { 130 struct i915_address_space *vm = vma->vm; 131 132 ret = -EBUSY; 133 if (!i915_vm_tryopen(vm)) 134 break; 135 136 list_move_tail(&vma->obj_link, &still_in_list); 137 spin_unlock(&obj->vma.lock); 138 139 if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE || 140 !i915_vma_is_active(vma)) 141 ret = i915_vma_unbind(vma); 142 143 i915_vm_close(vm); 144 spin_lock(&obj->vma.lock); 145 } 146 list_splice(&still_in_list, &obj->vma.list); 147 spin_unlock(&obj->vma.lock); 148 149 return ret; 150 } 151 152 static int 153 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, 154 struct drm_i915_gem_pwrite *args, 155 struct drm_file *file) 156 { 157 void *vaddr = obj->phys_handle->vaddr + args->offset; 158 char __user *user_data = u64_to_user_ptr(args->data_ptr); 159 160 /* 161 * We manually control the domain here and pretend that it 162 * remains coherent i.e. in the GTT domain, like shmem_pwrite. 163 */ 164 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); 165 166 if (copy_from_user(vaddr, user_data, args->size)) 167 return -EFAULT; 168 169 drm_clflush_virt_range(vaddr, args->size); 170 intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt); 171 172 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); 173 return 0; 174 } 175 176 static int 177 i915_gem_create(struct drm_file *file, 178 struct drm_i915_private *dev_priv, 179 u64 *size_p, 180 u32 *handle_p) 181 { 182 struct drm_i915_gem_object *obj; 183 u32 handle; 184 u64 size; 185 int ret; 186 187 size = round_up(*size_p, PAGE_SIZE); 188 if (size == 0) 189 return -EINVAL; 190 191 /* Allocate the new object */ 192 obj = i915_gem_object_create_shmem(dev_priv, size); 193 if (IS_ERR(obj)) 194 return PTR_ERR(obj); 195 196 ret = drm_gem_handle_create(file, &obj->base, &handle); 197 /* drop reference from allocate - handle holds it now */ 198 i915_gem_object_put(obj); 199 if (ret) 200 return ret; 201 202 *handle_p = handle; 203 *size_p = size; 204 return 0; 205 } 206 207 int 208 i915_gem_dumb_create(struct drm_file *file, 209 struct drm_device *dev, 210 struct drm_mode_create_dumb *args) 211 { 212 int cpp = DIV_ROUND_UP(args->bpp, 8); 213 u32 format; 214 215 switch (cpp) { 216 case 1: 217 format = DRM_FORMAT_C8; 218 break; 219 case 2: 220 format = DRM_FORMAT_RGB565; 221 break; 222 case 4: 223 format = DRM_FORMAT_XRGB8888; 224 break; 225 default: 226 return -EINVAL; 227 } 228 229 /* have to work out size/pitch and return them */ 230 args->pitch = ALIGN(args->width * cpp, 64); 231 232 /* align stride to page size so that we can remap */ 233 if (args->pitch > intel_plane_fb_max_stride(to_i915(dev), format, 234 DRM_FORMAT_MOD_LINEAR)) 235 args->pitch = ALIGN(args->pitch, 4096); 236 237 args->size = args->pitch * args->height; 238 return i915_gem_create(file, to_i915(dev), 239 &args->size, &args->handle); 240 } 241 242 /** 243 * Creates a new mm object and returns a handle to it. 244 * @dev: drm device pointer 245 * @data: ioctl data blob 246 * @file: drm file pointer 247 */ 248 int 249 i915_gem_create_ioctl(struct drm_device *dev, void *data, 250 struct drm_file *file) 251 { 252 struct drm_i915_private *dev_priv = to_i915(dev); 253 struct drm_i915_gem_create *args = data; 254 255 i915_gem_flush_free_objects(dev_priv); 256 257 return i915_gem_create(file, dev_priv, 258 &args->size, &args->handle); 259 } 260 261 static int 262 shmem_pread(struct page *page, int offset, int len, char __user *user_data, 263 bool needs_clflush) 264 { 265 char *vaddr; 266 int ret; 267 268 vaddr = kmap(page); 269 270 if (needs_clflush) 271 drm_clflush_virt_range(vaddr + offset, len); 272 273 ret = __copy_to_user(user_data, vaddr + offset, len); 274 275 kunmap(page); 276 277 return ret ? -EFAULT : 0; 278 } 279 280 static int 281 i915_gem_shmem_pread(struct drm_i915_gem_object *obj, 282 struct drm_i915_gem_pread *args) 283 { 284 unsigned int needs_clflush; 285 unsigned int idx, offset; 286 struct dma_fence *fence; 287 char __user *user_data; 288 u64 remain; 289 int ret; 290 291 ret = i915_gem_object_prepare_read(obj, &needs_clflush); 292 if (ret) 293 return ret; 294 295 fence = i915_gem_object_lock_fence(obj); 296 i915_gem_object_finish_access(obj); 297 if (!fence) 298 return -ENOMEM; 299 300 remain = args->size; 301 user_data = u64_to_user_ptr(args->data_ptr); 302 offset = offset_in_page(args->offset); 303 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { 304 struct page *page = i915_gem_object_get_page(obj, idx); 305 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset); 306 307 ret = shmem_pread(page, offset, length, user_data, 308 needs_clflush); 309 if (ret) 310 break; 311 312 remain -= length; 313 user_data += length; 314 offset = 0; 315 } 316 317 i915_gem_object_unlock_fence(obj, fence); 318 return ret; 319 } 320 321 static inline bool 322 gtt_user_read(struct io_mapping *mapping, 323 loff_t base, int offset, 324 char __user *user_data, int length) 325 { 326 void __iomem *vaddr; 327 unsigned long unwritten; 328 329 /* We can use the cpu mem copy function because this is X86. */ 330 vaddr = io_mapping_map_atomic_wc(mapping, base); 331 unwritten = __copy_to_user_inatomic(user_data, 332 (void __force *)vaddr + offset, 333 length); 334 io_mapping_unmap_atomic(vaddr); 335 if (unwritten) { 336 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE); 337 unwritten = copy_to_user(user_data, 338 (void __force *)vaddr + offset, 339 length); 340 io_mapping_unmap(vaddr); 341 } 342 return unwritten; 343 } 344 345 static int 346 i915_gem_gtt_pread(struct drm_i915_gem_object *obj, 347 const struct drm_i915_gem_pread *args) 348 { 349 struct drm_i915_private *i915 = to_i915(obj->base.dev); 350 struct i915_ggtt *ggtt = &i915->ggtt; 351 intel_wakeref_t wakeref; 352 struct drm_mm_node node; 353 struct dma_fence *fence; 354 void __user *user_data; 355 struct i915_vma *vma; 356 u64 remain, offset; 357 int ret; 358 359 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 360 vma = ERR_PTR(-ENODEV); 361 if (!i915_gem_object_is_tiled(obj)) 362 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 363 PIN_MAPPABLE | 364 PIN_NONBLOCK /* NOWARN */ | 365 PIN_NOEVICT); 366 if (!IS_ERR(vma)) { 367 node.start = i915_ggtt_offset(vma); 368 node.flags = 0; 369 } else { 370 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); 371 if (ret) 372 goto out_rpm; 373 GEM_BUG_ON(!drm_mm_node_allocated(&node)); 374 } 375 376 ret = i915_gem_object_lock_interruptible(obj); 377 if (ret) 378 goto out_unpin; 379 380 ret = i915_gem_object_set_to_gtt_domain(obj, false); 381 if (ret) { 382 i915_gem_object_unlock(obj); 383 goto out_unpin; 384 } 385 386 fence = i915_gem_object_lock_fence(obj); 387 i915_gem_object_unlock(obj); 388 if (!fence) { 389 ret = -ENOMEM; 390 goto out_unpin; 391 } 392 393 user_data = u64_to_user_ptr(args->data_ptr); 394 remain = args->size; 395 offset = args->offset; 396 397 while (remain > 0) { 398 /* Operation in this page 399 * 400 * page_base = page offset within aperture 401 * page_offset = offset within page 402 * page_length = bytes to copy for this page 403 */ 404 u32 page_base = node.start; 405 unsigned page_offset = offset_in_page(offset); 406 unsigned page_length = PAGE_SIZE - page_offset; 407 page_length = remain < page_length ? remain : page_length; 408 if (drm_mm_node_allocated(&node)) { 409 ggtt->vm.insert_page(&ggtt->vm, 410 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), 411 node.start, I915_CACHE_NONE, 0); 412 } else { 413 page_base += offset & PAGE_MASK; 414 } 415 416 if (gtt_user_read(&ggtt->iomap, page_base, page_offset, 417 user_data, page_length)) { 418 ret = -EFAULT; 419 break; 420 } 421 422 remain -= page_length; 423 user_data += page_length; 424 offset += page_length; 425 } 426 427 i915_gem_object_unlock_fence(obj, fence); 428 out_unpin: 429 if (drm_mm_node_allocated(&node)) { 430 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size); 431 remove_mappable_node(ggtt, &node); 432 } else { 433 i915_vma_unpin(vma); 434 } 435 out_rpm: 436 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 437 return ret; 438 } 439 440 /** 441 * Reads data from the object referenced by handle. 442 * @dev: drm device pointer 443 * @data: ioctl data blob 444 * @file: drm file pointer 445 * 446 * On error, the contents of *data are undefined. 447 */ 448 int 449 i915_gem_pread_ioctl(struct drm_device *dev, void *data, 450 struct drm_file *file) 451 { 452 struct drm_i915_gem_pread *args = data; 453 struct drm_i915_gem_object *obj; 454 int ret; 455 456 if (args->size == 0) 457 return 0; 458 459 if (!access_ok(u64_to_user_ptr(args->data_ptr), 460 args->size)) 461 return -EFAULT; 462 463 obj = i915_gem_object_lookup(file, args->handle); 464 if (!obj) 465 return -ENOENT; 466 467 /* Bounds check source. */ 468 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { 469 ret = -EINVAL; 470 goto out; 471 } 472 473 trace_i915_gem_object_pread(obj, args->offset, args->size); 474 475 ret = i915_gem_object_wait(obj, 476 I915_WAIT_INTERRUPTIBLE, 477 MAX_SCHEDULE_TIMEOUT); 478 if (ret) 479 goto out; 480 481 ret = i915_gem_object_pin_pages(obj); 482 if (ret) 483 goto out; 484 485 ret = i915_gem_shmem_pread(obj, args); 486 if (ret == -EFAULT || ret == -ENODEV) 487 ret = i915_gem_gtt_pread(obj, args); 488 489 i915_gem_object_unpin_pages(obj); 490 out: 491 i915_gem_object_put(obj); 492 return ret; 493 } 494 495 /* This is the fast write path which cannot handle 496 * page faults in the source data 497 */ 498 499 static inline bool 500 ggtt_write(struct io_mapping *mapping, 501 loff_t base, int offset, 502 char __user *user_data, int length) 503 { 504 void __iomem *vaddr; 505 unsigned long unwritten; 506 507 /* We can use the cpu mem copy function because this is X86. */ 508 vaddr = io_mapping_map_atomic_wc(mapping, base); 509 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, 510 user_data, length); 511 io_mapping_unmap_atomic(vaddr); 512 if (unwritten) { 513 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE); 514 unwritten = copy_from_user((void __force *)vaddr + offset, 515 user_data, length); 516 io_mapping_unmap(vaddr); 517 } 518 519 return unwritten; 520 } 521 522 /** 523 * This is the fast pwrite path, where we copy the data directly from the 524 * user into the GTT, uncached. 525 * @obj: i915 GEM object 526 * @args: pwrite arguments structure 527 */ 528 static int 529 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, 530 const struct drm_i915_gem_pwrite *args) 531 { 532 struct drm_i915_private *i915 = to_i915(obj->base.dev); 533 struct i915_ggtt *ggtt = &i915->ggtt; 534 struct intel_runtime_pm *rpm = &i915->runtime_pm; 535 intel_wakeref_t wakeref; 536 struct drm_mm_node node; 537 struct dma_fence *fence; 538 struct i915_vma *vma; 539 u64 remain, offset; 540 void __user *user_data; 541 int ret; 542 543 if (i915_gem_object_has_struct_page(obj)) { 544 /* 545 * Avoid waking the device up if we can fallback, as 546 * waking/resuming is very slow (worst-case 10-100 ms 547 * depending on PCI sleeps and our own resume time). 548 * This easily dwarfs any performance advantage from 549 * using the cache bypass of indirect GGTT access. 550 */ 551 wakeref = intel_runtime_pm_get_if_in_use(rpm); 552 if (!wakeref) 553 return -EFAULT; 554 } else { 555 /* No backing pages, no fallback, we must force GGTT access */ 556 wakeref = intel_runtime_pm_get(rpm); 557 } 558 559 vma = ERR_PTR(-ENODEV); 560 if (!i915_gem_object_is_tiled(obj)) 561 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 562 PIN_MAPPABLE | 563 PIN_NONBLOCK /* NOWARN */ | 564 PIN_NOEVICT); 565 if (!IS_ERR(vma)) { 566 node.start = i915_ggtt_offset(vma); 567 node.flags = 0; 568 } else { 569 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); 570 if (ret) 571 goto out_rpm; 572 GEM_BUG_ON(!drm_mm_node_allocated(&node)); 573 } 574 575 ret = i915_gem_object_lock_interruptible(obj); 576 if (ret) 577 goto out_unpin; 578 579 ret = i915_gem_object_set_to_gtt_domain(obj, true); 580 if (ret) { 581 i915_gem_object_unlock(obj); 582 goto out_unpin; 583 } 584 585 fence = i915_gem_object_lock_fence(obj); 586 i915_gem_object_unlock(obj); 587 if (!fence) { 588 ret = -ENOMEM; 589 goto out_unpin; 590 } 591 592 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); 593 594 user_data = u64_to_user_ptr(args->data_ptr); 595 offset = args->offset; 596 remain = args->size; 597 while (remain) { 598 /* Operation in this page 599 * 600 * page_base = page offset within aperture 601 * page_offset = offset within page 602 * page_length = bytes to copy for this page 603 */ 604 u32 page_base = node.start; 605 unsigned int page_offset = offset_in_page(offset); 606 unsigned int page_length = PAGE_SIZE - page_offset; 607 page_length = remain < page_length ? remain : page_length; 608 if (drm_mm_node_allocated(&node)) { 609 /* flush the write before we modify the GGTT */ 610 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 611 ggtt->vm.insert_page(&ggtt->vm, 612 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), 613 node.start, I915_CACHE_NONE, 0); 614 wmb(); /* flush modifications to the GGTT (insert_page) */ 615 } else { 616 page_base += offset & PAGE_MASK; 617 } 618 /* If we get a fault while copying data, then (presumably) our 619 * source page isn't available. Return the error and we'll 620 * retry in the slow path. 621 * If the object is non-shmem backed, we retry again with the 622 * path that handles page fault. 623 */ 624 if (ggtt_write(&ggtt->iomap, page_base, page_offset, 625 user_data, page_length)) { 626 ret = -EFAULT; 627 break; 628 } 629 630 remain -= page_length; 631 user_data += page_length; 632 offset += page_length; 633 } 634 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); 635 636 i915_gem_object_unlock_fence(obj, fence); 637 out_unpin: 638 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 639 if (drm_mm_node_allocated(&node)) { 640 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size); 641 remove_mappable_node(ggtt, &node); 642 } else { 643 i915_vma_unpin(vma); 644 } 645 out_rpm: 646 intel_runtime_pm_put(rpm, wakeref); 647 return ret; 648 } 649 650 /* Per-page copy function for the shmem pwrite fastpath. 651 * Flushes invalid cachelines before writing to the target if 652 * needs_clflush_before is set and flushes out any written cachelines after 653 * writing if needs_clflush is set. 654 */ 655 static int 656 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, 657 bool needs_clflush_before, 658 bool needs_clflush_after) 659 { 660 char *vaddr; 661 int ret; 662 663 vaddr = kmap(page); 664 665 if (needs_clflush_before) 666 drm_clflush_virt_range(vaddr + offset, len); 667 668 ret = __copy_from_user(vaddr + offset, user_data, len); 669 if (!ret && needs_clflush_after) 670 drm_clflush_virt_range(vaddr + offset, len); 671 672 kunmap(page); 673 674 return ret ? -EFAULT : 0; 675 } 676 677 static int 678 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, 679 const struct drm_i915_gem_pwrite *args) 680 { 681 unsigned int partial_cacheline_write; 682 unsigned int needs_clflush; 683 unsigned int offset, idx; 684 struct dma_fence *fence; 685 void __user *user_data; 686 u64 remain; 687 int ret; 688 689 ret = i915_gem_object_prepare_write(obj, &needs_clflush); 690 if (ret) 691 return ret; 692 693 fence = i915_gem_object_lock_fence(obj); 694 i915_gem_object_finish_access(obj); 695 if (!fence) 696 return -ENOMEM; 697 698 /* If we don't overwrite a cacheline completely we need to be 699 * careful to have up-to-date data by first clflushing. Don't 700 * overcomplicate things and flush the entire patch. 701 */ 702 partial_cacheline_write = 0; 703 if (needs_clflush & CLFLUSH_BEFORE) 704 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1; 705 706 user_data = u64_to_user_ptr(args->data_ptr); 707 remain = args->size; 708 offset = offset_in_page(args->offset); 709 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { 710 struct page *page = i915_gem_object_get_page(obj, idx); 711 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset); 712 713 ret = shmem_pwrite(page, offset, length, user_data, 714 (offset | length) & partial_cacheline_write, 715 needs_clflush & CLFLUSH_AFTER); 716 if (ret) 717 break; 718 719 remain -= length; 720 user_data += length; 721 offset = 0; 722 } 723 724 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); 725 i915_gem_object_unlock_fence(obj, fence); 726 727 return ret; 728 } 729 730 /** 731 * Writes data to the object referenced by handle. 732 * @dev: drm device 733 * @data: ioctl data blob 734 * @file: drm file 735 * 736 * On error, the contents of the buffer that were to be modified are undefined. 737 */ 738 int 739 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 740 struct drm_file *file) 741 { 742 struct drm_i915_gem_pwrite *args = data; 743 struct drm_i915_gem_object *obj; 744 int ret; 745 746 if (args->size == 0) 747 return 0; 748 749 if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size)) 750 return -EFAULT; 751 752 obj = i915_gem_object_lookup(file, args->handle); 753 if (!obj) 754 return -ENOENT; 755 756 /* Bounds check destination. */ 757 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { 758 ret = -EINVAL; 759 goto err; 760 } 761 762 /* Writes not allowed into this read-only object */ 763 if (i915_gem_object_is_readonly(obj)) { 764 ret = -EINVAL; 765 goto err; 766 } 767 768 trace_i915_gem_object_pwrite(obj, args->offset, args->size); 769 770 ret = -ENODEV; 771 if (obj->ops->pwrite) 772 ret = obj->ops->pwrite(obj, args); 773 if (ret != -ENODEV) 774 goto err; 775 776 ret = i915_gem_object_wait(obj, 777 I915_WAIT_INTERRUPTIBLE | 778 I915_WAIT_ALL, 779 MAX_SCHEDULE_TIMEOUT); 780 if (ret) 781 goto err; 782 783 ret = i915_gem_object_pin_pages(obj); 784 if (ret) 785 goto err; 786 787 ret = -EFAULT; 788 /* We can only do the GTT pwrite on untiled buffers, as otherwise 789 * it would end up going through the fenced access, and we'll get 790 * different detiling behavior between reading and writing. 791 * pread/pwrite currently are reading and writing from the CPU 792 * perspective, requiring manual detiling by the client. 793 */ 794 if (!i915_gem_object_has_struct_page(obj) || 795 cpu_write_needs_clflush(obj)) 796 /* Note that the gtt paths might fail with non-page-backed user 797 * pointers (e.g. gtt mappings when moving data between 798 * textures). Fallback to the shmem path in that case. 799 */ 800 ret = i915_gem_gtt_pwrite_fast(obj, args); 801 802 if (ret == -EFAULT || ret == -ENOSPC) { 803 if (obj->phys_handle) 804 ret = i915_gem_phys_pwrite(obj, args, file); 805 else 806 ret = i915_gem_shmem_pwrite(obj, args); 807 } 808 809 i915_gem_object_unpin_pages(obj); 810 err: 811 i915_gem_object_put(obj); 812 return ret; 813 } 814 815 /** 816 * Called when user space has done writes to this buffer 817 * @dev: drm device 818 * @data: ioctl data blob 819 * @file: drm file 820 */ 821 int 822 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 823 struct drm_file *file) 824 { 825 struct drm_i915_gem_sw_finish *args = data; 826 struct drm_i915_gem_object *obj; 827 828 obj = i915_gem_object_lookup(file, args->handle); 829 if (!obj) 830 return -ENOENT; 831 832 /* 833 * Proxy objects are barred from CPU access, so there is no 834 * need to ban sw_finish as it is a nop. 835 */ 836 837 /* Pinned buffers may be scanout, so flush the cache */ 838 i915_gem_object_flush_if_display(obj); 839 i915_gem_object_put(obj); 840 841 return 0; 842 } 843 844 void i915_gem_runtime_suspend(struct drm_i915_private *i915) 845 { 846 struct drm_i915_gem_object *obj, *on; 847 int i; 848 849 /* 850 * Only called during RPM suspend. All users of the userfault_list 851 * must be holding an RPM wakeref to ensure that this can not 852 * run concurrently with themselves (and use the struct_mutex for 853 * protection between themselves). 854 */ 855 856 list_for_each_entry_safe(obj, on, 857 &i915->ggtt.userfault_list, userfault_link) 858 __i915_gem_object_release_mmap(obj); 859 860 /* 861 * The fence will be lost when the device powers down. If any were 862 * in use by hardware (i.e. they are pinned), we should not be powering 863 * down! All other fences will be reacquired by the user upon waking. 864 */ 865 for (i = 0; i < i915->ggtt.num_fences; i++) { 866 struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i]; 867 868 /* 869 * Ideally we want to assert that the fence register is not 870 * live at this point (i.e. that no piece of code will be 871 * trying to write through fence + GTT, as that both violates 872 * our tracking of activity and associated locking/barriers, 873 * but also is illegal given that the hw is powered down). 874 * 875 * Previously we used reg->pin_count as a "liveness" indicator. 876 * That is not sufficient, and we need a more fine-grained 877 * tool if we want to have a sanity check here. 878 */ 879 880 if (!reg->vma) 881 continue; 882 883 GEM_BUG_ON(i915_vma_has_userfault(reg->vma)); 884 reg->dirty = true; 885 } 886 } 887 888 struct i915_vma * 889 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, 890 const struct i915_ggtt_view *view, 891 u64 size, 892 u64 alignment, 893 u64 flags) 894 { 895 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 896 struct i915_address_space *vm = &dev_priv->ggtt.vm; 897 898 return i915_gem_object_pin(obj, vm, view, size, alignment, 899 flags | PIN_GLOBAL); 900 } 901 902 struct i915_vma * 903 i915_gem_object_pin(struct drm_i915_gem_object *obj, 904 struct i915_address_space *vm, 905 const struct i915_ggtt_view *view, 906 u64 size, 907 u64 alignment, 908 u64 flags) 909 { 910 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 911 struct i915_vma *vma; 912 int ret; 913 914 if (i915_gem_object_never_bind_ggtt(obj)) 915 return ERR_PTR(-ENODEV); 916 917 if (flags & PIN_MAPPABLE && 918 (!view || view->type == I915_GGTT_VIEW_NORMAL)) { 919 /* If the required space is larger than the available 920 * aperture, we will not able to find a slot for the 921 * object and unbinding the object now will be in 922 * vain. Worse, doing so may cause us to ping-pong 923 * the object in and out of the Global GTT and 924 * waste a lot of cycles under the mutex. 925 */ 926 if (obj->base.size > dev_priv->ggtt.mappable_end) 927 return ERR_PTR(-E2BIG); 928 929 /* If NONBLOCK is set the caller is optimistically 930 * trying to cache the full object within the mappable 931 * aperture, and *must* have a fallback in place for 932 * situations where we cannot bind the object. We 933 * can be a little more lax here and use the fallback 934 * more often to avoid costly migrations of ourselves 935 * and other objects within the aperture. 936 * 937 * Half-the-aperture is used as a simple heuristic. 938 * More interesting would to do search for a free 939 * block prior to making the commitment to unbind. 940 * That caters for the self-harm case, and with a 941 * little more heuristics (e.g. NOFAULT, NOEVICT) 942 * we could try to minimise harm to others. 943 */ 944 if (flags & PIN_NONBLOCK && 945 obj->base.size > dev_priv->ggtt.mappable_end / 2) 946 return ERR_PTR(-ENOSPC); 947 } 948 949 vma = i915_vma_instance(obj, vm, view); 950 if (IS_ERR(vma)) 951 return vma; 952 953 if (i915_vma_misplaced(vma, size, alignment, flags)) { 954 if (flags & PIN_NONBLOCK) { 955 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) 956 return ERR_PTR(-ENOSPC); 957 958 if (flags & PIN_MAPPABLE && 959 vma->fence_size > dev_priv->ggtt.mappable_end / 2) 960 return ERR_PTR(-ENOSPC); 961 } 962 963 ret = i915_vma_unbind(vma); 964 if (ret) 965 return ERR_PTR(ret); 966 } 967 968 if (vma->fence && !i915_gem_object_is_tiled(obj)) { 969 mutex_lock(&vma->vm->mutex); 970 ret = i915_vma_revoke_fence(vma); 971 mutex_unlock(&vma->vm->mutex); 972 if (ret) 973 return ERR_PTR(ret); 974 } 975 976 ret = i915_vma_pin(vma, size, alignment, flags); 977 if (ret) 978 return ERR_PTR(ret); 979 980 return vma; 981 } 982 983 int 984 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 985 struct drm_file *file_priv) 986 { 987 struct drm_i915_private *i915 = to_i915(dev); 988 struct drm_i915_gem_madvise *args = data; 989 struct drm_i915_gem_object *obj; 990 int err; 991 992 switch (args->madv) { 993 case I915_MADV_DONTNEED: 994 case I915_MADV_WILLNEED: 995 break; 996 default: 997 return -EINVAL; 998 } 999 1000 obj = i915_gem_object_lookup(file_priv, args->handle); 1001 if (!obj) 1002 return -ENOENT; 1003 1004 err = mutex_lock_interruptible(&obj->mm.lock); 1005 if (err) 1006 goto out; 1007 1008 if (i915_gem_object_has_pages(obj) && 1009 i915_gem_object_is_tiled(obj) && 1010 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 1011 if (obj->mm.madv == I915_MADV_WILLNEED) { 1012 GEM_BUG_ON(!obj->mm.quirked); 1013 __i915_gem_object_unpin_pages(obj); 1014 obj->mm.quirked = false; 1015 } 1016 if (args->madv == I915_MADV_WILLNEED) { 1017 GEM_BUG_ON(obj->mm.quirked); 1018 __i915_gem_object_pin_pages(obj); 1019 obj->mm.quirked = true; 1020 } 1021 } 1022 1023 if (obj->mm.madv != __I915_MADV_PURGED) 1024 obj->mm.madv = args->madv; 1025 1026 if (i915_gem_object_has_pages(obj)) { 1027 struct list_head *list; 1028 1029 if (i915_gem_object_is_shrinkable(obj)) { 1030 unsigned long flags; 1031 1032 spin_lock_irqsave(&i915->mm.obj_lock, flags); 1033 1034 if (obj->mm.madv != I915_MADV_WILLNEED) 1035 list = &i915->mm.purge_list; 1036 else 1037 list = &i915->mm.shrink_list; 1038 list_move_tail(&obj->mm.link, list); 1039 1040 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 1041 } 1042 } 1043 1044 /* if the object is no longer attached, discard its backing storage */ 1045 if (obj->mm.madv == I915_MADV_DONTNEED && 1046 !i915_gem_object_has_pages(obj)) 1047 i915_gem_object_truncate(obj); 1048 1049 args->retained = obj->mm.madv != __I915_MADV_PURGED; 1050 mutex_unlock(&obj->mm.lock); 1051 1052 out: 1053 i915_gem_object_put(obj); 1054 return err; 1055 } 1056 1057 static int __intel_context_flush_retire(struct intel_context *ce) 1058 { 1059 struct intel_timeline *tl; 1060 1061 tl = intel_context_timeline_lock(ce); 1062 if (IS_ERR(tl)) 1063 return PTR_ERR(tl); 1064 1065 intel_context_timeline_unlock(tl); 1066 return 0; 1067 } 1068 1069 static int __intel_engines_record_defaults(struct intel_gt *gt) 1070 { 1071 struct i915_request *requests[I915_NUM_ENGINES] = {}; 1072 struct intel_engine_cs *engine; 1073 enum intel_engine_id id; 1074 int err = 0; 1075 1076 /* 1077 * As we reset the gpu during very early sanitisation, the current 1078 * register state on the GPU should reflect its defaults values. 1079 * We load a context onto the hw (with restore-inhibit), then switch 1080 * over to a second context to save that default register state. We 1081 * can then prime every new context with that state so they all start 1082 * from the same default HW values. 1083 */ 1084 1085 for_each_engine(engine, gt, id) { 1086 struct intel_context *ce; 1087 struct i915_request *rq; 1088 1089 /* We must be able to switch to something! */ 1090 GEM_BUG_ON(!engine->kernel_context); 1091 engine->serial++; /* force the kernel context switch */ 1092 1093 ce = intel_context_create(engine->kernel_context->gem_context, 1094 engine); 1095 if (IS_ERR(ce)) { 1096 err = PTR_ERR(ce); 1097 goto out; 1098 } 1099 1100 rq = intel_context_create_request(ce); 1101 if (IS_ERR(rq)) { 1102 err = PTR_ERR(rq); 1103 intel_context_put(ce); 1104 goto out; 1105 } 1106 1107 err = intel_engine_emit_ctx_wa(rq); 1108 if (err) 1109 goto err_rq; 1110 1111 err = intel_renderstate_emit(rq); 1112 if (err) 1113 goto err_rq; 1114 1115 err_rq: 1116 requests[id] = i915_request_get(rq); 1117 i915_request_add(rq); 1118 if (err) 1119 goto out; 1120 } 1121 1122 /* Flush the default context image to memory, and enable powersaving. */ 1123 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) { 1124 err = -EIO; 1125 goto out; 1126 } 1127 1128 for (id = 0; id < ARRAY_SIZE(requests); id++) { 1129 struct i915_request *rq; 1130 struct i915_vma *state; 1131 void *vaddr; 1132 1133 rq = requests[id]; 1134 if (!rq) 1135 continue; 1136 1137 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, 1138 &rq->hw_context->flags)); 1139 state = rq->hw_context->state; 1140 if (!state) 1141 continue; 1142 1143 /* Serialise with retirement on another CPU */ 1144 err = __intel_context_flush_retire(rq->hw_context); 1145 if (err) 1146 goto out; 1147 1148 /* We want to be able to unbind the state from the GGTT */ 1149 GEM_BUG_ON(intel_context_is_pinned(rq->hw_context)); 1150 1151 /* 1152 * As we will hold a reference to the logical state, it will 1153 * not be torn down with the context, and importantly the 1154 * object will hold onto its vma (making it possible for a 1155 * stray GTT write to corrupt our defaults). Unmap the vma 1156 * from the GTT to prevent such accidents and reclaim the 1157 * space. 1158 */ 1159 err = i915_vma_unbind(state); 1160 if (err) 1161 goto out; 1162 1163 i915_gem_object_lock(state->obj); 1164 err = i915_gem_object_set_to_cpu_domain(state->obj, false); 1165 i915_gem_object_unlock(state->obj); 1166 if (err) 1167 goto out; 1168 1169 i915_gem_object_set_cache_coherency(state->obj, I915_CACHE_LLC); 1170 1171 /* Check we can acquire the image of the context state */ 1172 vaddr = i915_gem_object_pin_map(state->obj, I915_MAP_FORCE_WB); 1173 if (IS_ERR(vaddr)) { 1174 err = PTR_ERR(vaddr); 1175 goto out; 1176 } 1177 1178 rq->engine->default_state = i915_gem_object_get(state->obj); 1179 i915_gem_object_unpin_map(state->obj); 1180 } 1181 1182 out: 1183 /* 1184 * If we have to abandon now, we expect the engines to be idle 1185 * and ready to be torn-down. The quickest way we can accomplish 1186 * this is by declaring ourselves wedged. 1187 */ 1188 if (err) 1189 intel_gt_set_wedged(gt); 1190 1191 for (id = 0; id < ARRAY_SIZE(requests); id++) { 1192 struct intel_context *ce; 1193 struct i915_request *rq; 1194 1195 rq = requests[id]; 1196 if (!rq) 1197 continue; 1198 1199 ce = rq->hw_context; 1200 i915_request_put(rq); 1201 intel_context_put(ce); 1202 } 1203 return err; 1204 } 1205 1206 static int intel_engines_verify_workarounds(struct intel_gt *gt) 1207 { 1208 struct intel_engine_cs *engine; 1209 enum intel_engine_id id; 1210 int err = 0; 1211 1212 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 1213 return 0; 1214 1215 for_each_engine(engine, gt, id) { 1216 if (intel_engine_verify_workarounds(engine, "load")) 1217 err = -EIO; 1218 } 1219 1220 return err; 1221 } 1222 1223 int i915_gem_init(struct drm_i915_private *dev_priv) 1224 { 1225 int ret; 1226 1227 /* We need to fallback to 4K pages if host doesn't support huge gtt. */ 1228 if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv)) 1229 mkwrite_device_info(dev_priv)->page_sizes = 1230 I915_GTT_PAGE_SIZE_4K; 1231 1232 intel_timelines_init(dev_priv); 1233 1234 ret = i915_gem_init_userptr(dev_priv); 1235 if (ret) 1236 return ret; 1237 1238 intel_uc_fetch_firmwares(&dev_priv->gt.uc); 1239 intel_wopcm_init(&dev_priv->wopcm); 1240 1241 /* This is just a security blanket to placate dragons. 1242 * On some systems, we very sporadically observe that the first TLBs 1243 * used by the CS may be stale, despite us poking the TLB reset. If 1244 * we hold the forcewake during initialisation these problems 1245 * just magically go away. 1246 */ 1247 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); 1248 1249 ret = i915_init_ggtt(dev_priv); 1250 if (ret) { 1251 GEM_BUG_ON(ret == -EIO); 1252 goto err_unlock; 1253 } 1254 1255 intel_gt_init(&dev_priv->gt); 1256 1257 ret = intel_engines_setup(&dev_priv->gt); 1258 if (ret) { 1259 GEM_BUG_ON(ret == -EIO); 1260 goto err_unlock; 1261 } 1262 1263 ret = i915_gem_init_contexts(dev_priv); 1264 if (ret) { 1265 GEM_BUG_ON(ret == -EIO); 1266 goto err_scratch; 1267 } 1268 1269 ret = intel_engines_init(&dev_priv->gt); 1270 if (ret) { 1271 GEM_BUG_ON(ret == -EIO); 1272 goto err_context; 1273 } 1274 1275 intel_uc_init(&dev_priv->gt.uc); 1276 1277 ret = intel_gt_init_hw(&dev_priv->gt); 1278 if (ret) 1279 goto err_uc_init; 1280 1281 /* Only when the HW is re-initialised, can we replay the requests */ 1282 ret = intel_gt_resume(&dev_priv->gt); 1283 if (ret) 1284 goto err_init_hw; 1285 1286 /* 1287 * Despite its name intel_init_clock_gating applies both display 1288 * clock gating workarounds; GT mmio workarounds and the occasional 1289 * GT power context workaround. Worse, sometimes it includes a context 1290 * register workaround which we need to apply before we record the 1291 * default HW state for all contexts. 1292 * 1293 * FIXME: break up the workarounds and apply them at the right time! 1294 */ 1295 intel_init_clock_gating(dev_priv); 1296 1297 ret = intel_engines_verify_workarounds(&dev_priv->gt); 1298 if (ret) 1299 goto err_gt; 1300 1301 ret = __intel_engines_record_defaults(&dev_priv->gt); 1302 if (ret) 1303 goto err_gt; 1304 1305 ret = i915_inject_probe_error(dev_priv, -ENODEV); 1306 if (ret) 1307 goto err_gt; 1308 1309 ret = i915_inject_probe_error(dev_priv, -EIO); 1310 if (ret) 1311 goto err_gt; 1312 1313 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); 1314 1315 return 0; 1316 1317 /* 1318 * Unwinding is complicated by that we want to handle -EIO to mean 1319 * disable GPU submission but keep KMS alive. We want to mark the 1320 * HW as irrevisibly wedged, but keep enough state around that the 1321 * driver doesn't explode during runtime. 1322 */ 1323 err_gt: 1324 intel_gt_set_wedged_on_init(&dev_priv->gt); 1325 i915_gem_suspend(dev_priv); 1326 i915_gem_suspend_late(dev_priv); 1327 1328 i915_gem_drain_workqueue(dev_priv); 1329 err_init_hw: 1330 intel_uc_fini_hw(&dev_priv->gt.uc); 1331 err_uc_init: 1332 if (ret != -EIO) { 1333 intel_uc_fini(&dev_priv->gt.uc); 1334 intel_engines_cleanup(&dev_priv->gt); 1335 } 1336 err_context: 1337 if (ret != -EIO) 1338 i915_gem_driver_release__contexts(dev_priv); 1339 err_scratch: 1340 intel_gt_driver_release(&dev_priv->gt); 1341 err_unlock: 1342 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); 1343 1344 if (ret != -EIO) { 1345 intel_uc_cleanup_firmwares(&dev_priv->gt.uc); 1346 i915_gem_cleanup_userptr(dev_priv); 1347 intel_timelines_fini(dev_priv); 1348 } 1349 1350 if (ret == -EIO) { 1351 /* 1352 * Allow engines or uC initialisation to fail by marking the GPU 1353 * as wedged. But we only want to do this when the GPU is angry, 1354 * for all other failure, such as an allocation failure, bail. 1355 */ 1356 if (!intel_gt_is_wedged(&dev_priv->gt)) { 1357 i915_probe_error(dev_priv, 1358 "Failed to initialize GPU, declaring it wedged!\n"); 1359 intel_gt_set_wedged(&dev_priv->gt); 1360 } 1361 1362 /* Minimal basic recovery for KMS */ 1363 ret = i915_ggtt_enable_hw(dev_priv); 1364 i915_gem_restore_gtt_mappings(dev_priv); 1365 i915_gem_restore_fences(&dev_priv->ggtt); 1366 intel_init_clock_gating(dev_priv); 1367 } 1368 1369 i915_gem_drain_freed_objects(dev_priv); 1370 return ret; 1371 } 1372 1373 void i915_gem_driver_register(struct drm_i915_private *i915) 1374 { 1375 i915_gem_driver_register__shrinker(i915); 1376 1377 intel_engines_driver_register(i915); 1378 } 1379 1380 void i915_gem_driver_unregister(struct drm_i915_private *i915) 1381 { 1382 i915_gem_driver_unregister__shrinker(i915); 1383 } 1384 1385 void i915_gem_driver_remove(struct drm_i915_private *dev_priv) 1386 { 1387 intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref); 1388 1389 i915_gem_suspend_late(dev_priv); 1390 intel_gt_driver_remove(&dev_priv->gt); 1391 1392 /* Flush any outstanding unpin_work. */ 1393 i915_gem_drain_workqueue(dev_priv); 1394 1395 intel_uc_fini_hw(&dev_priv->gt.uc); 1396 intel_uc_fini(&dev_priv->gt.uc); 1397 1398 i915_gem_drain_freed_objects(dev_priv); 1399 } 1400 1401 void i915_gem_driver_release(struct drm_i915_private *dev_priv) 1402 { 1403 intel_engines_cleanup(&dev_priv->gt); 1404 i915_gem_driver_release__contexts(dev_priv); 1405 intel_gt_driver_release(&dev_priv->gt); 1406 1407 intel_wa_list_free(&dev_priv->gt_wa_list); 1408 1409 intel_uc_cleanup_firmwares(&dev_priv->gt.uc); 1410 i915_gem_cleanup_userptr(dev_priv); 1411 intel_timelines_fini(dev_priv); 1412 1413 i915_gem_drain_freed_objects(dev_priv); 1414 1415 WARN_ON(!list_empty(&dev_priv->gem.contexts.list)); 1416 } 1417 1418 static void i915_gem_init__mm(struct drm_i915_private *i915) 1419 { 1420 spin_lock_init(&i915->mm.obj_lock); 1421 1422 init_llist_head(&i915->mm.free_list); 1423 1424 INIT_LIST_HEAD(&i915->mm.purge_list); 1425 INIT_LIST_HEAD(&i915->mm.shrink_list); 1426 1427 i915_gem_init__objects(i915); 1428 } 1429 1430 void i915_gem_init_early(struct drm_i915_private *dev_priv) 1431 { 1432 i915_gem_init__mm(dev_priv); 1433 1434 spin_lock_init(&dev_priv->fb_tracking.lock); 1435 } 1436 1437 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv) 1438 { 1439 i915_gem_drain_freed_objects(dev_priv); 1440 GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list)); 1441 GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count)); 1442 WARN_ON(dev_priv->mm.shrink_count); 1443 } 1444 1445 int i915_gem_freeze(struct drm_i915_private *dev_priv) 1446 { 1447 /* Discard all purgeable objects, let userspace recover those as 1448 * required after resuming. 1449 */ 1450 i915_gem_shrink_all(dev_priv); 1451 1452 return 0; 1453 } 1454 1455 int i915_gem_freeze_late(struct drm_i915_private *i915) 1456 { 1457 struct drm_i915_gem_object *obj; 1458 intel_wakeref_t wakeref; 1459 1460 /* 1461 * Called just before we write the hibernation image. 1462 * 1463 * We need to update the domain tracking to reflect that the CPU 1464 * will be accessing all the pages to create and restore from the 1465 * hibernation, and so upon restoration those pages will be in the 1466 * CPU domain. 1467 * 1468 * To make sure the hibernation image contains the latest state, 1469 * we update that state just before writing out the image. 1470 * 1471 * To try and reduce the hibernation image, we manually shrink 1472 * the objects as well, see i915_gem_freeze() 1473 */ 1474 1475 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1476 1477 i915_gem_shrink(i915, -1UL, NULL, ~0); 1478 i915_gem_drain_freed_objects(i915); 1479 1480 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) { 1481 i915_gem_object_lock(obj); 1482 WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true)); 1483 i915_gem_object_unlock(obj); 1484 } 1485 1486 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1487 1488 return 0; 1489 } 1490 1491 void i915_gem_release(struct drm_device *dev, struct drm_file *file) 1492 { 1493 struct drm_i915_file_private *file_priv = file->driver_priv; 1494 struct i915_request *request; 1495 1496 /* Clean up our request list when the client is going away, so that 1497 * later retire_requests won't dereference our soon-to-be-gone 1498 * file_priv. 1499 */ 1500 spin_lock(&file_priv->mm.lock); 1501 list_for_each_entry(request, &file_priv->mm.request_list, client_link) 1502 request->file_priv = NULL; 1503 spin_unlock(&file_priv->mm.lock); 1504 } 1505 1506 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) 1507 { 1508 struct drm_i915_file_private *file_priv; 1509 int ret; 1510 1511 DRM_DEBUG("\n"); 1512 1513 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); 1514 if (!file_priv) 1515 return -ENOMEM; 1516 1517 file->driver_priv = file_priv; 1518 file_priv->dev_priv = i915; 1519 file_priv->file = file; 1520 1521 spin_lock_init(&file_priv->mm.lock); 1522 INIT_LIST_HEAD(&file_priv->mm.request_list); 1523 1524 file_priv->bsd_engine = -1; 1525 file_priv->hang_timestamp = jiffies; 1526 1527 ret = i915_gem_context_open(i915, file); 1528 if (ret) 1529 kfree(file_priv); 1530 1531 return ret; 1532 } 1533 1534 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1535 #include "selftests/mock_gem_device.c" 1536 #include "selftests/i915_gem.c" 1537 #endif 1538