1 /* 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include "drmP.h" 29 #include "drm.h" 30 #include "i915_drm.h" 31 #include "i915_drv.h" 32 #include <linux/swap.h> 33 #include <linux/pci.h> 34 35 #define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) 36 37 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); 38 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); 39 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); 40 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, 41 int write); 42 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 43 uint64_t offset, 44 uint64_t size); 45 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj); 46 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj); 47 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, 48 unsigned alignment); 49 static int i915_gem_object_get_fence_reg(struct drm_gem_object *obj, bool write); 50 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); 51 static int i915_gem_evict_something(struct drm_device *dev); 52 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 53 struct drm_i915_gem_pwrite *args, 54 struct drm_file *file_priv); 55 56 int i915_gem_do_init(struct drm_device *dev, unsigned long start, 57 unsigned long end) 58 { 59 drm_i915_private_t *dev_priv = dev->dev_private; 60 61 if (start >= end || 62 (start & (PAGE_SIZE - 1)) != 0 || 63 (end & (PAGE_SIZE - 1)) != 0) { 64 return -EINVAL; 65 } 66 67 drm_mm_init(&dev_priv->mm.gtt_space, start, 68 end - start); 69 70 dev->gtt_total = (uint32_t) (end - start); 71 72 return 0; 73 } 74 75 int 76 i915_gem_init_ioctl(struct drm_device *dev, void *data, 77 struct drm_file *file_priv) 78 { 79 struct drm_i915_gem_init *args = data; 80 int ret; 81 82 mutex_lock(&dev->struct_mutex); 83 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end); 84 mutex_unlock(&dev->struct_mutex); 85 86 return ret; 87 } 88 89 int 90 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 91 struct drm_file *file_priv) 92 { 93 struct drm_i915_gem_get_aperture *args = data; 94 95 if (!(dev->driver->driver_features & DRIVER_GEM)) 96 return -ENODEV; 97 98 args->aper_size = dev->gtt_total; 99 args->aper_available_size = (args->aper_size - 100 atomic_read(&dev->pin_memory)); 101 102 return 0; 103 } 104 105 106 /** 107 * Creates a new mm object and returns a handle to it. 108 */ 109 int 110 i915_gem_create_ioctl(struct drm_device *dev, void *data, 111 struct drm_file *file_priv) 112 { 113 struct drm_i915_gem_create *args = data; 114 struct drm_gem_object *obj; 115 int handle, ret; 116 117 args->size = roundup(args->size, PAGE_SIZE); 118 119 /* Allocate the new object */ 120 obj = drm_gem_object_alloc(dev, args->size); 121 if (obj == NULL) 122 return -ENOMEM; 123 124 ret = drm_gem_handle_create(file_priv, obj, &handle); 125 mutex_lock(&dev->struct_mutex); 126 drm_gem_object_handle_unreference(obj); 127 mutex_unlock(&dev->struct_mutex); 128 129 if (ret) 130 return ret; 131 132 args->handle = handle; 133 134 return 0; 135 } 136 137 static inline int 138 fast_shmem_read(struct page **pages, 139 loff_t page_base, int page_offset, 140 char __user *data, 141 int length) 142 { 143 char __iomem *vaddr; 144 int unwritten; 145 146 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0); 147 if (vaddr == NULL) 148 return -ENOMEM; 149 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length); 150 kunmap_atomic(vaddr, KM_USER0); 151 152 if (unwritten) 153 return -EFAULT; 154 155 return 0; 156 } 157 158 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj) 159 { 160 drm_i915_private_t *dev_priv = obj->dev->dev_private; 161 struct drm_i915_gem_object *obj_priv = obj->driver_private; 162 163 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 164 obj_priv->tiling_mode != I915_TILING_NONE; 165 } 166 167 static inline int 168 slow_shmem_copy(struct page *dst_page, 169 int dst_offset, 170 struct page *src_page, 171 int src_offset, 172 int length) 173 { 174 char *dst_vaddr, *src_vaddr; 175 176 dst_vaddr = kmap_atomic(dst_page, KM_USER0); 177 if (dst_vaddr == NULL) 178 return -ENOMEM; 179 180 src_vaddr = kmap_atomic(src_page, KM_USER1); 181 if (src_vaddr == NULL) { 182 kunmap_atomic(dst_vaddr, KM_USER0); 183 return -ENOMEM; 184 } 185 186 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length); 187 188 kunmap_atomic(src_vaddr, KM_USER1); 189 kunmap_atomic(dst_vaddr, KM_USER0); 190 191 return 0; 192 } 193 194 static inline int 195 slow_shmem_bit17_copy(struct page *gpu_page, 196 int gpu_offset, 197 struct page *cpu_page, 198 int cpu_offset, 199 int length, 200 int is_read) 201 { 202 char *gpu_vaddr, *cpu_vaddr; 203 204 /* Use the unswizzled path if this page isn't affected. */ 205 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) { 206 if (is_read) 207 return slow_shmem_copy(cpu_page, cpu_offset, 208 gpu_page, gpu_offset, length); 209 else 210 return slow_shmem_copy(gpu_page, gpu_offset, 211 cpu_page, cpu_offset, length); 212 } 213 214 gpu_vaddr = kmap_atomic(gpu_page, KM_USER0); 215 if (gpu_vaddr == NULL) 216 return -ENOMEM; 217 218 cpu_vaddr = kmap_atomic(cpu_page, KM_USER1); 219 if (cpu_vaddr == NULL) { 220 kunmap_atomic(gpu_vaddr, KM_USER0); 221 return -ENOMEM; 222 } 223 224 /* Copy the data, XORing A6 with A17 (1). The user already knows he's 225 * XORing with the other bits (A9 for Y, A9 and A10 for X) 226 */ 227 while (length > 0) { 228 int cacheline_end = ALIGN(gpu_offset + 1, 64); 229 int this_length = min(cacheline_end - gpu_offset, length); 230 int swizzled_gpu_offset = gpu_offset ^ 64; 231 232 if (is_read) { 233 memcpy(cpu_vaddr + cpu_offset, 234 gpu_vaddr + swizzled_gpu_offset, 235 this_length); 236 } else { 237 memcpy(gpu_vaddr + swizzled_gpu_offset, 238 cpu_vaddr + cpu_offset, 239 this_length); 240 } 241 cpu_offset += this_length; 242 gpu_offset += this_length; 243 length -= this_length; 244 } 245 246 kunmap_atomic(cpu_vaddr, KM_USER1); 247 kunmap_atomic(gpu_vaddr, KM_USER0); 248 249 return 0; 250 } 251 252 /** 253 * This is the fast shmem pread path, which attempts to copy_from_user directly 254 * from the backing pages of the object to the user's address space. On a 255 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). 256 */ 257 static int 258 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, 259 struct drm_i915_gem_pread *args, 260 struct drm_file *file_priv) 261 { 262 struct drm_i915_gem_object *obj_priv = obj->driver_private; 263 ssize_t remain; 264 loff_t offset, page_base; 265 char __user *user_data; 266 int page_offset, page_length; 267 int ret; 268 269 user_data = (char __user *) (uintptr_t) args->data_ptr; 270 remain = args->size; 271 272 mutex_lock(&dev->struct_mutex); 273 274 ret = i915_gem_object_get_pages(obj); 275 if (ret != 0) 276 goto fail_unlock; 277 278 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset, 279 args->size); 280 if (ret != 0) 281 goto fail_put_pages; 282 283 obj_priv = obj->driver_private; 284 offset = args->offset; 285 286 while (remain > 0) { 287 /* Operation in this page 288 * 289 * page_base = page offset within aperture 290 * page_offset = offset within page 291 * page_length = bytes to copy for this page 292 */ 293 page_base = (offset & ~(PAGE_SIZE-1)); 294 page_offset = offset & (PAGE_SIZE-1); 295 page_length = remain; 296 if ((page_offset + remain) > PAGE_SIZE) 297 page_length = PAGE_SIZE - page_offset; 298 299 ret = fast_shmem_read(obj_priv->pages, 300 page_base, page_offset, 301 user_data, page_length); 302 if (ret) 303 goto fail_put_pages; 304 305 remain -= page_length; 306 user_data += page_length; 307 offset += page_length; 308 } 309 310 fail_put_pages: 311 i915_gem_object_put_pages(obj); 312 fail_unlock: 313 mutex_unlock(&dev->struct_mutex); 314 315 return ret; 316 } 317 318 /** 319 * This is the fallback shmem pread path, which allocates temporary storage 320 * in kernel space to copy_to_user into outside of the struct_mutex, so we 321 * can copy out of the object's backing pages while holding the struct mutex 322 * and not take page faults. 323 */ 324 static int 325 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, 326 struct drm_i915_gem_pread *args, 327 struct drm_file *file_priv) 328 { 329 struct drm_i915_gem_object *obj_priv = obj->driver_private; 330 struct mm_struct *mm = current->mm; 331 struct page **user_pages; 332 ssize_t remain; 333 loff_t offset, pinned_pages, i; 334 loff_t first_data_page, last_data_page, num_pages; 335 int shmem_page_index, shmem_page_offset; 336 int data_page_index, data_page_offset; 337 int page_length; 338 int ret; 339 uint64_t data_ptr = args->data_ptr; 340 int do_bit17_swizzling; 341 342 remain = args->size; 343 344 /* Pin the user pages containing the data. We can't fault while 345 * holding the struct mutex, yet we want to hold it while 346 * dereferencing the user data. 347 */ 348 first_data_page = data_ptr / PAGE_SIZE; 349 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 350 num_pages = last_data_page - first_data_page + 1; 351 352 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 353 if (user_pages == NULL) 354 return -ENOMEM; 355 356 down_read(&mm->mmap_sem); 357 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 358 num_pages, 1, 0, user_pages, NULL); 359 up_read(&mm->mmap_sem); 360 if (pinned_pages < num_pages) { 361 ret = -EFAULT; 362 goto fail_put_user_pages; 363 } 364 365 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 366 367 mutex_lock(&dev->struct_mutex); 368 369 ret = i915_gem_object_get_pages(obj); 370 if (ret != 0) 371 goto fail_unlock; 372 373 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset, 374 args->size); 375 if (ret != 0) 376 goto fail_put_pages; 377 378 obj_priv = obj->driver_private; 379 offset = args->offset; 380 381 while (remain > 0) { 382 /* Operation in this page 383 * 384 * shmem_page_index = page number within shmem file 385 * shmem_page_offset = offset within page in shmem file 386 * data_page_index = page number in get_user_pages return 387 * data_page_offset = offset with data_page_index page. 388 * page_length = bytes to copy for this page 389 */ 390 shmem_page_index = offset / PAGE_SIZE; 391 shmem_page_offset = offset & ~PAGE_MASK; 392 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 393 data_page_offset = data_ptr & ~PAGE_MASK; 394 395 page_length = remain; 396 if ((shmem_page_offset + page_length) > PAGE_SIZE) 397 page_length = PAGE_SIZE - shmem_page_offset; 398 if ((data_page_offset + page_length) > PAGE_SIZE) 399 page_length = PAGE_SIZE - data_page_offset; 400 401 if (do_bit17_swizzling) { 402 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 403 shmem_page_offset, 404 user_pages[data_page_index], 405 data_page_offset, 406 page_length, 407 1); 408 } else { 409 ret = slow_shmem_copy(user_pages[data_page_index], 410 data_page_offset, 411 obj_priv->pages[shmem_page_index], 412 shmem_page_offset, 413 page_length); 414 } 415 if (ret) 416 goto fail_put_pages; 417 418 remain -= page_length; 419 data_ptr += page_length; 420 offset += page_length; 421 } 422 423 fail_put_pages: 424 i915_gem_object_put_pages(obj); 425 fail_unlock: 426 mutex_unlock(&dev->struct_mutex); 427 fail_put_user_pages: 428 for (i = 0; i < pinned_pages; i++) { 429 SetPageDirty(user_pages[i]); 430 page_cache_release(user_pages[i]); 431 } 432 drm_free_large(user_pages); 433 434 return ret; 435 } 436 437 /** 438 * Reads data from the object referenced by handle. 439 * 440 * On error, the contents of *data are undefined. 441 */ 442 int 443 i915_gem_pread_ioctl(struct drm_device *dev, void *data, 444 struct drm_file *file_priv) 445 { 446 struct drm_i915_gem_pread *args = data; 447 struct drm_gem_object *obj; 448 struct drm_i915_gem_object *obj_priv; 449 int ret; 450 451 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 452 if (obj == NULL) 453 return -EBADF; 454 obj_priv = obj->driver_private; 455 456 /* Bounds check source. 457 * 458 * XXX: This could use review for overflow issues... 459 */ 460 if (args->offset > obj->size || args->size > obj->size || 461 args->offset + args->size > obj->size) { 462 drm_gem_object_unreference(obj); 463 return -EINVAL; 464 } 465 466 if (i915_gem_object_needs_bit17_swizzle(obj)) { 467 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv); 468 } else { 469 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv); 470 if (ret != 0) 471 ret = i915_gem_shmem_pread_slow(dev, obj, args, 472 file_priv); 473 } 474 475 drm_gem_object_unreference(obj); 476 477 return ret; 478 } 479 480 /* This is the fast write path which cannot handle 481 * page faults in the source data 482 */ 483 484 static inline int 485 fast_user_write(struct io_mapping *mapping, 486 loff_t page_base, int page_offset, 487 char __user *user_data, 488 int length) 489 { 490 char *vaddr_atomic; 491 unsigned long unwritten; 492 493 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); 494 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset, 495 user_data, length); 496 io_mapping_unmap_atomic(vaddr_atomic); 497 if (unwritten) 498 return -EFAULT; 499 return 0; 500 } 501 502 /* Here's the write path which can sleep for 503 * page faults 504 */ 505 506 static inline int 507 slow_kernel_write(struct io_mapping *mapping, 508 loff_t gtt_base, int gtt_offset, 509 struct page *user_page, int user_offset, 510 int length) 511 { 512 char *src_vaddr, *dst_vaddr; 513 unsigned long unwritten; 514 515 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base); 516 src_vaddr = kmap_atomic(user_page, KM_USER1); 517 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset, 518 src_vaddr + user_offset, 519 length); 520 kunmap_atomic(src_vaddr, KM_USER1); 521 io_mapping_unmap_atomic(dst_vaddr); 522 if (unwritten) 523 return -EFAULT; 524 return 0; 525 } 526 527 static inline int 528 fast_shmem_write(struct page **pages, 529 loff_t page_base, int page_offset, 530 char __user *data, 531 int length) 532 { 533 char __iomem *vaddr; 534 unsigned long unwritten; 535 536 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0); 537 if (vaddr == NULL) 538 return -ENOMEM; 539 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length); 540 kunmap_atomic(vaddr, KM_USER0); 541 542 if (unwritten) 543 return -EFAULT; 544 return 0; 545 } 546 547 /** 548 * This is the fast pwrite path, where we copy the data directly from the 549 * user into the GTT, uncached. 550 */ 551 static int 552 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 553 struct drm_i915_gem_pwrite *args, 554 struct drm_file *file_priv) 555 { 556 struct drm_i915_gem_object *obj_priv = obj->driver_private; 557 drm_i915_private_t *dev_priv = dev->dev_private; 558 ssize_t remain; 559 loff_t offset, page_base; 560 char __user *user_data; 561 int page_offset, page_length; 562 int ret; 563 564 user_data = (char __user *) (uintptr_t) args->data_ptr; 565 remain = args->size; 566 if (!access_ok(VERIFY_READ, user_data, remain)) 567 return -EFAULT; 568 569 570 mutex_lock(&dev->struct_mutex); 571 ret = i915_gem_object_pin(obj, 0); 572 if (ret) { 573 mutex_unlock(&dev->struct_mutex); 574 return ret; 575 } 576 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 577 if (ret) 578 goto fail; 579 580 obj_priv = obj->driver_private; 581 offset = obj_priv->gtt_offset + args->offset; 582 583 while (remain > 0) { 584 /* Operation in this page 585 * 586 * page_base = page offset within aperture 587 * page_offset = offset within page 588 * page_length = bytes to copy for this page 589 */ 590 page_base = (offset & ~(PAGE_SIZE-1)); 591 page_offset = offset & (PAGE_SIZE-1); 592 page_length = remain; 593 if ((page_offset + remain) > PAGE_SIZE) 594 page_length = PAGE_SIZE - page_offset; 595 596 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base, 597 page_offset, user_data, page_length); 598 599 /* If we get a fault while copying data, then (presumably) our 600 * source page isn't available. Return the error and we'll 601 * retry in the slow path. 602 */ 603 if (ret) 604 goto fail; 605 606 remain -= page_length; 607 user_data += page_length; 608 offset += page_length; 609 } 610 611 fail: 612 i915_gem_object_unpin(obj); 613 mutex_unlock(&dev->struct_mutex); 614 615 return ret; 616 } 617 618 /** 619 * This is the fallback GTT pwrite path, which uses get_user_pages to pin 620 * the memory and maps it using kmap_atomic for copying. 621 * 622 * This code resulted in x11perf -rgb10text consuming about 10% more CPU 623 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit). 624 */ 625 static int 626 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 627 struct drm_i915_gem_pwrite *args, 628 struct drm_file *file_priv) 629 { 630 struct drm_i915_gem_object *obj_priv = obj->driver_private; 631 drm_i915_private_t *dev_priv = dev->dev_private; 632 ssize_t remain; 633 loff_t gtt_page_base, offset; 634 loff_t first_data_page, last_data_page, num_pages; 635 loff_t pinned_pages, i; 636 struct page **user_pages; 637 struct mm_struct *mm = current->mm; 638 int gtt_page_offset, data_page_offset, data_page_index, page_length; 639 int ret; 640 uint64_t data_ptr = args->data_ptr; 641 642 remain = args->size; 643 644 /* Pin the user pages containing the data. We can't fault while 645 * holding the struct mutex, and all of the pwrite implementations 646 * want to hold it while dereferencing the user data. 647 */ 648 first_data_page = data_ptr / PAGE_SIZE; 649 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 650 num_pages = last_data_page - first_data_page + 1; 651 652 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 653 if (user_pages == NULL) 654 return -ENOMEM; 655 656 down_read(&mm->mmap_sem); 657 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 658 num_pages, 0, 0, user_pages, NULL); 659 up_read(&mm->mmap_sem); 660 if (pinned_pages < num_pages) { 661 ret = -EFAULT; 662 goto out_unpin_pages; 663 } 664 665 mutex_lock(&dev->struct_mutex); 666 ret = i915_gem_object_pin(obj, 0); 667 if (ret) 668 goto out_unlock; 669 670 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 671 if (ret) 672 goto out_unpin_object; 673 674 obj_priv = obj->driver_private; 675 offset = obj_priv->gtt_offset + args->offset; 676 677 while (remain > 0) { 678 /* Operation in this page 679 * 680 * gtt_page_base = page offset within aperture 681 * gtt_page_offset = offset within page in aperture 682 * data_page_index = page number in get_user_pages return 683 * data_page_offset = offset with data_page_index page. 684 * page_length = bytes to copy for this page 685 */ 686 gtt_page_base = offset & PAGE_MASK; 687 gtt_page_offset = offset & ~PAGE_MASK; 688 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 689 data_page_offset = data_ptr & ~PAGE_MASK; 690 691 page_length = remain; 692 if ((gtt_page_offset + page_length) > PAGE_SIZE) 693 page_length = PAGE_SIZE - gtt_page_offset; 694 if ((data_page_offset + page_length) > PAGE_SIZE) 695 page_length = PAGE_SIZE - data_page_offset; 696 697 ret = slow_kernel_write(dev_priv->mm.gtt_mapping, 698 gtt_page_base, gtt_page_offset, 699 user_pages[data_page_index], 700 data_page_offset, 701 page_length); 702 703 /* If we get a fault while copying data, then (presumably) our 704 * source page isn't available. Return the error and we'll 705 * retry in the slow path. 706 */ 707 if (ret) 708 goto out_unpin_object; 709 710 remain -= page_length; 711 offset += page_length; 712 data_ptr += page_length; 713 } 714 715 out_unpin_object: 716 i915_gem_object_unpin(obj); 717 out_unlock: 718 mutex_unlock(&dev->struct_mutex); 719 out_unpin_pages: 720 for (i = 0; i < pinned_pages; i++) 721 page_cache_release(user_pages[i]); 722 drm_free_large(user_pages); 723 724 return ret; 725 } 726 727 /** 728 * This is the fast shmem pwrite path, which attempts to directly 729 * copy_from_user into the kmapped pages backing the object. 730 */ 731 static int 732 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 733 struct drm_i915_gem_pwrite *args, 734 struct drm_file *file_priv) 735 { 736 struct drm_i915_gem_object *obj_priv = obj->driver_private; 737 ssize_t remain; 738 loff_t offset, page_base; 739 char __user *user_data; 740 int page_offset, page_length; 741 int ret; 742 743 user_data = (char __user *) (uintptr_t) args->data_ptr; 744 remain = args->size; 745 746 mutex_lock(&dev->struct_mutex); 747 748 ret = i915_gem_object_get_pages(obj); 749 if (ret != 0) 750 goto fail_unlock; 751 752 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 753 if (ret != 0) 754 goto fail_put_pages; 755 756 obj_priv = obj->driver_private; 757 offset = args->offset; 758 obj_priv->dirty = 1; 759 760 while (remain > 0) { 761 /* Operation in this page 762 * 763 * page_base = page offset within aperture 764 * page_offset = offset within page 765 * page_length = bytes to copy for this page 766 */ 767 page_base = (offset & ~(PAGE_SIZE-1)); 768 page_offset = offset & (PAGE_SIZE-1); 769 page_length = remain; 770 if ((page_offset + remain) > PAGE_SIZE) 771 page_length = PAGE_SIZE - page_offset; 772 773 ret = fast_shmem_write(obj_priv->pages, 774 page_base, page_offset, 775 user_data, page_length); 776 if (ret) 777 goto fail_put_pages; 778 779 remain -= page_length; 780 user_data += page_length; 781 offset += page_length; 782 } 783 784 fail_put_pages: 785 i915_gem_object_put_pages(obj); 786 fail_unlock: 787 mutex_unlock(&dev->struct_mutex); 788 789 return ret; 790 } 791 792 /** 793 * This is the fallback shmem pwrite path, which uses get_user_pages to pin 794 * the memory and maps it using kmap_atomic for copying. 795 * 796 * This avoids taking mmap_sem for faulting on the user's address while the 797 * struct_mutex is held. 798 */ 799 static int 800 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 801 struct drm_i915_gem_pwrite *args, 802 struct drm_file *file_priv) 803 { 804 struct drm_i915_gem_object *obj_priv = obj->driver_private; 805 struct mm_struct *mm = current->mm; 806 struct page **user_pages; 807 ssize_t remain; 808 loff_t offset, pinned_pages, i; 809 loff_t first_data_page, last_data_page, num_pages; 810 int shmem_page_index, shmem_page_offset; 811 int data_page_index, data_page_offset; 812 int page_length; 813 int ret; 814 uint64_t data_ptr = args->data_ptr; 815 int do_bit17_swizzling; 816 817 remain = args->size; 818 819 /* Pin the user pages containing the data. We can't fault while 820 * holding the struct mutex, and all of the pwrite implementations 821 * want to hold it while dereferencing the user data. 822 */ 823 first_data_page = data_ptr / PAGE_SIZE; 824 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 825 num_pages = last_data_page - first_data_page + 1; 826 827 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 828 if (user_pages == NULL) 829 return -ENOMEM; 830 831 down_read(&mm->mmap_sem); 832 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 833 num_pages, 0, 0, user_pages, NULL); 834 up_read(&mm->mmap_sem); 835 if (pinned_pages < num_pages) { 836 ret = -EFAULT; 837 goto fail_put_user_pages; 838 } 839 840 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 841 842 mutex_lock(&dev->struct_mutex); 843 844 ret = i915_gem_object_get_pages(obj); 845 if (ret != 0) 846 goto fail_unlock; 847 848 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 849 if (ret != 0) 850 goto fail_put_pages; 851 852 obj_priv = obj->driver_private; 853 offset = args->offset; 854 obj_priv->dirty = 1; 855 856 while (remain > 0) { 857 /* Operation in this page 858 * 859 * shmem_page_index = page number within shmem file 860 * shmem_page_offset = offset within page in shmem file 861 * data_page_index = page number in get_user_pages return 862 * data_page_offset = offset with data_page_index page. 863 * page_length = bytes to copy for this page 864 */ 865 shmem_page_index = offset / PAGE_SIZE; 866 shmem_page_offset = offset & ~PAGE_MASK; 867 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 868 data_page_offset = data_ptr & ~PAGE_MASK; 869 870 page_length = remain; 871 if ((shmem_page_offset + page_length) > PAGE_SIZE) 872 page_length = PAGE_SIZE - shmem_page_offset; 873 if ((data_page_offset + page_length) > PAGE_SIZE) 874 page_length = PAGE_SIZE - data_page_offset; 875 876 if (do_bit17_swizzling) { 877 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 878 shmem_page_offset, 879 user_pages[data_page_index], 880 data_page_offset, 881 page_length, 882 0); 883 } else { 884 ret = slow_shmem_copy(obj_priv->pages[shmem_page_index], 885 shmem_page_offset, 886 user_pages[data_page_index], 887 data_page_offset, 888 page_length); 889 } 890 if (ret) 891 goto fail_put_pages; 892 893 remain -= page_length; 894 data_ptr += page_length; 895 offset += page_length; 896 } 897 898 fail_put_pages: 899 i915_gem_object_put_pages(obj); 900 fail_unlock: 901 mutex_unlock(&dev->struct_mutex); 902 fail_put_user_pages: 903 for (i = 0; i < pinned_pages; i++) 904 page_cache_release(user_pages[i]); 905 drm_free_large(user_pages); 906 907 return ret; 908 } 909 910 /** 911 * Writes data to the object referenced by handle. 912 * 913 * On error, the contents of the buffer that were to be modified are undefined. 914 */ 915 int 916 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 917 struct drm_file *file_priv) 918 { 919 struct drm_i915_gem_pwrite *args = data; 920 struct drm_gem_object *obj; 921 struct drm_i915_gem_object *obj_priv; 922 int ret = 0; 923 924 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 925 if (obj == NULL) 926 return -EBADF; 927 obj_priv = obj->driver_private; 928 929 /* Bounds check destination. 930 * 931 * XXX: This could use review for overflow issues... 932 */ 933 if (args->offset > obj->size || args->size > obj->size || 934 args->offset + args->size > obj->size) { 935 drm_gem_object_unreference(obj); 936 return -EINVAL; 937 } 938 939 /* We can only do the GTT pwrite on untiled buffers, as otherwise 940 * it would end up going through the fenced access, and we'll get 941 * different detiling behavior between reading and writing. 942 * pread/pwrite currently are reading and writing from the CPU 943 * perspective, requiring manual detiling by the client. 944 */ 945 if (obj_priv->phys_obj) 946 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv); 947 else if (obj_priv->tiling_mode == I915_TILING_NONE && 948 dev->gtt_total != 0) { 949 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv); 950 if (ret == -EFAULT) { 951 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, 952 file_priv); 953 } 954 } else if (i915_gem_object_needs_bit17_swizzle(obj)) { 955 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv); 956 } else { 957 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv); 958 if (ret == -EFAULT) { 959 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, 960 file_priv); 961 } 962 } 963 964 #if WATCH_PWRITE 965 if (ret) 966 DRM_INFO("pwrite failed %d\n", ret); 967 #endif 968 969 drm_gem_object_unreference(obj); 970 971 return ret; 972 } 973 974 /** 975 * Called when user space prepares to use an object with the CPU, either 976 * through the mmap ioctl's mapping or a GTT mapping. 977 */ 978 int 979 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 980 struct drm_file *file_priv) 981 { 982 struct drm_i915_gem_set_domain *args = data; 983 struct drm_gem_object *obj; 984 uint32_t read_domains = args->read_domains; 985 uint32_t write_domain = args->write_domain; 986 int ret; 987 988 if (!(dev->driver->driver_features & DRIVER_GEM)) 989 return -ENODEV; 990 991 /* Only handle setting domains to types used by the CPU. */ 992 if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) 993 return -EINVAL; 994 995 if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) 996 return -EINVAL; 997 998 /* Having something in the write domain implies it's in the read 999 * domain, and only that read domain. Enforce that in the request. 1000 */ 1001 if (write_domain != 0 && read_domains != write_domain) 1002 return -EINVAL; 1003 1004 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1005 if (obj == NULL) 1006 return -EBADF; 1007 1008 mutex_lock(&dev->struct_mutex); 1009 #if WATCH_BUF 1010 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n", 1011 obj, obj->size, read_domains, write_domain); 1012 #endif 1013 if (read_domains & I915_GEM_DOMAIN_GTT) { 1014 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1015 1016 /* Silently promote "you're not bound, there was nothing to do" 1017 * to success, since the client was just asking us to 1018 * make sure everything was done. 1019 */ 1020 if (ret == -EINVAL) 1021 ret = 0; 1022 } else { 1023 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1024 } 1025 1026 drm_gem_object_unreference(obj); 1027 mutex_unlock(&dev->struct_mutex); 1028 return ret; 1029 } 1030 1031 /** 1032 * Called when user space has done writes to this buffer 1033 */ 1034 int 1035 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1036 struct drm_file *file_priv) 1037 { 1038 struct drm_i915_gem_sw_finish *args = data; 1039 struct drm_gem_object *obj; 1040 struct drm_i915_gem_object *obj_priv; 1041 int ret = 0; 1042 1043 if (!(dev->driver->driver_features & DRIVER_GEM)) 1044 return -ENODEV; 1045 1046 mutex_lock(&dev->struct_mutex); 1047 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1048 if (obj == NULL) { 1049 mutex_unlock(&dev->struct_mutex); 1050 return -EBADF; 1051 } 1052 1053 #if WATCH_BUF 1054 DRM_INFO("%s: sw_finish %d (%p %d)\n", 1055 __func__, args->handle, obj, obj->size); 1056 #endif 1057 obj_priv = obj->driver_private; 1058 1059 /* Pinned buffers may be scanout, so flush the cache */ 1060 if (obj_priv->pin_count) 1061 i915_gem_object_flush_cpu_write_domain(obj); 1062 1063 drm_gem_object_unreference(obj); 1064 mutex_unlock(&dev->struct_mutex); 1065 return ret; 1066 } 1067 1068 /** 1069 * Maps the contents of an object, returning the address it is mapped 1070 * into. 1071 * 1072 * While the mapping holds a reference on the contents of the object, it doesn't 1073 * imply a ref on the object itself. 1074 */ 1075 int 1076 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1077 struct drm_file *file_priv) 1078 { 1079 struct drm_i915_gem_mmap *args = data; 1080 struct drm_gem_object *obj; 1081 loff_t offset; 1082 unsigned long addr; 1083 1084 if (!(dev->driver->driver_features & DRIVER_GEM)) 1085 return -ENODEV; 1086 1087 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1088 if (obj == NULL) 1089 return -EBADF; 1090 1091 offset = args->offset; 1092 1093 down_write(¤t->mm->mmap_sem); 1094 addr = do_mmap(obj->filp, 0, args->size, 1095 PROT_READ | PROT_WRITE, MAP_SHARED, 1096 args->offset); 1097 up_write(¤t->mm->mmap_sem); 1098 mutex_lock(&dev->struct_mutex); 1099 drm_gem_object_unreference(obj); 1100 mutex_unlock(&dev->struct_mutex); 1101 if (IS_ERR((void *)addr)) 1102 return addr; 1103 1104 args->addr_ptr = (uint64_t) addr; 1105 1106 return 0; 1107 } 1108 1109 /** 1110 * i915_gem_fault - fault a page into the GTT 1111 * vma: VMA in question 1112 * vmf: fault info 1113 * 1114 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 1115 * from userspace. The fault handler takes care of binding the object to 1116 * the GTT (if needed), allocating and programming a fence register (again, 1117 * only if needed based on whether the old reg is still valid or the object 1118 * is tiled) and inserting a new PTE into the faulting process. 1119 * 1120 * Note that the faulting process may involve evicting existing objects 1121 * from the GTT and/or fence registers to make room. So performance may 1122 * suffer if the GTT working set is large or there are few fence registers 1123 * left. 1124 */ 1125 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1126 { 1127 struct drm_gem_object *obj = vma->vm_private_data; 1128 struct drm_device *dev = obj->dev; 1129 struct drm_i915_private *dev_priv = dev->dev_private; 1130 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1131 pgoff_t page_offset; 1132 unsigned long pfn; 1133 int ret = 0; 1134 bool write = !!(vmf->flags & FAULT_FLAG_WRITE); 1135 1136 /* We don't use vmf->pgoff since that has the fake offset */ 1137 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >> 1138 PAGE_SHIFT; 1139 1140 /* Now bind it into the GTT if needed */ 1141 mutex_lock(&dev->struct_mutex); 1142 if (!obj_priv->gtt_space) { 1143 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment); 1144 if (ret) { 1145 mutex_unlock(&dev->struct_mutex); 1146 return VM_FAULT_SIGBUS; 1147 } 1148 1149 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1150 if (ret) { 1151 mutex_unlock(&dev->struct_mutex); 1152 return VM_FAULT_SIGBUS; 1153 } 1154 1155 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 1156 } 1157 1158 /* Need a new fence register? */ 1159 if (obj_priv->fence_reg == I915_FENCE_REG_NONE && 1160 obj_priv->tiling_mode != I915_TILING_NONE) { 1161 ret = i915_gem_object_get_fence_reg(obj, write); 1162 if (ret) { 1163 mutex_unlock(&dev->struct_mutex); 1164 return VM_FAULT_SIGBUS; 1165 } 1166 } 1167 1168 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + 1169 page_offset; 1170 1171 /* Finally, remap it using the new GTT offset */ 1172 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn); 1173 1174 mutex_unlock(&dev->struct_mutex); 1175 1176 switch (ret) { 1177 case -ENOMEM: 1178 case -EAGAIN: 1179 return VM_FAULT_OOM; 1180 case -EFAULT: 1181 case -EINVAL: 1182 return VM_FAULT_SIGBUS; 1183 default: 1184 return VM_FAULT_NOPAGE; 1185 } 1186 } 1187 1188 /** 1189 * i915_gem_create_mmap_offset - create a fake mmap offset for an object 1190 * @obj: obj in question 1191 * 1192 * GEM memory mapping works by handing back to userspace a fake mmap offset 1193 * it can use in a subsequent mmap(2) call. The DRM core code then looks 1194 * up the object based on the offset and sets up the various memory mapping 1195 * structures. 1196 * 1197 * This routine allocates and attaches a fake offset for @obj. 1198 */ 1199 static int 1200 i915_gem_create_mmap_offset(struct drm_gem_object *obj) 1201 { 1202 struct drm_device *dev = obj->dev; 1203 struct drm_gem_mm *mm = dev->mm_private; 1204 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1205 struct drm_map_list *list; 1206 struct drm_local_map *map; 1207 int ret = 0; 1208 1209 /* Set the object up for mmap'ing */ 1210 list = &obj->map_list; 1211 list->map = drm_calloc(1, sizeof(struct drm_map_list), 1212 DRM_MEM_DRIVER); 1213 if (!list->map) 1214 return -ENOMEM; 1215 1216 map = list->map; 1217 map->type = _DRM_GEM; 1218 map->size = obj->size; 1219 map->handle = obj; 1220 1221 /* Get a DRM GEM mmap offset allocated... */ 1222 list->file_offset_node = drm_mm_search_free(&mm->offset_manager, 1223 obj->size / PAGE_SIZE, 0, 0); 1224 if (!list->file_offset_node) { 1225 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name); 1226 ret = -ENOMEM; 1227 goto out_free_list; 1228 } 1229 1230 list->file_offset_node = drm_mm_get_block(list->file_offset_node, 1231 obj->size / PAGE_SIZE, 0); 1232 if (!list->file_offset_node) { 1233 ret = -ENOMEM; 1234 goto out_free_list; 1235 } 1236 1237 list->hash.key = list->file_offset_node->start; 1238 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) { 1239 DRM_ERROR("failed to add to map hash\n"); 1240 goto out_free_mm; 1241 } 1242 1243 /* By now we should be all set, any drm_mmap request on the offset 1244 * below will get to our mmap & fault handler */ 1245 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT; 1246 1247 return 0; 1248 1249 out_free_mm: 1250 drm_mm_put_block(list->file_offset_node); 1251 out_free_list: 1252 drm_free(list->map, sizeof(struct drm_map_list), DRM_MEM_DRIVER); 1253 1254 return ret; 1255 } 1256 1257 static void 1258 i915_gem_free_mmap_offset(struct drm_gem_object *obj) 1259 { 1260 struct drm_device *dev = obj->dev; 1261 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1262 struct drm_gem_mm *mm = dev->mm_private; 1263 struct drm_map_list *list; 1264 1265 list = &obj->map_list; 1266 drm_ht_remove_item(&mm->offset_hash, &list->hash); 1267 1268 if (list->file_offset_node) { 1269 drm_mm_put_block(list->file_offset_node); 1270 list->file_offset_node = NULL; 1271 } 1272 1273 if (list->map) { 1274 drm_free(list->map, sizeof(struct drm_map), DRM_MEM_DRIVER); 1275 list->map = NULL; 1276 } 1277 1278 obj_priv->mmap_offset = 0; 1279 } 1280 1281 /** 1282 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1283 * @obj: object to check 1284 * 1285 * Return the required GTT alignment for an object, taking into account 1286 * potential fence register mapping if needed. 1287 */ 1288 static uint32_t 1289 i915_gem_get_gtt_alignment(struct drm_gem_object *obj) 1290 { 1291 struct drm_device *dev = obj->dev; 1292 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1293 int start, i; 1294 1295 /* 1296 * Minimum alignment is 4k (GTT page size), but might be greater 1297 * if a fence register is needed for the object. 1298 */ 1299 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE) 1300 return 4096; 1301 1302 /* 1303 * Previous chips need to be aligned to the size of the smallest 1304 * fence register that can contain the object. 1305 */ 1306 if (IS_I9XX(dev)) 1307 start = 1024*1024; 1308 else 1309 start = 512*1024; 1310 1311 for (i = start; i < obj->size; i <<= 1) 1312 ; 1313 1314 return i; 1315 } 1316 1317 /** 1318 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1319 * @dev: DRM device 1320 * @data: GTT mapping ioctl data 1321 * @file_priv: GEM object info 1322 * 1323 * Simply returns the fake offset to userspace so it can mmap it. 1324 * The mmap call will end up in drm_gem_mmap(), which will set things 1325 * up so we can get faults in the handler above. 1326 * 1327 * The fault handler will take care of binding the object into the GTT 1328 * (since it may have been evicted to make room for something), allocating 1329 * a fence register, and mapping the appropriate aperture address into 1330 * userspace. 1331 */ 1332 int 1333 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1334 struct drm_file *file_priv) 1335 { 1336 struct drm_i915_gem_mmap_gtt *args = data; 1337 struct drm_i915_private *dev_priv = dev->dev_private; 1338 struct drm_gem_object *obj; 1339 struct drm_i915_gem_object *obj_priv; 1340 int ret; 1341 1342 if (!(dev->driver->driver_features & DRIVER_GEM)) 1343 return -ENODEV; 1344 1345 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1346 if (obj == NULL) 1347 return -EBADF; 1348 1349 mutex_lock(&dev->struct_mutex); 1350 1351 obj_priv = obj->driver_private; 1352 1353 if (!obj_priv->mmap_offset) { 1354 ret = i915_gem_create_mmap_offset(obj); 1355 if (ret) { 1356 drm_gem_object_unreference(obj); 1357 mutex_unlock(&dev->struct_mutex); 1358 return ret; 1359 } 1360 } 1361 1362 args->offset = obj_priv->mmap_offset; 1363 1364 obj_priv->gtt_alignment = i915_gem_get_gtt_alignment(obj); 1365 1366 /* Make sure the alignment is correct for fence regs etc */ 1367 if (obj_priv->agp_mem && 1368 (obj_priv->gtt_offset & (obj_priv->gtt_alignment - 1))) { 1369 drm_gem_object_unreference(obj); 1370 mutex_unlock(&dev->struct_mutex); 1371 return -EINVAL; 1372 } 1373 1374 /* 1375 * Pull it into the GTT so that we have a page list (makes the 1376 * initial fault faster and any subsequent flushing possible). 1377 */ 1378 if (!obj_priv->agp_mem) { 1379 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment); 1380 if (ret) { 1381 drm_gem_object_unreference(obj); 1382 mutex_unlock(&dev->struct_mutex); 1383 return ret; 1384 } 1385 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 1386 } 1387 1388 drm_gem_object_unreference(obj); 1389 mutex_unlock(&dev->struct_mutex); 1390 1391 return 0; 1392 } 1393 1394 void 1395 i915_gem_object_put_pages(struct drm_gem_object *obj) 1396 { 1397 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1398 int page_count = obj->size / PAGE_SIZE; 1399 int i; 1400 1401 BUG_ON(obj_priv->pages_refcount == 0); 1402 1403 if (--obj_priv->pages_refcount != 0) 1404 return; 1405 1406 if (obj_priv->tiling_mode != I915_TILING_NONE) 1407 i915_gem_object_save_bit_17_swizzle(obj); 1408 1409 for (i = 0; i < page_count; i++) 1410 if (obj_priv->pages[i] != NULL) { 1411 if (obj_priv->dirty) 1412 set_page_dirty(obj_priv->pages[i]); 1413 mark_page_accessed(obj_priv->pages[i]); 1414 page_cache_release(obj_priv->pages[i]); 1415 } 1416 obj_priv->dirty = 0; 1417 1418 drm_free_large(obj_priv->pages); 1419 obj_priv->pages = NULL; 1420 } 1421 1422 static void 1423 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno) 1424 { 1425 struct drm_device *dev = obj->dev; 1426 drm_i915_private_t *dev_priv = dev->dev_private; 1427 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1428 1429 /* Add a reference if we're newly entering the active list. */ 1430 if (!obj_priv->active) { 1431 drm_gem_object_reference(obj); 1432 obj_priv->active = 1; 1433 } 1434 /* Move from whatever list we were on to the tail of execution. */ 1435 spin_lock(&dev_priv->mm.active_list_lock); 1436 list_move_tail(&obj_priv->list, 1437 &dev_priv->mm.active_list); 1438 spin_unlock(&dev_priv->mm.active_list_lock); 1439 obj_priv->last_rendering_seqno = seqno; 1440 } 1441 1442 static void 1443 i915_gem_object_move_to_flushing(struct drm_gem_object *obj) 1444 { 1445 struct drm_device *dev = obj->dev; 1446 drm_i915_private_t *dev_priv = dev->dev_private; 1447 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1448 1449 BUG_ON(!obj_priv->active); 1450 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list); 1451 obj_priv->last_rendering_seqno = 0; 1452 } 1453 1454 static void 1455 i915_gem_object_move_to_inactive(struct drm_gem_object *obj) 1456 { 1457 struct drm_device *dev = obj->dev; 1458 drm_i915_private_t *dev_priv = dev->dev_private; 1459 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1460 1461 i915_verify_inactive(dev, __FILE__, __LINE__); 1462 if (obj_priv->pin_count != 0) 1463 list_del_init(&obj_priv->list); 1464 else 1465 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 1466 1467 obj_priv->last_rendering_seqno = 0; 1468 if (obj_priv->active) { 1469 obj_priv->active = 0; 1470 drm_gem_object_unreference(obj); 1471 } 1472 i915_verify_inactive(dev, __FILE__, __LINE__); 1473 } 1474 1475 /** 1476 * Creates a new sequence number, emitting a write of it to the status page 1477 * plus an interrupt, which will trigger i915_user_interrupt_handler. 1478 * 1479 * Must be called with struct_lock held. 1480 * 1481 * Returned sequence numbers are nonzero on success. 1482 */ 1483 static uint32_t 1484 i915_add_request(struct drm_device *dev, uint32_t flush_domains) 1485 { 1486 drm_i915_private_t *dev_priv = dev->dev_private; 1487 struct drm_i915_gem_request *request; 1488 uint32_t seqno; 1489 int was_empty; 1490 RING_LOCALS; 1491 1492 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER); 1493 if (request == NULL) 1494 return 0; 1495 1496 /* Grab the seqno we're going to make this request be, and bump the 1497 * next (skipping 0 so it can be the reserved no-seqno value). 1498 */ 1499 seqno = dev_priv->mm.next_gem_seqno; 1500 dev_priv->mm.next_gem_seqno++; 1501 if (dev_priv->mm.next_gem_seqno == 0) 1502 dev_priv->mm.next_gem_seqno++; 1503 1504 BEGIN_LP_RING(4); 1505 OUT_RING(MI_STORE_DWORD_INDEX); 1506 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 1507 OUT_RING(seqno); 1508 1509 OUT_RING(MI_USER_INTERRUPT); 1510 ADVANCE_LP_RING(); 1511 1512 DRM_DEBUG("%d\n", seqno); 1513 1514 request->seqno = seqno; 1515 request->emitted_jiffies = jiffies; 1516 was_empty = list_empty(&dev_priv->mm.request_list); 1517 list_add_tail(&request->list, &dev_priv->mm.request_list); 1518 1519 /* Associate any objects on the flushing list matching the write 1520 * domain we're flushing with our flush. 1521 */ 1522 if (flush_domains != 0) { 1523 struct drm_i915_gem_object *obj_priv, *next; 1524 1525 list_for_each_entry_safe(obj_priv, next, 1526 &dev_priv->mm.flushing_list, list) { 1527 struct drm_gem_object *obj = obj_priv->obj; 1528 1529 if ((obj->write_domain & flush_domains) == 1530 obj->write_domain) { 1531 obj->write_domain = 0; 1532 i915_gem_object_move_to_active(obj, seqno); 1533 } 1534 } 1535 1536 } 1537 1538 if (was_empty && !dev_priv->mm.suspended) 1539 schedule_delayed_work(&dev_priv->mm.retire_work, HZ); 1540 return seqno; 1541 } 1542 1543 /** 1544 * Command execution barrier 1545 * 1546 * Ensures that all commands in the ring are finished 1547 * before signalling the CPU 1548 */ 1549 static uint32_t 1550 i915_retire_commands(struct drm_device *dev) 1551 { 1552 drm_i915_private_t *dev_priv = dev->dev_private; 1553 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; 1554 uint32_t flush_domains = 0; 1555 RING_LOCALS; 1556 1557 /* The sampler always gets flushed on i965 (sigh) */ 1558 if (IS_I965G(dev)) 1559 flush_domains |= I915_GEM_DOMAIN_SAMPLER; 1560 BEGIN_LP_RING(2); 1561 OUT_RING(cmd); 1562 OUT_RING(0); /* noop */ 1563 ADVANCE_LP_RING(); 1564 return flush_domains; 1565 } 1566 1567 /** 1568 * Moves buffers associated only with the given active seqno from the active 1569 * to inactive list, potentially freeing them. 1570 */ 1571 static void 1572 i915_gem_retire_request(struct drm_device *dev, 1573 struct drm_i915_gem_request *request) 1574 { 1575 drm_i915_private_t *dev_priv = dev->dev_private; 1576 1577 /* Move any buffers on the active list that are no longer referenced 1578 * by the ringbuffer to the flushing/inactive lists as appropriate. 1579 */ 1580 spin_lock(&dev_priv->mm.active_list_lock); 1581 while (!list_empty(&dev_priv->mm.active_list)) { 1582 struct drm_gem_object *obj; 1583 struct drm_i915_gem_object *obj_priv; 1584 1585 obj_priv = list_first_entry(&dev_priv->mm.active_list, 1586 struct drm_i915_gem_object, 1587 list); 1588 obj = obj_priv->obj; 1589 1590 /* If the seqno being retired doesn't match the oldest in the 1591 * list, then the oldest in the list must still be newer than 1592 * this seqno. 1593 */ 1594 if (obj_priv->last_rendering_seqno != request->seqno) 1595 goto out; 1596 1597 #if WATCH_LRU 1598 DRM_INFO("%s: retire %d moves to inactive list %p\n", 1599 __func__, request->seqno, obj); 1600 #endif 1601 1602 if (obj->write_domain != 0) 1603 i915_gem_object_move_to_flushing(obj); 1604 else { 1605 /* Take a reference on the object so it won't be 1606 * freed while the spinlock is held. The list 1607 * protection for this spinlock is safe when breaking 1608 * the lock like this since the next thing we do 1609 * is just get the head of the list again. 1610 */ 1611 drm_gem_object_reference(obj); 1612 i915_gem_object_move_to_inactive(obj); 1613 spin_unlock(&dev_priv->mm.active_list_lock); 1614 drm_gem_object_unreference(obj); 1615 spin_lock(&dev_priv->mm.active_list_lock); 1616 } 1617 } 1618 out: 1619 spin_unlock(&dev_priv->mm.active_list_lock); 1620 } 1621 1622 /** 1623 * Returns true if seq1 is later than seq2. 1624 */ 1625 static int 1626 i915_seqno_passed(uint32_t seq1, uint32_t seq2) 1627 { 1628 return (int32_t)(seq1 - seq2) >= 0; 1629 } 1630 1631 uint32_t 1632 i915_get_gem_seqno(struct drm_device *dev) 1633 { 1634 drm_i915_private_t *dev_priv = dev->dev_private; 1635 1636 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX); 1637 } 1638 1639 /** 1640 * This function clears the request list as sequence numbers are passed. 1641 */ 1642 void 1643 i915_gem_retire_requests(struct drm_device *dev) 1644 { 1645 drm_i915_private_t *dev_priv = dev->dev_private; 1646 uint32_t seqno; 1647 1648 if (!dev_priv->hw_status_page) 1649 return; 1650 1651 seqno = i915_get_gem_seqno(dev); 1652 1653 while (!list_empty(&dev_priv->mm.request_list)) { 1654 struct drm_i915_gem_request *request; 1655 uint32_t retiring_seqno; 1656 1657 request = list_first_entry(&dev_priv->mm.request_list, 1658 struct drm_i915_gem_request, 1659 list); 1660 retiring_seqno = request->seqno; 1661 1662 if (i915_seqno_passed(seqno, retiring_seqno) || 1663 dev_priv->mm.wedged) { 1664 i915_gem_retire_request(dev, request); 1665 1666 list_del(&request->list); 1667 drm_free(request, sizeof(*request), DRM_MEM_DRIVER); 1668 } else 1669 break; 1670 } 1671 } 1672 1673 void 1674 i915_gem_retire_work_handler(struct work_struct *work) 1675 { 1676 drm_i915_private_t *dev_priv; 1677 struct drm_device *dev; 1678 1679 dev_priv = container_of(work, drm_i915_private_t, 1680 mm.retire_work.work); 1681 dev = dev_priv->dev; 1682 1683 mutex_lock(&dev->struct_mutex); 1684 i915_gem_retire_requests(dev); 1685 if (!dev_priv->mm.suspended && 1686 !list_empty(&dev_priv->mm.request_list)) 1687 schedule_delayed_work(&dev_priv->mm.retire_work, HZ); 1688 mutex_unlock(&dev->struct_mutex); 1689 } 1690 1691 /** 1692 * Waits for a sequence number to be signaled, and cleans up the 1693 * request and object lists appropriately for that event. 1694 */ 1695 static int 1696 i915_wait_request(struct drm_device *dev, uint32_t seqno) 1697 { 1698 drm_i915_private_t *dev_priv = dev->dev_private; 1699 u32 ier; 1700 int ret = 0; 1701 1702 BUG_ON(seqno == 0); 1703 1704 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) { 1705 ier = I915_READ(IER); 1706 if (!ier) { 1707 DRM_ERROR("something (likely vbetool) disabled " 1708 "interrupts, re-enabling\n"); 1709 i915_driver_irq_preinstall(dev); 1710 i915_driver_irq_postinstall(dev); 1711 } 1712 1713 dev_priv->mm.waiting_gem_seqno = seqno; 1714 i915_user_irq_get(dev); 1715 ret = wait_event_interruptible(dev_priv->irq_queue, 1716 i915_seqno_passed(i915_get_gem_seqno(dev), 1717 seqno) || 1718 dev_priv->mm.wedged); 1719 i915_user_irq_put(dev); 1720 dev_priv->mm.waiting_gem_seqno = 0; 1721 } 1722 if (dev_priv->mm.wedged) 1723 ret = -EIO; 1724 1725 if (ret && ret != -ERESTARTSYS) 1726 DRM_ERROR("%s returns %d (awaiting %d at %d)\n", 1727 __func__, ret, seqno, i915_get_gem_seqno(dev)); 1728 1729 /* Directly dispatch request retiring. While we have the work queue 1730 * to handle this, the waiter on a request often wants an associated 1731 * buffer to have made it to the inactive list, and we would need 1732 * a separate wait queue to handle that. 1733 */ 1734 if (ret == 0) 1735 i915_gem_retire_requests(dev); 1736 1737 return ret; 1738 } 1739 1740 static void 1741 i915_gem_flush(struct drm_device *dev, 1742 uint32_t invalidate_domains, 1743 uint32_t flush_domains) 1744 { 1745 drm_i915_private_t *dev_priv = dev->dev_private; 1746 uint32_t cmd; 1747 RING_LOCALS; 1748 1749 #if WATCH_EXEC 1750 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__, 1751 invalidate_domains, flush_domains); 1752 #endif 1753 1754 if (flush_domains & I915_GEM_DOMAIN_CPU) 1755 drm_agp_chipset_flush(dev); 1756 1757 if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU | 1758 I915_GEM_DOMAIN_GTT)) { 1759 /* 1760 * read/write caches: 1761 * 1762 * I915_GEM_DOMAIN_RENDER is always invalidated, but is 1763 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is 1764 * also flushed at 2d versus 3d pipeline switches. 1765 * 1766 * read-only caches: 1767 * 1768 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if 1769 * MI_READ_FLUSH is set, and is always flushed on 965. 1770 * 1771 * I915_GEM_DOMAIN_COMMAND may not exist? 1772 * 1773 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is 1774 * invalidated when MI_EXE_FLUSH is set. 1775 * 1776 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is 1777 * invalidated with every MI_FLUSH. 1778 * 1779 * TLBs: 1780 * 1781 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND 1782 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and 1783 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER 1784 * are flushed at any MI_FLUSH. 1785 */ 1786 1787 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; 1788 if ((invalidate_domains|flush_domains) & 1789 I915_GEM_DOMAIN_RENDER) 1790 cmd &= ~MI_NO_WRITE_FLUSH; 1791 if (!IS_I965G(dev)) { 1792 /* 1793 * On the 965, the sampler cache always gets flushed 1794 * and this bit is reserved. 1795 */ 1796 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER) 1797 cmd |= MI_READ_FLUSH; 1798 } 1799 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) 1800 cmd |= MI_EXE_FLUSH; 1801 1802 #if WATCH_EXEC 1803 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd); 1804 #endif 1805 BEGIN_LP_RING(2); 1806 OUT_RING(cmd); 1807 OUT_RING(0); /* noop */ 1808 ADVANCE_LP_RING(); 1809 } 1810 } 1811 1812 /** 1813 * Ensures that all rendering to the object has completed and the object is 1814 * safe to unbind from the GTT or access from the CPU. 1815 */ 1816 static int 1817 i915_gem_object_wait_rendering(struct drm_gem_object *obj) 1818 { 1819 struct drm_device *dev = obj->dev; 1820 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1821 int ret; 1822 1823 /* This function only exists to support waiting for existing rendering, 1824 * not for emitting required flushes. 1825 */ 1826 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0); 1827 1828 /* If there is rendering queued on the buffer being evicted, wait for 1829 * it. 1830 */ 1831 if (obj_priv->active) { 1832 #if WATCH_BUF 1833 DRM_INFO("%s: object %p wait for seqno %08x\n", 1834 __func__, obj, obj_priv->last_rendering_seqno); 1835 #endif 1836 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno); 1837 if (ret != 0) 1838 return ret; 1839 } 1840 1841 return 0; 1842 } 1843 1844 /** 1845 * Unbinds an object from the GTT aperture. 1846 */ 1847 int 1848 i915_gem_object_unbind(struct drm_gem_object *obj) 1849 { 1850 struct drm_device *dev = obj->dev; 1851 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1852 loff_t offset; 1853 int ret = 0; 1854 1855 #if WATCH_BUF 1856 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj); 1857 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space); 1858 #endif 1859 if (obj_priv->gtt_space == NULL) 1860 return 0; 1861 1862 if (obj_priv->pin_count != 0) { 1863 DRM_ERROR("Attempting to unbind pinned buffer\n"); 1864 return -EINVAL; 1865 } 1866 1867 /* Move the object to the CPU domain to ensure that 1868 * any possible CPU writes while it's not in the GTT 1869 * are flushed when we go to remap it. This will 1870 * also ensure that all pending GPU writes are finished 1871 * before we unbind. 1872 */ 1873 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 1874 if (ret) { 1875 if (ret != -ERESTARTSYS) 1876 DRM_ERROR("set_domain failed: %d\n", ret); 1877 return ret; 1878 } 1879 1880 if (obj_priv->agp_mem != NULL) { 1881 drm_unbind_agp(obj_priv->agp_mem); 1882 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE); 1883 obj_priv->agp_mem = NULL; 1884 } 1885 1886 BUG_ON(obj_priv->active); 1887 1888 /* blow away mappings if mapped through GTT */ 1889 offset = ((loff_t) obj->map_list.hash.key) << PAGE_SHIFT; 1890 if (dev->dev_mapping) 1891 unmap_mapping_range(dev->dev_mapping, offset, obj->size, 1); 1892 1893 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) 1894 i915_gem_clear_fence_reg(obj); 1895 1896 i915_gem_object_put_pages(obj); 1897 1898 if (obj_priv->gtt_space) { 1899 atomic_dec(&dev->gtt_count); 1900 atomic_sub(obj->size, &dev->gtt_memory); 1901 1902 drm_mm_put_block(obj_priv->gtt_space); 1903 obj_priv->gtt_space = NULL; 1904 } 1905 1906 /* Remove ourselves from the LRU list if present. */ 1907 if (!list_empty(&obj_priv->list)) 1908 list_del_init(&obj_priv->list); 1909 1910 return 0; 1911 } 1912 1913 static int 1914 i915_gem_evict_something(struct drm_device *dev) 1915 { 1916 drm_i915_private_t *dev_priv = dev->dev_private; 1917 struct drm_gem_object *obj; 1918 struct drm_i915_gem_object *obj_priv; 1919 int ret = 0; 1920 1921 for (;;) { 1922 /* If there's an inactive buffer available now, grab it 1923 * and be done. 1924 */ 1925 if (!list_empty(&dev_priv->mm.inactive_list)) { 1926 obj_priv = list_first_entry(&dev_priv->mm.inactive_list, 1927 struct drm_i915_gem_object, 1928 list); 1929 obj = obj_priv->obj; 1930 BUG_ON(obj_priv->pin_count != 0); 1931 #if WATCH_LRU 1932 DRM_INFO("%s: evicting %p\n", __func__, obj); 1933 #endif 1934 BUG_ON(obj_priv->active); 1935 1936 /* Wait on the rendering and unbind the buffer. */ 1937 ret = i915_gem_object_unbind(obj); 1938 break; 1939 } 1940 1941 /* If we didn't get anything, but the ring is still processing 1942 * things, wait for one of those things to finish and hopefully 1943 * leave us a buffer to evict. 1944 */ 1945 if (!list_empty(&dev_priv->mm.request_list)) { 1946 struct drm_i915_gem_request *request; 1947 1948 request = list_first_entry(&dev_priv->mm.request_list, 1949 struct drm_i915_gem_request, 1950 list); 1951 1952 ret = i915_wait_request(dev, request->seqno); 1953 if (ret) 1954 break; 1955 1956 /* if waiting caused an object to become inactive, 1957 * then loop around and wait for it. Otherwise, we 1958 * assume that waiting freed and unbound something, 1959 * so there should now be some space in the GTT 1960 */ 1961 if (!list_empty(&dev_priv->mm.inactive_list)) 1962 continue; 1963 break; 1964 } 1965 1966 /* If we didn't have anything on the request list but there 1967 * are buffers awaiting a flush, emit one and try again. 1968 * When we wait on it, those buffers waiting for that flush 1969 * will get moved to inactive. 1970 */ 1971 if (!list_empty(&dev_priv->mm.flushing_list)) { 1972 obj_priv = list_first_entry(&dev_priv->mm.flushing_list, 1973 struct drm_i915_gem_object, 1974 list); 1975 obj = obj_priv->obj; 1976 1977 i915_gem_flush(dev, 1978 obj->write_domain, 1979 obj->write_domain); 1980 i915_add_request(dev, obj->write_domain); 1981 1982 obj = NULL; 1983 continue; 1984 } 1985 1986 DRM_ERROR("inactive empty %d request empty %d " 1987 "flushing empty %d\n", 1988 list_empty(&dev_priv->mm.inactive_list), 1989 list_empty(&dev_priv->mm.request_list), 1990 list_empty(&dev_priv->mm.flushing_list)); 1991 /* If we didn't do any of the above, there's nothing to be done 1992 * and we just can't fit it in. 1993 */ 1994 return -ENOMEM; 1995 } 1996 return ret; 1997 } 1998 1999 static int 2000 i915_gem_evict_everything(struct drm_device *dev) 2001 { 2002 int ret; 2003 2004 for (;;) { 2005 ret = i915_gem_evict_something(dev); 2006 if (ret != 0) 2007 break; 2008 } 2009 if (ret == -ENOMEM) 2010 return 0; 2011 return ret; 2012 } 2013 2014 int 2015 i915_gem_object_get_pages(struct drm_gem_object *obj) 2016 { 2017 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2018 int page_count, i; 2019 struct address_space *mapping; 2020 struct inode *inode; 2021 struct page *page; 2022 int ret; 2023 2024 if (obj_priv->pages_refcount++ != 0) 2025 return 0; 2026 2027 /* Get the list of pages out of our struct file. They'll be pinned 2028 * at this point until we release them. 2029 */ 2030 page_count = obj->size / PAGE_SIZE; 2031 BUG_ON(obj_priv->pages != NULL); 2032 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *)); 2033 if (obj_priv->pages == NULL) { 2034 DRM_ERROR("Faled to allocate page list\n"); 2035 obj_priv->pages_refcount--; 2036 return -ENOMEM; 2037 } 2038 2039 inode = obj->filp->f_path.dentry->d_inode; 2040 mapping = inode->i_mapping; 2041 for (i = 0; i < page_count; i++) { 2042 page = read_mapping_page(mapping, i, NULL); 2043 if (IS_ERR(page)) { 2044 ret = PTR_ERR(page); 2045 DRM_ERROR("read_mapping_page failed: %d\n", ret); 2046 i915_gem_object_put_pages(obj); 2047 return ret; 2048 } 2049 obj_priv->pages[i] = page; 2050 } 2051 2052 if (obj_priv->tiling_mode != I915_TILING_NONE) 2053 i915_gem_object_do_bit_17_swizzle(obj); 2054 2055 return 0; 2056 } 2057 2058 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg) 2059 { 2060 struct drm_gem_object *obj = reg->obj; 2061 struct drm_device *dev = obj->dev; 2062 drm_i915_private_t *dev_priv = dev->dev_private; 2063 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2064 int regnum = obj_priv->fence_reg; 2065 uint64_t val; 2066 2067 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2068 0xfffff000) << 32; 2069 val |= obj_priv->gtt_offset & 0xfffff000; 2070 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2071 if (obj_priv->tiling_mode == I915_TILING_Y) 2072 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2073 val |= I965_FENCE_REG_VALID; 2074 2075 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val); 2076 } 2077 2078 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg) 2079 { 2080 struct drm_gem_object *obj = reg->obj; 2081 struct drm_device *dev = obj->dev; 2082 drm_i915_private_t *dev_priv = dev->dev_private; 2083 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2084 int regnum = obj_priv->fence_reg; 2085 int tile_width; 2086 uint32_t fence_reg, val; 2087 uint32_t pitch_val; 2088 2089 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) || 2090 (obj_priv->gtt_offset & (obj->size - 1))) { 2091 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n", 2092 __func__, obj_priv->gtt_offset, obj->size); 2093 return; 2094 } 2095 2096 if (obj_priv->tiling_mode == I915_TILING_Y && 2097 HAS_128_BYTE_Y_TILING(dev)) 2098 tile_width = 128; 2099 else 2100 tile_width = 512; 2101 2102 /* Note: pitch better be a power of two tile widths */ 2103 pitch_val = obj_priv->stride / tile_width; 2104 pitch_val = ffs(pitch_val) - 1; 2105 2106 val = obj_priv->gtt_offset; 2107 if (obj_priv->tiling_mode == I915_TILING_Y) 2108 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2109 val |= I915_FENCE_SIZE_BITS(obj->size); 2110 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2111 val |= I830_FENCE_REG_VALID; 2112 2113 if (regnum < 8) 2114 fence_reg = FENCE_REG_830_0 + (regnum * 4); 2115 else 2116 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4); 2117 I915_WRITE(fence_reg, val); 2118 } 2119 2120 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg) 2121 { 2122 struct drm_gem_object *obj = reg->obj; 2123 struct drm_device *dev = obj->dev; 2124 drm_i915_private_t *dev_priv = dev->dev_private; 2125 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2126 int regnum = obj_priv->fence_reg; 2127 uint32_t val; 2128 uint32_t pitch_val; 2129 uint32_t fence_size_bits; 2130 2131 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) || 2132 (obj_priv->gtt_offset & (obj->size - 1))) { 2133 WARN(1, "%s: object 0x%08x not 512K or size aligned\n", 2134 __func__, obj_priv->gtt_offset); 2135 return; 2136 } 2137 2138 pitch_val = obj_priv->stride / 128; 2139 pitch_val = ffs(pitch_val) - 1; 2140 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL); 2141 2142 val = obj_priv->gtt_offset; 2143 if (obj_priv->tiling_mode == I915_TILING_Y) 2144 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2145 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size); 2146 WARN_ON(fence_size_bits & ~0x00000f00); 2147 val |= fence_size_bits; 2148 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2149 val |= I830_FENCE_REG_VALID; 2150 2151 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val); 2152 2153 } 2154 2155 /** 2156 * i915_gem_object_get_fence_reg - set up a fence reg for an object 2157 * @obj: object to map through a fence reg 2158 * @write: object is about to be written 2159 * 2160 * When mapping objects through the GTT, userspace wants to be able to write 2161 * to them without having to worry about swizzling if the object is tiled. 2162 * 2163 * This function walks the fence regs looking for a free one for @obj, 2164 * stealing one if it can't find any. 2165 * 2166 * It then sets up the reg based on the object's properties: address, pitch 2167 * and tiling format. 2168 */ 2169 static int 2170 i915_gem_object_get_fence_reg(struct drm_gem_object *obj, bool write) 2171 { 2172 struct drm_device *dev = obj->dev; 2173 struct drm_i915_private *dev_priv = dev->dev_private; 2174 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2175 struct drm_i915_fence_reg *reg = NULL; 2176 struct drm_i915_gem_object *old_obj_priv = NULL; 2177 int i, ret, avail; 2178 2179 switch (obj_priv->tiling_mode) { 2180 case I915_TILING_NONE: 2181 WARN(1, "allocating a fence for non-tiled object?\n"); 2182 break; 2183 case I915_TILING_X: 2184 if (!obj_priv->stride) 2185 return -EINVAL; 2186 WARN((obj_priv->stride & (512 - 1)), 2187 "object 0x%08x is X tiled but has non-512B pitch\n", 2188 obj_priv->gtt_offset); 2189 break; 2190 case I915_TILING_Y: 2191 if (!obj_priv->stride) 2192 return -EINVAL; 2193 WARN((obj_priv->stride & (128 - 1)), 2194 "object 0x%08x is Y tiled but has non-128B pitch\n", 2195 obj_priv->gtt_offset); 2196 break; 2197 } 2198 2199 /* First try to find a free reg */ 2200 try_again: 2201 avail = 0; 2202 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 2203 reg = &dev_priv->fence_regs[i]; 2204 if (!reg->obj) 2205 break; 2206 2207 old_obj_priv = reg->obj->driver_private; 2208 if (!old_obj_priv->pin_count) 2209 avail++; 2210 } 2211 2212 /* None available, try to steal one or wait for a user to finish */ 2213 if (i == dev_priv->num_fence_regs) { 2214 uint32_t seqno = dev_priv->mm.next_gem_seqno; 2215 loff_t offset; 2216 2217 if (avail == 0) 2218 return -ENOMEM; 2219 2220 for (i = dev_priv->fence_reg_start; 2221 i < dev_priv->num_fence_regs; i++) { 2222 uint32_t this_seqno; 2223 2224 reg = &dev_priv->fence_regs[i]; 2225 old_obj_priv = reg->obj->driver_private; 2226 2227 if (old_obj_priv->pin_count) 2228 continue; 2229 2230 /* i915 uses fences for GPU access to tiled buffers */ 2231 if (IS_I965G(dev) || !old_obj_priv->active) 2232 break; 2233 2234 /* find the seqno of the first available fence */ 2235 this_seqno = old_obj_priv->last_rendering_seqno; 2236 if (this_seqno != 0 && 2237 reg->obj->write_domain == 0 && 2238 i915_seqno_passed(seqno, this_seqno)) 2239 seqno = this_seqno; 2240 } 2241 2242 /* 2243 * Now things get ugly... we have to wait for one of the 2244 * objects to finish before trying again. 2245 */ 2246 if (i == dev_priv->num_fence_regs) { 2247 if (seqno == dev_priv->mm.next_gem_seqno) { 2248 i915_gem_flush(dev, 2249 I915_GEM_GPU_DOMAINS, 2250 I915_GEM_GPU_DOMAINS); 2251 seqno = i915_add_request(dev, 2252 I915_GEM_GPU_DOMAINS); 2253 if (seqno == 0) 2254 return -ENOMEM; 2255 } 2256 2257 ret = i915_wait_request(dev, seqno); 2258 if (ret) 2259 return ret; 2260 goto try_again; 2261 } 2262 2263 /* 2264 * Zap this virtual mapping so we can set up a fence again 2265 * for this object next time we need it. 2266 */ 2267 offset = ((loff_t) reg->obj->map_list.hash.key) << PAGE_SHIFT; 2268 if (dev->dev_mapping) 2269 unmap_mapping_range(dev->dev_mapping, offset, 2270 reg->obj->size, 1); 2271 old_obj_priv->fence_reg = I915_FENCE_REG_NONE; 2272 } 2273 2274 obj_priv->fence_reg = i; 2275 reg->obj = obj; 2276 2277 if (IS_I965G(dev)) 2278 i965_write_fence_reg(reg); 2279 else if (IS_I9XX(dev)) 2280 i915_write_fence_reg(reg); 2281 else 2282 i830_write_fence_reg(reg); 2283 2284 return 0; 2285 } 2286 2287 /** 2288 * i915_gem_clear_fence_reg - clear out fence register info 2289 * @obj: object to clear 2290 * 2291 * Zeroes out the fence register itself and clears out the associated 2292 * data structures in dev_priv and obj_priv. 2293 */ 2294 static void 2295 i915_gem_clear_fence_reg(struct drm_gem_object *obj) 2296 { 2297 struct drm_device *dev = obj->dev; 2298 drm_i915_private_t *dev_priv = dev->dev_private; 2299 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2300 2301 if (IS_I965G(dev)) 2302 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0); 2303 else { 2304 uint32_t fence_reg; 2305 2306 if (obj_priv->fence_reg < 8) 2307 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4; 2308 else 2309 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 2310 8) * 4; 2311 2312 I915_WRITE(fence_reg, 0); 2313 } 2314 2315 dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL; 2316 obj_priv->fence_reg = I915_FENCE_REG_NONE; 2317 } 2318 2319 /** 2320 * Finds free space in the GTT aperture and binds the object there. 2321 */ 2322 static int 2323 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) 2324 { 2325 struct drm_device *dev = obj->dev; 2326 drm_i915_private_t *dev_priv = dev->dev_private; 2327 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2328 struct drm_mm_node *free_space; 2329 int page_count, ret; 2330 2331 if (dev_priv->mm.suspended) 2332 return -EBUSY; 2333 if (alignment == 0) 2334 alignment = i915_gem_get_gtt_alignment(obj); 2335 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) { 2336 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2337 return -EINVAL; 2338 } 2339 2340 search_free: 2341 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2342 obj->size, alignment, 0); 2343 if (free_space != NULL) { 2344 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, 2345 alignment); 2346 if (obj_priv->gtt_space != NULL) { 2347 obj_priv->gtt_space->private = obj; 2348 obj_priv->gtt_offset = obj_priv->gtt_space->start; 2349 } 2350 } 2351 if (obj_priv->gtt_space == NULL) { 2352 bool lists_empty; 2353 2354 /* If the gtt is empty and we're still having trouble 2355 * fitting our object in, we're out of memory. 2356 */ 2357 #if WATCH_LRU 2358 DRM_INFO("%s: GTT full, evicting something\n", __func__); 2359 #endif 2360 spin_lock(&dev_priv->mm.active_list_lock); 2361 lists_empty = (list_empty(&dev_priv->mm.inactive_list) && 2362 list_empty(&dev_priv->mm.flushing_list) && 2363 list_empty(&dev_priv->mm.active_list)); 2364 spin_unlock(&dev_priv->mm.active_list_lock); 2365 if (lists_empty) { 2366 DRM_ERROR("GTT full, but LRU list empty\n"); 2367 return -ENOMEM; 2368 } 2369 2370 ret = i915_gem_evict_something(dev); 2371 if (ret != 0) { 2372 if (ret != -ERESTARTSYS) 2373 DRM_ERROR("Failed to evict a buffer %d\n", ret); 2374 return ret; 2375 } 2376 goto search_free; 2377 } 2378 2379 #if WATCH_BUF 2380 DRM_INFO("Binding object of size %d at 0x%08x\n", 2381 obj->size, obj_priv->gtt_offset); 2382 #endif 2383 ret = i915_gem_object_get_pages(obj); 2384 if (ret) { 2385 drm_mm_put_block(obj_priv->gtt_space); 2386 obj_priv->gtt_space = NULL; 2387 return ret; 2388 } 2389 2390 page_count = obj->size / PAGE_SIZE; 2391 /* Create an AGP memory structure pointing at our pages, and bind it 2392 * into the GTT. 2393 */ 2394 obj_priv->agp_mem = drm_agp_bind_pages(dev, 2395 obj_priv->pages, 2396 page_count, 2397 obj_priv->gtt_offset, 2398 obj_priv->agp_type); 2399 if (obj_priv->agp_mem == NULL) { 2400 i915_gem_object_put_pages(obj); 2401 drm_mm_put_block(obj_priv->gtt_space); 2402 obj_priv->gtt_space = NULL; 2403 return -ENOMEM; 2404 } 2405 atomic_inc(&dev->gtt_count); 2406 atomic_add(obj->size, &dev->gtt_memory); 2407 2408 /* Assert that the object is not currently in any GPU domain. As it 2409 * wasn't in the GTT, there shouldn't be any way it could have been in 2410 * a GPU cache 2411 */ 2412 BUG_ON(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)); 2413 BUG_ON(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)); 2414 2415 return 0; 2416 } 2417 2418 void 2419 i915_gem_clflush_object(struct drm_gem_object *obj) 2420 { 2421 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2422 2423 /* If we don't have a page list set up, then we're not pinned 2424 * to GPU, and we can ignore the cache flush because it'll happen 2425 * again at bind time. 2426 */ 2427 if (obj_priv->pages == NULL) 2428 return; 2429 2430 /* XXX: The 865 in particular appears to be weird in how it handles 2431 * cache flushing. We haven't figured it out, but the 2432 * clflush+agp_chipset_flush doesn't appear to successfully get the 2433 * data visible to the PGU, while wbinvd + agp_chipset_flush does. 2434 */ 2435 if (IS_I865G(obj->dev)) { 2436 wbinvd(); 2437 return; 2438 } 2439 2440 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE); 2441 } 2442 2443 /** Flushes any GPU write domain for the object if it's dirty. */ 2444 static void 2445 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj) 2446 { 2447 struct drm_device *dev = obj->dev; 2448 uint32_t seqno; 2449 2450 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) 2451 return; 2452 2453 /* Queue the GPU write cache flushing we need. */ 2454 i915_gem_flush(dev, 0, obj->write_domain); 2455 seqno = i915_add_request(dev, obj->write_domain); 2456 obj->write_domain = 0; 2457 i915_gem_object_move_to_active(obj, seqno); 2458 } 2459 2460 /** Flushes the GTT write domain for the object if it's dirty. */ 2461 static void 2462 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj) 2463 { 2464 if (obj->write_domain != I915_GEM_DOMAIN_GTT) 2465 return; 2466 2467 /* No actual flushing is required for the GTT write domain. Writes 2468 * to it immediately go to main memory as far as we know, so there's 2469 * no chipset flush. It also doesn't land in render cache. 2470 */ 2471 obj->write_domain = 0; 2472 } 2473 2474 /** Flushes the CPU write domain for the object if it's dirty. */ 2475 static void 2476 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) 2477 { 2478 struct drm_device *dev = obj->dev; 2479 2480 if (obj->write_domain != I915_GEM_DOMAIN_CPU) 2481 return; 2482 2483 i915_gem_clflush_object(obj); 2484 drm_agp_chipset_flush(dev); 2485 obj->write_domain = 0; 2486 } 2487 2488 /** 2489 * Moves a single object to the GTT read, and possibly write domain. 2490 * 2491 * This function returns when the move is complete, including waiting on 2492 * flushes to occur. 2493 */ 2494 int 2495 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) 2496 { 2497 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2498 int ret; 2499 2500 /* Not valid to be called on unbound objects. */ 2501 if (obj_priv->gtt_space == NULL) 2502 return -EINVAL; 2503 2504 i915_gem_object_flush_gpu_write_domain(obj); 2505 /* Wait on any GPU rendering and flushing to occur. */ 2506 ret = i915_gem_object_wait_rendering(obj); 2507 if (ret != 0) 2508 return ret; 2509 2510 /* If we're writing through the GTT domain, then CPU and GPU caches 2511 * will need to be invalidated at next use. 2512 */ 2513 if (write) 2514 obj->read_domains &= I915_GEM_DOMAIN_GTT; 2515 2516 i915_gem_object_flush_cpu_write_domain(obj); 2517 2518 /* It should now be out of any other write domains, and we can update 2519 * the domain values for our changes. 2520 */ 2521 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 2522 obj->read_domains |= I915_GEM_DOMAIN_GTT; 2523 if (write) { 2524 obj->write_domain = I915_GEM_DOMAIN_GTT; 2525 obj_priv->dirty = 1; 2526 } 2527 2528 return 0; 2529 } 2530 2531 /** 2532 * Moves a single object to the CPU read, and possibly write domain. 2533 * 2534 * This function returns when the move is complete, including waiting on 2535 * flushes to occur. 2536 */ 2537 static int 2538 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) 2539 { 2540 int ret; 2541 2542 i915_gem_object_flush_gpu_write_domain(obj); 2543 /* Wait on any GPU rendering and flushing to occur. */ 2544 ret = i915_gem_object_wait_rendering(obj); 2545 if (ret != 0) 2546 return ret; 2547 2548 i915_gem_object_flush_gtt_write_domain(obj); 2549 2550 /* If we have a partially-valid cache of the object in the CPU, 2551 * finish invalidating it and free the per-page flags. 2552 */ 2553 i915_gem_object_set_to_full_cpu_read_domain(obj); 2554 2555 /* Flush the CPU cache if it's still invalid. */ 2556 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 2557 i915_gem_clflush_object(obj); 2558 2559 obj->read_domains |= I915_GEM_DOMAIN_CPU; 2560 } 2561 2562 /* It should now be out of any other write domains, and we can update 2563 * the domain values for our changes. 2564 */ 2565 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 2566 2567 /* If we're writing through the CPU, then the GPU read domains will 2568 * need to be invalidated at next use. 2569 */ 2570 if (write) { 2571 obj->read_domains &= I915_GEM_DOMAIN_CPU; 2572 obj->write_domain = I915_GEM_DOMAIN_CPU; 2573 } 2574 2575 return 0; 2576 } 2577 2578 /* 2579 * Set the next domain for the specified object. This 2580 * may not actually perform the necessary flushing/invaliding though, 2581 * as that may want to be batched with other set_domain operations 2582 * 2583 * This is (we hope) the only really tricky part of gem. The goal 2584 * is fairly simple -- track which caches hold bits of the object 2585 * and make sure they remain coherent. A few concrete examples may 2586 * help to explain how it works. For shorthand, we use the notation 2587 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the 2588 * a pair of read and write domain masks. 2589 * 2590 * Case 1: the batch buffer 2591 * 2592 * 1. Allocated 2593 * 2. Written by CPU 2594 * 3. Mapped to GTT 2595 * 4. Read by GPU 2596 * 5. Unmapped from GTT 2597 * 6. Freed 2598 * 2599 * Let's take these a step at a time 2600 * 2601 * 1. Allocated 2602 * Pages allocated from the kernel may still have 2603 * cache contents, so we set them to (CPU, CPU) always. 2604 * 2. Written by CPU (using pwrite) 2605 * The pwrite function calls set_domain (CPU, CPU) and 2606 * this function does nothing (as nothing changes) 2607 * 3. Mapped by GTT 2608 * This function asserts that the object is not 2609 * currently in any GPU-based read or write domains 2610 * 4. Read by GPU 2611 * i915_gem_execbuffer calls set_domain (COMMAND, 0). 2612 * As write_domain is zero, this function adds in the 2613 * current read domains (CPU+COMMAND, 0). 2614 * flush_domains is set to CPU. 2615 * invalidate_domains is set to COMMAND 2616 * clflush is run to get data out of the CPU caches 2617 * then i915_dev_set_domain calls i915_gem_flush to 2618 * emit an MI_FLUSH and drm_agp_chipset_flush 2619 * 5. Unmapped from GTT 2620 * i915_gem_object_unbind calls set_domain (CPU, CPU) 2621 * flush_domains and invalidate_domains end up both zero 2622 * so no flushing/invalidating happens 2623 * 6. Freed 2624 * yay, done 2625 * 2626 * Case 2: The shared render buffer 2627 * 2628 * 1. Allocated 2629 * 2. Mapped to GTT 2630 * 3. Read/written by GPU 2631 * 4. set_domain to (CPU,CPU) 2632 * 5. Read/written by CPU 2633 * 6. Read/written by GPU 2634 * 2635 * 1. Allocated 2636 * Same as last example, (CPU, CPU) 2637 * 2. Mapped to GTT 2638 * Nothing changes (assertions find that it is not in the GPU) 2639 * 3. Read/written by GPU 2640 * execbuffer calls set_domain (RENDER, RENDER) 2641 * flush_domains gets CPU 2642 * invalidate_domains gets GPU 2643 * clflush (obj) 2644 * MI_FLUSH and drm_agp_chipset_flush 2645 * 4. set_domain (CPU, CPU) 2646 * flush_domains gets GPU 2647 * invalidate_domains gets CPU 2648 * wait_rendering (obj) to make sure all drawing is complete. 2649 * This will include an MI_FLUSH to get the data from GPU 2650 * to memory 2651 * clflush (obj) to invalidate the CPU cache 2652 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?) 2653 * 5. Read/written by CPU 2654 * cache lines are loaded and dirtied 2655 * 6. Read written by GPU 2656 * Same as last GPU access 2657 * 2658 * Case 3: The constant buffer 2659 * 2660 * 1. Allocated 2661 * 2. Written by CPU 2662 * 3. Read by GPU 2663 * 4. Updated (written) by CPU again 2664 * 5. Read by GPU 2665 * 2666 * 1. Allocated 2667 * (CPU, CPU) 2668 * 2. Written by CPU 2669 * (CPU, CPU) 2670 * 3. Read by GPU 2671 * (CPU+RENDER, 0) 2672 * flush_domains = CPU 2673 * invalidate_domains = RENDER 2674 * clflush (obj) 2675 * MI_FLUSH 2676 * drm_agp_chipset_flush 2677 * 4. Updated (written) by CPU again 2678 * (CPU, CPU) 2679 * flush_domains = 0 (no previous write domain) 2680 * invalidate_domains = 0 (no new read domains) 2681 * 5. Read by GPU 2682 * (CPU+RENDER, 0) 2683 * flush_domains = CPU 2684 * invalidate_domains = RENDER 2685 * clflush (obj) 2686 * MI_FLUSH 2687 * drm_agp_chipset_flush 2688 */ 2689 static void 2690 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) 2691 { 2692 struct drm_device *dev = obj->dev; 2693 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2694 uint32_t invalidate_domains = 0; 2695 uint32_t flush_domains = 0; 2696 2697 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU); 2698 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU); 2699 2700 #if WATCH_BUF 2701 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n", 2702 __func__, obj, 2703 obj->read_domains, obj->pending_read_domains, 2704 obj->write_domain, obj->pending_write_domain); 2705 #endif 2706 /* 2707 * If the object isn't moving to a new write domain, 2708 * let the object stay in multiple read domains 2709 */ 2710 if (obj->pending_write_domain == 0) 2711 obj->pending_read_domains |= obj->read_domains; 2712 else 2713 obj_priv->dirty = 1; 2714 2715 /* 2716 * Flush the current write domain if 2717 * the new read domains don't match. Invalidate 2718 * any read domains which differ from the old 2719 * write domain 2720 */ 2721 if (obj->write_domain && 2722 obj->write_domain != obj->pending_read_domains) { 2723 flush_domains |= obj->write_domain; 2724 invalidate_domains |= 2725 obj->pending_read_domains & ~obj->write_domain; 2726 } 2727 /* 2728 * Invalidate any read caches which may have 2729 * stale data. That is, any new read domains. 2730 */ 2731 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains; 2732 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) { 2733 #if WATCH_BUF 2734 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n", 2735 __func__, flush_domains, invalidate_domains); 2736 #endif 2737 i915_gem_clflush_object(obj); 2738 } 2739 2740 /* The actual obj->write_domain will be updated with 2741 * pending_write_domain after we emit the accumulated flush for all 2742 * of our domain changes in execbuffers (which clears objects' 2743 * write_domains). So if we have a current write domain that we 2744 * aren't changing, set pending_write_domain to that. 2745 */ 2746 if (flush_domains == 0 && obj->pending_write_domain == 0) 2747 obj->pending_write_domain = obj->write_domain; 2748 obj->read_domains = obj->pending_read_domains; 2749 2750 dev->invalidate_domains |= invalidate_domains; 2751 dev->flush_domains |= flush_domains; 2752 #if WATCH_BUF 2753 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n", 2754 __func__, 2755 obj->read_domains, obj->write_domain, 2756 dev->invalidate_domains, dev->flush_domains); 2757 #endif 2758 } 2759 2760 /** 2761 * Moves the object from a partially CPU read to a full one. 2762 * 2763 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(), 2764 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). 2765 */ 2766 static void 2767 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj) 2768 { 2769 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2770 2771 if (!obj_priv->page_cpu_valid) 2772 return; 2773 2774 /* If we're partially in the CPU read domain, finish moving it in. 2775 */ 2776 if (obj->read_domains & I915_GEM_DOMAIN_CPU) { 2777 int i; 2778 2779 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) { 2780 if (obj_priv->page_cpu_valid[i]) 2781 continue; 2782 drm_clflush_pages(obj_priv->pages + i, 1); 2783 } 2784 } 2785 2786 /* Free the page_cpu_valid mappings which are now stale, whether 2787 * or not we've got I915_GEM_DOMAIN_CPU. 2788 */ 2789 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE, 2790 DRM_MEM_DRIVER); 2791 obj_priv->page_cpu_valid = NULL; 2792 } 2793 2794 /** 2795 * Set the CPU read domain on a range of the object. 2796 * 2797 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's 2798 * not entirely valid. The page_cpu_valid member of the object flags which 2799 * pages have been flushed, and will be respected by 2800 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping 2801 * of the whole object. 2802 * 2803 * This function returns when the move is complete, including waiting on 2804 * flushes to occur. 2805 */ 2806 static int 2807 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 2808 uint64_t offset, uint64_t size) 2809 { 2810 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2811 int i, ret; 2812 2813 if (offset == 0 && size == obj->size) 2814 return i915_gem_object_set_to_cpu_domain(obj, 0); 2815 2816 i915_gem_object_flush_gpu_write_domain(obj); 2817 /* Wait on any GPU rendering and flushing to occur. */ 2818 ret = i915_gem_object_wait_rendering(obj); 2819 if (ret != 0) 2820 return ret; 2821 i915_gem_object_flush_gtt_write_domain(obj); 2822 2823 /* If we're already fully in the CPU read domain, we're done. */ 2824 if (obj_priv->page_cpu_valid == NULL && 2825 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0) 2826 return 0; 2827 2828 /* Otherwise, create/clear the per-page CPU read domain flag if we're 2829 * newly adding I915_GEM_DOMAIN_CPU 2830 */ 2831 if (obj_priv->page_cpu_valid == NULL) { 2832 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE, 2833 DRM_MEM_DRIVER); 2834 if (obj_priv->page_cpu_valid == NULL) 2835 return -ENOMEM; 2836 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) 2837 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE); 2838 2839 /* Flush the cache on any pages that are still invalid from the CPU's 2840 * perspective. 2841 */ 2842 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 2843 i++) { 2844 if (obj_priv->page_cpu_valid[i]) 2845 continue; 2846 2847 drm_clflush_pages(obj_priv->pages + i, 1); 2848 2849 obj_priv->page_cpu_valid[i] = 1; 2850 } 2851 2852 /* It should now be out of any other write domains, and we can update 2853 * the domain values for our changes. 2854 */ 2855 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 2856 2857 obj->read_domains |= I915_GEM_DOMAIN_CPU; 2858 2859 return 0; 2860 } 2861 2862 /** 2863 * Pin an object to the GTT and evaluate the relocations landing in it. 2864 */ 2865 static int 2866 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, 2867 struct drm_file *file_priv, 2868 struct drm_i915_gem_exec_object *entry, 2869 struct drm_i915_gem_relocation_entry *relocs) 2870 { 2871 struct drm_device *dev = obj->dev; 2872 drm_i915_private_t *dev_priv = dev->dev_private; 2873 struct drm_i915_gem_object *obj_priv = obj->driver_private; 2874 int i, ret; 2875 void __iomem *reloc_page; 2876 2877 /* Choose the GTT offset for our buffer and put it there. */ 2878 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment); 2879 if (ret) 2880 return ret; 2881 2882 entry->offset = obj_priv->gtt_offset; 2883 2884 /* Apply the relocations, using the GTT aperture to avoid cache 2885 * flushing requirements. 2886 */ 2887 for (i = 0; i < entry->relocation_count; i++) { 2888 struct drm_i915_gem_relocation_entry *reloc= &relocs[i]; 2889 struct drm_gem_object *target_obj; 2890 struct drm_i915_gem_object *target_obj_priv; 2891 uint32_t reloc_val, reloc_offset; 2892 uint32_t __iomem *reloc_entry; 2893 2894 target_obj = drm_gem_object_lookup(obj->dev, file_priv, 2895 reloc->target_handle); 2896 if (target_obj == NULL) { 2897 i915_gem_object_unpin(obj); 2898 return -EBADF; 2899 } 2900 target_obj_priv = target_obj->driver_private; 2901 2902 /* The target buffer should have appeared before us in the 2903 * exec_object list, so it should have a GTT space bound by now. 2904 */ 2905 if (target_obj_priv->gtt_space == NULL) { 2906 DRM_ERROR("No GTT space found for object %d\n", 2907 reloc->target_handle); 2908 drm_gem_object_unreference(target_obj); 2909 i915_gem_object_unpin(obj); 2910 return -EINVAL; 2911 } 2912 2913 if (reloc->offset > obj->size - 4) { 2914 DRM_ERROR("Relocation beyond object bounds: " 2915 "obj %p target %d offset %d size %d.\n", 2916 obj, reloc->target_handle, 2917 (int) reloc->offset, (int) obj->size); 2918 drm_gem_object_unreference(target_obj); 2919 i915_gem_object_unpin(obj); 2920 return -EINVAL; 2921 } 2922 if (reloc->offset & 3) { 2923 DRM_ERROR("Relocation not 4-byte aligned: " 2924 "obj %p target %d offset %d.\n", 2925 obj, reloc->target_handle, 2926 (int) reloc->offset); 2927 drm_gem_object_unreference(target_obj); 2928 i915_gem_object_unpin(obj); 2929 return -EINVAL; 2930 } 2931 2932 if (reloc->write_domain & I915_GEM_DOMAIN_CPU || 2933 reloc->read_domains & I915_GEM_DOMAIN_CPU) { 2934 DRM_ERROR("reloc with read/write CPU domains: " 2935 "obj %p target %d offset %d " 2936 "read %08x write %08x", 2937 obj, reloc->target_handle, 2938 (int) reloc->offset, 2939 reloc->read_domains, 2940 reloc->write_domain); 2941 drm_gem_object_unreference(target_obj); 2942 i915_gem_object_unpin(obj); 2943 return -EINVAL; 2944 } 2945 2946 if (reloc->write_domain && target_obj->pending_write_domain && 2947 reloc->write_domain != target_obj->pending_write_domain) { 2948 DRM_ERROR("Write domain conflict: " 2949 "obj %p target %d offset %d " 2950 "new %08x old %08x\n", 2951 obj, reloc->target_handle, 2952 (int) reloc->offset, 2953 reloc->write_domain, 2954 target_obj->pending_write_domain); 2955 drm_gem_object_unreference(target_obj); 2956 i915_gem_object_unpin(obj); 2957 return -EINVAL; 2958 } 2959 2960 #if WATCH_RELOC 2961 DRM_INFO("%s: obj %p offset %08x target %d " 2962 "read %08x write %08x gtt %08x " 2963 "presumed %08x delta %08x\n", 2964 __func__, 2965 obj, 2966 (int) reloc->offset, 2967 (int) reloc->target_handle, 2968 (int) reloc->read_domains, 2969 (int) reloc->write_domain, 2970 (int) target_obj_priv->gtt_offset, 2971 (int) reloc->presumed_offset, 2972 reloc->delta); 2973 #endif 2974 2975 target_obj->pending_read_domains |= reloc->read_domains; 2976 target_obj->pending_write_domain |= reloc->write_domain; 2977 2978 /* If the relocation already has the right value in it, no 2979 * more work needs to be done. 2980 */ 2981 if (target_obj_priv->gtt_offset == reloc->presumed_offset) { 2982 drm_gem_object_unreference(target_obj); 2983 continue; 2984 } 2985 2986 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 2987 if (ret != 0) { 2988 drm_gem_object_unreference(target_obj); 2989 i915_gem_object_unpin(obj); 2990 return -EINVAL; 2991 } 2992 2993 /* Map the page containing the relocation we're going to 2994 * perform. 2995 */ 2996 reloc_offset = obj_priv->gtt_offset + reloc->offset; 2997 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping, 2998 (reloc_offset & 2999 ~(PAGE_SIZE - 1))); 3000 reloc_entry = (uint32_t __iomem *)(reloc_page + 3001 (reloc_offset & (PAGE_SIZE - 1))); 3002 reloc_val = target_obj_priv->gtt_offset + reloc->delta; 3003 3004 #if WATCH_BUF 3005 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n", 3006 obj, (unsigned int) reloc->offset, 3007 readl(reloc_entry), reloc_val); 3008 #endif 3009 writel(reloc_val, reloc_entry); 3010 io_mapping_unmap_atomic(reloc_page); 3011 3012 /* The updated presumed offset for this entry will be 3013 * copied back out to the user. 3014 */ 3015 reloc->presumed_offset = target_obj_priv->gtt_offset; 3016 3017 drm_gem_object_unreference(target_obj); 3018 } 3019 3020 #if WATCH_BUF 3021 if (0) 3022 i915_gem_dump_object(obj, 128, __func__, ~0); 3023 #endif 3024 return 0; 3025 } 3026 3027 /** Dispatch a batchbuffer to the ring 3028 */ 3029 static int 3030 i915_dispatch_gem_execbuffer(struct drm_device *dev, 3031 struct drm_i915_gem_execbuffer *exec, 3032 struct drm_clip_rect *cliprects, 3033 uint64_t exec_offset) 3034 { 3035 drm_i915_private_t *dev_priv = dev->dev_private; 3036 int nbox = exec->num_cliprects; 3037 int i = 0, count; 3038 uint32_t exec_start, exec_len; 3039 RING_LOCALS; 3040 3041 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 3042 exec_len = (uint32_t) exec->batch_len; 3043 3044 if ((exec_start | exec_len) & 0x7) { 3045 DRM_ERROR("alignment\n"); 3046 return -EINVAL; 3047 } 3048 3049 if (!exec_start) 3050 return -EINVAL; 3051 3052 count = nbox ? nbox : 1; 3053 3054 for (i = 0; i < count; i++) { 3055 if (i < nbox) { 3056 int ret = i915_emit_box(dev, cliprects, i, 3057 exec->DR1, exec->DR4); 3058 if (ret) 3059 return ret; 3060 } 3061 3062 if (IS_I830(dev) || IS_845G(dev)) { 3063 BEGIN_LP_RING(4); 3064 OUT_RING(MI_BATCH_BUFFER); 3065 OUT_RING(exec_start | MI_BATCH_NON_SECURE); 3066 OUT_RING(exec_start + exec_len - 4); 3067 OUT_RING(0); 3068 ADVANCE_LP_RING(); 3069 } else { 3070 BEGIN_LP_RING(2); 3071 if (IS_I965G(dev)) { 3072 OUT_RING(MI_BATCH_BUFFER_START | 3073 (2 << 6) | 3074 MI_BATCH_NON_SECURE_I965); 3075 OUT_RING(exec_start); 3076 } else { 3077 OUT_RING(MI_BATCH_BUFFER_START | 3078 (2 << 6)); 3079 OUT_RING(exec_start | MI_BATCH_NON_SECURE); 3080 } 3081 ADVANCE_LP_RING(); 3082 } 3083 } 3084 3085 /* XXX breadcrumb */ 3086 return 0; 3087 } 3088 3089 /* Throttle our rendering by waiting until the ring has completed our requests 3090 * emitted over 20 msec ago. 3091 * 3092 * This should get us reasonable parallelism between CPU and GPU but also 3093 * relatively low latency when blocking on a particular request to finish. 3094 */ 3095 static int 3096 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv) 3097 { 3098 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; 3099 int ret = 0; 3100 uint32_t seqno; 3101 3102 mutex_lock(&dev->struct_mutex); 3103 seqno = i915_file_priv->mm.last_gem_throttle_seqno; 3104 i915_file_priv->mm.last_gem_throttle_seqno = 3105 i915_file_priv->mm.last_gem_seqno; 3106 if (seqno) 3107 ret = i915_wait_request(dev, seqno); 3108 mutex_unlock(&dev->struct_mutex); 3109 return ret; 3110 } 3111 3112 static int 3113 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object *exec_list, 3114 uint32_t buffer_count, 3115 struct drm_i915_gem_relocation_entry **relocs) 3116 { 3117 uint32_t reloc_count = 0, reloc_index = 0, i; 3118 int ret; 3119 3120 *relocs = NULL; 3121 for (i = 0; i < buffer_count; i++) { 3122 if (reloc_count + exec_list[i].relocation_count < reloc_count) 3123 return -EINVAL; 3124 reloc_count += exec_list[i].relocation_count; 3125 } 3126 3127 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs)); 3128 if (*relocs == NULL) 3129 return -ENOMEM; 3130 3131 for (i = 0; i < buffer_count; i++) { 3132 struct drm_i915_gem_relocation_entry __user *user_relocs; 3133 3134 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr; 3135 3136 ret = copy_from_user(&(*relocs)[reloc_index], 3137 user_relocs, 3138 exec_list[i].relocation_count * 3139 sizeof(**relocs)); 3140 if (ret != 0) { 3141 drm_free_large(*relocs); 3142 *relocs = NULL; 3143 return -EFAULT; 3144 } 3145 3146 reloc_index += exec_list[i].relocation_count; 3147 } 3148 3149 return 0; 3150 } 3151 3152 static int 3153 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object *exec_list, 3154 uint32_t buffer_count, 3155 struct drm_i915_gem_relocation_entry *relocs) 3156 { 3157 uint32_t reloc_count = 0, i; 3158 int ret = 0; 3159 3160 for (i = 0; i < buffer_count; i++) { 3161 struct drm_i915_gem_relocation_entry __user *user_relocs; 3162 int unwritten; 3163 3164 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr; 3165 3166 unwritten = copy_to_user(user_relocs, 3167 &relocs[reloc_count], 3168 exec_list[i].relocation_count * 3169 sizeof(*relocs)); 3170 3171 if (unwritten) { 3172 ret = -EFAULT; 3173 goto err; 3174 } 3175 3176 reloc_count += exec_list[i].relocation_count; 3177 } 3178 3179 err: 3180 drm_free_large(relocs); 3181 3182 return ret; 3183 } 3184 3185 int 3186 i915_gem_execbuffer(struct drm_device *dev, void *data, 3187 struct drm_file *file_priv) 3188 { 3189 drm_i915_private_t *dev_priv = dev->dev_private; 3190 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; 3191 struct drm_i915_gem_execbuffer *args = data; 3192 struct drm_i915_gem_exec_object *exec_list = NULL; 3193 struct drm_gem_object **object_list = NULL; 3194 struct drm_gem_object *batch_obj; 3195 struct drm_i915_gem_object *obj_priv; 3196 struct drm_clip_rect *cliprects = NULL; 3197 struct drm_i915_gem_relocation_entry *relocs; 3198 int ret, ret2, i, pinned = 0; 3199 uint64_t exec_offset; 3200 uint32_t seqno, flush_domains, reloc_index; 3201 int pin_tries; 3202 3203 #if WATCH_EXEC 3204 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", 3205 (int) args->buffers_ptr, args->buffer_count, args->batch_len); 3206 #endif 3207 3208 if (args->buffer_count < 1) { 3209 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); 3210 return -EINVAL; 3211 } 3212 /* Copy in the exec list from userland */ 3213 exec_list = drm_calloc_large(sizeof(*exec_list), args->buffer_count); 3214 object_list = drm_calloc_large(sizeof(*object_list), args->buffer_count); 3215 if (exec_list == NULL || object_list == NULL) { 3216 DRM_ERROR("Failed to allocate exec or object list " 3217 "for %d buffers\n", 3218 args->buffer_count); 3219 ret = -ENOMEM; 3220 goto pre_mutex_err; 3221 } 3222 ret = copy_from_user(exec_list, 3223 (struct drm_i915_relocation_entry __user *) 3224 (uintptr_t) args->buffers_ptr, 3225 sizeof(*exec_list) * args->buffer_count); 3226 if (ret != 0) { 3227 DRM_ERROR("copy %d exec entries failed %d\n", 3228 args->buffer_count, ret); 3229 goto pre_mutex_err; 3230 } 3231 3232 if (args->num_cliprects != 0) { 3233 cliprects = drm_calloc(args->num_cliprects, sizeof(*cliprects), 3234 DRM_MEM_DRIVER); 3235 if (cliprects == NULL) 3236 goto pre_mutex_err; 3237 3238 ret = copy_from_user(cliprects, 3239 (struct drm_clip_rect __user *) 3240 (uintptr_t) args->cliprects_ptr, 3241 sizeof(*cliprects) * args->num_cliprects); 3242 if (ret != 0) { 3243 DRM_ERROR("copy %d cliprects failed: %d\n", 3244 args->num_cliprects, ret); 3245 goto pre_mutex_err; 3246 } 3247 } 3248 3249 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count, 3250 &relocs); 3251 if (ret != 0) 3252 goto pre_mutex_err; 3253 3254 mutex_lock(&dev->struct_mutex); 3255 3256 i915_verify_inactive(dev, __FILE__, __LINE__); 3257 3258 if (dev_priv->mm.wedged) { 3259 DRM_ERROR("Execbuf while wedged\n"); 3260 mutex_unlock(&dev->struct_mutex); 3261 ret = -EIO; 3262 goto pre_mutex_err; 3263 } 3264 3265 if (dev_priv->mm.suspended) { 3266 DRM_ERROR("Execbuf while VT-switched.\n"); 3267 mutex_unlock(&dev->struct_mutex); 3268 ret = -EBUSY; 3269 goto pre_mutex_err; 3270 } 3271 3272 /* Look up object handles */ 3273 for (i = 0; i < args->buffer_count; i++) { 3274 object_list[i] = drm_gem_object_lookup(dev, file_priv, 3275 exec_list[i].handle); 3276 if (object_list[i] == NULL) { 3277 DRM_ERROR("Invalid object handle %d at index %d\n", 3278 exec_list[i].handle, i); 3279 ret = -EBADF; 3280 goto err; 3281 } 3282 3283 obj_priv = object_list[i]->driver_private; 3284 if (obj_priv->in_execbuffer) { 3285 DRM_ERROR("Object %p appears more than once in object list\n", 3286 object_list[i]); 3287 ret = -EBADF; 3288 goto err; 3289 } 3290 obj_priv->in_execbuffer = true; 3291 } 3292 3293 /* Pin and relocate */ 3294 for (pin_tries = 0; ; pin_tries++) { 3295 ret = 0; 3296 reloc_index = 0; 3297 3298 for (i = 0; i < args->buffer_count; i++) { 3299 object_list[i]->pending_read_domains = 0; 3300 object_list[i]->pending_write_domain = 0; 3301 ret = i915_gem_object_pin_and_relocate(object_list[i], 3302 file_priv, 3303 &exec_list[i], 3304 &relocs[reloc_index]); 3305 if (ret) 3306 break; 3307 pinned = i + 1; 3308 reloc_index += exec_list[i].relocation_count; 3309 } 3310 /* success */ 3311 if (ret == 0) 3312 break; 3313 3314 /* error other than GTT full, or we've already tried again */ 3315 if (ret != -ENOMEM || pin_tries >= 1) { 3316 if (ret != -ERESTARTSYS) 3317 DRM_ERROR("Failed to pin buffers %d\n", ret); 3318 goto err; 3319 } 3320 3321 /* unpin all of our buffers */ 3322 for (i = 0; i < pinned; i++) 3323 i915_gem_object_unpin(object_list[i]); 3324 pinned = 0; 3325 3326 /* evict everyone we can from the aperture */ 3327 ret = i915_gem_evict_everything(dev); 3328 if (ret) 3329 goto err; 3330 } 3331 3332 /* Set the pending read domains for the batch buffer to COMMAND */ 3333 batch_obj = object_list[args->buffer_count-1]; 3334 batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND; 3335 batch_obj->pending_write_domain = 0; 3336 3337 i915_verify_inactive(dev, __FILE__, __LINE__); 3338 3339 /* Zero the global flush/invalidate flags. These 3340 * will be modified as new domains are computed 3341 * for each object 3342 */ 3343 dev->invalidate_domains = 0; 3344 dev->flush_domains = 0; 3345 3346 for (i = 0; i < args->buffer_count; i++) { 3347 struct drm_gem_object *obj = object_list[i]; 3348 3349 /* Compute new gpu domains and update invalidate/flush */ 3350 i915_gem_object_set_to_gpu_domain(obj); 3351 } 3352 3353 i915_verify_inactive(dev, __FILE__, __LINE__); 3354 3355 if (dev->invalidate_domains | dev->flush_domains) { 3356 #if WATCH_EXEC 3357 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n", 3358 __func__, 3359 dev->invalidate_domains, 3360 dev->flush_domains); 3361 #endif 3362 i915_gem_flush(dev, 3363 dev->invalidate_domains, 3364 dev->flush_domains); 3365 if (dev->flush_domains) 3366 (void)i915_add_request(dev, dev->flush_domains); 3367 } 3368 3369 for (i = 0; i < args->buffer_count; i++) { 3370 struct drm_gem_object *obj = object_list[i]; 3371 3372 obj->write_domain = obj->pending_write_domain; 3373 } 3374 3375 i915_verify_inactive(dev, __FILE__, __LINE__); 3376 3377 #if WATCH_COHERENCY 3378 for (i = 0; i < args->buffer_count; i++) { 3379 i915_gem_object_check_coherency(object_list[i], 3380 exec_list[i].handle); 3381 } 3382 #endif 3383 3384 exec_offset = exec_list[args->buffer_count - 1].offset; 3385 3386 #if WATCH_EXEC 3387 i915_gem_dump_object(batch_obj, 3388 args->batch_len, 3389 __func__, 3390 ~0); 3391 #endif 3392 3393 /* Exec the batchbuffer */ 3394 ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset); 3395 if (ret) { 3396 DRM_ERROR("dispatch failed %d\n", ret); 3397 goto err; 3398 } 3399 3400 /* 3401 * Ensure that the commands in the batch buffer are 3402 * finished before the interrupt fires 3403 */ 3404 flush_domains = i915_retire_commands(dev); 3405 3406 i915_verify_inactive(dev, __FILE__, __LINE__); 3407 3408 /* 3409 * Get a seqno representing the execution of the current buffer, 3410 * which we can wait on. We would like to mitigate these interrupts, 3411 * likely by only creating seqnos occasionally (so that we have 3412 * *some* interrupts representing completion of buffers that we can 3413 * wait on when trying to clear up gtt space). 3414 */ 3415 seqno = i915_add_request(dev, flush_domains); 3416 BUG_ON(seqno == 0); 3417 i915_file_priv->mm.last_gem_seqno = seqno; 3418 for (i = 0; i < args->buffer_count; i++) { 3419 struct drm_gem_object *obj = object_list[i]; 3420 3421 i915_gem_object_move_to_active(obj, seqno); 3422 #if WATCH_LRU 3423 DRM_INFO("%s: move to exec list %p\n", __func__, obj); 3424 #endif 3425 } 3426 #if WATCH_LRU 3427 i915_dump_lru(dev, __func__); 3428 #endif 3429 3430 i915_verify_inactive(dev, __FILE__, __LINE__); 3431 3432 err: 3433 for (i = 0; i < pinned; i++) 3434 i915_gem_object_unpin(object_list[i]); 3435 3436 for (i = 0; i < args->buffer_count; i++) { 3437 if (object_list[i]) { 3438 obj_priv = object_list[i]->driver_private; 3439 obj_priv->in_execbuffer = false; 3440 } 3441 drm_gem_object_unreference(object_list[i]); 3442 } 3443 3444 mutex_unlock(&dev->struct_mutex); 3445 3446 if (!ret) { 3447 /* Copy the new buffer offsets back to the user's exec list. */ 3448 ret = copy_to_user((struct drm_i915_relocation_entry __user *) 3449 (uintptr_t) args->buffers_ptr, 3450 exec_list, 3451 sizeof(*exec_list) * args->buffer_count); 3452 if (ret) { 3453 ret = -EFAULT; 3454 DRM_ERROR("failed to copy %d exec entries " 3455 "back to user (%d)\n", 3456 args->buffer_count, ret); 3457 } 3458 } 3459 3460 /* Copy the updated relocations out regardless of current error 3461 * state. Failure to update the relocs would mean that the next 3462 * time userland calls execbuf, it would do so with presumed offset 3463 * state that didn't match the actual object state. 3464 */ 3465 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count, 3466 relocs); 3467 if (ret2 != 0) { 3468 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2); 3469 3470 if (ret == 0) 3471 ret = ret2; 3472 } 3473 3474 pre_mutex_err: 3475 drm_free_large(object_list); 3476 drm_free_large(exec_list); 3477 drm_free(cliprects, sizeof(*cliprects) * args->num_cliprects, 3478 DRM_MEM_DRIVER); 3479 3480 return ret; 3481 } 3482 3483 int 3484 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) 3485 { 3486 struct drm_device *dev = obj->dev; 3487 struct drm_i915_gem_object *obj_priv = obj->driver_private; 3488 int ret; 3489 3490 i915_verify_inactive(dev, __FILE__, __LINE__); 3491 if (obj_priv->gtt_space == NULL) { 3492 ret = i915_gem_object_bind_to_gtt(obj, alignment); 3493 if (ret != 0) { 3494 if (ret != -EBUSY && ret != -ERESTARTSYS) 3495 DRM_ERROR("Failure to bind: %d\n", ret); 3496 return ret; 3497 } 3498 } 3499 /* 3500 * Pre-965 chips need a fence register set up in order to 3501 * properly handle tiled surfaces. 3502 */ 3503 if (!IS_I965G(dev) && 3504 obj_priv->fence_reg == I915_FENCE_REG_NONE && 3505 obj_priv->tiling_mode != I915_TILING_NONE) { 3506 ret = i915_gem_object_get_fence_reg(obj, true); 3507 if (ret != 0) { 3508 if (ret != -EBUSY && ret != -ERESTARTSYS) 3509 DRM_ERROR("Failure to install fence: %d\n", 3510 ret); 3511 return ret; 3512 } 3513 } 3514 obj_priv->pin_count++; 3515 3516 /* If the object is not active and not pending a flush, 3517 * remove it from the inactive list 3518 */ 3519 if (obj_priv->pin_count == 1) { 3520 atomic_inc(&dev->pin_count); 3521 atomic_add(obj->size, &dev->pin_memory); 3522 if (!obj_priv->active && 3523 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | 3524 I915_GEM_DOMAIN_GTT)) == 0 && 3525 !list_empty(&obj_priv->list)) 3526 list_del_init(&obj_priv->list); 3527 } 3528 i915_verify_inactive(dev, __FILE__, __LINE__); 3529 3530 return 0; 3531 } 3532 3533 void 3534 i915_gem_object_unpin(struct drm_gem_object *obj) 3535 { 3536 struct drm_device *dev = obj->dev; 3537 drm_i915_private_t *dev_priv = dev->dev_private; 3538 struct drm_i915_gem_object *obj_priv = obj->driver_private; 3539 3540 i915_verify_inactive(dev, __FILE__, __LINE__); 3541 obj_priv->pin_count--; 3542 BUG_ON(obj_priv->pin_count < 0); 3543 BUG_ON(obj_priv->gtt_space == NULL); 3544 3545 /* If the object is no longer pinned, and is 3546 * neither active nor being flushed, then stick it on 3547 * the inactive list 3548 */ 3549 if (obj_priv->pin_count == 0) { 3550 if (!obj_priv->active && 3551 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | 3552 I915_GEM_DOMAIN_GTT)) == 0) 3553 list_move_tail(&obj_priv->list, 3554 &dev_priv->mm.inactive_list); 3555 atomic_dec(&dev->pin_count); 3556 atomic_sub(obj->size, &dev->pin_memory); 3557 } 3558 i915_verify_inactive(dev, __FILE__, __LINE__); 3559 } 3560 3561 int 3562 i915_gem_pin_ioctl(struct drm_device *dev, void *data, 3563 struct drm_file *file_priv) 3564 { 3565 struct drm_i915_gem_pin *args = data; 3566 struct drm_gem_object *obj; 3567 struct drm_i915_gem_object *obj_priv; 3568 int ret; 3569 3570 mutex_lock(&dev->struct_mutex); 3571 3572 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3573 if (obj == NULL) { 3574 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n", 3575 args->handle); 3576 mutex_unlock(&dev->struct_mutex); 3577 return -EBADF; 3578 } 3579 obj_priv = obj->driver_private; 3580 3581 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) { 3582 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 3583 args->handle); 3584 drm_gem_object_unreference(obj); 3585 mutex_unlock(&dev->struct_mutex); 3586 return -EINVAL; 3587 } 3588 3589 obj_priv->user_pin_count++; 3590 obj_priv->pin_filp = file_priv; 3591 if (obj_priv->user_pin_count == 1) { 3592 ret = i915_gem_object_pin(obj, args->alignment); 3593 if (ret != 0) { 3594 drm_gem_object_unreference(obj); 3595 mutex_unlock(&dev->struct_mutex); 3596 return ret; 3597 } 3598 } 3599 3600 /* XXX - flush the CPU caches for pinned objects 3601 * as the X server doesn't manage domains yet 3602 */ 3603 i915_gem_object_flush_cpu_write_domain(obj); 3604 args->offset = obj_priv->gtt_offset; 3605 drm_gem_object_unreference(obj); 3606 mutex_unlock(&dev->struct_mutex); 3607 3608 return 0; 3609 } 3610 3611 int 3612 i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 3613 struct drm_file *file_priv) 3614 { 3615 struct drm_i915_gem_pin *args = data; 3616 struct drm_gem_object *obj; 3617 struct drm_i915_gem_object *obj_priv; 3618 3619 mutex_lock(&dev->struct_mutex); 3620 3621 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3622 if (obj == NULL) { 3623 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n", 3624 args->handle); 3625 mutex_unlock(&dev->struct_mutex); 3626 return -EBADF; 3627 } 3628 3629 obj_priv = obj->driver_private; 3630 if (obj_priv->pin_filp != file_priv) { 3631 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 3632 args->handle); 3633 drm_gem_object_unreference(obj); 3634 mutex_unlock(&dev->struct_mutex); 3635 return -EINVAL; 3636 } 3637 obj_priv->user_pin_count--; 3638 if (obj_priv->user_pin_count == 0) { 3639 obj_priv->pin_filp = NULL; 3640 i915_gem_object_unpin(obj); 3641 } 3642 3643 drm_gem_object_unreference(obj); 3644 mutex_unlock(&dev->struct_mutex); 3645 return 0; 3646 } 3647 3648 int 3649 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 3650 struct drm_file *file_priv) 3651 { 3652 struct drm_i915_gem_busy *args = data; 3653 struct drm_gem_object *obj; 3654 struct drm_i915_gem_object *obj_priv; 3655 3656 mutex_lock(&dev->struct_mutex); 3657 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3658 if (obj == NULL) { 3659 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n", 3660 args->handle); 3661 mutex_unlock(&dev->struct_mutex); 3662 return -EBADF; 3663 } 3664 3665 /* Update the active list for the hardware's current position. 3666 * Otherwise this only updates on a delayed timer or when irqs are 3667 * actually unmasked, and our working set ends up being larger than 3668 * required. 3669 */ 3670 i915_gem_retire_requests(dev); 3671 3672 obj_priv = obj->driver_private; 3673 /* Don't count being on the flushing list against the object being 3674 * done. Otherwise, a buffer left on the flushing list but not getting 3675 * flushed (because nobody's flushing that domain) won't ever return 3676 * unbusy and get reused by libdrm's bo cache. The other expected 3677 * consumer of this interface, OpenGL's occlusion queries, also specs 3678 * that the objects get unbusy "eventually" without any interference. 3679 */ 3680 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0; 3681 3682 drm_gem_object_unreference(obj); 3683 mutex_unlock(&dev->struct_mutex); 3684 return 0; 3685 } 3686 3687 int 3688 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 3689 struct drm_file *file_priv) 3690 { 3691 return i915_gem_ring_throttle(dev, file_priv); 3692 } 3693 3694 int i915_gem_init_object(struct drm_gem_object *obj) 3695 { 3696 struct drm_i915_gem_object *obj_priv; 3697 3698 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER); 3699 if (obj_priv == NULL) 3700 return -ENOMEM; 3701 3702 /* 3703 * We've just allocated pages from the kernel, 3704 * so they've just been written by the CPU with 3705 * zeros. They'll need to be clflushed before we 3706 * use them with the GPU. 3707 */ 3708 obj->write_domain = I915_GEM_DOMAIN_CPU; 3709 obj->read_domains = I915_GEM_DOMAIN_CPU; 3710 3711 obj_priv->agp_type = AGP_USER_MEMORY; 3712 3713 obj->driver_private = obj_priv; 3714 obj_priv->obj = obj; 3715 obj_priv->fence_reg = I915_FENCE_REG_NONE; 3716 INIT_LIST_HEAD(&obj_priv->list); 3717 3718 return 0; 3719 } 3720 3721 void i915_gem_free_object(struct drm_gem_object *obj) 3722 { 3723 struct drm_device *dev = obj->dev; 3724 struct drm_i915_gem_object *obj_priv = obj->driver_private; 3725 3726 while (obj_priv->pin_count > 0) 3727 i915_gem_object_unpin(obj); 3728 3729 if (obj_priv->phys_obj) 3730 i915_gem_detach_phys_object(dev, obj); 3731 3732 i915_gem_object_unbind(obj); 3733 3734 i915_gem_free_mmap_offset(obj); 3735 3736 drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER); 3737 kfree(obj_priv->bit_17); 3738 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER); 3739 } 3740 3741 /** Unbinds all objects that are on the given buffer list. */ 3742 static int 3743 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head) 3744 { 3745 struct drm_gem_object *obj; 3746 struct drm_i915_gem_object *obj_priv; 3747 int ret; 3748 3749 while (!list_empty(head)) { 3750 obj_priv = list_first_entry(head, 3751 struct drm_i915_gem_object, 3752 list); 3753 obj = obj_priv->obj; 3754 3755 if (obj_priv->pin_count != 0) { 3756 DRM_ERROR("Pinned object in unbind list\n"); 3757 mutex_unlock(&dev->struct_mutex); 3758 return -EINVAL; 3759 } 3760 3761 ret = i915_gem_object_unbind(obj); 3762 if (ret != 0) { 3763 DRM_ERROR("Error unbinding object in LeaveVT: %d\n", 3764 ret); 3765 mutex_unlock(&dev->struct_mutex); 3766 return ret; 3767 } 3768 } 3769 3770 3771 return 0; 3772 } 3773 3774 int 3775 i915_gem_idle(struct drm_device *dev) 3776 { 3777 drm_i915_private_t *dev_priv = dev->dev_private; 3778 uint32_t seqno, cur_seqno, last_seqno; 3779 int stuck, ret; 3780 3781 mutex_lock(&dev->struct_mutex); 3782 3783 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) { 3784 mutex_unlock(&dev->struct_mutex); 3785 return 0; 3786 } 3787 3788 /* Hack! Don't let anybody do execbuf while we don't control the chip. 3789 * We need to replace this with a semaphore, or something. 3790 */ 3791 dev_priv->mm.suspended = 1; 3792 3793 /* Cancel the retire work handler, wait for it to finish if running 3794 */ 3795 mutex_unlock(&dev->struct_mutex); 3796 cancel_delayed_work_sync(&dev_priv->mm.retire_work); 3797 mutex_lock(&dev->struct_mutex); 3798 3799 i915_kernel_lost_context(dev); 3800 3801 /* Flush the GPU along with all non-CPU write domains 3802 */ 3803 i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT), 3804 ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)); 3805 seqno = i915_add_request(dev, ~I915_GEM_DOMAIN_CPU); 3806 3807 if (seqno == 0) { 3808 mutex_unlock(&dev->struct_mutex); 3809 return -ENOMEM; 3810 } 3811 3812 dev_priv->mm.waiting_gem_seqno = seqno; 3813 last_seqno = 0; 3814 stuck = 0; 3815 for (;;) { 3816 cur_seqno = i915_get_gem_seqno(dev); 3817 if (i915_seqno_passed(cur_seqno, seqno)) 3818 break; 3819 if (last_seqno == cur_seqno) { 3820 if (stuck++ > 100) { 3821 DRM_ERROR("hardware wedged\n"); 3822 dev_priv->mm.wedged = 1; 3823 DRM_WAKEUP(&dev_priv->irq_queue); 3824 break; 3825 } 3826 } 3827 msleep(10); 3828 last_seqno = cur_seqno; 3829 } 3830 dev_priv->mm.waiting_gem_seqno = 0; 3831 3832 i915_gem_retire_requests(dev); 3833 3834 spin_lock(&dev_priv->mm.active_list_lock); 3835 if (!dev_priv->mm.wedged) { 3836 /* Active and flushing should now be empty as we've 3837 * waited for a sequence higher than any pending execbuffer 3838 */ 3839 WARN_ON(!list_empty(&dev_priv->mm.active_list)); 3840 WARN_ON(!list_empty(&dev_priv->mm.flushing_list)); 3841 /* Request should now be empty as we've also waited 3842 * for the last request in the list 3843 */ 3844 WARN_ON(!list_empty(&dev_priv->mm.request_list)); 3845 } 3846 3847 /* Empty the active and flushing lists to inactive. If there's 3848 * anything left at this point, it means that we're wedged and 3849 * nothing good's going to happen by leaving them there. So strip 3850 * the GPU domains and just stuff them onto inactive. 3851 */ 3852 while (!list_empty(&dev_priv->mm.active_list)) { 3853 struct drm_i915_gem_object *obj_priv; 3854 3855 obj_priv = list_first_entry(&dev_priv->mm.active_list, 3856 struct drm_i915_gem_object, 3857 list); 3858 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS; 3859 i915_gem_object_move_to_inactive(obj_priv->obj); 3860 } 3861 spin_unlock(&dev_priv->mm.active_list_lock); 3862 3863 while (!list_empty(&dev_priv->mm.flushing_list)) { 3864 struct drm_i915_gem_object *obj_priv; 3865 3866 obj_priv = list_first_entry(&dev_priv->mm.flushing_list, 3867 struct drm_i915_gem_object, 3868 list); 3869 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS; 3870 i915_gem_object_move_to_inactive(obj_priv->obj); 3871 } 3872 3873 3874 /* Move all inactive buffers out of the GTT. */ 3875 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list); 3876 WARN_ON(!list_empty(&dev_priv->mm.inactive_list)); 3877 if (ret) { 3878 mutex_unlock(&dev->struct_mutex); 3879 return ret; 3880 } 3881 3882 i915_gem_cleanup_ringbuffer(dev); 3883 mutex_unlock(&dev->struct_mutex); 3884 3885 return 0; 3886 } 3887 3888 static int 3889 i915_gem_init_hws(struct drm_device *dev) 3890 { 3891 drm_i915_private_t *dev_priv = dev->dev_private; 3892 struct drm_gem_object *obj; 3893 struct drm_i915_gem_object *obj_priv; 3894 int ret; 3895 3896 /* If we need a physical address for the status page, it's already 3897 * initialized at driver load time. 3898 */ 3899 if (!I915_NEED_GFX_HWS(dev)) 3900 return 0; 3901 3902 obj = drm_gem_object_alloc(dev, 4096); 3903 if (obj == NULL) { 3904 DRM_ERROR("Failed to allocate status page\n"); 3905 return -ENOMEM; 3906 } 3907 obj_priv = obj->driver_private; 3908 obj_priv->agp_type = AGP_USER_CACHED_MEMORY; 3909 3910 ret = i915_gem_object_pin(obj, 4096); 3911 if (ret != 0) { 3912 drm_gem_object_unreference(obj); 3913 return ret; 3914 } 3915 3916 dev_priv->status_gfx_addr = obj_priv->gtt_offset; 3917 3918 dev_priv->hw_status_page = kmap(obj_priv->pages[0]); 3919 if (dev_priv->hw_status_page == NULL) { 3920 DRM_ERROR("Failed to map status page.\n"); 3921 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 3922 i915_gem_object_unpin(obj); 3923 drm_gem_object_unreference(obj); 3924 return -EINVAL; 3925 } 3926 dev_priv->hws_obj = obj; 3927 memset(dev_priv->hw_status_page, 0, PAGE_SIZE); 3928 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr); 3929 I915_READ(HWS_PGA); /* posting read */ 3930 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr); 3931 3932 return 0; 3933 } 3934 3935 static void 3936 i915_gem_cleanup_hws(struct drm_device *dev) 3937 { 3938 drm_i915_private_t *dev_priv = dev->dev_private; 3939 struct drm_gem_object *obj; 3940 struct drm_i915_gem_object *obj_priv; 3941 3942 if (dev_priv->hws_obj == NULL) 3943 return; 3944 3945 obj = dev_priv->hws_obj; 3946 obj_priv = obj->driver_private; 3947 3948 kunmap(obj_priv->pages[0]); 3949 i915_gem_object_unpin(obj); 3950 drm_gem_object_unreference(obj); 3951 dev_priv->hws_obj = NULL; 3952 3953 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 3954 dev_priv->hw_status_page = NULL; 3955 3956 /* Write high address into HWS_PGA when disabling. */ 3957 I915_WRITE(HWS_PGA, 0x1ffff000); 3958 } 3959 3960 int 3961 i915_gem_init_ringbuffer(struct drm_device *dev) 3962 { 3963 drm_i915_private_t *dev_priv = dev->dev_private; 3964 struct drm_gem_object *obj; 3965 struct drm_i915_gem_object *obj_priv; 3966 drm_i915_ring_buffer_t *ring = &dev_priv->ring; 3967 int ret; 3968 u32 head; 3969 3970 ret = i915_gem_init_hws(dev); 3971 if (ret != 0) 3972 return ret; 3973 3974 obj = drm_gem_object_alloc(dev, 128 * 1024); 3975 if (obj == NULL) { 3976 DRM_ERROR("Failed to allocate ringbuffer\n"); 3977 i915_gem_cleanup_hws(dev); 3978 return -ENOMEM; 3979 } 3980 obj_priv = obj->driver_private; 3981 3982 ret = i915_gem_object_pin(obj, 4096); 3983 if (ret != 0) { 3984 drm_gem_object_unreference(obj); 3985 i915_gem_cleanup_hws(dev); 3986 return ret; 3987 } 3988 3989 /* Set up the kernel mapping for the ring. */ 3990 ring->Size = obj->size; 3991 ring->tail_mask = obj->size - 1; 3992 3993 ring->map.offset = dev->agp->base + obj_priv->gtt_offset; 3994 ring->map.size = obj->size; 3995 ring->map.type = 0; 3996 ring->map.flags = 0; 3997 ring->map.mtrr = 0; 3998 3999 drm_core_ioremap_wc(&ring->map, dev); 4000 if (ring->map.handle == NULL) { 4001 DRM_ERROR("Failed to map ringbuffer.\n"); 4002 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring)); 4003 i915_gem_object_unpin(obj); 4004 drm_gem_object_unreference(obj); 4005 i915_gem_cleanup_hws(dev); 4006 return -EINVAL; 4007 } 4008 ring->ring_obj = obj; 4009 ring->virtual_start = ring->map.handle; 4010 4011 /* Stop the ring if it's running. */ 4012 I915_WRITE(PRB0_CTL, 0); 4013 I915_WRITE(PRB0_TAIL, 0); 4014 I915_WRITE(PRB0_HEAD, 0); 4015 4016 /* Initialize the ring. */ 4017 I915_WRITE(PRB0_START, obj_priv->gtt_offset); 4018 head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 4019 4020 /* G45 ring initialization fails to reset head to zero */ 4021 if (head != 0) { 4022 DRM_ERROR("Ring head not reset to zero " 4023 "ctl %08x head %08x tail %08x start %08x\n", 4024 I915_READ(PRB0_CTL), 4025 I915_READ(PRB0_HEAD), 4026 I915_READ(PRB0_TAIL), 4027 I915_READ(PRB0_START)); 4028 I915_WRITE(PRB0_HEAD, 0); 4029 4030 DRM_ERROR("Ring head forced to zero " 4031 "ctl %08x head %08x tail %08x start %08x\n", 4032 I915_READ(PRB0_CTL), 4033 I915_READ(PRB0_HEAD), 4034 I915_READ(PRB0_TAIL), 4035 I915_READ(PRB0_START)); 4036 } 4037 4038 I915_WRITE(PRB0_CTL, 4039 ((obj->size - 4096) & RING_NR_PAGES) | 4040 RING_NO_REPORT | 4041 RING_VALID); 4042 4043 head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 4044 4045 /* If the head is still not zero, the ring is dead */ 4046 if (head != 0) { 4047 DRM_ERROR("Ring initialization failed " 4048 "ctl %08x head %08x tail %08x start %08x\n", 4049 I915_READ(PRB0_CTL), 4050 I915_READ(PRB0_HEAD), 4051 I915_READ(PRB0_TAIL), 4052 I915_READ(PRB0_START)); 4053 return -EIO; 4054 } 4055 4056 /* Update our cache of the ring state */ 4057 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4058 i915_kernel_lost_context(dev); 4059 else { 4060 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 4061 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR; 4062 ring->space = ring->head - (ring->tail + 8); 4063 if (ring->space < 0) 4064 ring->space += ring->Size; 4065 } 4066 4067 return 0; 4068 } 4069 4070 void 4071 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 4072 { 4073 drm_i915_private_t *dev_priv = dev->dev_private; 4074 4075 if (dev_priv->ring.ring_obj == NULL) 4076 return; 4077 4078 drm_core_ioremapfree(&dev_priv->ring.map, dev); 4079 4080 i915_gem_object_unpin(dev_priv->ring.ring_obj); 4081 drm_gem_object_unreference(dev_priv->ring.ring_obj); 4082 dev_priv->ring.ring_obj = NULL; 4083 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring)); 4084 4085 i915_gem_cleanup_hws(dev); 4086 } 4087 4088 int 4089 i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 4090 struct drm_file *file_priv) 4091 { 4092 drm_i915_private_t *dev_priv = dev->dev_private; 4093 int ret; 4094 4095 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4096 return 0; 4097 4098 if (dev_priv->mm.wedged) { 4099 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 4100 dev_priv->mm.wedged = 0; 4101 } 4102 4103 mutex_lock(&dev->struct_mutex); 4104 dev_priv->mm.suspended = 0; 4105 4106 ret = i915_gem_init_ringbuffer(dev); 4107 if (ret != 0) { 4108 mutex_unlock(&dev->struct_mutex); 4109 return ret; 4110 } 4111 4112 spin_lock(&dev_priv->mm.active_list_lock); 4113 BUG_ON(!list_empty(&dev_priv->mm.active_list)); 4114 spin_unlock(&dev_priv->mm.active_list_lock); 4115 4116 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 4117 BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); 4118 BUG_ON(!list_empty(&dev_priv->mm.request_list)); 4119 mutex_unlock(&dev->struct_mutex); 4120 4121 drm_irq_install(dev); 4122 4123 return 0; 4124 } 4125 4126 int 4127 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 4128 struct drm_file *file_priv) 4129 { 4130 int ret; 4131 4132 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4133 return 0; 4134 4135 ret = i915_gem_idle(dev); 4136 drm_irq_uninstall(dev); 4137 4138 return ret; 4139 } 4140 4141 void 4142 i915_gem_lastclose(struct drm_device *dev) 4143 { 4144 int ret; 4145 4146 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4147 return; 4148 4149 ret = i915_gem_idle(dev); 4150 if (ret) 4151 DRM_ERROR("failed to idle hardware: %d\n", ret); 4152 } 4153 4154 void 4155 i915_gem_load(struct drm_device *dev) 4156 { 4157 drm_i915_private_t *dev_priv = dev->dev_private; 4158 4159 spin_lock_init(&dev_priv->mm.active_list_lock); 4160 INIT_LIST_HEAD(&dev_priv->mm.active_list); 4161 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 4162 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 4163 INIT_LIST_HEAD(&dev_priv->mm.request_list); 4164 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 4165 i915_gem_retire_work_handler); 4166 dev_priv->mm.next_gem_seqno = 1; 4167 4168 /* Old X drivers will take 0-2 for front, back, depth buffers */ 4169 dev_priv->fence_reg_start = 3; 4170 4171 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 4172 dev_priv->num_fence_regs = 16; 4173 else 4174 dev_priv->num_fence_regs = 8; 4175 4176 i915_gem_detect_bit_6_swizzle(dev); 4177 } 4178 4179 /* 4180 * Create a physically contiguous memory object for this object 4181 * e.g. for cursor + overlay regs 4182 */ 4183 int i915_gem_init_phys_object(struct drm_device *dev, 4184 int id, int size) 4185 { 4186 drm_i915_private_t *dev_priv = dev->dev_private; 4187 struct drm_i915_gem_phys_object *phys_obj; 4188 int ret; 4189 4190 if (dev_priv->mm.phys_objs[id - 1] || !size) 4191 return 0; 4192 4193 phys_obj = drm_calloc(1, sizeof(struct drm_i915_gem_phys_object), DRM_MEM_DRIVER); 4194 if (!phys_obj) 4195 return -ENOMEM; 4196 4197 phys_obj->id = id; 4198 4199 phys_obj->handle = drm_pci_alloc(dev, size, 0, 0xffffffff); 4200 if (!phys_obj->handle) { 4201 ret = -ENOMEM; 4202 goto kfree_obj; 4203 } 4204 #ifdef CONFIG_X86 4205 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); 4206 #endif 4207 4208 dev_priv->mm.phys_objs[id - 1] = phys_obj; 4209 4210 return 0; 4211 kfree_obj: 4212 drm_free(phys_obj, sizeof(struct drm_i915_gem_phys_object), DRM_MEM_DRIVER); 4213 return ret; 4214 } 4215 4216 void i915_gem_free_phys_object(struct drm_device *dev, int id) 4217 { 4218 drm_i915_private_t *dev_priv = dev->dev_private; 4219 struct drm_i915_gem_phys_object *phys_obj; 4220 4221 if (!dev_priv->mm.phys_objs[id - 1]) 4222 return; 4223 4224 phys_obj = dev_priv->mm.phys_objs[id - 1]; 4225 if (phys_obj->cur_obj) { 4226 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 4227 } 4228 4229 #ifdef CONFIG_X86 4230 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); 4231 #endif 4232 drm_pci_free(dev, phys_obj->handle); 4233 kfree(phys_obj); 4234 dev_priv->mm.phys_objs[id - 1] = NULL; 4235 } 4236 4237 void i915_gem_free_all_phys_object(struct drm_device *dev) 4238 { 4239 int i; 4240 4241 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 4242 i915_gem_free_phys_object(dev, i); 4243 } 4244 4245 void i915_gem_detach_phys_object(struct drm_device *dev, 4246 struct drm_gem_object *obj) 4247 { 4248 struct drm_i915_gem_object *obj_priv; 4249 int i; 4250 int ret; 4251 int page_count; 4252 4253 obj_priv = obj->driver_private; 4254 if (!obj_priv->phys_obj) 4255 return; 4256 4257 ret = i915_gem_object_get_pages(obj); 4258 if (ret) 4259 goto out; 4260 4261 page_count = obj->size / PAGE_SIZE; 4262 4263 for (i = 0; i < page_count; i++) { 4264 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0); 4265 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 4266 4267 memcpy(dst, src, PAGE_SIZE); 4268 kunmap_atomic(dst, KM_USER0); 4269 } 4270 drm_clflush_pages(obj_priv->pages, page_count); 4271 drm_agp_chipset_flush(dev); 4272 out: 4273 obj_priv->phys_obj->cur_obj = NULL; 4274 obj_priv->phys_obj = NULL; 4275 } 4276 4277 int 4278 i915_gem_attach_phys_object(struct drm_device *dev, 4279 struct drm_gem_object *obj, int id) 4280 { 4281 drm_i915_private_t *dev_priv = dev->dev_private; 4282 struct drm_i915_gem_object *obj_priv; 4283 int ret = 0; 4284 int page_count; 4285 int i; 4286 4287 if (id > I915_MAX_PHYS_OBJECT) 4288 return -EINVAL; 4289 4290 obj_priv = obj->driver_private; 4291 4292 if (obj_priv->phys_obj) { 4293 if (obj_priv->phys_obj->id == id) 4294 return 0; 4295 i915_gem_detach_phys_object(dev, obj); 4296 } 4297 4298 4299 /* create a new object */ 4300 if (!dev_priv->mm.phys_objs[id - 1]) { 4301 ret = i915_gem_init_phys_object(dev, id, 4302 obj->size); 4303 if (ret) { 4304 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); 4305 goto out; 4306 } 4307 } 4308 4309 /* bind to the object */ 4310 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1]; 4311 obj_priv->phys_obj->cur_obj = obj; 4312 4313 ret = i915_gem_object_get_pages(obj); 4314 if (ret) { 4315 DRM_ERROR("failed to get page list\n"); 4316 goto out; 4317 } 4318 4319 page_count = obj->size / PAGE_SIZE; 4320 4321 for (i = 0; i < page_count; i++) { 4322 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0); 4323 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 4324 4325 memcpy(dst, src, PAGE_SIZE); 4326 kunmap_atomic(src, KM_USER0); 4327 } 4328 4329 return 0; 4330 out: 4331 return ret; 4332 } 4333 4334 static int 4335 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 4336 struct drm_i915_gem_pwrite *args, 4337 struct drm_file *file_priv) 4338 { 4339 struct drm_i915_gem_object *obj_priv = obj->driver_private; 4340 void *obj_addr; 4341 int ret; 4342 char __user *user_data; 4343 4344 user_data = (char __user *) (uintptr_t) args->data_ptr; 4345 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset; 4346 4347 DRM_DEBUG("obj_addr %p, %lld\n", obj_addr, args->size); 4348 ret = copy_from_user(obj_addr, user_data, args->size); 4349 if (ret) 4350 return -EFAULT; 4351 4352 drm_agp_chipset_flush(dev); 4353 return 0; 4354 } 4355