1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #include <linux/highmem.h> 8 #include <linux/prime_numbers.h> 9 10 #include "gem/i915_gem_internal.h" 11 #include "gem/i915_gem_lmem.h" 12 #include "gem/i915_gem_region.h" 13 #include "gem/i915_gem_ttm.h" 14 #include "gem/i915_gem_ttm_move.h" 15 #include "gt/intel_engine_pm.h" 16 #include "gt/intel_gpu_commands.h" 17 #include "gt/intel_gt.h" 18 #include "gt/intel_gt_pm.h" 19 #include "gt/intel_migrate.h" 20 #include "i915_ttm_buddy_manager.h" 21 22 #include "huge_gem_object.h" 23 #include "i915_selftest.h" 24 #include "selftests/i915_random.h" 25 #include "selftests/igt_flush_test.h" 26 #include "selftests/igt_reset.h" 27 #include "selftests/igt_mmap.h" 28 29 struct tile { 30 unsigned int width; 31 unsigned int height; 32 unsigned int stride; 33 unsigned int size; 34 unsigned int tiling; 35 unsigned int swizzle; 36 }; 37 38 static u64 swizzle_bit(unsigned int bit, u64 offset) 39 { 40 return (offset & BIT_ULL(bit)) >> (bit - 6); 41 } 42 43 static u64 tiled_offset(const struct tile *tile, u64 v) 44 { 45 u64 x, y; 46 47 if (tile->tiling == I915_TILING_NONE) 48 return v; 49 50 y = div64_u64_rem(v, tile->stride, &x); 51 v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height; 52 53 if (tile->tiling == I915_TILING_X) { 54 v += y * tile->width; 55 v += div64_u64_rem(x, tile->width, &x) << tile->size; 56 v += x; 57 } else if (tile->width == 128) { 58 const unsigned int ytile_span = 16; 59 const unsigned int ytile_height = 512; 60 61 v += y * ytile_span; 62 v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 63 v += x; 64 } else { 65 const unsigned int ytile_span = 32; 66 const unsigned int ytile_height = 256; 67 68 v += y * ytile_span; 69 v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 70 v += x; 71 } 72 73 switch (tile->swizzle) { 74 case I915_BIT_6_SWIZZLE_9: 75 v ^= swizzle_bit(9, v); 76 break; 77 case I915_BIT_6_SWIZZLE_9_10: 78 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v); 79 break; 80 case I915_BIT_6_SWIZZLE_9_11: 81 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v); 82 break; 83 case I915_BIT_6_SWIZZLE_9_10_11: 84 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v); 85 break; 86 } 87 88 return v; 89 } 90 91 static int check_partial_mapping(struct drm_i915_gem_object *obj, 92 const struct tile *tile, 93 struct rnd_state *prng) 94 { 95 const unsigned long npages = obj->base.size / PAGE_SIZE; 96 struct drm_i915_private *i915 = to_i915(obj->base.dev); 97 struct i915_gtt_view view; 98 struct i915_vma *vma; 99 unsigned long page; 100 u32 __iomem *io; 101 struct page *p; 102 unsigned int n; 103 u64 offset; 104 u32 *cpu; 105 int err; 106 107 err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); 108 if (err) { 109 pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", 110 tile->tiling, tile->stride, err); 111 return err; 112 } 113 114 GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); 115 GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); 116 117 i915_gem_object_lock(obj, NULL); 118 err = i915_gem_object_set_to_gtt_domain(obj, true); 119 i915_gem_object_unlock(obj); 120 if (err) { 121 pr_err("Failed to flush to GTT write domain; err=%d\n", err); 122 return err; 123 } 124 125 page = i915_prandom_u32_max_state(npages, prng); 126 view = compute_partial_view(obj, page, MIN_CHUNK_PAGES); 127 128 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 129 if (IS_ERR(vma)) { 130 pr_err("Failed to pin partial view: offset=%lu; err=%d\n", 131 page, (int)PTR_ERR(vma)); 132 return PTR_ERR(vma); 133 } 134 135 n = page - view.partial.offset; 136 GEM_BUG_ON(n >= view.partial.size); 137 138 io = i915_vma_pin_iomap(vma); 139 i915_vma_unpin(vma); 140 if (IS_ERR(io)) { 141 pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", 142 page, (int)PTR_ERR(io)); 143 err = PTR_ERR(io); 144 goto out; 145 } 146 147 iowrite32(page, io + n * PAGE_SIZE / sizeof(*io)); 148 i915_vma_unpin_iomap(vma); 149 150 offset = tiled_offset(tile, page << PAGE_SHIFT); 151 if (offset >= obj->base.size) 152 goto out; 153 154 intel_gt_flush_ggtt_writes(to_gt(i915)); 155 156 p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); 157 cpu = kmap(p) + offset_in_page(offset); 158 drm_clflush_virt_range(cpu, sizeof(*cpu)); 159 if (*cpu != (u32)page) { 160 pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", 161 page, n, 162 view.partial.offset, 163 view.partial.size, 164 vma->size >> PAGE_SHIFT, 165 tile->tiling ? tile_row_pages(obj) : 0, 166 vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, 167 offset >> PAGE_SHIFT, 168 (unsigned int)offset_in_page(offset), 169 offset, 170 (u32)page, *cpu); 171 err = -EINVAL; 172 } 173 *cpu = 0; 174 drm_clflush_virt_range(cpu, sizeof(*cpu)); 175 kunmap(p); 176 177 out: 178 i915_gem_object_lock(obj, NULL); 179 i915_vma_destroy(vma); 180 i915_gem_object_unlock(obj); 181 return err; 182 } 183 184 static int check_partial_mappings(struct drm_i915_gem_object *obj, 185 const struct tile *tile, 186 unsigned long end_time) 187 { 188 const unsigned int nreal = obj->scratch / PAGE_SIZE; 189 const unsigned long npages = obj->base.size / PAGE_SIZE; 190 struct drm_i915_private *i915 = to_i915(obj->base.dev); 191 struct i915_vma *vma; 192 unsigned long page; 193 int err; 194 195 err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); 196 if (err) { 197 pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", 198 tile->tiling, tile->stride, err); 199 return err; 200 } 201 202 GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); 203 GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); 204 205 i915_gem_object_lock(obj, NULL); 206 err = i915_gem_object_set_to_gtt_domain(obj, true); 207 i915_gem_object_unlock(obj); 208 if (err) { 209 pr_err("Failed to flush to GTT write domain; err=%d\n", err); 210 return err; 211 } 212 213 for_each_prime_number_from(page, 1, npages) { 214 struct i915_gtt_view view = 215 compute_partial_view(obj, page, MIN_CHUNK_PAGES); 216 u32 __iomem *io; 217 struct page *p; 218 unsigned int n; 219 u64 offset; 220 u32 *cpu; 221 222 GEM_BUG_ON(view.partial.size > nreal); 223 cond_resched(); 224 225 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 226 if (IS_ERR(vma)) { 227 pr_err("Failed to pin partial view: offset=%lu; err=%d\n", 228 page, (int)PTR_ERR(vma)); 229 return PTR_ERR(vma); 230 } 231 232 n = page - view.partial.offset; 233 GEM_BUG_ON(n >= view.partial.size); 234 235 io = i915_vma_pin_iomap(vma); 236 i915_vma_unpin(vma); 237 if (IS_ERR(io)) { 238 pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", 239 page, (int)PTR_ERR(io)); 240 return PTR_ERR(io); 241 } 242 243 iowrite32(page, io + n * PAGE_SIZE / sizeof(*io)); 244 i915_vma_unpin_iomap(vma); 245 246 offset = tiled_offset(tile, page << PAGE_SHIFT); 247 if (offset >= obj->base.size) 248 continue; 249 250 intel_gt_flush_ggtt_writes(to_gt(i915)); 251 252 p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); 253 cpu = kmap(p) + offset_in_page(offset); 254 drm_clflush_virt_range(cpu, sizeof(*cpu)); 255 if (*cpu != (u32)page) { 256 pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", 257 page, n, 258 view.partial.offset, 259 view.partial.size, 260 vma->size >> PAGE_SHIFT, 261 tile->tiling ? tile_row_pages(obj) : 0, 262 vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, 263 offset >> PAGE_SHIFT, 264 (unsigned int)offset_in_page(offset), 265 offset, 266 (u32)page, *cpu); 267 err = -EINVAL; 268 } 269 *cpu = 0; 270 drm_clflush_virt_range(cpu, sizeof(*cpu)); 271 kunmap(p); 272 if (err) 273 return err; 274 275 i915_gem_object_lock(obj, NULL); 276 i915_vma_destroy(vma); 277 i915_gem_object_unlock(obj); 278 279 if (igt_timeout(end_time, 280 "%s: timed out after tiling=%d stride=%d\n", 281 __func__, tile->tiling, tile->stride)) 282 return -EINTR; 283 } 284 285 return 0; 286 } 287 288 static unsigned int 289 setup_tile_size(struct tile *tile, struct drm_i915_private *i915) 290 { 291 if (GRAPHICS_VER(i915) <= 2) { 292 tile->height = 16; 293 tile->width = 128; 294 tile->size = 11; 295 } else if (tile->tiling == I915_TILING_Y && 296 HAS_128_BYTE_Y_TILING(i915)) { 297 tile->height = 32; 298 tile->width = 128; 299 tile->size = 12; 300 } else { 301 tile->height = 8; 302 tile->width = 512; 303 tile->size = 12; 304 } 305 306 if (GRAPHICS_VER(i915) < 4) 307 return 8192 / tile->width; 308 else if (GRAPHICS_VER(i915) < 7) 309 return 128 * I965_FENCE_MAX_PITCH_VAL / tile->width; 310 else 311 return 128 * GEN7_FENCE_MAX_PITCH_VAL / tile->width; 312 } 313 314 static int igt_partial_tiling(void *arg) 315 { 316 const unsigned int nreal = 1 << 12; /* largest tile row x2 */ 317 struct drm_i915_private *i915 = arg; 318 struct drm_i915_gem_object *obj; 319 intel_wakeref_t wakeref; 320 int tiling; 321 int err; 322 323 if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 324 return 0; 325 326 /* We want to check the page mapping and fencing of a large object 327 * mmapped through the GTT. The object we create is larger than can 328 * possibly be mmaped as a whole, and so we must use partial GGTT vma. 329 * We then check that a write through each partial GGTT vma ends up 330 * in the right set of pages within the object, and with the expected 331 * tiling, which we verify by manual swizzling. 332 */ 333 334 obj = huge_gem_object(i915, 335 nreal << PAGE_SHIFT, 336 (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); 337 if (IS_ERR(obj)) 338 return PTR_ERR(obj); 339 340 err = i915_gem_object_pin_pages_unlocked(obj); 341 if (err) { 342 pr_err("Failed to allocate %u pages (%lu total), err=%d\n", 343 nreal, obj->base.size / PAGE_SIZE, err); 344 goto out; 345 } 346 347 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 348 349 if (1) { 350 IGT_TIMEOUT(end); 351 struct tile tile; 352 353 tile.height = 1; 354 tile.width = 1; 355 tile.size = 0; 356 tile.stride = 0; 357 tile.swizzle = I915_BIT_6_SWIZZLE_NONE; 358 tile.tiling = I915_TILING_NONE; 359 360 err = check_partial_mappings(obj, &tile, end); 361 if (err && err != -EINTR) 362 goto out_unlock; 363 } 364 365 for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) { 366 IGT_TIMEOUT(end); 367 unsigned int max_pitch; 368 unsigned int pitch; 369 struct tile tile; 370 371 if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) 372 /* 373 * The swizzling pattern is actually unknown as it 374 * varies based on physical address of each page. 375 * See i915_gem_detect_bit_6_swizzle(). 376 */ 377 break; 378 379 tile.tiling = tiling; 380 switch (tiling) { 381 case I915_TILING_X: 382 tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x; 383 break; 384 case I915_TILING_Y: 385 tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y; 386 break; 387 } 388 389 GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN); 390 if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || 391 tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) 392 continue; 393 394 max_pitch = setup_tile_size(&tile, i915); 395 396 for (pitch = max_pitch; pitch; pitch >>= 1) { 397 tile.stride = tile.width * pitch; 398 err = check_partial_mappings(obj, &tile, end); 399 if (err == -EINTR) 400 goto next_tiling; 401 if (err) 402 goto out_unlock; 403 404 if (pitch > 2 && GRAPHICS_VER(i915) >= 4) { 405 tile.stride = tile.width * (pitch - 1); 406 err = check_partial_mappings(obj, &tile, end); 407 if (err == -EINTR) 408 goto next_tiling; 409 if (err) 410 goto out_unlock; 411 } 412 413 if (pitch < max_pitch && GRAPHICS_VER(i915) >= 4) { 414 tile.stride = tile.width * (pitch + 1); 415 err = check_partial_mappings(obj, &tile, end); 416 if (err == -EINTR) 417 goto next_tiling; 418 if (err) 419 goto out_unlock; 420 } 421 } 422 423 if (GRAPHICS_VER(i915) >= 4) { 424 for_each_prime_number(pitch, max_pitch) { 425 tile.stride = tile.width * pitch; 426 err = check_partial_mappings(obj, &tile, end); 427 if (err == -EINTR) 428 goto next_tiling; 429 if (err) 430 goto out_unlock; 431 } 432 } 433 434 next_tiling: ; 435 } 436 437 out_unlock: 438 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 439 i915_gem_object_unpin_pages(obj); 440 out: 441 i915_gem_object_put(obj); 442 return err; 443 } 444 445 static int igt_smoke_tiling(void *arg) 446 { 447 const unsigned int nreal = 1 << 12; /* largest tile row x2 */ 448 struct drm_i915_private *i915 = arg; 449 struct drm_i915_gem_object *obj; 450 intel_wakeref_t wakeref; 451 I915_RND_STATE(prng); 452 unsigned long count; 453 IGT_TIMEOUT(end); 454 int err; 455 456 if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 457 return 0; 458 459 /* 460 * igt_partial_tiling() does an exhastive check of partial tiling 461 * chunking, but will undoubtably run out of time. Here, we do a 462 * randomised search and hope over many runs of 1s with different 463 * seeds we will do a thorough check. 464 * 465 * Remember to look at the st_seed if we see a flip-flop in BAT! 466 */ 467 468 if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) 469 return 0; 470 471 obj = huge_gem_object(i915, 472 nreal << PAGE_SHIFT, 473 (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); 474 if (IS_ERR(obj)) 475 return PTR_ERR(obj); 476 477 err = i915_gem_object_pin_pages_unlocked(obj); 478 if (err) { 479 pr_err("Failed to allocate %u pages (%lu total), err=%d\n", 480 nreal, obj->base.size / PAGE_SIZE, err); 481 goto out; 482 } 483 484 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 485 486 count = 0; 487 do { 488 struct tile tile; 489 490 tile.tiling = 491 i915_prandom_u32_max_state(I915_TILING_Y + 1, &prng); 492 switch (tile.tiling) { 493 case I915_TILING_NONE: 494 tile.height = 1; 495 tile.width = 1; 496 tile.size = 0; 497 tile.stride = 0; 498 tile.swizzle = I915_BIT_6_SWIZZLE_NONE; 499 break; 500 501 case I915_TILING_X: 502 tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x; 503 break; 504 case I915_TILING_Y: 505 tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y; 506 break; 507 } 508 509 if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || 510 tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) 511 continue; 512 513 if (tile.tiling != I915_TILING_NONE) { 514 unsigned int max_pitch = setup_tile_size(&tile, i915); 515 516 tile.stride = 517 i915_prandom_u32_max_state(max_pitch, &prng); 518 tile.stride = (1 + tile.stride) * tile.width; 519 if (GRAPHICS_VER(i915) < 4) 520 tile.stride = rounddown_pow_of_two(tile.stride); 521 } 522 523 err = check_partial_mapping(obj, &tile, &prng); 524 if (err) 525 break; 526 527 count++; 528 } while (!__igt_timeout(end, NULL)); 529 530 pr_info("%s: Completed %lu trials\n", __func__, count); 531 532 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 533 i915_gem_object_unpin_pages(obj); 534 out: 535 i915_gem_object_put(obj); 536 return err; 537 } 538 539 static int make_obj_busy(struct drm_i915_gem_object *obj) 540 { 541 struct drm_i915_private *i915 = to_i915(obj->base.dev); 542 struct intel_engine_cs *engine; 543 544 for_each_uabi_engine(engine, i915) { 545 struct i915_request *rq; 546 struct i915_vma *vma; 547 struct i915_gem_ww_ctx ww; 548 int err; 549 550 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 551 if (IS_ERR(vma)) 552 return PTR_ERR(vma); 553 554 i915_gem_ww_ctx_init(&ww, false); 555 retry: 556 err = i915_gem_object_lock(obj, &ww); 557 if (!err) 558 err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); 559 if (err) 560 goto err; 561 562 rq = intel_engine_create_kernel_request(engine); 563 if (IS_ERR(rq)) { 564 err = PTR_ERR(rq); 565 goto err_unpin; 566 } 567 568 err = i915_request_await_object(rq, vma->obj, true); 569 if (err == 0) 570 err = i915_vma_move_to_active(vma, rq, 571 EXEC_OBJECT_WRITE); 572 573 i915_request_add(rq); 574 err_unpin: 575 i915_vma_unpin(vma); 576 err: 577 if (err == -EDEADLK) { 578 err = i915_gem_ww_ctx_backoff(&ww); 579 if (!err) 580 goto retry; 581 } 582 i915_gem_ww_ctx_fini(&ww); 583 if (err) 584 return err; 585 } 586 587 i915_gem_object_put(obj); /* leave it only alive via its active ref */ 588 return 0; 589 } 590 591 static enum i915_mmap_type default_mapping(struct drm_i915_private *i915) 592 { 593 if (HAS_LMEM(i915)) 594 return I915_MMAP_TYPE_FIXED; 595 596 return I915_MMAP_TYPE_GTT; 597 } 598 599 static struct drm_i915_gem_object * 600 create_sys_or_internal(struct drm_i915_private *i915, 601 unsigned long size) 602 { 603 if (HAS_LMEM(i915)) { 604 struct intel_memory_region *sys_region = 605 i915->mm.regions[INTEL_REGION_SMEM]; 606 607 return __i915_gem_object_create_user(i915, size, &sys_region, 1); 608 } 609 610 return i915_gem_object_create_internal(i915, size); 611 } 612 613 static bool assert_mmap_offset(struct drm_i915_private *i915, 614 unsigned long size, 615 int expected) 616 { 617 struct drm_i915_gem_object *obj; 618 u64 offset; 619 int ret; 620 621 obj = create_sys_or_internal(i915, size); 622 if (IS_ERR(obj)) 623 return expected && expected == PTR_ERR(obj); 624 625 ret = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL); 626 i915_gem_object_put(obj); 627 628 return ret == expected; 629 } 630 631 static void disable_retire_worker(struct drm_i915_private *i915) 632 { 633 i915_gem_driver_unregister__shrinker(i915); 634 intel_gt_pm_get(to_gt(i915)); 635 cancel_delayed_work_sync(&to_gt(i915)->requests.retire_work); 636 } 637 638 static void restore_retire_worker(struct drm_i915_private *i915) 639 { 640 igt_flush_test(i915); 641 intel_gt_pm_put(to_gt(i915)); 642 i915_gem_driver_register__shrinker(i915); 643 } 644 645 static void mmap_offset_lock(struct drm_i915_private *i915) 646 __acquires(&i915->drm.vma_offset_manager->vm_lock) 647 { 648 write_lock(&i915->drm.vma_offset_manager->vm_lock); 649 } 650 651 static void mmap_offset_unlock(struct drm_i915_private *i915) 652 __releases(&i915->drm.vma_offset_manager->vm_lock) 653 { 654 write_unlock(&i915->drm.vma_offset_manager->vm_lock); 655 } 656 657 static int igt_mmap_offset_exhaustion(void *arg) 658 { 659 struct drm_i915_private *i915 = arg; 660 struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm; 661 struct drm_i915_gem_object *obj; 662 struct drm_mm_node *hole, *next; 663 int loop, err = 0; 664 u64 offset; 665 int enospc = HAS_LMEM(i915) ? -ENXIO : -ENOSPC; 666 667 /* Disable background reaper */ 668 disable_retire_worker(i915); 669 GEM_BUG_ON(!to_gt(i915)->awake); 670 intel_gt_retire_requests(to_gt(i915)); 671 i915_gem_drain_freed_objects(i915); 672 673 /* Trim the device mmap space to only a page */ 674 mmap_offset_lock(i915); 675 loop = 1; /* PAGE_SIZE units */ 676 list_for_each_entry_safe(hole, next, &mm->hole_stack, hole_stack) { 677 struct drm_mm_node *resv; 678 679 resv = kzalloc(sizeof(*resv), GFP_NOWAIT); 680 if (!resv) { 681 err = -ENOMEM; 682 goto out_park; 683 } 684 685 resv->start = drm_mm_hole_node_start(hole) + loop; 686 resv->size = hole->hole_size - loop; 687 resv->color = -1ul; 688 loop = 0; 689 690 if (!resv->size) { 691 kfree(resv); 692 continue; 693 } 694 695 pr_debug("Reserving hole [%llx + %llx]\n", 696 resv->start, resv->size); 697 698 err = drm_mm_reserve_node(mm, resv); 699 if (err) { 700 pr_err("Failed to trim VMA manager, err=%d\n", err); 701 kfree(resv); 702 goto out_park; 703 } 704 } 705 GEM_BUG_ON(!list_is_singular(&mm->hole_stack)); 706 mmap_offset_unlock(i915); 707 708 /* Just fits! */ 709 if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) { 710 pr_err("Unable to insert object into single page hole\n"); 711 err = -EINVAL; 712 goto out; 713 } 714 715 /* Too large */ 716 if (!assert_mmap_offset(i915, 2 * PAGE_SIZE, enospc)) { 717 pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n"); 718 err = -EINVAL; 719 goto out; 720 } 721 722 /* Fill the hole, further allocation attempts should then fail */ 723 obj = create_sys_or_internal(i915, PAGE_SIZE); 724 if (IS_ERR(obj)) { 725 err = PTR_ERR(obj); 726 pr_err("Unable to create object for reclaimed hole\n"); 727 goto out; 728 } 729 730 err = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL); 731 if (err) { 732 pr_err("Unable to insert object into reclaimed hole\n"); 733 goto err_obj; 734 } 735 736 if (!assert_mmap_offset(i915, PAGE_SIZE, enospc)) { 737 pr_err("Unexpectedly succeeded in inserting object into no holes!\n"); 738 err = -EINVAL; 739 goto err_obj; 740 } 741 742 i915_gem_object_put(obj); 743 744 /* Now fill with busy dead objects that we expect to reap */ 745 for (loop = 0; loop < 3; loop++) { 746 if (intel_gt_is_wedged(to_gt(i915))) 747 break; 748 749 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 750 if (IS_ERR(obj)) { 751 err = PTR_ERR(obj); 752 goto out; 753 } 754 755 err = make_obj_busy(obj); 756 if (err) { 757 pr_err("[loop %d] Failed to busy the object\n", loop); 758 goto err_obj; 759 } 760 } 761 762 out: 763 mmap_offset_lock(i915); 764 out_park: 765 drm_mm_for_each_node_safe(hole, next, mm) { 766 if (hole->color != -1ul) 767 continue; 768 769 drm_mm_remove_node(hole); 770 kfree(hole); 771 } 772 mmap_offset_unlock(i915); 773 restore_retire_worker(i915); 774 return err; 775 err_obj: 776 i915_gem_object_put(obj); 777 goto out; 778 } 779 780 static int gtt_set(struct drm_i915_gem_object *obj) 781 { 782 struct i915_vma *vma; 783 void __iomem *map; 784 int err = 0; 785 786 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); 787 if (IS_ERR(vma)) 788 return PTR_ERR(vma); 789 790 intel_gt_pm_get(vma->vm->gt); 791 map = i915_vma_pin_iomap(vma); 792 i915_vma_unpin(vma); 793 if (IS_ERR(map)) { 794 err = PTR_ERR(map); 795 goto out; 796 } 797 798 memset_io(map, POISON_INUSE, obj->base.size); 799 i915_vma_unpin_iomap(vma); 800 801 out: 802 intel_gt_pm_put(vma->vm->gt); 803 return err; 804 } 805 806 static int gtt_check(struct drm_i915_gem_object *obj) 807 { 808 struct i915_vma *vma; 809 void __iomem *map; 810 int err = 0; 811 812 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); 813 if (IS_ERR(vma)) 814 return PTR_ERR(vma); 815 816 intel_gt_pm_get(vma->vm->gt); 817 map = i915_vma_pin_iomap(vma); 818 i915_vma_unpin(vma); 819 if (IS_ERR(map)) { 820 err = PTR_ERR(map); 821 goto out; 822 } 823 824 if (memchr_inv((void __force *)map, POISON_FREE, obj->base.size)) { 825 pr_err("%s: Write via mmap did not land in backing store (GTT)\n", 826 obj->mm.region->name); 827 err = -EINVAL; 828 } 829 i915_vma_unpin_iomap(vma); 830 831 out: 832 intel_gt_pm_put(vma->vm->gt); 833 return err; 834 } 835 836 static int wc_set(struct drm_i915_gem_object *obj) 837 { 838 void *vaddr; 839 840 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 841 if (IS_ERR(vaddr)) 842 return PTR_ERR(vaddr); 843 844 memset(vaddr, POISON_INUSE, obj->base.size); 845 i915_gem_object_flush_map(obj); 846 i915_gem_object_unpin_map(obj); 847 848 return 0; 849 } 850 851 static int wc_check(struct drm_i915_gem_object *obj) 852 { 853 void *vaddr; 854 int err = 0; 855 856 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 857 if (IS_ERR(vaddr)) 858 return PTR_ERR(vaddr); 859 860 if (memchr_inv(vaddr, POISON_FREE, obj->base.size)) { 861 pr_err("%s: Write via mmap did not land in backing store (WC)\n", 862 obj->mm.region->name); 863 err = -EINVAL; 864 } 865 i915_gem_object_unpin_map(obj); 866 867 return err; 868 } 869 870 static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) 871 { 872 struct drm_i915_private *i915 = to_i915(obj->base.dev); 873 bool no_map; 874 875 if (obj->ops->mmap_offset) 876 return type == I915_MMAP_TYPE_FIXED; 877 else if (type == I915_MMAP_TYPE_FIXED) 878 return false; 879 880 if (type == I915_MMAP_TYPE_GTT && 881 !i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 882 return false; 883 884 i915_gem_object_lock(obj, NULL); 885 no_map = (type != I915_MMAP_TYPE_GTT && 886 !i915_gem_object_has_struct_page(obj) && 887 !i915_gem_object_has_iomem(obj)); 888 i915_gem_object_unlock(obj); 889 890 return !no_map; 891 } 892 893 #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24)) 894 static int __igt_mmap(struct drm_i915_private *i915, 895 struct drm_i915_gem_object *obj, 896 enum i915_mmap_type type) 897 { 898 struct vm_area_struct *area; 899 unsigned long addr; 900 int err, i; 901 u64 offset; 902 903 if (!can_mmap(obj, type)) 904 return 0; 905 906 err = wc_set(obj); 907 if (err == -ENXIO) 908 err = gtt_set(obj); 909 if (err) 910 return err; 911 912 err = __assign_mmap_offset(obj, type, &offset, NULL); 913 if (err) 914 return err; 915 916 addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED); 917 if (IS_ERR_VALUE(addr)) 918 return addr; 919 920 pr_debug("igt_mmap(%s, %d) @ %lx\n", obj->mm.region->name, type, addr); 921 922 mmap_read_lock(current->mm); 923 area = vma_lookup(current->mm, addr); 924 mmap_read_unlock(current->mm); 925 if (!area) { 926 pr_err("%s: Did not create a vm_area_struct for the mmap\n", 927 obj->mm.region->name); 928 err = -EINVAL; 929 goto out_unmap; 930 } 931 932 for (i = 0; i < obj->base.size / sizeof(u32); i++) { 933 u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux))); 934 u32 x; 935 936 if (get_user(x, ux)) { 937 pr_err("%s: Unable to read from mmap, offset:%zd\n", 938 obj->mm.region->name, i * sizeof(x)); 939 err = -EFAULT; 940 goto out_unmap; 941 } 942 943 if (x != expand32(POISON_INUSE)) { 944 pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n", 945 obj->mm.region->name, 946 i * sizeof(x), x, expand32(POISON_INUSE)); 947 err = -EINVAL; 948 goto out_unmap; 949 } 950 951 x = expand32(POISON_FREE); 952 if (put_user(x, ux)) { 953 pr_err("%s: Unable to write to mmap, offset:%zd\n", 954 obj->mm.region->name, i * sizeof(x)); 955 err = -EFAULT; 956 goto out_unmap; 957 } 958 } 959 960 if (type == I915_MMAP_TYPE_GTT) 961 intel_gt_flush_ggtt_writes(to_gt(i915)); 962 963 err = wc_check(obj); 964 if (err == -ENXIO) 965 err = gtt_check(obj); 966 out_unmap: 967 vm_munmap(addr, obj->base.size); 968 return err; 969 } 970 971 static int igt_mmap(void *arg) 972 { 973 struct drm_i915_private *i915 = arg; 974 struct intel_memory_region *mr; 975 enum intel_region_id id; 976 977 for_each_memory_region(mr, i915, id) { 978 unsigned long sizes[] = { 979 PAGE_SIZE, 980 mr->min_page_size, 981 SZ_4M, 982 }; 983 int i; 984 985 if (mr->private) 986 continue; 987 988 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 989 struct drm_i915_gem_object *obj; 990 int err; 991 992 obj = __i915_gem_object_create_user(i915, sizes[i], &mr, 1); 993 if (obj == ERR_PTR(-ENODEV)) 994 continue; 995 996 if (IS_ERR(obj)) 997 return PTR_ERR(obj); 998 999 err = __igt_mmap(i915, obj, I915_MMAP_TYPE_GTT); 1000 if (err == 0) 1001 err = __igt_mmap(i915, obj, I915_MMAP_TYPE_WC); 1002 if (err == 0) 1003 err = __igt_mmap(i915, obj, I915_MMAP_TYPE_FIXED); 1004 1005 i915_gem_object_put(obj); 1006 if (err) 1007 return err; 1008 } 1009 } 1010 1011 return 0; 1012 } 1013 1014 static void igt_close_objects(struct drm_i915_private *i915, 1015 struct list_head *objects) 1016 { 1017 struct drm_i915_gem_object *obj, *on; 1018 1019 list_for_each_entry_safe(obj, on, objects, st_link) { 1020 i915_gem_object_lock(obj, NULL); 1021 if (i915_gem_object_has_pinned_pages(obj)) 1022 i915_gem_object_unpin_pages(obj); 1023 /* No polluting the memory region between tests */ 1024 __i915_gem_object_put_pages(obj); 1025 i915_gem_object_unlock(obj); 1026 list_del(&obj->st_link); 1027 i915_gem_object_put(obj); 1028 } 1029 1030 cond_resched(); 1031 1032 i915_gem_drain_freed_objects(i915); 1033 } 1034 1035 static void igt_make_evictable(struct list_head *objects) 1036 { 1037 struct drm_i915_gem_object *obj; 1038 1039 list_for_each_entry(obj, objects, st_link) { 1040 i915_gem_object_lock(obj, NULL); 1041 if (i915_gem_object_has_pinned_pages(obj)) 1042 i915_gem_object_unpin_pages(obj); 1043 i915_gem_object_unlock(obj); 1044 } 1045 1046 cond_resched(); 1047 } 1048 1049 static int igt_fill_mappable(struct intel_memory_region *mr, 1050 struct list_head *objects) 1051 { 1052 u64 size, total; 1053 int err; 1054 1055 total = 0; 1056 size = mr->io_size; 1057 do { 1058 struct drm_i915_gem_object *obj; 1059 1060 obj = i915_gem_object_create_region(mr, size, 0, 0); 1061 if (IS_ERR(obj)) { 1062 err = PTR_ERR(obj); 1063 goto err_close; 1064 } 1065 1066 list_add(&obj->st_link, objects); 1067 1068 err = i915_gem_object_pin_pages_unlocked(obj); 1069 if (err) { 1070 if (err != -ENXIO && err != -ENOMEM) 1071 goto err_close; 1072 1073 if (size == mr->min_page_size) { 1074 err = 0; 1075 break; 1076 } 1077 1078 size >>= 1; 1079 continue; 1080 } 1081 1082 total += obj->base.size; 1083 } while (1); 1084 1085 pr_info("%s filled=%lluMiB\n", __func__, total >> 20); 1086 return 0; 1087 1088 err_close: 1089 igt_close_objects(mr->i915, objects); 1090 return err; 1091 } 1092 1093 static int ___igt_mmap_migrate(struct drm_i915_private *i915, 1094 struct drm_i915_gem_object *obj, 1095 unsigned long addr, 1096 bool unfaultable) 1097 { 1098 struct vm_area_struct *area; 1099 int err = 0, i; 1100 1101 pr_info("igt_mmap(%s, %d) @ %lx\n", 1102 obj->mm.region->name, I915_MMAP_TYPE_FIXED, addr); 1103 1104 mmap_read_lock(current->mm); 1105 area = vma_lookup(current->mm, addr); 1106 mmap_read_unlock(current->mm); 1107 if (!area) { 1108 pr_err("%s: Did not create a vm_area_struct for the mmap\n", 1109 obj->mm.region->name); 1110 err = -EINVAL; 1111 goto out_unmap; 1112 } 1113 1114 for (i = 0; i < obj->base.size / sizeof(u32); i++) { 1115 u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux))); 1116 u32 x; 1117 1118 if (get_user(x, ux)) { 1119 err = -EFAULT; 1120 if (!unfaultable) { 1121 pr_err("%s: Unable to read from mmap, offset:%zd\n", 1122 obj->mm.region->name, i * sizeof(x)); 1123 goto out_unmap; 1124 } 1125 1126 continue; 1127 } 1128 1129 if (unfaultable) { 1130 pr_err("%s: Faulted unmappable memory\n", 1131 obj->mm.region->name); 1132 err = -EINVAL; 1133 goto out_unmap; 1134 } 1135 1136 if (x != expand32(POISON_INUSE)) { 1137 pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n", 1138 obj->mm.region->name, 1139 i * sizeof(x), x, expand32(POISON_INUSE)); 1140 err = -EINVAL; 1141 goto out_unmap; 1142 } 1143 1144 x = expand32(POISON_FREE); 1145 if (put_user(x, ux)) { 1146 pr_err("%s: Unable to write to mmap, offset:%zd\n", 1147 obj->mm.region->name, i * sizeof(x)); 1148 err = -EFAULT; 1149 goto out_unmap; 1150 } 1151 } 1152 1153 if (unfaultable) { 1154 if (err == -EFAULT) 1155 err = 0; 1156 } else { 1157 obj->flags &= ~I915_BO_ALLOC_GPU_ONLY; 1158 err = wc_check(obj); 1159 } 1160 out_unmap: 1161 vm_munmap(addr, obj->base.size); 1162 return err; 1163 } 1164 1165 #define IGT_MMAP_MIGRATE_TOPDOWN (1 << 0) 1166 #define IGT_MMAP_MIGRATE_FILL (1 << 1) 1167 #define IGT_MMAP_MIGRATE_EVICTABLE (1 << 2) 1168 #define IGT_MMAP_MIGRATE_UNFAULTABLE (1 << 3) 1169 #define IGT_MMAP_MIGRATE_FAIL_GPU (1 << 4) 1170 static int __igt_mmap_migrate(struct intel_memory_region **placements, 1171 int n_placements, 1172 struct intel_memory_region *expected_mr, 1173 unsigned int flags) 1174 { 1175 struct drm_i915_private *i915 = placements[0]->i915; 1176 struct drm_i915_gem_object *obj; 1177 struct i915_request *rq = NULL; 1178 unsigned long addr; 1179 LIST_HEAD(objects); 1180 u64 offset; 1181 int err; 1182 1183 obj = __i915_gem_object_create_user(i915, PAGE_SIZE, 1184 placements, 1185 n_placements); 1186 if (IS_ERR(obj)) 1187 return PTR_ERR(obj); 1188 1189 if (flags & IGT_MMAP_MIGRATE_TOPDOWN) 1190 obj->flags |= I915_BO_ALLOC_GPU_ONLY; 1191 1192 err = __assign_mmap_offset(obj, I915_MMAP_TYPE_FIXED, &offset, NULL); 1193 if (err) 1194 goto out_put; 1195 1196 /* 1197 * This will eventually create a GEM context, due to opening dummy drm 1198 * file, which needs a tiny amount of mappable device memory for the top 1199 * level paging structures(and perhaps scratch), so make sure we 1200 * allocate early, to avoid tears. 1201 */ 1202 addr = igt_mmap_offset(i915, offset, obj->base.size, 1203 PROT_WRITE, MAP_SHARED); 1204 if (IS_ERR_VALUE(addr)) { 1205 err = addr; 1206 goto out_put; 1207 } 1208 1209 if (flags & IGT_MMAP_MIGRATE_FILL) { 1210 err = igt_fill_mappable(placements[0], &objects); 1211 if (err) 1212 goto out_put; 1213 } 1214 1215 err = i915_gem_object_lock(obj, NULL); 1216 if (err) 1217 goto out_put; 1218 1219 err = i915_gem_object_pin_pages(obj); 1220 if (err) { 1221 i915_gem_object_unlock(obj); 1222 goto out_put; 1223 } 1224 1225 err = intel_context_migrate_clear(to_gt(i915)->migrate.context, NULL, 1226 obj->mm.pages->sgl, obj->cache_level, 1227 i915_gem_object_is_lmem(obj), 1228 expand32(POISON_INUSE), &rq); 1229 i915_gem_object_unpin_pages(obj); 1230 if (rq) { 1231 err = dma_resv_reserve_fences(obj->base.resv, 1); 1232 if (!err) 1233 dma_resv_add_fence(obj->base.resv, &rq->fence, 1234 DMA_RESV_USAGE_KERNEL); 1235 i915_request_put(rq); 1236 } 1237 i915_gem_object_unlock(obj); 1238 if (err) 1239 goto out_put; 1240 1241 if (flags & IGT_MMAP_MIGRATE_EVICTABLE) 1242 igt_make_evictable(&objects); 1243 1244 if (flags & IGT_MMAP_MIGRATE_FAIL_GPU) { 1245 err = i915_gem_object_lock(obj, NULL); 1246 if (err) 1247 goto out_put; 1248 1249 /* 1250 * Ensure we only simulate the gpu failuire when faulting the 1251 * pages. 1252 */ 1253 err = i915_gem_object_wait_moving_fence(obj, true); 1254 i915_gem_object_unlock(obj); 1255 if (err) 1256 goto out_put; 1257 i915_ttm_migrate_set_failure_modes(true, false); 1258 } 1259 1260 err = ___igt_mmap_migrate(i915, obj, addr, 1261 flags & IGT_MMAP_MIGRATE_UNFAULTABLE); 1262 1263 if (!err && obj->mm.region != expected_mr) { 1264 pr_err("%s region mismatch %s\n", __func__, expected_mr->name); 1265 err = -EINVAL; 1266 } 1267 1268 if (flags & IGT_MMAP_MIGRATE_FAIL_GPU) { 1269 struct intel_gt *gt; 1270 unsigned int id; 1271 1272 i915_ttm_migrate_set_failure_modes(false, false); 1273 1274 for_each_gt(gt, i915, id) { 1275 intel_wakeref_t wakeref; 1276 bool wedged; 1277 1278 mutex_lock(>->reset.mutex); 1279 wedged = test_bit(I915_WEDGED, >->reset.flags); 1280 mutex_unlock(>->reset.mutex); 1281 if (!wedged) { 1282 pr_err("gt(%u) not wedged\n", id); 1283 err = -EINVAL; 1284 continue; 1285 } 1286 1287 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1288 igt_global_reset_lock(gt); 1289 intel_gt_reset(gt, ALL_ENGINES, NULL); 1290 igt_global_reset_unlock(gt); 1291 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1292 } 1293 1294 if (!i915_gem_object_has_unknown_state(obj)) { 1295 pr_err("object missing unknown_state\n"); 1296 err = -EINVAL; 1297 } 1298 } 1299 1300 out_put: 1301 i915_gem_object_put(obj); 1302 igt_close_objects(i915, &objects); 1303 return err; 1304 } 1305 1306 static int igt_mmap_migrate(void *arg) 1307 { 1308 struct drm_i915_private *i915 = arg; 1309 struct intel_memory_region *system = i915->mm.regions[INTEL_REGION_SMEM]; 1310 struct intel_memory_region *mr; 1311 enum intel_region_id id; 1312 1313 for_each_memory_region(mr, i915, id) { 1314 struct intel_memory_region *mixed[] = { mr, system }; 1315 struct intel_memory_region *single[] = { mr }; 1316 struct ttm_resource_manager *man = mr->region_private; 1317 resource_size_t saved_io_size; 1318 int err; 1319 1320 if (mr->private) 1321 continue; 1322 1323 if (!mr->io_size) 1324 continue; 1325 1326 /* 1327 * For testing purposes let's force small BAR, if not already 1328 * present. 1329 */ 1330 saved_io_size = mr->io_size; 1331 if (mr->io_size == mr->total) { 1332 resource_size_t io_size = mr->io_size; 1333 1334 io_size = rounddown_pow_of_two(io_size >> 1); 1335 if (io_size < PAGE_SIZE) 1336 continue; 1337 1338 mr->io_size = io_size; 1339 i915_ttm_buddy_man_force_visible_size(man, 1340 io_size >> PAGE_SHIFT); 1341 } 1342 1343 /* 1344 * Allocate in the mappable portion, should be no suprises here. 1345 */ 1346 err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), mr, 0); 1347 if (err) 1348 goto out_io_size; 1349 1350 /* 1351 * Allocate in the non-mappable portion, but force migrating to 1352 * the mappable portion on fault (LMEM -> LMEM) 1353 */ 1354 err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, 1355 IGT_MMAP_MIGRATE_TOPDOWN | 1356 IGT_MMAP_MIGRATE_FILL | 1357 IGT_MMAP_MIGRATE_EVICTABLE); 1358 if (err) 1359 goto out_io_size; 1360 1361 /* 1362 * Allocate in the non-mappable portion, but force spilling into 1363 * system memory on fault (LMEM -> SMEM) 1364 */ 1365 err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), system, 1366 IGT_MMAP_MIGRATE_TOPDOWN | 1367 IGT_MMAP_MIGRATE_FILL); 1368 if (err) 1369 goto out_io_size; 1370 1371 /* 1372 * Allocate in the non-mappable portion, but since the mappable 1373 * portion is already full, and we can't spill to system memory, 1374 * then we should expect the fault to fail. 1375 */ 1376 err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, 1377 IGT_MMAP_MIGRATE_TOPDOWN | 1378 IGT_MMAP_MIGRATE_FILL | 1379 IGT_MMAP_MIGRATE_UNFAULTABLE); 1380 if (err) 1381 goto out_io_size; 1382 1383 /* 1384 * Allocate in the non-mappable portion, but force migrating to 1385 * the mappable portion on fault (LMEM -> LMEM). We then also 1386 * simulate a gpu error when moving the pages when faulting the 1387 * pages, which should result in wedging the gpu and returning 1388 * SIGBUS in the fault handler, since we can't fallback to 1389 * memcpy. 1390 */ 1391 err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, 1392 IGT_MMAP_MIGRATE_TOPDOWN | 1393 IGT_MMAP_MIGRATE_FILL | 1394 IGT_MMAP_MIGRATE_EVICTABLE | 1395 IGT_MMAP_MIGRATE_FAIL_GPU | 1396 IGT_MMAP_MIGRATE_UNFAULTABLE); 1397 out_io_size: 1398 mr->io_size = saved_io_size; 1399 i915_ttm_buddy_man_force_visible_size(man, 1400 mr->io_size >> PAGE_SHIFT); 1401 if (err) 1402 return err; 1403 } 1404 1405 return 0; 1406 } 1407 1408 static const char *repr_mmap_type(enum i915_mmap_type type) 1409 { 1410 switch (type) { 1411 case I915_MMAP_TYPE_GTT: return "gtt"; 1412 case I915_MMAP_TYPE_WB: return "wb"; 1413 case I915_MMAP_TYPE_WC: return "wc"; 1414 case I915_MMAP_TYPE_UC: return "uc"; 1415 case I915_MMAP_TYPE_FIXED: return "fixed"; 1416 default: return "unknown"; 1417 } 1418 } 1419 1420 static bool can_access(struct drm_i915_gem_object *obj) 1421 { 1422 bool access; 1423 1424 i915_gem_object_lock(obj, NULL); 1425 access = i915_gem_object_has_struct_page(obj) || 1426 i915_gem_object_has_iomem(obj); 1427 i915_gem_object_unlock(obj); 1428 1429 return access; 1430 } 1431 1432 static int __igt_mmap_access(struct drm_i915_private *i915, 1433 struct drm_i915_gem_object *obj, 1434 enum i915_mmap_type type) 1435 { 1436 unsigned long __user *ptr; 1437 unsigned long A, B; 1438 unsigned long x, y; 1439 unsigned long addr; 1440 int err; 1441 u64 offset; 1442 1443 memset(&A, 0xAA, sizeof(A)); 1444 memset(&B, 0xBB, sizeof(B)); 1445 1446 if (!can_mmap(obj, type) || !can_access(obj)) 1447 return 0; 1448 1449 err = __assign_mmap_offset(obj, type, &offset, NULL); 1450 if (err) 1451 return err; 1452 1453 addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED); 1454 if (IS_ERR_VALUE(addr)) 1455 return addr; 1456 ptr = (unsigned long __user *)addr; 1457 1458 err = __put_user(A, ptr); 1459 if (err) { 1460 pr_err("%s(%s): failed to write into user mmap\n", 1461 obj->mm.region->name, repr_mmap_type(type)); 1462 goto out_unmap; 1463 } 1464 1465 intel_gt_flush_ggtt_writes(to_gt(i915)); 1466 1467 err = access_process_vm(current, addr, &x, sizeof(x), 0); 1468 if (err != sizeof(x)) { 1469 pr_err("%s(%s): access_process_vm() read failed\n", 1470 obj->mm.region->name, repr_mmap_type(type)); 1471 goto out_unmap; 1472 } 1473 1474 err = access_process_vm(current, addr, &B, sizeof(B), FOLL_WRITE); 1475 if (err != sizeof(B)) { 1476 pr_err("%s(%s): access_process_vm() write failed\n", 1477 obj->mm.region->name, repr_mmap_type(type)); 1478 goto out_unmap; 1479 } 1480 1481 intel_gt_flush_ggtt_writes(to_gt(i915)); 1482 1483 err = __get_user(y, ptr); 1484 if (err) { 1485 pr_err("%s(%s): failed to read from user mmap\n", 1486 obj->mm.region->name, repr_mmap_type(type)); 1487 goto out_unmap; 1488 } 1489 1490 if (x != A || y != B) { 1491 pr_err("%s(%s): failed to read/write values, found (%lx, %lx)\n", 1492 obj->mm.region->name, repr_mmap_type(type), 1493 x, y); 1494 err = -EINVAL; 1495 goto out_unmap; 1496 } 1497 1498 out_unmap: 1499 vm_munmap(addr, obj->base.size); 1500 return err; 1501 } 1502 1503 static int igt_mmap_access(void *arg) 1504 { 1505 struct drm_i915_private *i915 = arg; 1506 struct intel_memory_region *mr; 1507 enum intel_region_id id; 1508 1509 for_each_memory_region(mr, i915, id) { 1510 struct drm_i915_gem_object *obj; 1511 int err; 1512 1513 if (mr->private) 1514 continue; 1515 1516 obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); 1517 if (obj == ERR_PTR(-ENODEV)) 1518 continue; 1519 1520 if (IS_ERR(obj)) 1521 return PTR_ERR(obj); 1522 1523 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_GTT); 1524 if (err == 0) 1525 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WB); 1526 if (err == 0) 1527 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WC); 1528 if (err == 0) 1529 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_UC); 1530 if (err == 0) 1531 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_FIXED); 1532 1533 i915_gem_object_put(obj); 1534 if (err) 1535 return err; 1536 } 1537 1538 return 0; 1539 } 1540 1541 static int __igt_mmap_gpu(struct drm_i915_private *i915, 1542 struct drm_i915_gem_object *obj, 1543 enum i915_mmap_type type) 1544 { 1545 struct intel_engine_cs *engine; 1546 unsigned long addr; 1547 u32 __user *ux; 1548 u32 bbe; 1549 int err; 1550 u64 offset; 1551 1552 /* 1553 * Verify that the mmap access into the backing store aligns with 1554 * that of the GPU, i.e. that mmap is indeed writing into the same 1555 * page as being read by the GPU. 1556 */ 1557 1558 if (!can_mmap(obj, type)) 1559 return 0; 1560 1561 err = wc_set(obj); 1562 if (err == -ENXIO) 1563 err = gtt_set(obj); 1564 if (err) 1565 return err; 1566 1567 err = __assign_mmap_offset(obj, type, &offset, NULL); 1568 if (err) 1569 return err; 1570 1571 addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED); 1572 if (IS_ERR_VALUE(addr)) 1573 return addr; 1574 1575 ux = u64_to_user_ptr((u64)addr); 1576 bbe = MI_BATCH_BUFFER_END; 1577 if (put_user(bbe, ux)) { 1578 pr_err("%s: Unable to write to mmap\n", obj->mm.region->name); 1579 err = -EFAULT; 1580 goto out_unmap; 1581 } 1582 1583 if (type == I915_MMAP_TYPE_GTT) 1584 intel_gt_flush_ggtt_writes(to_gt(i915)); 1585 1586 for_each_uabi_engine(engine, i915) { 1587 struct i915_request *rq; 1588 struct i915_vma *vma; 1589 struct i915_gem_ww_ctx ww; 1590 1591 vma = i915_vma_instance(obj, engine->kernel_context->vm, NULL); 1592 if (IS_ERR(vma)) { 1593 err = PTR_ERR(vma); 1594 goto out_unmap; 1595 } 1596 1597 i915_gem_ww_ctx_init(&ww, false); 1598 retry: 1599 err = i915_gem_object_lock(obj, &ww); 1600 if (!err) 1601 err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); 1602 if (err) 1603 goto out_ww; 1604 1605 rq = i915_request_create(engine->kernel_context); 1606 if (IS_ERR(rq)) { 1607 err = PTR_ERR(rq); 1608 goto out_unpin; 1609 } 1610 1611 err = i915_request_await_object(rq, vma->obj, false); 1612 if (err == 0) 1613 err = i915_vma_move_to_active(vma, rq, 0); 1614 1615 err = engine->emit_bb_start(rq, vma->node.start, 0, 0); 1616 i915_request_get(rq); 1617 i915_request_add(rq); 1618 1619 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1620 struct drm_printer p = 1621 drm_info_printer(engine->i915->drm.dev); 1622 1623 pr_err("%s(%s, %s): Failed to execute batch\n", 1624 __func__, engine->name, obj->mm.region->name); 1625 intel_engine_dump(engine, &p, 1626 "%s\n", engine->name); 1627 1628 intel_gt_set_wedged(engine->gt); 1629 err = -EIO; 1630 } 1631 i915_request_put(rq); 1632 1633 out_unpin: 1634 i915_vma_unpin(vma); 1635 out_ww: 1636 if (err == -EDEADLK) { 1637 err = i915_gem_ww_ctx_backoff(&ww); 1638 if (!err) 1639 goto retry; 1640 } 1641 i915_gem_ww_ctx_fini(&ww); 1642 if (err) 1643 goto out_unmap; 1644 } 1645 1646 out_unmap: 1647 vm_munmap(addr, obj->base.size); 1648 return err; 1649 } 1650 1651 static int igt_mmap_gpu(void *arg) 1652 { 1653 struct drm_i915_private *i915 = arg; 1654 struct intel_memory_region *mr; 1655 enum intel_region_id id; 1656 1657 for_each_memory_region(mr, i915, id) { 1658 struct drm_i915_gem_object *obj; 1659 int err; 1660 1661 if (mr->private) 1662 continue; 1663 1664 obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); 1665 if (obj == ERR_PTR(-ENODEV)) 1666 continue; 1667 1668 if (IS_ERR(obj)) 1669 return PTR_ERR(obj); 1670 1671 err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_GTT); 1672 if (err == 0) 1673 err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_WC); 1674 if (err == 0) 1675 err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_FIXED); 1676 1677 i915_gem_object_put(obj); 1678 if (err) 1679 return err; 1680 } 1681 1682 return 0; 1683 } 1684 1685 static int check_present_pte(pte_t *pte, unsigned long addr, void *data) 1686 { 1687 if (!pte_present(*pte) || pte_none(*pte)) { 1688 pr_err("missing PTE:%lx\n", 1689 (addr - (unsigned long)data) >> PAGE_SHIFT); 1690 return -EINVAL; 1691 } 1692 1693 return 0; 1694 } 1695 1696 static int check_absent_pte(pte_t *pte, unsigned long addr, void *data) 1697 { 1698 if (pte_present(*pte) && !pte_none(*pte)) { 1699 pr_err("present PTE:%lx; expected to be revoked\n", 1700 (addr - (unsigned long)data) >> PAGE_SHIFT); 1701 return -EINVAL; 1702 } 1703 1704 return 0; 1705 } 1706 1707 static int check_present(unsigned long addr, unsigned long len) 1708 { 1709 return apply_to_page_range(current->mm, addr, len, 1710 check_present_pte, (void *)addr); 1711 } 1712 1713 static int check_absent(unsigned long addr, unsigned long len) 1714 { 1715 return apply_to_page_range(current->mm, addr, len, 1716 check_absent_pte, (void *)addr); 1717 } 1718 1719 static int prefault_range(u64 start, u64 len) 1720 { 1721 const char __user *addr, *end; 1722 char __maybe_unused c; 1723 int err; 1724 1725 addr = u64_to_user_ptr(start); 1726 end = addr + len; 1727 1728 for (; addr < end; addr += PAGE_SIZE) { 1729 err = __get_user(c, addr); 1730 if (err) 1731 return err; 1732 } 1733 1734 return __get_user(c, end - 1); 1735 } 1736 1737 static int __igt_mmap_revoke(struct drm_i915_private *i915, 1738 struct drm_i915_gem_object *obj, 1739 enum i915_mmap_type type) 1740 { 1741 unsigned long addr; 1742 int err; 1743 u64 offset; 1744 1745 if (!can_mmap(obj, type)) 1746 return 0; 1747 1748 err = __assign_mmap_offset(obj, type, &offset, NULL); 1749 if (err) 1750 return err; 1751 1752 addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED); 1753 if (IS_ERR_VALUE(addr)) 1754 return addr; 1755 1756 err = prefault_range(addr, obj->base.size); 1757 if (err) 1758 goto out_unmap; 1759 1760 err = check_present(addr, obj->base.size); 1761 if (err) { 1762 pr_err("%s: was not present\n", obj->mm.region->name); 1763 goto out_unmap; 1764 } 1765 1766 /* 1767 * After unbinding the object from the GGTT, its address may be reused 1768 * for other objects. Ergo we have to revoke the previous mmap PTE 1769 * access as it no longer points to the same object. 1770 */ 1771 i915_gem_object_lock(obj, NULL); 1772 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 1773 i915_gem_object_unlock(obj); 1774 if (err) { 1775 pr_err("Failed to unbind object!\n"); 1776 goto out_unmap; 1777 } 1778 1779 if (type != I915_MMAP_TYPE_GTT) { 1780 i915_gem_object_lock(obj, NULL); 1781 __i915_gem_object_put_pages(obj); 1782 i915_gem_object_unlock(obj); 1783 if (i915_gem_object_has_pages(obj)) { 1784 pr_err("Failed to put-pages object!\n"); 1785 err = -EINVAL; 1786 goto out_unmap; 1787 } 1788 } 1789 1790 err = check_absent(addr, obj->base.size); 1791 if (err) { 1792 pr_err("%s: was not absent\n", obj->mm.region->name); 1793 goto out_unmap; 1794 } 1795 1796 out_unmap: 1797 vm_munmap(addr, obj->base.size); 1798 return err; 1799 } 1800 1801 static int igt_mmap_revoke(void *arg) 1802 { 1803 struct drm_i915_private *i915 = arg; 1804 struct intel_memory_region *mr; 1805 enum intel_region_id id; 1806 1807 for_each_memory_region(mr, i915, id) { 1808 struct drm_i915_gem_object *obj; 1809 int err; 1810 1811 if (mr->private) 1812 continue; 1813 1814 obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); 1815 if (obj == ERR_PTR(-ENODEV)) 1816 continue; 1817 1818 if (IS_ERR(obj)) 1819 return PTR_ERR(obj); 1820 1821 err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT); 1822 if (err == 0) 1823 err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC); 1824 if (err == 0) 1825 err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_FIXED); 1826 1827 i915_gem_object_put(obj); 1828 if (err) 1829 return err; 1830 } 1831 1832 return 0; 1833 } 1834 1835 int i915_gem_mman_live_selftests(struct drm_i915_private *i915) 1836 { 1837 static const struct i915_subtest tests[] = { 1838 SUBTEST(igt_partial_tiling), 1839 SUBTEST(igt_smoke_tiling), 1840 SUBTEST(igt_mmap_offset_exhaustion), 1841 SUBTEST(igt_mmap), 1842 SUBTEST(igt_mmap_migrate), 1843 SUBTEST(igt_mmap_access), 1844 SUBTEST(igt_mmap_revoke), 1845 SUBTEST(igt_mmap_gpu), 1846 }; 1847 1848 return i915_live_subtests(tests, i915); 1849 } 1850