1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #include <linux/highmem.h> 8 #include <linux/prime_numbers.h> 9 10 #include "gem/i915_gem_internal.h" 11 #include "gem/i915_gem_region.h" 12 #include "gem/i915_gem_ttm.h" 13 #include "gt/intel_engine_pm.h" 14 #include "gt/intel_gpu_commands.h" 15 #include "gt/intel_gt.h" 16 #include "gt/intel_gt_pm.h" 17 #include "gt/intel_migrate.h" 18 #include "i915_ttm_buddy_manager.h" 19 20 #include "huge_gem_object.h" 21 #include "i915_selftest.h" 22 #include "selftests/i915_random.h" 23 #include "selftests/igt_flush_test.h" 24 #include "selftests/igt_mmap.h" 25 26 struct tile { 27 unsigned int width; 28 unsigned int height; 29 unsigned int stride; 30 unsigned int size; 31 unsigned int tiling; 32 unsigned int swizzle; 33 }; 34 35 static u64 swizzle_bit(unsigned int bit, u64 offset) 36 { 37 return (offset & BIT_ULL(bit)) >> (bit - 6); 38 } 39 40 static u64 tiled_offset(const struct tile *tile, u64 v) 41 { 42 u64 x, y; 43 44 if (tile->tiling == I915_TILING_NONE) 45 return v; 46 47 y = div64_u64_rem(v, tile->stride, &x); 48 v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height; 49 50 if (tile->tiling == I915_TILING_X) { 51 v += y * tile->width; 52 v += div64_u64_rem(x, tile->width, &x) << tile->size; 53 v += x; 54 } else if (tile->width == 128) { 55 const unsigned int ytile_span = 16; 56 const unsigned int ytile_height = 512; 57 58 v += y * ytile_span; 59 v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 60 v += x; 61 } else { 62 const unsigned int ytile_span = 32; 63 const unsigned int ytile_height = 256; 64 65 v += y * ytile_span; 66 v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 67 v += x; 68 } 69 70 switch (tile->swizzle) { 71 case I915_BIT_6_SWIZZLE_9: 72 v ^= swizzle_bit(9, v); 73 break; 74 case I915_BIT_6_SWIZZLE_9_10: 75 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v); 76 break; 77 case I915_BIT_6_SWIZZLE_9_11: 78 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v); 79 break; 80 case I915_BIT_6_SWIZZLE_9_10_11: 81 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v); 82 break; 83 } 84 85 return v; 86 } 87 88 static int check_partial_mapping(struct drm_i915_gem_object *obj, 89 const struct tile *tile, 90 struct rnd_state *prng) 91 { 92 const unsigned long npages = obj->base.size / PAGE_SIZE; 93 struct drm_i915_private *i915 = to_i915(obj->base.dev); 94 struct i915_ggtt_view view; 95 struct i915_vma *vma; 96 unsigned long page; 97 u32 __iomem *io; 98 struct page *p; 99 unsigned int n; 100 u64 offset; 101 u32 *cpu; 102 int err; 103 104 err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); 105 if (err) { 106 pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", 107 tile->tiling, tile->stride, err); 108 return err; 109 } 110 111 GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); 112 GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); 113 114 i915_gem_object_lock(obj, NULL); 115 err = i915_gem_object_set_to_gtt_domain(obj, true); 116 i915_gem_object_unlock(obj); 117 if (err) { 118 pr_err("Failed to flush to GTT write domain; err=%d\n", err); 119 return err; 120 } 121 122 page = i915_prandom_u32_max_state(npages, prng); 123 view = compute_partial_view(obj, page, MIN_CHUNK_PAGES); 124 125 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 126 if (IS_ERR(vma)) { 127 pr_err("Failed to pin partial view: offset=%lu; err=%d\n", 128 page, (int)PTR_ERR(vma)); 129 return PTR_ERR(vma); 130 } 131 132 n = page - view.partial.offset; 133 GEM_BUG_ON(n >= view.partial.size); 134 135 io = i915_vma_pin_iomap(vma); 136 i915_vma_unpin(vma); 137 if (IS_ERR(io)) { 138 pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", 139 page, (int)PTR_ERR(io)); 140 err = PTR_ERR(io); 141 goto out; 142 } 143 144 iowrite32(page, io + n * PAGE_SIZE / sizeof(*io)); 145 i915_vma_unpin_iomap(vma); 146 147 offset = tiled_offset(tile, page << PAGE_SHIFT); 148 if (offset >= obj->base.size) 149 goto out; 150 151 intel_gt_flush_ggtt_writes(to_gt(i915)); 152 153 p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); 154 cpu = kmap(p) + offset_in_page(offset); 155 drm_clflush_virt_range(cpu, sizeof(*cpu)); 156 if (*cpu != (u32)page) { 157 pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", 158 page, n, 159 view.partial.offset, 160 view.partial.size, 161 vma->size >> PAGE_SHIFT, 162 tile->tiling ? tile_row_pages(obj) : 0, 163 vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, 164 offset >> PAGE_SHIFT, 165 (unsigned int)offset_in_page(offset), 166 offset, 167 (u32)page, *cpu); 168 err = -EINVAL; 169 } 170 *cpu = 0; 171 drm_clflush_virt_range(cpu, sizeof(*cpu)); 172 kunmap(p); 173 174 out: 175 i915_gem_object_lock(obj, NULL); 176 i915_vma_destroy(vma); 177 i915_gem_object_unlock(obj); 178 return err; 179 } 180 181 static int check_partial_mappings(struct drm_i915_gem_object *obj, 182 const struct tile *tile, 183 unsigned long end_time) 184 { 185 const unsigned int nreal = obj->scratch / PAGE_SIZE; 186 const unsigned long npages = obj->base.size / PAGE_SIZE; 187 struct drm_i915_private *i915 = to_i915(obj->base.dev); 188 struct i915_vma *vma; 189 unsigned long page; 190 int err; 191 192 err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); 193 if (err) { 194 pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", 195 tile->tiling, tile->stride, err); 196 return err; 197 } 198 199 GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); 200 GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); 201 202 i915_gem_object_lock(obj, NULL); 203 err = i915_gem_object_set_to_gtt_domain(obj, true); 204 i915_gem_object_unlock(obj); 205 if (err) { 206 pr_err("Failed to flush to GTT write domain; err=%d\n", err); 207 return err; 208 } 209 210 for_each_prime_number_from(page, 1, npages) { 211 struct i915_ggtt_view view = 212 compute_partial_view(obj, page, MIN_CHUNK_PAGES); 213 u32 __iomem *io; 214 struct page *p; 215 unsigned int n; 216 u64 offset; 217 u32 *cpu; 218 219 GEM_BUG_ON(view.partial.size > nreal); 220 cond_resched(); 221 222 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 223 if (IS_ERR(vma)) { 224 pr_err("Failed to pin partial view: offset=%lu; err=%d\n", 225 page, (int)PTR_ERR(vma)); 226 return PTR_ERR(vma); 227 } 228 229 n = page - view.partial.offset; 230 GEM_BUG_ON(n >= view.partial.size); 231 232 io = i915_vma_pin_iomap(vma); 233 i915_vma_unpin(vma); 234 if (IS_ERR(io)) { 235 pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", 236 page, (int)PTR_ERR(io)); 237 return PTR_ERR(io); 238 } 239 240 iowrite32(page, io + n * PAGE_SIZE / sizeof(*io)); 241 i915_vma_unpin_iomap(vma); 242 243 offset = tiled_offset(tile, page << PAGE_SHIFT); 244 if (offset >= obj->base.size) 245 continue; 246 247 intel_gt_flush_ggtt_writes(to_gt(i915)); 248 249 p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); 250 cpu = kmap(p) + offset_in_page(offset); 251 drm_clflush_virt_range(cpu, sizeof(*cpu)); 252 if (*cpu != (u32)page) { 253 pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", 254 page, n, 255 view.partial.offset, 256 view.partial.size, 257 vma->size >> PAGE_SHIFT, 258 tile->tiling ? tile_row_pages(obj) : 0, 259 vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, 260 offset >> PAGE_SHIFT, 261 (unsigned int)offset_in_page(offset), 262 offset, 263 (u32)page, *cpu); 264 err = -EINVAL; 265 } 266 *cpu = 0; 267 drm_clflush_virt_range(cpu, sizeof(*cpu)); 268 kunmap(p); 269 if (err) 270 return err; 271 272 i915_gem_object_lock(obj, NULL); 273 i915_vma_destroy(vma); 274 i915_gem_object_unlock(obj); 275 276 if (igt_timeout(end_time, 277 "%s: timed out after tiling=%d stride=%d\n", 278 __func__, tile->tiling, tile->stride)) 279 return -EINTR; 280 } 281 282 return 0; 283 } 284 285 static unsigned int 286 setup_tile_size(struct tile *tile, struct drm_i915_private *i915) 287 { 288 if (GRAPHICS_VER(i915) <= 2) { 289 tile->height = 16; 290 tile->width = 128; 291 tile->size = 11; 292 } else if (tile->tiling == I915_TILING_Y && 293 HAS_128_BYTE_Y_TILING(i915)) { 294 tile->height = 32; 295 tile->width = 128; 296 tile->size = 12; 297 } else { 298 tile->height = 8; 299 tile->width = 512; 300 tile->size = 12; 301 } 302 303 if (GRAPHICS_VER(i915) < 4) 304 return 8192 / tile->width; 305 else if (GRAPHICS_VER(i915) < 7) 306 return 128 * I965_FENCE_MAX_PITCH_VAL / tile->width; 307 else 308 return 128 * GEN7_FENCE_MAX_PITCH_VAL / tile->width; 309 } 310 311 static int igt_partial_tiling(void *arg) 312 { 313 const unsigned int nreal = 1 << 12; /* largest tile row x2 */ 314 struct drm_i915_private *i915 = arg; 315 struct drm_i915_gem_object *obj; 316 intel_wakeref_t wakeref; 317 int tiling; 318 int err; 319 320 if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 321 return 0; 322 323 /* We want to check the page mapping and fencing of a large object 324 * mmapped through the GTT. The object we create is larger than can 325 * possibly be mmaped as a whole, and so we must use partial GGTT vma. 326 * We then check that a write through each partial GGTT vma ends up 327 * in the right set of pages within the object, and with the expected 328 * tiling, which we verify by manual swizzling. 329 */ 330 331 obj = huge_gem_object(i915, 332 nreal << PAGE_SHIFT, 333 (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); 334 if (IS_ERR(obj)) 335 return PTR_ERR(obj); 336 337 err = i915_gem_object_pin_pages_unlocked(obj); 338 if (err) { 339 pr_err("Failed to allocate %u pages (%lu total), err=%d\n", 340 nreal, obj->base.size / PAGE_SIZE, err); 341 goto out; 342 } 343 344 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 345 346 if (1) { 347 IGT_TIMEOUT(end); 348 struct tile tile; 349 350 tile.height = 1; 351 tile.width = 1; 352 tile.size = 0; 353 tile.stride = 0; 354 tile.swizzle = I915_BIT_6_SWIZZLE_NONE; 355 tile.tiling = I915_TILING_NONE; 356 357 err = check_partial_mappings(obj, &tile, end); 358 if (err && err != -EINTR) 359 goto out_unlock; 360 } 361 362 for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) { 363 IGT_TIMEOUT(end); 364 unsigned int max_pitch; 365 unsigned int pitch; 366 struct tile tile; 367 368 if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) 369 /* 370 * The swizzling pattern is actually unknown as it 371 * varies based on physical address of each page. 372 * See i915_gem_detect_bit_6_swizzle(). 373 */ 374 break; 375 376 tile.tiling = tiling; 377 switch (tiling) { 378 case I915_TILING_X: 379 tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x; 380 break; 381 case I915_TILING_Y: 382 tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y; 383 break; 384 } 385 386 GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN); 387 if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || 388 tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) 389 continue; 390 391 max_pitch = setup_tile_size(&tile, i915); 392 393 for (pitch = max_pitch; pitch; pitch >>= 1) { 394 tile.stride = tile.width * pitch; 395 err = check_partial_mappings(obj, &tile, end); 396 if (err == -EINTR) 397 goto next_tiling; 398 if (err) 399 goto out_unlock; 400 401 if (pitch > 2 && GRAPHICS_VER(i915) >= 4) { 402 tile.stride = tile.width * (pitch - 1); 403 err = check_partial_mappings(obj, &tile, end); 404 if (err == -EINTR) 405 goto next_tiling; 406 if (err) 407 goto out_unlock; 408 } 409 410 if (pitch < max_pitch && GRAPHICS_VER(i915) >= 4) { 411 tile.stride = tile.width * (pitch + 1); 412 err = check_partial_mappings(obj, &tile, end); 413 if (err == -EINTR) 414 goto next_tiling; 415 if (err) 416 goto out_unlock; 417 } 418 } 419 420 if (GRAPHICS_VER(i915) >= 4) { 421 for_each_prime_number(pitch, max_pitch) { 422 tile.stride = tile.width * pitch; 423 err = check_partial_mappings(obj, &tile, end); 424 if (err == -EINTR) 425 goto next_tiling; 426 if (err) 427 goto out_unlock; 428 } 429 } 430 431 next_tiling: ; 432 } 433 434 out_unlock: 435 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 436 i915_gem_object_unpin_pages(obj); 437 out: 438 i915_gem_object_put(obj); 439 return err; 440 } 441 442 static int igt_smoke_tiling(void *arg) 443 { 444 const unsigned int nreal = 1 << 12; /* largest tile row x2 */ 445 struct drm_i915_private *i915 = arg; 446 struct drm_i915_gem_object *obj; 447 intel_wakeref_t wakeref; 448 I915_RND_STATE(prng); 449 unsigned long count; 450 IGT_TIMEOUT(end); 451 int err; 452 453 if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 454 return 0; 455 456 /* 457 * igt_partial_tiling() does an exhastive check of partial tiling 458 * chunking, but will undoubtably run out of time. Here, we do a 459 * randomised search and hope over many runs of 1s with different 460 * seeds we will do a thorough check. 461 * 462 * Remember to look at the st_seed if we see a flip-flop in BAT! 463 */ 464 465 if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) 466 return 0; 467 468 obj = huge_gem_object(i915, 469 nreal << PAGE_SHIFT, 470 (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); 471 if (IS_ERR(obj)) 472 return PTR_ERR(obj); 473 474 err = i915_gem_object_pin_pages_unlocked(obj); 475 if (err) { 476 pr_err("Failed to allocate %u pages (%lu total), err=%d\n", 477 nreal, obj->base.size / PAGE_SIZE, err); 478 goto out; 479 } 480 481 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 482 483 count = 0; 484 do { 485 struct tile tile; 486 487 tile.tiling = 488 i915_prandom_u32_max_state(I915_TILING_Y + 1, &prng); 489 switch (tile.tiling) { 490 case I915_TILING_NONE: 491 tile.height = 1; 492 tile.width = 1; 493 tile.size = 0; 494 tile.stride = 0; 495 tile.swizzle = I915_BIT_6_SWIZZLE_NONE; 496 break; 497 498 case I915_TILING_X: 499 tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x; 500 break; 501 case I915_TILING_Y: 502 tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y; 503 break; 504 } 505 506 if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || 507 tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) 508 continue; 509 510 if (tile.tiling != I915_TILING_NONE) { 511 unsigned int max_pitch = setup_tile_size(&tile, i915); 512 513 tile.stride = 514 i915_prandom_u32_max_state(max_pitch, &prng); 515 tile.stride = (1 + tile.stride) * tile.width; 516 if (GRAPHICS_VER(i915) < 4) 517 tile.stride = rounddown_pow_of_two(tile.stride); 518 } 519 520 err = check_partial_mapping(obj, &tile, &prng); 521 if (err) 522 break; 523 524 count++; 525 } while (!__igt_timeout(end, NULL)); 526 527 pr_info("%s: Completed %lu trials\n", __func__, count); 528 529 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 530 i915_gem_object_unpin_pages(obj); 531 out: 532 i915_gem_object_put(obj); 533 return err; 534 } 535 536 static int make_obj_busy(struct drm_i915_gem_object *obj) 537 { 538 struct drm_i915_private *i915 = to_i915(obj->base.dev); 539 struct intel_engine_cs *engine; 540 541 for_each_uabi_engine(engine, i915) { 542 struct i915_request *rq; 543 struct i915_vma *vma; 544 struct i915_gem_ww_ctx ww; 545 int err; 546 547 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 548 if (IS_ERR(vma)) 549 return PTR_ERR(vma); 550 551 i915_gem_ww_ctx_init(&ww, false); 552 retry: 553 err = i915_gem_object_lock(obj, &ww); 554 if (!err) 555 err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); 556 if (err) 557 goto err; 558 559 rq = intel_engine_create_kernel_request(engine); 560 if (IS_ERR(rq)) { 561 err = PTR_ERR(rq); 562 goto err_unpin; 563 } 564 565 err = i915_request_await_object(rq, vma->obj, true); 566 if (err == 0) 567 err = i915_vma_move_to_active(vma, rq, 568 EXEC_OBJECT_WRITE); 569 570 i915_request_add(rq); 571 err_unpin: 572 i915_vma_unpin(vma); 573 err: 574 if (err == -EDEADLK) { 575 err = i915_gem_ww_ctx_backoff(&ww); 576 if (!err) 577 goto retry; 578 } 579 i915_gem_ww_ctx_fini(&ww); 580 if (err) 581 return err; 582 } 583 584 i915_gem_object_put(obj); /* leave it only alive via its active ref */ 585 return 0; 586 } 587 588 static enum i915_mmap_type default_mapping(struct drm_i915_private *i915) 589 { 590 if (HAS_LMEM(i915)) 591 return I915_MMAP_TYPE_FIXED; 592 593 return I915_MMAP_TYPE_GTT; 594 } 595 596 static struct drm_i915_gem_object * 597 create_sys_or_internal(struct drm_i915_private *i915, 598 unsigned long size) 599 { 600 if (HAS_LMEM(i915)) { 601 struct intel_memory_region *sys_region = 602 i915->mm.regions[INTEL_REGION_SMEM]; 603 604 return __i915_gem_object_create_user(i915, size, &sys_region, 1); 605 } 606 607 return i915_gem_object_create_internal(i915, size); 608 } 609 610 static bool assert_mmap_offset(struct drm_i915_private *i915, 611 unsigned long size, 612 int expected) 613 { 614 struct drm_i915_gem_object *obj; 615 u64 offset; 616 int ret; 617 618 obj = create_sys_or_internal(i915, size); 619 if (IS_ERR(obj)) 620 return expected && expected == PTR_ERR(obj); 621 622 ret = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL); 623 i915_gem_object_put(obj); 624 625 return ret == expected; 626 } 627 628 static void disable_retire_worker(struct drm_i915_private *i915) 629 { 630 i915_gem_driver_unregister__shrinker(i915); 631 intel_gt_pm_get(to_gt(i915)); 632 cancel_delayed_work_sync(&to_gt(i915)->requests.retire_work); 633 } 634 635 static void restore_retire_worker(struct drm_i915_private *i915) 636 { 637 igt_flush_test(i915); 638 intel_gt_pm_put(to_gt(i915)); 639 i915_gem_driver_register__shrinker(i915); 640 } 641 642 static void mmap_offset_lock(struct drm_i915_private *i915) 643 __acquires(&i915->drm.vma_offset_manager->vm_lock) 644 { 645 write_lock(&i915->drm.vma_offset_manager->vm_lock); 646 } 647 648 static void mmap_offset_unlock(struct drm_i915_private *i915) 649 __releases(&i915->drm.vma_offset_manager->vm_lock) 650 { 651 write_unlock(&i915->drm.vma_offset_manager->vm_lock); 652 } 653 654 static int igt_mmap_offset_exhaustion(void *arg) 655 { 656 struct drm_i915_private *i915 = arg; 657 struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm; 658 struct drm_i915_gem_object *obj; 659 struct drm_mm_node *hole, *next; 660 int loop, err = 0; 661 u64 offset; 662 int enospc = HAS_LMEM(i915) ? -ENXIO : -ENOSPC; 663 664 /* Disable background reaper */ 665 disable_retire_worker(i915); 666 GEM_BUG_ON(!to_gt(i915)->awake); 667 intel_gt_retire_requests(to_gt(i915)); 668 i915_gem_drain_freed_objects(i915); 669 670 /* Trim the device mmap space to only a page */ 671 mmap_offset_lock(i915); 672 loop = 1; /* PAGE_SIZE units */ 673 list_for_each_entry_safe(hole, next, &mm->hole_stack, hole_stack) { 674 struct drm_mm_node *resv; 675 676 resv = kzalloc(sizeof(*resv), GFP_NOWAIT); 677 if (!resv) { 678 err = -ENOMEM; 679 goto out_park; 680 } 681 682 resv->start = drm_mm_hole_node_start(hole) + loop; 683 resv->size = hole->hole_size - loop; 684 resv->color = -1ul; 685 loop = 0; 686 687 if (!resv->size) { 688 kfree(resv); 689 continue; 690 } 691 692 pr_debug("Reserving hole [%llx + %llx]\n", 693 resv->start, resv->size); 694 695 err = drm_mm_reserve_node(mm, resv); 696 if (err) { 697 pr_err("Failed to trim VMA manager, err=%d\n", err); 698 kfree(resv); 699 goto out_park; 700 } 701 } 702 GEM_BUG_ON(!list_is_singular(&mm->hole_stack)); 703 mmap_offset_unlock(i915); 704 705 /* Just fits! */ 706 if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) { 707 pr_err("Unable to insert object into single page hole\n"); 708 err = -EINVAL; 709 goto out; 710 } 711 712 /* Too large */ 713 if (!assert_mmap_offset(i915, 2 * PAGE_SIZE, enospc)) { 714 pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n"); 715 err = -EINVAL; 716 goto out; 717 } 718 719 /* Fill the hole, further allocation attempts should then fail */ 720 obj = create_sys_or_internal(i915, PAGE_SIZE); 721 if (IS_ERR(obj)) { 722 err = PTR_ERR(obj); 723 pr_err("Unable to create object for reclaimed hole\n"); 724 goto out; 725 } 726 727 err = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL); 728 if (err) { 729 pr_err("Unable to insert object into reclaimed hole\n"); 730 goto err_obj; 731 } 732 733 if (!assert_mmap_offset(i915, PAGE_SIZE, enospc)) { 734 pr_err("Unexpectedly succeeded in inserting object into no holes!\n"); 735 err = -EINVAL; 736 goto err_obj; 737 } 738 739 i915_gem_object_put(obj); 740 741 /* Now fill with busy dead objects that we expect to reap */ 742 for (loop = 0; loop < 3; loop++) { 743 if (intel_gt_is_wedged(to_gt(i915))) 744 break; 745 746 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 747 if (IS_ERR(obj)) { 748 err = PTR_ERR(obj); 749 goto out; 750 } 751 752 err = make_obj_busy(obj); 753 if (err) { 754 pr_err("[loop %d] Failed to busy the object\n", loop); 755 goto err_obj; 756 } 757 } 758 759 out: 760 mmap_offset_lock(i915); 761 out_park: 762 drm_mm_for_each_node_safe(hole, next, mm) { 763 if (hole->color != -1ul) 764 continue; 765 766 drm_mm_remove_node(hole); 767 kfree(hole); 768 } 769 mmap_offset_unlock(i915); 770 restore_retire_worker(i915); 771 return err; 772 err_obj: 773 i915_gem_object_put(obj); 774 goto out; 775 } 776 777 static int gtt_set(struct drm_i915_gem_object *obj) 778 { 779 struct i915_vma *vma; 780 void __iomem *map; 781 int err = 0; 782 783 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); 784 if (IS_ERR(vma)) 785 return PTR_ERR(vma); 786 787 intel_gt_pm_get(vma->vm->gt); 788 map = i915_vma_pin_iomap(vma); 789 i915_vma_unpin(vma); 790 if (IS_ERR(map)) { 791 err = PTR_ERR(map); 792 goto out; 793 } 794 795 memset_io(map, POISON_INUSE, obj->base.size); 796 i915_vma_unpin_iomap(vma); 797 798 out: 799 intel_gt_pm_put(vma->vm->gt); 800 return err; 801 } 802 803 static int gtt_check(struct drm_i915_gem_object *obj) 804 { 805 struct i915_vma *vma; 806 void __iomem *map; 807 int err = 0; 808 809 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); 810 if (IS_ERR(vma)) 811 return PTR_ERR(vma); 812 813 intel_gt_pm_get(vma->vm->gt); 814 map = i915_vma_pin_iomap(vma); 815 i915_vma_unpin(vma); 816 if (IS_ERR(map)) { 817 err = PTR_ERR(map); 818 goto out; 819 } 820 821 if (memchr_inv((void __force *)map, POISON_FREE, obj->base.size)) { 822 pr_err("%s: Write via mmap did not land in backing store (GTT)\n", 823 obj->mm.region->name); 824 err = -EINVAL; 825 } 826 i915_vma_unpin_iomap(vma); 827 828 out: 829 intel_gt_pm_put(vma->vm->gt); 830 return err; 831 } 832 833 static int wc_set(struct drm_i915_gem_object *obj) 834 { 835 void *vaddr; 836 837 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 838 if (IS_ERR(vaddr)) 839 return PTR_ERR(vaddr); 840 841 memset(vaddr, POISON_INUSE, obj->base.size); 842 i915_gem_object_flush_map(obj); 843 i915_gem_object_unpin_map(obj); 844 845 return 0; 846 } 847 848 static int wc_check(struct drm_i915_gem_object *obj) 849 { 850 void *vaddr; 851 int err = 0; 852 853 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 854 if (IS_ERR(vaddr)) 855 return PTR_ERR(vaddr); 856 857 if (memchr_inv(vaddr, POISON_FREE, obj->base.size)) { 858 pr_err("%s: Write via mmap did not land in backing store (WC)\n", 859 obj->mm.region->name); 860 err = -EINVAL; 861 } 862 i915_gem_object_unpin_map(obj); 863 864 return err; 865 } 866 867 static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) 868 { 869 struct drm_i915_private *i915 = to_i915(obj->base.dev); 870 bool no_map; 871 872 if (obj->ops->mmap_offset) 873 return type == I915_MMAP_TYPE_FIXED; 874 else if (type == I915_MMAP_TYPE_FIXED) 875 return false; 876 877 if (type == I915_MMAP_TYPE_GTT && 878 !i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 879 return false; 880 881 i915_gem_object_lock(obj, NULL); 882 no_map = (type != I915_MMAP_TYPE_GTT && 883 !i915_gem_object_has_struct_page(obj) && 884 !i915_gem_object_has_iomem(obj)); 885 i915_gem_object_unlock(obj); 886 887 return !no_map; 888 } 889 890 #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24)) 891 static int __igt_mmap(struct drm_i915_private *i915, 892 struct drm_i915_gem_object *obj, 893 enum i915_mmap_type type) 894 { 895 struct vm_area_struct *area; 896 unsigned long addr; 897 int err, i; 898 u64 offset; 899 900 if (!can_mmap(obj, type)) 901 return 0; 902 903 err = wc_set(obj); 904 if (err == -ENXIO) 905 err = gtt_set(obj); 906 if (err) 907 return err; 908 909 err = __assign_mmap_offset(obj, type, &offset, NULL); 910 if (err) 911 return err; 912 913 addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED); 914 if (IS_ERR_VALUE(addr)) 915 return addr; 916 917 pr_debug("igt_mmap(%s, %d) @ %lx\n", obj->mm.region->name, type, addr); 918 919 mmap_read_lock(current->mm); 920 area = vma_lookup(current->mm, addr); 921 mmap_read_unlock(current->mm); 922 if (!area) { 923 pr_err("%s: Did not create a vm_area_struct for the mmap\n", 924 obj->mm.region->name); 925 err = -EINVAL; 926 goto out_unmap; 927 } 928 929 for (i = 0; i < obj->base.size / sizeof(u32); i++) { 930 u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux))); 931 u32 x; 932 933 if (get_user(x, ux)) { 934 pr_err("%s: Unable to read from mmap, offset:%zd\n", 935 obj->mm.region->name, i * sizeof(x)); 936 err = -EFAULT; 937 goto out_unmap; 938 } 939 940 if (x != expand32(POISON_INUSE)) { 941 pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n", 942 obj->mm.region->name, 943 i * sizeof(x), x, expand32(POISON_INUSE)); 944 err = -EINVAL; 945 goto out_unmap; 946 } 947 948 x = expand32(POISON_FREE); 949 if (put_user(x, ux)) { 950 pr_err("%s: Unable to write to mmap, offset:%zd\n", 951 obj->mm.region->name, i * sizeof(x)); 952 err = -EFAULT; 953 goto out_unmap; 954 } 955 } 956 957 if (type == I915_MMAP_TYPE_GTT) 958 intel_gt_flush_ggtt_writes(to_gt(i915)); 959 960 err = wc_check(obj); 961 if (err == -ENXIO) 962 err = gtt_check(obj); 963 out_unmap: 964 vm_munmap(addr, obj->base.size); 965 return err; 966 } 967 968 static int igt_mmap(void *arg) 969 { 970 struct drm_i915_private *i915 = arg; 971 struct intel_memory_region *mr; 972 enum intel_region_id id; 973 974 for_each_memory_region(mr, i915, id) { 975 unsigned long sizes[] = { 976 PAGE_SIZE, 977 mr->min_page_size, 978 SZ_4M, 979 }; 980 int i; 981 982 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 983 struct drm_i915_gem_object *obj; 984 int err; 985 986 obj = __i915_gem_object_create_user(i915, sizes[i], &mr, 1); 987 if (obj == ERR_PTR(-ENODEV)) 988 continue; 989 990 if (IS_ERR(obj)) 991 return PTR_ERR(obj); 992 993 err = __igt_mmap(i915, obj, I915_MMAP_TYPE_GTT); 994 if (err == 0) 995 err = __igt_mmap(i915, obj, I915_MMAP_TYPE_WC); 996 if (err == 0) 997 err = __igt_mmap(i915, obj, I915_MMAP_TYPE_FIXED); 998 999 i915_gem_object_put(obj); 1000 if (err) 1001 return err; 1002 } 1003 } 1004 1005 return 0; 1006 } 1007 1008 static void igt_close_objects(struct drm_i915_private *i915, 1009 struct list_head *objects) 1010 { 1011 struct drm_i915_gem_object *obj, *on; 1012 1013 list_for_each_entry_safe(obj, on, objects, st_link) { 1014 i915_gem_object_lock(obj, NULL); 1015 if (i915_gem_object_has_pinned_pages(obj)) 1016 i915_gem_object_unpin_pages(obj); 1017 /* No polluting the memory region between tests */ 1018 __i915_gem_object_put_pages(obj); 1019 i915_gem_object_unlock(obj); 1020 list_del(&obj->st_link); 1021 i915_gem_object_put(obj); 1022 } 1023 1024 cond_resched(); 1025 1026 i915_gem_drain_freed_objects(i915); 1027 } 1028 1029 static void igt_make_evictable(struct list_head *objects) 1030 { 1031 struct drm_i915_gem_object *obj; 1032 1033 list_for_each_entry(obj, objects, st_link) { 1034 i915_gem_object_lock(obj, NULL); 1035 if (i915_gem_object_has_pinned_pages(obj)) 1036 i915_gem_object_unpin_pages(obj); 1037 i915_gem_object_unlock(obj); 1038 } 1039 1040 cond_resched(); 1041 } 1042 1043 static int igt_fill_mappable(struct intel_memory_region *mr, 1044 struct list_head *objects) 1045 { 1046 u64 size, total; 1047 int err; 1048 1049 total = 0; 1050 size = mr->io_size; 1051 do { 1052 struct drm_i915_gem_object *obj; 1053 1054 obj = i915_gem_object_create_region(mr, size, 0, 0); 1055 if (IS_ERR(obj)) { 1056 err = PTR_ERR(obj); 1057 goto err_close; 1058 } 1059 1060 list_add(&obj->st_link, objects); 1061 1062 err = i915_gem_object_pin_pages_unlocked(obj); 1063 if (err) { 1064 if (err != -ENXIO && err != -ENOMEM) 1065 goto err_close; 1066 1067 if (size == mr->min_page_size) { 1068 err = 0; 1069 break; 1070 } 1071 1072 size >>= 1; 1073 continue; 1074 } 1075 1076 total += obj->base.size; 1077 } while (1); 1078 1079 pr_info("%s filled=%lluMiB\n", __func__, total >> 20); 1080 return 0; 1081 1082 err_close: 1083 igt_close_objects(mr->i915, objects); 1084 return err; 1085 } 1086 1087 static int ___igt_mmap_migrate(struct drm_i915_private *i915, 1088 struct drm_i915_gem_object *obj, 1089 unsigned long addr, 1090 bool unfaultable) 1091 { 1092 struct vm_area_struct *area; 1093 int err = 0, i; 1094 1095 pr_info("igt_mmap(%s, %d) @ %lx\n", 1096 obj->mm.region->name, I915_MMAP_TYPE_FIXED, addr); 1097 1098 mmap_read_lock(current->mm); 1099 area = vma_lookup(current->mm, addr); 1100 mmap_read_unlock(current->mm); 1101 if (!area) { 1102 pr_err("%s: Did not create a vm_area_struct for the mmap\n", 1103 obj->mm.region->name); 1104 err = -EINVAL; 1105 goto out_unmap; 1106 } 1107 1108 for (i = 0; i < obj->base.size / sizeof(u32); i++) { 1109 u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux))); 1110 u32 x; 1111 1112 if (get_user(x, ux)) { 1113 err = -EFAULT; 1114 if (!unfaultable) { 1115 pr_err("%s: Unable to read from mmap, offset:%zd\n", 1116 obj->mm.region->name, i * sizeof(x)); 1117 goto out_unmap; 1118 } 1119 1120 continue; 1121 } 1122 1123 if (unfaultable) { 1124 pr_err("%s: Faulted unmappable memory\n", 1125 obj->mm.region->name); 1126 err = -EINVAL; 1127 goto out_unmap; 1128 } 1129 1130 if (x != expand32(POISON_INUSE)) { 1131 pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n", 1132 obj->mm.region->name, 1133 i * sizeof(x), x, expand32(POISON_INUSE)); 1134 err = -EINVAL; 1135 goto out_unmap; 1136 } 1137 1138 x = expand32(POISON_FREE); 1139 if (put_user(x, ux)) { 1140 pr_err("%s: Unable to write to mmap, offset:%zd\n", 1141 obj->mm.region->name, i * sizeof(x)); 1142 err = -EFAULT; 1143 goto out_unmap; 1144 } 1145 } 1146 1147 if (unfaultable) { 1148 if (err == -EFAULT) 1149 err = 0; 1150 } else { 1151 obj->flags &= ~I915_BO_ALLOC_GPU_ONLY; 1152 err = wc_check(obj); 1153 } 1154 out_unmap: 1155 vm_munmap(addr, obj->base.size); 1156 return err; 1157 } 1158 1159 #define IGT_MMAP_MIGRATE_TOPDOWN (1 << 0) 1160 #define IGT_MMAP_MIGRATE_FILL (1 << 1) 1161 #define IGT_MMAP_MIGRATE_EVICTABLE (1 << 2) 1162 #define IGT_MMAP_MIGRATE_UNFAULTABLE (1 << 3) 1163 static int __igt_mmap_migrate(struct intel_memory_region **placements, 1164 int n_placements, 1165 struct intel_memory_region *expected_mr, 1166 unsigned int flags) 1167 { 1168 struct drm_i915_private *i915 = placements[0]->i915; 1169 struct drm_i915_gem_object *obj; 1170 struct i915_request *rq = NULL; 1171 unsigned long addr; 1172 LIST_HEAD(objects); 1173 u64 offset; 1174 int err; 1175 1176 obj = __i915_gem_object_create_user(i915, PAGE_SIZE, 1177 placements, 1178 n_placements); 1179 if (IS_ERR(obj)) 1180 return PTR_ERR(obj); 1181 1182 if (flags & IGT_MMAP_MIGRATE_TOPDOWN) 1183 obj->flags |= I915_BO_ALLOC_GPU_ONLY; 1184 1185 err = __assign_mmap_offset(obj, I915_MMAP_TYPE_FIXED, &offset, NULL); 1186 if (err) 1187 goto out_put; 1188 1189 /* 1190 * This will eventually create a GEM context, due to opening dummy drm 1191 * file, which needs a tiny amount of mappable device memory for the top 1192 * level paging structures(and perhaps scratch), so make sure we 1193 * allocate early, to avoid tears. 1194 */ 1195 addr = igt_mmap_offset(i915, offset, obj->base.size, 1196 PROT_WRITE, MAP_SHARED); 1197 if (IS_ERR_VALUE(addr)) { 1198 err = addr; 1199 goto out_put; 1200 } 1201 1202 if (flags & IGT_MMAP_MIGRATE_FILL) { 1203 err = igt_fill_mappable(placements[0], &objects); 1204 if (err) 1205 goto out_put; 1206 } 1207 1208 err = i915_gem_object_lock(obj, NULL); 1209 if (err) 1210 goto out_put; 1211 1212 err = i915_gem_object_pin_pages(obj); 1213 if (err) { 1214 i915_gem_object_unlock(obj); 1215 goto out_put; 1216 } 1217 1218 err = intel_context_migrate_clear(to_gt(i915)->migrate.context, NULL, 1219 obj->mm.pages->sgl, obj->cache_level, 1220 i915_gem_object_is_lmem(obj), 1221 expand32(POISON_INUSE), &rq); 1222 i915_gem_object_unpin_pages(obj); 1223 if (rq) { 1224 dma_resv_add_fence(obj->base.resv, &rq->fence, 1225 DMA_RESV_USAGE_KERNEL); 1226 i915_request_put(rq); 1227 } 1228 i915_gem_object_unlock(obj); 1229 if (err) 1230 goto out_put; 1231 1232 if (flags & IGT_MMAP_MIGRATE_EVICTABLE) 1233 igt_make_evictable(&objects); 1234 1235 err = ___igt_mmap_migrate(i915, obj, addr, 1236 flags & IGT_MMAP_MIGRATE_UNFAULTABLE); 1237 if (!err && obj->mm.region != expected_mr) { 1238 pr_err("%s region mismatch %s\n", __func__, expected_mr->name); 1239 err = -EINVAL; 1240 } 1241 1242 out_put: 1243 i915_gem_object_put(obj); 1244 igt_close_objects(i915, &objects); 1245 return err; 1246 } 1247 1248 static int igt_mmap_migrate(void *arg) 1249 { 1250 struct drm_i915_private *i915 = arg; 1251 struct intel_memory_region *system = i915->mm.regions[INTEL_REGION_SMEM]; 1252 struct intel_memory_region *mr; 1253 enum intel_region_id id; 1254 1255 for_each_memory_region(mr, i915, id) { 1256 struct intel_memory_region *mixed[] = { mr, system }; 1257 struct intel_memory_region *single[] = { mr }; 1258 struct ttm_resource_manager *man = mr->region_private; 1259 resource_size_t saved_io_size; 1260 int err; 1261 1262 if (mr->private) 1263 continue; 1264 1265 if (!mr->io_size) 1266 continue; 1267 1268 /* 1269 * For testing purposes let's force small BAR, if not already 1270 * present. 1271 */ 1272 saved_io_size = mr->io_size; 1273 if (mr->io_size == mr->total) { 1274 resource_size_t io_size = mr->io_size; 1275 1276 io_size = rounddown_pow_of_two(io_size >> 1); 1277 if (io_size < PAGE_SIZE) 1278 continue; 1279 1280 mr->io_size = io_size; 1281 i915_ttm_buddy_man_force_visible_size(man, 1282 io_size >> PAGE_SHIFT); 1283 } 1284 1285 /* 1286 * Allocate in the mappable portion, should be no suprises here. 1287 */ 1288 err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), mr, 0); 1289 if (err) 1290 goto out_io_size; 1291 1292 /* 1293 * Allocate in the non-mappable portion, but force migrating to 1294 * the mappable portion on fault (LMEM -> LMEM) 1295 */ 1296 err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, 1297 IGT_MMAP_MIGRATE_TOPDOWN | 1298 IGT_MMAP_MIGRATE_FILL | 1299 IGT_MMAP_MIGRATE_EVICTABLE); 1300 if (err) 1301 goto out_io_size; 1302 1303 /* 1304 * Allocate in the non-mappable portion, but force spilling into 1305 * system memory on fault (LMEM -> SMEM) 1306 */ 1307 err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), system, 1308 IGT_MMAP_MIGRATE_TOPDOWN | 1309 IGT_MMAP_MIGRATE_FILL); 1310 if (err) 1311 goto out_io_size; 1312 1313 /* 1314 * Allocate in the non-mappable portion, but since the mappable 1315 * portion is already full, and we can't spill to system memory, 1316 * then we should expect the fault to fail. 1317 */ 1318 err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, 1319 IGT_MMAP_MIGRATE_TOPDOWN | 1320 IGT_MMAP_MIGRATE_FILL | 1321 IGT_MMAP_MIGRATE_UNFAULTABLE); 1322 out_io_size: 1323 mr->io_size = saved_io_size; 1324 i915_ttm_buddy_man_force_visible_size(man, 1325 mr->io_size >> PAGE_SHIFT); 1326 if (err) 1327 return err; 1328 } 1329 1330 return 0; 1331 } 1332 1333 static const char *repr_mmap_type(enum i915_mmap_type type) 1334 { 1335 switch (type) { 1336 case I915_MMAP_TYPE_GTT: return "gtt"; 1337 case I915_MMAP_TYPE_WB: return "wb"; 1338 case I915_MMAP_TYPE_WC: return "wc"; 1339 case I915_MMAP_TYPE_UC: return "uc"; 1340 case I915_MMAP_TYPE_FIXED: return "fixed"; 1341 default: return "unknown"; 1342 } 1343 } 1344 1345 static bool can_access(struct drm_i915_gem_object *obj) 1346 { 1347 bool access; 1348 1349 i915_gem_object_lock(obj, NULL); 1350 access = i915_gem_object_has_struct_page(obj) || 1351 i915_gem_object_has_iomem(obj); 1352 i915_gem_object_unlock(obj); 1353 1354 return access; 1355 } 1356 1357 static int __igt_mmap_access(struct drm_i915_private *i915, 1358 struct drm_i915_gem_object *obj, 1359 enum i915_mmap_type type) 1360 { 1361 unsigned long __user *ptr; 1362 unsigned long A, B; 1363 unsigned long x, y; 1364 unsigned long addr; 1365 int err; 1366 u64 offset; 1367 1368 memset(&A, 0xAA, sizeof(A)); 1369 memset(&B, 0xBB, sizeof(B)); 1370 1371 if (!can_mmap(obj, type) || !can_access(obj)) 1372 return 0; 1373 1374 err = __assign_mmap_offset(obj, type, &offset, NULL); 1375 if (err) 1376 return err; 1377 1378 addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED); 1379 if (IS_ERR_VALUE(addr)) 1380 return addr; 1381 ptr = (unsigned long __user *)addr; 1382 1383 err = __put_user(A, ptr); 1384 if (err) { 1385 pr_err("%s(%s): failed to write into user mmap\n", 1386 obj->mm.region->name, repr_mmap_type(type)); 1387 goto out_unmap; 1388 } 1389 1390 intel_gt_flush_ggtt_writes(to_gt(i915)); 1391 1392 err = access_process_vm(current, addr, &x, sizeof(x), 0); 1393 if (err != sizeof(x)) { 1394 pr_err("%s(%s): access_process_vm() read failed\n", 1395 obj->mm.region->name, repr_mmap_type(type)); 1396 goto out_unmap; 1397 } 1398 1399 err = access_process_vm(current, addr, &B, sizeof(B), FOLL_WRITE); 1400 if (err != sizeof(B)) { 1401 pr_err("%s(%s): access_process_vm() write failed\n", 1402 obj->mm.region->name, repr_mmap_type(type)); 1403 goto out_unmap; 1404 } 1405 1406 intel_gt_flush_ggtt_writes(to_gt(i915)); 1407 1408 err = __get_user(y, ptr); 1409 if (err) { 1410 pr_err("%s(%s): failed to read from user mmap\n", 1411 obj->mm.region->name, repr_mmap_type(type)); 1412 goto out_unmap; 1413 } 1414 1415 if (x != A || y != B) { 1416 pr_err("%s(%s): failed to read/write values, found (%lx, %lx)\n", 1417 obj->mm.region->name, repr_mmap_type(type), 1418 x, y); 1419 err = -EINVAL; 1420 goto out_unmap; 1421 } 1422 1423 out_unmap: 1424 vm_munmap(addr, obj->base.size); 1425 return err; 1426 } 1427 1428 static int igt_mmap_access(void *arg) 1429 { 1430 struct drm_i915_private *i915 = arg; 1431 struct intel_memory_region *mr; 1432 enum intel_region_id id; 1433 1434 for_each_memory_region(mr, i915, id) { 1435 struct drm_i915_gem_object *obj; 1436 int err; 1437 1438 obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); 1439 if (obj == ERR_PTR(-ENODEV)) 1440 continue; 1441 1442 if (IS_ERR(obj)) 1443 return PTR_ERR(obj); 1444 1445 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_GTT); 1446 if (err == 0) 1447 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WB); 1448 if (err == 0) 1449 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WC); 1450 if (err == 0) 1451 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_UC); 1452 if (err == 0) 1453 err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_FIXED); 1454 1455 i915_gem_object_put(obj); 1456 if (err) 1457 return err; 1458 } 1459 1460 return 0; 1461 } 1462 1463 static int __igt_mmap_gpu(struct drm_i915_private *i915, 1464 struct drm_i915_gem_object *obj, 1465 enum i915_mmap_type type) 1466 { 1467 struct intel_engine_cs *engine; 1468 unsigned long addr; 1469 u32 __user *ux; 1470 u32 bbe; 1471 int err; 1472 u64 offset; 1473 1474 /* 1475 * Verify that the mmap access into the backing store aligns with 1476 * that of the GPU, i.e. that mmap is indeed writing into the same 1477 * page as being read by the GPU. 1478 */ 1479 1480 if (!can_mmap(obj, type)) 1481 return 0; 1482 1483 err = wc_set(obj); 1484 if (err == -ENXIO) 1485 err = gtt_set(obj); 1486 if (err) 1487 return err; 1488 1489 err = __assign_mmap_offset(obj, type, &offset, NULL); 1490 if (err) 1491 return err; 1492 1493 addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED); 1494 if (IS_ERR_VALUE(addr)) 1495 return addr; 1496 1497 ux = u64_to_user_ptr((u64)addr); 1498 bbe = MI_BATCH_BUFFER_END; 1499 if (put_user(bbe, ux)) { 1500 pr_err("%s: Unable to write to mmap\n", obj->mm.region->name); 1501 err = -EFAULT; 1502 goto out_unmap; 1503 } 1504 1505 if (type == I915_MMAP_TYPE_GTT) 1506 intel_gt_flush_ggtt_writes(to_gt(i915)); 1507 1508 for_each_uabi_engine(engine, i915) { 1509 struct i915_request *rq; 1510 struct i915_vma *vma; 1511 struct i915_gem_ww_ctx ww; 1512 1513 vma = i915_vma_instance(obj, engine->kernel_context->vm, NULL); 1514 if (IS_ERR(vma)) { 1515 err = PTR_ERR(vma); 1516 goto out_unmap; 1517 } 1518 1519 i915_gem_ww_ctx_init(&ww, false); 1520 retry: 1521 err = i915_gem_object_lock(obj, &ww); 1522 if (!err) 1523 err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); 1524 if (err) 1525 goto out_ww; 1526 1527 rq = i915_request_create(engine->kernel_context); 1528 if (IS_ERR(rq)) { 1529 err = PTR_ERR(rq); 1530 goto out_unpin; 1531 } 1532 1533 err = i915_request_await_object(rq, vma->obj, false); 1534 if (err == 0) 1535 err = i915_vma_move_to_active(vma, rq, 0); 1536 1537 err = engine->emit_bb_start(rq, vma->node.start, 0, 0); 1538 i915_request_get(rq); 1539 i915_request_add(rq); 1540 1541 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1542 struct drm_printer p = 1543 drm_info_printer(engine->i915->drm.dev); 1544 1545 pr_err("%s(%s, %s): Failed to execute batch\n", 1546 __func__, engine->name, obj->mm.region->name); 1547 intel_engine_dump(engine, &p, 1548 "%s\n", engine->name); 1549 1550 intel_gt_set_wedged(engine->gt); 1551 err = -EIO; 1552 } 1553 i915_request_put(rq); 1554 1555 out_unpin: 1556 i915_vma_unpin(vma); 1557 out_ww: 1558 if (err == -EDEADLK) { 1559 err = i915_gem_ww_ctx_backoff(&ww); 1560 if (!err) 1561 goto retry; 1562 } 1563 i915_gem_ww_ctx_fini(&ww); 1564 if (err) 1565 goto out_unmap; 1566 } 1567 1568 out_unmap: 1569 vm_munmap(addr, obj->base.size); 1570 return err; 1571 } 1572 1573 static int igt_mmap_gpu(void *arg) 1574 { 1575 struct drm_i915_private *i915 = arg; 1576 struct intel_memory_region *mr; 1577 enum intel_region_id id; 1578 1579 for_each_memory_region(mr, i915, id) { 1580 struct drm_i915_gem_object *obj; 1581 int err; 1582 1583 obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); 1584 if (obj == ERR_PTR(-ENODEV)) 1585 continue; 1586 1587 if (IS_ERR(obj)) 1588 return PTR_ERR(obj); 1589 1590 err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_GTT); 1591 if (err == 0) 1592 err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_WC); 1593 if (err == 0) 1594 err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_FIXED); 1595 1596 i915_gem_object_put(obj); 1597 if (err) 1598 return err; 1599 } 1600 1601 return 0; 1602 } 1603 1604 static int check_present_pte(pte_t *pte, unsigned long addr, void *data) 1605 { 1606 if (!pte_present(*pte) || pte_none(*pte)) { 1607 pr_err("missing PTE:%lx\n", 1608 (addr - (unsigned long)data) >> PAGE_SHIFT); 1609 return -EINVAL; 1610 } 1611 1612 return 0; 1613 } 1614 1615 static int check_absent_pte(pte_t *pte, unsigned long addr, void *data) 1616 { 1617 if (pte_present(*pte) && !pte_none(*pte)) { 1618 pr_err("present PTE:%lx; expected to be revoked\n", 1619 (addr - (unsigned long)data) >> PAGE_SHIFT); 1620 return -EINVAL; 1621 } 1622 1623 return 0; 1624 } 1625 1626 static int check_present(unsigned long addr, unsigned long len) 1627 { 1628 return apply_to_page_range(current->mm, addr, len, 1629 check_present_pte, (void *)addr); 1630 } 1631 1632 static int check_absent(unsigned long addr, unsigned long len) 1633 { 1634 return apply_to_page_range(current->mm, addr, len, 1635 check_absent_pte, (void *)addr); 1636 } 1637 1638 static int prefault_range(u64 start, u64 len) 1639 { 1640 const char __user *addr, *end; 1641 char __maybe_unused c; 1642 int err; 1643 1644 addr = u64_to_user_ptr(start); 1645 end = addr + len; 1646 1647 for (; addr < end; addr += PAGE_SIZE) { 1648 err = __get_user(c, addr); 1649 if (err) 1650 return err; 1651 } 1652 1653 return __get_user(c, end - 1); 1654 } 1655 1656 static int __igt_mmap_revoke(struct drm_i915_private *i915, 1657 struct drm_i915_gem_object *obj, 1658 enum i915_mmap_type type) 1659 { 1660 unsigned long addr; 1661 int err; 1662 u64 offset; 1663 1664 if (!can_mmap(obj, type)) 1665 return 0; 1666 1667 err = __assign_mmap_offset(obj, type, &offset, NULL); 1668 if (err) 1669 return err; 1670 1671 addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED); 1672 if (IS_ERR_VALUE(addr)) 1673 return addr; 1674 1675 err = prefault_range(addr, obj->base.size); 1676 if (err) 1677 goto out_unmap; 1678 1679 err = check_present(addr, obj->base.size); 1680 if (err) { 1681 pr_err("%s: was not present\n", obj->mm.region->name); 1682 goto out_unmap; 1683 } 1684 1685 /* 1686 * After unbinding the object from the GGTT, its address may be reused 1687 * for other objects. Ergo we have to revoke the previous mmap PTE 1688 * access as it no longer points to the same object. 1689 */ 1690 i915_gem_object_lock(obj, NULL); 1691 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 1692 i915_gem_object_unlock(obj); 1693 if (err) { 1694 pr_err("Failed to unbind object!\n"); 1695 goto out_unmap; 1696 } 1697 1698 if (type != I915_MMAP_TYPE_GTT) { 1699 i915_gem_object_lock(obj, NULL); 1700 __i915_gem_object_put_pages(obj); 1701 i915_gem_object_unlock(obj); 1702 if (i915_gem_object_has_pages(obj)) { 1703 pr_err("Failed to put-pages object!\n"); 1704 err = -EINVAL; 1705 goto out_unmap; 1706 } 1707 } 1708 1709 err = check_absent(addr, obj->base.size); 1710 if (err) { 1711 pr_err("%s: was not absent\n", obj->mm.region->name); 1712 goto out_unmap; 1713 } 1714 1715 out_unmap: 1716 vm_munmap(addr, obj->base.size); 1717 return err; 1718 } 1719 1720 static int igt_mmap_revoke(void *arg) 1721 { 1722 struct drm_i915_private *i915 = arg; 1723 struct intel_memory_region *mr; 1724 enum intel_region_id id; 1725 1726 for_each_memory_region(mr, i915, id) { 1727 struct drm_i915_gem_object *obj; 1728 int err; 1729 1730 obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1); 1731 if (obj == ERR_PTR(-ENODEV)) 1732 continue; 1733 1734 if (IS_ERR(obj)) 1735 return PTR_ERR(obj); 1736 1737 err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT); 1738 if (err == 0) 1739 err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC); 1740 if (err == 0) 1741 err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_FIXED); 1742 1743 i915_gem_object_put(obj); 1744 if (err) 1745 return err; 1746 } 1747 1748 return 0; 1749 } 1750 1751 int i915_gem_mman_live_selftests(struct drm_i915_private *i915) 1752 { 1753 static const struct i915_subtest tests[] = { 1754 SUBTEST(igt_partial_tiling), 1755 SUBTEST(igt_smoke_tiling), 1756 SUBTEST(igt_mmap_offset_exhaustion), 1757 SUBTEST(igt_mmap), 1758 SUBTEST(igt_mmap_migrate), 1759 SUBTEST(igt_mmap_access), 1760 SUBTEST(igt_mmap_revoke), 1761 SUBTEST(igt_mmap_gpu), 1762 }; 1763 1764 return i915_subtests(tests, i915); 1765 } 1766