1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include "i915_selftest.h" 7 8 #include "gt/intel_context.h" 9 #include "gt/intel_engine_user.h" 10 #include "gt/intel_gt.h" 11 #include "gt/intel_gpu_commands.h" 12 #include "gem/i915_gem_lmem.h" 13 14 #include "selftests/igt_flush_test.h" 15 #include "selftests/mock_drm.h" 16 #include "selftests/i915_random.h" 17 #include "huge_gem_object.h" 18 #include "mock_context.h" 19 20 enum client_tiling { 21 CLIENT_TILING_LINEAR, 22 CLIENT_TILING_X, 23 CLIENT_TILING_Y, 24 CLIENT_NUM_TILING_TYPES 25 }; 26 27 #define WIDTH 512 28 #define HEIGHT 32 29 30 struct blit_buffer { 31 struct i915_vma *vma; 32 u32 start_val; 33 enum client_tiling tiling; 34 }; 35 36 struct tiled_blits { 37 struct intel_context *ce; 38 struct blit_buffer buffers[3]; 39 struct blit_buffer scratch; 40 struct i915_vma *batch; 41 u64 hole; 42 u32 width; 43 u32 height; 44 }; 45 46 static int prepare_blit(const struct tiled_blits *t, 47 struct blit_buffer *dst, 48 struct blit_buffer *src, 49 struct drm_i915_gem_object *batch) 50 { 51 const int ver = GRAPHICS_VER(to_i915(batch->base.dev)); 52 bool use_64b_reloc = ver >= 8; 53 u32 src_pitch, dst_pitch; 54 u32 cmd, *cs; 55 56 cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC); 57 if (IS_ERR(cs)) 58 return PTR_ERR(cs); 59 60 *cs++ = MI_LOAD_REGISTER_IMM(1); 61 *cs++ = i915_mmio_reg_offset(BCS_SWCTRL); 62 cmd = (BCS_SRC_Y | BCS_DST_Y) << 16; 63 if (src->tiling == CLIENT_TILING_Y) 64 cmd |= BCS_SRC_Y; 65 if (dst->tiling == CLIENT_TILING_Y) 66 cmd |= BCS_DST_Y; 67 *cs++ = cmd; 68 69 cmd = MI_FLUSH_DW; 70 if (ver >= 8) 71 cmd++; 72 *cs++ = cmd; 73 *cs++ = 0; 74 *cs++ = 0; 75 *cs++ = 0; 76 77 cmd = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (8 - 2); 78 if (ver >= 8) 79 cmd += 2; 80 81 src_pitch = t->width * 4; 82 if (src->tiling) { 83 cmd |= XY_SRC_COPY_BLT_SRC_TILED; 84 src_pitch /= 4; 85 } 86 87 dst_pitch = t->width * 4; 88 if (dst->tiling) { 89 cmd |= XY_SRC_COPY_BLT_DST_TILED; 90 dst_pitch /= 4; 91 } 92 93 *cs++ = cmd; 94 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | dst_pitch; 95 *cs++ = 0; 96 *cs++ = t->height << 16 | t->width; 97 *cs++ = lower_32_bits(dst->vma->node.start); 98 if (use_64b_reloc) 99 *cs++ = upper_32_bits(dst->vma->node.start); 100 *cs++ = 0; 101 *cs++ = src_pitch; 102 *cs++ = lower_32_bits(src->vma->node.start); 103 if (use_64b_reloc) 104 *cs++ = upper_32_bits(src->vma->node.start); 105 106 *cs++ = MI_BATCH_BUFFER_END; 107 108 i915_gem_object_flush_map(batch); 109 i915_gem_object_unpin_map(batch); 110 111 return 0; 112 } 113 114 static void tiled_blits_destroy_buffers(struct tiled_blits *t) 115 { 116 int i; 117 118 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) 119 i915_vma_put(t->buffers[i].vma); 120 121 i915_vma_put(t->scratch.vma); 122 i915_vma_put(t->batch); 123 } 124 125 static struct i915_vma * 126 __create_vma(struct tiled_blits *t, size_t size, bool lmem) 127 { 128 struct drm_i915_private *i915 = t->ce->vm->i915; 129 struct drm_i915_gem_object *obj; 130 struct i915_vma *vma; 131 132 if (lmem) 133 obj = i915_gem_object_create_lmem(i915, size, 0); 134 else 135 obj = i915_gem_object_create_shmem(i915, size); 136 if (IS_ERR(obj)) 137 return ERR_CAST(obj); 138 139 vma = i915_vma_instance(obj, t->ce->vm, NULL); 140 if (IS_ERR(vma)) 141 i915_gem_object_put(obj); 142 143 return vma; 144 } 145 146 static struct i915_vma *create_vma(struct tiled_blits *t, bool lmem) 147 { 148 return __create_vma(t, PAGE_ALIGN(t->width * t->height * 4), lmem); 149 } 150 151 static int tiled_blits_create_buffers(struct tiled_blits *t, 152 int width, int height, 153 struct rnd_state *prng) 154 { 155 struct drm_i915_private *i915 = t->ce->engine->i915; 156 int i; 157 158 t->width = width; 159 t->height = height; 160 161 t->batch = __create_vma(t, PAGE_SIZE, false); 162 if (IS_ERR(t->batch)) 163 return PTR_ERR(t->batch); 164 165 t->scratch.vma = create_vma(t, false); 166 if (IS_ERR(t->scratch.vma)) { 167 i915_vma_put(t->batch); 168 return PTR_ERR(t->scratch.vma); 169 } 170 171 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) { 172 struct i915_vma *vma; 173 174 vma = create_vma(t, HAS_LMEM(i915) && i % 2); 175 if (IS_ERR(vma)) { 176 tiled_blits_destroy_buffers(t); 177 return PTR_ERR(vma); 178 } 179 180 t->buffers[i].vma = vma; 181 t->buffers[i].tiling = 182 i915_prandom_u32_max_state(CLIENT_TILING_Y + 1, prng); 183 } 184 185 return 0; 186 } 187 188 static void fill_scratch(struct tiled_blits *t, u32 *vaddr, u32 val) 189 { 190 int i; 191 192 t->scratch.start_val = val; 193 for (i = 0; i < t->width * t->height; i++) 194 vaddr[i] = val++; 195 196 i915_gem_object_flush_map(t->scratch.vma->obj); 197 } 198 199 static u64 swizzle_bit(unsigned int bit, u64 offset) 200 { 201 return (offset & BIT_ULL(bit)) >> (bit - 6); 202 } 203 204 static u64 tiled_offset(const struct intel_gt *gt, 205 u64 v, 206 unsigned int stride, 207 enum client_tiling tiling) 208 { 209 unsigned int swizzle; 210 u64 x, y; 211 212 if (tiling == CLIENT_TILING_LINEAR) 213 return v; 214 215 y = div64_u64_rem(v, stride, &x); 216 217 if (tiling == CLIENT_TILING_X) { 218 v = div64_u64_rem(y, 8, &y) * stride * 8; 219 v += y * 512; 220 v += div64_u64_rem(x, 512, &x) << 12; 221 v += x; 222 223 swizzle = gt->ggtt->bit_6_swizzle_x; 224 } else { 225 const unsigned int ytile_span = 16; 226 const unsigned int ytile_height = 512; 227 228 v = div64_u64_rem(y, 32, &y) * stride * 32; 229 v += y * ytile_span; 230 v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 231 v += x; 232 233 swizzle = gt->ggtt->bit_6_swizzle_y; 234 } 235 236 switch (swizzle) { 237 case I915_BIT_6_SWIZZLE_9: 238 v ^= swizzle_bit(9, v); 239 break; 240 case I915_BIT_6_SWIZZLE_9_10: 241 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v); 242 break; 243 case I915_BIT_6_SWIZZLE_9_11: 244 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v); 245 break; 246 case I915_BIT_6_SWIZZLE_9_10_11: 247 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v); 248 break; 249 } 250 251 return v; 252 } 253 254 static const char *repr_tiling(enum client_tiling tiling) 255 { 256 switch (tiling) { 257 case CLIENT_TILING_LINEAR: return "linear"; 258 case CLIENT_TILING_X: return "X"; 259 case CLIENT_TILING_Y: return "Y"; 260 default: return "unknown"; 261 } 262 } 263 264 static int verify_buffer(const struct tiled_blits *t, 265 struct blit_buffer *buf, 266 struct rnd_state *prng) 267 { 268 const u32 *vaddr; 269 int ret = 0; 270 int x, y, p; 271 272 x = i915_prandom_u32_max_state(t->width, prng); 273 y = i915_prandom_u32_max_state(t->height, prng); 274 p = y * t->width + x; 275 276 vaddr = i915_gem_object_pin_map_unlocked(buf->vma->obj, I915_MAP_WC); 277 if (IS_ERR(vaddr)) 278 return PTR_ERR(vaddr); 279 280 if (vaddr[0] != buf->start_val) { 281 ret = -EINVAL; 282 } else { 283 u64 v = tiled_offset(buf->vma->vm->gt, 284 p * 4, t->width * 4, 285 buf->tiling); 286 287 if (vaddr[v / sizeof(*vaddr)] != buf->start_val + p) 288 ret = -EINVAL; 289 } 290 if (ret) { 291 pr_err("Invalid %s tiling detected at (%d, %d), start_val %x\n", 292 repr_tiling(buf->tiling), 293 x, y, buf->start_val); 294 igt_hexdump(vaddr, 4096); 295 } 296 297 i915_gem_object_unpin_map(buf->vma->obj); 298 return ret; 299 } 300 301 static int move_to_active(struct i915_vma *vma, 302 struct i915_request *rq, 303 unsigned int flags) 304 { 305 int err; 306 307 i915_vma_lock(vma); 308 err = i915_request_await_object(rq, vma->obj, false); 309 if (err == 0) 310 err = i915_vma_move_to_active(vma, rq, flags); 311 i915_vma_unlock(vma); 312 313 return err; 314 } 315 316 static int pin_buffer(struct i915_vma *vma, u64 addr) 317 { 318 int err; 319 320 if (drm_mm_node_allocated(&vma->node) && vma->node.start != addr) { 321 err = i915_vma_unbind(vma); 322 if (err) 323 return err; 324 } 325 326 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED | addr); 327 if (err) 328 return err; 329 330 return 0; 331 } 332 333 static int 334 tiled_blit(struct tiled_blits *t, 335 struct blit_buffer *dst, u64 dst_addr, 336 struct blit_buffer *src, u64 src_addr) 337 { 338 struct i915_request *rq; 339 int err; 340 341 err = pin_buffer(src->vma, src_addr); 342 if (err) { 343 pr_err("Cannot pin src @ %llx\n", src_addr); 344 return err; 345 } 346 347 err = pin_buffer(dst->vma, dst_addr); 348 if (err) { 349 pr_err("Cannot pin dst @ %llx\n", dst_addr); 350 goto err_src; 351 } 352 353 err = i915_vma_pin(t->batch, 0, 0, PIN_USER | PIN_HIGH); 354 if (err) { 355 pr_err("cannot pin batch\n"); 356 goto err_dst; 357 } 358 359 err = prepare_blit(t, dst, src, t->batch->obj); 360 if (err) 361 goto err_bb; 362 363 rq = intel_context_create_request(t->ce); 364 if (IS_ERR(rq)) { 365 err = PTR_ERR(rq); 366 goto err_bb; 367 } 368 369 err = move_to_active(t->batch, rq, 0); 370 if (!err) 371 err = move_to_active(src->vma, rq, 0); 372 if (!err) 373 err = move_to_active(dst->vma, rq, 0); 374 if (!err) 375 err = rq->engine->emit_bb_start(rq, 376 t->batch->node.start, 377 t->batch->node.size, 378 0); 379 i915_request_get(rq); 380 i915_request_add(rq); 381 if (i915_request_wait(rq, 0, HZ / 2) < 0) 382 err = -ETIME; 383 i915_request_put(rq); 384 385 dst->start_val = src->start_val; 386 err_bb: 387 i915_vma_unpin(t->batch); 388 err_dst: 389 i915_vma_unpin(dst->vma); 390 err_src: 391 i915_vma_unpin(src->vma); 392 return err; 393 } 394 395 static struct tiled_blits * 396 tiled_blits_create(struct intel_engine_cs *engine, struct rnd_state *prng) 397 { 398 struct drm_mm_node hole; 399 struct tiled_blits *t; 400 u64 hole_size; 401 int err; 402 403 t = kzalloc(sizeof(*t), GFP_KERNEL); 404 if (!t) 405 return ERR_PTR(-ENOMEM); 406 407 t->ce = intel_context_create(engine); 408 if (IS_ERR(t->ce)) { 409 err = PTR_ERR(t->ce); 410 goto err_free; 411 } 412 413 hole_size = 2 * PAGE_ALIGN(WIDTH * HEIGHT * 4); 414 hole_size *= 2; /* room to maneuver */ 415 hole_size += 2 * I915_GTT_MIN_ALIGNMENT; 416 417 mutex_lock(&t->ce->vm->mutex); 418 memset(&hole, 0, sizeof(hole)); 419 err = drm_mm_insert_node_in_range(&t->ce->vm->mm, &hole, 420 hole_size, 0, I915_COLOR_UNEVICTABLE, 421 0, U64_MAX, 422 DRM_MM_INSERT_BEST); 423 if (!err) 424 drm_mm_remove_node(&hole); 425 mutex_unlock(&t->ce->vm->mutex); 426 if (err) { 427 err = -ENODEV; 428 goto err_put; 429 } 430 431 t->hole = hole.start + I915_GTT_MIN_ALIGNMENT; 432 pr_info("Using hole at %llx\n", t->hole); 433 434 err = tiled_blits_create_buffers(t, WIDTH, HEIGHT, prng); 435 if (err) 436 goto err_put; 437 438 return t; 439 440 err_put: 441 intel_context_put(t->ce); 442 err_free: 443 kfree(t); 444 return ERR_PTR(err); 445 } 446 447 static void tiled_blits_destroy(struct tiled_blits *t) 448 { 449 tiled_blits_destroy_buffers(t); 450 451 intel_context_put(t->ce); 452 kfree(t); 453 } 454 455 static int tiled_blits_prepare(struct tiled_blits *t, 456 struct rnd_state *prng) 457 { 458 u64 offset = PAGE_ALIGN(t->width * t->height * 4); 459 u32 *map; 460 int err; 461 int i; 462 463 map = i915_gem_object_pin_map_unlocked(t->scratch.vma->obj, I915_MAP_WC); 464 if (IS_ERR(map)) 465 return PTR_ERR(map); 466 467 /* Use scratch to fill objects */ 468 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) { 469 fill_scratch(t, map, prandom_u32_state(prng)); 470 GEM_BUG_ON(verify_buffer(t, &t->scratch, prng)); 471 472 err = tiled_blit(t, 473 &t->buffers[i], t->hole + offset, 474 &t->scratch, t->hole); 475 if (err == 0) 476 err = verify_buffer(t, &t->buffers[i], prng); 477 if (err) { 478 pr_err("Failed to create buffer %d\n", i); 479 break; 480 } 481 } 482 483 i915_gem_object_unpin_map(t->scratch.vma->obj); 484 return err; 485 } 486 487 static int tiled_blits_bounce(struct tiled_blits *t, struct rnd_state *prng) 488 { 489 u64 offset = 490 round_up(t->width * t->height * 4, 2 * I915_GTT_MIN_ALIGNMENT); 491 int err; 492 493 /* We want to check position invariant tiling across GTT eviction */ 494 495 err = tiled_blit(t, 496 &t->buffers[1], t->hole + offset / 2, 497 &t->buffers[0], t->hole + 2 * offset); 498 if (err) 499 return err; 500 501 /* Reposition so that we overlap the old addresses, and slightly off */ 502 err = tiled_blit(t, 503 &t->buffers[2], t->hole + I915_GTT_MIN_ALIGNMENT, 504 &t->buffers[1], t->hole + 3 * offset / 2); 505 if (err) 506 return err; 507 508 err = verify_buffer(t, &t->buffers[2], prng); 509 if (err) 510 return err; 511 512 return 0; 513 } 514 515 static int __igt_client_tiled_blits(struct intel_engine_cs *engine, 516 struct rnd_state *prng) 517 { 518 struct tiled_blits *t; 519 int err; 520 521 t = tiled_blits_create(engine, prng); 522 if (IS_ERR(t)) 523 return PTR_ERR(t); 524 525 err = tiled_blits_prepare(t, prng); 526 if (err) 527 goto out; 528 529 err = tiled_blits_bounce(t, prng); 530 if (err) 531 goto out; 532 533 out: 534 tiled_blits_destroy(t); 535 return err; 536 } 537 538 static bool has_bit17_swizzle(int sw) 539 { 540 return (sw == I915_BIT_6_SWIZZLE_9_10_17 || 541 sw == I915_BIT_6_SWIZZLE_9_17); 542 } 543 544 static bool bad_swizzling(struct drm_i915_private *i915) 545 { 546 struct i915_ggtt *ggtt = &i915->ggtt; 547 548 if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) 549 return true; 550 551 if (has_bit17_swizzle(ggtt->bit_6_swizzle_x) || 552 has_bit17_swizzle(ggtt->bit_6_swizzle_y)) 553 return true; 554 555 return false; 556 } 557 558 static int igt_client_tiled_blits(void *arg) 559 { 560 struct drm_i915_private *i915 = arg; 561 I915_RND_STATE(prng); 562 int inst = 0; 563 564 /* Test requires explicit BLT tiling controls */ 565 if (GRAPHICS_VER(i915) < 4) 566 return 0; 567 568 if (bad_swizzling(i915)) /* Requires sane (sub-page) swizzling */ 569 return 0; 570 571 do { 572 struct intel_engine_cs *engine; 573 int err; 574 575 engine = intel_engine_lookup_user(i915, 576 I915_ENGINE_CLASS_COPY, 577 inst++); 578 if (!engine) 579 return 0; 580 581 err = __igt_client_tiled_blits(engine, &prng); 582 if (err == -ENODEV) 583 err = 0; 584 if (err) 585 return err; 586 } while (1); 587 } 588 589 int i915_gem_client_blt_live_selftests(struct drm_i915_private *i915) 590 { 591 static const struct i915_subtest tests[] = { 592 SUBTEST(igt_client_tiled_blits), 593 }; 594 595 if (intel_gt_is_wedged(to_gt(i915))) 596 return 0; 597 598 return i915_live_subtests(tests, i915); 599 } 600