1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "intel_context.h" 8 #include "intel_gpu_commands.h" 9 #include "intel_gt.h" 10 #include "intel_gtt.h" 11 #include "intel_migrate.h" 12 #include "intel_ring.h" 13 #include "gem/i915_gem_lmem.h" 14 15 struct insert_pte_data { 16 u64 offset; 17 }; 18 19 #define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */ 20 21 #define GET_CCS_BYTES(i915, size) (HAS_FLAT_CCS(i915) ? \ 22 DIV_ROUND_UP(size, NUM_BYTES_PER_CCS_BYTE) : 0) 23 static bool engine_supports_migration(struct intel_engine_cs *engine) 24 { 25 if (!engine) 26 return false; 27 28 /* 29 * We need the ability to prevent aribtration (MI_ARB_ON_OFF), 30 * the ability to write PTE using inline data (MI_STORE_DATA) 31 * and of course the ability to do the block transfer (blits). 32 */ 33 GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS); 34 35 return true; 36 } 37 38 static void xehpsdv_toggle_pdes(struct i915_address_space *vm, 39 struct i915_page_table *pt, 40 void *data) 41 { 42 struct insert_pte_data *d = data; 43 44 /* 45 * Insert a dummy PTE into every PT that will map to LMEM to ensure 46 * we have a correctly setup PDE structure for later use. 47 */ 48 vm->insert_page(vm, 0, d->offset, I915_CACHE_NONE, PTE_LM); 49 GEM_BUG_ON(!pt->is_compact); 50 d->offset += SZ_2M; 51 } 52 53 static void xehpsdv_insert_pte(struct i915_address_space *vm, 54 struct i915_page_table *pt, 55 void *data) 56 { 57 struct insert_pte_data *d = data; 58 59 /* 60 * We are playing tricks here, since the actual pt, from the hw 61 * pov, is only 256bytes with 32 entries, or 4096bytes with 512 62 * entries, but we are still guaranteed that the physical 63 * alignment is 64K underneath for the pt, and we are careful 64 * not to access the space in the void. 65 */ 66 vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, PTE_LM); 67 d->offset += SZ_64K; 68 } 69 70 static void insert_pte(struct i915_address_space *vm, 71 struct i915_page_table *pt, 72 void *data) 73 { 74 struct insert_pte_data *d = data; 75 76 vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, 77 i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0); 78 d->offset += PAGE_SIZE; 79 } 80 81 static struct i915_address_space *migrate_vm(struct intel_gt *gt) 82 { 83 struct i915_vm_pt_stash stash = {}; 84 struct i915_ppgtt *vm; 85 int err; 86 int i; 87 88 /* 89 * We construct a very special VM for use by all migration contexts, 90 * it is kept pinned so that it can be used at any time. As we need 91 * to pre-allocate the page directories for the migration VM, this 92 * limits us to only using a small number of prepared vma. 93 * 94 * To be able to pipeline and reschedule migration operations while 95 * avoiding unnecessary contention on the vm itself, the PTE updates 96 * are inline with the blits. All the blits use the same fixed 97 * addresses, with the backing store redirection being updated on the 98 * fly. Only 2 implicit vma are used for all migration operations. 99 * 100 * We lay the ppGTT out as: 101 * 102 * [0, CHUNK_SZ) -> first object 103 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second object 104 * [2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE 105 * 106 * By exposing the dma addresses of the page directories themselves 107 * within the ppGTT, we are then able to rewrite the PTE prior to use. 108 * But the PTE update and subsequent migration operation must be atomic, 109 * i.e. within the same non-preemptible window so that we do not switch 110 * to another migration context that overwrites the PTE. 111 * 112 * This changes quite a bit on platforms with HAS_64K_PAGES support, 113 * where we instead have three windows, each CHUNK_SIZE in size. The 114 * first is reserved for mapping system-memory, and that just uses the 115 * 512 entry layout using 4K GTT pages. The other two windows just map 116 * lmem pages and must use the new compact 32 entry layout using 64K GTT 117 * pages, which ensures we can address any lmem object that the user 118 * throws at us. We then also use the xehpsdv_toggle_pdes as a way of 119 * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the 120 * compact layout for each of these page-tables, that fall within the 121 * [CHUNK_SIZE, 3 * CHUNK_SIZE) range. 122 * 123 * We lay the ppGTT out as: 124 * 125 * [0, CHUNK_SZ) -> first window/object, maps smem 126 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src 127 * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst 128 * 129 * For the PTE window it's also quite different, since each PTE must 130 * point to some 64K page, one for each PT(since it's in lmem), and yet 131 * each is only <= 4096bytes, but since the unused space within that PTE 132 * range is never touched, this should be fine. 133 * 134 * So basically each PT now needs 64K of virtual memory, instead of 4K, 135 * which looks like: 136 * 137 * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE 138 */ 139 140 vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY); 141 if (IS_ERR(vm)) 142 return ERR_CAST(vm); 143 144 if (!vm->vm.allocate_va_range || !vm->vm.foreach) { 145 err = -ENODEV; 146 goto err_vm; 147 } 148 149 if (HAS_64K_PAGES(gt->i915)) 150 stash.pt_sz = I915_GTT_PAGE_SIZE_64K; 151 152 /* 153 * Each engine instance is assigned its own chunk in the VM, so 154 * that we can run multiple instances concurrently 155 */ 156 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { 157 struct intel_engine_cs *engine; 158 u64 base = (u64)i << 32; 159 struct insert_pte_data d = {}; 160 struct i915_gem_ww_ctx ww; 161 u64 sz; 162 163 engine = gt->engine_class[COPY_ENGINE_CLASS][i]; 164 if (!engine_supports_migration(engine)) 165 continue; 166 167 /* 168 * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need 169 * 4x2 page directories for source/destination. 170 */ 171 if (HAS_64K_PAGES(gt->i915)) 172 sz = 3 * CHUNK_SZ; 173 else 174 sz = 2 * CHUNK_SZ; 175 d.offset = base + sz; 176 177 /* 178 * We need another page directory setup so that we can write 179 * the 8x512 PTE in each chunk. 180 */ 181 if (HAS_64K_PAGES(gt->i915)) 182 sz += (sz / SZ_2M) * SZ_64K; 183 else 184 sz += (sz >> 12) * sizeof(u64); 185 186 err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz); 187 if (err) 188 goto err_vm; 189 190 for_i915_gem_ww(&ww, err, true) { 191 err = i915_vm_lock_objects(&vm->vm, &ww); 192 if (err) 193 continue; 194 err = i915_vm_map_pt_stash(&vm->vm, &stash); 195 if (err) 196 continue; 197 198 vm->vm.allocate_va_range(&vm->vm, &stash, base, sz); 199 } 200 i915_vm_free_pt_stash(&vm->vm, &stash); 201 if (err) 202 goto err_vm; 203 204 /* Now allow the GPU to rewrite the PTE via its own ppGTT */ 205 if (HAS_64K_PAGES(gt->i915)) { 206 vm->vm.foreach(&vm->vm, base, d.offset - base, 207 xehpsdv_insert_pte, &d); 208 d.offset = base + CHUNK_SZ; 209 vm->vm.foreach(&vm->vm, 210 d.offset, 211 2 * CHUNK_SZ, 212 xehpsdv_toggle_pdes, &d); 213 } else { 214 vm->vm.foreach(&vm->vm, base, d.offset - base, 215 insert_pte, &d); 216 } 217 } 218 219 return &vm->vm; 220 221 err_vm: 222 i915_vm_put(&vm->vm); 223 return ERR_PTR(err); 224 } 225 226 static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt) 227 { 228 struct intel_engine_cs *engine; 229 int i; 230 231 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { 232 engine = gt->engine_class[COPY_ENGINE_CLASS][i]; 233 if (engine_supports_migration(engine)) 234 return engine; 235 } 236 237 return NULL; 238 } 239 240 static struct intel_context *pinned_context(struct intel_gt *gt) 241 { 242 static struct lock_class_key key; 243 struct intel_engine_cs *engine; 244 struct i915_address_space *vm; 245 struct intel_context *ce; 246 247 engine = first_copy_engine(gt); 248 if (!engine) 249 return ERR_PTR(-ENODEV); 250 251 vm = migrate_vm(gt); 252 if (IS_ERR(vm)) 253 return ERR_CAST(vm); 254 255 ce = intel_engine_create_pinned_context(engine, vm, SZ_512K, 256 I915_GEM_HWS_MIGRATE, 257 &key, "migrate"); 258 i915_vm_put(vm); 259 return ce; 260 } 261 262 int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt) 263 { 264 struct intel_context *ce; 265 266 memset(m, 0, sizeof(*m)); 267 268 ce = pinned_context(gt); 269 if (IS_ERR(ce)) 270 return PTR_ERR(ce); 271 272 m->context = ce; 273 return 0; 274 } 275 276 static int random_index(unsigned int max) 277 { 278 return upper_32_bits(mul_u32_u32(get_random_u32(), max)); 279 } 280 281 static struct intel_context *__migrate_engines(struct intel_gt *gt) 282 { 283 struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE]; 284 struct intel_engine_cs *engine; 285 unsigned int count, i; 286 287 count = 0; 288 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { 289 engine = gt->engine_class[COPY_ENGINE_CLASS][i]; 290 if (engine_supports_migration(engine)) 291 engines[count++] = engine; 292 } 293 294 return intel_context_create(engines[random_index(count)]); 295 } 296 297 struct intel_context *intel_migrate_create_context(struct intel_migrate *m) 298 { 299 struct intel_context *ce; 300 301 /* 302 * We randomly distribute contexts across the engines upon constrction, 303 * as they all share the same pinned vm, and so in order to allow 304 * multiple blits to run in parallel, we must construct each blit 305 * to use a different range of the vm for its GTT. This has to be 306 * known at construction, so we can not use the late greedy load 307 * balancing of the virtual-engine. 308 */ 309 ce = __migrate_engines(m->context->engine->gt); 310 if (IS_ERR(ce)) 311 return ce; 312 313 ce->ring = NULL; 314 ce->ring_size = SZ_256K; 315 316 i915_vm_put(ce->vm); 317 ce->vm = i915_vm_get(m->context->vm); 318 319 return ce; 320 } 321 322 static inline struct sgt_dma sg_sgt(struct scatterlist *sg) 323 { 324 dma_addr_t addr = sg_dma_address(sg); 325 326 return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) }; 327 } 328 329 static int emit_no_arbitration(struct i915_request *rq) 330 { 331 u32 *cs; 332 333 cs = intel_ring_begin(rq, 2); 334 if (IS_ERR(cs)) 335 return PTR_ERR(cs); 336 337 /* Explicitly disable preemption for this request. */ 338 *cs++ = MI_ARB_ON_OFF; 339 *cs++ = MI_NOOP; 340 intel_ring_advance(rq, cs); 341 342 return 0; 343 } 344 345 static int max_pte_pkt_size(struct i915_request *rq, int pkt) 346 { 347 struct intel_ring *ring = rq->ring; 348 349 pkt = min_t(int, pkt, (ring->space - rq->reserved_space) / sizeof(u32) + 5); 350 pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5); 351 352 return pkt; 353 } 354 355 #define I915_EMIT_PTE_NUM_DWORDS 6 356 357 static int emit_pte(struct i915_request *rq, 358 struct sgt_dma *it, 359 enum i915_cache_level cache_level, 360 bool is_lmem, 361 u64 offset, 362 int length) 363 { 364 bool has_64K_pages = HAS_64K_PAGES(rq->engine->i915); 365 const u64 encode = rq->context->vm->pte_encode(0, cache_level, 366 is_lmem ? PTE_LM : 0); 367 struct intel_ring *ring = rq->ring; 368 int pkt, dword_length; 369 u32 total = 0; 370 u32 page_size; 371 u32 *hdr, *cs; 372 373 GEM_BUG_ON(GRAPHICS_VER(rq->engine->i915) < 8); 374 375 page_size = I915_GTT_PAGE_SIZE; 376 dword_length = 0x400; 377 378 /* Compute the page directory offset for the target address range */ 379 if (has_64K_pages) { 380 GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M)); 381 382 offset /= SZ_2M; 383 offset *= SZ_64K; 384 offset += 3 * CHUNK_SZ; 385 386 if (is_lmem) { 387 page_size = I915_GTT_PAGE_SIZE_64K; 388 dword_length = 0x40; 389 } 390 } else { 391 offset >>= 12; 392 offset *= sizeof(u64); 393 offset += 2 * CHUNK_SZ; 394 } 395 396 offset += (u64)rq->engine->instance << 32; 397 398 cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS); 399 if (IS_ERR(cs)) 400 return PTR_ERR(cs); 401 402 /* Pack as many PTE updates as possible into a single MI command */ 403 pkt = max_pte_pkt_size(rq, dword_length); 404 405 hdr = cs; 406 *cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */ 407 *cs++ = lower_32_bits(offset); 408 *cs++ = upper_32_bits(offset); 409 410 do { 411 if (cs - hdr >= pkt) { 412 int dword_rem; 413 414 *hdr += cs - hdr - 2; 415 *cs++ = MI_NOOP; 416 417 ring->emit = (void *)cs - ring->vaddr; 418 intel_ring_advance(rq, cs); 419 intel_ring_update_space(ring); 420 421 cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS); 422 if (IS_ERR(cs)) 423 return PTR_ERR(cs); 424 425 dword_rem = dword_length; 426 if (has_64K_pages) { 427 if (IS_ALIGNED(total, SZ_2M)) { 428 offset = round_up(offset, SZ_64K); 429 } else { 430 dword_rem = SZ_2M - (total & (SZ_2M - 1)); 431 dword_rem /= page_size; 432 dword_rem *= 2; 433 } 434 } 435 436 pkt = max_pte_pkt_size(rq, dword_rem); 437 438 hdr = cs; 439 *cs++ = MI_STORE_DATA_IMM | REG_BIT(21); 440 *cs++ = lower_32_bits(offset); 441 *cs++ = upper_32_bits(offset); 442 } 443 444 GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size)); 445 446 *cs++ = lower_32_bits(encode | it->dma); 447 *cs++ = upper_32_bits(encode | it->dma); 448 449 offset += 8; 450 total += page_size; 451 452 it->dma += page_size; 453 if (it->dma >= it->max) { 454 it->sg = __sg_next(it->sg); 455 if (!it->sg || sg_dma_len(it->sg) == 0) 456 break; 457 458 it->dma = sg_dma_address(it->sg); 459 it->max = it->dma + sg_dma_len(it->sg); 460 } 461 } while (total < length); 462 463 *hdr += cs - hdr - 2; 464 *cs++ = MI_NOOP; 465 466 ring->emit = (void *)cs - ring->vaddr; 467 intel_ring_advance(rq, cs); 468 intel_ring_update_space(ring); 469 470 return total; 471 } 472 473 static bool wa_1209644611_applies(int ver, u32 size) 474 { 475 u32 height = size >> PAGE_SHIFT; 476 477 if (ver != 11) 478 return false; 479 480 return height % 4 == 3 && height <= 8; 481 } 482 483 /** 484 * DOC: Flat-CCS - Memory compression for Local memory 485 * 486 * On Xe-HP and later devices, we use dedicated compression control state (CCS) 487 * stored in local memory for each surface, to support the 3D and media 488 * compression formats. 489 * 490 * The memory required for the CCS of the entire local memory is 1/256 of the 491 * local memory size. So before the kernel boot, the required memory is reserved 492 * for the CCS data and a secure register will be programmed with the CCS base 493 * address. 494 * 495 * Flat CCS data needs to be cleared when a lmem object is allocated. 496 * And CCS data can be copied in and out of CCS region through 497 * XY_CTRL_SURF_COPY_BLT. CPU can't access the CCS data directly. 498 * 499 * I915 supports Flat-CCS on lmem only objects. When an objects has smem in 500 * its preference list, on memory pressure, i915 needs to migrate the lmem 501 * content into smem. If the lmem object is Flat-CCS compressed by userspace, 502 * then i915 needs to decompress it. But I915 lack the required information 503 * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects. 504 * 505 * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can 506 * be temporarily evicted to smem, along with the auxiliary CCS state, where 507 * it can be potentially swapped-out at a later point, if required. 508 * If userspace later touches the evicted pages, then we always move 509 * the backing memory back to lmem, which includes restoring the saved CCS state, 510 * and potentially performing any required swap-in. 511 * 512 * For the migration of the lmem objects with smem in placement list, such as 513 * {lmem, smem}, objects are treated as non Flat-CCS capable objects. 514 */ 515 516 static inline u32 *i915_flush_dw(u32 *cmd, u32 flags) 517 { 518 *cmd++ = MI_FLUSH_DW | flags; 519 *cmd++ = 0; 520 *cmd++ = 0; 521 522 return cmd; 523 } 524 525 static int emit_copy_ccs(struct i915_request *rq, 526 u32 dst_offset, u8 dst_access, 527 u32 src_offset, u8 src_access, int size) 528 { 529 struct drm_i915_private *i915 = rq->engine->i915; 530 int mocs = rq->engine->gt->mocs.uc_index << 1; 531 u32 num_ccs_blks; 532 u32 *cs; 533 534 cs = intel_ring_begin(rq, 12); 535 if (IS_ERR(cs)) 536 return PTR_ERR(cs); 537 538 num_ccs_blks = DIV_ROUND_UP(GET_CCS_BYTES(i915, size), 539 NUM_CCS_BYTES_PER_BLOCK); 540 GEM_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER); 541 cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS); 542 543 /* 544 * The XY_CTRL_SURF_COPY_BLT instruction is used to copy the CCS 545 * data in and out of the CCS region. 546 * 547 * We can copy at most 1024 blocks of 256 bytes using one 548 * XY_CTRL_SURF_COPY_BLT instruction. 549 * 550 * In case we need to copy more than 1024 blocks, we need to add 551 * another instruction to the same batch buffer. 552 * 553 * 1024 blocks of 256 bytes of CCS represent a total 256KB of CCS. 554 * 555 * 256 KB of CCS represents 256 * 256 KB = 64 MB of LMEM. 556 */ 557 *cs++ = XY_CTRL_SURF_COPY_BLT | 558 src_access << SRC_ACCESS_TYPE_SHIFT | 559 dst_access << DST_ACCESS_TYPE_SHIFT | 560 ((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT; 561 *cs++ = src_offset; 562 *cs++ = rq->engine->instance | 563 FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs); 564 *cs++ = dst_offset; 565 *cs++ = rq->engine->instance | 566 FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs); 567 568 cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS); 569 *cs++ = MI_NOOP; 570 571 intel_ring_advance(rq, cs); 572 573 return 0; 574 } 575 576 static int emit_copy(struct i915_request *rq, 577 u32 dst_offset, u32 src_offset, int size) 578 { 579 const int ver = GRAPHICS_VER(rq->engine->i915); 580 u32 instance = rq->engine->instance; 581 u32 *cs; 582 583 cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6); 584 if (IS_ERR(cs)) 585 return PTR_ERR(cs); 586 587 if (ver >= 9 && !wa_1209644611_applies(ver, size)) { 588 *cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2); 589 *cs++ = BLT_DEPTH_32 | PAGE_SIZE; 590 *cs++ = 0; 591 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 592 *cs++ = dst_offset; 593 *cs++ = instance; 594 *cs++ = 0; 595 *cs++ = PAGE_SIZE; 596 *cs++ = src_offset; 597 *cs++ = instance; 598 } else if (ver >= 8) { 599 *cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2); 600 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; 601 *cs++ = 0; 602 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 603 *cs++ = dst_offset; 604 *cs++ = instance; 605 *cs++ = 0; 606 *cs++ = PAGE_SIZE; 607 *cs++ = src_offset; 608 *cs++ = instance; 609 } else { 610 GEM_BUG_ON(instance); 611 *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); 612 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; 613 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE; 614 *cs++ = dst_offset; 615 *cs++ = PAGE_SIZE; 616 *cs++ = src_offset; 617 } 618 619 intel_ring_advance(rq, cs); 620 return 0; 621 } 622 623 static u64 scatter_list_length(struct scatterlist *sg) 624 { 625 u64 len = 0; 626 627 while (sg && sg_dma_len(sg)) { 628 len += sg_dma_len(sg); 629 sg = sg_next(sg); 630 } 631 632 return len; 633 } 634 635 static int 636 calculate_chunk_sz(struct drm_i915_private *i915, bool src_is_lmem, 637 u64 bytes_to_cpy, u64 ccs_bytes_to_cpy) 638 { 639 if (ccs_bytes_to_cpy && !src_is_lmem) 640 /* 641 * When CHUNK_SZ is passed all the pages upto CHUNK_SZ 642 * will be taken for the blt. in Flat-ccs supported 643 * platform Smem obj will have more pages than required 644 * for main meory hence limit it to the required size 645 * for main memory 646 */ 647 return min_t(u64, bytes_to_cpy, CHUNK_SZ); 648 else 649 return CHUNK_SZ; 650 } 651 652 static void get_ccs_sg_sgt(struct sgt_dma *it, u64 bytes_to_cpy) 653 { 654 u64 len; 655 656 do { 657 GEM_BUG_ON(!it->sg || !sg_dma_len(it->sg)); 658 len = it->max - it->dma; 659 if (len > bytes_to_cpy) { 660 it->dma += bytes_to_cpy; 661 break; 662 } 663 664 bytes_to_cpy -= len; 665 666 it->sg = __sg_next(it->sg); 667 it->dma = sg_dma_address(it->sg); 668 it->max = it->dma + sg_dma_len(it->sg); 669 } while (bytes_to_cpy); 670 } 671 672 int 673 intel_context_migrate_copy(struct intel_context *ce, 674 const struct i915_deps *deps, 675 struct scatterlist *src, 676 enum i915_cache_level src_cache_level, 677 bool src_is_lmem, 678 struct scatterlist *dst, 679 enum i915_cache_level dst_cache_level, 680 bool dst_is_lmem, 681 struct i915_request **out) 682 { 683 struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst), it_ccs; 684 struct drm_i915_private *i915 = ce->engine->i915; 685 u64 ccs_bytes_to_cpy = 0, bytes_to_cpy; 686 enum i915_cache_level ccs_cache_level; 687 u32 src_offset, dst_offset; 688 u8 src_access, dst_access; 689 struct i915_request *rq; 690 u64 src_sz, dst_sz; 691 bool ccs_is_src, overwrite_ccs; 692 int err; 693 694 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm); 695 GEM_BUG_ON(IS_DGFX(ce->engine->i915) && (!src_is_lmem && !dst_is_lmem)); 696 *out = NULL; 697 698 GEM_BUG_ON(ce->ring->size < SZ_64K); 699 700 src_sz = scatter_list_length(src); 701 bytes_to_cpy = src_sz; 702 703 if (HAS_FLAT_CCS(i915) && src_is_lmem ^ dst_is_lmem) { 704 src_access = !src_is_lmem && dst_is_lmem; 705 dst_access = !src_access; 706 707 dst_sz = scatter_list_length(dst); 708 if (src_is_lmem) { 709 it_ccs = it_dst; 710 ccs_cache_level = dst_cache_level; 711 ccs_is_src = false; 712 } else if (dst_is_lmem) { 713 bytes_to_cpy = dst_sz; 714 it_ccs = it_src; 715 ccs_cache_level = src_cache_level; 716 ccs_is_src = true; 717 } 718 719 /* 720 * When there is a eviction of ccs needed smem will have the 721 * extra pages for the ccs data 722 * 723 * TO-DO: Want to move the size mismatch check to a WARN_ON, 724 * but still we have some requests of smem->lmem with same size. 725 * Need to fix it. 726 */ 727 ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0; 728 if (ccs_bytes_to_cpy) 729 get_ccs_sg_sgt(&it_ccs, bytes_to_cpy); 730 } 731 732 overwrite_ccs = HAS_FLAT_CCS(i915) && !ccs_bytes_to_cpy && dst_is_lmem; 733 734 src_offset = 0; 735 dst_offset = CHUNK_SZ; 736 if (HAS_64K_PAGES(ce->engine->i915)) { 737 src_offset = 0; 738 dst_offset = 0; 739 if (src_is_lmem) 740 src_offset = CHUNK_SZ; 741 if (dst_is_lmem) 742 dst_offset = 2 * CHUNK_SZ; 743 } 744 745 do { 746 int len; 747 748 rq = i915_request_create(ce); 749 if (IS_ERR(rq)) { 750 err = PTR_ERR(rq); 751 goto out_ce; 752 } 753 754 if (deps) { 755 err = i915_request_await_deps(rq, deps); 756 if (err) 757 goto out_rq; 758 759 if (rq->engine->emit_init_breadcrumb) { 760 err = rq->engine->emit_init_breadcrumb(rq); 761 if (err) 762 goto out_rq; 763 } 764 765 deps = NULL; 766 } 767 768 /* The PTE updates + copy must not be interrupted. */ 769 err = emit_no_arbitration(rq); 770 if (err) 771 goto out_rq; 772 773 src_sz = calculate_chunk_sz(i915, src_is_lmem, 774 bytes_to_cpy, ccs_bytes_to_cpy); 775 776 len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem, 777 src_offset, src_sz); 778 if (!len) { 779 err = -EINVAL; 780 goto out_rq; 781 } 782 if (len < 0) { 783 err = len; 784 goto out_rq; 785 } 786 787 err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem, 788 dst_offset, len); 789 if (err < 0) 790 goto out_rq; 791 if (err < len) { 792 err = -EINVAL; 793 goto out_rq; 794 } 795 796 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 797 if (err) 798 goto out_rq; 799 800 err = emit_copy(rq, dst_offset, src_offset, len); 801 if (err) 802 goto out_rq; 803 804 bytes_to_cpy -= len; 805 806 if (ccs_bytes_to_cpy) { 807 int ccs_sz; 808 809 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 810 if (err) 811 goto out_rq; 812 813 ccs_sz = GET_CCS_BYTES(i915, len); 814 err = emit_pte(rq, &it_ccs, ccs_cache_level, false, 815 ccs_is_src ? src_offset : dst_offset, 816 ccs_sz); 817 if (err < 0) 818 goto out_rq; 819 if (err < ccs_sz) { 820 err = -EINVAL; 821 goto out_rq; 822 } 823 824 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 825 if (err) 826 goto out_rq; 827 828 err = emit_copy_ccs(rq, dst_offset, dst_access, 829 src_offset, src_access, len); 830 if (err) 831 goto out_rq; 832 833 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 834 if (err) 835 goto out_rq; 836 ccs_bytes_to_cpy -= ccs_sz; 837 } else if (overwrite_ccs) { 838 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 839 if (err) 840 goto out_rq; 841 842 if (src_is_lmem) { 843 /* 844 * If the src is already in lmem, then we must 845 * be doing an lmem -> lmem transfer, and so 846 * should be safe to directly copy the CCS 847 * state. In this case we have either 848 * initialised the CCS aux state when first 849 * clearing the pages (since it is already 850 * allocated in lmem), or the user has 851 * potentially populated it, in which case we 852 * need to copy the CCS state as-is. 853 */ 854 err = emit_copy_ccs(rq, 855 dst_offset, INDIRECT_ACCESS, 856 src_offset, INDIRECT_ACCESS, 857 len); 858 } else { 859 /* 860 * While we can't always restore/manage the CCS 861 * state, we still need to ensure we don't leak 862 * the CCS state from the previous user, so make 863 * sure we overwrite it with something. 864 */ 865 err = emit_copy_ccs(rq, 866 dst_offset, INDIRECT_ACCESS, 867 dst_offset, DIRECT_ACCESS, 868 len); 869 } 870 871 if (err) 872 goto out_rq; 873 874 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 875 if (err) 876 goto out_rq; 877 } 878 879 /* Arbitration is re-enabled between requests. */ 880 out_rq: 881 if (*out) 882 i915_request_put(*out); 883 *out = i915_request_get(rq); 884 i915_request_add(rq); 885 886 if (err) 887 break; 888 889 if (!bytes_to_cpy && !ccs_bytes_to_cpy) { 890 if (src_is_lmem) 891 WARN_ON(it_src.sg && sg_dma_len(it_src.sg)); 892 else 893 WARN_ON(it_dst.sg && sg_dma_len(it_dst.sg)); 894 break; 895 } 896 897 if (WARN_ON(!it_src.sg || !sg_dma_len(it_src.sg) || 898 !it_dst.sg || !sg_dma_len(it_dst.sg) || 899 (ccs_bytes_to_cpy && (!it_ccs.sg || 900 !sg_dma_len(it_ccs.sg))))) { 901 err = -EINVAL; 902 break; 903 } 904 905 cond_resched(); 906 } while (1); 907 908 out_ce: 909 return err; 910 } 911 912 static int emit_clear(struct i915_request *rq, u32 offset, int size, 913 u32 value, bool is_lmem) 914 { 915 struct drm_i915_private *i915 = rq->engine->i915; 916 int mocs = rq->engine->gt->mocs.uc_index << 1; 917 const int ver = GRAPHICS_VER(i915); 918 int ring_sz; 919 u32 *cs; 920 921 GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX); 922 923 if (HAS_FLAT_CCS(i915) && ver >= 12) 924 ring_sz = XY_FAST_COLOR_BLT_DW; 925 else if (ver >= 8) 926 ring_sz = 8; 927 else 928 ring_sz = 6; 929 930 cs = intel_ring_begin(rq, ring_sz); 931 if (IS_ERR(cs)) 932 return PTR_ERR(cs); 933 934 if (HAS_FLAT_CCS(i915) && ver >= 12) { 935 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | 936 (XY_FAST_COLOR_BLT_DW - 2); 937 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | 938 (PAGE_SIZE - 1); 939 *cs++ = 0; 940 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 941 *cs++ = offset; 942 *cs++ = rq->engine->instance; 943 *cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; 944 /* BG7 */ 945 *cs++ = value; 946 *cs++ = 0; 947 *cs++ = 0; 948 *cs++ = 0; 949 /* BG11 */ 950 *cs++ = 0; 951 *cs++ = 0; 952 /* BG13 */ 953 *cs++ = 0; 954 *cs++ = 0; 955 *cs++ = 0; 956 } else if (ver >= 8) { 957 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2); 958 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; 959 *cs++ = 0; 960 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 961 *cs++ = offset; 962 *cs++ = rq->engine->instance; 963 *cs++ = value; 964 *cs++ = MI_NOOP; 965 } else { 966 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); 967 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; 968 *cs++ = 0; 969 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 970 *cs++ = offset; 971 *cs++ = value; 972 } 973 974 intel_ring_advance(rq, cs); 975 return 0; 976 } 977 978 int 979 intel_context_migrate_clear(struct intel_context *ce, 980 const struct i915_deps *deps, 981 struct scatterlist *sg, 982 enum i915_cache_level cache_level, 983 bool is_lmem, 984 u32 value, 985 struct i915_request **out) 986 { 987 struct drm_i915_private *i915 = ce->engine->i915; 988 struct sgt_dma it = sg_sgt(sg); 989 struct i915_request *rq; 990 u32 offset; 991 int err; 992 993 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm); 994 *out = NULL; 995 996 GEM_BUG_ON(ce->ring->size < SZ_64K); 997 998 offset = 0; 999 if (HAS_64K_PAGES(i915) && is_lmem) 1000 offset = CHUNK_SZ; 1001 1002 do { 1003 int len; 1004 1005 rq = i915_request_create(ce); 1006 if (IS_ERR(rq)) { 1007 err = PTR_ERR(rq); 1008 goto out_ce; 1009 } 1010 1011 if (deps) { 1012 err = i915_request_await_deps(rq, deps); 1013 if (err) 1014 goto out_rq; 1015 1016 if (rq->engine->emit_init_breadcrumb) { 1017 err = rq->engine->emit_init_breadcrumb(rq); 1018 if (err) 1019 goto out_rq; 1020 } 1021 1022 deps = NULL; 1023 } 1024 1025 /* The PTE updates + clear must not be interrupted. */ 1026 err = emit_no_arbitration(rq); 1027 if (err) 1028 goto out_rq; 1029 1030 len = emit_pte(rq, &it, cache_level, is_lmem, offset, CHUNK_SZ); 1031 if (len <= 0) { 1032 err = len; 1033 goto out_rq; 1034 } 1035 1036 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 1037 if (err) 1038 goto out_rq; 1039 1040 err = emit_clear(rq, offset, len, value, is_lmem); 1041 if (err) 1042 goto out_rq; 1043 1044 if (HAS_FLAT_CCS(i915) && is_lmem && !value) { 1045 /* 1046 * copy the content of memory into corresponding 1047 * ccs surface 1048 */ 1049 err = emit_copy_ccs(rq, offset, INDIRECT_ACCESS, offset, 1050 DIRECT_ACCESS, len); 1051 if (err) 1052 goto out_rq; 1053 } 1054 1055 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 1056 1057 /* Arbitration is re-enabled between requests. */ 1058 out_rq: 1059 if (*out) 1060 i915_request_put(*out); 1061 *out = i915_request_get(rq); 1062 i915_request_add(rq); 1063 if (err || !it.sg || !sg_dma_len(it.sg)) 1064 break; 1065 1066 cond_resched(); 1067 } while (1); 1068 1069 out_ce: 1070 return err; 1071 } 1072 1073 int intel_migrate_copy(struct intel_migrate *m, 1074 struct i915_gem_ww_ctx *ww, 1075 const struct i915_deps *deps, 1076 struct scatterlist *src, 1077 enum i915_cache_level src_cache_level, 1078 bool src_is_lmem, 1079 struct scatterlist *dst, 1080 enum i915_cache_level dst_cache_level, 1081 bool dst_is_lmem, 1082 struct i915_request **out) 1083 { 1084 struct intel_context *ce; 1085 int err; 1086 1087 *out = NULL; 1088 if (!m->context) 1089 return -ENODEV; 1090 1091 ce = intel_migrate_create_context(m); 1092 if (IS_ERR(ce)) 1093 ce = intel_context_get(m->context); 1094 GEM_BUG_ON(IS_ERR(ce)); 1095 1096 err = intel_context_pin_ww(ce, ww); 1097 if (err) 1098 goto out; 1099 1100 err = intel_context_migrate_copy(ce, deps, 1101 src, src_cache_level, src_is_lmem, 1102 dst, dst_cache_level, dst_is_lmem, 1103 out); 1104 1105 intel_context_unpin(ce); 1106 out: 1107 intel_context_put(ce); 1108 return err; 1109 } 1110 1111 int 1112 intel_migrate_clear(struct intel_migrate *m, 1113 struct i915_gem_ww_ctx *ww, 1114 const struct i915_deps *deps, 1115 struct scatterlist *sg, 1116 enum i915_cache_level cache_level, 1117 bool is_lmem, 1118 u32 value, 1119 struct i915_request **out) 1120 { 1121 struct intel_context *ce; 1122 int err; 1123 1124 *out = NULL; 1125 if (!m->context) 1126 return -ENODEV; 1127 1128 ce = intel_migrate_create_context(m); 1129 if (IS_ERR(ce)) 1130 ce = intel_context_get(m->context); 1131 GEM_BUG_ON(IS_ERR(ce)); 1132 1133 err = intel_context_pin_ww(ce, ww); 1134 if (err) 1135 goto out; 1136 1137 err = intel_context_migrate_clear(ce, deps, sg, cache_level, 1138 is_lmem, value, out); 1139 1140 intel_context_unpin(ce); 1141 out: 1142 intel_context_put(ce); 1143 return err; 1144 } 1145 1146 void intel_migrate_fini(struct intel_migrate *m) 1147 { 1148 struct intel_context *ce; 1149 1150 ce = fetch_and_zero(&m->context); 1151 if (!ce) 1152 return; 1153 1154 intel_engine_destroy_pinned_context(ce); 1155 } 1156 1157 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1158 #include "selftest_migrate.c" 1159 #endif 1160