1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 #include "intel_context.h"
8 #include "intel_gpu_commands.h"
9 #include "intel_gt.h"
10 #include "intel_gtt.h"
11 #include "intel_migrate.h"
12 #include "intel_ring.h"
13 #include "gem/i915_gem_lmem.h"
14 
15 struct insert_pte_data {
16 	u64 offset;
17 };
18 
19 #define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */
20 
21 #define GET_CCS_BYTES(i915, size)	(HAS_FLAT_CCS(i915) ? \
22 					 DIV_ROUND_UP(size, NUM_BYTES_PER_CCS_BYTE) : 0)
engine_supports_migration(struct intel_engine_cs * engine)23 static bool engine_supports_migration(struct intel_engine_cs *engine)
24 {
25 	if (!engine)
26 		return false;
27 
28 	/*
29 	 * We need the ability to prevent aribtration (MI_ARB_ON_OFF),
30 	 * the ability to write PTE using inline data (MI_STORE_DATA)
31 	 * and of course the ability to do the block transfer (blits).
32 	 */
33 	GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS);
34 
35 	return true;
36 }
37 
xehpsdv_toggle_pdes(struct i915_address_space * vm,struct i915_page_table * pt,void * data)38 static void xehpsdv_toggle_pdes(struct i915_address_space *vm,
39 				struct i915_page_table *pt,
40 				void *data)
41 {
42 	struct insert_pte_data *d = data;
43 
44 	/*
45 	 * Insert a dummy PTE into every PT that will map to LMEM to ensure
46 	 * we have a correctly setup PDE structure for later use.
47 	 */
48 	vm->insert_page(vm, 0, d->offset,
49 			i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE),
50 			PTE_LM);
51 	GEM_BUG_ON(!pt->is_compact);
52 	d->offset += SZ_2M;
53 }
54 
xehpsdv_insert_pte(struct i915_address_space * vm,struct i915_page_table * pt,void * data)55 static void xehpsdv_insert_pte(struct i915_address_space *vm,
56 			       struct i915_page_table *pt,
57 			       void *data)
58 {
59 	struct insert_pte_data *d = data;
60 
61 	/*
62 	 * We are playing tricks here, since the actual pt, from the hw
63 	 * pov, is only 256bytes with 32 entries, or 4096bytes with 512
64 	 * entries, but we are still guaranteed that the physical
65 	 * alignment is 64K underneath for the pt, and we are careful
66 	 * not to access the space in the void.
67 	 */
68 	vm->insert_page(vm, px_dma(pt), d->offset,
69 			i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE),
70 			PTE_LM);
71 	d->offset += SZ_64K;
72 }
73 
insert_pte(struct i915_address_space * vm,struct i915_page_table * pt,void * data)74 static void insert_pte(struct i915_address_space *vm,
75 		       struct i915_page_table *pt,
76 		       void *data)
77 {
78 	struct insert_pte_data *d = data;
79 
80 	vm->insert_page(vm, px_dma(pt), d->offset,
81 			i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE),
82 			i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0);
83 	d->offset += PAGE_SIZE;
84 }
85 
migrate_vm(struct intel_gt * gt)86 static struct i915_address_space *migrate_vm(struct intel_gt *gt)
87 {
88 	struct i915_vm_pt_stash stash = {};
89 	struct i915_ppgtt *vm;
90 	int err;
91 	int i;
92 
93 	/*
94 	 * We construct a very special VM for use by all migration contexts,
95 	 * it is kept pinned so that it can be used at any time. As we need
96 	 * to pre-allocate the page directories for the migration VM, this
97 	 * limits us to only using a small number of prepared vma.
98 	 *
99 	 * To be able to pipeline and reschedule migration operations while
100 	 * avoiding unnecessary contention on the vm itself, the PTE updates
101 	 * are inline with the blits. All the blits use the same fixed
102 	 * addresses, with the backing store redirection being updated on the
103 	 * fly. Only 2 implicit vma are used for all migration operations.
104 	 *
105 	 * We lay the ppGTT out as:
106 	 *
107 	 *	[0, CHUNK_SZ) -> first object
108 	 *	[CHUNK_SZ, 2 * CHUNK_SZ) -> second object
109 	 *	[2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE
110 	 *
111 	 * By exposing the dma addresses of the page directories themselves
112 	 * within the ppGTT, we are then able to rewrite the PTE prior to use.
113 	 * But the PTE update and subsequent migration operation must be atomic,
114 	 * i.e. within the same non-preemptible window so that we do not switch
115 	 * to another migration context that overwrites the PTE.
116 	 *
117 	 * This changes quite a bit on platforms with HAS_64K_PAGES support,
118 	 * where we instead have three windows, each CHUNK_SIZE in size. The
119 	 * first is reserved for mapping system-memory, and that just uses the
120 	 * 512 entry layout using 4K GTT pages. The other two windows just map
121 	 * lmem pages and must use the new compact 32 entry layout using 64K GTT
122 	 * pages, which ensures we can address any lmem object that the user
123 	 * throws at us. We then also use the xehpsdv_toggle_pdes as a way of
124 	 * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the
125 	 * compact layout for each of these page-tables, that fall within the
126 	 * [CHUNK_SIZE, 3 * CHUNK_SIZE) range.
127 	 *
128 	 * We lay the ppGTT out as:
129 	 *
130 	 * [0, CHUNK_SZ) -> first window/object, maps smem
131 	 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src
132 	 * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst
133 	 *
134 	 * For the PTE window it's also quite different, since each PTE must
135 	 * point to some 64K page, one for each PT(since it's in lmem), and yet
136 	 * each is only <= 4096bytes, but since the unused space within that PTE
137 	 * range is never touched, this should be fine.
138 	 *
139 	 * So basically each PT now needs 64K of virtual memory, instead of 4K,
140 	 * which looks like:
141 	 *
142 	 * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE
143 	 */
144 
145 	vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY);
146 	if (IS_ERR(vm))
147 		return ERR_CAST(vm);
148 
149 	if (!vm->vm.allocate_va_range || !vm->vm.foreach) {
150 		err = -ENODEV;
151 		goto err_vm;
152 	}
153 
154 	if (HAS_64K_PAGES(gt->i915))
155 		stash.pt_sz = I915_GTT_PAGE_SIZE_64K;
156 
157 	/*
158 	 * Each engine instance is assigned its own chunk in the VM, so
159 	 * that we can run multiple instances concurrently
160 	 */
161 	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
162 		struct intel_engine_cs *engine;
163 		u64 base = (u64)i << 32;
164 		struct insert_pte_data d = {};
165 		struct i915_gem_ww_ctx ww;
166 		u64 sz;
167 
168 		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
169 		if (!engine_supports_migration(engine))
170 			continue;
171 
172 		/*
173 		 * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need
174 		 * 4x2 page directories for source/destination.
175 		 */
176 		if (HAS_64K_PAGES(gt->i915))
177 			sz = 3 * CHUNK_SZ;
178 		else
179 			sz = 2 * CHUNK_SZ;
180 		d.offset = base + sz;
181 
182 		/*
183 		 * We need another page directory setup so that we can write
184 		 * the 8x512 PTE in each chunk.
185 		 */
186 		if (HAS_64K_PAGES(gt->i915))
187 			sz += (sz / SZ_2M) * SZ_64K;
188 		else
189 			sz += (sz >> 12) * sizeof(u64);
190 
191 		err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz);
192 		if (err)
193 			goto err_vm;
194 
195 		for_i915_gem_ww(&ww, err, true) {
196 			err = i915_vm_lock_objects(&vm->vm, &ww);
197 			if (err)
198 				continue;
199 			err = i915_vm_map_pt_stash(&vm->vm, &stash);
200 			if (err)
201 				continue;
202 
203 			vm->vm.allocate_va_range(&vm->vm, &stash, base, sz);
204 		}
205 		i915_vm_free_pt_stash(&vm->vm, &stash);
206 		if (err)
207 			goto err_vm;
208 
209 		/* Now allow the GPU to rewrite the PTE via its own ppGTT */
210 		if (HAS_64K_PAGES(gt->i915)) {
211 			vm->vm.foreach(&vm->vm, base, d.offset - base,
212 				       xehpsdv_insert_pte, &d);
213 			d.offset = base + CHUNK_SZ;
214 			vm->vm.foreach(&vm->vm,
215 				       d.offset,
216 				       2 * CHUNK_SZ,
217 				       xehpsdv_toggle_pdes, &d);
218 		} else {
219 			vm->vm.foreach(&vm->vm, base, d.offset - base,
220 				       insert_pte, &d);
221 		}
222 	}
223 
224 	return &vm->vm;
225 
226 err_vm:
227 	i915_vm_put(&vm->vm);
228 	return ERR_PTR(err);
229 }
230 
first_copy_engine(struct intel_gt * gt)231 static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt)
232 {
233 	struct intel_engine_cs *engine;
234 	int i;
235 
236 	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
237 		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
238 		if (engine_supports_migration(engine))
239 			return engine;
240 	}
241 
242 	return NULL;
243 }
244 
pinned_context(struct intel_gt * gt)245 static struct intel_context *pinned_context(struct intel_gt *gt)
246 {
247 	static struct lock_class_key key;
248 	struct intel_engine_cs *engine;
249 	struct i915_address_space *vm;
250 	struct intel_context *ce;
251 
252 	engine = first_copy_engine(gt);
253 	if (!engine)
254 		return ERR_PTR(-ENODEV);
255 
256 	vm = migrate_vm(gt);
257 	if (IS_ERR(vm))
258 		return ERR_CAST(vm);
259 
260 	ce = intel_engine_create_pinned_context(engine, vm, SZ_512K,
261 						I915_GEM_HWS_MIGRATE,
262 						&key, "migrate");
263 	i915_vm_put(vm);
264 	return ce;
265 }
266 
intel_migrate_init(struct intel_migrate * m,struct intel_gt * gt)267 int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt)
268 {
269 	struct intel_context *ce;
270 
271 	memset(m, 0, sizeof(*m));
272 
273 	ce = pinned_context(gt);
274 	if (IS_ERR(ce))
275 		return PTR_ERR(ce);
276 
277 	m->context = ce;
278 	return 0;
279 }
280 
random_index(unsigned int max)281 static int random_index(unsigned int max)
282 {
283 	return upper_32_bits(mul_u32_u32(get_random_u32(), max));
284 }
285 
__migrate_engines(struct intel_gt * gt)286 static struct intel_context *__migrate_engines(struct intel_gt *gt)
287 {
288 	struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE];
289 	struct intel_engine_cs *engine;
290 	unsigned int count, i;
291 
292 	count = 0;
293 	for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
294 		engine = gt->engine_class[COPY_ENGINE_CLASS][i];
295 		if (engine_supports_migration(engine))
296 			engines[count++] = engine;
297 	}
298 
299 	return intel_context_create(engines[random_index(count)]);
300 }
301 
intel_migrate_create_context(struct intel_migrate * m)302 struct intel_context *intel_migrate_create_context(struct intel_migrate *m)
303 {
304 	struct intel_context *ce;
305 
306 	/*
307 	 * We randomly distribute contexts across the engines upon constrction,
308 	 * as they all share the same pinned vm, and so in order to allow
309 	 * multiple blits to run in parallel, we must construct each blit
310 	 * to use a different range of the vm for its GTT. This has to be
311 	 * known at construction, so we can not use the late greedy load
312 	 * balancing of the virtual-engine.
313 	 */
314 	ce = __migrate_engines(m->context->engine->gt);
315 	if (IS_ERR(ce))
316 		return ce;
317 
318 	ce->ring = NULL;
319 	ce->ring_size = SZ_256K;
320 
321 	i915_vm_put(ce->vm);
322 	ce->vm = i915_vm_get(m->context->vm);
323 
324 	return ce;
325 }
326 
sg_sgt(struct scatterlist * sg)327 static inline struct sgt_dma sg_sgt(struct scatterlist *sg)
328 {
329 	dma_addr_t addr = sg_dma_address(sg);
330 
331 	return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) };
332 }
333 
emit_no_arbitration(struct i915_request * rq)334 static int emit_no_arbitration(struct i915_request *rq)
335 {
336 	u32 *cs;
337 
338 	cs = intel_ring_begin(rq, 2);
339 	if (IS_ERR(cs))
340 		return PTR_ERR(cs);
341 
342 	/* Explicitly disable preemption for this request. */
343 	*cs++ = MI_ARB_ON_OFF;
344 	*cs++ = MI_NOOP;
345 	intel_ring_advance(rq, cs);
346 
347 	return 0;
348 }
349 
max_pte_pkt_size(struct i915_request * rq,int pkt)350 static int max_pte_pkt_size(struct i915_request *rq, int pkt)
351 {
352 	struct intel_ring *ring = rq->ring;
353 
354 	pkt = min_t(int, pkt, (ring->space - rq->reserved_space) / sizeof(u32) + 5);
355 	pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5);
356 
357 	return pkt;
358 }
359 
360 #define I915_EMIT_PTE_NUM_DWORDS 6
361 
emit_pte(struct i915_request * rq,struct sgt_dma * it,unsigned int pat_index,bool is_lmem,u64 offset,int length)362 static int emit_pte(struct i915_request *rq,
363 		    struct sgt_dma *it,
364 		    unsigned int pat_index,
365 		    bool is_lmem,
366 		    u64 offset,
367 		    int length)
368 {
369 	bool has_64K_pages = HAS_64K_PAGES(rq->i915);
370 	const u64 encode = rq->context->vm->pte_encode(0, pat_index,
371 						       is_lmem ? PTE_LM : 0);
372 	struct intel_ring *ring = rq->ring;
373 	int pkt, dword_length;
374 	u32 total = 0;
375 	u32 page_size;
376 	u32 *hdr, *cs;
377 
378 	GEM_BUG_ON(GRAPHICS_VER(rq->i915) < 8);
379 
380 	page_size = I915_GTT_PAGE_SIZE;
381 	dword_length = 0x400;
382 
383 	/* Compute the page directory offset for the target address range */
384 	if (has_64K_pages) {
385 		GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M));
386 
387 		offset /= SZ_2M;
388 		offset *= SZ_64K;
389 		offset += 3 * CHUNK_SZ;
390 
391 		if (is_lmem) {
392 			page_size = I915_GTT_PAGE_SIZE_64K;
393 			dword_length = 0x40;
394 		}
395 	} else {
396 		offset >>= 12;
397 		offset *= sizeof(u64);
398 		offset += 2 * CHUNK_SZ;
399 	}
400 
401 	offset += (u64)rq->engine->instance << 32;
402 
403 	cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS);
404 	if (IS_ERR(cs))
405 		return PTR_ERR(cs);
406 
407 	/* Pack as many PTE updates as possible into a single MI command */
408 	pkt = max_pte_pkt_size(rq, dword_length);
409 
410 	hdr = cs;
411 	*cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */
412 	*cs++ = lower_32_bits(offset);
413 	*cs++ = upper_32_bits(offset);
414 
415 	do {
416 		if (cs - hdr >= pkt) {
417 			int dword_rem;
418 
419 			*hdr += cs - hdr - 2;
420 			*cs++ = MI_NOOP;
421 
422 			ring->emit = (void *)cs - ring->vaddr;
423 			intel_ring_advance(rq, cs);
424 			intel_ring_update_space(ring);
425 
426 			cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS);
427 			if (IS_ERR(cs))
428 				return PTR_ERR(cs);
429 
430 			dword_rem = dword_length;
431 			if (has_64K_pages) {
432 				if (IS_ALIGNED(total, SZ_2M)) {
433 					offset = round_up(offset, SZ_64K);
434 				} else {
435 					dword_rem = SZ_2M - (total & (SZ_2M - 1));
436 					dword_rem /= page_size;
437 					dword_rem *= 2;
438 				}
439 			}
440 
441 			pkt = max_pte_pkt_size(rq, dword_rem);
442 
443 			hdr = cs;
444 			*cs++ = MI_STORE_DATA_IMM | REG_BIT(21);
445 			*cs++ = lower_32_bits(offset);
446 			*cs++ = upper_32_bits(offset);
447 		}
448 
449 		GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size));
450 
451 		*cs++ = lower_32_bits(encode | it->dma);
452 		*cs++ = upper_32_bits(encode | it->dma);
453 
454 		offset += 8;
455 		total += page_size;
456 
457 		it->dma += page_size;
458 		if (it->dma >= it->max) {
459 			it->sg = __sg_next(it->sg);
460 			if (!it->sg || sg_dma_len(it->sg) == 0)
461 				break;
462 
463 			it->dma = sg_dma_address(it->sg);
464 			it->max = it->dma + sg_dma_len(it->sg);
465 		}
466 	} while (total < length);
467 
468 	*hdr += cs - hdr - 2;
469 	*cs++ = MI_NOOP;
470 
471 	ring->emit = (void *)cs - ring->vaddr;
472 	intel_ring_advance(rq, cs);
473 	intel_ring_update_space(ring);
474 
475 	return total;
476 }
477 
wa_1209644611_applies(int ver,u32 size)478 static bool wa_1209644611_applies(int ver, u32 size)
479 {
480 	u32 height = size >> PAGE_SHIFT;
481 
482 	if (ver != 11)
483 		return false;
484 
485 	return height % 4 == 3 && height <= 8;
486 }
487 
488 /**
489  * DOC: Flat-CCS - Memory compression for Local memory
490  *
491  * On Xe-HP and later devices, we use dedicated compression control state (CCS)
492  * stored in local memory for each surface, to support the 3D and media
493  * compression formats.
494  *
495  * The memory required for the CCS of the entire local memory is 1/256 of the
496  * local memory size. So before the kernel boot, the required memory is reserved
497  * for the CCS data and a secure register will be programmed with the CCS base
498  * address.
499  *
500  * Flat CCS data needs to be cleared when a lmem object is allocated.
501  * And CCS data can be copied in and out of CCS region through
502  * XY_CTRL_SURF_COPY_BLT. CPU can't access the CCS data directly.
503  *
504  * I915 supports Flat-CCS on lmem only objects. When an objects has smem in
505  * its preference list, on memory pressure, i915 needs to migrate the lmem
506  * content into smem. If the lmem object is Flat-CCS compressed by userspace,
507  * then i915 needs to decompress it. But I915 lack the required information
508  * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects.
509  *
510  * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can
511  * be temporarily evicted to smem, along with the auxiliary CCS state, where
512  * it can be potentially swapped-out at a later point, if required.
513  * If userspace later touches the evicted pages, then we always move
514  * the backing memory back to lmem, which includes restoring the saved CCS state,
515  * and potentially performing any required swap-in.
516  *
517  * For the migration of the lmem objects with smem in placement list, such as
518  * {lmem, smem}, objects are treated as non Flat-CCS capable objects.
519  */
520 
i915_flush_dw(u32 * cmd,u32 flags)521 static inline u32 *i915_flush_dw(u32 *cmd, u32 flags)
522 {
523 	*cmd++ = MI_FLUSH_DW | flags;
524 	*cmd++ = 0;
525 	*cmd++ = 0;
526 
527 	return cmd;
528 }
529 
emit_copy_ccs(struct i915_request * rq,u32 dst_offset,u8 dst_access,u32 src_offset,u8 src_access,int size)530 static int emit_copy_ccs(struct i915_request *rq,
531 			 u32 dst_offset, u8 dst_access,
532 			 u32 src_offset, u8 src_access, int size)
533 {
534 	struct drm_i915_private *i915 = rq->i915;
535 	int mocs = rq->engine->gt->mocs.uc_index << 1;
536 	u32 num_ccs_blks;
537 	u32 *cs;
538 
539 	cs = intel_ring_begin(rq, 12);
540 	if (IS_ERR(cs))
541 		return PTR_ERR(cs);
542 
543 	num_ccs_blks = DIV_ROUND_UP(GET_CCS_BYTES(i915, size),
544 				    NUM_CCS_BYTES_PER_BLOCK);
545 	GEM_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER);
546 	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
547 
548 	/*
549 	 * The XY_CTRL_SURF_COPY_BLT instruction is used to copy the CCS
550 	 * data in and out of the CCS region.
551 	 *
552 	 * We can copy at most 1024 blocks of 256 bytes using one
553 	 * XY_CTRL_SURF_COPY_BLT instruction.
554 	 *
555 	 * In case we need to copy more than 1024 blocks, we need to add
556 	 * another instruction to the same batch buffer.
557 	 *
558 	 * 1024 blocks of 256 bytes of CCS represent a total 256KB of CCS.
559 	 *
560 	 * 256 KB of CCS represents 256 * 256 KB = 64 MB of LMEM.
561 	 */
562 	*cs++ = XY_CTRL_SURF_COPY_BLT |
563 		src_access << SRC_ACCESS_TYPE_SHIFT |
564 		dst_access << DST_ACCESS_TYPE_SHIFT |
565 		((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT;
566 	*cs++ = src_offset;
567 	*cs++ = rq->engine->instance |
568 		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
569 	*cs++ = dst_offset;
570 	*cs++ = rq->engine->instance |
571 		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
572 
573 	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
574 	*cs++ = MI_NOOP;
575 
576 	intel_ring_advance(rq, cs);
577 
578 	return 0;
579 }
580 
emit_copy(struct i915_request * rq,u32 dst_offset,u32 src_offset,int size)581 static int emit_copy(struct i915_request *rq,
582 		     u32 dst_offset, u32 src_offset, int size)
583 {
584 	const int ver = GRAPHICS_VER(rq->i915);
585 	u32 instance = rq->engine->instance;
586 	u32 *cs;
587 
588 	cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6);
589 	if (IS_ERR(cs))
590 		return PTR_ERR(cs);
591 
592 	if (ver >= 9 && !wa_1209644611_applies(ver, size)) {
593 		*cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2);
594 		*cs++ = BLT_DEPTH_32 | PAGE_SIZE;
595 		*cs++ = 0;
596 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
597 		*cs++ = dst_offset;
598 		*cs++ = instance;
599 		*cs++ = 0;
600 		*cs++ = PAGE_SIZE;
601 		*cs++ = src_offset;
602 		*cs++ = instance;
603 	} else if (ver >= 8) {
604 		*cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2);
605 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
606 		*cs++ = 0;
607 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
608 		*cs++ = dst_offset;
609 		*cs++ = instance;
610 		*cs++ = 0;
611 		*cs++ = PAGE_SIZE;
612 		*cs++ = src_offset;
613 		*cs++ = instance;
614 	} else {
615 		GEM_BUG_ON(instance);
616 		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
617 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
618 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE;
619 		*cs++ = dst_offset;
620 		*cs++ = PAGE_SIZE;
621 		*cs++ = src_offset;
622 	}
623 
624 	intel_ring_advance(rq, cs);
625 	return 0;
626 }
627 
scatter_list_length(struct scatterlist * sg)628 static u64 scatter_list_length(struct scatterlist *sg)
629 {
630 	u64 len = 0;
631 
632 	while (sg && sg_dma_len(sg)) {
633 		len += sg_dma_len(sg);
634 		sg = sg_next(sg);
635 	}
636 
637 	return len;
638 }
639 
640 static int
calculate_chunk_sz(struct drm_i915_private * i915,bool src_is_lmem,u64 bytes_to_cpy,u64 ccs_bytes_to_cpy)641 calculate_chunk_sz(struct drm_i915_private *i915, bool src_is_lmem,
642 		   u64 bytes_to_cpy, u64 ccs_bytes_to_cpy)
643 {
644 	if (ccs_bytes_to_cpy && !src_is_lmem)
645 		/*
646 		 * When CHUNK_SZ is passed all the pages upto CHUNK_SZ
647 		 * will be taken for the blt. in Flat-ccs supported
648 		 * platform Smem obj will have more pages than required
649 		 * for main meory hence limit it to the required size
650 		 * for main memory
651 		 */
652 		return min_t(u64, bytes_to_cpy, CHUNK_SZ);
653 	else
654 		return CHUNK_SZ;
655 }
656 
get_ccs_sg_sgt(struct sgt_dma * it,u64 bytes_to_cpy)657 static void get_ccs_sg_sgt(struct sgt_dma *it, u64 bytes_to_cpy)
658 {
659 	u64 len;
660 
661 	do {
662 		GEM_BUG_ON(!it->sg || !sg_dma_len(it->sg));
663 		len = it->max - it->dma;
664 		if (len > bytes_to_cpy) {
665 			it->dma += bytes_to_cpy;
666 			break;
667 		}
668 
669 		bytes_to_cpy -= len;
670 
671 		it->sg = __sg_next(it->sg);
672 		it->dma = sg_dma_address(it->sg);
673 		it->max = it->dma + sg_dma_len(it->sg);
674 	} while (bytes_to_cpy);
675 }
676 
677 int
intel_context_migrate_copy(struct intel_context * ce,const struct i915_deps * deps,struct scatterlist * src,unsigned int src_pat_index,bool src_is_lmem,struct scatterlist * dst,unsigned int dst_pat_index,bool dst_is_lmem,struct i915_request ** out)678 intel_context_migrate_copy(struct intel_context *ce,
679 			   const struct i915_deps *deps,
680 			   struct scatterlist *src,
681 			   unsigned int src_pat_index,
682 			   bool src_is_lmem,
683 			   struct scatterlist *dst,
684 			   unsigned int dst_pat_index,
685 			   bool dst_is_lmem,
686 			   struct i915_request **out)
687 {
688 	struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst), it_ccs;
689 	struct drm_i915_private *i915 = ce->engine->i915;
690 	u64 ccs_bytes_to_cpy = 0, bytes_to_cpy;
691 	unsigned int ccs_pat_index;
692 	u32 src_offset, dst_offset;
693 	u8 src_access, dst_access;
694 	struct i915_request *rq;
695 	u64 src_sz, dst_sz;
696 	bool ccs_is_src, overwrite_ccs;
697 	int err;
698 
699 	GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
700 	GEM_BUG_ON(IS_DGFX(ce->engine->i915) && (!src_is_lmem && !dst_is_lmem));
701 	*out = NULL;
702 
703 	GEM_BUG_ON(ce->ring->size < SZ_64K);
704 
705 	src_sz = scatter_list_length(src);
706 	bytes_to_cpy = src_sz;
707 
708 	if (HAS_FLAT_CCS(i915) && src_is_lmem ^ dst_is_lmem) {
709 		src_access = !src_is_lmem && dst_is_lmem;
710 		dst_access = !src_access;
711 
712 		dst_sz = scatter_list_length(dst);
713 		if (src_is_lmem) {
714 			it_ccs = it_dst;
715 			ccs_pat_index = dst_pat_index;
716 			ccs_is_src = false;
717 		} else if (dst_is_lmem) {
718 			bytes_to_cpy = dst_sz;
719 			it_ccs = it_src;
720 			ccs_pat_index = src_pat_index;
721 			ccs_is_src = true;
722 		}
723 
724 		/*
725 		 * When there is a eviction of ccs needed smem will have the
726 		 * extra pages for the ccs data
727 		 *
728 		 * TO-DO: Want to move the size mismatch check to a WARN_ON,
729 		 * but still we have some requests of smem->lmem with same size.
730 		 * Need to fix it.
731 		 */
732 		ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0;
733 		if (ccs_bytes_to_cpy)
734 			get_ccs_sg_sgt(&it_ccs, bytes_to_cpy);
735 	}
736 
737 	overwrite_ccs = HAS_FLAT_CCS(i915) && !ccs_bytes_to_cpy && dst_is_lmem;
738 
739 	src_offset = 0;
740 	dst_offset = CHUNK_SZ;
741 	if (HAS_64K_PAGES(ce->engine->i915)) {
742 		src_offset = 0;
743 		dst_offset = 0;
744 		if (src_is_lmem)
745 			src_offset = CHUNK_SZ;
746 		if (dst_is_lmem)
747 			dst_offset = 2 * CHUNK_SZ;
748 	}
749 
750 	do {
751 		int len;
752 
753 		rq = i915_request_create(ce);
754 		if (IS_ERR(rq)) {
755 			err = PTR_ERR(rq);
756 			goto out_ce;
757 		}
758 
759 		if (deps) {
760 			err = i915_request_await_deps(rq, deps);
761 			if (err)
762 				goto out_rq;
763 
764 			if (rq->engine->emit_init_breadcrumb) {
765 				err = rq->engine->emit_init_breadcrumb(rq);
766 				if (err)
767 					goto out_rq;
768 			}
769 
770 			deps = NULL;
771 		}
772 
773 		/* The PTE updates + copy must not be interrupted. */
774 		err = emit_no_arbitration(rq);
775 		if (err)
776 			goto out_rq;
777 
778 		src_sz = calculate_chunk_sz(i915, src_is_lmem,
779 					    bytes_to_cpy, ccs_bytes_to_cpy);
780 
781 		len = emit_pte(rq, &it_src, src_pat_index, src_is_lmem,
782 			       src_offset, src_sz);
783 		if (!len) {
784 			err = -EINVAL;
785 			goto out_rq;
786 		}
787 		if (len < 0) {
788 			err = len;
789 			goto out_rq;
790 		}
791 
792 		err = emit_pte(rq, &it_dst, dst_pat_index, dst_is_lmem,
793 			       dst_offset, len);
794 		if (err < 0)
795 			goto out_rq;
796 		if (err < len) {
797 			err = -EINVAL;
798 			goto out_rq;
799 		}
800 
801 		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
802 		if (err)
803 			goto out_rq;
804 
805 		err = emit_copy(rq, dst_offset,	src_offset, len);
806 		if (err)
807 			goto out_rq;
808 
809 		bytes_to_cpy -= len;
810 
811 		if (ccs_bytes_to_cpy) {
812 			int ccs_sz;
813 
814 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
815 			if (err)
816 				goto out_rq;
817 
818 			ccs_sz = GET_CCS_BYTES(i915, len);
819 			err = emit_pte(rq, &it_ccs, ccs_pat_index, false,
820 				       ccs_is_src ? src_offset : dst_offset,
821 				       ccs_sz);
822 			if (err < 0)
823 				goto out_rq;
824 			if (err < ccs_sz) {
825 				err = -EINVAL;
826 				goto out_rq;
827 			}
828 
829 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
830 			if (err)
831 				goto out_rq;
832 
833 			err = emit_copy_ccs(rq, dst_offset, dst_access,
834 					    src_offset, src_access, len);
835 			if (err)
836 				goto out_rq;
837 
838 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
839 			if (err)
840 				goto out_rq;
841 			ccs_bytes_to_cpy -= ccs_sz;
842 		} else if (overwrite_ccs) {
843 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
844 			if (err)
845 				goto out_rq;
846 
847 			if (src_is_lmem) {
848 				/*
849 				 * If the src is already in lmem, then we must
850 				 * be doing an lmem -> lmem transfer, and so
851 				 * should be safe to directly copy the CCS
852 				 * state. In this case we have either
853 				 * initialised the CCS aux state when first
854 				 * clearing the pages (since it is already
855 				 * allocated in lmem), or the user has
856 				 * potentially populated it, in which case we
857 				 * need to copy the CCS state as-is.
858 				 */
859 				err = emit_copy_ccs(rq,
860 						    dst_offset, INDIRECT_ACCESS,
861 						    src_offset, INDIRECT_ACCESS,
862 						    len);
863 			} else {
864 				/*
865 				 * While we can't always restore/manage the CCS
866 				 * state, we still need to ensure we don't leak
867 				 * the CCS state from the previous user, so make
868 				 * sure we overwrite it with something.
869 				 */
870 				err = emit_copy_ccs(rq,
871 						    dst_offset, INDIRECT_ACCESS,
872 						    dst_offset, DIRECT_ACCESS,
873 						    len);
874 			}
875 
876 			if (err)
877 				goto out_rq;
878 
879 			err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
880 			if (err)
881 				goto out_rq;
882 		}
883 
884 		/* Arbitration is re-enabled between requests. */
885 out_rq:
886 		if (*out)
887 			i915_request_put(*out);
888 		*out = i915_request_get(rq);
889 		i915_request_add(rq);
890 
891 		if (err)
892 			break;
893 
894 		if (!bytes_to_cpy && !ccs_bytes_to_cpy) {
895 			if (src_is_lmem)
896 				WARN_ON(it_src.sg && sg_dma_len(it_src.sg));
897 			else
898 				WARN_ON(it_dst.sg && sg_dma_len(it_dst.sg));
899 			break;
900 		}
901 
902 		if (WARN_ON(!it_src.sg || !sg_dma_len(it_src.sg) ||
903 			    !it_dst.sg || !sg_dma_len(it_dst.sg) ||
904 			    (ccs_bytes_to_cpy && (!it_ccs.sg ||
905 						  !sg_dma_len(it_ccs.sg))))) {
906 			err = -EINVAL;
907 			break;
908 		}
909 
910 		cond_resched();
911 	} while (1);
912 
913 out_ce:
914 	return err;
915 }
916 
emit_clear(struct i915_request * rq,u32 offset,int size,u32 value,bool is_lmem)917 static int emit_clear(struct i915_request *rq, u32 offset, int size,
918 		      u32 value, bool is_lmem)
919 {
920 	struct drm_i915_private *i915 = rq->i915;
921 	int mocs = rq->engine->gt->mocs.uc_index << 1;
922 	const int ver = GRAPHICS_VER(i915);
923 	int ring_sz;
924 	u32 *cs;
925 
926 	GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX);
927 
928 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
929 		ring_sz = XY_FAST_COLOR_BLT_DW;
930 	else if (ver >= 8)
931 		ring_sz = 8;
932 	else
933 		ring_sz = 6;
934 
935 	cs = intel_ring_begin(rq, ring_sz);
936 	if (IS_ERR(cs))
937 		return PTR_ERR(cs);
938 
939 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
940 		*cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
941 			(XY_FAST_COLOR_BLT_DW - 2);
942 		*cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) |
943 			(PAGE_SIZE - 1);
944 		*cs++ = 0;
945 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
946 		*cs++ = offset;
947 		*cs++ = rq->engine->instance;
948 		*cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
949 		/* BG7 */
950 		*cs++ = value;
951 		*cs++ = 0;
952 		*cs++ = 0;
953 		*cs++ = 0;
954 		/* BG11 */
955 		*cs++ = 0;
956 		*cs++ = 0;
957 		/* BG13 */
958 		*cs++ = 0;
959 		*cs++ = 0;
960 		*cs++ = 0;
961 	} else if (ver >= 8) {
962 		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2);
963 		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
964 		*cs++ = 0;
965 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
966 		*cs++ = offset;
967 		*cs++ = rq->engine->instance;
968 		*cs++ = value;
969 		*cs++ = MI_NOOP;
970 	} else {
971 		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
972 		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
973 		*cs++ = 0;
974 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
975 		*cs++ = offset;
976 		*cs++ = value;
977 	}
978 
979 	intel_ring_advance(rq, cs);
980 	return 0;
981 }
982 
983 int
intel_context_migrate_clear(struct intel_context * ce,const struct i915_deps * deps,struct scatterlist * sg,unsigned int pat_index,bool is_lmem,u32 value,struct i915_request ** out)984 intel_context_migrate_clear(struct intel_context *ce,
985 			    const struct i915_deps *deps,
986 			    struct scatterlist *sg,
987 			    unsigned int pat_index,
988 			    bool is_lmem,
989 			    u32 value,
990 			    struct i915_request **out)
991 {
992 	struct drm_i915_private *i915 = ce->engine->i915;
993 	struct sgt_dma it = sg_sgt(sg);
994 	struct i915_request *rq;
995 	u32 offset;
996 	int err;
997 
998 	GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
999 	*out = NULL;
1000 
1001 	GEM_BUG_ON(ce->ring->size < SZ_64K);
1002 
1003 	offset = 0;
1004 	if (HAS_64K_PAGES(i915) && is_lmem)
1005 		offset = CHUNK_SZ;
1006 
1007 	do {
1008 		int len;
1009 
1010 		rq = i915_request_create(ce);
1011 		if (IS_ERR(rq)) {
1012 			err = PTR_ERR(rq);
1013 			goto out_ce;
1014 		}
1015 
1016 		if (deps) {
1017 			err = i915_request_await_deps(rq, deps);
1018 			if (err)
1019 				goto out_rq;
1020 
1021 			if (rq->engine->emit_init_breadcrumb) {
1022 				err = rq->engine->emit_init_breadcrumb(rq);
1023 				if (err)
1024 					goto out_rq;
1025 			}
1026 
1027 			deps = NULL;
1028 		}
1029 
1030 		/* The PTE updates + clear must not be interrupted. */
1031 		err = emit_no_arbitration(rq);
1032 		if (err)
1033 			goto out_rq;
1034 
1035 		len = emit_pte(rq, &it, pat_index, is_lmem, offset, CHUNK_SZ);
1036 		if (len <= 0) {
1037 			err = len;
1038 			goto out_rq;
1039 		}
1040 
1041 		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1042 		if (err)
1043 			goto out_rq;
1044 
1045 		err = emit_clear(rq, offset, len, value, is_lmem);
1046 		if (err)
1047 			goto out_rq;
1048 
1049 		if (HAS_FLAT_CCS(i915) && is_lmem && !value) {
1050 			/*
1051 			 * copy the content of memory into corresponding
1052 			 * ccs surface
1053 			 */
1054 			err = emit_copy_ccs(rq, offset, INDIRECT_ACCESS, offset,
1055 					    DIRECT_ACCESS, len);
1056 			if (err)
1057 				goto out_rq;
1058 		}
1059 
1060 		err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1061 
1062 		/* Arbitration is re-enabled between requests. */
1063 out_rq:
1064 		if (*out)
1065 			i915_request_put(*out);
1066 		*out = i915_request_get(rq);
1067 		i915_request_add(rq);
1068 		if (err || !it.sg || !sg_dma_len(it.sg))
1069 			break;
1070 
1071 		cond_resched();
1072 	} while (1);
1073 
1074 out_ce:
1075 	return err;
1076 }
1077 
intel_migrate_copy(struct intel_migrate * m,struct i915_gem_ww_ctx * ww,const struct i915_deps * deps,struct scatterlist * src,unsigned int src_pat_index,bool src_is_lmem,struct scatterlist * dst,unsigned int dst_pat_index,bool dst_is_lmem,struct i915_request ** out)1078 int intel_migrate_copy(struct intel_migrate *m,
1079 		       struct i915_gem_ww_ctx *ww,
1080 		       const struct i915_deps *deps,
1081 		       struct scatterlist *src,
1082 		       unsigned int src_pat_index,
1083 		       bool src_is_lmem,
1084 		       struct scatterlist *dst,
1085 		       unsigned int dst_pat_index,
1086 		       bool dst_is_lmem,
1087 		       struct i915_request **out)
1088 {
1089 	struct intel_context *ce;
1090 	int err;
1091 
1092 	*out = NULL;
1093 	if (!m->context)
1094 		return -ENODEV;
1095 
1096 	ce = intel_migrate_create_context(m);
1097 	if (IS_ERR(ce))
1098 		ce = intel_context_get(m->context);
1099 	GEM_BUG_ON(IS_ERR(ce));
1100 
1101 	err = intel_context_pin_ww(ce, ww);
1102 	if (err)
1103 		goto out;
1104 
1105 	err = intel_context_migrate_copy(ce, deps,
1106 					 src, src_pat_index, src_is_lmem,
1107 					 dst, dst_pat_index, dst_is_lmem,
1108 					 out);
1109 
1110 	intel_context_unpin(ce);
1111 out:
1112 	intel_context_put(ce);
1113 	return err;
1114 }
1115 
1116 int
intel_migrate_clear(struct intel_migrate * m,struct i915_gem_ww_ctx * ww,const struct i915_deps * deps,struct scatterlist * sg,unsigned int pat_index,bool is_lmem,u32 value,struct i915_request ** out)1117 intel_migrate_clear(struct intel_migrate *m,
1118 		    struct i915_gem_ww_ctx *ww,
1119 		    const struct i915_deps *deps,
1120 		    struct scatterlist *sg,
1121 		    unsigned int pat_index,
1122 		    bool is_lmem,
1123 		    u32 value,
1124 		    struct i915_request **out)
1125 {
1126 	struct intel_context *ce;
1127 	int err;
1128 
1129 	*out = NULL;
1130 	if (!m->context)
1131 		return -ENODEV;
1132 
1133 	ce = intel_migrate_create_context(m);
1134 	if (IS_ERR(ce))
1135 		ce = intel_context_get(m->context);
1136 	GEM_BUG_ON(IS_ERR(ce));
1137 
1138 	err = intel_context_pin_ww(ce, ww);
1139 	if (err)
1140 		goto out;
1141 
1142 	err = intel_context_migrate_clear(ce, deps, sg, pat_index,
1143 					  is_lmem, value, out);
1144 
1145 	intel_context_unpin(ce);
1146 out:
1147 	intel_context_put(ce);
1148 	return err;
1149 }
1150 
intel_migrate_fini(struct intel_migrate * m)1151 void intel_migrate_fini(struct intel_migrate *m)
1152 {
1153 	struct intel_context *ce;
1154 
1155 	ce = fetch_and_zero(&m->context);
1156 	if (!ce)
1157 		return;
1158 
1159 	intel_engine_destroy_pinned_context(ce);
1160 }
1161 
1162 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1163 #include "selftest_migrate.c"
1164 #endif
1165