1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include "gen7_renderclear.h"
7 #include "i915_drv.h"
8 #include "intel_gpu_commands.h"
9 
10 #define GT3_INLINE_DATA_DELAYS 0x1E00
11 #define batch_advance(Y, CS) GEM_BUG_ON((Y)->end != (CS))
12 
13 struct cb_kernel {
14 	const void *data;
15 	u32 size;
16 };
17 
18 #define CB_KERNEL(name) { .data = (name), .size = sizeof(name) }
19 
20 #include "ivb_clear_kernel.c"
21 static const struct cb_kernel cb_kernel_ivb = CB_KERNEL(ivb_clear_kernel);
22 
23 #include "hsw_clear_kernel.c"
24 static const struct cb_kernel cb_kernel_hsw = CB_KERNEL(hsw_clear_kernel);
25 
26 struct batch_chunk {
27 	struct i915_vma *vma;
28 	u32 offset;
29 	u32 *start;
30 	u32 *end;
31 	u32 max_items;
32 };
33 
34 struct batch_vals {
35 	u32 max_threads;
36 	u32 state_start;
37 	u32 surface_start;
38 	u32 surface_height;
39 	u32 surface_width;
40 	u32 size;
41 };
42 
43 static int num_primitives(const struct batch_vals *bv)
44 {
45 	/*
46 	 * We need to saturate the GPU with work in order to dispatch
47 	 * a shader on every HW thread, and clear the thread-local registers.
48 	 * In short, we have to dispatch work faster than the shaders can
49 	 * run in order to fill the EU and occupy each HW thread.
50 	 */
51 	return bv->max_threads;
52 }
53 
54 static void
55 batch_get_defaults(struct drm_i915_private *i915, struct batch_vals *bv)
56 {
57 	if (IS_HASWELL(i915)) {
58 		switch (INTEL_INFO(i915)->gt) {
59 		default:
60 		case 1:
61 			bv->max_threads = 70;
62 			break;
63 		case 2:
64 			bv->max_threads = 140;
65 			break;
66 		case 3:
67 			bv->max_threads = 280;
68 			break;
69 		}
70 		bv->surface_height = 16 * 16;
71 		bv->surface_width = 32 * 2 * 16;
72 	} else {
73 		switch (INTEL_INFO(i915)->gt) {
74 		default:
75 		case 1: /* including vlv */
76 			bv->max_threads = 36;
77 			break;
78 		case 2:
79 			bv->max_threads = 128;
80 			break;
81 		}
82 		bv->surface_height = 16 * 8;
83 		bv->surface_width = 32 * 16;
84 	}
85 	bv->state_start = round_up(SZ_1K + num_primitives(bv) * 64, SZ_4K);
86 	bv->surface_start = bv->state_start + SZ_4K;
87 	bv->size = bv->surface_start + bv->surface_height * bv->surface_width;
88 }
89 
90 static void batch_init(struct batch_chunk *bc,
91 		       struct i915_vma *vma,
92 		       u32 *start, u32 offset, u32 max_bytes)
93 {
94 	bc->vma = vma;
95 	bc->offset = offset;
96 	bc->start = start + bc->offset / sizeof(*bc->start);
97 	bc->end = bc->start;
98 	bc->max_items = max_bytes / sizeof(*bc->start);
99 }
100 
101 static u32 batch_offset(const struct batch_chunk *bc, u32 *cs)
102 {
103 	return (cs - bc->start) * sizeof(*bc->start) + bc->offset;
104 }
105 
106 static u32 batch_addr(const struct batch_chunk *bc)
107 {
108 	return bc->vma->node.start;
109 }
110 
111 static void batch_add(struct batch_chunk *bc, const u32 d)
112 {
113 	GEM_BUG_ON((bc->end - bc->start) >= bc->max_items);
114 	*bc->end++ = d;
115 }
116 
117 static u32 *batch_alloc_items(struct batch_chunk *bc, u32 align, u32 items)
118 {
119 	u32 *map;
120 
121 	if (align) {
122 		u32 *end = PTR_ALIGN(bc->end, align);
123 
124 		memset32(bc->end, 0, end - bc->end);
125 		bc->end = end;
126 	}
127 
128 	map = bc->end;
129 	bc->end += items;
130 
131 	return map;
132 }
133 
134 static u32 *batch_alloc_bytes(struct batch_chunk *bc, u32 align, u32 bytes)
135 {
136 	GEM_BUG_ON(!IS_ALIGNED(bytes, sizeof(*bc->start)));
137 	return batch_alloc_items(bc, align, bytes / sizeof(*bc->start));
138 }
139 
140 static u32
141 gen7_fill_surface_state(struct batch_chunk *state,
142 			const u32 dst_offset,
143 			const struct batch_vals *bv)
144 {
145 	u32 surface_h = bv->surface_height;
146 	u32 surface_w = bv->surface_width;
147 	u32 *cs = batch_alloc_items(state, 32, 8);
148 	u32 offset = batch_offset(state, cs);
149 
150 #define SURFACE_2D 1
151 #define SURFACEFORMAT_B8G8R8A8_UNORM 0x0C0
152 #define RENDER_CACHE_READ_WRITE 1
153 
154 	*cs++ = SURFACE_2D << 29 |
155 		(SURFACEFORMAT_B8G8R8A8_UNORM << 18) |
156 		(RENDER_CACHE_READ_WRITE << 8);
157 
158 	*cs++ = batch_addr(state) + dst_offset;
159 
160 	*cs++ = ((surface_h / 4 - 1) << 16) | (surface_w / 4 - 1);
161 	*cs++ = surface_w;
162 	*cs++ = 0;
163 	*cs++ = 0;
164 	*cs++ = 0;
165 #define SHADER_CHANNELS(r, g, b, a) \
166 	(((r) << 25) | ((g) << 22) | ((b) << 19) | ((a) << 16))
167 	*cs++ = SHADER_CHANNELS(4, 5, 6, 7);
168 	batch_advance(state, cs);
169 
170 	return offset;
171 }
172 
173 static u32
174 gen7_fill_binding_table(struct batch_chunk *state,
175 			const struct batch_vals *bv)
176 {
177 	u32 surface_start =
178 		gen7_fill_surface_state(state, bv->surface_start, bv);
179 	u32 *cs = batch_alloc_items(state, 32, 8);
180 	u32 offset = batch_offset(state, cs);
181 
182 	*cs++ = surface_start - state->offset;
183 	*cs++ = 0;
184 	*cs++ = 0;
185 	*cs++ = 0;
186 	*cs++ = 0;
187 	*cs++ = 0;
188 	*cs++ = 0;
189 	*cs++ = 0;
190 	batch_advance(state, cs);
191 
192 	return offset;
193 }
194 
195 static u32
196 gen7_fill_kernel_data(struct batch_chunk *state,
197 		      const u32 *data,
198 		      const u32 size)
199 {
200 	return batch_offset(state,
201 			    memcpy(batch_alloc_bytes(state, 64, size),
202 				   data, size));
203 }
204 
205 static u32
206 gen7_fill_interface_descriptor(struct batch_chunk *state,
207 			       const struct batch_vals *bv,
208 			       const struct cb_kernel *kernel,
209 			       unsigned int count)
210 {
211 	u32 kernel_offset =
212 		gen7_fill_kernel_data(state, kernel->data, kernel->size);
213 	u32 binding_table = gen7_fill_binding_table(state, bv);
214 	u32 *cs = batch_alloc_items(state, 32, 8 * count);
215 	u32 offset = batch_offset(state, cs);
216 
217 	*cs++ = kernel_offset;
218 	*cs++ = (1 << 7) | (1 << 13);
219 	*cs++ = 0;
220 	*cs++ = (binding_table - state->offset) | 1;
221 	*cs++ = 0;
222 	*cs++ = 0;
223 	*cs++ = 0;
224 	*cs++ = 0;
225 
226 	/* 1 - 63dummy idds */
227 	memset32(cs, 0x00, (count - 1) * 8);
228 	batch_advance(state, cs + (count - 1) * 8);
229 
230 	return offset;
231 }
232 
233 static void
234 gen7_emit_state_base_address(struct batch_chunk *batch,
235 			     u32 surface_state_base)
236 {
237 	u32 *cs = batch_alloc_items(batch, 0, 10);
238 
239 	*cs++ = STATE_BASE_ADDRESS | (10 - 2);
240 	/* general */
241 	*cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
242 	/* surface */
243 	*cs++ = (batch_addr(batch) + surface_state_base) | BASE_ADDRESS_MODIFY;
244 	/* dynamic */
245 	*cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
246 	/* indirect */
247 	*cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
248 	/* instruction */
249 	*cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
250 
251 	/* general/dynamic/indirect/instruction access Bound */
252 	*cs++ = 0;
253 	*cs++ = BASE_ADDRESS_MODIFY;
254 	*cs++ = 0;
255 	*cs++ = BASE_ADDRESS_MODIFY;
256 	batch_advance(batch, cs);
257 }
258 
259 static void
260 gen7_emit_vfe_state(struct batch_chunk *batch,
261 		    const struct batch_vals *bv,
262 		    u32 urb_size, u32 curbe_size,
263 		    u32 mode)
264 {
265 	u32 threads = bv->max_threads - 1;
266 	u32 *cs = batch_alloc_items(batch, 32, 8);
267 
268 	*cs++ = MEDIA_VFE_STATE | (8 - 2);
269 
270 	/* scratch buffer */
271 	*cs++ = 0;
272 
273 	/* number of threads & urb entries for GPGPU vs Media Mode */
274 	*cs++ = threads << 16 | 1 << 8 | mode << 2;
275 
276 	*cs++ = 0;
277 
278 	/* urb entry size & curbe size in 256 bits unit */
279 	*cs++ = urb_size << 16 | curbe_size;
280 
281 	/* scoreboard */
282 	*cs++ = 0;
283 	*cs++ = 0;
284 	*cs++ = 0;
285 	batch_advance(batch, cs);
286 }
287 
288 static void
289 gen7_emit_interface_descriptor_load(struct batch_chunk *batch,
290 				    const u32 interface_descriptor,
291 				    unsigned int count)
292 {
293 	u32 *cs = batch_alloc_items(batch, 8, 4);
294 
295 	*cs++ = MEDIA_INTERFACE_DESCRIPTOR_LOAD | (4 - 2);
296 	*cs++ = 0;
297 	*cs++ = count * 8 * sizeof(*cs);
298 
299 	/*
300 	 * interface descriptor address - it is relative to the dynamics base
301 	 * address
302 	 */
303 	*cs++ = interface_descriptor;
304 	batch_advance(batch, cs);
305 }
306 
307 static void
308 gen7_emit_media_object(struct batch_chunk *batch,
309 		       unsigned int media_object_index)
310 {
311 	unsigned int x_offset = (media_object_index % 16) * 64;
312 	unsigned int y_offset = (media_object_index / 16) * 16;
313 	unsigned int pkt = 6 + 3;
314 	u32 *cs;
315 
316 	cs = batch_alloc_items(batch, 8, pkt);
317 
318 	*cs++ = MEDIA_OBJECT | (pkt - 2);
319 
320 	/* interface descriptor offset */
321 	*cs++ = 0;
322 
323 	/* without indirect data */
324 	*cs++ = 0;
325 	*cs++ = 0;
326 
327 	/* scoreboard */
328 	*cs++ = 0;
329 	*cs++ = 0;
330 
331 	/* inline */
332 	*cs++ = y_offset << 16 | x_offset;
333 	*cs++ = 0;
334 	*cs++ = GT3_INLINE_DATA_DELAYS;
335 
336 	batch_advance(batch, cs);
337 }
338 
339 static void gen7_emit_pipeline_flush(struct batch_chunk *batch)
340 {
341 	u32 *cs = batch_alloc_items(batch, 0, 4);
342 
343 	*cs++ = GFX_OP_PIPE_CONTROL(4);
344 	*cs++ = PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
345 		PIPE_CONTROL_DEPTH_CACHE_FLUSH |
346 		PIPE_CONTROL_DC_FLUSH_ENABLE |
347 		PIPE_CONTROL_CS_STALL;
348 	*cs++ = 0;
349 	*cs++ = 0;
350 
351 	batch_advance(batch, cs);
352 }
353 
354 static void gen7_emit_pipeline_invalidate(struct batch_chunk *batch)
355 {
356 	u32 *cs = batch_alloc_items(batch, 0, 10);
357 
358 	/* ivb: Stall before STATE_CACHE_INVALIDATE */
359 	*cs++ = GFX_OP_PIPE_CONTROL(5);
360 	*cs++ = PIPE_CONTROL_STALL_AT_SCOREBOARD |
361 		PIPE_CONTROL_CS_STALL;
362 	*cs++ = 0;
363 	*cs++ = 0;
364 	*cs++ = 0;
365 
366 	*cs++ = GFX_OP_PIPE_CONTROL(5);
367 	*cs++ = PIPE_CONTROL_STATE_CACHE_INVALIDATE;
368 	*cs++ = 0;
369 	*cs++ = 0;
370 	*cs++ = 0;
371 
372 	batch_advance(batch, cs);
373 }
374 
375 static void emit_batch(struct i915_vma * const vma,
376 		       u32 *start,
377 		       const struct batch_vals *bv)
378 {
379 	struct drm_i915_private *i915 = vma->vm->i915;
380 	const unsigned int desc_count = 1;
381 	const unsigned int urb_size = 1;
382 	struct batch_chunk cmds, state;
383 	u32 descriptors;
384 	unsigned int i;
385 
386 	batch_init(&cmds, vma, start, 0, bv->state_start);
387 	batch_init(&state, vma, start, bv->state_start, SZ_4K);
388 
389 	descriptors = gen7_fill_interface_descriptor(&state, bv,
390 						     IS_HASWELL(i915) ?
391 						     &cb_kernel_hsw :
392 						     &cb_kernel_ivb,
393 						     desc_count);
394 
395 	/* Reset inherited context registers */
396 	gen7_emit_pipeline_flush(&cmds);
397 	gen7_emit_pipeline_invalidate(&cmds);
398 	batch_add(&cmds, MI_LOAD_REGISTER_IMM(2));
399 	batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_0_GEN7));
400 	batch_add(&cmds, 0xffff0000);
401 	batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_1));
402 	batch_add(&cmds, 0xffff0000 | PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
403 	gen7_emit_pipeline_invalidate(&cmds);
404 	gen7_emit_pipeline_flush(&cmds);
405 
406 	/* Switch to the media pipeline and our base address */
407 	gen7_emit_pipeline_invalidate(&cmds);
408 	batch_add(&cmds, PIPELINE_SELECT | PIPELINE_SELECT_MEDIA);
409 	batch_add(&cmds, MI_NOOP);
410 	gen7_emit_pipeline_invalidate(&cmds);
411 
412 	gen7_emit_pipeline_flush(&cmds);
413 	gen7_emit_state_base_address(&cmds, descriptors);
414 	gen7_emit_pipeline_invalidate(&cmds);
415 
416 	/* Set the clear-residual kernel state */
417 	gen7_emit_vfe_state(&cmds, bv, urb_size - 1, 0, 0);
418 	gen7_emit_interface_descriptor_load(&cmds, descriptors, desc_count);
419 
420 	/* Execute the kernel on all HW threads */
421 	for (i = 0; i < num_primitives(bv); i++)
422 		gen7_emit_media_object(&cmds, i);
423 
424 	batch_add(&cmds, MI_BATCH_BUFFER_END);
425 }
426 
427 int gen7_setup_clear_gpr_bb(struct intel_engine_cs * const engine,
428 			    struct i915_vma * const vma)
429 {
430 	struct batch_vals bv;
431 	u32 *batch;
432 
433 	batch_get_defaults(engine->i915, &bv);
434 	if (!vma)
435 		return bv.size;
436 
437 	GEM_BUG_ON(vma->obj->base.size < bv.size);
438 
439 	batch = i915_gem_object_pin_map(vma->obj, I915_MAP_WC);
440 	if (IS_ERR(batch))
441 		return PTR_ERR(batch);
442 
443 	emit_batch(vma, memset(batch, 0, bv.size), &bv);
444 
445 	i915_gem_object_flush_map(vma->obj);
446 	__i915_gem_object_release_map(vma->obj);
447 
448 	return 0;
449 }
450