1 /*
2  * Copyright © 2008-2010 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *    Zou Nan hai <nanhai.zou@intel.com>
26  *    Xiang Hai hao<haihao.xiang@intel.com>
27  *
28  */
29 
30 #include <linux/log2.h>
31 
32 #include "gem/i915_gem_context.h"
33 
34 #include "gen6_ppgtt.h"
35 #include "gen7_renderclear.h"
36 #include "i915_drv.h"
37 #include "i915_trace.h"
38 #include "intel_context.h"
39 #include "intel_gt.h"
40 #include "intel_gt_irq.h"
41 #include "intel_gt_pm_irq.h"
42 #include "intel_reset.h"
43 #include "intel_ring.h"
44 #include "intel_workarounds.h"
45 
46 /* Rough estimate of the typical request size, performing a flush,
47  * set-context and then emitting the batch.
48  */
49 #define LEGACY_REQUEST_SIZE 200
50 
51 static int
52 gen2_render_ring_flush(struct i915_request *rq, u32 mode)
53 {
54 	unsigned int num_store_dw;
55 	u32 cmd, *cs;
56 
57 	cmd = MI_FLUSH;
58 	num_store_dw = 0;
59 	if (mode & EMIT_INVALIDATE)
60 		cmd |= MI_READ_FLUSH;
61 	if (mode & EMIT_FLUSH)
62 		num_store_dw = 4;
63 
64 	cs = intel_ring_begin(rq, 2 + 3 * num_store_dw);
65 	if (IS_ERR(cs))
66 		return PTR_ERR(cs);
67 
68 	*cs++ = cmd;
69 	while (num_store_dw--) {
70 		*cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
71 		*cs++ = intel_gt_scratch_offset(rq->engine->gt,
72 						INTEL_GT_SCRATCH_FIELD_DEFAULT);
73 		*cs++ = 0;
74 	}
75 	*cs++ = MI_FLUSH | MI_NO_WRITE_FLUSH;
76 
77 	intel_ring_advance(rq, cs);
78 
79 	return 0;
80 }
81 
82 static int
83 gen4_render_ring_flush(struct i915_request *rq, u32 mode)
84 {
85 	u32 cmd, *cs;
86 	int i;
87 
88 	/*
89 	 * read/write caches:
90 	 *
91 	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
92 	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
93 	 * also flushed at 2d versus 3d pipeline switches.
94 	 *
95 	 * read-only caches:
96 	 *
97 	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
98 	 * MI_READ_FLUSH is set, and is always flushed on 965.
99 	 *
100 	 * I915_GEM_DOMAIN_COMMAND may not exist?
101 	 *
102 	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
103 	 * invalidated when MI_EXE_FLUSH is set.
104 	 *
105 	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
106 	 * invalidated with every MI_FLUSH.
107 	 *
108 	 * TLBs:
109 	 *
110 	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
111 	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
112 	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
113 	 * are flushed at any MI_FLUSH.
114 	 */
115 
116 	cmd = MI_FLUSH;
117 	if (mode & EMIT_INVALIDATE) {
118 		cmd |= MI_EXE_FLUSH;
119 		if (IS_G4X(rq->i915) || IS_GEN(rq->i915, 5))
120 			cmd |= MI_INVALIDATE_ISP;
121 	}
122 
123 	i = 2;
124 	if (mode & EMIT_INVALIDATE)
125 		i += 20;
126 
127 	cs = intel_ring_begin(rq, i);
128 	if (IS_ERR(cs))
129 		return PTR_ERR(cs);
130 
131 	*cs++ = cmd;
132 
133 	/*
134 	 * A random delay to let the CS invalidate take effect? Without this
135 	 * delay, the GPU relocation path fails as the CS does not see
136 	 * the updated contents. Just as important, if we apply the flushes
137 	 * to the EMIT_FLUSH branch (i.e. immediately after the relocation
138 	 * write and before the invalidate on the next batch), the relocations
139 	 * still fail. This implies that is a delay following invalidation
140 	 * that is required to reset the caches as opposed to a delay to
141 	 * ensure the memory is written.
142 	 */
143 	if (mode & EMIT_INVALIDATE) {
144 		*cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE;
145 		*cs++ = intel_gt_scratch_offset(rq->engine->gt,
146 						INTEL_GT_SCRATCH_FIELD_DEFAULT) |
147 			PIPE_CONTROL_GLOBAL_GTT;
148 		*cs++ = 0;
149 		*cs++ = 0;
150 
151 		for (i = 0; i < 12; i++)
152 			*cs++ = MI_FLUSH;
153 
154 		*cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE;
155 		*cs++ = intel_gt_scratch_offset(rq->engine->gt,
156 						INTEL_GT_SCRATCH_FIELD_DEFAULT) |
157 			PIPE_CONTROL_GLOBAL_GTT;
158 		*cs++ = 0;
159 		*cs++ = 0;
160 	}
161 
162 	*cs++ = cmd;
163 
164 	intel_ring_advance(rq, cs);
165 
166 	return 0;
167 }
168 
169 /*
170  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
171  * implementing two workarounds on gen6.  From section 1.4.7.1
172  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
173  *
174  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
175  * produced by non-pipelined state commands), software needs to first
176  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
177  * 0.
178  *
179  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
180  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
181  *
182  * And the workaround for these two requires this workaround first:
183  *
184  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
185  * BEFORE the pipe-control with a post-sync op and no write-cache
186  * flushes.
187  *
188  * And this last workaround is tricky because of the requirements on
189  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
190  * volume 2 part 1:
191  *
192  *     "1 of the following must also be set:
193  *      - Render Target Cache Flush Enable ([12] of DW1)
194  *      - Depth Cache Flush Enable ([0] of DW1)
195  *      - Stall at Pixel Scoreboard ([1] of DW1)
196  *      - Depth Stall ([13] of DW1)
197  *      - Post-Sync Operation ([13] of DW1)
198  *      - Notify Enable ([8] of DW1)"
199  *
200  * The cache flushes require the workaround flush that triggered this
201  * one, so we can't use it.  Depth stall would trigger the same.
202  * Post-sync nonzero is what triggered this second workaround, so we
203  * can't use that one either.  Notify enable is IRQs, which aren't
204  * really our business.  That leaves only stall at scoreboard.
205  */
206 static int
207 gen6_emit_post_sync_nonzero_flush(struct i915_request *rq)
208 {
209 	u32 scratch_addr =
210 		intel_gt_scratch_offset(rq->engine->gt,
211 					INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH);
212 	u32 *cs;
213 
214 	cs = intel_ring_begin(rq, 6);
215 	if (IS_ERR(cs))
216 		return PTR_ERR(cs);
217 
218 	*cs++ = GFX_OP_PIPE_CONTROL(5);
219 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
220 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
221 	*cs++ = 0; /* low dword */
222 	*cs++ = 0; /* high dword */
223 	*cs++ = MI_NOOP;
224 	intel_ring_advance(rq, cs);
225 
226 	cs = intel_ring_begin(rq, 6);
227 	if (IS_ERR(cs))
228 		return PTR_ERR(cs);
229 
230 	*cs++ = GFX_OP_PIPE_CONTROL(5);
231 	*cs++ = PIPE_CONTROL_QW_WRITE;
232 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
233 	*cs++ = 0;
234 	*cs++ = 0;
235 	*cs++ = MI_NOOP;
236 	intel_ring_advance(rq, cs);
237 
238 	return 0;
239 }
240 
241 static int
242 gen6_render_ring_flush(struct i915_request *rq, u32 mode)
243 {
244 	u32 scratch_addr =
245 		intel_gt_scratch_offset(rq->engine->gt,
246 					INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH);
247 	u32 *cs, flags = 0;
248 	int ret;
249 
250 	/* Force SNB workarounds for PIPE_CONTROL flushes */
251 	ret = gen6_emit_post_sync_nonzero_flush(rq);
252 	if (ret)
253 		return ret;
254 
255 	/* Just flush everything.  Experiments have shown that reducing the
256 	 * number of bits based on the write domains has little performance
257 	 * impact.
258 	 */
259 	if (mode & EMIT_FLUSH) {
260 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
261 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
262 		/*
263 		 * Ensure that any following seqno writes only happen
264 		 * when the render cache is indeed flushed.
265 		 */
266 		flags |= PIPE_CONTROL_CS_STALL;
267 	}
268 	if (mode & EMIT_INVALIDATE) {
269 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
270 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
271 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
272 		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
273 		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
274 		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
275 		/*
276 		 * TLB invalidate requires a post-sync write.
277 		 */
278 		flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
279 	}
280 
281 	cs = intel_ring_begin(rq, 4);
282 	if (IS_ERR(cs))
283 		return PTR_ERR(cs);
284 
285 	*cs++ = GFX_OP_PIPE_CONTROL(4);
286 	*cs++ = flags;
287 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
288 	*cs++ = 0;
289 	intel_ring_advance(rq, cs);
290 
291 	return 0;
292 }
293 
294 static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
295 {
296 	/* First we do the gen6_emit_post_sync_nonzero_flush w/a */
297 	*cs++ = GFX_OP_PIPE_CONTROL(4);
298 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
299 	*cs++ = 0;
300 	*cs++ = 0;
301 
302 	*cs++ = GFX_OP_PIPE_CONTROL(4);
303 	*cs++ = PIPE_CONTROL_QW_WRITE;
304 	*cs++ = intel_gt_scratch_offset(rq->engine->gt,
305 					INTEL_GT_SCRATCH_FIELD_DEFAULT) |
306 		PIPE_CONTROL_GLOBAL_GTT;
307 	*cs++ = 0;
308 
309 	/* Finally we can flush and with it emit the breadcrumb */
310 	*cs++ = GFX_OP_PIPE_CONTROL(4);
311 	*cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
312 		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
313 		 PIPE_CONTROL_DC_FLUSH_ENABLE |
314 		 PIPE_CONTROL_QW_WRITE |
315 		 PIPE_CONTROL_CS_STALL);
316 	*cs++ = i915_request_active_timeline(rq)->hwsp_offset |
317 		PIPE_CONTROL_GLOBAL_GTT;
318 	*cs++ = rq->fence.seqno;
319 
320 	*cs++ = MI_USER_INTERRUPT;
321 	*cs++ = MI_NOOP;
322 
323 	rq->tail = intel_ring_offset(rq, cs);
324 	assert_ring_tail_valid(rq->ring, rq->tail);
325 
326 	return cs;
327 }
328 
329 static int
330 gen7_render_ring_cs_stall_wa(struct i915_request *rq)
331 {
332 	u32 *cs;
333 
334 	cs = intel_ring_begin(rq, 4);
335 	if (IS_ERR(cs))
336 		return PTR_ERR(cs);
337 
338 	*cs++ = GFX_OP_PIPE_CONTROL(4);
339 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
340 	*cs++ = 0;
341 	*cs++ = 0;
342 	intel_ring_advance(rq, cs);
343 
344 	return 0;
345 }
346 
347 static int
348 gen7_render_ring_flush(struct i915_request *rq, u32 mode)
349 {
350 	u32 scratch_addr =
351 		intel_gt_scratch_offset(rq->engine->gt,
352 					INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH);
353 	u32 *cs, flags = 0;
354 
355 	/*
356 	 * Ensure that any following seqno writes only happen when the render
357 	 * cache is indeed flushed.
358 	 *
359 	 * Workaround: 4th PIPE_CONTROL command (except the ones with only
360 	 * read-cache invalidate bits set) must have the CS_STALL bit set. We
361 	 * don't try to be clever and just set it unconditionally.
362 	 */
363 	flags |= PIPE_CONTROL_CS_STALL;
364 
365 	/*
366 	 * CS_STALL suggests at least a post-sync write.
367 	 */
368 	flags |= PIPE_CONTROL_QW_WRITE;
369 	flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
370 
371 	/* Just flush everything.  Experiments have shown that reducing the
372 	 * number of bits based on the write domains has little performance
373 	 * impact.
374 	 */
375 	if (mode & EMIT_FLUSH) {
376 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
377 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
378 		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
379 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
380 	}
381 	if (mode & EMIT_INVALIDATE) {
382 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
383 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
384 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
385 		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
386 		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
387 		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
388 		flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
389 
390 		/* Workaround: we must issue a pipe_control with CS-stall bit
391 		 * set before a pipe_control command that has the state cache
392 		 * invalidate bit set. */
393 		gen7_render_ring_cs_stall_wa(rq);
394 	}
395 
396 	cs = intel_ring_begin(rq, 4);
397 	if (IS_ERR(cs))
398 		return PTR_ERR(cs);
399 
400 	*cs++ = GFX_OP_PIPE_CONTROL(4);
401 	*cs++ = flags;
402 	*cs++ = scratch_addr;
403 	*cs++ = 0;
404 	intel_ring_advance(rq, cs);
405 
406 	return 0;
407 }
408 
409 static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
410 {
411 	*cs++ = GFX_OP_PIPE_CONTROL(4);
412 	*cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
413 		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
414 		 PIPE_CONTROL_DC_FLUSH_ENABLE |
415 		 PIPE_CONTROL_FLUSH_ENABLE |
416 		 PIPE_CONTROL_QW_WRITE |
417 		 PIPE_CONTROL_GLOBAL_GTT_IVB |
418 		 PIPE_CONTROL_CS_STALL);
419 	*cs++ = i915_request_active_timeline(rq)->hwsp_offset;
420 	*cs++ = rq->fence.seqno;
421 
422 	*cs++ = MI_USER_INTERRUPT;
423 	*cs++ = MI_NOOP;
424 
425 	rq->tail = intel_ring_offset(rq, cs);
426 	assert_ring_tail_valid(rq->ring, rq->tail);
427 
428 	return cs;
429 }
430 
431 static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
432 {
433 	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
434 	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
435 
436 	*cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
437 	*cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;
438 	*cs++ = rq->fence.seqno;
439 
440 	*cs++ = MI_USER_INTERRUPT;
441 
442 	rq->tail = intel_ring_offset(rq, cs);
443 	assert_ring_tail_valid(rq->ring, rq->tail);
444 
445 	return cs;
446 }
447 
448 #define GEN7_XCS_WA 32
449 static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
450 {
451 	int i;
452 
453 	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
454 	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
455 
456 	*cs++ = MI_FLUSH_DW | MI_INVALIDATE_TLB |
457 		MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
458 	*cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;
459 	*cs++ = rq->fence.seqno;
460 
461 	for (i = 0; i < GEN7_XCS_WA; i++) {
462 		*cs++ = MI_STORE_DWORD_INDEX;
463 		*cs++ = I915_GEM_HWS_SEQNO_ADDR;
464 		*cs++ = rq->fence.seqno;
465 	}
466 
467 	*cs++ = MI_FLUSH_DW;
468 	*cs++ = 0;
469 	*cs++ = 0;
470 
471 	*cs++ = MI_USER_INTERRUPT;
472 	*cs++ = MI_NOOP;
473 
474 	rq->tail = intel_ring_offset(rq, cs);
475 	assert_ring_tail_valid(rq->ring, rq->tail);
476 
477 	return cs;
478 }
479 #undef GEN7_XCS_WA
480 
481 static void set_hwstam(struct intel_engine_cs *engine, u32 mask)
482 {
483 	/*
484 	 * Keep the render interrupt unmasked as this papers over
485 	 * lost interrupts following a reset.
486 	 */
487 	if (engine->class == RENDER_CLASS) {
488 		if (INTEL_GEN(engine->i915) >= 6)
489 			mask &= ~BIT(0);
490 		else
491 			mask &= ~I915_USER_INTERRUPT;
492 	}
493 
494 	intel_engine_set_hwsp_writemask(engine, mask);
495 }
496 
497 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys)
498 {
499 	u32 addr;
500 
501 	addr = lower_32_bits(phys);
502 	if (INTEL_GEN(engine->i915) >= 4)
503 		addr |= (phys >> 28) & 0xf0;
504 
505 	intel_uncore_write(engine->uncore, HWS_PGA, addr);
506 }
507 
508 static struct page *status_page(struct intel_engine_cs *engine)
509 {
510 	struct drm_i915_gem_object *obj = engine->status_page.vma->obj;
511 
512 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
513 	return sg_page(obj->mm.pages->sgl);
514 }
515 
516 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
517 {
518 	set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine))));
519 	set_hwstam(engine, ~0u);
520 }
521 
522 static void set_hwsp(struct intel_engine_cs *engine, u32 offset)
523 {
524 	i915_reg_t hwsp;
525 
526 	/*
527 	 * The ring status page addresses are no longer next to the rest of
528 	 * the ring registers as of gen7.
529 	 */
530 	if (IS_GEN(engine->i915, 7)) {
531 		switch (engine->id) {
532 		/*
533 		 * No more rings exist on Gen7. Default case is only to shut up
534 		 * gcc switch check warning.
535 		 */
536 		default:
537 			GEM_BUG_ON(engine->id);
538 			/* fallthrough */
539 		case RCS0:
540 			hwsp = RENDER_HWS_PGA_GEN7;
541 			break;
542 		case BCS0:
543 			hwsp = BLT_HWS_PGA_GEN7;
544 			break;
545 		case VCS0:
546 			hwsp = BSD_HWS_PGA_GEN7;
547 			break;
548 		case VECS0:
549 			hwsp = VEBOX_HWS_PGA_GEN7;
550 			break;
551 		}
552 	} else if (IS_GEN(engine->i915, 6)) {
553 		hwsp = RING_HWS_PGA_GEN6(engine->mmio_base);
554 	} else {
555 		hwsp = RING_HWS_PGA(engine->mmio_base);
556 	}
557 
558 	intel_uncore_write(engine->uncore, hwsp, offset);
559 	intel_uncore_posting_read(engine->uncore, hwsp);
560 }
561 
562 static void flush_cs_tlb(struct intel_engine_cs *engine)
563 {
564 	struct drm_i915_private *dev_priv = engine->i915;
565 
566 	if (!IS_GEN_RANGE(dev_priv, 6, 7))
567 		return;
568 
569 	/* ring should be idle before issuing a sync flush*/
570 	drm_WARN_ON(&dev_priv->drm,
571 		    (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
572 
573 	ENGINE_WRITE(engine, RING_INSTPM,
574 		     _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
575 					INSTPM_SYNC_FLUSH));
576 	if (intel_wait_for_register(engine->uncore,
577 				    RING_INSTPM(engine->mmio_base),
578 				    INSTPM_SYNC_FLUSH, 0,
579 				    1000))
580 		DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
581 			  engine->name);
582 }
583 
584 static void ring_setup_status_page(struct intel_engine_cs *engine)
585 {
586 	set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma));
587 	set_hwstam(engine, ~0u);
588 
589 	flush_cs_tlb(engine);
590 }
591 
592 static bool stop_ring(struct intel_engine_cs *engine)
593 {
594 	struct drm_i915_private *dev_priv = engine->i915;
595 
596 	if (INTEL_GEN(dev_priv) > 2) {
597 		ENGINE_WRITE(engine,
598 			     RING_MI_MODE, _MASKED_BIT_ENABLE(STOP_RING));
599 		if (intel_wait_for_register(engine->uncore,
600 					    RING_MI_MODE(engine->mmio_base),
601 					    MODE_IDLE,
602 					    MODE_IDLE,
603 					    1000)) {
604 			DRM_ERROR("%s : timed out trying to stop ring\n",
605 				  engine->name);
606 
607 			/*
608 			 * Sometimes we observe that the idle flag is not
609 			 * set even though the ring is empty. So double
610 			 * check before giving up.
611 			 */
612 			if (ENGINE_READ(engine, RING_HEAD) !=
613 			    ENGINE_READ(engine, RING_TAIL))
614 				return false;
615 		}
616 	}
617 
618 	ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL));
619 
620 	ENGINE_WRITE(engine, RING_HEAD, 0);
621 	ENGINE_WRITE(engine, RING_TAIL, 0);
622 
623 	/* The ring must be empty before it is disabled */
624 	ENGINE_WRITE(engine, RING_CTL, 0);
625 
626 	return (ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) == 0;
627 }
628 
629 static struct i915_address_space *vm_alias(struct i915_address_space *vm)
630 {
631 	if (i915_is_ggtt(vm))
632 		vm = &i915_vm_to_ggtt(vm)->alias->vm;
633 
634 	return vm;
635 }
636 
637 static void set_pp_dir(struct intel_engine_cs *engine)
638 {
639 	struct i915_address_space *vm = vm_alias(engine->gt->vm);
640 
641 	if (vm) {
642 		struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
643 
644 		ENGINE_WRITE(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G);
645 		ENGINE_WRITE(engine, RING_PP_DIR_BASE,
646 			     px_base(ppgtt->pd)->ggtt_offset << 10);
647 	}
648 }
649 
650 static int xcs_resume(struct intel_engine_cs *engine)
651 {
652 	struct drm_i915_private *dev_priv = engine->i915;
653 	struct intel_ring *ring = engine->legacy.ring;
654 	int ret = 0;
655 
656 	ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n",
657 		     ring->head, ring->tail);
658 
659 	intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL);
660 
661 	/* WaClearRingBufHeadRegAtInit:ctg,elk */
662 	if (!stop_ring(engine)) {
663 		/* G45 ring initialization often fails to reset head to zero */
664 		DRM_DEBUG_DRIVER("%s head not reset to zero "
665 				"ctl %08x head %08x tail %08x start %08x\n",
666 				engine->name,
667 				ENGINE_READ(engine, RING_CTL),
668 				ENGINE_READ(engine, RING_HEAD),
669 				ENGINE_READ(engine, RING_TAIL),
670 				ENGINE_READ(engine, RING_START));
671 
672 		if (!stop_ring(engine)) {
673 			DRM_ERROR("failed to set %s head to zero "
674 				  "ctl %08x head %08x tail %08x start %08x\n",
675 				  engine->name,
676 				  ENGINE_READ(engine, RING_CTL),
677 				  ENGINE_READ(engine, RING_HEAD),
678 				  ENGINE_READ(engine, RING_TAIL),
679 				  ENGINE_READ(engine, RING_START));
680 			ret = -EIO;
681 			goto out;
682 		}
683 	}
684 
685 	if (HWS_NEEDS_PHYSICAL(dev_priv))
686 		ring_setup_phys_status_page(engine);
687 	else
688 		ring_setup_status_page(engine);
689 
690 	intel_engine_reset_breadcrumbs(engine);
691 
692 	/* Enforce ordering by reading HEAD register back */
693 	ENGINE_POSTING_READ(engine, RING_HEAD);
694 
695 	/*
696 	 * Initialize the ring. This must happen _after_ we've cleared the ring
697 	 * registers with the above sequence (the readback of the HEAD registers
698 	 * also enforces ordering), otherwise the hw might lose the new ring
699 	 * register values.
700 	 */
701 	ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma));
702 
703 	/* Check that the ring offsets point within the ring! */
704 	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head));
705 	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail));
706 	intel_ring_update_space(ring);
707 
708 	set_pp_dir(engine);
709 
710 	/* First wake the ring up to an empty/idle ring */
711 	ENGINE_WRITE(engine, RING_HEAD, ring->head);
712 	ENGINE_WRITE(engine, RING_TAIL, ring->head);
713 	ENGINE_POSTING_READ(engine, RING_TAIL);
714 
715 	ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID);
716 
717 	/* If the head is still not zero, the ring is dead */
718 	if (intel_wait_for_register(engine->uncore,
719 				    RING_CTL(engine->mmio_base),
720 				    RING_VALID, RING_VALID,
721 				    50)) {
722 		DRM_ERROR("%s initialization failed "
723 			  "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
724 			  engine->name,
725 			  ENGINE_READ(engine, RING_CTL),
726 			  ENGINE_READ(engine, RING_CTL) & RING_VALID,
727 			  ENGINE_READ(engine, RING_HEAD), ring->head,
728 			  ENGINE_READ(engine, RING_TAIL), ring->tail,
729 			  ENGINE_READ(engine, RING_START),
730 			  i915_ggtt_offset(ring->vma));
731 		ret = -EIO;
732 		goto out;
733 	}
734 
735 	if (INTEL_GEN(dev_priv) > 2)
736 		ENGINE_WRITE(engine,
737 			     RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
738 
739 	/* Now awake, let it get started */
740 	if (ring->tail != ring->head) {
741 		ENGINE_WRITE(engine, RING_TAIL, ring->tail);
742 		ENGINE_POSTING_READ(engine, RING_TAIL);
743 	}
744 
745 	/* Papering over lost _interrupts_ immediately following the restart */
746 	intel_engine_signal_breadcrumbs(engine);
747 out:
748 	intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL);
749 
750 	return ret;
751 }
752 
753 static void reset_prepare(struct intel_engine_cs *engine)
754 {
755 	struct intel_uncore *uncore = engine->uncore;
756 	const u32 base = engine->mmio_base;
757 
758 	/*
759 	 * We stop engines, otherwise we might get failed reset and a
760 	 * dead gpu (on elk). Also as modern gpu as kbl can suffer
761 	 * from system hang if batchbuffer is progressing when
762 	 * the reset is issued, regardless of READY_TO_RESET ack.
763 	 * Thus assume it is best to stop engines on all gens
764 	 * where we have a gpu reset.
765 	 *
766 	 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES)
767 	 *
768 	 * WaMediaResetMainRingCleanup:ctg,elk (presumably)
769 	 *
770 	 * FIXME: Wa for more modern gens needs to be validated
771 	 */
772 	ENGINE_TRACE(engine, "\n");
773 
774 	if (intel_engine_stop_cs(engine))
775 		ENGINE_TRACE(engine, "timed out on STOP_RING\n");
776 
777 	intel_uncore_write_fw(uncore,
778 			      RING_HEAD(base),
779 			      intel_uncore_read_fw(uncore, RING_TAIL(base)));
780 	intel_uncore_posting_read_fw(uncore, RING_HEAD(base)); /* paranoia */
781 
782 	intel_uncore_write_fw(uncore, RING_HEAD(base), 0);
783 	intel_uncore_write_fw(uncore, RING_TAIL(base), 0);
784 	intel_uncore_posting_read_fw(uncore, RING_TAIL(base));
785 
786 	/* The ring must be empty before it is disabled */
787 	intel_uncore_write_fw(uncore, RING_CTL(base), 0);
788 
789 	/* Check acts as a post */
790 	if (intel_uncore_read_fw(uncore, RING_HEAD(base)))
791 		ENGINE_TRACE(engine, "ring head [%x] not parked\n",
792 			     intel_uncore_read_fw(uncore, RING_HEAD(base)));
793 }
794 
795 static void reset_rewind(struct intel_engine_cs *engine, bool stalled)
796 {
797 	struct i915_request *pos, *rq;
798 	unsigned long flags;
799 	u32 head;
800 
801 	rq = NULL;
802 	spin_lock_irqsave(&engine->active.lock, flags);
803 	list_for_each_entry(pos, &engine->active.requests, sched.link) {
804 		if (!i915_request_completed(pos)) {
805 			rq = pos;
806 			break;
807 		}
808 	}
809 
810 	/*
811 	 * The guilty request will get skipped on a hung engine.
812 	 *
813 	 * Users of client default contexts do not rely on logical
814 	 * state preserved between batches so it is safe to execute
815 	 * queued requests following the hang. Non default contexts
816 	 * rely on preserved state, so skipping a batch loses the
817 	 * evolution of the state and it needs to be considered corrupted.
818 	 * Executing more queued batches on top of corrupted state is
819 	 * risky. But we take the risk by trying to advance through
820 	 * the queued requests in order to make the client behaviour
821 	 * more predictable around resets, by not throwing away random
822 	 * amount of batches it has prepared for execution. Sophisticated
823 	 * clients can use gem_reset_stats_ioctl and dma fence status
824 	 * (exported via sync_file info ioctl on explicit fences) to observe
825 	 * when it loses the context state and should rebuild accordingly.
826 	 *
827 	 * The context ban, and ultimately the client ban, mechanism are safety
828 	 * valves if client submission ends up resulting in nothing more than
829 	 * subsequent hangs.
830 	 */
831 
832 	if (rq) {
833 		/*
834 		 * Try to restore the logical GPU state to match the
835 		 * continuation of the request queue. If we skip the
836 		 * context/PD restore, then the next request may try to execute
837 		 * assuming that its context is valid and loaded on the GPU and
838 		 * so may try to access invalid memory, prompting repeated GPU
839 		 * hangs.
840 		 *
841 		 * If the request was guilty, we still restore the logical
842 		 * state in case the next request requires it (e.g. the
843 		 * aliasing ppgtt), but skip over the hung batch.
844 		 *
845 		 * If the request was innocent, we try to replay the request
846 		 * with the restored context.
847 		 */
848 		__i915_request_reset(rq, stalled);
849 
850 		GEM_BUG_ON(rq->ring != engine->legacy.ring);
851 		head = rq->head;
852 	} else {
853 		head = engine->legacy.ring->tail;
854 	}
855 	engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head);
856 
857 	spin_unlock_irqrestore(&engine->active.lock, flags);
858 }
859 
860 static void reset_finish(struct intel_engine_cs *engine)
861 {
862 }
863 
864 static int rcs_resume(struct intel_engine_cs *engine)
865 {
866 	struct drm_i915_private *i915 = engine->i915;
867 	struct intel_uncore *uncore = engine->uncore;
868 
869 	/*
870 	 * Disable CONSTANT_BUFFER before it is loaded from the context
871 	 * image. For as it is loaded, it is executed and the stored
872 	 * address may no longer be valid, leading to a GPU hang.
873 	 *
874 	 * This imposes the requirement that userspace reload their
875 	 * CONSTANT_BUFFER on every batch, fortunately a requirement
876 	 * they are already accustomed to from before contexts were
877 	 * enabled.
878 	 */
879 	if (IS_GEN(i915, 4))
880 		intel_uncore_write(uncore, ECOSKPD,
881 			   _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE));
882 
883 	if (IS_GEN_RANGE(i915, 6, 7))
884 		intel_uncore_write(uncore, INSTPM,
885 				   _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
886 
887 	return xcs_resume(engine);
888 }
889 
890 static void reset_cancel(struct intel_engine_cs *engine)
891 {
892 	struct i915_request *request;
893 	unsigned long flags;
894 
895 	spin_lock_irqsave(&engine->active.lock, flags);
896 
897 	/* Mark all submitted requests as skipped. */
898 	list_for_each_entry(request, &engine->active.requests, sched.link) {
899 		i915_request_set_error_once(request, -EIO);
900 		i915_request_mark_complete(request);
901 	}
902 
903 	/* Remaining _unready_ requests will be nop'ed when submitted */
904 
905 	spin_unlock_irqrestore(&engine->active.lock, flags);
906 }
907 
908 static void i9xx_submit_request(struct i915_request *request)
909 {
910 	i915_request_submit(request);
911 	wmb(); /* paranoid flush writes out of the WCB before mmio */
912 
913 	ENGINE_WRITE(request->engine, RING_TAIL,
914 		     intel_ring_set_tail(request->ring, request->tail));
915 }
916 
917 static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs)
918 {
919 	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
920 	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
921 
922 	*cs++ = MI_FLUSH;
923 
924 	*cs++ = MI_STORE_DWORD_INDEX;
925 	*cs++ = I915_GEM_HWS_SEQNO_ADDR;
926 	*cs++ = rq->fence.seqno;
927 
928 	*cs++ = MI_USER_INTERRUPT;
929 	*cs++ = MI_NOOP;
930 
931 	rq->tail = intel_ring_offset(rq, cs);
932 	assert_ring_tail_valid(rq->ring, rq->tail);
933 
934 	return cs;
935 }
936 
937 #define GEN5_WA_STORES 8 /* must be at least 1! */
938 static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs)
939 {
940 	int i;
941 
942 	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
943 	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
944 
945 	*cs++ = MI_FLUSH;
946 
947 	BUILD_BUG_ON(GEN5_WA_STORES < 1);
948 	for (i = 0; i < GEN5_WA_STORES; i++) {
949 		*cs++ = MI_STORE_DWORD_INDEX;
950 		*cs++ = I915_GEM_HWS_SEQNO_ADDR;
951 		*cs++ = rq->fence.seqno;
952 	}
953 
954 	*cs++ = MI_USER_INTERRUPT;
955 
956 	rq->tail = intel_ring_offset(rq, cs);
957 	assert_ring_tail_valid(rq->ring, rq->tail);
958 
959 	return cs;
960 }
961 #undef GEN5_WA_STORES
962 
963 static void
964 gen5_irq_enable(struct intel_engine_cs *engine)
965 {
966 	gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask);
967 }
968 
969 static void
970 gen5_irq_disable(struct intel_engine_cs *engine)
971 {
972 	gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask);
973 }
974 
975 static void
976 i9xx_irq_enable(struct intel_engine_cs *engine)
977 {
978 	engine->i915->irq_mask &= ~engine->irq_enable_mask;
979 	intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask);
980 	intel_uncore_posting_read_fw(engine->uncore, GEN2_IMR);
981 }
982 
983 static void
984 i9xx_irq_disable(struct intel_engine_cs *engine)
985 {
986 	engine->i915->irq_mask |= engine->irq_enable_mask;
987 	intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask);
988 }
989 
990 static void
991 i8xx_irq_enable(struct intel_engine_cs *engine)
992 {
993 	struct drm_i915_private *i915 = engine->i915;
994 
995 	i915->irq_mask &= ~engine->irq_enable_mask;
996 	intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask);
997 	ENGINE_POSTING_READ16(engine, RING_IMR);
998 }
999 
1000 static void
1001 i8xx_irq_disable(struct intel_engine_cs *engine)
1002 {
1003 	struct drm_i915_private *i915 = engine->i915;
1004 
1005 	i915->irq_mask |= engine->irq_enable_mask;
1006 	intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask);
1007 }
1008 
1009 static int
1010 bsd_ring_flush(struct i915_request *rq, u32 mode)
1011 {
1012 	u32 *cs;
1013 
1014 	cs = intel_ring_begin(rq, 2);
1015 	if (IS_ERR(cs))
1016 		return PTR_ERR(cs);
1017 
1018 	*cs++ = MI_FLUSH;
1019 	*cs++ = MI_NOOP;
1020 	intel_ring_advance(rq, cs);
1021 	return 0;
1022 }
1023 
1024 static void
1025 gen6_irq_enable(struct intel_engine_cs *engine)
1026 {
1027 	ENGINE_WRITE(engine, RING_IMR,
1028 		     ~(engine->irq_enable_mask | engine->irq_keep_mask));
1029 
1030 	/* Flush/delay to ensure the RING_IMR is active before the GT IMR */
1031 	ENGINE_POSTING_READ(engine, RING_IMR);
1032 
1033 	gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask);
1034 }
1035 
1036 static void
1037 gen6_irq_disable(struct intel_engine_cs *engine)
1038 {
1039 	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
1040 	gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask);
1041 }
1042 
1043 static void
1044 hsw_vebox_irq_enable(struct intel_engine_cs *engine)
1045 {
1046 	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_enable_mask);
1047 
1048 	/* Flush/delay to ensure the RING_IMR is active before the GT IMR */
1049 	ENGINE_POSTING_READ(engine, RING_IMR);
1050 
1051 	gen6_gt_pm_unmask_irq(engine->gt, engine->irq_enable_mask);
1052 }
1053 
1054 static void
1055 hsw_vebox_irq_disable(struct intel_engine_cs *engine)
1056 {
1057 	ENGINE_WRITE(engine, RING_IMR, ~0);
1058 	gen6_gt_pm_mask_irq(engine->gt, engine->irq_enable_mask);
1059 }
1060 
1061 static int
1062 i965_emit_bb_start(struct i915_request *rq,
1063 		   u64 offset, u32 length,
1064 		   unsigned int dispatch_flags)
1065 {
1066 	u32 *cs;
1067 
1068 	cs = intel_ring_begin(rq, 2);
1069 	if (IS_ERR(cs))
1070 		return PTR_ERR(cs);
1071 
1072 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags &
1073 		I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965);
1074 	*cs++ = offset;
1075 	intel_ring_advance(rq, cs);
1076 
1077 	return 0;
1078 }
1079 
1080 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1081 #define I830_BATCH_LIMIT SZ_256K
1082 #define I830_TLB_ENTRIES (2)
1083 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1084 static int
1085 i830_emit_bb_start(struct i915_request *rq,
1086 		   u64 offset, u32 len,
1087 		   unsigned int dispatch_flags)
1088 {
1089 	u32 *cs, cs_offset =
1090 		intel_gt_scratch_offset(rq->engine->gt,
1091 					INTEL_GT_SCRATCH_FIELD_DEFAULT);
1092 
1093 	GEM_BUG_ON(rq->engine->gt->scratch->size < I830_WA_SIZE);
1094 
1095 	cs = intel_ring_begin(rq, 6);
1096 	if (IS_ERR(cs))
1097 		return PTR_ERR(cs);
1098 
1099 	/* Evict the invalid PTE TLBs */
1100 	*cs++ = COLOR_BLT_CMD | BLT_WRITE_RGBA;
1101 	*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096;
1102 	*cs++ = I830_TLB_ENTRIES << 16 | 4; /* load each page */
1103 	*cs++ = cs_offset;
1104 	*cs++ = 0xdeadbeef;
1105 	*cs++ = MI_NOOP;
1106 	intel_ring_advance(rq, cs);
1107 
1108 	if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
1109 		if (len > I830_BATCH_LIMIT)
1110 			return -ENOSPC;
1111 
1112 		cs = intel_ring_begin(rq, 6 + 2);
1113 		if (IS_ERR(cs))
1114 			return PTR_ERR(cs);
1115 
1116 		/* Blit the batch (which has now all relocs applied) to the
1117 		 * stable batch scratch bo area (so that the CS never
1118 		 * stumbles over its tlb invalidation bug) ...
1119 		 */
1120 		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
1121 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096;
1122 		*cs++ = DIV_ROUND_UP(len, 4096) << 16 | 4096;
1123 		*cs++ = cs_offset;
1124 		*cs++ = 4096;
1125 		*cs++ = offset;
1126 
1127 		*cs++ = MI_FLUSH;
1128 		*cs++ = MI_NOOP;
1129 		intel_ring_advance(rq, cs);
1130 
1131 		/* ... and execute it. */
1132 		offset = cs_offset;
1133 	}
1134 
1135 	cs = intel_ring_begin(rq, 2);
1136 	if (IS_ERR(cs))
1137 		return PTR_ERR(cs);
1138 
1139 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1140 	*cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
1141 		MI_BATCH_NON_SECURE);
1142 	intel_ring_advance(rq, cs);
1143 
1144 	return 0;
1145 }
1146 
1147 static int
1148 i915_emit_bb_start(struct i915_request *rq,
1149 		   u64 offset, u32 len,
1150 		   unsigned int dispatch_flags)
1151 {
1152 	u32 *cs;
1153 
1154 	cs = intel_ring_begin(rq, 2);
1155 	if (IS_ERR(cs))
1156 		return PTR_ERR(cs);
1157 
1158 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1159 	*cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
1160 		MI_BATCH_NON_SECURE);
1161 	intel_ring_advance(rq, cs);
1162 
1163 	return 0;
1164 }
1165 
1166 static void __ring_context_fini(struct intel_context *ce)
1167 {
1168 	i915_vma_put(ce->state);
1169 }
1170 
1171 static void ring_context_destroy(struct kref *ref)
1172 {
1173 	struct intel_context *ce = container_of(ref, typeof(*ce), ref);
1174 
1175 	GEM_BUG_ON(intel_context_is_pinned(ce));
1176 
1177 	if (ce->state)
1178 		__ring_context_fini(ce);
1179 
1180 	intel_context_fini(ce);
1181 	intel_context_free(ce);
1182 }
1183 
1184 static int __context_pin_ppgtt(struct intel_context *ce)
1185 {
1186 	struct i915_address_space *vm;
1187 	int err = 0;
1188 
1189 	vm = vm_alias(ce->vm);
1190 	if (vm)
1191 		err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)));
1192 
1193 	return err;
1194 }
1195 
1196 static void __context_unpin_ppgtt(struct intel_context *ce)
1197 {
1198 	struct i915_address_space *vm;
1199 
1200 	vm = vm_alias(ce->vm);
1201 	if (vm)
1202 		gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm));
1203 }
1204 
1205 static void ring_context_unpin(struct intel_context *ce)
1206 {
1207 	__context_unpin_ppgtt(ce);
1208 }
1209 
1210 static struct i915_vma *
1211 alloc_context_vma(struct intel_engine_cs *engine)
1212 {
1213 	struct drm_i915_private *i915 = engine->i915;
1214 	struct drm_i915_gem_object *obj;
1215 	struct i915_vma *vma;
1216 	int err;
1217 
1218 	obj = i915_gem_object_create_shmem(i915, engine->context_size);
1219 	if (IS_ERR(obj))
1220 		return ERR_CAST(obj);
1221 
1222 	/*
1223 	 * Try to make the context utilize L3 as well as LLC.
1224 	 *
1225 	 * On VLV we don't have L3 controls in the PTEs so we
1226 	 * shouldn't touch the cache level, especially as that
1227 	 * would make the object snooped which might have a
1228 	 * negative performance impact.
1229 	 *
1230 	 * Snooping is required on non-llc platforms in execlist
1231 	 * mode, but since all GGTT accesses use PAT entry 0 we
1232 	 * get snooping anyway regardless of cache_level.
1233 	 *
1234 	 * This is only applicable for Ivy Bridge devices since
1235 	 * later platforms don't have L3 control bits in the PTE.
1236 	 */
1237 	if (IS_IVYBRIDGE(i915))
1238 		i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);
1239 
1240 	if (engine->default_state) {
1241 		void *defaults, *vaddr;
1242 
1243 		vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1244 		if (IS_ERR(vaddr)) {
1245 			err = PTR_ERR(vaddr);
1246 			goto err_obj;
1247 		}
1248 
1249 		defaults = i915_gem_object_pin_map(engine->default_state,
1250 						   I915_MAP_WB);
1251 		if (IS_ERR(defaults)) {
1252 			err = PTR_ERR(defaults);
1253 			goto err_map;
1254 		}
1255 
1256 		memcpy(vaddr, defaults, engine->context_size);
1257 		i915_gem_object_unpin_map(engine->default_state);
1258 
1259 		i915_gem_object_flush_map(obj);
1260 		i915_gem_object_unpin_map(obj);
1261 	}
1262 
1263 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
1264 	if (IS_ERR(vma)) {
1265 		err = PTR_ERR(vma);
1266 		goto err_obj;
1267 	}
1268 
1269 	return vma;
1270 
1271 err_map:
1272 	i915_gem_object_unpin_map(obj);
1273 err_obj:
1274 	i915_gem_object_put(obj);
1275 	return ERR_PTR(err);
1276 }
1277 
1278 static int ring_context_alloc(struct intel_context *ce)
1279 {
1280 	struct intel_engine_cs *engine = ce->engine;
1281 
1282 	/* One ringbuffer to rule them all */
1283 	GEM_BUG_ON(!engine->legacy.ring);
1284 	ce->ring = engine->legacy.ring;
1285 	ce->timeline = intel_timeline_get(engine->legacy.timeline);
1286 
1287 	GEM_BUG_ON(ce->state);
1288 	if (engine->context_size) {
1289 		struct i915_vma *vma;
1290 
1291 		vma = alloc_context_vma(engine);
1292 		if (IS_ERR(vma))
1293 			return PTR_ERR(vma);
1294 
1295 		ce->state = vma;
1296 		if (engine->default_state)
1297 			__set_bit(CONTEXT_VALID_BIT, &ce->flags);
1298 	}
1299 
1300 	return 0;
1301 }
1302 
1303 static int ring_context_pin(struct intel_context *ce)
1304 {
1305 	return __context_pin_ppgtt(ce);
1306 }
1307 
1308 static void ring_context_reset(struct intel_context *ce)
1309 {
1310 	intel_ring_reset(ce->ring, ce->ring->emit);
1311 }
1312 
1313 static const struct intel_context_ops ring_context_ops = {
1314 	.alloc = ring_context_alloc,
1315 
1316 	.pin = ring_context_pin,
1317 	.unpin = ring_context_unpin,
1318 
1319 	.enter = intel_context_enter_engine,
1320 	.exit = intel_context_exit_engine,
1321 
1322 	.reset = ring_context_reset,
1323 	.destroy = ring_context_destroy,
1324 };
1325 
1326 static int load_pd_dir(struct i915_request *rq,
1327 		       const struct i915_ppgtt *ppgtt,
1328 		       u32 valid)
1329 {
1330 	const struct intel_engine_cs * const engine = rq->engine;
1331 	u32 *cs;
1332 
1333 	cs = intel_ring_begin(rq, 12);
1334 	if (IS_ERR(cs))
1335 		return PTR_ERR(cs);
1336 
1337 	*cs++ = MI_LOAD_REGISTER_IMM(1);
1338 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));
1339 	*cs++ = valid;
1340 
1341 	*cs++ = MI_LOAD_REGISTER_IMM(1);
1342 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
1343 	*cs++ = px_base(ppgtt->pd)->ggtt_offset << 10;
1344 
1345 	/* Stall until the page table load is complete? */
1346 	*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1347 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
1348 	*cs++ = intel_gt_scratch_offset(engine->gt,
1349 					INTEL_GT_SCRATCH_FIELD_DEFAULT);
1350 
1351 	*cs++ = MI_LOAD_REGISTER_IMM(1);
1352 	*cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base));
1353 	*cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE);
1354 
1355 	intel_ring_advance(rq, cs);
1356 
1357 	return rq->engine->emit_flush(rq, EMIT_FLUSH);
1358 }
1359 
1360 static inline int mi_set_context(struct i915_request *rq,
1361 				 struct intel_context *ce,
1362 				 u32 flags)
1363 {
1364 	struct drm_i915_private *i915 = rq->i915;
1365 	struct intel_engine_cs *engine = rq->engine;
1366 	enum intel_engine_id id;
1367 	const int num_engines =
1368 		IS_HASWELL(i915) ? RUNTIME_INFO(i915)->num_engines - 1 : 0;
1369 	bool force_restore = false;
1370 	int len;
1371 	u32 *cs;
1372 
1373 	len = 4;
1374 	if (IS_GEN(i915, 7))
1375 		len += 2 + (num_engines ? 4 * num_engines + 6 : 0);
1376 	else if (IS_GEN(i915, 5))
1377 		len += 2;
1378 	if (flags & MI_FORCE_RESTORE) {
1379 		GEM_BUG_ON(flags & MI_RESTORE_INHIBIT);
1380 		flags &= ~MI_FORCE_RESTORE;
1381 		force_restore = true;
1382 		len += 2;
1383 	}
1384 
1385 	cs = intel_ring_begin(rq, len);
1386 	if (IS_ERR(cs))
1387 		return PTR_ERR(cs);
1388 
1389 	/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
1390 	if (IS_GEN(i915, 7)) {
1391 		*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1392 		if (num_engines) {
1393 			struct intel_engine_cs *signaller;
1394 
1395 			*cs++ = MI_LOAD_REGISTER_IMM(num_engines);
1396 			for_each_engine(signaller, engine->gt, id) {
1397 				if (signaller == engine)
1398 					continue;
1399 
1400 				*cs++ = i915_mmio_reg_offset(
1401 					   RING_PSMI_CTL(signaller->mmio_base));
1402 				*cs++ = _MASKED_BIT_ENABLE(
1403 						GEN6_PSMI_SLEEP_MSG_DISABLE);
1404 			}
1405 		}
1406 	} else if (IS_GEN(i915, 5)) {
1407 		/*
1408 		 * This w/a is only listed for pre-production ilk a/b steppings,
1409 		 * but is also mentioned for programming the powerctx. To be
1410 		 * safe, just apply the workaround; we do not use SyncFlush so
1411 		 * this should never take effect and so be a no-op!
1412 		 */
1413 		*cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN;
1414 	}
1415 
1416 	if (force_restore) {
1417 		/*
1418 		 * The HW doesn't handle being told to restore the current
1419 		 * context very well. Quite often it likes goes to go off and
1420 		 * sulk, especially when it is meant to be reloading PP_DIR.
1421 		 * A very simple fix to force the reload is to simply switch
1422 		 * away from the current context and back again.
1423 		 *
1424 		 * Note that the kernel_context will contain random state
1425 		 * following the INHIBIT_RESTORE. We accept this since we
1426 		 * never use the kernel_context state; it is merely a
1427 		 * placeholder we use to flush other contexts.
1428 		 */
1429 		*cs++ = MI_SET_CONTEXT;
1430 		*cs++ = i915_ggtt_offset(engine->kernel_context->state) |
1431 			MI_MM_SPACE_GTT |
1432 			MI_RESTORE_INHIBIT;
1433 	}
1434 
1435 	*cs++ = MI_NOOP;
1436 	*cs++ = MI_SET_CONTEXT;
1437 	*cs++ = i915_ggtt_offset(ce->state) | flags;
1438 	/*
1439 	 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
1440 	 * WaMiSetContext_Hang:snb,ivb,vlv
1441 	 */
1442 	*cs++ = MI_NOOP;
1443 
1444 	if (IS_GEN(i915, 7)) {
1445 		if (num_engines) {
1446 			struct intel_engine_cs *signaller;
1447 			i915_reg_t last_reg = {}; /* keep gcc quiet */
1448 
1449 			*cs++ = MI_LOAD_REGISTER_IMM(num_engines);
1450 			for_each_engine(signaller, engine->gt, id) {
1451 				if (signaller == engine)
1452 					continue;
1453 
1454 				last_reg = RING_PSMI_CTL(signaller->mmio_base);
1455 				*cs++ = i915_mmio_reg_offset(last_reg);
1456 				*cs++ = _MASKED_BIT_DISABLE(
1457 						GEN6_PSMI_SLEEP_MSG_DISABLE);
1458 			}
1459 
1460 			/* Insert a delay before the next switch! */
1461 			*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1462 			*cs++ = i915_mmio_reg_offset(last_reg);
1463 			*cs++ = intel_gt_scratch_offset(engine->gt,
1464 							INTEL_GT_SCRATCH_FIELD_DEFAULT);
1465 			*cs++ = MI_NOOP;
1466 		}
1467 		*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1468 	} else if (IS_GEN(i915, 5)) {
1469 		*cs++ = MI_SUSPEND_FLUSH;
1470 	}
1471 
1472 	intel_ring_advance(rq, cs);
1473 
1474 	return 0;
1475 }
1476 
1477 static int remap_l3_slice(struct i915_request *rq, int slice)
1478 {
1479 	u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice];
1480 	int i;
1481 
1482 	if (!remap_info)
1483 		return 0;
1484 
1485 	cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2);
1486 	if (IS_ERR(cs))
1487 		return PTR_ERR(cs);
1488 
1489 	/*
1490 	 * Note: We do not worry about the concurrent register cacheline hang
1491 	 * here because no other code should access these registers other than
1492 	 * at initialization time.
1493 	 */
1494 	*cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4);
1495 	for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) {
1496 		*cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
1497 		*cs++ = remap_info[i];
1498 	}
1499 	*cs++ = MI_NOOP;
1500 	intel_ring_advance(rq, cs);
1501 
1502 	return 0;
1503 }
1504 
1505 static int remap_l3(struct i915_request *rq)
1506 {
1507 	struct i915_gem_context *ctx = i915_request_gem_context(rq);
1508 	int i, err;
1509 
1510 	if (!ctx || !ctx->remap_slice)
1511 		return 0;
1512 
1513 	for (i = 0; i < MAX_L3_SLICES; i++) {
1514 		if (!(ctx->remap_slice & BIT(i)))
1515 			continue;
1516 
1517 		err = remap_l3_slice(rq, i);
1518 		if (err)
1519 			return err;
1520 	}
1521 
1522 	ctx->remap_slice = 0;
1523 	return 0;
1524 }
1525 
1526 static int switch_mm(struct i915_request *rq, struct i915_address_space *vm)
1527 {
1528 	int ret;
1529 
1530 	if (!vm)
1531 		return 0;
1532 
1533 	ret = rq->engine->emit_flush(rq, EMIT_FLUSH);
1534 	if (ret)
1535 		return ret;
1536 
1537 	/*
1538 	 * Not only do we need a full barrier (post-sync write) after
1539 	 * invalidating the TLBs, but we need to wait a little bit
1540 	 * longer. Whether this is merely delaying us, or the
1541 	 * subsequent flush is a key part of serialising with the
1542 	 * post-sync op, this extra pass appears vital before a
1543 	 * mm switch!
1544 	 */
1545 	ret = load_pd_dir(rq, i915_vm_to_ppgtt(vm), PP_DIR_DCLV_2G);
1546 	if (ret)
1547 		return ret;
1548 
1549 	return rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1550 }
1551 
1552 static int clear_residuals(struct i915_request *rq)
1553 {
1554 	struct intel_engine_cs *engine = rq->engine;
1555 	int ret;
1556 
1557 	ret = switch_mm(rq, vm_alias(engine->kernel_context->vm));
1558 	if (ret)
1559 		return ret;
1560 
1561 	if (engine->kernel_context->state) {
1562 		ret = mi_set_context(rq,
1563 				     engine->kernel_context,
1564 				     MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT);
1565 		if (ret)
1566 			return ret;
1567 	}
1568 
1569 	ret = engine->emit_bb_start(rq,
1570 				    engine->wa_ctx.vma->node.start, 0,
1571 				    0);
1572 	if (ret)
1573 		return ret;
1574 
1575 	ret = engine->emit_flush(rq, EMIT_FLUSH);
1576 	if (ret)
1577 		return ret;
1578 
1579 	/* Always invalidate before the next switch_mm() */
1580 	return engine->emit_flush(rq, EMIT_INVALIDATE);
1581 }
1582 
1583 static int switch_context(struct i915_request *rq)
1584 {
1585 	struct intel_engine_cs *engine = rq->engine;
1586 	struct intel_context *ce = rq->context;
1587 	void **residuals = NULL;
1588 	int ret;
1589 
1590 	GEM_BUG_ON(HAS_EXECLISTS(rq->i915));
1591 
1592 	if (engine->wa_ctx.vma && ce != engine->kernel_context) {
1593 		if (engine->wa_ctx.vma->private != ce) {
1594 			ret = clear_residuals(rq);
1595 			if (ret)
1596 				return ret;
1597 
1598 			residuals = &engine->wa_ctx.vma->private;
1599 		}
1600 	}
1601 
1602 	ret = switch_mm(rq, vm_alias(ce->vm));
1603 	if (ret)
1604 		return ret;
1605 
1606 	if (ce->state) {
1607 		u32 flags;
1608 
1609 		GEM_BUG_ON(engine->id != RCS0);
1610 
1611 		/* For resource streamer on HSW+ and power context elsewhere */
1612 		BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN);
1613 		BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN);
1614 
1615 		flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT;
1616 		if (test_bit(CONTEXT_VALID_BIT, &ce->flags))
1617 			flags |= MI_RESTORE_EXT_STATE_EN;
1618 		else
1619 			flags |= MI_RESTORE_INHIBIT;
1620 
1621 		ret = mi_set_context(rq, ce, flags);
1622 		if (ret)
1623 			return ret;
1624 	}
1625 
1626 	ret = remap_l3(rq);
1627 	if (ret)
1628 		return ret;
1629 
1630 	/*
1631 	 * Now past the point of no return, this request _will_ be emitted.
1632 	 *
1633 	 * Or at least this preamble will be emitted, the request may be
1634 	 * interrupted prior to submitting the user payload. If so, we
1635 	 * still submit the "empty" request in order to preserve global
1636 	 * state tracking such as this, our tracking of the current
1637 	 * dirty context.
1638 	 */
1639 	if (residuals) {
1640 		intel_context_put(*residuals);
1641 		*residuals = intel_context_get(ce);
1642 	}
1643 
1644 	return 0;
1645 }
1646 
1647 static int ring_request_alloc(struct i915_request *request)
1648 {
1649 	int ret;
1650 
1651 	GEM_BUG_ON(!intel_context_is_pinned(request->context));
1652 	GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb);
1653 
1654 	/*
1655 	 * Flush enough space to reduce the likelihood of waiting after
1656 	 * we start building the request - in which case we will just
1657 	 * have to repeat work.
1658 	 */
1659 	request->reserved_space += LEGACY_REQUEST_SIZE;
1660 
1661 	/* Unconditionally invalidate GPU caches and TLBs. */
1662 	ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
1663 	if (ret)
1664 		return ret;
1665 
1666 	ret = switch_context(request);
1667 	if (ret)
1668 		return ret;
1669 
1670 	request->reserved_space -= LEGACY_REQUEST_SIZE;
1671 	return 0;
1672 }
1673 
1674 static void gen6_bsd_submit_request(struct i915_request *request)
1675 {
1676 	struct intel_uncore *uncore = request->engine->uncore;
1677 
1678 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1679 
1680        /* Every tail move must follow the sequence below */
1681 
1682 	/* Disable notification that the ring is IDLE. The GT
1683 	 * will then assume that it is busy and bring it out of rc6.
1684 	 */
1685 	intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
1686 			      _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1687 
1688 	/* Clear the context id. Here be magic! */
1689 	intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0);
1690 
1691 	/* Wait for the ring not to be idle, i.e. for it to wake up. */
1692 	if (__intel_wait_for_register_fw(uncore,
1693 					 GEN6_BSD_SLEEP_PSMI_CONTROL,
1694 					 GEN6_BSD_SLEEP_INDICATOR,
1695 					 0,
1696 					 1000, 0, NULL))
1697 		drm_err(&uncore->i915->drm,
1698 			"timed out waiting for the BSD ring to wake up\n");
1699 
1700 	/* Now that the ring is fully powered up, update the tail */
1701 	i9xx_submit_request(request);
1702 
1703 	/* Let the ring send IDLE messages to the GT again,
1704 	 * and so let it sleep to conserve power when idle.
1705 	 */
1706 	intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
1707 			      _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1708 
1709 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1710 }
1711 
1712 static int mi_flush_dw(struct i915_request *rq, u32 flags)
1713 {
1714 	u32 cmd, *cs;
1715 
1716 	cs = intel_ring_begin(rq, 4);
1717 	if (IS_ERR(cs))
1718 		return PTR_ERR(cs);
1719 
1720 	cmd = MI_FLUSH_DW;
1721 
1722 	/*
1723 	 * We always require a command barrier so that subsequent
1724 	 * commands, such as breadcrumb interrupts, are strictly ordered
1725 	 * wrt the contents of the write cache being flushed to memory
1726 	 * (and thus being coherent from the CPU).
1727 	 */
1728 	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1729 
1730 	/*
1731 	 * Bspec vol 1c.3 - blitter engine command streamer:
1732 	 * "If ENABLED, all TLBs will be invalidated once the flush
1733 	 * operation is complete. This bit is only valid when the
1734 	 * Post-Sync Operation field is a value of 1h or 3h."
1735 	 */
1736 	cmd |= flags;
1737 
1738 	*cs++ = cmd;
1739 	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
1740 	*cs++ = 0;
1741 	*cs++ = MI_NOOP;
1742 
1743 	intel_ring_advance(rq, cs);
1744 
1745 	return 0;
1746 }
1747 
1748 static int gen6_flush_dw(struct i915_request *rq, u32 mode, u32 invflags)
1749 {
1750 	return mi_flush_dw(rq, mode & EMIT_INVALIDATE ? invflags : 0);
1751 }
1752 
1753 static int gen6_bsd_ring_flush(struct i915_request *rq, u32 mode)
1754 {
1755 	return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB | MI_INVALIDATE_BSD);
1756 }
1757 
1758 static int
1759 hsw_emit_bb_start(struct i915_request *rq,
1760 		  u64 offset, u32 len,
1761 		  unsigned int dispatch_flags)
1762 {
1763 	u32 *cs;
1764 
1765 	cs = intel_ring_begin(rq, 2);
1766 	if (IS_ERR(cs))
1767 		return PTR_ERR(cs);
1768 
1769 	*cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?
1770 		0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW);
1771 	/* bit0-7 is the length on GEN6+ */
1772 	*cs++ = offset;
1773 	intel_ring_advance(rq, cs);
1774 
1775 	return 0;
1776 }
1777 
1778 static int
1779 gen6_emit_bb_start(struct i915_request *rq,
1780 		   u64 offset, u32 len,
1781 		   unsigned int dispatch_flags)
1782 {
1783 	u32 *cs;
1784 
1785 	cs = intel_ring_begin(rq, 2);
1786 	if (IS_ERR(cs))
1787 		return PTR_ERR(cs);
1788 
1789 	*cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?
1790 		0 : MI_BATCH_NON_SECURE_I965);
1791 	/* bit0-7 is the length on GEN6+ */
1792 	*cs++ = offset;
1793 	intel_ring_advance(rq, cs);
1794 
1795 	return 0;
1796 }
1797 
1798 /* Blitter support (SandyBridge+) */
1799 
1800 static int gen6_ring_flush(struct i915_request *rq, u32 mode)
1801 {
1802 	return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB);
1803 }
1804 
1805 static void i9xx_set_default_submission(struct intel_engine_cs *engine)
1806 {
1807 	engine->submit_request = i9xx_submit_request;
1808 
1809 	engine->park = NULL;
1810 	engine->unpark = NULL;
1811 }
1812 
1813 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
1814 {
1815 	i9xx_set_default_submission(engine);
1816 	engine->submit_request = gen6_bsd_submit_request;
1817 }
1818 
1819 static void ring_release(struct intel_engine_cs *engine)
1820 {
1821 	struct drm_i915_private *dev_priv = engine->i915;
1822 
1823 	drm_WARN_ON(&dev_priv->drm, INTEL_GEN(dev_priv) > 2 &&
1824 		    (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
1825 
1826 	intel_engine_cleanup_common(engine);
1827 
1828 	if (engine->wa_ctx.vma) {
1829 		intel_context_put(engine->wa_ctx.vma->private);
1830 		i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);
1831 	}
1832 
1833 	intel_ring_unpin(engine->legacy.ring);
1834 	intel_ring_put(engine->legacy.ring);
1835 
1836 	intel_timeline_unpin(engine->legacy.timeline);
1837 	intel_timeline_put(engine->legacy.timeline);
1838 }
1839 
1840 static void setup_irq(struct intel_engine_cs *engine)
1841 {
1842 	struct drm_i915_private *i915 = engine->i915;
1843 
1844 	if (INTEL_GEN(i915) >= 6) {
1845 		engine->irq_enable = gen6_irq_enable;
1846 		engine->irq_disable = gen6_irq_disable;
1847 	} else if (INTEL_GEN(i915) >= 5) {
1848 		engine->irq_enable = gen5_irq_enable;
1849 		engine->irq_disable = gen5_irq_disable;
1850 	} else if (INTEL_GEN(i915) >= 3) {
1851 		engine->irq_enable = i9xx_irq_enable;
1852 		engine->irq_disable = i9xx_irq_disable;
1853 	} else {
1854 		engine->irq_enable = i8xx_irq_enable;
1855 		engine->irq_disable = i8xx_irq_disable;
1856 	}
1857 }
1858 
1859 static void setup_common(struct intel_engine_cs *engine)
1860 {
1861 	struct drm_i915_private *i915 = engine->i915;
1862 
1863 	/* gen8+ are only supported with execlists */
1864 	GEM_BUG_ON(INTEL_GEN(i915) >= 8);
1865 
1866 	setup_irq(engine);
1867 
1868 	engine->resume = xcs_resume;
1869 	engine->reset.prepare = reset_prepare;
1870 	engine->reset.rewind = reset_rewind;
1871 	engine->reset.cancel = reset_cancel;
1872 	engine->reset.finish = reset_finish;
1873 
1874 	engine->cops = &ring_context_ops;
1875 	engine->request_alloc = ring_request_alloc;
1876 
1877 	/*
1878 	 * Using a global execution timeline; the previous final breadcrumb is
1879 	 * equivalent to our next initial bread so we can elide
1880 	 * engine->emit_init_breadcrumb().
1881 	 */
1882 	engine->emit_fini_breadcrumb = i9xx_emit_breadcrumb;
1883 	if (IS_GEN(i915, 5))
1884 		engine->emit_fini_breadcrumb = gen5_emit_breadcrumb;
1885 
1886 	engine->set_default_submission = i9xx_set_default_submission;
1887 
1888 	if (INTEL_GEN(i915) >= 6)
1889 		engine->emit_bb_start = gen6_emit_bb_start;
1890 	else if (INTEL_GEN(i915) >= 4)
1891 		engine->emit_bb_start = i965_emit_bb_start;
1892 	else if (IS_I830(i915) || IS_I845G(i915))
1893 		engine->emit_bb_start = i830_emit_bb_start;
1894 	else
1895 		engine->emit_bb_start = i915_emit_bb_start;
1896 }
1897 
1898 static void setup_rcs(struct intel_engine_cs *engine)
1899 {
1900 	struct drm_i915_private *i915 = engine->i915;
1901 
1902 	if (HAS_L3_DPF(i915))
1903 		engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1904 
1905 	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
1906 
1907 	if (INTEL_GEN(i915) >= 7) {
1908 		engine->emit_flush = gen7_render_ring_flush;
1909 		engine->emit_fini_breadcrumb = gen7_rcs_emit_breadcrumb;
1910 	} else if (IS_GEN(i915, 6)) {
1911 		engine->emit_flush = gen6_render_ring_flush;
1912 		engine->emit_fini_breadcrumb = gen6_rcs_emit_breadcrumb;
1913 	} else if (IS_GEN(i915, 5)) {
1914 		engine->emit_flush = gen4_render_ring_flush;
1915 	} else {
1916 		if (INTEL_GEN(i915) < 4)
1917 			engine->emit_flush = gen2_render_ring_flush;
1918 		else
1919 			engine->emit_flush = gen4_render_ring_flush;
1920 		engine->irq_enable_mask = I915_USER_INTERRUPT;
1921 	}
1922 
1923 	if (IS_HASWELL(i915))
1924 		engine->emit_bb_start = hsw_emit_bb_start;
1925 
1926 	engine->resume = rcs_resume;
1927 }
1928 
1929 static void setup_vcs(struct intel_engine_cs *engine)
1930 {
1931 	struct drm_i915_private *i915 = engine->i915;
1932 
1933 	if (INTEL_GEN(i915) >= 6) {
1934 		/* gen6 bsd needs a special wa for tail updates */
1935 		if (IS_GEN(i915, 6))
1936 			engine->set_default_submission = gen6_bsd_set_default_submission;
1937 		engine->emit_flush = gen6_bsd_ring_flush;
1938 		engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1939 
1940 		if (IS_GEN(i915, 6))
1941 			engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb;
1942 		else
1943 			engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb;
1944 	} else {
1945 		engine->emit_flush = bsd_ring_flush;
1946 		if (IS_GEN(i915, 5))
1947 			engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
1948 		else
1949 			engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1950 	}
1951 }
1952 
1953 static void setup_bcs(struct intel_engine_cs *engine)
1954 {
1955 	struct drm_i915_private *i915 = engine->i915;
1956 
1957 	engine->emit_flush = gen6_ring_flush;
1958 	engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
1959 
1960 	if (IS_GEN(i915, 6))
1961 		engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb;
1962 	else
1963 		engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb;
1964 }
1965 
1966 static void setup_vecs(struct intel_engine_cs *engine)
1967 {
1968 	struct drm_i915_private *i915 = engine->i915;
1969 
1970 	GEM_BUG_ON(INTEL_GEN(i915) < 7);
1971 
1972 	engine->emit_flush = gen6_ring_flush;
1973 	engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
1974 	engine->irq_enable = hsw_vebox_irq_enable;
1975 	engine->irq_disable = hsw_vebox_irq_disable;
1976 
1977 	engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb;
1978 }
1979 
1980 static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine,
1981 				    struct i915_vma * const vma)
1982 {
1983 	return gen7_setup_clear_gpr_bb(engine, vma);
1984 }
1985 
1986 static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)
1987 {
1988 	struct drm_i915_gem_object *obj;
1989 	struct i915_vma *vma;
1990 	int size;
1991 	int err;
1992 
1993 	size = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);
1994 	if (size <= 0)
1995 		return size;
1996 
1997 	size = ALIGN(size, PAGE_SIZE);
1998 	obj = i915_gem_object_create_internal(engine->i915, size);
1999 	if (IS_ERR(obj))
2000 		return PTR_ERR(obj);
2001 
2002 	vma = i915_vma_instance(obj, engine->gt->vm, NULL);
2003 	if (IS_ERR(vma)) {
2004 		err = PTR_ERR(vma);
2005 		goto err_obj;
2006 	}
2007 
2008 	vma->private = intel_context_create(engine); /* dummy residuals */
2009 	if (IS_ERR(vma->private)) {
2010 		err = PTR_ERR(vma->private);
2011 		goto err_obj;
2012 	}
2013 
2014 	err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
2015 	if (err)
2016 		goto err_private;
2017 
2018 	err = i915_vma_sync(vma);
2019 	if (err)
2020 		goto err_unpin;
2021 
2022 	err = gen7_ctx_switch_bb_setup(engine, vma);
2023 	if (err)
2024 		goto err_unpin;
2025 
2026 	engine->wa_ctx.vma = vma;
2027 	return 0;
2028 
2029 err_unpin:
2030 	i915_vma_unpin(vma);
2031 err_private:
2032 	intel_context_put(vma->private);
2033 err_obj:
2034 	i915_gem_object_put(obj);
2035 	return err;
2036 }
2037 
2038 int intel_ring_submission_setup(struct intel_engine_cs *engine)
2039 {
2040 	struct intel_timeline *timeline;
2041 	struct intel_ring *ring;
2042 	int err;
2043 
2044 	setup_common(engine);
2045 
2046 	switch (engine->class) {
2047 	case RENDER_CLASS:
2048 		setup_rcs(engine);
2049 		break;
2050 	case VIDEO_DECODE_CLASS:
2051 		setup_vcs(engine);
2052 		break;
2053 	case COPY_ENGINE_CLASS:
2054 		setup_bcs(engine);
2055 		break;
2056 	case VIDEO_ENHANCEMENT_CLASS:
2057 		setup_vecs(engine);
2058 		break;
2059 	default:
2060 		MISSING_CASE(engine->class);
2061 		return -ENODEV;
2062 	}
2063 
2064 	timeline = intel_timeline_create(engine->gt, engine->status_page.vma);
2065 	if (IS_ERR(timeline)) {
2066 		err = PTR_ERR(timeline);
2067 		goto err;
2068 	}
2069 	GEM_BUG_ON(timeline->has_initial_breadcrumb);
2070 
2071 	err = intel_timeline_pin(timeline);
2072 	if (err)
2073 		goto err_timeline;
2074 
2075 	ring = intel_engine_create_ring(engine, SZ_16K);
2076 	if (IS_ERR(ring)) {
2077 		err = PTR_ERR(ring);
2078 		goto err_timeline_unpin;
2079 	}
2080 
2081 	err = intel_ring_pin(ring);
2082 	if (err)
2083 		goto err_ring;
2084 
2085 	GEM_BUG_ON(engine->legacy.ring);
2086 	engine->legacy.ring = ring;
2087 	engine->legacy.timeline = timeline;
2088 
2089 	GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
2090 
2091 	if (IS_HASWELL(engine->i915) && engine->class == RENDER_CLASS) {
2092 		err = gen7_ctx_switch_bb_init(engine);
2093 		if (err)
2094 			goto err_ring_unpin;
2095 	}
2096 
2097 	/* Finally, take ownership and responsibility for cleanup! */
2098 	engine->release = ring_release;
2099 
2100 	return 0;
2101 
2102 err_ring_unpin:
2103 	intel_ring_unpin(ring);
2104 err_ring:
2105 	intel_ring_put(ring);
2106 err_timeline_unpin:
2107 	intel_timeline_unpin(timeline);
2108 err_timeline:
2109 	intel_timeline_put(timeline);
2110 err:
2111 	intel_engine_cleanup_common(engine);
2112 	return err;
2113 }
2114 
2115 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2116 #include "selftest_ring_submission.c"
2117 #endif
2118