xref: /openbmc/linux/drivers/gpu/drm/i915/gt/intel_gt.c (revision 1d1997db) 
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 #include "intel_gt.h"
8 #include "intel_gt_pm.h"
9 #include "intel_gt_requests.h"
10 #include "intel_mocs.h"
11 #include "intel_rc6.h"
12 #include "intel_rps.h"
13 #include "intel_uncore.h"
14 #include "intel_pm.h"
15 
16 void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915)
17 {
18 	gt->i915 = i915;
19 	gt->uncore = &i915->uncore;
20 
21 	spin_lock_init(&gt->irq_lock);
22 
23 	INIT_LIST_HEAD(&gt->closed_vma);
24 	spin_lock_init(&gt->closed_lock);
25 
26 	intel_gt_init_reset(gt);
27 	intel_gt_init_requests(gt);
28 	intel_gt_pm_init_early(gt);
29 
30 	intel_rps_init_early(&gt->rps);
31 	intel_uc_init_early(&gt->uc);
32 }
33 
34 void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt)
35 {
36 	gt->ggtt = ggtt;
37 
38 	intel_gt_sanitize(gt, false);
39 }
40 
41 static void init_unused_ring(struct intel_gt *gt, u32 base)
42 {
43 	struct intel_uncore *uncore = gt->uncore;
44 
45 	intel_uncore_write(uncore, RING_CTL(base), 0);
46 	intel_uncore_write(uncore, RING_HEAD(base), 0);
47 	intel_uncore_write(uncore, RING_TAIL(base), 0);
48 	intel_uncore_write(uncore, RING_START(base), 0);
49 }
50 
51 static void init_unused_rings(struct intel_gt *gt)
52 {
53 	struct drm_i915_private *i915 = gt->i915;
54 
55 	if (IS_I830(i915)) {
56 		init_unused_ring(gt, PRB1_BASE);
57 		init_unused_ring(gt, SRB0_BASE);
58 		init_unused_ring(gt, SRB1_BASE);
59 		init_unused_ring(gt, SRB2_BASE);
60 		init_unused_ring(gt, SRB3_BASE);
61 	} else if (IS_GEN(i915, 2)) {
62 		init_unused_ring(gt, SRB0_BASE);
63 		init_unused_ring(gt, SRB1_BASE);
64 	} else if (IS_GEN(i915, 3)) {
65 		init_unused_ring(gt, PRB1_BASE);
66 		init_unused_ring(gt, PRB2_BASE);
67 	}
68 }
69 
70 int intel_gt_init_hw(struct intel_gt *gt)
71 {
72 	struct drm_i915_private *i915 = gt->i915;
73 	struct intel_uncore *uncore = gt->uncore;
74 	int ret;
75 
76 	BUG_ON(!i915->kernel_context);
77 	ret = intel_gt_terminally_wedged(gt);
78 	if (ret)
79 		return ret;
80 
81 	gt->last_init_time = ktime_get();
82 
83 	/* Double layer security blanket, see i915_gem_init() */
84 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
85 
86 	if (HAS_EDRAM(i915) && INTEL_GEN(i915) < 9)
87 		intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));
88 
89 	if (IS_HASWELL(i915))
90 		intel_uncore_write(uncore,
91 				   MI_PREDICATE_RESULT_2,
92 				   IS_HSW_GT3(i915) ?
93 				   LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
94 
95 	/* Apply the GT workarounds... */
96 	intel_gt_apply_workarounds(gt);
97 	/* ...and determine whether they are sticking. */
98 	intel_gt_verify_workarounds(gt, "init");
99 
100 	intel_gt_init_swizzling(gt);
101 
102 	/*
103 	 * At least 830 can leave some of the unused rings
104 	 * "active" (ie. head != tail) after resume which
105 	 * will prevent c3 entry. Makes sure all unused rings
106 	 * are totally idle.
107 	 */
108 	init_unused_rings(gt);
109 
110 	ret = i915_ppgtt_init_hw(gt);
111 	if (ret) {
112 		DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
113 		goto out;
114 	}
115 
116 	/* We can't enable contexts until all firmware is loaded */
117 	ret = intel_uc_init_hw(&gt->uc);
118 	if (ret) {
119 		i915_probe_error(i915, "Enabling uc failed (%d)\n", ret);
120 		goto out;
121 	}
122 
123 	intel_mocs_init(gt);
124 
125 out:
126 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
127 	return ret;
128 }
129 
130 static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
131 {
132 	intel_uncore_rmw(uncore, reg, 0, set);
133 }
134 
135 static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
136 {
137 	intel_uncore_rmw(uncore, reg, clr, 0);
138 }
139 
140 static void clear_register(struct intel_uncore *uncore, i915_reg_t reg)
141 {
142 	intel_uncore_rmw(uncore, reg, 0, 0);
143 }
144 
145 static void gen8_clear_engine_error_register(struct intel_engine_cs *engine)
146 {
147 	GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);
148 	GEN6_RING_FAULT_REG_POSTING_READ(engine);
149 }
150 
151 void
152 intel_gt_clear_error_registers(struct intel_gt *gt,
153 			       intel_engine_mask_t engine_mask)
154 {
155 	struct drm_i915_private *i915 = gt->i915;
156 	struct intel_uncore *uncore = gt->uncore;
157 	u32 eir;
158 
159 	if (!IS_GEN(i915, 2))
160 		clear_register(uncore, PGTBL_ER);
161 
162 	if (INTEL_GEN(i915) < 4)
163 		clear_register(uncore, IPEIR(RENDER_RING_BASE));
164 	else
165 		clear_register(uncore, IPEIR_I965);
166 
167 	clear_register(uncore, EIR);
168 	eir = intel_uncore_read(uncore, EIR);
169 	if (eir) {
170 		/*
171 		 * some errors might have become stuck,
172 		 * mask them.
173 		 */
174 		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
175 		rmw_set(uncore, EMR, eir);
176 		intel_uncore_write(uncore, GEN2_IIR,
177 				   I915_MASTER_ERROR_INTERRUPT);
178 	}
179 
180 	if (INTEL_GEN(i915) >= 12) {
181 		rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID);
182 		intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);
183 	} else if (INTEL_GEN(i915) >= 8) {
184 		rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID);
185 		intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
186 	} else if (INTEL_GEN(i915) >= 6) {
187 		struct intel_engine_cs *engine;
188 		enum intel_engine_id id;
189 
190 		for_each_engine_masked(engine, gt, engine_mask, id)
191 			gen8_clear_engine_error_register(engine);
192 	}
193 }
194 
195 static void gen6_check_faults(struct intel_gt *gt)
196 {
197 	struct intel_engine_cs *engine;
198 	enum intel_engine_id id;
199 	u32 fault;
200 
201 	for_each_engine(engine, gt, id) {
202 		fault = GEN6_RING_FAULT_REG_READ(engine);
203 		if (fault & RING_FAULT_VALID) {
204 			DRM_DEBUG_DRIVER("Unexpected fault\n"
205 					 "\tAddr: 0x%08lx\n"
206 					 "\tAddress space: %s\n"
207 					 "\tSource ID: %d\n"
208 					 "\tType: %d\n",
209 					 fault & PAGE_MASK,
210 					 fault & RING_FAULT_GTTSEL_MASK ?
211 					 "GGTT" : "PPGTT",
212 					 RING_FAULT_SRCID(fault),
213 					 RING_FAULT_FAULT_TYPE(fault));
214 		}
215 	}
216 }
217 
218 static void gen8_check_faults(struct intel_gt *gt)
219 {
220 	struct intel_uncore *uncore = gt->uncore;
221 	i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;
222 	u32 fault;
223 
224 	if (INTEL_GEN(gt->i915) >= 12) {
225 		fault_reg = GEN12_RING_FAULT_REG;
226 		fault_data0_reg = GEN12_FAULT_TLB_DATA0;
227 		fault_data1_reg = GEN12_FAULT_TLB_DATA1;
228 	} else {
229 		fault_reg = GEN8_RING_FAULT_REG;
230 		fault_data0_reg = GEN8_FAULT_TLB_DATA0;
231 		fault_data1_reg = GEN8_FAULT_TLB_DATA1;
232 	}
233 
234 	fault = intel_uncore_read(uncore, fault_reg);
235 	if (fault & RING_FAULT_VALID) {
236 		u32 fault_data0, fault_data1;
237 		u64 fault_addr;
238 
239 		fault_data0 = intel_uncore_read(uncore, fault_data0_reg);
240 		fault_data1 = intel_uncore_read(uncore, fault_data1_reg);
241 
242 		fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
243 			     ((u64)fault_data0 << 12);
244 
245 		DRM_DEBUG_DRIVER("Unexpected fault\n"
246 				 "\tAddr: 0x%08x_%08x\n"
247 				 "\tAddress space: %s\n"
248 				 "\tEngine ID: %d\n"
249 				 "\tSource ID: %d\n"
250 				 "\tType: %d\n",
251 				 upper_32_bits(fault_addr),
252 				 lower_32_bits(fault_addr),
253 				 fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
254 				 GEN8_RING_FAULT_ENGINE_ID(fault),
255 				 RING_FAULT_SRCID(fault),
256 				 RING_FAULT_FAULT_TYPE(fault));
257 	}
258 }
259 
260 void intel_gt_check_and_clear_faults(struct intel_gt *gt)
261 {
262 	struct drm_i915_private *i915 = gt->i915;
263 
264 	/* From GEN8 onwards we only have one 'All Engine Fault Register' */
265 	if (INTEL_GEN(i915) >= 8)
266 		gen8_check_faults(gt);
267 	else if (INTEL_GEN(i915) >= 6)
268 		gen6_check_faults(gt);
269 	else
270 		return;
271 
272 	intel_gt_clear_error_registers(gt, ALL_ENGINES);
273 }
274 
275 void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
276 {
277 	struct intel_uncore *uncore = gt->uncore;
278 	intel_wakeref_t wakeref;
279 
280 	/*
281 	 * No actual flushing is required for the GTT write domain for reads
282 	 * from the GTT domain. Writes to it "immediately" go to main memory
283 	 * as far as we know, so there's no chipset flush. It also doesn't
284 	 * land in the GPU render cache.
285 	 *
286 	 * However, we do have to enforce the order so that all writes through
287 	 * the GTT land before any writes to the device, such as updates to
288 	 * the GATT itself.
289 	 *
290 	 * We also have to wait a bit for the writes to land from the GTT.
291 	 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
292 	 * timing. This issue has only been observed when switching quickly
293 	 * between GTT writes and CPU reads from inside the kernel on recent hw,
294 	 * and it appears to only affect discrete GTT blocks (i.e. on LLC
295 	 * system agents we cannot reproduce this behaviour, until Cannonlake
296 	 * that was!).
297 	 */
298 
299 	wmb();
300 
301 	if (INTEL_INFO(gt->i915)->has_coherent_ggtt)
302 		return;
303 
304 	intel_gt_chipset_flush(gt);
305 
306 	with_intel_runtime_pm(uncore->rpm, wakeref) {
307 		unsigned long flags;
308 
309 		spin_lock_irqsave(&uncore->lock, flags);
310 		intel_uncore_posting_read_fw(uncore,
311 					     RING_HEAD(RENDER_RING_BASE));
312 		spin_unlock_irqrestore(&uncore->lock, flags);
313 	}
314 }
315 
316 void intel_gt_chipset_flush(struct intel_gt *gt)
317 {
318 	wmb();
319 	if (INTEL_GEN(gt->i915) < 6)
320 		intel_gtt_chipset_flush();
321 }
322 
323 void intel_gt_driver_register(struct intel_gt *gt)
324 {
325 	intel_rps_driver_register(&gt->rps);
326 }
327 
328 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
329 {
330 	struct drm_i915_private *i915 = gt->i915;
331 	struct drm_i915_gem_object *obj;
332 	struct i915_vma *vma;
333 	int ret;
334 
335 	obj = i915_gem_object_create_stolen(i915, size);
336 	if (IS_ERR(obj))
337 		obj = i915_gem_object_create_internal(i915, size);
338 	if (IS_ERR(obj)) {
339 		DRM_ERROR("Failed to allocate scratch page\n");
340 		return PTR_ERR(obj);
341 	}
342 
343 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
344 	if (IS_ERR(vma)) {
345 		ret = PTR_ERR(vma);
346 		goto err_unref;
347 	}
348 
349 	ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
350 	if (ret)
351 		goto err_unref;
352 
353 	gt->scratch = i915_vma_make_unshrinkable(vma);
354 
355 	return 0;
356 
357 err_unref:
358 	i915_gem_object_put(obj);
359 	return ret;
360 }
361 
362 static void intel_gt_fini_scratch(struct intel_gt *gt)
363 {
364 	i915_vma_unpin_and_release(&gt->scratch, 0);
365 }
366 
367 int intel_gt_init(struct intel_gt *gt)
368 {
369 	int err;
370 
371 	err = intel_gt_init_scratch(gt, IS_GEN(gt->i915, 2) ? SZ_256K : SZ_4K);
372 	if (err)
373 		return err;
374 
375 	intel_gt_pm_init(gt);
376 
377 	return 0;
378 }
379 
380 void intel_gt_driver_remove(struct intel_gt *gt)
381 {
382 	GEM_BUG_ON(gt->awake);
383 }
384 
385 void intel_gt_driver_unregister(struct intel_gt *gt)
386 {
387 	intel_rps_driver_unregister(&gt->rps);
388 }
389 
390 void intel_gt_driver_release(struct intel_gt *gt)
391 {
392 	intel_gt_pm_fini(gt);
393 	intel_gt_fini_scratch(gt);
394 }
395 
396 void intel_gt_driver_late_release(struct intel_gt *gt)
397 {
398 	intel_uc_driver_late_release(&gt->uc);
399 	intel_gt_fini_reset(gt);
400 }
401