1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2016-2019 Intel Corporation
4  */
5 
6 #include <linux/circ_buf.h>
7 #include <linux/ktime.h>
8 #include <linux/time64.h>
9 #include <linux/string_helpers.h>
10 #include <linux/timekeeping.h>
11 
12 #include "i915_drv.h"
13 #include "intel_guc_ct.h"
14 #include "intel_guc_print.h"
15 
16 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
17 enum {
18 	CT_DEAD_ALIVE = 0,
19 	CT_DEAD_SETUP,
20 	CT_DEAD_WRITE,
21 	CT_DEAD_DEADLOCK,
22 	CT_DEAD_H2G_HAS_ROOM,
23 	CT_DEAD_READ,
24 	CT_DEAD_PROCESS_FAILED,
25 };
26 
27 static void ct_dead_ct_worker_func(struct work_struct *w);
28 
29 #define CT_DEAD(ct, reason)	\
30 	do { \
31 		if (!(ct)->dead_ct_reported) { \
32 			(ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \
33 			queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \
34 		} \
35 	} while (0)
36 #else
37 #define CT_DEAD(ct, reason)	do { } while (0)
38 #endif
39 
ct_to_guc(struct intel_guc_ct * ct)40 static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
41 {
42 	return container_of(ct, struct intel_guc, ct);
43 }
44 
45 #define CT_ERROR(_ct, _fmt, ...) \
46 	guc_err(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
47 #ifdef CONFIG_DRM_I915_DEBUG_GUC
48 #define CT_DEBUG(_ct, _fmt, ...) \
49 	guc_dbg(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
50 #else
51 #define CT_DEBUG(...)	do { } while (0)
52 #endif
53 #define CT_PROBE_ERROR(_ct, _fmt, ...) \
54 	guc_probe_error(ct_to_guc(ct), "CT: " _fmt, ##__VA_ARGS__)
55 
56 /**
57  * DOC: CTB Blob
58  *
59  * We allocate single blob to hold both CTB descriptors and buffers:
60  *
61  *      +--------+-----------------------------------------------+------+
62  *      | offset | contents                                      | size |
63  *      +========+===============================================+======+
64  *      | 0x0000 | H2G `CTB Descriptor`_ (send)                  |      |
65  *      +--------+-----------------------------------------------+  4K  |
66  *      | 0x0800 | G2H `CTB Descriptor`_ (recv)                  |      |
67  *      +--------+-----------------------------------------------+------+
68  *      | 0x1000 | H2G `CT Buffer`_ (send)                       | n*4K |
69  *      |        |                                               |      |
70  *      +--------+-----------------------------------------------+------+
71  *      | 0x1000 | G2H `CT Buffer`_ (recv)                       | m*4K |
72  *      | + n*4K |                                               |      |
73  *      +--------+-----------------------------------------------+------+
74  *
75  * Size of each `CT Buffer`_ must be multiple of 4K.
76  * We don't expect too many messages in flight at any time, unless we are
77  * using the GuC submission. In that case each request requires a minimum
78  * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
79  * enough space to avoid backpressure on the driver. We increase the size
80  * of the receive buffer (relative to the send) to ensure a G2H response
81  * CTB has a landing spot.
82  */
83 #define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
84 #define CTB_H2G_BUFFER_SIZE	(SZ_4K)
85 #define CTB_G2H_BUFFER_SIZE	(4 * CTB_H2G_BUFFER_SIZE)
86 #define G2H_ROOM_BUFFER_SIZE	(CTB_G2H_BUFFER_SIZE / 4)
87 
88 struct ct_request {
89 	struct list_head link;
90 	u32 fence;
91 	u32 status;
92 	u32 response_len;
93 	u32 *response_buf;
94 };
95 
96 struct ct_incoming_msg {
97 	struct list_head link;
98 	u32 size;
99 	u32 msg[];
100 };
101 
102 enum { CTB_SEND = 0, CTB_RECV = 1 };
103 
104 enum { CTB_OWNER_HOST = 0 };
105 
106 static void ct_receive_tasklet_func(struct tasklet_struct *t);
107 static void ct_incoming_request_worker_func(struct work_struct *w);
108 
109 /**
110  * intel_guc_ct_init_early - Initialize CT state without requiring device access
111  * @ct: pointer to CT struct
112  */
intel_guc_ct_init_early(struct intel_guc_ct * ct)113 void intel_guc_ct_init_early(struct intel_guc_ct *ct)
114 {
115 	spin_lock_init(&ct->ctbs.send.lock);
116 	spin_lock_init(&ct->ctbs.recv.lock);
117 	spin_lock_init(&ct->requests.lock);
118 	INIT_LIST_HEAD(&ct->requests.pending);
119 	INIT_LIST_HEAD(&ct->requests.incoming);
120 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
121 	INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func);
122 #endif
123 	INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);
124 	tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func);
125 	init_waitqueue_head(&ct->wq);
126 }
127 
guc_ct_buffer_desc_init(struct guc_ct_buffer_desc * desc)128 static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc)
129 {
130 	memset(desc, 0, sizeof(*desc));
131 }
132 
guc_ct_buffer_reset(struct intel_guc_ct_buffer * ctb)133 static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb)
134 {
135 	u32 space;
136 
137 	ctb->broken = false;
138 	ctb->tail = 0;
139 	ctb->head = 0;
140 	space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space;
141 	atomic_set(&ctb->space, space);
142 
143 	guc_ct_buffer_desc_init(ctb->desc);
144 }
145 
guc_ct_buffer_init(struct intel_guc_ct_buffer * ctb,struct guc_ct_buffer_desc * desc,u32 * cmds,u32 size_in_bytes,u32 resv_space)146 static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb,
147 			       struct guc_ct_buffer_desc *desc,
148 			       u32 *cmds, u32 size_in_bytes, u32 resv_space)
149 {
150 	GEM_BUG_ON(size_in_bytes % 4);
151 
152 	ctb->desc = desc;
153 	ctb->cmds = cmds;
154 	ctb->size = size_in_bytes / 4;
155 	ctb->resv_space = resv_space / 4;
156 
157 	guc_ct_buffer_reset(ctb);
158 }
159 
guc_action_control_ctb(struct intel_guc * guc,u32 control)160 static int guc_action_control_ctb(struct intel_guc *guc, u32 control)
161 {
162 	u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
163 		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
164 		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
165 		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB),
166 		FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control),
167 	};
168 	int ret;
169 
170 	GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE);
171 
172 	/* CT control must go over MMIO */
173 	ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0);
174 
175 	return ret > 0 ? -EPROTO : ret;
176 }
177 
ct_control_enable(struct intel_guc_ct * ct,bool enable)178 static int ct_control_enable(struct intel_guc_ct *ct, bool enable)
179 {
180 	int err;
181 
182 	err = guc_action_control_ctb(ct_to_guc(ct), enable ?
183 				     GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE);
184 	if (unlikely(err))
185 		CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n",
186 			       str_enable_disable(enable), ERR_PTR(err));
187 
188 	return err;
189 }
190 
ct_register_buffer(struct intel_guc_ct * ct,bool send,u32 desc_addr,u32 buff_addr,u32 size)191 static int ct_register_buffer(struct intel_guc_ct *ct, bool send,
192 			      u32 desc_addr, u32 buff_addr, u32 size)
193 {
194 	int err;
195 
196 	err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
197 				   GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY :
198 				   GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
199 				   desc_addr);
200 	if (unlikely(err))
201 		goto failed;
202 
203 	err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
204 				   GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY :
205 				   GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
206 				   buff_addr);
207 	if (unlikely(err))
208 		goto failed;
209 
210 	err = intel_guc_self_cfg32(ct_to_guc(ct), send ?
211 				   GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY :
212 				   GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
213 				   size);
214 	if (unlikely(err))
215 failed:
216 		CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n",
217 			       send ? "SEND" : "RECV", ERR_PTR(err));
218 
219 	return err;
220 }
221 
222 /**
223  * intel_guc_ct_init - Init buffer-based communication
224  * @ct: pointer to CT struct
225  *
226  * Allocate memory required for buffer-based communication.
227  *
228  * Return: 0 on success, a negative errno code on failure.
229  */
intel_guc_ct_init(struct intel_guc_ct * ct)230 int intel_guc_ct_init(struct intel_guc_ct *ct)
231 {
232 	struct intel_guc *guc = ct_to_guc(ct);
233 	struct guc_ct_buffer_desc *desc;
234 	u32 blob_size;
235 	u32 cmds_size;
236 	u32 resv_space;
237 	void *blob;
238 	u32 *cmds;
239 	int err;
240 
241 	err = i915_inject_probe_error(guc_to_gt(guc)->i915, -ENXIO);
242 	if (err)
243 		return err;
244 
245 	GEM_BUG_ON(ct->vma);
246 
247 	blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;
248 	err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob);
249 	if (unlikely(err)) {
250 		CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n",
251 			       blob_size, ERR_PTR(err));
252 		return err;
253 	}
254 
255 	CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size);
256 
257 	/* store pointers to desc and cmds for send ctb */
258 	desc = blob;
259 	cmds = blob + 2 * CTB_DESC_SIZE;
260 	cmds_size = CTB_H2G_BUFFER_SIZE;
261 	resv_space = 0;
262 	CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send",
263 		 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
264 		 resv_space);
265 
266 	guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space);
267 
268 	/* store pointers to desc and cmds for recv ctb */
269 	desc = blob + CTB_DESC_SIZE;
270 	cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE;
271 	cmds_size = CTB_G2H_BUFFER_SIZE;
272 	resv_space = G2H_ROOM_BUFFER_SIZE;
273 	CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv",
274 		 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
275 		 resv_space);
276 
277 	guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space);
278 
279 	return 0;
280 }
281 
282 /**
283  * intel_guc_ct_fini - Fini buffer-based communication
284  * @ct: pointer to CT struct
285  *
286  * Deallocate memory required for buffer-based communication.
287  */
intel_guc_ct_fini(struct intel_guc_ct * ct)288 void intel_guc_ct_fini(struct intel_guc_ct *ct)
289 {
290 	GEM_BUG_ON(ct->enabled);
291 
292 	tasklet_kill(&ct->receive_tasklet);
293 	i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP);
294 	memset(ct, 0, sizeof(*ct));
295 }
296 
297 /**
298  * intel_guc_ct_enable - Enable buffer based command transport.
299  * @ct: pointer to CT struct
300  *
301  * Return: 0 on success, a negative errno code on failure.
302  */
intel_guc_ct_enable(struct intel_guc_ct * ct)303 int intel_guc_ct_enable(struct intel_guc_ct *ct)
304 {
305 	struct intel_guc *guc = ct_to_guc(ct);
306 	u32 base, desc, cmds, size;
307 	void *blob;
308 	int err;
309 
310 	GEM_BUG_ON(ct->enabled);
311 
312 	/* vma should be already allocated and map'ed */
313 	GEM_BUG_ON(!ct->vma);
314 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj));
315 	base = intel_guc_ggtt_offset(guc, ct->vma);
316 
317 	/* blob should start with send descriptor */
318 	blob = __px_vaddr(ct->vma->obj);
319 	GEM_BUG_ON(blob != ct->ctbs.send.desc);
320 
321 	/* (re)initialize descriptors */
322 	guc_ct_buffer_reset(&ct->ctbs.send);
323 	guc_ct_buffer_reset(&ct->ctbs.recv);
324 
325 	/*
326 	 * Register both CT buffers starting with RECV buffer.
327 	 * Descriptors are in first half of the blob.
328 	 */
329 	desc = base + ptrdiff(ct->ctbs.recv.desc, blob);
330 	cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob);
331 	size = ct->ctbs.recv.size * 4;
332 	err = ct_register_buffer(ct, false, desc, cmds, size);
333 	if (unlikely(err))
334 		goto err_out;
335 
336 	desc = base + ptrdiff(ct->ctbs.send.desc, blob);
337 	cmds = base + ptrdiff(ct->ctbs.send.cmds, blob);
338 	size = ct->ctbs.send.size * 4;
339 	err = ct_register_buffer(ct, true, desc, cmds, size);
340 	if (unlikely(err))
341 		goto err_out;
342 
343 	err = ct_control_enable(ct, true);
344 	if (unlikely(err))
345 		goto err_out;
346 
347 	ct->enabled = true;
348 	ct->stall_time = KTIME_MAX;
349 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
350 	ct->dead_ct_reported = false;
351 	ct->dead_ct_reason = CT_DEAD_ALIVE;
352 #endif
353 
354 	return 0;
355 
356 err_out:
357 	CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err));
358 	CT_DEAD(ct, SETUP);
359 	return err;
360 }
361 
362 /**
363  * intel_guc_ct_disable - Disable buffer based command transport.
364  * @ct: pointer to CT struct
365  */
intel_guc_ct_disable(struct intel_guc_ct * ct)366 void intel_guc_ct_disable(struct intel_guc_ct *ct)
367 {
368 	struct intel_guc *guc = ct_to_guc(ct);
369 
370 	GEM_BUG_ON(!ct->enabled);
371 
372 	ct->enabled = false;
373 
374 	if (intel_guc_is_fw_running(guc)) {
375 		ct_control_enable(ct, false);
376 	}
377 }
378 
379 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
ct_track_lost_and_found(struct intel_guc_ct * ct,u32 fence,u32 action)380 static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action)
381 {
382 	unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found);
383 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
384 	unsigned long entries[SZ_32];
385 	unsigned int n;
386 
387 	n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
388 
389 	/* May be called under spinlock, so avoid sleeping */
390 	ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT);
391 #endif
392 	ct->requests.lost_and_found[lost].fence = fence;
393 	ct->requests.lost_and_found[lost].action = action;
394 }
395 #endif
396 
ct_get_next_fence(struct intel_guc_ct * ct)397 static u32 ct_get_next_fence(struct intel_guc_ct *ct)
398 {
399 	/* For now it's trivial */
400 	return ++ct->requests.last_fence;
401 }
402 
ct_write(struct intel_guc_ct * ct,const u32 * action,u32 len,u32 fence,u32 flags)403 static int ct_write(struct intel_guc_ct *ct,
404 		    const u32 *action,
405 		    u32 len /* in dwords */,
406 		    u32 fence, u32 flags)
407 {
408 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
409 	struct guc_ct_buffer_desc *desc = ctb->desc;
410 	u32 tail = ctb->tail;
411 	u32 size = ctb->size;
412 	u32 header;
413 	u32 hxg;
414 	u32 type;
415 	u32 *cmds = ctb->cmds;
416 	unsigned int i;
417 
418 	if (unlikely(desc->status))
419 		goto corrupted;
420 
421 	GEM_BUG_ON(tail > size);
422 
423 #ifdef CONFIG_DRM_I915_DEBUG_GUC
424 	if (unlikely(tail != READ_ONCE(desc->tail))) {
425 		CT_ERROR(ct, "Tail was modified %u != %u\n",
426 			 desc->tail, tail);
427 		desc->status |= GUC_CTB_STATUS_MISMATCH;
428 		goto corrupted;
429 	}
430 	if (unlikely(READ_ONCE(desc->head) >= size)) {
431 		CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
432 			 desc->head, size);
433 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
434 		goto corrupted;
435 	}
436 #endif
437 
438 	/*
439 	 * dw0: CT header (including fence)
440 	 * dw1: HXG header (including action code)
441 	 * dw2+: action data
442 	 */
443 	header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
444 		 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
445 		 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence);
446 
447 	type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST :
448 		GUC_HXG_TYPE_REQUEST;
449 	hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) |
450 		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION |
451 			   GUC_HXG_REQUEST_MSG_0_DATA0, action[0]);
452 
453 	CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n",
454 		 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]);
455 
456 	cmds[tail] = header;
457 	tail = (tail + 1) % size;
458 
459 	cmds[tail] = hxg;
460 	tail = (tail + 1) % size;
461 
462 	for (i = 1; i < len; i++) {
463 		cmds[tail] = action[i];
464 		tail = (tail + 1) % size;
465 	}
466 	GEM_BUG_ON(tail > size);
467 
468 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
469 	ct_track_lost_and_found(ct, fence,
470 				FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0]));
471 #endif
472 
473 	/*
474 	 * make sure H2G buffer update and LRC tail update (if this triggering a
475 	 * submission) are visible before updating the descriptor tail
476 	 */
477 	intel_guc_write_barrier(ct_to_guc(ct));
478 
479 	/* update local copies */
480 	ctb->tail = tail;
481 	GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN);
482 	atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space);
483 
484 	/* now update descriptor */
485 	WRITE_ONCE(desc->tail, tail);
486 
487 	return 0;
488 
489 corrupted:
490 	CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
491 		 desc->head, desc->tail, desc->status);
492 	CT_DEAD(ct, WRITE);
493 	ctb->broken = true;
494 	return -EPIPE;
495 }
496 
497 /**
498  * wait_for_ct_request_update - Wait for CT request state update.
499  * @ct:		pointer to CT
500  * @req:	pointer to pending request
501  * @status:	placeholder for status
502  *
503  * For each sent request, GuC shall send back CT response message.
504  * Our message handler will update status of tracked request once
505  * response message with given fence is received. Wait here and
506  * check for valid response status value.
507  *
508  * Return:
509  * *	0 response received (status is valid)
510  * *	-ETIMEDOUT no response within hardcoded timeout
511  */
wait_for_ct_request_update(struct intel_guc_ct * ct,struct ct_request * req,u32 * status)512 static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status)
513 {
514 	int err;
515 	bool ct_enabled;
516 
517 	/*
518 	 * Fast commands should complete in less than 10us, so sample quickly
519 	 * up to that length of time, then switch to a slower sleep-wait loop.
520 	 * No GuC command should ever take longer than 10ms but many GuC
521 	 * commands can be inflight at time, so use a 1s timeout on the slower
522 	 * sleep-wait loop.
523 	 */
524 #define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10
525 #define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000
526 #define done \
527 	(!(ct_enabled = intel_guc_ct_enabled(ct)) || \
528 	 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \
529 	 GUC_HXG_ORIGIN_GUC)
530 	err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS);
531 	if (err)
532 		err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS);
533 #undef done
534 	if (!ct_enabled)
535 		err = -ENODEV;
536 
537 	*status = req->status;
538 	return err;
539 }
540 
541 #define GUC_CTB_TIMEOUT_MS	1500
ct_deadlocked(struct intel_guc_ct * ct)542 static inline bool ct_deadlocked(struct intel_guc_ct *ct)
543 {
544 	long timeout = GUC_CTB_TIMEOUT_MS;
545 	bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout;
546 
547 	if (unlikely(ret)) {
548 		struct guc_ct_buffer_desc *send = ct->ctbs.send.desc;
549 		struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc;
550 
551 		CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n",
552 			 ktime_ms_delta(ktime_get(), ct->stall_time),
553 			 send->status, recv->status);
554 		CT_ERROR(ct, "H2G Space: %u (Bytes)\n",
555 			 atomic_read(&ct->ctbs.send.space) * 4);
556 		CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head);
557 		CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail);
558 		CT_ERROR(ct, "G2H Space: %u (Bytes)\n",
559 			 atomic_read(&ct->ctbs.recv.space) * 4);
560 		CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head);
561 		CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail);
562 
563 		CT_DEAD(ct, DEADLOCK);
564 		ct->ctbs.send.broken = true;
565 	}
566 
567 	return ret;
568 }
569 
g2h_has_room(struct intel_guc_ct * ct,u32 g2h_len_dw)570 static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw)
571 {
572 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
573 
574 	/*
575 	 * We leave a certain amount of space in the G2H CTB buffer for
576 	 * unexpected G2H CTBs (e.g. logging, engine hang, etc...)
577 	 */
578 	return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw;
579 }
580 
g2h_reserve_space(struct intel_guc_ct * ct,u32 g2h_len_dw)581 static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
582 {
583 	lockdep_assert_held(&ct->ctbs.send.lock);
584 
585 	GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw));
586 
587 	if (g2h_len_dw)
588 		atomic_sub(g2h_len_dw, &ct->ctbs.recv.space);
589 }
590 
g2h_release_space(struct intel_guc_ct * ct,u32 g2h_len_dw)591 static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
592 {
593 	atomic_add(g2h_len_dw, &ct->ctbs.recv.space);
594 }
595 
h2g_has_room(struct intel_guc_ct * ct,u32 len_dw)596 static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw)
597 {
598 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
599 	struct guc_ct_buffer_desc *desc = ctb->desc;
600 	u32 head;
601 	u32 space;
602 
603 	if (atomic_read(&ctb->space) >= len_dw)
604 		return true;
605 
606 	head = READ_ONCE(desc->head);
607 	if (unlikely(head > ctb->size)) {
608 		CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
609 			 head, ctb->size);
610 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
611 		ctb->broken = true;
612 		CT_DEAD(ct, H2G_HAS_ROOM);
613 		return false;
614 	}
615 
616 	space = CIRC_SPACE(ctb->tail, head, ctb->size);
617 	atomic_set(&ctb->space, space);
618 
619 	return space >= len_dw;
620 }
621 
has_room_nb(struct intel_guc_ct * ct,u32 h2g_dw,u32 g2h_dw)622 static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw)
623 {
624 	bool h2g = h2g_has_room(ct, h2g_dw);
625 	bool g2h = g2h_has_room(ct, g2h_dw);
626 
627 	lockdep_assert_held(&ct->ctbs.send.lock);
628 
629 	if (unlikely(!h2g || !g2h)) {
630 		if (ct->stall_time == KTIME_MAX)
631 			ct->stall_time = ktime_get();
632 
633 		/* Be paranoid and kick G2H tasklet to free credits */
634 		if (!g2h)
635 			tasklet_hi_schedule(&ct->receive_tasklet);
636 
637 		if (unlikely(ct_deadlocked(ct)))
638 			return -EPIPE;
639 		else
640 			return -EBUSY;
641 	}
642 
643 	ct->stall_time = KTIME_MAX;
644 	return 0;
645 }
646 
647 #define G2H_LEN_DW(f) ({ \
648 	typeof(f) f_ = (f); \
649 	FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \
650 	FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \
651 	GUC_CTB_HXG_MSG_MIN_LEN : 0; \
652 })
ct_send_nb(struct intel_guc_ct * ct,const u32 * action,u32 len,u32 flags)653 static int ct_send_nb(struct intel_guc_ct *ct,
654 		      const u32 *action,
655 		      u32 len,
656 		      u32 flags)
657 {
658 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
659 	unsigned long spin_flags;
660 	u32 g2h_len_dw = G2H_LEN_DW(flags);
661 	u32 fence;
662 	int ret;
663 
664 	spin_lock_irqsave(&ctb->lock, spin_flags);
665 
666 	ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw);
667 	if (unlikely(ret))
668 		goto out;
669 
670 	fence = ct_get_next_fence(ct);
671 	ret = ct_write(ct, action, len, fence, flags);
672 	if (unlikely(ret))
673 		goto out;
674 
675 	g2h_reserve_space(ct, g2h_len_dw);
676 	intel_guc_notify(ct_to_guc(ct));
677 
678 out:
679 	spin_unlock_irqrestore(&ctb->lock, spin_flags);
680 
681 	return ret;
682 }
683 
ct_send(struct intel_guc_ct * ct,const u32 * action,u32 len,u32 * response_buf,u32 response_buf_size,u32 * status)684 static int ct_send(struct intel_guc_ct *ct,
685 		   const u32 *action,
686 		   u32 len,
687 		   u32 *response_buf,
688 		   u32 response_buf_size,
689 		   u32 *status)
690 {
691 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
692 	struct ct_request request;
693 	unsigned long flags;
694 	unsigned int sleep_period_ms = 1;
695 	bool send_again;
696 	u32 fence;
697 	int err;
698 
699 	GEM_BUG_ON(!ct->enabled);
700 	GEM_BUG_ON(!len);
701 	GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN);
702 	GEM_BUG_ON(!response_buf && response_buf_size);
703 	might_sleep();
704 
705 resend:
706 	send_again = false;
707 
708 	/*
709 	 * We use a lazy spin wait loop here as we believe that if the CT
710 	 * buffers are sized correctly the flow control condition should be
711 	 * rare. Reserving the maximum size in the G2H credits as we don't know
712 	 * how big the response is going to be.
713 	 */
714 retry:
715 	spin_lock_irqsave(&ctb->lock, flags);
716 	if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) ||
717 		     !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) {
718 		if (ct->stall_time == KTIME_MAX)
719 			ct->stall_time = ktime_get();
720 		spin_unlock_irqrestore(&ctb->lock, flags);
721 
722 		if (unlikely(ct_deadlocked(ct)))
723 			return -EPIPE;
724 
725 		if (msleep_interruptible(sleep_period_ms))
726 			return -EINTR;
727 		sleep_period_ms = sleep_period_ms << 1;
728 
729 		goto retry;
730 	}
731 
732 	ct->stall_time = KTIME_MAX;
733 
734 	fence = ct_get_next_fence(ct);
735 	request.fence = fence;
736 	request.status = 0;
737 	request.response_len = response_buf_size;
738 	request.response_buf = response_buf;
739 
740 	spin_lock(&ct->requests.lock);
741 	list_add_tail(&request.link, &ct->requests.pending);
742 	spin_unlock(&ct->requests.lock);
743 
744 	err = ct_write(ct, action, len, fence, 0);
745 	g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
746 
747 	spin_unlock_irqrestore(&ctb->lock, flags);
748 
749 	if (unlikely(err))
750 		goto unlink;
751 
752 	intel_guc_notify(ct_to_guc(ct));
753 
754 	err = wait_for_ct_request_update(ct, &request, status);
755 	g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
756 	if (unlikely(err)) {
757 		if (err == -ENODEV)
758 			/* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress.
759 			 * In this case, output is debug rather than error info
760 			 */
761 			CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n",
762 				 action[0], request.fence);
763 		else
764 			CT_ERROR(ct, "No response for request %#x (fence %u)\n",
765 				 action[0], request.fence);
766 		goto unlink;
767 	}
768 
769 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
770 		CT_DEBUG(ct, "retrying request %#x (%u)\n", *action,
771 			 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status));
772 		send_again = true;
773 		goto unlink;
774 	}
775 
776 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) {
777 		err = -EIO;
778 		goto unlink;
779 	}
780 
781 	if (response_buf) {
782 		/* There shall be no data in the status */
783 		WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status));
784 		/* Return actual response len */
785 		err = request.response_len;
786 	} else {
787 		/* There shall be no response payload */
788 		WARN_ON(request.response_len);
789 		/* Return data decoded from the status dword */
790 		err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status);
791 	}
792 
793 unlink:
794 	spin_lock_irqsave(&ct->requests.lock, flags);
795 	list_del(&request.link);
796 	spin_unlock_irqrestore(&ct->requests.lock, flags);
797 
798 	if (unlikely(send_again))
799 		goto resend;
800 
801 	return err;
802 }
803 
804 /*
805  * Command Transport (CT) buffer based GuC send function.
806  */
intel_guc_ct_send(struct intel_guc_ct * ct,const u32 * action,u32 len,u32 * response_buf,u32 response_buf_size,u32 flags)807 int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
808 		      u32 *response_buf, u32 response_buf_size, u32 flags)
809 {
810 	u32 status = ~0; /* undefined */
811 	int ret;
812 
813 	if (unlikely(!ct->enabled)) {
814 		struct intel_guc *guc = ct_to_guc(ct);
815 		struct intel_uc *uc = container_of(guc, struct intel_uc, guc);
816 
817 		WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action);
818 		return -ENODEV;
819 	}
820 
821 	if (unlikely(ct->ctbs.send.broken))
822 		return -EPIPE;
823 
824 	if (flags & INTEL_GUC_CT_SEND_NB)
825 		return ct_send_nb(ct, action, len, flags);
826 
827 	ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);
828 	if (unlikely(ret < 0)) {
829 		if (ret != -ENODEV)
830 			CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",
831 				 action[0], ERR_PTR(ret), status);
832 	} else if (unlikely(ret)) {
833 		CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
834 			 action[0], ret, ret);
835 	}
836 
837 	return ret;
838 }
839 
ct_alloc_msg(u32 num_dwords)840 static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords)
841 {
842 	struct ct_incoming_msg *msg;
843 
844 	msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC);
845 	if (msg)
846 		msg->size = num_dwords;
847 	return msg;
848 }
849 
ct_free_msg(struct ct_incoming_msg * msg)850 static void ct_free_msg(struct ct_incoming_msg *msg)
851 {
852 	kfree(msg);
853 }
854 
855 /*
856  * Return: number available remaining dwords to read (0 if empty)
857  *         or a negative error code on failure
858  */
ct_read(struct intel_guc_ct * ct,struct ct_incoming_msg ** msg)859 static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)
860 {
861 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
862 	struct guc_ct_buffer_desc *desc = ctb->desc;
863 	u32 head = ctb->head;
864 	u32 tail = READ_ONCE(desc->tail);
865 	u32 size = ctb->size;
866 	u32 *cmds = ctb->cmds;
867 	s32 available;
868 	unsigned int len;
869 	unsigned int i;
870 	u32 header;
871 
872 	if (unlikely(ctb->broken))
873 		return -EPIPE;
874 
875 	if (unlikely(desc->status)) {
876 		u32 status = desc->status;
877 
878 		if (status & GUC_CTB_STATUS_UNUSED) {
879 			/*
880 			 * Potentially valid if a CLIENT_RESET request resulted in
881 			 * contexts/engines being reset. But should never happen as
882 			 * no contexts should be active when CLIENT_RESET is sent.
883 			 */
884 			CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n");
885 			status &= ~GUC_CTB_STATUS_UNUSED;
886 		}
887 
888 		if (status)
889 			goto corrupted;
890 	}
891 
892 	GEM_BUG_ON(head > size);
893 
894 #ifdef CONFIG_DRM_I915_DEBUG_GUC
895 	if (unlikely(head != READ_ONCE(desc->head))) {
896 		CT_ERROR(ct, "Head was modified %u != %u\n",
897 			 desc->head, head);
898 		desc->status |= GUC_CTB_STATUS_MISMATCH;
899 		goto corrupted;
900 	}
901 #endif
902 	if (unlikely(tail >= size)) {
903 		CT_ERROR(ct, "Invalid tail offset %u >= %u)\n",
904 			 tail, size);
905 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
906 		goto corrupted;
907 	}
908 
909 	/* tail == head condition indicates empty */
910 	available = tail - head;
911 	if (unlikely(available == 0)) {
912 		*msg = NULL;
913 		return 0;
914 	}
915 
916 	/* beware of buffer wrap case */
917 	if (unlikely(available < 0))
918 		available += size;
919 	CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size);
920 	GEM_BUG_ON(available < 0);
921 
922 	header = cmds[head];
923 	head = (head + 1) % size;
924 
925 	/* message len with header */
926 	len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN;
927 	if (unlikely(len > (u32)available)) {
928 		CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",
929 			 4, &header,
930 			 4 * (head + available - 1 > size ?
931 			      size - head : available - 1), &cmds[head],
932 			 4 * (head + available - 1 > size ?
933 			      available - 1 - size + head : 0), &cmds[0]);
934 		desc->status |= GUC_CTB_STATUS_UNDERFLOW;
935 		goto corrupted;
936 	}
937 
938 	*msg = ct_alloc_msg(len);
939 	if (!*msg) {
940 		CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n",
941 			 4, &header,
942 			 4 * (head + available - 1 > size ?
943 			      size - head : available - 1), &cmds[head],
944 			 4 * (head + available - 1 > size ?
945 			      available - 1 - size + head : 0), &cmds[0]);
946 		return available;
947 	}
948 
949 	(*msg)->msg[0] = header;
950 
951 	for (i = 1; i < len; i++) {
952 		(*msg)->msg[i] = cmds[head];
953 		head = (head + 1) % size;
954 	}
955 	CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg);
956 
957 	/* update local copies */
958 	ctb->head = head;
959 
960 	/* now update descriptor */
961 	WRITE_ONCE(desc->head, head);
962 
963 	intel_guc_write_barrier(ct_to_guc(ct));
964 
965 	return available - len;
966 
967 corrupted:
968 	CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
969 		 desc->head, desc->tail, desc->status);
970 	ctb->broken = true;
971 	CT_DEAD(ct, READ);
972 	return -EPIPE;
973 }
974 
975 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
ct_check_lost_and_found(struct intel_guc_ct * ct,u32 fence)976 static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
977 {
978 	unsigned int n;
979 	char *buf = NULL;
980 	bool found = false;
981 
982 	lockdep_assert_held(&ct->requests.lock);
983 
984 	for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) {
985 		if (ct->requests.lost_and_found[n].fence != fence)
986 			continue;
987 		found = true;
988 
989 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
990 		buf = kmalloc(SZ_4K, GFP_NOWAIT);
991 		if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack,
992 					       buf, SZ_4K, 0)) {
993 			CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s",
994 				 fence, ct->requests.lost_and_found[n].action, buf);
995 			break;
996 		}
997 #endif
998 		CT_ERROR(ct, "Fence %u was used by action %#04x\n",
999 			 fence, ct->requests.lost_and_found[n].action);
1000 		break;
1001 	}
1002 	kfree(buf);
1003 	return found;
1004 }
1005 #else
ct_check_lost_and_found(struct intel_guc_ct * ct,u32 fence)1006 static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1007 {
1008 	return false;
1009 }
1010 #endif
1011 
ct_handle_response(struct intel_guc_ct * ct,struct ct_incoming_msg * response)1012 static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response)
1013 {
1014 	u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]);
1015 	u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]);
1016 	const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN];
1017 	const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN];
1018 	u32 datalen = len - GUC_HXG_MSG_MIN_LEN;
1019 	struct ct_request *req;
1020 	unsigned long flags;
1021 	bool found = false;
1022 	int err = 0;
1023 
1024 	GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN);
1025 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC);
1026 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS &&
1027 		   FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY &&
1028 		   FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE);
1029 
1030 	CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]);
1031 
1032 	spin_lock_irqsave(&ct->requests.lock, flags);
1033 	list_for_each_entry(req, &ct->requests.pending, link) {
1034 		if (unlikely(fence != req->fence)) {
1035 			CT_DEBUG(ct, "request %u awaits response\n",
1036 				 req->fence);
1037 			continue;
1038 		}
1039 		if (unlikely(datalen > req->response_len)) {
1040 			CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n",
1041 				 req->fence, datalen, req->response_len);
1042 			datalen = min(datalen, req->response_len);
1043 			err = -EMSGSIZE;
1044 		}
1045 		if (datalen)
1046 			memcpy(req->response_buf, data, 4 * datalen);
1047 		req->response_len = datalen;
1048 		WRITE_ONCE(req->status, hxg[0]);
1049 		found = true;
1050 		break;
1051 	}
1052 	if (!found) {
1053 		CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n",
1054 			 len, hxg[0], fence, ct->requests.last_fence);
1055 		if (!ct_check_lost_and_found(ct, fence)) {
1056 			list_for_each_entry(req, &ct->requests.pending, link)
1057 				CT_ERROR(ct, "request %u awaits response\n",
1058 					 req->fence);
1059 		}
1060 		err = -ENOKEY;
1061 	}
1062 	spin_unlock_irqrestore(&ct->requests.lock, flags);
1063 
1064 	if (unlikely(err))
1065 		return err;
1066 
1067 	ct_free_msg(response);
1068 	return 0;
1069 }
1070 
ct_process_request(struct intel_guc_ct * ct,struct ct_incoming_msg * request)1071 static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1072 {
1073 	struct intel_guc *guc = ct_to_guc(ct);
1074 	const u32 *hxg;
1075 	const u32 *payload;
1076 	u32 hxg_len, action, len;
1077 	int ret;
1078 
1079 	hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1080 	hxg_len = request->size - GUC_CTB_MSG_MIN_LEN;
1081 	payload = &hxg[GUC_HXG_MSG_MIN_LEN];
1082 	action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1083 	len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1084 
1085 	CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);
1086 
1087 	switch (action) {
1088 	case INTEL_GUC_ACTION_DEFAULT:
1089 		ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
1090 		break;
1091 	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1092 		ret = intel_guc_deregister_done_process_msg(guc, payload,
1093 							    len);
1094 		break;
1095 	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1096 		ret = intel_guc_sched_done_process_msg(guc, payload, len);
1097 		break;
1098 	case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1099 		ret = intel_guc_context_reset_process_msg(guc, payload, len);
1100 		break;
1101 	case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1102 		ret = intel_guc_error_capture_process_msg(guc, payload, len);
1103 		if (unlikely(ret))
1104 			CT_ERROR(ct, "error capture notification failed %x %*ph\n",
1105 				 action, 4 * len, payload);
1106 		break;
1107 	case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1108 		ret = intel_guc_engine_failure_process_msg(guc, payload, len);
1109 		break;
1110 	case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1111 		intel_guc_log_handle_flush_event(&guc->log);
1112 		ret = 0;
1113 		break;
1114 	case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1115 		CT_ERROR(ct, "Received GuC crash dump notification!\n");
1116 		ret = 0;
1117 		break;
1118 	case INTEL_GUC_ACTION_NOTIFY_EXCEPTION:
1119 		CT_ERROR(ct, "Received GuC exception notification!\n");
1120 		ret = 0;
1121 		break;
1122 	default:
1123 		ret = -EOPNOTSUPP;
1124 		break;
1125 	}
1126 
1127 	if (unlikely(ret)) {
1128 		CT_ERROR(ct, "Failed to process request %04x (%pe)\n",
1129 			 action, ERR_PTR(ret));
1130 		return ret;
1131 	}
1132 
1133 	ct_free_msg(request);
1134 	return 0;
1135 }
1136 
ct_process_incoming_requests(struct intel_guc_ct * ct)1137 static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
1138 {
1139 	unsigned long flags;
1140 	struct ct_incoming_msg *request;
1141 	bool done;
1142 	int err;
1143 
1144 	spin_lock_irqsave(&ct->requests.lock, flags);
1145 	request = list_first_entry_or_null(&ct->requests.incoming,
1146 					   struct ct_incoming_msg, link);
1147 	if (request)
1148 		list_del(&request->link);
1149 	done = !!list_empty(&ct->requests.incoming);
1150 	spin_unlock_irqrestore(&ct->requests.lock, flags);
1151 
1152 	if (!request)
1153 		return true;
1154 
1155 	err = ct_process_request(ct, request);
1156 	if (unlikely(err)) {
1157 		CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1158 			 ERR_PTR(err), 4 * request->size, request->msg);
1159 		CT_DEAD(ct, PROCESS_FAILED);
1160 		ct_free_msg(request);
1161 	}
1162 
1163 	return done;
1164 }
1165 
ct_incoming_request_worker_func(struct work_struct * w)1166 static void ct_incoming_request_worker_func(struct work_struct *w)
1167 {
1168 	struct intel_guc_ct *ct =
1169 		container_of(w, struct intel_guc_ct, requests.worker);
1170 	bool done;
1171 
1172 	do {
1173 		done = ct_process_incoming_requests(ct);
1174 	} while (!done);
1175 }
1176 
ct_handle_event(struct intel_guc_ct * ct,struct ct_incoming_msg * request)1177 static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1178 {
1179 	const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1180 	u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1181 	unsigned long flags;
1182 
1183 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT);
1184 
1185 	/*
1186 	 * Adjusting the space must be done in IRQ or deadlock can occur as the
1187 	 * CTB processing in the below workqueue can send CTBs which creates a
1188 	 * circular dependency if the space was returned there.
1189 	 */
1190 	switch (action) {
1191 	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1192 	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1193 		g2h_release_space(ct, request->size);
1194 	}
1195 
1196 	spin_lock_irqsave(&ct->requests.lock, flags);
1197 	list_add_tail(&request->link, &ct->requests.incoming);
1198 	spin_unlock_irqrestore(&ct->requests.lock, flags);
1199 
1200 	queue_work(system_unbound_wq, &ct->requests.worker);
1201 	return 0;
1202 }
1203 
ct_handle_hxg(struct intel_guc_ct * ct,struct ct_incoming_msg * msg)1204 static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1205 {
1206 	u32 origin, type;
1207 	u32 *hxg;
1208 	int err;
1209 
1210 	if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN))
1211 		return -EBADMSG;
1212 
1213 	hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN];
1214 
1215 	origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1216 	if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1217 		err = -EPROTO;
1218 		goto failed;
1219 	}
1220 
1221 	type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1222 	switch (type) {
1223 	case GUC_HXG_TYPE_EVENT:
1224 		err = ct_handle_event(ct, msg);
1225 		break;
1226 	case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1227 	case GUC_HXG_TYPE_RESPONSE_FAILURE:
1228 	case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1229 		err = ct_handle_response(ct, msg);
1230 		break;
1231 	default:
1232 		err = -EOPNOTSUPP;
1233 	}
1234 
1235 	if (unlikely(err)) {
1236 failed:
1237 		CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n",
1238 			 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg);
1239 	}
1240 	return err;
1241 }
1242 
ct_handle_msg(struct intel_guc_ct * ct,struct ct_incoming_msg * msg)1243 static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1244 {
1245 	u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]);
1246 	int err;
1247 
1248 	if (format == GUC_CTB_FORMAT_HXG)
1249 		err = ct_handle_hxg(ct, msg);
1250 	else
1251 		err = -EOPNOTSUPP;
1252 
1253 	if (unlikely(err)) {
1254 		CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1255 			 ERR_PTR(err), 4 * msg->size, msg->msg);
1256 		ct_free_msg(msg);
1257 	}
1258 }
1259 
1260 /*
1261  * Return: number available remaining dwords to read (0 if empty)
1262  *         or a negative error code on failure
1263  */
ct_receive(struct intel_guc_ct * ct)1264 static int ct_receive(struct intel_guc_ct *ct)
1265 {
1266 	struct ct_incoming_msg *msg = NULL;
1267 	unsigned long flags;
1268 	int ret;
1269 
1270 	spin_lock_irqsave(&ct->ctbs.recv.lock, flags);
1271 	ret = ct_read(ct, &msg);
1272 	spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags);
1273 	if (ret < 0)
1274 		return ret;
1275 
1276 	if (msg)
1277 		ct_handle_msg(ct, msg);
1278 
1279 	return ret;
1280 }
1281 
ct_try_receive_message(struct intel_guc_ct * ct)1282 static void ct_try_receive_message(struct intel_guc_ct *ct)
1283 {
1284 	int ret;
1285 
1286 	if (GEM_WARN_ON(!ct->enabled))
1287 		return;
1288 
1289 	ret = ct_receive(ct);
1290 	if (ret > 0)
1291 		tasklet_hi_schedule(&ct->receive_tasklet);
1292 }
1293 
ct_receive_tasklet_func(struct tasklet_struct * t)1294 static void ct_receive_tasklet_func(struct tasklet_struct *t)
1295 {
1296 	struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet);
1297 
1298 	ct_try_receive_message(ct);
1299 }
1300 
1301 /*
1302  * When we're communicating with the GuC over CT, GuC uses events
1303  * to notify us about new messages being posted on the RECV buffer.
1304  */
intel_guc_ct_event_handler(struct intel_guc_ct * ct)1305 void intel_guc_ct_event_handler(struct intel_guc_ct *ct)
1306 {
1307 	if (unlikely(!ct->enabled)) {
1308 		WARN(1, "Unexpected GuC event received while CT disabled!\n");
1309 		return;
1310 	}
1311 
1312 	ct_try_receive_message(ct);
1313 }
1314 
intel_guc_ct_print_info(struct intel_guc_ct * ct,struct drm_printer * p)1315 void intel_guc_ct_print_info(struct intel_guc_ct *ct,
1316 			     struct drm_printer *p)
1317 {
1318 	drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled));
1319 
1320 	if (!ct->enabled)
1321 		return;
1322 
1323 	drm_printf(p, "H2G Space: %u\n",
1324 		   atomic_read(&ct->ctbs.send.space) * 4);
1325 	drm_printf(p, "Head: %u\n",
1326 		   ct->ctbs.send.desc->head);
1327 	drm_printf(p, "Tail: %u\n",
1328 		   ct->ctbs.send.desc->tail);
1329 	drm_printf(p, "G2H Space: %u\n",
1330 		   atomic_read(&ct->ctbs.recv.space) * 4);
1331 	drm_printf(p, "Head: %u\n",
1332 		   ct->ctbs.recv.desc->head);
1333 	drm_printf(p, "Tail: %u\n",
1334 		   ct->ctbs.recv.desc->tail);
1335 }
1336 
1337 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
ct_dead_ct_worker_func(struct work_struct * w)1338 static void ct_dead_ct_worker_func(struct work_struct *w)
1339 {
1340 	struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker);
1341 	struct intel_guc *guc = ct_to_guc(ct);
1342 
1343 	if (ct->dead_ct_reported)
1344 		return;
1345 
1346 	ct->dead_ct_reported = true;
1347 
1348 	guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason);
1349 	intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U);
1350 }
1351 #endif
1352