1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2016-2019 Intel Corporation
4  */
5 
6 #include <linux/circ_buf.h>
7 #include <linux/ktime.h>
8 #include <linux/time64.h>
9 #include <linux/string_helpers.h>
10 #include <linux/timekeeping.h>
11 
12 #include "i915_drv.h"
13 #include "intel_guc_ct.h"
14 #include "gt/intel_gt.h"
15 
16 static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
17 {
18 	return container_of(ct, struct intel_guc, ct);
19 }
20 
21 static inline struct intel_gt *ct_to_gt(struct intel_guc_ct *ct)
22 {
23 	return guc_to_gt(ct_to_guc(ct));
24 }
25 
26 static inline struct drm_i915_private *ct_to_i915(struct intel_guc_ct *ct)
27 {
28 	return ct_to_gt(ct)->i915;
29 }
30 
31 static inline struct drm_device *ct_to_drm(struct intel_guc_ct *ct)
32 {
33 	return &ct_to_i915(ct)->drm;
34 }
35 
36 #define CT_ERROR(_ct, _fmt, ...) \
37 	drm_err(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__)
38 #ifdef CONFIG_DRM_I915_DEBUG_GUC
39 #define CT_DEBUG(_ct, _fmt, ...) \
40 	drm_dbg(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__)
41 #else
42 #define CT_DEBUG(...)	do { } while (0)
43 #endif
44 #define CT_PROBE_ERROR(_ct, _fmt, ...) \
45 	i915_probe_error(ct_to_i915(ct), "CT: " _fmt, ##__VA_ARGS__)
46 
47 /**
48  * DOC: CTB Blob
49  *
50  * We allocate single blob to hold both CTB descriptors and buffers:
51  *
52  *      +--------+-----------------------------------------------+------+
53  *      | offset | contents                                      | size |
54  *      +========+===============================================+======+
55  *      | 0x0000 | H2G `CTB Descriptor`_ (send)                  |      |
56  *      +--------+-----------------------------------------------+  4K  |
57  *      | 0x0800 | G2H `CTB Descriptor`_ (recv)                  |      |
58  *      +--------+-----------------------------------------------+------+
59  *      | 0x1000 | H2G `CT Buffer`_ (send)                       | n*4K |
60  *      |        |                                               |      |
61  *      +--------+-----------------------------------------------+------+
62  *      | 0x1000 | G2H `CT Buffer`_ (recv)                       | m*4K |
63  *      | + n*4K |                                               |      |
64  *      +--------+-----------------------------------------------+------+
65  *
66  * Size of each `CT Buffer`_ must be multiple of 4K.
67  * We don't expect too many messages in flight at any time, unless we are
68  * using the GuC submission. In that case each request requires a minimum
69  * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
70  * enough space to avoid backpressure on the driver. We increase the size
71  * of the receive buffer (relative to the send) to ensure a G2H response
72  * CTB has a landing spot.
73  */
74 #define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
75 #define CTB_H2G_BUFFER_SIZE	(SZ_4K)
76 #define CTB_G2H_BUFFER_SIZE	(4 * CTB_H2G_BUFFER_SIZE)
77 #define G2H_ROOM_BUFFER_SIZE	(CTB_G2H_BUFFER_SIZE / 4)
78 
79 struct ct_request {
80 	struct list_head link;
81 	u32 fence;
82 	u32 status;
83 	u32 response_len;
84 	u32 *response_buf;
85 };
86 
87 struct ct_incoming_msg {
88 	struct list_head link;
89 	u32 size;
90 	u32 msg[];
91 };
92 
93 enum { CTB_SEND = 0, CTB_RECV = 1 };
94 
95 enum { CTB_OWNER_HOST = 0 };
96 
97 static void ct_receive_tasklet_func(struct tasklet_struct *t);
98 static void ct_incoming_request_worker_func(struct work_struct *w);
99 
100 /**
101  * intel_guc_ct_init_early - Initialize CT state without requiring device access
102  * @ct: pointer to CT struct
103  */
104 void intel_guc_ct_init_early(struct intel_guc_ct *ct)
105 {
106 	spin_lock_init(&ct->ctbs.send.lock);
107 	spin_lock_init(&ct->ctbs.recv.lock);
108 	spin_lock_init(&ct->requests.lock);
109 	INIT_LIST_HEAD(&ct->requests.pending);
110 	INIT_LIST_HEAD(&ct->requests.incoming);
111 	INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);
112 	tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func);
113 	init_waitqueue_head(&ct->wq);
114 }
115 
116 static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc)
117 {
118 	memset(desc, 0, sizeof(*desc));
119 }
120 
121 static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb)
122 {
123 	u32 space;
124 
125 	ctb->broken = false;
126 	ctb->tail = 0;
127 	ctb->head = 0;
128 	space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space;
129 	atomic_set(&ctb->space, space);
130 
131 	guc_ct_buffer_desc_init(ctb->desc);
132 }
133 
134 static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb,
135 			       struct guc_ct_buffer_desc *desc,
136 			       u32 *cmds, u32 size_in_bytes, u32 resv_space)
137 {
138 	GEM_BUG_ON(size_in_bytes % 4);
139 
140 	ctb->desc = desc;
141 	ctb->cmds = cmds;
142 	ctb->size = size_in_bytes / 4;
143 	ctb->resv_space = resv_space / 4;
144 
145 	guc_ct_buffer_reset(ctb);
146 }
147 
148 static int guc_action_control_ctb(struct intel_guc *guc, u32 control)
149 {
150 	u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
151 		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
152 		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
153 		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB),
154 		FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control),
155 	};
156 	int ret;
157 
158 	GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE);
159 
160 	/* CT control must go over MMIO */
161 	ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0);
162 
163 	return ret > 0 ? -EPROTO : ret;
164 }
165 
166 static int ct_control_enable(struct intel_guc_ct *ct, bool enable)
167 {
168 	int err;
169 
170 	err = guc_action_control_ctb(ct_to_guc(ct), enable ?
171 				     GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE);
172 	if (unlikely(err))
173 		CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n",
174 			       str_enable_disable(enable), ERR_PTR(err));
175 
176 	return err;
177 }
178 
179 static int ct_register_buffer(struct intel_guc_ct *ct, bool send,
180 			      u32 desc_addr, u32 buff_addr, u32 size)
181 {
182 	int err;
183 
184 	err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
185 				   GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY :
186 				   GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
187 				   desc_addr);
188 	if (unlikely(err))
189 		goto failed;
190 
191 	err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
192 				   GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY :
193 				   GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
194 				   buff_addr);
195 	if (unlikely(err))
196 		goto failed;
197 
198 	err = intel_guc_self_cfg32(ct_to_guc(ct), send ?
199 				   GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY :
200 				   GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
201 				   size);
202 	if (unlikely(err))
203 failed:
204 		CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n",
205 			       send ? "SEND" : "RECV", ERR_PTR(err));
206 
207 	return err;
208 }
209 
210 /**
211  * intel_guc_ct_init - Init buffer-based communication
212  * @ct: pointer to CT struct
213  *
214  * Allocate memory required for buffer-based communication.
215  *
216  * Return: 0 on success, a negative errno code on failure.
217  */
218 int intel_guc_ct_init(struct intel_guc_ct *ct)
219 {
220 	struct intel_guc *guc = ct_to_guc(ct);
221 	struct guc_ct_buffer_desc *desc;
222 	u32 blob_size;
223 	u32 cmds_size;
224 	u32 resv_space;
225 	void *blob;
226 	u32 *cmds;
227 	int err;
228 
229 	err = i915_inject_probe_error(guc_to_gt(guc)->i915, -ENXIO);
230 	if (err)
231 		return err;
232 
233 	GEM_BUG_ON(ct->vma);
234 
235 	blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;
236 	err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob);
237 	if (unlikely(err)) {
238 		CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n",
239 			       blob_size, ERR_PTR(err));
240 		return err;
241 	}
242 
243 	CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size);
244 
245 	/* store pointers to desc and cmds for send ctb */
246 	desc = blob;
247 	cmds = blob + 2 * CTB_DESC_SIZE;
248 	cmds_size = CTB_H2G_BUFFER_SIZE;
249 	resv_space = 0;
250 	CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send",
251 		 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
252 		 resv_space);
253 
254 	guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space);
255 
256 	/* store pointers to desc and cmds for recv ctb */
257 	desc = blob + CTB_DESC_SIZE;
258 	cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE;
259 	cmds_size = CTB_G2H_BUFFER_SIZE;
260 	resv_space = G2H_ROOM_BUFFER_SIZE;
261 	CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv",
262 		 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
263 		 resv_space);
264 
265 	guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space);
266 
267 	return 0;
268 }
269 
270 /**
271  * intel_guc_ct_fini - Fini buffer-based communication
272  * @ct: pointer to CT struct
273  *
274  * Deallocate memory required for buffer-based communication.
275  */
276 void intel_guc_ct_fini(struct intel_guc_ct *ct)
277 {
278 	GEM_BUG_ON(ct->enabled);
279 
280 	tasklet_kill(&ct->receive_tasklet);
281 	i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP);
282 	memset(ct, 0, sizeof(*ct));
283 }
284 
285 /**
286  * intel_guc_ct_enable - Enable buffer based command transport.
287  * @ct: pointer to CT struct
288  *
289  * Return: 0 on success, a negative errno code on failure.
290  */
291 int intel_guc_ct_enable(struct intel_guc_ct *ct)
292 {
293 	struct intel_guc *guc = ct_to_guc(ct);
294 	u32 base, desc, cmds, size;
295 	void *blob;
296 	int err;
297 
298 	GEM_BUG_ON(ct->enabled);
299 
300 	/* vma should be already allocated and map'ed */
301 	GEM_BUG_ON(!ct->vma);
302 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj));
303 	base = intel_guc_ggtt_offset(guc, ct->vma);
304 
305 	/* blob should start with send descriptor */
306 	blob = __px_vaddr(ct->vma->obj);
307 	GEM_BUG_ON(blob != ct->ctbs.send.desc);
308 
309 	/* (re)initialize descriptors */
310 	guc_ct_buffer_reset(&ct->ctbs.send);
311 	guc_ct_buffer_reset(&ct->ctbs.recv);
312 
313 	/*
314 	 * Register both CT buffers starting with RECV buffer.
315 	 * Descriptors are in first half of the blob.
316 	 */
317 	desc = base + ptrdiff(ct->ctbs.recv.desc, blob);
318 	cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob);
319 	size = ct->ctbs.recv.size * 4;
320 	err = ct_register_buffer(ct, false, desc, cmds, size);
321 	if (unlikely(err))
322 		goto err_out;
323 
324 	desc = base + ptrdiff(ct->ctbs.send.desc, blob);
325 	cmds = base + ptrdiff(ct->ctbs.send.cmds, blob);
326 	size = ct->ctbs.send.size * 4;
327 	err = ct_register_buffer(ct, true, desc, cmds, size);
328 	if (unlikely(err))
329 		goto err_out;
330 
331 	err = ct_control_enable(ct, true);
332 	if (unlikely(err))
333 		goto err_out;
334 
335 	ct->enabled = true;
336 	ct->stall_time = KTIME_MAX;
337 
338 	return 0;
339 
340 err_out:
341 	CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err));
342 	return err;
343 }
344 
345 /**
346  * intel_guc_ct_disable - Disable buffer based command transport.
347  * @ct: pointer to CT struct
348  */
349 void intel_guc_ct_disable(struct intel_guc_ct *ct)
350 {
351 	struct intel_guc *guc = ct_to_guc(ct);
352 
353 	GEM_BUG_ON(!ct->enabled);
354 
355 	ct->enabled = false;
356 
357 	if (intel_guc_is_fw_running(guc)) {
358 		ct_control_enable(ct, false);
359 	}
360 }
361 
362 static u32 ct_get_next_fence(struct intel_guc_ct *ct)
363 {
364 	/* For now it's trivial */
365 	return ++ct->requests.last_fence;
366 }
367 
368 static int ct_write(struct intel_guc_ct *ct,
369 		    const u32 *action,
370 		    u32 len /* in dwords */,
371 		    u32 fence, u32 flags)
372 {
373 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
374 	struct guc_ct_buffer_desc *desc = ctb->desc;
375 	u32 tail = ctb->tail;
376 	u32 size = ctb->size;
377 	u32 header;
378 	u32 hxg;
379 	u32 type;
380 	u32 *cmds = ctb->cmds;
381 	unsigned int i;
382 
383 	if (unlikely(desc->status))
384 		goto corrupted;
385 
386 	GEM_BUG_ON(tail > size);
387 
388 #ifdef CONFIG_DRM_I915_DEBUG_GUC
389 	if (unlikely(tail != READ_ONCE(desc->tail))) {
390 		CT_ERROR(ct, "Tail was modified %u != %u\n",
391 			 desc->tail, tail);
392 		desc->status |= GUC_CTB_STATUS_MISMATCH;
393 		goto corrupted;
394 	}
395 	if (unlikely(READ_ONCE(desc->head) >= size)) {
396 		CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
397 			 desc->head, size);
398 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
399 		goto corrupted;
400 	}
401 #endif
402 
403 	/*
404 	 * dw0: CT header (including fence)
405 	 * dw1: HXG header (including action code)
406 	 * dw2+: action data
407 	 */
408 	header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
409 		 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
410 		 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence);
411 
412 	type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_EVENT :
413 		GUC_HXG_TYPE_REQUEST;
414 	hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) |
415 		FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
416 			   GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
417 
418 	CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n",
419 		 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]);
420 
421 	cmds[tail] = header;
422 	tail = (tail + 1) % size;
423 
424 	cmds[tail] = hxg;
425 	tail = (tail + 1) % size;
426 
427 	for (i = 1; i < len; i++) {
428 		cmds[tail] = action[i];
429 		tail = (tail + 1) % size;
430 	}
431 	GEM_BUG_ON(tail > size);
432 
433 	/*
434 	 * make sure H2G buffer update and LRC tail update (if this triggering a
435 	 * submission) are visible before updating the descriptor tail
436 	 */
437 	intel_guc_write_barrier(ct_to_guc(ct));
438 
439 	/* update local copies */
440 	ctb->tail = tail;
441 	GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN);
442 	atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space);
443 
444 	/* now update descriptor */
445 	WRITE_ONCE(desc->tail, tail);
446 
447 	return 0;
448 
449 corrupted:
450 	CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
451 		 desc->head, desc->tail, desc->status);
452 	ctb->broken = true;
453 	return -EPIPE;
454 }
455 
456 /**
457  * wait_for_ct_request_update - Wait for CT request state update.
458  * @ct:		pointer to CT
459  * @req:	pointer to pending request
460  * @status:	placeholder for status
461  *
462  * For each sent request, GuC shall send back CT response message.
463  * Our message handler will update status of tracked request once
464  * response message with given fence is received. Wait here and
465  * check for valid response status value.
466  *
467  * Return:
468  * *	0 response received (status is valid)
469  * *	-ETIMEDOUT no response within hardcoded timeout
470  */
471 static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status)
472 {
473 	int err;
474 	bool ct_enabled;
475 
476 	/*
477 	 * Fast commands should complete in less than 10us, so sample quickly
478 	 * up to that length of time, then switch to a slower sleep-wait loop.
479 	 * No GuC command should ever take longer than 10ms but many GuC
480 	 * commands can be inflight at time, so use a 1s timeout on the slower
481 	 * sleep-wait loop.
482 	 */
483 #define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10
484 #define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000
485 #define done \
486 	(!(ct_enabled = intel_guc_ct_enabled(ct)) || \
487 	 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \
488 	 GUC_HXG_ORIGIN_GUC)
489 	err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS);
490 	if (err)
491 		err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS);
492 #undef done
493 	if (!ct_enabled)
494 		err = -ENODEV;
495 
496 	*status = req->status;
497 	return err;
498 }
499 
500 #define GUC_CTB_TIMEOUT_MS	1500
501 static inline bool ct_deadlocked(struct intel_guc_ct *ct)
502 {
503 	long timeout = GUC_CTB_TIMEOUT_MS;
504 	bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout;
505 
506 	if (unlikely(ret)) {
507 		struct guc_ct_buffer_desc *send = ct->ctbs.send.desc;
508 		struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc;
509 
510 		CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n",
511 			 ktime_ms_delta(ktime_get(), ct->stall_time),
512 			 send->status, recv->status);
513 		CT_ERROR(ct, "H2G Space: %u (Bytes)\n",
514 			 atomic_read(&ct->ctbs.send.space) * 4);
515 		CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head);
516 		CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail);
517 		CT_ERROR(ct, "G2H Space: %u (Bytes)\n",
518 			 atomic_read(&ct->ctbs.recv.space) * 4);
519 		CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head);
520 		CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail);
521 
522 		ct->ctbs.send.broken = true;
523 	}
524 
525 	return ret;
526 }
527 
528 static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw)
529 {
530 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
531 
532 	/*
533 	 * We leave a certain amount of space in the G2H CTB buffer for
534 	 * unexpected G2H CTBs (e.g. logging, engine hang, etc...)
535 	 */
536 	return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw;
537 }
538 
539 static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
540 {
541 	lockdep_assert_held(&ct->ctbs.send.lock);
542 
543 	GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw));
544 
545 	if (g2h_len_dw)
546 		atomic_sub(g2h_len_dw, &ct->ctbs.recv.space);
547 }
548 
549 static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
550 {
551 	atomic_add(g2h_len_dw, &ct->ctbs.recv.space);
552 }
553 
554 static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw)
555 {
556 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
557 	struct guc_ct_buffer_desc *desc = ctb->desc;
558 	u32 head;
559 	u32 space;
560 
561 	if (atomic_read(&ctb->space) >= len_dw)
562 		return true;
563 
564 	head = READ_ONCE(desc->head);
565 	if (unlikely(head > ctb->size)) {
566 		CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
567 			 head, ctb->size);
568 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
569 		ctb->broken = true;
570 		return false;
571 	}
572 
573 	space = CIRC_SPACE(ctb->tail, head, ctb->size);
574 	atomic_set(&ctb->space, space);
575 
576 	return space >= len_dw;
577 }
578 
579 static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw)
580 {
581 	bool h2g = h2g_has_room(ct, h2g_dw);
582 	bool g2h = g2h_has_room(ct, g2h_dw);
583 
584 	lockdep_assert_held(&ct->ctbs.send.lock);
585 
586 	if (unlikely(!h2g || !g2h)) {
587 		if (ct->stall_time == KTIME_MAX)
588 			ct->stall_time = ktime_get();
589 
590 		/* Be paranoid and kick G2H tasklet to free credits */
591 		if (!g2h)
592 			tasklet_hi_schedule(&ct->receive_tasklet);
593 
594 		if (unlikely(ct_deadlocked(ct)))
595 			return -EPIPE;
596 		else
597 			return -EBUSY;
598 	}
599 
600 	ct->stall_time = KTIME_MAX;
601 	return 0;
602 }
603 
604 #define G2H_LEN_DW(f) ({ \
605 	typeof(f) f_ = (f); \
606 	FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \
607 	FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \
608 	GUC_CTB_HXG_MSG_MIN_LEN : 0; \
609 })
610 static int ct_send_nb(struct intel_guc_ct *ct,
611 		      const u32 *action,
612 		      u32 len,
613 		      u32 flags)
614 {
615 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
616 	unsigned long spin_flags;
617 	u32 g2h_len_dw = G2H_LEN_DW(flags);
618 	u32 fence;
619 	int ret;
620 
621 	spin_lock_irqsave(&ctb->lock, spin_flags);
622 
623 	ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw);
624 	if (unlikely(ret))
625 		goto out;
626 
627 	fence = ct_get_next_fence(ct);
628 	ret = ct_write(ct, action, len, fence, flags);
629 	if (unlikely(ret))
630 		goto out;
631 
632 	g2h_reserve_space(ct, g2h_len_dw);
633 	intel_guc_notify(ct_to_guc(ct));
634 
635 out:
636 	spin_unlock_irqrestore(&ctb->lock, spin_flags);
637 
638 	return ret;
639 }
640 
641 static int ct_send(struct intel_guc_ct *ct,
642 		   const u32 *action,
643 		   u32 len,
644 		   u32 *response_buf,
645 		   u32 response_buf_size,
646 		   u32 *status)
647 {
648 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
649 	struct ct_request request;
650 	unsigned long flags;
651 	unsigned int sleep_period_ms = 1;
652 	bool send_again;
653 	u32 fence;
654 	int err;
655 
656 	GEM_BUG_ON(!ct->enabled);
657 	GEM_BUG_ON(!len);
658 	GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK);
659 	GEM_BUG_ON(!response_buf && response_buf_size);
660 	might_sleep();
661 
662 resend:
663 	send_again = false;
664 
665 	/*
666 	 * We use a lazy spin wait loop here as we believe that if the CT
667 	 * buffers are sized correctly the flow control condition should be
668 	 * rare. Reserving the maximum size in the G2H credits as we don't know
669 	 * how big the response is going to be.
670 	 */
671 retry:
672 	spin_lock_irqsave(&ctb->lock, flags);
673 	if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) ||
674 		     !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) {
675 		if (ct->stall_time == KTIME_MAX)
676 			ct->stall_time = ktime_get();
677 		spin_unlock_irqrestore(&ctb->lock, flags);
678 
679 		if (unlikely(ct_deadlocked(ct)))
680 			return -EPIPE;
681 
682 		if (msleep_interruptible(sleep_period_ms))
683 			return -EINTR;
684 		sleep_period_ms = sleep_period_ms << 1;
685 
686 		goto retry;
687 	}
688 
689 	ct->stall_time = KTIME_MAX;
690 
691 	fence = ct_get_next_fence(ct);
692 	request.fence = fence;
693 	request.status = 0;
694 	request.response_len = response_buf_size;
695 	request.response_buf = response_buf;
696 
697 	spin_lock(&ct->requests.lock);
698 	list_add_tail(&request.link, &ct->requests.pending);
699 	spin_unlock(&ct->requests.lock);
700 
701 	err = ct_write(ct, action, len, fence, 0);
702 	g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
703 
704 	spin_unlock_irqrestore(&ctb->lock, flags);
705 
706 	if (unlikely(err))
707 		goto unlink;
708 
709 	intel_guc_notify(ct_to_guc(ct));
710 
711 	err = wait_for_ct_request_update(ct, &request, status);
712 	g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
713 	if (unlikely(err)) {
714 		if (err == -ENODEV)
715 			/* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress.
716 			 * In this case, output is debug rather than error info
717 			 */
718 			CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n",
719 				 action[0], request.fence);
720 		else
721 			CT_ERROR(ct, "No response for request %#x (fence %u)\n",
722 				 action[0], request.fence);
723 		goto unlink;
724 	}
725 
726 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
727 		CT_DEBUG(ct, "retrying request %#x (%u)\n", *action,
728 			 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status));
729 		send_again = true;
730 		goto unlink;
731 	}
732 
733 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) {
734 		err = -EIO;
735 		goto unlink;
736 	}
737 
738 	if (response_buf) {
739 		/* There shall be no data in the status */
740 		WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status));
741 		/* Return actual response len */
742 		err = request.response_len;
743 	} else {
744 		/* There shall be no response payload */
745 		WARN_ON(request.response_len);
746 		/* Return data decoded from the status dword */
747 		err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status);
748 	}
749 
750 unlink:
751 	spin_lock_irqsave(&ct->requests.lock, flags);
752 	list_del(&request.link);
753 	spin_unlock_irqrestore(&ct->requests.lock, flags);
754 
755 	if (unlikely(send_again))
756 		goto resend;
757 
758 	return err;
759 }
760 
761 /*
762  * Command Transport (CT) buffer based GuC send function.
763  */
764 int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
765 		      u32 *response_buf, u32 response_buf_size, u32 flags)
766 {
767 	u32 status = ~0; /* undefined */
768 	int ret;
769 
770 	if (unlikely(!ct->enabled)) {
771 		struct intel_guc *guc = ct_to_guc(ct);
772 		struct intel_uc *uc = container_of(guc, struct intel_uc, guc);
773 
774 		WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action);
775 		return -ENODEV;
776 	}
777 
778 	if (unlikely(ct->ctbs.send.broken))
779 		return -EPIPE;
780 
781 	if (flags & INTEL_GUC_CT_SEND_NB)
782 		return ct_send_nb(ct, action, len, flags);
783 
784 	ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);
785 	if (unlikely(ret < 0)) {
786 		if (ret != -ENODEV)
787 			CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",
788 				 action[0], ERR_PTR(ret), status);
789 	} else if (unlikely(ret)) {
790 		CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
791 			 action[0], ret, ret);
792 	}
793 
794 	return ret;
795 }
796 
797 static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords)
798 {
799 	struct ct_incoming_msg *msg;
800 
801 	msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC);
802 	if (msg)
803 		msg->size = num_dwords;
804 	return msg;
805 }
806 
807 static void ct_free_msg(struct ct_incoming_msg *msg)
808 {
809 	kfree(msg);
810 }
811 
812 /*
813  * Return: number available remaining dwords to read (0 if empty)
814  *         or a negative error code on failure
815  */
816 static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)
817 {
818 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
819 	struct guc_ct_buffer_desc *desc = ctb->desc;
820 	u32 head = ctb->head;
821 	u32 tail = READ_ONCE(desc->tail);
822 	u32 size = ctb->size;
823 	u32 *cmds = ctb->cmds;
824 	s32 available;
825 	unsigned int len;
826 	unsigned int i;
827 	u32 header;
828 
829 	if (unlikely(ctb->broken))
830 		return -EPIPE;
831 
832 	if (unlikely(desc->status)) {
833 		u32 status = desc->status;
834 
835 		if (status & GUC_CTB_STATUS_UNUSED) {
836 			/*
837 			 * Potentially valid if a CLIENT_RESET request resulted in
838 			 * contexts/engines being reset. But should never happen as
839 			 * no contexts should be active when CLIENT_RESET is sent.
840 			 */
841 			CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n");
842 			status &= ~GUC_CTB_STATUS_UNUSED;
843 		}
844 
845 		if (status)
846 			goto corrupted;
847 	}
848 
849 	GEM_BUG_ON(head > size);
850 
851 #ifdef CONFIG_DRM_I915_DEBUG_GUC
852 	if (unlikely(head != READ_ONCE(desc->head))) {
853 		CT_ERROR(ct, "Head was modified %u != %u\n",
854 			 desc->head, head);
855 		desc->status |= GUC_CTB_STATUS_MISMATCH;
856 		goto corrupted;
857 	}
858 #endif
859 	if (unlikely(tail >= size)) {
860 		CT_ERROR(ct, "Invalid tail offset %u >= %u)\n",
861 			 tail, size);
862 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
863 		goto corrupted;
864 	}
865 
866 	/* tail == head condition indicates empty */
867 	available = tail - head;
868 	if (unlikely(available == 0)) {
869 		*msg = NULL;
870 		return 0;
871 	}
872 
873 	/* beware of buffer wrap case */
874 	if (unlikely(available < 0))
875 		available += size;
876 	CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size);
877 	GEM_BUG_ON(available < 0);
878 
879 	header = cmds[head];
880 	head = (head + 1) % size;
881 
882 	/* message len with header */
883 	len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN;
884 	if (unlikely(len > (u32)available)) {
885 		CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",
886 			 4, &header,
887 			 4 * (head + available - 1 > size ?
888 			      size - head : available - 1), &cmds[head],
889 			 4 * (head + available - 1 > size ?
890 			      available - 1 - size + head : 0), &cmds[0]);
891 		desc->status |= GUC_CTB_STATUS_UNDERFLOW;
892 		goto corrupted;
893 	}
894 
895 	*msg = ct_alloc_msg(len);
896 	if (!*msg) {
897 		CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n",
898 			 4, &header,
899 			 4 * (head + available - 1 > size ?
900 			      size - head : available - 1), &cmds[head],
901 			 4 * (head + available - 1 > size ?
902 			      available - 1 - size + head : 0), &cmds[0]);
903 		return available;
904 	}
905 
906 	(*msg)->msg[0] = header;
907 
908 	for (i = 1; i < len; i++) {
909 		(*msg)->msg[i] = cmds[head];
910 		head = (head + 1) % size;
911 	}
912 	CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg);
913 
914 	/* update local copies */
915 	ctb->head = head;
916 
917 	/* now update descriptor */
918 	WRITE_ONCE(desc->head, head);
919 
920 	return available - len;
921 
922 corrupted:
923 	CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
924 		 desc->head, desc->tail, desc->status);
925 	ctb->broken = true;
926 	return -EPIPE;
927 }
928 
929 static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response)
930 {
931 	u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]);
932 	u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]);
933 	const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN];
934 	const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN];
935 	u32 datalen = len - GUC_HXG_MSG_MIN_LEN;
936 	struct ct_request *req;
937 	unsigned long flags;
938 	bool found = false;
939 	int err = 0;
940 
941 	GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN);
942 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC);
943 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS &&
944 		   FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY &&
945 		   FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE);
946 
947 	CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]);
948 
949 	spin_lock_irqsave(&ct->requests.lock, flags);
950 	list_for_each_entry(req, &ct->requests.pending, link) {
951 		if (unlikely(fence != req->fence)) {
952 			CT_DEBUG(ct, "request %u awaits response\n",
953 				 req->fence);
954 			continue;
955 		}
956 		if (unlikely(datalen > req->response_len)) {
957 			CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n",
958 				 req->fence, datalen, req->response_len);
959 			datalen = min(datalen, req->response_len);
960 			err = -EMSGSIZE;
961 		}
962 		if (datalen)
963 			memcpy(req->response_buf, data, 4 * datalen);
964 		req->response_len = datalen;
965 		WRITE_ONCE(req->status, hxg[0]);
966 		found = true;
967 		break;
968 	}
969 	if (!found) {
970 		CT_ERROR(ct, "Unsolicited response (fence %u)\n", fence);
971 		CT_ERROR(ct, "Could not find fence=%u, last_fence=%u\n", fence,
972 			 ct->requests.last_fence);
973 		list_for_each_entry(req, &ct->requests.pending, link)
974 			CT_ERROR(ct, "request %u awaits response\n",
975 				 req->fence);
976 		err = -ENOKEY;
977 	}
978 	spin_unlock_irqrestore(&ct->requests.lock, flags);
979 
980 	if (unlikely(err))
981 		return err;
982 
983 	ct_free_msg(response);
984 	return 0;
985 }
986 
987 static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
988 {
989 	struct intel_guc *guc = ct_to_guc(ct);
990 	const u32 *hxg;
991 	const u32 *payload;
992 	u32 hxg_len, action, len;
993 	int ret;
994 
995 	hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
996 	hxg_len = request->size - GUC_CTB_MSG_MIN_LEN;
997 	payload = &hxg[GUC_HXG_MSG_MIN_LEN];
998 	action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
999 	len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1000 
1001 	CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);
1002 
1003 	switch (action) {
1004 	case INTEL_GUC_ACTION_DEFAULT:
1005 		ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
1006 		break;
1007 	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1008 		ret = intel_guc_deregister_done_process_msg(guc, payload,
1009 							    len);
1010 		break;
1011 	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1012 		ret = intel_guc_sched_done_process_msg(guc, payload, len);
1013 		break;
1014 	case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1015 		ret = intel_guc_context_reset_process_msg(guc, payload, len);
1016 		break;
1017 	case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1018 		ret = intel_guc_error_capture_process_msg(guc, payload, len);
1019 		if (unlikely(ret))
1020 			CT_ERROR(ct, "error capture notification failed %x %*ph\n",
1021 				 action, 4 * len, payload);
1022 		break;
1023 	case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1024 		ret = intel_guc_engine_failure_process_msg(guc, payload, len);
1025 		break;
1026 	case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1027 		intel_guc_log_handle_flush_event(&guc->log);
1028 		ret = 0;
1029 		break;
1030 	case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1031 		CT_ERROR(ct, "Received GuC crash dump notification!\n");
1032 		ret = 0;
1033 		break;
1034 	case INTEL_GUC_ACTION_NOTIFY_EXCEPTION:
1035 		CT_ERROR(ct, "Received GuC exception notification!\n");
1036 		ret = 0;
1037 		break;
1038 	default:
1039 		ret = -EOPNOTSUPP;
1040 		break;
1041 	}
1042 
1043 	if (unlikely(ret)) {
1044 		CT_ERROR(ct, "Failed to process request %04x (%pe)\n",
1045 			 action, ERR_PTR(ret));
1046 		return ret;
1047 	}
1048 
1049 	ct_free_msg(request);
1050 	return 0;
1051 }
1052 
1053 static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
1054 {
1055 	unsigned long flags;
1056 	struct ct_incoming_msg *request;
1057 	bool done;
1058 	int err;
1059 
1060 	spin_lock_irqsave(&ct->requests.lock, flags);
1061 	request = list_first_entry_or_null(&ct->requests.incoming,
1062 					   struct ct_incoming_msg, link);
1063 	if (request)
1064 		list_del(&request->link);
1065 	done = !!list_empty(&ct->requests.incoming);
1066 	spin_unlock_irqrestore(&ct->requests.lock, flags);
1067 
1068 	if (!request)
1069 		return true;
1070 
1071 	err = ct_process_request(ct, request);
1072 	if (unlikely(err)) {
1073 		CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1074 			 ERR_PTR(err), 4 * request->size, request->msg);
1075 		ct_free_msg(request);
1076 	}
1077 
1078 	return done;
1079 }
1080 
1081 static void ct_incoming_request_worker_func(struct work_struct *w)
1082 {
1083 	struct intel_guc_ct *ct =
1084 		container_of(w, struct intel_guc_ct, requests.worker);
1085 	bool done;
1086 
1087 	do {
1088 		done = ct_process_incoming_requests(ct);
1089 	} while (!done);
1090 }
1091 
1092 static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1093 {
1094 	const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1095 	u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1096 	unsigned long flags;
1097 
1098 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT);
1099 
1100 	/*
1101 	 * Adjusting the space must be done in IRQ or deadlock can occur as the
1102 	 * CTB processing in the below workqueue can send CTBs which creates a
1103 	 * circular dependency if the space was returned there.
1104 	 */
1105 	switch (action) {
1106 	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1107 	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1108 		g2h_release_space(ct, request->size);
1109 	}
1110 
1111 	spin_lock_irqsave(&ct->requests.lock, flags);
1112 	list_add_tail(&request->link, &ct->requests.incoming);
1113 	spin_unlock_irqrestore(&ct->requests.lock, flags);
1114 
1115 	queue_work(system_unbound_wq, &ct->requests.worker);
1116 	return 0;
1117 }
1118 
1119 static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1120 {
1121 	u32 origin, type;
1122 	u32 *hxg;
1123 	int err;
1124 
1125 	if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN))
1126 		return -EBADMSG;
1127 
1128 	hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN];
1129 
1130 	origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1131 	if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1132 		err = -EPROTO;
1133 		goto failed;
1134 	}
1135 
1136 	type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1137 	switch (type) {
1138 	case GUC_HXG_TYPE_EVENT:
1139 		err = ct_handle_event(ct, msg);
1140 		break;
1141 	case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1142 	case GUC_HXG_TYPE_RESPONSE_FAILURE:
1143 	case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1144 		err = ct_handle_response(ct, msg);
1145 		break;
1146 	default:
1147 		err = -EOPNOTSUPP;
1148 	}
1149 
1150 	if (unlikely(err)) {
1151 failed:
1152 		CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n",
1153 			 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg);
1154 	}
1155 	return err;
1156 }
1157 
1158 static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1159 {
1160 	u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]);
1161 	int err;
1162 
1163 	if (format == GUC_CTB_FORMAT_HXG)
1164 		err = ct_handle_hxg(ct, msg);
1165 	else
1166 		err = -EOPNOTSUPP;
1167 
1168 	if (unlikely(err)) {
1169 		CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1170 			 ERR_PTR(err), 4 * msg->size, msg->msg);
1171 		ct_free_msg(msg);
1172 	}
1173 }
1174 
1175 /*
1176  * Return: number available remaining dwords to read (0 if empty)
1177  *         or a negative error code on failure
1178  */
1179 static int ct_receive(struct intel_guc_ct *ct)
1180 {
1181 	struct ct_incoming_msg *msg = NULL;
1182 	unsigned long flags;
1183 	int ret;
1184 
1185 	spin_lock_irqsave(&ct->ctbs.recv.lock, flags);
1186 	ret = ct_read(ct, &msg);
1187 	spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags);
1188 	if (ret < 0)
1189 		return ret;
1190 
1191 	if (msg)
1192 		ct_handle_msg(ct, msg);
1193 
1194 	return ret;
1195 }
1196 
1197 static void ct_try_receive_message(struct intel_guc_ct *ct)
1198 {
1199 	int ret;
1200 
1201 	if (GEM_WARN_ON(!ct->enabled))
1202 		return;
1203 
1204 	ret = ct_receive(ct);
1205 	if (ret > 0)
1206 		tasklet_hi_schedule(&ct->receive_tasklet);
1207 }
1208 
1209 static void ct_receive_tasklet_func(struct tasklet_struct *t)
1210 {
1211 	struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet);
1212 
1213 	ct_try_receive_message(ct);
1214 }
1215 
1216 /*
1217  * When we're communicating with the GuC over CT, GuC uses events
1218  * to notify us about new messages being posted on the RECV buffer.
1219  */
1220 void intel_guc_ct_event_handler(struct intel_guc_ct *ct)
1221 {
1222 	if (unlikely(!ct->enabled)) {
1223 		WARN(1, "Unexpected GuC event received while CT disabled!\n");
1224 		return;
1225 	}
1226 
1227 	ct_try_receive_message(ct);
1228 }
1229 
1230 void intel_guc_ct_print_info(struct intel_guc_ct *ct,
1231 			     struct drm_printer *p)
1232 {
1233 	drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled));
1234 
1235 	if (!ct->enabled)
1236 		return;
1237 
1238 	drm_printf(p, "H2G Space: %u\n",
1239 		   atomic_read(&ct->ctbs.send.space) * 4);
1240 	drm_printf(p, "Head: %u\n",
1241 		   ct->ctbs.send.desc->head);
1242 	drm_printf(p, "Tail: %u\n",
1243 		   ct->ctbs.send.desc->tail);
1244 	drm_printf(p, "G2H Space: %u\n",
1245 		   atomic_read(&ct->ctbs.recv.space) * 4);
1246 	drm_printf(p, "Head: %u\n",
1247 		   ct->ctbs.recv.desc->head);
1248 	drm_printf(p, "Tail: %u\n",
1249 		   ct->ctbs.recv.desc->tail);
1250 }
1251