1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
4  * Copyright (C) 2017 Linaro Ltd.
5  */
6 
7 #include <linux/delay.h>
8 #include <linux/device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/interrupt.h>
11 #include <linux/iopoll.h>
12 #include <linux/kernel.h>
13 #include <linux/slab.h>
14 
15 #include "core.h"
16 #include "hfi_cmds.h"
17 #include "hfi_msgs.h"
18 #include "hfi_venus.h"
19 #include "hfi_venus_io.h"
20 #include "firmware.h"
21 
22 #define HFI_MASK_QHDR_TX_TYPE		0xff000000
23 #define HFI_MASK_QHDR_RX_TYPE		0x00ff0000
24 #define HFI_MASK_QHDR_PRI_TYPE		0x0000ff00
25 #define HFI_MASK_QHDR_ID_TYPE		0x000000ff
26 
27 #define HFI_HOST_TO_CTRL_CMD_Q		0
28 #define HFI_CTRL_TO_HOST_MSG_Q		1
29 #define HFI_CTRL_TO_HOST_DBG_Q		2
30 #define HFI_MASK_QHDR_STATUS		0x000000ff
31 
32 #define IFACEQ_NUM			3
33 #define IFACEQ_CMD_IDX			0
34 #define IFACEQ_MSG_IDX			1
35 #define IFACEQ_DBG_IDX			2
36 #define IFACEQ_MAX_BUF_COUNT		50
37 #define IFACEQ_MAX_PARALLEL_CLNTS	16
38 #define IFACEQ_DFLT_QHDR		0x01010000
39 
40 #define POLL_INTERVAL_US		50
41 
42 #define IFACEQ_MAX_PKT_SIZE		1024
43 #define IFACEQ_MED_PKT_SIZE		768
44 #define IFACEQ_MIN_PKT_SIZE		8
45 #define IFACEQ_VAR_SMALL_PKT_SIZE	100
46 #define IFACEQ_VAR_LARGE_PKT_SIZE	512
47 #define IFACEQ_VAR_HUGE_PKT_SIZE	(1024 * 12)
48 
49 struct hfi_queue_table_header {
50 	u32 version;
51 	u32 size;
52 	u32 qhdr0_offset;
53 	u32 qhdr_size;
54 	u32 num_q;
55 	u32 num_active_q;
56 };
57 
58 struct hfi_queue_header {
59 	u32 status;
60 	u32 start_addr;
61 	u32 type;
62 	u32 q_size;
63 	u32 pkt_size;
64 	u32 pkt_drop_cnt;
65 	u32 rx_wm;
66 	u32 tx_wm;
67 	u32 rx_req;
68 	u32 tx_req;
69 	u32 rx_irq_status;
70 	u32 tx_irq_status;
71 	u32 read_idx;
72 	u32 write_idx;
73 };
74 
75 #define IFACEQ_TABLE_SIZE	\
76 	(sizeof(struct hfi_queue_table_header) +	\
77 	 sizeof(struct hfi_queue_header) * IFACEQ_NUM)
78 
79 #define IFACEQ_QUEUE_SIZE	(IFACEQ_MAX_PKT_SIZE *	\
80 	IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
81 
82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i)	\
83 	(void *)(((ptr) + sizeof(struct hfi_queue_table_header)) +	\
84 		((i) * sizeof(struct hfi_queue_header)))
85 
86 #define QDSS_SIZE		SZ_4K
87 #define SFR_SIZE		SZ_4K
88 #define QUEUE_SIZE		\
89 	(IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
90 
91 #define ALIGNED_QDSS_SIZE	ALIGN(QDSS_SIZE, SZ_4K)
92 #define ALIGNED_SFR_SIZE	ALIGN(SFR_SIZE, SZ_4K)
93 #define ALIGNED_QUEUE_SIZE	ALIGN(QUEUE_SIZE, SZ_4K)
94 #define SHARED_QSIZE		ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
95 				      ALIGNED_QDSS_SIZE, SZ_1M)
96 
97 struct mem_desc {
98 	dma_addr_t da;	/* device address */
99 	void *kva;	/* kernel virtual address */
100 	u32 size;
101 	unsigned long attrs;
102 };
103 
104 struct iface_queue {
105 	struct hfi_queue_header *qhdr;
106 	struct mem_desc qmem;
107 };
108 
109 enum venus_state {
110 	VENUS_STATE_DEINIT = 1,
111 	VENUS_STATE_INIT,
112 };
113 
114 struct venus_hfi_device {
115 	struct venus_core *core;
116 	u32 irq_status;
117 	u32 last_packet_type;
118 	bool power_enabled;
119 	bool suspended;
120 	enum venus_state state;
121 	/* serialize read / write to the shared memory */
122 	struct mutex lock;
123 	struct completion pwr_collapse_prep;
124 	struct completion release_resource;
125 	struct mem_desc ifaceq_table;
126 	struct mem_desc sfr;
127 	struct iface_queue queues[IFACEQ_NUM];
128 	u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
129 	u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
130 };
131 
132 static bool venus_pkt_debug;
133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
134 static bool venus_sys_idle_indicator;
135 static bool venus_fw_low_power_mode = true;
136 static int venus_hw_rsp_timeout = 1000;
137 static bool venus_fw_coverage;
138 
139 static void venus_set_state(struct venus_hfi_device *hdev,
140 			    enum venus_state state)
141 {
142 	mutex_lock(&hdev->lock);
143 	hdev->state = state;
144 	mutex_unlock(&hdev->lock);
145 }
146 
147 static bool venus_is_valid_state(struct venus_hfi_device *hdev)
148 {
149 	return hdev->state != VENUS_STATE_DEINIT;
150 }
151 
152 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
153 {
154 	size_t pkt_size = *(u32 *)packet;
155 
156 	if (!venus_pkt_debug)
157 		return;
158 
159 	print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
160 		       pkt_size, true);
161 }
162 
163 static int venus_write_queue(struct venus_hfi_device *hdev,
164 			     struct iface_queue *queue,
165 			     void *packet, u32 *rx_req)
166 {
167 	struct hfi_queue_header *qhdr;
168 	u32 dwords, new_wr_idx;
169 	u32 empty_space, rd_idx, wr_idx, qsize;
170 	u32 *wr_ptr;
171 
172 	if (!queue->qmem.kva)
173 		return -EINVAL;
174 
175 	qhdr = queue->qhdr;
176 	if (!qhdr)
177 		return -EINVAL;
178 
179 	venus_dump_packet(hdev, packet);
180 
181 	dwords = (*(u32 *)packet) >> 2;
182 	if (!dwords)
183 		return -EINVAL;
184 
185 	rd_idx = qhdr->read_idx;
186 	wr_idx = qhdr->write_idx;
187 	qsize = qhdr->q_size;
188 	/* ensure rd/wr indices's are read from memory */
189 	rmb();
190 
191 	if (wr_idx >= rd_idx)
192 		empty_space = qsize - (wr_idx - rd_idx);
193 	else
194 		empty_space = rd_idx - wr_idx;
195 
196 	if (empty_space <= dwords) {
197 		qhdr->tx_req = 1;
198 		/* ensure tx_req is updated in memory */
199 		wmb();
200 		return -ENOSPC;
201 	}
202 
203 	qhdr->tx_req = 0;
204 	/* ensure tx_req is updated in memory */
205 	wmb();
206 
207 	new_wr_idx = wr_idx + dwords;
208 	wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
209 	if (new_wr_idx < qsize) {
210 		memcpy(wr_ptr, packet, dwords << 2);
211 	} else {
212 		size_t len;
213 
214 		new_wr_idx -= qsize;
215 		len = (dwords - new_wr_idx) << 2;
216 		memcpy(wr_ptr, packet, len);
217 		memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
218 	}
219 
220 	/* make sure packet is written before updating the write index */
221 	wmb();
222 
223 	qhdr->write_idx = new_wr_idx;
224 	*rx_req = qhdr->rx_req ? 1 : 0;
225 
226 	/* make sure write index is updated before an interrupt is raised */
227 	mb();
228 
229 	return 0;
230 }
231 
232 static int venus_read_queue(struct venus_hfi_device *hdev,
233 			    struct iface_queue *queue, void *pkt, u32 *tx_req)
234 {
235 	struct hfi_queue_header *qhdr;
236 	u32 dwords, new_rd_idx;
237 	u32 rd_idx, wr_idx, type, qsize;
238 	u32 *rd_ptr;
239 	u32 recv_request = 0;
240 	int ret = 0;
241 
242 	if (!queue->qmem.kva)
243 		return -EINVAL;
244 
245 	qhdr = queue->qhdr;
246 	if (!qhdr)
247 		return -EINVAL;
248 
249 	type = qhdr->type;
250 	rd_idx = qhdr->read_idx;
251 	wr_idx = qhdr->write_idx;
252 	qsize = qhdr->q_size;
253 
254 	/* make sure data is valid before using it */
255 	rmb();
256 
257 	/*
258 	 * Do not set receive request for debug queue, if set, Venus generates
259 	 * interrupt for debug messages even when there is no response message
260 	 * available. In general debug queue will not become full as it is being
261 	 * emptied out for every interrupt from Venus. Venus will anyway
262 	 * generates interrupt if it is full.
263 	 */
264 	if (type & HFI_CTRL_TO_HOST_MSG_Q)
265 		recv_request = 1;
266 
267 	if (rd_idx == wr_idx) {
268 		qhdr->rx_req = recv_request;
269 		*tx_req = 0;
270 		/* update rx_req field in memory */
271 		wmb();
272 		return -ENODATA;
273 	}
274 
275 	rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
276 	dwords = *rd_ptr >> 2;
277 	if (!dwords)
278 		return -EINVAL;
279 
280 	new_rd_idx = rd_idx + dwords;
281 	if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
282 		if (new_rd_idx < qsize) {
283 			memcpy(pkt, rd_ptr, dwords << 2);
284 		} else {
285 			size_t len;
286 
287 			new_rd_idx -= qsize;
288 			len = (dwords - new_rd_idx) << 2;
289 			memcpy(pkt, rd_ptr, len);
290 			memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
291 		}
292 	} else {
293 		/* bad packet received, dropping */
294 		new_rd_idx = qhdr->write_idx;
295 		ret = -EBADMSG;
296 	}
297 
298 	/* ensure the packet is read before updating read index */
299 	rmb();
300 
301 	qhdr->read_idx = new_rd_idx;
302 	/* ensure updating read index */
303 	wmb();
304 
305 	rd_idx = qhdr->read_idx;
306 	wr_idx = qhdr->write_idx;
307 	/* ensure rd/wr indices are read from memory */
308 	rmb();
309 
310 	if (rd_idx != wr_idx)
311 		qhdr->rx_req = 0;
312 	else
313 		qhdr->rx_req = recv_request;
314 
315 	*tx_req = qhdr->tx_req ? 1 : 0;
316 
317 	/* ensure rx_req is stored to memory and tx_req is loaded from memory */
318 	mb();
319 
320 	venus_dump_packet(hdev, pkt);
321 
322 	return ret;
323 }
324 
325 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
326 		       u32 size)
327 {
328 	struct device *dev = hdev->core->dev;
329 
330 	desc->attrs = DMA_ATTR_WRITE_COMBINE;
331 	desc->size = ALIGN(size, SZ_4K);
332 
333 	desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
334 				    desc->attrs);
335 	if (!desc->kva)
336 		return -ENOMEM;
337 
338 	return 0;
339 }
340 
341 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
342 {
343 	struct device *dev = hdev->core->dev;
344 
345 	dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
346 }
347 
348 static void venus_set_registers(struct venus_hfi_device *hdev)
349 {
350 	const struct venus_resources *res = hdev->core->res;
351 	const struct reg_val *tbl = res->reg_tbl;
352 	unsigned int count = res->reg_tbl_size;
353 	unsigned int i;
354 
355 	for (i = 0; i < count; i++)
356 		writel(tbl[i].value, hdev->core->base + tbl[i].reg);
357 }
358 
359 static void venus_soft_int(struct venus_hfi_device *hdev)
360 {
361 	void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;
362 	u32 clear_bit;
363 
364 	if (IS_V6(hdev->core))
365 		clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);
366 	else
367 		clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);
368 
369 	writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);
370 }
371 
372 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
373 					 void *pkt, bool sync)
374 {
375 	struct device *dev = hdev->core->dev;
376 	struct hfi_pkt_hdr *cmd_packet;
377 	struct iface_queue *queue;
378 	u32 rx_req;
379 	int ret;
380 
381 	if (!venus_is_valid_state(hdev))
382 		return -EINVAL;
383 
384 	cmd_packet = (struct hfi_pkt_hdr *)pkt;
385 	hdev->last_packet_type = cmd_packet->pkt_type;
386 
387 	queue = &hdev->queues[IFACEQ_CMD_IDX];
388 
389 	ret = venus_write_queue(hdev, queue, pkt, &rx_req);
390 	if (ret) {
391 		dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
392 		return ret;
393 	}
394 
395 	if (sync) {
396 		/*
397 		 * Inform video hardware to raise interrupt for synchronous
398 		 * commands
399 		 */
400 		queue = &hdev->queues[IFACEQ_MSG_IDX];
401 		queue->qhdr->rx_req = 1;
402 		/* ensure rx_req is updated in memory */
403 		wmb();
404 	}
405 
406 	if (rx_req)
407 		venus_soft_int(hdev);
408 
409 	return 0;
410 }
411 
412 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)
413 {
414 	int ret;
415 
416 	mutex_lock(&hdev->lock);
417 	ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);
418 	mutex_unlock(&hdev->lock);
419 
420 	return ret;
421 }
422 
423 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
424 				       u32 size, u32 addr, void *cookie)
425 {
426 	struct venus_hfi_device *hdev = to_hfi_priv(core);
427 	struct hfi_sys_set_resource_pkt *pkt;
428 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
429 	int ret;
430 
431 	if (id == VIDC_RESOURCE_NONE)
432 		return 0;
433 
434 	pkt = (struct hfi_sys_set_resource_pkt *)packet;
435 
436 	ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
437 	if (ret)
438 		return ret;
439 
440 	ret = venus_iface_cmdq_write(hdev, pkt, false);
441 	if (ret)
442 		return ret;
443 
444 	return 0;
445 }
446 
447 static int venus_boot_core(struct venus_hfi_device *hdev)
448 {
449 	struct device *dev = hdev->core->dev;
450 	static const unsigned int max_tries = 100;
451 	u32 ctrl_status = 0, mask_val;
452 	unsigned int count = 0;
453 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
454 	void __iomem *wrapper_base = hdev->core->wrapper_base;
455 	int ret = 0;
456 
457 	writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);
458 	if (IS_V6(hdev->core)) {
459 		mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);
460 		mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |
461 			      WRAPPER_INTR_MASK_A2HCPU_MASK);
462 	} else {
463 		mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;
464 	}
465 	writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);
466 	writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);
467 
468 	while (!ctrl_status && count < max_tries) {
469 		ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
470 		if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
471 			dev_err(dev, "invalid setting for UC_REGION\n");
472 			ret = -EINVAL;
473 			break;
474 		}
475 
476 		usleep_range(500, 1000);
477 		count++;
478 	}
479 
480 	if (count >= max_tries)
481 		ret = -ETIMEDOUT;
482 
483 	if (IS_V6(hdev->core)) {
484 		writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);
485 		writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);
486 	}
487 
488 	return ret;
489 }
490 
491 static u32 venus_hwversion(struct venus_hfi_device *hdev)
492 {
493 	struct device *dev = hdev->core->dev;
494 	void __iomem *wrapper_base = hdev->core->wrapper_base;
495 	u32 ver;
496 	u32 major, minor, step;
497 
498 	ver = readl(wrapper_base + WRAPPER_HW_VERSION);
499 	major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
500 	major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
501 	minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
502 	minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
503 	step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
504 
505 	dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);
506 
507 	return major;
508 }
509 
510 static int venus_run(struct venus_hfi_device *hdev)
511 {
512 	struct device *dev = hdev->core->dev;
513 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
514 	int ret;
515 
516 	/*
517 	 * Re-program all of the registers that get reset as a result of
518 	 * regulator_disable() and _enable()
519 	 */
520 	venus_set_registers(hdev);
521 
522 	writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);
523 	writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);
524 	writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);
525 	writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);
526 	if (hdev->sfr.da)
527 		writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);
528 
529 	ret = venus_boot_core(hdev);
530 	if (ret) {
531 		dev_err(dev, "failed to reset venus core\n");
532 		return ret;
533 	}
534 
535 	venus_hwversion(hdev);
536 
537 	return 0;
538 }
539 
540 static int venus_halt_axi(struct venus_hfi_device *hdev)
541 {
542 	void __iomem *wrapper_base = hdev->core->wrapper_base;
543 	void __iomem *vbif_base = hdev->core->vbif_base;
544 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
545 	void __iomem *aon_base = hdev->core->aon_base;
546 	struct device *dev = hdev->core->dev;
547 	u32 val;
548 	u32 mask_val;
549 	int ret;
550 
551 	if (IS_V6(hdev->core)) {
552 		writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);
553 
554 		if (hdev->core->res->num_vpp_pipes == 1)
555 			goto skip_aon_mvp_noc;
556 
557 		writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);
558 		ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,
559 					 val,
560 					 val & BIT(0),
561 					 POLL_INTERVAL_US,
562 					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
563 		if (ret)
564 			return -ETIMEDOUT;
565 
566 skip_aon_mvp_noc:
567 		mask_val = (BIT(2) | BIT(1) | BIT(0));
568 		writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
569 
570 		writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
571 		ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,
572 					 val,
573 					 val == 0,
574 					 POLL_INTERVAL_US,
575 					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
576 
577 		if (ret) {
578 			dev_err(dev, "DBLP Release: lpi_status %x\n", val);
579 			return -ETIMEDOUT;
580 		}
581 		return 0;
582 	}
583 
584 	if (IS_V4(hdev->core)) {
585 		val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);
586 		val |= WRAPPER_CPU_AXI_HALT_HALT;
587 		writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);
588 
589 		ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,
590 					 val,
591 					 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
592 					 POLL_INTERVAL_US,
593 					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
594 		if (ret) {
595 			dev_err(dev, "AXI bus port halt timeout\n");
596 			return ret;
597 		}
598 
599 		return 0;
600 	}
601 
602 	/* Halt AXI and AXI IMEM VBIF Access */
603 	val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);
604 	val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
605 	writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);
606 
607 	/* Request for AXI bus port halt */
608 	ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,
609 				 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
610 				 POLL_INTERVAL_US,
611 				 VBIF_AXI_HALT_ACK_TIMEOUT_US);
612 	if (ret) {
613 		dev_err(dev, "AXI bus port halt timeout\n");
614 		return ret;
615 	}
616 
617 	return 0;
618 }
619 
620 static int venus_power_off(struct venus_hfi_device *hdev)
621 {
622 	int ret;
623 
624 	if (!hdev->power_enabled)
625 		return 0;
626 
627 	ret = venus_set_hw_state_suspend(hdev->core);
628 	if (ret)
629 		return ret;
630 
631 	ret = venus_halt_axi(hdev);
632 	if (ret)
633 		return ret;
634 
635 	hdev->power_enabled = false;
636 
637 	return 0;
638 }
639 
640 static int venus_power_on(struct venus_hfi_device *hdev)
641 {
642 	int ret;
643 
644 	if (hdev->power_enabled)
645 		return 0;
646 
647 	ret = venus_set_hw_state_resume(hdev->core);
648 	if (ret)
649 		goto err;
650 
651 	ret = venus_run(hdev);
652 	if (ret)
653 		goto err_suspend;
654 
655 	hdev->power_enabled = true;
656 
657 	return 0;
658 
659 err_suspend:
660 	venus_set_hw_state_suspend(hdev->core);
661 err:
662 	hdev->power_enabled = false;
663 	return ret;
664 }
665 
666 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
667 					void *pkt)
668 {
669 	struct iface_queue *queue;
670 	u32 tx_req;
671 	int ret;
672 
673 	if (!venus_is_valid_state(hdev))
674 		return -EINVAL;
675 
676 	queue = &hdev->queues[IFACEQ_MSG_IDX];
677 
678 	ret = venus_read_queue(hdev, queue, pkt, &tx_req);
679 	if (ret)
680 		return ret;
681 
682 	if (tx_req)
683 		venus_soft_int(hdev);
684 
685 	return 0;
686 }
687 
688 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
689 {
690 	int ret;
691 
692 	mutex_lock(&hdev->lock);
693 	ret = venus_iface_msgq_read_nolock(hdev, pkt);
694 	mutex_unlock(&hdev->lock);
695 
696 	return ret;
697 }
698 
699 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
700 					void *pkt)
701 {
702 	struct iface_queue *queue;
703 	u32 tx_req;
704 	int ret;
705 
706 	ret = venus_is_valid_state(hdev);
707 	if (!ret)
708 		return -EINVAL;
709 
710 	queue = &hdev->queues[IFACEQ_DBG_IDX];
711 
712 	ret = venus_read_queue(hdev, queue, pkt, &tx_req);
713 	if (ret)
714 		return ret;
715 
716 	if (tx_req)
717 		venus_soft_int(hdev);
718 
719 	return 0;
720 }
721 
722 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
723 {
724 	int ret;
725 
726 	if (!pkt)
727 		return -EINVAL;
728 
729 	mutex_lock(&hdev->lock);
730 	ret = venus_iface_dbgq_read_nolock(hdev, pkt);
731 	mutex_unlock(&hdev->lock);
732 
733 	return ret;
734 }
735 
736 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
737 {
738 	qhdr->status = 1;
739 	qhdr->type = IFACEQ_DFLT_QHDR;
740 	qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
741 	qhdr->pkt_size = 0;
742 	qhdr->rx_wm = 1;
743 	qhdr->tx_wm = 1;
744 	qhdr->rx_req = 1;
745 	qhdr->tx_req = 0;
746 	qhdr->rx_irq_status = 0;
747 	qhdr->tx_irq_status = 0;
748 	qhdr->read_idx = 0;
749 	qhdr->write_idx = 0;
750 }
751 
752 static void venus_interface_queues_release(struct venus_hfi_device *hdev)
753 {
754 	mutex_lock(&hdev->lock);
755 
756 	venus_free(hdev, &hdev->ifaceq_table);
757 	venus_free(hdev, &hdev->sfr);
758 
759 	memset(hdev->queues, 0, sizeof(hdev->queues));
760 	memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
761 	memset(&hdev->sfr, 0, sizeof(hdev->sfr));
762 
763 	mutex_unlock(&hdev->lock);
764 }
765 
766 static int venus_interface_queues_init(struct venus_hfi_device *hdev)
767 {
768 	struct hfi_queue_table_header *tbl_hdr;
769 	struct iface_queue *queue;
770 	struct hfi_sfr *sfr;
771 	struct mem_desc desc = {0};
772 	unsigned int offset;
773 	unsigned int i;
774 	int ret;
775 
776 	ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
777 	if (ret)
778 		return ret;
779 
780 	hdev->ifaceq_table = desc;
781 	offset = IFACEQ_TABLE_SIZE;
782 
783 	for (i = 0; i < IFACEQ_NUM; i++) {
784 		queue = &hdev->queues[i];
785 		queue->qmem.da = desc.da + offset;
786 		queue->qmem.kva = desc.kva + offset;
787 		queue->qmem.size = IFACEQ_QUEUE_SIZE;
788 		offset += queue->qmem.size;
789 		queue->qhdr =
790 			IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
791 
792 		venus_set_qhdr_defaults(queue->qhdr);
793 
794 		queue->qhdr->start_addr = queue->qmem.da;
795 
796 		if (i == IFACEQ_CMD_IDX)
797 			queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
798 		else if (i == IFACEQ_MSG_IDX)
799 			queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
800 		else if (i == IFACEQ_DBG_IDX)
801 			queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
802 	}
803 
804 	tbl_hdr = hdev->ifaceq_table.kva;
805 	tbl_hdr->version = 0;
806 	tbl_hdr->size = IFACEQ_TABLE_SIZE;
807 	tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
808 	tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
809 	tbl_hdr->num_q = IFACEQ_NUM;
810 	tbl_hdr->num_active_q = IFACEQ_NUM;
811 
812 	/*
813 	 * Set receive request to zero on debug queue as there is no
814 	 * need of interrupt from video hardware for debug messages
815 	 */
816 	queue = &hdev->queues[IFACEQ_DBG_IDX];
817 	queue->qhdr->rx_req = 0;
818 
819 	ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
820 	if (ret) {
821 		hdev->sfr.da = 0;
822 	} else {
823 		hdev->sfr = desc;
824 		sfr = hdev->sfr.kva;
825 		sfr->buf_size = ALIGNED_SFR_SIZE;
826 	}
827 
828 	/* ensure table and queue header structs are settled in memory */
829 	wmb();
830 
831 	return 0;
832 }
833 
834 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
835 {
836 	struct hfi_sys_set_property_pkt *pkt;
837 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
838 
839 	pkt = (struct hfi_sys_set_property_pkt *)packet;
840 
841 	pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
842 
843 	return venus_iface_cmdq_write(hdev, pkt, false);
844 }
845 
846 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
847 {
848 	struct hfi_sys_set_property_pkt *pkt;
849 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
850 
851 	pkt = (struct hfi_sys_set_property_pkt *)packet;
852 
853 	pkt_sys_coverage_config(pkt, mode);
854 
855 	return venus_iface_cmdq_write(hdev, pkt, false);
856 }
857 
858 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
859 				      bool enable)
860 {
861 	struct hfi_sys_set_property_pkt *pkt;
862 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
863 
864 	if (!enable)
865 		return 0;
866 
867 	pkt = (struct hfi_sys_set_property_pkt *)packet;
868 
869 	pkt_sys_idle_indicator(pkt, enable);
870 
871 	return venus_iface_cmdq_write(hdev, pkt, false);
872 }
873 
874 static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
875 				       bool enable)
876 {
877 	struct hfi_sys_set_property_pkt *pkt;
878 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
879 
880 	pkt = (struct hfi_sys_set_property_pkt *)packet;
881 
882 	pkt_sys_power_control(pkt, enable);
883 
884 	return venus_iface_cmdq_write(hdev, pkt, false);
885 }
886 
887 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev)
888 {
889 	struct hfi_sys_set_property_pkt *pkt;
890 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
891 	const struct venus_resources *res = hdev->core->res;
892 	int ret;
893 
894 	pkt = (struct hfi_sys_set_property_pkt *)packet;
895 
896 	pkt_sys_ubwc_config(pkt, res->ubwc_conf);
897 
898 	ret = venus_iface_cmdq_write(hdev, pkt, false);
899 	if (ret)
900 		return ret;
901 
902 	return 0;
903 }
904 
905 static int venus_get_queue_size(struct venus_hfi_device *hdev,
906 				unsigned int index)
907 {
908 	struct hfi_queue_header *qhdr;
909 
910 	if (index >= IFACEQ_NUM)
911 		return -EINVAL;
912 
913 	qhdr = hdev->queues[index].qhdr;
914 	if (!qhdr)
915 		return -EINVAL;
916 
917 	return abs(qhdr->read_idx - qhdr->write_idx);
918 }
919 
920 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
921 {
922 	struct device *dev = hdev->core->dev;
923 	const struct venus_resources *res = hdev->core->res;
924 	int ret;
925 
926 	ret = venus_sys_set_debug(hdev, venus_fw_debug);
927 	if (ret)
928 		dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
929 
930 	/*
931 	 * Idle indicator is disabled by default on some 4xx firmware versions,
932 	 * enable it explicitly in order to make suspend functional by checking
933 	 * WFI (wait-for-interrupt) bit.
934 	 */
935 	if (IS_V4(hdev->core) || IS_V6(hdev->core))
936 		venus_sys_idle_indicator = true;
937 
938 	ret = venus_sys_set_idle_message(hdev, venus_sys_idle_indicator);
939 	if (ret)
940 		dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
941 
942 	ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
943 	if (ret)
944 		dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
945 			 ret);
946 
947 	/* For specific venus core, it is mandatory to set the UBWC configuration */
948 	if (res->ubwc_conf) {
949 		ret = venus_sys_set_ubwc_config(hdev);
950 		if (ret)
951 			dev_warn(dev, "setting ubwc config failed (%d)\n", ret);
952 	}
953 
954 	return ret;
955 }
956 
957 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)
958 {
959 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
960 	struct hfi_session_pkt pkt;
961 
962 	pkt_session_cmd(&pkt, pkt_type, inst);
963 
964 	return venus_iface_cmdq_write(hdev, &pkt, sync);
965 }
966 
967 static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
968 {
969 	struct device *dev = hdev->core->dev;
970 	void *packet = hdev->dbg_buf;
971 
972 	while (!venus_iface_dbgq_read(hdev, packet)) {
973 		struct hfi_msg_sys_coverage_pkt *pkt = packet;
974 
975 		if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
976 			struct hfi_msg_sys_debug_pkt *pkt = packet;
977 
978 			dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
979 		}
980 	}
981 }
982 
983 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
984 					bool wait)
985 {
986 	unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
987 	struct hfi_sys_pc_prep_pkt pkt;
988 	int ret;
989 
990 	init_completion(&hdev->pwr_collapse_prep);
991 
992 	pkt_sys_pc_prep(&pkt);
993 
994 	ret = venus_iface_cmdq_write(hdev, &pkt, false);
995 	if (ret)
996 		return ret;
997 
998 	if (!wait)
999 		return 0;
1000 
1001 	ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
1002 	if (!ret) {
1003 		venus_flush_debug_queue(hdev);
1004 		return -ETIMEDOUT;
1005 	}
1006 
1007 	return 0;
1008 }
1009 
1010 static int venus_are_queues_empty(struct venus_hfi_device *hdev)
1011 {
1012 	int ret1, ret2;
1013 
1014 	ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
1015 	if (ret1 < 0)
1016 		return ret1;
1017 
1018 	ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
1019 	if (ret2 < 0)
1020 		return ret2;
1021 
1022 	if (!ret1 && !ret2)
1023 		return 1;
1024 
1025 	return 0;
1026 }
1027 
1028 static void venus_sfr_print(struct venus_hfi_device *hdev)
1029 {
1030 	struct device *dev = hdev->core->dev;
1031 	struct hfi_sfr *sfr = hdev->sfr.kva;
1032 	void *p;
1033 
1034 	if (!sfr)
1035 		return;
1036 
1037 	p = memchr(sfr->data, '\0', sfr->buf_size);
1038 	/*
1039 	 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
1040 	 * that Venus is in the process of crashing.
1041 	 */
1042 	if (!p)
1043 		sfr->data[sfr->buf_size - 1] = '\0';
1044 
1045 	dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
1046 }
1047 
1048 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
1049 					void *packet)
1050 {
1051 	struct hfi_msg_event_notify_pkt *event_pkt = packet;
1052 
1053 	if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
1054 		return;
1055 
1056 	venus_set_state(hdev, VENUS_STATE_DEINIT);
1057 
1058 	venus_sfr_print(hdev);
1059 }
1060 
1061 static irqreturn_t venus_isr_thread(struct venus_core *core)
1062 {
1063 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1064 	const struct venus_resources *res;
1065 	void *pkt;
1066 	u32 msg_ret;
1067 
1068 	if (!hdev)
1069 		return IRQ_NONE;
1070 
1071 	res = hdev->core->res;
1072 	pkt = hdev->pkt_buf;
1073 
1074 
1075 	while (!venus_iface_msgq_read(hdev, pkt)) {
1076 		msg_ret = hfi_process_msg_packet(core, pkt);
1077 		switch (msg_ret) {
1078 		case HFI_MSG_EVENT_NOTIFY:
1079 			venus_process_msg_sys_error(hdev, pkt);
1080 			break;
1081 		case HFI_MSG_SYS_INIT:
1082 			venus_hfi_core_set_resource(core, res->vmem_id,
1083 						    res->vmem_size,
1084 						    res->vmem_addr,
1085 						    hdev);
1086 			break;
1087 		case HFI_MSG_SYS_RELEASE_RESOURCE:
1088 			complete(&hdev->release_resource);
1089 			break;
1090 		case HFI_MSG_SYS_PC_PREP:
1091 			complete(&hdev->pwr_collapse_prep);
1092 			break;
1093 		default:
1094 			break;
1095 		}
1096 	}
1097 
1098 	venus_flush_debug_queue(hdev);
1099 
1100 	return IRQ_HANDLED;
1101 }
1102 
1103 static irqreturn_t venus_isr(struct venus_core *core)
1104 {
1105 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1106 	u32 status;
1107 	void __iomem *cpu_cs_base;
1108 	void __iomem *wrapper_base;
1109 
1110 	if (!hdev)
1111 		return IRQ_NONE;
1112 
1113 	cpu_cs_base = hdev->core->cpu_cs_base;
1114 	wrapper_base = hdev->core->wrapper_base;
1115 
1116 	status = readl(wrapper_base + WRAPPER_INTR_STATUS);
1117 	if (IS_V6(core)) {
1118 		if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1119 		    status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||
1120 		    status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1121 			hdev->irq_status = status;
1122 	} else {
1123 		if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1124 		    status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
1125 		    status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1126 			hdev->irq_status = status;
1127 	}
1128 	writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);
1129 	if (!IS_V6(core))
1130 		writel(status, wrapper_base + WRAPPER_INTR_CLEAR);
1131 
1132 	return IRQ_WAKE_THREAD;
1133 }
1134 
1135 static int venus_core_init(struct venus_core *core)
1136 {
1137 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1138 	struct device *dev = core->dev;
1139 	struct hfi_sys_get_property_pkt version_pkt;
1140 	struct hfi_sys_init_pkt pkt;
1141 	int ret;
1142 
1143 	pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
1144 
1145 	venus_set_state(hdev, VENUS_STATE_INIT);
1146 
1147 	ret = venus_iface_cmdq_write(hdev, &pkt, false);
1148 	if (ret)
1149 		return ret;
1150 
1151 	pkt_sys_image_version(&version_pkt);
1152 
1153 	ret = venus_iface_cmdq_write(hdev, &version_pkt, false);
1154 	if (ret)
1155 		dev_warn(dev, "failed to send image version pkt to fw\n");
1156 
1157 	ret = venus_sys_set_default_properties(hdev);
1158 	if (ret)
1159 		return ret;
1160 
1161 	return 0;
1162 }
1163 
1164 static int venus_core_deinit(struct venus_core *core)
1165 {
1166 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1167 
1168 	venus_set_state(hdev, VENUS_STATE_DEINIT);
1169 	hdev->suspended = true;
1170 	hdev->power_enabled = false;
1171 
1172 	return 0;
1173 }
1174 
1175 static int venus_core_ping(struct venus_core *core, u32 cookie)
1176 {
1177 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1178 	struct hfi_sys_ping_pkt pkt;
1179 
1180 	pkt_sys_ping(&pkt, cookie);
1181 
1182 	return venus_iface_cmdq_write(hdev, &pkt, false);
1183 }
1184 
1185 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
1186 {
1187 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1188 	struct hfi_sys_test_ssr_pkt pkt;
1189 	int ret;
1190 
1191 	ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
1192 	if (ret)
1193 		return ret;
1194 
1195 	return venus_iface_cmdq_write(hdev, &pkt, false);
1196 }
1197 
1198 static int venus_session_init(struct venus_inst *inst, u32 session_type,
1199 			      u32 codec)
1200 {
1201 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1202 	struct hfi_session_init_pkt pkt;
1203 	int ret;
1204 
1205 	ret = venus_sys_set_debug(hdev, venus_fw_debug);
1206 	if (ret)
1207 		goto err;
1208 
1209 	ret = pkt_session_init(&pkt, inst, session_type, codec);
1210 	if (ret)
1211 		goto err;
1212 
1213 	ret = venus_iface_cmdq_write(hdev, &pkt, true);
1214 	if (ret)
1215 		goto err;
1216 
1217 	return 0;
1218 
1219 err:
1220 	venus_flush_debug_queue(hdev);
1221 	return ret;
1222 }
1223 
1224 static int venus_session_end(struct venus_inst *inst)
1225 {
1226 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1227 	struct device *dev = hdev->core->dev;
1228 
1229 	if (venus_fw_coverage) {
1230 		if (venus_sys_set_coverage(hdev, venus_fw_coverage))
1231 			dev_warn(dev, "fw coverage msg ON failed\n");
1232 	}
1233 
1234 	return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);
1235 }
1236 
1237 static int venus_session_abort(struct venus_inst *inst)
1238 {
1239 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1240 
1241 	venus_flush_debug_queue(hdev);
1242 
1243 	return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);
1244 }
1245 
1246 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
1247 {
1248 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1249 	struct hfi_session_flush_pkt pkt;
1250 	int ret;
1251 
1252 	ret = pkt_session_flush(&pkt, inst, flush_mode);
1253 	if (ret)
1254 		return ret;
1255 
1256 	return venus_iface_cmdq_write(hdev, &pkt, true);
1257 }
1258 
1259 static int venus_session_start(struct venus_inst *inst)
1260 {
1261 	return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);
1262 }
1263 
1264 static int venus_session_stop(struct venus_inst *inst)
1265 {
1266 	return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);
1267 }
1268 
1269 static int venus_session_continue(struct venus_inst *inst)
1270 {
1271 	return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);
1272 }
1273 
1274 static int venus_session_etb(struct venus_inst *inst,
1275 			     struct hfi_frame_data *in_frame)
1276 {
1277 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1278 	u32 session_type = inst->session_type;
1279 	int ret;
1280 
1281 	if (session_type == VIDC_SESSION_TYPE_DEC) {
1282 		struct hfi_session_empty_buffer_compressed_pkt pkt;
1283 
1284 		ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
1285 		if (ret)
1286 			return ret;
1287 
1288 		ret = venus_iface_cmdq_write(hdev, &pkt, false);
1289 	} else if (session_type == VIDC_SESSION_TYPE_ENC) {
1290 		struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
1291 
1292 		ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
1293 		if (ret)
1294 			return ret;
1295 
1296 		ret = venus_iface_cmdq_write(hdev, &pkt, false);
1297 	} else {
1298 		ret = -EINVAL;
1299 	}
1300 
1301 	return ret;
1302 }
1303 
1304 static int venus_session_ftb(struct venus_inst *inst,
1305 			     struct hfi_frame_data *out_frame)
1306 {
1307 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1308 	struct hfi_session_fill_buffer_pkt pkt;
1309 	int ret;
1310 
1311 	ret = pkt_session_ftb(&pkt, inst, out_frame);
1312 	if (ret)
1313 		return ret;
1314 
1315 	return venus_iface_cmdq_write(hdev, &pkt, false);
1316 }
1317 
1318 static int venus_session_set_buffers(struct venus_inst *inst,
1319 				     struct hfi_buffer_desc *bd)
1320 {
1321 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1322 	struct hfi_session_set_buffers_pkt *pkt;
1323 	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1324 	int ret;
1325 
1326 	if (bd->buffer_type == HFI_BUFFER_INPUT)
1327 		return 0;
1328 
1329 	pkt = (struct hfi_session_set_buffers_pkt *)packet;
1330 
1331 	ret = pkt_session_set_buffers(pkt, inst, bd);
1332 	if (ret)
1333 		return ret;
1334 
1335 	return venus_iface_cmdq_write(hdev, pkt, false);
1336 }
1337 
1338 static int venus_session_unset_buffers(struct venus_inst *inst,
1339 				       struct hfi_buffer_desc *bd)
1340 {
1341 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1342 	struct hfi_session_release_buffer_pkt *pkt;
1343 	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1344 	int ret;
1345 
1346 	if (bd->buffer_type == HFI_BUFFER_INPUT)
1347 		return 0;
1348 
1349 	pkt = (struct hfi_session_release_buffer_pkt *)packet;
1350 
1351 	ret = pkt_session_unset_buffers(pkt, inst, bd);
1352 	if (ret)
1353 		return ret;
1354 
1355 	return venus_iface_cmdq_write(hdev, pkt, true);
1356 }
1357 
1358 static int venus_session_load_res(struct venus_inst *inst)
1359 {
1360 	return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);
1361 }
1362 
1363 static int venus_session_release_res(struct venus_inst *inst)
1364 {
1365 	return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);
1366 }
1367 
1368 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1369 				       u32 seq_hdr_len)
1370 {
1371 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1372 	struct hfi_session_parse_sequence_header_pkt *pkt;
1373 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1374 	int ret;
1375 
1376 	pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
1377 
1378 	ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
1379 	if (ret)
1380 		return ret;
1381 
1382 	ret = venus_iface_cmdq_write(hdev, pkt, false);
1383 	if (ret)
1384 		return ret;
1385 
1386 	return 0;
1387 }
1388 
1389 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1390 				     u32 seq_hdr_len)
1391 {
1392 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1393 	struct hfi_session_get_sequence_header_pkt *pkt;
1394 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1395 	int ret;
1396 
1397 	pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
1398 
1399 	ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
1400 	if (ret)
1401 		return ret;
1402 
1403 	return venus_iface_cmdq_write(hdev, pkt, false);
1404 }
1405 
1406 static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
1407 				      void *pdata)
1408 {
1409 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1410 	struct hfi_session_set_property_pkt *pkt;
1411 	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1412 	int ret;
1413 
1414 	pkt = (struct hfi_session_set_property_pkt *)packet;
1415 
1416 	ret = pkt_session_set_property(pkt, inst, ptype, pdata);
1417 	if (ret == -ENOTSUPP)
1418 		return 0;
1419 	if (ret)
1420 		return ret;
1421 
1422 	return venus_iface_cmdq_write(hdev, pkt, false);
1423 }
1424 
1425 static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
1426 {
1427 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1428 	struct hfi_session_get_property_pkt pkt;
1429 	int ret;
1430 
1431 	ret = pkt_session_get_property(&pkt, inst, ptype);
1432 	if (ret)
1433 		return ret;
1434 
1435 	return venus_iface_cmdq_write(hdev, &pkt, true);
1436 }
1437 
1438 static int venus_resume(struct venus_core *core)
1439 {
1440 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1441 	int ret = 0;
1442 
1443 	mutex_lock(&hdev->lock);
1444 
1445 	if (!hdev->suspended)
1446 		goto unlock;
1447 
1448 	ret = venus_power_on(hdev);
1449 
1450 unlock:
1451 	if (!ret)
1452 		hdev->suspended = false;
1453 
1454 	mutex_unlock(&hdev->lock);
1455 
1456 	return ret;
1457 }
1458 
1459 static int venus_suspend_1xx(struct venus_core *core)
1460 {
1461 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1462 	struct device *dev = core->dev;
1463 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1464 	u32 ctrl_status;
1465 	int ret;
1466 
1467 	if (!hdev->power_enabled || hdev->suspended)
1468 		return 0;
1469 
1470 	mutex_lock(&hdev->lock);
1471 	ret = venus_is_valid_state(hdev);
1472 	mutex_unlock(&hdev->lock);
1473 
1474 	if (!ret) {
1475 		dev_err(dev, "bad state, cannot suspend\n");
1476 		return -EINVAL;
1477 	}
1478 
1479 	ret = venus_prepare_power_collapse(hdev, true);
1480 	if (ret) {
1481 		dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1482 		return ret;
1483 	}
1484 
1485 	mutex_lock(&hdev->lock);
1486 
1487 	if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
1488 		mutex_unlock(&hdev->lock);
1489 		return -EINVAL;
1490 	}
1491 
1492 	ret = venus_are_queues_empty(hdev);
1493 	if (ret < 0 || !ret) {
1494 		mutex_unlock(&hdev->lock);
1495 		return -EINVAL;
1496 	}
1497 
1498 	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1499 	if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
1500 		mutex_unlock(&hdev->lock);
1501 		return -EINVAL;
1502 	}
1503 
1504 	ret = venus_power_off(hdev);
1505 	if (ret) {
1506 		mutex_unlock(&hdev->lock);
1507 		return ret;
1508 	}
1509 
1510 	hdev->suspended = true;
1511 
1512 	mutex_unlock(&hdev->lock);
1513 
1514 	return 0;
1515 }
1516 
1517 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
1518 {
1519 	void __iomem *wrapper_base = hdev->core->wrapper_base;
1520 	void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1521 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1522 	u32 ctrl_status, cpu_status;
1523 
1524 	if (IS_V6(hdev->core))
1525 		cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1526 	else
1527 		cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1528 	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1529 
1530 	if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1531 	    ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1532 		return true;
1533 
1534 	return false;
1535 }
1536 
1537 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
1538 {
1539 	void __iomem *wrapper_base = hdev->core->wrapper_base;
1540 	void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1541 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1542 	u32 ctrl_status, cpu_status;
1543 
1544 	if (IS_V6(hdev->core))
1545 		cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1546 	else
1547 		cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1548 	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1549 
1550 	if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1551 	    ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1552 		return true;
1553 
1554 	return false;
1555 }
1556 
1557 static int venus_suspend_3xx(struct venus_core *core)
1558 {
1559 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1560 	struct device *dev = core->dev;
1561 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1562 	u32 ctrl_status;
1563 	bool val;
1564 	int ret;
1565 
1566 	if (!hdev->power_enabled || hdev->suspended)
1567 		return 0;
1568 
1569 	mutex_lock(&hdev->lock);
1570 	ret = venus_is_valid_state(hdev);
1571 	mutex_unlock(&hdev->lock);
1572 
1573 	if (!ret) {
1574 		dev_err(dev, "bad state, cannot suspend\n");
1575 		return -EINVAL;
1576 	}
1577 
1578 	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1579 	if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1580 		goto power_off;
1581 
1582 	/*
1583 	 * Power collapse sequence for Venus 3xx and 4xx versions:
1584 	 * 1. Check for ARM9 and video core to be idle by checking WFI bit
1585 	 *    (bit 0) in CPU status register and by checking Idle (bit 30) in
1586 	 *    Control status register for video core.
1587 	 * 2. Send a command to prepare for power collapse.
1588 	 * 3. Check for WFI and PC_READY bits.
1589 	 */
1590 	ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
1591 				 1500, 100 * 1500);
1592 	if (ret) {
1593 		dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret);
1594 		return ret;
1595 	}
1596 
1597 	ret = venus_prepare_power_collapse(hdev, false);
1598 	if (ret) {
1599 		dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1600 		return ret;
1601 	}
1602 
1603 	ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
1604 				 1500, 100 * 1500);
1605 	if (ret)
1606 		return ret;
1607 
1608 power_off:
1609 	mutex_lock(&hdev->lock);
1610 
1611 	ret = venus_power_off(hdev);
1612 	if (ret) {
1613 		dev_err(dev, "venus_power_off (%d)\n", ret);
1614 		mutex_unlock(&hdev->lock);
1615 		return ret;
1616 	}
1617 
1618 	hdev->suspended = true;
1619 
1620 	mutex_unlock(&hdev->lock);
1621 
1622 	return 0;
1623 }
1624 
1625 static int venus_suspend(struct venus_core *core)
1626 {
1627 	if (IS_V3(core) || IS_V4(core) || IS_V6(core))
1628 		return venus_suspend_3xx(core);
1629 
1630 	return venus_suspend_1xx(core);
1631 }
1632 
1633 static const struct hfi_ops venus_hfi_ops = {
1634 	.core_init			= venus_core_init,
1635 	.core_deinit			= venus_core_deinit,
1636 	.core_ping			= venus_core_ping,
1637 	.core_trigger_ssr		= venus_core_trigger_ssr,
1638 
1639 	.session_init			= venus_session_init,
1640 	.session_end			= venus_session_end,
1641 	.session_abort			= venus_session_abort,
1642 	.session_flush			= venus_session_flush,
1643 	.session_start			= venus_session_start,
1644 	.session_stop			= venus_session_stop,
1645 	.session_continue		= venus_session_continue,
1646 	.session_etb			= venus_session_etb,
1647 	.session_ftb			= venus_session_ftb,
1648 	.session_set_buffers		= venus_session_set_buffers,
1649 	.session_unset_buffers		= venus_session_unset_buffers,
1650 	.session_load_res		= venus_session_load_res,
1651 	.session_release_res		= venus_session_release_res,
1652 	.session_parse_seq_hdr		= venus_session_parse_seq_hdr,
1653 	.session_get_seq_hdr		= venus_session_get_seq_hdr,
1654 	.session_set_property		= venus_session_set_property,
1655 	.session_get_property		= venus_session_get_property,
1656 
1657 	.resume				= venus_resume,
1658 	.suspend			= venus_suspend,
1659 
1660 	.isr				= venus_isr,
1661 	.isr_thread			= venus_isr_thread,
1662 };
1663 
1664 void venus_hfi_destroy(struct venus_core *core)
1665 {
1666 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1667 
1668 	core->priv = NULL;
1669 	venus_interface_queues_release(hdev);
1670 	mutex_destroy(&hdev->lock);
1671 	kfree(hdev);
1672 	core->ops = NULL;
1673 }
1674 
1675 int venus_hfi_create(struct venus_core *core)
1676 {
1677 	struct venus_hfi_device *hdev;
1678 	int ret;
1679 
1680 	hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
1681 	if (!hdev)
1682 		return -ENOMEM;
1683 
1684 	mutex_init(&hdev->lock);
1685 
1686 	hdev->core = core;
1687 	hdev->suspended = true;
1688 	core->priv = hdev;
1689 	core->ops = &venus_hfi_ops;
1690 
1691 	ret = venus_interface_queues_init(hdev);
1692 	if (ret)
1693 		goto err_kfree;
1694 
1695 	return 0;
1696 
1697 err_kfree:
1698 	kfree(hdev);
1699 	core->priv = NULL;
1700 	core->ops = NULL;
1701 	return ret;
1702 }
1703 
1704 void venus_hfi_queues_reinit(struct venus_core *core)
1705 {
1706 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1707 	struct hfi_queue_table_header *tbl_hdr;
1708 	struct iface_queue *queue;
1709 	struct hfi_sfr *sfr;
1710 	unsigned int i;
1711 
1712 	mutex_lock(&hdev->lock);
1713 
1714 	for (i = 0; i < IFACEQ_NUM; i++) {
1715 		queue = &hdev->queues[i];
1716 		queue->qhdr =
1717 			IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
1718 
1719 		venus_set_qhdr_defaults(queue->qhdr);
1720 
1721 		queue->qhdr->start_addr = queue->qmem.da;
1722 
1723 		if (i == IFACEQ_CMD_IDX)
1724 			queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
1725 		else if (i == IFACEQ_MSG_IDX)
1726 			queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
1727 		else if (i == IFACEQ_DBG_IDX)
1728 			queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
1729 	}
1730 
1731 	tbl_hdr = hdev->ifaceq_table.kva;
1732 	tbl_hdr->version = 0;
1733 	tbl_hdr->size = IFACEQ_TABLE_SIZE;
1734 	tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
1735 	tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
1736 	tbl_hdr->num_q = IFACEQ_NUM;
1737 	tbl_hdr->num_active_q = IFACEQ_NUM;
1738 
1739 	/*
1740 	 * Set receive request to zero on debug queue as there is no
1741 	 * need of interrupt from video hardware for debug messages
1742 	 */
1743 	queue = &hdev->queues[IFACEQ_DBG_IDX];
1744 	queue->qhdr->rx_req = 0;
1745 
1746 	sfr = hdev->sfr.kva;
1747 	sfr->buf_size = ALIGNED_SFR_SIZE;
1748 
1749 	/* ensure table and queue header structs are settled in memory */
1750 	wmb();
1751 
1752 	mutex_unlock(&hdev->lock);
1753 }
1754