1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
4  * Copyright (C) 2017 Linaro Ltd.
5  */
6 
7 #include <linux/delay.h>
8 #include <linux/device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/interrupt.h>
11 #include <linux/iopoll.h>
12 #include <linux/kernel.h>
13 #include <linux/slab.h>
14 
15 #include "core.h"
16 #include "hfi_cmds.h"
17 #include "hfi_msgs.h"
18 #include "hfi_venus.h"
19 #include "hfi_venus_io.h"
20 #include "firmware.h"
21 
22 #define HFI_MASK_QHDR_TX_TYPE		0xff000000
23 #define HFI_MASK_QHDR_RX_TYPE		0x00ff0000
24 #define HFI_MASK_QHDR_PRI_TYPE		0x0000ff00
25 #define HFI_MASK_QHDR_ID_TYPE		0x000000ff
26 
27 #define HFI_HOST_TO_CTRL_CMD_Q		0
28 #define HFI_CTRL_TO_HOST_MSG_Q		1
29 #define HFI_CTRL_TO_HOST_DBG_Q		2
30 #define HFI_MASK_QHDR_STATUS		0x000000ff
31 
32 #define IFACEQ_NUM			3
33 #define IFACEQ_CMD_IDX			0
34 #define IFACEQ_MSG_IDX			1
35 #define IFACEQ_DBG_IDX			2
36 #define IFACEQ_MAX_BUF_COUNT		50
37 #define IFACEQ_MAX_PARALLEL_CLNTS	16
38 #define IFACEQ_DFLT_QHDR		0x01010000
39 
40 #define POLL_INTERVAL_US		50
41 
42 #define IFACEQ_MAX_PKT_SIZE		1024
43 #define IFACEQ_MED_PKT_SIZE		768
44 #define IFACEQ_MIN_PKT_SIZE		8
45 #define IFACEQ_VAR_SMALL_PKT_SIZE	100
46 #define IFACEQ_VAR_LARGE_PKT_SIZE	512
47 #define IFACEQ_VAR_HUGE_PKT_SIZE	(1024 * 12)
48 
49 struct hfi_queue_table_header {
50 	u32 version;
51 	u32 size;
52 	u32 qhdr0_offset;
53 	u32 qhdr_size;
54 	u32 num_q;
55 	u32 num_active_q;
56 };
57 
58 struct hfi_queue_header {
59 	u32 status;
60 	u32 start_addr;
61 	u32 type;
62 	u32 q_size;
63 	u32 pkt_size;
64 	u32 pkt_drop_cnt;
65 	u32 rx_wm;
66 	u32 tx_wm;
67 	u32 rx_req;
68 	u32 tx_req;
69 	u32 rx_irq_status;
70 	u32 tx_irq_status;
71 	u32 read_idx;
72 	u32 write_idx;
73 };
74 
75 #define IFACEQ_TABLE_SIZE	\
76 	(sizeof(struct hfi_queue_table_header) +	\
77 	 sizeof(struct hfi_queue_header) * IFACEQ_NUM)
78 
79 #define IFACEQ_QUEUE_SIZE	(IFACEQ_MAX_PKT_SIZE *	\
80 	IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
81 
82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i)	\
83 	(void *)(((ptr) + sizeof(struct hfi_queue_table_header)) +	\
84 		((i) * sizeof(struct hfi_queue_header)))
85 
86 #define QDSS_SIZE		SZ_4K
87 #define SFR_SIZE		SZ_4K
88 #define QUEUE_SIZE		\
89 	(IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
90 
91 #define ALIGNED_QDSS_SIZE	ALIGN(QDSS_SIZE, SZ_4K)
92 #define ALIGNED_SFR_SIZE	ALIGN(SFR_SIZE, SZ_4K)
93 #define ALIGNED_QUEUE_SIZE	ALIGN(QUEUE_SIZE, SZ_4K)
94 #define SHARED_QSIZE		ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
95 				      ALIGNED_QDSS_SIZE, SZ_1M)
96 
97 struct mem_desc {
98 	dma_addr_t da;	/* device address */
99 	void *kva;	/* kernel virtual address */
100 	u32 size;
101 	unsigned long attrs;
102 };
103 
104 struct iface_queue {
105 	struct hfi_queue_header *qhdr;
106 	struct mem_desc qmem;
107 };
108 
109 enum venus_state {
110 	VENUS_STATE_DEINIT = 1,
111 	VENUS_STATE_INIT,
112 };
113 
114 struct venus_hfi_device {
115 	struct venus_core *core;
116 	u32 irq_status;
117 	u32 last_packet_type;
118 	bool power_enabled;
119 	bool suspended;
120 	enum venus_state state;
121 	/* serialize read / write to the shared memory */
122 	struct mutex lock;
123 	struct completion pwr_collapse_prep;
124 	struct completion release_resource;
125 	struct mem_desc ifaceq_table;
126 	struct mem_desc sfr;
127 	struct iface_queue queues[IFACEQ_NUM];
128 	u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
129 	u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
130 };
131 
132 static bool venus_pkt_debug;
133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
134 static bool venus_fw_low_power_mode = true;
135 static int venus_hw_rsp_timeout = 1000;
136 static bool venus_fw_coverage;
137 
venus_set_state(struct venus_hfi_device * hdev,enum venus_state state)138 static void venus_set_state(struct venus_hfi_device *hdev,
139 			    enum venus_state state)
140 {
141 	mutex_lock(&hdev->lock);
142 	hdev->state = state;
143 	mutex_unlock(&hdev->lock);
144 }
145 
venus_is_valid_state(struct venus_hfi_device * hdev)146 static bool venus_is_valid_state(struct venus_hfi_device *hdev)
147 {
148 	return hdev->state != VENUS_STATE_DEINIT;
149 }
150 
venus_dump_packet(struct venus_hfi_device * hdev,const void * packet)151 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
152 {
153 	size_t pkt_size = *(u32 *)packet;
154 
155 	if (!venus_pkt_debug)
156 		return;
157 
158 	print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
159 		       pkt_size, true);
160 }
161 
venus_write_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * packet,u32 * rx_req)162 static int venus_write_queue(struct venus_hfi_device *hdev,
163 			     struct iface_queue *queue,
164 			     void *packet, u32 *rx_req)
165 {
166 	struct hfi_queue_header *qhdr;
167 	u32 dwords, new_wr_idx;
168 	u32 empty_space, rd_idx, wr_idx, qsize;
169 	u32 *wr_ptr;
170 
171 	if (!queue->qmem.kva)
172 		return -EINVAL;
173 
174 	qhdr = queue->qhdr;
175 	if (!qhdr)
176 		return -EINVAL;
177 
178 	venus_dump_packet(hdev, packet);
179 
180 	dwords = (*(u32 *)packet) >> 2;
181 	if (!dwords)
182 		return -EINVAL;
183 
184 	rd_idx = qhdr->read_idx;
185 	wr_idx = qhdr->write_idx;
186 	qsize = qhdr->q_size;
187 	/* ensure rd/wr indices's are read from memory */
188 	rmb();
189 
190 	if (wr_idx >= rd_idx)
191 		empty_space = qsize - (wr_idx - rd_idx);
192 	else
193 		empty_space = rd_idx - wr_idx;
194 
195 	if (empty_space <= dwords) {
196 		qhdr->tx_req = 1;
197 		/* ensure tx_req is updated in memory */
198 		wmb();
199 		return -ENOSPC;
200 	}
201 
202 	qhdr->tx_req = 0;
203 	/* ensure tx_req is updated in memory */
204 	wmb();
205 
206 	new_wr_idx = wr_idx + dwords;
207 	wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
208 
209 	if (wr_ptr < (u32 *)queue->qmem.kva ||
210 	    wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr)))
211 		return -EINVAL;
212 
213 	if (new_wr_idx < qsize) {
214 		memcpy(wr_ptr, packet, dwords << 2);
215 	} else {
216 		size_t len;
217 
218 		new_wr_idx -= qsize;
219 		len = (dwords - new_wr_idx) << 2;
220 		memcpy(wr_ptr, packet, len);
221 		memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
222 	}
223 
224 	/* make sure packet is written before updating the write index */
225 	wmb();
226 
227 	qhdr->write_idx = new_wr_idx;
228 	*rx_req = qhdr->rx_req ? 1 : 0;
229 
230 	/* make sure write index is updated before an interrupt is raised */
231 	mb();
232 
233 	return 0;
234 }
235 
venus_read_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * pkt,u32 * tx_req)236 static int venus_read_queue(struct venus_hfi_device *hdev,
237 			    struct iface_queue *queue, void *pkt, u32 *tx_req)
238 {
239 	struct hfi_queue_header *qhdr;
240 	u32 dwords, new_rd_idx;
241 	u32 rd_idx, wr_idx, type, qsize;
242 	u32 *rd_ptr;
243 	u32 recv_request = 0;
244 	int ret = 0;
245 
246 	if (!queue->qmem.kva)
247 		return -EINVAL;
248 
249 	qhdr = queue->qhdr;
250 	if (!qhdr)
251 		return -EINVAL;
252 
253 	type = qhdr->type;
254 	rd_idx = qhdr->read_idx;
255 	wr_idx = qhdr->write_idx;
256 	qsize = qhdr->q_size;
257 
258 	/* make sure data is valid before using it */
259 	rmb();
260 
261 	/*
262 	 * Do not set receive request for debug queue, if set, Venus generates
263 	 * interrupt for debug messages even when there is no response message
264 	 * available. In general debug queue will not become full as it is being
265 	 * emptied out for every interrupt from Venus. Venus will anyway
266 	 * generates interrupt if it is full.
267 	 */
268 	if (type & HFI_CTRL_TO_HOST_MSG_Q)
269 		recv_request = 1;
270 
271 	if (rd_idx == wr_idx) {
272 		qhdr->rx_req = recv_request;
273 		*tx_req = 0;
274 		/* update rx_req field in memory */
275 		wmb();
276 		return -ENODATA;
277 	}
278 
279 	rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
280 
281 	if (rd_ptr < (u32 *)queue->qmem.kva ||
282 	    rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr)))
283 		return -EINVAL;
284 
285 	dwords = *rd_ptr >> 2;
286 	if (!dwords)
287 		return -EINVAL;
288 
289 	new_rd_idx = rd_idx + dwords;
290 	if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
291 		if (new_rd_idx < qsize) {
292 			memcpy(pkt, rd_ptr, dwords << 2);
293 		} else {
294 			size_t len;
295 
296 			new_rd_idx -= qsize;
297 			len = (dwords - new_rd_idx) << 2;
298 			memcpy(pkt, rd_ptr, len);
299 			memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
300 		}
301 	} else {
302 		/* bad packet received, dropping */
303 		new_rd_idx = qhdr->write_idx;
304 		ret = -EBADMSG;
305 	}
306 
307 	/* ensure the packet is read before updating read index */
308 	rmb();
309 
310 	qhdr->read_idx = new_rd_idx;
311 	/* ensure updating read index */
312 	wmb();
313 
314 	rd_idx = qhdr->read_idx;
315 	wr_idx = qhdr->write_idx;
316 	/* ensure rd/wr indices are read from memory */
317 	rmb();
318 
319 	if (rd_idx != wr_idx)
320 		qhdr->rx_req = 0;
321 	else
322 		qhdr->rx_req = recv_request;
323 
324 	*tx_req = qhdr->tx_req ? 1 : 0;
325 
326 	/* ensure rx_req is stored to memory and tx_req is loaded from memory */
327 	mb();
328 
329 	venus_dump_packet(hdev, pkt);
330 
331 	return ret;
332 }
333 
venus_alloc(struct venus_hfi_device * hdev,struct mem_desc * desc,u32 size)334 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
335 		       u32 size)
336 {
337 	struct device *dev = hdev->core->dev;
338 
339 	desc->attrs = DMA_ATTR_WRITE_COMBINE;
340 	desc->size = ALIGN(size, SZ_4K);
341 
342 	desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
343 				    desc->attrs);
344 	if (!desc->kva)
345 		return -ENOMEM;
346 
347 	return 0;
348 }
349 
venus_free(struct venus_hfi_device * hdev,struct mem_desc * mem)350 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
351 {
352 	struct device *dev = hdev->core->dev;
353 
354 	dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
355 }
356 
venus_set_registers(struct venus_hfi_device * hdev)357 static void venus_set_registers(struct venus_hfi_device *hdev)
358 {
359 	const struct venus_resources *res = hdev->core->res;
360 	const struct reg_val *tbl = res->reg_tbl;
361 	unsigned int count = res->reg_tbl_size;
362 	unsigned int i;
363 
364 	for (i = 0; i < count; i++)
365 		writel(tbl[i].value, hdev->core->base + tbl[i].reg);
366 }
367 
venus_soft_int(struct venus_hfi_device * hdev)368 static void venus_soft_int(struct venus_hfi_device *hdev)
369 {
370 	void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;
371 	u32 clear_bit;
372 
373 	if (IS_V6(hdev->core))
374 		clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);
375 	else
376 		clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);
377 
378 	writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);
379 }
380 
venus_iface_cmdq_write_nolock(struct venus_hfi_device * hdev,void * pkt,bool sync)381 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
382 					 void *pkt, bool sync)
383 {
384 	struct device *dev = hdev->core->dev;
385 	struct hfi_pkt_hdr *cmd_packet;
386 	struct iface_queue *queue;
387 	u32 rx_req;
388 	int ret;
389 
390 	if (!venus_is_valid_state(hdev))
391 		return -EINVAL;
392 
393 	cmd_packet = (struct hfi_pkt_hdr *)pkt;
394 	hdev->last_packet_type = cmd_packet->pkt_type;
395 
396 	queue = &hdev->queues[IFACEQ_CMD_IDX];
397 
398 	ret = venus_write_queue(hdev, queue, pkt, &rx_req);
399 	if (ret) {
400 		dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
401 		return ret;
402 	}
403 
404 	if (sync) {
405 		/*
406 		 * Inform video hardware to raise interrupt for synchronous
407 		 * commands
408 		 */
409 		queue = &hdev->queues[IFACEQ_MSG_IDX];
410 		queue->qhdr->rx_req = 1;
411 		/* ensure rx_req is updated in memory */
412 		wmb();
413 	}
414 
415 	if (rx_req)
416 		venus_soft_int(hdev);
417 
418 	return 0;
419 }
420 
venus_iface_cmdq_write(struct venus_hfi_device * hdev,void * pkt,bool sync)421 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)
422 {
423 	int ret;
424 
425 	mutex_lock(&hdev->lock);
426 	ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);
427 	mutex_unlock(&hdev->lock);
428 
429 	return ret;
430 }
431 
venus_hfi_core_set_resource(struct venus_core * core,u32 id,u32 size,u32 addr,void * cookie)432 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
433 				       u32 size, u32 addr, void *cookie)
434 {
435 	struct venus_hfi_device *hdev = to_hfi_priv(core);
436 	struct hfi_sys_set_resource_pkt *pkt;
437 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
438 	int ret;
439 
440 	if (id == VIDC_RESOURCE_NONE)
441 		return 0;
442 
443 	pkt = (struct hfi_sys_set_resource_pkt *)packet;
444 
445 	ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
446 	if (ret)
447 		return ret;
448 
449 	ret = venus_iface_cmdq_write(hdev, pkt, false);
450 	if (ret)
451 		return ret;
452 
453 	return 0;
454 }
455 
venus_boot_core(struct venus_hfi_device * hdev)456 static int venus_boot_core(struct venus_hfi_device *hdev)
457 {
458 	struct device *dev = hdev->core->dev;
459 	static const unsigned int max_tries = 100;
460 	u32 ctrl_status = 0, mask_val = 0;
461 	unsigned int count = 0;
462 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
463 	void __iomem *wrapper_base = hdev->core->wrapper_base;
464 	int ret = 0;
465 
466 	if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
467 		mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);
468 		mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |
469 			      WRAPPER_INTR_MASK_A2HCPU_MASK);
470 	} else {
471 		mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;
472 	}
473 
474 	writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);
475 	if (IS_V1(hdev->core))
476 		writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);
477 
478 	writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);
479 	while (!ctrl_status && count < max_tries) {
480 		ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
481 		if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
482 			dev_err(dev, "invalid setting for UC_REGION\n");
483 			ret = -EINVAL;
484 			break;
485 		}
486 
487 		usleep_range(500, 1000);
488 		count++;
489 	}
490 
491 	if (count >= max_tries)
492 		ret = -ETIMEDOUT;
493 
494 	if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
495 		writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);
496 		writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);
497 	}
498 
499 	return ret;
500 }
501 
venus_hwversion(struct venus_hfi_device * hdev)502 static u32 venus_hwversion(struct venus_hfi_device *hdev)
503 {
504 	struct device *dev = hdev->core->dev;
505 	void __iomem *wrapper_base = hdev->core->wrapper_base;
506 	u32 ver;
507 	u32 major, minor, step;
508 
509 	ver = readl(wrapper_base + WRAPPER_HW_VERSION);
510 	major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
511 	major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
512 	minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
513 	minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
514 	step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
515 
516 	dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);
517 
518 	return major;
519 }
520 
venus_run(struct venus_hfi_device * hdev)521 static int venus_run(struct venus_hfi_device *hdev)
522 {
523 	struct device *dev = hdev->core->dev;
524 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
525 	int ret;
526 
527 	/*
528 	 * Re-program all of the registers that get reset as a result of
529 	 * regulator_disable() and _enable()
530 	 */
531 	venus_set_registers(hdev);
532 
533 	writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);
534 	writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);
535 	writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);
536 	writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);
537 	if (hdev->sfr.da)
538 		writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);
539 
540 	ret = venus_boot_core(hdev);
541 	if (ret) {
542 		dev_err(dev, "failed to reset venus core\n");
543 		return ret;
544 	}
545 
546 	venus_hwversion(hdev);
547 
548 	return 0;
549 }
550 
venus_halt_axi(struct venus_hfi_device * hdev)551 static int venus_halt_axi(struct venus_hfi_device *hdev)
552 {
553 	void __iomem *wrapper_base = hdev->core->wrapper_base;
554 	void __iomem *vbif_base = hdev->core->vbif_base;
555 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
556 	void __iomem *aon_base = hdev->core->aon_base;
557 	struct device *dev = hdev->core->dev;
558 	u32 val;
559 	u32 mask_val;
560 	int ret;
561 
562 	if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
563 		writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);
564 
565 		if (IS_IRIS2_1(hdev->core))
566 			goto skip_aon_mvp_noc;
567 
568 		writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);
569 		ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,
570 					 val,
571 					 val & BIT(0),
572 					 POLL_INTERVAL_US,
573 					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
574 		if (ret)
575 			return -ETIMEDOUT;
576 
577 skip_aon_mvp_noc:
578 		mask_val = (BIT(2) | BIT(1) | BIT(0));
579 		writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
580 
581 		writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
582 		ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,
583 					 val,
584 					 val == 0,
585 					 POLL_INTERVAL_US,
586 					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
587 
588 		if (ret) {
589 			dev_err(dev, "DBLP Release: lpi_status %x\n", val);
590 			return -ETIMEDOUT;
591 		}
592 		return 0;
593 	}
594 
595 	if (IS_V4(hdev->core)) {
596 		val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);
597 		val |= WRAPPER_CPU_AXI_HALT_HALT;
598 		writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);
599 
600 		ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,
601 					 val,
602 					 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
603 					 POLL_INTERVAL_US,
604 					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
605 		if (ret) {
606 			dev_err(dev, "AXI bus port halt timeout\n");
607 			return ret;
608 		}
609 
610 		return 0;
611 	}
612 
613 	/* Halt AXI and AXI IMEM VBIF Access */
614 	val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);
615 	val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
616 	writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);
617 
618 	/* Request for AXI bus port halt */
619 	ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,
620 				 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
621 				 POLL_INTERVAL_US,
622 				 VBIF_AXI_HALT_ACK_TIMEOUT_US);
623 	if (ret) {
624 		dev_err(dev, "AXI bus port halt timeout\n");
625 		return ret;
626 	}
627 
628 	return 0;
629 }
630 
venus_power_off(struct venus_hfi_device * hdev)631 static int venus_power_off(struct venus_hfi_device *hdev)
632 {
633 	int ret;
634 
635 	if (!hdev->power_enabled)
636 		return 0;
637 
638 	ret = venus_set_hw_state_suspend(hdev->core);
639 	if (ret)
640 		return ret;
641 
642 	ret = venus_halt_axi(hdev);
643 	if (ret)
644 		return ret;
645 
646 	hdev->power_enabled = false;
647 
648 	return 0;
649 }
650 
venus_power_on(struct venus_hfi_device * hdev)651 static int venus_power_on(struct venus_hfi_device *hdev)
652 {
653 	int ret;
654 
655 	if (hdev->power_enabled)
656 		return 0;
657 
658 	ret = venus_set_hw_state_resume(hdev->core);
659 	if (ret)
660 		goto err;
661 
662 	ret = venus_run(hdev);
663 	if (ret)
664 		goto err_suspend;
665 
666 	hdev->power_enabled = true;
667 
668 	return 0;
669 
670 err_suspend:
671 	venus_set_hw_state_suspend(hdev->core);
672 err:
673 	hdev->power_enabled = false;
674 	return ret;
675 }
676 
venus_iface_msgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)677 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
678 					void *pkt)
679 {
680 	struct iface_queue *queue;
681 	u32 tx_req;
682 	int ret;
683 
684 	if (!venus_is_valid_state(hdev))
685 		return -EINVAL;
686 
687 	queue = &hdev->queues[IFACEQ_MSG_IDX];
688 
689 	ret = venus_read_queue(hdev, queue, pkt, &tx_req);
690 	if (ret)
691 		return ret;
692 
693 	if (tx_req)
694 		venus_soft_int(hdev);
695 
696 	return 0;
697 }
698 
venus_iface_msgq_read(struct venus_hfi_device * hdev,void * pkt)699 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
700 {
701 	int ret;
702 
703 	mutex_lock(&hdev->lock);
704 	ret = venus_iface_msgq_read_nolock(hdev, pkt);
705 	mutex_unlock(&hdev->lock);
706 
707 	return ret;
708 }
709 
venus_iface_dbgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)710 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
711 					void *pkt)
712 {
713 	struct iface_queue *queue;
714 	u32 tx_req;
715 	int ret;
716 
717 	ret = venus_is_valid_state(hdev);
718 	if (!ret)
719 		return -EINVAL;
720 
721 	queue = &hdev->queues[IFACEQ_DBG_IDX];
722 
723 	ret = venus_read_queue(hdev, queue, pkt, &tx_req);
724 	if (ret)
725 		return ret;
726 
727 	if (tx_req)
728 		venus_soft_int(hdev);
729 
730 	return 0;
731 }
732 
venus_iface_dbgq_read(struct venus_hfi_device * hdev,void * pkt)733 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
734 {
735 	int ret;
736 
737 	if (!pkt)
738 		return -EINVAL;
739 
740 	mutex_lock(&hdev->lock);
741 	ret = venus_iface_dbgq_read_nolock(hdev, pkt);
742 	mutex_unlock(&hdev->lock);
743 
744 	return ret;
745 }
746 
venus_set_qhdr_defaults(struct hfi_queue_header * qhdr)747 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
748 {
749 	qhdr->status = 1;
750 	qhdr->type = IFACEQ_DFLT_QHDR;
751 	qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
752 	qhdr->pkt_size = 0;
753 	qhdr->rx_wm = 1;
754 	qhdr->tx_wm = 1;
755 	qhdr->rx_req = 1;
756 	qhdr->tx_req = 0;
757 	qhdr->rx_irq_status = 0;
758 	qhdr->tx_irq_status = 0;
759 	qhdr->read_idx = 0;
760 	qhdr->write_idx = 0;
761 }
762 
venus_interface_queues_release(struct venus_hfi_device * hdev)763 static void venus_interface_queues_release(struct venus_hfi_device *hdev)
764 {
765 	mutex_lock(&hdev->lock);
766 
767 	venus_free(hdev, &hdev->ifaceq_table);
768 	venus_free(hdev, &hdev->sfr);
769 
770 	memset(hdev->queues, 0, sizeof(hdev->queues));
771 	memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
772 	memset(&hdev->sfr, 0, sizeof(hdev->sfr));
773 
774 	mutex_unlock(&hdev->lock);
775 }
776 
venus_interface_queues_init(struct venus_hfi_device * hdev)777 static int venus_interface_queues_init(struct venus_hfi_device *hdev)
778 {
779 	struct hfi_queue_table_header *tbl_hdr;
780 	struct iface_queue *queue;
781 	struct hfi_sfr *sfr;
782 	struct mem_desc desc = {0};
783 	unsigned int offset;
784 	unsigned int i;
785 	int ret;
786 
787 	ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
788 	if (ret)
789 		return ret;
790 
791 	hdev->ifaceq_table = desc;
792 	offset = IFACEQ_TABLE_SIZE;
793 
794 	for (i = 0; i < IFACEQ_NUM; i++) {
795 		queue = &hdev->queues[i];
796 		queue->qmem.da = desc.da + offset;
797 		queue->qmem.kva = desc.kva + offset;
798 		queue->qmem.size = IFACEQ_QUEUE_SIZE;
799 		offset += queue->qmem.size;
800 		queue->qhdr =
801 			IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
802 
803 		venus_set_qhdr_defaults(queue->qhdr);
804 
805 		queue->qhdr->start_addr = queue->qmem.da;
806 
807 		if (i == IFACEQ_CMD_IDX)
808 			queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
809 		else if (i == IFACEQ_MSG_IDX)
810 			queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
811 		else if (i == IFACEQ_DBG_IDX)
812 			queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
813 	}
814 
815 	tbl_hdr = hdev->ifaceq_table.kva;
816 	tbl_hdr->version = 0;
817 	tbl_hdr->size = IFACEQ_TABLE_SIZE;
818 	tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
819 	tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
820 	tbl_hdr->num_q = IFACEQ_NUM;
821 	tbl_hdr->num_active_q = IFACEQ_NUM;
822 
823 	/*
824 	 * Set receive request to zero on debug queue as there is no
825 	 * need of interrupt from video hardware for debug messages
826 	 */
827 	queue = &hdev->queues[IFACEQ_DBG_IDX];
828 	queue->qhdr->rx_req = 0;
829 
830 	ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
831 	if (ret) {
832 		hdev->sfr.da = 0;
833 	} else {
834 		hdev->sfr = desc;
835 		sfr = hdev->sfr.kva;
836 		sfr->buf_size = ALIGNED_SFR_SIZE;
837 	}
838 
839 	/* ensure table and queue header structs are settled in memory */
840 	wmb();
841 
842 	return 0;
843 }
844 
venus_sys_set_debug(struct venus_hfi_device * hdev,u32 debug)845 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
846 {
847 	struct hfi_sys_set_property_pkt *pkt;
848 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
849 
850 	pkt = (struct hfi_sys_set_property_pkt *)packet;
851 
852 	pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
853 
854 	return venus_iface_cmdq_write(hdev, pkt, false);
855 }
856 
venus_sys_set_coverage(struct venus_hfi_device * hdev,u32 mode)857 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
858 {
859 	struct hfi_sys_set_property_pkt *pkt;
860 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
861 
862 	pkt = (struct hfi_sys_set_property_pkt *)packet;
863 
864 	pkt_sys_coverage_config(pkt, mode);
865 
866 	return venus_iface_cmdq_write(hdev, pkt, false);
867 }
868 
venus_sys_set_idle_message(struct venus_hfi_device * hdev,bool enable)869 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
870 				      bool enable)
871 {
872 	struct hfi_sys_set_property_pkt *pkt;
873 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
874 
875 	if (!enable)
876 		return 0;
877 
878 	pkt = (struct hfi_sys_set_property_pkt *)packet;
879 
880 	pkt_sys_idle_indicator(pkt, enable);
881 
882 	return venus_iface_cmdq_write(hdev, pkt, false);
883 }
884 
venus_sys_set_power_control(struct venus_hfi_device * hdev,bool enable)885 static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
886 				       bool enable)
887 {
888 	struct hfi_sys_set_property_pkt *pkt;
889 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
890 
891 	pkt = (struct hfi_sys_set_property_pkt *)packet;
892 
893 	pkt_sys_power_control(pkt, enable);
894 
895 	return venus_iface_cmdq_write(hdev, pkt, false);
896 }
897 
venus_sys_set_ubwc_config(struct venus_hfi_device * hdev)898 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev)
899 {
900 	struct hfi_sys_set_property_pkt *pkt;
901 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
902 	const struct venus_resources *res = hdev->core->res;
903 	int ret;
904 
905 	pkt = (struct hfi_sys_set_property_pkt *)packet;
906 
907 	pkt_sys_ubwc_config(pkt, res->ubwc_conf);
908 
909 	ret = venus_iface_cmdq_write(hdev, pkt, false);
910 	if (ret)
911 		return ret;
912 
913 	return 0;
914 }
915 
venus_get_queue_size(struct venus_hfi_device * hdev,unsigned int index)916 static int venus_get_queue_size(struct venus_hfi_device *hdev,
917 				unsigned int index)
918 {
919 	struct hfi_queue_header *qhdr;
920 
921 	if (index >= IFACEQ_NUM)
922 		return -EINVAL;
923 
924 	qhdr = hdev->queues[index].qhdr;
925 	if (!qhdr)
926 		return -EINVAL;
927 
928 	return abs(qhdr->read_idx - qhdr->write_idx);
929 }
930 
venus_sys_set_default_properties(struct venus_hfi_device * hdev)931 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
932 {
933 	struct device *dev = hdev->core->dev;
934 	const struct venus_resources *res = hdev->core->res;
935 	int ret;
936 
937 	ret = venus_sys_set_debug(hdev, venus_fw_debug);
938 	if (ret)
939 		dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
940 
941 	/* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */
942 	if (IS_V1(hdev->core)) {
943 		ret = venus_sys_set_idle_message(hdev, false);
944 		if (ret)
945 			dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
946 	}
947 
948 	ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
949 	if (ret)
950 		dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
951 			 ret);
952 
953 	/* For specific venus core, it is mandatory to set the UBWC configuration */
954 	if (res->ubwc_conf) {
955 		ret = venus_sys_set_ubwc_config(hdev);
956 		if (ret)
957 			dev_warn(dev, "setting ubwc config failed (%d)\n", ret);
958 	}
959 
960 	return ret;
961 }
962 
venus_session_cmd(struct venus_inst * inst,u32 pkt_type,bool sync)963 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)
964 {
965 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
966 	struct hfi_session_pkt pkt;
967 
968 	pkt_session_cmd(&pkt, pkt_type, inst);
969 
970 	return venus_iface_cmdq_write(hdev, &pkt, sync);
971 }
972 
venus_flush_debug_queue(struct venus_hfi_device * hdev)973 static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
974 {
975 	struct device *dev = hdev->core->dev;
976 	void *packet = hdev->dbg_buf;
977 
978 	while (!venus_iface_dbgq_read(hdev, packet)) {
979 		struct hfi_msg_sys_coverage_pkt *pkt = packet;
980 
981 		if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
982 			struct hfi_msg_sys_debug_pkt *pkt = packet;
983 
984 			dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
985 		}
986 	}
987 }
988 
venus_prepare_power_collapse(struct venus_hfi_device * hdev,bool wait)989 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
990 					bool wait)
991 {
992 	unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
993 	struct hfi_sys_pc_prep_pkt pkt;
994 	int ret;
995 
996 	init_completion(&hdev->pwr_collapse_prep);
997 
998 	pkt_sys_pc_prep(&pkt);
999 
1000 	ret = venus_iface_cmdq_write(hdev, &pkt, false);
1001 	if (ret)
1002 		return ret;
1003 
1004 	if (!wait)
1005 		return 0;
1006 
1007 	ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
1008 	if (!ret) {
1009 		venus_flush_debug_queue(hdev);
1010 		return -ETIMEDOUT;
1011 	}
1012 
1013 	return 0;
1014 }
1015 
venus_are_queues_empty(struct venus_hfi_device * hdev)1016 static int venus_are_queues_empty(struct venus_hfi_device *hdev)
1017 {
1018 	int ret1, ret2;
1019 
1020 	ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
1021 	if (ret1 < 0)
1022 		return ret1;
1023 
1024 	ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
1025 	if (ret2 < 0)
1026 		return ret2;
1027 
1028 	if (!ret1 && !ret2)
1029 		return 1;
1030 
1031 	return 0;
1032 }
1033 
venus_sfr_print(struct venus_hfi_device * hdev)1034 static void venus_sfr_print(struct venus_hfi_device *hdev)
1035 {
1036 	struct device *dev = hdev->core->dev;
1037 	struct hfi_sfr *sfr = hdev->sfr.kva;
1038 	void *p;
1039 
1040 	if (!sfr)
1041 		return;
1042 
1043 	p = memchr(sfr->data, '\0', sfr->buf_size);
1044 	/*
1045 	 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
1046 	 * that Venus is in the process of crashing.
1047 	 */
1048 	if (!p)
1049 		sfr->data[sfr->buf_size - 1] = '\0';
1050 
1051 	dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
1052 }
1053 
venus_process_msg_sys_error(struct venus_hfi_device * hdev,void * packet)1054 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
1055 					void *packet)
1056 {
1057 	struct hfi_msg_event_notify_pkt *event_pkt = packet;
1058 
1059 	if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
1060 		return;
1061 
1062 	venus_set_state(hdev, VENUS_STATE_DEINIT);
1063 
1064 	venus_sfr_print(hdev);
1065 }
1066 
venus_isr_thread(struct venus_core * core)1067 static irqreturn_t venus_isr_thread(struct venus_core *core)
1068 {
1069 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1070 	const struct venus_resources *res;
1071 	void *pkt;
1072 	u32 msg_ret;
1073 
1074 	if (!hdev)
1075 		return IRQ_NONE;
1076 
1077 	res = hdev->core->res;
1078 	pkt = hdev->pkt_buf;
1079 
1080 
1081 	while (!venus_iface_msgq_read(hdev, pkt)) {
1082 		msg_ret = hfi_process_msg_packet(core, pkt);
1083 		switch (msg_ret) {
1084 		case HFI_MSG_EVENT_NOTIFY:
1085 			venus_process_msg_sys_error(hdev, pkt);
1086 			break;
1087 		case HFI_MSG_SYS_INIT:
1088 			venus_hfi_core_set_resource(core, res->vmem_id,
1089 						    res->vmem_size,
1090 						    res->vmem_addr,
1091 						    hdev);
1092 			break;
1093 		case HFI_MSG_SYS_RELEASE_RESOURCE:
1094 			complete(&hdev->release_resource);
1095 			break;
1096 		case HFI_MSG_SYS_PC_PREP:
1097 			complete(&hdev->pwr_collapse_prep);
1098 			break;
1099 		default:
1100 			break;
1101 		}
1102 	}
1103 
1104 	venus_flush_debug_queue(hdev);
1105 
1106 	return IRQ_HANDLED;
1107 }
1108 
venus_isr(struct venus_core * core)1109 static irqreturn_t venus_isr(struct venus_core *core)
1110 {
1111 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1112 	u32 status;
1113 	void __iomem *cpu_cs_base;
1114 	void __iomem *wrapper_base;
1115 
1116 	if (!hdev)
1117 		return IRQ_NONE;
1118 
1119 	cpu_cs_base = hdev->core->cpu_cs_base;
1120 	wrapper_base = hdev->core->wrapper_base;
1121 
1122 	status = readl(wrapper_base + WRAPPER_INTR_STATUS);
1123 	if (IS_IRIS2(core) || IS_IRIS2_1(core)) {
1124 		if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1125 		    status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||
1126 		    status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1127 			hdev->irq_status = status;
1128 	} else {
1129 		if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1130 		    status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
1131 		    status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1132 			hdev->irq_status = status;
1133 	}
1134 	writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);
1135 	if (!(IS_IRIS2(core) || IS_IRIS2_1(core)))
1136 		writel(status, wrapper_base + WRAPPER_INTR_CLEAR);
1137 
1138 	return IRQ_WAKE_THREAD;
1139 }
1140 
venus_core_init(struct venus_core * core)1141 static int venus_core_init(struct venus_core *core)
1142 {
1143 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1144 	struct device *dev = core->dev;
1145 	struct hfi_sys_get_property_pkt version_pkt;
1146 	struct hfi_sys_init_pkt pkt;
1147 	int ret;
1148 
1149 	pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
1150 
1151 	venus_set_state(hdev, VENUS_STATE_INIT);
1152 
1153 	ret = venus_iface_cmdq_write(hdev, &pkt, false);
1154 	if (ret)
1155 		return ret;
1156 
1157 	pkt_sys_image_version(&version_pkt);
1158 
1159 	ret = venus_iface_cmdq_write(hdev, &version_pkt, false);
1160 	if (ret)
1161 		dev_warn(dev, "failed to send image version pkt to fw\n");
1162 
1163 	ret = venus_sys_set_default_properties(hdev);
1164 	if (ret)
1165 		return ret;
1166 
1167 	return 0;
1168 }
1169 
venus_core_deinit(struct venus_core * core)1170 static int venus_core_deinit(struct venus_core *core)
1171 {
1172 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1173 
1174 	venus_set_state(hdev, VENUS_STATE_DEINIT);
1175 	hdev->suspended = true;
1176 	hdev->power_enabled = false;
1177 
1178 	return 0;
1179 }
1180 
venus_core_ping(struct venus_core * core,u32 cookie)1181 static int venus_core_ping(struct venus_core *core, u32 cookie)
1182 {
1183 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1184 	struct hfi_sys_ping_pkt pkt;
1185 
1186 	pkt_sys_ping(&pkt, cookie);
1187 
1188 	return venus_iface_cmdq_write(hdev, &pkt, false);
1189 }
1190 
venus_core_trigger_ssr(struct venus_core * core,u32 trigger_type)1191 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
1192 {
1193 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1194 	struct hfi_sys_test_ssr_pkt pkt;
1195 	int ret;
1196 
1197 	ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
1198 	if (ret)
1199 		return ret;
1200 
1201 	return venus_iface_cmdq_write(hdev, &pkt, false);
1202 }
1203 
venus_session_init(struct venus_inst * inst,u32 session_type,u32 codec)1204 static int venus_session_init(struct venus_inst *inst, u32 session_type,
1205 			      u32 codec)
1206 {
1207 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1208 	struct hfi_session_init_pkt pkt;
1209 	int ret;
1210 
1211 	ret = venus_sys_set_debug(hdev, venus_fw_debug);
1212 	if (ret)
1213 		goto err;
1214 
1215 	ret = pkt_session_init(&pkt, inst, session_type, codec);
1216 	if (ret)
1217 		goto err;
1218 
1219 	ret = venus_iface_cmdq_write(hdev, &pkt, true);
1220 	if (ret)
1221 		goto err;
1222 
1223 	return 0;
1224 
1225 err:
1226 	venus_flush_debug_queue(hdev);
1227 	return ret;
1228 }
1229 
venus_session_end(struct venus_inst * inst)1230 static int venus_session_end(struct venus_inst *inst)
1231 {
1232 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1233 	struct device *dev = hdev->core->dev;
1234 
1235 	if (venus_fw_coverage) {
1236 		if (venus_sys_set_coverage(hdev, venus_fw_coverage))
1237 			dev_warn(dev, "fw coverage msg ON failed\n");
1238 	}
1239 
1240 	return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);
1241 }
1242 
venus_session_abort(struct venus_inst * inst)1243 static int venus_session_abort(struct venus_inst *inst)
1244 {
1245 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1246 
1247 	venus_flush_debug_queue(hdev);
1248 
1249 	return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);
1250 }
1251 
venus_session_flush(struct venus_inst * inst,u32 flush_mode)1252 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
1253 {
1254 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1255 	struct hfi_session_flush_pkt pkt;
1256 	int ret;
1257 
1258 	ret = pkt_session_flush(&pkt, inst, flush_mode);
1259 	if (ret)
1260 		return ret;
1261 
1262 	return venus_iface_cmdq_write(hdev, &pkt, true);
1263 }
1264 
venus_session_start(struct venus_inst * inst)1265 static int venus_session_start(struct venus_inst *inst)
1266 {
1267 	return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);
1268 }
1269 
venus_session_stop(struct venus_inst * inst)1270 static int venus_session_stop(struct venus_inst *inst)
1271 {
1272 	return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);
1273 }
1274 
venus_session_continue(struct venus_inst * inst)1275 static int venus_session_continue(struct venus_inst *inst)
1276 {
1277 	return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);
1278 }
1279 
venus_session_etb(struct venus_inst * inst,struct hfi_frame_data * in_frame)1280 static int venus_session_etb(struct venus_inst *inst,
1281 			     struct hfi_frame_data *in_frame)
1282 {
1283 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1284 	u32 session_type = inst->session_type;
1285 	int ret;
1286 
1287 	if (session_type == VIDC_SESSION_TYPE_DEC) {
1288 		struct hfi_session_empty_buffer_compressed_pkt pkt;
1289 
1290 		ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
1291 		if (ret)
1292 			return ret;
1293 
1294 		ret = venus_iface_cmdq_write(hdev, &pkt, false);
1295 	} else if (session_type == VIDC_SESSION_TYPE_ENC) {
1296 		struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
1297 
1298 		ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
1299 		if (ret)
1300 			return ret;
1301 
1302 		ret = venus_iface_cmdq_write(hdev, &pkt, false);
1303 	} else {
1304 		ret = -EINVAL;
1305 	}
1306 
1307 	return ret;
1308 }
1309 
venus_session_ftb(struct venus_inst * inst,struct hfi_frame_data * out_frame)1310 static int venus_session_ftb(struct venus_inst *inst,
1311 			     struct hfi_frame_data *out_frame)
1312 {
1313 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1314 	struct hfi_session_fill_buffer_pkt pkt;
1315 	int ret;
1316 
1317 	ret = pkt_session_ftb(&pkt, inst, out_frame);
1318 	if (ret)
1319 		return ret;
1320 
1321 	return venus_iface_cmdq_write(hdev, &pkt, false);
1322 }
1323 
venus_session_set_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1324 static int venus_session_set_buffers(struct venus_inst *inst,
1325 				     struct hfi_buffer_desc *bd)
1326 {
1327 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1328 	struct hfi_session_set_buffers_pkt *pkt;
1329 	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1330 	int ret;
1331 
1332 	if (bd->buffer_type == HFI_BUFFER_INPUT)
1333 		return 0;
1334 
1335 	pkt = (struct hfi_session_set_buffers_pkt *)packet;
1336 
1337 	ret = pkt_session_set_buffers(pkt, inst, bd);
1338 	if (ret)
1339 		return ret;
1340 
1341 	return venus_iface_cmdq_write(hdev, pkt, false);
1342 }
1343 
venus_session_unset_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1344 static int venus_session_unset_buffers(struct venus_inst *inst,
1345 				       struct hfi_buffer_desc *bd)
1346 {
1347 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1348 	struct hfi_session_release_buffer_pkt *pkt;
1349 	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1350 	int ret;
1351 
1352 	if (bd->buffer_type == HFI_BUFFER_INPUT)
1353 		return 0;
1354 
1355 	pkt = (struct hfi_session_release_buffer_pkt *)packet;
1356 
1357 	ret = pkt_session_unset_buffers(pkt, inst, bd);
1358 	if (ret)
1359 		return ret;
1360 
1361 	return venus_iface_cmdq_write(hdev, pkt, true);
1362 }
1363 
venus_session_load_res(struct venus_inst * inst)1364 static int venus_session_load_res(struct venus_inst *inst)
1365 {
1366 	return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);
1367 }
1368 
venus_session_release_res(struct venus_inst * inst)1369 static int venus_session_release_res(struct venus_inst *inst)
1370 {
1371 	return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);
1372 }
1373 
venus_session_parse_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1374 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1375 				       u32 seq_hdr_len)
1376 {
1377 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1378 	struct hfi_session_parse_sequence_header_pkt *pkt;
1379 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1380 	int ret;
1381 
1382 	pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
1383 
1384 	ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
1385 	if (ret)
1386 		return ret;
1387 
1388 	ret = venus_iface_cmdq_write(hdev, pkt, false);
1389 	if (ret)
1390 		return ret;
1391 
1392 	return 0;
1393 }
1394 
venus_session_get_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1395 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1396 				     u32 seq_hdr_len)
1397 {
1398 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1399 	struct hfi_session_get_sequence_header_pkt *pkt;
1400 	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1401 	int ret;
1402 
1403 	pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
1404 
1405 	ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
1406 	if (ret)
1407 		return ret;
1408 
1409 	return venus_iface_cmdq_write(hdev, pkt, false);
1410 }
1411 
venus_session_set_property(struct venus_inst * inst,u32 ptype,void * pdata)1412 static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
1413 				      void *pdata)
1414 {
1415 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1416 	struct hfi_session_set_property_pkt *pkt;
1417 	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1418 	int ret;
1419 
1420 	pkt = (struct hfi_session_set_property_pkt *)packet;
1421 
1422 	ret = pkt_session_set_property(pkt, inst, ptype, pdata);
1423 	if (ret == -ENOTSUPP)
1424 		return 0;
1425 	if (ret)
1426 		return ret;
1427 
1428 	return venus_iface_cmdq_write(hdev, pkt, false);
1429 }
1430 
venus_session_get_property(struct venus_inst * inst,u32 ptype)1431 static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
1432 {
1433 	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1434 	struct hfi_session_get_property_pkt pkt;
1435 	int ret;
1436 
1437 	ret = pkt_session_get_property(&pkt, inst, ptype);
1438 	if (ret)
1439 		return ret;
1440 
1441 	return venus_iface_cmdq_write(hdev, &pkt, true);
1442 }
1443 
venus_resume(struct venus_core * core)1444 static int venus_resume(struct venus_core *core)
1445 {
1446 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1447 	int ret = 0;
1448 
1449 	mutex_lock(&hdev->lock);
1450 
1451 	if (!hdev->suspended)
1452 		goto unlock;
1453 
1454 	ret = venus_power_on(hdev);
1455 
1456 unlock:
1457 	if (!ret)
1458 		hdev->suspended = false;
1459 
1460 	mutex_unlock(&hdev->lock);
1461 
1462 	return ret;
1463 }
1464 
venus_suspend_1xx(struct venus_core * core)1465 static int venus_suspend_1xx(struct venus_core *core)
1466 {
1467 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1468 	struct device *dev = core->dev;
1469 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1470 	u32 ctrl_status;
1471 	int ret;
1472 
1473 	if (!hdev->power_enabled || hdev->suspended)
1474 		return 0;
1475 
1476 	mutex_lock(&hdev->lock);
1477 	ret = venus_is_valid_state(hdev);
1478 	mutex_unlock(&hdev->lock);
1479 
1480 	if (!ret) {
1481 		dev_err(dev, "bad state, cannot suspend\n");
1482 		return -EINVAL;
1483 	}
1484 
1485 	ret = venus_prepare_power_collapse(hdev, true);
1486 	if (ret) {
1487 		dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1488 		return ret;
1489 	}
1490 
1491 	mutex_lock(&hdev->lock);
1492 
1493 	if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
1494 		mutex_unlock(&hdev->lock);
1495 		return -EINVAL;
1496 	}
1497 
1498 	ret = venus_are_queues_empty(hdev);
1499 	if (ret < 0 || !ret) {
1500 		mutex_unlock(&hdev->lock);
1501 		return -EINVAL;
1502 	}
1503 
1504 	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1505 	if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
1506 		mutex_unlock(&hdev->lock);
1507 		return -EINVAL;
1508 	}
1509 
1510 	ret = venus_power_off(hdev);
1511 	if (ret) {
1512 		mutex_unlock(&hdev->lock);
1513 		return ret;
1514 	}
1515 
1516 	hdev->suspended = true;
1517 
1518 	mutex_unlock(&hdev->lock);
1519 
1520 	return 0;
1521 }
1522 
venus_cpu_and_video_core_idle(struct venus_hfi_device * hdev)1523 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
1524 {
1525 	void __iomem *wrapper_base = hdev->core->wrapper_base;
1526 	void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1527 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1528 	u32 ctrl_status, cpu_status;
1529 
1530 	if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1531 		cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1532 	else
1533 		cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1534 	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1535 
1536 	if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1537 	    ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1538 		return true;
1539 
1540 	return false;
1541 }
1542 
venus_cpu_idle_and_pc_ready(struct venus_hfi_device * hdev)1543 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
1544 {
1545 	void __iomem *wrapper_base = hdev->core->wrapper_base;
1546 	void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1547 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1548 	u32 ctrl_status, cpu_status;
1549 
1550 	if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1551 		cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1552 	else
1553 		cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1554 	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1555 
1556 	if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1557 	    ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1558 		return true;
1559 
1560 	return false;
1561 }
1562 
venus_suspend_3xx(struct venus_core * core)1563 static int venus_suspend_3xx(struct venus_core *core)
1564 {
1565 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1566 	struct device *dev = core->dev;
1567 	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1568 	u32 ctrl_status;
1569 	bool val;
1570 	int ret;
1571 
1572 	if (!hdev->power_enabled || hdev->suspended)
1573 		return 0;
1574 
1575 	mutex_lock(&hdev->lock);
1576 	ret = venus_is_valid_state(hdev);
1577 	mutex_unlock(&hdev->lock);
1578 
1579 	if (!ret) {
1580 		dev_err(dev, "bad state, cannot suspend\n");
1581 		return -EINVAL;
1582 	}
1583 
1584 	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1585 	if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1586 		goto power_off;
1587 
1588 	/*
1589 	 * Power collapse sequence for Venus 3xx and 4xx versions:
1590 	 * 1. Check for ARM9 and video core to be idle by checking WFI bit
1591 	 *    (bit 0) in CPU status register and by checking Idle (bit 30) in
1592 	 *    Control status register for video core.
1593 	 * 2. Send a command to prepare for power collapse.
1594 	 * 3. Check for WFI and PC_READY bits.
1595 	 */
1596 	ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
1597 				 1500, 100 * 1500);
1598 	if (ret) {
1599 		dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret);
1600 		return ret;
1601 	}
1602 
1603 	ret = venus_prepare_power_collapse(hdev, false);
1604 	if (ret) {
1605 		dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1606 		return ret;
1607 	}
1608 
1609 	ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
1610 				 1500, 100 * 1500);
1611 	if (ret)
1612 		return ret;
1613 
1614 power_off:
1615 	mutex_lock(&hdev->lock);
1616 
1617 	ret = venus_power_off(hdev);
1618 	if (ret) {
1619 		dev_err(dev, "venus_power_off (%d)\n", ret);
1620 		mutex_unlock(&hdev->lock);
1621 		return ret;
1622 	}
1623 
1624 	hdev->suspended = true;
1625 
1626 	mutex_unlock(&hdev->lock);
1627 
1628 	return 0;
1629 }
1630 
venus_suspend(struct venus_core * core)1631 static int venus_suspend(struct venus_core *core)
1632 {
1633 	if (IS_V3(core) || IS_V4(core) || IS_V6(core))
1634 		return venus_suspend_3xx(core);
1635 
1636 	return venus_suspend_1xx(core);
1637 }
1638 
1639 static const struct hfi_ops venus_hfi_ops = {
1640 	.core_init			= venus_core_init,
1641 	.core_deinit			= venus_core_deinit,
1642 	.core_ping			= venus_core_ping,
1643 	.core_trigger_ssr		= venus_core_trigger_ssr,
1644 
1645 	.session_init			= venus_session_init,
1646 	.session_end			= venus_session_end,
1647 	.session_abort			= venus_session_abort,
1648 	.session_flush			= venus_session_flush,
1649 	.session_start			= venus_session_start,
1650 	.session_stop			= venus_session_stop,
1651 	.session_continue		= venus_session_continue,
1652 	.session_etb			= venus_session_etb,
1653 	.session_ftb			= venus_session_ftb,
1654 	.session_set_buffers		= venus_session_set_buffers,
1655 	.session_unset_buffers		= venus_session_unset_buffers,
1656 	.session_load_res		= venus_session_load_res,
1657 	.session_release_res		= venus_session_release_res,
1658 	.session_parse_seq_hdr		= venus_session_parse_seq_hdr,
1659 	.session_get_seq_hdr		= venus_session_get_seq_hdr,
1660 	.session_set_property		= venus_session_set_property,
1661 	.session_get_property		= venus_session_get_property,
1662 
1663 	.resume				= venus_resume,
1664 	.suspend			= venus_suspend,
1665 
1666 	.isr				= venus_isr,
1667 	.isr_thread			= venus_isr_thread,
1668 };
1669 
venus_hfi_destroy(struct venus_core * core)1670 void venus_hfi_destroy(struct venus_core *core)
1671 {
1672 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1673 
1674 	core->priv = NULL;
1675 	venus_interface_queues_release(hdev);
1676 	mutex_destroy(&hdev->lock);
1677 	kfree(hdev);
1678 	core->ops = NULL;
1679 }
1680 
venus_hfi_create(struct venus_core * core)1681 int venus_hfi_create(struct venus_core *core)
1682 {
1683 	struct venus_hfi_device *hdev;
1684 	int ret;
1685 
1686 	hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
1687 	if (!hdev)
1688 		return -ENOMEM;
1689 
1690 	mutex_init(&hdev->lock);
1691 
1692 	hdev->core = core;
1693 	hdev->suspended = true;
1694 	core->priv = hdev;
1695 	core->ops = &venus_hfi_ops;
1696 
1697 	ret = venus_interface_queues_init(hdev);
1698 	if (ret)
1699 		goto err_kfree;
1700 
1701 	return 0;
1702 
1703 err_kfree:
1704 	kfree(hdev);
1705 	core->priv = NULL;
1706 	core->ops = NULL;
1707 	return ret;
1708 }
1709 
venus_hfi_queues_reinit(struct venus_core * core)1710 void venus_hfi_queues_reinit(struct venus_core *core)
1711 {
1712 	struct venus_hfi_device *hdev = to_hfi_priv(core);
1713 	struct hfi_queue_table_header *tbl_hdr;
1714 	struct iface_queue *queue;
1715 	struct hfi_sfr *sfr;
1716 	unsigned int i;
1717 
1718 	mutex_lock(&hdev->lock);
1719 
1720 	for (i = 0; i < IFACEQ_NUM; i++) {
1721 		queue = &hdev->queues[i];
1722 		queue->qhdr =
1723 			IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
1724 
1725 		venus_set_qhdr_defaults(queue->qhdr);
1726 
1727 		queue->qhdr->start_addr = queue->qmem.da;
1728 
1729 		if (i == IFACEQ_CMD_IDX)
1730 			queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
1731 		else if (i == IFACEQ_MSG_IDX)
1732 			queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
1733 		else if (i == IFACEQ_DBG_IDX)
1734 			queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
1735 	}
1736 
1737 	tbl_hdr = hdev->ifaceq_table.kva;
1738 	tbl_hdr->version = 0;
1739 	tbl_hdr->size = IFACEQ_TABLE_SIZE;
1740 	tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
1741 	tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
1742 	tbl_hdr->num_q = IFACEQ_NUM;
1743 	tbl_hdr->num_active_q = IFACEQ_NUM;
1744 
1745 	/*
1746 	 * Set receive request to zero on debug queue as there is no
1747 	 * need of interrupt from video hardware for debug messages
1748 	 */
1749 	queue = &hdev->queues[IFACEQ_DBG_IDX];
1750 	queue->qhdr->rx_req = 0;
1751 
1752 	sfr = hdev->sfr.kva;
1753 	sfr->buf_size = ALIGNED_SFR_SIZE;
1754 
1755 	/* ensure table and queue header structs are settled in memory */
1756 	wmb();
1757 
1758 	mutex_unlock(&hdev->lock);
1759 }
1760