1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2017 Linaro Ltd.
5 */
6
7 #include <linux/delay.h>
8 #include <linux/device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/interrupt.h>
11 #include <linux/iopoll.h>
12 #include <linux/kernel.h>
13 #include <linux/slab.h>
14
15 #include "core.h"
16 #include "hfi_cmds.h"
17 #include "hfi_msgs.h"
18 #include "hfi_venus.h"
19 #include "hfi_venus_io.h"
20 #include "firmware.h"
21
22 #define HFI_MASK_QHDR_TX_TYPE 0xff000000
23 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000
24 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00
25 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff
26
27 #define HFI_HOST_TO_CTRL_CMD_Q 0
28 #define HFI_CTRL_TO_HOST_MSG_Q 1
29 #define HFI_CTRL_TO_HOST_DBG_Q 2
30 #define HFI_MASK_QHDR_STATUS 0x000000ff
31
32 #define IFACEQ_NUM 3
33 #define IFACEQ_CMD_IDX 0
34 #define IFACEQ_MSG_IDX 1
35 #define IFACEQ_DBG_IDX 2
36 #define IFACEQ_MAX_BUF_COUNT 50
37 #define IFACEQ_MAX_PARALLEL_CLNTS 16
38 #define IFACEQ_DFLT_QHDR 0x01010000
39
40 #define POLL_INTERVAL_US 50
41
42 #define IFACEQ_MAX_PKT_SIZE 1024
43 #define IFACEQ_MED_PKT_SIZE 768
44 #define IFACEQ_MIN_PKT_SIZE 8
45 #define IFACEQ_VAR_SMALL_PKT_SIZE 100
46 #define IFACEQ_VAR_LARGE_PKT_SIZE 512
47 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12)
48
49 struct hfi_queue_table_header {
50 u32 version;
51 u32 size;
52 u32 qhdr0_offset;
53 u32 qhdr_size;
54 u32 num_q;
55 u32 num_active_q;
56 };
57
58 struct hfi_queue_header {
59 u32 status;
60 u32 start_addr;
61 u32 type;
62 u32 q_size;
63 u32 pkt_size;
64 u32 pkt_drop_cnt;
65 u32 rx_wm;
66 u32 tx_wm;
67 u32 rx_req;
68 u32 tx_req;
69 u32 rx_irq_status;
70 u32 tx_irq_status;
71 u32 read_idx;
72 u32 write_idx;
73 };
74
75 #define IFACEQ_TABLE_SIZE \
76 (sizeof(struct hfi_queue_table_header) + \
77 sizeof(struct hfi_queue_header) * IFACEQ_NUM)
78
79 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \
80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
81
82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \
83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \
84 ((i) * sizeof(struct hfi_queue_header)))
85
86 #define QDSS_SIZE SZ_4K
87 #define SFR_SIZE SZ_4K
88 #define QUEUE_SIZE \
89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
90
91 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K)
92 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K)
93 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K)
94 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
95 ALIGNED_QDSS_SIZE, SZ_1M)
96
97 struct mem_desc {
98 dma_addr_t da; /* device address */
99 void *kva; /* kernel virtual address */
100 u32 size;
101 unsigned long attrs;
102 };
103
104 struct iface_queue {
105 struct hfi_queue_header *qhdr;
106 struct mem_desc qmem;
107 };
108
109 enum venus_state {
110 VENUS_STATE_DEINIT = 1,
111 VENUS_STATE_INIT,
112 };
113
114 struct venus_hfi_device {
115 struct venus_core *core;
116 u32 irq_status;
117 u32 last_packet_type;
118 bool power_enabled;
119 bool suspended;
120 enum venus_state state;
121 /* serialize read / write to the shared memory */
122 struct mutex lock;
123 struct completion pwr_collapse_prep;
124 struct completion release_resource;
125 struct mem_desc ifaceq_table;
126 struct mem_desc sfr;
127 struct iface_queue queues[IFACEQ_NUM];
128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
130 };
131
132 static bool venus_pkt_debug;
133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
134 static bool venus_fw_low_power_mode = true;
135 static int venus_hw_rsp_timeout = 1000;
136 static bool venus_fw_coverage;
137
venus_set_state(struct venus_hfi_device * hdev,enum venus_state state)138 static void venus_set_state(struct venus_hfi_device *hdev,
139 enum venus_state state)
140 {
141 mutex_lock(&hdev->lock);
142 hdev->state = state;
143 mutex_unlock(&hdev->lock);
144 }
145
venus_is_valid_state(struct venus_hfi_device * hdev)146 static bool venus_is_valid_state(struct venus_hfi_device *hdev)
147 {
148 return hdev->state != VENUS_STATE_DEINIT;
149 }
150
venus_dump_packet(struct venus_hfi_device * hdev,const void * packet)151 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
152 {
153 size_t pkt_size = *(u32 *)packet;
154
155 if (!venus_pkt_debug)
156 return;
157
158 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
159 pkt_size, true);
160 }
161
venus_write_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * packet,u32 * rx_req)162 static int venus_write_queue(struct venus_hfi_device *hdev,
163 struct iface_queue *queue,
164 void *packet, u32 *rx_req)
165 {
166 struct hfi_queue_header *qhdr;
167 u32 dwords, new_wr_idx;
168 u32 empty_space, rd_idx, wr_idx, qsize;
169 u32 *wr_ptr;
170
171 if (!queue->qmem.kva)
172 return -EINVAL;
173
174 qhdr = queue->qhdr;
175 if (!qhdr)
176 return -EINVAL;
177
178 venus_dump_packet(hdev, packet);
179
180 dwords = (*(u32 *)packet) >> 2;
181 if (!dwords)
182 return -EINVAL;
183
184 rd_idx = qhdr->read_idx;
185 wr_idx = qhdr->write_idx;
186 qsize = qhdr->q_size;
187 /* ensure rd/wr indices's are read from memory */
188 rmb();
189
190 if (qsize > IFACEQ_QUEUE_SIZE / 4)
191 return -EINVAL;
192
193 if (wr_idx >= rd_idx)
194 empty_space = qsize - (wr_idx - rd_idx);
195 else
196 empty_space = rd_idx - wr_idx;
197
198 if (empty_space <= dwords) {
199 qhdr->tx_req = 1;
200 /* ensure tx_req is updated in memory */
201 wmb();
202 return -ENOSPC;
203 }
204
205 qhdr->tx_req = 0;
206 /* ensure tx_req is updated in memory */
207 wmb();
208
209 new_wr_idx = wr_idx + dwords;
210 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
211
212 if (wr_ptr < (u32 *)queue->qmem.kva ||
213 wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr)))
214 return -EINVAL;
215
216 if (new_wr_idx < qsize) {
217 memcpy(wr_ptr, packet, dwords << 2);
218 } else {
219 size_t len;
220
221 new_wr_idx -= qsize;
222 len = (dwords - new_wr_idx) << 2;
223 memcpy(wr_ptr, packet, len);
224 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
225 }
226
227 /* make sure packet is written before updating the write index */
228 wmb();
229
230 qhdr->write_idx = new_wr_idx;
231 *rx_req = qhdr->rx_req ? 1 : 0;
232
233 /* make sure write index is updated before an interrupt is raised */
234 mb();
235
236 return 0;
237 }
238
venus_read_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * pkt,u32 * tx_req)239 static int venus_read_queue(struct venus_hfi_device *hdev,
240 struct iface_queue *queue, void *pkt, u32 *tx_req)
241 {
242 struct hfi_queue_header *qhdr;
243 u32 dwords, new_rd_idx;
244 u32 rd_idx, wr_idx, type, qsize;
245 u32 *rd_ptr;
246 u32 recv_request = 0;
247 int ret = 0;
248
249 if (!queue->qmem.kva)
250 return -EINVAL;
251
252 qhdr = queue->qhdr;
253 if (!qhdr)
254 return -EINVAL;
255
256 type = qhdr->type;
257 rd_idx = qhdr->read_idx;
258 wr_idx = qhdr->write_idx;
259 qsize = qhdr->q_size;
260
261 if (qsize > IFACEQ_QUEUE_SIZE / 4)
262 return -EINVAL;
263
264 /* make sure data is valid before using it */
265 rmb();
266
267 /*
268 * Do not set receive request for debug queue, if set, Venus generates
269 * interrupt for debug messages even when there is no response message
270 * available. In general debug queue will not become full as it is being
271 * emptied out for every interrupt from Venus. Venus will anyway
272 * generates interrupt if it is full.
273 */
274 if (type & HFI_CTRL_TO_HOST_MSG_Q)
275 recv_request = 1;
276
277 if (rd_idx == wr_idx) {
278 qhdr->rx_req = recv_request;
279 *tx_req = 0;
280 /* update rx_req field in memory */
281 wmb();
282 return -ENODATA;
283 }
284
285 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
286
287 if (rd_ptr < (u32 *)queue->qmem.kva ||
288 rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr)))
289 return -EINVAL;
290
291 dwords = *rd_ptr >> 2;
292 if (!dwords)
293 return -EINVAL;
294
295 new_rd_idx = rd_idx + dwords;
296 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
297 if (new_rd_idx < qsize) {
298 memcpy(pkt, rd_ptr, dwords << 2);
299 } else {
300 size_t len;
301
302 new_rd_idx -= qsize;
303 len = (dwords - new_rd_idx) << 2;
304 memcpy(pkt, rd_ptr, len);
305 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
306 }
307 } else {
308 /* bad packet received, dropping */
309 new_rd_idx = qhdr->write_idx;
310 ret = -EBADMSG;
311 }
312
313 /* ensure the packet is read before updating read index */
314 rmb();
315
316 qhdr->read_idx = new_rd_idx;
317 /* ensure updating read index */
318 wmb();
319
320 rd_idx = qhdr->read_idx;
321 wr_idx = qhdr->write_idx;
322 /* ensure rd/wr indices are read from memory */
323 rmb();
324
325 if (rd_idx != wr_idx)
326 qhdr->rx_req = 0;
327 else
328 qhdr->rx_req = recv_request;
329
330 *tx_req = qhdr->tx_req ? 1 : 0;
331
332 /* ensure rx_req is stored to memory and tx_req is loaded from memory */
333 mb();
334
335 venus_dump_packet(hdev, pkt);
336
337 return ret;
338 }
339
venus_alloc(struct venus_hfi_device * hdev,struct mem_desc * desc,u32 size)340 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
341 u32 size)
342 {
343 struct device *dev = hdev->core->dev;
344
345 desc->attrs = DMA_ATTR_WRITE_COMBINE;
346 desc->size = ALIGN(size, SZ_4K);
347
348 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
349 desc->attrs);
350 if (!desc->kva)
351 return -ENOMEM;
352
353 return 0;
354 }
355
venus_free(struct venus_hfi_device * hdev,struct mem_desc * mem)356 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
357 {
358 struct device *dev = hdev->core->dev;
359
360 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
361 }
362
venus_set_registers(struct venus_hfi_device * hdev)363 static void venus_set_registers(struct venus_hfi_device *hdev)
364 {
365 const struct venus_resources *res = hdev->core->res;
366 const struct reg_val *tbl = res->reg_tbl;
367 unsigned int count = res->reg_tbl_size;
368 unsigned int i;
369
370 for (i = 0; i < count; i++)
371 writel(tbl[i].value, hdev->core->base + tbl[i].reg);
372 }
373
venus_soft_int(struct venus_hfi_device * hdev)374 static void venus_soft_int(struct venus_hfi_device *hdev)
375 {
376 void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;
377 u32 clear_bit;
378
379 if (IS_V6(hdev->core))
380 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);
381 else
382 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);
383
384 writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);
385 }
386
venus_iface_cmdq_write_nolock(struct venus_hfi_device * hdev,void * pkt,bool sync)387 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
388 void *pkt, bool sync)
389 {
390 struct device *dev = hdev->core->dev;
391 struct hfi_pkt_hdr *cmd_packet;
392 struct iface_queue *queue;
393 u32 rx_req;
394 int ret;
395
396 if (!venus_is_valid_state(hdev))
397 return -EINVAL;
398
399 cmd_packet = (struct hfi_pkt_hdr *)pkt;
400 hdev->last_packet_type = cmd_packet->pkt_type;
401
402 queue = &hdev->queues[IFACEQ_CMD_IDX];
403
404 ret = venus_write_queue(hdev, queue, pkt, &rx_req);
405 if (ret) {
406 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
407 return ret;
408 }
409
410 if (sync) {
411 /*
412 * Inform video hardware to raise interrupt for synchronous
413 * commands
414 */
415 queue = &hdev->queues[IFACEQ_MSG_IDX];
416 queue->qhdr->rx_req = 1;
417 /* ensure rx_req is updated in memory */
418 wmb();
419 }
420
421 if (rx_req)
422 venus_soft_int(hdev);
423
424 return 0;
425 }
426
venus_iface_cmdq_write(struct venus_hfi_device * hdev,void * pkt,bool sync)427 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)
428 {
429 int ret;
430
431 mutex_lock(&hdev->lock);
432 ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);
433 mutex_unlock(&hdev->lock);
434
435 return ret;
436 }
437
venus_hfi_core_set_resource(struct venus_core * core,u32 id,u32 size,u32 addr,void * cookie)438 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
439 u32 size, u32 addr, void *cookie)
440 {
441 struct venus_hfi_device *hdev = to_hfi_priv(core);
442 struct hfi_sys_set_resource_pkt *pkt;
443 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
444 int ret;
445
446 if (id == VIDC_RESOURCE_NONE)
447 return 0;
448
449 pkt = (struct hfi_sys_set_resource_pkt *)packet;
450
451 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
452 if (ret)
453 return ret;
454
455 ret = venus_iface_cmdq_write(hdev, pkt, false);
456 if (ret)
457 return ret;
458
459 return 0;
460 }
461
venus_boot_core(struct venus_hfi_device * hdev)462 static int venus_boot_core(struct venus_hfi_device *hdev)
463 {
464 struct device *dev = hdev->core->dev;
465 static const unsigned int max_tries = 100;
466 u32 ctrl_status = 0, mask_val = 0;
467 unsigned int count = 0;
468 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
469 void __iomem *wrapper_base = hdev->core->wrapper_base;
470 int ret = 0;
471
472 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
473 mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);
474 mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |
475 WRAPPER_INTR_MASK_A2HCPU_MASK);
476 } else {
477 mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;
478 }
479
480 writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);
481 if (IS_V1(hdev->core))
482 writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);
483
484 writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);
485 while (!ctrl_status && count < max_tries) {
486 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
487 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
488 dev_err(dev, "invalid setting for UC_REGION\n");
489 ret = -EINVAL;
490 break;
491 }
492
493 usleep_range(500, 1000);
494 count++;
495 }
496
497 if (count >= max_tries)
498 ret = -ETIMEDOUT;
499
500 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
501 writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);
502 writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);
503 }
504
505 return ret;
506 }
507
venus_hwversion(struct venus_hfi_device * hdev)508 static u32 venus_hwversion(struct venus_hfi_device *hdev)
509 {
510 struct device *dev = hdev->core->dev;
511 void __iomem *wrapper_base = hdev->core->wrapper_base;
512 u32 ver;
513 u32 major, minor, step;
514
515 ver = readl(wrapper_base + WRAPPER_HW_VERSION);
516 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
517 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
518 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
519 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
520 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
521
522 dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);
523
524 return major;
525 }
526
venus_run(struct venus_hfi_device * hdev)527 static int venus_run(struct venus_hfi_device *hdev)
528 {
529 struct device *dev = hdev->core->dev;
530 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
531 int ret;
532
533 /*
534 * Re-program all of the registers that get reset as a result of
535 * regulator_disable() and _enable()
536 */
537 venus_set_registers(hdev);
538
539 writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);
540 writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);
541 writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);
542 writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);
543 if (hdev->sfr.da)
544 writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);
545
546 ret = venus_boot_core(hdev);
547 if (ret) {
548 dev_err(dev, "failed to reset venus core\n");
549 return ret;
550 }
551
552 venus_hwversion(hdev);
553
554 return 0;
555 }
556
venus_halt_axi(struct venus_hfi_device * hdev)557 static int venus_halt_axi(struct venus_hfi_device *hdev)
558 {
559 void __iomem *wrapper_base = hdev->core->wrapper_base;
560 void __iomem *vbif_base = hdev->core->vbif_base;
561 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
562 void __iomem *aon_base = hdev->core->aon_base;
563 struct device *dev = hdev->core->dev;
564 u32 val;
565 u32 mask_val;
566 int ret;
567
568 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
569 writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);
570
571 if (IS_IRIS2_1(hdev->core))
572 goto skip_aon_mvp_noc;
573
574 writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);
575 ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,
576 val,
577 val & BIT(0),
578 POLL_INTERVAL_US,
579 VBIF_AXI_HALT_ACK_TIMEOUT_US);
580 if (ret)
581 return -ETIMEDOUT;
582
583 skip_aon_mvp_noc:
584 mask_val = (BIT(2) | BIT(1) | BIT(0));
585 writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
586
587 writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
588 ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,
589 val,
590 val == 0,
591 POLL_INTERVAL_US,
592 VBIF_AXI_HALT_ACK_TIMEOUT_US);
593
594 if (ret) {
595 dev_err(dev, "DBLP Release: lpi_status %x\n", val);
596 return -ETIMEDOUT;
597 }
598 return 0;
599 }
600
601 if (IS_V4(hdev->core)) {
602 val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);
603 val |= WRAPPER_CPU_AXI_HALT_HALT;
604 writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);
605
606 ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,
607 val,
608 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
609 POLL_INTERVAL_US,
610 VBIF_AXI_HALT_ACK_TIMEOUT_US);
611 if (ret) {
612 dev_err(dev, "AXI bus port halt timeout\n");
613 return ret;
614 }
615
616 return 0;
617 }
618
619 /* Halt AXI and AXI IMEM VBIF Access */
620 val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);
621 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
622 writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);
623
624 /* Request for AXI bus port halt */
625 ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,
626 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
627 POLL_INTERVAL_US,
628 VBIF_AXI_HALT_ACK_TIMEOUT_US);
629 if (ret) {
630 dev_err(dev, "AXI bus port halt timeout\n");
631 return ret;
632 }
633
634 return 0;
635 }
636
venus_power_off(struct venus_hfi_device * hdev)637 static int venus_power_off(struct venus_hfi_device *hdev)
638 {
639 int ret;
640
641 if (!hdev->power_enabled)
642 return 0;
643
644 ret = venus_set_hw_state_suspend(hdev->core);
645 if (ret)
646 return ret;
647
648 ret = venus_halt_axi(hdev);
649 if (ret)
650 return ret;
651
652 hdev->power_enabled = false;
653
654 return 0;
655 }
656
venus_power_on(struct venus_hfi_device * hdev)657 static int venus_power_on(struct venus_hfi_device *hdev)
658 {
659 int ret;
660
661 if (hdev->power_enabled)
662 return 0;
663
664 ret = venus_set_hw_state_resume(hdev->core);
665 if (ret)
666 goto err;
667
668 ret = venus_run(hdev);
669 if (ret)
670 goto err_suspend;
671
672 hdev->power_enabled = true;
673
674 return 0;
675
676 err_suspend:
677 venus_set_hw_state_suspend(hdev->core);
678 err:
679 hdev->power_enabled = false;
680 return ret;
681 }
682
venus_iface_msgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)683 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
684 void *pkt)
685 {
686 struct iface_queue *queue;
687 u32 tx_req;
688 int ret;
689
690 if (!venus_is_valid_state(hdev))
691 return -EINVAL;
692
693 queue = &hdev->queues[IFACEQ_MSG_IDX];
694
695 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
696 if (ret)
697 return ret;
698
699 if (tx_req)
700 venus_soft_int(hdev);
701
702 return 0;
703 }
704
venus_iface_msgq_read(struct venus_hfi_device * hdev,void * pkt)705 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
706 {
707 int ret;
708
709 mutex_lock(&hdev->lock);
710 ret = venus_iface_msgq_read_nolock(hdev, pkt);
711 mutex_unlock(&hdev->lock);
712
713 return ret;
714 }
715
venus_iface_dbgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)716 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
717 void *pkt)
718 {
719 struct iface_queue *queue;
720 u32 tx_req;
721 int ret;
722
723 ret = venus_is_valid_state(hdev);
724 if (!ret)
725 return -EINVAL;
726
727 queue = &hdev->queues[IFACEQ_DBG_IDX];
728
729 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
730 if (ret)
731 return ret;
732
733 if (tx_req)
734 venus_soft_int(hdev);
735
736 return 0;
737 }
738
venus_iface_dbgq_read(struct venus_hfi_device * hdev,void * pkt)739 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
740 {
741 int ret;
742
743 if (!pkt)
744 return -EINVAL;
745
746 mutex_lock(&hdev->lock);
747 ret = venus_iface_dbgq_read_nolock(hdev, pkt);
748 mutex_unlock(&hdev->lock);
749
750 return ret;
751 }
752
venus_set_qhdr_defaults(struct hfi_queue_header * qhdr)753 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
754 {
755 qhdr->status = 1;
756 qhdr->type = IFACEQ_DFLT_QHDR;
757 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
758 qhdr->pkt_size = 0;
759 qhdr->rx_wm = 1;
760 qhdr->tx_wm = 1;
761 qhdr->rx_req = 1;
762 qhdr->tx_req = 0;
763 qhdr->rx_irq_status = 0;
764 qhdr->tx_irq_status = 0;
765 qhdr->read_idx = 0;
766 qhdr->write_idx = 0;
767 }
768
venus_interface_queues_release(struct venus_hfi_device * hdev)769 static void venus_interface_queues_release(struct venus_hfi_device *hdev)
770 {
771 mutex_lock(&hdev->lock);
772
773 venus_free(hdev, &hdev->ifaceq_table);
774 venus_free(hdev, &hdev->sfr);
775
776 memset(hdev->queues, 0, sizeof(hdev->queues));
777 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
778 memset(&hdev->sfr, 0, sizeof(hdev->sfr));
779
780 mutex_unlock(&hdev->lock);
781 }
782
venus_interface_queues_init(struct venus_hfi_device * hdev)783 static int venus_interface_queues_init(struct venus_hfi_device *hdev)
784 {
785 struct hfi_queue_table_header *tbl_hdr;
786 struct iface_queue *queue;
787 struct hfi_sfr *sfr;
788 struct mem_desc desc = {0};
789 unsigned int offset;
790 unsigned int i;
791 int ret;
792
793 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
794 if (ret)
795 return ret;
796
797 hdev->ifaceq_table = desc;
798 offset = IFACEQ_TABLE_SIZE;
799
800 for (i = 0; i < IFACEQ_NUM; i++) {
801 queue = &hdev->queues[i];
802 queue->qmem.da = desc.da + offset;
803 queue->qmem.kva = desc.kva + offset;
804 queue->qmem.size = IFACEQ_QUEUE_SIZE;
805 offset += queue->qmem.size;
806 queue->qhdr =
807 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
808
809 venus_set_qhdr_defaults(queue->qhdr);
810
811 queue->qhdr->start_addr = queue->qmem.da;
812
813 if (i == IFACEQ_CMD_IDX)
814 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
815 else if (i == IFACEQ_MSG_IDX)
816 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
817 else if (i == IFACEQ_DBG_IDX)
818 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
819 }
820
821 tbl_hdr = hdev->ifaceq_table.kva;
822 tbl_hdr->version = 0;
823 tbl_hdr->size = IFACEQ_TABLE_SIZE;
824 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
825 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
826 tbl_hdr->num_q = IFACEQ_NUM;
827 tbl_hdr->num_active_q = IFACEQ_NUM;
828
829 /*
830 * Set receive request to zero on debug queue as there is no
831 * need of interrupt from video hardware for debug messages
832 */
833 queue = &hdev->queues[IFACEQ_DBG_IDX];
834 queue->qhdr->rx_req = 0;
835
836 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
837 if (ret) {
838 hdev->sfr.da = 0;
839 } else {
840 hdev->sfr = desc;
841 sfr = hdev->sfr.kva;
842 sfr->buf_size = ALIGNED_SFR_SIZE;
843 }
844
845 /* ensure table and queue header structs are settled in memory */
846 wmb();
847
848 return 0;
849 }
850
venus_sys_set_debug(struct venus_hfi_device * hdev,u32 debug)851 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
852 {
853 struct hfi_sys_set_property_pkt *pkt;
854 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
855
856 pkt = (struct hfi_sys_set_property_pkt *)packet;
857
858 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
859
860 return venus_iface_cmdq_write(hdev, pkt, false);
861 }
862
venus_sys_set_coverage(struct venus_hfi_device * hdev,u32 mode)863 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
864 {
865 struct hfi_sys_set_property_pkt *pkt;
866 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
867
868 pkt = (struct hfi_sys_set_property_pkt *)packet;
869
870 pkt_sys_coverage_config(pkt, mode);
871
872 return venus_iface_cmdq_write(hdev, pkt, false);
873 }
874
venus_sys_set_idle_message(struct venus_hfi_device * hdev,bool enable)875 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
876 bool enable)
877 {
878 struct hfi_sys_set_property_pkt *pkt;
879 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
880
881 if (!enable)
882 return 0;
883
884 pkt = (struct hfi_sys_set_property_pkt *)packet;
885
886 pkt_sys_idle_indicator(pkt, enable);
887
888 return venus_iface_cmdq_write(hdev, pkt, false);
889 }
890
venus_sys_set_power_control(struct venus_hfi_device * hdev,bool enable)891 static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
892 bool enable)
893 {
894 struct hfi_sys_set_property_pkt *pkt;
895 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
896
897 pkt = (struct hfi_sys_set_property_pkt *)packet;
898
899 pkt_sys_power_control(pkt, enable);
900
901 return venus_iface_cmdq_write(hdev, pkt, false);
902 }
903
venus_sys_set_ubwc_config(struct venus_hfi_device * hdev)904 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev)
905 {
906 struct hfi_sys_set_property_pkt *pkt;
907 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
908 const struct venus_resources *res = hdev->core->res;
909 int ret;
910
911 pkt = (struct hfi_sys_set_property_pkt *)packet;
912
913 pkt_sys_ubwc_config(pkt, res->ubwc_conf);
914
915 ret = venus_iface_cmdq_write(hdev, pkt, false);
916 if (ret)
917 return ret;
918
919 return 0;
920 }
921
venus_get_queue_size(struct venus_hfi_device * hdev,unsigned int index)922 static int venus_get_queue_size(struct venus_hfi_device *hdev,
923 unsigned int index)
924 {
925 struct hfi_queue_header *qhdr;
926
927 if (index >= IFACEQ_NUM)
928 return -EINVAL;
929
930 qhdr = hdev->queues[index].qhdr;
931 if (!qhdr)
932 return -EINVAL;
933
934 return abs(qhdr->read_idx - qhdr->write_idx);
935 }
936
venus_sys_set_default_properties(struct venus_hfi_device * hdev)937 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
938 {
939 struct device *dev = hdev->core->dev;
940 const struct venus_resources *res = hdev->core->res;
941 int ret;
942
943 ret = venus_sys_set_debug(hdev, venus_fw_debug);
944 if (ret)
945 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
946
947 /* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */
948 if (IS_V1(hdev->core)) {
949 ret = venus_sys_set_idle_message(hdev, false);
950 if (ret)
951 dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
952 }
953
954 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
955 if (ret)
956 dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
957 ret);
958
959 /* For specific venus core, it is mandatory to set the UBWC configuration */
960 if (res->ubwc_conf) {
961 ret = venus_sys_set_ubwc_config(hdev);
962 if (ret)
963 dev_warn(dev, "setting ubwc config failed (%d)\n", ret);
964 }
965
966 return ret;
967 }
968
venus_session_cmd(struct venus_inst * inst,u32 pkt_type,bool sync)969 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)
970 {
971 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
972 struct hfi_session_pkt pkt;
973
974 pkt_session_cmd(&pkt, pkt_type, inst);
975
976 return venus_iface_cmdq_write(hdev, &pkt, sync);
977 }
978
venus_flush_debug_queue(struct venus_hfi_device * hdev)979 static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
980 {
981 struct device *dev = hdev->core->dev;
982 void *packet = hdev->dbg_buf;
983
984 while (!venus_iface_dbgq_read(hdev, packet)) {
985 struct hfi_msg_sys_coverage_pkt *pkt = packet;
986
987 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
988 struct hfi_msg_sys_debug_pkt *pkt = packet;
989
990 dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
991 }
992 }
993 }
994
venus_prepare_power_collapse(struct venus_hfi_device * hdev,bool wait)995 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
996 bool wait)
997 {
998 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
999 struct hfi_sys_pc_prep_pkt pkt;
1000 int ret;
1001
1002 init_completion(&hdev->pwr_collapse_prep);
1003
1004 pkt_sys_pc_prep(&pkt);
1005
1006 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1007 if (ret)
1008 return ret;
1009
1010 if (!wait)
1011 return 0;
1012
1013 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
1014 if (!ret) {
1015 venus_flush_debug_queue(hdev);
1016 return -ETIMEDOUT;
1017 }
1018
1019 return 0;
1020 }
1021
venus_are_queues_empty(struct venus_hfi_device * hdev)1022 static int venus_are_queues_empty(struct venus_hfi_device *hdev)
1023 {
1024 int ret1, ret2;
1025
1026 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
1027 if (ret1 < 0)
1028 return ret1;
1029
1030 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
1031 if (ret2 < 0)
1032 return ret2;
1033
1034 if (!ret1 && !ret2)
1035 return 1;
1036
1037 return 0;
1038 }
1039
venus_sfr_print(struct venus_hfi_device * hdev)1040 static void venus_sfr_print(struct venus_hfi_device *hdev)
1041 {
1042 struct device *dev = hdev->core->dev;
1043 struct hfi_sfr *sfr = hdev->sfr.kva;
1044 u32 size;
1045 void *p;
1046
1047 if (!sfr)
1048 return;
1049
1050 size = sfr->buf_size;
1051 if (!size)
1052 return;
1053
1054 if (size > ALIGNED_SFR_SIZE)
1055 size = ALIGNED_SFR_SIZE;
1056
1057 p = memchr(sfr->data, '\0', size);
1058 /*
1059 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
1060 * that Venus is in the process of crashing.
1061 */
1062 if (!p)
1063 sfr->data[size - 1] = '\0';
1064
1065 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
1066 }
1067
venus_process_msg_sys_error(struct venus_hfi_device * hdev,void * packet)1068 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
1069 void *packet)
1070 {
1071 struct hfi_msg_event_notify_pkt *event_pkt = packet;
1072
1073 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
1074 return;
1075
1076 venus_set_state(hdev, VENUS_STATE_DEINIT);
1077
1078 venus_sfr_print(hdev);
1079 }
1080
venus_isr_thread(struct venus_core * core)1081 static irqreturn_t venus_isr_thread(struct venus_core *core)
1082 {
1083 struct venus_hfi_device *hdev = to_hfi_priv(core);
1084 const struct venus_resources *res;
1085 void *pkt;
1086 u32 msg_ret;
1087
1088 if (!hdev)
1089 return IRQ_NONE;
1090
1091 res = hdev->core->res;
1092 pkt = hdev->pkt_buf;
1093
1094
1095 while (!venus_iface_msgq_read(hdev, pkt)) {
1096 msg_ret = hfi_process_msg_packet(core, pkt);
1097 switch (msg_ret) {
1098 case HFI_MSG_EVENT_NOTIFY:
1099 venus_process_msg_sys_error(hdev, pkt);
1100 break;
1101 case HFI_MSG_SYS_INIT:
1102 venus_hfi_core_set_resource(core, res->vmem_id,
1103 res->vmem_size,
1104 res->vmem_addr,
1105 hdev);
1106 break;
1107 case HFI_MSG_SYS_RELEASE_RESOURCE:
1108 complete(&hdev->release_resource);
1109 break;
1110 case HFI_MSG_SYS_PC_PREP:
1111 complete(&hdev->pwr_collapse_prep);
1112 break;
1113 default:
1114 break;
1115 }
1116 }
1117
1118 venus_flush_debug_queue(hdev);
1119
1120 return IRQ_HANDLED;
1121 }
1122
venus_isr(struct venus_core * core)1123 static irqreturn_t venus_isr(struct venus_core *core)
1124 {
1125 struct venus_hfi_device *hdev = to_hfi_priv(core);
1126 u32 status;
1127 void __iomem *cpu_cs_base;
1128 void __iomem *wrapper_base;
1129
1130 if (!hdev)
1131 return IRQ_NONE;
1132
1133 cpu_cs_base = hdev->core->cpu_cs_base;
1134 wrapper_base = hdev->core->wrapper_base;
1135
1136 status = readl(wrapper_base + WRAPPER_INTR_STATUS);
1137 if (IS_IRIS2(core) || IS_IRIS2_1(core)) {
1138 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1139 status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||
1140 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1141 hdev->irq_status = status;
1142 } else {
1143 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1144 status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
1145 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1146 hdev->irq_status = status;
1147 }
1148 writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);
1149 if (!(IS_IRIS2(core) || IS_IRIS2_1(core)))
1150 writel(status, wrapper_base + WRAPPER_INTR_CLEAR);
1151
1152 return IRQ_WAKE_THREAD;
1153 }
1154
venus_core_init(struct venus_core * core)1155 static int venus_core_init(struct venus_core *core)
1156 {
1157 struct venus_hfi_device *hdev = to_hfi_priv(core);
1158 struct device *dev = core->dev;
1159 struct hfi_sys_get_property_pkt version_pkt;
1160 struct hfi_sys_init_pkt pkt;
1161 int ret;
1162
1163 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
1164
1165 venus_set_state(hdev, VENUS_STATE_INIT);
1166
1167 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1168 if (ret)
1169 return ret;
1170
1171 pkt_sys_image_version(&version_pkt);
1172
1173 ret = venus_iface_cmdq_write(hdev, &version_pkt, false);
1174 if (ret)
1175 dev_warn(dev, "failed to send image version pkt to fw\n");
1176
1177 ret = venus_sys_set_default_properties(hdev);
1178 if (ret)
1179 return ret;
1180
1181 return 0;
1182 }
1183
venus_core_deinit(struct venus_core * core)1184 static int venus_core_deinit(struct venus_core *core)
1185 {
1186 struct venus_hfi_device *hdev = to_hfi_priv(core);
1187
1188 venus_set_state(hdev, VENUS_STATE_DEINIT);
1189 hdev->suspended = true;
1190 hdev->power_enabled = false;
1191
1192 return 0;
1193 }
1194
venus_core_ping(struct venus_core * core,u32 cookie)1195 static int venus_core_ping(struct venus_core *core, u32 cookie)
1196 {
1197 struct venus_hfi_device *hdev = to_hfi_priv(core);
1198 struct hfi_sys_ping_pkt pkt;
1199
1200 pkt_sys_ping(&pkt, cookie);
1201
1202 return venus_iface_cmdq_write(hdev, &pkt, false);
1203 }
1204
venus_core_trigger_ssr(struct venus_core * core,u32 trigger_type)1205 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
1206 {
1207 struct venus_hfi_device *hdev = to_hfi_priv(core);
1208 struct hfi_sys_test_ssr_pkt pkt;
1209 int ret;
1210
1211 ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
1212 if (ret)
1213 return ret;
1214
1215 return venus_iface_cmdq_write(hdev, &pkt, false);
1216 }
1217
venus_session_init(struct venus_inst * inst,u32 session_type,u32 codec)1218 static int venus_session_init(struct venus_inst *inst, u32 session_type,
1219 u32 codec)
1220 {
1221 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1222 struct hfi_session_init_pkt pkt;
1223 int ret;
1224
1225 ret = venus_sys_set_debug(hdev, venus_fw_debug);
1226 if (ret)
1227 goto err;
1228
1229 ret = pkt_session_init(&pkt, inst, session_type, codec);
1230 if (ret)
1231 goto err;
1232
1233 ret = venus_iface_cmdq_write(hdev, &pkt, true);
1234 if (ret)
1235 goto err;
1236
1237 return 0;
1238
1239 err:
1240 venus_flush_debug_queue(hdev);
1241 return ret;
1242 }
1243
venus_session_end(struct venus_inst * inst)1244 static int venus_session_end(struct venus_inst *inst)
1245 {
1246 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1247 struct device *dev = hdev->core->dev;
1248
1249 if (venus_fw_coverage) {
1250 if (venus_sys_set_coverage(hdev, venus_fw_coverage))
1251 dev_warn(dev, "fw coverage msg ON failed\n");
1252 }
1253
1254 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);
1255 }
1256
venus_session_abort(struct venus_inst * inst)1257 static int venus_session_abort(struct venus_inst *inst)
1258 {
1259 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1260
1261 venus_flush_debug_queue(hdev);
1262
1263 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);
1264 }
1265
venus_session_flush(struct venus_inst * inst,u32 flush_mode)1266 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
1267 {
1268 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1269 struct hfi_session_flush_pkt pkt;
1270 int ret;
1271
1272 ret = pkt_session_flush(&pkt, inst, flush_mode);
1273 if (ret)
1274 return ret;
1275
1276 return venus_iface_cmdq_write(hdev, &pkt, true);
1277 }
1278
venus_session_start(struct venus_inst * inst)1279 static int venus_session_start(struct venus_inst *inst)
1280 {
1281 return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);
1282 }
1283
venus_session_stop(struct venus_inst * inst)1284 static int venus_session_stop(struct venus_inst *inst)
1285 {
1286 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);
1287 }
1288
venus_session_continue(struct venus_inst * inst)1289 static int venus_session_continue(struct venus_inst *inst)
1290 {
1291 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);
1292 }
1293
venus_session_etb(struct venus_inst * inst,struct hfi_frame_data * in_frame)1294 static int venus_session_etb(struct venus_inst *inst,
1295 struct hfi_frame_data *in_frame)
1296 {
1297 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1298 u32 session_type = inst->session_type;
1299 int ret;
1300
1301 if (session_type == VIDC_SESSION_TYPE_DEC) {
1302 struct hfi_session_empty_buffer_compressed_pkt pkt;
1303
1304 ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
1305 if (ret)
1306 return ret;
1307
1308 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1309 } else if (session_type == VIDC_SESSION_TYPE_ENC) {
1310 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
1311
1312 ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
1313 if (ret)
1314 return ret;
1315
1316 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1317 } else {
1318 ret = -EINVAL;
1319 }
1320
1321 return ret;
1322 }
1323
venus_session_ftb(struct venus_inst * inst,struct hfi_frame_data * out_frame)1324 static int venus_session_ftb(struct venus_inst *inst,
1325 struct hfi_frame_data *out_frame)
1326 {
1327 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1328 struct hfi_session_fill_buffer_pkt pkt;
1329 int ret;
1330
1331 ret = pkt_session_ftb(&pkt, inst, out_frame);
1332 if (ret)
1333 return ret;
1334
1335 return venus_iface_cmdq_write(hdev, &pkt, false);
1336 }
1337
venus_session_set_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1338 static int venus_session_set_buffers(struct venus_inst *inst,
1339 struct hfi_buffer_desc *bd)
1340 {
1341 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1342 struct hfi_session_set_buffers_pkt *pkt;
1343 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1344 int ret;
1345
1346 if (bd->buffer_type == HFI_BUFFER_INPUT)
1347 return 0;
1348
1349 pkt = (struct hfi_session_set_buffers_pkt *)packet;
1350
1351 ret = pkt_session_set_buffers(pkt, inst, bd);
1352 if (ret)
1353 return ret;
1354
1355 return venus_iface_cmdq_write(hdev, pkt, false);
1356 }
1357
venus_session_unset_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1358 static int venus_session_unset_buffers(struct venus_inst *inst,
1359 struct hfi_buffer_desc *bd)
1360 {
1361 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1362 struct hfi_session_release_buffer_pkt *pkt;
1363 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1364 int ret;
1365
1366 if (bd->buffer_type == HFI_BUFFER_INPUT)
1367 return 0;
1368
1369 pkt = (struct hfi_session_release_buffer_pkt *)packet;
1370
1371 ret = pkt_session_unset_buffers(pkt, inst, bd);
1372 if (ret)
1373 return ret;
1374
1375 return venus_iface_cmdq_write(hdev, pkt, true);
1376 }
1377
venus_session_load_res(struct venus_inst * inst)1378 static int venus_session_load_res(struct venus_inst *inst)
1379 {
1380 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);
1381 }
1382
venus_session_release_res(struct venus_inst * inst)1383 static int venus_session_release_res(struct venus_inst *inst)
1384 {
1385 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);
1386 }
1387
venus_session_parse_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1388 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1389 u32 seq_hdr_len)
1390 {
1391 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1392 struct hfi_session_parse_sequence_header_pkt *pkt;
1393 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1394 int ret;
1395
1396 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
1397
1398 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
1399 if (ret)
1400 return ret;
1401
1402 ret = venus_iface_cmdq_write(hdev, pkt, false);
1403 if (ret)
1404 return ret;
1405
1406 return 0;
1407 }
1408
venus_session_get_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1409 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1410 u32 seq_hdr_len)
1411 {
1412 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1413 struct hfi_session_get_sequence_header_pkt *pkt;
1414 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1415 int ret;
1416
1417 pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
1418
1419 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
1420 if (ret)
1421 return ret;
1422
1423 return venus_iface_cmdq_write(hdev, pkt, false);
1424 }
1425
venus_session_set_property(struct venus_inst * inst,u32 ptype,void * pdata)1426 static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
1427 void *pdata)
1428 {
1429 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1430 struct hfi_session_set_property_pkt *pkt;
1431 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1432 int ret;
1433
1434 pkt = (struct hfi_session_set_property_pkt *)packet;
1435
1436 ret = pkt_session_set_property(pkt, inst, ptype, pdata);
1437 if (ret == -ENOTSUPP)
1438 return 0;
1439 if (ret)
1440 return ret;
1441
1442 return venus_iface_cmdq_write(hdev, pkt, false);
1443 }
1444
venus_session_get_property(struct venus_inst * inst,u32 ptype)1445 static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
1446 {
1447 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1448 struct hfi_session_get_property_pkt pkt;
1449 int ret;
1450
1451 ret = pkt_session_get_property(&pkt, inst, ptype);
1452 if (ret)
1453 return ret;
1454
1455 return venus_iface_cmdq_write(hdev, &pkt, true);
1456 }
1457
venus_resume(struct venus_core * core)1458 static int venus_resume(struct venus_core *core)
1459 {
1460 struct venus_hfi_device *hdev = to_hfi_priv(core);
1461 int ret = 0;
1462
1463 mutex_lock(&hdev->lock);
1464
1465 if (!hdev->suspended)
1466 goto unlock;
1467
1468 ret = venus_power_on(hdev);
1469
1470 unlock:
1471 if (!ret)
1472 hdev->suspended = false;
1473
1474 mutex_unlock(&hdev->lock);
1475
1476 return ret;
1477 }
1478
venus_suspend_1xx(struct venus_core * core)1479 static int venus_suspend_1xx(struct venus_core *core)
1480 {
1481 struct venus_hfi_device *hdev = to_hfi_priv(core);
1482 struct device *dev = core->dev;
1483 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1484 u32 ctrl_status;
1485 int ret;
1486
1487 if (!hdev->power_enabled || hdev->suspended)
1488 return 0;
1489
1490 mutex_lock(&hdev->lock);
1491 ret = venus_is_valid_state(hdev);
1492 mutex_unlock(&hdev->lock);
1493
1494 if (!ret) {
1495 dev_err(dev, "bad state, cannot suspend\n");
1496 return -EINVAL;
1497 }
1498
1499 ret = venus_prepare_power_collapse(hdev, true);
1500 if (ret) {
1501 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1502 return ret;
1503 }
1504
1505 mutex_lock(&hdev->lock);
1506
1507 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
1508 mutex_unlock(&hdev->lock);
1509 return -EINVAL;
1510 }
1511
1512 ret = venus_are_queues_empty(hdev);
1513 if (ret < 0 || !ret) {
1514 mutex_unlock(&hdev->lock);
1515 return -EINVAL;
1516 }
1517
1518 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1519 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
1520 mutex_unlock(&hdev->lock);
1521 return -EINVAL;
1522 }
1523
1524 ret = venus_power_off(hdev);
1525 if (ret) {
1526 mutex_unlock(&hdev->lock);
1527 return ret;
1528 }
1529
1530 hdev->suspended = true;
1531
1532 mutex_unlock(&hdev->lock);
1533
1534 return 0;
1535 }
1536
venus_cpu_and_video_core_idle(struct venus_hfi_device * hdev)1537 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
1538 {
1539 void __iomem *wrapper_base = hdev->core->wrapper_base;
1540 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1541 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1542 u32 ctrl_status, cpu_status;
1543
1544 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1545 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1546 else
1547 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1548 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1549
1550 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1551 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1552 return true;
1553
1554 return false;
1555 }
1556
venus_cpu_idle_and_pc_ready(struct venus_hfi_device * hdev)1557 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
1558 {
1559 void __iomem *wrapper_base = hdev->core->wrapper_base;
1560 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1561 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1562 u32 ctrl_status, cpu_status;
1563
1564 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1565 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1566 else
1567 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1568 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1569
1570 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1571 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1572 return true;
1573
1574 return false;
1575 }
1576
venus_suspend_3xx(struct venus_core * core)1577 static int venus_suspend_3xx(struct venus_core *core)
1578 {
1579 struct venus_hfi_device *hdev = to_hfi_priv(core);
1580 struct device *dev = core->dev;
1581 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1582 u32 ctrl_status;
1583 bool val;
1584 int ret;
1585
1586 if (!hdev->power_enabled || hdev->suspended)
1587 return 0;
1588
1589 mutex_lock(&hdev->lock);
1590 ret = venus_is_valid_state(hdev);
1591 mutex_unlock(&hdev->lock);
1592
1593 if (!ret) {
1594 dev_err(dev, "bad state, cannot suspend\n");
1595 return -EINVAL;
1596 }
1597
1598 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1599 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1600 goto power_off;
1601
1602 /*
1603 * Power collapse sequence for Venus 3xx and 4xx versions:
1604 * 1. Check for ARM9 and video core to be idle by checking WFI bit
1605 * (bit 0) in CPU status register and by checking Idle (bit 30) in
1606 * Control status register for video core.
1607 * 2. Send a command to prepare for power collapse.
1608 * 3. Check for WFI and PC_READY bits.
1609 */
1610 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
1611 1500, 100 * 1500);
1612 if (ret) {
1613 dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret);
1614 return ret;
1615 }
1616
1617 ret = venus_prepare_power_collapse(hdev, false);
1618 if (ret) {
1619 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1620 return ret;
1621 }
1622
1623 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
1624 1500, 100 * 1500);
1625 if (ret)
1626 return ret;
1627
1628 power_off:
1629 mutex_lock(&hdev->lock);
1630
1631 ret = venus_power_off(hdev);
1632 if (ret) {
1633 dev_err(dev, "venus_power_off (%d)\n", ret);
1634 mutex_unlock(&hdev->lock);
1635 return ret;
1636 }
1637
1638 hdev->suspended = true;
1639
1640 mutex_unlock(&hdev->lock);
1641
1642 return 0;
1643 }
1644
venus_suspend(struct venus_core * core)1645 static int venus_suspend(struct venus_core *core)
1646 {
1647 if (IS_V3(core) || IS_V4(core) || IS_V6(core))
1648 return venus_suspend_3xx(core);
1649
1650 return venus_suspend_1xx(core);
1651 }
1652
1653 static const struct hfi_ops venus_hfi_ops = {
1654 .core_init = venus_core_init,
1655 .core_deinit = venus_core_deinit,
1656 .core_ping = venus_core_ping,
1657 .core_trigger_ssr = venus_core_trigger_ssr,
1658
1659 .session_init = venus_session_init,
1660 .session_end = venus_session_end,
1661 .session_abort = venus_session_abort,
1662 .session_flush = venus_session_flush,
1663 .session_start = venus_session_start,
1664 .session_stop = venus_session_stop,
1665 .session_continue = venus_session_continue,
1666 .session_etb = venus_session_etb,
1667 .session_ftb = venus_session_ftb,
1668 .session_set_buffers = venus_session_set_buffers,
1669 .session_unset_buffers = venus_session_unset_buffers,
1670 .session_load_res = venus_session_load_res,
1671 .session_release_res = venus_session_release_res,
1672 .session_parse_seq_hdr = venus_session_parse_seq_hdr,
1673 .session_get_seq_hdr = venus_session_get_seq_hdr,
1674 .session_set_property = venus_session_set_property,
1675 .session_get_property = venus_session_get_property,
1676
1677 .resume = venus_resume,
1678 .suspend = venus_suspend,
1679
1680 .isr = venus_isr,
1681 .isr_thread = venus_isr_thread,
1682 };
1683
venus_hfi_destroy(struct venus_core * core)1684 void venus_hfi_destroy(struct venus_core *core)
1685 {
1686 struct venus_hfi_device *hdev = to_hfi_priv(core);
1687
1688 core->priv = NULL;
1689 venus_interface_queues_release(hdev);
1690 mutex_destroy(&hdev->lock);
1691 kfree(hdev);
1692 core->ops = NULL;
1693 }
1694
venus_hfi_create(struct venus_core * core)1695 int venus_hfi_create(struct venus_core *core)
1696 {
1697 struct venus_hfi_device *hdev;
1698 int ret;
1699
1700 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
1701 if (!hdev)
1702 return -ENOMEM;
1703
1704 mutex_init(&hdev->lock);
1705
1706 hdev->core = core;
1707 hdev->suspended = true;
1708 core->priv = hdev;
1709 core->ops = &venus_hfi_ops;
1710
1711 ret = venus_interface_queues_init(hdev);
1712 if (ret)
1713 goto err_kfree;
1714
1715 return 0;
1716
1717 err_kfree:
1718 kfree(hdev);
1719 core->priv = NULL;
1720 core->ops = NULL;
1721 return ret;
1722 }
1723
venus_hfi_queues_reinit(struct venus_core * core)1724 void venus_hfi_queues_reinit(struct venus_core *core)
1725 {
1726 struct venus_hfi_device *hdev = to_hfi_priv(core);
1727 struct hfi_queue_table_header *tbl_hdr;
1728 struct iface_queue *queue;
1729 struct hfi_sfr *sfr;
1730 unsigned int i;
1731
1732 mutex_lock(&hdev->lock);
1733
1734 for (i = 0; i < IFACEQ_NUM; i++) {
1735 queue = &hdev->queues[i];
1736 queue->qhdr =
1737 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
1738
1739 venus_set_qhdr_defaults(queue->qhdr);
1740
1741 queue->qhdr->start_addr = queue->qmem.da;
1742
1743 if (i == IFACEQ_CMD_IDX)
1744 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
1745 else if (i == IFACEQ_MSG_IDX)
1746 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
1747 else if (i == IFACEQ_DBG_IDX)
1748 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
1749 }
1750
1751 tbl_hdr = hdev->ifaceq_table.kva;
1752 tbl_hdr->version = 0;
1753 tbl_hdr->size = IFACEQ_TABLE_SIZE;
1754 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
1755 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
1756 tbl_hdr->num_q = IFACEQ_NUM;
1757 tbl_hdr->num_active_q = IFACEQ_NUM;
1758
1759 /*
1760 * Set receive request to zero on debug queue as there is no
1761 * need of interrupt from video hardware for debug messages
1762 */
1763 queue = &hdev->queues[IFACEQ_DBG_IDX];
1764 queue->qhdr->rx_req = 0;
1765
1766 sfr = hdev->sfr.kva;
1767 sfr->buf_size = ALIGNED_SFR_SIZE;
1768
1769 /* ensure table and queue header structs are settled in memory */
1770 wmb();
1771
1772 mutex_unlock(&hdev->lock);
1773 }
1774