1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2016 MediaTek Inc.
4 * Author: Andrew-CT Chen <andrew-ct.chen@mediatek.com>
5 */
6 #include <linux/clk.h>
7 #include <linux/debugfs.h>
8 #include <linux/firmware.h>
9 #include <linux/interrupt.h>
10 #include <linux/iommu.h>
11 #include <linux/module.h>
12 #include <linux/of.h>
13 #include <linux/of_platform.h>
14 #include <linux/of_reserved_mem.h>
15 #include <linux/platform_device.h>
16 #include <linux/sched.h>
17 #include <linux/sizes.h>
18 #include <linux/dma-mapping.h>
19 
20 #include "mtk_vpu.h"
21 
22 /*
23  * VPU (video processor unit) is a tiny processor controlling video hardware
24  * related to video codec, scaling and color format converting.
25  * VPU interfaces with other blocks by share memory and interrupt.
26  */
27 
28 #define INIT_TIMEOUT_MS		2000U
29 #define IPI_TIMEOUT_MS		2000U
30 #define VPU_IDLE_TIMEOUT_MS	1000U
31 #define VPU_FW_VER_LEN		16
32 
33 /* maximum program/data TCM (Tightly-Coupled Memory) size */
34 #define VPU_PTCM_SIZE		(96 * SZ_1K)
35 #define VPU_DTCM_SIZE		(32 * SZ_1K)
36 /* the offset to get data tcm address */
37 #define VPU_DTCM_OFFSET		0x18000UL
38 /* daynamic allocated maximum extended memory size */
39 #define VPU_EXT_P_SIZE		SZ_1M
40 #define VPU_EXT_D_SIZE		SZ_4M
41 /* maximum binary firmware size */
42 #define VPU_P_FW_SIZE		(VPU_PTCM_SIZE + VPU_EXT_P_SIZE)
43 #define VPU_D_FW_SIZE		(VPU_DTCM_SIZE + VPU_EXT_D_SIZE)
44 /* the size of share buffer between Host and  VPU */
45 #define SHARE_BUF_SIZE		48
46 
47 /* binary firmware name */
48 #define VPU_P_FW		"vpu_p.bin"
49 #define VPU_D_FW		"vpu_d.bin"
50 #define VPU_P_FW_NEW		"mediatek/mt8173/vpu_p.bin"
51 #define VPU_D_FW_NEW		"mediatek/mt8173/vpu_d.bin"
52 
53 #define VPU_RESET		0x0
54 #define VPU_TCM_CFG		0x0008
55 #define VPU_PMEM_EXT0_ADDR	0x000C
56 #define VPU_PMEM_EXT1_ADDR	0x0010
57 #define VPU_TO_HOST		0x001C
58 #define VPU_DMEM_EXT0_ADDR	0x0014
59 #define VPU_DMEM_EXT1_ADDR	0x0018
60 #define HOST_TO_VPU		0x0024
61 #define VPU_IDLE_REG		0x002C
62 #define VPU_INT_STATUS		0x0034
63 #define VPU_PC_REG		0x0060
64 #define VPU_SP_REG		0x0064
65 #define VPU_RA_REG		0x0068
66 #define VPU_WDT_REG		0x0084
67 
68 /* vpu inter-processor communication interrupt */
69 #define VPU_IPC_INT		BIT(8)
70 /* vpu idle state */
71 #define VPU_IDLE_STATE		BIT(23)
72 
73 /**
74  * enum vpu_fw_type - VPU firmware type
75  *
76  * @P_FW: program firmware
77  * @D_FW: data firmware
78  *
79  */
80 enum vpu_fw_type {
81 	P_FW,
82 	D_FW,
83 };
84 
85 /**
86  * struct vpu_mem - VPU extended program/data memory information
87  *
88  * @va:		the kernel virtual memory address of VPU extended memory
89  * @pa:		the physical memory address of VPU extended memory
90  *
91  */
92 struct vpu_mem {
93 	void *va;
94 	dma_addr_t pa;
95 };
96 
97 /**
98  * struct vpu_regs - VPU TCM and configuration registers
99  *
100  * @tcm:	the register for VPU Tightly-Coupled Memory
101  * @cfg:	the register for VPU configuration
102  * @irq:	the irq number for VPU interrupt
103  */
104 struct vpu_regs {
105 	void __iomem *tcm;
106 	void __iomem *cfg;
107 	int irq;
108 };
109 
110 /**
111  * struct vpu_wdt_handler - VPU watchdog reset handler
112  *
113  * @reset_func:	reset handler
114  * @priv:	private data
115  */
116 struct vpu_wdt_handler {
117 	void (*reset_func)(void *);
118 	void *priv;
119 };
120 
121 /**
122  * struct vpu_wdt - VPU watchdog workqueue
123  *
124  * @handler:	VPU watchdog reset handler
125  * @ws:		workstruct for VPU watchdog
126  * @wq:		workqueue for VPU watchdog
127  */
128 struct vpu_wdt {
129 	struct vpu_wdt_handler handler[VPU_RST_MAX];
130 	struct work_struct ws;
131 	struct workqueue_struct *wq;
132 };
133 
134 /**
135  * struct vpu_run - VPU initialization status
136  *
137  * @signaled:		the signal of vpu initialization completed
138  * @fw_ver:		VPU firmware version
139  * @dec_capability:	decoder capability which is not used for now and
140  *			the value is reserved for future use
141  * @enc_capability:	encoder capability which is not used for now and
142  *			the value is reserved for future use
143  * @wq:			wait queue for VPU initialization status
144  */
145 struct vpu_run {
146 	u32 signaled;
147 	char fw_ver[VPU_FW_VER_LEN];
148 	unsigned int	dec_capability;
149 	unsigned int	enc_capability;
150 	wait_queue_head_t wq;
151 };
152 
153 /**
154  * struct vpu_ipi_desc - VPU IPI descriptor
155  *
156  * @handler:	IPI handler
157  * @name:	the name of IPI handler
158  * @priv:	the private data of IPI handler
159  */
160 struct vpu_ipi_desc {
161 	ipi_handler_t handler;
162 	const char *name;
163 	void *priv;
164 };
165 
166 /**
167  * struct share_obj - DTCM (Data Tightly-Coupled Memory) buffer shared with
168  *		      AP and VPU
169  *
170  * @id:		IPI id
171  * @len:	share buffer length
172  * @share_buf:	share buffer data
173  */
174 struct share_obj {
175 	s32 id;
176 	u32 len;
177 	unsigned char share_buf[SHARE_BUF_SIZE];
178 };
179 
180 /**
181  * struct mtk_vpu - vpu driver data
182  * @extmem:		VPU extended memory information
183  * @reg:		VPU TCM and configuration registers
184  * @run:		VPU initialization status
185  * @wdt:		VPU watchdog workqueue
186  * @ipi_desc:		VPU IPI descriptor
187  * @recv_buf:		VPU DTCM share buffer for receiving. The
188  *			receive buffer is only accessed in interrupt context.
189  * @send_buf:		VPU DTCM share buffer for sending
190  * @dev:		VPU struct device
191  * @clk:		VPU clock on/off
192  * @fw_loaded:		indicate VPU firmware loaded
193  * @enable_4GB:		VPU 4GB mode on/off
194  * @vpu_mutex:		protect mtk_vpu (except recv_buf) and ensure only
195  *			one client to use VPU service at a time. For example,
196  *			suppose a client is using VPU to decode VP8.
197  *			If the other client wants to encode VP8,
198  *			it has to wait until VP8 decode completes.
199  * @wdt_refcnt:		WDT reference count to make sure the watchdog can be
200  *			disabled if no other client is using VPU service
201  * @ack_wq:		The wait queue for each codec and mdp. When sleeping
202  *			processes wake up, they will check the condition
203  *			"ipi_id_ack" to run the corresponding action or
204  *			go back to sleep.
205  * @ipi_id_ack:		The ACKs for registered IPI function sending
206  *			interrupt to VPU
207  *
208  */
209 struct mtk_vpu {
210 	struct vpu_mem extmem[2];
211 	struct vpu_regs reg;
212 	struct vpu_run run;
213 	struct vpu_wdt wdt;
214 	struct vpu_ipi_desc ipi_desc[IPI_MAX];
215 	struct share_obj __iomem *recv_buf;
216 	struct share_obj __iomem *send_buf;
217 	struct device *dev;
218 	struct clk *clk;
219 	bool fw_loaded;
220 	bool enable_4GB;
221 	struct mutex vpu_mutex; /* for protecting vpu data data structure */
222 	u32 wdt_refcnt;
223 	wait_queue_head_t ack_wq;
224 	bool ipi_id_ack[IPI_MAX];
225 };
226 
227 static inline void vpu_cfg_writel(struct mtk_vpu *vpu, u32 val, u32 offset)
228 {
229 	writel(val, vpu->reg.cfg + offset);
230 }
231 
232 static inline u32 vpu_cfg_readl(struct mtk_vpu *vpu, u32 offset)
233 {
234 	return readl(vpu->reg.cfg + offset);
235 }
236 
237 static inline bool vpu_running(struct mtk_vpu *vpu)
238 {
239 	return vpu_cfg_readl(vpu, VPU_RESET) & BIT(0);
240 }
241 
242 static void vpu_clock_disable(struct mtk_vpu *vpu)
243 {
244 	/* Disable VPU watchdog */
245 	mutex_lock(&vpu->vpu_mutex);
246 	if (!--vpu->wdt_refcnt)
247 		vpu_cfg_writel(vpu,
248 			       vpu_cfg_readl(vpu, VPU_WDT_REG) & ~(1L << 31),
249 			       VPU_WDT_REG);
250 	mutex_unlock(&vpu->vpu_mutex);
251 
252 	clk_disable(vpu->clk);
253 }
254 
255 static int vpu_clock_enable(struct mtk_vpu *vpu)
256 {
257 	int ret;
258 
259 	ret = clk_enable(vpu->clk);
260 	if (ret)
261 		return ret;
262 	/* Enable VPU watchdog */
263 	mutex_lock(&vpu->vpu_mutex);
264 	if (!vpu->wdt_refcnt++)
265 		vpu_cfg_writel(vpu,
266 			       vpu_cfg_readl(vpu, VPU_WDT_REG) | (1L << 31),
267 			       VPU_WDT_REG);
268 	mutex_unlock(&vpu->vpu_mutex);
269 
270 	return ret;
271 }
272 
273 static void vpu_dump_status(struct mtk_vpu *vpu)
274 {
275 	dev_info(vpu->dev,
276 		 "vpu: run %x, pc = 0x%x, ra = 0x%x, sp = 0x%x, idle = 0x%x\n"
277 		 "vpu: int %x, hv = 0x%x, vh = 0x%x, wdt = 0x%x\n",
278 		 vpu_running(vpu), vpu_cfg_readl(vpu, VPU_PC_REG),
279 		 vpu_cfg_readl(vpu, VPU_RA_REG), vpu_cfg_readl(vpu, VPU_SP_REG),
280 		 vpu_cfg_readl(vpu, VPU_IDLE_REG),
281 		 vpu_cfg_readl(vpu, VPU_INT_STATUS),
282 		 vpu_cfg_readl(vpu, HOST_TO_VPU),
283 		 vpu_cfg_readl(vpu, VPU_TO_HOST),
284 		 vpu_cfg_readl(vpu, VPU_WDT_REG));
285 }
286 
287 int vpu_ipi_register(struct platform_device *pdev,
288 		     enum ipi_id id, ipi_handler_t handler,
289 		     const char *name, void *priv)
290 {
291 	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
292 	struct vpu_ipi_desc *ipi_desc;
293 
294 	if (!vpu) {
295 		dev_err(&pdev->dev, "vpu device in not ready\n");
296 		return -EPROBE_DEFER;
297 	}
298 
299 	if (id < IPI_MAX && handler) {
300 		ipi_desc = vpu->ipi_desc;
301 		ipi_desc[id].name = name;
302 		ipi_desc[id].handler = handler;
303 		ipi_desc[id].priv = priv;
304 		return 0;
305 	}
306 
307 	dev_err(&pdev->dev, "register vpu ipi id %d with invalid arguments\n",
308 		id);
309 	return -EINVAL;
310 }
311 EXPORT_SYMBOL_GPL(vpu_ipi_register);
312 
313 int vpu_ipi_send(struct platform_device *pdev,
314 		 enum ipi_id id, void *buf,
315 		 unsigned int len)
316 {
317 	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
318 	struct share_obj __iomem *send_obj = vpu->send_buf;
319 	unsigned long timeout;
320 	int ret = 0;
321 
322 	if (id <= IPI_VPU_INIT || id >= IPI_MAX ||
323 	    len > sizeof(send_obj->share_buf) || !buf) {
324 		dev_err(vpu->dev, "failed to send ipi message\n");
325 		return -EINVAL;
326 	}
327 
328 	ret = vpu_clock_enable(vpu);
329 	if (ret) {
330 		dev_err(vpu->dev, "failed to enable vpu clock\n");
331 		return ret;
332 	}
333 	if (!vpu_running(vpu)) {
334 		dev_err(vpu->dev, "vpu_ipi_send: VPU is not running\n");
335 		ret = -EINVAL;
336 		goto clock_disable;
337 	}
338 
339 	mutex_lock(&vpu->vpu_mutex);
340 
341 	 /* Wait until VPU receives the last command */
342 	timeout = jiffies + msecs_to_jiffies(IPI_TIMEOUT_MS);
343 	do {
344 		if (time_after(jiffies, timeout)) {
345 			dev_err(vpu->dev, "vpu_ipi_send: IPI timeout!\n");
346 			ret = -EIO;
347 			vpu_dump_status(vpu);
348 			goto mut_unlock;
349 		}
350 	} while (vpu_cfg_readl(vpu, HOST_TO_VPU));
351 
352 	memcpy_toio(send_obj->share_buf, buf, len);
353 	writel(len, &send_obj->len);
354 	writel(id, &send_obj->id);
355 
356 	vpu->ipi_id_ack[id] = false;
357 	/* send the command to VPU */
358 	vpu_cfg_writel(vpu, 0x1, HOST_TO_VPU);
359 
360 	mutex_unlock(&vpu->vpu_mutex);
361 
362 	/* wait for VPU's ACK */
363 	timeout = msecs_to_jiffies(IPI_TIMEOUT_MS);
364 	ret = wait_event_timeout(vpu->ack_wq, vpu->ipi_id_ack[id], timeout);
365 	vpu->ipi_id_ack[id] = false;
366 	if (ret == 0) {
367 		dev_err(vpu->dev, "vpu ipi %d ack time out !\n", id);
368 		ret = -EIO;
369 		vpu_dump_status(vpu);
370 		goto clock_disable;
371 	}
372 	vpu_clock_disable(vpu);
373 
374 	return 0;
375 
376 mut_unlock:
377 	mutex_unlock(&vpu->vpu_mutex);
378 clock_disable:
379 	vpu_clock_disable(vpu);
380 
381 	return ret;
382 }
383 EXPORT_SYMBOL_GPL(vpu_ipi_send);
384 
385 static void vpu_wdt_reset_func(struct work_struct *ws)
386 {
387 	struct vpu_wdt *wdt = container_of(ws, struct vpu_wdt, ws);
388 	struct mtk_vpu *vpu = container_of(wdt, struct mtk_vpu, wdt);
389 	struct vpu_wdt_handler *handler = wdt->handler;
390 	int index, ret;
391 
392 	dev_info(vpu->dev, "vpu reset\n");
393 	ret = vpu_clock_enable(vpu);
394 	if (ret) {
395 		dev_err(vpu->dev, "[VPU] wdt enables clock failed %d\n", ret);
396 		return;
397 	}
398 	mutex_lock(&vpu->vpu_mutex);
399 	vpu_cfg_writel(vpu, 0x0, VPU_RESET);
400 	vpu->fw_loaded = false;
401 	mutex_unlock(&vpu->vpu_mutex);
402 	vpu_clock_disable(vpu);
403 
404 	for (index = 0; index < VPU_RST_MAX; index++) {
405 		if (handler[index].reset_func) {
406 			handler[index].reset_func(handler[index].priv);
407 			dev_dbg(vpu->dev, "wdt handler func %d\n", index);
408 		}
409 	}
410 }
411 
412 int vpu_wdt_reg_handler(struct platform_device *pdev,
413 			void wdt_reset(void *),
414 			void *priv, enum rst_id id)
415 {
416 	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
417 	struct vpu_wdt_handler *handler;
418 
419 	if (!vpu) {
420 		dev_err(&pdev->dev, "vpu device in not ready\n");
421 		return -EPROBE_DEFER;
422 	}
423 
424 	handler = vpu->wdt.handler;
425 
426 	if (id < VPU_RST_MAX && wdt_reset) {
427 		dev_dbg(vpu->dev, "wdt register id %d\n", id);
428 		mutex_lock(&vpu->vpu_mutex);
429 		handler[id].reset_func = wdt_reset;
430 		handler[id].priv = priv;
431 		mutex_unlock(&vpu->vpu_mutex);
432 		return 0;
433 	}
434 
435 	dev_err(vpu->dev, "register vpu wdt handler failed\n");
436 	return -EINVAL;
437 }
438 EXPORT_SYMBOL_GPL(vpu_wdt_reg_handler);
439 
440 unsigned int vpu_get_vdec_hw_capa(struct platform_device *pdev)
441 {
442 	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
443 
444 	return vpu->run.dec_capability;
445 }
446 EXPORT_SYMBOL_GPL(vpu_get_vdec_hw_capa);
447 
448 unsigned int vpu_get_venc_hw_capa(struct platform_device *pdev)
449 {
450 	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
451 
452 	return vpu->run.enc_capability;
453 }
454 EXPORT_SYMBOL_GPL(vpu_get_venc_hw_capa);
455 
456 void *vpu_mapping_dm_addr(struct platform_device *pdev,
457 			  u32 dtcm_dmem_addr)
458 {
459 	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
460 
461 	if (!dtcm_dmem_addr ||
462 	    (dtcm_dmem_addr > (VPU_DTCM_SIZE + VPU_EXT_D_SIZE))) {
463 		dev_err(vpu->dev, "invalid virtual data memory address\n");
464 		return ERR_PTR(-EINVAL);
465 	}
466 
467 	if (dtcm_dmem_addr < VPU_DTCM_SIZE)
468 		return (__force void *)(dtcm_dmem_addr + vpu->reg.tcm +
469 					VPU_DTCM_OFFSET);
470 
471 	return vpu->extmem[D_FW].va + (dtcm_dmem_addr - VPU_DTCM_SIZE);
472 }
473 EXPORT_SYMBOL_GPL(vpu_mapping_dm_addr);
474 
475 struct platform_device *vpu_get_plat_device(struct platform_device *pdev)
476 {
477 	struct device *dev = &pdev->dev;
478 	struct device_node *vpu_node;
479 	struct platform_device *vpu_pdev;
480 
481 	vpu_node = of_parse_phandle(dev->of_node, "mediatek,vpu", 0);
482 	if (!vpu_node) {
483 		dev_err(dev, "can't get vpu node\n");
484 		return NULL;
485 	}
486 
487 	vpu_pdev = of_find_device_by_node(vpu_node);
488 	of_node_put(vpu_node);
489 	if (WARN_ON(!vpu_pdev)) {
490 		dev_err(dev, "vpu pdev failed\n");
491 		return NULL;
492 	}
493 
494 	return vpu_pdev;
495 }
496 EXPORT_SYMBOL_GPL(vpu_get_plat_device);
497 
498 /* load vpu program/data memory */
499 static int load_requested_vpu(struct mtk_vpu *vpu,
500 			      u8 fw_type)
501 {
502 	size_t tcm_size = fw_type ? VPU_DTCM_SIZE : VPU_PTCM_SIZE;
503 	size_t fw_size = fw_type ? VPU_D_FW_SIZE : VPU_P_FW_SIZE;
504 	char *fw_name = fw_type ? VPU_D_FW : VPU_P_FW;
505 	char *fw_new_name = fw_type ? VPU_D_FW_NEW : VPU_P_FW_NEW;
506 	const struct firmware *vpu_fw;
507 	size_t dl_size = 0;
508 	size_t extra_fw_size = 0;
509 	void *dest;
510 	int ret;
511 
512 	ret = request_firmware(&vpu_fw, fw_new_name, vpu->dev);
513 	if (ret < 0) {
514 		dev_info(vpu->dev, "Failed to load %s, %d, retry\n",
515 			 fw_new_name, ret);
516 
517 		ret = request_firmware(&vpu_fw, fw_name, vpu->dev);
518 		if (ret < 0) {
519 			dev_err(vpu->dev, "Failed to load %s, %d\n", fw_name,
520 				ret);
521 			return ret;
522 		}
523 	}
524 	dl_size = vpu_fw->size;
525 	if (dl_size > fw_size) {
526 		dev_err(vpu->dev, "fw %s size %zu is abnormal\n", fw_name,
527 			dl_size);
528 		release_firmware(vpu_fw);
529 		return  -EFBIG;
530 	}
531 	dev_dbg(vpu->dev, "Downloaded fw %s size: %zu.\n",
532 		fw_name,
533 		dl_size);
534 	/* reset VPU */
535 	vpu_cfg_writel(vpu, 0x0, VPU_RESET);
536 
537 	/* handle extended firmware size */
538 	if (dl_size > tcm_size) {
539 		dev_dbg(vpu->dev, "fw size %zu > limited fw size %zu\n",
540 			dl_size, tcm_size);
541 		extra_fw_size = dl_size - tcm_size;
542 		dev_dbg(vpu->dev, "extra_fw_size %zu\n", extra_fw_size);
543 		dl_size = tcm_size;
544 	}
545 	dest = (__force void *)vpu->reg.tcm;
546 	if (fw_type == D_FW)
547 		dest += VPU_DTCM_OFFSET;
548 	memcpy(dest, vpu_fw->data, dl_size);
549 	/* download to extended memory if need */
550 	if (extra_fw_size > 0) {
551 		dest = vpu->extmem[fw_type].va;
552 		dev_dbg(vpu->dev, "download extended memory type %x\n",
553 			fw_type);
554 		memcpy(dest, vpu_fw->data + tcm_size, extra_fw_size);
555 	}
556 
557 	release_firmware(vpu_fw);
558 
559 	return 0;
560 }
561 
562 int vpu_load_firmware(struct platform_device *pdev)
563 {
564 	struct mtk_vpu *vpu;
565 	struct device *dev;
566 	struct vpu_run *run;
567 	int ret;
568 
569 	if (!pdev) {
570 		pr_err("VPU platform device is invalid\n");
571 		return -EINVAL;
572 	}
573 
574 	dev = &pdev->dev;
575 
576 	vpu = platform_get_drvdata(pdev);
577 	run = &vpu->run;
578 
579 	mutex_lock(&vpu->vpu_mutex);
580 	if (vpu->fw_loaded) {
581 		mutex_unlock(&vpu->vpu_mutex);
582 		return 0;
583 	}
584 	mutex_unlock(&vpu->vpu_mutex);
585 
586 	ret = vpu_clock_enable(vpu);
587 	if (ret) {
588 		dev_err(dev, "enable clock failed %d\n", ret);
589 		return ret;
590 	}
591 
592 	mutex_lock(&vpu->vpu_mutex);
593 
594 	run->signaled = false;
595 	dev_dbg(vpu->dev, "firmware request\n");
596 	/* Downloading program firmware to device*/
597 	ret = load_requested_vpu(vpu, P_FW);
598 	if (ret < 0) {
599 		dev_err(dev, "Failed to request %s, %d\n", VPU_P_FW, ret);
600 		goto OUT_LOAD_FW;
601 	}
602 
603 	/* Downloading data firmware to device */
604 	ret = load_requested_vpu(vpu, D_FW);
605 	if (ret < 0) {
606 		dev_err(dev, "Failed to request %s, %d\n", VPU_D_FW, ret);
607 		goto OUT_LOAD_FW;
608 	}
609 
610 	vpu->fw_loaded = true;
611 	/* boot up vpu */
612 	vpu_cfg_writel(vpu, 0x1, VPU_RESET);
613 
614 	ret = wait_event_interruptible_timeout(run->wq,
615 					       run->signaled,
616 					       msecs_to_jiffies(INIT_TIMEOUT_MS)
617 					       );
618 	if (ret == 0) {
619 		ret = -ETIME;
620 		dev_err(dev, "wait vpu initialization timeout!\n");
621 		goto OUT_LOAD_FW;
622 	} else if (-ERESTARTSYS == ret) {
623 		dev_err(dev, "wait vpu interrupted by a signal!\n");
624 		goto OUT_LOAD_FW;
625 	}
626 
627 	ret = 0;
628 	dev_info(dev, "vpu is ready. Fw version %s\n", run->fw_ver);
629 
630 OUT_LOAD_FW:
631 	mutex_unlock(&vpu->vpu_mutex);
632 	vpu_clock_disable(vpu);
633 
634 	return ret;
635 }
636 EXPORT_SYMBOL_GPL(vpu_load_firmware);
637 
638 static void vpu_init_ipi_handler(const void *data, unsigned int len, void *priv)
639 {
640 	struct mtk_vpu *vpu = priv;
641 	const struct vpu_run *run = data;
642 
643 	vpu->run.signaled = run->signaled;
644 	strscpy(vpu->run.fw_ver, run->fw_ver, sizeof(vpu->run.fw_ver));
645 	vpu->run.dec_capability = run->dec_capability;
646 	vpu->run.enc_capability = run->enc_capability;
647 	wake_up_interruptible(&vpu->run.wq);
648 }
649 
650 #ifdef CONFIG_DEBUG_FS
651 static ssize_t vpu_debug_read(struct file *file, char __user *user_buf,
652 			      size_t count, loff_t *ppos)
653 {
654 	char buf[256];
655 	unsigned int len;
656 	unsigned int running, pc, vpu_to_host, host_to_vpu, wdt, idle, ra, sp;
657 	int ret;
658 	struct device *dev = file->private_data;
659 	struct mtk_vpu *vpu = dev_get_drvdata(dev);
660 
661 	ret = vpu_clock_enable(vpu);
662 	if (ret) {
663 		dev_err(vpu->dev, "[VPU] enable clock failed %d\n", ret);
664 		return 0;
665 	}
666 
667 	/* vpu register status */
668 	running = vpu_running(vpu);
669 	pc = vpu_cfg_readl(vpu, VPU_PC_REG);
670 	wdt = vpu_cfg_readl(vpu, VPU_WDT_REG);
671 	host_to_vpu = vpu_cfg_readl(vpu, HOST_TO_VPU);
672 	vpu_to_host = vpu_cfg_readl(vpu, VPU_TO_HOST);
673 	ra = vpu_cfg_readl(vpu, VPU_RA_REG);
674 	sp = vpu_cfg_readl(vpu, VPU_SP_REG);
675 	idle = vpu_cfg_readl(vpu, VPU_IDLE_REG);
676 
677 	vpu_clock_disable(vpu);
678 
679 	if (running) {
680 		len = snprintf(buf, sizeof(buf), "VPU is running\n\n"
681 		"FW Version: %s\n"
682 		"PC: 0x%x\n"
683 		"WDT: 0x%x\n"
684 		"Host to VPU: 0x%x\n"
685 		"VPU to Host: 0x%x\n"
686 		"SP: 0x%x\n"
687 		"RA: 0x%x\n"
688 		"idle: 0x%x\n",
689 		vpu->run.fw_ver, pc, wdt,
690 		host_to_vpu, vpu_to_host, sp, ra, idle);
691 	} else {
692 		len = snprintf(buf, sizeof(buf), "VPU not running\n");
693 	}
694 
695 	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
696 }
697 
698 static const struct file_operations vpu_debug_fops = {
699 	.open = simple_open,
700 	.read = vpu_debug_read,
701 };
702 #endif /* CONFIG_DEBUG_FS */
703 
704 static void vpu_free_ext_mem(struct mtk_vpu *vpu, u8 fw_type)
705 {
706 	struct device *dev = vpu->dev;
707 	size_t fw_ext_size = fw_type ? VPU_EXT_D_SIZE : VPU_EXT_P_SIZE;
708 
709 	dma_free_coherent(dev, fw_ext_size, vpu->extmem[fw_type].va,
710 			  vpu->extmem[fw_type].pa);
711 }
712 
713 static int vpu_alloc_ext_mem(struct mtk_vpu *vpu, u32 fw_type)
714 {
715 	struct device *dev = vpu->dev;
716 	size_t fw_ext_size = fw_type ? VPU_EXT_D_SIZE : VPU_EXT_P_SIZE;
717 	u32 vpu_ext_mem0 = fw_type ? VPU_DMEM_EXT0_ADDR : VPU_PMEM_EXT0_ADDR;
718 	u32 vpu_ext_mem1 = fw_type ? VPU_DMEM_EXT1_ADDR : VPU_PMEM_EXT1_ADDR;
719 	u32 offset_4gb = vpu->enable_4GB ? 0x40000000 : 0;
720 
721 	vpu->extmem[fw_type].va = dma_alloc_coherent(dev,
722 					       fw_ext_size,
723 					       &vpu->extmem[fw_type].pa,
724 					       GFP_KERNEL);
725 	if (!vpu->extmem[fw_type].va) {
726 		dev_err(dev, "Failed to allocate the extended program memory\n");
727 		return -ENOMEM;
728 	}
729 
730 	/* Disable extend0. Enable extend1 */
731 	vpu_cfg_writel(vpu, 0x1, vpu_ext_mem0);
732 	vpu_cfg_writel(vpu, (vpu->extmem[fw_type].pa & 0xFFFFF000) + offset_4gb,
733 		       vpu_ext_mem1);
734 
735 	dev_info(dev, "%s extend memory phy=0x%llx virt=0x%p\n",
736 		 fw_type ? "Data" : "Program",
737 		 (unsigned long long)vpu->extmem[fw_type].pa,
738 		 vpu->extmem[fw_type].va);
739 
740 	return 0;
741 }
742 
743 static void vpu_ipi_handler(struct mtk_vpu *vpu)
744 {
745 	struct share_obj __iomem *rcv_obj = vpu->recv_buf;
746 	struct vpu_ipi_desc *ipi_desc = vpu->ipi_desc;
747 	unsigned char data[SHARE_BUF_SIZE];
748 	s32 id = readl(&rcv_obj->id);
749 
750 	memcpy_fromio(data, rcv_obj->share_buf, sizeof(data));
751 	if (id < IPI_MAX && ipi_desc[id].handler) {
752 		ipi_desc[id].handler(data, readl(&rcv_obj->len),
753 				     ipi_desc[id].priv);
754 		if (id > IPI_VPU_INIT) {
755 			vpu->ipi_id_ack[id] = true;
756 			wake_up(&vpu->ack_wq);
757 		}
758 	} else {
759 		dev_err(vpu->dev, "No such ipi id = %d\n", id);
760 	}
761 }
762 
763 static int vpu_ipi_init(struct mtk_vpu *vpu)
764 {
765 	/* Disable VPU to host interrupt */
766 	vpu_cfg_writel(vpu, 0x0, VPU_TO_HOST);
767 
768 	/* shared buffer initialization */
769 	vpu->recv_buf = vpu->reg.tcm + VPU_DTCM_OFFSET;
770 	vpu->send_buf = vpu->recv_buf + 1;
771 	memset_io(vpu->recv_buf, 0, sizeof(struct share_obj));
772 	memset_io(vpu->send_buf, 0, sizeof(struct share_obj));
773 
774 	return 0;
775 }
776 
777 static irqreturn_t vpu_irq_handler(int irq, void *priv)
778 {
779 	struct mtk_vpu *vpu = priv;
780 	u32 vpu_to_host;
781 	int ret;
782 
783 	/*
784 	 * Clock should have been enabled already.
785 	 * Enable again in case vpu_ipi_send times out
786 	 * and has disabled the clock.
787 	 */
788 	ret = clk_enable(vpu->clk);
789 	if (ret) {
790 		dev_err(vpu->dev, "[VPU] enable clock failed %d\n", ret);
791 		return IRQ_NONE;
792 	}
793 	vpu_to_host = vpu_cfg_readl(vpu, VPU_TO_HOST);
794 	if (vpu_to_host & VPU_IPC_INT) {
795 		vpu_ipi_handler(vpu);
796 	} else {
797 		dev_err(vpu->dev, "vpu watchdog timeout! 0x%x", vpu_to_host);
798 		queue_work(vpu->wdt.wq, &vpu->wdt.ws);
799 	}
800 
801 	/* VPU won't send another interrupt until we set VPU_TO_HOST to 0. */
802 	vpu_cfg_writel(vpu, 0x0, VPU_TO_HOST);
803 	clk_disable(vpu->clk);
804 
805 	return IRQ_HANDLED;
806 }
807 
808 #ifdef CONFIG_DEBUG_FS
809 static struct dentry *vpu_debugfs;
810 #endif
811 static int mtk_vpu_probe(struct platform_device *pdev)
812 {
813 	struct mtk_vpu *vpu;
814 	struct device *dev;
815 	int ret = 0;
816 
817 	dev_dbg(&pdev->dev, "initialization\n");
818 
819 	dev = &pdev->dev;
820 	vpu = devm_kzalloc(dev, sizeof(*vpu), GFP_KERNEL);
821 	if (!vpu)
822 		return -ENOMEM;
823 
824 	vpu->dev = &pdev->dev;
825 	vpu->reg.tcm = devm_platform_ioremap_resource_byname(pdev, "tcm");
826 	if (IS_ERR((__force void *)vpu->reg.tcm))
827 		return PTR_ERR((__force void *)vpu->reg.tcm);
828 
829 	vpu->reg.cfg = devm_platform_ioremap_resource_byname(pdev, "cfg_reg");
830 	if (IS_ERR((__force void *)vpu->reg.cfg))
831 		return PTR_ERR((__force void *)vpu->reg.cfg);
832 
833 	/* Get VPU clock */
834 	vpu->clk = devm_clk_get(dev, "main");
835 	if (IS_ERR(vpu->clk)) {
836 		dev_err(dev, "get vpu clock failed\n");
837 		return PTR_ERR(vpu->clk);
838 	}
839 
840 	platform_set_drvdata(pdev, vpu);
841 
842 	ret = clk_prepare(vpu->clk);
843 	if (ret) {
844 		dev_err(dev, "prepare vpu clock failed\n");
845 		return ret;
846 	}
847 
848 	/* VPU watchdog */
849 	vpu->wdt.wq = create_singlethread_workqueue("vpu_wdt");
850 	if (!vpu->wdt.wq) {
851 		dev_err(dev, "initialize wdt workqueue failed\n");
852 		ret = -ENOMEM;
853 		goto clk_unprepare;
854 	}
855 	INIT_WORK(&vpu->wdt.ws, vpu_wdt_reset_func);
856 	mutex_init(&vpu->vpu_mutex);
857 
858 	ret = vpu_clock_enable(vpu);
859 	if (ret) {
860 		dev_err(dev, "enable vpu clock failed\n");
861 		goto workqueue_destroy;
862 	}
863 
864 	dev_dbg(dev, "vpu ipi init\n");
865 	ret = vpu_ipi_init(vpu);
866 	if (ret) {
867 		dev_err(dev, "Failed to init ipi\n");
868 		goto disable_vpu_clk;
869 	}
870 
871 	/* register vpu initialization IPI */
872 	ret = vpu_ipi_register(pdev, IPI_VPU_INIT, vpu_init_ipi_handler,
873 			       "vpu_init", vpu);
874 	if (ret) {
875 		dev_err(dev, "Failed to register IPI_VPU_INIT\n");
876 		goto vpu_mutex_destroy;
877 	}
878 
879 #ifdef CONFIG_DEBUG_FS
880 	vpu_debugfs = debugfs_create_file("mtk_vpu", S_IRUGO, NULL, (void *)dev,
881 					  &vpu_debug_fops);
882 #endif
883 
884 	/* Set PTCM to 96K and DTCM to 32K */
885 	vpu_cfg_writel(vpu, 0x2, VPU_TCM_CFG);
886 
887 	vpu->enable_4GB = !!(totalram_pages() > (SZ_2G >> PAGE_SHIFT));
888 	dev_info(dev, "4GB mode %u\n", vpu->enable_4GB);
889 
890 	if (vpu->enable_4GB) {
891 		ret = of_reserved_mem_device_init(dev);
892 		if (ret)
893 			dev_info(dev, "init reserved memory failed\n");
894 			/* continue to use dynamic allocation if failed */
895 	}
896 
897 	ret = vpu_alloc_ext_mem(vpu, D_FW);
898 	if (ret) {
899 		dev_err(dev, "Allocate DM failed\n");
900 		goto remove_debugfs;
901 	}
902 
903 	ret = vpu_alloc_ext_mem(vpu, P_FW);
904 	if (ret) {
905 		dev_err(dev, "Allocate PM failed\n");
906 		goto free_d_mem;
907 	}
908 
909 	init_waitqueue_head(&vpu->run.wq);
910 	init_waitqueue_head(&vpu->ack_wq);
911 
912 	ret = platform_get_irq(pdev, 0);
913 	if (ret < 0)
914 		goto free_p_mem;
915 	vpu->reg.irq = ret;
916 	ret = devm_request_irq(dev, vpu->reg.irq, vpu_irq_handler, 0,
917 			       pdev->name, vpu);
918 	if (ret) {
919 		dev_err(dev, "failed to request irq\n");
920 		goto free_p_mem;
921 	}
922 
923 	vpu_clock_disable(vpu);
924 	dev_dbg(dev, "initialization completed\n");
925 
926 	return 0;
927 
928 free_p_mem:
929 	vpu_free_ext_mem(vpu, P_FW);
930 free_d_mem:
931 	vpu_free_ext_mem(vpu, D_FW);
932 remove_debugfs:
933 	of_reserved_mem_device_release(dev);
934 #ifdef CONFIG_DEBUG_FS
935 	debugfs_remove(vpu_debugfs);
936 #endif
937 	memset(vpu->ipi_desc, 0, sizeof(struct vpu_ipi_desc) * IPI_MAX);
938 vpu_mutex_destroy:
939 	mutex_destroy(&vpu->vpu_mutex);
940 disable_vpu_clk:
941 	vpu_clock_disable(vpu);
942 workqueue_destroy:
943 	destroy_workqueue(vpu->wdt.wq);
944 clk_unprepare:
945 	clk_unprepare(vpu->clk);
946 
947 	return ret;
948 }
949 
950 static const struct of_device_id mtk_vpu_match[] = {
951 	{
952 		.compatible = "mediatek,mt8173-vpu",
953 	},
954 	{},
955 };
956 MODULE_DEVICE_TABLE(of, mtk_vpu_match);
957 
958 static void mtk_vpu_remove(struct platform_device *pdev)
959 {
960 	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
961 
962 #ifdef CONFIG_DEBUG_FS
963 	debugfs_remove(vpu_debugfs);
964 #endif
965 	if (vpu->wdt.wq)
966 		destroy_workqueue(vpu->wdt.wq);
967 	vpu_free_ext_mem(vpu, P_FW);
968 	vpu_free_ext_mem(vpu, D_FW);
969 	mutex_destroy(&vpu->vpu_mutex);
970 	clk_unprepare(vpu->clk);
971 }
972 
973 static int mtk_vpu_suspend(struct device *dev)
974 {
975 	struct mtk_vpu *vpu = dev_get_drvdata(dev);
976 	unsigned long timeout;
977 	int ret;
978 
979 	ret = vpu_clock_enable(vpu);
980 	if (ret) {
981 		dev_err(dev, "failed to enable vpu clock\n");
982 		return ret;
983 	}
984 
985 	if (!vpu_running(vpu)) {
986 		vpu_clock_disable(vpu);
987 		clk_unprepare(vpu->clk);
988 		return 0;
989 	}
990 
991 	mutex_lock(&vpu->vpu_mutex);
992 	/* disable vpu timer interrupt */
993 	vpu_cfg_writel(vpu, vpu_cfg_readl(vpu, VPU_INT_STATUS) | VPU_IDLE_STATE,
994 		       VPU_INT_STATUS);
995 	/* check if vpu is idle for system suspend */
996 	timeout = jiffies + msecs_to_jiffies(VPU_IDLE_TIMEOUT_MS);
997 	do {
998 		if (time_after(jiffies, timeout)) {
999 			dev_err(dev, "vpu idle timeout\n");
1000 			mutex_unlock(&vpu->vpu_mutex);
1001 			vpu_clock_disable(vpu);
1002 			return -EIO;
1003 		}
1004 	} while (!vpu_cfg_readl(vpu, VPU_IDLE_REG));
1005 
1006 	mutex_unlock(&vpu->vpu_mutex);
1007 	vpu_clock_disable(vpu);
1008 	clk_unprepare(vpu->clk);
1009 
1010 	return 0;
1011 }
1012 
1013 static int mtk_vpu_resume(struct device *dev)
1014 {
1015 	struct mtk_vpu *vpu = dev_get_drvdata(dev);
1016 	int ret;
1017 
1018 	clk_prepare(vpu->clk);
1019 	ret = vpu_clock_enable(vpu);
1020 	if (ret) {
1021 		clk_unprepare(vpu->clk);
1022 		dev_err(dev, "failed to enable vpu clock\n");
1023 		return ret;
1024 	}
1025 
1026 	mutex_lock(&vpu->vpu_mutex);
1027 	/* enable vpu timer interrupt */
1028 	vpu_cfg_writel(vpu,
1029 		       vpu_cfg_readl(vpu, VPU_INT_STATUS) & ~(VPU_IDLE_STATE),
1030 		       VPU_INT_STATUS);
1031 	mutex_unlock(&vpu->vpu_mutex);
1032 	vpu_clock_disable(vpu);
1033 
1034 	return 0;
1035 }
1036 
1037 static const struct dev_pm_ops mtk_vpu_pm = {
1038 	.suspend = mtk_vpu_suspend,
1039 	.resume = mtk_vpu_resume,
1040 };
1041 
1042 static struct platform_driver mtk_vpu_driver = {
1043 	.probe	= mtk_vpu_probe,
1044 	.remove_new = mtk_vpu_remove,
1045 	.driver	= {
1046 		.name	= "mtk_vpu",
1047 		.pm = &mtk_vpu_pm,
1048 		.of_match_table = mtk_vpu_match,
1049 	},
1050 };
1051 
1052 module_platform_driver(mtk_vpu_driver);
1053 
1054 MODULE_LICENSE("GPL v2");
1055 MODULE_DESCRIPTION("Mediatek Video Processor Unit driver");
1056