1 // SPDX-License-Identifier: GPL-2.0
7 #include <linux/clk-provider.h>
8 #include <linux/dma-mapping.h>
17 #include <linux/mailbox/mtk-cmdq-mailbox.h>
21 #define CMDQ_NUM_CMD(t)			(t->cmd_buf_size / CMDQ_INST_SIZE)
95 	WARN_ON(clk_bulk_enable(cmdq->pdata->gce_num, cmdq->clocks));  in cmdq_sw_ddr_enable()
98 		writel(GCE_DDR_EN | GCE_CTRL_BY_SW, cmdq->base + GCE_GCTL_VALUE);  in cmdq_sw_ddr_enable()
100 		writel(GCE_CTRL_BY_SW, cmdq->base + GCE_GCTL_VALUE);  in cmdq_sw_ddr_enable()
102 	clk_bulk_disable(cmdq->pdata->gce_num, cmdq->clocks);  in cmdq_sw_ddr_enable()
107 	struct cmdq *cmdq = container_of(chan->mbox, struct cmdq, mbox);  in cmdq_get_shift_pa()
109 	return cmdq->pdata->shift;  in cmdq_get_shift_pa()
117 	writel(CMDQ_THR_SUSPEND, thread->base + CMDQ_THR_SUSPEND_TASK);  in cmdq_thread_suspend()
120 	if (!(readl(thread->base + CMDQ_THR_ENABLE_TASK) & CMDQ_THR_ENABLED))  in cmdq_thread_suspend()
123 	if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_CURR_STATUS,  in cmdq_thread_suspend()
125 		dev_err(cmdq->mbox.dev, "suspend GCE thread 0x%x failed\n",  in cmdq_thread_suspend()
126 			(u32)(thread->base - cmdq->base));  in cmdq_thread_suspend()
127 		return -EFAULT;  in cmdq_thread_suspend()
135 	writel(CMDQ_THR_RESUME, thread->base + CMDQ_THR_SUSPEND_TASK);  in cmdq_thread_resume()
143 	WARN_ON(clk_bulk_enable(cmdq->pdata->gce_num, cmdq->clocks));  in cmdq_init()
144 	if (cmdq->pdata->control_by_sw)  in cmdq_init()
146 	if (cmdq->pdata->sw_ddr_en)  in cmdq_init()
150 		writel(gctl_regval, cmdq->base + GCE_GCTL_VALUE);  in cmdq_init()
152 	writel(CMDQ_THR_ACTIVE_SLOT_CYCLES, cmdq->base + CMDQ_THR_SLOT_CYCLES);  in cmdq_init()
154 		writel(i, cmdq->base + CMDQ_SYNC_TOKEN_UPDATE);  in cmdq_init()
155 	clk_bulk_disable(cmdq->pdata->gce_num, cmdq->clocks);  in cmdq_init()
162 	writel(CMDQ_THR_DO_WARM_RESET, thread->base + CMDQ_THR_WARM_RESET);  in cmdq_thread_reset()
163 	if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_WARM_RESET,  in cmdq_thread_reset()
166 		dev_err(cmdq->mbox.dev, "reset GCE thread 0x%x failed\n",  in cmdq_thread_reset()
167 			(u32)(thread->base - cmdq->base));  in cmdq_thread_reset()
168 		return -EFAULT;  in cmdq_thread_reset()
177 	writel(CMDQ_THR_DISABLED, thread->base + CMDQ_THR_ENABLE_TASK);  in cmdq_thread_disable()
180 /* notify GCE to re-fetch commands by setting GCE thread PC */
183 	writel(readl(thread->base + CMDQ_THR_CURR_ADDR),  in cmdq_thread_invalidate_fetched_data()
184 	       thread->base + CMDQ_THR_CURR_ADDR);  in cmdq_thread_invalidate_fetched_data()
189 	struct device *dev = task->cmdq->mbox.dev;  in cmdq_task_insert_into_thread()
190 	struct cmdq_thread *thread = task->thread;  in cmdq_task_insert_into_thread()
192 			&thread->task_busy_list, typeof(*task), list_entry);  in cmdq_task_insert_into_thread()
193 	u64 *prev_task_base = prev_task->pkt->va_base;  in cmdq_task_insert_into_thread()
196 	dma_sync_single_for_cpu(dev, prev_task->pa_base,  in cmdq_task_insert_into_thread()
197 				prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE);  in cmdq_task_insert_into_thread()
198 	prev_task_base[CMDQ_NUM_CMD(prev_task->pkt) - 1] =  in cmdq_task_insert_into_thread()
200 		(task->pa_base >> task->cmdq->pdata->shift);  in cmdq_task_insert_into_thread()
201 	dma_sync_single_for_device(dev, prev_task->pa_base,  in cmdq_task_insert_into_thread()
202 				   prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE);  in cmdq_task_insert_into_thread()
209 	return readl(thread->base + CMDQ_THR_WAIT_TOKEN) & CMDQ_THR_IS_WAITING;  in cmdq_thread_is_in_wfe()
217 	data.pkt = task->pkt;  in cmdq_task_exec_done()
218 	mbox_chan_received_data(task->thread->chan, &data);  in cmdq_task_exec_done()
220 	list_del(&task->list_entry);  in cmdq_task_exec_done()
225 	struct cmdq_thread *thread = task->thread;  in cmdq_task_handle_error()
227 	struct cmdq *cmdq = task->cmdq;  in cmdq_task_handle_error()
229 	dev_err(cmdq->mbox.dev, "task 0x%p error\n", task);  in cmdq_task_handle_error()
231 	next_task = list_first_entry_or_null(&thread->task_busy_list,  in cmdq_task_handle_error()
234 		writel(next_task->pa_base >> cmdq->pdata->shift,  in cmdq_task_handle_error()
235 		       thread->base + CMDQ_THR_CURR_ADDR);  in cmdq_task_handle_error()
246 	irq_flag = readl(thread->base + CMDQ_THR_IRQ_STATUS);  in cmdq_thread_irq_handler()
247 	writel(~irq_flag, thread->base + CMDQ_THR_IRQ_STATUS);  in cmdq_thread_irq_handler()
252 	 * reset / disable this GCE thread, so we need to check the enable  in cmdq_thread_irq_handler()
253 	 * bit of this GCE thread.  in cmdq_thread_irq_handler()
255 	if (!(readl(thread->base + CMDQ_THR_ENABLE_TASK) & CMDQ_THR_ENABLED))  in cmdq_thread_irq_handler()
265 	curr_pa = readl(thread->base + CMDQ_THR_CURR_ADDR) << cmdq->pdata->shift;  in cmdq_thread_irq_handler()
267 	list_for_each_entry_safe(task, tmp, &thread->task_busy_list,  in cmdq_thread_irq_handler()
269 		task_end_pa = task->pa_base + task->pkt->cmd_buf_size;  in cmdq_thread_irq_handler()
270 		if (curr_pa >= task->pa_base && curr_pa < task_end_pa)  in cmdq_thread_irq_handler()
273 		if (!curr_task || curr_pa == task_end_pa - CMDQ_INST_SIZE) {  in cmdq_thread_irq_handler()
277 			cmdq_task_exec_done(task, -ENOEXEC);  in cmdq_thread_irq_handler()
286 	if (list_empty(&thread->task_busy_list)) {  in cmdq_thread_irq_handler()
288 		clk_bulk_disable(cmdq->pdata->gce_num, cmdq->clocks);  in cmdq_thread_irq_handler()
298 	irq_status = readl(cmdq->base + CMDQ_CURR_IRQ_STATUS) & cmdq->irq_mask;  in cmdq_irq_handler()
299 	if (!(irq_status ^ cmdq->irq_mask))  in cmdq_irq_handler()
302 	for_each_clear_bit(bit, &irq_status, cmdq->pdata->thread_nr) {  in cmdq_irq_handler()
303 		struct cmdq_thread *thread = &cmdq->thread[bit];  in cmdq_irq_handler()
305 		spin_lock_irqsave(&thread->chan->lock, flags);  in cmdq_irq_handler()
307 		spin_unlock_irqrestore(&thread->chan->lock, flags);  in cmdq_irq_handler()
320 	cmdq->suspended = true;  in cmdq_suspend()
322 	for (i = 0; i < cmdq->pdata->thread_nr; i++) {  in cmdq_suspend()
323 		thread = &cmdq->thread[i];  in cmdq_suspend()
324 		if (!list_empty(&thread->task_busy_list)) {  in cmdq_suspend()
333 	if (cmdq->pdata->sw_ddr_en)  in cmdq_suspend()
336 	clk_bulk_unprepare(cmdq->pdata->gce_num, cmdq->clocks);  in cmdq_suspend()
345 	WARN_ON(clk_bulk_prepare(cmdq->pdata->gce_num, cmdq->clocks));  in cmdq_resume()
346 	cmdq->suspended = false;  in cmdq_resume()
348 	if (cmdq->pdata->sw_ddr_en)  in cmdq_resume()
358 	if (cmdq->pdata->sw_ddr_en)  in cmdq_remove()
361 	clk_bulk_unprepare(cmdq->pdata->gce_num, cmdq->clocks);  in cmdq_remove()
368 	struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv;  in cmdq_mbox_send_data()
369 	struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev);  in cmdq_mbox_send_data()
374 	WARN_ON(cmdq->suspended);  in cmdq_mbox_send_data()
378 		return -ENOMEM;  in cmdq_mbox_send_data()
380 	task->cmdq = cmdq;  in cmdq_mbox_send_data()
381 	INIT_LIST_HEAD(&task->list_entry);  in cmdq_mbox_send_data()
382 	task->pa_base = pkt->pa_base;  in cmdq_mbox_send_data()
383 	task->thread = thread;  in cmdq_mbox_send_data()
384 	task->pkt = pkt;  in cmdq_mbox_send_data()
386 	if (list_empty(&thread->task_busy_list)) {  in cmdq_mbox_send_data()
387 		WARN_ON(clk_bulk_enable(cmdq->pdata->gce_num, cmdq->clocks));  in cmdq_mbox_send_data()
397 		writel(task->pa_base >> cmdq->pdata->shift,  in cmdq_mbox_send_data()
398 		       thread->base + CMDQ_THR_CURR_ADDR);  in cmdq_mbox_send_data()
399 		writel((task->pa_base + pkt->cmd_buf_size) >> cmdq->pdata->shift,  in cmdq_mbox_send_data()
400 		       thread->base + CMDQ_THR_END_ADDR);  in cmdq_mbox_send_data()
402 		writel(thread->priority, thread->base + CMDQ_THR_PRIORITY);  in cmdq_mbox_send_data()
403 		writel(CMDQ_THR_IRQ_EN, thread->base + CMDQ_THR_IRQ_ENABLE);  in cmdq_mbox_send_data()
404 		writel(CMDQ_THR_ENABLED, thread->base + CMDQ_THR_ENABLE_TASK);  in cmdq_mbox_send_data()
407 		curr_pa = readl(thread->base + CMDQ_THR_CURR_ADDR) <<  in cmdq_mbox_send_data()
408 			cmdq->pdata->shift;  in cmdq_mbox_send_data()
409 		end_pa = readl(thread->base + CMDQ_THR_END_ADDR) <<  in cmdq_mbox_send_data()
410 			cmdq->pdata->shift;  in cmdq_mbox_send_data()
412 		if (curr_pa == end_pa - CMDQ_INST_SIZE ||  in cmdq_mbox_send_data()
415 			writel(task->pa_base >> cmdq->pdata->shift,  in cmdq_mbox_send_data()
416 			       thread->base + CMDQ_THR_CURR_ADDR);  in cmdq_mbox_send_data()
421 		writel((task->pa_base + pkt->cmd_buf_size) >> cmdq->pdata->shift,  in cmdq_mbox_send_data()
422 		       thread->base + CMDQ_THR_END_ADDR);  in cmdq_mbox_send_data()
425 	list_move_tail(&task->list_entry, &thread->task_busy_list);  in cmdq_mbox_send_data()
437 	struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv;  in cmdq_mbox_shutdown()
438 	struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev);  in cmdq_mbox_shutdown()
442 	spin_lock_irqsave(&thread->chan->lock, flags);  in cmdq_mbox_shutdown()
443 	if (list_empty(&thread->task_busy_list))  in cmdq_mbox_shutdown()
450 	if (list_empty(&thread->task_busy_list))  in cmdq_mbox_shutdown()
453 	list_for_each_entry_safe(task, tmp, &thread->task_busy_list,  in cmdq_mbox_shutdown()
455 		cmdq_task_exec_done(task, -ECONNABORTED);  in cmdq_mbox_shutdown()
460 	clk_bulk_disable(cmdq->pdata->gce_num, cmdq->clocks);  in cmdq_mbox_shutdown()
464 	 * The thread->task_busy_list empty means thread already disable. The  in cmdq_mbox_shutdown()
469 	spin_unlock_irqrestore(&thread->chan->lock, flags);  in cmdq_mbox_shutdown()
474 	struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv;  in cmdq_mbox_flush()
476 	struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev);  in cmdq_mbox_flush()
481 	spin_lock_irqsave(&thread->chan->lock, flags);  in cmdq_mbox_flush()
482 	if (list_empty(&thread->task_busy_list))  in cmdq_mbox_flush()
489 	list_for_each_entry_safe(task, tmp, &thread->task_busy_list,  in cmdq_mbox_flush()
491 		data.sta = -ECONNABORTED;  in cmdq_mbox_flush()
492 		data.pkt = task->pkt;  in cmdq_mbox_flush()
493 		mbox_chan_received_data(task->thread->chan, &data);  in cmdq_mbox_flush()
494 		list_del(&task->list_entry);  in cmdq_mbox_flush()
500 	clk_bulk_disable(cmdq->pdata->gce_num, cmdq->clocks);  in cmdq_mbox_flush()
503 	spin_unlock_irqrestore(&thread->chan->lock, flags);  in cmdq_mbox_flush()
508 	spin_unlock_irqrestore(&thread->chan->lock, flags);  in cmdq_mbox_flush()
509 	if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_ENABLE_TASK,  in cmdq_mbox_flush()
511 		dev_err(cmdq->mbox.dev, "Fail to wait GCE thread 0x%x done\n",  in cmdq_mbox_flush()
512 			(u32)(thread->base - cmdq->base));  in cmdq_mbox_flush()
514 		return -EFAULT;  in cmdq_mbox_flush()
529 	int ind = sp->args[0];  in cmdq_xlate()
532 	if (ind >= mbox->num_chans)  in cmdq_xlate()
533 		return ERR_PTR(-EINVAL);  in cmdq_xlate()
535 	thread = (struct cmdq_thread *)mbox->chans[ind].con_priv;  in cmdq_xlate()
536 	thread->priority = sp->args[1];  in cmdq_xlate()
537 	thread->chan = &mbox->chans[ind];  in cmdq_xlate()
539 	return &mbox->chans[ind];  in cmdq_xlate()
544 	struct device *dev = &pdev->dev;  in cmdq_probe()
547 	struct device_node *phandle = dev->of_node;  in cmdq_probe()
550 	static const char * const clk_name = "gce";  in cmdq_probe()
555 		return -ENOMEM;  in cmdq_probe()
557 	cmdq->base = devm_platform_ioremap_resource(pdev, 0);  in cmdq_probe()
558 	if (IS_ERR(cmdq->base))  in cmdq_probe()
559 		return PTR_ERR(cmdq->base);  in cmdq_probe()
561 	cmdq->irq = platform_get_irq(pdev, 0);  in cmdq_probe()
562 	if (cmdq->irq < 0)  in cmdq_probe()
563 		return cmdq->irq;  in cmdq_probe()
565 	cmdq->pdata = device_get_match_data(dev);  in cmdq_probe()
566 	if (!cmdq->pdata) {  in cmdq_probe()
568 		return -EINVAL;  in cmdq_probe()
571 	cmdq->irq_mask = GENMASK(cmdq->pdata->thread_nr - 1, 0);  in cmdq_probe()
574 		dev, cmdq->base, cmdq->irq);  in cmdq_probe()
576 	if (cmdq->pdata->gce_num > 1) {  in cmdq_probe()
577 		for_each_child_of_node(phandle->parent, node) {  in cmdq_probe()
579 			if (alias_id >= 0 && alias_id < cmdq->pdata->gce_num) {  in cmdq_probe()
580 				cmdq->clocks[alias_id].id = clk_names[alias_id];  in cmdq_probe()
581 				cmdq->clocks[alias_id].clk = of_clk_get(node, 0);  in cmdq_probe()
582 				if (IS_ERR(cmdq->clocks[alias_id].clk)) {  in cmdq_probe()
585 							     PTR_ERR(cmdq->clocks[alias_id].clk),  in cmdq_probe()
586 							     "failed to get gce clk: %d\n",  in cmdq_probe()
592 		cmdq->clocks[alias_id].id = clk_name;  in cmdq_probe()
593 		cmdq->clocks[alias_id].clk = devm_clk_get(&pdev->dev, clk_name);  in cmdq_probe()
594 		if (IS_ERR(cmdq->clocks[alias_id].clk)) {  in cmdq_probe()
595 			return dev_err_probe(dev, PTR_ERR(cmdq->clocks[alias_id].clk),  in cmdq_probe()
596 					     "failed to get gce clk\n");  in cmdq_probe()
600 	cmdq->mbox.dev = dev;  in cmdq_probe()
601 	cmdq->mbox.chans = devm_kcalloc(dev, cmdq->pdata->thread_nr,  in cmdq_probe()
602 					sizeof(*cmdq->mbox.chans), GFP_KERNEL);  in cmdq_probe()
603 	if (!cmdq->mbox.chans)  in cmdq_probe()
604 		return -ENOMEM;  in cmdq_probe()
606 	cmdq->mbox.num_chans = cmdq->pdata->thread_nr;  in cmdq_probe()
607 	cmdq->mbox.ops = &cmdq_mbox_chan_ops;  in cmdq_probe()
608 	cmdq->mbox.of_xlate = cmdq_xlate;  in cmdq_probe()
611 	cmdq->mbox.txdone_irq = false;  in cmdq_probe()
612 	cmdq->mbox.txdone_poll = false;  in cmdq_probe()
614 	cmdq->thread = devm_kcalloc(dev, cmdq->pdata->thread_nr,  in cmdq_probe()
615 					sizeof(*cmdq->thread), GFP_KERNEL);  in cmdq_probe()
616 	if (!cmdq->thread)  in cmdq_probe()
617 		return -ENOMEM;  in cmdq_probe()
619 	for (i = 0; i < cmdq->pdata->thread_nr; i++) {  in cmdq_probe()
620 		cmdq->thread[i].base = cmdq->base + CMDQ_THR_BASE +  in cmdq_probe()
622 		INIT_LIST_HEAD(&cmdq->thread[i].task_busy_list);  in cmdq_probe()
623 		cmdq->mbox.chans[i].con_priv = (void *)&cmdq->thread[i];  in cmdq_probe()
628 	WARN_ON(clk_bulk_prepare(cmdq->pdata->gce_num, cmdq->clocks));  in cmdq_probe()
632 	err = devm_request_irq(dev, cmdq->irq, cmdq_irq_handler, IRQF_SHARED,  in cmdq_probe()
639 	err = devm_mbox_controller_register(dev, &cmdq->mbox);  in cmdq_probe()
697 	{.compatible = "mediatek,mt8173-gce", .data = (void *)&gce_plat_v2},
698 	{.compatible = "mediatek,mt8183-gce", .data = (void *)&gce_plat_v3},
699 	{.compatible = "mediatek,mt8186-gce", .data = (void *)&gce_plat_v7},
700 	{.compatible = "mediatek,mt6779-gce", .data = (void *)&gce_plat_v4},
701 	{.compatible = "mediatek,mt8192-gce", .data = (void *)&gce_plat_v5},
702 	{.compatible = "mediatek,mt8195-gce", .data = (void *)&gce_plat_v6},