1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2021, MediaTek Inc.
4  * Copyright (c) 2021-2022, Intel Corporation.
5  *
6  * Authors:
7  *  Amir Hanania <amir.hanania@intel.com>
8  *  Haijun Liu <haijun.liu@mediatek.com>
9  *  Moises Veleta <moises.veleta@intel.com>
10  *  Ricardo Martinez <ricardo.martinez@linux.intel.com>
11  *  Sreehari Kancharla <sreehari.kancharla@intel.com>
12  *
13  * Contributors:
14  *  Andy Shevchenko <andriy.shevchenko@linux.intel.com>
15  *  Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
16  *  Eliot Lee <eliot.lee@intel.com>
17  */
18 
19 #include <linux/bits.h>
20 #include <linux/bitops.h>
21 #include <linux/delay.h>
22 #include <linux/device.h>
23 #include <linux/dmapool.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/dma-direction.h>
26 #include <linux/gfp.h>
27 #include <linux/io.h>
28 #include <linux/io-64-nonatomic-lo-hi.h>
29 #include <linux/iopoll.h>
30 #include <linux/irqreturn.h>
31 #include <linux/kernel.h>
32 #include <linux/kthread.h>
33 #include <linux/list.h>
34 #include <linux/netdevice.h>
35 #include <linux/pci.h>
36 #include <linux/pm_runtime.h>
37 #include <linux/sched.h>
38 #include <linux/skbuff.h>
39 #include <linux/slab.h>
40 #include <linux/spinlock.h>
41 #include <linux/types.h>
42 #include <linux/wait.h>
43 #include <linux/workqueue.h>
44 
45 #include "t7xx_cldma.h"
46 #include "t7xx_hif_cldma.h"
47 #include "t7xx_mhccif.h"
48 #include "t7xx_pci.h"
49 #include "t7xx_pcie_mac.h"
50 #include "t7xx_port_proxy.h"
51 #include "t7xx_reg.h"
52 #include "t7xx_state_monitor.h"
53 
54 #define MAX_TX_BUDGET			16
55 #define MAX_RX_BUDGET			16
56 
57 #define CHECK_Q_STOP_TIMEOUT_US		1000000
58 #define CHECK_Q_STOP_STEP_US		10000
59 
60 #define CLDMA_JUMBO_BUFF_SZ		(63 * 1024 + sizeof(struct ccci_header))
61 
62 static void md_cd_queue_struct_reset(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl,
63 				     enum mtk_txrx tx_rx, unsigned int index)
64 {
65 	queue->dir = tx_rx;
66 	queue->index = index;
67 	queue->md_ctrl = md_ctrl;
68 	queue->tr_ring = NULL;
69 	queue->tr_done = NULL;
70 	queue->tx_next = NULL;
71 }
72 
73 static void md_cd_queue_struct_init(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl,
74 				    enum mtk_txrx tx_rx, unsigned int index)
75 {
76 	md_cd_queue_struct_reset(queue, md_ctrl, tx_rx, index);
77 	init_waitqueue_head(&queue->req_wq);
78 	spin_lock_init(&queue->ring_lock);
79 }
80 
81 static void t7xx_cldma_gpd_set_data_ptr(struct cldma_gpd *gpd, dma_addr_t data_ptr)
82 {
83 	gpd->data_buff_bd_ptr_h = cpu_to_le32(upper_32_bits(data_ptr));
84 	gpd->data_buff_bd_ptr_l = cpu_to_le32(lower_32_bits(data_ptr));
85 }
86 
87 static void t7xx_cldma_gpd_set_next_ptr(struct cldma_gpd *gpd, dma_addr_t next_ptr)
88 {
89 	gpd->next_gpd_ptr_h = cpu_to_le32(upper_32_bits(next_ptr));
90 	gpd->next_gpd_ptr_l = cpu_to_le32(lower_32_bits(next_ptr));
91 }
92 
93 static int t7xx_cldma_alloc_and_map_skb(struct cldma_ctrl *md_ctrl, struct cldma_request *req,
94 					size_t size, gfp_t gfp_mask)
95 {
96 	req->skb = __dev_alloc_skb(size, gfp_mask);
97 	if (!req->skb)
98 		return -ENOMEM;
99 
100 	req->mapped_buff = dma_map_single(md_ctrl->dev, req->skb->data, size, DMA_FROM_DEVICE);
101 	if (dma_mapping_error(md_ctrl->dev, req->mapped_buff)) {
102 		dev_kfree_skb_any(req->skb);
103 		req->skb = NULL;
104 		req->mapped_buff = 0;
105 		dev_err(md_ctrl->dev, "DMA mapping failed\n");
106 		return -ENOMEM;
107 	}
108 
109 	return 0;
110 }
111 
112 static int t7xx_cldma_gpd_rx_from_q(struct cldma_queue *queue, int budget, bool *over_budget)
113 {
114 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
115 	unsigned int hwo_polling_count = 0;
116 	struct t7xx_cldma_hw *hw_info;
117 	bool rx_not_done = true;
118 	unsigned long flags;
119 	int count = 0;
120 
121 	hw_info = &md_ctrl->hw_info;
122 
123 	do {
124 		struct cldma_request *req;
125 		struct cldma_gpd *gpd;
126 		struct sk_buff *skb;
127 		int ret;
128 
129 		req = queue->tr_done;
130 		if (!req)
131 			return -ENODATA;
132 
133 		gpd = req->gpd;
134 		if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) {
135 			dma_addr_t gpd_addr;
136 
137 			if (!pci_device_is_present(to_pci_dev(md_ctrl->dev))) {
138 				dev_err(md_ctrl->dev, "PCIe Link disconnected\n");
139 				return -ENODEV;
140 			}
141 
142 			gpd_addr = ioread64(hw_info->ap_pdn_base + REG_CLDMA_DL_CURRENT_ADDRL_0 +
143 					    queue->index * sizeof(u64));
144 			if (req->gpd_addr == gpd_addr || hwo_polling_count++ >= 100)
145 				return 0;
146 
147 			udelay(1);
148 			continue;
149 		}
150 
151 		hwo_polling_count = 0;
152 		skb = req->skb;
153 
154 		if (req->mapped_buff) {
155 			dma_unmap_single(md_ctrl->dev, req->mapped_buff,
156 					 queue->tr_ring->pkt_size, DMA_FROM_DEVICE);
157 			req->mapped_buff = 0;
158 		}
159 
160 		skb->len = 0;
161 		skb_reset_tail_pointer(skb);
162 		skb_put(skb, le16_to_cpu(gpd->data_buff_len));
163 
164 		ret = md_ctrl->recv_skb(queue, skb);
165 		/* Break processing, will try again later */
166 		if (ret < 0)
167 			return ret;
168 
169 		req->skb = NULL;
170 		t7xx_cldma_gpd_set_data_ptr(gpd, 0);
171 
172 		spin_lock_irqsave(&queue->ring_lock, flags);
173 		queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
174 		spin_unlock_irqrestore(&queue->ring_lock, flags);
175 		req = queue->rx_refill;
176 
177 		ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, queue->tr_ring->pkt_size, GFP_KERNEL);
178 		if (ret)
179 			return ret;
180 
181 		gpd = req->gpd;
182 		t7xx_cldma_gpd_set_data_ptr(gpd, req->mapped_buff);
183 		gpd->data_buff_len = 0;
184 		gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
185 
186 		spin_lock_irqsave(&queue->ring_lock, flags);
187 		queue->rx_refill = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
188 		spin_unlock_irqrestore(&queue->ring_lock, flags);
189 
190 		rx_not_done = ++count < budget || !need_resched();
191 	} while (rx_not_done);
192 
193 	*over_budget = true;
194 	return 0;
195 }
196 
197 static int t7xx_cldma_gpd_rx_collect(struct cldma_queue *queue, int budget)
198 {
199 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
200 	struct t7xx_cldma_hw *hw_info;
201 	unsigned int pending_rx_int;
202 	bool over_budget = false;
203 	unsigned long flags;
204 	int ret;
205 
206 	hw_info = &md_ctrl->hw_info;
207 
208 	do {
209 		ret = t7xx_cldma_gpd_rx_from_q(queue, budget, &over_budget);
210 		if (ret == -ENODATA)
211 			return 0;
212 		else if (ret)
213 			return ret;
214 
215 		pending_rx_int = 0;
216 
217 		spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
218 		if (md_ctrl->rxq_active & BIT(queue->index)) {
219 			if (!t7xx_cldma_hw_queue_status(hw_info, queue->index, MTK_RX))
220 				t7xx_cldma_hw_resume_queue(hw_info, queue->index, MTK_RX);
221 
222 			pending_rx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index),
223 								  MTK_RX);
224 			if (pending_rx_int) {
225 				t7xx_cldma_hw_rx_done(hw_info, pending_rx_int);
226 
227 				if (over_budget) {
228 					spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
229 					return -EAGAIN;
230 				}
231 			}
232 		}
233 		spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
234 	} while (pending_rx_int);
235 
236 	return 0;
237 }
238 
239 static void t7xx_cldma_rx_done(struct work_struct *work)
240 {
241 	struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work);
242 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
243 	int value;
244 
245 	value = t7xx_cldma_gpd_rx_collect(queue, queue->budget);
246 	if (value && md_ctrl->rxq_active & BIT(queue->index)) {
247 		queue_work(queue->worker, &queue->cldma_work);
248 		return;
249 	}
250 
251 	t7xx_cldma_clear_ip_busy(&md_ctrl->hw_info);
252 	t7xx_cldma_hw_irq_en_txrx(&md_ctrl->hw_info, queue->index, MTK_RX);
253 	t7xx_cldma_hw_irq_en_eq(&md_ctrl->hw_info, queue->index, MTK_RX);
254 	pm_runtime_mark_last_busy(md_ctrl->dev);
255 	pm_runtime_put_autosuspend(md_ctrl->dev);
256 }
257 
258 static int t7xx_cldma_gpd_tx_collect(struct cldma_queue *queue)
259 {
260 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
261 	unsigned int dma_len, count = 0;
262 	struct cldma_request *req;
263 	struct cldma_gpd *gpd;
264 	unsigned long flags;
265 	dma_addr_t dma_free;
266 	struct sk_buff *skb;
267 
268 	while (!kthread_should_stop()) {
269 		spin_lock_irqsave(&queue->ring_lock, flags);
270 		req = queue->tr_done;
271 		if (!req) {
272 			spin_unlock_irqrestore(&queue->ring_lock, flags);
273 			break;
274 		}
275 		gpd = req->gpd;
276 		if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) {
277 			spin_unlock_irqrestore(&queue->ring_lock, flags);
278 			break;
279 		}
280 		queue->budget++;
281 		dma_free = req->mapped_buff;
282 		dma_len = le16_to_cpu(gpd->data_buff_len);
283 		skb = req->skb;
284 		req->skb = NULL;
285 		queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
286 		spin_unlock_irqrestore(&queue->ring_lock, flags);
287 
288 		count++;
289 		dma_unmap_single(md_ctrl->dev, dma_free, dma_len, DMA_TO_DEVICE);
290 		dev_kfree_skb_any(skb);
291 	}
292 
293 	if (count)
294 		wake_up_nr(&queue->req_wq, count);
295 
296 	return count;
297 }
298 
299 static void t7xx_cldma_txq_empty_hndl(struct cldma_queue *queue)
300 {
301 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
302 	struct cldma_request *req;
303 	dma_addr_t ul_curr_addr;
304 	unsigned long flags;
305 	bool pending_gpd;
306 
307 	if (!(md_ctrl->txq_active & BIT(queue->index)))
308 		return;
309 
310 	spin_lock_irqsave(&queue->ring_lock, flags);
311 	req = list_prev_entry_circular(queue->tx_next, &queue->tr_ring->gpd_ring, entry);
312 	spin_unlock_irqrestore(&queue->ring_lock, flags);
313 
314 	pending_gpd = (req->gpd->flags & GPD_FLAGS_HWO) && req->skb;
315 
316 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
317 	if (pending_gpd) {
318 		struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
319 
320 		/* Check current processing TGPD, 64-bit address is in a table by Q index */
321 		ul_curr_addr = ioread64(hw_info->ap_pdn_base + REG_CLDMA_UL_CURRENT_ADDRL_0 +
322 					queue->index * sizeof(u64));
323 		if (req->gpd_addr != ul_curr_addr) {
324 			spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
325 			dev_err(md_ctrl->dev, "CLDMA%d queue %d is not empty\n",
326 				md_ctrl->hif_id, queue->index);
327 			return;
328 		}
329 
330 		t7xx_cldma_hw_resume_queue(hw_info, queue->index, MTK_TX);
331 	}
332 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
333 }
334 
335 static void t7xx_cldma_tx_done(struct work_struct *work)
336 {
337 	struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work);
338 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
339 	struct t7xx_cldma_hw *hw_info;
340 	unsigned int l2_tx_int;
341 	unsigned long flags;
342 
343 	hw_info = &md_ctrl->hw_info;
344 	t7xx_cldma_gpd_tx_collect(queue);
345 	l2_tx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index) | EQ_STA_BIT(queue->index),
346 					     MTK_TX);
347 	if (l2_tx_int & EQ_STA_BIT(queue->index)) {
348 		t7xx_cldma_hw_tx_done(hw_info, EQ_STA_BIT(queue->index));
349 		t7xx_cldma_txq_empty_hndl(queue);
350 	}
351 
352 	if (l2_tx_int & BIT(queue->index)) {
353 		t7xx_cldma_hw_tx_done(hw_info, BIT(queue->index));
354 		queue_work(queue->worker, &queue->cldma_work);
355 		return;
356 	}
357 
358 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
359 	if (md_ctrl->txq_active & BIT(queue->index)) {
360 		t7xx_cldma_clear_ip_busy(hw_info);
361 		t7xx_cldma_hw_irq_en_eq(hw_info, queue->index, MTK_TX);
362 		t7xx_cldma_hw_irq_en_txrx(hw_info, queue->index, MTK_TX);
363 	}
364 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
365 
366 	pm_runtime_mark_last_busy(md_ctrl->dev);
367 	pm_runtime_put_autosuspend(md_ctrl->dev);
368 }
369 
370 static void t7xx_cldma_ring_free(struct cldma_ctrl *md_ctrl,
371 				 struct cldma_ring *ring, enum dma_data_direction tx_rx)
372 {
373 	struct cldma_request *req_cur, *req_next;
374 
375 	list_for_each_entry_safe(req_cur, req_next, &ring->gpd_ring, entry) {
376 		if (req_cur->mapped_buff && req_cur->skb) {
377 			dma_unmap_single(md_ctrl->dev, req_cur->mapped_buff,
378 					 ring->pkt_size, tx_rx);
379 			req_cur->mapped_buff = 0;
380 		}
381 
382 		dev_kfree_skb_any(req_cur->skb);
383 
384 		if (req_cur->gpd)
385 			dma_pool_free(md_ctrl->gpd_dmapool, req_cur->gpd, req_cur->gpd_addr);
386 
387 		list_del(&req_cur->entry);
388 		kfree(req_cur);
389 	}
390 }
391 
392 static struct cldma_request *t7xx_alloc_rx_request(struct cldma_ctrl *md_ctrl, size_t pkt_size)
393 {
394 	struct cldma_request *req;
395 	int val;
396 
397 	req = kzalloc(sizeof(*req), GFP_KERNEL);
398 	if (!req)
399 		return NULL;
400 
401 	req->gpd = dma_pool_zalloc(md_ctrl->gpd_dmapool, GFP_KERNEL, &req->gpd_addr);
402 	if (!req->gpd)
403 		goto err_free_req;
404 
405 	val = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, pkt_size, GFP_KERNEL);
406 	if (val)
407 		goto err_free_pool;
408 
409 	return req;
410 
411 err_free_pool:
412 	dma_pool_free(md_ctrl->gpd_dmapool, req->gpd, req->gpd_addr);
413 
414 err_free_req:
415 	kfree(req);
416 
417 	return NULL;
418 }
419 
420 static int t7xx_cldma_rx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring)
421 {
422 	struct cldma_request *req;
423 	struct cldma_gpd *gpd;
424 	int i;
425 
426 	INIT_LIST_HEAD(&ring->gpd_ring);
427 	ring->length = MAX_RX_BUDGET;
428 
429 	for (i = 0; i < ring->length; i++) {
430 		req = t7xx_alloc_rx_request(md_ctrl, ring->pkt_size);
431 		if (!req) {
432 			t7xx_cldma_ring_free(md_ctrl, ring, DMA_FROM_DEVICE);
433 			return -ENOMEM;
434 		}
435 
436 		gpd = req->gpd;
437 		t7xx_cldma_gpd_set_data_ptr(gpd, req->mapped_buff);
438 		gpd->rx_data_allow_len = cpu_to_le16(ring->pkt_size);
439 		gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
440 		INIT_LIST_HEAD(&req->entry);
441 		list_add_tail(&req->entry, &ring->gpd_ring);
442 	}
443 
444 	/* Link previous GPD to next GPD, circular */
445 	list_for_each_entry(req, &ring->gpd_ring, entry) {
446 		t7xx_cldma_gpd_set_next_ptr(gpd, req->gpd_addr);
447 		gpd = req->gpd;
448 	}
449 
450 	return 0;
451 }
452 
453 static struct cldma_request *t7xx_alloc_tx_request(struct cldma_ctrl *md_ctrl)
454 {
455 	struct cldma_request *req;
456 
457 	req = kzalloc(sizeof(*req), GFP_KERNEL);
458 	if (!req)
459 		return NULL;
460 
461 	req->gpd = dma_pool_zalloc(md_ctrl->gpd_dmapool, GFP_KERNEL, &req->gpd_addr);
462 	if (!req->gpd) {
463 		kfree(req);
464 		return NULL;
465 	}
466 
467 	return req;
468 }
469 
470 static int t7xx_cldma_tx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring)
471 {
472 	struct cldma_request *req;
473 	struct cldma_gpd *gpd;
474 	int i;
475 
476 	INIT_LIST_HEAD(&ring->gpd_ring);
477 	ring->length = MAX_TX_BUDGET;
478 
479 	for (i = 0; i < ring->length; i++) {
480 		req = t7xx_alloc_tx_request(md_ctrl);
481 		if (!req) {
482 			t7xx_cldma_ring_free(md_ctrl, ring, DMA_TO_DEVICE);
483 			return -ENOMEM;
484 		}
485 
486 		gpd = req->gpd;
487 		gpd->flags = GPD_FLAGS_IOC;
488 		INIT_LIST_HEAD(&req->entry);
489 		list_add_tail(&req->entry, &ring->gpd_ring);
490 	}
491 
492 	/* Link previous GPD to next GPD, circular */
493 	list_for_each_entry(req, &ring->gpd_ring, entry) {
494 		t7xx_cldma_gpd_set_next_ptr(gpd, req->gpd_addr);
495 		gpd = req->gpd;
496 	}
497 
498 	return 0;
499 }
500 
501 /**
502  * t7xx_cldma_q_reset() - Reset CLDMA request pointers to their initial values.
503  * @queue: Pointer to the queue structure.
504  *
505  * Called with ring_lock (unless called during initialization phase)
506  */
507 static void t7xx_cldma_q_reset(struct cldma_queue *queue)
508 {
509 	struct cldma_request *req;
510 
511 	req = list_first_entry(&queue->tr_ring->gpd_ring, struct cldma_request, entry);
512 	queue->tr_done = req;
513 	queue->budget = queue->tr_ring->length;
514 
515 	if (queue->dir == MTK_TX)
516 		queue->tx_next = req;
517 	else
518 		queue->rx_refill = req;
519 }
520 
521 static void t7xx_cldma_rxq_init(struct cldma_queue *queue)
522 {
523 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
524 
525 	queue->dir = MTK_RX;
526 	queue->tr_ring = &md_ctrl->rx_ring[queue->index];
527 	t7xx_cldma_q_reset(queue);
528 }
529 
530 static void t7xx_cldma_txq_init(struct cldma_queue *queue)
531 {
532 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
533 
534 	queue->dir = MTK_TX;
535 	queue->tr_ring = &md_ctrl->tx_ring[queue->index];
536 	t7xx_cldma_q_reset(queue);
537 }
538 
539 static void t7xx_cldma_enable_irq(struct cldma_ctrl *md_ctrl)
540 {
541 	t7xx_pcie_mac_set_int(md_ctrl->t7xx_dev, md_ctrl->hw_info.phy_interrupt_id);
542 }
543 
544 static void t7xx_cldma_disable_irq(struct cldma_ctrl *md_ctrl)
545 {
546 	t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, md_ctrl->hw_info.phy_interrupt_id);
547 }
548 
549 static void t7xx_cldma_irq_work_cb(struct cldma_ctrl *md_ctrl)
550 {
551 	unsigned long l2_tx_int_msk, l2_rx_int_msk, l2_tx_int, l2_rx_int, val;
552 	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
553 	int i;
554 
555 	/* L2 raw interrupt status */
556 	l2_tx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TISAR0);
557 	l2_rx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2RISAR0);
558 	l2_tx_int_msk = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TIMR0);
559 	l2_rx_int_msk = ioread32(hw_info->ap_ao_base + REG_CLDMA_L2RIMR0);
560 	l2_tx_int &= ~l2_tx_int_msk;
561 	l2_rx_int &= ~l2_rx_int_msk;
562 
563 	if (l2_tx_int) {
564 		if (l2_tx_int & (TQ_ERR_INT_BITMASK | TQ_ACTIVE_START_ERR_INT_BITMASK)) {
565 			/* Read and clear L3 TX interrupt status */
566 			val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0);
567 			iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0);
568 			val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1);
569 			iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1);
570 		}
571 
572 		t7xx_cldma_hw_tx_done(hw_info, l2_tx_int);
573 		if (l2_tx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) {
574 			for_each_set_bit(i, &l2_tx_int, L2_INT_BIT_COUNT) {
575 				if (i < CLDMA_TXQ_NUM) {
576 					pm_runtime_get(md_ctrl->dev);
577 					t7xx_cldma_hw_irq_dis_eq(hw_info, i, MTK_TX);
578 					t7xx_cldma_hw_irq_dis_txrx(hw_info, i, MTK_TX);
579 					queue_work(md_ctrl->txq[i].worker,
580 						   &md_ctrl->txq[i].cldma_work);
581 				} else {
582 					t7xx_cldma_txq_empty_hndl(&md_ctrl->txq[i - CLDMA_TXQ_NUM]);
583 				}
584 			}
585 		}
586 	}
587 
588 	if (l2_rx_int) {
589 		if (l2_rx_int & (RQ_ERR_INT_BITMASK | RQ_ACTIVE_START_ERR_INT_BITMASK)) {
590 			/* Read and clear L3 RX interrupt status */
591 			val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0);
592 			iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0);
593 			val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1);
594 			iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1);
595 		}
596 
597 		t7xx_cldma_hw_rx_done(hw_info, l2_rx_int);
598 		if (l2_rx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) {
599 			l2_rx_int |= l2_rx_int >> CLDMA_RXQ_NUM;
600 			for_each_set_bit(i, &l2_rx_int, CLDMA_RXQ_NUM) {
601 				pm_runtime_get(md_ctrl->dev);
602 				t7xx_cldma_hw_irq_dis_eq(hw_info, i, MTK_RX);
603 				t7xx_cldma_hw_irq_dis_txrx(hw_info, i, MTK_RX);
604 				queue_work(md_ctrl->rxq[i].worker, &md_ctrl->rxq[i].cldma_work);
605 			}
606 		}
607 	}
608 }
609 
610 static bool t7xx_cldma_qs_are_active(struct cldma_ctrl *md_ctrl)
611 {
612 	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
613 	unsigned int tx_active;
614 	unsigned int rx_active;
615 
616 	if (!pci_device_is_present(to_pci_dev(md_ctrl->dev)))
617 		return false;
618 
619 	tx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, MTK_TX);
620 	rx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, MTK_RX);
621 
622 	return tx_active || rx_active;
623 }
624 
625 /**
626  * t7xx_cldma_stop() - Stop CLDMA.
627  * @md_ctrl: CLDMA context structure.
628  *
629  * Stop TX and RX queues. Disable L1 and L2 interrupts.
630  * Clear status registers.
631  *
632  * Return:
633  * * 0		- Success.
634  * * -ERROR	- Error code from polling cldma_queues_active.
635  */
636 int t7xx_cldma_stop(struct cldma_ctrl *md_ctrl)
637 {
638 	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
639 	bool active;
640 	int i, ret;
641 
642 	md_ctrl->rxq_active = 0;
643 	t7xx_cldma_hw_stop_all_qs(hw_info, MTK_RX);
644 	md_ctrl->txq_active = 0;
645 	t7xx_cldma_hw_stop_all_qs(hw_info, MTK_TX);
646 	md_ctrl->txq_started = 0;
647 	t7xx_cldma_disable_irq(md_ctrl);
648 	t7xx_cldma_hw_stop(hw_info, MTK_RX);
649 	t7xx_cldma_hw_stop(hw_info, MTK_TX);
650 	t7xx_cldma_hw_tx_done(hw_info, CLDMA_L2TISAR0_ALL_INT_MASK);
651 	t7xx_cldma_hw_rx_done(hw_info, CLDMA_L2RISAR0_ALL_INT_MASK);
652 
653 	if (md_ctrl->is_late_init) {
654 		for (i = 0; i < CLDMA_TXQ_NUM; i++)
655 			flush_work(&md_ctrl->txq[i].cldma_work);
656 
657 		for (i = 0; i < CLDMA_RXQ_NUM; i++)
658 			flush_work(&md_ctrl->rxq[i].cldma_work);
659 	}
660 
661 	ret = read_poll_timeout(t7xx_cldma_qs_are_active, active, !active, CHECK_Q_STOP_STEP_US,
662 				CHECK_Q_STOP_TIMEOUT_US, true, md_ctrl);
663 	if (ret)
664 		dev_err(md_ctrl->dev, "Could not stop CLDMA%d queues", md_ctrl->hif_id);
665 
666 	return ret;
667 }
668 
669 static void t7xx_cldma_late_release(struct cldma_ctrl *md_ctrl)
670 {
671 	int i;
672 
673 	if (!md_ctrl->is_late_init)
674 		return;
675 
676 	for (i = 0; i < CLDMA_TXQ_NUM; i++)
677 		t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE);
678 
679 	for (i = 0; i < CLDMA_RXQ_NUM; i++)
680 		t7xx_cldma_ring_free(md_ctrl, &md_ctrl->rx_ring[i], DMA_FROM_DEVICE);
681 
682 	dma_pool_destroy(md_ctrl->gpd_dmapool);
683 	md_ctrl->gpd_dmapool = NULL;
684 	md_ctrl->is_late_init = false;
685 }
686 
687 void t7xx_cldma_reset(struct cldma_ctrl *md_ctrl)
688 {
689 	unsigned long flags;
690 	int i;
691 
692 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
693 	md_ctrl->txq_active = 0;
694 	md_ctrl->rxq_active = 0;
695 	t7xx_cldma_disable_irq(md_ctrl);
696 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
697 
698 	for (i = 0; i < CLDMA_TXQ_NUM; i++) {
699 		cancel_work_sync(&md_ctrl->txq[i].cldma_work);
700 
701 		spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
702 		md_cd_queue_struct_reset(&md_ctrl->txq[i], md_ctrl, MTK_TX, i);
703 		spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
704 	}
705 
706 	for (i = 0; i < CLDMA_RXQ_NUM; i++) {
707 		cancel_work_sync(&md_ctrl->rxq[i].cldma_work);
708 
709 		spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
710 		md_cd_queue_struct_reset(&md_ctrl->rxq[i], md_ctrl, MTK_RX, i);
711 		spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
712 	}
713 
714 	t7xx_cldma_late_release(md_ctrl);
715 }
716 
717 /**
718  * t7xx_cldma_start() - Start CLDMA.
719  * @md_ctrl: CLDMA context structure.
720  *
721  * Set TX/RX start address.
722  * Start all RX queues and enable L2 interrupt.
723  */
724 void t7xx_cldma_start(struct cldma_ctrl *md_ctrl)
725 {
726 	unsigned long flags;
727 
728 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
729 	if (md_ctrl->is_late_init) {
730 		struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
731 		int i;
732 
733 		t7xx_cldma_enable_irq(md_ctrl);
734 
735 		for (i = 0; i < CLDMA_TXQ_NUM; i++) {
736 			if (md_ctrl->txq[i].tr_done)
737 				t7xx_cldma_hw_set_start_addr(hw_info, i,
738 							     md_ctrl->txq[i].tr_done->gpd_addr,
739 							     MTK_TX);
740 		}
741 
742 		for (i = 0; i < CLDMA_RXQ_NUM; i++) {
743 			if (md_ctrl->rxq[i].tr_done)
744 				t7xx_cldma_hw_set_start_addr(hw_info, i,
745 							     md_ctrl->rxq[i].tr_done->gpd_addr,
746 							     MTK_RX);
747 		}
748 
749 		/* Enable L2 interrupt */
750 		t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, MTK_RX);
751 		t7xx_cldma_hw_start(hw_info);
752 		md_ctrl->txq_started = 0;
753 		md_ctrl->txq_active |= TXRX_STATUS_BITMASK;
754 		md_ctrl->rxq_active |= TXRX_STATUS_BITMASK;
755 	}
756 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
757 }
758 
759 static void t7xx_cldma_clear_txq(struct cldma_ctrl *md_ctrl, int qnum)
760 {
761 	struct cldma_queue *txq = &md_ctrl->txq[qnum];
762 	struct cldma_request *req;
763 	struct cldma_gpd *gpd;
764 	unsigned long flags;
765 
766 	spin_lock_irqsave(&txq->ring_lock, flags);
767 	t7xx_cldma_q_reset(txq);
768 	list_for_each_entry(req, &txq->tr_ring->gpd_ring, entry) {
769 		gpd = req->gpd;
770 		gpd->flags &= ~GPD_FLAGS_HWO;
771 		t7xx_cldma_gpd_set_data_ptr(gpd, 0);
772 		gpd->data_buff_len = 0;
773 		dev_kfree_skb_any(req->skb);
774 		req->skb = NULL;
775 	}
776 	spin_unlock_irqrestore(&txq->ring_lock, flags);
777 }
778 
779 static int t7xx_cldma_clear_rxq(struct cldma_ctrl *md_ctrl, int qnum)
780 {
781 	struct cldma_queue *rxq = &md_ctrl->rxq[qnum];
782 	struct cldma_request *req;
783 	struct cldma_gpd *gpd;
784 	unsigned long flags;
785 	int ret = 0;
786 
787 	spin_lock_irqsave(&rxq->ring_lock, flags);
788 	t7xx_cldma_q_reset(rxq);
789 	list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) {
790 		gpd = req->gpd;
791 		gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
792 		gpd->data_buff_len = 0;
793 
794 		if (req->skb) {
795 			req->skb->len = 0;
796 			skb_reset_tail_pointer(req->skb);
797 		}
798 	}
799 
800 	list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) {
801 		if (req->skb)
802 			continue;
803 
804 		ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, rxq->tr_ring->pkt_size, GFP_ATOMIC);
805 		if (ret)
806 			break;
807 
808 		t7xx_cldma_gpd_set_data_ptr(req->gpd, req->mapped_buff);
809 	}
810 	spin_unlock_irqrestore(&rxq->ring_lock, flags);
811 
812 	return ret;
813 }
814 
815 void t7xx_cldma_clear_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx)
816 {
817 	int i;
818 
819 	if (tx_rx == MTK_TX) {
820 		for (i = 0; i < CLDMA_TXQ_NUM; i++)
821 			t7xx_cldma_clear_txq(md_ctrl, i);
822 	} else {
823 		for (i = 0; i < CLDMA_RXQ_NUM; i++)
824 			t7xx_cldma_clear_rxq(md_ctrl, i);
825 	}
826 }
827 
828 void t7xx_cldma_stop_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx)
829 {
830 	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
831 	unsigned long flags;
832 
833 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
834 	t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, tx_rx);
835 	t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, tx_rx);
836 	if (tx_rx == MTK_RX)
837 		md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK;
838 	else
839 		md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK;
840 	t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx);
841 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
842 }
843 
844 static int t7xx_cldma_gpd_handle_tx_request(struct cldma_queue *queue, struct cldma_request *tx_req,
845 					    struct sk_buff *skb)
846 {
847 	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
848 	struct cldma_gpd *gpd = tx_req->gpd;
849 	unsigned long flags;
850 
851 	/* Update GPD */
852 	tx_req->mapped_buff = dma_map_single(md_ctrl->dev, skb->data, skb->len, DMA_TO_DEVICE);
853 
854 	if (dma_mapping_error(md_ctrl->dev, tx_req->mapped_buff)) {
855 		dev_err(md_ctrl->dev, "DMA mapping failed\n");
856 		return -ENOMEM;
857 	}
858 
859 	t7xx_cldma_gpd_set_data_ptr(gpd, tx_req->mapped_buff);
860 	gpd->data_buff_len = cpu_to_le16(skb->len);
861 
862 	/* This lock must cover TGPD setting, as even without a resume operation,
863 	 * CLDMA can send next HWO=1 if last TGPD just finished.
864 	 */
865 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
866 	if (md_ctrl->txq_active & BIT(queue->index))
867 		gpd->flags |= GPD_FLAGS_HWO;
868 
869 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
870 
871 	tx_req->skb = skb;
872 	return 0;
873 }
874 
875 /* Called with cldma_lock */
876 static void t7xx_cldma_hw_start_send(struct cldma_ctrl *md_ctrl, int qno,
877 				     struct cldma_request *prev_req)
878 {
879 	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
880 
881 	/* Check whether the device was powered off (CLDMA start address is not set) */
882 	if (!t7xx_cldma_tx_addr_is_set(hw_info, qno)) {
883 		t7xx_cldma_hw_init(hw_info);
884 		t7xx_cldma_hw_set_start_addr(hw_info, qno, prev_req->gpd_addr, MTK_TX);
885 		md_ctrl->txq_started &= ~BIT(qno);
886 	}
887 
888 	if (!t7xx_cldma_hw_queue_status(hw_info, qno, MTK_TX)) {
889 		if (md_ctrl->txq_started & BIT(qno))
890 			t7xx_cldma_hw_resume_queue(hw_info, qno, MTK_TX);
891 		else
892 			t7xx_cldma_hw_start_queue(hw_info, qno, MTK_TX);
893 
894 		md_ctrl->txq_started |= BIT(qno);
895 	}
896 }
897 
898 /**
899  * t7xx_cldma_set_recv_skb() - Set the callback to handle RX packets.
900  * @md_ctrl: CLDMA context structure.
901  * @recv_skb: Receiving skb callback.
902  */
903 void t7xx_cldma_set_recv_skb(struct cldma_ctrl *md_ctrl,
904 			     int (*recv_skb)(struct cldma_queue *queue, struct sk_buff *skb))
905 {
906 	md_ctrl->recv_skb = recv_skb;
907 }
908 
909 /**
910  * t7xx_cldma_send_skb() - Send control data to modem.
911  * @md_ctrl: CLDMA context structure.
912  * @qno: Queue number.
913  * @skb: Socket buffer.
914  *
915  * Return:
916  * * 0		- Success.
917  * * -ENOMEM	- Allocation failure.
918  * * -EINVAL	- Invalid queue request.
919  * * -EIO	- Queue is not active.
920  * * -ETIMEDOUT	- Timeout waiting for the device to wake up.
921  */
922 int t7xx_cldma_send_skb(struct cldma_ctrl *md_ctrl, int qno, struct sk_buff *skb)
923 {
924 	struct cldma_request *tx_req;
925 	struct cldma_queue *queue;
926 	unsigned long flags;
927 	int ret;
928 
929 	if (qno >= CLDMA_TXQ_NUM)
930 		return -EINVAL;
931 
932 	ret = pm_runtime_resume_and_get(md_ctrl->dev);
933 	if (ret < 0 && ret != -EACCES)
934 		return ret;
935 
936 	t7xx_pci_disable_sleep(md_ctrl->t7xx_dev);
937 	queue = &md_ctrl->txq[qno];
938 
939 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
940 	if (!(md_ctrl->txq_active & BIT(qno))) {
941 		ret = -EIO;
942 		spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
943 		goto allow_sleep;
944 	}
945 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
946 
947 	do {
948 		spin_lock_irqsave(&queue->ring_lock, flags);
949 		tx_req = queue->tx_next;
950 		if (queue->budget > 0 && !tx_req->skb) {
951 			struct list_head *gpd_ring = &queue->tr_ring->gpd_ring;
952 
953 			queue->budget--;
954 			t7xx_cldma_gpd_handle_tx_request(queue, tx_req, skb);
955 			queue->tx_next = list_next_entry_circular(tx_req, gpd_ring, entry);
956 			spin_unlock_irqrestore(&queue->ring_lock, flags);
957 
958 			if (!t7xx_pci_sleep_disable_complete(md_ctrl->t7xx_dev)) {
959 				ret = -ETIMEDOUT;
960 				break;
961 			}
962 
963 			/* Protect the access to the modem for queues operations (resume/start)
964 			 * which access shared locations by all the queues.
965 			 * cldma_lock is independent of ring_lock which is per queue.
966 			 */
967 			spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
968 			t7xx_cldma_hw_start_send(md_ctrl, qno, tx_req);
969 			spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
970 
971 			break;
972 		}
973 		spin_unlock_irqrestore(&queue->ring_lock, flags);
974 
975 		if (!t7xx_pci_sleep_disable_complete(md_ctrl->t7xx_dev)) {
976 			ret = -ETIMEDOUT;
977 			break;
978 		}
979 
980 		if (!t7xx_cldma_hw_queue_status(&md_ctrl->hw_info, qno, MTK_TX)) {
981 			spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
982 			t7xx_cldma_hw_resume_queue(&md_ctrl->hw_info, qno, MTK_TX);
983 			spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
984 		}
985 
986 		ret = wait_event_interruptible_exclusive(queue->req_wq, queue->budget > 0);
987 	} while (!ret);
988 
989 allow_sleep:
990 	t7xx_pci_enable_sleep(md_ctrl->t7xx_dev);
991 	pm_runtime_mark_last_busy(md_ctrl->dev);
992 	pm_runtime_put_autosuspend(md_ctrl->dev);
993 	return ret;
994 }
995 
996 static int t7xx_cldma_late_init(struct cldma_ctrl *md_ctrl)
997 {
998 	char dma_pool_name[32];
999 	int i, j, ret;
1000 
1001 	if (md_ctrl->is_late_init) {
1002 		dev_err(md_ctrl->dev, "CLDMA late init was already done\n");
1003 		return -EALREADY;
1004 	}
1005 
1006 	snprintf(dma_pool_name, sizeof(dma_pool_name), "cldma_req_hif%d", md_ctrl->hif_id);
1007 
1008 	md_ctrl->gpd_dmapool = dma_pool_create(dma_pool_name, md_ctrl->dev,
1009 					       sizeof(struct cldma_gpd), GPD_DMAPOOL_ALIGN, 0);
1010 	if (!md_ctrl->gpd_dmapool) {
1011 		dev_err(md_ctrl->dev, "DMA pool alloc fail\n");
1012 		return -ENOMEM;
1013 	}
1014 
1015 	for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1016 		ret = t7xx_cldma_tx_ring_init(md_ctrl, &md_ctrl->tx_ring[i]);
1017 		if (ret) {
1018 			dev_err(md_ctrl->dev, "control TX ring init fail\n");
1019 			goto err_free_tx_ring;
1020 		}
1021 
1022 		md_ctrl->tx_ring[i].pkt_size = CLDMA_MTU;
1023 	}
1024 
1025 	for (j = 0; j < CLDMA_RXQ_NUM; j++) {
1026 		md_ctrl->rx_ring[j].pkt_size = CLDMA_MTU;
1027 
1028 		if (j == CLDMA_RXQ_NUM - 1)
1029 			md_ctrl->rx_ring[j].pkt_size = CLDMA_JUMBO_BUFF_SZ;
1030 
1031 		ret = t7xx_cldma_rx_ring_init(md_ctrl, &md_ctrl->rx_ring[j]);
1032 		if (ret) {
1033 			dev_err(md_ctrl->dev, "Control RX ring init fail\n");
1034 			goto err_free_rx_ring;
1035 		}
1036 	}
1037 
1038 	for (i = 0; i < CLDMA_TXQ_NUM; i++)
1039 		t7xx_cldma_txq_init(&md_ctrl->txq[i]);
1040 
1041 	for (j = 0; j < CLDMA_RXQ_NUM; j++)
1042 		t7xx_cldma_rxq_init(&md_ctrl->rxq[j]);
1043 
1044 	md_ctrl->is_late_init = true;
1045 	return 0;
1046 
1047 err_free_rx_ring:
1048 	while (j--)
1049 		t7xx_cldma_ring_free(md_ctrl, &md_ctrl->rx_ring[j], DMA_FROM_DEVICE);
1050 
1051 err_free_tx_ring:
1052 	while (i--)
1053 		t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE);
1054 
1055 	return ret;
1056 }
1057 
1058 static void __iomem *t7xx_pcie_addr_transfer(void __iomem *addr, u32 addr_trs1, u32 phy_addr)
1059 {
1060 	return addr + phy_addr - addr_trs1;
1061 }
1062 
1063 static void t7xx_hw_info_init(struct cldma_ctrl *md_ctrl)
1064 {
1065 	struct t7xx_addr_base *pbase = &md_ctrl->t7xx_dev->base_addr;
1066 	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1067 	u32 phy_ao_base, phy_pd_base;
1068 
1069 	if (md_ctrl->hif_id != CLDMA_ID_MD)
1070 		return;
1071 
1072 	phy_ao_base = CLDMA1_AO_BASE;
1073 	phy_pd_base = CLDMA1_PD_BASE;
1074 	hw_info->phy_interrupt_id = CLDMA1_INT;
1075 	hw_info->hw_mode = MODE_BIT_64;
1076 	hw_info->ap_ao_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base,
1077 						      pbase->pcie_dev_reg_trsl_addr, phy_ao_base);
1078 	hw_info->ap_pdn_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base,
1079 						       pbase->pcie_dev_reg_trsl_addr, phy_pd_base);
1080 }
1081 
1082 static int t7xx_cldma_default_recv_skb(struct cldma_queue *queue, struct sk_buff *skb)
1083 {
1084 	dev_kfree_skb_any(skb);
1085 	return 0;
1086 }
1087 
1088 int t7xx_cldma_alloc(enum cldma_id hif_id, struct t7xx_pci_dev *t7xx_dev)
1089 {
1090 	struct device *dev = &t7xx_dev->pdev->dev;
1091 	struct cldma_ctrl *md_ctrl;
1092 
1093 	md_ctrl = devm_kzalloc(dev, sizeof(*md_ctrl), GFP_KERNEL);
1094 	if (!md_ctrl)
1095 		return -ENOMEM;
1096 
1097 	md_ctrl->t7xx_dev = t7xx_dev;
1098 	md_ctrl->dev = dev;
1099 	md_ctrl->hif_id = hif_id;
1100 	md_ctrl->recv_skb = t7xx_cldma_default_recv_skb;
1101 	t7xx_hw_info_init(md_ctrl);
1102 	t7xx_dev->md->md_ctrl[hif_id] = md_ctrl;
1103 	return 0;
1104 }
1105 
1106 static void t7xx_cldma_resume_early(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1107 {
1108 	struct cldma_ctrl *md_ctrl = entity_param;
1109 	struct t7xx_cldma_hw *hw_info;
1110 	unsigned long flags;
1111 	int qno_t;
1112 
1113 	hw_info = &md_ctrl->hw_info;
1114 
1115 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1116 	t7xx_cldma_hw_restore(hw_info);
1117 	for (qno_t = 0; qno_t < CLDMA_TXQ_NUM; qno_t++) {
1118 		t7xx_cldma_hw_set_start_addr(hw_info, qno_t, md_ctrl->txq[qno_t].tx_next->gpd_addr,
1119 					     MTK_TX);
1120 		t7xx_cldma_hw_set_start_addr(hw_info, qno_t, md_ctrl->rxq[qno_t].tr_done->gpd_addr,
1121 					     MTK_RX);
1122 	}
1123 	t7xx_cldma_enable_irq(md_ctrl);
1124 	t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, MTK_RX);
1125 	md_ctrl->rxq_active |= TXRX_STATUS_BITMASK;
1126 	t7xx_cldma_hw_irq_en_eq(hw_info, CLDMA_ALL_Q, MTK_RX);
1127 	t7xx_cldma_hw_irq_en_txrx(hw_info, CLDMA_ALL_Q, MTK_RX);
1128 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1129 }
1130 
1131 static int t7xx_cldma_resume(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1132 {
1133 	struct cldma_ctrl *md_ctrl = entity_param;
1134 	unsigned long flags;
1135 
1136 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1137 	md_ctrl->txq_active |= TXRX_STATUS_BITMASK;
1138 	t7xx_cldma_hw_irq_en_txrx(&md_ctrl->hw_info, CLDMA_ALL_Q, MTK_TX);
1139 	t7xx_cldma_hw_irq_en_eq(&md_ctrl->hw_info, CLDMA_ALL_Q, MTK_TX);
1140 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1141 
1142 	if (md_ctrl->hif_id == CLDMA_ID_MD)
1143 		t7xx_mhccif_mask_clr(t7xx_dev, D2H_SW_INT_MASK);
1144 
1145 	return 0;
1146 }
1147 
1148 static void t7xx_cldma_suspend_late(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1149 {
1150 	struct cldma_ctrl *md_ctrl = entity_param;
1151 	struct t7xx_cldma_hw *hw_info;
1152 	unsigned long flags;
1153 
1154 	hw_info = &md_ctrl->hw_info;
1155 
1156 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1157 	t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, MTK_RX);
1158 	t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, MTK_RX);
1159 	md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK;
1160 	t7xx_cldma_hw_stop_all_qs(hw_info, MTK_RX);
1161 	t7xx_cldma_clear_ip_busy(hw_info);
1162 	t7xx_cldma_disable_irq(md_ctrl);
1163 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1164 }
1165 
1166 static int t7xx_cldma_suspend(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1167 {
1168 	struct cldma_ctrl *md_ctrl = entity_param;
1169 	struct t7xx_cldma_hw *hw_info;
1170 	unsigned long flags;
1171 
1172 	if (md_ctrl->hif_id == CLDMA_ID_MD)
1173 		t7xx_mhccif_mask_set(t7xx_dev, D2H_SW_INT_MASK);
1174 
1175 	hw_info = &md_ctrl->hw_info;
1176 
1177 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1178 	t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, MTK_TX);
1179 	t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, MTK_TX);
1180 	md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK;
1181 	t7xx_cldma_hw_stop_all_qs(hw_info, MTK_TX);
1182 	md_ctrl->txq_started = 0;
1183 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1184 
1185 	return 0;
1186 }
1187 
1188 static int t7xx_cldma_pm_init(struct cldma_ctrl *md_ctrl)
1189 {
1190 	md_ctrl->pm_entity = kzalloc(sizeof(*md_ctrl->pm_entity), GFP_KERNEL);
1191 	if (!md_ctrl->pm_entity)
1192 		return -ENOMEM;
1193 
1194 	md_ctrl->pm_entity->entity_param = md_ctrl;
1195 
1196 	if (md_ctrl->hif_id == CLDMA_ID_MD)
1197 		md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL1;
1198 	else
1199 		md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL2;
1200 
1201 	md_ctrl->pm_entity->suspend = t7xx_cldma_suspend;
1202 	md_ctrl->pm_entity->suspend_late = t7xx_cldma_suspend_late;
1203 	md_ctrl->pm_entity->resume = t7xx_cldma_resume;
1204 	md_ctrl->pm_entity->resume_early = t7xx_cldma_resume_early;
1205 
1206 	return t7xx_pci_pm_entity_register(md_ctrl->t7xx_dev, md_ctrl->pm_entity);
1207 }
1208 
1209 static int t7xx_cldma_pm_uninit(struct cldma_ctrl *md_ctrl)
1210 {
1211 	if (!md_ctrl->pm_entity)
1212 		return -EINVAL;
1213 
1214 	t7xx_pci_pm_entity_unregister(md_ctrl->t7xx_dev, md_ctrl->pm_entity);
1215 	kfree(md_ctrl->pm_entity);
1216 	md_ctrl->pm_entity = NULL;
1217 	return 0;
1218 }
1219 
1220 void t7xx_cldma_hif_hw_init(struct cldma_ctrl *md_ctrl)
1221 {
1222 	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1223 	unsigned long flags;
1224 
1225 	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1226 	t7xx_cldma_hw_stop(hw_info, MTK_TX);
1227 	t7xx_cldma_hw_stop(hw_info, MTK_RX);
1228 	t7xx_cldma_hw_rx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK);
1229 	t7xx_cldma_hw_tx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK);
1230 	t7xx_cldma_hw_init(hw_info);
1231 	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1232 }
1233 
1234 static irqreturn_t t7xx_cldma_isr_handler(int irq, void *data)
1235 {
1236 	struct cldma_ctrl *md_ctrl = data;
1237 	u32 interrupt;
1238 
1239 	interrupt = md_ctrl->hw_info.phy_interrupt_id;
1240 	t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, interrupt);
1241 	t7xx_cldma_irq_work_cb(md_ctrl);
1242 	t7xx_pcie_mac_clear_int_status(md_ctrl->t7xx_dev, interrupt);
1243 	t7xx_pcie_mac_set_int(md_ctrl->t7xx_dev, interrupt);
1244 	return IRQ_HANDLED;
1245 }
1246 
1247 static void t7xx_cldma_destroy_wqs(struct cldma_ctrl *md_ctrl)
1248 {
1249 	int i;
1250 
1251 	for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1252 		if (md_ctrl->txq[i].worker) {
1253 			destroy_workqueue(md_ctrl->txq[i].worker);
1254 			md_ctrl->txq[i].worker = NULL;
1255 		}
1256 	}
1257 
1258 	for (i = 0; i < CLDMA_RXQ_NUM; i++) {
1259 		if (md_ctrl->rxq[i].worker) {
1260 			destroy_workqueue(md_ctrl->rxq[i].worker);
1261 			md_ctrl->rxq[i].worker = NULL;
1262 		}
1263 	}
1264 }
1265 
1266 /**
1267  * t7xx_cldma_init() - Initialize CLDMA.
1268  * @md_ctrl: CLDMA context structure.
1269  *
1270  * Allocate and initialize device power management entity.
1271  * Initialize HIF TX/RX queue structure.
1272  * Register CLDMA callback ISR with PCIe driver.
1273  *
1274  * Return:
1275  * * 0		- Success.
1276  * * -ERROR	- Error code from failure sub-initializations.
1277  */
1278 int t7xx_cldma_init(struct cldma_ctrl *md_ctrl)
1279 {
1280 	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1281 	int ret, i;
1282 
1283 	md_ctrl->txq_active = 0;
1284 	md_ctrl->rxq_active = 0;
1285 	md_ctrl->is_late_init = false;
1286 
1287 	ret = t7xx_cldma_pm_init(md_ctrl);
1288 	if (ret)
1289 		return ret;
1290 
1291 	spin_lock_init(&md_ctrl->cldma_lock);
1292 
1293 	for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1294 		md_cd_queue_struct_init(&md_ctrl->txq[i], md_ctrl, MTK_TX, i);
1295 		md_ctrl->txq[i].worker =
1296 			alloc_ordered_workqueue("md_hif%d_tx%d_worker",
1297 					WQ_MEM_RECLAIM | (i ? 0 : WQ_HIGHPRI),
1298 					md_ctrl->hif_id, i);
1299 		if (!md_ctrl->txq[i].worker)
1300 			goto err_workqueue;
1301 
1302 		INIT_WORK(&md_ctrl->txq[i].cldma_work, t7xx_cldma_tx_done);
1303 	}
1304 
1305 	for (i = 0; i < CLDMA_RXQ_NUM; i++) {
1306 		md_cd_queue_struct_init(&md_ctrl->rxq[i], md_ctrl, MTK_RX, i);
1307 		INIT_WORK(&md_ctrl->rxq[i].cldma_work, t7xx_cldma_rx_done);
1308 
1309 		md_ctrl->rxq[i].worker =
1310 			alloc_ordered_workqueue("md_hif%d_rx%d_worker",
1311 						WQ_MEM_RECLAIM,
1312 						md_ctrl->hif_id, i);
1313 		if (!md_ctrl->rxq[i].worker)
1314 			goto err_workqueue;
1315 	}
1316 
1317 	t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, hw_info->phy_interrupt_id);
1318 	md_ctrl->t7xx_dev->intr_handler[hw_info->phy_interrupt_id] = t7xx_cldma_isr_handler;
1319 	md_ctrl->t7xx_dev->intr_thread[hw_info->phy_interrupt_id] = NULL;
1320 	md_ctrl->t7xx_dev->callback_param[hw_info->phy_interrupt_id] = md_ctrl;
1321 	t7xx_pcie_mac_clear_int_status(md_ctrl->t7xx_dev, hw_info->phy_interrupt_id);
1322 	return 0;
1323 
1324 err_workqueue:
1325 	t7xx_cldma_destroy_wqs(md_ctrl);
1326 	t7xx_cldma_pm_uninit(md_ctrl);
1327 	return -ENOMEM;
1328 }
1329 
1330 void t7xx_cldma_switch_cfg(struct cldma_ctrl *md_ctrl)
1331 {
1332 	t7xx_cldma_late_release(md_ctrl);
1333 	t7xx_cldma_late_init(md_ctrl);
1334 }
1335 
1336 void t7xx_cldma_exit(struct cldma_ctrl *md_ctrl)
1337 {
1338 	t7xx_cldma_stop(md_ctrl);
1339 	t7xx_cldma_late_release(md_ctrl);
1340 	t7xx_cldma_destroy_wqs(md_ctrl);
1341 	t7xx_cldma_pm_uninit(md_ctrl);
1342 }
1343