1 // SPDX-License-Identifier: ISC
2 /* Copyright (C) 2020 MediaTek Inc. */
3 
4 #include "mt7915.h"
5 #include "../dma.h"
6 #include "mac.h"
7 
8 static int
9 mt7915_init_tx_queues(struct mt7915_phy *phy, int idx, int n_desc, int ring_base)
10 {
11 	struct mt7915_dev *dev = phy->dev;
12 
13 	if (mtk_wed_device_active(&phy->dev->mt76.mmio.wed)) {
14 		ring_base = MT_WED_TX_RING_BASE;
15 		idx -= MT_TXQ_ID(0);
16 	}
17 
18 	return mt76_connac_init_tx_queues(phy->mt76, idx, n_desc, ring_base,
19 					  MT_WED_Q_TX(idx));
20 }
21 
22 static int mt7915_poll_tx(struct napi_struct *napi, int budget)
23 {
24 	struct mt7915_dev *dev;
25 
26 	dev = container_of(napi, struct mt7915_dev, mt76.tx_napi);
27 
28 	mt76_connac_tx_cleanup(&dev->mt76);
29 	if (napi_complete_done(napi, 0))
30 		mt7915_irq_enable(dev, MT_INT_TX_DONE_MCU);
31 
32 	return 0;
33 }
34 
35 static void mt7915_dma_config(struct mt7915_dev *dev)
36 {
37 #define Q_CONFIG(q, wfdma, int, id) do {		\
38 		if (wfdma)				\
39 			dev->wfdma_mask |= (1 << (q));	\
40 		dev->q_int_mask[(q)] = int;		\
41 		dev->q_id[(q)] = id;			\
42 	} while (0)
43 
44 #define MCUQ_CONFIG(q, wfdma, int, id)	Q_CONFIG(q, (wfdma), (int), (id))
45 #define RXQ_CONFIG(q, wfdma, int, id)	Q_CONFIG(__RXQ(q), (wfdma), (int), (id))
46 #define TXQ_CONFIG(q, wfdma, int, id)	Q_CONFIG(__TXQ(q), (wfdma), (int), (id))
47 
48 	if (is_mt7915(&dev->mt76)) {
49 		RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0, MT7915_RXQ_BAND0);
50 		RXQ_CONFIG(MT_RXQ_MCU, WFDMA1, MT_INT_RX_DONE_WM, MT7915_RXQ_MCU_WM);
51 		RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA1, MT_INT_RX_DONE_WA, MT7915_RXQ_MCU_WA);
52 		RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1, MT7915_RXQ_BAND1);
53 		RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA1, MT_INT_RX_DONE_WA_EXT, MT7915_RXQ_MCU_WA_EXT);
54 		RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA1, MT_INT_RX_DONE_WA_MAIN, MT7915_RXQ_MCU_WA);
55 		TXQ_CONFIG(0, WFDMA1, MT_INT_TX_DONE_BAND0, MT7915_TXQ_BAND0);
56 		TXQ_CONFIG(1, WFDMA1, MT_INT_TX_DONE_BAND1, MT7915_TXQ_BAND1);
57 		MCUQ_CONFIG(MT_MCUQ_WM, WFDMA1, MT_INT_TX_DONE_MCU_WM, MT7915_TXQ_MCU_WM);
58 		MCUQ_CONFIG(MT_MCUQ_WA, WFDMA1, MT_INT_TX_DONE_MCU_WA, MT7915_TXQ_MCU_WA);
59 		MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA1, MT_INT_TX_DONE_FWDL, MT7915_TXQ_FWDL);
60 	} else {
61 		RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0_MT7916, MT7916_RXQ_BAND0);
62 		RXQ_CONFIG(MT_RXQ_MCU, WFDMA0, MT_INT_RX_DONE_WM, MT7916_RXQ_MCU_WM);
63 		RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA0, MT_INT_RX_DONE_WA, MT7916_RXQ_MCU_WA);
64 		RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1_MT7916, MT7916_RXQ_BAND1);
65 		RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA0, MT_INT_RX_DONE_WA_EXT_MT7916, MT7916_RXQ_MCU_WA_EXT);
66 		RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_RX_DONE_WA_MAIN_MT7916, MT7916_RXQ_MCU_WA_MAIN);
67 		TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, MT7915_TXQ_BAND0);
68 		TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, MT7915_TXQ_BAND1);
69 		MCUQ_CONFIG(MT_MCUQ_WM, WFDMA0, MT_INT_TX_DONE_MCU_WM, MT7915_TXQ_MCU_WM);
70 		MCUQ_CONFIG(MT_MCUQ_WA, WFDMA0, MT_INT_TX_DONE_MCU_WA_MT7916, MT7915_TXQ_MCU_WA);
71 		MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA0, MT_INT_TX_DONE_FWDL, MT7915_TXQ_FWDL);
72 	}
73 }
74 
75 static void __mt7915_dma_prefetch(struct mt7915_dev *dev, u32 ofs)
76 {
77 #define PREFETCH(_base, _depth)	((_base) << 16 | (_depth))
78 	u32 base = 0;
79 
80 	/* prefetch SRAM wrapping boundary for tx/rx ring. */
81 	mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_FWDL) + ofs, PREFETCH(0x0, 0x4));
82 	mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WM) + ofs, PREFETCH(0x40, 0x4));
83 	mt76_wr(dev, MT_TXQ_EXT_CTRL(0) + ofs, PREFETCH(0x80, 0x4));
84 	mt76_wr(dev, MT_TXQ_EXT_CTRL(1) + ofs, PREFETCH(0xc0, 0x4));
85 	mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WA) + ofs, PREFETCH(0x100, 0x4));
86 
87 	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU) + ofs,
88 		PREFETCH(0x140, 0x4));
89 	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU_WA) + ofs,
90 		PREFETCH(0x180, 0x4));
91 	if (!is_mt7915(&dev->mt76)) {
92 		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN_WA) + ofs,
93 			PREFETCH(0x1c0, 0x4));
94 		base = 0x40;
95 	}
96 	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_BAND1_WA) + ofs,
97 		PREFETCH(0x1c0 + base, 0x4));
98 	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN) + ofs,
99 		PREFETCH(0x200 + base, 0x4));
100 	mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_BAND1) + ofs,
101 		PREFETCH(0x240 + base, 0x4));
102 
103 	/* for mt7915, the ring which is next the last
104 	 * used ring must be initialized.
105 	 */
106 	if (is_mt7915(&dev->mt76)) {
107 		ofs += 0x4;
108 		mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WA) + ofs,
109 			PREFETCH(0x140, 0x0));
110 		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_BAND1_WA) + ofs,
111 			PREFETCH(0x200 + base, 0x0));
112 		mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_BAND1) + ofs,
113 			PREFETCH(0x280 + base, 0x0));
114 	}
115 }
116 
117 void mt7915_dma_prefetch(struct mt7915_dev *dev)
118 {
119 	__mt7915_dma_prefetch(dev, 0);
120 	if (dev->hif2)
121 		__mt7915_dma_prefetch(dev, MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0));
122 }
123 
124 static void mt7915_dma_disable(struct mt7915_dev *dev, bool rst)
125 {
126 	struct mt76_dev *mdev = &dev->mt76;
127 	u32 hif1_ofs = 0;
128 
129 	if (dev->hif2)
130 		hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0);
131 
132 	/* reset */
133 	if (rst) {
134 		mt76_clear(dev, MT_WFDMA0_RST,
135 			   MT_WFDMA0_RST_DMASHDL_ALL_RST |
136 			   MT_WFDMA0_RST_LOGIC_RST);
137 
138 		mt76_set(dev, MT_WFDMA0_RST,
139 			 MT_WFDMA0_RST_DMASHDL_ALL_RST |
140 			 MT_WFDMA0_RST_LOGIC_RST);
141 
142 		if (is_mt7915(mdev)) {
143 			mt76_clear(dev, MT_WFDMA1_RST,
144 				   MT_WFDMA1_RST_DMASHDL_ALL_RST |
145 				   MT_WFDMA1_RST_LOGIC_RST);
146 
147 			mt76_set(dev, MT_WFDMA1_RST,
148 				 MT_WFDMA1_RST_DMASHDL_ALL_RST |
149 				 MT_WFDMA1_RST_LOGIC_RST);
150 		}
151 
152 		if (dev->hif2) {
153 			mt76_clear(dev, MT_WFDMA0_RST + hif1_ofs,
154 				   MT_WFDMA0_RST_DMASHDL_ALL_RST |
155 				   MT_WFDMA0_RST_LOGIC_RST);
156 
157 			mt76_set(dev, MT_WFDMA0_RST + hif1_ofs,
158 				 MT_WFDMA0_RST_DMASHDL_ALL_RST |
159 				 MT_WFDMA0_RST_LOGIC_RST);
160 
161 			if (is_mt7915(mdev)) {
162 				mt76_clear(dev, MT_WFDMA1_RST + hif1_ofs,
163 					   MT_WFDMA1_RST_DMASHDL_ALL_RST |
164 					   MT_WFDMA1_RST_LOGIC_RST);
165 
166 				mt76_set(dev, MT_WFDMA1_RST + hif1_ofs,
167 					 MT_WFDMA1_RST_DMASHDL_ALL_RST |
168 					 MT_WFDMA1_RST_LOGIC_RST);
169 			}
170 		}
171 	}
172 
173 	/* disable */
174 	mt76_clear(dev, MT_WFDMA0_GLO_CFG,
175 		   MT_WFDMA0_GLO_CFG_TX_DMA_EN |
176 		   MT_WFDMA0_GLO_CFG_RX_DMA_EN |
177 		   MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
178 		   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO |
179 		   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
180 
181 	if (is_mt7915(mdev))
182 		mt76_clear(dev, MT_WFDMA1_GLO_CFG,
183 			   MT_WFDMA1_GLO_CFG_TX_DMA_EN |
184 			   MT_WFDMA1_GLO_CFG_RX_DMA_EN |
185 			   MT_WFDMA1_GLO_CFG_OMIT_TX_INFO |
186 			   MT_WFDMA1_GLO_CFG_OMIT_RX_INFO |
187 			   MT_WFDMA1_GLO_CFG_OMIT_RX_INFO_PFET2);
188 
189 	if (dev->hif2) {
190 		mt76_clear(dev, MT_WFDMA0_GLO_CFG + hif1_ofs,
191 			   MT_WFDMA0_GLO_CFG_TX_DMA_EN |
192 			   MT_WFDMA0_GLO_CFG_RX_DMA_EN |
193 			   MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
194 			   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO |
195 			   MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
196 
197 		if (is_mt7915(mdev))
198 			mt76_clear(dev, MT_WFDMA1_GLO_CFG + hif1_ofs,
199 				   MT_WFDMA1_GLO_CFG_TX_DMA_EN |
200 				   MT_WFDMA1_GLO_CFG_RX_DMA_EN |
201 				   MT_WFDMA1_GLO_CFG_OMIT_TX_INFO |
202 				   MT_WFDMA1_GLO_CFG_OMIT_RX_INFO |
203 				   MT_WFDMA1_GLO_CFG_OMIT_RX_INFO_PFET2);
204 	}
205 }
206 
207 static int mt7915_dma_enable(struct mt7915_dev *dev)
208 {
209 	struct mt76_dev *mdev = &dev->mt76;
210 	u32 hif1_ofs = 0;
211 	u32 irq_mask;
212 
213 	if (dev->hif2)
214 		hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0);
215 
216 	/* reset dma idx */
217 	mt76_wr(dev, MT_WFDMA0_RST_DTX_PTR, ~0);
218 	if (is_mt7915(mdev))
219 		mt76_wr(dev, MT_WFDMA1_RST_DTX_PTR, ~0);
220 	if (dev->hif2) {
221 		mt76_wr(dev, MT_WFDMA0_RST_DTX_PTR + hif1_ofs, ~0);
222 		if (is_mt7915(mdev))
223 			mt76_wr(dev, MT_WFDMA1_RST_DTX_PTR + hif1_ofs, ~0);
224 	}
225 
226 	/* configure delay interrupt off */
227 	mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG0, 0);
228 	if (is_mt7915(mdev)) {
229 		mt76_wr(dev, MT_WFDMA1_PRI_DLY_INT_CFG0, 0);
230 	} else {
231 		mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG1, 0);
232 		mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG2, 0);
233 	}
234 
235 	if (dev->hif2) {
236 		mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG0 + hif1_ofs, 0);
237 		if (is_mt7915(mdev)) {
238 			mt76_wr(dev, MT_WFDMA1_PRI_DLY_INT_CFG0 +
239 				hif1_ofs, 0);
240 		} else {
241 			mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG1 +
242 				hif1_ofs, 0);
243 			mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG2 +
244 				hif1_ofs, 0);
245 		}
246 	}
247 
248 	/* configure perfetch settings */
249 	mt7915_dma_prefetch(dev);
250 
251 	/* hif wait WFDMA idle */
252 	mt76_set(dev, MT_WFDMA0_BUSY_ENA,
253 		 MT_WFDMA0_BUSY_ENA_TX_FIFO0 |
254 		 MT_WFDMA0_BUSY_ENA_TX_FIFO1 |
255 		 MT_WFDMA0_BUSY_ENA_RX_FIFO);
256 
257 	if (is_mt7915(mdev))
258 		mt76_set(dev, MT_WFDMA1_BUSY_ENA,
259 			 MT_WFDMA1_BUSY_ENA_TX_FIFO0 |
260 			 MT_WFDMA1_BUSY_ENA_TX_FIFO1 |
261 			 MT_WFDMA1_BUSY_ENA_RX_FIFO);
262 
263 	if (dev->hif2) {
264 		mt76_set(dev, MT_WFDMA0_BUSY_ENA + hif1_ofs,
265 			 MT_WFDMA0_PCIE1_BUSY_ENA_TX_FIFO0 |
266 			 MT_WFDMA0_PCIE1_BUSY_ENA_TX_FIFO1 |
267 			 MT_WFDMA0_PCIE1_BUSY_ENA_RX_FIFO);
268 
269 		if (is_mt7915(mdev))
270 			mt76_set(dev, MT_WFDMA1_BUSY_ENA + hif1_ofs,
271 				 MT_WFDMA1_PCIE1_BUSY_ENA_TX_FIFO0 |
272 				 MT_WFDMA1_PCIE1_BUSY_ENA_TX_FIFO1 |
273 				 MT_WFDMA1_PCIE1_BUSY_ENA_RX_FIFO);
274 	}
275 
276 	mt76_poll(dev, MT_WFDMA_EXT_CSR_HIF_MISC,
277 		  MT_WFDMA_EXT_CSR_HIF_MISC_BUSY, 0, 1000);
278 
279 	/* set WFDMA Tx/Rx */
280 	mt76_set(dev, MT_WFDMA0_GLO_CFG,
281 		 MT_WFDMA0_GLO_CFG_TX_DMA_EN |
282 		 MT_WFDMA0_GLO_CFG_RX_DMA_EN |
283 		 MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
284 		 MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
285 
286 	if (is_mt7915(mdev))
287 		mt76_set(dev, MT_WFDMA1_GLO_CFG,
288 			 MT_WFDMA1_GLO_CFG_TX_DMA_EN |
289 			 MT_WFDMA1_GLO_CFG_RX_DMA_EN |
290 			 MT_WFDMA1_GLO_CFG_OMIT_TX_INFO |
291 			 MT_WFDMA1_GLO_CFG_OMIT_RX_INFO);
292 
293 	if (dev->hif2) {
294 		mt76_set(dev, MT_WFDMA0_GLO_CFG + hif1_ofs,
295 			 MT_WFDMA0_GLO_CFG_TX_DMA_EN |
296 			 MT_WFDMA0_GLO_CFG_RX_DMA_EN |
297 			 MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
298 			 MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
299 
300 		if (is_mt7915(mdev))
301 			mt76_set(dev, MT_WFDMA1_GLO_CFG + hif1_ofs,
302 				 MT_WFDMA1_GLO_CFG_TX_DMA_EN |
303 				 MT_WFDMA1_GLO_CFG_RX_DMA_EN |
304 				 MT_WFDMA1_GLO_CFG_OMIT_TX_INFO |
305 				 MT_WFDMA1_GLO_CFG_OMIT_RX_INFO);
306 
307 		mt76_set(dev, MT_WFDMA_HOST_CONFIG,
308 			 MT_WFDMA_HOST_CONFIG_PDMA_BAND);
309 	}
310 
311 	/* enable interrupts for TX/RX rings */
312 	irq_mask = MT_INT_RX_DONE_MCU |
313 		   MT_INT_TX_DONE_MCU |
314 		   MT_INT_MCU_CMD;
315 
316 	if (!dev->phy.band_idx)
317 		irq_mask |= MT_INT_BAND0_RX_DONE;
318 
319 	if (dev->dbdc_support || dev->phy.band_idx)
320 		irq_mask |= MT_INT_BAND1_RX_DONE;
321 
322 	if (mtk_wed_device_active(&dev->mt76.mmio.wed)) {
323 		u32 wed_irq_mask = irq_mask;
324 
325 		wed_irq_mask |= MT_INT_TX_DONE_BAND0 | MT_INT_TX_DONE_BAND1;
326 		mt76_wr(dev, MT_INT_WED_MASK_CSR, wed_irq_mask);
327 		mtk_wed_device_start(&dev->mt76.mmio.wed, wed_irq_mask);
328 	}
329 
330 	mt7915_irq_enable(dev, irq_mask);
331 
332 	return 0;
333 }
334 
335 int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2)
336 {
337 	struct mt76_dev *mdev = &dev->mt76;
338 	u32 wa_rx_base, wa_rx_idx;
339 	u32 hif1_ofs = 0;
340 	int ret;
341 
342 	mt7915_dma_config(dev);
343 
344 	mt76_dma_attach(&dev->mt76);
345 
346 	if (dev->hif2)
347 		hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0);
348 
349 	mt7915_dma_disable(dev, true);
350 
351 	if (mtk_wed_device_active(&dev->mt76.mmio.wed)) {
352 		mt76_set(dev, MT_WFDMA_HOST_CONFIG, MT_WFDMA_HOST_CONFIG_WED);
353 
354 		mt76_wr(dev, MT_WFDMA_WED_RING_CONTROL,
355 			FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_TX0, 18) |
356 			FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_TX1, 19) |
357 			FIELD_PREP(MT_WFDMA_WED_RING_CONTROL_RX1, 1));
358 	} else {
359 		mt76_clear(dev, MT_WFDMA_HOST_CONFIG, MT_WFDMA_HOST_CONFIG_WED);
360 	}
361 
362 	/* init tx queue */
363 	ret = mt7915_init_tx_queues(&dev->phy,
364 				    MT_TXQ_ID(dev->phy.band_idx),
365 				    MT7915_TX_RING_SIZE,
366 				    MT_TXQ_RING_BASE(0));
367 	if (ret)
368 		return ret;
369 
370 	if (phy2) {
371 		ret = mt7915_init_tx_queues(phy2,
372 					    MT_TXQ_ID(phy2->band_idx),
373 					    MT7915_TX_RING_SIZE,
374 					    MT_TXQ_RING_BASE(1));
375 		if (ret)
376 			return ret;
377 	}
378 
379 	/* command to WM */
380 	ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_WM,
381 				  MT_MCUQ_ID(MT_MCUQ_WM),
382 				  MT7915_TX_MCU_RING_SIZE,
383 				  MT_MCUQ_RING_BASE(MT_MCUQ_WM));
384 	if (ret)
385 		return ret;
386 
387 	/* command to WA */
388 	ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_WA,
389 				  MT_MCUQ_ID(MT_MCUQ_WA),
390 				  MT7915_TX_MCU_RING_SIZE,
391 				  MT_MCUQ_RING_BASE(MT_MCUQ_WA));
392 	if (ret)
393 		return ret;
394 
395 	/* firmware download */
396 	ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_FWDL,
397 				  MT_MCUQ_ID(MT_MCUQ_FWDL),
398 				  MT7915_TX_FWDL_RING_SIZE,
399 				  MT_MCUQ_RING_BASE(MT_MCUQ_FWDL));
400 	if (ret)
401 		return ret;
402 
403 	/* event from WM */
404 	ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU],
405 			       MT_RXQ_ID(MT_RXQ_MCU),
406 			       MT7915_RX_MCU_RING_SIZE,
407 			       MT_RX_BUF_SIZE,
408 			       MT_RXQ_RING_BASE(MT_RXQ_MCU));
409 	if (ret)
410 		return ret;
411 
412 	/* event from WA */
413 	if (mtk_wed_device_active(&dev->mt76.mmio.wed)) {
414 		wa_rx_base = MT_WED_RX_RING_BASE;
415 		wa_rx_idx = MT7915_RXQ_MCU_WA;
416 		dev->mt76.q_rx[MT_RXQ_MCU_WA].flags = MT_WED_Q_TXFREE;
417 	} else {
418 		wa_rx_base = MT_RXQ_RING_BASE(MT_RXQ_MCU_WA);
419 		wa_rx_idx = MT_RXQ_ID(MT_RXQ_MCU_WA);
420 	}
421 	ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU_WA],
422 			       wa_rx_idx, MT7915_RX_MCU_RING_SIZE,
423 			       MT_RX_BUF_SIZE, wa_rx_base);
424 	if (ret)
425 		return ret;
426 
427 	/* rx data queue for band0 */
428 	if (!dev->phy.band_idx) {
429 		ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN],
430 				       MT_RXQ_ID(MT_RXQ_MAIN),
431 				       MT7915_RX_RING_SIZE,
432 				       MT_RX_BUF_SIZE,
433 				       MT_RXQ_RING_BASE(MT_RXQ_MAIN));
434 		if (ret)
435 			return ret;
436 	}
437 
438 	/* tx free notify event from WA for band0 */
439 	if (!is_mt7915(mdev)) {
440 		ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN_WA],
441 				       MT_RXQ_ID(MT_RXQ_MAIN_WA),
442 				       MT7915_RX_MCU_RING_SIZE,
443 				       MT_RX_BUF_SIZE,
444 				       MT_RXQ_RING_BASE(MT_RXQ_MAIN_WA));
445 		if (ret)
446 			return ret;
447 	}
448 
449 	if (dev->dbdc_support || dev->phy.band_idx) {
450 		/* rx data queue for band1 */
451 		ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1],
452 				       MT_RXQ_ID(MT_RXQ_BAND1),
453 				       MT7915_RX_RING_SIZE,
454 				       MT_RX_BUF_SIZE,
455 				       MT_RXQ_RING_BASE(MT_RXQ_BAND1) + hif1_ofs);
456 		if (ret)
457 			return ret;
458 
459 		/* tx free notify event from WA for band1 */
460 		ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1_WA],
461 				       MT_RXQ_ID(MT_RXQ_BAND1_WA),
462 				       MT7915_RX_MCU_RING_SIZE,
463 				       MT_RX_BUF_SIZE,
464 				       MT_RXQ_RING_BASE(MT_RXQ_BAND1_WA) + hif1_ofs);
465 		if (ret)
466 			return ret;
467 	}
468 
469 	ret = mt76_init_queues(dev, mt76_dma_rx_poll);
470 	if (ret < 0)
471 		return ret;
472 
473 	netif_napi_add_tx(&dev->mt76.tx_napi_dev, &dev->mt76.tx_napi,
474 			  mt7915_poll_tx);
475 	napi_enable(&dev->mt76.tx_napi);
476 
477 	mt7915_dma_enable(dev);
478 
479 	return 0;
480 }
481 
482 void mt7915_dma_cleanup(struct mt7915_dev *dev)
483 {
484 	mt7915_dma_disable(dev, true);
485 
486 	mt76_dma_cleanup(&dev->mt76);
487 }
488