1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2020  Realtek Corporation
3  */
4 
5 #include <linux/pci.h>
6 
7 #include "mac.h"
8 #include "pci.h"
9 #include "reg.h"
10 #include "ser.h"
11 
12 static bool rtw89_pci_disable_clkreq;
13 static bool rtw89_pci_disable_aspm_l1;
14 static bool rtw89_pci_disable_l1ss;
15 module_param_named(disable_clkreq, rtw89_pci_disable_clkreq, bool, 0644);
16 module_param_named(disable_aspm_l1, rtw89_pci_disable_aspm_l1, bool, 0644);
17 module_param_named(disable_aspm_l1ss, rtw89_pci_disable_l1ss, bool, 0644);
18 MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support");
19 MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support");
20 MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
21 
22 static int rtw89_pci_rst_bdram_pcie(struct rtw89_dev *rtwdev)
23 {
24 	u32 val;
25 	int ret;
26 
27 	rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1,
28 		      rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) | B_AX_RST_BDRAM);
29 
30 	ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM),
31 				       1, RTW89_PCI_POLL_BDRAM_RST_CNT, false,
32 				       rtwdev, R_AX_PCIE_INIT_CFG1);
33 
34 	if (ret)
35 		return -EBUSY;
36 
37 	return 0;
38 }
39 
40 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
41 				struct rtw89_pci_dma_ring *bd_ring,
42 				u32 cur_idx, bool tx)
43 {
44 	u32 cnt, cur_rp, wp, rp, len;
45 
46 	rp = bd_ring->rp;
47 	wp = bd_ring->wp;
48 	len = bd_ring->len;
49 
50 	cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
51 	if (tx)
52 		cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp);
53 	else
54 		cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp);
55 
56 	bd_ring->rp = cur_rp;
57 
58 	return cnt;
59 }
60 
61 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
62 				 struct rtw89_pci_tx_ring *tx_ring)
63 {
64 	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
65 	u32 addr_idx = bd_ring->addr.idx;
66 	u32 cnt, idx;
67 
68 	idx = rtw89_read32(rtwdev, addr_idx);
69 	cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true);
70 
71 	return cnt;
72 }
73 
74 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
75 				    struct rtw89_pci *rtwpci,
76 				    u32 cnt, bool release_all)
77 {
78 	struct rtw89_pci_tx_data *tx_data;
79 	struct sk_buff *skb;
80 	u32 qlen;
81 
82 	while (cnt--) {
83 		skb = skb_dequeue(&rtwpci->h2c_queue);
84 		if (!skb) {
85 			rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
86 			return;
87 		}
88 		skb_queue_tail(&rtwpci->h2c_release_queue, skb);
89 	}
90 
91 	qlen = skb_queue_len(&rtwpci->h2c_release_queue);
92 	if (!release_all)
93 	       qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0;
94 
95 	while (qlen--) {
96 		skb = skb_dequeue(&rtwpci->h2c_release_queue);
97 		if (!skb) {
98 			rtw89_err(rtwdev, "failed to release fwcmd\n");
99 			return;
100 		}
101 		tx_data = RTW89_PCI_TX_SKB_CB(skb);
102 		dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
103 				 DMA_TO_DEVICE);
104 		dev_kfree_skb_any(skb);
105 	}
106 }
107 
108 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
109 				       struct rtw89_pci *rtwpci)
110 {
111 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
112 	u32 cnt;
113 
114 	cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
115 	if (!cnt)
116 		return;
117 	rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false);
118 }
119 
120 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
121 				 struct rtw89_pci_rx_ring *rx_ring)
122 {
123 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
124 	u32 addr_idx = bd_ring->addr.idx;
125 	u32 cnt, idx;
126 
127 	idx = rtw89_read32(rtwdev, addr_idx);
128 	cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false);
129 
130 	return cnt;
131 }
132 
133 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
134 				       struct sk_buff *skb)
135 {
136 	struct rtw89_pci_rx_info *rx_info;
137 	dma_addr_t dma;
138 
139 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
140 	dma = rx_info->dma;
141 	dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
142 				DMA_FROM_DEVICE);
143 }
144 
145 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
146 					  struct sk_buff *skb)
147 {
148 	struct rtw89_pci_rx_info *rx_info;
149 	dma_addr_t dma;
150 
151 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
152 	dma = rx_info->dma;
153 	dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
154 				   DMA_FROM_DEVICE);
155 }
156 
157 static int rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
158 				      struct sk_buff *skb)
159 {
160 	struct rtw89_pci_rxbd_info *rxbd_info;
161 	struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
162 
163 	rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data;
164 	rx_info->fs = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_FS);
165 	rx_info->ls = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_LS);
166 	rx_info->len = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_WRITE_SIZE);
167 	rx_info->tag = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_TAG);
168 
169 	return 0;
170 }
171 
172 static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable)
173 {
174 	const struct rtw89_pci_info *info = rtwdev->pci_info;
175 	const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
176 	const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2;
177 
178 	if (enable) {
179 		rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
180 		if (dma_stop2->addr)
181 			rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
182 	} else {
183 		rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
184 		if (dma_stop2->addr)
185 			rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
186 	}
187 }
188 
189 static void rtw89_pci_ctrl_txdma_fw_ch_pcie(struct rtw89_dev *rtwdev, bool enable)
190 {
191 	const struct rtw89_pci_info *info = rtwdev->pci_info;
192 	const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
193 
194 	if (enable)
195 		rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
196 	else
197 		rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
198 }
199 
200 static bool
201 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
202 		      struct sk_buff *new,
203 		      const struct sk_buff *skb, u32 offset,
204 		      const struct rtw89_pci_rx_info *rx_info,
205 		      const struct rtw89_rx_desc_info *desc_info)
206 {
207 	u32 copy_len = rx_info->len - offset;
208 
209 	if (unlikely(skb_tailroom(new) < copy_len)) {
210 		rtw89_debug(rtwdev, RTW89_DBG_TXRX,
211 			    "invalid rx data length bd_len=%d desc_len=%d offset=%d (fs=%d ls=%d)\n",
212 			    rx_info->len, desc_info->pkt_size, offset, fs, ls);
213 		rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ",
214 			       skb->data, rx_info->len);
215 		/* length of a single segment skb is desc_info->pkt_size */
216 		if (fs && ls) {
217 			copy_len = desc_info->pkt_size;
218 		} else {
219 			rtw89_info(rtwdev, "drop rx data due to invalid length\n");
220 			return false;
221 		}
222 	}
223 
224 	skb_put_data(new, skb->data + offset, copy_len);
225 
226 	return true;
227 }
228 
229 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
230 				       struct rtw89_pci_rx_ring *rx_ring)
231 {
232 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
233 	struct rtw89_pci_rx_info *rx_info;
234 	struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc;
235 	struct sk_buff *new = rx_ring->diliver_skb;
236 	struct sk_buff *skb;
237 	u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
238 	u32 offset;
239 	u32 cnt = 1;
240 	bool fs, ls;
241 	int ret;
242 
243 	skb = rx_ring->buf[bd_ring->wp];
244 	rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
245 
246 	ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
247 	if (ret) {
248 		rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
249 			  bd_ring->wp, ret);
250 		goto err_sync_device;
251 	}
252 
253 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
254 	fs = rx_info->fs;
255 	ls = rx_info->ls;
256 
257 	if (fs) {
258 		if (new) {
259 			rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
260 				    "skb should not be ready before first segment start\n");
261 			goto err_sync_device;
262 		}
263 		if (desc_info->ready) {
264 			rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
265 			goto err_sync_device;
266 		}
267 
268 		rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size);
269 
270 		new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size);
271 		if (!new)
272 			goto err_sync_device;
273 
274 		rx_ring->diliver_skb = new;
275 
276 		/* first segment has RX desc */
277 		offset = desc_info->offset + desc_info->rxd_len;
278 	} else {
279 		offset = sizeof(struct rtw89_pci_rxbd_info);
280 		if (!new) {
281 			rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n");
282 			goto err_sync_device;
283 		}
284 	}
285 	if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
286 		goto err_sync_device;
287 	rtw89_pci_sync_skb_for_device(rtwdev, skb);
288 	rtw89_pci_rxbd_increase(rx_ring, 1);
289 
290 	if (!desc_info->ready) {
291 		rtw89_warn(rtwdev, "no rx desc information\n");
292 		goto err_free_resource;
293 	}
294 	if (ls) {
295 		rtw89_core_rx(rtwdev, desc_info, new);
296 		rx_ring->diliver_skb = NULL;
297 		desc_info->ready = false;
298 	}
299 
300 	return cnt;
301 
302 err_sync_device:
303 	rtw89_pci_sync_skb_for_device(rtwdev, skb);
304 	rtw89_pci_rxbd_increase(rx_ring, 1);
305 err_free_resource:
306 	if (new)
307 		dev_kfree_skb_any(new);
308 	rx_ring->diliver_skb = NULL;
309 	desc_info->ready = false;
310 
311 	return cnt;
312 }
313 
314 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
315 				   struct rtw89_pci_rx_ring *rx_ring,
316 				   u32 cnt)
317 {
318 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
319 	u32 rx_cnt;
320 
321 	while (cnt && rtwdev->napi_budget_countdown > 0) {
322 		rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
323 		if (!rx_cnt) {
324 			rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
325 
326 			/* skip the rest RXBD bufs */
327 			rtw89_pci_rxbd_increase(rx_ring, cnt);
328 			break;
329 		}
330 
331 		cnt -= rx_cnt;
332 	}
333 
334 	rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
335 }
336 
337 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
338 				  struct rtw89_pci *rtwpci, int budget)
339 {
340 	struct rtw89_pci_rx_ring *rx_ring;
341 	int countdown = rtwdev->napi_budget_countdown;
342 	u32 cnt;
343 
344 	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ];
345 
346 	cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
347 	if (!cnt)
348 		return 0;
349 
350 	cnt = min_t(u32, budget, cnt);
351 
352 	rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
353 
354 	/* In case of flushing pending SKBs, the countdown may exceed. */
355 	if (rtwdev->napi_budget_countdown <= 0)
356 		return budget;
357 
358 	return budget - countdown;
359 }
360 
361 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
362 				struct rtw89_pci_tx_ring *tx_ring,
363 				struct sk_buff *skb, u8 tx_status)
364 {
365 	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
366 	struct ieee80211_tx_info *info;
367 
368 	rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status == RTW89_TX_DONE);
369 
370 	info = IEEE80211_SKB_CB(skb);
371 	ieee80211_tx_info_clear_status(info);
372 
373 	if (info->flags & IEEE80211_TX_CTL_NO_ACK)
374 		info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
375 	if (tx_status == RTW89_TX_DONE) {
376 		info->flags |= IEEE80211_TX_STAT_ACK;
377 		tx_ring->tx_acked++;
378 	} else {
379 		if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
380 			rtw89_debug(rtwdev, RTW89_DBG_FW,
381 				    "failed to TX of status %x\n", tx_status);
382 		switch (tx_status) {
383 		case RTW89_TX_RETRY_LIMIT:
384 			tx_ring->tx_retry_lmt++;
385 			break;
386 		case RTW89_TX_LIFE_TIME:
387 			tx_ring->tx_life_time++;
388 			break;
389 		case RTW89_TX_MACID_DROP:
390 			tx_ring->tx_mac_id_drop++;
391 			break;
392 		default:
393 			rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
394 			break;
395 		}
396 	}
397 
398 	ieee80211_tx_status_ni(rtwdev->hw, skb);
399 }
400 
401 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
402 {
403 	struct rtw89_pci_tx_wd *txwd;
404 	u32 cnt;
405 
406 	cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
407 	while (cnt--) {
408 		txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
409 		if (!txwd) {
410 			rtw89_warn(rtwdev, "No busy txwd pages available\n");
411 			break;
412 		}
413 
414 		list_del_init(&txwd->list);
415 
416 		/* this skb has been freed by RPP */
417 		if (skb_queue_len(&txwd->queue) == 0)
418 			rtw89_pci_enqueue_txwd(tx_ring, txwd);
419 	}
420 }
421 
422 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
423 					struct rtw89_pci_tx_ring *tx_ring)
424 {
425 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
426 	struct rtw89_pci_tx_wd *txwd;
427 	int i;
428 
429 	for (i = 0; i < wd_ring->page_num; i++) {
430 		txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
431 		if (!txwd)
432 			break;
433 
434 		list_del_init(&txwd->list);
435 	}
436 }
437 
438 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
439 				       struct rtw89_pci_tx_ring *tx_ring,
440 				       struct rtw89_pci_tx_wd *txwd, u16 seq,
441 				       u8 tx_status)
442 {
443 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
444 	struct rtw89_pci_tx_data *tx_data;
445 	struct sk_buff *skb, *tmp;
446 	u8 txch = tx_ring->txch;
447 
448 	if (!list_empty(&txwd->list)) {
449 		rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
450 		/* In low power mode, RPP can receive before updating of TX BD.
451 		 * In normal mode, it should not happen so give it a warning.
452 		 */
453 		if (!rtwpci->low_power && !list_empty(&txwd->list))
454 			rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
455 				   txch, seq);
456 	}
457 
458 	skb_queue_walk_safe(&txwd->queue, skb, tmp) {
459 		skb_unlink(skb, &txwd->queue);
460 
461 		tx_data = RTW89_PCI_TX_SKB_CB(skb);
462 		dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
463 				 DMA_TO_DEVICE);
464 
465 		rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
466 	}
467 
468 	if (list_empty(&txwd->list))
469 		rtw89_pci_enqueue_txwd(tx_ring, txwd);
470 }
471 
472 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,
473 				  struct rtw89_pci_rpp_fmt *rpp)
474 {
475 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
476 	struct rtw89_pci_tx_ring *tx_ring;
477 	struct rtw89_pci_tx_wd_ring *wd_ring;
478 	struct rtw89_pci_tx_wd *txwd;
479 	u16 seq;
480 	u8 qsel, tx_status, txch;
481 
482 	seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ);
483 	qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL);
484 	tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS);
485 	txch = rtw89_core_get_ch_dma(rtwdev, qsel);
486 
487 	if (txch == RTW89_TXCH_CH12) {
488 		rtw89_warn(rtwdev, "should no fwcmd release report\n");
489 		return;
490 	}
491 
492 	tx_ring = &rtwpci->tx_rings[txch];
493 	wd_ring = &tx_ring->wd_ring;
494 	txwd = &wd_ring->pages[seq];
495 
496 	rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status);
497 }
498 
499 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
500 					       struct rtw89_pci_tx_ring *tx_ring)
501 {
502 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
503 	struct rtw89_pci_tx_wd *txwd;
504 	int i;
505 
506 	for (i = 0; i < wd_ring->page_num; i++) {
507 		txwd = &wd_ring->pages[i];
508 
509 		if (!list_empty(&txwd->list))
510 			continue;
511 
512 		rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP);
513 	}
514 }
515 
516 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
517 				     struct rtw89_pci_rx_ring *rx_ring,
518 				     u32 max_cnt)
519 {
520 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
521 	struct rtw89_pci_rx_info *rx_info;
522 	struct rtw89_pci_rpp_fmt *rpp;
523 	struct rtw89_rx_desc_info desc_info = {};
524 	struct sk_buff *skb;
525 	u32 cnt = 0;
526 	u32 rpp_size = sizeof(struct rtw89_pci_rpp_fmt);
527 	u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
528 	u32 offset;
529 	int ret;
530 
531 	skb = rx_ring->buf[bd_ring->wp];
532 	rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
533 
534 	ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
535 	if (ret) {
536 		rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
537 			  bd_ring->wp, ret);
538 		goto err_sync_device;
539 	}
540 
541 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
542 	if (!rx_info->fs || !rx_info->ls) {
543 		rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
544 		return cnt;
545 	}
546 
547 	rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size);
548 
549 	/* first segment has RX desc */
550 	offset = desc_info.offset + desc_info.rxd_len;
551 	for (; offset + rpp_size <= rx_info->len; offset += rpp_size) {
552 		rpp = (struct rtw89_pci_rpp_fmt *)(skb->data + offset);
553 		rtw89_pci_release_rpp(rtwdev, rpp);
554 	}
555 
556 	rtw89_pci_sync_skb_for_device(rtwdev, skb);
557 	rtw89_pci_rxbd_increase(rx_ring, 1);
558 	cnt++;
559 
560 	return cnt;
561 
562 err_sync_device:
563 	rtw89_pci_sync_skb_for_device(rtwdev, skb);
564 	return 0;
565 }
566 
567 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
568 				 struct rtw89_pci_rx_ring *rx_ring,
569 				 u32 cnt)
570 {
571 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
572 	u32 release_cnt;
573 
574 	while (cnt) {
575 		release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt);
576 		if (!release_cnt) {
577 			rtw89_err(rtwdev, "failed to release TX skbs\n");
578 
579 			/* skip the rest RXBD bufs */
580 			rtw89_pci_rxbd_increase(rx_ring, cnt);
581 			break;
582 		}
583 
584 		cnt -= release_cnt;
585 	}
586 
587 	rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
588 }
589 
590 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
591 				  struct rtw89_pci *rtwpci, int budget)
592 {
593 	struct rtw89_pci_rx_ring *rx_ring;
594 	u32 cnt;
595 	int work_done;
596 
597 	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
598 
599 	spin_lock_bh(&rtwpci->trx_lock);
600 
601 	cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
602 	if (cnt == 0)
603 		goto out_unlock;
604 
605 	rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
606 
607 out_unlock:
608 	spin_unlock_bh(&rtwpci->trx_lock);
609 
610 	/* always release all RPQ */
611 	work_done = min_t(int, cnt, budget);
612 	rtwdev->napi_budget_countdown -= work_done;
613 
614 	return work_done;
615 }
616 
617 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
618 				      struct rtw89_pci *rtwpci)
619 {
620 	struct rtw89_pci_rx_ring *rx_ring;
621 	struct rtw89_pci_dma_ring *bd_ring;
622 	u32 reg_idx;
623 	u16 hw_idx, hw_idx_next, host_idx;
624 	int i;
625 
626 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
627 		rx_ring = &rtwpci->rx_rings[i];
628 		bd_ring = &rx_ring->bd_ring;
629 
630 		reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
631 		hw_idx = FIELD_GET(TXBD_HW_IDX_MASK, reg_idx);
632 		host_idx = FIELD_GET(TXBD_HOST_IDX_MASK, reg_idx);
633 		hw_idx_next = (hw_idx + 1) % bd_ring->len;
634 
635 		if (hw_idx_next == host_idx)
636 			rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i);
637 
638 		rtw89_debug(rtwdev, RTW89_DBG_TXRX,
639 			    "%d RXD unavailable, idx=0x%08x, len=%d\n",
640 			    i, reg_idx, bd_ring->len);
641 	}
642 }
643 
644 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
645 			       struct rtw89_pci *rtwpci,
646 			       struct rtw89_pci_isrs *isrs)
647 {
648 	isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
649 	isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
650 	isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
651 
652 	rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
653 	rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]);
654 	rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]);
655 }
656 EXPORT_SYMBOL(rtw89_pci_recognize_intrs);
657 
658 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
659 				  struct rtw89_pci *rtwpci,
660 				  struct rtw89_pci_isrs *isrs)
661 {
662 	isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
663 	isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ?
664 			      rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
665 	isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ?
666 			rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
667 	isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ?
668 			rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
669 
670 	if (isrs->halt_c2h_isrs)
671 		rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
672 	if (isrs->isrs[0])
673 		rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]);
674 	if (isrs->isrs[1])
675 		rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]);
676 }
677 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1);
678 
679 static void rtw89_pci_clear_isr0(struct rtw89_dev *rtwdev, u32 isr00)
680 {
681 	/* write 1 clear */
682 	rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isr00);
683 }
684 
685 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
686 {
687 	rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
688 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]);
689 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]);
690 }
691 EXPORT_SYMBOL(rtw89_pci_enable_intr);
692 
693 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
694 {
695 	rtw89_write32(rtwdev, R_AX_HIMR0, 0);
696 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0);
697 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0);
698 }
699 EXPORT_SYMBOL(rtw89_pci_disable_intr);
700 
701 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
702 {
703 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs);
704 	rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
705 	rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]);
706 	rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]);
707 }
708 EXPORT_SYMBOL(rtw89_pci_enable_intr_v1);
709 
710 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
711 {
712 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0);
713 }
714 EXPORT_SYMBOL(rtw89_pci_disable_intr_v1);
715 
716 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
717 {
718 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
719 	unsigned long flags;
720 
721 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
722 	rtw89_chip_disable_intr(rtwdev, rtwpci);
723 	rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START);
724 	rtw89_chip_enable_intr(rtwdev, rtwpci);
725 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
726 }
727 
728 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
729 {
730 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
731 	unsigned long flags;
732 
733 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
734 	rtw89_chip_disable_intr(rtwdev, rtwpci);
735 	rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
736 	rtw89_chip_enable_intr(rtwdev, rtwpci);
737 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
738 }
739 
740 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
741 {
742 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
743 	int budget = NAPI_POLL_WEIGHT;
744 
745 	/* To prevent RXQ get stuck due to run out of budget. */
746 	rtwdev->napi_budget_countdown = budget;
747 
748 	rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
749 	rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
750 }
751 
752 static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
753 {
754 	struct rtw89_dev *rtwdev = dev;
755 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
756 	struct rtw89_pci_isrs isrs;
757 	unsigned long flags;
758 
759 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
760 	rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
761 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
762 
763 	if (unlikely(isrs.isrs[0] & B_AX_RDU_INT))
764 		rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
765 
766 	if (unlikely(isrs.halt_c2h_isrs & B_AX_HALT_C2H_INT_EN))
767 		rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
768 
769 	if (unlikely(isrs.halt_c2h_isrs & B_AX_WDT_TIMEOUT_INT_EN))
770 		rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
771 
772 	if (unlikely(rtwpci->under_recovery))
773 		goto enable_intr;
774 
775 	if (unlikely(rtwpci->low_power)) {
776 		rtw89_pci_low_power_interrupt_handler(rtwdev);
777 		goto enable_intr;
778 	}
779 
780 	if (likely(rtwpci->running)) {
781 		local_bh_disable();
782 		napi_schedule(&rtwdev->napi);
783 		local_bh_enable();
784 	}
785 
786 	return IRQ_HANDLED;
787 
788 enable_intr:
789 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
790 	if (likely(rtwpci->running))
791 		rtw89_chip_enable_intr(rtwdev, rtwpci);
792 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
793 	return IRQ_HANDLED;
794 }
795 
796 static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
797 {
798 	struct rtw89_dev *rtwdev = dev;
799 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
800 	unsigned long flags;
801 	irqreturn_t irqret = IRQ_WAKE_THREAD;
802 
803 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
804 
805 	/* If interrupt event is on the road, it is still trigger interrupt
806 	 * even we have done pci_stop() to turn off IMR.
807 	 */
808 	if (unlikely(!rtwpci->running)) {
809 		irqret = IRQ_HANDLED;
810 		goto exit;
811 	}
812 
813 	rtw89_chip_disable_intr(rtwdev, rtwpci);
814 exit:
815 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
816 
817 	return irqret;
818 }
819 
820 #define DEF_TXCHADDRS_TYPE1(info, txch, v...) \
821 	[RTW89_TXCH_##txch] = { \
822 		.num = R_AX_##txch##_TXBD_NUM ##v, \
823 		.idx = R_AX_##txch##_TXBD_IDX ##v, \
824 		.bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
825 		.desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
826 		.desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
827 	}
828 
829 #define DEF_TXCHADDRS(info, txch, v...) \
830 	[RTW89_TXCH_##txch] = { \
831 		.num = R_AX_##txch##_TXBD_NUM, \
832 		.idx = R_AX_##txch##_TXBD_IDX, \
833 		.bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
834 		.desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
835 		.desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
836 	}
837 
838 #define DEF_RXCHADDRS(info, rxch, v...) \
839 	[RTW89_RXCH_##rxch] = { \
840 		.num = R_AX_##rxch##_RXBD_NUM ##v, \
841 		.idx = R_AX_##rxch##_RXBD_IDX ##v, \
842 		.desa_l = R_AX_##rxch##_RXBD_DESA_L ##v, \
843 		.desa_h = R_AX_##rxch##_RXBD_DESA_H ##v, \
844 	}
845 
846 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
847 	.tx = {
848 		DEF_TXCHADDRS(info, ACH0),
849 		DEF_TXCHADDRS(info, ACH1),
850 		DEF_TXCHADDRS(info, ACH2),
851 		DEF_TXCHADDRS(info, ACH3),
852 		DEF_TXCHADDRS(info, ACH4),
853 		DEF_TXCHADDRS(info, ACH5),
854 		DEF_TXCHADDRS(info, ACH6),
855 		DEF_TXCHADDRS(info, ACH7),
856 		DEF_TXCHADDRS(info, CH8),
857 		DEF_TXCHADDRS(info, CH9),
858 		DEF_TXCHADDRS_TYPE1(info, CH10),
859 		DEF_TXCHADDRS_TYPE1(info, CH11),
860 		DEF_TXCHADDRS(info, CH12),
861 	},
862 	.rx = {
863 		DEF_RXCHADDRS(info, RXQ),
864 		DEF_RXCHADDRS(info, RPQ),
865 	},
866 };
867 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set);
868 
869 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = {
870 	.tx = {
871 		DEF_TXCHADDRS(info, ACH0, _V1),
872 		DEF_TXCHADDRS(info, ACH1, _V1),
873 		DEF_TXCHADDRS(info, ACH2, _V1),
874 		DEF_TXCHADDRS(info, ACH3, _V1),
875 		DEF_TXCHADDRS(info, ACH4, _V1),
876 		DEF_TXCHADDRS(info, ACH5, _V1),
877 		DEF_TXCHADDRS(info, ACH6, _V1),
878 		DEF_TXCHADDRS(info, ACH7, _V1),
879 		DEF_TXCHADDRS(info, CH8, _V1),
880 		DEF_TXCHADDRS(info, CH9, _V1),
881 		DEF_TXCHADDRS_TYPE1(info, CH10, _V1),
882 		DEF_TXCHADDRS_TYPE1(info, CH11, _V1),
883 		DEF_TXCHADDRS(info, CH12, _V1),
884 	},
885 	.rx = {
886 		DEF_RXCHADDRS(info, RXQ, _V1),
887 		DEF_RXCHADDRS(info, RPQ, _V1),
888 	},
889 };
890 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1);
891 
892 #undef DEF_TXCHADDRS_TYPE1
893 #undef DEF_TXCHADDRS
894 #undef DEF_RXCHADDRS
895 
896 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
897 				    enum rtw89_tx_channel txch,
898 				    const struct rtw89_pci_ch_dma_addr **addr)
899 {
900 	const struct rtw89_pci_info *info = rtwdev->pci_info;
901 
902 	if (txch >= RTW89_TXCH_NUM)
903 		return -EINVAL;
904 
905 	*addr = &info->dma_addr_set->tx[txch];
906 
907 	return 0;
908 }
909 
910 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
911 				    enum rtw89_rx_channel rxch,
912 				    const struct rtw89_pci_ch_dma_addr **addr)
913 {
914 	const struct rtw89_pci_info *info = rtwdev->pci_info;
915 
916 	if (rxch >= RTW89_RXCH_NUM)
917 		return -EINVAL;
918 
919 	*addr = &info->dma_addr_set->rx[rxch];
920 
921 	return 0;
922 }
923 
924 static u32 rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring *ring)
925 {
926 	struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring;
927 
928 	/* reserved 1 desc check ring is full or not */
929 	if (bd_ring->rp > bd_ring->wp)
930 		return bd_ring->rp - bd_ring->wp - 1;
931 
932 	return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1;
933 }
934 
935 static
936 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
937 {
938 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
939 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
940 	u32 cnt;
941 
942 	spin_lock_bh(&rtwpci->trx_lock);
943 	rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
944 	cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
945 	spin_unlock_bh(&rtwpci->trx_lock);
946 
947 	return cnt;
948 }
949 
950 static
951 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
952 						   u8 txch)
953 {
954 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
955 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
956 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
957 	u32 cnt;
958 
959 	spin_lock_bh(&rtwpci->trx_lock);
960 	cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
961 	cnt = min(cnt, wd_ring->curr_num);
962 	spin_unlock_bh(&rtwpci->trx_lock);
963 
964 	return cnt;
965 }
966 
967 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
968 						     u8 txch)
969 {
970 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
971 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
972 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
973 	const struct rtw89_chip_info *chip = rtwdev->chip;
974 	u32 bd_cnt, wd_cnt, min_cnt = 0;
975 	struct rtw89_pci_rx_ring *rx_ring;
976 	enum rtw89_debug_mask debug_mask;
977 	u32 cnt;
978 
979 	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
980 
981 	spin_lock_bh(&rtwpci->trx_lock);
982 	bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
983 	wd_cnt = wd_ring->curr_num;
984 
985 	if (wd_cnt == 0 || bd_cnt == 0) {
986 		cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
987 		if (cnt)
988 			rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
989 		else if (wd_cnt == 0)
990 			goto out_unlock;
991 
992 		bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
993 		if (bd_cnt == 0)
994 			rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
995 	}
996 
997 	bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
998 	wd_cnt = wd_ring->curr_num;
999 	min_cnt = min(bd_cnt, wd_cnt);
1000 	if (min_cnt == 0) {
1001 		/* This message can be frequently shown in low power mode or
1002 		 * high traffic with small FIFO chips, and we have recognized it as normal
1003 		 * behavior, so print with mask RTW89_DBG_TXRX in these situations.
1004 		 */
1005 		if (rtwpci->low_power || chip->small_fifo_size)
1006 			debug_mask = RTW89_DBG_TXRX;
1007 		else
1008 			debug_mask = RTW89_DBG_UNEXP;
1009 
1010 		rtw89_debug(rtwdev, debug_mask,
1011 			    "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n",
1012 			    wd_cnt, bd_cnt);
1013 	}
1014 
1015 out_unlock:
1016 	spin_unlock_bh(&rtwpci->trx_lock);
1017 
1018 	return min_cnt;
1019 }
1020 
1021 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1022 						   u8 txch)
1023 {
1024 	if (rtwdev->hci.paused)
1025 		return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1026 
1027 	if (txch == RTW89_TXCH_CH12)
1028 		return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1029 
1030 	return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1031 }
1032 
1033 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1034 {
1035 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1036 	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1037 	u32 host_idx, addr;
1038 
1039 	spin_lock_bh(&rtwpci->trx_lock);
1040 
1041 	addr = bd_ring->addr.idx;
1042 	host_idx = bd_ring->wp;
1043 	rtw89_write16(rtwdev, addr, host_idx);
1044 
1045 	spin_unlock_bh(&rtwpci->trx_lock);
1046 }
1047 
1048 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1049 					int n_txbd)
1050 {
1051 	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1052 	u32 host_idx, len;
1053 
1054 	len = bd_ring->len;
1055 	host_idx = bd_ring->wp + n_txbd;
1056 	host_idx = host_idx < len ? host_idx : host_idx - len;
1057 
1058 	bd_ring->wp = host_idx;
1059 }
1060 
1061 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1062 {
1063 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1064 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1065 
1066 	if (rtwdev->hci.paused) {
1067 		set_bit(txch, rtwpci->kick_map);
1068 		return;
1069 	}
1070 
1071 	__rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1072 }
1073 
1074 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1075 {
1076 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1077 	struct rtw89_pci_tx_ring *tx_ring;
1078 	int txch;
1079 
1080 	for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1081 		if (!test_and_clear_bit(txch, rtwpci->kick_map))
1082 			continue;
1083 
1084 		tx_ring = &rtwpci->tx_rings[txch];
1085 		__rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1086 	}
1087 }
1088 
1089 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1090 {
1091 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1092 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1093 	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1094 	u32 cur_idx, cur_rp;
1095 	u8 i;
1096 
1097 	/* Because the time taked by the I/O is a bit dynamic, it's hard to
1098 	 * define a reasonable fixed total timeout to use read_poll_timeout*
1099 	 * helper. Instead, we can ensure a reasonable polling times, so we
1100 	 * just use for loop with udelay here.
1101 	 */
1102 	for (i = 0; i < 60; i++) {
1103 		cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
1104 		cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
1105 		if (cur_rp == bd_ring->wp)
1106 			return;
1107 
1108 		udelay(1);
1109 	}
1110 
1111 	if (!drop)
1112 		rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1113 }
1114 
1115 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1116 					bool drop)
1117 {
1118 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1119 	u8 i;
1120 
1121 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1122 		/* It may be unnecessary to flush FWCMD queue. */
1123 		if (i == RTW89_TXCH_CH12)
1124 			continue;
1125 		if (info->tx_dma_ch_mask & BIT(i))
1126 			continue;
1127 
1128 		if (txchs & BIT(i))
1129 			__pci_flush_txch(rtwdev, i, drop);
1130 	}
1131 }
1132 
1133 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1134 				       bool drop)
1135 {
1136 	__rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1137 }
1138 
1139 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1140 			       void *txaddr_info_addr, u32 total_len,
1141 			       dma_addr_t dma, u8 *add_info_nr)
1142 {
1143 	struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr;
1144 
1145 	txaddr_info->length = cpu_to_le16(total_len);
1146 	txaddr_info->option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS |
1147 					  RTW89_PCI_ADDR_NUM(1));
1148 	txaddr_info->dma = cpu_to_le32(dma);
1149 
1150 	*add_info_nr = 1;
1151 
1152 	return sizeof(*txaddr_info);
1153 }
1154 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info);
1155 
1156 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1157 				  void *txaddr_info_addr, u32 total_len,
1158 				  dma_addr_t dma, u8 *add_info_nr)
1159 {
1160 	struct rtw89_pci_tx_addr_info_32_v1 *txaddr_info = txaddr_info_addr;
1161 	u32 remain = total_len;
1162 	u32 len;
1163 	u16 length_option;
1164 	int n;
1165 
1166 	for (n = 0; n < RTW89_TXADDR_INFO_NR_V1 && remain; n++) {
1167 		len = remain >= TXADDR_INFO_LENTHG_V1_MAX ?
1168 		      TXADDR_INFO_LENTHG_V1_MAX : remain;
1169 		remain -= len;
1170 
1171 		length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) |
1172 				FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) |
1173 				FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0);
1174 		txaddr_info->length_opt = cpu_to_le16(length_option);
1175 		txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma));
1176 		txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma));
1177 
1178 		dma += len;
1179 		txaddr_info++;
1180 	}
1181 
1182 	WARN_ONCE(remain, "length overflow remain=%u total_len=%u",
1183 		  remain, total_len);
1184 
1185 	*add_info_nr = n;
1186 
1187 	return n * sizeof(*txaddr_info);
1188 }
1189 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info_v1);
1190 
1191 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1192 				 struct rtw89_pci_tx_ring *tx_ring,
1193 				 struct rtw89_pci_tx_wd *txwd,
1194 				 struct rtw89_core_tx_request *tx_req)
1195 {
1196 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1197 	const struct rtw89_chip_info *chip = rtwdev->chip;
1198 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1199 	struct rtw89_txwd_info *txwd_info;
1200 	struct rtw89_pci_tx_wp_info *txwp_info;
1201 	void *txaddr_info_addr;
1202 	struct pci_dev *pdev = rtwpci->pdev;
1203 	struct sk_buff *skb = tx_req->skb;
1204 	struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1205 	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
1206 	bool en_wd_info = desc_info->en_wd_info;
1207 	u32 txwd_len;
1208 	u32 txwp_len;
1209 	u32 txaddr_info_len;
1210 	dma_addr_t dma;
1211 	int ret;
1212 
1213 	dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1214 	if (dma_mapping_error(&pdev->dev, dma)) {
1215 		rtw89_err(rtwdev, "failed to map skb dma data\n");
1216 		ret = -EBUSY;
1217 		goto err;
1218 	}
1219 
1220 	tx_data->dma = dma;
1221 	rcu_assign_pointer(skb_data->wait, NULL);
1222 
1223 	txwp_len = sizeof(*txwp_info);
1224 	txwd_len = chip->txwd_body_size;
1225 	txwd_len += en_wd_info ? sizeof(*txwd_info) : 0;
1226 
1227 	txwp_info = txwd->vaddr + txwd_len;
1228 	txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID);
1229 	txwp_info->seq1 = 0;
1230 	txwp_info->seq2 = 0;
1231 	txwp_info->seq3 = 0;
1232 
1233 	tx_ring->tx_cnt++;
1234 	txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len;
1235 	txaddr_info_len =
1236 		rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len,
1237 					    dma, &desc_info->addr_info_nr);
1238 
1239 	txwd->len = txwd_len + txwp_len + txaddr_info_len;
1240 
1241 	rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr);
1242 
1243 	skb_queue_tail(&txwd->queue, skb);
1244 
1245 	return 0;
1246 
1247 err:
1248 	return ret;
1249 }
1250 
1251 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1252 				  struct rtw89_pci_tx_ring *tx_ring,
1253 				  struct rtw89_pci_tx_bd_32 *txbd,
1254 				  struct rtw89_core_tx_request *tx_req)
1255 {
1256 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1257 	const struct rtw89_chip_info *chip = rtwdev->chip;
1258 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1259 	void *txdesc;
1260 	int txdesc_size = chip->h2c_desc_size;
1261 	struct pci_dev *pdev = rtwpci->pdev;
1262 	struct sk_buff *skb = tx_req->skb;
1263 	struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1264 	dma_addr_t dma;
1265 
1266 	txdesc = skb_push(skb, txdesc_size);
1267 	memset(txdesc, 0, txdesc_size);
1268 	rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1269 
1270 	dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1271 	if (dma_mapping_error(&pdev->dev, dma)) {
1272 		rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1273 		return -EBUSY;
1274 	}
1275 
1276 	tx_data->dma = dma;
1277 	txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS);
1278 	txbd->length = cpu_to_le16(skb->len);
1279 	txbd->dma = cpu_to_le32(tx_data->dma);
1280 	skb_queue_tail(&rtwpci->h2c_queue, skb);
1281 
1282 	rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1283 
1284 	return 0;
1285 }
1286 
1287 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1288 				 struct rtw89_pci_tx_ring *tx_ring,
1289 				 struct rtw89_pci_tx_bd_32 *txbd,
1290 				 struct rtw89_core_tx_request *tx_req)
1291 {
1292 	struct rtw89_pci_tx_wd *txwd;
1293 	int ret;
1294 
1295 	/* FWCMD queue doesn't have wd pages. Instead, it submits the CMD
1296 	 * buffer with WD BODY only. So here we don't need to check the free
1297 	 * pages of the wd ring.
1298 	 */
1299 	if (tx_ring->txch == RTW89_TXCH_CH12)
1300 		return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1301 
1302 	txwd = rtw89_pci_dequeue_txwd(tx_ring);
1303 	if (!txwd) {
1304 		rtw89_err(rtwdev, "no available TXWD\n");
1305 		ret = -ENOSPC;
1306 		goto err;
1307 	}
1308 
1309 	ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1310 	if (ret) {
1311 		rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1312 		goto err_enqueue_wd;
1313 	}
1314 
1315 	list_add_tail(&txwd->list, &tx_ring->busy_pages);
1316 
1317 	txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS);
1318 	txbd->length = cpu_to_le16(txwd->len);
1319 	txbd->dma = cpu_to_le32(txwd->paddr);
1320 
1321 	rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1322 
1323 	return 0;
1324 
1325 err_enqueue_wd:
1326 	rtw89_pci_enqueue_txwd(tx_ring, txwd);
1327 err:
1328 	return ret;
1329 }
1330 
1331 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1332 			      u8 txch)
1333 {
1334 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1335 	struct rtw89_pci_tx_ring *tx_ring;
1336 	struct rtw89_pci_tx_bd_32 *txbd;
1337 	u32 n_avail_txbd;
1338 	int ret = 0;
1339 
1340 	/* check the tx type and dma channel for fw cmd queue */
1341 	if ((txch == RTW89_TXCH_CH12 ||
1342 	     tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) &&
1343 	    (txch != RTW89_TXCH_CH12 ||
1344 	     tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) {
1345 		rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1346 		return -EINVAL;
1347 	}
1348 
1349 	tx_ring = &rtwpci->tx_rings[txch];
1350 	spin_lock_bh(&rtwpci->trx_lock);
1351 
1352 	n_avail_txbd = rtw89_pci_get_avail_txbd_num(tx_ring);
1353 	if (n_avail_txbd == 0) {
1354 		rtw89_err(rtwdev, "no available TXBD\n");
1355 		ret = -ENOSPC;
1356 		goto err_unlock;
1357 	}
1358 
1359 	txbd = rtw89_pci_get_next_txbd(tx_ring);
1360 	ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1361 	if (ret) {
1362 		rtw89_err(rtwdev, "failed to submit TXBD\n");
1363 		goto err_unlock;
1364 	}
1365 
1366 	spin_unlock_bh(&rtwpci->trx_lock);
1367 	return 0;
1368 
1369 err_unlock:
1370 	spin_unlock_bh(&rtwpci->trx_lock);
1371 	return ret;
1372 }
1373 
1374 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1375 {
1376 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1377 	int ret;
1378 
1379 	ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma);
1380 	if (ret) {
1381 		rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1382 		return ret;
1383 	}
1384 
1385 	return 0;
1386 }
1387 
1388 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM] = {
1389 	[RTW89_TXCH_ACH0] = {.start_idx = 0,  .max_num = 5, .min_num = 2},
1390 	[RTW89_TXCH_ACH1] = {.start_idx = 5,  .max_num = 5, .min_num = 2},
1391 	[RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1392 	[RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1393 	[RTW89_TXCH_ACH4] = {.start_idx = 20, .max_num = 5, .min_num = 2},
1394 	[RTW89_TXCH_ACH5] = {.start_idx = 25, .max_num = 5, .min_num = 2},
1395 	[RTW89_TXCH_ACH6] = {.start_idx = 30, .max_num = 5, .min_num = 2},
1396 	[RTW89_TXCH_ACH7] = {.start_idx = 35, .max_num = 5, .min_num = 2},
1397 	[RTW89_TXCH_CH8]  = {.start_idx = 40, .max_num = 5, .min_num = 1},
1398 	[RTW89_TXCH_CH9]  = {.start_idx = 45, .max_num = 5, .min_num = 1},
1399 	[RTW89_TXCH_CH10] = {.start_idx = 50, .max_num = 5, .min_num = 1},
1400 	[RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1},
1401 	[RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1},
1402 };
1403 EXPORT_SYMBOL(rtw89_bd_ram_table_dual);
1404 
1405 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM] = {
1406 	[RTW89_TXCH_ACH0] = {.start_idx = 0,  .max_num = 5, .min_num = 2},
1407 	[RTW89_TXCH_ACH1] = {.start_idx = 5,  .max_num = 5, .min_num = 2},
1408 	[RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1409 	[RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1410 	[RTW89_TXCH_CH8]  = {.start_idx = 20, .max_num = 4, .min_num = 1},
1411 	[RTW89_TXCH_CH9]  = {.start_idx = 24, .max_num = 4, .min_num = 1},
1412 	[RTW89_TXCH_CH12] = {.start_idx = 28, .max_num = 4, .min_num = 1},
1413 };
1414 EXPORT_SYMBOL(rtw89_bd_ram_table_single);
1415 
1416 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1417 {
1418 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1419 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1420 	const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table;
1421 	struct rtw89_pci_tx_ring *tx_ring;
1422 	struct rtw89_pci_rx_ring *rx_ring;
1423 	struct rtw89_pci_dma_ring *bd_ring;
1424 	const struct rtw89_pci_bd_ram *bd_ram;
1425 	u32 addr_num;
1426 	u32 addr_bdram;
1427 	u32 addr_desa_l;
1428 	u32 val32;
1429 	int i;
1430 
1431 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1432 		if (info->tx_dma_ch_mask & BIT(i))
1433 			continue;
1434 
1435 		tx_ring = &rtwpci->tx_rings[i];
1436 		bd_ring = &tx_ring->bd_ring;
1437 		bd_ram = &bd_ram_table[i];
1438 		addr_num = bd_ring->addr.num;
1439 		addr_bdram = bd_ring->addr.bdram;
1440 		addr_desa_l = bd_ring->addr.desa_l;
1441 		bd_ring->wp = 0;
1442 		bd_ring->rp = 0;
1443 
1444 		val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
1445 			FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
1446 			FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
1447 
1448 		rtw89_write16(rtwdev, addr_num, bd_ring->len);
1449 		rtw89_write32(rtwdev, addr_bdram, val32);
1450 		rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1451 	}
1452 
1453 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
1454 		rx_ring = &rtwpci->rx_rings[i];
1455 		bd_ring = &rx_ring->bd_ring;
1456 		addr_num = bd_ring->addr.num;
1457 		addr_desa_l = bd_ring->addr.desa_l;
1458 		bd_ring->wp = 0;
1459 		bd_ring->rp = 0;
1460 		rx_ring->diliver_skb = NULL;
1461 		rx_ring->diliver_desc.ready = false;
1462 
1463 		rtw89_write16(rtwdev, addr_num, bd_ring->len);
1464 		rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1465 	}
1466 }
1467 
1468 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1469 				      struct rtw89_pci_tx_ring *tx_ring)
1470 {
1471 	rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1472 	rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1473 }
1474 
1475 static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1476 {
1477 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1478 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1479 	int txch;
1480 
1481 	rtw89_pci_reset_trx_rings(rtwdev);
1482 
1483 	spin_lock_bh(&rtwpci->trx_lock);
1484 	for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1485 		if (info->tx_dma_ch_mask & BIT(txch))
1486 			continue;
1487 		if (txch == RTW89_TXCH_CH12) {
1488 			rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1489 						skb_queue_len(&rtwpci->h2c_queue), true);
1490 			continue;
1491 		}
1492 		rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]);
1493 	}
1494 	spin_unlock_bh(&rtwpci->trx_lock);
1495 }
1496 
1497 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1498 {
1499 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1500 	unsigned long flags;
1501 
1502 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
1503 	rtwpci->running = true;
1504 	rtw89_chip_enable_intr(rtwdev, rtwpci);
1505 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1506 }
1507 
1508 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1509 {
1510 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1511 	unsigned long flags;
1512 
1513 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
1514 	rtwpci->running = false;
1515 	rtw89_chip_disable_intr(rtwdev, rtwpci);
1516 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1517 }
1518 
1519 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1520 {
1521 	rtw89_core_napi_start(rtwdev);
1522 	rtw89_pci_enable_intr_lock(rtwdev);
1523 
1524 	return 0;
1525 }
1526 
1527 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1528 {
1529 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1530 	struct pci_dev *pdev = rtwpci->pdev;
1531 
1532 	rtw89_pci_disable_intr_lock(rtwdev);
1533 	synchronize_irq(pdev->irq);
1534 	rtw89_core_napi_stop(rtwdev);
1535 }
1536 
1537 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1538 {
1539 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1540 	struct pci_dev *pdev = rtwpci->pdev;
1541 
1542 	if (pause) {
1543 		rtw89_pci_disable_intr_lock(rtwdev);
1544 		synchronize_irq(pdev->irq);
1545 		if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1546 			napi_synchronize(&rtwdev->napi);
1547 	} else {
1548 		rtw89_pci_enable_intr_lock(rtwdev);
1549 		rtw89_pci_tx_kick_off_pending(rtwdev);
1550 	}
1551 }
1552 
1553 static
1554 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1555 {
1556 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1557 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1558 	const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power;
1559 	const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set;
1560 	struct rtw89_pci_tx_ring *tx_ring;
1561 	struct rtw89_pci_rx_ring *rx_ring;
1562 	int i;
1563 
1564 	if (WARN(!bd_idx_addr, "only HCI with low power mode needs this\n"))
1565 		return;
1566 
1567 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1568 		tx_ring = &rtwpci->tx_rings[i];
1569 		tx_ring->bd_ring.addr.idx = low_power ?
1570 					    bd_idx_addr->tx_bd_addrs[i] :
1571 					    dma_addr_set->tx[i].idx;
1572 	}
1573 
1574 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
1575 		rx_ring = &rtwpci->rx_rings[i];
1576 		rx_ring->bd_ring.addr.idx = low_power ?
1577 					    bd_idx_addr->rx_bd_addrs[i] :
1578 					    dma_addr_set->rx[i].idx;
1579 	}
1580 }
1581 
1582 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
1583 {
1584 	enum rtw89_pci_intr_mask_cfg cfg;
1585 
1586 	WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
1587 
1588 	cfg = low_power ? RTW89_PCI_INTR_MASK_LOW_POWER : RTW89_PCI_INTR_MASK_NORMAL;
1589 	rtw89_chip_config_intr_mask(rtwdev, cfg);
1590 	rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
1591 }
1592 
1593 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
1594 
1595 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
1596 {
1597 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1598 	u32 val = readl(rtwpci->mmap + addr);
1599 	int count;
1600 
1601 	for (count = 0; ; count++) {
1602 		if (val != RTW89_R32_DEAD)
1603 			return val;
1604 		if (count >= MAC_REG_POOL_COUNT) {
1605 			rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
1606 			return RTW89_R32_DEAD;
1607 		}
1608 		rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
1609 		val = readl(rtwpci->mmap + addr);
1610 	}
1611 
1612 	return val;
1613 }
1614 
1615 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
1616 {
1617 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1618 	u32 addr32, val32, shift;
1619 
1620 	if (!ACCESS_CMAC(addr))
1621 		return readb(rtwpci->mmap + addr);
1622 
1623 	addr32 = addr & ~0x3;
1624 	shift = (addr & 0x3) * 8;
1625 	val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1626 	return val32 >> shift;
1627 }
1628 
1629 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
1630 {
1631 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1632 	u32 addr32, val32, shift;
1633 
1634 	if (!ACCESS_CMAC(addr))
1635 		return readw(rtwpci->mmap + addr);
1636 
1637 	addr32 = addr & ~0x3;
1638 	shift = (addr & 0x3) * 8;
1639 	val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1640 	return val32 >> shift;
1641 }
1642 
1643 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
1644 {
1645 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1646 
1647 	if (!ACCESS_CMAC(addr))
1648 		return readl(rtwpci->mmap + addr);
1649 
1650 	return rtw89_pci_ops_read32_cmac(rtwdev, addr);
1651 }
1652 
1653 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
1654 {
1655 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1656 
1657 	writeb(data, rtwpci->mmap + addr);
1658 }
1659 
1660 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
1661 {
1662 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1663 
1664 	writew(data, rtwpci->mmap + addr);
1665 }
1666 
1667 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
1668 {
1669 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1670 
1671 	writel(data, rtwpci->mmap + addr);
1672 }
1673 
1674 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
1675 {
1676 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1677 
1678 	if (enable)
1679 		rtw89_write32_set(rtwdev, info->init_cfg_reg,
1680 				  info->rxhci_en_bit | info->txhci_en_bit);
1681 	else
1682 		rtw89_write32_clr(rtwdev, info->init_cfg_reg,
1683 				  info->rxhci_en_bit | info->txhci_en_bit);
1684 }
1685 
1686 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
1687 {
1688 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
1689 	u32 reg, mask;
1690 
1691 	if (chip_id == RTL8852C) {
1692 		reg = R_AX_HAXI_INIT_CFG1;
1693 		mask = B_AX_STOP_AXI_MST;
1694 	} else {
1695 		reg = R_AX_PCIE_DMA_STOP1;
1696 		mask = B_AX_STOP_PCIEIO;
1697 	}
1698 
1699 	if (enable)
1700 		rtw89_write32_clr(rtwdev, reg, mask);
1701 	else
1702 		rtw89_write32_set(rtwdev, reg, mask);
1703 }
1704 
1705 static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
1706 {
1707 	rtw89_pci_ctrl_dma_io(rtwdev, enable);
1708 	rtw89_pci_ctrl_dma_trx(rtwdev, enable);
1709 }
1710 
1711 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
1712 {
1713 	u16 val;
1714 
1715 	rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F);
1716 
1717 	val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
1718 	switch (speed) {
1719 	case PCIE_PHY_GEN1:
1720 		if (addr < 0x20)
1721 			val = u16_replace_bits(val, MDIO_PG0_G1, B_AX_MDIO_PHY_ADDR_MASK);
1722 		else
1723 			val = u16_replace_bits(val, MDIO_PG1_G1, B_AX_MDIO_PHY_ADDR_MASK);
1724 		break;
1725 	case PCIE_PHY_GEN2:
1726 		if (addr < 0x20)
1727 			val = u16_replace_bits(val, MDIO_PG0_G2, B_AX_MDIO_PHY_ADDR_MASK);
1728 		else
1729 			val = u16_replace_bits(val, MDIO_PG1_G2, B_AX_MDIO_PHY_ADDR_MASK);
1730 		break;
1731 	default:
1732 		rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
1733 		return -EINVAL;
1734 	}
1735 	rtw89_write16(rtwdev, R_AX_MDIO_CFG, val);
1736 	rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit);
1737 
1738 	return read_poll_timeout(rtw89_read16, val, !(val & rw_bit), 10, 2000,
1739 				 false, rtwdev, R_AX_MDIO_CFG);
1740 }
1741 
1742 static int
1743 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
1744 {
1745 	int ret;
1746 
1747 	ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
1748 	if (ret) {
1749 		rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
1750 		return ret;
1751 	}
1752 	*val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
1753 
1754 	return 0;
1755 }
1756 
1757 static int
1758 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
1759 {
1760 	int ret;
1761 
1762 	rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
1763 	ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
1764 	if (ret) {
1765 		rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
1766 		return ret;
1767 	}
1768 
1769 	return 0;
1770 }
1771 
1772 static int
1773 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
1774 {
1775 	u32 shift;
1776 	int ret;
1777 	u16 val;
1778 
1779 	ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1780 	if (ret)
1781 		return ret;
1782 
1783 	shift = __ffs(mask);
1784 	val &= ~mask;
1785 	val |= ((data << shift) & mask);
1786 
1787 	ret = rtw89_write16_mdio(rtwdev, addr, val, speed);
1788 	if (ret)
1789 		return ret;
1790 
1791 	return 0;
1792 }
1793 
1794 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1795 {
1796 	int ret;
1797 	u16 val;
1798 
1799 	ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1800 	if (ret)
1801 		return ret;
1802 	ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed);
1803 	if (ret)
1804 		return ret;
1805 
1806 	return 0;
1807 }
1808 
1809 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1810 {
1811 	int ret;
1812 	u16 val;
1813 
1814 	ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1815 	if (ret)
1816 		return ret;
1817 	ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed);
1818 	if (ret)
1819 		return ret;
1820 
1821 	return 0;
1822 }
1823 
1824 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
1825 				       u8 data)
1826 {
1827 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1828 	struct pci_dev *pdev = rtwpci->pdev;
1829 
1830 	return pci_write_config_byte(pdev, addr, data);
1831 }
1832 
1833 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
1834 				      u8 *value)
1835 {
1836 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1837 	struct pci_dev *pdev = rtwpci->pdev;
1838 
1839 	return pci_read_config_byte(pdev, addr, value);
1840 }
1841 
1842 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
1843 				     u8 bit)
1844 {
1845 	u8 value;
1846 	int ret;
1847 
1848 	ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
1849 	if (ret)
1850 		return ret;
1851 
1852 	value |= bit;
1853 	ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
1854 
1855 	return ret;
1856 }
1857 
1858 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
1859 				     u8 bit)
1860 {
1861 	u8 value;
1862 	int ret;
1863 
1864 	ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
1865 	if (ret)
1866 		return ret;
1867 
1868 	value &= ~bit;
1869 	ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
1870 
1871 	return ret;
1872 }
1873 
1874 static int
1875 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
1876 {
1877 	u16 val, tar;
1878 	int ret;
1879 
1880 	/* Enable counter */
1881 	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val);
1882 	if (ret)
1883 		return ret;
1884 	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
1885 				 phy_rate);
1886 	if (ret)
1887 		return ret;
1888 	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN,
1889 				 phy_rate);
1890 	if (ret)
1891 		return ret;
1892 
1893 	fsleep(300);
1894 
1895 	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar);
1896 	if (ret)
1897 		return ret;
1898 	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
1899 				 phy_rate);
1900 	if (ret)
1901 		return ret;
1902 
1903 	tar = tar & 0x0FFF;
1904 	if (tar == 0 || tar == 0x0FFF) {
1905 		rtw89_err(rtwdev, "[ERR]Get target failed.\n");
1906 		return -EINVAL;
1907 	}
1908 
1909 	*target = tar;
1910 
1911 	return 0;
1912 }
1913 
1914 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
1915 {
1916 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
1917 	int ret;
1918 
1919 	if (chip_id != RTL8852B && chip_id != RTL8851B)
1920 		return 0;
1921 
1922 	ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
1923 				      PCIE_AUTOK_4, PCIE_PHY_GEN1);
1924 	return ret;
1925 }
1926 
1927 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
1928 {
1929 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
1930 	enum rtw89_pcie_phy phy_rate;
1931 	u16 val16, mgn_set, div_set, tar;
1932 	u8 val8, bdr_ori;
1933 	bool l1_flag = false;
1934 	int ret = 0;
1935 
1936 	if (chip_id != RTL8852B && chip_id != RTL8851B)
1937 		return 0;
1938 
1939 	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
1940 	if (ret) {
1941 		rtw89_err(rtwdev, "[ERR]pci config read %X\n",
1942 			  RTW89_PCIE_PHY_RATE);
1943 		return ret;
1944 	}
1945 
1946 	if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x1) {
1947 		phy_rate = PCIE_PHY_GEN1;
1948 	} else if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x2) {
1949 		phy_rate = PCIE_PHY_GEN2;
1950 	} else {
1951 		rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
1952 		return -EOPNOTSUPP;
1953 	}
1954 	/* Disable L1BD */
1955 	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori);
1956 	if (ret) {
1957 		rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
1958 		return ret;
1959 	}
1960 
1961 	if (bdr_ori & RTW89_PCIE_BIT_L1) {
1962 		ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
1963 						  bdr_ori & ~RTW89_PCIE_BIT_L1);
1964 		if (ret) {
1965 			rtw89_err(rtwdev, "[ERR]pci config write %X\n",
1966 				  RTW89_PCIE_L1_CTRL);
1967 			return ret;
1968 		}
1969 		l1_flag = true;
1970 	}
1971 
1972 	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
1973 	if (ret) {
1974 		rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
1975 		goto end;
1976 	}
1977 
1978 	if (val16 & B_AX_CALIB_EN) {
1979 		ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
1980 					 val16 & ~B_AX_CALIB_EN, phy_rate);
1981 		if (ret) {
1982 			rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
1983 			goto end;
1984 		}
1985 	}
1986 
1987 	if (!autook_en)
1988 		goto end;
1989 	/* Set div */
1990 	ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate);
1991 	if (ret) {
1992 		rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
1993 		goto end;
1994 	}
1995 
1996 	/* Obtain div and margin */
1997 	ret = __get_target(rtwdev, &tar, phy_rate);
1998 	if (ret) {
1999 		rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
2000 		goto end;
2001 	}
2002 
2003 	mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar;
2004 
2005 	if (mgn_set >= 128) {
2006 		div_set = 0x0003;
2007 		mgn_set = 0x000F;
2008 	} else if (mgn_set >= 64) {
2009 		div_set = 0x0003;
2010 		mgn_set >>= 3;
2011 	} else if (mgn_set >= 32) {
2012 		div_set = 0x0002;
2013 		mgn_set >>= 2;
2014 	} else if (mgn_set >= 16) {
2015 		div_set = 0x0001;
2016 		mgn_set >>= 1;
2017 	} else if (mgn_set == 0) {
2018 		rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
2019 		goto end;
2020 	} else {
2021 		div_set = 0x0000;
2022 	}
2023 
2024 	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2025 	if (ret) {
2026 		rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2027 		goto end;
2028 	}
2029 
2030 	val16 |= u16_encode_bits(div_set, B_AX_DIV);
2031 
2032 	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate);
2033 	if (ret) {
2034 		rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2035 		goto end;
2036 	}
2037 
2038 	ret = __get_target(rtwdev, &tar, phy_rate);
2039 	if (ret) {
2040 		rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2041 		goto end;
2042 	}
2043 
2044 	rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2045 		    tar, div_set, mgn_set);
2046 	ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2047 				 (tar & 0x0FFF) | (mgn_set << 12), phy_rate);
2048 	if (ret) {
2049 		rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2050 		goto end;
2051 	}
2052 
2053 	/* Enable function */
2054 	ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate);
2055 	if (ret) {
2056 		rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2057 		goto end;
2058 	}
2059 
2060 	/* CLK delay = 0 */
2061 	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2062 					  PCIE_CLKDLY_HW_0);
2063 
2064 end:
2065 	/* Set L1BD to ori */
2066 	if (l1_flag) {
2067 		ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2068 						  bdr_ori);
2069 		if (ret) {
2070 			rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2071 				  RTW89_PCIE_L1_CTRL);
2072 			return ret;
2073 		}
2074 	}
2075 
2076 	return ret;
2077 }
2078 
2079 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2080 {
2081 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2082 	int ret;
2083 
2084 	if (chip_id == RTL8852A) {
2085 		ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2086 					     PCIE_PHY_GEN1);
2087 		if (ret)
2088 			return ret;
2089 		ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2090 					     PCIE_PHY_GEN2);
2091 		if (ret)
2092 			return ret;
2093 	} else if (chip_id == RTL8852C) {
2094 		rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2095 				  B_AX_DEGLITCH);
2096 		rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2097 				  B_AX_DEGLITCH);
2098 	}
2099 
2100 	return 0;
2101 }
2102 
2103 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2104 {
2105 	if (rtwdev->chip->chip_id != RTL8852A)
2106 		return;
2107 
2108 	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2109 }
2110 
2111 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2112 {
2113 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2114 
2115 	if (chip_id != RTL8852A && chip_id != RTL8852B && chip_id != RTL8851B)
2116 		return;
2117 
2118 	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2119 }
2120 
2121 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2122 {
2123 	int ret;
2124 
2125 	if (rtwdev->chip->chip_id != RTL8852A)
2126 		return 0;
2127 
2128 	ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2129 				     PCIE_PHY_GEN1);
2130 	if (ret)
2131 		return ret;
2132 
2133 	ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2134 				     PCIE_PHY_GEN2);
2135 	if (ret)
2136 		return ret;
2137 
2138 	return 0;
2139 }
2140 
2141 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2142 {
2143 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2144 
2145 	if (chip_id != RTL8852A && chip_id != RTL8852B && chip_id != RTL8851B)
2146 		return;
2147 
2148 	rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2149 }
2150 
2151 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2152 {
2153 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2154 
2155 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
2156 		rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2157 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2158 		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2159 				  B_AX_PCIE_DIS_WLSUS_AFT_PDN);
2160 	} else if (rtwdev->chip->chip_id == RTL8852C) {
2161 		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2162 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2163 	}
2164 }
2165 
2166 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2167 {
2168 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2169 
2170 	if (chip_id != RTL8852B && chip_id != RTL8851B)
2171 		return 0;
2172 
2173 	return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2174 				       PCIE_DPHY_DLY_25US, PCIE_PHY_GEN1);
2175 }
2176 
2177 static void rtw89_pci_power_wake(struct rtw89_dev *rtwdev, bool pwr_up)
2178 {
2179 	if (pwr_up)
2180 		rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2181 	else
2182 		rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2183 }
2184 
2185 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2186 {
2187 	if (rtwdev->chip->chip_id != RTL8852C)
2188 		return;
2189 
2190 	rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2191 	rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2192 }
2193 
2194 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2195 {
2196 	if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2197 		return;
2198 
2199 	rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2200 }
2201 
2202 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2203 {
2204 	if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2205 		return;
2206 
2207 	rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2208 			  B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2209 	rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2210 	rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2211 			  B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2212 }
2213 
2214 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2215 {
2216 	if (rtwdev->chip->chip_id != RTL8852C)
2217 		return;
2218 
2219 	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2220 }
2221 
2222 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2223 {
2224 	if (rtwdev->chip->chip_id != RTL8852C)
2225 		return;
2226 
2227 	rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2228 }
2229 
2230 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2231 {
2232 	if (rtwdev->chip->chip_id == RTL8852C)
2233 		return;
2234 
2235 	rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2236 			  B_AX_SIC_EN_FORCE_CLKREQ);
2237 }
2238 
2239 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2240 {
2241 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2242 	u32 lbc;
2243 
2244 	if (rtwdev->chip->chip_id == RTL8852C)
2245 		return;
2246 
2247 	lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2248 	if (info->lbc_en == MAC_AX_PCIE_ENABLE) {
2249 		lbc = u32_replace_bits(lbc, info->lbc_tmr, B_AX_LBC_TIMER);
2250 		lbc |= B_AX_LBC_FLAG | B_AX_LBC_EN;
2251 		rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2252 	} else {
2253 		lbc &= ~B_AX_LBC_EN;
2254 	}
2255 	rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2256 }
2257 
2258 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2259 {
2260 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2261 	u32 val32;
2262 
2263 	if (rtwdev->chip->chip_id != RTL8852C)
2264 		return;
2265 
2266 	if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) {
2267 		val32 = FIELD_PREP(B_AX_PCIE_WDT_TIMER_M1_MASK,
2268 				   info->io_rcy_tmr);
2269 		rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32);
2270 		rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32);
2271 		rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32);
2272 
2273 		rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2274 		rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2275 		rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2276 	} else {
2277 		rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2278 		rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2279 		rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2280 	}
2281 
2282 	rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2283 }
2284 
2285 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2286 {
2287 	if (rtwdev->chip->chip_id == RTL8852C)
2288 		return;
2289 
2290 	rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2291 			  B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG);
2292 
2293 	if (rtwdev->chip->chip_id == RTL8852A)
2294 		rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2295 				  B_AX_EN_CHKDSC_NO_RX_STUCK);
2296 }
2297 
2298 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2299 {
2300 	if (rtwdev->chip->chip_id == RTL8852C)
2301 		return;
2302 
2303 	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2304 			  B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG);
2305 }
2306 
2307 static void rtw89_pci_clr_idx_all(struct rtw89_dev *rtwdev)
2308 {
2309 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2310 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2311 	u32 val = B_AX_CLR_ACH0_IDX | B_AX_CLR_ACH1_IDX | B_AX_CLR_ACH2_IDX |
2312 		  B_AX_CLR_ACH3_IDX | B_AX_CLR_CH8_IDX | B_AX_CLR_CH9_IDX |
2313 		  B_AX_CLR_CH12_IDX;
2314 	u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg;
2315 	u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg;
2316 
2317 	if (chip_id == RTL8852A || chip_id == RTL8852C)
2318 		val |= B_AX_CLR_ACH4_IDX | B_AX_CLR_ACH5_IDX |
2319 		       B_AX_CLR_ACH6_IDX | B_AX_CLR_ACH7_IDX;
2320 	/* clear DMA indexes */
2321 	rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val);
2322 	if (chip_id == RTL8852A || chip_id == RTL8852C)
2323 		rtw89_write32_set(rtwdev, txbd_rwptr_clr2,
2324 				  B_AX_CLR_CH10_IDX | B_AX_CLR_CH11_IDX);
2325 	rtw89_write32_set(rtwdev, rxbd_rwptr_clr,
2326 			  B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX);
2327 }
2328 
2329 static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
2330 {
2331 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2332 	u32 ret, check, dma_busy;
2333 	u32 dma_busy1 = info->dma_busy1.addr;
2334 	u32 dma_busy2 = info->dma_busy2_reg;
2335 
2336 	check = info->dma_busy1.mask;
2337 
2338 	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2339 				10, 100, false, rtwdev, dma_busy1);
2340 	if (ret)
2341 		return ret;
2342 
2343 	if (!dma_busy2)
2344 		return 0;
2345 
2346 	check = B_AX_CH10_BUSY | B_AX_CH11_BUSY;
2347 
2348 	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2349 				10, 100, false, rtwdev, dma_busy2);
2350 	if (ret)
2351 		return ret;
2352 
2353 	return 0;
2354 }
2355 
2356 static int rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
2357 {
2358 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2359 	u32 ret, check, dma_busy;
2360 	u32 dma_busy3 = info->dma_busy3_reg;
2361 
2362 	check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY;
2363 
2364 	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2365 				10, 100, false, rtwdev, dma_busy3);
2366 	if (ret)
2367 		return ret;
2368 
2369 	return 0;
2370 }
2371 
2372 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
2373 {
2374 	u32 ret;
2375 
2376 	ret = rtw89_poll_txdma_ch_idle_pcie(rtwdev);
2377 	if (ret) {
2378 		rtw89_err(rtwdev, "txdma ch busy\n");
2379 		return ret;
2380 	}
2381 
2382 	ret = rtw89_poll_rxdma_ch_idle_pcie(rtwdev);
2383 	if (ret) {
2384 		rtw89_err(rtwdev, "rxdma ch busy\n");
2385 		return ret;
2386 	}
2387 
2388 	return 0;
2389 }
2390 
2391 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
2392 {
2393 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2394 	enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode;
2395 	enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode;
2396 	enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode;
2397 	enum mac_ax_tag_mode tag_mode = info->tag_mode;
2398 	enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl;
2399 	enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl;
2400 	enum mac_ax_tx_burst tx_burst = info->tx_burst;
2401 	enum mac_ax_rx_burst rx_burst = info->rx_burst;
2402 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2403 	u8 cv = rtwdev->hal.cv;
2404 	u32 val32;
2405 
2406 	if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2407 		if (chip_id == RTL8852A && cv == CHIP_CBV)
2408 			rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2409 	} else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2410 		if (chip_id == RTL8852A || chip_id == RTL8852B)
2411 			rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2412 	}
2413 
2414 	if (rxbd_trunc_mode == MAC_AX_BD_TRUNC) {
2415 		if (chip_id == RTL8852A && cv == CHIP_CBV)
2416 			rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2417 	} else if (rxbd_trunc_mode == MAC_AX_BD_NORM) {
2418 		if (chip_id == RTL8852A || chip_id == RTL8852B)
2419 			rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2420 	}
2421 
2422 	if (rxbd_mode == MAC_AX_RXBD_PKT) {
2423 		rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2424 	} else if (rxbd_mode == MAC_AX_RXBD_SEP) {
2425 		rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2426 
2427 		if (chip_id == RTL8852A || chip_id == RTL8852B)
2428 			rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
2429 					   B_AX_PCIE_RX_APPLEN_MASK, 0);
2430 	}
2431 
2432 	if (chip_id == RTL8852A || chip_id == RTL8852B) {
2433 		rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst);
2434 		rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst);
2435 	} else if (chip_id == RTL8852C) {
2436 		rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst);
2437 		rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst);
2438 	}
2439 
2440 	if (chip_id == RTL8852A || chip_id == RTL8852B) {
2441 		if (tag_mode == MAC_AX_TAG_SGL) {
2442 			val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
2443 					    ~B_AX_LATENCY_CONTROL;
2444 			rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2445 		} else if (tag_mode == MAC_AX_TAG_MULTI) {
2446 			val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
2447 					    B_AX_LATENCY_CONTROL;
2448 			rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2449 		}
2450 	}
2451 
2452 	rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask,
2453 			   info->multi_tag_num);
2454 
2455 	if (chip_id == RTL8852A || chip_id == RTL8852B) {
2456 		rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
2457 				   wd_dma_idle_intvl);
2458 		rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
2459 				   wd_dma_act_intvl);
2460 	} else if (chip_id == RTL8852C) {
2461 		rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
2462 				   wd_dma_idle_intvl);
2463 		rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
2464 				   wd_dma_act_intvl);
2465 	}
2466 
2467 	if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2468 		rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2469 				  B_AX_HOST_ADDR_INFO_8B_SEL);
2470 		rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2471 	} else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2472 		rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2473 				  B_AX_HOST_ADDR_INFO_8B_SEL);
2474 		rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2475 	}
2476 
2477 	return 0;
2478 }
2479 
2480 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
2481 {
2482 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2483 
2484 	if (rtwdev->chip->chip_id == RTL8852A) {
2485 		/* ltr sw trigger */
2486 		rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
2487 	}
2488 	info->ltr_set(rtwdev, false);
2489 	rtw89_pci_ctrl_dma_all(rtwdev, false);
2490 	rtw89_pci_clr_idx_all(rtwdev);
2491 
2492 	return 0;
2493 }
2494 
2495 static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
2496 {
2497 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2498 	int ret;
2499 
2500 	rtw89_pci_rxdma_prefth(rtwdev);
2501 	rtw89_pci_l1off_pwroff(rtwdev);
2502 	rtw89_pci_deglitch_setting(rtwdev);
2503 	ret = rtw89_pci_l2_rxen_lat(rtwdev);
2504 	if (ret) {
2505 		rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
2506 		return ret;
2507 	}
2508 
2509 	rtw89_pci_aphy_pwrcut(rtwdev);
2510 	rtw89_pci_hci_ldo(rtwdev);
2511 	rtw89_pci_dphy_delay(rtwdev);
2512 
2513 	ret = rtw89_pci_autok_x(rtwdev);
2514 	if (ret) {
2515 		rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
2516 		return ret;
2517 	}
2518 
2519 	ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
2520 	if (ret) {
2521 		rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
2522 		return ret;
2523 	}
2524 
2525 	rtw89_pci_power_wake(rtwdev, true);
2526 	rtw89_pci_autoload_hang(rtwdev);
2527 	rtw89_pci_l12_vmain(rtwdev);
2528 	rtw89_pci_gen2_force_ib(rtwdev);
2529 	rtw89_pci_l1_ent_lat(rtwdev);
2530 	rtw89_pci_wd_exit_l1(rtwdev);
2531 	rtw89_pci_set_sic(rtwdev);
2532 	rtw89_pci_set_lbc(rtwdev);
2533 	rtw89_pci_set_io_rcy(rtwdev);
2534 	rtw89_pci_set_dbg(rtwdev);
2535 	rtw89_pci_set_keep_reg(rtwdev);
2536 
2537 	rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
2538 
2539 	/* stop DMA activities */
2540 	rtw89_pci_ctrl_dma_all(rtwdev, false);
2541 
2542 	ret = rtw89_pci_poll_dma_all_idle(rtwdev);
2543 	if (ret) {
2544 		rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
2545 		return ret;
2546 	}
2547 
2548 	rtw89_pci_clr_idx_all(rtwdev);
2549 	rtw89_pci_mode_op(rtwdev);
2550 
2551 	/* fill TRX BD indexes */
2552 	rtw89_pci_ops_reset(rtwdev);
2553 
2554 	ret = rtw89_pci_rst_bdram_pcie(rtwdev);
2555 	if (ret) {
2556 		rtw89_warn(rtwdev, "reset bdram busy\n");
2557 		return ret;
2558 	}
2559 
2560 	/* disable all channels except to FW CMD channel to download firmware */
2561 	rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, false);
2562 	rtw89_pci_ctrl_txdma_fw_ch_pcie(rtwdev, true);
2563 
2564 	/* start DMA activities */
2565 	rtw89_pci_ctrl_dma_all(rtwdev, true);
2566 
2567 	return 0;
2568 }
2569 
2570 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
2571 {
2572 	u32 val;
2573 
2574 	if (!en)
2575 		return 0;
2576 
2577 	val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2578 	if (rtw89_pci_ltr_is_err_reg_val(val))
2579 		return -EINVAL;
2580 	val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2581 	if (rtw89_pci_ltr_is_err_reg_val(val))
2582 		return -EINVAL;
2583 	val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
2584 	if (rtw89_pci_ltr_is_err_reg_val(val))
2585 		return -EINVAL;
2586 	val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
2587 	if (rtw89_pci_ltr_is_err_reg_val(val))
2588 		return -EINVAL;
2589 
2590 	rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
2591 						   B_AX_LTR_WD_NOEMP_CHK);
2592 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
2593 			   PCI_LTR_SPC_500US);
2594 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2595 			   PCI_LTR_IDLE_TIMER_3_2MS);
2596 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2597 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2598 	rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
2599 	rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
2600 
2601 	return 0;
2602 }
2603 EXPORT_SYMBOL(rtw89_pci_ltr_set);
2604 
2605 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
2606 {
2607 	u32 dec_ctrl;
2608 	u32 val32;
2609 
2610 	val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2611 	if (rtw89_pci_ltr_is_err_reg_val(val32))
2612 		return -EINVAL;
2613 	val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2614 	if (rtw89_pci_ltr_is_err_reg_val(val32))
2615 		return -EINVAL;
2616 	dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
2617 	if (rtw89_pci_ltr_is_err_reg_val(dec_ctrl))
2618 		return -EINVAL;
2619 	val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
2620 	if (rtw89_pci_ltr_is_err_reg_val(val32))
2621 		return -EINVAL;
2622 	val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
2623 	if (rtw89_pci_ltr_is_err_reg_val(val32))
2624 		return -EINVAL;
2625 
2626 	if (!en) {
2627 		dec_ctrl &= ~(LTR_EN_BITS | B_AX_LTR_IDX_DRV_MASK | B_AX_LTR_HW_DEC_EN);
2628 		dec_ctrl |= FIELD_PREP(B_AX_LTR_IDX_DRV_MASK, PCIE_LTR_IDX_IDLE) |
2629 			    B_AX_LTR_REQ_DRV;
2630 	} else {
2631 		dec_ctrl |= B_AX_LTR_HW_DEC_EN;
2632 	}
2633 
2634 	dec_ctrl &= ~B_AX_LTR_SPACE_IDX_V1_MASK;
2635 	dec_ctrl |= FIELD_PREP(B_AX_LTR_SPACE_IDX_V1_MASK, PCI_LTR_SPC_500US);
2636 
2637 	if (en)
2638 		rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
2639 				  B_AX_LTR_WD_NOEMP_CHK_V1 | B_AX_LTR_HW_EN);
2640 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2641 			   PCI_LTR_IDLE_TIMER_3_2MS);
2642 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2643 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2644 	rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl);
2645 	rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003);
2646 	rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b);
2647 
2648 	return 0;
2649 }
2650 EXPORT_SYMBOL(rtw89_pci_ltr_set_v1);
2651 
2652 static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev)
2653 {
2654 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2655 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2656 	int ret;
2657 
2658 	ret = info->ltr_set(rtwdev, true);
2659 	if (ret) {
2660 		rtw89_err(rtwdev, "pci ltr set fail\n");
2661 		return ret;
2662 	}
2663 	if (chip_id == RTL8852A) {
2664 		/* ltr sw trigger */
2665 		rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
2666 	}
2667 	if (chip_id == RTL8852A || chip_id == RTL8852B) {
2668 		/* ADDR info 8-byte mode */
2669 		rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2670 				  B_AX_HOST_ADDR_INFO_8B_SEL);
2671 		rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2672 	}
2673 
2674 	/* enable DMA for all queues */
2675 	rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, true);
2676 
2677 	/* Release PCI IO */
2678 	rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
2679 			  B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO);
2680 
2681 	return 0;
2682 }
2683 
2684 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
2685 				  struct pci_dev *pdev)
2686 {
2687 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2688 	int ret;
2689 
2690 	ret = pci_enable_device(pdev);
2691 	if (ret) {
2692 		rtw89_err(rtwdev, "failed to enable pci device\n");
2693 		return ret;
2694 	}
2695 
2696 	pci_set_master(pdev);
2697 	pci_set_drvdata(pdev, rtwdev->hw);
2698 
2699 	rtwpci->pdev = pdev;
2700 
2701 	return 0;
2702 }
2703 
2704 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
2705 				     struct pci_dev *pdev)
2706 {
2707 	pci_disable_device(pdev);
2708 }
2709 
2710 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
2711 				   struct pci_dev *pdev)
2712 {
2713 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2714 	unsigned long resource_len;
2715 	u8 bar_id = 2;
2716 	int ret;
2717 
2718 	ret = pci_request_regions(pdev, KBUILD_MODNAME);
2719 	if (ret) {
2720 		rtw89_err(rtwdev, "failed to request pci regions\n");
2721 		goto err;
2722 	}
2723 
2724 	ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2725 	if (ret) {
2726 		rtw89_err(rtwdev, "failed to set dma mask to 32-bit\n");
2727 		goto err_release_regions;
2728 	}
2729 
2730 	ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
2731 	if (ret) {
2732 		rtw89_err(rtwdev, "failed to set consistent dma mask to 32-bit\n");
2733 		goto err_release_regions;
2734 	}
2735 
2736 	resource_len = pci_resource_len(pdev, bar_id);
2737 	rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len);
2738 	if (!rtwpci->mmap) {
2739 		rtw89_err(rtwdev, "failed to map pci io\n");
2740 		ret = -EIO;
2741 		goto err_release_regions;
2742 	}
2743 
2744 	return 0;
2745 
2746 err_release_regions:
2747 	pci_release_regions(pdev);
2748 err:
2749 	return ret;
2750 }
2751 
2752 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
2753 				    struct pci_dev *pdev)
2754 {
2755 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2756 
2757 	if (rtwpci->mmap) {
2758 		pci_iounmap(pdev, rtwpci->mmap);
2759 		pci_release_regions(pdev);
2760 	}
2761 }
2762 
2763 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
2764 				      struct pci_dev *pdev,
2765 				      struct rtw89_pci_tx_ring *tx_ring)
2766 {
2767 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
2768 	u8 *head = wd_ring->head;
2769 	dma_addr_t dma = wd_ring->dma;
2770 	u32 page_size = wd_ring->page_size;
2771 	u32 page_num = wd_ring->page_num;
2772 	u32 ring_sz = page_size * page_num;
2773 
2774 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2775 	wd_ring->head = NULL;
2776 }
2777 
2778 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
2779 				   struct pci_dev *pdev,
2780 				   struct rtw89_pci_tx_ring *tx_ring)
2781 {
2782 	int ring_sz;
2783 	u8 *head;
2784 	dma_addr_t dma;
2785 
2786 	head = tx_ring->bd_ring.head;
2787 	dma = tx_ring->bd_ring.dma;
2788 	ring_sz = tx_ring->bd_ring.desc_size * tx_ring->bd_ring.len;
2789 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2790 
2791 	tx_ring->bd_ring.head = NULL;
2792 }
2793 
2794 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
2795 				    struct pci_dev *pdev)
2796 {
2797 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2798 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2799 	struct rtw89_pci_tx_ring *tx_ring;
2800 	int i;
2801 
2802 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
2803 		if (info->tx_dma_ch_mask & BIT(i))
2804 			continue;
2805 		tx_ring = &rtwpci->tx_rings[i];
2806 		rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
2807 		rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
2808 	}
2809 }
2810 
2811 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
2812 				   struct pci_dev *pdev,
2813 				   struct rtw89_pci_rx_ring *rx_ring)
2814 {
2815 	struct rtw89_pci_rx_info *rx_info;
2816 	struct sk_buff *skb;
2817 	dma_addr_t dma;
2818 	u32 buf_sz;
2819 	u8 *head;
2820 	int ring_sz = rx_ring->bd_ring.desc_size * rx_ring->bd_ring.len;
2821 	int i;
2822 
2823 	buf_sz = rx_ring->buf_sz;
2824 	for (i = 0; i < rx_ring->bd_ring.len; i++) {
2825 		skb = rx_ring->buf[i];
2826 		if (!skb)
2827 			continue;
2828 
2829 		rx_info = RTW89_PCI_RX_SKB_CB(skb);
2830 		dma = rx_info->dma;
2831 		dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
2832 		dev_kfree_skb(skb);
2833 		rx_ring->buf[i] = NULL;
2834 	}
2835 
2836 	head = rx_ring->bd_ring.head;
2837 	dma = rx_ring->bd_ring.dma;
2838 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2839 
2840 	rx_ring->bd_ring.head = NULL;
2841 }
2842 
2843 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
2844 				    struct pci_dev *pdev)
2845 {
2846 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2847 	struct rtw89_pci_rx_ring *rx_ring;
2848 	int i;
2849 
2850 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
2851 		rx_ring = &rtwpci->rx_rings[i];
2852 		rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
2853 	}
2854 }
2855 
2856 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
2857 				     struct pci_dev *pdev)
2858 {
2859 	rtw89_pci_free_rx_rings(rtwdev, pdev);
2860 	rtw89_pci_free_tx_rings(rtwdev, pdev);
2861 }
2862 
2863 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
2864 				struct rtw89_pci_rx_ring *rx_ring,
2865 				struct sk_buff *skb, int buf_sz, u32 idx)
2866 {
2867 	struct rtw89_pci_rx_info *rx_info;
2868 	struct rtw89_pci_rx_bd_32 *rx_bd;
2869 	dma_addr_t dma;
2870 
2871 	if (!skb)
2872 		return -EINVAL;
2873 
2874 	dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
2875 	if (dma_mapping_error(&pdev->dev, dma))
2876 		return -EBUSY;
2877 
2878 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
2879 	rx_bd = RTW89_PCI_RX_BD(rx_ring, idx);
2880 
2881 	memset(rx_bd, 0, sizeof(*rx_bd));
2882 	rx_bd->buf_size = cpu_to_le16(buf_sz);
2883 	rx_bd->dma = cpu_to_le32(dma);
2884 	rx_info->dma = dma;
2885 
2886 	return 0;
2887 }
2888 
2889 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
2890 				      struct pci_dev *pdev,
2891 				      struct rtw89_pci_tx_ring *tx_ring,
2892 				      enum rtw89_tx_channel txch)
2893 {
2894 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
2895 	struct rtw89_pci_tx_wd *txwd;
2896 	dma_addr_t dma;
2897 	dma_addr_t cur_paddr;
2898 	u8 *head;
2899 	u8 *cur_vaddr;
2900 	u32 page_size = RTW89_PCI_TXWD_PAGE_SIZE;
2901 	u32 page_num = RTW89_PCI_TXWD_NUM_MAX;
2902 	u32 ring_sz = page_size * page_num;
2903 	u32 page_offset;
2904 	int i;
2905 
2906 	/* FWCMD queue doesn't use txwd as pages */
2907 	if (txch == RTW89_TXCH_CH12)
2908 		return 0;
2909 
2910 	head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
2911 	if (!head)
2912 		return -ENOMEM;
2913 
2914 	INIT_LIST_HEAD(&wd_ring->free_pages);
2915 	wd_ring->head = head;
2916 	wd_ring->dma = dma;
2917 	wd_ring->page_size = page_size;
2918 	wd_ring->page_num = page_num;
2919 
2920 	page_offset = 0;
2921 	for (i = 0; i < page_num; i++) {
2922 		txwd = &wd_ring->pages[i];
2923 		cur_paddr = dma + page_offset;
2924 		cur_vaddr = head + page_offset;
2925 
2926 		skb_queue_head_init(&txwd->queue);
2927 		INIT_LIST_HEAD(&txwd->list);
2928 		txwd->paddr = cur_paddr;
2929 		txwd->vaddr = cur_vaddr;
2930 		txwd->len = page_size;
2931 		txwd->seq = i;
2932 		rtw89_pci_enqueue_txwd(tx_ring, txwd);
2933 
2934 		page_offset += page_size;
2935 	}
2936 
2937 	return 0;
2938 }
2939 
2940 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
2941 				   struct pci_dev *pdev,
2942 				   struct rtw89_pci_tx_ring *tx_ring,
2943 				   u32 desc_size, u32 len,
2944 				   enum rtw89_tx_channel txch)
2945 {
2946 	const struct rtw89_pci_ch_dma_addr *txch_addr;
2947 	int ring_sz = desc_size * len;
2948 	u8 *head;
2949 	dma_addr_t dma;
2950 	int ret;
2951 
2952 	ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
2953 	if (ret) {
2954 		rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
2955 		goto err;
2956 	}
2957 
2958 	ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr);
2959 	if (ret) {
2960 		rtw89_err(rtwdev, "failed to get address of txch %d", txch);
2961 		goto err_free_wd_ring;
2962 	}
2963 
2964 	head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
2965 	if (!head) {
2966 		ret = -ENOMEM;
2967 		goto err_free_wd_ring;
2968 	}
2969 
2970 	INIT_LIST_HEAD(&tx_ring->busy_pages);
2971 	tx_ring->bd_ring.head = head;
2972 	tx_ring->bd_ring.dma = dma;
2973 	tx_ring->bd_ring.len = len;
2974 	tx_ring->bd_ring.desc_size = desc_size;
2975 	tx_ring->bd_ring.addr = *txch_addr;
2976 	tx_ring->bd_ring.wp = 0;
2977 	tx_ring->bd_ring.rp = 0;
2978 	tx_ring->txch = txch;
2979 
2980 	return 0;
2981 
2982 err_free_wd_ring:
2983 	rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
2984 err:
2985 	return ret;
2986 }
2987 
2988 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
2989 				    struct pci_dev *pdev)
2990 {
2991 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2992 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2993 	struct rtw89_pci_tx_ring *tx_ring;
2994 	u32 desc_size;
2995 	u32 len;
2996 	u32 i, tx_allocated;
2997 	int ret;
2998 
2999 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
3000 		if (info->tx_dma_ch_mask & BIT(i))
3001 			continue;
3002 		tx_ring = &rtwpci->tx_rings[i];
3003 		desc_size = sizeof(struct rtw89_pci_tx_bd_32);
3004 		len = RTW89_PCI_TXBD_NUM_MAX;
3005 		ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
3006 					      desc_size, len, i);
3007 		if (ret) {
3008 			rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
3009 			goto err_free;
3010 		}
3011 	}
3012 
3013 	return 0;
3014 
3015 err_free:
3016 	tx_allocated = i;
3017 	for (i = 0; i < tx_allocated; i++) {
3018 		tx_ring = &rtwpci->tx_rings[i];
3019 		rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3020 	}
3021 
3022 	return ret;
3023 }
3024 
3025 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
3026 				   struct pci_dev *pdev,
3027 				   struct rtw89_pci_rx_ring *rx_ring,
3028 				   u32 desc_size, u32 len, u32 rxch)
3029 {
3030 	const struct rtw89_pci_ch_dma_addr *rxch_addr;
3031 	struct sk_buff *skb;
3032 	u8 *head;
3033 	dma_addr_t dma;
3034 	int ring_sz = desc_size * len;
3035 	int buf_sz = RTW89_PCI_RX_BUF_SIZE;
3036 	int i, allocated;
3037 	int ret;
3038 
3039 	ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr);
3040 	if (ret) {
3041 		rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
3042 		return ret;
3043 	}
3044 
3045 	head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3046 	if (!head) {
3047 		ret = -ENOMEM;
3048 		goto err;
3049 	}
3050 
3051 	rx_ring->bd_ring.head = head;
3052 	rx_ring->bd_ring.dma = dma;
3053 	rx_ring->bd_ring.len = len;
3054 	rx_ring->bd_ring.desc_size = desc_size;
3055 	rx_ring->bd_ring.addr = *rxch_addr;
3056 	rx_ring->bd_ring.wp = 0;
3057 	rx_ring->bd_ring.rp = 0;
3058 	rx_ring->buf_sz = buf_sz;
3059 	rx_ring->diliver_skb = NULL;
3060 	rx_ring->diliver_desc.ready = false;
3061 
3062 	for (i = 0; i < len; i++) {
3063 		skb = dev_alloc_skb(buf_sz);
3064 		if (!skb) {
3065 			ret = -ENOMEM;
3066 			goto err_free;
3067 		}
3068 
3069 		memset(skb->data, 0, buf_sz);
3070 		rx_ring->buf[i] = skb;
3071 		ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3072 					   buf_sz, i);
3073 		if (ret) {
3074 			rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3075 			dev_kfree_skb_any(skb);
3076 			rx_ring->buf[i] = NULL;
3077 			goto err_free;
3078 		}
3079 	}
3080 
3081 	return 0;
3082 
3083 err_free:
3084 	allocated = i;
3085 	for (i = 0; i < allocated; i++) {
3086 		skb = rx_ring->buf[i];
3087 		if (!skb)
3088 			continue;
3089 		dma = *((dma_addr_t *)skb->cb);
3090 		dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3091 		dev_kfree_skb(skb);
3092 		rx_ring->buf[i] = NULL;
3093 	}
3094 
3095 	head = rx_ring->bd_ring.head;
3096 	dma = rx_ring->bd_ring.dma;
3097 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3098 
3099 	rx_ring->bd_ring.head = NULL;
3100 err:
3101 	return ret;
3102 }
3103 
3104 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3105 				    struct pci_dev *pdev)
3106 {
3107 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3108 	struct rtw89_pci_rx_ring *rx_ring;
3109 	u32 desc_size;
3110 	u32 len;
3111 	int i, rx_allocated;
3112 	int ret;
3113 
3114 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
3115 		rx_ring = &rtwpci->rx_rings[i];
3116 		desc_size = sizeof(struct rtw89_pci_rx_bd_32);
3117 		len = RTW89_PCI_RXBD_NUM_MAX;
3118 		ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3119 					      desc_size, len, i);
3120 		if (ret) {
3121 			rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3122 			goto err_free;
3123 		}
3124 	}
3125 
3126 	return 0;
3127 
3128 err_free:
3129 	rx_allocated = i;
3130 	for (i = 0; i < rx_allocated; i++) {
3131 		rx_ring = &rtwpci->rx_rings[i];
3132 		rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3133 	}
3134 
3135 	return ret;
3136 }
3137 
3138 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3139 				     struct pci_dev *pdev)
3140 {
3141 	int ret;
3142 
3143 	ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3144 	if (ret) {
3145 		rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3146 		goto err;
3147 	}
3148 
3149 	ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3150 	if (ret) {
3151 		rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3152 		goto err_free_tx_rings;
3153 	}
3154 
3155 	return 0;
3156 
3157 err_free_tx_rings:
3158 	rtw89_pci_free_tx_rings(rtwdev, pdev);
3159 err:
3160 	return ret;
3161 }
3162 
3163 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3164 			       struct rtw89_pci *rtwpci)
3165 {
3166 	skb_queue_head_init(&rtwpci->h2c_queue);
3167 	skb_queue_head_init(&rtwpci->h2c_release_queue);
3168 }
3169 
3170 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3171 				    struct pci_dev *pdev)
3172 {
3173 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3174 	int ret;
3175 
3176 	ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3177 	if (ret) {
3178 		rtw89_err(rtwdev, "failed to setup pci mapping\n");
3179 		goto err;
3180 	}
3181 
3182 	ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3183 	if (ret) {
3184 		rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3185 		goto err_pci_unmap;
3186 	}
3187 
3188 	rtw89_pci_h2c_init(rtwdev, rtwpci);
3189 
3190 	spin_lock_init(&rtwpci->irq_lock);
3191 	spin_lock_init(&rtwpci->trx_lock);
3192 
3193 	return 0;
3194 
3195 err_pci_unmap:
3196 	rtw89_pci_clear_mapping(rtwdev, pdev);
3197 err:
3198 	return ret;
3199 }
3200 
3201 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3202 				     struct pci_dev *pdev)
3203 {
3204 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3205 
3206 	rtw89_pci_free_trx_rings(rtwdev, pdev);
3207 	rtw89_pci_clear_mapping(rtwdev, pdev);
3208 	rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3209 				skb_queue_len(&rtwpci->h2c_queue), true);
3210 }
3211 
3212 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3213 {
3214 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3215 	const struct rtw89_chip_info *chip = rtwdev->chip;
3216 	u32 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN;
3217 
3218 	if (chip->chip_id == RTL8851B)
3219 		hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN_WKARND;
3220 
3221 	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0;
3222 
3223 	if (rtwpci->under_recovery) {
3224 		rtwpci->intrs[0] = hs0isr_ind_int_en;
3225 		rtwpci->intrs[1] = 0;
3226 	} else {
3227 		rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3228 				   B_AX_RXDMA_INT_EN |
3229 				   B_AX_RXP1DMA_INT_EN |
3230 				   B_AX_RPQDMA_INT_EN |
3231 				   B_AX_RXDMA_STUCK_INT_EN |
3232 				   B_AX_RDU_INT_EN |
3233 				   B_AX_RPQBD_FULL_INT_EN |
3234 				   hs0isr_ind_int_en;
3235 
3236 		rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN;
3237 	}
3238 }
3239 EXPORT_SYMBOL(rtw89_pci_config_intr_mask);
3240 
3241 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3242 {
3243 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3244 
3245 	rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN;
3246 	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3247 	rtwpci->intrs[0] = 0;
3248 	rtwpci->intrs[1] = 0;
3249 }
3250 
3251 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
3252 {
3253 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3254 
3255 	rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN |
3256 			    B_AX_HS1ISR_IND_INT_EN |
3257 			    B_AX_HS0ISR_IND_INT_EN;
3258 	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3259 	rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3260 			   B_AX_RXDMA_INT_EN |
3261 			   B_AX_RXP1DMA_INT_EN |
3262 			   B_AX_RPQDMA_INT_EN |
3263 			   B_AX_RXDMA_STUCK_INT_EN |
3264 			   B_AX_RDU_INT_EN |
3265 			   B_AX_RPQBD_FULL_INT_EN;
3266 	rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3267 }
3268 
3269 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
3270 {
3271 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3272 
3273 	rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN |
3274 			    B_AX_HS0ISR_IND_INT_EN;
3275 	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3276 	rtwpci->intrs[0] = 0;
3277 	rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3278 }
3279 
3280 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
3281 {
3282 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3283 
3284 	if (rtwpci->under_recovery)
3285 		rtw89_pci_recovery_intr_mask_v1(rtwdev);
3286 	else if (rtwpci->low_power)
3287 		rtw89_pci_low_power_intr_mask_v1(rtwdev);
3288 	else
3289 		rtw89_pci_default_intr_mask_v1(rtwdev);
3290 }
3291 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1);
3292 
3293 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
3294 				 struct pci_dev *pdev)
3295 {
3296 	unsigned long flags = 0;
3297 	int ret;
3298 
3299 	flags |= PCI_IRQ_LEGACY | PCI_IRQ_MSI;
3300 	ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
3301 	if (ret < 0) {
3302 		rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
3303 		goto err;
3304 	}
3305 
3306 	ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
3307 					rtw89_pci_interrupt_handler,
3308 					rtw89_pci_interrupt_threadfn,
3309 					IRQF_SHARED, KBUILD_MODNAME, rtwdev);
3310 	if (ret) {
3311 		rtw89_err(rtwdev, "failed to request threaded irq\n");
3312 		goto err_free_vector;
3313 	}
3314 
3315 	rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET);
3316 
3317 	return 0;
3318 
3319 err_free_vector:
3320 	pci_free_irq_vectors(pdev);
3321 err:
3322 	return ret;
3323 }
3324 
3325 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
3326 			       struct pci_dev *pdev)
3327 {
3328 	devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
3329 	pci_free_irq_vectors(pdev);
3330 }
3331 
3332 static u16 gray_code_to_bin(u16 gray_code, u32 bit_num)
3333 {
3334 	u16 bin = 0, gray_bit;
3335 	u32 bit_idx;
3336 
3337 	for (bit_idx = 0; bit_idx < bit_num; bit_idx++) {
3338 		gray_bit = (gray_code >> bit_idx) & 0x1;
3339 		if (bit_num - bit_idx > 1)
3340 			gray_bit ^= (gray_code >> (bit_idx + 1)) & 0x1;
3341 		bin |= (gray_bit << bit_idx);
3342 	}
3343 
3344 	return bin;
3345 }
3346 
3347 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
3348 {
3349 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3350 	struct pci_dev *pdev = rtwpci->pdev;
3351 	u16 val16, filter_out_val;
3352 	u32 val, phy_offset;
3353 	int ret;
3354 
3355 	if (rtwdev->chip->chip_id != RTL8852C)
3356 		return 0;
3357 
3358 	val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
3359 	if (val == B_AX_ASPM_CTRL_L1)
3360 		return 0;
3361 
3362 	ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
3363 	if (ret)
3364 		return ret;
3365 
3366 	val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val);
3367 	if (val == RTW89_PCIE_GEN1_SPEED) {
3368 		phy_offset = R_RAC_DIRECT_OFFSET_G1;
3369 	} else if (val == RTW89_PCIE_GEN2_SPEED) {
3370 		phy_offset = R_RAC_DIRECT_OFFSET_G2;
3371 		val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
3372 		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
3373 				  val16 | B_PCIE_BIT_PINOUT_DIS);
3374 		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
3375 				  val16 & ~B_PCIE_BIT_RD_SEL);
3376 
3377 		val16 = rtw89_read16_mask(rtwdev,
3378 					  phy_offset + RAC_ANA1F * RAC_MULT,
3379 					  FILTER_OUT_EQ_MASK);
3380 		val16 = gray_code_to_bin(val16, hweight16(val16));
3381 		filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
3382 					      RAC_MULT);
3383 		filter_out_val &= ~REG_FILTER_OUT_MASK;
3384 		filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16);
3385 
3386 		rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
3387 			      filter_out_val);
3388 		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
3389 				  B_BAC_EQ_SEL);
3390 		rtw89_write16_set(rtwdev,
3391 				  R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT,
3392 				  B_PCIE_BIT_PSAVE);
3393 	} else {
3394 		return -EOPNOTSUPP;
3395 	}
3396 	rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
3397 			  B_PCIE_BIT_PSAVE);
3398 
3399 	return 0;
3400 }
3401 
3402 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
3403 {
3404 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3405 	int ret;
3406 
3407 	if (rtw89_pci_disable_clkreq)
3408 		return;
3409 
3410 	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
3411 					  PCIE_CLKDLY_HW_30US);
3412 	if (ret)
3413 		rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
3414 
3415 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3416 		if (enable)
3417 			ret = rtw89_pci_config_byte_set(rtwdev,
3418 							RTW89_PCIE_L1_CTRL,
3419 							RTW89_PCIE_BIT_CLK);
3420 		else
3421 			ret = rtw89_pci_config_byte_clr(rtwdev,
3422 							RTW89_PCIE_L1_CTRL,
3423 							RTW89_PCIE_BIT_CLK);
3424 		if (ret)
3425 			rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
3426 				  enable ? "set" : "unset", ret);
3427 	} else if (chip_id == RTL8852C) {
3428 		rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
3429 				  B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL);
3430 		if (enable)
3431 			rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
3432 					  B_AX_CLK_REQ_N);
3433 		else
3434 			rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
3435 					  B_AX_CLK_REQ_N);
3436 	}
3437 }
3438 
3439 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
3440 {
3441 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3442 	u8 value = 0;
3443 	int ret;
3444 
3445 	if (rtw89_pci_disable_aspm_l1)
3446 		return;
3447 
3448 	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value);
3449 	if (ret)
3450 		rtw89_err(rtwdev, "failed to read ASPM Delay\n");
3451 
3452 	value &= ~(RTW89_L1DLY_MASK | RTW89_L0DLY_MASK);
3453 	value |= FIELD_PREP(RTW89_L1DLY_MASK, PCIE_L1DLY_16US) |
3454 		 FIELD_PREP(RTW89_L0DLY_MASK, PCIE_L0SDLY_4US);
3455 
3456 	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value);
3457 	if (ret)
3458 		rtw89_err(rtwdev, "failed to read ASPM Delay\n");
3459 
3460 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3461 		if (enable)
3462 			ret = rtw89_pci_config_byte_set(rtwdev,
3463 							RTW89_PCIE_L1_CTRL,
3464 							RTW89_PCIE_BIT_L1);
3465 		else
3466 			ret = rtw89_pci_config_byte_clr(rtwdev,
3467 							RTW89_PCIE_L1_CTRL,
3468 							RTW89_PCIE_BIT_L1);
3469 	} else if (chip_id == RTL8852C) {
3470 		if (enable)
3471 			rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3472 					  B_AX_ASPM_CTRL_L1);
3473 		else
3474 			rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3475 					  B_AX_ASPM_CTRL_L1);
3476 	}
3477 	if (ret)
3478 		rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
3479 			  enable ? "set" : "unset", ret);
3480 }
3481 
3482 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
3483 {
3484 	struct rtw89_traffic_stats *stats = &rtwdev->stats;
3485 	enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
3486 	enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
3487 	u32 val = 0;
3488 
3489 	if (!rtwdev->scanning &&
3490 	    (tx_tfc_lv >= RTW89_TFC_HIGH || rx_tfc_lv >= RTW89_TFC_HIGH))
3491 		val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL |
3492 		      FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) |
3493 		      FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) |
3494 		      FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64);
3495 
3496 	rtw89_write32(rtwdev, R_AX_INT_MIT_RX, val);
3497 }
3498 
3499 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
3500 {
3501 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3502 	struct pci_dev *pdev = rtwpci->pdev;
3503 	u16 link_ctrl;
3504 	int ret;
3505 
3506 	/* Though there is standard PCIE configuration space to set the
3507 	 * link control register, but by Realtek's design, driver should
3508 	 * check if host supports CLKREQ/ASPM to enable the HW module.
3509 	 *
3510 	 * These functions are implemented by two HW modules associated,
3511 	 * one is responsible to access PCIE configuration space to
3512 	 * follow the host settings, and another is in charge of doing
3513 	 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
3514 	 * the host does not support it, and due to some reasons or wrong
3515 	 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
3516 	 * loss if HW misbehaves on the link.
3517 	 *
3518 	 * Hence it's designed that driver should first check the PCIE
3519 	 * configuration space is sync'ed and enabled, then driver can turn
3520 	 * on the other module that is actually working on the mechanism.
3521 	 */
3522 	ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
3523 	if (ret) {
3524 		rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
3525 		return;
3526 	}
3527 
3528 	if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
3529 		rtw89_pci_clkreq_set(rtwdev, true);
3530 
3531 	if (link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)
3532 		rtw89_pci_aspm_set(rtwdev, true);
3533 }
3534 
3535 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
3536 {
3537 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3538 	int ret;
3539 
3540 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3541 		if (enable)
3542 			ret = rtw89_pci_config_byte_set(rtwdev,
3543 							RTW89_PCIE_TIMER_CTRL,
3544 							RTW89_PCIE_BIT_L1SUB);
3545 		else
3546 			ret = rtw89_pci_config_byte_clr(rtwdev,
3547 							RTW89_PCIE_TIMER_CTRL,
3548 							RTW89_PCIE_BIT_L1SUB);
3549 		if (ret)
3550 			rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
3551 				  enable ? "set" : "unset", ret);
3552 	} else if (chip_id == RTL8852C) {
3553 		ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
3554 						RTW89_PCIE_BIT_ASPM_L11 |
3555 						RTW89_PCIE_BIT_PCI_L11);
3556 		if (ret)
3557 			rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
3558 		if (enable)
3559 			rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3560 					  B_AX_L1SUB_DISABLE);
3561 		else
3562 			rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3563 					  B_AX_L1SUB_DISABLE);
3564 	}
3565 }
3566 
3567 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
3568 {
3569 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3570 	struct pci_dev *pdev = rtwpci->pdev;
3571 	u32 l1ss_cap_ptr, l1ss_ctrl;
3572 
3573 	if (rtw89_pci_disable_l1ss)
3574 		return;
3575 
3576 	l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
3577 	if (!l1ss_cap_ptr)
3578 		return;
3579 
3580 	pci_read_config_dword(pdev, l1ss_cap_ptr + PCI_L1SS_CTL1, &l1ss_ctrl);
3581 
3582 	if (l1ss_ctrl & PCI_L1SS_CTL1_L1SS_MASK)
3583 		rtw89_pci_l1ss_set(rtwdev, true);
3584 }
3585 
3586 static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev)
3587 {
3588 	int ret = 0;
3589 	u32 sts;
3590 	u32 busy = B_AX_PCIEIO_BUSY | B_AX_PCIEIO_TX_BUSY | B_AX_PCIEIO_RX_BUSY;
3591 
3592 	ret = read_poll_timeout_atomic(rtw89_read32, sts, (sts & busy) == 0x0,
3593 				       10, 1000, false, rtwdev,
3594 				       R_AX_PCIE_DMA_BUSY1);
3595 	if (ret) {
3596 		rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
3597 			  rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
3598 		return -EINVAL;
3599 	}
3600 	return ret;
3601 }
3602 
3603 static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
3604 {
3605 	u32 val;
3606 	int ret;
3607 
3608 	if (rtwdev->chip->chip_id == RTL8852C)
3609 		return 0;
3610 
3611 	rtw89_pci_ctrl_dma_all(rtwdev, false);
3612 	ret = rtw89_pci_poll_io_idle(rtwdev);
3613 	if (ret) {
3614 		val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
3615 		rtw89_debug(rtwdev, RTW89_DBG_HCI,
3616 			    "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n",
3617 			    R_AX_DBG_ERR_FLAG, val);
3618 		if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0)
3619 			rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
3620 		if (val & B_AX_RX_STUCK)
3621 			rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
3622 		rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
3623 		ret = rtw89_pci_poll_io_idle(rtwdev);
3624 		val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
3625 		rtw89_debug(rtwdev, RTW89_DBG_HCI,
3626 			    "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n",
3627 			    R_AX_DBG_ERR_FLAG, val);
3628 	}
3629 
3630 	return ret;
3631 }
3632 
3633 
3634 
3635 static int rtw89_pci_rst_bdram(struct rtw89_dev *rtwdev)
3636 {
3637 	int ret = 0;
3638 	u32 val32, sts;
3639 
3640 	val32 = B_AX_RST_BDRAM;
3641 	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
3642 
3643 	ret = read_poll_timeout_atomic(rtw89_read32, sts,
3644 				       (sts & B_AX_RST_BDRAM) == 0x0, 1, 100,
3645 				       true, rtwdev, R_AX_PCIE_INIT_CFG1);
3646 	return ret;
3647 }
3648 
3649 static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev)
3650 {
3651 	u32 ret;
3652 
3653 	if (rtwdev->chip->chip_id == RTL8852C)
3654 		return 0;
3655 
3656 	rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
3657 	rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
3658 	rtw89_pci_clr_idx_all(rtwdev);
3659 
3660 	ret = rtw89_pci_rst_bdram(rtwdev);
3661 	if (ret)
3662 		return ret;
3663 
3664 	rtw89_pci_ctrl_dma_all(rtwdev, true);
3665 	return ret;
3666 }
3667 
3668 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
3669 					  enum rtw89_lv1_rcvy_step step)
3670 {
3671 	int ret;
3672 
3673 	switch (step) {
3674 	case RTW89_LV1_RCVY_STEP_1:
3675 		ret = rtw89_pci_lv1rst_stop_dma(rtwdev);
3676 		if (ret)
3677 			rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
3678 
3679 		break;
3680 
3681 	case RTW89_LV1_RCVY_STEP_2:
3682 		ret = rtw89_pci_lv1rst_start_dma(rtwdev);
3683 		if (ret)
3684 			rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
3685 		break;
3686 
3687 	default:
3688 		return -EINVAL;
3689 	}
3690 
3691 	return ret;
3692 }
3693 
3694 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
3695 {
3696 	rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
3697 		   rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
3698 	rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
3699 		   rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
3700 	rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
3701 		   rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
3702 }
3703 
3704 static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget)
3705 {
3706 	struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
3707 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3708 	unsigned long flags;
3709 	int work_done;
3710 
3711 	rtwdev->napi_budget_countdown = budget;
3712 
3713 	rtw89_pci_clear_isr0(rtwdev, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT);
3714 	work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
3715 	if (work_done == budget)
3716 		return budget;
3717 
3718 	rtw89_pci_clear_isr0(rtwdev, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT | B_AX_RDU_INT);
3719 	work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
3720 	if (work_done < budget && napi_complete_done(napi, work_done)) {
3721 		spin_lock_irqsave(&rtwpci->irq_lock, flags);
3722 		if (likely(rtwpci->running))
3723 			rtw89_chip_enable_intr(rtwdev, rtwpci);
3724 		spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
3725 	}
3726 
3727 	return work_done;
3728 }
3729 
3730 static int __maybe_unused rtw89_pci_suspend(struct device *dev)
3731 {
3732 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
3733 	struct rtw89_dev *rtwdev = hw->priv;
3734 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3735 
3736 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3737 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
3738 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3739 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3740 		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
3741 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
3742 		rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
3743 				  B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
3744 	} else {
3745 		rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3746 				  B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
3747 	}
3748 
3749 	return 0;
3750 }
3751 
3752 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
3753 {
3754 	if (rtwdev->chip->chip_id == RTL8852C)
3755 		return;
3756 
3757 	/* Hardware need write the reg twice to ensure the setting work */
3758 	rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
3759 				    RTW89_PCIE_BIT_CFG_RST_MSTATE);
3760 	rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
3761 				    RTW89_PCIE_BIT_CFG_RST_MSTATE);
3762 }
3763 
3764 static int __maybe_unused rtw89_pci_resume(struct device *dev)
3765 {
3766 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
3767 	struct rtw89_dev *rtwdev = hw->priv;
3768 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3769 
3770 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3771 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
3772 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3773 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3774 		rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
3775 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
3776 		rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
3777 				  B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
3778 	} else {
3779 		rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3780 				  B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
3781 		rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3782 				  B_AX_SEL_REQ_ENTR_L1);
3783 	}
3784 	rtw89_pci_l2_hci_ldo(rtwdev);
3785 	rtw89_pci_filter_out(rtwdev);
3786 	rtw89_pci_link_cfg(rtwdev);
3787 	rtw89_pci_l1ss_cfg(rtwdev);
3788 
3789 	return 0;
3790 }
3791 
3792 SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
3793 EXPORT_SYMBOL(rtw89_pm_ops);
3794 
3795 static const struct rtw89_hci_ops rtw89_pci_ops = {
3796 	.tx_write	= rtw89_pci_ops_tx_write,
3797 	.tx_kick_off	= rtw89_pci_ops_tx_kick_off,
3798 	.flush_queues	= rtw89_pci_ops_flush_queues,
3799 	.reset		= rtw89_pci_ops_reset,
3800 	.start		= rtw89_pci_ops_start,
3801 	.stop		= rtw89_pci_ops_stop,
3802 	.pause		= rtw89_pci_ops_pause,
3803 	.switch_mode	= rtw89_pci_ops_switch_mode,
3804 	.recalc_int_mit = rtw89_pci_recalc_int_mit,
3805 
3806 	.read8		= rtw89_pci_ops_read8,
3807 	.read16		= rtw89_pci_ops_read16,
3808 	.read32		= rtw89_pci_ops_read32,
3809 	.write8		= rtw89_pci_ops_write8,
3810 	.write16	= rtw89_pci_ops_write16,
3811 	.write32	= rtw89_pci_ops_write32,
3812 
3813 	.mac_pre_init	= rtw89_pci_ops_mac_pre_init,
3814 	.mac_post_init	= rtw89_pci_ops_mac_post_init,
3815 	.deinit		= rtw89_pci_ops_deinit,
3816 
3817 	.check_and_reclaim_tx_resource = rtw89_pci_check_and_reclaim_tx_resource,
3818 	.mac_lv1_rcvy	= rtw89_pci_ops_mac_lv1_recovery,
3819 	.dump_err_status = rtw89_pci_ops_dump_err_status,
3820 	.napi_poll	= rtw89_pci_napi_poll,
3821 
3822 	.recovery_start = rtw89_pci_ops_recovery_start,
3823 	.recovery_complete = rtw89_pci_ops_recovery_complete,
3824 
3825 	.ctrl_txdma_ch	= rtw89_pci_ctrl_txdma_ch_pcie,
3826 	.ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_pcie,
3827 	.ctrl_trxhci	= rtw89_pci_ctrl_dma_trx,
3828 	.poll_txdma_ch	= rtw89_poll_txdma_ch_idle_pcie,
3829 	.clr_idx_all	= rtw89_pci_clr_idx_all,
3830 	.clear		= rtw89_pci_clear_resource,
3831 	.disable_intr	= rtw89_pci_disable_intr_lock,
3832 	.enable_intr	= rtw89_pci_enable_intr_lock,
3833 	.rst_bdram	= rtw89_pci_rst_bdram_pcie,
3834 };
3835 
3836 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3837 {
3838 	struct rtw89_dev *rtwdev;
3839 	const struct rtw89_driver_info *info;
3840 	const struct rtw89_pci_info *pci_info;
3841 	int ret;
3842 
3843 	info = (const struct rtw89_driver_info *)id->driver_data;
3844 
3845 	rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
3846 					  sizeof(struct rtw89_pci),
3847 					  info->chip);
3848 	if (!rtwdev) {
3849 		dev_err(&pdev->dev, "failed to allocate hw\n");
3850 		return -ENOMEM;
3851 	}
3852 
3853 	pci_info = info->bus.pci;
3854 
3855 	rtwdev->pci_info = info->bus.pci;
3856 	rtwdev->hci.ops = &rtw89_pci_ops;
3857 	rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
3858 	rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
3859 	rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
3860 
3861 	SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
3862 
3863 	ret = rtw89_core_init(rtwdev);
3864 	if (ret) {
3865 		rtw89_err(rtwdev, "failed to initialise core\n");
3866 		goto err_release_hw;
3867 	}
3868 
3869 	ret = rtw89_pci_claim_device(rtwdev, pdev);
3870 	if (ret) {
3871 		rtw89_err(rtwdev, "failed to claim pci device\n");
3872 		goto err_core_deinit;
3873 	}
3874 
3875 	ret = rtw89_pci_setup_resource(rtwdev, pdev);
3876 	if (ret) {
3877 		rtw89_err(rtwdev, "failed to setup pci resource\n");
3878 		goto err_declaim_pci;
3879 	}
3880 
3881 	ret = rtw89_chip_info_setup(rtwdev);
3882 	if (ret) {
3883 		rtw89_err(rtwdev, "failed to setup chip information\n");
3884 		goto err_clear_resource;
3885 	}
3886 
3887 	rtw89_pci_filter_out(rtwdev);
3888 	rtw89_pci_link_cfg(rtwdev);
3889 	rtw89_pci_l1ss_cfg(rtwdev);
3890 
3891 	rtw89_core_napi_init(rtwdev);
3892 
3893 	ret = rtw89_pci_request_irq(rtwdev, pdev);
3894 	if (ret) {
3895 		rtw89_err(rtwdev, "failed to request pci irq\n");
3896 		goto err_deinit_napi;
3897 	}
3898 
3899 	ret = rtw89_core_register(rtwdev);
3900 	if (ret) {
3901 		rtw89_err(rtwdev, "failed to register core\n");
3902 		goto err_free_irq;
3903 	}
3904 
3905 	return 0;
3906 
3907 err_free_irq:
3908 	rtw89_pci_free_irq(rtwdev, pdev);
3909 err_deinit_napi:
3910 	rtw89_core_napi_deinit(rtwdev);
3911 err_clear_resource:
3912 	rtw89_pci_clear_resource(rtwdev, pdev);
3913 err_declaim_pci:
3914 	rtw89_pci_declaim_device(rtwdev, pdev);
3915 err_core_deinit:
3916 	rtw89_core_deinit(rtwdev);
3917 err_release_hw:
3918 	rtw89_free_ieee80211_hw(rtwdev);
3919 
3920 	return ret;
3921 }
3922 EXPORT_SYMBOL(rtw89_pci_probe);
3923 
3924 void rtw89_pci_remove(struct pci_dev *pdev)
3925 {
3926 	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
3927 	struct rtw89_dev *rtwdev;
3928 
3929 	rtwdev = hw->priv;
3930 
3931 	rtw89_pci_free_irq(rtwdev, pdev);
3932 	rtw89_core_napi_deinit(rtwdev);
3933 	rtw89_core_unregister(rtwdev);
3934 	rtw89_pci_clear_resource(rtwdev, pdev);
3935 	rtw89_pci_declaim_device(rtwdev, pdev);
3936 	rtw89_core_deinit(rtwdev);
3937 	rtw89_free_ieee80211_hw(rtwdev);
3938 }
3939 EXPORT_SYMBOL(rtw89_pci_remove);
3940 
3941 MODULE_AUTHOR("Realtek Corporation");
3942 MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver");
3943 MODULE_LICENSE("Dual BSD/GPL");
3944