22 static int rtw89_pci_rst_bdram_pcie(struct rtw89_dev *rtwdev)  in rtw89_pci_rst_bdram_pcie()
40 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,  in rtw89_pci_dma_recalc()
61 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,  in rtw89_pci_txbd_recalc()
74 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,  in rtw89_pci_release_fwcmd()
108 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,  in rtw89_pci_reclaim_tx_fwcmd()
120 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,  in rtw89_pci_rxbd_recalc()
133 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,  in rtw89_pci_sync_skb_for_cpu()
145 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,  in rtw89_pci_sync_skb_for_device()
157 static int rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,  in rtw89_pci_rxbd_info_update()
172 static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable)  in rtw89_pci_ctrl_txdma_ch_pcie()
189 static void rtw89_pci_ctrl_txdma_fw_ch_pcie(struct rtw89_dev *rtwdev, bool enable)  in rtw89_pci_ctrl_txdma_fw_ch_pcie()
201 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,  in rtw89_skb_put_rx_data()
229 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,  in rtw89_pci_rxbd_deliver_skbs()
314 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,  in rtw89_pci_rxbd_deliver()
337 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,  in rtw89_pci_poll_rxq_dma()
361 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,  in rtw89_pci_tx_status()
401 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)  in rtw89_pci_reclaim_txbd()
422 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,  in rtw89_pci_release_busy_txwd()
438 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,  in rtw89_pci_release_txwd_skb()
472 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,  in rtw89_pci_release_rpp()
499 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,  in rtw89_pci_release_pending_txwd_skb()
516 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,  in rtw89_pci_release_tx_skbs()
567 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,  in rtw89_pci_release_tx()
590 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,  in rtw89_pci_poll_rpq_dma()
617 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,  in rtw89_pci_isr_rxd_unavail()
644 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,  in rtw89_pci_recognize_intrs()
658 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,  in rtw89_pci_recognize_intrs_v1()
679 static void rtw89_pci_clear_isr0(struct rtw89_dev *rtwdev, u32 isr00)  in rtw89_pci_clear_isr0()
685 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)  in rtw89_pci_enable_intr()
693 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)  in rtw89_pci_disable_intr()
701 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)  in rtw89_pci_enable_intr_v1()
710 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)  in rtw89_pci_disable_intr_v1()
716 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_recovery_start()
728 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_recovery_complete()
740 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)  in rtw89_pci_low_power_interrupt_handler()
754 	struct rtw89_dev *rtwdev = dev;  in rtw89_pci_interrupt_threadfn()  local
798 	struct rtw89_dev *rtwdev = dev;  in rtw89_pci_interrupt_handler()  local
896 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,  in rtw89_pci_get_txch_addrs()
910 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,  in rtw89_pci_get_rxch_addrs()
936 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)  in __rtw89_pci_check_and_reclaim_tx_fwcmd_resource()
951 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,  in __rtw89_pci_check_and_reclaim_tx_resource_noio()
968 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,  in __rtw89_pci_check_and_reclaim_tx_resource()
1022 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,  in rtw89_pci_check_and_reclaim_tx_resource()
1034 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)  in __rtw89_pci_tx_kick_off()
1049 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,  in rtw89_pci_tx_bd_ring_update()
1062 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)  in rtw89_pci_ops_tx_kick_off()
1075 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)  in rtw89_pci_tx_kick_off_pending()
1090 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)  in __pci_flush_txch()
1116 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,  in __rtw89_pci_ops_flush_txchs()
1134 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,  in rtw89_pci_ops_flush_queues()
1140 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,  in rtw89_pci_fill_txaddr_info()
1157 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,  in rtw89_pci_fill_txaddr_info_v1()
1192 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,  in rtw89_pci_txwd_submit()
1252 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,  in rtw89_pci_fwcmd_submit()
1288 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,  in rtw89_pci_txbd_submit()
1332 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,  in rtw89_pci_tx_write()
1375 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)  in rtw89_pci_ops_tx_write()
1417 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)  in rtw89_pci_reset_trx_rings()
1469 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,  in rtw89_pci_release_tx_ring()
1476 static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_reset()
1498 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)  in rtw89_pci_enable_intr_lock()
1509 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)  in rtw89_pci_disable_intr_lock()
1520 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_start()
1528 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_stop()
1538 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)  in rtw89_pci_ops_pause()
1555 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)  in rtw89_pci_switch_bd_idx_addr()
1583 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)  in rtw89_pci_ops_switch_mode()
1596 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)  in rtw89_pci_ops_read32_cmac()
1616 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)  in rtw89_pci_ops_read8()
1630 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)  in rtw89_pci_ops_read16()
1644 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)  in rtw89_pci_ops_read32()
1654 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)  in rtw89_pci_ops_write8()
1661 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)  in rtw89_pci_ops_write16()
1668 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)  in rtw89_pci_ops_write32()
1675 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)  in rtw89_pci_ctrl_dma_trx()
1687 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)  in rtw89_pci_ctrl_dma_io()
1706 static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)  in rtw89_pci_ctrl_dma_all()
1712 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)  in rtw89_pci_check_mdio()
1744 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)  in rtw89_read16_mdio()
1759 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)  in rtw89_write16_mdio()
1774 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)  in rtw89_write16_mdio_mask()
1795 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)  in rtw89_write16_mdio_set()
1810 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)  in rtw89_write16_mdio_clr()
1825 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,  in rtw89_pci_write_config_byte()
1834 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,  in rtw89_pci_read_config_byte()
1843 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,  in rtw89_pci_config_byte_set()
1859 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,  in rtw89_pci_config_byte_clr()
1876 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)  in __get_target()
1915 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)  in rtw89_pci_autok_x()
1928 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)  in rtw89_pci_auto_refclk_cal()
2080 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)  in rtw89_pci_deglitch_setting()
2104 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)  in rtw89_pci_rxdma_prefth()
2112 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)  in rtw89_pci_l1off_pwroff()
2122 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)  in rtw89_pci_l2_rxen_lat()
2142 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)  in rtw89_pci_aphy_pwrcut()
2152 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)  in rtw89_pci_hci_ldo()
2167 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)  in rtw89_pci_dphy_delay()
2178 static void rtw89_pci_power_wake(struct rtw89_dev *rtwdev, bool pwr_up)  in rtw89_pci_power_wake()
2186 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)  in rtw89_pci_autoload_hang()
2195 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)  in rtw89_pci_l12_vmain()
2203 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)  in rtw89_pci_gen2_force_ib()
2215 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)  in rtw89_pci_l1_ent_lat()
2223 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)  in rtw89_pci_wd_exit_l1()
2231 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)  in rtw89_pci_set_sic()
2240 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)  in rtw89_pci_set_lbc()
2259 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)  in rtw89_pci_set_io_rcy()
2286 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)  in rtw89_pci_set_dbg()
2299 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)  in rtw89_pci_set_keep_reg()
2308 static void rtw89_pci_clr_idx_all(struct rtw89_dev *rtwdev)  in rtw89_pci_clr_idx_all()
2330 static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev)  in rtw89_poll_txdma_ch_idle_pcie()
2357 static int rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev *rtwdev)  in rtw89_poll_rxdma_ch_idle_pcie()
2373 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)  in rtw89_pci_poll_dma_all_idle()
2392 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)  in rtw89_pci_mode_op()
2481 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_deinit()
2496 static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_mac_pre_init()
2571 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)  in rtw89_pci_ltr_set()
2606 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)  in rtw89_pci_ltr_set_v1()
2653 static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_mac_post_init()
2685 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,  in rtw89_pci_claim_device()
2705 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,  in rtw89_pci_declaim_device()
2711 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,  in rtw89_pci_setup_mapping()
2753 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,  in rtw89_pci_clear_mapping()
2764 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,  in rtw89_pci_free_tx_wd_ring()
2779 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,  in rtw89_pci_free_tx_ring()
2795 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,  in rtw89_pci_free_tx_rings()
2812 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,  in rtw89_pci_free_rx_ring()
2844 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,  in rtw89_pci_free_rx_rings()
2857 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,  in rtw89_pci_free_trx_rings()
2864 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,  in rtw89_pci_init_rx_bd()
2890 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,  in rtw89_pci_alloc_tx_wd_ring()
2941 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,  in rtw89_pci_alloc_tx_ring()
2989 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,  in rtw89_pci_alloc_tx_rings()
3026 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,  in rtw89_pci_alloc_rx_ring()
3105 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,  in rtw89_pci_alloc_rx_rings()
3139 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,  in rtw89_pci_alloc_trx_rings()
3164 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,  in rtw89_pci_h2c_init()
3171 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,  in rtw89_pci_setup_resource()
3202 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,  in rtw89_pci_clear_resource()
3213 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)  in rtw89_pci_config_intr_mask()
3242 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)  in rtw89_pci_recovery_intr_mask_v1()
3252 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)  in rtw89_pci_default_intr_mask_v1()
3270 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)  in rtw89_pci_low_power_intr_mask_v1()
3281 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)  in rtw89_pci_config_intr_mask_v1()
3294 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,  in rtw89_pci_request_irq()
3326 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,  in rtw89_pci_free_irq()
3348 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)  in rtw89_pci_filter_out()
3403 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)  in rtw89_pci_clkreq_set()
3440 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)  in rtw89_pci_aspm_set()
3483 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)  in rtw89_pci_recalc_int_mit()
3500 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)  in rtw89_pci_link_cfg()
3536 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)  in rtw89_pci_l1ss_set()
3568 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)  in rtw89_pci_l1ss_cfg()
3587 static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev)  in rtw89_pci_poll_io_idle()
3604 static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)  in rtw89_pci_lv1rst_stop_dma()
3636 static int rtw89_pci_rst_bdram(struct rtw89_dev *rtwdev)  in rtw89_pci_rst_bdram()
3650 static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev)  in rtw89_pci_lv1rst_start_dma()
3669 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,  in rtw89_pci_ops_mac_lv1_recovery()
3695 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)  in rtw89_pci_ops_dump_err_status()
3707 	struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);  in rtw89_pci_napi_poll()  local
3734 	struct rtw89_dev *rtwdev = hw->priv;  in rtw89_pci_suspend()  local
3753 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)  in rtw89_pci_l2_hci_ldo()
3768 	struct rtw89_dev *rtwdev = hw->priv;  in rtw89_pci_resume()  local
3839 	struct rtw89_dev *rtwdev;  in rtw89_pci_probe()  local
3928 	struct rtw89_dev *rtwdev;  in rtw89_pci_remove()  local