1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright(c) 2018-2019 Realtek Corporation 3 */ 4 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include "main.h" 8 #include "pci.h" 9 #include "reg.h" 10 #include "tx.h" 11 #include "rx.h" 12 #include "fw.h" 13 #include "ps.h" 14 #include "debug.h" 15 16 static bool rtw_disable_msi; 17 static bool rtw_pci_disable_aspm; 18 module_param_named(disable_msi, rtw_disable_msi, bool, 0644); 19 module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644); 20 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support"); 21 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support"); 22 23 static u32 rtw_pci_tx_queue_idx_addr[] = { 24 [RTW_TX_QUEUE_BK] = RTK_PCI_TXBD_IDX_BKQ, 25 [RTW_TX_QUEUE_BE] = RTK_PCI_TXBD_IDX_BEQ, 26 [RTW_TX_QUEUE_VI] = RTK_PCI_TXBD_IDX_VIQ, 27 [RTW_TX_QUEUE_VO] = RTK_PCI_TXBD_IDX_VOQ, 28 [RTW_TX_QUEUE_MGMT] = RTK_PCI_TXBD_IDX_MGMTQ, 29 [RTW_TX_QUEUE_HI0] = RTK_PCI_TXBD_IDX_HI0Q, 30 [RTW_TX_QUEUE_H2C] = RTK_PCI_TXBD_IDX_H2CQ, 31 }; 32 33 static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb, u8 queue) 34 { 35 switch (queue) { 36 case RTW_TX_QUEUE_BCN: 37 return TX_DESC_QSEL_BEACON; 38 case RTW_TX_QUEUE_H2C: 39 return TX_DESC_QSEL_H2C; 40 case RTW_TX_QUEUE_MGMT: 41 return TX_DESC_QSEL_MGMT; 42 case RTW_TX_QUEUE_HI0: 43 return TX_DESC_QSEL_HIGH; 44 default: 45 return skb->priority; 46 } 47 }; 48 49 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr) 50 { 51 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 52 53 return readb(rtwpci->mmap + addr); 54 } 55 56 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr) 57 { 58 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 59 60 return readw(rtwpci->mmap + addr); 61 } 62 63 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr) 64 { 65 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 66 67 return readl(rtwpci->mmap + addr); 68 } 69 70 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val) 71 { 72 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 73 74 writeb(val, rtwpci->mmap + addr); 75 } 76 77 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val) 78 { 79 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 80 81 writew(val, rtwpci->mmap + addr); 82 } 83 84 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val) 85 { 86 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 87 88 writel(val, rtwpci->mmap + addr); 89 } 90 91 static inline void *rtw_pci_get_tx_desc(struct rtw_pci_tx_ring *tx_ring, u8 idx) 92 { 93 int offset = tx_ring->r.desc_size * idx; 94 95 return tx_ring->r.head + offset; 96 } 97 98 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev, 99 struct rtw_pci_tx_ring *tx_ring) 100 { 101 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 102 struct rtw_pci_tx_data *tx_data; 103 struct sk_buff *skb, *tmp; 104 dma_addr_t dma; 105 106 /* free every skb remained in tx list */ 107 skb_queue_walk_safe(&tx_ring->queue, skb, tmp) { 108 __skb_unlink(skb, &tx_ring->queue); 109 tx_data = rtw_pci_get_tx_data(skb); 110 dma = tx_data->dma; 111 112 dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE); 113 dev_kfree_skb_any(skb); 114 } 115 } 116 117 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev, 118 struct rtw_pci_tx_ring *tx_ring) 119 { 120 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 121 u8 *head = tx_ring->r.head; 122 u32 len = tx_ring->r.len; 123 int ring_sz = len * tx_ring->r.desc_size; 124 125 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); 126 127 /* free the ring itself */ 128 dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma); 129 tx_ring->r.head = NULL; 130 } 131 132 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev, 133 struct rtw_pci_rx_ring *rx_ring) 134 { 135 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 136 struct sk_buff *skb; 137 int buf_sz = RTK_PCI_RX_BUF_SIZE; 138 dma_addr_t dma; 139 int i; 140 141 for (i = 0; i < rx_ring->r.len; i++) { 142 skb = rx_ring->buf[i]; 143 if (!skb) 144 continue; 145 146 dma = *((dma_addr_t *)skb->cb); 147 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 148 dev_kfree_skb(skb); 149 rx_ring->buf[i] = NULL; 150 } 151 } 152 153 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev, 154 struct rtw_pci_rx_ring *rx_ring) 155 { 156 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 157 u8 *head = rx_ring->r.head; 158 int ring_sz = rx_ring->r.desc_size * rx_ring->r.len; 159 160 rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring); 161 162 dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma); 163 } 164 165 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev) 166 { 167 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 168 struct rtw_pci_tx_ring *tx_ring; 169 struct rtw_pci_rx_ring *rx_ring; 170 int i; 171 172 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { 173 tx_ring = &rtwpci->tx_rings[i]; 174 rtw_pci_free_tx_ring(rtwdev, tx_ring); 175 } 176 177 for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) { 178 rx_ring = &rtwpci->rx_rings[i]; 179 rtw_pci_free_rx_ring(rtwdev, rx_ring); 180 } 181 } 182 183 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev, 184 struct rtw_pci_tx_ring *tx_ring, 185 u8 desc_size, u32 len) 186 { 187 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 188 int ring_sz = desc_size * len; 189 dma_addr_t dma; 190 u8 *head; 191 192 if (len > TRX_BD_IDX_MASK) { 193 rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len); 194 return -EINVAL; 195 } 196 197 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 198 if (!head) { 199 rtw_err(rtwdev, "failed to allocate tx ring\n"); 200 return -ENOMEM; 201 } 202 203 skb_queue_head_init(&tx_ring->queue); 204 tx_ring->r.head = head; 205 tx_ring->r.dma = dma; 206 tx_ring->r.len = len; 207 tx_ring->r.desc_size = desc_size; 208 tx_ring->r.wp = 0; 209 tx_ring->r.rp = 0; 210 211 return 0; 212 } 213 214 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb, 215 struct rtw_pci_rx_ring *rx_ring, 216 u32 idx, u32 desc_sz) 217 { 218 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 219 struct rtw_pci_rx_buffer_desc *buf_desc; 220 int buf_sz = RTK_PCI_RX_BUF_SIZE; 221 dma_addr_t dma; 222 223 if (!skb) 224 return -EINVAL; 225 226 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE); 227 if (dma_mapping_error(&pdev->dev, dma)) 228 return -EBUSY; 229 230 *((dma_addr_t *)skb->cb) = dma; 231 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 232 idx * desc_sz); 233 memset(buf_desc, 0, sizeof(*buf_desc)); 234 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); 235 buf_desc->dma = cpu_to_le32(dma); 236 237 return 0; 238 } 239 240 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma, 241 struct rtw_pci_rx_ring *rx_ring, 242 u32 idx, u32 desc_sz) 243 { 244 struct device *dev = rtwdev->dev; 245 struct rtw_pci_rx_buffer_desc *buf_desc; 246 int buf_sz = RTK_PCI_RX_BUF_SIZE; 247 248 dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE); 249 250 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 251 idx * desc_sz); 252 memset(buf_desc, 0, sizeof(*buf_desc)); 253 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); 254 buf_desc->dma = cpu_to_le32(dma); 255 } 256 257 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev, 258 struct rtw_pci_rx_ring *rx_ring, 259 u8 desc_size, u32 len) 260 { 261 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 262 struct sk_buff *skb = NULL; 263 dma_addr_t dma; 264 u8 *head; 265 int ring_sz = desc_size * len; 266 int buf_sz = RTK_PCI_RX_BUF_SIZE; 267 int i, allocated; 268 int ret = 0; 269 270 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 271 if (!head) { 272 rtw_err(rtwdev, "failed to allocate rx ring\n"); 273 return -ENOMEM; 274 } 275 rx_ring->r.head = head; 276 277 for (i = 0; i < len; i++) { 278 skb = dev_alloc_skb(buf_sz); 279 if (!skb) { 280 allocated = i; 281 ret = -ENOMEM; 282 goto err_out; 283 } 284 285 memset(skb->data, 0, buf_sz); 286 rx_ring->buf[i] = skb; 287 ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size); 288 if (ret) { 289 allocated = i; 290 dev_kfree_skb_any(skb); 291 goto err_out; 292 } 293 } 294 295 rx_ring->r.dma = dma; 296 rx_ring->r.len = len; 297 rx_ring->r.desc_size = desc_size; 298 rx_ring->r.wp = 0; 299 rx_ring->r.rp = 0; 300 301 return 0; 302 303 err_out: 304 for (i = 0; i < allocated; i++) { 305 skb = rx_ring->buf[i]; 306 if (!skb) 307 continue; 308 dma = *((dma_addr_t *)skb->cb); 309 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 310 dev_kfree_skb_any(skb); 311 rx_ring->buf[i] = NULL; 312 } 313 dma_free_coherent(&pdev->dev, ring_sz, head, dma); 314 315 rtw_err(rtwdev, "failed to init rx buffer\n"); 316 317 return ret; 318 } 319 320 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev) 321 { 322 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 323 struct rtw_pci_tx_ring *tx_ring; 324 struct rtw_pci_rx_ring *rx_ring; 325 struct rtw_chip_info *chip = rtwdev->chip; 326 int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0; 327 int tx_desc_size, rx_desc_size; 328 u32 len; 329 int ret; 330 331 tx_desc_size = chip->tx_buf_desc_sz; 332 333 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { 334 tx_ring = &rtwpci->tx_rings[i]; 335 len = max_num_of_tx_queue(i); 336 ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len); 337 if (ret) 338 goto out; 339 } 340 341 rx_desc_size = chip->rx_buf_desc_sz; 342 343 for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) { 344 rx_ring = &rtwpci->rx_rings[j]; 345 ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size, 346 RTK_MAX_RX_DESC_NUM); 347 if (ret) 348 goto out; 349 } 350 351 return 0; 352 353 out: 354 tx_alloced = i; 355 for (i = 0; i < tx_alloced; i++) { 356 tx_ring = &rtwpci->tx_rings[i]; 357 rtw_pci_free_tx_ring(rtwdev, tx_ring); 358 } 359 360 rx_alloced = j; 361 for (j = 0; j < rx_alloced; j++) { 362 rx_ring = &rtwpci->rx_rings[j]; 363 rtw_pci_free_rx_ring(rtwdev, rx_ring); 364 } 365 366 return ret; 367 } 368 369 static void rtw_pci_deinit(struct rtw_dev *rtwdev) 370 { 371 rtw_pci_free_trx_ring(rtwdev); 372 } 373 374 static int rtw_pci_init(struct rtw_dev *rtwdev) 375 { 376 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 377 int ret = 0; 378 379 rtwpci->irq_mask[0] = IMR_HIGHDOK | 380 IMR_MGNTDOK | 381 IMR_BKDOK | 382 IMR_BEDOK | 383 IMR_VIDOK | 384 IMR_VODOK | 385 IMR_ROK | 386 IMR_BCNDMAINT_E | 387 IMR_C2HCMD | 388 0; 389 rtwpci->irq_mask[1] = IMR_TXFOVW | 390 0; 391 rtwpci->irq_mask[3] = IMR_H2CDOK | 392 0; 393 spin_lock_init(&rtwpci->irq_lock); 394 spin_lock_init(&rtwpci->hwirq_lock); 395 ret = rtw_pci_init_trx_ring(rtwdev); 396 397 return ret; 398 } 399 400 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev) 401 { 402 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 403 u32 len; 404 u8 tmp; 405 dma_addr_t dma; 406 407 tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3); 408 rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7); 409 410 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma; 411 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma); 412 413 if (!rtw_chip_wcpu_11n(rtwdev)) { 414 len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len; 415 dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma; 416 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0; 417 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0; 418 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK); 419 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma); 420 } 421 422 len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len; 423 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma; 424 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0; 425 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0; 426 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK); 427 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma); 428 429 len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len; 430 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma; 431 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0; 432 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0; 433 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK); 434 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma); 435 436 len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len; 437 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma; 438 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0; 439 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0; 440 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK); 441 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma); 442 443 len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len; 444 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma; 445 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0; 446 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0; 447 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK); 448 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma); 449 450 len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len; 451 dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma; 452 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0; 453 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0; 454 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK); 455 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma); 456 457 len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len; 458 dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma; 459 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0; 460 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0; 461 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK); 462 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma); 463 464 len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len; 465 dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma; 466 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0; 467 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0; 468 rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK); 469 rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma); 470 471 /* reset read/write point */ 472 rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff); 473 474 /* reset H2C Queue index in a single write */ 475 if (rtw_chip_wcpu_11ac(rtwdev)) 476 rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR, 477 BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX); 478 } 479 480 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev) 481 { 482 rtw_pci_reset_buf_desc(rtwdev); 483 } 484 485 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev, 486 struct rtw_pci *rtwpci, bool exclude_rx) 487 { 488 unsigned long flags; 489 u32 imr0_unmask = exclude_rx ? IMR_ROK : 0; 490 491 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 492 493 rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask); 494 rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]); 495 if (rtw_chip_wcpu_11ac(rtwdev)) 496 rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]); 497 498 rtwpci->irq_enabled = true; 499 500 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 501 } 502 503 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev, 504 struct rtw_pci *rtwpci) 505 { 506 unsigned long flags; 507 508 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 509 510 if (!rtwpci->irq_enabled) 511 goto out; 512 513 rtw_write32(rtwdev, RTK_PCI_HIMR0, 0); 514 rtw_write32(rtwdev, RTK_PCI_HIMR1, 0); 515 if (rtw_chip_wcpu_11ac(rtwdev)) 516 rtw_write32(rtwdev, RTK_PCI_HIMR3, 0); 517 518 rtwpci->irq_enabled = false; 519 520 out: 521 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 522 } 523 524 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) 525 { 526 /* reset dma and rx tag */ 527 rtw_write32_set(rtwdev, RTK_PCI_CTRL, 528 BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN); 529 rtwpci->rx_tag = 0; 530 } 531 532 static int rtw_pci_setup(struct rtw_dev *rtwdev) 533 { 534 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 535 536 rtw_pci_reset_trx_ring(rtwdev); 537 rtw_pci_dma_reset(rtwdev, rtwpci); 538 539 return 0; 540 } 541 542 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) 543 { 544 struct rtw_pci_tx_ring *tx_ring; 545 u8 queue; 546 547 rtw_pci_reset_trx_ring(rtwdev); 548 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { 549 tx_ring = &rtwpci->tx_rings[queue]; 550 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); 551 } 552 } 553 554 static void rtw_pci_napi_start(struct rtw_dev *rtwdev) 555 { 556 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 557 558 if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) 559 return; 560 561 napi_enable(&rtwpci->napi); 562 } 563 564 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev) 565 { 566 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 567 568 if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) 569 return; 570 571 napi_synchronize(&rtwpci->napi); 572 napi_disable(&rtwpci->napi); 573 } 574 575 static int rtw_pci_start(struct rtw_dev *rtwdev) 576 { 577 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 578 579 rtw_pci_napi_start(rtwdev); 580 581 spin_lock_bh(&rtwpci->irq_lock); 582 rtwpci->running = true; 583 rtw_pci_enable_interrupt(rtwdev, rtwpci, false); 584 spin_unlock_bh(&rtwpci->irq_lock); 585 586 return 0; 587 } 588 589 static void rtw_pci_stop(struct rtw_dev *rtwdev) 590 { 591 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 592 struct pci_dev *pdev = rtwpci->pdev; 593 594 spin_lock_bh(&rtwpci->irq_lock); 595 rtwpci->running = false; 596 rtw_pci_disable_interrupt(rtwdev, rtwpci); 597 spin_unlock_bh(&rtwpci->irq_lock); 598 599 synchronize_irq(pdev->irq); 600 rtw_pci_napi_stop(rtwdev); 601 602 spin_lock_bh(&rtwpci->irq_lock); 603 rtw_pci_dma_release(rtwdev, rtwpci); 604 spin_unlock_bh(&rtwpci->irq_lock); 605 } 606 607 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev) 608 { 609 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 610 struct rtw_pci_tx_ring *tx_ring; 611 bool tx_empty = true; 612 u8 queue; 613 614 if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) 615 goto enter_deep_ps; 616 617 lockdep_assert_held(&rtwpci->irq_lock); 618 619 /* Deep PS state is not allowed to TX-DMA */ 620 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { 621 /* BCN queue is rsvd page, does not have DMA interrupt 622 * H2C queue is managed by firmware 623 */ 624 if (queue == RTW_TX_QUEUE_BCN || 625 queue == RTW_TX_QUEUE_H2C) 626 continue; 627 628 tx_ring = &rtwpci->tx_rings[queue]; 629 630 /* check if there is any skb DMAing */ 631 if (skb_queue_len(&tx_ring->queue)) { 632 tx_empty = false; 633 break; 634 } 635 } 636 637 if (!tx_empty) { 638 rtw_dbg(rtwdev, RTW_DBG_PS, 639 "TX path not empty, cannot enter deep power save state\n"); 640 return; 641 } 642 enter_deep_ps: 643 set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags); 644 rtw_power_mode_change(rtwdev, true); 645 } 646 647 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev) 648 { 649 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 650 651 lockdep_assert_held(&rtwpci->irq_lock); 652 653 if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 654 rtw_power_mode_change(rtwdev, false); 655 } 656 657 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter) 658 { 659 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 660 661 spin_lock_bh(&rtwpci->irq_lock); 662 663 if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 664 rtw_pci_deep_ps_enter(rtwdev); 665 666 if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 667 rtw_pci_deep_ps_leave(rtwdev); 668 669 spin_unlock_bh(&rtwpci->irq_lock); 670 } 671 672 static u8 ac_to_hwq[] = { 673 [IEEE80211_AC_VO] = RTW_TX_QUEUE_VO, 674 [IEEE80211_AC_VI] = RTW_TX_QUEUE_VI, 675 [IEEE80211_AC_BE] = RTW_TX_QUEUE_BE, 676 [IEEE80211_AC_BK] = RTW_TX_QUEUE_BK, 677 }; 678 679 static_assert(ARRAY_SIZE(ac_to_hwq) == IEEE80211_NUM_ACS); 680 681 static u8 rtw_hw_queue_mapping(struct sk_buff *skb) 682 { 683 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 684 __le16 fc = hdr->frame_control; 685 u8 q_mapping = skb_get_queue_mapping(skb); 686 u8 queue; 687 688 if (unlikely(ieee80211_is_beacon(fc))) 689 queue = RTW_TX_QUEUE_BCN; 690 else if (unlikely(ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc))) 691 queue = RTW_TX_QUEUE_MGMT; 692 else if (is_broadcast_ether_addr(hdr->addr1) || 693 is_multicast_ether_addr(hdr->addr1)) 694 queue = RTW_TX_QUEUE_HI0; 695 else if (WARN_ON_ONCE(q_mapping >= ARRAY_SIZE(ac_to_hwq))) 696 queue = ac_to_hwq[IEEE80211_AC_BE]; 697 else 698 queue = ac_to_hwq[q_mapping]; 699 700 return queue; 701 } 702 703 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci, 704 struct rtw_pci_tx_ring *ring) 705 { 706 struct sk_buff *prev = skb_dequeue(&ring->queue); 707 struct rtw_pci_tx_data *tx_data; 708 dma_addr_t dma; 709 710 if (!prev) 711 return; 712 713 tx_data = rtw_pci_get_tx_data(prev); 714 dma = tx_data->dma; 715 dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE); 716 dev_kfree_skb_any(prev); 717 } 718 719 static void rtw_pci_dma_check(struct rtw_dev *rtwdev, 720 struct rtw_pci_rx_ring *rx_ring, 721 u32 idx) 722 { 723 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 724 struct rtw_chip_info *chip = rtwdev->chip; 725 struct rtw_pci_rx_buffer_desc *buf_desc; 726 u32 desc_sz = chip->rx_buf_desc_sz; 727 u16 total_pkt_size; 728 729 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 730 idx * desc_sz); 731 total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size); 732 733 /* rx tag mismatch, throw a warning */ 734 if (total_pkt_size != rtwpci->rx_tag) 735 rtw_warn(rtwdev, "pci bus timeout, check dma status\n"); 736 737 rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX; 738 } 739 740 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q) 741 { 742 u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q]; 743 u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2); 744 745 return FIELD_GET(TRX_BD_IDX_MASK, bd_idx); 746 } 747 748 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop) 749 { 750 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 751 struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q]; 752 u32 cur_rp; 753 u8 i; 754 755 /* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a 756 * bit dynamic, it's hard to define a reasonable fixed total timeout to 757 * use read_poll_timeout* helper. Instead, we can ensure a reasonable 758 * polling times, so we just use for loop with udelay here. 759 */ 760 for (i = 0; i < 30; i++) { 761 cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q); 762 if (cur_rp == ring->r.wp) 763 return; 764 765 udelay(1); 766 } 767 768 if (!drop) 769 rtw_warn(rtwdev, "timed out to flush pci tx ring[%d]\n", pci_q); 770 } 771 772 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues, 773 bool drop) 774 { 775 u8 q; 776 777 for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) { 778 /* It may be not necessary to flush BCN and H2C tx queues. */ 779 if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C) 780 continue; 781 782 if (pci_queues & BIT(q)) 783 __pci_flush_queue(rtwdev, q, drop); 784 } 785 } 786 787 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop) 788 { 789 u32 pci_queues = 0; 790 u8 i; 791 792 /* If all of the hardware queues are requested to flush, 793 * flush all of the pci queues. 794 */ 795 if (queues == BIT(rtwdev->hw->queues) - 1) { 796 pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1; 797 } else { 798 for (i = 0; i < rtwdev->hw->queues; i++) 799 if (queues & BIT(i)) 800 pci_queues |= BIT(ac_to_hwq[i]); 801 } 802 803 __rtw_pci_flush_queues(rtwdev, pci_queues, drop); 804 } 805 806 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, u8 queue) 807 { 808 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 809 struct rtw_pci_tx_ring *ring; 810 u32 bd_idx; 811 812 ring = &rtwpci->tx_rings[queue]; 813 bd_idx = rtw_pci_tx_queue_idx_addr[queue]; 814 815 spin_lock_bh(&rtwpci->irq_lock); 816 if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) 817 rtw_pci_deep_ps_leave(rtwdev); 818 rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK); 819 spin_unlock_bh(&rtwpci->irq_lock); 820 } 821 822 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev) 823 { 824 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 825 u8 queue; 826 827 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) 828 if (test_and_clear_bit(queue, rtwpci->tx_queued)) 829 rtw_pci_tx_kick_off_queue(rtwdev, queue); 830 } 831 832 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev, 833 struct rtw_tx_pkt_info *pkt_info, 834 struct sk_buff *skb, u8 queue) 835 { 836 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 837 struct rtw_chip_info *chip = rtwdev->chip; 838 struct rtw_pci_tx_ring *ring; 839 struct rtw_pci_tx_data *tx_data; 840 dma_addr_t dma; 841 u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz; 842 u32 tx_buf_desc_sz = chip->tx_buf_desc_sz; 843 u32 size; 844 u32 psb_len; 845 u8 *pkt_desc; 846 struct rtw_pci_tx_buffer_desc *buf_desc; 847 848 ring = &rtwpci->tx_rings[queue]; 849 850 size = skb->len; 851 852 if (queue == RTW_TX_QUEUE_BCN) 853 rtw_pci_release_rsvd_page(rtwpci, ring); 854 else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len)) 855 return -ENOSPC; 856 857 pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz); 858 memset(pkt_desc, 0, tx_pkt_desc_sz); 859 pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue); 860 rtw_tx_fill_tx_desc(pkt_info, skb); 861 dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len, 862 DMA_TO_DEVICE); 863 if (dma_mapping_error(&rtwpci->pdev->dev, dma)) 864 return -EBUSY; 865 866 /* after this we got dma mapped, there is no way back */ 867 buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz); 868 memset(buf_desc, 0, tx_buf_desc_sz); 869 psb_len = (skb->len - 1) / 128 + 1; 870 if (queue == RTW_TX_QUEUE_BCN) 871 psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET; 872 873 buf_desc[0].psb_len = cpu_to_le16(psb_len); 874 buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz); 875 buf_desc[0].dma = cpu_to_le32(dma); 876 buf_desc[1].buf_size = cpu_to_le16(size); 877 buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz); 878 879 tx_data = rtw_pci_get_tx_data(skb); 880 tx_data->dma = dma; 881 tx_data->sn = pkt_info->sn; 882 883 spin_lock_bh(&rtwpci->irq_lock); 884 885 skb_queue_tail(&ring->queue, skb); 886 887 if (queue == RTW_TX_QUEUE_BCN) 888 goto out_unlock; 889 890 /* update write-index, and kick it off later */ 891 set_bit(queue, rtwpci->tx_queued); 892 if (++ring->r.wp >= ring->r.len) 893 ring->r.wp = 0; 894 895 out_unlock: 896 spin_unlock_bh(&rtwpci->irq_lock); 897 898 return 0; 899 } 900 901 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, 902 u32 size) 903 { 904 struct sk_buff *skb; 905 struct rtw_tx_pkt_info pkt_info = {0}; 906 u8 reg_bcn_work; 907 int ret; 908 909 skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size); 910 if (!skb) 911 return -ENOMEM; 912 913 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN); 914 if (ret) { 915 rtw_err(rtwdev, "failed to write rsvd page data\n"); 916 return ret; 917 } 918 919 /* reserved pages go through beacon queue */ 920 reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK); 921 reg_bcn_work |= BIT_PCI_BCNQ_FLAG; 922 rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work); 923 924 return 0; 925 } 926 927 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size) 928 { 929 struct sk_buff *skb; 930 struct rtw_tx_pkt_info pkt_info = {0}; 931 int ret; 932 933 skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size); 934 if (!skb) 935 return -ENOMEM; 936 937 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C); 938 if (ret) { 939 rtw_err(rtwdev, "failed to write h2c data\n"); 940 return ret; 941 } 942 943 rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C); 944 945 return 0; 946 } 947 948 static int rtw_pci_tx_write(struct rtw_dev *rtwdev, 949 struct rtw_tx_pkt_info *pkt_info, 950 struct sk_buff *skb) 951 { 952 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 953 struct rtw_pci_tx_ring *ring; 954 u8 queue = rtw_hw_queue_mapping(skb); 955 int ret; 956 957 ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue); 958 if (ret) 959 return ret; 960 961 ring = &rtwpci->tx_rings[queue]; 962 spin_lock_bh(&rtwpci->irq_lock); 963 if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) { 964 ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb)); 965 ring->queue_stopped = true; 966 } 967 spin_unlock_bh(&rtwpci->irq_lock); 968 969 return 0; 970 } 971 972 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, 973 u8 hw_queue) 974 { 975 struct ieee80211_hw *hw = rtwdev->hw; 976 struct ieee80211_tx_info *info; 977 struct rtw_pci_tx_ring *ring; 978 struct rtw_pci_tx_data *tx_data; 979 struct sk_buff *skb; 980 u32 count; 981 u32 bd_idx_addr; 982 u32 bd_idx, cur_rp, rp_idx; 983 u16 q_map; 984 985 ring = &rtwpci->tx_rings[hw_queue]; 986 987 bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue]; 988 bd_idx = rtw_read32(rtwdev, bd_idx_addr); 989 cur_rp = bd_idx >> 16; 990 cur_rp &= TRX_BD_IDX_MASK; 991 rp_idx = ring->r.rp; 992 if (cur_rp >= ring->r.rp) 993 count = cur_rp - ring->r.rp; 994 else 995 count = ring->r.len - (ring->r.rp - cur_rp); 996 997 while (count--) { 998 skb = skb_dequeue(&ring->queue); 999 if (!skb) { 1000 rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n", 1001 count, hw_queue, bd_idx, ring->r.rp, cur_rp); 1002 break; 1003 } 1004 tx_data = rtw_pci_get_tx_data(skb); 1005 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, 1006 DMA_TO_DEVICE); 1007 1008 /* just free command packets from host to card */ 1009 if (hw_queue == RTW_TX_QUEUE_H2C) { 1010 dev_kfree_skb_irq(skb); 1011 continue; 1012 } 1013 1014 if (ring->queue_stopped && 1015 avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) { 1016 q_map = skb_get_queue_mapping(skb); 1017 ieee80211_wake_queue(hw, q_map); 1018 ring->queue_stopped = false; 1019 } 1020 1021 if (++rp_idx >= ring->r.len) 1022 rp_idx = 0; 1023 1024 skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz); 1025 1026 info = IEEE80211_SKB_CB(skb); 1027 1028 /* enqueue to wait for tx report */ 1029 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { 1030 rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn); 1031 continue; 1032 } 1033 1034 /* always ACK for others, then they won't be marked as drop */ 1035 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 1036 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; 1037 else 1038 info->flags |= IEEE80211_TX_STAT_ACK; 1039 1040 ieee80211_tx_info_clear_status(info); 1041 ieee80211_tx_status_irqsafe(hw, skb); 1042 } 1043 1044 ring->r.rp = cur_rp; 1045 } 1046 1047 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev) 1048 { 1049 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1050 struct napi_struct *napi = &rtwpci->napi; 1051 1052 napi_schedule(napi); 1053 } 1054 1055 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev, 1056 struct rtw_pci *rtwpci) 1057 { 1058 struct rtw_pci_rx_ring *ring; 1059 int count = 0; 1060 u32 tmp, cur_wp; 1061 1062 ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; 1063 tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ); 1064 cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK); 1065 if (cur_wp >= ring->r.wp) 1066 count = cur_wp - ring->r.wp; 1067 else 1068 count = ring->r.len - (ring->r.wp - cur_wp); 1069 1070 return count; 1071 } 1072 1073 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, 1074 u8 hw_queue, u32 limit) 1075 { 1076 struct rtw_chip_info *chip = rtwdev->chip; 1077 struct napi_struct *napi = &rtwpci->napi; 1078 struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; 1079 struct rtw_rx_pkt_stat pkt_stat; 1080 struct ieee80211_rx_status rx_status; 1081 struct sk_buff *skb, *new; 1082 u32 cur_rp = ring->r.rp; 1083 u32 count, rx_done = 0; 1084 u32 pkt_offset; 1085 u32 pkt_desc_sz = chip->rx_pkt_desc_sz; 1086 u32 buf_desc_sz = chip->rx_buf_desc_sz; 1087 u32 new_len; 1088 u8 *rx_desc; 1089 dma_addr_t dma; 1090 1091 count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci); 1092 count = min(count, limit); 1093 1094 while (count--) { 1095 rtw_pci_dma_check(rtwdev, ring, cur_rp); 1096 skb = ring->buf[cur_rp]; 1097 dma = *((dma_addr_t *)skb->cb); 1098 dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE, 1099 DMA_FROM_DEVICE); 1100 rx_desc = skb->data; 1101 chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status); 1102 1103 /* offset from rx_desc to payload */ 1104 pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz + 1105 pkt_stat.shift; 1106 1107 /* allocate a new skb for this frame, 1108 * discard the frame if none available 1109 */ 1110 new_len = pkt_stat.pkt_len + pkt_offset; 1111 new = dev_alloc_skb(new_len); 1112 if (WARN_ONCE(!new, "rx routine starvation\n")) 1113 goto next_rp; 1114 1115 /* put the DMA data including rx_desc from phy to new skb */ 1116 skb_put_data(new, skb->data, new_len); 1117 1118 if (pkt_stat.is_c2h) { 1119 rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new); 1120 } else { 1121 /* remove rx_desc */ 1122 skb_pull(new, pkt_offset); 1123 1124 rtw_rx_stats(rtwdev, pkt_stat.vif, new); 1125 memcpy(new->cb, &rx_status, sizeof(rx_status)); 1126 ieee80211_rx_napi(rtwdev->hw, NULL, new, napi); 1127 rx_done++; 1128 } 1129 1130 next_rp: 1131 /* new skb delivered to mac80211, re-enable original skb DMA */ 1132 rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp, 1133 buf_desc_sz); 1134 1135 /* host read next element in ring */ 1136 if (++cur_rp >= ring->r.len) 1137 cur_rp = 0; 1138 } 1139 1140 ring->r.rp = cur_rp; 1141 /* 'rp', the last position we have read, is seen as previous posistion 1142 * of 'wp' that is used to calculate 'count' next time. 1143 */ 1144 ring->r.wp = cur_rp; 1145 rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp); 1146 1147 return rx_done; 1148 } 1149 1150 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev, 1151 struct rtw_pci *rtwpci, u32 *irq_status) 1152 { 1153 unsigned long flags; 1154 1155 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 1156 1157 irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0); 1158 irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1); 1159 if (rtw_chip_wcpu_11ac(rtwdev)) 1160 irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3); 1161 else 1162 irq_status[3] = 0; 1163 irq_status[0] &= rtwpci->irq_mask[0]; 1164 irq_status[1] &= rtwpci->irq_mask[1]; 1165 irq_status[3] &= rtwpci->irq_mask[3]; 1166 rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]); 1167 rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]); 1168 if (rtw_chip_wcpu_11ac(rtwdev)) 1169 rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]); 1170 1171 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 1172 } 1173 1174 static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev) 1175 { 1176 struct rtw_dev *rtwdev = dev; 1177 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1178 1179 /* disable RTW PCI interrupt to avoid more interrupts before the end of 1180 * thread function 1181 * 1182 * disable HIMR here to also avoid new HISR flag being raised before 1183 * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs 1184 * are cleared, the edge-triggered interrupt will not be generated when 1185 * a new HISR flag is set. 1186 */ 1187 rtw_pci_disable_interrupt(rtwdev, rtwpci); 1188 1189 return IRQ_WAKE_THREAD; 1190 } 1191 1192 static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev) 1193 { 1194 struct rtw_dev *rtwdev = dev; 1195 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1196 u32 irq_status[4]; 1197 bool rx = false; 1198 1199 spin_lock_bh(&rtwpci->irq_lock); 1200 rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status); 1201 1202 if (irq_status[0] & IMR_MGNTDOK) 1203 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT); 1204 if (irq_status[0] & IMR_HIGHDOK) 1205 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0); 1206 if (irq_status[0] & IMR_BEDOK) 1207 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE); 1208 if (irq_status[0] & IMR_BKDOK) 1209 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK); 1210 if (irq_status[0] & IMR_VODOK) 1211 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO); 1212 if (irq_status[0] & IMR_VIDOK) 1213 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI); 1214 if (irq_status[3] & IMR_H2CDOK) 1215 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C); 1216 if (irq_status[0] & IMR_ROK) { 1217 rtw_pci_rx_isr(rtwdev); 1218 rx = true; 1219 } 1220 if (unlikely(irq_status[0] & IMR_C2HCMD)) 1221 rtw_fw_c2h_cmd_isr(rtwdev); 1222 1223 /* all of the jobs for this interrupt have been done */ 1224 if (rtwpci->running) 1225 rtw_pci_enable_interrupt(rtwdev, rtwpci, rx); 1226 spin_unlock_bh(&rtwpci->irq_lock); 1227 1228 return IRQ_HANDLED; 1229 } 1230 1231 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev, 1232 struct pci_dev *pdev) 1233 { 1234 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1235 unsigned long len; 1236 u8 bar_id = 2; 1237 int ret; 1238 1239 ret = pci_request_regions(pdev, KBUILD_MODNAME); 1240 if (ret) { 1241 rtw_err(rtwdev, "failed to request pci regions\n"); 1242 return ret; 1243 } 1244 1245 len = pci_resource_len(pdev, bar_id); 1246 rtwpci->mmap = pci_iomap(pdev, bar_id, len); 1247 if (!rtwpci->mmap) { 1248 pci_release_regions(pdev); 1249 rtw_err(rtwdev, "failed to map pci memory\n"); 1250 return -ENOMEM; 1251 } 1252 1253 return 0; 1254 } 1255 1256 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev, 1257 struct pci_dev *pdev) 1258 { 1259 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1260 1261 if (rtwpci->mmap) { 1262 pci_iounmap(pdev, rtwpci->mmap); 1263 pci_release_regions(pdev); 1264 } 1265 } 1266 1267 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data) 1268 { 1269 u16 write_addr; 1270 u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK); 1271 u8 flag; 1272 u8 cnt; 1273 1274 write_addr = addr & BITS_DBI_ADDR_MASK; 1275 write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN); 1276 rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data); 1277 rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr); 1278 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16); 1279 1280 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1281 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); 1282 if (flag == 0) 1283 return; 1284 1285 udelay(10); 1286 } 1287 1288 WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr); 1289 } 1290 1291 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value) 1292 { 1293 u16 read_addr = addr & BITS_DBI_ADDR_MASK; 1294 u8 flag; 1295 u8 cnt; 1296 1297 rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr); 1298 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16); 1299 1300 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1301 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); 1302 if (flag == 0) { 1303 read_addr = REG_DBI_RDATA_V1 + (addr & 3); 1304 *value = rtw_read8(rtwdev, read_addr); 1305 return 0; 1306 } 1307 1308 udelay(10); 1309 } 1310 1311 WARN(1, "failed to read DBI register, addr=0x%04x\n", addr); 1312 return -EIO; 1313 } 1314 1315 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1) 1316 { 1317 u8 page; 1318 u8 wflag; 1319 u8 cnt; 1320 1321 rtw_write16(rtwdev, REG_MDIO_V1, data); 1322 1323 page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1; 1324 page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2; 1325 rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK); 1326 rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page); 1327 rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1); 1328 1329 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1330 wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG, 1331 BIT_MDIO_WFLAG_V1); 1332 if (wflag == 0) 1333 return; 1334 1335 udelay(10); 1336 } 1337 1338 WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr); 1339 } 1340 1341 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable) 1342 { 1343 u8 value; 1344 int ret; 1345 1346 if (rtw_pci_disable_aspm) 1347 return; 1348 1349 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1350 if (ret) { 1351 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); 1352 return; 1353 } 1354 1355 if (enable) 1356 value |= BIT_CLKREQ_SW_EN; 1357 else 1358 value &= ~BIT_CLKREQ_SW_EN; 1359 1360 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1361 } 1362 1363 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable) 1364 { 1365 u8 value; 1366 int ret; 1367 1368 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1369 if (ret) { 1370 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); 1371 return; 1372 } 1373 1374 if (enable) 1375 value &= ~BIT_CLKREQ_N_PAD; 1376 else 1377 value |= BIT_CLKREQ_N_PAD; 1378 1379 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1380 } 1381 1382 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable) 1383 { 1384 u8 value; 1385 int ret; 1386 1387 if (rtw_pci_disable_aspm) 1388 return; 1389 1390 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1391 if (ret) { 1392 rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret); 1393 return; 1394 } 1395 1396 if (enable) 1397 value |= BIT_L1_SW_EN; 1398 else 1399 value &= ~BIT_L1_SW_EN; 1400 1401 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1402 } 1403 1404 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter) 1405 { 1406 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1407 1408 /* Like CLKREQ, ASPM is also implemented by two HW modules, and can 1409 * only be enabled when host supports it. 1410 * 1411 * And ASPM mechanism should be enabled when driver/firmware enters 1412 * power save mode, without having heavy traffic. Because we've 1413 * experienced some inter-operability issues that the link tends 1414 * to enter L1 state on the fly even when driver is having high 1415 * throughput. This is probably because the ASPM behavior slightly 1416 * varies from different SOC. 1417 */ 1418 if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)) 1419 return; 1420 1421 if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) || 1422 (!enter && atomic_inc_return(&rtwpci->link_usage) == 1)) 1423 rtw_pci_aspm_set(rtwdev, enter); 1424 } 1425 1426 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev) 1427 { 1428 struct rtw_chip_info *chip = rtwdev->chip; 1429 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1430 struct pci_dev *pdev = rtwpci->pdev; 1431 u16 link_ctrl; 1432 int ret; 1433 1434 /* RTL8822CE has enabled REFCLK auto calibration, it does not need 1435 * to add clock delay to cover the REFCLK timing gap. 1436 */ 1437 if (chip->id == RTW_CHIP_TYPE_8822C) 1438 rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0); 1439 1440 /* Though there is standard PCIE configuration space to set the 1441 * link control register, but by Realtek's design, driver should 1442 * check if host supports CLKREQ/ASPM to enable the HW module. 1443 * 1444 * These functions are implemented by two HW modules associated, 1445 * one is responsible to access PCIE configuration space to 1446 * follow the host settings, and another is in charge of doing 1447 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes 1448 * the host does not support it, and due to some reasons or wrong 1449 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device 1450 * loss if HW misbehaves on the link. 1451 * 1452 * Hence it's designed that driver should first check the PCIE 1453 * configuration space is sync'ed and enabled, then driver can turn 1454 * on the other module that is actually working on the mechanism. 1455 */ 1456 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl); 1457 if (ret) { 1458 rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret); 1459 return; 1460 } 1461 1462 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN) 1463 rtw_pci_clkreq_set(rtwdev, true); 1464 1465 rtwpci->link_ctrl = link_ctrl; 1466 } 1467 1468 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev) 1469 { 1470 struct rtw_chip_info *chip = rtwdev->chip; 1471 1472 switch (chip->id) { 1473 case RTW_CHIP_TYPE_8822C: 1474 if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D) 1475 rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG, 1476 BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1); 1477 break; 1478 default: 1479 break; 1480 } 1481 } 1482 1483 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev) 1484 { 1485 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1486 struct rtw_chip_info *chip = rtwdev->chip; 1487 struct pci_dev *pdev = rtwpci->pdev; 1488 const struct rtw_intf_phy_para *para; 1489 u16 cut; 1490 u16 value; 1491 u16 offset; 1492 int i; 1493 int ret; 1494 1495 cut = BIT(0) << rtwdev->hal.cut_version; 1496 1497 for (i = 0; i < chip->intf_table->n_gen1_para; i++) { 1498 para = &chip->intf_table->gen1_para[i]; 1499 if (!(para->cut_mask & cut)) 1500 continue; 1501 if (para->offset == 0xffff) 1502 break; 1503 offset = para->offset; 1504 value = para->value; 1505 if (para->ip_sel == RTW_IP_SEL_PHY) 1506 rtw_mdio_write(rtwdev, offset, value, true); 1507 else 1508 rtw_dbi_write8(rtwdev, offset, value); 1509 } 1510 1511 for (i = 0; i < chip->intf_table->n_gen2_para; i++) { 1512 para = &chip->intf_table->gen2_para[i]; 1513 if (!(para->cut_mask & cut)) 1514 continue; 1515 if (para->offset == 0xffff) 1516 break; 1517 offset = para->offset; 1518 value = para->value; 1519 if (para->ip_sel == RTW_IP_SEL_PHY) 1520 rtw_mdio_write(rtwdev, offset, value, false); 1521 else 1522 rtw_dbi_write8(rtwdev, offset, value); 1523 } 1524 1525 rtw_pci_link_cfg(rtwdev); 1526 1527 /* Disable 8821ce completion timeout by default */ 1528 if (chip->id == RTW_CHIP_TYPE_8821C) { 1529 ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2, 1530 PCI_EXP_DEVCTL2_COMP_TMOUT_DIS); 1531 if (ret) 1532 rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n", 1533 ret); 1534 } 1535 } 1536 1537 static int __maybe_unused rtw_pci_suspend(struct device *dev) 1538 { 1539 struct ieee80211_hw *hw = dev_get_drvdata(dev); 1540 struct rtw_dev *rtwdev = hw->priv; 1541 struct rtw_chip_info *chip = rtwdev->chip; 1542 struct rtw_efuse *efuse = &rtwdev->efuse; 1543 1544 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) 1545 rtw_pci_clkreq_pad_low(rtwdev, true); 1546 return 0; 1547 } 1548 1549 static int __maybe_unused rtw_pci_resume(struct device *dev) 1550 { 1551 struct ieee80211_hw *hw = dev_get_drvdata(dev); 1552 struct rtw_dev *rtwdev = hw->priv; 1553 struct rtw_chip_info *chip = rtwdev->chip; 1554 struct rtw_efuse *efuse = &rtwdev->efuse; 1555 1556 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) 1557 rtw_pci_clkreq_pad_low(rtwdev, false); 1558 return 0; 1559 } 1560 1561 SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume); 1562 EXPORT_SYMBOL(rtw_pm_ops); 1563 1564 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1565 { 1566 int ret; 1567 1568 ret = pci_enable_device(pdev); 1569 if (ret) { 1570 rtw_err(rtwdev, "failed to enable pci device\n"); 1571 return ret; 1572 } 1573 1574 pci_set_master(pdev); 1575 pci_set_drvdata(pdev, rtwdev->hw); 1576 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev); 1577 1578 return 0; 1579 } 1580 1581 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1582 { 1583 pci_clear_master(pdev); 1584 pci_disable_device(pdev); 1585 } 1586 1587 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1588 { 1589 struct rtw_pci *rtwpci; 1590 int ret; 1591 1592 rtwpci = (struct rtw_pci *)rtwdev->priv; 1593 rtwpci->pdev = pdev; 1594 1595 /* after this driver can access to hw registers */ 1596 ret = rtw_pci_io_mapping(rtwdev, pdev); 1597 if (ret) { 1598 rtw_err(rtwdev, "failed to request pci io region\n"); 1599 goto err_out; 1600 } 1601 1602 ret = rtw_pci_init(rtwdev); 1603 if (ret) { 1604 rtw_err(rtwdev, "failed to allocate pci resources\n"); 1605 goto err_io_unmap; 1606 } 1607 1608 return 0; 1609 1610 err_io_unmap: 1611 rtw_pci_io_unmapping(rtwdev, pdev); 1612 1613 err_out: 1614 return ret; 1615 } 1616 1617 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1618 { 1619 rtw_pci_deinit(rtwdev); 1620 rtw_pci_io_unmapping(rtwdev, pdev); 1621 } 1622 1623 static struct rtw_hci_ops rtw_pci_ops = { 1624 .tx_write = rtw_pci_tx_write, 1625 .tx_kick_off = rtw_pci_tx_kick_off, 1626 .flush_queues = rtw_pci_flush_queues, 1627 .setup = rtw_pci_setup, 1628 .start = rtw_pci_start, 1629 .stop = rtw_pci_stop, 1630 .deep_ps = rtw_pci_deep_ps, 1631 .link_ps = rtw_pci_link_ps, 1632 .interface_cfg = rtw_pci_interface_cfg, 1633 1634 .read8 = rtw_pci_read8, 1635 .read16 = rtw_pci_read16, 1636 .read32 = rtw_pci_read32, 1637 .write8 = rtw_pci_write8, 1638 .write16 = rtw_pci_write16, 1639 .write32 = rtw_pci_write32, 1640 .write_data_rsvd_page = rtw_pci_write_data_rsvd_page, 1641 .write_data_h2c = rtw_pci_write_data_h2c, 1642 }; 1643 1644 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1645 { 1646 unsigned int flags = PCI_IRQ_LEGACY; 1647 int ret; 1648 1649 if (!rtw_disable_msi) 1650 flags |= PCI_IRQ_MSI; 1651 1652 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags); 1653 if (ret < 0) { 1654 rtw_err(rtwdev, "failed to alloc PCI irq vectors\n"); 1655 return ret; 1656 } 1657 1658 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq, 1659 rtw_pci_interrupt_handler, 1660 rtw_pci_interrupt_threadfn, 1661 IRQF_SHARED, KBUILD_MODNAME, rtwdev); 1662 if (ret) { 1663 rtw_err(rtwdev, "failed to request irq %d\n", ret); 1664 pci_free_irq_vectors(pdev); 1665 } 1666 1667 return ret; 1668 } 1669 1670 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1671 { 1672 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev); 1673 pci_free_irq_vectors(pdev); 1674 } 1675 1676 static int rtw_pci_napi_poll(struct napi_struct *napi, int budget) 1677 { 1678 struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi); 1679 struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev, 1680 priv); 1681 int work_done = 0; 1682 1683 if (rtwpci->rx_no_aspm) 1684 rtw_pci_link_ps(rtwdev, false); 1685 1686 while (work_done < budget) { 1687 u32 work_done_once; 1688 1689 work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU, 1690 budget - work_done); 1691 if (work_done_once == 0) 1692 break; 1693 work_done += work_done_once; 1694 } 1695 if (work_done < budget) { 1696 napi_complete_done(napi, work_done); 1697 spin_lock_bh(&rtwpci->irq_lock); 1698 if (rtwpci->running) 1699 rtw_pci_enable_interrupt(rtwdev, rtwpci, false); 1700 spin_unlock_bh(&rtwpci->irq_lock); 1701 /* When ISR happens during polling and before napi_complete 1702 * while no further data is received. Data on the dma_ring will 1703 * not be processed immediately. Check whether dma ring is 1704 * empty and perform napi_schedule accordingly. 1705 */ 1706 if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci)) 1707 napi_schedule(napi); 1708 } 1709 if (rtwpci->rx_no_aspm) 1710 rtw_pci_link_ps(rtwdev, true); 1711 1712 return work_done; 1713 } 1714 1715 static void rtw_pci_napi_init(struct rtw_dev *rtwdev) 1716 { 1717 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1718 1719 init_dummy_netdev(&rtwpci->netdev); 1720 netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll, 1721 NAPI_POLL_WEIGHT); 1722 } 1723 1724 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev) 1725 { 1726 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1727 1728 rtw_pci_napi_stop(rtwdev); 1729 netif_napi_del(&rtwpci->napi); 1730 } 1731 1732 int rtw_pci_probe(struct pci_dev *pdev, 1733 const struct pci_device_id *id) 1734 { 1735 struct pci_dev *bridge = pci_upstream_bridge(pdev); 1736 struct ieee80211_hw *hw; 1737 struct rtw_dev *rtwdev; 1738 struct rtw_pci *rtwpci; 1739 int drv_data_size; 1740 int ret; 1741 1742 drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci); 1743 hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops); 1744 if (!hw) { 1745 dev_err(&pdev->dev, "failed to allocate hw\n"); 1746 return -ENOMEM; 1747 } 1748 1749 rtwdev = hw->priv; 1750 rtwdev->hw = hw; 1751 rtwdev->dev = &pdev->dev; 1752 rtwdev->chip = (struct rtw_chip_info *)id->driver_data; 1753 rtwdev->hci.ops = &rtw_pci_ops; 1754 rtwdev->hci.type = RTW_HCI_TYPE_PCIE; 1755 1756 rtwpci = (struct rtw_pci *)rtwdev->priv; 1757 atomic_set(&rtwpci->link_usage, 1); 1758 1759 ret = rtw_core_init(rtwdev); 1760 if (ret) 1761 goto err_release_hw; 1762 1763 rtw_dbg(rtwdev, RTW_DBG_PCI, 1764 "rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n", 1765 pdev->vendor, pdev->device, pdev->revision); 1766 1767 ret = rtw_pci_claim(rtwdev, pdev); 1768 if (ret) { 1769 rtw_err(rtwdev, "failed to claim pci device\n"); 1770 goto err_deinit_core; 1771 } 1772 1773 ret = rtw_pci_setup_resource(rtwdev, pdev); 1774 if (ret) { 1775 rtw_err(rtwdev, "failed to setup pci resources\n"); 1776 goto err_pci_declaim; 1777 } 1778 1779 rtw_pci_napi_init(rtwdev); 1780 1781 ret = rtw_chip_info_setup(rtwdev); 1782 if (ret) { 1783 rtw_err(rtwdev, "failed to setup chip information\n"); 1784 goto err_destroy_pci; 1785 } 1786 1787 /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */ 1788 if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL) 1789 rtwpci->rx_no_aspm = true; 1790 1791 rtw_pci_phy_cfg(rtwdev); 1792 1793 ret = rtw_register_hw(rtwdev, hw); 1794 if (ret) { 1795 rtw_err(rtwdev, "failed to register hw\n"); 1796 goto err_destroy_pci; 1797 } 1798 1799 ret = rtw_pci_request_irq(rtwdev, pdev); 1800 if (ret) { 1801 ieee80211_unregister_hw(hw); 1802 goto err_destroy_pci; 1803 } 1804 1805 return 0; 1806 1807 err_destroy_pci: 1808 rtw_pci_napi_deinit(rtwdev); 1809 rtw_pci_destroy(rtwdev, pdev); 1810 1811 err_pci_declaim: 1812 rtw_pci_declaim(rtwdev, pdev); 1813 1814 err_deinit_core: 1815 rtw_core_deinit(rtwdev); 1816 1817 err_release_hw: 1818 ieee80211_free_hw(hw); 1819 1820 return ret; 1821 } 1822 EXPORT_SYMBOL(rtw_pci_probe); 1823 1824 void rtw_pci_remove(struct pci_dev *pdev) 1825 { 1826 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 1827 struct rtw_dev *rtwdev; 1828 struct rtw_pci *rtwpci; 1829 1830 if (!hw) 1831 return; 1832 1833 rtwdev = hw->priv; 1834 rtwpci = (struct rtw_pci *)rtwdev->priv; 1835 1836 rtw_unregister_hw(rtwdev, hw); 1837 rtw_pci_disable_interrupt(rtwdev, rtwpci); 1838 rtw_pci_napi_deinit(rtwdev); 1839 rtw_pci_destroy(rtwdev, pdev); 1840 rtw_pci_declaim(rtwdev, pdev); 1841 rtw_pci_free_irq(rtwdev, pdev); 1842 rtw_core_deinit(rtwdev); 1843 ieee80211_free_hw(hw); 1844 } 1845 EXPORT_SYMBOL(rtw_pci_remove); 1846 1847 void rtw_pci_shutdown(struct pci_dev *pdev) 1848 { 1849 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 1850 struct rtw_dev *rtwdev; 1851 struct rtw_chip_info *chip; 1852 1853 if (!hw) 1854 return; 1855 1856 rtwdev = hw->priv; 1857 chip = rtwdev->chip; 1858 1859 if (chip->ops->shutdown) 1860 chip->ops->shutdown(rtwdev); 1861 1862 pci_set_power_state(pdev, PCI_D3hot); 1863 } 1864 EXPORT_SYMBOL(rtw_pci_shutdown); 1865 1866 MODULE_AUTHOR("Realtek Corporation"); 1867 MODULE_DESCRIPTION("Realtek 802.11ac wireless PCI driver"); 1868 MODULE_LICENSE("Dual BSD/GPL"); 1869