1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/errno.h> 10 #include <linux/pci.h> 11 #include <linux/device.h> 12 #include <linux/netdevice.h> 13 #include <linux/etherdevice.h> 14 #include <linux/u64_stats_sync.h> 15 #include <linux/slab.h> 16 #include <linux/interrupt.h> 17 #include <linux/skbuff.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/prefetch.h> 20 #include <linux/cpumask.h> 21 #include <asm/barrier.h> 22 23 #include "hinic_common.h" 24 #include "hinic_hw_if.h" 25 #include "hinic_hw_wqe.h" 26 #include "hinic_hw_wq.h" 27 #include "hinic_hw_qp.h" 28 #include "hinic_hw_dev.h" 29 #include "hinic_rx.h" 30 #include "hinic_dev.h" 31 32 #define RX_IRQ_NO_PENDING 0 33 #define RX_IRQ_NO_COALESC 0 34 #define RX_IRQ_NO_LLI_TIMER 0 35 #define RX_IRQ_NO_CREDIT 0 36 #define RX_IRQ_NO_RESEND_TIMER 0 37 #define HINIC_RX_BUFFER_WRITE 16 38 39 /** 40 * hinic_rxq_clean_stats - Clean the statistics of specific queue 41 * @rxq: Logical Rx Queue 42 **/ 43 void hinic_rxq_clean_stats(struct hinic_rxq *rxq) 44 { 45 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 46 47 u64_stats_update_begin(&rxq_stats->syncp); 48 rxq_stats->pkts = 0; 49 rxq_stats->bytes = 0; 50 u64_stats_update_end(&rxq_stats->syncp); 51 } 52 53 /** 54 * hinic_rxq_get_stats - get statistics of Rx Queue 55 * @rxq: Logical Rx Queue 56 * @stats: return updated stats here 57 **/ 58 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) 59 { 60 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 61 unsigned int start; 62 63 u64_stats_update_begin(&stats->syncp); 64 do { 65 start = u64_stats_fetch_begin(&rxq_stats->syncp); 66 stats->pkts = rxq_stats->pkts; 67 stats->bytes = rxq_stats->bytes; 68 } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); 69 u64_stats_update_end(&stats->syncp); 70 } 71 72 /** 73 * rxq_stats_init - Initialize the statistics of specific queue 74 * @rxq: Logical Rx Queue 75 **/ 76 static void rxq_stats_init(struct hinic_rxq *rxq) 77 { 78 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 79 80 u64_stats_init(&rxq_stats->syncp); 81 hinic_rxq_clean_stats(rxq); 82 } 83 84 static void rx_csum(struct hinic_rxq *rxq, u16 cons_idx, 85 struct sk_buff *skb) 86 { 87 struct net_device *netdev = rxq->netdev; 88 struct hinic_rq_cqe *cqe; 89 struct hinic_rq *rq; 90 u32 csum_err; 91 u32 status; 92 93 rq = rxq->rq; 94 cqe = rq->cqe[cons_idx]; 95 status = be32_to_cpu(cqe->status); 96 csum_err = HINIC_RQ_CQE_STATUS_GET(status, CSUM_ERR); 97 98 if (!(netdev->features & NETIF_F_RXCSUM)) 99 return; 100 101 if (!csum_err) 102 skb->ip_summed = CHECKSUM_UNNECESSARY; 103 else 104 skb->ip_summed = CHECKSUM_NONE; 105 } 106 /** 107 * rx_alloc_skb - allocate skb and map it to dma address 108 * @rxq: rx queue 109 * @dma_addr: returned dma address for the skb 110 * 111 * Return skb 112 **/ 113 static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq, 114 dma_addr_t *dma_addr) 115 { 116 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 117 struct hinic_hwdev *hwdev = nic_dev->hwdev; 118 struct hinic_hwif *hwif = hwdev->hwif; 119 struct pci_dev *pdev = hwif->pdev; 120 struct sk_buff *skb; 121 dma_addr_t addr; 122 int err; 123 124 skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz); 125 if (!skb) { 126 netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n"); 127 return NULL; 128 } 129 130 addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz, 131 DMA_FROM_DEVICE); 132 err = dma_mapping_error(&pdev->dev, addr); 133 if (err) { 134 dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err); 135 goto err_rx_map; 136 } 137 138 *dma_addr = addr; 139 return skb; 140 141 err_rx_map: 142 dev_kfree_skb_any(skb); 143 return NULL; 144 } 145 146 /** 147 * rx_unmap_skb - unmap the dma address of the skb 148 * @rxq: rx queue 149 * @dma_addr: dma address of the skb 150 **/ 151 static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr) 152 { 153 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 154 struct hinic_hwdev *hwdev = nic_dev->hwdev; 155 struct hinic_hwif *hwif = hwdev->hwif; 156 struct pci_dev *pdev = hwif->pdev; 157 158 dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz, 159 DMA_FROM_DEVICE); 160 } 161 162 /** 163 * rx_free_skb - unmap and free skb 164 * @rxq: rx queue 165 * @skb: skb to free 166 * @dma_addr: dma address of the skb 167 **/ 168 static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb, 169 dma_addr_t dma_addr) 170 { 171 rx_unmap_skb(rxq, dma_addr); 172 dev_kfree_skb_any(skb); 173 } 174 175 /** 176 * rx_alloc_pkts - allocate pkts in rx queue 177 * @rxq: rx queue 178 * 179 * Return number of skbs allocated 180 **/ 181 static int rx_alloc_pkts(struct hinic_rxq *rxq) 182 { 183 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 184 struct hinic_rq_wqe *rq_wqe; 185 unsigned int free_wqebbs; 186 struct hinic_sge sge; 187 dma_addr_t dma_addr; 188 struct sk_buff *skb; 189 u16 prod_idx; 190 int i; 191 192 free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 193 194 /* Limit the allocation chunks */ 195 if (free_wqebbs > nic_dev->rx_weight) 196 free_wqebbs = nic_dev->rx_weight; 197 198 for (i = 0; i < free_wqebbs; i++) { 199 skb = rx_alloc_skb(rxq, &dma_addr); 200 if (!skb) { 201 netdev_err(rxq->netdev, "Failed to alloc Rx skb\n"); 202 goto skb_out; 203 } 204 205 hinic_set_sge(&sge, dma_addr, skb->len); 206 207 rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 208 &prod_idx); 209 if (!rq_wqe) { 210 rx_free_skb(rxq, skb, dma_addr); 211 goto skb_out; 212 } 213 214 hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge); 215 216 hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb); 217 } 218 219 skb_out: 220 if (i) { 221 wmb(); /* write all the wqes before update PI */ 222 223 hinic_rq_update(rxq->rq, prod_idx); 224 } 225 226 return i; 227 } 228 229 /** 230 * free_all_rx_skbs - free all skbs in rx queue 231 * @rxq: rx queue 232 **/ 233 static void free_all_rx_skbs(struct hinic_rxq *rxq) 234 { 235 struct hinic_rq *rq = rxq->rq; 236 struct hinic_hw_wqe *hw_wqe; 237 struct hinic_sge sge; 238 u16 ci; 239 240 while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) { 241 if (IS_ERR(hw_wqe)) 242 break; 243 244 hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge); 245 246 hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE); 247 248 rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge)); 249 } 250 } 251 252 /** 253 * rx_recv_jumbo_pkt - Rx handler for jumbo pkt 254 * @rxq: rx queue 255 * @head_skb: the first skb in the list 256 * @left_pkt_len: left size of the pkt exclude head skb 257 * @ci: consumer index 258 * 259 * Return number of wqes that used for the left of the pkt 260 **/ 261 static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb, 262 unsigned int left_pkt_len, u16 ci) 263 { 264 struct sk_buff *skb, *curr_skb = head_skb; 265 struct hinic_rq_wqe *rq_wqe; 266 unsigned int curr_len; 267 struct hinic_sge sge; 268 int num_wqes = 0; 269 270 while (left_pkt_len > 0) { 271 rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 272 &skb, &ci); 273 274 num_wqes++; 275 276 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 277 278 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 279 280 prefetch(skb->data); 281 282 curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ : 283 left_pkt_len; 284 285 left_pkt_len -= curr_len; 286 287 __skb_put(skb, curr_len); 288 289 if (curr_skb == head_skb) 290 skb_shinfo(head_skb)->frag_list = skb; 291 else 292 curr_skb->next = skb; 293 294 head_skb->len += skb->len; 295 head_skb->data_len += skb->len; 296 head_skb->truesize += skb->truesize; 297 298 curr_skb = skb; 299 } 300 301 return num_wqes; 302 } 303 304 /** 305 * rxq_recv - Rx handler 306 * @rxq: rx queue 307 * @budget: maximum pkts to process 308 * 309 * Return number of pkts received 310 **/ 311 static int rxq_recv(struct hinic_rxq *rxq, int budget) 312 { 313 struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq); 314 u64 pkt_len = 0, rx_bytes = 0; 315 struct hinic_rq_wqe *rq_wqe; 316 unsigned int free_wqebbs; 317 int num_wqes, pkts = 0; 318 struct hinic_sge sge; 319 struct sk_buff *skb; 320 u16 ci; 321 322 while (pkts < budget) { 323 num_wqes = 0; 324 325 rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb, 326 &ci); 327 if (!rq_wqe) 328 break; 329 330 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 331 332 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 333 334 rx_csum(rxq, ci, skb); 335 336 prefetch(skb->data); 337 338 pkt_len = sge.len; 339 340 if (pkt_len <= HINIC_RX_BUF_SZ) { 341 __skb_put(skb, pkt_len); 342 } else { 343 __skb_put(skb, HINIC_RX_BUF_SZ); 344 num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len - 345 HINIC_RX_BUF_SZ, ci); 346 } 347 348 hinic_rq_put_wqe(rxq->rq, ci, 349 (num_wqes + 1) * HINIC_RQ_WQE_SIZE); 350 351 skb_record_rx_queue(skb, qp->q_id); 352 skb->protocol = eth_type_trans(skb, rxq->netdev); 353 354 napi_gro_receive(&rxq->napi, skb); 355 356 pkts++; 357 rx_bytes += pkt_len; 358 } 359 360 free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 361 if (free_wqebbs > HINIC_RX_BUFFER_WRITE) 362 rx_alloc_pkts(rxq); 363 364 u64_stats_update_begin(&rxq->rxq_stats.syncp); 365 rxq->rxq_stats.pkts += pkts; 366 rxq->rxq_stats.bytes += rx_bytes; 367 u64_stats_update_end(&rxq->rxq_stats.syncp); 368 369 return pkts; 370 } 371 372 static int rx_poll(struct napi_struct *napi, int budget) 373 { 374 struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi); 375 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 376 struct hinic_rq *rq = rxq->rq; 377 int pkts; 378 379 pkts = rxq_recv(rxq, budget); 380 if (pkts >= budget) 381 return budget; 382 383 napi_complete(napi); 384 hinic_hwdev_set_msix_state(nic_dev->hwdev, 385 rq->msix_entry, 386 HINIC_MSIX_ENABLE); 387 388 return pkts; 389 } 390 391 static void rx_add_napi(struct hinic_rxq *rxq) 392 { 393 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 394 395 netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight); 396 napi_enable(&rxq->napi); 397 } 398 399 static void rx_del_napi(struct hinic_rxq *rxq) 400 { 401 napi_disable(&rxq->napi); 402 netif_napi_del(&rxq->napi); 403 } 404 405 static irqreturn_t rx_irq(int irq, void *data) 406 { 407 struct hinic_rxq *rxq = (struct hinic_rxq *)data; 408 struct hinic_rq *rq = rxq->rq; 409 struct hinic_dev *nic_dev; 410 411 /* Disable the interrupt until napi will be completed */ 412 nic_dev = netdev_priv(rxq->netdev); 413 hinic_hwdev_set_msix_state(nic_dev->hwdev, 414 rq->msix_entry, 415 HINIC_MSIX_DISABLE); 416 417 nic_dev = netdev_priv(rxq->netdev); 418 hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry); 419 420 napi_schedule(&rxq->napi); 421 return IRQ_HANDLED; 422 } 423 424 static int rx_request_irq(struct hinic_rxq *rxq) 425 { 426 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 427 struct hinic_hwdev *hwdev = nic_dev->hwdev; 428 struct hinic_rq *rq = rxq->rq; 429 struct hinic_qp *qp; 430 struct cpumask mask; 431 int err; 432 433 rx_add_napi(rxq); 434 435 hinic_hwdev_msix_set(hwdev, rq->msix_entry, 436 RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC, 437 RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT, 438 RX_IRQ_NO_RESEND_TIMER); 439 440 err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq); 441 if (err) { 442 rx_del_napi(rxq); 443 return err; 444 } 445 446 qp = container_of(rq, struct hinic_qp, rq); 447 cpumask_set_cpu(qp->q_id % num_online_cpus(), &mask); 448 return irq_set_affinity_hint(rq->irq, &mask); 449 } 450 451 static void rx_free_irq(struct hinic_rxq *rxq) 452 { 453 struct hinic_rq *rq = rxq->rq; 454 455 irq_set_affinity_hint(rq->irq, NULL); 456 free_irq(rq->irq, rxq); 457 rx_del_napi(rxq); 458 } 459 460 /** 461 * hinic_init_rxq - Initialize the Rx Queue 462 * @rxq: Logical Rx Queue 463 * @rq: Hardware Rx Queue to connect the Logical queue with 464 * @netdev: network device to connect the Logical queue with 465 * 466 * Return 0 - Success, negative - Failure 467 **/ 468 int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq, 469 struct net_device *netdev) 470 { 471 struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq); 472 int err, pkts, irqname_len; 473 474 rxq->netdev = netdev; 475 rxq->rq = rq; 476 477 rxq_stats_init(rxq); 478 479 irqname_len = snprintf(NULL, 0, "hinic_rxq%d", qp->q_id) + 1; 480 rxq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL); 481 if (!rxq->irq_name) 482 return -ENOMEM; 483 484 sprintf(rxq->irq_name, "hinic_rxq%d", qp->q_id); 485 486 pkts = rx_alloc_pkts(rxq); 487 if (!pkts) { 488 err = -ENOMEM; 489 goto err_rx_pkts; 490 } 491 492 err = rx_request_irq(rxq); 493 if (err) { 494 netdev_err(netdev, "Failed to request Rx irq\n"); 495 goto err_req_rx_irq; 496 } 497 498 return 0; 499 500 err_req_rx_irq: 501 err_rx_pkts: 502 free_all_rx_skbs(rxq); 503 devm_kfree(&netdev->dev, rxq->irq_name); 504 return err; 505 } 506 507 /** 508 * hinic_clean_rxq - Clean the Rx Queue 509 * @rxq: Logical Rx Queue 510 **/ 511 void hinic_clean_rxq(struct hinic_rxq *rxq) 512 { 513 struct net_device *netdev = rxq->netdev; 514 515 rx_free_irq(rxq); 516 517 free_all_rx_skbs(rxq); 518 devm_kfree(&netdev->dev, rxq->irq_name); 519 } 520