1 /* 2 * Huawei HiNIC PCI Express Linux driver 3 * Copyright(c) 2017 Huawei Technologies Co., Ltd 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * for more details. 13 * 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/errno.h> 19 #include <linux/pci.h> 20 #include <linux/device.h> 21 #include <linux/netdevice.h> 22 #include <linux/etherdevice.h> 23 #include <linux/u64_stats_sync.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/skbuff.h> 27 #include <linux/dma-mapping.h> 28 #include <linux/prefetch.h> 29 #include <linux/cpumask.h> 30 #include <asm/barrier.h> 31 32 #include "hinic_common.h" 33 #include "hinic_hw_if.h" 34 #include "hinic_hw_wqe.h" 35 #include "hinic_hw_wq.h" 36 #include "hinic_hw_qp.h" 37 #include "hinic_hw_dev.h" 38 #include "hinic_rx.h" 39 #include "hinic_dev.h" 40 41 #define RX_IRQ_NO_PENDING 0 42 #define RX_IRQ_NO_COALESC 0 43 #define RX_IRQ_NO_LLI_TIMER 0 44 #define RX_IRQ_NO_CREDIT 0 45 #define RX_IRQ_NO_RESEND_TIMER 0 46 47 /** 48 * hinic_rxq_clean_stats - Clean the statistics of specific queue 49 * @rxq: Logical Rx Queue 50 **/ 51 void hinic_rxq_clean_stats(struct hinic_rxq *rxq) 52 { 53 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 54 55 u64_stats_update_begin(&rxq_stats->syncp); 56 rxq_stats->pkts = 0; 57 rxq_stats->bytes = 0; 58 u64_stats_update_end(&rxq_stats->syncp); 59 } 60 61 /** 62 * hinic_rxq_get_stats - get statistics of Rx Queue 63 * @rxq: Logical Rx Queue 64 * @stats: return updated stats here 65 **/ 66 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) 67 { 68 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 69 unsigned int start; 70 71 u64_stats_update_begin(&stats->syncp); 72 do { 73 start = u64_stats_fetch_begin(&rxq_stats->syncp); 74 stats->pkts = rxq_stats->pkts; 75 stats->bytes = rxq_stats->bytes; 76 } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); 77 u64_stats_update_end(&stats->syncp); 78 } 79 80 /** 81 * rxq_stats_init - Initialize the statistics of specific queue 82 * @rxq: Logical Rx Queue 83 **/ 84 static void rxq_stats_init(struct hinic_rxq *rxq) 85 { 86 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 87 88 u64_stats_init(&rxq_stats->syncp); 89 hinic_rxq_clean_stats(rxq); 90 } 91 92 /** 93 * rx_alloc_skb - allocate skb and map it to dma address 94 * @rxq: rx queue 95 * @dma_addr: returned dma address for the skb 96 * 97 * Return skb 98 **/ 99 static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq, 100 dma_addr_t *dma_addr) 101 { 102 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 103 struct hinic_hwdev *hwdev = nic_dev->hwdev; 104 struct hinic_hwif *hwif = hwdev->hwif; 105 struct pci_dev *pdev = hwif->pdev; 106 struct sk_buff *skb; 107 dma_addr_t addr; 108 int err; 109 110 skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz); 111 if (!skb) { 112 netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n"); 113 return NULL; 114 } 115 116 addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz, 117 DMA_FROM_DEVICE); 118 err = dma_mapping_error(&pdev->dev, addr); 119 if (err) { 120 dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err); 121 goto err_rx_map; 122 } 123 124 *dma_addr = addr; 125 return skb; 126 127 err_rx_map: 128 dev_kfree_skb_any(skb); 129 return NULL; 130 } 131 132 /** 133 * rx_unmap_skb - unmap the dma address of the skb 134 * @rxq: rx queue 135 * @dma_addr: dma address of the skb 136 **/ 137 static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr) 138 { 139 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 140 struct hinic_hwdev *hwdev = nic_dev->hwdev; 141 struct hinic_hwif *hwif = hwdev->hwif; 142 struct pci_dev *pdev = hwif->pdev; 143 144 dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz, 145 DMA_FROM_DEVICE); 146 } 147 148 /** 149 * rx_free_skb - unmap and free skb 150 * @rxq: rx queue 151 * @skb: skb to free 152 * @dma_addr: dma address of the skb 153 **/ 154 static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb, 155 dma_addr_t dma_addr) 156 { 157 rx_unmap_skb(rxq, dma_addr); 158 dev_kfree_skb_any(skb); 159 } 160 161 /** 162 * rx_alloc_pkts - allocate pkts in rx queue 163 * @rxq: rx queue 164 * 165 * Return number of skbs allocated 166 **/ 167 static int rx_alloc_pkts(struct hinic_rxq *rxq) 168 { 169 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 170 struct hinic_rq_wqe *rq_wqe; 171 unsigned int free_wqebbs; 172 struct hinic_sge sge; 173 dma_addr_t dma_addr; 174 struct sk_buff *skb; 175 u16 prod_idx; 176 int i; 177 178 free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 179 180 /* Limit the allocation chunks */ 181 if (free_wqebbs > nic_dev->rx_weight) 182 free_wqebbs = nic_dev->rx_weight; 183 184 for (i = 0; i < free_wqebbs; i++) { 185 skb = rx_alloc_skb(rxq, &dma_addr); 186 if (!skb) { 187 netdev_err(rxq->netdev, "Failed to alloc Rx skb\n"); 188 goto skb_out; 189 } 190 191 hinic_set_sge(&sge, dma_addr, skb->len); 192 193 rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 194 &prod_idx); 195 if (!rq_wqe) { 196 rx_free_skb(rxq, skb, dma_addr); 197 goto skb_out; 198 } 199 200 hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge); 201 202 hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb); 203 } 204 205 skb_out: 206 if (i) { 207 wmb(); /* write all the wqes before update PI */ 208 209 hinic_rq_update(rxq->rq, prod_idx); 210 } 211 212 tasklet_schedule(&rxq->rx_task); 213 return i; 214 } 215 216 /** 217 * free_all_rx_skbs - free all skbs in rx queue 218 * @rxq: rx queue 219 **/ 220 static void free_all_rx_skbs(struct hinic_rxq *rxq) 221 { 222 struct hinic_rq *rq = rxq->rq; 223 struct hinic_hw_wqe *hw_wqe; 224 struct hinic_sge sge; 225 u16 ci; 226 227 while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) { 228 if (IS_ERR(hw_wqe)) 229 break; 230 231 hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge); 232 233 hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE); 234 235 rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge)); 236 } 237 } 238 239 /** 240 * rx_alloc_task - tasklet for queue allocation 241 * @data: rx queue 242 **/ 243 static void rx_alloc_task(unsigned long data) 244 { 245 struct hinic_rxq *rxq = (struct hinic_rxq *)data; 246 247 (void)rx_alloc_pkts(rxq); 248 } 249 250 /** 251 * rx_recv_jumbo_pkt - Rx handler for jumbo pkt 252 * @rxq: rx queue 253 * @head_skb: the first skb in the list 254 * @left_pkt_len: left size of the pkt exclude head skb 255 * @ci: consumer index 256 * 257 * Return number of wqes that used for the left of the pkt 258 **/ 259 static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb, 260 unsigned int left_pkt_len, u16 ci) 261 { 262 struct sk_buff *skb, *curr_skb = head_skb; 263 struct hinic_rq_wqe *rq_wqe; 264 unsigned int curr_len; 265 struct hinic_sge sge; 266 int num_wqes = 0; 267 268 while (left_pkt_len > 0) { 269 rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 270 &skb, &ci); 271 272 num_wqes++; 273 274 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 275 276 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 277 278 prefetch(skb->data); 279 280 curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ : 281 left_pkt_len; 282 283 left_pkt_len -= curr_len; 284 285 __skb_put(skb, curr_len); 286 287 if (curr_skb == head_skb) 288 skb_shinfo(head_skb)->frag_list = skb; 289 else 290 curr_skb->next = skb; 291 292 head_skb->len += skb->len; 293 head_skb->data_len += skb->len; 294 head_skb->truesize += skb->truesize; 295 296 curr_skb = skb; 297 } 298 299 return num_wqes; 300 } 301 302 /** 303 * rxq_recv - Rx handler 304 * @rxq: rx queue 305 * @budget: maximum pkts to process 306 * 307 * Return number of pkts received 308 **/ 309 static int rxq_recv(struct hinic_rxq *rxq, int budget) 310 { 311 struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq); 312 u64 pkt_len = 0, rx_bytes = 0; 313 struct hinic_rq_wqe *rq_wqe; 314 int num_wqes, pkts = 0; 315 struct hinic_sge sge; 316 struct sk_buff *skb; 317 u16 ci; 318 319 while (pkts < budget) { 320 num_wqes = 0; 321 322 rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb, 323 &ci); 324 if (!rq_wqe) 325 break; 326 327 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 328 329 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 330 331 prefetch(skb->data); 332 333 pkt_len = sge.len; 334 335 if (pkt_len <= HINIC_RX_BUF_SZ) { 336 __skb_put(skb, pkt_len); 337 } else { 338 __skb_put(skb, HINIC_RX_BUF_SZ); 339 num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len - 340 HINIC_RX_BUF_SZ, ci); 341 } 342 343 hinic_rq_put_wqe(rxq->rq, ci, 344 (num_wqes + 1) * HINIC_RQ_WQE_SIZE); 345 346 skb_record_rx_queue(skb, qp->q_id); 347 skb->protocol = eth_type_trans(skb, rxq->netdev); 348 349 napi_gro_receive(&rxq->napi, skb); 350 351 pkts++; 352 rx_bytes += pkt_len; 353 } 354 355 if (pkts) 356 tasklet_schedule(&rxq->rx_task); /* rx_alloc_pkts */ 357 358 u64_stats_update_begin(&rxq->rxq_stats.syncp); 359 rxq->rxq_stats.pkts += pkts; 360 rxq->rxq_stats.bytes += rx_bytes; 361 u64_stats_update_end(&rxq->rxq_stats.syncp); 362 363 return pkts; 364 } 365 366 static int rx_poll(struct napi_struct *napi, int budget) 367 { 368 struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi); 369 struct hinic_rq *rq = rxq->rq; 370 int pkts; 371 372 pkts = rxq_recv(rxq, budget); 373 if (pkts >= budget) 374 return budget; 375 376 napi_complete(napi); 377 enable_irq(rq->irq); 378 return pkts; 379 } 380 381 static void rx_add_napi(struct hinic_rxq *rxq) 382 { 383 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 384 385 netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight); 386 napi_enable(&rxq->napi); 387 } 388 389 static void rx_del_napi(struct hinic_rxq *rxq) 390 { 391 napi_disable(&rxq->napi); 392 netif_napi_del(&rxq->napi); 393 } 394 395 static irqreturn_t rx_irq(int irq, void *data) 396 { 397 struct hinic_rxq *rxq = (struct hinic_rxq *)data; 398 struct hinic_rq *rq = rxq->rq; 399 struct hinic_dev *nic_dev; 400 401 /* Disable the interrupt until napi will be completed */ 402 disable_irq_nosync(rq->irq); 403 404 nic_dev = netdev_priv(rxq->netdev); 405 hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry); 406 407 napi_schedule(&rxq->napi); 408 return IRQ_HANDLED; 409 } 410 411 static int rx_request_irq(struct hinic_rxq *rxq) 412 { 413 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 414 struct hinic_hwdev *hwdev = nic_dev->hwdev; 415 struct hinic_rq *rq = rxq->rq; 416 struct hinic_qp *qp; 417 struct cpumask mask; 418 int err; 419 420 rx_add_napi(rxq); 421 422 hinic_hwdev_msix_set(hwdev, rq->msix_entry, 423 RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC, 424 RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT, 425 RX_IRQ_NO_RESEND_TIMER); 426 427 err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq); 428 if (err) { 429 rx_del_napi(rxq); 430 return err; 431 } 432 433 qp = container_of(rq, struct hinic_qp, rq); 434 cpumask_set_cpu(qp->q_id % num_online_cpus(), &mask); 435 return irq_set_affinity_hint(rq->irq, &mask); 436 } 437 438 static void rx_free_irq(struct hinic_rxq *rxq) 439 { 440 struct hinic_rq *rq = rxq->rq; 441 442 irq_set_affinity_hint(rq->irq, NULL); 443 free_irq(rq->irq, rxq); 444 rx_del_napi(rxq); 445 } 446 447 /** 448 * hinic_init_rxq - Initialize the Rx Queue 449 * @rxq: Logical Rx Queue 450 * @rq: Hardware Rx Queue to connect the Logical queue with 451 * @netdev: network device to connect the Logical queue with 452 * 453 * Return 0 - Success, negative - Failure 454 **/ 455 int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq, 456 struct net_device *netdev) 457 { 458 struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq); 459 int err, pkts, irqname_len; 460 461 rxq->netdev = netdev; 462 rxq->rq = rq; 463 464 rxq_stats_init(rxq); 465 466 irqname_len = snprintf(NULL, 0, "hinic_rxq%d", qp->q_id) + 1; 467 rxq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL); 468 if (!rxq->irq_name) 469 return -ENOMEM; 470 471 sprintf(rxq->irq_name, "hinic_rxq%d", qp->q_id); 472 473 tasklet_init(&rxq->rx_task, rx_alloc_task, (unsigned long)rxq); 474 475 pkts = rx_alloc_pkts(rxq); 476 if (!pkts) { 477 err = -ENOMEM; 478 goto err_rx_pkts; 479 } 480 481 err = rx_request_irq(rxq); 482 if (err) { 483 netdev_err(netdev, "Failed to request Rx irq\n"); 484 goto err_req_rx_irq; 485 } 486 487 return 0; 488 489 err_req_rx_irq: 490 err_rx_pkts: 491 tasklet_kill(&rxq->rx_task); 492 free_all_rx_skbs(rxq); 493 devm_kfree(&netdev->dev, rxq->irq_name); 494 return err; 495 } 496 497 /** 498 * hinic_clean_rxq - Clean the Rx Queue 499 * @rxq: Logical Rx Queue 500 **/ 501 void hinic_clean_rxq(struct hinic_rxq *rxq) 502 { 503 struct net_device *netdev = rxq->netdev; 504 505 rx_free_irq(rxq); 506 507 tasklet_kill(&rxq->rx_task); 508 free_all_rx_skbs(rxq); 509 devm_kfree(&netdev->dev, rxq->irq_name); 510 } 511