1 /* 2 * Copyright (C) 2015 Cavium, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of version 2 of the GNU General Public License 6 * as published by the Free Software Foundation. 7 */ 8 9 #include <linux/pci.h> 10 #include <linux/netdevice.h> 11 #include <linux/ip.h> 12 #include <linux/etherdevice.h> 13 #include <net/ip.h> 14 #include <net/tso.h> 15 16 #include "nic_reg.h" 17 #include "nic.h" 18 #include "q_struct.h" 19 #include "nicvf_queues.h" 20 21 static void nicvf_get_page(struct nicvf *nic) 22 { 23 if (!nic->rb_pageref || !nic->rb_page) 24 return; 25 26 page_ref_add(nic->rb_page, nic->rb_pageref); 27 nic->rb_pageref = 0; 28 } 29 30 /* Poll a register for a specific value */ 31 static int nicvf_poll_reg(struct nicvf *nic, int qidx, 32 u64 reg, int bit_pos, int bits, int val) 33 { 34 u64 bit_mask; 35 u64 reg_val; 36 int timeout = 10; 37 38 bit_mask = (1ULL << bits) - 1; 39 bit_mask = (bit_mask << bit_pos); 40 41 while (timeout) { 42 reg_val = nicvf_queue_reg_read(nic, reg, qidx); 43 if (((reg_val & bit_mask) >> bit_pos) == val) 44 return 0; 45 usleep_range(1000, 2000); 46 timeout--; 47 } 48 netdev_err(nic->netdev, "Poll on reg 0x%llx failed\n", reg); 49 return 1; 50 } 51 52 /* Allocate memory for a queue's descriptors */ 53 static int nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem, 54 int q_len, int desc_size, int align_bytes) 55 { 56 dmem->q_len = q_len; 57 dmem->size = (desc_size * q_len) + align_bytes; 58 /* Save address, need it while freeing */ 59 dmem->unalign_base = dma_zalloc_coherent(&nic->pdev->dev, dmem->size, 60 &dmem->dma, GFP_KERNEL); 61 if (!dmem->unalign_base) 62 return -ENOMEM; 63 64 /* Align memory address for 'align_bytes' */ 65 dmem->phys_base = NICVF_ALIGNED_ADDR((u64)dmem->dma, align_bytes); 66 dmem->base = dmem->unalign_base + (dmem->phys_base - dmem->dma); 67 return 0; 68 } 69 70 /* Free queue's descriptor memory */ 71 static void nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem) 72 { 73 if (!dmem) 74 return; 75 76 dma_free_coherent(&nic->pdev->dev, dmem->size, 77 dmem->unalign_base, dmem->dma); 78 dmem->unalign_base = NULL; 79 dmem->base = NULL; 80 } 81 82 /* Allocate buffer for packet reception 83 * HW returns memory address where packet is DMA'ed but not a pointer 84 * into RBDR ring, so save buffer address at the start of fragment and 85 * align the start address to a cache aligned address 86 */ 87 static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, gfp_t gfp, 88 u32 buf_len, u64 **rbuf) 89 { 90 int order = (PAGE_SIZE <= 4096) ? PAGE_ALLOC_COSTLY_ORDER : 0; 91 92 /* Check if request can be accomodated in previous allocated page */ 93 if (nic->rb_page && 94 ((nic->rb_page_offset + buf_len) < (PAGE_SIZE << order))) { 95 nic->rb_pageref++; 96 goto ret; 97 } 98 99 nicvf_get_page(nic); 100 nic->rb_page = NULL; 101 102 /* Allocate a new page */ 103 if (!nic->rb_page) { 104 nic->rb_page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN, 105 order); 106 if (!nic->rb_page) { 107 this_cpu_inc(nic->pnicvf->drv_stats-> 108 rcv_buffer_alloc_failures); 109 return -ENOMEM; 110 } 111 nic->rb_page_offset = 0; 112 } 113 114 ret: 115 *rbuf = (u64 *)((u64)page_address(nic->rb_page) + nic->rb_page_offset); 116 nic->rb_page_offset += buf_len; 117 118 return 0; 119 } 120 121 /* Build skb around receive buffer */ 122 static struct sk_buff *nicvf_rb_ptr_to_skb(struct nicvf *nic, 123 u64 rb_ptr, int len) 124 { 125 void *data; 126 struct sk_buff *skb; 127 128 data = phys_to_virt(rb_ptr); 129 130 /* Now build an skb to give to stack */ 131 skb = build_skb(data, RCV_FRAG_LEN); 132 if (!skb) { 133 put_page(virt_to_page(data)); 134 return NULL; 135 } 136 137 prefetch(skb->data); 138 return skb; 139 } 140 141 /* Allocate RBDR ring and populate receive buffers */ 142 static int nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr, 143 int ring_len, int buf_size) 144 { 145 int idx; 146 u64 *rbuf; 147 struct rbdr_entry_t *desc; 148 int err; 149 150 err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len, 151 sizeof(struct rbdr_entry_t), 152 NICVF_RCV_BUF_ALIGN_BYTES); 153 if (err) 154 return err; 155 156 rbdr->desc = rbdr->dmem.base; 157 /* Buffer size has to be in multiples of 128 bytes */ 158 rbdr->dma_size = buf_size; 159 rbdr->enable = true; 160 rbdr->thresh = RBDR_THRESH; 161 162 nic->rb_page = NULL; 163 for (idx = 0; idx < ring_len; idx++) { 164 err = nicvf_alloc_rcv_buffer(nic, GFP_KERNEL, RCV_FRAG_LEN, 165 &rbuf); 166 if (err) 167 return err; 168 169 desc = GET_RBDR_DESC(rbdr, idx); 170 desc->buf_addr = virt_to_phys(rbuf) >> NICVF_RCV_BUF_ALIGN; 171 } 172 173 nicvf_get_page(nic); 174 175 return 0; 176 } 177 178 /* Free RBDR ring and its receive buffers */ 179 static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr) 180 { 181 int head, tail; 182 u64 buf_addr; 183 struct rbdr_entry_t *desc; 184 185 if (!rbdr) 186 return; 187 188 rbdr->enable = false; 189 if (!rbdr->dmem.base) 190 return; 191 192 head = rbdr->head; 193 tail = rbdr->tail; 194 195 /* Free SKBs */ 196 while (head != tail) { 197 desc = GET_RBDR_DESC(rbdr, head); 198 buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN; 199 put_page(virt_to_page(phys_to_virt(buf_addr))); 200 head++; 201 head &= (rbdr->dmem.q_len - 1); 202 } 203 /* Free SKB of tail desc */ 204 desc = GET_RBDR_DESC(rbdr, tail); 205 buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN; 206 put_page(virt_to_page(phys_to_virt(buf_addr))); 207 208 /* Free RBDR ring */ 209 nicvf_free_q_desc_mem(nic, &rbdr->dmem); 210 } 211 212 /* Refill receive buffer descriptors with new buffers. 213 */ 214 static void nicvf_refill_rbdr(struct nicvf *nic, gfp_t gfp) 215 { 216 struct queue_set *qs = nic->qs; 217 int rbdr_idx = qs->rbdr_cnt; 218 int tail, qcount; 219 int refill_rb_cnt; 220 struct rbdr *rbdr; 221 struct rbdr_entry_t *desc; 222 u64 *rbuf; 223 int new_rb = 0; 224 225 refill: 226 if (!rbdr_idx) 227 return; 228 rbdr_idx--; 229 rbdr = &qs->rbdr[rbdr_idx]; 230 /* Check if it's enabled */ 231 if (!rbdr->enable) 232 goto next_rbdr; 233 234 /* Get no of desc's to be refilled */ 235 qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx); 236 qcount &= 0x7FFFF; 237 /* Doorbell can be ringed with a max of ring size minus 1 */ 238 if (qcount >= (qs->rbdr_len - 1)) 239 goto next_rbdr; 240 else 241 refill_rb_cnt = qs->rbdr_len - qcount - 1; 242 243 /* Start filling descs from tail */ 244 tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3; 245 while (refill_rb_cnt) { 246 tail++; 247 tail &= (rbdr->dmem.q_len - 1); 248 249 if (nicvf_alloc_rcv_buffer(nic, gfp, RCV_FRAG_LEN, &rbuf)) 250 break; 251 252 desc = GET_RBDR_DESC(rbdr, tail); 253 desc->buf_addr = virt_to_phys(rbuf) >> NICVF_RCV_BUF_ALIGN; 254 refill_rb_cnt--; 255 new_rb++; 256 } 257 258 nicvf_get_page(nic); 259 260 /* make sure all memory stores are done before ringing doorbell */ 261 smp_wmb(); 262 263 /* Check if buffer allocation failed */ 264 if (refill_rb_cnt) 265 nic->rb_alloc_fail = true; 266 else 267 nic->rb_alloc_fail = false; 268 269 /* Notify HW */ 270 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, 271 rbdr_idx, new_rb); 272 next_rbdr: 273 /* Re-enable RBDR interrupts only if buffer allocation is success */ 274 if (!nic->rb_alloc_fail && rbdr->enable && 275 netif_running(nic->pnicvf->netdev)) 276 nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx); 277 278 if (rbdr_idx) 279 goto refill; 280 } 281 282 /* Alloc rcv buffers in non-atomic mode for better success */ 283 void nicvf_rbdr_work(struct work_struct *work) 284 { 285 struct nicvf *nic = container_of(work, struct nicvf, rbdr_work.work); 286 287 nicvf_refill_rbdr(nic, GFP_KERNEL); 288 if (nic->rb_alloc_fail) 289 schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10)); 290 else 291 nic->rb_work_scheduled = false; 292 } 293 294 /* In Softirq context, alloc rcv buffers in atomic mode */ 295 void nicvf_rbdr_task(unsigned long data) 296 { 297 struct nicvf *nic = (struct nicvf *)data; 298 299 nicvf_refill_rbdr(nic, GFP_ATOMIC); 300 if (nic->rb_alloc_fail) { 301 nic->rb_work_scheduled = true; 302 schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10)); 303 } 304 } 305 306 /* Initialize completion queue */ 307 static int nicvf_init_cmp_queue(struct nicvf *nic, 308 struct cmp_queue *cq, int q_len) 309 { 310 int err; 311 312 err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE, 313 NICVF_CQ_BASE_ALIGN_BYTES); 314 if (err) 315 return err; 316 317 cq->desc = cq->dmem.base; 318 cq->thresh = pass1_silicon(nic->pdev) ? 0 : CMP_QUEUE_CQE_THRESH; 319 nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1; 320 321 return 0; 322 } 323 324 static void nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq) 325 { 326 if (!cq) 327 return; 328 if (!cq->dmem.base) 329 return; 330 331 nicvf_free_q_desc_mem(nic, &cq->dmem); 332 } 333 334 /* Initialize transmit queue */ 335 static int nicvf_init_snd_queue(struct nicvf *nic, 336 struct snd_queue *sq, int q_len) 337 { 338 int err; 339 340 err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE, 341 NICVF_SQ_BASE_ALIGN_BYTES); 342 if (err) 343 return err; 344 345 sq->desc = sq->dmem.base; 346 sq->skbuff = kcalloc(q_len, sizeof(u64), GFP_KERNEL); 347 if (!sq->skbuff) 348 return -ENOMEM; 349 sq->head = 0; 350 sq->tail = 0; 351 atomic_set(&sq->free_cnt, q_len - 1); 352 sq->thresh = SND_QUEUE_THRESH; 353 354 /* Preallocate memory for TSO segment's header */ 355 sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev, 356 q_len * TSO_HEADER_SIZE, 357 &sq->tso_hdrs_phys, GFP_KERNEL); 358 if (!sq->tso_hdrs) 359 return -ENOMEM; 360 361 return 0; 362 } 363 364 static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq) 365 { 366 struct sk_buff *skb; 367 368 if (!sq) 369 return; 370 if (!sq->dmem.base) 371 return; 372 373 if (sq->tso_hdrs) 374 dma_free_coherent(&nic->pdev->dev, 375 sq->dmem.q_len * TSO_HEADER_SIZE, 376 sq->tso_hdrs, sq->tso_hdrs_phys); 377 378 /* Free pending skbs in the queue */ 379 smp_rmb(); 380 while (sq->head != sq->tail) { 381 skb = (struct sk_buff *)sq->skbuff[sq->head]; 382 if (skb) 383 dev_kfree_skb_any(skb); 384 sq->head++; 385 sq->head &= (sq->dmem.q_len - 1); 386 } 387 kfree(sq->skbuff); 388 nicvf_free_q_desc_mem(nic, &sq->dmem); 389 } 390 391 static void nicvf_reclaim_snd_queue(struct nicvf *nic, 392 struct queue_set *qs, int qidx) 393 { 394 /* Disable send queue */ 395 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0); 396 /* Check if SQ is stopped */ 397 if (nicvf_poll_reg(nic, qidx, NIC_QSET_SQ_0_7_STATUS, 21, 1, 0x01)) 398 return; 399 /* Reset send queue */ 400 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); 401 } 402 403 static void nicvf_reclaim_rcv_queue(struct nicvf *nic, 404 struct queue_set *qs, int qidx) 405 { 406 union nic_mbx mbx = {}; 407 408 /* Make sure all packets in the pipeline are written back into mem */ 409 mbx.msg.msg = NIC_MBOX_MSG_RQ_SW_SYNC; 410 nicvf_send_msg_to_pf(nic, &mbx); 411 } 412 413 static void nicvf_reclaim_cmp_queue(struct nicvf *nic, 414 struct queue_set *qs, int qidx) 415 { 416 /* Disable timer threshold (doesn't get reset upon CQ reset */ 417 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0); 418 /* Disable completion queue */ 419 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, 0); 420 /* Reset completion queue */ 421 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); 422 } 423 424 static void nicvf_reclaim_rbdr(struct nicvf *nic, 425 struct rbdr *rbdr, int qidx) 426 { 427 u64 tmp, fifo_state; 428 int timeout = 10; 429 430 /* Save head and tail pointers for feeing up buffers */ 431 rbdr->head = nicvf_queue_reg_read(nic, 432 NIC_QSET_RBDR_0_1_HEAD, 433 qidx) >> 3; 434 rbdr->tail = nicvf_queue_reg_read(nic, 435 NIC_QSET_RBDR_0_1_TAIL, 436 qidx) >> 3; 437 438 /* If RBDR FIFO is in 'FAIL' state then do a reset first 439 * before relaiming. 440 */ 441 fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx); 442 if (((fifo_state >> 62) & 0x03) == 0x3) 443 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, 444 qidx, NICVF_RBDR_RESET); 445 446 /* Disable RBDR */ 447 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0); 448 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) 449 return; 450 while (1) { 451 tmp = nicvf_queue_reg_read(nic, 452 NIC_QSET_RBDR_0_1_PREFETCH_STATUS, 453 qidx); 454 if ((tmp & 0xFFFFFFFF) == ((tmp >> 32) & 0xFFFFFFFF)) 455 break; 456 usleep_range(1000, 2000); 457 timeout--; 458 if (!timeout) { 459 netdev_err(nic->netdev, 460 "Failed polling on prefetch status\n"); 461 return; 462 } 463 } 464 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, 465 qidx, NICVF_RBDR_RESET); 466 467 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02)) 468 return; 469 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0x00); 470 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) 471 return; 472 } 473 474 void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features) 475 { 476 u64 rq_cfg; 477 int sqs; 478 479 rq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_RQ_GEN_CFG, 0); 480 481 /* Enable first VLAN stripping */ 482 if (features & NETIF_F_HW_VLAN_CTAG_RX) 483 rq_cfg |= (1ULL << 25); 484 else 485 rq_cfg &= ~(1ULL << 25); 486 nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, rq_cfg); 487 488 /* Configure Secondary Qsets, if any */ 489 for (sqs = 0; sqs < nic->sqs_count; sqs++) 490 if (nic->snicvf[sqs]) 491 nicvf_queue_reg_write(nic->snicvf[sqs], 492 NIC_QSET_RQ_GEN_CFG, 0, rq_cfg); 493 } 494 495 static void nicvf_reset_rcv_queue_stats(struct nicvf *nic) 496 { 497 union nic_mbx mbx = {}; 498 499 /* Reset all RQ/SQ and VF stats */ 500 mbx.reset_stat.msg = NIC_MBOX_MSG_RESET_STAT_COUNTER; 501 mbx.reset_stat.rx_stat_mask = 0x3FFF; 502 mbx.reset_stat.tx_stat_mask = 0x1F; 503 mbx.reset_stat.rq_stat_mask = 0xFFFF; 504 mbx.reset_stat.sq_stat_mask = 0xFFFF; 505 nicvf_send_msg_to_pf(nic, &mbx); 506 } 507 508 /* Configures receive queue */ 509 static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, 510 int qidx, bool enable) 511 { 512 union nic_mbx mbx = {}; 513 struct rcv_queue *rq; 514 struct rq_cfg rq_cfg; 515 516 rq = &qs->rq[qidx]; 517 rq->enable = enable; 518 519 /* Disable receive queue */ 520 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0); 521 522 if (!rq->enable) { 523 nicvf_reclaim_rcv_queue(nic, qs, qidx); 524 return; 525 } 526 527 rq->cq_qs = qs->vnic_id; 528 rq->cq_idx = qidx; 529 rq->start_rbdr_qs = qs->vnic_id; 530 rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1; 531 rq->cont_rbdr_qs = qs->vnic_id; 532 rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1; 533 /* all writes of RBDR data to be loaded into L2 Cache as well*/ 534 rq->caching = 1; 535 536 /* Send a mailbox msg to PF to config RQ */ 537 mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; 538 mbx.rq.qs_num = qs->vnic_id; 539 mbx.rq.rq_num = qidx; 540 mbx.rq.cfg = (rq->caching << 26) | (rq->cq_qs << 19) | 541 (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) | 542 (rq->cont_qs_rbdr_idx << 8) | 543 (rq->start_rbdr_qs << 1) | (rq->start_qs_rbdr_idx); 544 nicvf_send_msg_to_pf(nic, &mbx); 545 546 mbx.rq.msg = NIC_MBOX_MSG_RQ_BP_CFG; 547 mbx.rq.cfg = BIT_ULL(63) | BIT_ULL(62) | 548 (RQ_PASS_RBDR_LVL << 16) | (RQ_PASS_CQ_LVL << 8) | 549 (qs->vnic_id << 0); 550 nicvf_send_msg_to_pf(nic, &mbx); 551 552 /* RQ drop config 553 * Enable CQ drop to reserve sufficient CQEs for all tx packets 554 */ 555 mbx.rq.msg = NIC_MBOX_MSG_RQ_DROP_CFG; 556 mbx.rq.cfg = BIT_ULL(63) | BIT_ULL(62) | 557 (RQ_PASS_RBDR_LVL << 40) | (RQ_DROP_RBDR_LVL << 32) | 558 (RQ_PASS_CQ_LVL << 16) | (RQ_DROP_CQ_LVL << 8); 559 nicvf_send_msg_to_pf(nic, &mbx); 560 561 if (!nic->sqs_mode && (qidx == 0)) { 562 /* Enable checking L3/L4 length and TCP/UDP checksums */ 563 nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, 564 (BIT(24) | BIT(23) | BIT(21))); 565 nicvf_config_vlan_stripping(nic, nic->netdev->features); 566 } 567 568 /* Enable Receive queue */ 569 memset(&rq_cfg, 0, sizeof(struct rq_cfg)); 570 rq_cfg.ena = 1; 571 rq_cfg.tcp_ena = 0; 572 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, *(u64 *)&rq_cfg); 573 } 574 575 /* Configures completion queue */ 576 void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs, 577 int qidx, bool enable) 578 { 579 struct cmp_queue *cq; 580 struct cq_cfg cq_cfg; 581 582 cq = &qs->cq[qidx]; 583 cq->enable = enable; 584 585 if (!cq->enable) { 586 nicvf_reclaim_cmp_queue(nic, qs, qidx); 587 return; 588 } 589 590 /* Reset completion queue */ 591 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); 592 593 if (!cq->enable) 594 return; 595 596 spin_lock_init(&cq->lock); 597 /* Set completion queue base address */ 598 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE, 599 qidx, (u64)(cq->dmem.phys_base)); 600 601 /* Enable Completion queue */ 602 memset(&cq_cfg, 0, sizeof(struct cq_cfg)); 603 cq_cfg.ena = 1; 604 cq_cfg.reset = 0; 605 cq_cfg.caching = 0; 606 cq_cfg.qsize = CMP_QSIZE; 607 cq_cfg.avg_con = 0; 608 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(u64 *)&cq_cfg); 609 610 /* Set threshold value for interrupt generation */ 611 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh); 612 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, 613 qidx, CMP_QUEUE_TIMER_THRESH); 614 } 615 616 /* Configures transmit queue */ 617 static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, 618 int qidx, bool enable) 619 { 620 union nic_mbx mbx = {}; 621 struct snd_queue *sq; 622 struct sq_cfg sq_cfg; 623 624 sq = &qs->sq[qidx]; 625 sq->enable = enable; 626 627 if (!sq->enable) { 628 nicvf_reclaim_snd_queue(nic, qs, qidx); 629 return; 630 } 631 632 /* Reset send queue */ 633 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); 634 635 sq->cq_qs = qs->vnic_id; 636 sq->cq_idx = qidx; 637 638 /* Send a mailbox msg to PF to config SQ */ 639 mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG; 640 mbx.sq.qs_num = qs->vnic_id; 641 mbx.sq.sq_num = qidx; 642 mbx.sq.sqs_mode = nic->sqs_mode; 643 mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx; 644 nicvf_send_msg_to_pf(nic, &mbx); 645 646 /* Set queue base address */ 647 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, 648 qidx, (u64)(sq->dmem.phys_base)); 649 650 /* Enable send queue & set queue size */ 651 memset(&sq_cfg, 0, sizeof(struct sq_cfg)); 652 sq_cfg.ena = 1; 653 sq_cfg.reset = 0; 654 sq_cfg.ldwb = 0; 655 sq_cfg.qsize = SND_QSIZE; 656 sq_cfg.tstmp_bgx_intf = 0; 657 sq_cfg.cq_limit = 0; 658 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(u64 *)&sq_cfg); 659 660 /* Set threshold value for interrupt generation */ 661 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh); 662 663 /* Set queue:cpu affinity for better load distribution */ 664 if (cpu_online(qidx)) { 665 cpumask_set_cpu(qidx, &sq->affinity_mask); 666 netif_set_xps_queue(nic->netdev, 667 &sq->affinity_mask, qidx); 668 } 669 } 670 671 /* Configures receive buffer descriptor ring */ 672 static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs, 673 int qidx, bool enable) 674 { 675 struct rbdr *rbdr; 676 struct rbdr_cfg rbdr_cfg; 677 678 rbdr = &qs->rbdr[qidx]; 679 nicvf_reclaim_rbdr(nic, rbdr, qidx); 680 if (!enable) 681 return; 682 683 /* Set descriptor base address */ 684 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE, 685 qidx, (u64)(rbdr->dmem.phys_base)); 686 687 /* Enable RBDR & set queue size */ 688 /* Buffer size should be in multiples of 128 bytes */ 689 memset(&rbdr_cfg, 0, sizeof(struct rbdr_cfg)); 690 rbdr_cfg.ena = 1; 691 rbdr_cfg.reset = 0; 692 rbdr_cfg.ldwb = 0; 693 rbdr_cfg.qsize = RBDR_SIZE; 694 rbdr_cfg.avg_con = 0; 695 rbdr_cfg.lines = rbdr->dma_size / 128; 696 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, 697 qidx, *(u64 *)&rbdr_cfg); 698 699 /* Notify HW */ 700 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, 701 qidx, qs->rbdr_len - 1); 702 703 /* Set threshold value for interrupt generation */ 704 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH, 705 qidx, rbdr->thresh - 1); 706 } 707 708 /* Requests PF to assign and enable Qset */ 709 void nicvf_qset_config(struct nicvf *nic, bool enable) 710 { 711 union nic_mbx mbx = {}; 712 struct queue_set *qs = nic->qs; 713 struct qs_cfg *qs_cfg; 714 715 if (!qs) { 716 netdev_warn(nic->netdev, 717 "Qset is still not allocated, don't init queues\n"); 718 return; 719 } 720 721 qs->enable = enable; 722 qs->vnic_id = nic->vf_id; 723 724 /* Send a mailbox msg to PF to config Qset */ 725 mbx.qs.msg = NIC_MBOX_MSG_QS_CFG; 726 mbx.qs.num = qs->vnic_id; 727 mbx.qs.sqs_count = nic->sqs_count; 728 729 mbx.qs.cfg = 0; 730 qs_cfg = (struct qs_cfg *)&mbx.qs.cfg; 731 if (qs->enable) { 732 qs_cfg->ena = 1; 733 #ifdef __BIG_ENDIAN 734 qs_cfg->be = 1; 735 #endif 736 qs_cfg->vnic = qs->vnic_id; 737 } 738 nicvf_send_msg_to_pf(nic, &mbx); 739 } 740 741 static void nicvf_free_resources(struct nicvf *nic) 742 { 743 int qidx; 744 struct queue_set *qs = nic->qs; 745 746 /* Free receive buffer descriptor ring */ 747 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 748 nicvf_free_rbdr(nic, &qs->rbdr[qidx]); 749 750 /* Free completion queue */ 751 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 752 nicvf_free_cmp_queue(nic, &qs->cq[qidx]); 753 754 /* Free send queue */ 755 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 756 nicvf_free_snd_queue(nic, &qs->sq[qidx]); 757 } 758 759 static int nicvf_alloc_resources(struct nicvf *nic) 760 { 761 int qidx; 762 struct queue_set *qs = nic->qs; 763 764 /* Alloc receive buffer descriptor ring */ 765 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { 766 if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len, 767 DMA_BUFFER_LEN)) 768 goto alloc_fail; 769 } 770 771 /* Alloc send queue */ 772 for (qidx = 0; qidx < qs->sq_cnt; qidx++) { 773 if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len)) 774 goto alloc_fail; 775 } 776 777 /* Alloc completion queue */ 778 for (qidx = 0; qidx < qs->cq_cnt; qidx++) { 779 if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len)) 780 goto alloc_fail; 781 } 782 783 return 0; 784 alloc_fail: 785 nicvf_free_resources(nic); 786 return -ENOMEM; 787 } 788 789 int nicvf_set_qset_resources(struct nicvf *nic) 790 { 791 struct queue_set *qs; 792 793 qs = devm_kzalloc(&nic->pdev->dev, sizeof(*qs), GFP_KERNEL); 794 if (!qs) 795 return -ENOMEM; 796 nic->qs = qs; 797 798 /* Set count of each queue */ 799 qs->rbdr_cnt = DEFAULT_RBDR_CNT; 800 qs->rq_cnt = min_t(u8, MAX_RCV_QUEUES_PER_QS, num_online_cpus()); 801 qs->sq_cnt = min_t(u8, MAX_SND_QUEUES_PER_QS, num_online_cpus()); 802 qs->cq_cnt = max_t(u8, qs->rq_cnt, qs->sq_cnt); 803 804 /* Set queue lengths */ 805 qs->rbdr_len = RCV_BUF_COUNT; 806 qs->sq_len = SND_QUEUE_LEN; 807 qs->cq_len = CMP_QUEUE_LEN; 808 809 nic->rx_queues = qs->rq_cnt; 810 nic->tx_queues = qs->sq_cnt; 811 812 return 0; 813 } 814 815 int nicvf_config_data_transfer(struct nicvf *nic, bool enable) 816 { 817 bool disable = false; 818 struct queue_set *qs = nic->qs; 819 int qidx; 820 821 if (!qs) 822 return 0; 823 824 if (enable) { 825 if (nicvf_alloc_resources(nic)) 826 return -ENOMEM; 827 828 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 829 nicvf_snd_queue_config(nic, qs, qidx, enable); 830 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 831 nicvf_cmp_queue_config(nic, qs, qidx, enable); 832 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 833 nicvf_rbdr_config(nic, qs, qidx, enable); 834 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 835 nicvf_rcv_queue_config(nic, qs, qidx, enable); 836 } else { 837 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 838 nicvf_rcv_queue_config(nic, qs, qidx, disable); 839 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 840 nicvf_rbdr_config(nic, qs, qidx, disable); 841 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 842 nicvf_snd_queue_config(nic, qs, qidx, disable); 843 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 844 nicvf_cmp_queue_config(nic, qs, qidx, disable); 845 846 nicvf_free_resources(nic); 847 } 848 849 /* Reset RXQ's stats. 850 * SQ's stats will get reset automatically once SQ is reset. 851 */ 852 nicvf_reset_rcv_queue_stats(nic); 853 854 return 0; 855 } 856 857 /* Get a free desc from SQ 858 * returns descriptor ponter & descriptor number 859 */ 860 static inline int nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt) 861 { 862 int qentry; 863 864 qentry = sq->tail; 865 atomic_sub(desc_cnt, &sq->free_cnt); 866 sq->tail += desc_cnt; 867 sq->tail &= (sq->dmem.q_len - 1); 868 869 return qentry; 870 } 871 872 /* Free descriptor back to SQ for future use */ 873 void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt) 874 { 875 atomic_add(desc_cnt, &sq->free_cnt); 876 sq->head += desc_cnt; 877 sq->head &= (sq->dmem.q_len - 1); 878 } 879 880 static inline int nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry) 881 { 882 qentry++; 883 qentry &= (sq->dmem.q_len - 1); 884 return qentry; 885 } 886 887 void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx) 888 { 889 u64 sq_cfg; 890 891 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); 892 sq_cfg |= NICVF_SQ_EN; 893 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); 894 /* Ring doorbell so that H/W restarts processing SQEs */ 895 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0); 896 } 897 898 void nicvf_sq_disable(struct nicvf *nic, int qidx) 899 { 900 u64 sq_cfg; 901 902 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); 903 sq_cfg &= ~NICVF_SQ_EN; 904 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); 905 } 906 907 void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq, 908 int qidx) 909 { 910 u64 head, tail; 911 struct sk_buff *skb; 912 struct nicvf *nic = netdev_priv(netdev); 913 struct sq_hdr_subdesc *hdr; 914 915 head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4; 916 tail = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_TAIL, qidx) >> 4; 917 while (sq->head != head) { 918 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head); 919 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) { 920 nicvf_put_sq_desc(sq, 1); 921 continue; 922 } 923 skb = (struct sk_buff *)sq->skbuff[sq->head]; 924 if (skb) 925 dev_kfree_skb_any(skb); 926 atomic64_add(1, (atomic64_t *)&netdev->stats.tx_packets); 927 atomic64_add(hdr->tot_len, 928 (atomic64_t *)&netdev->stats.tx_bytes); 929 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); 930 } 931 } 932 933 /* Calculate no of SQ subdescriptors needed to transmit all 934 * segments of this TSO packet. 935 * Taken from 'Tilera network driver' with a minor modification. 936 */ 937 static int nicvf_tso_count_subdescs(struct sk_buff *skb) 938 { 939 struct skb_shared_info *sh = skb_shinfo(skb); 940 unsigned int sh_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 941 unsigned int data_len = skb->len - sh_len; 942 unsigned int p_len = sh->gso_size; 943 long f_id = -1; /* id of the current fragment */ 944 long f_size = skb_headlen(skb) - sh_len; /* current fragment size */ 945 long f_used = 0; /* bytes used from the current fragment */ 946 long n; /* size of the current piece of payload */ 947 int num_edescs = 0; 948 int segment; 949 950 for (segment = 0; segment < sh->gso_segs; segment++) { 951 unsigned int p_used = 0; 952 953 /* One edesc for header and for each piece of the payload. */ 954 for (num_edescs++; p_used < p_len; num_edescs++) { 955 /* Advance as needed. */ 956 while (f_used >= f_size) { 957 f_id++; 958 f_size = skb_frag_size(&sh->frags[f_id]); 959 f_used = 0; 960 } 961 962 /* Use bytes from the current fragment. */ 963 n = p_len - p_used; 964 if (n > f_size - f_used) 965 n = f_size - f_used; 966 f_used += n; 967 p_used += n; 968 } 969 970 /* The last segment may be less than gso_size. */ 971 data_len -= p_len; 972 if (data_len < p_len) 973 p_len = data_len; 974 } 975 976 /* '+ gso_segs' for SQ_HDR_SUDESCs for each segment */ 977 return num_edescs + sh->gso_segs; 978 } 979 980 #define POST_CQE_DESC_COUNT 2 981 982 /* Get the number of SQ descriptors needed to xmit this skb */ 983 static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb) 984 { 985 int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT; 986 987 if (skb_shinfo(skb)->gso_size && !nic->hw_tso) { 988 subdesc_cnt = nicvf_tso_count_subdescs(skb); 989 return subdesc_cnt; 990 } 991 992 /* Dummy descriptors to get TSO pkt completion notification */ 993 if (nic->t88 && nic->hw_tso && skb_shinfo(skb)->gso_size) 994 subdesc_cnt += POST_CQE_DESC_COUNT; 995 996 if (skb_shinfo(skb)->nr_frags) 997 subdesc_cnt += skb_shinfo(skb)->nr_frags; 998 999 return subdesc_cnt; 1000 } 1001 1002 /* Add SQ HEADER subdescriptor. 1003 * First subdescriptor for every send descriptor. 1004 */ 1005 static inline void 1006 nicvf_sq_add_hdr_subdesc(struct nicvf *nic, struct snd_queue *sq, int qentry, 1007 int subdesc_cnt, struct sk_buff *skb, int len) 1008 { 1009 int proto; 1010 struct sq_hdr_subdesc *hdr; 1011 1012 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry); 1013 memset(hdr, 0, SND_QUEUE_DESC_SIZE); 1014 hdr->subdesc_type = SQ_DESC_TYPE_HEADER; 1015 1016 if (nic->t88 && nic->hw_tso && skb_shinfo(skb)->gso_size) { 1017 /* post_cqe = 0, to avoid HW posting a CQE for every TSO 1018 * segment transmitted on 88xx. 1019 */ 1020 hdr->subdesc_cnt = subdesc_cnt - POST_CQE_DESC_COUNT; 1021 } else { 1022 sq->skbuff[qentry] = (u64)skb; 1023 /* Enable notification via CQE after processing SQE */ 1024 hdr->post_cqe = 1; 1025 /* No of subdescriptors following this */ 1026 hdr->subdesc_cnt = subdesc_cnt; 1027 } 1028 hdr->tot_len = len; 1029 1030 /* Offload checksum calculation to HW */ 1031 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1032 hdr->csum_l3 = 1; /* Enable IP csum calculation */ 1033 hdr->l3_offset = skb_network_offset(skb); 1034 hdr->l4_offset = skb_transport_offset(skb); 1035 1036 proto = ip_hdr(skb)->protocol; 1037 switch (proto) { 1038 case IPPROTO_TCP: 1039 hdr->csum_l4 = SEND_L4_CSUM_TCP; 1040 break; 1041 case IPPROTO_UDP: 1042 hdr->csum_l4 = SEND_L4_CSUM_UDP; 1043 break; 1044 case IPPROTO_SCTP: 1045 hdr->csum_l4 = SEND_L4_CSUM_SCTP; 1046 break; 1047 } 1048 } 1049 1050 if (nic->hw_tso && skb_shinfo(skb)->gso_size) { 1051 hdr->tso = 1; 1052 hdr->tso_start = skb_transport_offset(skb) + tcp_hdrlen(skb); 1053 hdr->tso_max_paysize = skb_shinfo(skb)->gso_size; 1054 /* For non-tunneled pkts, point this to L2 ethertype */ 1055 hdr->inner_l3_offset = skb_network_offset(skb) - 2; 1056 this_cpu_inc(nic->pnicvf->drv_stats->tx_tso); 1057 } 1058 } 1059 1060 /* SQ GATHER subdescriptor 1061 * Must follow HDR descriptor 1062 */ 1063 static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry, 1064 int size, u64 data) 1065 { 1066 struct sq_gather_subdesc *gather; 1067 1068 qentry &= (sq->dmem.q_len - 1); 1069 gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry); 1070 1071 memset(gather, 0, SND_QUEUE_DESC_SIZE); 1072 gather->subdesc_type = SQ_DESC_TYPE_GATHER; 1073 gather->ld_type = NIC_SEND_LD_TYPE_E_LDD; 1074 gather->size = size; 1075 gather->addr = data; 1076 } 1077 1078 /* Add HDR + IMMEDIATE subdescriptors right after descriptors of a TSO 1079 * packet so that a CQE is posted as a notifation for transmission of 1080 * TSO packet. 1081 */ 1082 static inline void nicvf_sq_add_cqe_subdesc(struct snd_queue *sq, int qentry, 1083 int tso_sqe, struct sk_buff *skb) 1084 { 1085 struct sq_imm_subdesc *imm; 1086 struct sq_hdr_subdesc *hdr; 1087 1088 sq->skbuff[qentry] = (u64)skb; 1089 1090 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry); 1091 memset(hdr, 0, SND_QUEUE_DESC_SIZE); 1092 hdr->subdesc_type = SQ_DESC_TYPE_HEADER; 1093 /* Enable notification via CQE after processing SQE */ 1094 hdr->post_cqe = 1; 1095 /* There is no packet to transmit here */ 1096 hdr->dont_send = 1; 1097 hdr->subdesc_cnt = POST_CQE_DESC_COUNT - 1; 1098 hdr->tot_len = 1; 1099 /* Actual TSO header SQE index, needed for cleanup */ 1100 hdr->rsvd2 = tso_sqe; 1101 1102 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1103 imm = (struct sq_imm_subdesc *)GET_SQ_DESC(sq, qentry); 1104 memset(imm, 0, SND_QUEUE_DESC_SIZE); 1105 imm->subdesc_type = SQ_DESC_TYPE_IMMEDIATE; 1106 imm->len = 1; 1107 } 1108 1109 static inline void nicvf_sq_doorbell(struct nicvf *nic, struct sk_buff *skb, 1110 int sq_num, int desc_cnt) 1111 { 1112 struct netdev_queue *txq; 1113 1114 txq = netdev_get_tx_queue(nic->pnicvf->netdev, 1115 skb_get_queue_mapping(skb)); 1116 1117 netdev_tx_sent_queue(txq, skb->len); 1118 1119 /* make sure all memory stores are done before ringing doorbell */ 1120 smp_wmb(); 1121 1122 /* Inform HW to xmit all TSO segments */ 1123 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, 1124 sq_num, desc_cnt); 1125 } 1126 1127 /* Segment a TSO packet into 'gso_size' segments and append 1128 * them to SQ for transfer 1129 */ 1130 static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq, 1131 int sq_num, int qentry, struct sk_buff *skb) 1132 { 1133 struct tso_t tso; 1134 int seg_subdescs = 0, desc_cnt = 0; 1135 int seg_len, total_len, data_left; 1136 int hdr_qentry = qentry; 1137 int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 1138 1139 tso_start(skb, &tso); 1140 total_len = skb->len - hdr_len; 1141 while (total_len > 0) { 1142 char *hdr; 1143 1144 /* Save Qentry for adding HDR_SUBDESC at the end */ 1145 hdr_qentry = qentry; 1146 1147 data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len); 1148 total_len -= data_left; 1149 1150 /* Add segment's header */ 1151 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1152 hdr = sq->tso_hdrs + qentry * TSO_HEADER_SIZE; 1153 tso_build_hdr(skb, hdr, &tso, data_left, total_len == 0); 1154 nicvf_sq_add_gather_subdesc(sq, qentry, hdr_len, 1155 sq->tso_hdrs_phys + 1156 qentry * TSO_HEADER_SIZE); 1157 /* HDR_SUDESC + GATHER */ 1158 seg_subdescs = 2; 1159 seg_len = hdr_len; 1160 1161 /* Add segment's payload fragments */ 1162 while (data_left > 0) { 1163 int size; 1164 1165 size = min_t(int, tso.size, data_left); 1166 1167 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1168 nicvf_sq_add_gather_subdesc(sq, qentry, size, 1169 virt_to_phys(tso.data)); 1170 seg_subdescs++; 1171 seg_len += size; 1172 1173 data_left -= size; 1174 tso_build_data(skb, &tso, size); 1175 } 1176 nicvf_sq_add_hdr_subdesc(nic, sq, hdr_qentry, 1177 seg_subdescs - 1, skb, seg_len); 1178 sq->skbuff[hdr_qentry] = (u64)NULL; 1179 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1180 1181 desc_cnt += seg_subdescs; 1182 } 1183 /* Save SKB in the last segment for freeing */ 1184 sq->skbuff[hdr_qentry] = (u64)skb; 1185 1186 nicvf_sq_doorbell(nic, skb, sq_num, desc_cnt); 1187 1188 this_cpu_inc(nic->pnicvf->drv_stats->tx_tso); 1189 return 1; 1190 } 1191 1192 /* Append an skb to a SQ for packet transfer. */ 1193 int nicvf_sq_append_skb(struct nicvf *nic, struct snd_queue *sq, 1194 struct sk_buff *skb, u8 sq_num) 1195 { 1196 int i, size; 1197 int subdesc_cnt, tso_sqe = 0; 1198 int qentry; 1199 1200 subdesc_cnt = nicvf_sq_subdesc_required(nic, skb); 1201 if (subdesc_cnt > atomic_read(&sq->free_cnt)) 1202 goto append_fail; 1203 1204 qentry = nicvf_get_sq_desc(sq, subdesc_cnt); 1205 1206 /* Check if its a TSO packet */ 1207 if (skb_shinfo(skb)->gso_size && !nic->hw_tso) 1208 return nicvf_sq_append_tso(nic, sq, sq_num, qentry, skb); 1209 1210 /* Add SQ header subdesc */ 1211 nicvf_sq_add_hdr_subdesc(nic, sq, qentry, subdesc_cnt - 1, 1212 skb, skb->len); 1213 tso_sqe = qentry; 1214 1215 /* Add SQ gather subdescs */ 1216 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1217 size = skb_is_nonlinear(skb) ? skb_headlen(skb) : skb->len; 1218 nicvf_sq_add_gather_subdesc(sq, qentry, size, virt_to_phys(skb->data)); 1219 1220 /* Check for scattered buffer */ 1221 if (!skb_is_nonlinear(skb)) 1222 goto doorbell; 1223 1224 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1225 const struct skb_frag_struct *frag; 1226 1227 frag = &skb_shinfo(skb)->frags[i]; 1228 1229 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1230 size = skb_frag_size(frag); 1231 nicvf_sq_add_gather_subdesc(sq, qentry, size, 1232 virt_to_phys( 1233 skb_frag_address(frag))); 1234 } 1235 1236 doorbell: 1237 if (nic->t88 && skb_shinfo(skb)->gso_size) { 1238 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1239 nicvf_sq_add_cqe_subdesc(sq, qentry, tso_sqe, skb); 1240 } 1241 1242 nicvf_sq_doorbell(nic, skb, sq_num, subdesc_cnt); 1243 1244 return 1; 1245 1246 append_fail: 1247 /* Use original PCI dev for debug log */ 1248 nic = nic->pnicvf; 1249 netdev_dbg(nic->netdev, "Not enough SQ descriptors to xmit pkt\n"); 1250 return 0; 1251 } 1252 1253 static inline unsigned frag_num(unsigned i) 1254 { 1255 #ifdef __BIG_ENDIAN 1256 return (i & ~3) + 3 - (i & 3); 1257 #else 1258 return i; 1259 #endif 1260 } 1261 1262 /* Returns SKB for a received packet */ 1263 struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx) 1264 { 1265 int frag; 1266 int payload_len = 0; 1267 struct sk_buff *skb = NULL; 1268 struct page *page; 1269 int offset; 1270 u16 *rb_lens = NULL; 1271 u64 *rb_ptrs = NULL; 1272 1273 rb_lens = (void *)cqe_rx + (3 * sizeof(u64)); 1274 /* Except 88xx pass1 on all other chips CQE_RX2_S is added to 1275 * CQE_RX at word6, hence buffer pointers move by word 1276 * 1277 * Use existing 'hw_tso' flag which will be set for all chips 1278 * except 88xx pass1 instead of a additional cache line 1279 * access (or miss) by using pci dev's revision. 1280 */ 1281 if (!nic->hw_tso) 1282 rb_ptrs = (void *)cqe_rx + (6 * sizeof(u64)); 1283 else 1284 rb_ptrs = (void *)cqe_rx + (7 * sizeof(u64)); 1285 1286 netdev_dbg(nic->netdev, "%s rb_cnt %d rb0_ptr %llx rb0_sz %d\n", 1287 __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz); 1288 1289 for (frag = 0; frag < cqe_rx->rb_cnt; frag++) { 1290 payload_len = rb_lens[frag_num(frag)]; 1291 if (!frag) { 1292 /* First fragment */ 1293 skb = nicvf_rb_ptr_to_skb(nic, 1294 *rb_ptrs - cqe_rx->align_pad, 1295 payload_len); 1296 if (!skb) 1297 return NULL; 1298 skb_reserve(skb, cqe_rx->align_pad); 1299 skb_put(skb, payload_len); 1300 } else { 1301 /* Add fragments */ 1302 page = virt_to_page(phys_to_virt(*rb_ptrs)); 1303 offset = phys_to_virt(*rb_ptrs) - page_address(page); 1304 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 1305 offset, payload_len, RCV_FRAG_LEN); 1306 } 1307 /* Next buffer pointer */ 1308 rb_ptrs++; 1309 } 1310 return skb; 1311 } 1312 1313 static u64 nicvf_int_type_to_mask(int int_type, int q_idx) 1314 { 1315 u64 reg_val; 1316 1317 switch (int_type) { 1318 case NICVF_INTR_CQ: 1319 reg_val = ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT); 1320 break; 1321 case NICVF_INTR_SQ: 1322 reg_val = ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT); 1323 break; 1324 case NICVF_INTR_RBDR: 1325 reg_val = ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT); 1326 break; 1327 case NICVF_INTR_PKT_DROP: 1328 reg_val = (1ULL << NICVF_INTR_PKT_DROP_SHIFT); 1329 break; 1330 case NICVF_INTR_TCP_TIMER: 1331 reg_val = (1ULL << NICVF_INTR_TCP_TIMER_SHIFT); 1332 break; 1333 case NICVF_INTR_MBOX: 1334 reg_val = (1ULL << NICVF_INTR_MBOX_SHIFT); 1335 break; 1336 case NICVF_INTR_QS_ERR: 1337 reg_val = (1ULL << NICVF_INTR_QS_ERR_SHIFT); 1338 break; 1339 default: 1340 reg_val = 0; 1341 } 1342 1343 return reg_val; 1344 } 1345 1346 /* Enable interrupt */ 1347 void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx) 1348 { 1349 u64 mask = nicvf_int_type_to_mask(int_type, q_idx); 1350 1351 if (!mask) { 1352 netdev_dbg(nic->netdev, 1353 "Failed to enable interrupt: unknown type\n"); 1354 return; 1355 } 1356 nicvf_reg_write(nic, NIC_VF_ENA_W1S, 1357 nicvf_reg_read(nic, NIC_VF_ENA_W1S) | mask); 1358 } 1359 1360 /* Disable interrupt */ 1361 void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx) 1362 { 1363 u64 mask = nicvf_int_type_to_mask(int_type, q_idx); 1364 1365 if (!mask) { 1366 netdev_dbg(nic->netdev, 1367 "Failed to disable interrupt: unknown type\n"); 1368 return; 1369 } 1370 1371 nicvf_reg_write(nic, NIC_VF_ENA_W1C, mask); 1372 } 1373 1374 /* Clear interrupt */ 1375 void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx) 1376 { 1377 u64 mask = nicvf_int_type_to_mask(int_type, q_idx); 1378 1379 if (!mask) { 1380 netdev_dbg(nic->netdev, 1381 "Failed to clear interrupt: unknown type\n"); 1382 return; 1383 } 1384 1385 nicvf_reg_write(nic, NIC_VF_INT, mask); 1386 } 1387 1388 /* Check if interrupt is enabled */ 1389 int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx) 1390 { 1391 u64 mask = nicvf_int_type_to_mask(int_type, q_idx); 1392 /* If interrupt type is unknown, we treat it disabled. */ 1393 if (!mask) { 1394 netdev_dbg(nic->netdev, 1395 "Failed to check interrupt enable: unknown type\n"); 1396 return 0; 1397 } 1398 1399 return mask & nicvf_reg_read(nic, NIC_VF_ENA_W1S); 1400 } 1401 1402 void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx) 1403 { 1404 struct rcv_queue *rq; 1405 1406 #define GET_RQ_STATS(reg) \ 1407 nicvf_reg_read(nic, NIC_QSET_RQ_0_7_STAT_0_1 |\ 1408 (rq_idx << NIC_Q_NUM_SHIFT) | (reg << 3)) 1409 1410 rq = &nic->qs->rq[rq_idx]; 1411 rq->stats.bytes = GET_RQ_STATS(RQ_SQ_STATS_OCTS); 1412 rq->stats.pkts = GET_RQ_STATS(RQ_SQ_STATS_PKTS); 1413 } 1414 1415 void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx) 1416 { 1417 struct snd_queue *sq; 1418 1419 #define GET_SQ_STATS(reg) \ 1420 nicvf_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1 |\ 1421 (sq_idx << NIC_Q_NUM_SHIFT) | (reg << 3)) 1422 1423 sq = &nic->qs->sq[sq_idx]; 1424 sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS); 1425 sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS); 1426 } 1427 1428 /* Check for errors in the receive cmp.queue entry */ 1429 int nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cqe_rx_t *cqe_rx) 1430 { 1431 if (!cqe_rx->err_level && !cqe_rx->err_opcode) 1432 return 0; 1433 1434 if (netif_msg_rx_err(nic)) 1435 netdev_err(nic->netdev, 1436 "%s: RX error CQE err_level 0x%x err_opcode 0x%x\n", 1437 nic->netdev->name, 1438 cqe_rx->err_level, cqe_rx->err_opcode); 1439 1440 switch (cqe_rx->err_opcode) { 1441 case CQ_RX_ERROP_RE_PARTIAL: 1442 this_cpu_inc(nic->drv_stats->rx_bgx_truncated_pkts); 1443 break; 1444 case CQ_RX_ERROP_RE_JABBER: 1445 this_cpu_inc(nic->drv_stats->rx_jabber_errs); 1446 break; 1447 case CQ_RX_ERROP_RE_FCS: 1448 this_cpu_inc(nic->drv_stats->rx_fcs_errs); 1449 break; 1450 case CQ_RX_ERROP_RE_RX_CTL: 1451 this_cpu_inc(nic->drv_stats->rx_bgx_errs); 1452 break; 1453 case CQ_RX_ERROP_PREL2_ERR: 1454 this_cpu_inc(nic->drv_stats->rx_prel2_errs); 1455 break; 1456 case CQ_RX_ERROP_L2_MAL: 1457 this_cpu_inc(nic->drv_stats->rx_l2_hdr_malformed); 1458 break; 1459 case CQ_RX_ERROP_L2_OVERSIZE: 1460 this_cpu_inc(nic->drv_stats->rx_oversize); 1461 break; 1462 case CQ_RX_ERROP_L2_UNDERSIZE: 1463 this_cpu_inc(nic->drv_stats->rx_undersize); 1464 break; 1465 case CQ_RX_ERROP_L2_LENMISM: 1466 this_cpu_inc(nic->drv_stats->rx_l2_len_mismatch); 1467 break; 1468 case CQ_RX_ERROP_L2_PCLP: 1469 this_cpu_inc(nic->drv_stats->rx_l2_pclp); 1470 break; 1471 case CQ_RX_ERROP_IP_NOT: 1472 this_cpu_inc(nic->drv_stats->rx_ip_ver_errs); 1473 break; 1474 case CQ_RX_ERROP_IP_CSUM_ERR: 1475 this_cpu_inc(nic->drv_stats->rx_ip_csum_errs); 1476 break; 1477 case CQ_RX_ERROP_IP_MAL: 1478 this_cpu_inc(nic->drv_stats->rx_ip_hdr_malformed); 1479 break; 1480 case CQ_RX_ERROP_IP_MALD: 1481 this_cpu_inc(nic->drv_stats->rx_ip_payload_malformed); 1482 break; 1483 case CQ_RX_ERROP_IP_HOP: 1484 this_cpu_inc(nic->drv_stats->rx_ip_ttl_errs); 1485 break; 1486 case CQ_RX_ERROP_L3_PCLP: 1487 this_cpu_inc(nic->drv_stats->rx_l3_pclp); 1488 break; 1489 case CQ_RX_ERROP_L4_MAL: 1490 this_cpu_inc(nic->drv_stats->rx_l4_malformed); 1491 break; 1492 case CQ_RX_ERROP_L4_CHK: 1493 this_cpu_inc(nic->drv_stats->rx_l4_csum_errs); 1494 break; 1495 case CQ_RX_ERROP_UDP_LEN: 1496 this_cpu_inc(nic->drv_stats->rx_udp_len_errs); 1497 break; 1498 case CQ_RX_ERROP_L4_PORT: 1499 this_cpu_inc(nic->drv_stats->rx_l4_port_errs); 1500 break; 1501 case CQ_RX_ERROP_TCP_FLAG: 1502 this_cpu_inc(nic->drv_stats->rx_tcp_flag_errs); 1503 break; 1504 case CQ_RX_ERROP_TCP_OFFSET: 1505 this_cpu_inc(nic->drv_stats->rx_tcp_offset_errs); 1506 break; 1507 case CQ_RX_ERROP_L4_PCLP: 1508 this_cpu_inc(nic->drv_stats->rx_l4_pclp); 1509 break; 1510 case CQ_RX_ERROP_RBDR_TRUNC: 1511 this_cpu_inc(nic->drv_stats->rx_truncated_pkts); 1512 break; 1513 } 1514 1515 return 1; 1516 } 1517 1518 /* Check for errors in the send cmp.queue entry */ 1519 int nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cqe_send_t *cqe_tx) 1520 { 1521 switch (cqe_tx->send_status) { 1522 case CQ_TX_ERROP_GOOD: 1523 return 0; 1524 case CQ_TX_ERROP_DESC_FAULT: 1525 this_cpu_inc(nic->drv_stats->tx_desc_fault); 1526 break; 1527 case CQ_TX_ERROP_HDR_CONS_ERR: 1528 this_cpu_inc(nic->drv_stats->tx_hdr_cons_err); 1529 break; 1530 case CQ_TX_ERROP_SUBDC_ERR: 1531 this_cpu_inc(nic->drv_stats->tx_subdesc_err); 1532 break; 1533 case CQ_TX_ERROP_MAX_SIZE_VIOL: 1534 this_cpu_inc(nic->drv_stats->tx_max_size_exceeded); 1535 break; 1536 case CQ_TX_ERROP_IMM_SIZE_OFLOW: 1537 this_cpu_inc(nic->drv_stats->tx_imm_size_oflow); 1538 break; 1539 case CQ_TX_ERROP_DATA_SEQUENCE_ERR: 1540 this_cpu_inc(nic->drv_stats->tx_data_seq_err); 1541 break; 1542 case CQ_TX_ERROP_MEM_SEQUENCE_ERR: 1543 this_cpu_inc(nic->drv_stats->tx_mem_seq_err); 1544 break; 1545 case CQ_TX_ERROP_LOCK_VIOL: 1546 this_cpu_inc(nic->drv_stats->tx_lock_viol); 1547 break; 1548 case CQ_TX_ERROP_DATA_FAULT: 1549 this_cpu_inc(nic->drv_stats->tx_data_fault); 1550 break; 1551 case CQ_TX_ERROP_TSTMP_CONFLICT: 1552 this_cpu_inc(nic->drv_stats->tx_tstmp_conflict); 1553 break; 1554 case CQ_TX_ERROP_TSTMP_TIMEOUT: 1555 this_cpu_inc(nic->drv_stats->tx_tstmp_timeout); 1556 break; 1557 case CQ_TX_ERROP_MEM_FAULT: 1558 this_cpu_inc(nic->drv_stats->tx_mem_fault); 1559 break; 1560 case CQ_TX_ERROP_CK_OVERLAP: 1561 this_cpu_inc(nic->drv_stats->tx_csum_overlap); 1562 break; 1563 case CQ_TX_ERROP_CK_OFLOW: 1564 this_cpu_inc(nic->drv_stats->tx_csum_overflow); 1565 break; 1566 } 1567 1568 return 1; 1569 } 1570