1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along with 16 * this program; if not, write to the Free Software Foundation, Inc., 17 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * The full GNU General Public License is included in this distribution in 20 * the file called "COPYING". 21 * 22 * Contact Information: 23 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 * 26 ******************************************************************************/ 27 28 #include "i40e.h" 29 30 static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size, 31 u32 td_tag) 32 { 33 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA | 34 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | 35 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | 36 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | 37 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); 38 } 39 40 /** 41 * i40e_program_fdir_filter - Program a Flow Director filter 42 * @fdir_input: Packet data that will be filter parameters 43 * @pf: The pf pointer 44 * @add: True for add/update, False for remove 45 **/ 46 int i40e_program_fdir_filter(struct i40e_fdir_data *fdir_data, 47 struct i40e_pf *pf, bool add) 48 { 49 struct i40e_filter_program_desc *fdir_desc; 50 struct i40e_tx_buffer *tx_buf; 51 struct i40e_tx_desc *tx_desc; 52 struct i40e_ring *tx_ring; 53 struct i40e_vsi *vsi; 54 struct device *dev; 55 dma_addr_t dma; 56 u32 td_cmd = 0; 57 u16 i; 58 59 /* find existing FDIR VSI */ 60 vsi = NULL; 61 for (i = 0; i < pf->hw.func_caps.num_vsis; i++) 62 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) 63 vsi = pf->vsi[i]; 64 if (!vsi) 65 return -ENOENT; 66 67 tx_ring = &vsi->tx_rings[0]; 68 dev = tx_ring->dev; 69 70 dma = dma_map_single(dev, fdir_data->raw_packet, 71 I40E_FDIR_MAX_RAW_PACKET_LOOKUP, DMA_TO_DEVICE); 72 if (dma_mapping_error(dev, dma)) 73 goto dma_fail; 74 75 /* grab the next descriptor */ 76 fdir_desc = I40E_TX_FDIRDESC(tx_ring, tx_ring->next_to_use); 77 tx_buf = &tx_ring->tx_bi[tx_ring->next_to_use]; 78 tx_ring->next_to_use++; 79 if (tx_ring->next_to_use == tx_ring->count) 80 tx_ring->next_to_use = 0; 81 82 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32((fdir_data->q_index 83 << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) 84 & I40E_TXD_FLTR_QW0_QINDEX_MASK); 85 86 fdir_desc->qindex_flex_ptype_vsi |= cpu_to_le32((fdir_data->flex_off 87 << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) 88 & I40E_TXD_FLTR_QW0_FLEXOFF_MASK); 89 90 fdir_desc->qindex_flex_ptype_vsi |= cpu_to_le32((fdir_data->pctype 91 << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) 92 & I40E_TXD_FLTR_QW0_PCTYPE_MASK); 93 94 /* Use LAN VSI Id if not programmed by user */ 95 if (fdir_data->dest_vsi == 0) 96 fdir_desc->qindex_flex_ptype_vsi |= 97 cpu_to_le32((pf->vsi[pf->lan_vsi]->id) 98 << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT); 99 else 100 fdir_desc->qindex_flex_ptype_vsi |= 101 cpu_to_le32((fdir_data->dest_vsi 102 << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) 103 & I40E_TXD_FLTR_QW0_DEST_VSI_MASK); 104 105 fdir_desc->dtype_cmd_cntindex = 106 cpu_to_le32(I40E_TX_DESC_DTYPE_FILTER_PROG); 107 108 if (add) 109 fdir_desc->dtype_cmd_cntindex |= cpu_to_le32( 110 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE 111 << I40E_TXD_FLTR_QW1_PCMD_SHIFT); 112 else 113 fdir_desc->dtype_cmd_cntindex |= cpu_to_le32( 114 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE 115 << I40E_TXD_FLTR_QW1_PCMD_SHIFT); 116 117 fdir_desc->dtype_cmd_cntindex |= cpu_to_le32((fdir_data->dest_ctl 118 << I40E_TXD_FLTR_QW1_DEST_SHIFT) 119 & I40E_TXD_FLTR_QW1_DEST_MASK); 120 121 fdir_desc->dtype_cmd_cntindex |= cpu_to_le32( 122 (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) 123 & I40E_TXD_FLTR_QW1_FD_STATUS_MASK); 124 125 if (fdir_data->cnt_index != 0) { 126 fdir_desc->dtype_cmd_cntindex |= 127 cpu_to_le32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK); 128 fdir_desc->dtype_cmd_cntindex |= 129 cpu_to_le32((fdir_data->cnt_index 130 << I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) 131 & I40E_TXD_FLTR_QW1_CNTINDEX_MASK); 132 } 133 134 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id); 135 136 /* Now program a dummy descriptor */ 137 tx_desc = I40E_TX_DESC(tx_ring, tx_ring->next_to_use); 138 tx_buf = &tx_ring->tx_bi[tx_ring->next_to_use]; 139 tx_ring->next_to_use++; 140 if (tx_ring->next_to_use == tx_ring->count) 141 tx_ring->next_to_use = 0; 142 143 tx_desc->buffer_addr = cpu_to_le64(dma); 144 td_cmd = I40E_TX_DESC_CMD_EOP | 145 I40E_TX_DESC_CMD_RS | 146 I40E_TX_DESC_CMD_DUMMY; 147 148 tx_desc->cmd_type_offset_bsz = 149 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_LOOKUP, 0); 150 151 /* Mark the data descriptor to be watched */ 152 tx_buf->next_to_watch = tx_desc; 153 154 /* Force memory writes to complete before letting h/w 155 * know there are new descriptors to fetch. (Only 156 * applicable for weak-ordered memory model archs, 157 * such as IA-64). 158 */ 159 wmb(); 160 161 writel(tx_ring->next_to_use, tx_ring->tail); 162 return 0; 163 164 dma_fail: 165 return -1; 166 } 167 168 /** 169 * i40e_fd_handle_status - check the Programming Status for FD 170 * @rx_ring: the Rx ring for this descriptor 171 * @qw: the descriptor data 172 * @prog_id: the id originally used for programming 173 * 174 * This is used to verify if the FD programming or invalidation 175 * requested by SW to the HW is successful or not and take actions accordingly. 176 **/ 177 static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u32 qw, u8 prog_id) 178 { 179 struct pci_dev *pdev = rx_ring->vsi->back->pdev; 180 u32 error; 181 182 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> 183 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT; 184 185 /* for now just print the Status */ 186 dev_info(&pdev->dev, "FD programming id %02x, Status %08x\n", 187 prog_id, error); 188 } 189 190 /** 191 * i40e_unmap_tx_resource - Release a Tx buffer 192 * @ring: the ring that owns the buffer 193 * @tx_buffer: the buffer to free 194 **/ 195 static inline void i40e_unmap_tx_resource(struct i40e_ring *ring, 196 struct i40e_tx_buffer *tx_buffer) 197 { 198 if (tx_buffer->dma) { 199 if (tx_buffer->tx_flags & I40E_TX_FLAGS_MAPPED_AS_PAGE) 200 dma_unmap_page(ring->dev, 201 tx_buffer->dma, 202 tx_buffer->length, 203 DMA_TO_DEVICE); 204 else 205 dma_unmap_single(ring->dev, 206 tx_buffer->dma, 207 tx_buffer->length, 208 DMA_TO_DEVICE); 209 } 210 tx_buffer->dma = 0; 211 tx_buffer->time_stamp = 0; 212 } 213 214 /** 215 * i40e_clean_tx_ring - Free any empty Tx buffers 216 * @tx_ring: ring to be cleaned 217 **/ 218 void i40e_clean_tx_ring(struct i40e_ring *tx_ring) 219 { 220 struct i40e_tx_buffer *tx_buffer; 221 unsigned long bi_size; 222 u16 i; 223 224 /* ring already cleared, nothing to do */ 225 if (!tx_ring->tx_bi) 226 return; 227 228 /* Free all the Tx ring sk_buffs */ 229 for (i = 0; i < tx_ring->count; i++) { 230 tx_buffer = &tx_ring->tx_bi[i]; 231 i40e_unmap_tx_resource(tx_ring, tx_buffer); 232 if (tx_buffer->skb) 233 dev_kfree_skb_any(tx_buffer->skb); 234 tx_buffer->skb = NULL; 235 } 236 237 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 238 memset(tx_ring->tx_bi, 0, bi_size); 239 240 /* Zero out the descriptor ring */ 241 memset(tx_ring->desc, 0, tx_ring->size); 242 243 tx_ring->next_to_use = 0; 244 tx_ring->next_to_clean = 0; 245 } 246 247 /** 248 * i40e_free_tx_resources - Free Tx resources per queue 249 * @tx_ring: Tx descriptor ring for a specific queue 250 * 251 * Free all transmit software resources 252 **/ 253 void i40e_free_tx_resources(struct i40e_ring *tx_ring) 254 { 255 i40e_clean_tx_ring(tx_ring); 256 kfree(tx_ring->tx_bi); 257 tx_ring->tx_bi = NULL; 258 259 if (tx_ring->desc) { 260 dma_free_coherent(tx_ring->dev, tx_ring->size, 261 tx_ring->desc, tx_ring->dma); 262 tx_ring->desc = NULL; 263 } 264 } 265 266 /** 267 * i40e_get_tx_pending - how many tx descriptors not processed 268 * @tx_ring: the ring of descriptors 269 * 270 * Since there is no access to the ring head register 271 * in XL710, we need to use our local copies 272 **/ 273 static u32 i40e_get_tx_pending(struct i40e_ring *ring) 274 { 275 u32 ntu = ((ring->next_to_clean <= ring->next_to_use) 276 ? ring->next_to_use 277 : ring->next_to_use + ring->count); 278 return ntu - ring->next_to_clean; 279 } 280 281 /** 282 * i40e_check_tx_hang - Is there a hang in the Tx queue 283 * @tx_ring: the ring of descriptors 284 **/ 285 static bool i40e_check_tx_hang(struct i40e_ring *tx_ring) 286 { 287 u32 tx_pending = i40e_get_tx_pending(tx_ring); 288 bool ret = false; 289 290 clear_check_for_tx_hang(tx_ring); 291 292 /* Check for a hung queue, but be thorough. This verifies 293 * that a transmit has been completed since the previous 294 * check AND there is at least one packet pending. The 295 * ARMED bit is set to indicate a potential hang. The 296 * bit is cleared if a pause frame is received to remove 297 * false hang detection due to PFC or 802.3x frames. By 298 * requiring this to fail twice we avoid races with 299 * PFC clearing the ARMED bit and conditions where we 300 * run the check_tx_hang logic with a transmit completion 301 * pending but without time to complete it yet. 302 */ 303 if ((tx_ring->tx_stats.tx_done_old == tx_ring->tx_stats.packets) && 304 tx_pending) { 305 /* make sure it is true for two checks in a row */ 306 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED, 307 &tx_ring->state); 308 } else { 309 /* update completed stats and disarm the hang check */ 310 tx_ring->tx_stats.tx_done_old = tx_ring->tx_stats.packets; 311 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state); 312 } 313 314 return ret; 315 } 316 317 /** 318 * i40e_clean_tx_irq - Reclaim resources after transmit completes 319 * @tx_ring: tx ring to clean 320 * @budget: how many cleans we're allowed 321 * 322 * Returns true if there's any budget left (e.g. the clean is finished) 323 **/ 324 static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget) 325 { 326 u16 i = tx_ring->next_to_clean; 327 struct i40e_tx_buffer *tx_buf; 328 struct i40e_tx_desc *tx_desc; 329 unsigned int total_packets = 0; 330 unsigned int total_bytes = 0; 331 332 tx_buf = &tx_ring->tx_bi[i]; 333 tx_desc = I40E_TX_DESC(tx_ring, i); 334 335 for (; budget; budget--) { 336 struct i40e_tx_desc *eop_desc; 337 338 eop_desc = tx_buf->next_to_watch; 339 340 /* if next_to_watch is not set then there is no work pending */ 341 if (!eop_desc) 342 break; 343 344 /* if the descriptor isn't done, no work yet to do */ 345 if (!(eop_desc->cmd_type_offset_bsz & 346 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 347 break; 348 349 /* count the packet as being completed */ 350 tx_ring->tx_stats.completed++; 351 tx_buf->next_to_watch = NULL; 352 tx_buf->time_stamp = 0; 353 354 /* set memory barrier before eop_desc is verified */ 355 rmb(); 356 357 do { 358 i40e_unmap_tx_resource(tx_ring, tx_buf); 359 360 /* clear dtype status */ 361 tx_desc->cmd_type_offset_bsz &= 362 ~cpu_to_le64(I40E_TXD_QW1_DTYPE_MASK); 363 364 if (likely(tx_desc == eop_desc)) { 365 eop_desc = NULL; 366 367 dev_kfree_skb_any(tx_buf->skb); 368 tx_buf->skb = NULL; 369 370 total_bytes += tx_buf->bytecount; 371 total_packets += tx_buf->gso_segs; 372 } 373 374 tx_buf++; 375 tx_desc++; 376 i++; 377 if (unlikely(i == tx_ring->count)) { 378 i = 0; 379 tx_buf = tx_ring->tx_bi; 380 tx_desc = I40E_TX_DESC(tx_ring, 0); 381 } 382 } while (eop_desc); 383 } 384 385 tx_ring->next_to_clean = i; 386 tx_ring->tx_stats.bytes += total_bytes; 387 tx_ring->tx_stats.packets += total_packets; 388 tx_ring->q_vector->tx.total_bytes += total_bytes; 389 tx_ring->q_vector->tx.total_packets += total_packets; 390 if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) { 391 /* schedule immediate reset if we believe we hung */ 392 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n" 393 " VSI <%d>\n" 394 " Tx Queue <%d>\n" 395 " next_to_use <%x>\n" 396 " next_to_clean <%x>\n", 397 tx_ring->vsi->seid, 398 tx_ring->queue_index, 399 tx_ring->next_to_use, i); 400 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n" 401 " time_stamp <%lx>\n" 402 " jiffies <%lx>\n", 403 tx_ring->tx_bi[i].time_stamp, jiffies); 404 405 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 406 407 dev_info(tx_ring->dev, 408 "tx hang detected on queue %d, resetting adapter\n", 409 tx_ring->queue_index); 410 411 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev); 412 413 /* the adapter is about to reset, no point in enabling stuff */ 414 return true; 415 } 416 417 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 418 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 419 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 420 /* Make sure that anybody stopping the queue after this 421 * sees the new next_to_clean. 422 */ 423 smp_mb(); 424 if (__netif_subqueue_stopped(tx_ring->netdev, 425 tx_ring->queue_index) && 426 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) { 427 netif_wake_subqueue(tx_ring->netdev, 428 tx_ring->queue_index); 429 ++tx_ring->tx_stats.restart_queue; 430 } 431 } 432 433 return budget > 0; 434 } 435 436 /** 437 * i40e_set_new_dynamic_itr - Find new ITR level 438 * @rc: structure containing ring performance data 439 * 440 * Stores a new ITR value based on packets and byte counts during 441 * the last interrupt. The advantage of per interrupt computation 442 * is faster updates and more accurate ITR for the current traffic 443 * pattern. Constants in this function were computed based on 444 * theoretical maximum wire speed and thresholds were set based on 445 * testing data as well as attempting to minimize response time 446 * while increasing bulk throughput. 447 **/ 448 static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc) 449 { 450 enum i40e_latency_range new_latency_range = rc->latency_range; 451 u32 new_itr = rc->itr; 452 int bytes_per_int; 453 454 if (rc->total_packets == 0 || !rc->itr) 455 return; 456 457 /* simple throttlerate management 458 * 0-10MB/s lowest (100000 ints/s) 459 * 10-20MB/s low (20000 ints/s) 460 * 20-1249MB/s bulk (8000 ints/s) 461 */ 462 bytes_per_int = rc->total_bytes / rc->itr; 463 switch (rc->itr) { 464 case I40E_LOWEST_LATENCY: 465 if (bytes_per_int > 10) 466 new_latency_range = I40E_LOW_LATENCY; 467 break; 468 case I40E_LOW_LATENCY: 469 if (bytes_per_int > 20) 470 new_latency_range = I40E_BULK_LATENCY; 471 else if (bytes_per_int <= 10) 472 new_latency_range = I40E_LOWEST_LATENCY; 473 break; 474 case I40E_BULK_LATENCY: 475 if (bytes_per_int <= 20) 476 rc->latency_range = I40E_LOW_LATENCY; 477 break; 478 } 479 480 switch (new_latency_range) { 481 case I40E_LOWEST_LATENCY: 482 new_itr = I40E_ITR_100K; 483 break; 484 case I40E_LOW_LATENCY: 485 new_itr = I40E_ITR_20K; 486 break; 487 case I40E_BULK_LATENCY: 488 new_itr = I40E_ITR_8K; 489 break; 490 default: 491 break; 492 } 493 494 if (new_itr != rc->itr) { 495 /* do an exponential smoothing */ 496 new_itr = (10 * new_itr * rc->itr) / 497 ((9 * new_itr) + rc->itr); 498 rc->itr = new_itr & I40E_MAX_ITR; 499 } 500 501 rc->total_bytes = 0; 502 rc->total_packets = 0; 503 } 504 505 /** 506 * i40e_update_dynamic_itr - Adjust ITR based on bytes per int 507 * @q_vector: the vector to adjust 508 **/ 509 static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector) 510 { 511 u16 vector = q_vector->vsi->base_vector + q_vector->v_idx; 512 struct i40e_hw *hw = &q_vector->vsi->back->hw; 513 u32 reg_addr; 514 u16 old_itr; 515 516 reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1); 517 old_itr = q_vector->rx.itr; 518 i40e_set_new_dynamic_itr(&q_vector->rx); 519 if (old_itr != q_vector->rx.itr) 520 wr32(hw, reg_addr, q_vector->rx.itr); 521 522 reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1); 523 old_itr = q_vector->tx.itr; 524 i40e_set_new_dynamic_itr(&q_vector->tx); 525 if (old_itr != q_vector->tx.itr) 526 wr32(hw, reg_addr, q_vector->tx.itr); 527 528 i40e_flush(hw); 529 } 530 531 /** 532 * i40e_clean_programming_status - clean the programming status descriptor 533 * @rx_ring: the rx ring that has this descriptor 534 * @rx_desc: the rx descriptor written back by HW 535 * 536 * Flow director should handle FD_FILTER_STATUS to check its filter programming 537 * status being successful or not and take actions accordingly. FCoE should 538 * handle its context/filter programming/invalidation status and take actions. 539 * 540 **/ 541 static void i40e_clean_programming_status(struct i40e_ring *rx_ring, 542 union i40e_rx_desc *rx_desc) 543 { 544 u64 qw; 545 u8 id; 546 547 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 548 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> 549 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT; 550 551 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) 552 i40e_fd_handle_status(rx_ring, qw, id); 553 } 554 555 /** 556 * i40e_setup_tx_descriptors - Allocate the Tx descriptors 557 * @tx_ring: the tx ring to set up 558 * 559 * Return 0 on success, negative on error 560 **/ 561 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring) 562 { 563 struct device *dev = tx_ring->dev; 564 int bi_size; 565 566 if (!dev) 567 return -ENOMEM; 568 569 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 570 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL); 571 if (!tx_ring->tx_bi) 572 goto err; 573 574 /* round up to nearest 4K */ 575 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc); 576 tx_ring->size = ALIGN(tx_ring->size, 4096); 577 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 578 &tx_ring->dma, GFP_KERNEL); 579 if (!tx_ring->desc) { 580 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 581 tx_ring->size); 582 goto err; 583 } 584 585 tx_ring->next_to_use = 0; 586 tx_ring->next_to_clean = 0; 587 return 0; 588 589 err: 590 kfree(tx_ring->tx_bi); 591 tx_ring->tx_bi = NULL; 592 return -ENOMEM; 593 } 594 595 /** 596 * i40e_clean_rx_ring - Free Rx buffers 597 * @rx_ring: ring to be cleaned 598 **/ 599 void i40e_clean_rx_ring(struct i40e_ring *rx_ring) 600 { 601 struct device *dev = rx_ring->dev; 602 struct i40e_rx_buffer *rx_bi; 603 unsigned long bi_size; 604 u16 i; 605 606 /* ring already cleared, nothing to do */ 607 if (!rx_ring->rx_bi) 608 return; 609 610 /* Free all the Rx ring sk_buffs */ 611 for (i = 0; i < rx_ring->count; i++) { 612 rx_bi = &rx_ring->rx_bi[i]; 613 if (rx_bi->dma) { 614 dma_unmap_single(dev, 615 rx_bi->dma, 616 rx_ring->rx_buf_len, 617 DMA_FROM_DEVICE); 618 rx_bi->dma = 0; 619 } 620 if (rx_bi->skb) { 621 dev_kfree_skb(rx_bi->skb); 622 rx_bi->skb = NULL; 623 } 624 if (rx_bi->page) { 625 if (rx_bi->page_dma) { 626 dma_unmap_page(dev, 627 rx_bi->page_dma, 628 PAGE_SIZE / 2, 629 DMA_FROM_DEVICE); 630 rx_bi->page_dma = 0; 631 } 632 __free_page(rx_bi->page); 633 rx_bi->page = NULL; 634 rx_bi->page_offset = 0; 635 } 636 } 637 638 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 639 memset(rx_ring->rx_bi, 0, bi_size); 640 641 /* Zero out the descriptor ring */ 642 memset(rx_ring->desc, 0, rx_ring->size); 643 644 rx_ring->next_to_clean = 0; 645 rx_ring->next_to_use = 0; 646 } 647 648 /** 649 * i40e_free_rx_resources - Free Rx resources 650 * @rx_ring: ring to clean the resources from 651 * 652 * Free all receive software resources 653 **/ 654 void i40e_free_rx_resources(struct i40e_ring *rx_ring) 655 { 656 i40e_clean_rx_ring(rx_ring); 657 kfree(rx_ring->rx_bi); 658 rx_ring->rx_bi = NULL; 659 660 if (rx_ring->desc) { 661 dma_free_coherent(rx_ring->dev, rx_ring->size, 662 rx_ring->desc, rx_ring->dma); 663 rx_ring->desc = NULL; 664 } 665 } 666 667 /** 668 * i40e_setup_rx_descriptors - Allocate Rx descriptors 669 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 670 * 671 * Returns 0 on success, negative on failure 672 **/ 673 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) 674 { 675 struct device *dev = rx_ring->dev; 676 int bi_size; 677 678 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 679 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL); 680 if (!rx_ring->rx_bi) 681 goto err; 682 683 /* Round up to nearest 4K */ 684 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring) 685 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc) 686 : rx_ring->count * sizeof(union i40e_32byte_rx_desc); 687 rx_ring->size = ALIGN(rx_ring->size, 4096); 688 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 689 &rx_ring->dma, GFP_KERNEL); 690 691 if (!rx_ring->desc) { 692 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 693 rx_ring->size); 694 goto err; 695 } 696 697 rx_ring->next_to_clean = 0; 698 rx_ring->next_to_use = 0; 699 700 return 0; 701 err: 702 kfree(rx_ring->rx_bi); 703 rx_ring->rx_bi = NULL; 704 return -ENOMEM; 705 } 706 707 /** 708 * i40e_release_rx_desc - Store the new tail and head values 709 * @rx_ring: ring to bump 710 * @val: new head index 711 **/ 712 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val) 713 { 714 rx_ring->next_to_use = val; 715 /* Force memory writes to complete before letting h/w 716 * know there are new descriptors to fetch. (Only 717 * applicable for weak-ordered memory model archs, 718 * such as IA-64). 719 */ 720 wmb(); 721 writel(val, rx_ring->tail); 722 } 723 724 /** 725 * i40e_alloc_rx_buffers - Replace used receive buffers; packet split 726 * @rx_ring: ring to place buffers on 727 * @cleaned_count: number of buffers to replace 728 **/ 729 void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count) 730 { 731 u16 i = rx_ring->next_to_use; 732 union i40e_rx_desc *rx_desc; 733 struct i40e_rx_buffer *bi; 734 struct sk_buff *skb; 735 736 /* do nothing if no valid netdev defined */ 737 if (!rx_ring->netdev || !cleaned_count) 738 return; 739 740 while (cleaned_count--) { 741 rx_desc = I40E_RX_DESC(rx_ring, i); 742 bi = &rx_ring->rx_bi[i]; 743 skb = bi->skb; 744 745 if (!skb) { 746 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 747 rx_ring->rx_buf_len); 748 if (!skb) { 749 rx_ring->rx_stats.alloc_rx_buff_failed++; 750 goto no_buffers; 751 } 752 /* initialize queue mapping */ 753 skb_record_rx_queue(skb, rx_ring->queue_index); 754 bi->skb = skb; 755 } 756 757 if (!bi->dma) { 758 bi->dma = dma_map_single(rx_ring->dev, 759 skb->data, 760 rx_ring->rx_buf_len, 761 DMA_FROM_DEVICE); 762 if (dma_mapping_error(rx_ring->dev, bi->dma)) { 763 rx_ring->rx_stats.alloc_rx_buff_failed++; 764 bi->dma = 0; 765 goto no_buffers; 766 } 767 } 768 769 if (ring_is_ps_enabled(rx_ring)) { 770 if (!bi->page) { 771 bi->page = alloc_page(GFP_ATOMIC); 772 if (!bi->page) { 773 rx_ring->rx_stats.alloc_rx_page_failed++; 774 goto no_buffers; 775 } 776 } 777 778 if (!bi->page_dma) { 779 /* use a half page if we're re-using */ 780 bi->page_offset ^= PAGE_SIZE / 2; 781 bi->page_dma = dma_map_page(rx_ring->dev, 782 bi->page, 783 bi->page_offset, 784 PAGE_SIZE / 2, 785 DMA_FROM_DEVICE); 786 if (dma_mapping_error(rx_ring->dev, 787 bi->page_dma)) { 788 rx_ring->rx_stats.alloc_rx_page_failed++; 789 bi->page_dma = 0; 790 goto no_buffers; 791 } 792 } 793 794 /* Refresh the desc even if buffer_addrs didn't change 795 * because each write-back erases this info. 796 */ 797 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); 798 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); 799 } else { 800 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); 801 rx_desc->read.hdr_addr = 0; 802 } 803 i++; 804 if (i == rx_ring->count) 805 i = 0; 806 } 807 808 no_buffers: 809 if (rx_ring->next_to_use != i) 810 i40e_release_rx_desc(rx_ring, i); 811 } 812 813 /** 814 * i40e_receive_skb - Send a completed packet up the stack 815 * @rx_ring: rx ring in play 816 * @skb: packet to send up 817 * @vlan_tag: vlan tag for packet 818 **/ 819 static void i40e_receive_skb(struct i40e_ring *rx_ring, 820 struct sk_buff *skb, u16 vlan_tag) 821 { 822 struct i40e_q_vector *q_vector = rx_ring->q_vector; 823 struct i40e_vsi *vsi = rx_ring->vsi; 824 u64 flags = vsi->back->flags; 825 826 if (vlan_tag & VLAN_VID_MASK) 827 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); 828 829 if (flags & I40E_FLAG_IN_NETPOLL) 830 netif_rx(skb); 831 else 832 napi_gro_receive(&q_vector->napi, skb); 833 } 834 835 /** 836 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum 837 * @vsi: the VSI we care about 838 * @skb: skb currently being received and modified 839 * @rx_status: status value of last descriptor in packet 840 * @rx_error: error value of last descriptor in packet 841 **/ 842 static inline void i40e_rx_checksum(struct i40e_vsi *vsi, 843 struct sk_buff *skb, 844 u32 rx_status, 845 u32 rx_error) 846 { 847 skb->ip_summed = CHECKSUM_NONE; 848 849 /* Rx csum enabled and ip headers found? */ 850 if (!(vsi->netdev->features & NETIF_F_RXCSUM && 851 rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT))) 852 return; 853 854 /* IP or L4 checksum error */ 855 if (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | 856 (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { 857 vsi->back->hw_csum_rx_error++; 858 return; 859 } 860 861 skb->ip_summed = CHECKSUM_UNNECESSARY; 862 } 863 864 /** 865 * i40e_rx_hash - returns the hash value from the Rx descriptor 866 * @ring: descriptor ring 867 * @rx_desc: specific descriptor 868 **/ 869 static inline u32 i40e_rx_hash(struct i40e_ring *ring, 870 union i40e_rx_desc *rx_desc) 871 { 872 if (ring->netdev->features & NETIF_F_RXHASH) { 873 if ((le64_to_cpu(rx_desc->wb.qword1.status_error_len) >> 874 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT) & 875 I40E_RX_DESC_FLTSTAT_RSS_HASH) 876 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss); 877 } 878 return 0; 879 } 880 881 /** 882 * i40e_clean_rx_irq - Reclaim resources after receive completes 883 * @rx_ring: rx ring to clean 884 * @budget: how many cleans we're allowed 885 * 886 * Returns true if there's any budget left (e.g. the clean is finished) 887 **/ 888 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) 889 { 890 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 891 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo; 892 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); 893 const int current_node = numa_node_id(); 894 struct i40e_vsi *vsi = rx_ring->vsi; 895 u16 i = rx_ring->next_to_clean; 896 union i40e_rx_desc *rx_desc; 897 u32 rx_error, rx_status; 898 u64 qword; 899 900 rx_desc = I40E_RX_DESC(rx_ring, i); 901 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 902 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) 903 >> I40E_RXD_QW1_STATUS_SHIFT; 904 905 while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) { 906 union i40e_rx_desc *next_rxd; 907 struct i40e_rx_buffer *rx_bi; 908 struct sk_buff *skb; 909 u16 vlan_tag; 910 if (i40e_rx_is_programming_status(qword)) { 911 i40e_clean_programming_status(rx_ring, rx_desc); 912 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd); 913 goto next_desc; 914 } 915 rx_bi = &rx_ring->rx_bi[i]; 916 skb = rx_bi->skb; 917 prefetch(skb->data); 918 919 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) 920 >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; 921 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) 922 >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; 923 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) 924 >> I40E_RXD_QW1_LENGTH_SPH_SHIFT; 925 926 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) 927 >> I40E_RXD_QW1_ERROR_SHIFT; 928 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT); 929 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT); 930 931 rx_bi->skb = NULL; 932 933 /* This memory barrier is needed to keep us from reading 934 * any other fields out of the rx_desc until we know the 935 * STATUS_DD bit is set 936 */ 937 rmb(); 938 939 /* Get the header and possibly the whole packet 940 * If this is an skb from previous receive dma will be 0 941 */ 942 if (rx_bi->dma) { 943 u16 len; 944 945 if (rx_hbo) 946 len = I40E_RX_HDR_SIZE; 947 else if (rx_sph) 948 len = rx_header_len; 949 else if (rx_packet_len) 950 len = rx_packet_len; /* 1buf/no split found */ 951 else 952 len = rx_header_len; /* split always mode */ 953 954 skb_put(skb, len); 955 dma_unmap_single(rx_ring->dev, 956 rx_bi->dma, 957 rx_ring->rx_buf_len, 958 DMA_FROM_DEVICE); 959 rx_bi->dma = 0; 960 } 961 962 /* Get the rest of the data if this was a header split */ 963 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) { 964 965 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 966 rx_bi->page, 967 rx_bi->page_offset, 968 rx_packet_len); 969 970 skb->len += rx_packet_len; 971 skb->data_len += rx_packet_len; 972 skb->truesize += rx_packet_len; 973 974 if ((page_count(rx_bi->page) == 1) && 975 (page_to_nid(rx_bi->page) == current_node)) 976 get_page(rx_bi->page); 977 else 978 rx_bi->page = NULL; 979 980 dma_unmap_page(rx_ring->dev, 981 rx_bi->page_dma, 982 PAGE_SIZE / 2, 983 DMA_FROM_DEVICE); 984 rx_bi->page_dma = 0; 985 } 986 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd); 987 988 if (unlikely( 989 !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) { 990 struct i40e_rx_buffer *next_buffer; 991 992 next_buffer = &rx_ring->rx_bi[i]; 993 994 if (ring_is_ps_enabled(rx_ring)) { 995 rx_bi->skb = next_buffer->skb; 996 rx_bi->dma = next_buffer->dma; 997 next_buffer->skb = skb; 998 next_buffer->dma = 0; 999 } 1000 rx_ring->rx_stats.non_eop_descs++; 1001 goto next_desc; 1002 } 1003 1004 /* ERR_MASK will only have valid bits if EOP set */ 1005 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { 1006 dev_kfree_skb_any(skb); 1007 goto next_desc; 1008 } 1009 1010 skb->rxhash = i40e_rx_hash(rx_ring, rx_desc); 1011 i40e_rx_checksum(vsi, skb, rx_status, rx_error); 1012 1013 /* probably a little skewed due to removing CRC */ 1014 total_rx_bytes += skb->len; 1015 total_rx_packets++; 1016 1017 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1018 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT) 1019 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) 1020 : 0; 1021 i40e_receive_skb(rx_ring, skb, vlan_tag); 1022 1023 rx_ring->netdev->last_rx = jiffies; 1024 budget--; 1025 next_desc: 1026 rx_desc->wb.qword1.status_error_len = 0; 1027 if (!budget) 1028 break; 1029 1030 cleaned_count++; 1031 /* return some buffers to hardware, one at a time is too slow */ 1032 if (cleaned_count >= I40E_RX_BUFFER_WRITE) { 1033 i40e_alloc_rx_buffers(rx_ring, cleaned_count); 1034 cleaned_count = 0; 1035 } 1036 1037 /* use prefetched values */ 1038 rx_desc = next_rxd; 1039 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1040 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) 1041 >> I40E_RXD_QW1_STATUS_SHIFT; 1042 } 1043 1044 rx_ring->next_to_clean = i; 1045 rx_ring->rx_stats.packets += total_rx_packets; 1046 rx_ring->rx_stats.bytes += total_rx_bytes; 1047 rx_ring->q_vector->rx.total_packets += total_rx_packets; 1048 rx_ring->q_vector->rx.total_bytes += total_rx_bytes; 1049 1050 if (cleaned_count) 1051 i40e_alloc_rx_buffers(rx_ring, cleaned_count); 1052 1053 return budget > 0; 1054 } 1055 1056 /** 1057 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine 1058 * @napi: napi struct with our devices info in it 1059 * @budget: amount of work driver is allowed to do this pass, in packets 1060 * 1061 * This function will clean all queues associated with a q_vector. 1062 * 1063 * Returns the amount of work done 1064 **/ 1065 int i40e_napi_poll(struct napi_struct *napi, int budget) 1066 { 1067 struct i40e_q_vector *q_vector = 1068 container_of(napi, struct i40e_q_vector, napi); 1069 struct i40e_vsi *vsi = q_vector->vsi; 1070 bool clean_complete = true; 1071 int budget_per_ring; 1072 int i; 1073 1074 if (test_bit(__I40E_DOWN, &vsi->state)) { 1075 napi_complete(napi); 1076 return 0; 1077 } 1078 1079 /* We attempt to distribute budget to each Rx queue fairly, but don't 1080 * allow the budget to go below 1 because that would exit polling early. 1081 * Since the actual Tx work is minimal, we can give the Tx a larger 1082 * budget and be more aggressive about cleaning up the Tx descriptors. 1083 */ 1084 budget_per_ring = max(budget/q_vector->num_ringpairs, 1); 1085 for (i = 0; i < q_vector->num_ringpairs; i++) { 1086 clean_complete &= i40e_clean_tx_irq(q_vector->tx.ring[i], 1087 vsi->work_limit); 1088 clean_complete &= i40e_clean_rx_irq(q_vector->rx.ring[i], 1089 budget_per_ring); 1090 } 1091 1092 /* If work not completed, return budget and polling will return */ 1093 if (!clean_complete) 1094 return budget; 1095 1096 /* Work is done so exit the polling mode and re-enable the interrupt */ 1097 napi_complete(napi); 1098 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) || 1099 ITR_IS_DYNAMIC(vsi->tx_itr_setting)) 1100 i40e_update_dynamic_itr(q_vector); 1101 1102 if (!test_bit(__I40E_DOWN, &vsi->state)) { 1103 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 1104 i40e_irq_dynamic_enable(vsi, 1105 q_vector->v_idx + vsi->base_vector); 1106 } else { 1107 struct i40e_hw *hw = &vsi->back->hw; 1108 /* We re-enable the queue 0 cause, but 1109 * don't worry about dynamic_enable 1110 * because we left it on for the other 1111 * possible interrupts during napi 1112 */ 1113 u32 qval = rd32(hw, I40E_QINT_RQCTL(0)); 1114 qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; 1115 wr32(hw, I40E_QINT_RQCTL(0), qval); 1116 1117 qval = rd32(hw, I40E_QINT_TQCTL(0)); 1118 qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; 1119 wr32(hw, I40E_QINT_TQCTL(0), qval); 1120 i40e_flush(hw); 1121 } 1122 } 1123 1124 return 0; 1125 } 1126 1127 /** 1128 * i40e_atr - Add a Flow Director ATR filter 1129 * @tx_ring: ring to add programming descriptor to 1130 * @skb: send buffer 1131 * @flags: send flags 1132 * @protocol: wire protocol 1133 **/ 1134 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb, 1135 u32 flags, __be16 protocol) 1136 { 1137 struct i40e_filter_program_desc *fdir_desc; 1138 struct i40e_pf *pf = tx_ring->vsi->back; 1139 union { 1140 unsigned char *network; 1141 struct iphdr *ipv4; 1142 struct ipv6hdr *ipv6; 1143 } hdr; 1144 struct tcphdr *th; 1145 unsigned int hlen; 1146 u32 flex_ptype, dtype_cmd; 1147 1148 /* make sure ATR is enabled */ 1149 if (!(pf->flags & I40E_FLAG_FDIR_ATR_ENABLED)) 1150 return; 1151 1152 /* if sampling is disabled do nothing */ 1153 if (!tx_ring->atr_sample_rate) 1154 return; 1155 1156 tx_ring->atr_count++; 1157 1158 /* snag network header to get L4 type and address */ 1159 hdr.network = skb_network_header(skb); 1160 1161 /* Currently only IPv4/IPv6 with TCP is supported */ 1162 if (protocol == htons(ETH_P_IP)) { 1163 if (hdr.ipv4->protocol != IPPROTO_TCP) 1164 return; 1165 1166 /* access ihl as a u8 to avoid unaligned access on ia64 */ 1167 hlen = (hdr.network[0] & 0x0F) << 2; 1168 } else if (protocol == htons(ETH_P_IPV6)) { 1169 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1170 return; 1171 1172 hlen = sizeof(struct ipv6hdr); 1173 } else { 1174 return; 1175 } 1176 1177 th = (struct tcphdr *)(hdr.network + hlen); 1178 1179 /* sample on all syn/fin packets or once every atr sample rate */ 1180 if (!th->fin && !th->syn && (tx_ring->atr_count < tx_ring->atr_sample_rate)) 1181 return; 1182 1183 tx_ring->atr_count = 0; 1184 1185 /* grab the next descriptor */ 1186 fdir_desc = I40E_TX_FDIRDESC(tx_ring, tx_ring->next_to_use); 1187 tx_ring->next_to_use++; 1188 if (tx_ring->next_to_use == tx_ring->count) 1189 tx_ring->next_to_use = 0; 1190 1191 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 1192 I40E_TXD_FLTR_QW0_QINDEX_MASK; 1193 flex_ptype |= (protocol == htons(ETH_P_IP)) ? 1194 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << 1195 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : 1196 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << 1197 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 1198 1199 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; 1200 1201 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG; 1202 1203 dtype_cmd |= th->fin ? 1204 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 1205 I40E_TXD_FLTR_QW1_PCMD_SHIFT) : 1206 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 1207 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 1208 1209 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << 1210 I40E_TXD_FLTR_QW1_DEST_SHIFT; 1211 1212 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << 1213 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; 1214 1215 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype); 1216 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd); 1217 } 1218 1219 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) 1220 /** 1221 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW 1222 * @skb: send buffer 1223 * @tx_ring: ring to send buffer on 1224 * @flags: the tx flags to be set 1225 * 1226 * Checks the skb and set up correspondingly several generic transmit flags 1227 * related to VLAN tagging for the HW, such as VLAN, DCB, etc. 1228 * 1229 * Returns error code indicate the frame should be dropped upon error and the 1230 * otherwise returns 0 to indicate the flags has been set properly. 1231 **/ 1232 static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb, 1233 struct i40e_ring *tx_ring, 1234 u32 *flags) 1235 { 1236 __be16 protocol = skb->protocol; 1237 u32 tx_flags = 0; 1238 1239 /* if we have a HW VLAN tag being added, default to the HW one */ 1240 if (vlan_tx_tag_present(skb)) { 1241 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT; 1242 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 1243 /* else if it is a SW VLAN, check the next protocol and store the tag */ 1244 } else if (protocol == __constant_htons(ETH_P_8021Q)) { 1245 struct vlan_hdr *vhdr, _vhdr; 1246 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 1247 if (!vhdr) 1248 return -EINVAL; 1249 1250 protocol = vhdr->h_vlan_encapsulated_proto; 1251 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT; 1252 tx_flags |= I40E_TX_FLAGS_SW_VLAN; 1253 } 1254 1255 /* Insert 802.1p priority into VLAN header */ 1256 if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) && 1257 ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) || 1258 (skb->priority != TC_PRIO_CONTROL))) { 1259 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK; 1260 tx_flags |= (skb->priority & 0x7) << 1261 I40E_TX_FLAGS_VLAN_PRIO_SHIFT; 1262 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) { 1263 struct vlan_ethhdr *vhdr; 1264 if (skb_header_cloned(skb) && 1265 pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) 1266 return -ENOMEM; 1267 vhdr = (struct vlan_ethhdr *)skb->data; 1268 vhdr->h_vlan_TCI = htons(tx_flags >> 1269 I40E_TX_FLAGS_VLAN_SHIFT); 1270 } else { 1271 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 1272 } 1273 } 1274 *flags = tx_flags; 1275 return 0; 1276 } 1277 1278 /** 1279 * i40e_tx_csum - is checksum offload requested 1280 * @tx_ring: ptr to the ring to send 1281 * @skb: ptr to the skb we're sending 1282 * @tx_flags: the collected send information 1283 * @protocol: the send protocol 1284 * 1285 * Returns true if checksum offload is requested 1286 **/ 1287 static bool i40e_tx_csum(struct i40e_ring *tx_ring, struct sk_buff *skb, 1288 u32 tx_flags, __be16 protocol) 1289 { 1290 if ((skb->ip_summed != CHECKSUM_PARTIAL) && 1291 !(tx_flags & I40E_TX_FLAGS_TXSW)) { 1292 if (!(tx_flags & I40E_TX_FLAGS_HW_VLAN)) 1293 return false; 1294 } 1295 1296 return skb->ip_summed == CHECKSUM_PARTIAL; 1297 } 1298 1299 /** 1300 * i40e_tso - set up the tso context descriptor 1301 * @tx_ring: ptr to the ring to send 1302 * @skb: ptr to the skb we're sending 1303 * @tx_flags: the collected send information 1304 * @protocol: the send protocol 1305 * @hdr_len: ptr to the size of the packet header 1306 * @cd_tunneling: ptr to context descriptor bits 1307 * 1308 * Returns 0 if no TSO can happen, 1 if tso is going, or error 1309 **/ 1310 static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb, 1311 u32 tx_flags, __be16 protocol, u8 *hdr_len, 1312 u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling) 1313 { 1314 u32 cd_cmd, cd_tso_len, cd_mss; 1315 struct tcphdr *tcph; 1316 struct iphdr *iph; 1317 u32 l4len; 1318 int err; 1319 struct ipv6hdr *ipv6h; 1320 1321 if (!skb_is_gso(skb)) 1322 return 0; 1323 1324 if (skb_header_cloned(skb)) { 1325 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 1326 if (err) 1327 return err; 1328 } 1329 1330 if (protocol == __constant_htons(ETH_P_IP)) { 1331 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb); 1332 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb); 1333 iph->tot_len = 0; 1334 iph->check = 0; 1335 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 1336 0, IPPROTO_TCP, 0); 1337 } else if (skb_is_gso_v6(skb)) { 1338 1339 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb) 1340 : ipv6_hdr(skb); 1341 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb); 1342 ipv6h->payload_len = 0; 1343 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr, 1344 0, IPPROTO_TCP, 0); 1345 } 1346 1347 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb); 1348 *hdr_len = (skb->encapsulation 1349 ? (skb_inner_transport_header(skb) - skb->data) 1350 : skb_transport_offset(skb)) + l4len; 1351 1352 /* find the field values */ 1353 cd_cmd = I40E_TX_CTX_DESC_TSO; 1354 cd_tso_len = skb->len - *hdr_len; 1355 cd_mss = skb_shinfo(skb)->gso_size; 1356 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) 1357 | ((u64)cd_tso_len 1358 << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) 1359 | ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT); 1360 return 1; 1361 } 1362 1363 /** 1364 * i40e_tx_enable_csum - Enable Tx checksum offloads 1365 * @skb: send buffer 1366 * @tx_flags: Tx flags currently set 1367 * @td_cmd: Tx descriptor command bits to set 1368 * @td_offset: Tx descriptor header offsets to set 1369 * @cd_tunneling: ptr to context desc bits 1370 **/ 1371 static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags, 1372 u32 *td_cmd, u32 *td_offset, 1373 struct i40e_ring *tx_ring, 1374 u32 *cd_tunneling) 1375 { 1376 struct ipv6hdr *this_ipv6_hdr; 1377 unsigned int this_tcp_hdrlen; 1378 struct iphdr *this_ip_hdr; 1379 u32 network_hdr_len; 1380 u8 l4_hdr = 0; 1381 1382 if (skb->encapsulation) { 1383 network_hdr_len = skb_inner_network_header_len(skb); 1384 this_ip_hdr = inner_ip_hdr(skb); 1385 this_ipv6_hdr = inner_ipv6_hdr(skb); 1386 this_tcp_hdrlen = inner_tcp_hdrlen(skb); 1387 1388 if (tx_flags & I40E_TX_FLAGS_IPV4) { 1389 1390 if (tx_flags & I40E_TX_FLAGS_TSO) { 1391 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4; 1392 ip_hdr(skb)->check = 0; 1393 } else { 1394 *cd_tunneling |= 1395 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 1396 } 1397 } else if (tx_flags & I40E_TX_FLAGS_IPV6) { 1398 if (tx_flags & I40E_TX_FLAGS_TSO) { 1399 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6; 1400 ip_hdr(skb)->check = 0; 1401 } else { 1402 *cd_tunneling |= 1403 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 1404 } 1405 } 1406 1407 /* Now set the ctx descriptor fields */ 1408 *cd_tunneling |= (skb_network_header_len(skb) >> 2) << 1409 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT | 1410 I40E_TXD_CTX_UDP_TUNNELING | 1411 ((skb_inner_network_offset(skb) - 1412 skb_transport_offset(skb)) >> 1) << 1413 I40E_TXD_CTX_QW0_NATLEN_SHIFT; 1414 1415 } else { 1416 network_hdr_len = skb_network_header_len(skb); 1417 this_ip_hdr = ip_hdr(skb); 1418 this_ipv6_hdr = ipv6_hdr(skb); 1419 this_tcp_hdrlen = tcp_hdrlen(skb); 1420 } 1421 1422 /* Enable IP checksum offloads */ 1423 if (tx_flags & I40E_TX_FLAGS_IPV4) { 1424 l4_hdr = this_ip_hdr->protocol; 1425 /* the stack computes the IP header already, the only time we 1426 * need the hardware to recompute it is in the case of TSO. 1427 */ 1428 if (tx_flags & I40E_TX_FLAGS_TSO) { 1429 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; 1430 this_ip_hdr->check = 0; 1431 } else { 1432 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; 1433 } 1434 /* Now set the td_offset for IP header length */ 1435 *td_offset = (network_hdr_len >> 2) << 1436 I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 1437 } else if (tx_flags & I40E_TX_FLAGS_IPV6) { 1438 l4_hdr = this_ipv6_hdr->nexthdr; 1439 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; 1440 /* Now set the td_offset for IP header length */ 1441 *td_offset = (network_hdr_len >> 2) << 1442 I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 1443 } 1444 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */ 1445 *td_offset |= (skb_network_offset(skb) >> 1) << 1446 I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 1447 1448 /* Enable L4 checksum offloads */ 1449 switch (l4_hdr) { 1450 case IPPROTO_TCP: 1451 /* enable checksum offloads */ 1452 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 1453 *td_offset |= (this_tcp_hdrlen >> 2) << 1454 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 1455 break; 1456 case IPPROTO_SCTP: 1457 /* enable SCTP checksum offload */ 1458 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; 1459 *td_offset |= (sizeof(struct sctphdr) >> 2) << 1460 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 1461 break; 1462 case IPPROTO_UDP: 1463 /* enable UDP checksum offload */ 1464 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; 1465 *td_offset |= (sizeof(struct udphdr) >> 2) << 1466 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 1467 break; 1468 default: 1469 break; 1470 } 1471 } 1472 1473 /** 1474 * i40e_create_tx_ctx Build the Tx context descriptor 1475 * @tx_ring: ring to create the descriptor on 1476 * @cd_type_cmd_tso_mss: Quad Word 1 1477 * @cd_tunneling: Quad Word 0 - bits 0-31 1478 * @cd_l2tag2: Quad Word 0 - bits 32-63 1479 **/ 1480 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring, 1481 const u64 cd_type_cmd_tso_mss, 1482 const u32 cd_tunneling, const u32 cd_l2tag2) 1483 { 1484 struct i40e_tx_context_desc *context_desc; 1485 1486 if (!cd_type_cmd_tso_mss && !cd_tunneling && !cd_l2tag2) 1487 return; 1488 1489 /* grab the next descriptor */ 1490 context_desc = I40E_TX_CTXTDESC(tx_ring, tx_ring->next_to_use); 1491 tx_ring->next_to_use++; 1492 if (tx_ring->next_to_use == tx_ring->count) 1493 tx_ring->next_to_use = 0; 1494 1495 /* cpu_to_le32 and assign to struct fields */ 1496 context_desc->tunneling_params = cpu_to_le32(cd_tunneling); 1497 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2); 1498 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss); 1499 } 1500 1501 /** 1502 * i40e_tx_map - Build the Tx descriptor 1503 * @tx_ring: ring to send buffer on 1504 * @skb: send buffer 1505 * @first: first buffer info buffer to use 1506 * @tx_flags: collected send information 1507 * @hdr_len: size of the packet header 1508 * @td_cmd: the command field in the descriptor 1509 * @td_offset: offset for checksum or crc 1510 **/ 1511 static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb, 1512 struct i40e_tx_buffer *first, u32 tx_flags, 1513 const u8 hdr_len, u32 td_cmd, u32 td_offset) 1514 { 1515 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1516 unsigned int data_len = skb->data_len; 1517 unsigned int size = skb_headlen(skb); 1518 struct device *dev = tx_ring->dev; 1519 u32 paylen = skb->len - hdr_len; 1520 u16 i = tx_ring->next_to_use; 1521 struct i40e_tx_buffer *tx_bi; 1522 struct i40e_tx_desc *tx_desc; 1523 u32 buf_offset = 0; 1524 u32 td_tag = 0; 1525 dma_addr_t dma; 1526 u16 gso_segs; 1527 1528 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); 1529 if (dma_mapping_error(dev, dma)) 1530 goto dma_error; 1531 1532 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) { 1533 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; 1534 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >> 1535 I40E_TX_FLAGS_VLAN_SHIFT; 1536 } 1537 1538 tx_desc = I40E_TX_DESC(tx_ring, i); 1539 for (;;) { 1540 while (size > I40E_MAX_DATA_PER_TXD) { 1541 tx_desc->buffer_addr = cpu_to_le64(dma + buf_offset); 1542 tx_desc->cmd_type_offset_bsz = 1543 build_ctob(td_cmd, td_offset, 1544 I40E_MAX_DATA_PER_TXD, td_tag); 1545 1546 buf_offset += I40E_MAX_DATA_PER_TXD; 1547 size -= I40E_MAX_DATA_PER_TXD; 1548 1549 tx_desc++; 1550 i++; 1551 if (i == tx_ring->count) { 1552 tx_desc = I40E_TX_DESC(tx_ring, 0); 1553 i = 0; 1554 } 1555 } 1556 1557 tx_bi = &tx_ring->tx_bi[i]; 1558 tx_bi->length = buf_offset + size; 1559 tx_bi->tx_flags = tx_flags; 1560 tx_bi->dma = dma; 1561 1562 tx_desc->buffer_addr = cpu_to_le64(dma + buf_offset); 1563 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, 1564 size, td_tag); 1565 1566 if (likely(!data_len)) 1567 break; 1568 1569 size = skb_frag_size(frag); 1570 data_len -= size; 1571 buf_offset = 0; 1572 tx_flags |= I40E_TX_FLAGS_MAPPED_AS_PAGE; 1573 1574 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); 1575 if (dma_mapping_error(dev, dma)) 1576 goto dma_error; 1577 1578 tx_desc++; 1579 i++; 1580 if (i == tx_ring->count) { 1581 tx_desc = I40E_TX_DESC(tx_ring, 0); 1582 i = 0; 1583 } 1584 1585 frag++; 1586 } 1587 1588 tx_desc->cmd_type_offset_bsz |= 1589 cpu_to_le64((u64)I40E_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT); 1590 1591 i++; 1592 if (i == tx_ring->count) 1593 i = 0; 1594 1595 tx_ring->next_to_use = i; 1596 1597 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO)) 1598 gso_segs = skb_shinfo(skb)->gso_segs; 1599 else 1600 gso_segs = 1; 1601 1602 /* multiply data chunks by size of headers */ 1603 tx_bi->bytecount = paylen + (gso_segs * hdr_len); 1604 tx_bi->gso_segs = gso_segs; 1605 tx_bi->skb = skb; 1606 1607 /* set the timestamp and next to watch values */ 1608 first->time_stamp = jiffies; 1609 first->next_to_watch = tx_desc; 1610 1611 /* Force memory writes to complete before letting h/w 1612 * know there are new descriptors to fetch. (Only 1613 * applicable for weak-ordered memory model archs, 1614 * such as IA-64). 1615 */ 1616 wmb(); 1617 1618 writel(i, tx_ring->tail); 1619 return; 1620 1621 dma_error: 1622 dev_info(dev, "TX DMA map failed\n"); 1623 1624 /* clear dma mappings for failed tx_bi map */ 1625 for (;;) { 1626 tx_bi = &tx_ring->tx_bi[i]; 1627 i40e_unmap_tx_resource(tx_ring, tx_bi); 1628 if (tx_bi == first) 1629 break; 1630 if (i == 0) 1631 i = tx_ring->count; 1632 i--; 1633 } 1634 1635 dev_kfree_skb_any(skb); 1636 1637 tx_ring->next_to_use = i; 1638 } 1639 1640 /** 1641 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions 1642 * @tx_ring: the ring to be checked 1643 * @size: the size buffer we want to assure is available 1644 * 1645 * Returns -EBUSY if a stop is needed, else 0 1646 **/ 1647 static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) 1648 { 1649 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 1650 smp_mb(); 1651 1652 /* Check again in a case another CPU has just made room available. */ 1653 if (likely(I40E_DESC_UNUSED(tx_ring) < size)) 1654 return -EBUSY; 1655 1656 /* A reprieve! - use start_queue because it doesn't call schedule */ 1657 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 1658 ++tx_ring->tx_stats.restart_queue; 1659 return 0; 1660 } 1661 1662 /** 1663 * i40e_maybe_stop_tx - 1st level check for tx stop conditions 1664 * @tx_ring: the ring to be checked 1665 * @size: the size buffer we want to assure is available 1666 * 1667 * Returns 0 if stop is not needed 1668 **/ 1669 static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) 1670 { 1671 if (likely(I40E_DESC_UNUSED(tx_ring) >= size)) 1672 return 0; 1673 return __i40e_maybe_stop_tx(tx_ring, size); 1674 } 1675 1676 /** 1677 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed 1678 * @skb: send buffer 1679 * @tx_ring: ring to send buffer on 1680 * 1681 * Returns number of data descriptors needed for this skb. Returns 0 to indicate 1682 * there is not enough descriptors available in this ring since we need at least 1683 * one descriptor. 1684 **/ 1685 static int i40e_xmit_descriptor_count(struct sk_buff *skb, 1686 struct i40e_ring *tx_ring) 1687 { 1688 #if PAGE_SIZE > I40E_MAX_DATA_PER_TXD 1689 unsigned int f; 1690 #endif 1691 int count = 0; 1692 1693 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, 1694 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, 1695 * + 2 desc gap to keep tail from touching head, 1696 * + 1 desc for context descriptor, 1697 * otherwise try next time 1698 */ 1699 #if PAGE_SIZE > I40E_MAX_DATA_PER_TXD 1700 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1701 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 1702 #else 1703 count += skb_shinfo(skb)->nr_frags; 1704 #endif 1705 count += TXD_USE_COUNT(skb_headlen(skb)); 1706 if (i40e_maybe_stop_tx(tx_ring, count + 3)) { 1707 tx_ring->tx_stats.tx_busy++; 1708 return 0; 1709 } 1710 return count; 1711 } 1712 1713 /** 1714 * i40e_xmit_frame_ring - Sends buffer on Tx ring 1715 * @skb: send buffer 1716 * @tx_ring: ring to send buffer on 1717 * 1718 * Returns NETDEV_TX_OK if sent, else an error code 1719 **/ 1720 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, 1721 struct i40e_ring *tx_ring) 1722 { 1723 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT; 1724 u32 cd_tunneling = 0, cd_l2tag2 = 0; 1725 struct i40e_tx_buffer *first; 1726 u32 td_offset = 0; 1727 u32 tx_flags = 0; 1728 __be16 protocol; 1729 u32 td_cmd = 0; 1730 u8 hdr_len = 0; 1731 int tso; 1732 if (0 == i40e_xmit_descriptor_count(skb, tx_ring)) 1733 return NETDEV_TX_BUSY; 1734 1735 /* prepare the xmit flags */ 1736 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags)) 1737 goto out_drop; 1738 1739 /* obtain protocol of skb */ 1740 protocol = skb->protocol; 1741 1742 /* record the location of the first descriptor for this packet */ 1743 first = &tx_ring->tx_bi[tx_ring->next_to_use]; 1744 1745 /* setup IPv4/IPv6 offloads */ 1746 if (protocol == __constant_htons(ETH_P_IP)) 1747 tx_flags |= I40E_TX_FLAGS_IPV4; 1748 else if (protocol == __constant_htons(ETH_P_IPV6)) 1749 tx_flags |= I40E_TX_FLAGS_IPV6; 1750 1751 tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len, 1752 &cd_type_cmd_tso_mss, &cd_tunneling); 1753 1754 if (tso < 0) 1755 goto out_drop; 1756 else if (tso) 1757 tx_flags |= I40E_TX_FLAGS_TSO; 1758 1759 skb_tx_timestamp(skb); 1760 1761 /* Always offload the checksum, since it's in the data descriptor */ 1762 if (i40e_tx_csum(tx_ring, skb, tx_flags, protocol)) 1763 tx_flags |= I40E_TX_FLAGS_CSUM; 1764 1765 /* always enable offload insertion */ 1766 td_cmd |= I40E_TX_DESC_CMD_ICRC; 1767 1768 if (tx_flags & I40E_TX_FLAGS_CSUM) 1769 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset, 1770 tx_ring, &cd_tunneling); 1771 1772 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss, 1773 cd_tunneling, cd_l2tag2); 1774 1775 /* Add Flow Director ATR if it's enabled. 1776 * 1777 * NOTE: this must always be directly before the data descriptor. 1778 */ 1779 i40e_atr(tx_ring, skb, tx_flags, protocol); 1780 1781 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len, 1782 td_cmd, td_offset); 1783 1784 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED); 1785 1786 return NETDEV_TX_OK; 1787 1788 out_drop: 1789 dev_kfree_skb_any(skb); 1790 return NETDEV_TX_OK; 1791 } 1792 1793 /** 1794 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer 1795 * @skb: send buffer 1796 * @netdev: network interface device structure 1797 * 1798 * Returns NETDEV_TX_OK if sent, else an error code 1799 **/ 1800 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1801 { 1802 struct i40e_netdev_priv *np = netdev_priv(netdev); 1803 struct i40e_vsi *vsi = np->vsi; 1804 struct i40e_ring *tx_ring = &vsi->tx_rings[skb->queue_mapping]; 1805 1806 /* hardware can't handle really short frames, hardware padding works 1807 * beyond this point 1808 */ 1809 if (unlikely(skb->len < I40E_MIN_TX_LEN)) { 1810 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len)) 1811 return NETDEV_TX_OK; 1812 skb->len = I40E_MIN_TX_LEN; 1813 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN); 1814 } 1815 1816 return i40e_xmit_frame_ring(skb, tx_ring); 1817 } 1818