1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2014 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/prefetch.h> 28 #include "i40e.h" 29 #include "i40e_prototype.h" 30 31 static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size, 32 u32 td_tag) 33 { 34 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA | 35 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | 36 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | 37 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | 38 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); 39 } 40 41 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) 42 /** 43 * i40e_program_fdir_filter - Program a Flow Director filter 44 * @fdir_data: Packet data that will be filter parameters 45 * @raw_packet: the pre-allocated packet buffer for FDir 46 * @pf: The pf pointer 47 * @add: True for add/update, False for remove 48 **/ 49 int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet, 50 struct i40e_pf *pf, bool add) 51 { 52 struct i40e_filter_program_desc *fdir_desc; 53 struct i40e_tx_buffer *tx_buf; 54 struct i40e_tx_desc *tx_desc; 55 struct i40e_ring *tx_ring; 56 unsigned int fpt, dcc; 57 struct i40e_vsi *vsi; 58 struct device *dev; 59 dma_addr_t dma; 60 u32 td_cmd = 0; 61 u16 i; 62 63 /* find existing FDIR VSI */ 64 vsi = NULL; 65 for (i = 0; i < pf->num_alloc_vsi; i++) 66 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) 67 vsi = pf->vsi[i]; 68 if (!vsi) 69 return -ENOENT; 70 71 tx_ring = vsi->tx_rings[0]; 72 dev = tx_ring->dev; 73 74 dma = dma_map_single(dev, raw_packet, 75 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE); 76 if (dma_mapping_error(dev, dma)) 77 goto dma_fail; 78 79 /* grab the next descriptor */ 80 i = tx_ring->next_to_use; 81 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i); 82 83 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0; 84 85 fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 86 I40E_TXD_FLTR_QW0_QINDEX_MASK; 87 88 fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) & 89 I40E_TXD_FLTR_QW0_FLEXOFF_MASK; 90 91 fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) & 92 I40E_TXD_FLTR_QW0_PCTYPE_MASK; 93 94 /* Use LAN VSI Id if not programmed by user */ 95 if (fdir_data->dest_vsi == 0) 96 fpt |= (pf->vsi[pf->lan_vsi]->id) << 97 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; 98 else 99 fpt |= ((u32)fdir_data->dest_vsi << 100 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) & 101 I40E_TXD_FLTR_QW0_DEST_VSI_MASK; 102 103 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt); 104 105 dcc = I40E_TX_DESC_DTYPE_FILTER_PROG; 106 107 if (add) 108 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 109 I40E_TXD_FLTR_QW1_PCMD_SHIFT; 110 else 111 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 112 I40E_TXD_FLTR_QW1_PCMD_SHIFT; 113 114 dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) & 115 I40E_TXD_FLTR_QW1_DEST_MASK; 116 117 dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) & 118 I40E_TXD_FLTR_QW1_FD_STATUS_MASK; 119 120 if (fdir_data->cnt_index != 0) { 121 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK; 122 dcc |= ((u32)fdir_data->cnt_index << 123 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 124 I40E_TXD_FLTR_QW1_CNTINDEX_MASK; 125 } 126 127 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc); 128 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id); 129 130 /* Now program a dummy descriptor */ 131 i = tx_ring->next_to_use; 132 tx_desc = I40E_TX_DESC(tx_ring, i); 133 tx_buf = &tx_ring->tx_bi[i]; 134 135 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0; 136 137 /* record length, and DMA address */ 138 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE); 139 dma_unmap_addr_set(tx_buf, dma, dma); 140 141 tx_desc->buffer_addr = cpu_to_le64(dma); 142 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY; 143 144 tx_desc->cmd_type_offset_bsz = 145 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0); 146 147 /* set the timestamp */ 148 tx_buf->time_stamp = jiffies; 149 150 /* Force memory writes to complete before letting h/w 151 * know there are new descriptors to fetch. (Only 152 * applicable for weak-ordered memory model archs, 153 * such as IA-64). 154 */ 155 wmb(); 156 157 /* Mark the data descriptor to be watched */ 158 tx_buf->next_to_watch = tx_desc; 159 160 writel(tx_ring->next_to_use, tx_ring->tail); 161 return 0; 162 163 dma_fail: 164 return -1; 165 } 166 167 #define IP_HEADER_OFFSET 14 168 #define I40E_UDPIP_DUMMY_PACKET_LEN 42 169 /** 170 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters 171 * @vsi: pointer to the targeted VSI 172 * @fd_data: the flow director data required for the FDir descriptor 173 * @raw_packet: the pre-allocated packet buffer for FDir 174 * @add: true adds a filter, false removes it 175 * 176 * Returns 0 if the filters were successfully added or removed 177 **/ 178 static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi, 179 struct i40e_fdir_filter *fd_data, 180 u8 *raw_packet, bool add) 181 { 182 struct i40e_pf *pf = vsi->back; 183 struct udphdr *udp; 184 struct iphdr *ip; 185 bool err = false; 186 int ret; 187 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 188 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0, 189 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 190 191 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN); 192 193 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 194 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET 195 + sizeof(struct iphdr)); 196 197 ip->daddr = fd_data->dst_ip[0]; 198 udp->dest = fd_data->dst_port; 199 ip->saddr = fd_data->src_ip[0]; 200 udp->source = fd_data->src_port; 201 202 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; 203 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 204 if (ret) { 205 dev_info(&pf->pdev->dev, 206 "Filter command send failed for PCTYPE %d (ret = %d)\n", 207 fd_data->pctype, ret); 208 err = true; 209 } else { 210 dev_info(&pf->pdev->dev, 211 "Filter OK for PCTYPE %d (ret = %d)\n", 212 fd_data->pctype, ret); 213 } 214 215 return err ? -EOPNOTSUPP : 0; 216 } 217 218 #define I40E_TCPIP_DUMMY_PACKET_LEN 54 219 /** 220 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters 221 * @vsi: pointer to the targeted VSI 222 * @fd_data: the flow director data required for the FDir descriptor 223 * @raw_packet: the pre-allocated packet buffer for FDir 224 * @add: true adds a filter, false removes it 225 * 226 * Returns 0 if the filters were successfully added or removed 227 **/ 228 static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi, 229 struct i40e_fdir_filter *fd_data, 230 u8 *raw_packet, bool add) 231 { 232 struct i40e_pf *pf = vsi->back; 233 struct tcphdr *tcp; 234 struct iphdr *ip; 235 bool err = false; 236 int ret; 237 /* Dummy packet */ 238 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 239 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0, 240 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11, 241 0x0, 0x72, 0, 0, 0, 0}; 242 243 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN); 244 245 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 246 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET 247 + sizeof(struct iphdr)); 248 249 ip->daddr = fd_data->dst_ip[0]; 250 tcp->dest = fd_data->dst_port; 251 ip->saddr = fd_data->src_ip[0]; 252 tcp->source = fd_data->src_port; 253 254 if (add) { 255 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) { 256 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n"); 257 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED; 258 } 259 } 260 261 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP; 262 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 263 264 if (ret) { 265 dev_info(&pf->pdev->dev, 266 "Filter command send failed for PCTYPE %d (ret = %d)\n", 267 fd_data->pctype, ret); 268 err = true; 269 } else { 270 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n", 271 fd_data->pctype, ret); 272 } 273 274 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP; 275 276 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 277 if (ret) { 278 dev_info(&pf->pdev->dev, 279 "Filter command send failed for PCTYPE %d (ret = %d)\n", 280 fd_data->pctype, ret); 281 err = true; 282 } else { 283 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n", 284 fd_data->pctype, ret); 285 } 286 287 return err ? -EOPNOTSUPP : 0; 288 } 289 290 /** 291 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for 292 * a specific flow spec 293 * @vsi: pointer to the targeted VSI 294 * @fd_data: the flow director data required for the FDir descriptor 295 * @raw_packet: the pre-allocated packet buffer for FDir 296 * @add: true adds a filter, false removes it 297 * 298 * Always returns -EOPNOTSUPP 299 **/ 300 static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi, 301 struct i40e_fdir_filter *fd_data, 302 u8 *raw_packet, bool add) 303 { 304 return -EOPNOTSUPP; 305 } 306 307 #define I40E_IP_DUMMY_PACKET_LEN 34 308 /** 309 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for 310 * a specific flow spec 311 * @vsi: pointer to the targeted VSI 312 * @fd_data: the flow director data required for the FDir descriptor 313 * @raw_packet: the pre-allocated packet buffer for FDir 314 * @add: true adds a filter, false removes it 315 * 316 * Returns 0 if the filters were successfully added or removed 317 **/ 318 static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi, 319 struct i40e_fdir_filter *fd_data, 320 u8 *raw_packet, bool add) 321 { 322 struct i40e_pf *pf = vsi->back; 323 struct iphdr *ip; 324 bool err = false; 325 int ret; 326 int i; 327 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 328 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0, 329 0, 0, 0, 0}; 330 331 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN); 332 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 333 334 ip->saddr = fd_data->src_ip[0]; 335 ip->daddr = fd_data->dst_ip[0]; 336 ip->protocol = 0; 337 338 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER; 339 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) { 340 fd_data->pctype = i; 341 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 342 343 if (ret) { 344 dev_info(&pf->pdev->dev, 345 "Filter command send failed for PCTYPE %d (ret = %d)\n", 346 fd_data->pctype, ret); 347 err = true; 348 } else { 349 dev_info(&pf->pdev->dev, 350 "Filter OK for PCTYPE %d (ret = %d)\n", 351 fd_data->pctype, ret); 352 } 353 } 354 355 return err ? -EOPNOTSUPP : 0; 356 } 357 358 /** 359 * i40e_add_del_fdir - Build raw packets to add/del fdir filter 360 * @vsi: pointer to the targeted VSI 361 * @cmd: command to get or set RX flow classification rules 362 * @add: true adds a filter, false removes it 363 * 364 **/ 365 int i40e_add_del_fdir(struct i40e_vsi *vsi, 366 struct i40e_fdir_filter *input, bool add) 367 { 368 struct i40e_pf *pf = vsi->back; 369 u8 *raw_packet; 370 int ret; 371 372 /* Populate the Flow Director that we have at the moment 373 * and allocate the raw packet buffer for the calling functions 374 */ 375 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 376 if (!raw_packet) 377 return -ENOMEM; 378 379 switch (input->flow_type & ~FLOW_EXT) { 380 case TCP_V4_FLOW: 381 ret = i40e_add_del_fdir_tcpv4(vsi, input, raw_packet, 382 add); 383 break; 384 case UDP_V4_FLOW: 385 ret = i40e_add_del_fdir_udpv4(vsi, input, raw_packet, 386 add); 387 break; 388 case SCTP_V4_FLOW: 389 ret = i40e_add_del_fdir_sctpv4(vsi, input, raw_packet, 390 add); 391 break; 392 case IPV4_FLOW: 393 ret = i40e_add_del_fdir_ipv4(vsi, input, raw_packet, 394 add); 395 break; 396 case IP_USER_FLOW: 397 switch (input->ip4_proto) { 398 case IPPROTO_TCP: 399 ret = i40e_add_del_fdir_tcpv4(vsi, input, 400 raw_packet, add); 401 break; 402 case IPPROTO_UDP: 403 ret = i40e_add_del_fdir_udpv4(vsi, input, 404 raw_packet, add); 405 break; 406 case IPPROTO_SCTP: 407 ret = i40e_add_del_fdir_sctpv4(vsi, input, 408 raw_packet, add); 409 break; 410 default: 411 ret = i40e_add_del_fdir_ipv4(vsi, input, 412 raw_packet, add); 413 break; 414 } 415 break; 416 default: 417 dev_info(&pf->pdev->dev, "Could not specify spec type %d\n", 418 input->flow_type); 419 ret = -EINVAL; 420 } 421 422 kfree(raw_packet); 423 return ret; 424 } 425 426 /** 427 * i40e_fd_handle_status - check the Programming Status for FD 428 * @rx_ring: the Rx ring for this descriptor 429 * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor. 430 * @prog_id: the id originally used for programming 431 * 432 * This is used to verify if the FD programming or invalidation 433 * requested by SW to the HW is successful or not and take actions accordingly. 434 **/ 435 static void i40e_fd_handle_status(struct i40e_ring *rx_ring, 436 union i40e_rx_desc *rx_desc, u8 prog_id) 437 { 438 struct i40e_pf *pf = rx_ring->vsi->back; 439 struct pci_dev *pdev = pf->pdev; 440 u32 fcnt_prog, fcnt_avail; 441 u32 error; 442 u64 qw; 443 444 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 445 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> 446 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT; 447 448 if (error == (0x1 << I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) { 449 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n", 450 rx_desc->wb.qword0.hi_dword.fd_id); 451 452 /* filter programming failed most likely due to table full */ 453 fcnt_prog = i40e_get_current_fd_count(pf); 454 fcnt_avail = i40e_get_fd_cnt_all(pf); 455 /* If ATR is running fcnt_prog can quickly change, 456 * if we are very close to full, it makes sense to disable 457 * FD ATR/SB and then re-enable it when there is room. 458 */ 459 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) { 460 /* Turn off ATR first */ 461 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) { 462 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED; 463 dev_warn(&pdev->dev, "FD filter space full, ATR for further flows will be turned off\n"); 464 pf->auto_disable_flags |= 465 I40E_FLAG_FD_ATR_ENABLED; 466 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT; 467 } else if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 468 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 469 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n"); 470 pf->auto_disable_flags |= 471 I40E_FLAG_FD_SB_ENABLED; 472 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT; 473 } 474 } else { 475 dev_info(&pdev->dev, "FD filter programming error\n"); 476 } 477 } else if (error == 478 (0x1 << I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) { 479 if (I40E_DEBUG_FD & pf->hw.debug_mask) 480 dev_info(&pdev->dev, "ntuple filter loc = %d, could not be removed\n", 481 rx_desc->wb.qword0.hi_dword.fd_id); 482 } 483 } 484 485 /** 486 * i40e_unmap_and_free_tx_resource - Release a Tx buffer 487 * @ring: the ring that owns the buffer 488 * @tx_buffer: the buffer to free 489 **/ 490 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring, 491 struct i40e_tx_buffer *tx_buffer) 492 { 493 if (tx_buffer->skb) { 494 dev_kfree_skb_any(tx_buffer->skb); 495 if (dma_unmap_len(tx_buffer, len)) 496 dma_unmap_single(ring->dev, 497 dma_unmap_addr(tx_buffer, dma), 498 dma_unmap_len(tx_buffer, len), 499 DMA_TO_DEVICE); 500 } else if (dma_unmap_len(tx_buffer, len)) { 501 dma_unmap_page(ring->dev, 502 dma_unmap_addr(tx_buffer, dma), 503 dma_unmap_len(tx_buffer, len), 504 DMA_TO_DEVICE); 505 } 506 tx_buffer->next_to_watch = NULL; 507 tx_buffer->skb = NULL; 508 dma_unmap_len_set(tx_buffer, len, 0); 509 /* tx_buffer must be completely set up in the transmit path */ 510 } 511 512 /** 513 * i40e_clean_tx_ring - Free any empty Tx buffers 514 * @tx_ring: ring to be cleaned 515 **/ 516 void i40e_clean_tx_ring(struct i40e_ring *tx_ring) 517 { 518 unsigned long bi_size; 519 u16 i; 520 521 /* ring already cleared, nothing to do */ 522 if (!tx_ring->tx_bi) 523 return; 524 525 /* Free all the Tx ring sk_buffs */ 526 for (i = 0; i < tx_ring->count; i++) 527 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]); 528 529 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 530 memset(tx_ring->tx_bi, 0, bi_size); 531 532 /* Zero out the descriptor ring */ 533 memset(tx_ring->desc, 0, tx_ring->size); 534 535 tx_ring->next_to_use = 0; 536 tx_ring->next_to_clean = 0; 537 538 if (!tx_ring->netdev) 539 return; 540 541 /* cleanup Tx queue statistics */ 542 netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev, 543 tx_ring->queue_index)); 544 } 545 546 /** 547 * i40e_free_tx_resources - Free Tx resources per queue 548 * @tx_ring: Tx descriptor ring for a specific queue 549 * 550 * Free all transmit software resources 551 **/ 552 void i40e_free_tx_resources(struct i40e_ring *tx_ring) 553 { 554 i40e_clean_tx_ring(tx_ring); 555 kfree(tx_ring->tx_bi); 556 tx_ring->tx_bi = NULL; 557 558 if (tx_ring->desc) { 559 dma_free_coherent(tx_ring->dev, tx_ring->size, 560 tx_ring->desc, tx_ring->dma); 561 tx_ring->desc = NULL; 562 } 563 } 564 565 /** 566 * i40e_get_tx_pending - how many tx descriptors not processed 567 * @tx_ring: the ring of descriptors 568 * 569 * Since there is no access to the ring head register 570 * in XL710, we need to use our local copies 571 **/ 572 static u32 i40e_get_tx_pending(struct i40e_ring *ring) 573 { 574 u32 ntu = ((ring->next_to_clean <= ring->next_to_use) 575 ? ring->next_to_use 576 : ring->next_to_use + ring->count); 577 return ntu - ring->next_to_clean; 578 } 579 580 /** 581 * i40e_check_tx_hang - Is there a hang in the Tx queue 582 * @tx_ring: the ring of descriptors 583 **/ 584 static bool i40e_check_tx_hang(struct i40e_ring *tx_ring) 585 { 586 u32 tx_pending = i40e_get_tx_pending(tx_ring); 587 bool ret = false; 588 589 clear_check_for_tx_hang(tx_ring); 590 591 /* Check for a hung queue, but be thorough. This verifies 592 * that a transmit has been completed since the previous 593 * check AND there is at least one packet pending. The 594 * ARMED bit is set to indicate a potential hang. The 595 * bit is cleared if a pause frame is received to remove 596 * false hang detection due to PFC or 802.3x frames. By 597 * requiring this to fail twice we avoid races with 598 * PFC clearing the ARMED bit and conditions where we 599 * run the check_tx_hang logic with a transmit completion 600 * pending but without time to complete it yet. 601 */ 602 if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) && 603 tx_pending) { 604 /* make sure it is true for two checks in a row */ 605 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED, 606 &tx_ring->state); 607 } else { 608 /* update completed stats and disarm the hang check */ 609 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets; 610 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state); 611 } 612 613 return ret; 614 } 615 616 /** 617 * i40e_get_head - Retrieve head from head writeback 618 * @tx_ring: tx ring to fetch head of 619 * 620 * Returns value of Tx ring head based on value stored 621 * in head write-back location 622 **/ 623 static inline u32 i40e_get_head(struct i40e_ring *tx_ring) 624 { 625 void *head = (struct i40e_tx_desc *)tx_ring->desc + tx_ring->count; 626 627 return le32_to_cpu(*(volatile __le32 *)head); 628 } 629 630 /** 631 * i40e_clean_tx_irq - Reclaim resources after transmit completes 632 * @tx_ring: tx ring to clean 633 * @budget: how many cleans we're allowed 634 * 635 * Returns true if there's any budget left (e.g. the clean is finished) 636 **/ 637 static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget) 638 { 639 u16 i = tx_ring->next_to_clean; 640 struct i40e_tx_buffer *tx_buf; 641 struct i40e_tx_desc *tx_head; 642 struct i40e_tx_desc *tx_desc; 643 unsigned int total_packets = 0; 644 unsigned int total_bytes = 0; 645 646 tx_buf = &tx_ring->tx_bi[i]; 647 tx_desc = I40E_TX_DESC(tx_ring, i); 648 i -= tx_ring->count; 649 650 tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring)); 651 652 do { 653 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 654 655 /* if next_to_watch is not set then there is no work pending */ 656 if (!eop_desc) 657 break; 658 659 /* prevent any other reads prior to eop_desc */ 660 read_barrier_depends(); 661 662 /* we have caught up to head, no work left to do */ 663 if (tx_head == tx_desc) 664 break; 665 666 /* clear next_to_watch to prevent false hangs */ 667 tx_buf->next_to_watch = NULL; 668 669 /* update the statistics for this packet */ 670 total_bytes += tx_buf->bytecount; 671 total_packets += tx_buf->gso_segs; 672 673 /* free the skb */ 674 dev_kfree_skb_any(tx_buf->skb); 675 676 /* unmap skb header data */ 677 dma_unmap_single(tx_ring->dev, 678 dma_unmap_addr(tx_buf, dma), 679 dma_unmap_len(tx_buf, len), 680 DMA_TO_DEVICE); 681 682 /* clear tx_buffer data */ 683 tx_buf->skb = NULL; 684 dma_unmap_len_set(tx_buf, len, 0); 685 686 /* unmap remaining buffers */ 687 while (tx_desc != eop_desc) { 688 689 tx_buf++; 690 tx_desc++; 691 i++; 692 if (unlikely(!i)) { 693 i -= tx_ring->count; 694 tx_buf = tx_ring->tx_bi; 695 tx_desc = I40E_TX_DESC(tx_ring, 0); 696 } 697 698 /* unmap any remaining paged data */ 699 if (dma_unmap_len(tx_buf, len)) { 700 dma_unmap_page(tx_ring->dev, 701 dma_unmap_addr(tx_buf, dma), 702 dma_unmap_len(tx_buf, len), 703 DMA_TO_DEVICE); 704 dma_unmap_len_set(tx_buf, len, 0); 705 } 706 } 707 708 /* move us one more past the eop_desc for start of next pkt */ 709 tx_buf++; 710 tx_desc++; 711 i++; 712 if (unlikely(!i)) { 713 i -= tx_ring->count; 714 tx_buf = tx_ring->tx_bi; 715 tx_desc = I40E_TX_DESC(tx_ring, 0); 716 } 717 718 /* update budget accounting */ 719 budget--; 720 } while (likely(budget)); 721 722 i += tx_ring->count; 723 tx_ring->next_to_clean = i; 724 u64_stats_update_begin(&tx_ring->syncp); 725 tx_ring->stats.bytes += total_bytes; 726 tx_ring->stats.packets += total_packets; 727 u64_stats_update_end(&tx_ring->syncp); 728 tx_ring->q_vector->tx.total_bytes += total_bytes; 729 tx_ring->q_vector->tx.total_packets += total_packets; 730 731 if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) { 732 /* schedule immediate reset if we believe we hung */ 733 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n" 734 " VSI <%d>\n" 735 " Tx Queue <%d>\n" 736 " next_to_use <%x>\n" 737 " next_to_clean <%x>\n", 738 tx_ring->vsi->seid, 739 tx_ring->queue_index, 740 tx_ring->next_to_use, i); 741 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n" 742 " time_stamp <%lx>\n" 743 " jiffies <%lx>\n", 744 tx_ring->tx_bi[i].time_stamp, jiffies); 745 746 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 747 748 dev_info(tx_ring->dev, 749 "tx hang detected on queue %d, resetting adapter\n", 750 tx_ring->queue_index); 751 752 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev); 753 754 /* the adapter is about to reset, no point in enabling stuff */ 755 return true; 756 } 757 758 netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev, 759 tx_ring->queue_index), 760 total_packets, total_bytes); 761 762 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 763 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 764 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 765 /* Make sure that anybody stopping the queue after this 766 * sees the new next_to_clean. 767 */ 768 smp_mb(); 769 if (__netif_subqueue_stopped(tx_ring->netdev, 770 tx_ring->queue_index) && 771 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) { 772 netif_wake_subqueue(tx_ring->netdev, 773 tx_ring->queue_index); 774 ++tx_ring->tx_stats.restart_queue; 775 } 776 } 777 778 return budget > 0; 779 } 780 781 /** 782 * i40e_set_new_dynamic_itr - Find new ITR level 783 * @rc: structure containing ring performance data 784 * 785 * Stores a new ITR value based on packets and byte counts during 786 * the last interrupt. The advantage of per interrupt computation 787 * is faster updates and more accurate ITR for the current traffic 788 * pattern. Constants in this function were computed based on 789 * theoretical maximum wire speed and thresholds were set based on 790 * testing data as well as attempting to minimize response time 791 * while increasing bulk throughput. 792 **/ 793 static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc) 794 { 795 enum i40e_latency_range new_latency_range = rc->latency_range; 796 u32 new_itr = rc->itr; 797 int bytes_per_int; 798 799 if (rc->total_packets == 0 || !rc->itr) 800 return; 801 802 /* simple throttlerate management 803 * 0-10MB/s lowest (100000 ints/s) 804 * 10-20MB/s low (20000 ints/s) 805 * 20-1249MB/s bulk (8000 ints/s) 806 */ 807 bytes_per_int = rc->total_bytes / rc->itr; 808 switch (rc->itr) { 809 case I40E_LOWEST_LATENCY: 810 if (bytes_per_int > 10) 811 new_latency_range = I40E_LOW_LATENCY; 812 break; 813 case I40E_LOW_LATENCY: 814 if (bytes_per_int > 20) 815 new_latency_range = I40E_BULK_LATENCY; 816 else if (bytes_per_int <= 10) 817 new_latency_range = I40E_LOWEST_LATENCY; 818 break; 819 case I40E_BULK_LATENCY: 820 if (bytes_per_int <= 20) 821 rc->latency_range = I40E_LOW_LATENCY; 822 break; 823 } 824 825 switch (new_latency_range) { 826 case I40E_LOWEST_LATENCY: 827 new_itr = I40E_ITR_100K; 828 break; 829 case I40E_LOW_LATENCY: 830 new_itr = I40E_ITR_20K; 831 break; 832 case I40E_BULK_LATENCY: 833 new_itr = I40E_ITR_8K; 834 break; 835 default: 836 break; 837 } 838 839 if (new_itr != rc->itr) { 840 /* do an exponential smoothing */ 841 new_itr = (10 * new_itr * rc->itr) / 842 ((9 * new_itr) + rc->itr); 843 rc->itr = new_itr & I40E_MAX_ITR; 844 } 845 846 rc->total_bytes = 0; 847 rc->total_packets = 0; 848 } 849 850 /** 851 * i40e_update_dynamic_itr - Adjust ITR based on bytes per int 852 * @q_vector: the vector to adjust 853 **/ 854 static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector) 855 { 856 u16 vector = q_vector->vsi->base_vector + q_vector->v_idx; 857 struct i40e_hw *hw = &q_vector->vsi->back->hw; 858 u32 reg_addr; 859 u16 old_itr; 860 861 reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1); 862 old_itr = q_vector->rx.itr; 863 i40e_set_new_dynamic_itr(&q_vector->rx); 864 if (old_itr != q_vector->rx.itr) 865 wr32(hw, reg_addr, q_vector->rx.itr); 866 867 reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1); 868 old_itr = q_vector->tx.itr; 869 i40e_set_new_dynamic_itr(&q_vector->tx); 870 if (old_itr != q_vector->tx.itr) 871 wr32(hw, reg_addr, q_vector->tx.itr); 872 } 873 874 /** 875 * i40e_clean_programming_status - clean the programming status descriptor 876 * @rx_ring: the rx ring that has this descriptor 877 * @rx_desc: the rx descriptor written back by HW 878 * 879 * Flow director should handle FD_FILTER_STATUS to check its filter programming 880 * status being successful or not and take actions accordingly. FCoE should 881 * handle its context/filter programming/invalidation status and take actions. 882 * 883 **/ 884 static void i40e_clean_programming_status(struct i40e_ring *rx_ring, 885 union i40e_rx_desc *rx_desc) 886 { 887 u64 qw; 888 u8 id; 889 890 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 891 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> 892 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT; 893 894 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) 895 i40e_fd_handle_status(rx_ring, rx_desc, id); 896 } 897 898 /** 899 * i40e_setup_tx_descriptors - Allocate the Tx descriptors 900 * @tx_ring: the tx ring to set up 901 * 902 * Return 0 on success, negative on error 903 **/ 904 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring) 905 { 906 struct device *dev = tx_ring->dev; 907 int bi_size; 908 909 if (!dev) 910 return -ENOMEM; 911 912 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 913 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL); 914 if (!tx_ring->tx_bi) 915 goto err; 916 917 /* round up to nearest 4K */ 918 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc); 919 /* add u32 for head writeback, align after this takes care of 920 * guaranteeing this is at least one cache line in size 921 */ 922 tx_ring->size += sizeof(u32); 923 tx_ring->size = ALIGN(tx_ring->size, 4096); 924 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 925 &tx_ring->dma, GFP_KERNEL); 926 if (!tx_ring->desc) { 927 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 928 tx_ring->size); 929 goto err; 930 } 931 932 tx_ring->next_to_use = 0; 933 tx_ring->next_to_clean = 0; 934 return 0; 935 936 err: 937 kfree(tx_ring->tx_bi); 938 tx_ring->tx_bi = NULL; 939 return -ENOMEM; 940 } 941 942 /** 943 * i40e_clean_rx_ring - Free Rx buffers 944 * @rx_ring: ring to be cleaned 945 **/ 946 void i40e_clean_rx_ring(struct i40e_ring *rx_ring) 947 { 948 struct device *dev = rx_ring->dev; 949 struct i40e_rx_buffer *rx_bi; 950 unsigned long bi_size; 951 u16 i; 952 953 /* ring already cleared, nothing to do */ 954 if (!rx_ring->rx_bi) 955 return; 956 957 /* Free all the Rx ring sk_buffs */ 958 for (i = 0; i < rx_ring->count; i++) { 959 rx_bi = &rx_ring->rx_bi[i]; 960 if (rx_bi->dma) { 961 dma_unmap_single(dev, 962 rx_bi->dma, 963 rx_ring->rx_buf_len, 964 DMA_FROM_DEVICE); 965 rx_bi->dma = 0; 966 } 967 if (rx_bi->skb) { 968 dev_kfree_skb(rx_bi->skb); 969 rx_bi->skb = NULL; 970 } 971 if (rx_bi->page) { 972 if (rx_bi->page_dma) { 973 dma_unmap_page(dev, 974 rx_bi->page_dma, 975 PAGE_SIZE / 2, 976 DMA_FROM_DEVICE); 977 rx_bi->page_dma = 0; 978 } 979 __free_page(rx_bi->page); 980 rx_bi->page = NULL; 981 rx_bi->page_offset = 0; 982 } 983 } 984 985 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 986 memset(rx_ring->rx_bi, 0, bi_size); 987 988 /* Zero out the descriptor ring */ 989 memset(rx_ring->desc, 0, rx_ring->size); 990 991 rx_ring->next_to_clean = 0; 992 rx_ring->next_to_use = 0; 993 } 994 995 /** 996 * i40e_free_rx_resources - Free Rx resources 997 * @rx_ring: ring to clean the resources from 998 * 999 * Free all receive software resources 1000 **/ 1001 void i40e_free_rx_resources(struct i40e_ring *rx_ring) 1002 { 1003 i40e_clean_rx_ring(rx_ring); 1004 kfree(rx_ring->rx_bi); 1005 rx_ring->rx_bi = NULL; 1006 1007 if (rx_ring->desc) { 1008 dma_free_coherent(rx_ring->dev, rx_ring->size, 1009 rx_ring->desc, rx_ring->dma); 1010 rx_ring->desc = NULL; 1011 } 1012 } 1013 1014 /** 1015 * i40e_setup_rx_descriptors - Allocate Rx descriptors 1016 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 1017 * 1018 * Returns 0 on success, negative on failure 1019 **/ 1020 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) 1021 { 1022 struct device *dev = rx_ring->dev; 1023 int bi_size; 1024 1025 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 1026 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL); 1027 if (!rx_ring->rx_bi) 1028 goto err; 1029 1030 /* Round up to nearest 4K */ 1031 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring) 1032 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc) 1033 : rx_ring->count * sizeof(union i40e_32byte_rx_desc); 1034 rx_ring->size = ALIGN(rx_ring->size, 4096); 1035 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 1036 &rx_ring->dma, GFP_KERNEL); 1037 1038 if (!rx_ring->desc) { 1039 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 1040 rx_ring->size); 1041 goto err; 1042 } 1043 1044 rx_ring->next_to_clean = 0; 1045 rx_ring->next_to_use = 0; 1046 1047 return 0; 1048 err: 1049 kfree(rx_ring->rx_bi); 1050 rx_ring->rx_bi = NULL; 1051 return -ENOMEM; 1052 } 1053 1054 /** 1055 * i40e_release_rx_desc - Store the new tail and head values 1056 * @rx_ring: ring to bump 1057 * @val: new head index 1058 **/ 1059 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val) 1060 { 1061 rx_ring->next_to_use = val; 1062 /* Force memory writes to complete before letting h/w 1063 * know there are new descriptors to fetch. (Only 1064 * applicable for weak-ordered memory model archs, 1065 * such as IA-64). 1066 */ 1067 wmb(); 1068 writel(val, rx_ring->tail); 1069 } 1070 1071 /** 1072 * i40e_alloc_rx_buffers - Replace used receive buffers; packet split 1073 * @rx_ring: ring to place buffers on 1074 * @cleaned_count: number of buffers to replace 1075 **/ 1076 void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count) 1077 { 1078 u16 i = rx_ring->next_to_use; 1079 union i40e_rx_desc *rx_desc; 1080 struct i40e_rx_buffer *bi; 1081 struct sk_buff *skb; 1082 1083 /* do nothing if no valid netdev defined */ 1084 if (!rx_ring->netdev || !cleaned_count) 1085 return; 1086 1087 while (cleaned_count--) { 1088 rx_desc = I40E_RX_DESC(rx_ring, i); 1089 bi = &rx_ring->rx_bi[i]; 1090 skb = bi->skb; 1091 1092 if (!skb) { 1093 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 1094 rx_ring->rx_buf_len); 1095 if (!skb) { 1096 rx_ring->rx_stats.alloc_buff_failed++; 1097 goto no_buffers; 1098 } 1099 /* initialize queue mapping */ 1100 skb_record_rx_queue(skb, rx_ring->queue_index); 1101 bi->skb = skb; 1102 } 1103 1104 if (!bi->dma) { 1105 bi->dma = dma_map_single(rx_ring->dev, 1106 skb->data, 1107 rx_ring->rx_buf_len, 1108 DMA_FROM_DEVICE); 1109 if (dma_mapping_error(rx_ring->dev, bi->dma)) { 1110 rx_ring->rx_stats.alloc_buff_failed++; 1111 bi->dma = 0; 1112 goto no_buffers; 1113 } 1114 } 1115 1116 if (ring_is_ps_enabled(rx_ring)) { 1117 if (!bi->page) { 1118 bi->page = alloc_page(GFP_ATOMIC); 1119 if (!bi->page) { 1120 rx_ring->rx_stats.alloc_page_failed++; 1121 goto no_buffers; 1122 } 1123 } 1124 1125 if (!bi->page_dma) { 1126 /* use a half page if we're re-using */ 1127 bi->page_offset ^= PAGE_SIZE / 2; 1128 bi->page_dma = dma_map_page(rx_ring->dev, 1129 bi->page, 1130 bi->page_offset, 1131 PAGE_SIZE / 2, 1132 DMA_FROM_DEVICE); 1133 if (dma_mapping_error(rx_ring->dev, 1134 bi->page_dma)) { 1135 rx_ring->rx_stats.alloc_page_failed++; 1136 bi->page_dma = 0; 1137 goto no_buffers; 1138 } 1139 } 1140 1141 /* Refresh the desc even if buffer_addrs didn't change 1142 * because each write-back erases this info. 1143 */ 1144 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); 1145 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); 1146 } else { 1147 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); 1148 rx_desc->read.hdr_addr = 0; 1149 } 1150 i++; 1151 if (i == rx_ring->count) 1152 i = 0; 1153 } 1154 1155 no_buffers: 1156 if (rx_ring->next_to_use != i) 1157 i40e_release_rx_desc(rx_ring, i); 1158 } 1159 1160 /** 1161 * i40e_receive_skb - Send a completed packet up the stack 1162 * @rx_ring: rx ring in play 1163 * @skb: packet to send up 1164 * @vlan_tag: vlan tag for packet 1165 **/ 1166 static void i40e_receive_skb(struct i40e_ring *rx_ring, 1167 struct sk_buff *skb, u16 vlan_tag) 1168 { 1169 struct i40e_q_vector *q_vector = rx_ring->q_vector; 1170 struct i40e_vsi *vsi = rx_ring->vsi; 1171 u64 flags = vsi->back->flags; 1172 1173 if (vlan_tag & VLAN_VID_MASK) 1174 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); 1175 1176 if (flags & I40E_FLAG_IN_NETPOLL) 1177 netif_rx(skb); 1178 else 1179 napi_gro_receive(&q_vector->napi, skb); 1180 } 1181 1182 /** 1183 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum 1184 * @vsi: the VSI we care about 1185 * @skb: skb currently being received and modified 1186 * @rx_status: status value of last descriptor in packet 1187 * @rx_error: error value of last descriptor in packet 1188 * @rx_ptype: ptype value of last descriptor in packet 1189 **/ 1190 static inline void i40e_rx_checksum(struct i40e_vsi *vsi, 1191 struct sk_buff *skb, 1192 u32 rx_status, 1193 u32 rx_error, 1194 u16 rx_ptype) 1195 { 1196 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(rx_ptype); 1197 bool ipv4 = false, ipv6 = false; 1198 bool ipv4_tunnel, ipv6_tunnel; 1199 __wsum rx_udp_csum; 1200 struct iphdr *iph; 1201 __sum16 csum; 1202 1203 ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) && 1204 (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4); 1205 ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) && 1206 (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4); 1207 1208 skb->encapsulation = ipv4_tunnel || ipv6_tunnel; 1209 skb->ip_summed = CHECKSUM_NONE; 1210 1211 /* Rx csum enabled and ip headers found? */ 1212 if (!(vsi->netdev->features & NETIF_F_RXCSUM)) 1213 return; 1214 1215 /* did the hardware decode the packet and checksum? */ 1216 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT))) 1217 return; 1218 1219 /* both known and outer_ip must be set for the below code to work */ 1220 if (!(decoded.known && decoded.outer_ip)) 1221 return; 1222 1223 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1224 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) 1225 ipv4 = true; 1226 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1227 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) 1228 ipv6 = true; 1229 1230 if (ipv4 && 1231 (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | 1232 (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT)))) 1233 goto checksum_fail; 1234 1235 /* likely incorrect csum if alternate IP extension headers found */ 1236 if (ipv6 && 1237 decoded.inner_prot == I40E_RX_PTYPE_INNER_PROT_TCP && 1238 rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) && 1239 rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) 1240 /* don't increment checksum err here, non-fatal err */ 1241 return; 1242 1243 /* there was some L4 error, count error and punt packet to the stack */ 1244 if (rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)) 1245 goto checksum_fail; 1246 1247 /* handle packets that were not able to be checksummed due 1248 * to arrival speed, in this case the stack can compute 1249 * the csum. 1250 */ 1251 if (rx_error & (1 << I40E_RX_DESC_ERROR_PPRS_SHIFT)) 1252 return; 1253 1254 /* If VXLAN traffic has an outer UDPv4 checksum we need to check 1255 * it in the driver, hardware does not do it for us. 1256 * Since L3L4P bit was set we assume a valid IHL value (>=5) 1257 * so the total length of IPv4 header is IHL*4 bytes 1258 * The UDP_0 bit *may* bet set if the *inner* header is UDP 1259 */ 1260 if (ipv4_tunnel && 1261 (decoded.inner_prot != I40E_RX_PTYPE_INNER_PROT_UDP) && 1262 !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) { 1263 skb->transport_header = skb->mac_header + 1264 sizeof(struct ethhdr) + 1265 (ip_hdr(skb)->ihl * 4); 1266 1267 /* Add 4 bytes for VLAN tagged packets */ 1268 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) || 1269 skb->protocol == htons(ETH_P_8021AD)) 1270 ? VLAN_HLEN : 0; 1271 1272 rx_udp_csum = udp_csum(skb); 1273 iph = ip_hdr(skb); 1274 csum = csum_tcpudp_magic( 1275 iph->saddr, iph->daddr, 1276 (skb->len - skb_transport_offset(skb)), 1277 IPPROTO_UDP, rx_udp_csum); 1278 1279 if (udp_hdr(skb)->check != csum) 1280 goto checksum_fail; 1281 } 1282 1283 skb->ip_summed = CHECKSUM_UNNECESSARY; 1284 1285 return; 1286 1287 checksum_fail: 1288 vsi->back->hw_csum_rx_error++; 1289 } 1290 1291 /** 1292 * i40e_rx_hash - returns the hash value from the Rx descriptor 1293 * @ring: descriptor ring 1294 * @rx_desc: specific descriptor 1295 **/ 1296 static inline u32 i40e_rx_hash(struct i40e_ring *ring, 1297 union i40e_rx_desc *rx_desc) 1298 { 1299 const __le64 rss_mask = 1300 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH << 1301 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT); 1302 1303 if ((ring->netdev->features & NETIF_F_RXHASH) && 1304 (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) 1305 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss); 1306 else 1307 return 0; 1308 } 1309 1310 /** 1311 * i40e_ptype_to_hash - get a hash type 1312 * @ptype: the ptype value from the descriptor 1313 * 1314 * Returns a hash type to be used by skb_set_hash 1315 **/ 1316 static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype) 1317 { 1318 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype); 1319 1320 if (!decoded.known) 1321 return PKT_HASH_TYPE_NONE; 1322 1323 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1324 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4) 1325 return PKT_HASH_TYPE_L4; 1326 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1327 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3) 1328 return PKT_HASH_TYPE_L3; 1329 else 1330 return PKT_HASH_TYPE_L2; 1331 } 1332 1333 /** 1334 * i40e_clean_rx_irq - Reclaim resources after receive completes 1335 * @rx_ring: rx ring to clean 1336 * @budget: how many cleans we're allowed 1337 * 1338 * Returns true if there's any budget left (e.g. the clean is finished) 1339 **/ 1340 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) 1341 { 1342 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1343 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo; 1344 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); 1345 const int current_node = numa_node_id(); 1346 struct i40e_vsi *vsi = rx_ring->vsi; 1347 u16 i = rx_ring->next_to_clean; 1348 union i40e_rx_desc *rx_desc; 1349 u32 rx_error, rx_status; 1350 u8 rx_ptype; 1351 u64 qword; 1352 1353 if (budget <= 0) 1354 return 0; 1355 1356 rx_desc = I40E_RX_DESC(rx_ring, i); 1357 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1358 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> 1359 I40E_RXD_QW1_STATUS_SHIFT; 1360 1361 while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) { 1362 union i40e_rx_desc *next_rxd; 1363 struct i40e_rx_buffer *rx_bi; 1364 struct sk_buff *skb; 1365 u16 vlan_tag; 1366 if (i40e_rx_is_programming_status(qword)) { 1367 i40e_clean_programming_status(rx_ring, rx_desc); 1368 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd); 1369 goto next_desc; 1370 } 1371 rx_bi = &rx_ring->rx_bi[i]; 1372 skb = rx_bi->skb; 1373 prefetch(skb->data); 1374 1375 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 1376 I40E_RXD_QW1_LENGTH_PBUF_SHIFT; 1377 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >> 1378 I40E_RXD_QW1_LENGTH_HBUF_SHIFT; 1379 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >> 1380 I40E_RXD_QW1_LENGTH_SPH_SHIFT; 1381 1382 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >> 1383 I40E_RXD_QW1_ERROR_SHIFT; 1384 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT); 1385 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT); 1386 1387 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> 1388 I40E_RXD_QW1_PTYPE_SHIFT; 1389 rx_bi->skb = NULL; 1390 1391 /* This memory barrier is needed to keep us from reading 1392 * any other fields out of the rx_desc until we know the 1393 * STATUS_DD bit is set 1394 */ 1395 rmb(); 1396 1397 /* Get the header and possibly the whole packet 1398 * If this is an skb from previous receive dma will be 0 1399 */ 1400 if (rx_bi->dma) { 1401 u16 len; 1402 1403 if (rx_hbo) 1404 len = I40E_RX_HDR_SIZE; 1405 else if (rx_sph) 1406 len = rx_header_len; 1407 else if (rx_packet_len) 1408 len = rx_packet_len; /* 1buf/no split found */ 1409 else 1410 len = rx_header_len; /* split always mode */ 1411 1412 skb_put(skb, len); 1413 dma_unmap_single(rx_ring->dev, 1414 rx_bi->dma, 1415 rx_ring->rx_buf_len, 1416 DMA_FROM_DEVICE); 1417 rx_bi->dma = 0; 1418 } 1419 1420 /* Get the rest of the data if this was a header split */ 1421 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) { 1422 1423 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 1424 rx_bi->page, 1425 rx_bi->page_offset, 1426 rx_packet_len); 1427 1428 skb->len += rx_packet_len; 1429 skb->data_len += rx_packet_len; 1430 skb->truesize += rx_packet_len; 1431 1432 if ((page_count(rx_bi->page) == 1) && 1433 (page_to_nid(rx_bi->page) == current_node)) 1434 get_page(rx_bi->page); 1435 else 1436 rx_bi->page = NULL; 1437 1438 dma_unmap_page(rx_ring->dev, 1439 rx_bi->page_dma, 1440 PAGE_SIZE / 2, 1441 DMA_FROM_DEVICE); 1442 rx_bi->page_dma = 0; 1443 } 1444 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd); 1445 1446 if (unlikely( 1447 !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) { 1448 struct i40e_rx_buffer *next_buffer; 1449 1450 next_buffer = &rx_ring->rx_bi[i]; 1451 1452 if (ring_is_ps_enabled(rx_ring)) { 1453 rx_bi->skb = next_buffer->skb; 1454 rx_bi->dma = next_buffer->dma; 1455 next_buffer->skb = skb; 1456 next_buffer->dma = 0; 1457 } 1458 rx_ring->rx_stats.non_eop_descs++; 1459 goto next_desc; 1460 } 1461 1462 /* ERR_MASK will only have valid bits if EOP set */ 1463 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { 1464 dev_kfree_skb_any(skb); 1465 /* TODO: shouldn't we increment a counter indicating the 1466 * drop? 1467 */ 1468 goto next_desc; 1469 } 1470 1471 skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc), 1472 i40e_ptype_to_hash(rx_ptype)); 1473 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) { 1474 i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status & 1475 I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >> 1476 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT); 1477 rx_ring->last_rx_timestamp = jiffies; 1478 } 1479 1480 /* probably a little skewed due to removing CRC */ 1481 total_rx_bytes += skb->len; 1482 total_rx_packets++; 1483 1484 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1485 1486 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype); 1487 1488 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT) 1489 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) 1490 : 0; 1491 i40e_receive_skb(rx_ring, skb, vlan_tag); 1492 1493 rx_ring->netdev->last_rx = jiffies; 1494 budget--; 1495 next_desc: 1496 rx_desc->wb.qword1.status_error_len = 0; 1497 if (!budget) 1498 break; 1499 1500 cleaned_count++; 1501 /* return some buffers to hardware, one at a time is too slow */ 1502 if (cleaned_count >= I40E_RX_BUFFER_WRITE) { 1503 i40e_alloc_rx_buffers(rx_ring, cleaned_count); 1504 cleaned_count = 0; 1505 } 1506 1507 /* use prefetched values */ 1508 rx_desc = next_rxd; 1509 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1510 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> 1511 I40E_RXD_QW1_STATUS_SHIFT; 1512 } 1513 1514 rx_ring->next_to_clean = i; 1515 u64_stats_update_begin(&rx_ring->syncp); 1516 rx_ring->stats.packets += total_rx_packets; 1517 rx_ring->stats.bytes += total_rx_bytes; 1518 u64_stats_update_end(&rx_ring->syncp); 1519 rx_ring->q_vector->rx.total_packets += total_rx_packets; 1520 rx_ring->q_vector->rx.total_bytes += total_rx_bytes; 1521 1522 if (cleaned_count) 1523 i40e_alloc_rx_buffers(rx_ring, cleaned_count); 1524 1525 return budget > 0; 1526 } 1527 1528 /** 1529 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine 1530 * @napi: napi struct with our devices info in it 1531 * @budget: amount of work driver is allowed to do this pass, in packets 1532 * 1533 * This function will clean all queues associated with a q_vector. 1534 * 1535 * Returns the amount of work done 1536 **/ 1537 int i40e_napi_poll(struct napi_struct *napi, int budget) 1538 { 1539 struct i40e_q_vector *q_vector = 1540 container_of(napi, struct i40e_q_vector, napi); 1541 struct i40e_vsi *vsi = q_vector->vsi; 1542 struct i40e_ring *ring; 1543 bool clean_complete = true; 1544 int budget_per_ring; 1545 1546 if (test_bit(__I40E_DOWN, &vsi->state)) { 1547 napi_complete(napi); 1548 return 0; 1549 } 1550 1551 /* Since the actual Tx work is minimal, we can give the Tx a larger 1552 * budget and be more aggressive about cleaning up the Tx descriptors. 1553 */ 1554 i40e_for_each_ring(ring, q_vector->tx) 1555 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit); 1556 1557 /* We attempt to distribute budget to each Rx queue fairly, but don't 1558 * allow the budget to go below 1 because that would exit polling early. 1559 */ 1560 budget_per_ring = max(budget/q_vector->num_ringpairs, 1); 1561 1562 i40e_for_each_ring(ring, q_vector->rx) 1563 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring); 1564 1565 /* If work not completed, return budget and polling will return */ 1566 if (!clean_complete) 1567 return budget; 1568 1569 /* Work is done so exit the polling mode and re-enable the interrupt */ 1570 napi_complete(napi); 1571 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) || 1572 ITR_IS_DYNAMIC(vsi->tx_itr_setting)) 1573 i40e_update_dynamic_itr(q_vector); 1574 1575 if (!test_bit(__I40E_DOWN, &vsi->state)) { 1576 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 1577 i40e_irq_dynamic_enable(vsi, 1578 q_vector->v_idx + vsi->base_vector); 1579 } else { 1580 struct i40e_hw *hw = &vsi->back->hw; 1581 /* We re-enable the queue 0 cause, but 1582 * don't worry about dynamic_enable 1583 * because we left it on for the other 1584 * possible interrupts during napi 1585 */ 1586 u32 qval = rd32(hw, I40E_QINT_RQCTL(0)); 1587 qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; 1588 wr32(hw, I40E_QINT_RQCTL(0), qval); 1589 1590 qval = rd32(hw, I40E_QINT_TQCTL(0)); 1591 qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; 1592 wr32(hw, I40E_QINT_TQCTL(0), qval); 1593 1594 i40e_irq_dynamic_enable_icr0(vsi->back); 1595 } 1596 } 1597 1598 return 0; 1599 } 1600 1601 /** 1602 * i40e_atr - Add a Flow Director ATR filter 1603 * @tx_ring: ring to add programming descriptor to 1604 * @skb: send buffer 1605 * @flags: send flags 1606 * @protocol: wire protocol 1607 **/ 1608 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb, 1609 u32 flags, __be16 protocol) 1610 { 1611 struct i40e_filter_program_desc *fdir_desc; 1612 struct i40e_pf *pf = tx_ring->vsi->back; 1613 union { 1614 unsigned char *network; 1615 struct iphdr *ipv4; 1616 struct ipv6hdr *ipv6; 1617 } hdr; 1618 struct tcphdr *th; 1619 unsigned int hlen; 1620 u32 flex_ptype, dtype_cmd; 1621 u16 i; 1622 1623 /* make sure ATR is enabled */ 1624 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED)) 1625 return; 1626 1627 /* if sampling is disabled do nothing */ 1628 if (!tx_ring->atr_sample_rate) 1629 return; 1630 1631 /* snag network header to get L4 type and address */ 1632 hdr.network = skb_network_header(skb); 1633 1634 /* Currently only IPv4/IPv6 with TCP is supported */ 1635 if (protocol == htons(ETH_P_IP)) { 1636 if (hdr.ipv4->protocol != IPPROTO_TCP) 1637 return; 1638 1639 /* access ihl as a u8 to avoid unaligned access on ia64 */ 1640 hlen = (hdr.network[0] & 0x0F) << 2; 1641 } else if (protocol == htons(ETH_P_IPV6)) { 1642 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1643 return; 1644 1645 hlen = sizeof(struct ipv6hdr); 1646 } else { 1647 return; 1648 } 1649 1650 th = (struct tcphdr *)(hdr.network + hlen); 1651 1652 /* Due to lack of space, no more new filters can be programmed */ 1653 if (th->syn && (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) 1654 return; 1655 1656 tx_ring->atr_count++; 1657 1658 /* sample on all syn/fin/rst packets or once every atr sample rate */ 1659 if (!th->fin && 1660 !th->syn && 1661 !th->rst && 1662 (tx_ring->atr_count < tx_ring->atr_sample_rate)) 1663 return; 1664 1665 tx_ring->atr_count = 0; 1666 1667 /* grab the next descriptor */ 1668 i = tx_ring->next_to_use; 1669 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i); 1670 1671 i++; 1672 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 1673 1674 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 1675 I40E_TXD_FLTR_QW0_QINDEX_MASK; 1676 flex_ptype |= (protocol == htons(ETH_P_IP)) ? 1677 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << 1678 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : 1679 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << 1680 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 1681 1682 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; 1683 1684 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG; 1685 1686 dtype_cmd |= (th->fin || th->rst) ? 1687 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 1688 I40E_TXD_FLTR_QW1_PCMD_SHIFT) : 1689 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 1690 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 1691 1692 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << 1693 I40E_TXD_FLTR_QW1_DEST_SHIFT; 1694 1695 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << 1696 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; 1697 1698 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK; 1699 dtype_cmd |= 1700 ((u32)pf->fd_atr_cnt_idx << I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 1701 I40E_TXD_FLTR_QW1_CNTINDEX_MASK; 1702 1703 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype); 1704 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd); 1705 } 1706 1707 /** 1708 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW 1709 * @skb: send buffer 1710 * @tx_ring: ring to send buffer on 1711 * @flags: the tx flags to be set 1712 * 1713 * Checks the skb and set up correspondingly several generic transmit flags 1714 * related to VLAN tagging for the HW, such as VLAN, DCB, etc. 1715 * 1716 * Returns error code indicate the frame should be dropped upon error and the 1717 * otherwise returns 0 to indicate the flags has been set properly. 1718 **/ 1719 static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb, 1720 struct i40e_ring *tx_ring, 1721 u32 *flags) 1722 { 1723 __be16 protocol = skb->protocol; 1724 u32 tx_flags = 0; 1725 1726 /* if we have a HW VLAN tag being added, default to the HW one */ 1727 if (vlan_tx_tag_present(skb)) { 1728 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT; 1729 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 1730 /* else if it is a SW VLAN, check the next protocol and store the tag */ 1731 } else if (protocol == htons(ETH_P_8021Q)) { 1732 struct vlan_hdr *vhdr, _vhdr; 1733 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 1734 if (!vhdr) 1735 return -EINVAL; 1736 1737 protocol = vhdr->h_vlan_encapsulated_proto; 1738 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT; 1739 tx_flags |= I40E_TX_FLAGS_SW_VLAN; 1740 } 1741 1742 /* Insert 802.1p priority into VLAN header */ 1743 if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) && 1744 ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) || 1745 (skb->priority != TC_PRIO_CONTROL))) { 1746 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK; 1747 tx_flags |= (skb->priority & 0x7) << 1748 I40E_TX_FLAGS_VLAN_PRIO_SHIFT; 1749 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) { 1750 struct vlan_ethhdr *vhdr; 1751 int rc; 1752 1753 rc = skb_cow_head(skb, 0); 1754 if (rc < 0) 1755 return rc; 1756 vhdr = (struct vlan_ethhdr *)skb->data; 1757 vhdr->h_vlan_TCI = htons(tx_flags >> 1758 I40E_TX_FLAGS_VLAN_SHIFT); 1759 } else { 1760 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 1761 } 1762 } 1763 *flags = tx_flags; 1764 return 0; 1765 } 1766 1767 /** 1768 * i40e_tso - set up the tso context descriptor 1769 * @tx_ring: ptr to the ring to send 1770 * @skb: ptr to the skb we're sending 1771 * @tx_flags: the collected send information 1772 * @protocol: the send protocol 1773 * @hdr_len: ptr to the size of the packet header 1774 * @cd_tunneling: ptr to context descriptor bits 1775 * 1776 * Returns 0 if no TSO can happen, 1 if tso is going, or error 1777 **/ 1778 static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb, 1779 u32 tx_flags, __be16 protocol, u8 *hdr_len, 1780 u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling) 1781 { 1782 u32 cd_cmd, cd_tso_len, cd_mss; 1783 struct ipv6hdr *ipv6h; 1784 struct tcphdr *tcph; 1785 struct iphdr *iph; 1786 u32 l4len; 1787 int err; 1788 1789 if (!skb_is_gso(skb)) 1790 return 0; 1791 1792 err = skb_cow_head(skb, 0); 1793 if (err < 0) 1794 return err; 1795 1796 if (protocol == htons(ETH_P_IP)) { 1797 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb); 1798 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb); 1799 iph->tot_len = 0; 1800 iph->check = 0; 1801 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 1802 0, IPPROTO_TCP, 0); 1803 } else if (skb_is_gso_v6(skb)) { 1804 1805 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb) 1806 : ipv6_hdr(skb); 1807 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb); 1808 ipv6h->payload_len = 0; 1809 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr, 1810 0, IPPROTO_TCP, 0); 1811 } 1812 1813 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb); 1814 *hdr_len = (skb->encapsulation 1815 ? (skb_inner_transport_header(skb) - skb->data) 1816 : skb_transport_offset(skb)) + l4len; 1817 1818 /* find the field values */ 1819 cd_cmd = I40E_TX_CTX_DESC_TSO; 1820 cd_tso_len = skb->len - *hdr_len; 1821 cd_mss = skb_shinfo(skb)->gso_size; 1822 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | 1823 ((u64)cd_tso_len << 1824 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | 1825 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT); 1826 return 1; 1827 } 1828 1829 /** 1830 * i40e_tsyn - set up the tsyn context descriptor 1831 * @tx_ring: ptr to the ring to send 1832 * @skb: ptr to the skb we're sending 1833 * @tx_flags: the collected send information 1834 * 1835 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen 1836 **/ 1837 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb, 1838 u32 tx_flags, u64 *cd_type_cmd_tso_mss) 1839 { 1840 struct i40e_pf *pf; 1841 1842 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) 1843 return 0; 1844 1845 /* Tx timestamps cannot be sampled when doing TSO */ 1846 if (tx_flags & I40E_TX_FLAGS_TSO) 1847 return 0; 1848 1849 /* only timestamp the outbound packet if the user has requested it and 1850 * we are not already transmitting a packet to be timestamped 1851 */ 1852 pf = i40e_netdev_to_pf(tx_ring->netdev); 1853 if (pf->ptp_tx && !pf->ptp_tx_skb) { 1854 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1855 pf->ptp_tx_skb = skb_get(skb); 1856 } else { 1857 return 0; 1858 } 1859 1860 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN << 1861 I40E_TXD_CTX_QW1_CMD_SHIFT; 1862 1863 return 1; 1864 } 1865 1866 /** 1867 * i40e_tx_enable_csum - Enable Tx checksum offloads 1868 * @skb: send buffer 1869 * @tx_flags: Tx flags currently set 1870 * @td_cmd: Tx descriptor command bits to set 1871 * @td_offset: Tx descriptor header offsets to set 1872 * @cd_tunneling: ptr to context desc bits 1873 **/ 1874 static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags, 1875 u32 *td_cmd, u32 *td_offset, 1876 struct i40e_ring *tx_ring, 1877 u32 *cd_tunneling) 1878 { 1879 struct ipv6hdr *this_ipv6_hdr; 1880 unsigned int this_tcp_hdrlen; 1881 struct iphdr *this_ip_hdr; 1882 u32 network_hdr_len; 1883 u8 l4_hdr = 0; 1884 1885 if (skb->encapsulation) { 1886 network_hdr_len = skb_inner_network_header_len(skb); 1887 this_ip_hdr = inner_ip_hdr(skb); 1888 this_ipv6_hdr = inner_ipv6_hdr(skb); 1889 this_tcp_hdrlen = inner_tcp_hdrlen(skb); 1890 1891 if (tx_flags & I40E_TX_FLAGS_IPV4) { 1892 1893 if (tx_flags & I40E_TX_FLAGS_TSO) { 1894 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4; 1895 ip_hdr(skb)->check = 0; 1896 } else { 1897 *cd_tunneling |= 1898 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 1899 } 1900 } else if (tx_flags & I40E_TX_FLAGS_IPV6) { 1901 if (tx_flags & I40E_TX_FLAGS_TSO) { 1902 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6; 1903 ip_hdr(skb)->check = 0; 1904 } else { 1905 *cd_tunneling |= 1906 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 1907 } 1908 } 1909 1910 /* Now set the ctx descriptor fields */ 1911 *cd_tunneling |= (skb_network_header_len(skb) >> 2) << 1912 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT | 1913 I40E_TXD_CTX_UDP_TUNNELING | 1914 ((skb_inner_network_offset(skb) - 1915 skb_transport_offset(skb)) >> 1) << 1916 I40E_TXD_CTX_QW0_NATLEN_SHIFT; 1917 1918 } else { 1919 network_hdr_len = skb_network_header_len(skb); 1920 this_ip_hdr = ip_hdr(skb); 1921 this_ipv6_hdr = ipv6_hdr(skb); 1922 this_tcp_hdrlen = tcp_hdrlen(skb); 1923 } 1924 1925 /* Enable IP checksum offloads */ 1926 if (tx_flags & I40E_TX_FLAGS_IPV4) { 1927 l4_hdr = this_ip_hdr->protocol; 1928 /* the stack computes the IP header already, the only time we 1929 * need the hardware to recompute it is in the case of TSO. 1930 */ 1931 if (tx_flags & I40E_TX_FLAGS_TSO) { 1932 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; 1933 this_ip_hdr->check = 0; 1934 } else { 1935 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; 1936 } 1937 /* Now set the td_offset for IP header length */ 1938 *td_offset = (network_hdr_len >> 2) << 1939 I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 1940 } else if (tx_flags & I40E_TX_FLAGS_IPV6) { 1941 l4_hdr = this_ipv6_hdr->nexthdr; 1942 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; 1943 /* Now set the td_offset for IP header length */ 1944 *td_offset = (network_hdr_len >> 2) << 1945 I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 1946 } 1947 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */ 1948 *td_offset |= (skb_network_offset(skb) >> 1) << 1949 I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 1950 1951 /* Enable L4 checksum offloads */ 1952 switch (l4_hdr) { 1953 case IPPROTO_TCP: 1954 /* enable checksum offloads */ 1955 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 1956 *td_offset |= (this_tcp_hdrlen >> 2) << 1957 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 1958 break; 1959 case IPPROTO_SCTP: 1960 /* enable SCTP checksum offload */ 1961 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; 1962 *td_offset |= (sizeof(struct sctphdr) >> 2) << 1963 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 1964 break; 1965 case IPPROTO_UDP: 1966 /* enable UDP checksum offload */ 1967 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; 1968 *td_offset |= (sizeof(struct udphdr) >> 2) << 1969 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 1970 break; 1971 default: 1972 break; 1973 } 1974 } 1975 1976 /** 1977 * i40e_create_tx_ctx Build the Tx context descriptor 1978 * @tx_ring: ring to create the descriptor on 1979 * @cd_type_cmd_tso_mss: Quad Word 1 1980 * @cd_tunneling: Quad Word 0 - bits 0-31 1981 * @cd_l2tag2: Quad Word 0 - bits 32-63 1982 **/ 1983 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring, 1984 const u64 cd_type_cmd_tso_mss, 1985 const u32 cd_tunneling, const u32 cd_l2tag2) 1986 { 1987 struct i40e_tx_context_desc *context_desc; 1988 int i = tx_ring->next_to_use; 1989 1990 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) && 1991 !cd_tunneling && !cd_l2tag2) 1992 return; 1993 1994 /* grab the next descriptor */ 1995 context_desc = I40E_TX_CTXTDESC(tx_ring, i); 1996 1997 i++; 1998 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 1999 2000 /* cpu_to_le32 and assign to struct fields */ 2001 context_desc->tunneling_params = cpu_to_le32(cd_tunneling); 2002 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2); 2003 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss); 2004 } 2005 2006 /** 2007 * i40e_tx_map - Build the Tx descriptor 2008 * @tx_ring: ring to send buffer on 2009 * @skb: send buffer 2010 * @first: first buffer info buffer to use 2011 * @tx_flags: collected send information 2012 * @hdr_len: size of the packet header 2013 * @td_cmd: the command field in the descriptor 2014 * @td_offset: offset for checksum or crc 2015 **/ 2016 static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb, 2017 struct i40e_tx_buffer *first, u32 tx_flags, 2018 const u8 hdr_len, u32 td_cmd, u32 td_offset) 2019 { 2020 unsigned int data_len = skb->data_len; 2021 unsigned int size = skb_headlen(skb); 2022 struct skb_frag_struct *frag; 2023 struct i40e_tx_buffer *tx_bi; 2024 struct i40e_tx_desc *tx_desc; 2025 u16 i = tx_ring->next_to_use; 2026 u32 td_tag = 0; 2027 dma_addr_t dma; 2028 u16 gso_segs; 2029 2030 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) { 2031 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; 2032 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >> 2033 I40E_TX_FLAGS_VLAN_SHIFT; 2034 } 2035 2036 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO)) 2037 gso_segs = skb_shinfo(skb)->gso_segs; 2038 else 2039 gso_segs = 1; 2040 2041 /* multiply data chunks by size of headers */ 2042 first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len); 2043 first->gso_segs = gso_segs; 2044 first->skb = skb; 2045 first->tx_flags = tx_flags; 2046 2047 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 2048 2049 tx_desc = I40E_TX_DESC(tx_ring, i); 2050 tx_bi = first; 2051 2052 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 2053 if (dma_mapping_error(tx_ring->dev, dma)) 2054 goto dma_error; 2055 2056 /* record length, and DMA address */ 2057 dma_unmap_len_set(tx_bi, len, size); 2058 dma_unmap_addr_set(tx_bi, dma, dma); 2059 2060 tx_desc->buffer_addr = cpu_to_le64(dma); 2061 2062 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) { 2063 tx_desc->cmd_type_offset_bsz = 2064 build_ctob(td_cmd, td_offset, 2065 I40E_MAX_DATA_PER_TXD, td_tag); 2066 2067 tx_desc++; 2068 i++; 2069 if (i == tx_ring->count) { 2070 tx_desc = I40E_TX_DESC(tx_ring, 0); 2071 i = 0; 2072 } 2073 2074 dma += I40E_MAX_DATA_PER_TXD; 2075 size -= I40E_MAX_DATA_PER_TXD; 2076 2077 tx_desc->buffer_addr = cpu_to_le64(dma); 2078 } 2079 2080 if (likely(!data_len)) 2081 break; 2082 2083 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, 2084 size, td_tag); 2085 2086 tx_desc++; 2087 i++; 2088 if (i == tx_ring->count) { 2089 tx_desc = I40E_TX_DESC(tx_ring, 0); 2090 i = 0; 2091 } 2092 2093 size = skb_frag_size(frag); 2094 data_len -= size; 2095 2096 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 2097 DMA_TO_DEVICE); 2098 2099 tx_bi = &tx_ring->tx_bi[i]; 2100 } 2101 2102 /* Place RS bit on last descriptor of any packet that spans across the 2103 * 4th descriptor (WB_STRIDE aka 0x3) in a 64B cacheline. 2104 */ 2105 #define WB_STRIDE 0x3 2106 if (((i & WB_STRIDE) != WB_STRIDE) && 2107 (first <= &tx_ring->tx_bi[i]) && 2108 (first >= &tx_ring->tx_bi[i & ~WB_STRIDE])) { 2109 tx_desc->cmd_type_offset_bsz = 2110 build_ctob(td_cmd, td_offset, size, td_tag) | 2111 cpu_to_le64((u64)I40E_TX_DESC_CMD_EOP << 2112 I40E_TXD_QW1_CMD_SHIFT); 2113 } else { 2114 tx_desc->cmd_type_offset_bsz = 2115 build_ctob(td_cmd, td_offset, size, td_tag) | 2116 cpu_to_le64((u64)I40E_TXD_CMD << 2117 I40E_TXD_QW1_CMD_SHIFT); 2118 } 2119 2120 netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev, 2121 tx_ring->queue_index), 2122 first->bytecount); 2123 2124 /* set the timestamp */ 2125 first->time_stamp = jiffies; 2126 2127 /* Force memory writes to complete before letting h/w 2128 * know there are new descriptors to fetch. (Only 2129 * applicable for weak-ordered memory model archs, 2130 * such as IA-64). 2131 */ 2132 wmb(); 2133 2134 /* set next_to_watch value indicating a packet is present */ 2135 first->next_to_watch = tx_desc; 2136 2137 i++; 2138 if (i == tx_ring->count) 2139 i = 0; 2140 2141 tx_ring->next_to_use = i; 2142 2143 /* notify HW of packet */ 2144 writel(i, tx_ring->tail); 2145 2146 return; 2147 2148 dma_error: 2149 dev_info(tx_ring->dev, "TX DMA map failed\n"); 2150 2151 /* clear dma mappings for failed tx_bi map */ 2152 for (;;) { 2153 tx_bi = &tx_ring->tx_bi[i]; 2154 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi); 2155 if (tx_bi == first) 2156 break; 2157 if (i == 0) 2158 i = tx_ring->count; 2159 i--; 2160 } 2161 2162 tx_ring->next_to_use = i; 2163 } 2164 2165 /** 2166 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions 2167 * @tx_ring: the ring to be checked 2168 * @size: the size buffer we want to assure is available 2169 * 2170 * Returns -EBUSY if a stop is needed, else 0 2171 **/ 2172 static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) 2173 { 2174 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 2175 /* Memory barrier before checking head and tail */ 2176 smp_mb(); 2177 2178 /* Check again in a case another CPU has just made room available. */ 2179 if (likely(I40E_DESC_UNUSED(tx_ring) < size)) 2180 return -EBUSY; 2181 2182 /* A reprieve! - use start_queue because it doesn't call schedule */ 2183 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 2184 ++tx_ring->tx_stats.restart_queue; 2185 return 0; 2186 } 2187 2188 /** 2189 * i40e_maybe_stop_tx - 1st level check for tx stop conditions 2190 * @tx_ring: the ring to be checked 2191 * @size: the size buffer we want to assure is available 2192 * 2193 * Returns 0 if stop is not needed 2194 **/ 2195 static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) 2196 { 2197 if (likely(I40E_DESC_UNUSED(tx_ring) >= size)) 2198 return 0; 2199 return __i40e_maybe_stop_tx(tx_ring, size); 2200 } 2201 2202 /** 2203 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed 2204 * @skb: send buffer 2205 * @tx_ring: ring to send buffer on 2206 * 2207 * Returns number of data descriptors needed for this skb. Returns 0 to indicate 2208 * there is not enough descriptors available in this ring since we need at least 2209 * one descriptor. 2210 **/ 2211 static int i40e_xmit_descriptor_count(struct sk_buff *skb, 2212 struct i40e_ring *tx_ring) 2213 { 2214 unsigned int f; 2215 int count = 0; 2216 2217 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, 2218 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, 2219 * + 4 desc gap to avoid the cache line where head is, 2220 * + 1 desc for context descriptor, 2221 * otherwise try next time 2222 */ 2223 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 2224 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 2225 2226 count += TXD_USE_COUNT(skb_headlen(skb)); 2227 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) { 2228 tx_ring->tx_stats.tx_busy++; 2229 return 0; 2230 } 2231 return count; 2232 } 2233 2234 /** 2235 * i40e_xmit_frame_ring - Sends buffer on Tx ring 2236 * @skb: send buffer 2237 * @tx_ring: ring to send buffer on 2238 * 2239 * Returns NETDEV_TX_OK if sent, else an error code 2240 **/ 2241 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, 2242 struct i40e_ring *tx_ring) 2243 { 2244 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT; 2245 u32 cd_tunneling = 0, cd_l2tag2 = 0; 2246 struct i40e_tx_buffer *first; 2247 u32 td_offset = 0; 2248 u32 tx_flags = 0; 2249 __be16 protocol; 2250 u32 td_cmd = 0; 2251 u8 hdr_len = 0; 2252 int tsyn; 2253 int tso; 2254 if (0 == i40e_xmit_descriptor_count(skb, tx_ring)) 2255 return NETDEV_TX_BUSY; 2256 2257 /* prepare the xmit flags */ 2258 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags)) 2259 goto out_drop; 2260 2261 /* obtain protocol of skb */ 2262 protocol = skb->protocol; 2263 2264 /* record the location of the first descriptor for this packet */ 2265 first = &tx_ring->tx_bi[tx_ring->next_to_use]; 2266 2267 /* setup IPv4/IPv6 offloads */ 2268 if (protocol == htons(ETH_P_IP)) 2269 tx_flags |= I40E_TX_FLAGS_IPV4; 2270 else if (protocol == htons(ETH_P_IPV6)) 2271 tx_flags |= I40E_TX_FLAGS_IPV6; 2272 2273 tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len, 2274 &cd_type_cmd_tso_mss, &cd_tunneling); 2275 2276 if (tso < 0) 2277 goto out_drop; 2278 else if (tso) 2279 tx_flags |= I40E_TX_FLAGS_TSO; 2280 2281 skb_tx_timestamp(skb); 2282 2283 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss); 2284 2285 if (tsyn) 2286 tx_flags |= I40E_TX_FLAGS_TSYN; 2287 2288 /* always enable CRC insertion offload */ 2289 td_cmd |= I40E_TX_DESC_CMD_ICRC; 2290 2291 /* Always offload the checksum, since it's in the data descriptor */ 2292 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2293 tx_flags |= I40E_TX_FLAGS_CSUM; 2294 2295 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset, 2296 tx_ring, &cd_tunneling); 2297 } 2298 2299 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss, 2300 cd_tunneling, cd_l2tag2); 2301 2302 /* Add Flow Director ATR if it's enabled. 2303 * 2304 * NOTE: this must always be directly before the data descriptor. 2305 */ 2306 i40e_atr(tx_ring, skb, tx_flags, protocol); 2307 2308 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len, 2309 td_cmd, td_offset); 2310 2311 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED); 2312 2313 return NETDEV_TX_OK; 2314 2315 out_drop: 2316 dev_kfree_skb_any(skb); 2317 return NETDEV_TX_OK; 2318 } 2319 2320 /** 2321 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer 2322 * @skb: send buffer 2323 * @netdev: network interface device structure 2324 * 2325 * Returns NETDEV_TX_OK if sent, else an error code 2326 **/ 2327 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 2328 { 2329 struct i40e_netdev_priv *np = netdev_priv(netdev); 2330 struct i40e_vsi *vsi = np->vsi; 2331 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping]; 2332 2333 /* hardware can't handle really short frames, hardware padding works 2334 * beyond this point 2335 */ 2336 if (unlikely(skb->len < I40E_MIN_TX_LEN)) { 2337 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len)) 2338 return NETDEV_TX_OK; 2339 skb->len = I40E_MIN_TX_LEN; 2340 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN); 2341 } 2342 2343 return i40e_xmit_frame_ring(skb, tx_ring); 2344 } 2345