1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2016 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/prefetch.h> 28 #include <net/busy_poll.h> 29 #include <linux/bpf_trace.h> 30 #include "i40e.h" 31 #include "i40e_trace.h" 32 #include "i40e_prototype.h" 33 34 static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size, 35 u32 td_tag) 36 { 37 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA | 38 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | 39 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | 40 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | 41 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); 42 } 43 44 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) 45 /** 46 * i40e_fdir - Generate a Flow Director descriptor based on fdata 47 * @tx_ring: Tx ring to send buffer on 48 * @fdata: Flow director filter data 49 * @add: Indicate if we are adding a rule or deleting one 50 * 51 **/ 52 static void i40e_fdir(struct i40e_ring *tx_ring, 53 struct i40e_fdir_filter *fdata, bool add) 54 { 55 struct i40e_filter_program_desc *fdir_desc; 56 struct i40e_pf *pf = tx_ring->vsi->back; 57 u32 flex_ptype, dtype_cmd; 58 u16 i; 59 60 /* grab the next descriptor */ 61 i = tx_ring->next_to_use; 62 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i); 63 64 i++; 65 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 66 67 flex_ptype = I40E_TXD_FLTR_QW0_QINDEX_MASK & 68 (fdata->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT); 69 70 flex_ptype |= I40E_TXD_FLTR_QW0_FLEXOFF_MASK & 71 (fdata->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT); 72 73 flex_ptype |= I40E_TXD_FLTR_QW0_PCTYPE_MASK & 74 (fdata->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 75 76 flex_ptype |= I40E_TXD_FLTR_QW0_PCTYPE_MASK & 77 (fdata->flex_offset << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT); 78 79 /* Use LAN VSI Id if not programmed by user */ 80 flex_ptype |= I40E_TXD_FLTR_QW0_DEST_VSI_MASK & 81 ((u32)(fdata->dest_vsi ? : pf->vsi[pf->lan_vsi]->id) << 82 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT); 83 84 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG; 85 86 dtype_cmd |= add ? 87 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 88 I40E_TXD_FLTR_QW1_PCMD_SHIFT : 89 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 90 I40E_TXD_FLTR_QW1_PCMD_SHIFT; 91 92 dtype_cmd |= I40E_TXD_FLTR_QW1_DEST_MASK & 93 (fdata->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT); 94 95 dtype_cmd |= I40E_TXD_FLTR_QW1_FD_STATUS_MASK & 96 (fdata->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT); 97 98 if (fdata->cnt_index) { 99 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK; 100 dtype_cmd |= I40E_TXD_FLTR_QW1_CNTINDEX_MASK & 101 ((u32)fdata->cnt_index << 102 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT); 103 } 104 105 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype); 106 fdir_desc->rsvd = cpu_to_le32(0); 107 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd); 108 fdir_desc->fd_id = cpu_to_le32(fdata->fd_id); 109 } 110 111 #define I40E_FD_CLEAN_DELAY 10 112 /** 113 * i40e_program_fdir_filter - Program a Flow Director filter 114 * @fdir_data: Packet data that will be filter parameters 115 * @raw_packet: the pre-allocated packet buffer for FDir 116 * @pf: The PF pointer 117 * @add: True for add/update, False for remove 118 **/ 119 static int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, 120 u8 *raw_packet, struct i40e_pf *pf, 121 bool add) 122 { 123 struct i40e_tx_buffer *tx_buf, *first; 124 struct i40e_tx_desc *tx_desc; 125 struct i40e_ring *tx_ring; 126 struct i40e_vsi *vsi; 127 struct device *dev; 128 dma_addr_t dma; 129 u32 td_cmd = 0; 130 u16 i; 131 132 /* find existing FDIR VSI */ 133 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 134 if (!vsi) 135 return -ENOENT; 136 137 tx_ring = vsi->tx_rings[0]; 138 dev = tx_ring->dev; 139 140 /* we need two descriptors to add/del a filter and we can wait */ 141 for (i = I40E_FD_CLEAN_DELAY; I40E_DESC_UNUSED(tx_ring) < 2; i--) { 142 if (!i) 143 return -EAGAIN; 144 msleep_interruptible(1); 145 } 146 147 dma = dma_map_single(dev, raw_packet, 148 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE); 149 if (dma_mapping_error(dev, dma)) 150 goto dma_fail; 151 152 /* grab the next descriptor */ 153 i = tx_ring->next_to_use; 154 first = &tx_ring->tx_bi[i]; 155 i40e_fdir(tx_ring, fdir_data, add); 156 157 /* Now program a dummy descriptor */ 158 i = tx_ring->next_to_use; 159 tx_desc = I40E_TX_DESC(tx_ring, i); 160 tx_buf = &tx_ring->tx_bi[i]; 161 162 tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0; 163 164 memset(tx_buf, 0, sizeof(struct i40e_tx_buffer)); 165 166 /* record length, and DMA address */ 167 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE); 168 dma_unmap_addr_set(tx_buf, dma, dma); 169 170 tx_desc->buffer_addr = cpu_to_le64(dma); 171 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY; 172 173 tx_buf->tx_flags = I40E_TX_FLAGS_FD_SB; 174 tx_buf->raw_buf = (void *)raw_packet; 175 176 tx_desc->cmd_type_offset_bsz = 177 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0); 178 179 /* Force memory writes to complete before letting h/w 180 * know there are new descriptors to fetch. 181 */ 182 wmb(); 183 184 /* Mark the data descriptor to be watched */ 185 first->next_to_watch = tx_desc; 186 187 writel(tx_ring->next_to_use, tx_ring->tail); 188 return 0; 189 190 dma_fail: 191 return -1; 192 } 193 194 #define IP_HEADER_OFFSET 14 195 #define I40E_UDPIP_DUMMY_PACKET_LEN 42 196 /** 197 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters 198 * @vsi: pointer to the targeted VSI 199 * @fd_data: the flow director data required for the FDir descriptor 200 * @add: true adds a filter, false removes it 201 * 202 * Returns 0 if the filters were successfully added or removed 203 **/ 204 static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi, 205 struct i40e_fdir_filter *fd_data, 206 bool add) 207 { 208 struct i40e_pf *pf = vsi->back; 209 struct udphdr *udp; 210 struct iphdr *ip; 211 u8 *raw_packet; 212 int ret; 213 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 214 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0, 215 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 216 217 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 218 if (!raw_packet) 219 return -ENOMEM; 220 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN); 221 222 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 223 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET 224 + sizeof(struct iphdr)); 225 226 ip->daddr = fd_data->dst_ip; 227 udp->dest = fd_data->dst_port; 228 ip->saddr = fd_data->src_ip; 229 udp->source = fd_data->src_port; 230 231 if (fd_data->flex_filter) { 232 u8 *payload = raw_packet + I40E_UDPIP_DUMMY_PACKET_LEN; 233 __be16 pattern = fd_data->flex_word; 234 u16 off = fd_data->flex_offset; 235 236 *((__force __be16 *)(payload + off)) = pattern; 237 } 238 239 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; 240 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 241 if (ret) { 242 dev_info(&pf->pdev->dev, 243 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 244 fd_data->pctype, fd_data->fd_id, ret); 245 /* Free the packet buffer since it wasn't added to the ring */ 246 kfree(raw_packet); 247 return -EOPNOTSUPP; 248 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 249 if (add) 250 dev_info(&pf->pdev->dev, 251 "Filter OK for PCTYPE %d loc = %d\n", 252 fd_data->pctype, fd_data->fd_id); 253 else 254 dev_info(&pf->pdev->dev, 255 "Filter deleted for PCTYPE %d loc = %d\n", 256 fd_data->pctype, fd_data->fd_id); 257 } 258 259 if (add) 260 pf->fd_udp4_filter_cnt++; 261 else 262 pf->fd_udp4_filter_cnt--; 263 264 return 0; 265 } 266 267 #define I40E_TCPIP_DUMMY_PACKET_LEN 54 268 /** 269 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters 270 * @vsi: pointer to the targeted VSI 271 * @fd_data: the flow director data required for the FDir descriptor 272 * @add: true adds a filter, false removes it 273 * 274 * Returns 0 if the filters were successfully added or removed 275 **/ 276 static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi, 277 struct i40e_fdir_filter *fd_data, 278 bool add) 279 { 280 struct i40e_pf *pf = vsi->back; 281 struct tcphdr *tcp; 282 struct iphdr *ip; 283 u8 *raw_packet; 284 int ret; 285 /* Dummy packet */ 286 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 287 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0, 288 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11, 289 0x0, 0x72, 0, 0, 0, 0}; 290 291 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 292 if (!raw_packet) 293 return -ENOMEM; 294 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN); 295 296 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 297 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET 298 + sizeof(struct iphdr)); 299 300 ip->daddr = fd_data->dst_ip; 301 tcp->dest = fd_data->dst_port; 302 ip->saddr = fd_data->src_ip; 303 tcp->source = fd_data->src_port; 304 305 if (fd_data->flex_filter) { 306 u8 *payload = raw_packet + I40E_TCPIP_DUMMY_PACKET_LEN; 307 __be16 pattern = fd_data->flex_word; 308 u16 off = fd_data->flex_offset; 309 310 *((__force __be16 *)(payload + off)) = pattern; 311 } 312 313 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP; 314 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 315 if (ret) { 316 dev_info(&pf->pdev->dev, 317 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 318 fd_data->pctype, fd_data->fd_id, ret); 319 /* Free the packet buffer since it wasn't added to the ring */ 320 kfree(raw_packet); 321 return -EOPNOTSUPP; 322 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 323 if (add) 324 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d loc = %d)\n", 325 fd_data->pctype, fd_data->fd_id); 326 else 327 dev_info(&pf->pdev->dev, 328 "Filter deleted for PCTYPE %d loc = %d\n", 329 fd_data->pctype, fd_data->fd_id); 330 } 331 332 if (add) { 333 pf->fd_tcp4_filter_cnt++; 334 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 335 I40E_DEBUG_FD & pf->hw.debug_mask) 336 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n"); 337 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED; 338 } else { 339 pf->fd_tcp4_filter_cnt--; 340 } 341 342 return 0; 343 } 344 345 #define I40E_SCTPIP_DUMMY_PACKET_LEN 46 346 /** 347 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for 348 * a specific flow spec 349 * @vsi: pointer to the targeted VSI 350 * @fd_data: the flow director data required for the FDir descriptor 351 * @add: true adds a filter, false removes it 352 * 353 * Returns 0 if the filters were successfully added or removed 354 **/ 355 static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi, 356 struct i40e_fdir_filter *fd_data, 357 bool add) 358 { 359 struct i40e_pf *pf = vsi->back; 360 struct sctphdr *sctp; 361 struct iphdr *ip; 362 u8 *raw_packet; 363 int ret; 364 /* Dummy packet */ 365 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 366 0x45, 0, 0, 0x20, 0, 0, 0x40, 0, 0x40, 0x84, 0, 0, 0, 0, 0, 0, 367 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 368 369 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 370 if (!raw_packet) 371 return -ENOMEM; 372 memcpy(raw_packet, packet, I40E_SCTPIP_DUMMY_PACKET_LEN); 373 374 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 375 sctp = (struct sctphdr *)(raw_packet + IP_HEADER_OFFSET 376 + sizeof(struct iphdr)); 377 378 ip->daddr = fd_data->dst_ip; 379 sctp->dest = fd_data->dst_port; 380 ip->saddr = fd_data->src_ip; 381 sctp->source = fd_data->src_port; 382 383 if (fd_data->flex_filter) { 384 u8 *payload = raw_packet + I40E_SCTPIP_DUMMY_PACKET_LEN; 385 __be16 pattern = fd_data->flex_word; 386 u16 off = fd_data->flex_offset; 387 388 *((__force __be16 *)(payload + off)) = pattern; 389 } 390 391 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_SCTP; 392 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 393 if (ret) { 394 dev_info(&pf->pdev->dev, 395 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 396 fd_data->pctype, fd_data->fd_id, ret); 397 /* Free the packet buffer since it wasn't added to the ring */ 398 kfree(raw_packet); 399 return -EOPNOTSUPP; 400 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 401 if (add) 402 dev_info(&pf->pdev->dev, 403 "Filter OK for PCTYPE %d loc = %d\n", 404 fd_data->pctype, fd_data->fd_id); 405 else 406 dev_info(&pf->pdev->dev, 407 "Filter deleted for PCTYPE %d loc = %d\n", 408 fd_data->pctype, fd_data->fd_id); 409 } 410 411 if (add) 412 pf->fd_sctp4_filter_cnt++; 413 else 414 pf->fd_sctp4_filter_cnt--; 415 416 return 0; 417 } 418 419 #define I40E_IP_DUMMY_PACKET_LEN 34 420 /** 421 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for 422 * a specific flow spec 423 * @vsi: pointer to the targeted VSI 424 * @fd_data: the flow director data required for the FDir descriptor 425 * @add: true adds a filter, false removes it 426 * 427 * Returns 0 if the filters were successfully added or removed 428 **/ 429 static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi, 430 struct i40e_fdir_filter *fd_data, 431 bool add) 432 { 433 struct i40e_pf *pf = vsi->back; 434 struct iphdr *ip; 435 u8 *raw_packet; 436 int ret; 437 int i; 438 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 439 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0, 440 0, 0, 0, 0}; 441 442 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER; 443 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) { 444 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 445 if (!raw_packet) 446 return -ENOMEM; 447 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN); 448 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 449 450 ip->saddr = fd_data->src_ip; 451 ip->daddr = fd_data->dst_ip; 452 ip->protocol = 0; 453 454 if (fd_data->flex_filter) { 455 u8 *payload = raw_packet + I40E_IP_DUMMY_PACKET_LEN; 456 __be16 pattern = fd_data->flex_word; 457 u16 off = fd_data->flex_offset; 458 459 *((__force __be16 *)(payload + off)) = pattern; 460 } 461 462 fd_data->pctype = i; 463 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 464 if (ret) { 465 dev_info(&pf->pdev->dev, 466 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 467 fd_data->pctype, fd_data->fd_id, ret); 468 /* The packet buffer wasn't added to the ring so we 469 * need to free it now. 470 */ 471 kfree(raw_packet); 472 return -EOPNOTSUPP; 473 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 474 if (add) 475 dev_info(&pf->pdev->dev, 476 "Filter OK for PCTYPE %d loc = %d\n", 477 fd_data->pctype, fd_data->fd_id); 478 else 479 dev_info(&pf->pdev->dev, 480 "Filter deleted for PCTYPE %d loc = %d\n", 481 fd_data->pctype, fd_data->fd_id); 482 } 483 } 484 485 if (add) 486 pf->fd_ip4_filter_cnt++; 487 else 488 pf->fd_ip4_filter_cnt--; 489 490 return 0; 491 } 492 493 /** 494 * i40e_add_del_fdir - Build raw packets to add/del fdir filter 495 * @vsi: pointer to the targeted VSI 496 * @cmd: command to get or set RX flow classification rules 497 * @add: true adds a filter, false removes it 498 * 499 **/ 500 int i40e_add_del_fdir(struct i40e_vsi *vsi, 501 struct i40e_fdir_filter *input, bool add) 502 { 503 struct i40e_pf *pf = vsi->back; 504 int ret; 505 506 switch (input->flow_type & ~FLOW_EXT) { 507 case TCP_V4_FLOW: 508 ret = i40e_add_del_fdir_tcpv4(vsi, input, add); 509 break; 510 case UDP_V4_FLOW: 511 ret = i40e_add_del_fdir_udpv4(vsi, input, add); 512 break; 513 case SCTP_V4_FLOW: 514 ret = i40e_add_del_fdir_sctpv4(vsi, input, add); 515 break; 516 case IP_USER_FLOW: 517 switch (input->ip4_proto) { 518 case IPPROTO_TCP: 519 ret = i40e_add_del_fdir_tcpv4(vsi, input, add); 520 break; 521 case IPPROTO_UDP: 522 ret = i40e_add_del_fdir_udpv4(vsi, input, add); 523 break; 524 case IPPROTO_SCTP: 525 ret = i40e_add_del_fdir_sctpv4(vsi, input, add); 526 break; 527 case IPPROTO_IP: 528 ret = i40e_add_del_fdir_ipv4(vsi, input, add); 529 break; 530 default: 531 /* We cannot support masking based on protocol */ 532 dev_info(&pf->pdev->dev, "Unsupported IPv4 protocol 0x%02x\n", 533 input->ip4_proto); 534 return -EINVAL; 535 } 536 break; 537 default: 538 dev_info(&pf->pdev->dev, "Unsupported flow type 0x%02x\n", 539 input->flow_type); 540 return -EINVAL; 541 } 542 543 /* The buffer allocated here will be normally be freed by 544 * i40e_clean_fdir_tx_irq() as it reclaims resources after transmit 545 * completion. In the event of an error adding the buffer to the FDIR 546 * ring, it will immediately be freed. It may also be freed by 547 * i40e_clean_tx_ring() when closing the VSI. 548 */ 549 return ret; 550 } 551 552 /** 553 * i40e_fd_handle_status - check the Programming Status for FD 554 * @rx_ring: the Rx ring for this descriptor 555 * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor. 556 * @prog_id: the id originally used for programming 557 * 558 * This is used to verify if the FD programming or invalidation 559 * requested by SW to the HW is successful or not and take actions accordingly. 560 **/ 561 static void i40e_fd_handle_status(struct i40e_ring *rx_ring, 562 union i40e_rx_desc *rx_desc, u8 prog_id) 563 { 564 struct i40e_pf *pf = rx_ring->vsi->back; 565 struct pci_dev *pdev = pf->pdev; 566 u32 fcnt_prog, fcnt_avail; 567 u32 error; 568 u64 qw; 569 570 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 571 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> 572 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT; 573 574 if (error == BIT(I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) { 575 pf->fd_inv = le32_to_cpu(rx_desc->wb.qword0.hi_dword.fd_id); 576 if ((rx_desc->wb.qword0.hi_dword.fd_id != 0) || 577 (I40E_DEBUG_FD & pf->hw.debug_mask)) 578 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n", 579 pf->fd_inv); 580 581 /* Check if the programming error is for ATR. 582 * If so, auto disable ATR and set a state for 583 * flush in progress. Next time we come here if flush is in 584 * progress do nothing, once flush is complete the state will 585 * be cleared. 586 */ 587 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 588 return; 589 590 pf->fd_add_err++; 591 /* store the current atr filter count */ 592 pf->fd_atr_cnt = i40e_get_current_atr_cnt(pf); 593 594 if ((rx_desc->wb.qword0.hi_dword.fd_id == 0) && 595 pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) { 596 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED; 597 set_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 598 } 599 600 /* filter programming failed most likely due to table full */ 601 fcnt_prog = i40e_get_global_fd_count(pf); 602 fcnt_avail = pf->fdir_pf_filter_count; 603 /* If ATR is running fcnt_prog can quickly change, 604 * if we are very close to full, it makes sense to disable 605 * FD ATR/SB and then re-enable it when there is room. 606 */ 607 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) { 608 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 609 !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED)) { 610 pf->flags |= I40E_FLAG_FD_SB_AUTO_DISABLED; 611 if (I40E_DEBUG_FD & pf->hw.debug_mask) 612 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n"); 613 } 614 } 615 } else if (error == BIT(I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) { 616 if (I40E_DEBUG_FD & pf->hw.debug_mask) 617 dev_info(&pdev->dev, "ntuple filter fd_id = %d, could not be removed\n", 618 rx_desc->wb.qword0.hi_dword.fd_id); 619 } 620 } 621 622 /** 623 * i40e_unmap_and_free_tx_resource - Release a Tx buffer 624 * @ring: the ring that owns the buffer 625 * @tx_buffer: the buffer to free 626 **/ 627 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring, 628 struct i40e_tx_buffer *tx_buffer) 629 { 630 if (tx_buffer->skb) { 631 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB) 632 kfree(tx_buffer->raw_buf); 633 else if (ring_is_xdp(ring)) 634 page_frag_free(tx_buffer->raw_buf); 635 else 636 dev_kfree_skb_any(tx_buffer->skb); 637 if (dma_unmap_len(tx_buffer, len)) 638 dma_unmap_single(ring->dev, 639 dma_unmap_addr(tx_buffer, dma), 640 dma_unmap_len(tx_buffer, len), 641 DMA_TO_DEVICE); 642 } else if (dma_unmap_len(tx_buffer, len)) { 643 dma_unmap_page(ring->dev, 644 dma_unmap_addr(tx_buffer, dma), 645 dma_unmap_len(tx_buffer, len), 646 DMA_TO_DEVICE); 647 } 648 649 tx_buffer->next_to_watch = NULL; 650 tx_buffer->skb = NULL; 651 dma_unmap_len_set(tx_buffer, len, 0); 652 /* tx_buffer must be completely set up in the transmit path */ 653 } 654 655 /** 656 * i40e_clean_tx_ring - Free any empty Tx buffers 657 * @tx_ring: ring to be cleaned 658 **/ 659 void i40e_clean_tx_ring(struct i40e_ring *tx_ring) 660 { 661 unsigned long bi_size; 662 u16 i; 663 664 /* ring already cleared, nothing to do */ 665 if (!tx_ring->tx_bi) 666 return; 667 668 /* Free all the Tx ring sk_buffs */ 669 for (i = 0; i < tx_ring->count; i++) 670 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]); 671 672 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 673 memset(tx_ring->tx_bi, 0, bi_size); 674 675 /* Zero out the descriptor ring */ 676 memset(tx_ring->desc, 0, tx_ring->size); 677 678 tx_ring->next_to_use = 0; 679 tx_ring->next_to_clean = 0; 680 681 if (!tx_ring->netdev) 682 return; 683 684 /* cleanup Tx queue statistics */ 685 netdev_tx_reset_queue(txring_txq(tx_ring)); 686 } 687 688 /** 689 * i40e_free_tx_resources - Free Tx resources per queue 690 * @tx_ring: Tx descriptor ring for a specific queue 691 * 692 * Free all transmit software resources 693 **/ 694 void i40e_free_tx_resources(struct i40e_ring *tx_ring) 695 { 696 i40e_clean_tx_ring(tx_ring); 697 kfree(tx_ring->tx_bi); 698 tx_ring->tx_bi = NULL; 699 700 if (tx_ring->desc) { 701 dma_free_coherent(tx_ring->dev, tx_ring->size, 702 tx_ring->desc, tx_ring->dma); 703 tx_ring->desc = NULL; 704 } 705 } 706 707 /** 708 * i40e_get_tx_pending - how many tx descriptors not processed 709 * @tx_ring: the ring of descriptors 710 * 711 * Since there is no access to the ring head register 712 * in XL710, we need to use our local copies 713 **/ 714 u32 i40e_get_tx_pending(struct i40e_ring *ring) 715 { 716 u32 head, tail; 717 718 head = i40e_get_head(ring); 719 tail = readl(ring->tail); 720 721 if (head != tail) 722 return (head < tail) ? 723 tail - head : (tail + ring->count - head); 724 725 return 0; 726 } 727 728 #define WB_STRIDE 4 729 730 /** 731 * i40e_clean_tx_irq - Reclaim resources after transmit completes 732 * @vsi: the VSI we care about 733 * @tx_ring: Tx ring to clean 734 * @napi_budget: Used to determine if we are in netpoll 735 * 736 * Returns true if there's any budget left (e.g. the clean is finished) 737 **/ 738 static bool i40e_clean_tx_irq(struct i40e_vsi *vsi, 739 struct i40e_ring *tx_ring, int napi_budget) 740 { 741 u16 i = tx_ring->next_to_clean; 742 struct i40e_tx_buffer *tx_buf; 743 struct i40e_tx_desc *tx_head; 744 struct i40e_tx_desc *tx_desc; 745 unsigned int total_bytes = 0, total_packets = 0; 746 unsigned int budget = vsi->work_limit; 747 748 tx_buf = &tx_ring->tx_bi[i]; 749 tx_desc = I40E_TX_DESC(tx_ring, i); 750 i -= tx_ring->count; 751 752 tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring)); 753 754 do { 755 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 756 757 /* if next_to_watch is not set then there is no work pending */ 758 if (!eop_desc) 759 break; 760 761 /* prevent any other reads prior to eop_desc */ 762 read_barrier_depends(); 763 764 i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf); 765 /* we have caught up to head, no work left to do */ 766 if (tx_head == tx_desc) 767 break; 768 769 /* clear next_to_watch to prevent false hangs */ 770 tx_buf->next_to_watch = NULL; 771 772 /* update the statistics for this packet */ 773 total_bytes += tx_buf->bytecount; 774 total_packets += tx_buf->gso_segs; 775 776 /* free the skb/XDP data */ 777 if (ring_is_xdp(tx_ring)) 778 page_frag_free(tx_buf->raw_buf); 779 else 780 napi_consume_skb(tx_buf->skb, napi_budget); 781 782 /* unmap skb header data */ 783 dma_unmap_single(tx_ring->dev, 784 dma_unmap_addr(tx_buf, dma), 785 dma_unmap_len(tx_buf, len), 786 DMA_TO_DEVICE); 787 788 /* clear tx_buffer data */ 789 tx_buf->skb = NULL; 790 dma_unmap_len_set(tx_buf, len, 0); 791 792 /* unmap remaining buffers */ 793 while (tx_desc != eop_desc) { 794 i40e_trace(clean_tx_irq_unmap, 795 tx_ring, tx_desc, tx_buf); 796 797 tx_buf++; 798 tx_desc++; 799 i++; 800 if (unlikely(!i)) { 801 i -= tx_ring->count; 802 tx_buf = tx_ring->tx_bi; 803 tx_desc = I40E_TX_DESC(tx_ring, 0); 804 } 805 806 /* unmap any remaining paged data */ 807 if (dma_unmap_len(tx_buf, len)) { 808 dma_unmap_page(tx_ring->dev, 809 dma_unmap_addr(tx_buf, dma), 810 dma_unmap_len(tx_buf, len), 811 DMA_TO_DEVICE); 812 dma_unmap_len_set(tx_buf, len, 0); 813 } 814 } 815 816 /* move us one more past the eop_desc for start of next pkt */ 817 tx_buf++; 818 tx_desc++; 819 i++; 820 if (unlikely(!i)) { 821 i -= tx_ring->count; 822 tx_buf = tx_ring->tx_bi; 823 tx_desc = I40E_TX_DESC(tx_ring, 0); 824 } 825 826 prefetch(tx_desc); 827 828 /* update budget accounting */ 829 budget--; 830 } while (likely(budget)); 831 832 i += tx_ring->count; 833 tx_ring->next_to_clean = i; 834 u64_stats_update_begin(&tx_ring->syncp); 835 tx_ring->stats.bytes += total_bytes; 836 tx_ring->stats.packets += total_packets; 837 u64_stats_update_end(&tx_ring->syncp); 838 tx_ring->q_vector->tx.total_bytes += total_bytes; 839 tx_ring->q_vector->tx.total_packets += total_packets; 840 841 if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) { 842 /* check to see if there are < 4 descriptors 843 * waiting to be written back, then kick the hardware to force 844 * them to be written back in case we stay in NAPI. 845 * In this mode on X722 we do not enable Interrupt. 846 */ 847 unsigned int j = i40e_get_tx_pending(tx_ring); 848 849 if (budget && 850 ((j / WB_STRIDE) == 0) && (j > 0) && 851 !test_bit(__I40E_VSI_DOWN, vsi->state) && 852 (I40E_DESC_UNUSED(tx_ring) != tx_ring->count)) 853 tx_ring->arm_wb = true; 854 } 855 856 if (ring_is_xdp(tx_ring)) 857 return !!budget; 858 859 /* notify netdev of completed buffers */ 860 netdev_tx_completed_queue(txring_txq(tx_ring), 861 total_packets, total_bytes); 862 863 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 864 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 865 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 866 /* Make sure that anybody stopping the queue after this 867 * sees the new next_to_clean. 868 */ 869 smp_mb(); 870 if (__netif_subqueue_stopped(tx_ring->netdev, 871 tx_ring->queue_index) && 872 !test_bit(__I40E_VSI_DOWN, vsi->state)) { 873 netif_wake_subqueue(tx_ring->netdev, 874 tx_ring->queue_index); 875 ++tx_ring->tx_stats.restart_queue; 876 } 877 } 878 879 return !!budget; 880 } 881 882 /** 883 * i40e_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled 884 * @vsi: the VSI we care about 885 * @q_vector: the vector on which to enable writeback 886 * 887 **/ 888 static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi, 889 struct i40e_q_vector *q_vector) 890 { 891 u16 flags = q_vector->tx.ring[0].flags; 892 u32 val; 893 894 if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR)) 895 return; 896 897 if (q_vector->arm_wb_state) 898 return; 899 900 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 901 val = I40E_PFINT_DYN_CTLN_WB_ON_ITR_MASK | 902 I40E_PFINT_DYN_CTLN_ITR_INDX_MASK; /* set noitr */ 903 904 wr32(&vsi->back->hw, 905 I40E_PFINT_DYN_CTLN(q_vector->v_idx + vsi->base_vector - 1), 906 val); 907 } else { 908 val = I40E_PFINT_DYN_CTL0_WB_ON_ITR_MASK | 909 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK; /* set noitr */ 910 911 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val); 912 } 913 q_vector->arm_wb_state = true; 914 } 915 916 /** 917 * i40e_force_wb - Issue SW Interrupt so HW does a wb 918 * @vsi: the VSI we care about 919 * @q_vector: the vector on which to force writeback 920 * 921 **/ 922 void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector) 923 { 924 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 925 u32 val = I40E_PFINT_DYN_CTLN_INTENA_MASK | 926 I40E_PFINT_DYN_CTLN_ITR_INDX_MASK | /* set noitr */ 927 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK | 928 I40E_PFINT_DYN_CTLN_SW_ITR_INDX_ENA_MASK; 929 /* allow 00 to be written to the index */ 930 931 wr32(&vsi->back->hw, 932 I40E_PFINT_DYN_CTLN(q_vector->v_idx + 933 vsi->base_vector - 1), val); 934 } else { 935 u32 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 936 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK | /* set noitr */ 937 I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK | 938 I40E_PFINT_DYN_CTL0_SW_ITR_INDX_ENA_MASK; 939 /* allow 00 to be written to the index */ 940 941 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val); 942 } 943 } 944 945 /** 946 * i40e_set_new_dynamic_itr - Find new ITR level 947 * @rc: structure containing ring performance data 948 * 949 * Returns true if ITR changed, false if not 950 * 951 * Stores a new ITR value based on packets and byte counts during 952 * the last interrupt. The advantage of per interrupt computation 953 * is faster updates and more accurate ITR for the current traffic 954 * pattern. Constants in this function were computed based on 955 * theoretical maximum wire speed and thresholds were set based on 956 * testing data as well as attempting to minimize response time 957 * while increasing bulk throughput. 958 **/ 959 static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc) 960 { 961 enum i40e_latency_range new_latency_range = rc->latency_range; 962 struct i40e_q_vector *qv = rc->ring->q_vector; 963 u32 new_itr = rc->itr; 964 int bytes_per_int; 965 int usecs; 966 967 if (rc->total_packets == 0 || !rc->itr) 968 return false; 969 970 /* simple throttlerate management 971 * 0-10MB/s lowest (50000 ints/s) 972 * 10-20MB/s low (20000 ints/s) 973 * 20-1249MB/s bulk (18000 ints/s) 974 * > 40000 Rx packets per second (8000 ints/s) 975 * 976 * The math works out because the divisor is in 10^(-6) which 977 * turns the bytes/us input value into MB/s values, but 978 * make sure to use usecs, as the register values written 979 * are in 2 usec increments in the ITR registers, and make sure 980 * to use the smoothed values that the countdown timer gives us. 981 */ 982 usecs = (rc->itr << 1) * ITR_COUNTDOWN_START; 983 bytes_per_int = rc->total_bytes / usecs; 984 985 switch (new_latency_range) { 986 case I40E_LOWEST_LATENCY: 987 if (bytes_per_int > 10) 988 new_latency_range = I40E_LOW_LATENCY; 989 break; 990 case I40E_LOW_LATENCY: 991 if (bytes_per_int > 20) 992 new_latency_range = I40E_BULK_LATENCY; 993 else if (bytes_per_int <= 10) 994 new_latency_range = I40E_LOWEST_LATENCY; 995 break; 996 case I40E_BULK_LATENCY: 997 case I40E_ULTRA_LATENCY: 998 default: 999 if (bytes_per_int <= 20) 1000 new_latency_range = I40E_LOW_LATENCY; 1001 break; 1002 } 1003 1004 /* this is to adjust RX more aggressively when streaming small 1005 * packets. The value of 40000 was picked as it is just beyond 1006 * what the hardware can receive per second if in low latency 1007 * mode. 1008 */ 1009 #define RX_ULTRA_PACKET_RATE 40000 1010 1011 if ((((rc->total_packets * 1000000) / usecs) > RX_ULTRA_PACKET_RATE) && 1012 (&qv->rx == rc)) 1013 new_latency_range = I40E_ULTRA_LATENCY; 1014 1015 rc->latency_range = new_latency_range; 1016 1017 switch (new_latency_range) { 1018 case I40E_LOWEST_LATENCY: 1019 new_itr = I40E_ITR_50K; 1020 break; 1021 case I40E_LOW_LATENCY: 1022 new_itr = I40E_ITR_20K; 1023 break; 1024 case I40E_BULK_LATENCY: 1025 new_itr = I40E_ITR_18K; 1026 break; 1027 case I40E_ULTRA_LATENCY: 1028 new_itr = I40E_ITR_8K; 1029 break; 1030 default: 1031 break; 1032 } 1033 1034 rc->total_bytes = 0; 1035 rc->total_packets = 0; 1036 1037 if (new_itr != rc->itr) { 1038 rc->itr = new_itr; 1039 return true; 1040 } 1041 1042 return false; 1043 } 1044 1045 /** 1046 * i40e_rx_is_programming_status - check for programming status descriptor 1047 * @qw: qword representing status_error_len in CPU ordering 1048 * 1049 * The value of in the descriptor length field indicate if this 1050 * is a programming status descriptor for flow director or FCoE 1051 * by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise 1052 * it is a packet descriptor. 1053 **/ 1054 static inline bool i40e_rx_is_programming_status(u64 qw) 1055 { 1056 /* The Rx filter programming status and SPH bit occupy the same 1057 * spot in the descriptor. Since we don't support packet split we 1058 * can just reuse the bit as an indication that this is a 1059 * programming status descriptor. 1060 */ 1061 return qw & I40E_RXD_QW1_LENGTH_SPH_MASK; 1062 } 1063 1064 /** 1065 * i40e_clean_programming_status - clean the programming status descriptor 1066 * @rx_ring: the rx ring that has this descriptor 1067 * @rx_desc: the rx descriptor written back by HW 1068 * @qw: qword representing status_error_len in CPU ordering 1069 * 1070 * Flow director should handle FD_FILTER_STATUS to check its filter programming 1071 * status being successful or not and take actions accordingly. FCoE should 1072 * handle its context/filter programming/invalidation status and take actions. 1073 * 1074 **/ 1075 static void i40e_clean_programming_status(struct i40e_ring *rx_ring, 1076 union i40e_rx_desc *rx_desc, 1077 u64 qw) 1078 { 1079 u32 ntc = rx_ring->next_to_clean + 1; 1080 u8 id; 1081 1082 /* fetch, update, and store next to clean */ 1083 ntc = (ntc < rx_ring->count) ? ntc : 0; 1084 rx_ring->next_to_clean = ntc; 1085 1086 prefetch(I40E_RX_DESC(rx_ring, ntc)); 1087 1088 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> 1089 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT; 1090 1091 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) 1092 i40e_fd_handle_status(rx_ring, rx_desc, id); 1093 } 1094 1095 /** 1096 * i40e_setup_tx_descriptors - Allocate the Tx descriptors 1097 * @tx_ring: the tx ring to set up 1098 * 1099 * Return 0 on success, negative on error 1100 **/ 1101 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring) 1102 { 1103 struct device *dev = tx_ring->dev; 1104 int bi_size; 1105 1106 if (!dev) 1107 return -ENOMEM; 1108 1109 /* warn if we are about to overwrite the pointer */ 1110 WARN_ON(tx_ring->tx_bi); 1111 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 1112 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL); 1113 if (!tx_ring->tx_bi) 1114 goto err; 1115 1116 u64_stats_init(&tx_ring->syncp); 1117 1118 /* round up to nearest 4K */ 1119 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc); 1120 /* add u32 for head writeback, align after this takes care of 1121 * guaranteeing this is at least one cache line in size 1122 */ 1123 tx_ring->size += sizeof(u32); 1124 tx_ring->size = ALIGN(tx_ring->size, 4096); 1125 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 1126 &tx_ring->dma, GFP_KERNEL); 1127 if (!tx_ring->desc) { 1128 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 1129 tx_ring->size); 1130 goto err; 1131 } 1132 1133 tx_ring->next_to_use = 0; 1134 tx_ring->next_to_clean = 0; 1135 return 0; 1136 1137 err: 1138 kfree(tx_ring->tx_bi); 1139 tx_ring->tx_bi = NULL; 1140 return -ENOMEM; 1141 } 1142 1143 /** 1144 * i40e_clean_rx_ring - Free Rx buffers 1145 * @rx_ring: ring to be cleaned 1146 **/ 1147 void i40e_clean_rx_ring(struct i40e_ring *rx_ring) 1148 { 1149 unsigned long bi_size; 1150 u16 i; 1151 1152 /* ring already cleared, nothing to do */ 1153 if (!rx_ring->rx_bi) 1154 return; 1155 1156 if (rx_ring->skb) { 1157 dev_kfree_skb(rx_ring->skb); 1158 rx_ring->skb = NULL; 1159 } 1160 1161 /* Free all the Rx ring sk_buffs */ 1162 for (i = 0; i < rx_ring->count; i++) { 1163 struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i]; 1164 1165 if (!rx_bi->page) 1166 continue; 1167 1168 /* Invalidate cache lines that may have been written to by 1169 * device so that we avoid corrupting memory. 1170 */ 1171 dma_sync_single_range_for_cpu(rx_ring->dev, 1172 rx_bi->dma, 1173 rx_bi->page_offset, 1174 rx_ring->rx_buf_len, 1175 DMA_FROM_DEVICE); 1176 1177 /* free resources associated with mapping */ 1178 dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma, 1179 i40e_rx_pg_size(rx_ring), 1180 DMA_FROM_DEVICE, 1181 I40E_RX_DMA_ATTR); 1182 1183 __page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias); 1184 1185 rx_bi->page = NULL; 1186 rx_bi->page_offset = 0; 1187 } 1188 1189 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 1190 memset(rx_ring->rx_bi, 0, bi_size); 1191 1192 /* Zero out the descriptor ring */ 1193 memset(rx_ring->desc, 0, rx_ring->size); 1194 1195 rx_ring->next_to_alloc = 0; 1196 rx_ring->next_to_clean = 0; 1197 rx_ring->next_to_use = 0; 1198 } 1199 1200 /** 1201 * i40e_free_rx_resources - Free Rx resources 1202 * @rx_ring: ring to clean the resources from 1203 * 1204 * Free all receive software resources 1205 **/ 1206 void i40e_free_rx_resources(struct i40e_ring *rx_ring) 1207 { 1208 i40e_clean_rx_ring(rx_ring); 1209 rx_ring->xdp_prog = NULL; 1210 kfree(rx_ring->rx_bi); 1211 rx_ring->rx_bi = NULL; 1212 1213 if (rx_ring->desc) { 1214 dma_free_coherent(rx_ring->dev, rx_ring->size, 1215 rx_ring->desc, rx_ring->dma); 1216 rx_ring->desc = NULL; 1217 } 1218 } 1219 1220 /** 1221 * i40e_setup_rx_descriptors - Allocate Rx descriptors 1222 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 1223 * 1224 * Returns 0 on success, negative on failure 1225 **/ 1226 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) 1227 { 1228 struct device *dev = rx_ring->dev; 1229 int bi_size; 1230 1231 /* warn if we are about to overwrite the pointer */ 1232 WARN_ON(rx_ring->rx_bi); 1233 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 1234 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL); 1235 if (!rx_ring->rx_bi) 1236 goto err; 1237 1238 u64_stats_init(&rx_ring->syncp); 1239 1240 /* Round up to nearest 4K */ 1241 rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc); 1242 rx_ring->size = ALIGN(rx_ring->size, 4096); 1243 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 1244 &rx_ring->dma, GFP_KERNEL); 1245 1246 if (!rx_ring->desc) { 1247 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 1248 rx_ring->size); 1249 goto err; 1250 } 1251 1252 rx_ring->next_to_alloc = 0; 1253 rx_ring->next_to_clean = 0; 1254 rx_ring->next_to_use = 0; 1255 1256 rx_ring->xdp_prog = rx_ring->vsi->xdp_prog; 1257 1258 return 0; 1259 err: 1260 kfree(rx_ring->rx_bi); 1261 rx_ring->rx_bi = NULL; 1262 return -ENOMEM; 1263 } 1264 1265 /** 1266 * i40e_release_rx_desc - Store the new tail and head values 1267 * @rx_ring: ring to bump 1268 * @val: new head index 1269 **/ 1270 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val) 1271 { 1272 rx_ring->next_to_use = val; 1273 1274 /* update next to alloc since we have filled the ring */ 1275 rx_ring->next_to_alloc = val; 1276 1277 /* Force memory writes to complete before letting h/w 1278 * know there are new descriptors to fetch. (Only 1279 * applicable for weak-ordered memory model archs, 1280 * such as IA-64). 1281 */ 1282 wmb(); 1283 writel(val, rx_ring->tail); 1284 } 1285 1286 /** 1287 * i40e_rx_offset - Return expected offset into page to access data 1288 * @rx_ring: Ring we are requesting offset of 1289 * 1290 * Returns the offset value for ring into the data buffer. 1291 */ 1292 static inline unsigned int i40e_rx_offset(struct i40e_ring *rx_ring) 1293 { 1294 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0; 1295 } 1296 1297 /** 1298 * i40e_alloc_mapped_page - recycle or make a new page 1299 * @rx_ring: ring to use 1300 * @bi: rx_buffer struct to modify 1301 * 1302 * Returns true if the page was successfully allocated or 1303 * reused. 1304 **/ 1305 static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring, 1306 struct i40e_rx_buffer *bi) 1307 { 1308 struct page *page = bi->page; 1309 dma_addr_t dma; 1310 1311 /* since we are recycling buffers we should seldom need to alloc */ 1312 if (likely(page)) { 1313 rx_ring->rx_stats.page_reuse_count++; 1314 return true; 1315 } 1316 1317 /* alloc new page for storage */ 1318 page = dev_alloc_pages(i40e_rx_pg_order(rx_ring)); 1319 if (unlikely(!page)) { 1320 rx_ring->rx_stats.alloc_page_failed++; 1321 return false; 1322 } 1323 1324 /* map page for use */ 1325 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1326 i40e_rx_pg_size(rx_ring), 1327 DMA_FROM_DEVICE, 1328 I40E_RX_DMA_ATTR); 1329 1330 /* if mapping failed free memory back to system since 1331 * there isn't much point in holding memory we can't use 1332 */ 1333 if (dma_mapping_error(rx_ring->dev, dma)) { 1334 __free_pages(page, i40e_rx_pg_order(rx_ring)); 1335 rx_ring->rx_stats.alloc_page_failed++; 1336 return false; 1337 } 1338 1339 bi->dma = dma; 1340 bi->page = page; 1341 bi->page_offset = i40e_rx_offset(rx_ring); 1342 1343 /* initialize pagecnt_bias to 1 representing we fully own page */ 1344 bi->pagecnt_bias = 1; 1345 1346 return true; 1347 } 1348 1349 /** 1350 * i40e_receive_skb - Send a completed packet up the stack 1351 * @rx_ring: rx ring in play 1352 * @skb: packet to send up 1353 * @vlan_tag: vlan tag for packet 1354 **/ 1355 static void i40e_receive_skb(struct i40e_ring *rx_ring, 1356 struct sk_buff *skb, u16 vlan_tag) 1357 { 1358 struct i40e_q_vector *q_vector = rx_ring->q_vector; 1359 1360 if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 1361 (vlan_tag & VLAN_VID_MASK)) 1362 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); 1363 1364 napi_gro_receive(&q_vector->napi, skb); 1365 } 1366 1367 /** 1368 * i40e_alloc_rx_buffers - Replace used receive buffers 1369 * @rx_ring: ring to place buffers on 1370 * @cleaned_count: number of buffers to replace 1371 * 1372 * Returns false if all allocations were successful, true if any fail 1373 **/ 1374 bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count) 1375 { 1376 u16 ntu = rx_ring->next_to_use; 1377 union i40e_rx_desc *rx_desc; 1378 struct i40e_rx_buffer *bi; 1379 1380 /* do nothing if no valid netdev defined */ 1381 if (!rx_ring->netdev || !cleaned_count) 1382 return false; 1383 1384 rx_desc = I40E_RX_DESC(rx_ring, ntu); 1385 bi = &rx_ring->rx_bi[ntu]; 1386 1387 do { 1388 if (!i40e_alloc_mapped_page(rx_ring, bi)) 1389 goto no_buffers; 1390 1391 /* sync the buffer for use by the device */ 1392 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1393 bi->page_offset, 1394 rx_ring->rx_buf_len, 1395 DMA_FROM_DEVICE); 1396 1397 /* Refresh the desc even if buffer_addrs didn't change 1398 * because each write-back erases this info. 1399 */ 1400 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1401 1402 rx_desc++; 1403 bi++; 1404 ntu++; 1405 if (unlikely(ntu == rx_ring->count)) { 1406 rx_desc = I40E_RX_DESC(rx_ring, 0); 1407 bi = rx_ring->rx_bi; 1408 ntu = 0; 1409 } 1410 1411 /* clear the status bits for the next_to_use descriptor */ 1412 rx_desc->wb.qword1.status_error_len = 0; 1413 1414 cleaned_count--; 1415 } while (cleaned_count); 1416 1417 if (rx_ring->next_to_use != ntu) 1418 i40e_release_rx_desc(rx_ring, ntu); 1419 1420 return false; 1421 1422 no_buffers: 1423 if (rx_ring->next_to_use != ntu) 1424 i40e_release_rx_desc(rx_ring, ntu); 1425 1426 /* make sure to come back via polling to try again after 1427 * allocation failure 1428 */ 1429 return true; 1430 } 1431 1432 /** 1433 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum 1434 * @vsi: the VSI we care about 1435 * @skb: skb currently being received and modified 1436 * @rx_desc: the receive descriptor 1437 **/ 1438 static inline void i40e_rx_checksum(struct i40e_vsi *vsi, 1439 struct sk_buff *skb, 1440 union i40e_rx_desc *rx_desc) 1441 { 1442 struct i40e_rx_ptype_decoded decoded; 1443 u32 rx_error, rx_status; 1444 bool ipv4, ipv6; 1445 u8 ptype; 1446 u64 qword; 1447 1448 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1449 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT; 1450 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >> 1451 I40E_RXD_QW1_ERROR_SHIFT; 1452 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> 1453 I40E_RXD_QW1_STATUS_SHIFT; 1454 decoded = decode_rx_desc_ptype(ptype); 1455 1456 skb->ip_summed = CHECKSUM_NONE; 1457 1458 skb_checksum_none_assert(skb); 1459 1460 /* Rx csum enabled and ip headers found? */ 1461 if (!(vsi->netdev->features & NETIF_F_RXCSUM)) 1462 return; 1463 1464 /* did the hardware decode the packet and checksum? */ 1465 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT))) 1466 return; 1467 1468 /* both known and outer_ip must be set for the below code to work */ 1469 if (!(decoded.known && decoded.outer_ip)) 1470 return; 1471 1472 ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) && 1473 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4); 1474 ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) && 1475 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6); 1476 1477 if (ipv4 && 1478 (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) | 1479 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT)))) 1480 goto checksum_fail; 1481 1482 /* likely incorrect csum if alternate IP extension headers found */ 1483 if (ipv6 && 1484 rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) 1485 /* don't increment checksum err here, non-fatal err */ 1486 return; 1487 1488 /* there was some L4 error, count error and punt packet to the stack */ 1489 if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT)) 1490 goto checksum_fail; 1491 1492 /* handle packets that were not able to be checksummed due 1493 * to arrival speed, in this case the stack can compute 1494 * the csum. 1495 */ 1496 if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT)) 1497 return; 1498 1499 /* If there is an outer header present that might contain a checksum 1500 * we need to bump the checksum level by 1 to reflect the fact that 1501 * we are indicating we validated the inner checksum. 1502 */ 1503 if (decoded.tunnel_type >= I40E_RX_PTYPE_TUNNEL_IP_GRENAT) 1504 skb->csum_level = 1; 1505 1506 /* Only report checksum unnecessary for TCP, UDP, or SCTP */ 1507 switch (decoded.inner_prot) { 1508 case I40E_RX_PTYPE_INNER_PROT_TCP: 1509 case I40E_RX_PTYPE_INNER_PROT_UDP: 1510 case I40E_RX_PTYPE_INNER_PROT_SCTP: 1511 skb->ip_summed = CHECKSUM_UNNECESSARY; 1512 /* fall though */ 1513 default: 1514 break; 1515 } 1516 1517 return; 1518 1519 checksum_fail: 1520 vsi->back->hw_csum_rx_error++; 1521 } 1522 1523 /** 1524 * i40e_ptype_to_htype - get a hash type 1525 * @ptype: the ptype value from the descriptor 1526 * 1527 * Returns a hash type to be used by skb_set_hash 1528 **/ 1529 static inline int i40e_ptype_to_htype(u8 ptype) 1530 { 1531 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype); 1532 1533 if (!decoded.known) 1534 return PKT_HASH_TYPE_NONE; 1535 1536 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1537 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4) 1538 return PKT_HASH_TYPE_L4; 1539 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1540 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3) 1541 return PKT_HASH_TYPE_L3; 1542 else 1543 return PKT_HASH_TYPE_L2; 1544 } 1545 1546 /** 1547 * i40e_rx_hash - set the hash value in the skb 1548 * @ring: descriptor ring 1549 * @rx_desc: specific descriptor 1550 **/ 1551 static inline void i40e_rx_hash(struct i40e_ring *ring, 1552 union i40e_rx_desc *rx_desc, 1553 struct sk_buff *skb, 1554 u8 rx_ptype) 1555 { 1556 u32 hash; 1557 const __le64 rss_mask = 1558 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH << 1559 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT); 1560 1561 if (!(ring->netdev->features & NETIF_F_RXHASH)) 1562 return; 1563 1564 if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) { 1565 hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss); 1566 skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype)); 1567 } 1568 } 1569 1570 /** 1571 * i40e_process_skb_fields - Populate skb header fields from Rx descriptor 1572 * @rx_ring: rx descriptor ring packet is being transacted on 1573 * @rx_desc: pointer to the EOP Rx descriptor 1574 * @skb: pointer to current skb being populated 1575 * @rx_ptype: the packet type decoded by hardware 1576 * 1577 * This function checks the ring, descriptor, and packet information in 1578 * order to populate the hash, checksum, VLAN, protocol, and 1579 * other fields within the skb. 1580 **/ 1581 static inline 1582 void i40e_process_skb_fields(struct i40e_ring *rx_ring, 1583 union i40e_rx_desc *rx_desc, struct sk_buff *skb, 1584 u8 rx_ptype) 1585 { 1586 u64 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1587 u32 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> 1588 I40E_RXD_QW1_STATUS_SHIFT; 1589 u32 tsynvalid = rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK; 1590 u32 tsyn = (rx_status & I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >> 1591 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT; 1592 1593 if (unlikely(tsynvalid)) 1594 i40e_ptp_rx_hwtstamp(rx_ring->vsi->back, skb, tsyn); 1595 1596 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype); 1597 1598 i40e_rx_checksum(rx_ring->vsi, skb, rx_desc); 1599 1600 skb_record_rx_queue(skb, rx_ring->queue_index); 1601 1602 /* modifies the skb - consumes the enet header */ 1603 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1604 } 1605 1606 /** 1607 * i40e_cleanup_headers - Correct empty headers 1608 * @rx_ring: rx descriptor ring packet is being transacted on 1609 * @skb: pointer to current skb being fixed 1610 * @rx_desc: pointer to the EOP Rx descriptor 1611 * 1612 * Also address the case where we are pulling data in on pages only 1613 * and as such no data is present in the skb header. 1614 * 1615 * In addition if skb is not at least 60 bytes we need to pad it so that 1616 * it is large enough to qualify as a valid Ethernet frame. 1617 * 1618 * Returns true if an error was encountered and skb was freed. 1619 **/ 1620 static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb, 1621 union i40e_rx_desc *rx_desc) 1622 1623 { 1624 /* XDP packets use error pointer so abort at this point */ 1625 if (IS_ERR(skb)) 1626 return true; 1627 1628 /* ERR_MASK will only have valid bits if EOP set, and 1629 * what we are doing here is actually checking 1630 * I40E_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in 1631 * the error field 1632 */ 1633 if (unlikely(i40e_test_staterr(rx_desc, 1634 BIT(I40E_RXD_QW1_ERROR_SHIFT)))) { 1635 dev_kfree_skb_any(skb); 1636 return true; 1637 } 1638 1639 /* if eth_skb_pad returns an error the skb was freed */ 1640 if (eth_skb_pad(skb)) 1641 return true; 1642 1643 return false; 1644 } 1645 1646 /** 1647 * i40e_reuse_rx_page - page flip buffer and store it back on the ring 1648 * @rx_ring: rx descriptor ring to store buffers on 1649 * @old_buff: donor buffer to have page reused 1650 * 1651 * Synchronizes page for reuse by the adapter 1652 **/ 1653 static void i40e_reuse_rx_page(struct i40e_ring *rx_ring, 1654 struct i40e_rx_buffer *old_buff) 1655 { 1656 struct i40e_rx_buffer *new_buff; 1657 u16 nta = rx_ring->next_to_alloc; 1658 1659 new_buff = &rx_ring->rx_bi[nta]; 1660 1661 /* update, and store next to alloc */ 1662 nta++; 1663 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1664 1665 /* transfer page from old buffer to new buffer */ 1666 new_buff->dma = old_buff->dma; 1667 new_buff->page = old_buff->page; 1668 new_buff->page_offset = old_buff->page_offset; 1669 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1670 } 1671 1672 /** 1673 * i40e_page_is_reusable - check if any reuse is possible 1674 * @page: page struct to check 1675 * 1676 * A page is not reusable if it was allocated under low memory 1677 * conditions, or it's not in the same NUMA node as this CPU. 1678 */ 1679 static inline bool i40e_page_is_reusable(struct page *page) 1680 { 1681 return (page_to_nid(page) == numa_mem_id()) && 1682 !page_is_pfmemalloc(page); 1683 } 1684 1685 /** 1686 * i40e_can_reuse_rx_page - Determine if this page can be reused by 1687 * the adapter for another receive 1688 * 1689 * @rx_buffer: buffer containing the page 1690 * 1691 * If page is reusable, rx_buffer->page_offset is adjusted to point to 1692 * an unused region in the page. 1693 * 1694 * For small pages, @truesize will be a constant value, half the size 1695 * of the memory at page. We'll attempt to alternate between high and 1696 * low halves of the page, with one half ready for use by the hardware 1697 * and the other half being consumed by the stack. We use the page 1698 * ref count to determine whether the stack has finished consuming the 1699 * portion of this page that was passed up with a previous packet. If 1700 * the page ref count is >1, we'll assume the "other" half page is 1701 * still busy, and this page cannot be reused. 1702 * 1703 * For larger pages, @truesize will be the actual space used by the 1704 * received packet (adjusted upward to an even multiple of the cache 1705 * line size). This will advance through the page by the amount 1706 * actually consumed by the received packets while there is still 1707 * space for a buffer. Each region of larger pages will be used at 1708 * most once, after which the page will not be reused. 1709 * 1710 * In either case, if the page is reusable its refcount is increased. 1711 **/ 1712 static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer) 1713 { 1714 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1715 struct page *page = rx_buffer->page; 1716 1717 /* Is any reuse possible? */ 1718 if (unlikely(!i40e_page_is_reusable(page))) 1719 return false; 1720 1721 #if (PAGE_SIZE < 8192) 1722 /* if we are only owner of page we can reuse it */ 1723 if (unlikely((page_count(page) - pagecnt_bias) > 1)) 1724 return false; 1725 #else 1726 #define I40E_LAST_OFFSET \ 1727 (SKB_WITH_OVERHEAD(PAGE_SIZE) - I40E_RXBUFFER_2048) 1728 if (rx_buffer->page_offset > I40E_LAST_OFFSET) 1729 return false; 1730 #endif 1731 1732 /* If we have drained the page fragment pool we need to update 1733 * the pagecnt_bias and page count so that we fully restock the 1734 * number of references the driver holds. 1735 */ 1736 if (unlikely(!pagecnt_bias)) { 1737 page_ref_add(page, USHRT_MAX); 1738 rx_buffer->pagecnt_bias = USHRT_MAX; 1739 } 1740 1741 return true; 1742 } 1743 1744 /** 1745 * i40e_add_rx_frag - Add contents of Rx buffer to sk_buff 1746 * @rx_ring: rx descriptor ring to transact packets on 1747 * @rx_buffer: buffer containing page to add 1748 * @skb: sk_buff to place the data into 1749 * @size: packet length from rx_desc 1750 * 1751 * This function will add the data contained in rx_buffer->page to the skb. 1752 * It will just attach the page as a frag to the skb. 1753 * 1754 * The function will then update the page offset. 1755 **/ 1756 static void i40e_add_rx_frag(struct i40e_ring *rx_ring, 1757 struct i40e_rx_buffer *rx_buffer, 1758 struct sk_buff *skb, 1759 unsigned int size) 1760 { 1761 #if (PAGE_SIZE < 8192) 1762 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 1763 #else 1764 unsigned int truesize = SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring)); 1765 #endif 1766 1767 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1768 rx_buffer->page_offset, size, truesize); 1769 1770 /* page is being used so we must update the page offset */ 1771 #if (PAGE_SIZE < 8192) 1772 rx_buffer->page_offset ^= truesize; 1773 #else 1774 rx_buffer->page_offset += truesize; 1775 #endif 1776 } 1777 1778 /** 1779 * i40e_get_rx_buffer - Fetch Rx buffer and synchronize data for use 1780 * @rx_ring: rx descriptor ring to transact packets on 1781 * @size: size of buffer to add to skb 1782 * 1783 * This function will pull an Rx buffer from the ring and synchronize it 1784 * for use by the CPU. 1785 */ 1786 static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring, 1787 const unsigned int size) 1788 { 1789 struct i40e_rx_buffer *rx_buffer; 1790 1791 rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean]; 1792 prefetchw(rx_buffer->page); 1793 1794 /* we are reusing so sync this buffer for CPU use */ 1795 dma_sync_single_range_for_cpu(rx_ring->dev, 1796 rx_buffer->dma, 1797 rx_buffer->page_offset, 1798 size, 1799 DMA_FROM_DEVICE); 1800 1801 /* We have pulled a buffer for use, so decrement pagecnt_bias */ 1802 rx_buffer->pagecnt_bias--; 1803 1804 return rx_buffer; 1805 } 1806 1807 /** 1808 * i40e_construct_skb - Allocate skb and populate it 1809 * @rx_ring: rx descriptor ring to transact packets on 1810 * @rx_buffer: rx buffer to pull data from 1811 * @xdp: xdp_buff pointing to the data 1812 * 1813 * This function allocates an skb. It then populates it with the page 1814 * data from the current receive descriptor, taking care to set up the 1815 * skb correctly. 1816 */ 1817 static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring, 1818 struct i40e_rx_buffer *rx_buffer, 1819 struct xdp_buff *xdp) 1820 { 1821 unsigned int size = xdp->data_end - xdp->data; 1822 #if (PAGE_SIZE < 8192) 1823 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 1824 #else 1825 unsigned int truesize = SKB_DATA_ALIGN(size); 1826 #endif 1827 unsigned int headlen; 1828 struct sk_buff *skb; 1829 1830 /* prefetch first cache line of first page */ 1831 prefetch(xdp->data); 1832 #if L1_CACHE_BYTES < 128 1833 prefetch(xdp->data + L1_CACHE_BYTES); 1834 #endif 1835 1836 /* allocate a skb to store the frags */ 1837 skb = __napi_alloc_skb(&rx_ring->q_vector->napi, 1838 I40E_RX_HDR_SIZE, 1839 GFP_ATOMIC | __GFP_NOWARN); 1840 if (unlikely(!skb)) 1841 return NULL; 1842 1843 /* Determine available headroom for copy */ 1844 headlen = size; 1845 if (headlen > I40E_RX_HDR_SIZE) 1846 headlen = eth_get_headlen(xdp->data, I40E_RX_HDR_SIZE); 1847 1848 /* align pull length to size of long to optimize memcpy performance */ 1849 memcpy(__skb_put(skb, headlen), xdp->data, 1850 ALIGN(headlen, sizeof(long))); 1851 1852 /* update all of the pointers */ 1853 size -= headlen; 1854 if (size) { 1855 skb_add_rx_frag(skb, 0, rx_buffer->page, 1856 rx_buffer->page_offset + headlen, 1857 size, truesize); 1858 1859 /* buffer is used by skb, update page_offset */ 1860 #if (PAGE_SIZE < 8192) 1861 rx_buffer->page_offset ^= truesize; 1862 #else 1863 rx_buffer->page_offset += truesize; 1864 #endif 1865 } else { 1866 /* buffer is unused, reset bias back to rx_buffer */ 1867 rx_buffer->pagecnt_bias++; 1868 } 1869 1870 return skb; 1871 } 1872 1873 /** 1874 * i40e_build_skb - Build skb around an existing buffer 1875 * @rx_ring: Rx descriptor ring to transact packets on 1876 * @rx_buffer: Rx buffer to pull data from 1877 * @xdp: xdp_buff pointing to the data 1878 * 1879 * This function builds an skb around an existing Rx buffer, taking care 1880 * to set up the skb correctly and avoid any memcpy overhead. 1881 */ 1882 static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring, 1883 struct i40e_rx_buffer *rx_buffer, 1884 struct xdp_buff *xdp) 1885 { 1886 unsigned int size = xdp->data_end - xdp->data; 1887 #if (PAGE_SIZE < 8192) 1888 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 1889 #else 1890 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1891 SKB_DATA_ALIGN(I40E_SKB_PAD + size); 1892 #endif 1893 struct sk_buff *skb; 1894 1895 /* prefetch first cache line of first page */ 1896 prefetch(xdp->data); 1897 #if L1_CACHE_BYTES < 128 1898 prefetch(xdp->data + L1_CACHE_BYTES); 1899 #endif 1900 /* build an skb around the page buffer */ 1901 skb = build_skb(xdp->data_hard_start, truesize); 1902 if (unlikely(!skb)) 1903 return NULL; 1904 1905 /* update pointers within the skb to store the data */ 1906 skb_reserve(skb, I40E_SKB_PAD); 1907 __skb_put(skb, size); 1908 1909 /* buffer is used by skb, update page_offset */ 1910 #if (PAGE_SIZE < 8192) 1911 rx_buffer->page_offset ^= truesize; 1912 #else 1913 rx_buffer->page_offset += truesize; 1914 #endif 1915 1916 return skb; 1917 } 1918 1919 /** 1920 * i40e_put_rx_buffer - Clean up used buffer and either recycle or free 1921 * @rx_ring: rx descriptor ring to transact packets on 1922 * @rx_buffer: rx buffer to pull data from 1923 * 1924 * This function will clean up the contents of the rx_buffer. It will 1925 * either recycle the bufer or unmap it and free the associated resources. 1926 */ 1927 static void i40e_put_rx_buffer(struct i40e_ring *rx_ring, 1928 struct i40e_rx_buffer *rx_buffer) 1929 { 1930 if (i40e_can_reuse_rx_page(rx_buffer)) { 1931 /* hand second half of page back to the ring */ 1932 i40e_reuse_rx_page(rx_ring, rx_buffer); 1933 rx_ring->rx_stats.page_reuse_count++; 1934 } else { 1935 /* we are not reusing the buffer so unmap it */ 1936 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1937 i40e_rx_pg_size(rx_ring), 1938 DMA_FROM_DEVICE, I40E_RX_DMA_ATTR); 1939 __page_frag_cache_drain(rx_buffer->page, 1940 rx_buffer->pagecnt_bias); 1941 } 1942 1943 /* clear contents of buffer_info */ 1944 rx_buffer->page = NULL; 1945 } 1946 1947 /** 1948 * i40e_is_non_eop - process handling of non-EOP buffers 1949 * @rx_ring: Rx ring being processed 1950 * @rx_desc: Rx descriptor for current buffer 1951 * @skb: Current socket buffer containing buffer in progress 1952 * 1953 * This function updates next to clean. If the buffer is an EOP buffer 1954 * this function exits returning false, otherwise it will place the 1955 * sk_buff in the next buffer to be chained and return true indicating 1956 * that this is in fact a non-EOP buffer. 1957 **/ 1958 static bool i40e_is_non_eop(struct i40e_ring *rx_ring, 1959 union i40e_rx_desc *rx_desc, 1960 struct sk_buff *skb) 1961 { 1962 u32 ntc = rx_ring->next_to_clean + 1; 1963 1964 /* fetch, update, and store next to clean */ 1965 ntc = (ntc < rx_ring->count) ? ntc : 0; 1966 rx_ring->next_to_clean = ntc; 1967 1968 prefetch(I40E_RX_DESC(rx_ring, ntc)); 1969 1970 /* if we are the last buffer then there is nothing else to do */ 1971 #define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT) 1972 if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF))) 1973 return false; 1974 1975 rx_ring->rx_stats.non_eop_descs++; 1976 1977 return true; 1978 } 1979 1980 #define I40E_XDP_PASS 0 1981 #define I40E_XDP_CONSUMED 1 1982 #define I40E_XDP_TX 2 1983 1984 static int i40e_xmit_xdp_ring(struct xdp_buff *xdp, 1985 struct i40e_ring *xdp_ring); 1986 1987 /** 1988 * i40e_run_xdp - run an XDP program 1989 * @rx_ring: Rx ring being processed 1990 * @xdp: XDP buffer containing the frame 1991 **/ 1992 static struct sk_buff *i40e_run_xdp(struct i40e_ring *rx_ring, 1993 struct xdp_buff *xdp) 1994 { 1995 int result = I40E_XDP_PASS; 1996 struct i40e_ring *xdp_ring; 1997 struct bpf_prog *xdp_prog; 1998 u32 act; 1999 2000 rcu_read_lock(); 2001 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 2002 2003 if (!xdp_prog) 2004 goto xdp_out; 2005 2006 act = bpf_prog_run_xdp(xdp_prog, xdp); 2007 switch (act) { 2008 case XDP_PASS: 2009 break; 2010 case XDP_TX: 2011 xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; 2012 result = i40e_xmit_xdp_ring(xdp, xdp_ring); 2013 break; 2014 default: 2015 bpf_warn_invalid_xdp_action(act); 2016 case XDP_ABORTED: 2017 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 2018 /* fallthrough -- handle aborts by dropping packet */ 2019 case XDP_DROP: 2020 result = I40E_XDP_CONSUMED; 2021 break; 2022 } 2023 xdp_out: 2024 rcu_read_unlock(); 2025 return ERR_PTR(-result); 2026 } 2027 2028 /** 2029 * i40e_rx_buffer_flip - adjusted rx_buffer to point to an unused region 2030 * @rx_ring: Rx ring 2031 * @rx_buffer: Rx buffer to adjust 2032 * @size: Size of adjustment 2033 **/ 2034 static void i40e_rx_buffer_flip(struct i40e_ring *rx_ring, 2035 struct i40e_rx_buffer *rx_buffer, 2036 unsigned int size) 2037 { 2038 #if (PAGE_SIZE < 8192) 2039 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 2040 2041 rx_buffer->page_offset ^= truesize; 2042 #else 2043 unsigned int truesize = SKB_DATA_ALIGN(i40e_rx_offset(rx_ring) + size); 2044 2045 rx_buffer->page_offset += truesize; 2046 #endif 2047 } 2048 2049 /** 2050 * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 2051 * @rx_ring: rx descriptor ring to transact packets on 2052 * @budget: Total limit on number of packets to process 2053 * 2054 * This function provides a "bounce buffer" approach to Rx interrupt 2055 * processing. The advantage to this is that on systems that have 2056 * expensive overhead for IOMMU access this provides a means of avoiding 2057 * it by maintaining the mapping of the page to the system. 2058 * 2059 * Returns amount of work completed 2060 **/ 2061 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) 2062 { 2063 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 2064 struct sk_buff *skb = rx_ring->skb; 2065 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); 2066 bool failure = false, xdp_xmit = false; 2067 2068 while (likely(total_rx_packets < budget)) { 2069 struct i40e_rx_buffer *rx_buffer; 2070 union i40e_rx_desc *rx_desc; 2071 struct xdp_buff xdp; 2072 unsigned int size; 2073 u16 vlan_tag; 2074 u8 rx_ptype; 2075 u64 qword; 2076 2077 /* return some buffers to hardware, one at a time is too slow */ 2078 if (cleaned_count >= I40E_RX_BUFFER_WRITE) { 2079 failure = failure || 2080 i40e_alloc_rx_buffers(rx_ring, cleaned_count); 2081 cleaned_count = 0; 2082 } 2083 2084 rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean); 2085 2086 /* status_error_len will always be zero for unused descriptors 2087 * because it's cleared in cleanup, and overlaps with hdr_addr 2088 * which is always zero because packet split isn't used, if the 2089 * hardware wrote DD then the length will be non-zero 2090 */ 2091 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 2092 2093 /* This memory barrier is needed to keep us from reading 2094 * any other fields out of the rx_desc until we have 2095 * verified the descriptor has been written back. 2096 */ 2097 dma_rmb(); 2098 2099 if (unlikely(i40e_rx_is_programming_status(qword))) { 2100 i40e_clean_programming_status(rx_ring, rx_desc, qword); 2101 continue; 2102 } 2103 size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 2104 I40E_RXD_QW1_LENGTH_PBUF_SHIFT; 2105 if (!size) 2106 break; 2107 2108 i40e_trace(clean_rx_irq, rx_ring, rx_desc, skb); 2109 rx_buffer = i40e_get_rx_buffer(rx_ring, size); 2110 2111 /* retrieve a buffer from the ring */ 2112 if (!skb) { 2113 xdp.data = page_address(rx_buffer->page) + 2114 rx_buffer->page_offset; 2115 xdp.data_hard_start = xdp.data - 2116 i40e_rx_offset(rx_ring); 2117 xdp.data_end = xdp.data + size; 2118 2119 skb = i40e_run_xdp(rx_ring, &xdp); 2120 } 2121 2122 if (IS_ERR(skb)) { 2123 if (PTR_ERR(skb) == -I40E_XDP_TX) { 2124 xdp_xmit = true; 2125 i40e_rx_buffer_flip(rx_ring, rx_buffer, size); 2126 } else { 2127 rx_buffer->pagecnt_bias++; 2128 } 2129 total_rx_bytes += size; 2130 total_rx_packets++; 2131 } else if (skb) { 2132 i40e_add_rx_frag(rx_ring, rx_buffer, skb, size); 2133 } else if (ring_uses_build_skb(rx_ring)) { 2134 skb = i40e_build_skb(rx_ring, rx_buffer, &xdp); 2135 } else { 2136 skb = i40e_construct_skb(rx_ring, rx_buffer, &xdp); 2137 } 2138 2139 /* exit if we failed to retrieve a buffer */ 2140 if (!skb) { 2141 rx_ring->rx_stats.alloc_buff_failed++; 2142 rx_buffer->pagecnt_bias++; 2143 break; 2144 } 2145 2146 i40e_put_rx_buffer(rx_ring, rx_buffer); 2147 cleaned_count++; 2148 2149 if (i40e_is_non_eop(rx_ring, rx_desc, skb)) 2150 continue; 2151 2152 if (i40e_cleanup_headers(rx_ring, skb, rx_desc)) { 2153 skb = NULL; 2154 continue; 2155 } 2156 2157 /* probably a little skewed due to removing CRC */ 2158 total_rx_bytes += skb->len; 2159 2160 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 2161 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> 2162 I40E_RXD_QW1_PTYPE_SHIFT; 2163 2164 /* populate checksum, VLAN, and protocol */ 2165 i40e_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); 2166 2167 vlan_tag = (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) ? 2168 le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0; 2169 2170 i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb); 2171 i40e_receive_skb(rx_ring, skb, vlan_tag); 2172 skb = NULL; 2173 2174 /* update budget accounting */ 2175 total_rx_packets++; 2176 } 2177 2178 if (xdp_xmit) { 2179 struct i40e_ring *xdp_ring; 2180 2181 xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; 2182 2183 /* Force memory writes to complete before letting h/w 2184 * know there are new descriptors to fetch. 2185 */ 2186 wmb(); 2187 2188 writel(xdp_ring->next_to_use, xdp_ring->tail); 2189 } 2190 2191 rx_ring->skb = skb; 2192 2193 u64_stats_update_begin(&rx_ring->syncp); 2194 rx_ring->stats.packets += total_rx_packets; 2195 rx_ring->stats.bytes += total_rx_bytes; 2196 u64_stats_update_end(&rx_ring->syncp); 2197 rx_ring->q_vector->rx.total_packets += total_rx_packets; 2198 rx_ring->q_vector->rx.total_bytes += total_rx_bytes; 2199 2200 /* guarantee a trip back through this routine if there was a failure */ 2201 return failure ? budget : total_rx_packets; 2202 } 2203 2204 static u32 i40e_buildreg_itr(const int type, const u16 itr) 2205 { 2206 u32 val; 2207 2208 val = I40E_PFINT_DYN_CTLN_INTENA_MASK | 2209 /* Don't clear PBA because that can cause lost interrupts that 2210 * came in while we were cleaning/polling 2211 */ 2212 (type << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | 2213 (itr << I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT); 2214 2215 return val; 2216 } 2217 2218 /* a small macro to shorten up some long lines */ 2219 #define INTREG I40E_PFINT_DYN_CTLN 2220 static inline int get_rx_itr(struct i40e_vsi *vsi, int idx) 2221 { 2222 return vsi->rx_rings[idx]->rx_itr_setting; 2223 } 2224 2225 static inline int get_tx_itr(struct i40e_vsi *vsi, int idx) 2226 { 2227 return vsi->tx_rings[idx]->tx_itr_setting; 2228 } 2229 2230 /** 2231 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt 2232 * @vsi: the VSI we care about 2233 * @q_vector: q_vector for which itr is being updated and interrupt enabled 2234 * 2235 **/ 2236 static inline void i40e_update_enable_itr(struct i40e_vsi *vsi, 2237 struct i40e_q_vector *q_vector) 2238 { 2239 struct i40e_hw *hw = &vsi->back->hw; 2240 bool rx = false, tx = false; 2241 u32 rxval, txval; 2242 int vector; 2243 int idx = q_vector->v_idx; 2244 int rx_itr_setting, tx_itr_setting; 2245 2246 vector = (q_vector->v_idx + vsi->base_vector); 2247 2248 /* avoid dynamic calculation if in countdown mode OR if 2249 * all dynamic is disabled 2250 */ 2251 rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0); 2252 2253 rx_itr_setting = get_rx_itr(vsi, idx); 2254 tx_itr_setting = get_tx_itr(vsi, idx); 2255 2256 if (q_vector->itr_countdown > 0 || 2257 (!ITR_IS_DYNAMIC(rx_itr_setting) && 2258 !ITR_IS_DYNAMIC(tx_itr_setting))) { 2259 goto enable_int; 2260 } 2261 2262 if (ITR_IS_DYNAMIC(tx_itr_setting)) { 2263 rx = i40e_set_new_dynamic_itr(&q_vector->rx); 2264 rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr); 2265 } 2266 2267 if (ITR_IS_DYNAMIC(tx_itr_setting)) { 2268 tx = i40e_set_new_dynamic_itr(&q_vector->tx); 2269 txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr); 2270 } 2271 2272 if (rx || tx) { 2273 /* get the higher of the two ITR adjustments and 2274 * use the same value for both ITR registers 2275 * when in adaptive mode (Rx and/or Tx) 2276 */ 2277 u16 itr = max(q_vector->tx.itr, q_vector->rx.itr); 2278 2279 q_vector->tx.itr = q_vector->rx.itr = itr; 2280 txval = i40e_buildreg_itr(I40E_TX_ITR, itr); 2281 tx = true; 2282 rxval = i40e_buildreg_itr(I40E_RX_ITR, itr); 2283 rx = true; 2284 } 2285 2286 /* only need to enable the interrupt once, but need 2287 * to possibly update both ITR values 2288 */ 2289 if (rx) { 2290 /* set the INTENA_MSK_MASK so that this first write 2291 * won't actually enable the interrupt, instead just 2292 * updating the ITR (it's bit 31 PF and VF) 2293 */ 2294 rxval |= BIT(31); 2295 /* don't check _DOWN because interrupt isn't being enabled */ 2296 wr32(hw, INTREG(vector - 1), rxval); 2297 } 2298 2299 enable_int: 2300 if (!test_bit(__I40E_VSI_DOWN, vsi->state)) 2301 wr32(hw, INTREG(vector - 1), txval); 2302 2303 if (q_vector->itr_countdown) 2304 q_vector->itr_countdown--; 2305 else 2306 q_vector->itr_countdown = ITR_COUNTDOWN_START; 2307 } 2308 2309 /** 2310 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine 2311 * @napi: napi struct with our devices info in it 2312 * @budget: amount of work driver is allowed to do this pass, in packets 2313 * 2314 * This function will clean all queues associated with a q_vector. 2315 * 2316 * Returns the amount of work done 2317 **/ 2318 int i40e_napi_poll(struct napi_struct *napi, int budget) 2319 { 2320 struct i40e_q_vector *q_vector = 2321 container_of(napi, struct i40e_q_vector, napi); 2322 struct i40e_vsi *vsi = q_vector->vsi; 2323 struct i40e_ring *ring; 2324 bool clean_complete = true; 2325 bool arm_wb = false; 2326 int budget_per_ring; 2327 int work_done = 0; 2328 2329 if (test_bit(__I40E_VSI_DOWN, vsi->state)) { 2330 napi_complete(napi); 2331 return 0; 2332 } 2333 2334 /* Since the actual Tx work is minimal, we can give the Tx a larger 2335 * budget and be more aggressive about cleaning up the Tx descriptors. 2336 */ 2337 i40e_for_each_ring(ring, q_vector->tx) { 2338 if (!i40e_clean_tx_irq(vsi, ring, budget)) { 2339 clean_complete = false; 2340 continue; 2341 } 2342 arm_wb |= ring->arm_wb; 2343 ring->arm_wb = false; 2344 } 2345 2346 /* Handle case where we are called by netpoll with a budget of 0 */ 2347 if (budget <= 0) 2348 goto tx_only; 2349 2350 /* We attempt to distribute budget to each Rx queue fairly, but don't 2351 * allow the budget to go below 1 because that would exit polling early. 2352 */ 2353 budget_per_ring = max(budget/q_vector->num_ringpairs, 1); 2354 2355 i40e_for_each_ring(ring, q_vector->rx) { 2356 int cleaned = i40e_clean_rx_irq(ring, budget_per_ring); 2357 2358 work_done += cleaned; 2359 /* if we clean as many as budgeted, we must not be done */ 2360 if (cleaned >= budget_per_ring) 2361 clean_complete = false; 2362 } 2363 2364 /* If work not completed, return budget and polling will return */ 2365 if (!clean_complete) { 2366 const cpumask_t *aff_mask = &q_vector->affinity_mask; 2367 int cpu_id = smp_processor_id(); 2368 2369 /* It is possible that the interrupt affinity has changed but, 2370 * if the cpu is pegged at 100%, polling will never exit while 2371 * traffic continues and the interrupt will be stuck on this 2372 * cpu. We check to make sure affinity is correct before we 2373 * continue to poll, otherwise we must stop polling so the 2374 * interrupt can move to the correct cpu. 2375 */ 2376 if (likely(cpumask_test_cpu(cpu_id, aff_mask) || 2377 !(vsi->back->flags & I40E_FLAG_MSIX_ENABLED))) { 2378 tx_only: 2379 if (arm_wb) { 2380 q_vector->tx.ring[0].tx_stats.tx_force_wb++; 2381 i40e_enable_wb_on_itr(vsi, q_vector); 2382 } 2383 return budget; 2384 } 2385 } 2386 2387 if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR) 2388 q_vector->arm_wb_state = false; 2389 2390 /* Work is done so exit the polling mode and re-enable the interrupt */ 2391 napi_complete_done(napi, work_done); 2392 2393 /* If we're prematurely stopping polling to fix the interrupt 2394 * affinity we want to make sure polling starts back up so we 2395 * issue a call to i40e_force_wb which triggers a SW interrupt. 2396 */ 2397 if (!clean_complete) 2398 i40e_force_wb(vsi, q_vector); 2399 else if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED)) 2400 i40e_irq_dynamic_enable_icr0(vsi->back, false); 2401 else 2402 i40e_update_enable_itr(vsi, q_vector); 2403 2404 return min(work_done, budget - 1); 2405 } 2406 2407 /** 2408 * i40e_atr - Add a Flow Director ATR filter 2409 * @tx_ring: ring to add programming descriptor to 2410 * @skb: send buffer 2411 * @tx_flags: send tx flags 2412 **/ 2413 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb, 2414 u32 tx_flags) 2415 { 2416 struct i40e_filter_program_desc *fdir_desc; 2417 struct i40e_pf *pf = tx_ring->vsi->back; 2418 union { 2419 unsigned char *network; 2420 struct iphdr *ipv4; 2421 struct ipv6hdr *ipv6; 2422 } hdr; 2423 struct tcphdr *th; 2424 unsigned int hlen; 2425 u32 flex_ptype, dtype_cmd; 2426 int l4_proto; 2427 u16 i; 2428 2429 /* make sure ATR is enabled */ 2430 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED)) 2431 return; 2432 2433 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) 2434 return; 2435 2436 /* if sampling is disabled do nothing */ 2437 if (!tx_ring->atr_sample_rate) 2438 return; 2439 2440 /* Currently only IPv4/IPv6 with TCP is supported */ 2441 if (!(tx_flags & (I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6))) 2442 return; 2443 2444 /* snag network header to get L4 type and address */ 2445 hdr.network = (tx_flags & I40E_TX_FLAGS_UDP_TUNNEL) ? 2446 skb_inner_network_header(skb) : skb_network_header(skb); 2447 2448 /* Note: tx_flags gets modified to reflect inner protocols in 2449 * tx_enable_csum function if encap is enabled. 2450 */ 2451 if (tx_flags & I40E_TX_FLAGS_IPV4) { 2452 /* access ihl as u8 to avoid unaligned access on ia64 */ 2453 hlen = (hdr.network[0] & 0x0F) << 2; 2454 l4_proto = hdr.ipv4->protocol; 2455 } else { 2456 hlen = hdr.network - skb->data; 2457 l4_proto = ipv6_find_hdr(skb, &hlen, IPPROTO_TCP, NULL, NULL); 2458 hlen -= hdr.network - skb->data; 2459 } 2460 2461 if (l4_proto != IPPROTO_TCP) 2462 return; 2463 2464 th = (struct tcphdr *)(hdr.network + hlen); 2465 2466 /* Due to lack of space, no more new filters can be programmed */ 2467 if (th->syn && (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED)) 2468 return; 2469 if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) { 2470 /* HW ATR eviction will take care of removing filters on FIN 2471 * and RST packets. 2472 */ 2473 if (th->fin || th->rst) 2474 return; 2475 } 2476 2477 tx_ring->atr_count++; 2478 2479 /* sample on all syn/fin/rst packets or once every atr sample rate */ 2480 if (!th->fin && 2481 !th->syn && 2482 !th->rst && 2483 (tx_ring->atr_count < tx_ring->atr_sample_rate)) 2484 return; 2485 2486 tx_ring->atr_count = 0; 2487 2488 /* grab the next descriptor */ 2489 i = tx_ring->next_to_use; 2490 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i); 2491 2492 i++; 2493 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2494 2495 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 2496 I40E_TXD_FLTR_QW0_QINDEX_MASK; 2497 flex_ptype |= (tx_flags & I40E_TX_FLAGS_IPV4) ? 2498 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << 2499 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : 2500 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << 2501 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 2502 2503 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; 2504 2505 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG; 2506 2507 dtype_cmd |= (th->fin || th->rst) ? 2508 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 2509 I40E_TXD_FLTR_QW1_PCMD_SHIFT) : 2510 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 2511 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 2512 2513 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << 2514 I40E_TXD_FLTR_QW1_DEST_SHIFT; 2515 2516 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << 2517 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; 2518 2519 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK; 2520 if (!(tx_flags & I40E_TX_FLAGS_UDP_TUNNEL)) 2521 dtype_cmd |= 2522 ((u32)I40E_FD_ATR_STAT_IDX(pf->hw.pf_id) << 2523 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 2524 I40E_TXD_FLTR_QW1_CNTINDEX_MASK; 2525 else 2526 dtype_cmd |= 2527 ((u32)I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id) << 2528 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 2529 I40E_TXD_FLTR_QW1_CNTINDEX_MASK; 2530 2531 if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) 2532 dtype_cmd |= I40E_TXD_FLTR_QW1_ATR_MASK; 2533 2534 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype); 2535 fdir_desc->rsvd = cpu_to_le32(0); 2536 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd); 2537 fdir_desc->fd_id = cpu_to_le32(0); 2538 } 2539 2540 /** 2541 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW 2542 * @skb: send buffer 2543 * @tx_ring: ring to send buffer on 2544 * @flags: the tx flags to be set 2545 * 2546 * Checks the skb and set up correspondingly several generic transmit flags 2547 * related to VLAN tagging for the HW, such as VLAN, DCB, etc. 2548 * 2549 * Returns error code indicate the frame should be dropped upon error and the 2550 * otherwise returns 0 to indicate the flags has been set properly. 2551 **/ 2552 static inline int i40e_tx_prepare_vlan_flags(struct sk_buff *skb, 2553 struct i40e_ring *tx_ring, 2554 u32 *flags) 2555 { 2556 __be16 protocol = skb->protocol; 2557 u32 tx_flags = 0; 2558 2559 if (protocol == htons(ETH_P_8021Q) && 2560 !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { 2561 /* When HW VLAN acceleration is turned off by the user the 2562 * stack sets the protocol to 8021q so that the driver 2563 * can take any steps required to support the SW only 2564 * VLAN handling. In our case the driver doesn't need 2565 * to take any further steps so just set the protocol 2566 * to the encapsulated ethertype. 2567 */ 2568 skb->protocol = vlan_get_protocol(skb); 2569 goto out; 2570 } 2571 2572 /* if we have a HW VLAN tag being added, default to the HW one */ 2573 if (skb_vlan_tag_present(skb)) { 2574 tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT; 2575 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 2576 /* else if it is a SW VLAN, check the next protocol and store the tag */ 2577 } else if (protocol == htons(ETH_P_8021Q)) { 2578 struct vlan_hdr *vhdr, _vhdr; 2579 2580 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 2581 if (!vhdr) 2582 return -EINVAL; 2583 2584 protocol = vhdr->h_vlan_encapsulated_proto; 2585 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT; 2586 tx_flags |= I40E_TX_FLAGS_SW_VLAN; 2587 } 2588 2589 if (!(tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED)) 2590 goto out; 2591 2592 /* Insert 802.1p priority into VLAN header */ 2593 if ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) || 2594 (skb->priority != TC_PRIO_CONTROL)) { 2595 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK; 2596 tx_flags |= (skb->priority & 0x7) << 2597 I40E_TX_FLAGS_VLAN_PRIO_SHIFT; 2598 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) { 2599 struct vlan_ethhdr *vhdr; 2600 int rc; 2601 2602 rc = skb_cow_head(skb, 0); 2603 if (rc < 0) 2604 return rc; 2605 vhdr = (struct vlan_ethhdr *)skb->data; 2606 vhdr->h_vlan_TCI = htons(tx_flags >> 2607 I40E_TX_FLAGS_VLAN_SHIFT); 2608 } else { 2609 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 2610 } 2611 } 2612 2613 out: 2614 *flags = tx_flags; 2615 return 0; 2616 } 2617 2618 /** 2619 * i40e_tso - set up the tso context descriptor 2620 * @first: pointer to first Tx buffer for xmit 2621 * @hdr_len: ptr to the size of the packet header 2622 * @cd_type_cmd_tso_mss: Quad Word 1 2623 * 2624 * Returns 0 if no TSO can happen, 1 if tso is going, or error 2625 **/ 2626 static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len, 2627 u64 *cd_type_cmd_tso_mss) 2628 { 2629 struct sk_buff *skb = first->skb; 2630 u64 cd_cmd, cd_tso_len, cd_mss; 2631 union { 2632 struct iphdr *v4; 2633 struct ipv6hdr *v6; 2634 unsigned char *hdr; 2635 } ip; 2636 union { 2637 struct tcphdr *tcp; 2638 struct udphdr *udp; 2639 unsigned char *hdr; 2640 } l4; 2641 u32 paylen, l4_offset; 2642 u16 gso_segs, gso_size; 2643 int err; 2644 2645 if (skb->ip_summed != CHECKSUM_PARTIAL) 2646 return 0; 2647 2648 if (!skb_is_gso(skb)) 2649 return 0; 2650 2651 err = skb_cow_head(skb, 0); 2652 if (err < 0) 2653 return err; 2654 2655 ip.hdr = skb_network_header(skb); 2656 l4.hdr = skb_transport_header(skb); 2657 2658 /* initialize outer IP header fields */ 2659 if (ip.v4->version == 4) { 2660 ip.v4->tot_len = 0; 2661 ip.v4->check = 0; 2662 } else { 2663 ip.v6->payload_len = 0; 2664 } 2665 2666 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 2667 SKB_GSO_GRE_CSUM | 2668 SKB_GSO_IPXIP4 | 2669 SKB_GSO_IPXIP6 | 2670 SKB_GSO_UDP_TUNNEL | 2671 SKB_GSO_UDP_TUNNEL_CSUM)) { 2672 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && 2673 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) { 2674 l4.udp->len = 0; 2675 2676 /* determine offset of outer transport header */ 2677 l4_offset = l4.hdr - skb->data; 2678 2679 /* remove payload length from outer checksum */ 2680 paylen = skb->len - l4_offset; 2681 csum_replace_by_diff(&l4.udp->check, 2682 (__force __wsum)htonl(paylen)); 2683 } 2684 2685 /* reset pointers to inner headers */ 2686 ip.hdr = skb_inner_network_header(skb); 2687 l4.hdr = skb_inner_transport_header(skb); 2688 2689 /* initialize inner IP header fields */ 2690 if (ip.v4->version == 4) { 2691 ip.v4->tot_len = 0; 2692 ip.v4->check = 0; 2693 } else { 2694 ip.v6->payload_len = 0; 2695 } 2696 } 2697 2698 /* determine offset of inner transport header */ 2699 l4_offset = l4.hdr - skb->data; 2700 2701 /* remove payload length from inner checksum */ 2702 paylen = skb->len - l4_offset; 2703 csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); 2704 2705 /* compute length of segmentation header */ 2706 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 2707 2708 /* pull values out of skb_shinfo */ 2709 gso_size = skb_shinfo(skb)->gso_size; 2710 gso_segs = skb_shinfo(skb)->gso_segs; 2711 2712 /* update GSO size and bytecount with header size */ 2713 first->gso_segs = gso_segs; 2714 first->bytecount += (first->gso_segs - 1) * *hdr_len; 2715 2716 /* find the field values */ 2717 cd_cmd = I40E_TX_CTX_DESC_TSO; 2718 cd_tso_len = skb->len - *hdr_len; 2719 cd_mss = gso_size; 2720 *cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | 2721 (cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | 2722 (cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT); 2723 return 1; 2724 } 2725 2726 /** 2727 * i40e_tsyn - set up the tsyn context descriptor 2728 * @tx_ring: ptr to the ring to send 2729 * @skb: ptr to the skb we're sending 2730 * @tx_flags: the collected send information 2731 * @cd_type_cmd_tso_mss: Quad Word 1 2732 * 2733 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen 2734 **/ 2735 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb, 2736 u32 tx_flags, u64 *cd_type_cmd_tso_mss) 2737 { 2738 struct i40e_pf *pf; 2739 2740 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) 2741 return 0; 2742 2743 /* Tx timestamps cannot be sampled when doing TSO */ 2744 if (tx_flags & I40E_TX_FLAGS_TSO) 2745 return 0; 2746 2747 /* only timestamp the outbound packet if the user has requested it and 2748 * we are not already transmitting a packet to be timestamped 2749 */ 2750 pf = i40e_netdev_to_pf(tx_ring->netdev); 2751 if (!(pf->flags & I40E_FLAG_PTP)) 2752 return 0; 2753 2754 if (pf->ptp_tx && 2755 !test_and_set_bit_lock(__I40E_PTP_TX_IN_PROGRESS, pf->state)) { 2756 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2757 pf->ptp_tx_start = jiffies; 2758 pf->ptp_tx_skb = skb_get(skb); 2759 } else { 2760 pf->tx_hwtstamp_skipped++; 2761 return 0; 2762 } 2763 2764 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN << 2765 I40E_TXD_CTX_QW1_CMD_SHIFT; 2766 2767 return 1; 2768 } 2769 2770 /** 2771 * i40e_tx_enable_csum - Enable Tx checksum offloads 2772 * @skb: send buffer 2773 * @tx_flags: pointer to Tx flags currently set 2774 * @td_cmd: Tx descriptor command bits to set 2775 * @td_offset: Tx descriptor header offsets to set 2776 * @tx_ring: Tx descriptor ring 2777 * @cd_tunneling: ptr to context desc bits 2778 **/ 2779 static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags, 2780 u32 *td_cmd, u32 *td_offset, 2781 struct i40e_ring *tx_ring, 2782 u32 *cd_tunneling) 2783 { 2784 union { 2785 struct iphdr *v4; 2786 struct ipv6hdr *v6; 2787 unsigned char *hdr; 2788 } ip; 2789 union { 2790 struct tcphdr *tcp; 2791 struct udphdr *udp; 2792 unsigned char *hdr; 2793 } l4; 2794 unsigned char *exthdr; 2795 u32 offset, cmd = 0; 2796 __be16 frag_off; 2797 u8 l4_proto = 0; 2798 2799 if (skb->ip_summed != CHECKSUM_PARTIAL) 2800 return 0; 2801 2802 ip.hdr = skb_network_header(skb); 2803 l4.hdr = skb_transport_header(skb); 2804 2805 /* compute outer L2 header size */ 2806 offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 2807 2808 if (skb->encapsulation) { 2809 u32 tunnel = 0; 2810 /* define outer network header type */ 2811 if (*tx_flags & I40E_TX_FLAGS_IPV4) { 2812 tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ? 2813 I40E_TX_CTX_EXT_IP_IPV4 : 2814 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 2815 2816 l4_proto = ip.v4->protocol; 2817 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { 2818 tunnel |= I40E_TX_CTX_EXT_IP_IPV6; 2819 2820 exthdr = ip.hdr + sizeof(*ip.v6); 2821 l4_proto = ip.v6->nexthdr; 2822 if (l4.hdr != exthdr) 2823 ipv6_skip_exthdr(skb, exthdr - skb->data, 2824 &l4_proto, &frag_off); 2825 } 2826 2827 /* define outer transport */ 2828 switch (l4_proto) { 2829 case IPPROTO_UDP: 2830 tunnel |= I40E_TXD_CTX_UDP_TUNNELING; 2831 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 2832 break; 2833 case IPPROTO_GRE: 2834 tunnel |= I40E_TXD_CTX_GRE_TUNNELING; 2835 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 2836 break; 2837 case IPPROTO_IPIP: 2838 case IPPROTO_IPV6: 2839 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 2840 l4.hdr = skb_inner_network_header(skb); 2841 break; 2842 default: 2843 if (*tx_flags & I40E_TX_FLAGS_TSO) 2844 return -1; 2845 2846 skb_checksum_help(skb); 2847 return 0; 2848 } 2849 2850 /* compute outer L3 header size */ 2851 tunnel |= ((l4.hdr - ip.hdr) / 4) << 2852 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT; 2853 2854 /* switch IP header pointer from outer to inner header */ 2855 ip.hdr = skb_inner_network_header(skb); 2856 2857 /* compute tunnel header size */ 2858 tunnel |= ((ip.hdr - l4.hdr) / 2) << 2859 I40E_TXD_CTX_QW0_NATLEN_SHIFT; 2860 2861 /* indicate if we need to offload outer UDP header */ 2862 if ((*tx_flags & I40E_TX_FLAGS_TSO) && 2863 !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && 2864 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) 2865 tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK; 2866 2867 /* record tunnel offload values */ 2868 *cd_tunneling |= tunnel; 2869 2870 /* switch L4 header pointer from outer to inner */ 2871 l4.hdr = skb_inner_transport_header(skb); 2872 l4_proto = 0; 2873 2874 /* reset type as we transition from outer to inner headers */ 2875 *tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6); 2876 if (ip.v4->version == 4) 2877 *tx_flags |= I40E_TX_FLAGS_IPV4; 2878 if (ip.v6->version == 6) 2879 *tx_flags |= I40E_TX_FLAGS_IPV6; 2880 } 2881 2882 /* Enable IP checksum offloads */ 2883 if (*tx_flags & I40E_TX_FLAGS_IPV4) { 2884 l4_proto = ip.v4->protocol; 2885 /* the stack computes the IP header already, the only time we 2886 * need the hardware to recompute it is in the case of TSO. 2887 */ 2888 cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ? 2889 I40E_TX_DESC_CMD_IIPT_IPV4_CSUM : 2890 I40E_TX_DESC_CMD_IIPT_IPV4; 2891 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { 2892 cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; 2893 2894 exthdr = ip.hdr + sizeof(*ip.v6); 2895 l4_proto = ip.v6->nexthdr; 2896 if (l4.hdr != exthdr) 2897 ipv6_skip_exthdr(skb, exthdr - skb->data, 2898 &l4_proto, &frag_off); 2899 } 2900 2901 /* compute inner L3 header size */ 2902 offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 2903 2904 /* Enable L4 checksum offloads */ 2905 switch (l4_proto) { 2906 case IPPROTO_TCP: 2907 /* enable checksum offloads */ 2908 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 2909 offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2910 break; 2911 case IPPROTO_SCTP: 2912 /* enable SCTP checksum offload */ 2913 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; 2914 offset |= (sizeof(struct sctphdr) >> 2) << 2915 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2916 break; 2917 case IPPROTO_UDP: 2918 /* enable UDP checksum offload */ 2919 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; 2920 offset |= (sizeof(struct udphdr) >> 2) << 2921 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2922 break; 2923 default: 2924 if (*tx_flags & I40E_TX_FLAGS_TSO) 2925 return -1; 2926 skb_checksum_help(skb); 2927 return 0; 2928 } 2929 2930 *td_cmd |= cmd; 2931 *td_offset |= offset; 2932 2933 return 1; 2934 } 2935 2936 /** 2937 * i40e_create_tx_ctx Build the Tx context descriptor 2938 * @tx_ring: ring to create the descriptor on 2939 * @cd_type_cmd_tso_mss: Quad Word 1 2940 * @cd_tunneling: Quad Word 0 - bits 0-31 2941 * @cd_l2tag2: Quad Word 0 - bits 32-63 2942 **/ 2943 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring, 2944 const u64 cd_type_cmd_tso_mss, 2945 const u32 cd_tunneling, const u32 cd_l2tag2) 2946 { 2947 struct i40e_tx_context_desc *context_desc; 2948 int i = tx_ring->next_to_use; 2949 2950 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) && 2951 !cd_tunneling && !cd_l2tag2) 2952 return; 2953 2954 /* grab the next descriptor */ 2955 context_desc = I40E_TX_CTXTDESC(tx_ring, i); 2956 2957 i++; 2958 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2959 2960 /* cpu_to_le32 and assign to struct fields */ 2961 context_desc->tunneling_params = cpu_to_le32(cd_tunneling); 2962 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2); 2963 context_desc->rsvd = cpu_to_le16(0); 2964 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss); 2965 } 2966 2967 /** 2968 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions 2969 * @tx_ring: the ring to be checked 2970 * @size: the size buffer we want to assure is available 2971 * 2972 * Returns -EBUSY if a stop is needed, else 0 2973 **/ 2974 int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) 2975 { 2976 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 2977 /* Memory barrier before checking head and tail */ 2978 smp_mb(); 2979 2980 /* Check again in a case another CPU has just made room available. */ 2981 if (likely(I40E_DESC_UNUSED(tx_ring) < size)) 2982 return -EBUSY; 2983 2984 /* A reprieve! - use start_queue because it doesn't call schedule */ 2985 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 2986 ++tx_ring->tx_stats.restart_queue; 2987 return 0; 2988 } 2989 2990 /** 2991 * __i40e_chk_linearize - Check if there are more than 8 buffers per packet 2992 * @skb: send buffer 2993 * 2994 * Note: Our HW can't DMA more than 8 buffers to build a packet on the wire 2995 * and so we need to figure out the cases where we need to linearize the skb. 2996 * 2997 * For TSO we need to count the TSO header and segment payload separately. 2998 * As such we need to check cases where we have 7 fragments or more as we 2999 * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for 3000 * the segment payload in the first descriptor, and another 7 for the 3001 * fragments. 3002 **/ 3003 bool __i40e_chk_linearize(struct sk_buff *skb) 3004 { 3005 const struct skb_frag_struct *frag, *stale; 3006 int nr_frags, sum; 3007 3008 /* no need to check if number of frags is less than 7 */ 3009 nr_frags = skb_shinfo(skb)->nr_frags; 3010 if (nr_frags < (I40E_MAX_BUFFER_TXD - 1)) 3011 return false; 3012 3013 /* We need to walk through the list and validate that each group 3014 * of 6 fragments totals at least gso_size. 3015 */ 3016 nr_frags -= I40E_MAX_BUFFER_TXD - 2; 3017 frag = &skb_shinfo(skb)->frags[0]; 3018 3019 /* Initialize size to the negative value of gso_size minus 1. We 3020 * use this as the worst case scenerio in which the frag ahead 3021 * of us only provides one byte which is why we are limited to 6 3022 * descriptors for a single transmit as the header and previous 3023 * fragment are already consuming 2 descriptors. 3024 */ 3025 sum = 1 - skb_shinfo(skb)->gso_size; 3026 3027 /* Add size of frags 0 through 4 to create our initial sum */ 3028 sum += skb_frag_size(frag++); 3029 sum += skb_frag_size(frag++); 3030 sum += skb_frag_size(frag++); 3031 sum += skb_frag_size(frag++); 3032 sum += skb_frag_size(frag++); 3033 3034 /* Walk through fragments adding latest fragment, testing it, and 3035 * then removing stale fragments from the sum. 3036 */ 3037 stale = &skb_shinfo(skb)->frags[0]; 3038 for (;;) { 3039 sum += skb_frag_size(frag++); 3040 3041 /* if sum is negative we failed to make sufficient progress */ 3042 if (sum < 0) 3043 return true; 3044 3045 if (!nr_frags--) 3046 break; 3047 3048 sum -= skb_frag_size(stale++); 3049 } 3050 3051 return false; 3052 } 3053 3054 /** 3055 * i40e_tx_map - Build the Tx descriptor 3056 * @tx_ring: ring to send buffer on 3057 * @skb: send buffer 3058 * @first: first buffer info buffer to use 3059 * @tx_flags: collected send information 3060 * @hdr_len: size of the packet header 3061 * @td_cmd: the command field in the descriptor 3062 * @td_offset: offset for checksum or crc 3063 * 3064 * Returns 0 on success, -1 on failure to DMA 3065 **/ 3066 static inline int i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb, 3067 struct i40e_tx_buffer *first, u32 tx_flags, 3068 const u8 hdr_len, u32 td_cmd, u32 td_offset) 3069 { 3070 unsigned int data_len = skb->data_len; 3071 unsigned int size = skb_headlen(skb); 3072 struct skb_frag_struct *frag; 3073 struct i40e_tx_buffer *tx_bi; 3074 struct i40e_tx_desc *tx_desc; 3075 u16 i = tx_ring->next_to_use; 3076 u32 td_tag = 0; 3077 dma_addr_t dma; 3078 u16 desc_count = 1; 3079 3080 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) { 3081 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; 3082 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >> 3083 I40E_TX_FLAGS_VLAN_SHIFT; 3084 } 3085 3086 first->tx_flags = tx_flags; 3087 3088 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3089 3090 tx_desc = I40E_TX_DESC(tx_ring, i); 3091 tx_bi = first; 3092 3093 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 3094 unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED; 3095 3096 if (dma_mapping_error(tx_ring->dev, dma)) 3097 goto dma_error; 3098 3099 /* record length, and DMA address */ 3100 dma_unmap_len_set(tx_bi, len, size); 3101 dma_unmap_addr_set(tx_bi, dma, dma); 3102 3103 /* align size to end of page */ 3104 max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1); 3105 tx_desc->buffer_addr = cpu_to_le64(dma); 3106 3107 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) { 3108 tx_desc->cmd_type_offset_bsz = 3109 build_ctob(td_cmd, td_offset, 3110 max_data, td_tag); 3111 3112 tx_desc++; 3113 i++; 3114 desc_count++; 3115 3116 if (i == tx_ring->count) { 3117 tx_desc = I40E_TX_DESC(tx_ring, 0); 3118 i = 0; 3119 } 3120 3121 dma += max_data; 3122 size -= max_data; 3123 3124 max_data = I40E_MAX_DATA_PER_TXD_ALIGNED; 3125 tx_desc->buffer_addr = cpu_to_le64(dma); 3126 } 3127 3128 if (likely(!data_len)) 3129 break; 3130 3131 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, 3132 size, td_tag); 3133 3134 tx_desc++; 3135 i++; 3136 desc_count++; 3137 3138 if (i == tx_ring->count) { 3139 tx_desc = I40E_TX_DESC(tx_ring, 0); 3140 i = 0; 3141 } 3142 3143 size = skb_frag_size(frag); 3144 data_len -= size; 3145 3146 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3147 DMA_TO_DEVICE); 3148 3149 tx_bi = &tx_ring->tx_bi[i]; 3150 } 3151 3152 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 3153 3154 i++; 3155 if (i == tx_ring->count) 3156 i = 0; 3157 3158 tx_ring->next_to_use = i; 3159 3160 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED); 3161 3162 /* write last descriptor with EOP bit */ 3163 td_cmd |= I40E_TX_DESC_CMD_EOP; 3164 3165 /* We can OR these values together as they both are checked against 3166 * 4 below and at this point desc_count will be used as a boolean value 3167 * after this if/else block. 3168 */ 3169 desc_count |= ++tx_ring->packet_stride; 3170 3171 /* Algorithm to optimize tail and RS bit setting: 3172 * if queue is stopped 3173 * mark RS bit 3174 * reset packet counter 3175 * else if xmit_more is supported and is true 3176 * advance packet counter to 4 3177 * reset desc_count to 0 3178 * 3179 * if desc_count >= 4 3180 * mark RS bit 3181 * reset packet counter 3182 * if desc_count > 0 3183 * update tail 3184 * 3185 * Note: If there are less than 4 descriptors 3186 * pending and interrupts were disabled the service task will 3187 * trigger a force WB. 3188 */ 3189 if (netif_xmit_stopped(txring_txq(tx_ring))) { 3190 goto do_rs; 3191 } else if (skb->xmit_more) { 3192 /* set stride to arm on next packet and reset desc_count */ 3193 tx_ring->packet_stride = WB_STRIDE; 3194 desc_count = 0; 3195 } else if (desc_count >= WB_STRIDE) { 3196 do_rs: 3197 /* write last descriptor with RS bit set */ 3198 td_cmd |= I40E_TX_DESC_CMD_RS; 3199 tx_ring->packet_stride = 0; 3200 } 3201 3202 tx_desc->cmd_type_offset_bsz = 3203 build_ctob(td_cmd, td_offset, size, td_tag); 3204 3205 /* Force memory writes to complete before letting h/w know there 3206 * are new descriptors to fetch. 3207 * 3208 * We also use this memory barrier to make certain all of the 3209 * status bits have been updated before next_to_watch is written. 3210 */ 3211 wmb(); 3212 3213 /* set next_to_watch value indicating a packet is present */ 3214 first->next_to_watch = tx_desc; 3215 3216 /* notify HW of packet */ 3217 if (desc_count) { 3218 writel(i, tx_ring->tail); 3219 3220 /* we need this if more than one processor can write to our tail 3221 * at a time, it synchronizes IO on IA64/Altix systems 3222 */ 3223 mmiowb(); 3224 } 3225 3226 return 0; 3227 3228 dma_error: 3229 dev_info(tx_ring->dev, "TX DMA map failed\n"); 3230 3231 /* clear dma mappings for failed tx_bi map */ 3232 for (;;) { 3233 tx_bi = &tx_ring->tx_bi[i]; 3234 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi); 3235 if (tx_bi == first) 3236 break; 3237 if (i == 0) 3238 i = tx_ring->count; 3239 i--; 3240 } 3241 3242 tx_ring->next_to_use = i; 3243 3244 return -1; 3245 } 3246 3247 /** 3248 * i40e_xmit_xdp_ring - transmits an XDP buffer to an XDP Tx ring 3249 * @xdp: data to transmit 3250 * @xdp_ring: XDP Tx ring 3251 **/ 3252 static int i40e_xmit_xdp_ring(struct xdp_buff *xdp, 3253 struct i40e_ring *xdp_ring) 3254 { 3255 u32 size = xdp->data_end - xdp->data; 3256 u16 i = xdp_ring->next_to_use; 3257 struct i40e_tx_buffer *tx_bi; 3258 struct i40e_tx_desc *tx_desc; 3259 dma_addr_t dma; 3260 3261 if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) { 3262 xdp_ring->tx_stats.tx_busy++; 3263 return I40E_XDP_CONSUMED; 3264 } 3265 3266 dma = dma_map_single(xdp_ring->dev, xdp->data, size, DMA_TO_DEVICE); 3267 if (dma_mapping_error(xdp_ring->dev, dma)) 3268 return I40E_XDP_CONSUMED; 3269 3270 tx_bi = &xdp_ring->tx_bi[i]; 3271 tx_bi->bytecount = size; 3272 tx_bi->gso_segs = 1; 3273 tx_bi->raw_buf = xdp->data; 3274 3275 /* record length, and DMA address */ 3276 dma_unmap_len_set(tx_bi, len, size); 3277 dma_unmap_addr_set(tx_bi, dma, dma); 3278 3279 tx_desc = I40E_TX_DESC(xdp_ring, i); 3280 tx_desc->buffer_addr = cpu_to_le64(dma); 3281 tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC 3282 | I40E_TXD_CMD, 3283 0, size, 0); 3284 3285 /* Make certain all of the status bits have been updated 3286 * before next_to_watch is written. 3287 */ 3288 smp_wmb(); 3289 3290 i++; 3291 if (i == xdp_ring->count) 3292 i = 0; 3293 3294 tx_bi->next_to_watch = tx_desc; 3295 xdp_ring->next_to_use = i; 3296 3297 return I40E_XDP_TX; 3298 } 3299 3300 /** 3301 * i40e_xmit_frame_ring - Sends buffer on Tx ring 3302 * @skb: send buffer 3303 * @tx_ring: ring to send buffer on 3304 * 3305 * Returns NETDEV_TX_OK if sent, else an error code 3306 **/ 3307 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, 3308 struct i40e_ring *tx_ring) 3309 { 3310 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT; 3311 u32 cd_tunneling = 0, cd_l2tag2 = 0; 3312 struct i40e_tx_buffer *first; 3313 u32 td_offset = 0; 3314 u32 tx_flags = 0; 3315 __be16 protocol; 3316 u32 td_cmd = 0; 3317 u8 hdr_len = 0; 3318 int tso, count; 3319 int tsyn; 3320 3321 /* prefetch the data, we'll need it later */ 3322 prefetch(skb->data); 3323 3324 i40e_trace(xmit_frame_ring, skb, tx_ring); 3325 3326 count = i40e_xmit_descriptor_count(skb); 3327 if (i40e_chk_linearize(skb, count)) { 3328 if (__skb_linearize(skb)) { 3329 dev_kfree_skb_any(skb); 3330 return NETDEV_TX_OK; 3331 } 3332 count = i40e_txd_use_count(skb->len); 3333 tx_ring->tx_stats.tx_linearize++; 3334 } 3335 3336 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, 3337 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, 3338 * + 4 desc gap to avoid the cache line where head is, 3339 * + 1 desc for context descriptor, 3340 * otherwise try next time 3341 */ 3342 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) { 3343 tx_ring->tx_stats.tx_busy++; 3344 return NETDEV_TX_BUSY; 3345 } 3346 3347 /* record the location of the first descriptor for this packet */ 3348 first = &tx_ring->tx_bi[tx_ring->next_to_use]; 3349 first->skb = skb; 3350 first->bytecount = skb->len; 3351 first->gso_segs = 1; 3352 3353 /* prepare the xmit flags */ 3354 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags)) 3355 goto out_drop; 3356 3357 /* obtain protocol of skb */ 3358 protocol = vlan_get_protocol(skb); 3359 3360 /* setup IPv4/IPv6 offloads */ 3361 if (protocol == htons(ETH_P_IP)) 3362 tx_flags |= I40E_TX_FLAGS_IPV4; 3363 else if (protocol == htons(ETH_P_IPV6)) 3364 tx_flags |= I40E_TX_FLAGS_IPV6; 3365 3366 tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss); 3367 3368 if (tso < 0) 3369 goto out_drop; 3370 else if (tso) 3371 tx_flags |= I40E_TX_FLAGS_TSO; 3372 3373 /* Always offload the checksum, since it's in the data descriptor */ 3374 tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset, 3375 tx_ring, &cd_tunneling); 3376 if (tso < 0) 3377 goto out_drop; 3378 3379 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss); 3380 3381 if (tsyn) 3382 tx_flags |= I40E_TX_FLAGS_TSYN; 3383 3384 skb_tx_timestamp(skb); 3385 3386 /* always enable CRC insertion offload */ 3387 td_cmd |= I40E_TX_DESC_CMD_ICRC; 3388 3389 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss, 3390 cd_tunneling, cd_l2tag2); 3391 3392 /* Add Flow Director ATR if it's enabled. 3393 * 3394 * NOTE: this must always be directly before the data descriptor. 3395 */ 3396 i40e_atr(tx_ring, skb, tx_flags); 3397 3398 if (i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len, 3399 td_cmd, td_offset)) 3400 goto cleanup_tx_tstamp; 3401 3402 return NETDEV_TX_OK; 3403 3404 out_drop: 3405 i40e_trace(xmit_frame_ring_drop, first->skb, tx_ring); 3406 dev_kfree_skb_any(first->skb); 3407 first->skb = NULL; 3408 cleanup_tx_tstamp: 3409 if (unlikely(tx_flags & I40E_TX_FLAGS_TSYN)) { 3410 struct i40e_pf *pf = i40e_netdev_to_pf(tx_ring->netdev); 3411 3412 dev_kfree_skb_any(pf->ptp_tx_skb); 3413 pf->ptp_tx_skb = NULL; 3414 clear_bit_unlock(__I40E_PTP_TX_IN_PROGRESS, pf->state); 3415 } 3416 3417 return NETDEV_TX_OK; 3418 } 3419 3420 /** 3421 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer 3422 * @skb: send buffer 3423 * @netdev: network interface device structure 3424 * 3425 * Returns NETDEV_TX_OK if sent, else an error code 3426 **/ 3427 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3428 { 3429 struct i40e_netdev_priv *np = netdev_priv(netdev); 3430 struct i40e_vsi *vsi = np->vsi; 3431 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping]; 3432 3433 /* hardware can't handle really short frames, hardware padding works 3434 * beyond this point 3435 */ 3436 if (skb_put_padto(skb, I40E_MIN_TX_LEN)) 3437 return NETDEV_TX_OK; 3438 3439 return i40e_xmit_frame_ring(skb, tx_ring); 3440 } 3441