1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2016 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/prefetch.h> 28 #include <net/busy_poll.h> 29 #include <linux/bpf_trace.h> 30 #include <net/xdp.h> 31 #include "i40e.h" 32 #include "i40e_trace.h" 33 #include "i40e_prototype.h" 34 35 static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size, 36 u32 td_tag) 37 { 38 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA | 39 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | 40 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | 41 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | 42 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); 43 } 44 45 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) 46 /** 47 * i40e_fdir - Generate a Flow Director descriptor based on fdata 48 * @tx_ring: Tx ring to send buffer on 49 * @fdata: Flow director filter data 50 * @add: Indicate if we are adding a rule or deleting one 51 * 52 **/ 53 static void i40e_fdir(struct i40e_ring *tx_ring, 54 struct i40e_fdir_filter *fdata, bool add) 55 { 56 struct i40e_filter_program_desc *fdir_desc; 57 struct i40e_pf *pf = tx_ring->vsi->back; 58 u32 flex_ptype, dtype_cmd; 59 u16 i; 60 61 /* grab the next descriptor */ 62 i = tx_ring->next_to_use; 63 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i); 64 65 i++; 66 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 67 68 flex_ptype = I40E_TXD_FLTR_QW0_QINDEX_MASK & 69 (fdata->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT); 70 71 flex_ptype |= I40E_TXD_FLTR_QW0_FLEXOFF_MASK & 72 (fdata->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT); 73 74 flex_ptype |= I40E_TXD_FLTR_QW0_PCTYPE_MASK & 75 (fdata->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 76 77 flex_ptype |= I40E_TXD_FLTR_QW0_PCTYPE_MASK & 78 (fdata->flex_offset << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT); 79 80 /* Use LAN VSI Id if not programmed by user */ 81 flex_ptype |= I40E_TXD_FLTR_QW0_DEST_VSI_MASK & 82 ((u32)(fdata->dest_vsi ? : pf->vsi[pf->lan_vsi]->id) << 83 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT); 84 85 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG; 86 87 dtype_cmd |= add ? 88 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 89 I40E_TXD_FLTR_QW1_PCMD_SHIFT : 90 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 91 I40E_TXD_FLTR_QW1_PCMD_SHIFT; 92 93 dtype_cmd |= I40E_TXD_FLTR_QW1_DEST_MASK & 94 (fdata->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT); 95 96 dtype_cmd |= I40E_TXD_FLTR_QW1_FD_STATUS_MASK & 97 (fdata->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT); 98 99 if (fdata->cnt_index) { 100 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK; 101 dtype_cmd |= I40E_TXD_FLTR_QW1_CNTINDEX_MASK & 102 ((u32)fdata->cnt_index << 103 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT); 104 } 105 106 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype); 107 fdir_desc->rsvd = cpu_to_le32(0); 108 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd); 109 fdir_desc->fd_id = cpu_to_le32(fdata->fd_id); 110 } 111 112 #define I40E_FD_CLEAN_DELAY 10 113 /** 114 * i40e_program_fdir_filter - Program a Flow Director filter 115 * @fdir_data: Packet data that will be filter parameters 116 * @raw_packet: the pre-allocated packet buffer for FDir 117 * @pf: The PF pointer 118 * @add: True for add/update, False for remove 119 **/ 120 static int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, 121 u8 *raw_packet, struct i40e_pf *pf, 122 bool add) 123 { 124 struct i40e_tx_buffer *tx_buf, *first; 125 struct i40e_tx_desc *tx_desc; 126 struct i40e_ring *tx_ring; 127 struct i40e_vsi *vsi; 128 struct device *dev; 129 dma_addr_t dma; 130 u32 td_cmd = 0; 131 u16 i; 132 133 /* find existing FDIR VSI */ 134 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 135 if (!vsi) 136 return -ENOENT; 137 138 tx_ring = vsi->tx_rings[0]; 139 dev = tx_ring->dev; 140 141 /* we need two descriptors to add/del a filter and we can wait */ 142 for (i = I40E_FD_CLEAN_DELAY; I40E_DESC_UNUSED(tx_ring) < 2; i--) { 143 if (!i) 144 return -EAGAIN; 145 msleep_interruptible(1); 146 } 147 148 dma = dma_map_single(dev, raw_packet, 149 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE); 150 if (dma_mapping_error(dev, dma)) 151 goto dma_fail; 152 153 /* grab the next descriptor */ 154 i = tx_ring->next_to_use; 155 first = &tx_ring->tx_bi[i]; 156 i40e_fdir(tx_ring, fdir_data, add); 157 158 /* Now program a dummy descriptor */ 159 i = tx_ring->next_to_use; 160 tx_desc = I40E_TX_DESC(tx_ring, i); 161 tx_buf = &tx_ring->tx_bi[i]; 162 163 tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0; 164 165 memset(tx_buf, 0, sizeof(struct i40e_tx_buffer)); 166 167 /* record length, and DMA address */ 168 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE); 169 dma_unmap_addr_set(tx_buf, dma, dma); 170 171 tx_desc->buffer_addr = cpu_to_le64(dma); 172 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY; 173 174 tx_buf->tx_flags = I40E_TX_FLAGS_FD_SB; 175 tx_buf->raw_buf = (void *)raw_packet; 176 177 tx_desc->cmd_type_offset_bsz = 178 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0); 179 180 /* Force memory writes to complete before letting h/w 181 * know there are new descriptors to fetch. 182 */ 183 wmb(); 184 185 /* Mark the data descriptor to be watched */ 186 first->next_to_watch = tx_desc; 187 188 writel(tx_ring->next_to_use, tx_ring->tail); 189 return 0; 190 191 dma_fail: 192 return -1; 193 } 194 195 #define IP_HEADER_OFFSET 14 196 #define I40E_UDPIP_DUMMY_PACKET_LEN 42 197 /** 198 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters 199 * @vsi: pointer to the targeted VSI 200 * @fd_data: the flow director data required for the FDir descriptor 201 * @add: true adds a filter, false removes it 202 * 203 * Returns 0 if the filters were successfully added or removed 204 **/ 205 static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi, 206 struct i40e_fdir_filter *fd_data, 207 bool add) 208 { 209 struct i40e_pf *pf = vsi->back; 210 struct udphdr *udp; 211 struct iphdr *ip; 212 u8 *raw_packet; 213 int ret; 214 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 215 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0, 216 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 217 218 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 219 if (!raw_packet) 220 return -ENOMEM; 221 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN); 222 223 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 224 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET 225 + sizeof(struct iphdr)); 226 227 ip->daddr = fd_data->dst_ip; 228 udp->dest = fd_data->dst_port; 229 ip->saddr = fd_data->src_ip; 230 udp->source = fd_data->src_port; 231 232 if (fd_data->flex_filter) { 233 u8 *payload = raw_packet + I40E_UDPIP_DUMMY_PACKET_LEN; 234 __be16 pattern = fd_data->flex_word; 235 u16 off = fd_data->flex_offset; 236 237 *((__force __be16 *)(payload + off)) = pattern; 238 } 239 240 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; 241 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 242 if (ret) { 243 dev_info(&pf->pdev->dev, 244 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 245 fd_data->pctype, fd_data->fd_id, ret); 246 /* Free the packet buffer since it wasn't added to the ring */ 247 kfree(raw_packet); 248 return -EOPNOTSUPP; 249 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 250 if (add) 251 dev_info(&pf->pdev->dev, 252 "Filter OK for PCTYPE %d loc = %d\n", 253 fd_data->pctype, fd_data->fd_id); 254 else 255 dev_info(&pf->pdev->dev, 256 "Filter deleted for PCTYPE %d loc = %d\n", 257 fd_data->pctype, fd_data->fd_id); 258 } 259 260 if (add) 261 pf->fd_udp4_filter_cnt++; 262 else 263 pf->fd_udp4_filter_cnt--; 264 265 return 0; 266 } 267 268 #define I40E_TCPIP_DUMMY_PACKET_LEN 54 269 /** 270 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters 271 * @vsi: pointer to the targeted VSI 272 * @fd_data: the flow director data required for the FDir descriptor 273 * @add: true adds a filter, false removes it 274 * 275 * Returns 0 if the filters were successfully added or removed 276 **/ 277 static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi, 278 struct i40e_fdir_filter *fd_data, 279 bool add) 280 { 281 struct i40e_pf *pf = vsi->back; 282 struct tcphdr *tcp; 283 struct iphdr *ip; 284 u8 *raw_packet; 285 int ret; 286 /* Dummy packet */ 287 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 288 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0, 289 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11, 290 0x0, 0x72, 0, 0, 0, 0}; 291 292 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 293 if (!raw_packet) 294 return -ENOMEM; 295 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN); 296 297 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 298 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET 299 + sizeof(struct iphdr)); 300 301 ip->daddr = fd_data->dst_ip; 302 tcp->dest = fd_data->dst_port; 303 ip->saddr = fd_data->src_ip; 304 tcp->source = fd_data->src_port; 305 306 if (fd_data->flex_filter) { 307 u8 *payload = raw_packet + I40E_TCPIP_DUMMY_PACKET_LEN; 308 __be16 pattern = fd_data->flex_word; 309 u16 off = fd_data->flex_offset; 310 311 *((__force __be16 *)(payload + off)) = pattern; 312 } 313 314 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP; 315 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 316 if (ret) { 317 dev_info(&pf->pdev->dev, 318 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 319 fd_data->pctype, fd_data->fd_id, ret); 320 /* Free the packet buffer since it wasn't added to the ring */ 321 kfree(raw_packet); 322 return -EOPNOTSUPP; 323 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 324 if (add) 325 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d loc = %d)\n", 326 fd_data->pctype, fd_data->fd_id); 327 else 328 dev_info(&pf->pdev->dev, 329 "Filter deleted for PCTYPE %d loc = %d\n", 330 fd_data->pctype, fd_data->fd_id); 331 } 332 333 if (add) { 334 pf->fd_tcp4_filter_cnt++; 335 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 336 I40E_DEBUG_FD & pf->hw.debug_mask) 337 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n"); 338 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED; 339 } else { 340 pf->fd_tcp4_filter_cnt--; 341 } 342 343 return 0; 344 } 345 346 #define I40E_SCTPIP_DUMMY_PACKET_LEN 46 347 /** 348 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for 349 * a specific flow spec 350 * @vsi: pointer to the targeted VSI 351 * @fd_data: the flow director data required for the FDir descriptor 352 * @add: true adds a filter, false removes it 353 * 354 * Returns 0 if the filters were successfully added or removed 355 **/ 356 static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi, 357 struct i40e_fdir_filter *fd_data, 358 bool add) 359 { 360 struct i40e_pf *pf = vsi->back; 361 struct sctphdr *sctp; 362 struct iphdr *ip; 363 u8 *raw_packet; 364 int ret; 365 /* Dummy packet */ 366 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 367 0x45, 0, 0, 0x20, 0, 0, 0x40, 0, 0x40, 0x84, 0, 0, 0, 0, 0, 0, 368 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 369 370 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 371 if (!raw_packet) 372 return -ENOMEM; 373 memcpy(raw_packet, packet, I40E_SCTPIP_DUMMY_PACKET_LEN); 374 375 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 376 sctp = (struct sctphdr *)(raw_packet + IP_HEADER_OFFSET 377 + sizeof(struct iphdr)); 378 379 ip->daddr = fd_data->dst_ip; 380 sctp->dest = fd_data->dst_port; 381 ip->saddr = fd_data->src_ip; 382 sctp->source = fd_data->src_port; 383 384 if (fd_data->flex_filter) { 385 u8 *payload = raw_packet + I40E_SCTPIP_DUMMY_PACKET_LEN; 386 __be16 pattern = fd_data->flex_word; 387 u16 off = fd_data->flex_offset; 388 389 *((__force __be16 *)(payload + off)) = pattern; 390 } 391 392 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_SCTP; 393 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 394 if (ret) { 395 dev_info(&pf->pdev->dev, 396 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 397 fd_data->pctype, fd_data->fd_id, ret); 398 /* Free the packet buffer since it wasn't added to the ring */ 399 kfree(raw_packet); 400 return -EOPNOTSUPP; 401 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 402 if (add) 403 dev_info(&pf->pdev->dev, 404 "Filter OK for PCTYPE %d loc = %d\n", 405 fd_data->pctype, fd_data->fd_id); 406 else 407 dev_info(&pf->pdev->dev, 408 "Filter deleted for PCTYPE %d loc = %d\n", 409 fd_data->pctype, fd_data->fd_id); 410 } 411 412 if (add) 413 pf->fd_sctp4_filter_cnt++; 414 else 415 pf->fd_sctp4_filter_cnt--; 416 417 return 0; 418 } 419 420 #define I40E_IP_DUMMY_PACKET_LEN 34 421 /** 422 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for 423 * a specific flow spec 424 * @vsi: pointer to the targeted VSI 425 * @fd_data: the flow director data required for the FDir descriptor 426 * @add: true adds a filter, false removes it 427 * 428 * Returns 0 if the filters were successfully added or removed 429 **/ 430 static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi, 431 struct i40e_fdir_filter *fd_data, 432 bool add) 433 { 434 struct i40e_pf *pf = vsi->back; 435 struct iphdr *ip; 436 u8 *raw_packet; 437 int ret; 438 int i; 439 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 440 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0, 441 0, 0, 0, 0}; 442 443 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER; 444 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) { 445 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 446 if (!raw_packet) 447 return -ENOMEM; 448 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN); 449 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 450 451 ip->saddr = fd_data->src_ip; 452 ip->daddr = fd_data->dst_ip; 453 ip->protocol = 0; 454 455 if (fd_data->flex_filter) { 456 u8 *payload = raw_packet + I40E_IP_DUMMY_PACKET_LEN; 457 __be16 pattern = fd_data->flex_word; 458 u16 off = fd_data->flex_offset; 459 460 *((__force __be16 *)(payload + off)) = pattern; 461 } 462 463 fd_data->pctype = i; 464 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 465 if (ret) { 466 dev_info(&pf->pdev->dev, 467 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 468 fd_data->pctype, fd_data->fd_id, ret); 469 /* The packet buffer wasn't added to the ring so we 470 * need to free it now. 471 */ 472 kfree(raw_packet); 473 return -EOPNOTSUPP; 474 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 475 if (add) 476 dev_info(&pf->pdev->dev, 477 "Filter OK for PCTYPE %d loc = %d\n", 478 fd_data->pctype, fd_data->fd_id); 479 else 480 dev_info(&pf->pdev->dev, 481 "Filter deleted for PCTYPE %d loc = %d\n", 482 fd_data->pctype, fd_data->fd_id); 483 } 484 } 485 486 if (add) 487 pf->fd_ip4_filter_cnt++; 488 else 489 pf->fd_ip4_filter_cnt--; 490 491 return 0; 492 } 493 494 /** 495 * i40e_add_del_fdir - Build raw packets to add/del fdir filter 496 * @vsi: pointer to the targeted VSI 497 * @cmd: command to get or set RX flow classification rules 498 * @add: true adds a filter, false removes it 499 * 500 **/ 501 int i40e_add_del_fdir(struct i40e_vsi *vsi, 502 struct i40e_fdir_filter *input, bool add) 503 { 504 struct i40e_pf *pf = vsi->back; 505 int ret; 506 507 switch (input->flow_type & ~FLOW_EXT) { 508 case TCP_V4_FLOW: 509 ret = i40e_add_del_fdir_tcpv4(vsi, input, add); 510 break; 511 case UDP_V4_FLOW: 512 ret = i40e_add_del_fdir_udpv4(vsi, input, add); 513 break; 514 case SCTP_V4_FLOW: 515 ret = i40e_add_del_fdir_sctpv4(vsi, input, add); 516 break; 517 case IP_USER_FLOW: 518 switch (input->ip4_proto) { 519 case IPPROTO_TCP: 520 ret = i40e_add_del_fdir_tcpv4(vsi, input, add); 521 break; 522 case IPPROTO_UDP: 523 ret = i40e_add_del_fdir_udpv4(vsi, input, add); 524 break; 525 case IPPROTO_SCTP: 526 ret = i40e_add_del_fdir_sctpv4(vsi, input, add); 527 break; 528 case IPPROTO_IP: 529 ret = i40e_add_del_fdir_ipv4(vsi, input, add); 530 break; 531 default: 532 /* We cannot support masking based on protocol */ 533 dev_info(&pf->pdev->dev, "Unsupported IPv4 protocol 0x%02x\n", 534 input->ip4_proto); 535 return -EINVAL; 536 } 537 break; 538 default: 539 dev_info(&pf->pdev->dev, "Unsupported flow type 0x%02x\n", 540 input->flow_type); 541 return -EINVAL; 542 } 543 544 /* The buffer allocated here will be normally be freed by 545 * i40e_clean_fdir_tx_irq() as it reclaims resources after transmit 546 * completion. In the event of an error adding the buffer to the FDIR 547 * ring, it will immediately be freed. It may also be freed by 548 * i40e_clean_tx_ring() when closing the VSI. 549 */ 550 return ret; 551 } 552 553 /** 554 * i40e_fd_handle_status - check the Programming Status for FD 555 * @rx_ring: the Rx ring for this descriptor 556 * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor. 557 * @prog_id: the id originally used for programming 558 * 559 * This is used to verify if the FD programming or invalidation 560 * requested by SW to the HW is successful or not and take actions accordingly. 561 **/ 562 static void i40e_fd_handle_status(struct i40e_ring *rx_ring, 563 union i40e_rx_desc *rx_desc, u8 prog_id) 564 { 565 struct i40e_pf *pf = rx_ring->vsi->back; 566 struct pci_dev *pdev = pf->pdev; 567 u32 fcnt_prog, fcnt_avail; 568 u32 error; 569 u64 qw; 570 571 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 572 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> 573 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT; 574 575 if (error == BIT(I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) { 576 pf->fd_inv = le32_to_cpu(rx_desc->wb.qword0.hi_dword.fd_id); 577 if ((rx_desc->wb.qword0.hi_dword.fd_id != 0) || 578 (I40E_DEBUG_FD & pf->hw.debug_mask)) 579 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n", 580 pf->fd_inv); 581 582 /* Check if the programming error is for ATR. 583 * If so, auto disable ATR and set a state for 584 * flush in progress. Next time we come here if flush is in 585 * progress do nothing, once flush is complete the state will 586 * be cleared. 587 */ 588 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 589 return; 590 591 pf->fd_add_err++; 592 /* store the current atr filter count */ 593 pf->fd_atr_cnt = i40e_get_current_atr_cnt(pf); 594 595 if ((rx_desc->wb.qword0.hi_dword.fd_id == 0) && 596 pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) { 597 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED; 598 set_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 599 } 600 601 /* filter programming failed most likely due to table full */ 602 fcnt_prog = i40e_get_global_fd_count(pf); 603 fcnt_avail = pf->fdir_pf_filter_count; 604 /* If ATR is running fcnt_prog can quickly change, 605 * if we are very close to full, it makes sense to disable 606 * FD ATR/SB and then re-enable it when there is room. 607 */ 608 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) { 609 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 610 !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED)) { 611 pf->flags |= I40E_FLAG_FD_SB_AUTO_DISABLED; 612 if (I40E_DEBUG_FD & pf->hw.debug_mask) 613 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n"); 614 } 615 } 616 } else if (error == BIT(I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) { 617 if (I40E_DEBUG_FD & pf->hw.debug_mask) 618 dev_info(&pdev->dev, "ntuple filter fd_id = %d, could not be removed\n", 619 rx_desc->wb.qword0.hi_dword.fd_id); 620 } 621 } 622 623 /** 624 * i40e_unmap_and_free_tx_resource - Release a Tx buffer 625 * @ring: the ring that owns the buffer 626 * @tx_buffer: the buffer to free 627 **/ 628 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring, 629 struct i40e_tx_buffer *tx_buffer) 630 { 631 if (tx_buffer->skb) { 632 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB) 633 kfree(tx_buffer->raw_buf); 634 else if (ring_is_xdp(ring)) 635 page_frag_free(tx_buffer->raw_buf); 636 else 637 dev_kfree_skb_any(tx_buffer->skb); 638 if (dma_unmap_len(tx_buffer, len)) 639 dma_unmap_single(ring->dev, 640 dma_unmap_addr(tx_buffer, dma), 641 dma_unmap_len(tx_buffer, len), 642 DMA_TO_DEVICE); 643 } else if (dma_unmap_len(tx_buffer, len)) { 644 dma_unmap_page(ring->dev, 645 dma_unmap_addr(tx_buffer, dma), 646 dma_unmap_len(tx_buffer, len), 647 DMA_TO_DEVICE); 648 } 649 650 tx_buffer->next_to_watch = NULL; 651 tx_buffer->skb = NULL; 652 dma_unmap_len_set(tx_buffer, len, 0); 653 /* tx_buffer must be completely set up in the transmit path */ 654 } 655 656 /** 657 * i40e_clean_tx_ring - Free any empty Tx buffers 658 * @tx_ring: ring to be cleaned 659 **/ 660 void i40e_clean_tx_ring(struct i40e_ring *tx_ring) 661 { 662 unsigned long bi_size; 663 u16 i; 664 665 /* ring already cleared, nothing to do */ 666 if (!tx_ring->tx_bi) 667 return; 668 669 /* Free all the Tx ring sk_buffs */ 670 for (i = 0; i < tx_ring->count; i++) 671 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]); 672 673 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 674 memset(tx_ring->tx_bi, 0, bi_size); 675 676 /* Zero out the descriptor ring */ 677 memset(tx_ring->desc, 0, tx_ring->size); 678 679 tx_ring->next_to_use = 0; 680 tx_ring->next_to_clean = 0; 681 682 if (!tx_ring->netdev) 683 return; 684 685 /* cleanup Tx queue statistics */ 686 netdev_tx_reset_queue(txring_txq(tx_ring)); 687 } 688 689 /** 690 * i40e_free_tx_resources - Free Tx resources per queue 691 * @tx_ring: Tx descriptor ring for a specific queue 692 * 693 * Free all transmit software resources 694 **/ 695 void i40e_free_tx_resources(struct i40e_ring *tx_ring) 696 { 697 i40e_clean_tx_ring(tx_ring); 698 kfree(tx_ring->tx_bi); 699 tx_ring->tx_bi = NULL; 700 701 if (tx_ring->desc) { 702 dma_free_coherent(tx_ring->dev, tx_ring->size, 703 tx_ring->desc, tx_ring->dma); 704 tx_ring->desc = NULL; 705 } 706 } 707 708 /** 709 * i40e_get_tx_pending - how many tx descriptors not processed 710 * @tx_ring: the ring of descriptors 711 * 712 * Since there is no access to the ring head register 713 * in XL710, we need to use our local copies 714 **/ 715 u32 i40e_get_tx_pending(struct i40e_ring *ring) 716 { 717 u32 head, tail; 718 719 head = i40e_get_head(ring); 720 tail = readl(ring->tail); 721 722 if (head != tail) 723 return (head < tail) ? 724 tail - head : (tail + ring->count - head); 725 726 return 0; 727 } 728 729 #define WB_STRIDE 4 730 731 /** 732 * i40e_clean_tx_irq - Reclaim resources after transmit completes 733 * @vsi: the VSI we care about 734 * @tx_ring: Tx ring to clean 735 * @napi_budget: Used to determine if we are in netpoll 736 * 737 * Returns true if there's any budget left (e.g. the clean is finished) 738 **/ 739 static bool i40e_clean_tx_irq(struct i40e_vsi *vsi, 740 struct i40e_ring *tx_ring, int napi_budget) 741 { 742 u16 i = tx_ring->next_to_clean; 743 struct i40e_tx_buffer *tx_buf; 744 struct i40e_tx_desc *tx_head; 745 struct i40e_tx_desc *tx_desc; 746 unsigned int total_bytes = 0, total_packets = 0; 747 unsigned int budget = vsi->work_limit; 748 749 tx_buf = &tx_ring->tx_bi[i]; 750 tx_desc = I40E_TX_DESC(tx_ring, i); 751 i -= tx_ring->count; 752 753 tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring)); 754 755 do { 756 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 757 758 /* if next_to_watch is not set then there is no work pending */ 759 if (!eop_desc) 760 break; 761 762 /* prevent any other reads prior to eop_desc */ 763 smp_rmb(); 764 765 i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf); 766 /* we have caught up to head, no work left to do */ 767 if (tx_head == tx_desc) 768 break; 769 770 /* clear next_to_watch to prevent false hangs */ 771 tx_buf->next_to_watch = NULL; 772 773 /* update the statistics for this packet */ 774 total_bytes += tx_buf->bytecount; 775 total_packets += tx_buf->gso_segs; 776 777 /* free the skb/XDP data */ 778 if (ring_is_xdp(tx_ring)) 779 page_frag_free(tx_buf->raw_buf); 780 else 781 napi_consume_skb(tx_buf->skb, napi_budget); 782 783 /* unmap skb header data */ 784 dma_unmap_single(tx_ring->dev, 785 dma_unmap_addr(tx_buf, dma), 786 dma_unmap_len(tx_buf, len), 787 DMA_TO_DEVICE); 788 789 /* clear tx_buffer data */ 790 tx_buf->skb = NULL; 791 dma_unmap_len_set(tx_buf, len, 0); 792 793 /* unmap remaining buffers */ 794 while (tx_desc != eop_desc) { 795 i40e_trace(clean_tx_irq_unmap, 796 tx_ring, tx_desc, tx_buf); 797 798 tx_buf++; 799 tx_desc++; 800 i++; 801 if (unlikely(!i)) { 802 i -= tx_ring->count; 803 tx_buf = tx_ring->tx_bi; 804 tx_desc = I40E_TX_DESC(tx_ring, 0); 805 } 806 807 /* unmap any remaining paged data */ 808 if (dma_unmap_len(tx_buf, len)) { 809 dma_unmap_page(tx_ring->dev, 810 dma_unmap_addr(tx_buf, dma), 811 dma_unmap_len(tx_buf, len), 812 DMA_TO_DEVICE); 813 dma_unmap_len_set(tx_buf, len, 0); 814 } 815 } 816 817 /* move us one more past the eop_desc for start of next pkt */ 818 tx_buf++; 819 tx_desc++; 820 i++; 821 if (unlikely(!i)) { 822 i -= tx_ring->count; 823 tx_buf = tx_ring->tx_bi; 824 tx_desc = I40E_TX_DESC(tx_ring, 0); 825 } 826 827 prefetch(tx_desc); 828 829 /* update budget accounting */ 830 budget--; 831 } while (likely(budget)); 832 833 i += tx_ring->count; 834 tx_ring->next_to_clean = i; 835 u64_stats_update_begin(&tx_ring->syncp); 836 tx_ring->stats.bytes += total_bytes; 837 tx_ring->stats.packets += total_packets; 838 u64_stats_update_end(&tx_ring->syncp); 839 tx_ring->q_vector->tx.total_bytes += total_bytes; 840 tx_ring->q_vector->tx.total_packets += total_packets; 841 842 if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) { 843 /* check to see if there are < 4 descriptors 844 * waiting to be written back, then kick the hardware to force 845 * them to be written back in case we stay in NAPI. 846 * In this mode on X722 we do not enable Interrupt. 847 */ 848 unsigned int j = i40e_get_tx_pending(tx_ring); 849 850 if (budget && 851 ((j / WB_STRIDE) == 0) && (j > 0) && 852 !test_bit(__I40E_VSI_DOWN, vsi->state) && 853 (I40E_DESC_UNUSED(tx_ring) != tx_ring->count)) 854 tx_ring->arm_wb = true; 855 } 856 857 if (ring_is_xdp(tx_ring)) 858 return !!budget; 859 860 /* notify netdev of completed buffers */ 861 netdev_tx_completed_queue(txring_txq(tx_ring), 862 total_packets, total_bytes); 863 864 #define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2)) 865 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 866 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 867 /* Make sure that anybody stopping the queue after this 868 * sees the new next_to_clean. 869 */ 870 smp_mb(); 871 if (__netif_subqueue_stopped(tx_ring->netdev, 872 tx_ring->queue_index) && 873 !test_bit(__I40E_VSI_DOWN, vsi->state)) { 874 netif_wake_subqueue(tx_ring->netdev, 875 tx_ring->queue_index); 876 ++tx_ring->tx_stats.restart_queue; 877 } 878 } 879 880 return !!budget; 881 } 882 883 /** 884 * i40e_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled 885 * @vsi: the VSI we care about 886 * @q_vector: the vector on which to enable writeback 887 * 888 **/ 889 static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi, 890 struct i40e_q_vector *q_vector) 891 { 892 u16 flags = q_vector->tx.ring[0].flags; 893 u32 val; 894 895 if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR)) 896 return; 897 898 if (q_vector->arm_wb_state) 899 return; 900 901 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 902 val = I40E_PFINT_DYN_CTLN_WB_ON_ITR_MASK | 903 I40E_PFINT_DYN_CTLN_ITR_INDX_MASK; /* set noitr */ 904 905 wr32(&vsi->back->hw, 906 I40E_PFINT_DYN_CTLN(q_vector->v_idx + vsi->base_vector - 1), 907 val); 908 } else { 909 val = I40E_PFINT_DYN_CTL0_WB_ON_ITR_MASK | 910 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK; /* set noitr */ 911 912 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val); 913 } 914 q_vector->arm_wb_state = true; 915 } 916 917 /** 918 * i40e_force_wb - Issue SW Interrupt so HW does a wb 919 * @vsi: the VSI we care about 920 * @q_vector: the vector on which to force writeback 921 * 922 **/ 923 void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector) 924 { 925 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 926 u32 val = I40E_PFINT_DYN_CTLN_INTENA_MASK | 927 I40E_PFINT_DYN_CTLN_ITR_INDX_MASK | /* set noitr */ 928 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK | 929 I40E_PFINT_DYN_CTLN_SW_ITR_INDX_ENA_MASK; 930 /* allow 00 to be written to the index */ 931 932 wr32(&vsi->back->hw, 933 I40E_PFINT_DYN_CTLN(q_vector->v_idx + 934 vsi->base_vector - 1), val); 935 } else { 936 u32 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 937 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK | /* set noitr */ 938 I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK | 939 I40E_PFINT_DYN_CTL0_SW_ITR_INDX_ENA_MASK; 940 /* allow 00 to be written to the index */ 941 942 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val); 943 } 944 } 945 946 /** 947 * i40e_set_new_dynamic_itr - Find new ITR level 948 * @rc: structure containing ring performance data 949 * 950 * Returns true if ITR changed, false if not 951 * 952 * Stores a new ITR value based on packets and byte counts during 953 * the last interrupt. The advantage of per interrupt computation 954 * is faster updates and more accurate ITR for the current traffic 955 * pattern. Constants in this function were computed based on 956 * theoretical maximum wire speed and thresholds were set based on 957 * testing data as well as attempting to minimize response time 958 * while increasing bulk throughput. 959 **/ 960 static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc) 961 { 962 enum i40e_latency_range new_latency_range = rc->latency_range; 963 u32 new_itr = rc->itr; 964 int bytes_per_usec; 965 unsigned int usecs, estimated_usecs; 966 967 if (rc->total_packets == 0 || !rc->itr) 968 return false; 969 970 usecs = (rc->itr << 1) * ITR_COUNTDOWN_START; 971 bytes_per_usec = rc->total_bytes / usecs; 972 973 /* The calculations in this algorithm depend on interrupts actually 974 * firing at the ITR rate. This may not happen if the packet rate is 975 * really low, or if we've been napi polling. Check to make sure 976 * that's not the case before we continue. 977 */ 978 estimated_usecs = jiffies_to_usecs(jiffies - rc->last_itr_update); 979 if (estimated_usecs > usecs) { 980 new_latency_range = I40E_LOW_LATENCY; 981 goto reset_latency; 982 } 983 984 /* simple throttlerate management 985 * 0-10MB/s lowest (50000 ints/s) 986 * 10-20MB/s low (20000 ints/s) 987 * 20-1249MB/s bulk (18000 ints/s) 988 * 989 * The math works out because the divisor is in 10^(-6) which 990 * turns the bytes/us input value into MB/s values, but 991 * make sure to use usecs, as the register values written 992 * are in 2 usec increments in the ITR registers, and make sure 993 * to use the smoothed values that the countdown timer gives us. 994 */ 995 switch (new_latency_range) { 996 case I40E_LOWEST_LATENCY: 997 if (bytes_per_usec > 10) 998 new_latency_range = I40E_LOW_LATENCY; 999 break; 1000 case I40E_LOW_LATENCY: 1001 if (bytes_per_usec > 20) 1002 new_latency_range = I40E_BULK_LATENCY; 1003 else if (bytes_per_usec <= 10) 1004 new_latency_range = I40E_LOWEST_LATENCY; 1005 break; 1006 case I40E_BULK_LATENCY: 1007 default: 1008 if (bytes_per_usec <= 20) 1009 new_latency_range = I40E_LOW_LATENCY; 1010 break; 1011 } 1012 1013 reset_latency: 1014 rc->latency_range = new_latency_range; 1015 1016 switch (new_latency_range) { 1017 case I40E_LOWEST_LATENCY: 1018 new_itr = I40E_ITR_50K; 1019 break; 1020 case I40E_LOW_LATENCY: 1021 new_itr = I40E_ITR_20K; 1022 break; 1023 case I40E_BULK_LATENCY: 1024 new_itr = I40E_ITR_18K; 1025 break; 1026 default: 1027 break; 1028 } 1029 1030 rc->total_bytes = 0; 1031 rc->total_packets = 0; 1032 rc->last_itr_update = jiffies; 1033 1034 if (new_itr != rc->itr) { 1035 rc->itr = new_itr; 1036 return true; 1037 } 1038 return false; 1039 } 1040 1041 /** 1042 * i40e_reuse_rx_page - page flip buffer and store it back on the ring 1043 * @rx_ring: rx descriptor ring to store buffers on 1044 * @old_buff: donor buffer to have page reused 1045 * 1046 * Synchronizes page for reuse by the adapter 1047 **/ 1048 static void i40e_reuse_rx_page(struct i40e_ring *rx_ring, 1049 struct i40e_rx_buffer *old_buff) 1050 { 1051 struct i40e_rx_buffer *new_buff; 1052 u16 nta = rx_ring->next_to_alloc; 1053 1054 new_buff = &rx_ring->rx_bi[nta]; 1055 1056 /* update, and store next to alloc */ 1057 nta++; 1058 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1059 1060 /* transfer page from old buffer to new buffer */ 1061 new_buff->dma = old_buff->dma; 1062 new_buff->page = old_buff->page; 1063 new_buff->page_offset = old_buff->page_offset; 1064 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1065 } 1066 1067 /** 1068 * i40e_rx_is_programming_status - check for programming status descriptor 1069 * @qw: qword representing status_error_len in CPU ordering 1070 * 1071 * The value of in the descriptor length field indicate if this 1072 * is a programming status descriptor for flow director or FCoE 1073 * by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise 1074 * it is a packet descriptor. 1075 **/ 1076 static inline bool i40e_rx_is_programming_status(u64 qw) 1077 { 1078 /* The Rx filter programming status and SPH bit occupy the same 1079 * spot in the descriptor. Since we don't support packet split we 1080 * can just reuse the bit as an indication that this is a 1081 * programming status descriptor. 1082 */ 1083 return qw & I40E_RXD_QW1_LENGTH_SPH_MASK; 1084 } 1085 1086 /** 1087 * i40e_clean_programming_status - clean the programming status descriptor 1088 * @rx_ring: the rx ring that has this descriptor 1089 * @rx_desc: the rx descriptor written back by HW 1090 * @qw: qword representing status_error_len in CPU ordering 1091 * 1092 * Flow director should handle FD_FILTER_STATUS to check its filter programming 1093 * status being successful or not and take actions accordingly. FCoE should 1094 * handle its context/filter programming/invalidation status and take actions. 1095 * 1096 **/ 1097 static void i40e_clean_programming_status(struct i40e_ring *rx_ring, 1098 union i40e_rx_desc *rx_desc, 1099 u64 qw) 1100 { 1101 struct i40e_rx_buffer *rx_buffer; 1102 u32 ntc = rx_ring->next_to_clean; 1103 u8 id; 1104 1105 /* fetch, update, and store next to clean */ 1106 rx_buffer = &rx_ring->rx_bi[ntc++]; 1107 ntc = (ntc < rx_ring->count) ? ntc : 0; 1108 rx_ring->next_to_clean = ntc; 1109 1110 prefetch(I40E_RX_DESC(rx_ring, ntc)); 1111 1112 /* place unused page back on the ring */ 1113 i40e_reuse_rx_page(rx_ring, rx_buffer); 1114 rx_ring->rx_stats.page_reuse_count++; 1115 1116 /* clear contents of buffer_info */ 1117 rx_buffer->page = NULL; 1118 1119 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> 1120 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT; 1121 1122 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) 1123 i40e_fd_handle_status(rx_ring, rx_desc, id); 1124 } 1125 1126 /** 1127 * i40e_setup_tx_descriptors - Allocate the Tx descriptors 1128 * @tx_ring: the tx ring to set up 1129 * 1130 * Return 0 on success, negative on error 1131 **/ 1132 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring) 1133 { 1134 struct device *dev = tx_ring->dev; 1135 int bi_size; 1136 1137 if (!dev) 1138 return -ENOMEM; 1139 1140 /* warn if we are about to overwrite the pointer */ 1141 WARN_ON(tx_ring->tx_bi); 1142 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 1143 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL); 1144 if (!tx_ring->tx_bi) 1145 goto err; 1146 1147 u64_stats_init(&tx_ring->syncp); 1148 1149 /* round up to nearest 4K */ 1150 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc); 1151 /* add u32 for head writeback, align after this takes care of 1152 * guaranteeing this is at least one cache line in size 1153 */ 1154 tx_ring->size += sizeof(u32); 1155 tx_ring->size = ALIGN(tx_ring->size, 4096); 1156 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 1157 &tx_ring->dma, GFP_KERNEL); 1158 if (!tx_ring->desc) { 1159 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 1160 tx_ring->size); 1161 goto err; 1162 } 1163 1164 tx_ring->next_to_use = 0; 1165 tx_ring->next_to_clean = 0; 1166 return 0; 1167 1168 err: 1169 kfree(tx_ring->tx_bi); 1170 tx_ring->tx_bi = NULL; 1171 return -ENOMEM; 1172 } 1173 1174 /** 1175 * i40e_clean_rx_ring - Free Rx buffers 1176 * @rx_ring: ring to be cleaned 1177 **/ 1178 void i40e_clean_rx_ring(struct i40e_ring *rx_ring) 1179 { 1180 unsigned long bi_size; 1181 u16 i; 1182 1183 /* ring already cleared, nothing to do */ 1184 if (!rx_ring->rx_bi) 1185 return; 1186 1187 if (rx_ring->skb) { 1188 dev_kfree_skb(rx_ring->skb); 1189 rx_ring->skb = NULL; 1190 } 1191 1192 /* Free all the Rx ring sk_buffs */ 1193 for (i = 0; i < rx_ring->count; i++) { 1194 struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i]; 1195 1196 if (!rx_bi->page) 1197 continue; 1198 1199 /* Invalidate cache lines that may have been written to by 1200 * device so that we avoid corrupting memory. 1201 */ 1202 dma_sync_single_range_for_cpu(rx_ring->dev, 1203 rx_bi->dma, 1204 rx_bi->page_offset, 1205 rx_ring->rx_buf_len, 1206 DMA_FROM_DEVICE); 1207 1208 /* free resources associated with mapping */ 1209 dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma, 1210 i40e_rx_pg_size(rx_ring), 1211 DMA_FROM_DEVICE, 1212 I40E_RX_DMA_ATTR); 1213 1214 __page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias); 1215 1216 rx_bi->page = NULL; 1217 rx_bi->page_offset = 0; 1218 } 1219 1220 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 1221 memset(rx_ring->rx_bi, 0, bi_size); 1222 1223 /* Zero out the descriptor ring */ 1224 memset(rx_ring->desc, 0, rx_ring->size); 1225 1226 rx_ring->next_to_alloc = 0; 1227 rx_ring->next_to_clean = 0; 1228 rx_ring->next_to_use = 0; 1229 } 1230 1231 /** 1232 * i40e_free_rx_resources - Free Rx resources 1233 * @rx_ring: ring to clean the resources from 1234 * 1235 * Free all receive software resources 1236 **/ 1237 void i40e_free_rx_resources(struct i40e_ring *rx_ring) 1238 { 1239 i40e_clean_rx_ring(rx_ring); 1240 if (rx_ring->vsi->type == I40E_VSI_MAIN) 1241 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 1242 rx_ring->xdp_prog = NULL; 1243 kfree(rx_ring->rx_bi); 1244 rx_ring->rx_bi = NULL; 1245 1246 if (rx_ring->desc) { 1247 dma_free_coherent(rx_ring->dev, rx_ring->size, 1248 rx_ring->desc, rx_ring->dma); 1249 rx_ring->desc = NULL; 1250 } 1251 } 1252 1253 /** 1254 * i40e_setup_rx_descriptors - Allocate Rx descriptors 1255 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 1256 * 1257 * Returns 0 on success, negative on failure 1258 **/ 1259 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) 1260 { 1261 struct device *dev = rx_ring->dev; 1262 int err = -ENOMEM; 1263 int bi_size; 1264 1265 /* warn if we are about to overwrite the pointer */ 1266 WARN_ON(rx_ring->rx_bi); 1267 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 1268 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL); 1269 if (!rx_ring->rx_bi) 1270 goto err; 1271 1272 u64_stats_init(&rx_ring->syncp); 1273 1274 /* Round up to nearest 4K */ 1275 rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc); 1276 rx_ring->size = ALIGN(rx_ring->size, 4096); 1277 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 1278 &rx_ring->dma, GFP_KERNEL); 1279 1280 if (!rx_ring->desc) { 1281 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 1282 rx_ring->size); 1283 goto err; 1284 } 1285 1286 rx_ring->next_to_alloc = 0; 1287 rx_ring->next_to_clean = 0; 1288 rx_ring->next_to_use = 0; 1289 1290 /* XDP RX-queue info only needed for RX rings exposed to XDP */ 1291 if (rx_ring->vsi->type == I40E_VSI_MAIN) { 1292 err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, 1293 rx_ring->queue_index); 1294 if (err < 0) 1295 goto err; 1296 } 1297 1298 rx_ring->xdp_prog = rx_ring->vsi->xdp_prog; 1299 1300 return 0; 1301 err: 1302 kfree(rx_ring->rx_bi); 1303 rx_ring->rx_bi = NULL; 1304 return err; 1305 } 1306 1307 /** 1308 * i40e_release_rx_desc - Store the new tail and head values 1309 * @rx_ring: ring to bump 1310 * @val: new head index 1311 **/ 1312 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val) 1313 { 1314 rx_ring->next_to_use = val; 1315 1316 /* update next to alloc since we have filled the ring */ 1317 rx_ring->next_to_alloc = val; 1318 1319 /* Force memory writes to complete before letting h/w 1320 * know there are new descriptors to fetch. (Only 1321 * applicable for weak-ordered memory model archs, 1322 * such as IA-64). 1323 */ 1324 wmb(); 1325 writel(val, rx_ring->tail); 1326 } 1327 1328 /** 1329 * i40e_rx_offset - Return expected offset into page to access data 1330 * @rx_ring: Ring we are requesting offset of 1331 * 1332 * Returns the offset value for ring into the data buffer. 1333 */ 1334 static inline unsigned int i40e_rx_offset(struct i40e_ring *rx_ring) 1335 { 1336 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0; 1337 } 1338 1339 /** 1340 * i40e_alloc_mapped_page - recycle or make a new page 1341 * @rx_ring: ring to use 1342 * @bi: rx_buffer struct to modify 1343 * 1344 * Returns true if the page was successfully allocated or 1345 * reused. 1346 **/ 1347 static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring, 1348 struct i40e_rx_buffer *bi) 1349 { 1350 struct page *page = bi->page; 1351 dma_addr_t dma; 1352 1353 /* since we are recycling buffers we should seldom need to alloc */ 1354 if (likely(page)) { 1355 rx_ring->rx_stats.page_reuse_count++; 1356 return true; 1357 } 1358 1359 /* alloc new page for storage */ 1360 page = dev_alloc_pages(i40e_rx_pg_order(rx_ring)); 1361 if (unlikely(!page)) { 1362 rx_ring->rx_stats.alloc_page_failed++; 1363 return false; 1364 } 1365 1366 /* map page for use */ 1367 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1368 i40e_rx_pg_size(rx_ring), 1369 DMA_FROM_DEVICE, 1370 I40E_RX_DMA_ATTR); 1371 1372 /* if mapping failed free memory back to system since 1373 * there isn't much point in holding memory we can't use 1374 */ 1375 if (dma_mapping_error(rx_ring->dev, dma)) { 1376 __free_pages(page, i40e_rx_pg_order(rx_ring)); 1377 rx_ring->rx_stats.alloc_page_failed++; 1378 return false; 1379 } 1380 1381 bi->dma = dma; 1382 bi->page = page; 1383 bi->page_offset = i40e_rx_offset(rx_ring); 1384 1385 /* initialize pagecnt_bias to 1 representing we fully own page */ 1386 bi->pagecnt_bias = 1; 1387 1388 return true; 1389 } 1390 1391 /** 1392 * i40e_receive_skb - Send a completed packet up the stack 1393 * @rx_ring: rx ring in play 1394 * @skb: packet to send up 1395 * @vlan_tag: vlan tag for packet 1396 **/ 1397 static void i40e_receive_skb(struct i40e_ring *rx_ring, 1398 struct sk_buff *skb, u16 vlan_tag) 1399 { 1400 struct i40e_q_vector *q_vector = rx_ring->q_vector; 1401 1402 if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 1403 (vlan_tag & VLAN_VID_MASK)) 1404 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); 1405 1406 napi_gro_receive(&q_vector->napi, skb); 1407 } 1408 1409 /** 1410 * i40e_alloc_rx_buffers - Replace used receive buffers 1411 * @rx_ring: ring to place buffers on 1412 * @cleaned_count: number of buffers to replace 1413 * 1414 * Returns false if all allocations were successful, true if any fail 1415 **/ 1416 bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count) 1417 { 1418 u16 ntu = rx_ring->next_to_use; 1419 union i40e_rx_desc *rx_desc; 1420 struct i40e_rx_buffer *bi; 1421 1422 /* do nothing if no valid netdev defined */ 1423 if (!rx_ring->netdev || !cleaned_count) 1424 return false; 1425 1426 rx_desc = I40E_RX_DESC(rx_ring, ntu); 1427 bi = &rx_ring->rx_bi[ntu]; 1428 1429 do { 1430 if (!i40e_alloc_mapped_page(rx_ring, bi)) 1431 goto no_buffers; 1432 1433 /* sync the buffer for use by the device */ 1434 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1435 bi->page_offset, 1436 rx_ring->rx_buf_len, 1437 DMA_FROM_DEVICE); 1438 1439 /* Refresh the desc even if buffer_addrs didn't change 1440 * because each write-back erases this info. 1441 */ 1442 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1443 1444 rx_desc++; 1445 bi++; 1446 ntu++; 1447 if (unlikely(ntu == rx_ring->count)) { 1448 rx_desc = I40E_RX_DESC(rx_ring, 0); 1449 bi = rx_ring->rx_bi; 1450 ntu = 0; 1451 } 1452 1453 /* clear the status bits for the next_to_use descriptor */ 1454 rx_desc->wb.qword1.status_error_len = 0; 1455 1456 cleaned_count--; 1457 } while (cleaned_count); 1458 1459 if (rx_ring->next_to_use != ntu) 1460 i40e_release_rx_desc(rx_ring, ntu); 1461 1462 return false; 1463 1464 no_buffers: 1465 if (rx_ring->next_to_use != ntu) 1466 i40e_release_rx_desc(rx_ring, ntu); 1467 1468 /* make sure to come back via polling to try again after 1469 * allocation failure 1470 */ 1471 return true; 1472 } 1473 1474 /** 1475 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum 1476 * @vsi: the VSI we care about 1477 * @skb: skb currently being received and modified 1478 * @rx_desc: the receive descriptor 1479 **/ 1480 static inline void i40e_rx_checksum(struct i40e_vsi *vsi, 1481 struct sk_buff *skb, 1482 union i40e_rx_desc *rx_desc) 1483 { 1484 struct i40e_rx_ptype_decoded decoded; 1485 u32 rx_error, rx_status; 1486 bool ipv4, ipv6; 1487 u8 ptype; 1488 u64 qword; 1489 1490 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1491 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT; 1492 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >> 1493 I40E_RXD_QW1_ERROR_SHIFT; 1494 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> 1495 I40E_RXD_QW1_STATUS_SHIFT; 1496 decoded = decode_rx_desc_ptype(ptype); 1497 1498 skb->ip_summed = CHECKSUM_NONE; 1499 1500 skb_checksum_none_assert(skb); 1501 1502 /* Rx csum enabled and ip headers found? */ 1503 if (!(vsi->netdev->features & NETIF_F_RXCSUM)) 1504 return; 1505 1506 /* did the hardware decode the packet and checksum? */ 1507 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT))) 1508 return; 1509 1510 /* both known and outer_ip must be set for the below code to work */ 1511 if (!(decoded.known && decoded.outer_ip)) 1512 return; 1513 1514 ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) && 1515 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4); 1516 ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) && 1517 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6); 1518 1519 if (ipv4 && 1520 (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) | 1521 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT)))) 1522 goto checksum_fail; 1523 1524 /* likely incorrect csum if alternate IP extension headers found */ 1525 if (ipv6 && 1526 rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) 1527 /* don't increment checksum err here, non-fatal err */ 1528 return; 1529 1530 /* there was some L4 error, count error and punt packet to the stack */ 1531 if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT)) 1532 goto checksum_fail; 1533 1534 /* handle packets that were not able to be checksummed due 1535 * to arrival speed, in this case the stack can compute 1536 * the csum. 1537 */ 1538 if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT)) 1539 return; 1540 1541 /* If there is an outer header present that might contain a checksum 1542 * we need to bump the checksum level by 1 to reflect the fact that 1543 * we are indicating we validated the inner checksum. 1544 */ 1545 if (decoded.tunnel_type >= I40E_RX_PTYPE_TUNNEL_IP_GRENAT) 1546 skb->csum_level = 1; 1547 1548 /* Only report checksum unnecessary for TCP, UDP, or SCTP */ 1549 switch (decoded.inner_prot) { 1550 case I40E_RX_PTYPE_INNER_PROT_TCP: 1551 case I40E_RX_PTYPE_INNER_PROT_UDP: 1552 case I40E_RX_PTYPE_INNER_PROT_SCTP: 1553 skb->ip_summed = CHECKSUM_UNNECESSARY; 1554 /* fall though */ 1555 default: 1556 break; 1557 } 1558 1559 return; 1560 1561 checksum_fail: 1562 vsi->back->hw_csum_rx_error++; 1563 } 1564 1565 /** 1566 * i40e_ptype_to_htype - get a hash type 1567 * @ptype: the ptype value from the descriptor 1568 * 1569 * Returns a hash type to be used by skb_set_hash 1570 **/ 1571 static inline int i40e_ptype_to_htype(u8 ptype) 1572 { 1573 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype); 1574 1575 if (!decoded.known) 1576 return PKT_HASH_TYPE_NONE; 1577 1578 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1579 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4) 1580 return PKT_HASH_TYPE_L4; 1581 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1582 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3) 1583 return PKT_HASH_TYPE_L3; 1584 else 1585 return PKT_HASH_TYPE_L2; 1586 } 1587 1588 /** 1589 * i40e_rx_hash - set the hash value in the skb 1590 * @ring: descriptor ring 1591 * @rx_desc: specific descriptor 1592 **/ 1593 static inline void i40e_rx_hash(struct i40e_ring *ring, 1594 union i40e_rx_desc *rx_desc, 1595 struct sk_buff *skb, 1596 u8 rx_ptype) 1597 { 1598 u32 hash; 1599 const __le64 rss_mask = 1600 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH << 1601 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT); 1602 1603 if (!(ring->netdev->features & NETIF_F_RXHASH)) 1604 return; 1605 1606 if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) { 1607 hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss); 1608 skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype)); 1609 } 1610 } 1611 1612 /** 1613 * i40e_process_skb_fields - Populate skb header fields from Rx descriptor 1614 * @rx_ring: rx descriptor ring packet is being transacted on 1615 * @rx_desc: pointer to the EOP Rx descriptor 1616 * @skb: pointer to current skb being populated 1617 * @rx_ptype: the packet type decoded by hardware 1618 * 1619 * This function checks the ring, descriptor, and packet information in 1620 * order to populate the hash, checksum, VLAN, protocol, and 1621 * other fields within the skb. 1622 **/ 1623 static inline 1624 void i40e_process_skb_fields(struct i40e_ring *rx_ring, 1625 union i40e_rx_desc *rx_desc, struct sk_buff *skb, 1626 u8 rx_ptype) 1627 { 1628 u64 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1629 u32 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> 1630 I40E_RXD_QW1_STATUS_SHIFT; 1631 u32 tsynvalid = rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK; 1632 u32 tsyn = (rx_status & I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >> 1633 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT; 1634 1635 if (unlikely(tsynvalid)) 1636 i40e_ptp_rx_hwtstamp(rx_ring->vsi->back, skb, tsyn); 1637 1638 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype); 1639 1640 i40e_rx_checksum(rx_ring->vsi, skb, rx_desc); 1641 1642 skb_record_rx_queue(skb, rx_ring->queue_index); 1643 1644 /* modifies the skb - consumes the enet header */ 1645 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1646 } 1647 1648 /** 1649 * i40e_cleanup_headers - Correct empty headers 1650 * @rx_ring: rx descriptor ring packet is being transacted on 1651 * @skb: pointer to current skb being fixed 1652 * @rx_desc: pointer to the EOP Rx descriptor 1653 * 1654 * Also address the case where we are pulling data in on pages only 1655 * and as such no data is present in the skb header. 1656 * 1657 * In addition if skb is not at least 60 bytes we need to pad it so that 1658 * it is large enough to qualify as a valid Ethernet frame. 1659 * 1660 * Returns true if an error was encountered and skb was freed. 1661 **/ 1662 static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb, 1663 union i40e_rx_desc *rx_desc) 1664 1665 { 1666 /* XDP packets use error pointer so abort at this point */ 1667 if (IS_ERR(skb)) 1668 return true; 1669 1670 /* ERR_MASK will only have valid bits if EOP set, and 1671 * what we are doing here is actually checking 1672 * I40E_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in 1673 * the error field 1674 */ 1675 if (unlikely(i40e_test_staterr(rx_desc, 1676 BIT(I40E_RXD_QW1_ERROR_SHIFT)))) { 1677 dev_kfree_skb_any(skb); 1678 return true; 1679 } 1680 1681 /* if eth_skb_pad returns an error the skb was freed */ 1682 if (eth_skb_pad(skb)) 1683 return true; 1684 1685 return false; 1686 } 1687 1688 /** 1689 * i40e_page_is_reusable - check if any reuse is possible 1690 * @page: page struct to check 1691 * 1692 * A page is not reusable if it was allocated under low memory 1693 * conditions, or it's not in the same NUMA node as this CPU. 1694 */ 1695 static inline bool i40e_page_is_reusable(struct page *page) 1696 { 1697 return (page_to_nid(page) == numa_mem_id()) && 1698 !page_is_pfmemalloc(page); 1699 } 1700 1701 /** 1702 * i40e_can_reuse_rx_page - Determine if this page can be reused by 1703 * the adapter for another receive 1704 * 1705 * @rx_buffer: buffer containing the page 1706 * 1707 * If page is reusable, rx_buffer->page_offset is adjusted to point to 1708 * an unused region in the page. 1709 * 1710 * For small pages, @truesize will be a constant value, half the size 1711 * of the memory at page. We'll attempt to alternate between high and 1712 * low halves of the page, with one half ready for use by the hardware 1713 * and the other half being consumed by the stack. We use the page 1714 * ref count to determine whether the stack has finished consuming the 1715 * portion of this page that was passed up with a previous packet. If 1716 * the page ref count is >1, we'll assume the "other" half page is 1717 * still busy, and this page cannot be reused. 1718 * 1719 * For larger pages, @truesize will be the actual space used by the 1720 * received packet (adjusted upward to an even multiple of the cache 1721 * line size). This will advance through the page by the amount 1722 * actually consumed by the received packets while there is still 1723 * space for a buffer. Each region of larger pages will be used at 1724 * most once, after which the page will not be reused. 1725 * 1726 * In either case, if the page is reusable its refcount is increased. 1727 **/ 1728 static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer) 1729 { 1730 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1731 struct page *page = rx_buffer->page; 1732 1733 /* Is any reuse possible? */ 1734 if (unlikely(!i40e_page_is_reusable(page))) 1735 return false; 1736 1737 #if (PAGE_SIZE < 8192) 1738 /* if we are only owner of page we can reuse it */ 1739 if (unlikely((page_count(page) - pagecnt_bias) > 1)) 1740 return false; 1741 #else 1742 #define I40E_LAST_OFFSET \ 1743 (SKB_WITH_OVERHEAD(PAGE_SIZE) - I40E_RXBUFFER_2048) 1744 if (rx_buffer->page_offset > I40E_LAST_OFFSET) 1745 return false; 1746 #endif 1747 1748 /* If we have drained the page fragment pool we need to update 1749 * the pagecnt_bias and page count so that we fully restock the 1750 * number of references the driver holds. 1751 */ 1752 if (unlikely(!pagecnt_bias)) { 1753 page_ref_add(page, USHRT_MAX); 1754 rx_buffer->pagecnt_bias = USHRT_MAX; 1755 } 1756 1757 return true; 1758 } 1759 1760 /** 1761 * i40e_add_rx_frag - Add contents of Rx buffer to sk_buff 1762 * @rx_ring: rx descriptor ring to transact packets on 1763 * @rx_buffer: buffer containing page to add 1764 * @skb: sk_buff to place the data into 1765 * @size: packet length from rx_desc 1766 * 1767 * This function will add the data contained in rx_buffer->page to the skb. 1768 * It will just attach the page as a frag to the skb. 1769 * 1770 * The function will then update the page offset. 1771 **/ 1772 static void i40e_add_rx_frag(struct i40e_ring *rx_ring, 1773 struct i40e_rx_buffer *rx_buffer, 1774 struct sk_buff *skb, 1775 unsigned int size) 1776 { 1777 #if (PAGE_SIZE < 8192) 1778 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 1779 #else 1780 unsigned int truesize = SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring)); 1781 #endif 1782 1783 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1784 rx_buffer->page_offset, size, truesize); 1785 1786 /* page is being used so we must update the page offset */ 1787 #if (PAGE_SIZE < 8192) 1788 rx_buffer->page_offset ^= truesize; 1789 #else 1790 rx_buffer->page_offset += truesize; 1791 #endif 1792 } 1793 1794 /** 1795 * i40e_get_rx_buffer - Fetch Rx buffer and synchronize data for use 1796 * @rx_ring: rx descriptor ring to transact packets on 1797 * @size: size of buffer to add to skb 1798 * 1799 * This function will pull an Rx buffer from the ring and synchronize it 1800 * for use by the CPU. 1801 */ 1802 static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring, 1803 const unsigned int size) 1804 { 1805 struct i40e_rx_buffer *rx_buffer; 1806 1807 rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean]; 1808 prefetchw(rx_buffer->page); 1809 1810 /* we are reusing so sync this buffer for CPU use */ 1811 dma_sync_single_range_for_cpu(rx_ring->dev, 1812 rx_buffer->dma, 1813 rx_buffer->page_offset, 1814 size, 1815 DMA_FROM_DEVICE); 1816 1817 /* We have pulled a buffer for use, so decrement pagecnt_bias */ 1818 rx_buffer->pagecnt_bias--; 1819 1820 return rx_buffer; 1821 } 1822 1823 /** 1824 * i40e_construct_skb - Allocate skb and populate it 1825 * @rx_ring: rx descriptor ring to transact packets on 1826 * @rx_buffer: rx buffer to pull data from 1827 * @xdp: xdp_buff pointing to the data 1828 * 1829 * This function allocates an skb. It then populates it with the page 1830 * data from the current receive descriptor, taking care to set up the 1831 * skb correctly. 1832 */ 1833 static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring, 1834 struct i40e_rx_buffer *rx_buffer, 1835 struct xdp_buff *xdp) 1836 { 1837 unsigned int size = xdp->data_end - xdp->data; 1838 #if (PAGE_SIZE < 8192) 1839 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 1840 #else 1841 unsigned int truesize = SKB_DATA_ALIGN(size); 1842 #endif 1843 unsigned int headlen; 1844 struct sk_buff *skb; 1845 1846 /* prefetch first cache line of first page */ 1847 prefetch(xdp->data); 1848 #if L1_CACHE_BYTES < 128 1849 prefetch(xdp->data + L1_CACHE_BYTES); 1850 #endif 1851 1852 /* allocate a skb to store the frags */ 1853 skb = __napi_alloc_skb(&rx_ring->q_vector->napi, 1854 I40E_RX_HDR_SIZE, 1855 GFP_ATOMIC | __GFP_NOWARN); 1856 if (unlikely(!skb)) 1857 return NULL; 1858 1859 /* Determine available headroom for copy */ 1860 headlen = size; 1861 if (headlen > I40E_RX_HDR_SIZE) 1862 headlen = eth_get_headlen(xdp->data, I40E_RX_HDR_SIZE); 1863 1864 /* align pull length to size of long to optimize memcpy performance */ 1865 memcpy(__skb_put(skb, headlen), xdp->data, 1866 ALIGN(headlen, sizeof(long))); 1867 1868 /* update all of the pointers */ 1869 size -= headlen; 1870 if (size) { 1871 skb_add_rx_frag(skb, 0, rx_buffer->page, 1872 rx_buffer->page_offset + headlen, 1873 size, truesize); 1874 1875 /* buffer is used by skb, update page_offset */ 1876 #if (PAGE_SIZE < 8192) 1877 rx_buffer->page_offset ^= truesize; 1878 #else 1879 rx_buffer->page_offset += truesize; 1880 #endif 1881 } else { 1882 /* buffer is unused, reset bias back to rx_buffer */ 1883 rx_buffer->pagecnt_bias++; 1884 } 1885 1886 return skb; 1887 } 1888 1889 /** 1890 * i40e_build_skb - Build skb around an existing buffer 1891 * @rx_ring: Rx descriptor ring to transact packets on 1892 * @rx_buffer: Rx buffer to pull data from 1893 * @xdp: xdp_buff pointing to the data 1894 * 1895 * This function builds an skb around an existing Rx buffer, taking care 1896 * to set up the skb correctly and avoid any memcpy overhead. 1897 */ 1898 static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring, 1899 struct i40e_rx_buffer *rx_buffer, 1900 struct xdp_buff *xdp) 1901 { 1902 unsigned int size = xdp->data_end - xdp->data; 1903 #if (PAGE_SIZE < 8192) 1904 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 1905 #else 1906 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1907 SKB_DATA_ALIGN(I40E_SKB_PAD + size); 1908 #endif 1909 struct sk_buff *skb; 1910 1911 /* prefetch first cache line of first page */ 1912 prefetch(xdp->data); 1913 #if L1_CACHE_BYTES < 128 1914 prefetch(xdp->data + L1_CACHE_BYTES); 1915 #endif 1916 /* build an skb around the page buffer */ 1917 skb = build_skb(xdp->data_hard_start, truesize); 1918 if (unlikely(!skb)) 1919 return NULL; 1920 1921 /* update pointers within the skb to store the data */ 1922 skb_reserve(skb, I40E_SKB_PAD); 1923 __skb_put(skb, size); 1924 1925 /* buffer is used by skb, update page_offset */ 1926 #if (PAGE_SIZE < 8192) 1927 rx_buffer->page_offset ^= truesize; 1928 #else 1929 rx_buffer->page_offset += truesize; 1930 #endif 1931 1932 return skb; 1933 } 1934 1935 /** 1936 * i40e_put_rx_buffer - Clean up used buffer and either recycle or free 1937 * @rx_ring: rx descriptor ring to transact packets on 1938 * @rx_buffer: rx buffer to pull data from 1939 * 1940 * This function will clean up the contents of the rx_buffer. It will 1941 * either recycle the bufer or unmap it and free the associated resources. 1942 */ 1943 static void i40e_put_rx_buffer(struct i40e_ring *rx_ring, 1944 struct i40e_rx_buffer *rx_buffer) 1945 { 1946 if (i40e_can_reuse_rx_page(rx_buffer)) { 1947 /* hand second half of page back to the ring */ 1948 i40e_reuse_rx_page(rx_ring, rx_buffer); 1949 rx_ring->rx_stats.page_reuse_count++; 1950 } else { 1951 /* we are not reusing the buffer so unmap it */ 1952 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1953 i40e_rx_pg_size(rx_ring), 1954 DMA_FROM_DEVICE, I40E_RX_DMA_ATTR); 1955 __page_frag_cache_drain(rx_buffer->page, 1956 rx_buffer->pagecnt_bias); 1957 } 1958 1959 /* clear contents of buffer_info */ 1960 rx_buffer->page = NULL; 1961 } 1962 1963 /** 1964 * i40e_is_non_eop - process handling of non-EOP buffers 1965 * @rx_ring: Rx ring being processed 1966 * @rx_desc: Rx descriptor for current buffer 1967 * @skb: Current socket buffer containing buffer in progress 1968 * 1969 * This function updates next to clean. If the buffer is an EOP buffer 1970 * this function exits returning false, otherwise it will place the 1971 * sk_buff in the next buffer to be chained and return true indicating 1972 * that this is in fact a non-EOP buffer. 1973 **/ 1974 static bool i40e_is_non_eop(struct i40e_ring *rx_ring, 1975 union i40e_rx_desc *rx_desc, 1976 struct sk_buff *skb) 1977 { 1978 u32 ntc = rx_ring->next_to_clean + 1; 1979 1980 /* fetch, update, and store next to clean */ 1981 ntc = (ntc < rx_ring->count) ? ntc : 0; 1982 rx_ring->next_to_clean = ntc; 1983 1984 prefetch(I40E_RX_DESC(rx_ring, ntc)); 1985 1986 /* if we are the last buffer then there is nothing else to do */ 1987 #define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT) 1988 if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF))) 1989 return false; 1990 1991 rx_ring->rx_stats.non_eop_descs++; 1992 1993 return true; 1994 } 1995 1996 #define I40E_XDP_PASS 0 1997 #define I40E_XDP_CONSUMED 1 1998 #define I40E_XDP_TX 2 1999 2000 static int i40e_xmit_xdp_ring(struct xdp_buff *xdp, 2001 struct i40e_ring *xdp_ring); 2002 2003 /** 2004 * i40e_run_xdp - run an XDP program 2005 * @rx_ring: Rx ring being processed 2006 * @xdp: XDP buffer containing the frame 2007 **/ 2008 static struct sk_buff *i40e_run_xdp(struct i40e_ring *rx_ring, 2009 struct xdp_buff *xdp) 2010 { 2011 int result = I40E_XDP_PASS; 2012 struct i40e_ring *xdp_ring; 2013 struct bpf_prog *xdp_prog; 2014 u32 act; 2015 2016 rcu_read_lock(); 2017 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 2018 2019 if (!xdp_prog) 2020 goto xdp_out; 2021 2022 act = bpf_prog_run_xdp(xdp_prog, xdp); 2023 switch (act) { 2024 case XDP_PASS: 2025 break; 2026 case XDP_TX: 2027 xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; 2028 result = i40e_xmit_xdp_ring(xdp, xdp_ring); 2029 break; 2030 default: 2031 bpf_warn_invalid_xdp_action(act); 2032 case XDP_ABORTED: 2033 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 2034 /* fallthrough -- handle aborts by dropping packet */ 2035 case XDP_DROP: 2036 result = I40E_XDP_CONSUMED; 2037 break; 2038 } 2039 xdp_out: 2040 rcu_read_unlock(); 2041 return ERR_PTR(-result); 2042 } 2043 2044 /** 2045 * i40e_rx_buffer_flip - adjusted rx_buffer to point to an unused region 2046 * @rx_ring: Rx ring 2047 * @rx_buffer: Rx buffer to adjust 2048 * @size: Size of adjustment 2049 **/ 2050 static void i40e_rx_buffer_flip(struct i40e_ring *rx_ring, 2051 struct i40e_rx_buffer *rx_buffer, 2052 unsigned int size) 2053 { 2054 #if (PAGE_SIZE < 8192) 2055 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 2056 2057 rx_buffer->page_offset ^= truesize; 2058 #else 2059 unsigned int truesize = SKB_DATA_ALIGN(i40e_rx_offset(rx_ring) + size); 2060 2061 rx_buffer->page_offset += truesize; 2062 #endif 2063 } 2064 2065 /** 2066 * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 2067 * @rx_ring: rx descriptor ring to transact packets on 2068 * @budget: Total limit on number of packets to process 2069 * 2070 * This function provides a "bounce buffer" approach to Rx interrupt 2071 * processing. The advantage to this is that on systems that have 2072 * expensive overhead for IOMMU access this provides a means of avoiding 2073 * it by maintaining the mapping of the page to the system. 2074 * 2075 * Returns amount of work completed 2076 **/ 2077 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) 2078 { 2079 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 2080 struct sk_buff *skb = rx_ring->skb; 2081 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); 2082 bool failure = false, xdp_xmit = false; 2083 struct xdp_buff xdp; 2084 2085 xdp.rxq = &rx_ring->xdp_rxq; 2086 2087 while (likely(total_rx_packets < (unsigned int)budget)) { 2088 struct i40e_rx_buffer *rx_buffer; 2089 union i40e_rx_desc *rx_desc; 2090 unsigned int size; 2091 u16 vlan_tag; 2092 u8 rx_ptype; 2093 u64 qword; 2094 2095 /* return some buffers to hardware, one at a time is too slow */ 2096 if (cleaned_count >= I40E_RX_BUFFER_WRITE) { 2097 failure = failure || 2098 i40e_alloc_rx_buffers(rx_ring, cleaned_count); 2099 cleaned_count = 0; 2100 } 2101 2102 rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean); 2103 2104 /* status_error_len will always be zero for unused descriptors 2105 * because it's cleared in cleanup, and overlaps with hdr_addr 2106 * which is always zero because packet split isn't used, if the 2107 * hardware wrote DD then the length will be non-zero 2108 */ 2109 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 2110 2111 /* This memory barrier is needed to keep us from reading 2112 * any other fields out of the rx_desc until we have 2113 * verified the descriptor has been written back. 2114 */ 2115 dma_rmb(); 2116 2117 if (unlikely(i40e_rx_is_programming_status(qword))) { 2118 i40e_clean_programming_status(rx_ring, rx_desc, qword); 2119 cleaned_count++; 2120 continue; 2121 } 2122 size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 2123 I40E_RXD_QW1_LENGTH_PBUF_SHIFT; 2124 if (!size) 2125 break; 2126 2127 i40e_trace(clean_rx_irq, rx_ring, rx_desc, skb); 2128 rx_buffer = i40e_get_rx_buffer(rx_ring, size); 2129 2130 /* retrieve a buffer from the ring */ 2131 if (!skb) { 2132 xdp.data = page_address(rx_buffer->page) + 2133 rx_buffer->page_offset; 2134 xdp_set_data_meta_invalid(&xdp); 2135 xdp.data_hard_start = xdp.data - 2136 i40e_rx_offset(rx_ring); 2137 xdp.data_end = xdp.data + size; 2138 2139 skb = i40e_run_xdp(rx_ring, &xdp); 2140 } 2141 2142 if (IS_ERR(skb)) { 2143 if (PTR_ERR(skb) == -I40E_XDP_TX) { 2144 xdp_xmit = true; 2145 i40e_rx_buffer_flip(rx_ring, rx_buffer, size); 2146 } else { 2147 rx_buffer->pagecnt_bias++; 2148 } 2149 total_rx_bytes += size; 2150 total_rx_packets++; 2151 } else if (skb) { 2152 i40e_add_rx_frag(rx_ring, rx_buffer, skb, size); 2153 } else if (ring_uses_build_skb(rx_ring)) { 2154 skb = i40e_build_skb(rx_ring, rx_buffer, &xdp); 2155 } else { 2156 skb = i40e_construct_skb(rx_ring, rx_buffer, &xdp); 2157 } 2158 2159 /* exit if we failed to retrieve a buffer */ 2160 if (!skb) { 2161 rx_ring->rx_stats.alloc_buff_failed++; 2162 rx_buffer->pagecnt_bias++; 2163 break; 2164 } 2165 2166 i40e_put_rx_buffer(rx_ring, rx_buffer); 2167 cleaned_count++; 2168 2169 if (i40e_is_non_eop(rx_ring, rx_desc, skb)) 2170 continue; 2171 2172 if (i40e_cleanup_headers(rx_ring, skb, rx_desc)) { 2173 skb = NULL; 2174 continue; 2175 } 2176 2177 /* probably a little skewed due to removing CRC */ 2178 total_rx_bytes += skb->len; 2179 2180 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 2181 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> 2182 I40E_RXD_QW1_PTYPE_SHIFT; 2183 2184 /* populate checksum, VLAN, and protocol */ 2185 i40e_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); 2186 2187 vlan_tag = (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) ? 2188 le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0; 2189 2190 i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb); 2191 i40e_receive_skb(rx_ring, skb, vlan_tag); 2192 skb = NULL; 2193 2194 /* update budget accounting */ 2195 total_rx_packets++; 2196 } 2197 2198 if (xdp_xmit) { 2199 struct i40e_ring *xdp_ring; 2200 2201 xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; 2202 2203 /* Force memory writes to complete before letting h/w 2204 * know there are new descriptors to fetch. 2205 */ 2206 wmb(); 2207 2208 writel(xdp_ring->next_to_use, xdp_ring->tail); 2209 } 2210 2211 rx_ring->skb = skb; 2212 2213 u64_stats_update_begin(&rx_ring->syncp); 2214 rx_ring->stats.packets += total_rx_packets; 2215 rx_ring->stats.bytes += total_rx_bytes; 2216 u64_stats_update_end(&rx_ring->syncp); 2217 rx_ring->q_vector->rx.total_packets += total_rx_packets; 2218 rx_ring->q_vector->rx.total_bytes += total_rx_bytes; 2219 2220 /* guarantee a trip back through this routine if there was a failure */ 2221 return failure ? budget : (int)total_rx_packets; 2222 } 2223 2224 static u32 i40e_buildreg_itr(const int type, const u16 itr) 2225 { 2226 u32 val; 2227 2228 val = I40E_PFINT_DYN_CTLN_INTENA_MASK | 2229 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 2230 (type << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | 2231 (itr << I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT); 2232 2233 return val; 2234 } 2235 2236 /* a small macro to shorten up some long lines */ 2237 #define INTREG I40E_PFINT_DYN_CTLN 2238 static inline int get_rx_itr(struct i40e_vsi *vsi, int idx) 2239 { 2240 return vsi->rx_rings[idx]->rx_itr_setting; 2241 } 2242 2243 static inline int get_tx_itr(struct i40e_vsi *vsi, int idx) 2244 { 2245 return vsi->tx_rings[idx]->tx_itr_setting; 2246 } 2247 2248 /** 2249 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt 2250 * @vsi: the VSI we care about 2251 * @q_vector: q_vector for which itr is being updated and interrupt enabled 2252 * 2253 **/ 2254 static inline void i40e_update_enable_itr(struct i40e_vsi *vsi, 2255 struct i40e_q_vector *q_vector) 2256 { 2257 struct i40e_hw *hw = &vsi->back->hw; 2258 bool rx = false, tx = false; 2259 u32 rxval, txval; 2260 int vector; 2261 int idx = q_vector->v_idx; 2262 int rx_itr_setting, tx_itr_setting; 2263 2264 /* If we don't have MSIX, then we only need to re-enable icr0 */ 2265 if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED)) { 2266 i40e_irq_dynamic_enable_icr0(vsi->back); 2267 return; 2268 } 2269 2270 vector = (q_vector->v_idx + vsi->base_vector); 2271 2272 /* avoid dynamic calculation if in countdown mode OR if 2273 * all dynamic is disabled 2274 */ 2275 rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0); 2276 2277 rx_itr_setting = get_rx_itr(vsi, idx); 2278 tx_itr_setting = get_tx_itr(vsi, idx); 2279 2280 if (q_vector->itr_countdown > 0 || 2281 (!ITR_IS_DYNAMIC(rx_itr_setting) && 2282 !ITR_IS_DYNAMIC(tx_itr_setting))) { 2283 goto enable_int; 2284 } 2285 2286 if (ITR_IS_DYNAMIC(rx_itr_setting)) { 2287 rx = i40e_set_new_dynamic_itr(&q_vector->rx); 2288 rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr); 2289 } 2290 2291 if (ITR_IS_DYNAMIC(tx_itr_setting)) { 2292 tx = i40e_set_new_dynamic_itr(&q_vector->tx); 2293 txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr); 2294 } 2295 2296 if (rx || tx) { 2297 /* get the higher of the two ITR adjustments and 2298 * use the same value for both ITR registers 2299 * when in adaptive mode (Rx and/or Tx) 2300 */ 2301 u16 itr = max(q_vector->tx.itr, q_vector->rx.itr); 2302 2303 q_vector->tx.itr = q_vector->rx.itr = itr; 2304 txval = i40e_buildreg_itr(I40E_TX_ITR, itr); 2305 tx = true; 2306 rxval = i40e_buildreg_itr(I40E_RX_ITR, itr); 2307 rx = true; 2308 } 2309 2310 /* only need to enable the interrupt once, but need 2311 * to possibly update both ITR values 2312 */ 2313 if (rx) { 2314 /* set the INTENA_MSK_MASK so that this first write 2315 * won't actually enable the interrupt, instead just 2316 * updating the ITR (it's bit 31 PF and VF) 2317 */ 2318 rxval |= BIT(31); 2319 /* don't check _DOWN because interrupt isn't being enabled */ 2320 wr32(hw, INTREG(vector - 1), rxval); 2321 } 2322 2323 enable_int: 2324 if (!test_bit(__I40E_VSI_DOWN, vsi->state)) 2325 wr32(hw, INTREG(vector - 1), txval); 2326 2327 if (q_vector->itr_countdown) 2328 q_vector->itr_countdown--; 2329 else 2330 q_vector->itr_countdown = ITR_COUNTDOWN_START; 2331 } 2332 2333 /** 2334 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine 2335 * @napi: napi struct with our devices info in it 2336 * @budget: amount of work driver is allowed to do this pass, in packets 2337 * 2338 * This function will clean all queues associated with a q_vector. 2339 * 2340 * Returns the amount of work done 2341 **/ 2342 int i40e_napi_poll(struct napi_struct *napi, int budget) 2343 { 2344 struct i40e_q_vector *q_vector = 2345 container_of(napi, struct i40e_q_vector, napi); 2346 struct i40e_vsi *vsi = q_vector->vsi; 2347 struct i40e_ring *ring; 2348 bool clean_complete = true; 2349 bool arm_wb = false; 2350 int budget_per_ring; 2351 int work_done = 0; 2352 2353 if (test_bit(__I40E_VSI_DOWN, vsi->state)) { 2354 napi_complete(napi); 2355 return 0; 2356 } 2357 2358 /* Since the actual Tx work is minimal, we can give the Tx a larger 2359 * budget and be more aggressive about cleaning up the Tx descriptors. 2360 */ 2361 i40e_for_each_ring(ring, q_vector->tx) { 2362 if (!i40e_clean_tx_irq(vsi, ring, budget)) { 2363 clean_complete = false; 2364 continue; 2365 } 2366 arm_wb |= ring->arm_wb; 2367 ring->arm_wb = false; 2368 } 2369 2370 /* Handle case where we are called by netpoll with a budget of 0 */ 2371 if (budget <= 0) 2372 goto tx_only; 2373 2374 /* We attempt to distribute budget to each Rx queue fairly, but don't 2375 * allow the budget to go below 1 because that would exit polling early. 2376 */ 2377 budget_per_ring = max(budget/q_vector->num_ringpairs, 1); 2378 2379 i40e_for_each_ring(ring, q_vector->rx) { 2380 int cleaned = i40e_clean_rx_irq(ring, budget_per_ring); 2381 2382 work_done += cleaned; 2383 /* if we clean as many as budgeted, we must not be done */ 2384 if (cleaned >= budget_per_ring) 2385 clean_complete = false; 2386 } 2387 2388 /* If work not completed, return budget and polling will return */ 2389 if (!clean_complete) { 2390 int cpu_id = smp_processor_id(); 2391 2392 /* It is possible that the interrupt affinity has changed but, 2393 * if the cpu is pegged at 100%, polling will never exit while 2394 * traffic continues and the interrupt will be stuck on this 2395 * cpu. We check to make sure affinity is correct before we 2396 * continue to poll, otherwise we must stop polling so the 2397 * interrupt can move to the correct cpu. 2398 */ 2399 if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) { 2400 /* Tell napi that we are done polling */ 2401 napi_complete_done(napi, work_done); 2402 2403 /* Force an interrupt */ 2404 i40e_force_wb(vsi, q_vector); 2405 2406 /* Return budget-1 so that polling stops */ 2407 return budget - 1; 2408 } 2409 tx_only: 2410 if (arm_wb) { 2411 q_vector->tx.ring[0].tx_stats.tx_force_wb++; 2412 i40e_enable_wb_on_itr(vsi, q_vector); 2413 } 2414 return budget; 2415 } 2416 2417 if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR) 2418 q_vector->arm_wb_state = false; 2419 2420 /* Work is done so exit the polling mode and re-enable the interrupt */ 2421 napi_complete_done(napi, work_done); 2422 2423 i40e_update_enable_itr(vsi, q_vector); 2424 2425 return min(work_done, budget - 1); 2426 } 2427 2428 /** 2429 * i40e_atr - Add a Flow Director ATR filter 2430 * @tx_ring: ring to add programming descriptor to 2431 * @skb: send buffer 2432 * @tx_flags: send tx flags 2433 **/ 2434 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb, 2435 u32 tx_flags) 2436 { 2437 struct i40e_filter_program_desc *fdir_desc; 2438 struct i40e_pf *pf = tx_ring->vsi->back; 2439 union { 2440 unsigned char *network; 2441 struct iphdr *ipv4; 2442 struct ipv6hdr *ipv6; 2443 } hdr; 2444 struct tcphdr *th; 2445 unsigned int hlen; 2446 u32 flex_ptype, dtype_cmd; 2447 int l4_proto; 2448 u16 i; 2449 2450 /* make sure ATR is enabled */ 2451 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED)) 2452 return; 2453 2454 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) 2455 return; 2456 2457 /* if sampling is disabled do nothing */ 2458 if (!tx_ring->atr_sample_rate) 2459 return; 2460 2461 /* Currently only IPv4/IPv6 with TCP is supported */ 2462 if (!(tx_flags & (I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6))) 2463 return; 2464 2465 /* snag network header to get L4 type and address */ 2466 hdr.network = (tx_flags & I40E_TX_FLAGS_UDP_TUNNEL) ? 2467 skb_inner_network_header(skb) : skb_network_header(skb); 2468 2469 /* Note: tx_flags gets modified to reflect inner protocols in 2470 * tx_enable_csum function if encap is enabled. 2471 */ 2472 if (tx_flags & I40E_TX_FLAGS_IPV4) { 2473 /* access ihl as u8 to avoid unaligned access on ia64 */ 2474 hlen = (hdr.network[0] & 0x0F) << 2; 2475 l4_proto = hdr.ipv4->protocol; 2476 } else { 2477 /* find the start of the innermost ipv6 header */ 2478 unsigned int inner_hlen = hdr.network - skb->data; 2479 unsigned int h_offset = inner_hlen; 2480 2481 /* this function updates h_offset to the end of the header */ 2482 l4_proto = 2483 ipv6_find_hdr(skb, &h_offset, IPPROTO_TCP, NULL, NULL); 2484 /* hlen will contain our best estimate of the tcp header */ 2485 hlen = h_offset - inner_hlen; 2486 } 2487 2488 if (l4_proto != IPPROTO_TCP) 2489 return; 2490 2491 th = (struct tcphdr *)(hdr.network + hlen); 2492 2493 /* Due to lack of space, no more new filters can be programmed */ 2494 if (th->syn && (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED)) 2495 return; 2496 if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) { 2497 /* HW ATR eviction will take care of removing filters on FIN 2498 * and RST packets. 2499 */ 2500 if (th->fin || th->rst) 2501 return; 2502 } 2503 2504 tx_ring->atr_count++; 2505 2506 /* sample on all syn/fin/rst packets or once every atr sample rate */ 2507 if (!th->fin && 2508 !th->syn && 2509 !th->rst && 2510 (tx_ring->atr_count < tx_ring->atr_sample_rate)) 2511 return; 2512 2513 tx_ring->atr_count = 0; 2514 2515 /* grab the next descriptor */ 2516 i = tx_ring->next_to_use; 2517 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i); 2518 2519 i++; 2520 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2521 2522 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 2523 I40E_TXD_FLTR_QW0_QINDEX_MASK; 2524 flex_ptype |= (tx_flags & I40E_TX_FLAGS_IPV4) ? 2525 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << 2526 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : 2527 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << 2528 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 2529 2530 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; 2531 2532 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG; 2533 2534 dtype_cmd |= (th->fin || th->rst) ? 2535 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 2536 I40E_TXD_FLTR_QW1_PCMD_SHIFT) : 2537 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 2538 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 2539 2540 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << 2541 I40E_TXD_FLTR_QW1_DEST_SHIFT; 2542 2543 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << 2544 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; 2545 2546 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK; 2547 if (!(tx_flags & I40E_TX_FLAGS_UDP_TUNNEL)) 2548 dtype_cmd |= 2549 ((u32)I40E_FD_ATR_STAT_IDX(pf->hw.pf_id) << 2550 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 2551 I40E_TXD_FLTR_QW1_CNTINDEX_MASK; 2552 else 2553 dtype_cmd |= 2554 ((u32)I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id) << 2555 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 2556 I40E_TXD_FLTR_QW1_CNTINDEX_MASK; 2557 2558 if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) 2559 dtype_cmd |= I40E_TXD_FLTR_QW1_ATR_MASK; 2560 2561 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype); 2562 fdir_desc->rsvd = cpu_to_le32(0); 2563 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd); 2564 fdir_desc->fd_id = cpu_to_le32(0); 2565 } 2566 2567 /** 2568 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW 2569 * @skb: send buffer 2570 * @tx_ring: ring to send buffer on 2571 * @flags: the tx flags to be set 2572 * 2573 * Checks the skb and set up correspondingly several generic transmit flags 2574 * related to VLAN tagging for the HW, such as VLAN, DCB, etc. 2575 * 2576 * Returns error code indicate the frame should be dropped upon error and the 2577 * otherwise returns 0 to indicate the flags has been set properly. 2578 **/ 2579 static inline int i40e_tx_prepare_vlan_flags(struct sk_buff *skb, 2580 struct i40e_ring *tx_ring, 2581 u32 *flags) 2582 { 2583 __be16 protocol = skb->protocol; 2584 u32 tx_flags = 0; 2585 2586 if (protocol == htons(ETH_P_8021Q) && 2587 !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { 2588 /* When HW VLAN acceleration is turned off by the user the 2589 * stack sets the protocol to 8021q so that the driver 2590 * can take any steps required to support the SW only 2591 * VLAN handling. In our case the driver doesn't need 2592 * to take any further steps so just set the protocol 2593 * to the encapsulated ethertype. 2594 */ 2595 skb->protocol = vlan_get_protocol(skb); 2596 goto out; 2597 } 2598 2599 /* if we have a HW VLAN tag being added, default to the HW one */ 2600 if (skb_vlan_tag_present(skb)) { 2601 tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT; 2602 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 2603 /* else if it is a SW VLAN, check the next protocol and store the tag */ 2604 } else if (protocol == htons(ETH_P_8021Q)) { 2605 struct vlan_hdr *vhdr, _vhdr; 2606 2607 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 2608 if (!vhdr) 2609 return -EINVAL; 2610 2611 protocol = vhdr->h_vlan_encapsulated_proto; 2612 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT; 2613 tx_flags |= I40E_TX_FLAGS_SW_VLAN; 2614 } 2615 2616 if (!(tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED)) 2617 goto out; 2618 2619 /* Insert 802.1p priority into VLAN header */ 2620 if ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) || 2621 (skb->priority != TC_PRIO_CONTROL)) { 2622 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK; 2623 tx_flags |= (skb->priority & 0x7) << 2624 I40E_TX_FLAGS_VLAN_PRIO_SHIFT; 2625 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) { 2626 struct vlan_ethhdr *vhdr; 2627 int rc; 2628 2629 rc = skb_cow_head(skb, 0); 2630 if (rc < 0) 2631 return rc; 2632 vhdr = (struct vlan_ethhdr *)skb->data; 2633 vhdr->h_vlan_TCI = htons(tx_flags >> 2634 I40E_TX_FLAGS_VLAN_SHIFT); 2635 } else { 2636 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 2637 } 2638 } 2639 2640 out: 2641 *flags = tx_flags; 2642 return 0; 2643 } 2644 2645 /** 2646 * i40e_tso - set up the tso context descriptor 2647 * @first: pointer to first Tx buffer for xmit 2648 * @hdr_len: ptr to the size of the packet header 2649 * @cd_type_cmd_tso_mss: Quad Word 1 2650 * 2651 * Returns 0 if no TSO can happen, 1 if tso is going, or error 2652 **/ 2653 static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len, 2654 u64 *cd_type_cmd_tso_mss) 2655 { 2656 struct sk_buff *skb = first->skb; 2657 u64 cd_cmd, cd_tso_len, cd_mss; 2658 union { 2659 struct iphdr *v4; 2660 struct ipv6hdr *v6; 2661 unsigned char *hdr; 2662 } ip; 2663 union { 2664 struct tcphdr *tcp; 2665 struct udphdr *udp; 2666 unsigned char *hdr; 2667 } l4; 2668 u32 paylen, l4_offset; 2669 u16 gso_segs, gso_size; 2670 int err; 2671 2672 if (skb->ip_summed != CHECKSUM_PARTIAL) 2673 return 0; 2674 2675 if (!skb_is_gso(skb)) 2676 return 0; 2677 2678 err = skb_cow_head(skb, 0); 2679 if (err < 0) 2680 return err; 2681 2682 ip.hdr = skb_network_header(skb); 2683 l4.hdr = skb_transport_header(skb); 2684 2685 /* initialize outer IP header fields */ 2686 if (ip.v4->version == 4) { 2687 ip.v4->tot_len = 0; 2688 ip.v4->check = 0; 2689 } else { 2690 ip.v6->payload_len = 0; 2691 } 2692 2693 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 2694 SKB_GSO_GRE_CSUM | 2695 SKB_GSO_IPXIP4 | 2696 SKB_GSO_IPXIP6 | 2697 SKB_GSO_UDP_TUNNEL | 2698 SKB_GSO_UDP_TUNNEL_CSUM)) { 2699 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && 2700 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) { 2701 l4.udp->len = 0; 2702 2703 /* determine offset of outer transport header */ 2704 l4_offset = l4.hdr - skb->data; 2705 2706 /* remove payload length from outer checksum */ 2707 paylen = skb->len - l4_offset; 2708 csum_replace_by_diff(&l4.udp->check, 2709 (__force __wsum)htonl(paylen)); 2710 } 2711 2712 /* reset pointers to inner headers */ 2713 ip.hdr = skb_inner_network_header(skb); 2714 l4.hdr = skb_inner_transport_header(skb); 2715 2716 /* initialize inner IP header fields */ 2717 if (ip.v4->version == 4) { 2718 ip.v4->tot_len = 0; 2719 ip.v4->check = 0; 2720 } else { 2721 ip.v6->payload_len = 0; 2722 } 2723 } 2724 2725 /* determine offset of inner transport header */ 2726 l4_offset = l4.hdr - skb->data; 2727 2728 /* remove payload length from inner checksum */ 2729 paylen = skb->len - l4_offset; 2730 csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); 2731 2732 /* compute length of segmentation header */ 2733 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 2734 2735 /* pull values out of skb_shinfo */ 2736 gso_size = skb_shinfo(skb)->gso_size; 2737 gso_segs = skb_shinfo(skb)->gso_segs; 2738 2739 /* update GSO size and bytecount with header size */ 2740 first->gso_segs = gso_segs; 2741 first->bytecount += (first->gso_segs - 1) * *hdr_len; 2742 2743 /* find the field values */ 2744 cd_cmd = I40E_TX_CTX_DESC_TSO; 2745 cd_tso_len = skb->len - *hdr_len; 2746 cd_mss = gso_size; 2747 *cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | 2748 (cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | 2749 (cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT); 2750 return 1; 2751 } 2752 2753 /** 2754 * i40e_tsyn - set up the tsyn context descriptor 2755 * @tx_ring: ptr to the ring to send 2756 * @skb: ptr to the skb we're sending 2757 * @tx_flags: the collected send information 2758 * @cd_type_cmd_tso_mss: Quad Word 1 2759 * 2760 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen 2761 **/ 2762 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb, 2763 u32 tx_flags, u64 *cd_type_cmd_tso_mss) 2764 { 2765 struct i40e_pf *pf; 2766 2767 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) 2768 return 0; 2769 2770 /* Tx timestamps cannot be sampled when doing TSO */ 2771 if (tx_flags & I40E_TX_FLAGS_TSO) 2772 return 0; 2773 2774 /* only timestamp the outbound packet if the user has requested it and 2775 * we are not already transmitting a packet to be timestamped 2776 */ 2777 pf = i40e_netdev_to_pf(tx_ring->netdev); 2778 if (!(pf->flags & I40E_FLAG_PTP)) 2779 return 0; 2780 2781 if (pf->ptp_tx && 2782 !test_and_set_bit_lock(__I40E_PTP_TX_IN_PROGRESS, pf->state)) { 2783 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2784 pf->ptp_tx_start = jiffies; 2785 pf->ptp_tx_skb = skb_get(skb); 2786 } else { 2787 pf->tx_hwtstamp_skipped++; 2788 return 0; 2789 } 2790 2791 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN << 2792 I40E_TXD_CTX_QW1_CMD_SHIFT; 2793 2794 return 1; 2795 } 2796 2797 /** 2798 * i40e_tx_enable_csum - Enable Tx checksum offloads 2799 * @skb: send buffer 2800 * @tx_flags: pointer to Tx flags currently set 2801 * @td_cmd: Tx descriptor command bits to set 2802 * @td_offset: Tx descriptor header offsets to set 2803 * @tx_ring: Tx descriptor ring 2804 * @cd_tunneling: ptr to context desc bits 2805 **/ 2806 static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags, 2807 u32 *td_cmd, u32 *td_offset, 2808 struct i40e_ring *tx_ring, 2809 u32 *cd_tunneling) 2810 { 2811 union { 2812 struct iphdr *v4; 2813 struct ipv6hdr *v6; 2814 unsigned char *hdr; 2815 } ip; 2816 union { 2817 struct tcphdr *tcp; 2818 struct udphdr *udp; 2819 unsigned char *hdr; 2820 } l4; 2821 unsigned char *exthdr; 2822 u32 offset, cmd = 0; 2823 __be16 frag_off; 2824 u8 l4_proto = 0; 2825 2826 if (skb->ip_summed != CHECKSUM_PARTIAL) 2827 return 0; 2828 2829 ip.hdr = skb_network_header(skb); 2830 l4.hdr = skb_transport_header(skb); 2831 2832 /* compute outer L2 header size */ 2833 offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 2834 2835 if (skb->encapsulation) { 2836 u32 tunnel = 0; 2837 /* define outer network header type */ 2838 if (*tx_flags & I40E_TX_FLAGS_IPV4) { 2839 tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ? 2840 I40E_TX_CTX_EXT_IP_IPV4 : 2841 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 2842 2843 l4_proto = ip.v4->protocol; 2844 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { 2845 tunnel |= I40E_TX_CTX_EXT_IP_IPV6; 2846 2847 exthdr = ip.hdr + sizeof(*ip.v6); 2848 l4_proto = ip.v6->nexthdr; 2849 if (l4.hdr != exthdr) 2850 ipv6_skip_exthdr(skb, exthdr - skb->data, 2851 &l4_proto, &frag_off); 2852 } 2853 2854 /* define outer transport */ 2855 switch (l4_proto) { 2856 case IPPROTO_UDP: 2857 tunnel |= I40E_TXD_CTX_UDP_TUNNELING; 2858 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 2859 break; 2860 case IPPROTO_GRE: 2861 tunnel |= I40E_TXD_CTX_GRE_TUNNELING; 2862 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 2863 break; 2864 case IPPROTO_IPIP: 2865 case IPPROTO_IPV6: 2866 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 2867 l4.hdr = skb_inner_network_header(skb); 2868 break; 2869 default: 2870 if (*tx_flags & I40E_TX_FLAGS_TSO) 2871 return -1; 2872 2873 skb_checksum_help(skb); 2874 return 0; 2875 } 2876 2877 /* compute outer L3 header size */ 2878 tunnel |= ((l4.hdr - ip.hdr) / 4) << 2879 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT; 2880 2881 /* switch IP header pointer from outer to inner header */ 2882 ip.hdr = skb_inner_network_header(skb); 2883 2884 /* compute tunnel header size */ 2885 tunnel |= ((ip.hdr - l4.hdr) / 2) << 2886 I40E_TXD_CTX_QW0_NATLEN_SHIFT; 2887 2888 /* indicate if we need to offload outer UDP header */ 2889 if ((*tx_flags & I40E_TX_FLAGS_TSO) && 2890 !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && 2891 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) 2892 tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK; 2893 2894 /* record tunnel offload values */ 2895 *cd_tunneling |= tunnel; 2896 2897 /* switch L4 header pointer from outer to inner */ 2898 l4.hdr = skb_inner_transport_header(skb); 2899 l4_proto = 0; 2900 2901 /* reset type as we transition from outer to inner headers */ 2902 *tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6); 2903 if (ip.v4->version == 4) 2904 *tx_flags |= I40E_TX_FLAGS_IPV4; 2905 if (ip.v6->version == 6) 2906 *tx_flags |= I40E_TX_FLAGS_IPV6; 2907 } 2908 2909 /* Enable IP checksum offloads */ 2910 if (*tx_flags & I40E_TX_FLAGS_IPV4) { 2911 l4_proto = ip.v4->protocol; 2912 /* the stack computes the IP header already, the only time we 2913 * need the hardware to recompute it is in the case of TSO. 2914 */ 2915 cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ? 2916 I40E_TX_DESC_CMD_IIPT_IPV4_CSUM : 2917 I40E_TX_DESC_CMD_IIPT_IPV4; 2918 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { 2919 cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; 2920 2921 exthdr = ip.hdr + sizeof(*ip.v6); 2922 l4_proto = ip.v6->nexthdr; 2923 if (l4.hdr != exthdr) 2924 ipv6_skip_exthdr(skb, exthdr - skb->data, 2925 &l4_proto, &frag_off); 2926 } 2927 2928 /* compute inner L3 header size */ 2929 offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 2930 2931 /* Enable L4 checksum offloads */ 2932 switch (l4_proto) { 2933 case IPPROTO_TCP: 2934 /* enable checksum offloads */ 2935 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 2936 offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2937 break; 2938 case IPPROTO_SCTP: 2939 /* enable SCTP checksum offload */ 2940 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; 2941 offset |= (sizeof(struct sctphdr) >> 2) << 2942 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2943 break; 2944 case IPPROTO_UDP: 2945 /* enable UDP checksum offload */ 2946 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; 2947 offset |= (sizeof(struct udphdr) >> 2) << 2948 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2949 break; 2950 default: 2951 if (*tx_flags & I40E_TX_FLAGS_TSO) 2952 return -1; 2953 skb_checksum_help(skb); 2954 return 0; 2955 } 2956 2957 *td_cmd |= cmd; 2958 *td_offset |= offset; 2959 2960 return 1; 2961 } 2962 2963 /** 2964 * i40e_create_tx_ctx Build the Tx context descriptor 2965 * @tx_ring: ring to create the descriptor on 2966 * @cd_type_cmd_tso_mss: Quad Word 1 2967 * @cd_tunneling: Quad Word 0 - bits 0-31 2968 * @cd_l2tag2: Quad Word 0 - bits 32-63 2969 **/ 2970 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring, 2971 const u64 cd_type_cmd_tso_mss, 2972 const u32 cd_tunneling, const u32 cd_l2tag2) 2973 { 2974 struct i40e_tx_context_desc *context_desc; 2975 int i = tx_ring->next_to_use; 2976 2977 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) && 2978 !cd_tunneling && !cd_l2tag2) 2979 return; 2980 2981 /* grab the next descriptor */ 2982 context_desc = I40E_TX_CTXTDESC(tx_ring, i); 2983 2984 i++; 2985 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2986 2987 /* cpu_to_le32 and assign to struct fields */ 2988 context_desc->tunneling_params = cpu_to_le32(cd_tunneling); 2989 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2); 2990 context_desc->rsvd = cpu_to_le16(0); 2991 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss); 2992 } 2993 2994 /** 2995 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions 2996 * @tx_ring: the ring to be checked 2997 * @size: the size buffer we want to assure is available 2998 * 2999 * Returns -EBUSY if a stop is needed, else 0 3000 **/ 3001 int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) 3002 { 3003 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 3004 /* Memory barrier before checking head and tail */ 3005 smp_mb(); 3006 3007 /* Check again in a case another CPU has just made room available. */ 3008 if (likely(I40E_DESC_UNUSED(tx_ring) < size)) 3009 return -EBUSY; 3010 3011 /* A reprieve! - use start_queue because it doesn't call schedule */ 3012 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 3013 ++tx_ring->tx_stats.restart_queue; 3014 return 0; 3015 } 3016 3017 /** 3018 * __i40e_chk_linearize - Check if there are more than 8 buffers per packet 3019 * @skb: send buffer 3020 * 3021 * Note: Our HW can't DMA more than 8 buffers to build a packet on the wire 3022 * and so we need to figure out the cases where we need to linearize the skb. 3023 * 3024 * For TSO we need to count the TSO header and segment payload separately. 3025 * As such we need to check cases where we have 7 fragments or more as we 3026 * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for 3027 * the segment payload in the first descriptor, and another 7 for the 3028 * fragments. 3029 **/ 3030 bool __i40e_chk_linearize(struct sk_buff *skb) 3031 { 3032 const struct skb_frag_struct *frag, *stale; 3033 int nr_frags, sum; 3034 3035 /* no need to check if number of frags is less than 7 */ 3036 nr_frags = skb_shinfo(skb)->nr_frags; 3037 if (nr_frags < (I40E_MAX_BUFFER_TXD - 1)) 3038 return false; 3039 3040 /* We need to walk through the list and validate that each group 3041 * of 6 fragments totals at least gso_size. 3042 */ 3043 nr_frags -= I40E_MAX_BUFFER_TXD - 2; 3044 frag = &skb_shinfo(skb)->frags[0]; 3045 3046 /* Initialize size to the negative value of gso_size minus 1. We 3047 * use this as the worst case scenerio in which the frag ahead 3048 * of us only provides one byte which is why we are limited to 6 3049 * descriptors for a single transmit as the header and previous 3050 * fragment are already consuming 2 descriptors. 3051 */ 3052 sum = 1 - skb_shinfo(skb)->gso_size; 3053 3054 /* Add size of frags 0 through 4 to create our initial sum */ 3055 sum += skb_frag_size(frag++); 3056 sum += skb_frag_size(frag++); 3057 sum += skb_frag_size(frag++); 3058 sum += skb_frag_size(frag++); 3059 sum += skb_frag_size(frag++); 3060 3061 /* Walk through fragments adding latest fragment, testing it, and 3062 * then removing stale fragments from the sum. 3063 */ 3064 stale = &skb_shinfo(skb)->frags[0]; 3065 for (;;) { 3066 sum += skb_frag_size(frag++); 3067 3068 /* if sum is negative we failed to make sufficient progress */ 3069 if (sum < 0) 3070 return true; 3071 3072 if (!nr_frags--) 3073 break; 3074 3075 sum -= skb_frag_size(stale++); 3076 } 3077 3078 return false; 3079 } 3080 3081 /** 3082 * i40e_tx_map - Build the Tx descriptor 3083 * @tx_ring: ring to send buffer on 3084 * @skb: send buffer 3085 * @first: first buffer info buffer to use 3086 * @tx_flags: collected send information 3087 * @hdr_len: size of the packet header 3088 * @td_cmd: the command field in the descriptor 3089 * @td_offset: offset for checksum or crc 3090 * 3091 * Returns 0 on success, -1 on failure to DMA 3092 **/ 3093 static inline int i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb, 3094 struct i40e_tx_buffer *first, u32 tx_flags, 3095 const u8 hdr_len, u32 td_cmd, u32 td_offset) 3096 { 3097 unsigned int data_len = skb->data_len; 3098 unsigned int size = skb_headlen(skb); 3099 struct skb_frag_struct *frag; 3100 struct i40e_tx_buffer *tx_bi; 3101 struct i40e_tx_desc *tx_desc; 3102 u16 i = tx_ring->next_to_use; 3103 u32 td_tag = 0; 3104 dma_addr_t dma; 3105 u16 desc_count = 1; 3106 3107 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) { 3108 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; 3109 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >> 3110 I40E_TX_FLAGS_VLAN_SHIFT; 3111 } 3112 3113 first->tx_flags = tx_flags; 3114 3115 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3116 3117 tx_desc = I40E_TX_DESC(tx_ring, i); 3118 tx_bi = first; 3119 3120 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 3121 unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED; 3122 3123 if (dma_mapping_error(tx_ring->dev, dma)) 3124 goto dma_error; 3125 3126 /* record length, and DMA address */ 3127 dma_unmap_len_set(tx_bi, len, size); 3128 dma_unmap_addr_set(tx_bi, dma, dma); 3129 3130 /* align size to end of page */ 3131 max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1); 3132 tx_desc->buffer_addr = cpu_to_le64(dma); 3133 3134 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) { 3135 tx_desc->cmd_type_offset_bsz = 3136 build_ctob(td_cmd, td_offset, 3137 max_data, td_tag); 3138 3139 tx_desc++; 3140 i++; 3141 desc_count++; 3142 3143 if (i == tx_ring->count) { 3144 tx_desc = I40E_TX_DESC(tx_ring, 0); 3145 i = 0; 3146 } 3147 3148 dma += max_data; 3149 size -= max_data; 3150 3151 max_data = I40E_MAX_DATA_PER_TXD_ALIGNED; 3152 tx_desc->buffer_addr = cpu_to_le64(dma); 3153 } 3154 3155 if (likely(!data_len)) 3156 break; 3157 3158 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, 3159 size, td_tag); 3160 3161 tx_desc++; 3162 i++; 3163 desc_count++; 3164 3165 if (i == tx_ring->count) { 3166 tx_desc = I40E_TX_DESC(tx_ring, 0); 3167 i = 0; 3168 } 3169 3170 size = skb_frag_size(frag); 3171 data_len -= size; 3172 3173 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3174 DMA_TO_DEVICE); 3175 3176 tx_bi = &tx_ring->tx_bi[i]; 3177 } 3178 3179 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 3180 3181 i++; 3182 if (i == tx_ring->count) 3183 i = 0; 3184 3185 tx_ring->next_to_use = i; 3186 3187 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED); 3188 3189 /* write last descriptor with EOP bit */ 3190 td_cmd |= I40E_TX_DESC_CMD_EOP; 3191 3192 /* We OR these values together to check both against 4 (WB_STRIDE) 3193 * below. This is safe since we don't re-use desc_count afterwards. 3194 */ 3195 desc_count |= ++tx_ring->packet_stride; 3196 3197 if (desc_count >= WB_STRIDE) { 3198 /* write last descriptor with RS bit set */ 3199 td_cmd |= I40E_TX_DESC_CMD_RS; 3200 tx_ring->packet_stride = 0; 3201 } 3202 3203 tx_desc->cmd_type_offset_bsz = 3204 build_ctob(td_cmd, td_offset, size, td_tag); 3205 3206 /* Force memory writes to complete before letting h/w know there 3207 * are new descriptors to fetch. 3208 * 3209 * We also use this memory barrier to make certain all of the 3210 * status bits have been updated before next_to_watch is written. 3211 */ 3212 wmb(); 3213 3214 /* set next_to_watch value indicating a packet is present */ 3215 first->next_to_watch = tx_desc; 3216 3217 /* notify HW of packet */ 3218 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) { 3219 writel(i, tx_ring->tail); 3220 3221 /* we need this if more than one processor can write to our tail 3222 * at a time, it synchronizes IO on IA64/Altix systems 3223 */ 3224 mmiowb(); 3225 } 3226 3227 return 0; 3228 3229 dma_error: 3230 dev_info(tx_ring->dev, "TX DMA map failed\n"); 3231 3232 /* clear dma mappings for failed tx_bi map */ 3233 for (;;) { 3234 tx_bi = &tx_ring->tx_bi[i]; 3235 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi); 3236 if (tx_bi == first) 3237 break; 3238 if (i == 0) 3239 i = tx_ring->count; 3240 i--; 3241 } 3242 3243 tx_ring->next_to_use = i; 3244 3245 return -1; 3246 } 3247 3248 /** 3249 * i40e_xmit_xdp_ring - transmits an XDP buffer to an XDP Tx ring 3250 * @xdp: data to transmit 3251 * @xdp_ring: XDP Tx ring 3252 **/ 3253 static int i40e_xmit_xdp_ring(struct xdp_buff *xdp, 3254 struct i40e_ring *xdp_ring) 3255 { 3256 u32 size = xdp->data_end - xdp->data; 3257 u16 i = xdp_ring->next_to_use; 3258 struct i40e_tx_buffer *tx_bi; 3259 struct i40e_tx_desc *tx_desc; 3260 dma_addr_t dma; 3261 3262 if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) { 3263 xdp_ring->tx_stats.tx_busy++; 3264 return I40E_XDP_CONSUMED; 3265 } 3266 3267 dma = dma_map_single(xdp_ring->dev, xdp->data, size, DMA_TO_DEVICE); 3268 if (dma_mapping_error(xdp_ring->dev, dma)) 3269 return I40E_XDP_CONSUMED; 3270 3271 tx_bi = &xdp_ring->tx_bi[i]; 3272 tx_bi->bytecount = size; 3273 tx_bi->gso_segs = 1; 3274 tx_bi->raw_buf = xdp->data; 3275 3276 /* record length, and DMA address */ 3277 dma_unmap_len_set(tx_bi, len, size); 3278 dma_unmap_addr_set(tx_bi, dma, dma); 3279 3280 tx_desc = I40E_TX_DESC(xdp_ring, i); 3281 tx_desc->buffer_addr = cpu_to_le64(dma); 3282 tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC 3283 | I40E_TXD_CMD, 3284 0, size, 0); 3285 3286 /* Make certain all of the status bits have been updated 3287 * before next_to_watch is written. 3288 */ 3289 smp_wmb(); 3290 3291 i++; 3292 if (i == xdp_ring->count) 3293 i = 0; 3294 3295 tx_bi->next_to_watch = tx_desc; 3296 xdp_ring->next_to_use = i; 3297 3298 return I40E_XDP_TX; 3299 } 3300 3301 /** 3302 * i40e_xmit_frame_ring - Sends buffer on Tx ring 3303 * @skb: send buffer 3304 * @tx_ring: ring to send buffer on 3305 * 3306 * Returns NETDEV_TX_OK if sent, else an error code 3307 **/ 3308 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, 3309 struct i40e_ring *tx_ring) 3310 { 3311 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT; 3312 u32 cd_tunneling = 0, cd_l2tag2 = 0; 3313 struct i40e_tx_buffer *first; 3314 u32 td_offset = 0; 3315 u32 tx_flags = 0; 3316 __be16 protocol; 3317 u32 td_cmd = 0; 3318 u8 hdr_len = 0; 3319 int tso, count; 3320 int tsyn; 3321 3322 /* prefetch the data, we'll need it later */ 3323 prefetch(skb->data); 3324 3325 i40e_trace(xmit_frame_ring, skb, tx_ring); 3326 3327 count = i40e_xmit_descriptor_count(skb); 3328 if (i40e_chk_linearize(skb, count)) { 3329 if (__skb_linearize(skb)) { 3330 dev_kfree_skb_any(skb); 3331 return NETDEV_TX_OK; 3332 } 3333 count = i40e_txd_use_count(skb->len); 3334 tx_ring->tx_stats.tx_linearize++; 3335 } 3336 3337 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, 3338 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, 3339 * + 4 desc gap to avoid the cache line where head is, 3340 * + 1 desc for context descriptor, 3341 * otherwise try next time 3342 */ 3343 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) { 3344 tx_ring->tx_stats.tx_busy++; 3345 return NETDEV_TX_BUSY; 3346 } 3347 3348 /* record the location of the first descriptor for this packet */ 3349 first = &tx_ring->tx_bi[tx_ring->next_to_use]; 3350 first->skb = skb; 3351 first->bytecount = skb->len; 3352 first->gso_segs = 1; 3353 3354 /* prepare the xmit flags */ 3355 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags)) 3356 goto out_drop; 3357 3358 /* obtain protocol of skb */ 3359 protocol = vlan_get_protocol(skb); 3360 3361 /* setup IPv4/IPv6 offloads */ 3362 if (protocol == htons(ETH_P_IP)) 3363 tx_flags |= I40E_TX_FLAGS_IPV4; 3364 else if (protocol == htons(ETH_P_IPV6)) 3365 tx_flags |= I40E_TX_FLAGS_IPV6; 3366 3367 tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss); 3368 3369 if (tso < 0) 3370 goto out_drop; 3371 else if (tso) 3372 tx_flags |= I40E_TX_FLAGS_TSO; 3373 3374 /* Always offload the checksum, since it's in the data descriptor */ 3375 tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset, 3376 tx_ring, &cd_tunneling); 3377 if (tso < 0) 3378 goto out_drop; 3379 3380 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss); 3381 3382 if (tsyn) 3383 tx_flags |= I40E_TX_FLAGS_TSYN; 3384 3385 skb_tx_timestamp(skb); 3386 3387 /* always enable CRC insertion offload */ 3388 td_cmd |= I40E_TX_DESC_CMD_ICRC; 3389 3390 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss, 3391 cd_tunneling, cd_l2tag2); 3392 3393 /* Add Flow Director ATR if it's enabled. 3394 * 3395 * NOTE: this must always be directly before the data descriptor. 3396 */ 3397 i40e_atr(tx_ring, skb, tx_flags); 3398 3399 if (i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len, 3400 td_cmd, td_offset)) 3401 goto cleanup_tx_tstamp; 3402 3403 return NETDEV_TX_OK; 3404 3405 out_drop: 3406 i40e_trace(xmit_frame_ring_drop, first->skb, tx_ring); 3407 dev_kfree_skb_any(first->skb); 3408 first->skb = NULL; 3409 cleanup_tx_tstamp: 3410 if (unlikely(tx_flags & I40E_TX_FLAGS_TSYN)) { 3411 struct i40e_pf *pf = i40e_netdev_to_pf(tx_ring->netdev); 3412 3413 dev_kfree_skb_any(pf->ptp_tx_skb); 3414 pf->ptp_tx_skb = NULL; 3415 clear_bit_unlock(__I40E_PTP_TX_IN_PROGRESS, pf->state); 3416 } 3417 3418 return NETDEV_TX_OK; 3419 } 3420 3421 /** 3422 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer 3423 * @skb: send buffer 3424 * @netdev: network interface device structure 3425 * 3426 * Returns NETDEV_TX_OK if sent, else an error code 3427 **/ 3428 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3429 { 3430 struct i40e_netdev_priv *np = netdev_priv(netdev); 3431 struct i40e_vsi *vsi = np->vsi; 3432 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping]; 3433 3434 /* hardware can't handle really short frames, hardware padding works 3435 * beyond this point 3436 */ 3437 if (skb_put_padto(skb, I40E_MIN_TX_LEN)) 3438 return NETDEV_TX_OK; 3439 3440 return i40e_xmit_frame_ring(skb, tx_ring); 3441 } 3442