1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2015 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include <linux/pci.h> 12 #include <linux/tcp.h> 13 #include <linux/ip.h> 14 #include <linux/in.h> 15 #include <linux/ipv6.h> 16 #include <linux/slab.h> 17 #include <net/ipv6.h> 18 #include <linux/if_ether.h> 19 #include <linux/highmem.h> 20 #include <linux/moduleparam.h> 21 #include <linux/cache.h> 22 #include "net_driver.h" 23 #include "efx.h" 24 #include "io.h" 25 #include "nic.h" 26 #include "tx.h" 27 #include "workarounds.h" 28 #include "ef10_regs.h" 29 30 /* Efx legacy TCP segmentation acceleration. 31 * 32 * Utilises firmware support to go faster than GSO (but not as fast as TSOv2). 33 * 34 * Requires TX checksum offload support. 35 */ 36 37 #define PTR_DIFF(p1, p2) ((u8 *)(p1) - (u8 *)(p2)) 38 39 /** 40 * struct tso_state - TSO state for an SKB 41 * @out_len: Remaining length in current segment 42 * @seqnum: Current sequence number 43 * @ipv4_id: Current IPv4 ID, host endian 44 * @packet_space: Remaining space in current packet 45 * @dma_addr: DMA address of current position 46 * @in_len: Remaining length in current SKB fragment 47 * @unmap_len: Length of SKB fragment 48 * @unmap_addr: DMA address of SKB fragment 49 * @protocol: Network protocol (after any VLAN header) 50 * @ip_off: Offset of IP header 51 * @tcp_off: Offset of TCP header 52 * @header_len: Number of bytes of header 53 * @ip_base_len: IPv4 tot_len or IPv6 payload_len, before TCP payload 54 * @header_dma_addr: Header DMA address 55 * @header_unmap_len: Header DMA mapped length 56 * 57 * The state used during segmentation. It is put into this data structure 58 * just to make it easy to pass into inline functions. 59 */ 60 struct tso_state { 61 /* Output position */ 62 unsigned int out_len; 63 unsigned int seqnum; 64 u16 ipv4_id; 65 unsigned int packet_space; 66 67 /* Input position */ 68 dma_addr_t dma_addr; 69 unsigned int in_len; 70 unsigned int unmap_len; 71 dma_addr_t unmap_addr; 72 73 __be16 protocol; 74 unsigned int ip_off; 75 unsigned int tcp_off; 76 unsigned int header_len; 77 unsigned int ip_base_len; 78 dma_addr_t header_dma_addr; 79 unsigned int header_unmap_len; 80 }; 81 82 static inline void prefetch_ptr(struct efx_tx_queue *tx_queue) 83 { 84 unsigned int insert_ptr = efx_tx_queue_get_insert_index(tx_queue); 85 char *ptr; 86 87 ptr = (char *) (tx_queue->buffer + insert_ptr); 88 prefetch(ptr); 89 prefetch(ptr + 0x80); 90 91 ptr = (char *) (((efx_qword_t *)tx_queue->txd.buf.addr) + insert_ptr); 92 prefetch(ptr); 93 prefetch(ptr + 0x80); 94 } 95 96 /** 97 * efx_tx_queue_insert - push descriptors onto the TX queue 98 * @tx_queue: Efx TX queue 99 * @dma_addr: DMA address of fragment 100 * @len: Length of fragment 101 * @final_buffer: The final buffer inserted into the queue 102 * 103 * Push descriptors onto the TX queue. 104 */ 105 static void efx_tx_queue_insert(struct efx_tx_queue *tx_queue, 106 dma_addr_t dma_addr, unsigned int len, 107 struct efx_tx_buffer **final_buffer) 108 { 109 struct efx_tx_buffer *buffer; 110 unsigned int dma_len; 111 112 EFX_WARN_ON_ONCE_PARANOID(len <= 0); 113 114 while (1) { 115 buffer = efx_tx_queue_get_insert_buffer(tx_queue); 116 ++tx_queue->insert_count; 117 118 EFX_WARN_ON_ONCE_PARANOID(tx_queue->insert_count - 119 tx_queue->read_count >= 120 tx_queue->efx->txq_entries); 121 122 buffer->dma_addr = dma_addr; 123 124 dma_len = tx_queue->efx->type->tx_limit_len(tx_queue, 125 dma_addr, len); 126 127 /* If there's space for everything this is our last buffer. */ 128 if (dma_len >= len) 129 break; 130 131 buffer->len = dma_len; 132 buffer->flags = EFX_TX_BUF_CONT; 133 dma_addr += dma_len; 134 len -= dma_len; 135 } 136 137 EFX_WARN_ON_ONCE_PARANOID(!len); 138 buffer->len = len; 139 *final_buffer = buffer; 140 } 141 142 /* 143 * Verify that our various assumptions about sk_buffs and the conditions 144 * under which TSO will be attempted hold true. Return the protocol number. 145 */ 146 static __be16 efx_tso_check_protocol(struct sk_buff *skb) 147 { 148 __be16 protocol = skb->protocol; 149 150 EFX_WARN_ON_ONCE_PARANOID(((struct ethhdr *)skb->data)->h_proto != 151 protocol); 152 if (protocol == htons(ETH_P_8021Q)) { 153 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 154 155 protocol = veh->h_vlan_encapsulated_proto; 156 } 157 158 if (protocol == htons(ETH_P_IP)) { 159 EFX_WARN_ON_ONCE_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP); 160 } else { 161 EFX_WARN_ON_ONCE_PARANOID(protocol != htons(ETH_P_IPV6)); 162 EFX_WARN_ON_ONCE_PARANOID(ipv6_hdr(skb)->nexthdr != NEXTHDR_TCP); 163 } 164 EFX_WARN_ON_ONCE_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data) + 165 (tcp_hdr(skb)->doff << 2u)) > 166 skb_headlen(skb)); 167 168 return protocol; 169 } 170 171 /* Parse the SKB header and initialise state. */ 172 static int tso_start(struct tso_state *st, struct efx_nic *efx, 173 struct efx_tx_queue *tx_queue, 174 const struct sk_buff *skb) 175 { 176 struct device *dma_dev = &efx->pci_dev->dev; 177 unsigned int header_len, in_len; 178 dma_addr_t dma_addr; 179 180 st->ip_off = skb_network_header(skb) - skb->data; 181 st->tcp_off = skb_transport_header(skb) - skb->data; 182 header_len = st->tcp_off + (tcp_hdr(skb)->doff << 2u); 183 in_len = skb_headlen(skb) - header_len; 184 st->header_len = header_len; 185 st->in_len = in_len; 186 if (st->protocol == htons(ETH_P_IP)) { 187 st->ip_base_len = st->header_len - st->ip_off; 188 st->ipv4_id = ntohs(ip_hdr(skb)->id); 189 } else { 190 st->ip_base_len = st->header_len - st->tcp_off; 191 st->ipv4_id = 0; 192 } 193 st->seqnum = ntohl(tcp_hdr(skb)->seq); 194 195 EFX_WARN_ON_ONCE_PARANOID(tcp_hdr(skb)->urg); 196 EFX_WARN_ON_ONCE_PARANOID(tcp_hdr(skb)->syn); 197 EFX_WARN_ON_ONCE_PARANOID(tcp_hdr(skb)->rst); 198 199 st->out_len = skb->len - header_len; 200 201 dma_addr = dma_map_single(dma_dev, skb->data, 202 skb_headlen(skb), DMA_TO_DEVICE); 203 st->header_dma_addr = dma_addr; 204 st->header_unmap_len = skb_headlen(skb); 205 st->dma_addr = dma_addr + header_len; 206 st->unmap_len = 0; 207 208 return unlikely(dma_mapping_error(dma_dev, dma_addr)) ? -ENOMEM : 0; 209 } 210 211 static int tso_get_fragment(struct tso_state *st, struct efx_nic *efx, 212 skb_frag_t *frag) 213 { 214 st->unmap_addr = skb_frag_dma_map(&efx->pci_dev->dev, frag, 0, 215 skb_frag_size(frag), DMA_TO_DEVICE); 216 if (likely(!dma_mapping_error(&efx->pci_dev->dev, st->unmap_addr))) { 217 st->unmap_len = skb_frag_size(frag); 218 st->in_len = skb_frag_size(frag); 219 st->dma_addr = st->unmap_addr; 220 return 0; 221 } 222 return -ENOMEM; 223 } 224 225 226 /** 227 * tso_fill_packet_with_fragment - form descriptors for the current fragment 228 * @tx_queue: Efx TX queue 229 * @skb: Socket buffer 230 * @st: TSO state 231 * 232 * Form descriptors for the current fragment, until we reach the end 233 * of fragment or end-of-packet. 234 */ 235 static void tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue, 236 const struct sk_buff *skb, 237 struct tso_state *st) 238 { 239 struct efx_tx_buffer *buffer; 240 int n; 241 242 if (st->in_len == 0) 243 return; 244 if (st->packet_space == 0) 245 return; 246 247 EFX_WARN_ON_ONCE_PARANOID(st->in_len <= 0); 248 EFX_WARN_ON_ONCE_PARANOID(st->packet_space <= 0); 249 250 n = min(st->in_len, st->packet_space); 251 252 st->packet_space -= n; 253 st->out_len -= n; 254 st->in_len -= n; 255 256 efx_tx_queue_insert(tx_queue, st->dma_addr, n, &buffer); 257 258 if (st->out_len == 0) { 259 /* Transfer ownership of the skb */ 260 buffer->skb = skb; 261 buffer->flags = EFX_TX_BUF_SKB; 262 } else if (st->packet_space != 0) { 263 buffer->flags = EFX_TX_BUF_CONT; 264 } 265 266 if (st->in_len == 0) { 267 /* Transfer ownership of the DMA mapping */ 268 buffer->unmap_len = st->unmap_len; 269 buffer->dma_offset = buffer->unmap_len - buffer->len; 270 st->unmap_len = 0; 271 } 272 273 st->dma_addr += n; 274 } 275 276 277 #define TCP_FLAGS_OFFSET 13 278 279 /** 280 * tso_start_new_packet - generate a new header and prepare for the new packet 281 * @tx_queue: Efx TX queue 282 * @skb: Socket buffer 283 * @st: TSO state 284 * 285 * Generate a new header and prepare for the new packet. Return 0 on 286 * success, or -%ENOMEM if failed to alloc header, or other negative error. 287 */ 288 static int tso_start_new_packet(struct efx_tx_queue *tx_queue, 289 const struct sk_buff *skb, 290 struct tso_state *st) 291 { 292 struct efx_tx_buffer *buffer = 293 efx_tx_queue_get_insert_buffer(tx_queue); 294 bool is_last = st->out_len <= skb_shinfo(skb)->gso_size; 295 u8 tcp_flags_mask, tcp_flags; 296 297 if (!is_last) { 298 st->packet_space = skb_shinfo(skb)->gso_size; 299 tcp_flags_mask = 0x09; /* mask out FIN and PSH */ 300 } else { 301 st->packet_space = st->out_len; 302 tcp_flags_mask = 0x00; 303 } 304 305 if (WARN_ON(!st->header_unmap_len)) 306 return -EINVAL; 307 /* Send the original headers with a TSO option descriptor 308 * in front 309 */ 310 tcp_flags = ((u8 *)tcp_hdr(skb))[TCP_FLAGS_OFFSET] & ~tcp_flags_mask; 311 312 buffer->flags = EFX_TX_BUF_OPTION; 313 buffer->len = 0; 314 buffer->unmap_len = 0; 315 EFX_POPULATE_QWORD_5(buffer->option, 316 ESF_DZ_TX_DESC_IS_OPT, 1, 317 ESF_DZ_TX_OPTION_TYPE, 318 ESE_DZ_TX_OPTION_DESC_TSO, 319 ESF_DZ_TX_TSO_TCP_FLAGS, tcp_flags, 320 ESF_DZ_TX_TSO_IP_ID, st->ipv4_id, 321 ESF_DZ_TX_TSO_TCP_SEQNO, st->seqnum); 322 ++tx_queue->insert_count; 323 324 /* We mapped the headers in tso_start(). Unmap them 325 * when the last segment is completed. 326 */ 327 buffer = efx_tx_queue_get_insert_buffer(tx_queue); 328 buffer->dma_addr = st->header_dma_addr; 329 buffer->len = st->header_len; 330 if (is_last) { 331 buffer->flags = EFX_TX_BUF_CONT | EFX_TX_BUF_MAP_SINGLE; 332 buffer->unmap_len = st->header_unmap_len; 333 buffer->dma_offset = 0; 334 /* Ensure we only unmap them once in case of a 335 * later DMA mapping error and rollback 336 */ 337 st->header_unmap_len = 0; 338 } else { 339 buffer->flags = EFX_TX_BUF_CONT; 340 buffer->unmap_len = 0; 341 } 342 ++tx_queue->insert_count; 343 344 st->seqnum += skb_shinfo(skb)->gso_size; 345 346 /* Linux leaves suitable gaps in the IP ID space for us to fill. */ 347 ++st->ipv4_id; 348 349 return 0; 350 } 351 352 /** 353 * efx_enqueue_skb_tso - segment and transmit a TSO socket buffer 354 * @tx_queue: Efx TX queue 355 * @skb: Socket buffer 356 * @data_mapped: Did we map the data? Always set to true 357 * by this on success. 358 * 359 * Context: You must hold netif_tx_lock() to call this function. 360 * 361 * Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if 362 * @skb was not enqueued. @skb is consumed unless return value is 363 * %EINVAL. 364 */ 365 int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue, 366 struct sk_buff *skb, 367 bool *data_mapped) 368 { 369 struct efx_nic *efx = tx_queue->efx; 370 int frag_i, rc; 371 struct tso_state state; 372 373 if (tx_queue->tso_version != 1) 374 return -EINVAL; 375 376 prefetch(skb->data); 377 378 /* Find the packet protocol and sanity-check it */ 379 state.protocol = efx_tso_check_protocol(skb); 380 381 EFX_WARN_ON_ONCE_PARANOID(tx_queue->write_count != tx_queue->insert_count); 382 383 rc = tso_start(&state, efx, tx_queue, skb); 384 if (rc) 385 goto fail; 386 387 if (likely(state.in_len == 0)) { 388 /* Grab the first payload fragment. */ 389 EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->nr_frags < 1); 390 frag_i = 0; 391 rc = tso_get_fragment(&state, efx, 392 skb_shinfo(skb)->frags + frag_i); 393 if (rc) 394 goto fail; 395 } else { 396 /* Payload starts in the header area. */ 397 frag_i = -1; 398 } 399 400 rc = tso_start_new_packet(tx_queue, skb, &state); 401 if (rc) 402 goto fail; 403 404 prefetch_ptr(tx_queue); 405 406 while (1) { 407 tso_fill_packet_with_fragment(tx_queue, skb, &state); 408 409 /* Move onto the next fragment? */ 410 if (state.in_len == 0) { 411 if (++frag_i >= skb_shinfo(skb)->nr_frags) 412 /* End of payload reached. */ 413 break; 414 rc = tso_get_fragment(&state, efx, 415 skb_shinfo(skb)->frags + frag_i); 416 if (rc) 417 goto fail; 418 } 419 420 /* Start at new packet? */ 421 if (state.packet_space == 0) { 422 rc = tso_start_new_packet(tx_queue, skb, &state); 423 if (rc) 424 goto fail; 425 } 426 } 427 428 *data_mapped = true; 429 430 return 0; 431 432 fail: 433 if (rc == -ENOMEM) 434 netif_err(efx, tx_err, efx->net_dev, 435 "Out of memory for TSO headers, or DMA mapping error\n"); 436 else 437 netif_err(efx, tx_err, efx->net_dev, "TSO failed, rc = %d\n", rc); 438 439 /* Free the DMA mapping we were in the process of writing out */ 440 if (state.unmap_len) { 441 dma_unmap_page(&efx->pci_dev->dev, state.unmap_addr, 442 state.unmap_len, DMA_TO_DEVICE); 443 } 444 445 /* Free the header DMA mapping */ 446 if (state.header_unmap_len) 447 dma_unmap_single(&efx->pci_dev->dev, state.header_dma_addr, 448 state.header_unmap_len, DMA_TO_DEVICE); 449 450 return rc; 451 } 452