1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
3
4 #include <linux/etherdevice.h>
5 #include <net/ip6_checksum.h>
6 #include <net/page_pool/helpers.h>
7 #include <net/inet_ecn.h>
8 #include <linux/iopoll.h>
9 #include <linux/sctp.h>
10 #include <linux/pci.h>
11 #include <net/tcp.h>
12 #include <net/ip.h>
13
14 #include "wx_type.h"
15 #include "wx_lib.h"
16 #include "wx_hw.h"
17
18 /* Lookup table mapping the HW PTYPE to the bit field for decoding */
19 static struct wx_dec_ptype wx_ptype_lookup[256] = {
20 /* L2: mac */
21 [0x11] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
22 [0x12] = WX_PTT(L2, NONE, NONE, NONE, TS, PAY2),
23 [0x13] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
24 [0x14] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
25 [0x15] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
26 [0x16] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
27 [0x17] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
28
29 /* L2: ethertype filter */
30 [0x18 ... 0x1F] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
31
32 /* L3: ip non-tunnel */
33 [0x21] = WX_PTT(IP, FGV4, NONE, NONE, NONE, PAY3),
34 [0x22] = WX_PTT(IP, IPV4, NONE, NONE, NONE, PAY3),
35 [0x23] = WX_PTT(IP, IPV4, NONE, NONE, UDP, PAY4),
36 [0x24] = WX_PTT(IP, IPV4, NONE, NONE, TCP, PAY4),
37 [0x25] = WX_PTT(IP, IPV4, NONE, NONE, SCTP, PAY4),
38 [0x29] = WX_PTT(IP, FGV6, NONE, NONE, NONE, PAY3),
39 [0x2A] = WX_PTT(IP, IPV6, NONE, NONE, NONE, PAY3),
40 [0x2B] = WX_PTT(IP, IPV6, NONE, NONE, UDP, PAY3),
41 [0x2C] = WX_PTT(IP, IPV6, NONE, NONE, TCP, PAY4),
42 [0x2D] = WX_PTT(IP, IPV6, NONE, NONE, SCTP, PAY4),
43
44 /* L2: fcoe */
45 [0x30 ... 0x34] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
46 [0x38 ... 0x3C] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
47
48 /* IPv4 --> IPv4/IPv6 */
49 [0x81] = WX_PTT(IP, IPV4, IPIP, FGV4, NONE, PAY3),
50 [0x82] = WX_PTT(IP, IPV4, IPIP, IPV4, NONE, PAY3),
51 [0x83] = WX_PTT(IP, IPV4, IPIP, IPV4, UDP, PAY4),
52 [0x84] = WX_PTT(IP, IPV4, IPIP, IPV4, TCP, PAY4),
53 [0x85] = WX_PTT(IP, IPV4, IPIP, IPV4, SCTP, PAY4),
54 [0x89] = WX_PTT(IP, IPV4, IPIP, FGV6, NONE, PAY3),
55 [0x8A] = WX_PTT(IP, IPV4, IPIP, IPV6, NONE, PAY3),
56 [0x8B] = WX_PTT(IP, IPV4, IPIP, IPV6, UDP, PAY4),
57 [0x8C] = WX_PTT(IP, IPV4, IPIP, IPV6, TCP, PAY4),
58 [0x8D] = WX_PTT(IP, IPV4, IPIP, IPV6, SCTP, PAY4),
59
60 /* IPv4 --> GRE/NAT --> NONE/IPv4/IPv6 */
61 [0x90] = WX_PTT(IP, IPV4, IG, NONE, NONE, PAY3),
62 [0x91] = WX_PTT(IP, IPV4, IG, FGV4, NONE, PAY3),
63 [0x92] = WX_PTT(IP, IPV4, IG, IPV4, NONE, PAY3),
64 [0x93] = WX_PTT(IP, IPV4, IG, IPV4, UDP, PAY4),
65 [0x94] = WX_PTT(IP, IPV4, IG, IPV4, TCP, PAY4),
66 [0x95] = WX_PTT(IP, IPV4, IG, IPV4, SCTP, PAY4),
67 [0x99] = WX_PTT(IP, IPV4, IG, FGV6, NONE, PAY3),
68 [0x9A] = WX_PTT(IP, IPV4, IG, IPV6, NONE, PAY3),
69 [0x9B] = WX_PTT(IP, IPV4, IG, IPV6, UDP, PAY4),
70 [0x9C] = WX_PTT(IP, IPV4, IG, IPV6, TCP, PAY4),
71 [0x9D] = WX_PTT(IP, IPV4, IG, IPV6, SCTP, PAY4),
72
73 /* IPv4 --> GRE/NAT --> MAC --> NONE/IPv4/IPv6 */
74 [0xA0] = WX_PTT(IP, IPV4, IGM, NONE, NONE, PAY3),
75 [0xA1] = WX_PTT(IP, IPV4, IGM, FGV4, NONE, PAY3),
76 [0xA2] = WX_PTT(IP, IPV4, IGM, IPV4, NONE, PAY3),
77 [0xA3] = WX_PTT(IP, IPV4, IGM, IPV4, UDP, PAY4),
78 [0xA4] = WX_PTT(IP, IPV4, IGM, IPV4, TCP, PAY4),
79 [0xA5] = WX_PTT(IP, IPV4, IGM, IPV4, SCTP, PAY4),
80 [0xA9] = WX_PTT(IP, IPV4, IGM, FGV6, NONE, PAY3),
81 [0xAA] = WX_PTT(IP, IPV4, IGM, IPV6, NONE, PAY3),
82 [0xAB] = WX_PTT(IP, IPV4, IGM, IPV6, UDP, PAY4),
83 [0xAC] = WX_PTT(IP, IPV4, IGM, IPV6, TCP, PAY4),
84 [0xAD] = WX_PTT(IP, IPV4, IGM, IPV6, SCTP, PAY4),
85
86 /* IPv4 --> GRE/NAT --> MAC+VLAN --> NONE/IPv4/IPv6 */
87 [0xB0] = WX_PTT(IP, IPV4, IGMV, NONE, NONE, PAY3),
88 [0xB1] = WX_PTT(IP, IPV4, IGMV, FGV4, NONE, PAY3),
89 [0xB2] = WX_PTT(IP, IPV4, IGMV, IPV4, NONE, PAY3),
90 [0xB3] = WX_PTT(IP, IPV4, IGMV, IPV4, UDP, PAY4),
91 [0xB4] = WX_PTT(IP, IPV4, IGMV, IPV4, TCP, PAY4),
92 [0xB5] = WX_PTT(IP, IPV4, IGMV, IPV4, SCTP, PAY4),
93 [0xB9] = WX_PTT(IP, IPV4, IGMV, FGV6, NONE, PAY3),
94 [0xBA] = WX_PTT(IP, IPV4, IGMV, IPV6, NONE, PAY3),
95 [0xBB] = WX_PTT(IP, IPV4, IGMV, IPV6, UDP, PAY4),
96 [0xBC] = WX_PTT(IP, IPV4, IGMV, IPV6, TCP, PAY4),
97 [0xBD] = WX_PTT(IP, IPV4, IGMV, IPV6, SCTP, PAY4),
98
99 /* IPv6 --> IPv4/IPv6 */
100 [0xC1] = WX_PTT(IP, IPV6, IPIP, FGV4, NONE, PAY3),
101 [0xC2] = WX_PTT(IP, IPV6, IPIP, IPV4, NONE, PAY3),
102 [0xC3] = WX_PTT(IP, IPV6, IPIP, IPV4, UDP, PAY4),
103 [0xC4] = WX_PTT(IP, IPV6, IPIP, IPV4, TCP, PAY4),
104 [0xC5] = WX_PTT(IP, IPV6, IPIP, IPV4, SCTP, PAY4),
105 [0xC9] = WX_PTT(IP, IPV6, IPIP, FGV6, NONE, PAY3),
106 [0xCA] = WX_PTT(IP, IPV6, IPIP, IPV6, NONE, PAY3),
107 [0xCB] = WX_PTT(IP, IPV6, IPIP, IPV6, UDP, PAY4),
108 [0xCC] = WX_PTT(IP, IPV6, IPIP, IPV6, TCP, PAY4),
109 [0xCD] = WX_PTT(IP, IPV6, IPIP, IPV6, SCTP, PAY4),
110
111 /* IPv6 --> GRE/NAT -> NONE/IPv4/IPv6 */
112 [0xD0] = WX_PTT(IP, IPV6, IG, NONE, NONE, PAY3),
113 [0xD1] = WX_PTT(IP, IPV6, IG, FGV4, NONE, PAY3),
114 [0xD2] = WX_PTT(IP, IPV6, IG, IPV4, NONE, PAY3),
115 [0xD3] = WX_PTT(IP, IPV6, IG, IPV4, UDP, PAY4),
116 [0xD4] = WX_PTT(IP, IPV6, IG, IPV4, TCP, PAY4),
117 [0xD5] = WX_PTT(IP, IPV6, IG, IPV4, SCTP, PAY4),
118 [0xD9] = WX_PTT(IP, IPV6, IG, FGV6, NONE, PAY3),
119 [0xDA] = WX_PTT(IP, IPV6, IG, IPV6, NONE, PAY3),
120 [0xDB] = WX_PTT(IP, IPV6, IG, IPV6, UDP, PAY4),
121 [0xDC] = WX_PTT(IP, IPV6, IG, IPV6, TCP, PAY4),
122 [0xDD] = WX_PTT(IP, IPV6, IG, IPV6, SCTP, PAY4),
123
124 /* IPv6 --> GRE/NAT -> MAC -> NONE/IPv4/IPv6 */
125 [0xE0] = WX_PTT(IP, IPV6, IGM, NONE, NONE, PAY3),
126 [0xE1] = WX_PTT(IP, IPV6, IGM, FGV4, NONE, PAY3),
127 [0xE2] = WX_PTT(IP, IPV6, IGM, IPV4, NONE, PAY3),
128 [0xE3] = WX_PTT(IP, IPV6, IGM, IPV4, UDP, PAY4),
129 [0xE4] = WX_PTT(IP, IPV6, IGM, IPV4, TCP, PAY4),
130 [0xE5] = WX_PTT(IP, IPV6, IGM, IPV4, SCTP, PAY4),
131 [0xE9] = WX_PTT(IP, IPV6, IGM, FGV6, NONE, PAY3),
132 [0xEA] = WX_PTT(IP, IPV6, IGM, IPV6, NONE, PAY3),
133 [0xEB] = WX_PTT(IP, IPV6, IGM, IPV6, UDP, PAY4),
134 [0xEC] = WX_PTT(IP, IPV6, IGM, IPV6, TCP, PAY4),
135 [0xED] = WX_PTT(IP, IPV6, IGM, IPV6, SCTP, PAY4),
136
137 /* IPv6 --> GRE/NAT -> MAC--> NONE/IPv */
138 [0xF0] = WX_PTT(IP, IPV6, IGMV, NONE, NONE, PAY3),
139 [0xF1] = WX_PTT(IP, IPV6, IGMV, FGV4, NONE, PAY3),
140 [0xF2] = WX_PTT(IP, IPV6, IGMV, IPV4, NONE, PAY3),
141 [0xF3] = WX_PTT(IP, IPV6, IGMV, IPV4, UDP, PAY4),
142 [0xF4] = WX_PTT(IP, IPV6, IGMV, IPV4, TCP, PAY4),
143 [0xF5] = WX_PTT(IP, IPV6, IGMV, IPV4, SCTP, PAY4),
144 [0xF9] = WX_PTT(IP, IPV6, IGMV, FGV6, NONE, PAY3),
145 [0xFA] = WX_PTT(IP, IPV6, IGMV, IPV6, NONE, PAY3),
146 [0xFB] = WX_PTT(IP, IPV6, IGMV, IPV6, UDP, PAY4),
147 [0xFC] = WX_PTT(IP, IPV6, IGMV, IPV6, TCP, PAY4),
148 [0xFD] = WX_PTT(IP, IPV6, IGMV, IPV6, SCTP, PAY4),
149 };
150
wx_decode_ptype(const u8 ptype)151 static struct wx_dec_ptype wx_decode_ptype(const u8 ptype)
152 {
153 return wx_ptype_lookup[ptype];
154 }
155
156 /* wx_test_staterr - tests bits in Rx descriptor status and error fields */
wx_test_staterr(union wx_rx_desc * rx_desc,const u32 stat_err_bits)157 static __le32 wx_test_staterr(union wx_rx_desc *rx_desc,
158 const u32 stat_err_bits)
159 {
160 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);
161 }
162
wx_dma_sync_frag(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer)163 static void wx_dma_sync_frag(struct wx_ring *rx_ring,
164 struct wx_rx_buffer *rx_buffer)
165 {
166 struct sk_buff *skb = rx_buffer->skb;
167 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
168
169 dma_sync_single_range_for_cpu(rx_ring->dev,
170 WX_CB(skb)->dma,
171 skb_frag_off(frag),
172 skb_frag_size(frag),
173 DMA_FROM_DEVICE);
174
175 /* If the page was released, just unmap it. */
176 if (unlikely(WX_CB(skb)->page_released))
177 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
178 }
179
wx_get_rx_buffer(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff ** skb,int * rx_buffer_pgcnt)180 static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring,
181 union wx_rx_desc *rx_desc,
182 struct sk_buff **skb,
183 int *rx_buffer_pgcnt)
184 {
185 struct wx_rx_buffer *rx_buffer;
186 unsigned int size;
187
188 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
189 size = le16_to_cpu(rx_desc->wb.upper.length);
190
191 #if (PAGE_SIZE < 8192)
192 *rx_buffer_pgcnt = page_count(rx_buffer->page);
193 #else
194 *rx_buffer_pgcnt = 0;
195 #endif
196
197 prefetchw(rx_buffer->page);
198 *skb = rx_buffer->skb;
199
200 /* Delay unmapping of the first packet. It carries the header
201 * information, HW may still access the header after the writeback.
202 * Only unmap it when EOP is reached
203 */
204 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) {
205 if (!*skb)
206 goto skip_sync;
207 } else {
208 if (*skb)
209 wx_dma_sync_frag(rx_ring, rx_buffer);
210 }
211
212 /* we are reusing so sync this buffer for CPU use */
213 dma_sync_single_range_for_cpu(rx_ring->dev,
214 rx_buffer->dma,
215 rx_buffer->page_offset,
216 size,
217 DMA_FROM_DEVICE);
218 skip_sync:
219 return rx_buffer;
220 }
221
wx_put_rx_buffer(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,struct sk_buff * skb,int rx_buffer_pgcnt)222 static void wx_put_rx_buffer(struct wx_ring *rx_ring,
223 struct wx_rx_buffer *rx_buffer,
224 struct sk_buff *skb,
225 int rx_buffer_pgcnt)
226 {
227 if (!IS_ERR(skb) && WX_CB(skb)->dma == rx_buffer->dma)
228 /* the page has been released from the ring */
229 WX_CB(skb)->page_released = true;
230
231 /* clear contents of rx_buffer */
232 rx_buffer->page = NULL;
233 rx_buffer->skb = NULL;
234 }
235
wx_build_skb(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,union wx_rx_desc * rx_desc)236 static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring,
237 struct wx_rx_buffer *rx_buffer,
238 union wx_rx_desc *rx_desc)
239 {
240 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
241 #if (PAGE_SIZE < 8192)
242 unsigned int truesize = WX_RX_BUFSZ;
243 #else
244 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
245 #endif
246 struct sk_buff *skb = rx_buffer->skb;
247
248 if (!skb) {
249 void *page_addr = page_address(rx_buffer->page) +
250 rx_buffer->page_offset;
251
252 /* prefetch first cache line of first page */
253 prefetch(page_addr);
254 #if L1_CACHE_BYTES < 128
255 prefetch(page_addr + L1_CACHE_BYTES);
256 #endif
257
258 /* allocate a skb to store the frags */
259 skb = napi_alloc_skb(&rx_ring->q_vector->napi, WX_RXBUFFER_256);
260 if (unlikely(!skb))
261 return NULL;
262
263 /* we will be copying header into skb->data in
264 * pskb_may_pull so it is in our interest to prefetch
265 * it now to avoid a possible cache miss
266 */
267 prefetchw(skb->data);
268
269 if (size <= WX_RXBUFFER_256) {
270 memcpy(__skb_put(skb, size), page_addr,
271 ALIGN(size, sizeof(long)));
272 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, true);
273 return skb;
274 }
275
276 skb_mark_for_recycle(skb);
277
278 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))
279 WX_CB(skb)->dma = rx_buffer->dma;
280
281 skb_add_rx_frag(skb, 0, rx_buffer->page,
282 rx_buffer->page_offset,
283 size, truesize);
284 goto out;
285
286 } else {
287 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
288 rx_buffer->page_offset, size, truesize);
289 }
290
291 out:
292 #if (PAGE_SIZE < 8192)
293 /* flip page offset to other buffer */
294 rx_buffer->page_offset ^= truesize;
295 #else
296 /* move offset up to the next cache line */
297 rx_buffer->page_offset += truesize;
298 #endif
299
300 return skb;
301 }
302
wx_alloc_mapped_page(struct wx_ring * rx_ring,struct wx_rx_buffer * bi)303 static bool wx_alloc_mapped_page(struct wx_ring *rx_ring,
304 struct wx_rx_buffer *bi)
305 {
306 struct page *page = bi->page;
307 dma_addr_t dma;
308
309 /* since we are recycling buffers we should seldom need to alloc */
310 if (likely(page))
311 return true;
312
313 page = page_pool_dev_alloc_pages(rx_ring->page_pool);
314 if (unlikely(!page))
315 return false;
316 dma = page_pool_get_dma_addr(page);
317
318 bi->page_dma = dma;
319 bi->page = page;
320 bi->page_offset = 0;
321
322 return true;
323 }
324
325 /**
326 * wx_alloc_rx_buffers - Replace used receive buffers
327 * @rx_ring: ring to place buffers on
328 * @cleaned_count: number of buffers to replace
329 **/
wx_alloc_rx_buffers(struct wx_ring * rx_ring,u16 cleaned_count)330 void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count)
331 {
332 u16 i = rx_ring->next_to_use;
333 union wx_rx_desc *rx_desc;
334 struct wx_rx_buffer *bi;
335
336 /* nothing to do */
337 if (!cleaned_count)
338 return;
339
340 rx_desc = WX_RX_DESC(rx_ring, i);
341 bi = &rx_ring->rx_buffer_info[i];
342 i -= rx_ring->count;
343
344 do {
345 if (!wx_alloc_mapped_page(rx_ring, bi))
346 break;
347
348 /* sync the buffer for use by the device */
349 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
350 bi->page_offset,
351 WX_RX_BUFSZ,
352 DMA_FROM_DEVICE);
353
354 rx_desc->read.pkt_addr =
355 cpu_to_le64(bi->page_dma + bi->page_offset);
356
357 rx_desc++;
358 bi++;
359 i++;
360 if (unlikely(!i)) {
361 rx_desc = WX_RX_DESC(rx_ring, 0);
362 bi = rx_ring->rx_buffer_info;
363 i -= rx_ring->count;
364 }
365
366 /* clear the status bits for the next_to_use descriptor */
367 rx_desc->wb.upper.status_error = 0;
368
369 cleaned_count--;
370 } while (cleaned_count);
371
372 i += rx_ring->count;
373
374 if (rx_ring->next_to_use != i) {
375 rx_ring->next_to_use = i;
376 /* update next to alloc since we have filled the ring */
377 rx_ring->next_to_alloc = i;
378
379 /* Force memory writes to complete before letting h/w
380 * know there are new descriptors to fetch. (Only
381 * applicable for weak-ordered memory model archs,
382 * such as IA-64).
383 */
384 wmb();
385 writel(i, rx_ring->tail);
386 }
387 }
388
wx_desc_unused(struct wx_ring * ring)389 u16 wx_desc_unused(struct wx_ring *ring)
390 {
391 u16 ntc = ring->next_to_clean;
392 u16 ntu = ring->next_to_use;
393
394 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
395 }
396
397 /**
398 * wx_is_non_eop - process handling of non-EOP buffers
399 * @rx_ring: Rx ring being processed
400 * @rx_desc: Rx descriptor for current buffer
401 * @skb: Current socket buffer containing buffer in progress
402 *
403 * This function updates next to clean. If the buffer is an EOP buffer
404 * this function exits returning false, otherwise it will place the
405 * sk_buff in the next buffer to be chained and return true indicating
406 * that this is in fact a non-EOP buffer.
407 **/
wx_is_non_eop(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)408 static bool wx_is_non_eop(struct wx_ring *rx_ring,
409 union wx_rx_desc *rx_desc,
410 struct sk_buff *skb)
411 {
412 u32 ntc = rx_ring->next_to_clean + 1;
413
414 /* fetch, update, and store next to clean */
415 ntc = (ntc < rx_ring->count) ? ntc : 0;
416 rx_ring->next_to_clean = ntc;
417
418 prefetch(WX_RX_DESC(rx_ring, ntc));
419
420 /* if we are the last buffer then there is nothing else to do */
421 if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)))
422 return false;
423
424 rx_ring->rx_buffer_info[ntc].skb = skb;
425
426 return true;
427 }
428
wx_pull_tail(struct sk_buff * skb)429 static void wx_pull_tail(struct sk_buff *skb)
430 {
431 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
432 unsigned int pull_len;
433 unsigned char *va;
434
435 /* it is valid to use page_address instead of kmap since we are
436 * working with pages allocated out of the lomem pool per
437 * alloc_page(GFP_ATOMIC)
438 */
439 va = skb_frag_address(frag);
440
441 /* we need the header to contain the greater of either ETH_HLEN or
442 * 60 bytes if the skb->len is less than 60 for skb_pad.
443 */
444 pull_len = eth_get_headlen(skb->dev, va, WX_RXBUFFER_256);
445
446 /* align pull length to size of long to optimize memcpy performance */
447 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
448
449 /* update all of the pointers */
450 skb_frag_size_sub(frag, pull_len);
451 skb_frag_off_add(frag, pull_len);
452 skb->data_len -= pull_len;
453 skb->tail += pull_len;
454 }
455
456 /**
457 * wx_cleanup_headers - Correct corrupted or empty headers
458 * @rx_ring: rx descriptor ring packet is being transacted on
459 * @rx_desc: pointer to the EOP Rx descriptor
460 * @skb: pointer to current skb being fixed
461 *
462 * Check for corrupted packet headers caused by senders on the local L2
463 * embedded NIC switch not setting up their Tx Descriptors right. These
464 * should be very rare.
465 *
466 * Also address the case where we are pulling data in on pages only
467 * and as such no data is present in the skb header.
468 *
469 * In addition if skb is not at least 60 bytes we need to pad it so that
470 * it is large enough to qualify as a valid Ethernet frame.
471 *
472 * Returns true if an error was encountered and skb was freed.
473 **/
wx_cleanup_headers(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)474 static bool wx_cleanup_headers(struct wx_ring *rx_ring,
475 union wx_rx_desc *rx_desc,
476 struct sk_buff *skb)
477 {
478 struct net_device *netdev = rx_ring->netdev;
479
480 /* verify that the packet does not have any known errors */
481 if (!netdev ||
482 unlikely(wx_test_staterr(rx_desc, WX_RXD_ERR_RXE) &&
483 !(netdev->features & NETIF_F_RXALL))) {
484 dev_kfree_skb_any(skb);
485 return true;
486 }
487
488 /* place header in linear portion of buffer */
489 if (!skb_headlen(skb))
490 wx_pull_tail(skb);
491
492 /* if eth_skb_pad returns an error the skb was freed */
493 if (eth_skb_pad(skb))
494 return true;
495
496 return false;
497 }
498
wx_rx_hash(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)499 static void wx_rx_hash(struct wx_ring *ring,
500 union wx_rx_desc *rx_desc,
501 struct sk_buff *skb)
502 {
503 u16 rss_type;
504
505 if (!(ring->netdev->features & NETIF_F_RXHASH))
506 return;
507
508 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
509 WX_RXD_RSSTYPE_MASK;
510
511 if (!rss_type)
512 return;
513
514 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
515 (WX_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
516 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
517 }
518
519 /**
520 * wx_rx_checksum - indicate in skb if hw indicated a good cksum
521 * @ring: structure containing ring specific data
522 * @rx_desc: current Rx descriptor being processed
523 * @skb: skb currently being received and modified
524 **/
wx_rx_checksum(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)525 static void wx_rx_checksum(struct wx_ring *ring,
526 union wx_rx_desc *rx_desc,
527 struct sk_buff *skb)
528 {
529 struct wx_dec_ptype dptype = wx_decode_ptype(WX_RXD_PKTTYPE(rx_desc));
530
531 skb_checksum_none_assert(skb);
532 /* Rx csum disabled */
533 if (!(ring->netdev->features & NETIF_F_RXCSUM))
534 return;
535
536 /* if IPv4 header checksum error */
537 if ((wx_test_staterr(rx_desc, WX_RXD_STAT_IPCS) &&
538 wx_test_staterr(rx_desc, WX_RXD_ERR_IPE)) ||
539 (wx_test_staterr(rx_desc, WX_RXD_STAT_OUTERIPCS) &&
540 wx_test_staterr(rx_desc, WX_RXD_ERR_OUTERIPER))) {
541 ring->rx_stats.csum_err++;
542 return;
543 }
544
545 /* L4 checksum offload flag must set for the below code to work */
546 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_L4CS))
547 return;
548
549 /* Hardware can't guarantee csum if IPv6 Dest Header found */
550 if (dptype.prot != WX_DEC_PTYPE_PROT_SCTP && WX_RXD_IPV6EX(rx_desc))
551 return;
552
553 /* if L4 checksum error */
554 if (wx_test_staterr(rx_desc, WX_RXD_ERR_TCPE)) {
555 ring->rx_stats.csum_err++;
556 return;
557 }
558
559 /* It must be a TCP or UDP or SCTP packet with a valid checksum */
560 skb->ip_summed = CHECKSUM_UNNECESSARY;
561
562 /* If there is an outer header present that might contain a checksum
563 * we need to bump the checksum level by 1 to reflect the fact that
564 * we are indicating we validated the inner checksum.
565 */
566 if (dptype.etype >= WX_DEC_PTYPE_ETYPE_IG)
567 __skb_incr_checksum_unnecessary(skb);
568 ring->rx_stats.csum_good_cnt++;
569 }
570
wx_rx_vlan(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)571 static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc,
572 struct sk_buff *skb)
573 {
574 u16 ethertype;
575 u8 idx = 0;
576
577 if ((ring->netdev->features &
578 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) &&
579 wx_test_staterr(rx_desc, WX_RXD_STAT_VP)) {
580 idx = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
581 0x1c0) >> 6;
582 ethertype = ring->q_vector->wx->tpid[idx];
583 __vlan_hwaccel_put_tag(skb, htons(ethertype),
584 le16_to_cpu(rx_desc->wb.upper.vlan));
585 }
586 }
587
588 /**
589 * wx_process_skb_fields - Populate skb header fields from Rx descriptor
590 * @rx_ring: rx descriptor ring packet is being transacted on
591 * @rx_desc: pointer to the EOP Rx descriptor
592 * @skb: pointer to current skb being populated
593 *
594 * This function checks the ring, descriptor, and packet information in
595 * order to populate the hash, checksum, protocol, and
596 * other fields within the skb.
597 **/
wx_process_skb_fields(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)598 static void wx_process_skb_fields(struct wx_ring *rx_ring,
599 union wx_rx_desc *rx_desc,
600 struct sk_buff *skb)
601 {
602 wx_rx_hash(rx_ring, rx_desc, skb);
603 wx_rx_checksum(rx_ring, rx_desc, skb);
604 wx_rx_vlan(rx_ring, rx_desc, skb);
605 skb_record_rx_queue(skb, rx_ring->queue_index);
606 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
607 }
608
609 /**
610 * wx_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
611 * @q_vector: structure containing interrupt and ring information
612 * @rx_ring: rx descriptor ring to transact packets on
613 * @budget: Total limit on number of packets to process
614 *
615 * This function provides a "bounce buffer" approach to Rx interrupt
616 * processing. The advantage to this is that on systems that have
617 * expensive overhead for IOMMU access this provides a means of avoiding
618 * it by maintaining the mapping of the page to the system.
619 *
620 * Returns amount of work completed.
621 **/
wx_clean_rx_irq(struct wx_q_vector * q_vector,struct wx_ring * rx_ring,int budget)622 static int wx_clean_rx_irq(struct wx_q_vector *q_vector,
623 struct wx_ring *rx_ring,
624 int budget)
625 {
626 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
627 u16 cleaned_count = wx_desc_unused(rx_ring);
628
629 do {
630 struct wx_rx_buffer *rx_buffer;
631 union wx_rx_desc *rx_desc;
632 struct sk_buff *skb;
633 int rx_buffer_pgcnt;
634
635 /* return some buffers to hardware, one at a time is too slow */
636 if (cleaned_count >= WX_RX_BUFFER_WRITE) {
637 wx_alloc_rx_buffers(rx_ring, cleaned_count);
638 cleaned_count = 0;
639 }
640
641 rx_desc = WX_RX_DESC(rx_ring, rx_ring->next_to_clean);
642 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_DD))
643 break;
644
645 /* This memory barrier is needed to keep us from reading
646 * any other fields out of the rx_desc until we know the
647 * descriptor has been written back
648 */
649 dma_rmb();
650
651 rx_buffer = wx_get_rx_buffer(rx_ring, rx_desc, &skb, &rx_buffer_pgcnt);
652
653 /* retrieve a buffer from the ring */
654 skb = wx_build_skb(rx_ring, rx_buffer, rx_desc);
655
656 /* exit if we failed to retrieve a buffer */
657 if (!skb) {
658 break;
659 }
660
661 wx_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);
662 cleaned_count++;
663
664 /* place incomplete frames back on ring for completion */
665 if (wx_is_non_eop(rx_ring, rx_desc, skb))
666 continue;
667
668 /* verify the packet layout is correct */
669 if (wx_cleanup_headers(rx_ring, rx_desc, skb))
670 continue;
671
672 /* probably a little skewed due to removing CRC */
673 total_rx_bytes += skb->len;
674
675 /* populate checksum, timestamp, VLAN, and protocol */
676 wx_process_skb_fields(rx_ring, rx_desc, skb);
677 napi_gro_receive(&q_vector->napi, skb);
678
679 /* update budget accounting */
680 total_rx_packets++;
681 } while (likely(total_rx_packets < budget));
682
683 u64_stats_update_begin(&rx_ring->syncp);
684 rx_ring->stats.packets += total_rx_packets;
685 rx_ring->stats.bytes += total_rx_bytes;
686 u64_stats_update_end(&rx_ring->syncp);
687 q_vector->rx.total_packets += total_rx_packets;
688 q_vector->rx.total_bytes += total_rx_bytes;
689
690 return total_rx_packets;
691 }
692
wx_txring_txq(const struct wx_ring * ring)693 static struct netdev_queue *wx_txring_txq(const struct wx_ring *ring)
694 {
695 return netdev_get_tx_queue(ring->netdev, ring->queue_index);
696 }
697
698 /**
699 * wx_clean_tx_irq - Reclaim resources after transmit completes
700 * @q_vector: structure containing interrupt and ring information
701 * @tx_ring: tx ring to clean
702 * @napi_budget: Used to determine if we are in netpoll
703 **/
wx_clean_tx_irq(struct wx_q_vector * q_vector,struct wx_ring * tx_ring,int napi_budget)704 static bool wx_clean_tx_irq(struct wx_q_vector *q_vector,
705 struct wx_ring *tx_ring, int napi_budget)
706 {
707 unsigned int budget = q_vector->wx->tx_work_limit;
708 unsigned int total_bytes = 0, total_packets = 0;
709 unsigned int i = tx_ring->next_to_clean;
710 struct wx_tx_buffer *tx_buffer;
711 union wx_tx_desc *tx_desc;
712
713 if (!netif_carrier_ok(tx_ring->netdev))
714 return true;
715
716 tx_buffer = &tx_ring->tx_buffer_info[i];
717 tx_desc = WX_TX_DESC(tx_ring, i);
718 i -= tx_ring->count;
719
720 do {
721 union wx_tx_desc *eop_desc = tx_buffer->next_to_watch;
722
723 /* if next_to_watch is not set then there is no work pending */
724 if (!eop_desc)
725 break;
726
727 /* prevent any other reads prior to eop_desc */
728 smp_rmb();
729
730 /* if DD is not set pending work has not been completed */
731 if (!(eop_desc->wb.status & cpu_to_le32(WX_TXD_STAT_DD)))
732 break;
733
734 /* clear next_to_watch to prevent false hangs */
735 tx_buffer->next_to_watch = NULL;
736
737 /* update the statistics for this packet */
738 total_bytes += tx_buffer->bytecount;
739 total_packets += tx_buffer->gso_segs;
740
741 /* free the skb */
742 napi_consume_skb(tx_buffer->skb, napi_budget);
743
744 /* unmap skb header data */
745 dma_unmap_single(tx_ring->dev,
746 dma_unmap_addr(tx_buffer, dma),
747 dma_unmap_len(tx_buffer, len),
748 DMA_TO_DEVICE);
749
750 /* clear tx_buffer data */
751 dma_unmap_len_set(tx_buffer, len, 0);
752
753 /* unmap remaining buffers */
754 while (tx_desc != eop_desc) {
755 tx_buffer++;
756 tx_desc++;
757 i++;
758 if (unlikely(!i)) {
759 i -= tx_ring->count;
760 tx_buffer = tx_ring->tx_buffer_info;
761 tx_desc = WX_TX_DESC(tx_ring, 0);
762 }
763
764 /* unmap any remaining paged data */
765 if (dma_unmap_len(tx_buffer, len)) {
766 dma_unmap_page(tx_ring->dev,
767 dma_unmap_addr(tx_buffer, dma),
768 dma_unmap_len(tx_buffer, len),
769 DMA_TO_DEVICE);
770 dma_unmap_len_set(tx_buffer, len, 0);
771 }
772 }
773
774 /* move us one more past the eop_desc for start of next pkt */
775 tx_buffer++;
776 tx_desc++;
777 i++;
778 if (unlikely(!i)) {
779 i -= tx_ring->count;
780 tx_buffer = tx_ring->tx_buffer_info;
781 tx_desc = WX_TX_DESC(tx_ring, 0);
782 }
783
784 /* issue prefetch for next Tx descriptor */
785 prefetch(tx_desc);
786
787 /* update budget accounting */
788 budget--;
789 } while (likely(budget));
790
791 i += tx_ring->count;
792 tx_ring->next_to_clean = i;
793 u64_stats_update_begin(&tx_ring->syncp);
794 tx_ring->stats.bytes += total_bytes;
795 tx_ring->stats.packets += total_packets;
796 u64_stats_update_end(&tx_ring->syncp);
797 q_vector->tx.total_bytes += total_bytes;
798 q_vector->tx.total_packets += total_packets;
799
800 netdev_tx_completed_queue(wx_txring_txq(tx_ring),
801 total_packets, total_bytes);
802
803 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
804 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
805 (wx_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
806 /* Make sure that anybody stopping the queue after this
807 * sees the new next_to_clean.
808 */
809 smp_mb();
810
811 if (__netif_subqueue_stopped(tx_ring->netdev,
812 tx_ring->queue_index) &&
813 netif_running(tx_ring->netdev))
814 netif_wake_subqueue(tx_ring->netdev,
815 tx_ring->queue_index);
816 }
817
818 return !!budget;
819 }
820
821 /**
822 * wx_poll - NAPI polling RX/TX cleanup routine
823 * @napi: napi struct with our devices info in it
824 * @budget: amount of work driver is allowed to do this pass, in packets
825 *
826 * This function will clean all queues associated with a q_vector.
827 **/
wx_poll(struct napi_struct * napi,int budget)828 static int wx_poll(struct napi_struct *napi, int budget)
829 {
830 struct wx_q_vector *q_vector = container_of(napi, struct wx_q_vector, napi);
831 int per_ring_budget, work_done = 0;
832 struct wx *wx = q_vector->wx;
833 bool clean_complete = true;
834 struct wx_ring *ring;
835
836 wx_for_each_ring(ring, q_vector->tx) {
837 if (!wx_clean_tx_irq(q_vector, ring, budget))
838 clean_complete = false;
839 }
840
841 /* Exit if we are called by netpoll */
842 if (budget <= 0)
843 return budget;
844
845 /* attempt to distribute budget to each queue fairly, but don't allow
846 * the budget to go below 1 because we'll exit polling
847 */
848 if (q_vector->rx.count > 1)
849 per_ring_budget = max(budget / q_vector->rx.count, 1);
850 else
851 per_ring_budget = budget;
852
853 wx_for_each_ring(ring, q_vector->rx) {
854 int cleaned = wx_clean_rx_irq(q_vector, ring, per_ring_budget);
855
856 work_done += cleaned;
857 if (cleaned >= per_ring_budget)
858 clean_complete = false;
859 }
860
861 /* If all work not completed, return budget and keep polling */
862 if (!clean_complete)
863 return budget;
864
865 /* all work done, exit the polling mode */
866 if (likely(napi_complete_done(napi, work_done))) {
867 if (netif_running(wx->netdev))
868 wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));
869 }
870
871 return min(work_done, budget - 1);
872 }
873
wx_maybe_stop_tx(struct wx_ring * tx_ring,u16 size)874 static int wx_maybe_stop_tx(struct wx_ring *tx_ring, u16 size)
875 {
876 if (likely(wx_desc_unused(tx_ring) >= size))
877 return 0;
878
879 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
880
881 /* For the next check */
882 smp_mb();
883
884 /* We need to check again in a case another CPU has just
885 * made room available.
886 */
887 if (likely(wx_desc_unused(tx_ring) < size))
888 return -EBUSY;
889
890 /* A reprieve! - use start_queue because it doesn't call schedule */
891 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
892
893 return 0;
894 }
895
wx_tx_cmd_type(u32 tx_flags)896 static u32 wx_tx_cmd_type(u32 tx_flags)
897 {
898 /* set type for advanced descriptor with frame checksum insertion */
899 u32 cmd_type = WX_TXD_DTYP_DATA | WX_TXD_IFCS;
900
901 /* set HW vlan bit if vlan is present */
902 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_HW_VLAN, WX_TXD_VLE);
903 /* set segmentation enable bits for TSO/FSO */
904 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSO, WX_TXD_TSE);
905 /* set timestamp bit if present */
906 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSTAMP, WX_TXD_MAC_TSTAMP);
907 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_LINKSEC, WX_TXD_LINKSEC);
908
909 return cmd_type;
910 }
911
wx_tx_olinfo_status(union wx_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)912 static void wx_tx_olinfo_status(union wx_tx_desc *tx_desc,
913 u32 tx_flags, unsigned int paylen)
914 {
915 u32 olinfo_status = paylen << WX_TXD_PAYLEN_SHIFT;
916
917 /* enable L4 checksum for TSO and TX checksum offload */
918 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CSUM, WX_TXD_L4CS);
919 /* enable IPv4 checksum for TSO */
920 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPV4, WX_TXD_IIPCS);
921 /* enable outer IPv4 checksum for TSO */
922 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_OUTER_IPV4,
923 WX_TXD_EIPCS);
924 /* Check Context must be set if Tx switch is enabled, which it
925 * always is for case where virtual functions are running
926 */
927 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CC, WX_TXD_CC);
928 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPSEC,
929 WX_TXD_IPSEC);
930 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
931 }
932
wx_tx_map(struct wx_ring * tx_ring,struct wx_tx_buffer * first,const u8 hdr_len)933 static void wx_tx_map(struct wx_ring *tx_ring,
934 struct wx_tx_buffer *first,
935 const u8 hdr_len)
936 {
937 struct sk_buff *skb = first->skb;
938 struct wx_tx_buffer *tx_buffer;
939 u32 tx_flags = first->tx_flags;
940 u16 i = tx_ring->next_to_use;
941 unsigned int data_len, size;
942 union wx_tx_desc *tx_desc;
943 skb_frag_t *frag;
944 dma_addr_t dma;
945 u32 cmd_type;
946
947 cmd_type = wx_tx_cmd_type(tx_flags);
948 tx_desc = WX_TX_DESC(tx_ring, i);
949 wx_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
950
951 size = skb_headlen(skb);
952 data_len = skb->data_len;
953 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
954
955 tx_buffer = first;
956
957 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
958 if (dma_mapping_error(tx_ring->dev, dma))
959 goto dma_error;
960
961 /* record length, and DMA address */
962 dma_unmap_len_set(tx_buffer, len, size);
963 dma_unmap_addr_set(tx_buffer, dma, dma);
964
965 tx_desc->read.buffer_addr = cpu_to_le64(dma);
966
967 while (unlikely(size > WX_MAX_DATA_PER_TXD)) {
968 tx_desc->read.cmd_type_len =
969 cpu_to_le32(cmd_type ^ WX_MAX_DATA_PER_TXD);
970
971 i++;
972 tx_desc++;
973 if (i == tx_ring->count) {
974 tx_desc = WX_TX_DESC(tx_ring, 0);
975 i = 0;
976 }
977 tx_desc->read.olinfo_status = 0;
978
979 dma += WX_MAX_DATA_PER_TXD;
980 size -= WX_MAX_DATA_PER_TXD;
981
982 tx_desc->read.buffer_addr = cpu_to_le64(dma);
983 }
984
985 if (likely(!data_len))
986 break;
987
988 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
989
990 i++;
991 tx_desc++;
992 if (i == tx_ring->count) {
993 tx_desc = WX_TX_DESC(tx_ring, 0);
994 i = 0;
995 }
996 tx_desc->read.olinfo_status = 0;
997
998 size = skb_frag_size(frag);
999
1000 data_len -= size;
1001
1002 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1003 DMA_TO_DEVICE);
1004
1005 tx_buffer = &tx_ring->tx_buffer_info[i];
1006 }
1007
1008 /* write last descriptor with RS and EOP bits */
1009 cmd_type |= size | WX_TXD_EOP | WX_TXD_RS;
1010 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1011
1012 netdev_tx_sent_queue(wx_txring_txq(tx_ring), first->bytecount);
1013
1014 skb_tx_timestamp(skb);
1015
1016 /* Force memory writes to complete before letting h/w know there
1017 * are new descriptors to fetch. (Only applicable for weak-ordered
1018 * memory model archs, such as IA-64).
1019 *
1020 * We also need this memory barrier to make certain all of the
1021 * status bits have been updated before next_to_watch is written.
1022 */
1023 wmb();
1024
1025 /* set next_to_watch value indicating a packet is present */
1026 first->next_to_watch = tx_desc;
1027
1028 i++;
1029 if (i == tx_ring->count)
1030 i = 0;
1031
1032 tx_ring->next_to_use = i;
1033
1034 wx_maybe_stop_tx(tx_ring, DESC_NEEDED);
1035
1036 if (netif_xmit_stopped(wx_txring_txq(tx_ring)) || !netdev_xmit_more())
1037 writel(i, tx_ring->tail);
1038
1039 return;
1040 dma_error:
1041 dev_err(tx_ring->dev, "TX DMA map failed\n");
1042
1043 /* clear dma mappings for failed tx_buffer_info map */
1044 for (;;) {
1045 tx_buffer = &tx_ring->tx_buffer_info[i];
1046 if (dma_unmap_len(tx_buffer, len))
1047 dma_unmap_page(tx_ring->dev,
1048 dma_unmap_addr(tx_buffer, dma),
1049 dma_unmap_len(tx_buffer, len),
1050 DMA_TO_DEVICE);
1051 dma_unmap_len_set(tx_buffer, len, 0);
1052 if (tx_buffer == first)
1053 break;
1054 if (i == 0)
1055 i += tx_ring->count;
1056 i--;
1057 }
1058
1059 dev_kfree_skb_any(first->skb);
1060 first->skb = NULL;
1061
1062 tx_ring->next_to_use = i;
1063 }
1064
wx_tx_ctxtdesc(struct wx_ring * tx_ring,u32 vlan_macip_lens,u32 fcoe_sof_eof,u32 type_tucmd,u32 mss_l4len_idx)1065 static void wx_tx_ctxtdesc(struct wx_ring *tx_ring, u32 vlan_macip_lens,
1066 u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
1067 {
1068 struct wx_tx_context_desc *context_desc;
1069 u16 i = tx_ring->next_to_use;
1070
1071 context_desc = WX_TX_CTXTDESC(tx_ring, i);
1072 i++;
1073 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1074
1075 /* set bits to identify this as an advanced context descriptor */
1076 type_tucmd |= WX_TXD_DTYP_CTXT;
1077 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
1078 context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);
1079 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
1080 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
1081 }
1082
wx_get_ipv6_proto(struct sk_buff * skb,int offset,u8 * nexthdr)1083 static void wx_get_ipv6_proto(struct sk_buff *skb, int offset, u8 *nexthdr)
1084 {
1085 struct ipv6hdr *hdr = (struct ipv6hdr *)(skb->data + offset);
1086
1087 *nexthdr = hdr->nexthdr;
1088 offset += sizeof(struct ipv6hdr);
1089 while (ipv6_ext_hdr(*nexthdr)) {
1090 struct ipv6_opt_hdr _hdr, *hp;
1091
1092 if (*nexthdr == NEXTHDR_NONE)
1093 return;
1094 hp = skb_header_pointer(skb, offset, sizeof(_hdr), &_hdr);
1095 if (!hp)
1096 return;
1097 if (*nexthdr == NEXTHDR_FRAGMENT)
1098 break;
1099 *nexthdr = hp->nexthdr;
1100 }
1101 }
1102
1103 union network_header {
1104 struct iphdr *ipv4;
1105 struct ipv6hdr *ipv6;
1106 void *raw;
1107 };
1108
wx_encode_tx_desc_ptype(const struct wx_tx_buffer * first)1109 static u8 wx_encode_tx_desc_ptype(const struct wx_tx_buffer *first)
1110 {
1111 u8 tun_prot = 0, l4_prot = 0, ptype = 0;
1112 struct sk_buff *skb = first->skb;
1113
1114 if (skb->encapsulation) {
1115 union network_header hdr;
1116
1117 switch (first->protocol) {
1118 case htons(ETH_P_IP):
1119 tun_prot = ip_hdr(skb)->protocol;
1120 ptype = WX_PTYPE_TUN_IPV4;
1121 break;
1122 case htons(ETH_P_IPV6):
1123 wx_get_ipv6_proto(skb, skb_network_offset(skb), &tun_prot);
1124 ptype = WX_PTYPE_TUN_IPV6;
1125 break;
1126 default:
1127 return ptype;
1128 }
1129
1130 if (tun_prot == IPPROTO_IPIP) {
1131 hdr.raw = (void *)inner_ip_hdr(skb);
1132 ptype |= WX_PTYPE_PKT_IPIP;
1133 } else if (tun_prot == IPPROTO_UDP) {
1134 hdr.raw = (void *)inner_ip_hdr(skb);
1135 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1136 skb->inner_protocol != htons(ETH_P_TEB)) {
1137 ptype |= WX_PTYPE_PKT_IG;
1138 } else {
1139 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1140 == htons(ETH_P_8021Q))
1141 ptype |= WX_PTYPE_PKT_IGMV;
1142 else
1143 ptype |= WX_PTYPE_PKT_IGM;
1144 }
1145
1146 } else if (tun_prot == IPPROTO_GRE) {
1147 hdr.raw = (void *)inner_ip_hdr(skb);
1148 if (skb->inner_protocol == htons(ETH_P_IP) ||
1149 skb->inner_protocol == htons(ETH_P_IPV6)) {
1150 ptype |= WX_PTYPE_PKT_IG;
1151 } else {
1152 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1153 == htons(ETH_P_8021Q))
1154 ptype |= WX_PTYPE_PKT_IGMV;
1155 else
1156 ptype |= WX_PTYPE_PKT_IGM;
1157 }
1158 } else {
1159 return ptype;
1160 }
1161
1162 switch (hdr.ipv4->version) {
1163 case IPVERSION:
1164 l4_prot = hdr.ipv4->protocol;
1165 break;
1166 case 6:
1167 wx_get_ipv6_proto(skb, skb_inner_network_offset(skb), &l4_prot);
1168 ptype |= WX_PTYPE_PKT_IPV6;
1169 break;
1170 default:
1171 return ptype;
1172 }
1173 } else {
1174 switch (first->protocol) {
1175 case htons(ETH_P_IP):
1176 l4_prot = ip_hdr(skb)->protocol;
1177 ptype = WX_PTYPE_PKT_IP;
1178 break;
1179 case htons(ETH_P_IPV6):
1180 wx_get_ipv6_proto(skb, skb_network_offset(skb), &l4_prot);
1181 ptype = WX_PTYPE_PKT_IP | WX_PTYPE_PKT_IPV6;
1182 break;
1183 default:
1184 return WX_PTYPE_PKT_MAC | WX_PTYPE_TYP_MAC;
1185 }
1186 }
1187 switch (l4_prot) {
1188 case IPPROTO_TCP:
1189 ptype |= WX_PTYPE_TYP_TCP;
1190 break;
1191 case IPPROTO_UDP:
1192 ptype |= WX_PTYPE_TYP_UDP;
1193 break;
1194 case IPPROTO_SCTP:
1195 ptype |= WX_PTYPE_TYP_SCTP;
1196 break;
1197 default:
1198 ptype |= WX_PTYPE_TYP_IP;
1199 break;
1200 }
1201
1202 return ptype;
1203 }
1204
wx_tso(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 * hdr_len,u8 ptype)1205 static int wx_tso(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1206 u8 *hdr_len, u8 ptype)
1207 {
1208 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1209 struct net_device *netdev = tx_ring->netdev;
1210 u32 l4len, tunhdr_eiplen_tunlen = 0;
1211 struct sk_buff *skb = first->skb;
1212 bool enc = skb->encapsulation;
1213 struct ipv6hdr *ipv6h;
1214 struct tcphdr *tcph;
1215 struct iphdr *iph;
1216 u8 tun_prot = 0;
1217 int err;
1218
1219 if (skb->ip_summed != CHECKSUM_PARTIAL)
1220 return 0;
1221
1222 if (!skb_is_gso(skb))
1223 return 0;
1224
1225 err = skb_cow_head(skb, 0);
1226 if (err < 0)
1227 return err;
1228
1229 /* indicates the inner headers in the skbuff are valid. */
1230 iph = enc ? inner_ip_hdr(skb) : ip_hdr(skb);
1231 if (iph->version == 4) {
1232 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1233 iph->tot_len = 0;
1234 iph->check = 0;
1235 tcph->check = ~csum_tcpudp_magic(iph->saddr,
1236 iph->daddr, 0,
1237 IPPROTO_TCP, 0);
1238 first->tx_flags |= WX_TX_FLAGS_TSO |
1239 WX_TX_FLAGS_CSUM |
1240 WX_TX_FLAGS_IPV4 |
1241 WX_TX_FLAGS_CC;
1242 } else if (iph->version == 6 && skb_is_gso_v6(skb)) {
1243 ipv6h = enc ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);
1244 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1245 ipv6h->payload_len = 0;
1246 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1247 &ipv6h->daddr, 0,
1248 IPPROTO_TCP, 0);
1249 first->tx_flags |= WX_TX_FLAGS_TSO |
1250 WX_TX_FLAGS_CSUM |
1251 WX_TX_FLAGS_CC;
1252 }
1253
1254 /* compute header lengths */
1255 l4len = enc ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1256 *hdr_len = enc ? (skb_inner_transport_header(skb) - skb->data) :
1257 skb_transport_offset(skb);
1258 *hdr_len += l4len;
1259
1260 /* update gso size and bytecount with header size */
1261 first->gso_segs = skb_shinfo(skb)->gso_segs;
1262 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1263
1264 /* mss_l4len_id: use 0 as index for TSO */
1265 mss_l4len_idx = l4len << WX_TXD_L4LEN_SHIFT;
1266 mss_l4len_idx |= skb_shinfo(skb)->gso_size << WX_TXD_MSS_SHIFT;
1267
1268 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
1269 if (enc) {
1270 switch (first->protocol) {
1271 case htons(ETH_P_IP):
1272 tun_prot = ip_hdr(skb)->protocol;
1273 first->tx_flags |= WX_TX_FLAGS_OUTER_IPV4;
1274 break;
1275 case htons(ETH_P_IPV6):
1276 tun_prot = ipv6_hdr(skb)->nexthdr;
1277 break;
1278 default:
1279 break;
1280 }
1281 switch (tun_prot) {
1282 case IPPROTO_UDP:
1283 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1284 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1285 WX_TXD_OUTER_IPLEN_SHIFT) |
1286 (((skb_inner_mac_header(skb) -
1287 skb_transport_header(skb)) >> 1) <<
1288 WX_TXD_TUNNEL_LEN_SHIFT);
1289 break;
1290 case IPPROTO_GRE:
1291 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1292 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1293 WX_TXD_OUTER_IPLEN_SHIFT) |
1294 (((skb_inner_mac_header(skb) -
1295 skb_transport_header(skb)) >> 1) <<
1296 WX_TXD_TUNNEL_LEN_SHIFT);
1297 break;
1298 case IPPROTO_IPIP:
1299 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1300 (char *)ip_hdr(skb)) >> 2) <<
1301 WX_TXD_OUTER_IPLEN_SHIFT;
1302 break;
1303 default:
1304 break;
1305 }
1306 vlan_macip_lens = skb_inner_network_header_len(skb) >> 1;
1307 } else {
1308 vlan_macip_lens = skb_network_header_len(skb) >> 1;
1309 }
1310
1311 vlan_macip_lens |= skb_network_offset(skb) << WX_TXD_MACLEN_SHIFT;
1312 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1313
1314 type_tucmd = ptype << 24;
1315 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1316 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1317 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1318 WX_TX_FLAGS_HW_VLAN,
1319 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1320 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1321 type_tucmd, mss_l4len_idx);
1322
1323 return 1;
1324 }
1325
wx_tx_csum(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 ptype)1326 static void wx_tx_csum(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1327 u8 ptype)
1328 {
1329 u32 tunhdr_eiplen_tunlen = 0, vlan_macip_lens = 0;
1330 struct net_device *netdev = tx_ring->netdev;
1331 u32 mss_l4len_idx = 0, type_tucmd;
1332 struct sk_buff *skb = first->skb;
1333 u8 tun_prot = 0;
1334
1335 if (skb->ip_summed != CHECKSUM_PARTIAL) {
1336 if (!(first->tx_flags & WX_TX_FLAGS_HW_VLAN) &&
1337 !(first->tx_flags & WX_TX_FLAGS_CC))
1338 return;
1339 vlan_macip_lens = skb_network_offset(skb) <<
1340 WX_TXD_MACLEN_SHIFT;
1341 } else {
1342 u8 l4_prot = 0;
1343 union {
1344 struct iphdr *ipv4;
1345 struct ipv6hdr *ipv6;
1346 u8 *raw;
1347 } network_hdr;
1348 union {
1349 struct tcphdr *tcphdr;
1350 u8 *raw;
1351 } transport_hdr;
1352
1353 if (skb->encapsulation) {
1354 network_hdr.raw = skb_inner_network_header(skb);
1355 transport_hdr.raw = skb_inner_transport_header(skb);
1356 vlan_macip_lens = skb_network_offset(skb) <<
1357 WX_TXD_MACLEN_SHIFT;
1358 switch (first->protocol) {
1359 case htons(ETH_P_IP):
1360 tun_prot = ip_hdr(skb)->protocol;
1361 break;
1362 case htons(ETH_P_IPV6):
1363 tun_prot = ipv6_hdr(skb)->nexthdr;
1364 break;
1365 default:
1366 return;
1367 }
1368 switch (tun_prot) {
1369 case IPPROTO_UDP:
1370 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1371 tunhdr_eiplen_tunlen |=
1372 ((skb_network_header_len(skb) >> 2) <<
1373 WX_TXD_OUTER_IPLEN_SHIFT) |
1374 (((skb_inner_mac_header(skb) -
1375 skb_transport_header(skb)) >> 1) <<
1376 WX_TXD_TUNNEL_LEN_SHIFT);
1377 break;
1378 case IPPROTO_GRE:
1379 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1380 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1381 WX_TXD_OUTER_IPLEN_SHIFT) |
1382 (((skb_inner_mac_header(skb) -
1383 skb_transport_header(skb)) >> 1) <<
1384 WX_TXD_TUNNEL_LEN_SHIFT);
1385 break;
1386 case IPPROTO_IPIP:
1387 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1388 (char *)ip_hdr(skb)) >> 2) <<
1389 WX_TXD_OUTER_IPLEN_SHIFT;
1390 break;
1391 default:
1392 break;
1393 }
1394
1395 } else {
1396 network_hdr.raw = skb_network_header(skb);
1397 transport_hdr.raw = skb_transport_header(skb);
1398 vlan_macip_lens = skb_network_offset(skb) <<
1399 WX_TXD_MACLEN_SHIFT;
1400 }
1401
1402 switch (network_hdr.ipv4->version) {
1403 case IPVERSION:
1404 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1405 l4_prot = network_hdr.ipv4->protocol;
1406 break;
1407 case 6:
1408 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1409 l4_prot = network_hdr.ipv6->nexthdr;
1410 break;
1411 default:
1412 break;
1413 }
1414
1415 switch (l4_prot) {
1416 case IPPROTO_TCP:
1417 mss_l4len_idx = (transport_hdr.tcphdr->doff * 4) <<
1418 WX_TXD_L4LEN_SHIFT;
1419 break;
1420 case IPPROTO_SCTP:
1421 mss_l4len_idx = sizeof(struct sctphdr) <<
1422 WX_TXD_L4LEN_SHIFT;
1423 break;
1424 case IPPROTO_UDP:
1425 mss_l4len_idx = sizeof(struct udphdr) <<
1426 WX_TXD_L4LEN_SHIFT;
1427 break;
1428 default:
1429 break;
1430 }
1431
1432 /* update TX checksum flag */
1433 first->tx_flags |= WX_TX_FLAGS_CSUM;
1434 }
1435 first->tx_flags |= WX_TX_FLAGS_CC;
1436 /* vlan_macip_lens: MACLEN, VLAN tag */
1437 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1438
1439 type_tucmd = ptype << 24;
1440 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1441 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1442 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1443 WX_TX_FLAGS_HW_VLAN,
1444 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1445 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1446 type_tucmd, mss_l4len_idx);
1447 }
1448
wx_xmit_frame_ring(struct sk_buff * skb,struct wx_ring * tx_ring)1449 static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
1450 struct wx_ring *tx_ring)
1451 {
1452 u16 count = TXD_USE_COUNT(skb_headlen(skb));
1453 struct wx_tx_buffer *first;
1454 u8 hdr_len = 0, ptype;
1455 unsigned short f;
1456 u32 tx_flags = 0;
1457 int tso;
1458
1459 /* need: 1 descriptor per page * PAGE_SIZE/WX_MAX_DATA_PER_TXD,
1460 * + 1 desc for skb_headlen/WX_MAX_DATA_PER_TXD,
1461 * + 2 desc gap to keep tail from touching head,
1462 * + 1 desc for context descriptor,
1463 * otherwise try next time
1464 */
1465 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1466 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->
1467 frags[f]));
1468
1469 if (wx_maybe_stop_tx(tx_ring, count + 3))
1470 return NETDEV_TX_BUSY;
1471
1472 /* record the location of the first descriptor for this packet */
1473 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1474 first->skb = skb;
1475 first->bytecount = skb->len;
1476 first->gso_segs = 1;
1477
1478 /* if we have a HW VLAN tag being added default to the HW one */
1479 if (skb_vlan_tag_present(skb)) {
1480 tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT;
1481 tx_flags |= WX_TX_FLAGS_HW_VLAN;
1482 }
1483
1484 /* record initial flags and protocol */
1485 first->tx_flags = tx_flags;
1486 first->protocol = vlan_get_protocol(skb);
1487
1488 ptype = wx_encode_tx_desc_ptype(first);
1489
1490 tso = wx_tso(tx_ring, first, &hdr_len, ptype);
1491 if (tso < 0)
1492 goto out_drop;
1493 else if (!tso)
1494 wx_tx_csum(tx_ring, first, ptype);
1495 wx_tx_map(tx_ring, first, hdr_len);
1496
1497 return NETDEV_TX_OK;
1498 out_drop:
1499 dev_kfree_skb_any(first->skb);
1500 first->skb = NULL;
1501
1502 return NETDEV_TX_OK;
1503 }
1504
wx_xmit_frame(struct sk_buff * skb,struct net_device * netdev)1505 netdev_tx_t wx_xmit_frame(struct sk_buff *skb,
1506 struct net_device *netdev)
1507 {
1508 unsigned int r_idx = skb->queue_mapping;
1509 struct wx *wx = netdev_priv(netdev);
1510 struct wx_ring *tx_ring;
1511
1512 if (!netif_carrier_ok(netdev)) {
1513 dev_kfree_skb_any(skb);
1514 return NETDEV_TX_OK;
1515 }
1516
1517 /* The minimum packet size for olinfo paylen is 17 so pad the skb
1518 * in order to meet this minimum size requirement.
1519 */
1520 if (skb_put_padto(skb, 17))
1521 return NETDEV_TX_OK;
1522
1523 if (r_idx >= wx->num_tx_queues)
1524 r_idx = r_idx % wx->num_tx_queues;
1525 tx_ring = wx->tx_ring[r_idx];
1526
1527 return wx_xmit_frame_ring(skb, tx_ring);
1528 }
1529 EXPORT_SYMBOL(wx_xmit_frame);
1530
wx_napi_enable_all(struct wx * wx)1531 void wx_napi_enable_all(struct wx *wx)
1532 {
1533 struct wx_q_vector *q_vector;
1534 int q_idx;
1535
1536 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1537 q_vector = wx->q_vector[q_idx];
1538 napi_enable(&q_vector->napi);
1539 }
1540 }
1541 EXPORT_SYMBOL(wx_napi_enable_all);
1542
wx_napi_disable_all(struct wx * wx)1543 void wx_napi_disable_all(struct wx *wx)
1544 {
1545 struct wx_q_vector *q_vector;
1546 int q_idx;
1547
1548 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1549 q_vector = wx->q_vector[q_idx];
1550 napi_disable(&q_vector->napi);
1551 }
1552 }
1553 EXPORT_SYMBOL(wx_napi_disable_all);
1554
1555 /**
1556 * wx_set_rss_queues: Allocate queues for RSS
1557 * @wx: board private structure to initialize
1558 *
1559 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try
1560 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1561 *
1562 **/
wx_set_rss_queues(struct wx * wx)1563 static void wx_set_rss_queues(struct wx *wx)
1564 {
1565 wx->num_rx_queues = wx->mac.max_rx_queues;
1566 wx->num_tx_queues = wx->mac.max_tx_queues;
1567 }
1568
wx_set_num_queues(struct wx * wx)1569 static void wx_set_num_queues(struct wx *wx)
1570 {
1571 /* Start with base case */
1572 wx->num_rx_queues = 1;
1573 wx->num_tx_queues = 1;
1574 wx->queues_per_pool = 1;
1575
1576 wx_set_rss_queues(wx);
1577 }
1578
1579 /**
1580 * wx_acquire_msix_vectors - acquire MSI-X vectors
1581 * @wx: board private structure
1582 *
1583 * Attempts to acquire a suitable range of MSI-X vector interrupts. Will
1584 * return a negative error code if unable to acquire MSI-X vectors for any
1585 * reason.
1586 */
wx_acquire_msix_vectors(struct wx * wx)1587 static int wx_acquire_msix_vectors(struct wx *wx)
1588 {
1589 struct irq_affinity affd = {0, };
1590 int nvecs, i;
1591
1592 nvecs = min_t(int, num_online_cpus(), wx->mac.max_msix_vectors);
1593
1594 wx->msix_entries = kcalloc(nvecs,
1595 sizeof(struct msix_entry),
1596 GFP_KERNEL);
1597 if (!wx->msix_entries)
1598 return -ENOMEM;
1599
1600 nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,
1601 nvecs,
1602 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
1603 &affd);
1604 if (nvecs < 0) {
1605 wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);
1606 kfree(wx->msix_entries);
1607 wx->msix_entries = NULL;
1608 return nvecs;
1609 }
1610
1611 for (i = 0; i < nvecs; i++) {
1612 wx->msix_entries[i].entry = i;
1613 wx->msix_entries[i].vector = pci_irq_vector(wx->pdev, i);
1614 }
1615
1616 /* one for msix_other */
1617 nvecs -= 1;
1618 wx->num_q_vectors = nvecs;
1619 wx->num_rx_queues = nvecs;
1620 wx->num_tx_queues = nvecs;
1621
1622 return 0;
1623 }
1624
1625 /**
1626 * wx_set_interrupt_capability - set MSI-X or MSI if supported
1627 * @wx: board private structure to initialize
1628 *
1629 * Attempt to configure the interrupts using the best available
1630 * capabilities of the hardware and the kernel.
1631 **/
wx_set_interrupt_capability(struct wx * wx)1632 static int wx_set_interrupt_capability(struct wx *wx)
1633 {
1634 struct pci_dev *pdev = wx->pdev;
1635 int nvecs, ret;
1636
1637 /* We will try to get MSI-X interrupts first */
1638 ret = wx_acquire_msix_vectors(wx);
1639 if (ret == 0 || (ret == -ENOMEM))
1640 return ret;
1641
1642 wx->num_rx_queues = 1;
1643 wx->num_tx_queues = 1;
1644 wx->num_q_vectors = 1;
1645
1646 /* minmum one for queue, one for misc*/
1647 nvecs = 1;
1648 nvecs = pci_alloc_irq_vectors(pdev, nvecs,
1649 nvecs, PCI_IRQ_MSI | PCI_IRQ_LEGACY);
1650 if (nvecs == 1) {
1651 if (pdev->msi_enabled)
1652 wx_err(wx, "Fallback to MSI.\n");
1653 else
1654 wx_err(wx, "Fallback to LEGACY.\n");
1655 } else {
1656 wx_err(wx, "Failed to allocate MSI/LEGACY interrupts. Error: %d\n", nvecs);
1657 return nvecs;
1658 }
1659
1660 pdev->irq = pci_irq_vector(pdev, 0);
1661 wx->num_q_vectors = 1;
1662
1663 return 0;
1664 }
1665
1666 /**
1667 * wx_cache_ring_rss - Descriptor ring to register mapping for RSS
1668 * @wx: board private structure to initialize
1669 *
1670 * Cache the descriptor ring offsets for RSS, ATR, FCoE, and SR-IOV.
1671 *
1672 **/
wx_cache_ring_rss(struct wx * wx)1673 static void wx_cache_ring_rss(struct wx *wx)
1674 {
1675 u16 i;
1676
1677 for (i = 0; i < wx->num_rx_queues; i++)
1678 wx->rx_ring[i]->reg_idx = i;
1679
1680 for (i = 0; i < wx->num_tx_queues; i++)
1681 wx->tx_ring[i]->reg_idx = i;
1682 }
1683
wx_add_ring(struct wx_ring * ring,struct wx_ring_container * head)1684 static void wx_add_ring(struct wx_ring *ring, struct wx_ring_container *head)
1685 {
1686 ring->next = head->ring;
1687 head->ring = ring;
1688 head->count++;
1689 }
1690
1691 /**
1692 * wx_alloc_q_vector - Allocate memory for a single interrupt vector
1693 * @wx: board private structure to initialize
1694 * @v_count: q_vectors allocated on wx, used for ring interleaving
1695 * @v_idx: index of vector in wx struct
1696 * @txr_count: total number of Tx rings to allocate
1697 * @txr_idx: index of first Tx ring to allocate
1698 * @rxr_count: total number of Rx rings to allocate
1699 * @rxr_idx: index of first Rx ring to allocate
1700 *
1701 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1702 **/
wx_alloc_q_vector(struct wx * wx,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)1703 static int wx_alloc_q_vector(struct wx *wx,
1704 unsigned int v_count, unsigned int v_idx,
1705 unsigned int txr_count, unsigned int txr_idx,
1706 unsigned int rxr_count, unsigned int rxr_idx)
1707 {
1708 struct wx_q_vector *q_vector;
1709 int ring_count, default_itr;
1710 struct wx_ring *ring;
1711
1712 /* note this will allocate space for the ring structure as well! */
1713 ring_count = txr_count + rxr_count;
1714
1715 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
1716 GFP_KERNEL);
1717 if (!q_vector)
1718 return -ENOMEM;
1719
1720 /* initialize NAPI */
1721 netif_napi_add(wx->netdev, &q_vector->napi,
1722 wx_poll);
1723
1724 /* tie q_vector and wx together */
1725 wx->q_vector[v_idx] = q_vector;
1726 q_vector->wx = wx;
1727 q_vector->v_idx = v_idx;
1728 if (cpu_online(v_idx))
1729 q_vector->numa_node = cpu_to_node(v_idx);
1730
1731 /* initialize pointer to rings */
1732 ring = q_vector->ring;
1733
1734 if (wx->mac.type == wx_mac_sp)
1735 default_itr = WX_12K_ITR;
1736 else
1737 default_itr = WX_7K_ITR;
1738 /* initialize ITR */
1739 if (txr_count && !rxr_count)
1740 /* tx only vector */
1741 q_vector->itr = wx->tx_itr_setting ?
1742 default_itr : wx->tx_itr_setting;
1743 else
1744 /* rx or rx/tx vector */
1745 q_vector->itr = wx->rx_itr_setting ?
1746 default_itr : wx->rx_itr_setting;
1747
1748 while (txr_count) {
1749 /* assign generic ring traits */
1750 ring->dev = &wx->pdev->dev;
1751 ring->netdev = wx->netdev;
1752
1753 /* configure backlink on ring */
1754 ring->q_vector = q_vector;
1755
1756 /* update q_vector Tx values */
1757 wx_add_ring(ring, &q_vector->tx);
1758
1759 /* apply Tx specific ring traits */
1760 ring->count = wx->tx_ring_count;
1761
1762 ring->queue_index = txr_idx;
1763
1764 /* assign ring to wx */
1765 wx->tx_ring[txr_idx] = ring;
1766
1767 /* update count and index */
1768 txr_count--;
1769 txr_idx += v_count;
1770
1771 /* push pointer to next ring */
1772 ring++;
1773 }
1774
1775 while (rxr_count) {
1776 /* assign generic ring traits */
1777 ring->dev = &wx->pdev->dev;
1778 ring->netdev = wx->netdev;
1779
1780 /* configure backlink on ring */
1781 ring->q_vector = q_vector;
1782
1783 /* update q_vector Rx values */
1784 wx_add_ring(ring, &q_vector->rx);
1785
1786 /* apply Rx specific ring traits */
1787 ring->count = wx->rx_ring_count;
1788 ring->queue_index = rxr_idx;
1789
1790 /* assign ring to wx */
1791 wx->rx_ring[rxr_idx] = ring;
1792
1793 /* update count and index */
1794 rxr_count--;
1795 rxr_idx += v_count;
1796
1797 /* push pointer to next ring */
1798 ring++;
1799 }
1800
1801 return 0;
1802 }
1803
1804 /**
1805 * wx_free_q_vector - Free memory allocated for specific interrupt vector
1806 * @wx: board private structure to initialize
1807 * @v_idx: Index of vector to be freed
1808 *
1809 * This function frees the memory allocated to the q_vector. In addition if
1810 * NAPI is enabled it will delete any references to the NAPI struct prior
1811 * to freeing the q_vector.
1812 **/
wx_free_q_vector(struct wx * wx,int v_idx)1813 static void wx_free_q_vector(struct wx *wx, int v_idx)
1814 {
1815 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
1816 struct wx_ring *ring;
1817
1818 wx_for_each_ring(ring, q_vector->tx)
1819 wx->tx_ring[ring->queue_index] = NULL;
1820
1821 wx_for_each_ring(ring, q_vector->rx)
1822 wx->rx_ring[ring->queue_index] = NULL;
1823
1824 wx->q_vector[v_idx] = NULL;
1825 netif_napi_del(&q_vector->napi);
1826 kfree_rcu(q_vector, rcu);
1827 }
1828
1829 /**
1830 * wx_alloc_q_vectors - Allocate memory for interrupt vectors
1831 * @wx: board private structure to initialize
1832 *
1833 * We allocate one q_vector per queue interrupt. If allocation fails we
1834 * return -ENOMEM.
1835 **/
wx_alloc_q_vectors(struct wx * wx)1836 static int wx_alloc_q_vectors(struct wx *wx)
1837 {
1838 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1839 unsigned int rxr_remaining = wx->num_rx_queues;
1840 unsigned int txr_remaining = wx->num_tx_queues;
1841 unsigned int q_vectors = wx->num_q_vectors;
1842 int rqpv, tqpv;
1843 int err;
1844
1845 for (; v_idx < q_vectors; v_idx++) {
1846 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1847 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1848 err = wx_alloc_q_vector(wx, q_vectors, v_idx,
1849 tqpv, txr_idx,
1850 rqpv, rxr_idx);
1851
1852 if (err)
1853 goto err_out;
1854
1855 /* update counts and index */
1856 rxr_remaining -= rqpv;
1857 txr_remaining -= tqpv;
1858 rxr_idx++;
1859 txr_idx++;
1860 }
1861
1862 return 0;
1863
1864 err_out:
1865 wx->num_tx_queues = 0;
1866 wx->num_rx_queues = 0;
1867 wx->num_q_vectors = 0;
1868
1869 while (v_idx--)
1870 wx_free_q_vector(wx, v_idx);
1871
1872 return -ENOMEM;
1873 }
1874
1875 /**
1876 * wx_free_q_vectors - Free memory allocated for interrupt vectors
1877 * @wx: board private structure to initialize
1878 *
1879 * This function frees the memory allocated to the q_vectors. In addition if
1880 * NAPI is enabled it will delete any references to the NAPI struct prior
1881 * to freeing the q_vector.
1882 **/
wx_free_q_vectors(struct wx * wx)1883 static void wx_free_q_vectors(struct wx *wx)
1884 {
1885 int v_idx = wx->num_q_vectors;
1886
1887 wx->num_tx_queues = 0;
1888 wx->num_rx_queues = 0;
1889 wx->num_q_vectors = 0;
1890
1891 while (v_idx--)
1892 wx_free_q_vector(wx, v_idx);
1893 }
1894
wx_reset_interrupt_capability(struct wx * wx)1895 void wx_reset_interrupt_capability(struct wx *wx)
1896 {
1897 struct pci_dev *pdev = wx->pdev;
1898
1899 if (!pdev->msi_enabled && !pdev->msix_enabled)
1900 return;
1901
1902 if (pdev->msix_enabled) {
1903 kfree(wx->msix_entries);
1904 wx->msix_entries = NULL;
1905 }
1906 pci_free_irq_vectors(wx->pdev);
1907 }
1908 EXPORT_SYMBOL(wx_reset_interrupt_capability);
1909
1910 /**
1911 * wx_clear_interrupt_scheme - Clear the current interrupt scheme settings
1912 * @wx: board private structure to clear interrupt scheme on
1913 *
1914 * We go through and clear interrupt specific resources and reset the structure
1915 * to pre-load conditions
1916 **/
wx_clear_interrupt_scheme(struct wx * wx)1917 void wx_clear_interrupt_scheme(struct wx *wx)
1918 {
1919 wx_free_q_vectors(wx);
1920 wx_reset_interrupt_capability(wx);
1921 }
1922 EXPORT_SYMBOL(wx_clear_interrupt_scheme);
1923
wx_init_interrupt_scheme(struct wx * wx)1924 int wx_init_interrupt_scheme(struct wx *wx)
1925 {
1926 int ret;
1927
1928 /* Number of supported queues */
1929 wx_set_num_queues(wx);
1930
1931 /* Set interrupt mode */
1932 ret = wx_set_interrupt_capability(wx);
1933 if (ret) {
1934 wx_err(wx, "Allocate irq vectors for failed.\n");
1935 return ret;
1936 }
1937
1938 /* Allocate memory for queues */
1939 ret = wx_alloc_q_vectors(wx);
1940 if (ret) {
1941 wx_err(wx, "Unable to allocate memory for queue vectors.\n");
1942 wx_reset_interrupt_capability(wx);
1943 return ret;
1944 }
1945
1946 wx_cache_ring_rss(wx);
1947
1948 return 0;
1949 }
1950 EXPORT_SYMBOL(wx_init_interrupt_scheme);
1951
wx_msix_clean_rings(int __always_unused irq,void * data)1952 irqreturn_t wx_msix_clean_rings(int __always_unused irq, void *data)
1953 {
1954 struct wx_q_vector *q_vector = data;
1955
1956 /* EIAM disabled interrupts (on this vector) for us */
1957 if (q_vector->rx.ring || q_vector->tx.ring)
1958 napi_schedule_irqoff(&q_vector->napi);
1959
1960 return IRQ_HANDLED;
1961 }
1962 EXPORT_SYMBOL(wx_msix_clean_rings);
1963
wx_free_irq(struct wx * wx)1964 void wx_free_irq(struct wx *wx)
1965 {
1966 struct pci_dev *pdev = wx->pdev;
1967 int vector;
1968
1969 if (!(pdev->msix_enabled)) {
1970 free_irq(pdev->irq, wx);
1971 return;
1972 }
1973
1974 for (vector = 0; vector < wx->num_q_vectors; vector++) {
1975 struct wx_q_vector *q_vector = wx->q_vector[vector];
1976 struct msix_entry *entry = &wx->msix_entries[vector];
1977
1978 /* free only the irqs that were actually requested */
1979 if (!q_vector->rx.ring && !q_vector->tx.ring)
1980 continue;
1981
1982 free_irq(entry->vector, q_vector);
1983 }
1984
1985 if (wx->mac.type == wx_mac_em)
1986 free_irq(wx->msix_entries[vector].vector, wx);
1987 }
1988 EXPORT_SYMBOL(wx_free_irq);
1989
1990 /**
1991 * wx_setup_isb_resources - allocate interrupt status resources
1992 * @wx: board private structure
1993 *
1994 * Return 0 on success, negative on failure
1995 **/
wx_setup_isb_resources(struct wx * wx)1996 int wx_setup_isb_resources(struct wx *wx)
1997 {
1998 struct pci_dev *pdev = wx->pdev;
1999
2000 wx->isb_mem = dma_alloc_coherent(&pdev->dev,
2001 sizeof(u32) * 4,
2002 &wx->isb_dma,
2003 GFP_KERNEL);
2004 if (!wx->isb_mem) {
2005 wx_err(wx, "Alloc isb_mem failed\n");
2006 return -ENOMEM;
2007 }
2008
2009 return 0;
2010 }
2011 EXPORT_SYMBOL(wx_setup_isb_resources);
2012
2013 /**
2014 * wx_free_isb_resources - allocate all queues Rx resources
2015 * @wx: board private structure
2016 *
2017 * Return 0 on success, negative on failure
2018 **/
wx_free_isb_resources(struct wx * wx)2019 void wx_free_isb_resources(struct wx *wx)
2020 {
2021 struct pci_dev *pdev = wx->pdev;
2022
2023 dma_free_coherent(&pdev->dev, sizeof(u32) * 4,
2024 wx->isb_mem, wx->isb_dma);
2025 wx->isb_mem = NULL;
2026 }
2027 EXPORT_SYMBOL(wx_free_isb_resources);
2028
wx_misc_isb(struct wx * wx,enum wx_isb_idx idx)2029 u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)
2030 {
2031 u32 cur_tag = 0;
2032
2033 cur_tag = wx->isb_mem[WX_ISB_HEADER];
2034 wx->isb_tag[idx] = cur_tag;
2035
2036 return (__force u32)cpu_to_le32(wx->isb_mem[idx]);
2037 }
2038 EXPORT_SYMBOL(wx_misc_isb);
2039
2040 /**
2041 * wx_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
2042 * @wx: pointer to wx struct
2043 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
2044 * @queue: queue to map the corresponding interrupt to
2045 * @msix_vector: the vector to map to the corresponding queue
2046 *
2047 **/
wx_set_ivar(struct wx * wx,s8 direction,u16 queue,u16 msix_vector)2048 static void wx_set_ivar(struct wx *wx, s8 direction,
2049 u16 queue, u16 msix_vector)
2050 {
2051 u32 ivar, index;
2052
2053 if (direction == -1) {
2054 /* other causes */
2055 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2056 index = 0;
2057 ivar = rd32(wx, WX_PX_MISC_IVAR);
2058 ivar &= ~(0xFF << index);
2059 ivar |= (msix_vector << index);
2060 wr32(wx, WX_PX_MISC_IVAR, ivar);
2061 } else {
2062 /* tx or rx causes */
2063 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2064 index = ((16 * (queue & 1)) + (8 * direction));
2065 ivar = rd32(wx, WX_PX_IVAR(queue >> 1));
2066 ivar &= ~(0xFF << index);
2067 ivar |= (msix_vector << index);
2068 wr32(wx, WX_PX_IVAR(queue >> 1), ivar);
2069 }
2070 }
2071
2072 /**
2073 * wx_write_eitr - write EITR register in hardware specific way
2074 * @q_vector: structure containing interrupt and ring information
2075 *
2076 * This function is made to be called by ethtool and by the driver
2077 * when it needs to update EITR registers at runtime. Hardware
2078 * specific quirks/differences are taken care of here.
2079 */
wx_write_eitr(struct wx_q_vector * q_vector)2080 static void wx_write_eitr(struct wx_q_vector *q_vector)
2081 {
2082 struct wx *wx = q_vector->wx;
2083 int v_idx = q_vector->v_idx;
2084 u32 itr_reg;
2085
2086 if (wx->mac.type == wx_mac_sp)
2087 itr_reg = q_vector->itr & WX_SP_MAX_EITR;
2088 else
2089 itr_reg = q_vector->itr & WX_EM_MAX_EITR;
2090
2091 itr_reg |= WX_PX_ITR_CNT_WDIS;
2092
2093 wr32(wx, WX_PX_ITR(v_idx), itr_reg);
2094 }
2095
2096 /**
2097 * wx_configure_vectors - Configure vectors for hardware
2098 * @wx: board private structure
2099 *
2100 * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/LEGACY
2101 * interrupts.
2102 **/
wx_configure_vectors(struct wx * wx)2103 void wx_configure_vectors(struct wx *wx)
2104 {
2105 struct pci_dev *pdev = wx->pdev;
2106 u32 eitrsel = 0;
2107 u16 v_idx;
2108
2109 if (pdev->msix_enabled) {
2110 /* Populate MSIX to EITR Select */
2111 wr32(wx, WX_PX_ITRSEL, eitrsel);
2112 /* use EIAM to auto-mask when MSI-X interrupt is asserted
2113 * this saves a register write for every interrupt
2114 */
2115 wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);
2116 } else {
2117 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
2118 * specifically only auto mask tx and rx interrupts.
2119 */
2120 wr32(wx, WX_PX_GPIE, 0);
2121 }
2122
2123 /* Populate the IVAR table and set the ITR values to the
2124 * corresponding register.
2125 */
2126 for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {
2127 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2128 struct wx_ring *ring;
2129
2130 wx_for_each_ring(ring, q_vector->rx)
2131 wx_set_ivar(wx, 0, ring->reg_idx, v_idx);
2132
2133 wx_for_each_ring(ring, q_vector->tx)
2134 wx_set_ivar(wx, 1, ring->reg_idx, v_idx);
2135
2136 wx_write_eitr(q_vector);
2137 }
2138
2139 wx_set_ivar(wx, -1, 0, v_idx);
2140 if (pdev->msix_enabled)
2141 wr32(wx, WX_PX_ITR(v_idx), 1950);
2142 }
2143 EXPORT_SYMBOL(wx_configure_vectors);
2144
2145 /**
2146 * wx_clean_rx_ring - Free Rx Buffers per Queue
2147 * @rx_ring: ring to free buffers from
2148 **/
wx_clean_rx_ring(struct wx_ring * rx_ring)2149 static void wx_clean_rx_ring(struct wx_ring *rx_ring)
2150 {
2151 struct wx_rx_buffer *rx_buffer;
2152 u16 i = rx_ring->next_to_clean;
2153
2154 rx_buffer = &rx_ring->rx_buffer_info[i];
2155
2156 /* Free all the Rx ring sk_buffs */
2157 while (i != rx_ring->next_to_alloc) {
2158 if (rx_buffer->skb) {
2159 struct sk_buff *skb = rx_buffer->skb;
2160
2161 if (WX_CB(skb)->page_released)
2162 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2163
2164 dev_kfree_skb(skb);
2165 }
2166
2167 /* Invalidate cache lines that may have been written to by
2168 * device so that we avoid corrupting memory.
2169 */
2170 dma_sync_single_range_for_cpu(rx_ring->dev,
2171 rx_buffer->dma,
2172 rx_buffer->page_offset,
2173 WX_RX_BUFSZ,
2174 DMA_FROM_DEVICE);
2175
2176 /* free resources associated with mapping */
2177 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2178
2179 i++;
2180 rx_buffer++;
2181 if (i == rx_ring->count) {
2182 i = 0;
2183 rx_buffer = rx_ring->rx_buffer_info;
2184 }
2185 }
2186
2187 rx_ring->next_to_alloc = 0;
2188 rx_ring->next_to_clean = 0;
2189 rx_ring->next_to_use = 0;
2190 }
2191
2192 /**
2193 * wx_clean_all_rx_rings - Free Rx Buffers for all queues
2194 * @wx: board private structure
2195 **/
wx_clean_all_rx_rings(struct wx * wx)2196 void wx_clean_all_rx_rings(struct wx *wx)
2197 {
2198 int i;
2199
2200 for (i = 0; i < wx->num_rx_queues; i++)
2201 wx_clean_rx_ring(wx->rx_ring[i]);
2202 }
2203 EXPORT_SYMBOL(wx_clean_all_rx_rings);
2204
2205 /**
2206 * wx_free_rx_resources - Free Rx Resources
2207 * @rx_ring: ring to clean the resources from
2208 *
2209 * Free all receive software resources
2210 **/
wx_free_rx_resources(struct wx_ring * rx_ring)2211 static void wx_free_rx_resources(struct wx_ring *rx_ring)
2212 {
2213 wx_clean_rx_ring(rx_ring);
2214 kvfree(rx_ring->rx_buffer_info);
2215 rx_ring->rx_buffer_info = NULL;
2216
2217 /* if not set, then don't free */
2218 if (!rx_ring->desc)
2219 return;
2220
2221 dma_free_coherent(rx_ring->dev, rx_ring->size,
2222 rx_ring->desc, rx_ring->dma);
2223
2224 rx_ring->desc = NULL;
2225
2226 if (rx_ring->page_pool) {
2227 page_pool_destroy(rx_ring->page_pool);
2228 rx_ring->page_pool = NULL;
2229 }
2230 }
2231
2232 /**
2233 * wx_free_all_rx_resources - Free Rx Resources for All Queues
2234 * @wx: pointer to hardware structure
2235 *
2236 * Free all receive software resources
2237 **/
wx_free_all_rx_resources(struct wx * wx)2238 static void wx_free_all_rx_resources(struct wx *wx)
2239 {
2240 int i;
2241
2242 for (i = 0; i < wx->num_rx_queues; i++)
2243 wx_free_rx_resources(wx->rx_ring[i]);
2244 }
2245
2246 /**
2247 * wx_clean_tx_ring - Free Tx Buffers
2248 * @tx_ring: ring to be cleaned
2249 **/
wx_clean_tx_ring(struct wx_ring * tx_ring)2250 static void wx_clean_tx_ring(struct wx_ring *tx_ring)
2251 {
2252 struct wx_tx_buffer *tx_buffer;
2253 u16 i = tx_ring->next_to_clean;
2254
2255 tx_buffer = &tx_ring->tx_buffer_info[i];
2256
2257 while (i != tx_ring->next_to_use) {
2258 union wx_tx_desc *eop_desc, *tx_desc;
2259
2260 /* Free all the Tx ring sk_buffs */
2261 dev_kfree_skb_any(tx_buffer->skb);
2262
2263 /* unmap skb header data */
2264 dma_unmap_single(tx_ring->dev,
2265 dma_unmap_addr(tx_buffer, dma),
2266 dma_unmap_len(tx_buffer, len),
2267 DMA_TO_DEVICE);
2268
2269 /* check for eop_desc to determine the end of the packet */
2270 eop_desc = tx_buffer->next_to_watch;
2271 tx_desc = WX_TX_DESC(tx_ring, i);
2272
2273 /* unmap remaining buffers */
2274 while (tx_desc != eop_desc) {
2275 tx_buffer++;
2276 tx_desc++;
2277 i++;
2278 if (unlikely(i == tx_ring->count)) {
2279 i = 0;
2280 tx_buffer = tx_ring->tx_buffer_info;
2281 tx_desc = WX_TX_DESC(tx_ring, 0);
2282 }
2283
2284 /* unmap any remaining paged data */
2285 if (dma_unmap_len(tx_buffer, len))
2286 dma_unmap_page(tx_ring->dev,
2287 dma_unmap_addr(tx_buffer, dma),
2288 dma_unmap_len(tx_buffer, len),
2289 DMA_TO_DEVICE);
2290 }
2291
2292 /* move us one more past the eop_desc for start of next pkt */
2293 tx_buffer++;
2294 i++;
2295 if (unlikely(i == tx_ring->count)) {
2296 i = 0;
2297 tx_buffer = tx_ring->tx_buffer_info;
2298 }
2299 }
2300
2301 netdev_tx_reset_queue(wx_txring_txq(tx_ring));
2302
2303 /* reset next_to_use and next_to_clean */
2304 tx_ring->next_to_use = 0;
2305 tx_ring->next_to_clean = 0;
2306 }
2307
2308 /**
2309 * wx_clean_all_tx_rings - Free Tx Buffers for all queues
2310 * @wx: board private structure
2311 **/
wx_clean_all_tx_rings(struct wx * wx)2312 void wx_clean_all_tx_rings(struct wx *wx)
2313 {
2314 int i;
2315
2316 for (i = 0; i < wx->num_tx_queues; i++)
2317 wx_clean_tx_ring(wx->tx_ring[i]);
2318 }
2319 EXPORT_SYMBOL(wx_clean_all_tx_rings);
2320
2321 /**
2322 * wx_free_tx_resources - Free Tx Resources per Queue
2323 * @tx_ring: Tx descriptor ring for a specific queue
2324 *
2325 * Free all transmit software resources
2326 **/
wx_free_tx_resources(struct wx_ring * tx_ring)2327 static void wx_free_tx_resources(struct wx_ring *tx_ring)
2328 {
2329 wx_clean_tx_ring(tx_ring);
2330 kvfree(tx_ring->tx_buffer_info);
2331 tx_ring->tx_buffer_info = NULL;
2332
2333 /* if not set, then don't free */
2334 if (!tx_ring->desc)
2335 return;
2336
2337 dma_free_coherent(tx_ring->dev, tx_ring->size,
2338 tx_ring->desc, tx_ring->dma);
2339 tx_ring->desc = NULL;
2340 }
2341
2342 /**
2343 * wx_free_all_tx_resources - Free Tx Resources for All Queues
2344 * @wx: pointer to hardware structure
2345 *
2346 * Free all transmit software resources
2347 **/
wx_free_all_tx_resources(struct wx * wx)2348 static void wx_free_all_tx_resources(struct wx *wx)
2349 {
2350 int i;
2351
2352 for (i = 0; i < wx->num_tx_queues; i++)
2353 wx_free_tx_resources(wx->tx_ring[i]);
2354 }
2355
wx_free_resources(struct wx * wx)2356 void wx_free_resources(struct wx *wx)
2357 {
2358 wx_free_isb_resources(wx);
2359 wx_free_all_rx_resources(wx);
2360 wx_free_all_tx_resources(wx);
2361 }
2362 EXPORT_SYMBOL(wx_free_resources);
2363
wx_alloc_page_pool(struct wx_ring * rx_ring)2364 static int wx_alloc_page_pool(struct wx_ring *rx_ring)
2365 {
2366 int ret = 0;
2367
2368 struct page_pool_params pp_params = {
2369 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
2370 .order = 0,
2371 .pool_size = rx_ring->size,
2372 .nid = dev_to_node(rx_ring->dev),
2373 .dev = rx_ring->dev,
2374 .dma_dir = DMA_FROM_DEVICE,
2375 .offset = 0,
2376 .max_len = PAGE_SIZE,
2377 };
2378
2379 rx_ring->page_pool = page_pool_create(&pp_params);
2380 if (IS_ERR(rx_ring->page_pool)) {
2381 ret = PTR_ERR(rx_ring->page_pool);
2382 rx_ring->page_pool = NULL;
2383 }
2384
2385 return ret;
2386 }
2387
2388 /**
2389 * wx_setup_rx_resources - allocate Rx resources (Descriptors)
2390 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2391 *
2392 * Returns 0 on success, negative on failure
2393 **/
wx_setup_rx_resources(struct wx_ring * rx_ring)2394 static int wx_setup_rx_resources(struct wx_ring *rx_ring)
2395 {
2396 struct device *dev = rx_ring->dev;
2397 int orig_node = dev_to_node(dev);
2398 int numa_node = NUMA_NO_NODE;
2399 int size, ret;
2400
2401 size = sizeof(struct wx_rx_buffer) * rx_ring->count;
2402
2403 if (rx_ring->q_vector)
2404 numa_node = rx_ring->q_vector->numa_node;
2405
2406 rx_ring->rx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2407 if (!rx_ring->rx_buffer_info)
2408 rx_ring->rx_buffer_info = kvmalloc(size, GFP_KERNEL);
2409 if (!rx_ring->rx_buffer_info)
2410 goto err;
2411
2412 /* Round up to nearest 4K */
2413 rx_ring->size = rx_ring->count * sizeof(union wx_rx_desc);
2414 rx_ring->size = ALIGN(rx_ring->size, 4096);
2415
2416 set_dev_node(dev, numa_node);
2417 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2418 &rx_ring->dma, GFP_KERNEL);
2419 if (!rx_ring->desc) {
2420 set_dev_node(dev, orig_node);
2421 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2422 &rx_ring->dma, GFP_KERNEL);
2423 }
2424
2425 if (!rx_ring->desc)
2426 goto err;
2427
2428 rx_ring->next_to_clean = 0;
2429 rx_ring->next_to_use = 0;
2430
2431 ret = wx_alloc_page_pool(rx_ring);
2432 if (ret < 0) {
2433 dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);
2434 goto err_desc;
2435 }
2436
2437 return 0;
2438
2439 err_desc:
2440 dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
2441 err:
2442 kvfree(rx_ring->rx_buffer_info);
2443 rx_ring->rx_buffer_info = NULL;
2444 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
2445 return -ENOMEM;
2446 }
2447
2448 /**
2449 * wx_setup_all_rx_resources - allocate all queues Rx resources
2450 * @wx: pointer to hardware structure
2451 *
2452 * If this function returns with an error, then it's possible one or
2453 * more of the rings is populated (while the rest are not). It is the
2454 * callers duty to clean those orphaned rings.
2455 *
2456 * Return 0 on success, negative on failure
2457 **/
wx_setup_all_rx_resources(struct wx * wx)2458 static int wx_setup_all_rx_resources(struct wx *wx)
2459 {
2460 int i, err = 0;
2461
2462 for (i = 0; i < wx->num_rx_queues; i++) {
2463 err = wx_setup_rx_resources(wx->rx_ring[i]);
2464 if (!err)
2465 continue;
2466
2467 wx_err(wx, "Allocation for Rx Queue %u failed\n", i);
2468 goto err_setup_rx;
2469 }
2470
2471 return 0;
2472 err_setup_rx:
2473 /* rewind the index freeing the rings as we go */
2474 while (i--)
2475 wx_free_rx_resources(wx->rx_ring[i]);
2476 return err;
2477 }
2478
2479 /**
2480 * wx_setup_tx_resources - allocate Tx resources (Descriptors)
2481 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2482 *
2483 * Return 0 on success, negative on failure
2484 **/
wx_setup_tx_resources(struct wx_ring * tx_ring)2485 static int wx_setup_tx_resources(struct wx_ring *tx_ring)
2486 {
2487 struct device *dev = tx_ring->dev;
2488 int orig_node = dev_to_node(dev);
2489 int numa_node = NUMA_NO_NODE;
2490 int size;
2491
2492 size = sizeof(struct wx_tx_buffer) * tx_ring->count;
2493
2494 if (tx_ring->q_vector)
2495 numa_node = tx_ring->q_vector->numa_node;
2496
2497 tx_ring->tx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2498 if (!tx_ring->tx_buffer_info)
2499 tx_ring->tx_buffer_info = kvmalloc(size, GFP_KERNEL);
2500 if (!tx_ring->tx_buffer_info)
2501 goto err;
2502
2503 /* round up to nearest 4K */
2504 tx_ring->size = tx_ring->count * sizeof(union wx_tx_desc);
2505 tx_ring->size = ALIGN(tx_ring->size, 4096);
2506
2507 set_dev_node(dev, numa_node);
2508 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2509 &tx_ring->dma, GFP_KERNEL);
2510 if (!tx_ring->desc) {
2511 set_dev_node(dev, orig_node);
2512 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2513 &tx_ring->dma, GFP_KERNEL);
2514 }
2515
2516 if (!tx_ring->desc)
2517 goto err;
2518
2519 tx_ring->next_to_use = 0;
2520 tx_ring->next_to_clean = 0;
2521
2522 return 0;
2523
2524 err:
2525 kvfree(tx_ring->tx_buffer_info);
2526 tx_ring->tx_buffer_info = NULL;
2527 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
2528 return -ENOMEM;
2529 }
2530
2531 /**
2532 * wx_setup_all_tx_resources - allocate all queues Tx resources
2533 * @wx: pointer to private structure
2534 *
2535 * If this function returns with an error, then it's possible one or
2536 * more of the rings is populated (while the rest are not). It is the
2537 * callers duty to clean those orphaned rings.
2538 *
2539 * Return 0 on success, negative on failure
2540 **/
wx_setup_all_tx_resources(struct wx * wx)2541 static int wx_setup_all_tx_resources(struct wx *wx)
2542 {
2543 int i, err = 0;
2544
2545 for (i = 0; i < wx->num_tx_queues; i++) {
2546 err = wx_setup_tx_resources(wx->tx_ring[i]);
2547 if (!err)
2548 continue;
2549
2550 wx_err(wx, "Allocation for Tx Queue %u failed\n", i);
2551 goto err_setup_tx;
2552 }
2553
2554 return 0;
2555 err_setup_tx:
2556 /* rewind the index freeing the rings as we go */
2557 while (i--)
2558 wx_free_tx_resources(wx->tx_ring[i]);
2559 return err;
2560 }
2561
wx_setup_resources(struct wx * wx)2562 int wx_setup_resources(struct wx *wx)
2563 {
2564 int err;
2565
2566 /* allocate transmit descriptors */
2567 err = wx_setup_all_tx_resources(wx);
2568 if (err)
2569 return err;
2570
2571 /* allocate receive descriptors */
2572 err = wx_setup_all_rx_resources(wx);
2573 if (err)
2574 goto err_free_tx;
2575
2576 err = wx_setup_isb_resources(wx);
2577 if (err)
2578 goto err_free_rx;
2579
2580 return 0;
2581
2582 err_free_rx:
2583 wx_free_all_rx_resources(wx);
2584 err_free_tx:
2585 wx_free_all_tx_resources(wx);
2586
2587 return err;
2588 }
2589 EXPORT_SYMBOL(wx_setup_resources);
2590
2591 /**
2592 * wx_get_stats64 - Get System Network Statistics
2593 * @netdev: network interface device structure
2594 * @stats: storage space for 64bit statistics
2595 */
wx_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)2596 void wx_get_stats64(struct net_device *netdev,
2597 struct rtnl_link_stats64 *stats)
2598 {
2599 struct wx *wx = netdev_priv(netdev);
2600 int i;
2601
2602 rcu_read_lock();
2603 for (i = 0; i < wx->num_rx_queues; i++) {
2604 struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);
2605 u64 bytes, packets;
2606 unsigned int start;
2607
2608 if (ring) {
2609 do {
2610 start = u64_stats_fetch_begin(&ring->syncp);
2611 packets = ring->stats.packets;
2612 bytes = ring->stats.bytes;
2613 } while (u64_stats_fetch_retry(&ring->syncp, start));
2614 stats->rx_packets += packets;
2615 stats->rx_bytes += bytes;
2616 }
2617 }
2618
2619 for (i = 0; i < wx->num_tx_queues; i++) {
2620 struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);
2621 u64 bytes, packets;
2622 unsigned int start;
2623
2624 if (ring) {
2625 do {
2626 start = u64_stats_fetch_begin(&ring->syncp);
2627 packets = ring->stats.packets;
2628 bytes = ring->stats.bytes;
2629 } while (u64_stats_fetch_retry(&ring->syncp,
2630 start));
2631 stats->tx_packets += packets;
2632 stats->tx_bytes += bytes;
2633 }
2634 }
2635
2636 rcu_read_unlock();
2637 }
2638 EXPORT_SYMBOL(wx_get_stats64);
2639
wx_set_features(struct net_device * netdev,netdev_features_t features)2640 int wx_set_features(struct net_device *netdev, netdev_features_t features)
2641 {
2642 netdev_features_t changed = netdev->features ^ features;
2643 struct wx *wx = netdev_priv(netdev);
2644
2645 if (changed & NETIF_F_RXHASH)
2646 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN,
2647 WX_RDB_RA_CTL_RSS_EN);
2648 else
2649 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN, 0);
2650
2651 netdev->features = features;
2652
2653 if (changed &
2654 (NETIF_F_HW_VLAN_CTAG_RX |
2655 NETIF_F_HW_VLAN_STAG_RX))
2656 wx_set_rx_mode(netdev);
2657
2658 return 0;
2659 }
2660 EXPORT_SYMBOL(wx_set_features);
2661
2662 MODULE_LICENSE("GPL");
2663