1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/pci.h>
9 #include <linux/etherdevice.h>
10 #include <linux/vmalloc.h>
11
12 #include "octep_config.h"
13 #include "octep_main.h"
14
octep_oq_reset_indices(struct octep_oq * oq)15 static void octep_oq_reset_indices(struct octep_oq *oq)
16 {
17 oq->host_read_idx = 0;
18 oq->host_refill_idx = 0;
19 oq->refill_count = 0;
20 oq->last_pkt_count = 0;
21 oq->pkts_pending = 0;
22 }
23
24 /**
25 * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.
26 *
27 * @oq: Octeon Rx queue data structure.
28 *
29 * Return: 0, if successfully filled receive buffers for all descriptors.
30 * -1, if failed to allocate a buffer or failed to map for DMA.
31 */
octep_oq_fill_ring_buffers(struct octep_oq * oq)32 static int octep_oq_fill_ring_buffers(struct octep_oq *oq)
33 {
34 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
35 struct page *page;
36 u32 i;
37
38 for (i = 0; i < oq->max_count; i++) {
39 page = dev_alloc_page();
40 if (unlikely(!page)) {
41 dev_err(oq->dev, "Rx buffer alloc failed\n");
42 goto rx_buf_alloc_err;
43 }
44 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,
45 PAGE_SIZE,
46 DMA_FROM_DEVICE);
47 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {
48 dev_err(oq->dev,
49 "OQ-%d buffer alloc: DMA mapping error!\n",
50 oq->q_no);
51 put_page(page);
52 goto dma_map_err;
53 }
54 oq->buff_info[i].page = page;
55 }
56
57 return 0;
58
59 dma_map_err:
60 rx_buf_alloc_err:
61 while (i) {
62 i--;
63 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);
64 put_page(oq->buff_info[i].page);
65 oq->buff_info[i].page = NULL;
66 }
67
68 return -1;
69 }
70
71 /**
72 * octep_oq_refill() - refill buffers for used Rx ring descriptors.
73 *
74 * @oct: Octeon device private data structure.
75 * @oq: Octeon Rx queue data structure.
76 *
77 * Return: number of descriptors successfully refilled with receive buffers.
78 */
octep_oq_refill(struct octep_device * oct,struct octep_oq * oq)79 static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq)
80 {
81 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
82 struct page *page;
83 u32 refill_idx, i;
84
85 refill_idx = oq->host_refill_idx;
86 for (i = 0; i < oq->refill_count; i++) {
87 page = dev_alloc_page();
88 if (unlikely(!page)) {
89 dev_err(oq->dev, "refill: rx buffer alloc failed\n");
90 oq->stats.alloc_failures++;
91 break;
92 }
93
94 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
95 PAGE_SIZE, DMA_FROM_DEVICE);
96 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
97 dev_err(oq->dev,
98 "OQ-%d buffer refill: DMA mapping error!\n",
99 oq->q_no);
100 put_page(page);
101 oq->stats.alloc_failures++;
102 break;
103 }
104 oq->buff_info[refill_idx].page = page;
105 refill_idx++;
106 if (refill_idx == oq->max_count)
107 refill_idx = 0;
108 }
109 oq->host_refill_idx = refill_idx;
110 oq->refill_count -= i;
111
112 return i;
113 }
114
115 /**
116 * octep_setup_oq() - Setup a Rx queue.
117 *
118 * @oct: Octeon device private data structure.
119 * @q_no: Rx queue number to be setup.
120 *
121 * Allocate resources for a Rx queue.
122 */
octep_setup_oq(struct octep_device * oct,int q_no)123 static int octep_setup_oq(struct octep_device *oct, int q_no)
124 {
125 struct octep_oq *oq;
126 u32 desc_ring_size;
127
128 oq = vzalloc(sizeof(*oq));
129 if (!oq)
130 goto create_oq_fail;
131 oct->oq[q_no] = oq;
132
133 oq->octep_dev = oct;
134 oq->netdev = oct->netdev;
135 oq->dev = &oct->pdev->dev;
136 oq->q_no = q_no;
137 oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);
138 oq->ring_size_mask = oq->max_count - 1;
139 oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);
140 oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE;
141
142 /* When the hardware/firmware supports additional capabilities,
143 * additional header is filled-in by Octeon after length field in
144 * Rx packets. this header contains additional packet information.
145 */
146 if (oct->caps_enabled)
147 oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE;
148
149 oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);
150
151 desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE;
152 oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,
153 &oq->desc_ring_dma, GFP_KERNEL);
154
155 if (unlikely(!oq->desc_ring)) {
156 dev_err(oq->dev,
157 "Failed to allocate DMA memory for OQ-%d !!\n", q_no);
158 goto desc_dma_alloc_err;
159 }
160
161 oq->buff_info = vcalloc(oq->max_count, OCTEP_OQ_RECVBUF_SIZE);
162 if (unlikely(!oq->buff_info)) {
163 dev_err(&oct->pdev->dev,
164 "Failed to allocate buffer info for OQ-%d\n", q_no);
165 goto buf_list_err;
166 }
167
168 if (octep_oq_fill_ring_buffers(oq))
169 goto oq_fill_buff_err;
170
171 octep_oq_reset_indices(oq);
172 oct->hw_ops.setup_oq_regs(oct, q_no);
173 oct->num_oqs++;
174
175 return 0;
176
177 oq_fill_buff_err:
178 vfree(oq->buff_info);
179 oq->buff_info = NULL;
180 buf_list_err:
181 dma_free_coherent(oq->dev, desc_ring_size,
182 oq->desc_ring, oq->desc_ring_dma);
183 oq->desc_ring = NULL;
184 desc_dma_alloc_err:
185 vfree(oq);
186 oct->oq[q_no] = NULL;
187 create_oq_fail:
188 return -1;
189 }
190
191 /**
192 * octep_oq_free_ring_buffers() - Free ring buffers.
193 *
194 * @oq: Octeon Rx queue data structure.
195 *
196 * Free receive buffers in unused Rx queue descriptors.
197 */
octep_oq_free_ring_buffers(struct octep_oq * oq)198 static void octep_oq_free_ring_buffers(struct octep_oq *oq)
199 {
200 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
201 int i;
202
203 if (!oq->desc_ring || !oq->buff_info)
204 return;
205
206 for (i = 0; i < oq->max_count; i++) {
207 if (oq->buff_info[i].page) {
208 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,
209 PAGE_SIZE, DMA_FROM_DEVICE);
210 put_page(oq->buff_info[i].page);
211 oq->buff_info[i].page = NULL;
212 desc_ring[i].buffer_ptr = 0;
213 }
214 }
215 octep_oq_reset_indices(oq);
216 }
217
218 /**
219 * octep_free_oq() - Free Rx queue resources.
220 *
221 * @oq: Octeon Rx queue data structure.
222 *
223 * Free all resources of a Rx queue.
224 */
octep_free_oq(struct octep_oq * oq)225 static int octep_free_oq(struct octep_oq *oq)
226 {
227 struct octep_device *oct = oq->octep_dev;
228 int q_no = oq->q_no;
229
230 octep_oq_free_ring_buffers(oq);
231
232 vfree(oq->buff_info);
233
234 if (oq->desc_ring)
235 dma_free_coherent(oq->dev,
236 oq->max_count * OCTEP_OQ_DESC_SIZE,
237 oq->desc_ring, oq->desc_ring_dma);
238
239 vfree(oq);
240 oct->oq[q_no] = NULL;
241 oct->num_oqs--;
242 return 0;
243 }
244
245 /**
246 * octep_setup_oqs() - setup resources for all Rx queues.
247 *
248 * @oct: Octeon device private data structure.
249 */
octep_setup_oqs(struct octep_device * oct)250 int octep_setup_oqs(struct octep_device *oct)
251 {
252 int i, retval = 0;
253
254 oct->num_oqs = 0;
255 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
256 retval = octep_setup_oq(oct, i);
257 if (retval) {
258 dev_err(&oct->pdev->dev,
259 "Failed to setup OQ(RxQ)-%d.\n", i);
260 goto oq_setup_err;
261 }
262 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);
263 }
264
265 return 0;
266
267 oq_setup_err:
268 while (i) {
269 i--;
270 octep_free_oq(oct->oq[i]);
271 }
272 return -1;
273 }
274
275 /**
276 * octep_oq_dbell_init() - Initialize Rx queue doorbell.
277 *
278 * @oct: Octeon device private data structure.
279 *
280 * Write number of descriptors to Rx queue doorbell register.
281 */
octep_oq_dbell_init(struct octep_device * oct)282 void octep_oq_dbell_init(struct octep_device *oct)
283 {
284 int i;
285
286 for (i = 0; i < oct->num_oqs; i++)
287 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
288 }
289
290 /**
291 * octep_free_oqs() - Free resources of all Rx queues.
292 *
293 * @oct: Octeon device private data structure.
294 */
octep_free_oqs(struct octep_device * oct)295 void octep_free_oqs(struct octep_device *oct)
296 {
297 int i;
298
299 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
300 if (!oct->oq[i])
301 continue;
302 octep_free_oq(oct->oq[i]);
303 dev_dbg(&oct->pdev->dev,
304 "Successfully freed OQ(RxQ)-%d.\n", i);
305 }
306 }
307
308 /**
309 * octep_oq_check_hw_for_pkts() - Check for new Rx packets.
310 *
311 * @oct: Octeon device private data structure.
312 * @oq: Octeon Rx queue data structure.
313 *
314 * Return: packets received after previous check.
315 */
octep_oq_check_hw_for_pkts(struct octep_device * oct,struct octep_oq * oq)316 static int octep_oq_check_hw_for_pkts(struct octep_device *oct,
317 struct octep_oq *oq)
318 {
319 u32 pkt_count, new_pkts;
320
321 pkt_count = readl(oq->pkts_sent_reg);
322 new_pkts = pkt_count - oq->last_pkt_count;
323
324 /* Clear the hardware packets counter register if the rx queue is
325 * being processed continuously with-in a single interrupt and
326 * reached half its max value.
327 * this counter is not cleared every time read, to save write cycles.
328 */
329 if (unlikely(pkt_count > 0xF0000000U)) {
330 writel(pkt_count, oq->pkts_sent_reg);
331 pkt_count = readl(oq->pkts_sent_reg);
332 new_pkts += pkt_count;
333 }
334 oq->last_pkt_count = pkt_count;
335 oq->pkts_pending += new_pkts;
336 return new_pkts;
337 }
338
339 /**
340 * octep_oq_next_pkt() - Move to the next packet in Rx queue.
341 *
342 * @oq: Octeon Rx queue data structure.
343 * @buff_info: Current packet buffer info.
344 * @read_idx: Current packet index in the ring.
345 * @desc_used: Current packet descriptor number.
346 *
347 * Free the resources associated with a packet.
348 * Increment packet index in the ring and packet descriptor number.
349 */
octep_oq_next_pkt(struct octep_oq * oq,struct octep_rx_buffer * buff_info,u32 * read_idx,u32 * desc_used)350 static void octep_oq_next_pkt(struct octep_oq *oq,
351 struct octep_rx_buffer *buff_info,
352 u32 *read_idx, u32 *desc_used)
353 {
354 dma_unmap_page(oq->dev, oq->desc_ring[*read_idx].buffer_ptr,
355 PAGE_SIZE, DMA_FROM_DEVICE);
356 buff_info->page = NULL;
357 (*read_idx)++;
358 (*desc_used)++;
359 if (*read_idx == oq->max_count)
360 *read_idx = 0;
361 }
362
363 /**
364 * octep_oq_drop_rx() - Free the resources associated with a packet.
365 *
366 * @oq: Octeon Rx queue data structure.
367 * @buff_info: Current packet buffer info.
368 * @read_idx: Current packet index in the ring.
369 * @desc_used: Current packet descriptor number.
370 *
371 */
octep_oq_drop_rx(struct octep_oq * oq,struct octep_rx_buffer * buff_info,u32 * read_idx,u32 * desc_used)372 static void octep_oq_drop_rx(struct octep_oq *oq,
373 struct octep_rx_buffer *buff_info,
374 u32 *read_idx, u32 *desc_used)
375 {
376 int data_len = buff_info->len - oq->max_single_buffer_size;
377
378 while (data_len > 0) {
379 octep_oq_next_pkt(oq, buff_info, read_idx, desc_used);
380 data_len -= oq->buffer_size;
381 };
382 }
383
384 /**
385 * __octep_oq_process_rx() - Process hardware Rx queue and push to stack.
386 *
387 * @oct: Octeon device private data structure.
388 * @oq: Octeon Rx queue data structure.
389 * @pkts_to_process: number of packets to be processed.
390 *
391 * Process the new packets in Rx queue.
392 * Packets larger than single Rx buffer arrive in consecutive descriptors.
393 * But, count returned by the API only accounts full packets, not fragments.
394 *
395 * Return: number of packets processed and pushed to stack.
396 */
__octep_oq_process_rx(struct octep_device * oct,struct octep_oq * oq,u16 pkts_to_process)397 static int __octep_oq_process_rx(struct octep_device *oct,
398 struct octep_oq *oq, u16 pkts_to_process)
399 {
400 struct octep_oq_resp_hw_ext *resp_hw_ext = NULL;
401 struct octep_rx_buffer *buff_info;
402 struct octep_oq_resp_hw *resp_hw;
403 u32 pkt, rx_bytes, desc_used;
404 struct sk_buff *skb;
405 u16 data_offset;
406 u32 read_idx;
407
408 read_idx = oq->host_read_idx;
409 rx_bytes = 0;
410 desc_used = 0;
411 for (pkt = 0; pkt < pkts_to_process; pkt++) {
412 buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];
413 resp_hw = page_address(buff_info->page);
414
415 /* Swap the length field that is in Big-Endian to CPU */
416 buff_info->len = be64_to_cpu(resp_hw->length);
417 if (oct->caps_enabled & OCTEP_CAP_RX_CHECKSUM) {
418 /* Extended response header is immediately after
419 * response header (resp_hw)
420 */
421 resp_hw_ext = (struct octep_oq_resp_hw_ext *)
422 (resp_hw + 1);
423 buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE;
424 /* Packet Data is immediately after
425 * extended response header.
426 */
427 data_offset = OCTEP_OQ_RESP_HW_SIZE +
428 OCTEP_OQ_RESP_HW_EXT_SIZE;
429 } else {
430 /* Data is immediately after
431 * Hardware Rx response header.
432 */
433 data_offset = OCTEP_OQ_RESP_HW_SIZE;
434 }
435
436 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);
437
438 skb = build_skb((void *)resp_hw, PAGE_SIZE);
439 if (!skb) {
440 octep_oq_drop_rx(oq, buff_info,
441 &read_idx, &desc_used);
442 oq->stats.alloc_failures++;
443 continue;
444 }
445 skb_reserve(skb, data_offset);
446
447 rx_bytes += buff_info->len;
448
449 if (buff_info->len <= oq->max_single_buffer_size) {
450 skb_put(skb, buff_info->len);
451 } else {
452 struct skb_shared_info *shinfo;
453 u16 data_len;
454
455 /* Head fragment includes response header(s);
456 * subsequent fragments contains only data.
457 */
458 skb_put(skb, oq->max_single_buffer_size);
459 shinfo = skb_shinfo(skb);
460 data_len = buff_info->len - oq->max_single_buffer_size;
461 while (data_len) {
462 buff_info = (struct octep_rx_buffer *)
463 &oq->buff_info[read_idx];
464 if (data_len < oq->buffer_size) {
465 buff_info->len = data_len;
466 data_len = 0;
467 } else {
468 buff_info->len = oq->buffer_size;
469 data_len -= oq->buffer_size;
470 }
471
472 skb_add_rx_frag(skb, shinfo->nr_frags,
473 buff_info->page, 0,
474 buff_info->len,
475 buff_info->len);
476
477 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);
478 }
479 }
480
481 skb->dev = oq->netdev;
482 skb->protocol = eth_type_trans(skb, skb->dev);
483 if (resp_hw_ext &&
484 resp_hw_ext->csum_verified == OCTEP_CSUM_VERIFIED)
485 skb->ip_summed = CHECKSUM_UNNECESSARY;
486 else
487 skb->ip_summed = CHECKSUM_NONE;
488 napi_gro_receive(oq->napi, skb);
489 }
490
491 oq->host_read_idx = read_idx;
492 oq->refill_count += desc_used;
493 oq->stats.packets += pkt;
494 oq->stats.bytes += rx_bytes;
495
496 return pkt;
497 }
498
499 /**
500 * octep_oq_process_rx() - Process Rx queue.
501 *
502 * @oq: Octeon Rx queue data structure.
503 * @budget: max number of packets can be processed in one invocation.
504 *
505 * Check for newly received packets and process them.
506 * Keeps checking for new packets until budget is used or no new packets seen.
507 *
508 * Return: number of packets processed.
509 */
octep_oq_process_rx(struct octep_oq * oq,int budget)510 int octep_oq_process_rx(struct octep_oq *oq, int budget)
511 {
512 u32 pkts_available, pkts_processed, total_pkts_processed;
513 struct octep_device *oct = oq->octep_dev;
514
515 pkts_available = 0;
516 pkts_processed = 0;
517 total_pkts_processed = 0;
518 while (total_pkts_processed < budget) {
519 /* update pending count only when current one exhausted */
520 if (oq->pkts_pending == 0)
521 octep_oq_check_hw_for_pkts(oct, oq);
522 pkts_available = min(budget - total_pkts_processed,
523 oq->pkts_pending);
524 if (!pkts_available)
525 break;
526
527 pkts_processed = __octep_oq_process_rx(oct, oq,
528 pkts_available);
529 oq->pkts_pending -= pkts_processed;
530 total_pkts_processed += pkts_processed;
531 }
532
533 if (oq->refill_count >= oq->refill_threshold) {
534 u32 desc_refilled = octep_oq_refill(oct, oq);
535
536 /* flush pending writes before updating credits */
537 wmb();
538 writel(desc_refilled, oq->pkts_credit_reg);
539 }
540
541 return total_pkts_processed;
542 }
543