1 /**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2016 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more details.
17 ***********************************************************************/
18 #include <linux/pci.h>
19 #include <linux/netdevice.h>
20 #include <linux/vmalloc.h>
21 #include "liquidio_common.h"
22 #include "octeon_droq.h"
23 #include "octeon_iq.h"
24 #include "response_manager.h"
25 #include "octeon_device.h"
26 #include "octeon_main.h"
27 #include "octeon_network.h"
28 #include "cn66xx_regs.h"
29 #include "cn66xx_device.h"
30 #include "cn23xx_pf_device.h"
31 #include "cn23xx_vf_device.h"
32
33 struct niclist {
34 struct list_head list;
35 void *ptr;
36 };
37
38 struct __dispatch {
39 struct list_head list;
40 struct octeon_recv_info *rinfo;
41 octeon_dispatch_fn_t disp_fn;
42 };
43
44 /** Get the argument that the user set when registering dispatch
45 * function for a given opcode/subcode.
46 * @param octeon_dev - the octeon device pointer.
47 * @param opcode - the opcode for which the dispatch argument
48 * is to be checked.
49 * @param subcode - the subcode for which the dispatch argument
50 * is to be checked.
51 * @return Success: void * (argument to the dispatch function)
52 * @return Failure: NULL
53 *
54 */
octeon_get_dispatch_arg(struct octeon_device * octeon_dev,u16 opcode,u16 subcode)55 void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
56 u16 opcode, u16 subcode)
57 {
58 int idx;
59 struct list_head *dispatch;
60 void *fn_arg = NULL;
61 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
62
63 idx = combined_opcode & OCTEON_OPCODE_MASK;
64
65 spin_lock_bh(&octeon_dev->dispatch.lock);
66
67 if (octeon_dev->dispatch.count == 0) {
68 spin_unlock_bh(&octeon_dev->dispatch.lock);
69 return NULL;
70 }
71
72 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
73 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
74 } else {
75 list_for_each(dispatch,
76 &octeon_dev->dispatch.dlist[idx].list) {
77 if (((struct octeon_dispatch *)dispatch)->opcode ==
78 combined_opcode) {
79 fn_arg = ((struct octeon_dispatch *)
80 dispatch)->arg;
81 break;
82 }
83 }
84 }
85
86 spin_unlock_bh(&octeon_dev->dispatch.lock);
87 return fn_arg;
88 }
89
90 /** Check for packets on Droq. This function should be called with lock held.
91 * @param droq - Droq on which count is checked.
92 * @return Returns packet count.
93 */
octeon_droq_check_hw_for_pkts(struct octeon_droq * droq)94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
95 {
96 u32 pkt_count = 0;
97 u32 last_count;
98
99 pkt_count = readl(droq->pkts_sent_reg);
100
101 last_count = pkt_count - droq->pkt_count;
102 droq->pkt_count = pkt_count;
103
104 /* we shall write to cnts at napi irq enable or end of droq tasklet */
105 if (last_count)
106 atomic_add(last_count, &droq->pkts_pending);
107
108 return last_count;
109 }
110 EXPORT_SYMBOL_GPL(octeon_droq_check_hw_for_pkts);
111
octeon_droq_compute_max_packet_bufs(struct octeon_droq * droq)112 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
113 {
114 u32 count = 0;
115
116 /* max_empty_descs is the max. no. of descs that can have no buffers.
117 * If the empty desc count goes beyond this value, we cannot safely
118 * read in a 64K packet sent by Octeon
119 * (64K is max pkt size from Octeon)
120 */
121 droq->max_empty_descs = 0;
122
123 do {
124 droq->max_empty_descs++;
125 count += droq->buffer_size;
126 } while (count < (64 * 1024));
127
128 droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
129 }
130
octeon_droq_reset_indices(struct octeon_droq * droq)131 static void octeon_droq_reset_indices(struct octeon_droq *droq)
132 {
133 droq->read_idx = 0;
134 droq->write_idx = 0;
135 droq->refill_idx = 0;
136 droq->refill_count = 0;
137 atomic_set(&droq->pkts_pending, 0);
138 }
139
140 static void
octeon_droq_destroy_ring_buffers(struct octeon_device * oct,struct octeon_droq * droq)141 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
142 struct octeon_droq *droq)
143 {
144 u32 i;
145 struct octeon_skb_page_info *pg_info;
146
147 for (i = 0; i < droq->max_count; i++) {
148 pg_info = &droq->recv_buf_list[i].pg_info;
149 if (!pg_info)
150 continue;
151
152 if (pg_info->dma)
153 lio_unmap_ring(oct->pci_dev,
154 (u64)pg_info->dma);
155 pg_info->dma = 0;
156
157 if (pg_info->page)
158 recv_buffer_destroy(droq->recv_buf_list[i].buffer,
159 pg_info);
160
161 droq->recv_buf_list[i].buffer = NULL;
162 }
163
164 octeon_droq_reset_indices(droq);
165 }
166
167 static int
octeon_droq_setup_ring_buffers(struct octeon_device * oct,struct octeon_droq * droq)168 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
169 struct octeon_droq *droq)
170 {
171 u32 i;
172 void *buf;
173 struct octeon_droq_desc *desc_ring = droq->desc_ring;
174
175 for (i = 0; i < droq->max_count; i++) {
176 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
177
178 if (!buf) {
179 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
180 __func__);
181 droq->stats.rx_alloc_failure++;
182 return -ENOMEM;
183 }
184
185 droq->recv_buf_list[i].buffer = buf;
186 droq->recv_buf_list[i].data = get_rbd(buf);
187 desc_ring[i].info_ptr = 0;
188 desc_ring[i].buffer_ptr =
189 lio_map_ring(droq->recv_buf_list[i].buffer);
190 }
191
192 octeon_droq_reset_indices(droq);
193
194 octeon_droq_compute_max_packet_bufs(droq);
195
196 return 0;
197 }
198
octeon_delete_droq(struct octeon_device * oct,u32 q_no)199 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
200 {
201 struct octeon_droq *droq = oct->droq[q_no];
202
203 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
204
205 octeon_droq_destroy_ring_buffers(oct, droq);
206 vfree(droq->recv_buf_list);
207
208 if (droq->desc_ring)
209 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
210 droq->desc_ring, droq->desc_ring_dma);
211
212 memset(droq, 0, OCT_DROQ_SIZE);
213 oct->io_qmask.oq &= ~(1ULL << q_no);
214 vfree(oct->droq[q_no]);
215 oct->droq[q_no] = NULL;
216 oct->num_oqs--;
217
218 return 0;
219 }
220 EXPORT_SYMBOL_GPL(octeon_delete_droq);
221
octeon_init_droq(struct octeon_device * oct,u32 q_no,u32 num_descs,u32 desc_size,void * app_ctx)222 int octeon_init_droq(struct octeon_device *oct,
223 u32 q_no,
224 u32 num_descs,
225 u32 desc_size,
226 void *app_ctx)
227 {
228 struct octeon_droq *droq;
229 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
230 u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
231 int numa_node = dev_to_node(&oct->pci_dev->dev);
232
233 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
234
235 droq = oct->droq[q_no];
236 memset(droq, 0, OCT_DROQ_SIZE);
237
238 droq->oct_dev = oct;
239 droq->q_no = q_no;
240 if (app_ctx)
241 droq->app_ctx = app_ctx;
242 else
243 droq->app_ctx = (void *)(size_t)q_no;
244
245 c_num_descs = num_descs;
246 c_buf_size = desc_size;
247 if (OCTEON_CN6XXX(oct)) {
248 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);
249
250 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
251 c_refill_threshold =
252 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
253 } else if (OCTEON_CN23XX_PF(oct)) {
254 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf);
255
256 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
257 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
258 } else if (OCTEON_CN23XX_VF(oct)) {
259 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf);
260
261 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
262 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
263 } else {
264 return 1;
265 }
266
267 droq->max_count = c_num_descs;
268 droq->buffer_size = c_buf_size;
269
270 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
271 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
272 (dma_addr_t *)&droq->desc_ring_dma);
273
274 if (!droq->desc_ring) {
275 dev_err(&oct->pci_dev->dev,
276 "Output queue %d ring alloc failed\n", q_no);
277 return 1;
278 }
279
280 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
281 q_no, droq->desc_ring, droq->desc_ring_dma);
282 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
283 droq->max_count);
284
285 droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE),
286 numa_node);
287 if (!droq->recv_buf_list)
288 droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE));
289 if (!droq->recv_buf_list) {
290 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
291 goto init_droq_fail;
292 }
293
294 if (octeon_droq_setup_ring_buffers(oct, droq))
295 goto init_droq_fail;
296
297 droq->pkts_per_intr = c_pkts_per_intr;
298 droq->refill_threshold = c_refill_threshold;
299
300 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
301 droq->max_empty_descs);
302
303 INIT_LIST_HEAD(&droq->dispatch_list);
304
305 /* For 56xx Pass1, this function won't be called, so no checks. */
306 oct->fn_list.setup_oq_regs(oct, q_no);
307
308 oct->io_qmask.oq |= BIT_ULL(q_no);
309
310 return 0;
311
312 init_droq_fail:
313 octeon_delete_droq(oct, q_no);
314 return 1;
315 }
316
317 /* octeon_create_recv_info
318 * Parameters:
319 * octeon_dev - pointer to the octeon device structure
320 * droq - droq in which the packet arrived.
321 * buf_cnt - no. of buffers used by the packet.
322 * idx - index in the descriptor for the first buffer in the packet.
323 * Description:
324 * Allocates a recv_info_t and copies the buffer addresses for packet data
325 * into the recv_pkt space which starts at an 8B offset from recv_info_t.
326 * Flags the descriptors for refill later. If available descriptors go
327 * below the threshold to receive a 64K pkt, new buffers are first allocated
328 * before the recv_pkt_t is created.
329 * This routine will be called in interrupt context.
330 * Returns:
331 * Success: Pointer to recv_info_t
332 * Failure: NULL.
333 */
octeon_create_recv_info(struct octeon_device * octeon_dev,struct octeon_droq * droq,u32 buf_cnt,u32 idx)334 static inline struct octeon_recv_info *octeon_create_recv_info(
335 struct octeon_device *octeon_dev,
336 struct octeon_droq *droq,
337 u32 buf_cnt,
338 u32 idx)
339 {
340 struct octeon_droq_info *info;
341 struct octeon_recv_pkt *recv_pkt;
342 struct octeon_recv_info *recv_info;
343 u32 i, bytes_left;
344 struct octeon_skb_page_info *pg_info;
345
346 info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data;
347
348 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
349 if (!recv_info)
350 return NULL;
351
352 recv_pkt = recv_info->recv_pkt;
353 recv_pkt->rh = info->rh;
354 recv_pkt->length = (u32)info->length;
355 recv_pkt->buffer_count = (u16)buf_cnt;
356 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
357
358 i = 0;
359 bytes_left = (u32)info->length;
360
361 while (buf_cnt) {
362 {
363 pg_info = &droq->recv_buf_list[idx].pg_info;
364
365 lio_unmap_ring(octeon_dev->pci_dev,
366 (u64)pg_info->dma);
367 pg_info->page = NULL;
368 pg_info->dma = 0;
369 }
370
371 recv_pkt->buffer_size[i] =
372 (bytes_left >=
373 droq->buffer_size) ? droq->buffer_size : bytes_left;
374
375 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
376 droq->recv_buf_list[idx].buffer = NULL;
377
378 idx = incr_index(idx, 1, droq->max_count);
379 bytes_left -= droq->buffer_size;
380 i++;
381 buf_cnt--;
382 }
383
384 return recv_info;
385 }
386
387 /* If we were not able to refill all buffers, try to move around
388 * the buffers that were not dispatched.
389 */
390 static inline u32
octeon_droq_refill_pullup_descs(struct octeon_droq * droq,struct octeon_droq_desc * desc_ring)391 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
392 struct octeon_droq_desc *desc_ring)
393 {
394 u32 desc_refilled = 0;
395
396 u32 refill_index = droq->refill_idx;
397
398 while (refill_index != droq->read_idx) {
399 if (droq->recv_buf_list[refill_index].buffer) {
400 droq->recv_buf_list[droq->refill_idx].buffer =
401 droq->recv_buf_list[refill_index].buffer;
402 droq->recv_buf_list[droq->refill_idx].data =
403 droq->recv_buf_list[refill_index].data;
404 desc_ring[droq->refill_idx].buffer_ptr =
405 desc_ring[refill_index].buffer_ptr;
406 droq->recv_buf_list[refill_index].buffer = NULL;
407 desc_ring[refill_index].buffer_ptr = 0;
408 do {
409 droq->refill_idx = incr_index(droq->refill_idx,
410 1,
411 droq->max_count);
412 desc_refilled++;
413 droq->refill_count--;
414 } while (droq->recv_buf_list[droq->refill_idx].buffer);
415 }
416 refill_index = incr_index(refill_index, 1, droq->max_count);
417 } /* while */
418 return desc_refilled;
419 }
420
421 /* octeon_droq_refill
422 * Parameters:
423 * droq - droq in which descriptors require new buffers.
424 * Description:
425 * Called during normal DROQ processing in interrupt mode or by the poll
426 * thread to refill the descriptors from which buffers were dispatched
427 * to upper layers. Attempts to allocate new buffers. If that fails, moves
428 * up buffers (that were not dispatched) to form a contiguous ring.
429 * Returns:
430 * No of descriptors refilled.
431 */
432 static u32
octeon_droq_refill(struct octeon_device * octeon_dev,struct octeon_droq * droq)433 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
434 {
435 struct octeon_droq_desc *desc_ring;
436 void *buf = NULL;
437 u8 *data;
438 u32 desc_refilled = 0;
439 struct octeon_skb_page_info *pg_info;
440
441 desc_ring = droq->desc_ring;
442
443 while (droq->refill_count && (desc_refilled < droq->max_count)) {
444 /* If a valid buffer exists (happens if there is no dispatch),
445 * reuse the buffer, else allocate.
446 */
447 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
448 pg_info =
449 &droq->recv_buf_list[droq->refill_idx].pg_info;
450 /* Either recycle the existing pages or go for
451 * new page alloc
452 */
453 if (pg_info->page)
454 buf = recv_buffer_reuse(octeon_dev, pg_info);
455 else
456 buf = recv_buffer_alloc(octeon_dev, pg_info);
457 /* If a buffer could not be allocated, no point in
458 * continuing
459 */
460 if (!buf) {
461 droq->stats.rx_alloc_failure++;
462 break;
463 }
464 droq->recv_buf_list[droq->refill_idx].buffer =
465 buf;
466 data = get_rbd(buf);
467 } else {
468 data = get_rbd(droq->recv_buf_list
469 [droq->refill_idx].buffer);
470 }
471
472 droq->recv_buf_list[droq->refill_idx].data = data;
473
474 desc_ring[droq->refill_idx].buffer_ptr =
475 lio_map_ring(droq->recv_buf_list[
476 droq->refill_idx].buffer);
477
478 droq->refill_idx = incr_index(droq->refill_idx, 1,
479 droq->max_count);
480 desc_refilled++;
481 droq->refill_count--;
482 }
483
484 if (droq->refill_count)
485 desc_refilled +=
486 octeon_droq_refill_pullup_descs(droq, desc_ring);
487
488 /* if droq->refill_count
489 * The refill count would not change in pass two. We only moved buffers
490 * to close the gap in the ring, but we would still have the same no. of
491 * buffers to refill.
492 */
493 return desc_refilled;
494 }
495
496 /** check if we can allocate packets to get out of oom.
497 * @param droq - Droq being checked.
498 * @return 1 if fails to refill minimum
499 */
octeon_retry_droq_refill(struct octeon_droq * droq)500 int octeon_retry_droq_refill(struct octeon_droq *droq)
501 {
502 struct octeon_device *oct = droq->oct_dev;
503 int desc_refilled, reschedule = 1;
504 u32 pkts_credit;
505
506 pkts_credit = readl(droq->pkts_credit_reg);
507 desc_refilled = octeon_droq_refill(oct, droq);
508 if (desc_refilled) {
509 /* Flush the droq descriptor data to memory to be sure
510 * that when we update the credits the data in memory
511 * is accurate.
512 */
513 wmb();
514 writel(desc_refilled, droq->pkts_credit_reg);
515
516 if (pkts_credit + desc_refilled >= CN23XX_SLI_DEF_BP)
517 reschedule = 0;
518 }
519
520 return reschedule;
521 }
522
523 static inline u32
octeon_droq_get_bufcount(u32 buf_size,u32 total_len)524 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
525 {
526 return DIV_ROUND_UP(total_len, buf_size);
527 }
528
529 static int
octeon_droq_dispatch_pkt(struct octeon_device * oct,struct octeon_droq * droq,union octeon_rh * rh,struct octeon_droq_info * info)530 octeon_droq_dispatch_pkt(struct octeon_device *oct,
531 struct octeon_droq *droq,
532 union octeon_rh *rh,
533 struct octeon_droq_info *info)
534 {
535 u32 cnt;
536 octeon_dispatch_fn_t disp_fn;
537 struct octeon_recv_info *rinfo;
538
539 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
540
541 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
542 (u16)rh->r.subcode);
543 if (disp_fn) {
544 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
545 if (rinfo) {
546 struct __dispatch *rdisp = rinfo->rsvd;
547
548 rdisp->rinfo = rinfo;
549 rdisp->disp_fn = disp_fn;
550 rinfo->recv_pkt->rh = *rh;
551 list_add_tail(&rdisp->list,
552 &droq->dispatch_list);
553 } else {
554 droq->stats.dropped_nomem++;
555 }
556 } else {
557 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
558 (unsigned int)rh->r.opcode,
559 (unsigned int)rh->r.subcode);
560 droq->stats.dropped_nodispatch++;
561 }
562
563 return cnt;
564 }
565
octeon_droq_drop_packets(struct octeon_device * oct,struct octeon_droq * droq,u32 cnt)566 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
567 struct octeon_droq *droq,
568 u32 cnt)
569 {
570 u32 i = 0, buf_cnt;
571 struct octeon_droq_info *info;
572
573 for (i = 0; i < cnt; i++) {
574 info = (struct octeon_droq_info *)
575 droq->recv_buf_list[droq->read_idx].data;
576 octeon_swap_8B_data((u64 *)info, 2);
577
578 if (info->length) {
579 info->length += OCTNET_FRM_LENGTH_SIZE;
580 droq->stats.bytes_received += info->length;
581 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
582 (u32)info->length);
583 } else {
584 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
585 buf_cnt = 1;
586 }
587
588 droq->read_idx = incr_index(droq->read_idx, buf_cnt,
589 droq->max_count);
590 droq->refill_count += buf_cnt;
591 }
592 }
593
594 static u32
octeon_droq_fast_process_packets(struct octeon_device * oct,struct octeon_droq * droq,u32 pkts_to_process)595 octeon_droq_fast_process_packets(struct octeon_device *oct,
596 struct octeon_droq *droq,
597 u32 pkts_to_process)
598 {
599 u32 pkt, total_len = 0, pkt_count, retval;
600 struct octeon_droq_info *info;
601 union octeon_rh *rh;
602
603 pkt_count = pkts_to_process;
604
605 for (pkt = 0; pkt < pkt_count; pkt++) {
606 u32 pkt_len = 0;
607 struct sk_buff *nicbuf = NULL;
608 struct octeon_skb_page_info *pg_info;
609 void *buf;
610
611 info = (struct octeon_droq_info *)
612 droq->recv_buf_list[droq->read_idx].data;
613 octeon_swap_8B_data((u64 *)info, 2);
614
615 if (!info->length) {
616 dev_err(&oct->pci_dev->dev,
617 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
618 droq->q_no, droq->read_idx, pkt_count);
619 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
620 (u8 *)info,
621 OCT_DROQ_INFO_SIZE);
622 break;
623 }
624
625 /* Len of resp hdr in included in the received data len. */
626 rh = &info->rh;
627
628 info->length += OCTNET_FRM_LENGTH_SIZE;
629 rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64));
630 total_len += (u32)info->length;
631 if (opcode_slow_path(rh)) {
632 u32 buf_cnt;
633
634 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
635 droq->read_idx = incr_index(droq->read_idx,
636 buf_cnt, droq->max_count);
637 droq->refill_count += buf_cnt;
638 } else {
639 if (info->length <= droq->buffer_size) {
640 pkt_len = (u32)info->length;
641 nicbuf = droq->recv_buf_list[
642 droq->read_idx].buffer;
643 pg_info = &droq->recv_buf_list[
644 droq->read_idx].pg_info;
645 if (recv_buffer_recycle(oct, pg_info))
646 pg_info->page = NULL;
647 droq->recv_buf_list[droq->read_idx].buffer =
648 NULL;
649
650 droq->read_idx = incr_index(droq->read_idx, 1,
651 droq->max_count);
652 droq->refill_count++;
653 } else {
654 nicbuf = octeon_fast_packet_alloc((u32)
655 info->length);
656 pkt_len = 0;
657 /* nicbuf allocation can fail. We'll handle it
658 * inside the loop.
659 */
660 while (pkt_len < info->length) {
661 int cpy_len, idx = droq->read_idx;
662
663 cpy_len = ((pkt_len + droq->buffer_size)
664 > info->length) ?
665 ((u32)info->length - pkt_len) :
666 droq->buffer_size;
667
668 if (nicbuf) {
669 octeon_fast_packet_next(droq,
670 nicbuf,
671 cpy_len,
672 idx);
673 buf = droq->recv_buf_list[
674 idx].buffer;
675 recv_buffer_fast_free(buf);
676 droq->recv_buf_list[idx].buffer
677 = NULL;
678 } else {
679 droq->stats.rx_alloc_failure++;
680 }
681
682 pkt_len += cpy_len;
683 droq->read_idx =
684 incr_index(droq->read_idx, 1,
685 droq->max_count);
686 droq->refill_count++;
687 }
688 }
689
690 if (nicbuf) {
691 if (droq->ops.fptr) {
692 droq->ops.fptr(oct->octeon_id,
693 nicbuf, pkt_len,
694 rh, &droq->napi,
695 droq->ops.farg);
696 } else {
697 recv_buffer_free(nicbuf);
698 }
699 }
700 }
701
702 if (droq->refill_count >= droq->refill_threshold) {
703 int desc_refilled = octeon_droq_refill(oct, droq);
704
705 if (desc_refilled) {
706 /* Flush the droq descriptor data to memory to
707 * be sure that when we update the credits the
708 * data in memory is accurate.
709 */
710 wmb();
711 writel(desc_refilled, droq->pkts_credit_reg);
712 }
713 }
714 } /* for (each packet)... */
715
716 /* Increment refill_count by the number of buffers processed. */
717 droq->stats.pkts_received += pkt;
718 droq->stats.bytes_received += total_len;
719
720 retval = pkt;
721 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
722 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
723
724 droq->stats.dropped_toomany += (pkts_to_process - pkt);
725 retval = pkts_to_process;
726 }
727
728 atomic_sub(retval, &droq->pkts_pending);
729
730 if (droq->refill_count >= droq->refill_threshold &&
731 readl(droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) {
732 octeon_droq_check_hw_for_pkts(droq);
733
734 /* Make sure there are no pkts_pending */
735 if (!atomic_read(&droq->pkts_pending))
736 octeon_schedule_rxq_oom_work(oct, droq);
737 }
738
739 return retval;
740 }
741
742 int
octeon_droq_process_packets(struct octeon_device * oct,struct octeon_droq * droq,u32 budget)743 octeon_droq_process_packets(struct octeon_device *oct,
744 struct octeon_droq *droq,
745 u32 budget)
746 {
747 u32 pkt_count = 0;
748 struct list_head *tmp, *tmp2;
749
750 octeon_droq_check_hw_for_pkts(droq);
751 pkt_count = atomic_read(&droq->pkts_pending);
752
753 if (!pkt_count)
754 return 0;
755
756 if (pkt_count > budget)
757 pkt_count = budget;
758
759 octeon_droq_fast_process_packets(oct, droq, pkt_count);
760
761 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
762 struct __dispatch *rdisp = (struct __dispatch *)tmp;
763
764 list_del(tmp);
765 rdisp->disp_fn(rdisp->rinfo,
766 octeon_get_dispatch_arg
767 (oct,
768 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
769 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
770 }
771
772 /* If there are packets pending. schedule tasklet again */
773 if (atomic_read(&droq->pkts_pending))
774 return 1;
775
776 return 0;
777 }
778 EXPORT_SYMBOL_GPL(octeon_droq_process_packets);
779
780 /*
781 * Utility function to poll for packets. check_hw_for_packets must be
782 * called before calling this routine.
783 */
784
785 int
octeon_droq_process_poll_pkts(struct octeon_device * oct,struct octeon_droq * droq,u32 budget)786 octeon_droq_process_poll_pkts(struct octeon_device *oct,
787 struct octeon_droq *droq, u32 budget)
788 {
789 struct list_head *tmp, *tmp2;
790 u32 pkts_available = 0, pkts_processed = 0;
791 u32 total_pkts_processed = 0;
792
793 if (budget > droq->max_count)
794 budget = droq->max_count;
795
796 while (total_pkts_processed < budget) {
797 octeon_droq_check_hw_for_pkts(droq);
798
799 pkts_available = min((budget - total_pkts_processed),
800 (u32)(atomic_read(&droq->pkts_pending)));
801
802 if (pkts_available == 0)
803 break;
804
805 pkts_processed =
806 octeon_droq_fast_process_packets(oct, droq,
807 pkts_available);
808
809 total_pkts_processed += pkts_processed;
810 }
811
812 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
813 struct __dispatch *rdisp = (struct __dispatch *)tmp;
814
815 list_del(tmp);
816 rdisp->disp_fn(rdisp->rinfo,
817 octeon_get_dispatch_arg
818 (oct,
819 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
820 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
821 }
822
823 return total_pkts_processed;
824 }
825
826 /* Enable Pkt Interrupt */
827 int
octeon_enable_irq(struct octeon_device * oct,u32 q_no)828 octeon_enable_irq(struct octeon_device *oct, u32 q_no)
829 {
830 switch (oct->chip_id) {
831 case OCTEON_CN66XX:
832 case OCTEON_CN68XX: {
833 struct octeon_cn6xxx *cn6xxx =
834 (struct octeon_cn6xxx *)oct->chip;
835 unsigned long flags;
836 u32 value;
837
838 spin_lock_irqsave
839 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
840 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);
841 value |= (1 << q_no);
842 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value);
843 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
844 value |= (1 << q_no);
845 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value);
846
847 /* don't bother flushing the enables */
848
849 spin_unlock_irqrestore
850 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
851 }
852 break;
853 case OCTEON_CN23XX_PF_VID:
854 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
855 break;
856
857 case OCTEON_CN23XX_VF_VID:
858 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
859 break;
860 default:
861 dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__);
862 return 1;
863 }
864
865 return 0;
866 }
867
octeon_register_droq_ops(struct octeon_device * oct,u32 q_no,struct octeon_droq_ops * ops)868 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
869 struct octeon_droq_ops *ops)
870 {
871 struct octeon_config *oct_cfg = NULL;
872 struct octeon_droq *droq;
873
874 oct_cfg = octeon_get_conf(oct);
875
876 if (!oct_cfg)
877 return -EINVAL;
878
879 if (!(ops)) {
880 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
881 __func__);
882 return -EINVAL;
883 }
884
885 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
886 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
887 __func__, q_no, (oct->num_oqs - 1));
888 return -EINVAL;
889 }
890
891 droq = oct->droq[q_no];
892 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
893
894 return 0;
895 }
896
octeon_unregister_droq_ops(struct octeon_device * oct,u32 q_no)897 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
898 {
899 struct octeon_config *oct_cfg = NULL;
900 struct octeon_droq *droq;
901
902 oct_cfg = octeon_get_conf(oct);
903
904 if (!oct_cfg)
905 return -EINVAL;
906
907 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
908 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
909 __func__, q_no, oct->num_oqs - 1);
910 return -EINVAL;
911 }
912
913 droq = oct->droq[q_no];
914
915 if (!droq) {
916 dev_info(&oct->pci_dev->dev,
917 "Droq id (%d) not available.\n", q_no);
918 return 0;
919 }
920
921 droq->ops.fptr = NULL;
922 droq->ops.farg = NULL;
923 droq->ops.drop_on_max = 0;
924
925 return 0;
926 }
927 EXPORT_SYMBOL_GPL(octeon_unregister_droq_ops);
928
octeon_create_droq(struct octeon_device * oct,u32 q_no,u32 num_descs,u32 desc_size,void * app_ctx)929 int octeon_create_droq(struct octeon_device *oct,
930 u32 q_no, u32 num_descs,
931 u32 desc_size, void *app_ctx)
932 {
933 struct octeon_droq *droq;
934 int numa_node = dev_to_node(&oct->pci_dev->dev);
935
936 if (oct->droq[q_no]) {
937 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
938 q_no);
939 return 1;
940 }
941
942 /* Allocate the DS for the new droq. */
943 droq = vmalloc_node(sizeof(*droq), numa_node);
944 if (!droq)
945 droq = vmalloc(sizeof(*droq));
946 if (!droq)
947 return -1;
948
949 memset(droq, 0, sizeof(struct octeon_droq));
950
951 /*Disable the pkt o/p for this Q */
952 octeon_set_droq_pkt_op(oct, q_no, 0);
953 oct->droq[q_no] = droq;
954
955 /* Initialize the Droq */
956 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {
957 vfree(oct->droq[q_no]);
958 oct->droq[q_no] = NULL;
959 return -1;
960 }
961
962 oct->num_oqs++;
963
964 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
965 oct->num_oqs);
966
967 /* Global Droq register settings */
968
969 /* As of now not required, as setting are done for all 32 Droqs at
970 * the same time.
971 */
972 return 0;
973 }
974