1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more details. 17 ***********************************************************************/ 18 #include <linux/pci.h> 19 #include <linux/netdevice.h> 20 #include <linux/vmalloc.h> 21 #include "liquidio_common.h" 22 #include "octeon_droq.h" 23 #include "octeon_iq.h" 24 #include "response_manager.h" 25 #include "octeon_device.h" 26 #include "octeon_main.h" 27 #include "octeon_network.h" 28 #include "cn66xx_regs.h" 29 #include "cn66xx_device.h" 30 #include "cn23xx_pf_device.h" 31 #include "cn23xx_vf_device.h" 32 33 struct niclist { 34 struct list_head list; 35 void *ptr; 36 }; 37 38 struct __dispatch { 39 struct list_head list; 40 struct octeon_recv_info *rinfo; 41 octeon_dispatch_fn_t disp_fn; 42 }; 43 44 /** Get the argument that the user set when registering dispatch 45 * function for a given opcode/subcode. 46 * @param octeon_dev - the octeon device pointer. 47 * @param opcode - the opcode for which the dispatch argument 48 * is to be checked. 49 * @param subcode - the subcode for which the dispatch argument 50 * is to be checked. 51 * @return Success: void * (argument to the dispatch function) 52 * @return Failure: NULL 53 * 54 */ 55 void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 56 u16 opcode, u16 subcode) 57 { 58 int idx; 59 struct list_head *dispatch; 60 void *fn_arg = NULL; 61 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 62 63 idx = combined_opcode & OCTEON_OPCODE_MASK; 64 65 spin_lock_bh(&octeon_dev->dispatch.lock); 66 67 if (octeon_dev->dispatch.count == 0) { 68 spin_unlock_bh(&octeon_dev->dispatch.lock); 69 return NULL; 70 } 71 72 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 73 fn_arg = octeon_dev->dispatch.dlist[idx].arg; 74 } else { 75 list_for_each(dispatch, 76 &octeon_dev->dispatch.dlist[idx].list) { 77 if (((struct octeon_dispatch *)dispatch)->opcode == 78 combined_opcode) { 79 fn_arg = ((struct octeon_dispatch *) 80 dispatch)->arg; 81 break; 82 } 83 } 84 } 85 86 spin_unlock_bh(&octeon_dev->dispatch.lock); 87 return fn_arg; 88 } 89 90 /** Check for packets on Droq. This function should be called with lock held. 91 * @param droq - Droq on which count is checked. 92 * @return Returns packet count. 93 */ 94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) 95 { 96 u32 pkt_count = 0; 97 u32 last_count; 98 99 pkt_count = readl(droq->pkts_sent_reg); 100 101 last_count = pkt_count - droq->pkt_count; 102 droq->pkt_count = pkt_count; 103 104 /* we shall write to cnts at napi irq enable or end of droq tasklet */ 105 if (last_count) 106 atomic_add(last_count, &droq->pkts_pending); 107 108 return last_count; 109 } 110 EXPORT_SYMBOL_GPL(octeon_droq_check_hw_for_pkts); 111 112 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 113 { 114 u32 count = 0; 115 116 /* max_empty_descs is the max. no. of descs that can have no buffers. 117 * If the empty desc count goes beyond this value, we cannot safely 118 * read in a 64K packet sent by Octeon 119 * (64K is max pkt size from Octeon) 120 */ 121 droq->max_empty_descs = 0; 122 123 do { 124 droq->max_empty_descs++; 125 count += droq->buffer_size; 126 } while (count < (64 * 1024)); 127 128 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 129 } 130 131 static void octeon_droq_reset_indices(struct octeon_droq *droq) 132 { 133 droq->read_idx = 0; 134 droq->write_idx = 0; 135 droq->refill_idx = 0; 136 droq->refill_count = 0; 137 atomic_set(&droq->pkts_pending, 0); 138 } 139 140 static void 141 octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 142 struct octeon_droq *droq) 143 { 144 u32 i; 145 struct octeon_skb_page_info *pg_info; 146 147 for (i = 0; i < droq->max_count; i++) { 148 pg_info = &droq->recv_buf_list[i].pg_info; 149 if (!pg_info) 150 continue; 151 152 if (pg_info->dma) 153 lio_unmap_ring(oct->pci_dev, 154 (u64)pg_info->dma); 155 pg_info->dma = 0; 156 157 if (pg_info->page) 158 recv_buffer_destroy(droq->recv_buf_list[i].buffer, 159 pg_info); 160 161 droq->recv_buf_list[i].buffer = NULL; 162 } 163 164 octeon_droq_reset_indices(droq); 165 } 166 167 static int 168 octeon_droq_setup_ring_buffers(struct octeon_device *oct, 169 struct octeon_droq *droq) 170 { 171 u32 i; 172 void *buf; 173 struct octeon_droq_desc *desc_ring = droq->desc_ring; 174 175 for (i = 0; i < droq->max_count; i++) { 176 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); 177 178 if (!buf) { 179 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 180 __func__); 181 droq->stats.rx_alloc_failure++; 182 return -ENOMEM; 183 } 184 185 droq->recv_buf_list[i].buffer = buf; 186 droq->recv_buf_list[i].data = get_rbd(buf); 187 desc_ring[i].info_ptr = 0; 188 desc_ring[i].buffer_ptr = 189 lio_map_ring(droq->recv_buf_list[i].buffer); 190 } 191 192 octeon_droq_reset_indices(droq); 193 194 octeon_droq_compute_max_packet_bufs(droq); 195 196 return 0; 197 } 198 199 int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 200 { 201 struct octeon_droq *droq = oct->droq[q_no]; 202 203 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 204 205 octeon_droq_destroy_ring_buffers(oct, droq); 206 vfree(droq->recv_buf_list); 207 208 if (droq->desc_ring) 209 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 210 droq->desc_ring, droq->desc_ring_dma); 211 212 memset(droq, 0, OCT_DROQ_SIZE); 213 oct->io_qmask.oq &= ~(1ULL << q_no); 214 vfree(oct->droq[q_no]); 215 oct->droq[q_no] = NULL; 216 oct->num_oqs--; 217 218 return 0; 219 } 220 EXPORT_SYMBOL_GPL(octeon_delete_droq); 221 222 int octeon_init_droq(struct octeon_device *oct, 223 u32 q_no, 224 u32 num_descs, 225 u32 desc_size, 226 void *app_ctx) 227 { 228 struct octeon_droq *droq; 229 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 230 u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 231 int numa_node = dev_to_node(&oct->pci_dev->dev); 232 233 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 234 235 droq = oct->droq[q_no]; 236 memset(droq, 0, OCT_DROQ_SIZE); 237 238 droq->oct_dev = oct; 239 droq->q_no = q_no; 240 if (app_ctx) 241 droq->app_ctx = app_ctx; 242 else 243 droq->app_ctx = (void *)(size_t)q_no; 244 245 c_num_descs = num_descs; 246 c_buf_size = desc_size; 247 if (OCTEON_CN6XXX(oct)) { 248 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); 249 250 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 251 c_refill_threshold = 252 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 253 } else if (OCTEON_CN23XX_PF(oct)) { 254 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); 255 256 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 257 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 258 } else if (OCTEON_CN23XX_VF(oct)) { 259 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); 260 261 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 262 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 263 } else { 264 return 1; 265 } 266 267 droq->max_count = c_num_descs; 268 droq->buffer_size = c_buf_size; 269 270 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 271 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 272 (dma_addr_t *)&droq->desc_ring_dma); 273 274 if (!droq->desc_ring) { 275 dev_err(&oct->pci_dev->dev, 276 "Output queue %d ring alloc failed\n", q_no); 277 return 1; 278 } 279 280 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 281 q_no, droq->desc_ring, droq->desc_ring_dma); 282 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 283 droq->max_count); 284 285 droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE), 286 numa_node); 287 if (!droq->recv_buf_list) 288 droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE)); 289 if (!droq->recv_buf_list) { 290 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 291 goto init_droq_fail; 292 } 293 294 if (octeon_droq_setup_ring_buffers(oct, droq)) 295 goto init_droq_fail; 296 297 droq->pkts_per_intr = c_pkts_per_intr; 298 droq->refill_threshold = c_refill_threshold; 299 300 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 301 droq->max_empty_descs); 302 303 INIT_LIST_HEAD(&droq->dispatch_list); 304 305 /* For 56xx Pass1, this function won't be called, so no checks. */ 306 oct->fn_list.setup_oq_regs(oct, q_no); 307 308 oct->io_qmask.oq |= BIT_ULL(q_no); 309 310 return 0; 311 312 init_droq_fail: 313 octeon_delete_droq(oct, q_no); 314 return 1; 315 } 316 317 /* octeon_create_recv_info 318 * Parameters: 319 * octeon_dev - pointer to the octeon device structure 320 * droq - droq in which the packet arrived. 321 * buf_cnt - no. of buffers used by the packet. 322 * idx - index in the descriptor for the first buffer in the packet. 323 * Description: 324 * Allocates a recv_info_t and copies the buffer addresses for packet data 325 * into the recv_pkt space which starts at an 8B offset from recv_info_t. 326 * Flags the descriptors for refill later. If available descriptors go 327 * below the threshold to receive a 64K pkt, new buffers are first allocated 328 * before the recv_pkt_t is created. 329 * This routine will be called in interrupt context. 330 * Returns: 331 * Success: Pointer to recv_info_t 332 * Failure: NULL. 333 */ 334 static inline struct octeon_recv_info *octeon_create_recv_info( 335 struct octeon_device *octeon_dev, 336 struct octeon_droq *droq, 337 u32 buf_cnt, 338 u32 idx) 339 { 340 struct octeon_droq_info *info; 341 struct octeon_recv_pkt *recv_pkt; 342 struct octeon_recv_info *recv_info; 343 u32 i, bytes_left; 344 struct octeon_skb_page_info *pg_info; 345 346 info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data; 347 348 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 349 if (!recv_info) 350 return NULL; 351 352 recv_pkt = recv_info->recv_pkt; 353 recv_pkt->rh = info->rh; 354 recv_pkt->length = (u32)info->length; 355 recv_pkt->buffer_count = (u16)buf_cnt; 356 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 357 358 i = 0; 359 bytes_left = (u32)info->length; 360 361 while (buf_cnt) { 362 { 363 pg_info = &droq->recv_buf_list[idx].pg_info; 364 365 lio_unmap_ring(octeon_dev->pci_dev, 366 (u64)pg_info->dma); 367 pg_info->page = NULL; 368 pg_info->dma = 0; 369 } 370 371 recv_pkt->buffer_size[i] = 372 (bytes_left >= 373 droq->buffer_size) ? droq->buffer_size : bytes_left; 374 375 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 376 droq->recv_buf_list[idx].buffer = NULL; 377 378 idx = incr_index(idx, 1, droq->max_count); 379 bytes_left -= droq->buffer_size; 380 i++; 381 buf_cnt--; 382 } 383 384 return recv_info; 385 } 386 387 /* If we were not able to refill all buffers, try to move around 388 * the buffers that were not dispatched. 389 */ 390 static inline u32 391 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 392 struct octeon_droq_desc *desc_ring) 393 { 394 u32 desc_refilled = 0; 395 396 u32 refill_index = droq->refill_idx; 397 398 while (refill_index != droq->read_idx) { 399 if (droq->recv_buf_list[refill_index].buffer) { 400 droq->recv_buf_list[droq->refill_idx].buffer = 401 droq->recv_buf_list[refill_index].buffer; 402 droq->recv_buf_list[droq->refill_idx].data = 403 droq->recv_buf_list[refill_index].data; 404 desc_ring[droq->refill_idx].buffer_ptr = 405 desc_ring[refill_index].buffer_ptr; 406 droq->recv_buf_list[refill_index].buffer = NULL; 407 desc_ring[refill_index].buffer_ptr = 0; 408 do { 409 droq->refill_idx = incr_index(droq->refill_idx, 410 1, 411 droq->max_count); 412 desc_refilled++; 413 droq->refill_count--; 414 } while (droq->recv_buf_list[droq->refill_idx].buffer); 415 } 416 refill_index = incr_index(refill_index, 1, droq->max_count); 417 } /* while */ 418 return desc_refilled; 419 } 420 421 /* octeon_droq_refill 422 * Parameters: 423 * droq - droq in which descriptors require new buffers. 424 * Description: 425 * Called during normal DROQ processing in interrupt mode or by the poll 426 * thread to refill the descriptors from which buffers were dispatched 427 * to upper layers. Attempts to allocate new buffers. If that fails, moves 428 * up buffers (that were not dispatched) to form a contiguous ring. 429 * Returns: 430 * No of descriptors refilled. 431 */ 432 static u32 433 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 434 { 435 struct octeon_droq_desc *desc_ring; 436 void *buf = NULL; 437 u8 *data; 438 u32 desc_refilled = 0; 439 struct octeon_skb_page_info *pg_info; 440 441 desc_ring = droq->desc_ring; 442 443 while (droq->refill_count && (desc_refilled < droq->max_count)) { 444 /* If a valid buffer exists (happens if there is no dispatch), 445 * reuse the buffer, else allocate. 446 */ 447 if (!droq->recv_buf_list[droq->refill_idx].buffer) { 448 pg_info = 449 &droq->recv_buf_list[droq->refill_idx].pg_info; 450 /* Either recycle the existing pages or go for 451 * new page alloc 452 */ 453 if (pg_info->page) 454 buf = recv_buffer_reuse(octeon_dev, pg_info); 455 else 456 buf = recv_buffer_alloc(octeon_dev, pg_info); 457 /* If a buffer could not be allocated, no point in 458 * continuing 459 */ 460 if (!buf) { 461 droq->stats.rx_alloc_failure++; 462 break; 463 } 464 droq->recv_buf_list[droq->refill_idx].buffer = 465 buf; 466 data = get_rbd(buf); 467 } else { 468 data = get_rbd(droq->recv_buf_list 469 [droq->refill_idx].buffer); 470 } 471 472 droq->recv_buf_list[droq->refill_idx].data = data; 473 474 desc_ring[droq->refill_idx].buffer_ptr = 475 lio_map_ring(droq->recv_buf_list[ 476 droq->refill_idx].buffer); 477 478 droq->refill_idx = incr_index(droq->refill_idx, 1, 479 droq->max_count); 480 desc_refilled++; 481 droq->refill_count--; 482 } 483 484 if (droq->refill_count) 485 desc_refilled += 486 octeon_droq_refill_pullup_descs(droq, desc_ring); 487 488 /* if droq->refill_count 489 * The refill count would not change in pass two. We only moved buffers 490 * to close the gap in the ring, but we would still have the same no. of 491 * buffers to refill. 492 */ 493 return desc_refilled; 494 } 495 496 /** check if we can allocate packets to get out of oom. 497 * @param droq - Droq being checked. 498 * @return 1 if fails to refill minimum 499 */ 500 int octeon_retry_droq_refill(struct octeon_droq *droq) 501 { 502 struct octeon_device *oct = droq->oct_dev; 503 int desc_refilled, reschedule = 1; 504 u32 pkts_credit; 505 506 pkts_credit = readl(droq->pkts_credit_reg); 507 desc_refilled = octeon_droq_refill(oct, droq); 508 if (desc_refilled) { 509 /* Flush the droq descriptor data to memory to be sure 510 * that when we update the credits the data in memory 511 * is accurate. 512 */ 513 wmb(); 514 writel(desc_refilled, droq->pkts_credit_reg); 515 516 if (pkts_credit + desc_refilled >= CN23XX_SLI_DEF_BP) 517 reschedule = 0; 518 } 519 520 return reschedule; 521 } 522 523 static inline u32 524 octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 525 { 526 return DIV_ROUND_UP(total_len, buf_size); 527 } 528 529 static int 530 octeon_droq_dispatch_pkt(struct octeon_device *oct, 531 struct octeon_droq *droq, 532 union octeon_rh *rh, 533 struct octeon_droq_info *info) 534 { 535 u32 cnt; 536 octeon_dispatch_fn_t disp_fn; 537 struct octeon_recv_info *rinfo; 538 539 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 540 541 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 542 (u16)rh->r.subcode); 543 if (disp_fn) { 544 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 545 if (rinfo) { 546 struct __dispatch *rdisp = rinfo->rsvd; 547 548 rdisp->rinfo = rinfo; 549 rdisp->disp_fn = disp_fn; 550 rinfo->recv_pkt->rh = *rh; 551 list_add_tail(&rdisp->list, 552 &droq->dispatch_list); 553 } else { 554 droq->stats.dropped_nomem++; 555 } 556 } else { 557 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", 558 (unsigned int)rh->r.opcode, 559 (unsigned int)rh->r.subcode); 560 droq->stats.dropped_nodispatch++; 561 } 562 563 return cnt; 564 } 565 566 static inline void octeon_droq_drop_packets(struct octeon_device *oct, 567 struct octeon_droq *droq, 568 u32 cnt) 569 { 570 u32 i = 0, buf_cnt; 571 struct octeon_droq_info *info; 572 573 for (i = 0; i < cnt; i++) { 574 info = (struct octeon_droq_info *) 575 droq->recv_buf_list[droq->read_idx].data; 576 octeon_swap_8B_data((u64 *)info, 2); 577 578 if (info->length) { 579 info->length += OCTNET_FRM_LENGTH_SIZE; 580 droq->stats.bytes_received += info->length; 581 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 582 (u32)info->length); 583 } else { 584 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 585 buf_cnt = 1; 586 } 587 588 droq->read_idx = incr_index(droq->read_idx, buf_cnt, 589 droq->max_count); 590 droq->refill_count += buf_cnt; 591 } 592 } 593 594 static u32 595 octeon_droq_fast_process_packets(struct octeon_device *oct, 596 struct octeon_droq *droq, 597 u32 pkts_to_process) 598 { 599 u32 pkt, total_len = 0, pkt_count, retval; 600 struct octeon_droq_info *info; 601 union octeon_rh *rh; 602 603 pkt_count = pkts_to_process; 604 605 for (pkt = 0; pkt < pkt_count; pkt++) { 606 u32 pkt_len = 0; 607 struct sk_buff *nicbuf = NULL; 608 struct octeon_skb_page_info *pg_info; 609 void *buf; 610 611 info = (struct octeon_droq_info *) 612 droq->recv_buf_list[droq->read_idx].data; 613 octeon_swap_8B_data((u64 *)info, 2); 614 615 if (!info->length) { 616 dev_err(&oct->pci_dev->dev, 617 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 618 droq->q_no, droq->read_idx, pkt_count); 619 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 620 (u8 *)info, 621 OCT_DROQ_INFO_SIZE); 622 break; 623 } 624 625 /* Len of resp hdr in included in the received data len. */ 626 rh = &info->rh; 627 628 info->length += OCTNET_FRM_LENGTH_SIZE; 629 rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64)); 630 total_len += (u32)info->length; 631 if (opcode_slow_path(rh)) { 632 u32 buf_cnt; 633 634 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 635 droq->read_idx = incr_index(droq->read_idx, 636 buf_cnt, droq->max_count); 637 droq->refill_count += buf_cnt; 638 } else { 639 if (info->length <= droq->buffer_size) { 640 pkt_len = (u32)info->length; 641 nicbuf = droq->recv_buf_list[ 642 droq->read_idx].buffer; 643 pg_info = &droq->recv_buf_list[ 644 droq->read_idx].pg_info; 645 if (recv_buffer_recycle(oct, pg_info)) 646 pg_info->page = NULL; 647 droq->recv_buf_list[droq->read_idx].buffer = 648 NULL; 649 650 droq->read_idx = incr_index(droq->read_idx, 1, 651 droq->max_count); 652 droq->refill_count++; 653 } else { 654 nicbuf = octeon_fast_packet_alloc((u32) 655 info->length); 656 pkt_len = 0; 657 /* nicbuf allocation can fail. We'll handle it 658 * inside the loop. 659 */ 660 while (pkt_len < info->length) { 661 int cpy_len, idx = droq->read_idx; 662 663 cpy_len = ((pkt_len + droq->buffer_size) 664 > info->length) ? 665 ((u32)info->length - pkt_len) : 666 droq->buffer_size; 667 668 if (nicbuf) { 669 octeon_fast_packet_next(droq, 670 nicbuf, 671 cpy_len, 672 idx); 673 buf = droq->recv_buf_list[ 674 idx].buffer; 675 recv_buffer_fast_free(buf); 676 droq->recv_buf_list[idx].buffer 677 = NULL; 678 } else { 679 droq->stats.rx_alloc_failure++; 680 } 681 682 pkt_len += cpy_len; 683 droq->read_idx = 684 incr_index(droq->read_idx, 1, 685 droq->max_count); 686 droq->refill_count++; 687 } 688 } 689 690 if (nicbuf) { 691 if (droq->ops.fptr) { 692 droq->ops.fptr(oct->octeon_id, 693 nicbuf, pkt_len, 694 rh, &droq->napi, 695 droq->ops.farg); 696 } else { 697 recv_buffer_free(nicbuf); 698 } 699 } 700 } 701 702 if (droq->refill_count >= droq->refill_threshold) { 703 int desc_refilled = octeon_droq_refill(oct, droq); 704 705 if (desc_refilled) { 706 /* Flush the droq descriptor data to memory to 707 * be sure that when we update the credits the 708 * data in memory is accurate. 709 */ 710 wmb(); 711 writel(desc_refilled, droq->pkts_credit_reg); 712 } 713 } 714 } /* for (each packet)... */ 715 716 /* Increment refill_count by the number of buffers processed. */ 717 droq->stats.pkts_received += pkt; 718 droq->stats.bytes_received += total_len; 719 720 retval = pkt; 721 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 722 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 723 724 droq->stats.dropped_toomany += (pkts_to_process - pkt); 725 retval = pkts_to_process; 726 } 727 728 atomic_sub(retval, &droq->pkts_pending); 729 730 if (droq->refill_count >= droq->refill_threshold && 731 readl(droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) { 732 octeon_droq_check_hw_for_pkts(droq); 733 734 /* Make sure there are no pkts_pending */ 735 if (!atomic_read(&droq->pkts_pending)) 736 octeon_schedule_rxq_oom_work(oct, droq); 737 } 738 739 return retval; 740 } 741 742 int 743 octeon_droq_process_packets(struct octeon_device *oct, 744 struct octeon_droq *droq, 745 u32 budget) 746 { 747 u32 pkt_count = 0; 748 struct list_head *tmp, *tmp2; 749 750 octeon_droq_check_hw_for_pkts(droq); 751 pkt_count = atomic_read(&droq->pkts_pending); 752 753 if (!pkt_count) 754 return 0; 755 756 if (pkt_count > budget) 757 pkt_count = budget; 758 759 octeon_droq_fast_process_packets(oct, droq, pkt_count); 760 761 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 762 struct __dispatch *rdisp = (struct __dispatch *)tmp; 763 764 list_del(tmp); 765 rdisp->disp_fn(rdisp->rinfo, 766 octeon_get_dispatch_arg 767 (oct, 768 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 769 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 770 } 771 772 /* If there are packets pending. schedule tasklet again */ 773 if (atomic_read(&droq->pkts_pending)) 774 return 1; 775 776 return 0; 777 } 778 EXPORT_SYMBOL_GPL(octeon_droq_process_packets); 779 780 /* 781 * Utility function to poll for packets. check_hw_for_packets must be 782 * called before calling this routine. 783 */ 784 785 int 786 octeon_droq_process_poll_pkts(struct octeon_device *oct, 787 struct octeon_droq *droq, u32 budget) 788 { 789 struct list_head *tmp, *tmp2; 790 u32 pkts_available = 0, pkts_processed = 0; 791 u32 total_pkts_processed = 0; 792 793 if (budget > droq->max_count) 794 budget = droq->max_count; 795 796 while (total_pkts_processed < budget) { 797 octeon_droq_check_hw_for_pkts(droq); 798 799 pkts_available = min((budget - total_pkts_processed), 800 (u32)(atomic_read(&droq->pkts_pending))); 801 802 if (pkts_available == 0) 803 break; 804 805 pkts_processed = 806 octeon_droq_fast_process_packets(oct, droq, 807 pkts_available); 808 809 total_pkts_processed += pkts_processed; 810 } 811 812 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 813 struct __dispatch *rdisp = (struct __dispatch *)tmp; 814 815 list_del(tmp); 816 rdisp->disp_fn(rdisp->rinfo, 817 octeon_get_dispatch_arg 818 (oct, 819 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 820 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 821 } 822 823 return total_pkts_processed; 824 } 825 826 /* Enable Pkt Interrupt */ 827 int 828 octeon_enable_irq(struct octeon_device *oct, u32 q_no) 829 { 830 switch (oct->chip_id) { 831 case OCTEON_CN66XX: 832 case OCTEON_CN68XX: { 833 struct octeon_cn6xxx *cn6xxx = 834 (struct octeon_cn6xxx *)oct->chip; 835 unsigned long flags; 836 u32 value; 837 838 spin_lock_irqsave 839 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 840 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB); 841 value |= (1 << q_no); 842 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value); 843 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB); 844 value |= (1 << q_no); 845 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value); 846 847 /* don't bother flushing the enables */ 848 849 spin_unlock_irqrestore 850 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 851 } 852 break; 853 case OCTEON_CN23XX_PF_VID: 854 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 855 break; 856 857 case OCTEON_CN23XX_VF_VID: 858 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 859 break; 860 default: 861 dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__); 862 return 1; 863 } 864 865 return 0; 866 } 867 868 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 869 struct octeon_droq_ops *ops) 870 { 871 struct octeon_config *oct_cfg = NULL; 872 struct octeon_droq *droq; 873 874 oct_cfg = octeon_get_conf(oct); 875 876 if (!oct_cfg) 877 return -EINVAL; 878 879 if (!(ops)) { 880 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 881 __func__); 882 return -EINVAL; 883 } 884 885 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 886 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 887 __func__, q_no, (oct->num_oqs - 1)); 888 return -EINVAL; 889 } 890 891 droq = oct->droq[q_no]; 892 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 893 894 return 0; 895 } 896 897 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 898 { 899 struct octeon_config *oct_cfg = NULL; 900 struct octeon_droq *droq; 901 902 oct_cfg = octeon_get_conf(oct); 903 904 if (!oct_cfg) 905 return -EINVAL; 906 907 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 908 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 909 __func__, q_no, oct->num_oqs - 1); 910 return -EINVAL; 911 } 912 913 droq = oct->droq[q_no]; 914 915 if (!droq) { 916 dev_info(&oct->pci_dev->dev, 917 "Droq id (%d) not available.\n", q_no); 918 return 0; 919 } 920 921 droq->ops.fptr = NULL; 922 droq->ops.farg = NULL; 923 droq->ops.drop_on_max = 0; 924 925 return 0; 926 } 927 EXPORT_SYMBOL_GPL(octeon_unregister_droq_ops); 928 929 int octeon_create_droq(struct octeon_device *oct, 930 u32 q_no, u32 num_descs, 931 u32 desc_size, void *app_ctx) 932 { 933 struct octeon_droq *droq; 934 int numa_node = dev_to_node(&oct->pci_dev->dev); 935 936 if (oct->droq[q_no]) { 937 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 938 q_no); 939 return 1; 940 } 941 942 /* Allocate the DS for the new droq. */ 943 droq = vmalloc_node(sizeof(*droq), numa_node); 944 if (!droq) 945 droq = vmalloc(sizeof(*droq)); 946 if (!droq) 947 return -1; 948 949 memset(droq, 0, sizeof(struct octeon_droq)); 950 951 /*Disable the pkt o/p for this Q */ 952 octeon_set_droq_pkt_op(oct, q_no, 0); 953 oct->droq[q_no] = droq; 954 955 /* Initialize the Droq */ 956 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { 957 vfree(oct->droq[q_no]); 958 oct->droq[q_no] = NULL; 959 return -1; 960 } 961 962 oct->num_oqs++; 963 964 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 965 oct->num_oqs); 966 967 /* Global Droq register settings */ 968 969 /* As of now not required, as setting are done for all 32 Droqs at 970 * the same time. 971 */ 972 return 0; 973 } 974