1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more details. 17 ***********************************************************************/ 18 #include <linux/pci.h> 19 #include <linux/netdevice.h> 20 #include <linux/vmalloc.h> 21 #include "liquidio_common.h" 22 #include "octeon_droq.h" 23 #include "octeon_iq.h" 24 #include "response_manager.h" 25 #include "octeon_device.h" 26 #include "octeon_main.h" 27 #include "octeon_network.h" 28 #include "cn66xx_regs.h" 29 #include "cn66xx_device.h" 30 #include "cn23xx_pf_device.h" 31 #include "cn23xx_vf_device.h" 32 33 struct niclist { 34 struct list_head list; 35 void *ptr; 36 }; 37 38 struct __dispatch { 39 struct list_head list; 40 struct octeon_recv_info *rinfo; 41 octeon_dispatch_fn_t disp_fn; 42 }; 43 44 /** Get the argument that the user set when registering dispatch 45 * function for a given opcode/subcode. 46 * @param octeon_dev - the octeon device pointer. 47 * @param opcode - the opcode for which the dispatch argument 48 * is to be checked. 49 * @param subcode - the subcode for which the dispatch argument 50 * is to be checked. 51 * @return Success: void * (argument to the dispatch function) 52 * @return Failure: NULL 53 * 54 */ 55 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 56 u16 opcode, u16 subcode) 57 { 58 int idx; 59 struct list_head *dispatch; 60 void *fn_arg = NULL; 61 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 62 63 idx = combined_opcode & OCTEON_OPCODE_MASK; 64 65 spin_lock_bh(&octeon_dev->dispatch.lock); 66 67 if (octeon_dev->dispatch.count == 0) { 68 spin_unlock_bh(&octeon_dev->dispatch.lock); 69 return NULL; 70 } 71 72 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 73 fn_arg = octeon_dev->dispatch.dlist[idx].arg; 74 } else { 75 list_for_each(dispatch, 76 &octeon_dev->dispatch.dlist[idx].list) { 77 if (((struct octeon_dispatch *)dispatch)->opcode == 78 combined_opcode) { 79 fn_arg = ((struct octeon_dispatch *) 80 dispatch)->arg; 81 break; 82 } 83 } 84 } 85 86 spin_unlock_bh(&octeon_dev->dispatch.lock); 87 return fn_arg; 88 } 89 90 /** Check for packets on Droq. This function should be called with lock held. 91 * @param droq - Droq on which count is checked. 92 * @return Returns packet count. 93 */ 94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) 95 { 96 u32 pkt_count = 0; 97 u32 last_count; 98 99 pkt_count = readl(droq->pkts_sent_reg); 100 101 last_count = pkt_count - droq->pkt_count; 102 droq->pkt_count = pkt_count; 103 104 /* we shall write to cnts at napi irq enable or end of droq tasklet */ 105 if (last_count) 106 atomic_add(last_count, &droq->pkts_pending); 107 108 return last_count; 109 } 110 111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 112 { 113 u32 count = 0; 114 115 /* max_empty_descs is the max. no. of descs that can have no buffers. 116 * If the empty desc count goes beyond this value, we cannot safely 117 * read in a 64K packet sent by Octeon 118 * (64K is max pkt size from Octeon) 119 */ 120 droq->max_empty_descs = 0; 121 122 do { 123 droq->max_empty_descs++; 124 count += droq->buffer_size; 125 } while (count < (64 * 1024)); 126 127 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 128 } 129 130 static void octeon_droq_reset_indices(struct octeon_droq *droq) 131 { 132 droq->read_idx = 0; 133 droq->write_idx = 0; 134 droq->refill_idx = 0; 135 droq->refill_count = 0; 136 atomic_set(&droq->pkts_pending, 0); 137 } 138 139 static void 140 octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 141 struct octeon_droq *droq) 142 { 143 u32 i; 144 struct octeon_skb_page_info *pg_info; 145 146 for (i = 0; i < droq->max_count; i++) { 147 pg_info = &droq->recv_buf_list[i].pg_info; 148 149 if (pg_info->dma) 150 lio_unmap_ring(oct->pci_dev, 151 (u64)pg_info->dma); 152 pg_info->dma = 0; 153 154 if (pg_info->page) 155 recv_buffer_destroy(droq->recv_buf_list[i].buffer, 156 pg_info); 157 158 droq->recv_buf_list[i].buffer = NULL; 159 } 160 161 octeon_droq_reset_indices(droq); 162 } 163 164 static int 165 octeon_droq_setup_ring_buffers(struct octeon_device *oct, 166 struct octeon_droq *droq) 167 { 168 u32 i; 169 void *buf; 170 struct octeon_droq_desc *desc_ring = droq->desc_ring; 171 172 for (i = 0; i < droq->max_count; i++) { 173 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); 174 175 if (!buf) { 176 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 177 __func__); 178 droq->stats.rx_alloc_failure++; 179 return -ENOMEM; 180 } 181 182 droq->recv_buf_list[i].buffer = buf; 183 droq->recv_buf_list[i].data = get_rbd(buf); 184 desc_ring[i].info_ptr = 0; 185 desc_ring[i].buffer_ptr = 186 lio_map_ring(droq->recv_buf_list[i].buffer); 187 } 188 189 octeon_droq_reset_indices(droq); 190 191 octeon_droq_compute_max_packet_bufs(droq); 192 193 return 0; 194 } 195 196 int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 197 { 198 struct octeon_droq *droq = oct->droq[q_no]; 199 200 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 201 202 octeon_droq_destroy_ring_buffers(oct, droq); 203 vfree(droq->recv_buf_list); 204 205 if (droq->desc_ring) 206 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 207 droq->desc_ring, droq->desc_ring_dma); 208 209 memset(droq, 0, OCT_DROQ_SIZE); 210 211 return 0; 212 } 213 214 int octeon_init_droq(struct octeon_device *oct, 215 u32 q_no, 216 u32 num_descs, 217 u32 desc_size, 218 void *app_ctx) 219 { 220 struct octeon_droq *droq; 221 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 222 u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 223 int numa_node = dev_to_node(&oct->pci_dev->dev); 224 225 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 226 227 droq = oct->droq[q_no]; 228 memset(droq, 0, OCT_DROQ_SIZE); 229 230 droq->oct_dev = oct; 231 droq->q_no = q_no; 232 if (app_ctx) 233 droq->app_ctx = app_ctx; 234 else 235 droq->app_ctx = (void *)(size_t)q_no; 236 237 c_num_descs = num_descs; 238 c_buf_size = desc_size; 239 if (OCTEON_CN6XXX(oct)) { 240 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); 241 242 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 243 c_refill_threshold = 244 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 245 } else if (OCTEON_CN23XX_PF(oct)) { 246 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); 247 248 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 249 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 250 } else if (OCTEON_CN23XX_VF(oct)) { 251 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); 252 253 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 254 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 255 } else { 256 return 1; 257 } 258 259 droq->max_count = c_num_descs; 260 droq->buffer_size = c_buf_size; 261 262 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 263 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 264 (dma_addr_t *)&droq->desc_ring_dma); 265 266 if (!droq->desc_ring) { 267 dev_err(&oct->pci_dev->dev, 268 "Output queue %d ring alloc failed\n", q_no); 269 return 1; 270 } 271 272 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 273 q_no, droq->desc_ring, droq->desc_ring_dma); 274 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 275 droq->max_count); 276 277 droq->recv_buf_list = (struct octeon_recv_buffer *) 278 vmalloc_node(droq->max_count * 279 OCT_DROQ_RECVBUF_SIZE, 280 numa_node); 281 if (!droq->recv_buf_list) 282 droq->recv_buf_list = (struct octeon_recv_buffer *) 283 vmalloc(droq->max_count * 284 OCT_DROQ_RECVBUF_SIZE); 285 if (!droq->recv_buf_list) { 286 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 287 goto init_droq_fail; 288 } 289 290 if (octeon_droq_setup_ring_buffers(oct, droq)) 291 goto init_droq_fail; 292 293 droq->pkts_per_intr = c_pkts_per_intr; 294 droq->refill_threshold = c_refill_threshold; 295 296 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 297 droq->max_empty_descs); 298 299 spin_lock_init(&droq->lock); 300 301 INIT_LIST_HEAD(&droq->dispatch_list); 302 303 /* For 56xx Pass1, this function won't be called, so no checks. */ 304 oct->fn_list.setup_oq_regs(oct, q_no); 305 306 oct->io_qmask.oq |= BIT_ULL(q_no); 307 308 return 0; 309 310 init_droq_fail: 311 octeon_delete_droq(oct, q_no); 312 return 1; 313 } 314 315 /* octeon_create_recv_info 316 * Parameters: 317 * octeon_dev - pointer to the octeon device structure 318 * droq - droq in which the packet arrived. 319 * buf_cnt - no. of buffers used by the packet. 320 * idx - index in the descriptor for the first buffer in the packet. 321 * Description: 322 * Allocates a recv_info_t and copies the buffer addresses for packet data 323 * into the recv_pkt space which starts at an 8B offset from recv_info_t. 324 * Flags the descriptors for refill later. If available descriptors go 325 * below the threshold to receive a 64K pkt, new buffers are first allocated 326 * before the recv_pkt_t is created. 327 * This routine will be called in interrupt context. 328 * Returns: 329 * Success: Pointer to recv_info_t 330 * Failure: NULL. 331 * Locks: 332 * The droq->lock is held when this routine is called. 333 */ 334 static inline struct octeon_recv_info *octeon_create_recv_info( 335 struct octeon_device *octeon_dev, 336 struct octeon_droq *droq, 337 u32 buf_cnt, 338 u32 idx) 339 { 340 struct octeon_droq_info *info; 341 struct octeon_recv_pkt *recv_pkt; 342 struct octeon_recv_info *recv_info; 343 u32 i, bytes_left; 344 struct octeon_skb_page_info *pg_info; 345 346 info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data; 347 348 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 349 if (!recv_info) 350 return NULL; 351 352 recv_pkt = recv_info->recv_pkt; 353 recv_pkt->rh = info->rh; 354 recv_pkt->length = (u32)info->length; 355 recv_pkt->buffer_count = (u16)buf_cnt; 356 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 357 358 i = 0; 359 bytes_left = (u32)info->length; 360 361 while (buf_cnt) { 362 { 363 pg_info = &droq->recv_buf_list[idx].pg_info; 364 365 lio_unmap_ring(octeon_dev->pci_dev, 366 (u64)pg_info->dma); 367 pg_info->page = NULL; 368 pg_info->dma = 0; 369 } 370 371 recv_pkt->buffer_size[i] = 372 (bytes_left >= 373 droq->buffer_size) ? droq->buffer_size : bytes_left; 374 375 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 376 droq->recv_buf_list[idx].buffer = NULL; 377 378 idx = incr_index(idx, 1, droq->max_count); 379 bytes_left -= droq->buffer_size; 380 i++; 381 buf_cnt--; 382 } 383 384 return recv_info; 385 } 386 387 /* If we were not able to refill all buffers, try to move around 388 * the buffers that were not dispatched. 389 */ 390 static inline u32 391 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 392 struct octeon_droq_desc *desc_ring) 393 { 394 u32 desc_refilled = 0; 395 396 u32 refill_index = droq->refill_idx; 397 398 while (refill_index != droq->read_idx) { 399 if (droq->recv_buf_list[refill_index].buffer) { 400 droq->recv_buf_list[droq->refill_idx].buffer = 401 droq->recv_buf_list[refill_index].buffer; 402 droq->recv_buf_list[droq->refill_idx].data = 403 droq->recv_buf_list[refill_index].data; 404 desc_ring[droq->refill_idx].buffer_ptr = 405 desc_ring[refill_index].buffer_ptr; 406 droq->recv_buf_list[refill_index].buffer = NULL; 407 desc_ring[refill_index].buffer_ptr = 0; 408 do { 409 droq->refill_idx = incr_index(droq->refill_idx, 410 1, 411 droq->max_count); 412 desc_refilled++; 413 droq->refill_count--; 414 } while (droq->recv_buf_list[droq->refill_idx].buffer); 415 } 416 refill_index = incr_index(refill_index, 1, droq->max_count); 417 } /* while */ 418 return desc_refilled; 419 } 420 421 /* octeon_droq_refill 422 * Parameters: 423 * droq - droq in which descriptors require new buffers. 424 * Description: 425 * Called during normal DROQ processing in interrupt mode or by the poll 426 * thread to refill the descriptors from which buffers were dispatched 427 * to upper layers. Attempts to allocate new buffers. If that fails, moves 428 * up buffers (that were not dispatched) to form a contiguous ring. 429 * Returns: 430 * No of descriptors refilled. 431 * Locks: 432 * This routine is called with droq->lock held. 433 */ 434 static u32 435 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 436 { 437 struct octeon_droq_desc *desc_ring; 438 void *buf = NULL; 439 u8 *data; 440 u32 desc_refilled = 0; 441 struct octeon_skb_page_info *pg_info; 442 443 desc_ring = droq->desc_ring; 444 445 while (droq->refill_count && (desc_refilled < droq->max_count)) { 446 /* If a valid buffer exists (happens if there is no dispatch), 447 * reuse 448 * the buffer, else allocate. 449 */ 450 if (!droq->recv_buf_list[droq->refill_idx].buffer) { 451 pg_info = 452 &droq->recv_buf_list[droq->refill_idx].pg_info; 453 /* Either recycle the existing pages or go for 454 * new page alloc 455 */ 456 if (pg_info->page) 457 buf = recv_buffer_reuse(octeon_dev, pg_info); 458 else 459 buf = recv_buffer_alloc(octeon_dev, pg_info); 460 /* If a buffer could not be allocated, no point in 461 * continuing 462 */ 463 if (!buf) { 464 droq->stats.rx_alloc_failure++; 465 break; 466 } 467 droq->recv_buf_list[droq->refill_idx].buffer = 468 buf; 469 data = get_rbd(buf); 470 } else { 471 data = get_rbd(droq->recv_buf_list 472 [droq->refill_idx].buffer); 473 } 474 475 droq->recv_buf_list[droq->refill_idx].data = data; 476 477 desc_ring[droq->refill_idx].buffer_ptr = 478 lio_map_ring(droq->recv_buf_list[ 479 droq->refill_idx].buffer); 480 481 droq->refill_idx = incr_index(droq->refill_idx, 1, 482 droq->max_count); 483 desc_refilled++; 484 droq->refill_count--; 485 } 486 487 if (droq->refill_count) 488 desc_refilled += 489 octeon_droq_refill_pullup_descs(droq, desc_ring); 490 491 /* if droq->refill_count 492 * The refill count would not change in pass two. We only moved buffers 493 * to close the gap in the ring, but we would still have the same no. of 494 * buffers to refill. 495 */ 496 return desc_refilled; 497 } 498 499 /** check if we can allocate packets to get out of oom. 500 * @param droq - Droq being checked. 501 * @return does not return anything 502 */ 503 void octeon_droq_check_oom(struct octeon_droq *droq) 504 { 505 int desc_refilled; 506 struct octeon_device *oct = droq->oct_dev; 507 508 if (readl(droq->pkts_credit_reg) <= CN23XX_SLI_DEF_BP) { 509 spin_lock_bh(&droq->lock); 510 desc_refilled = octeon_droq_refill(oct, droq); 511 if (desc_refilled) { 512 /* Flush the droq descriptor data to memory to be sure 513 * that when we update the credits the data in memory 514 * is accurate. 515 */ 516 wmb(); 517 writel(desc_refilled, droq->pkts_credit_reg); 518 /* make sure mmio write completes */ 519 mmiowb(); 520 } 521 spin_unlock_bh(&droq->lock); 522 } 523 } 524 525 static inline u32 526 octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 527 { 528 return ((total_len + buf_size - 1) / buf_size); 529 } 530 531 static int 532 octeon_droq_dispatch_pkt(struct octeon_device *oct, 533 struct octeon_droq *droq, 534 union octeon_rh *rh, 535 struct octeon_droq_info *info) 536 { 537 u32 cnt; 538 octeon_dispatch_fn_t disp_fn; 539 struct octeon_recv_info *rinfo; 540 541 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 542 543 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 544 (u16)rh->r.subcode); 545 if (disp_fn) { 546 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 547 if (rinfo) { 548 struct __dispatch *rdisp = rinfo->rsvd; 549 550 rdisp->rinfo = rinfo; 551 rdisp->disp_fn = disp_fn; 552 rinfo->recv_pkt->rh = *rh; 553 list_add_tail(&rdisp->list, 554 &droq->dispatch_list); 555 } else { 556 droq->stats.dropped_nomem++; 557 } 558 } else { 559 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", 560 (unsigned int)rh->r.opcode, 561 (unsigned int)rh->r.subcode); 562 droq->stats.dropped_nodispatch++; 563 } 564 565 return cnt; 566 } 567 568 static inline void octeon_droq_drop_packets(struct octeon_device *oct, 569 struct octeon_droq *droq, 570 u32 cnt) 571 { 572 u32 i = 0, buf_cnt; 573 struct octeon_droq_info *info; 574 575 for (i = 0; i < cnt; i++) { 576 info = (struct octeon_droq_info *) 577 droq->recv_buf_list[droq->read_idx].data; 578 octeon_swap_8B_data((u64 *)info, 2); 579 580 if (info->length) { 581 info->length += OCTNET_FRM_LENGTH_SIZE; 582 droq->stats.bytes_received += info->length; 583 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 584 (u32)info->length); 585 } else { 586 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 587 buf_cnt = 1; 588 } 589 590 droq->read_idx = incr_index(droq->read_idx, buf_cnt, 591 droq->max_count); 592 droq->refill_count += buf_cnt; 593 } 594 } 595 596 static u32 597 octeon_droq_fast_process_packets(struct octeon_device *oct, 598 struct octeon_droq *droq, 599 u32 pkts_to_process) 600 { 601 struct octeon_droq_info *info; 602 union octeon_rh *rh; 603 u32 pkt, total_len = 0, pkt_count; 604 605 pkt_count = pkts_to_process; 606 607 for (pkt = 0; pkt < pkt_count; pkt++) { 608 u32 pkt_len = 0; 609 struct sk_buff *nicbuf = NULL; 610 struct octeon_skb_page_info *pg_info; 611 void *buf; 612 613 info = (struct octeon_droq_info *) 614 droq->recv_buf_list[droq->read_idx].data; 615 octeon_swap_8B_data((u64 *)info, 2); 616 617 if (!info->length) { 618 dev_err(&oct->pci_dev->dev, 619 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 620 droq->q_no, droq->read_idx, pkt_count); 621 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 622 (u8 *)info, 623 OCT_DROQ_INFO_SIZE); 624 break; 625 } 626 627 /* Len of resp hdr in included in the received data len. */ 628 rh = &info->rh; 629 630 info->length += OCTNET_FRM_LENGTH_SIZE; 631 rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64)); 632 total_len += (u32)info->length; 633 if (opcode_slow_path(rh)) { 634 u32 buf_cnt; 635 636 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 637 droq->read_idx = incr_index(droq->read_idx, 638 buf_cnt, droq->max_count); 639 droq->refill_count += buf_cnt; 640 } else { 641 if (info->length <= droq->buffer_size) { 642 pkt_len = (u32)info->length; 643 nicbuf = droq->recv_buf_list[ 644 droq->read_idx].buffer; 645 pg_info = &droq->recv_buf_list[ 646 droq->read_idx].pg_info; 647 if (recv_buffer_recycle(oct, pg_info)) 648 pg_info->page = NULL; 649 droq->recv_buf_list[droq->read_idx].buffer = 650 NULL; 651 652 droq->read_idx = incr_index(droq->read_idx, 1, 653 droq->max_count); 654 droq->refill_count++; 655 } else { 656 nicbuf = octeon_fast_packet_alloc((u32) 657 info->length); 658 pkt_len = 0; 659 /* nicbuf allocation can fail. We'll handle it 660 * inside the loop. 661 */ 662 while (pkt_len < info->length) { 663 int cpy_len, idx = droq->read_idx; 664 665 cpy_len = ((pkt_len + droq->buffer_size) 666 > info->length) ? 667 ((u32)info->length - pkt_len) : 668 droq->buffer_size; 669 670 if (nicbuf) { 671 octeon_fast_packet_next(droq, 672 nicbuf, 673 cpy_len, 674 idx); 675 buf = droq->recv_buf_list[ 676 idx].buffer; 677 recv_buffer_fast_free(buf); 678 droq->recv_buf_list[idx].buffer 679 = NULL; 680 } else { 681 droq->stats.rx_alloc_failure++; 682 } 683 684 pkt_len += cpy_len; 685 droq->read_idx = 686 incr_index(droq->read_idx, 1, 687 droq->max_count); 688 droq->refill_count++; 689 } 690 } 691 692 if (nicbuf) { 693 if (droq->ops.fptr) { 694 droq->ops.fptr(oct->octeon_id, 695 nicbuf, pkt_len, 696 rh, &droq->napi, 697 droq->ops.farg); 698 } else { 699 recv_buffer_free(nicbuf); 700 } 701 } 702 } 703 704 if (droq->refill_count >= droq->refill_threshold) { 705 int desc_refilled = octeon_droq_refill(oct, droq); 706 707 /* Flush the droq descriptor data to memory to be sure 708 * that when we update the credits the data in memory 709 * is accurate. 710 */ 711 wmb(); 712 writel((desc_refilled), droq->pkts_credit_reg); 713 /* make sure mmio write completes */ 714 mmiowb(); 715 } 716 717 } /* for (each packet)... */ 718 719 /* Increment refill_count by the number of buffers processed. */ 720 droq->stats.pkts_received += pkt; 721 droq->stats.bytes_received += total_len; 722 723 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 724 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 725 726 droq->stats.dropped_toomany += (pkts_to_process - pkt); 727 return pkts_to_process; 728 } 729 730 return pkt; 731 } 732 733 int 734 octeon_droq_process_packets(struct octeon_device *oct, 735 struct octeon_droq *droq, 736 u32 budget) 737 { 738 u32 pkt_count = 0, pkts_processed = 0; 739 struct list_head *tmp, *tmp2; 740 741 /* Grab the droq lock */ 742 spin_lock(&droq->lock); 743 744 octeon_droq_check_hw_for_pkts(droq); 745 pkt_count = atomic_read(&droq->pkts_pending); 746 747 if (!pkt_count) { 748 spin_unlock(&droq->lock); 749 return 0; 750 } 751 752 if (pkt_count > budget) 753 pkt_count = budget; 754 755 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); 756 757 atomic_sub(pkts_processed, &droq->pkts_pending); 758 759 /* Release the spin lock */ 760 spin_unlock(&droq->lock); 761 762 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 763 struct __dispatch *rdisp = (struct __dispatch *)tmp; 764 765 list_del(tmp); 766 rdisp->disp_fn(rdisp->rinfo, 767 octeon_get_dispatch_arg 768 (oct, 769 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 770 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 771 } 772 773 /* If there are packets pending. schedule tasklet again */ 774 if (atomic_read(&droq->pkts_pending)) 775 return 1; 776 777 return 0; 778 } 779 780 /** 781 * Utility function to poll for packets. check_hw_for_packets must be 782 * called before calling this routine. 783 */ 784 785 static int 786 octeon_droq_process_poll_pkts(struct octeon_device *oct, 787 struct octeon_droq *droq, u32 budget) 788 { 789 struct list_head *tmp, *tmp2; 790 u32 pkts_available = 0, pkts_processed = 0; 791 u32 total_pkts_processed = 0; 792 793 if (budget > droq->max_count) 794 budget = droq->max_count; 795 796 spin_lock(&droq->lock); 797 798 while (total_pkts_processed < budget) { 799 octeon_droq_check_hw_for_pkts(droq); 800 801 pkts_available = min((budget - total_pkts_processed), 802 (u32)(atomic_read(&droq->pkts_pending))); 803 804 if (pkts_available == 0) 805 break; 806 807 pkts_processed = 808 octeon_droq_fast_process_packets(oct, droq, 809 pkts_available); 810 811 atomic_sub(pkts_processed, &droq->pkts_pending); 812 813 total_pkts_processed += pkts_processed; 814 } 815 816 spin_unlock(&droq->lock); 817 818 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 819 struct __dispatch *rdisp = (struct __dispatch *)tmp; 820 821 list_del(tmp); 822 rdisp->disp_fn(rdisp->rinfo, 823 octeon_get_dispatch_arg 824 (oct, 825 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 826 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 827 } 828 829 return total_pkts_processed; 830 } 831 832 int 833 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, 834 u32 arg) 835 { 836 struct octeon_droq *droq; 837 838 droq = oct->droq[q_no]; 839 840 if (cmd == POLL_EVENT_PROCESS_PKTS) 841 return octeon_droq_process_poll_pkts(oct, droq, arg); 842 843 if (cmd == POLL_EVENT_PENDING_PKTS) { 844 u32 pkt_cnt = atomic_read(&droq->pkts_pending); 845 846 return octeon_droq_process_packets(oct, droq, pkt_cnt); 847 } 848 849 if (cmd == POLL_EVENT_ENABLE_INTR) { 850 u32 value; 851 unsigned long flags; 852 853 /* Enable Pkt Interrupt */ 854 switch (oct->chip_id) { 855 case OCTEON_CN66XX: 856 case OCTEON_CN68XX: { 857 struct octeon_cn6xxx *cn6xxx = 858 (struct octeon_cn6xxx *)oct->chip; 859 spin_lock_irqsave 860 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 861 value = 862 octeon_read_csr(oct, 863 CN6XXX_SLI_PKT_TIME_INT_ENB); 864 value |= (1 << q_no); 865 octeon_write_csr(oct, 866 CN6XXX_SLI_PKT_TIME_INT_ENB, 867 value); 868 value = 869 octeon_read_csr(oct, 870 CN6XXX_SLI_PKT_CNT_INT_ENB); 871 value |= (1 << q_no); 872 octeon_write_csr(oct, 873 CN6XXX_SLI_PKT_CNT_INT_ENB, 874 value); 875 876 /* don't bother flushing the enables */ 877 878 spin_unlock_irqrestore 879 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 880 return 0; 881 } 882 break; 883 case OCTEON_CN23XX_PF_VID: { 884 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 885 } 886 break; 887 888 case OCTEON_CN23XX_VF_VID: 889 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 890 break; 891 } 892 return 0; 893 } 894 895 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); 896 return -EINVAL; 897 } 898 899 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 900 struct octeon_droq_ops *ops) 901 { 902 struct octeon_droq *droq; 903 unsigned long flags; 904 struct octeon_config *oct_cfg = NULL; 905 906 oct_cfg = octeon_get_conf(oct); 907 908 if (!oct_cfg) 909 return -EINVAL; 910 911 if (!(ops)) { 912 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 913 __func__); 914 return -EINVAL; 915 } 916 917 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 918 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 919 __func__, q_no, (oct->num_oqs - 1)); 920 return -EINVAL; 921 } 922 923 droq = oct->droq[q_no]; 924 925 spin_lock_irqsave(&droq->lock, flags); 926 927 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 928 929 spin_unlock_irqrestore(&droq->lock, flags); 930 931 return 0; 932 } 933 934 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 935 { 936 unsigned long flags; 937 struct octeon_droq *droq; 938 struct octeon_config *oct_cfg = NULL; 939 940 oct_cfg = octeon_get_conf(oct); 941 942 if (!oct_cfg) 943 return -EINVAL; 944 945 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 946 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 947 __func__, q_no, oct->num_oqs - 1); 948 return -EINVAL; 949 } 950 951 droq = oct->droq[q_no]; 952 953 if (!droq) { 954 dev_info(&oct->pci_dev->dev, 955 "Droq id (%d) not available.\n", q_no); 956 return 0; 957 } 958 959 spin_lock_irqsave(&droq->lock, flags); 960 961 droq->ops.fptr = NULL; 962 droq->ops.farg = NULL; 963 droq->ops.drop_on_max = 0; 964 965 spin_unlock_irqrestore(&droq->lock, flags); 966 967 return 0; 968 } 969 970 int octeon_create_droq(struct octeon_device *oct, 971 u32 q_no, u32 num_descs, 972 u32 desc_size, void *app_ctx) 973 { 974 struct octeon_droq *droq; 975 int numa_node = dev_to_node(&oct->pci_dev->dev); 976 977 if (oct->droq[q_no]) { 978 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 979 q_no); 980 return 1; 981 } 982 983 /* Allocate the DS for the new droq. */ 984 droq = vmalloc_node(sizeof(*droq), numa_node); 985 if (!droq) 986 droq = vmalloc(sizeof(*droq)); 987 if (!droq) 988 return -1; 989 990 memset(droq, 0, sizeof(struct octeon_droq)); 991 992 /*Disable the pkt o/p for this Q */ 993 octeon_set_droq_pkt_op(oct, q_no, 0); 994 oct->droq[q_no] = droq; 995 996 /* Initialize the Droq */ 997 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { 998 vfree(oct->droq[q_no]); 999 oct->droq[q_no] = NULL; 1000 return -1; 1001 } 1002 1003 oct->num_oqs++; 1004 1005 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 1006 oct->num_oqs); 1007 1008 /* Global Droq register settings */ 1009 1010 /* As of now not required, as setting are done for all 32 Droqs at 1011 * the same time. 1012 */ 1013 return 0; 1014 } 1015