1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more details. 17 ***********************************************************************/ 18 #include <linux/pci.h> 19 #include <linux/netdevice.h> 20 #include <linux/vmalloc.h> 21 #include "liquidio_common.h" 22 #include "octeon_droq.h" 23 #include "octeon_iq.h" 24 #include "response_manager.h" 25 #include "octeon_device.h" 26 #include "octeon_main.h" 27 #include "octeon_network.h" 28 #include "cn66xx_regs.h" 29 #include "cn66xx_device.h" 30 #include "cn23xx_pf_device.h" 31 #include "cn23xx_vf_device.h" 32 33 struct niclist { 34 struct list_head list; 35 void *ptr; 36 }; 37 38 struct __dispatch { 39 struct list_head list; 40 struct octeon_recv_info *rinfo; 41 octeon_dispatch_fn_t disp_fn; 42 }; 43 44 /** Get the argument that the user set when registering dispatch 45 * function for a given opcode/subcode. 46 * @param octeon_dev - the octeon device pointer. 47 * @param opcode - the opcode for which the dispatch argument 48 * is to be checked. 49 * @param subcode - the subcode for which the dispatch argument 50 * is to be checked. 51 * @return Success: void * (argument to the dispatch function) 52 * @return Failure: NULL 53 * 54 */ 55 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 56 u16 opcode, u16 subcode) 57 { 58 int idx; 59 struct list_head *dispatch; 60 void *fn_arg = NULL; 61 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 62 63 idx = combined_opcode & OCTEON_OPCODE_MASK; 64 65 spin_lock_bh(&octeon_dev->dispatch.lock); 66 67 if (octeon_dev->dispatch.count == 0) { 68 spin_unlock_bh(&octeon_dev->dispatch.lock); 69 return NULL; 70 } 71 72 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 73 fn_arg = octeon_dev->dispatch.dlist[idx].arg; 74 } else { 75 list_for_each(dispatch, 76 &octeon_dev->dispatch.dlist[idx].list) { 77 if (((struct octeon_dispatch *)dispatch)->opcode == 78 combined_opcode) { 79 fn_arg = ((struct octeon_dispatch *) 80 dispatch)->arg; 81 break; 82 } 83 } 84 } 85 86 spin_unlock_bh(&octeon_dev->dispatch.lock); 87 return fn_arg; 88 } 89 90 /** Check for packets on Droq. This function should be called with lock held. 91 * @param droq - Droq on which count is checked. 92 * @return Returns packet count. 93 */ 94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) 95 { 96 u32 pkt_count = 0; 97 u32 last_count; 98 99 pkt_count = readl(droq->pkts_sent_reg); 100 101 last_count = pkt_count - droq->pkt_count; 102 droq->pkt_count = pkt_count; 103 104 /* we shall write to cnts at napi irq enable or end of droq tasklet */ 105 if (last_count) 106 atomic_add(last_count, &droq->pkts_pending); 107 108 return last_count; 109 } 110 111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 112 { 113 u32 count = 0; 114 115 /* max_empty_descs is the max. no. of descs that can have no buffers. 116 * If the empty desc count goes beyond this value, we cannot safely 117 * read in a 64K packet sent by Octeon 118 * (64K is max pkt size from Octeon) 119 */ 120 droq->max_empty_descs = 0; 121 122 do { 123 droq->max_empty_descs++; 124 count += droq->buffer_size; 125 } while (count < (64 * 1024)); 126 127 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 128 } 129 130 static void octeon_droq_reset_indices(struct octeon_droq *droq) 131 { 132 droq->read_idx = 0; 133 droq->write_idx = 0; 134 droq->refill_idx = 0; 135 droq->refill_count = 0; 136 atomic_set(&droq->pkts_pending, 0); 137 } 138 139 static void 140 octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 141 struct octeon_droq *droq) 142 { 143 u32 i; 144 struct octeon_skb_page_info *pg_info; 145 146 for (i = 0; i < droq->max_count; i++) { 147 pg_info = &droq->recv_buf_list[i].pg_info; 148 149 if (pg_info->dma) 150 lio_unmap_ring(oct->pci_dev, 151 (u64)pg_info->dma); 152 pg_info->dma = 0; 153 154 if (pg_info->page) 155 recv_buffer_destroy(droq->recv_buf_list[i].buffer, 156 pg_info); 157 158 droq->recv_buf_list[i].buffer = NULL; 159 } 160 161 octeon_droq_reset_indices(droq); 162 } 163 164 static int 165 octeon_droq_setup_ring_buffers(struct octeon_device *oct, 166 struct octeon_droq *droq) 167 { 168 u32 i; 169 void *buf; 170 struct octeon_droq_desc *desc_ring = droq->desc_ring; 171 172 for (i = 0; i < droq->max_count; i++) { 173 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); 174 175 if (!buf) { 176 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 177 __func__); 178 droq->stats.rx_alloc_failure++; 179 return -ENOMEM; 180 } 181 182 droq->recv_buf_list[i].buffer = buf; 183 droq->recv_buf_list[i].data = get_rbd(buf); 184 droq->info_list[i].length = 0; 185 186 /* map ring buffers into memory */ 187 desc_ring[i].info_ptr = lio_map_ring_info(droq, i); 188 desc_ring[i].buffer_ptr = 189 lio_map_ring(droq->recv_buf_list[i].buffer); 190 } 191 192 octeon_droq_reset_indices(droq); 193 194 octeon_droq_compute_max_packet_bufs(droq); 195 196 return 0; 197 } 198 199 int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 200 { 201 struct octeon_droq *droq = oct->droq[q_no]; 202 203 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 204 205 octeon_droq_destroy_ring_buffers(oct, droq); 206 vfree(droq->recv_buf_list); 207 208 if (droq->info_base_addr) 209 lio_free_info_buffer(oct, droq); 210 211 if (droq->desc_ring) 212 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 213 droq->desc_ring, droq->desc_ring_dma); 214 215 memset(droq, 0, OCT_DROQ_SIZE); 216 217 return 0; 218 } 219 220 int octeon_init_droq(struct octeon_device *oct, 221 u32 q_no, 222 u32 num_descs, 223 u32 desc_size, 224 void *app_ctx) 225 { 226 struct octeon_droq *droq; 227 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 228 u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 229 int numa_node = dev_to_node(&oct->pci_dev->dev); 230 231 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 232 233 droq = oct->droq[q_no]; 234 memset(droq, 0, OCT_DROQ_SIZE); 235 236 droq->oct_dev = oct; 237 droq->q_no = q_no; 238 if (app_ctx) 239 droq->app_ctx = app_ctx; 240 else 241 droq->app_ctx = (void *)(size_t)q_no; 242 243 c_num_descs = num_descs; 244 c_buf_size = desc_size; 245 if (OCTEON_CN6XXX(oct)) { 246 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); 247 248 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 249 c_refill_threshold = 250 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 251 } else if (OCTEON_CN23XX_PF(oct)) { 252 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); 253 254 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 255 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 256 } else if (OCTEON_CN23XX_VF(oct)) { 257 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); 258 259 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 260 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 261 } else { 262 return 1; 263 } 264 265 droq->max_count = c_num_descs; 266 droq->buffer_size = c_buf_size; 267 268 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 269 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 270 (dma_addr_t *)&droq->desc_ring_dma); 271 272 if (!droq->desc_ring) { 273 dev_err(&oct->pci_dev->dev, 274 "Output queue %d ring alloc failed\n", q_no); 275 return 1; 276 } 277 278 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 279 q_no, droq->desc_ring, droq->desc_ring_dma); 280 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 281 droq->max_count); 282 283 droq->info_list = lio_alloc_info_buffer(oct, droq); 284 if (!droq->info_list) { 285 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n"); 286 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 287 droq->desc_ring, droq->desc_ring_dma); 288 return 1; 289 } 290 291 droq->recv_buf_list = (struct octeon_recv_buffer *) 292 vmalloc_node(droq->max_count * 293 OCT_DROQ_RECVBUF_SIZE, 294 numa_node); 295 if (!droq->recv_buf_list) 296 droq->recv_buf_list = (struct octeon_recv_buffer *) 297 vmalloc(droq->max_count * 298 OCT_DROQ_RECVBUF_SIZE); 299 if (!droq->recv_buf_list) { 300 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 301 goto init_droq_fail; 302 } 303 304 if (octeon_droq_setup_ring_buffers(oct, droq)) 305 goto init_droq_fail; 306 307 droq->pkts_per_intr = c_pkts_per_intr; 308 droq->refill_threshold = c_refill_threshold; 309 310 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 311 droq->max_empty_descs); 312 313 spin_lock_init(&droq->lock); 314 315 INIT_LIST_HEAD(&droq->dispatch_list); 316 317 /* For 56xx Pass1, this function won't be called, so no checks. */ 318 oct->fn_list.setup_oq_regs(oct, q_no); 319 320 oct->io_qmask.oq |= BIT_ULL(q_no); 321 322 return 0; 323 324 init_droq_fail: 325 octeon_delete_droq(oct, q_no); 326 return 1; 327 } 328 329 /* octeon_create_recv_info 330 * Parameters: 331 * octeon_dev - pointer to the octeon device structure 332 * droq - droq in which the packet arrived. 333 * buf_cnt - no. of buffers used by the packet. 334 * idx - index in the descriptor for the first buffer in the packet. 335 * Description: 336 * Allocates a recv_info_t and copies the buffer addresses for packet data 337 * into the recv_pkt space which starts at an 8B offset from recv_info_t. 338 * Flags the descriptors for refill later. If available descriptors go 339 * below the threshold to receive a 64K pkt, new buffers are first allocated 340 * before the recv_pkt_t is created. 341 * This routine will be called in interrupt context. 342 * Returns: 343 * Success: Pointer to recv_info_t 344 * Failure: NULL. 345 * Locks: 346 * The droq->lock is held when this routine is called. 347 */ 348 static inline struct octeon_recv_info *octeon_create_recv_info( 349 struct octeon_device *octeon_dev, 350 struct octeon_droq *droq, 351 u32 buf_cnt, 352 u32 idx) 353 { 354 struct octeon_droq_info *info; 355 struct octeon_recv_pkt *recv_pkt; 356 struct octeon_recv_info *recv_info; 357 u32 i, bytes_left; 358 struct octeon_skb_page_info *pg_info; 359 360 info = &droq->info_list[idx]; 361 362 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 363 if (!recv_info) 364 return NULL; 365 366 recv_pkt = recv_info->recv_pkt; 367 recv_pkt->rh = info->rh; 368 recv_pkt->length = (u32)info->length; 369 recv_pkt->buffer_count = (u16)buf_cnt; 370 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 371 372 i = 0; 373 bytes_left = (u32)info->length; 374 375 while (buf_cnt) { 376 { 377 pg_info = &droq->recv_buf_list[idx].pg_info; 378 379 lio_unmap_ring(octeon_dev->pci_dev, 380 (u64)pg_info->dma); 381 pg_info->page = NULL; 382 pg_info->dma = 0; 383 } 384 385 recv_pkt->buffer_size[i] = 386 (bytes_left >= 387 droq->buffer_size) ? droq->buffer_size : bytes_left; 388 389 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 390 droq->recv_buf_list[idx].buffer = NULL; 391 392 idx = incr_index(idx, 1, droq->max_count); 393 bytes_left -= droq->buffer_size; 394 i++; 395 buf_cnt--; 396 } 397 398 return recv_info; 399 } 400 401 /* If we were not able to refill all buffers, try to move around 402 * the buffers that were not dispatched. 403 */ 404 static inline u32 405 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 406 struct octeon_droq_desc *desc_ring) 407 { 408 u32 desc_refilled = 0; 409 410 u32 refill_index = droq->refill_idx; 411 412 while (refill_index != droq->read_idx) { 413 if (droq->recv_buf_list[refill_index].buffer) { 414 droq->recv_buf_list[droq->refill_idx].buffer = 415 droq->recv_buf_list[refill_index].buffer; 416 droq->recv_buf_list[droq->refill_idx].data = 417 droq->recv_buf_list[refill_index].data; 418 desc_ring[droq->refill_idx].buffer_ptr = 419 desc_ring[refill_index].buffer_ptr; 420 droq->recv_buf_list[refill_index].buffer = NULL; 421 desc_ring[refill_index].buffer_ptr = 0; 422 do { 423 droq->refill_idx = incr_index(droq->refill_idx, 424 1, 425 droq->max_count); 426 desc_refilled++; 427 droq->refill_count--; 428 } while (droq->recv_buf_list[droq->refill_idx]. 429 buffer); 430 } 431 refill_index = incr_index(refill_index, 1, droq->max_count); 432 } /* while */ 433 return desc_refilled; 434 } 435 436 /* octeon_droq_refill 437 * Parameters: 438 * droq - droq in which descriptors require new buffers. 439 * Description: 440 * Called during normal DROQ processing in interrupt mode or by the poll 441 * thread to refill the descriptors from which buffers were dispatched 442 * to upper layers. Attempts to allocate new buffers. If that fails, moves 443 * up buffers (that were not dispatched) to form a contiguous ring. 444 * Returns: 445 * No of descriptors refilled. 446 * Locks: 447 * This routine is called with droq->lock held. 448 */ 449 static u32 450 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 451 { 452 struct octeon_droq_desc *desc_ring; 453 void *buf = NULL; 454 u8 *data; 455 u32 desc_refilled = 0; 456 struct octeon_skb_page_info *pg_info; 457 458 desc_ring = droq->desc_ring; 459 460 while (droq->refill_count && (desc_refilled < droq->max_count)) { 461 /* If a valid buffer exists (happens if there is no dispatch), 462 * reuse 463 * the buffer, else allocate. 464 */ 465 if (!droq->recv_buf_list[droq->refill_idx].buffer) { 466 pg_info = 467 &droq->recv_buf_list[droq->refill_idx].pg_info; 468 /* Either recycle the existing pages or go for 469 * new page alloc 470 */ 471 if (pg_info->page) 472 buf = recv_buffer_reuse(octeon_dev, pg_info); 473 else 474 buf = recv_buffer_alloc(octeon_dev, pg_info); 475 /* If a buffer could not be allocated, no point in 476 * continuing 477 */ 478 if (!buf) { 479 droq->stats.rx_alloc_failure++; 480 break; 481 } 482 droq->recv_buf_list[droq->refill_idx].buffer = 483 buf; 484 data = get_rbd(buf); 485 } else { 486 data = get_rbd(droq->recv_buf_list 487 [droq->refill_idx].buffer); 488 } 489 490 droq->recv_buf_list[droq->refill_idx].data = data; 491 492 desc_ring[droq->refill_idx].buffer_ptr = 493 lio_map_ring(droq->recv_buf_list[droq-> 494 refill_idx].buffer); 495 /* Reset any previous values in the length field. */ 496 droq->info_list[droq->refill_idx].length = 0; 497 498 droq->refill_idx = incr_index(droq->refill_idx, 1, 499 droq->max_count); 500 desc_refilled++; 501 droq->refill_count--; 502 } 503 504 if (droq->refill_count) 505 desc_refilled += 506 octeon_droq_refill_pullup_descs(droq, desc_ring); 507 508 /* if droq->refill_count 509 * The refill count would not change in pass two. We only moved buffers 510 * to close the gap in the ring, but we would still have the same no. of 511 * buffers to refill. 512 */ 513 return desc_refilled; 514 } 515 516 /** check if we can allocate packets to get out of oom. 517 * @param droq - Droq being checked. 518 * @return does not return anything 519 */ 520 void octeon_droq_check_oom(struct octeon_droq *droq) 521 { 522 int desc_refilled; 523 struct octeon_device *oct = droq->oct_dev; 524 525 if (readl(droq->pkts_credit_reg) <= CN23XX_SLI_DEF_BP) { 526 spin_lock_bh(&droq->lock); 527 desc_refilled = octeon_droq_refill(oct, droq); 528 if (desc_refilled) { 529 /* Flush the droq descriptor data to memory to be sure 530 * that when we update the credits the data in memory 531 * is accurate. 532 */ 533 wmb(); 534 writel(desc_refilled, droq->pkts_credit_reg); 535 /* make sure mmio write completes */ 536 mmiowb(); 537 } 538 spin_unlock_bh(&droq->lock); 539 } 540 } 541 542 static inline u32 543 octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 544 { 545 u32 buf_cnt = 0; 546 547 while (total_len > (buf_size * buf_cnt)) 548 buf_cnt++; 549 return buf_cnt; 550 } 551 552 static int 553 octeon_droq_dispatch_pkt(struct octeon_device *oct, 554 struct octeon_droq *droq, 555 union octeon_rh *rh, 556 struct octeon_droq_info *info) 557 { 558 u32 cnt; 559 octeon_dispatch_fn_t disp_fn; 560 struct octeon_recv_info *rinfo; 561 562 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 563 564 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 565 (u16)rh->r.subcode); 566 if (disp_fn) { 567 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 568 if (rinfo) { 569 struct __dispatch *rdisp = rinfo->rsvd; 570 571 rdisp->rinfo = rinfo; 572 rdisp->disp_fn = disp_fn; 573 rinfo->recv_pkt->rh = *rh; 574 list_add_tail(&rdisp->list, 575 &droq->dispatch_list); 576 } else { 577 droq->stats.dropped_nomem++; 578 } 579 } else { 580 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", 581 (unsigned int)rh->r.opcode, 582 (unsigned int)rh->r.subcode); 583 droq->stats.dropped_nodispatch++; 584 } 585 586 return cnt; 587 } 588 589 static inline void octeon_droq_drop_packets(struct octeon_device *oct, 590 struct octeon_droq *droq, 591 u32 cnt) 592 { 593 u32 i = 0, buf_cnt; 594 struct octeon_droq_info *info; 595 596 for (i = 0; i < cnt; i++) { 597 info = &droq->info_list[droq->read_idx]; 598 octeon_swap_8B_data((u64 *)info, 2); 599 600 if (info->length) { 601 info->length -= OCT_RH_SIZE; 602 droq->stats.bytes_received += info->length; 603 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 604 (u32)info->length); 605 } else { 606 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 607 buf_cnt = 1; 608 } 609 610 droq->read_idx = incr_index(droq->read_idx, buf_cnt, 611 droq->max_count); 612 droq->refill_count += buf_cnt; 613 } 614 } 615 616 static u32 617 octeon_droq_fast_process_packets(struct octeon_device *oct, 618 struct octeon_droq *droq, 619 u32 pkts_to_process) 620 { 621 struct octeon_droq_info *info; 622 union octeon_rh *rh; 623 u32 pkt, total_len = 0, pkt_count; 624 625 pkt_count = pkts_to_process; 626 627 for (pkt = 0; pkt < pkt_count; pkt++) { 628 u32 pkt_len = 0; 629 struct sk_buff *nicbuf = NULL; 630 struct octeon_skb_page_info *pg_info; 631 void *buf; 632 633 info = &droq->info_list[droq->read_idx]; 634 octeon_swap_8B_data((u64 *)info, 2); 635 636 if (!info->length) { 637 dev_err(&oct->pci_dev->dev, 638 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 639 droq->q_no, droq->read_idx, pkt_count); 640 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 641 (u8 *)info, 642 OCT_DROQ_INFO_SIZE); 643 break; 644 } 645 646 /* Len of resp hdr in included in the received data len. */ 647 info->length -= OCT_RH_SIZE; 648 rh = &info->rh; 649 650 total_len += (u32)info->length; 651 if (opcode_slow_path(rh)) { 652 u32 buf_cnt; 653 654 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 655 droq->read_idx = incr_index(droq->read_idx, 656 buf_cnt, droq->max_count); 657 droq->refill_count += buf_cnt; 658 } else { 659 if (info->length <= droq->buffer_size) { 660 pkt_len = (u32)info->length; 661 nicbuf = droq->recv_buf_list[ 662 droq->read_idx].buffer; 663 pg_info = &droq->recv_buf_list[ 664 droq->read_idx].pg_info; 665 if (recv_buffer_recycle(oct, pg_info)) 666 pg_info->page = NULL; 667 droq->recv_buf_list[droq->read_idx].buffer = 668 NULL; 669 670 droq->read_idx = incr_index(droq->read_idx, 1, 671 droq->max_count); 672 droq->refill_count++; 673 } else { 674 nicbuf = octeon_fast_packet_alloc((u32) 675 info->length); 676 pkt_len = 0; 677 /* nicbuf allocation can fail. We'll handle it 678 * inside the loop. 679 */ 680 while (pkt_len < info->length) { 681 int cpy_len, idx = droq->read_idx; 682 683 cpy_len = ((pkt_len + droq->buffer_size) 684 > info->length) ? 685 ((u32)info->length - pkt_len) : 686 droq->buffer_size; 687 688 if (nicbuf) { 689 octeon_fast_packet_next(droq, 690 nicbuf, 691 cpy_len, 692 idx); 693 buf = droq->recv_buf_list[idx]. 694 buffer; 695 recv_buffer_fast_free(buf); 696 droq->recv_buf_list[idx].buffer 697 = NULL; 698 } else { 699 droq->stats.rx_alloc_failure++; 700 } 701 702 pkt_len += cpy_len; 703 droq->read_idx = 704 incr_index(droq->read_idx, 1, 705 droq->max_count); 706 droq->refill_count++; 707 } 708 } 709 710 if (nicbuf) { 711 if (droq->ops.fptr) { 712 droq->ops.fptr(oct->octeon_id, 713 nicbuf, pkt_len, 714 rh, &droq->napi, 715 droq->ops.farg); 716 } else { 717 recv_buffer_free(nicbuf); 718 } 719 } 720 } 721 722 if (droq->refill_count >= droq->refill_threshold) { 723 int desc_refilled = octeon_droq_refill(oct, droq); 724 725 /* Flush the droq descriptor data to memory to be sure 726 * that when we update the credits the data in memory 727 * is accurate. 728 */ 729 wmb(); 730 writel((desc_refilled), droq->pkts_credit_reg); 731 /* make sure mmio write completes */ 732 mmiowb(); 733 } 734 735 } /* for (each packet)... */ 736 737 /* Increment refill_count by the number of buffers processed. */ 738 droq->stats.pkts_received += pkt; 739 droq->stats.bytes_received += total_len; 740 741 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 742 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 743 744 droq->stats.dropped_toomany += (pkts_to_process - pkt); 745 return pkts_to_process; 746 } 747 748 return pkt; 749 } 750 751 int 752 octeon_droq_process_packets(struct octeon_device *oct, 753 struct octeon_droq *droq, 754 u32 budget) 755 { 756 u32 pkt_count = 0, pkts_processed = 0; 757 struct list_head *tmp, *tmp2; 758 759 /* Grab the droq lock */ 760 spin_lock(&droq->lock); 761 762 octeon_droq_check_hw_for_pkts(droq); 763 pkt_count = atomic_read(&droq->pkts_pending); 764 765 if (!pkt_count) { 766 spin_unlock(&droq->lock); 767 return 0; 768 } 769 770 if (pkt_count > budget) 771 pkt_count = budget; 772 773 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); 774 775 atomic_sub(pkts_processed, &droq->pkts_pending); 776 777 /* Release the spin lock */ 778 spin_unlock(&droq->lock); 779 780 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 781 struct __dispatch *rdisp = (struct __dispatch *)tmp; 782 783 list_del(tmp); 784 rdisp->disp_fn(rdisp->rinfo, 785 octeon_get_dispatch_arg 786 (oct, 787 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 788 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 789 } 790 791 /* If there are packets pending. schedule tasklet again */ 792 if (atomic_read(&droq->pkts_pending)) 793 return 1; 794 795 return 0; 796 } 797 798 /** 799 * Utility function to poll for packets. check_hw_for_packets must be 800 * called before calling this routine. 801 */ 802 803 static int 804 octeon_droq_process_poll_pkts(struct octeon_device *oct, 805 struct octeon_droq *droq, u32 budget) 806 { 807 struct list_head *tmp, *tmp2; 808 u32 pkts_available = 0, pkts_processed = 0; 809 u32 total_pkts_processed = 0; 810 811 if (budget > droq->max_count) 812 budget = droq->max_count; 813 814 spin_lock(&droq->lock); 815 816 while (total_pkts_processed < budget) { 817 octeon_droq_check_hw_for_pkts(droq); 818 819 pkts_available = min((budget - total_pkts_processed), 820 (u32)(atomic_read(&droq->pkts_pending))); 821 822 if (pkts_available == 0) 823 break; 824 825 pkts_processed = 826 octeon_droq_fast_process_packets(oct, droq, 827 pkts_available); 828 829 atomic_sub(pkts_processed, &droq->pkts_pending); 830 831 total_pkts_processed += pkts_processed; 832 } 833 834 spin_unlock(&droq->lock); 835 836 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 837 struct __dispatch *rdisp = (struct __dispatch *)tmp; 838 839 list_del(tmp); 840 rdisp->disp_fn(rdisp->rinfo, 841 octeon_get_dispatch_arg 842 (oct, 843 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 844 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 845 } 846 847 return total_pkts_processed; 848 } 849 850 int 851 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, 852 u32 arg) 853 { 854 struct octeon_droq *droq; 855 856 droq = oct->droq[q_no]; 857 858 if (cmd == POLL_EVENT_PROCESS_PKTS) 859 return octeon_droq_process_poll_pkts(oct, droq, arg); 860 861 if (cmd == POLL_EVENT_PENDING_PKTS) { 862 u32 pkt_cnt = atomic_read(&droq->pkts_pending); 863 864 return octeon_droq_process_packets(oct, droq, pkt_cnt); 865 } 866 867 if (cmd == POLL_EVENT_ENABLE_INTR) { 868 u32 value; 869 unsigned long flags; 870 871 /* Enable Pkt Interrupt */ 872 switch (oct->chip_id) { 873 case OCTEON_CN66XX: 874 case OCTEON_CN68XX: { 875 struct octeon_cn6xxx *cn6xxx = 876 (struct octeon_cn6xxx *)oct->chip; 877 spin_lock_irqsave 878 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 879 value = 880 octeon_read_csr(oct, 881 CN6XXX_SLI_PKT_TIME_INT_ENB); 882 value |= (1 << q_no); 883 octeon_write_csr(oct, 884 CN6XXX_SLI_PKT_TIME_INT_ENB, 885 value); 886 value = 887 octeon_read_csr(oct, 888 CN6XXX_SLI_PKT_CNT_INT_ENB); 889 value |= (1 << q_no); 890 octeon_write_csr(oct, 891 CN6XXX_SLI_PKT_CNT_INT_ENB, 892 value); 893 894 /* don't bother flushing the enables */ 895 896 spin_unlock_irqrestore 897 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 898 return 0; 899 } 900 break; 901 case OCTEON_CN23XX_PF_VID: { 902 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 903 } 904 break; 905 906 case OCTEON_CN23XX_VF_VID: 907 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 908 break; 909 } 910 return 0; 911 } 912 913 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); 914 return -EINVAL; 915 } 916 917 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 918 struct octeon_droq_ops *ops) 919 { 920 struct octeon_droq *droq; 921 unsigned long flags; 922 struct octeon_config *oct_cfg = NULL; 923 924 oct_cfg = octeon_get_conf(oct); 925 926 if (!oct_cfg) 927 return -EINVAL; 928 929 if (!(ops)) { 930 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 931 __func__); 932 return -EINVAL; 933 } 934 935 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 936 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 937 __func__, q_no, (oct->num_oqs - 1)); 938 return -EINVAL; 939 } 940 941 droq = oct->droq[q_no]; 942 943 spin_lock_irqsave(&droq->lock, flags); 944 945 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 946 947 spin_unlock_irqrestore(&droq->lock, flags); 948 949 return 0; 950 } 951 952 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 953 { 954 unsigned long flags; 955 struct octeon_droq *droq; 956 struct octeon_config *oct_cfg = NULL; 957 958 oct_cfg = octeon_get_conf(oct); 959 960 if (!oct_cfg) 961 return -EINVAL; 962 963 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 964 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 965 __func__, q_no, oct->num_oqs - 1); 966 return -EINVAL; 967 } 968 969 droq = oct->droq[q_no]; 970 971 if (!droq) { 972 dev_info(&oct->pci_dev->dev, 973 "Droq id (%d) not available.\n", q_no); 974 return 0; 975 } 976 977 spin_lock_irqsave(&droq->lock, flags); 978 979 droq->ops.fptr = NULL; 980 droq->ops.farg = NULL; 981 droq->ops.drop_on_max = 0; 982 983 spin_unlock_irqrestore(&droq->lock, flags); 984 985 return 0; 986 } 987 988 int octeon_create_droq(struct octeon_device *oct, 989 u32 q_no, u32 num_descs, 990 u32 desc_size, void *app_ctx) 991 { 992 struct octeon_droq *droq; 993 int numa_node = dev_to_node(&oct->pci_dev->dev); 994 995 if (oct->droq[q_no]) { 996 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 997 q_no); 998 return 1; 999 } 1000 1001 /* Allocate the DS for the new droq. */ 1002 droq = vmalloc_node(sizeof(*droq), numa_node); 1003 if (!droq) 1004 droq = vmalloc(sizeof(*droq)); 1005 if (!droq) 1006 return -1; 1007 1008 memset(droq, 0, sizeof(struct octeon_droq)); 1009 1010 /*Disable the pkt o/p for this Q */ 1011 octeon_set_droq_pkt_op(oct, q_no, 0); 1012 oct->droq[q_no] = droq; 1013 1014 /* Initialize the Droq */ 1015 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { 1016 vfree(oct->droq[q_no]); 1017 oct->droq[q_no] = NULL; 1018 return -1; 1019 } 1020 1021 oct->num_oqs++; 1022 1023 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 1024 oct->num_oqs); 1025 1026 /* Global Droq register settings */ 1027 1028 /* As of now not required, as setting are done for all 32 Droqs at 1029 * the same time. 1030 */ 1031 return 0; 1032 } 1033