1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2015 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more 17 * details. 18 * 19 * This file may also be available under a different license from Cavium. 20 * Contact Cavium, Inc. for more information 21 **********************************************************************/ 22 #include <linux/pci.h> 23 #include <linux/netdevice.h> 24 #include <linux/vmalloc.h> 25 #include "liquidio_common.h" 26 #include "octeon_droq.h" 27 #include "octeon_iq.h" 28 #include "response_manager.h" 29 #include "octeon_device.h" 30 #include "octeon_main.h" 31 #include "octeon_network.h" 32 #include "cn66xx_regs.h" 33 #include "cn66xx_device.h" 34 35 #define CVM_MIN(d1, d2) (((d1) < (d2)) ? (d1) : (d2)) 36 #define CVM_MAX(d1, d2) (((d1) > (d2)) ? (d1) : (d2)) 37 38 struct niclist { 39 struct list_head list; 40 void *ptr; 41 }; 42 43 struct __dispatch { 44 struct list_head list; 45 struct octeon_recv_info *rinfo; 46 octeon_dispatch_fn_t disp_fn; 47 }; 48 49 /** Get the argument that the user set when registering dispatch 50 * function for a given opcode/subcode. 51 * @param octeon_dev - the octeon device pointer. 52 * @param opcode - the opcode for which the dispatch argument 53 * is to be checked. 54 * @param subcode - the subcode for which the dispatch argument 55 * is to be checked. 56 * @return Success: void * (argument to the dispatch function) 57 * @return Failure: NULL 58 * 59 */ 60 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 61 u16 opcode, u16 subcode) 62 { 63 int idx; 64 struct list_head *dispatch; 65 void *fn_arg = NULL; 66 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 67 68 idx = combined_opcode & OCTEON_OPCODE_MASK; 69 70 spin_lock_bh(&octeon_dev->dispatch.lock); 71 72 if (octeon_dev->dispatch.count == 0) { 73 spin_unlock_bh(&octeon_dev->dispatch.lock); 74 return NULL; 75 } 76 77 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 78 fn_arg = octeon_dev->dispatch.dlist[idx].arg; 79 } else { 80 list_for_each(dispatch, 81 &octeon_dev->dispatch.dlist[idx].list) { 82 if (((struct octeon_dispatch *)dispatch)->opcode == 83 combined_opcode) { 84 fn_arg = ((struct octeon_dispatch *) 85 dispatch)->arg; 86 break; 87 } 88 } 89 } 90 91 spin_unlock_bh(&octeon_dev->dispatch.lock); 92 return fn_arg; 93 } 94 95 /** Check for packets on Droq. This function should be called with 96 * lock held. 97 * @param droq - Droq on which count is checked. 98 * @return Returns packet count. 99 */ 100 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) 101 { 102 u32 pkt_count = 0; 103 104 pkt_count = readl(droq->pkts_sent_reg); 105 if (pkt_count) { 106 atomic_add(pkt_count, &droq->pkts_pending); 107 writel(pkt_count, droq->pkts_sent_reg); 108 } 109 110 return pkt_count; 111 } 112 113 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 114 { 115 u32 count = 0; 116 117 /* max_empty_descs is the max. no. of descs that can have no buffers. 118 * If the empty desc count goes beyond this value, we cannot safely 119 * read in a 64K packet sent by Octeon 120 * (64K is max pkt size from Octeon) 121 */ 122 droq->max_empty_descs = 0; 123 124 do { 125 droq->max_empty_descs++; 126 count += droq->buffer_size; 127 } while (count < (64 * 1024)); 128 129 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 130 } 131 132 static void octeon_droq_reset_indices(struct octeon_droq *droq) 133 { 134 droq->read_idx = 0; 135 droq->write_idx = 0; 136 droq->refill_idx = 0; 137 droq->refill_count = 0; 138 atomic_set(&droq->pkts_pending, 0); 139 } 140 141 static void 142 octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 143 struct octeon_droq *droq) 144 { 145 u32 i; 146 struct octeon_skb_page_info *pg_info; 147 148 for (i = 0; i < droq->max_count; i++) { 149 pg_info = &droq->recv_buf_list[i].pg_info; 150 151 if (pg_info->dma) 152 lio_unmap_ring(oct->pci_dev, 153 (u64)pg_info->dma); 154 pg_info->dma = 0; 155 156 if (pg_info->page) 157 recv_buffer_destroy(droq->recv_buf_list[i].buffer, 158 pg_info); 159 160 if (droq->desc_ring && droq->desc_ring[i].info_ptr) 161 lio_unmap_ring_info(oct->pci_dev, 162 (u64)droq-> 163 desc_ring[i].info_ptr, 164 OCT_DROQ_INFO_SIZE); 165 droq->recv_buf_list[i].buffer = NULL; 166 } 167 168 octeon_droq_reset_indices(droq); 169 } 170 171 static int 172 octeon_droq_setup_ring_buffers(struct octeon_device *oct, 173 struct octeon_droq *droq) 174 { 175 u32 i; 176 void *buf; 177 struct octeon_droq_desc *desc_ring = droq->desc_ring; 178 179 for (i = 0; i < droq->max_count; i++) { 180 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); 181 182 if (!buf) { 183 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 184 __func__); 185 droq->stats.rx_alloc_failure++; 186 return -ENOMEM; 187 } 188 189 droq->recv_buf_list[i].buffer = buf; 190 droq->recv_buf_list[i].data = get_rbd(buf); 191 droq->info_list[i].length = 0; 192 193 /* map ring buffers into memory */ 194 desc_ring[i].info_ptr = lio_map_ring_info(droq, i); 195 desc_ring[i].buffer_ptr = 196 lio_map_ring(droq->recv_buf_list[i].buffer); 197 } 198 199 octeon_droq_reset_indices(droq); 200 201 octeon_droq_compute_max_packet_bufs(droq); 202 203 return 0; 204 } 205 206 int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 207 { 208 struct octeon_droq *droq = oct->droq[q_no]; 209 210 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 211 212 octeon_droq_destroy_ring_buffers(oct, droq); 213 vfree(droq->recv_buf_list); 214 215 if (droq->info_base_addr) 216 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list, 217 droq->info_alloc_size, 218 droq->info_base_addr, 219 droq->info_list_dma); 220 221 if (droq->desc_ring) 222 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 223 droq->desc_ring, droq->desc_ring_dma); 224 225 memset(droq, 0, OCT_DROQ_SIZE); 226 227 return 0; 228 } 229 230 int octeon_init_droq(struct octeon_device *oct, 231 u32 q_no, 232 u32 num_descs, 233 u32 desc_size, 234 void *app_ctx) 235 { 236 struct octeon_droq *droq; 237 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 238 u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 239 int orig_node = dev_to_node(&oct->pci_dev->dev); 240 int numa_node = cpu_to_node(q_no % num_online_cpus()); 241 242 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 243 244 droq = oct->droq[q_no]; 245 memset(droq, 0, OCT_DROQ_SIZE); 246 247 droq->oct_dev = oct; 248 droq->q_no = q_no; 249 if (app_ctx) 250 droq->app_ctx = app_ctx; 251 else 252 droq->app_ctx = (void *)(size_t)q_no; 253 254 c_num_descs = num_descs; 255 c_buf_size = desc_size; 256 if (OCTEON_CN6XXX(oct)) { 257 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf); 258 259 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 260 c_refill_threshold = 261 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 262 } else { 263 return 1; 264 } 265 266 droq->max_count = c_num_descs; 267 droq->buffer_size = c_buf_size; 268 269 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 270 set_dev_node(&oct->pci_dev->dev, numa_node); 271 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 272 (dma_addr_t *)&droq->desc_ring_dma); 273 set_dev_node(&oct->pci_dev->dev, orig_node); 274 if (!droq->desc_ring) 275 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 276 (dma_addr_t *)&droq->desc_ring_dma); 277 278 if (!droq->desc_ring) { 279 dev_err(&oct->pci_dev->dev, 280 "Output queue %d ring alloc failed\n", q_no); 281 return 1; 282 } 283 284 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 285 q_no, droq->desc_ring, droq->desc_ring_dma); 286 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 287 droq->max_count); 288 289 droq->info_list = 290 cnnic_numa_alloc_aligned_dma((droq->max_count * 291 OCT_DROQ_INFO_SIZE), 292 &droq->info_alloc_size, 293 &droq->info_base_addr, 294 numa_node); 295 if (!droq->info_list) { 296 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n"); 297 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 298 droq->desc_ring, droq->desc_ring_dma); 299 return 1; 300 } 301 302 droq->recv_buf_list = (struct octeon_recv_buffer *) 303 vmalloc_node(droq->max_count * 304 OCT_DROQ_RECVBUF_SIZE, 305 numa_node); 306 if (!droq->recv_buf_list) 307 droq->recv_buf_list = (struct octeon_recv_buffer *) 308 vmalloc(droq->max_count * 309 OCT_DROQ_RECVBUF_SIZE); 310 if (!droq->recv_buf_list) { 311 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 312 goto init_droq_fail; 313 } 314 315 if (octeon_droq_setup_ring_buffers(oct, droq)) 316 goto init_droq_fail; 317 318 droq->pkts_per_intr = c_pkts_per_intr; 319 droq->refill_threshold = c_refill_threshold; 320 321 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 322 droq->max_empty_descs); 323 324 spin_lock_init(&droq->lock); 325 326 INIT_LIST_HEAD(&droq->dispatch_list); 327 328 /* For 56xx Pass1, this function won't be called, so no checks. */ 329 oct->fn_list.setup_oq_regs(oct, q_no); 330 331 oct->io_qmask.oq |= (1ULL << q_no); 332 333 return 0; 334 335 init_droq_fail: 336 octeon_delete_droq(oct, q_no); 337 return 1; 338 } 339 340 /* octeon_create_recv_info 341 * Parameters: 342 * octeon_dev - pointer to the octeon device structure 343 * droq - droq in which the packet arrived. 344 * buf_cnt - no. of buffers used by the packet. 345 * idx - index in the descriptor for the first buffer in the packet. 346 * Description: 347 * Allocates a recv_info_t and copies the buffer addresses for packet data 348 * into the recv_pkt space which starts at an 8B offset from recv_info_t. 349 * Flags the descriptors for refill later. If available descriptors go 350 * below the threshold to receive a 64K pkt, new buffers are first allocated 351 * before the recv_pkt_t is created. 352 * This routine will be called in interrupt context. 353 * Returns: 354 * Success: Pointer to recv_info_t 355 * Failure: NULL. 356 * Locks: 357 * The droq->lock is held when this routine is called. 358 */ 359 static inline struct octeon_recv_info *octeon_create_recv_info( 360 struct octeon_device *octeon_dev, 361 struct octeon_droq *droq, 362 u32 buf_cnt, 363 u32 idx) 364 { 365 struct octeon_droq_info *info; 366 struct octeon_recv_pkt *recv_pkt; 367 struct octeon_recv_info *recv_info; 368 u32 i, bytes_left; 369 struct octeon_skb_page_info *pg_info; 370 371 info = &droq->info_list[idx]; 372 373 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 374 if (!recv_info) 375 return NULL; 376 377 recv_pkt = recv_info->recv_pkt; 378 recv_pkt->rh = info->rh; 379 recv_pkt->length = (u32)info->length; 380 recv_pkt->buffer_count = (u16)buf_cnt; 381 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 382 383 i = 0; 384 bytes_left = (u32)info->length; 385 386 while (buf_cnt) { 387 { 388 pg_info = &droq->recv_buf_list[idx].pg_info; 389 390 lio_unmap_ring(octeon_dev->pci_dev, 391 (u64)pg_info->dma); 392 pg_info->page = NULL; 393 pg_info->dma = 0; 394 } 395 396 recv_pkt->buffer_size[i] = 397 (bytes_left >= 398 droq->buffer_size) ? droq->buffer_size : bytes_left; 399 400 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 401 droq->recv_buf_list[idx].buffer = NULL; 402 403 INCR_INDEX_BY1(idx, droq->max_count); 404 bytes_left -= droq->buffer_size; 405 i++; 406 buf_cnt--; 407 } 408 409 return recv_info; 410 } 411 412 /* If we were not able to refill all buffers, try to move around 413 * the buffers that were not dispatched. 414 */ 415 static inline u32 416 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 417 struct octeon_droq_desc *desc_ring) 418 { 419 u32 desc_refilled = 0; 420 421 u32 refill_index = droq->refill_idx; 422 423 while (refill_index != droq->read_idx) { 424 if (droq->recv_buf_list[refill_index].buffer) { 425 droq->recv_buf_list[droq->refill_idx].buffer = 426 droq->recv_buf_list[refill_index].buffer; 427 droq->recv_buf_list[droq->refill_idx].data = 428 droq->recv_buf_list[refill_index].data; 429 desc_ring[droq->refill_idx].buffer_ptr = 430 desc_ring[refill_index].buffer_ptr; 431 droq->recv_buf_list[refill_index].buffer = NULL; 432 desc_ring[refill_index].buffer_ptr = 0; 433 do { 434 INCR_INDEX_BY1(droq->refill_idx, 435 droq->max_count); 436 desc_refilled++; 437 droq->refill_count--; 438 } while (droq->recv_buf_list[droq->refill_idx]. 439 buffer); 440 } 441 INCR_INDEX_BY1(refill_index, droq->max_count); 442 } /* while */ 443 return desc_refilled; 444 } 445 446 /* octeon_droq_refill 447 * Parameters: 448 * droq - droq in which descriptors require new buffers. 449 * Description: 450 * Called during normal DROQ processing in interrupt mode or by the poll 451 * thread to refill the descriptors from which buffers were dispatched 452 * to upper layers. Attempts to allocate new buffers. If that fails, moves 453 * up buffers (that were not dispatched) to form a contiguous ring. 454 * Returns: 455 * No of descriptors refilled. 456 * Locks: 457 * This routine is called with droq->lock held. 458 */ 459 static u32 460 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 461 { 462 struct octeon_droq_desc *desc_ring; 463 void *buf = NULL; 464 u8 *data; 465 u32 desc_refilled = 0; 466 struct octeon_skb_page_info *pg_info; 467 468 desc_ring = droq->desc_ring; 469 470 while (droq->refill_count && (desc_refilled < droq->max_count)) { 471 /* If a valid buffer exists (happens if there is no dispatch), 472 * reuse 473 * the buffer, else allocate. 474 */ 475 if (!droq->recv_buf_list[droq->refill_idx].buffer) { 476 pg_info = 477 &droq->recv_buf_list[droq->refill_idx].pg_info; 478 /* Either recycle the existing pages or go for 479 * new page alloc 480 */ 481 if (pg_info->page) 482 buf = recv_buffer_reuse(octeon_dev, pg_info); 483 else 484 buf = recv_buffer_alloc(octeon_dev, pg_info); 485 /* If a buffer could not be allocated, no point in 486 * continuing 487 */ 488 if (!buf) { 489 droq->stats.rx_alloc_failure++; 490 break; 491 } 492 droq->recv_buf_list[droq->refill_idx].buffer = 493 buf; 494 data = get_rbd(buf); 495 } else { 496 data = get_rbd(droq->recv_buf_list 497 [droq->refill_idx].buffer); 498 } 499 500 droq->recv_buf_list[droq->refill_idx].data = data; 501 502 desc_ring[droq->refill_idx].buffer_ptr = 503 lio_map_ring(droq->recv_buf_list[droq-> 504 refill_idx].buffer); 505 /* Reset any previous values in the length field. */ 506 droq->info_list[droq->refill_idx].length = 0; 507 508 INCR_INDEX_BY1(droq->refill_idx, droq->max_count); 509 desc_refilled++; 510 droq->refill_count--; 511 } 512 513 if (droq->refill_count) 514 desc_refilled += 515 octeon_droq_refill_pullup_descs(droq, desc_ring); 516 517 /* if droq->refill_count 518 * The refill count would not change in pass two. We only moved buffers 519 * to close the gap in the ring, but we would still have the same no. of 520 * buffers to refill. 521 */ 522 return desc_refilled; 523 } 524 525 static inline u32 526 octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 527 { 528 u32 buf_cnt = 0; 529 530 while (total_len > (buf_size * buf_cnt)) 531 buf_cnt++; 532 return buf_cnt; 533 } 534 535 static int 536 octeon_droq_dispatch_pkt(struct octeon_device *oct, 537 struct octeon_droq *droq, 538 union octeon_rh *rh, 539 struct octeon_droq_info *info) 540 { 541 u32 cnt; 542 octeon_dispatch_fn_t disp_fn; 543 struct octeon_recv_info *rinfo; 544 545 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 546 547 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 548 (u16)rh->r.subcode); 549 if (disp_fn) { 550 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 551 if (rinfo) { 552 struct __dispatch *rdisp = rinfo->rsvd; 553 554 rdisp->rinfo = rinfo; 555 rdisp->disp_fn = disp_fn; 556 rinfo->recv_pkt->rh = *rh; 557 list_add_tail(&rdisp->list, 558 &droq->dispatch_list); 559 } else { 560 droq->stats.dropped_nomem++; 561 } 562 } else { 563 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", 564 (unsigned int)rh->r.opcode, 565 (unsigned int)rh->r.subcode); 566 droq->stats.dropped_nodispatch++; 567 } /* else (dispatch_fn ... */ 568 569 return cnt; 570 } 571 572 static inline void octeon_droq_drop_packets(struct octeon_device *oct, 573 struct octeon_droq *droq, 574 u32 cnt) 575 { 576 u32 i = 0, buf_cnt; 577 struct octeon_droq_info *info; 578 579 for (i = 0; i < cnt; i++) { 580 info = &droq->info_list[droq->read_idx]; 581 octeon_swap_8B_data((u64 *)info, 2); 582 583 if (info->length) { 584 info->length -= OCT_RH_SIZE; 585 droq->stats.bytes_received += info->length; 586 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 587 (u32)info->length); 588 } else { 589 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 590 buf_cnt = 1; 591 } 592 593 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); 594 droq->refill_count += buf_cnt; 595 } 596 } 597 598 static u32 599 octeon_droq_fast_process_packets(struct octeon_device *oct, 600 struct octeon_droq *droq, 601 u32 pkts_to_process) 602 { 603 struct octeon_droq_info *info; 604 union octeon_rh *rh; 605 u32 pkt, total_len = 0, pkt_count; 606 607 pkt_count = pkts_to_process; 608 609 for (pkt = 0; pkt < pkt_count; pkt++) { 610 u32 pkt_len = 0; 611 struct sk_buff *nicbuf = NULL; 612 struct octeon_skb_page_info *pg_info; 613 void *buf; 614 615 info = &droq->info_list[droq->read_idx]; 616 octeon_swap_8B_data((u64 *)info, 2); 617 618 if (!info->length) { 619 dev_err(&oct->pci_dev->dev, 620 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 621 droq->q_no, droq->read_idx, pkt_count); 622 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 623 (u8 *)info, 624 OCT_DROQ_INFO_SIZE); 625 break; 626 } 627 628 /* Len of resp hdr in included in the received data len. */ 629 info->length -= OCT_RH_SIZE; 630 rh = &info->rh; 631 632 total_len += (u32)info->length; 633 if (OPCODE_SLOW_PATH(rh)) { 634 u32 buf_cnt; 635 636 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 637 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); 638 droq->refill_count += buf_cnt; 639 } else { 640 if (info->length <= droq->buffer_size) { 641 pkt_len = (u32)info->length; 642 nicbuf = droq->recv_buf_list[ 643 droq->read_idx].buffer; 644 pg_info = &droq->recv_buf_list[ 645 droq->read_idx].pg_info; 646 if (recv_buffer_recycle(oct, pg_info)) 647 pg_info->page = NULL; 648 droq->recv_buf_list[droq->read_idx].buffer = 649 NULL; 650 651 INCR_INDEX_BY1(droq->read_idx, droq->max_count); 652 droq->refill_count++; 653 } else { 654 nicbuf = octeon_fast_packet_alloc((u32) 655 info->length); 656 pkt_len = 0; 657 /* nicbuf allocation can fail. We'll handle it 658 * inside the loop. 659 */ 660 while (pkt_len < info->length) { 661 int cpy_len, idx = droq->read_idx; 662 663 cpy_len = ((pkt_len + droq->buffer_size) 664 > info->length) ? 665 ((u32)info->length - pkt_len) : 666 droq->buffer_size; 667 668 if (nicbuf) { 669 octeon_fast_packet_next(droq, 670 nicbuf, 671 cpy_len, 672 idx); 673 buf = droq->recv_buf_list[idx]. 674 buffer; 675 recv_buffer_fast_free(buf); 676 droq->recv_buf_list[idx].buffer 677 = NULL; 678 } else { 679 droq->stats.rx_alloc_failure++; 680 } 681 682 pkt_len += cpy_len; 683 INCR_INDEX_BY1(droq->read_idx, 684 droq->max_count); 685 droq->refill_count++; 686 } 687 } 688 689 if (nicbuf) { 690 if (droq->ops.fptr) { 691 droq->ops.fptr(oct->octeon_id, 692 nicbuf, pkt_len, 693 rh, &droq->napi, 694 droq->ops.farg); 695 } else { 696 recv_buffer_free(nicbuf); 697 } 698 } 699 } 700 701 if (droq->refill_count >= droq->refill_threshold) { 702 int desc_refilled = octeon_droq_refill(oct, droq); 703 704 /* Flush the droq descriptor data to memory to be sure 705 * that when we update the credits the data in memory 706 * is accurate. 707 */ 708 wmb(); 709 writel((desc_refilled), droq->pkts_credit_reg); 710 /* make sure mmio write completes */ 711 mmiowb(); 712 } 713 714 } /* for (each packet)... */ 715 716 /* Increment refill_count by the number of buffers processed. */ 717 droq->stats.pkts_received += pkt; 718 droq->stats.bytes_received += total_len; 719 720 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 721 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 722 723 droq->stats.dropped_toomany += (pkts_to_process - pkt); 724 return pkts_to_process; 725 } 726 727 return pkt; 728 } 729 730 int 731 octeon_droq_process_packets(struct octeon_device *oct, 732 struct octeon_droq *droq, 733 u32 budget) 734 { 735 u32 pkt_count = 0, pkts_processed = 0; 736 struct list_head *tmp, *tmp2; 737 738 pkt_count = atomic_read(&droq->pkts_pending); 739 if (!pkt_count) 740 return 0; 741 742 if (pkt_count > budget) 743 pkt_count = budget; 744 745 /* Grab the droq lock */ 746 spin_lock(&droq->lock); 747 748 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); 749 750 atomic_sub(pkts_processed, &droq->pkts_pending); 751 752 /* Release the spin lock */ 753 spin_unlock(&droq->lock); 754 755 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 756 struct __dispatch *rdisp = (struct __dispatch *)tmp; 757 758 list_del(tmp); 759 rdisp->disp_fn(rdisp->rinfo, 760 octeon_get_dispatch_arg 761 (oct, 762 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 763 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 764 } 765 766 /* If there are packets pending. schedule tasklet again */ 767 if (atomic_read(&droq->pkts_pending)) 768 return 1; 769 770 return 0; 771 } 772 773 /** 774 * Utility function to poll for packets. check_hw_for_packets must be 775 * called before calling this routine. 776 */ 777 778 static int 779 octeon_droq_process_poll_pkts(struct octeon_device *oct, 780 struct octeon_droq *droq, u32 budget) 781 { 782 struct list_head *tmp, *tmp2; 783 u32 pkts_available = 0, pkts_processed = 0; 784 u32 total_pkts_processed = 0; 785 786 if (budget > droq->max_count) 787 budget = droq->max_count; 788 789 spin_lock(&droq->lock); 790 791 while (total_pkts_processed < budget) { 792 pkts_available = 793 CVM_MIN((budget - total_pkts_processed), 794 (u32)(atomic_read(&droq->pkts_pending))); 795 796 if (pkts_available == 0) 797 break; 798 799 pkts_processed = 800 octeon_droq_fast_process_packets(oct, droq, 801 pkts_available); 802 803 atomic_sub(pkts_processed, &droq->pkts_pending); 804 805 total_pkts_processed += pkts_processed; 806 807 octeon_droq_check_hw_for_pkts(droq); 808 } 809 810 spin_unlock(&droq->lock); 811 812 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 813 struct __dispatch *rdisp = (struct __dispatch *)tmp; 814 815 list_del(tmp); 816 rdisp->disp_fn(rdisp->rinfo, 817 octeon_get_dispatch_arg 818 (oct, 819 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 820 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 821 } 822 823 return total_pkts_processed; 824 } 825 826 int 827 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, 828 u32 arg) 829 { 830 struct octeon_droq *droq; 831 832 droq = oct->droq[q_no]; 833 834 if (cmd == POLL_EVENT_PROCESS_PKTS) 835 return octeon_droq_process_poll_pkts(oct, droq, arg); 836 837 if (cmd == POLL_EVENT_PENDING_PKTS) { 838 u32 pkt_cnt = atomic_read(&droq->pkts_pending); 839 840 return octeon_droq_process_packets(oct, droq, pkt_cnt); 841 } 842 843 if (cmd == POLL_EVENT_ENABLE_INTR) { 844 u32 value; 845 unsigned long flags; 846 847 /* Enable Pkt Interrupt */ 848 switch (oct->chip_id) { 849 case OCTEON_CN66XX: 850 case OCTEON_CN68XX: { 851 struct octeon_cn6xxx *cn6xxx = 852 (struct octeon_cn6xxx *)oct->chip; 853 spin_lock_irqsave 854 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 855 value = 856 octeon_read_csr(oct, 857 CN6XXX_SLI_PKT_TIME_INT_ENB); 858 value |= (1 << q_no); 859 octeon_write_csr(oct, 860 CN6XXX_SLI_PKT_TIME_INT_ENB, 861 value); 862 value = 863 octeon_read_csr(oct, 864 CN6XXX_SLI_PKT_CNT_INT_ENB); 865 value |= (1 << q_no); 866 octeon_write_csr(oct, 867 CN6XXX_SLI_PKT_CNT_INT_ENB, 868 value); 869 870 /* don't bother flushing the enables */ 871 872 spin_unlock_irqrestore 873 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 874 return 0; 875 } 876 break; 877 } 878 879 return 0; 880 } 881 882 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); 883 return -EINVAL; 884 } 885 886 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 887 struct octeon_droq_ops *ops) 888 { 889 struct octeon_droq *droq; 890 unsigned long flags; 891 struct octeon_config *oct_cfg = NULL; 892 893 oct_cfg = octeon_get_conf(oct); 894 895 if (!oct_cfg) 896 return -EINVAL; 897 898 if (!(ops)) { 899 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 900 __func__); 901 return -EINVAL; 902 } 903 904 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 905 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 906 __func__, q_no, (oct->num_oqs - 1)); 907 return -EINVAL; 908 } 909 910 droq = oct->droq[q_no]; 911 912 spin_lock_irqsave(&droq->lock, flags); 913 914 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 915 916 spin_unlock_irqrestore(&droq->lock, flags); 917 918 return 0; 919 } 920 921 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 922 { 923 unsigned long flags; 924 struct octeon_droq *droq; 925 struct octeon_config *oct_cfg = NULL; 926 927 oct_cfg = octeon_get_conf(oct); 928 929 if (!oct_cfg) 930 return -EINVAL; 931 932 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 933 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 934 __func__, q_no, oct->num_oqs - 1); 935 return -EINVAL; 936 } 937 938 droq = oct->droq[q_no]; 939 940 if (!droq) { 941 dev_info(&oct->pci_dev->dev, 942 "Droq id (%d) not available.\n", q_no); 943 return 0; 944 } 945 946 spin_lock_irqsave(&droq->lock, flags); 947 948 droq->ops.fptr = NULL; 949 droq->ops.farg = NULL; 950 droq->ops.drop_on_max = 0; 951 952 spin_unlock_irqrestore(&droq->lock, flags); 953 954 return 0; 955 } 956 957 int octeon_create_droq(struct octeon_device *oct, 958 u32 q_no, u32 num_descs, 959 u32 desc_size, void *app_ctx) 960 { 961 struct octeon_droq *droq; 962 int numa_node = cpu_to_node(q_no % num_online_cpus()); 963 964 if (oct->droq[q_no]) { 965 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 966 q_no); 967 return 1; 968 } 969 970 /* Allocate the DS for the new droq. */ 971 droq = vmalloc_node(sizeof(*droq), numa_node); 972 if (!droq) 973 droq = vmalloc(sizeof(*droq)); 974 if (!droq) 975 goto create_droq_fail; 976 memset(droq, 0, sizeof(struct octeon_droq)); 977 978 /*Disable the pkt o/p for this Q */ 979 octeon_set_droq_pkt_op(oct, q_no, 0); 980 oct->droq[q_no] = droq; 981 982 /* Initialize the Droq */ 983 octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx); 984 985 oct->num_oqs++; 986 987 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 988 oct->num_oqs); 989 990 /* Global Droq register settings */ 991 992 /* As of now not required, as setting are done for all 32 Droqs at 993 * the same time. 994 */ 995 return 0; 996 997 create_droq_fail: 998 octeon_delete_droq(oct, q_no); 999 return -ENOMEM; 1000 } 1001