1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2015 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more 17 * details. 18 * 19 * This file may also be available under a different license from Cavium. 20 * Contact Cavium, Inc. for more information 21 **********************************************************************/ 22 #include <linux/pci.h> 23 #include <linux/netdevice.h> 24 #include <linux/vmalloc.h> 25 #include "liquidio_common.h" 26 #include "octeon_droq.h" 27 #include "octeon_iq.h" 28 #include "response_manager.h" 29 #include "octeon_device.h" 30 #include "octeon_main.h" 31 #include "octeon_network.h" 32 #include "cn66xx_regs.h" 33 #include "cn66xx_device.h" 34 #include "cn23xx_pf_device.h" 35 36 #define CVM_MIN(d1, d2) (((d1) < (d2)) ? (d1) : (d2)) 37 #define CVM_MAX(d1, d2) (((d1) > (d2)) ? (d1) : (d2)) 38 39 struct niclist { 40 struct list_head list; 41 void *ptr; 42 }; 43 44 struct __dispatch { 45 struct list_head list; 46 struct octeon_recv_info *rinfo; 47 octeon_dispatch_fn_t disp_fn; 48 }; 49 50 /** Get the argument that the user set when registering dispatch 51 * function for a given opcode/subcode. 52 * @param octeon_dev - the octeon device pointer. 53 * @param opcode - the opcode for which the dispatch argument 54 * is to be checked. 55 * @param subcode - the subcode for which the dispatch argument 56 * is to be checked. 57 * @return Success: void * (argument to the dispatch function) 58 * @return Failure: NULL 59 * 60 */ 61 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 62 u16 opcode, u16 subcode) 63 { 64 int idx; 65 struct list_head *dispatch; 66 void *fn_arg = NULL; 67 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 68 69 idx = combined_opcode & OCTEON_OPCODE_MASK; 70 71 spin_lock_bh(&octeon_dev->dispatch.lock); 72 73 if (octeon_dev->dispatch.count == 0) { 74 spin_unlock_bh(&octeon_dev->dispatch.lock); 75 return NULL; 76 } 77 78 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 79 fn_arg = octeon_dev->dispatch.dlist[idx].arg; 80 } else { 81 list_for_each(dispatch, 82 &octeon_dev->dispatch.dlist[idx].list) { 83 if (((struct octeon_dispatch *)dispatch)->opcode == 84 combined_opcode) { 85 fn_arg = ((struct octeon_dispatch *) 86 dispatch)->arg; 87 break; 88 } 89 } 90 } 91 92 spin_unlock_bh(&octeon_dev->dispatch.lock); 93 return fn_arg; 94 } 95 96 /** Check for packets on Droq. This function should be called with lock held. 97 * @param droq - Droq on which count is checked. 98 * @return Returns packet count. 99 */ 100 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) 101 { 102 u32 pkt_count = 0; 103 u32 last_count; 104 105 pkt_count = readl(droq->pkts_sent_reg); 106 107 last_count = pkt_count - droq->pkt_count; 108 droq->pkt_count = pkt_count; 109 110 /* we shall write to cnts at napi irq enable or end of droq tasklet */ 111 if (last_count) 112 atomic_add(last_count, &droq->pkts_pending); 113 114 return last_count; 115 } 116 117 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 118 { 119 u32 count = 0; 120 121 /* max_empty_descs is the max. no. of descs that can have no buffers. 122 * If the empty desc count goes beyond this value, we cannot safely 123 * read in a 64K packet sent by Octeon 124 * (64K is max pkt size from Octeon) 125 */ 126 droq->max_empty_descs = 0; 127 128 do { 129 droq->max_empty_descs++; 130 count += droq->buffer_size; 131 } while (count < (64 * 1024)); 132 133 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 134 } 135 136 static void octeon_droq_reset_indices(struct octeon_droq *droq) 137 { 138 droq->read_idx = 0; 139 droq->write_idx = 0; 140 droq->refill_idx = 0; 141 droq->refill_count = 0; 142 atomic_set(&droq->pkts_pending, 0); 143 } 144 145 static void 146 octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 147 struct octeon_droq *droq) 148 { 149 u32 i; 150 struct octeon_skb_page_info *pg_info; 151 152 for (i = 0; i < droq->max_count; i++) { 153 pg_info = &droq->recv_buf_list[i].pg_info; 154 155 if (pg_info->dma) 156 lio_unmap_ring(oct->pci_dev, 157 (u64)pg_info->dma); 158 pg_info->dma = 0; 159 160 if (pg_info->page) 161 recv_buffer_destroy(droq->recv_buf_list[i].buffer, 162 pg_info); 163 164 if (droq->desc_ring && droq->desc_ring[i].info_ptr) 165 lio_unmap_ring_info(oct->pci_dev, 166 (u64)droq-> 167 desc_ring[i].info_ptr, 168 OCT_DROQ_INFO_SIZE); 169 droq->recv_buf_list[i].buffer = NULL; 170 } 171 172 octeon_droq_reset_indices(droq); 173 } 174 175 static int 176 octeon_droq_setup_ring_buffers(struct octeon_device *oct, 177 struct octeon_droq *droq) 178 { 179 u32 i; 180 void *buf; 181 struct octeon_droq_desc *desc_ring = droq->desc_ring; 182 183 for (i = 0; i < droq->max_count; i++) { 184 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); 185 186 if (!buf) { 187 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 188 __func__); 189 droq->stats.rx_alloc_failure++; 190 return -ENOMEM; 191 } 192 193 droq->recv_buf_list[i].buffer = buf; 194 droq->recv_buf_list[i].data = get_rbd(buf); 195 droq->info_list[i].length = 0; 196 197 /* map ring buffers into memory */ 198 desc_ring[i].info_ptr = lio_map_ring_info(droq, i); 199 desc_ring[i].buffer_ptr = 200 lio_map_ring(droq->recv_buf_list[i].buffer); 201 } 202 203 octeon_droq_reset_indices(droq); 204 205 octeon_droq_compute_max_packet_bufs(droq); 206 207 return 0; 208 } 209 210 int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 211 { 212 struct octeon_droq *droq = oct->droq[q_no]; 213 214 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 215 216 octeon_droq_destroy_ring_buffers(oct, droq); 217 vfree(droq->recv_buf_list); 218 219 if (droq->info_base_addr) 220 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list, 221 droq->info_alloc_size, 222 droq->info_base_addr, 223 droq->info_list_dma); 224 225 if (droq->desc_ring) 226 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 227 droq->desc_ring, droq->desc_ring_dma); 228 229 memset(droq, 0, OCT_DROQ_SIZE); 230 231 return 0; 232 } 233 234 int octeon_init_droq(struct octeon_device *oct, 235 u32 q_no, 236 u32 num_descs, 237 u32 desc_size, 238 void *app_ctx) 239 { 240 struct octeon_droq *droq; 241 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 242 u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 243 int orig_node = dev_to_node(&oct->pci_dev->dev); 244 int numa_node = cpu_to_node(q_no % num_online_cpus()); 245 246 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 247 248 droq = oct->droq[q_no]; 249 memset(droq, 0, OCT_DROQ_SIZE); 250 251 droq->oct_dev = oct; 252 droq->q_no = q_no; 253 if (app_ctx) 254 droq->app_ctx = app_ctx; 255 else 256 droq->app_ctx = (void *)(size_t)q_no; 257 258 c_num_descs = num_descs; 259 c_buf_size = desc_size; 260 if (OCTEON_CN6XXX(oct)) { 261 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf); 262 263 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 264 c_refill_threshold = 265 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 266 } else if (OCTEON_CN23XX_PF(oct)) { 267 struct octeon_config *conf23 = CHIP_FIELD(oct, cn23xx_pf, conf); 268 269 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 270 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 271 } else { 272 return 1; 273 } 274 275 droq->max_count = c_num_descs; 276 droq->buffer_size = c_buf_size; 277 278 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 279 set_dev_node(&oct->pci_dev->dev, numa_node); 280 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 281 (dma_addr_t *)&droq->desc_ring_dma); 282 set_dev_node(&oct->pci_dev->dev, orig_node); 283 if (!droq->desc_ring) 284 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 285 (dma_addr_t *)&droq->desc_ring_dma); 286 287 if (!droq->desc_ring) { 288 dev_err(&oct->pci_dev->dev, 289 "Output queue %d ring alloc failed\n", q_no); 290 return 1; 291 } 292 293 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 294 q_no, droq->desc_ring, droq->desc_ring_dma); 295 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 296 droq->max_count); 297 298 droq->info_list = 299 cnnic_numa_alloc_aligned_dma((droq->max_count * 300 OCT_DROQ_INFO_SIZE), 301 &droq->info_alloc_size, 302 &droq->info_base_addr, 303 numa_node); 304 if (!droq->info_list) { 305 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n"); 306 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 307 droq->desc_ring, droq->desc_ring_dma); 308 return 1; 309 } 310 311 droq->recv_buf_list = (struct octeon_recv_buffer *) 312 vmalloc_node(droq->max_count * 313 OCT_DROQ_RECVBUF_SIZE, 314 numa_node); 315 if (!droq->recv_buf_list) 316 droq->recv_buf_list = (struct octeon_recv_buffer *) 317 vmalloc(droq->max_count * 318 OCT_DROQ_RECVBUF_SIZE); 319 if (!droq->recv_buf_list) { 320 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 321 goto init_droq_fail; 322 } 323 324 if (octeon_droq_setup_ring_buffers(oct, droq)) 325 goto init_droq_fail; 326 327 droq->pkts_per_intr = c_pkts_per_intr; 328 droq->refill_threshold = c_refill_threshold; 329 330 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 331 droq->max_empty_descs); 332 333 spin_lock_init(&droq->lock); 334 335 INIT_LIST_HEAD(&droq->dispatch_list); 336 337 /* For 56xx Pass1, this function won't be called, so no checks. */ 338 oct->fn_list.setup_oq_regs(oct, q_no); 339 340 oct->io_qmask.oq |= (1ULL << q_no); 341 342 return 0; 343 344 init_droq_fail: 345 octeon_delete_droq(oct, q_no); 346 return 1; 347 } 348 349 /* octeon_create_recv_info 350 * Parameters: 351 * octeon_dev - pointer to the octeon device structure 352 * droq - droq in which the packet arrived. 353 * buf_cnt - no. of buffers used by the packet. 354 * idx - index in the descriptor for the first buffer in the packet. 355 * Description: 356 * Allocates a recv_info_t and copies the buffer addresses for packet data 357 * into the recv_pkt space which starts at an 8B offset from recv_info_t. 358 * Flags the descriptors for refill later. If available descriptors go 359 * below the threshold to receive a 64K pkt, new buffers are first allocated 360 * before the recv_pkt_t is created. 361 * This routine will be called in interrupt context. 362 * Returns: 363 * Success: Pointer to recv_info_t 364 * Failure: NULL. 365 * Locks: 366 * The droq->lock is held when this routine is called. 367 */ 368 static inline struct octeon_recv_info *octeon_create_recv_info( 369 struct octeon_device *octeon_dev, 370 struct octeon_droq *droq, 371 u32 buf_cnt, 372 u32 idx) 373 { 374 struct octeon_droq_info *info; 375 struct octeon_recv_pkt *recv_pkt; 376 struct octeon_recv_info *recv_info; 377 u32 i, bytes_left; 378 struct octeon_skb_page_info *pg_info; 379 380 info = &droq->info_list[idx]; 381 382 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 383 if (!recv_info) 384 return NULL; 385 386 recv_pkt = recv_info->recv_pkt; 387 recv_pkt->rh = info->rh; 388 recv_pkt->length = (u32)info->length; 389 recv_pkt->buffer_count = (u16)buf_cnt; 390 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 391 392 i = 0; 393 bytes_left = (u32)info->length; 394 395 while (buf_cnt) { 396 { 397 pg_info = &droq->recv_buf_list[idx].pg_info; 398 399 lio_unmap_ring(octeon_dev->pci_dev, 400 (u64)pg_info->dma); 401 pg_info->page = NULL; 402 pg_info->dma = 0; 403 } 404 405 recv_pkt->buffer_size[i] = 406 (bytes_left >= 407 droq->buffer_size) ? droq->buffer_size : bytes_left; 408 409 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 410 droq->recv_buf_list[idx].buffer = NULL; 411 412 INCR_INDEX_BY1(idx, droq->max_count); 413 bytes_left -= droq->buffer_size; 414 i++; 415 buf_cnt--; 416 } 417 418 return recv_info; 419 } 420 421 /* If we were not able to refill all buffers, try to move around 422 * the buffers that were not dispatched. 423 */ 424 static inline u32 425 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 426 struct octeon_droq_desc *desc_ring) 427 { 428 u32 desc_refilled = 0; 429 430 u32 refill_index = droq->refill_idx; 431 432 while (refill_index != droq->read_idx) { 433 if (droq->recv_buf_list[refill_index].buffer) { 434 droq->recv_buf_list[droq->refill_idx].buffer = 435 droq->recv_buf_list[refill_index].buffer; 436 droq->recv_buf_list[droq->refill_idx].data = 437 droq->recv_buf_list[refill_index].data; 438 desc_ring[droq->refill_idx].buffer_ptr = 439 desc_ring[refill_index].buffer_ptr; 440 droq->recv_buf_list[refill_index].buffer = NULL; 441 desc_ring[refill_index].buffer_ptr = 0; 442 do { 443 INCR_INDEX_BY1(droq->refill_idx, 444 droq->max_count); 445 desc_refilled++; 446 droq->refill_count--; 447 } while (droq->recv_buf_list[droq->refill_idx]. 448 buffer); 449 } 450 INCR_INDEX_BY1(refill_index, droq->max_count); 451 } /* while */ 452 return desc_refilled; 453 } 454 455 /* octeon_droq_refill 456 * Parameters: 457 * droq - droq in which descriptors require new buffers. 458 * Description: 459 * Called during normal DROQ processing in interrupt mode or by the poll 460 * thread to refill the descriptors from which buffers were dispatched 461 * to upper layers. Attempts to allocate new buffers. If that fails, moves 462 * up buffers (that were not dispatched) to form a contiguous ring. 463 * Returns: 464 * No of descriptors refilled. 465 * Locks: 466 * This routine is called with droq->lock held. 467 */ 468 static u32 469 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 470 { 471 struct octeon_droq_desc *desc_ring; 472 void *buf = NULL; 473 u8 *data; 474 u32 desc_refilled = 0; 475 struct octeon_skb_page_info *pg_info; 476 477 desc_ring = droq->desc_ring; 478 479 while (droq->refill_count && (desc_refilled < droq->max_count)) { 480 /* If a valid buffer exists (happens if there is no dispatch), 481 * reuse 482 * the buffer, else allocate. 483 */ 484 if (!droq->recv_buf_list[droq->refill_idx].buffer) { 485 pg_info = 486 &droq->recv_buf_list[droq->refill_idx].pg_info; 487 /* Either recycle the existing pages or go for 488 * new page alloc 489 */ 490 if (pg_info->page) 491 buf = recv_buffer_reuse(octeon_dev, pg_info); 492 else 493 buf = recv_buffer_alloc(octeon_dev, pg_info); 494 /* If a buffer could not be allocated, no point in 495 * continuing 496 */ 497 if (!buf) { 498 droq->stats.rx_alloc_failure++; 499 break; 500 } 501 droq->recv_buf_list[droq->refill_idx].buffer = 502 buf; 503 data = get_rbd(buf); 504 } else { 505 data = get_rbd(droq->recv_buf_list 506 [droq->refill_idx].buffer); 507 } 508 509 droq->recv_buf_list[droq->refill_idx].data = data; 510 511 desc_ring[droq->refill_idx].buffer_ptr = 512 lio_map_ring(droq->recv_buf_list[droq-> 513 refill_idx].buffer); 514 /* Reset any previous values in the length field. */ 515 droq->info_list[droq->refill_idx].length = 0; 516 517 INCR_INDEX_BY1(droq->refill_idx, droq->max_count); 518 desc_refilled++; 519 droq->refill_count--; 520 } 521 522 if (droq->refill_count) 523 desc_refilled += 524 octeon_droq_refill_pullup_descs(droq, desc_ring); 525 526 /* if droq->refill_count 527 * The refill count would not change in pass two. We only moved buffers 528 * to close the gap in the ring, but we would still have the same no. of 529 * buffers to refill. 530 */ 531 return desc_refilled; 532 } 533 534 static inline u32 535 octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 536 { 537 u32 buf_cnt = 0; 538 539 while (total_len > (buf_size * buf_cnt)) 540 buf_cnt++; 541 return buf_cnt; 542 } 543 544 static int 545 octeon_droq_dispatch_pkt(struct octeon_device *oct, 546 struct octeon_droq *droq, 547 union octeon_rh *rh, 548 struct octeon_droq_info *info) 549 { 550 u32 cnt; 551 octeon_dispatch_fn_t disp_fn; 552 struct octeon_recv_info *rinfo; 553 554 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 555 556 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 557 (u16)rh->r.subcode); 558 if (disp_fn) { 559 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 560 if (rinfo) { 561 struct __dispatch *rdisp = rinfo->rsvd; 562 563 rdisp->rinfo = rinfo; 564 rdisp->disp_fn = disp_fn; 565 rinfo->recv_pkt->rh = *rh; 566 list_add_tail(&rdisp->list, 567 &droq->dispatch_list); 568 } else { 569 droq->stats.dropped_nomem++; 570 } 571 } else { 572 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", 573 (unsigned int)rh->r.opcode, 574 (unsigned int)rh->r.subcode); 575 droq->stats.dropped_nodispatch++; 576 } 577 578 return cnt; 579 } 580 581 static inline void octeon_droq_drop_packets(struct octeon_device *oct, 582 struct octeon_droq *droq, 583 u32 cnt) 584 { 585 u32 i = 0, buf_cnt; 586 struct octeon_droq_info *info; 587 588 for (i = 0; i < cnt; i++) { 589 info = &droq->info_list[droq->read_idx]; 590 octeon_swap_8B_data((u64 *)info, 2); 591 592 if (info->length) { 593 info->length -= OCT_RH_SIZE; 594 droq->stats.bytes_received += info->length; 595 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 596 (u32)info->length); 597 } else { 598 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 599 buf_cnt = 1; 600 } 601 602 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); 603 droq->refill_count += buf_cnt; 604 } 605 } 606 607 static u32 608 octeon_droq_fast_process_packets(struct octeon_device *oct, 609 struct octeon_droq *droq, 610 u32 pkts_to_process) 611 { 612 struct octeon_droq_info *info; 613 union octeon_rh *rh; 614 u32 pkt, total_len = 0, pkt_count; 615 616 pkt_count = pkts_to_process; 617 618 for (pkt = 0; pkt < pkt_count; pkt++) { 619 u32 pkt_len = 0; 620 struct sk_buff *nicbuf = NULL; 621 struct octeon_skb_page_info *pg_info; 622 void *buf; 623 624 info = &droq->info_list[droq->read_idx]; 625 octeon_swap_8B_data((u64 *)info, 2); 626 627 if (!info->length) { 628 dev_err(&oct->pci_dev->dev, 629 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 630 droq->q_no, droq->read_idx, pkt_count); 631 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 632 (u8 *)info, 633 OCT_DROQ_INFO_SIZE); 634 break; 635 } 636 637 /* Len of resp hdr in included in the received data len. */ 638 info->length -= OCT_RH_SIZE; 639 rh = &info->rh; 640 641 total_len += (u32)info->length; 642 if (OPCODE_SLOW_PATH(rh)) { 643 u32 buf_cnt; 644 645 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 646 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); 647 droq->refill_count += buf_cnt; 648 } else { 649 if (info->length <= droq->buffer_size) { 650 pkt_len = (u32)info->length; 651 nicbuf = droq->recv_buf_list[ 652 droq->read_idx].buffer; 653 pg_info = &droq->recv_buf_list[ 654 droq->read_idx].pg_info; 655 if (recv_buffer_recycle(oct, pg_info)) 656 pg_info->page = NULL; 657 droq->recv_buf_list[droq->read_idx].buffer = 658 NULL; 659 660 INCR_INDEX_BY1(droq->read_idx, droq->max_count); 661 droq->refill_count++; 662 } else { 663 nicbuf = octeon_fast_packet_alloc((u32) 664 info->length); 665 pkt_len = 0; 666 /* nicbuf allocation can fail. We'll handle it 667 * inside the loop. 668 */ 669 while (pkt_len < info->length) { 670 int cpy_len, idx = droq->read_idx; 671 672 cpy_len = ((pkt_len + droq->buffer_size) 673 > info->length) ? 674 ((u32)info->length - pkt_len) : 675 droq->buffer_size; 676 677 if (nicbuf) { 678 octeon_fast_packet_next(droq, 679 nicbuf, 680 cpy_len, 681 idx); 682 buf = droq->recv_buf_list[idx]. 683 buffer; 684 recv_buffer_fast_free(buf); 685 droq->recv_buf_list[idx].buffer 686 = NULL; 687 } else { 688 droq->stats.rx_alloc_failure++; 689 } 690 691 pkt_len += cpy_len; 692 INCR_INDEX_BY1(droq->read_idx, 693 droq->max_count); 694 droq->refill_count++; 695 } 696 } 697 698 if (nicbuf) { 699 if (droq->ops.fptr) { 700 droq->ops.fptr(oct->octeon_id, 701 nicbuf, pkt_len, 702 rh, &droq->napi, 703 droq->ops.farg); 704 } else { 705 recv_buffer_free(nicbuf); 706 } 707 } 708 } 709 710 if (droq->refill_count >= droq->refill_threshold) { 711 int desc_refilled = octeon_droq_refill(oct, droq); 712 713 /* Flush the droq descriptor data to memory to be sure 714 * that when we update the credits the data in memory 715 * is accurate. 716 */ 717 wmb(); 718 writel((desc_refilled), droq->pkts_credit_reg); 719 /* make sure mmio write completes */ 720 mmiowb(); 721 } 722 723 } /* for (each packet)... */ 724 725 /* Increment refill_count by the number of buffers processed. */ 726 droq->stats.pkts_received += pkt; 727 droq->stats.bytes_received += total_len; 728 729 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 730 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 731 732 droq->stats.dropped_toomany += (pkts_to_process - pkt); 733 return pkts_to_process; 734 } 735 736 return pkt; 737 } 738 739 int 740 octeon_droq_process_packets(struct octeon_device *oct, 741 struct octeon_droq *droq, 742 u32 budget) 743 { 744 u32 pkt_count = 0, pkts_processed = 0; 745 struct list_head *tmp, *tmp2; 746 747 /* Grab the droq lock */ 748 spin_lock(&droq->lock); 749 750 octeon_droq_check_hw_for_pkts(droq); 751 pkt_count = atomic_read(&droq->pkts_pending); 752 753 if (!pkt_count) { 754 spin_unlock(&droq->lock); 755 return 0; 756 } 757 758 if (pkt_count > budget) 759 pkt_count = budget; 760 761 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); 762 763 atomic_sub(pkts_processed, &droq->pkts_pending); 764 765 /* Release the spin lock */ 766 spin_unlock(&droq->lock); 767 768 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 769 struct __dispatch *rdisp = (struct __dispatch *)tmp; 770 771 list_del(tmp); 772 rdisp->disp_fn(rdisp->rinfo, 773 octeon_get_dispatch_arg 774 (oct, 775 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 776 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 777 } 778 779 /* If there are packets pending. schedule tasklet again */ 780 if (atomic_read(&droq->pkts_pending)) 781 return 1; 782 783 return 0; 784 } 785 786 /** 787 * Utility function to poll for packets. check_hw_for_packets must be 788 * called before calling this routine. 789 */ 790 791 static int 792 octeon_droq_process_poll_pkts(struct octeon_device *oct, 793 struct octeon_droq *droq, u32 budget) 794 { 795 struct list_head *tmp, *tmp2; 796 u32 pkts_available = 0, pkts_processed = 0; 797 u32 total_pkts_processed = 0; 798 799 if (budget > droq->max_count) 800 budget = droq->max_count; 801 802 spin_lock(&droq->lock); 803 804 while (total_pkts_processed < budget) { 805 octeon_droq_check_hw_for_pkts(droq); 806 807 pkts_available = 808 CVM_MIN((budget - total_pkts_processed), 809 (u32)(atomic_read(&droq->pkts_pending))); 810 811 if (pkts_available == 0) 812 break; 813 814 pkts_processed = 815 octeon_droq_fast_process_packets(oct, droq, 816 pkts_available); 817 818 atomic_sub(pkts_processed, &droq->pkts_pending); 819 820 total_pkts_processed += pkts_processed; 821 } 822 823 spin_unlock(&droq->lock); 824 825 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 826 struct __dispatch *rdisp = (struct __dispatch *)tmp; 827 828 list_del(tmp); 829 rdisp->disp_fn(rdisp->rinfo, 830 octeon_get_dispatch_arg 831 (oct, 832 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 833 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 834 } 835 836 return total_pkts_processed; 837 } 838 839 int 840 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, 841 u32 arg) 842 { 843 struct octeon_droq *droq; 844 845 droq = oct->droq[q_no]; 846 847 if (cmd == POLL_EVENT_PROCESS_PKTS) 848 return octeon_droq_process_poll_pkts(oct, droq, arg); 849 850 if (cmd == POLL_EVENT_PENDING_PKTS) { 851 u32 pkt_cnt = atomic_read(&droq->pkts_pending); 852 853 return octeon_droq_process_packets(oct, droq, pkt_cnt); 854 } 855 856 if (cmd == POLL_EVENT_ENABLE_INTR) { 857 u32 value; 858 unsigned long flags; 859 860 /* Enable Pkt Interrupt */ 861 switch (oct->chip_id) { 862 case OCTEON_CN66XX: 863 case OCTEON_CN68XX: { 864 struct octeon_cn6xxx *cn6xxx = 865 (struct octeon_cn6xxx *)oct->chip; 866 spin_lock_irqsave 867 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 868 value = 869 octeon_read_csr(oct, 870 CN6XXX_SLI_PKT_TIME_INT_ENB); 871 value |= (1 << q_no); 872 octeon_write_csr(oct, 873 CN6XXX_SLI_PKT_TIME_INT_ENB, 874 value); 875 value = 876 octeon_read_csr(oct, 877 CN6XXX_SLI_PKT_CNT_INT_ENB); 878 value |= (1 << q_no); 879 octeon_write_csr(oct, 880 CN6XXX_SLI_PKT_CNT_INT_ENB, 881 value); 882 883 /* don't bother flushing the enables */ 884 885 spin_unlock_irqrestore 886 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 887 return 0; 888 } 889 break; 890 case OCTEON_CN23XX_PF_VID: { 891 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 892 } 893 break; 894 } 895 return 0; 896 } 897 898 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); 899 return -EINVAL; 900 } 901 902 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 903 struct octeon_droq_ops *ops) 904 { 905 struct octeon_droq *droq; 906 unsigned long flags; 907 struct octeon_config *oct_cfg = NULL; 908 909 oct_cfg = octeon_get_conf(oct); 910 911 if (!oct_cfg) 912 return -EINVAL; 913 914 if (!(ops)) { 915 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 916 __func__); 917 return -EINVAL; 918 } 919 920 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 921 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 922 __func__, q_no, (oct->num_oqs - 1)); 923 return -EINVAL; 924 } 925 926 droq = oct->droq[q_no]; 927 928 spin_lock_irqsave(&droq->lock, flags); 929 930 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 931 932 spin_unlock_irqrestore(&droq->lock, flags); 933 934 return 0; 935 } 936 937 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 938 { 939 unsigned long flags; 940 struct octeon_droq *droq; 941 struct octeon_config *oct_cfg = NULL; 942 943 oct_cfg = octeon_get_conf(oct); 944 945 if (!oct_cfg) 946 return -EINVAL; 947 948 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 949 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 950 __func__, q_no, oct->num_oqs - 1); 951 return -EINVAL; 952 } 953 954 droq = oct->droq[q_no]; 955 956 if (!droq) { 957 dev_info(&oct->pci_dev->dev, 958 "Droq id (%d) not available.\n", q_no); 959 return 0; 960 } 961 962 spin_lock_irqsave(&droq->lock, flags); 963 964 droq->ops.fptr = NULL; 965 droq->ops.farg = NULL; 966 droq->ops.drop_on_max = 0; 967 968 spin_unlock_irqrestore(&droq->lock, flags); 969 970 return 0; 971 } 972 973 int octeon_create_droq(struct octeon_device *oct, 974 u32 q_no, u32 num_descs, 975 u32 desc_size, void *app_ctx) 976 { 977 struct octeon_droq *droq; 978 int numa_node = cpu_to_node(q_no % num_online_cpus()); 979 980 if (oct->droq[q_no]) { 981 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 982 q_no); 983 return 1; 984 } 985 986 /* Allocate the DS for the new droq. */ 987 droq = vmalloc_node(sizeof(*droq), numa_node); 988 if (!droq) 989 droq = vmalloc(sizeof(*droq)); 990 if (!droq) 991 return -1; 992 993 memset(droq, 0, sizeof(struct octeon_droq)); 994 995 /*Disable the pkt o/p for this Q */ 996 octeon_set_droq_pkt_op(oct, q_no, 0); 997 oct->droq[q_no] = droq; 998 999 /* Initialize the Droq */ 1000 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { 1001 vfree(oct->droq[q_no]); 1002 oct->droq[q_no] = NULL; 1003 return -1; 1004 } 1005 1006 oct->num_oqs++; 1007 1008 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 1009 oct->num_oqs); 1010 1011 /* Global Droq register settings */ 1012 1013 /* As of now not required, as setting are done for all 32 Droqs at 1014 * the same time. 1015 */ 1016 return 0; 1017 } 1018