1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more details. 17 ***********************************************************************/ 18 #include <linux/pci.h> 19 #include <linux/netdevice.h> 20 #include <linux/vmalloc.h> 21 #include "liquidio_common.h" 22 #include "octeon_droq.h" 23 #include "octeon_iq.h" 24 #include "response_manager.h" 25 #include "octeon_device.h" 26 #include "octeon_main.h" 27 #include "octeon_network.h" 28 #include "cn66xx_regs.h" 29 #include "cn66xx_device.h" 30 #include "cn23xx_pf_device.h" 31 #include "cn23xx_vf_device.h" 32 33 struct niclist { 34 struct list_head list; 35 void *ptr; 36 }; 37 38 struct __dispatch { 39 struct list_head list; 40 struct octeon_recv_info *rinfo; 41 octeon_dispatch_fn_t disp_fn; 42 }; 43 44 /** Get the argument that the user set when registering dispatch 45 * function for a given opcode/subcode. 46 * @param octeon_dev - the octeon device pointer. 47 * @param opcode - the opcode for which the dispatch argument 48 * is to be checked. 49 * @param subcode - the subcode for which the dispatch argument 50 * is to be checked. 51 * @return Success: void * (argument to the dispatch function) 52 * @return Failure: NULL 53 * 54 */ 55 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 56 u16 opcode, u16 subcode) 57 { 58 int idx; 59 struct list_head *dispatch; 60 void *fn_arg = NULL; 61 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 62 63 idx = combined_opcode & OCTEON_OPCODE_MASK; 64 65 spin_lock_bh(&octeon_dev->dispatch.lock); 66 67 if (octeon_dev->dispatch.count == 0) { 68 spin_unlock_bh(&octeon_dev->dispatch.lock); 69 return NULL; 70 } 71 72 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 73 fn_arg = octeon_dev->dispatch.dlist[idx].arg; 74 } else { 75 list_for_each(dispatch, 76 &octeon_dev->dispatch.dlist[idx].list) { 77 if (((struct octeon_dispatch *)dispatch)->opcode == 78 combined_opcode) { 79 fn_arg = ((struct octeon_dispatch *) 80 dispatch)->arg; 81 break; 82 } 83 } 84 } 85 86 spin_unlock_bh(&octeon_dev->dispatch.lock); 87 return fn_arg; 88 } 89 90 /** Check for packets on Droq. This function should be called with lock held. 91 * @param droq - Droq on which count is checked. 92 * @return Returns packet count. 93 */ 94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) 95 { 96 u32 pkt_count = 0; 97 u32 last_count; 98 99 pkt_count = readl(droq->pkts_sent_reg); 100 101 last_count = pkt_count - droq->pkt_count; 102 droq->pkt_count = pkt_count; 103 104 /* we shall write to cnts at napi irq enable or end of droq tasklet */ 105 if (last_count) 106 atomic_add(last_count, &droq->pkts_pending); 107 108 return last_count; 109 } 110 111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 112 { 113 u32 count = 0; 114 115 /* max_empty_descs is the max. no. of descs that can have no buffers. 116 * If the empty desc count goes beyond this value, we cannot safely 117 * read in a 64K packet sent by Octeon 118 * (64K is max pkt size from Octeon) 119 */ 120 droq->max_empty_descs = 0; 121 122 do { 123 droq->max_empty_descs++; 124 count += droq->buffer_size; 125 } while (count < (64 * 1024)); 126 127 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 128 } 129 130 static void octeon_droq_reset_indices(struct octeon_droq *droq) 131 { 132 droq->read_idx = 0; 133 droq->write_idx = 0; 134 droq->refill_idx = 0; 135 droq->refill_count = 0; 136 atomic_set(&droq->pkts_pending, 0); 137 } 138 139 static void 140 octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 141 struct octeon_droq *droq) 142 { 143 u32 i; 144 struct octeon_skb_page_info *pg_info; 145 146 for (i = 0; i < droq->max_count; i++) { 147 pg_info = &droq->recv_buf_list[i].pg_info; 148 149 if (pg_info->dma) 150 lio_unmap_ring(oct->pci_dev, 151 (u64)pg_info->dma); 152 pg_info->dma = 0; 153 154 if (pg_info->page) 155 recv_buffer_destroy(droq->recv_buf_list[i].buffer, 156 pg_info); 157 158 if (droq->desc_ring && droq->desc_ring[i].info_ptr) 159 lio_unmap_ring_info(oct->pci_dev, 160 (u64)droq-> 161 desc_ring[i].info_ptr, 162 OCT_DROQ_INFO_SIZE); 163 droq->recv_buf_list[i].buffer = NULL; 164 } 165 166 octeon_droq_reset_indices(droq); 167 } 168 169 static int 170 octeon_droq_setup_ring_buffers(struct octeon_device *oct, 171 struct octeon_droq *droq) 172 { 173 u32 i; 174 void *buf; 175 struct octeon_droq_desc *desc_ring = droq->desc_ring; 176 177 for (i = 0; i < droq->max_count; i++) { 178 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); 179 180 if (!buf) { 181 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 182 __func__); 183 droq->stats.rx_alloc_failure++; 184 return -ENOMEM; 185 } 186 187 droq->recv_buf_list[i].buffer = buf; 188 droq->recv_buf_list[i].data = get_rbd(buf); 189 droq->info_list[i].length = 0; 190 191 /* map ring buffers into memory */ 192 desc_ring[i].info_ptr = lio_map_ring_info(droq, i); 193 desc_ring[i].buffer_ptr = 194 lio_map_ring(droq->recv_buf_list[i].buffer); 195 } 196 197 octeon_droq_reset_indices(droq); 198 199 octeon_droq_compute_max_packet_bufs(droq); 200 201 return 0; 202 } 203 204 int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 205 { 206 struct octeon_droq *droq = oct->droq[q_no]; 207 208 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 209 210 octeon_droq_destroy_ring_buffers(oct, droq); 211 vfree(droq->recv_buf_list); 212 213 if (droq->info_base_addr) 214 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list, 215 droq->info_alloc_size, 216 droq->info_base_addr, 217 droq->info_list_dma); 218 219 if (droq->desc_ring) 220 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 221 droq->desc_ring, droq->desc_ring_dma); 222 223 memset(droq, 0, OCT_DROQ_SIZE); 224 225 return 0; 226 } 227 228 int octeon_init_droq(struct octeon_device *oct, 229 u32 q_no, 230 u32 num_descs, 231 u32 desc_size, 232 void *app_ctx) 233 { 234 struct octeon_droq *droq; 235 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 236 u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 237 int orig_node = dev_to_node(&oct->pci_dev->dev); 238 int numa_node = cpu_to_node(q_no % num_online_cpus()); 239 240 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 241 242 droq = oct->droq[q_no]; 243 memset(droq, 0, OCT_DROQ_SIZE); 244 245 droq->oct_dev = oct; 246 droq->q_no = q_no; 247 if (app_ctx) 248 droq->app_ctx = app_ctx; 249 else 250 droq->app_ctx = (void *)(size_t)q_no; 251 252 c_num_descs = num_descs; 253 c_buf_size = desc_size; 254 if (OCTEON_CN6XXX(oct)) { 255 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); 256 257 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 258 c_refill_threshold = 259 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 260 } else if (OCTEON_CN23XX_PF(oct)) { 261 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); 262 263 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 264 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 265 } else if (OCTEON_CN23XX_VF(oct)) { 266 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); 267 268 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 269 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 270 } else { 271 return 1; 272 } 273 274 droq->max_count = c_num_descs; 275 droq->buffer_size = c_buf_size; 276 277 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 278 set_dev_node(&oct->pci_dev->dev, numa_node); 279 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 280 (dma_addr_t *)&droq->desc_ring_dma); 281 set_dev_node(&oct->pci_dev->dev, orig_node); 282 if (!droq->desc_ring) 283 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 284 (dma_addr_t *)&droq->desc_ring_dma); 285 286 if (!droq->desc_ring) { 287 dev_err(&oct->pci_dev->dev, 288 "Output queue %d ring alloc failed\n", q_no); 289 return 1; 290 } 291 292 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 293 q_no, droq->desc_ring, droq->desc_ring_dma); 294 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 295 droq->max_count); 296 297 droq->info_list = 298 cnnic_numa_alloc_aligned_dma((droq->max_count * 299 OCT_DROQ_INFO_SIZE), 300 &droq->info_alloc_size, 301 &droq->info_base_addr, 302 numa_node); 303 if (!droq->info_list) { 304 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n"); 305 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 306 droq->desc_ring, droq->desc_ring_dma); 307 return 1; 308 } 309 310 droq->recv_buf_list = (struct octeon_recv_buffer *) 311 vmalloc_node(droq->max_count * 312 OCT_DROQ_RECVBUF_SIZE, 313 numa_node); 314 if (!droq->recv_buf_list) 315 droq->recv_buf_list = (struct octeon_recv_buffer *) 316 vmalloc(droq->max_count * 317 OCT_DROQ_RECVBUF_SIZE); 318 if (!droq->recv_buf_list) { 319 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 320 goto init_droq_fail; 321 } 322 323 if (octeon_droq_setup_ring_buffers(oct, droq)) 324 goto init_droq_fail; 325 326 droq->pkts_per_intr = c_pkts_per_intr; 327 droq->refill_threshold = c_refill_threshold; 328 329 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 330 droq->max_empty_descs); 331 332 spin_lock_init(&droq->lock); 333 334 INIT_LIST_HEAD(&droq->dispatch_list); 335 336 /* For 56xx Pass1, this function won't be called, so no checks. */ 337 oct->fn_list.setup_oq_regs(oct, q_no); 338 339 oct->io_qmask.oq |= BIT_ULL(q_no); 340 341 return 0; 342 343 init_droq_fail: 344 octeon_delete_droq(oct, q_no); 345 return 1; 346 } 347 348 /* octeon_create_recv_info 349 * Parameters: 350 * octeon_dev - pointer to the octeon device structure 351 * droq - droq in which the packet arrived. 352 * buf_cnt - no. of buffers used by the packet. 353 * idx - index in the descriptor for the first buffer in the packet. 354 * Description: 355 * Allocates a recv_info_t and copies the buffer addresses for packet data 356 * into the recv_pkt space which starts at an 8B offset from recv_info_t. 357 * Flags the descriptors for refill later. If available descriptors go 358 * below the threshold to receive a 64K pkt, new buffers are first allocated 359 * before the recv_pkt_t is created. 360 * This routine will be called in interrupt context. 361 * Returns: 362 * Success: Pointer to recv_info_t 363 * Failure: NULL. 364 * Locks: 365 * The droq->lock is held when this routine is called. 366 */ 367 static inline struct octeon_recv_info *octeon_create_recv_info( 368 struct octeon_device *octeon_dev, 369 struct octeon_droq *droq, 370 u32 buf_cnt, 371 u32 idx) 372 { 373 struct octeon_droq_info *info; 374 struct octeon_recv_pkt *recv_pkt; 375 struct octeon_recv_info *recv_info; 376 u32 i, bytes_left; 377 struct octeon_skb_page_info *pg_info; 378 379 info = &droq->info_list[idx]; 380 381 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 382 if (!recv_info) 383 return NULL; 384 385 recv_pkt = recv_info->recv_pkt; 386 recv_pkt->rh = info->rh; 387 recv_pkt->length = (u32)info->length; 388 recv_pkt->buffer_count = (u16)buf_cnt; 389 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 390 391 i = 0; 392 bytes_left = (u32)info->length; 393 394 while (buf_cnt) { 395 { 396 pg_info = &droq->recv_buf_list[idx].pg_info; 397 398 lio_unmap_ring(octeon_dev->pci_dev, 399 (u64)pg_info->dma); 400 pg_info->page = NULL; 401 pg_info->dma = 0; 402 } 403 404 recv_pkt->buffer_size[i] = 405 (bytes_left >= 406 droq->buffer_size) ? droq->buffer_size : bytes_left; 407 408 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 409 droq->recv_buf_list[idx].buffer = NULL; 410 411 idx = incr_index(idx, 1, droq->max_count); 412 bytes_left -= droq->buffer_size; 413 i++; 414 buf_cnt--; 415 } 416 417 return recv_info; 418 } 419 420 /* If we were not able to refill all buffers, try to move around 421 * the buffers that were not dispatched. 422 */ 423 static inline u32 424 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 425 struct octeon_droq_desc *desc_ring) 426 { 427 u32 desc_refilled = 0; 428 429 u32 refill_index = droq->refill_idx; 430 431 while (refill_index != droq->read_idx) { 432 if (droq->recv_buf_list[refill_index].buffer) { 433 droq->recv_buf_list[droq->refill_idx].buffer = 434 droq->recv_buf_list[refill_index].buffer; 435 droq->recv_buf_list[droq->refill_idx].data = 436 droq->recv_buf_list[refill_index].data; 437 desc_ring[droq->refill_idx].buffer_ptr = 438 desc_ring[refill_index].buffer_ptr; 439 droq->recv_buf_list[refill_index].buffer = NULL; 440 desc_ring[refill_index].buffer_ptr = 0; 441 do { 442 droq->refill_idx = incr_index(droq->refill_idx, 443 1, 444 droq->max_count); 445 desc_refilled++; 446 droq->refill_count--; 447 } while (droq->recv_buf_list[droq->refill_idx]. 448 buffer); 449 } 450 refill_index = incr_index(refill_index, 1, droq->max_count); 451 } /* while */ 452 return desc_refilled; 453 } 454 455 /* octeon_droq_refill 456 * Parameters: 457 * droq - droq in which descriptors require new buffers. 458 * Description: 459 * Called during normal DROQ processing in interrupt mode or by the poll 460 * thread to refill the descriptors from which buffers were dispatched 461 * to upper layers. Attempts to allocate new buffers. If that fails, moves 462 * up buffers (that were not dispatched) to form a contiguous ring. 463 * Returns: 464 * No of descriptors refilled. 465 * Locks: 466 * This routine is called with droq->lock held. 467 */ 468 static u32 469 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 470 { 471 struct octeon_droq_desc *desc_ring; 472 void *buf = NULL; 473 u8 *data; 474 u32 desc_refilled = 0; 475 struct octeon_skb_page_info *pg_info; 476 477 desc_ring = droq->desc_ring; 478 479 while (droq->refill_count && (desc_refilled < droq->max_count)) { 480 /* If a valid buffer exists (happens if there is no dispatch), 481 * reuse 482 * the buffer, else allocate. 483 */ 484 if (!droq->recv_buf_list[droq->refill_idx].buffer) { 485 pg_info = 486 &droq->recv_buf_list[droq->refill_idx].pg_info; 487 /* Either recycle the existing pages or go for 488 * new page alloc 489 */ 490 if (pg_info->page) 491 buf = recv_buffer_reuse(octeon_dev, pg_info); 492 else 493 buf = recv_buffer_alloc(octeon_dev, pg_info); 494 /* If a buffer could not be allocated, no point in 495 * continuing 496 */ 497 if (!buf) { 498 droq->stats.rx_alloc_failure++; 499 break; 500 } 501 droq->recv_buf_list[droq->refill_idx].buffer = 502 buf; 503 data = get_rbd(buf); 504 } else { 505 data = get_rbd(droq->recv_buf_list 506 [droq->refill_idx].buffer); 507 } 508 509 droq->recv_buf_list[droq->refill_idx].data = data; 510 511 desc_ring[droq->refill_idx].buffer_ptr = 512 lio_map_ring(droq->recv_buf_list[droq-> 513 refill_idx].buffer); 514 /* Reset any previous values in the length field. */ 515 droq->info_list[droq->refill_idx].length = 0; 516 517 droq->refill_idx = incr_index(droq->refill_idx, 1, 518 droq->max_count); 519 desc_refilled++; 520 droq->refill_count--; 521 } 522 523 if (droq->refill_count) 524 desc_refilled += 525 octeon_droq_refill_pullup_descs(droq, desc_ring); 526 527 /* if droq->refill_count 528 * The refill count would not change in pass two. We only moved buffers 529 * to close the gap in the ring, but we would still have the same no. of 530 * buffers to refill. 531 */ 532 return desc_refilled; 533 } 534 535 static inline u32 536 octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 537 { 538 u32 buf_cnt = 0; 539 540 while (total_len > (buf_size * buf_cnt)) 541 buf_cnt++; 542 return buf_cnt; 543 } 544 545 static int 546 octeon_droq_dispatch_pkt(struct octeon_device *oct, 547 struct octeon_droq *droq, 548 union octeon_rh *rh, 549 struct octeon_droq_info *info) 550 { 551 u32 cnt; 552 octeon_dispatch_fn_t disp_fn; 553 struct octeon_recv_info *rinfo; 554 555 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 556 557 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 558 (u16)rh->r.subcode); 559 if (disp_fn) { 560 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 561 if (rinfo) { 562 struct __dispatch *rdisp = rinfo->rsvd; 563 564 rdisp->rinfo = rinfo; 565 rdisp->disp_fn = disp_fn; 566 rinfo->recv_pkt->rh = *rh; 567 list_add_tail(&rdisp->list, 568 &droq->dispatch_list); 569 } else { 570 droq->stats.dropped_nomem++; 571 } 572 } else { 573 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", 574 (unsigned int)rh->r.opcode, 575 (unsigned int)rh->r.subcode); 576 droq->stats.dropped_nodispatch++; 577 } 578 579 return cnt; 580 } 581 582 static inline void octeon_droq_drop_packets(struct octeon_device *oct, 583 struct octeon_droq *droq, 584 u32 cnt) 585 { 586 u32 i = 0, buf_cnt; 587 struct octeon_droq_info *info; 588 589 for (i = 0; i < cnt; i++) { 590 info = &droq->info_list[droq->read_idx]; 591 octeon_swap_8B_data((u64 *)info, 2); 592 593 if (info->length) { 594 info->length -= OCT_RH_SIZE; 595 droq->stats.bytes_received += info->length; 596 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 597 (u32)info->length); 598 } else { 599 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 600 buf_cnt = 1; 601 } 602 603 droq->read_idx = incr_index(droq->read_idx, buf_cnt, 604 droq->max_count); 605 droq->refill_count += buf_cnt; 606 } 607 } 608 609 static u32 610 octeon_droq_fast_process_packets(struct octeon_device *oct, 611 struct octeon_droq *droq, 612 u32 pkts_to_process) 613 { 614 struct octeon_droq_info *info; 615 union octeon_rh *rh; 616 u32 pkt, total_len = 0, pkt_count; 617 618 pkt_count = pkts_to_process; 619 620 for (pkt = 0; pkt < pkt_count; pkt++) { 621 u32 pkt_len = 0; 622 struct sk_buff *nicbuf = NULL; 623 struct octeon_skb_page_info *pg_info; 624 void *buf; 625 626 info = &droq->info_list[droq->read_idx]; 627 octeon_swap_8B_data((u64 *)info, 2); 628 629 if (!info->length) { 630 dev_err(&oct->pci_dev->dev, 631 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 632 droq->q_no, droq->read_idx, pkt_count); 633 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 634 (u8 *)info, 635 OCT_DROQ_INFO_SIZE); 636 break; 637 } 638 639 /* Len of resp hdr in included in the received data len. */ 640 info->length -= OCT_RH_SIZE; 641 rh = &info->rh; 642 643 total_len += (u32)info->length; 644 if (opcode_slow_path(rh)) { 645 u32 buf_cnt; 646 647 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 648 droq->read_idx = incr_index(droq->read_idx, 649 buf_cnt, droq->max_count); 650 droq->refill_count += buf_cnt; 651 } else { 652 if (info->length <= droq->buffer_size) { 653 pkt_len = (u32)info->length; 654 nicbuf = droq->recv_buf_list[ 655 droq->read_idx].buffer; 656 pg_info = &droq->recv_buf_list[ 657 droq->read_idx].pg_info; 658 if (recv_buffer_recycle(oct, pg_info)) 659 pg_info->page = NULL; 660 droq->recv_buf_list[droq->read_idx].buffer = 661 NULL; 662 663 droq->read_idx = incr_index(droq->read_idx, 1, 664 droq->max_count); 665 droq->refill_count++; 666 } else { 667 nicbuf = octeon_fast_packet_alloc((u32) 668 info->length); 669 pkt_len = 0; 670 /* nicbuf allocation can fail. We'll handle it 671 * inside the loop. 672 */ 673 while (pkt_len < info->length) { 674 int cpy_len, idx = droq->read_idx; 675 676 cpy_len = ((pkt_len + droq->buffer_size) 677 > info->length) ? 678 ((u32)info->length - pkt_len) : 679 droq->buffer_size; 680 681 if (nicbuf) { 682 octeon_fast_packet_next(droq, 683 nicbuf, 684 cpy_len, 685 idx); 686 buf = droq->recv_buf_list[idx]. 687 buffer; 688 recv_buffer_fast_free(buf); 689 droq->recv_buf_list[idx].buffer 690 = NULL; 691 } else { 692 droq->stats.rx_alloc_failure++; 693 } 694 695 pkt_len += cpy_len; 696 droq->read_idx = 697 incr_index(droq->read_idx, 1, 698 droq->max_count); 699 droq->refill_count++; 700 } 701 } 702 703 if (nicbuf) { 704 if (droq->ops.fptr) { 705 droq->ops.fptr(oct->octeon_id, 706 nicbuf, pkt_len, 707 rh, &droq->napi, 708 droq->ops.farg); 709 } else { 710 recv_buffer_free(nicbuf); 711 } 712 } 713 } 714 715 if (droq->refill_count >= droq->refill_threshold) { 716 int desc_refilled = octeon_droq_refill(oct, droq); 717 718 /* Flush the droq descriptor data to memory to be sure 719 * that when we update the credits the data in memory 720 * is accurate. 721 */ 722 wmb(); 723 writel((desc_refilled), droq->pkts_credit_reg); 724 /* make sure mmio write completes */ 725 mmiowb(); 726 } 727 728 } /* for (each packet)... */ 729 730 /* Increment refill_count by the number of buffers processed. */ 731 droq->stats.pkts_received += pkt; 732 droq->stats.bytes_received += total_len; 733 734 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 735 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 736 737 droq->stats.dropped_toomany += (pkts_to_process - pkt); 738 return pkts_to_process; 739 } 740 741 return pkt; 742 } 743 744 int 745 octeon_droq_process_packets(struct octeon_device *oct, 746 struct octeon_droq *droq, 747 u32 budget) 748 { 749 u32 pkt_count = 0, pkts_processed = 0; 750 struct list_head *tmp, *tmp2; 751 752 /* Grab the droq lock */ 753 spin_lock(&droq->lock); 754 755 octeon_droq_check_hw_for_pkts(droq); 756 pkt_count = atomic_read(&droq->pkts_pending); 757 758 if (!pkt_count) { 759 spin_unlock(&droq->lock); 760 return 0; 761 } 762 763 if (pkt_count > budget) 764 pkt_count = budget; 765 766 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); 767 768 atomic_sub(pkts_processed, &droq->pkts_pending); 769 770 /* Release the spin lock */ 771 spin_unlock(&droq->lock); 772 773 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 774 struct __dispatch *rdisp = (struct __dispatch *)tmp; 775 776 list_del(tmp); 777 rdisp->disp_fn(rdisp->rinfo, 778 octeon_get_dispatch_arg 779 (oct, 780 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 781 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 782 } 783 784 /* If there are packets pending. schedule tasklet again */ 785 if (atomic_read(&droq->pkts_pending)) 786 return 1; 787 788 return 0; 789 } 790 791 /** 792 * Utility function to poll for packets. check_hw_for_packets must be 793 * called before calling this routine. 794 */ 795 796 static int 797 octeon_droq_process_poll_pkts(struct octeon_device *oct, 798 struct octeon_droq *droq, u32 budget) 799 { 800 struct list_head *tmp, *tmp2; 801 u32 pkts_available = 0, pkts_processed = 0; 802 u32 total_pkts_processed = 0; 803 804 if (budget > droq->max_count) 805 budget = droq->max_count; 806 807 spin_lock(&droq->lock); 808 809 while (total_pkts_processed < budget) { 810 octeon_droq_check_hw_for_pkts(droq); 811 812 pkts_available = min((budget - total_pkts_processed), 813 (u32)(atomic_read(&droq->pkts_pending))); 814 815 if (pkts_available == 0) 816 break; 817 818 pkts_processed = 819 octeon_droq_fast_process_packets(oct, droq, 820 pkts_available); 821 822 atomic_sub(pkts_processed, &droq->pkts_pending); 823 824 total_pkts_processed += pkts_processed; 825 } 826 827 spin_unlock(&droq->lock); 828 829 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 830 struct __dispatch *rdisp = (struct __dispatch *)tmp; 831 832 list_del(tmp); 833 rdisp->disp_fn(rdisp->rinfo, 834 octeon_get_dispatch_arg 835 (oct, 836 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 837 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 838 } 839 840 return total_pkts_processed; 841 } 842 843 int 844 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, 845 u32 arg) 846 { 847 struct octeon_droq *droq; 848 849 droq = oct->droq[q_no]; 850 851 if (cmd == POLL_EVENT_PROCESS_PKTS) 852 return octeon_droq_process_poll_pkts(oct, droq, arg); 853 854 if (cmd == POLL_EVENT_PENDING_PKTS) { 855 u32 pkt_cnt = atomic_read(&droq->pkts_pending); 856 857 return octeon_droq_process_packets(oct, droq, pkt_cnt); 858 } 859 860 if (cmd == POLL_EVENT_ENABLE_INTR) { 861 u32 value; 862 unsigned long flags; 863 864 /* Enable Pkt Interrupt */ 865 switch (oct->chip_id) { 866 case OCTEON_CN66XX: 867 case OCTEON_CN68XX: { 868 struct octeon_cn6xxx *cn6xxx = 869 (struct octeon_cn6xxx *)oct->chip; 870 spin_lock_irqsave 871 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 872 value = 873 octeon_read_csr(oct, 874 CN6XXX_SLI_PKT_TIME_INT_ENB); 875 value |= (1 << q_no); 876 octeon_write_csr(oct, 877 CN6XXX_SLI_PKT_TIME_INT_ENB, 878 value); 879 value = 880 octeon_read_csr(oct, 881 CN6XXX_SLI_PKT_CNT_INT_ENB); 882 value |= (1 << q_no); 883 octeon_write_csr(oct, 884 CN6XXX_SLI_PKT_CNT_INT_ENB, 885 value); 886 887 /* don't bother flushing the enables */ 888 889 spin_unlock_irqrestore 890 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 891 return 0; 892 } 893 break; 894 case OCTEON_CN23XX_PF_VID: { 895 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 896 } 897 break; 898 899 case OCTEON_CN23XX_VF_VID: 900 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 901 break; 902 } 903 return 0; 904 } 905 906 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); 907 return -EINVAL; 908 } 909 910 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 911 struct octeon_droq_ops *ops) 912 { 913 struct octeon_droq *droq; 914 unsigned long flags; 915 struct octeon_config *oct_cfg = NULL; 916 917 oct_cfg = octeon_get_conf(oct); 918 919 if (!oct_cfg) 920 return -EINVAL; 921 922 if (!(ops)) { 923 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 924 __func__); 925 return -EINVAL; 926 } 927 928 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 929 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 930 __func__, q_no, (oct->num_oqs - 1)); 931 return -EINVAL; 932 } 933 934 droq = oct->droq[q_no]; 935 936 spin_lock_irqsave(&droq->lock, flags); 937 938 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 939 940 spin_unlock_irqrestore(&droq->lock, flags); 941 942 return 0; 943 } 944 945 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 946 { 947 unsigned long flags; 948 struct octeon_droq *droq; 949 struct octeon_config *oct_cfg = NULL; 950 951 oct_cfg = octeon_get_conf(oct); 952 953 if (!oct_cfg) 954 return -EINVAL; 955 956 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 957 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 958 __func__, q_no, oct->num_oqs - 1); 959 return -EINVAL; 960 } 961 962 droq = oct->droq[q_no]; 963 964 if (!droq) { 965 dev_info(&oct->pci_dev->dev, 966 "Droq id (%d) not available.\n", q_no); 967 return 0; 968 } 969 970 spin_lock_irqsave(&droq->lock, flags); 971 972 droq->ops.fptr = NULL; 973 droq->ops.farg = NULL; 974 droq->ops.drop_on_max = 0; 975 976 spin_unlock_irqrestore(&droq->lock, flags); 977 978 return 0; 979 } 980 981 int octeon_create_droq(struct octeon_device *oct, 982 u32 q_no, u32 num_descs, 983 u32 desc_size, void *app_ctx) 984 { 985 struct octeon_droq *droq; 986 int numa_node = cpu_to_node(q_no % num_online_cpus()); 987 988 if (oct->droq[q_no]) { 989 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 990 q_no); 991 return 1; 992 } 993 994 /* Allocate the DS for the new droq. */ 995 droq = vmalloc_node(sizeof(*droq), numa_node); 996 if (!droq) 997 droq = vmalloc(sizeof(*droq)); 998 if (!droq) 999 return -1; 1000 1001 memset(droq, 0, sizeof(struct octeon_droq)); 1002 1003 /*Disable the pkt o/p for this Q */ 1004 octeon_set_droq_pkt_op(oct, q_no, 0); 1005 oct->droq[q_no] = droq; 1006 1007 /* Initialize the Droq */ 1008 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { 1009 vfree(oct->droq[q_no]); 1010 oct->droq[q_no] = NULL; 1011 return -1; 1012 } 1013 1014 oct->num_oqs++; 1015 1016 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 1017 oct->num_oqs); 1018 1019 /* Global Droq register settings */ 1020 1021 /* As of now not required, as setting are done for all 32 Droqs at 1022 * the same time. 1023 */ 1024 return 0; 1025 } 1026