1f21fb3edSRaghu Vatsavayi /********************************************************************** 2f21fb3edSRaghu Vatsavayi * Author: Cavium, Inc. 3f21fb3edSRaghu Vatsavayi * 4f21fb3edSRaghu Vatsavayi * Contact: support@cavium.com 5f21fb3edSRaghu Vatsavayi * Please include "LiquidIO" in the subject. 6f21fb3edSRaghu Vatsavayi * 750579d3dSRaghu Vatsavayi * Copyright (c) 2003-2016 Cavium, Inc. 8f21fb3edSRaghu Vatsavayi * 9f21fb3edSRaghu Vatsavayi * This file is free software; you can redistribute it and/or modify 10f21fb3edSRaghu Vatsavayi * it under the terms of the GNU General Public License, Version 2, as 11f21fb3edSRaghu Vatsavayi * published by the Free Software Foundation. 12f21fb3edSRaghu Vatsavayi * 13f21fb3edSRaghu Vatsavayi * This file is distributed in the hope that it will be useful, but 14f21fb3edSRaghu Vatsavayi * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15f21fb3edSRaghu Vatsavayi * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 1650579d3dSRaghu Vatsavayi * NONINFRINGEMENT. See the GNU General Public License for more details. 1750579d3dSRaghu Vatsavayi ***********************************************************************/ 18f21fb3edSRaghu Vatsavayi #include <linux/pci.h> 19f21fb3edSRaghu Vatsavayi #include <linux/netdevice.h> 205b173cf9SRaghu Vatsavayi #include <linux/vmalloc.h> 21f21fb3edSRaghu Vatsavayi #include "liquidio_common.h" 22f21fb3edSRaghu Vatsavayi #include "octeon_droq.h" 23f21fb3edSRaghu Vatsavayi #include "octeon_iq.h" 24f21fb3edSRaghu Vatsavayi #include "response_manager.h" 25f21fb3edSRaghu Vatsavayi #include "octeon_device.h" 26f21fb3edSRaghu Vatsavayi #include "octeon_main.h" 27f21fb3edSRaghu Vatsavayi #include "octeon_network.h" 28f21fb3edSRaghu Vatsavayi #include "cn66xx_regs.h" 29f21fb3edSRaghu Vatsavayi #include "cn66xx_device.h" 305b823514SRaghu Vatsavayi #include "cn23xx_pf_device.h" 319217c3cfSRaghu Vatsavayi #include "cn23xx_vf_device.h" 32f21fb3edSRaghu Vatsavayi 33f21fb3edSRaghu Vatsavayi struct niclist { 34f21fb3edSRaghu Vatsavayi struct list_head list; 35f21fb3edSRaghu Vatsavayi void *ptr; 36f21fb3edSRaghu Vatsavayi }; 37f21fb3edSRaghu Vatsavayi 38f21fb3edSRaghu Vatsavayi struct __dispatch { 39f21fb3edSRaghu Vatsavayi struct list_head list; 40f21fb3edSRaghu Vatsavayi struct octeon_recv_info *rinfo; 41f21fb3edSRaghu Vatsavayi octeon_dispatch_fn_t disp_fn; 42f21fb3edSRaghu Vatsavayi }; 43f21fb3edSRaghu Vatsavayi 44f21fb3edSRaghu Vatsavayi /** Get the argument that the user set when registering dispatch 45f21fb3edSRaghu Vatsavayi * function for a given opcode/subcode. 46f21fb3edSRaghu Vatsavayi * @param octeon_dev - the octeon device pointer. 47f21fb3edSRaghu Vatsavayi * @param opcode - the opcode for which the dispatch argument 48f21fb3edSRaghu Vatsavayi * is to be checked. 49f21fb3edSRaghu Vatsavayi * @param subcode - the subcode for which the dispatch argument 50f21fb3edSRaghu Vatsavayi * is to be checked. 51f21fb3edSRaghu Vatsavayi * @return Success: void * (argument to the dispatch function) 52f21fb3edSRaghu Vatsavayi * @return Failure: NULL 53f21fb3edSRaghu Vatsavayi * 54f21fb3edSRaghu Vatsavayi */ 55bf534588SVijaya Mohan Guvva void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 56f21fb3edSRaghu Vatsavayi u16 opcode, u16 subcode) 57f21fb3edSRaghu Vatsavayi { 58f21fb3edSRaghu Vatsavayi int idx; 59f21fb3edSRaghu Vatsavayi struct list_head *dispatch; 60f21fb3edSRaghu Vatsavayi void *fn_arg = NULL; 61f21fb3edSRaghu Vatsavayi u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 62f21fb3edSRaghu Vatsavayi 63f21fb3edSRaghu Vatsavayi idx = combined_opcode & OCTEON_OPCODE_MASK; 64f21fb3edSRaghu Vatsavayi 65f21fb3edSRaghu Vatsavayi spin_lock_bh(&octeon_dev->dispatch.lock); 66f21fb3edSRaghu Vatsavayi 67f21fb3edSRaghu Vatsavayi if (octeon_dev->dispatch.count == 0) { 68f21fb3edSRaghu Vatsavayi spin_unlock_bh(&octeon_dev->dispatch.lock); 69f21fb3edSRaghu Vatsavayi return NULL; 70f21fb3edSRaghu Vatsavayi } 71f21fb3edSRaghu Vatsavayi 72f21fb3edSRaghu Vatsavayi if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 73f21fb3edSRaghu Vatsavayi fn_arg = octeon_dev->dispatch.dlist[idx].arg; 74f21fb3edSRaghu Vatsavayi } else { 75f21fb3edSRaghu Vatsavayi list_for_each(dispatch, 76f21fb3edSRaghu Vatsavayi &octeon_dev->dispatch.dlist[idx].list) { 77f21fb3edSRaghu Vatsavayi if (((struct octeon_dispatch *)dispatch)->opcode == 78f21fb3edSRaghu Vatsavayi combined_opcode) { 79f21fb3edSRaghu Vatsavayi fn_arg = ((struct octeon_dispatch *) 80f21fb3edSRaghu Vatsavayi dispatch)->arg; 81f21fb3edSRaghu Vatsavayi break; 82f21fb3edSRaghu Vatsavayi } 83f21fb3edSRaghu Vatsavayi } 84f21fb3edSRaghu Vatsavayi } 85f21fb3edSRaghu Vatsavayi 86f21fb3edSRaghu Vatsavayi spin_unlock_bh(&octeon_dev->dispatch.lock); 87f21fb3edSRaghu Vatsavayi return fn_arg; 88f21fb3edSRaghu Vatsavayi } 89f21fb3edSRaghu Vatsavayi 90cd8b1eb4SRaghu Vatsavayi /** Check for packets on Droq. This function should be called with lock held. 91a2c64b67SRaghu Vatsavayi * @param droq - Droq on which count is checked. 92a2c64b67SRaghu Vatsavayi * @return Returns packet count. 93a2c64b67SRaghu Vatsavayi */ 94a7d5a3dcSRaghu Vatsavayi u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) 95f21fb3edSRaghu Vatsavayi { 96f21fb3edSRaghu Vatsavayi u32 pkt_count = 0; 97cd8b1eb4SRaghu Vatsavayi u32 last_count; 98f21fb3edSRaghu Vatsavayi 99f21fb3edSRaghu Vatsavayi pkt_count = readl(droq->pkts_sent_reg); 100f21fb3edSRaghu Vatsavayi 101cd8b1eb4SRaghu Vatsavayi last_count = pkt_count - droq->pkt_count; 102cd8b1eb4SRaghu Vatsavayi droq->pkt_count = pkt_count; 103cd8b1eb4SRaghu Vatsavayi 104cd8b1eb4SRaghu Vatsavayi /* we shall write to cnts at napi irq enable or end of droq tasklet */ 105cd8b1eb4SRaghu Vatsavayi if (last_count) 106cd8b1eb4SRaghu Vatsavayi atomic_add(last_count, &droq->pkts_pending); 107cd8b1eb4SRaghu Vatsavayi 108cd8b1eb4SRaghu Vatsavayi return last_count; 109f21fb3edSRaghu Vatsavayi } 110f21fb3edSRaghu Vatsavayi 111f21fb3edSRaghu Vatsavayi static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 112f21fb3edSRaghu Vatsavayi { 113f21fb3edSRaghu Vatsavayi u32 count = 0; 114f21fb3edSRaghu Vatsavayi 115f21fb3edSRaghu Vatsavayi /* max_empty_descs is the max. no. of descs that can have no buffers. 116f21fb3edSRaghu Vatsavayi * If the empty desc count goes beyond this value, we cannot safely 117f21fb3edSRaghu Vatsavayi * read in a 64K packet sent by Octeon 118f21fb3edSRaghu Vatsavayi * (64K is max pkt size from Octeon) 119f21fb3edSRaghu Vatsavayi */ 120f21fb3edSRaghu Vatsavayi droq->max_empty_descs = 0; 121f21fb3edSRaghu Vatsavayi 122f21fb3edSRaghu Vatsavayi do { 123f21fb3edSRaghu Vatsavayi droq->max_empty_descs++; 124f21fb3edSRaghu Vatsavayi count += droq->buffer_size; 125f21fb3edSRaghu Vatsavayi } while (count < (64 * 1024)); 126f21fb3edSRaghu Vatsavayi 127f21fb3edSRaghu Vatsavayi droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 128f21fb3edSRaghu Vatsavayi } 129f21fb3edSRaghu Vatsavayi 130f21fb3edSRaghu Vatsavayi static void octeon_droq_reset_indices(struct octeon_droq *droq) 131f21fb3edSRaghu Vatsavayi { 132f21fb3edSRaghu Vatsavayi droq->read_idx = 0; 133f21fb3edSRaghu Vatsavayi droq->write_idx = 0; 134f21fb3edSRaghu Vatsavayi droq->refill_idx = 0; 135f21fb3edSRaghu Vatsavayi droq->refill_count = 0; 136f21fb3edSRaghu Vatsavayi atomic_set(&droq->pkts_pending, 0); 137f21fb3edSRaghu Vatsavayi } 138f21fb3edSRaghu Vatsavayi 139f21fb3edSRaghu Vatsavayi static void 140f21fb3edSRaghu Vatsavayi octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 141f21fb3edSRaghu Vatsavayi struct octeon_droq *droq) 142f21fb3edSRaghu Vatsavayi { 143f21fb3edSRaghu Vatsavayi u32 i; 144cabeb13bSRaghu Vatsavayi struct octeon_skb_page_info *pg_info; 145f21fb3edSRaghu Vatsavayi 146f21fb3edSRaghu Vatsavayi for (i = 0; i < droq->max_count; i++) { 147cabeb13bSRaghu Vatsavayi pg_info = &droq->recv_buf_list[i].pg_info; 148689062a1SRick Farrington if (!pg_info) 149689062a1SRick Farrington continue; 150cabeb13bSRaghu Vatsavayi 151cabeb13bSRaghu Vatsavayi if (pg_info->dma) 152cabeb13bSRaghu Vatsavayi lio_unmap_ring(oct->pci_dev, 153cabeb13bSRaghu Vatsavayi (u64)pg_info->dma); 154cabeb13bSRaghu Vatsavayi pg_info->dma = 0; 155cabeb13bSRaghu Vatsavayi 156cabeb13bSRaghu Vatsavayi if (pg_info->page) 157cabeb13bSRaghu Vatsavayi recv_buffer_destroy(droq->recv_buf_list[i].buffer, 158cabeb13bSRaghu Vatsavayi pg_info); 159cabeb13bSRaghu Vatsavayi 160f21fb3edSRaghu Vatsavayi droq->recv_buf_list[i].buffer = NULL; 161f21fb3edSRaghu Vatsavayi } 162f21fb3edSRaghu Vatsavayi 163f21fb3edSRaghu Vatsavayi octeon_droq_reset_indices(droq); 164f21fb3edSRaghu Vatsavayi } 165f21fb3edSRaghu Vatsavayi 166f21fb3edSRaghu Vatsavayi static int 167f21fb3edSRaghu Vatsavayi octeon_droq_setup_ring_buffers(struct octeon_device *oct, 168f21fb3edSRaghu Vatsavayi struct octeon_droq *droq) 169f21fb3edSRaghu Vatsavayi { 170f21fb3edSRaghu Vatsavayi u32 i; 171f21fb3edSRaghu Vatsavayi void *buf; 172f21fb3edSRaghu Vatsavayi struct octeon_droq_desc *desc_ring = droq->desc_ring; 173f21fb3edSRaghu Vatsavayi 174f21fb3edSRaghu Vatsavayi for (i = 0; i < droq->max_count; i++) { 175cabeb13bSRaghu Vatsavayi buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); 176f21fb3edSRaghu Vatsavayi 177f21fb3edSRaghu Vatsavayi if (!buf) { 178f21fb3edSRaghu Vatsavayi dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 179f21fb3edSRaghu Vatsavayi __func__); 180cabeb13bSRaghu Vatsavayi droq->stats.rx_alloc_failure++; 181f21fb3edSRaghu Vatsavayi return -ENOMEM; 182f21fb3edSRaghu Vatsavayi } 183f21fb3edSRaghu Vatsavayi 184f21fb3edSRaghu Vatsavayi droq->recv_buf_list[i].buffer = buf; 185f21fb3edSRaghu Vatsavayi droq->recv_buf_list[i].data = get_rbd(buf); 186c4ee5d81SPrasad Kanneganti desc_ring[i].info_ptr = 0; 187f21fb3edSRaghu Vatsavayi desc_ring[i].buffer_ptr = 188cabeb13bSRaghu Vatsavayi lio_map_ring(droq->recv_buf_list[i].buffer); 189f21fb3edSRaghu Vatsavayi } 190f21fb3edSRaghu Vatsavayi 191f21fb3edSRaghu Vatsavayi octeon_droq_reset_indices(droq); 192f21fb3edSRaghu Vatsavayi 193f21fb3edSRaghu Vatsavayi octeon_droq_compute_max_packet_bufs(droq); 194f21fb3edSRaghu Vatsavayi 195f21fb3edSRaghu Vatsavayi return 0; 196f21fb3edSRaghu Vatsavayi } 197f21fb3edSRaghu Vatsavayi 198f21fb3edSRaghu Vatsavayi int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 199f21fb3edSRaghu Vatsavayi { 200f21fb3edSRaghu Vatsavayi struct octeon_droq *droq = oct->droq[q_no]; 201f21fb3edSRaghu Vatsavayi 202f21fb3edSRaghu Vatsavayi dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 203f21fb3edSRaghu Vatsavayi 204f21fb3edSRaghu Vatsavayi octeon_droq_destroy_ring_buffers(oct, droq); 205f21fb3edSRaghu Vatsavayi vfree(droq->recv_buf_list); 206f21fb3edSRaghu Vatsavayi 207f21fb3edSRaghu Vatsavayi if (droq->desc_ring) 208f21fb3edSRaghu Vatsavayi lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 209f21fb3edSRaghu Vatsavayi droq->desc_ring, droq->desc_ring_dma); 210f21fb3edSRaghu Vatsavayi 211f21fb3edSRaghu Vatsavayi memset(droq, 0, OCT_DROQ_SIZE); 212c1550fdeSIntiyaz Basha oct->io_qmask.oq &= ~(1ULL << q_no); 213c1550fdeSIntiyaz Basha vfree(oct->droq[q_no]); 214c1550fdeSIntiyaz Basha oct->droq[q_no] = NULL; 215c1550fdeSIntiyaz Basha oct->num_oqs--; 216f21fb3edSRaghu Vatsavayi 217f21fb3edSRaghu Vatsavayi return 0; 218f21fb3edSRaghu Vatsavayi } 219f21fb3edSRaghu Vatsavayi 220f21fb3edSRaghu Vatsavayi int octeon_init_droq(struct octeon_device *oct, 221f21fb3edSRaghu Vatsavayi u32 q_no, 222f21fb3edSRaghu Vatsavayi u32 num_descs, 223f21fb3edSRaghu Vatsavayi u32 desc_size, 224f21fb3edSRaghu Vatsavayi void *app_ctx) 225f21fb3edSRaghu Vatsavayi { 226f21fb3edSRaghu Vatsavayi struct octeon_droq *droq; 227f21fb3edSRaghu Vatsavayi u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 228f21fb3edSRaghu Vatsavayi u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 229b3ca9af0SVSR Burru int numa_node = dev_to_node(&oct->pci_dev->dev); 230f21fb3edSRaghu Vatsavayi 231f21fb3edSRaghu Vatsavayi dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 232f21fb3edSRaghu Vatsavayi 233f21fb3edSRaghu Vatsavayi droq = oct->droq[q_no]; 234f21fb3edSRaghu Vatsavayi memset(droq, 0, OCT_DROQ_SIZE); 235f21fb3edSRaghu Vatsavayi 236f21fb3edSRaghu Vatsavayi droq->oct_dev = oct; 237f21fb3edSRaghu Vatsavayi droq->q_no = q_no; 238f21fb3edSRaghu Vatsavayi if (app_ctx) 239f21fb3edSRaghu Vatsavayi droq->app_ctx = app_ctx; 240f21fb3edSRaghu Vatsavayi else 241f21fb3edSRaghu Vatsavayi droq->app_ctx = (void *)(size_t)q_no; 242f21fb3edSRaghu Vatsavayi 243f21fb3edSRaghu Vatsavayi c_num_descs = num_descs; 244f21fb3edSRaghu Vatsavayi c_buf_size = desc_size; 245f21fb3edSRaghu Vatsavayi if (OCTEON_CN6XXX(oct)) { 24697a25326SRaghu Vatsavayi struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); 247f21fb3edSRaghu Vatsavayi 248f21fb3edSRaghu Vatsavayi c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 24996ae48b7SRaghu Vatsavayi c_refill_threshold = 25096ae48b7SRaghu Vatsavayi (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 2515b823514SRaghu Vatsavayi } else if (OCTEON_CN23XX_PF(oct)) { 25297a25326SRaghu Vatsavayi struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); 2535b823514SRaghu Vatsavayi 2545b823514SRaghu Vatsavayi c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 2555b823514SRaghu Vatsavayi c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 2569217c3cfSRaghu Vatsavayi } else if (OCTEON_CN23XX_VF(oct)) { 2579217c3cfSRaghu Vatsavayi struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); 2589217c3cfSRaghu Vatsavayi 2599217c3cfSRaghu Vatsavayi c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 2609217c3cfSRaghu Vatsavayi c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 26196ae48b7SRaghu Vatsavayi } else { 26296ae48b7SRaghu Vatsavayi return 1; 263f21fb3edSRaghu Vatsavayi } 264f21fb3edSRaghu Vatsavayi 265f21fb3edSRaghu Vatsavayi droq->max_count = c_num_descs; 266f21fb3edSRaghu Vatsavayi droq->buffer_size = c_buf_size; 267f21fb3edSRaghu Vatsavayi 268f21fb3edSRaghu Vatsavayi desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 269f21fb3edSRaghu Vatsavayi droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 270f21fb3edSRaghu Vatsavayi (dma_addr_t *)&droq->desc_ring_dma); 271f21fb3edSRaghu Vatsavayi 272f21fb3edSRaghu Vatsavayi if (!droq->desc_ring) { 273f21fb3edSRaghu Vatsavayi dev_err(&oct->pci_dev->dev, 274f21fb3edSRaghu Vatsavayi "Output queue %d ring alloc failed\n", q_no); 275f21fb3edSRaghu Vatsavayi return 1; 276f21fb3edSRaghu Vatsavayi } 277f21fb3edSRaghu Vatsavayi 278f21fb3edSRaghu Vatsavayi dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 279f21fb3edSRaghu Vatsavayi q_no, droq->desc_ring, droq->desc_ring_dma); 280f21fb3edSRaghu Vatsavayi dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 281f21fb3edSRaghu Vatsavayi droq->max_count); 282f21fb3edSRaghu Vatsavayi 2838aa639e1SYueHaibing droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE), 28496ae48b7SRaghu Vatsavayi numa_node); 28596ae48b7SRaghu Vatsavayi if (!droq->recv_buf_list) 2868aa639e1SYueHaibing droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE)); 287f21fb3edSRaghu Vatsavayi if (!droq->recv_buf_list) { 288f21fb3edSRaghu Vatsavayi dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 289f21fb3edSRaghu Vatsavayi goto init_droq_fail; 290f21fb3edSRaghu Vatsavayi } 291f21fb3edSRaghu Vatsavayi 292f21fb3edSRaghu Vatsavayi if (octeon_droq_setup_ring_buffers(oct, droq)) 293f21fb3edSRaghu Vatsavayi goto init_droq_fail; 294f21fb3edSRaghu Vatsavayi 295f21fb3edSRaghu Vatsavayi droq->pkts_per_intr = c_pkts_per_intr; 296f21fb3edSRaghu Vatsavayi droq->refill_threshold = c_refill_threshold; 297f21fb3edSRaghu Vatsavayi 298f21fb3edSRaghu Vatsavayi dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 299f21fb3edSRaghu Vatsavayi droq->max_empty_descs); 300f21fb3edSRaghu Vatsavayi 301f21fb3edSRaghu Vatsavayi INIT_LIST_HEAD(&droq->dispatch_list); 302f21fb3edSRaghu Vatsavayi 303f21fb3edSRaghu Vatsavayi /* For 56xx Pass1, this function won't be called, so no checks. */ 304f21fb3edSRaghu Vatsavayi oct->fn_list.setup_oq_regs(oct, q_no); 305f21fb3edSRaghu Vatsavayi 306763185a3SRaghu Vatsavayi oct->io_qmask.oq |= BIT_ULL(q_no); 307f21fb3edSRaghu Vatsavayi 308f21fb3edSRaghu Vatsavayi return 0; 309f21fb3edSRaghu Vatsavayi 310f21fb3edSRaghu Vatsavayi init_droq_fail: 311f21fb3edSRaghu Vatsavayi octeon_delete_droq(oct, q_no); 312f21fb3edSRaghu Vatsavayi return 1; 313f21fb3edSRaghu Vatsavayi } 314f21fb3edSRaghu Vatsavayi 315f21fb3edSRaghu Vatsavayi /* octeon_create_recv_info 316f21fb3edSRaghu Vatsavayi * Parameters: 317f21fb3edSRaghu Vatsavayi * octeon_dev - pointer to the octeon device structure 318f21fb3edSRaghu Vatsavayi * droq - droq in which the packet arrived. 319f21fb3edSRaghu Vatsavayi * buf_cnt - no. of buffers used by the packet. 320f21fb3edSRaghu Vatsavayi * idx - index in the descriptor for the first buffer in the packet. 321f21fb3edSRaghu Vatsavayi * Description: 322f21fb3edSRaghu Vatsavayi * Allocates a recv_info_t and copies the buffer addresses for packet data 323f21fb3edSRaghu Vatsavayi * into the recv_pkt space which starts at an 8B offset from recv_info_t. 324f21fb3edSRaghu Vatsavayi * Flags the descriptors for refill later. If available descriptors go 325f21fb3edSRaghu Vatsavayi * below the threshold to receive a 64K pkt, new buffers are first allocated 326f21fb3edSRaghu Vatsavayi * before the recv_pkt_t is created. 327f21fb3edSRaghu Vatsavayi * This routine will be called in interrupt context. 328f21fb3edSRaghu Vatsavayi * Returns: 329f21fb3edSRaghu Vatsavayi * Success: Pointer to recv_info_t 330f21fb3edSRaghu Vatsavayi * Failure: NULL. 331f21fb3edSRaghu Vatsavayi */ 332f21fb3edSRaghu Vatsavayi static inline struct octeon_recv_info *octeon_create_recv_info( 333f21fb3edSRaghu Vatsavayi struct octeon_device *octeon_dev, 334f21fb3edSRaghu Vatsavayi struct octeon_droq *droq, 335f21fb3edSRaghu Vatsavayi u32 buf_cnt, 336f21fb3edSRaghu Vatsavayi u32 idx) 337f21fb3edSRaghu Vatsavayi { 338f21fb3edSRaghu Vatsavayi struct octeon_droq_info *info; 339f21fb3edSRaghu Vatsavayi struct octeon_recv_pkt *recv_pkt; 340f21fb3edSRaghu Vatsavayi struct octeon_recv_info *recv_info; 341f21fb3edSRaghu Vatsavayi u32 i, bytes_left; 342cabeb13bSRaghu Vatsavayi struct octeon_skb_page_info *pg_info; 343f21fb3edSRaghu Vatsavayi 344c4ee5d81SPrasad Kanneganti info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data; 345f21fb3edSRaghu Vatsavayi 346f21fb3edSRaghu Vatsavayi recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 347f21fb3edSRaghu Vatsavayi if (!recv_info) 348f21fb3edSRaghu Vatsavayi return NULL; 349f21fb3edSRaghu Vatsavayi 350f21fb3edSRaghu Vatsavayi recv_pkt = recv_info->recv_pkt; 351f21fb3edSRaghu Vatsavayi recv_pkt->rh = info->rh; 352f21fb3edSRaghu Vatsavayi recv_pkt->length = (u32)info->length; 353f21fb3edSRaghu Vatsavayi recv_pkt->buffer_count = (u16)buf_cnt; 354f21fb3edSRaghu Vatsavayi recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 355f21fb3edSRaghu Vatsavayi 356f21fb3edSRaghu Vatsavayi i = 0; 357f21fb3edSRaghu Vatsavayi bytes_left = (u32)info->length; 358f21fb3edSRaghu Vatsavayi 359f21fb3edSRaghu Vatsavayi while (buf_cnt) { 360cabeb13bSRaghu Vatsavayi { 361cabeb13bSRaghu Vatsavayi pg_info = &droq->recv_buf_list[idx].pg_info; 362cabeb13bSRaghu Vatsavayi 363f21fb3edSRaghu Vatsavayi lio_unmap_ring(octeon_dev->pci_dev, 364cabeb13bSRaghu Vatsavayi (u64)pg_info->dma); 365cabeb13bSRaghu Vatsavayi pg_info->page = NULL; 366cabeb13bSRaghu Vatsavayi pg_info->dma = 0; 367cabeb13bSRaghu Vatsavayi } 368f21fb3edSRaghu Vatsavayi 369f21fb3edSRaghu Vatsavayi recv_pkt->buffer_size[i] = 370f21fb3edSRaghu Vatsavayi (bytes_left >= 371f21fb3edSRaghu Vatsavayi droq->buffer_size) ? droq->buffer_size : bytes_left; 372f21fb3edSRaghu Vatsavayi 373f21fb3edSRaghu Vatsavayi recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 374f21fb3edSRaghu Vatsavayi droq->recv_buf_list[idx].buffer = NULL; 375f21fb3edSRaghu Vatsavayi 37697a25326SRaghu Vatsavayi idx = incr_index(idx, 1, droq->max_count); 377f21fb3edSRaghu Vatsavayi bytes_left -= droq->buffer_size; 378f21fb3edSRaghu Vatsavayi i++; 379f21fb3edSRaghu Vatsavayi buf_cnt--; 380f21fb3edSRaghu Vatsavayi } 381f21fb3edSRaghu Vatsavayi 382f21fb3edSRaghu Vatsavayi return recv_info; 383f21fb3edSRaghu Vatsavayi } 384f21fb3edSRaghu Vatsavayi 385f21fb3edSRaghu Vatsavayi /* If we were not able to refill all buffers, try to move around 386f21fb3edSRaghu Vatsavayi * the buffers that were not dispatched. 387f21fb3edSRaghu Vatsavayi */ 388f21fb3edSRaghu Vatsavayi static inline u32 389f21fb3edSRaghu Vatsavayi octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 390f21fb3edSRaghu Vatsavayi struct octeon_droq_desc *desc_ring) 391f21fb3edSRaghu Vatsavayi { 392f21fb3edSRaghu Vatsavayi u32 desc_refilled = 0; 393f21fb3edSRaghu Vatsavayi 394f21fb3edSRaghu Vatsavayi u32 refill_index = droq->refill_idx; 395f21fb3edSRaghu Vatsavayi 396f21fb3edSRaghu Vatsavayi while (refill_index != droq->read_idx) { 397f21fb3edSRaghu Vatsavayi if (droq->recv_buf_list[refill_index].buffer) { 398f21fb3edSRaghu Vatsavayi droq->recv_buf_list[droq->refill_idx].buffer = 399f21fb3edSRaghu Vatsavayi droq->recv_buf_list[refill_index].buffer; 400f21fb3edSRaghu Vatsavayi droq->recv_buf_list[droq->refill_idx].data = 401f21fb3edSRaghu Vatsavayi droq->recv_buf_list[refill_index].data; 402f21fb3edSRaghu Vatsavayi desc_ring[droq->refill_idx].buffer_ptr = 403f21fb3edSRaghu Vatsavayi desc_ring[refill_index].buffer_ptr; 404f21fb3edSRaghu Vatsavayi droq->recv_buf_list[refill_index].buffer = NULL; 405f21fb3edSRaghu Vatsavayi desc_ring[refill_index].buffer_ptr = 0; 406f21fb3edSRaghu Vatsavayi do { 40797a25326SRaghu Vatsavayi droq->refill_idx = incr_index(droq->refill_idx, 40897a25326SRaghu Vatsavayi 1, 409f21fb3edSRaghu Vatsavayi droq->max_count); 410f21fb3edSRaghu Vatsavayi desc_refilled++; 411f21fb3edSRaghu Vatsavayi droq->refill_count--; 4129ae122c6SSatanand Burla } while (droq->recv_buf_list[droq->refill_idx].buffer); 413f21fb3edSRaghu Vatsavayi } 41497a25326SRaghu Vatsavayi refill_index = incr_index(refill_index, 1, droq->max_count); 415f21fb3edSRaghu Vatsavayi } /* while */ 416f21fb3edSRaghu Vatsavayi return desc_refilled; 417f21fb3edSRaghu Vatsavayi } 418f21fb3edSRaghu Vatsavayi 419f21fb3edSRaghu Vatsavayi /* octeon_droq_refill 420f21fb3edSRaghu Vatsavayi * Parameters: 421f21fb3edSRaghu Vatsavayi * droq - droq in which descriptors require new buffers. 422f21fb3edSRaghu Vatsavayi * Description: 423f21fb3edSRaghu Vatsavayi * Called during normal DROQ processing in interrupt mode or by the poll 424f21fb3edSRaghu Vatsavayi * thread to refill the descriptors from which buffers were dispatched 425f21fb3edSRaghu Vatsavayi * to upper layers. Attempts to allocate new buffers. If that fails, moves 426f21fb3edSRaghu Vatsavayi * up buffers (that were not dispatched) to form a contiguous ring. 427f21fb3edSRaghu Vatsavayi * Returns: 428f21fb3edSRaghu Vatsavayi * No of descriptors refilled. 429f21fb3edSRaghu Vatsavayi */ 430f21fb3edSRaghu Vatsavayi static u32 431f21fb3edSRaghu Vatsavayi octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 432f21fb3edSRaghu Vatsavayi { 433f21fb3edSRaghu Vatsavayi struct octeon_droq_desc *desc_ring; 434f21fb3edSRaghu Vatsavayi void *buf = NULL; 435f21fb3edSRaghu Vatsavayi u8 *data; 436f21fb3edSRaghu Vatsavayi u32 desc_refilled = 0; 437cabeb13bSRaghu Vatsavayi struct octeon_skb_page_info *pg_info; 438f21fb3edSRaghu Vatsavayi 439f21fb3edSRaghu Vatsavayi desc_ring = droq->desc_ring; 440f21fb3edSRaghu Vatsavayi 441f21fb3edSRaghu Vatsavayi while (droq->refill_count && (desc_refilled < droq->max_count)) { 442f21fb3edSRaghu Vatsavayi /* If a valid buffer exists (happens if there is no dispatch), 4434b6e326bSIntiyaz Basha * reuse the buffer, else allocate. 444f21fb3edSRaghu Vatsavayi */ 445f21fb3edSRaghu Vatsavayi if (!droq->recv_buf_list[droq->refill_idx].buffer) { 446cabeb13bSRaghu Vatsavayi pg_info = 447cabeb13bSRaghu Vatsavayi &droq->recv_buf_list[droq->refill_idx].pg_info; 448cabeb13bSRaghu Vatsavayi /* Either recycle the existing pages or go for 449cabeb13bSRaghu Vatsavayi * new page alloc 450cabeb13bSRaghu Vatsavayi */ 451cabeb13bSRaghu Vatsavayi if (pg_info->page) 452cabeb13bSRaghu Vatsavayi buf = recv_buffer_reuse(octeon_dev, pg_info); 453cabeb13bSRaghu Vatsavayi else 454cabeb13bSRaghu Vatsavayi buf = recv_buffer_alloc(octeon_dev, pg_info); 455f21fb3edSRaghu Vatsavayi /* If a buffer could not be allocated, no point in 456f21fb3edSRaghu Vatsavayi * continuing 457f21fb3edSRaghu Vatsavayi */ 458cabeb13bSRaghu Vatsavayi if (!buf) { 459cabeb13bSRaghu Vatsavayi droq->stats.rx_alloc_failure++; 460f21fb3edSRaghu Vatsavayi break; 461cabeb13bSRaghu Vatsavayi } 462f21fb3edSRaghu Vatsavayi droq->recv_buf_list[droq->refill_idx].buffer = 463f21fb3edSRaghu Vatsavayi buf; 464f21fb3edSRaghu Vatsavayi data = get_rbd(buf); 465f21fb3edSRaghu Vatsavayi } else { 466f21fb3edSRaghu Vatsavayi data = get_rbd(droq->recv_buf_list 467f21fb3edSRaghu Vatsavayi [droq->refill_idx].buffer); 468f21fb3edSRaghu Vatsavayi } 469f21fb3edSRaghu Vatsavayi 470f21fb3edSRaghu Vatsavayi droq->recv_buf_list[droq->refill_idx].data = data; 471f21fb3edSRaghu Vatsavayi 472f21fb3edSRaghu Vatsavayi desc_ring[droq->refill_idx].buffer_ptr = 4739ae122c6SSatanand Burla lio_map_ring(droq->recv_buf_list[ 4749ae122c6SSatanand Burla droq->refill_idx].buffer); 475f21fb3edSRaghu Vatsavayi 47697a25326SRaghu Vatsavayi droq->refill_idx = incr_index(droq->refill_idx, 1, 47797a25326SRaghu Vatsavayi droq->max_count); 478f21fb3edSRaghu Vatsavayi desc_refilled++; 479f21fb3edSRaghu Vatsavayi droq->refill_count--; 480f21fb3edSRaghu Vatsavayi } 481f21fb3edSRaghu Vatsavayi 482f21fb3edSRaghu Vatsavayi if (droq->refill_count) 483f21fb3edSRaghu Vatsavayi desc_refilled += 484f21fb3edSRaghu Vatsavayi octeon_droq_refill_pullup_descs(droq, desc_ring); 485f21fb3edSRaghu Vatsavayi 486f21fb3edSRaghu Vatsavayi /* if droq->refill_count 487f21fb3edSRaghu Vatsavayi * The refill count would not change in pass two. We only moved buffers 488f21fb3edSRaghu Vatsavayi * to close the gap in the ring, but we would still have the same no. of 489f21fb3edSRaghu Vatsavayi * buffers to refill. 490f21fb3edSRaghu Vatsavayi */ 491f21fb3edSRaghu Vatsavayi return desc_refilled; 492f21fb3edSRaghu Vatsavayi } 493f21fb3edSRaghu Vatsavayi 494031d4f12SSatanand Burla /** check if we can allocate packets to get out of oom. 495031d4f12SSatanand Burla * @param droq - Droq being checked. 4964b6e326bSIntiyaz Basha * @return 1 if fails to refill minimum 497031d4f12SSatanand Burla */ 4984b6e326bSIntiyaz Basha int octeon_retry_droq_refill(struct octeon_droq *droq) 499031d4f12SSatanand Burla { 500031d4f12SSatanand Burla struct octeon_device *oct = droq->oct_dev; 5014b6e326bSIntiyaz Basha int desc_refilled, reschedule = 1; 5024b6e326bSIntiyaz Basha u32 pkts_credit; 503031d4f12SSatanand Burla 5044b6e326bSIntiyaz Basha pkts_credit = readl(droq->pkts_credit_reg); 505031d4f12SSatanand Burla desc_refilled = octeon_droq_refill(oct, droq); 506031d4f12SSatanand Burla if (desc_refilled) { 507031d4f12SSatanand Burla /* Flush the droq descriptor data to memory to be sure 508031d4f12SSatanand Burla * that when we update the credits the data in memory 509031d4f12SSatanand Burla * is accurate. 510031d4f12SSatanand Burla */ 511031d4f12SSatanand Burla wmb(); 512031d4f12SSatanand Burla writel(desc_refilled, droq->pkts_credit_reg); 5134b6e326bSIntiyaz Basha 5144b6e326bSIntiyaz Basha if (pkts_credit + desc_refilled >= CN23XX_SLI_DEF_BP) 5154b6e326bSIntiyaz Basha reschedule = 0; 516031d4f12SSatanand Burla } 5174b6e326bSIntiyaz Basha 5184b6e326bSIntiyaz Basha return reschedule; 519031d4f12SSatanand Burla } 520031d4f12SSatanand Burla 521f21fb3edSRaghu Vatsavayi static inline u32 522f21fb3edSRaghu Vatsavayi octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 523f21fb3edSRaghu Vatsavayi { 524f8a1988fSzhong jiang return DIV_ROUND_UP(total_len, buf_size); 525f21fb3edSRaghu Vatsavayi } 526f21fb3edSRaghu Vatsavayi 527f21fb3edSRaghu Vatsavayi static int 528f21fb3edSRaghu Vatsavayi octeon_droq_dispatch_pkt(struct octeon_device *oct, 529f21fb3edSRaghu Vatsavayi struct octeon_droq *droq, 530f21fb3edSRaghu Vatsavayi union octeon_rh *rh, 531f21fb3edSRaghu Vatsavayi struct octeon_droq_info *info) 532f21fb3edSRaghu Vatsavayi { 533f21fb3edSRaghu Vatsavayi u32 cnt; 534f21fb3edSRaghu Vatsavayi octeon_dispatch_fn_t disp_fn; 535f21fb3edSRaghu Vatsavayi struct octeon_recv_info *rinfo; 536f21fb3edSRaghu Vatsavayi 537f21fb3edSRaghu Vatsavayi cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 538f21fb3edSRaghu Vatsavayi 539f21fb3edSRaghu Vatsavayi disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 540f21fb3edSRaghu Vatsavayi (u16)rh->r.subcode); 541f21fb3edSRaghu Vatsavayi if (disp_fn) { 542f21fb3edSRaghu Vatsavayi rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 543f21fb3edSRaghu Vatsavayi if (rinfo) { 544f21fb3edSRaghu Vatsavayi struct __dispatch *rdisp = rinfo->rsvd; 545f21fb3edSRaghu Vatsavayi 546f21fb3edSRaghu Vatsavayi rdisp->rinfo = rinfo; 547f21fb3edSRaghu Vatsavayi rdisp->disp_fn = disp_fn; 548f21fb3edSRaghu Vatsavayi rinfo->recv_pkt->rh = *rh; 549f21fb3edSRaghu Vatsavayi list_add_tail(&rdisp->list, 550f21fb3edSRaghu Vatsavayi &droq->dispatch_list); 551f21fb3edSRaghu Vatsavayi } else { 552f21fb3edSRaghu Vatsavayi droq->stats.dropped_nomem++; 553f21fb3edSRaghu Vatsavayi } 554f21fb3edSRaghu Vatsavayi } else { 555a2c64b67SRaghu Vatsavayi dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", 556a2c64b67SRaghu Vatsavayi (unsigned int)rh->r.opcode, 557a2c64b67SRaghu Vatsavayi (unsigned int)rh->r.subcode); 558f21fb3edSRaghu Vatsavayi droq->stats.dropped_nodispatch++; 5599ded1a51SRaghu Vatsavayi } 560f21fb3edSRaghu Vatsavayi 561f21fb3edSRaghu Vatsavayi return cnt; 562f21fb3edSRaghu Vatsavayi } 563f21fb3edSRaghu Vatsavayi 564f21fb3edSRaghu Vatsavayi static inline void octeon_droq_drop_packets(struct octeon_device *oct, 565f21fb3edSRaghu Vatsavayi struct octeon_droq *droq, 566f21fb3edSRaghu Vatsavayi u32 cnt) 567f21fb3edSRaghu Vatsavayi { 568f21fb3edSRaghu Vatsavayi u32 i = 0, buf_cnt; 569f21fb3edSRaghu Vatsavayi struct octeon_droq_info *info; 570f21fb3edSRaghu Vatsavayi 571f21fb3edSRaghu Vatsavayi for (i = 0; i < cnt; i++) { 572c4ee5d81SPrasad Kanneganti info = (struct octeon_droq_info *) 573c4ee5d81SPrasad Kanneganti droq->recv_buf_list[droq->read_idx].data; 574f21fb3edSRaghu Vatsavayi octeon_swap_8B_data((u64 *)info, 2); 575f21fb3edSRaghu Vatsavayi 576f21fb3edSRaghu Vatsavayi if (info->length) { 577c4ee5d81SPrasad Kanneganti info->length += OCTNET_FRM_LENGTH_SIZE; 578f21fb3edSRaghu Vatsavayi droq->stats.bytes_received += info->length; 579f21fb3edSRaghu Vatsavayi buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 580f21fb3edSRaghu Vatsavayi (u32)info->length); 581f21fb3edSRaghu Vatsavayi } else { 582f21fb3edSRaghu Vatsavayi dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 583f21fb3edSRaghu Vatsavayi buf_cnt = 1; 584f21fb3edSRaghu Vatsavayi } 585f21fb3edSRaghu Vatsavayi 58697a25326SRaghu Vatsavayi droq->read_idx = incr_index(droq->read_idx, buf_cnt, 58797a25326SRaghu Vatsavayi droq->max_count); 588f21fb3edSRaghu Vatsavayi droq->refill_count += buf_cnt; 589f21fb3edSRaghu Vatsavayi } 590f21fb3edSRaghu Vatsavayi } 591f21fb3edSRaghu Vatsavayi 592f21fb3edSRaghu Vatsavayi static u32 593f21fb3edSRaghu Vatsavayi octeon_droq_fast_process_packets(struct octeon_device *oct, 594f21fb3edSRaghu Vatsavayi struct octeon_droq *droq, 595f21fb3edSRaghu Vatsavayi u32 pkts_to_process) 596f21fb3edSRaghu Vatsavayi { 5974b6e326bSIntiyaz Basha u32 pkt, total_len = 0, pkt_count, retval; 598f21fb3edSRaghu Vatsavayi struct octeon_droq_info *info; 599f21fb3edSRaghu Vatsavayi union octeon_rh *rh; 600f21fb3edSRaghu Vatsavayi 601f21fb3edSRaghu Vatsavayi pkt_count = pkts_to_process; 602f21fb3edSRaghu Vatsavayi 603f21fb3edSRaghu Vatsavayi for (pkt = 0; pkt < pkt_count; pkt++) { 604f21fb3edSRaghu Vatsavayi u32 pkt_len = 0; 605f21fb3edSRaghu Vatsavayi struct sk_buff *nicbuf = NULL; 606cabeb13bSRaghu Vatsavayi struct octeon_skb_page_info *pg_info; 607cabeb13bSRaghu Vatsavayi void *buf; 608f21fb3edSRaghu Vatsavayi 609c4ee5d81SPrasad Kanneganti info = (struct octeon_droq_info *) 610c4ee5d81SPrasad Kanneganti droq->recv_buf_list[droq->read_idx].data; 611f21fb3edSRaghu Vatsavayi octeon_swap_8B_data((u64 *)info, 2); 612f21fb3edSRaghu Vatsavayi 613f21fb3edSRaghu Vatsavayi if (!info->length) { 614f21fb3edSRaghu Vatsavayi dev_err(&oct->pci_dev->dev, 615f21fb3edSRaghu Vatsavayi "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 616f21fb3edSRaghu Vatsavayi droq->q_no, droq->read_idx, pkt_count); 617f21fb3edSRaghu Vatsavayi print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 618f21fb3edSRaghu Vatsavayi (u8 *)info, 619f21fb3edSRaghu Vatsavayi OCT_DROQ_INFO_SIZE); 620f21fb3edSRaghu Vatsavayi break; 621f21fb3edSRaghu Vatsavayi } 622f21fb3edSRaghu Vatsavayi 623f21fb3edSRaghu Vatsavayi /* Len of resp hdr in included in the received data len. */ 624f21fb3edSRaghu Vatsavayi rh = &info->rh; 625f21fb3edSRaghu Vatsavayi 626c4ee5d81SPrasad Kanneganti info->length += OCTNET_FRM_LENGTH_SIZE; 627c4ee5d81SPrasad Kanneganti rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64)); 628f21fb3edSRaghu Vatsavayi total_len += (u32)info->length; 62997a25326SRaghu Vatsavayi if (opcode_slow_path(rh)) { 630f21fb3edSRaghu Vatsavayi u32 buf_cnt; 631f21fb3edSRaghu Vatsavayi 632f21fb3edSRaghu Vatsavayi buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 63397a25326SRaghu Vatsavayi droq->read_idx = incr_index(droq->read_idx, 63497a25326SRaghu Vatsavayi buf_cnt, droq->max_count); 635f21fb3edSRaghu Vatsavayi droq->refill_count += buf_cnt; 636f21fb3edSRaghu Vatsavayi } else { 637f21fb3edSRaghu Vatsavayi if (info->length <= droq->buffer_size) { 638f21fb3edSRaghu Vatsavayi pkt_len = (u32)info->length; 639f21fb3edSRaghu Vatsavayi nicbuf = droq->recv_buf_list[ 640f21fb3edSRaghu Vatsavayi droq->read_idx].buffer; 641cabeb13bSRaghu Vatsavayi pg_info = &droq->recv_buf_list[ 642cabeb13bSRaghu Vatsavayi droq->read_idx].pg_info; 643cabeb13bSRaghu Vatsavayi if (recv_buffer_recycle(oct, pg_info)) 644cabeb13bSRaghu Vatsavayi pg_info->page = NULL; 645f21fb3edSRaghu Vatsavayi droq->recv_buf_list[droq->read_idx].buffer = 646f21fb3edSRaghu Vatsavayi NULL; 647a2c64b67SRaghu Vatsavayi 64897a25326SRaghu Vatsavayi droq->read_idx = incr_index(droq->read_idx, 1, 64997a25326SRaghu Vatsavayi droq->max_count); 650f21fb3edSRaghu Vatsavayi droq->refill_count++; 651f21fb3edSRaghu Vatsavayi } else { 652cabeb13bSRaghu Vatsavayi nicbuf = octeon_fast_packet_alloc((u32) 653f21fb3edSRaghu Vatsavayi info->length); 654f21fb3edSRaghu Vatsavayi pkt_len = 0; 655f21fb3edSRaghu Vatsavayi /* nicbuf allocation can fail. We'll handle it 656f21fb3edSRaghu Vatsavayi * inside the loop. 657f21fb3edSRaghu Vatsavayi */ 658f21fb3edSRaghu Vatsavayi while (pkt_len < info->length) { 659cabeb13bSRaghu Vatsavayi int cpy_len, idx = droq->read_idx; 660f21fb3edSRaghu Vatsavayi 661cabeb13bSRaghu Vatsavayi cpy_len = ((pkt_len + droq->buffer_size) 662cabeb13bSRaghu Vatsavayi > info->length) ? 663f21fb3edSRaghu Vatsavayi ((u32)info->length - pkt_len) : 664f21fb3edSRaghu Vatsavayi droq->buffer_size; 665f21fb3edSRaghu Vatsavayi 666f21fb3edSRaghu Vatsavayi if (nicbuf) { 667f21fb3edSRaghu Vatsavayi octeon_fast_packet_next(droq, 668f21fb3edSRaghu Vatsavayi nicbuf, 669f21fb3edSRaghu Vatsavayi cpy_len, 670cabeb13bSRaghu Vatsavayi idx); 6719ae122c6SSatanand Burla buf = droq->recv_buf_list[ 6729ae122c6SSatanand Burla idx].buffer; 673cabeb13bSRaghu Vatsavayi recv_buffer_fast_free(buf); 674cabeb13bSRaghu Vatsavayi droq->recv_buf_list[idx].buffer 675cabeb13bSRaghu Vatsavayi = NULL; 676cabeb13bSRaghu Vatsavayi } else { 677cabeb13bSRaghu Vatsavayi droq->stats.rx_alloc_failure++; 678f21fb3edSRaghu Vatsavayi } 679f21fb3edSRaghu Vatsavayi 680f21fb3edSRaghu Vatsavayi pkt_len += cpy_len; 68197a25326SRaghu Vatsavayi droq->read_idx = 68297a25326SRaghu Vatsavayi incr_index(droq->read_idx, 1, 683f21fb3edSRaghu Vatsavayi droq->max_count); 684f21fb3edSRaghu Vatsavayi droq->refill_count++; 685f21fb3edSRaghu Vatsavayi } 686f21fb3edSRaghu Vatsavayi } 687f21fb3edSRaghu Vatsavayi 688f21fb3edSRaghu Vatsavayi if (nicbuf) { 689cabeb13bSRaghu Vatsavayi if (droq->ops.fptr) { 690f21fb3edSRaghu Vatsavayi droq->ops.fptr(oct->octeon_id, 691f21fb3edSRaghu Vatsavayi nicbuf, pkt_len, 6920cece6c5SRaghu Vatsavayi rh, &droq->napi, 6930cece6c5SRaghu Vatsavayi droq->ops.farg); 694cabeb13bSRaghu Vatsavayi } else { 695f21fb3edSRaghu Vatsavayi recv_buffer_free(nicbuf); 696f21fb3edSRaghu Vatsavayi } 697f21fb3edSRaghu Vatsavayi } 698cabeb13bSRaghu Vatsavayi } 699f21fb3edSRaghu Vatsavayi 700f21fb3edSRaghu Vatsavayi if (droq->refill_count >= droq->refill_threshold) { 701f21fb3edSRaghu Vatsavayi int desc_refilled = octeon_droq_refill(oct, droq); 702f21fb3edSRaghu Vatsavayi 7034b6e326bSIntiyaz Basha if (desc_refilled) { 7044b6e326bSIntiyaz Basha /* Flush the droq descriptor data to memory to 7054b6e326bSIntiyaz Basha * be sure that when we update the credits the 7064b6e326bSIntiyaz Basha * data in memory is accurate. 707f21fb3edSRaghu Vatsavayi */ 708f21fb3edSRaghu Vatsavayi wmb(); 7094b6e326bSIntiyaz Basha writel(desc_refilled, droq->pkts_credit_reg); 710f21fb3edSRaghu Vatsavayi } 7114b6e326bSIntiyaz Basha } 712f21fb3edSRaghu Vatsavayi } /* for (each packet)... */ 713f21fb3edSRaghu Vatsavayi 714f21fb3edSRaghu Vatsavayi /* Increment refill_count by the number of buffers processed. */ 715f21fb3edSRaghu Vatsavayi droq->stats.pkts_received += pkt; 716f21fb3edSRaghu Vatsavayi droq->stats.bytes_received += total_len; 717f21fb3edSRaghu Vatsavayi 7184b6e326bSIntiyaz Basha retval = pkt; 719f21fb3edSRaghu Vatsavayi if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 720f21fb3edSRaghu Vatsavayi octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 721f21fb3edSRaghu Vatsavayi 722f21fb3edSRaghu Vatsavayi droq->stats.dropped_toomany += (pkts_to_process - pkt); 7234b6e326bSIntiyaz Basha retval = pkts_to_process; 724f21fb3edSRaghu Vatsavayi } 725f21fb3edSRaghu Vatsavayi 7264b6e326bSIntiyaz Basha atomic_sub(retval, &droq->pkts_pending); 7274b6e326bSIntiyaz Basha 7284b6e326bSIntiyaz Basha if (droq->refill_count >= droq->refill_threshold && 7294b6e326bSIntiyaz Basha readl(droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) { 7304b6e326bSIntiyaz Basha octeon_droq_check_hw_for_pkts(droq); 7314b6e326bSIntiyaz Basha 7324b6e326bSIntiyaz Basha /* Make sure there are no pkts_pending */ 7334b6e326bSIntiyaz Basha if (!atomic_read(&droq->pkts_pending)) 7344b6e326bSIntiyaz Basha octeon_schedule_rxq_oom_work(oct, droq); 7354b6e326bSIntiyaz Basha } 7364b6e326bSIntiyaz Basha 7374b6e326bSIntiyaz Basha return retval; 738f21fb3edSRaghu Vatsavayi } 739f21fb3edSRaghu Vatsavayi 740f21fb3edSRaghu Vatsavayi int 741f21fb3edSRaghu Vatsavayi octeon_droq_process_packets(struct octeon_device *oct, 742f21fb3edSRaghu Vatsavayi struct octeon_droq *droq, 743f21fb3edSRaghu Vatsavayi u32 budget) 744f21fb3edSRaghu Vatsavayi { 7454b6e326bSIntiyaz Basha u32 pkt_count = 0; 746f21fb3edSRaghu Vatsavayi struct list_head *tmp, *tmp2; 747f21fb3edSRaghu Vatsavayi 748cd8b1eb4SRaghu Vatsavayi octeon_droq_check_hw_for_pkts(droq); 749f21fb3edSRaghu Vatsavayi pkt_count = atomic_read(&droq->pkts_pending); 750cd8b1eb4SRaghu Vatsavayi 7518bf6edcdSIntiyaz Basha if (!pkt_count) 752f21fb3edSRaghu Vatsavayi return 0; 753f21fb3edSRaghu Vatsavayi 754f21fb3edSRaghu Vatsavayi if (pkt_count > budget) 755f21fb3edSRaghu Vatsavayi pkt_count = budget; 756f21fb3edSRaghu Vatsavayi 7574b6e326bSIntiyaz Basha octeon_droq_fast_process_packets(oct, droq, pkt_count); 758f21fb3edSRaghu Vatsavayi 759f21fb3edSRaghu Vatsavayi list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 760f21fb3edSRaghu Vatsavayi struct __dispatch *rdisp = (struct __dispatch *)tmp; 761f21fb3edSRaghu Vatsavayi 762f21fb3edSRaghu Vatsavayi list_del(tmp); 763f21fb3edSRaghu Vatsavayi rdisp->disp_fn(rdisp->rinfo, 764f21fb3edSRaghu Vatsavayi octeon_get_dispatch_arg 765f21fb3edSRaghu Vatsavayi (oct, 766f21fb3edSRaghu Vatsavayi (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 767f21fb3edSRaghu Vatsavayi (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 768f21fb3edSRaghu Vatsavayi } 769f21fb3edSRaghu Vatsavayi 770f21fb3edSRaghu Vatsavayi /* If there are packets pending. schedule tasklet again */ 771f21fb3edSRaghu Vatsavayi if (atomic_read(&droq->pkts_pending)) 772f21fb3edSRaghu Vatsavayi return 1; 773f21fb3edSRaghu Vatsavayi 774f21fb3edSRaghu Vatsavayi return 0; 775f21fb3edSRaghu Vatsavayi } 776f21fb3edSRaghu Vatsavayi 777f21fb3edSRaghu Vatsavayi /** 778f21fb3edSRaghu Vatsavayi * Utility function to poll for packets. check_hw_for_packets must be 779f21fb3edSRaghu Vatsavayi * called before calling this routine. 780f21fb3edSRaghu Vatsavayi */ 781f21fb3edSRaghu Vatsavayi 7825eb297a9SIntiyaz Basha int 783f21fb3edSRaghu Vatsavayi octeon_droq_process_poll_pkts(struct octeon_device *oct, 784f21fb3edSRaghu Vatsavayi struct octeon_droq *droq, u32 budget) 785f21fb3edSRaghu Vatsavayi { 786f21fb3edSRaghu Vatsavayi struct list_head *tmp, *tmp2; 787f21fb3edSRaghu Vatsavayi u32 pkts_available = 0, pkts_processed = 0; 788f21fb3edSRaghu Vatsavayi u32 total_pkts_processed = 0; 789f21fb3edSRaghu Vatsavayi 790f21fb3edSRaghu Vatsavayi if (budget > droq->max_count) 791f21fb3edSRaghu Vatsavayi budget = droq->max_count; 792f21fb3edSRaghu Vatsavayi 793f21fb3edSRaghu Vatsavayi while (total_pkts_processed < budget) { 794cd8b1eb4SRaghu Vatsavayi octeon_droq_check_hw_for_pkts(droq); 795cd8b1eb4SRaghu Vatsavayi 79697a25326SRaghu Vatsavayi pkts_available = min((budget - total_pkts_processed), 797f21fb3edSRaghu Vatsavayi (u32)(atomic_read(&droq->pkts_pending))); 798f21fb3edSRaghu Vatsavayi 799f21fb3edSRaghu Vatsavayi if (pkts_available == 0) 800f21fb3edSRaghu Vatsavayi break; 801f21fb3edSRaghu Vatsavayi 802f21fb3edSRaghu Vatsavayi pkts_processed = 803f21fb3edSRaghu Vatsavayi octeon_droq_fast_process_packets(oct, droq, 804f21fb3edSRaghu Vatsavayi pkts_available); 805f21fb3edSRaghu Vatsavayi 806f21fb3edSRaghu Vatsavayi total_pkts_processed += pkts_processed; 807f21fb3edSRaghu Vatsavayi } 808f21fb3edSRaghu Vatsavayi 809f21fb3edSRaghu Vatsavayi list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 810f21fb3edSRaghu Vatsavayi struct __dispatch *rdisp = (struct __dispatch *)tmp; 811f21fb3edSRaghu Vatsavayi 812f21fb3edSRaghu Vatsavayi list_del(tmp); 813f21fb3edSRaghu Vatsavayi rdisp->disp_fn(rdisp->rinfo, 814f21fb3edSRaghu Vatsavayi octeon_get_dispatch_arg 815f21fb3edSRaghu Vatsavayi (oct, 816f21fb3edSRaghu Vatsavayi (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 817f21fb3edSRaghu Vatsavayi (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 818f21fb3edSRaghu Vatsavayi } 819f21fb3edSRaghu Vatsavayi 820f21fb3edSRaghu Vatsavayi return total_pkts_processed; 821f21fb3edSRaghu Vatsavayi } 822f21fb3edSRaghu Vatsavayi 823f21fb3edSRaghu Vatsavayi /* Enable Pkt Interrupt */ 8245eb297a9SIntiyaz Basha int 8255eb297a9SIntiyaz Basha octeon_enable_irq(struct octeon_device *oct, u32 q_no) 8265eb297a9SIntiyaz Basha { 827f21fb3edSRaghu Vatsavayi switch (oct->chip_id) { 828f21fb3edSRaghu Vatsavayi case OCTEON_CN66XX: 829f21fb3edSRaghu Vatsavayi case OCTEON_CN68XX: { 830f21fb3edSRaghu Vatsavayi struct octeon_cn6xxx *cn6xxx = 831f21fb3edSRaghu Vatsavayi (struct octeon_cn6xxx *)oct->chip; 8325eb297a9SIntiyaz Basha unsigned long flags; 8335eb297a9SIntiyaz Basha u32 value; 8345eb297a9SIntiyaz Basha 835f21fb3edSRaghu Vatsavayi spin_lock_irqsave 836f21fb3edSRaghu Vatsavayi (&cn6xxx->lock_for_droq_int_enb_reg, flags); 8375eb297a9SIntiyaz Basha value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB); 838f21fb3edSRaghu Vatsavayi value |= (1 << q_no); 8395eb297a9SIntiyaz Basha octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value); 8405eb297a9SIntiyaz Basha value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB); 841f21fb3edSRaghu Vatsavayi value |= (1 << q_no); 8425eb297a9SIntiyaz Basha octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value); 843f21fb3edSRaghu Vatsavayi 844f21fb3edSRaghu Vatsavayi /* don't bother flushing the enables */ 845f21fb3edSRaghu Vatsavayi 846f21fb3edSRaghu Vatsavayi spin_unlock_irqrestore 847f21fb3edSRaghu Vatsavayi (&cn6xxx->lock_for_droq_int_enb_reg, flags); 848f21fb3edSRaghu Vatsavayi } 849f21fb3edSRaghu Vatsavayi break; 8505eb297a9SIntiyaz Basha case OCTEON_CN23XX_PF_VID: 8519ded1a51SRaghu Vatsavayi lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 8529ded1a51SRaghu Vatsavayi break; 8539217c3cfSRaghu Vatsavayi 8549217c3cfSRaghu Vatsavayi case OCTEON_CN23XX_VF_VID: 8559217c3cfSRaghu Vatsavayi lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 8569217c3cfSRaghu Vatsavayi break; 8575eb297a9SIntiyaz Basha default: 8585eb297a9SIntiyaz Basha dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__); 8595eb297a9SIntiyaz Basha return 1; 860f21fb3edSRaghu Vatsavayi } 861f21fb3edSRaghu Vatsavayi 8625eb297a9SIntiyaz Basha return 0; 863f21fb3edSRaghu Vatsavayi } 864f21fb3edSRaghu Vatsavayi 865f21fb3edSRaghu Vatsavayi int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 866f21fb3edSRaghu Vatsavayi struct octeon_droq_ops *ops) 867f21fb3edSRaghu Vatsavayi { 868f21fb3edSRaghu Vatsavayi struct octeon_config *oct_cfg = NULL; 8698bf6edcdSIntiyaz Basha struct octeon_droq *droq; 870f21fb3edSRaghu Vatsavayi 871f21fb3edSRaghu Vatsavayi oct_cfg = octeon_get_conf(oct); 872f21fb3edSRaghu Vatsavayi 873f21fb3edSRaghu Vatsavayi if (!oct_cfg) 874f21fb3edSRaghu Vatsavayi return -EINVAL; 875f21fb3edSRaghu Vatsavayi 876f21fb3edSRaghu Vatsavayi if (!(ops)) { 877f21fb3edSRaghu Vatsavayi dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 878f21fb3edSRaghu Vatsavayi __func__); 879f21fb3edSRaghu Vatsavayi return -EINVAL; 880f21fb3edSRaghu Vatsavayi } 881f21fb3edSRaghu Vatsavayi 882f21fb3edSRaghu Vatsavayi if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 883f21fb3edSRaghu Vatsavayi dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 884f21fb3edSRaghu Vatsavayi __func__, q_no, (oct->num_oqs - 1)); 885f21fb3edSRaghu Vatsavayi return -EINVAL; 886f21fb3edSRaghu Vatsavayi } 887f21fb3edSRaghu Vatsavayi 888f21fb3edSRaghu Vatsavayi droq = oct->droq[q_no]; 889f21fb3edSRaghu Vatsavayi memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 890f21fb3edSRaghu Vatsavayi 891f21fb3edSRaghu Vatsavayi return 0; 892f21fb3edSRaghu Vatsavayi } 893f21fb3edSRaghu Vatsavayi 894f21fb3edSRaghu Vatsavayi int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 895f21fb3edSRaghu Vatsavayi { 896f21fb3edSRaghu Vatsavayi struct octeon_config *oct_cfg = NULL; 8978bf6edcdSIntiyaz Basha struct octeon_droq *droq; 898f21fb3edSRaghu Vatsavayi 899f21fb3edSRaghu Vatsavayi oct_cfg = octeon_get_conf(oct); 900f21fb3edSRaghu Vatsavayi 901f21fb3edSRaghu Vatsavayi if (!oct_cfg) 902f21fb3edSRaghu Vatsavayi return -EINVAL; 903f21fb3edSRaghu Vatsavayi 904f21fb3edSRaghu Vatsavayi if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 905f21fb3edSRaghu Vatsavayi dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 906f21fb3edSRaghu Vatsavayi __func__, q_no, oct->num_oqs - 1); 907f21fb3edSRaghu Vatsavayi return -EINVAL; 908f21fb3edSRaghu Vatsavayi } 909f21fb3edSRaghu Vatsavayi 910f21fb3edSRaghu Vatsavayi droq = oct->droq[q_no]; 911f21fb3edSRaghu Vatsavayi 912f21fb3edSRaghu Vatsavayi if (!droq) { 913f21fb3edSRaghu Vatsavayi dev_info(&oct->pci_dev->dev, 914f21fb3edSRaghu Vatsavayi "Droq id (%d) not available.\n", q_no); 915f21fb3edSRaghu Vatsavayi return 0; 916f21fb3edSRaghu Vatsavayi } 917f21fb3edSRaghu Vatsavayi 918f21fb3edSRaghu Vatsavayi droq->ops.fptr = NULL; 9190cece6c5SRaghu Vatsavayi droq->ops.farg = NULL; 920f21fb3edSRaghu Vatsavayi droq->ops.drop_on_max = 0; 921f21fb3edSRaghu Vatsavayi 922f21fb3edSRaghu Vatsavayi return 0; 923f21fb3edSRaghu Vatsavayi } 924f21fb3edSRaghu Vatsavayi 925f21fb3edSRaghu Vatsavayi int octeon_create_droq(struct octeon_device *oct, 926f21fb3edSRaghu Vatsavayi u32 q_no, u32 num_descs, 927f21fb3edSRaghu Vatsavayi u32 desc_size, void *app_ctx) 928f21fb3edSRaghu Vatsavayi { 929f21fb3edSRaghu Vatsavayi struct octeon_droq *droq; 930b3ca9af0SVSR Burru int numa_node = dev_to_node(&oct->pci_dev->dev); 931f21fb3edSRaghu Vatsavayi 932f21fb3edSRaghu Vatsavayi if (oct->droq[q_no]) { 933f21fb3edSRaghu Vatsavayi dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 934f21fb3edSRaghu Vatsavayi q_no); 935f21fb3edSRaghu Vatsavayi return 1; 936f21fb3edSRaghu Vatsavayi } 937f21fb3edSRaghu Vatsavayi 938f21fb3edSRaghu Vatsavayi /* Allocate the DS for the new droq. */ 93996ae48b7SRaghu Vatsavayi droq = vmalloc_node(sizeof(*droq), numa_node); 94096ae48b7SRaghu Vatsavayi if (!droq) 941f21fb3edSRaghu Vatsavayi droq = vmalloc(sizeof(*droq)); 942f21fb3edSRaghu Vatsavayi if (!droq) 943515e752dSRaghu Vatsavayi return -1; 944515e752dSRaghu Vatsavayi 945f21fb3edSRaghu Vatsavayi memset(droq, 0, sizeof(struct octeon_droq)); 946f21fb3edSRaghu Vatsavayi 947f21fb3edSRaghu Vatsavayi /*Disable the pkt o/p for this Q */ 948f21fb3edSRaghu Vatsavayi octeon_set_droq_pkt_op(oct, q_no, 0); 949f21fb3edSRaghu Vatsavayi oct->droq[q_no] = droq; 950f21fb3edSRaghu Vatsavayi 951f21fb3edSRaghu Vatsavayi /* Initialize the Droq */ 952515e752dSRaghu Vatsavayi if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { 953515e752dSRaghu Vatsavayi vfree(oct->droq[q_no]); 954515e752dSRaghu Vatsavayi oct->droq[q_no] = NULL; 955515e752dSRaghu Vatsavayi return -1; 956515e752dSRaghu Vatsavayi } 957f21fb3edSRaghu Vatsavayi 958f21fb3edSRaghu Vatsavayi oct->num_oqs++; 959f21fb3edSRaghu Vatsavayi 960f21fb3edSRaghu Vatsavayi dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 961f21fb3edSRaghu Vatsavayi oct->num_oqs); 962f21fb3edSRaghu Vatsavayi 963f21fb3edSRaghu Vatsavayi /* Global Droq register settings */ 964f21fb3edSRaghu Vatsavayi 965f21fb3edSRaghu Vatsavayi /* As of now not required, as setting are done for all 32 Droqs at 966f21fb3edSRaghu Vatsavayi * the same time. 967f21fb3edSRaghu Vatsavayi */ 968f21fb3edSRaghu Vatsavayi return 0; 969f21fb3edSRaghu Vatsavayi } 970