151dce24bSJeff Kirsher // SPDX-License-Identifier: GPL-2.0 251dce24bSJeff Kirsher /* Copyright(c) 1999 - 2018 Intel Corporation. */ 38af3c33fSJeff Kirsher 48af3c33fSJeff Kirsher #include "ixgbe.h" 58af3c33fSJeff Kirsher #include "ixgbe_sriov.h" 68af3c33fSJeff Kirsher 7800bd607SAlexander Duyck #ifdef CONFIG_IXGBE_DCB 873079ea0SAlexander Duyck /** 973079ea0SAlexander Duyck * ixgbe_cache_ring_dcb_sriov - Descriptor ring to register mapping for SR-IOV 1073079ea0SAlexander Duyck * @adapter: board private structure to initialize 1173079ea0SAlexander Duyck * 1273079ea0SAlexander Duyck * Cache the descriptor ring offsets for SR-IOV to the assigned rings. It 1373079ea0SAlexander Duyck * will also try to cache the proper offsets if RSS/FCoE are enabled along 1473079ea0SAlexander Duyck * with VMDq. 1573079ea0SAlexander Duyck * 1673079ea0SAlexander Duyck **/ 1773079ea0SAlexander Duyck static bool ixgbe_cache_ring_dcb_sriov(struct ixgbe_adapter *adapter) 1873079ea0SAlexander Duyck { 1973079ea0SAlexander Duyck #ifdef IXGBE_FCOE 2073079ea0SAlexander Duyck struct ixgbe_ring_feature *fcoe = &adapter->ring_feature[RING_F_FCOE]; 2173079ea0SAlexander Duyck #endif /* IXGBE_FCOE */ 2273079ea0SAlexander Duyck struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ]; 2373079ea0SAlexander Duyck int i; 24b5f69ccfSAlexander Duyck u16 reg_idx, pool; 250efbf12bSAlexander Duyck u8 tcs = adapter->hw_tcs; 2673079ea0SAlexander Duyck 2773079ea0SAlexander Duyck /* verify we have DCB queueing enabled before proceeding */ 2873079ea0SAlexander Duyck if (tcs <= 1) 2973079ea0SAlexander Duyck return false; 3073079ea0SAlexander Duyck 3173079ea0SAlexander Duyck /* verify we have VMDq enabled before proceeding */ 3273079ea0SAlexander Duyck if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 3373079ea0SAlexander Duyck return false; 3473079ea0SAlexander Duyck 3573079ea0SAlexander Duyck /* start at VMDq register offset for SR-IOV enabled setups */ 3673079ea0SAlexander Duyck reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask); 37b5f69ccfSAlexander Duyck for (i = 0, pool = 0; i < adapter->num_rx_queues; i++, reg_idx++) { 3873079ea0SAlexander Duyck /* If we are greater than indices move to next pool */ 39b5f69ccfSAlexander Duyck if ((reg_idx & ~vmdq->mask) >= tcs) { 40b5f69ccfSAlexander Duyck pool++; 4173079ea0SAlexander Duyck reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask); 42b5f69ccfSAlexander Duyck } 4373079ea0SAlexander Duyck adapter->rx_ring[i]->reg_idx = reg_idx; 44b5f69ccfSAlexander Duyck adapter->rx_ring[i]->netdev = pool ? NULL : adapter->netdev; 4573079ea0SAlexander Duyck } 4673079ea0SAlexander Duyck 4773079ea0SAlexander Duyck reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask); 4873079ea0SAlexander Duyck for (i = 0; i < adapter->num_tx_queues; i++, reg_idx++) { 4973079ea0SAlexander Duyck /* If we are greater than indices move to next pool */ 5073079ea0SAlexander Duyck if ((reg_idx & ~vmdq->mask) >= tcs) 5173079ea0SAlexander Duyck reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask); 5273079ea0SAlexander Duyck adapter->tx_ring[i]->reg_idx = reg_idx; 5373079ea0SAlexander Duyck } 5473079ea0SAlexander Duyck 5573079ea0SAlexander Duyck #ifdef IXGBE_FCOE 5673079ea0SAlexander Duyck /* nothing to do if FCoE is disabled */ 5773079ea0SAlexander Duyck if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) 5873079ea0SAlexander Duyck return true; 5973079ea0SAlexander Duyck 6073079ea0SAlexander Duyck /* The work is already done if the FCoE ring is shared */ 6173079ea0SAlexander Duyck if (fcoe->offset < tcs) 6273079ea0SAlexander Duyck return true; 6373079ea0SAlexander Duyck 6473079ea0SAlexander Duyck /* The FCoE rings exist separately, we need to move their reg_idx */ 6573079ea0SAlexander Duyck if (fcoe->indices) { 6673079ea0SAlexander Duyck u16 queues_per_pool = __ALIGN_MASK(1, ~vmdq->mask); 6773079ea0SAlexander Duyck u8 fcoe_tc = ixgbe_fcoe_get_tc(adapter); 6873079ea0SAlexander Duyck 6973079ea0SAlexander Duyck reg_idx = (vmdq->offset + vmdq->indices) * queues_per_pool; 7073079ea0SAlexander Duyck for (i = fcoe->offset; i < adapter->num_rx_queues; i++) { 7173079ea0SAlexander Duyck reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask) + fcoe_tc; 7273079ea0SAlexander Duyck adapter->rx_ring[i]->reg_idx = reg_idx; 73b5f69ccfSAlexander Duyck adapter->rx_ring[i]->netdev = adapter->netdev; 7473079ea0SAlexander Duyck reg_idx++; 7573079ea0SAlexander Duyck } 7673079ea0SAlexander Duyck 7773079ea0SAlexander Duyck reg_idx = (vmdq->offset + vmdq->indices) * queues_per_pool; 7873079ea0SAlexander Duyck for (i = fcoe->offset; i < adapter->num_tx_queues; i++) { 7973079ea0SAlexander Duyck reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask) + fcoe_tc; 8073079ea0SAlexander Duyck adapter->tx_ring[i]->reg_idx = reg_idx; 8173079ea0SAlexander Duyck reg_idx++; 8273079ea0SAlexander Duyck } 8373079ea0SAlexander Duyck } 8473079ea0SAlexander Duyck 8573079ea0SAlexander Duyck #endif /* IXGBE_FCOE */ 8673079ea0SAlexander Duyck return true; 8773079ea0SAlexander Duyck } 8873079ea0SAlexander Duyck 898af3c33fSJeff Kirsher /* ixgbe_get_first_reg_idx - Return first register index associated with ring */ 908af3c33fSJeff Kirsher static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc, 918af3c33fSJeff Kirsher unsigned int *tx, unsigned int *rx) 928af3c33fSJeff Kirsher { 938af3c33fSJeff Kirsher struct ixgbe_hw *hw = &adapter->hw; 940efbf12bSAlexander Duyck u8 num_tcs = adapter->hw_tcs; 958af3c33fSJeff Kirsher 968af3c33fSJeff Kirsher *tx = 0; 978af3c33fSJeff Kirsher *rx = 0; 988af3c33fSJeff Kirsher 998af3c33fSJeff Kirsher switch (hw->mac.type) { 1008af3c33fSJeff Kirsher case ixgbe_mac_82598EB: 1014ae63730SAlexander Duyck /* TxQs/TC: 4 RxQs/TC: 8 */ 1024ae63730SAlexander Duyck *tx = tc << 2; /* 0, 4, 8, 12, 16, 20, 24, 28 */ 1034ae63730SAlexander Duyck *rx = tc << 3; /* 0, 8, 16, 24, 32, 40, 48, 56 */ 1048af3c33fSJeff Kirsher break; 1058af3c33fSJeff Kirsher case ixgbe_mac_82599EB: 1068af3c33fSJeff Kirsher case ixgbe_mac_X540: 1079a75a1acSDon Skidmore case ixgbe_mac_X550: 1089a75a1acSDon Skidmore case ixgbe_mac_X550EM_x: 10949425dfcSMark Rustad case ixgbe_mac_x550em_a: 1108af3c33fSJeff Kirsher if (num_tcs > 4) { 1114ae63730SAlexander Duyck /* 1124ae63730SAlexander Duyck * TCs : TC0/1 TC2/3 TC4-7 1134ae63730SAlexander Duyck * TxQs/TC: 32 16 8 1144ae63730SAlexander Duyck * RxQs/TC: 16 16 16 1154ae63730SAlexander Duyck */ 1168af3c33fSJeff Kirsher *rx = tc << 4; 1174ae63730SAlexander Duyck if (tc < 3) 1184ae63730SAlexander Duyck *tx = tc << 5; /* 0, 32, 64 */ 1194ae63730SAlexander Duyck else if (tc < 5) 1204ae63730SAlexander Duyck *tx = (tc + 2) << 4; /* 80, 96 */ 1214ae63730SAlexander Duyck else 1224ae63730SAlexander Duyck *tx = (tc + 8) << 3; /* 104, 112, 120 */ 1238af3c33fSJeff Kirsher } else { 1244ae63730SAlexander Duyck /* 1254ae63730SAlexander Duyck * TCs : TC0 TC1 TC2/3 1264ae63730SAlexander Duyck * TxQs/TC: 64 32 16 1274ae63730SAlexander Duyck * RxQs/TC: 32 32 32 1284ae63730SAlexander Duyck */ 1298af3c33fSJeff Kirsher *rx = tc << 5; 1304ae63730SAlexander Duyck if (tc < 2) 1314ae63730SAlexander Duyck *tx = tc << 6; /* 0, 64 */ 1324ae63730SAlexander Duyck else 1334ae63730SAlexander Duyck *tx = (tc + 4) << 4; /* 96, 112 */ 1348af3c33fSJeff Kirsher } 13527e40255SGustavo A. R. Silva break; 1368af3c33fSJeff Kirsher default: 1378af3c33fSJeff Kirsher break; 1388af3c33fSJeff Kirsher } 1398af3c33fSJeff Kirsher } 1408af3c33fSJeff Kirsher 1418af3c33fSJeff Kirsher /** 1428af3c33fSJeff Kirsher * ixgbe_cache_ring_dcb - Descriptor ring to register mapping for DCB 1438af3c33fSJeff Kirsher * @adapter: board private structure to initialize 1448af3c33fSJeff Kirsher * 1458af3c33fSJeff Kirsher * Cache the descriptor ring offsets for DCB to the assigned rings. 1468af3c33fSJeff Kirsher * 1478af3c33fSJeff Kirsher **/ 1484ae63730SAlexander Duyck static bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter) 1498af3c33fSJeff Kirsher { 1500efbf12bSAlexander Duyck u8 num_tcs = adapter->hw_tcs; 1514ae63730SAlexander Duyck unsigned int tx_idx, rx_idx; 1524ae63730SAlexander Duyck int tc, offset, rss_i, i; 1538af3c33fSJeff Kirsher 1544ae63730SAlexander Duyck /* verify we have DCB queueing enabled before proceeding */ 1554ae63730SAlexander Duyck if (num_tcs <= 1) 1568af3c33fSJeff Kirsher return false; 1578af3c33fSJeff Kirsher 1584ae63730SAlexander Duyck rss_i = adapter->ring_feature[RING_F_RSS].indices; 1598af3c33fSJeff Kirsher 1604ae63730SAlexander Duyck for (tc = 0, offset = 0; tc < num_tcs; tc++, offset += rss_i) { 1614ae63730SAlexander Duyck ixgbe_get_first_reg_idx(adapter, tc, &tx_idx, &rx_idx); 1624ae63730SAlexander Duyck for (i = 0; i < rss_i; i++, tx_idx++, rx_idx++) { 1634ae63730SAlexander Duyck adapter->tx_ring[offset + i]->reg_idx = tx_idx; 1644ae63730SAlexander Duyck adapter->rx_ring[offset + i]->reg_idx = rx_idx; 165b5f69ccfSAlexander Duyck adapter->rx_ring[offset + i]->netdev = adapter->netdev; 1664ae63730SAlexander Duyck adapter->tx_ring[offset + i]->dcb_tc = tc; 1674ae63730SAlexander Duyck adapter->rx_ring[offset + i]->dcb_tc = tc; 1688af3c33fSJeff Kirsher } 1698af3c33fSJeff Kirsher } 1708af3c33fSJeff Kirsher 1718af3c33fSJeff Kirsher return true; 1728af3c33fSJeff Kirsher } 173d411a936SAlexander Duyck 1748af3c33fSJeff Kirsher #endif 1758af3c33fSJeff Kirsher /** 1768af3c33fSJeff Kirsher * ixgbe_cache_ring_sriov - Descriptor ring to register mapping for sriov 1778af3c33fSJeff Kirsher * @adapter: board private structure to initialize 1788af3c33fSJeff Kirsher * 1798af3c33fSJeff Kirsher * SR-IOV doesn't use any descriptor rings but changes the default if 1808af3c33fSJeff Kirsher * no other mapping is used. 1818af3c33fSJeff Kirsher * 1828af3c33fSJeff Kirsher */ 18373079ea0SAlexander Duyck static bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter) 1848af3c33fSJeff Kirsher { 18573079ea0SAlexander Duyck #ifdef IXGBE_FCOE 18673079ea0SAlexander Duyck struct ixgbe_ring_feature *fcoe = &adapter->ring_feature[RING_F_FCOE]; 18773079ea0SAlexander Duyck #endif /* IXGBE_FCOE */ 18873079ea0SAlexander Duyck struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ]; 18973079ea0SAlexander Duyck struct ixgbe_ring_feature *rss = &adapter->ring_feature[RING_F_RSS]; 190b5f69ccfSAlexander Duyck u16 reg_idx, pool; 19173079ea0SAlexander Duyck int i; 19273079ea0SAlexander Duyck 19373079ea0SAlexander Duyck /* only proceed if VMDq is enabled */ 19473079ea0SAlexander Duyck if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED)) 1958af3c33fSJeff Kirsher return false; 19673079ea0SAlexander Duyck 19773079ea0SAlexander Duyck /* start at VMDq register offset for SR-IOV enabled setups */ 198b5f69ccfSAlexander Duyck pool = 0; 19973079ea0SAlexander Duyck reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask); 20073079ea0SAlexander Duyck for (i = 0; i < adapter->num_rx_queues; i++, reg_idx++) { 20173079ea0SAlexander Duyck #ifdef IXGBE_FCOE 20273079ea0SAlexander Duyck /* Allow first FCoE queue to be mapped as RSS */ 20373079ea0SAlexander Duyck if (fcoe->offset && (i > fcoe->offset)) 20473079ea0SAlexander Duyck break; 20573079ea0SAlexander Duyck #endif 20673079ea0SAlexander Duyck /* If we are greater than indices move to next pool */ 207b5f69ccfSAlexander Duyck if ((reg_idx & ~vmdq->mask) >= rss->indices) { 208b5f69ccfSAlexander Duyck pool++; 20973079ea0SAlexander Duyck reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask); 210b5f69ccfSAlexander Duyck } 21173079ea0SAlexander Duyck adapter->rx_ring[i]->reg_idx = reg_idx; 212b5f69ccfSAlexander Duyck adapter->rx_ring[i]->netdev = pool ? NULL : adapter->netdev; 21373079ea0SAlexander Duyck } 21473079ea0SAlexander Duyck 21573079ea0SAlexander Duyck #ifdef IXGBE_FCOE 21673079ea0SAlexander Duyck /* FCoE uses a linear block of queues so just assigning 1:1 */ 217b5f69ccfSAlexander Duyck for (; i < adapter->num_rx_queues; i++, reg_idx++) { 21873079ea0SAlexander Duyck adapter->rx_ring[i]->reg_idx = reg_idx; 219b5f69ccfSAlexander Duyck adapter->rx_ring[i]->netdev = adapter->netdev; 220b5f69ccfSAlexander Duyck } 22173079ea0SAlexander Duyck 22273079ea0SAlexander Duyck #endif 22373079ea0SAlexander Duyck reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask); 22473079ea0SAlexander Duyck for (i = 0; i < adapter->num_tx_queues; i++, reg_idx++) { 22573079ea0SAlexander Duyck #ifdef IXGBE_FCOE 22673079ea0SAlexander Duyck /* Allow first FCoE queue to be mapped as RSS */ 22773079ea0SAlexander Duyck if (fcoe->offset && (i > fcoe->offset)) 22873079ea0SAlexander Duyck break; 22973079ea0SAlexander Duyck #endif 23073079ea0SAlexander Duyck /* If we are greater than indices move to next pool */ 23173079ea0SAlexander Duyck if ((reg_idx & rss->mask) >= rss->indices) 23273079ea0SAlexander Duyck reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask); 23373079ea0SAlexander Duyck adapter->tx_ring[i]->reg_idx = reg_idx; 23473079ea0SAlexander Duyck } 23573079ea0SAlexander Duyck 23673079ea0SAlexander Duyck #ifdef IXGBE_FCOE 23773079ea0SAlexander Duyck /* FCoE uses a linear block of queues so just assigning 1:1 */ 23873079ea0SAlexander Duyck for (; i < adapter->num_tx_queues; i++, reg_idx++) 23973079ea0SAlexander Duyck adapter->tx_ring[i]->reg_idx = reg_idx; 24073079ea0SAlexander Duyck 24173079ea0SAlexander Duyck #endif 24273079ea0SAlexander Duyck 24373079ea0SAlexander Duyck return true; 2448af3c33fSJeff Kirsher } 2458af3c33fSJeff Kirsher 2468af3c33fSJeff Kirsher /** 247d411a936SAlexander Duyck * ixgbe_cache_ring_rss - Descriptor ring to register mapping for RSS 248d411a936SAlexander Duyck * @adapter: board private structure to initialize 249d411a936SAlexander Duyck * 250d411a936SAlexander Duyck * Cache the descriptor ring offsets for RSS to the assigned rings. 251d411a936SAlexander Duyck * 252d411a936SAlexander Duyck **/ 253d411a936SAlexander Duyck static bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter) 254d411a936SAlexander Duyck { 25533fdc82fSJohn Fastabend int i, reg_idx; 256d411a936SAlexander Duyck 257b5f69ccfSAlexander Duyck for (i = 0; i < adapter->num_rx_queues; i++) { 258d411a936SAlexander Duyck adapter->rx_ring[i]->reg_idx = i; 259b5f69ccfSAlexander Duyck adapter->rx_ring[i]->netdev = adapter->netdev; 260b5f69ccfSAlexander Duyck } 26133fdc82fSJohn Fastabend for (i = 0, reg_idx = 0; i < adapter->num_tx_queues; i++, reg_idx++) 26233fdc82fSJohn Fastabend adapter->tx_ring[i]->reg_idx = reg_idx; 26333fdc82fSJohn Fastabend for (i = 0; i < adapter->num_xdp_queues; i++, reg_idx++) 26433fdc82fSJohn Fastabend adapter->xdp_ring[i]->reg_idx = reg_idx; 265d411a936SAlexander Duyck 266d411a936SAlexander Duyck return true; 267d411a936SAlexander Duyck } 268d411a936SAlexander Duyck 269d411a936SAlexander Duyck /** 2708af3c33fSJeff Kirsher * ixgbe_cache_ring_register - Descriptor ring to register mapping 2718af3c33fSJeff Kirsher * @adapter: board private structure to initialize 2728af3c33fSJeff Kirsher * 2738af3c33fSJeff Kirsher * Once we know the feature-set enabled for the device, we'll cache 2748af3c33fSJeff Kirsher * the register offset the descriptor ring is assigned to. 2758af3c33fSJeff Kirsher * 2768af3c33fSJeff Kirsher * Note, the order the various feature calls is important. It must start with 2778af3c33fSJeff Kirsher * the "most" features enabled at the same time, then trickle down to the 2788af3c33fSJeff Kirsher * least amount of features turned on at once. 2798af3c33fSJeff Kirsher **/ 2808af3c33fSJeff Kirsher static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter) 2818af3c33fSJeff Kirsher { 2828af3c33fSJeff Kirsher /* start with default case */ 2838af3c33fSJeff Kirsher adapter->rx_ring[0]->reg_idx = 0; 2848af3c33fSJeff Kirsher adapter->tx_ring[0]->reg_idx = 0; 2858af3c33fSJeff Kirsher 28673079ea0SAlexander Duyck #ifdef CONFIG_IXGBE_DCB 28773079ea0SAlexander Duyck if (ixgbe_cache_ring_dcb_sriov(adapter)) 28873079ea0SAlexander Duyck return; 28973079ea0SAlexander Duyck 29073079ea0SAlexander Duyck if (ixgbe_cache_ring_dcb(adapter)) 29173079ea0SAlexander Duyck return; 29273079ea0SAlexander Duyck 29373079ea0SAlexander Duyck #endif 2948af3c33fSJeff Kirsher if (ixgbe_cache_ring_sriov(adapter)) 2958af3c33fSJeff Kirsher return; 2968af3c33fSJeff Kirsher 297d411a936SAlexander Duyck ixgbe_cache_ring_rss(adapter); 2988af3c33fSJeff Kirsher } 2998af3c33fSJeff Kirsher 30033fdc82fSJohn Fastabend static int ixgbe_xdp_queues(struct ixgbe_adapter *adapter) 30133fdc82fSJohn Fastabend { 302*4fe81585SJason Xing int queues; 303*4fe81585SJason Xing 304*4fe81585SJason Xing queues = min_t(int, IXGBE_MAX_XDP_QS, nr_cpu_ids); 305*4fe81585SJason Xing return adapter->xdp_prog ? queues : 0; 30633fdc82fSJohn Fastabend } 30733fdc82fSJohn Fastabend 3082bf1a87bSEmil Tantilov #define IXGBE_RSS_64Q_MASK 0x3F 309d411a936SAlexander Duyck #define IXGBE_RSS_16Q_MASK 0xF 310d411a936SAlexander Duyck #define IXGBE_RSS_8Q_MASK 0x7 311d411a936SAlexander Duyck #define IXGBE_RSS_4Q_MASK 0x3 312d411a936SAlexander Duyck #define IXGBE_RSS_2Q_MASK 0x1 313d411a936SAlexander Duyck #define IXGBE_RSS_DISABLED_MASK 0x0 314d411a936SAlexander Duyck 315d411a936SAlexander Duyck #ifdef CONFIG_IXGBE_DCB 31673079ea0SAlexander Duyck /** 31773079ea0SAlexander Duyck * ixgbe_set_dcb_sriov_queues: Allocate queues for SR-IOV devices w/ DCB 31873079ea0SAlexander Duyck * @adapter: board private structure to initialize 31973079ea0SAlexander Duyck * 32073079ea0SAlexander Duyck * When SR-IOV (Single Root IO Virtualiztion) is enabled, allocate queues 32173079ea0SAlexander Duyck * and VM pools where appropriate. Also assign queues based on DCB 32273079ea0SAlexander Duyck * priorities and map accordingly.. 32373079ea0SAlexander Duyck * 32473079ea0SAlexander Duyck **/ 32573079ea0SAlexander Duyck static bool ixgbe_set_dcb_sriov_queues(struct ixgbe_adapter *adapter) 32673079ea0SAlexander Duyck { 32773079ea0SAlexander Duyck int i; 32873079ea0SAlexander Duyck u16 vmdq_i = adapter->ring_feature[RING_F_VMDQ].limit; 32973079ea0SAlexander Duyck u16 vmdq_m = 0; 33073079ea0SAlexander Duyck #ifdef IXGBE_FCOE 33173079ea0SAlexander Duyck u16 fcoe_i = 0; 33273079ea0SAlexander Duyck #endif 3330efbf12bSAlexander Duyck u8 tcs = adapter->hw_tcs; 33473079ea0SAlexander Duyck 33573079ea0SAlexander Duyck /* verify we have DCB queueing enabled before proceeding */ 33673079ea0SAlexander Duyck if (tcs <= 1) 33773079ea0SAlexander Duyck return false; 33873079ea0SAlexander Duyck 33973079ea0SAlexander Duyck /* verify we have VMDq enabled before proceeding */ 34073079ea0SAlexander Duyck if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 34173079ea0SAlexander Duyck return false; 34273079ea0SAlexander Duyck 3434e039c16SAlexander Duyck /* limit VMDq instances on the PF by number of Tx queues */ 3444e039c16SAlexander Duyck vmdq_i = min_t(u16, vmdq_i, MAX_TX_QUEUES / tcs); 3454e039c16SAlexander Duyck 34673079ea0SAlexander Duyck /* Add starting offset to total pool count */ 34773079ea0SAlexander Duyck vmdq_i += adapter->ring_feature[RING_F_VMDQ].offset; 34873079ea0SAlexander Duyck 34973079ea0SAlexander Duyck /* 16 pools w/ 8 TC per pool */ 35073079ea0SAlexander Duyck if (tcs > 4) { 35173079ea0SAlexander Duyck vmdq_i = min_t(u16, vmdq_i, 16); 35273079ea0SAlexander Duyck vmdq_m = IXGBE_82599_VMDQ_8Q_MASK; 35373079ea0SAlexander Duyck /* 32 pools w/ 4 TC per pool */ 35473079ea0SAlexander Duyck } else { 35573079ea0SAlexander Duyck vmdq_i = min_t(u16, vmdq_i, 32); 35673079ea0SAlexander Duyck vmdq_m = IXGBE_82599_VMDQ_4Q_MASK; 35773079ea0SAlexander Duyck } 35873079ea0SAlexander Duyck 35973079ea0SAlexander Duyck #ifdef IXGBE_FCOE 36073079ea0SAlexander Duyck /* queues in the remaining pools are available for FCoE */ 36173079ea0SAlexander Duyck fcoe_i = (128 / __ALIGN_MASK(1, ~vmdq_m)) - vmdq_i; 36273079ea0SAlexander Duyck 36373079ea0SAlexander Duyck #endif 36473079ea0SAlexander Duyck /* remove the starting offset from the pool count */ 36573079ea0SAlexander Duyck vmdq_i -= adapter->ring_feature[RING_F_VMDQ].offset; 36673079ea0SAlexander Duyck 36773079ea0SAlexander Duyck /* save features for later use */ 36873079ea0SAlexander Duyck adapter->ring_feature[RING_F_VMDQ].indices = vmdq_i; 36973079ea0SAlexander Duyck adapter->ring_feature[RING_F_VMDQ].mask = vmdq_m; 37073079ea0SAlexander Duyck 37173079ea0SAlexander Duyck /* 37273079ea0SAlexander Duyck * We do not support DCB, VMDq, and RSS all simultaneously 37373079ea0SAlexander Duyck * so we will disable RSS since it is the lowest priority 37473079ea0SAlexander Duyck */ 37573079ea0SAlexander Duyck adapter->ring_feature[RING_F_RSS].indices = 1; 37673079ea0SAlexander Duyck adapter->ring_feature[RING_F_RSS].mask = IXGBE_RSS_DISABLED_MASK; 37773079ea0SAlexander Duyck 37839cb681bSAlexander Duyck /* disable ATR as it is not supported when VMDq is enabled */ 37939cb681bSAlexander Duyck adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 38039cb681bSAlexander Duyck 38173079ea0SAlexander Duyck adapter->num_rx_pools = vmdq_i; 38273079ea0SAlexander Duyck adapter->num_rx_queues_per_pool = tcs; 38373079ea0SAlexander Duyck 38473079ea0SAlexander Duyck adapter->num_tx_queues = vmdq_i * tcs; 38533fdc82fSJohn Fastabend adapter->num_xdp_queues = 0; 38673079ea0SAlexander Duyck adapter->num_rx_queues = vmdq_i * tcs; 38773079ea0SAlexander Duyck 38873079ea0SAlexander Duyck #ifdef IXGBE_FCOE 38973079ea0SAlexander Duyck if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { 39073079ea0SAlexander Duyck struct ixgbe_ring_feature *fcoe; 39173079ea0SAlexander Duyck 39273079ea0SAlexander Duyck fcoe = &adapter->ring_feature[RING_F_FCOE]; 39373079ea0SAlexander Duyck 39473079ea0SAlexander Duyck /* limit ourselves based on feature limits */ 39573079ea0SAlexander Duyck fcoe_i = min_t(u16, fcoe_i, fcoe->limit); 39673079ea0SAlexander Duyck 39773079ea0SAlexander Duyck if (fcoe_i) { 39873079ea0SAlexander Duyck /* alloc queues for FCoE separately */ 39973079ea0SAlexander Duyck fcoe->indices = fcoe_i; 40073079ea0SAlexander Duyck fcoe->offset = vmdq_i * tcs; 40173079ea0SAlexander Duyck 40273079ea0SAlexander Duyck /* add queues to adapter */ 40373079ea0SAlexander Duyck adapter->num_tx_queues += fcoe_i; 40473079ea0SAlexander Duyck adapter->num_rx_queues += fcoe_i; 40573079ea0SAlexander Duyck } else if (tcs > 1) { 40673079ea0SAlexander Duyck /* use queue belonging to FcoE TC */ 40773079ea0SAlexander Duyck fcoe->indices = 1; 40873079ea0SAlexander Duyck fcoe->offset = ixgbe_fcoe_get_tc(adapter); 40973079ea0SAlexander Duyck } else { 41073079ea0SAlexander Duyck adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; 41173079ea0SAlexander Duyck 41273079ea0SAlexander Duyck fcoe->indices = 0; 41373079ea0SAlexander Duyck fcoe->offset = 0; 41473079ea0SAlexander Duyck } 41573079ea0SAlexander Duyck } 41673079ea0SAlexander Duyck 41773079ea0SAlexander Duyck #endif /* IXGBE_FCOE */ 41873079ea0SAlexander Duyck /* configure TC to queue mapping */ 41973079ea0SAlexander Duyck for (i = 0; i < tcs; i++) 42073079ea0SAlexander Duyck netdev_set_tc_queue(adapter->netdev, i, 1, i); 42173079ea0SAlexander Duyck 42273079ea0SAlexander Duyck return true; 42373079ea0SAlexander Duyck } 42473079ea0SAlexander Duyck 425d411a936SAlexander Duyck static bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter) 426d411a936SAlexander Duyck { 427d411a936SAlexander Duyck struct net_device *dev = adapter->netdev; 428d411a936SAlexander Duyck struct ixgbe_ring_feature *f; 429d411a936SAlexander Duyck int rss_i, rss_m, i; 430d411a936SAlexander Duyck int tcs; 431d411a936SAlexander Duyck 432d411a936SAlexander Duyck /* Map queue offset and counts onto allocated tx queues */ 4330efbf12bSAlexander Duyck tcs = adapter->hw_tcs; 434d411a936SAlexander Duyck 435d411a936SAlexander Duyck /* verify we have DCB queueing enabled before proceeding */ 436d411a936SAlexander Duyck if (tcs <= 1) 437d411a936SAlexander Duyck return false; 438d411a936SAlexander Duyck 439d411a936SAlexander Duyck /* determine the upper limit for our current DCB mode */ 440d411a936SAlexander Duyck rss_i = dev->num_tx_queues / tcs; 441d411a936SAlexander Duyck if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 442d411a936SAlexander Duyck /* 8 TC w/ 4 queues per TC */ 443d411a936SAlexander Duyck rss_i = min_t(u16, rss_i, 4); 444d411a936SAlexander Duyck rss_m = IXGBE_RSS_4Q_MASK; 445d411a936SAlexander Duyck } else if (tcs > 4) { 446d411a936SAlexander Duyck /* 8 TC w/ 8 queues per TC */ 447d411a936SAlexander Duyck rss_i = min_t(u16, rss_i, 8); 448d411a936SAlexander Duyck rss_m = IXGBE_RSS_8Q_MASK; 449d411a936SAlexander Duyck } else { 450d411a936SAlexander Duyck /* 4 TC w/ 16 queues per TC */ 451d411a936SAlexander Duyck rss_i = min_t(u16, rss_i, 16); 452d411a936SAlexander Duyck rss_m = IXGBE_RSS_16Q_MASK; 453d411a936SAlexander Duyck } 454d411a936SAlexander Duyck 455d411a936SAlexander Duyck /* set RSS mask and indices */ 456d411a936SAlexander Duyck f = &adapter->ring_feature[RING_F_RSS]; 457d411a936SAlexander Duyck rss_i = min_t(int, rss_i, f->limit); 458d411a936SAlexander Duyck f->indices = rss_i; 459d411a936SAlexander Duyck f->mask = rss_m; 460d411a936SAlexander Duyck 46139cb681bSAlexander Duyck /* disable ATR as it is not supported when multiple TCs are enabled */ 46239cb681bSAlexander Duyck adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 46339cb681bSAlexander Duyck 464d411a936SAlexander Duyck #ifdef IXGBE_FCOE 465d411a936SAlexander Duyck /* FCoE enabled queues require special configuration indexed 466d411a936SAlexander Duyck * by feature specific indices and offset. Here we map FCoE 467d411a936SAlexander Duyck * indices onto the DCB queue pairs allowing FCoE to own 468d411a936SAlexander Duyck * configuration later. 469d411a936SAlexander Duyck */ 470d411a936SAlexander Duyck if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { 471d411a936SAlexander Duyck u8 tc = ixgbe_fcoe_get_tc(adapter); 472d411a936SAlexander Duyck 473d411a936SAlexander Duyck f = &adapter->ring_feature[RING_F_FCOE]; 474d411a936SAlexander Duyck f->indices = min_t(u16, rss_i, f->limit); 475d411a936SAlexander Duyck f->offset = rss_i * tc; 476d411a936SAlexander Duyck } 477d411a936SAlexander Duyck 478d411a936SAlexander Duyck #endif /* IXGBE_FCOE */ 479d411a936SAlexander Duyck for (i = 0; i < tcs; i++) 480d411a936SAlexander Duyck netdev_set_tc_queue(dev, i, rss_i, rss_i * i); 481d411a936SAlexander Duyck 482d411a936SAlexander Duyck adapter->num_tx_queues = rss_i * tcs; 48333fdc82fSJohn Fastabend adapter->num_xdp_queues = 0; 484d411a936SAlexander Duyck adapter->num_rx_queues = rss_i * tcs; 485d411a936SAlexander Duyck 486d411a936SAlexander Duyck return true; 487d411a936SAlexander Duyck } 488d411a936SAlexander Duyck 489d411a936SAlexander Duyck #endif 4908af3c33fSJeff Kirsher /** 49173079ea0SAlexander Duyck * ixgbe_set_sriov_queues - Allocate queues for SR-IOV devices 49273079ea0SAlexander Duyck * @adapter: board private structure to initialize 49373079ea0SAlexander Duyck * 49473079ea0SAlexander Duyck * When SR-IOV (Single Root IO Virtualiztion) is enabled, allocate queues 49573079ea0SAlexander Duyck * and VM pools where appropriate. If RSS is available, then also try and 49673079ea0SAlexander Duyck * enable RSS and map accordingly. 49773079ea0SAlexander Duyck * 49873079ea0SAlexander Duyck **/ 49973079ea0SAlexander Duyck static bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter) 50073079ea0SAlexander Duyck { 50173079ea0SAlexander Duyck u16 vmdq_i = adapter->ring_feature[RING_F_VMDQ].limit; 50273079ea0SAlexander Duyck u16 vmdq_m = 0; 50373079ea0SAlexander Duyck u16 rss_i = adapter->ring_feature[RING_F_RSS].limit; 50473079ea0SAlexander Duyck u16 rss_m = IXGBE_RSS_DISABLED_MASK; 50573079ea0SAlexander Duyck #ifdef IXGBE_FCOE 50673079ea0SAlexander Duyck u16 fcoe_i = 0; 50773079ea0SAlexander Duyck #endif 50873079ea0SAlexander Duyck 50973079ea0SAlexander Duyck /* only proceed if SR-IOV is enabled */ 51073079ea0SAlexander Duyck if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 51173079ea0SAlexander Duyck return false; 51273079ea0SAlexander Duyck 5134e039c16SAlexander Duyck /* limit l2fwd RSS based on total Tx queue limit */ 5144e039c16SAlexander Duyck rss_i = min_t(u16, rss_i, MAX_TX_QUEUES / vmdq_i); 5154e039c16SAlexander Duyck 51673079ea0SAlexander Duyck /* Add starting offset to total pool count */ 51773079ea0SAlexander Duyck vmdq_i += adapter->ring_feature[RING_F_VMDQ].offset; 51873079ea0SAlexander Duyck 51973079ea0SAlexander Duyck /* double check we are limited to maximum pools */ 52073079ea0SAlexander Duyck vmdq_i = min_t(u16, IXGBE_MAX_VMDQ_INDICES, vmdq_i); 52173079ea0SAlexander Duyck 52273079ea0SAlexander Duyck /* 64 pool mode with 2 queues per pool */ 5234e039c16SAlexander Duyck if (vmdq_i > 32) { 52473079ea0SAlexander Duyck vmdq_m = IXGBE_82599_VMDQ_2Q_MASK; 52573079ea0SAlexander Duyck rss_m = IXGBE_RSS_2Q_MASK; 52673079ea0SAlexander Duyck rss_i = min_t(u16, rss_i, 2); 527e24fcf28SAlexander Duyck /* 32 pool mode with up to 4 queues per pool */ 52873079ea0SAlexander Duyck } else { 52973079ea0SAlexander Duyck vmdq_m = IXGBE_82599_VMDQ_4Q_MASK; 53073079ea0SAlexander Duyck rss_m = IXGBE_RSS_4Q_MASK; 531e24fcf28SAlexander Duyck /* We can support 4, 2, or 1 queues */ 532e24fcf28SAlexander Duyck rss_i = (rss_i > 3) ? 4 : (rss_i > 1) ? 2 : 1; 53373079ea0SAlexander Duyck } 53473079ea0SAlexander Duyck 53573079ea0SAlexander Duyck #ifdef IXGBE_FCOE 53673079ea0SAlexander Duyck /* queues in the remaining pools are available for FCoE */ 53773079ea0SAlexander Duyck fcoe_i = 128 - (vmdq_i * __ALIGN_MASK(1, ~vmdq_m)); 53873079ea0SAlexander Duyck 53973079ea0SAlexander Duyck #endif 54073079ea0SAlexander Duyck /* remove the starting offset from the pool count */ 54173079ea0SAlexander Duyck vmdq_i -= adapter->ring_feature[RING_F_VMDQ].offset; 54273079ea0SAlexander Duyck 54373079ea0SAlexander Duyck /* save features for later use */ 54473079ea0SAlexander Duyck adapter->ring_feature[RING_F_VMDQ].indices = vmdq_i; 54573079ea0SAlexander Duyck adapter->ring_feature[RING_F_VMDQ].mask = vmdq_m; 54673079ea0SAlexander Duyck 54773079ea0SAlexander Duyck /* limit RSS based on user input and save for later use */ 54873079ea0SAlexander Duyck adapter->ring_feature[RING_F_RSS].indices = rss_i; 54973079ea0SAlexander Duyck adapter->ring_feature[RING_F_RSS].mask = rss_m; 55073079ea0SAlexander Duyck 55173079ea0SAlexander Duyck adapter->num_rx_pools = vmdq_i; 55273079ea0SAlexander Duyck adapter->num_rx_queues_per_pool = rss_i; 55373079ea0SAlexander Duyck 55473079ea0SAlexander Duyck adapter->num_rx_queues = vmdq_i * rss_i; 55573079ea0SAlexander Duyck adapter->num_tx_queues = vmdq_i * rss_i; 55633fdc82fSJohn Fastabend adapter->num_xdp_queues = 0; 55773079ea0SAlexander Duyck 55873079ea0SAlexander Duyck /* disable ATR as it is not supported when VMDq is enabled */ 55973079ea0SAlexander Duyck adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 56073079ea0SAlexander Duyck 56173079ea0SAlexander Duyck #ifdef IXGBE_FCOE 56273079ea0SAlexander Duyck /* 56373079ea0SAlexander Duyck * FCoE can use rings from adjacent buffers to allow RSS 56473079ea0SAlexander Duyck * like behavior. To account for this we need to add the 56573079ea0SAlexander Duyck * FCoE indices to the total ring count. 56673079ea0SAlexander Duyck */ 56773079ea0SAlexander Duyck if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { 56873079ea0SAlexander Duyck struct ixgbe_ring_feature *fcoe; 56973079ea0SAlexander Duyck 57073079ea0SAlexander Duyck fcoe = &adapter->ring_feature[RING_F_FCOE]; 57173079ea0SAlexander Duyck 57273079ea0SAlexander Duyck /* limit ourselves based on feature limits */ 57373079ea0SAlexander Duyck fcoe_i = min_t(u16, fcoe_i, fcoe->limit); 57473079ea0SAlexander Duyck 57573079ea0SAlexander Duyck if (vmdq_i > 1 && fcoe_i) { 57673079ea0SAlexander Duyck /* alloc queues for FCoE separately */ 57773079ea0SAlexander Duyck fcoe->indices = fcoe_i; 57873079ea0SAlexander Duyck fcoe->offset = vmdq_i * rss_i; 57973079ea0SAlexander Duyck } else { 58073079ea0SAlexander Duyck /* merge FCoE queues with RSS queues */ 58173079ea0SAlexander Duyck fcoe_i = min_t(u16, fcoe_i + rss_i, num_online_cpus()); 58273079ea0SAlexander Duyck 58373079ea0SAlexander Duyck /* limit indices to rss_i if MSI-X is disabled */ 58473079ea0SAlexander Duyck if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) 58573079ea0SAlexander Duyck fcoe_i = rss_i; 58673079ea0SAlexander Duyck 58773079ea0SAlexander Duyck /* attempt to reserve some queues for just FCoE */ 58873079ea0SAlexander Duyck fcoe->indices = min_t(u16, fcoe_i, fcoe->limit); 58973079ea0SAlexander Duyck fcoe->offset = fcoe_i - fcoe->indices; 59073079ea0SAlexander Duyck 59173079ea0SAlexander Duyck fcoe_i -= rss_i; 59273079ea0SAlexander Duyck } 59373079ea0SAlexander Duyck 59473079ea0SAlexander Duyck /* add queues to adapter */ 59573079ea0SAlexander Duyck adapter->num_tx_queues += fcoe_i; 59673079ea0SAlexander Duyck adapter->num_rx_queues += fcoe_i; 59773079ea0SAlexander Duyck } 59873079ea0SAlexander Duyck 59973079ea0SAlexander Duyck #endif 600646bb57cSAlexander Duyck /* To support macvlan offload we have to use num_tc to 601646bb57cSAlexander Duyck * restrict the queues that can be used by the device. 602646bb57cSAlexander Duyck * By doing this we can avoid reporting a false number of 603646bb57cSAlexander Duyck * queues. 604646bb57cSAlexander Duyck */ 605646bb57cSAlexander Duyck if (vmdq_i > 1) 606646bb57cSAlexander Duyck netdev_set_num_tc(adapter->netdev, 1); 607646bb57cSAlexander Duyck 60849cfbeb7SAlexander Duyck /* populate TC0 for use by pool 0 */ 60949cfbeb7SAlexander Duyck netdev_set_tc_queue(adapter->netdev, 0, 61049cfbeb7SAlexander Duyck adapter->num_rx_queues_per_pool, 0); 61149cfbeb7SAlexander Duyck 61273079ea0SAlexander Duyck return true; 61373079ea0SAlexander Duyck } 61473079ea0SAlexander Duyck 61573079ea0SAlexander Duyck /** 61649ce9c2cSBen Hutchings * ixgbe_set_rss_queues - Allocate queues for RSS 6178af3c33fSJeff Kirsher * @adapter: board private structure to initialize 6188af3c33fSJeff Kirsher * 6198af3c33fSJeff Kirsher * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try 6208af3c33fSJeff Kirsher * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU. 6218af3c33fSJeff Kirsher * 6228af3c33fSJeff Kirsher **/ 6230b7f5d0bSAlexander Duyck static bool ixgbe_set_rss_queues(struct ixgbe_adapter *adapter) 6248af3c33fSJeff Kirsher { 6252bf1a87bSEmil Tantilov struct ixgbe_hw *hw = &adapter->hw; 6260b7f5d0bSAlexander Duyck struct ixgbe_ring_feature *f; 6270b7f5d0bSAlexander Duyck u16 rss_i; 6288af3c33fSJeff Kirsher 6290b7f5d0bSAlexander Duyck /* set mask for 16 queue limit of RSS */ 6300b7f5d0bSAlexander Duyck f = &adapter->ring_feature[RING_F_RSS]; 6310b7f5d0bSAlexander Duyck rss_i = f->limit; 6320b7f5d0bSAlexander Duyck 6330b7f5d0bSAlexander Duyck f->indices = rss_i; 6342bf1a87bSEmil Tantilov 6352bf1a87bSEmil Tantilov if (hw->mac.type < ixgbe_mac_X550) 636d411a936SAlexander Duyck f->mask = IXGBE_RSS_16Q_MASK; 6372bf1a87bSEmil Tantilov else 6382bf1a87bSEmil Tantilov f->mask = IXGBE_RSS_64Q_MASK; 6398af3c33fSJeff Kirsher 64039cb681bSAlexander Duyck /* disable ATR by default, it will be configured below */ 64139cb681bSAlexander Duyck adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 64239cb681bSAlexander Duyck 6438af3c33fSJeff Kirsher /* 6440b7f5d0bSAlexander Duyck * Use Flow Director in addition to RSS to ensure the best 6458af3c33fSJeff Kirsher * distribution of flows across cores, even when an FDIR flow 6468af3c33fSJeff Kirsher * isn't matched. 6478af3c33fSJeff Kirsher */ 64839cb681bSAlexander Duyck if (rss_i > 1 && adapter->atr_sample_rate) { 6490b7f5d0bSAlexander Duyck f = &adapter->ring_feature[RING_F_FDIR]; 6500b7f5d0bSAlexander Duyck 651d3cb9869SAlexander Duyck rss_i = f->indices = f->limit; 65239cb681bSAlexander Duyck 65339cb681bSAlexander Duyck if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) 65439cb681bSAlexander Duyck adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; 6558af3c33fSJeff Kirsher } 6560b7f5d0bSAlexander Duyck 657d411a936SAlexander Duyck #ifdef IXGBE_FCOE 658d411a936SAlexander Duyck /* 659d411a936SAlexander Duyck * FCoE can exist on the same rings as standard network traffic 660d411a936SAlexander Duyck * however it is preferred to avoid that if possible. In order 661d411a936SAlexander Duyck * to get the best performance we allocate as many FCoE queues 662d411a936SAlexander Duyck * as we can and we place them at the end of the ring array to 663d411a936SAlexander Duyck * avoid sharing queues with standard RSS on systems with 24 or 664d411a936SAlexander Duyck * more CPUs. 665d411a936SAlexander Duyck */ 666d411a936SAlexander Duyck if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { 667d411a936SAlexander Duyck struct net_device *dev = adapter->netdev; 668d411a936SAlexander Duyck u16 fcoe_i; 669d411a936SAlexander Duyck 670d411a936SAlexander Duyck f = &adapter->ring_feature[RING_F_FCOE]; 671d411a936SAlexander Duyck 672d411a936SAlexander Duyck /* merge FCoE queues with RSS queues */ 673d411a936SAlexander Duyck fcoe_i = min_t(u16, f->limit + rss_i, num_online_cpus()); 674d411a936SAlexander Duyck fcoe_i = min_t(u16, fcoe_i, dev->num_tx_queues); 675d411a936SAlexander Duyck 676d411a936SAlexander Duyck /* limit indices to rss_i if MSI-X is disabled */ 677d411a936SAlexander Duyck if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) 678d411a936SAlexander Duyck fcoe_i = rss_i; 679d411a936SAlexander Duyck 680d411a936SAlexander Duyck /* attempt to reserve some queues for just FCoE */ 681d411a936SAlexander Duyck f->indices = min_t(u16, fcoe_i, f->limit); 682d411a936SAlexander Duyck f->offset = fcoe_i - f->indices; 683d411a936SAlexander Duyck rss_i = max_t(u16, fcoe_i, rss_i); 684d411a936SAlexander Duyck } 685d411a936SAlexander Duyck 686d411a936SAlexander Duyck #endif /* IXGBE_FCOE */ 6870b7f5d0bSAlexander Duyck adapter->num_rx_queues = rss_i; 6880b7f5d0bSAlexander Duyck adapter->num_tx_queues = rss_i; 68933fdc82fSJohn Fastabend adapter->num_xdp_queues = ixgbe_xdp_queues(adapter); 6900b7f5d0bSAlexander Duyck 6910b7f5d0bSAlexander Duyck return true; 6928af3c33fSJeff Kirsher } 6938af3c33fSJeff Kirsher 6948af3c33fSJeff Kirsher /** 69549ce9c2cSBen Hutchings * ixgbe_set_num_queues - Allocate queues for device, feature dependent 6968af3c33fSJeff Kirsher * @adapter: board private structure to initialize 6978af3c33fSJeff Kirsher * 6988af3c33fSJeff Kirsher * This is the top level queue allocation routine. The order here is very 6998af3c33fSJeff Kirsher * important, starting with the "most" number of features turned on at once, 7008af3c33fSJeff Kirsher * and ending with the smallest set of features. This way large combinations 7018af3c33fSJeff Kirsher * can be allocated if they're turned on, and smaller combinations are the 7028af3c33fSJeff Kirsher * fallthrough conditions. 7038af3c33fSJeff Kirsher * 7048af3c33fSJeff Kirsher **/ 705ac802f5dSAlexander Duyck static void ixgbe_set_num_queues(struct ixgbe_adapter *adapter) 7068af3c33fSJeff Kirsher { 7078af3c33fSJeff Kirsher /* Start with base case */ 7088af3c33fSJeff Kirsher adapter->num_rx_queues = 1; 7098af3c33fSJeff Kirsher adapter->num_tx_queues = 1; 71033fdc82fSJohn Fastabend adapter->num_xdp_queues = 0; 711ff815fb2SAlexander Duyck adapter->num_rx_pools = 1; 7128af3c33fSJeff Kirsher adapter->num_rx_queues_per_pool = 1; 7138af3c33fSJeff Kirsher 71473079ea0SAlexander Duyck #ifdef CONFIG_IXGBE_DCB 71573079ea0SAlexander Duyck if (ixgbe_set_dcb_sriov_queues(adapter)) 716ac802f5dSAlexander Duyck return; 7178af3c33fSJeff Kirsher 7188af3c33fSJeff Kirsher if (ixgbe_set_dcb_queues(adapter)) 719ac802f5dSAlexander Duyck return; 7208af3c33fSJeff Kirsher 7218af3c33fSJeff Kirsher #endif 72273079ea0SAlexander Duyck if (ixgbe_set_sriov_queues(adapter)) 72373079ea0SAlexander Duyck return; 72473079ea0SAlexander Duyck 725ac802f5dSAlexander Duyck ixgbe_set_rss_queues(adapter); 7268af3c33fSJeff Kirsher } 7278af3c33fSJeff Kirsher 7283bcf3446SJacob Keller /** 7293bcf3446SJacob Keller * ixgbe_acquire_msix_vectors - acquire MSI-X vectors 7303bcf3446SJacob Keller * @adapter: board private structure 7313bcf3446SJacob Keller * 7323bcf3446SJacob Keller * Attempts to acquire a suitable range of MSI-X vector interrupts. Will 7333bcf3446SJacob Keller * return a negative error code if unable to acquire MSI-X vectors for any 7343bcf3446SJacob Keller * reason. 7353bcf3446SJacob Keller */ 7363bcf3446SJacob Keller static int ixgbe_acquire_msix_vectors(struct ixgbe_adapter *adapter) 7378af3c33fSJeff Kirsher { 7383bcf3446SJacob Keller struct ixgbe_hw *hw = &adapter->hw; 7393bcf3446SJacob Keller int i, vectors, vector_threshold; 7408af3c33fSJeff Kirsher 74133fdc82fSJohn Fastabend /* We start by asking for one vector per queue pair with XDP queues 74233fdc82fSJohn Fastabend * being stacked with TX queues. 74333fdc82fSJohn Fastabend */ 7443bcf3446SJacob Keller vectors = max(adapter->num_rx_queues, adapter->num_tx_queues); 74533fdc82fSJohn Fastabend vectors = max(vectors, adapter->num_xdp_queues); 7463bcf3446SJacob Keller 7473bcf3446SJacob Keller /* It is easy to be greedy for MSI-X vectors. However, it really 7483bcf3446SJacob Keller * doesn't do much good if we have a lot more vectors than CPUs. We'll 7493bcf3446SJacob Keller * be somewhat conservative and only ask for (roughly) the same number 7503bcf3446SJacob Keller * of vectors as there are CPUs. 7513bcf3446SJacob Keller */ 7523bcf3446SJacob Keller vectors = min_t(int, vectors, num_online_cpus()); 7533bcf3446SJacob Keller 7543bcf3446SJacob Keller /* Some vectors are necessary for non-queue interrupts */ 7553bcf3446SJacob Keller vectors += NON_Q_VECTORS; 7563bcf3446SJacob Keller 7573bcf3446SJacob Keller /* Hardware can only support a maximum of hw.mac->max_msix_vectors. 7583bcf3446SJacob Keller * With features such as RSS and VMDq, we can easily surpass the 7593bcf3446SJacob Keller * number of Rx and Tx descriptor queues supported by our device. 7603bcf3446SJacob Keller * Thus, we cap the maximum in the rare cases where the CPU count also 7613bcf3446SJacob Keller * exceeds our vector limit 7623bcf3446SJacob Keller */ 7633bcf3446SJacob Keller vectors = min_t(int, vectors, hw->mac.max_msix_vectors); 7643bcf3446SJacob Keller 7653bcf3446SJacob Keller /* We want a minimum of two MSI-X vectors for (1) a TxQ[0] + RxQ[0] 7663bcf3446SJacob Keller * handler, and (2) an Other (Link Status Change, etc.) handler. 7678af3c33fSJeff Kirsher */ 7688af3c33fSJeff Kirsher vector_threshold = MIN_MSIX_COUNT; 7698af3c33fSJeff Kirsher 770027bb561SJacob Keller adapter->msix_entries = kcalloc(vectors, 771027bb561SJacob Keller sizeof(struct msix_entry), 772027bb561SJacob Keller GFP_KERNEL); 773027bb561SJacob Keller if (!adapter->msix_entries) 774027bb561SJacob Keller return -ENOMEM; 775027bb561SJacob Keller 776027bb561SJacob Keller for (i = 0; i < vectors; i++) 777027bb561SJacob Keller adapter->msix_entries[i].entry = i; 778027bb561SJacob Keller 779b45e620cSAlexander Gordeev vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 780b45e620cSAlexander Gordeev vector_threshold, vectors); 7818af3c33fSJeff Kirsher 782b45e620cSAlexander Gordeev if (vectors < 0) { 783493043e5SJacob Keller /* A negative count of allocated vectors indicates an error in 784493043e5SJacob Keller * acquiring within the specified range of MSI-X vectors 7858af3c33fSJeff Kirsher */ 786493043e5SJacob Keller e_dev_warn("Failed to allocate MSI-X interrupts. Err: %d\n", 787493043e5SJacob Keller vectors); 788493043e5SJacob Keller 7898af3c33fSJeff Kirsher adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; 7908af3c33fSJeff Kirsher kfree(adapter->msix_entries); 7918af3c33fSJeff Kirsher adapter->msix_entries = NULL; 792d7de3c6eSJacob Keller 793d7de3c6eSJacob Keller return vectors; 794d7de3c6eSJacob Keller } 795d7de3c6eSJacob Keller 796d7de3c6eSJacob Keller /* we successfully allocated some number of vectors within our 797d7de3c6eSJacob Keller * requested range. 798d7de3c6eSJacob Keller */ 799d7de3c6eSJacob Keller adapter->flags |= IXGBE_FLAG_MSIX_ENABLED; 800d7de3c6eSJacob Keller 801d7de3c6eSJacob Keller /* Adjust for only the vectors we'll use, which is minimum 802d7de3c6eSJacob Keller * of max_q_vectors, or the number of vectors we were allocated. 8038af3c33fSJeff Kirsher */ 80449c7ffbeSAlexander Duyck vectors -= NON_Q_VECTORS; 805d7de3c6eSJacob Keller adapter->num_q_vectors = min_t(int, vectors, adapter->max_q_vectors); 806d7de3c6eSJacob Keller 807d7de3c6eSJacob Keller return 0; 8088af3c33fSJeff Kirsher } 8098af3c33fSJeff Kirsher 8108af3c33fSJeff Kirsher static void ixgbe_add_ring(struct ixgbe_ring *ring, 8118af3c33fSJeff Kirsher struct ixgbe_ring_container *head) 8128af3c33fSJeff Kirsher { 8138af3c33fSJeff Kirsher ring->next = head->ring; 8148af3c33fSJeff Kirsher head->ring = ring; 8158af3c33fSJeff Kirsher head->count++; 816b4ded832SAlexander Duyck head->next_update = jiffies + 1; 8178af3c33fSJeff Kirsher } 8188af3c33fSJeff Kirsher 8198af3c33fSJeff Kirsher /** 8208af3c33fSJeff Kirsher * ixgbe_alloc_q_vector - Allocate memory for a single interrupt vector 8218af3c33fSJeff Kirsher * @adapter: board private structure to initialize 822d0bfcdfdSAlexander Duyck * @v_count: q_vectors allocated on adapter, used for ring interleaving 8238af3c33fSJeff Kirsher * @v_idx: index of vector in adapter struct 824d0bfcdfdSAlexander Duyck * @txr_count: total number of Tx rings to allocate 825d0bfcdfdSAlexander Duyck * @txr_idx: index of first Tx ring to allocate 82633fdc82fSJohn Fastabend * @xdp_count: total number of XDP rings to allocate 82733fdc82fSJohn Fastabend * @xdp_idx: index of first XDP ring to allocate 828d0bfcdfdSAlexander Duyck * @rxr_count: total number of Rx rings to allocate 829d0bfcdfdSAlexander Duyck * @rxr_idx: index of first Rx ring to allocate 8308af3c33fSJeff Kirsher * 8318af3c33fSJeff Kirsher * We allocate one q_vector. If allocation fails we return -ENOMEM. 8328af3c33fSJeff Kirsher **/ 833d0bfcdfdSAlexander Duyck static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, 834d0bfcdfdSAlexander Duyck int v_count, int v_idx, 8358af3c33fSJeff Kirsher int txr_count, int txr_idx, 83633fdc82fSJohn Fastabend int xdp_count, int xdp_idx, 8378af3c33fSJeff Kirsher int rxr_count, int rxr_idx) 8388af3c33fSJeff Kirsher { 839780e354dSAlexander Duyck int node = dev_to_node(&adapter->pdev->dev); 8408af3c33fSJeff Kirsher struct ixgbe_q_vector *q_vector; 8418af3c33fSJeff Kirsher struct ixgbe_ring *ring; 8428af3c33fSJeff Kirsher int cpu = -1; 843439bb9edSGustavo A. R. Silva int ring_count; 8440efbf12bSAlexander Duyck u8 tcs = adapter->hw_tcs; 8458af3c33fSJeff Kirsher 84633fdc82fSJohn Fastabend ring_count = txr_count + rxr_count + xdp_count; 8478af3c33fSJeff Kirsher 8488af3c33fSJeff Kirsher /* customize cpu for Flow Director mapping */ 849fd786b7bSAlexander Duyck if ((tcs <= 1) && !(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { 850fd786b7bSAlexander Duyck u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; 851fd786b7bSAlexander Duyck if (rss_i > 1 && adapter->atr_sample_rate) { 852780e354dSAlexander Duyck cpu = cpumask_local_spread(v_idx, node); 8538af3c33fSJeff Kirsher node = cpu_to_node(cpu); 8548af3c33fSJeff Kirsher } 8558af3c33fSJeff Kirsher } 8568af3c33fSJeff Kirsher 8578af3c33fSJeff Kirsher /* allocate q_vector and rings */ 858439bb9edSGustavo A. R. Silva q_vector = kzalloc_node(struct_size(q_vector, ring, ring_count), 859439bb9edSGustavo A. R. Silva GFP_KERNEL, node); 8608af3c33fSJeff Kirsher if (!q_vector) 861439bb9edSGustavo A. R. Silva q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 862439bb9edSGustavo A. R. Silva GFP_KERNEL); 8638af3c33fSJeff Kirsher if (!q_vector) 8648af3c33fSJeff Kirsher return -ENOMEM; 8658af3c33fSJeff Kirsher 8668af3c33fSJeff Kirsher /* setup affinity mask and node */ 8678af3c33fSJeff Kirsher if (cpu != -1) 8688af3c33fSJeff Kirsher cpumask_set_cpu(cpu, &q_vector->affinity_mask); 8698af3c33fSJeff Kirsher q_vector->numa_node = node; 8708af3c33fSJeff Kirsher 871245f292dSAlexander Duyck #ifdef CONFIG_IXGBE_DCA 872245f292dSAlexander Duyck /* initialize CPU for DCA */ 873245f292dSAlexander Duyck q_vector->cpu = -1; 874245f292dSAlexander Duyck 875245f292dSAlexander Duyck #endif 8768af3c33fSJeff Kirsher /* initialize NAPI */ 8778af3c33fSJeff Kirsher netif_napi_add(adapter->netdev, &q_vector->napi, 8788af3c33fSJeff Kirsher ixgbe_poll, 64); 8798af3c33fSJeff Kirsher 8808af3c33fSJeff Kirsher /* tie q_vector and adapter together */ 8818af3c33fSJeff Kirsher adapter->q_vector[v_idx] = q_vector; 8828af3c33fSJeff Kirsher q_vector->adapter = adapter; 8838af3c33fSJeff Kirsher q_vector->v_idx = v_idx; 8848af3c33fSJeff Kirsher 8858af3c33fSJeff Kirsher /* initialize work limits */ 8868af3c33fSJeff Kirsher q_vector->tx.work_limit = adapter->tx_work_limit; 8878af3c33fSJeff Kirsher 888b4ded832SAlexander Duyck /* Initialize setting for adaptive ITR */ 889b4ded832SAlexander Duyck q_vector->tx.itr = IXGBE_ITR_ADAPTIVE_MAX_USECS | 890b4ded832SAlexander Duyck IXGBE_ITR_ADAPTIVE_LATENCY; 891b4ded832SAlexander Duyck q_vector->rx.itr = IXGBE_ITR_ADAPTIVE_MAX_USECS | 892b4ded832SAlexander Duyck IXGBE_ITR_ADAPTIVE_LATENCY; 8938af3c33fSJeff Kirsher 8943af3361eSEmil Tantilov /* intialize ITR */ 8953af3361eSEmil Tantilov if (txr_count && !rxr_count) { 8963af3361eSEmil Tantilov /* tx only vector */ 8973af3361eSEmil Tantilov if (adapter->tx_itr_setting == 1) 8988ac34f10SAlexander Duyck q_vector->itr = IXGBE_12K_ITR; 8993af3361eSEmil Tantilov else 9003af3361eSEmil Tantilov q_vector->itr = adapter->tx_itr_setting; 9013af3361eSEmil Tantilov } else { 9023af3361eSEmil Tantilov /* rx or rx/tx vector */ 9033af3361eSEmil Tantilov if (adapter->rx_itr_setting == 1) 9043af3361eSEmil Tantilov q_vector->itr = IXGBE_20K_ITR; 9053af3361eSEmil Tantilov else 9063af3361eSEmil Tantilov q_vector->itr = adapter->rx_itr_setting; 9073af3361eSEmil Tantilov } 9083af3361eSEmil Tantilov 909b4ded832SAlexander Duyck /* initialize pointer to rings */ 910b4ded832SAlexander Duyck ring = q_vector->ring; 911b4ded832SAlexander Duyck 9128af3c33fSJeff Kirsher while (txr_count) { 9138af3c33fSJeff Kirsher /* assign generic ring traits */ 9148af3c33fSJeff Kirsher ring->dev = &adapter->pdev->dev; 9158af3c33fSJeff Kirsher ring->netdev = adapter->netdev; 9168af3c33fSJeff Kirsher 9178af3c33fSJeff Kirsher /* configure backlink on ring */ 9188af3c33fSJeff Kirsher ring->q_vector = q_vector; 9198af3c33fSJeff Kirsher 9208af3c33fSJeff Kirsher /* update q_vector Tx values */ 9218af3c33fSJeff Kirsher ixgbe_add_ring(ring, &q_vector->tx); 9228af3c33fSJeff Kirsher 9238af3c33fSJeff Kirsher /* apply Tx specific ring traits */ 9248af3c33fSJeff Kirsher ring->count = adapter->tx_ring_count; 9258af3c33fSJeff Kirsher ring->queue_index = txr_idx; 9268af3c33fSJeff Kirsher 9278af3c33fSJeff Kirsher /* assign ring to adapter */ 928f140ad9fSCiara Loftus WRITE_ONCE(adapter->tx_ring[txr_idx], ring); 9298af3c33fSJeff Kirsher 9308af3c33fSJeff Kirsher /* update count and index */ 9318af3c33fSJeff Kirsher txr_count--; 932d0bfcdfdSAlexander Duyck txr_idx += v_count; 9338af3c33fSJeff Kirsher 9348af3c33fSJeff Kirsher /* push pointer to next ring */ 9358af3c33fSJeff Kirsher ring++; 9368af3c33fSJeff Kirsher } 9378af3c33fSJeff Kirsher 93833fdc82fSJohn Fastabend while (xdp_count) { 93933fdc82fSJohn Fastabend /* assign generic ring traits */ 94033fdc82fSJohn Fastabend ring->dev = &adapter->pdev->dev; 94133fdc82fSJohn Fastabend ring->netdev = adapter->netdev; 94233fdc82fSJohn Fastabend 94333fdc82fSJohn Fastabend /* configure backlink on ring */ 94433fdc82fSJohn Fastabend ring->q_vector = q_vector; 94533fdc82fSJohn Fastabend 94633fdc82fSJohn Fastabend /* update q_vector Tx values */ 94733fdc82fSJohn Fastabend ixgbe_add_ring(ring, &q_vector->tx); 94833fdc82fSJohn Fastabend 94933fdc82fSJohn Fastabend /* apply Tx specific ring traits */ 95033fdc82fSJohn Fastabend ring->count = adapter->tx_ring_count; 95133fdc82fSJohn Fastabend ring->queue_index = xdp_idx; 95233fdc82fSJohn Fastabend set_ring_xdp(ring); 953*4fe81585SJason Xing spin_lock_init(&ring->tx_lock); 95433fdc82fSJohn Fastabend 95533fdc82fSJohn Fastabend /* assign ring to adapter */ 956f140ad9fSCiara Loftus WRITE_ONCE(adapter->xdp_ring[xdp_idx], ring); 95733fdc82fSJohn Fastabend 95833fdc82fSJohn Fastabend /* update count and index */ 95933fdc82fSJohn Fastabend xdp_count--; 96033fdc82fSJohn Fastabend xdp_idx++; 96133fdc82fSJohn Fastabend 96233fdc82fSJohn Fastabend /* push pointer to next ring */ 96333fdc82fSJohn Fastabend ring++; 96433fdc82fSJohn Fastabend } 96533fdc82fSJohn Fastabend 9668af3c33fSJeff Kirsher while (rxr_count) { 9678af3c33fSJeff Kirsher /* assign generic ring traits */ 9688af3c33fSJeff Kirsher ring->dev = &adapter->pdev->dev; 9698af3c33fSJeff Kirsher ring->netdev = adapter->netdev; 9708af3c33fSJeff Kirsher 9718af3c33fSJeff Kirsher /* configure backlink on ring */ 9728af3c33fSJeff Kirsher ring->q_vector = q_vector; 9738af3c33fSJeff Kirsher 9748af3c33fSJeff Kirsher /* update q_vector Rx values */ 9758af3c33fSJeff Kirsher ixgbe_add_ring(ring, &q_vector->rx); 9768af3c33fSJeff Kirsher 9778af3c33fSJeff Kirsher /* 9788af3c33fSJeff Kirsher * 82599 errata, UDP frames with a 0 checksum 9798af3c33fSJeff Kirsher * can be marked as checksum errors. 9808af3c33fSJeff Kirsher */ 9818af3c33fSJeff Kirsher if (adapter->hw.mac.type == ixgbe_mac_82599EB) 9828af3c33fSJeff Kirsher set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state); 9838af3c33fSJeff Kirsher 984b2db497eSAlexander Duyck #ifdef IXGBE_FCOE 985b2db497eSAlexander Duyck if (adapter->netdev->features & NETIF_F_FCOE_MTU) { 986b2db497eSAlexander Duyck struct ixgbe_ring_feature *f; 987b2db497eSAlexander Duyck f = &adapter->ring_feature[RING_F_FCOE]; 988e4b317e9SAlexander Duyck if ((rxr_idx >= f->offset) && 989e4b317e9SAlexander Duyck (rxr_idx < f->offset + f->indices)) 99057efd44cSAlexander Duyck set_bit(__IXGBE_RX_FCOE, &ring->state); 991b2db497eSAlexander Duyck } 992b2db497eSAlexander Duyck 993b2db497eSAlexander Duyck #endif /* IXGBE_FCOE */ 9948af3c33fSJeff Kirsher /* apply Rx specific ring traits */ 9958af3c33fSJeff Kirsher ring->count = adapter->rx_ring_count; 9968af3c33fSJeff Kirsher ring->queue_index = rxr_idx; 9978af3c33fSJeff Kirsher 9988af3c33fSJeff Kirsher /* assign ring to adapter */ 999f140ad9fSCiara Loftus WRITE_ONCE(adapter->rx_ring[rxr_idx], ring); 10008af3c33fSJeff Kirsher 10018af3c33fSJeff Kirsher /* update count and index */ 10028af3c33fSJeff Kirsher rxr_count--; 1003d0bfcdfdSAlexander Duyck rxr_idx += v_count; 10048af3c33fSJeff Kirsher 10058af3c33fSJeff Kirsher /* push pointer to next ring */ 10068af3c33fSJeff Kirsher ring++; 10078af3c33fSJeff Kirsher } 10088af3c33fSJeff Kirsher 10098af3c33fSJeff Kirsher return 0; 10108af3c33fSJeff Kirsher } 10118af3c33fSJeff Kirsher 10128af3c33fSJeff Kirsher /** 10138af3c33fSJeff Kirsher * ixgbe_free_q_vector - Free memory allocated for specific interrupt vector 10148af3c33fSJeff Kirsher * @adapter: board private structure to initialize 10158af3c33fSJeff Kirsher * @v_idx: Index of vector to be freed 10168af3c33fSJeff Kirsher * 10178af3c33fSJeff Kirsher * This function frees the memory allocated to the q_vector. In addition if 10188af3c33fSJeff Kirsher * NAPI is enabled it will delete any references to the NAPI struct prior 10198af3c33fSJeff Kirsher * to freeing the q_vector. 10208af3c33fSJeff Kirsher **/ 10218af3c33fSJeff Kirsher static void ixgbe_free_q_vector(struct ixgbe_adapter *adapter, int v_idx) 10228af3c33fSJeff Kirsher { 10238af3c33fSJeff Kirsher struct ixgbe_q_vector *q_vector = adapter->q_vector[v_idx]; 10248af3c33fSJeff Kirsher struct ixgbe_ring *ring; 10258af3c33fSJeff Kirsher 102690382dcaSJohn Fastabend ixgbe_for_each_ring(ring, q_vector->tx) { 102790382dcaSJohn Fastabend if (ring_is_xdp(ring)) 1028f140ad9fSCiara Loftus WRITE_ONCE(adapter->xdp_ring[ring->queue_index], NULL); 102990382dcaSJohn Fastabend else 1030f140ad9fSCiara Loftus WRITE_ONCE(adapter->tx_ring[ring->queue_index], NULL); 103190382dcaSJohn Fastabend } 10328af3c33fSJeff Kirsher 10338af3c33fSJeff Kirsher ixgbe_for_each_ring(ring, q_vector->rx) 1034f140ad9fSCiara Loftus WRITE_ONCE(adapter->rx_ring[ring->queue_index], NULL); 10358af3c33fSJeff Kirsher 10368af3c33fSJeff Kirsher adapter->q_vector[v_idx] = NULL; 10375198d545SJakub Kicinski __netif_napi_del(&q_vector->napi); 10388af3c33fSJeff Kirsher 1039*4fe81585SJason Xing if (static_key_enabled(&ixgbe_xdp_locking_key)) 1040*4fe81585SJason Xing static_branch_dec(&ixgbe_xdp_locking_key); 1041*4fe81585SJason Xing 10428af3c33fSJeff Kirsher /* 10435198d545SJakub Kicinski * after a call to __netif_napi_del() napi may still be used and 10448af3c33fSJeff Kirsher * ixgbe_get_stats64() might access the rings on this vector, 10458af3c33fSJeff Kirsher * we must wait a grace period before freeing it. 10468af3c33fSJeff Kirsher */ 10478af3c33fSJeff Kirsher kfree_rcu(q_vector, rcu); 10488af3c33fSJeff Kirsher } 10498af3c33fSJeff Kirsher 10508af3c33fSJeff Kirsher /** 10518af3c33fSJeff Kirsher * ixgbe_alloc_q_vectors - Allocate memory for interrupt vectors 10528af3c33fSJeff Kirsher * @adapter: board private structure to initialize 10538af3c33fSJeff Kirsher * 10548af3c33fSJeff Kirsher * We allocate one q_vector per queue interrupt. If allocation fails we 10558af3c33fSJeff Kirsher * return -ENOMEM. 10568af3c33fSJeff Kirsher **/ 10578af3c33fSJeff Kirsher static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter) 10588af3c33fSJeff Kirsher { 105949c7ffbeSAlexander Duyck int q_vectors = adapter->num_q_vectors; 10608af3c33fSJeff Kirsher int rxr_remaining = adapter->num_rx_queues; 10618af3c33fSJeff Kirsher int txr_remaining = adapter->num_tx_queues; 106233fdc82fSJohn Fastabend int xdp_remaining = adapter->num_xdp_queues; 106333fdc82fSJohn Fastabend int rxr_idx = 0, txr_idx = 0, xdp_idx = 0, v_idx = 0; 1064d0bcacd0SBjörn Töpel int err, i; 10658af3c33fSJeff Kirsher 10668af3c33fSJeff Kirsher /* only one q_vector if MSI-X is disabled. */ 10678af3c33fSJeff Kirsher if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) 10688af3c33fSJeff Kirsher q_vectors = 1; 10698af3c33fSJeff Kirsher 107033fdc82fSJohn Fastabend if (q_vectors >= (rxr_remaining + txr_remaining + xdp_remaining)) { 1071d0bfcdfdSAlexander Duyck for (; rxr_remaining; v_idx++) { 1072d0bfcdfdSAlexander Duyck err = ixgbe_alloc_q_vector(adapter, q_vectors, v_idx, 107333fdc82fSJohn Fastabend 0, 0, 0, 0, 1, rxr_idx); 10748af3c33fSJeff Kirsher 10758af3c33fSJeff Kirsher if (err) 10768af3c33fSJeff Kirsher goto err_out; 10778af3c33fSJeff Kirsher 10788af3c33fSJeff Kirsher /* update counts and index */ 1079d0bfcdfdSAlexander Duyck rxr_remaining--; 1080d0bfcdfdSAlexander Duyck rxr_idx++; 10818af3c33fSJeff Kirsher } 10828af3c33fSJeff Kirsher } 10838af3c33fSJeff Kirsher 1084d0bfcdfdSAlexander Duyck for (; v_idx < q_vectors; v_idx++) { 1085d0bfcdfdSAlexander Duyck int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 1086d0bfcdfdSAlexander Duyck int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 108733fdc82fSJohn Fastabend int xqpv = DIV_ROUND_UP(xdp_remaining, q_vectors - v_idx); 108833fdc82fSJohn Fastabend 1089d0bfcdfdSAlexander Duyck err = ixgbe_alloc_q_vector(adapter, q_vectors, v_idx, 10908af3c33fSJeff Kirsher tqpv, txr_idx, 109133fdc82fSJohn Fastabend xqpv, xdp_idx, 10928af3c33fSJeff Kirsher rqpv, rxr_idx); 10938af3c33fSJeff Kirsher 10948af3c33fSJeff Kirsher if (err) 10958af3c33fSJeff Kirsher goto err_out; 10968af3c33fSJeff Kirsher 10978af3c33fSJeff Kirsher /* update counts and index */ 10988af3c33fSJeff Kirsher rxr_remaining -= rqpv; 10998af3c33fSJeff Kirsher txr_remaining -= tqpv; 110033fdc82fSJohn Fastabend xdp_remaining -= xqpv; 1101d0bfcdfdSAlexander Duyck rxr_idx++; 1102d0bfcdfdSAlexander Duyck txr_idx++; 110333fdc82fSJohn Fastabend xdp_idx += xqpv; 11048af3c33fSJeff Kirsher } 11058af3c33fSJeff Kirsher 1106d0bcacd0SBjörn Töpel for (i = 0; i < adapter->num_rx_queues; i++) { 1107d0bcacd0SBjörn Töpel if (adapter->rx_ring[i]) 1108d0bcacd0SBjörn Töpel adapter->rx_ring[i]->ring_idx = i; 1109d0bcacd0SBjörn Töpel } 1110d0bcacd0SBjörn Töpel 1111d0bcacd0SBjörn Töpel for (i = 0; i < adapter->num_tx_queues; i++) { 1112d0bcacd0SBjörn Töpel if (adapter->tx_ring[i]) 1113d0bcacd0SBjörn Töpel adapter->tx_ring[i]->ring_idx = i; 1114d0bcacd0SBjörn Töpel } 1115d0bcacd0SBjörn Töpel 1116d0bcacd0SBjörn Töpel for (i = 0; i < adapter->num_xdp_queues; i++) { 1117d0bcacd0SBjörn Töpel if (adapter->xdp_ring[i]) 1118d0bcacd0SBjörn Töpel adapter->xdp_ring[i]->ring_idx = i; 1119d0bcacd0SBjörn Töpel } 1120d0bcacd0SBjörn Töpel 11218af3c33fSJeff Kirsher return 0; 11228af3c33fSJeff Kirsher 11238af3c33fSJeff Kirsher err_out: 112449c7ffbeSAlexander Duyck adapter->num_tx_queues = 0; 112533fdc82fSJohn Fastabend adapter->num_xdp_queues = 0; 112649c7ffbeSAlexander Duyck adapter->num_rx_queues = 0; 112749c7ffbeSAlexander Duyck adapter->num_q_vectors = 0; 112849c7ffbeSAlexander Duyck 112949c7ffbeSAlexander Duyck while (v_idx--) 11308af3c33fSJeff Kirsher ixgbe_free_q_vector(adapter, v_idx); 11318af3c33fSJeff Kirsher 11328af3c33fSJeff Kirsher return -ENOMEM; 11338af3c33fSJeff Kirsher } 11348af3c33fSJeff Kirsher 11358af3c33fSJeff Kirsher /** 11368af3c33fSJeff Kirsher * ixgbe_free_q_vectors - Free memory allocated for interrupt vectors 11378af3c33fSJeff Kirsher * @adapter: board private structure to initialize 11388af3c33fSJeff Kirsher * 11398af3c33fSJeff Kirsher * This function frees the memory allocated to the q_vectors. In addition if 11408af3c33fSJeff Kirsher * NAPI is enabled it will delete any references to the NAPI struct prior 11418af3c33fSJeff Kirsher * to freeing the q_vector. 11428af3c33fSJeff Kirsher **/ 11438af3c33fSJeff Kirsher static void ixgbe_free_q_vectors(struct ixgbe_adapter *adapter) 11448af3c33fSJeff Kirsher { 114549c7ffbeSAlexander Duyck int v_idx = adapter->num_q_vectors; 11468af3c33fSJeff Kirsher 114749c7ffbeSAlexander Duyck adapter->num_tx_queues = 0; 114833fdc82fSJohn Fastabend adapter->num_xdp_queues = 0; 114949c7ffbeSAlexander Duyck adapter->num_rx_queues = 0; 115049c7ffbeSAlexander Duyck adapter->num_q_vectors = 0; 11518af3c33fSJeff Kirsher 115249c7ffbeSAlexander Duyck while (v_idx--) 11538af3c33fSJeff Kirsher ixgbe_free_q_vector(adapter, v_idx); 11548af3c33fSJeff Kirsher } 11558af3c33fSJeff Kirsher 11568af3c33fSJeff Kirsher static void ixgbe_reset_interrupt_capability(struct ixgbe_adapter *adapter) 11578af3c33fSJeff Kirsher { 11588af3c33fSJeff Kirsher if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 11598af3c33fSJeff Kirsher adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; 11608af3c33fSJeff Kirsher pci_disable_msix(adapter->pdev); 11618af3c33fSJeff Kirsher kfree(adapter->msix_entries); 11628af3c33fSJeff Kirsher adapter->msix_entries = NULL; 11638af3c33fSJeff Kirsher } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) { 11648af3c33fSJeff Kirsher adapter->flags &= ~IXGBE_FLAG_MSI_ENABLED; 11658af3c33fSJeff Kirsher pci_disable_msi(adapter->pdev); 11668af3c33fSJeff Kirsher } 11678af3c33fSJeff Kirsher } 11688af3c33fSJeff Kirsher 11698af3c33fSJeff Kirsher /** 11708af3c33fSJeff Kirsher * ixgbe_set_interrupt_capability - set MSI-X or MSI if supported 11718af3c33fSJeff Kirsher * @adapter: board private structure to initialize 11728af3c33fSJeff Kirsher * 11738af3c33fSJeff Kirsher * Attempt to configure the interrupts using the best available 11748af3c33fSJeff Kirsher * capabilities of the hardware and the kernel. 11758af3c33fSJeff Kirsher **/ 1176ac802f5dSAlexander Duyck static void ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter) 11778af3c33fSJeff Kirsher { 11783bcf3446SJacob Keller int err; 11798af3c33fSJeff Kirsher 11803bcf3446SJacob Keller /* We will try to get MSI-X interrupts first */ 11813bcf3446SJacob Keller if (!ixgbe_acquire_msix_vectors(adapter)) 1182ac802f5dSAlexander Duyck return; 11838af3c33fSJeff Kirsher 1184eec66731SJacob Keller /* At this point, we do not have MSI-X capabilities. We need to 1185eec66731SJacob Keller * reconfigure or disable various features which require MSI-X 1186eec66731SJacob Keller * capability. 1187eec66731SJacob Keller */ 1188eec66731SJacob Keller 1189c1c55f63SJacob Keller /* Disable DCB unless we only have a single traffic class */ 11900efbf12bSAlexander Duyck if (adapter->hw_tcs > 1) { 1191c1c55f63SJacob Keller e_dev_warn("Number of DCB TCs exceeds number of available queues. Disabling DCB support.\n"); 1192b724e9f2SAlexander Duyck netdev_reset_tc(adapter->netdev); 119339cb681bSAlexander Duyck 1194b724e9f2SAlexander Duyck if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1195b724e9f2SAlexander Duyck adapter->hw.fc.requested_mode = adapter->last_lfc_mode; 1196b724e9f2SAlexander Duyck 1197b724e9f2SAlexander Duyck adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED; 1198b724e9f2SAlexander Duyck adapter->temp_dcb_cfg.pfc_mode_enable = false; 1199b724e9f2SAlexander Duyck adapter->dcb_cfg.pfc_mode_enable = false; 1200b724e9f2SAlexander Duyck } 1201d786cf7bSJacob Keller 12020efbf12bSAlexander Duyck adapter->hw_tcs = 0; 1203b724e9f2SAlexander Duyck adapter->dcb_cfg.num_tcs.pg_tcs = 1; 1204b724e9f2SAlexander Duyck adapter->dcb_cfg.num_tcs.pfc_tcs = 1; 1205b724e9f2SAlexander Duyck 1206d786cf7bSJacob Keller /* Disable SR-IOV support */ 1207d786cf7bSJacob Keller e_dev_warn("Disabling SR-IOV support\n"); 12088af3c33fSJeff Kirsher ixgbe_disable_sriov(adapter); 12098af3c33fSJeff Kirsher 1210d786cf7bSJacob Keller /* Disable RSS */ 1211d786cf7bSJacob Keller e_dev_warn("Disabling RSS support\n"); 1212fbe7ca7fSAlexander Duyck adapter->ring_feature[RING_F_RSS].limit = 1; 1213b724e9f2SAlexander Duyck 1214eec66731SJacob Keller /* recalculate number of queues now that many features have been 1215eec66731SJacob Keller * changed or disabled. 1216eec66731SJacob Keller */ 1217ac802f5dSAlexander Duyck ixgbe_set_num_queues(adapter); 121849c7ffbeSAlexander Duyck adapter->num_q_vectors = 1; 121949c7ffbeSAlexander Duyck 12208af3c33fSJeff Kirsher err = pci_enable_msi(adapter->pdev); 12215d31b48aSJacob Keller if (err) 12225d31b48aSJacob Keller e_dev_warn("Failed to allocate MSI interrupt, falling back to legacy. Error: %d\n", 12236ec1b71fSJacob Keller err); 12245d31b48aSJacob Keller else 1225ac802f5dSAlexander Duyck adapter->flags |= IXGBE_FLAG_MSI_ENABLED; 12268af3c33fSJeff Kirsher } 12278af3c33fSJeff Kirsher 12288af3c33fSJeff Kirsher /** 12298af3c33fSJeff Kirsher * ixgbe_init_interrupt_scheme - Determine proper interrupt scheme 12308af3c33fSJeff Kirsher * @adapter: board private structure to initialize 12318af3c33fSJeff Kirsher * 12328af3c33fSJeff Kirsher * We determine which interrupt scheme to use based on... 12338af3c33fSJeff Kirsher * - Kernel support (MSI, MSI-X) 12348af3c33fSJeff Kirsher * - which can be user-defined (via MODULE_PARAM) 12358af3c33fSJeff Kirsher * - Hardware queue count (num_*_queues) 12368af3c33fSJeff Kirsher * - defined by miscellaneous hardware support/features (RSS, etc.) 12378af3c33fSJeff Kirsher **/ 12388af3c33fSJeff Kirsher int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter) 12398af3c33fSJeff Kirsher { 12408af3c33fSJeff Kirsher int err; 12418af3c33fSJeff Kirsher 12428af3c33fSJeff Kirsher /* Number of supported queues */ 1243ac802f5dSAlexander Duyck ixgbe_set_num_queues(adapter); 12448af3c33fSJeff Kirsher 1245ac802f5dSAlexander Duyck /* Set interrupt mode */ 1246ac802f5dSAlexander Duyck ixgbe_set_interrupt_capability(adapter); 12478af3c33fSJeff Kirsher 12488af3c33fSJeff Kirsher err = ixgbe_alloc_q_vectors(adapter); 12498af3c33fSJeff Kirsher if (err) { 12508af3c33fSJeff Kirsher e_dev_err("Unable to allocate memory for queue vectors\n"); 12518af3c33fSJeff Kirsher goto err_alloc_q_vectors; 12528af3c33fSJeff Kirsher } 12538af3c33fSJeff Kirsher 12548af3c33fSJeff Kirsher ixgbe_cache_ring_register(adapter); 12558af3c33fSJeff Kirsher 125633fdc82fSJohn Fastabend e_dev_info("Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u XDP Queue count = %u\n", 12578af3c33fSJeff Kirsher (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled", 125833fdc82fSJohn Fastabend adapter->num_rx_queues, adapter->num_tx_queues, 125933fdc82fSJohn Fastabend adapter->num_xdp_queues); 12608af3c33fSJeff Kirsher 12618af3c33fSJeff Kirsher set_bit(__IXGBE_DOWN, &adapter->state); 12628af3c33fSJeff Kirsher 12638af3c33fSJeff Kirsher return 0; 12648af3c33fSJeff Kirsher 12658af3c33fSJeff Kirsher err_alloc_q_vectors: 12668af3c33fSJeff Kirsher ixgbe_reset_interrupt_capability(adapter); 12678af3c33fSJeff Kirsher return err; 12688af3c33fSJeff Kirsher } 12698af3c33fSJeff Kirsher 12708af3c33fSJeff Kirsher /** 12718af3c33fSJeff Kirsher * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings 12728af3c33fSJeff Kirsher * @adapter: board private structure to clear interrupt scheme on 12738af3c33fSJeff Kirsher * 12748af3c33fSJeff Kirsher * We go through and clear interrupt specific resources and reset the structure 12758af3c33fSJeff Kirsher * to pre-load conditions 12768af3c33fSJeff Kirsher **/ 12778af3c33fSJeff Kirsher void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter) 12788af3c33fSJeff Kirsher { 12798af3c33fSJeff Kirsher adapter->num_tx_queues = 0; 128033fdc82fSJohn Fastabend adapter->num_xdp_queues = 0; 12818af3c33fSJeff Kirsher adapter->num_rx_queues = 0; 12828af3c33fSJeff Kirsher 12838af3c33fSJeff Kirsher ixgbe_free_q_vectors(adapter); 12848af3c33fSJeff Kirsher ixgbe_reset_interrupt_capability(adapter); 12858af3c33fSJeff Kirsher } 12868af3c33fSJeff Kirsher 12878af3c33fSJeff Kirsher void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens, 128859259470SShannon Nelson u32 fceof_saidx, u32 type_tucmd, u32 mss_l4len_idx) 12898af3c33fSJeff Kirsher { 12908af3c33fSJeff Kirsher struct ixgbe_adv_tx_context_desc *context_desc; 12918af3c33fSJeff Kirsher u16 i = tx_ring->next_to_use; 12928af3c33fSJeff Kirsher 12938af3c33fSJeff Kirsher context_desc = IXGBE_TX_CTXTDESC(tx_ring, i); 12948af3c33fSJeff Kirsher 12958af3c33fSJeff Kirsher i++; 12968af3c33fSJeff Kirsher tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 12978af3c33fSJeff Kirsher 12988af3c33fSJeff Kirsher /* set bits to identify this as an advanced context descriptor */ 12998af3c33fSJeff Kirsher type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 13008af3c33fSJeff Kirsher 13018af3c33fSJeff Kirsher context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 130259259470SShannon Nelson context_desc->fceof_saidx = cpu_to_le32(fceof_saidx); 13038af3c33fSJeff Kirsher context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 13048af3c33fSJeff Kirsher context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 13058af3c33fSJeff Kirsher } 13068af3c33fSJeff Kirsher 1307