1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019, Intel Corporation. */ 3 4 #include <net/xdp_sock_drv.h> 5 #include "ice_base.h" 6 #include "ice_lib.h" 7 #include "ice_dcb_lib.h" 8 #include "ice_sriov.h" 9 10 /** 11 * __ice_vsi_get_qs_contig - Assign a contiguous chunk of queues to VSI 12 * @qs_cfg: gathered variables needed for PF->VSI queues assignment 13 * 14 * Return 0 on success and -ENOMEM in case of no left space in PF queue bitmap 15 */ 16 static int __ice_vsi_get_qs_contig(struct ice_qs_cfg *qs_cfg) 17 { 18 unsigned int offset, i; 19 20 mutex_lock(qs_cfg->qs_mutex); 21 offset = bitmap_find_next_zero_area(qs_cfg->pf_map, qs_cfg->pf_map_size, 22 0, qs_cfg->q_count, 0); 23 if (offset >= qs_cfg->pf_map_size) { 24 mutex_unlock(qs_cfg->qs_mutex); 25 return -ENOMEM; 26 } 27 28 bitmap_set(qs_cfg->pf_map, offset, qs_cfg->q_count); 29 for (i = 0; i < qs_cfg->q_count; i++) 30 qs_cfg->vsi_map[i + qs_cfg->vsi_map_offset] = (u16)(i + offset); 31 mutex_unlock(qs_cfg->qs_mutex); 32 33 return 0; 34 } 35 36 /** 37 * __ice_vsi_get_qs_sc - Assign a scattered queues from PF to VSI 38 * @qs_cfg: gathered variables needed for pf->vsi queues assignment 39 * 40 * Return 0 on success and -ENOMEM in case of no left space in PF queue bitmap 41 */ 42 static int __ice_vsi_get_qs_sc(struct ice_qs_cfg *qs_cfg) 43 { 44 unsigned int i, index = 0; 45 46 mutex_lock(qs_cfg->qs_mutex); 47 for (i = 0; i < qs_cfg->q_count; i++) { 48 index = find_next_zero_bit(qs_cfg->pf_map, 49 qs_cfg->pf_map_size, index); 50 if (index >= qs_cfg->pf_map_size) 51 goto err_scatter; 52 set_bit(index, qs_cfg->pf_map); 53 qs_cfg->vsi_map[i + qs_cfg->vsi_map_offset] = (u16)index; 54 } 55 mutex_unlock(qs_cfg->qs_mutex); 56 57 return 0; 58 err_scatter: 59 for (index = 0; index < i; index++) { 60 clear_bit(qs_cfg->vsi_map[index], qs_cfg->pf_map); 61 qs_cfg->vsi_map[index + qs_cfg->vsi_map_offset] = 0; 62 } 63 mutex_unlock(qs_cfg->qs_mutex); 64 65 return -ENOMEM; 66 } 67 68 /** 69 * ice_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 70 * @pf: the PF being configured 71 * @pf_q: the PF queue 72 * @ena: enable or disable state of the queue 73 * 74 * This routine will wait for the given Rx queue of the PF to reach the 75 * enabled or disabled state. 76 * Returns -ETIMEDOUT in case of failing to reach the requested state after 77 * multiple retries; else will return 0 in case of success. 78 */ 79 static int ice_pf_rxq_wait(struct ice_pf *pf, int pf_q, bool ena) 80 { 81 int i; 82 83 for (i = 0; i < ICE_Q_WAIT_MAX_RETRY; i++) { 84 if (ena == !!(rd32(&pf->hw, QRX_CTRL(pf_q)) & 85 QRX_CTRL_QENA_STAT_M)) 86 return 0; 87 88 usleep_range(20, 40); 89 } 90 91 return -ETIMEDOUT; 92 } 93 94 /** 95 * ice_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 96 * @vsi: the VSI being configured 97 * @v_idx: index of the vector in the VSI struct 98 * 99 * We allocate one q_vector and set default value for ITR setting associated 100 * with this q_vector. If allocation fails we return -ENOMEM. 101 */ 102 static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx) 103 { 104 struct ice_pf *pf = vsi->back; 105 struct ice_q_vector *q_vector; 106 int err; 107 108 /* allocate q_vector */ 109 q_vector = kzalloc(sizeof(*q_vector), GFP_KERNEL); 110 if (!q_vector) 111 return -ENOMEM; 112 113 q_vector->vsi = vsi; 114 q_vector->v_idx = v_idx; 115 q_vector->tx.itr_setting = ICE_DFLT_TX_ITR; 116 q_vector->rx.itr_setting = ICE_DFLT_RX_ITR; 117 q_vector->tx.itr_mode = ITR_DYNAMIC; 118 q_vector->rx.itr_mode = ITR_DYNAMIC; 119 q_vector->tx.type = ICE_TX_CONTAINER; 120 q_vector->rx.type = ICE_RX_CONTAINER; 121 q_vector->irq.index = -ENOENT; 122 123 if (vsi->type == ICE_VSI_VF) { 124 q_vector->reg_idx = ice_calc_vf_reg_idx(vsi->vf, q_vector); 125 goto out; 126 } else if (vsi->type == ICE_VSI_CTRL && vsi->vf) { 127 struct ice_vsi *ctrl_vsi = ice_get_vf_ctrl_vsi(pf, vsi); 128 129 if (ctrl_vsi) { 130 if (unlikely(!ctrl_vsi->q_vectors)) { 131 err = -ENOENT; 132 goto err_free_q_vector; 133 } 134 135 q_vector->irq = ctrl_vsi->q_vectors[0]->irq; 136 goto skip_alloc; 137 } 138 } 139 140 q_vector->irq = ice_alloc_irq(pf, vsi->irq_dyn_alloc); 141 if (q_vector->irq.index < 0) { 142 err = -ENOMEM; 143 goto err_free_q_vector; 144 } 145 146 skip_alloc: 147 q_vector->reg_idx = q_vector->irq.index; 148 149 /* only set affinity_mask if the CPU is online */ 150 if (cpu_online(v_idx)) 151 cpumask_set_cpu(v_idx, &q_vector->affinity_mask); 152 153 /* This will not be called in the driver load path because the netdev 154 * will not be created yet. All other cases with register the NAPI 155 * handler here (i.e. resume, reset/rebuild, etc.) 156 */ 157 if (vsi->netdev) 158 netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll); 159 160 out: 161 /* tie q_vector and VSI together */ 162 vsi->q_vectors[v_idx] = q_vector; 163 164 return 0; 165 166 err_free_q_vector: 167 kfree(q_vector); 168 169 return err; 170 } 171 172 /** 173 * ice_free_q_vector - Free memory allocated for a specific interrupt vector 174 * @vsi: VSI having the memory freed 175 * @v_idx: index of the vector to be freed 176 */ 177 static void ice_free_q_vector(struct ice_vsi *vsi, int v_idx) 178 { 179 struct ice_q_vector *q_vector; 180 struct ice_pf *pf = vsi->back; 181 struct ice_tx_ring *tx_ring; 182 struct ice_rx_ring *rx_ring; 183 struct device *dev; 184 185 dev = ice_pf_to_dev(pf); 186 if (!vsi->q_vectors[v_idx]) { 187 dev_dbg(dev, "Queue vector at index %d not found\n", v_idx); 188 return; 189 } 190 q_vector = vsi->q_vectors[v_idx]; 191 192 ice_for_each_tx_ring(tx_ring, q_vector->tx) 193 tx_ring->q_vector = NULL; 194 ice_for_each_rx_ring(rx_ring, q_vector->rx) 195 rx_ring->q_vector = NULL; 196 197 /* only VSI with an associated netdev is set up with NAPI */ 198 if (vsi->netdev) 199 netif_napi_del(&q_vector->napi); 200 201 /* release MSIX interrupt if q_vector had interrupt allocated */ 202 if (q_vector->irq.index < 0) 203 goto free_q_vector; 204 205 /* only free last VF ctrl vsi interrupt */ 206 if (vsi->type == ICE_VSI_CTRL && vsi->vf && 207 ice_get_vf_ctrl_vsi(pf, vsi)) 208 goto free_q_vector; 209 210 ice_free_irq(pf, q_vector->irq); 211 212 free_q_vector: 213 kfree(q_vector); 214 vsi->q_vectors[v_idx] = NULL; 215 } 216 217 /** 218 * ice_cfg_itr_gran - set the ITR granularity to 2 usecs if not already set 219 * @hw: board specific structure 220 */ 221 static void ice_cfg_itr_gran(struct ice_hw *hw) 222 { 223 u32 regval = rd32(hw, GLINT_CTL); 224 225 /* no need to update global register if ITR gran is already set */ 226 if (!(regval & GLINT_CTL_DIS_AUTOMASK_M) && 227 (((regval & GLINT_CTL_ITR_GRAN_200_M) >> 228 GLINT_CTL_ITR_GRAN_200_S) == ICE_ITR_GRAN_US) && 229 (((regval & GLINT_CTL_ITR_GRAN_100_M) >> 230 GLINT_CTL_ITR_GRAN_100_S) == ICE_ITR_GRAN_US) && 231 (((regval & GLINT_CTL_ITR_GRAN_50_M) >> 232 GLINT_CTL_ITR_GRAN_50_S) == ICE_ITR_GRAN_US) && 233 (((regval & GLINT_CTL_ITR_GRAN_25_M) >> 234 GLINT_CTL_ITR_GRAN_25_S) == ICE_ITR_GRAN_US)) 235 return; 236 237 regval = ((ICE_ITR_GRAN_US << GLINT_CTL_ITR_GRAN_200_S) & 238 GLINT_CTL_ITR_GRAN_200_M) | 239 ((ICE_ITR_GRAN_US << GLINT_CTL_ITR_GRAN_100_S) & 240 GLINT_CTL_ITR_GRAN_100_M) | 241 ((ICE_ITR_GRAN_US << GLINT_CTL_ITR_GRAN_50_S) & 242 GLINT_CTL_ITR_GRAN_50_M) | 243 ((ICE_ITR_GRAN_US << GLINT_CTL_ITR_GRAN_25_S) & 244 GLINT_CTL_ITR_GRAN_25_M); 245 wr32(hw, GLINT_CTL, regval); 246 } 247 248 /** 249 * ice_calc_txq_handle - calculate the queue handle 250 * @vsi: VSI that ring belongs to 251 * @ring: ring to get the absolute queue index 252 * @tc: traffic class number 253 */ 254 static u16 ice_calc_txq_handle(struct ice_vsi *vsi, struct ice_tx_ring *ring, u8 tc) 255 { 256 WARN_ONCE(ice_ring_is_xdp(ring) && tc, "XDP ring can't belong to TC other than 0\n"); 257 258 if (ring->ch) 259 return ring->q_index - ring->ch->base_q; 260 261 /* Idea here for calculation is that we subtract the number of queue 262 * count from TC that ring belongs to from it's absolute queue index 263 * and as a result we get the queue's index within TC. 264 */ 265 return ring->q_index - vsi->tc_cfg.tc_info[tc].qoffset; 266 } 267 268 /** 269 * ice_eswitch_calc_txq_handle 270 * @ring: pointer to ring which unique index is needed 271 * 272 * To correctly work with many netdevs ring->q_index of Tx rings on switchdev 273 * VSI can repeat. Hardware ring setup requires unique q_index. Calculate it 274 * here by finding index in vsi->tx_rings of this ring. 275 * 276 * Return ICE_INVAL_Q_INDEX when index wasn't found. Should never happen, 277 * because VSI is get from ring->vsi, so it has to be present in this VSI. 278 */ 279 static u16 ice_eswitch_calc_txq_handle(struct ice_tx_ring *ring) 280 { 281 struct ice_vsi *vsi = ring->vsi; 282 int i; 283 284 ice_for_each_txq(vsi, i) { 285 if (vsi->tx_rings[i] == ring) 286 return i; 287 } 288 289 return ICE_INVAL_Q_INDEX; 290 } 291 292 /** 293 * ice_cfg_xps_tx_ring - Configure XPS for a Tx ring 294 * @ring: The Tx ring to configure 295 * 296 * This enables/disables XPS for a given Tx descriptor ring 297 * based on the TCs enabled for the VSI that ring belongs to. 298 */ 299 static void ice_cfg_xps_tx_ring(struct ice_tx_ring *ring) 300 { 301 if (!ring->q_vector || !ring->netdev) 302 return; 303 304 /* We only initialize XPS once, so as not to overwrite user settings */ 305 if (test_and_set_bit(ICE_TX_XPS_INIT_DONE, ring->xps_state)) 306 return; 307 308 netif_set_xps_queue(ring->netdev, &ring->q_vector->affinity_mask, 309 ring->q_index); 310 } 311 312 /** 313 * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance 314 * @ring: The Tx ring to configure 315 * @tlan_ctx: Pointer to the Tx LAN queue context structure to be initialized 316 * @pf_q: queue index in the PF space 317 * 318 * Configure the Tx descriptor ring in TLAN context. 319 */ 320 static void 321 ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) 322 { 323 struct ice_vsi *vsi = ring->vsi; 324 struct ice_hw *hw = &vsi->back->hw; 325 326 tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S; 327 328 tlan_ctx->port_num = vsi->port_info->lport; 329 330 /* Transmit Queue Length */ 331 tlan_ctx->qlen = ring->count; 332 333 ice_set_cgd_num(tlan_ctx, ring->dcb_tc); 334 335 /* PF number */ 336 tlan_ctx->pf_num = hw->pf_id; 337 338 /* queue belongs to a specific VSI type 339 * VF / VM index should be programmed per vmvf_type setting: 340 * for vmvf_type = VF, it is VF number between 0-256 341 * for vmvf_type = VM, it is VM number between 0-767 342 * for PF or EMP this field should be set to zero 343 */ 344 switch (vsi->type) { 345 case ICE_VSI_LB: 346 case ICE_VSI_CTRL: 347 case ICE_VSI_PF: 348 if (ring->ch) 349 tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; 350 else 351 tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; 352 break; 353 case ICE_VSI_VF: 354 /* Firmware expects vmvf_num to be absolute VF ID */ 355 tlan_ctx->vmvf_num = hw->func_caps.vf_base_id + vsi->vf->vf_id; 356 tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VF; 357 break; 358 case ICE_VSI_SWITCHDEV_CTRL: 359 tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; 360 break; 361 default: 362 return; 363 } 364 365 /* make sure the context is associated with the right VSI */ 366 if (ring->ch) 367 tlan_ctx->src_vsi = ring->ch->vsi_num; 368 else 369 tlan_ctx->src_vsi = ice_get_hw_vsi_num(hw, vsi->idx); 370 371 /* Restrict Tx timestamps to the PF VSI */ 372 switch (vsi->type) { 373 case ICE_VSI_PF: 374 tlan_ctx->tsyn_ena = 1; 375 break; 376 default: 377 break; 378 } 379 380 tlan_ctx->tso_ena = ICE_TX_LEGACY; 381 tlan_ctx->tso_qnum = pf_q; 382 383 /* Legacy or Advanced Host Interface: 384 * 0: Advanced Host Interface 385 * 1: Legacy Host Interface 386 */ 387 tlan_ctx->legacy_int = ICE_TX_LEGACY; 388 } 389 390 /** 391 * ice_rx_offset - Return expected offset into page to access data 392 * @rx_ring: Ring we are requesting offset of 393 * 394 * Returns the offset value for ring into the data buffer. 395 */ 396 static unsigned int ice_rx_offset(struct ice_rx_ring *rx_ring) 397 { 398 if (ice_ring_uses_build_skb(rx_ring)) 399 return ICE_SKB_PAD; 400 return 0; 401 } 402 403 /** 404 * ice_setup_rx_ctx - Configure a receive ring context 405 * @ring: The Rx ring to configure 406 * 407 * Configure the Rx descriptor ring in RLAN context. 408 */ 409 static int ice_setup_rx_ctx(struct ice_rx_ring *ring) 410 { 411 struct ice_vsi *vsi = ring->vsi; 412 u32 rxdid = ICE_RXDID_FLEX_NIC; 413 struct ice_rlan_ctx rlan_ctx; 414 struct ice_hw *hw; 415 u16 pf_q; 416 int err; 417 418 hw = &vsi->back->hw; 419 420 /* what is Rx queue number in global space of 2K Rx queues */ 421 pf_q = vsi->rxq_map[ring->q_index]; 422 423 /* clear the context structure first */ 424 memset(&rlan_ctx, 0, sizeof(rlan_ctx)); 425 426 /* Receive Queue Base Address. 427 * Indicates the starting address of the descriptor queue defined in 428 * 128 Byte units. 429 */ 430 rlan_ctx.base = ring->dma >> ICE_RLAN_BASE_S; 431 432 rlan_ctx.qlen = ring->count; 433 434 /* Receive Packet Data Buffer Size. 435 * The Packet Data Buffer Size is defined in 128 byte units. 436 */ 437 rlan_ctx.dbuf = DIV_ROUND_UP(ring->rx_buf_len, 438 BIT_ULL(ICE_RLAN_CTX_DBUF_S)); 439 440 /* use 32 byte descriptors */ 441 rlan_ctx.dsize = 1; 442 443 /* Strip the Ethernet CRC bytes before the packet is posted to host 444 * memory. 445 */ 446 rlan_ctx.crcstrip = !(ring->flags & ICE_RX_FLAGS_CRC_STRIP_DIS); 447 448 /* L2TSEL flag defines the reported L2 Tags in the receive descriptor 449 * and it needs to remain 1 for non-DVM capable configurations to not 450 * break backward compatibility for VF drivers. Setting this field to 0 451 * will cause the single/outer VLAN tag to be stripped to the L2TAG2_2ND 452 * field in the Rx descriptor. Setting it to 1 allows the VLAN tag to 453 * be stripped in L2TAG1 of the Rx descriptor, which is where VFs will 454 * check for the tag 455 */ 456 if (ice_is_dvm_ena(hw)) 457 if (vsi->type == ICE_VSI_VF && 458 ice_vf_is_port_vlan_ena(vsi->vf)) 459 rlan_ctx.l2tsel = 1; 460 else 461 rlan_ctx.l2tsel = 0; 462 else 463 rlan_ctx.l2tsel = 1; 464 465 rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT; 466 rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT; 467 rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT; 468 469 /* This controls whether VLAN is stripped from inner headers 470 * The VLAN in the inner L2 header is stripped to the receive 471 * descriptor if enabled by this flag. 472 */ 473 rlan_ctx.showiv = 0; 474 475 /* Max packet size for this queue - must not be set to a larger value 476 * than 5 x DBUF 477 */ 478 rlan_ctx.rxmax = min_t(u32, vsi->max_frame, 479 ICE_MAX_CHAINED_RX_BUFS * ring->rx_buf_len); 480 481 /* Rx queue threshold in units of 64 */ 482 rlan_ctx.lrxqthresh = 1; 483 484 /* Enable Flexible Descriptors in the queue context which 485 * allows this driver to select a specific receive descriptor format 486 * increasing context priority to pick up profile ID; default is 0x01; 487 * setting to 0x03 to ensure profile is programming if prev context is 488 * of same priority 489 */ 490 if (vsi->type != ICE_VSI_VF) 491 ice_write_qrxflxp_cntxt(hw, pf_q, rxdid, 0x3, true); 492 else 493 ice_write_qrxflxp_cntxt(hw, pf_q, ICE_RXDID_LEGACY_1, 0x3, 494 false); 495 496 /* Absolute queue number out of 2K needs to be passed */ 497 err = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q); 498 if (err) { 499 dev_err(ice_pf_to_dev(vsi->back), "Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n", 500 pf_q, err); 501 return -EIO; 502 } 503 504 if (vsi->type == ICE_VSI_VF) 505 return 0; 506 507 /* configure Rx buffer alignment */ 508 if (!vsi->netdev || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) 509 ice_clear_ring_build_skb_ena(ring); 510 else 511 ice_set_ring_build_skb_ena(ring); 512 513 ring->rx_offset = ice_rx_offset(ring); 514 515 /* init queue specific tail register */ 516 ring->tail = hw->hw_addr + QRX_TAIL(pf_q); 517 writel(0, ring->tail); 518 519 return 0; 520 } 521 522 /** 523 * ice_vsi_cfg_rxq - Configure an Rx queue 524 * @ring: the ring being configured 525 * 526 * Return 0 on success and a negative value on error. 527 */ 528 int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) 529 { 530 struct device *dev = ice_pf_to_dev(ring->vsi->back); 531 u32 num_bufs = ICE_RX_DESC_UNUSED(ring); 532 int err; 533 534 ring->rx_buf_len = ring->vsi->rx_buf_len; 535 536 if (ring->vsi->type == ICE_VSI_PF) { 537 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { 538 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 539 ring->q_index, 540 ring->q_vector->napi.napi_id, 541 ring->rx_buf_len); 542 if (err) 543 return err; 544 } 545 546 ring->xsk_pool = ice_xsk_pool(ring); 547 if (ring->xsk_pool) { 548 xdp_rxq_info_unreg(&ring->xdp_rxq); 549 550 ring->rx_buf_len = 551 xsk_pool_get_rx_frame_size(ring->xsk_pool); 552 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 553 ring->q_index, 554 ring->q_vector->napi.napi_id, 555 ring->rx_buf_len); 556 if (err) 557 return err; 558 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 559 MEM_TYPE_XSK_BUFF_POOL, 560 NULL); 561 if (err) 562 return err; 563 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 564 565 dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 566 ring->q_index); 567 } else { 568 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { 569 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 570 ring->q_index, 571 ring->q_vector->napi.napi_id, 572 ring->rx_buf_len); 573 if (err) 574 return err; 575 } 576 577 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 578 MEM_TYPE_PAGE_SHARED, 579 NULL); 580 if (err) 581 return err; 582 } 583 } 584 585 xdp_init_buff(&ring->xdp, ice_rx_pg_size(ring) / 2, &ring->xdp_rxq); 586 ring->xdp.data = NULL; 587 err = ice_setup_rx_ctx(ring); 588 if (err) { 589 dev_err(dev, "ice_setup_rx_ctx failed for RxQ %d, err %d\n", 590 ring->q_index, err); 591 return err; 592 } 593 594 if (ring->xsk_pool) { 595 bool ok; 596 597 if (!xsk_buff_can_alloc(ring->xsk_pool, num_bufs)) { 598 dev_warn(dev, "XSK buffer pool does not provide enough addresses to fill %d buffers on Rx ring %d\n", 599 num_bufs, ring->q_index); 600 dev_warn(dev, "Change Rx ring/fill queue size to avoid performance issues\n"); 601 602 return 0; 603 } 604 605 ok = ice_alloc_rx_bufs_zc(ring, num_bufs); 606 if (!ok) { 607 u16 pf_q = ring->vsi->rxq_map[ring->q_index]; 608 609 dev_info(dev, "Failed to allocate some buffers on XSK buffer pool enabled Rx ring %d (pf_q %d)\n", 610 ring->q_index, pf_q); 611 } 612 613 return 0; 614 } 615 616 ice_alloc_rx_bufs(ring, num_bufs); 617 618 return 0; 619 } 620 621 /** 622 * __ice_vsi_get_qs - helper function for assigning queues from PF to VSI 623 * @qs_cfg: gathered variables needed for pf->vsi queues assignment 624 * 625 * This function first tries to find contiguous space. If it is not successful, 626 * it tries with the scatter approach. 627 * 628 * Return 0 on success and -ENOMEM in case of no left space in PF queue bitmap 629 */ 630 int __ice_vsi_get_qs(struct ice_qs_cfg *qs_cfg) 631 { 632 int ret = 0; 633 634 ret = __ice_vsi_get_qs_contig(qs_cfg); 635 if (ret) { 636 /* contig failed, so try with scatter approach */ 637 qs_cfg->mapping_mode = ICE_VSI_MAP_SCATTER; 638 qs_cfg->q_count = min_t(unsigned int, qs_cfg->q_count, 639 qs_cfg->scatter_count); 640 ret = __ice_vsi_get_qs_sc(qs_cfg); 641 } 642 return ret; 643 } 644 645 /** 646 * ice_vsi_ctrl_one_rx_ring - start/stop VSI's Rx ring with no busy wait 647 * @vsi: the VSI being configured 648 * @ena: start or stop the Rx ring 649 * @rxq_idx: 0-based Rx queue index for the VSI passed in 650 * @wait: wait or don't wait for configuration to finish in hardware 651 * 652 * Return 0 on success and negative on error. 653 */ 654 int 655 ice_vsi_ctrl_one_rx_ring(struct ice_vsi *vsi, bool ena, u16 rxq_idx, bool wait) 656 { 657 int pf_q = vsi->rxq_map[rxq_idx]; 658 struct ice_pf *pf = vsi->back; 659 struct ice_hw *hw = &pf->hw; 660 u32 rx_reg; 661 662 rx_reg = rd32(hw, QRX_CTRL(pf_q)); 663 664 /* Skip if the queue is already in the requested state */ 665 if (ena == !!(rx_reg & QRX_CTRL_QENA_STAT_M)) 666 return 0; 667 668 /* turn on/off the queue */ 669 if (ena) 670 rx_reg |= QRX_CTRL_QENA_REQ_M; 671 else 672 rx_reg &= ~QRX_CTRL_QENA_REQ_M; 673 wr32(hw, QRX_CTRL(pf_q), rx_reg); 674 675 if (!wait) 676 return 0; 677 678 ice_flush(hw); 679 return ice_pf_rxq_wait(pf, pf_q, ena); 680 } 681 682 /** 683 * ice_vsi_wait_one_rx_ring - wait for a VSI's Rx ring to be stopped/started 684 * @vsi: the VSI being configured 685 * @ena: true/false to verify Rx ring has been enabled/disabled respectively 686 * @rxq_idx: 0-based Rx queue index for the VSI passed in 687 * 688 * This routine will wait for the given Rx queue of the VSI to reach the 689 * enabled or disabled state. Returns -ETIMEDOUT in case of failing to reach 690 * the requested state after multiple retries; else will return 0 in case of 691 * success. 692 */ 693 int ice_vsi_wait_one_rx_ring(struct ice_vsi *vsi, bool ena, u16 rxq_idx) 694 { 695 int pf_q = vsi->rxq_map[rxq_idx]; 696 struct ice_pf *pf = vsi->back; 697 698 return ice_pf_rxq_wait(pf, pf_q, ena); 699 } 700 701 /** 702 * ice_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 703 * @vsi: the VSI being configured 704 * 705 * We allocate one q_vector per queue interrupt. If allocation fails we 706 * return -ENOMEM. 707 */ 708 int ice_vsi_alloc_q_vectors(struct ice_vsi *vsi) 709 { 710 struct device *dev = ice_pf_to_dev(vsi->back); 711 u16 v_idx; 712 int err; 713 714 if (vsi->q_vectors[0]) { 715 dev_dbg(dev, "VSI %d has existing q_vectors\n", vsi->vsi_num); 716 return -EEXIST; 717 } 718 719 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) { 720 err = ice_vsi_alloc_q_vector(vsi, v_idx); 721 if (err) 722 goto err_out; 723 } 724 725 return 0; 726 727 err_out: 728 while (v_idx--) 729 ice_free_q_vector(vsi, v_idx); 730 731 dev_err(dev, "Failed to allocate %d q_vector for VSI %d, ret=%d\n", 732 vsi->num_q_vectors, vsi->vsi_num, err); 733 vsi->num_q_vectors = 0; 734 return err; 735 } 736 737 /** 738 * ice_vsi_map_rings_to_vectors - Map VSI rings to interrupt vectors 739 * @vsi: the VSI being configured 740 * 741 * This function maps descriptor rings to the queue-specific vectors allotted 742 * through the MSI-X enabling code. On a constrained vector budget, we map Tx 743 * and Rx rings to the vector as "efficiently" as possible. 744 */ 745 void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi) 746 { 747 int q_vectors = vsi->num_q_vectors; 748 u16 tx_rings_rem, rx_rings_rem; 749 int v_id; 750 751 /* initially assigning remaining rings count to VSIs num queue value */ 752 tx_rings_rem = vsi->num_txq; 753 rx_rings_rem = vsi->num_rxq; 754 755 for (v_id = 0; v_id < q_vectors; v_id++) { 756 struct ice_q_vector *q_vector = vsi->q_vectors[v_id]; 757 u8 tx_rings_per_v, rx_rings_per_v; 758 u16 q_id, q_base; 759 760 /* Tx rings mapping to vector */ 761 tx_rings_per_v = (u8)DIV_ROUND_UP(tx_rings_rem, 762 q_vectors - v_id); 763 q_vector->num_ring_tx = tx_rings_per_v; 764 q_vector->tx.tx_ring = NULL; 765 q_vector->tx.itr_idx = ICE_TX_ITR; 766 q_base = vsi->num_txq - tx_rings_rem; 767 768 for (q_id = q_base; q_id < (q_base + tx_rings_per_v); q_id++) { 769 struct ice_tx_ring *tx_ring = vsi->tx_rings[q_id]; 770 771 tx_ring->q_vector = q_vector; 772 tx_ring->next = q_vector->tx.tx_ring; 773 q_vector->tx.tx_ring = tx_ring; 774 } 775 tx_rings_rem -= tx_rings_per_v; 776 777 /* Rx rings mapping to vector */ 778 rx_rings_per_v = (u8)DIV_ROUND_UP(rx_rings_rem, 779 q_vectors - v_id); 780 q_vector->num_ring_rx = rx_rings_per_v; 781 q_vector->rx.rx_ring = NULL; 782 q_vector->rx.itr_idx = ICE_RX_ITR; 783 q_base = vsi->num_rxq - rx_rings_rem; 784 785 for (q_id = q_base; q_id < (q_base + rx_rings_per_v); q_id++) { 786 struct ice_rx_ring *rx_ring = vsi->rx_rings[q_id]; 787 788 rx_ring->q_vector = q_vector; 789 rx_ring->next = q_vector->rx.rx_ring; 790 q_vector->rx.rx_ring = rx_ring; 791 } 792 rx_rings_rem -= rx_rings_per_v; 793 } 794 } 795 796 /** 797 * ice_vsi_free_q_vectors - Free memory allocated for interrupt vectors 798 * @vsi: the VSI having memory freed 799 */ 800 void ice_vsi_free_q_vectors(struct ice_vsi *vsi) 801 { 802 int v_idx; 803 804 ice_for_each_q_vector(vsi, v_idx) 805 ice_free_q_vector(vsi, v_idx); 806 807 vsi->num_q_vectors = 0; 808 } 809 810 /** 811 * ice_vsi_cfg_txq - Configure single Tx queue 812 * @vsi: the VSI that queue belongs to 813 * @ring: Tx ring to be configured 814 * @qg_buf: queue group buffer 815 */ 816 int 817 ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, 818 struct ice_aqc_add_tx_qgrp *qg_buf) 819 { 820 u8 buf_len = struct_size(qg_buf, txqs, 1); 821 struct ice_tlan_ctx tlan_ctx = { 0 }; 822 struct ice_aqc_add_txqs_perq *txq; 823 struct ice_channel *ch = ring->ch; 824 struct ice_pf *pf = vsi->back; 825 struct ice_hw *hw = &pf->hw; 826 int status; 827 u16 pf_q; 828 u8 tc; 829 830 /* Configure XPS */ 831 ice_cfg_xps_tx_ring(ring); 832 833 pf_q = ring->reg_idx; 834 ice_setup_tx_ctx(ring, &tlan_ctx, pf_q); 835 /* copy context contents into the qg_buf */ 836 qg_buf->txqs[0].txq_id = cpu_to_le16(pf_q); 837 ice_set_ctx(hw, (u8 *)&tlan_ctx, qg_buf->txqs[0].txq_ctx, 838 ice_tlan_ctx_info); 839 840 /* init queue specific tail reg. It is referred as 841 * transmit comm scheduler queue doorbell. 842 */ 843 ring->tail = hw->hw_addr + QTX_COMM_DBELL(pf_q); 844 845 if (IS_ENABLED(CONFIG_DCB)) 846 tc = ring->dcb_tc; 847 else 848 tc = 0; 849 850 /* Add unique software queue handle of the Tx queue per 851 * TC into the VSI Tx ring 852 */ 853 if (vsi->type == ICE_VSI_SWITCHDEV_CTRL) { 854 ring->q_handle = ice_eswitch_calc_txq_handle(ring); 855 856 if (ring->q_handle == ICE_INVAL_Q_INDEX) 857 return -ENODEV; 858 } else { 859 ring->q_handle = ice_calc_txq_handle(vsi, ring, tc); 860 } 861 862 if (ch) 863 status = ice_ena_vsi_txq(vsi->port_info, ch->ch_vsi->idx, 0, 864 ring->q_handle, 1, qg_buf, buf_len, 865 NULL); 866 else 867 status = ice_ena_vsi_txq(vsi->port_info, vsi->idx, tc, 868 ring->q_handle, 1, qg_buf, buf_len, 869 NULL); 870 if (status) { 871 dev_err(ice_pf_to_dev(pf), "Failed to set LAN Tx queue context, error: %d\n", 872 status); 873 return status; 874 } 875 876 /* Add Tx Queue TEID into the VSI Tx ring from the 877 * response. This will complete configuring and 878 * enabling the queue. 879 */ 880 txq = &qg_buf->txqs[0]; 881 if (pf_q == le16_to_cpu(txq->txq_id)) 882 ring->txq_teid = le32_to_cpu(txq->q_teid); 883 884 return 0; 885 } 886 887 /** 888 * ice_cfg_itr - configure the initial interrupt throttle values 889 * @hw: pointer to the HW structure 890 * @q_vector: interrupt vector that's being configured 891 * 892 * Configure interrupt throttling values for the ring containers that are 893 * associated with the interrupt vector passed in. 894 */ 895 void ice_cfg_itr(struct ice_hw *hw, struct ice_q_vector *q_vector) 896 { 897 ice_cfg_itr_gran(hw); 898 899 if (q_vector->num_ring_rx) 900 ice_write_itr(&q_vector->rx, q_vector->rx.itr_setting); 901 902 if (q_vector->num_ring_tx) 903 ice_write_itr(&q_vector->tx, q_vector->tx.itr_setting); 904 905 ice_write_intrl(q_vector, q_vector->intrl); 906 } 907 908 /** 909 * ice_cfg_txq_interrupt - configure interrupt on Tx queue 910 * @vsi: the VSI being configured 911 * @txq: Tx queue being mapped to MSI-X vector 912 * @msix_idx: MSI-X vector index within the function 913 * @itr_idx: ITR index of the interrupt cause 914 * 915 * Configure interrupt on Tx queue by associating Tx queue to MSI-X vector 916 * within the function space. 917 */ 918 void 919 ice_cfg_txq_interrupt(struct ice_vsi *vsi, u16 txq, u16 msix_idx, u16 itr_idx) 920 { 921 struct ice_pf *pf = vsi->back; 922 struct ice_hw *hw = &pf->hw; 923 u32 val; 924 925 itr_idx = (itr_idx << QINT_TQCTL_ITR_INDX_S) & QINT_TQCTL_ITR_INDX_M; 926 927 val = QINT_TQCTL_CAUSE_ENA_M | itr_idx | 928 ((msix_idx << QINT_TQCTL_MSIX_INDX_S) & QINT_TQCTL_MSIX_INDX_M); 929 930 wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), val); 931 if (ice_is_xdp_ena_vsi(vsi)) { 932 u32 xdp_txq = txq + vsi->num_xdp_txq; 933 934 wr32(hw, QINT_TQCTL(vsi->txq_map[xdp_txq]), 935 val); 936 } 937 ice_flush(hw); 938 } 939 940 /** 941 * ice_cfg_rxq_interrupt - configure interrupt on Rx queue 942 * @vsi: the VSI being configured 943 * @rxq: Rx queue being mapped to MSI-X vector 944 * @msix_idx: MSI-X vector index within the function 945 * @itr_idx: ITR index of the interrupt cause 946 * 947 * Configure interrupt on Rx queue by associating Rx queue to MSI-X vector 948 * within the function space. 949 */ 950 void 951 ice_cfg_rxq_interrupt(struct ice_vsi *vsi, u16 rxq, u16 msix_idx, u16 itr_idx) 952 { 953 struct ice_pf *pf = vsi->back; 954 struct ice_hw *hw = &pf->hw; 955 u32 val; 956 957 itr_idx = (itr_idx << QINT_RQCTL_ITR_INDX_S) & QINT_RQCTL_ITR_INDX_M; 958 959 val = QINT_RQCTL_CAUSE_ENA_M | itr_idx | 960 ((msix_idx << QINT_RQCTL_MSIX_INDX_S) & QINT_RQCTL_MSIX_INDX_M); 961 962 wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), val); 963 964 ice_flush(hw); 965 } 966 967 /** 968 * ice_trigger_sw_intr - trigger a software interrupt 969 * @hw: pointer to the HW structure 970 * @q_vector: interrupt vector to trigger the software interrupt for 971 */ 972 void ice_trigger_sw_intr(struct ice_hw *hw, struct ice_q_vector *q_vector) 973 { 974 wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 975 (ICE_ITR_NONE << GLINT_DYN_CTL_ITR_INDX_S) | 976 GLINT_DYN_CTL_SWINT_TRIG_M | 977 GLINT_DYN_CTL_INTENA_M); 978 } 979 980 /** 981 * ice_vsi_stop_tx_ring - Disable single Tx ring 982 * @vsi: the VSI being configured 983 * @rst_src: reset source 984 * @rel_vmvf_num: Relative ID of VF/VM 985 * @ring: Tx ring to be stopped 986 * @txq_meta: Meta data of Tx ring to be stopped 987 */ 988 int 989 ice_vsi_stop_tx_ring(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, 990 u16 rel_vmvf_num, struct ice_tx_ring *ring, 991 struct ice_txq_meta *txq_meta) 992 { 993 struct ice_pf *pf = vsi->back; 994 struct ice_q_vector *q_vector; 995 struct ice_hw *hw = &pf->hw; 996 int status; 997 u32 val; 998 999 /* clear cause_ena bit for disabled queues */ 1000 val = rd32(hw, QINT_TQCTL(ring->reg_idx)); 1001 val &= ~QINT_TQCTL_CAUSE_ENA_M; 1002 wr32(hw, QINT_TQCTL(ring->reg_idx), val); 1003 1004 /* software is expected to wait for 100 ns */ 1005 ndelay(100); 1006 1007 /* trigger a software interrupt for the vector 1008 * associated to the queue to schedule NAPI handler 1009 */ 1010 q_vector = ring->q_vector; 1011 if (q_vector && !(vsi->vf && ice_is_vf_disabled(vsi->vf))) 1012 ice_trigger_sw_intr(hw, q_vector); 1013 1014 status = ice_dis_vsi_txq(vsi->port_info, txq_meta->vsi_idx, 1015 txq_meta->tc, 1, &txq_meta->q_handle, 1016 &txq_meta->q_id, &txq_meta->q_teid, rst_src, 1017 rel_vmvf_num, NULL); 1018 1019 /* if the disable queue command was exercised during an 1020 * active reset flow, -EBUSY is returned. 1021 * This is not an error as the reset operation disables 1022 * queues at the hardware level anyway. 1023 */ 1024 if (status == -EBUSY) { 1025 dev_dbg(ice_pf_to_dev(vsi->back), "Reset in progress. LAN Tx queues already disabled\n"); 1026 } else if (status == -ENOENT) { 1027 dev_dbg(ice_pf_to_dev(vsi->back), "LAN Tx queues do not exist, nothing to disable\n"); 1028 } else if (status) { 1029 dev_dbg(ice_pf_to_dev(vsi->back), "Failed to disable LAN Tx queues, error: %d\n", 1030 status); 1031 return status; 1032 } 1033 1034 return 0; 1035 } 1036 1037 /** 1038 * ice_fill_txq_meta - Prepare the Tx queue's meta data 1039 * @vsi: VSI that ring belongs to 1040 * @ring: ring that txq_meta will be based on 1041 * @txq_meta: a helper struct that wraps Tx queue's information 1042 * 1043 * Set up a helper struct that will contain all the necessary fields that 1044 * are needed for stopping Tx queue 1045 */ 1046 void 1047 ice_fill_txq_meta(struct ice_vsi *vsi, struct ice_tx_ring *ring, 1048 struct ice_txq_meta *txq_meta) 1049 { 1050 struct ice_channel *ch = ring->ch; 1051 u8 tc; 1052 1053 if (IS_ENABLED(CONFIG_DCB)) 1054 tc = ring->dcb_tc; 1055 else 1056 tc = 0; 1057 1058 txq_meta->q_id = ring->reg_idx; 1059 txq_meta->q_teid = ring->txq_teid; 1060 txq_meta->q_handle = ring->q_handle; 1061 if (ch) { 1062 txq_meta->vsi_idx = ch->ch_vsi->idx; 1063 txq_meta->tc = 0; 1064 } else { 1065 txq_meta->vsi_idx = vsi->idx; 1066 txq_meta->tc = tc; 1067 } 1068 } 1069