1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, Intel Corporation. */ 3 4 #include "ice_sched.h" 5 6 /** 7 * ice_sched_add_root_node - Insert the Tx scheduler root node in SW DB 8 * @pi: port information structure 9 * @info: Scheduler element information from firmware 10 * 11 * This function inserts the root node of the scheduling tree topology 12 * to the SW DB. 13 */ 14 static enum ice_status 15 ice_sched_add_root_node(struct ice_port_info *pi, 16 struct ice_aqc_txsched_elem_data *info) 17 { 18 struct ice_sched_node *root; 19 struct ice_hw *hw; 20 21 if (!pi) 22 return ICE_ERR_PARAM; 23 24 hw = pi->hw; 25 26 root = devm_kzalloc(ice_hw_to_dev(hw), sizeof(*root), GFP_KERNEL); 27 if (!root) 28 return ICE_ERR_NO_MEMORY; 29 30 /* coverity[suspicious_sizeof] */ 31 root->children = devm_kcalloc(ice_hw_to_dev(hw), hw->max_children[0], 32 sizeof(*root), GFP_KERNEL); 33 if (!root->children) { 34 devm_kfree(ice_hw_to_dev(hw), root); 35 return ICE_ERR_NO_MEMORY; 36 } 37 38 memcpy(&root->info, info, sizeof(*info)); 39 pi->root = root; 40 return 0; 41 } 42 43 /** 44 * ice_sched_find_node_by_teid - Find the Tx scheduler node in SW DB 45 * @start_node: pointer to the starting ice_sched_node struct in a sub-tree 46 * @teid: node TEID to search 47 * 48 * This function searches for a node matching the TEID in the scheduling tree 49 * from the SW DB. The search is recursive and is restricted by the number of 50 * layers it has searched through; stopping at the max supported layer. 51 * 52 * This function needs to be called when holding the port_info->sched_lock 53 */ 54 struct ice_sched_node * 55 ice_sched_find_node_by_teid(struct ice_sched_node *start_node, u32 teid) 56 { 57 u16 i; 58 59 /* The TEID is same as that of the start_node */ 60 if (ICE_TXSCHED_GET_NODE_TEID(start_node) == teid) 61 return start_node; 62 63 /* The node has no children or is at the max layer */ 64 if (!start_node->num_children || 65 start_node->tx_sched_layer >= ICE_AQC_TOPO_MAX_LEVEL_NUM || 66 start_node->info.data.elem_type == ICE_AQC_ELEM_TYPE_LEAF) 67 return NULL; 68 69 /* Check if TEID matches to any of the children nodes */ 70 for (i = 0; i < start_node->num_children; i++) 71 if (ICE_TXSCHED_GET_NODE_TEID(start_node->children[i]) == teid) 72 return start_node->children[i]; 73 74 /* Search within each child's sub-tree */ 75 for (i = 0; i < start_node->num_children; i++) { 76 struct ice_sched_node *tmp; 77 78 tmp = ice_sched_find_node_by_teid(start_node->children[i], 79 teid); 80 if (tmp) 81 return tmp; 82 } 83 84 return NULL; 85 } 86 87 /** 88 * ice_aqc_send_sched_elem_cmd - send scheduling elements cmd 89 * @hw: pointer to the HW struct 90 * @cmd_opc: cmd opcode 91 * @elems_req: number of elements to request 92 * @buf: pointer to buffer 93 * @buf_size: buffer size in bytes 94 * @elems_resp: returns total number of elements response 95 * @cd: pointer to command details structure or NULL 96 * 97 * This function sends a scheduling elements cmd (cmd_opc) 98 */ 99 static enum ice_status 100 ice_aqc_send_sched_elem_cmd(struct ice_hw *hw, enum ice_adminq_opc cmd_opc, 101 u16 elems_req, void *buf, u16 buf_size, 102 u16 *elems_resp, struct ice_sq_cd *cd) 103 { 104 struct ice_aqc_sched_elem_cmd *cmd; 105 struct ice_aq_desc desc; 106 enum ice_status status; 107 108 cmd = &desc.params.sched_elem_cmd; 109 ice_fill_dflt_direct_cmd_desc(&desc, cmd_opc); 110 cmd->num_elem_req = cpu_to_le16(elems_req); 111 desc.flags |= cpu_to_le16(ICE_AQ_FLAG_RD); 112 status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd); 113 if (!status && elems_resp) 114 *elems_resp = le16_to_cpu(cmd->num_elem_resp); 115 116 return status; 117 } 118 119 /** 120 * ice_aq_query_sched_elems - query scheduler elements 121 * @hw: pointer to the HW struct 122 * @elems_req: number of elements to query 123 * @buf: pointer to buffer 124 * @buf_size: buffer size in bytes 125 * @elems_ret: returns total number of elements returned 126 * @cd: pointer to command details structure or NULL 127 * 128 * Query scheduling elements (0x0404) 129 */ 130 enum ice_status 131 ice_aq_query_sched_elems(struct ice_hw *hw, u16 elems_req, 132 struct ice_aqc_txsched_elem_data *buf, u16 buf_size, 133 u16 *elems_ret, struct ice_sq_cd *cd) 134 { 135 return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_get_sched_elems, 136 elems_req, (void *)buf, buf_size, 137 elems_ret, cd); 138 } 139 140 /** 141 * ice_sched_add_node - Insert the Tx scheduler node in SW DB 142 * @pi: port information structure 143 * @layer: Scheduler layer of the node 144 * @info: Scheduler element information from firmware 145 * 146 * This function inserts a scheduler node to the SW DB. 147 */ 148 enum ice_status 149 ice_sched_add_node(struct ice_port_info *pi, u8 layer, 150 struct ice_aqc_txsched_elem_data *info) 151 { 152 struct ice_aqc_txsched_elem_data elem; 153 struct ice_sched_node *parent; 154 struct ice_sched_node *node; 155 enum ice_status status; 156 struct ice_hw *hw; 157 158 if (!pi) 159 return ICE_ERR_PARAM; 160 161 hw = pi->hw; 162 163 /* A valid parent node should be there */ 164 parent = ice_sched_find_node_by_teid(pi->root, 165 le32_to_cpu(info->parent_teid)); 166 if (!parent) { 167 ice_debug(hw, ICE_DBG_SCHED, "Parent Node not found for parent_teid=0x%x\n", 168 le32_to_cpu(info->parent_teid)); 169 return ICE_ERR_PARAM; 170 } 171 172 /* query the current node information from FW before adding it 173 * to the SW DB 174 */ 175 status = ice_sched_query_elem(hw, le32_to_cpu(info->node_teid), &elem); 176 if (status) 177 return status; 178 179 node = devm_kzalloc(ice_hw_to_dev(hw), sizeof(*node), GFP_KERNEL); 180 if (!node) 181 return ICE_ERR_NO_MEMORY; 182 if (hw->max_children[layer]) { 183 /* coverity[suspicious_sizeof] */ 184 node->children = devm_kcalloc(ice_hw_to_dev(hw), 185 hw->max_children[layer], 186 sizeof(*node), GFP_KERNEL); 187 if (!node->children) { 188 devm_kfree(ice_hw_to_dev(hw), node); 189 return ICE_ERR_NO_MEMORY; 190 } 191 } 192 193 node->in_use = true; 194 node->parent = parent; 195 node->tx_sched_layer = layer; 196 parent->children[parent->num_children++] = node; 197 node->info = elem; 198 return 0; 199 } 200 201 /** 202 * ice_aq_delete_sched_elems - delete scheduler elements 203 * @hw: pointer to the HW struct 204 * @grps_req: number of groups to delete 205 * @buf: pointer to buffer 206 * @buf_size: buffer size in bytes 207 * @grps_del: returns total number of elements deleted 208 * @cd: pointer to command details structure or NULL 209 * 210 * Delete scheduling elements (0x040F) 211 */ 212 static enum ice_status 213 ice_aq_delete_sched_elems(struct ice_hw *hw, u16 grps_req, 214 struct ice_aqc_delete_elem *buf, u16 buf_size, 215 u16 *grps_del, struct ice_sq_cd *cd) 216 { 217 return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_delete_sched_elems, 218 grps_req, (void *)buf, buf_size, 219 grps_del, cd); 220 } 221 222 /** 223 * ice_sched_remove_elems - remove nodes from HW 224 * @hw: pointer to the HW struct 225 * @parent: pointer to the parent node 226 * @num_nodes: number of nodes 227 * @node_teids: array of node teids to be deleted 228 * 229 * This function remove nodes from HW 230 */ 231 static enum ice_status 232 ice_sched_remove_elems(struct ice_hw *hw, struct ice_sched_node *parent, 233 u16 num_nodes, u32 *node_teids) 234 { 235 struct ice_aqc_delete_elem *buf; 236 u16 i, num_groups_removed = 0; 237 enum ice_status status; 238 u16 buf_size; 239 240 buf_size = struct_size(buf, teid, num_nodes); 241 buf = devm_kzalloc(ice_hw_to_dev(hw), buf_size, GFP_KERNEL); 242 if (!buf) 243 return ICE_ERR_NO_MEMORY; 244 245 buf->hdr.parent_teid = parent->info.node_teid; 246 buf->hdr.num_elems = cpu_to_le16(num_nodes); 247 for (i = 0; i < num_nodes; i++) 248 buf->teid[i] = cpu_to_le32(node_teids[i]); 249 250 status = ice_aq_delete_sched_elems(hw, 1, buf, buf_size, 251 &num_groups_removed, NULL); 252 if (status || num_groups_removed != 1) 253 ice_debug(hw, ICE_DBG_SCHED, "remove node failed FW error %d\n", 254 hw->adminq.sq_last_status); 255 256 devm_kfree(ice_hw_to_dev(hw), buf); 257 return status; 258 } 259 260 /** 261 * ice_sched_get_first_node - get the first node of the given layer 262 * @pi: port information structure 263 * @parent: pointer the base node of the subtree 264 * @layer: layer number 265 * 266 * This function retrieves the first node of the given layer from the subtree 267 */ 268 static struct ice_sched_node * 269 ice_sched_get_first_node(struct ice_port_info *pi, 270 struct ice_sched_node *parent, u8 layer) 271 { 272 return pi->sib_head[parent->tc_num][layer]; 273 } 274 275 /** 276 * ice_sched_get_tc_node - get pointer to TC node 277 * @pi: port information structure 278 * @tc: TC number 279 * 280 * This function returns the TC node pointer 281 */ 282 struct ice_sched_node *ice_sched_get_tc_node(struct ice_port_info *pi, u8 tc) 283 { 284 u8 i; 285 286 if (!pi || !pi->root) 287 return NULL; 288 for (i = 0; i < pi->root->num_children; i++) 289 if (pi->root->children[i]->tc_num == tc) 290 return pi->root->children[i]; 291 return NULL; 292 } 293 294 /** 295 * ice_free_sched_node - Free a Tx scheduler node from SW DB 296 * @pi: port information structure 297 * @node: pointer to the ice_sched_node struct 298 * 299 * This function frees up a node from SW DB as well as from HW 300 * 301 * This function needs to be called with the port_info->sched_lock held 302 */ 303 void ice_free_sched_node(struct ice_port_info *pi, struct ice_sched_node *node) 304 { 305 struct ice_sched_node *parent; 306 struct ice_hw *hw = pi->hw; 307 u8 i, j; 308 309 /* Free the children before freeing up the parent node 310 * The parent array is updated below and that shifts the nodes 311 * in the array. So always pick the first child if num children > 0 312 */ 313 while (node->num_children) 314 ice_free_sched_node(pi, node->children[0]); 315 316 /* Leaf, TC and root nodes can't be deleted by SW */ 317 if (node->tx_sched_layer >= hw->sw_entry_point_layer && 318 node->info.data.elem_type != ICE_AQC_ELEM_TYPE_TC && 319 node->info.data.elem_type != ICE_AQC_ELEM_TYPE_ROOT_PORT && 320 node->info.data.elem_type != ICE_AQC_ELEM_TYPE_LEAF) { 321 u32 teid = le32_to_cpu(node->info.node_teid); 322 323 ice_sched_remove_elems(hw, node->parent, 1, &teid); 324 } 325 parent = node->parent; 326 /* root has no parent */ 327 if (parent) { 328 struct ice_sched_node *p; 329 330 /* update the parent */ 331 for (i = 0; i < parent->num_children; i++) 332 if (parent->children[i] == node) { 333 for (j = i + 1; j < parent->num_children; j++) 334 parent->children[j - 1] = 335 parent->children[j]; 336 parent->num_children--; 337 break; 338 } 339 340 p = ice_sched_get_first_node(pi, node, node->tx_sched_layer); 341 while (p) { 342 if (p->sibling == node) { 343 p->sibling = node->sibling; 344 break; 345 } 346 p = p->sibling; 347 } 348 349 /* update the sibling head if head is getting removed */ 350 if (pi->sib_head[node->tc_num][node->tx_sched_layer] == node) 351 pi->sib_head[node->tc_num][node->tx_sched_layer] = 352 node->sibling; 353 } 354 355 /* leaf nodes have no children */ 356 if (node->children) 357 devm_kfree(ice_hw_to_dev(hw), node->children); 358 devm_kfree(ice_hw_to_dev(hw), node); 359 } 360 361 /** 362 * ice_aq_get_dflt_topo - gets default scheduler topology 363 * @hw: pointer to the HW struct 364 * @lport: logical port number 365 * @buf: pointer to buffer 366 * @buf_size: buffer size in bytes 367 * @num_branches: returns total number of queue to port branches 368 * @cd: pointer to command details structure or NULL 369 * 370 * Get default scheduler topology (0x400) 371 */ 372 static enum ice_status 373 ice_aq_get_dflt_topo(struct ice_hw *hw, u8 lport, 374 struct ice_aqc_get_topo_elem *buf, u16 buf_size, 375 u8 *num_branches, struct ice_sq_cd *cd) 376 { 377 struct ice_aqc_get_topo *cmd; 378 struct ice_aq_desc desc; 379 enum ice_status status; 380 381 cmd = &desc.params.get_topo; 382 ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_dflt_topo); 383 cmd->port_num = lport; 384 status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd); 385 if (!status && num_branches) 386 *num_branches = cmd->num_branches; 387 388 return status; 389 } 390 391 /** 392 * ice_aq_add_sched_elems - adds scheduling element 393 * @hw: pointer to the HW struct 394 * @grps_req: the number of groups that are requested to be added 395 * @buf: pointer to buffer 396 * @buf_size: buffer size in bytes 397 * @grps_added: returns total number of groups added 398 * @cd: pointer to command details structure or NULL 399 * 400 * Add scheduling elements (0x0401) 401 */ 402 static enum ice_status 403 ice_aq_add_sched_elems(struct ice_hw *hw, u16 grps_req, 404 struct ice_aqc_add_elem *buf, u16 buf_size, 405 u16 *grps_added, struct ice_sq_cd *cd) 406 { 407 return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_add_sched_elems, 408 grps_req, (void *)buf, buf_size, 409 grps_added, cd); 410 } 411 412 /** 413 * ice_aq_cfg_sched_elems - configures scheduler elements 414 * @hw: pointer to the HW struct 415 * @elems_req: number of elements to configure 416 * @buf: pointer to buffer 417 * @buf_size: buffer size in bytes 418 * @elems_cfgd: returns total number of elements configured 419 * @cd: pointer to command details structure or NULL 420 * 421 * Configure scheduling elements (0x0403) 422 */ 423 static enum ice_status 424 ice_aq_cfg_sched_elems(struct ice_hw *hw, u16 elems_req, 425 struct ice_aqc_txsched_elem_data *buf, u16 buf_size, 426 u16 *elems_cfgd, struct ice_sq_cd *cd) 427 { 428 return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_cfg_sched_elems, 429 elems_req, (void *)buf, buf_size, 430 elems_cfgd, cd); 431 } 432 433 /** 434 * ice_aq_move_sched_elems - move scheduler elements 435 * @hw: pointer to the HW struct 436 * @grps_req: number of groups to move 437 * @buf: pointer to buffer 438 * @buf_size: buffer size in bytes 439 * @grps_movd: returns total number of groups moved 440 * @cd: pointer to command details structure or NULL 441 * 442 * Move scheduling elements (0x0408) 443 */ 444 static enum ice_status 445 ice_aq_move_sched_elems(struct ice_hw *hw, u16 grps_req, 446 struct ice_aqc_move_elem *buf, u16 buf_size, 447 u16 *grps_movd, struct ice_sq_cd *cd) 448 { 449 return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_move_sched_elems, 450 grps_req, (void *)buf, buf_size, 451 grps_movd, cd); 452 } 453 454 /** 455 * ice_aq_suspend_sched_elems - suspend scheduler elements 456 * @hw: pointer to the HW struct 457 * @elems_req: number of elements to suspend 458 * @buf: pointer to buffer 459 * @buf_size: buffer size in bytes 460 * @elems_ret: returns total number of elements suspended 461 * @cd: pointer to command details structure or NULL 462 * 463 * Suspend scheduling elements (0x0409) 464 */ 465 static enum ice_status 466 ice_aq_suspend_sched_elems(struct ice_hw *hw, u16 elems_req, __le32 *buf, 467 u16 buf_size, u16 *elems_ret, struct ice_sq_cd *cd) 468 { 469 return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_suspend_sched_elems, 470 elems_req, (void *)buf, buf_size, 471 elems_ret, cd); 472 } 473 474 /** 475 * ice_aq_resume_sched_elems - resume scheduler elements 476 * @hw: pointer to the HW struct 477 * @elems_req: number of elements to resume 478 * @buf: pointer to buffer 479 * @buf_size: buffer size in bytes 480 * @elems_ret: returns total number of elements resumed 481 * @cd: pointer to command details structure or NULL 482 * 483 * resume scheduling elements (0x040A) 484 */ 485 static enum ice_status 486 ice_aq_resume_sched_elems(struct ice_hw *hw, u16 elems_req, __le32 *buf, 487 u16 buf_size, u16 *elems_ret, struct ice_sq_cd *cd) 488 { 489 return ice_aqc_send_sched_elem_cmd(hw, ice_aqc_opc_resume_sched_elems, 490 elems_req, (void *)buf, buf_size, 491 elems_ret, cd); 492 } 493 494 /** 495 * ice_aq_query_sched_res - query scheduler resource 496 * @hw: pointer to the HW struct 497 * @buf_size: buffer size in bytes 498 * @buf: pointer to buffer 499 * @cd: pointer to command details structure or NULL 500 * 501 * Query scheduler resource allocation (0x0412) 502 */ 503 static enum ice_status 504 ice_aq_query_sched_res(struct ice_hw *hw, u16 buf_size, 505 struct ice_aqc_query_txsched_res_resp *buf, 506 struct ice_sq_cd *cd) 507 { 508 struct ice_aq_desc desc; 509 510 ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_query_sched_res); 511 return ice_aq_send_cmd(hw, &desc, buf, buf_size, cd); 512 } 513 514 /** 515 * ice_sched_suspend_resume_elems - suspend or resume HW nodes 516 * @hw: pointer to the HW struct 517 * @num_nodes: number of nodes 518 * @node_teids: array of node teids to be suspended or resumed 519 * @suspend: true means suspend / false means resume 520 * 521 * This function suspends or resumes HW nodes 522 */ 523 static enum ice_status 524 ice_sched_suspend_resume_elems(struct ice_hw *hw, u8 num_nodes, u32 *node_teids, 525 bool suspend) 526 { 527 u16 i, buf_size, num_elem_ret = 0; 528 enum ice_status status; 529 __le32 *buf; 530 531 buf_size = sizeof(*buf) * num_nodes; 532 buf = devm_kzalloc(ice_hw_to_dev(hw), buf_size, GFP_KERNEL); 533 if (!buf) 534 return ICE_ERR_NO_MEMORY; 535 536 for (i = 0; i < num_nodes; i++) 537 buf[i] = cpu_to_le32(node_teids[i]); 538 539 if (suspend) 540 status = ice_aq_suspend_sched_elems(hw, num_nodes, buf, 541 buf_size, &num_elem_ret, 542 NULL); 543 else 544 status = ice_aq_resume_sched_elems(hw, num_nodes, buf, 545 buf_size, &num_elem_ret, 546 NULL); 547 if (status || num_elem_ret != num_nodes) 548 ice_debug(hw, ICE_DBG_SCHED, "suspend/resume failed\n"); 549 550 devm_kfree(ice_hw_to_dev(hw), buf); 551 return status; 552 } 553 554 /** 555 * ice_alloc_lan_q_ctx - allocate LAN queue contexts for the given VSI and TC 556 * @hw: pointer to the HW struct 557 * @vsi_handle: VSI handle 558 * @tc: TC number 559 * @new_numqs: number of queues 560 */ 561 static enum ice_status 562 ice_alloc_lan_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 new_numqs) 563 { 564 struct ice_vsi_ctx *vsi_ctx; 565 struct ice_q_ctx *q_ctx; 566 567 vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle); 568 if (!vsi_ctx) 569 return ICE_ERR_PARAM; 570 /* allocate LAN queue contexts */ 571 if (!vsi_ctx->lan_q_ctx[tc]) { 572 vsi_ctx->lan_q_ctx[tc] = devm_kcalloc(ice_hw_to_dev(hw), 573 new_numqs, 574 sizeof(*q_ctx), 575 GFP_KERNEL); 576 if (!vsi_ctx->lan_q_ctx[tc]) 577 return ICE_ERR_NO_MEMORY; 578 vsi_ctx->num_lan_q_entries[tc] = new_numqs; 579 return 0; 580 } 581 /* num queues are increased, update the queue contexts */ 582 if (new_numqs > vsi_ctx->num_lan_q_entries[tc]) { 583 u16 prev_num = vsi_ctx->num_lan_q_entries[tc]; 584 585 q_ctx = devm_kcalloc(ice_hw_to_dev(hw), new_numqs, 586 sizeof(*q_ctx), GFP_KERNEL); 587 if (!q_ctx) 588 return ICE_ERR_NO_MEMORY; 589 memcpy(q_ctx, vsi_ctx->lan_q_ctx[tc], 590 prev_num * sizeof(*q_ctx)); 591 devm_kfree(ice_hw_to_dev(hw), vsi_ctx->lan_q_ctx[tc]); 592 vsi_ctx->lan_q_ctx[tc] = q_ctx; 593 vsi_ctx->num_lan_q_entries[tc] = new_numqs; 594 } 595 return 0; 596 } 597 598 /** 599 * ice_alloc_rdma_q_ctx - allocate RDMA queue contexts for the given VSI and TC 600 * @hw: pointer to the HW struct 601 * @vsi_handle: VSI handle 602 * @tc: TC number 603 * @new_numqs: number of queues 604 */ 605 static enum ice_status 606 ice_alloc_rdma_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 new_numqs) 607 { 608 struct ice_vsi_ctx *vsi_ctx; 609 struct ice_q_ctx *q_ctx; 610 611 vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle); 612 if (!vsi_ctx) 613 return ICE_ERR_PARAM; 614 /* allocate RDMA queue contexts */ 615 if (!vsi_ctx->rdma_q_ctx[tc]) { 616 vsi_ctx->rdma_q_ctx[tc] = devm_kcalloc(ice_hw_to_dev(hw), 617 new_numqs, 618 sizeof(*q_ctx), 619 GFP_KERNEL); 620 if (!vsi_ctx->rdma_q_ctx[tc]) 621 return ICE_ERR_NO_MEMORY; 622 vsi_ctx->num_rdma_q_entries[tc] = new_numqs; 623 return 0; 624 } 625 /* num queues are increased, update the queue contexts */ 626 if (new_numqs > vsi_ctx->num_rdma_q_entries[tc]) { 627 u16 prev_num = vsi_ctx->num_rdma_q_entries[tc]; 628 629 q_ctx = devm_kcalloc(ice_hw_to_dev(hw), new_numqs, 630 sizeof(*q_ctx), GFP_KERNEL); 631 if (!q_ctx) 632 return ICE_ERR_NO_MEMORY; 633 memcpy(q_ctx, vsi_ctx->rdma_q_ctx[tc], 634 prev_num * sizeof(*q_ctx)); 635 devm_kfree(ice_hw_to_dev(hw), vsi_ctx->rdma_q_ctx[tc]); 636 vsi_ctx->rdma_q_ctx[tc] = q_ctx; 637 vsi_ctx->num_rdma_q_entries[tc] = new_numqs; 638 } 639 return 0; 640 } 641 642 /** 643 * ice_aq_rl_profile - performs a rate limiting task 644 * @hw: pointer to the HW struct 645 * @opcode: opcode for add, query, or remove profile(s) 646 * @num_profiles: the number of profiles 647 * @buf: pointer to buffer 648 * @buf_size: buffer size in bytes 649 * @num_processed: number of processed add or remove profile(s) to return 650 * @cd: pointer to command details structure 651 * 652 * RL profile function to add, query, or remove profile(s) 653 */ 654 static enum ice_status 655 ice_aq_rl_profile(struct ice_hw *hw, enum ice_adminq_opc opcode, 656 u16 num_profiles, struct ice_aqc_rl_profile_elem *buf, 657 u16 buf_size, u16 *num_processed, struct ice_sq_cd *cd) 658 { 659 struct ice_aqc_rl_profile *cmd; 660 struct ice_aq_desc desc; 661 enum ice_status status; 662 663 cmd = &desc.params.rl_profile; 664 665 ice_fill_dflt_direct_cmd_desc(&desc, opcode); 666 desc.flags |= cpu_to_le16(ICE_AQ_FLAG_RD); 667 cmd->num_profiles = cpu_to_le16(num_profiles); 668 status = ice_aq_send_cmd(hw, &desc, buf, buf_size, cd); 669 if (!status && num_processed) 670 *num_processed = le16_to_cpu(cmd->num_processed); 671 return status; 672 } 673 674 /** 675 * ice_aq_add_rl_profile - adds rate limiting profile(s) 676 * @hw: pointer to the HW struct 677 * @num_profiles: the number of profile(s) to be add 678 * @buf: pointer to buffer 679 * @buf_size: buffer size in bytes 680 * @num_profiles_added: total number of profiles added to return 681 * @cd: pointer to command details structure 682 * 683 * Add RL profile (0x0410) 684 */ 685 static enum ice_status 686 ice_aq_add_rl_profile(struct ice_hw *hw, u16 num_profiles, 687 struct ice_aqc_rl_profile_elem *buf, u16 buf_size, 688 u16 *num_profiles_added, struct ice_sq_cd *cd) 689 { 690 return ice_aq_rl_profile(hw, ice_aqc_opc_add_rl_profiles, num_profiles, 691 buf, buf_size, num_profiles_added, cd); 692 } 693 694 /** 695 * ice_aq_remove_rl_profile - removes RL profile(s) 696 * @hw: pointer to the HW struct 697 * @num_profiles: the number of profile(s) to remove 698 * @buf: pointer to buffer 699 * @buf_size: buffer size in bytes 700 * @num_profiles_removed: total number of profiles removed to return 701 * @cd: pointer to command details structure or NULL 702 * 703 * Remove RL profile (0x0415) 704 */ 705 static enum ice_status 706 ice_aq_remove_rl_profile(struct ice_hw *hw, u16 num_profiles, 707 struct ice_aqc_rl_profile_elem *buf, u16 buf_size, 708 u16 *num_profiles_removed, struct ice_sq_cd *cd) 709 { 710 return ice_aq_rl_profile(hw, ice_aqc_opc_remove_rl_profiles, 711 num_profiles, buf, buf_size, 712 num_profiles_removed, cd); 713 } 714 715 /** 716 * ice_sched_del_rl_profile - remove RL profile 717 * @hw: pointer to the HW struct 718 * @rl_info: rate limit profile information 719 * 720 * If the profile ID is not referenced anymore, it removes profile ID with 721 * its associated parameters from HW DB,and locally. The caller needs to 722 * hold scheduler lock. 723 */ 724 static enum ice_status 725 ice_sched_del_rl_profile(struct ice_hw *hw, 726 struct ice_aqc_rl_profile_info *rl_info) 727 { 728 struct ice_aqc_rl_profile_elem *buf; 729 u16 num_profiles_removed; 730 enum ice_status status; 731 u16 num_profiles = 1; 732 733 if (rl_info->prof_id_ref != 0) 734 return ICE_ERR_IN_USE; 735 736 /* Safe to remove profile ID */ 737 buf = &rl_info->profile; 738 status = ice_aq_remove_rl_profile(hw, num_profiles, buf, sizeof(*buf), 739 &num_profiles_removed, NULL); 740 if (status || num_profiles_removed != num_profiles) 741 return ICE_ERR_CFG; 742 743 /* Delete stale entry now */ 744 list_del(&rl_info->list_entry); 745 devm_kfree(ice_hw_to_dev(hw), rl_info); 746 return status; 747 } 748 749 /** 750 * ice_sched_clear_rl_prof - clears RL prof entries 751 * @pi: port information structure 752 * 753 * This function removes all RL profile from HW as well as from SW DB. 754 */ 755 static void ice_sched_clear_rl_prof(struct ice_port_info *pi) 756 { 757 u16 ln; 758 759 for (ln = 0; ln < pi->hw->num_tx_sched_layers; ln++) { 760 struct ice_aqc_rl_profile_info *rl_prof_elem; 761 struct ice_aqc_rl_profile_info *rl_prof_tmp; 762 763 list_for_each_entry_safe(rl_prof_elem, rl_prof_tmp, 764 &pi->rl_prof_list[ln], list_entry) { 765 struct ice_hw *hw = pi->hw; 766 enum ice_status status; 767 768 rl_prof_elem->prof_id_ref = 0; 769 status = ice_sched_del_rl_profile(hw, rl_prof_elem); 770 if (status) { 771 ice_debug(hw, ICE_DBG_SCHED, "Remove rl profile failed\n"); 772 /* On error, free mem required */ 773 list_del(&rl_prof_elem->list_entry); 774 devm_kfree(ice_hw_to_dev(hw), rl_prof_elem); 775 } 776 } 777 } 778 } 779 780 /** 781 * ice_sched_clear_agg - clears the aggregator related information 782 * @hw: pointer to the hardware structure 783 * 784 * This function removes aggregator list and free up aggregator related memory 785 * previously allocated. 786 */ 787 void ice_sched_clear_agg(struct ice_hw *hw) 788 { 789 struct ice_sched_agg_info *agg_info; 790 struct ice_sched_agg_info *atmp; 791 792 list_for_each_entry_safe(agg_info, atmp, &hw->agg_list, list_entry) { 793 struct ice_sched_agg_vsi_info *agg_vsi_info; 794 struct ice_sched_agg_vsi_info *vtmp; 795 796 list_for_each_entry_safe(agg_vsi_info, vtmp, 797 &agg_info->agg_vsi_list, list_entry) { 798 list_del(&agg_vsi_info->list_entry); 799 devm_kfree(ice_hw_to_dev(hw), agg_vsi_info); 800 } 801 list_del(&agg_info->list_entry); 802 devm_kfree(ice_hw_to_dev(hw), agg_info); 803 } 804 } 805 806 /** 807 * ice_sched_clear_tx_topo - clears the scheduler tree nodes 808 * @pi: port information structure 809 * 810 * This function removes all the nodes from HW as well as from SW DB. 811 */ 812 static void ice_sched_clear_tx_topo(struct ice_port_info *pi) 813 { 814 if (!pi) 815 return; 816 /* remove RL profiles related lists */ 817 ice_sched_clear_rl_prof(pi); 818 if (pi->root) { 819 ice_free_sched_node(pi, pi->root); 820 pi->root = NULL; 821 } 822 } 823 824 /** 825 * ice_sched_clear_port - clear the scheduler elements from SW DB for a port 826 * @pi: port information structure 827 * 828 * Cleanup scheduling elements from SW DB 829 */ 830 void ice_sched_clear_port(struct ice_port_info *pi) 831 { 832 if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY) 833 return; 834 835 pi->port_state = ICE_SCHED_PORT_STATE_INIT; 836 mutex_lock(&pi->sched_lock); 837 ice_sched_clear_tx_topo(pi); 838 mutex_unlock(&pi->sched_lock); 839 mutex_destroy(&pi->sched_lock); 840 } 841 842 /** 843 * ice_sched_cleanup_all - cleanup scheduler elements from SW DB for all ports 844 * @hw: pointer to the HW struct 845 * 846 * Cleanup scheduling elements from SW DB for all the ports 847 */ 848 void ice_sched_cleanup_all(struct ice_hw *hw) 849 { 850 if (!hw) 851 return; 852 853 if (hw->layer_info) { 854 devm_kfree(ice_hw_to_dev(hw), hw->layer_info); 855 hw->layer_info = NULL; 856 } 857 858 ice_sched_clear_port(hw->port_info); 859 860 hw->num_tx_sched_layers = 0; 861 hw->num_tx_sched_phys_layers = 0; 862 hw->flattened_layers = 0; 863 hw->max_cgds = 0; 864 } 865 866 /** 867 * ice_sched_add_elems - add nodes to HW and SW DB 868 * @pi: port information structure 869 * @tc_node: pointer to the branch node 870 * @parent: pointer to the parent node 871 * @layer: layer number to add nodes 872 * @num_nodes: number of nodes 873 * @num_nodes_added: pointer to num nodes added 874 * @first_node_teid: if new nodes are added then return the TEID of first node 875 * 876 * This function add nodes to HW as well as to SW DB for a given layer 877 */ 878 static enum ice_status 879 ice_sched_add_elems(struct ice_port_info *pi, struct ice_sched_node *tc_node, 880 struct ice_sched_node *parent, u8 layer, u16 num_nodes, 881 u16 *num_nodes_added, u32 *first_node_teid) 882 { 883 struct ice_sched_node *prev, *new_node; 884 struct ice_aqc_add_elem *buf; 885 u16 i, num_groups_added = 0; 886 enum ice_status status = 0; 887 struct ice_hw *hw = pi->hw; 888 size_t buf_size; 889 u32 teid; 890 891 buf_size = struct_size(buf, generic, num_nodes); 892 buf = devm_kzalloc(ice_hw_to_dev(hw), buf_size, GFP_KERNEL); 893 if (!buf) 894 return ICE_ERR_NO_MEMORY; 895 896 buf->hdr.parent_teid = parent->info.node_teid; 897 buf->hdr.num_elems = cpu_to_le16(num_nodes); 898 for (i = 0; i < num_nodes; i++) { 899 buf->generic[i].parent_teid = parent->info.node_teid; 900 buf->generic[i].data.elem_type = ICE_AQC_ELEM_TYPE_SE_GENERIC; 901 buf->generic[i].data.valid_sections = 902 ICE_AQC_ELEM_VALID_GENERIC | ICE_AQC_ELEM_VALID_CIR | 903 ICE_AQC_ELEM_VALID_EIR; 904 buf->generic[i].data.generic = 0; 905 buf->generic[i].data.cir_bw.bw_profile_idx = 906 cpu_to_le16(ICE_SCHED_DFLT_RL_PROF_ID); 907 buf->generic[i].data.cir_bw.bw_alloc = 908 cpu_to_le16(ICE_SCHED_DFLT_BW_WT); 909 buf->generic[i].data.eir_bw.bw_profile_idx = 910 cpu_to_le16(ICE_SCHED_DFLT_RL_PROF_ID); 911 buf->generic[i].data.eir_bw.bw_alloc = 912 cpu_to_le16(ICE_SCHED_DFLT_BW_WT); 913 } 914 915 status = ice_aq_add_sched_elems(hw, 1, buf, buf_size, 916 &num_groups_added, NULL); 917 if (status || num_groups_added != 1) { 918 ice_debug(hw, ICE_DBG_SCHED, "add node failed FW Error %d\n", 919 hw->adminq.sq_last_status); 920 devm_kfree(ice_hw_to_dev(hw), buf); 921 return ICE_ERR_CFG; 922 } 923 924 *num_nodes_added = num_nodes; 925 /* add nodes to the SW DB */ 926 for (i = 0; i < num_nodes; i++) { 927 status = ice_sched_add_node(pi, layer, &buf->generic[i]); 928 if (status) { 929 ice_debug(hw, ICE_DBG_SCHED, "add nodes in SW DB failed status =%d\n", 930 status); 931 break; 932 } 933 934 teid = le32_to_cpu(buf->generic[i].node_teid); 935 new_node = ice_sched_find_node_by_teid(parent, teid); 936 if (!new_node) { 937 ice_debug(hw, ICE_DBG_SCHED, "Node is missing for teid =%d\n", teid); 938 break; 939 } 940 941 new_node->sibling = NULL; 942 new_node->tc_num = tc_node->tc_num; 943 944 /* add it to previous node sibling pointer */ 945 /* Note: siblings are not linked across branches */ 946 prev = ice_sched_get_first_node(pi, tc_node, layer); 947 if (prev && prev != new_node) { 948 while (prev->sibling) 949 prev = prev->sibling; 950 prev->sibling = new_node; 951 } 952 953 /* initialize the sibling head */ 954 if (!pi->sib_head[tc_node->tc_num][layer]) 955 pi->sib_head[tc_node->tc_num][layer] = new_node; 956 957 if (i == 0) 958 *first_node_teid = teid; 959 } 960 961 devm_kfree(ice_hw_to_dev(hw), buf); 962 return status; 963 } 964 965 /** 966 * ice_sched_add_nodes_to_hw_layer - Add nodes to HW layer 967 * @pi: port information structure 968 * @tc_node: pointer to TC node 969 * @parent: pointer to parent node 970 * @layer: layer number to add nodes 971 * @num_nodes: number of nodes to be added 972 * @first_node_teid: pointer to the first node TEID 973 * @num_nodes_added: pointer to number of nodes added 974 * 975 * Add nodes into specific HW layer. 976 */ 977 static enum ice_status 978 ice_sched_add_nodes_to_hw_layer(struct ice_port_info *pi, 979 struct ice_sched_node *tc_node, 980 struct ice_sched_node *parent, u8 layer, 981 u16 num_nodes, u32 *first_node_teid, 982 u16 *num_nodes_added) 983 { 984 u16 max_child_nodes; 985 986 *num_nodes_added = 0; 987 988 if (!num_nodes) 989 return 0; 990 991 if (!parent || layer < pi->hw->sw_entry_point_layer) 992 return ICE_ERR_PARAM; 993 994 /* max children per node per layer */ 995 max_child_nodes = pi->hw->max_children[parent->tx_sched_layer]; 996 997 /* current number of children + required nodes exceed max children */ 998 if ((parent->num_children + num_nodes) > max_child_nodes) { 999 /* Fail if the parent is a TC node */ 1000 if (parent == tc_node) 1001 return ICE_ERR_CFG; 1002 return ICE_ERR_MAX_LIMIT; 1003 } 1004 1005 return ice_sched_add_elems(pi, tc_node, parent, layer, num_nodes, 1006 num_nodes_added, first_node_teid); 1007 } 1008 1009 /** 1010 * ice_sched_add_nodes_to_layer - Add nodes to a given layer 1011 * @pi: port information structure 1012 * @tc_node: pointer to TC node 1013 * @parent: pointer to parent node 1014 * @layer: layer number to add nodes 1015 * @num_nodes: number of nodes to be added 1016 * @first_node_teid: pointer to the first node TEID 1017 * @num_nodes_added: pointer to number of nodes added 1018 * 1019 * This function add nodes to a given layer. 1020 */ 1021 static enum ice_status 1022 ice_sched_add_nodes_to_layer(struct ice_port_info *pi, 1023 struct ice_sched_node *tc_node, 1024 struct ice_sched_node *parent, u8 layer, 1025 u16 num_nodes, u32 *first_node_teid, 1026 u16 *num_nodes_added) 1027 { 1028 u32 *first_teid_ptr = first_node_teid; 1029 u16 new_num_nodes = num_nodes; 1030 enum ice_status status = 0; 1031 1032 *num_nodes_added = 0; 1033 while (*num_nodes_added < num_nodes) { 1034 u16 max_child_nodes, num_added = 0; 1035 /* cppcheck-suppress unusedVariable */ 1036 u32 temp; 1037 1038 status = ice_sched_add_nodes_to_hw_layer(pi, tc_node, parent, 1039 layer, new_num_nodes, 1040 first_teid_ptr, 1041 &num_added); 1042 if (!status) 1043 *num_nodes_added += num_added; 1044 /* added more nodes than requested ? */ 1045 if (*num_nodes_added > num_nodes) { 1046 ice_debug(pi->hw, ICE_DBG_SCHED, "added extra nodes %d %d\n", num_nodes, 1047 *num_nodes_added); 1048 status = ICE_ERR_CFG; 1049 break; 1050 } 1051 /* break if all the nodes are added successfully */ 1052 if (!status && (*num_nodes_added == num_nodes)) 1053 break; 1054 /* break if the error is not max limit */ 1055 if (status && status != ICE_ERR_MAX_LIMIT) 1056 break; 1057 /* Exceeded the max children */ 1058 max_child_nodes = pi->hw->max_children[parent->tx_sched_layer]; 1059 /* utilize all the spaces if the parent is not full */ 1060 if (parent->num_children < max_child_nodes) { 1061 new_num_nodes = max_child_nodes - parent->num_children; 1062 } else { 1063 /* This parent is full, try the next sibling */ 1064 parent = parent->sibling; 1065 /* Don't modify the first node TEID memory if the 1066 * first node was added already in the above call. 1067 * Instead send some temp memory for all other 1068 * recursive calls. 1069 */ 1070 if (num_added) 1071 first_teid_ptr = &temp; 1072 1073 new_num_nodes = num_nodes - *num_nodes_added; 1074 } 1075 } 1076 return status; 1077 } 1078 1079 /** 1080 * ice_sched_get_qgrp_layer - get the current queue group layer number 1081 * @hw: pointer to the HW struct 1082 * 1083 * This function returns the current queue group layer number 1084 */ 1085 static u8 ice_sched_get_qgrp_layer(struct ice_hw *hw) 1086 { 1087 /* It's always total layers - 1, the array is 0 relative so -2 */ 1088 return hw->num_tx_sched_layers - ICE_QGRP_LAYER_OFFSET; 1089 } 1090 1091 /** 1092 * ice_sched_get_vsi_layer - get the current VSI layer number 1093 * @hw: pointer to the HW struct 1094 * 1095 * This function returns the current VSI layer number 1096 */ 1097 static u8 ice_sched_get_vsi_layer(struct ice_hw *hw) 1098 { 1099 /* Num Layers VSI layer 1100 * 9 6 1101 * 7 4 1102 * 5 or less sw_entry_point_layer 1103 */ 1104 /* calculate the VSI layer based on number of layers. */ 1105 if (hw->num_tx_sched_layers > ICE_VSI_LAYER_OFFSET + 1) { 1106 u8 layer = hw->num_tx_sched_layers - ICE_VSI_LAYER_OFFSET; 1107 1108 if (layer > hw->sw_entry_point_layer) 1109 return layer; 1110 } 1111 return hw->sw_entry_point_layer; 1112 } 1113 1114 /** 1115 * ice_sched_get_agg_layer - get the current aggregator layer number 1116 * @hw: pointer to the HW struct 1117 * 1118 * This function returns the current aggregator layer number 1119 */ 1120 static u8 ice_sched_get_agg_layer(struct ice_hw *hw) 1121 { 1122 /* Num Layers aggregator layer 1123 * 9 4 1124 * 7 or less sw_entry_point_layer 1125 */ 1126 /* calculate the aggregator layer based on number of layers. */ 1127 if (hw->num_tx_sched_layers > ICE_AGG_LAYER_OFFSET + 1) { 1128 u8 layer = hw->num_tx_sched_layers - ICE_AGG_LAYER_OFFSET; 1129 1130 if (layer > hw->sw_entry_point_layer) 1131 return layer; 1132 } 1133 return hw->sw_entry_point_layer; 1134 } 1135 1136 /** 1137 * ice_rm_dflt_leaf_node - remove the default leaf node in the tree 1138 * @pi: port information structure 1139 * 1140 * This function removes the leaf node that was created by the FW 1141 * during initialization 1142 */ 1143 static void ice_rm_dflt_leaf_node(struct ice_port_info *pi) 1144 { 1145 struct ice_sched_node *node; 1146 1147 node = pi->root; 1148 while (node) { 1149 if (!node->num_children) 1150 break; 1151 node = node->children[0]; 1152 } 1153 if (node && node->info.data.elem_type == ICE_AQC_ELEM_TYPE_LEAF) { 1154 u32 teid = le32_to_cpu(node->info.node_teid); 1155 enum ice_status status; 1156 1157 /* remove the default leaf node */ 1158 status = ice_sched_remove_elems(pi->hw, node->parent, 1, &teid); 1159 if (!status) 1160 ice_free_sched_node(pi, node); 1161 } 1162 } 1163 1164 /** 1165 * ice_sched_rm_dflt_nodes - free the default nodes in the tree 1166 * @pi: port information structure 1167 * 1168 * This function frees all the nodes except root and TC that were created by 1169 * the FW during initialization 1170 */ 1171 static void ice_sched_rm_dflt_nodes(struct ice_port_info *pi) 1172 { 1173 struct ice_sched_node *node; 1174 1175 ice_rm_dflt_leaf_node(pi); 1176 1177 /* remove the default nodes except TC and root nodes */ 1178 node = pi->root; 1179 while (node) { 1180 if (node->tx_sched_layer >= pi->hw->sw_entry_point_layer && 1181 node->info.data.elem_type != ICE_AQC_ELEM_TYPE_TC && 1182 node->info.data.elem_type != ICE_AQC_ELEM_TYPE_ROOT_PORT) { 1183 ice_free_sched_node(pi, node); 1184 break; 1185 } 1186 1187 if (!node->num_children) 1188 break; 1189 node = node->children[0]; 1190 } 1191 } 1192 1193 /** 1194 * ice_sched_init_port - Initialize scheduler by querying information from FW 1195 * @pi: port info structure for the tree to cleanup 1196 * 1197 * This function is the initial call to find the total number of Tx scheduler 1198 * resources, default topology created by firmware and storing the information 1199 * in SW DB. 1200 */ 1201 enum ice_status ice_sched_init_port(struct ice_port_info *pi) 1202 { 1203 struct ice_aqc_get_topo_elem *buf; 1204 enum ice_status status; 1205 struct ice_hw *hw; 1206 u8 num_branches; 1207 u16 num_elems; 1208 u8 i, j; 1209 1210 if (!pi) 1211 return ICE_ERR_PARAM; 1212 hw = pi->hw; 1213 1214 /* Query the Default Topology from FW */ 1215 buf = devm_kzalloc(ice_hw_to_dev(hw), ICE_AQ_MAX_BUF_LEN, GFP_KERNEL); 1216 if (!buf) 1217 return ICE_ERR_NO_MEMORY; 1218 1219 /* Query default scheduling tree topology */ 1220 status = ice_aq_get_dflt_topo(hw, pi->lport, buf, ICE_AQ_MAX_BUF_LEN, 1221 &num_branches, NULL); 1222 if (status) 1223 goto err_init_port; 1224 1225 /* num_branches should be between 1-8 */ 1226 if (num_branches < 1 || num_branches > ICE_TXSCHED_MAX_BRANCHES) { 1227 ice_debug(hw, ICE_DBG_SCHED, "num_branches unexpected %d\n", 1228 num_branches); 1229 status = ICE_ERR_PARAM; 1230 goto err_init_port; 1231 } 1232 1233 /* get the number of elements on the default/first branch */ 1234 num_elems = le16_to_cpu(buf[0].hdr.num_elems); 1235 1236 /* num_elems should always be between 1-9 */ 1237 if (num_elems < 1 || num_elems > ICE_AQC_TOPO_MAX_LEVEL_NUM) { 1238 ice_debug(hw, ICE_DBG_SCHED, "num_elems unexpected %d\n", 1239 num_elems); 1240 status = ICE_ERR_PARAM; 1241 goto err_init_port; 1242 } 1243 1244 /* If the last node is a leaf node then the index of the queue group 1245 * layer is two less than the number of elements. 1246 */ 1247 if (num_elems > 2 && buf[0].generic[num_elems - 1].data.elem_type == 1248 ICE_AQC_ELEM_TYPE_LEAF) 1249 pi->last_node_teid = 1250 le32_to_cpu(buf[0].generic[num_elems - 2].node_teid); 1251 else 1252 pi->last_node_teid = 1253 le32_to_cpu(buf[0].generic[num_elems - 1].node_teid); 1254 1255 /* Insert the Tx Sched root node */ 1256 status = ice_sched_add_root_node(pi, &buf[0].generic[0]); 1257 if (status) 1258 goto err_init_port; 1259 1260 /* Parse the default tree and cache the information */ 1261 for (i = 0; i < num_branches; i++) { 1262 num_elems = le16_to_cpu(buf[i].hdr.num_elems); 1263 1264 /* Skip root element as already inserted */ 1265 for (j = 1; j < num_elems; j++) { 1266 /* update the sw entry point */ 1267 if (buf[0].generic[j].data.elem_type == 1268 ICE_AQC_ELEM_TYPE_ENTRY_POINT) 1269 hw->sw_entry_point_layer = j; 1270 1271 status = ice_sched_add_node(pi, j, &buf[i].generic[j]); 1272 if (status) 1273 goto err_init_port; 1274 } 1275 } 1276 1277 /* Remove the default nodes. */ 1278 if (pi->root) 1279 ice_sched_rm_dflt_nodes(pi); 1280 1281 /* initialize the port for handling the scheduler tree */ 1282 pi->port_state = ICE_SCHED_PORT_STATE_READY; 1283 mutex_init(&pi->sched_lock); 1284 for (i = 0; i < ICE_AQC_TOPO_MAX_LEVEL_NUM; i++) 1285 INIT_LIST_HEAD(&pi->rl_prof_list[i]); 1286 1287 err_init_port: 1288 if (status && pi->root) { 1289 ice_free_sched_node(pi, pi->root); 1290 pi->root = NULL; 1291 } 1292 1293 devm_kfree(ice_hw_to_dev(hw), buf); 1294 return status; 1295 } 1296 1297 /** 1298 * ice_sched_query_res_alloc - query the FW for num of logical sched layers 1299 * @hw: pointer to the HW struct 1300 * 1301 * query FW for allocated scheduler resources and store in HW struct 1302 */ 1303 enum ice_status ice_sched_query_res_alloc(struct ice_hw *hw) 1304 { 1305 struct ice_aqc_query_txsched_res_resp *buf; 1306 enum ice_status status = 0; 1307 __le16 max_sibl; 1308 u16 i; 1309 1310 if (hw->layer_info) 1311 return status; 1312 1313 buf = devm_kzalloc(ice_hw_to_dev(hw), sizeof(*buf), GFP_KERNEL); 1314 if (!buf) 1315 return ICE_ERR_NO_MEMORY; 1316 1317 status = ice_aq_query_sched_res(hw, sizeof(*buf), buf, NULL); 1318 if (status) 1319 goto sched_query_out; 1320 1321 hw->num_tx_sched_layers = le16_to_cpu(buf->sched_props.logical_levels); 1322 hw->num_tx_sched_phys_layers = 1323 le16_to_cpu(buf->sched_props.phys_levels); 1324 hw->flattened_layers = buf->sched_props.flattening_bitmap; 1325 hw->max_cgds = buf->sched_props.max_pf_cgds; 1326 1327 /* max sibling group size of current layer refers to the max children 1328 * of the below layer node. 1329 * layer 1 node max children will be layer 2 max sibling group size 1330 * layer 2 node max children will be layer 3 max sibling group size 1331 * and so on. This array will be populated from root (index 0) to 1332 * qgroup layer 7. Leaf node has no children. 1333 */ 1334 for (i = 0; i < hw->num_tx_sched_layers - 1; i++) { 1335 max_sibl = buf->layer_props[i + 1].max_sibl_grp_sz; 1336 hw->max_children[i] = le16_to_cpu(max_sibl); 1337 } 1338 1339 hw->layer_info = devm_kmemdup(ice_hw_to_dev(hw), buf->layer_props, 1340 (hw->num_tx_sched_layers * 1341 sizeof(*hw->layer_info)), 1342 GFP_KERNEL); 1343 if (!hw->layer_info) { 1344 status = ICE_ERR_NO_MEMORY; 1345 goto sched_query_out; 1346 } 1347 1348 sched_query_out: 1349 devm_kfree(ice_hw_to_dev(hw), buf); 1350 return status; 1351 } 1352 1353 /** 1354 * ice_sched_get_psm_clk_freq - determine the PSM clock frequency 1355 * @hw: pointer to the HW struct 1356 * 1357 * Determine the PSM clock frequency and store in HW struct 1358 */ 1359 void ice_sched_get_psm_clk_freq(struct ice_hw *hw) 1360 { 1361 u32 val, clk_src; 1362 1363 val = rd32(hw, GLGEN_CLKSTAT_SRC); 1364 clk_src = (val & GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_M) >> 1365 GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_S; 1366 1367 #define PSM_CLK_SRC_367_MHZ 0x0 1368 #define PSM_CLK_SRC_416_MHZ 0x1 1369 #define PSM_CLK_SRC_446_MHZ 0x2 1370 #define PSM_CLK_SRC_390_MHZ 0x3 1371 1372 switch (clk_src) { 1373 case PSM_CLK_SRC_367_MHZ: 1374 hw->psm_clk_freq = ICE_PSM_CLK_367MHZ_IN_HZ; 1375 break; 1376 case PSM_CLK_SRC_416_MHZ: 1377 hw->psm_clk_freq = ICE_PSM_CLK_416MHZ_IN_HZ; 1378 break; 1379 case PSM_CLK_SRC_446_MHZ: 1380 hw->psm_clk_freq = ICE_PSM_CLK_446MHZ_IN_HZ; 1381 break; 1382 case PSM_CLK_SRC_390_MHZ: 1383 hw->psm_clk_freq = ICE_PSM_CLK_390MHZ_IN_HZ; 1384 break; 1385 default: 1386 ice_debug(hw, ICE_DBG_SCHED, "PSM clk_src unexpected %u\n", 1387 clk_src); 1388 /* fall back to a safe default */ 1389 hw->psm_clk_freq = ICE_PSM_CLK_446MHZ_IN_HZ; 1390 } 1391 } 1392 1393 /** 1394 * ice_sched_find_node_in_subtree - Find node in part of base node subtree 1395 * @hw: pointer to the HW struct 1396 * @base: pointer to the base node 1397 * @node: pointer to the node to search 1398 * 1399 * This function checks whether a given node is part of the base node 1400 * subtree or not 1401 */ 1402 static bool 1403 ice_sched_find_node_in_subtree(struct ice_hw *hw, struct ice_sched_node *base, 1404 struct ice_sched_node *node) 1405 { 1406 u8 i; 1407 1408 for (i = 0; i < base->num_children; i++) { 1409 struct ice_sched_node *child = base->children[i]; 1410 1411 if (node == child) 1412 return true; 1413 1414 if (child->tx_sched_layer > node->tx_sched_layer) 1415 return false; 1416 1417 /* this recursion is intentional, and wouldn't 1418 * go more than 8 calls 1419 */ 1420 if (ice_sched_find_node_in_subtree(hw, child, node)) 1421 return true; 1422 } 1423 return false; 1424 } 1425 1426 /** 1427 * ice_sched_get_free_qgrp - Scan all queue group siblings and find a free node 1428 * @pi: port information structure 1429 * @vsi_node: software VSI handle 1430 * @qgrp_node: first queue group node identified for scanning 1431 * @owner: LAN or RDMA 1432 * 1433 * This function retrieves a free LAN or RDMA queue group node by scanning 1434 * qgrp_node and its siblings for the queue group with the fewest number 1435 * of queues currently assigned. 1436 */ 1437 static struct ice_sched_node * 1438 ice_sched_get_free_qgrp(struct ice_port_info *pi, 1439 struct ice_sched_node *vsi_node, 1440 struct ice_sched_node *qgrp_node, u8 owner) 1441 { 1442 struct ice_sched_node *min_qgrp; 1443 u8 min_children; 1444 1445 if (!qgrp_node) 1446 return qgrp_node; 1447 min_children = qgrp_node->num_children; 1448 if (!min_children) 1449 return qgrp_node; 1450 min_qgrp = qgrp_node; 1451 /* scan all queue groups until find a node which has less than the 1452 * minimum number of children. This way all queue group nodes get 1453 * equal number of shares and active. The bandwidth will be equally 1454 * distributed across all queues. 1455 */ 1456 while (qgrp_node) { 1457 /* make sure the qgroup node is part of the VSI subtree */ 1458 if (ice_sched_find_node_in_subtree(pi->hw, vsi_node, qgrp_node)) 1459 if (qgrp_node->num_children < min_children && 1460 qgrp_node->owner == owner) { 1461 /* replace the new min queue group node */ 1462 min_qgrp = qgrp_node; 1463 min_children = min_qgrp->num_children; 1464 /* break if it has no children, */ 1465 if (!min_children) 1466 break; 1467 } 1468 qgrp_node = qgrp_node->sibling; 1469 } 1470 return min_qgrp; 1471 } 1472 1473 /** 1474 * ice_sched_get_free_qparent - Get a free LAN or RDMA queue group node 1475 * @pi: port information structure 1476 * @vsi_handle: software VSI handle 1477 * @tc: branch number 1478 * @owner: LAN or RDMA 1479 * 1480 * This function retrieves a free LAN or RDMA queue group node 1481 */ 1482 struct ice_sched_node * 1483 ice_sched_get_free_qparent(struct ice_port_info *pi, u16 vsi_handle, u8 tc, 1484 u8 owner) 1485 { 1486 struct ice_sched_node *vsi_node, *qgrp_node; 1487 struct ice_vsi_ctx *vsi_ctx; 1488 u16 max_children; 1489 u8 qgrp_layer; 1490 1491 qgrp_layer = ice_sched_get_qgrp_layer(pi->hw); 1492 max_children = pi->hw->max_children[qgrp_layer]; 1493 1494 vsi_ctx = ice_get_vsi_ctx(pi->hw, vsi_handle); 1495 if (!vsi_ctx) 1496 return NULL; 1497 vsi_node = vsi_ctx->sched.vsi_node[tc]; 1498 /* validate invalid VSI ID */ 1499 if (!vsi_node) 1500 return NULL; 1501 1502 /* get the first queue group node from VSI sub-tree */ 1503 qgrp_node = ice_sched_get_first_node(pi, vsi_node, qgrp_layer); 1504 while (qgrp_node) { 1505 /* make sure the qgroup node is part of the VSI subtree */ 1506 if (ice_sched_find_node_in_subtree(pi->hw, vsi_node, qgrp_node)) 1507 if (qgrp_node->num_children < max_children && 1508 qgrp_node->owner == owner) 1509 break; 1510 qgrp_node = qgrp_node->sibling; 1511 } 1512 1513 /* Select the best queue group */ 1514 return ice_sched_get_free_qgrp(pi, vsi_node, qgrp_node, owner); 1515 } 1516 1517 /** 1518 * ice_sched_get_vsi_node - Get a VSI node based on VSI ID 1519 * @pi: pointer to the port information structure 1520 * @tc_node: pointer to the TC node 1521 * @vsi_handle: software VSI handle 1522 * 1523 * This function retrieves a VSI node for a given VSI ID from a given 1524 * TC branch 1525 */ 1526 static struct ice_sched_node * 1527 ice_sched_get_vsi_node(struct ice_port_info *pi, struct ice_sched_node *tc_node, 1528 u16 vsi_handle) 1529 { 1530 struct ice_sched_node *node; 1531 u8 vsi_layer; 1532 1533 vsi_layer = ice_sched_get_vsi_layer(pi->hw); 1534 node = ice_sched_get_first_node(pi, tc_node, vsi_layer); 1535 1536 /* Check whether it already exists */ 1537 while (node) { 1538 if (node->vsi_handle == vsi_handle) 1539 return node; 1540 node = node->sibling; 1541 } 1542 1543 return node; 1544 } 1545 1546 /** 1547 * ice_sched_get_agg_node - Get an aggregator node based on aggregator ID 1548 * @pi: pointer to the port information structure 1549 * @tc_node: pointer to the TC node 1550 * @agg_id: aggregator ID 1551 * 1552 * This function retrieves an aggregator node for a given aggregator ID from 1553 * a given TC branch 1554 */ 1555 static struct ice_sched_node * 1556 ice_sched_get_agg_node(struct ice_port_info *pi, struct ice_sched_node *tc_node, 1557 u32 agg_id) 1558 { 1559 struct ice_sched_node *node; 1560 struct ice_hw *hw = pi->hw; 1561 u8 agg_layer; 1562 1563 if (!hw) 1564 return NULL; 1565 agg_layer = ice_sched_get_agg_layer(hw); 1566 node = ice_sched_get_first_node(pi, tc_node, agg_layer); 1567 1568 /* Check whether it already exists */ 1569 while (node) { 1570 if (node->agg_id == agg_id) 1571 return node; 1572 node = node->sibling; 1573 } 1574 1575 return node; 1576 } 1577 1578 /** 1579 * ice_sched_calc_vsi_child_nodes - calculate number of VSI child nodes 1580 * @hw: pointer to the HW struct 1581 * @num_qs: number of queues 1582 * @num_nodes: num nodes array 1583 * 1584 * This function calculates the number of VSI child nodes based on the 1585 * number of queues. 1586 */ 1587 static void 1588 ice_sched_calc_vsi_child_nodes(struct ice_hw *hw, u16 num_qs, u16 *num_nodes) 1589 { 1590 u16 num = num_qs; 1591 u8 i, qgl, vsil; 1592 1593 qgl = ice_sched_get_qgrp_layer(hw); 1594 vsil = ice_sched_get_vsi_layer(hw); 1595 1596 /* calculate num nodes from queue group to VSI layer */ 1597 for (i = qgl; i > vsil; i--) { 1598 /* round to the next integer if there is a remainder */ 1599 num = DIV_ROUND_UP(num, hw->max_children[i]); 1600 1601 /* need at least one node */ 1602 num_nodes[i] = num ? num : 1; 1603 } 1604 } 1605 1606 /** 1607 * ice_sched_add_vsi_child_nodes - add VSI child nodes to tree 1608 * @pi: port information structure 1609 * @vsi_handle: software VSI handle 1610 * @tc_node: pointer to the TC node 1611 * @num_nodes: pointer to the num nodes that needs to be added per layer 1612 * @owner: node owner (LAN or RDMA) 1613 * 1614 * This function adds the VSI child nodes to tree. It gets called for 1615 * LAN and RDMA separately. 1616 */ 1617 static enum ice_status 1618 ice_sched_add_vsi_child_nodes(struct ice_port_info *pi, u16 vsi_handle, 1619 struct ice_sched_node *tc_node, u16 *num_nodes, 1620 u8 owner) 1621 { 1622 struct ice_sched_node *parent, *node; 1623 struct ice_hw *hw = pi->hw; 1624 enum ice_status status; 1625 u32 first_node_teid; 1626 u16 num_added = 0; 1627 u8 i, qgl, vsil; 1628 1629 qgl = ice_sched_get_qgrp_layer(hw); 1630 vsil = ice_sched_get_vsi_layer(hw); 1631 parent = ice_sched_get_vsi_node(pi, tc_node, vsi_handle); 1632 for (i = vsil + 1; i <= qgl; i++) { 1633 if (!parent) 1634 return ICE_ERR_CFG; 1635 1636 status = ice_sched_add_nodes_to_layer(pi, tc_node, parent, i, 1637 num_nodes[i], 1638 &first_node_teid, 1639 &num_added); 1640 if (status || num_nodes[i] != num_added) 1641 return ICE_ERR_CFG; 1642 1643 /* The newly added node can be a new parent for the next 1644 * layer nodes 1645 */ 1646 if (num_added) { 1647 parent = ice_sched_find_node_by_teid(tc_node, 1648 first_node_teid); 1649 node = parent; 1650 while (node) { 1651 node->owner = owner; 1652 node = node->sibling; 1653 } 1654 } else { 1655 parent = parent->children[0]; 1656 } 1657 } 1658 1659 return 0; 1660 } 1661 1662 /** 1663 * ice_sched_calc_vsi_support_nodes - calculate number of VSI support nodes 1664 * @pi: pointer to the port info structure 1665 * @tc_node: pointer to TC node 1666 * @num_nodes: pointer to num nodes array 1667 * 1668 * This function calculates the number of supported nodes needed to add this 1669 * VSI into Tx tree including the VSI, parent and intermediate nodes in below 1670 * layers 1671 */ 1672 static void 1673 ice_sched_calc_vsi_support_nodes(struct ice_port_info *pi, 1674 struct ice_sched_node *tc_node, u16 *num_nodes) 1675 { 1676 struct ice_sched_node *node; 1677 u8 vsil; 1678 int i; 1679 1680 vsil = ice_sched_get_vsi_layer(pi->hw); 1681 for (i = vsil; i >= pi->hw->sw_entry_point_layer; i--) 1682 /* Add intermediate nodes if TC has no children and 1683 * need at least one node for VSI 1684 */ 1685 if (!tc_node->num_children || i == vsil) { 1686 num_nodes[i]++; 1687 } else { 1688 /* If intermediate nodes are reached max children 1689 * then add a new one. 1690 */ 1691 node = ice_sched_get_first_node(pi, tc_node, (u8)i); 1692 /* scan all the siblings */ 1693 while (node) { 1694 if (node->num_children < pi->hw->max_children[i]) 1695 break; 1696 node = node->sibling; 1697 } 1698 1699 /* tree has one intermediate node to add this new VSI. 1700 * So no need to calculate supported nodes for below 1701 * layers. 1702 */ 1703 if (node) 1704 break; 1705 /* all the nodes are full, allocate a new one */ 1706 num_nodes[i]++; 1707 } 1708 } 1709 1710 /** 1711 * ice_sched_add_vsi_support_nodes - add VSI supported nodes into Tx tree 1712 * @pi: port information structure 1713 * @vsi_handle: software VSI handle 1714 * @tc_node: pointer to TC node 1715 * @num_nodes: pointer to num nodes array 1716 * 1717 * This function adds the VSI supported nodes into Tx tree including the 1718 * VSI, its parent and intermediate nodes in below layers 1719 */ 1720 static enum ice_status 1721 ice_sched_add_vsi_support_nodes(struct ice_port_info *pi, u16 vsi_handle, 1722 struct ice_sched_node *tc_node, u16 *num_nodes) 1723 { 1724 struct ice_sched_node *parent = tc_node; 1725 enum ice_status status; 1726 u32 first_node_teid; 1727 u16 num_added = 0; 1728 u8 i, vsil; 1729 1730 if (!pi) 1731 return ICE_ERR_PARAM; 1732 1733 vsil = ice_sched_get_vsi_layer(pi->hw); 1734 for (i = pi->hw->sw_entry_point_layer; i <= vsil; i++) { 1735 status = ice_sched_add_nodes_to_layer(pi, tc_node, parent, 1736 i, num_nodes[i], 1737 &first_node_teid, 1738 &num_added); 1739 if (status || num_nodes[i] != num_added) 1740 return ICE_ERR_CFG; 1741 1742 /* The newly added node can be a new parent for the next 1743 * layer nodes 1744 */ 1745 if (num_added) 1746 parent = ice_sched_find_node_by_teid(tc_node, 1747 first_node_teid); 1748 else 1749 parent = parent->children[0]; 1750 1751 if (!parent) 1752 return ICE_ERR_CFG; 1753 1754 if (i == vsil) 1755 parent->vsi_handle = vsi_handle; 1756 } 1757 1758 return 0; 1759 } 1760 1761 /** 1762 * ice_sched_add_vsi_to_topo - add a new VSI into tree 1763 * @pi: port information structure 1764 * @vsi_handle: software VSI handle 1765 * @tc: TC number 1766 * 1767 * This function adds a new VSI into scheduler tree 1768 */ 1769 static enum ice_status 1770 ice_sched_add_vsi_to_topo(struct ice_port_info *pi, u16 vsi_handle, u8 tc) 1771 { 1772 u16 num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 }; 1773 struct ice_sched_node *tc_node; 1774 1775 tc_node = ice_sched_get_tc_node(pi, tc); 1776 if (!tc_node) 1777 return ICE_ERR_PARAM; 1778 1779 /* calculate number of supported nodes needed for this VSI */ 1780 ice_sched_calc_vsi_support_nodes(pi, tc_node, num_nodes); 1781 1782 /* add VSI supported nodes to TC subtree */ 1783 return ice_sched_add_vsi_support_nodes(pi, vsi_handle, tc_node, 1784 num_nodes); 1785 } 1786 1787 /** 1788 * ice_sched_update_vsi_child_nodes - update VSI child nodes 1789 * @pi: port information structure 1790 * @vsi_handle: software VSI handle 1791 * @tc: TC number 1792 * @new_numqs: new number of max queues 1793 * @owner: owner of this subtree 1794 * 1795 * This function updates the VSI child nodes based on the number of queues 1796 */ 1797 static enum ice_status 1798 ice_sched_update_vsi_child_nodes(struct ice_port_info *pi, u16 vsi_handle, 1799 u8 tc, u16 new_numqs, u8 owner) 1800 { 1801 u16 new_num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 }; 1802 struct ice_sched_node *vsi_node; 1803 struct ice_sched_node *tc_node; 1804 struct ice_vsi_ctx *vsi_ctx; 1805 enum ice_status status = 0; 1806 struct ice_hw *hw = pi->hw; 1807 u16 prev_numqs; 1808 1809 tc_node = ice_sched_get_tc_node(pi, tc); 1810 if (!tc_node) 1811 return ICE_ERR_CFG; 1812 1813 vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle); 1814 if (!vsi_node) 1815 return ICE_ERR_CFG; 1816 1817 vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle); 1818 if (!vsi_ctx) 1819 return ICE_ERR_PARAM; 1820 1821 if (owner == ICE_SCHED_NODE_OWNER_LAN) 1822 prev_numqs = vsi_ctx->sched.max_lanq[tc]; 1823 else 1824 prev_numqs = vsi_ctx->sched.max_rdmaq[tc]; 1825 /* num queues are not changed or less than the previous number */ 1826 if (new_numqs <= prev_numqs) 1827 return status; 1828 if (owner == ICE_SCHED_NODE_OWNER_LAN) { 1829 status = ice_alloc_lan_q_ctx(hw, vsi_handle, tc, new_numqs); 1830 if (status) 1831 return status; 1832 } else { 1833 status = ice_alloc_rdma_q_ctx(hw, vsi_handle, tc, new_numqs); 1834 if (status) 1835 return status; 1836 } 1837 1838 if (new_numqs) 1839 ice_sched_calc_vsi_child_nodes(hw, new_numqs, new_num_nodes); 1840 /* Keep the max number of queue configuration all the time. Update the 1841 * tree only if number of queues > previous number of queues. This may 1842 * leave some extra nodes in the tree if number of queues < previous 1843 * number but that wouldn't harm anything. Removing those extra nodes 1844 * may complicate the code if those nodes are part of SRL or 1845 * individually rate limited. 1846 */ 1847 status = ice_sched_add_vsi_child_nodes(pi, vsi_handle, tc_node, 1848 new_num_nodes, owner); 1849 if (status) 1850 return status; 1851 if (owner == ICE_SCHED_NODE_OWNER_LAN) 1852 vsi_ctx->sched.max_lanq[tc] = new_numqs; 1853 else 1854 vsi_ctx->sched.max_rdmaq[tc] = new_numqs; 1855 1856 return 0; 1857 } 1858 1859 /** 1860 * ice_sched_cfg_vsi - configure the new/existing VSI 1861 * @pi: port information structure 1862 * @vsi_handle: software VSI handle 1863 * @tc: TC number 1864 * @maxqs: max number of queues 1865 * @owner: LAN or RDMA 1866 * @enable: TC enabled or disabled 1867 * 1868 * This function adds/updates VSI nodes based on the number of queues. If TC is 1869 * enabled and VSI is in suspended state then resume the VSI back. If TC is 1870 * disabled then suspend the VSI if it is not already. 1871 */ 1872 enum ice_status 1873 ice_sched_cfg_vsi(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u16 maxqs, 1874 u8 owner, bool enable) 1875 { 1876 struct ice_sched_node *vsi_node, *tc_node; 1877 struct ice_vsi_ctx *vsi_ctx; 1878 enum ice_status status = 0; 1879 struct ice_hw *hw = pi->hw; 1880 1881 ice_debug(pi->hw, ICE_DBG_SCHED, "add/config VSI %d\n", vsi_handle); 1882 tc_node = ice_sched_get_tc_node(pi, tc); 1883 if (!tc_node) 1884 return ICE_ERR_PARAM; 1885 vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle); 1886 if (!vsi_ctx) 1887 return ICE_ERR_PARAM; 1888 vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle); 1889 1890 /* suspend the VSI if TC is not enabled */ 1891 if (!enable) { 1892 if (vsi_node && vsi_node->in_use) { 1893 u32 teid = le32_to_cpu(vsi_node->info.node_teid); 1894 1895 status = ice_sched_suspend_resume_elems(hw, 1, &teid, 1896 true); 1897 if (!status) 1898 vsi_node->in_use = false; 1899 } 1900 return status; 1901 } 1902 1903 /* TC is enabled, if it is a new VSI then add it to the tree */ 1904 if (!vsi_node) { 1905 status = ice_sched_add_vsi_to_topo(pi, vsi_handle, tc); 1906 if (status) 1907 return status; 1908 1909 vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle); 1910 if (!vsi_node) 1911 return ICE_ERR_CFG; 1912 1913 vsi_ctx->sched.vsi_node[tc] = vsi_node; 1914 vsi_node->in_use = true; 1915 /* invalidate the max queues whenever VSI gets added first time 1916 * into the scheduler tree (boot or after reset). We need to 1917 * recreate the child nodes all the time in these cases. 1918 */ 1919 vsi_ctx->sched.max_lanq[tc] = 0; 1920 vsi_ctx->sched.max_rdmaq[tc] = 0; 1921 } 1922 1923 /* update the VSI child nodes */ 1924 status = ice_sched_update_vsi_child_nodes(pi, vsi_handle, tc, maxqs, 1925 owner); 1926 if (status) 1927 return status; 1928 1929 /* TC is enabled, resume the VSI if it is in the suspend state */ 1930 if (!vsi_node->in_use) { 1931 u32 teid = le32_to_cpu(vsi_node->info.node_teid); 1932 1933 status = ice_sched_suspend_resume_elems(hw, 1, &teid, false); 1934 if (!status) 1935 vsi_node->in_use = true; 1936 } 1937 1938 return status; 1939 } 1940 1941 /** 1942 * ice_sched_rm_agg_vsi_info - remove aggregator related VSI info entry 1943 * @pi: port information structure 1944 * @vsi_handle: software VSI handle 1945 * 1946 * This function removes single aggregator VSI info entry from 1947 * aggregator list. 1948 */ 1949 static void ice_sched_rm_agg_vsi_info(struct ice_port_info *pi, u16 vsi_handle) 1950 { 1951 struct ice_sched_agg_info *agg_info; 1952 struct ice_sched_agg_info *atmp; 1953 1954 list_for_each_entry_safe(agg_info, atmp, &pi->hw->agg_list, 1955 list_entry) { 1956 struct ice_sched_agg_vsi_info *agg_vsi_info; 1957 struct ice_sched_agg_vsi_info *vtmp; 1958 1959 list_for_each_entry_safe(agg_vsi_info, vtmp, 1960 &agg_info->agg_vsi_list, list_entry) 1961 if (agg_vsi_info->vsi_handle == vsi_handle) { 1962 list_del(&agg_vsi_info->list_entry); 1963 devm_kfree(ice_hw_to_dev(pi->hw), 1964 agg_vsi_info); 1965 return; 1966 } 1967 } 1968 } 1969 1970 /** 1971 * ice_sched_is_leaf_node_present - check for a leaf node in the sub-tree 1972 * @node: pointer to the sub-tree node 1973 * 1974 * This function checks for a leaf node presence in a given sub-tree node. 1975 */ 1976 static bool ice_sched_is_leaf_node_present(struct ice_sched_node *node) 1977 { 1978 u8 i; 1979 1980 for (i = 0; i < node->num_children; i++) 1981 if (ice_sched_is_leaf_node_present(node->children[i])) 1982 return true; 1983 /* check for a leaf node */ 1984 return (node->info.data.elem_type == ICE_AQC_ELEM_TYPE_LEAF); 1985 } 1986 1987 /** 1988 * ice_sched_rm_vsi_cfg - remove the VSI and its children nodes 1989 * @pi: port information structure 1990 * @vsi_handle: software VSI handle 1991 * @owner: LAN or RDMA 1992 * 1993 * This function removes the VSI and its LAN or RDMA children nodes from the 1994 * scheduler tree. 1995 */ 1996 static enum ice_status 1997 ice_sched_rm_vsi_cfg(struct ice_port_info *pi, u16 vsi_handle, u8 owner) 1998 { 1999 enum ice_status status = ICE_ERR_PARAM; 2000 struct ice_vsi_ctx *vsi_ctx; 2001 u8 i; 2002 2003 ice_debug(pi->hw, ICE_DBG_SCHED, "removing VSI %d\n", vsi_handle); 2004 if (!ice_is_vsi_valid(pi->hw, vsi_handle)) 2005 return status; 2006 mutex_lock(&pi->sched_lock); 2007 vsi_ctx = ice_get_vsi_ctx(pi->hw, vsi_handle); 2008 if (!vsi_ctx) 2009 goto exit_sched_rm_vsi_cfg; 2010 2011 ice_for_each_traffic_class(i) { 2012 struct ice_sched_node *vsi_node, *tc_node; 2013 u8 j = 0; 2014 2015 tc_node = ice_sched_get_tc_node(pi, i); 2016 if (!tc_node) 2017 continue; 2018 2019 vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle); 2020 if (!vsi_node) 2021 continue; 2022 2023 if (ice_sched_is_leaf_node_present(vsi_node)) { 2024 ice_debug(pi->hw, ICE_DBG_SCHED, "VSI has leaf nodes in TC %d\n", i); 2025 status = ICE_ERR_IN_USE; 2026 goto exit_sched_rm_vsi_cfg; 2027 } 2028 while (j < vsi_node->num_children) { 2029 if (vsi_node->children[j]->owner == owner) { 2030 ice_free_sched_node(pi, vsi_node->children[j]); 2031 2032 /* reset the counter again since the num 2033 * children will be updated after node removal 2034 */ 2035 j = 0; 2036 } else { 2037 j++; 2038 } 2039 } 2040 /* remove the VSI if it has no children */ 2041 if (!vsi_node->num_children) { 2042 ice_free_sched_node(pi, vsi_node); 2043 vsi_ctx->sched.vsi_node[i] = NULL; 2044 2045 /* clean up aggregator related VSI info if any */ 2046 ice_sched_rm_agg_vsi_info(pi, vsi_handle); 2047 } 2048 if (owner == ICE_SCHED_NODE_OWNER_LAN) 2049 vsi_ctx->sched.max_lanq[i] = 0; 2050 else 2051 vsi_ctx->sched.max_rdmaq[i] = 0; 2052 } 2053 status = 0; 2054 2055 exit_sched_rm_vsi_cfg: 2056 mutex_unlock(&pi->sched_lock); 2057 return status; 2058 } 2059 2060 /** 2061 * ice_rm_vsi_lan_cfg - remove VSI and its LAN children nodes 2062 * @pi: port information structure 2063 * @vsi_handle: software VSI handle 2064 * 2065 * This function clears the VSI and its LAN children nodes from scheduler tree 2066 * for all TCs. 2067 */ 2068 enum ice_status ice_rm_vsi_lan_cfg(struct ice_port_info *pi, u16 vsi_handle) 2069 { 2070 return ice_sched_rm_vsi_cfg(pi, vsi_handle, ICE_SCHED_NODE_OWNER_LAN); 2071 } 2072 2073 /** 2074 * ice_get_agg_info - get the aggregator ID 2075 * @hw: pointer to the hardware structure 2076 * @agg_id: aggregator ID 2077 * 2078 * This function validates aggregator ID. The function returns info if 2079 * aggregator ID is present in list otherwise it returns null. 2080 */ 2081 static struct ice_sched_agg_info * 2082 ice_get_agg_info(struct ice_hw *hw, u32 agg_id) 2083 { 2084 struct ice_sched_agg_info *agg_info; 2085 2086 list_for_each_entry(agg_info, &hw->agg_list, list_entry) 2087 if (agg_info->agg_id == agg_id) 2088 return agg_info; 2089 2090 return NULL; 2091 } 2092 2093 /** 2094 * ice_sched_get_free_vsi_parent - Find a free parent node in aggregator subtree 2095 * @hw: pointer to the HW struct 2096 * @node: pointer to a child node 2097 * @num_nodes: num nodes count array 2098 * 2099 * This function walks through the aggregator subtree to find a free parent 2100 * node 2101 */ 2102 static struct ice_sched_node * 2103 ice_sched_get_free_vsi_parent(struct ice_hw *hw, struct ice_sched_node *node, 2104 u16 *num_nodes) 2105 { 2106 u8 l = node->tx_sched_layer; 2107 u8 vsil, i; 2108 2109 vsil = ice_sched_get_vsi_layer(hw); 2110 2111 /* Is it VSI parent layer ? */ 2112 if (l == vsil - 1) 2113 return (node->num_children < hw->max_children[l]) ? node : NULL; 2114 2115 /* We have intermediate nodes. Let's walk through the subtree. If the 2116 * intermediate node has space to add a new node then clear the count 2117 */ 2118 if (node->num_children < hw->max_children[l]) 2119 num_nodes[l] = 0; 2120 /* The below recursive call is intentional and wouldn't go more than 2121 * 2 or 3 iterations. 2122 */ 2123 2124 for (i = 0; i < node->num_children; i++) { 2125 struct ice_sched_node *parent; 2126 2127 parent = ice_sched_get_free_vsi_parent(hw, node->children[i], 2128 num_nodes); 2129 if (parent) 2130 return parent; 2131 } 2132 2133 return NULL; 2134 } 2135 2136 /** 2137 * ice_sched_update_parent - update the new parent in SW DB 2138 * @new_parent: pointer to a new parent node 2139 * @node: pointer to a child node 2140 * 2141 * This function removes the child from the old parent and adds it to a new 2142 * parent 2143 */ 2144 static void 2145 ice_sched_update_parent(struct ice_sched_node *new_parent, 2146 struct ice_sched_node *node) 2147 { 2148 struct ice_sched_node *old_parent; 2149 u8 i, j; 2150 2151 old_parent = node->parent; 2152 2153 /* update the old parent children */ 2154 for (i = 0; i < old_parent->num_children; i++) 2155 if (old_parent->children[i] == node) { 2156 for (j = i + 1; j < old_parent->num_children; j++) 2157 old_parent->children[j - 1] = 2158 old_parent->children[j]; 2159 old_parent->num_children--; 2160 break; 2161 } 2162 2163 /* now move the node to a new parent */ 2164 new_parent->children[new_parent->num_children++] = node; 2165 node->parent = new_parent; 2166 node->info.parent_teid = new_parent->info.node_teid; 2167 } 2168 2169 /** 2170 * ice_sched_move_nodes - move child nodes to a given parent 2171 * @pi: port information structure 2172 * @parent: pointer to parent node 2173 * @num_items: number of child nodes to be moved 2174 * @list: pointer to child node teids 2175 * 2176 * This function move the child nodes to a given parent. 2177 */ 2178 static enum ice_status 2179 ice_sched_move_nodes(struct ice_port_info *pi, struct ice_sched_node *parent, 2180 u16 num_items, u32 *list) 2181 { 2182 struct ice_aqc_move_elem *buf; 2183 struct ice_sched_node *node; 2184 enum ice_status status = 0; 2185 u16 i, grps_movd = 0; 2186 struct ice_hw *hw; 2187 u16 buf_len; 2188 2189 hw = pi->hw; 2190 2191 if (!parent || !num_items) 2192 return ICE_ERR_PARAM; 2193 2194 /* Does parent have enough space */ 2195 if (parent->num_children + num_items > 2196 hw->max_children[parent->tx_sched_layer]) 2197 return ICE_ERR_AQ_FULL; 2198 2199 buf_len = struct_size(buf, teid, 1); 2200 buf = kzalloc(buf_len, GFP_KERNEL); 2201 if (!buf) 2202 return ICE_ERR_NO_MEMORY; 2203 2204 for (i = 0; i < num_items; i++) { 2205 node = ice_sched_find_node_by_teid(pi->root, list[i]); 2206 if (!node) { 2207 status = ICE_ERR_PARAM; 2208 goto move_err_exit; 2209 } 2210 2211 buf->hdr.src_parent_teid = node->info.parent_teid; 2212 buf->hdr.dest_parent_teid = parent->info.node_teid; 2213 buf->teid[0] = node->info.node_teid; 2214 buf->hdr.num_elems = cpu_to_le16(1); 2215 status = ice_aq_move_sched_elems(hw, 1, buf, buf_len, 2216 &grps_movd, NULL); 2217 if (status && grps_movd != 1) { 2218 status = ICE_ERR_CFG; 2219 goto move_err_exit; 2220 } 2221 2222 /* update the SW DB */ 2223 ice_sched_update_parent(parent, node); 2224 } 2225 2226 move_err_exit: 2227 kfree(buf); 2228 return status; 2229 } 2230 2231 /** 2232 * ice_sched_move_vsi_to_agg - move VSI to aggregator node 2233 * @pi: port information structure 2234 * @vsi_handle: software VSI handle 2235 * @agg_id: aggregator ID 2236 * @tc: TC number 2237 * 2238 * This function moves a VSI to an aggregator node or its subtree. 2239 * Intermediate nodes may be created if required. 2240 */ 2241 static enum ice_status 2242 ice_sched_move_vsi_to_agg(struct ice_port_info *pi, u16 vsi_handle, u32 agg_id, 2243 u8 tc) 2244 { 2245 struct ice_sched_node *vsi_node, *agg_node, *tc_node, *parent; 2246 u16 num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 }; 2247 u32 first_node_teid, vsi_teid; 2248 enum ice_status status; 2249 u16 num_nodes_added; 2250 u8 aggl, vsil, i; 2251 2252 tc_node = ice_sched_get_tc_node(pi, tc); 2253 if (!tc_node) 2254 return ICE_ERR_CFG; 2255 2256 agg_node = ice_sched_get_agg_node(pi, tc_node, agg_id); 2257 if (!agg_node) 2258 return ICE_ERR_DOES_NOT_EXIST; 2259 2260 vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle); 2261 if (!vsi_node) 2262 return ICE_ERR_DOES_NOT_EXIST; 2263 2264 /* Is this VSI already part of given aggregator? */ 2265 if (ice_sched_find_node_in_subtree(pi->hw, agg_node, vsi_node)) 2266 return 0; 2267 2268 aggl = ice_sched_get_agg_layer(pi->hw); 2269 vsil = ice_sched_get_vsi_layer(pi->hw); 2270 2271 /* set intermediate node count to 1 between aggregator and VSI layers */ 2272 for (i = aggl + 1; i < vsil; i++) 2273 num_nodes[i] = 1; 2274 2275 /* Check if the aggregator subtree has any free node to add the VSI */ 2276 for (i = 0; i < agg_node->num_children; i++) { 2277 parent = ice_sched_get_free_vsi_parent(pi->hw, 2278 agg_node->children[i], 2279 num_nodes); 2280 if (parent) 2281 goto move_nodes; 2282 } 2283 2284 /* add new nodes */ 2285 parent = agg_node; 2286 for (i = aggl + 1; i < vsil; i++) { 2287 status = ice_sched_add_nodes_to_layer(pi, tc_node, parent, i, 2288 num_nodes[i], 2289 &first_node_teid, 2290 &num_nodes_added); 2291 if (status || num_nodes[i] != num_nodes_added) 2292 return ICE_ERR_CFG; 2293 2294 /* The newly added node can be a new parent for the next 2295 * layer nodes 2296 */ 2297 if (num_nodes_added) 2298 parent = ice_sched_find_node_by_teid(tc_node, 2299 first_node_teid); 2300 else 2301 parent = parent->children[0]; 2302 2303 if (!parent) 2304 return ICE_ERR_CFG; 2305 } 2306 2307 move_nodes: 2308 vsi_teid = le32_to_cpu(vsi_node->info.node_teid); 2309 return ice_sched_move_nodes(pi, parent, 1, &vsi_teid); 2310 } 2311 2312 /** 2313 * ice_move_all_vsi_to_dflt_agg - move all VSI(s) to default aggregator 2314 * @pi: port information structure 2315 * @agg_info: aggregator info 2316 * @tc: traffic class number 2317 * @rm_vsi_info: true or false 2318 * 2319 * This function move all the VSI(s) to the default aggregator and delete 2320 * aggregator VSI info based on passed in boolean parameter rm_vsi_info. The 2321 * caller holds the scheduler lock. 2322 */ 2323 static enum ice_status 2324 ice_move_all_vsi_to_dflt_agg(struct ice_port_info *pi, 2325 struct ice_sched_agg_info *agg_info, u8 tc, 2326 bool rm_vsi_info) 2327 { 2328 struct ice_sched_agg_vsi_info *agg_vsi_info; 2329 struct ice_sched_agg_vsi_info *tmp; 2330 enum ice_status status = 0; 2331 2332 list_for_each_entry_safe(agg_vsi_info, tmp, &agg_info->agg_vsi_list, 2333 list_entry) { 2334 u16 vsi_handle = agg_vsi_info->vsi_handle; 2335 2336 /* Move VSI to default aggregator */ 2337 if (!ice_is_tc_ena(agg_vsi_info->tc_bitmap[0], tc)) 2338 continue; 2339 2340 status = ice_sched_move_vsi_to_agg(pi, vsi_handle, 2341 ICE_DFLT_AGG_ID, tc); 2342 if (status) 2343 break; 2344 2345 clear_bit(tc, agg_vsi_info->tc_bitmap); 2346 if (rm_vsi_info && !agg_vsi_info->tc_bitmap[0]) { 2347 list_del(&agg_vsi_info->list_entry); 2348 devm_kfree(ice_hw_to_dev(pi->hw), agg_vsi_info); 2349 } 2350 } 2351 2352 return status; 2353 } 2354 2355 /** 2356 * ice_sched_is_agg_inuse - check whether the aggregator is in use or not 2357 * @pi: port information structure 2358 * @node: node pointer 2359 * 2360 * This function checks whether the aggregator is attached with any VSI or not. 2361 */ 2362 static bool 2363 ice_sched_is_agg_inuse(struct ice_port_info *pi, struct ice_sched_node *node) 2364 { 2365 u8 vsil, i; 2366 2367 vsil = ice_sched_get_vsi_layer(pi->hw); 2368 if (node->tx_sched_layer < vsil - 1) { 2369 for (i = 0; i < node->num_children; i++) 2370 if (ice_sched_is_agg_inuse(pi, node->children[i])) 2371 return true; 2372 return false; 2373 } else { 2374 return node->num_children ? true : false; 2375 } 2376 } 2377 2378 /** 2379 * ice_sched_rm_agg_cfg - remove the aggregator node 2380 * @pi: port information structure 2381 * @agg_id: aggregator ID 2382 * @tc: TC number 2383 * 2384 * This function removes the aggregator node and intermediate nodes if any 2385 * from the given TC 2386 */ 2387 static enum ice_status 2388 ice_sched_rm_agg_cfg(struct ice_port_info *pi, u32 agg_id, u8 tc) 2389 { 2390 struct ice_sched_node *tc_node, *agg_node; 2391 struct ice_hw *hw = pi->hw; 2392 2393 tc_node = ice_sched_get_tc_node(pi, tc); 2394 if (!tc_node) 2395 return ICE_ERR_CFG; 2396 2397 agg_node = ice_sched_get_agg_node(pi, tc_node, agg_id); 2398 if (!agg_node) 2399 return ICE_ERR_DOES_NOT_EXIST; 2400 2401 /* Can't remove the aggregator node if it has children */ 2402 if (ice_sched_is_agg_inuse(pi, agg_node)) 2403 return ICE_ERR_IN_USE; 2404 2405 /* need to remove the whole subtree if aggregator node is the 2406 * only child. 2407 */ 2408 while (agg_node->tx_sched_layer > hw->sw_entry_point_layer) { 2409 struct ice_sched_node *parent = agg_node->parent; 2410 2411 if (!parent) 2412 return ICE_ERR_CFG; 2413 2414 if (parent->num_children > 1) 2415 break; 2416 2417 agg_node = parent; 2418 } 2419 2420 ice_free_sched_node(pi, agg_node); 2421 return 0; 2422 } 2423 2424 /** 2425 * ice_rm_agg_cfg_tc - remove aggregator configuration for TC 2426 * @pi: port information structure 2427 * @agg_info: aggregator ID 2428 * @tc: TC number 2429 * @rm_vsi_info: bool value true or false 2430 * 2431 * This function removes aggregator reference to VSI of given TC. It removes 2432 * the aggregator configuration completely for requested TC. The caller needs 2433 * to hold the scheduler lock. 2434 */ 2435 static enum ice_status 2436 ice_rm_agg_cfg_tc(struct ice_port_info *pi, struct ice_sched_agg_info *agg_info, 2437 u8 tc, bool rm_vsi_info) 2438 { 2439 enum ice_status status = 0; 2440 2441 /* If nothing to remove - return success */ 2442 if (!ice_is_tc_ena(agg_info->tc_bitmap[0], tc)) 2443 goto exit_rm_agg_cfg_tc; 2444 2445 status = ice_move_all_vsi_to_dflt_agg(pi, agg_info, tc, rm_vsi_info); 2446 if (status) 2447 goto exit_rm_agg_cfg_tc; 2448 2449 /* Delete aggregator node(s) */ 2450 status = ice_sched_rm_agg_cfg(pi, agg_info->agg_id, tc); 2451 if (status) 2452 goto exit_rm_agg_cfg_tc; 2453 2454 clear_bit(tc, agg_info->tc_bitmap); 2455 exit_rm_agg_cfg_tc: 2456 return status; 2457 } 2458 2459 /** 2460 * ice_save_agg_tc_bitmap - save aggregator TC bitmap 2461 * @pi: port information structure 2462 * @agg_id: aggregator ID 2463 * @tc_bitmap: 8 bits TC bitmap 2464 * 2465 * Save aggregator TC bitmap. This function needs to be called with scheduler 2466 * lock held. 2467 */ 2468 static enum ice_status 2469 ice_save_agg_tc_bitmap(struct ice_port_info *pi, u32 agg_id, 2470 unsigned long *tc_bitmap) 2471 { 2472 struct ice_sched_agg_info *agg_info; 2473 2474 agg_info = ice_get_agg_info(pi->hw, agg_id); 2475 if (!agg_info) 2476 return ICE_ERR_PARAM; 2477 bitmap_copy(agg_info->replay_tc_bitmap, tc_bitmap, 2478 ICE_MAX_TRAFFIC_CLASS); 2479 return 0; 2480 } 2481 2482 /** 2483 * ice_sched_add_agg_cfg - create an aggregator node 2484 * @pi: port information structure 2485 * @agg_id: aggregator ID 2486 * @tc: TC number 2487 * 2488 * This function creates an aggregator node and intermediate nodes if required 2489 * for the given TC 2490 */ 2491 static enum ice_status 2492 ice_sched_add_agg_cfg(struct ice_port_info *pi, u32 agg_id, u8 tc) 2493 { 2494 struct ice_sched_node *parent, *agg_node, *tc_node; 2495 u16 num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 }; 2496 enum ice_status status = 0; 2497 struct ice_hw *hw = pi->hw; 2498 u32 first_node_teid; 2499 u16 num_nodes_added; 2500 u8 i, aggl; 2501 2502 tc_node = ice_sched_get_tc_node(pi, tc); 2503 if (!tc_node) 2504 return ICE_ERR_CFG; 2505 2506 agg_node = ice_sched_get_agg_node(pi, tc_node, agg_id); 2507 /* Does Agg node already exist ? */ 2508 if (agg_node) 2509 return status; 2510 2511 aggl = ice_sched_get_agg_layer(hw); 2512 2513 /* need one node in Agg layer */ 2514 num_nodes[aggl] = 1; 2515 2516 /* Check whether the intermediate nodes have space to add the 2517 * new aggregator. If they are full, then SW needs to allocate a new 2518 * intermediate node on those layers 2519 */ 2520 for (i = hw->sw_entry_point_layer; i < aggl; i++) { 2521 parent = ice_sched_get_first_node(pi, tc_node, i); 2522 2523 /* scan all the siblings */ 2524 while (parent) { 2525 if (parent->num_children < hw->max_children[i]) 2526 break; 2527 parent = parent->sibling; 2528 } 2529 2530 /* all the nodes are full, reserve one for this layer */ 2531 if (!parent) 2532 num_nodes[i]++; 2533 } 2534 2535 /* add the aggregator node */ 2536 parent = tc_node; 2537 for (i = hw->sw_entry_point_layer; i <= aggl; i++) { 2538 if (!parent) 2539 return ICE_ERR_CFG; 2540 2541 status = ice_sched_add_nodes_to_layer(pi, tc_node, parent, i, 2542 num_nodes[i], 2543 &first_node_teid, 2544 &num_nodes_added); 2545 if (status || num_nodes[i] != num_nodes_added) 2546 return ICE_ERR_CFG; 2547 2548 /* The newly added node can be a new parent for the next 2549 * layer nodes 2550 */ 2551 if (num_nodes_added) { 2552 parent = ice_sched_find_node_by_teid(tc_node, 2553 first_node_teid); 2554 /* register aggregator ID with the aggregator node */ 2555 if (parent && i == aggl) 2556 parent->agg_id = agg_id; 2557 } else { 2558 parent = parent->children[0]; 2559 } 2560 } 2561 2562 return 0; 2563 } 2564 2565 /** 2566 * ice_sched_cfg_agg - configure aggregator node 2567 * @pi: port information structure 2568 * @agg_id: aggregator ID 2569 * @agg_type: aggregator type queue, VSI, or aggregator group 2570 * @tc_bitmap: bits TC bitmap 2571 * 2572 * It registers a unique aggregator node into scheduler services. It 2573 * allows a user to register with a unique ID to track it's resources. 2574 * The aggregator type determines if this is a queue group, VSI group 2575 * or aggregator group. It then creates the aggregator node(s) for requested 2576 * TC(s) or removes an existing aggregator node including its configuration 2577 * if indicated via tc_bitmap. Call ice_rm_agg_cfg to release aggregator 2578 * resources and remove aggregator ID. 2579 * This function needs to be called with scheduler lock held. 2580 */ 2581 static enum ice_status 2582 ice_sched_cfg_agg(struct ice_port_info *pi, u32 agg_id, 2583 enum ice_agg_type agg_type, unsigned long *tc_bitmap) 2584 { 2585 struct ice_sched_agg_info *agg_info; 2586 enum ice_status status = 0; 2587 struct ice_hw *hw = pi->hw; 2588 u8 tc; 2589 2590 agg_info = ice_get_agg_info(hw, agg_id); 2591 if (!agg_info) { 2592 /* Create new entry for new aggregator ID */ 2593 agg_info = devm_kzalloc(ice_hw_to_dev(hw), sizeof(*agg_info), 2594 GFP_KERNEL); 2595 if (!agg_info) 2596 return ICE_ERR_NO_MEMORY; 2597 2598 agg_info->agg_id = agg_id; 2599 agg_info->agg_type = agg_type; 2600 agg_info->tc_bitmap[0] = 0; 2601 2602 /* Initialize the aggregator VSI list head */ 2603 INIT_LIST_HEAD(&agg_info->agg_vsi_list); 2604 2605 /* Add new entry in aggregator list */ 2606 list_add(&agg_info->list_entry, &hw->agg_list); 2607 } 2608 /* Create aggregator node(s) for requested TC(s) */ 2609 ice_for_each_traffic_class(tc) { 2610 if (!ice_is_tc_ena(*tc_bitmap, tc)) { 2611 /* Delete aggregator cfg TC if it exists previously */ 2612 status = ice_rm_agg_cfg_tc(pi, agg_info, tc, false); 2613 if (status) 2614 break; 2615 continue; 2616 } 2617 2618 /* Check if aggregator node for TC already exists */ 2619 if (ice_is_tc_ena(agg_info->tc_bitmap[0], tc)) 2620 continue; 2621 2622 /* Create new aggregator node for TC */ 2623 status = ice_sched_add_agg_cfg(pi, agg_id, tc); 2624 if (status) 2625 break; 2626 2627 /* Save aggregator node's TC information */ 2628 set_bit(tc, agg_info->tc_bitmap); 2629 } 2630 2631 return status; 2632 } 2633 2634 /** 2635 * ice_cfg_agg - config aggregator node 2636 * @pi: port information structure 2637 * @agg_id: aggregator ID 2638 * @agg_type: aggregator type queue, VSI, or aggregator group 2639 * @tc_bitmap: bits TC bitmap 2640 * 2641 * This function configures aggregator node(s). 2642 */ 2643 enum ice_status 2644 ice_cfg_agg(struct ice_port_info *pi, u32 agg_id, enum ice_agg_type agg_type, 2645 u8 tc_bitmap) 2646 { 2647 unsigned long bitmap = tc_bitmap; 2648 enum ice_status status; 2649 2650 mutex_lock(&pi->sched_lock); 2651 status = ice_sched_cfg_agg(pi, agg_id, agg_type, 2652 (unsigned long *)&bitmap); 2653 if (!status) 2654 status = ice_save_agg_tc_bitmap(pi, agg_id, 2655 (unsigned long *)&bitmap); 2656 mutex_unlock(&pi->sched_lock); 2657 return status; 2658 } 2659 2660 /** 2661 * ice_get_agg_vsi_info - get the aggregator ID 2662 * @agg_info: aggregator info 2663 * @vsi_handle: software VSI handle 2664 * 2665 * The function returns aggregator VSI info based on VSI handle. This function 2666 * needs to be called with scheduler lock held. 2667 */ 2668 static struct ice_sched_agg_vsi_info * 2669 ice_get_agg_vsi_info(struct ice_sched_agg_info *agg_info, u16 vsi_handle) 2670 { 2671 struct ice_sched_agg_vsi_info *agg_vsi_info; 2672 2673 list_for_each_entry(agg_vsi_info, &agg_info->agg_vsi_list, list_entry) 2674 if (agg_vsi_info->vsi_handle == vsi_handle) 2675 return agg_vsi_info; 2676 2677 return NULL; 2678 } 2679 2680 /** 2681 * ice_get_vsi_agg_info - get the aggregator info of VSI 2682 * @hw: pointer to the hardware structure 2683 * @vsi_handle: Sw VSI handle 2684 * 2685 * The function returns aggregator info of VSI represented via vsi_handle. The 2686 * VSI has in this case a different aggregator than the default one. This 2687 * function needs to be called with scheduler lock held. 2688 */ 2689 static struct ice_sched_agg_info * 2690 ice_get_vsi_agg_info(struct ice_hw *hw, u16 vsi_handle) 2691 { 2692 struct ice_sched_agg_info *agg_info; 2693 2694 list_for_each_entry(agg_info, &hw->agg_list, list_entry) { 2695 struct ice_sched_agg_vsi_info *agg_vsi_info; 2696 2697 agg_vsi_info = ice_get_agg_vsi_info(agg_info, vsi_handle); 2698 if (agg_vsi_info) 2699 return agg_info; 2700 } 2701 return NULL; 2702 } 2703 2704 /** 2705 * ice_save_agg_vsi_tc_bitmap - save aggregator VSI TC bitmap 2706 * @pi: port information structure 2707 * @agg_id: aggregator ID 2708 * @vsi_handle: software VSI handle 2709 * @tc_bitmap: TC bitmap of enabled TC(s) 2710 * 2711 * Save VSI to aggregator TC bitmap. This function needs to call with scheduler 2712 * lock held. 2713 */ 2714 static enum ice_status 2715 ice_save_agg_vsi_tc_bitmap(struct ice_port_info *pi, u32 agg_id, u16 vsi_handle, 2716 unsigned long *tc_bitmap) 2717 { 2718 struct ice_sched_agg_vsi_info *agg_vsi_info; 2719 struct ice_sched_agg_info *agg_info; 2720 2721 agg_info = ice_get_agg_info(pi->hw, agg_id); 2722 if (!agg_info) 2723 return ICE_ERR_PARAM; 2724 /* check if entry already exist */ 2725 agg_vsi_info = ice_get_agg_vsi_info(agg_info, vsi_handle); 2726 if (!agg_vsi_info) 2727 return ICE_ERR_PARAM; 2728 bitmap_copy(agg_vsi_info->replay_tc_bitmap, tc_bitmap, 2729 ICE_MAX_TRAFFIC_CLASS); 2730 return 0; 2731 } 2732 2733 /** 2734 * ice_sched_assoc_vsi_to_agg - associate/move VSI to new/default aggregator 2735 * @pi: port information structure 2736 * @agg_id: aggregator ID 2737 * @vsi_handle: software VSI handle 2738 * @tc_bitmap: TC bitmap of enabled TC(s) 2739 * 2740 * This function moves VSI to a new or default aggregator node. If VSI is 2741 * already associated to the aggregator node then no operation is performed on 2742 * the tree. This function needs to be called with scheduler lock held. 2743 */ 2744 static enum ice_status 2745 ice_sched_assoc_vsi_to_agg(struct ice_port_info *pi, u32 agg_id, 2746 u16 vsi_handle, unsigned long *tc_bitmap) 2747 { 2748 struct ice_sched_agg_vsi_info *agg_vsi_info; 2749 struct ice_sched_agg_info *agg_info; 2750 enum ice_status status = 0; 2751 struct ice_hw *hw = pi->hw; 2752 u8 tc; 2753 2754 if (!ice_is_vsi_valid(pi->hw, vsi_handle)) 2755 return ICE_ERR_PARAM; 2756 agg_info = ice_get_agg_info(hw, agg_id); 2757 if (!agg_info) 2758 return ICE_ERR_PARAM; 2759 /* check if entry already exist */ 2760 agg_vsi_info = ice_get_agg_vsi_info(agg_info, vsi_handle); 2761 if (!agg_vsi_info) { 2762 /* Create new entry for VSI under aggregator list */ 2763 agg_vsi_info = devm_kzalloc(ice_hw_to_dev(hw), 2764 sizeof(*agg_vsi_info), GFP_KERNEL); 2765 if (!agg_vsi_info) 2766 return ICE_ERR_PARAM; 2767 2768 /* add VSI ID into the aggregator list */ 2769 agg_vsi_info->vsi_handle = vsi_handle; 2770 list_add(&agg_vsi_info->list_entry, &agg_info->agg_vsi_list); 2771 } 2772 /* Move VSI node to new aggregator node for requested TC(s) */ 2773 ice_for_each_traffic_class(tc) { 2774 if (!ice_is_tc_ena(*tc_bitmap, tc)) 2775 continue; 2776 2777 /* Move VSI to new aggregator */ 2778 status = ice_sched_move_vsi_to_agg(pi, vsi_handle, agg_id, tc); 2779 if (status) 2780 break; 2781 2782 set_bit(tc, agg_vsi_info->tc_bitmap); 2783 } 2784 return status; 2785 } 2786 2787 /** 2788 * ice_sched_rm_unused_rl_prof - remove unused RL profile 2789 * @pi: port information structure 2790 * 2791 * This function removes unused rate limit profiles from the HW and 2792 * SW DB. The caller needs to hold scheduler lock. 2793 */ 2794 static void ice_sched_rm_unused_rl_prof(struct ice_port_info *pi) 2795 { 2796 u16 ln; 2797 2798 for (ln = 0; ln < pi->hw->num_tx_sched_layers; ln++) { 2799 struct ice_aqc_rl_profile_info *rl_prof_elem; 2800 struct ice_aqc_rl_profile_info *rl_prof_tmp; 2801 2802 list_for_each_entry_safe(rl_prof_elem, rl_prof_tmp, 2803 &pi->rl_prof_list[ln], list_entry) { 2804 if (!ice_sched_del_rl_profile(pi->hw, rl_prof_elem)) 2805 ice_debug(pi->hw, ICE_DBG_SCHED, "Removed rl profile\n"); 2806 } 2807 } 2808 } 2809 2810 /** 2811 * ice_sched_update_elem - update element 2812 * @hw: pointer to the HW struct 2813 * @node: pointer to node 2814 * @info: node info to update 2815 * 2816 * Update the HW DB, and local SW DB of node. Update the scheduling 2817 * parameters of node from argument info data buffer (Info->data buf) and 2818 * returns success or error on config sched element failure. The caller 2819 * needs to hold scheduler lock. 2820 */ 2821 static enum ice_status 2822 ice_sched_update_elem(struct ice_hw *hw, struct ice_sched_node *node, 2823 struct ice_aqc_txsched_elem_data *info) 2824 { 2825 struct ice_aqc_txsched_elem_data buf; 2826 enum ice_status status; 2827 u16 elem_cfgd = 0; 2828 u16 num_elems = 1; 2829 2830 buf = *info; 2831 /* Parent TEID is reserved field in this aq call */ 2832 buf.parent_teid = 0; 2833 /* Element type is reserved field in this aq call */ 2834 buf.data.elem_type = 0; 2835 /* Flags is reserved field in this aq call */ 2836 buf.data.flags = 0; 2837 2838 /* Update HW DB */ 2839 /* Configure element node */ 2840 status = ice_aq_cfg_sched_elems(hw, num_elems, &buf, sizeof(buf), 2841 &elem_cfgd, NULL); 2842 if (status || elem_cfgd != num_elems) { 2843 ice_debug(hw, ICE_DBG_SCHED, "Config sched elem error\n"); 2844 return ICE_ERR_CFG; 2845 } 2846 2847 /* Config success case */ 2848 /* Now update local SW DB */ 2849 /* Only copy the data portion of info buffer */ 2850 node->info.data = info->data; 2851 return status; 2852 } 2853 2854 /** 2855 * ice_sched_cfg_node_bw_alloc - configure node BW weight/alloc params 2856 * @hw: pointer to the HW struct 2857 * @node: sched node to configure 2858 * @rl_type: rate limit type CIR, EIR, or shared 2859 * @bw_alloc: BW weight/allocation 2860 * 2861 * This function configures node element's BW allocation. 2862 */ 2863 static enum ice_status 2864 ice_sched_cfg_node_bw_alloc(struct ice_hw *hw, struct ice_sched_node *node, 2865 enum ice_rl_type rl_type, u16 bw_alloc) 2866 { 2867 struct ice_aqc_txsched_elem_data buf; 2868 struct ice_aqc_txsched_elem *data; 2869 2870 buf = node->info; 2871 data = &buf.data; 2872 if (rl_type == ICE_MIN_BW) { 2873 data->valid_sections |= ICE_AQC_ELEM_VALID_CIR; 2874 data->cir_bw.bw_alloc = cpu_to_le16(bw_alloc); 2875 } else if (rl_type == ICE_MAX_BW) { 2876 data->valid_sections |= ICE_AQC_ELEM_VALID_EIR; 2877 data->eir_bw.bw_alloc = cpu_to_le16(bw_alloc); 2878 } else { 2879 return ICE_ERR_PARAM; 2880 } 2881 2882 /* Configure element */ 2883 return ice_sched_update_elem(hw, node, &buf); 2884 } 2885 2886 /** 2887 * ice_move_vsi_to_agg - moves VSI to new or default aggregator 2888 * @pi: port information structure 2889 * @agg_id: aggregator ID 2890 * @vsi_handle: software VSI handle 2891 * @tc_bitmap: TC bitmap of enabled TC(s) 2892 * 2893 * Move or associate VSI to a new or default aggregator node. 2894 */ 2895 enum ice_status 2896 ice_move_vsi_to_agg(struct ice_port_info *pi, u32 agg_id, u16 vsi_handle, 2897 u8 tc_bitmap) 2898 { 2899 unsigned long bitmap = tc_bitmap; 2900 enum ice_status status; 2901 2902 mutex_lock(&pi->sched_lock); 2903 status = ice_sched_assoc_vsi_to_agg(pi, agg_id, vsi_handle, 2904 (unsigned long *)&bitmap); 2905 if (!status) 2906 status = ice_save_agg_vsi_tc_bitmap(pi, agg_id, vsi_handle, 2907 (unsigned long *)&bitmap); 2908 mutex_unlock(&pi->sched_lock); 2909 return status; 2910 } 2911 2912 /** 2913 * ice_set_clear_cir_bw - set or clear CIR BW 2914 * @bw_t_info: bandwidth type information structure 2915 * @bw: bandwidth in Kbps - Kilo bits per sec 2916 * 2917 * Save or clear CIR bandwidth (BW) in the passed param bw_t_info. 2918 */ 2919 static void ice_set_clear_cir_bw(struct ice_bw_type_info *bw_t_info, u32 bw) 2920 { 2921 if (bw == ICE_SCHED_DFLT_BW) { 2922 clear_bit(ICE_BW_TYPE_CIR, bw_t_info->bw_t_bitmap); 2923 bw_t_info->cir_bw.bw = 0; 2924 } else { 2925 /* Save type of BW information */ 2926 set_bit(ICE_BW_TYPE_CIR, bw_t_info->bw_t_bitmap); 2927 bw_t_info->cir_bw.bw = bw; 2928 } 2929 } 2930 2931 /** 2932 * ice_set_clear_eir_bw - set or clear EIR BW 2933 * @bw_t_info: bandwidth type information structure 2934 * @bw: bandwidth in Kbps - Kilo bits per sec 2935 * 2936 * Save or clear EIR bandwidth (BW) in the passed param bw_t_info. 2937 */ 2938 static void ice_set_clear_eir_bw(struct ice_bw_type_info *bw_t_info, u32 bw) 2939 { 2940 if (bw == ICE_SCHED_DFLT_BW) { 2941 clear_bit(ICE_BW_TYPE_EIR, bw_t_info->bw_t_bitmap); 2942 bw_t_info->eir_bw.bw = 0; 2943 } else { 2944 /* EIR BW and Shared BW profiles are mutually exclusive and 2945 * hence only one of them may be set for any given element. 2946 * First clear earlier saved shared BW information. 2947 */ 2948 clear_bit(ICE_BW_TYPE_SHARED, bw_t_info->bw_t_bitmap); 2949 bw_t_info->shared_bw = 0; 2950 /* save EIR BW information */ 2951 set_bit(ICE_BW_TYPE_EIR, bw_t_info->bw_t_bitmap); 2952 bw_t_info->eir_bw.bw = bw; 2953 } 2954 } 2955 2956 /** 2957 * ice_set_clear_shared_bw - set or clear shared BW 2958 * @bw_t_info: bandwidth type information structure 2959 * @bw: bandwidth in Kbps - Kilo bits per sec 2960 * 2961 * Save or clear shared bandwidth (BW) in the passed param bw_t_info. 2962 */ 2963 static void ice_set_clear_shared_bw(struct ice_bw_type_info *bw_t_info, u32 bw) 2964 { 2965 if (bw == ICE_SCHED_DFLT_BW) { 2966 clear_bit(ICE_BW_TYPE_SHARED, bw_t_info->bw_t_bitmap); 2967 bw_t_info->shared_bw = 0; 2968 } else { 2969 /* EIR BW and Shared BW profiles are mutually exclusive and 2970 * hence only one of them may be set for any given element. 2971 * First clear earlier saved EIR BW information. 2972 */ 2973 clear_bit(ICE_BW_TYPE_EIR, bw_t_info->bw_t_bitmap); 2974 bw_t_info->eir_bw.bw = 0; 2975 /* save shared BW information */ 2976 set_bit(ICE_BW_TYPE_SHARED, bw_t_info->bw_t_bitmap); 2977 bw_t_info->shared_bw = bw; 2978 } 2979 } 2980 2981 /** 2982 * ice_sched_calc_wakeup - calculate RL profile wakeup parameter 2983 * @hw: pointer to the HW struct 2984 * @bw: bandwidth in Kbps 2985 * 2986 * This function calculates the wakeup parameter of RL profile. 2987 */ 2988 static u16 ice_sched_calc_wakeup(struct ice_hw *hw, s32 bw) 2989 { 2990 s64 bytes_per_sec, wakeup_int, wakeup_a, wakeup_b, wakeup_f; 2991 s32 wakeup_f_int; 2992 u16 wakeup = 0; 2993 2994 /* Get the wakeup integer value */ 2995 bytes_per_sec = div64_long(((s64)bw * 1000), BITS_PER_BYTE); 2996 wakeup_int = div64_long(hw->psm_clk_freq, bytes_per_sec); 2997 if (wakeup_int > 63) { 2998 wakeup = (u16)((1 << 15) | wakeup_int); 2999 } else { 3000 /* Calculate fraction value up to 4 decimals 3001 * Convert Integer value to a constant multiplier 3002 */ 3003 wakeup_b = (s64)ICE_RL_PROF_MULTIPLIER * wakeup_int; 3004 wakeup_a = div64_long((s64)ICE_RL_PROF_MULTIPLIER * 3005 hw->psm_clk_freq, bytes_per_sec); 3006 3007 /* Get Fraction value */ 3008 wakeup_f = wakeup_a - wakeup_b; 3009 3010 /* Round up the Fractional value via Ceil(Fractional value) */ 3011 if (wakeup_f > div64_long(ICE_RL_PROF_MULTIPLIER, 2)) 3012 wakeup_f += 1; 3013 3014 wakeup_f_int = (s32)div64_long(wakeup_f * ICE_RL_PROF_FRACTION, 3015 ICE_RL_PROF_MULTIPLIER); 3016 wakeup |= (u16)(wakeup_int << 9); 3017 wakeup |= (u16)(0x1ff & wakeup_f_int); 3018 } 3019 3020 return wakeup; 3021 } 3022 3023 /** 3024 * ice_sched_bw_to_rl_profile - convert BW to profile parameters 3025 * @hw: pointer to the HW struct 3026 * @bw: bandwidth in Kbps 3027 * @profile: profile parameters to return 3028 * 3029 * This function converts the BW to profile structure format. 3030 */ 3031 static enum ice_status 3032 ice_sched_bw_to_rl_profile(struct ice_hw *hw, u32 bw, 3033 struct ice_aqc_rl_profile_elem *profile) 3034 { 3035 enum ice_status status = ICE_ERR_PARAM; 3036 s64 bytes_per_sec, ts_rate, mv_tmp; 3037 bool found = false; 3038 s32 encode = 0; 3039 s64 mv = 0; 3040 s32 i; 3041 3042 /* Bw settings range is from 0.5Mb/sec to 100Gb/sec */ 3043 if (bw < ICE_SCHED_MIN_BW || bw > ICE_SCHED_MAX_BW) 3044 return status; 3045 3046 /* Bytes per second from Kbps */ 3047 bytes_per_sec = div64_long(((s64)bw * 1000), BITS_PER_BYTE); 3048 3049 /* encode is 6 bits but really useful are 5 bits */ 3050 for (i = 0; i < 64; i++) { 3051 u64 pow_result = BIT_ULL(i); 3052 3053 ts_rate = div64_long((s64)hw->psm_clk_freq, 3054 pow_result * ICE_RL_PROF_TS_MULTIPLIER); 3055 if (ts_rate <= 0) 3056 continue; 3057 3058 /* Multiplier value */ 3059 mv_tmp = div64_long(bytes_per_sec * ICE_RL_PROF_MULTIPLIER, 3060 ts_rate); 3061 3062 /* Round to the nearest ICE_RL_PROF_MULTIPLIER */ 3063 mv = round_up_64bit(mv_tmp, ICE_RL_PROF_MULTIPLIER); 3064 3065 /* First multiplier value greater than the given 3066 * accuracy bytes 3067 */ 3068 if (mv > ICE_RL_PROF_ACCURACY_BYTES) { 3069 encode = i; 3070 found = true; 3071 break; 3072 } 3073 } 3074 if (found) { 3075 u16 wm; 3076 3077 wm = ice_sched_calc_wakeup(hw, bw); 3078 profile->rl_multiply = cpu_to_le16(mv); 3079 profile->wake_up_calc = cpu_to_le16(wm); 3080 profile->rl_encode = cpu_to_le16(encode); 3081 status = 0; 3082 } else { 3083 status = ICE_ERR_DOES_NOT_EXIST; 3084 } 3085 3086 return status; 3087 } 3088 3089 /** 3090 * ice_sched_add_rl_profile - add RL profile 3091 * @pi: port information structure 3092 * @rl_type: type of rate limit BW - min, max, or shared 3093 * @bw: bandwidth in Kbps - Kilo bits per sec 3094 * @layer_num: specifies in which layer to create profile 3095 * 3096 * This function first checks the existing list for corresponding BW 3097 * parameter. If it exists, it returns the associated profile otherwise 3098 * it creates a new rate limit profile for requested BW, and adds it to 3099 * the HW DB and local list. It returns the new profile or null on error. 3100 * The caller needs to hold the scheduler lock. 3101 */ 3102 static struct ice_aqc_rl_profile_info * 3103 ice_sched_add_rl_profile(struct ice_port_info *pi, 3104 enum ice_rl_type rl_type, u32 bw, u8 layer_num) 3105 { 3106 struct ice_aqc_rl_profile_info *rl_prof_elem; 3107 u16 profiles_added = 0, num_profiles = 1; 3108 struct ice_aqc_rl_profile_elem *buf; 3109 enum ice_status status; 3110 struct ice_hw *hw; 3111 u8 profile_type; 3112 3113 if (layer_num >= ICE_AQC_TOPO_MAX_LEVEL_NUM) 3114 return NULL; 3115 switch (rl_type) { 3116 case ICE_MIN_BW: 3117 profile_type = ICE_AQC_RL_PROFILE_TYPE_CIR; 3118 break; 3119 case ICE_MAX_BW: 3120 profile_type = ICE_AQC_RL_PROFILE_TYPE_EIR; 3121 break; 3122 case ICE_SHARED_BW: 3123 profile_type = ICE_AQC_RL_PROFILE_TYPE_SRL; 3124 break; 3125 default: 3126 return NULL; 3127 } 3128 3129 if (!pi) 3130 return NULL; 3131 hw = pi->hw; 3132 list_for_each_entry(rl_prof_elem, &pi->rl_prof_list[layer_num], 3133 list_entry) 3134 if ((rl_prof_elem->profile.flags & ICE_AQC_RL_PROFILE_TYPE_M) == 3135 profile_type && rl_prof_elem->bw == bw) 3136 /* Return existing profile ID info */ 3137 return rl_prof_elem; 3138 3139 /* Create new profile ID */ 3140 rl_prof_elem = devm_kzalloc(ice_hw_to_dev(hw), sizeof(*rl_prof_elem), 3141 GFP_KERNEL); 3142 3143 if (!rl_prof_elem) 3144 return NULL; 3145 3146 status = ice_sched_bw_to_rl_profile(hw, bw, &rl_prof_elem->profile); 3147 if (status) 3148 goto exit_add_rl_prof; 3149 3150 rl_prof_elem->bw = bw; 3151 /* layer_num is zero relative, and fw expects level from 1 to 9 */ 3152 rl_prof_elem->profile.level = layer_num + 1; 3153 rl_prof_elem->profile.flags = profile_type; 3154 rl_prof_elem->profile.max_burst_size = cpu_to_le16(hw->max_burst_size); 3155 3156 /* Create new entry in HW DB */ 3157 buf = &rl_prof_elem->profile; 3158 status = ice_aq_add_rl_profile(hw, num_profiles, buf, sizeof(*buf), 3159 &profiles_added, NULL); 3160 if (status || profiles_added != num_profiles) 3161 goto exit_add_rl_prof; 3162 3163 /* Good entry - add in the list */ 3164 rl_prof_elem->prof_id_ref = 0; 3165 list_add(&rl_prof_elem->list_entry, &pi->rl_prof_list[layer_num]); 3166 return rl_prof_elem; 3167 3168 exit_add_rl_prof: 3169 devm_kfree(ice_hw_to_dev(hw), rl_prof_elem); 3170 return NULL; 3171 } 3172 3173 /** 3174 * ice_sched_cfg_node_bw_lmt - configure node sched params 3175 * @hw: pointer to the HW struct 3176 * @node: sched node to configure 3177 * @rl_type: rate limit type CIR, EIR, or shared 3178 * @rl_prof_id: rate limit profile ID 3179 * 3180 * This function configures node element's BW limit. 3181 */ 3182 static enum ice_status 3183 ice_sched_cfg_node_bw_lmt(struct ice_hw *hw, struct ice_sched_node *node, 3184 enum ice_rl_type rl_type, u16 rl_prof_id) 3185 { 3186 struct ice_aqc_txsched_elem_data buf; 3187 struct ice_aqc_txsched_elem *data; 3188 3189 buf = node->info; 3190 data = &buf.data; 3191 switch (rl_type) { 3192 case ICE_MIN_BW: 3193 data->valid_sections |= ICE_AQC_ELEM_VALID_CIR; 3194 data->cir_bw.bw_profile_idx = cpu_to_le16(rl_prof_id); 3195 break; 3196 case ICE_MAX_BW: 3197 /* EIR BW and Shared BW profiles are mutually exclusive and 3198 * hence only one of them may be set for any given element 3199 */ 3200 if (data->valid_sections & ICE_AQC_ELEM_VALID_SHARED) 3201 return ICE_ERR_CFG; 3202 data->valid_sections |= ICE_AQC_ELEM_VALID_EIR; 3203 data->eir_bw.bw_profile_idx = cpu_to_le16(rl_prof_id); 3204 break; 3205 case ICE_SHARED_BW: 3206 /* Check for removing shared BW */ 3207 if (rl_prof_id == ICE_SCHED_NO_SHARED_RL_PROF_ID) { 3208 /* remove shared profile */ 3209 data->valid_sections &= ~ICE_AQC_ELEM_VALID_SHARED; 3210 data->srl_id = 0; /* clear SRL field */ 3211 3212 /* enable back EIR to default profile */ 3213 data->valid_sections |= ICE_AQC_ELEM_VALID_EIR; 3214 data->eir_bw.bw_profile_idx = 3215 cpu_to_le16(ICE_SCHED_DFLT_RL_PROF_ID); 3216 break; 3217 } 3218 /* EIR BW and Shared BW profiles are mutually exclusive and 3219 * hence only one of them may be set for any given element 3220 */ 3221 if ((data->valid_sections & ICE_AQC_ELEM_VALID_EIR) && 3222 (le16_to_cpu(data->eir_bw.bw_profile_idx) != 3223 ICE_SCHED_DFLT_RL_PROF_ID)) 3224 return ICE_ERR_CFG; 3225 /* EIR BW is set to default, disable it */ 3226 data->valid_sections &= ~ICE_AQC_ELEM_VALID_EIR; 3227 /* Okay to enable shared BW now */ 3228 data->valid_sections |= ICE_AQC_ELEM_VALID_SHARED; 3229 data->srl_id = cpu_to_le16(rl_prof_id); 3230 break; 3231 default: 3232 /* Unknown rate limit type */ 3233 return ICE_ERR_PARAM; 3234 } 3235 3236 /* Configure element */ 3237 return ice_sched_update_elem(hw, node, &buf); 3238 } 3239 3240 /** 3241 * ice_sched_get_node_rl_prof_id - get node's rate limit profile ID 3242 * @node: sched node 3243 * @rl_type: rate limit type 3244 * 3245 * If existing profile matches, it returns the corresponding rate 3246 * limit profile ID, otherwise it returns an invalid ID as error. 3247 */ 3248 static u16 3249 ice_sched_get_node_rl_prof_id(struct ice_sched_node *node, 3250 enum ice_rl_type rl_type) 3251 { 3252 u16 rl_prof_id = ICE_SCHED_INVAL_PROF_ID; 3253 struct ice_aqc_txsched_elem *data; 3254 3255 data = &node->info.data; 3256 switch (rl_type) { 3257 case ICE_MIN_BW: 3258 if (data->valid_sections & ICE_AQC_ELEM_VALID_CIR) 3259 rl_prof_id = le16_to_cpu(data->cir_bw.bw_profile_idx); 3260 break; 3261 case ICE_MAX_BW: 3262 if (data->valid_sections & ICE_AQC_ELEM_VALID_EIR) 3263 rl_prof_id = le16_to_cpu(data->eir_bw.bw_profile_idx); 3264 break; 3265 case ICE_SHARED_BW: 3266 if (data->valid_sections & ICE_AQC_ELEM_VALID_SHARED) 3267 rl_prof_id = le16_to_cpu(data->srl_id); 3268 break; 3269 default: 3270 break; 3271 } 3272 3273 return rl_prof_id; 3274 } 3275 3276 /** 3277 * ice_sched_get_rl_prof_layer - selects rate limit profile creation layer 3278 * @pi: port information structure 3279 * @rl_type: type of rate limit BW - min, max, or shared 3280 * @layer_index: layer index 3281 * 3282 * This function returns requested profile creation layer. 3283 */ 3284 static u8 3285 ice_sched_get_rl_prof_layer(struct ice_port_info *pi, enum ice_rl_type rl_type, 3286 u8 layer_index) 3287 { 3288 struct ice_hw *hw = pi->hw; 3289 3290 if (layer_index >= hw->num_tx_sched_layers) 3291 return ICE_SCHED_INVAL_LAYER_NUM; 3292 switch (rl_type) { 3293 case ICE_MIN_BW: 3294 if (hw->layer_info[layer_index].max_cir_rl_profiles) 3295 return layer_index; 3296 break; 3297 case ICE_MAX_BW: 3298 if (hw->layer_info[layer_index].max_eir_rl_profiles) 3299 return layer_index; 3300 break; 3301 case ICE_SHARED_BW: 3302 /* if current layer doesn't support SRL profile creation 3303 * then try a layer up or down. 3304 */ 3305 if (hw->layer_info[layer_index].max_srl_profiles) 3306 return layer_index; 3307 else if (layer_index < hw->num_tx_sched_layers - 1 && 3308 hw->layer_info[layer_index + 1].max_srl_profiles) 3309 return layer_index + 1; 3310 else if (layer_index > 0 && 3311 hw->layer_info[layer_index - 1].max_srl_profiles) 3312 return layer_index - 1; 3313 break; 3314 default: 3315 break; 3316 } 3317 return ICE_SCHED_INVAL_LAYER_NUM; 3318 } 3319 3320 /** 3321 * ice_sched_get_srl_node - get shared rate limit node 3322 * @node: tree node 3323 * @srl_layer: shared rate limit layer 3324 * 3325 * This function returns SRL node to be used for shared rate limit purpose. 3326 * The caller needs to hold scheduler lock. 3327 */ 3328 static struct ice_sched_node * 3329 ice_sched_get_srl_node(struct ice_sched_node *node, u8 srl_layer) 3330 { 3331 if (srl_layer > node->tx_sched_layer) 3332 return node->children[0]; 3333 else if (srl_layer < node->tx_sched_layer) 3334 /* Node can't be created without a parent. It will always 3335 * have a valid parent except root node. 3336 */ 3337 return node->parent; 3338 else 3339 return node; 3340 } 3341 3342 /** 3343 * ice_sched_rm_rl_profile - remove RL profile ID 3344 * @pi: port information structure 3345 * @layer_num: layer number where profiles are saved 3346 * @profile_type: profile type like EIR, CIR, or SRL 3347 * @profile_id: profile ID to remove 3348 * 3349 * This function removes rate limit profile from layer 'layer_num' of type 3350 * 'profile_type' and profile ID as 'profile_id'. The caller needs to hold 3351 * scheduler lock. 3352 */ 3353 static enum ice_status 3354 ice_sched_rm_rl_profile(struct ice_port_info *pi, u8 layer_num, u8 profile_type, 3355 u16 profile_id) 3356 { 3357 struct ice_aqc_rl_profile_info *rl_prof_elem; 3358 enum ice_status status = 0; 3359 3360 if (layer_num >= ICE_AQC_TOPO_MAX_LEVEL_NUM) 3361 return ICE_ERR_PARAM; 3362 /* Check the existing list for RL profile */ 3363 list_for_each_entry(rl_prof_elem, &pi->rl_prof_list[layer_num], 3364 list_entry) 3365 if ((rl_prof_elem->profile.flags & ICE_AQC_RL_PROFILE_TYPE_M) == 3366 profile_type && 3367 le16_to_cpu(rl_prof_elem->profile.profile_id) == 3368 profile_id) { 3369 if (rl_prof_elem->prof_id_ref) 3370 rl_prof_elem->prof_id_ref--; 3371 3372 /* Remove old profile ID from database */ 3373 status = ice_sched_del_rl_profile(pi->hw, rl_prof_elem); 3374 if (status && status != ICE_ERR_IN_USE) 3375 ice_debug(pi->hw, ICE_DBG_SCHED, "Remove rl profile failed\n"); 3376 break; 3377 } 3378 if (status == ICE_ERR_IN_USE) 3379 status = 0; 3380 return status; 3381 } 3382 3383 /** 3384 * ice_sched_set_node_bw_dflt - set node's bandwidth limit to default 3385 * @pi: port information structure 3386 * @node: pointer to node structure 3387 * @rl_type: rate limit type min, max, or shared 3388 * @layer_num: layer number where RL profiles are saved 3389 * 3390 * This function configures node element's BW rate limit profile ID of 3391 * type CIR, EIR, or SRL to default. This function needs to be called 3392 * with the scheduler lock held. 3393 */ 3394 static enum ice_status 3395 ice_sched_set_node_bw_dflt(struct ice_port_info *pi, 3396 struct ice_sched_node *node, 3397 enum ice_rl_type rl_type, u8 layer_num) 3398 { 3399 enum ice_status status; 3400 struct ice_hw *hw; 3401 u8 profile_type; 3402 u16 rl_prof_id; 3403 u16 old_id; 3404 3405 hw = pi->hw; 3406 switch (rl_type) { 3407 case ICE_MIN_BW: 3408 profile_type = ICE_AQC_RL_PROFILE_TYPE_CIR; 3409 rl_prof_id = ICE_SCHED_DFLT_RL_PROF_ID; 3410 break; 3411 case ICE_MAX_BW: 3412 profile_type = ICE_AQC_RL_PROFILE_TYPE_EIR; 3413 rl_prof_id = ICE_SCHED_DFLT_RL_PROF_ID; 3414 break; 3415 case ICE_SHARED_BW: 3416 profile_type = ICE_AQC_RL_PROFILE_TYPE_SRL; 3417 /* No SRL is configured for default case */ 3418 rl_prof_id = ICE_SCHED_NO_SHARED_RL_PROF_ID; 3419 break; 3420 default: 3421 return ICE_ERR_PARAM; 3422 } 3423 /* Save existing RL prof ID for later clean up */ 3424 old_id = ice_sched_get_node_rl_prof_id(node, rl_type); 3425 /* Configure BW scheduling parameters */ 3426 status = ice_sched_cfg_node_bw_lmt(hw, node, rl_type, rl_prof_id); 3427 if (status) 3428 return status; 3429 3430 /* Remove stale RL profile ID */ 3431 if (old_id == ICE_SCHED_DFLT_RL_PROF_ID || 3432 old_id == ICE_SCHED_INVAL_PROF_ID) 3433 return 0; 3434 3435 return ice_sched_rm_rl_profile(pi, layer_num, profile_type, old_id); 3436 } 3437 3438 /** 3439 * ice_sched_set_eir_srl_excl - set EIR/SRL exclusiveness 3440 * @pi: port information structure 3441 * @node: pointer to node structure 3442 * @layer_num: layer number where rate limit profiles are saved 3443 * @rl_type: rate limit type min, max, or shared 3444 * @bw: bandwidth value 3445 * 3446 * This function prepares node element's bandwidth to SRL or EIR exclusively. 3447 * EIR BW and Shared BW profiles are mutually exclusive and hence only one of 3448 * them may be set for any given element. This function needs to be called 3449 * with the scheduler lock held. 3450 */ 3451 static enum ice_status 3452 ice_sched_set_eir_srl_excl(struct ice_port_info *pi, 3453 struct ice_sched_node *node, 3454 u8 layer_num, enum ice_rl_type rl_type, u32 bw) 3455 { 3456 if (rl_type == ICE_SHARED_BW) { 3457 /* SRL node passed in this case, it may be different node */ 3458 if (bw == ICE_SCHED_DFLT_BW) 3459 /* SRL being removed, ice_sched_cfg_node_bw_lmt() 3460 * enables EIR to default. EIR is not set in this 3461 * case, so no additional action is required. 3462 */ 3463 return 0; 3464 3465 /* SRL being configured, set EIR to default here. 3466 * ice_sched_cfg_node_bw_lmt() disables EIR when it 3467 * configures SRL 3468 */ 3469 return ice_sched_set_node_bw_dflt(pi, node, ICE_MAX_BW, 3470 layer_num); 3471 } else if (rl_type == ICE_MAX_BW && 3472 node->info.data.valid_sections & ICE_AQC_ELEM_VALID_SHARED) { 3473 /* Remove Shared profile. Set default shared BW call 3474 * removes shared profile for a node. 3475 */ 3476 return ice_sched_set_node_bw_dflt(pi, node, 3477 ICE_SHARED_BW, 3478 layer_num); 3479 } 3480 return 0; 3481 } 3482 3483 /** 3484 * ice_sched_set_node_bw - set node's bandwidth 3485 * @pi: port information structure 3486 * @node: tree node 3487 * @rl_type: rate limit type min, max, or shared 3488 * @bw: bandwidth in Kbps - Kilo bits per sec 3489 * @layer_num: layer number 3490 * 3491 * This function adds new profile corresponding to requested BW, configures 3492 * node's RL profile ID of type CIR, EIR, or SRL, and removes old profile 3493 * ID from local database. The caller needs to hold scheduler lock. 3494 */ 3495 static enum ice_status 3496 ice_sched_set_node_bw(struct ice_port_info *pi, struct ice_sched_node *node, 3497 enum ice_rl_type rl_type, u32 bw, u8 layer_num) 3498 { 3499 struct ice_aqc_rl_profile_info *rl_prof_info; 3500 enum ice_status status = ICE_ERR_PARAM; 3501 struct ice_hw *hw = pi->hw; 3502 u16 old_id, rl_prof_id; 3503 3504 rl_prof_info = ice_sched_add_rl_profile(pi, rl_type, bw, layer_num); 3505 if (!rl_prof_info) 3506 return status; 3507 3508 rl_prof_id = le16_to_cpu(rl_prof_info->profile.profile_id); 3509 3510 /* Save existing RL prof ID for later clean up */ 3511 old_id = ice_sched_get_node_rl_prof_id(node, rl_type); 3512 /* Configure BW scheduling parameters */ 3513 status = ice_sched_cfg_node_bw_lmt(hw, node, rl_type, rl_prof_id); 3514 if (status) 3515 return status; 3516 3517 /* New changes has been applied */ 3518 /* Increment the profile ID reference count */ 3519 rl_prof_info->prof_id_ref++; 3520 3521 /* Check for old ID removal */ 3522 if ((old_id == ICE_SCHED_DFLT_RL_PROF_ID && rl_type != ICE_SHARED_BW) || 3523 old_id == ICE_SCHED_INVAL_PROF_ID || old_id == rl_prof_id) 3524 return 0; 3525 3526 return ice_sched_rm_rl_profile(pi, layer_num, 3527 rl_prof_info->profile.flags & 3528 ICE_AQC_RL_PROFILE_TYPE_M, old_id); 3529 } 3530 3531 /** 3532 * ice_sched_set_node_bw_lmt - set node's BW limit 3533 * @pi: port information structure 3534 * @node: tree node 3535 * @rl_type: rate limit type min, max, or shared 3536 * @bw: bandwidth in Kbps - Kilo bits per sec 3537 * 3538 * It updates node's BW limit parameters like BW RL profile ID of type CIR, 3539 * EIR, or SRL. The caller needs to hold scheduler lock. 3540 */ 3541 static enum ice_status 3542 ice_sched_set_node_bw_lmt(struct ice_port_info *pi, struct ice_sched_node *node, 3543 enum ice_rl_type rl_type, u32 bw) 3544 { 3545 struct ice_sched_node *cfg_node = node; 3546 enum ice_status status; 3547 3548 struct ice_hw *hw; 3549 u8 layer_num; 3550 3551 if (!pi) 3552 return ICE_ERR_PARAM; 3553 hw = pi->hw; 3554 /* Remove unused RL profile IDs from HW and SW DB */ 3555 ice_sched_rm_unused_rl_prof(pi); 3556 layer_num = ice_sched_get_rl_prof_layer(pi, rl_type, 3557 node->tx_sched_layer); 3558 if (layer_num >= hw->num_tx_sched_layers) 3559 return ICE_ERR_PARAM; 3560 3561 if (rl_type == ICE_SHARED_BW) { 3562 /* SRL node may be different */ 3563 cfg_node = ice_sched_get_srl_node(node, layer_num); 3564 if (!cfg_node) 3565 return ICE_ERR_CFG; 3566 } 3567 /* EIR BW and Shared BW profiles are mutually exclusive and 3568 * hence only one of them may be set for any given element 3569 */ 3570 status = ice_sched_set_eir_srl_excl(pi, cfg_node, layer_num, rl_type, 3571 bw); 3572 if (status) 3573 return status; 3574 if (bw == ICE_SCHED_DFLT_BW) 3575 return ice_sched_set_node_bw_dflt(pi, cfg_node, rl_type, 3576 layer_num); 3577 return ice_sched_set_node_bw(pi, cfg_node, rl_type, bw, layer_num); 3578 } 3579 3580 /** 3581 * ice_sched_set_node_bw_dflt_lmt - set node's BW limit to default 3582 * @pi: port information structure 3583 * @node: pointer to node structure 3584 * @rl_type: rate limit type min, max, or shared 3585 * 3586 * This function configures node element's BW rate limit profile ID of 3587 * type CIR, EIR, or SRL to default. This function needs to be called 3588 * with the scheduler lock held. 3589 */ 3590 static enum ice_status 3591 ice_sched_set_node_bw_dflt_lmt(struct ice_port_info *pi, 3592 struct ice_sched_node *node, 3593 enum ice_rl_type rl_type) 3594 { 3595 return ice_sched_set_node_bw_lmt(pi, node, rl_type, 3596 ICE_SCHED_DFLT_BW); 3597 } 3598 3599 /** 3600 * ice_sched_validate_srl_node - Check node for SRL applicability 3601 * @node: sched node to configure 3602 * @sel_layer: selected SRL layer 3603 * 3604 * This function checks if the SRL can be applied to a selected layer node on 3605 * behalf of the requested node (first argument). This function needs to be 3606 * called with scheduler lock held. 3607 */ 3608 static enum ice_status 3609 ice_sched_validate_srl_node(struct ice_sched_node *node, u8 sel_layer) 3610 { 3611 /* SRL profiles are not available on all layers. Check if the 3612 * SRL profile can be applied to a node above or below the 3613 * requested node. SRL configuration is possible only if the 3614 * selected layer's node has single child. 3615 */ 3616 if (sel_layer == node->tx_sched_layer || 3617 ((sel_layer == node->tx_sched_layer + 1) && 3618 node->num_children == 1) || 3619 ((sel_layer == node->tx_sched_layer - 1) && 3620 (node->parent && node->parent->num_children == 1))) 3621 return 0; 3622 3623 return ICE_ERR_CFG; 3624 } 3625 3626 /** 3627 * ice_sched_save_q_bw - save queue node's BW information 3628 * @q_ctx: queue context structure 3629 * @rl_type: rate limit type min, max, or shared 3630 * @bw: bandwidth in Kbps - Kilo bits per sec 3631 * 3632 * Save BW information of queue type node for post replay use. 3633 */ 3634 static enum ice_status 3635 ice_sched_save_q_bw(struct ice_q_ctx *q_ctx, enum ice_rl_type rl_type, u32 bw) 3636 { 3637 switch (rl_type) { 3638 case ICE_MIN_BW: 3639 ice_set_clear_cir_bw(&q_ctx->bw_t_info, bw); 3640 break; 3641 case ICE_MAX_BW: 3642 ice_set_clear_eir_bw(&q_ctx->bw_t_info, bw); 3643 break; 3644 case ICE_SHARED_BW: 3645 ice_set_clear_shared_bw(&q_ctx->bw_t_info, bw); 3646 break; 3647 default: 3648 return ICE_ERR_PARAM; 3649 } 3650 return 0; 3651 } 3652 3653 /** 3654 * ice_sched_set_q_bw_lmt - sets queue BW limit 3655 * @pi: port information structure 3656 * @vsi_handle: sw VSI handle 3657 * @tc: traffic class 3658 * @q_handle: software queue handle 3659 * @rl_type: min, max, or shared 3660 * @bw: bandwidth in Kbps 3661 * 3662 * This function sets BW limit of queue scheduling node. 3663 */ 3664 static enum ice_status 3665 ice_sched_set_q_bw_lmt(struct ice_port_info *pi, u16 vsi_handle, u8 tc, 3666 u16 q_handle, enum ice_rl_type rl_type, u32 bw) 3667 { 3668 enum ice_status status = ICE_ERR_PARAM; 3669 struct ice_sched_node *node; 3670 struct ice_q_ctx *q_ctx; 3671 3672 if (!ice_is_vsi_valid(pi->hw, vsi_handle)) 3673 return ICE_ERR_PARAM; 3674 mutex_lock(&pi->sched_lock); 3675 q_ctx = ice_get_lan_q_ctx(pi->hw, vsi_handle, tc, q_handle); 3676 if (!q_ctx) 3677 goto exit_q_bw_lmt; 3678 node = ice_sched_find_node_by_teid(pi->root, q_ctx->q_teid); 3679 if (!node) { 3680 ice_debug(pi->hw, ICE_DBG_SCHED, "Wrong q_teid\n"); 3681 goto exit_q_bw_lmt; 3682 } 3683 3684 /* Return error if it is not a leaf node */ 3685 if (node->info.data.elem_type != ICE_AQC_ELEM_TYPE_LEAF) 3686 goto exit_q_bw_lmt; 3687 3688 /* SRL bandwidth layer selection */ 3689 if (rl_type == ICE_SHARED_BW) { 3690 u8 sel_layer; /* selected layer */ 3691 3692 sel_layer = ice_sched_get_rl_prof_layer(pi, rl_type, 3693 node->tx_sched_layer); 3694 if (sel_layer >= pi->hw->num_tx_sched_layers) { 3695 status = ICE_ERR_PARAM; 3696 goto exit_q_bw_lmt; 3697 } 3698 status = ice_sched_validate_srl_node(node, sel_layer); 3699 if (status) 3700 goto exit_q_bw_lmt; 3701 } 3702 3703 if (bw == ICE_SCHED_DFLT_BW) 3704 status = ice_sched_set_node_bw_dflt_lmt(pi, node, rl_type); 3705 else 3706 status = ice_sched_set_node_bw_lmt(pi, node, rl_type, bw); 3707 3708 if (!status) 3709 status = ice_sched_save_q_bw(q_ctx, rl_type, bw); 3710 3711 exit_q_bw_lmt: 3712 mutex_unlock(&pi->sched_lock); 3713 return status; 3714 } 3715 3716 /** 3717 * ice_cfg_q_bw_lmt - configure queue BW limit 3718 * @pi: port information structure 3719 * @vsi_handle: sw VSI handle 3720 * @tc: traffic class 3721 * @q_handle: software queue handle 3722 * @rl_type: min, max, or shared 3723 * @bw: bandwidth in Kbps 3724 * 3725 * This function configures BW limit of queue scheduling node. 3726 */ 3727 enum ice_status 3728 ice_cfg_q_bw_lmt(struct ice_port_info *pi, u16 vsi_handle, u8 tc, 3729 u16 q_handle, enum ice_rl_type rl_type, u32 bw) 3730 { 3731 return ice_sched_set_q_bw_lmt(pi, vsi_handle, tc, q_handle, rl_type, 3732 bw); 3733 } 3734 3735 /** 3736 * ice_cfg_q_bw_dflt_lmt - configure queue BW default limit 3737 * @pi: port information structure 3738 * @vsi_handle: sw VSI handle 3739 * @tc: traffic class 3740 * @q_handle: software queue handle 3741 * @rl_type: min, max, or shared 3742 * 3743 * This function configures BW default limit of queue scheduling node. 3744 */ 3745 enum ice_status 3746 ice_cfg_q_bw_dflt_lmt(struct ice_port_info *pi, u16 vsi_handle, u8 tc, 3747 u16 q_handle, enum ice_rl_type rl_type) 3748 { 3749 return ice_sched_set_q_bw_lmt(pi, vsi_handle, tc, q_handle, rl_type, 3750 ICE_SCHED_DFLT_BW); 3751 } 3752 3753 /** 3754 * ice_cfg_rl_burst_size - Set burst size value 3755 * @hw: pointer to the HW struct 3756 * @bytes: burst size in bytes 3757 * 3758 * This function configures/set the burst size to requested new value. The new 3759 * burst size value is used for future rate limit calls. It doesn't change the 3760 * existing or previously created RL profiles. 3761 */ 3762 enum ice_status ice_cfg_rl_burst_size(struct ice_hw *hw, u32 bytes) 3763 { 3764 u16 burst_size_to_prog; 3765 3766 if (bytes < ICE_MIN_BURST_SIZE_ALLOWED || 3767 bytes > ICE_MAX_BURST_SIZE_ALLOWED) 3768 return ICE_ERR_PARAM; 3769 if (ice_round_to_num(bytes, 64) <= 3770 ICE_MAX_BURST_SIZE_64_BYTE_GRANULARITY) { 3771 /* 64 byte granularity case */ 3772 /* Disable MSB granularity bit */ 3773 burst_size_to_prog = ICE_64_BYTE_GRANULARITY; 3774 /* round number to nearest 64 byte granularity */ 3775 bytes = ice_round_to_num(bytes, 64); 3776 /* The value is in 64 byte chunks */ 3777 burst_size_to_prog |= (u16)(bytes / 64); 3778 } else { 3779 /* k bytes granularity case */ 3780 /* Enable MSB granularity bit */ 3781 burst_size_to_prog = ICE_KBYTE_GRANULARITY; 3782 /* round number to nearest 1024 granularity */ 3783 bytes = ice_round_to_num(bytes, 1024); 3784 /* check rounding doesn't go beyond allowed */ 3785 if (bytes > ICE_MAX_BURST_SIZE_KBYTE_GRANULARITY) 3786 bytes = ICE_MAX_BURST_SIZE_KBYTE_GRANULARITY; 3787 /* The value is in k bytes */ 3788 burst_size_to_prog |= (u16)(bytes / 1024); 3789 } 3790 hw->max_burst_size = burst_size_to_prog; 3791 return 0; 3792 } 3793 3794 /** 3795 * ice_sched_replay_node_prio - re-configure node priority 3796 * @hw: pointer to the HW struct 3797 * @node: sched node to configure 3798 * @priority: priority value 3799 * 3800 * This function configures node element's priority value. It 3801 * needs to be called with scheduler lock held. 3802 */ 3803 static enum ice_status 3804 ice_sched_replay_node_prio(struct ice_hw *hw, struct ice_sched_node *node, 3805 u8 priority) 3806 { 3807 struct ice_aqc_txsched_elem_data buf; 3808 struct ice_aqc_txsched_elem *data; 3809 enum ice_status status; 3810 3811 buf = node->info; 3812 data = &buf.data; 3813 data->valid_sections |= ICE_AQC_ELEM_VALID_GENERIC; 3814 data->generic = priority; 3815 3816 /* Configure element */ 3817 status = ice_sched_update_elem(hw, node, &buf); 3818 return status; 3819 } 3820 3821 /** 3822 * ice_sched_replay_node_bw - replay node(s) BW 3823 * @hw: pointer to the HW struct 3824 * @node: sched node to configure 3825 * @bw_t_info: BW type information 3826 * 3827 * This function restores node's BW from bw_t_info. The caller needs 3828 * to hold the scheduler lock. 3829 */ 3830 static enum ice_status 3831 ice_sched_replay_node_bw(struct ice_hw *hw, struct ice_sched_node *node, 3832 struct ice_bw_type_info *bw_t_info) 3833 { 3834 struct ice_port_info *pi = hw->port_info; 3835 enum ice_status status = ICE_ERR_PARAM; 3836 u16 bw_alloc; 3837 3838 if (!node) 3839 return status; 3840 if (bitmap_empty(bw_t_info->bw_t_bitmap, ICE_BW_TYPE_CNT)) 3841 return 0; 3842 if (test_bit(ICE_BW_TYPE_PRIO, bw_t_info->bw_t_bitmap)) { 3843 status = ice_sched_replay_node_prio(hw, node, 3844 bw_t_info->generic); 3845 if (status) 3846 return status; 3847 } 3848 if (test_bit(ICE_BW_TYPE_CIR, bw_t_info->bw_t_bitmap)) { 3849 status = ice_sched_set_node_bw_lmt(pi, node, ICE_MIN_BW, 3850 bw_t_info->cir_bw.bw); 3851 if (status) 3852 return status; 3853 } 3854 if (test_bit(ICE_BW_TYPE_CIR_WT, bw_t_info->bw_t_bitmap)) { 3855 bw_alloc = bw_t_info->cir_bw.bw_alloc; 3856 status = ice_sched_cfg_node_bw_alloc(hw, node, ICE_MIN_BW, 3857 bw_alloc); 3858 if (status) 3859 return status; 3860 } 3861 if (test_bit(ICE_BW_TYPE_EIR, bw_t_info->bw_t_bitmap)) { 3862 status = ice_sched_set_node_bw_lmt(pi, node, ICE_MAX_BW, 3863 bw_t_info->eir_bw.bw); 3864 if (status) 3865 return status; 3866 } 3867 if (test_bit(ICE_BW_TYPE_EIR_WT, bw_t_info->bw_t_bitmap)) { 3868 bw_alloc = bw_t_info->eir_bw.bw_alloc; 3869 status = ice_sched_cfg_node_bw_alloc(hw, node, ICE_MAX_BW, 3870 bw_alloc); 3871 if (status) 3872 return status; 3873 } 3874 if (test_bit(ICE_BW_TYPE_SHARED, bw_t_info->bw_t_bitmap)) 3875 status = ice_sched_set_node_bw_lmt(pi, node, ICE_SHARED_BW, 3876 bw_t_info->shared_bw); 3877 return status; 3878 } 3879 3880 /** 3881 * ice_sched_get_ena_tc_bitmap - get enabled TC bitmap 3882 * @pi: port info struct 3883 * @tc_bitmap: 8 bits TC bitmap to check 3884 * @ena_tc_bitmap: 8 bits enabled TC bitmap to return 3885 * 3886 * This function returns enabled TC bitmap in variable ena_tc_bitmap. Some TCs 3887 * may be missing, it returns enabled TCs. This function needs to be called with 3888 * scheduler lock held. 3889 */ 3890 static void 3891 ice_sched_get_ena_tc_bitmap(struct ice_port_info *pi, 3892 unsigned long *tc_bitmap, 3893 unsigned long *ena_tc_bitmap) 3894 { 3895 u8 tc; 3896 3897 /* Some TC(s) may be missing after reset, adjust for replay */ 3898 ice_for_each_traffic_class(tc) 3899 if (ice_is_tc_ena(*tc_bitmap, tc) && 3900 (ice_sched_get_tc_node(pi, tc))) 3901 set_bit(tc, ena_tc_bitmap); 3902 } 3903 3904 /** 3905 * ice_sched_replay_agg - recreate aggregator node(s) 3906 * @hw: pointer to the HW struct 3907 * 3908 * This function recreate aggregator type nodes which are not replayed earlier. 3909 * It also replay aggregator BW information. These aggregator nodes are not 3910 * associated with VSI type node yet. 3911 */ 3912 void ice_sched_replay_agg(struct ice_hw *hw) 3913 { 3914 struct ice_port_info *pi = hw->port_info; 3915 struct ice_sched_agg_info *agg_info; 3916 3917 mutex_lock(&pi->sched_lock); 3918 list_for_each_entry(agg_info, &hw->agg_list, list_entry) 3919 /* replay aggregator (re-create aggregator node) */ 3920 if (!bitmap_equal(agg_info->tc_bitmap, agg_info->replay_tc_bitmap, 3921 ICE_MAX_TRAFFIC_CLASS)) { 3922 DECLARE_BITMAP(replay_bitmap, ICE_MAX_TRAFFIC_CLASS); 3923 enum ice_status status; 3924 3925 bitmap_zero(replay_bitmap, ICE_MAX_TRAFFIC_CLASS); 3926 ice_sched_get_ena_tc_bitmap(pi, 3927 agg_info->replay_tc_bitmap, 3928 replay_bitmap); 3929 status = ice_sched_cfg_agg(hw->port_info, 3930 agg_info->agg_id, 3931 ICE_AGG_TYPE_AGG, 3932 replay_bitmap); 3933 if (status) { 3934 dev_info(ice_hw_to_dev(hw), 3935 "Replay agg id[%d] failed\n", 3936 agg_info->agg_id); 3937 /* Move on to next one */ 3938 continue; 3939 } 3940 } 3941 mutex_unlock(&pi->sched_lock); 3942 } 3943 3944 /** 3945 * ice_sched_replay_agg_vsi_preinit - Agg/VSI replay pre initialization 3946 * @hw: pointer to the HW struct 3947 * 3948 * This function initialize aggregator(s) TC bitmap to zero. A required 3949 * preinit step for replaying aggregators. 3950 */ 3951 void ice_sched_replay_agg_vsi_preinit(struct ice_hw *hw) 3952 { 3953 struct ice_port_info *pi = hw->port_info; 3954 struct ice_sched_agg_info *agg_info; 3955 3956 mutex_lock(&pi->sched_lock); 3957 list_for_each_entry(agg_info, &hw->agg_list, list_entry) { 3958 struct ice_sched_agg_vsi_info *agg_vsi_info; 3959 3960 agg_info->tc_bitmap[0] = 0; 3961 list_for_each_entry(agg_vsi_info, &agg_info->agg_vsi_list, 3962 list_entry) 3963 agg_vsi_info->tc_bitmap[0] = 0; 3964 } 3965 mutex_unlock(&pi->sched_lock); 3966 } 3967 3968 /** 3969 * ice_sched_replay_vsi_agg - replay aggregator & VSI to aggregator node(s) 3970 * @hw: pointer to the HW struct 3971 * @vsi_handle: software VSI handle 3972 * 3973 * This function replays aggregator node, VSI to aggregator type nodes, and 3974 * their node bandwidth information. This function needs to be called with 3975 * scheduler lock held. 3976 */ 3977 static enum ice_status 3978 ice_sched_replay_vsi_agg(struct ice_hw *hw, u16 vsi_handle) 3979 { 3980 DECLARE_BITMAP(replay_bitmap, ICE_MAX_TRAFFIC_CLASS); 3981 struct ice_sched_agg_vsi_info *agg_vsi_info; 3982 struct ice_port_info *pi = hw->port_info; 3983 struct ice_sched_agg_info *agg_info; 3984 enum ice_status status; 3985 3986 bitmap_zero(replay_bitmap, ICE_MAX_TRAFFIC_CLASS); 3987 if (!ice_is_vsi_valid(hw, vsi_handle)) 3988 return ICE_ERR_PARAM; 3989 agg_info = ice_get_vsi_agg_info(hw, vsi_handle); 3990 if (!agg_info) 3991 return 0; /* Not present in list - default Agg case */ 3992 agg_vsi_info = ice_get_agg_vsi_info(agg_info, vsi_handle); 3993 if (!agg_vsi_info) 3994 return 0; /* Not present in list - default Agg case */ 3995 ice_sched_get_ena_tc_bitmap(pi, agg_info->replay_tc_bitmap, 3996 replay_bitmap); 3997 /* Replay aggregator node associated to vsi_handle */ 3998 status = ice_sched_cfg_agg(hw->port_info, agg_info->agg_id, 3999 ICE_AGG_TYPE_AGG, replay_bitmap); 4000 if (status) 4001 return status; 4002 4003 bitmap_zero(replay_bitmap, ICE_MAX_TRAFFIC_CLASS); 4004 ice_sched_get_ena_tc_bitmap(pi, agg_vsi_info->replay_tc_bitmap, 4005 replay_bitmap); 4006 /* Move this VSI (vsi_handle) to above aggregator */ 4007 return ice_sched_assoc_vsi_to_agg(pi, agg_info->agg_id, vsi_handle, 4008 replay_bitmap); 4009 } 4010 4011 /** 4012 * ice_replay_vsi_agg - replay VSI to aggregator node 4013 * @hw: pointer to the HW struct 4014 * @vsi_handle: software VSI handle 4015 * 4016 * This function replays association of VSI to aggregator type nodes, and 4017 * node bandwidth information. 4018 */ 4019 enum ice_status ice_replay_vsi_agg(struct ice_hw *hw, u16 vsi_handle) 4020 { 4021 struct ice_port_info *pi = hw->port_info; 4022 enum ice_status status; 4023 4024 mutex_lock(&pi->sched_lock); 4025 status = ice_sched_replay_vsi_agg(hw, vsi_handle); 4026 mutex_unlock(&pi->sched_lock); 4027 return status; 4028 } 4029 4030 /** 4031 * ice_sched_replay_q_bw - replay queue type node BW 4032 * @pi: port information structure 4033 * @q_ctx: queue context structure 4034 * 4035 * This function replays queue type node bandwidth. This function needs to be 4036 * called with scheduler lock held. 4037 */ 4038 enum ice_status 4039 ice_sched_replay_q_bw(struct ice_port_info *pi, struct ice_q_ctx *q_ctx) 4040 { 4041 struct ice_sched_node *q_node; 4042 4043 /* Following also checks the presence of node in tree */ 4044 q_node = ice_sched_find_node_by_teid(pi->root, q_ctx->q_teid); 4045 if (!q_node) 4046 return ICE_ERR_PARAM; 4047 return ice_sched_replay_node_bw(pi->hw, q_node, &q_ctx->bw_t_info); 4048 } 4049