1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Physcial Function ethernet driver 3 * 4 * Copyright (C) 2021 Marvell. 5 */ 6 #include <linux/netdevice.h> 7 #include <linux/etherdevice.h> 8 #include <linux/inetdevice.h> 9 #include <linux/rhashtable.h> 10 #include <linux/bitfield.h> 11 #include <net/flow_dissector.h> 12 #include <net/pkt_cls.h> 13 #include <net/tc_act/tc_gact.h> 14 #include <net/tc_act/tc_mirred.h> 15 #include <net/tc_act/tc_vlan.h> 16 #include <net/ipv6.h> 17 18 #include "cn10k.h" 19 #include "otx2_common.h" 20 21 /* Egress rate limiting definitions */ 22 #define MAX_BURST_EXPONENT 0x0FULL 23 #define MAX_BURST_MANTISSA 0xFFULL 24 #define MAX_BURST_SIZE 130816ULL 25 #define MAX_RATE_DIVIDER_EXPONENT 12ULL 26 #define MAX_RATE_EXPONENT 0x0FULL 27 #define MAX_RATE_MANTISSA 0xFFULL 28 29 /* Bitfields in NIX_TLX_PIR register */ 30 #define TLX_RATE_MANTISSA GENMASK_ULL(8, 1) 31 #define TLX_RATE_EXPONENT GENMASK_ULL(12, 9) 32 #define TLX_RATE_DIVIDER_EXPONENT GENMASK_ULL(16, 13) 33 #define TLX_BURST_MANTISSA GENMASK_ULL(36, 29) 34 #define TLX_BURST_EXPONENT GENMASK_ULL(40, 37) 35 36 struct otx2_tc_flow_stats { 37 u64 bytes; 38 u64 pkts; 39 u64 used; 40 }; 41 42 struct otx2_tc_flow { 43 struct rhash_head node; 44 unsigned long cookie; 45 unsigned int bitpos; 46 struct rcu_head rcu; 47 struct otx2_tc_flow_stats stats; 48 spinlock_t lock; /* lock for stats */ 49 u16 rq; 50 u16 entry; 51 u16 leaf_profile; 52 bool is_act_police; 53 }; 54 55 int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic) 56 { 57 struct otx2_tc_info *tc = &nic->tc_info; 58 59 if (!nic->flow_cfg->max_flows || is_otx2_vf(nic->pcifunc)) 60 return 0; 61 62 /* Max flows changed, free the existing bitmap */ 63 kfree(tc->tc_entries_bitmap); 64 65 tc->tc_entries_bitmap = 66 kcalloc(BITS_TO_LONGS(nic->flow_cfg->max_flows), 67 sizeof(long), GFP_KERNEL); 68 if (!tc->tc_entries_bitmap) { 69 netdev_err(nic->netdev, 70 "Unable to alloc TC flow entries bitmap\n"); 71 return -ENOMEM; 72 } 73 74 return 0; 75 } 76 EXPORT_SYMBOL(otx2_tc_alloc_ent_bitmap); 77 78 static void otx2_get_egress_burst_cfg(u32 burst, u32 *burst_exp, 79 u32 *burst_mantissa) 80 { 81 unsigned int tmp; 82 83 /* Burst is calculated as 84 * ((256 + BURST_MANTISSA) << (1 + BURST_EXPONENT)) / 256 85 * Max supported burst size is 130,816 bytes. 86 */ 87 burst = min_t(u32, burst, MAX_BURST_SIZE); 88 if (burst) { 89 *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0; 90 tmp = burst - rounddown_pow_of_two(burst); 91 if (burst < MAX_BURST_MANTISSA) 92 *burst_mantissa = tmp * 2; 93 else 94 *burst_mantissa = tmp / (1ULL << (*burst_exp - 7)); 95 } else { 96 *burst_exp = MAX_BURST_EXPONENT; 97 *burst_mantissa = MAX_BURST_MANTISSA; 98 } 99 } 100 101 static void otx2_get_egress_rate_cfg(u32 maxrate, u32 *exp, 102 u32 *mantissa, u32 *div_exp) 103 { 104 unsigned int tmp; 105 106 /* Rate calculation by hardware 107 * 108 * PIR_ADD = ((256 + mantissa) << exp) / 256 109 * rate = (2 * PIR_ADD) / ( 1 << div_exp) 110 * The resultant rate is in Mbps. 111 */ 112 113 /* 2Mbps to 100Gbps can be expressed with div_exp = 0. 114 * Setting this to '0' will ease the calculation of 115 * exponent and mantissa. 116 */ 117 *div_exp = 0; 118 119 if (maxrate) { 120 *exp = ilog2(maxrate) ? ilog2(maxrate) - 1 : 0; 121 tmp = maxrate - rounddown_pow_of_two(maxrate); 122 if (maxrate < MAX_RATE_MANTISSA) 123 *mantissa = tmp * 2; 124 else 125 *mantissa = tmp / (1ULL << (*exp - 7)); 126 } else { 127 /* Instead of disabling rate limiting, set all values to max */ 128 *exp = MAX_RATE_EXPONENT; 129 *mantissa = MAX_RATE_MANTISSA; 130 } 131 } 132 133 static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, u32 burst, u32 maxrate) 134 { 135 struct otx2_hw *hw = &nic->hw; 136 struct nix_txschq_config *req; 137 u32 burst_exp, burst_mantissa; 138 u32 exp, mantissa, div_exp; 139 int txschq, err; 140 141 /* All SQs share the same TL4, so pick the first scheduler */ 142 txschq = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; 143 144 /* Get exponent and mantissa values from the desired rate */ 145 otx2_get_egress_burst_cfg(burst, &burst_exp, &burst_mantissa); 146 otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp); 147 148 mutex_lock(&nic->mbox.lock); 149 req = otx2_mbox_alloc_msg_nix_txschq_cfg(&nic->mbox); 150 if (!req) { 151 mutex_unlock(&nic->mbox.lock); 152 return -ENOMEM; 153 } 154 155 req->lvl = NIX_TXSCH_LVL_TL4; 156 req->num_regs = 1; 157 req->reg[0] = NIX_AF_TL4X_PIR(txschq); 158 req->regval[0] = FIELD_PREP(TLX_BURST_EXPONENT, burst_exp) | 159 FIELD_PREP(TLX_BURST_MANTISSA, burst_mantissa) | 160 FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | 161 FIELD_PREP(TLX_RATE_EXPONENT, exp) | 162 FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); 163 164 err = otx2_sync_mbox_msg(&nic->mbox); 165 mutex_unlock(&nic->mbox.lock); 166 return err; 167 } 168 169 static int otx2_tc_validate_flow(struct otx2_nic *nic, 170 struct flow_action *actions, 171 struct netlink_ext_ack *extack) 172 { 173 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 174 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 175 return -EINVAL; 176 } 177 178 if (!flow_action_has_entries(actions)) { 179 NL_SET_ERR_MSG_MOD(extack, "MATCHALL offload called with no action"); 180 return -EINVAL; 181 } 182 183 if (!flow_offload_has_one_action(actions)) { 184 NL_SET_ERR_MSG_MOD(extack, 185 "Egress MATCHALL offload supports only 1 policing action"); 186 return -EINVAL; 187 } 188 return 0; 189 } 190 191 static int otx2_tc_egress_matchall_install(struct otx2_nic *nic, 192 struct tc_cls_matchall_offload *cls) 193 { 194 struct netlink_ext_ack *extack = cls->common.extack; 195 struct flow_action *actions = &cls->rule->action; 196 struct flow_action_entry *entry; 197 u32 rate; 198 int err; 199 200 err = otx2_tc_validate_flow(nic, actions, extack); 201 if (err) 202 return err; 203 204 if (nic->flags & OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED) { 205 NL_SET_ERR_MSG_MOD(extack, 206 "Only one Egress MATCHALL ratelimiter can be offloaded"); 207 return -ENOMEM; 208 } 209 210 entry = &cls->rule->action.entries[0]; 211 switch (entry->id) { 212 case FLOW_ACTION_POLICE: 213 if (entry->police.rate_pkt_ps) { 214 NL_SET_ERR_MSG_MOD(extack, "QoS offload not support packets per second"); 215 return -EOPNOTSUPP; 216 } 217 /* Convert bytes per second to Mbps */ 218 rate = entry->police.rate_bytes_ps * 8; 219 rate = max_t(u32, rate / 1000000, 1); 220 err = otx2_set_matchall_egress_rate(nic, entry->police.burst, rate); 221 if (err) 222 return err; 223 nic->flags |= OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED; 224 break; 225 default: 226 NL_SET_ERR_MSG_MOD(extack, 227 "Only police action is supported with Egress MATCHALL offload"); 228 return -EOPNOTSUPP; 229 } 230 231 return 0; 232 } 233 234 static int otx2_tc_egress_matchall_delete(struct otx2_nic *nic, 235 struct tc_cls_matchall_offload *cls) 236 { 237 struct netlink_ext_ack *extack = cls->common.extack; 238 int err; 239 240 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 241 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 242 return -EINVAL; 243 } 244 245 err = otx2_set_matchall_egress_rate(nic, 0, 0); 246 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED; 247 return err; 248 } 249 250 static int otx2_tc_act_set_police(struct otx2_nic *nic, 251 struct otx2_tc_flow *node, 252 struct flow_cls_offload *f, 253 u64 rate, u32 burst, u32 mark, 254 struct npc_install_flow_req *req, bool pps) 255 { 256 struct netlink_ext_ack *extack = f->common.extack; 257 struct otx2_hw *hw = &nic->hw; 258 int rq_idx, rc; 259 260 rq_idx = find_first_zero_bit(&nic->rq_bmap, hw->rx_queues); 261 if (rq_idx >= hw->rx_queues) { 262 NL_SET_ERR_MSG_MOD(extack, "Police action rules exceeded"); 263 return -EINVAL; 264 } 265 266 mutex_lock(&nic->mbox.lock); 267 268 rc = cn10k_alloc_leaf_profile(nic, &node->leaf_profile); 269 if (rc) { 270 mutex_unlock(&nic->mbox.lock); 271 return rc; 272 } 273 274 rc = cn10k_set_ipolicer_rate(nic, node->leaf_profile, burst, rate, pps); 275 if (rc) 276 goto free_leaf; 277 278 rc = cn10k_map_unmap_rq_policer(nic, rq_idx, node->leaf_profile, true); 279 if (rc) 280 goto free_leaf; 281 282 mutex_unlock(&nic->mbox.lock); 283 284 req->match_id = mark & 0xFFFFULL; 285 req->index = rq_idx; 286 req->op = NIX_RX_ACTIONOP_UCAST; 287 set_bit(rq_idx, &nic->rq_bmap); 288 node->is_act_police = true; 289 node->rq = rq_idx; 290 291 return 0; 292 293 free_leaf: 294 if (cn10k_free_leaf_profile(nic, node->leaf_profile)) 295 netdev_err(nic->netdev, 296 "Unable to free leaf bandwidth profile(%d)\n", 297 node->leaf_profile); 298 mutex_unlock(&nic->mbox.lock); 299 return rc; 300 } 301 302 static int otx2_tc_parse_actions(struct otx2_nic *nic, 303 struct flow_action *flow_action, 304 struct npc_install_flow_req *req, 305 struct flow_cls_offload *f, 306 struct otx2_tc_flow *node) 307 { 308 struct netlink_ext_ack *extack = f->common.extack; 309 struct flow_action_entry *act; 310 struct net_device *target; 311 struct otx2_nic *priv; 312 u32 burst, mark = 0; 313 u8 nr_police = 0; 314 bool pps = false; 315 u64 rate; 316 int i; 317 318 if (!flow_action_has_entries(flow_action)) { 319 NL_SET_ERR_MSG_MOD(extack, "no tc actions specified"); 320 return -EINVAL; 321 } 322 323 flow_action_for_each(i, act, flow_action) { 324 switch (act->id) { 325 case FLOW_ACTION_DROP: 326 req->op = NIX_RX_ACTIONOP_DROP; 327 return 0; 328 case FLOW_ACTION_ACCEPT: 329 req->op = NIX_RX_ACTION_DEFAULT; 330 return 0; 331 case FLOW_ACTION_REDIRECT_INGRESS: 332 target = act->dev; 333 priv = netdev_priv(target); 334 /* npc_install_flow_req doesn't support passing a target pcifunc */ 335 if (rvu_get_pf(nic->pcifunc) != rvu_get_pf(priv->pcifunc)) { 336 NL_SET_ERR_MSG_MOD(extack, 337 "can't redirect to other pf/vf"); 338 return -EOPNOTSUPP; 339 } 340 req->vf = priv->pcifunc & RVU_PFVF_FUNC_MASK; 341 req->op = NIX_RX_ACTION_DEFAULT; 342 return 0; 343 case FLOW_ACTION_VLAN_POP: 344 req->vtag0_valid = true; 345 /* use RX_VTAG_TYPE7 which is initialized to strip vlan tag */ 346 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7; 347 break; 348 case FLOW_ACTION_POLICE: 349 /* Ingress ratelimiting is not supported on OcteonTx2 */ 350 if (is_dev_otx2(nic->pdev)) { 351 NL_SET_ERR_MSG_MOD(extack, 352 "Ingress policing not supported on this platform"); 353 return -EOPNOTSUPP; 354 } 355 356 if (act->police.rate_bytes_ps > 0) { 357 rate = act->police.rate_bytes_ps * 8; 358 burst = act->police.burst; 359 } else if (act->police.rate_pkt_ps > 0) { 360 /* The algorithm used to calculate rate 361 * mantissa, exponent values for a given token 362 * rate (token can be byte or packet) requires 363 * token rate to be mutiplied by 8. 364 */ 365 rate = act->police.rate_pkt_ps * 8; 366 burst = act->police.burst_pkt; 367 pps = true; 368 } 369 nr_police++; 370 break; 371 case FLOW_ACTION_MARK: 372 mark = act->mark; 373 break; 374 default: 375 return -EOPNOTSUPP; 376 } 377 } 378 379 if (nr_police > 1) { 380 NL_SET_ERR_MSG_MOD(extack, 381 "rate limit police offload requires a single action"); 382 return -EOPNOTSUPP; 383 } 384 385 if (nr_police) 386 return otx2_tc_act_set_police(nic, node, f, rate, burst, 387 mark, req, pps); 388 389 return 0; 390 } 391 392 static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, 393 struct flow_cls_offload *f, 394 struct npc_install_flow_req *req) 395 { 396 struct netlink_ext_ack *extack = f->common.extack; 397 struct flow_msg *flow_spec = &req->packet; 398 struct flow_msg *flow_mask = &req->mask; 399 struct flow_dissector *dissector; 400 struct flow_rule *rule; 401 u8 ip_proto = 0; 402 403 rule = flow_cls_offload_flow_rule(f); 404 dissector = rule->match.dissector; 405 406 if ((dissector->used_keys & 407 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 408 BIT(FLOW_DISSECTOR_KEY_BASIC) | 409 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 410 BIT(FLOW_DISSECTOR_KEY_VLAN) | 411 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 412 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 413 BIT(FLOW_DISSECTOR_KEY_PORTS) | 414 BIT(FLOW_DISSECTOR_KEY_IP)))) { 415 netdev_info(nic->netdev, "unsupported flow used key 0x%x", 416 dissector->used_keys); 417 return -EOPNOTSUPP; 418 } 419 420 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 421 struct flow_match_basic match; 422 423 flow_rule_match_basic(rule, &match); 424 425 /* All EtherTypes can be matched, no hw limitation */ 426 flow_spec->etype = match.key->n_proto; 427 flow_mask->etype = match.mask->n_proto; 428 req->features |= BIT_ULL(NPC_ETYPE); 429 430 if (match.mask->ip_proto && 431 (match.key->ip_proto != IPPROTO_TCP && 432 match.key->ip_proto != IPPROTO_UDP && 433 match.key->ip_proto != IPPROTO_SCTP && 434 match.key->ip_proto != IPPROTO_ICMP && 435 match.key->ip_proto != IPPROTO_ICMPV6)) { 436 netdev_info(nic->netdev, 437 "ip_proto=0x%x not supported\n", 438 match.key->ip_proto); 439 return -EOPNOTSUPP; 440 } 441 if (match.mask->ip_proto) 442 ip_proto = match.key->ip_proto; 443 444 if (ip_proto == IPPROTO_UDP) 445 req->features |= BIT_ULL(NPC_IPPROTO_UDP); 446 else if (ip_proto == IPPROTO_TCP) 447 req->features |= BIT_ULL(NPC_IPPROTO_TCP); 448 else if (ip_proto == IPPROTO_SCTP) 449 req->features |= BIT_ULL(NPC_IPPROTO_SCTP); 450 else if (ip_proto == IPPROTO_ICMP) 451 req->features |= BIT_ULL(NPC_IPPROTO_ICMP); 452 else if (ip_proto == IPPROTO_ICMPV6) 453 req->features |= BIT_ULL(NPC_IPPROTO_ICMP6); 454 } 455 456 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 457 struct flow_match_eth_addrs match; 458 459 flow_rule_match_eth_addrs(rule, &match); 460 if (!is_zero_ether_addr(match.mask->src)) { 461 NL_SET_ERR_MSG_MOD(extack, "src mac match not supported"); 462 return -EOPNOTSUPP; 463 } 464 465 if (!is_zero_ether_addr(match.mask->dst)) { 466 ether_addr_copy(flow_spec->dmac, (u8 *)&match.key->dst); 467 ether_addr_copy(flow_mask->dmac, 468 (u8 *)&match.mask->dst); 469 req->features |= BIT_ULL(NPC_DMAC); 470 } 471 } 472 473 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { 474 struct flow_match_ip match; 475 476 flow_rule_match_ip(rule, &match); 477 if ((ntohs(flow_spec->etype) != ETH_P_IP) && 478 match.mask->tos) { 479 NL_SET_ERR_MSG_MOD(extack, "tos not supported"); 480 return -EOPNOTSUPP; 481 } 482 if (match.mask->ttl) { 483 NL_SET_ERR_MSG_MOD(extack, "ttl not supported"); 484 return -EOPNOTSUPP; 485 } 486 flow_spec->tos = match.key->tos; 487 flow_mask->tos = match.mask->tos; 488 req->features |= BIT_ULL(NPC_TOS); 489 } 490 491 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 492 struct flow_match_vlan match; 493 u16 vlan_tci, vlan_tci_mask; 494 495 flow_rule_match_vlan(rule, &match); 496 497 if (ntohs(match.key->vlan_tpid) != ETH_P_8021Q) { 498 netdev_err(nic->netdev, "vlan tpid 0x%x not supported\n", 499 ntohs(match.key->vlan_tpid)); 500 return -EOPNOTSUPP; 501 } 502 503 if (match.mask->vlan_id || 504 match.mask->vlan_dei || 505 match.mask->vlan_priority) { 506 vlan_tci = match.key->vlan_id | 507 match.key->vlan_dei << 12 | 508 match.key->vlan_priority << 13; 509 510 vlan_tci_mask = match.mask->vlan_id | 511 match.mask->vlan_dei << 12 | 512 match.mask->vlan_priority << 13; 513 514 flow_spec->vlan_tci = htons(vlan_tci); 515 flow_mask->vlan_tci = htons(vlan_tci_mask); 516 req->features |= BIT_ULL(NPC_OUTER_VID); 517 } 518 } 519 520 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 521 struct flow_match_ipv4_addrs match; 522 523 flow_rule_match_ipv4_addrs(rule, &match); 524 525 flow_spec->ip4dst = match.key->dst; 526 flow_mask->ip4dst = match.mask->dst; 527 req->features |= BIT_ULL(NPC_DIP_IPV4); 528 529 flow_spec->ip4src = match.key->src; 530 flow_mask->ip4src = match.mask->src; 531 req->features |= BIT_ULL(NPC_SIP_IPV4); 532 } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 533 struct flow_match_ipv6_addrs match; 534 535 flow_rule_match_ipv6_addrs(rule, &match); 536 537 if (ipv6_addr_loopback(&match.key->dst) || 538 ipv6_addr_loopback(&match.key->src)) { 539 NL_SET_ERR_MSG_MOD(extack, 540 "Flow matching IPv6 loopback addr not supported"); 541 return -EOPNOTSUPP; 542 } 543 544 if (!ipv6_addr_any(&match.mask->dst)) { 545 memcpy(&flow_spec->ip6dst, 546 (struct in6_addr *)&match.key->dst, 547 sizeof(flow_spec->ip6dst)); 548 memcpy(&flow_mask->ip6dst, 549 (struct in6_addr *)&match.mask->dst, 550 sizeof(flow_spec->ip6dst)); 551 req->features |= BIT_ULL(NPC_DIP_IPV6); 552 } 553 554 if (!ipv6_addr_any(&match.mask->src)) { 555 memcpy(&flow_spec->ip6src, 556 (struct in6_addr *)&match.key->src, 557 sizeof(flow_spec->ip6src)); 558 memcpy(&flow_mask->ip6src, 559 (struct in6_addr *)&match.mask->src, 560 sizeof(flow_spec->ip6src)); 561 req->features |= BIT_ULL(NPC_SIP_IPV6); 562 } 563 } 564 565 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 566 struct flow_match_ports match; 567 568 flow_rule_match_ports(rule, &match); 569 570 flow_spec->dport = match.key->dst; 571 flow_mask->dport = match.mask->dst; 572 if (ip_proto == IPPROTO_UDP) 573 req->features |= BIT_ULL(NPC_DPORT_UDP); 574 else if (ip_proto == IPPROTO_TCP) 575 req->features |= BIT_ULL(NPC_DPORT_TCP); 576 else if (ip_proto == IPPROTO_SCTP) 577 req->features |= BIT_ULL(NPC_DPORT_SCTP); 578 579 flow_spec->sport = match.key->src; 580 flow_mask->sport = match.mask->src; 581 if (ip_proto == IPPROTO_UDP) 582 req->features |= BIT_ULL(NPC_SPORT_UDP); 583 else if (ip_proto == IPPROTO_TCP) 584 req->features |= BIT_ULL(NPC_SPORT_TCP); 585 else if (ip_proto == IPPROTO_SCTP) 586 req->features |= BIT_ULL(NPC_SPORT_SCTP); 587 } 588 589 return otx2_tc_parse_actions(nic, &rule->action, req, f, node); 590 } 591 592 static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry) 593 { 594 struct npc_delete_flow_req *req; 595 int err; 596 597 mutex_lock(&nic->mbox.lock); 598 req = otx2_mbox_alloc_msg_npc_delete_flow(&nic->mbox); 599 if (!req) { 600 mutex_unlock(&nic->mbox.lock); 601 return -ENOMEM; 602 } 603 604 req->entry = entry; 605 606 /* Send message to AF */ 607 err = otx2_sync_mbox_msg(&nic->mbox); 608 if (err) { 609 netdev_err(nic->netdev, "Failed to delete MCAM flow entry %d\n", 610 entry); 611 mutex_unlock(&nic->mbox.lock); 612 return -EFAULT; 613 } 614 mutex_unlock(&nic->mbox.lock); 615 616 return 0; 617 } 618 619 static int otx2_tc_del_flow(struct otx2_nic *nic, 620 struct flow_cls_offload *tc_flow_cmd) 621 { 622 struct otx2_flow_config *flow_cfg = nic->flow_cfg; 623 struct otx2_tc_info *tc_info = &nic->tc_info; 624 struct otx2_tc_flow *flow_node; 625 int err; 626 627 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 628 &tc_flow_cmd->cookie, 629 tc_info->flow_ht_params); 630 if (!flow_node) { 631 netdev_err(nic->netdev, "tc flow not found for cookie 0x%lx\n", 632 tc_flow_cmd->cookie); 633 return -EINVAL; 634 } 635 636 if (flow_node->is_act_police) { 637 mutex_lock(&nic->mbox.lock); 638 639 err = cn10k_map_unmap_rq_policer(nic, flow_node->rq, 640 flow_node->leaf_profile, false); 641 if (err) 642 netdev_err(nic->netdev, 643 "Unmapping RQ %d & profile %d failed\n", 644 flow_node->rq, flow_node->leaf_profile); 645 646 err = cn10k_free_leaf_profile(nic, flow_node->leaf_profile); 647 if (err) 648 netdev_err(nic->netdev, 649 "Unable to free leaf bandwidth profile(%d)\n", 650 flow_node->leaf_profile); 651 652 __clear_bit(flow_node->rq, &nic->rq_bmap); 653 654 mutex_unlock(&nic->mbox.lock); 655 } 656 657 otx2_del_mcam_flow_entry(nic, flow_node->entry); 658 659 WARN_ON(rhashtable_remove_fast(&nic->tc_info.flow_table, 660 &flow_node->node, 661 nic->tc_info.flow_ht_params)); 662 kfree_rcu(flow_node, rcu); 663 664 clear_bit(flow_node->bitpos, tc_info->tc_entries_bitmap); 665 flow_cfg->nr_flows--; 666 667 return 0; 668 } 669 670 static int otx2_tc_add_flow(struct otx2_nic *nic, 671 struct flow_cls_offload *tc_flow_cmd) 672 { 673 struct netlink_ext_ack *extack = tc_flow_cmd->common.extack; 674 struct otx2_flow_config *flow_cfg = nic->flow_cfg; 675 struct otx2_tc_info *tc_info = &nic->tc_info; 676 struct otx2_tc_flow *new_node, *old_node; 677 struct npc_install_flow_req *req, dummy; 678 int rc, err; 679 680 if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)) 681 return -ENOMEM; 682 683 if (bitmap_full(tc_info->tc_entries_bitmap, flow_cfg->max_flows)) { 684 NL_SET_ERR_MSG_MOD(extack, 685 "Free MCAM entry not available to add the flow"); 686 return -ENOMEM; 687 } 688 689 /* allocate memory for the new flow and it's node */ 690 new_node = kzalloc(sizeof(*new_node), GFP_KERNEL); 691 if (!new_node) 692 return -ENOMEM; 693 spin_lock_init(&new_node->lock); 694 new_node->cookie = tc_flow_cmd->cookie; 695 696 memset(&dummy, 0, sizeof(struct npc_install_flow_req)); 697 698 rc = otx2_tc_prepare_flow(nic, new_node, tc_flow_cmd, &dummy); 699 if (rc) { 700 kfree_rcu(new_node, rcu); 701 return rc; 702 } 703 704 /* If a flow exists with the same cookie, delete it */ 705 old_node = rhashtable_lookup_fast(&tc_info->flow_table, 706 &tc_flow_cmd->cookie, 707 tc_info->flow_ht_params); 708 if (old_node) 709 otx2_tc_del_flow(nic, tc_flow_cmd); 710 711 mutex_lock(&nic->mbox.lock); 712 req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); 713 if (!req) { 714 mutex_unlock(&nic->mbox.lock); 715 rc = -ENOMEM; 716 goto free_leaf; 717 } 718 719 memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr)); 720 memcpy(req, &dummy, sizeof(struct npc_install_flow_req)); 721 722 new_node->bitpos = find_first_zero_bit(tc_info->tc_entries_bitmap, 723 flow_cfg->max_flows); 724 req->channel = nic->hw.rx_chan_base; 725 req->entry = flow_cfg->flow_ent[flow_cfg->max_flows - new_node->bitpos - 1]; 726 req->intf = NIX_INTF_RX; 727 req->set_cntr = 1; 728 new_node->entry = req->entry; 729 730 /* Send message to AF */ 731 rc = otx2_sync_mbox_msg(&nic->mbox); 732 if (rc) { 733 NL_SET_ERR_MSG_MOD(extack, "Failed to install MCAM flow entry"); 734 mutex_unlock(&nic->mbox.lock); 735 kfree_rcu(new_node, rcu); 736 goto free_leaf; 737 } 738 mutex_unlock(&nic->mbox.lock); 739 740 /* add new flow to flow-table */ 741 rc = rhashtable_insert_fast(&nic->tc_info.flow_table, &new_node->node, 742 nic->tc_info.flow_ht_params); 743 if (rc) { 744 otx2_del_mcam_flow_entry(nic, req->entry); 745 kfree_rcu(new_node, rcu); 746 goto free_leaf; 747 } 748 749 set_bit(new_node->bitpos, tc_info->tc_entries_bitmap); 750 flow_cfg->nr_flows++; 751 752 return 0; 753 754 free_leaf: 755 if (new_node->is_act_police) { 756 mutex_lock(&nic->mbox.lock); 757 758 err = cn10k_map_unmap_rq_policer(nic, new_node->rq, 759 new_node->leaf_profile, false); 760 if (err) 761 netdev_err(nic->netdev, 762 "Unmapping RQ %d & profile %d failed\n", 763 new_node->rq, new_node->leaf_profile); 764 err = cn10k_free_leaf_profile(nic, new_node->leaf_profile); 765 if (err) 766 netdev_err(nic->netdev, 767 "Unable to free leaf bandwidth profile(%d)\n", 768 new_node->leaf_profile); 769 770 __clear_bit(new_node->rq, &nic->rq_bmap); 771 772 mutex_unlock(&nic->mbox.lock); 773 } 774 775 return rc; 776 } 777 778 static int otx2_tc_get_flow_stats(struct otx2_nic *nic, 779 struct flow_cls_offload *tc_flow_cmd) 780 { 781 struct otx2_tc_info *tc_info = &nic->tc_info; 782 struct npc_mcam_get_stats_req *req; 783 struct npc_mcam_get_stats_rsp *rsp; 784 struct otx2_tc_flow_stats *stats; 785 struct otx2_tc_flow *flow_node; 786 int err; 787 788 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 789 &tc_flow_cmd->cookie, 790 tc_info->flow_ht_params); 791 if (!flow_node) { 792 netdev_info(nic->netdev, "tc flow not found for cookie %lx", 793 tc_flow_cmd->cookie); 794 return -EINVAL; 795 } 796 797 mutex_lock(&nic->mbox.lock); 798 799 req = otx2_mbox_alloc_msg_npc_mcam_entry_stats(&nic->mbox); 800 if (!req) { 801 mutex_unlock(&nic->mbox.lock); 802 return -ENOMEM; 803 } 804 805 req->entry = flow_node->entry; 806 807 err = otx2_sync_mbox_msg(&nic->mbox); 808 if (err) { 809 netdev_err(nic->netdev, "Failed to get stats for MCAM flow entry %d\n", 810 req->entry); 811 mutex_unlock(&nic->mbox.lock); 812 return -EFAULT; 813 } 814 815 rsp = (struct npc_mcam_get_stats_rsp *)otx2_mbox_get_rsp 816 (&nic->mbox.mbox, 0, &req->hdr); 817 if (IS_ERR(rsp)) { 818 mutex_unlock(&nic->mbox.lock); 819 return PTR_ERR(rsp); 820 } 821 822 mutex_unlock(&nic->mbox.lock); 823 824 if (!rsp->stat_ena) 825 return -EINVAL; 826 827 stats = &flow_node->stats; 828 829 spin_lock(&flow_node->lock); 830 flow_stats_update(&tc_flow_cmd->stats, 0x0, rsp->stat - stats->pkts, 0x0, 0x0, 831 FLOW_ACTION_HW_STATS_IMMEDIATE); 832 stats->pkts = rsp->stat; 833 spin_unlock(&flow_node->lock); 834 835 return 0; 836 } 837 838 static int otx2_setup_tc_cls_flower(struct otx2_nic *nic, 839 struct flow_cls_offload *cls_flower) 840 { 841 switch (cls_flower->command) { 842 case FLOW_CLS_REPLACE: 843 return otx2_tc_add_flow(nic, cls_flower); 844 case FLOW_CLS_DESTROY: 845 return otx2_tc_del_flow(nic, cls_flower); 846 case FLOW_CLS_STATS: 847 return otx2_tc_get_flow_stats(nic, cls_flower); 848 default: 849 return -EOPNOTSUPP; 850 } 851 } 852 853 static int otx2_tc_ingress_matchall_install(struct otx2_nic *nic, 854 struct tc_cls_matchall_offload *cls) 855 { 856 struct netlink_ext_ack *extack = cls->common.extack; 857 struct flow_action *actions = &cls->rule->action; 858 struct flow_action_entry *entry; 859 u64 rate; 860 int err; 861 862 err = otx2_tc_validate_flow(nic, actions, extack); 863 if (err) 864 return err; 865 866 if (nic->flags & OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED) { 867 NL_SET_ERR_MSG_MOD(extack, 868 "Only one ingress MATCHALL ratelimitter can be offloaded"); 869 return -ENOMEM; 870 } 871 872 entry = &cls->rule->action.entries[0]; 873 switch (entry->id) { 874 case FLOW_ACTION_POLICE: 875 /* Ingress ratelimiting is not supported on OcteonTx2 */ 876 if (is_dev_otx2(nic->pdev)) { 877 NL_SET_ERR_MSG_MOD(extack, 878 "Ingress policing not supported on this platform"); 879 return -EOPNOTSUPP; 880 } 881 882 err = cn10k_alloc_matchall_ipolicer(nic); 883 if (err) 884 return err; 885 886 /* Convert to bits per second */ 887 rate = entry->police.rate_bytes_ps * 8; 888 err = cn10k_set_matchall_ipolicer_rate(nic, entry->police.burst, rate); 889 if (err) 890 return err; 891 nic->flags |= OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED; 892 break; 893 default: 894 NL_SET_ERR_MSG_MOD(extack, 895 "Only police action supported with Ingress MATCHALL offload"); 896 return -EOPNOTSUPP; 897 } 898 899 return 0; 900 } 901 902 static int otx2_tc_ingress_matchall_delete(struct otx2_nic *nic, 903 struct tc_cls_matchall_offload *cls) 904 { 905 struct netlink_ext_ack *extack = cls->common.extack; 906 int err; 907 908 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 909 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 910 return -EINVAL; 911 } 912 913 err = cn10k_free_matchall_ipolicer(nic); 914 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED; 915 return err; 916 } 917 918 static int otx2_setup_tc_ingress_matchall(struct otx2_nic *nic, 919 struct tc_cls_matchall_offload *cls_matchall) 920 { 921 switch (cls_matchall->command) { 922 case TC_CLSMATCHALL_REPLACE: 923 return otx2_tc_ingress_matchall_install(nic, cls_matchall); 924 case TC_CLSMATCHALL_DESTROY: 925 return otx2_tc_ingress_matchall_delete(nic, cls_matchall); 926 case TC_CLSMATCHALL_STATS: 927 default: 928 break; 929 } 930 931 return -EOPNOTSUPP; 932 } 933 934 static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type, 935 void *type_data, void *cb_priv) 936 { 937 struct otx2_nic *nic = cb_priv; 938 939 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data)) 940 return -EOPNOTSUPP; 941 942 switch (type) { 943 case TC_SETUP_CLSFLOWER: 944 return otx2_setup_tc_cls_flower(nic, type_data); 945 case TC_SETUP_CLSMATCHALL: 946 return otx2_setup_tc_ingress_matchall(nic, type_data); 947 default: 948 break; 949 } 950 951 return -EOPNOTSUPP; 952 } 953 954 static int otx2_setup_tc_egress_matchall(struct otx2_nic *nic, 955 struct tc_cls_matchall_offload *cls_matchall) 956 { 957 switch (cls_matchall->command) { 958 case TC_CLSMATCHALL_REPLACE: 959 return otx2_tc_egress_matchall_install(nic, cls_matchall); 960 case TC_CLSMATCHALL_DESTROY: 961 return otx2_tc_egress_matchall_delete(nic, cls_matchall); 962 case TC_CLSMATCHALL_STATS: 963 default: 964 break; 965 } 966 967 return -EOPNOTSUPP; 968 } 969 970 static int otx2_setup_tc_block_egress_cb(enum tc_setup_type type, 971 void *type_data, void *cb_priv) 972 { 973 struct otx2_nic *nic = cb_priv; 974 975 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data)) 976 return -EOPNOTSUPP; 977 978 switch (type) { 979 case TC_SETUP_CLSMATCHALL: 980 return otx2_setup_tc_egress_matchall(nic, type_data); 981 default: 982 break; 983 } 984 985 return -EOPNOTSUPP; 986 } 987 988 static LIST_HEAD(otx2_block_cb_list); 989 990 static int otx2_setup_tc_block(struct net_device *netdev, 991 struct flow_block_offload *f) 992 { 993 struct otx2_nic *nic = netdev_priv(netdev); 994 flow_setup_cb_t *cb; 995 bool ingress; 996 997 if (f->block_shared) 998 return -EOPNOTSUPP; 999 1000 if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) { 1001 cb = otx2_setup_tc_block_ingress_cb; 1002 ingress = true; 1003 } else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) { 1004 cb = otx2_setup_tc_block_egress_cb; 1005 ingress = false; 1006 } else { 1007 return -EOPNOTSUPP; 1008 } 1009 1010 return flow_block_cb_setup_simple(f, &otx2_block_cb_list, cb, 1011 nic, nic, ingress); 1012 } 1013 1014 int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type, 1015 void *type_data) 1016 { 1017 switch (type) { 1018 case TC_SETUP_BLOCK: 1019 return otx2_setup_tc_block(netdev, type_data); 1020 default: 1021 return -EOPNOTSUPP; 1022 } 1023 } 1024 1025 static const struct rhashtable_params tc_flow_ht_params = { 1026 .head_offset = offsetof(struct otx2_tc_flow, node), 1027 .key_offset = offsetof(struct otx2_tc_flow, cookie), 1028 .key_len = sizeof(((struct otx2_tc_flow *)0)->cookie), 1029 .automatic_shrinking = true, 1030 }; 1031 1032 int otx2_init_tc(struct otx2_nic *nic) 1033 { 1034 struct otx2_tc_info *tc = &nic->tc_info; 1035 int err; 1036 1037 /* Exclude receive queue 0 being used for police action */ 1038 set_bit(0, &nic->rq_bmap); 1039 1040 if (!nic->flow_cfg) { 1041 netdev_err(nic->netdev, 1042 "Can't init TC, nic->flow_cfg is not setup\n"); 1043 return -EINVAL; 1044 } 1045 1046 err = otx2_tc_alloc_ent_bitmap(nic); 1047 if (err) 1048 return err; 1049 1050 tc->flow_ht_params = tc_flow_ht_params; 1051 return rhashtable_init(&tc->flow_table, &tc->flow_ht_params); 1052 } 1053 1054 void otx2_shutdown_tc(struct otx2_nic *nic) 1055 { 1056 struct otx2_tc_info *tc = &nic->tc_info; 1057 1058 kfree(tc->tc_entries_bitmap); 1059 rhashtable_destroy(&tc->flow_table); 1060 } 1061