1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2021 Marvell. 5 * 6 */ 7 8 #include <linux/netdevice.h> 9 #include <linux/etherdevice.h> 10 #include <linux/inetdevice.h> 11 #include <linux/rhashtable.h> 12 #include <linux/bitfield.h> 13 #include <net/flow_dissector.h> 14 #include <net/pkt_cls.h> 15 #include <net/tc_act/tc_gact.h> 16 #include <net/tc_act/tc_mirred.h> 17 #include <net/tc_act/tc_vlan.h> 18 #include <net/ipv6.h> 19 20 #include "cn10k.h" 21 #include "otx2_common.h" 22 23 /* Egress rate limiting definitions */ 24 #define MAX_BURST_EXPONENT 0x0FULL 25 #define MAX_BURST_MANTISSA 0xFFULL 26 #define MAX_BURST_SIZE 130816ULL 27 #define MAX_RATE_DIVIDER_EXPONENT 12ULL 28 #define MAX_RATE_EXPONENT 0x0FULL 29 #define MAX_RATE_MANTISSA 0xFFULL 30 31 /* Bitfields in NIX_TLX_PIR register */ 32 #define TLX_RATE_MANTISSA GENMASK_ULL(8, 1) 33 #define TLX_RATE_EXPONENT GENMASK_ULL(12, 9) 34 #define TLX_RATE_DIVIDER_EXPONENT GENMASK_ULL(16, 13) 35 #define TLX_BURST_MANTISSA GENMASK_ULL(36, 29) 36 #define TLX_BURST_EXPONENT GENMASK_ULL(40, 37) 37 38 struct otx2_tc_flow_stats { 39 u64 bytes; 40 u64 pkts; 41 u64 used; 42 }; 43 44 struct otx2_tc_flow { 45 struct rhash_head node; 46 unsigned long cookie; 47 unsigned int bitpos; 48 struct rcu_head rcu; 49 struct otx2_tc_flow_stats stats; 50 spinlock_t lock; /* lock for stats */ 51 u16 rq; 52 u16 entry; 53 u16 leaf_profile; 54 bool is_act_police; 55 }; 56 57 int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic) 58 { 59 struct otx2_tc_info *tc = &nic->tc_info; 60 61 if (!nic->flow_cfg->max_flows) 62 return 0; 63 64 /* Max flows changed, free the existing bitmap */ 65 kfree(tc->tc_entries_bitmap); 66 67 tc->tc_entries_bitmap = 68 kcalloc(BITS_TO_LONGS(nic->flow_cfg->max_flows), 69 sizeof(long), GFP_KERNEL); 70 if (!tc->tc_entries_bitmap) { 71 netdev_err(nic->netdev, 72 "Unable to alloc TC flow entries bitmap\n"); 73 return -ENOMEM; 74 } 75 76 return 0; 77 } 78 EXPORT_SYMBOL(otx2_tc_alloc_ent_bitmap); 79 80 static void otx2_get_egress_burst_cfg(u32 burst, u32 *burst_exp, 81 u32 *burst_mantissa) 82 { 83 unsigned int tmp; 84 85 /* Burst is calculated as 86 * ((256 + BURST_MANTISSA) << (1 + BURST_EXPONENT)) / 256 87 * Max supported burst size is 130,816 bytes. 88 */ 89 burst = min_t(u32, burst, MAX_BURST_SIZE); 90 if (burst) { 91 *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0; 92 tmp = burst - rounddown_pow_of_two(burst); 93 if (burst < MAX_BURST_MANTISSA) 94 *burst_mantissa = tmp * 2; 95 else 96 *burst_mantissa = tmp / (1ULL << (*burst_exp - 7)); 97 } else { 98 *burst_exp = MAX_BURST_EXPONENT; 99 *burst_mantissa = MAX_BURST_MANTISSA; 100 } 101 } 102 103 static void otx2_get_egress_rate_cfg(u32 maxrate, u32 *exp, 104 u32 *mantissa, u32 *div_exp) 105 { 106 unsigned int tmp; 107 108 /* Rate calculation by hardware 109 * 110 * PIR_ADD = ((256 + mantissa) << exp) / 256 111 * rate = (2 * PIR_ADD) / ( 1 << div_exp) 112 * The resultant rate is in Mbps. 113 */ 114 115 /* 2Mbps to 100Gbps can be expressed with div_exp = 0. 116 * Setting this to '0' will ease the calculation of 117 * exponent and mantissa. 118 */ 119 *div_exp = 0; 120 121 if (maxrate) { 122 *exp = ilog2(maxrate) ? ilog2(maxrate) - 1 : 0; 123 tmp = maxrate - rounddown_pow_of_two(maxrate); 124 if (maxrate < MAX_RATE_MANTISSA) 125 *mantissa = tmp * 2; 126 else 127 *mantissa = tmp / (1ULL << (*exp - 7)); 128 } else { 129 /* Instead of disabling rate limiting, set all values to max */ 130 *exp = MAX_RATE_EXPONENT; 131 *mantissa = MAX_RATE_MANTISSA; 132 } 133 } 134 135 static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, u32 burst, u32 maxrate) 136 { 137 struct otx2_hw *hw = &nic->hw; 138 struct nix_txschq_config *req; 139 u32 burst_exp, burst_mantissa; 140 u32 exp, mantissa, div_exp; 141 int txschq, err; 142 143 /* All SQs share the same TL4, so pick the first scheduler */ 144 txschq = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; 145 146 /* Get exponent and mantissa values from the desired rate */ 147 otx2_get_egress_burst_cfg(burst, &burst_exp, &burst_mantissa); 148 otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp); 149 150 mutex_lock(&nic->mbox.lock); 151 req = otx2_mbox_alloc_msg_nix_txschq_cfg(&nic->mbox); 152 if (!req) { 153 mutex_unlock(&nic->mbox.lock); 154 return -ENOMEM; 155 } 156 157 req->lvl = NIX_TXSCH_LVL_TL4; 158 req->num_regs = 1; 159 req->reg[0] = NIX_AF_TL4X_PIR(txschq); 160 req->regval[0] = FIELD_PREP(TLX_BURST_EXPONENT, burst_exp) | 161 FIELD_PREP(TLX_BURST_MANTISSA, burst_mantissa) | 162 FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | 163 FIELD_PREP(TLX_RATE_EXPONENT, exp) | 164 FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); 165 166 err = otx2_sync_mbox_msg(&nic->mbox); 167 mutex_unlock(&nic->mbox.lock); 168 return err; 169 } 170 171 static int otx2_tc_validate_flow(struct otx2_nic *nic, 172 struct flow_action *actions, 173 struct netlink_ext_ack *extack) 174 { 175 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 176 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 177 return -EINVAL; 178 } 179 180 if (!flow_action_has_entries(actions)) { 181 NL_SET_ERR_MSG_MOD(extack, "MATCHALL offload called with no action"); 182 return -EINVAL; 183 } 184 185 if (!flow_offload_has_one_action(actions)) { 186 NL_SET_ERR_MSG_MOD(extack, 187 "Egress MATCHALL offload supports only 1 policing action"); 188 return -EINVAL; 189 } 190 return 0; 191 } 192 193 static int otx2_policer_validate(const struct flow_action *action, 194 const struct flow_action_entry *act, 195 struct netlink_ext_ack *extack) 196 { 197 if (act->police.exceed.act_id != FLOW_ACTION_DROP) { 198 NL_SET_ERR_MSG_MOD(extack, 199 "Offload not supported when exceed action is not drop"); 200 return -EOPNOTSUPP; 201 } 202 203 if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && 204 act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { 205 NL_SET_ERR_MSG_MOD(extack, 206 "Offload not supported when conform action is not pipe or ok"); 207 return -EOPNOTSUPP; 208 } 209 210 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && 211 !flow_action_is_last_entry(action, act)) { 212 NL_SET_ERR_MSG_MOD(extack, 213 "Offload not supported when conform action is ok, but action is not last"); 214 return -EOPNOTSUPP; 215 } 216 217 if (act->police.peakrate_bytes_ps || 218 act->police.avrate || act->police.overhead) { 219 NL_SET_ERR_MSG_MOD(extack, 220 "Offload not supported when peakrate/avrate/overhead is configured"); 221 return -EOPNOTSUPP; 222 } 223 224 return 0; 225 } 226 227 static int otx2_tc_egress_matchall_install(struct otx2_nic *nic, 228 struct tc_cls_matchall_offload *cls) 229 { 230 struct netlink_ext_ack *extack = cls->common.extack; 231 struct flow_action *actions = &cls->rule->action; 232 struct flow_action_entry *entry; 233 u32 rate; 234 int err; 235 236 err = otx2_tc_validate_flow(nic, actions, extack); 237 if (err) 238 return err; 239 240 if (nic->flags & OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED) { 241 NL_SET_ERR_MSG_MOD(extack, 242 "Only one Egress MATCHALL ratelimiter can be offloaded"); 243 return -ENOMEM; 244 } 245 246 entry = &cls->rule->action.entries[0]; 247 switch (entry->id) { 248 case FLOW_ACTION_POLICE: 249 err = otx2_policer_validate(&cls->rule->action, entry, extack); 250 if (err) 251 return err; 252 253 if (entry->police.rate_pkt_ps) { 254 NL_SET_ERR_MSG_MOD(extack, "QoS offload not support packets per second"); 255 return -EOPNOTSUPP; 256 } 257 /* Convert bytes per second to Mbps */ 258 rate = entry->police.rate_bytes_ps * 8; 259 rate = max_t(u32, rate / 1000000, 1); 260 err = otx2_set_matchall_egress_rate(nic, entry->police.burst, rate); 261 if (err) 262 return err; 263 nic->flags |= OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED; 264 break; 265 default: 266 NL_SET_ERR_MSG_MOD(extack, 267 "Only police action is supported with Egress MATCHALL offload"); 268 return -EOPNOTSUPP; 269 } 270 271 return 0; 272 } 273 274 static int otx2_tc_egress_matchall_delete(struct otx2_nic *nic, 275 struct tc_cls_matchall_offload *cls) 276 { 277 struct netlink_ext_ack *extack = cls->common.extack; 278 int err; 279 280 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 281 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 282 return -EINVAL; 283 } 284 285 err = otx2_set_matchall_egress_rate(nic, 0, 0); 286 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED; 287 return err; 288 } 289 290 static int otx2_tc_act_set_police(struct otx2_nic *nic, 291 struct otx2_tc_flow *node, 292 struct flow_cls_offload *f, 293 u64 rate, u32 burst, u32 mark, 294 struct npc_install_flow_req *req, bool pps) 295 { 296 struct netlink_ext_ack *extack = f->common.extack; 297 struct otx2_hw *hw = &nic->hw; 298 int rq_idx, rc; 299 300 rq_idx = find_first_zero_bit(&nic->rq_bmap, hw->rx_queues); 301 if (rq_idx >= hw->rx_queues) { 302 NL_SET_ERR_MSG_MOD(extack, "Police action rules exceeded"); 303 return -EINVAL; 304 } 305 306 mutex_lock(&nic->mbox.lock); 307 308 rc = cn10k_alloc_leaf_profile(nic, &node->leaf_profile); 309 if (rc) { 310 mutex_unlock(&nic->mbox.lock); 311 return rc; 312 } 313 314 rc = cn10k_set_ipolicer_rate(nic, node->leaf_profile, burst, rate, pps); 315 if (rc) 316 goto free_leaf; 317 318 rc = cn10k_map_unmap_rq_policer(nic, rq_idx, node->leaf_profile, true); 319 if (rc) 320 goto free_leaf; 321 322 mutex_unlock(&nic->mbox.lock); 323 324 req->match_id = mark & 0xFFFFULL; 325 req->index = rq_idx; 326 req->op = NIX_RX_ACTIONOP_UCAST; 327 set_bit(rq_idx, &nic->rq_bmap); 328 node->is_act_police = true; 329 node->rq = rq_idx; 330 331 return 0; 332 333 free_leaf: 334 if (cn10k_free_leaf_profile(nic, node->leaf_profile)) 335 netdev_err(nic->netdev, 336 "Unable to free leaf bandwidth profile(%d)\n", 337 node->leaf_profile); 338 mutex_unlock(&nic->mbox.lock); 339 return rc; 340 } 341 342 static int otx2_tc_parse_actions(struct otx2_nic *nic, 343 struct flow_action *flow_action, 344 struct npc_install_flow_req *req, 345 struct flow_cls_offload *f, 346 struct otx2_tc_flow *node) 347 { 348 struct netlink_ext_ack *extack = f->common.extack; 349 struct flow_action_entry *act; 350 struct net_device *target; 351 struct otx2_nic *priv; 352 u32 burst, mark = 0; 353 u8 nr_police = 0; 354 bool pps = false; 355 u64 rate; 356 int err; 357 int i; 358 359 if (!flow_action_has_entries(flow_action)) { 360 NL_SET_ERR_MSG_MOD(extack, "no tc actions specified"); 361 return -EINVAL; 362 } 363 364 flow_action_for_each(i, act, flow_action) { 365 switch (act->id) { 366 case FLOW_ACTION_DROP: 367 req->op = NIX_RX_ACTIONOP_DROP; 368 return 0; 369 case FLOW_ACTION_ACCEPT: 370 req->op = NIX_RX_ACTION_DEFAULT; 371 return 0; 372 case FLOW_ACTION_REDIRECT_INGRESS: 373 target = act->dev; 374 priv = netdev_priv(target); 375 /* npc_install_flow_req doesn't support passing a target pcifunc */ 376 if (rvu_get_pf(nic->pcifunc) != rvu_get_pf(priv->pcifunc)) { 377 NL_SET_ERR_MSG_MOD(extack, 378 "can't redirect to other pf/vf"); 379 return -EOPNOTSUPP; 380 } 381 req->vf = priv->pcifunc & RVU_PFVF_FUNC_MASK; 382 req->op = NIX_RX_ACTION_DEFAULT; 383 return 0; 384 case FLOW_ACTION_VLAN_POP: 385 req->vtag0_valid = true; 386 /* use RX_VTAG_TYPE7 which is initialized to strip vlan tag */ 387 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7; 388 break; 389 case FLOW_ACTION_POLICE: 390 /* Ingress ratelimiting is not supported on OcteonTx2 */ 391 if (is_dev_otx2(nic->pdev)) { 392 NL_SET_ERR_MSG_MOD(extack, 393 "Ingress policing not supported on this platform"); 394 return -EOPNOTSUPP; 395 } 396 397 err = otx2_policer_validate(flow_action, act, extack); 398 if (err) 399 return err; 400 401 if (act->police.rate_bytes_ps > 0) { 402 rate = act->police.rate_bytes_ps * 8; 403 burst = act->police.burst; 404 } else if (act->police.rate_pkt_ps > 0) { 405 /* The algorithm used to calculate rate 406 * mantissa, exponent values for a given token 407 * rate (token can be byte or packet) requires 408 * token rate to be mutiplied by 8. 409 */ 410 rate = act->police.rate_pkt_ps * 8; 411 burst = act->police.burst_pkt; 412 pps = true; 413 } 414 nr_police++; 415 break; 416 case FLOW_ACTION_MARK: 417 mark = act->mark; 418 break; 419 default: 420 return -EOPNOTSUPP; 421 } 422 } 423 424 if (nr_police > 1) { 425 NL_SET_ERR_MSG_MOD(extack, 426 "rate limit police offload requires a single action"); 427 return -EOPNOTSUPP; 428 } 429 430 if (nr_police) 431 return otx2_tc_act_set_police(nic, node, f, rate, burst, 432 mark, req, pps); 433 434 return 0; 435 } 436 437 static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, 438 struct flow_cls_offload *f, 439 struct npc_install_flow_req *req) 440 { 441 struct netlink_ext_ack *extack = f->common.extack; 442 struct flow_msg *flow_spec = &req->packet; 443 struct flow_msg *flow_mask = &req->mask; 444 struct flow_dissector *dissector; 445 struct flow_rule *rule; 446 u8 ip_proto = 0; 447 448 rule = flow_cls_offload_flow_rule(f); 449 dissector = rule->match.dissector; 450 451 if ((dissector->used_keys & 452 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 453 BIT(FLOW_DISSECTOR_KEY_BASIC) | 454 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 455 BIT(FLOW_DISSECTOR_KEY_VLAN) | 456 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 457 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 458 BIT(FLOW_DISSECTOR_KEY_PORTS) | 459 BIT(FLOW_DISSECTOR_KEY_IP)))) { 460 netdev_info(nic->netdev, "unsupported flow used key 0x%x", 461 dissector->used_keys); 462 return -EOPNOTSUPP; 463 } 464 465 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 466 struct flow_match_basic match; 467 468 flow_rule_match_basic(rule, &match); 469 470 /* All EtherTypes can be matched, no hw limitation */ 471 flow_spec->etype = match.key->n_proto; 472 flow_mask->etype = match.mask->n_proto; 473 req->features |= BIT_ULL(NPC_ETYPE); 474 475 if (match.mask->ip_proto && 476 (match.key->ip_proto != IPPROTO_TCP && 477 match.key->ip_proto != IPPROTO_UDP && 478 match.key->ip_proto != IPPROTO_SCTP && 479 match.key->ip_proto != IPPROTO_ICMP && 480 match.key->ip_proto != IPPROTO_ICMPV6)) { 481 netdev_info(nic->netdev, 482 "ip_proto=0x%x not supported\n", 483 match.key->ip_proto); 484 return -EOPNOTSUPP; 485 } 486 if (match.mask->ip_proto) 487 ip_proto = match.key->ip_proto; 488 489 if (ip_proto == IPPROTO_UDP) 490 req->features |= BIT_ULL(NPC_IPPROTO_UDP); 491 else if (ip_proto == IPPROTO_TCP) 492 req->features |= BIT_ULL(NPC_IPPROTO_TCP); 493 else if (ip_proto == IPPROTO_SCTP) 494 req->features |= BIT_ULL(NPC_IPPROTO_SCTP); 495 else if (ip_proto == IPPROTO_ICMP) 496 req->features |= BIT_ULL(NPC_IPPROTO_ICMP); 497 else if (ip_proto == IPPROTO_ICMPV6) 498 req->features |= BIT_ULL(NPC_IPPROTO_ICMP6); 499 } 500 501 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 502 struct flow_match_eth_addrs match; 503 504 flow_rule_match_eth_addrs(rule, &match); 505 if (!is_zero_ether_addr(match.mask->src)) { 506 NL_SET_ERR_MSG_MOD(extack, "src mac match not supported"); 507 return -EOPNOTSUPP; 508 } 509 510 if (!is_zero_ether_addr(match.mask->dst)) { 511 ether_addr_copy(flow_spec->dmac, (u8 *)&match.key->dst); 512 ether_addr_copy(flow_mask->dmac, 513 (u8 *)&match.mask->dst); 514 req->features |= BIT_ULL(NPC_DMAC); 515 } 516 } 517 518 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { 519 struct flow_match_ip match; 520 521 flow_rule_match_ip(rule, &match); 522 if ((ntohs(flow_spec->etype) != ETH_P_IP) && 523 match.mask->tos) { 524 NL_SET_ERR_MSG_MOD(extack, "tos not supported"); 525 return -EOPNOTSUPP; 526 } 527 if (match.mask->ttl) { 528 NL_SET_ERR_MSG_MOD(extack, "ttl not supported"); 529 return -EOPNOTSUPP; 530 } 531 flow_spec->tos = match.key->tos; 532 flow_mask->tos = match.mask->tos; 533 req->features |= BIT_ULL(NPC_TOS); 534 } 535 536 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 537 struct flow_match_vlan match; 538 u16 vlan_tci, vlan_tci_mask; 539 540 flow_rule_match_vlan(rule, &match); 541 542 if (ntohs(match.key->vlan_tpid) != ETH_P_8021Q) { 543 netdev_err(nic->netdev, "vlan tpid 0x%x not supported\n", 544 ntohs(match.key->vlan_tpid)); 545 return -EOPNOTSUPP; 546 } 547 548 if (match.mask->vlan_id || 549 match.mask->vlan_dei || 550 match.mask->vlan_priority) { 551 vlan_tci = match.key->vlan_id | 552 match.key->vlan_dei << 12 | 553 match.key->vlan_priority << 13; 554 555 vlan_tci_mask = match.mask->vlan_id | 556 match.mask->vlan_dei << 12 | 557 match.mask->vlan_priority << 13; 558 559 flow_spec->vlan_tci = htons(vlan_tci); 560 flow_mask->vlan_tci = htons(vlan_tci_mask); 561 req->features |= BIT_ULL(NPC_OUTER_VID); 562 } 563 } 564 565 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 566 struct flow_match_ipv4_addrs match; 567 568 flow_rule_match_ipv4_addrs(rule, &match); 569 570 flow_spec->ip4dst = match.key->dst; 571 flow_mask->ip4dst = match.mask->dst; 572 req->features |= BIT_ULL(NPC_DIP_IPV4); 573 574 flow_spec->ip4src = match.key->src; 575 flow_mask->ip4src = match.mask->src; 576 req->features |= BIT_ULL(NPC_SIP_IPV4); 577 } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 578 struct flow_match_ipv6_addrs match; 579 580 flow_rule_match_ipv6_addrs(rule, &match); 581 582 if (ipv6_addr_loopback(&match.key->dst) || 583 ipv6_addr_loopback(&match.key->src)) { 584 NL_SET_ERR_MSG_MOD(extack, 585 "Flow matching IPv6 loopback addr not supported"); 586 return -EOPNOTSUPP; 587 } 588 589 if (!ipv6_addr_any(&match.mask->dst)) { 590 memcpy(&flow_spec->ip6dst, 591 (struct in6_addr *)&match.key->dst, 592 sizeof(flow_spec->ip6dst)); 593 memcpy(&flow_mask->ip6dst, 594 (struct in6_addr *)&match.mask->dst, 595 sizeof(flow_spec->ip6dst)); 596 req->features |= BIT_ULL(NPC_DIP_IPV6); 597 } 598 599 if (!ipv6_addr_any(&match.mask->src)) { 600 memcpy(&flow_spec->ip6src, 601 (struct in6_addr *)&match.key->src, 602 sizeof(flow_spec->ip6src)); 603 memcpy(&flow_mask->ip6src, 604 (struct in6_addr *)&match.mask->src, 605 sizeof(flow_spec->ip6src)); 606 req->features |= BIT_ULL(NPC_SIP_IPV6); 607 } 608 } 609 610 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 611 struct flow_match_ports match; 612 613 flow_rule_match_ports(rule, &match); 614 615 flow_spec->dport = match.key->dst; 616 flow_mask->dport = match.mask->dst; 617 if (ip_proto == IPPROTO_UDP) 618 req->features |= BIT_ULL(NPC_DPORT_UDP); 619 else if (ip_proto == IPPROTO_TCP) 620 req->features |= BIT_ULL(NPC_DPORT_TCP); 621 else if (ip_proto == IPPROTO_SCTP) 622 req->features |= BIT_ULL(NPC_DPORT_SCTP); 623 624 flow_spec->sport = match.key->src; 625 flow_mask->sport = match.mask->src; 626 if (ip_proto == IPPROTO_UDP) 627 req->features |= BIT_ULL(NPC_SPORT_UDP); 628 else if (ip_proto == IPPROTO_TCP) 629 req->features |= BIT_ULL(NPC_SPORT_TCP); 630 else if (ip_proto == IPPROTO_SCTP) 631 req->features |= BIT_ULL(NPC_SPORT_SCTP); 632 } 633 634 return otx2_tc_parse_actions(nic, &rule->action, req, f, node); 635 } 636 637 static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry) 638 { 639 struct npc_delete_flow_req *req; 640 int err; 641 642 mutex_lock(&nic->mbox.lock); 643 req = otx2_mbox_alloc_msg_npc_delete_flow(&nic->mbox); 644 if (!req) { 645 mutex_unlock(&nic->mbox.lock); 646 return -ENOMEM; 647 } 648 649 req->entry = entry; 650 651 /* Send message to AF */ 652 err = otx2_sync_mbox_msg(&nic->mbox); 653 if (err) { 654 netdev_err(nic->netdev, "Failed to delete MCAM flow entry %d\n", 655 entry); 656 mutex_unlock(&nic->mbox.lock); 657 return -EFAULT; 658 } 659 mutex_unlock(&nic->mbox.lock); 660 661 return 0; 662 } 663 664 static int otx2_tc_del_flow(struct otx2_nic *nic, 665 struct flow_cls_offload *tc_flow_cmd) 666 { 667 struct otx2_flow_config *flow_cfg = nic->flow_cfg; 668 struct otx2_tc_info *tc_info = &nic->tc_info; 669 struct otx2_tc_flow *flow_node; 670 int err; 671 672 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 673 &tc_flow_cmd->cookie, 674 tc_info->flow_ht_params); 675 if (!flow_node) { 676 netdev_err(nic->netdev, "tc flow not found for cookie 0x%lx\n", 677 tc_flow_cmd->cookie); 678 return -EINVAL; 679 } 680 681 if (flow_node->is_act_police) { 682 mutex_lock(&nic->mbox.lock); 683 684 err = cn10k_map_unmap_rq_policer(nic, flow_node->rq, 685 flow_node->leaf_profile, false); 686 if (err) 687 netdev_err(nic->netdev, 688 "Unmapping RQ %d & profile %d failed\n", 689 flow_node->rq, flow_node->leaf_profile); 690 691 err = cn10k_free_leaf_profile(nic, flow_node->leaf_profile); 692 if (err) 693 netdev_err(nic->netdev, 694 "Unable to free leaf bandwidth profile(%d)\n", 695 flow_node->leaf_profile); 696 697 __clear_bit(flow_node->rq, &nic->rq_bmap); 698 699 mutex_unlock(&nic->mbox.lock); 700 } 701 702 otx2_del_mcam_flow_entry(nic, flow_node->entry); 703 704 WARN_ON(rhashtable_remove_fast(&nic->tc_info.flow_table, 705 &flow_node->node, 706 nic->tc_info.flow_ht_params)); 707 kfree_rcu(flow_node, rcu); 708 709 clear_bit(flow_node->bitpos, tc_info->tc_entries_bitmap); 710 flow_cfg->nr_flows--; 711 712 return 0; 713 } 714 715 static int otx2_tc_add_flow(struct otx2_nic *nic, 716 struct flow_cls_offload *tc_flow_cmd) 717 { 718 struct netlink_ext_ack *extack = tc_flow_cmd->common.extack; 719 struct otx2_flow_config *flow_cfg = nic->flow_cfg; 720 struct otx2_tc_info *tc_info = &nic->tc_info; 721 struct otx2_tc_flow *new_node, *old_node; 722 struct npc_install_flow_req *req, dummy; 723 int rc, err; 724 725 if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)) 726 return -ENOMEM; 727 728 if (bitmap_full(tc_info->tc_entries_bitmap, flow_cfg->max_flows)) { 729 NL_SET_ERR_MSG_MOD(extack, 730 "Free MCAM entry not available to add the flow"); 731 return -ENOMEM; 732 } 733 734 /* allocate memory for the new flow and it's node */ 735 new_node = kzalloc(sizeof(*new_node), GFP_KERNEL); 736 if (!new_node) 737 return -ENOMEM; 738 spin_lock_init(&new_node->lock); 739 new_node->cookie = tc_flow_cmd->cookie; 740 741 memset(&dummy, 0, sizeof(struct npc_install_flow_req)); 742 743 rc = otx2_tc_prepare_flow(nic, new_node, tc_flow_cmd, &dummy); 744 if (rc) { 745 kfree_rcu(new_node, rcu); 746 return rc; 747 } 748 749 /* If a flow exists with the same cookie, delete it */ 750 old_node = rhashtable_lookup_fast(&tc_info->flow_table, 751 &tc_flow_cmd->cookie, 752 tc_info->flow_ht_params); 753 if (old_node) 754 otx2_tc_del_flow(nic, tc_flow_cmd); 755 756 mutex_lock(&nic->mbox.lock); 757 req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); 758 if (!req) { 759 mutex_unlock(&nic->mbox.lock); 760 rc = -ENOMEM; 761 goto free_leaf; 762 } 763 764 memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr)); 765 memcpy(req, &dummy, sizeof(struct npc_install_flow_req)); 766 767 new_node->bitpos = find_first_zero_bit(tc_info->tc_entries_bitmap, 768 flow_cfg->max_flows); 769 req->channel = nic->hw.rx_chan_base; 770 req->entry = flow_cfg->flow_ent[flow_cfg->max_flows - new_node->bitpos - 1]; 771 req->intf = NIX_INTF_RX; 772 req->set_cntr = 1; 773 new_node->entry = req->entry; 774 775 /* Send message to AF */ 776 rc = otx2_sync_mbox_msg(&nic->mbox); 777 if (rc) { 778 NL_SET_ERR_MSG_MOD(extack, "Failed to install MCAM flow entry"); 779 mutex_unlock(&nic->mbox.lock); 780 kfree_rcu(new_node, rcu); 781 goto free_leaf; 782 } 783 mutex_unlock(&nic->mbox.lock); 784 785 /* add new flow to flow-table */ 786 rc = rhashtable_insert_fast(&nic->tc_info.flow_table, &new_node->node, 787 nic->tc_info.flow_ht_params); 788 if (rc) { 789 otx2_del_mcam_flow_entry(nic, req->entry); 790 kfree_rcu(new_node, rcu); 791 goto free_leaf; 792 } 793 794 set_bit(new_node->bitpos, tc_info->tc_entries_bitmap); 795 flow_cfg->nr_flows++; 796 797 return 0; 798 799 free_leaf: 800 if (new_node->is_act_police) { 801 mutex_lock(&nic->mbox.lock); 802 803 err = cn10k_map_unmap_rq_policer(nic, new_node->rq, 804 new_node->leaf_profile, false); 805 if (err) 806 netdev_err(nic->netdev, 807 "Unmapping RQ %d & profile %d failed\n", 808 new_node->rq, new_node->leaf_profile); 809 err = cn10k_free_leaf_profile(nic, new_node->leaf_profile); 810 if (err) 811 netdev_err(nic->netdev, 812 "Unable to free leaf bandwidth profile(%d)\n", 813 new_node->leaf_profile); 814 815 __clear_bit(new_node->rq, &nic->rq_bmap); 816 817 mutex_unlock(&nic->mbox.lock); 818 } 819 820 return rc; 821 } 822 823 static int otx2_tc_get_flow_stats(struct otx2_nic *nic, 824 struct flow_cls_offload *tc_flow_cmd) 825 { 826 struct otx2_tc_info *tc_info = &nic->tc_info; 827 struct npc_mcam_get_stats_req *req; 828 struct npc_mcam_get_stats_rsp *rsp; 829 struct otx2_tc_flow_stats *stats; 830 struct otx2_tc_flow *flow_node; 831 int err; 832 833 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 834 &tc_flow_cmd->cookie, 835 tc_info->flow_ht_params); 836 if (!flow_node) { 837 netdev_info(nic->netdev, "tc flow not found for cookie %lx", 838 tc_flow_cmd->cookie); 839 return -EINVAL; 840 } 841 842 mutex_lock(&nic->mbox.lock); 843 844 req = otx2_mbox_alloc_msg_npc_mcam_entry_stats(&nic->mbox); 845 if (!req) { 846 mutex_unlock(&nic->mbox.lock); 847 return -ENOMEM; 848 } 849 850 req->entry = flow_node->entry; 851 852 err = otx2_sync_mbox_msg(&nic->mbox); 853 if (err) { 854 netdev_err(nic->netdev, "Failed to get stats for MCAM flow entry %d\n", 855 req->entry); 856 mutex_unlock(&nic->mbox.lock); 857 return -EFAULT; 858 } 859 860 rsp = (struct npc_mcam_get_stats_rsp *)otx2_mbox_get_rsp 861 (&nic->mbox.mbox, 0, &req->hdr); 862 if (IS_ERR(rsp)) { 863 mutex_unlock(&nic->mbox.lock); 864 return PTR_ERR(rsp); 865 } 866 867 mutex_unlock(&nic->mbox.lock); 868 869 if (!rsp->stat_ena) 870 return -EINVAL; 871 872 stats = &flow_node->stats; 873 874 spin_lock(&flow_node->lock); 875 flow_stats_update(&tc_flow_cmd->stats, 0x0, rsp->stat - stats->pkts, 0x0, 0x0, 876 FLOW_ACTION_HW_STATS_IMMEDIATE); 877 stats->pkts = rsp->stat; 878 spin_unlock(&flow_node->lock); 879 880 return 0; 881 } 882 883 static int otx2_setup_tc_cls_flower(struct otx2_nic *nic, 884 struct flow_cls_offload *cls_flower) 885 { 886 switch (cls_flower->command) { 887 case FLOW_CLS_REPLACE: 888 return otx2_tc_add_flow(nic, cls_flower); 889 case FLOW_CLS_DESTROY: 890 return otx2_tc_del_flow(nic, cls_flower); 891 case FLOW_CLS_STATS: 892 return otx2_tc_get_flow_stats(nic, cls_flower); 893 default: 894 return -EOPNOTSUPP; 895 } 896 } 897 898 static int otx2_tc_ingress_matchall_install(struct otx2_nic *nic, 899 struct tc_cls_matchall_offload *cls) 900 { 901 struct netlink_ext_ack *extack = cls->common.extack; 902 struct flow_action *actions = &cls->rule->action; 903 struct flow_action_entry *entry; 904 u64 rate; 905 int err; 906 907 err = otx2_tc_validate_flow(nic, actions, extack); 908 if (err) 909 return err; 910 911 if (nic->flags & OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED) { 912 NL_SET_ERR_MSG_MOD(extack, 913 "Only one ingress MATCHALL ratelimitter can be offloaded"); 914 return -ENOMEM; 915 } 916 917 entry = &cls->rule->action.entries[0]; 918 switch (entry->id) { 919 case FLOW_ACTION_POLICE: 920 /* Ingress ratelimiting is not supported on OcteonTx2 */ 921 if (is_dev_otx2(nic->pdev)) { 922 NL_SET_ERR_MSG_MOD(extack, 923 "Ingress policing not supported on this platform"); 924 return -EOPNOTSUPP; 925 } 926 927 err = cn10k_alloc_matchall_ipolicer(nic); 928 if (err) 929 return err; 930 931 /* Convert to bits per second */ 932 rate = entry->police.rate_bytes_ps * 8; 933 err = cn10k_set_matchall_ipolicer_rate(nic, entry->police.burst, rate); 934 if (err) 935 return err; 936 nic->flags |= OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED; 937 break; 938 default: 939 NL_SET_ERR_MSG_MOD(extack, 940 "Only police action supported with Ingress MATCHALL offload"); 941 return -EOPNOTSUPP; 942 } 943 944 return 0; 945 } 946 947 static int otx2_tc_ingress_matchall_delete(struct otx2_nic *nic, 948 struct tc_cls_matchall_offload *cls) 949 { 950 struct netlink_ext_ack *extack = cls->common.extack; 951 int err; 952 953 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 954 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 955 return -EINVAL; 956 } 957 958 err = cn10k_free_matchall_ipolicer(nic); 959 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED; 960 return err; 961 } 962 963 static int otx2_setup_tc_ingress_matchall(struct otx2_nic *nic, 964 struct tc_cls_matchall_offload *cls_matchall) 965 { 966 switch (cls_matchall->command) { 967 case TC_CLSMATCHALL_REPLACE: 968 return otx2_tc_ingress_matchall_install(nic, cls_matchall); 969 case TC_CLSMATCHALL_DESTROY: 970 return otx2_tc_ingress_matchall_delete(nic, cls_matchall); 971 case TC_CLSMATCHALL_STATS: 972 default: 973 break; 974 } 975 976 return -EOPNOTSUPP; 977 } 978 979 static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type, 980 void *type_data, void *cb_priv) 981 { 982 struct otx2_nic *nic = cb_priv; 983 984 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data)) 985 return -EOPNOTSUPP; 986 987 switch (type) { 988 case TC_SETUP_CLSFLOWER: 989 return otx2_setup_tc_cls_flower(nic, type_data); 990 case TC_SETUP_CLSMATCHALL: 991 return otx2_setup_tc_ingress_matchall(nic, type_data); 992 default: 993 break; 994 } 995 996 return -EOPNOTSUPP; 997 } 998 999 static int otx2_setup_tc_egress_matchall(struct otx2_nic *nic, 1000 struct tc_cls_matchall_offload *cls_matchall) 1001 { 1002 switch (cls_matchall->command) { 1003 case TC_CLSMATCHALL_REPLACE: 1004 return otx2_tc_egress_matchall_install(nic, cls_matchall); 1005 case TC_CLSMATCHALL_DESTROY: 1006 return otx2_tc_egress_matchall_delete(nic, cls_matchall); 1007 case TC_CLSMATCHALL_STATS: 1008 default: 1009 break; 1010 } 1011 1012 return -EOPNOTSUPP; 1013 } 1014 1015 static int otx2_setup_tc_block_egress_cb(enum tc_setup_type type, 1016 void *type_data, void *cb_priv) 1017 { 1018 struct otx2_nic *nic = cb_priv; 1019 1020 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data)) 1021 return -EOPNOTSUPP; 1022 1023 switch (type) { 1024 case TC_SETUP_CLSMATCHALL: 1025 return otx2_setup_tc_egress_matchall(nic, type_data); 1026 default: 1027 break; 1028 } 1029 1030 return -EOPNOTSUPP; 1031 } 1032 1033 static LIST_HEAD(otx2_block_cb_list); 1034 1035 static int otx2_setup_tc_block(struct net_device *netdev, 1036 struct flow_block_offload *f) 1037 { 1038 struct otx2_nic *nic = netdev_priv(netdev); 1039 flow_setup_cb_t *cb; 1040 bool ingress; 1041 1042 if (f->block_shared) 1043 return -EOPNOTSUPP; 1044 1045 if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) { 1046 cb = otx2_setup_tc_block_ingress_cb; 1047 ingress = true; 1048 } else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) { 1049 cb = otx2_setup_tc_block_egress_cb; 1050 ingress = false; 1051 } else { 1052 return -EOPNOTSUPP; 1053 } 1054 1055 return flow_block_cb_setup_simple(f, &otx2_block_cb_list, cb, 1056 nic, nic, ingress); 1057 } 1058 1059 int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type, 1060 void *type_data) 1061 { 1062 switch (type) { 1063 case TC_SETUP_BLOCK: 1064 return otx2_setup_tc_block(netdev, type_data); 1065 default: 1066 return -EOPNOTSUPP; 1067 } 1068 } 1069 EXPORT_SYMBOL(otx2_setup_tc); 1070 1071 static const struct rhashtable_params tc_flow_ht_params = { 1072 .head_offset = offsetof(struct otx2_tc_flow, node), 1073 .key_offset = offsetof(struct otx2_tc_flow, cookie), 1074 .key_len = sizeof(((struct otx2_tc_flow *)0)->cookie), 1075 .automatic_shrinking = true, 1076 }; 1077 1078 int otx2_init_tc(struct otx2_nic *nic) 1079 { 1080 struct otx2_tc_info *tc = &nic->tc_info; 1081 int err; 1082 1083 /* Exclude receive queue 0 being used for police action */ 1084 set_bit(0, &nic->rq_bmap); 1085 1086 if (!nic->flow_cfg) { 1087 netdev_err(nic->netdev, 1088 "Can't init TC, nic->flow_cfg is not setup\n"); 1089 return -EINVAL; 1090 } 1091 1092 err = otx2_tc_alloc_ent_bitmap(nic); 1093 if (err) 1094 return err; 1095 1096 tc->flow_ht_params = tc_flow_ht_params; 1097 return rhashtable_init(&tc->flow_table, &tc->flow_ht_params); 1098 } 1099 EXPORT_SYMBOL(otx2_init_tc); 1100 1101 void otx2_shutdown_tc(struct otx2_nic *nic) 1102 { 1103 struct otx2_tc_info *tc = &nic->tc_info; 1104 1105 kfree(tc->tc_entries_bitmap); 1106 rhashtable_destroy(&tc->flow_table); 1107 } 1108 EXPORT_SYMBOL(otx2_shutdown_tc); 1109