1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2017 Broadcom Limited 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 */ 9 10 #include <linux/netdevice.h> 11 #include <linux/inetdevice.h> 12 #include <linux/if_vlan.h> 13 #include <net/flow_dissector.h> 14 #include <net/pkt_cls.h> 15 #include <net/tc_act/tc_gact.h> 16 #include <net/tc_act/tc_skbedit.h> 17 #include <net/tc_act/tc_mirred.h> 18 #include <net/tc_act/tc_vlan.h> 19 20 #include "bnxt_hsi.h" 21 #include "bnxt.h" 22 #include "bnxt_sriov.h" 23 #include "bnxt_tc.h" 24 #include "bnxt_vfr.h" 25 26 #ifdef CONFIG_BNXT_FLOWER_OFFLOAD 27 28 #define BNXT_FID_INVALID 0xffff 29 #define VLAN_TCI(vid, prio) ((vid) | ((prio) << VLAN_PRIO_SHIFT)) 30 31 /* Return the dst fid of the func for flow forwarding 32 * For PFs: src_fid is the fid of the PF 33 * For VF-reps: src_fid the fid of the VF 34 */ 35 static u16 bnxt_flow_get_dst_fid(struct bnxt *pf_bp, struct net_device *dev) 36 { 37 struct bnxt *bp; 38 39 /* check if dev belongs to the same switch */ 40 if (!switchdev_port_same_parent_id(pf_bp->dev, dev)) { 41 netdev_info(pf_bp->dev, "dev(ifindex=%d) not on same switch", 42 dev->ifindex); 43 return BNXT_FID_INVALID; 44 } 45 46 /* Is dev a VF-rep? */ 47 if (dev != pf_bp->dev) 48 return bnxt_vf_rep_get_fid(dev); 49 50 bp = netdev_priv(dev); 51 return bp->pf.fw_fid; 52 } 53 54 static int bnxt_tc_parse_redir(struct bnxt *bp, 55 struct bnxt_tc_actions *actions, 56 const struct tc_action *tc_act) 57 { 58 int ifindex = tcf_mirred_ifindex(tc_act); 59 struct net_device *dev; 60 u16 dst_fid; 61 62 dev = __dev_get_by_index(dev_net(bp->dev), ifindex); 63 if (!dev) { 64 netdev_info(bp->dev, "no dev for ifindex=%d", ifindex); 65 return -EINVAL; 66 } 67 68 /* find the FID from dev */ 69 dst_fid = bnxt_flow_get_dst_fid(bp, dev); 70 if (dst_fid == BNXT_FID_INVALID) { 71 netdev_info(bp->dev, "can't get fid for ifindex=%d", ifindex); 72 return -EINVAL; 73 } 74 75 actions->flags |= BNXT_TC_ACTION_FLAG_FWD; 76 actions->dst_fid = dst_fid; 77 actions->dst_dev = dev; 78 return 0; 79 } 80 81 static void bnxt_tc_parse_vlan(struct bnxt *bp, 82 struct bnxt_tc_actions *actions, 83 const struct tc_action *tc_act) 84 { 85 if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_POP) { 86 actions->flags |= BNXT_TC_ACTION_FLAG_POP_VLAN; 87 } else if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_PUSH) { 88 actions->flags |= BNXT_TC_ACTION_FLAG_PUSH_VLAN; 89 actions->push_vlan_tci = htons(tcf_vlan_push_vid(tc_act)); 90 actions->push_vlan_tpid = tcf_vlan_push_proto(tc_act); 91 } 92 } 93 94 static int bnxt_tc_parse_actions(struct bnxt *bp, 95 struct bnxt_tc_actions *actions, 96 struct tcf_exts *tc_exts) 97 { 98 const struct tc_action *tc_act; 99 LIST_HEAD(tc_actions); 100 int rc; 101 102 if (!tcf_exts_has_actions(tc_exts)) { 103 netdev_info(bp->dev, "no actions"); 104 return -EINVAL; 105 } 106 107 tcf_exts_to_list(tc_exts, &tc_actions); 108 list_for_each_entry(tc_act, &tc_actions, list) { 109 /* Drop action */ 110 if (is_tcf_gact_shot(tc_act)) { 111 actions->flags |= BNXT_TC_ACTION_FLAG_DROP; 112 return 0; /* don't bother with other actions */ 113 } 114 115 /* Redirect action */ 116 if (is_tcf_mirred_egress_redirect(tc_act)) { 117 rc = bnxt_tc_parse_redir(bp, actions, tc_act); 118 if (rc) 119 return rc; 120 continue; 121 } 122 123 /* Push/pop VLAN */ 124 if (is_tcf_vlan(tc_act)) { 125 bnxt_tc_parse_vlan(bp, actions, tc_act); 126 continue; 127 } 128 } 129 130 return 0; 131 } 132 133 #define GET_KEY(flow_cmd, key_type) \ 134 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\ 135 (flow_cmd)->key) 136 #define GET_MASK(flow_cmd, key_type) \ 137 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\ 138 (flow_cmd)->mask) 139 140 static int bnxt_tc_parse_flow(struct bnxt *bp, 141 struct tc_cls_flower_offload *tc_flow_cmd, 142 struct bnxt_tc_flow *flow) 143 { 144 struct flow_dissector *dissector = tc_flow_cmd->dissector; 145 u16 addr_type = 0; 146 147 /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */ 148 if ((dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) == 0 || 149 (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) == 0) { 150 netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%x", 151 dissector->used_keys); 152 return -EOPNOTSUPP; 153 } 154 155 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_CONTROL)) { 156 struct flow_dissector_key_control *key = 157 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_CONTROL); 158 159 addr_type = key->addr_type; 160 } 161 162 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC)) { 163 struct flow_dissector_key_basic *key = 164 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC); 165 struct flow_dissector_key_basic *mask = 166 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC); 167 168 flow->l2_key.ether_type = key->n_proto; 169 flow->l2_mask.ether_type = mask->n_proto; 170 171 if (key->n_proto == htons(ETH_P_IP) || 172 key->n_proto == htons(ETH_P_IPV6)) { 173 flow->l4_key.ip_proto = key->ip_proto; 174 flow->l4_mask.ip_proto = mask->ip_proto; 175 } 176 } 177 178 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 179 struct flow_dissector_key_eth_addrs *key = 180 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS); 181 struct flow_dissector_key_eth_addrs *mask = 182 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS); 183 184 flow->flags |= BNXT_TC_FLOW_FLAGS_ETH_ADDRS; 185 ether_addr_copy(flow->l2_key.dmac, key->dst); 186 ether_addr_copy(flow->l2_mask.dmac, mask->dst); 187 ether_addr_copy(flow->l2_key.smac, key->src); 188 ether_addr_copy(flow->l2_mask.smac, mask->src); 189 } 190 191 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN)) { 192 struct flow_dissector_key_vlan *key = 193 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN); 194 struct flow_dissector_key_vlan *mask = 195 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN); 196 197 flow->l2_key.inner_vlan_tci = 198 cpu_to_be16(VLAN_TCI(key->vlan_id, key->vlan_priority)); 199 flow->l2_mask.inner_vlan_tci = 200 cpu_to_be16((VLAN_TCI(mask->vlan_id, mask->vlan_priority))); 201 flow->l2_key.inner_vlan_tpid = htons(ETH_P_8021Q); 202 flow->l2_mask.inner_vlan_tpid = htons(0xffff); 203 flow->l2_key.num_vlans = 1; 204 } 205 206 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 207 struct flow_dissector_key_ipv4_addrs *key = 208 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 209 struct flow_dissector_key_ipv4_addrs *mask = 210 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 211 212 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS; 213 flow->l3_key.ipv4.daddr.s_addr = key->dst; 214 flow->l3_mask.ipv4.daddr.s_addr = mask->dst; 215 flow->l3_key.ipv4.saddr.s_addr = key->src; 216 flow->l3_mask.ipv4.saddr.s_addr = mask->src; 217 } else if (dissector_uses_key(dissector, 218 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 219 struct flow_dissector_key_ipv6_addrs *key = 220 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 221 struct flow_dissector_key_ipv6_addrs *mask = 222 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 223 224 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS; 225 flow->l3_key.ipv6.daddr = key->dst; 226 flow->l3_mask.ipv6.daddr = mask->dst; 227 flow->l3_key.ipv6.saddr = key->src; 228 flow->l3_mask.ipv6.saddr = mask->src; 229 } 230 231 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_PORTS)) { 232 struct flow_dissector_key_ports *key = 233 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS); 234 struct flow_dissector_key_ports *mask = 235 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS); 236 237 flow->flags |= BNXT_TC_FLOW_FLAGS_PORTS; 238 flow->l4_key.ports.dport = key->dst; 239 flow->l4_mask.ports.dport = mask->dst; 240 flow->l4_key.ports.sport = key->src; 241 flow->l4_mask.ports.sport = mask->src; 242 } 243 244 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ICMP)) { 245 struct flow_dissector_key_icmp *key = 246 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP); 247 struct flow_dissector_key_icmp *mask = 248 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP); 249 250 flow->flags |= BNXT_TC_FLOW_FLAGS_ICMP; 251 flow->l4_key.icmp.type = key->type; 252 flow->l4_key.icmp.code = key->code; 253 flow->l4_mask.icmp.type = mask->type; 254 flow->l4_mask.icmp.code = mask->code; 255 } 256 257 return bnxt_tc_parse_actions(bp, &flow->actions, tc_flow_cmd->exts); 258 } 259 260 static int bnxt_hwrm_cfa_flow_free(struct bnxt *bp, __le16 flow_handle) 261 { 262 struct hwrm_cfa_flow_free_input req = { 0 }; 263 int rc; 264 265 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_FREE, -1, -1); 266 req.flow_handle = flow_handle; 267 268 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 269 if (rc) 270 netdev_info(bp->dev, "Error: %s: flow_handle=0x%x rc=%d", 271 __func__, flow_handle, rc); 272 return rc; 273 } 274 275 static int ipv6_mask_len(struct in6_addr *mask) 276 { 277 int mask_len = 0, i; 278 279 for (i = 0; i < 4; i++) 280 mask_len += inet_mask_len(mask->s6_addr32[i]); 281 282 return mask_len; 283 } 284 285 static bool is_wildcard(void *mask, int len) 286 { 287 const u8 *p = mask; 288 int i; 289 290 for (i = 0; i < len; i++) { 291 if (p[i] != 0) 292 return false; 293 } 294 return true; 295 } 296 297 static int bnxt_hwrm_cfa_flow_alloc(struct bnxt *bp, struct bnxt_tc_flow *flow, 298 __le16 ref_flow_handle, __le16 *flow_handle) 299 { 300 struct hwrm_cfa_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr; 301 struct bnxt_tc_actions *actions = &flow->actions; 302 struct bnxt_tc_l3_key *l3_mask = &flow->l3_mask; 303 struct bnxt_tc_l3_key *l3_key = &flow->l3_key; 304 struct hwrm_cfa_flow_alloc_input req = { 0 }; 305 u16 flow_flags = 0, action_flags = 0; 306 int rc; 307 308 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_ALLOC, -1, -1); 309 310 req.src_fid = cpu_to_le16(flow->src_fid); 311 req.ref_flow_handle = ref_flow_handle; 312 req.ethertype = flow->l2_key.ether_type; 313 req.ip_proto = flow->l4_key.ip_proto; 314 315 if (flow->flags & BNXT_TC_FLOW_FLAGS_ETH_ADDRS) { 316 memcpy(req.dmac, flow->l2_key.dmac, ETH_ALEN); 317 memcpy(req.smac, flow->l2_key.smac, ETH_ALEN); 318 } 319 320 if (flow->l2_key.num_vlans > 0) { 321 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE; 322 /* FW expects the inner_vlan_tci value to be set 323 * in outer_vlan_tci when num_vlans is 1 (which is 324 * always the case in TC.) 325 */ 326 req.outer_vlan_tci = flow->l2_key.inner_vlan_tci; 327 } 328 329 /* If all IP and L4 fields are wildcarded then this is an L2 flow */ 330 if (is_wildcard(&l3_mask, sizeof(l3_mask)) && 331 is_wildcard(&flow->l4_mask, sizeof(flow->l4_mask))) { 332 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2; 333 } else { 334 flow_flags |= flow->l2_key.ether_type == htons(ETH_P_IP) ? 335 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4 : 336 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6; 337 338 if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV4_ADDRS) { 339 req.ip_dst[0] = l3_key->ipv4.daddr.s_addr; 340 req.ip_dst_mask_len = 341 inet_mask_len(l3_mask->ipv4.daddr.s_addr); 342 req.ip_src[0] = l3_key->ipv4.saddr.s_addr; 343 req.ip_src_mask_len = 344 inet_mask_len(l3_mask->ipv4.saddr.s_addr); 345 } else if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV6_ADDRS) { 346 memcpy(req.ip_dst, l3_key->ipv6.daddr.s6_addr32, 347 sizeof(req.ip_dst)); 348 req.ip_dst_mask_len = 349 ipv6_mask_len(&l3_mask->ipv6.daddr); 350 memcpy(req.ip_src, l3_key->ipv6.saddr.s6_addr32, 351 sizeof(req.ip_src)); 352 req.ip_src_mask_len = 353 ipv6_mask_len(&l3_mask->ipv6.saddr); 354 } 355 } 356 357 if (flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) { 358 req.l4_src_port = flow->l4_key.ports.sport; 359 req.l4_src_port_mask = flow->l4_mask.ports.sport; 360 req.l4_dst_port = flow->l4_key.ports.dport; 361 req.l4_dst_port_mask = flow->l4_mask.ports.dport; 362 } else if (flow->flags & BNXT_TC_FLOW_FLAGS_ICMP) { 363 /* l4 ports serve as type/code when ip_proto is ICMP */ 364 req.l4_src_port = htons(flow->l4_key.icmp.type); 365 req.l4_src_port_mask = htons(flow->l4_mask.icmp.type); 366 req.l4_dst_port = htons(flow->l4_key.icmp.code); 367 req.l4_dst_port_mask = htons(flow->l4_mask.icmp.code); 368 } 369 req.flags = cpu_to_le16(flow_flags); 370 371 if (actions->flags & BNXT_TC_ACTION_FLAG_DROP) { 372 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP; 373 } else { 374 if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) { 375 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD; 376 req.dst_fid = cpu_to_le16(actions->dst_fid); 377 } 378 if (actions->flags & BNXT_TC_ACTION_FLAG_PUSH_VLAN) { 379 action_flags |= 380 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE; 381 req.l2_rewrite_vlan_tpid = actions->push_vlan_tpid; 382 req.l2_rewrite_vlan_tci = actions->push_vlan_tci; 383 memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN); 384 memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN); 385 } 386 if (actions->flags & BNXT_TC_ACTION_FLAG_POP_VLAN) { 387 action_flags |= 388 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE; 389 /* Rewrite config with tpid = 0 implies vlan pop */ 390 req.l2_rewrite_vlan_tpid = 0; 391 memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN); 392 memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN); 393 } 394 } 395 req.action_flags = cpu_to_le16(action_flags); 396 397 mutex_lock(&bp->hwrm_cmd_lock); 398 399 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 400 if (!rc) 401 *flow_handle = resp->flow_handle; 402 403 mutex_unlock(&bp->hwrm_cmd_lock); 404 405 return rc; 406 } 407 408 /* Add val to accum while handling a possible wraparound 409 * of val. Eventhough val is of type u64, its actual width 410 * is denoted by mask and will wrap-around beyond that width. 411 */ 412 static void accumulate_val(u64 *accum, u64 val, u64 mask) 413 { 414 #define low_bits(x, mask) ((x) & (mask)) 415 #define high_bits(x, mask) ((x) & ~(mask)) 416 bool wrapped = val < low_bits(*accum, mask); 417 418 *accum = high_bits(*accum, mask) + val; 419 if (wrapped) 420 *accum += (mask + 1); 421 } 422 423 /* The HW counters' width is much less than 64bits. 424 * Handle possible wrap-around while updating the stat counters 425 */ 426 static void bnxt_flow_stats_fix_wraparound(struct bnxt_tc_info *tc_info, 427 struct bnxt_tc_flow_stats *stats, 428 struct bnxt_tc_flow_stats *hw_stats) 429 { 430 accumulate_val(&stats->bytes, hw_stats->bytes, tc_info->bytes_mask); 431 accumulate_val(&stats->packets, hw_stats->packets, 432 tc_info->packets_mask); 433 } 434 435 /* Fix possible wraparound of the stats queried from HW, calculate 436 * the delta from prev_stats, and also update the prev_stats. 437 * The HW flow stats are fetched under the hwrm_cmd_lock mutex. 438 * This routine is best called while under the mutex so that the 439 * stats processing happens atomically. 440 */ 441 static void bnxt_flow_stats_calc(struct bnxt_tc_info *tc_info, 442 struct bnxt_tc_flow *flow, 443 struct bnxt_tc_flow_stats *stats) 444 { 445 struct bnxt_tc_flow_stats *acc_stats, *prev_stats; 446 447 acc_stats = &flow->stats; 448 bnxt_flow_stats_fix_wraparound(tc_info, acc_stats, stats); 449 450 prev_stats = &flow->prev_stats; 451 stats->bytes = acc_stats->bytes - prev_stats->bytes; 452 stats->packets = acc_stats->packets - prev_stats->packets; 453 *prev_stats = *acc_stats; 454 } 455 456 static int bnxt_hwrm_cfa_flow_stats_get(struct bnxt *bp, 457 __le16 flow_handle, 458 struct bnxt_tc_flow *flow, 459 struct bnxt_tc_flow_stats *stats) 460 { 461 struct hwrm_cfa_flow_stats_output *resp = bp->hwrm_cmd_resp_addr; 462 struct hwrm_cfa_flow_stats_input req = { 0 }; 463 int rc; 464 465 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_STATS, -1, -1); 466 req.num_flows = cpu_to_le16(1); 467 req.flow_handle_0 = flow_handle; 468 469 mutex_lock(&bp->hwrm_cmd_lock); 470 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 471 if (!rc) { 472 stats->packets = le64_to_cpu(resp->packet_0); 473 stats->bytes = le64_to_cpu(resp->byte_0); 474 bnxt_flow_stats_calc(&bp->tc_info, flow, stats); 475 } else { 476 netdev_info(bp->dev, "error rc=%d", rc); 477 } 478 479 mutex_unlock(&bp->hwrm_cmd_lock); 480 return rc; 481 } 482 483 static int bnxt_tc_put_l2_node(struct bnxt *bp, 484 struct bnxt_tc_flow_node *flow_node) 485 { 486 struct bnxt_tc_l2_node *l2_node = flow_node->l2_node; 487 struct bnxt_tc_info *tc_info = &bp->tc_info; 488 int rc; 489 490 /* remove flow_node from the L2 shared flow list */ 491 list_del(&flow_node->l2_list_node); 492 if (--l2_node->refcount == 0) { 493 rc = rhashtable_remove_fast(&tc_info->l2_table, &l2_node->node, 494 tc_info->l2_ht_params); 495 if (rc) 496 netdev_err(bp->dev, 497 "Error: %s: rhashtable_remove_fast: %d", 498 __func__, rc); 499 kfree_rcu(l2_node, rcu); 500 } 501 return 0; 502 } 503 504 static struct bnxt_tc_l2_node * 505 bnxt_tc_get_l2_node(struct bnxt *bp, struct rhashtable *l2_table, 506 struct rhashtable_params ht_params, 507 struct bnxt_tc_l2_key *l2_key) 508 { 509 struct bnxt_tc_l2_node *l2_node; 510 int rc; 511 512 l2_node = rhashtable_lookup_fast(l2_table, l2_key, ht_params); 513 if (!l2_node) { 514 l2_node = kzalloc(sizeof(*l2_node), GFP_KERNEL); 515 if (!l2_node) { 516 rc = -ENOMEM; 517 return NULL; 518 } 519 520 l2_node->key = *l2_key; 521 rc = rhashtable_insert_fast(l2_table, &l2_node->node, 522 ht_params); 523 if (rc) { 524 kfree(l2_node); 525 netdev_err(bp->dev, 526 "Error: %s: rhashtable_insert_fast: %d", 527 __func__, rc); 528 return NULL; 529 } 530 INIT_LIST_HEAD(&l2_node->common_l2_flows); 531 } 532 return l2_node; 533 } 534 535 /* Get the ref_flow_handle for a flow by checking if there are any other 536 * flows that share the same L2 key as this flow. 537 */ 538 static int 539 bnxt_tc_get_ref_flow_handle(struct bnxt *bp, struct bnxt_tc_flow *flow, 540 struct bnxt_tc_flow_node *flow_node, 541 __le16 *ref_flow_handle) 542 { 543 struct bnxt_tc_info *tc_info = &bp->tc_info; 544 struct bnxt_tc_flow_node *ref_flow_node; 545 struct bnxt_tc_l2_node *l2_node; 546 547 l2_node = bnxt_tc_get_l2_node(bp, &tc_info->l2_table, 548 tc_info->l2_ht_params, 549 &flow->l2_key); 550 if (!l2_node) 551 return -1; 552 553 /* If any other flow is using this l2_node, use it's flow_handle 554 * as the ref_flow_handle 555 */ 556 if (l2_node->refcount > 0) { 557 ref_flow_node = list_first_entry(&l2_node->common_l2_flows, 558 struct bnxt_tc_flow_node, 559 l2_list_node); 560 *ref_flow_handle = ref_flow_node->flow_handle; 561 } else { 562 *ref_flow_handle = cpu_to_le16(0xffff); 563 } 564 565 /* Insert the l2_node into the flow_node so that subsequent flows 566 * with a matching l2 key can use the flow_handle of this flow 567 * as their ref_flow_handle 568 */ 569 flow_node->l2_node = l2_node; 570 list_add(&flow_node->l2_list_node, &l2_node->common_l2_flows); 571 l2_node->refcount++; 572 return 0; 573 } 574 575 /* After the flow parsing is done, this routine is used for checking 576 * if there are any aspects of the flow that prevent it from being 577 * offloaded. 578 */ 579 static bool bnxt_tc_can_offload(struct bnxt *bp, struct bnxt_tc_flow *flow) 580 { 581 /* If L4 ports are specified then ip_proto must be TCP or UDP */ 582 if ((flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) && 583 (flow->l4_key.ip_proto != IPPROTO_TCP && 584 flow->l4_key.ip_proto != IPPROTO_UDP)) { 585 netdev_info(bp->dev, "Cannot offload non-TCP/UDP (%d) ports", 586 flow->l4_key.ip_proto); 587 return false; 588 } 589 590 return true; 591 } 592 593 static int __bnxt_tc_del_flow(struct bnxt *bp, 594 struct bnxt_tc_flow_node *flow_node) 595 { 596 struct bnxt_tc_info *tc_info = &bp->tc_info; 597 int rc; 598 599 /* send HWRM cmd to free the flow-id */ 600 bnxt_hwrm_cfa_flow_free(bp, flow_node->flow_handle); 601 602 mutex_lock(&tc_info->lock); 603 604 /* release reference to l2 node */ 605 bnxt_tc_put_l2_node(bp, flow_node); 606 607 mutex_unlock(&tc_info->lock); 608 609 rc = rhashtable_remove_fast(&tc_info->flow_table, &flow_node->node, 610 tc_info->flow_ht_params); 611 if (rc) 612 netdev_err(bp->dev, "Error: %s: rhashtable_remove_fast rc=%d", 613 __func__, rc); 614 615 kfree_rcu(flow_node, rcu); 616 return 0; 617 } 618 619 /* Add a new flow or replace an existing flow. 620 * Notes on locking: 621 * There are essentially two critical sections here. 622 * 1. while adding a new flow 623 * a) lookup l2-key 624 * b) issue HWRM cmd and get flow_handle 625 * c) link l2-key with flow 626 * 2. while deleting a flow 627 * a) unlinking l2-key from flow 628 * A lock is needed to protect these two critical sections. 629 * 630 * The hash-tables are already protected by the rhashtable API. 631 */ 632 static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid, 633 struct tc_cls_flower_offload *tc_flow_cmd) 634 { 635 struct bnxt_tc_flow_node *new_node, *old_node; 636 struct bnxt_tc_info *tc_info = &bp->tc_info; 637 struct bnxt_tc_flow *flow; 638 __le16 ref_flow_handle; 639 int rc; 640 641 /* allocate memory for the new flow and it's node */ 642 new_node = kzalloc(sizeof(*new_node), GFP_KERNEL); 643 if (!new_node) { 644 rc = -ENOMEM; 645 goto done; 646 } 647 new_node->cookie = tc_flow_cmd->cookie; 648 flow = &new_node->flow; 649 650 rc = bnxt_tc_parse_flow(bp, tc_flow_cmd, flow); 651 if (rc) 652 goto free_node; 653 flow->src_fid = src_fid; 654 655 if (!bnxt_tc_can_offload(bp, flow)) { 656 rc = -ENOSPC; 657 goto free_node; 658 } 659 660 /* If a flow exists with the same cookie, delete it */ 661 old_node = rhashtable_lookup_fast(&tc_info->flow_table, 662 &tc_flow_cmd->cookie, 663 tc_info->flow_ht_params); 664 if (old_node) 665 __bnxt_tc_del_flow(bp, old_node); 666 667 /* Check if the L2 part of the flow has been offloaded already. 668 * If so, bump up it's refcnt and get it's reference handle. 669 */ 670 mutex_lock(&tc_info->lock); 671 rc = bnxt_tc_get_ref_flow_handle(bp, flow, new_node, &ref_flow_handle); 672 if (rc) 673 goto unlock; 674 675 /* send HWRM cmd to alloc the flow */ 676 rc = bnxt_hwrm_cfa_flow_alloc(bp, flow, ref_flow_handle, 677 &new_node->flow_handle); 678 if (rc) 679 goto put_l2; 680 681 /* add new flow to flow-table */ 682 rc = rhashtable_insert_fast(&tc_info->flow_table, &new_node->node, 683 tc_info->flow_ht_params); 684 if (rc) 685 goto hwrm_flow_free; 686 687 mutex_unlock(&tc_info->lock); 688 return 0; 689 690 hwrm_flow_free: 691 bnxt_hwrm_cfa_flow_free(bp, new_node->flow_handle); 692 put_l2: 693 bnxt_tc_put_l2_node(bp, new_node); 694 unlock: 695 mutex_unlock(&tc_info->lock); 696 free_node: 697 kfree(new_node); 698 done: 699 netdev_err(bp->dev, "Error: %s: cookie=0x%lx error=%d", 700 __func__, tc_flow_cmd->cookie, rc); 701 return rc; 702 } 703 704 static int bnxt_tc_del_flow(struct bnxt *bp, 705 struct tc_cls_flower_offload *tc_flow_cmd) 706 { 707 struct bnxt_tc_info *tc_info = &bp->tc_info; 708 struct bnxt_tc_flow_node *flow_node; 709 710 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 711 &tc_flow_cmd->cookie, 712 tc_info->flow_ht_params); 713 if (!flow_node) { 714 netdev_info(bp->dev, "ERROR: no flow_node for cookie %lx", 715 tc_flow_cmd->cookie); 716 return -EINVAL; 717 } 718 719 return __bnxt_tc_del_flow(bp, flow_node); 720 } 721 722 static int bnxt_tc_get_flow_stats(struct bnxt *bp, 723 struct tc_cls_flower_offload *tc_flow_cmd) 724 { 725 struct bnxt_tc_info *tc_info = &bp->tc_info; 726 struct bnxt_tc_flow_node *flow_node; 727 struct bnxt_tc_flow_stats stats; 728 int rc; 729 730 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 731 &tc_flow_cmd->cookie, 732 tc_info->flow_ht_params); 733 if (!flow_node) { 734 netdev_info(bp->dev, "Error: no flow_node for cookie %lx", 735 tc_flow_cmd->cookie); 736 return -1; 737 } 738 739 rc = bnxt_hwrm_cfa_flow_stats_get(bp, flow_node->flow_handle, 740 &flow_node->flow, &stats); 741 if (rc) 742 return rc; 743 744 tcf_exts_stats_update(tc_flow_cmd->exts, stats.bytes, stats.packets, 0); 745 return 0; 746 } 747 748 int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, 749 struct tc_cls_flower_offload *cls_flower) 750 { 751 int rc = 0; 752 753 if (!is_classid_clsact_ingress(cls_flower->common.classid) || 754 cls_flower->common.chain_index) 755 return -EOPNOTSUPP; 756 757 switch (cls_flower->command) { 758 case TC_CLSFLOWER_REPLACE: 759 rc = bnxt_tc_add_flow(bp, src_fid, cls_flower); 760 break; 761 762 case TC_CLSFLOWER_DESTROY: 763 rc = bnxt_tc_del_flow(bp, cls_flower); 764 break; 765 766 case TC_CLSFLOWER_STATS: 767 rc = bnxt_tc_get_flow_stats(bp, cls_flower); 768 break; 769 } 770 return rc; 771 } 772 773 static const struct rhashtable_params bnxt_tc_flow_ht_params = { 774 .head_offset = offsetof(struct bnxt_tc_flow_node, node), 775 .key_offset = offsetof(struct bnxt_tc_flow_node, cookie), 776 .key_len = sizeof(((struct bnxt_tc_flow_node *)0)->cookie), 777 .automatic_shrinking = true 778 }; 779 780 static const struct rhashtable_params bnxt_tc_l2_ht_params = { 781 .head_offset = offsetof(struct bnxt_tc_l2_node, node), 782 .key_offset = offsetof(struct bnxt_tc_l2_node, key), 783 .key_len = BNXT_TC_L2_KEY_LEN, 784 .automatic_shrinking = true 785 }; 786 787 /* convert counter width in bits to a mask */ 788 #define mask(width) ((u64)~0 >> (64 - (width))) 789 790 int bnxt_init_tc(struct bnxt *bp) 791 { 792 struct bnxt_tc_info *tc_info = &bp->tc_info; 793 int rc; 794 795 if (bp->hwrm_spec_code < 0x10800) { 796 netdev_warn(bp->dev, 797 "Firmware does not support TC flower offload.\n"); 798 return -ENOTSUPP; 799 } 800 mutex_init(&tc_info->lock); 801 802 /* Counter widths are programmed by FW */ 803 tc_info->bytes_mask = mask(36); 804 tc_info->packets_mask = mask(28); 805 806 tc_info->flow_ht_params = bnxt_tc_flow_ht_params; 807 rc = rhashtable_init(&tc_info->flow_table, &tc_info->flow_ht_params); 808 if (rc) 809 return rc; 810 811 tc_info->l2_ht_params = bnxt_tc_l2_ht_params; 812 rc = rhashtable_init(&tc_info->l2_table, &tc_info->l2_ht_params); 813 if (rc) 814 goto destroy_flow_table; 815 816 tc_info->enabled = true; 817 bp->dev->hw_features |= NETIF_F_HW_TC; 818 bp->dev->features |= NETIF_F_HW_TC; 819 return 0; 820 821 destroy_flow_table: 822 rhashtable_destroy(&tc_info->flow_table); 823 return rc; 824 } 825 826 void bnxt_shutdown_tc(struct bnxt *bp) 827 { 828 struct bnxt_tc_info *tc_info = &bp->tc_info; 829 830 if (!tc_info->enabled) 831 return; 832 833 rhashtable_destroy(&tc_info->flow_table); 834 rhashtable_destroy(&tc_info->l2_table); 835 } 836 837 #else 838 #endif 839