1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2017 Broadcom Limited 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 */ 9 10 #include <linux/netdevice.h> 11 #include <linux/inetdevice.h> 12 #include <linux/if_vlan.h> 13 #include <net/flow_dissector.h> 14 #include <net/pkt_cls.h> 15 #include <net/tc_act/tc_gact.h> 16 #include <net/tc_act/tc_skbedit.h> 17 #include <net/tc_act/tc_mirred.h> 18 #include <net/tc_act/tc_vlan.h> 19 #include <net/tc_act/tc_tunnel_key.h> 20 21 #include "bnxt_hsi.h" 22 #include "bnxt.h" 23 #include "bnxt_sriov.h" 24 #include "bnxt_tc.h" 25 #include "bnxt_vfr.h" 26 27 #define BNXT_FID_INVALID 0xffff 28 #define VLAN_TCI(vid, prio) ((vid) | ((prio) << VLAN_PRIO_SHIFT)) 29 30 /* Return the dst fid of the func for flow forwarding 31 * For PFs: src_fid is the fid of the PF 32 * For VF-reps: src_fid the fid of the VF 33 */ 34 static u16 bnxt_flow_get_dst_fid(struct bnxt *pf_bp, struct net_device *dev) 35 { 36 struct bnxt *bp; 37 38 /* check if dev belongs to the same switch */ 39 if (!switchdev_port_same_parent_id(pf_bp->dev, dev)) { 40 netdev_info(pf_bp->dev, "dev(ifindex=%d) not on same switch", 41 dev->ifindex); 42 return BNXT_FID_INVALID; 43 } 44 45 /* Is dev a VF-rep? */ 46 if (bnxt_dev_is_vf_rep(dev)) 47 return bnxt_vf_rep_get_fid(dev); 48 49 bp = netdev_priv(dev); 50 return bp->pf.fw_fid; 51 } 52 53 static int bnxt_tc_parse_redir(struct bnxt *bp, 54 struct bnxt_tc_actions *actions, 55 const struct tc_action *tc_act) 56 { 57 struct net_device *dev = tcf_mirred_dev(tc_act); 58 59 if (!dev) { 60 netdev_info(bp->dev, "no dev in mirred action"); 61 return -EINVAL; 62 } 63 64 actions->flags |= BNXT_TC_ACTION_FLAG_FWD; 65 actions->dst_dev = dev; 66 return 0; 67 } 68 69 static void bnxt_tc_parse_vlan(struct bnxt *bp, 70 struct bnxt_tc_actions *actions, 71 const struct tc_action *tc_act) 72 { 73 if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_POP) { 74 actions->flags |= BNXT_TC_ACTION_FLAG_POP_VLAN; 75 } else if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_PUSH) { 76 actions->flags |= BNXT_TC_ACTION_FLAG_PUSH_VLAN; 77 actions->push_vlan_tci = htons(tcf_vlan_push_vid(tc_act)); 78 actions->push_vlan_tpid = tcf_vlan_push_proto(tc_act); 79 } 80 } 81 82 static int bnxt_tc_parse_tunnel_set(struct bnxt *bp, 83 struct bnxt_tc_actions *actions, 84 const struct tc_action *tc_act) 85 { 86 struct ip_tunnel_info *tun_info = tcf_tunnel_info(tc_act); 87 struct ip_tunnel_key *tun_key = &tun_info->key; 88 89 if (ip_tunnel_info_af(tun_info) != AF_INET) { 90 netdev_info(bp->dev, "only IPv4 tunnel-encap is supported"); 91 return -EOPNOTSUPP; 92 } 93 94 actions->tun_encap_key = *tun_key; 95 actions->flags |= BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP; 96 return 0; 97 } 98 99 static int bnxt_tc_parse_actions(struct bnxt *bp, 100 struct bnxt_tc_actions *actions, 101 struct tcf_exts *tc_exts) 102 { 103 const struct tc_action *tc_act; 104 LIST_HEAD(tc_actions); 105 int rc; 106 107 if (!tcf_exts_has_actions(tc_exts)) { 108 netdev_info(bp->dev, "no actions"); 109 return -EINVAL; 110 } 111 112 tcf_exts_to_list(tc_exts, &tc_actions); 113 list_for_each_entry(tc_act, &tc_actions, list) { 114 /* Drop action */ 115 if (is_tcf_gact_shot(tc_act)) { 116 actions->flags |= BNXT_TC_ACTION_FLAG_DROP; 117 return 0; /* don't bother with other actions */ 118 } 119 120 /* Redirect action */ 121 if (is_tcf_mirred_egress_redirect(tc_act)) { 122 rc = bnxt_tc_parse_redir(bp, actions, tc_act); 123 if (rc) 124 return rc; 125 continue; 126 } 127 128 /* Push/pop VLAN */ 129 if (is_tcf_vlan(tc_act)) { 130 bnxt_tc_parse_vlan(bp, actions, tc_act); 131 continue; 132 } 133 134 /* Tunnel encap */ 135 if (is_tcf_tunnel_set(tc_act)) { 136 rc = bnxt_tc_parse_tunnel_set(bp, actions, tc_act); 137 if (rc) 138 return rc; 139 continue; 140 } 141 142 /* Tunnel decap */ 143 if (is_tcf_tunnel_release(tc_act)) { 144 actions->flags |= BNXT_TC_ACTION_FLAG_TUNNEL_DECAP; 145 continue; 146 } 147 } 148 149 if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) { 150 if (actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) { 151 /* dst_fid is PF's fid */ 152 actions->dst_fid = bp->pf.fw_fid; 153 } else { 154 /* find the FID from dst_dev */ 155 actions->dst_fid = 156 bnxt_flow_get_dst_fid(bp, actions->dst_dev); 157 if (actions->dst_fid == BNXT_FID_INVALID) 158 return -EINVAL; 159 } 160 } 161 162 return 0; 163 } 164 165 #define GET_KEY(flow_cmd, key_type) \ 166 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\ 167 (flow_cmd)->key) 168 #define GET_MASK(flow_cmd, key_type) \ 169 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\ 170 (flow_cmd)->mask) 171 172 static int bnxt_tc_parse_flow(struct bnxt *bp, 173 struct tc_cls_flower_offload *tc_flow_cmd, 174 struct bnxt_tc_flow *flow) 175 { 176 struct flow_dissector *dissector = tc_flow_cmd->dissector; 177 u16 addr_type = 0; 178 179 /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */ 180 if ((dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) == 0 || 181 (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) == 0) { 182 netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%x", 183 dissector->used_keys); 184 return -EOPNOTSUPP; 185 } 186 187 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_CONTROL)) { 188 struct flow_dissector_key_control *key = 189 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_CONTROL); 190 191 addr_type = key->addr_type; 192 } 193 194 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC)) { 195 struct flow_dissector_key_basic *key = 196 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC); 197 struct flow_dissector_key_basic *mask = 198 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC); 199 200 flow->l2_key.ether_type = key->n_proto; 201 flow->l2_mask.ether_type = mask->n_proto; 202 203 if (key->n_proto == htons(ETH_P_IP) || 204 key->n_proto == htons(ETH_P_IPV6)) { 205 flow->l4_key.ip_proto = key->ip_proto; 206 flow->l4_mask.ip_proto = mask->ip_proto; 207 } 208 } 209 210 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 211 struct flow_dissector_key_eth_addrs *key = 212 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS); 213 struct flow_dissector_key_eth_addrs *mask = 214 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS); 215 216 flow->flags |= BNXT_TC_FLOW_FLAGS_ETH_ADDRS; 217 ether_addr_copy(flow->l2_key.dmac, key->dst); 218 ether_addr_copy(flow->l2_mask.dmac, mask->dst); 219 ether_addr_copy(flow->l2_key.smac, key->src); 220 ether_addr_copy(flow->l2_mask.smac, mask->src); 221 } 222 223 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN)) { 224 struct flow_dissector_key_vlan *key = 225 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN); 226 struct flow_dissector_key_vlan *mask = 227 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN); 228 229 flow->l2_key.inner_vlan_tci = 230 cpu_to_be16(VLAN_TCI(key->vlan_id, key->vlan_priority)); 231 flow->l2_mask.inner_vlan_tci = 232 cpu_to_be16((VLAN_TCI(mask->vlan_id, mask->vlan_priority))); 233 flow->l2_key.inner_vlan_tpid = htons(ETH_P_8021Q); 234 flow->l2_mask.inner_vlan_tpid = htons(0xffff); 235 flow->l2_key.num_vlans = 1; 236 } 237 238 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 239 struct flow_dissector_key_ipv4_addrs *key = 240 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 241 struct flow_dissector_key_ipv4_addrs *mask = 242 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 243 244 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS; 245 flow->l3_key.ipv4.daddr.s_addr = key->dst; 246 flow->l3_mask.ipv4.daddr.s_addr = mask->dst; 247 flow->l3_key.ipv4.saddr.s_addr = key->src; 248 flow->l3_mask.ipv4.saddr.s_addr = mask->src; 249 } else if (dissector_uses_key(dissector, 250 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 251 struct flow_dissector_key_ipv6_addrs *key = 252 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 253 struct flow_dissector_key_ipv6_addrs *mask = 254 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 255 256 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS; 257 flow->l3_key.ipv6.daddr = key->dst; 258 flow->l3_mask.ipv6.daddr = mask->dst; 259 flow->l3_key.ipv6.saddr = key->src; 260 flow->l3_mask.ipv6.saddr = mask->src; 261 } 262 263 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_PORTS)) { 264 struct flow_dissector_key_ports *key = 265 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS); 266 struct flow_dissector_key_ports *mask = 267 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS); 268 269 flow->flags |= BNXT_TC_FLOW_FLAGS_PORTS; 270 flow->l4_key.ports.dport = key->dst; 271 flow->l4_mask.ports.dport = mask->dst; 272 flow->l4_key.ports.sport = key->src; 273 flow->l4_mask.ports.sport = mask->src; 274 } 275 276 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ICMP)) { 277 struct flow_dissector_key_icmp *key = 278 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP); 279 struct flow_dissector_key_icmp *mask = 280 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP); 281 282 flow->flags |= BNXT_TC_FLOW_FLAGS_ICMP; 283 flow->l4_key.icmp.type = key->type; 284 flow->l4_key.icmp.code = key->code; 285 flow->l4_mask.icmp.type = mask->type; 286 flow->l4_mask.icmp.code = mask->code; 287 } 288 289 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) { 290 struct flow_dissector_key_control *key = 291 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_CONTROL); 292 293 addr_type = key->addr_type; 294 } 295 296 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { 297 struct flow_dissector_key_ipv4_addrs *key = 298 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); 299 struct flow_dissector_key_ipv4_addrs *mask = 300 GET_MASK(tc_flow_cmd, 301 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); 302 303 flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS; 304 flow->tun_key.u.ipv4.dst = key->dst; 305 flow->tun_mask.u.ipv4.dst = mask->dst; 306 flow->tun_key.u.ipv4.src = key->src; 307 flow->tun_mask.u.ipv4.src = mask->src; 308 } else if (dissector_uses_key(dissector, 309 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { 310 return -EOPNOTSUPP; 311 } 312 313 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 314 struct flow_dissector_key_keyid *key = 315 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_KEYID); 316 struct flow_dissector_key_keyid *mask = 317 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_KEYID); 318 319 flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_ID; 320 flow->tun_key.tun_id = key32_to_tunnel_id(key->keyid); 321 flow->tun_mask.tun_id = key32_to_tunnel_id(mask->keyid); 322 } 323 324 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) { 325 struct flow_dissector_key_ports *key = 326 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_PORTS); 327 struct flow_dissector_key_ports *mask = 328 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_PORTS); 329 330 flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_PORTS; 331 flow->tun_key.tp_dst = key->dst; 332 flow->tun_mask.tp_dst = mask->dst; 333 flow->tun_key.tp_src = key->src; 334 flow->tun_mask.tp_src = mask->src; 335 } 336 337 return bnxt_tc_parse_actions(bp, &flow->actions, tc_flow_cmd->exts); 338 } 339 340 static int bnxt_hwrm_cfa_flow_free(struct bnxt *bp, __le16 flow_handle) 341 { 342 struct hwrm_cfa_flow_free_input req = { 0 }; 343 int rc; 344 345 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_FREE, -1, -1); 346 req.flow_handle = flow_handle; 347 348 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 349 if (rc) 350 netdev_info(bp->dev, "Error: %s: flow_handle=0x%x rc=%d", 351 __func__, flow_handle, rc); 352 return rc; 353 } 354 355 static int ipv6_mask_len(struct in6_addr *mask) 356 { 357 int mask_len = 0, i; 358 359 for (i = 0; i < 4; i++) 360 mask_len += inet_mask_len(mask->s6_addr32[i]); 361 362 return mask_len; 363 } 364 365 static bool is_wildcard(void *mask, int len) 366 { 367 const u8 *p = mask; 368 int i; 369 370 for (i = 0; i < len; i++) { 371 if (p[i] != 0) 372 return false; 373 } 374 return true; 375 } 376 377 static int bnxt_hwrm_cfa_flow_alloc(struct bnxt *bp, struct bnxt_tc_flow *flow, 378 __le16 ref_flow_handle, 379 __le32 tunnel_handle, __le16 *flow_handle) 380 { 381 struct hwrm_cfa_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr; 382 struct bnxt_tc_actions *actions = &flow->actions; 383 struct bnxt_tc_l3_key *l3_mask = &flow->l3_mask; 384 struct bnxt_tc_l3_key *l3_key = &flow->l3_key; 385 struct hwrm_cfa_flow_alloc_input req = { 0 }; 386 u16 flow_flags = 0, action_flags = 0; 387 int rc; 388 389 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_ALLOC, -1, -1); 390 391 req.src_fid = cpu_to_le16(flow->src_fid); 392 req.ref_flow_handle = ref_flow_handle; 393 394 if (actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP || 395 actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) { 396 req.tunnel_handle = tunnel_handle; 397 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL; 398 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL; 399 } 400 401 req.ethertype = flow->l2_key.ether_type; 402 req.ip_proto = flow->l4_key.ip_proto; 403 404 if (flow->flags & BNXT_TC_FLOW_FLAGS_ETH_ADDRS) { 405 memcpy(req.dmac, flow->l2_key.dmac, ETH_ALEN); 406 memcpy(req.smac, flow->l2_key.smac, ETH_ALEN); 407 } 408 409 if (flow->l2_key.num_vlans > 0) { 410 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE; 411 /* FW expects the inner_vlan_tci value to be set 412 * in outer_vlan_tci when num_vlans is 1 (which is 413 * always the case in TC.) 414 */ 415 req.outer_vlan_tci = flow->l2_key.inner_vlan_tci; 416 } 417 418 /* If all IP and L4 fields are wildcarded then this is an L2 flow */ 419 if (is_wildcard(l3_mask, sizeof(*l3_mask)) && 420 is_wildcard(&flow->l4_mask, sizeof(flow->l4_mask))) { 421 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2; 422 } else { 423 flow_flags |= flow->l2_key.ether_type == htons(ETH_P_IP) ? 424 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4 : 425 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6; 426 427 if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV4_ADDRS) { 428 req.ip_dst[0] = l3_key->ipv4.daddr.s_addr; 429 req.ip_dst_mask_len = 430 inet_mask_len(l3_mask->ipv4.daddr.s_addr); 431 req.ip_src[0] = l3_key->ipv4.saddr.s_addr; 432 req.ip_src_mask_len = 433 inet_mask_len(l3_mask->ipv4.saddr.s_addr); 434 } else if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV6_ADDRS) { 435 memcpy(req.ip_dst, l3_key->ipv6.daddr.s6_addr32, 436 sizeof(req.ip_dst)); 437 req.ip_dst_mask_len = 438 ipv6_mask_len(&l3_mask->ipv6.daddr); 439 memcpy(req.ip_src, l3_key->ipv6.saddr.s6_addr32, 440 sizeof(req.ip_src)); 441 req.ip_src_mask_len = 442 ipv6_mask_len(&l3_mask->ipv6.saddr); 443 } 444 } 445 446 if (flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) { 447 req.l4_src_port = flow->l4_key.ports.sport; 448 req.l4_src_port_mask = flow->l4_mask.ports.sport; 449 req.l4_dst_port = flow->l4_key.ports.dport; 450 req.l4_dst_port_mask = flow->l4_mask.ports.dport; 451 } else if (flow->flags & BNXT_TC_FLOW_FLAGS_ICMP) { 452 /* l4 ports serve as type/code when ip_proto is ICMP */ 453 req.l4_src_port = htons(flow->l4_key.icmp.type); 454 req.l4_src_port_mask = htons(flow->l4_mask.icmp.type); 455 req.l4_dst_port = htons(flow->l4_key.icmp.code); 456 req.l4_dst_port_mask = htons(flow->l4_mask.icmp.code); 457 } 458 req.flags = cpu_to_le16(flow_flags); 459 460 if (actions->flags & BNXT_TC_ACTION_FLAG_DROP) { 461 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP; 462 } else { 463 if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) { 464 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD; 465 req.dst_fid = cpu_to_le16(actions->dst_fid); 466 } 467 if (actions->flags & BNXT_TC_ACTION_FLAG_PUSH_VLAN) { 468 action_flags |= 469 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE; 470 req.l2_rewrite_vlan_tpid = actions->push_vlan_tpid; 471 req.l2_rewrite_vlan_tci = actions->push_vlan_tci; 472 memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN); 473 memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN); 474 } 475 if (actions->flags & BNXT_TC_ACTION_FLAG_POP_VLAN) { 476 action_flags |= 477 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE; 478 /* Rewrite config with tpid = 0 implies vlan pop */ 479 req.l2_rewrite_vlan_tpid = 0; 480 memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN); 481 memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN); 482 } 483 } 484 req.action_flags = cpu_to_le16(action_flags); 485 486 mutex_lock(&bp->hwrm_cmd_lock); 487 488 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 489 if (!rc) 490 *flow_handle = resp->flow_handle; 491 492 mutex_unlock(&bp->hwrm_cmd_lock); 493 494 return rc; 495 } 496 497 static int hwrm_cfa_decap_filter_alloc(struct bnxt *bp, 498 struct bnxt_tc_flow *flow, 499 struct bnxt_tc_l2_key *l2_info, 500 __le32 ref_decap_handle, 501 __le32 *decap_filter_handle) 502 { 503 struct hwrm_cfa_decap_filter_alloc_output *resp = 504 bp->hwrm_cmd_resp_addr; 505 struct hwrm_cfa_decap_filter_alloc_input req = { 0 }; 506 struct ip_tunnel_key *tun_key = &flow->tun_key; 507 u32 enables = 0; 508 int rc; 509 510 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_DECAP_FILTER_ALLOC, -1, -1); 511 512 req.flags = cpu_to_le32(CFA_DECAP_FILTER_ALLOC_REQ_FLAGS_OVS_TUNNEL); 513 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE | 514 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL; 515 req.tunnel_type = CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN; 516 req.ip_protocol = CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP; 517 518 if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_ID) { 519 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_ID; 520 /* tunnel_id is wrongly defined in hsi defn. as __le32 */ 521 req.tunnel_id = tunnel_id_to_key32(tun_key->tun_id); 522 } 523 524 if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS) { 525 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR; 526 ether_addr_copy(req.dst_macaddr, l2_info->dmac); 527 } 528 if (l2_info->num_vlans) { 529 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_IVLAN_VID; 530 req.t_ivlan_vid = l2_info->inner_vlan_tci; 531 } 532 533 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE; 534 req.ethertype = htons(ETH_P_IP); 535 536 if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS) { 537 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | 538 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | 539 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE; 540 req.ip_addr_type = CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 541 req.dst_ipaddr[0] = tun_key->u.ipv4.dst; 542 req.src_ipaddr[0] = tun_key->u.ipv4.src; 543 } 544 545 if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_PORTS) { 546 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_PORT; 547 req.dst_port = tun_key->tp_dst; 548 } 549 550 /* Eventhough the decap_handle returned by hwrm_cfa_decap_filter_alloc 551 * is defined as __le32, l2_ctxt_ref_id is defined in HSI as __le16. 552 */ 553 req.l2_ctxt_ref_id = (__force __le16)ref_decap_handle; 554 req.enables = cpu_to_le32(enables); 555 556 mutex_lock(&bp->hwrm_cmd_lock); 557 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 558 if (!rc) 559 *decap_filter_handle = resp->decap_filter_id; 560 else 561 netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc); 562 mutex_unlock(&bp->hwrm_cmd_lock); 563 564 return rc; 565 } 566 567 static int hwrm_cfa_decap_filter_free(struct bnxt *bp, 568 __le32 decap_filter_handle) 569 { 570 struct hwrm_cfa_decap_filter_free_input req = { 0 }; 571 int rc; 572 573 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_DECAP_FILTER_FREE, -1, -1); 574 req.decap_filter_id = decap_filter_handle; 575 576 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 577 if (rc) 578 netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc); 579 return rc; 580 } 581 582 static int hwrm_cfa_encap_record_alloc(struct bnxt *bp, 583 struct ip_tunnel_key *encap_key, 584 struct bnxt_tc_l2_key *l2_info, 585 __le32 *encap_record_handle) 586 { 587 struct hwrm_cfa_encap_record_alloc_output *resp = 588 bp->hwrm_cmd_resp_addr; 589 struct hwrm_cfa_encap_record_alloc_input req = { 0 }; 590 struct hwrm_cfa_encap_data_vxlan *encap = 591 (struct hwrm_cfa_encap_data_vxlan *)&req.encap_data; 592 struct hwrm_vxlan_ipv4_hdr *encap_ipv4 = 593 (struct hwrm_vxlan_ipv4_hdr *)encap->l3; 594 int rc; 595 596 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ENCAP_RECORD_ALLOC, -1, -1); 597 598 req.encap_type = CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN; 599 600 ether_addr_copy(encap->dst_mac_addr, l2_info->dmac); 601 ether_addr_copy(encap->src_mac_addr, l2_info->smac); 602 if (l2_info->num_vlans) { 603 encap->num_vlan_tags = l2_info->num_vlans; 604 encap->ovlan_tci = l2_info->inner_vlan_tci; 605 encap->ovlan_tpid = l2_info->inner_vlan_tpid; 606 } 607 608 encap_ipv4->ver_hlen = 4 << VXLAN_IPV4_HDR_VER_HLEN_VERSION_SFT; 609 encap_ipv4->ver_hlen |= 5 << VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_SFT; 610 encap_ipv4->ttl = encap_key->ttl; 611 612 encap_ipv4->dest_ip_addr = encap_key->u.ipv4.dst; 613 encap_ipv4->src_ip_addr = encap_key->u.ipv4.src; 614 encap_ipv4->protocol = IPPROTO_UDP; 615 616 encap->dst_port = encap_key->tp_dst; 617 encap->vni = tunnel_id_to_key32(encap_key->tun_id); 618 619 mutex_lock(&bp->hwrm_cmd_lock); 620 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 621 if (!rc) 622 *encap_record_handle = resp->encap_record_id; 623 else 624 netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc); 625 mutex_unlock(&bp->hwrm_cmd_lock); 626 627 return rc; 628 } 629 630 static int hwrm_cfa_encap_record_free(struct bnxt *bp, 631 __le32 encap_record_handle) 632 { 633 struct hwrm_cfa_encap_record_free_input req = { 0 }; 634 int rc; 635 636 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ENCAP_RECORD_FREE, -1, -1); 637 req.encap_record_id = encap_record_handle; 638 639 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 640 if (rc) 641 netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc); 642 return rc; 643 } 644 645 static int bnxt_tc_put_l2_node(struct bnxt *bp, 646 struct bnxt_tc_flow_node *flow_node) 647 { 648 struct bnxt_tc_l2_node *l2_node = flow_node->l2_node; 649 struct bnxt_tc_info *tc_info = bp->tc_info; 650 int rc; 651 652 /* remove flow_node from the L2 shared flow list */ 653 list_del(&flow_node->l2_list_node); 654 if (--l2_node->refcount == 0) { 655 rc = rhashtable_remove_fast(&tc_info->l2_table, &l2_node->node, 656 tc_info->l2_ht_params); 657 if (rc) 658 netdev_err(bp->dev, 659 "Error: %s: rhashtable_remove_fast: %d", 660 __func__, rc); 661 kfree_rcu(l2_node, rcu); 662 } 663 return 0; 664 } 665 666 static struct bnxt_tc_l2_node * 667 bnxt_tc_get_l2_node(struct bnxt *bp, struct rhashtable *l2_table, 668 struct rhashtable_params ht_params, 669 struct bnxt_tc_l2_key *l2_key) 670 { 671 struct bnxt_tc_l2_node *l2_node; 672 int rc; 673 674 l2_node = rhashtable_lookup_fast(l2_table, l2_key, ht_params); 675 if (!l2_node) { 676 l2_node = kzalloc(sizeof(*l2_node), GFP_KERNEL); 677 if (!l2_node) { 678 rc = -ENOMEM; 679 return NULL; 680 } 681 682 l2_node->key = *l2_key; 683 rc = rhashtable_insert_fast(l2_table, &l2_node->node, 684 ht_params); 685 if (rc) { 686 kfree_rcu(l2_node, rcu); 687 netdev_err(bp->dev, 688 "Error: %s: rhashtable_insert_fast: %d", 689 __func__, rc); 690 return NULL; 691 } 692 INIT_LIST_HEAD(&l2_node->common_l2_flows); 693 } 694 return l2_node; 695 } 696 697 /* Get the ref_flow_handle for a flow by checking if there are any other 698 * flows that share the same L2 key as this flow. 699 */ 700 static int 701 bnxt_tc_get_ref_flow_handle(struct bnxt *bp, struct bnxt_tc_flow *flow, 702 struct bnxt_tc_flow_node *flow_node, 703 __le16 *ref_flow_handle) 704 { 705 struct bnxt_tc_info *tc_info = bp->tc_info; 706 struct bnxt_tc_flow_node *ref_flow_node; 707 struct bnxt_tc_l2_node *l2_node; 708 709 l2_node = bnxt_tc_get_l2_node(bp, &tc_info->l2_table, 710 tc_info->l2_ht_params, 711 &flow->l2_key); 712 if (!l2_node) 713 return -1; 714 715 /* If any other flow is using this l2_node, use it's flow_handle 716 * as the ref_flow_handle 717 */ 718 if (l2_node->refcount > 0) { 719 ref_flow_node = list_first_entry(&l2_node->common_l2_flows, 720 struct bnxt_tc_flow_node, 721 l2_list_node); 722 *ref_flow_handle = ref_flow_node->flow_handle; 723 } else { 724 *ref_flow_handle = cpu_to_le16(0xffff); 725 } 726 727 /* Insert the l2_node into the flow_node so that subsequent flows 728 * with a matching l2 key can use the flow_handle of this flow 729 * as their ref_flow_handle 730 */ 731 flow_node->l2_node = l2_node; 732 list_add(&flow_node->l2_list_node, &l2_node->common_l2_flows); 733 l2_node->refcount++; 734 return 0; 735 } 736 737 /* After the flow parsing is done, this routine is used for checking 738 * if there are any aspects of the flow that prevent it from being 739 * offloaded. 740 */ 741 static bool bnxt_tc_can_offload(struct bnxt *bp, struct bnxt_tc_flow *flow) 742 { 743 /* If L4 ports are specified then ip_proto must be TCP or UDP */ 744 if ((flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) && 745 (flow->l4_key.ip_proto != IPPROTO_TCP && 746 flow->l4_key.ip_proto != IPPROTO_UDP)) { 747 netdev_info(bp->dev, "Cannot offload non-TCP/UDP (%d) ports", 748 flow->l4_key.ip_proto); 749 return false; 750 } 751 752 return true; 753 } 754 755 /* Returns the final refcount of the node on success 756 * or a -ve error code on failure 757 */ 758 static int bnxt_tc_put_tunnel_node(struct bnxt *bp, 759 struct rhashtable *tunnel_table, 760 struct rhashtable_params *ht_params, 761 struct bnxt_tc_tunnel_node *tunnel_node) 762 { 763 int rc; 764 765 if (--tunnel_node->refcount == 0) { 766 rc = rhashtable_remove_fast(tunnel_table, &tunnel_node->node, 767 *ht_params); 768 if (rc) { 769 netdev_err(bp->dev, "rhashtable_remove_fast rc=%d", rc); 770 rc = -1; 771 } 772 kfree_rcu(tunnel_node, rcu); 773 return rc; 774 } else { 775 return tunnel_node->refcount; 776 } 777 } 778 779 /* Get (or add) either encap or decap tunnel node from/to the supplied 780 * hash table. 781 */ 782 static struct bnxt_tc_tunnel_node * 783 bnxt_tc_get_tunnel_node(struct bnxt *bp, struct rhashtable *tunnel_table, 784 struct rhashtable_params *ht_params, 785 struct ip_tunnel_key *tun_key) 786 { 787 struct bnxt_tc_tunnel_node *tunnel_node; 788 int rc; 789 790 tunnel_node = rhashtable_lookup_fast(tunnel_table, tun_key, *ht_params); 791 if (!tunnel_node) { 792 tunnel_node = kzalloc(sizeof(*tunnel_node), GFP_KERNEL); 793 if (!tunnel_node) { 794 rc = -ENOMEM; 795 goto err; 796 } 797 798 tunnel_node->key = *tun_key; 799 tunnel_node->tunnel_handle = INVALID_TUNNEL_HANDLE; 800 rc = rhashtable_insert_fast(tunnel_table, &tunnel_node->node, 801 *ht_params); 802 if (rc) { 803 kfree_rcu(tunnel_node, rcu); 804 goto err; 805 } 806 } 807 tunnel_node->refcount++; 808 return tunnel_node; 809 err: 810 netdev_info(bp->dev, "error rc=%d", rc); 811 return NULL; 812 } 813 814 static int bnxt_tc_get_ref_decap_handle(struct bnxt *bp, 815 struct bnxt_tc_flow *flow, 816 struct bnxt_tc_l2_key *l2_key, 817 struct bnxt_tc_flow_node *flow_node, 818 __le32 *ref_decap_handle) 819 { 820 struct bnxt_tc_info *tc_info = bp->tc_info; 821 struct bnxt_tc_flow_node *ref_flow_node; 822 struct bnxt_tc_l2_node *decap_l2_node; 823 824 decap_l2_node = bnxt_tc_get_l2_node(bp, &tc_info->decap_l2_table, 825 tc_info->decap_l2_ht_params, 826 l2_key); 827 if (!decap_l2_node) 828 return -1; 829 830 /* If any other flow is using this decap_l2_node, use it's decap_handle 831 * as the ref_decap_handle 832 */ 833 if (decap_l2_node->refcount > 0) { 834 ref_flow_node = 835 list_first_entry(&decap_l2_node->common_l2_flows, 836 struct bnxt_tc_flow_node, 837 decap_l2_list_node); 838 *ref_decap_handle = ref_flow_node->decap_node->tunnel_handle; 839 } else { 840 *ref_decap_handle = INVALID_TUNNEL_HANDLE; 841 } 842 843 /* Insert the l2_node into the flow_node so that subsequent flows 844 * with a matching decap l2 key can use the decap_filter_handle of 845 * this flow as their ref_decap_handle 846 */ 847 flow_node->decap_l2_node = decap_l2_node; 848 list_add(&flow_node->decap_l2_list_node, 849 &decap_l2_node->common_l2_flows); 850 decap_l2_node->refcount++; 851 return 0; 852 } 853 854 static void bnxt_tc_put_decap_l2_node(struct bnxt *bp, 855 struct bnxt_tc_flow_node *flow_node) 856 { 857 struct bnxt_tc_l2_node *decap_l2_node = flow_node->decap_l2_node; 858 struct bnxt_tc_info *tc_info = bp->tc_info; 859 int rc; 860 861 /* remove flow_node from the decap L2 sharing flow list */ 862 list_del(&flow_node->decap_l2_list_node); 863 if (--decap_l2_node->refcount == 0) { 864 rc = rhashtable_remove_fast(&tc_info->decap_l2_table, 865 &decap_l2_node->node, 866 tc_info->decap_l2_ht_params); 867 if (rc) 868 netdev_err(bp->dev, "rhashtable_remove_fast rc=%d", rc); 869 kfree_rcu(decap_l2_node, rcu); 870 } 871 } 872 873 static void bnxt_tc_put_decap_handle(struct bnxt *bp, 874 struct bnxt_tc_flow_node *flow_node) 875 { 876 __le32 decap_handle = flow_node->decap_node->tunnel_handle; 877 struct bnxt_tc_info *tc_info = bp->tc_info; 878 int rc; 879 880 if (flow_node->decap_l2_node) 881 bnxt_tc_put_decap_l2_node(bp, flow_node); 882 883 rc = bnxt_tc_put_tunnel_node(bp, &tc_info->decap_table, 884 &tc_info->decap_ht_params, 885 flow_node->decap_node); 886 if (!rc && decap_handle != INVALID_TUNNEL_HANDLE) 887 hwrm_cfa_decap_filter_free(bp, decap_handle); 888 } 889 890 static int bnxt_tc_resolve_tunnel_hdrs(struct bnxt *bp, 891 struct ip_tunnel_key *tun_key, 892 struct bnxt_tc_l2_key *l2_info) 893 { 894 #ifdef CONFIG_INET 895 struct net_device *real_dst_dev = bp->dev; 896 struct flowi4 flow = { {0} }; 897 struct net_device *dst_dev; 898 struct neighbour *nbr; 899 struct rtable *rt; 900 int rc; 901 902 flow.flowi4_proto = IPPROTO_UDP; 903 flow.fl4_dport = tun_key->tp_dst; 904 flow.daddr = tun_key->u.ipv4.dst; 905 906 rt = ip_route_output_key(dev_net(real_dst_dev), &flow); 907 if (IS_ERR(rt)) { 908 netdev_info(bp->dev, "no route to %pI4b", &flow.daddr); 909 return -EOPNOTSUPP; 910 } 911 912 /* The route must either point to the real_dst_dev or a dst_dev that 913 * uses the real_dst_dev. 914 */ 915 dst_dev = rt->dst.dev; 916 if (is_vlan_dev(dst_dev)) { 917 #if IS_ENABLED(CONFIG_VLAN_8021Q) 918 struct vlan_dev_priv *vlan = vlan_dev_priv(dst_dev); 919 920 if (vlan->real_dev != real_dst_dev) { 921 netdev_info(bp->dev, 922 "dst_dev(%s) doesn't use PF-if(%s)", 923 netdev_name(dst_dev), 924 netdev_name(real_dst_dev)); 925 rc = -EOPNOTSUPP; 926 goto put_rt; 927 } 928 l2_info->inner_vlan_tci = htons(vlan->vlan_id); 929 l2_info->inner_vlan_tpid = vlan->vlan_proto; 930 l2_info->num_vlans = 1; 931 #endif 932 } else if (dst_dev != real_dst_dev) { 933 netdev_info(bp->dev, 934 "dst_dev(%s) for %pI4b is not PF-if(%s)", 935 netdev_name(dst_dev), &flow.daddr, 936 netdev_name(real_dst_dev)); 937 rc = -EOPNOTSUPP; 938 goto put_rt; 939 } 940 941 nbr = dst_neigh_lookup(&rt->dst, &flow.daddr); 942 if (!nbr) { 943 netdev_info(bp->dev, "can't lookup neighbor for %pI4b", 944 &flow.daddr); 945 rc = -EOPNOTSUPP; 946 goto put_rt; 947 } 948 949 tun_key->u.ipv4.src = flow.saddr; 950 tun_key->ttl = ip4_dst_hoplimit(&rt->dst); 951 neigh_ha_snapshot(l2_info->dmac, nbr, dst_dev); 952 ether_addr_copy(l2_info->smac, dst_dev->dev_addr); 953 neigh_release(nbr); 954 ip_rt_put(rt); 955 956 return 0; 957 put_rt: 958 ip_rt_put(rt); 959 return rc; 960 #else 961 return -EOPNOTSUPP; 962 #endif 963 } 964 965 static int bnxt_tc_get_decap_handle(struct bnxt *bp, struct bnxt_tc_flow *flow, 966 struct bnxt_tc_flow_node *flow_node, 967 __le32 *decap_filter_handle) 968 { 969 struct ip_tunnel_key *decap_key = &flow->tun_key; 970 struct bnxt_tc_info *tc_info = bp->tc_info; 971 struct bnxt_tc_l2_key l2_info = { {0} }; 972 struct bnxt_tc_tunnel_node *decap_node; 973 struct ip_tunnel_key tun_key = { 0 }; 974 struct bnxt_tc_l2_key *decap_l2_info; 975 __le32 ref_decap_handle; 976 int rc; 977 978 /* Check if there's another flow using the same tunnel decap. 979 * If not, add this tunnel to the table and resolve the other 980 * tunnel header fileds 981 */ 982 decap_node = bnxt_tc_get_tunnel_node(bp, &tc_info->decap_table, 983 &tc_info->decap_ht_params, 984 decap_key); 985 if (!decap_node) 986 return -ENOMEM; 987 988 flow_node->decap_node = decap_node; 989 990 if (decap_node->tunnel_handle != INVALID_TUNNEL_HANDLE) 991 goto done; 992 993 /* Resolve the L2 fields for tunnel decap 994 * Resolve the route for remote vtep (saddr) of the decap key 995 * Find it's next-hop mac addrs 996 */ 997 tun_key.u.ipv4.dst = flow->tun_key.u.ipv4.src; 998 tun_key.tp_dst = flow->tun_key.tp_dst; 999 rc = bnxt_tc_resolve_tunnel_hdrs(bp, &tun_key, &l2_info); 1000 if (rc) 1001 goto put_decap; 1002 1003 decap_l2_info = &decap_node->l2_info; 1004 /* decap smac is wildcarded */ 1005 ether_addr_copy(decap_l2_info->dmac, l2_info.smac); 1006 if (l2_info.num_vlans) { 1007 decap_l2_info->num_vlans = l2_info.num_vlans; 1008 decap_l2_info->inner_vlan_tpid = l2_info.inner_vlan_tpid; 1009 decap_l2_info->inner_vlan_tci = l2_info.inner_vlan_tci; 1010 } 1011 flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS; 1012 1013 /* For getting a decap_filter_handle we first need to check if 1014 * there are any other decap flows that share the same tunnel L2 1015 * key and if so, pass that flow's decap_filter_handle as the 1016 * ref_decap_handle for this flow. 1017 */ 1018 rc = bnxt_tc_get_ref_decap_handle(bp, flow, decap_l2_info, flow_node, 1019 &ref_decap_handle); 1020 if (rc) 1021 goto put_decap; 1022 1023 /* Issue the hwrm cmd to allocate a decap filter handle */ 1024 rc = hwrm_cfa_decap_filter_alloc(bp, flow, decap_l2_info, 1025 ref_decap_handle, 1026 &decap_node->tunnel_handle); 1027 if (rc) 1028 goto put_decap_l2; 1029 1030 done: 1031 *decap_filter_handle = decap_node->tunnel_handle; 1032 return 0; 1033 1034 put_decap_l2: 1035 bnxt_tc_put_decap_l2_node(bp, flow_node); 1036 put_decap: 1037 bnxt_tc_put_tunnel_node(bp, &tc_info->decap_table, 1038 &tc_info->decap_ht_params, 1039 flow_node->decap_node); 1040 return rc; 1041 } 1042 1043 static void bnxt_tc_put_encap_handle(struct bnxt *bp, 1044 struct bnxt_tc_tunnel_node *encap_node) 1045 { 1046 __le32 encap_handle = encap_node->tunnel_handle; 1047 struct bnxt_tc_info *tc_info = bp->tc_info; 1048 int rc; 1049 1050 rc = bnxt_tc_put_tunnel_node(bp, &tc_info->encap_table, 1051 &tc_info->encap_ht_params, encap_node); 1052 if (!rc && encap_handle != INVALID_TUNNEL_HANDLE) 1053 hwrm_cfa_encap_record_free(bp, encap_handle); 1054 } 1055 1056 /* Lookup the tunnel encap table and check if there's an encap_handle 1057 * alloc'd already. 1058 * If not, query L2 info via a route lookup and issue an encap_record_alloc 1059 * cmd to FW. 1060 */ 1061 static int bnxt_tc_get_encap_handle(struct bnxt *bp, struct bnxt_tc_flow *flow, 1062 struct bnxt_tc_flow_node *flow_node, 1063 __le32 *encap_handle) 1064 { 1065 struct ip_tunnel_key *encap_key = &flow->actions.tun_encap_key; 1066 struct bnxt_tc_info *tc_info = bp->tc_info; 1067 struct bnxt_tc_tunnel_node *encap_node; 1068 int rc; 1069 1070 /* Check if there's another flow using the same tunnel encap. 1071 * If not, add this tunnel to the table and resolve the other 1072 * tunnel header fileds 1073 */ 1074 encap_node = bnxt_tc_get_tunnel_node(bp, &tc_info->encap_table, 1075 &tc_info->encap_ht_params, 1076 encap_key); 1077 if (!encap_node) 1078 return -ENOMEM; 1079 1080 flow_node->encap_node = encap_node; 1081 1082 if (encap_node->tunnel_handle != INVALID_TUNNEL_HANDLE) 1083 goto done; 1084 1085 rc = bnxt_tc_resolve_tunnel_hdrs(bp, encap_key, &encap_node->l2_info); 1086 if (rc) 1087 goto put_encap; 1088 1089 /* Allocate a new tunnel encap record */ 1090 rc = hwrm_cfa_encap_record_alloc(bp, encap_key, &encap_node->l2_info, 1091 &encap_node->tunnel_handle); 1092 if (rc) 1093 goto put_encap; 1094 1095 done: 1096 *encap_handle = encap_node->tunnel_handle; 1097 return 0; 1098 1099 put_encap: 1100 bnxt_tc_put_tunnel_node(bp, &tc_info->encap_table, 1101 &tc_info->encap_ht_params, encap_node); 1102 return rc; 1103 } 1104 1105 static void bnxt_tc_put_tunnel_handle(struct bnxt *bp, 1106 struct bnxt_tc_flow *flow, 1107 struct bnxt_tc_flow_node *flow_node) 1108 { 1109 if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP) 1110 bnxt_tc_put_decap_handle(bp, flow_node); 1111 else if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) 1112 bnxt_tc_put_encap_handle(bp, flow_node->encap_node); 1113 } 1114 1115 static int bnxt_tc_get_tunnel_handle(struct bnxt *bp, 1116 struct bnxt_tc_flow *flow, 1117 struct bnxt_tc_flow_node *flow_node, 1118 __le32 *tunnel_handle) 1119 { 1120 if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP) 1121 return bnxt_tc_get_decap_handle(bp, flow, flow_node, 1122 tunnel_handle); 1123 else if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) 1124 return bnxt_tc_get_encap_handle(bp, flow, flow_node, 1125 tunnel_handle); 1126 else 1127 return 0; 1128 } 1129 static int __bnxt_tc_del_flow(struct bnxt *bp, 1130 struct bnxt_tc_flow_node *flow_node) 1131 { 1132 struct bnxt_tc_info *tc_info = bp->tc_info; 1133 int rc; 1134 1135 /* send HWRM cmd to free the flow-id */ 1136 bnxt_hwrm_cfa_flow_free(bp, flow_node->flow_handle); 1137 1138 mutex_lock(&tc_info->lock); 1139 1140 /* release references to any tunnel encap/decap nodes */ 1141 bnxt_tc_put_tunnel_handle(bp, &flow_node->flow, flow_node); 1142 1143 /* release reference to l2 node */ 1144 bnxt_tc_put_l2_node(bp, flow_node); 1145 1146 mutex_unlock(&tc_info->lock); 1147 1148 rc = rhashtable_remove_fast(&tc_info->flow_table, &flow_node->node, 1149 tc_info->flow_ht_params); 1150 if (rc) 1151 netdev_err(bp->dev, "Error: %s: rhashtable_remove_fast rc=%d", 1152 __func__, rc); 1153 1154 kfree_rcu(flow_node, rcu); 1155 return 0; 1156 } 1157 1158 static void bnxt_tc_set_src_fid(struct bnxt *bp, struct bnxt_tc_flow *flow, 1159 u16 src_fid) 1160 { 1161 if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP) 1162 flow->src_fid = bp->pf.fw_fid; 1163 else 1164 flow->src_fid = src_fid; 1165 } 1166 1167 /* Add a new flow or replace an existing flow. 1168 * Notes on locking: 1169 * There are essentially two critical sections here. 1170 * 1. while adding a new flow 1171 * a) lookup l2-key 1172 * b) issue HWRM cmd and get flow_handle 1173 * c) link l2-key with flow 1174 * 2. while deleting a flow 1175 * a) unlinking l2-key from flow 1176 * A lock is needed to protect these two critical sections. 1177 * 1178 * The hash-tables are already protected by the rhashtable API. 1179 */ 1180 static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid, 1181 struct tc_cls_flower_offload *tc_flow_cmd) 1182 { 1183 struct bnxt_tc_flow_node *new_node, *old_node; 1184 struct bnxt_tc_info *tc_info = bp->tc_info; 1185 struct bnxt_tc_flow *flow; 1186 __le32 tunnel_handle = 0; 1187 __le16 ref_flow_handle; 1188 int rc; 1189 1190 /* allocate memory for the new flow and it's node */ 1191 new_node = kzalloc(sizeof(*new_node), GFP_KERNEL); 1192 if (!new_node) { 1193 rc = -ENOMEM; 1194 goto done; 1195 } 1196 new_node->cookie = tc_flow_cmd->cookie; 1197 flow = &new_node->flow; 1198 1199 rc = bnxt_tc_parse_flow(bp, tc_flow_cmd, flow); 1200 if (rc) 1201 goto free_node; 1202 1203 bnxt_tc_set_src_fid(bp, flow, src_fid); 1204 1205 if (!bnxt_tc_can_offload(bp, flow)) { 1206 rc = -ENOSPC; 1207 goto free_node; 1208 } 1209 1210 /* If a flow exists with the same cookie, delete it */ 1211 old_node = rhashtable_lookup_fast(&tc_info->flow_table, 1212 &tc_flow_cmd->cookie, 1213 tc_info->flow_ht_params); 1214 if (old_node) 1215 __bnxt_tc_del_flow(bp, old_node); 1216 1217 /* Check if the L2 part of the flow has been offloaded already. 1218 * If so, bump up it's refcnt and get it's reference handle. 1219 */ 1220 mutex_lock(&tc_info->lock); 1221 rc = bnxt_tc_get_ref_flow_handle(bp, flow, new_node, &ref_flow_handle); 1222 if (rc) 1223 goto unlock; 1224 1225 /* If the flow involves tunnel encap/decap, get tunnel_handle */ 1226 rc = bnxt_tc_get_tunnel_handle(bp, flow, new_node, &tunnel_handle); 1227 if (rc) 1228 goto put_l2; 1229 1230 /* send HWRM cmd to alloc the flow */ 1231 rc = bnxt_hwrm_cfa_flow_alloc(bp, flow, ref_flow_handle, 1232 tunnel_handle, &new_node->flow_handle); 1233 if (rc) 1234 goto put_tunnel; 1235 1236 flow->lastused = jiffies; 1237 spin_lock_init(&flow->stats_lock); 1238 /* add new flow to flow-table */ 1239 rc = rhashtable_insert_fast(&tc_info->flow_table, &new_node->node, 1240 tc_info->flow_ht_params); 1241 if (rc) 1242 goto hwrm_flow_free; 1243 1244 mutex_unlock(&tc_info->lock); 1245 return 0; 1246 1247 hwrm_flow_free: 1248 bnxt_hwrm_cfa_flow_free(bp, new_node->flow_handle); 1249 put_tunnel: 1250 bnxt_tc_put_tunnel_handle(bp, flow, new_node); 1251 put_l2: 1252 bnxt_tc_put_l2_node(bp, new_node); 1253 unlock: 1254 mutex_unlock(&tc_info->lock); 1255 free_node: 1256 kfree_rcu(new_node, rcu); 1257 done: 1258 netdev_err(bp->dev, "Error: %s: cookie=0x%lx error=%d", 1259 __func__, tc_flow_cmd->cookie, rc); 1260 return rc; 1261 } 1262 1263 static int bnxt_tc_del_flow(struct bnxt *bp, 1264 struct tc_cls_flower_offload *tc_flow_cmd) 1265 { 1266 struct bnxt_tc_info *tc_info = bp->tc_info; 1267 struct bnxt_tc_flow_node *flow_node; 1268 1269 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 1270 &tc_flow_cmd->cookie, 1271 tc_info->flow_ht_params); 1272 if (!flow_node) { 1273 netdev_info(bp->dev, "ERROR: no flow_node for cookie %lx", 1274 tc_flow_cmd->cookie); 1275 return -EINVAL; 1276 } 1277 1278 return __bnxt_tc_del_flow(bp, flow_node); 1279 } 1280 1281 static int bnxt_tc_get_flow_stats(struct bnxt *bp, 1282 struct tc_cls_flower_offload *tc_flow_cmd) 1283 { 1284 struct bnxt_tc_flow_stats stats, *curr_stats, *prev_stats; 1285 struct bnxt_tc_info *tc_info = bp->tc_info; 1286 struct bnxt_tc_flow_node *flow_node; 1287 struct bnxt_tc_flow *flow; 1288 unsigned long lastused; 1289 1290 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 1291 &tc_flow_cmd->cookie, 1292 tc_info->flow_ht_params); 1293 if (!flow_node) { 1294 netdev_info(bp->dev, "Error: no flow_node for cookie %lx", 1295 tc_flow_cmd->cookie); 1296 return -1; 1297 } 1298 1299 flow = &flow_node->flow; 1300 curr_stats = &flow->stats; 1301 prev_stats = &flow->prev_stats; 1302 1303 spin_lock(&flow->stats_lock); 1304 stats.packets = curr_stats->packets - prev_stats->packets; 1305 stats.bytes = curr_stats->bytes - prev_stats->bytes; 1306 *prev_stats = *curr_stats; 1307 lastused = flow->lastused; 1308 spin_unlock(&flow->stats_lock); 1309 1310 tcf_exts_stats_update(tc_flow_cmd->exts, stats.bytes, stats.packets, 1311 lastused); 1312 return 0; 1313 } 1314 1315 static int 1316 bnxt_hwrm_cfa_flow_stats_get(struct bnxt *bp, int num_flows, 1317 struct bnxt_tc_stats_batch stats_batch[]) 1318 { 1319 struct hwrm_cfa_flow_stats_output *resp = bp->hwrm_cmd_resp_addr; 1320 struct hwrm_cfa_flow_stats_input req = { 0 }; 1321 __le16 *req_flow_handles = &req.flow_handle_0; 1322 int rc, i; 1323 1324 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_STATS, -1, -1); 1325 req.num_flows = cpu_to_le16(num_flows); 1326 for (i = 0; i < num_flows; i++) { 1327 struct bnxt_tc_flow_node *flow_node = stats_batch[i].flow_node; 1328 1329 req_flow_handles[i] = flow_node->flow_handle; 1330 } 1331 1332 mutex_lock(&bp->hwrm_cmd_lock); 1333 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 1334 if (!rc) { 1335 __le64 *resp_packets = &resp->packet_0; 1336 __le64 *resp_bytes = &resp->byte_0; 1337 1338 for (i = 0; i < num_flows; i++) { 1339 stats_batch[i].hw_stats.packets = 1340 le64_to_cpu(resp_packets[i]); 1341 stats_batch[i].hw_stats.bytes = 1342 le64_to_cpu(resp_bytes[i]); 1343 } 1344 } else { 1345 netdev_info(bp->dev, "error rc=%d", rc); 1346 } 1347 1348 mutex_unlock(&bp->hwrm_cmd_lock); 1349 return rc; 1350 } 1351 1352 /* Add val to accum while handling a possible wraparound 1353 * of val. Eventhough val is of type u64, its actual width 1354 * is denoted by mask and will wrap-around beyond that width. 1355 */ 1356 static void accumulate_val(u64 *accum, u64 val, u64 mask) 1357 { 1358 #define low_bits(x, mask) ((x) & (mask)) 1359 #define high_bits(x, mask) ((x) & ~(mask)) 1360 bool wrapped = val < low_bits(*accum, mask); 1361 1362 *accum = high_bits(*accum, mask) + val; 1363 if (wrapped) 1364 *accum += (mask + 1); 1365 } 1366 1367 /* The HW counters' width is much less than 64bits. 1368 * Handle possible wrap-around while updating the stat counters 1369 */ 1370 static void bnxt_flow_stats_accum(struct bnxt_tc_info *tc_info, 1371 struct bnxt_tc_flow_stats *acc_stats, 1372 struct bnxt_tc_flow_stats *hw_stats) 1373 { 1374 accumulate_val(&acc_stats->bytes, hw_stats->bytes, tc_info->bytes_mask); 1375 accumulate_val(&acc_stats->packets, hw_stats->packets, 1376 tc_info->packets_mask); 1377 } 1378 1379 static int 1380 bnxt_tc_flow_stats_batch_update(struct bnxt *bp, int num_flows, 1381 struct bnxt_tc_stats_batch stats_batch[]) 1382 { 1383 struct bnxt_tc_info *tc_info = bp->tc_info; 1384 int rc, i; 1385 1386 rc = bnxt_hwrm_cfa_flow_stats_get(bp, num_flows, stats_batch); 1387 if (rc) 1388 return rc; 1389 1390 for (i = 0; i < num_flows; i++) { 1391 struct bnxt_tc_flow_node *flow_node = stats_batch[i].flow_node; 1392 struct bnxt_tc_flow *flow = &flow_node->flow; 1393 1394 spin_lock(&flow->stats_lock); 1395 bnxt_flow_stats_accum(tc_info, &flow->stats, 1396 &stats_batch[i].hw_stats); 1397 if (flow->stats.packets != flow->prev_stats.packets) 1398 flow->lastused = jiffies; 1399 spin_unlock(&flow->stats_lock); 1400 } 1401 1402 return 0; 1403 } 1404 1405 static int 1406 bnxt_tc_flow_stats_batch_prep(struct bnxt *bp, 1407 struct bnxt_tc_stats_batch stats_batch[], 1408 int *num_flows) 1409 { 1410 struct bnxt_tc_info *tc_info = bp->tc_info; 1411 struct rhashtable_iter *iter = &tc_info->iter; 1412 void *flow_node; 1413 int rc, i; 1414 1415 rhashtable_walk_start(iter); 1416 1417 rc = 0; 1418 for (i = 0; i < BNXT_FLOW_STATS_BATCH_MAX; i++) { 1419 flow_node = rhashtable_walk_next(iter); 1420 if (IS_ERR(flow_node)) { 1421 i = 0; 1422 if (PTR_ERR(flow_node) == -EAGAIN) { 1423 continue; 1424 } else { 1425 rc = PTR_ERR(flow_node); 1426 goto done; 1427 } 1428 } 1429 1430 /* No more flows */ 1431 if (!flow_node) 1432 goto done; 1433 1434 stats_batch[i].flow_node = flow_node; 1435 } 1436 done: 1437 rhashtable_walk_stop(iter); 1438 *num_flows = i; 1439 return rc; 1440 } 1441 1442 void bnxt_tc_flow_stats_work(struct bnxt *bp) 1443 { 1444 struct bnxt_tc_info *tc_info = bp->tc_info; 1445 int num_flows, rc; 1446 1447 num_flows = atomic_read(&tc_info->flow_table.nelems); 1448 if (!num_flows) 1449 return; 1450 1451 rhashtable_walk_enter(&tc_info->flow_table, &tc_info->iter); 1452 1453 for (;;) { 1454 rc = bnxt_tc_flow_stats_batch_prep(bp, tc_info->stats_batch, 1455 &num_flows); 1456 if (rc) { 1457 if (rc == -EAGAIN) 1458 continue; 1459 break; 1460 } 1461 1462 if (!num_flows) 1463 break; 1464 1465 bnxt_tc_flow_stats_batch_update(bp, num_flows, 1466 tc_info->stats_batch); 1467 } 1468 1469 rhashtable_walk_exit(&tc_info->iter); 1470 } 1471 1472 int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, 1473 struct tc_cls_flower_offload *cls_flower) 1474 { 1475 int rc = 0; 1476 1477 switch (cls_flower->command) { 1478 case TC_CLSFLOWER_REPLACE: 1479 rc = bnxt_tc_add_flow(bp, src_fid, cls_flower); 1480 break; 1481 1482 case TC_CLSFLOWER_DESTROY: 1483 rc = bnxt_tc_del_flow(bp, cls_flower); 1484 break; 1485 1486 case TC_CLSFLOWER_STATS: 1487 rc = bnxt_tc_get_flow_stats(bp, cls_flower); 1488 break; 1489 } 1490 return rc; 1491 } 1492 1493 static const struct rhashtable_params bnxt_tc_flow_ht_params = { 1494 .head_offset = offsetof(struct bnxt_tc_flow_node, node), 1495 .key_offset = offsetof(struct bnxt_tc_flow_node, cookie), 1496 .key_len = sizeof(((struct bnxt_tc_flow_node *)0)->cookie), 1497 .automatic_shrinking = true 1498 }; 1499 1500 static const struct rhashtable_params bnxt_tc_l2_ht_params = { 1501 .head_offset = offsetof(struct bnxt_tc_l2_node, node), 1502 .key_offset = offsetof(struct bnxt_tc_l2_node, key), 1503 .key_len = BNXT_TC_L2_KEY_LEN, 1504 .automatic_shrinking = true 1505 }; 1506 1507 static const struct rhashtable_params bnxt_tc_decap_l2_ht_params = { 1508 .head_offset = offsetof(struct bnxt_tc_l2_node, node), 1509 .key_offset = offsetof(struct bnxt_tc_l2_node, key), 1510 .key_len = BNXT_TC_L2_KEY_LEN, 1511 .automatic_shrinking = true 1512 }; 1513 1514 static const struct rhashtable_params bnxt_tc_tunnel_ht_params = { 1515 .head_offset = offsetof(struct bnxt_tc_tunnel_node, node), 1516 .key_offset = offsetof(struct bnxt_tc_tunnel_node, key), 1517 .key_len = sizeof(struct ip_tunnel_key), 1518 .automatic_shrinking = true 1519 }; 1520 1521 /* convert counter width in bits to a mask */ 1522 #define mask(width) ((u64)~0 >> (64 - (width))) 1523 1524 int bnxt_init_tc(struct bnxt *bp) 1525 { 1526 struct bnxt_tc_info *tc_info; 1527 int rc; 1528 1529 if (bp->hwrm_spec_code < 0x10803) { 1530 netdev_warn(bp->dev, 1531 "Firmware does not support TC flower offload.\n"); 1532 return -ENOTSUPP; 1533 } 1534 1535 tc_info = kzalloc(sizeof(*tc_info), GFP_KERNEL); 1536 if (!tc_info) 1537 return -ENOMEM; 1538 mutex_init(&tc_info->lock); 1539 1540 /* Counter widths are programmed by FW */ 1541 tc_info->bytes_mask = mask(36); 1542 tc_info->packets_mask = mask(28); 1543 1544 tc_info->flow_ht_params = bnxt_tc_flow_ht_params; 1545 rc = rhashtable_init(&tc_info->flow_table, &tc_info->flow_ht_params); 1546 if (rc) 1547 goto free_tc_info; 1548 1549 tc_info->l2_ht_params = bnxt_tc_l2_ht_params; 1550 rc = rhashtable_init(&tc_info->l2_table, &tc_info->l2_ht_params); 1551 if (rc) 1552 goto destroy_flow_table; 1553 1554 tc_info->decap_l2_ht_params = bnxt_tc_decap_l2_ht_params; 1555 rc = rhashtable_init(&tc_info->decap_l2_table, 1556 &tc_info->decap_l2_ht_params); 1557 if (rc) 1558 goto destroy_l2_table; 1559 1560 tc_info->decap_ht_params = bnxt_tc_tunnel_ht_params; 1561 rc = rhashtable_init(&tc_info->decap_table, 1562 &tc_info->decap_ht_params); 1563 if (rc) 1564 goto destroy_decap_l2_table; 1565 1566 tc_info->encap_ht_params = bnxt_tc_tunnel_ht_params; 1567 rc = rhashtable_init(&tc_info->encap_table, 1568 &tc_info->encap_ht_params); 1569 if (rc) 1570 goto destroy_decap_table; 1571 1572 tc_info->enabled = true; 1573 bp->dev->hw_features |= NETIF_F_HW_TC; 1574 bp->dev->features |= NETIF_F_HW_TC; 1575 bp->tc_info = tc_info; 1576 return 0; 1577 1578 destroy_decap_table: 1579 rhashtable_destroy(&tc_info->decap_table); 1580 destroy_decap_l2_table: 1581 rhashtable_destroy(&tc_info->decap_l2_table); 1582 destroy_l2_table: 1583 rhashtable_destroy(&tc_info->l2_table); 1584 destroy_flow_table: 1585 rhashtable_destroy(&tc_info->flow_table); 1586 free_tc_info: 1587 kfree(tc_info); 1588 return rc; 1589 } 1590 1591 void bnxt_shutdown_tc(struct bnxt *bp) 1592 { 1593 struct bnxt_tc_info *tc_info = bp->tc_info; 1594 1595 if (!bnxt_tc_flower_enabled(bp)) 1596 return; 1597 1598 rhashtable_destroy(&tc_info->flow_table); 1599 rhashtable_destroy(&tc_info->l2_table); 1600 rhashtable_destroy(&tc_info->decap_l2_table); 1601 rhashtable_destroy(&tc_info->decap_table); 1602 rhashtable_destroy(&tc_info->encap_table); 1603 kfree(tc_info); 1604 bp->tc_info = NULL; 1605 } 1606