1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2023, Advanced Micro Devices, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include "tc_encap_actions.h" 12 #include "tc.h" 13 #include "mae.h" 14 #include <net/vxlan.h> 15 #include <net/geneve.h> 16 #include <net/netevent.h> 17 #include <net/arp.h> 18 19 static const struct rhashtable_params efx_neigh_ht_params = { 20 .key_len = offsetof(struct efx_neigh_binder, ha), 21 .key_offset = 0, 22 .head_offset = offsetof(struct efx_neigh_binder, linkage), 23 }; 24 25 static const struct rhashtable_params efx_tc_encap_ht_params = { 26 .key_len = offsetofend(struct efx_tc_encap_action, key), 27 .key_offset = 0, 28 .head_offset = offsetof(struct efx_tc_encap_action, linkage), 29 }; 30 31 static void efx_tc_encap_free(void *ptr, void *__unused) 32 { 33 struct efx_tc_encap_action *enc = ptr; 34 35 WARN_ON(refcount_read(&enc->ref)); 36 kfree(enc); 37 } 38 39 static void efx_neigh_free(void *ptr, void *__unused) 40 { 41 struct efx_neigh_binder *neigh = ptr; 42 43 WARN_ON(refcount_read(&neigh->ref)); 44 WARN_ON(!list_empty(&neigh->users)); 45 put_net_track(neigh->net, &neigh->ns_tracker); 46 netdev_put(neigh->egdev, &neigh->dev_tracker); 47 kfree(neigh); 48 } 49 50 int efx_tc_init_encap_actions(struct efx_nic *efx) 51 { 52 int rc; 53 54 rc = rhashtable_init(&efx->tc->neigh_ht, &efx_neigh_ht_params); 55 if (rc < 0) 56 goto fail_neigh_ht; 57 rc = rhashtable_init(&efx->tc->encap_ht, &efx_tc_encap_ht_params); 58 if (rc < 0) 59 goto fail_encap_ht; 60 return 0; 61 fail_encap_ht: 62 rhashtable_destroy(&efx->tc->neigh_ht); 63 fail_neigh_ht: 64 return rc; 65 } 66 67 /* Only call this in init failure teardown. 68 * Normal exit should fini instead as there may be entries in the table. 69 */ 70 void efx_tc_destroy_encap_actions(struct efx_nic *efx) 71 { 72 rhashtable_destroy(&efx->tc->encap_ht); 73 rhashtable_destroy(&efx->tc->neigh_ht); 74 } 75 76 void efx_tc_fini_encap_actions(struct efx_nic *efx) 77 { 78 rhashtable_free_and_destroy(&efx->tc->encap_ht, efx_tc_encap_free, NULL); 79 rhashtable_free_and_destroy(&efx->tc->neigh_ht, efx_neigh_free, NULL); 80 } 81 82 static void efx_neigh_update(struct work_struct *work); 83 84 static int efx_bind_neigh(struct efx_nic *efx, 85 struct efx_tc_encap_action *encap, struct net *net, 86 struct netlink_ext_ack *extack) 87 { 88 struct efx_neigh_binder *neigh, *old; 89 struct flowi6 flow6 = {}; 90 struct flowi4 flow4 = {}; 91 int rc; 92 93 /* GCC stupidly thinks that only values explicitly listed in the enum 94 * definition can _possibly_ be sensible case values, so without this 95 * cast it complains about the IPv6 versions. 96 */ 97 switch ((int)encap->type) { 98 case EFX_ENCAP_TYPE_VXLAN: 99 case EFX_ENCAP_TYPE_GENEVE: 100 flow4.flowi4_proto = IPPROTO_UDP; 101 flow4.fl4_dport = encap->key.tp_dst; 102 flow4.flowi4_tos = encap->key.tos; 103 flow4.daddr = encap->key.u.ipv4.dst; 104 flow4.saddr = encap->key.u.ipv4.src; 105 break; 106 case EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6: 107 case EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6: 108 flow6.flowi6_proto = IPPROTO_UDP; 109 flow6.fl6_dport = encap->key.tp_dst; 110 flow6.flowlabel = ip6_make_flowinfo(encap->key.tos, 111 encap->key.label); 112 flow6.daddr = encap->key.u.ipv6.dst; 113 flow6.saddr = encap->key.u.ipv6.src; 114 break; 115 default: 116 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported encap type %d", 117 (int)encap->type); 118 return -EOPNOTSUPP; 119 } 120 121 neigh = kzalloc(sizeof(*neigh), GFP_KERNEL_ACCOUNT); 122 if (!neigh) 123 return -ENOMEM; 124 neigh->net = get_net_track(net, &neigh->ns_tracker, GFP_KERNEL_ACCOUNT); 125 neigh->dst_ip = flow4.daddr; 126 neigh->dst_ip6 = flow6.daddr; 127 128 old = rhashtable_lookup_get_insert_fast(&efx->tc->neigh_ht, 129 &neigh->linkage, 130 efx_neigh_ht_params); 131 if (old) { 132 /* don't need our new entry */ 133 put_net_track(neigh->net, &neigh->ns_tracker); 134 kfree(neigh); 135 if (!refcount_inc_not_zero(&old->ref)) 136 return -EAGAIN; 137 /* existing entry found, ref taken */ 138 neigh = old; 139 } else { 140 /* New entry. We need to initiate a lookup */ 141 struct neighbour *n; 142 struct rtable *rt; 143 144 if (encap->type & EFX_ENCAP_FLAG_IPV6) { 145 #if IS_ENABLED(CONFIG_IPV6) 146 struct dst_entry *dst; 147 148 dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &flow6, 149 NULL); 150 rc = PTR_ERR_OR_ZERO(dst); 151 if (rc) { 152 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup route for IPv6 encap"); 153 goto out_free; 154 } 155 neigh->egdev = dst->dev; 156 netdev_hold(neigh->egdev, &neigh->dev_tracker, 157 GFP_KERNEL_ACCOUNT); 158 neigh->ttl = ip6_dst_hoplimit(dst); 159 n = dst_neigh_lookup(dst, &flow6.daddr); 160 dst_release(dst); 161 #else 162 /* We shouldn't ever get here, because if IPv6 isn't 163 * enabled how did someone create an IPv6 tunnel_key? 164 */ 165 rc = -EOPNOTSUPP; 166 NL_SET_ERR_MSG_MOD(extack, "No IPv6 support (neigh bind)"); 167 #endif 168 } else { 169 rt = ip_route_output_key(net, &flow4); 170 if (IS_ERR_OR_NULL(rt)) { 171 rc = PTR_ERR_OR_ZERO(rt); 172 if (!rc) 173 rc = -EIO; 174 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup route for encap"); 175 goto out_free; 176 } 177 neigh->egdev = rt->dst.dev; 178 netdev_hold(neigh->egdev, &neigh->dev_tracker, 179 GFP_KERNEL_ACCOUNT); 180 neigh->ttl = ip4_dst_hoplimit(&rt->dst); 181 n = dst_neigh_lookup(&rt->dst, &flow4.daddr); 182 ip_rt_put(rt); 183 } 184 if (!n) { 185 rc = -ENETUNREACH; 186 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup neighbour for encap"); 187 netdev_put(neigh->egdev, &neigh->dev_tracker); 188 goto out_free; 189 } 190 refcount_set(&neigh->ref, 1); 191 INIT_LIST_HEAD(&neigh->users); 192 read_lock_bh(&n->lock); 193 ether_addr_copy(neigh->ha, n->ha); 194 neigh->n_valid = n->nud_state & NUD_VALID; 195 read_unlock_bh(&n->lock); 196 rwlock_init(&neigh->lock); 197 INIT_WORK(&neigh->work, efx_neigh_update); 198 neigh->efx = efx; 199 neigh->used = jiffies; 200 if (!neigh->n_valid) 201 /* Prod ARP to find us a neighbour */ 202 neigh_event_send(n, NULL); 203 neigh_release(n); 204 } 205 /* Add us to this neigh */ 206 encap->neigh = neigh; 207 list_add_tail(&encap->list, &neigh->users); 208 return 0; 209 210 out_free: 211 /* cleanup common to several error paths */ 212 rhashtable_remove_fast(&efx->tc->neigh_ht, &neigh->linkage, 213 efx_neigh_ht_params); 214 synchronize_rcu(); 215 put_net_track(net, &neigh->ns_tracker); 216 kfree(neigh); 217 return rc; 218 } 219 220 static void efx_free_neigh(struct efx_neigh_binder *neigh) 221 { 222 struct efx_nic *efx = neigh->efx; 223 224 rhashtable_remove_fast(&efx->tc->neigh_ht, &neigh->linkage, 225 efx_neigh_ht_params); 226 synchronize_rcu(); 227 netdev_put(neigh->egdev, &neigh->dev_tracker); 228 put_net_track(neigh->net, &neigh->ns_tracker); 229 kfree(neigh); 230 } 231 232 static void efx_release_neigh(struct efx_nic *efx, 233 struct efx_tc_encap_action *encap) 234 { 235 struct efx_neigh_binder *neigh = encap->neigh; 236 237 if (!neigh) 238 return; 239 list_del(&encap->list); 240 encap->neigh = NULL; 241 if (!refcount_dec_and_test(&neigh->ref)) 242 return; /* still in use */ 243 efx_free_neigh(neigh); 244 } 245 246 static void efx_gen_tun_header_eth(struct efx_tc_encap_action *encap, u16 proto) 247 { 248 struct efx_neigh_binder *neigh = encap->neigh; 249 struct ethhdr *eth; 250 251 encap->encap_hdr_len = sizeof(*eth); 252 eth = (struct ethhdr *)encap->encap_hdr; 253 254 if (encap->neigh->n_valid) 255 ether_addr_copy(eth->h_dest, neigh->ha); 256 else 257 eth_zero_addr(eth->h_dest); 258 ether_addr_copy(eth->h_source, neigh->egdev->dev_addr); 259 eth->h_proto = htons(proto); 260 } 261 262 static void efx_gen_tun_header_ipv4(struct efx_tc_encap_action *encap, u8 ipproto, u8 len) 263 { 264 struct efx_neigh_binder *neigh = encap->neigh; 265 struct ip_tunnel_key *key = &encap->key; 266 struct iphdr *ip; 267 268 ip = (struct iphdr *)(encap->encap_hdr + encap->encap_hdr_len); 269 encap->encap_hdr_len += sizeof(*ip); 270 271 ip->daddr = key->u.ipv4.dst; 272 ip->saddr = key->u.ipv4.src; 273 ip->ttl = neigh->ttl; 274 ip->protocol = ipproto; 275 ip->version = 0x4; 276 ip->ihl = 0x5; 277 ip->tot_len = cpu_to_be16(ip->ihl * 4 + len); 278 ip_send_check(ip); 279 } 280 281 #ifdef CONFIG_IPV6 282 static void efx_gen_tun_header_ipv6(struct efx_tc_encap_action *encap, u8 ipproto, u8 len) 283 { 284 struct efx_neigh_binder *neigh = encap->neigh; 285 struct ip_tunnel_key *key = &encap->key; 286 struct ipv6hdr *ip; 287 288 ip = (struct ipv6hdr *)(encap->encap_hdr + encap->encap_hdr_len); 289 encap->encap_hdr_len += sizeof(*ip); 290 291 ip6_flow_hdr(ip, key->tos, key->label); 292 ip->daddr = key->u.ipv6.dst; 293 ip->saddr = key->u.ipv6.src; 294 ip->hop_limit = neigh->ttl; 295 ip->nexthdr = ipproto; 296 ip->version = 0x6; 297 ip->payload_len = cpu_to_be16(len); 298 } 299 #endif 300 301 static void efx_gen_tun_header_udp(struct efx_tc_encap_action *encap, u8 len) 302 { 303 struct ip_tunnel_key *key = &encap->key; 304 struct udphdr *udp; 305 306 udp = (struct udphdr *)(encap->encap_hdr + encap->encap_hdr_len); 307 encap->encap_hdr_len += sizeof(*udp); 308 309 udp->dest = key->tp_dst; 310 udp->len = cpu_to_be16(sizeof(*udp) + len); 311 } 312 313 static void efx_gen_tun_header_vxlan(struct efx_tc_encap_action *encap) 314 { 315 struct ip_tunnel_key *key = &encap->key; 316 struct vxlanhdr *vxlan; 317 318 vxlan = (struct vxlanhdr *)(encap->encap_hdr + encap->encap_hdr_len); 319 encap->encap_hdr_len += sizeof(*vxlan); 320 321 vxlan->vx_flags = VXLAN_HF_VNI; 322 vxlan->vx_vni = vxlan_vni_field(tunnel_id_to_key32(key->tun_id)); 323 } 324 325 static void efx_gen_tun_header_geneve(struct efx_tc_encap_action *encap) 326 { 327 struct ip_tunnel_key *key = &encap->key; 328 struct genevehdr *geneve; 329 u32 vni; 330 331 geneve = (struct genevehdr *)(encap->encap_hdr + encap->encap_hdr_len); 332 encap->encap_hdr_len += sizeof(*geneve); 333 334 geneve->proto_type = htons(ETH_P_TEB); 335 /* convert tun_id to host-endian so we can use host arithmetic to 336 * extract individual bytes. 337 */ 338 vni = ntohl(tunnel_id_to_key32(key->tun_id)); 339 geneve->vni[0] = vni >> 16; 340 geneve->vni[1] = vni >> 8; 341 geneve->vni[2] = vni; 342 } 343 344 #define vxlan_header_l4_len (sizeof(struct udphdr) + sizeof(struct vxlanhdr)) 345 #define vxlan4_header_len (sizeof(struct ethhdr) + sizeof(struct iphdr) + vxlan_header_l4_len) 346 static void efx_gen_vxlan_header_ipv4(struct efx_tc_encap_action *encap) 347 { 348 BUILD_BUG_ON(sizeof(encap->encap_hdr) < vxlan4_header_len); 349 efx_gen_tun_header_eth(encap, ETH_P_IP); 350 efx_gen_tun_header_ipv4(encap, IPPROTO_UDP, vxlan_header_l4_len); 351 efx_gen_tun_header_udp(encap, sizeof(struct vxlanhdr)); 352 efx_gen_tun_header_vxlan(encap); 353 } 354 355 #define geneve_header_l4_len (sizeof(struct udphdr) + sizeof(struct genevehdr)) 356 #define geneve4_header_len (sizeof(struct ethhdr) + sizeof(struct iphdr) + geneve_header_l4_len) 357 static void efx_gen_geneve_header_ipv4(struct efx_tc_encap_action *encap) 358 { 359 BUILD_BUG_ON(sizeof(encap->encap_hdr) < geneve4_header_len); 360 efx_gen_tun_header_eth(encap, ETH_P_IP); 361 efx_gen_tun_header_ipv4(encap, IPPROTO_UDP, geneve_header_l4_len); 362 efx_gen_tun_header_udp(encap, sizeof(struct genevehdr)); 363 efx_gen_tun_header_geneve(encap); 364 } 365 366 #ifdef CONFIG_IPV6 367 #define vxlan6_header_len (sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + vxlan_header_l4_len) 368 static void efx_gen_vxlan_header_ipv6(struct efx_tc_encap_action *encap) 369 { 370 BUILD_BUG_ON(sizeof(encap->encap_hdr) < vxlan6_header_len); 371 efx_gen_tun_header_eth(encap, ETH_P_IPV6); 372 efx_gen_tun_header_ipv6(encap, IPPROTO_UDP, vxlan_header_l4_len); 373 efx_gen_tun_header_udp(encap, sizeof(struct vxlanhdr)); 374 efx_gen_tun_header_vxlan(encap); 375 } 376 377 #define geneve6_header_len (sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + geneve_header_l4_len) 378 static void efx_gen_geneve_header_ipv6(struct efx_tc_encap_action *encap) 379 { 380 BUILD_BUG_ON(sizeof(encap->encap_hdr) < geneve6_header_len); 381 efx_gen_tun_header_eth(encap, ETH_P_IPV6); 382 efx_gen_tun_header_ipv6(encap, IPPROTO_UDP, geneve_header_l4_len); 383 efx_gen_tun_header_udp(encap, sizeof(struct genevehdr)); 384 efx_gen_tun_header_geneve(encap); 385 } 386 #endif 387 388 static void efx_gen_encap_header(struct efx_nic *efx, 389 struct efx_tc_encap_action *encap) 390 { 391 encap->n_valid = encap->neigh->n_valid; 392 393 /* GCC stupidly thinks that only values explicitly listed in the enum 394 * definition can _possibly_ be sensible case values, so without this 395 * cast it complains about the IPv6 versions. 396 */ 397 switch ((int)encap->type) { 398 case EFX_ENCAP_TYPE_VXLAN: 399 efx_gen_vxlan_header_ipv4(encap); 400 break; 401 case EFX_ENCAP_TYPE_GENEVE: 402 efx_gen_geneve_header_ipv4(encap); 403 break; 404 #ifdef CONFIG_IPV6 405 case EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6: 406 efx_gen_vxlan_header_ipv6(encap); 407 break; 408 case EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6: 409 efx_gen_geneve_header_ipv6(encap); 410 break; 411 #endif 412 default: 413 /* unhandled encap type, can't happen */ 414 if (net_ratelimit()) 415 netif_err(efx, drv, efx->net_dev, 416 "Bogus encap type %d, can't generate\n", 417 encap->type); 418 419 /* Use fallback action. */ 420 encap->n_valid = false; 421 break; 422 } 423 } 424 425 static void efx_tc_update_encap(struct efx_nic *efx, 426 struct efx_tc_encap_action *encap) 427 { 428 struct efx_tc_action_set_list *acts, *fallback; 429 struct efx_tc_flow_rule *rule; 430 struct efx_tc_action_set *act; 431 int rc; 432 433 if (encap->n_valid) { 434 /* Make sure no rules are using this encap while we change it */ 435 list_for_each_entry(act, &encap->users, encap_user) { 436 acts = act->user; 437 if (WARN_ON(!acts)) /* can't happen */ 438 continue; 439 rule = container_of(acts, struct efx_tc_flow_rule, acts); 440 if (rule->fallback) 441 fallback = rule->fallback; 442 else /* fallback fallback: deliver to PF */ 443 fallback = &efx->tc->facts.pf; 444 rc = efx_mae_update_rule(efx, fallback->fw_id, 445 rule->fw_id); 446 if (rc) 447 netif_err(efx, drv, efx->net_dev, 448 "Failed to update (f) rule %08x rc %d\n", 449 rule->fw_id, rc); 450 else 451 netif_dbg(efx, drv, efx->net_dev, "Updated (f) rule %08x\n", 452 rule->fw_id); 453 } 454 } 455 456 /* Make sure we don't leak arbitrary bytes on the wire; 457 * set an all-0s ethernet header. A successful call to 458 * efx_gen_encap_header() will overwrite this. 459 */ 460 memset(encap->encap_hdr, 0, sizeof(encap->encap_hdr)); 461 encap->encap_hdr_len = ETH_HLEN; 462 463 if (encap->neigh) { 464 read_lock_bh(&encap->neigh->lock); 465 efx_gen_encap_header(efx, encap); 466 read_unlock_bh(&encap->neigh->lock); 467 } else { 468 encap->n_valid = false; 469 } 470 471 rc = efx_mae_update_encap_md(efx, encap); 472 if (rc) { 473 netif_err(efx, drv, efx->net_dev, 474 "Failed to update encap hdr %08x rc %d\n", 475 encap->fw_id, rc); 476 return; 477 } 478 netif_dbg(efx, drv, efx->net_dev, "Updated encap hdr %08x\n", 479 encap->fw_id); 480 if (!encap->n_valid) 481 return; 482 /* Update rule users: use the action if they are now ready */ 483 list_for_each_entry(act, &encap->users, encap_user) { 484 acts = act->user; 485 if (WARN_ON(!acts)) /* can't happen */ 486 continue; 487 rule = container_of(acts, struct efx_tc_flow_rule, acts); 488 if (!efx_tc_check_ready(efx, rule)) 489 continue; 490 rc = efx_mae_update_rule(efx, acts->fw_id, rule->fw_id); 491 if (rc) 492 netif_err(efx, drv, efx->net_dev, 493 "Failed to update rule %08x rc %d\n", 494 rule->fw_id, rc); 495 else 496 netif_dbg(efx, drv, efx->net_dev, "Updated rule %08x\n", 497 rule->fw_id); 498 } 499 } 500 501 static void efx_neigh_update(struct work_struct *work) 502 { 503 struct efx_neigh_binder *neigh = container_of(work, struct efx_neigh_binder, work); 504 struct efx_tc_encap_action *encap; 505 struct efx_nic *efx = neigh->efx; 506 507 mutex_lock(&efx->tc->mutex); 508 list_for_each_entry(encap, &neigh->users, list) 509 efx_tc_update_encap(neigh->efx, encap); 510 /* release ref taken in efx_neigh_event() */ 511 if (refcount_dec_and_test(&neigh->ref)) 512 efx_free_neigh(neigh); 513 mutex_unlock(&efx->tc->mutex); 514 } 515 516 static int efx_neigh_event(struct efx_nic *efx, struct neighbour *n) 517 { 518 struct efx_neigh_binder keys = {NULL}, *neigh; 519 bool n_valid, ipv6 = false; 520 char ha[ETH_ALEN]; 521 size_t keysize; 522 523 if (WARN_ON(!efx->tc)) 524 return NOTIFY_DONE; 525 526 if (n->tbl == &arp_tbl) { 527 keysize = sizeof(keys.dst_ip); 528 #if IS_ENABLED(CONFIG_IPV6) 529 } else if (n->tbl == ipv6_stub->nd_tbl) { 530 ipv6 = true; 531 keysize = sizeof(keys.dst_ip6); 532 #endif 533 } else { 534 return NOTIFY_DONE; 535 } 536 if (!n->parms) { 537 netif_warn(efx, drv, efx->net_dev, "neigh_event with no parms!\n"); 538 return NOTIFY_DONE; 539 } 540 keys.net = read_pnet(&n->parms->net); 541 if (n->tbl->key_len != keysize) { 542 netif_warn(efx, drv, efx->net_dev, "neigh_event with bad key_len %u\n", 543 n->tbl->key_len); 544 return NOTIFY_DONE; 545 } 546 read_lock_bh(&n->lock); /* Get a consistent view */ 547 memcpy(ha, n->ha, ETH_ALEN); 548 n_valid = (n->nud_state & NUD_VALID) && !n->dead; 549 read_unlock_bh(&n->lock); 550 if (ipv6) 551 memcpy(&keys.dst_ip6, n->primary_key, n->tbl->key_len); 552 else 553 memcpy(&keys.dst_ip, n->primary_key, n->tbl->key_len); 554 rcu_read_lock(); 555 neigh = rhashtable_lookup_fast(&efx->tc->neigh_ht, &keys, 556 efx_neigh_ht_params); 557 if (!neigh || neigh->dying) 558 /* We're not interested in this neighbour */ 559 goto done; 560 write_lock_bh(&neigh->lock); 561 if (n_valid == neigh->n_valid && !memcmp(ha, neigh->ha, ETH_ALEN)) { 562 write_unlock_bh(&neigh->lock); 563 /* Nothing has changed; no work to do */ 564 goto done; 565 } 566 neigh->n_valid = n_valid; 567 memcpy(neigh->ha, ha, ETH_ALEN); 568 write_unlock_bh(&neigh->lock); 569 if (refcount_inc_not_zero(&neigh->ref)) { 570 rcu_read_unlock(); 571 if (!schedule_work(&neigh->work)) 572 /* failed to schedule, release the ref we just took */ 573 if (refcount_dec_and_test(&neigh->ref)) 574 efx_free_neigh(neigh); 575 } else { 576 done: 577 rcu_read_unlock(); 578 } 579 return NOTIFY_DONE; 580 } 581 582 bool efx_tc_check_ready(struct efx_nic *efx, struct efx_tc_flow_rule *rule) 583 { 584 struct efx_tc_action_set *act; 585 586 /* Encap actions can only be offloaded if they have valid 587 * neighbour info for the outer Ethernet header. 588 */ 589 list_for_each_entry(act, &rule->acts.list, list) 590 if (act->encap_md && !act->encap_md->n_valid) 591 return false; 592 return true; 593 } 594 595 struct efx_tc_encap_action *efx_tc_flower_create_encap_md( 596 struct efx_nic *efx, const struct ip_tunnel_info *info, 597 struct net_device *egdev, struct netlink_ext_ack *extack) 598 { 599 enum efx_encap_type type = efx_tc_indr_netdev_type(egdev); 600 struct efx_tc_encap_action *encap, *old; 601 struct efx_rep *to_efv; 602 s64 rc; 603 604 if (type == EFX_ENCAP_TYPE_NONE) { 605 /* dest is not an encap device */ 606 NL_SET_ERR_MSG_MOD(extack, "Not a (supported) tunnel device but tunnel_key is set"); 607 return ERR_PTR(-EOPNOTSUPP); 608 } 609 rc = efx_mae_check_encap_type_supported(efx, type); 610 if (rc < 0) { 611 NL_SET_ERR_MSG_MOD(extack, "Firmware reports no support for this tunnel type"); 612 return ERR_PTR(rc); 613 } 614 /* No support yet for Geneve options */ 615 if (info->options_len) { 616 NL_SET_ERR_MSG_MOD(extack, "Unsupported tunnel options"); 617 return ERR_PTR(-EOPNOTSUPP); 618 } 619 switch (info->mode) { 620 case IP_TUNNEL_INFO_TX: 621 break; 622 case IP_TUNNEL_INFO_TX | IP_TUNNEL_INFO_IPV6: 623 type |= EFX_ENCAP_FLAG_IPV6; 624 break; 625 default: 626 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported tunnel mode %u", 627 info->mode); 628 return ERR_PTR(-EOPNOTSUPP); 629 } 630 encap = kzalloc(sizeof(*encap), GFP_KERNEL_ACCOUNT); 631 if (!encap) 632 return ERR_PTR(-ENOMEM); 633 encap->type = type; 634 encap->key = info->key; 635 INIT_LIST_HEAD(&encap->users); 636 old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_ht, 637 &encap->linkage, 638 efx_tc_encap_ht_params); 639 if (old) { 640 /* don't need our new entry */ 641 kfree(encap); 642 if (!refcount_inc_not_zero(&old->ref)) 643 return ERR_PTR(-EAGAIN); 644 /* existing entry found, ref taken */ 645 return old; 646 } 647 648 rc = efx_bind_neigh(efx, encap, dev_net(egdev), extack); 649 if (rc < 0) 650 goto out_remove; 651 to_efv = efx_tc_flower_lookup_efv(efx, encap->neigh->egdev); 652 if (IS_ERR(to_efv)) { 653 /* neigh->egdev isn't ours */ 654 NL_SET_ERR_MSG_MOD(extack, "Tunnel egress device not on switch"); 655 rc = PTR_ERR(to_efv); 656 goto out_release; 657 } 658 rc = efx_tc_flower_external_mport(efx, to_efv); 659 if (rc < 0) { 660 NL_SET_ERR_MSG_MOD(extack, "Failed to identify tunnel egress m-port"); 661 goto out_release; 662 } 663 encap->dest_mport = rc; 664 read_lock_bh(&encap->neigh->lock); 665 efx_gen_encap_header(efx, encap); 666 read_unlock_bh(&encap->neigh->lock); 667 668 rc = efx_mae_allocate_encap_md(efx, encap); 669 if (rc < 0) { 670 NL_SET_ERR_MSG_MOD(extack, "Failed to write tunnel header to hw"); 671 goto out_release; 672 } 673 674 /* ref and return */ 675 refcount_set(&encap->ref, 1); 676 return encap; 677 out_release: 678 efx_release_neigh(efx, encap); 679 out_remove: 680 rhashtable_remove_fast(&efx->tc->encap_ht, &encap->linkage, 681 efx_tc_encap_ht_params); 682 kfree(encap); 683 return ERR_PTR(rc); 684 } 685 686 void efx_tc_flower_release_encap_md(struct efx_nic *efx, 687 struct efx_tc_encap_action *encap) 688 { 689 if (!refcount_dec_and_test(&encap->ref)) 690 return; /* still in use */ 691 efx_release_neigh(efx, encap); 692 rhashtable_remove_fast(&efx->tc->encap_ht, &encap->linkage, 693 efx_tc_encap_ht_params); 694 efx_mae_free_encap_md(efx, encap); 695 kfree(encap); 696 } 697 698 static void efx_tc_remove_neigh_users(struct efx_nic *efx, struct efx_neigh_binder *neigh) 699 { 700 struct efx_tc_encap_action *encap, *next; 701 702 list_for_each_entry_safe(encap, next, &neigh->users, list) { 703 /* Should cause neigh usage count to fall to zero, freeing it */ 704 efx_release_neigh(efx, encap); 705 /* The encap has lost its neigh, so it's now unready */ 706 efx_tc_update_encap(efx, encap); 707 } 708 } 709 710 void efx_tc_unregister_egdev(struct efx_nic *efx, struct net_device *net_dev) 711 { 712 struct efx_neigh_binder *neigh; 713 struct rhashtable_iter walk; 714 715 mutex_lock(&efx->tc->mutex); 716 rhashtable_walk_enter(&efx->tc->neigh_ht, &walk); 717 rhashtable_walk_start(&walk); 718 while ((neigh = rhashtable_walk_next(&walk)) != NULL) { 719 if (IS_ERR(neigh)) 720 continue; 721 if (neigh->egdev != net_dev) 722 continue; 723 neigh->dying = true; 724 rhashtable_walk_stop(&walk); 725 synchronize_rcu(); /* Make sure any updates see dying flag */ 726 efx_tc_remove_neigh_users(efx, neigh); /* might sleep */ 727 rhashtable_walk_start(&walk); 728 } 729 rhashtable_walk_stop(&walk); 730 rhashtable_walk_exit(&walk); 731 mutex_unlock(&efx->tc->mutex); 732 } 733 734 int efx_tc_netevent_event(struct efx_nic *efx, unsigned long event, 735 void *ptr) 736 { 737 if (efx->type->is_vf) 738 return NOTIFY_DONE; 739 740 switch (event) { 741 case NETEVENT_NEIGH_UPDATE: 742 return efx_neigh_event(efx, ptr); 743 default: 744 return NOTIFY_DONE; 745 } 746 } 747