1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2021 Mellanox Technologies. */ 3 4 #include <net/fib_notifier.h> 5 #include <net/nexthop.h> 6 #include <net/ip_tunnels.h> 7 #include "tc_tun_encap.h" 8 #include "en_tc.h" 9 #include "tc_tun.h" 10 #include "rep/tc.h" 11 #include "diag/en_tc_tracepoint.h" 12 13 enum { 14 MLX5E_ROUTE_ENTRY_VALID = BIT(0), 15 }; 16 17 static int mlx5e_set_int_port_tunnel(struct mlx5e_priv *priv, 18 struct mlx5_flow_attr *attr, 19 struct mlx5e_encap_entry *e, 20 int out_index) 21 { 22 struct net_device *route_dev; 23 int err = 0; 24 25 route_dev = dev_get_by_index(dev_net(e->out_dev), e->route_dev_ifindex); 26 27 if (!route_dev || !netif_is_ovs_master(route_dev)) 28 goto out; 29 30 err = mlx5e_set_fwd_to_int_port_actions(priv, attr, e->route_dev_ifindex, 31 MLX5E_TC_INT_PORT_EGRESS, 32 &attr->action, out_index); 33 34 out: 35 if (route_dev) 36 dev_put(route_dev); 37 38 return err; 39 } 40 41 struct mlx5e_route_key { 42 int ip_version; 43 union { 44 __be32 v4; 45 struct in6_addr v6; 46 } endpoint_ip; 47 }; 48 49 struct mlx5e_route_entry { 50 struct mlx5e_route_key key; 51 struct list_head encap_entries; 52 struct list_head decap_flows; 53 u32 flags; 54 struct hlist_node hlist; 55 refcount_t refcnt; 56 int tunnel_dev_index; 57 struct rcu_head rcu; 58 }; 59 60 struct mlx5e_tc_tun_encap { 61 struct mlx5e_priv *priv; 62 struct notifier_block fib_nb; 63 spinlock_t route_lock; /* protects route_tbl */ 64 unsigned long route_tbl_last_update; 65 DECLARE_HASHTABLE(route_tbl, 8); 66 }; 67 68 static bool mlx5e_route_entry_valid(struct mlx5e_route_entry *r) 69 { 70 return r->flags & MLX5E_ROUTE_ENTRY_VALID; 71 } 72 73 int mlx5e_tc_set_attr_rx_tun(struct mlx5e_tc_flow *flow, 74 struct mlx5_flow_spec *spec) 75 { 76 struct mlx5_esw_flow_attr *esw_attr = flow->attr->esw_attr; 77 struct mlx5_rx_tun_attr *tun_attr; 78 void *daddr, *saddr; 79 u8 ip_version; 80 81 tun_attr = kvzalloc(sizeof(*tun_attr), GFP_KERNEL); 82 if (!tun_attr) 83 return -ENOMEM; 84 85 esw_attr->rx_tun_attr = tun_attr; 86 ip_version = mlx5e_tc_get_ip_version(spec, true); 87 88 if (ip_version == 4) { 89 daddr = MLX5_ADDR_OF(fte_match_param, spec->match_value, 90 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 91 saddr = MLX5_ADDR_OF(fte_match_param, spec->match_value, 92 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 93 tun_attr->dst_ip.v4 = *(__be32 *)daddr; 94 tun_attr->src_ip.v4 = *(__be32 *)saddr; 95 if (!tun_attr->dst_ip.v4 || !tun_attr->src_ip.v4) 96 return 0; 97 } 98 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) 99 else if (ip_version == 6) { 100 int ipv6_size = MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6); 101 102 daddr = MLX5_ADDR_OF(fte_match_param, spec->match_value, 103 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 104 saddr = MLX5_ADDR_OF(fte_match_param, spec->match_value, 105 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6); 106 memcpy(&tun_attr->dst_ip.v6, daddr, ipv6_size); 107 memcpy(&tun_attr->src_ip.v6, saddr, ipv6_size); 108 if (ipv6_addr_any(&tun_attr->dst_ip.v6) || 109 ipv6_addr_any(&tun_attr->src_ip.v6)) 110 return 0; 111 } 112 #endif 113 /* Only set the flag if both src and dst ip addresses exist. They are 114 * required to establish routing. 115 */ 116 flow_flag_set(flow, TUN_RX); 117 flow->attr->tun_ip_version = ip_version; 118 return 0; 119 } 120 121 static bool mlx5e_tc_flow_all_encaps_valid(struct mlx5_esw_flow_attr *esw_attr) 122 { 123 bool all_flow_encaps_valid = true; 124 int i; 125 126 /* Flow can be associated with multiple encap entries. 127 * Before offloading the flow verify that all of them have 128 * a valid neighbour. 129 */ 130 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 131 if (!(esw_attr->dests[i].flags & MLX5_ESW_DEST_ENCAP)) 132 continue; 133 if (!(esw_attr->dests[i].flags & MLX5_ESW_DEST_ENCAP_VALID)) { 134 all_flow_encaps_valid = false; 135 break; 136 } 137 } 138 139 return all_flow_encaps_valid; 140 } 141 142 void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv, 143 struct mlx5e_encap_entry *e, 144 struct list_head *flow_list) 145 { 146 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 147 struct mlx5_pkt_reformat_params reformat_params; 148 struct mlx5_esw_flow_attr *esw_attr; 149 struct mlx5_flow_handle *rule; 150 struct mlx5_flow_attr *attr; 151 struct mlx5_flow_spec *spec; 152 struct mlx5e_tc_flow *flow; 153 int err; 154 155 if (e->flags & MLX5_ENCAP_ENTRY_NO_ROUTE) 156 return; 157 158 memset(&reformat_params, 0, sizeof(reformat_params)); 159 reformat_params.type = e->reformat_type; 160 reformat_params.size = e->encap_size; 161 reformat_params.data = e->encap_header; 162 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, 163 &reformat_params, 164 MLX5_FLOW_NAMESPACE_FDB); 165 if (IS_ERR(e->pkt_reformat)) { 166 mlx5_core_warn(priv->mdev, "Failed to offload cached encapsulation header, %lu\n", 167 PTR_ERR(e->pkt_reformat)); 168 return; 169 } 170 e->flags |= MLX5_ENCAP_ENTRY_VALID; 171 mlx5e_rep_queue_neigh_stats_work(priv); 172 173 list_for_each_entry(flow, flow_list, tmp_list) { 174 if (!mlx5e_is_offloaded_flow(flow) || !flow_flag_test(flow, SLOW)) 175 continue; 176 177 spec = &flow->attr->parse_attr->spec; 178 179 attr = mlx5e_tc_get_encap_attr(flow); 180 esw_attr = attr->esw_attr; 181 esw_attr->dests[flow->tmp_entry_index].pkt_reformat = e->pkt_reformat; 182 esw_attr->dests[flow->tmp_entry_index].flags |= MLX5_ESW_DEST_ENCAP_VALID; 183 184 /* Do not offload flows with unresolved neighbors */ 185 if (!mlx5e_tc_flow_all_encaps_valid(esw_attr)) 186 continue; 187 188 err = mlx5e_tc_offload_flow_post_acts(flow); 189 if (err) { 190 mlx5_core_warn(priv->mdev, "Failed to update flow post acts, %d\n", 191 err); 192 continue; 193 } 194 195 /* update from slow path rule to encap rule */ 196 rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, flow->attr); 197 if (IS_ERR(rule)) { 198 mlx5e_tc_unoffload_flow_post_acts(flow); 199 err = PTR_ERR(rule); 200 mlx5_core_warn(priv->mdev, "Failed to update cached encapsulation flow, %d\n", 201 err); 202 continue; 203 } 204 205 mlx5e_tc_unoffload_from_slow_path(esw, flow); 206 flow->rule[0] = rule; 207 /* was unset when slow path rule removed */ 208 flow_flag_set(flow, OFFLOADED); 209 } 210 } 211 212 void mlx5e_tc_encap_flows_del(struct mlx5e_priv *priv, 213 struct mlx5e_encap_entry *e, 214 struct list_head *flow_list) 215 { 216 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 217 struct mlx5_esw_flow_attr *esw_attr; 218 struct mlx5_flow_handle *rule; 219 struct mlx5_flow_attr *attr; 220 struct mlx5_flow_spec *spec; 221 struct mlx5e_tc_flow *flow; 222 int err; 223 224 list_for_each_entry(flow, flow_list, tmp_list) { 225 if (!mlx5e_is_offloaded_flow(flow)) 226 continue; 227 228 attr = mlx5e_tc_get_encap_attr(flow); 229 esw_attr = attr->esw_attr; 230 /* mark the flow's encap dest as non-valid */ 231 esw_attr->dests[flow->tmp_entry_index].flags &= ~MLX5_ESW_DEST_ENCAP_VALID; 232 esw_attr->dests[flow->tmp_entry_index].pkt_reformat = NULL; 233 234 /* Clear pkt_reformat before checking slow path flag. Because 235 * in next iteration, the same flow is already set slow path 236 * flag, but still need to clear the pkt_reformat. 237 */ 238 if (flow_flag_test(flow, SLOW)) 239 continue; 240 241 /* update from encap rule to slow path rule */ 242 spec = &flow->attr->parse_attr->spec; 243 rule = mlx5e_tc_offload_to_slow_path(esw, flow, spec); 244 245 if (IS_ERR(rule)) { 246 err = PTR_ERR(rule); 247 mlx5_core_warn(priv->mdev, "Failed to update slow path (encap) flow, %d\n", 248 err); 249 continue; 250 } 251 252 mlx5e_tc_unoffload_fdb_rules(esw, flow, flow->attr); 253 mlx5e_tc_unoffload_flow_post_acts(flow); 254 flow->rule[0] = rule; 255 /* was unset when fast path rule removed */ 256 flow_flag_set(flow, OFFLOADED); 257 } 258 259 /* we know that the encap is valid */ 260 e->flags &= ~MLX5_ENCAP_ENTRY_VALID; 261 mlx5_packet_reformat_dealloc(priv->mdev, e->pkt_reformat); 262 e->pkt_reformat = NULL; 263 } 264 265 static void mlx5e_take_tmp_flow(struct mlx5e_tc_flow *flow, 266 struct list_head *flow_list, 267 int index) 268 { 269 if (IS_ERR(mlx5e_flow_get(flow))) { 270 /* Flow is being deleted concurrently. Wait for it to be 271 * unoffloaded from hardware, otherwise deleting encap will 272 * fail. 273 */ 274 wait_for_completion(&flow->del_hw_done); 275 return; 276 } 277 wait_for_completion(&flow->init_done); 278 279 flow->tmp_entry_index = index; 280 list_add(&flow->tmp_list, flow_list); 281 } 282 283 /* Takes reference to all flows attached to encap and adds the flows to 284 * flow_list using 'tmp_list' list_head in mlx5e_tc_flow. 285 */ 286 void mlx5e_take_all_encap_flows(struct mlx5e_encap_entry *e, struct list_head *flow_list) 287 { 288 struct encap_flow_item *efi; 289 struct mlx5e_tc_flow *flow; 290 291 list_for_each_entry(efi, &e->flows, list) { 292 flow = container_of(efi, struct mlx5e_tc_flow, encaps[efi->index]); 293 mlx5e_take_tmp_flow(flow, flow_list, efi->index); 294 } 295 } 296 297 /* Takes reference to all flows attached to route and adds the flows to 298 * flow_list using 'tmp_list' list_head in mlx5e_tc_flow. 299 */ 300 static void mlx5e_take_all_route_decap_flows(struct mlx5e_route_entry *r, 301 struct list_head *flow_list) 302 { 303 struct mlx5e_tc_flow *flow; 304 305 list_for_each_entry(flow, &r->decap_flows, decap_routes) 306 mlx5e_take_tmp_flow(flow, flow_list, 0); 307 } 308 309 typedef bool (match_cb)(struct mlx5e_encap_entry *); 310 311 static struct mlx5e_encap_entry * 312 mlx5e_get_next_matching_encap(struct mlx5e_neigh_hash_entry *nhe, 313 struct mlx5e_encap_entry *e, 314 match_cb match) 315 { 316 struct mlx5e_encap_entry *next = NULL; 317 318 retry: 319 rcu_read_lock(); 320 321 /* find encap with non-zero reference counter value */ 322 for (next = e ? 323 list_next_or_null_rcu(&nhe->encap_list, 324 &e->encap_list, 325 struct mlx5e_encap_entry, 326 encap_list) : 327 list_first_or_null_rcu(&nhe->encap_list, 328 struct mlx5e_encap_entry, 329 encap_list); 330 next; 331 next = list_next_or_null_rcu(&nhe->encap_list, 332 &next->encap_list, 333 struct mlx5e_encap_entry, 334 encap_list)) 335 if (mlx5e_encap_take(next)) 336 break; 337 338 rcu_read_unlock(); 339 340 /* release starting encap */ 341 if (e) 342 mlx5e_encap_put(netdev_priv(e->out_dev), e); 343 if (!next) 344 return next; 345 346 /* wait for encap to be fully initialized */ 347 wait_for_completion(&next->res_ready); 348 /* continue searching if encap entry is not in valid state after completion */ 349 if (!match(next)) { 350 e = next; 351 goto retry; 352 } 353 354 return next; 355 } 356 357 static bool mlx5e_encap_valid(struct mlx5e_encap_entry *e) 358 { 359 return e->flags & MLX5_ENCAP_ENTRY_VALID; 360 } 361 362 static struct mlx5e_encap_entry * 363 mlx5e_get_next_valid_encap(struct mlx5e_neigh_hash_entry *nhe, 364 struct mlx5e_encap_entry *e) 365 { 366 return mlx5e_get_next_matching_encap(nhe, e, mlx5e_encap_valid); 367 } 368 369 static bool mlx5e_encap_initialized(struct mlx5e_encap_entry *e) 370 { 371 return e->compl_result >= 0; 372 } 373 374 struct mlx5e_encap_entry * 375 mlx5e_get_next_init_encap(struct mlx5e_neigh_hash_entry *nhe, 376 struct mlx5e_encap_entry *e) 377 { 378 return mlx5e_get_next_matching_encap(nhe, e, mlx5e_encap_initialized); 379 } 380 381 void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe) 382 { 383 struct mlx5e_neigh *m_neigh = &nhe->m_neigh; 384 struct mlx5e_encap_entry *e = NULL; 385 struct mlx5e_tc_flow *flow; 386 struct mlx5_fc *counter; 387 struct neigh_table *tbl; 388 bool neigh_used = false; 389 struct neighbour *n; 390 u64 lastuse; 391 392 if (m_neigh->family == AF_INET) 393 tbl = &arp_tbl; 394 #if IS_ENABLED(CONFIG_IPV6) 395 else if (m_neigh->family == AF_INET6) 396 tbl = ipv6_stub->nd_tbl; 397 #endif 398 else 399 return; 400 401 /* mlx5e_get_next_valid_encap() releases previous encap before returning 402 * next one. 403 */ 404 while ((e = mlx5e_get_next_valid_encap(nhe, e)) != NULL) { 405 struct mlx5e_priv *priv = netdev_priv(e->out_dev); 406 struct encap_flow_item *efi, *tmp; 407 struct mlx5_eswitch *esw; 408 LIST_HEAD(flow_list); 409 410 esw = priv->mdev->priv.eswitch; 411 mutex_lock(&esw->offloads.encap_tbl_lock); 412 list_for_each_entry_safe(efi, tmp, &e->flows, list) { 413 flow = container_of(efi, struct mlx5e_tc_flow, 414 encaps[efi->index]); 415 if (IS_ERR(mlx5e_flow_get(flow))) 416 continue; 417 list_add(&flow->tmp_list, &flow_list); 418 419 if (mlx5e_is_offloaded_flow(flow)) { 420 counter = mlx5e_tc_get_counter(flow); 421 lastuse = mlx5_fc_query_lastuse(counter); 422 if (time_after((unsigned long)lastuse, nhe->reported_lastuse)) { 423 neigh_used = true; 424 break; 425 } 426 } 427 } 428 mutex_unlock(&esw->offloads.encap_tbl_lock); 429 430 mlx5e_put_flow_list(priv, &flow_list); 431 if (neigh_used) { 432 /* release current encap before breaking the loop */ 433 mlx5e_encap_put(priv, e); 434 break; 435 } 436 } 437 438 trace_mlx5e_tc_update_neigh_used_value(nhe, neigh_used); 439 440 if (neigh_used) { 441 nhe->reported_lastuse = jiffies; 442 443 /* find the relevant neigh according to the cached device and 444 * dst ip pair 445 */ 446 n = neigh_lookup(tbl, &m_neigh->dst_ip, READ_ONCE(nhe->neigh_dev)); 447 if (!n) 448 return; 449 450 neigh_event_send(n, NULL); 451 neigh_release(n); 452 } 453 } 454 455 static void mlx5e_encap_dealloc(struct mlx5e_priv *priv, struct mlx5e_encap_entry *e) 456 { 457 WARN_ON(!list_empty(&e->flows)); 458 459 if (e->compl_result > 0) { 460 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e); 461 462 if (e->flags & MLX5_ENCAP_ENTRY_VALID) 463 mlx5_packet_reformat_dealloc(priv->mdev, e->pkt_reformat); 464 } 465 466 kfree(e->tun_info); 467 kfree(e->encap_header); 468 kfree_rcu(e, rcu); 469 } 470 471 static void mlx5e_decap_dealloc(struct mlx5e_priv *priv, 472 struct mlx5e_decap_entry *d) 473 { 474 WARN_ON(!list_empty(&d->flows)); 475 476 if (!d->compl_result) 477 mlx5_packet_reformat_dealloc(priv->mdev, d->pkt_reformat); 478 479 kfree_rcu(d, rcu); 480 } 481 482 void mlx5e_encap_put(struct mlx5e_priv *priv, struct mlx5e_encap_entry *e) 483 { 484 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 485 486 if (!refcount_dec_and_mutex_lock(&e->refcnt, &esw->offloads.encap_tbl_lock)) 487 return; 488 list_del(&e->route_list); 489 hash_del_rcu(&e->encap_hlist); 490 mutex_unlock(&esw->offloads.encap_tbl_lock); 491 492 mlx5e_encap_dealloc(priv, e); 493 } 494 495 static void mlx5e_encap_put_locked(struct mlx5e_priv *priv, struct mlx5e_encap_entry *e) 496 { 497 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 498 499 lockdep_assert_held(&esw->offloads.encap_tbl_lock); 500 501 if (!refcount_dec_and_test(&e->refcnt)) 502 return; 503 list_del(&e->route_list); 504 hash_del_rcu(&e->encap_hlist); 505 mlx5e_encap_dealloc(priv, e); 506 } 507 508 static void mlx5e_decap_put(struct mlx5e_priv *priv, struct mlx5e_decap_entry *d) 509 { 510 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 511 512 if (!refcount_dec_and_mutex_lock(&d->refcnt, &esw->offloads.decap_tbl_lock)) 513 return; 514 hash_del_rcu(&d->hlist); 515 mutex_unlock(&esw->offloads.decap_tbl_lock); 516 517 mlx5e_decap_dealloc(priv, d); 518 } 519 520 static void mlx5e_detach_encap_route(struct mlx5e_priv *priv, 521 struct mlx5e_tc_flow *flow, 522 int out_index); 523 524 void mlx5e_detach_encap(struct mlx5e_priv *priv, 525 struct mlx5e_tc_flow *flow, 526 struct mlx5_flow_attr *attr, 527 int out_index) 528 { 529 struct mlx5e_encap_entry *e = flow->encaps[out_index].e; 530 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 531 532 if (!mlx5e_is_eswitch_flow(flow)) 533 return; 534 535 if (attr->esw_attr->dests[out_index].flags & 536 MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 537 mlx5e_detach_encap_route(priv, flow, out_index); 538 539 /* flow wasn't fully initialized */ 540 if (!e) 541 return; 542 543 mutex_lock(&esw->offloads.encap_tbl_lock); 544 list_del(&flow->encaps[out_index].list); 545 flow->encaps[out_index].e = NULL; 546 if (!refcount_dec_and_test(&e->refcnt)) { 547 mutex_unlock(&esw->offloads.encap_tbl_lock); 548 return; 549 } 550 list_del(&e->route_list); 551 hash_del_rcu(&e->encap_hlist); 552 mutex_unlock(&esw->offloads.encap_tbl_lock); 553 554 mlx5e_encap_dealloc(priv, e); 555 } 556 557 void mlx5e_detach_decap(struct mlx5e_priv *priv, 558 struct mlx5e_tc_flow *flow) 559 { 560 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 561 struct mlx5e_decap_entry *d = flow->decap_reformat; 562 563 if (!d) 564 return; 565 566 mutex_lock(&esw->offloads.decap_tbl_lock); 567 list_del(&flow->l3_to_l2_reformat); 568 flow->decap_reformat = NULL; 569 570 if (!refcount_dec_and_test(&d->refcnt)) { 571 mutex_unlock(&esw->offloads.decap_tbl_lock); 572 return; 573 } 574 hash_del_rcu(&d->hlist); 575 mutex_unlock(&esw->offloads.decap_tbl_lock); 576 577 mlx5e_decap_dealloc(priv, d); 578 } 579 580 bool mlx5e_tc_tun_encap_info_equal_generic(struct mlx5e_encap_key *a, 581 struct mlx5e_encap_key *b) 582 { 583 return memcmp(a->ip_tun_key, b->ip_tun_key, sizeof(*a->ip_tun_key)) == 0 && 584 a->tc_tunnel->tunnel_type == b->tc_tunnel->tunnel_type; 585 } 586 587 bool mlx5e_tc_tun_encap_info_equal_options(struct mlx5e_encap_key *a, 588 struct mlx5e_encap_key *b, 589 __be16 tun_flags) 590 { 591 struct ip_tunnel_info *a_info; 592 struct ip_tunnel_info *b_info; 593 bool a_has_opts, b_has_opts; 594 595 if (!mlx5e_tc_tun_encap_info_equal_generic(a, b)) 596 return false; 597 598 a_has_opts = !!(a->ip_tun_key->tun_flags & tun_flags); 599 b_has_opts = !!(b->ip_tun_key->tun_flags & tun_flags); 600 601 /* keys are equal when both don't have any options attached */ 602 if (!a_has_opts && !b_has_opts) 603 return true; 604 605 if (a_has_opts != b_has_opts) 606 return false; 607 608 /* options stored in memory next to ip_tunnel_info struct */ 609 a_info = container_of(a->ip_tun_key, struct ip_tunnel_info, key); 610 b_info = container_of(b->ip_tun_key, struct ip_tunnel_info, key); 611 612 return a_info->options_len == b_info->options_len && 613 !memcmp(ip_tunnel_info_opts(a_info), 614 ip_tunnel_info_opts(b_info), 615 a_info->options_len); 616 } 617 618 static int cmp_decap_info(struct mlx5e_decap_key *a, 619 struct mlx5e_decap_key *b) 620 { 621 return memcmp(&a->key, &b->key, sizeof(b->key)); 622 } 623 624 static int hash_encap_info(struct mlx5e_encap_key *key) 625 { 626 return jhash(key->ip_tun_key, sizeof(*key->ip_tun_key), 627 key->tc_tunnel->tunnel_type); 628 } 629 630 static int hash_decap_info(struct mlx5e_decap_key *key) 631 { 632 return jhash(&key->key, sizeof(key->key), 0); 633 } 634 635 bool mlx5e_encap_take(struct mlx5e_encap_entry *e) 636 { 637 return refcount_inc_not_zero(&e->refcnt); 638 } 639 640 static bool mlx5e_decap_take(struct mlx5e_decap_entry *e) 641 { 642 return refcount_inc_not_zero(&e->refcnt); 643 } 644 645 static struct mlx5e_encap_entry * 646 mlx5e_encap_get(struct mlx5e_priv *priv, struct mlx5e_encap_key *key, 647 uintptr_t hash_key) 648 { 649 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 650 struct mlx5e_encap_key e_key; 651 struct mlx5e_encap_entry *e; 652 653 hash_for_each_possible_rcu(esw->offloads.encap_tbl, e, 654 encap_hlist, hash_key) { 655 e_key.ip_tun_key = &e->tun_info->key; 656 e_key.tc_tunnel = e->tunnel; 657 if (e->tunnel->encap_info_equal(&e_key, key) && 658 mlx5e_encap_take(e)) 659 return e; 660 } 661 662 return NULL; 663 } 664 665 static struct mlx5e_decap_entry * 666 mlx5e_decap_get(struct mlx5e_priv *priv, struct mlx5e_decap_key *key, 667 uintptr_t hash_key) 668 { 669 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 670 struct mlx5e_decap_key r_key; 671 struct mlx5e_decap_entry *e; 672 673 hash_for_each_possible_rcu(esw->offloads.decap_tbl, e, 674 hlist, hash_key) { 675 r_key = e->key; 676 if (!cmp_decap_info(&r_key, key) && 677 mlx5e_decap_take(e)) 678 return e; 679 } 680 return NULL; 681 } 682 683 struct ip_tunnel_info *mlx5e_dup_tun_info(const struct ip_tunnel_info *tun_info) 684 { 685 size_t tun_size = sizeof(*tun_info) + tun_info->options_len; 686 687 return kmemdup(tun_info, tun_size, GFP_KERNEL); 688 } 689 690 static bool is_duplicated_encap_entry(struct mlx5e_priv *priv, 691 struct mlx5e_tc_flow *flow, 692 int out_index, 693 struct mlx5e_encap_entry *e, 694 struct netlink_ext_ack *extack) 695 { 696 int i; 697 698 for (i = 0; i < out_index; i++) { 699 if (flow->encaps[i].e != e) 700 continue; 701 NL_SET_ERR_MSG_MOD(extack, "can't duplicate encap action"); 702 netdev_err(priv->netdev, "can't duplicate encap action\n"); 703 return true; 704 } 705 706 return false; 707 } 708 709 static int mlx5e_set_vf_tunnel(struct mlx5_eswitch *esw, 710 struct mlx5_flow_attr *attr, 711 struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts, 712 struct net_device *out_dev, 713 int route_dev_ifindex, 714 int out_index) 715 { 716 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 717 struct net_device *route_dev; 718 u16 vport_num; 719 int err = 0; 720 u32 data; 721 722 route_dev = dev_get_by_index(dev_net(out_dev), route_dev_ifindex); 723 724 if (!route_dev || route_dev->netdev_ops != &mlx5e_netdev_ops || 725 !mlx5e_tc_is_vf_tunnel(out_dev, route_dev)) 726 goto out; 727 728 err = mlx5e_tc_query_route_vport(out_dev, route_dev, &vport_num); 729 if (err) 730 goto out; 731 732 attr->dest_chain = 0; 733 attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 734 esw_attr->dests[out_index].flags |= MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE; 735 data = mlx5_eswitch_get_vport_metadata_for_set(esw_attr->in_mdev->priv.eswitch, 736 vport_num); 737 err = mlx5e_tc_match_to_reg_set_and_get_id(esw->dev, mod_hdr_acts, 738 MLX5_FLOW_NAMESPACE_FDB, 739 VPORT_TO_REG, data); 740 if (err >= 0) { 741 esw_attr->dests[out_index].src_port_rewrite_act_id = err; 742 err = 0; 743 } 744 745 out: 746 if (route_dev) 747 dev_put(route_dev); 748 return err; 749 } 750 751 static int mlx5e_update_vf_tunnel(struct mlx5_eswitch *esw, 752 struct mlx5_esw_flow_attr *attr, 753 struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts, 754 struct net_device *out_dev, 755 int route_dev_ifindex, 756 int out_index) 757 { 758 int act_id = attr->dests[out_index].src_port_rewrite_act_id; 759 struct net_device *route_dev; 760 u16 vport_num; 761 int err = 0; 762 u32 data; 763 764 route_dev = dev_get_by_index(dev_net(out_dev), route_dev_ifindex); 765 766 if (!route_dev || route_dev->netdev_ops != &mlx5e_netdev_ops || 767 !mlx5e_tc_is_vf_tunnel(out_dev, route_dev)) { 768 err = -ENODEV; 769 goto out; 770 } 771 772 err = mlx5e_tc_query_route_vport(out_dev, route_dev, &vport_num); 773 if (err) 774 goto out; 775 776 data = mlx5_eswitch_get_vport_metadata_for_set(attr->in_mdev->priv.eswitch, 777 vport_num); 778 mlx5e_tc_match_to_reg_mod_hdr_change(esw->dev, mod_hdr_acts, VPORT_TO_REG, act_id, data); 779 780 out: 781 if (route_dev) 782 dev_put(route_dev); 783 return err; 784 } 785 786 static unsigned int mlx5e_route_tbl_get_last_update(struct mlx5e_priv *priv) 787 { 788 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 789 struct mlx5_rep_uplink_priv *uplink_priv; 790 struct mlx5e_rep_priv *uplink_rpriv; 791 struct mlx5e_tc_tun_encap *encap; 792 unsigned int ret; 793 794 uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); 795 uplink_priv = &uplink_rpriv->uplink_priv; 796 encap = uplink_priv->encap; 797 798 spin_lock_bh(&encap->route_lock); 799 ret = encap->route_tbl_last_update; 800 spin_unlock_bh(&encap->route_lock); 801 return ret; 802 } 803 804 static int mlx5e_attach_encap_route(struct mlx5e_priv *priv, 805 struct mlx5e_tc_flow *flow, 806 struct mlx5_flow_attr *attr, 807 struct mlx5e_encap_entry *e, 808 bool new_encap_entry, 809 unsigned long tbl_time_before, 810 int out_index); 811 812 int mlx5e_attach_encap(struct mlx5e_priv *priv, 813 struct mlx5e_tc_flow *flow, 814 struct mlx5_flow_attr *attr, 815 struct net_device *mirred_dev, 816 int out_index, 817 struct netlink_ext_ack *extack, 818 struct net_device **encap_dev) 819 { 820 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 821 struct mlx5e_tc_flow_parse_attr *parse_attr; 822 const struct ip_tunnel_info *tun_info; 823 const struct mlx5e_mpls_info *mpls_info; 824 unsigned long tbl_time_before = 0; 825 struct mlx5e_encap_entry *e; 826 struct mlx5e_encap_key key; 827 bool entry_created = false; 828 unsigned short family; 829 uintptr_t hash_key; 830 int err = 0; 831 832 lockdep_assert_held(&esw->offloads.encap_tbl_lock); 833 834 parse_attr = attr->parse_attr; 835 tun_info = parse_attr->tun_info[out_index]; 836 mpls_info = &parse_attr->mpls_info[out_index]; 837 family = ip_tunnel_info_af(tun_info); 838 key.ip_tun_key = &tun_info->key; 839 key.tc_tunnel = mlx5e_get_tc_tun(mirred_dev); 840 if (!key.tc_tunnel) { 841 NL_SET_ERR_MSG_MOD(extack, "Unsupported tunnel"); 842 return -EOPNOTSUPP; 843 } 844 845 hash_key = hash_encap_info(&key); 846 847 e = mlx5e_encap_get(priv, &key, hash_key); 848 849 /* must verify if encap is valid or not */ 850 if (e) { 851 /* Check that entry was not already attached to this flow */ 852 if (is_duplicated_encap_entry(priv, flow, out_index, e, extack)) { 853 err = -EOPNOTSUPP; 854 goto out_err; 855 } 856 857 goto attach_flow; 858 } 859 860 e = kzalloc(sizeof(*e), GFP_KERNEL); 861 if (!e) { 862 err = -ENOMEM; 863 goto out_err; 864 } 865 866 refcount_set(&e->refcnt, 1); 867 init_completion(&e->res_ready); 868 entry_created = true; 869 INIT_LIST_HEAD(&e->route_list); 870 871 tun_info = mlx5e_dup_tun_info(tun_info); 872 if (!tun_info) { 873 err = -ENOMEM; 874 goto out_err_init; 875 } 876 e->tun_info = tun_info; 877 memcpy(&e->mpls_info, mpls_info, sizeof(*mpls_info)); 878 err = mlx5e_tc_tun_init_encap_attr(mirred_dev, priv, e, extack); 879 if (err) 880 goto out_err_init; 881 882 INIT_LIST_HEAD(&e->flows); 883 hash_add_rcu(esw->offloads.encap_tbl, &e->encap_hlist, hash_key); 884 tbl_time_before = mlx5e_route_tbl_get_last_update(priv); 885 886 if (family == AF_INET) 887 err = mlx5e_tc_tun_create_header_ipv4(priv, mirred_dev, e); 888 else if (family == AF_INET6) 889 err = mlx5e_tc_tun_create_header_ipv6(priv, mirred_dev, e); 890 891 complete_all(&e->res_ready); 892 if (err) { 893 e->compl_result = err; 894 goto out_err; 895 } 896 e->compl_result = 1; 897 898 attach_flow: 899 err = mlx5e_attach_encap_route(priv, flow, attr, e, entry_created, 900 tbl_time_before, out_index); 901 if (err) 902 goto out_err; 903 904 err = mlx5e_set_int_port_tunnel(priv, attr, e, out_index); 905 if (err == -EOPNOTSUPP) { 906 /* If device doesn't support int port offload, 907 * redirect to uplink vport. 908 */ 909 mlx5_core_dbg(priv->mdev, "attaching int port as encap dev not supported, using uplink\n"); 910 err = 0; 911 } else if (err) { 912 goto out_err; 913 } 914 915 flow->encaps[out_index].e = e; 916 list_add(&flow->encaps[out_index].list, &e->flows); 917 flow->encaps[out_index].index = out_index; 918 *encap_dev = e->out_dev; 919 if (e->flags & MLX5_ENCAP_ENTRY_VALID) { 920 attr->esw_attr->dests[out_index].pkt_reformat = e->pkt_reformat; 921 attr->esw_attr->dests[out_index].flags |= MLX5_ESW_DEST_ENCAP_VALID; 922 } else { 923 flow_flag_set(flow, SLOW); 924 } 925 926 return err; 927 928 out_err: 929 if (e) 930 mlx5e_encap_put_locked(priv, e); 931 return err; 932 933 out_err_init: 934 kfree(tun_info); 935 kfree(e); 936 return err; 937 } 938 939 int mlx5e_attach_decap(struct mlx5e_priv *priv, 940 struct mlx5e_tc_flow *flow, 941 struct netlink_ext_ack *extack) 942 { 943 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 944 struct mlx5_esw_flow_attr *attr = flow->attr->esw_attr; 945 struct mlx5_pkt_reformat_params reformat_params; 946 struct mlx5e_decap_entry *d; 947 struct mlx5e_decap_key key; 948 uintptr_t hash_key; 949 int err = 0; 950 951 if (sizeof(attr->eth) > MLX5_CAP_ESW(priv->mdev, max_encap_header_size)) { 952 NL_SET_ERR_MSG_MOD(extack, 953 "encap header larger than max supported"); 954 return -EOPNOTSUPP; 955 } 956 957 key.key = attr->eth; 958 hash_key = hash_decap_info(&key); 959 mutex_lock(&esw->offloads.decap_tbl_lock); 960 d = mlx5e_decap_get(priv, &key, hash_key); 961 if (d) { 962 mutex_unlock(&esw->offloads.decap_tbl_lock); 963 wait_for_completion(&d->res_ready); 964 mutex_lock(&esw->offloads.decap_tbl_lock); 965 if (d->compl_result) { 966 err = -EREMOTEIO; 967 goto out_free; 968 } 969 goto found; 970 } 971 972 d = kzalloc(sizeof(*d), GFP_KERNEL); 973 if (!d) { 974 err = -ENOMEM; 975 goto out_err; 976 } 977 978 d->key = key; 979 refcount_set(&d->refcnt, 1); 980 init_completion(&d->res_ready); 981 INIT_LIST_HEAD(&d->flows); 982 hash_add_rcu(esw->offloads.decap_tbl, &d->hlist, hash_key); 983 mutex_unlock(&esw->offloads.decap_tbl_lock); 984 985 memset(&reformat_params, 0, sizeof(reformat_params)); 986 reformat_params.type = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 987 reformat_params.size = sizeof(attr->eth); 988 reformat_params.data = &attr->eth; 989 d->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, 990 &reformat_params, 991 MLX5_FLOW_NAMESPACE_FDB); 992 if (IS_ERR(d->pkt_reformat)) { 993 err = PTR_ERR(d->pkt_reformat); 994 d->compl_result = err; 995 } 996 mutex_lock(&esw->offloads.decap_tbl_lock); 997 complete_all(&d->res_ready); 998 if (err) 999 goto out_free; 1000 1001 found: 1002 flow->decap_reformat = d; 1003 attr->decap_pkt_reformat = d->pkt_reformat; 1004 list_add(&flow->l3_to_l2_reformat, &d->flows); 1005 mutex_unlock(&esw->offloads.decap_tbl_lock); 1006 return 0; 1007 1008 out_free: 1009 mutex_unlock(&esw->offloads.decap_tbl_lock); 1010 mlx5e_decap_put(priv, d); 1011 return err; 1012 1013 out_err: 1014 mutex_unlock(&esw->offloads.decap_tbl_lock); 1015 return err; 1016 } 1017 1018 int mlx5e_tc_tun_encap_dests_set(struct mlx5e_priv *priv, 1019 struct mlx5e_tc_flow *flow, 1020 struct mlx5_flow_attr *attr, 1021 struct netlink_ext_ack *extack, 1022 bool *vf_tun) 1023 { 1024 struct mlx5e_tc_flow_parse_attr *parse_attr; 1025 struct mlx5_esw_flow_attr *esw_attr; 1026 struct net_device *encap_dev = NULL; 1027 struct mlx5e_rep_priv *rpriv; 1028 struct mlx5e_priv *out_priv; 1029 struct mlx5_eswitch *esw; 1030 int out_index; 1031 int err = 0; 1032 1033 if (!mlx5e_is_eswitch_flow(flow)) 1034 return 0; 1035 1036 parse_attr = attr->parse_attr; 1037 esw_attr = attr->esw_attr; 1038 *vf_tun = false; 1039 1040 esw = priv->mdev->priv.eswitch; 1041 mutex_lock(&esw->offloads.encap_tbl_lock); 1042 for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) { 1043 struct net_device *out_dev; 1044 int mirred_ifindex; 1045 1046 if (!(esw_attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP)) 1047 continue; 1048 1049 mirred_ifindex = parse_attr->mirred_ifindex[out_index]; 1050 out_dev = dev_get_by_index(dev_net(priv->netdev), mirred_ifindex); 1051 if (!out_dev) { 1052 NL_SET_ERR_MSG_MOD(extack, "Requested mirred device not found"); 1053 err = -ENODEV; 1054 goto out; 1055 } 1056 err = mlx5e_attach_encap(priv, flow, attr, out_dev, out_index, 1057 extack, &encap_dev); 1058 dev_put(out_dev); 1059 if (err) 1060 goto out; 1061 1062 if (esw_attr->dests[out_index].flags & 1063 MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE && 1064 !esw_attr->dest_int_port) 1065 *vf_tun = true; 1066 1067 out_priv = netdev_priv(encap_dev); 1068 rpriv = out_priv->ppriv; 1069 esw_attr->dests[out_index].rep = rpriv->rep; 1070 esw_attr->dests[out_index].mdev = out_priv->mdev; 1071 } 1072 1073 if (*vf_tun && esw_attr->out_count > 1) { 1074 NL_SET_ERR_MSG_MOD(extack, "VF tunnel encap with mirroring is not supported"); 1075 err = -EOPNOTSUPP; 1076 goto out; 1077 } 1078 1079 out: 1080 mutex_unlock(&esw->offloads.encap_tbl_lock); 1081 return err; 1082 } 1083 1084 void mlx5e_tc_tun_encap_dests_unset(struct mlx5e_priv *priv, 1085 struct mlx5e_tc_flow *flow, 1086 struct mlx5_flow_attr *attr) 1087 { 1088 struct mlx5_esw_flow_attr *esw_attr; 1089 int out_index; 1090 1091 if (!mlx5e_is_eswitch_flow(flow)) 1092 return; 1093 1094 esw_attr = attr->esw_attr; 1095 1096 for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) { 1097 if (!(esw_attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP)) 1098 continue; 1099 1100 mlx5e_detach_encap(flow->priv, flow, attr, out_index); 1101 kfree(attr->parse_attr->tun_info[out_index]); 1102 } 1103 } 1104 1105 static int cmp_route_info(struct mlx5e_route_key *a, 1106 struct mlx5e_route_key *b) 1107 { 1108 if (a->ip_version == 4 && b->ip_version == 4) 1109 return memcmp(&a->endpoint_ip.v4, &b->endpoint_ip.v4, 1110 sizeof(a->endpoint_ip.v4)); 1111 else if (a->ip_version == 6 && b->ip_version == 6) 1112 return memcmp(&a->endpoint_ip.v6, &b->endpoint_ip.v6, 1113 sizeof(a->endpoint_ip.v6)); 1114 return 1; 1115 } 1116 1117 static u32 hash_route_info(struct mlx5e_route_key *key) 1118 { 1119 if (key->ip_version == 4) 1120 return jhash(&key->endpoint_ip.v4, sizeof(key->endpoint_ip.v4), 0); 1121 return jhash(&key->endpoint_ip.v6, sizeof(key->endpoint_ip.v6), 0); 1122 } 1123 1124 static void mlx5e_route_dealloc(struct mlx5e_priv *priv, 1125 struct mlx5e_route_entry *r) 1126 { 1127 WARN_ON(!list_empty(&r->decap_flows)); 1128 WARN_ON(!list_empty(&r->encap_entries)); 1129 1130 kfree_rcu(r, rcu); 1131 } 1132 1133 static void mlx5e_route_put(struct mlx5e_priv *priv, struct mlx5e_route_entry *r) 1134 { 1135 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1136 1137 if (!refcount_dec_and_mutex_lock(&r->refcnt, &esw->offloads.encap_tbl_lock)) 1138 return; 1139 1140 hash_del_rcu(&r->hlist); 1141 mutex_unlock(&esw->offloads.encap_tbl_lock); 1142 1143 mlx5e_route_dealloc(priv, r); 1144 } 1145 1146 static void mlx5e_route_put_locked(struct mlx5e_priv *priv, struct mlx5e_route_entry *r) 1147 { 1148 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1149 1150 lockdep_assert_held(&esw->offloads.encap_tbl_lock); 1151 1152 if (!refcount_dec_and_test(&r->refcnt)) 1153 return; 1154 hash_del_rcu(&r->hlist); 1155 mlx5e_route_dealloc(priv, r); 1156 } 1157 1158 static struct mlx5e_route_entry * 1159 mlx5e_route_get(struct mlx5e_tc_tun_encap *encap, struct mlx5e_route_key *key, 1160 u32 hash_key) 1161 { 1162 struct mlx5e_route_key r_key; 1163 struct mlx5e_route_entry *r; 1164 1165 hash_for_each_possible(encap->route_tbl, r, hlist, hash_key) { 1166 r_key = r->key; 1167 if (!cmp_route_info(&r_key, key) && 1168 refcount_inc_not_zero(&r->refcnt)) 1169 return r; 1170 } 1171 return NULL; 1172 } 1173 1174 static struct mlx5e_route_entry * 1175 mlx5e_route_get_create(struct mlx5e_priv *priv, 1176 struct mlx5e_route_key *key, 1177 int tunnel_dev_index, 1178 unsigned long *route_tbl_change_time) 1179 { 1180 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1181 struct mlx5_rep_uplink_priv *uplink_priv; 1182 struct mlx5e_rep_priv *uplink_rpriv; 1183 struct mlx5e_tc_tun_encap *encap; 1184 struct mlx5e_route_entry *r; 1185 u32 hash_key; 1186 1187 uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); 1188 uplink_priv = &uplink_rpriv->uplink_priv; 1189 encap = uplink_priv->encap; 1190 1191 hash_key = hash_route_info(key); 1192 spin_lock_bh(&encap->route_lock); 1193 r = mlx5e_route_get(encap, key, hash_key); 1194 spin_unlock_bh(&encap->route_lock); 1195 if (r) { 1196 if (!mlx5e_route_entry_valid(r)) { 1197 mlx5e_route_put_locked(priv, r); 1198 return ERR_PTR(-EINVAL); 1199 } 1200 return r; 1201 } 1202 1203 r = kzalloc(sizeof(*r), GFP_KERNEL); 1204 if (!r) 1205 return ERR_PTR(-ENOMEM); 1206 1207 r->key = *key; 1208 r->flags |= MLX5E_ROUTE_ENTRY_VALID; 1209 r->tunnel_dev_index = tunnel_dev_index; 1210 refcount_set(&r->refcnt, 1); 1211 INIT_LIST_HEAD(&r->decap_flows); 1212 INIT_LIST_HEAD(&r->encap_entries); 1213 1214 spin_lock_bh(&encap->route_lock); 1215 *route_tbl_change_time = encap->route_tbl_last_update; 1216 hash_add(encap->route_tbl, &r->hlist, hash_key); 1217 spin_unlock_bh(&encap->route_lock); 1218 1219 return r; 1220 } 1221 1222 static struct mlx5e_route_entry * 1223 mlx5e_route_lookup_for_update(struct mlx5e_tc_tun_encap *encap, struct mlx5e_route_key *key) 1224 { 1225 u32 hash_key = hash_route_info(key); 1226 struct mlx5e_route_entry *r; 1227 1228 spin_lock_bh(&encap->route_lock); 1229 encap->route_tbl_last_update = jiffies; 1230 r = mlx5e_route_get(encap, key, hash_key); 1231 spin_unlock_bh(&encap->route_lock); 1232 1233 return r; 1234 } 1235 1236 struct mlx5e_tc_fib_event_data { 1237 struct work_struct work; 1238 unsigned long event; 1239 struct mlx5e_route_entry *r; 1240 struct net_device *ul_dev; 1241 }; 1242 1243 static void mlx5e_tc_fib_event_work(struct work_struct *work); 1244 static struct mlx5e_tc_fib_event_data * 1245 mlx5e_tc_init_fib_work(unsigned long event, struct net_device *ul_dev, gfp_t flags) 1246 { 1247 struct mlx5e_tc_fib_event_data *fib_work; 1248 1249 fib_work = kzalloc(sizeof(*fib_work), flags); 1250 if (WARN_ON(!fib_work)) 1251 return NULL; 1252 1253 INIT_WORK(&fib_work->work, mlx5e_tc_fib_event_work); 1254 fib_work->event = event; 1255 fib_work->ul_dev = ul_dev; 1256 1257 return fib_work; 1258 } 1259 1260 static int 1261 mlx5e_route_enqueue_update(struct mlx5e_priv *priv, 1262 struct mlx5e_route_entry *r, 1263 unsigned long event) 1264 { 1265 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1266 struct mlx5e_tc_fib_event_data *fib_work; 1267 struct mlx5e_rep_priv *uplink_rpriv; 1268 struct net_device *ul_dev; 1269 1270 uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); 1271 ul_dev = uplink_rpriv->netdev; 1272 1273 fib_work = mlx5e_tc_init_fib_work(event, ul_dev, GFP_KERNEL); 1274 if (!fib_work) 1275 return -ENOMEM; 1276 1277 dev_hold(ul_dev); 1278 refcount_inc(&r->refcnt); 1279 fib_work->r = r; 1280 queue_work(priv->wq, &fib_work->work); 1281 1282 return 0; 1283 } 1284 1285 int mlx5e_attach_decap_route(struct mlx5e_priv *priv, 1286 struct mlx5e_tc_flow *flow) 1287 { 1288 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1289 unsigned long tbl_time_before, tbl_time_after; 1290 struct mlx5e_tc_flow_parse_attr *parse_attr; 1291 struct mlx5_flow_attr *attr = flow->attr; 1292 struct mlx5_esw_flow_attr *esw_attr; 1293 struct mlx5e_route_entry *r; 1294 struct mlx5e_route_key key; 1295 int err = 0; 1296 1297 esw_attr = attr->esw_attr; 1298 parse_attr = attr->parse_attr; 1299 mutex_lock(&esw->offloads.encap_tbl_lock); 1300 if (!esw_attr->rx_tun_attr) 1301 goto out; 1302 1303 tbl_time_before = mlx5e_route_tbl_get_last_update(priv); 1304 tbl_time_after = tbl_time_before; 1305 err = mlx5e_tc_tun_route_lookup(priv, &parse_attr->spec, attr, parse_attr->filter_dev); 1306 if (err || !esw_attr->rx_tun_attr->decap_vport) 1307 goto out; 1308 1309 key.ip_version = attr->tun_ip_version; 1310 if (key.ip_version == 4) 1311 key.endpoint_ip.v4 = esw_attr->rx_tun_attr->dst_ip.v4; 1312 else 1313 key.endpoint_ip.v6 = esw_attr->rx_tun_attr->dst_ip.v6; 1314 1315 r = mlx5e_route_get_create(priv, &key, parse_attr->filter_dev->ifindex, 1316 &tbl_time_after); 1317 if (IS_ERR(r)) { 1318 err = PTR_ERR(r); 1319 goto out; 1320 } 1321 /* Routing changed concurrently. FIB event handler might have missed new 1322 * entry, schedule update. 1323 */ 1324 if (tbl_time_before != tbl_time_after) { 1325 err = mlx5e_route_enqueue_update(priv, r, FIB_EVENT_ENTRY_REPLACE); 1326 if (err) { 1327 mlx5e_route_put_locked(priv, r); 1328 goto out; 1329 } 1330 } 1331 1332 flow->decap_route = r; 1333 list_add(&flow->decap_routes, &r->decap_flows); 1334 mutex_unlock(&esw->offloads.encap_tbl_lock); 1335 return 0; 1336 1337 out: 1338 mutex_unlock(&esw->offloads.encap_tbl_lock); 1339 return err; 1340 } 1341 1342 static int mlx5e_attach_encap_route(struct mlx5e_priv *priv, 1343 struct mlx5e_tc_flow *flow, 1344 struct mlx5_flow_attr *attr, 1345 struct mlx5e_encap_entry *e, 1346 bool new_encap_entry, 1347 unsigned long tbl_time_before, 1348 int out_index) 1349 { 1350 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1351 unsigned long tbl_time_after = tbl_time_before; 1352 struct mlx5e_tc_flow_parse_attr *parse_attr; 1353 const struct ip_tunnel_info *tun_info; 1354 struct mlx5_esw_flow_attr *esw_attr; 1355 struct mlx5e_route_entry *r; 1356 struct mlx5e_route_key key; 1357 unsigned short family; 1358 int err = 0; 1359 1360 esw_attr = attr->esw_attr; 1361 parse_attr = attr->parse_attr; 1362 tun_info = parse_attr->tun_info[out_index]; 1363 family = ip_tunnel_info_af(tun_info); 1364 1365 if (family == AF_INET) { 1366 key.endpoint_ip.v4 = tun_info->key.u.ipv4.src; 1367 key.ip_version = 4; 1368 } else if (family == AF_INET6) { 1369 key.endpoint_ip.v6 = tun_info->key.u.ipv6.src; 1370 key.ip_version = 6; 1371 } 1372 1373 err = mlx5e_set_vf_tunnel(esw, attr, &parse_attr->mod_hdr_acts, e->out_dev, 1374 e->route_dev_ifindex, out_index); 1375 if (err || !(esw_attr->dests[out_index].flags & 1376 MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)) 1377 return err; 1378 1379 r = mlx5e_route_get_create(priv, &key, parse_attr->mirred_ifindex[out_index], 1380 &tbl_time_after); 1381 if (IS_ERR(r)) 1382 return PTR_ERR(r); 1383 /* Routing changed concurrently. FIB event handler might have missed new 1384 * entry, schedule update. 1385 */ 1386 if (tbl_time_before != tbl_time_after) { 1387 err = mlx5e_route_enqueue_update(priv, r, FIB_EVENT_ENTRY_REPLACE); 1388 if (err) { 1389 mlx5e_route_put_locked(priv, r); 1390 return err; 1391 } 1392 } 1393 1394 flow->encap_routes[out_index].r = r; 1395 if (new_encap_entry) 1396 list_add(&e->route_list, &r->encap_entries); 1397 flow->encap_routes[out_index].index = out_index; 1398 return 0; 1399 } 1400 1401 void mlx5e_detach_decap_route(struct mlx5e_priv *priv, 1402 struct mlx5e_tc_flow *flow) 1403 { 1404 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1405 struct mlx5e_route_entry *r = flow->decap_route; 1406 1407 if (!r) 1408 return; 1409 1410 mutex_lock(&esw->offloads.encap_tbl_lock); 1411 list_del(&flow->decap_routes); 1412 flow->decap_route = NULL; 1413 1414 if (!refcount_dec_and_test(&r->refcnt)) { 1415 mutex_unlock(&esw->offloads.encap_tbl_lock); 1416 return; 1417 } 1418 hash_del_rcu(&r->hlist); 1419 mutex_unlock(&esw->offloads.encap_tbl_lock); 1420 1421 mlx5e_route_dealloc(priv, r); 1422 } 1423 1424 static void mlx5e_detach_encap_route(struct mlx5e_priv *priv, 1425 struct mlx5e_tc_flow *flow, 1426 int out_index) 1427 { 1428 struct mlx5e_route_entry *r = flow->encap_routes[out_index].r; 1429 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1430 struct mlx5e_encap_entry *e, *tmp; 1431 1432 if (!r) 1433 return; 1434 1435 mutex_lock(&esw->offloads.encap_tbl_lock); 1436 flow->encap_routes[out_index].r = NULL; 1437 1438 if (!refcount_dec_and_test(&r->refcnt)) { 1439 mutex_unlock(&esw->offloads.encap_tbl_lock); 1440 return; 1441 } 1442 list_for_each_entry_safe(e, tmp, &r->encap_entries, route_list) 1443 list_del_init(&e->route_list); 1444 hash_del_rcu(&r->hlist); 1445 mutex_unlock(&esw->offloads.encap_tbl_lock); 1446 1447 mlx5e_route_dealloc(priv, r); 1448 } 1449 1450 static void mlx5e_invalidate_encap(struct mlx5e_priv *priv, 1451 struct mlx5e_encap_entry *e, 1452 struct list_head *encap_flows) 1453 { 1454 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1455 struct mlx5e_tc_flow *flow; 1456 1457 list_for_each_entry(flow, encap_flows, tmp_list) { 1458 struct mlx5_esw_flow_attr *esw_attr; 1459 struct mlx5_flow_attr *attr; 1460 1461 if (!mlx5e_is_offloaded_flow(flow)) 1462 continue; 1463 1464 attr = mlx5e_tc_get_encap_attr(flow); 1465 esw_attr = attr->esw_attr; 1466 1467 if (flow_flag_test(flow, SLOW)) 1468 mlx5e_tc_unoffload_from_slow_path(esw, flow); 1469 else 1470 mlx5e_tc_unoffload_fdb_rules(esw, flow, flow->attr); 1471 1472 mlx5e_tc_detach_mod_hdr(priv, flow, attr); 1473 attr->modify_hdr = NULL; 1474 1475 esw_attr->dests[flow->tmp_entry_index].flags &= 1476 ~MLX5_ESW_DEST_ENCAP_VALID; 1477 esw_attr->dests[flow->tmp_entry_index].pkt_reformat = NULL; 1478 } 1479 1480 e->flags |= MLX5_ENCAP_ENTRY_NO_ROUTE; 1481 if (e->flags & MLX5_ENCAP_ENTRY_VALID) { 1482 e->flags &= ~MLX5_ENCAP_ENTRY_VALID; 1483 mlx5_packet_reformat_dealloc(priv->mdev, e->pkt_reformat); 1484 e->pkt_reformat = NULL; 1485 } 1486 } 1487 1488 static void mlx5e_reoffload_encap(struct mlx5e_priv *priv, 1489 struct net_device *tunnel_dev, 1490 struct mlx5e_encap_entry *e, 1491 struct list_head *encap_flows) 1492 { 1493 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1494 struct mlx5e_tc_flow *flow; 1495 int err; 1496 1497 err = ip_tunnel_info_af(e->tun_info) == AF_INET ? 1498 mlx5e_tc_tun_update_header_ipv4(priv, tunnel_dev, e) : 1499 mlx5e_tc_tun_update_header_ipv6(priv, tunnel_dev, e); 1500 if (err) 1501 mlx5_core_warn(priv->mdev, "Failed to update encap header, %d", err); 1502 e->flags &= ~MLX5_ENCAP_ENTRY_NO_ROUTE; 1503 1504 list_for_each_entry(flow, encap_flows, tmp_list) { 1505 struct mlx5e_tc_flow_parse_attr *parse_attr; 1506 struct mlx5_esw_flow_attr *esw_attr; 1507 struct mlx5_flow_handle *rule; 1508 struct mlx5_flow_attr *attr; 1509 struct mlx5_flow_spec *spec; 1510 1511 if (flow_flag_test(flow, FAILED)) 1512 continue; 1513 1514 spec = &flow->attr->parse_attr->spec; 1515 1516 attr = mlx5e_tc_get_encap_attr(flow); 1517 esw_attr = attr->esw_attr; 1518 parse_attr = attr->parse_attr; 1519 1520 err = mlx5e_update_vf_tunnel(esw, esw_attr, &parse_attr->mod_hdr_acts, 1521 e->out_dev, e->route_dev_ifindex, 1522 flow->tmp_entry_index); 1523 if (err) { 1524 mlx5_core_warn(priv->mdev, "Failed to update VF tunnel err=%d", err); 1525 continue; 1526 } 1527 1528 err = mlx5e_tc_attach_mod_hdr(priv, flow, attr); 1529 if (err) { 1530 mlx5_core_warn(priv->mdev, "Failed to update flow mod_hdr err=%d", 1531 err); 1532 continue; 1533 } 1534 1535 if (e->flags & MLX5_ENCAP_ENTRY_VALID) { 1536 esw_attr->dests[flow->tmp_entry_index].pkt_reformat = e->pkt_reformat; 1537 esw_attr->dests[flow->tmp_entry_index].flags |= MLX5_ESW_DEST_ENCAP_VALID; 1538 if (!mlx5e_tc_flow_all_encaps_valid(esw_attr)) 1539 goto offload_to_slow_path; 1540 1541 err = mlx5e_tc_offload_flow_post_acts(flow); 1542 if (err) { 1543 mlx5_core_warn(priv->mdev, "Failed to update flow post acts, %d\n", 1544 err); 1545 goto offload_to_slow_path; 1546 } 1547 1548 /* update from slow path rule to encap rule */ 1549 rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, flow->attr); 1550 if (IS_ERR(rule)) { 1551 mlx5e_tc_unoffload_flow_post_acts(flow); 1552 err = PTR_ERR(rule); 1553 mlx5_core_warn(priv->mdev, "Failed to update cached encapsulation flow, %d\n", 1554 err); 1555 } else { 1556 flow->rule[0] = rule; 1557 } 1558 } else { 1559 offload_to_slow_path: 1560 rule = mlx5e_tc_offload_to_slow_path(esw, flow, spec); 1561 /* mark the flow's encap dest as non-valid */ 1562 esw_attr->dests[flow->tmp_entry_index].flags &= 1563 ~MLX5_ESW_DEST_ENCAP_VALID; 1564 1565 if (IS_ERR(rule)) { 1566 err = PTR_ERR(rule); 1567 mlx5_core_warn(priv->mdev, "Failed to update slow path (encap) flow, %d\n", 1568 err); 1569 } else { 1570 flow->rule[0] = rule; 1571 } 1572 } 1573 flow_flag_set(flow, OFFLOADED); 1574 } 1575 } 1576 1577 static int mlx5e_update_route_encaps(struct mlx5e_priv *priv, 1578 struct mlx5e_route_entry *r, 1579 struct list_head *flow_list, 1580 bool replace) 1581 { 1582 struct net_device *tunnel_dev; 1583 struct mlx5e_encap_entry *e; 1584 1585 tunnel_dev = __dev_get_by_index(dev_net(priv->netdev), r->tunnel_dev_index); 1586 if (!tunnel_dev) 1587 return -ENODEV; 1588 1589 list_for_each_entry(e, &r->encap_entries, route_list) { 1590 LIST_HEAD(encap_flows); 1591 1592 mlx5e_take_all_encap_flows(e, &encap_flows); 1593 if (list_empty(&encap_flows)) 1594 continue; 1595 1596 if (mlx5e_route_entry_valid(r)) 1597 mlx5e_invalidate_encap(priv, e, &encap_flows); 1598 1599 if (!replace) { 1600 list_splice(&encap_flows, flow_list); 1601 continue; 1602 } 1603 1604 mlx5e_reoffload_encap(priv, tunnel_dev, e, &encap_flows); 1605 list_splice(&encap_flows, flow_list); 1606 } 1607 1608 return 0; 1609 } 1610 1611 static void mlx5e_unoffload_flow_list(struct mlx5e_priv *priv, 1612 struct list_head *flow_list) 1613 { 1614 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1615 struct mlx5e_tc_flow *flow; 1616 1617 list_for_each_entry(flow, flow_list, tmp_list) 1618 if (mlx5e_is_offloaded_flow(flow)) 1619 mlx5e_tc_unoffload_fdb_rules(esw, flow, flow->attr); 1620 } 1621 1622 static void mlx5e_reoffload_decap(struct mlx5e_priv *priv, 1623 struct list_head *decap_flows) 1624 { 1625 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1626 struct mlx5e_tc_flow *flow; 1627 1628 list_for_each_entry(flow, decap_flows, tmp_list) { 1629 struct mlx5e_tc_flow_parse_attr *parse_attr; 1630 struct mlx5_flow_attr *attr = flow->attr; 1631 struct mlx5_flow_handle *rule; 1632 struct mlx5_flow_spec *spec; 1633 int err; 1634 1635 if (flow_flag_test(flow, FAILED)) 1636 continue; 1637 1638 parse_attr = attr->parse_attr; 1639 spec = &parse_attr->spec; 1640 err = mlx5e_tc_tun_route_lookup(priv, spec, attr, parse_attr->filter_dev); 1641 if (err) { 1642 mlx5_core_warn(priv->mdev, "Failed to lookup route for flow, %d\n", 1643 err); 1644 continue; 1645 } 1646 1647 rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, attr); 1648 if (IS_ERR(rule)) { 1649 err = PTR_ERR(rule); 1650 mlx5_core_warn(priv->mdev, "Failed to update cached decap flow, %d\n", 1651 err); 1652 } else { 1653 flow->rule[0] = rule; 1654 flow_flag_set(flow, OFFLOADED); 1655 } 1656 } 1657 } 1658 1659 static int mlx5e_update_route_decap_flows(struct mlx5e_priv *priv, 1660 struct mlx5e_route_entry *r, 1661 struct list_head *flow_list, 1662 bool replace) 1663 { 1664 struct net_device *tunnel_dev; 1665 LIST_HEAD(decap_flows); 1666 1667 tunnel_dev = __dev_get_by_index(dev_net(priv->netdev), r->tunnel_dev_index); 1668 if (!tunnel_dev) 1669 return -ENODEV; 1670 1671 mlx5e_take_all_route_decap_flows(r, &decap_flows); 1672 if (mlx5e_route_entry_valid(r)) 1673 mlx5e_unoffload_flow_list(priv, &decap_flows); 1674 if (replace) 1675 mlx5e_reoffload_decap(priv, &decap_flows); 1676 1677 list_splice(&decap_flows, flow_list); 1678 1679 return 0; 1680 } 1681 1682 static void mlx5e_tc_fib_event_work(struct work_struct *work) 1683 { 1684 struct mlx5e_tc_fib_event_data *event_data = 1685 container_of(work, struct mlx5e_tc_fib_event_data, work); 1686 struct net_device *ul_dev = event_data->ul_dev; 1687 struct mlx5e_priv *priv = netdev_priv(ul_dev); 1688 struct mlx5e_route_entry *r = event_data->r; 1689 struct mlx5_eswitch *esw; 1690 LIST_HEAD(flow_list); 1691 bool replace; 1692 int err; 1693 1694 /* sync with concurrent neigh updates */ 1695 rtnl_lock(); 1696 esw = priv->mdev->priv.eswitch; 1697 mutex_lock(&esw->offloads.encap_tbl_lock); 1698 replace = event_data->event == FIB_EVENT_ENTRY_REPLACE; 1699 1700 if (!mlx5e_route_entry_valid(r) && !replace) 1701 goto out; 1702 1703 err = mlx5e_update_route_encaps(priv, r, &flow_list, replace); 1704 if (err) 1705 mlx5_core_warn(priv->mdev, "Failed to update route encaps, %d\n", 1706 err); 1707 1708 err = mlx5e_update_route_decap_flows(priv, r, &flow_list, replace); 1709 if (err) 1710 mlx5_core_warn(priv->mdev, "Failed to update route decap flows, %d\n", 1711 err); 1712 1713 if (replace) 1714 r->flags |= MLX5E_ROUTE_ENTRY_VALID; 1715 out: 1716 mutex_unlock(&esw->offloads.encap_tbl_lock); 1717 rtnl_unlock(); 1718 1719 mlx5e_put_flow_list(priv, &flow_list); 1720 mlx5e_route_put(priv, event_data->r); 1721 dev_put(event_data->ul_dev); 1722 kfree(event_data); 1723 } 1724 1725 static struct mlx5e_tc_fib_event_data * 1726 mlx5e_init_fib_work_ipv4(struct mlx5e_priv *priv, 1727 struct net_device *ul_dev, 1728 struct mlx5e_tc_tun_encap *encap, 1729 unsigned long event, 1730 struct fib_notifier_info *info) 1731 { 1732 struct fib_entry_notifier_info *fen_info; 1733 struct mlx5e_tc_fib_event_data *fib_work; 1734 struct mlx5e_route_entry *r; 1735 struct mlx5e_route_key key; 1736 struct net_device *fib_dev; 1737 1738 fen_info = container_of(info, struct fib_entry_notifier_info, info); 1739 if (fen_info->fi->nh) 1740 return NULL; 1741 fib_dev = fib_info_nh(fen_info->fi, 0)->fib_nh_dev; 1742 if (!fib_dev || fib_dev->netdev_ops != &mlx5e_netdev_ops || 1743 fen_info->dst_len != 32) 1744 return NULL; 1745 1746 fib_work = mlx5e_tc_init_fib_work(event, ul_dev, GFP_ATOMIC); 1747 if (!fib_work) 1748 return ERR_PTR(-ENOMEM); 1749 1750 key.endpoint_ip.v4 = htonl(fen_info->dst); 1751 key.ip_version = 4; 1752 1753 /* Can't fail after this point because releasing reference to r 1754 * requires obtaining sleeping mutex which we can't do in atomic 1755 * context. 1756 */ 1757 r = mlx5e_route_lookup_for_update(encap, &key); 1758 if (!r) 1759 goto out; 1760 fib_work->r = r; 1761 dev_hold(ul_dev); 1762 1763 return fib_work; 1764 1765 out: 1766 kfree(fib_work); 1767 return NULL; 1768 } 1769 1770 static struct mlx5e_tc_fib_event_data * 1771 mlx5e_init_fib_work_ipv6(struct mlx5e_priv *priv, 1772 struct net_device *ul_dev, 1773 struct mlx5e_tc_tun_encap *encap, 1774 unsigned long event, 1775 struct fib_notifier_info *info) 1776 { 1777 struct fib6_entry_notifier_info *fen_info; 1778 struct mlx5e_tc_fib_event_data *fib_work; 1779 struct mlx5e_route_entry *r; 1780 struct mlx5e_route_key key; 1781 struct net_device *fib_dev; 1782 1783 fen_info = container_of(info, struct fib6_entry_notifier_info, info); 1784 fib_dev = fib6_info_nh_dev(fen_info->rt); 1785 if (fib_dev->netdev_ops != &mlx5e_netdev_ops || 1786 fen_info->rt->fib6_dst.plen != 128) 1787 return NULL; 1788 1789 fib_work = mlx5e_tc_init_fib_work(event, ul_dev, GFP_ATOMIC); 1790 if (!fib_work) 1791 return ERR_PTR(-ENOMEM); 1792 1793 memcpy(&key.endpoint_ip.v6, &fen_info->rt->fib6_dst.addr, 1794 sizeof(fen_info->rt->fib6_dst.addr)); 1795 key.ip_version = 6; 1796 1797 /* Can't fail after this point because releasing reference to r 1798 * requires obtaining sleeping mutex which we can't do in atomic 1799 * context. 1800 */ 1801 r = mlx5e_route_lookup_for_update(encap, &key); 1802 if (!r) 1803 goto out; 1804 fib_work->r = r; 1805 dev_hold(ul_dev); 1806 1807 return fib_work; 1808 1809 out: 1810 kfree(fib_work); 1811 return NULL; 1812 } 1813 1814 static int mlx5e_tc_tun_fib_event(struct notifier_block *nb, unsigned long event, void *ptr) 1815 { 1816 struct mlx5e_tc_fib_event_data *fib_work; 1817 struct fib_notifier_info *info = ptr; 1818 struct mlx5e_tc_tun_encap *encap; 1819 struct net_device *ul_dev; 1820 struct mlx5e_priv *priv; 1821 1822 encap = container_of(nb, struct mlx5e_tc_tun_encap, fib_nb); 1823 priv = encap->priv; 1824 ul_dev = priv->netdev; 1825 priv = netdev_priv(ul_dev); 1826 1827 switch (event) { 1828 case FIB_EVENT_ENTRY_REPLACE: 1829 case FIB_EVENT_ENTRY_DEL: 1830 if (info->family == AF_INET) 1831 fib_work = mlx5e_init_fib_work_ipv4(priv, ul_dev, encap, event, info); 1832 else if (info->family == AF_INET6) 1833 fib_work = mlx5e_init_fib_work_ipv6(priv, ul_dev, encap, event, info); 1834 else 1835 return NOTIFY_DONE; 1836 1837 if (!IS_ERR_OR_NULL(fib_work)) { 1838 queue_work(priv->wq, &fib_work->work); 1839 } else if (IS_ERR(fib_work)) { 1840 NL_SET_ERR_MSG_MOD(info->extack, "Failed to init fib work"); 1841 mlx5_core_warn(priv->mdev, "Failed to init fib work, %ld\n", 1842 PTR_ERR(fib_work)); 1843 } 1844 1845 break; 1846 default: 1847 return NOTIFY_DONE; 1848 } 1849 1850 return NOTIFY_DONE; 1851 } 1852 1853 struct mlx5e_tc_tun_encap *mlx5e_tc_tun_init(struct mlx5e_priv *priv) 1854 { 1855 struct mlx5e_tc_tun_encap *encap; 1856 int err; 1857 1858 encap = kvzalloc(sizeof(*encap), GFP_KERNEL); 1859 if (!encap) 1860 return ERR_PTR(-ENOMEM); 1861 1862 encap->priv = priv; 1863 encap->fib_nb.notifier_call = mlx5e_tc_tun_fib_event; 1864 spin_lock_init(&encap->route_lock); 1865 hash_init(encap->route_tbl); 1866 err = register_fib_notifier(dev_net(priv->netdev), &encap->fib_nb, 1867 NULL, NULL); 1868 if (err) { 1869 kvfree(encap); 1870 return ERR_PTR(err); 1871 } 1872 1873 return encap; 1874 } 1875 1876 void mlx5e_tc_tun_cleanup(struct mlx5e_tc_tun_encap *encap) 1877 { 1878 if (!encap) 1879 return; 1880 1881 unregister_fib_notifier(dev_net(encap->priv->netdev), &encap->fib_nb); 1882 flush_workqueue(encap->priv->wq); /* flush fib event works */ 1883 kvfree(encap); 1884 } 1885