1 /* 2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/skbuff.h> 35 #include <linux/if_arp.h> 36 #include <linux/netdevice.h> 37 #include <linux/if.h> 38 #include <linux/if_vlan.h> 39 #include <net/udp_tunnel.h> 40 #include <net/sch_generic.h> 41 #include <linux/netfilter.h> 42 #include <rdma/ib_addr.h> 43 44 #include "rxe.h" 45 #include "rxe_net.h" 46 #include "rxe_loc.h" 47 48 static struct rxe_recv_sockets recv_sockets; 49 50 struct device *rxe_dma_device(struct rxe_dev *rxe) 51 { 52 struct net_device *ndev; 53 54 ndev = rxe->ndev; 55 56 if (is_vlan_dev(ndev)) 57 ndev = vlan_dev_real_dev(ndev); 58 59 return ndev->dev.parent; 60 } 61 62 int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid) 63 { 64 int err; 65 unsigned char ll_addr[ETH_ALEN]; 66 67 ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr); 68 err = dev_mc_add(rxe->ndev, ll_addr); 69 70 return err; 71 } 72 73 int rxe_mcast_delete(struct rxe_dev *rxe, union ib_gid *mgid) 74 { 75 int err; 76 unsigned char ll_addr[ETH_ALEN]; 77 78 ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr); 79 err = dev_mc_del(rxe->ndev, ll_addr); 80 81 return err; 82 } 83 84 static struct dst_entry *rxe_find_route4(struct net_device *ndev, 85 struct in_addr *saddr, 86 struct in_addr *daddr) 87 { 88 struct rtable *rt; 89 struct flowi4 fl = { { 0 } }; 90 91 memset(&fl, 0, sizeof(fl)); 92 fl.flowi4_oif = ndev->ifindex; 93 memcpy(&fl.saddr, saddr, sizeof(*saddr)); 94 memcpy(&fl.daddr, daddr, sizeof(*daddr)); 95 fl.flowi4_proto = IPPROTO_UDP; 96 97 rt = ip_route_output_key(&init_net, &fl); 98 if (IS_ERR(rt)) { 99 pr_err_ratelimited("no route to %pI4\n", &daddr->s_addr); 100 return NULL; 101 } 102 103 return &rt->dst; 104 } 105 106 #if IS_ENABLED(CONFIG_IPV6) 107 static struct dst_entry *rxe_find_route6(struct net_device *ndev, 108 struct in6_addr *saddr, 109 struct in6_addr *daddr) 110 { 111 struct dst_entry *ndst; 112 struct flowi6 fl6 = { { 0 } }; 113 114 memset(&fl6, 0, sizeof(fl6)); 115 fl6.flowi6_oif = ndev->ifindex; 116 memcpy(&fl6.saddr, saddr, sizeof(*saddr)); 117 memcpy(&fl6.daddr, daddr, sizeof(*daddr)); 118 fl6.flowi6_proto = IPPROTO_UDP; 119 120 ndst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(recv_sockets.sk6->sk), 121 recv_sockets.sk6->sk, &fl6, 122 NULL); 123 if (unlikely(IS_ERR(ndst))) { 124 pr_err_ratelimited("no route to %pI6\n", daddr); 125 return NULL; 126 } 127 128 if (unlikely(ndst->error)) { 129 pr_err("no route to %pI6\n", daddr); 130 goto put; 131 } 132 133 return ndst; 134 put: 135 dst_release(ndst); 136 return NULL; 137 } 138 139 #else 140 141 static struct dst_entry *rxe_find_route6(struct net_device *ndev, 142 struct in6_addr *saddr, 143 struct in6_addr *daddr) 144 { 145 return NULL; 146 } 147 148 #endif 149 150 static struct dst_entry *rxe_find_route(struct net_device *ndev, 151 struct rxe_qp *qp, 152 struct rxe_av *av) 153 { 154 struct dst_entry *dst = NULL; 155 156 if (qp_type(qp) == IB_QPT_RC) 157 dst = sk_dst_get(qp->sk->sk); 158 159 if (!dst || !dst_check(dst, qp->dst_cookie)) { 160 if (dst) 161 dst_release(dst); 162 163 if (av->network_type == RDMA_NETWORK_IPV4) { 164 struct in_addr *saddr; 165 struct in_addr *daddr; 166 167 saddr = &av->sgid_addr._sockaddr_in.sin_addr; 168 daddr = &av->dgid_addr._sockaddr_in.sin_addr; 169 dst = rxe_find_route4(ndev, saddr, daddr); 170 } else if (av->network_type == RDMA_NETWORK_IPV6) { 171 struct in6_addr *saddr6; 172 struct in6_addr *daddr6; 173 174 saddr6 = &av->sgid_addr._sockaddr_in6.sin6_addr; 175 daddr6 = &av->dgid_addr._sockaddr_in6.sin6_addr; 176 dst = rxe_find_route6(ndev, saddr6, daddr6); 177 #if IS_ENABLED(CONFIG_IPV6) 178 if (dst) 179 qp->dst_cookie = 180 rt6_get_cookie((struct rt6_info *)dst); 181 #endif 182 } 183 184 if (dst && (qp_type(qp) == IB_QPT_RC)) { 185 dst_hold(dst); 186 sk_dst_set(qp->sk->sk, dst); 187 } 188 } 189 return dst; 190 } 191 192 static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) 193 { 194 struct udphdr *udph; 195 struct net_device *ndev = skb->dev; 196 struct net_device *rdev = ndev; 197 struct rxe_dev *rxe = rxe_get_dev_from_net(ndev); 198 struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); 199 200 if (!rxe && is_vlan_dev(rdev)) { 201 rdev = vlan_dev_real_dev(ndev); 202 rxe = rxe_get_dev_from_net(rdev); 203 } 204 if (!rxe) 205 goto drop; 206 207 if (skb_linearize(skb)) { 208 pr_err("skb_linearize failed\n"); 209 ib_device_put(&rxe->ib_dev); 210 goto drop; 211 } 212 213 udph = udp_hdr(skb); 214 pkt->rxe = rxe; 215 pkt->port_num = 1; 216 pkt->hdr = (u8 *)(udph + 1); 217 pkt->mask = RXE_GRH_MASK; 218 pkt->paylen = be16_to_cpu(udph->len) - sizeof(*udph); 219 220 rxe_rcv(skb); 221 222 /* 223 * FIXME: this is in the wrong place, it needs to be done when pkt is 224 * destroyed 225 */ 226 ib_device_put(&rxe->ib_dev); 227 228 return 0; 229 drop: 230 kfree_skb(skb); 231 232 return 0; 233 } 234 235 static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port, 236 bool ipv6) 237 { 238 int err; 239 struct socket *sock; 240 struct udp_port_cfg udp_cfg = { }; 241 struct udp_tunnel_sock_cfg tnl_cfg = { }; 242 243 if (ipv6) { 244 udp_cfg.family = AF_INET6; 245 udp_cfg.ipv6_v6only = 1; 246 } else { 247 udp_cfg.family = AF_INET; 248 } 249 250 udp_cfg.local_udp_port = port; 251 252 /* Create UDP socket */ 253 err = udp_sock_create(net, &udp_cfg, &sock); 254 if (err < 0) { 255 pr_err("failed to create udp socket. err = %d\n", err); 256 return ERR_PTR(err); 257 } 258 259 tnl_cfg.encap_type = 1; 260 tnl_cfg.encap_rcv = rxe_udp_encap_recv; 261 262 /* Setup UDP tunnel */ 263 setup_udp_tunnel_sock(net, sock, &tnl_cfg); 264 265 return sock; 266 } 267 268 static void rxe_release_udp_tunnel(struct socket *sk) 269 { 270 if (sk) 271 udp_tunnel_sock_release(sk); 272 } 273 274 static void prepare_udp_hdr(struct sk_buff *skb, __be16 src_port, 275 __be16 dst_port) 276 { 277 struct udphdr *udph; 278 279 __skb_push(skb, sizeof(*udph)); 280 skb_reset_transport_header(skb); 281 udph = udp_hdr(skb); 282 283 udph->dest = dst_port; 284 udph->source = src_port; 285 udph->len = htons(skb->len); 286 udph->check = 0; 287 } 288 289 static void prepare_ipv4_hdr(struct dst_entry *dst, struct sk_buff *skb, 290 __be32 saddr, __be32 daddr, __u8 proto, 291 __u8 tos, __u8 ttl, __be16 df, bool xnet) 292 { 293 struct iphdr *iph; 294 295 skb_scrub_packet(skb, xnet); 296 297 skb_clear_hash(skb); 298 skb_dst_set(skb, dst_clone(dst)); 299 memset(IPCB(skb), 0, sizeof(*IPCB(skb))); 300 301 skb_push(skb, sizeof(struct iphdr)); 302 skb_reset_network_header(skb); 303 304 iph = ip_hdr(skb); 305 306 iph->version = IPVERSION; 307 iph->ihl = sizeof(struct iphdr) >> 2; 308 iph->frag_off = df; 309 iph->protocol = proto; 310 iph->tos = tos; 311 iph->daddr = daddr; 312 iph->saddr = saddr; 313 iph->ttl = ttl; 314 __ip_select_ident(dev_net(dst->dev), iph, 315 skb_shinfo(skb)->gso_segs ?: 1); 316 iph->tot_len = htons(skb->len); 317 ip_send_check(iph); 318 } 319 320 static void prepare_ipv6_hdr(struct dst_entry *dst, struct sk_buff *skb, 321 struct in6_addr *saddr, struct in6_addr *daddr, 322 __u8 proto, __u8 prio, __u8 ttl) 323 { 324 struct ipv6hdr *ip6h; 325 326 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); 327 IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED 328 | IPSKB_REROUTED); 329 skb_dst_set(skb, dst_clone(dst)); 330 331 __skb_push(skb, sizeof(*ip6h)); 332 skb_reset_network_header(skb); 333 ip6h = ipv6_hdr(skb); 334 ip6_flow_hdr(ip6h, prio, htonl(0)); 335 ip6h->payload_len = htons(skb->len); 336 ip6h->nexthdr = proto; 337 ip6h->hop_limit = ttl; 338 ip6h->daddr = *daddr; 339 ip6h->saddr = *saddr; 340 ip6h->payload_len = htons(skb->len - sizeof(*ip6h)); 341 } 342 343 static int prepare4(struct rxe_pkt_info *pkt, struct sk_buff *skb) 344 { 345 struct rxe_qp *qp = pkt->qp; 346 struct dst_entry *dst; 347 bool xnet = false; 348 __be16 df = htons(IP_DF); 349 struct rxe_av *av = rxe_get_av(pkt); 350 struct in_addr *saddr = &av->sgid_addr._sockaddr_in.sin_addr; 351 struct in_addr *daddr = &av->dgid_addr._sockaddr_in.sin_addr; 352 353 dst = rxe_find_route(skb->dev, qp, av); 354 if (!dst) { 355 pr_err("Host not reachable\n"); 356 return -EHOSTUNREACH; 357 } 358 359 prepare_udp_hdr(skb, cpu_to_be16(qp->src_port), 360 cpu_to_be16(ROCE_V2_UDP_DPORT)); 361 362 prepare_ipv4_hdr(dst, skb, saddr->s_addr, daddr->s_addr, IPPROTO_UDP, 363 av->grh.traffic_class, av->grh.hop_limit, df, xnet); 364 365 dst_release(dst); 366 return 0; 367 } 368 369 static int prepare6(struct rxe_pkt_info *pkt, struct sk_buff *skb) 370 { 371 struct rxe_qp *qp = pkt->qp; 372 struct dst_entry *dst; 373 struct rxe_av *av = rxe_get_av(pkt); 374 struct in6_addr *saddr = &av->sgid_addr._sockaddr_in6.sin6_addr; 375 struct in6_addr *daddr = &av->dgid_addr._sockaddr_in6.sin6_addr; 376 377 dst = rxe_find_route(skb->dev, qp, av); 378 if (!dst) { 379 pr_err("Host not reachable\n"); 380 return -EHOSTUNREACH; 381 } 382 383 prepare_udp_hdr(skb, cpu_to_be16(qp->src_port), 384 cpu_to_be16(ROCE_V2_UDP_DPORT)); 385 386 prepare_ipv6_hdr(dst, skb, saddr, daddr, IPPROTO_UDP, 387 av->grh.traffic_class, 388 av->grh.hop_limit); 389 390 dst_release(dst); 391 return 0; 392 } 393 394 int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb, u32 *crc) 395 { 396 int err = 0; 397 398 if (skb->protocol == htons(ETH_P_IP)) 399 err = prepare4(pkt, skb); 400 else if (skb->protocol == htons(ETH_P_IPV6)) 401 err = prepare6(pkt, skb); 402 403 *crc = rxe_icrc_hdr(pkt, skb); 404 405 if (ether_addr_equal(skb->dev->dev_addr, rxe_get_av(pkt)->dmac)) 406 pkt->mask |= RXE_LOOPBACK_MASK; 407 408 return err; 409 } 410 411 static void rxe_skb_tx_dtor(struct sk_buff *skb) 412 { 413 struct sock *sk = skb->sk; 414 struct rxe_qp *qp = sk->sk_user_data; 415 int skb_out = atomic_dec_return(&qp->skb_out); 416 417 if (unlikely(qp->need_req_skb && 418 skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW)) 419 rxe_run_task(&qp->req.task, 1); 420 421 rxe_drop_ref(qp); 422 } 423 424 int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb) 425 { 426 int err; 427 428 skb->destructor = rxe_skb_tx_dtor; 429 skb->sk = pkt->qp->sk->sk; 430 431 rxe_add_ref(pkt->qp); 432 atomic_inc(&pkt->qp->skb_out); 433 434 if (skb->protocol == htons(ETH_P_IP)) { 435 err = ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); 436 } else if (skb->protocol == htons(ETH_P_IPV6)) { 437 err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); 438 } else { 439 pr_err("Unknown layer 3 protocol: %d\n", skb->protocol); 440 atomic_dec(&pkt->qp->skb_out); 441 rxe_drop_ref(pkt->qp); 442 kfree_skb(skb); 443 return -EINVAL; 444 } 445 446 if (unlikely(net_xmit_eval(err))) { 447 pr_debug("error sending packet: %d\n", err); 448 return -EAGAIN; 449 } 450 451 return 0; 452 } 453 454 void rxe_loopback(struct sk_buff *skb) 455 { 456 rxe_rcv(skb); 457 } 458 459 struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, 460 int paylen, struct rxe_pkt_info *pkt) 461 { 462 unsigned int hdr_len; 463 struct sk_buff *skb = NULL; 464 struct net_device *ndev; 465 const struct ib_gid_attr *attr; 466 const int port_num = 1; 467 468 attr = rdma_get_gid_attr(&rxe->ib_dev, port_num, av->grh.sgid_index); 469 if (IS_ERR(attr)) 470 return NULL; 471 472 if (av->network_type == RDMA_NETWORK_IPV4) 473 hdr_len = ETH_HLEN + sizeof(struct udphdr) + 474 sizeof(struct iphdr); 475 else 476 hdr_len = ETH_HLEN + sizeof(struct udphdr) + 477 sizeof(struct ipv6hdr); 478 479 rcu_read_lock(); 480 ndev = rdma_read_gid_attr_ndev_rcu(attr); 481 if (IS_ERR(ndev)) { 482 rcu_read_unlock(); 483 goto out; 484 } 485 skb = alloc_skb(paylen + hdr_len + LL_RESERVED_SPACE(ndev), 486 GFP_ATOMIC); 487 488 if (unlikely(!skb)) { 489 rcu_read_unlock(); 490 goto out; 491 } 492 493 skb_reserve(skb, hdr_len + LL_RESERVED_SPACE(ndev)); 494 495 /* FIXME: hold reference to this netdev until life of this skb. */ 496 skb->dev = ndev; 497 rcu_read_unlock(); 498 499 if (av->network_type == RDMA_NETWORK_IPV4) 500 skb->protocol = htons(ETH_P_IP); 501 else 502 skb->protocol = htons(ETH_P_IPV6); 503 504 pkt->rxe = rxe; 505 pkt->port_num = port_num; 506 pkt->hdr = skb_put_zero(skb, paylen); 507 pkt->mask |= RXE_GRH_MASK; 508 509 out: 510 rdma_put_gid_attr(attr); 511 return skb; 512 } 513 514 /* 515 * this is required by rxe_cfg to match rxe devices in 516 * /sys/class/infiniband up with their underlying ethernet devices 517 */ 518 const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num) 519 { 520 return rxe->ndev->name; 521 } 522 523 int rxe_net_add(const char *ibdev_name, struct net_device *ndev) 524 { 525 int err; 526 struct rxe_dev *rxe = NULL; 527 528 rxe = ib_alloc_device(rxe_dev, ib_dev); 529 if (!rxe) 530 return -ENOMEM; 531 532 rxe->ndev = ndev; 533 534 err = rxe_add(rxe, ndev->mtu, ibdev_name); 535 if (err) { 536 ib_dealloc_device(&rxe->ib_dev); 537 return err; 538 } 539 540 return 0; 541 } 542 543 static void rxe_port_event(struct rxe_dev *rxe, 544 enum ib_event_type event) 545 { 546 struct ib_event ev; 547 548 ev.device = &rxe->ib_dev; 549 ev.element.port_num = 1; 550 ev.event = event; 551 552 ib_dispatch_event(&ev); 553 } 554 555 /* Caller must hold net_info_lock */ 556 void rxe_port_up(struct rxe_dev *rxe) 557 { 558 struct rxe_port *port; 559 560 port = &rxe->port; 561 port->attr.state = IB_PORT_ACTIVE; 562 563 rxe_port_event(rxe, IB_EVENT_PORT_ACTIVE); 564 dev_info(&rxe->ib_dev.dev, "set active\n"); 565 } 566 567 /* Caller must hold net_info_lock */ 568 void rxe_port_down(struct rxe_dev *rxe) 569 { 570 struct rxe_port *port; 571 572 port = &rxe->port; 573 port->attr.state = IB_PORT_DOWN; 574 575 rxe_port_event(rxe, IB_EVENT_PORT_ERR); 576 rxe_counter_inc(rxe, RXE_CNT_LINK_DOWNED); 577 dev_info(&rxe->ib_dev.dev, "set down\n"); 578 } 579 580 void rxe_set_port_state(struct rxe_dev *rxe) 581 { 582 if (netif_running(rxe->ndev) && netif_carrier_ok(rxe->ndev)) 583 rxe_port_up(rxe); 584 else 585 rxe_port_down(rxe); 586 } 587 588 static int rxe_notify(struct notifier_block *not_blk, 589 unsigned long event, 590 void *arg) 591 { 592 struct net_device *ndev = netdev_notifier_info_to_dev(arg); 593 struct rxe_dev *rxe = rxe_get_dev_from_net(ndev); 594 595 if (!rxe) 596 return NOTIFY_OK; 597 598 switch (event) { 599 case NETDEV_UNREGISTER: 600 ib_unregister_device_queued(&rxe->ib_dev); 601 break; 602 case NETDEV_UP: 603 rxe_port_up(rxe); 604 break; 605 case NETDEV_DOWN: 606 rxe_port_down(rxe); 607 break; 608 case NETDEV_CHANGEMTU: 609 pr_info("%s changed mtu to %d\n", ndev->name, ndev->mtu); 610 rxe_set_mtu(rxe, ndev->mtu); 611 break; 612 case NETDEV_CHANGE: 613 rxe_set_port_state(rxe); 614 break; 615 case NETDEV_REBOOT: 616 case NETDEV_GOING_DOWN: 617 case NETDEV_CHANGEADDR: 618 case NETDEV_CHANGENAME: 619 case NETDEV_FEAT_CHANGE: 620 default: 621 pr_info("ignoring netdev event = %ld for %s\n", 622 event, ndev->name); 623 break; 624 } 625 626 ib_device_put(&rxe->ib_dev); 627 return NOTIFY_OK; 628 } 629 630 static struct notifier_block rxe_net_notifier = { 631 .notifier_call = rxe_notify, 632 }; 633 634 static int rxe_net_ipv4_init(void) 635 { 636 recv_sockets.sk4 = rxe_setup_udp_tunnel(&init_net, 637 htons(ROCE_V2_UDP_DPORT), false); 638 if (IS_ERR(recv_sockets.sk4)) { 639 recv_sockets.sk4 = NULL; 640 pr_err("Failed to create IPv4 UDP tunnel\n"); 641 return -1; 642 } 643 644 return 0; 645 } 646 647 static int rxe_net_ipv6_init(void) 648 { 649 #if IS_ENABLED(CONFIG_IPV6) 650 651 recv_sockets.sk6 = rxe_setup_udp_tunnel(&init_net, 652 htons(ROCE_V2_UDP_DPORT), true); 653 if (IS_ERR(recv_sockets.sk6)) { 654 recv_sockets.sk6 = NULL; 655 pr_err("Failed to create IPv6 UDP tunnel\n"); 656 return -1; 657 } 658 #endif 659 return 0; 660 } 661 662 void rxe_net_exit(void) 663 { 664 rxe_release_udp_tunnel(recv_sockets.sk6); 665 rxe_release_udp_tunnel(recv_sockets.sk4); 666 unregister_netdevice_notifier(&rxe_net_notifier); 667 } 668 669 int rxe_net_init(void) 670 { 671 int err; 672 673 recv_sockets.sk6 = NULL; 674 675 err = rxe_net_ipv4_init(); 676 if (err) 677 return err; 678 err = rxe_net_ipv6_init(); 679 if (err) 680 goto err_out; 681 err = register_netdevice_notifier(&rxe_net_notifier); 682 if (err) { 683 pr_err("Failed to register netdev notifier\n"); 684 goto err_out; 685 } 686 return 0; 687 err_out: 688 rxe_net_exit(); 689 return err; 690 } 691