1 // SPDX-License-Identifier: GPL-2.0 2 /* XDP sockets 3 * 4 * AF_XDP sockets allows a channel between XDP programs and userspace 5 * applications. 6 * Copyright(c) 2018 Intel Corporation. 7 * 8 * Author(s): Björn Töpel <bjorn.topel@intel.com> 9 * Magnus Karlsson <magnus.karlsson@intel.com> 10 */ 11 12 #define pr_fmt(fmt) "AF_XDP: %s: " fmt, __func__ 13 14 #include <linux/if_xdp.h> 15 #include <linux/init.h> 16 #include <linux/sched/mm.h> 17 #include <linux/sched/signal.h> 18 #include <linux/sched/task.h> 19 #include <linux/socket.h> 20 #include <linux/file.h> 21 #include <linux/uaccess.h> 22 #include <linux/net.h> 23 #include <linux/netdevice.h> 24 #include <linux/rculist.h> 25 #include <net/xdp_sock.h> 26 #include <net/xdp.h> 27 28 #include "xsk_queue.h" 29 #include "xdp_umem.h" 30 #include "xsk.h" 31 32 #define TX_BATCH_SIZE 16 33 34 static DEFINE_PER_CPU(struct list_head, xskmap_flush_list); 35 36 bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs) 37 { 38 return READ_ONCE(xs->rx) && READ_ONCE(xs->umem) && 39 READ_ONCE(xs->umem->fq); 40 } 41 42 bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt) 43 { 44 return xskq_cons_has_entries(umem->fq, cnt); 45 } 46 EXPORT_SYMBOL(xsk_umem_has_addrs); 47 48 bool xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr) 49 { 50 return xskq_cons_peek_addr(umem->fq, addr, umem); 51 } 52 EXPORT_SYMBOL(xsk_umem_peek_addr); 53 54 void xsk_umem_release_addr(struct xdp_umem *umem) 55 { 56 xskq_cons_release(umem->fq); 57 } 58 EXPORT_SYMBOL(xsk_umem_release_addr); 59 60 void xsk_set_rx_need_wakeup(struct xdp_umem *umem) 61 { 62 if (umem->need_wakeup & XDP_WAKEUP_RX) 63 return; 64 65 umem->fq->ring->flags |= XDP_RING_NEED_WAKEUP; 66 umem->need_wakeup |= XDP_WAKEUP_RX; 67 } 68 EXPORT_SYMBOL(xsk_set_rx_need_wakeup); 69 70 void xsk_set_tx_need_wakeup(struct xdp_umem *umem) 71 { 72 struct xdp_sock *xs; 73 74 if (umem->need_wakeup & XDP_WAKEUP_TX) 75 return; 76 77 rcu_read_lock(); 78 list_for_each_entry_rcu(xs, &umem->xsk_list, list) { 79 xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP; 80 } 81 rcu_read_unlock(); 82 83 umem->need_wakeup |= XDP_WAKEUP_TX; 84 } 85 EXPORT_SYMBOL(xsk_set_tx_need_wakeup); 86 87 void xsk_clear_rx_need_wakeup(struct xdp_umem *umem) 88 { 89 if (!(umem->need_wakeup & XDP_WAKEUP_RX)) 90 return; 91 92 umem->fq->ring->flags &= ~XDP_RING_NEED_WAKEUP; 93 umem->need_wakeup &= ~XDP_WAKEUP_RX; 94 } 95 EXPORT_SYMBOL(xsk_clear_rx_need_wakeup); 96 97 void xsk_clear_tx_need_wakeup(struct xdp_umem *umem) 98 { 99 struct xdp_sock *xs; 100 101 if (!(umem->need_wakeup & XDP_WAKEUP_TX)) 102 return; 103 104 rcu_read_lock(); 105 list_for_each_entry_rcu(xs, &umem->xsk_list, list) { 106 xs->tx->ring->flags &= ~XDP_RING_NEED_WAKEUP; 107 } 108 rcu_read_unlock(); 109 110 umem->need_wakeup &= ~XDP_WAKEUP_TX; 111 } 112 EXPORT_SYMBOL(xsk_clear_tx_need_wakeup); 113 114 bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem) 115 { 116 return umem->flags & XDP_UMEM_USES_NEED_WAKEUP; 117 } 118 EXPORT_SYMBOL(xsk_umem_uses_need_wakeup); 119 120 /* If a buffer crosses a page boundary, we need to do 2 memcpy's, one for 121 * each page. This is only required in copy mode. 122 */ 123 static void __xsk_rcv_memcpy(struct xdp_umem *umem, u64 addr, void *from_buf, 124 u32 len, u32 metalen) 125 { 126 void *to_buf = xdp_umem_get_data(umem, addr); 127 128 addr = xsk_umem_add_offset_to_addr(addr); 129 if (xskq_cons_crosses_non_contig_pg(umem, addr, len + metalen)) { 130 void *next_pg_addr = umem->pages[(addr >> PAGE_SHIFT) + 1].addr; 131 u64 page_start = addr & ~(PAGE_SIZE - 1); 132 u64 first_len = PAGE_SIZE - (addr - page_start); 133 134 memcpy(to_buf, from_buf, first_len); 135 memcpy(next_pg_addr, from_buf + first_len, 136 len + metalen - first_len); 137 138 return; 139 } 140 141 memcpy(to_buf, from_buf, len + metalen); 142 } 143 144 static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) 145 { 146 u64 offset = xs->umem->headroom; 147 u64 addr, memcpy_addr; 148 void *from_buf; 149 u32 metalen; 150 int err; 151 152 if (!xskq_cons_peek_addr(xs->umem->fq, &addr, xs->umem) || 153 len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) { 154 xs->rx_dropped++; 155 return -ENOSPC; 156 } 157 158 if (unlikely(xdp_data_meta_unsupported(xdp))) { 159 from_buf = xdp->data; 160 metalen = 0; 161 } else { 162 from_buf = xdp->data_meta; 163 metalen = xdp->data - xdp->data_meta; 164 } 165 166 memcpy_addr = xsk_umem_adjust_offset(xs->umem, addr, offset); 167 __xsk_rcv_memcpy(xs->umem, memcpy_addr, from_buf, len, metalen); 168 169 offset += metalen; 170 addr = xsk_umem_adjust_offset(xs->umem, addr, offset); 171 err = xskq_prod_reserve_desc(xs->rx, addr, len); 172 if (!err) { 173 xskq_cons_release(xs->umem->fq); 174 xdp_return_buff(xdp); 175 return 0; 176 } 177 178 xs->rx_dropped++; 179 return err; 180 } 181 182 static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) 183 { 184 int err = xskq_prod_reserve_desc(xs->rx, xdp->handle, len); 185 186 if (err) 187 xs->rx_dropped++; 188 189 return err; 190 } 191 192 static bool xsk_is_bound(struct xdp_sock *xs) 193 { 194 if (READ_ONCE(xs->state) == XSK_BOUND) { 195 /* Matches smp_wmb() in bind(). */ 196 smp_rmb(); 197 return true; 198 } 199 return false; 200 } 201 202 static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 203 { 204 u32 len; 205 206 if (!xsk_is_bound(xs)) 207 return -EINVAL; 208 209 if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) 210 return -EINVAL; 211 212 len = xdp->data_end - xdp->data; 213 214 return (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ? 215 __xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len); 216 } 217 218 static void xsk_flush(struct xdp_sock *xs) 219 { 220 xskq_prod_submit(xs->rx); 221 __xskq_cons_release(xs->umem->fq); 222 sock_def_readable(&xs->sk); 223 } 224 225 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 226 { 227 u32 metalen = xdp->data - xdp->data_meta; 228 u32 len = xdp->data_end - xdp->data; 229 u64 offset = xs->umem->headroom; 230 void *buffer; 231 u64 addr; 232 int err; 233 234 spin_lock_bh(&xs->rx_lock); 235 236 if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) { 237 err = -EINVAL; 238 goto out_unlock; 239 } 240 241 if (!xskq_cons_peek_addr(xs->umem->fq, &addr, xs->umem) || 242 len > xs->umem->chunk_size_nohr - XDP_PACKET_HEADROOM) { 243 err = -ENOSPC; 244 goto out_drop; 245 } 246 247 addr = xsk_umem_adjust_offset(xs->umem, addr, offset); 248 buffer = xdp_umem_get_data(xs->umem, addr); 249 memcpy(buffer, xdp->data_meta, len + metalen); 250 251 addr = xsk_umem_adjust_offset(xs->umem, addr, metalen); 252 err = xskq_prod_reserve_desc(xs->rx, addr, len); 253 if (err) 254 goto out_drop; 255 256 xskq_cons_release(xs->umem->fq); 257 xskq_prod_submit(xs->rx); 258 259 spin_unlock_bh(&xs->rx_lock); 260 261 xs->sk.sk_data_ready(&xs->sk); 262 return 0; 263 264 out_drop: 265 xs->rx_dropped++; 266 out_unlock: 267 spin_unlock_bh(&xs->rx_lock); 268 return err; 269 } 270 271 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp) 272 { 273 struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list); 274 int err; 275 276 err = xsk_rcv(xs, xdp); 277 if (err) 278 return err; 279 280 if (!xs->flush_node.prev) 281 list_add(&xs->flush_node, flush_list); 282 283 return 0; 284 } 285 286 void __xsk_map_flush(void) 287 { 288 struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list); 289 struct xdp_sock *xs, *tmp; 290 291 list_for_each_entry_safe(xs, tmp, flush_list, flush_node) { 292 xsk_flush(xs); 293 __list_del_clearprev(&xs->flush_node); 294 } 295 } 296 297 void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries) 298 { 299 xskq_prod_submit_n(umem->cq, nb_entries); 300 } 301 EXPORT_SYMBOL(xsk_umem_complete_tx); 302 303 void xsk_umem_consume_tx_done(struct xdp_umem *umem) 304 { 305 struct xdp_sock *xs; 306 307 rcu_read_lock(); 308 list_for_each_entry_rcu(xs, &umem->xsk_list, list) { 309 __xskq_cons_release(xs->tx); 310 xs->sk.sk_write_space(&xs->sk); 311 } 312 rcu_read_unlock(); 313 } 314 EXPORT_SYMBOL(xsk_umem_consume_tx_done); 315 316 bool xsk_umem_consume_tx(struct xdp_umem *umem, struct xdp_desc *desc) 317 { 318 struct xdp_sock *xs; 319 320 rcu_read_lock(); 321 list_for_each_entry_rcu(xs, &umem->xsk_list, list) { 322 if (!xskq_cons_peek_desc(xs->tx, desc, umem)) 323 continue; 324 325 /* This is the backpreassure mechanism for the Tx path. 326 * Reserve space in the completion queue and only proceed 327 * if there is space in it. This avoids having to implement 328 * any buffering in the Tx path. 329 */ 330 if (xskq_prod_reserve_addr(umem->cq, desc->addr)) 331 goto out; 332 333 xskq_cons_release(xs->tx); 334 rcu_read_unlock(); 335 return true; 336 } 337 338 out: 339 rcu_read_unlock(); 340 return false; 341 } 342 EXPORT_SYMBOL(xsk_umem_consume_tx); 343 344 static int xsk_wakeup(struct xdp_sock *xs, u8 flags) 345 { 346 struct net_device *dev = xs->dev; 347 int err; 348 349 rcu_read_lock(); 350 err = dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags); 351 rcu_read_unlock(); 352 353 return err; 354 } 355 356 static int xsk_zc_xmit(struct xdp_sock *xs) 357 { 358 return xsk_wakeup(xs, XDP_WAKEUP_TX); 359 } 360 361 static void xsk_destruct_skb(struct sk_buff *skb) 362 { 363 u64 addr = (u64)(long)skb_shinfo(skb)->destructor_arg; 364 struct xdp_sock *xs = xdp_sk(skb->sk); 365 unsigned long flags; 366 367 spin_lock_irqsave(&xs->tx_completion_lock, flags); 368 xskq_prod_submit_addr(xs->umem->cq, addr); 369 spin_unlock_irqrestore(&xs->tx_completion_lock, flags); 370 371 sock_wfree(skb); 372 } 373 374 static int xsk_generic_xmit(struct sock *sk) 375 { 376 struct xdp_sock *xs = xdp_sk(sk); 377 u32 max_batch = TX_BATCH_SIZE; 378 bool sent_frame = false; 379 struct xdp_desc desc; 380 struct sk_buff *skb; 381 int err = 0; 382 383 mutex_lock(&xs->mutex); 384 385 if (xs->queue_id >= xs->dev->real_num_tx_queues) 386 goto out; 387 388 while (xskq_cons_peek_desc(xs->tx, &desc, xs->umem)) { 389 char *buffer; 390 u64 addr; 391 u32 len; 392 393 if (max_batch-- == 0) { 394 err = -EAGAIN; 395 goto out; 396 } 397 398 len = desc.len; 399 skb = sock_alloc_send_skb(sk, len, 1, &err); 400 if (unlikely(!skb)) { 401 err = -EAGAIN; 402 goto out; 403 } 404 405 skb_put(skb, len); 406 addr = desc.addr; 407 buffer = xdp_umem_get_data(xs->umem, addr); 408 err = skb_store_bits(skb, 0, buffer, len); 409 /* This is the backpreassure mechanism for the Tx path. 410 * Reserve space in the completion queue and only proceed 411 * if there is space in it. This avoids having to implement 412 * any buffering in the Tx path. 413 */ 414 if (unlikely(err) || xskq_prod_reserve(xs->umem->cq)) { 415 kfree_skb(skb); 416 goto out; 417 } 418 419 skb->dev = xs->dev; 420 skb->priority = sk->sk_priority; 421 skb->mark = sk->sk_mark; 422 skb_shinfo(skb)->destructor_arg = (void *)(long)desc.addr; 423 skb->destructor = xsk_destruct_skb; 424 425 err = dev_direct_xmit(skb, xs->queue_id); 426 xskq_cons_release(xs->tx); 427 /* Ignore NET_XMIT_CN as packet might have been sent */ 428 if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) { 429 /* SKB completed but not sent */ 430 err = -EBUSY; 431 goto out; 432 } 433 434 sent_frame = true; 435 } 436 437 out: 438 if (sent_frame) 439 sk->sk_write_space(sk); 440 441 mutex_unlock(&xs->mutex); 442 return err; 443 } 444 445 static int __xsk_sendmsg(struct sock *sk) 446 { 447 struct xdp_sock *xs = xdp_sk(sk); 448 449 if (unlikely(!(xs->dev->flags & IFF_UP))) 450 return -ENETDOWN; 451 if (unlikely(!xs->tx)) 452 return -ENOBUFS; 453 454 return xs->zc ? xsk_zc_xmit(xs) : xsk_generic_xmit(sk); 455 } 456 457 static int xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) 458 { 459 bool need_wait = !(m->msg_flags & MSG_DONTWAIT); 460 struct sock *sk = sock->sk; 461 struct xdp_sock *xs = xdp_sk(sk); 462 463 if (unlikely(!xsk_is_bound(xs))) 464 return -ENXIO; 465 if (unlikely(need_wait)) 466 return -EOPNOTSUPP; 467 468 return __xsk_sendmsg(sk); 469 } 470 471 static __poll_t xsk_poll(struct file *file, struct socket *sock, 472 struct poll_table_struct *wait) 473 { 474 __poll_t mask = datagram_poll(file, sock, wait); 475 struct sock *sk = sock->sk; 476 struct xdp_sock *xs = xdp_sk(sk); 477 struct xdp_umem *umem; 478 479 if (unlikely(!xsk_is_bound(xs))) 480 return mask; 481 482 umem = xs->umem; 483 484 if (umem->need_wakeup) { 485 if (xs->zc) 486 xsk_wakeup(xs, umem->need_wakeup); 487 else 488 /* Poll needs to drive Tx also in copy mode */ 489 __xsk_sendmsg(sk); 490 } 491 492 if (xs->rx && !xskq_prod_is_empty(xs->rx)) 493 mask |= EPOLLIN | EPOLLRDNORM; 494 if (xs->tx && !xskq_cons_is_full(xs->tx)) 495 mask |= EPOLLOUT | EPOLLWRNORM; 496 497 return mask; 498 } 499 500 static int xsk_init_queue(u32 entries, struct xsk_queue **queue, 501 bool umem_queue) 502 { 503 struct xsk_queue *q; 504 505 if (entries == 0 || *queue || !is_power_of_2(entries)) 506 return -EINVAL; 507 508 q = xskq_create(entries, umem_queue); 509 if (!q) 510 return -ENOMEM; 511 512 /* Make sure queue is ready before it can be seen by others */ 513 smp_wmb(); 514 WRITE_ONCE(*queue, q); 515 return 0; 516 } 517 518 static void xsk_unbind_dev(struct xdp_sock *xs) 519 { 520 struct net_device *dev = xs->dev; 521 522 if (xs->state != XSK_BOUND) 523 return; 524 WRITE_ONCE(xs->state, XSK_UNBOUND); 525 526 /* Wait for driver to stop using the xdp socket. */ 527 xdp_del_sk_umem(xs->umem, xs); 528 xs->dev = NULL; 529 synchronize_net(); 530 dev_put(dev); 531 } 532 533 static struct xsk_map *xsk_get_map_list_entry(struct xdp_sock *xs, 534 struct xdp_sock ***map_entry) 535 { 536 struct xsk_map *map = NULL; 537 struct xsk_map_node *node; 538 539 *map_entry = NULL; 540 541 spin_lock_bh(&xs->map_list_lock); 542 node = list_first_entry_or_null(&xs->map_list, struct xsk_map_node, 543 node); 544 if (node) { 545 WARN_ON(xsk_map_inc(node->map)); 546 map = node->map; 547 *map_entry = node->map_entry; 548 } 549 spin_unlock_bh(&xs->map_list_lock); 550 return map; 551 } 552 553 static void xsk_delete_from_maps(struct xdp_sock *xs) 554 { 555 /* This function removes the current XDP socket from all the 556 * maps it resides in. We need to take extra care here, due to 557 * the two locks involved. Each map has a lock synchronizing 558 * updates to the entries, and each socket has a lock that 559 * synchronizes access to the list of maps (map_list). For 560 * deadlock avoidance the locks need to be taken in the order 561 * "map lock"->"socket map list lock". We start off by 562 * accessing the socket map list, and take a reference to the 563 * map to guarantee existence between the 564 * xsk_get_map_list_entry() and xsk_map_try_sock_delete() 565 * calls. Then we ask the map to remove the socket, which 566 * tries to remove the socket from the map. Note that there 567 * might be updates to the map between 568 * xsk_get_map_list_entry() and xsk_map_try_sock_delete(). 569 */ 570 struct xdp_sock **map_entry = NULL; 571 struct xsk_map *map; 572 573 while ((map = xsk_get_map_list_entry(xs, &map_entry))) { 574 xsk_map_try_sock_delete(map, xs, map_entry); 575 xsk_map_put(map); 576 } 577 } 578 579 static int xsk_release(struct socket *sock) 580 { 581 struct sock *sk = sock->sk; 582 struct xdp_sock *xs = xdp_sk(sk); 583 struct net *net; 584 585 if (!sk) 586 return 0; 587 588 net = sock_net(sk); 589 590 mutex_lock(&net->xdp.lock); 591 sk_del_node_init_rcu(sk); 592 mutex_unlock(&net->xdp.lock); 593 594 local_bh_disable(); 595 sock_prot_inuse_add(net, sk->sk_prot, -1); 596 local_bh_enable(); 597 598 xsk_delete_from_maps(xs); 599 mutex_lock(&xs->mutex); 600 xsk_unbind_dev(xs); 601 mutex_unlock(&xs->mutex); 602 603 xskq_destroy(xs->rx); 604 xskq_destroy(xs->tx); 605 606 sock_orphan(sk); 607 sock->sk = NULL; 608 609 sk_refcnt_debug_release(sk); 610 sock_put(sk); 611 612 return 0; 613 } 614 615 static struct socket *xsk_lookup_xsk_from_fd(int fd) 616 { 617 struct socket *sock; 618 int err; 619 620 sock = sockfd_lookup(fd, &err); 621 if (!sock) 622 return ERR_PTR(-ENOTSOCK); 623 624 if (sock->sk->sk_family != PF_XDP) { 625 sockfd_put(sock); 626 return ERR_PTR(-ENOPROTOOPT); 627 } 628 629 return sock; 630 } 631 632 /* Check if umem pages are contiguous. 633 * If zero-copy mode, use the DMA address to do the page contiguity check 634 * For all other modes we use addr (kernel virtual address) 635 * Store the result in the low bits of addr. 636 */ 637 static void xsk_check_page_contiguity(struct xdp_umem *umem, u32 flags) 638 { 639 struct xdp_umem_page *pgs = umem->pages; 640 int i, is_contig; 641 642 for (i = 0; i < umem->npgs - 1; i++) { 643 is_contig = (flags & XDP_ZEROCOPY) ? 644 (pgs[i].dma + PAGE_SIZE == pgs[i + 1].dma) : 645 (pgs[i].addr + PAGE_SIZE == pgs[i + 1].addr); 646 pgs[i].addr += is_contig << XSK_NEXT_PG_CONTIG_SHIFT; 647 } 648 } 649 650 static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) 651 { 652 struct sockaddr_xdp *sxdp = (struct sockaddr_xdp *)addr; 653 struct sock *sk = sock->sk; 654 struct xdp_sock *xs = xdp_sk(sk); 655 struct net_device *dev; 656 u32 flags, qid; 657 int err = 0; 658 659 if (addr_len < sizeof(struct sockaddr_xdp)) 660 return -EINVAL; 661 if (sxdp->sxdp_family != AF_XDP) 662 return -EINVAL; 663 664 flags = sxdp->sxdp_flags; 665 if (flags & ~(XDP_SHARED_UMEM | XDP_COPY | XDP_ZEROCOPY | 666 XDP_USE_NEED_WAKEUP)) 667 return -EINVAL; 668 669 rtnl_lock(); 670 mutex_lock(&xs->mutex); 671 if (xs->state != XSK_READY) { 672 err = -EBUSY; 673 goto out_release; 674 } 675 676 dev = dev_get_by_index(sock_net(sk), sxdp->sxdp_ifindex); 677 if (!dev) { 678 err = -ENODEV; 679 goto out_release; 680 } 681 682 if (!xs->rx && !xs->tx) { 683 err = -EINVAL; 684 goto out_unlock; 685 } 686 687 qid = sxdp->sxdp_queue_id; 688 689 if (flags & XDP_SHARED_UMEM) { 690 struct xdp_sock *umem_xs; 691 struct socket *sock; 692 693 if ((flags & XDP_COPY) || (flags & XDP_ZEROCOPY) || 694 (flags & XDP_USE_NEED_WAKEUP)) { 695 /* Cannot specify flags for shared sockets. */ 696 err = -EINVAL; 697 goto out_unlock; 698 } 699 700 if (xs->umem) { 701 /* We have already our own. */ 702 err = -EINVAL; 703 goto out_unlock; 704 } 705 706 sock = xsk_lookup_xsk_from_fd(sxdp->sxdp_shared_umem_fd); 707 if (IS_ERR(sock)) { 708 err = PTR_ERR(sock); 709 goto out_unlock; 710 } 711 712 umem_xs = xdp_sk(sock->sk); 713 if (!xsk_is_bound(umem_xs)) { 714 err = -EBADF; 715 sockfd_put(sock); 716 goto out_unlock; 717 } 718 if (umem_xs->dev != dev || umem_xs->queue_id != qid) { 719 err = -EINVAL; 720 sockfd_put(sock); 721 goto out_unlock; 722 } 723 724 xdp_get_umem(umem_xs->umem); 725 WRITE_ONCE(xs->umem, umem_xs->umem); 726 sockfd_put(sock); 727 } else if (!xs->umem || !xdp_umem_validate_queues(xs->umem)) { 728 err = -EINVAL; 729 goto out_unlock; 730 } else { 731 /* This xsk has its own umem. */ 732 xskq_set_umem(xs->umem->fq, xs->umem->size, 733 xs->umem->chunk_mask); 734 xskq_set_umem(xs->umem->cq, xs->umem->size, 735 xs->umem->chunk_mask); 736 737 err = xdp_umem_assign_dev(xs->umem, dev, qid, flags); 738 if (err) 739 goto out_unlock; 740 741 xsk_check_page_contiguity(xs->umem, flags); 742 } 743 744 xs->dev = dev; 745 xs->zc = xs->umem->zc; 746 xs->queue_id = qid; 747 xskq_set_umem(xs->rx, xs->umem->size, xs->umem->chunk_mask); 748 xskq_set_umem(xs->tx, xs->umem->size, xs->umem->chunk_mask); 749 xdp_add_sk_umem(xs->umem, xs); 750 751 out_unlock: 752 if (err) { 753 dev_put(dev); 754 } else { 755 /* Matches smp_rmb() in bind() for shared umem 756 * sockets, and xsk_is_bound(). 757 */ 758 smp_wmb(); 759 WRITE_ONCE(xs->state, XSK_BOUND); 760 } 761 out_release: 762 mutex_unlock(&xs->mutex); 763 rtnl_unlock(); 764 return err; 765 } 766 767 struct xdp_umem_reg_v1 { 768 __u64 addr; /* Start of packet data area */ 769 __u64 len; /* Length of packet data area */ 770 __u32 chunk_size; 771 __u32 headroom; 772 }; 773 774 static int xsk_setsockopt(struct socket *sock, int level, int optname, 775 char __user *optval, unsigned int optlen) 776 { 777 struct sock *sk = sock->sk; 778 struct xdp_sock *xs = xdp_sk(sk); 779 int err; 780 781 if (level != SOL_XDP) 782 return -ENOPROTOOPT; 783 784 switch (optname) { 785 case XDP_RX_RING: 786 case XDP_TX_RING: 787 { 788 struct xsk_queue **q; 789 int entries; 790 791 if (optlen < sizeof(entries)) 792 return -EINVAL; 793 if (copy_from_user(&entries, optval, sizeof(entries))) 794 return -EFAULT; 795 796 mutex_lock(&xs->mutex); 797 if (xs->state != XSK_READY) { 798 mutex_unlock(&xs->mutex); 799 return -EBUSY; 800 } 801 q = (optname == XDP_TX_RING) ? &xs->tx : &xs->rx; 802 err = xsk_init_queue(entries, q, false); 803 if (!err && optname == XDP_TX_RING) 804 /* Tx needs to be explicitly woken up the first time */ 805 xs->tx->ring->flags |= XDP_RING_NEED_WAKEUP; 806 mutex_unlock(&xs->mutex); 807 return err; 808 } 809 case XDP_UMEM_REG: 810 { 811 size_t mr_size = sizeof(struct xdp_umem_reg); 812 struct xdp_umem_reg mr = {}; 813 struct xdp_umem *umem; 814 815 if (optlen < sizeof(struct xdp_umem_reg_v1)) 816 return -EINVAL; 817 else if (optlen < sizeof(mr)) 818 mr_size = sizeof(struct xdp_umem_reg_v1); 819 820 if (copy_from_user(&mr, optval, mr_size)) 821 return -EFAULT; 822 823 mutex_lock(&xs->mutex); 824 if (xs->state != XSK_READY || xs->umem) { 825 mutex_unlock(&xs->mutex); 826 return -EBUSY; 827 } 828 829 umem = xdp_umem_create(&mr); 830 if (IS_ERR(umem)) { 831 mutex_unlock(&xs->mutex); 832 return PTR_ERR(umem); 833 } 834 835 /* Make sure umem is ready before it can be seen by others */ 836 smp_wmb(); 837 WRITE_ONCE(xs->umem, umem); 838 mutex_unlock(&xs->mutex); 839 return 0; 840 } 841 case XDP_UMEM_FILL_RING: 842 case XDP_UMEM_COMPLETION_RING: 843 { 844 struct xsk_queue **q; 845 int entries; 846 847 if (copy_from_user(&entries, optval, sizeof(entries))) 848 return -EFAULT; 849 850 mutex_lock(&xs->mutex); 851 if (xs->state != XSK_READY) { 852 mutex_unlock(&xs->mutex); 853 return -EBUSY; 854 } 855 if (!xs->umem) { 856 mutex_unlock(&xs->mutex); 857 return -EINVAL; 858 } 859 860 q = (optname == XDP_UMEM_FILL_RING) ? &xs->umem->fq : 861 &xs->umem->cq; 862 err = xsk_init_queue(entries, q, true); 863 mutex_unlock(&xs->mutex); 864 return err; 865 } 866 default: 867 break; 868 } 869 870 return -ENOPROTOOPT; 871 } 872 873 static void xsk_enter_rxtx_offsets(struct xdp_ring_offset_v1 *ring) 874 { 875 ring->producer = offsetof(struct xdp_rxtx_ring, ptrs.producer); 876 ring->consumer = offsetof(struct xdp_rxtx_ring, ptrs.consumer); 877 ring->desc = offsetof(struct xdp_rxtx_ring, desc); 878 } 879 880 static void xsk_enter_umem_offsets(struct xdp_ring_offset_v1 *ring) 881 { 882 ring->producer = offsetof(struct xdp_umem_ring, ptrs.producer); 883 ring->consumer = offsetof(struct xdp_umem_ring, ptrs.consumer); 884 ring->desc = offsetof(struct xdp_umem_ring, desc); 885 } 886 887 static int xsk_getsockopt(struct socket *sock, int level, int optname, 888 char __user *optval, int __user *optlen) 889 { 890 struct sock *sk = sock->sk; 891 struct xdp_sock *xs = xdp_sk(sk); 892 int len; 893 894 if (level != SOL_XDP) 895 return -ENOPROTOOPT; 896 897 if (get_user(len, optlen)) 898 return -EFAULT; 899 if (len < 0) 900 return -EINVAL; 901 902 switch (optname) { 903 case XDP_STATISTICS: 904 { 905 struct xdp_statistics stats; 906 907 if (len < sizeof(stats)) 908 return -EINVAL; 909 910 mutex_lock(&xs->mutex); 911 stats.rx_dropped = xs->rx_dropped; 912 stats.rx_invalid_descs = xskq_nb_invalid_descs(xs->rx); 913 stats.tx_invalid_descs = xskq_nb_invalid_descs(xs->tx); 914 mutex_unlock(&xs->mutex); 915 916 if (copy_to_user(optval, &stats, sizeof(stats))) 917 return -EFAULT; 918 if (put_user(sizeof(stats), optlen)) 919 return -EFAULT; 920 921 return 0; 922 } 923 case XDP_MMAP_OFFSETS: 924 { 925 struct xdp_mmap_offsets off; 926 struct xdp_mmap_offsets_v1 off_v1; 927 bool flags_supported = true; 928 void *to_copy; 929 930 if (len < sizeof(off_v1)) 931 return -EINVAL; 932 else if (len < sizeof(off)) 933 flags_supported = false; 934 935 if (flags_supported) { 936 /* xdp_ring_offset is identical to xdp_ring_offset_v1 937 * except for the flags field added to the end. 938 */ 939 xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *) 940 &off.rx); 941 xsk_enter_rxtx_offsets((struct xdp_ring_offset_v1 *) 942 &off.tx); 943 xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *) 944 &off.fr); 945 xsk_enter_umem_offsets((struct xdp_ring_offset_v1 *) 946 &off.cr); 947 off.rx.flags = offsetof(struct xdp_rxtx_ring, 948 ptrs.flags); 949 off.tx.flags = offsetof(struct xdp_rxtx_ring, 950 ptrs.flags); 951 off.fr.flags = offsetof(struct xdp_umem_ring, 952 ptrs.flags); 953 off.cr.flags = offsetof(struct xdp_umem_ring, 954 ptrs.flags); 955 956 len = sizeof(off); 957 to_copy = &off; 958 } else { 959 xsk_enter_rxtx_offsets(&off_v1.rx); 960 xsk_enter_rxtx_offsets(&off_v1.tx); 961 xsk_enter_umem_offsets(&off_v1.fr); 962 xsk_enter_umem_offsets(&off_v1.cr); 963 964 len = sizeof(off_v1); 965 to_copy = &off_v1; 966 } 967 968 if (copy_to_user(optval, to_copy, len)) 969 return -EFAULT; 970 if (put_user(len, optlen)) 971 return -EFAULT; 972 973 return 0; 974 } 975 case XDP_OPTIONS: 976 { 977 struct xdp_options opts = {}; 978 979 if (len < sizeof(opts)) 980 return -EINVAL; 981 982 mutex_lock(&xs->mutex); 983 if (xs->zc) 984 opts.flags |= XDP_OPTIONS_ZEROCOPY; 985 mutex_unlock(&xs->mutex); 986 987 len = sizeof(opts); 988 if (copy_to_user(optval, &opts, len)) 989 return -EFAULT; 990 if (put_user(len, optlen)) 991 return -EFAULT; 992 993 return 0; 994 } 995 default: 996 break; 997 } 998 999 return -EOPNOTSUPP; 1000 } 1001 1002 static int xsk_mmap(struct file *file, struct socket *sock, 1003 struct vm_area_struct *vma) 1004 { 1005 loff_t offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; 1006 unsigned long size = vma->vm_end - vma->vm_start; 1007 struct xdp_sock *xs = xdp_sk(sock->sk); 1008 struct xsk_queue *q = NULL; 1009 struct xdp_umem *umem; 1010 unsigned long pfn; 1011 struct page *qpg; 1012 1013 if (READ_ONCE(xs->state) != XSK_READY) 1014 return -EBUSY; 1015 1016 if (offset == XDP_PGOFF_RX_RING) { 1017 q = READ_ONCE(xs->rx); 1018 } else if (offset == XDP_PGOFF_TX_RING) { 1019 q = READ_ONCE(xs->tx); 1020 } else { 1021 umem = READ_ONCE(xs->umem); 1022 if (!umem) 1023 return -EINVAL; 1024 1025 /* Matches the smp_wmb() in XDP_UMEM_REG */ 1026 smp_rmb(); 1027 if (offset == XDP_UMEM_PGOFF_FILL_RING) 1028 q = READ_ONCE(umem->fq); 1029 else if (offset == XDP_UMEM_PGOFF_COMPLETION_RING) 1030 q = READ_ONCE(umem->cq); 1031 } 1032 1033 if (!q) 1034 return -EINVAL; 1035 1036 /* Matches the smp_wmb() in xsk_init_queue */ 1037 smp_rmb(); 1038 qpg = virt_to_head_page(q->ring); 1039 if (size > page_size(qpg)) 1040 return -EINVAL; 1041 1042 pfn = virt_to_phys(q->ring) >> PAGE_SHIFT; 1043 return remap_pfn_range(vma, vma->vm_start, pfn, 1044 size, vma->vm_page_prot); 1045 } 1046 1047 static int xsk_notifier(struct notifier_block *this, 1048 unsigned long msg, void *ptr) 1049 { 1050 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 1051 struct net *net = dev_net(dev); 1052 struct sock *sk; 1053 1054 switch (msg) { 1055 case NETDEV_UNREGISTER: 1056 mutex_lock(&net->xdp.lock); 1057 sk_for_each(sk, &net->xdp.list) { 1058 struct xdp_sock *xs = xdp_sk(sk); 1059 1060 mutex_lock(&xs->mutex); 1061 if (xs->dev == dev) { 1062 sk->sk_err = ENETDOWN; 1063 if (!sock_flag(sk, SOCK_DEAD)) 1064 sk->sk_error_report(sk); 1065 1066 xsk_unbind_dev(xs); 1067 1068 /* Clear device references in umem. */ 1069 xdp_umem_clear_dev(xs->umem); 1070 } 1071 mutex_unlock(&xs->mutex); 1072 } 1073 mutex_unlock(&net->xdp.lock); 1074 break; 1075 } 1076 return NOTIFY_DONE; 1077 } 1078 1079 static struct proto xsk_proto = { 1080 .name = "XDP", 1081 .owner = THIS_MODULE, 1082 .obj_size = sizeof(struct xdp_sock), 1083 }; 1084 1085 static const struct proto_ops xsk_proto_ops = { 1086 .family = PF_XDP, 1087 .owner = THIS_MODULE, 1088 .release = xsk_release, 1089 .bind = xsk_bind, 1090 .connect = sock_no_connect, 1091 .socketpair = sock_no_socketpair, 1092 .accept = sock_no_accept, 1093 .getname = sock_no_getname, 1094 .poll = xsk_poll, 1095 .ioctl = sock_no_ioctl, 1096 .listen = sock_no_listen, 1097 .shutdown = sock_no_shutdown, 1098 .setsockopt = xsk_setsockopt, 1099 .getsockopt = xsk_getsockopt, 1100 .sendmsg = xsk_sendmsg, 1101 .recvmsg = sock_no_recvmsg, 1102 .mmap = xsk_mmap, 1103 .sendpage = sock_no_sendpage, 1104 }; 1105 1106 static void xsk_destruct(struct sock *sk) 1107 { 1108 struct xdp_sock *xs = xdp_sk(sk); 1109 1110 if (!sock_flag(sk, SOCK_DEAD)) 1111 return; 1112 1113 xdp_put_umem(xs->umem); 1114 1115 sk_refcnt_debug_dec(sk); 1116 } 1117 1118 static int xsk_create(struct net *net, struct socket *sock, int protocol, 1119 int kern) 1120 { 1121 struct sock *sk; 1122 struct xdp_sock *xs; 1123 1124 if (!ns_capable(net->user_ns, CAP_NET_RAW)) 1125 return -EPERM; 1126 if (sock->type != SOCK_RAW) 1127 return -ESOCKTNOSUPPORT; 1128 1129 if (protocol) 1130 return -EPROTONOSUPPORT; 1131 1132 sock->state = SS_UNCONNECTED; 1133 1134 sk = sk_alloc(net, PF_XDP, GFP_KERNEL, &xsk_proto, kern); 1135 if (!sk) 1136 return -ENOBUFS; 1137 1138 sock->ops = &xsk_proto_ops; 1139 1140 sock_init_data(sock, sk); 1141 1142 sk->sk_family = PF_XDP; 1143 1144 sk->sk_destruct = xsk_destruct; 1145 sk_refcnt_debug_inc(sk); 1146 1147 sock_set_flag(sk, SOCK_RCU_FREE); 1148 1149 xs = xdp_sk(sk); 1150 xs->state = XSK_READY; 1151 mutex_init(&xs->mutex); 1152 spin_lock_init(&xs->rx_lock); 1153 spin_lock_init(&xs->tx_completion_lock); 1154 1155 INIT_LIST_HEAD(&xs->map_list); 1156 spin_lock_init(&xs->map_list_lock); 1157 1158 mutex_lock(&net->xdp.lock); 1159 sk_add_node_rcu(sk, &net->xdp.list); 1160 mutex_unlock(&net->xdp.lock); 1161 1162 local_bh_disable(); 1163 sock_prot_inuse_add(net, &xsk_proto, 1); 1164 local_bh_enable(); 1165 1166 return 0; 1167 } 1168 1169 static const struct net_proto_family xsk_family_ops = { 1170 .family = PF_XDP, 1171 .create = xsk_create, 1172 .owner = THIS_MODULE, 1173 }; 1174 1175 static struct notifier_block xsk_netdev_notifier = { 1176 .notifier_call = xsk_notifier, 1177 }; 1178 1179 static int __net_init xsk_net_init(struct net *net) 1180 { 1181 mutex_init(&net->xdp.lock); 1182 INIT_HLIST_HEAD(&net->xdp.list); 1183 return 0; 1184 } 1185 1186 static void __net_exit xsk_net_exit(struct net *net) 1187 { 1188 WARN_ON_ONCE(!hlist_empty(&net->xdp.list)); 1189 } 1190 1191 static struct pernet_operations xsk_net_ops = { 1192 .init = xsk_net_init, 1193 .exit = xsk_net_exit, 1194 }; 1195 1196 static int __init xsk_init(void) 1197 { 1198 int err, cpu; 1199 1200 err = proto_register(&xsk_proto, 0 /* no slab */); 1201 if (err) 1202 goto out; 1203 1204 err = sock_register(&xsk_family_ops); 1205 if (err) 1206 goto out_proto; 1207 1208 err = register_pernet_subsys(&xsk_net_ops); 1209 if (err) 1210 goto out_sk; 1211 1212 err = register_netdevice_notifier(&xsk_netdev_notifier); 1213 if (err) 1214 goto out_pernet; 1215 1216 for_each_possible_cpu(cpu) 1217 INIT_LIST_HEAD(&per_cpu(xskmap_flush_list, cpu)); 1218 return 0; 1219 1220 out_pernet: 1221 unregister_pernet_subsys(&xsk_net_ops); 1222 out_sk: 1223 sock_unregister(PF_XDP); 1224 out_proto: 1225 proto_unregister(&xsk_proto); 1226 out: 1227 return err; 1228 } 1229 1230 fs_initcall(xsk_init); 1231