1c0c77d8fSBjörn Töpel // SPDX-License-Identifier: GPL-2.0 2c0c77d8fSBjörn Töpel /* XDP user-space packet buffer 3c0c77d8fSBjörn Töpel * Copyright(c) 2018 Intel Corporation. 4c0c77d8fSBjörn Töpel */ 5c0c77d8fSBjörn Töpel 6c0c77d8fSBjörn Töpel #include <linux/init.h> 7c0c77d8fSBjörn Töpel #include <linux/sched/mm.h> 8c0c77d8fSBjörn Töpel #include <linux/sched/signal.h> 9c0c77d8fSBjörn Töpel #include <linux/sched/task.h> 10c0c77d8fSBjörn Töpel #include <linux/uaccess.h> 11c0c77d8fSBjörn Töpel #include <linux/slab.h> 12c0c77d8fSBjörn Töpel #include <linux/bpf.h> 13c0c77d8fSBjörn Töpel #include <linux/mm.h> 1484c6b868SJakub Kicinski #include <linux/netdevice.h> 1584c6b868SJakub Kicinski #include <linux/rtnetlink.h> 16c0c77d8fSBjörn Töpel 17c0c77d8fSBjörn Töpel #include "xdp_umem.h" 18e61e62b9SBjörn Töpel #include "xsk_queue.h" 19c0c77d8fSBjörn Töpel 20bbff2f32SBjörn Töpel #define XDP_UMEM_MIN_CHUNK_SIZE 2048 21c0c77d8fSBjörn Töpel 22ac98d8aaSMagnus Karlsson void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 23ac98d8aaSMagnus Karlsson { 24ac98d8aaSMagnus Karlsson unsigned long flags; 25ac98d8aaSMagnus Karlsson 26ac98d8aaSMagnus Karlsson spin_lock_irqsave(&umem->xsk_list_lock, flags); 27ac98d8aaSMagnus Karlsson list_add_rcu(&xs->list, &umem->xsk_list); 28ac98d8aaSMagnus Karlsson spin_unlock_irqrestore(&umem->xsk_list_lock, flags); 29ac98d8aaSMagnus Karlsson } 30ac98d8aaSMagnus Karlsson 31ac98d8aaSMagnus Karlsson void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 32ac98d8aaSMagnus Karlsson { 33ac98d8aaSMagnus Karlsson unsigned long flags; 34ac98d8aaSMagnus Karlsson 35ac98d8aaSMagnus Karlsson if (xs->dev) { 36ac98d8aaSMagnus Karlsson spin_lock_irqsave(&umem->xsk_list_lock, flags); 37ac98d8aaSMagnus Karlsson list_del_rcu(&xs->list); 38ac98d8aaSMagnus Karlsson spin_unlock_irqrestore(&umem->xsk_list_lock, flags); 39ac98d8aaSMagnus Karlsson 40ac98d8aaSMagnus Karlsson if (umem->zc) 41ac98d8aaSMagnus Karlsson synchronize_net(); 42ac98d8aaSMagnus Karlsson } 43ac98d8aaSMagnus Karlsson } 44ac98d8aaSMagnus Karlsson 45c9b47cc1SMagnus Karlsson /* The umem is stored both in the _rx struct and the _tx struct as we do 46c9b47cc1SMagnus Karlsson * not know if the device has more tx queues than rx, or the opposite. 47c9b47cc1SMagnus Karlsson * This might also change during run time. 48c9b47cc1SMagnus Karlsson */ 49c9b47cc1SMagnus Karlsson static void xdp_reg_umem_at_qid(struct net_device *dev, struct xdp_umem *umem, 50c9b47cc1SMagnus Karlsson u16 queue_id) 5184c6b868SJakub Kicinski { 52c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 53c9b47cc1SMagnus Karlsson dev->_rx[queue_id].umem = umem; 54c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 55c9b47cc1SMagnus Karlsson dev->_tx[queue_id].umem = umem; 56c9b47cc1SMagnus Karlsson } 5784c6b868SJakub Kicinski 581661d346SJakub Kicinski struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, 59c9b47cc1SMagnus Karlsson u16 queue_id) 60c9b47cc1SMagnus Karlsson { 61c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 62c9b47cc1SMagnus Karlsson return dev->_rx[queue_id].umem; 63c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 64c9b47cc1SMagnus Karlsson return dev->_tx[queue_id].umem; 6584c6b868SJakub Kicinski 66c9b47cc1SMagnus Karlsson return NULL; 67c9b47cc1SMagnus Karlsson } 6884c6b868SJakub Kicinski 69c9b47cc1SMagnus Karlsson static void xdp_clear_umem_at_qid(struct net_device *dev, u16 queue_id) 70c9b47cc1SMagnus Karlsson { 71*a41b4f3cSMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 72c9b47cc1SMagnus Karlsson dev->_rx[queue_id].umem = NULL; 73*a41b4f3cSMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 74c9b47cc1SMagnus Karlsson dev->_tx[queue_id].umem = NULL; 7584c6b868SJakub Kicinski } 7684c6b868SJakub Kicinski 77173d3adbSBjörn Töpel int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, 78c9b47cc1SMagnus Karlsson u16 queue_id, u16 flags) 79173d3adbSBjörn Töpel { 80173d3adbSBjörn Töpel bool force_zc, force_copy; 81173d3adbSBjörn Töpel struct netdev_bpf bpf; 82c9b47cc1SMagnus Karlsson int err = 0; 83173d3adbSBjörn Töpel 84173d3adbSBjörn Töpel force_zc = flags & XDP_ZEROCOPY; 85173d3adbSBjörn Töpel force_copy = flags & XDP_COPY; 86173d3adbSBjörn Töpel 87173d3adbSBjörn Töpel if (force_zc && force_copy) 88173d3adbSBjörn Töpel return -EINVAL; 89173d3adbSBjörn Töpel 90173d3adbSBjörn Töpel rtnl_lock(); 91c9b47cc1SMagnus Karlsson if (xdp_get_umem_from_qid(dev, queue_id)) { 92c9b47cc1SMagnus Karlsson err = -EBUSY; 93c9b47cc1SMagnus Karlsson goto out_rtnl_unlock; 94c9b47cc1SMagnus Karlsson } 95c9b47cc1SMagnus Karlsson 96c9b47cc1SMagnus Karlsson xdp_reg_umem_at_qid(dev, umem, queue_id); 97c9b47cc1SMagnus Karlsson umem->dev = dev; 98c9b47cc1SMagnus Karlsson umem->queue_id = queue_id; 99c9b47cc1SMagnus Karlsson if (force_copy) 100c9b47cc1SMagnus Karlsson /* For copy-mode, we are done. */ 101c9b47cc1SMagnus Karlsson goto out_rtnl_unlock; 102c9b47cc1SMagnus Karlsson 103c9b47cc1SMagnus Karlsson if (!dev->netdev_ops->ndo_bpf || 104c9b47cc1SMagnus Karlsson !dev->netdev_ops->ndo_xsk_async_xmit) { 105c9b47cc1SMagnus Karlsson err = -EOPNOTSUPP; 106c9b47cc1SMagnus Karlsson goto err_unreg_umem; 10784c6b868SJakub Kicinski } 108173d3adbSBjörn Töpel 109173d3adbSBjörn Töpel bpf.command = XDP_SETUP_XSK_UMEM; 110173d3adbSBjörn Töpel bpf.xsk.umem = umem; 111173d3adbSBjörn Töpel bpf.xsk.queue_id = queue_id; 112173d3adbSBjörn Töpel 113173d3adbSBjörn Töpel err = dev->netdev_ops->ndo_bpf(dev, &bpf); 114f734607eSJakub Kicinski if (err) 115c9b47cc1SMagnus Karlsson goto err_unreg_umem; 11684c6b868SJakub Kicinski rtnl_unlock(); 117173d3adbSBjörn Töpel 118f734607eSJakub Kicinski dev_hold(dev); 119173d3adbSBjörn Töpel umem->zc = true; 120173d3adbSBjörn Töpel return 0; 12184c6b868SJakub Kicinski 122c9b47cc1SMagnus Karlsson err_unreg_umem: 123c9b47cc1SMagnus Karlsson xdp_clear_umem_at_qid(dev, queue_id); 124c9b47cc1SMagnus Karlsson if (!force_zc) 125c9b47cc1SMagnus Karlsson err = 0; /* fallback to copy mode */ 126c9b47cc1SMagnus Karlsson out_rtnl_unlock: 12784c6b868SJakub Kicinski rtnl_unlock(); 128c9b47cc1SMagnus Karlsson return err; 129173d3adbSBjörn Töpel } 130173d3adbSBjörn Töpel 131ac98d8aaSMagnus Karlsson static void xdp_umem_clear_dev(struct xdp_umem *umem) 132173d3adbSBjörn Töpel { 133173d3adbSBjörn Töpel struct netdev_bpf bpf; 134173d3adbSBjörn Töpel int err; 135173d3adbSBjörn Töpel 136c9b47cc1SMagnus Karlsson if (umem->zc) { 137173d3adbSBjörn Töpel bpf.command = XDP_SETUP_XSK_UMEM; 138173d3adbSBjörn Töpel bpf.xsk.umem = NULL; 139173d3adbSBjörn Töpel bpf.xsk.queue_id = umem->queue_id; 140173d3adbSBjörn Töpel 141173d3adbSBjörn Töpel rtnl_lock(); 142173d3adbSBjörn Töpel err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf); 143173d3adbSBjörn Töpel rtnl_unlock(); 144173d3adbSBjörn Töpel 145173d3adbSBjörn Töpel if (err) 146173d3adbSBjörn Töpel WARN(1, "failed to disable umem!\n"); 147c9b47cc1SMagnus Karlsson } 148173d3adbSBjörn Töpel 149c9b47cc1SMagnus Karlsson if (umem->dev) { 150c9b47cc1SMagnus Karlsson rtnl_lock(); 151c9b47cc1SMagnus Karlsson xdp_clear_umem_at_qid(umem->dev, umem->queue_id); 152c9b47cc1SMagnus Karlsson rtnl_unlock(); 153c9b47cc1SMagnus Karlsson } 154c9b47cc1SMagnus Karlsson 155c9b47cc1SMagnus Karlsson if (umem->zc) { 156173d3adbSBjörn Töpel dev_put(umem->dev); 157c9b47cc1SMagnus Karlsson umem->zc = false; 158173d3adbSBjörn Töpel } 159173d3adbSBjörn Töpel } 160173d3adbSBjörn Töpel 161c0c77d8fSBjörn Töpel static void xdp_umem_unpin_pages(struct xdp_umem *umem) 162c0c77d8fSBjörn Töpel { 163c0c77d8fSBjörn Töpel unsigned int i; 164c0c77d8fSBjörn Töpel 165c0c77d8fSBjörn Töpel for (i = 0; i < umem->npgs; i++) { 166c0c77d8fSBjörn Töpel struct page *page = umem->pgs[i]; 167c0c77d8fSBjörn Töpel 168c0c77d8fSBjörn Töpel set_page_dirty_lock(page); 169c0c77d8fSBjörn Töpel put_page(page); 170c0c77d8fSBjörn Töpel } 171c0c77d8fSBjörn Töpel 172c0c77d8fSBjörn Töpel kfree(umem->pgs); 173c0c77d8fSBjörn Töpel umem->pgs = NULL; 174c0c77d8fSBjörn Töpel } 175c0c77d8fSBjörn Töpel 176c0c77d8fSBjörn Töpel static void xdp_umem_unaccount_pages(struct xdp_umem *umem) 177c0c77d8fSBjörn Töpel { 178c09290c5SDaniel Borkmann if (umem->user) { 179c0c77d8fSBjörn Töpel atomic_long_sub(umem->npgs, &umem->user->locked_vm); 180c0c77d8fSBjörn Töpel free_uid(umem->user); 181c0c77d8fSBjörn Töpel } 182c09290c5SDaniel Borkmann } 183c0c77d8fSBjörn Töpel 184c0c77d8fSBjörn Töpel static void xdp_umem_release(struct xdp_umem *umem) 185c0c77d8fSBjörn Töpel { 186c0c77d8fSBjörn Töpel struct task_struct *task; 187c0c77d8fSBjörn Töpel struct mm_struct *mm; 188c0c77d8fSBjörn Töpel 189173d3adbSBjörn Töpel xdp_umem_clear_dev(umem); 190173d3adbSBjörn Töpel 191423f3832SMagnus Karlsson if (umem->fq) { 192423f3832SMagnus Karlsson xskq_destroy(umem->fq); 193423f3832SMagnus Karlsson umem->fq = NULL; 194423f3832SMagnus Karlsson } 195423f3832SMagnus Karlsson 196fe230832SMagnus Karlsson if (umem->cq) { 197fe230832SMagnus Karlsson xskq_destroy(umem->cq); 198fe230832SMagnus Karlsson umem->cq = NULL; 199fe230832SMagnus Karlsson } 200fe230832SMagnus Karlsson 201f5bd9138SJakub Kicinski xsk_reuseq_destroy(umem); 202f5bd9138SJakub Kicinski 203c0c77d8fSBjörn Töpel xdp_umem_unpin_pages(umem); 204c0c77d8fSBjörn Töpel 205c0c77d8fSBjörn Töpel task = get_pid_task(umem->pid, PIDTYPE_PID); 206c0c77d8fSBjörn Töpel put_pid(umem->pid); 207c0c77d8fSBjörn Töpel if (!task) 208c0c77d8fSBjörn Töpel goto out; 209c0c77d8fSBjörn Töpel mm = get_task_mm(task); 210c0c77d8fSBjörn Töpel put_task_struct(task); 211c0c77d8fSBjörn Töpel if (!mm) 212c0c77d8fSBjörn Töpel goto out; 213c0c77d8fSBjörn Töpel 214c0c77d8fSBjörn Töpel mmput(mm); 2158aef7340SBjörn Töpel kfree(umem->pages); 2168aef7340SBjörn Töpel umem->pages = NULL; 2178aef7340SBjörn Töpel 218c0c77d8fSBjörn Töpel xdp_umem_unaccount_pages(umem); 219c0c77d8fSBjörn Töpel out: 220c0c77d8fSBjörn Töpel kfree(umem); 221c0c77d8fSBjörn Töpel } 222c0c77d8fSBjörn Töpel 223c0c77d8fSBjörn Töpel static void xdp_umem_release_deferred(struct work_struct *work) 224c0c77d8fSBjörn Töpel { 225c0c77d8fSBjörn Töpel struct xdp_umem *umem = container_of(work, struct xdp_umem, work); 226c0c77d8fSBjörn Töpel 227c0c77d8fSBjörn Töpel xdp_umem_release(umem); 228c0c77d8fSBjörn Töpel } 229c0c77d8fSBjörn Töpel 230c0c77d8fSBjörn Töpel void xdp_get_umem(struct xdp_umem *umem) 231c0c77d8fSBjörn Töpel { 232d3b42f14SBjörn Töpel refcount_inc(&umem->users); 233c0c77d8fSBjörn Töpel } 234c0c77d8fSBjörn Töpel 235c0c77d8fSBjörn Töpel void xdp_put_umem(struct xdp_umem *umem) 236c0c77d8fSBjörn Töpel { 237c0c77d8fSBjörn Töpel if (!umem) 238c0c77d8fSBjörn Töpel return; 239c0c77d8fSBjörn Töpel 240d3b42f14SBjörn Töpel if (refcount_dec_and_test(&umem->users)) { 241c0c77d8fSBjörn Töpel INIT_WORK(&umem->work, xdp_umem_release_deferred); 242c0c77d8fSBjörn Töpel schedule_work(&umem->work); 243c0c77d8fSBjörn Töpel } 244c0c77d8fSBjörn Töpel } 245c0c77d8fSBjörn Töpel 246c0c77d8fSBjörn Töpel static int xdp_umem_pin_pages(struct xdp_umem *umem) 247c0c77d8fSBjörn Töpel { 248c0c77d8fSBjörn Töpel unsigned int gup_flags = FOLL_WRITE; 249c0c77d8fSBjörn Töpel long npgs; 250c0c77d8fSBjörn Töpel int err; 251c0c77d8fSBjörn Töpel 252a343993cSBjörn Töpel umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs), 253a343993cSBjörn Töpel GFP_KERNEL | __GFP_NOWARN); 254c0c77d8fSBjörn Töpel if (!umem->pgs) 255c0c77d8fSBjörn Töpel return -ENOMEM; 256c0c77d8fSBjörn Töpel 257c0c77d8fSBjörn Töpel down_write(¤t->mm->mmap_sem); 258c0c77d8fSBjörn Töpel npgs = get_user_pages(umem->address, umem->npgs, 259c0c77d8fSBjörn Töpel gup_flags, &umem->pgs[0], NULL); 260c0c77d8fSBjörn Töpel up_write(¤t->mm->mmap_sem); 261c0c77d8fSBjörn Töpel 262c0c77d8fSBjörn Töpel if (npgs != umem->npgs) { 263c0c77d8fSBjörn Töpel if (npgs >= 0) { 264c0c77d8fSBjörn Töpel umem->npgs = npgs; 265c0c77d8fSBjörn Töpel err = -ENOMEM; 266c0c77d8fSBjörn Töpel goto out_pin; 267c0c77d8fSBjörn Töpel } 268c0c77d8fSBjörn Töpel err = npgs; 269c0c77d8fSBjörn Töpel goto out_pgs; 270c0c77d8fSBjörn Töpel } 271c0c77d8fSBjörn Töpel return 0; 272c0c77d8fSBjörn Töpel 273c0c77d8fSBjörn Töpel out_pin: 274c0c77d8fSBjörn Töpel xdp_umem_unpin_pages(umem); 275c0c77d8fSBjörn Töpel out_pgs: 276c0c77d8fSBjörn Töpel kfree(umem->pgs); 277c0c77d8fSBjörn Töpel umem->pgs = NULL; 278c0c77d8fSBjörn Töpel return err; 279c0c77d8fSBjörn Töpel } 280c0c77d8fSBjörn Töpel 281c0c77d8fSBjörn Töpel static int xdp_umem_account_pages(struct xdp_umem *umem) 282c0c77d8fSBjörn Töpel { 283c0c77d8fSBjörn Töpel unsigned long lock_limit, new_npgs, old_npgs; 284c0c77d8fSBjörn Töpel 285c0c77d8fSBjörn Töpel if (capable(CAP_IPC_LOCK)) 286c0c77d8fSBjörn Töpel return 0; 287c0c77d8fSBjörn Töpel 288c0c77d8fSBjörn Töpel lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 289c0c77d8fSBjörn Töpel umem->user = get_uid(current_user()); 290c0c77d8fSBjörn Töpel 291c0c77d8fSBjörn Töpel do { 292c0c77d8fSBjörn Töpel old_npgs = atomic_long_read(&umem->user->locked_vm); 293c0c77d8fSBjörn Töpel new_npgs = old_npgs + umem->npgs; 294c0c77d8fSBjörn Töpel if (new_npgs > lock_limit) { 295c0c77d8fSBjörn Töpel free_uid(umem->user); 296c0c77d8fSBjörn Töpel umem->user = NULL; 297c0c77d8fSBjörn Töpel return -ENOBUFS; 298c0c77d8fSBjörn Töpel } 299c0c77d8fSBjörn Töpel } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs, 300c0c77d8fSBjörn Töpel new_npgs) != old_npgs); 301c0c77d8fSBjörn Töpel return 0; 302c0c77d8fSBjörn Töpel } 303c0c77d8fSBjörn Töpel 304a49049eaSBjörn Töpel static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) 305c0c77d8fSBjörn Töpel { 306bbff2f32SBjörn Töpel u32 chunk_size = mr->chunk_size, headroom = mr->headroom; 307bbff2f32SBjörn Töpel unsigned int chunks, chunks_per_page; 308c0c77d8fSBjörn Töpel u64 addr = mr->addr, size = mr->len; 3098aef7340SBjörn Töpel int size_chk, err, i; 310c0c77d8fSBjörn Töpel 311bbff2f32SBjörn Töpel if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) { 312c0c77d8fSBjörn Töpel /* Strictly speaking we could support this, if: 313c0c77d8fSBjörn Töpel * - huge pages, or* 314c0c77d8fSBjörn Töpel * - using an IOMMU, or 315c0c77d8fSBjörn Töpel * - making sure the memory area is consecutive 316c0c77d8fSBjörn Töpel * but for now, we simply say "computer says no". 317c0c77d8fSBjörn Töpel */ 318c0c77d8fSBjörn Töpel return -EINVAL; 319c0c77d8fSBjörn Töpel } 320c0c77d8fSBjörn Töpel 321bbff2f32SBjörn Töpel if (!is_power_of_2(chunk_size)) 322c0c77d8fSBjörn Töpel return -EINVAL; 323c0c77d8fSBjörn Töpel 324c0c77d8fSBjörn Töpel if (!PAGE_ALIGNED(addr)) { 325c0c77d8fSBjörn Töpel /* Memory area has to be page size aligned. For 326c0c77d8fSBjörn Töpel * simplicity, this might change. 327c0c77d8fSBjörn Töpel */ 328c0c77d8fSBjörn Töpel return -EINVAL; 329c0c77d8fSBjörn Töpel } 330c0c77d8fSBjörn Töpel 331c0c77d8fSBjörn Töpel if ((addr + size) < addr) 332c0c77d8fSBjörn Töpel return -EINVAL; 333c0c77d8fSBjörn Töpel 334bbff2f32SBjörn Töpel chunks = (unsigned int)div_u64(size, chunk_size); 335bbff2f32SBjörn Töpel if (chunks == 0) 336c0c77d8fSBjörn Töpel return -EINVAL; 337c0c77d8fSBjörn Töpel 338bbff2f32SBjörn Töpel chunks_per_page = PAGE_SIZE / chunk_size; 339bbff2f32SBjörn Töpel if (chunks < chunks_per_page || chunks % chunks_per_page) 340c0c77d8fSBjörn Töpel return -EINVAL; 341c0c77d8fSBjörn Töpel 342bbff2f32SBjörn Töpel headroom = ALIGN(headroom, 64); 343c0c77d8fSBjörn Töpel 344bbff2f32SBjörn Töpel size_chk = chunk_size - headroom - XDP_PACKET_HEADROOM; 345c0c77d8fSBjörn Töpel if (size_chk < 0) 346c0c77d8fSBjörn Töpel return -EINVAL; 347c0c77d8fSBjörn Töpel 348c0c77d8fSBjörn Töpel umem->pid = get_task_pid(current, PIDTYPE_PID); 349c0c77d8fSBjörn Töpel umem->address = (unsigned long)addr; 35093ee30f3SMagnus Karlsson umem->chunk_mask = ~((u64)chunk_size - 1); 35193ee30f3SMagnus Karlsson umem->size = size; 352bbff2f32SBjörn Töpel umem->headroom = headroom; 353bbff2f32SBjörn Töpel umem->chunk_size_nohr = chunk_size - headroom; 354c0c77d8fSBjörn Töpel umem->npgs = size / PAGE_SIZE; 355c0c77d8fSBjörn Töpel umem->pgs = NULL; 356c0c77d8fSBjörn Töpel umem->user = NULL; 357ac98d8aaSMagnus Karlsson INIT_LIST_HEAD(&umem->xsk_list); 358ac98d8aaSMagnus Karlsson spin_lock_init(&umem->xsk_list_lock); 359c0c77d8fSBjörn Töpel 360d3b42f14SBjörn Töpel refcount_set(&umem->users, 1); 361c0c77d8fSBjörn Töpel 362c0c77d8fSBjörn Töpel err = xdp_umem_account_pages(umem); 363c0c77d8fSBjörn Töpel if (err) 364c0c77d8fSBjörn Töpel goto out; 365c0c77d8fSBjörn Töpel 366c0c77d8fSBjörn Töpel err = xdp_umem_pin_pages(umem); 367c0c77d8fSBjörn Töpel if (err) 368c0c77d8fSBjörn Töpel goto out_account; 3698aef7340SBjörn Töpel 3708aef7340SBjörn Töpel umem->pages = kcalloc(umem->npgs, sizeof(*umem->pages), GFP_KERNEL); 3718aef7340SBjörn Töpel if (!umem->pages) { 3728aef7340SBjörn Töpel err = -ENOMEM; 3738aef7340SBjörn Töpel goto out_account; 3748aef7340SBjörn Töpel } 3758aef7340SBjörn Töpel 3768aef7340SBjörn Töpel for (i = 0; i < umem->npgs; i++) 3778aef7340SBjörn Töpel umem->pages[i].addr = page_address(umem->pgs[i]); 3788aef7340SBjörn Töpel 379c0c77d8fSBjörn Töpel return 0; 380c0c77d8fSBjörn Töpel 381c0c77d8fSBjörn Töpel out_account: 382c0c77d8fSBjörn Töpel xdp_umem_unaccount_pages(umem); 383c0c77d8fSBjörn Töpel out: 384c0c77d8fSBjörn Töpel put_pid(umem->pid); 385c0c77d8fSBjörn Töpel return err; 386c0c77d8fSBjörn Töpel } 387965a9909SMagnus Karlsson 388a49049eaSBjörn Töpel struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr) 389a49049eaSBjörn Töpel { 390a49049eaSBjörn Töpel struct xdp_umem *umem; 391a49049eaSBjörn Töpel int err; 392a49049eaSBjörn Töpel 393a49049eaSBjörn Töpel umem = kzalloc(sizeof(*umem), GFP_KERNEL); 394a49049eaSBjörn Töpel if (!umem) 395a49049eaSBjörn Töpel return ERR_PTR(-ENOMEM); 396a49049eaSBjörn Töpel 397a49049eaSBjörn Töpel err = xdp_umem_reg(umem, mr); 398a49049eaSBjörn Töpel if (err) { 399a49049eaSBjörn Töpel kfree(umem); 400a49049eaSBjörn Töpel return ERR_PTR(err); 401a49049eaSBjörn Töpel } 402a49049eaSBjörn Töpel 403a49049eaSBjörn Töpel return umem; 404a49049eaSBjörn Töpel } 405a49049eaSBjörn Töpel 406965a9909SMagnus Karlsson bool xdp_umem_validate_queues(struct xdp_umem *umem) 407965a9909SMagnus Karlsson { 408da60cf00SBjörn Töpel return umem->fq && umem->cq; 409965a9909SMagnus Karlsson } 410