1 c0c77d8fSBjörn Töpel // SPDX-License-Identifier: GPL-2.0 2 c0c77d8fSBjörn Töpel /* XDP user-space packet buffer 3 c0c77d8fSBjörn Töpel * Copyright(c) 2018 Intel Corporation. 4 c0c77d8fSBjörn Töpel */ 5 c0c77d8fSBjörn Töpel 6 c0c77d8fSBjörn Töpel #include <linux/init.h> 7 c0c77d8fSBjörn Töpel #include <linux/sched/mm.h> 8 c0c77d8fSBjörn Töpel #include <linux/sched/signal.h> 9 c0c77d8fSBjörn Töpel #include <linux/sched/task.h> 10 c0c77d8fSBjörn Töpel #include <linux/uaccess.h> 11 c0c77d8fSBjörn Töpel #include <linux/slab.h> 12 c0c77d8fSBjörn Töpel #include <linux/bpf.h> 13 c0c77d8fSBjörn Töpel #include <linux/mm.h> 14 84c6b868SJakub Kicinski #include <linux/netdevice.h> 15 84c6b868SJakub Kicinski #include <linux/rtnetlink.h> 16 c0c77d8fSBjörn Töpel 17 c0c77d8fSBjörn Töpel #include "xdp_umem.h" 18 e61e62b9SBjörn Töpel #include "xsk_queue.h" 19 c0c77d8fSBjörn Töpel 20 bbff2f32SBjörn Töpel #define XDP_UMEM_MIN_CHUNK_SIZE 2048 21 c0c77d8fSBjörn Töpel 22 ac98d8aaSMagnus Karlsson void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 23 ac98d8aaSMagnus Karlsson { 24 ac98d8aaSMagnus Karlsson unsigned long flags; 25 ac98d8aaSMagnus Karlsson 26 ac98d8aaSMagnus Karlsson spin_lock_irqsave(&umem->xsk_list_lock, flags); 27 ac98d8aaSMagnus Karlsson list_add_rcu(&xs->list, &umem->xsk_list); 28 ac98d8aaSMagnus Karlsson spin_unlock_irqrestore(&umem->xsk_list_lock, flags); 29 ac98d8aaSMagnus Karlsson } 30 ac98d8aaSMagnus Karlsson 31 ac98d8aaSMagnus Karlsson void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs) 32 ac98d8aaSMagnus Karlsson { 33 ac98d8aaSMagnus Karlsson unsigned long flags; 34 ac98d8aaSMagnus Karlsson 35 ac98d8aaSMagnus Karlsson if (xs->dev) { 36 ac98d8aaSMagnus Karlsson spin_lock_irqsave(&umem->xsk_list_lock, flags); 37 ac98d8aaSMagnus Karlsson list_del_rcu(&xs->list); 38 ac98d8aaSMagnus Karlsson spin_unlock_irqrestore(&umem->xsk_list_lock, flags); 39 ac98d8aaSMagnus Karlsson 40 ac98d8aaSMagnus Karlsson if (umem->zc) 41 ac98d8aaSMagnus Karlsson synchronize_net(); 42 ac98d8aaSMagnus Karlsson } 43 ac98d8aaSMagnus Karlsson } 44 ac98d8aaSMagnus Karlsson 45 *c9b47cc1SMagnus Karlsson /* The umem is stored both in the _rx struct and the _tx struct as we do 46 *c9b47cc1SMagnus Karlsson * not know if the device has more tx queues than rx, or the opposite. 47 *c9b47cc1SMagnus Karlsson * This might also change during run time. 48 *c9b47cc1SMagnus Karlsson */ 49 *c9b47cc1SMagnus Karlsson static void xdp_reg_umem_at_qid(struct net_device *dev, struct xdp_umem *umem, 50 *c9b47cc1SMagnus Karlsson u16 queue_id) 51 84c6b868SJakub Kicinski { 52 *c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 53 *c9b47cc1SMagnus Karlsson dev->_rx[queue_id].umem = umem; 54 *c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 55 *c9b47cc1SMagnus Karlsson dev->_tx[queue_id].umem = umem; 56 *c9b47cc1SMagnus Karlsson } 57 84c6b868SJakub Kicinski 58 *c9b47cc1SMagnus Karlsson static struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, 59 *c9b47cc1SMagnus Karlsson u16 queue_id) 60 *c9b47cc1SMagnus Karlsson { 61 *c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_rx_queues) 62 *c9b47cc1SMagnus Karlsson return dev->_rx[queue_id].umem; 63 *c9b47cc1SMagnus Karlsson if (queue_id < dev->real_num_tx_queues) 64 *c9b47cc1SMagnus Karlsson return dev->_tx[queue_id].umem; 65 84c6b868SJakub Kicinski 66 *c9b47cc1SMagnus Karlsson return NULL; 67 *c9b47cc1SMagnus Karlsson } 68 84c6b868SJakub Kicinski 69 *c9b47cc1SMagnus Karlsson static void xdp_clear_umem_at_qid(struct net_device *dev, u16 queue_id) 70 *c9b47cc1SMagnus Karlsson { 71 *c9b47cc1SMagnus Karlsson /* Zero out the entry independent on how many queues are configured 72 *c9b47cc1SMagnus Karlsson * at this point in time, as it might be used in the future. 73 *c9b47cc1SMagnus Karlsson */ 74 *c9b47cc1SMagnus Karlsson if (queue_id < dev->num_rx_queues) 75 *c9b47cc1SMagnus Karlsson dev->_rx[queue_id].umem = NULL; 76 *c9b47cc1SMagnus Karlsson if (queue_id < dev->num_tx_queues) 77 *c9b47cc1SMagnus Karlsson dev->_tx[queue_id].umem = NULL; 78 84c6b868SJakub Kicinski } 79 84c6b868SJakub Kicinski 80 173d3adbSBjörn Töpel int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, 81 *c9b47cc1SMagnus Karlsson u16 queue_id, u16 flags) 82 173d3adbSBjörn Töpel { 83 173d3adbSBjörn Töpel bool force_zc, force_copy; 84 173d3adbSBjörn Töpel struct netdev_bpf bpf; 85 *c9b47cc1SMagnus Karlsson int err = 0; 86 173d3adbSBjörn Töpel 87 173d3adbSBjörn Töpel force_zc = flags & XDP_ZEROCOPY; 88 173d3adbSBjörn Töpel force_copy = flags & XDP_COPY; 89 173d3adbSBjörn Töpel 90 173d3adbSBjörn Töpel if (force_zc && force_copy) 91 173d3adbSBjörn Töpel return -EINVAL; 92 173d3adbSBjörn Töpel 93 173d3adbSBjörn Töpel rtnl_lock(); 94 *c9b47cc1SMagnus Karlsson if (xdp_get_umem_from_qid(dev, queue_id)) { 95 *c9b47cc1SMagnus Karlsson err = -EBUSY; 96 *c9b47cc1SMagnus Karlsson goto out_rtnl_unlock; 97 *c9b47cc1SMagnus Karlsson } 98 *c9b47cc1SMagnus Karlsson 99 *c9b47cc1SMagnus Karlsson xdp_reg_umem_at_qid(dev, umem, queue_id); 100 *c9b47cc1SMagnus Karlsson umem->dev = dev; 101 *c9b47cc1SMagnus Karlsson umem->queue_id = queue_id; 102 *c9b47cc1SMagnus Karlsson if (force_copy) 103 *c9b47cc1SMagnus Karlsson /* For copy-mode, we are done. */ 104 *c9b47cc1SMagnus Karlsson goto out_rtnl_unlock; 105 *c9b47cc1SMagnus Karlsson 106 *c9b47cc1SMagnus Karlsson if (!dev->netdev_ops->ndo_bpf || 107 *c9b47cc1SMagnus Karlsson !dev->netdev_ops->ndo_xsk_async_xmit) { 108 *c9b47cc1SMagnus Karlsson err = -EOPNOTSUPP; 109 *c9b47cc1SMagnus Karlsson goto err_unreg_umem; 110 84c6b868SJakub Kicinski } 111 173d3adbSBjörn Töpel 112 173d3adbSBjörn Töpel bpf.command = XDP_SETUP_XSK_UMEM; 113 173d3adbSBjörn Töpel bpf.xsk.umem = umem; 114 173d3adbSBjörn Töpel bpf.xsk.queue_id = queue_id; 115 173d3adbSBjörn Töpel 116 173d3adbSBjörn Töpel err = dev->netdev_ops->ndo_bpf(dev, &bpf); 117 f734607eSJakub Kicinski if (err) 118 *c9b47cc1SMagnus Karlsson goto err_unreg_umem; 119 84c6b868SJakub Kicinski rtnl_unlock(); 120 173d3adbSBjörn Töpel 121 f734607eSJakub Kicinski dev_hold(dev); 122 173d3adbSBjörn Töpel umem->zc = true; 123 173d3adbSBjörn Töpel return 0; 124 84c6b868SJakub Kicinski 125 *c9b47cc1SMagnus Karlsson err_unreg_umem: 126 *c9b47cc1SMagnus Karlsson xdp_clear_umem_at_qid(dev, queue_id); 127 *c9b47cc1SMagnus Karlsson if (!force_zc) 128 *c9b47cc1SMagnus Karlsson err = 0; /* fallback to copy mode */ 129 *c9b47cc1SMagnus Karlsson out_rtnl_unlock: 130 84c6b868SJakub Kicinski rtnl_unlock(); 131 *c9b47cc1SMagnus Karlsson return err; 132 173d3adbSBjörn Töpel } 133 173d3adbSBjörn Töpel 134 ac98d8aaSMagnus Karlsson static void xdp_umem_clear_dev(struct xdp_umem *umem) 135 173d3adbSBjörn Töpel { 136 173d3adbSBjörn Töpel struct netdev_bpf bpf; 137 173d3adbSBjörn Töpel int err; 138 173d3adbSBjörn Töpel 139 *c9b47cc1SMagnus Karlsson if (umem->zc) { 140 173d3adbSBjörn Töpel bpf.command = XDP_SETUP_XSK_UMEM; 141 173d3adbSBjörn Töpel bpf.xsk.umem = NULL; 142 173d3adbSBjörn Töpel bpf.xsk.queue_id = umem->queue_id; 143 173d3adbSBjörn Töpel 144 173d3adbSBjörn Töpel rtnl_lock(); 145 173d3adbSBjörn Töpel err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf); 146 173d3adbSBjörn Töpel rtnl_unlock(); 147 173d3adbSBjörn Töpel 148 173d3adbSBjörn Töpel if (err) 149 173d3adbSBjörn Töpel WARN(1, "failed to disable umem!\n"); 150 *c9b47cc1SMagnus Karlsson } 151 173d3adbSBjörn Töpel 152 *c9b47cc1SMagnus Karlsson if (umem->dev) { 153 *c9b47cc1SMagnus Karlsson rtnl_lock(); 154 *c9b47cc1SMagnus Karlsson xdp_clear_umem_at_qid(umem->dev, umem->queue_id); 155 *c9b47cc1SMagnus Karlsson rtnl_unlock(); 156 *c9b47cc1SMagnus Karlsson } 157 *c9b47cc1SMagnus Karlsson 158 *c9b47cc1SMagnus Karlsson if (umem->zc) { 159 173d3adbSBjörn Töpel dev_put(umem->dev); 160 *c9b47cc1SMagnus Karlsson umem->zc = false; 161 173d3adbSBjörn Töpel } 162 173d3adbSBjörn Töpel } 163 173d3adbSBjörn Töpel 164 c0c77d8fSBjörn Töpel static void xdp_umem_unpin_pages(struct xdp_umem *umem) 165 c0c77d8fSBjörn Töpel { 166 c0c77d8fSBjörn Töpel unsigned int i; 167 c0c77d8fSBjörn Töpel 168 c0c77d8fSBjörn Töpel for (i = 0; i < umem->npgs; i++) { 169 c0c77d8fSBjörn Töpel struct page *page = umem->pgs[i]; 170 c0c77d8fSBjörn Töpel 171 c0c77d8fSBjörn Töpel set_page_dirty_lock(page); 172 c0c77d8fSBjörn Töpel put_page(page); 173 c0c77d8fSBjörn Töpel } 174 c0c77d8fSBjörn Töpel 175 c0c77d8fSBjörn Töpel kfree(umem->pgs); 176 c0c77d8fSBjörn Töpel umem->pgs = NULL; 177 c0c77d8fSBjörn Töpel } 178 c0c77d8fSBjörn Töpel 179 c0c77d8fSBjörn Töpel static void xdp_umem_unaccount_pages(struct xdp_umem *umem) 180 c0c77d8fSBjörn Töpel { 181 c09290c5SDaniel Borkmann if (umem->user) { 182 c0c77d8fSBjörn Töpel atomic_long_sub(umem->npgs, &umem->user->locked_vm); 183 c0c77d8fSBjörn Töpel free_uid(umem->user); 184 c0c77d8fSBjörn Töpel } 185 c09290c5SDaniel Borkmann } 186 c0c77d8fSBjörn Töpel 187 c0c77d8fSBjörn Töpel static void xdp_umem_release(struct xdp_umem *umem) 188 c0c77d8fSBjörn Töpel { 189 c0c77d8fSBjörn Töpel struct task_struct *task; 190 c0c77d8fSBjörn Töpel struct mm_struct *mm; 191 c0c77d8fSBjörn Töpel 192 173d3adbSBjörn Töpel xdp_umem_clear_dev(umem); 193 173d3adbSBjörn Töpel 194 423f3832SMagnus Karlsson if (umem->fq) { 195 423f3832SMagnus Karlsson xskq_destroy(umem->fq); 196 423f3832SMagnus Karlsson umem->fq = NULL; 197 423f3832SMagnus Karlsson } 198 423f3832SMagnus Karlsson 199 fe230832SMagnus Karlsson if (umem->cq) { 200 fe230832SMagnus Karlsson xskq_destroy(umem->cq); 201 fe230832SMagnus Karlsson umem->cq = NULL; 202 fe230832SMagnus Karlsson } 203 fe230832SMagnus Karlsson 204 f5bd9138SJakub Kicinski xsk_reuseq_destroy(umem); 205 f5bd9138SJakub Kicinski 206 c0c77d8fSBjörn Töpel xdp_umem_unpin_pages(umem); 207 c0c77d8fSBjörn Töpel 208 c0c77d8fSBjörn Töpel task = get_pid_task(umem->pid, PIDTYPE_PID); 209 c0c77d8fSBjörn Töpel put_pid(umem->pid); 210 c0c77d8fSBjörn Töpel if (!task) 211 c0c77d8fSBjörn Töpel goto out; 212 c0c77d8fSBjörn Töpel mm = get_task_mm(task); 213 c0c77d8fSBjörn Töpel put_task_struct(task); 214 c0c77d8fSBjörn Töpel if (!mm) 215 c0c77d8fSBjörn Töpel goto out; 216 c0c77d8fSBjörn Töpel 217 c0c77d8fSBjörn Töpel mmput(mm); 218 8aef7340SBjörn Töpel kfree(umem->pages); 219 8aef7340SBjörn Töpel umem->pages = NULL; 220 8aef7340SBjörn Töpel 221 c0c77d8fSBjörn Töpel xdp_umem_unaccount_pages(umem); 222 c0c77d8fSBjörn Töpel out: 223 c0c77d8fSBjörn Töpel kfree(umem); 224 c0c77d8fSBjörn Töpel } 225 c0c77d8fSBjörn Töpel 226 c0c77d8fSBjörn Töpel static void xdp_umem_release_deferred(struct work_struct *work) 227 c0c77d8fSBjörn Töpel { 228 c0c77d8fSBjörn Töpel struct xdp_umem *umem = container_of(work, struct xdp_umem, work); 229 c0c77d8fSBjörn Töpel 230 c0c77d8fSBjörn Töpel xdp_umem_release(umem); 231 c0c77d8fSBjörn Töpel } 232 c0c77d8fSBjörn Töpel 233 c0c77d8fSBjörn Töpel void xdp_get_umem(struct xdp_umem *umem) 234 c0c77d8fSBjörn Töpel { 235 d3b42f14SBjörn Töpel refcount_inc(&umem->users); 236 c0c77d8fSBjörn Töpel } 237 c0c77d8fSBjörn Töpel 238 c0c77d8fSBjörn Töpel void xdp_put_umem(struct xdp_umem *umem) 239 c0c77d8fSBjörn Töpel { 240 c0c77d8fSBjörn Töpel if (!umem) 241 c0c77d8fSBjörn Töpel return; 242 c0c77d8fSBjörn Töpel 243 d3b42f14SBjörn Töpel if (refcount_dec_and_test(&umem->users)) { 244 c0c77d8fSBjörn Töpel INIT_WORK(&umem->work, xdp_umem_release_deferred); 245 c0c77d8fSBjörn Töpel schedule_work(&umem->work); 246 c0c77d8fSBjörn Töpel } 247 c0c77d8fSBjörn Töpel } 248 c0c77d8fSBjörn Töpel 249 c0c77d8fSBjörn Töpel static int xdp_umem_pin_pages(struct xdp_umem *umem) 250 c0c77d8fSBjörn Töpel { 251 c0c77d8fSBjörn Töpel unsigned int gup_flags = FOLL_WRITE; 252 c0c77d8fSBjörn Töpel long npgs; 253 c0c77d8fSBjörn Töpel int err; 254 c0c77d8fSBjörn Töpel 255 a343993cSBjörn Töpel umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs), 256 a343993cSBjörn Töpel GFP_KERNEL | __GFP_NOWARN); 257 c0c77d8fSBjörn Töpel if (!umem->pgs) 258 c0c77d8fSBjörn Töpel return -ENOMEM; 259 c0c77d8fSBjörn Töpel 260 c0c77d8fSBjörn Töpel down_write(¤t->mm->mmap_sem); 261 c0c77d8fSBjörn Töpel npgs = get_user_pages(umem->address, umem->npgs, 262 c0c77d8fSBjörn Töpel gup_flags, &umem->pgs[0], NULL); 263 c0c77d8fSBjörn Töpel up_write(¤t->mm->mmap_sem); 264 c0c77d8fSBjörn Töpel 265 c0c77d8fSBjörn Töpel if (npgs != umem->npgs) { 266 c0c77d8fSBjörn Töpel if (npgs >= 0) { 267 c0c77d8fSBjörn Töpel umem->npgs = npgs; 268 c0c77d8fSBjörn Töpel err = -ENOMEM; 269 c0c77d8fSBjörn Töpel goto out_pin; 270 c0c77d8fSBjörn Töpel } 271 c0c77d8fSBjörn Töpel err = npgs; 272 c0c77d8fSBjörn Töpel goto out_pgs; 273 c0c77d8fSBjörn Töpel } 274 c0c77d8fSBjörn Töpel return 0; 275 c0c77d8fSBjörn Töpel 276 c0c77d8fSBjörn Töpel out_pin: 277 c0c77d8fSBjörn Töpel xdp_umem_unpin_pages(umem); 278 c0c77d8fSBjörn Töpel out_pgs: 279 c0c77d8fSBjörn Töpel kfree(umem->pgs); 280 c0c77d8fSBjörn Töpel umem->pgs = NULL; 281 c0c77d8fSBjörn Töpel return err; 282 c0c77d8fSBjörn Töpel } 283 c0c77d8fSBjörn Töpel 284 c0c77d8fSBjörn Töpel static int xdp_umem_account_pages(struct xdp_umem *umem) 285 c0c77d8fSBjörn Töpel { 286 c0c77d8fSBjörn Töpel unsigned long lock_limit, new_npgs, old_npgs; 287 c0c77d8fSBjörn Töpel 288 c0c77d8fSBjörn Töpel if (capable(CAP_IPC_LOCK)) 289 c0c77d8fSBjörn Töpel return 0; 290 c0c77d8fSBjörn Töpel 291 c0c77d8fSBjörn Töpel lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 292 c0c77d8fSBjörn Töpel umem->user = get_uid(current_user()); 293 c0c77d8fSBjörn Töpel 294 c0c77d8fSBjörn Töpel do { 295 c0c77d8fSBjörn Töpel old_npgs = atomic_long_read(&umem->user->locked_vm); 296 c0c77d8fSBjörn Töpel new_npgs = old_npgs + umem->npgs; 297 c0c77d8fSBjörn Töpel if (new_npgs > lock_limit) { 298 c0c77d8fSBjörn Töpel free_uid(umem->user); 299 c0c77d8fSBjörn Töpel umem->user = NULL; 300 c0c77d8fSBjörn Töpel return -ENOBUFS; 301 c0c77d8fSBjörn Töpel } 302 c0c77d8fSBjörn Töpel } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs, 303 c0c77d8fSBjörn Töpel new_npgs) != old_npgs); 304 c0c77d8fSBjörn Töpel return 0; 305 c0c77d8fSBjörn Töpel } 306 c0c77d8fSBjörn Töpel 307 a49049eaSBjörn Töpel static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) 308 c0c77d8fSBjörn Töpel { 309 bbff2f32SBjörn Töpel u32 chunk_size = mr->chunk_size, headroom = mr->headroom; 310 bbff2f32SBjörn Töpel unsigned int chunks, chunks_per_page; 311 c0c77d8fSBjörn Töpel u64 addr = mr->addr, size = mr->len; 312 8aef7340SBjörn Töpel int size_chk, err, i; 313 c0c77d8fSBjörn Töpel 314 bbff2f32SBjörn Töpel if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) { 315 c0c77d8fSBjörn Töpel /* Strictly speaking we could support this, if: 316 c0c77d8fSBjörn Töpel * - huge pages, or* 317 c0c77d8fSBjörn Töpel * - using an IOMMU, or 318 c0c77d8fSBjörn Töpel * - making sure the memory area is consecutive 319 c0c77d8fSBjörn Töpel * but for now, we simply say "computer says no". 320 c0c77d8fSBjörn Töpel */ 321 c0c77d8fSBjörn Töpel return -EINVAL; 322 c0c77d8fSBjörn Töpel } 323 c0c77d8fSBjörn Töpel 324 bbff2f32SBjörn Töpel if (!is_power_of_2(chunk_size)) 325 c0c77d8fSBjörn Töpel return -EINVAL; 326 c0c77d8fSBjörn Töpel 327 c0c77d8fSBjörn Töpel if (!PAGE_ALIGNED(addr)) { 328 c0c77d8fSBjörn Töpel /* Memory area has to be page size aligned. For 329 c0c77d8fSBjörn Töpel * simplicity, this might change. 330 c0c77d8fSBjörn Töpel */ 331 c0c77d8fSBjörn Töpel return -EINVAL; 332 c0c77d8fSBjörn Töpel } 333 c0c77d8fSBjörn Töpel 334 c0c77d8fSBjörn Töpel if ((addr + size) < addr) 335 c0c77d8fSBjörn Töpel return -EINVAL; 336 c0c77d8fSBjörn Töpel 337 bbff2f32SBjörn Töpel chunks = (unsigned int)div_u64(size, chunk_size); 338 bbff2f32SBjörn Töpel if (chunks == 0) 339 c0c77d8fSBjörn Töpel return -EINVAL; 340 c0c77d8fSBjörn Töpel 341 bbff2f32SBjörn Töpel chunks_per_page = PAGE_SIZE / chunk_size; 342 bbff2f32SBjörn Töpel if (chunks < chunks_per_page || chunks % chunks_per_page) 343 c0c77d8fSBjörn Töpel return -EINVAL; 344 c0c77d8fSBjörn Töpel 345 bbff2f32SBjörn Töpel headroom = ALIGN(headroom, 64); 346 c0c77d8fSBjörn Töpel 347 bbff2f32SBjörn Töpel size_chk = chunk_size - headroom - XDP_PACKET_HEADROOM; 348 c0c77d8fSBjörn Töpel if (size_chk < 0) 349 c0c77d8fSBjörn Töpel return -EINVAL; 350 c0c77d8fSBjörn Töpel 351 c0c77d8fSBjörn Töpel umem->pid = get_task_pid(current, PIDTYPE_PID); 352 c0c77d8fSBjörn Töpel umem->address = (unsigned long)addr; 353 93ee30f3SMagnus Karlsson umem->chunk_mask = ~((u64)chunk_size - 1); 354 93ee30f3SMagnus Karlsson umem->size = size; 355 bbff2f32SBjörn Töpel umem->headroom = headroom; 356 bbff2f32SBjörn Töpel umem->chunk_size_nohr = chunk_size - headroom; 357 c0c77d8fSBjörn Töpel umem->npgs = size / PAGE_SIZE; 358 c0c77d8fSBjörn Töpel umem->pgs = NULL; 359 c0c77d8fSBjörn Töpel umem->user = NULL; 360 ac98d8aaSMagnus Karlsson INIT_LIST_HEAD(&umem->xsk_list); 361 ac98d8aaSMagnus Karlsson spin_lock_init(&umem->xsk_list_lock); 362 c0c77d8fSBjörn Töpel 363 d3b42f14SBjörn Töpel refcount_set(&umem->users, 1); 364 c0c77d8fSBjörn Töpel 365 c0c77d8fSBjörn Töpel err = xdp_umem_account_pages(umem); 366 c0c77d8fSBjörn Töpel if (err) 367 c0c77d8fSBjörn Töpel goto out; 368 c0c77d8fSBjörn Töpel 369 c0c77d8fSBjörn Töpel err = xdp_umem_pin_pages(umem); 370 c0c77d8fSBjörn Töpel if (err) 371 c0c77d8fSBjörn Töpel goto out_account; 372 8aef7340SBjörn Töpel 373 8aef7340SBjörn Töpel umem->pages = kcalloc(umem->npgs, sizeof(*umem->pages), GFP_KERNEL); 374 8aef7340SBjörn Töpel if (!umem->pages) { 375 8aef7340SBjörn Töpel err = -ENOMEM; 376 8aef7340SBjörn Töpel goto out_account; 377 8aef7340SBjörn Töpel } 378 8aef7340SBjörn Töpel 379 8aef7340SBjörn Töpel for (i = 0; i < umem->npgs; i++) 380 8aef7340SBjörn Töpel umem->pages[i].addr = page_address(umem->pgs[i]); 381 8aef7340SBjörn Töpel 382 c0c77d8fSBjörn Töpel return 0; 383 c0c77d8fSBjörn Töpel 384 c0c77d8fSBjörn Töpel out_account: 385 c0c77d8fSBjörn Töpel xdp_umem_unaccount_pages(umem); 386 c0c77d8fSBjörn Töpel out: 387 c0c77d8fSBjörn Töpel put_pid(umem->pid); 388 c0c77d8fSBjörn Töpel return err; 389 c0c77d8fSBjörn Töpel } 390 965a9909SMagnus Karlsson 391 a49049eaSBjörn Töpel struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr) 392 a49049eaSBjörn Töpel { 393 a49049eaSBjörn Töpel struct xdp_umem *umem; 394 a49049eaSBjörn Töpel int err; 395 a49049eaSBjörn Töpel 396 a49049eaSBjörn Töpel umem = kzalloc(sizeof(*umem), GFP_KERNEL); 397 a49049eaSBjörn Töpel if (!umem) 398 a49049eaSBjörn Töpel return ERR_PTR(-ENOMEM); 399 a49049eaSBjörn Töpel 400 a49049eaSBjörn Töpel err = xdp_umem_reg(umem, mr); 401 a49049eaSBjörn Töpel if (err) { 402 a49049eaSBjörn Töpel kfree(umem); 403 a49049eaSBjörn Töpel return ERR_PTR(err); 404 a49049eaSBjörn Töpel } 405 a49049eaSBjörn Töpel 406 a49049eaSBjörn Töpel return umem; 407 a49049eaSBjörn Töpel } 408 a49049eaSBjörn Töpel 409 965a9909SMagnus Karlsson bool xdp_umem_validate_queues(struct xdp_umem *umem) 410 965a9909SMagnus Karlsson { 411 da60cf00SBjörn Töpel return umem->fq && umem->cq; 412 965a9909SMagnus Karlsson } 413