xdp_umem.c (c4655761d3cf62bf5f86650e79349c1bfa5c6285) | xdp_umem.c (1c1efc2af158869795d3334a12fed2afd9c51539) |
---|---|
1// SPDX-License-Identifier: GPL-2.0 2/* XDP user-space packet buffer 3 * Copyright(c) 2018 Intel Corporation. 4 */ 5 6#include <linux/init.h> 7#include <linux/sched/mm.h> 8#include <linux/sched/signal.h> --- 33 unchanged lines hidden (view full) --- 42 if (!xs->tx) 43 return; 44 45 spin_lock_irqsave(&umem->xsk_tx_list_lock, flags); 46 list_del_rcu(&xs->list); 47 spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags); 48} 49 | 1// SPDX-License-Identifier: GPL-2.0 2/* XDP user-space packet buffer 3 * Copyright(c) 2018 Intel Corporation. 4 */ 5 6#include <linux/init.h> 7#include <linux/sched/mm.h> 8#include <linux/sched/signal.h> --- 33 unchanged lines hidden (view full) --- 42 if (!xs->tx) 43 return; 44 45 spin_lock_irqsave(&umem->xsk_tx_list_lock, flags); 46 list_del_rcu(&xs->list); 47 spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags); 48} 49 |
50/* The umem is stored both in the _rx struct and the _tx struct as we do 51 * not know if the device has more tx queues than rx, or the opposite. 52 * This might also change during run time. 53 */ 54static int xsk_reg_pool_at_qid(struct net_device *dev, 55 struct xsk_buff_pool *pool, 56 u16 queue_id) | 50static void xdp_umem_unpin_pages(struct xdp_umem *umem) |
57{ | 51{ |
58 if (queue_id >= max_t(unsigned int, 59 dev->real_num_rx_queues, 60 dev->real_num_tx_queues)) 61 return -EINVAL; | 52 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true); |
62 | 53 |
63 if (queue_id < dev->real_num_rx_queues) 64 dev->_rx[queue_id].pool = pool; 65 if (queue_id < dev->real_num_tx_queues) 66 dev->_tx[queue_id].pool = pool; 67 68 return 0; | 54 kfree(umem->pgs); 55 umem->pgs = NULL; |
69} 70 | 56} 57 |
71struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev, 72 u16 queue_id) | 58static void xdp_umem_unaccount_pages(struct xdp_umem *umem) |
73{ | 59{ |
74 if (queue_id < dev->real_num_rx_queues) 75 return dev->_rx[queue_id].pool; 76 if (queue_id < dev->real_num_tx_queues) 77 return dev->_tx[queue_id].pool; 78 79 return NULL; | 60 if (umem->user) { 61 atomic_long_sub(umem->npgs, &umem->user->locked_vm); 62 free_uid(umem->user); 63 } |
80} | 64} |
81EXPORT_SYMBOL(xsk_get_pool_from_qid); | |
82 | 65 |
83static void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id) | 66void xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, 67 u16 queue_id) |
84{ | 68{ |
85 if (queue_id < dev->real_num_rx_queues) 86 dev->_rx[queue_id].pool = NULL; 87 if (queue_id < dev->real_num_tx_queues) 88 dev->_tx[queue_id].pool = NULL; 89} 90 91int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev, 92 u16 queue_id, u16 flags) 93{ 94 bool force_zc, force_copy; 95 struct netdev_bpf bpf; 96 int err = 0; 97 98 ASSERT_RTNL(); 99 100 force_zc = flags & XDP_ZEROCOPY; 101 force_copy = flags & XDP_COPY; 102 103 if (force_zc && force_copy) 104 return -EINVAL; 105 106 if (xsk_get_pool_from_qid(dev, queue_id)) 107 return -EBUSY; 108 109 err = xsk_reg_pool_at_qid(dev, umem->pool, queue_id); 110 if (err) 111 return err; 112 | |
113 umem->dev = dev; 114 umem->queue_id = queue_id; 115 | 69 umem->dev = dev; 70 umem->queue_id = queue_id; 71 |
116 if (flags & XDP_USE_NEED_WAKEUP) { 117 umem->flags |= XDP_UMEM_USES_NEED_WAKEUP; 118 /* Tx needs to be explicitly woken up the first time. 119 * Also for supporting drivers that do not implement this 120 * feature. They will always have to call sendto(). 121 */ 122 xsk_set_tx_need_wakeup(umem->pool); 123 } 124 | |
125 dev_hold(dev); | 72 dev_hold(dev); |
126 127 if (force_copy) 128 /* For copy-mode, we are done. */ 129 return 0; 130 131 if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xsk_wakeup) { 132 err = -EOPNOTSUPP; 133 goto err_unreg_umem; 134 } 135 136 bpf.command = XDP_SETUP_XSK_POOL; 137 bpf.xsk.pool = umem->pool; 138 bpf.xsk.queue_id = queue_id; 139 140 err = dev->netdev_ops->ndo_bpf(dev, &bpf); 141 if (err) 142 goto err_unreg_umem; 143 144 umem->zc = true; 145 return 0; 146 147err_unreg_umem: 148 if (!force_zc) 149 err = 0; /* fallback to copy mode */ 150 if (err) 151 xsk_clear_pool_at_qid(dev, queue_id); 152 return err; | |
153} 154 155void xdp_umem_clear_dev(struct xdp_umem *umem) 156{ | 73} 74 75void xdp_umem_clear_dev(struct xdp_umem *umem) 76{ |
157 struct netdev_bpf bpf; 158 int err; 159 160 ASSERT_RTNL(); 161 162 if (!umem->dev) 163 return; 164 165 if (umem->zc) { 166 bpf.command = XDP_SETUP_XSK_POOL; 167 bpf.xsk.pool = NULL; 168 bpf.xsk.queue_id = umem->queue_id; 169 170 err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf); 171 172 if (err) 173 WARN(1, "failed to disable umem!\n"); 174 } 175 176 xsk_clear_pool_at_qid(umem->dev, umem->queue_id); 177 | |
178 dev_put(umem->dev); 179 umem->dev = NULL; 180 umem->zc = false; 181} 182 | 77 dev_put(umem->dev); 78 umem->dev = NULL; 79 umem->zc = false; 80} 81 |
183static void xdp_umem_unpin_pages(struct xdp_umem *umem) 184{ 185 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true); 186 187 kfree(umem->pgs); 188 umem->pgs = NULL; 189} 190 191static void xdp_umem_unaccount_pages(struct xdp_umem *umem) 192{ 193 if (umem->user) { 194 atomic_long_sub(umem->npgs, &umem->user->locked_vm); 195 free_uid(umem->user); 196 } 197} 198 | |
199static void xdp_umem_release(struct xdp_umem *umem) 200{ | 82static void xdp_umem_release(struct xdp_umem *umem) 83{ |
201 rtnl_lock(); | |
202 xdp_umem_clear_dev(umem); | 84 xdp_umem_clear_dev(umem); |
203 rtnl_unlock(); | |
204 205 ida_simple_remove(&umem_ida, umem->id); 206 207 if (umem->fq) { 208 xskq_destroy(umem->fq); 209 umem->fq = NULL; 210 } 211 212 if (umem->cq) { 213 xskq_destroy(umem->cq); 214 umem->cq = NULL; 215 } 216 | 85 86 ida_simple_remove(&umem_ida, umem->id); 87 88 if (umem->fq) { 89 xskq_destroy(umem->fq); 90 umem->fq = NULL; 91 } 92 93 if (umem->cq) { 94 xskq_destroy(umem->cq); 95 umem->cq = NULL; 96 } 97 |
217 xp_destroy(umem->pool); | |
218 xdp_umem_unpin_pages(umem); 219 220 xdp_umem_unaccount_pages(umem); 221 kfree(umem); 222} 223 | 98 xdp_umem_unpin_pages(umem); 99 100 xdp_umem_unaccount_pages(umem); 101 kfree(umem); 102} 103 |
224static void xdp_umem_release_deferred(struct work_struct *work) 225{ 226 struct xdp_umem *umem = container_of(work, struct xdp_umem, work); 227 228 xdp_umem_release(umem); 229} 230 | |
231void xdp_get_umem(struct xdp_umem *umem) 232{ 233 refcount_inc(&umem->users); 234} 235 236void xdp_put_umem(struct xdp_umem *umem) 237{ 238 if (!umem) 239 return; 240 | 104void xdp_get_umem(struct xdp_umem *umem) 105{ 106 refcount_inc(&umem->users); 107} 108 109void xdp_put_umem(struct xdp_umem *umem) 110{ 111 if (!umem) 112 return; 113 |
241 if (refcount_dec_and_test(&umem->users)) { 242 INIT_WORK(&umem->work, xdp_umem_release_deferred); 243 schedule_work(&umem->work); 244 } | 114 if (refcount_dec_and_test(&umem->users)) 115 xdp_umem_release(umem); |
245} 246 247static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address) 248{ 249 unsigned int gup_flags = FOLL_WRITE; 250 long npgs; 251 int err; 252 --- 99 unchanged lines hidden (view full) --- 352 } 353 354 if (headroom >= chunk_size - XDP_PACKET_HEADROOM) 355 return -EINVAL; 356 357 umem->size = size; 358 umem->headroom = headroom; 359 umem->chunk_size = chunk_size; | 116} 117 118static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address) 119{ 120 unsigned int gup_flags = FOLL_WRITE; 121 long npgs; 122 int err; 123 --- 99 unchanged lines hidden (view full) --- 223 } 224 225 if (headroom >= chunk_size - XDP_PACKET_HEADROOM) 226 return -EINVAL; 227 228 umem->size = size; 229 umem->headroom = headroom; 230 umem->chunk_size = chunk_size; |
231 umem->chunks = chunks; |
|
360 umem->npgs = (u32)npgs; 361 umem->pgs = NULL; 362 umem->user = NULL; 363 umem->flags = mr->flags; 364 INIT_LIST_HEAD(&umem->xsk_tx_list); 365 spin_lock_init(&umem->xsk_tx_list_lock); 366 367 refcount_set(&umem->users, 1); 368 369 err = xdp_umem_account_pages(umem); 370 if (err) 371 return err; 372 373 err = xdp_umem_pin_pages(umem, (unsigned long)addr); 374 if (err) 375 goto out_account; 376 | 232 umem->npgs = (u32)npgs; 233 umem->pgs = NULL; 234 umem->user = NULL; 235 umem->flags = mr->flags; 236 INIT_LIST_HEAD(&umem->xsk_tx_list); 237 spin_lock_init(&umem->xsk_tx_list_lock); 238 239 refcount_set(&umem->users, 1); 240 241 err = xdp_umem_account_pages(umem); 242 if (err) 243 return err; 244 245 err = xdp_umem_pin_pages(umem, (unsigned long)addr); 246 if (err) 247 goto out_account; 248 |
377 umem->pool = xp_create(umem, chunks, chunk_size, headroom, size, 378 unaligned_chunks); 379 if (!umem->pool) { 380 err = -ENOMEM; 381 goto out_pin; 382 } | |
383 return 0; 384 | 249 return 0; 250 |
385out_pin: 386 xdp_umem_unpin_pages(umem); | |
387out_account: 388 xdp_umem_unaccount_pages(umem); 389 return err; 390} 391 392struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr) 393{ 394 struct xdp_umem *umem; --- 27 unchanged lines hidden --- | 251out_account: 252 xdp_umem_unaccount_pages(umem); 253 return err; 254} 255 256struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr) 257{ 258 struct xdp_umem *umem; --- 27 unchanged lines hidden --- |