xdp_umem.c (c4655761d3cf62bf5f86650e79349c1bfa5c6285) xdp_umem.c (1c1efc2af158869795d3334a12fed2afd9c51539)
1// SPDX-License-Identifier: GPL-2.0
2/* XDP user-space packet buffer
3 * Copyright(c) 2018 Intel Corporation.
4 */
5
6#include <linux/init.h>
7#include <linux/sched/mm.h>
8#include <linux/sched/signal.h>

--- 33 unchanged lines hidden (view full) ---

42 if (!xs->tx)
43 return;
44
45 spin_lock_irqsave(&umem->xsk_tx_list_lock, flags);
46 list_del_rcu(&xs->list);
47 spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags);
48}
49
1// SPDX-License-Identifier: GPL-2.0
2/* XDP user-space packet buffer
3 * Copyright(c) 2018 Intel Corporation.
4 */
5
6#include <linux/init.h>
7#include <linux/sched/mm.h>
8#include <linux/sched/signal.h>

--- 33 unchanged lines hidden (view full) ---

42 if (!xs->tx)
43 return;
44
45 spin_lock_irqsave(&umem->xsk_tx_list_lock, flags);
46 list_del_rcu(&xs->list);
47 spin_unlock_irqrestore(&umem->xsk_tx_list_lock, flags);
48}
49
50/* The umem is stored both in the _rx struct and the _tx struct as we do
51 * not know if the device has more tx queues than rx, or the opposite.
52 * This might also change during run time.
53 */
54static int xsk_reg_pool_at_qid(struct net_device *dev,
55 struct xsk_buff_pool *pool,
56 u16 queue_id)
50static void xdp_umem_unpin_pages(struct xdp_umem *umem)
57{
51{
58 if (queue_id >= max_t(unsigned int,
59 dev->real_num_rx_queues,
60 dev->real_num_tx_queues))
61 return -EINVAL;
52 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true);
62
53
63 if (queue_id < dev->real_num_rx_queues)
64 dev->_rx[queue_id].pool = pool;
65 if (queue_id < dev->real_num_tx_queues)
66 dev->_tx[queue_id].pool = pool;
67
68 return 0;
54 kfree(umem->pgs);
55 umem->pgs = NULL;
69}
70
56}
57
71struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
72 u16 queue_id)
58static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
73{
59{
74 if (queue_id < dev->real_num_rx_queues)
75 return dev->_rx[queue_id].pool;
76 if (queue_id < dev->real_num_tx_queues)
77 return dev->_tx[queue_id].pool;
78
79 return NULL;
60 if (umem->user) {
61 atomic_long_sub(umem->npgs, &umem->user->locked_vm);
62 free_uid(umem->user);
63 }
80}
64}
81EXPORT_SYMBOL(xsk_get_pool_from_qid);
82
65
83static void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id)
66void xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,
67 u16 queue_id)
84{
68{
85 if (queue_id < dev->real_num_rx_queues)
86 dev->_rx[queue_id].pool = NULL;
87 if (queue_id < dev->real_num_tx_queues)
88 dev->_tx[queue_id].pool = NULL;
89}
90
91int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,
92 u16 queue_id, u16 flags)
93{
94 bool force_zc, force_copy;
95 struct netdev_bpf bpf;
96 int err = 0;
97
98 ASSERT_RTNL();
99
100 force_zc = flags & XDP_ZEROCOPY;
101 force_copy = flags & XDP_COPY;
102
103 if (force_zc && force_copy)
104 return -EINVAL;
105
106 if (xsk_get_pool_from_qid(dev, queue_id))
107 return -EBUSY;
108
109 err = xsk_reg_pool_at_qid(dev, umem->pool, queue_id);
110 if (err)
111 return err;
112
113 umem->dev = dev;
114 umem->queue_id = queue_id;
115
69 umem->dev = dev;
70 umem->queue_id = queue_id;
71
116 if (flags & XDP_USE_NEED_WAKEUP) {
117 umem->flags |= XDP_UMEM_USES_NEED_WAKEUP;
118 /* Tx needs to be explicitly woken up the first time.
119 * Also for supporting drivers that do not implement this
120 * feature. They will always have to call sendto().
121 */
122 xsk_set_tx_need_wakeup(umem->pool);
123 }
124
125 dev_hold(dev);
72 dev_hold(dev);
126
127 if (force_copy)
128 /* For copy-mode, we are done. */
129 return 0;
130
131 if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xsk_wakeup) {
132 err = -EOPNOTSUPP;
133 goto err_unreg_umem;
134 }
135
136 bpf.command = XDP_SETUP_XSK_POOL;
137 bpf.xsk.pool = umem->pool;
138 bpf.xsk.queue_id = queue_id;
139
140 err = dev->netdev_ops->ndo_bpf(dev, &bpf);
141 if (err)
142 goto err_unreg_umem;
143
144 umem->zc = true;
145 return 0;
146
147err_unreg_umem:
148 if (!force_zc)
149 err = 0; /* fallback to copy mode */
150 if (err)
151 xsk_clear_pool_at_qid(dev, queue_id);
152 return err;
153}
154
155void xdp_umem_clear_dev(struct xdp_umem *umem)
156{
73}
74
75void xdp_umem_clear_dev(struct xdp_umem *umem)
76{
157 struct netdev_bpf bpf;
158 int err;
159
160 ASSERT_RTNL();
161
162 if (!umem->dev)
163 return;
164
165 if (umem->zc) {
166 bpf.command = XDP_SETUP_XSK_POOL;
167 bpf.xsk.pool = NULL;
168 bpf.xsk.queue_id = umem->queue_id;
169
170 err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf);
171
172 if (err)
173 WARN(1, "failed to disable umem!\n");
174 }
175
176 xsk_clear_pool_at_qid(umem->dev, umem->queue_id);
177
178 dev_put(umem->dev);
179 umem->dev = NULL;
180 umem->zc = false;
181}
182
77 dev_put(umem->dev);
78 umem->dev = NULL;
79 umem->zc = false;
80}
81
183static void xdp_umem_unpin_pages(struct xdp_umem *umem)
184{
185 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true);
186
187 kfree(umem->pgs);
188 umem->pgs = NULL;
189}
190
191static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
192{
193 if (umem->user) {
194 atomic_long_sub(umem->npgs, &umem->user->locked_vm);
195 free_uid(umem->user);
196 }
197}
198
199static void xdp_umem_release(struct xdp_umem *umem)
200{
82static void xdp_umem_release(struct xdp_umem *umem)
83{
201 rtnl_lock();
202 xdp_umem_clear_dev(umem);
84 xdp_umem_clear_dev(umem);
203 rtnl_unlock();
204
205 ida_simple_remove(&umem_ida, umem->id);
206
207 if (umem->fq) {
208 xskq_destroy(umem->fq);
209 umem->fq = NULL;
210 }
211
212 if (umem->cq) {
213 xskq_destroy(umem->cq);
214 umem->cq = NULL;
215 }
216
85
86 ida_simple_remove(&umem_ida, umem->id);
87
88 if (umem->fq) {
89 xskq_destroy(umem->fq);
90 umem->fq = NULL;
91 }
92
93 if (umem->cq) {
94 xskq_destroy(umem->cq);
95 umem->cq = NULL;
96 }
97
217 xp_destroy(umem->pool);
218 xdp_umem_unpin_pages(umem);
219
220 xdp_umem_unaccount_pages(umem);
221 kfree(umem);
222}
223
98 xdp_umem_unpin_pages(umem);
99
100 xdp_umem_unaccount_pages(umem);
101 kfree(umem);
102}
103
224static void xdp_umem_release_deferred(struct work_struct *work)
225{
226 struct xdp_umem *umem = container_of(work, struct xdp_umem, work);
227
228 xdp_umem_release(umem);
229}
230
231void xdp_get_umem(struct xdp_umem *umem)
232{
233 refcount_inc(&umem->users);
234}
235
236void xdp_put_umem(struct xdp_umem *umem)
237{
238 if (!umem)
239 return;
240
104void xdp_get_umem(struct xdp_umem *umem)
105{
106 refcount_inc(&umem->users);
107}
108
109void xdp_put_umem(struct xdp_umem *umem)
110{
111 if (!umem)
112 return;
113
241 if (refcount_dec_and_test(&umem->users)) {
242 INIT_WORK(&umem->work, xdp_umem_release_deferred);
243 schedule_work(&umem->work);
244 }
114 if (refcount_dec_and_test(&umem->users))
115 xdp_umem_release(umem);
245}
246
247static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address)
248{
249 unsigned int gup_flags = FOLL_WRITE;
250 long npgs;
251 int err;
252

--- 99 unchanged lines hidden (view full) ---

352 }
353
354 if (headroom >= chunk_size - XDP_PACKET_HEADROOM)
355 return -EINVAL;
356
357 umem->size = size;
358 umem->headroom = headroom;
359 umem->chunk_size = chunk_size;
116}
117
118static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address)
119{
120 unsigned int gup_flags = FOLL_WRITE;
121 long npgs;
122 int err;
123

--- 99 unchanged lines hidden (view full) ---

223 }
224
225 if (headroom >= chunk_size - XDP_PACKET_HEADROOM)
226 return -EINVAL;
227
228 umem->size = size;
229 umem->headroom = headroom;
230 umem->chunk_size = chunk_size;
231 umem->chunks = chunks;
360 umem->npgs = (u32)npgs;
361 umem->pgs = NULL;
362 umem->user = NULL;
363 umem->flags = mr->flags;
364 INIT_LIST_HEAD(&umem->xsk_tx_list);
365 spin_lock_init(&umem->xsk_tx_list_lock);
366
367 refcount_set(&umem->users, 1);
368
369 err = xdp_umem_account_pages(umem);
370 if (err)
371 return err;
372
373 err = xdp_umem_pin_pages(umem, (unsigned long)addr);
374 if (err)
375 goto out_account;
376
232 umem->npgs = (u32)npgs;
233 umem->pgs = NULL;
234 umem->user = NULL;
235 umem->flags = mr->flags;
236 INIT_LIST_HEAD(&umem->xsk_tx_list);
237 spin_lock_init(&umem->xsk_tx_list_lock);
238
239 refcount_set(&umem->users, 1);
240
241 err = xdp_umem_account_pages(umem);
242 if (err)
243 return err;
244
245 err = xdp_umem_pin_pages(umem, (unsigned long)addr);
246 if (err)
247 goto out_account;
248
377 umem->pool = xp_create(umem, chunks, chunk_size, headroom, size,
378 unaligned_chunks);
379 if (!umem->pool) {
380 err = -ENOMEM;
381 goto out_pin;
382 }
383 return 0;
384
249 return 0;
250
385out_pin:
386 xdp_umem_unpin_pages(umem);
387out_account:
388 xdp_umem_unaccount_pages(umem);
389 return err;
390}
391
392struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr)
393{
394 struct xdp_umem *umem;

--- 27 unchanged lines hidden ---
251out_account:
252 xdp_umem_unaccount_pages(umem);
253 return err;
254}
255
256struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr)
257{
258 struct xdp_umem *umem;

--- 27 unchanged lines hidden ---