1 // SPDX-License-Identifier: GPL-2.0 2 /* XDP user-space packet buffer 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 */ 14 15 #include <linux/init.h> 16 #include <linux/sched/mm.h> 17 #include <linux/sched/signal.h> 18 #include <linux/sched/task.h> 19 #include <linux/uaccess.h> 20 #include <linux/slab.h> 21 #include <linux/bpf.h> 22 #include <linux/mm.h> 23 24 #include "xdp_umem.h" 25 26 #define XDP_UMEM_MIN_FRAME_SIZE 2048 27 28 int xdp_umem_create(struct xdp_umem **umem) 29 { 30 *umem = kzalloc(sizeof(**umem), GFP_KERNEL); 31 32 if (!(*umem)) 33 return -ENOMEM; 34 35 return 0; 36 } 37 38 static void xdp_umem_unpin_pages(struct xdp_umem *umem) 39 { 40 unsigned int i; 41 42 if (umem->pgs) { 43 for (i = 0; i < umem->npgs; i++) { 44 struct page *page = umem->pgs[i]; 45 46 set_page_dirty_lock(page); 47 put_page(page); 48 } 49 50 kfree(umem->pgs); 51 umem->pgs = NULL; 52 } 53 } 54 55 static void xdp_umem_unaccount_pages(struct xdp_umem *umem) 56 { 57 if (umem->user) { 58 atomic_long_sub(umem->npgs, &umem->user->locked_vm); 59 free_uid(umem->user); 60 } 61 } 62 63 static void xdp_umem_release(struct xdp_umem *umem) 64 { 65 struct task_struct *task; 66 struct mm_struct *mm; 67 68 if (umem->fq) { 69 xskq_destroy(umem->fq); 70 umem->fq = NULL; 71 } 72 73 if (umem->cq) { 74 xskq_destroy(umem->cq); 75 umem->cq = NULL; 76 } 77 78 if (umem->pgs) { 79 xdp_umem_unpin_pages(umem); 80 81 task = get_pid_task(umem->pid, PIDTYPE_PID); 82 put_pid(umem->pid); 83 if (!task) 84 goto out; 85 mm = get_task_mm(task); 86 put_task_struct(task); 87 if (!mm) 88 goto out; 89 90 mmput(mm); 91 umem->pgs = NULL; 92 } 93 94 xdp_umem_unaccount_pages(umem); 95 out: 96 kfree(umem); 97 } 98 99 static void xdp_umem_release_deferred(struct work_struct *work) 100 { 101 struct xdp_umem *umem = container_of(work, struct xdp_umem, work); 102 103 xdp_umem_release(umem); 104 } 105 106 void xdp_get_umem(struct xdp_umem *umem) 107 { 108 atomic_inc(&umem->users); 109 } 110 111 void xdp_put_umem(struct xdp_umem *umem) 112 { 113 if (!umem) 114 return; 115 116 if (atomic_dec_and_test(&umem->users)) { 117 INIT_WORK(&umem->work, xdp_umem_release_deferred); 118 schedule_work(&umem->work); 119 } 120 } 121 122 static int xdp_umem_pin_pages(struct xdp_umem *umem) 123 { 124 unsigned int gup_flags = FOLL_WRITE; 125 long npgs; 126 int err; 127 128 umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs), GFP_KERNEL); 129 if (!umem->pgs) 130 return -ENOMEM; 131 132 down_write(¤t->mm->mmap_sem); 133 npgs = get_user_pages(umem->address, umem->npgs, 134 gup_flags, &umem->pgs[0], NULL); 135 up_write(¤t->mm->mmap_sem); 136 137 if (npgs != umem->npgs) { 138 if (npgs >= 0) { 139 umem->npgs = npgs; 140 err = -ENOMEM; 141 goto out_pin; 142 } 143 err = npgs; 144 goto out_pgs; 145 } 146 return 0; 147 148 out_pin: 149 xdp_umem_unpin_pages(umem); 150 out_pgs: 151 kfree(umem->pgs); 152 umem->pgs = NULL; 153 return err; 154 } 155 156 static int xdp_umem_account_pages(struct xdp_umem *umem) 157 { 158 unsigned long lock_limit, new_npgs, old_npgs; 159 160 if (capable(CAP_IPC_LOCK)) 161 return 0; 162 163 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 164 umem->user = get_uid(current_user()); 165 166 do { 167 old_npgs = atomic_long_read(&umem->user->locked_vm); 168 new_npgs = old_npgs + umem->npgs; 169 if (new_npgs > lock_limit) { 170 free_uid(umem->user); 171 umem->user = NULL; 172 return -ENOBUFS; 173 } 174 } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs, 175 new_npgs) != old_npgs); 176 return 0; 177 } 178 179 int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) 180 { 181 u32 frame_size = mr->frame_size, frame_headroom = mr->frame_headroom; 182 u64 addr = mr->addr, size = mr->len; 183 unsigned int nframes, nfpp; 184 int size_chk, err; 185 186 if (!umem) 187 return -EINVAL; 188 189 if (frame_size < XDP_UMEM_MIN_FRAME_SIZE || frame_size > PAGE_SIZE) { 190 /* Strictly speaking we could support this, if: 191 * - huge pages, or* 192 * - using an IOMMU, or 193 * - making sure the memory area is consecutive 194 * but for now, we simply say "computer says no". 195 */ 196 return -EINVAL; 197 } 198 199 if (!is_power_of_2(frame_size)) 200 return -EINVAL; 201 202 if (!PAGE_ALIGNED(addr)) { 203 /* Memory area has to be page size aligned. For 204 * simplicity, this might change. 205 */ 206 return -EINVAL; 207 } 208 209 if ((addr + size) < addr) 210 return -EINVAL; 211 212 nframes = (unsigned int)div_u64(size, frame_size); 213 if (nframes == 0 || nframes > UINT_MAX) 214 return -EINVAL; 215 216 nfpp = PAGE_SIZE / frame_size; 217 if (nframes < nfpp || nframes % nfpp) 218 return -EINVAL; 219 220 frame_headroom = ALIGN(frame_headroom, 64); 221 222 size_chk = frame_size - frame_headroom - XDP_PACKET_HEADROOM; 223 if (size_chk < 0) 224 return -EINVAL; 225 226 umem->pid = get_task_pid(current, PIDTYPE_PID); 227 umem->size = (size_t)size; 228 umem->address = (unsigned long)addr; 229 umem->props.frame_size = frame_size; 230 umem->props.nframes = nframes; 231 umem->frame_headroom = frame_headroom; 232 umem->npgs = size / PAGE_SIZE; 233 umem->pgs = NULL; 234 umem->user = NULL; 235 236 umem->frame_size_log2 = ilog2(frame_size); 237 umem->nfpp_mask = nfpp - 1; 238 umem->nfpplog2 = ilog2(nfpp); 239 atomic_set(&umem->users, 1); 240 241 err = xdp_umem_account_pages(umem); 242 if (err) 243 goto out; 244 245 err = xdp_umem_pin_pages(umem); 246 if (err) 247 goto out_account; 248 return 0; 249 250 out_account: 251 xdp_umem_unaccount_pages(umem); 252 out: 253 put_pid(umem->pid); 254 return err; 255 } 256 257 bool xdp_umem_validate_queues(struct xdp_umem *umem) 258 { 259 return (umem->fq && umem->cq); 260 } 261