core.c (9a25c1df24a6fea9dc79eec950453c4e00f707fd) core.c (88dca4ca5a93d2c09e5bbc6a62fbfc3af83c4fca)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Linux Socket Filter - Kernel level socket filtering
4 *
5 * Based on the design of the Berkeley Packet Filter. The new
6 * internal format has been designed by PLUMgrid:
7 *
8 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com

--- 68 unchanged lines hidden (view full) ---

77
78struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags)
79{
80 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
81 struct bpf_prog_aux *aux;
82 struct bpf_prog *fp;
83
84 size = round_up(size, PAGE_SIZE);
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Linux Socket Filter - Kernel level socket filtering
4 *
5 * Based on the design of the Berkeley Packet Filter. The new
6 * internal format has been designed by PLUMgrid:
7 *
8 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com

--- 68 unchanged lines hidden (view full) ---

77
78struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags)
79{
80 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
81 struct bpf_prog_aux *aux;
82 struct bpf_prog *fp;
83
84 size = round_up(size, PAGE_SIZE);
85 fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);
85 fp = __vmalloc(size, gfp_flags);
86 if (fp == NULL)
87 return NULL;
88
89 aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags);
90 if (aux == NULL) {
91 vfree(fp);
92 return NULL;
93 }

--- 133 unchanged lines hidden (view full) ---

227 if (pages <= fp_old->pages)
228 return fp_old;
229
230 delta = pages - fp_old->pages;
231 ret = __bpf_prog_charge(fp_old->aux->user, delta);
232 if (ret)
233 return NULL;
234
86 if (fp == NULL)
87 return NULL;
88
89 aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags);
90 if (aux == NULL) {
91 vfree(fp);
92 return NULL;
93 }

--- 133 unchanged lines hidden (view full) ---

227 if (pages <= fp_old->pages)
228 return fp_old;
229
230 delta = pages - fp_old->pages;
231 ret = __bpf_prog_charge(fp_old->aux->user, delta);
232 if (ret)
233 return NULL;
234
235 fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);
235 fp = __vmalloc(size, gfp_flags);
236 if (fp == NULL) {
237 __bpf_prog_uncharge(fp_old->aux->user, delta);
238 } else {
239 memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);
240 fp->pages = pages;
241 fp->aux->prog = fp;
242
243 /* We keep fp->aux from fp_old around in the new

--- 13 unchanged lines hidden (view full) ---

257 kfree(fp->aux->poke_tab);
258 kfree(fp->aux);
259 }
260 vfree(fp);
261}
262
263int bpf_prog_calc_tag(struct bpf_prog *fp)
264{
236 if (fp == NULL) {
237 __bpf_prog_uncharge(fp_old->aux->user, delta);
238 } else {
239 memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);
240 fp->pages = pages;
241 fp->aux->prog = fp;
242
243 /* We keep fp->aux from fp_old around in the new

--- 13 unchanged lines hidden (view full) ---

257 kfree(fp->aux->poke_tab);
258 kfree(fp->aux);
259 }
260 vfree(fp);
261}
262
263int bpf_prog_calc_tag(struct bpf_prog *fp)
264{
265 const u32 bits_offset = SHA_MESSAGE_BYTES - sizeof(__be64);
265 const u32 bits_offset = SHA1_BLOCK_SIZE - sizeof(__be64);
266 u32 raw_size = bpf_prog_tag_scratch_size(fp);
266 u32 raw_size = bpf_prog_tag_scratch_size(fp);
267 u32 digest[SHA_DIGEST_WORDS];
268 u32 ws[SHA_WORKSPACE_WORDS];
267 u32 digest[SHA1_DIGEST_WORDS];
268 u32 ws[SHA1_WORKSPACE_WORDS];
269 u32 i, bsize, psize, blocks;
270 struct bpf_insn *dst;
271 bool was_ld_map;
272 u8 *raw, *todo;
273 __be32 *result;
274 __be64 *bits;
275
276 raw = vmalloc(raw_size);
277 if (!raw)
278 return -ENOMEM;
279
269 u32 i, bsize, psize, blocks;
270 struct bpf_insn *dst;
271 bool was_ld_map;
272 u8 *raw, *todo;
273 __be32 *result;
274 __be64 *bits;
275
276 raw = vmalloc(raw_size);
277 if (!raw)
278 return -ENOMEM;
279
280 sha_init(digest);
280 sha1_init(digest);
281 memset(ws, 0, sizeof(ws));
282
283 /* We need to take out the map fd for the digest calculation
284 * since they are unstable from user space side.
285 */
286 dst = (void *)raw;
287 for (i = 0, was_ld_map = false; i < fp->len; i++) {
288 dst[i] = fp->insnsi[i];

--- 14 unchanged lines hidden (view full) ---

303 was_ld_map = false;
304 }
305 }
306
307 psize = bpf_prog_insn_size(fp);
308 memset(&raw[psize], 0, raw_size - psize);
309 raw[psize++] = 0x80;
310
281 memset(ws, 0, sizeof(ws));
282
283 /* We need to take out the map fd for the digest calculation
284 * since they are unstable from user space side.
285 */
286 dst = (void *)raw;
287 for (i = 0, was_ld_map = false; i < fp->len; i++) {
288 dst[i] = fp->insnsi[i];

--- 14 unchanged lines hidden (view full) ---

303 was_ld_map = false;
304 }
305 }
306
307 psize = bpf_prog_insn_size(fp);
308 memset(&raw[psize], 0, raw_size - psize);
309 raw[psize++] = 0x80;
310
311 bsize = round_up(psize, SHA_MESSAGE_BYTES);
312 blocks = bsize / SHA_MESSAGE_BYTES;
311 bsize = round_up(psize, SHA1_BLOCK_SIZE);
312 blocks = bsize / SHA1_BLOCK_SIZE;
313 todo = raw;
314 if (bsize - psize >= sizeof(__be64)) {
315 bits = (__be64 *)(todo + bsize - sizeof(__be64));
316 } else {
317 bits = (__be64 *)(todo + bsize + bits_offset);
318 blocks++;
319 }
320 *bits = cpu_to_be64((psize - 1) << 3);
321
322 while (blocks--) {
313 todo = raw;
314 if (bsize - psize >= sizeof(__be64)) {
315 bits = (__be64 *)(todo + bsize - sizeof(__be64));
316 } else {
317 bits = (__be64 *)(todo + bsize + bits_offset);
318 blocks++;
319 }
320 *bits = cpu_to_be64((psize - 1) << 3);
321
322 while (blocks--) {
323 sha_transform(digest, todo, ws);
324 todo += SHA_MESSAGE_BYTES;
323 sha1_transform(digest, todo, ws);
324 todo += SHA1_BLOCK_SIZE;
325 }
326
327 result = (__force __be32 *)digest;
325 }
326
327 result = (__force __be32 *)digest;
328 for (i = 0; i < SHA_DIGEST_WORDS; i++)
328 for (i = 0; i < SHA1_DIGEST_WORDS; i++)
329 result[i] = cpu_to_be32(digest[i]);
330 memcpy(fp->tag, result, sizeof(fp->tag));
331
332 vfree(raw);
333 return 0;
334}
335
336static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old,

--- 304 unchanged lines hidden (view full) ---

641{
642 return list_empty(&fp->aux->ksym.lnode) ||
643 fp->aux->ksym.lnode.prev == LIST_POISON2;
644}
645
646void bpf_prog_kallsyms_add(struct bpf_prog *fp)
647{
648 if (!bpf_prog_kallsyms_candidate(fp) ||
329 result[i] = cpu_to_be32(digest[i]);
330 memcpy(fp->tag, result, sizeof(fp->tag));
331
332 vfree(raw);
333 return 0;
334}
335
336static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old,

--- 304 unchanged lines hidden (view full) ---

641{
642 return list_empty(&fp->aux->ksym.lnode) ||
643 fp->aux->ksym.lnode.prev == LIST_POISON2;
644}
645
646void bpf_prog_kallsyms_add(struct bpf_prog *fp)
647{
648 if (!bpf_prog_kallsyms_candidate(fp) ||
649 !bpf_capable())
649 !capable(CAP_SYS_ADMIN))
650 return;
651
652 bpf_prog_ksym_set_addr(fp);
653 bpf_prog_ksym_set_name(fp);
654 fp->aux->ksym.prog = true;
655
656 bpf_ksym_add(&fp->aux->ksym);
657}

--- 426 unchanged lines hidden (view full) ---

1084}
1085
1086static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other,
1087 gfp_t gfp_extra_flags)
1088{
1089 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
1090 struct bpf_prog *fp;
1091
650 return;
651
652 bpf_prog_ksym_set_addr(fp);
653 bpf_prog_ksym_set_name(fp);
654 fp->aux->ksym.prog = true;
655
656 bpf_ksym_add(&fp->aux->ksym);
657}

--- 426 unchanged lines hidden (view full) ---

1084}
1085
1086static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other,
1087 gfp_t gfp_extra_flags)
1088{
1089 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
1090 struct bpf_prog *fp;
1091
1092 fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags, PAGE_KERNEL);
1092 fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags);
1093 if (fp != NULL) {
1094 /* aux->prog still points to the fp_other one, so
1095 * when promoting the clone to the real program,
1096 * this still needs to be adapted.
1097 */
1098 memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE);
1099 }
1100

--- 437 unchanged lines hidden (view full) ---

1538 tail_call_cnt++;
1539
1540 prog = READ_ONCE(array->ptrs[index]);
1541 if (!prog)
1542 goto out;
1543
1544 /* ARG1 at this point is guaranteed to point to CTX from
1545 * the verifier side due to the fact that the tail call is
1093 if (fp != NULL) {
1094 /* aux->prog still points to the fp_other one, so
1095 * when promoting the clone to the real program,
1096 * this still needs to be adapted.
1097 */
1098 memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE);
1099 }
1100

--- 437 unchanged lines hidden (view full) ---

1538 tail_call_cnt++;
1539
1540 prog = READ_ONCE(array->ptrs[index]);
1541 if (!prog)
1542 goto out;
1543
1544 /* ARG1 at this point is guaranteed to point to CTX from
1545 * the verifier side due to the fact that the tail call is
1546 * handled like a helper, that is, bpf_tail_call_proto,
1546 * handeled like a helper, that is, bpf_tail_call_proto,
1547 * where arg1_type is ARG_PTR_TO_CTX.
1548 */
1549 insn = prog->insnsi;
1550 goto select_insn;
1551out:
1552 CONT;
1553 }
1554 JMP_JA:

--- 576 unchanged lines hidden (view full) ---

2131
2132 state = &get_cpu_var(bpf_user_rnd_state);
2133 res = prandom_u32_state(state);
2134 put_cpu_var(bpf_user_rnd_state);
2135
2136 return res;
2137}
2138
1547 * where arg1_type is ARG_PTR_TO_CTX.
1548 */
1549 insn = prog->insnsi;
1550 goto select_insn;
1551out:
1552 CONT;
1553 }
1554 JMP_JA:

--- 576 unchanged lines hidden (view full) ---

2131
2132 state = &get_cpu_var(bpf_user_rnd_state);
2133 res = prandom_u32_state(state);
2134 put_cpu_var(bpf_user_rnd_state);
2135
2136 return res;
2137}
2138
2139BPF_CALL_0(bpf_get_raw_cpu_id)
2140{
2141 return raw_smp_processor_id();
2142}
2143
2144/* Weak definitions of helper functions in case we don't have bpf syscall. */
2145const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;
2146const struct bpf_func_proto bpf_map_update_elem_proto __weak;
2147const struct bpf_func_proto bpf_map_delete_elem_proto __weak;
2148const struct bpf_func_proto bpf_map_push_elem_proto __weak;
2149const struct bpf_func_proto bpf_map_pop_elem_proto __weak;
2150const struct bpf_func_proto bpf_map_peek_elem_proto __weak;
2151const struct bpf_func_proto bpf_spin_lock_proto __weak;
2152const struct bpf_func_proto bpf_spin_unlock_proto __weak;
2153const struct bpf_func_proto bpf_jiffies64_proto __weak;
2154
2155const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;
2156const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;
2157const struct bpf_func_proto bpf_get_numa_node_id_proto __weak;
2158const struct bpf_func_proto bpf_ktime_get_ns_proto __weak;
2139/* Weak definitions of helper functions in case we don't have bpf syscall. */
2140const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;
2141const struct bpf_func_proto bpf_map_update_elem_proto __weak;
2142const struct bpf_func_proto bpf_map_delete_elem_proto __weak;
2143const struct bpf_func_proto bpf_map_push_elem_proto __weak;
2144const struct bpf_func_proto bpf_map_pop_elem_proto __weak;
2145const struct bpf_func_proto bpf_map_peek_elem_proto __weak;
2146const struct bpf_func_proto bpf_spin_lock_proto __weak;
2147const struct bpf_func_proto bpf_spin_unlock_proto __weak;
2148const struct bpf_func_proto bpf_jiffies64_proto __weak;
2149
2150const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;
2151const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;
2152const struct bpf_func_proto bpf_get_numa_node_id_proto __weak;
2153const struct bpf_func_proto bpf_ktime_get_ns_proto __weak;
2159const struct bpf_func_proto bpf_ktime_get_boot_ns_proto __weak;
2160
2161const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak;
2162const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak;
2163const struct bpf_func_proto bpf_get_current_comm_proto __weak;
2164const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak;
2165const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto __weak;
2166const struct bpf_func_proto bpf_get_local_storage_proto __weak;
2167const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak;

--- 78 unchanged lines hidden --- 		2154
2155const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak;
2156const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak;
2157const struct bpf_func_proto bpf_get_current_comm_proto __weak;
2158const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak;
2159const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto __weak;
2160const struct bpf_func_proto bpf_get_local_storage_proto __weak;
2161const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak;

--- 78 unchanged lines hidden ---