test_run.c (ea68a3e9d14e9e0bf017d178fb4bd53b6deb1482) test_run.c (75dcef8d3609d0b1d3497d6ed4809096513e0b83)
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2017 Facebook
3 */
4#include <linux/bpf.h>
5#include <linux/btf.h>
6#include <linux/btf_ids.h>
7#include <linux/slab.h>
8#include <linux/init.h>

--- 83 unchanged lines hidden (view full) ---

92
93/* We put this struct at the head of each page with a context and frame
94 * initialised when the page is allocated, so we don't have to do this on each
95 * repetition of the test run.
96 */
97struct xdp_page_head {
98 struct xdp_buff orig_ctx;
99 struct xdp_buff ctx;
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2017 Facebook
3 */
4#include <linux/bpf.h>
5#include <linux/btf.h>
6#include <linux/btf_ids.h>
7#include <linux/slab.h>
8#include <linux/init.h>

--- 83 unchanged lines hidden (view full) ---

92
93/* We put this struct at the head of each page with a context and frame
94 * initialised when the page is allocated, so we don't have to do this on each
95 * repetition of the test run.
96 */
97struct xdp_page_head {
98 struct xdp_buff orig_ctx;
99 struct xdp_buff ctx;
100 union {
101 /* ::data_hard_start starts here */
102 DECLARE_FLEX_ARRAY(struct xdp_frame, frame);
103 DECLARE_FLEX_ARRAY(u8, data);
104 };
100 struct xdp_frame frm;
101 u8 data[];
105};
106
107struct xdp_test_data {
108 struct xdp_buff *orig_ctx;
109 struct xdp_rxq_info rxq;
110 struct net_device *dev;
111 struct page_pool *pp;
112 struct xdp_frame **frames;
113 struct sk_buff **skbs;
114 struct xdp_mem_info mem;
115 u32 batch_size;
116 u32 frame_cnt;
117};
118
102};
103
104struct xdp_test_data {
105 struct xdp_buff *orig_ctx;
106 struct xdp_rxq_info rxq;
107 struct net_device *dev;
108 struct page_pool *pp;
109 struct xdp_frame **frames;
110 struct sk_buff **skbs;
111 struct xdp_mem_info mem;
112 u32 batch_size;
113 u32 frame_cnt;
114};
115
119/* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE
120 * must be updated accordingly this gets changed, otherwise BPF selftests
121 * will fail.
122 */
123#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
124#define TEST_XDP_MAX_BATCH 256
125
126static void xdp_test_run_init_page(struct page *page, void *arg)
127{
128 struct xdp_page_head *head = phys_to_virt(page_to_phys(page));
129 struct xdp_buff *new_ctx, *orig_ctx;
130 u32 headroom = XDP_PACKET_HEADROOM;
131 struct xdp_test_data *xdp = arg;
132 size_t frm_len, meta_len;
133 struct xdp_frame *frm;
134 void *data;
135
136 orig_ctx = xdp->orig_ctx;
137 frm_len = orig_ctx->data_end - orig_ctx->data_meta;
138 meta_len = orig_ctx->data - orig_ctx->data_meta;
139 headroom -= meta_len;
140
141 new_ctx = &head->ctx;
116#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
117#define TEST_XDP_MAX_BATCH 256
118
119static void xdp_test_run_init_page(struct page *page, void *arg)
120{
121 struct xdp_page_head *head = phys_to_virt(page_to_phys(page));
122 struct xdp_buff *new_ctx, *orig_ctx;
123 u32 headroom = XDP_PACKET_HEADROOM;
124 struct xdp_test_data *xdp = arg;
125 size_t frm_len, meta_len;
126 struct xdp_frame *frm;
127 void *data;
128
129 orig_ctx = xdp->orig_ctx;
130 frm_len = orig_ctx->data_end - orig_ctx->data_meta;
131 meta_len = orig_ctx->data - orig_ctx->data_meta;
132 headroom -= meta_len;
133
134 new_ctx = &head->ctx;
142 frm = head->frame;
143 data = head->data;
135 frm = &head->frm;
136 data = &head->data;
144 memcpy(data + headroom, orig_ctx->data_meta, frm_len);
145
146 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);
147 xdp_prepare_buff(new_ctx, data, headroom, frm_len, true);
148 new_ctx->data = new_ctx->data_meta + meta_len;
149
150 xdp_update_frame_from_buff(new_ctx, frm);
151 frm->mem = new_ctx->rxq->mem;

--- 58 unchanged lines hidden (view full) ---

210static void xdp_test_run_teardown(struct xdp_test_data *xdp)
211{
212 xdp_unreg_mem_model(&xdp->mem);
213 page_pool_destroy(xdp->pp);
214 kfree(xdp->frames);
215 kfree(xdp->skbs);
216}
217
137 memcpy(data + headroom, orig_ctx->data_meta, frm_len);
138
139 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);
140 xdp_prepare_buff(new_ctx, data, headroom, frm_len, true);
141 new_ctx->data = new_ctx->data_meta + meta_len;
142
143 xdp_update_frame_from_buff(new_ctx, frm);
144 frm->mem = new_ctx->rxq->mem;

--- 58 unchanged lines hidden (view full) ---

203static void xdp_test_run_teardown(struct xdp_test_data *xdp)
204{
205 xdp_unreg_mem_model(&xdp->mem);
206 page_pool_destroy(xdp->pp);
207 kfree(xdp->frames);
208 kfree(xdp->skbs);
209}
210
211static bool frame_was_changed(const struct xdp_page_head *head)
212{
213 /* xdp_scrub_frame() zeroes the data pointer, flags is the last field,
214 * i.e. has the highest chances to be overwritten. If those two are
215 * untouched, it's most likely safe to skip the context reset.
216 */
217 return head->frm.data != head->orig_ctx.data ||
218 head->frm.flags != head->orig_ctx.flags;
219}
220
218static bool ctx_was_changed(struct xdp_page_head *head)
219{
220 return head->orig_ctx.data != head->ctx.data ||
221 head->orig_ctx.data_meta != head->ctx.data_meta ||
222 head->orig_ctx.data_end != head->ctx.data_end;
223}
224
225static void reset_ctx(struct xdp_page_head *head)
226{
221static bool ctx_was_changed(struct xdp_page_head *head)
222{
223 return head->orig_ctx.data != head->ctx.data ||
224 head->orig_ctx.data_meta != head->ctx.data_meta ||
225 head->orig_ctx.data_end != head->ctx.data_end;
226}
227
228static void reset_ctx(struct xdp_page_head *head)
229{
227 if (likely(!ctx_was_changed(head)))
230 if (likely(!frame_was_changed(head) && !ctx_was_changed(head)))
228 return;
229
230 head->ctx.data = head->orig_ctx.data;
231 head->ctx.data_meta = head->orig_ctx.data_meta;
232 head->ctx.data_end = head->orig_ctx.data_end;
231 return;
232
233 head->ctx.data = head->orig_ctx.data;
234 head->ctx.data_meta = head->orig_ctx.data_meta;
235 head->ctx.data_end = head->orig_ctx.data_end;
233 xdp_update_frame_from_buff(&head->ctx, head->frame);
236 xdp_update_frame_from_buff(&head->ctx, &head->frm);
234}
235
236static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
237 struct sk_buff **skbs,
238 struct net_device *dev)
239{
240 gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
241 int i, n;

--- 45 unchanged lines hidden (view full) ---

287 if (!page) {
288 err = -ENOMEM;
289 goto out;
290 }
291
292 head = phys_to_virt(page_to_phys(page));
293 reset_ctx(head);
294 ctx = &head->ctx;
237}
238
239static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
240 struct sk_buff **skbs,
241 struct net_device *dev)
242{
243 gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
244 int i, n;

--- 45 unchanged lines hidden (view full) ---

290 if (!page) {
291 err = -ENOMEM;
292 goto out;
293 }
294
295 head = phys_to_virt(page_to_phys(page));
296 reset_ctx(head);
297 ctx = &head->ctx;
295 frm = head->frame;
298 frm = &head->frm;
296 xdp->frame_cnt++;
297
298 act = bpf_prog_run_xdp(prog, ctx);
299
300 /* if program changed pkt bounds we need to update the xdp_frame */
301 if (unlikely(ctx_was_changed(head))) {
302 ret = xdp_update_frame_from_buff(ctx, frm);
303 if (ret) {

--- 229 unchanged lines hidden (view full) ---

533 return (long)arg;
534}
535
536int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
537{
538 return (long)arg->a;
539}
540
299 xdp->frame_cnt++;
300
301 act = bpf_prog_run_xdp(prog, ctx);
302
303 /* if program changed pkt bounds we need to update the xdp_frame */
304 if (unlikely(ctx_was_changed(head))) {
305 ret = xdp_update_frame_from_buff(ctx, frm);
306 if (ret) {

--- 229 unchanged lines hidden (view full) ---

536 return (long)arg;
537}
538
539int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
540{
541 return (long)arg->a;
542}
543
544__bpf_kfunc u32 bpf_fentry_test9(u32 *a)
545{
546 return *a;
547}
548
541__bpf_kfunc int bpf_modify_return_test(int a, int *b)
542{
543 *b += 1;
544 return a + *b;
545}
546
547__bpf_kfunc u64 bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
548{

--- 13 unchanged lines hidden (view full) ---

562long noinline bpf_kfunc_call_test4(signed char a, short b, int c, long d)
563{
564 /* Provoke the compiler to assume that the caller has sign-extended a,
565 * b and c on platforms where this is required (e.g. s390x).
566 */
567 return (long)a + (long)b + (long)c + d;
568}
569
549__bpf_kfunc int bpf_modify_return_test(int a, int *b)
550{
551 *b += 1;
552 return a + *b;
553}
554
555__bpf_kfunc u64 bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
556{

--- 13 unchanged lines hidden (view full) ---

570long noinline bpf_kfunc_call_test4(signed char a, short b, int c, long d)
571{
572 /* Provoke the compiler to assume that the caller has sign-extended a,
573 * b and c on platforms where this is required (e.g. s390x).
574 */
575 return (long)a + (long)b + (long)c + d;
576}
577
578int noinline bpf_fentry_shadow_test(int a)
579{
580 return a + 1;
581}
582
570struct prog_test_member1 {
571 int a;
572};
573
574struct prog_test_member {
575 struct prog_test_member1 m;
576 int c;
577};

--- 15 unchanged lines hidden (view full) ---

593
594__bpf_kfunc struct prog_test_ref_kfunc *
595bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr)
596{
597 refcount_inc(&prog_test_struct.cnt);
598 return &prog_test_struct;
599}
600
583struct prog_test_member1 {
584 int a;
585};
586
587struct prog_test_member {
588 struct prog_test_member1 m;
589 int c;
590};

--- 15 unchanged lines hidden (view full) ---

606
607__bpf_kfunc struct prog_test_ref_kfunc *
608bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr)
609{
610 refcount_inc(&prog_test_struct.cnt);
611 return &prog_test_struct;
612}
613
614__bpf_kfunc void bpf_kfunc_call_test_offset(struct prog_test_ref_kfunc *p)
615{
616 WARN_ON_ONCE(1);
617}
618
601__bpf_kfunc struct prog_test_member *
602bpf_kfunc_call_memb_acquire(void)
603{
604 WARN_ON_ONCE(1);
605 return NULL;
606}
607
608__bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
609{
619__bpf_kfunc struct prog_test_member *
620bpf_kfunc_call_memb_acquire(void)
621{
622 WARN_ON_ONCE(1);
623 return NULL;
624}
625
626__bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
627{
610 if (!p)
611 return;
612
613 refcount_dec(&p->cnt);
614}
615
616__bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p)
617{
618}
619
620__bpf_kfunc void bpf_kfunc_call_memb1_release(struct prog_test_member1 *p)

--- 118 unchanged lines hidden (view full) ---

739}
740
741__bpf_kfunc void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len)
742{
743}
744
745__bpf_kfunc void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p)
746{
628 refcount_dec(&p->cnt);
629}
630
631__bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p)
632{
633}
634
635__bpf_kfunc void bpf_kfunc_call_memb1_release(struct prog_test_member1 *p)

--- 118 unchanged lines hidden (view full) ---

754}
755
756__bpf_kfunc void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len)
757{
758}
759
760__bpf_kfunc void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p)
761{
762 /* p != NULL, but p->cnt could be 0 */
747}
748
749__bpf_kfunc void bpf_kfunc_call_test_destructive(void)
750{
751}
752
753__bpf_kfunc static u32 bpf_kfunc_call_test_static_unused_arg(u32 arg, u32 unused)
754{

--- 31 unchanged lines hidden (view full) ---

786BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1)
787BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2)
788BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1)
789BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2)
790BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3)
791BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1)
792BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1)
793BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2)
763}
764
765__bpf_kfunc void bpf_kfunc_call_test_destructive(void)
766{
767}
768
769__bpf_kfunc static u32 bpf_kfunc_call_test_static_unused_arg(u32 arg, u32 unused)
770{

--- 31 unchanged lines hidden (view full) ---

802BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1)
803BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2)
804BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1)
805BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2)
806BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3)
807BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1)
808BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1)
809BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2)
794BTF_ID_FLAGS(func, bpf_kfunc_call_test_ref, KF_TRUSTED_ARGS)
810BTF_ID_FLAGS(func, bpf_kfunc_call_test_ref, KF_TRUSTED_ARGS | KF_RCU)
795BTF_ID_FLAGS(func, bpf_kfunc_call_test_destructive, KF_DESTRUCTIVE)
796BTF_ID_FLAGS(func, bpf_kfunc_call_test_static_unused_arg)
811BTF_ID_FLAGS(func, bpf_kfunc_call_test_destructive, KF_DESTRUCTIVE)
812BTF_ID_FLAGS(func, bpf_kfunc_call_test_static_unused_arg)
813BTF_ID_FLAGS(func, bpf_kfunc_call_test_offset)
797BTF_SET8_END(test_sk_check_kfunc_ids)
798
799static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
800 u32 size, u32 headroom, u32 tailroom)
801{
802 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
803 void *data;
804

--- 33 unchanged lines hidden (view full) ---

838 case BPF_TRACE_FEXIT:
839 if (bpf_fentry_test1(1) != 2 ||
840 bpf_fentry_test2(2, 3) != 5 ||
841 bpf_fentry_test3(4, 5, 6) != 15 ||
842 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
843 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
844 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
845 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
814BTF_SET8_END(test_sk_check_kfunc_ids)
815
816static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
817 u32 size, u32 headroom, u32 tailroom)
818{
819 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
820 void *data;
821

--- 33 unchanged lines hidden (view full) ---

855 case BPF_TRACE_FEXIT:
856 if (bpf_fentry_test1(1) != 2 ||
857 bpf_fentry_test2(2, 3) != 5 ||
858 bpf_fentry_test3(4, 5, 6) != 15 ||
859 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
860 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
861 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
862 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
846 bpf_fentry_test8(&arg) != 0)
863 bpf_fentry_test8(&arg) != 0 ||
864 bpf_fentry_test9(&retval) != 0)
847 goto out;
848 break;
849 case BPF_MODIFY_RETURN:
850 ret = bpf_modify_return_test(1, &b);
851 if (b != 2)
852 side_effect = 1;
853 break;
854 default:

--- 860 unchanged lines hidden --- 		865 goto out;
866 break;
867 case BPF_MODIFY_RETURN:
868 ret = bpf_modify_return_test(1, &b);
869 if (b != 2)
870 side_effect = 1;
871 break;
872 default:

--- 860 unchanged lines hidden ---