16ac22d03SDave Marchevsky // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
26ac22d03SDave Marchevsky
36ac22d03SDave Marchevsky #ifndef _GNU_SOURCE
46ac22d03SDave Marchevsky #define _GNU_SOURCE
56ac22d03SDave Marchevsky #endif
66ac22d03SDave Marchevsky
76ac22d03SDave Marchevsky #include <errno.h>
86ac22d03SDave Marchevsky #include <stdlib.h>
96ac22d03SDave Marchevsky #include <linux/err.h>
106ac22d03SDave Marchevsky #include <linux/kernel.h>
116ac22d03SDave Marchevsky #include <bpf/bpf.h>
126ac22d03SDave Marchevsky #include "bpf-utils.h"
136ac22d03SDave Marchevsky #include "debug.h"
146ac22d03SDave Marchevsky
156ac22d03SDave Marchevsky struct bpil_array_desc {
166ac22d03SDave Marchevsky int array_offset; /* e.g. offset of jited_prog_insns */
176ac22d03SDave Marchevsky int count_offset; /* e.g. offset of jited_prog_len */
186ac22d03SDave Marchevsky int size_offset; /* > 0: offset of rec size,
196ac22d03SDave Marchevsky * < 0: fix size of -size_offset
206ac22d03SDave Marchevsky */
216ac22d03SDave Marchevsky };
226ac22d03SDave Marchevsky
236ac22d03SDave Marchevsky static struct bpil_array_desc bpil_array_desc[] = {
246ac22d03SDave Marchevsky [PERF_BPIL_JITED_INSNS] = {
256ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, jited_prog_insns),
266ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, jited_prog_len),
276ac22d03SDave Marchevsky -1,
286ac22d03SDave Marchevsky },
296ac22d03SDave Marchevsky [PERF_BPIL_XLATED_INSNS] = {
306ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, xlated_prog_insns),
316ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, xlated_prog_len),
326ac22d03SDave Marchevsky -1,
336ac22d03SDave Marchevsky },
346ac22d03SDave Marchevsky [PERF_BPIL_MAP_IDS] = {
356ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, map_ids),
366ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, nr_map_ids),
376ac22d03SDave Marchevsky -(int)sizeof(__u32),
386ac22d03SDave Marchevsky },
396ac22d03SDave Marchevsky [PERF_BPIL_JITED_KSYMS] = {
406ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, jited_ksyms),
416ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, nr_jited_ksyms),
426ac22d03SDave Marchevsky -(int)sizeof(__u64),
436ac22d03SDave Marchevsky },
446ac22d03SDave Marchevsky [PERF_BPIL_JITED_FUNC_LENS] = {
456ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, jited_func_lens),
466ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, nr_jited_func_lens),
476ac22d03SDave Marchevsky -(int)sizeof(__u32),
486ac22d03SDave Marchevsky },
496ac22d03SDave Marchevsky [PERF_BPIL_FUNC_INFO] = {
506ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, func_info),
516ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, nr_func_info),
526ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, func_info_rec_size),
536ac22d03SDave Marchevsky },
546ac22d03SDave Marchevsky [PERF_BPIL_LINE_INFO] = {
556ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, line_info),
566ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, nr_line_info),
576ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, line_info_rec_size),
586ac22d03SDave Marchevsky },
596ac22d03SDave Marchevsky [PERF_BPIL_JITED_LINE_INFO] = {
606ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, jited_line_info),
616ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, nr_jited_line_info),
626ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, jited_line_info_rec_size),
636ac22d03SDave Marchevsky },
646ac22d03SDave Marchevsky [PERF_BPIL_PROG_TAGS] = {
656ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, prog_tags),
666ac22d03SDave Marchevsky offsetof(struct bpf_prog_info, nr_prog_tags),
676ac22d03SDave Marchevsky -(int)sizeof(__u8) * BPF_TAG_SIZE,
686ac22d03SDave Marchevsky },
696ac22d03SDave Marchevsky
706ac22d03SDave Marchevsky };
716ac22d03SDave Marchevsky
bpf_prog_info_read_offset_u32(struct bpf_prog_info * info,int offset)726ac22d03SDave Marchevsky static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
736ac22d03SDave Marchevsky int offset)
746ac22d03SDave Marchevsky {
756ac22d03SDave Marchevsky __u32 *array = (__u32 *)info;
766ac22d03SDave Marchevsky
776ac22d03SDave Marchevsky if (offset >= 0)
786ac22d03SDave Marchevsky return array[offset / sizeof(__u32)];
796ac22d03SDave Marchevsky return -(int)offset;
806ac22d03SDave Marchevsky }
816ac22d03SDave Marchevsky
bpf_prog_info_read_offset_u64(struct bpf_prog_info * info,int offset)826ac22d03SDave Marchevsky static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
836ac22d03SDave Marchevsky int offset)
846ac22d03SDave Marchevsky {
856ac22d03SDave Marchevsky __u64 *array = (__u64 *)info;
866ac22d03SDave Marchevsky
876ac22d03SDave Marchevsky if (offset >= 0)
886ac22d03SDave Marchevsky return array[offset / sizeof(__u64)];
896ac22d03SDave Marchevsky return -(int)offset;
906ac22d03SDave Marchevsky }
916ac22d03SDave Marchevsky
bpf_prog_info_set_offset_u32(struct bpf_prog_info * info,int offset,__u32 val)926ac22d03SDave Marchevsky static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
936ac22d03SDave Marchevsky __u32 val)
946ac22d03SDave Marchevsky {
956ac22d03SDave Marchevsky __u32 *array = (__u32 *)info;
966ac22d03SDave Marchevsky
976ac22d03SDave Marchevsky if (offset >= 0)
986ac22d03SDave Marchevsky array[offset / sizeof(__u32)] = val;
996ac22d03SDave Marchevsky }
1006ac22d03SDave Marchevsky
bpf_prog_info_set_offset_u64(struct bpf_prog_info * info,int offset,__u64 val)1016ac22d03SDave Marchevsky static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
1026ac22d03SDave Marchevsky __u64 val)
1036ac22d03SDave Marchevsky {
1046ac22d03SDave Marchevsky __u64 *array = (__u64 *)info;
1056ac22d03SDave Marchevsky
1066ac22d03SDave Marchevsky if (offset >= 0)
1076ac22d03SDave Marchevsky array[offset / sizeof(__u64)] = val;
1086ac22d03SDave Marchevsky }
1096ac22d03SDave Marchevsky
1106ac22d03SDave Marchevsky struct perf_bpil *
get_bpf_prog_info_linear(int fd,__u64 arrays)1116ac22d03SDave Marchevsky get_bpf_prog_info_linear(int fd, __u64 arrays)
1126ac22d03SDave Marchevsky {
1136ac22d03SDave Marchevsky struct bpf_prog_info info = {};
1146ac22d03SDave Marchevsky struct perf_bpil *info_linear;
1156ac22d03SDave Marchevsky __u32 info_len = sizeof(info);
1166ac22d03SDave Marchevsky __u32 data_len = 0;
1176ac22d03SDave Marchevsky int i, err;
1186ac22d03SDave Marchevsky void *ptr;
1196ac22d03SDave Marchevsky
1206ac22d03SDave Marchevsky if (arrays >> PERF_BPIL_LAST_ARRAY)
1216ac22d03SDave Marchevsky return ERR_PTR(-EINVAL);
1226ac22d03SDave Marchevsky
1236ac22d03SDave Marchevsky /* step 1: get array dimensions */
1246ac22d03SDave Marchevsky err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
1256ac22d03SDave Marchevsky if (err) {
1266ac22d03SDave Marchevsky pr_debug("can't get prog info: %s", strerror(errno));
1276ac22d03SDave Marchevsky return ERR_PTR(-EFAULT);
1286ac22d03SDave Marchevsky }
1296ac22d03SDave Marchevsky
1306ac22d03SDave Marchevsky /* step 2: calculate total size of all arrays */
1316ac22d03SDave Marchevsky for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) {
1326ac22d03SDave Marchevsky bool include_array = (arrays & (1UL << i)) > 0;
1336ac22d03SDave Marchevsky struct bpil_array_desc *desc;
1346ac22d03SDave Marchevsky __u32 count, size;
1356ac22d03SDave Marchevsky
1366ac22d03SDave Marchevsky desc = bpil_array_desc + i;
1376ac22d03SDave Marchevsky
1386ac22d03SDave Marchevsky /* kernel is too old to support this field */
1396ac22d03SDave Marchevsky if (info_len < desc->array_offset + sizeof(__u32) ||
1406ac22d03SDave Marchevsky info_len < desc->count_offset + sizeof(__u32) ||
1416ac22d03SDave Marchevsky (desc->size_offset > 0 && info_len < (__u32)desc->size_offset))
1426ac22d03SDave Marchevsky include_array = false;
1436ac22d03SDave Marchevsky
1446ac22d03SDave Marchevsky if (!include_array) {
1456ac22d03SDave Marchevsky arrays &= ~(1UL << i); /* clear the bit */
1466ac22d03SDave Marchevsky continue;
1476ac22d03SDave Marchevsky }
1486ac22d03SDave Marchevsky
1496ac22d03SDave Marchevsky count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
1506ac22d03SDave Marchevsky size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
1516ac22d03SDave Marchevsky
152*579d6c6dSIan Rogers data_len += roundup(count * size, sizeof(__u64));
1536ac22d03SDave Marchevsky }
1546ac22d03SDave Marchevsky
1556ac22d03SDave Marchevsky /* step 3: allocate continuous memory */
1566ac22d03SDave Marchevsky info_linear = malloc(sizeof(struct perf_bpil) + data_len);
1576ac22d03SDave Marchevsky if (!info_linear)
1586ac22d03SDave Marchevsky return ERR_PTR(-ENOMEM);
1596ac22d03SDave Marchevsky
1606ac22d03SDave Marchevsky /* step 4: fill data to info_linear->info */
1616ac22d03SDave Marchevsky info_linear->arrays = arrays;
1626ac22d03SDave Marchevsky memset(&info_linear->info, 0, sizeof(info));
1636ac22d03SDave Marchevsky ptr = info_linear->data;
1646ac22d03SDave Marchevsky
1656ac22d03SDave Marchevsky for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) {
1666ac22d03SDave Marchevsky struct bpil_array_desc *desc;
1676ac22d03SDave Marchevsky __u32 count, size;
1686ac22d03SDave Marchevsky
1696ac22d03SDave Marchevsky if ((arrays & (1UL << i)) == 0)
1706ac22d03SDave Marchevsky continue;
1716ac22d03SDave Marchevsky
1726ac22d03SDave Marchevsky desc = bpil_array_desc + i;
1736ac22d03SDave Marchevsky count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
1746ac22d03SDave Marchevsky size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
1756ac22d03SDave Marchevsky bpf_prog_info_set_offset_u32(&info_linear->info,
1766ac22d03SDave Marchevsky desc->count_offset, count);
1776ac22d03SDave Marchevsky bpf_prog_info_set_offset_u32(&info_linear->info,
1786ac22d03SDave Marchevsky desc->size_offset, size);
1796ac22d03SDave Marchevsky bpf_prog_info_set_offset_u64(&info_linear->info,
1806ac22d03SDave Marchevsky desc->array_offset,
1816ac22d03SDave Marchevsky ptr_to_u64(ptr));
182*579d6c6dSIan Rogers ptr += roundup(count * size, sizeof(__u64));
1836ac22d03SDave Marchevsky }
1846ac22d03SDave Marchevsky
1856ac22d03SDave Marchevsky /* step 5: call syscall again to get required arrays */
1866ac22d03SDave Marchevsky err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
1876ac22d03SDave Marchevsky if (err) {
1886ac22d03SDave Marchevsky pr_debug("can't get prog info: %s", strerror(errno));
1896ac22d03SDave Marchevsky free(info_linear);
1906ac22d03SDave Marchevsky return ERR_PTR(-EFAULT);
1916ac22d03SDave Marchevsky }
1926ac22d03SDave Marchevsky
1936ac22d03SDave Marchevsky /* step 6: verify the data */
1946ac22d03SDave Marchevsky for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) {
1956ac22d03SDave Marchevsky struct bpil_array_desc *desc;
1966ac22d03SDave Marchevsky __u32 v1, v2;
1976ac22d03SDave Marchevsky
1986ac22d03SDave Marchevsky if ((arrays & (1UL << i)) == 0)
1996ac22d03SDave Marchevsky continue;
2006ac22d03SDave Marchevsky
2016ac22d03SDave Marchevsky desc = bpil_array_desc + i;
2026ac22d03SDave Marchevsky v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
2036ac22d03SDave Marchevsky v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
2046ac22d03SDave Marchevsky desc->count_offset);
2056ac22d03SDave Marchevsky if (v1 != v2)
2066ac22d03SDave Marchevsky pr_warning("%s: mismatch in element count\n", __func__);
2076ac22d03SDave Marchevsky
2086ac22d03SDave Marchevsky v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
2096ac22d03SDave Marchevsky v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
2106ac22d03SDave Marchevsky desc->size_offset);
2116ac22d03SDave Marchevsky if (v1 != v2)
2126ac22d03SDave Marchevsky pr_warning("%s: mismatch in rec size\n", __func__);
2136ac22d03SDave Marchevsky }
2146ac22d03SDave Marchevsky
2156ac22d03SDave Marchevsky /* step 7: update info_len and data_len */
2166ac22d03SDave Marchevsky info_linear->info_len = sizeof(struct bpf_prog_info);
2176ac22d03SDave Marchevsky info_linear->data_len = data_len;
2186ac22d03SDave Marchevsky
2196ac22d03SDave Marchevsky return info_linear;
2206ac22d03SDave Marchevsky }
2216ac22d03SDave Marchevsky
bpil_addr_to_offs(struct perf_bpil * info_linear)2226ac22d03SDave Marchevsky void bpil_addr_to_offs(struct perf_bpil *info_linear)
2236ac22d03SDave Marchevsky {
2246ac22d03SDave Marchevsky int i;
2256ac22d03SDave Marchevsky
2266ac22d03SDave Marchevsky for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) {
2276ac22d03SDave Marchevsky struct bpil_array_desc *desc;
2286ac22d03SDave Marchevsky __u64 addr, offs;
2296ac22d03SDave Marchevsky
2306ac22d03SDave Marchevsky if ((info_linear->arrays & (1UL << i)) == 0)
2316ac22d03SDave Marchevsky continue;
2326ac22d03SDave Marchevsky
2336ac22d03SDave Marchevsky desc = bpil_array_desc + i;
2346ac22d03SDave Marchevsky addr = bpf_prog_info_read_offset_u64(&info_linear->info,
2356ac22d03SDave Marchevsky desc->array_offset);
2366ac22d03SDave Marchevsky offs = addr - ptr_to_u64(info_linear->data);
2376ac22d03SDave Marchevsky bpf_prog_info_set_offset_u64(&info_linear->info,
2386ac22d03SDave Marchevsky desc->array_offset, offs);
2396ac22d03SDave Marchevsky }
2406ac22d03SDave Marchevsky }
2416ac22d03SDave Marchevsky
bpil_offs_to_addr(struct perf_bpil * info_linear)2426ac22d03SDave Marchevsky void bpil_offs_to_addr(struct perf_bpil *info_linear)
2436ac22d03SDave Marchevsky {
2446ac22d03SDave Marchevsky int i;
2456ac22d03SDave Marchevsky
2466ac22d03SDave Marchevsky for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) {
2476ac22d03SDave Marchevsky struct bpil_array_desc *desc;
2486ac22d03SDave Marchevsky __u64 addr, offs;
2496ac22d03SDave Marchevsky
2506ac22d03SDave Marchevsky if ((info_linear->arrays & (1UL << i)) == 0)
2516ac22d03SDave Marchevsky continue;
2526ac22d03SDave Marchevsky
2536ac22d03SDave Marchevsky desc = bpil_array_desc + i;
2546ac22d03SDave Marchevsky offs = bpf_prog_info_read_offset_u64(&info_linear->info,
2556ac22d03SDave Marchevsky desc->array_offset);
2566ac22d03SDave Marchevsky addr = offs + ptr_to_u64(info_linear->data);
2576ac22d03SDave Marchevsky bpf_prog_info_set_offset_u64(&info_linear->info,
2586ac22d03SDave Marchevsky desc->array_offset, addr);
2596ac22d03SDave Marchevsky }
2606ac22d03SDave Marchevsky }
261