1#!/usr/bin/env python3
2# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
3#
4# Copyright (C) 2021 Isovalent, Inc.
5
6import argparse
7import re
8import os, sys
9
10LINUX_ROOT = os.path.abspath(os.path.join(__file__,
11    os.pardir, os.pardir, os.pardir, os.pardir, os.pardir))
12BPFTOOL_DIR = os.path.join(LINUX_ROOT, 'tools/bpf/bpftool')
13retval = 0
14
15class BlockParser(object):
16    """
17    A parser for extracting set of values from blocks such as enums.
18    @reader: a pointer to the open file to parse
19    """
20    def __init__(self, reader):
21        self.reader = reader
22
23    def search_block(self, start_marker):
24        """
25        Search for a given structure in a file.
26        @start_marker: regex marking the beginning of a structure to parse
27        """
28        offset = self.reader.tell()
29        array_start = re.search(start_marker, self.reader.read())
30        if array_start is None:
31            raise Exception('Failed to find start of block')
32        self.reader.seek(offset + array_start.start())
33
34    def parse(self, pattern, end_marker):
35        """
36        Parse a block and return a set of values. Values to extract must be
37        on separate lines in the file.
38        @pattern: pattern used to identify the values to extract
39        @end_marker: regex marking the end of the block to parse
40        """
41        entries = set()
42        while True:
43            line = self.reader.readline()
44            if not line or re.match(end_marker, line):
45                break
46            capture = pattern.search(line)
47            if capture and pattern.groups >= 1:
48                entries.add(capture.group(1))
49        return entries
50
51class ArrayParser(BlockParser):
52    """
53    A parser for extracting dicionaries of values from some BPF-related arrays.
54    @reader: a pointer to the open file to parse
55    @array_name: name of the array to parse
56    """
57    end_marker = re.compile('^};')
58
59    def __init__(self, reader, array_name):
60        self.array_name = array_name
61        self.start_marker = re.compile(f'(static )?const char \* const {self.array_name}\[.*\] = {{\n')
62        super().__init__(reader)
63
64    def search_block(self):
65        """
66        Search for the given array in a file.
67        """
68        super().search_block(self.start_marker);
69
70    def parse(self):
71        """
72        Parse a block and return data as a dictionary. Items to extract must be
73        on separate lines in the file.
74        """
75        pattern = re.compile('\[(BPF_\w*)\]\s*= "(.*)",?$')
76        entries = {}
77        while True:
78            line = self.reader.readline()
79            if line == '' or re.match(self.end_marker, line):
80                break
81            capture = pattern.search(line)
82            if capture:
83                entries[capture.group(1)] = capture.group(2)
84        return entries
85
86class InlineListParser(BlockParser):
87    """
88    A parser for extracting set of values from inline lists.
89    """
90    def parse(self, pattern, end_marker):
91        """
92        Parse a block and return a set of values. Multiple values to extract
93        can be on a same line in the file.
94        @pattern: pattern used to identify the values to extract
95        @end_marker: regex marking the end of the block to parse
96        """
97        entries = set()
98        while True:
99            line = self.reader.readline()
100            if not line:
101                break
102            entries.update(pattern.findall(line))
103            if re.search(end_marker, line):
104                break
105        return entries
106
107class FileExtractor(object):
108    """
109    A generic reader for extracting data from a given file. This class contains
110    several helper methods that wrap arround parser objects to extract values
111    from different structures.
112    This class does not offer a way to set a filename, which is expected to be
113    defined in children classes.
114    """
115    def __init__(self):
116        self.reader = open(self.filename, 'r')
117
118    def close(self):
119        """
120        Close the file used by the parser.
121        """
122        self.reader.close()
123
124    def reset_read(self):
125        """
126        Reset the file position indicator for this parser. This is useful when
127        parsing several structures in the file without respecting the order in
128        which those structures appear in the file.
129        """
130        self.reader.seek(0)
131
132    def get_types_from_array(self, array_name):
133        """
134        Search for and parse an array associating names to BPF_* enum members,
135        for example:
136
137            const char * const prog_type_name[] = {
138                    [BPF_PROG_TYPE_UNSPEC]                  = "unspec",
139                    [BPF_PROG_TYPE_SOCKET_FILTER]           = "socket_filter",
140                    [BPF_PROG_TYPE_KPROBE]                  = "kprobe",
141            };
142
143        Return a dictionary with the enum member names as keys and the
144        associated names as values, for example:
145
146            {'BPF_PROG_TYPE_UNSPEC': 'unspec',
147             'BPF_PROG_TYPE_SOCKET_FILTER': 'socket_filter',
148             'BPF_PROG_TYPE_KPROBE': 'kprobe'}
149
150        @array_name: name of the array to parse
151        """
152        array_parser = ArrayParser(self.reader, array_name)
153        array_parser.search_block()
154        return array_parser.parse()
155
156    def get_enum(self, enum_name):
157        """
158        Search for and parse an enum containing BPF_* members, for example:
159
160            enum bpf_prog_type {
161                    BPF_PROG_TYPE_UNSPEC,
162                    BPF_PROG_TYPE_SOCKET_FILTER,
163                    BPF_PROG_TYPE_KPROBE,
164            };
165
166        Return a set containing all member names, for example:
167
168            {'BPF_PROG_TYPE_UNSPEC',
169             'BPF_PROG_TYPE_SOCKET_FILTER',
170             'BPF_PROG_TYPE_KPROBE'}
171
172        @enum_name: name of the enum to parse
173        """
174        start_marker = re.compile(f'enum {enum_name} {{\n')
175        pattern = re.compile('^\s*(BPF_\w+),?$')
176        end_marker = re.compile('^};')
177        parser = BlockParser(self.reader)
178        parser.search_block(start_marker)
179        return parser.parse(pattern, end_marker)
180
181    def __get_description_list(self, start_marker, pattern, end_marker):
182        parser = InlineListParser(self.reader)
183        parser.search_block(start_marker)
184        return parser.parse(pattern, end_marker)
185
186    def get_rst_list(self, block_name):
187        """
188        Search for and parse a list of type names from RST documentation, for
189        example:
190
191             |       *TYPE* := {
192             |               **socket** | **kprobe** |
193             |               **kretprobe**
194             |       }
195
196        Return a set containing all type names, for example:
197
198            {'socket', 'kprobe', 'kretprobe'}
199
200        @block_name: name of the blog to parse, 'TYPE' in the example
201        """
202        start_marker = re.compile(f'\*{block_name}\* := {{')
203        pattern = re.compile('\*\*([\w/-]+)\*\*')
204        end_marker = re.compile('}\n')
205        return self.__get_description_list(start_marker, pattern, end_marker)
206
207    def get_help_list(self, block_name):
208        """
209        Search for and parse a list of type names from a help message in
210        bpftool, for example:
211
212            "       TYPE := { socket | kprobe |\\n"
213            "               kretprobe }\\n"
214
215        Return a set containing all type names, for example:
216
217            {'socket', 'kprobe', 'kretprobe'}
218
219        @block_name: name of the blog to parse, 'TYPE' in the example
220        """
221        start_marker = re.compile(f'"\s*{block_name} := {{')
222        pattern = re.compile('([\w/]+) [|}]')
223        end_marker = re.compile('}')
224        return self.__get_description_list(start_marker, pattern, end_marker)
225
226    def get_help_list_macro(self, macro):
227        """
228        Search for and parse a list of values from a help message starting with
229        a macro in bpftool, for example:
230
231            "       " HELP_SPEC_OPTIONS " |\\n"
232            "                    {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} }\\n"
233
234        Return a set containing all item names, for example:
235
236            {'-f', '--bpffs', '-m', '--mapcompat', '-n', '--nomount'}
237
238        @macro: macro starting the block, 'HELP_SPEC_OPTIONS' in the example
239        """
240        start_marker = re.compile(f'"\s*{macro}\s*" [|}}]')
241        pattern = re.compile('([\w-]+) ?(?:\||}[ }\]])')
242        end_marker = re.compile('}\\\\n')
243        return self.__get_description_list(start_marker, pattern, end_marker)
244
245    def default_options(self):
246        """
247        Return the default options contained in HELP_SPEC_OPTIONS
248        """
249        return { '-j', '--json', '-p', '--pretty', '-d', '--debug' }
250
251    def get_bashcomp_list(self, block_name):
252        """
253        Search for and parse a list of type names from a variable in bash
254        completion file, for example:
255
256            local BPFTOOL_PROG_LOAD_TYPES='socket kprobe \\
257                kretprobe'
258
259        Return a set containing all type names, for example:
260
261            {'socket', 'kprobe', 'kretprobe'}
262
263        @block_name: name of the blog to parse, 'TYPE' in the example
264        """
265        start_marker = re.compile(f'local {block_name}=\'')
266        pattern = re.compile('(?:.*=\')?([\w/]+)')
267        end_marker = re.compile('\'$')
268        return self.__get_description_list(start_marker, pattern, end_marker)
269
270class SourceFileExtractor(FileExtractor):
271    """
272    An abstract extractor for a source file with usage message.
273    This class does not offer a way to set a filename, which is expected to be
274    defined in children classes.
275    """
276    def get_options(self):
277        return self.default_options().union(self.get_help_list_macro('HELP_SPEC_OPTIONS'))
278
279class ProgFileExtractor(SourceFileExtractor):
280    """
281    An extractor for bpftool's prog.c.
282    """
283    filename = os.path.join(BPFTOOL_DIR, 'prog.c')
284
285    def get_prog_types(self):
286        return self.get_types_from_array('prog_type_name')
287
288    def get_attach_types(self):
289        return self.get_types_from_array('attach_type_strings')
290
291    def get_prog_attach_help(self):
292        return self.get_help_list('ATTACH_TYPE')
293
294class MapFileExtractor(SourceFileExtractor):
295    """
296    An extractor for bpftool's map.c.
297    """
298    filename = os.path.join(BPFTOOL_DIR, 'map.c')
299
300    def get_map_types(self):
301        return self.get_types_from_array('map_type_name')
302
303    def get_map_help(self):
304        return self.get_help_list('TYPE')
305
306class CgroupFileExtractor(SourceFileExtractor):
307    """
308    An extractor for bpftool's cgroup.c.
309    """
310    filename = os.path.join(BPFTOOL_DIR, 'cgroup.c')
311
312    def get_prog_attach_help(self):
313        return self.get_help_list('ATTACH_TYPE')
314
315class CommonFileExtractor(SourceFileExtractor):
316    """
317    An extractor for bpftool's common.c.
318    """
319    filename = os.path.join(BPFTOOL_DIR, 'common.c')
320
321    def __init__(self):
322        super().__init__()
323        self.attach_types = {}
324
325    def get_attach_types(self):
326        if not self.attach_types:
327            self.attach_types = self.get_types_from_array('attach_type_name')
328        return self.attach_types
329
330    def get_cgroup_attach_types(self):
331        if not self.attach_types:
332            self.get_attach_types()
333        cgroup_types = {}
334        for (key, value) in self.attach_types.items():
335            if key.find('BPF_CGROUP') != -1:
336                cgroup_types[key] = value
337        return cgroup_types
338
339class GenericSourceExtractor(SourceFileExtractor):
340    """
341    An extractor for generic source code files.
342    """
343    filename = ""
344
345    def __init__(self, filename):
346        self.filename = os.path.join(BPFTOOL_DIR, filename)
347        super().__init__()
348
349class BpfHeaderExtractor(FileExtractor):
350    """
351    An extractor for the UAPI BPF header.
352    """
353    filename = os.path.join(LINUX_ROOT, 'tools/include/uapi/linux/bpf.h')
354
355    def get_prog_types(self):
356        return self.get_enum('bpf_prog_type')
357
358    def get_map_types(self):
359        return self.get_enum('bpf_map_type')
360
361    def get_attach_types(self):
362        return self.get_enum('bpf_attach_type')
363
364class ManPageExtractor(FileExtractor):
365    """
366    An abstract extractor for an RST documentation page.
367    This class does not offer a way to set a filename, which is expected to be
368    defined in children classes.
369    """
370    def get_options(self):
371        return self.get_rst_list('OPTIONS')
372
373class ManProgExtractor(ManPageExtractor):
374    """
375    An extractor for bpftool-prog.rst.
376    """
377    filename = os.path.join(BPFTOOL_DIR, 'Documentation/bpftool-prog.rst')
378
379    def get_attach_types(self):
380        return self.get_rst_list('ATTACH_TYPE')
381
382class ManMapExtractor(ManPageExtractor):
383    """
384    An extractor for bpftool-map.rst.
385    """
386    filename = os.path.join(BPFTOOL_DIR, 'Documentation/bpftool-map.rst')
387
388    def get_map_types(self):
389        return self.get_rst_list('TYPE')
390
391class ManCgroupExtractor(ManPageExtractor):
392    """
393    An extractor for bpftool-cgroup.rst.
394    """
395    filename = os.path.join(BPFTOOL_DIR, 'Documentation/bpftool-cgroup.rst')
396
397    def get_attach_types(self):
398        return self.get_rst_list('ATTACH_TYPE')
399
400class ManGenericExtractor(ManPageExtractor):
401    """
402    An extractor for generic RST documentation pages.
403    """
404    filename = ""
405
406    def __init__(self, filename):
407        self.filename = os.path.join(BPFTOOL_DIR, filename)
408        super().__init__()
409
410class BashcompExtractor(FileExtractor):
411    """
412    An extractor for bpftool's bash completion file.
413    """
414    filename = os.path.join(BPFTOOL_DIR, 'bash-completion/bpftool')
415
416    def get_prog_attach_types(self):
417        return self.get_bashcomp_list('BPFTOOL_PROG_ATTACH_TYPES')
418
419    def get_map_types(self):
420        return self.get_bashcomp_list('BPFTOOL_MAP_CREATE_TYPES')
421
422    def get_cgroup_attach_types(self):
423        return self.get_bashcomp_list('BPFTOOL_CGROUP_ATTACH_TYPES')
424
425def verify(first_set, second_set, message):
426    """
427    Print all values that differ between two sets.
428    @first_set: one set to compare
429    @second_set: another set to compare
430    @message: message to print for values belonging to only one of the sets
431    """
432    global retval
433    diff = first_set.symmetric_difference(second_set)
434    if diff:
435        print(message, diff)
436        retval = 1
437
438def main():
439    # No arguments supported at this time, but print usage for -h|--help
440    argParser = argparse.ArgumentParser(description="""
441    Verify that bpftool's code, help messages, documentation and bash
442    completion are all in sync on program types, map types, attach types, and
443    options. Also check that bpftool is in sync with the UAPI BPF header.
444    """)
445    args = argParser.parse_args()
446
447    # Map types (enum)
448
449    bpf_info = BpfHeaderExtractor()
450    ref = bpf_info.get_map_types()
451
452    map_info = MapFileExtractor()
453    source_map_items = map_info.get_map_types()
454    map_types_enum = set(source_map_items.keys())
455
456    verify(ref, map_types_enum,
457            f'Comparing BPF header (enum bpf_map_type) and {MapFileExtractor.filename} (map_type_name):')
458
459    # Map types (names)
460
461    source_map_types = set(source_map_items.values())
462    source_map_types.discard('unspec')
463
464    help_map_types = map_info.get_map_help()
465    help_map_options = map_info.get_options()
466    map_info.close()
467
468    man_map_info = ManMapExtractor()
469    man_map_options = man_map_info.get_options()
470    man_map_types = man_map_info.get_map_types()
471    man_map_info.close()
472
473    bashcomp_info = BashcompExtractor()
474    bashcomp_map_types = bashcomp_info.get_map_types()
475
476    verify(source_map_types, help_map_types,
477            f'Comparing {MapFileExtractor.filename} (map_type_name) and {MapFileExtractor.filename} (do_help() TYPE):')
478    verify(source_map_types, man_map_types,
479            f'Comparing {MapFileExtractor.filename} (map_type_name) and {ManMapExtractor.filename} (TYPE):')
480    verify(help_map_options, man_map_options,
481            f'Comparing {MapFileExtractor.filename} (do_help() OPTIONS) and {ManMapExtractor.filename} (OPTIONS):')
482    verify(source_map_types, bashcomp_map_types,
483            f'Comparing {MapFileExtractor.filename} (map_type_name) and {BashcompExtractor.filename} (BPFTOOL_MAP_CREATE_TYPES):')
484
485    # Program types (enum)
486
487    ref = bpf_info.get_prog_types()
488
489    prog_info = ProgFileExtractor()
490    prog_types = set(prog_info.get_prog_types().keys())
491
492    verify(ref, prog_types,
493            f'Comparing BPF header (enum bpf_prog_type) and {ProgFileExtractor.filename} (prog_type_name):')
494
495    # Attach types (enum)
496
497    ref = bpf_info.get_attach_types()
498    bpf_info.close()
499
500    common_info = CommonFileExtractor()
501    attach_types = common_info.get_attach_types()
502
503    verify(ref, attach_types,
504            f'Comparing BPF header (enum bpf_attach_type) and {CommonFileExtractor.filename} (attach_type_name):')
505
506    # Attach types (names)
507
508    source_prog_attach_types = set(prog_info.get_attach_types().values())
509
510    help_prog_attach_types = prog_info.get_prog_attach_help()
511    help_prog_options = prog_info.get_options()
512    prog_info.close()
513
514    man_prog_info = ManProgExtractor()
515    man_prog_options = man_prog_info.get_options()
516    man_prog_attach_types = man_prog_info.get_attach_types()
517    man_prog_info.close()
518
519    bashcomp_info.reset_read() # We stopped at map types, rewind
520    bashcomp_prog_attach_types = bashcomp_info.get_prog_attach_types()
521
522    verify(source_prog_attach_types, help_prog_attach_types,
523            f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ProgFileExtractor.filename} (do_help() ATTACH_TYPE):')
524    verify(source_prog_attach_types, man_prog_attach_types,
525            f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ManProgExtractor.filename} (ATTACH_TYPE):')
526    verify(help_prog_options, man_prog_options,
527            f'Comparing {ProgFileExtractor.filename} (do_help() OPTIONS) and {ManProgExtractor.filename} (OPTIONS):')
528    verify(source_prog_attach_types, bashcomp_prog_attach_types,
529            f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_PROG_ATTACH_TYPES):')
530
531    # Cgroup attach types
532
533    source_cgroup_attach_types = set(common_info.get_cgroup_attach_types().values())
534    common_info.close()
535
536    cgroup_info = CgroupFileExtractor()
537    help_cgroup_attach_types = cgroup_info.get_prog_attach_help()
538    help_cgroup_options = cgroup_info.get_options()
539    cgroup_info.close()
540
541    man_cgroup_info = ManCgroupExtractor()
542    man_cgroup_options = man_cgroup_info.get_options()
543    man_cgroup_attach_types = man_cgroup_info.get_attach_types()
544    man_cgroup_info.close()
545
546    bashcomp_cgroup_attach_types = bashcomp_info.get_cgroup_attach_types()
547    bashcomp_info.close()
548
549    verify(source_cgroup_attach_types, help_cgroup_attach_types,
550            f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {CgroupFileExtractor.filename} (do_help() ATTACH_TYPE):')
551    verify(source_cgroup_attach_types, man_cgroup_attach_types,
552            f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {ManCgroupExtractor.filename} (ATTACH_TYPE):')
553    verify(help_cgroup_options, man_cgroup_options,
554            f'Comparing {CgroupFileExtractor.filename} (do_help() OPTIONS) and {ManCgroupExtractor.filename} (OPTIONS):')
555    verify(source_cgroup_attach_types, bashcomp_cgroup_attach_types,
556            f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_CGROUP_ATTACH_TYPES):')
557
558    # Options for remaining commands
559
560    for cmd in [ 'btf', 'feature', 'gen', 'iter', 'link', 'net', 'perf', 'struct_ops', ]:
561        source_info = GenericSourceExtractor(cmd + '.c')
562        help_cmd_options = source_info.get_options()
563        source_info.close()
564
565        man_cmd_info = ManGenericExtractor(os.path.join('Documentation', 'bpftool-' + cmd + '.rst'))
566        man_cmd_options = man_cmd_info.get_options()
567        man_cmd_info.close()
568
569        verify(help_cmd_options, man_cmd_options,
570                f'Comparing {source_info.filename} (do_help() OPTIONS) and {man_cmd_info.filename} (OPTIONS):')
571
572    source_main_info = GenericSourceExtractor('main.c')
573    help_main_options = source_main_info.get_options()
574    source_main_info.close()
575
576    man_main_info = ManGenericExtractor(os.path.join('Documentation', 'bpftool.rst'))
577    man_main_options = man_main_info.get_options()
578    man_main_info.close()
579
580    verify(help_main_options, man_main_options,
581            f'Comparing {source_main_info.filename} (do_help() OPTIONS) and {man_main_info.filename} (OPTIONS):')
582
583    sys.exit(retval)
584
585if __name__ == "__main__":
586    main()
587