1#!/usr/bin/env python3 2# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 3# 4# Copyright (C) 2021 Isovalent, Inc. 5 6import argparse 7import re 8import os, sys 9 10LINUX_ROOT = os.path.abspath(os.path.join(__file__, 11 os.pardir, os.pardir, os.pardir, os.pardir, os.pardir)) 12BPFTOOL_DIR = os.getenv('BPFTOOL_DIR', 13 os.path.join(LINUX_ROOT, 'tools/bpf/bpftool')) 14BPFTOOL_BASHCOMP_DIR = os.getenv('BPFTOOL_BASHCOMP_DIR', 15 os.path.join(BPFTOOL_DIR, 'bash-completion')) 16BPFTOOL_DOC_DIR = os.getenv('BPFTOOL_DOC_DIR', 17 os.path.join(BPFTOOL_DIR, 'Documentation')) 18INCLUDE_DIR = os.getenv('INCLUDE_DIR', 19 os.path.join(LINUX_ROOT, 'tools/include')) 20 21retval = 0 22 23class BlockParser(object): 24 """ 25 A parser for extracting set of values from blocks such as enums. 26 @reader: a pointer to the open file to parse 27 """ 28 def __init__(self, reader): 29 self.reader = reader 30 31 def search_block(self, start_marker): 32 """ 33 Search for a given structure in a file. 34 @start_marker: regex marking the beginning of a structure to parse 35 """ 36 offset = self.reader.tell() 37 array_start = re.search(start_marker, self.reader.read()) 38 if array_start is None: 39 raise Exception('Failed to find start of block') 40 self.reader.seek(offset + array_start.start()) 41 42 def parse(self, pattern, end_marker): 43 """ 44 Parse a block and return a set of values. Values to extract must be 45 on separate lines in the file. 46 @pattern: pattern used to identify the values to extract 47 @end_marker: regex marking the end of the block to parse 48 """ 49 entries = set() 50 while True: 51 line = self.reader.readline() 52 if not line or re.match(end_marker, line): 53 break 54 capture = pattern.search(line) 55 if capture and pattern.groups >= 1: 56 entries.add(capture.group(1)) 57 return entries 58 59class ArrayParser(BlockParser): 60 """ 61 A parser for extracting a set of values from some BPF-related arrays. 62 @reader: a pointer to the open file to parse 63 @array_name: name of the array to parse 64 """ 65 end_marker = re.compile('^};') 66 67 def __init__(self, reader, array_name): 68 self.array_name = array_name 69 self.start_marker = re.compile(f'(static )?const bool {self.array_name}\[.*\] = {{\n') 70 super().__init__(reader) 71 72 def search_block(self): 73 """ 74 Search for the given array in a file. 75 """ 76 super().search_block(self.start_marker); 77 78 def parse(self): 79 """ 80 Parse a block and return data as a dictionary. Items to extract must be 81 on separate lines in the file. 82 """ 83 pattern = re.compile('\[(BPF_\w*)\]\s*= (true|false),?$') 84 entries = set() 85 while True: 86 line = self.reader.readline() 87 if line == '' or re.match(self.end_marker, line): 88 break 89 capture = pattern.search(line) 90 if capture: 91 entries |= {capture.group(1)} 92 return entries 93 94class InlineListParser(BlockParser): 95 """ 96 A parser for extracting set of values from inline lists. 97 """ 98 def parse(self, pattern, end_marker): 99 """ 100 Parse a block and return a set of values. Multiple values to extract 101 can be on a same line in the file. 102 @pattern: pattern used to identify the values to extract 103 @end_marker: regex marking the end of the block to parse 104 """ 105 entries = set() 106 while True: 107 line = self.reader.readline() 108 if not line: 109 break 110 entries.update(pattern.findall(line)) 111 if re.search(end_marker, line): 112 break 113 return entries 114 115class FileExtractor(object): 116 """ 117 A generic reader for extracting data from a given file. This class contains 118 several helper methods that wrap around parser objects to extract values 119 from different structures. 120 This class does not offer a way to set a filename, which is expected to be 121 defined in children classes. 122 """ 123 def __init__(self): 124 self.reader = open(self.filename, 'r') 125 126 def close(self): 127 """ 128 Close the file used by the parser. 129 """ 130 self.reader.close() 131 132 def reset_read(self): 133 """ 134 Reset the file position indicator for this parser. This is useful when 135 parsing several structures in the file without respecting the order in 136 which those structures appear in the file. 137 """ 138 self.reader.seek(0) 139 140 def get_types_from_array(self, array_name): 141 """ 142 Search for and parse a list of allowed BPF_* enum members, for example: 143 144 const bool prog_type_name[] = { 145 [BPF_PROG_TYPE_UNSPEC] = true, 146 [BPF_PROG_TYPE_SOCKET_FILTER] = true, 147 [BPF_PROG_TYPE_KPROBE] = true, 148 }; 149 150 Return a set of the enum members, for example: 151 152 {'BPF_PROG_TYPE_UNSPEC', 153 'BPF_PROG_TYPE_SOCKET_FILTER', 154 'BPF_PROG_TYPE_KPROBE'} 155 156 @array_name: name of the array to parse 157 """ 158 array_parser = ArrayParser(self.reader, array_name) 159 array_parser.search_block() 160 return array_parser.parse() 161 162 def get_enum(self, enum_name): 163 """ 164 Search for and parse an enum containing BPF_* members, for example: 165 166 enum bpf_prog_type { 167 BPF_PROG_TYPE_UNSPEC, 168 BPF_PROG_TYPE_SOCKET_FILTER, 169 BPF_PROG_TYPE_KPROBE, 170 }; 171 172 Return a set containing all member names, for example: 173 174 {'BPF_PROG_TYPE_UNSPEC', 175 'BPF_PROG_TYPE_SOCKET_FILTER', 176 'BPF_PROG_TYPE_KPROBE'} 177 178 @enum_name: name of the enum to parse 179 """ 180 start_marker = re.compile(f'enum {enum_name} {{\n') 181 pattern = re.compile('^\s*(BPF_\w+),?(\s+/\*.*\*/)?$') 182 end_marker = re.compile('^};') 183 parser = BlockParser(self.reader) 184 parser.search_block(start_marker) 185 return parser.parse(pattern, end_marker) 186 187 def make_enum_map(self, names, enum_prefix): 188 """ 189 Search for and parse an enum containing BPF_* members, just as get_enum 190 does. However, instead of just returning a set of the variant names, 191 also generate a textual representation from them by (assuming and) 192 removing a provided prefix and lowercasing the remainder. Then return a 193 dict mapping from name to textual representation. 194 195 @enum_values: a set of enum values; e.g., as retrieved by get_enum 196 @enum_prefix: the prefix to remove from each of the variants to infer 197 textual representation 198 """ 199 mapping = {} 200 for name in names: 201 if not name.startswith(enum_prefix): 202 raise Exception(f"enum variant {name} does not start with {enum_prefix}") 203 text = name[len(enum_prefix):].lower() 204 mapping[name] = text 205 206 return mapping 207 208 def __get_description_list(self, start_marker, pattern, end_marker): 209 parser = InlineListParser(self.reader) 210 parser.search_block(start_marker) 211 return parser.parse(pattern, end_marker) 212 213 def get_rst_list(self, block_name): 214 """ 215 Search for and parse a list of type names from RST documentation, for 216 example: 217 218 | *TYPE* := { 219 | **socket** | **kprobe** | 220 | **kretprobe** 221 | } 222 223 Return a set containing all type names, for example: 224 225 {'socket', 'kprobe', 'kretprobe'} 226 227 @block_name: name of the blog to parse, 'TYPE' in the example 228 """ 229 start_marker = re.compile(f'\*{block_name}\* := {{') 230 pattern = re.compile('\*\*([\w/-]+)\*\*') 231 end_marker = re.compile('}\n') 232 return self.__get_description_list(start_marker, pattern, end_marker) 233 234 def get_help_list(self, block_name): 235 """ 236 Search for and parse a list of type names from a help message in 237 bpftool, for example: 238 239 " TYPE := { socket | kprobe |\\n" 240 " kretprobe }\\n" 241 242 Return a set containing all type names, for example: 243 244 {'socket', 'kprobe', 'kretprobe'} 245 246 @block_name: name of the blog to parse, 'TYPE' in the example 247 """ 248 start_marker = re.compile(f'"\s*{block_name} := {{') 249 pattern = re.compile('([\w/]+) [|}]') 250 end_marker = re.compile('}') 251 return self.__get_description_list(start_marker, pattern, end_marker) 252 253 def get_help_list_macro(self, macro): 254 """ 255 Search for and parse a list of values from a help message starting with 256 a macro in bpftool, for example: 257 258 " " HELP_SPEC_OPTIONS " |\\n" 259 " {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} }\\n" 260 261 Return a set containing all item names, for example: 262 263 {'-f', '--bpffs', '-m', '--mapcompat', '-n', '--nomount'} 264 265 @macro: macro starting the block, 'HELP_SPEC_OPTIONS' in the example 266 """ 267 start_marker = re.compile(f'"\s*{macro}\s*" [|}}]') 268 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]])') 269 end_marker = re.compile('}\\\\n') 270 return self.__get_description_list(start_marker, pattern, end_marker) 271 272 def get_bashcomp_list(self, block_name): 273 """ 274 Search for and parse a list of type names from a variable in bash 275 completion file, for example: 276 277 local BPFTOOL_PROG_LOAD_TYPES='socket kprobe \\ 278 kretprobe' 279 280 Return a set containing all type names, for example: 281 282 {'socket', 'kprobe', 'kretprobe'} 283 284 @block_name: name of the blog to parse, 'TYPE' in the example 285 """ 286 start_marker = re.compile(f'local {block_name}=\'') 287 pattern = re.compile('(?:.*=\')?([\w/]+)') 288 end_marker = re.compile('\'$') 289 return self.__get_description_list(start_marker, pattern, end_marker) 290 291class SourceFileExtractor(FileExtractor): 292 """ 293 An abstract extractor for a source file with usage message. 294 This class does not offer a way to set a filename, which is expected to be 295 defined in children classes. 296 """ 297 def get_options(self): 298 return self.get_help_list_macro('HELP_SPEC_OPTIONS') 299 300class MainHeaderFileExtractor(SourceFileExtractor): 301 """ 302 An extractor for bpftool's main.h 303 """ 304 filename = os.path.join(BPFTOOL_DIR, 'main.h') 305 306 def get_common_options(self): 307 """ 308 Parse the list of common options in main.h (options that apply to all 309 commands), which looks to the lists of options in other source files 310 but has different start and end markers: 311 312 "OPTIONS := { {-j|--json} [{-p|--pretty}] | {-d|--debug}" 313 314 Return a set containing all options, such as: 315 316 {'-p', '-d', '--pretty', '--debug', '--json', '-j'} 317 """ 318 start_marker = re.compile(f'"OPTIONS :=') 319 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]"])') 320 end_marker = re.compile('#define') 321 322 parser = InlineListParser(self.reader) 323 parser.search_block(start_marker) 324 return parser.parse(pattern, end_marker) 325 326class ManSubstitutionsExtractor(SourceFileExtractor): 327 """ 328 An extractor for substitutions.rst 329 """ 330 filename = os.path.join(BPFTOOL_DOC_DIR, 'substitutions.rst') 331 332 def get_common_options(self): 333 """ 334 Parse the list of common options in substitutions.rst (options that 335 apply to all commands). 336 337 Return a set containing all options, such as: 338 339 {'-p', '-d', '--pretty', '--debug', '--json', '-j'} 340 """ 341 start_marker = re.compile('\|COMMON_OPTIONS\| replace:: {') 342 pattern = re.compile('\*\*([\w/-]+)\*\*') 343 end_marker = re.compile('}$') 344 345 parser = InlineListParser(self.reader) 346 parser.search_block(start_marker) 347 return parser.parse(pattern, end_marker) 348 349class ProgFileExtractor(SourceFileExtractor): 350 """ 351 An extractor for bpftool's prog.c. 352 """ 353 filename = os.path.join(BPFTOOL_DIR, 'prog.c') 354 355 def get_attach_types(self): 356 types = self.get_types_from_array('attach_types') 357 return self.make_enum_map(types, 'BPF_') 358 359 def get_prog_attach_help(self): 360 return self.get_help_list('ATTACH_TYPE') 361 362class MapFileExtractor(SourceFileExtractor): 363 """ 364 An extractor for bpftool's map.c. 365 """ 366 filename = os.path.join(BPFTOOL_DIR, 'map.c') 367 368 def get_map_help(self): 369 return self.get_help_list('TYPE') 370 371class CgroupFileExtractor(SourceFileExtractor): 372 """ 373 An extractor for bpftool's cgroup.c. 374 """ 375 filename = os.path.join(BPFTOOL_DIR, 'cgroup.c') 376 377 def get_prog_attach_help(self): 378 return self.get_help_list('ATTACH_TYPE') 379 380class GenericSourceExtractor(SourceFileExtractor): 381 """ 382 An extractor for generic source code files. 383 """ 384 filename = "" 385 386 def __init__(self, filename): 387 self.filename = os.path.join(BPFTOOL_DIR, filename) 388 super().__init__() 389 390class BpfHeaderExtractor(FileExtractor): 391 """ 392 An extractor for the UAPI BPF header. 393 """ 394 filename = os.path.join(INCLUDE_DIR, 'uapi/linux/bpf.h') 395 396 def __init__(self): 397 super().__init__() 398 self.attach_types = {} 399 400 def get_prog_types(self): 401 return self.get_enum('bpf_prog_type') 402 403 def get_map_type_map(self): 404 names = self.get_enum('bpf_map_type') 405 return self.make_enum_map(names, 'BPF_MAP_TYPE_') 406 407 def get_attach_type_map(self): 408 if not self.attach_types: 409 names = self.get_enum('bpf_attach_type') 410 self.attach_types = self.make_enum_map(names, 'BPF_') 411 return self.attach_types 412 413 def get_cgroup_attach_type_map(self): 414 if not self.attach_types: 415 self.get_attach_type_map() 416 return {name: text for name, text in self.attach_types.items() 417 if name.startswith('BPF_CGROUP')} 418 419class ManPageExtractor(FileExtractor): 420 """ 421 An abstract extractor for an RST documentation page. 422 This class does not offer a way to set a filename, which is expected to be 423 defined in children classes. 424 """ 425 def get_options(self): 426 return self.get_rst_list('OPTIONS') 427 428class ManProgExtractor(ManPageExtractor): 429 """ 430 An extractor for bpftool-prog.rst. 431 """ 432 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-prog.rst') 433 434 def get_attach_types(self): 435 return self.get_rst_list('ATTACH_TYPE') 436 437class ManMapExtractor(ManPageExtractor): 438 """ 439 An extractor for bpftool-map.rst. 440 """ 441 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-map.rst') 442 443 def get_map_types(self): 444 return self.get_rst_list('TYPE') 445 446class ManCgroupExtractor(ManPageExtractor): 447 """ 448 An extractor for bpftool-cgroup.rst. 449 """ 450 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-cgroup.rst') 451 452 def get_attach_types(self): 453 return self.get_rst_list('ATTACH_TYPE') 454 455class ManGenericExtractor(ManPageExtractor): 456 """ 457 An extractor for generic RST documentation pages. 458 """ 459 filename = "" 460 461 def __init__(self, filename): 462 self.filename = os.path.join(BPFTOOL_DIR, filename) 463 super().__init__() 464 465class BashcompExtractor(FileExtractor): 466 """ 467 An extractor for bpftool's bash completion file. 468 """ 469 filename = os.path.join(BPFTOOL_BASHCOMP_DIR, 'bpftool') 470 471 def get_prog_attach_types(self): 472 return self.get_bashcomp_list('BPFTOOL_PROG_ATTACH_TYPES') 473 474def verify(first_set, second_set, message): 475 """ 476 Print all values that differ between two sets. 477 @first_set: one set to compare 478 @second_set: another set to compare 479 @message: message to print for values belonging to only one of the sets 480 """ 481 global retval 482 diff = first_set.symmetric_difference(second_set) 483 if diff: 484 print(message, diff) 485 retval = 1 486 487def main(): 488 # No arguments supported at this time, but print usage for -h|--help 489 argParser = argparse.ArgumentParser(description=""" 490 Verify that bpftool's code, help messages, documentation and bash 491 completion are all in sync on program types, map types, attach types, and 492 options. Also check that bpftool is in sync with the UAPI BPF header. 493 """) 494 args = argParser.parse_args() 495 496 bpf_info = BpfHeaderExtractor() 497 498 # Map types (names) 499 500 map_info = MapFileExtractor() 501 source_map_types = set(bpf_info.get_map_type_map().values()) 502 source_map_types.discard('unspec') 503 504 # BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED and BPF_MAP_TYPE_CGROUP_STORAGE 505 # share the same enum value and source_map_types picks 506 # BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED/cgroup_storage_deprecated. 507 # Replace 'cgroup_storage_deprecated' with 'cgroup_storage' 508 # so it aligns with what `bpftool map help` shows. 509 source_map_types.remove('cgroup_storage_deprecated') 510 source_map_types.add('cgroup_storage') 511 512 help_map_types = map_info.get_map_help() 513 help_map_options = map_info.get_options() 514 map_info.close() 515 516 man_map_info = ManMapExtractor() 517 man_map_options = man_map_info.get_options() 518 man_map_types = man_map_info.get_map_types() 519 man_map_info.close() 520 521 verify(source_map_types, help_map_types, 522 f'Comparing {BpfHeaderExtractor.filename} (bpf_map_type) and {MapFileExtractor.filename} (do_help() TYPE):') 523 verify(source_map_types, man_map_types, 524 f'Comparing {BpfHeaderExtractor.filename} (bpf_map_type) and {ManMapExtractor.filename} (TYPE):') 525 verify(help_map_options, man_map_options, 526 f'Comparing {MapFileExtractor.filename} (do_help() OPTIONS) and {ManMapExtractor.filename} (OPTIONS):') 527 528 # Attach types (names) 529 530 prog_info = ProgFileExtractor() 531 source_prog_attach_types = set(prog_info.get_attach_types().values()) 532 533 help_prog_attach_types = prog_info.get_prog_attach_help() 534 help_prog_options = prog_info.get_options() 535 prog_info.close() 536 537 man_prog_info = ManProgExtractor() 538 man_prog_options = man_prog_info.get_options() 539 man_prog_attach_types = man_prog_info.get_attach_types() 540 man_prog_info.close() 541 542 543 bashcomp_info = BashcompExtractor() 544 bashcomp_prog_attach_types = bashcomp_info.get_prog_attach_types() 545 bashcomp_info.close() 546 547 verify(source_prog_attach_types, help_prog_attach_types, 548 f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {ProgFileExtractor.filename} (do_help() ATTACH_TYPE):') 549 verify(source_prog_attach_types, man_prog_attach_types, 550 f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {ManProgExtractor.filename} (ATTACH_TYPE):') 551 verify(help_prog_options, man_prog_options, 552 f'Comparing {ProgFileExtractor.filename} (do_help() OPTIONS) and {ManProgExtractor.filename} (OPTIONS):') 553 verify(source_prog_attach_types, bashcomp_prog_attach_types, 554 f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {BashcompExtractor.filename} (BPFTOOL_PROG_ATTACH_TYPES):') 555 556 # Cgroup attach types 557 source_cgroup_attach_types = set(bpf_info.get_cgroup_attach_type_map().values()) 558 bpf_info.close() 559 560 cgroup_info = CgroupFileExtractor() 561 help_cgroup_attach_types = cgroup_info.get_prog_attach_help() 562 help_cgroup_options = cgroup_info.get_options() 563 cgroup_info.close() 564 565 man_cgroup_info = ManCgroupExtractor() 566 man_cgroup_options = man_cgroup_info.get_options() 567 man_cgroup_attach_types = man_cgroup_info.get_attach_types() 568 man_cgroup_info.close() 569 570 verify(source_cgroup_attach_types, help_cgroup_attach_types, 571 f'Comparing {BpfHeaderExtractor.filename} (bpf_attach_type) and {CgroupFileExtractor.filename} (do_help() ATTACH_TYPE):') 572 verify(source_cgroup_attach_types, man_cgroup_attach_types, 573 f'Comparing {BpfHeaderExtractor.filename} (bpf_attach_type) and {ManCgroupExtractor.filename} (ATTACH_TYPE):') 574 verify(help_cgroup_options, man_cgroup_options, 575 f'Comparing {CgroupFileExtractor.filename} (do_help() OPTIONS) and {ManCgroupExtractor.filename} (OPTIONS):') 576 577 # Options for remaining commands 578 579 for cmd in [ 'btf', 'feature', 'gen', 'iter', 'link', 'net', 'perf', 'struct_ops', ]: 580 source_info = GenericSourceExtractor(cmd + '.c') 581 help_cmd_options = source_info.get_options() 582 source_info.close() 583 584 man_cmd_info = ManGenericExtractor(os.path.join(BPFTOOL_DOC_DIR, 'bpftool-' + cmd + '.rst')) 585 man_cmd_options = man_cmd_info.get_options() 586 man_cmd_info.close() 587 588 verify(help_cmd_options, man_cmd_options, 589 f'Comparing {source_info.filename} (do_help() OPTIONS) and {man_cmd_info.filename} (OPTIONS):') 590 591 source_main_info = GenericSourceExtractor('main.c') 592 help_main_options = source_main_info.get_options() 593 source_main_info.close() 594 595 man_main_info = ManGenericExtractor(os.path.join(BPFTOOL_DOC_DIR, 'bpftool.rst')) 596 man_main_options = man_main_info.get_options() 597 man_main_info.close() 598 599 verify(help_main_options, man_main_options, 600 f'Comparing {source_main_info.filename} (do_help() OPTIONS) and {man_main_info.filename} (OPTIONS):') 601 602 # Compare common options (options that apply to all commands) 603 604 main_hdr_info = MainHeaderFileExtractor() 605 source_common_options = main_hdr_info.get_common_options() 606 main_hdr_info.close() 607 608 man_substitutions = ManSubstitutionsExtractor() 609 man_common_options = man_substitutions.get_common_options() 610 man_substitutions.close() 611 612 verify(source_common_options, man_common_options, 613 f'Comparing common options from {main_hdr_info.filename} (HELP_SPEC_OPTIONS) and {man_substitutions.filename}:') 614 615 sys.exit(retval) 616 617if __name__ == "__main__": 618 main() 619