1#!/usr/bin/env python3 2# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 3# 4# Copyright (C) 2021 Isovalent, Inc. 5 6import argparse 7import re 8import os, sys 9 10LINUX_ROOT = os.path.abspath(os.path.join(__file__, 11 os.pardir, os.pardir, os.pardir, os.pardir, os.pardir)) 12BPFTOOL_DIR = os.getenv('BPFTOOL_DIR', 13 os.path.join(LINUX_ROOT, 'tools/bpf/bpftool')) 14BPFTOOL_BASHCOMP_DIR = os.getenv('BPFTOOL_BASHCOMP_DIR', 15 os.path.join(BPFTOOL_DIR, 'bash-completion')) 16BPFTOOL_DOC_DIR = os.getenv('BPFTOOL_DOC_DIR', 17 os.path.join(BPFTOOL_DIR, 'Documentation')) 18INCLUDE_DIR = os.getenv('INCLUDE_DIR', 19 os.path.join(LINUX_ROOT, 'tools/include')) 20 21retval = 0 22 23class BlockParser(object): 24 """ 25 A parser for extracting set of values from blocks such as enums. 26 @reader: a pointer to the open file to parse 27 """ 28 def __init__(self, reader): 29 self.reader = reader 30 31 def search_block(self, start_marker): 32 """ 33 Search for a given structure in a file. 34 @start_marker: regex marking the beginning of a structure to parse 35 """ 36 offset = self.reader.tell() 37 array_start = re.search(start_marker, self.reader.read()) 38 if array_start is None: 39 raise Exception('Failed to find start of block') 40 self.reader.seek(offset + array_start.start()) 41 42 def parse(self, pattern, end_marker): 43 """ 44 Parse a block and return a set of values. Values to extract must be 45 on separate lines in the file. 46 @pattern: pattern used to identify the values to extract 47 @end_marker: regex marking the end of the block to parse 48 """ 49 entries = set() 50 while True: 51 line = self.reader.readline() 52 if not line or re.match(end_marker, line): 53 break 54 capture = pattern.search(line) 55 if capture and pattern.groups >= 1: 56 entries.add(capture.group(1)) 57 return entries 58 59class ArrayParser(BlockParser): 60 """ 61 A parser for extracting a set of values from some BPF-related arrays. 62 @reader: a pointer to the open file to parse 63 @array_name: name of the array to parse 64 """ 65 end_marker = re.compile('^};') 66 67 def __init__(self, reader, array_name): 68 self.array_name = array_name 69 self.start_marker = re.compile(f'(static )?const bool {self.array_name}\[.*\] = {{\n') 70 super().__init__(reader) 71 72 def search_block(self): 73 """ 74 Search for the given array in a file. 75 """ 76 super().search_block(self.start_marker); 77 78 def parse(self): 79 """ 80 Parse a block and return data as a dictionary. Items to extract must be 81 on separate lines in the file. 82 """ 83 pattern = re.compile('\[(BPF_\w*)\]\s*= (true|false),?$') 84 entries = set() 85 while True: 86 line = self.reader.readline() 87 if line == '' or re.match(self.end_marker, line): 88 break 89 capture = pattern.search(line) 90 if capture: 91 entries |= {capture.group(1)} 92 return entries 93 94class InlineListParser(BlockParser): 95 """ 96 A parser for extracting set of values from inline lists. 97 """ 98 def parse(self, pattern, end_marker): 99 """ 100 Parse a block and return a set of values. Multiple values to extract 101 can be on a same line in the file. 102 @pattern: pattern used to identify the values to extract 103 @end_marker: regex marking the end of the block to parse 104 """ 105 entries = set() 106 while True: 107 line = self.reader.readline() 108 if not line: 109 break 110 entries.update(pattern.findall(line)) 111 if re.search(end_marker, line): 112 break 113 return entries 114 115class FileExtractor(object): 116 """ 117 A generic reader for extracting data from a given file. This class contains 118 several helper methods that wrap around parser objects to extract values 119 from different structures. 120 This class does not offer a way to set a filename, which is expected to be 121 defined in children classes. 122 """ 123 def __init__(self): 124 self.reader = open(self.filename, 'r') 125 126 def close(self): 127 """ 128 Close the file used by the parser. 129 """ 130 self.reader.close() 131 132 def reset_read(self): 133 """ 134 Reset the file position indicator for this parser. This is useful when 135 parsing several structures in the file without respecting the order in 136 which those structures appear in the file. 137 """ 138 self.reader.seek(0) 139 140 def get_types_from_array(self, array_name): 141 """ 142 Search for and parse a list of allowed BPF_* enum members, for example: 143 144 const bool prog_type_name[] = { 145 [BPF_PROG_TYPE_UNSPEC] = true, 146 [BPF_PROG_TYPE_SOCKET_FILTER] = true, 147 [BPF_PROG_TYPE_KPROBE] = true, 148 }; 149 150 Return a set of the enum members, for example: 151 152 {'BPF_PROG_TYPE_UNSPEC', 153 'BPF_PROG_TYPE_SOCKET_FILTER', 154 'BPF_PROG_TYPE_KPROBE'} 155 156 @array_name: name of the array to parse 157 """ 158 array_parser = ArrayParser(self.reader, array_name) 159 array_parser.search_block() 160 return array_parser.parse() 161 162 def get_enum(self, enum_name): 163 """ 164 Search for and parse an enum containing BPF_* members, for example: 165 166 enum bpf_prog_type { 167 BPF_PROG_TYPE_UNSPEC, 168 BPF_PROG_TYPE_SOCKET_FILTER, 169 BPF_PROG_TYPE_KPROBE, 170 }; 171 172 Return a set containing all member names, for example: 173 174 {'BPF_PROG_TYPE_UNSPEC', 175 'BPF_PROG_TYPE_SOCKET_FILTER', 176 'BPF_PROG_TYPE_KPROBE'} 177 178 @enum_name: name of the enum to parse 179 """ 180 start_marker = re.compile(f'enum {enum_name} {{\n') 181 pattern = re.compile('^\s*(BPF_\w+),?(\s+/\*.*\*/)?$') 182 end_marker = re.compile('^};') 183 parser = BlockParser(self.reader) 184 parser.search_block(start_marker) 185 return parser.parse(pattern, end_marker) 186 187 def make_enum_map(self, names, enum_prefix): 188 """ 189 Search for and parse an enum containing BPF_* members, just as get_enum 190 does. However, instead of just returning a set of the variant names, 191 also generate a textual representation from them by (assuming and) 192 removing a provided prefix and lowercasing the remainder. Then return a 193 dict mapping from name to textual representation. 194 195 @enum_values: a set of enum values; e.g., as retrieved by get_enum 196 @enum_prefix: the prefix to remove from each of the variants to infer 197 textual representation 198 """ 199 mapping = {} 200 for name in names: 201 if not name.startswith(enum_prefix): 202 raise Exception(f"enum variant {name} does not start with {enum_prefix}") 203 text = name[len(enum_prefix):].lower() 204 mapping[name] = text 205 206 return mapping 207 208 def __get_description_list(self, start_marker, pattern, end_marker): 209 parser = InlineListParser(self.reader) 210 parser.search_block(start_marker) 211 return parser.parse(pattern, end_marker) 212 213 def get_rst_list(self, block_name): 214 """ 215 Search for and parse a list of type names from RST documentation, for 216 example: 217 218 | *TYPE* := { 219 | **socket** | **kprobe** | 220 | **kretprobe** 221 | } 222 223 Return a set containing all type names, for example: 224 225 {'socket', 'kprobe', 'kretprobe'} 226 227 @block_name: name of the blog to parse, 'TYPE' in the example 228 """ 229 start_marker = re.compile(f'\*{block_name}\* := {{') 230 pattern = re.compile('\*\*([\w/-]+)\*\*') 231 end_marker = re.compile('}\n') 232 return self.__get_description_list(start_marker, pattern, end_marker) 233 234 def get_help_list(self, block_name): 235 """ 236 Search for and parse a list of type names from a help message in 237 bpftool, for example: 238 239 " TYPE := { socket | kprobe |\\n" 240 " kretprobe }\\n" 241 242 Return a set containing all type names, for example: 243 244 {'socket', 'kprobe', 'kretprobe'} 245 246 @block_name: name of the blog to parse, 'TYPE' in the example 247 """ 248 start_marker = re.compile(f'"\s*{block_name} := {{') 249 pattern = re.compile('([\w/]+) [|}]') 250 end_marker = re.compile('}') 251 return self.__get_description_list(start_marker, pattern, end_marker) 252 253 def get_help_list_macro(self, macro): 254 """ 255 Search for and parse a list of values from a help message starting with 256 a macro in bpftool, for example: 257 258 " " HELP_SPEC_OPTIONS " |\\n" 259 " {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} }\\n" 260 261 Return a set containing all item names, for example: 262 263 {'-f', '--bpffs', '-m', '--mapcompat', '-n', '--nomount'} 264 265 @macro: macro starting the block, 'HELP_SPEC_OPTIONS' in the example 266 """ 267 start_marker = re.compile(f'"\s*{macro}\s*" [|}}]') 268 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]])') 269 end_marker = re.compile('}\\\\n') 270 return self.__get_description_list(start_marker, pattern, end_marker) 271 272 def get_bashcomp_list(self, block_name): 273 """ 274 Search for and parse a list of type names from a variable in bash 275 completion file, for example: 276 277 local BPFTOOL_PROG_LOAD_TYPES='socket kprobe \\ 278 kretprobe' 279 280 Return a set containing all type names, for example: 281 282 {'socket', 'kprobe', 'kretprobe'} 283 284 @block_name: name of the blog to parse, 'TYPE' in the example 285 """ 286 start_marker = re.compile(f'local {block_name}=\'') 287 pattern = re.compile('(?:.*=\')?([\w/]+)') 288 end_marker = re.compile('\'$') 289 return self.__get_description_list(start_marker, pattern, end_marker) 290 291class SourceFileExtractor(FileExtractor): 292 """ 293 An abstract extractor for a source file with usage message. 294 This class does not offer a way to set a filename, which is expected to be 295 defined in children classes. 296 """ 297 def get_options(self): 298 return self.get_help_list_macro('HELP_SPEC_OPTIONS') 299 300class MainHeaderFileExtractor(SourceFileExtractor): 301 """ 302 An extractor for bpftool's main.h 303 """ 304 filename = os.path.join(BPFTOOL_DIR, 'main.h') 305 306 def get_common_options(self): 307 """ 308 Parse the list of common options in main.h (options that apply to all 309 commands), which looks to the lists of options in other source files 310 but has different start and end markers: 311 312 "OPTIONS := { {-j|--json} [{-p|--pretty}] | {-d|--debug} | {-l|--legacy}" 313 314 Return a set containing all options, such as: 315 316 {'-p', '-d', '--legacy', '--pretty', '--debug', '--json', '-l', '-j'} 317 """ 318 start_marker = re.compile(f'"OPTIONS :=') 319 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]"])') 320 end_marker = re.compile('#define') 321 322 parser = InlineListParser(self.reader) 323 parser.search_block(start_marker) 324 return parser.parse(pattern, end_marker) 325 326class ManSubstitutionsExtractor(SourceFileExtractor): 327 """ 328 An extractor for substitutions.rst 329 """ 330 filename = os.path.join(BPFTOOL_DOC_DIR, 'substitutions.rst') 331 332 def get_common_options(self): 333 """ 334 Parse the list of common options in substitutions.rst (options that 335 apply to all commands). 336 337 Return a set containing all options, such as: 338 339 {'-p', '-d', '--legacy', '--pretty', '--debug', '--json', '-l', '-j'} 340 """ 341 start_marker = re.compile('\|COMMON_OPTIONS\| replace:: {') 342 pattern = re.compile('\*\*([\w/-]+)\*\*') 343 end_marker = re.compile('}$') 344 345 parser = InlineListParser(self.reader) 346 parser.search_block(start_marker) 347 return parser.parse(pattern, end_marker) 348 349class ProgFileExtractor(SourceFileExtractor): 350 """ 351 An extractor for bpftool's prog.c. 352 """ 353 filename = os.path.join(BPFTOOL_DIR, 'prog.c') 354 355 def get_attach_types(self): 356 types = self.get_types_from_array('attach_types') 357 return self.make_enum_map(types, 'BPF_') 358 359 def get_prog_attach_help(self): 360 return self.get_help_list('ATTACH_TYPE') 361 362class MapFileExtractor(SourceFileExtractor): 363 """ 364 An extractor for bpftool's map.c. 365 """ 366 filename = os.path.join(BPFTOOL_DIR, 'map.c') 367 368 def get_map_help(self): 369 return self.get_help_list('TYPE') 370 371class CgroupFileExtractor(SourceFileExtractor): 372 """ 373 An extractor for bpftool's cgroup.c. 374 """ 375 filename = os.path.join(BPFTOOL_DIR, 'cgroup.c') 376 377 def get_prog_attach_help(self): 378 return self.get_help_list('ATTACH_TYPE') 379 380class GenericSourceExtractor(SourceFileExtractor): 381 """ 382 An extractor for generic source code files. 383 """ 384 filename = "" 385 386 def __init__(self, filename): 387 self.filename = os.path.join(BPFTOOL_DIR, filename) 388 super().__init__() 389 390class BpfHeaderExtractor(FileExtractor): 391 """ 392 An extractor for the UAPI BPF header. 393 """ 394 filename = os.path.join(INCLUDE_DIR, 'uapi/linux/bpf.h') 395 396 def __init__(self): 397 super().__init__() 398 self.attach_types = {} 399 400 def get_prog_types(self): 401 return self.get_enum('bpf_prog_type') 402 403 def get_map_type_map(self): 404 names = self.get_enum('bpf_map_type') 405 return self.make_enum_map(names, 'BPF_MAP_TYPE_') 406 407 def get_attach_type_map(self): 408 if not self.attach_types: 409 names = self.get_enum('bpf_attach_type') 410 self.attach_types = self.make_enum_map(names, 'BPF_') 411 return self.attach_types 412 413 def get_cgroup_attach_type_map(self): 414 if not self.attach_types: 415 self.get_attach_type_map() 416 return {name: text for name, text in self.attach_types.items() 417 if name.startswith('BPF_CGROUP')} 418 419class ManPageExtractor(FileExtractor): 420 """ 421 An abstract extractor for an RST documentation page. 422 This class does not offer a way to set a filename, which is expected to be 423 defined in children classes. 424 """ 425 def get_options(self): 426 return self.get_rst_list('OPTIONS') 427 428class ManProgExtractor(ManPageExtractor): 429 """ 430 An extractor for bpftool-prog.rst. 431 """ 432 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-prog.rst') 433 434 def get_attach_types(self): 435 return self.get_rst_list('ATTACH_TYPE') 436 437class ManMapExtractor(ManPageExtractor): 438 """ 439 An extractor for bpftool-map.rst. 440 """ 441 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-map.rst') 442 443 def get_map_types(self): 444 return self.get_rst_list('TYPE') 445 446class ManCgroupExtractor(ManPageExtractor): 447 """ 448 An extractor for bpftool-cgroup.rst. 449 """ 450 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-cgroup.rst') 451 452 def get_attach_types(self): 453 return self.get_rst_list('ATTACH_TYPE') 454 455class ManGenericExtractor(ManPageExtractor): 456 """ 457 An extractor for generic RST documentation pages. 458 """ 459 filename = "" 460 461 def __init__(self, filename): 462 self.filename = os.path.join(BPFTOOL_DIR, filename) 463 super().__init__() 464 465class BashcompExtractor(FileExtractor): 466 """ 467 An extractor for bpftool's bash completion file. 468 """ 469 filename = os.path.join(BPFTOOL_BASHCOMP_DIR, 'bpftool') 470 471 def get_prog_attach_types(self): 472 return self.get_bashcomp_list('BPFTOOL_PROG_ATTACH_TYPES') 473 474def verify(first_set, second_set, message): 475 """ 476 Print all values that differ between two sets. 477 @first_set: one set to compare 478 @second_set: another set to compare 479 @message: message to print for values belonging to only one of the sets 480 """ 481 global retval 482 diff = first_set.symmetric_difference(second_set) 483 if diff: 484 print(message, diff) 485 retval = 1 486 487def main(): 488 # No arguments supported at this time, but print usage for -h|--help 489 argParser = argparse.ArgumentParser(description=""" 490 Verify that bpftool's code, help messages, documentation and bash 491 completion are all in sync on program types, map types, attach types, and 492 options. Also check that bpftool is in sync with the UAPI BPF header. 493 """) 494 args = argParser.parse_args() 495 496 bpf_info = BpfHeaderExtractor() 497 498 # Map types (names) 499 500 map_info = MapFileExtractor() 501 source_map_types = set(bpf_info.get_map_type_map().values()) 502 source_map_types.discard('unspec') 503 504 help_map_types = map_info.get_map_help() 505 help_map_options = map_info.get_options() 506 map_info.close() 507 508 man_map_info = ManMapExtractor() 509 man_map_options = man_map_info.get_options() 510 man_map_types = man_map_info.get_map_types() 511 man_map_info.close() 512 513 verify(source_map_types, help_map_types, 514 f'Comparing {BpfHeaderExtractor.filename} (bpf_map_type) and {MapFileExtractor.filename} (do_help() TYPE):') 515 verify(source_map_types, man_map_types, 516 f'Comparing {BpfHeaderExtractor.filename} (bpf_map_type) and {ManMapExtractor.filename} (TYPE):') 517 verify(help_map_options, man_map_options, 518 f'Comparing {MapFileExtractor.filename} (do_help() OPTIONS) and {ManMapExtractor.filename} (OPTIONS):') 519 520 # Attach types (names) 521 522 prog_info = ProgFileExtractor() 523 source_prog_attach_types = set(prog_info.get_attach_types().values()) 524 525 help_prog_attach_types = prog_info.get_prog_attach_help() 526 help_prog_options = prog_info.get_options() 527 prog_info.close() 528 529 man_prog_info = ManProgExtractor() 530 man_prog_options = man_prog_info.get_options() 531 man_prog_attach_types = man_prog_info.get_attach_types() 532 man_prog_info.close() 533 534 535 bashcomp_info = BashcompExtractor() 536 bashcomp_prog_attach_types = bashcomp_info.get_prog_attach_types() 537 bashcomp_info.close() 538 539 verify(source_prog_attach_types, help_prog_attach_types, 540 f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {ProgFileExtractor.filename} (do_help() ATTACH_TYPE):') 541 verify(source_prog_attach_types, man_prog_attach_types, 542 f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {ManProgExtractor.filename} (ATTACH_TYPE):') 543 verify(help_prog_options, man_prog_options, 544 f'Comparing {ProgFileExtractor.filename} (do_help() OPTIONS) and {ManProgExtractor.filename} (OPTIONS):') 545 verify(source_prog_attach_types, bashcomp_prog_attach_types, 546 f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {BashcompExtractor.filename} (BPFTOOL_PROG_ATTACH_TYPES):') 547 548 # Cgroup attach types 549 source_cgroup_attach_types = set(bpf_info.get_cgroup_attach_type_map().values()) 550 bpf_info.close() 551 552 cgroup_info = CgroupFileExtractor() 553 help_cgroup_attach_types = cgroup_info.get_prog_attach_help() 554 help_cgroup_options = cgroup_info.get_options() 555 cgroup_info.close() 556 557 man_cgroup_info = ManCgroupExtractor() 558 man_cgroup_options = man_cgroup_info.get_options() 559 man_cgroup_attach_types = man_cgroup_info.get_attach_types() 560 man_cgroup_info.close() 561 562 verify(source_cgroup_attach_types, help_cgroup_attach_types, 563 f'Comparing {BpfHeaderExtractor.filename} (bpf_attach_type) and {CgroupFileExtractor.filename} (do_help() ATTACH_TYPE):') 564 verify(source_cgroup_attach_types, man_cgroup_attach_types, 565 f'Comparing {BpfHeaderExtractor.filename} (bpf_attach_type) and {ManCgroupExtractor.filename} (ATTACH_TYPE):') 566 verify(help_cgroup_options, man_cgroup_options, 567 f'Comparing {CgroupFileExtractor.filename} (do_help() OPTIONS) and {ManCgroupExtractor.filename} (OPTIONS):') 568 569 # Options for remaining commands 570 571 for cmd in [ 'btf', 'feature', 'gen', 'iter', 'link', 'net', 'perf', 'struct_ops', ]: 572 source_info = GenericSourceExtractor(cmd + '.c') 573 help_cmd_options = source_info.get_options() 574 source_info.close() 575 576 man_cmd_info = ManGenericExtractor(os.path.join(BPFTOOL_DOC_DIR, 'bpftool-' + cmd + '.rst')) 577 man_cmd_options = man_cmd_info.get_options() 578 man_cmd_info.close() 579 580 verify(help_cmd_options, man_cmd_options, 581 f'Comparing {source_info.filename} (do_help() OPTIONS) and {man_cmd_info.filename} (OPTIONS):') 582 583 source_main_info = GenericSourceExtractor('main.c') 584 help_main_options = source_main_info.get_options() 585 source_main_info.close() 586 587 man_main_info = ManGenericExtractor(os.path.join(BPFTOOL_DOC_DIR, 'bpftool.rst')) 588 man_main_options = man_main_info.get_options() 589 man_main_info.close() 590 591 verify(help_main_options, man_main_options, 592 f'Comparing {source_main_info.filename} (do_help() OPTIONS) and {man_main_info.filename} (OPTIONS):') 593 594 # Compare common options (options that apply to all commands) 595 596 main_hdr_info = MainHeaderFileExtractor() 597 source_common_options = main_hdr_info.get_common_options() 598 main_hdr_info.close() 599 600 man_substitutions = ManSubstitutionsExtractor() 601 man_common_options = man_substitutions.get_common_options() 602 man_substitutions.close() 603 604 verify(source_common_options, man_common_options, 605 f'Comparing common options from {main_hdr_info.filename} (HELP_SPEC_OPTIONS) and {man_substitutions.filename}:') 606 607 sys.exit(retval) 608 609if __name__ == "__main__": 610 main() 611