1#!/usr/bin/env python3 2# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 3# 4# Copyright (C) 2021 Isovalent, Inc. 5 6import argparse 7import re 8import os, sys 9 10LINUX_ROOT = os.path.abspath(os.path.join(__file__, 11 os.pardir, os.pardir, os.pardir, os.pardir, os.pardir)) 12BPFTOOL_DIR = os.path.join(LINUX_ROOT, 'tools/bpf/bpftool') 13retval = 0 14 15class BlockParser(object): 16 """ 17 A parser for extracting set of values from blocks such as enums. 18 @reader: a pointer to the open file to parse 19 """ 20 def __init__(self, reader): 21 self.reader = reader 22 23 def search_block(self, start_marker): 24 """ 25 Search for a given structure in a file. 26 @start_marker: regex marking the beginning of a structure to parse 27 """ 28 offset = self.reader.tell() 29 array_start = re.search(start_marker, self.reader.read()) 30 if array_start is None: 31 raise Exception('Failed to find start of block') 32 self.reader.seek(offset + array_start.start()) 33 34 def parse(self, pattern, end_marker): 35 """ 36 Parse a block and return a set of values. Values to extract must be 37 on separate lines in the file. 38 @pattern: pattern used to identify the values to extract 39 @end_marker: regex marking the end of the block to parse 40 """ 41 entries = set() 42 while True: 43 line = self.reader.readline() 44 if not line or re.match(end_marker, line): 45 break 46 capture = pattern.search(line) 47 if capture and pattern.groups >= 1: 48 entries.add(capture.group(1)) 49 return entries 50 51class ArrayParser(BlockParser): 52 """ 53 A parser for extracting dicionaries of values from some BPF-related arrays. 54 @reader: a pointer to the open file to parse 55 @array_name: name of the array to parse 56 """ 57 end_marker = re.compile('^};') 58 59 def __init__(self, reader, array_name): 60 self.array_name = array_name 61 self.start_marker = re.compile(f'(static )?const char \* const {self.array_name}\[.*\] = {{\n') 62 super().__init__(reader) 63 64 def search_block(self): 65 """ 66 Search for the given array in a file. 67 """ 68 super().search_block(self.start_marker); 69 70 def parse(self): 71 """ 72 Parse a block and return data as a dictionary. Items to extract must be 73 on separate lines in the file. 74 """ 75 pattern = re.compile('\[(BPF_\w*)\]\s*= "(.*)",?$') 76 entries = {} 77 while True: 78 line = self.reader.readline() 79 if line == '' or re.match(self.end_marker, line): 80 break 81 capture = pattern.search(line) 82 if capture: 83 entries[capture.group(1)] = capture.group(2) 84 return entries 85 86class InlineListParser(BlockParser): 87 """ 88 A parser for extracting set of values from inline lists. 89 """ 90 def parse(self, pattern, end_marker): 91 """ 92 Parse a block and return a set of values. Multiple values to extract 93 can be on a same line in the file. 94 @pattern: pattern used to identify the values to extract 95 @end_marker: regex marking the end of the block to parse 96 """ 97 entries = set() 98 while True: 99 line = self.reader.readline() 100 if not line: 101 break 102 entries.update(pattern.findall(line)) 103 if re.search(end_marker, line): 104 break 105 return entries 106 107class FileExtractor(object): 108 """ 109 A generic reader for extracting data from a given file. This class contains 110 several helper methods that wrap arround parser objects to extract values 111 from different structures. 112 This class does not offer a way to set a filename, which is expected to be 113 defined in children classes. 114 """ 115 def __init__(self): 116 self.reader = open(self.filename, 'r') 117 118 def close(self): 119 """ 120 Close the file used by the parser. 121 """ 122 self.reader.close() 123 124 def reset_read(self): 125 """ 126 Reset the file position indicator for this parser. This is useful when 127 parsing several structures in the file without respecting the order in 128 which those structures appear in the file. 129 """ 130 self.reader.seek(0) 131 132 def get_types_from_array(self, array_name): 133 """ 134 Search for and parse an array associating names to BPF_* enum members, 135 for example: 136 137 const char * const prog_type_name[] = { 138 [BPF_PROG_TYPE_UNSPEC] = "unspec", 139 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", 140 [BPF_PROG_TYPE_KPROBE] = "kprobe", 141 }; 142 143 Return a dictionary with the enum member names as keys and the 144 associated names as values, for example: 145 146 {'BPF_PROG_TYPE_UNSPEC': 'unspec', 147 'BPF_PROG_TYPE_SOCKET_FILTER': 'socket_filter', 148 'BPF_PROG_TYPE_KPROBE': 'kprobe'} 149 150 @array_name: name of the array to parse 151 """ 152 array_parser = ArrayParser(self.reader, array_name) 153 array_parser.search_block() 154 return array_parser.parse() 155 156 def get_enum(self, enum_name): 157 """ 158 Search for and parse an enum containing BPF_* members, for example: 159 160 enum bpf_prog_type { 161 BPF_PROG_TYPE_UNSPEC, 162 BPF_PROG_TYPE_SOCKET_FILTER, 163 BPF_PROG_TYPE_KPROBE, 164 }; 165 166 Return a set containing all member names, for example: 167 168 {'BPF_PROG_TYPE_UNSPEC', 169 'BPF_PROG_TYPE_SOCKET_FILTER', 170 'BPF_PROG_TYPE_KPROBE'} 171 172 @enum_name: name of the enum to parse 173 """ 174 start_marker = re.compile(f'enum {enum_name} {{\n') 175 pattern = re.compile('^\s*(BPF_\w+),?$') 176 end_marker = re.compile('^};') 177 parser = BlockParser(self.reader) 178 parser.search_block(start_marker) 179 return parser.parse(pattern, end_marker) 180 181 def __get_description_list(self, start_marker, pattern, end_marker): 182 parser = InlineListParser(self.reader) 183 parser.search_block(start_marker) 184 return parser.parse(pattern, end_marker) 185 186 def get_rst_list(self, block_name): 187 """ 188 Search for and parse a list of type names from RST documentation, for 189 example: 190 191 | *TYPE* := { 192 | **socket** | **kprobe** | 193 | **kretprobe** 194 | } 195 196 Return a set containing all type names, for example: 197 198 {'socket', 'kprobe', 'kretprobe'} 199 200 @block_name: name of the blog to parse, 'TYPE' in the example 201 """ 202 start_marker = re.compile(f'\*{block_name}\* := {{') 203 pattern = re.compile('\*\*([\w/-]+)\*\*') 204 end_marker = re.compile('}\n') 205 return self.__get_description_list(start_marker, pattern, end_marker) 206 207 def get_help_list(self, block_name): 208 """ 209 Search for and parse a list of type names from a help message in 210 bpftool, for example: 211 212 " TYPE := { socket | kprobe |\\n" 213 " kretprobe }\\n" 214 215 Return a set containing all type names, for example: 216 217 {'socket', 'kprobe', 'kretprobe'} 218 219 @block_name: name of the blog to parse, 'TYPE' in the example 220 """ 221 start_marker = re.compile(f'"\s*{block_name} := {{') 222 pattern = re.compile('([\w/]+) [|}]') 223 end_marker = re.compile('}') 224 return self.__get_description_list(start_marker, pattern, end_marker) 225 226 def get_help_list_macro(self, macro): 227 """ 228 Search for and parse a list of values from a help message starting with 229 a macro in bpftool, for example: 230 231 " " HELP_SPEC_OPTIONS " |\\n" 232 " {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} }\\n" 233 234 Return a set containing all item names, for example: 235 236 {'-f', '--bpffs', '-m', '--mapcompat', '-n', '--nomount'} 237 238 @macro: macro starting the block, 'HELP_SPEC_OPTIONS' in the example 239 """ 240 start_marker = re.compile(f'"\s*{macro}\s*" [|}}]') 241 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]])') 242 end_marker = re.compile('}\\\\n') 243 return self.__get_description_list(start_marker, pattern, end_marker) 244 245 def default_options(self): 246 """ 247 Return the default options contained in HELP_SPEC_OPTIONS 248 """ 249 return { '-j', '--json', '-p', '--pretty', '-d', '--debug' } 250 251 def get_bashcomp_list(self, block_name): 252 """ 253 Search for and parse a list of type names from a variable in bash 254 completion file, for example: 255 256 local BPFTOOL_PROG_LOAD_TYPES='socket kprobe \\ 257 kretprobe' 258 259 Return a set containing all type names, for example: 260 261 {'socket', 'kprobe', 'kretprobe'} 262 263 @block_name: name of the blog to parse, 'TYPE' in the example 264 """ 265 start_marker = re.compile(f'local {block_name}=\'') 266 pattern = re.compile('(?:.*=\')?([\w/]+)') 267 end_marker = re.compile('\'$') 268 return self.__get_description_list(start_marker, pattern, end_marker) 269 270class SourceFileExtractor(FileExtractor): 271 """ 272 An abstract extractor for a source file with usage message. 273 This class does not offer a way to set a filename, which is expected to be 274 defined in children classes. 275 """ 276 def get_options(self): 277 return self.default_options().union(self.get_help_list_macro('HELP_SPEC_OPTIONS')) 278 279class ProgFileExtractor(SourceFileExtractor): 280 """ 281 An extractor for bpftool's prog.c. 282 """ 283 filename = os.path.join(BPFTOOL_DIR, 'prog.c') 284 285 def get_prog_types(self): 286 return self.get_types_from_array('prog_type_name') 287 288 def get_attach_types(self): 289 return self.get_types_from_array('attach_type_strings') 290 291 def get_prog_attach_help(self): 292 return self.get_help_list('ATTACH_TYPE') 293 294class MapFileExtractor(SourceFileExtractor): 295 """ 296 An extractor for bpftool's map.c. 297 """ 298 filename = os.path.join(BPFTOOL_DIR, 'map.c') 299 300 def get_map_types(self): 301 return self.get_types_from_array('map_type_name') 302 303 def get_map_help(self): 304 return self.get_help_list('TYPE') 305 306class CgroupFileExtractor(SourceFileExtractor): 307 """ 308 An extractor for bpftool's cgroup.c. 309 """ 310 filename = os.path.join(BPFTOOL_DIR, 'cgroup.c') 311 312 def get_prog_attach_help(self): 313 return self.get_help_list('ATTACH_TYPE') 314 315class CommonFileExtractor(SourceFileExtractor): 316 """ 317 An extractor for bpftool's common.c. 318 """ 319 filename = os.path.join(BPFTOOL_DIR, 'common.c') 320 321 def __init__(self): 322 super().__init__() 323 self.attach_types = {} 324 325 def get_attach_types(self): 326 if not self.attach_types: 327 self.attach_types = self.get_types_from_array('attach_type_name') 328 return self.attach_types 329 330 def get_cgroup_attach_types(self): 331 if not self.attach_types: 332 self.get_attach_types() 333 cgroup_types = {} 334 for (key, value) in self.attach_types.items(): 335 if key.find('BPF_CGROUP') != -1: 336 cgroup_types[key] = value 337 return cgroup_types 338 339class GenericSourceExtractor(SourceFileExtractor): 340 """ 341 An extractor for generic source code files. 342 """ 343 filename = "" 344 345 def __init__(self, filename): 346 self.filename = os.path.join(BPFTOOL_DIR, filename) 347 super().__init__() 348 349class BpfHeaderExtractor(FileExtractor): 350 """ 351 An extractor for the UAPI BPF header. 352 """ 353 filename = os.path.join(LINUX_ROOT, 'tools/include/uapi/linux/bpf.h') 354 355 def get_prog_types(self): 356 return self.get_enum('bpf_prog_type') 357 358 def get_map_types(self): 359 return self.get_enum('bpf_map_type') 360 361 def get_attach_types(self): 362 return self.get_enum('bpf_attach_type') 363 364class ManPageExtractor(FileExtractor): 365 """ 366 An abstract extractor for an RST documentation page. 367 This class does not offer a way to set a filename, which is expected to be 368 defined in children classes. 369 """ 370 def get_options(self): 371 return self.get_rst_list('OPTIONS') 372 373class ManProgExtractor(ManPageExtractor): 374 """ 375 An extractor for bpftool-prog.rst. 376 """ 377 filename = os.path.join(BPFTOOL_DIR, 'Documentation/bpftool-prog.rst') 378 379 def get_attach_types(self): 380 return self.get_rst_list('ATTACH_TYPE') 381 382class ManMapExtractor(ManPageExtractor): 383 """ 384 An extractor for bpftool-map.rst. 385 """ 386 filename = os.path.join(BPFTOOL_DIR, 'Documentation/bpftool-map.rst') 387 388 def get_map_types(self): 389 return self.get_rst_list('TYPE') 390 391class ManCgroupExtractor(ManPageExtractor): 392 """ 393 An extractor for bpftool-cgroup.rst. 394 """ 395 filename = os.path.join(BPFTOOL_DIR, 'Documentation/bpftool-cgroup.rst') 396 397 def get_attach_types(self): 398 return self.get_rst_list('ATTACH_TYPE') 399 400class ManGenericExtractor(ManPageExtractor): 401 """ 402 An extractor for generic RST documentation pages. 403 """ 404 filename = "" 405 406 def __init__(self, filename): 407 self.filename = os.path.join(BPFTOOL_DIR, filename) 408 super().__init__() 409 410class BashcompExtractor(FileExtractor): 411 """ 412 An extractor for bpftool's bash completion file. 413 """ 414 filename = os.path.join(BPFTOOL_DIR, 'bash-completion/bpftool') 415 416 def get_prog_attach_types(self): 417 return self.get_bashcomp_list('BPFTOOL_PROG_ATTACH_TYPES') 418 419 def get_map_types(self): 420 return self.get_bashcomp_list('BPFTOOL_MAP_CREATE_TYPES') 421 422 def get_cgroup_attach_types(self): 423 return self.get_bashcomp_list('BPFTOOL_CGROUP_ATTACH_TYPES') 424 425def verify(first_set, second_set, message): 426 """ 427 Print all values that differ between two sets. 428 @first_set: one set to compare 429 @second_set: another set to compare 430 @message: message to print for values belonging to only one of the sets 431 """ 432 global retval 433 diff = first_set.symmetric_difference(second_set) 434 if diff: 435 print(message, diff) 436 retval = 1 437 438def main(): 439 # No arguments supported at this time, but print usage for -h|--help 440 argParser = argparse.ArgumentParser(description=""" 441 Verify that bpftool's code, help messages, documentation and bash 442 completion are all in sync on program types, map types, attach types, and 443 options. Also check that bpftool is in sync with the UAPI BPF header. 444 """) 445 args = argParser.parse_args() 446 447 # Map types (enum) 448 449 bpf_info = BpfHeaderExtractor() 450 ref = bpf_info.get_map_types() 451 452 map_info = MapFileExtractor() 453 source_map_items = map_info.get_map_types() 454 map_types_enum = set(source_map_items.keys()) 455 456 verify(ref, map_types_enum, 457 f'Comparing BPF header (enum bpf_map_type) and {MapFileExtractor.filename} (map_type_name):') 458 459 # Map types (names) 460 461 source_map_types = set(source_map_items.values()) 462 source_map_types.discard('unspec') 463 464 help_map_types = map_info.get_map_help() 465 help_map_options = map_info.get_options() 466 map_info.close() 467 468 man_map_info = ManMapExtractor() 469 man_map_options = man_map_info.get_options() 470 man_map_types = man_map_info.get_map_types() 471 man_map_info.close() 472 473 bashcomp_info = BashcompExtractor() 474 bashcomp_map_types = bashcomp_info.get_map_types() 475 476 verify(source_map_types, help_map_types, 477 f'Comparing {MapFileExtractor.filename} (map_type_name) and {MapFileExtractor.filename} (do_help() TYPE):') 478 verify(source_map_types, man_map_types, 479 f'Comparing {MapFileExtractor.filename} (map_type_name) and {ManMapExtractor.filename} (TYPE):') 480 verify(help_map_options, man_map_options, 481 f'Comparing {MapFileExtractor.filename} (do_help() OPTIONS) and {ManMapExtractor.filename} (OPTIONS):') 482 verify(source_map_types, bashcomp_map_types, 483 f'Comparing {MapFileExtractor.filename} (map_type_name) and {BashcompExtractor.filename} (BPFTOOL_MAP_CREATE_TYPES):') 484 485 # Program types (enum) 486 487 ref = bpf_info.get_prog_types() 488 489 prog_info = ProgFileExtractor() 490 prog_types = set(prog_info.get_prog_types().keys()) 491 492 verify(ref, prog_types, 493 f'Comparing BPF header (enum bpf_prog_type) and {ProgFileExtractor.filename} (prog_type_name):') 494 495 # Attach types (enum) 496 497 ref = bpf_info.get_attach_types() 498 bpf_info.close() 499 500 common_info = CommonFileExtractor() 501 attach_types = common_info.get_attach_types() 502 503 verify(ref, attach_types, 504 f'Comparing BPF header (enum bpf_attach_type) and {CommonFileExtractor.filename} (attach_type_name):') 505 506 # Attach types (names) 507 508 source_prog_attach_types = set(prog_info.get_attach_types().values()) 509 510 help_prog_attach_types = prog_info.get_prog_attach_help() 511 help_prog_options = prog_info.get_options() 512 prog_info.close() 513 514 man_prog_info = ManProgExtractor() 515 man_prog_options = man_prog_info.get_options() 516 man_prog_attach_types = man_prog_info.get_attach_types() 517 man_prog_info.close() 518 519 bashcomp_info.reset_read() # We stopped at map types, rewind 520 bashcomp_prog_attach_types = bashcomp_info.get_prog_attach_types() 521 522 verify(source_prog_attach_types, help_prog_attach_types, 523 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ProgFileExtractor.filename} (do_help() ATTACH_TYPE):') 524 verify(source_prog_attach_types, man_prog_attach_types, 525 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ManProgExtractor.filename} (ATTACH_TYPE):') 526 verify(help_prog_options, man_prog_options, 527 f'Comparing {ProgFileExtractor.filename} (do_help() OPTIONS) and {ManProgExtractor.filename} (OPTIONS):') 528 verify(source_prog_attach_types, bashcomp_prog_attach_types, 529 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_PROG_ATTACH_TYPES):') 530 531 # Cgroup attach types 532 533 source_cgroup_attach_types = set(common_info.get_cgroup_attach_types().values()) 534 common_info.close() 535 536 cgroup_info = CgroupFileExtractor() 537 help_cgroup_attach_types = cgroup_info.get_prog_attach_help() 538 help_cgroup_options = cgroup_info.get_options() 539 cgroup_info.close() 540 541 man_cgroup_info = ManCgroupExtractor() 542 man_cgroup_options = man_cgroup_info.get_options() 543 man_cgroup_attach_types = man_cgroup_info.get_attach_types() 544 man_cgroup_info.close() 545 546 bashcomp_cgroup_attach_types = bashcomp_info.get_cgroup_attach_types() 547 bashcomp_info.close() 548 549 verify(source_cgroup_attach_types, help_cgroup_attach_types, 550 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {CgroupFileExtractor.filename} (do_help() ATTACH_TYPE):') 551 verify(source_cgroup_attach_types, man_cgroup_attach_types, 552 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {ManCgroupExtractor.filename} (ATTACH_TYPE):') 553 verify(help_cgroup_options, man_cgroup_options, 554 f'Comparing {CgroupFileExtractor.filename} (do_help() OPTIONS) and {ManCgroupExtractor.filename} (OPTIONS):') 555 verify(source_cgroup_attach_types, bashcomp_cgroup_attach_types, 556 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_CGROUP_ATTACH_TYPES):') 557 558 # Options for remaining commands 559 560 for cmd in [ 'btf', 'feature', 'gen', 'iter', 'link', 'net', 'perf', 'struct_ops', ]: 561 source_info = GenericSourceExtractor(cmd + '.c') 562 help_cmd_options = source_info.get_options() 563 source_info.close() 564 565 man_cmd_info = ManGenericExtractor(os.path.join('Documentation', 'bpftool-' + cmd + '.rst')) 566 man_cmd_options = man_cmd_info.get_options() 567 man_cmd_info.close() 568 569 verify(help_cmd_options, man_cmd_options, 570 f'Comparing {source_info.filename} (do_help() OPTIONS) and {man_cmd_info.filename} (OPTIONS):') 571 572 source_main_info = GenericSourceExtractor('main.c') 573 help_main_options = source_main_info.get_options() 574 source_main_info.close() 575 576 man_main_info = ManGenericExtractor(os.path.join('Documentation', 'bpftool.rst')) 577 man_main_options = man_main_info.get_options() 578 man_main_info.close() 579 580 verify(help_main_options, man_main_options, 581 f'Comparing {source_main_info.filename} (do_help() OPTIONS) and {man_main_info.filename} (OPTIONS):') 582 583 sys.exit(retval) 584 585if __name__ == "__main__": 586 main() 587