1#!/usr/bin/env python 2 3""" 4This script determines the given package's openbmc dependencies from its 5configure.ac file where it downloads, configures, builds, and installs each of 6these dependencies. Then the given package is configured, built, and installed 7prior to executing its unit tests. 8""" 9 10from git import Repo 11from urlparse import urljoin 12from subprocess import check_call, call, CalledProcessError 13import os 14import sys 15import argparse 16import multiprocessing 17import re 18 19 20class DepTree(): 21 """ 22 Represents package dependency tree, where each node is a DepTree with a 23 name and DepTree children. 24 """ 25 26 def __init__(self, name): 27 """ 28 Create new DepTree. 29 30 Parameter descriptions: 31 name Name of new tree node. 32 """ 33 self.name = name 34 self.children = list() 35 36 def AddChild(self, name): 37 """ 38 Add new child node to current node. 39 40 Parameter descriptions: 41 name Name of new child 42 """ 43 new_child = DepTree(name) 44 self.children.append(new_child) 45 return new_child 46 47 def AddChildNode(self, node): 48 """ 49 Add existing child node to current node. 50 51 Parameter descriptions: 52 node Tree node to add 53 """ 54 self.children.append(node) 55 56 def RemoveChild(self, name): 57 """ 58 Remove child node. 59 60 Parameter descriptions: 61 name Name of child to remove 62 """ 63 for child in self.children: 64 if child.name == name: 65 self.children.remove(child) 66 return 67 68 def GetNode(self, name): 69 """ 70 Return node with matching name. Return None if not found. 71 72 Parameter descriptions: 73 name Name of node to return 74 """ 75 if self.name == name: 76 return self 77 for child in self.children: 78 node = child.GetNode(name) 79 if node: 80 return node 81 return None 82 83 def GetParentNode(self, name, parent_node=None): 84 """ 85 Return parent of node with matching name. Return none if not found. 86 87 Parameter descriptions: 88 name Name of node to get parent of 89 parent_node Parent of current node 90 """ 91 if self.name == name: 92 return parent_node 93 for child in self.children: 94 found_node = child.GetParentNode(name, self) 95 if found_node: 96 return found_node 97 return None 98 99 def GetPath(self, name, path=None): 100 """ 101 Return list of node names from head to matching name. 102 Return None if not found. 103 104 Parameter descriptions: 105 name Name of node 106 path List of node names from head to current node 107 """ 108 if not path: 109 path = [] 110 if self.name == name: 111 path.append(self.name) 112 return path 113 for child in self.children: 114 match = child.GetPath(name, path + [self.name]) 115 if match: 116 return match 117 return None 118 119 def GetPathRegex(self, name, regex_str, path=None): 120 """ 121 Return list of node paths that end in name, or match regex_str. 122 Return empty list if not found. 123 124 Parameter descriptions: 125 name Name of node to search for 126 regex_str Regex string to match node names 127 path Path of node names from head to current node 128 """ 129 new_paths = [] 130 if not path: 131 path = [] 132 match = re.match(regex_str, self.name) 133 if (self.name == name) or (match): 134 new_paths.append(path + [self.name]) 135 for child in self.children: 136 return_paths = None 137 full_path = path + [self.name] 138 return_paths = child.GetPathRegex(name, regex_str, full_path) 139 for i in return_paths: 140 new_paths.append(i) 141 return new_paths 142 143 def MoveNode(self, from_name, to_name): 144 """ 145 Mode existing from_name node to become child of to_name node. 146 147 Parameter descriptions: 148 from_name Name of node to make a child of to_name 149 to_name Name of node to make parent of from_name 150 """ 151 parent_from_node = self.GetParentNode(from_name) 152 from_node = self.GetNode(from_name) 153 parent_from_node.RemoveChild(from_name) 154 to_node = self.GetNode(to_name) 155 to_node.AddChildNode(from_node) 156 157 def ReorderDeps(self, name, regex_str): 158 """ 159 Reorder dependency tree. If tree contains nodes with names that 160 match 'name' and 'regex_str', move 'regex_str' nodes that are 161 to the right of 'name' node, so that they become children of the 162 'name' node. 163 164 Parameter descriptions: 165 name Name of node to look for 166 regex_str Regex string to match names to 167 """ 168 name_path = self.GetPath(name) 169 if not name_path: 170 return 171 paths = self.GetPathRegex(name, regex_str) 172 is_name_in_paths = False 173 name_index = 0 174 for i in range(len(paths)): 175 path = paths[i] 176 if path[-1] == name: 177 is_name_in_paths = True 178 name_index = i 179 break 180 if not is_name_in_paths: 181 return 182 for i in range(name_index + 1, len(paths)): 183 path = paths[i] 184 if name in path: 185 continue 186 from_name = path[-1] 187 self.MoveNode(from_name, name) 188 189 def GetInstallList(self): 190 """ 191 Return post-order list of node names. 192 193 Parameter descriptions: 194 """ 195 install_list = [] 196 for child in self.children: 197 child_install_list = child.GetInstallList() 198 install_list.extend(child_install_list) 199 install_list.append(self.name) 200 return install_list 201 202 def PrintTree(self, level=0): 203 """ 204 Print pre-order node names with indentation denoting node depth level. 205 206 Parameter descriptions: 207 level Current depth level 208 """ 209 INDENT_PER_LEVEL = 4 210 print ' ' * (level * INDENT_PER_LEVEL) + self.name 211 for child in self.children: 212 child.PrintTree(level + 1) 213 214 215def check_call_cmd(dir, *cmd): 216 """ 217 Verbose prints the directory location the given command is called from and 218 the command, then executes the command using check_call. 219 220 Parameter descriptions: 221 dir Directory location command is to be called from 222 cmd List of parameters constructing the complete command 223 """ 224 printline(dir, ">", " ".join(cmd)) 225 check_call(cmd) 226 227 228def clone_pkg(pkg): 229 """ 230 Clone the given openbmc package's git repository from gerrit into 231 the WORKSPACE location 232 233 Parameter descriptions: 234 pkg Name of the package to clone 235 """ 236 pkg_dir = os.path.join(WORKSPACE, pkg) 237 if os.path.exists(os.path.join(pkg_dir, '.git')): 238 return pkg_dir 239 pkg_repo = urljoin('https://gerrit.openbmc-project.xyz/openbmc/', pkg) 240 os.mkdir(pkg_dir) 241 printline(pkg_dir, "> git clone", pkg_repo, "./") 242 return Repo.clone_from(pkg_repo, pkg_dir).working_dir 243 244 245def get_deps(configure_ac): 246 """ 247 Parse the given 'configure.ac' file for package dependencies and return 248 a list of the dependencies found. 249 250 Parameter descriptions: 251 configure_ac Opened 'configure.ac' file object 252 """ 253 line = "" 254 dep_pkgs = set() 255 for cfg_line in configure_ac: 256 # Remove whitespace & newline 257 cfg_line = cfg_line.rstrip() 258 # Check for line breaks 259 if cfg_line.endswith('\\'): 260 line += str(cfg_line[:-1]) 261 continue 262 line = line+cfg_line 263 264 # Find any defined dependency 265 line_has = lambda x: x if x in line else None 266 macros = set(filter(line_has, DEPENDENCIES.iterkeys())) 267 if len(macros) == 1: 268 macro = ''.join(macros) 269 deps = filter(line_has, DEPENDENCIES[macro].iterkeys()) 270 dep_pkgs.update(map(lambda x: DEPENDENCIES[macro][x], deps)) 271 272 line = "" 273 deps = list(dep_pkgs) 274 275 return deps 276 277 278make_parallel = [ 279 'make', 280 # Run enough jobs to saturate all the cpus 281 '-j', str(multiprocessing.cpu_count()), 282 # Don't start more jobs if the load avg is too high 283 '-l', str(multiprocessing.cpu_count()), 284 # Synchronize the output so logs aren't intermixed in stdout / stderr 285 '-O', 286] 287 288def install_deps(dep_list): 289 """ 290 Install each package in the ordered dep_list. 291 292 Parameter descriptions: 293 dep_list Ordered list of dependencies 294 """ 295 for pkg in dep_list: 296 pkgdir = os.path.join(WORKSPACE, pkg) 297 # Build & install this package 298 conf_flags = [ 299 '--disable-silent-rules', 300 '--enable-tests', 301 '--enable-code-coverage', 302 '--enable-valgrind', 303 ] 304 os.chdir(pkgdir) 305 # Add any necessary configure flags for package 306 if CONFIGURE_FLAGS.get(pkg) is not None: 307 conf_flags.extend(CONFIGURE_FLAGS.get(pkg)) 308 check_call_cmd(pkgdir, './bootstrap.sh') 309 check_call_cmd(pkgdir, './configure', *conf_flags) 310 check_call_cmd(pkgdir, *make_parallel) 311 check_call_cmd(pkgdir, *(make_parallel + [ 'install' ])) 312 313 314def build_dep_tree(pkg, pkgdir, dep_added, head, dep_tree=None): 315 """ 316 For each package(pkg), starting with the package to be unit tested, 317 parse its 'configure.ac' file from within the package's directory(pkgdir) 318 for each package dependency defined recursively doing the same thing 319 on each package found as a dependency. 320 321 Parameter descriptions: 322 pkg Name of the package 323 pkgdir Directory where package source is located 324 dep_added Current list of dependencies and added status 325 head Head node of the dependency tree 326 dep_tree Current dependency tree node 327 """ 328 if not dep_tree: 329 dep_tree = head 330 os.chdir(pkgdir) 331 # Open package's configure.ac 332 with open("/root/.depcache", "r") as depcache: 333 cached = depcache.readline() 334 with open("configure.ac", "rt") as configure_ac: 335 # Retrieve dependency list from package's configure.ac 336 configure_ac_deps = get_deps(configure_ac) 337 for dep_pkg in configure_ac_deps: 338 if dep_pkg in cached: 339 continue 340 # Dependency package not already known 341 if dep_added.get(dep_pkg) is None: 342 # Dependency package not added 343 new_child = dep_tree.AddChild(dep_pkg) 344 dep_added[dep_pkg] = False 345 dep_pkgdir = clone_pkg(dep_pkg) 346 # Determine this dependency package's 347 # dependencies and add them before 348 # returning to add this package 349 dep_added = build_dep_tree(dep_pkg, 350 dep_pkgdir, 351 dep_added, 352 head, 353 new_child) 354 else: 355 # Dependency package known and added 356 if dep_added[dep_pkg]: 357 continue 358 else: 359 # Cyclic dependency failure 360 raise Exception("Cyclic dependencies found in "+pkg) 361 362 if not dep_added[pkg]: 363 dep_added[pkg] = True 364 365 return dep_added 366 367def make_target_exists(target): 368 """ 369 Runs a check against the makefile in the current directory to determine 370 if the target exists so that it can be built. 371 372 Parameter descriptions: 373 target The make target we are checking 374 """ 375 try: 376 cmd = [ 'make', '-n', target ] 377 with open(os.devnull, 'w') as devnull: 378 check_call(cmd, stdout=devnull, stderr=devnull) 379 return True 380 except CalledProcessError: 381 return False 382 383def run_unit_tests(top_dir): 384 """ 385 Runs the unit tests for the package via `make check` 386 387 Parameter descriptions: 388 top_dir The root directory of our project 389 """ 390 try: 391 cmd = make_parallel + [ 'check' ] 392 for i in range(0, args.repeat): 393 check_call_cmd(top_dir, *cmd) 394 except CalledProcessError: 395 for root, _, files in os.walk(top_dir): 396 if 'test-suite.log' not in files: 397 continue 398 check_call_cmd(root, 'cat', os.path.join(root, 'test-suite.log')) 399 raise Exception('Unit tests failed') 400 401 402def maybe_run_valgrind(top_dir): 403 """ 404 Potentially runs the unit tests through valgrind for the package 405 via `make check-valgrind`. If the package does not have valgrind testing 406 then it just skips over this. 407 408 Parameter descriptions: 409 top_dir The root directory of our project 410 """ 411 if not make_target_exists('check-valgrind'): 412 return 413 414 try: 415 cmd = make_parallel + [ 'check-valgrind' ] 416 check_call_cmd(top_dir, *cmd) 417 except CalledProcessError: 418 for root, _, files in os.walk(top_dir): 419 for f in files: 420 if re.search('test-suite-[a-z]+.log', f) is None: 421 continue 422 check_call_cmd(root, 'cat', os.path.join(root, f)) 423 raise Exception('Valgrind tests failed') 424 425def maybe_run_coverage(top_dir): 426 """ 427 Potentially runs the unit tests through code coverage for the package 428 via `make check-code-coverage`. If the package does not have code coverage 429 testing then it just skips over this. 430 431 Parameter descriptions: 432 top_dir The root directory of our project 433 """ 434 if not make_target_exists('check-code-coverage'): 435 return 436 437 # Actually run code coverage 438 try: 439 cmd = make_parallel + [ 'check-code-coverage' ] 440 check_call_cmd(top_dir, *cmd) 441 except CalledProcessError: 442 raise Exception('Code coverage failed') 443 444if __name__ == '__main__': 445 # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS] 446 CONFIGURE_FLAGS = { 447 'phosphor-objmgr': ['--enable-unpatched-systemd'], 448 'sdbusplus': ['--enable-transaction'], 449 'phosphor-logging': 450 ['--enable-metadata-processing', 451 'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml'] 452 } 453 454 # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO] 455 DEPENDENCIES = { 456 'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'}, 457 'AC_CHECK_HEADER': { 458 'host-ipmid': 'phosphor-host-ipmid', 459 'sdbusplus': 'sdbusplus', 460 'phosphor-logging/log.hpp': 'phosphor-logging', 461 }, 462 'AC_PATH_PROG': {'sdbus++': 'sdbusplus'}, 463 'PKG_CHECK_MODULES': { 464 'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces', 465 'openpower-dbus-interfaces': 'openpower-dbus-interfaces', 466 'ibm-dbus-interfaces': 'ibm-dbus-interfaces', 467 'sdbusplus': 'sdbusplus', 468 'phosphor-logging': 'phosphor-logging', 469 }, 470 } 471 472 # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING] 473 DEPENDENCIES_REGEX = { 474 'phosphor-logging': '\S+-dbus-interfaces$' 475 } 476 477 # Set command line arguments 478 parser = argparse.ArgumentParser() 479 parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True, 480 help="Workspace directory location(i.e. /home)") 481 parser.add_argument("-p", "--package", dest="PACKAGE", required=True, 482 help="OpenBMC package to be unit tested") 483 parser.add_argument("-v", "--verbose", action="store_true", 484 help="Print additional package status messages") 485 parser.add_argument("-r", "--repeat", help="Repeat tests N times", 486 type=int, default=1) 487 args = parser.parse_args(sys.argv[1:]) 488 WORKSPACE = args.WORKSPACE 489 UNIT_TEST_PKG = args.PACKAGE 490 if args.verbose: 491 def printline(*line): 492 for arg in line: 493 print arg, 494 print 495 else: 496 printline = lambda *l: None 497 498 # First validate code formattting if repo has style formatting files. 499 # The format-code.sh checks for these files. 500 CODE_SCAN_DIR = WORKSPACE + "/" + UNIT_TEST_PKG 501 check_call_cmd(WORKSPACE, "./format-code.sh", CODE_SCAN_DIR) 502 503 # The rest of this script is CI testing, which currently only supports 504 # Automake based repos. Check if this repo is Automake, if not exit 505 if not os.path.isfile(CODE_SCAN_DIR + "/configure.ac"): 506 print "Not a supported repo for CI Tests, exit" 507 quit() 508 509 prev_umask = os.umask(000) 510 # Determine dependencies and add them 511 dep_added = dict() 512 dep_added[UNIT_TEST_PKG] = False 513 # Create dependency tree 514 dep_tree = DepTree(UNIT_TEST_PKG) 515 build_dep_tree(UNIT_TEST_PKG, 516 os.path.join(WORKSPACE, UNIT_TEST_PKG), 517 dep_added, 518 dep_tree) 519 520 # Reorder Dependency Tree 521 for pkg_name, regex_str in DEPENDENCIES_REGEX.iteritems(): 522 dep_tree.ReorderDeps(pkg_name, regex_str) 523 if args.verbose: 524 dep_tree.PrintTree() 525 install_list = dep_tree.GetInstallList() 526 # install reordered dependencies 527 install_deps(install_list) 528 top_dir = os.path.join(WORKSPACE, UNIT_TEST_PKG) 529 os.chdir(top_dir) 530 # Refresh dynamic linker run time bindings for dependencies 531 check_call_cmd(top_dir, 'ldconfig') 532 # Run package unit tests 533 run_unit_tests(top_dir) 534 maybe_run_valgrind(top_dir) 535 maybe_run_coverage(top_dir) 536 537 os.umask(prev_umask) 538