1#!/usr/bin/env python 2 3""" 4This script determines the given package's openbmc dependencies from its 5configure.ac file where it downloads, configures, builds, and installs each of 6these dependencies. Then the given package is configured, built, and installed 7prior to executing its unit tests. 8""" 9 10from git import Repo 11from urlparse import urljoin 12from subprocess import check_call, call, CalledProcessError 13import os 14import sys 15import argparse 16import multiprocessing 17import re 18 19 20class DepTree(): 21 """ 22 Represents package dependency tree, where each node is a DepTree with a 23 name and DepTree children. 24 """ 25 26 def __init__(self, name): 27 """ 28 Create new DepTree. 29 30 Parameter descriptions: 31 name Name of new tree node. 32 """ 33 self.name = name 34 self.children = list() 35 36 def AddChild(self, name): 37 """ 38 Add new child node to current node. 39 40 Parameter descriptions: 41 name Name of new child 42 """ 43 new_child = DepTree(name) 44 self.children.append(new_child) 45 return new_child 46 47 def AddChildNode(self, node): 48 """ 49 Add existing child node to current node. 50 51 Parameter descriptions: 52 node Tree node to add 53 """ 54 self.children.append(node) 55 56 def RemoveChild(self, name): 57 """ 58 Remove child node. 59 60 Parameter descriptions: 61 name Name of child to remove 62 """ 63 for child in self.children: 64 if child.name == name: 65 self.children.remove(child) 66 return 67 68 def GetNode(self, name): 69 """ 70 Return node with matching name. Return None if not found. 71 72 Parameter descriptions: 73 name Name of node to return 74 """ 75 if self.name == name: 76 return self 77 for child in self.children: 78 node = child.GetNode(name) 79 if node: 80 return node 81 return None 82 83 def GetParentNode(self, name, parent_node=None): 84 """ 85 Return parent of node with matching name. Return none if not found. 86 87 Parameter descriptions: 88 name Name of node to get parent of 89 parent_node Parent of current node 90 """ 91 if self.name == name: 92 return parent_node 93 for child in self.children: 94 found_node = child.GetParentNode(name, self) 95 if found_node: 96 return found_node 97 return None 98 99 def GetPath(self, name, path=None): 100 """ 101 Return list of node names from head to matching name. 102 Return None if not found. 103 104 Parameter descriptions: 105 name Name of node 106 path List of node names from head to current node 107 """ 108 if not path: 109 path = [] 110 if self.name == name: 111 path.append(self.name) 112 return path 113 for child in self.children: 114 match = child.GetPath(name, path + [self.name]) 115 if match: 116 return match 117 return None 118 119 def GetPathRegex(self, name, regex_str, path=None): 120 """ 121 Return list of node paths that end in name, or match regex_str. 122 Return empty list if not found. 123 124 Parameter descriptions: 125 name Name of node to search for 126 regex_str Regex string to match node names 127 path Path of node names from head to current node 128 """ 129 new_paths = [] 130 if not path: 131 path = [] 132 match = re.match(regex_str, self.name) 133 if (self.name == name) or (match): 134 new_paths.append(path + [self.name]) 135 for child in self.children: 136 return_paths = None 137 full_path = path + [self.name] 138 return_paths = child.GetPathRegex(name, regex_str, full_path) 139 for i in return_paths: 140 new_paths.append(i) 141 return new_paths 142 143 def MoveNode(self, from_name, to_name): 144 """ 145 Mode existing from_name node to become child of to_name node. 146 147 Parameter descriptions: 148 from_name Name of node to make a child of to_name 149 to_name Name of node to make parent of from_name 150 """ 151 parent_from_node = self.GetParentNode(from_name) 152 from_node = self.GetNode(from_name) 153 parent_from_node.RemoveChild(from_name) 154 to_node = self.GetNode(to_name) 155 to_node.AddChildNode(from_node) 156 157 def ReorderDeps(self, name, regex_str): 158 """ 159 Reorder dependency tree. If tree contains nodes with names that 160 match 'name' and 'regex_str', move 'regex_str' nodes that are 161 to the right of 'name' node, so that they become children of the 162 'name' node. 163 164 Parameter descriptions: 165 name Name of node to look for 166 regex_str Regex string to match names to 167 """ 168 name_path = self.GetPath(name) 169 if not name_path: 170 return 171 paths = self.GetPathRegex(name, regex_str) 172 is_name_in_paths = False 173 name_index = 0 174 for i in range(len(paths)): 175 path = paths[i] 176 if path[-1] == name: 177 is_name_in_paths = True 178 name_index = i 179 break 180 if not is_name_in_paths: 181 return 182 for i in range(name_index + 1, len(paths)): 183 path = paths[i] 184 if name in path: 185 continue 186 from_name = path[-1] 187 self.MoveNode(from_name, name) 188 189 def GetInstallList(self): 190 """ 191 Return post-order list of node names. 192 193 Parameter descriptions: 194 """ 195 install_list = [] 196 for child in self.children: 197 child_install_list = child.GetInstallList() 198 install_list.extend(child_install_list) 199 install_list.append(self.name) 200 return install_list 201 202 def PrintTree(self, level=0): 203 """ 204 Print pre-order node names with indentation denoting node depth level. 205 206 Parameter descriptions: 207 level Current depth level 208 """ 209 INDENT_PER_LEVEL = 4 210 print ' ' * (level * INDENT_PER_LEVEL) + self.name 211 for child in self.children: 212 child.PrintTree(level + 1) 213 214 215def check_call_cmd(dir, *cmd): 216 """ 217 Verbose prints the directory location the given command is called from and 218 the command, then executes the command using check_call. 219 220 Parameter descriptions: 221 dir Directory location command is to be called from 222 cmd List of parameters constructing the complete command 223 """ 224 printline(dir, ">", " ".join(cmd)) 225 check_call(cmd) 226 227 228def clone_pkg(pkg): 229 """ 230 Clone the given openbmc package's git repository from gerrit into 231 the WORKSPACE location 232 233 Parameter descriptions: 234 pkg Name of the package to clone 235 """ 236 pkg_dir = os.path.join(WORKSPACE, pkg) 237 if os.path.exists(os.path.join(pkg_dir, '.git')): 238 return pkg_dir 239 pkg_repo = urljoin('https://gerrit.openbmc-project.xyz/openbmc/', pkg) 240 os.mkdir(pkg_dir) 241 printline(pkg_dir, "> git clone", pkg_repo, "./") 242 return Repo.clone_from(pkg_repo, pkg_dir).working_dir 243 244 245def get_deps(configure_ac): 246 """ 247 Parse the given 'configure.ac' file for package dependencies and return 248 a list of the dependencies found. 249 250 Parameter descriptions: 251 configure_ac Opened 'configure.ac' file object 252 """ 253 line = "" 254 dep_pkgs = set() 255 for cfg_line in configure_ac: 256 # Remove whitespace & newline 257 cfg_line = cfg_line.rstrip() 258 # Check for line breaks 259 if cfg_line.endswith('\\'): 260 line += str(cfg_line[:-1]) 261 continue 262 line = line+cfg_line 263 264 # Find any defined dependency 265 line_has = lambda x: x if x in line else None 266 macros = set(filter(line_has, DEPENDENCIES.iterkeys())) 267 if len(macros) == 1: 268 macro = ''.join(macros) 269 deps = filter(line_has, DEPENDENCIES[macro].iterkeys()) 270 dep_pkgs.update(map(lambda x: DEPENDENCIES[macro][x], deps)) 271 272 line = "" 273 deps = list(dep_pkgs) 274 275 return deps 276 277 278make_parallel = [ 279 'make', 280 # Run enough jobs to saturate all the cpus 281 '-j', str(multiprocessing.cpu_count()), 282 # Don't start more jobs if the load avg is too high 283 '-l', str(multiprocessing.cpu_count()), 284 # Synchronize the output so logs aren't intermixed in stdout / stderr 285 '-O', 286] 287 288def install_deps(dep_list): 289 """ 290 Install each package in the ordered dep_list. 291 292 Parameter descriptions: 293 dep_list Ordered list of dependencies 294 """ 295 for pkg in dep_list: 296 pkgdir = os.path.join(WORKSPACE, pkg) 297 # Build & install this package 298 conf_flags = [ 299 '--disable-silent-rules', 300 '--enable-tests', 301 '--enable-code-coverage', 302 '--enable-valgrind', 303 ] 304 os.chdir(pkgdir) 305 # Add any necessary configure flags for package 306 if CONFIGURE_FLAGS.get(pkg) is not None: 307 conf_flags.extend(CONFIGURE_FLAGS.get(pkg)) 308 check_call_cmd(pkgdir, './bootstrap.sh') 309 check_call_cmd(pkgdir, './configure', *conf_flags) 310 check_call_cmd(pkgdir, *make_parallel) 311 check_call_cmd(pkgdir, *(make_parallel + [ 'install' ])) 312 313 314def build_dep_tree(pkg, pkgdir, dep_added, head, dep_tree=None): 315 """ 316 For each package(pkg), starting with the package to be unit tested, 317 parse its 'configure.ac' file from within the package's directory(pkgdir) 318 for each package dependency defined recursively doing the same thing 319 on each package found as a dependency. 320 321 Parameter descriptions: 322 pkg Name of the package 323 pkgdir Directory where package source is located 324 dep_added Current list of dependencies and added status 325 head Head node of the dependency tree 326 dep_tree Current dependency tree node 327 """ 328 if not dep_tree: 329 dep_tree = head 330 os.chdir(pkgdir) 331 # Open package's configure.ac 332 with open("/root/.depcache", "r") as depcache: 333 cached = depcache.readline() 334 with open("configure.ac", "rt") as configure_ac: 335 # Retrieve dependency list from package's configure.ac 336 configure_ac_deps = get_deps(configure_ac) 337 for dep_pkg in configure_ac_deps: 338 if dep_pkg in cached: 339 continue 340 # Dependency package not already known 341 if dep_added.get(dep_pkg) is None: 342 # Dependency package not added 343 new_child = dep_tree.AddChild(dep_pkg) 344 dep_added[dep_pkg] = False 345 dep_pkgdir = clone_pkg(dep_pkg) 346 # Determine this dependency package's 347 # dependencies and add them before 348 # returning to add this package 349 dep_added = build_dep_tree(dep_pkg, 350 dep_pkgdir, 351 dep_added, 352 head, 353 new_child) 354 else: 355 # Dependency package known and added 356 if dep_added[dep_pkg]: 357 continue 358 else: 359 # Cyclic dependency failure 360 raise Exception("Cyclic dependencies found in "+pkg) 361 362 if not dep_added[pkg]: 363 dep_added[pkg] = True 364 365 return dep_added 366 367 368if __name__ == '__main__': 369 # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS] 370 CONFIGURE_FLAGS = { 371 'phosphor-objmgr': ['--enable-unpatched-systemd'], 372 'sdbusplus': ['--enable-transaction'], 373 'phosphor-logging': 374 ['--enable-metadata-processing', 375 'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml'] 376 } 377 378 # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO] 379 DEPENDENCIES = { 380 'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'}, 381 'AC_CHECK_HEADER': { 382 'host-ipmid': 'phosphor-host-ipmid', 383 'sdbusplus': 'sdbusplus', 384 'phosphor-logging/log.hpp': 'phosphor-logging', 385 }, 386 'AC_PATH_PROG': {'sdbus++': 'sdbusplus'}, 387 'PKG_CHECK_MODULES': { 388 'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces', 389 'openpower-dbus-interfaces': 'openpower-dbus-interfaces', 390 'ibm-dbus-interfaces': 'ibm-dbus-interfaces', 391 'sdbusplus': 'sdbusplus', 392 'phosphor-logging': 'phosphor-logging', 393 }, 394 } 395 396 # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING] 397 DEPENDENCIES_REGEX = { 398 'phosphor-logging': '\S+-dbus-interfaces$' 399 } 400 401 # Set command line arguments 402 parser = argparse.ArgumentParser() 403 parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True, 404 help="Workspace directory location(i.e. /home)") 405 parser.add_argument("-p", "--package", dest="PACKAGE", required=True, 406 help="OpenBMC package to be unit tested") 407 parser.add_argument("-v", "--verbose", action="store_true", 408 help="Print additional package status messages") 409 parser.add_argument("-r", "--repeat", help="Repeat tests N times", 410 type=int, default=1) 411 args = parser.parse_args(sys.argv[1:]) 412 WORKSPACE = args.WORKSPACE 413 UNIT_TEST_PKG = args.PACKAGE 414 if args.verbose: 415 def printline(*line): 416 for arg in line: 417 print arg, 418 print 419 else: 420 printline = lambda *l: None 421 422 # First validate code formattting if repo has style formatting files. 423 # The format-code.sh checks for these files. 424 CODE_SCAN_DIR = WORKSPACE + "/" + UNIT_TEST_PKG 425 check_call_cmd(WORKSPACE, "./format-code.sh", CODE_SCAN_DIR) 426 427 # The rest of this script is CI testing, which currently only supports 428 # Automake based repos. Check if this repo is Automake, if not exit 429 if not os.path.isfile(CODE_SCAN_DIR + "/configure.ac"): 430 print "Not a supported repo for CI Tests, exit" 431 quit() 432 433 prev_umask = os.umask(000) 434 # Determine dependencies and add them 435 dep_added = dict() 436 dep_added[UNIT_TEST_PKG] = False 437 # Create dependency tree 438 dep_tree = DepTree(UNIT_TEST_PKG) 439 build_dep_tree(UNIT_TEST_PKG, 440 os.path.join(WORKSPACE, UNIT_TEST_PKG), 441 dep_added, 442 dep_tree) 443 444 # Reorder Dependency Tree 445 for pkg_name, regex_str in DEPENDENCIES_REGEX.iteritems(): 446 dep_tree.ReorderDeps(pkg_name, regex_str) 447 if args.verbose: 448 dep_tree.PrintTree() 449 install_list = dep_tree.GetInstallList() 450 # install reordered dependencies 451 install_deps(install_list) 452 os.chdir(os.path.join(WORKSPACE, UNIT_TEST_PKG)) 453 # Refresh dynamic linker run time bindings for dependencies 454 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), 'ldconfig') 455 # Run package unit tests 456 try: 457 cmd = make_parallel + [ 'check' ] 458 for i in range(0, args.repeat): 459 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), *cmd) 460 except CalledProcessError: 461 for root, _, files in os.walk(os.path.join(WORKSPACE, UNIT_TEST_PKG)): 462 if 'test-suite.log' not in files: 463 continue 464 check_call_cmd(root, 'cat', os.path.join(root, 'test-suite.log')) 465 raise Exception('Unit tests failed') 466 467 with open(os.devnull, 'w') as devnull: 468 # Run unit tests through valgrind if it exists 469 top_dir = os.path.join(WORKSPACE, UNIT_TEST_PKG) 470 try: 471 cmd = [ 'make', '-n', 'check-valgrind' ] 472 check_call(cmd, stdout=devnull, stderr=devnull) 473 try: 474 cmd = make_parallel + [ 'check-valgrind' ] 475 check_call_cmd(top_dir, *cmd) 476 except CalledProcessError: 477 for root, _, files in os.walk(top_dir): 478 for f in files: 479 if re.search('test-suite-[a-z]+.log', f) is None: 480 continue 481 check_call_cmd(root, 'cat', os.path.join(root, f)) 482 raise Exception('Valgrind tests failed') 483 except CalledProcessError: 484 pass 485 486 # Run code coverage if possible 487 try: 488 cmd = [ 'make', '-n', 'check-code-coverage' ] 489 check_call(cmd, stdout=devnull, stderr=devnull) 490 try: 491 cmd = make_parallel + [ 'check-code-coverage' ] 492 check_call_cmd(top_dir, *cmd) 493 except CalledProcessError: 494 raise Exception('Code coverage failed') 495 except CalledProcessError: 496 pass 497 498 os.umask(prev_umask) 499