1#!/usr/bin/env python 2 3""" 4This script determines the given package's openbmc dependencies from its 5configure.ac file where it downloads, configures, builds, and installs each of 6these dependencies. Then the given package is configured, built, and installed 7prior to executing its unit tests. 8""" 9 10from git import Repo 11from urlparse import urljoin 12from subprocess import check_call, call, CalledProcessError 13import os 14import sys 15import argparse 16import re 17 18 19class DepTree(): 20 """ 21 Represents package dependency tree, where each node is a DepTree with a 22 name and DepTree children. 23 """ 24 25 def __init__(self, name): 26 """ 27 Create new DepTree. 28 29 Parameter descriptions: 30 name Name of new tree node. 31 """ 32 self.name = name 33 self.children = list() 34 35 def AddChild(self, name): 36 """ 37 Add new child node to current node. 38 39 Parameter descriptions: 40 name Name of new child 41 """ 42 new_child = DepTree(name) 43 self.children.append(new_child) 44 return new_child 45 46 def AddChildNode(self, node): 47 """ 48 Add existing child node to current node. 49 50 Parameter descriptions: 51 node Tree node to add 52 """ 53 self.children.append(node) 54 55 def RemoveChild(self, name): 56 """ 57 Remove child node. 58 59 Parameter descriptions: 60 name Name of child to remove 61 """ 62 for child in self.children: 63 if child.name == name: 64 self.children.remove(child) 65 return 66 67 def GetNode(self, name): 68 """ 69 Return node with matching name. Return None if not found. 70 71 Parameter descriptions: 72 name Name of node to return 73 """ 74 if self.name == name: 75 return self 76 for child in self.children: 77 node = child.GetNode(name) 78 if node: 79 return node 80 return None 81 82 def GetParentNode(self, name, parent_node=None): 83 """ 84 Return parent of node with matching name. Return none if not found. 85 86 Parameter descriptions: 87 name Name of node to get parent of 88 parent_node Parent of current node 89 """ 90 if self.name == name: 91 return parent_node 92 for child in self.children: 93 found_node = child.GetParentNode(name, self) 94 if found_node: 95 return found_node 96 return None 97 98 def GetPath(self, name, path=None): 99 """ 100 Return list of node names from head to matching name. 101 Return None if not found. 102 103 Parameter descriptions: 104 name Name of node 105 path List of node names from head to current node 106 """ 107 if not path: 108 path = [] 109 if self.name == name: 110 path.append(self.name) 111 return path 112 for child in self.children: 113 match = child.GetPath(name, path + [self.name]) 114 if match: 115 return match 116 return None 117 118 def GetPathRegex(self, name, regex_str, path=None): 119 """ 120 Return list of node paths that end in name, or match regex_str. 121 Return empty list if not found. 122 123 Parameter descriptions: 124 name Name of node to search for 125 regex_str Regex string to match node names 126 path Path of node names from head to current node 127 """ 128 new_paths = [] 129 if not path: 130 path = [] 131 match = re.match(regex_str, self.name) 132 if (self.name == name) or (match): 133 new_paths.append(path + [self.name]) 134 for child in self.children: 135 return_paths = None 136 full_path = path + [self.name] 137 return_paths = child.GetPathRegex(name, regex_str, full_path) 138 for i in return_paths: 139 new_paths.append(i) 140 return new_paths 141 142 def MoveNode(self, from_name, to_name): 143 """ 144 Mode existing from_name node to become child of to_name node. 145 146 Parameter descriptions: 147 from_name Name of node to make a child of to_name 148 to_name Name of node to make parent of from_name 149 """ 150 parent_from_node = self.GetParentNode(from_name) 151 from_node = self.GetNode(from_name) 152 parent_from_node.RemoveChild(from_name) 153 to_node = self.GetNode(to_name) 154 to_node.AddChildNode(from_node) 155 156 def ReorderDeps(self, name, regex_str): 157 """ 158 Reorder dependency tree. If tree contains nodes with names that 159 match 'name' and 'regex_str', move 'regex_str' nodes that are 160 to the right of 'name' node, so that they become children of the 161 'name' node. 162 163 Parameter descriptions: 164 name Name of node to look for 165 regex_str Regex string to match names to 166 """ 167 name_path = self.GetPath(name) 168 if not name_path: 169 return 170 paths = self.GetPathRegex(name, regex_str) 171 is_name_in_paths = False 172 name_index = 0 173 for i in range(len(paths)): 174 path = paths[i] 175 if path[-1] == name: 176 is_name_in_paths = True 177 name_index = i 178 break 179 if not is_name_in_paths: 180 return 181 for i in range(name_index + 1, len(paths)): 182 path = paths[i] 183 if name in path: 184 continue 185 from_name = path[-1] 186 self.MoveNode(from_name, name) 187 188 def GetInstallList(self): 189 """ 190 Return post-order list of node names. 191 192 Parameter descriptions: 193 """ 194 install_list = [] 195 for child in self.children: 196 child_install_list = child.GetInstallList() 197 install_list.extend(child_install_list) 198 install_list.append(self.name) 199 return install_list 200 201 def PrintTree(self, level=0): 202 """ 203 Print pre-order node names with indentation denoting node depth level. 204 205 Parameter descriptions: 206 level Current depth level 207 """ 208 INDENT_PER_LEVEL = 4 209 print ' ' * (level * INDENT_PER_LEVEL) + self.name 210 for child in self.children: 211 child.PrintTree(level + 1) 212 213 214def check_call_cmd(dir, *cmd): 215 """ 216 Verbose prints the directory location the given command is called from and 217 the command, then executes the command using check_call. 218 219 Parameter descriptions: 220 dir Directory location command is to be called from 221 cmd List of parameters constructing the complete command 222 """ 223 printline(dir, ">", " ".join(cmd)) 224 check_call(cmd) 225 226 227def clone_pkg(pkg): 228 """ 229 Clone the given openbmc package's git repository from gerrit into 230 the WORKSPACE location 231 232 Parameter descriptions: 233 pkg Name of the package to clone 234 """ 235 pkg_dir = os.path.join(WORKSPACE, pkg) 236 if os.path.exists(os.path.join(pkg_dir, '.git')): 237 return pkg_dir 238 pkg_repo = urljoin('https://gerrit.openbmc-project.xyz/openbmc/', pkg) 239 os.mkdir(pkg_dir) 240 printline(pkg_dir, "> git clone", pkg_repo, "./") 241 return Repo.clone_from(pkg_repo, pkg_dir).working_dir 242 243 244def get_deps(configure_ac): 245 """ 246 Parse the given 'configure.ac' file for package dependencies and return 247 a list of the dependencies found. 248 249 Parameter descriptions: 250 configure_ac Opened 'configure.ac' file object 251 """ 252 line = "" 253 dep_pkgs = set() 254 for cfg_line in configure_ac: 255 # Remove whitespace & newline 256 cfg_line = cfg_line.rstrip() 257 # Check for line breaks 258 if cfg_line.endswith('\\'): 259 line += str(cfg_line[:-1]) 260 continue 261 line = line+cfg_line 262 263 # Find any defined dependency 264 line_has = lambda x: x if x in line else None 265 macros = set(filter(line_has, DEPENDENCIES.iterkeys())) 266 if len(macros) == 1: 267 macro = ''.join(macros) 268 deps = filter(line_has, DEPENDENCIES[macro].iterkeys()) 269 dep_pkgs.update(map(lambda x: DEPENDENCIES[macro][x], deps)) 270 271 line = "" 272 deps = list(dep_pkgs) 273 274 return deps 275 276 277def install_deps(dep_list): 278 """ 279 Install each package in the ordered dep_list. 280 281 Parameter descriptions: 282 dep_list Ordered list of dependencies 283 """ 284 for pkg in dep_list: 285 pkgdir = os.path.join(WORKSPACE, pkg) 286 # Build & install this package 287 conf_flags = [ 288 '--disable-silent-rules', 289 '--enable-tests', 290 '--enable-code-coverage', 291 '--enable-valgrind', 292 ] 293 os.chdir(pkgdir) 294 # Add any necessary configure flags for package 295 if CONFIGURE_FLAGS.get(pkg) is not None: 296 conf_flags.extend(CONFIGURE_FLAGS.get(pkg)) 297 check_call_cmd(pkgdir, './bootstrap.sh') 298 check_call_cmd(pkgdir, './configure', *conf_flags) 299 check_call_cmd(pkgdir, 'make') 300 check_call_cmd(pkgdir, 'make', 'install') 301 302 303def build_dep_tree(pkg, pkgdir, dep_added, head, dep_tree=None): 304 """ 305 For each package(pkg), starting with the package to be unit tested, 306 parse its 'configure.ac' file from within the package's directory(pkgdir) 307 for each package dependency defined recursively doing the same thing 308 on each package found as a dependency. 309 310 Parameter descriptions: 311 pkg Name of the package 312 pkgdir Directory where package source is located 313 dep_added Current list of dependencies and added status 314 head Head node of the dependency tree 315 dep_tree Current dependency tree node 316 """ 317 if not dep_tree: 318 dep_tree = head 319 os.chdir(pkgdir) 320 # Open package's configure.ac 321 with open("/root/.depcache", "r") as depcache: 322 cached = depcache.readline() 323 with open("configure.ac", "rt") as configure_ac: 324 # Retrieve dependency list from package's configure.ac 325 configure_ac_deps = get_deps(configure_ac) 326 for dep_pkg in configure_ac_deps: 327 if dep_pkg in cached: 328 continue 329 # Dependency package not already known 330 if dep_added.get(dep_pkg) is None: 331 # Dependency package not added 332 new_child = dep_tree.AddChild(dep_pkg) 333 dep_added[dep_pkg] = False 334 dep_pkgdir = clone_pkg(dep_pkg) 335 # Determine this dependency package's 336 # dependencies and add them before 337 # returning to add this package 338 dep_added = build_dep_tree(dep_pkg, 339 dep_pkgdir, 340 dep_added, 341 head, 342 new_child) 343 else: 344 # Dependency package known and added 345 if dep_added[dep_pkg]: 346 continue 347 else: 348 # Cyclic dependency failure 349 raise Exception("Cyclic dependencies found in "+pkg) 350 351 if not dep_added[pkg]: 352 dep_added[pkg] = True 353 354 return dep_added 355 356 357if __name__ == '__main__': 358 # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS] 359 CONFIGURE_FLAGS = { 360 'phosphor-objmgr': ['--enable-unpatched-systemd'], 361 'sdbusplus': ['--enable-transaction'], 362 'phosphor-logging': 363 ['--enable-metadata-processing', 364 'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml'] 365 } 366 367 # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO] 368 DEPENDENCIES = { 369 'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'}, 370 'AC_CHECK_HEADER': { 371 'host-ipmid': 'phosphor-host-ipmid', 372 'sdbusplus': 'sdbusplus', 373 'phosphor-logging/log.hpp': 'phosphor-logging', 374 }, 375 'AC_PATH_PROG': {'sdbus++': 'sdbusplus'}, 376 'PKG_CHECK_MODULES': { 377 'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces', 378 'openpower-dbus-interfaces': 'openpower-dbus-interfaces', 379 'ibm-dbus-interfaces': 'ibm-dbus-interfaces', 380 'sdbusplus': 'sdbusplus', 381 'phosphor-logging': 'phosphor-logging', 382 }, 383 } 384 385 # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING] 386 DEPENDENCIES_REGEX = { 387 'phosphor-logging': '\S+-dbus-interfaces$' 388 } 389 390 # Set command line arguments 391 parser = argparse.ArgumentParser() 392 parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True, 393 help="Workspace directory location(i.e. /home)") 394 parser.add_argument("-p", "--package", dest="PACKAGE", required=True, 395 help="OpenBMC package to be unit tested") 396 parser.add_argument("-v", "--verbose", action="store_true", 397 help="Print additional package status messages") 398 parser.add_argument("-r", "--repeat", help="Repeat tests N times", 399 type=int, default=1) 400 args = parser.parse_args(sys.argv[1:]) 401 WORKSPACE = args.WORKSPACE 402 UNIT_TEST_PKG = args.PACKAGE 403 if args.verbose: 404 def printline(*line): 405 for arg in line: 406 print arg, 407 print 408 else: 409 printline = lambda *l: None 410 411 # First validate code formattting if repo has style formatting files. 412 # The format-code.sh checks for these files. 413 CODE_SCAN_DIR = WORKSPACE + "/" + UNIT_TEST_PKG 414 check_call_cmd(WORKSPACE, "./format-code.sh", CODE_SCAN_DIR) 415 416 # The rest of this script is CI testing, which currently only supports 417 # Automake based repos. Check if this repo is Automake, if not exit 418 if not os.path.isfile(CODE_SCAN_DIR + "/configure.ac"): 419 print "Not a supported repo for CI Tests, exit" 420 quit() 421 422 prev_umask = os.umask(000) 423 # Determine dependencies and add them 424 dep_added = dict() 425 dep_added[UNIT_TEST_PKG] = False 426 # Create dependency tree 427 dep_tree = DepTree(UNIT_TEST_PKG) 428 build_dep_tree(UNIT_TEST_PKG, 429 os.path.join(WORKSPACE, UNIT_TEST_PKG), 430 dep_added, 431 dep_tree) 432 433 # Reorder Dependency Tree 434 for pkg_name, regex_str in DEPENDENCIES_REGEX.iteritems(): 435 dep_tree.ReorderDeps(pkg_name, regex_str) 436 if args.verbose: 437 dep_tree.PrintTree() 438 install_list = dep_tree.GetInstallList() 439 # install reordered dependencies 440 install_deps(install_list) 441 os.chdir(os.path.join(WORKSPACE, UNIT_TEST_PKG)) 442 # Refresh dynamic linker run time bindings for dependencies 443 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), 'ldconfig') 444 # Run package unit tests 445 try: 446 cmd = [ 'make', 'check' ] 447 for i in range(0, args.repeat): 448 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), *cmd) 449 except CalledProcessError: 450 for root, _, files in os.walk(os.path.join(WORKSPACE, UNIT_TEST_PKG)): 451 if 'test-suite.log' not in files: 452 continue 453 check_call_cmd(root, 'cat', os.path.join(root, 'test-suite.log')) 454 raise Exception('Unit tests failed') 455 456 with open(os.devnull, 'w') as devnull: 457 # Run unit tests through valgrind if it exists 458 top_dir = os.path.join(WORKSPACE, UNIT_TEST_PKG) 459 try: 460 cmd = [ 'make', '-n', 'check-valgrind' ] 461 check_call(cmd, stdout=devnull, stderr=devnull) 462 try: 463 cmd = [ 'make', 'check-valgrind' ] 464 check_call_cmd(top_dir, *cmd) 465 except CalledProcessError: 466 for root, _, files in os.walk(top_dir): 467 for f in files: 468 if re.search('test-suite-[a-z]+.log', f) is None: 469 continue 470 check_call_cmd(root, 'cat', os.path.join(root, f)) 471 raise Exception('Valgrind tests failed') 472 except CalledProcessError: 473 pass 474 475 # Run code coverage if possible 476 try: 477 cmd = [ 'make', '-n', 'check-code-coverage' ] 478 check_call(cmd, stdout=devnull, stderr=devnull) 479 try: 480 cmd = [ 'make', 'check-code-coverage' ] 481 check_call_cmd(top_dir, *cmd) 482 except CalledProcessError: 483 raise Exception('Code coverage failed') 484 except CalledProcessError: 485 pass 486 487 os.umask(prev_umask) 488