1#!/usr/bin/env python 2 3""" 4This script determines the given package's openbmc dependencies from its 5configure.ac file where it downloads, configures, builds, and installs each of 6these dependencies. Then the given package is configured, built, and installed 7prior to executing its unit tests. 8""" 9 10from git import Repo 11from urlparse import urljoin 12from subprocess import check_call, call, CalledProcessError 13import os 14import sys 15import argparse 16import re 17 18 19class DepTree(): 20 """ 21 Represents package dependency tree, where each node is a DepTree with a 22 name and DepTree children. 23 """ 24 25 def __init__(self, name): 26 """ 27 Create new DepTree. 28 29 Parameter descriptions: 30 name Name of new tree node. 31 """ 32 self.name = name 33 self.children = list() 34 35 def AddChild(self, name): 36 """ 37 Add new child node to current node. 38 39 Parameter descriptions: 40 name Name of new child 41 """ 42 new_child = DepTree(name) 43 self.children.append(new_child) 44 return new_child 45 46 def AddChildNode(self, node): 47 """ 48 Add existing child node to current node. 49 50 Parameter descriptions: 51 node Tree node to add 52 """ 53 self.children.append(node) 54 55 def RemoveChild(self, name): 56 """ 57 Remove child node. 58 59 Parameter descriptions: 60 name Name of child to remove 61 """ 62 for child in self.children: 63 if child.name == name: 64 self.children.remove(child) 65 return 66 67 def GetNode(self, name): 68 """ 69 Return node with matching name. Return None if not found. 70 71 Parameter descriptions: 72 name Name of node to return 73 """ 74 if self.name == name: 75 return self 76 for child in self.children: 77 node = child.GetNode(name) 78 if node: 79 return node 80 return None 81 82 def GetParentNode(self, name, parent_node=None): 83 """ 84 Return parent of node with matching name. Return none if not found. 85 86 Parameter descriptions: 87 name Name of node to get parent of 88 parent_node Parent of current node 89 """ 90 if self.name == name: 91 return parent_node 92 for child in self.children: 93 found_node = child.GetParentNode(name, self) 94 if found_node: 95 return found_node 96 return None 97 98 def GetPath(self, name, path=None): 99 """ 100 Return list of node names from head to matching name. 101 Return None if not found. 102 103 Parameter descriptions: 104 name Name of node 105 path List of node names from head to current node 106 """ 107 if not path: 108 path = [] 109 if self.name == name: 110 path.append(self.name) 111 return path 112 for child in self.children: 113 match = child.GetPath(name, path + [self.name]) 114 if match: 115 return match 116 return None 117 118 def GetPathRegex(self, name, regex_str, path=None): 119 """ 120 Return list of node paths that end in name, or match regex_str. 121 Return empty list if not found. 122 123 Parameter descriptions: 124 name Name of node to search for 125 regex_str Regex string to match node names 126 path Path of node names from head to current node 127 """ 128 new_paths = [] 129 if not path: 130 path = [] 131 match = re.match(regex_str, self.name) 132 if (self.name == name) or (match): 133 new_paths.append(path + [self.name]) 134 for child in self.children: 135 return_paths = None 136 full_path = path + [self.name] 137 return_paths = child.GetPathRegex(name, regex_str, full_path) 138 for i in return_paths: 139 new_paths.append(i) 140 return new_paths 141 142 def MoveNode(self, from_name, to_name): 143 """ 144 Mode existing from_name node to become child of to_name node. 145 146 Parameter descriptions: 147 from_name Name of node to make a child of to_name 148 to_name Name of node to make parent of from_name 149 """ 150 parent_from_node = self.GetParentNode(from_name) 151 from_node = self.GetNode(from_name) 152 parent_from_node.RemoveChild(from_name) 153 to_node = self.GetNode(to_name) 154 to_node.AddChildNode(from_node) 155 156 def ReorderDeps(self, name, regex_str): 157 """ 158 Reorder dependency tree. If tree contains nodes with names that 159 match 'name' and 'regex_str', move 'regex_str' nodes that are 160 to the right of 'name' node, so that they become children of the 161 'name' node. 162 163 Parameter descriptions: 164 name Name of node to look for 165 regex_str Regex string to match names to 166 """ 167 name_path = self.GetPath(name) 168 if not name_path: 169 return 170 paths = self.GetPathRegex(name, regex_str) 171 is_name_in_paths = False 172 name_index = 0 173 for i in range(len(paths)): 174 path = paths[i] 175 if path[-1] == name: 176 is_name_in_paths = True 177 name_index = i 178 break 179 if not is_name_in_paths: 180 return 181 for i in range(name_index + 1, len(paths)): 182 path = paths[i] 183 if name in path: 184 continue 185 from_name = path[-1] 186 self.MoveNode(from_name, name) 187 188 def GetInstallList(self): 189 """ 190 Return post-order list of node names. 191 192 Parameter descriptions: 193 """ 194 install_list = [] 195 for child in self.children: 196 child_install_list = child.GetInstallList() 197 install_list.extend(child_install_list) 198 install_list.append(self.name) 199 return install_list 200 201 def PrintTree(self, level=0): 202 """ 203 Print pre-order node names with indentation denoting node depth level. 204 205 Parameter descriptions: 206 level Current depth level 207 """ 208 INDENT_PER_LEVEL = 4 209 print ' ' * (level * INDENT_PER_LEVEL) + self.name 210 for child in self.children: 211 child.PrintTree(level + 1) 212 213 214def check_call_cmd(dir, *cmd): 215 """ 216 Verbose prints the directory location the given command is called from and 217 the command, then executes the command using check_call. 218 219 Parameter descriptions: 220 dir Directory location command is to be called from 221 cmd List of parameters constructing the complete command 222 """ 223 printline(dir, ">", " ".join(cmd)) 224 check_call(cmd) 225 226 227def clone_pkg(pkg): 228 """ 229 Clone the given openbmc package's git repository from gerrit into 230 the WORKSPACE location 231 232 Parameter descriptions: 233 pkg Name of the package to clone 234 """ 235 pkg_dir = os.path.join(WORKSPACE, pkg) 236 if os.path.exists(os.path.join(pkg_dir, '.git')): 237 return pkg_dir 238 pkg_repo = urljoin('https://gerrit.openbmc-project.xyz/openbmc/', pkg) 239 os.mkdir(pkg_dir) 240 printline(pkg_dir, "> git clone", pkg_repo, "./") 241 return Repo.clone_from(pkg_repo, pkg_dir).working_dir 242 243 244def get_deps(configure_ac): 245 """ 246 Parse the given 'configure.ac' file for package dependencies and return 247 a list of the dependencies found. 248 249 Parameter descriptions: 250 configure_ac Opened 'configure.ac' file object 251 """ 252 line = "" 253 dep_pkgs = set() 254 for cfg_line in configure_ac: 255 # Remove whitespace & newline 256 cfg_line = cfg_line.rstrip() 257 # Check for line breaks 258 if cfg_line.endswith('\\'): 259 line += str(cfg_line[:-1]) 260 continue 261 line = line+cfg_line 262 263 # Find any defined dependency 264 line_has = lambda x: x if x in line else None 265 macros = set(filter(line_has, DEPENDENCIES.iterkeys())) 266 if len(macros) == 1: 267 macro = ''.join(macros) 268 deps = filter(line_has, DEPENDENCIES[macro].iterkeys()) 269 dep_pkgs.update(map(lambda x: DEPENDENCIES[macro][x], deps)) 270 271 line = "" 272 deps = list(dep_pkgs) 273 274 return deps 275 276 277def install_deps(dep_list): 278 """ 279 Install each package in the ordered dep_list. 280 281 Parameter descriptions: 282 dep_list Ordered list of dependencies 283 """ 284 for pkg in dep_list: 285 pkgdir = os.path.join(WORKSPACE, pkg) 286 # Build & install this package 287 conf_flags = [ 288 '--disable-silent-rules', 289 '--enable-code-coverage', 290 '--enable-valgrind', 291 ] 292 os.chdir(pkgdir) 293 # Add any necessary configure flags for package 294 if CONFIGURE_FLAGS.get(pkg) is not None: 295 conf_flags.extend(CONFIGURE_FLAGS.get(pkg)) 296 check_call_cmd(pkgdir, './bootstrap.sh') 297 check_call_cmd(pkgdir, './configure', *conf_flags) 298 check_call_cmd(pkgdir, 'make') 299 check_call_cmd(pkgdir, 'make', 'install') 300 301 302def build_dep_tree(pkg, pkgdir, dep_added, head, dep_tree=None): 303 """ 304 For each package(pkg), starting with the package to be unit tested, 305 parse its 'configure.ac' file from within the package's directory(pkgdir) 306 for each package dependency defined recursively doing the same thing 307 on each package found as a dependency. 308 309 Parameter descriptions: 310 pkg Name of the package 311 pkgdir Directory where package source is located 312 dep_added Current list of dependencies and added status 313 head Head node of the dependency tree 314 dep_tree Current dependency tree node 315 """ 316 if not dep_tree: 317 dep_tree = head 318 os.chdir(pkgdir) 319 # Open package's configure.ac 320 with open("/root/.depcache", "r") as depcache: 321 cached = depcache.readline() 322 with open("configure.ac", "rt") as configure_ac: 323 # Retrieve dependency list from package's configure.ac 324 configure_ac_deps = get_deps(configure_ac) 325 for dep_pkg in configure_ac_deps: 326 if dep_pkg in cached: 327 continue 328 # Dependency package not already known 329 if dep_added.get(dep_pkg) is None: 330 # Dependency package not added 331 new_child = dep_tree.AddChild(dep_pkg) 332 dep_added[dep_pkg] = False 333 dep_pkgdir = clone_pkg(dep_pkg) 334 # Determine this dependency package's 335 # dependencies and add them before 336 # returning to add this package 337 dep_added = build_dep_tree(dep_pkg, 338 dep_pkgdir, 339 dep_added, 340 head, 341 new_child) 342 else: 343 # Dependency package known and added 344 if dep_added[dep_pkg]: 345 continue 346 else: 347 # Cyclic dependency failure 348 raise Exception("Cyclic dependencies found in "+pkg) 349 350 if not dep_added[pkg]: 351 dep_added[pkg] = True 352 353 return dep_added 354 355 356if __name__ == '__main__': 357 # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS] 358 CONFIGURE_FLAGS = { 359 'phosphor-objmgr': ['--enable-unpatched-systemd'], 360 'sdbusplus': ['--enable-transaction'], 361 'phosphor-logging': 362 ['--enable-metadata-processing', 363 'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml'] 364 } 365 366 # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO] 367 DEPENDENCIES = { 368 'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'}, 369 'AC_CHECK_HEADER': { 370 'host-ipmid': 'phosphor-host-ipmid', 371 'sdbusplus': 'sdbusplus', 372 'phosphor-logging/log.hpp': 'phosphor-logging', 373 }, 374 'AC_PATH_PROG': {'sdbus++': 'sdbusplus'}, 375 'PKG_CHECK_MODULES': { 376 'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces', 377 'openpower-dbus-interfaces': 'openpower-dbus-interfaces', 378 'ibm-dbus-interfaces': 'ibm-dbus-interfaces', 379 'sdbusplus': 'sdbusplus', 380 'phosphor-logging': 'phosphor-logging', 381 }, 382 } 383 384 # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING] 385 DEPENDENCIES_REGEX = { 386 'phosphor-logging': '\S+-dbus-interfaces$' 387 } 388 389 # Set command line arguments 390 parser = argparse.ArgumentParser() 391 parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True, 392 help="Workspace directory location(i.e. /home)") 393 parser.add_argument("-p", "--package", dest="PACKAGE", required=True, 394 help="OpenBMC package to be unit tested") 395 parser.add_argument("-v", "--verbose", action="store_true", 396 help="Print additional package status messages") 397 parser.add_argument("-r", "--repeat", help="Repeat tests N times", 398 type=int, default=1) 399 args = parser.parse_args(sys.argv[1:]) 400 WORKSPACE = args.WORKSPACE 401 UNIT_TEST_PKG = args.PACKAGE 402 if args.verbose: 403 def printline(*line): 404 for arg in line: 405 print arg, 406 print 407 else: 408 printline = lambda *l: None 409 410 # First validate code formattting if repo has style formatting files. 411 # The format-code.sh checks for these files. 412 CODE_SCAN_DIR = WORKSPACE + "/" + UNIT_TEST_PKG 413 check_call_cmd(WORKSPACE, "./format-code.sh", CODE_SCAN_DIR) 414 415 # The rest of this script is CI testing, which currently only supports 416 # Automake based repos. Check if this repo is Automake, if not exit 417 if not os.path.isfile(CODE_SCAN_DIR + "/configure.ac"): 418 print "Not a supported repo for CI Tests, exit" 419 quit() 420 421 prev_umask = os.umask(000) 422 # Determine dependencies and add them 423 dep_added = dict() 424 dep_added[UNIT_TEST_PKG] = False 425 # Create dependency tree 426 dep_tree = DepTree(UNIT_TEST_PKG) 427 build_dep_tree(UNIT_TEST_PKG, 428 os.path.join(WORKSPACE, UNIT_TEST_PKG), 429 dep_added, 430 dep_tree) 431 432 # Reorder Dependency Tree 433 for pkg_name, regex_str in DEPENDENCIES_REGEX.iteritems(): 434 dep_tree.ReorderDeps(pkg_name, regex_str) 435 if args.verbose: 436 dep_tree.PrintTree() 437 install_list = dep_tree.GetInstallList() 438 # install reordered dependencies 439 install_deps(install_list) 440 os.chdir(os.path.join(WORKSPACE, UNIT_TEST_PKG)) 441 # Refresh dynamic linker run time bindings for dependencies 442 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), 'ldconfig') 443 # Run package unit tests 444 try: 445 cmd = [ 'make', 'check' ] 446 for i in range(0, args.repeat): 447 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), *cmd) 448 except CalledProcessError: 449 for root, _, files in os.walk(os.path.join(WORKSPACE, UNIT_TEST_PKG)): 450 if 'test-suite.log' not in files: 451 continue 452 check_call_cmd(root, 'cat', os.path.join(root, 'test-suite.log')) 453 raise Exception('Unit tests failed') 454 455 with open(os.devnull, 'w') as devnull: 456 # Run unit tests through valgrind if it exists 457 top_dir = os.path.join(WORKSPACE, UNIT_TEST_PKG) 458 try: 459 cmd = [ 'make', '-n', 'check-valgrind' ] 460 check_call(cmd, stdout=devnull, stderr=devnull) 461 try: 462 cmd = [ 'make', 'check-valgrind' ] 463 check_call_cmd(top_dir, *cmd) 464 except CalledProcessError: 465 for root, _, files in os.walk(top_dir): 466 for f in files: 467 if re.search('test-suite-[a-z]+.log', f) is None: 468 continue 469 check_call_cmd(root, 'cat', os.path.join(root, f)) 470 raise Exception('Valgrind tests failed') 471 except CalledProcessError: 472 pass 473 474 # Run code coverage if possible 475 try: 476 cmd = [ 'make', '-n', 'check-code-coverage' ] 477 check_call(cmd, stdout=devnull, stderr=devnull) 478 try: 479 cmd = [ 'make', 'check-code-coverage' ] 480 check_call_cmd(top_dir, *cmd) 481 except CalledProcessError: 482 raise Exception('Code coverage failed') 483 except CalledProcessError: 484 pass 485 486 os.umask(prev_umask) 487