1#!/usr/bin/env python 2 3""" 4This script determines the given package's openbmc dependencies from its 5configure.ac file where it downloads, configures, builds, and installs each of 6these dependencies. Then the given package is configured, built, and installed 7prior to executing its unit tests. 8""" 9 10from git import Repo 11from urlparse import urljoin 12from subprocess import check_call, call, CalledProcessError 13import os 14import sys 15import argparse 16import re 17 18 19class DepTree(): 20 """ 21 Represents package dependency tree, where each node is a DepTree with a 22 name and DepTree children. 23 """ 24 25 def __init__(self, name): 26 """ 27 Create new DepTree. 28 29 Parameter descriptions: 30 name Name of new tree node. 31 """ 32 self.name = name 33 self.children = list() 34 35 def AddChild(self, name): 36 """ 37 Add new child node to current node. 38 39 Parameter descriptions: 40 name Name of new child 41 """ 42 new_child = DepTree(name) 43 self.children.append(new_child) 44 return new_child 45 46 def AddChildNode(self, node): 47 """ 48 Add existing child node to current node. 49 50 Parameter descriptions: 51 node Tree node to add 52 """ 53 self.children.append(node) 54 55 def RemoveChild(self, name): 56 """ 57 Remove child node. 58 59 Parameter descriptions: 60 name Name of child to remove 61 """ 62 for child in self.children: 63 if child.name == name: 64 self.children.remove(child) 65 return 66 67 def GetNode(self, name): 68 """ 69 Return node with matching name. Return None if not found. 70 71 Parameter descriptions: 72 name Name of node to return 73 """ 74 if self.name == name: 75 return self 76 for child in self.children: 77 node = child.GetNode(name) 78 if node: 79 return node 80 return None 81 82 def GetParentNode(self, name, parent_node=None): 83 """ 84 Return parent of node with matching name. Return none if not found. 85 86 Parameter descriptions: 87 name Name of node to get parent of 88 parent_node Parent of current node 89 """ 90 if self.name == name: 91 return parent_node 92 for child in self.children: 93 found_node = child.GetParentNode(name, self) 94 if found_node: 95 return found_node 96 return None 97 98 def GetPath(self, name, path=None): 99 """ 100 Return list of node names from head to matching name. 101 Return None if not found. 102 103 Parameter descriptions: 104 name Name of node 105 path List of node names from head to current node 106 """ 107 if not path: 108 path = [] 109 if self.name == name: 110 path.append(self.name) 111 return path 112 for child in self.children: 113 match = child.GetPath(name, path + [self.name]) 114 if match: 115 return match 116 return None 117 118 def GetPathRegex(self, name, regex_str, path=None): 119 """ 120 Return list of node paths that end in name, or match regex_str. 121 Return empty list if not found. 122 123 Parameter descriptions: 124 name Name of node to search for 125 regex_str Regex string to match node names 126 path Path of node names from head to current node 127 """ 128 new_paths = [] 129 if not path: 130 path = [] 131 match = re.match(regex_str, self.name) 132 if (self.name == name) or (match): 133 new_paths.append(path + [self.name]) 134 for child in self.children: 135 return_paths = None 136 full_path = path + [self.name] 137 return_paths = child.GetPathRegex(name, regex_str, full_path) 138 for i in return_paths: 139 new_paths.append(i) 140 return new_paths 141 142 def MoveNode(self, from_name, to_name): 143 """ 144 Mode existing from_name node to become child of to_name node. 145 146 Parameter descriptions: 147 from_name Name of node to make a child of to_name 148 to_name Name of node to make parent of from_name 149 """ 150 parent_from_node = self.GetParentNode(from_name) 151 from_node = self.GetNode(from_name) 152 parent_from_node.RemoveChild(from_name) 153 to_node = self.GetNode(to_name) 154 to_node.AddChildNode(from_node) 155 156 def ReorderDeps(self, name, regex_str): 157 """ 158 Reorder dependency tree. If tree contains nodes with names that 159 match 'name' and 'regex_str', move 'regex_str' nodes that are 160 to the right of 'name' node, so that they become children of the 161 'name' node. 162 163 Parameter descriptions: 164 name Name of node to look for 165 regex_str Regex string to match names to 166 """ 167 name_path = self.GetPath(name) 168 if not name_path: 169 return 170 paths = self.GetPathRegex(name, regex_str) 171 is_name_in_paths = False 172 name_index = 0 173 for i in range(len(paths)): 174 path = paths[i] 175 if path[-1] == name: 176 is_name_in_paths = True 177 name_index = i 178 break 179 if not is_name_in_paths: 180 return 181 for i in range(name_index + 1, len(paths)): 182 path = paths[i] 183 if name in path: 184 continue 185 from_name = path[-1] 186 self.MoveNode(from_name, name) 187 188 def GetInstallList(self): 189 """ 190 Return post-order list of node names. 191 192 Parameter descriptions: 193 """ 194 install_list = [] 195 for child in self.children: 196 child_install_list = child.GetInstallList() 197 install_list.extend(child_install_list) 198 install_list.append(self.name) 199 return install_list 200 201 def PrintTree(self, level=0): 202 """ 203 Print pre-order node names with indentation denoting node depth level. 204 205 Parameter descriptions: 206 level Current depth level 207 """ 208 INDENT_PER_LEVEL = 4 209 print ' ' * (level * INDENT_PER_LEVEL) + self.name 210 for child in self.children: 211 child.PrintTree(level + 1) 212 213 214def check_call_cmd(dir, *cmd): 215 """ 216 Verbose prints the directory location the given command is called from and 217 the command, then executes the command using check_call. 218 219 Parameter descriptions: 220 dir Directory location command is to be called from 221 cmd List of parameters constructing the complete command 222 """ 223 printline(dir, ">", " ".join(cmd)) 224 check_call(cmd) 225 226 227def clone_pkg(pkg): 228 """ 229 Clone the given openbmc package's git repository from gerrit into 230 the WORKSPACE location 231 232 Parameter descriptions: 233 pkg Name of the package to clone 234 """ 235 pkg_dir = os.path.join(WORKSPACE, pkg) 236 if os.path.exists(os.path.join(pkg_dir, '.git')): 237 return pkg_dir 238 pkg_repo = urljoin('https://gerrit.openbmc-project.xyz/openbmc/', pkg) 239 os.mkdir(pkg_dir) 240 printline(pkg_dir, "> git clone", pkg_repo, "./") 241 return Repo.clone_from(pkg_repo, pkg_dir).working_dir 242 243 244def get_deps(configure_ac): 245 """ 246 Parse the given 'configure.ac' file for package dependencies and return 247 a list of the dependencies found. 248 249 Parameter descriptions: 250 configure_ac Opened 'configure.ac' file object 251 """ 252 line = "" 253 dep_pkgs = set() 254 for cfg_line in configure_ac: 255 # Remove whitespace & newline 256 cfg_line = cfg_line.rstrip() 257 # Check for line breaks 258 if cfg_line.endswith('\\'): 259 line += str(cfg_line[:-1]) 260 continue 261 line = line+cfg_line 262 263 # Find any defined dependency 264 line_has = lambda x: x if x in line else None 265 macros = set(filter(line_has, DEPENDENCIES.iterkeys())) 266 if len(macros) == 1: 267 macro = ''.join(macros) 268 deps = filter(line_has, DEPENDENCIES[macro].iterkeys()) 269 dep_pkgs.update(map(lambda x: DEPENDENCIES[macro][x], deps)) 270 271 line = "" 272 deps = list(dep_pkgs) 273 274 return deps 275 276 277def install_deps(dep_list): 278 """ 279 Install each package in the ordered dep_list. 280 281 Parameter descriptions: 282 dep_list Ordered list of dependencies 283 """ 284 for pkg in dep_list: 285 pkgdir = os.path.join(WORKSPACE, pkg) 286 # Build & install this package 287 conf_flags = [ 288 '--disable-silent-rules', 289 ] 290 os.chdir(pkgdir) 291 # Add any necessary configure flags for package 292 if CONFIGURE_FLAGS.get(pkg) is not None: 293 conf_flags.extend(CONFIGURE_FLAGS.get(pkg)) 294 check_call_cmd(pkgdir, './bootstrap.sh') 295 check_call_cmd(pkgdir, './configure', *conf_flags) 296 check_call_cmd(pkgdir, 'make') 297 check_call_cmd(pkgdir, 'make', 'install') 298 299 300def build_dep_tree(pkg, pkgdir, dep_added, head, dep_tree=None): 301 """ 302 For each package(pkg), starting with the package to be unit tested, 303 parse its 'configure.ac' file from within the package's directory(pkgdir) 304 for each package dependency defined recursively doing the same thing 305 on each package found as a dependency. 306 307 Parameter descriptions: 308 pkg Name of the package 309 pkgdir Directory where package source is located 310 dep_added Current list of dependencies and added status 311 head Head node of the dependency tree 312 dep_tree Current dependency tree node 313 """ 314 if not dep_tree: 315 dep_tree = head 316 os.chdir(pkgdir) 317 # Open package's configure.ac 318 with open("/root/.depcache", "r") as depcache: 319 cached = depcache.readline() 320 with open("configure.ac", "rt") as configure_ac: 321 # Retrieve dependency list from package's configure.ac 322 configure_ac_deps = get_deps(configure_ac) 323 for dep_pkg in configure_ac_deps: 324 if dep_pkg in cached: 325 continue 326 # Dependency package not already known 327 if dep_added.get(dep_pkg) is None: 328 # Dependency package not added 329 new_child = dep_tree.AddChild(dep_pkg) 330 dep_added[dep_pkg] = False 331 dep_pkgdir = clone_pkg(dep_pkg) 332 # Determine this dependency package's 333 # dependencies and add them before 334 # returning to add this package 335 dep_added = build_dep_tree(dep_pkg, 336 dep_pkgdir, 337 dep_added, 338 head, 339 new_child) 340 else: 341 # Dependency package known and added 342 if dep_added[dep_pkg]: 343 continue 344 else: 345 # Cyclic dependency failure 346 raise Exception("Cyclic dependencies found in "+pkg) 347 348 if not dep_added[pkg]: 349 dep_added[pkg] = True 350 351 return dep_added 352 353 354if __name__ == '__main__': 355 # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS] 356 CONFIGURE_FLAGS = { 357 'phosphor-objmgr': ['--enable-unpatched-systemd'], 358 'sdbusplus': ['--enable-transaction'], 359 'phosphor-logging': 360 ['--enable-metadata-processing', 361 'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml'] 362 } 363 364 # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO] 365 DEPENDENCIES = { 366 'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'}, 367 'AC_CHECK_HEADER': { 368 'host-ipmid': 'phosphor-host-ipmid', 369 'sdbusplus': 'sdbusplus', 370 'phosphor-logging/log.hpp': 'phosphor-logging', 371 }, 372 'AC_PATH_PROG': {'sdbus++': 'sdbusplus'}, 373 'PKG_CHECK_MODULES': { 374 'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces', 375 'openpower-dbus-interfaces': 'openpower-dbus-interfaces', 376 'ibm-dbus-interfaces': 'ibm-dbus-interfaces', 377 'sdbusplus': 'sdbusplus', 378 'phosphor-logging': 'phosphor-logging', 379 }, 380 } 381 382 # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING] 383 DEPENDENCIES_REGEX = { 384 'phosphor-logging': '\S+-dbus-interfaces$' 385 } 386 387 # Set command line arguments 388 parser = argparse.ArgumentParser() 389 parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True, 390 help="Workspace directory location(i.e. /home)") 391 parser.add_argument("-p", "--package", dest="PACKAGE", required=True, 392 help="OpenBMC package to be unit tested") 393 parser.add_argument("-v", "--verbose", action="store_true", 394 help="Print additional package status messages") 395 parser.add_argument("-r", "--repeat", help="Repeat tests N times", 396 type=int, default=1) 397 args = parser.parse_args(sys.argv[1:]) 398 WORKSPACE = args.WORKSPACE 399 UNIT_TEST_PKG = args.PACKAGE 400 if args.verbose: 401 def printline(*line): 402 for arg in line: 403 print arg, 404 print 405 else: 406 printline = lambda *l: None 407 408 # First validate code formattting if repo has style formatting files. 409 # The format-code.sh checks for these files. 410 CODE_SCAN_DIR = WORKSPACE + "/" + UNIT_TEST_PKG 411 check_call_cmd(WORKSPACE, "./format-code.sh", CODE_SCAN_DIR) 412 413 # The rest of this script is CI testing, which currently only supports 414 # Automake based repos. Check if this repo is Automake, if not exit 415 if not os.path.isfile(CODE_SCAN_DIR + "/configure.ac"): 416 print "Not a supported repo for CI Tests, exit" 417 quit() 418 419 prev_umask = os.umask(000) 420 # Determine dependencies and add them 421 dep_added = dict() 422 dep_added[UNIT_TEST_PKG] = False 423 # Create dependency tree 424 dep_tree = DepTree(UNIT_TEST_PKG) 425 build_dep_tree(UNIT_TEST_PKG, 426 os.path.join(WORKSPACE, UNIT_TEST_PKG), 427 dep_added, 428 dep_tree) 429 430 # Reorder Dependency Tree 431 for pkg_name, regex_str in DEPENDENCIES_REGEX.iteritems(): 432 dep_tree.ReorderDeps(pkg_name, regex_str) 433 if args.verbose: 434 dep_tree.PrintTree() 435 install_list = dep_tree.GetInstallList() 436 # install reordered dependencies 437 install_deps(install_list) 438 os.chdir(os.path.join(WORKSPACE, UNIT_TEST_PKG)) 439 # Refresh dynamic linker run time bindings for dependencies 440 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), 'ldconfig') 441 # Run package unit tests 442 try: 443 cmd = [ 'make', 'check' ] 444 for i in range(0, args.repeat): 445 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), *cmd) 446 except CalledProcessError: 447 for root, _, files in os.walk(os.path.join(WORKSPACE, UNIT_TEST_PKG)): 448 if 'test-suite.log' not in files: 449 continue 450 check_call_cmd(root, 'cat', os.path.join(root, 'test-suite.log')) 451 raise Exception('Unit tests failed') 452 os.umask(prev_umask) 453