1#!/usr/bin/env python 2 3""" 4This script determines the given package's openbmc dependencies from its 5configure.ac file where it downloads, configures, builds, and installs each of 6these dependencies. Then the given package is configured, built, and installed 7prior to executing its unit tests. 8""" 9 10from git import Repo 11from urlparse import urljoin 12from subprocess import check_call, call, CalledProcessError 13import os 14import sys 15import argparse 16import re 17 18 19class DepTree(): 20 """ 21 Represents package dependency tree, where each node is a DepTree with a 22 name and DepTree children. 23 """ 24 25 def __init__(self, name): 26 """ 27 Create new DepTree. 28 29 Parameter descriptions: 30 name Name of new tree node. 31 """ 32 self.name = name 33 self.children = list() 34 35 def AddChild(self, name): 36 """ 37 Add new child node to current node. 38 39 Parameter descriptions: 40 name Name of new child 41 """ 42 new_child = DepTree(name) 43 self.children.append(new_child) 44 return new_child 45 46 def AddChildNode(self, node): 47 """ 48 Add existing child node to current node. 49 50 Parameter descriptions: 51 node Tree node to add 52 """ 53 self.children.append(node) 54 55 def RemoveChild(self, name): 56 """ 57 Remove child node. 58 59 Parameter descriptions: 60 name Name of child to remove 61 """ 62 for child in self.children: 63 if child.name == name: 64 self.children.remove(child) 65 return 66 67 def GetNode(self, name): 68 """ 69 Return node with matching name. Return None if not found. 70 71 Parameter descriptions: 72 name Name of node to return 73 """ 74 if self.name == name: 75 return self 76 for child in self.children: 77 node = child.GetNode(name) 78 if node: 79 return node 80 return None 81 82 def GetParentNode(self, name, parent_node=None): 83 """ 84 Return parent of node with matching name. Return none if not found. 85 86 Parameter descriptions: 87 name Name of node to get parent of 88 parent_node Parent of current node 89 """ 90 if self.name == name: 91 return parent_node 92 for child in self.children: 93 found_node = child.GetParentNode(name, self) 94 if found_node: 95 return found_node 96 return None 97 98 def GetPath(self, name, path=None): 99 """ 100 Return list of node names from head to matching name. 101 Return None if not found. 102 103 Parameter descriptions: 104 name Name of node 105 path List of node names from head to current node 106 """ 107 if not path: 108 path = [] 109 if self.name == name: 110 path.append(self.name) 111 return path 112 for child in self.children: 113 match = child.GetPath(name, path + [self.name]) 114 if match: 115 return match 116 return None 117 118 def GetPathRegex(self, name, regex_str, path=None): 119 """ 120 Return list of node paths that end in name, or match regex_str. 121 Return empty list if not found. 122 123 Parameter descriptions: 124 name Name of node to search for 125 regex_str Regex string to match node names 126 path Path of node names from head to current node 127 """ 128 new_paths = [] 129 if not path: 130 path = [] 131 match = re.match(regex_str, self.name) 132 if (self.name == name) or (match): 133 new_paths.append(path + [self.name]) 134 for child in self.children: 135 return_paths = None 136 full_path = path + [self.name] 137 return_paths = child.GetPathRegex(name, regex_str, full_path) 138 for i in return_paths: 139 new_paths.append(i) 140 return new_paths 141 142 def MoveNode(self, from_name, to_name): 143 """ 144 Mode existing from_name node to become child of to_name node. 145 146 Parameter descriptions: 147 from_name Name of node to make a child of to_name 148 to_name Name of node to make parent of from_name 149 """ 150 parent_from_node = self.GetParentNode(from_name) 151 from_node = self.GetNode(from_name) 152 parent_from_node.RemoveChild(from_name) 153 to_node = self.GetNode(to_name) 154 to_node.AddChildNode(from_node) 155 156 def ReorderDeps(self, name, regex_str): 157 """ 158 Reorder dependency tree. If tree contains nodes with names that 159 match 'name' and 'regex_str', move 'regex_str' nodes that are 160 to the right of 'name' node, so that they become children of the 161 'name' node. 162 163 Parameter descriptions: 164 name Name of node to look for 165 regex_str Regex string to match names to 166 """ 167 name_path = self.GetPath(name) 168 if not name_path: 169 return 170 paths = self.GetPathRegex(name, regex_str) 171 is_name_in_paths = False 172 name_index = 0 173 for i in range(len(paths)): 174 path = paths[i] 175 if path[-1] == name: 176 is_name_in_paths = True 177 name_index = i 178 break 179 if not is_name_in_paths: 180 return 181 for i in range(name_index + 1, len(paths)): 182 path = paths[i] 183 if name in path: 184 continue 185 from_name = path[-1] 186 self.MoveNode(from_name, name) 187 188 def GetInstallList(self): 189 """ 190 Return post-order list of node names. 191 192 Parameter descriptions: 193 """ 194 install_list = [] 195 for child in self.children: 196 child_install_list = child.GetInstallList() 197 install_list.extend(child_install_list) 198 install_list.append(self.name) 199 return install_list 200 201 def PrintTree(self, level=0): 202 """ 203 Print pre-order node names with indentation denoting node depth level. 204 205 Parameter descriptions: 206 level Current depth level 207 """ 208 INDENT_PER_LEVEL = 4 209 print ' ' * (level * INDENT_PER_LEVEL) + self.name 210 for child in self.children: 211 child.PrintTree(level + 1) 212 213 214def check_call_cmd(dir, *cmd): 215 """ 216 Verbose prints the directory location the given command is called from and 217 the command, then executes the command using check_call. 218 219 Parameter descriptions: 220 dir Directory location command is to be called from 221 cmd List of parameters constructing the complete command 222 """ 223 printline(dir, ">", " ".join(cmd)) 224 check_call(cmd) 225 226 227def clone_pkg(pkg): 228 """ 229 Clone the given openbmc package's git repository from gerrit into 230 the WORKSPACE location 231 232 Parameter descriptions: 233 pkg Name of the package to clone 234 """ 235 pkg_dir = os.path.join(WORKSPACE, pkg) 236 if os.path.exists(os.path.join(pkg_dir, '.git')): 237 return pkg_dir 238 pkg_repo = urljoin('https://gerrit.openbmc-project.xyz/openbmc/', pkg) 239 os.mkdir(pkg_dir) 240 printline(pkg_dir, "> git clone", pkg_repo, "./") 241 return Repo.clone_from(pkg_repo, pkg_dir).working_dir 242 243 244def get_deps(configure_ac): 245 """ 246 Parse the given 'configure.ac' file for package dependencies and return 247 a list of the dependencies found. 248 249 Parameter descriptions: 250 configure_ac Opened 'configure.ac' file object 251 """ 252 line = "" 253 dep_pkgs = set() 254 for cfg_line in configure_ac: 255 # Remove whitespace & newline 256 cfg_line = cfg_line.rstrip() 257 # Check for line breaks 258 if cfg_line.endswith('\\'): 259 line += str(cfg_line[:-1]) 260 continue 261 line = line+cfg_line 262 263 # Find any defined dependency 264 line_has = lambda x: x if x in line else None 265 macros = set(filter(line_has, DEPENDENCIES.iterkeys())) 266 if len(macros) == 1: 267 macro = ''.join(macros) 268 deps = filter(line_has, DEPENDENCIES[macro].iterkeys()) 269 dep_pkgs.update(map(lambda x: DEPENDENCIES[macro][x], deps)) 270 271 line = "" 272 deps = list(dep_pkgs) 273 274 return deps 275 276 277def install_deps(dep_list): 278 """ 279 Install each package in the ordered dep_list. 280 281 Parameter descriptions: 282 dep_list Ordered list of dependencies 283 """ 284 for pkg in dep_list: 285 pkgdir = os.path.join(WORKSPACE, pkg) 286 # Build & install this package 287 conf_flags = [] 288 os.chdir(pkgdir) 289 # Add any necessary configure flags for package 290 if CONFIGURE_FLAGS.get(pkg) is not None: 291 conf_flags.extend(CONFIGURE_FLAGS.get(pkg)) 292 check_call_cmd(pkgdir, './bootstrap.sh') 293 check_call_cmd(pkgdir, './configure', *conf_flags) 294 check_call_cmd(pkgdir, 'make') 295 check_call_cmd(pkgdir, 'make', 'install') 296 297 298def build_dep_tree(pkg, pkgdir, dep_added, head, dep_tree=None): 299 """ 300 For each package(pkg), starting with the package to be unit tested, 301 parse its 'configure.ac' file from within the package's directory(pkgdir) 302 for each package dependency defined recursively doing the same thing 303 on each package found as a dependency. 304 305 Parameter descriptions: 306 pkg Name of the package 307 pkgdir Directory where package source is located 308 dep_added Current list of dependencies and added status 309 head Head node of the dependency tree 310 dep_tree Current dependency tree node 311 """ 312 if not dep_tree: 313 dep_tree = head 314 os.chdir(pkgdir) 315 # Open package's configure.ac 316 with open("/root/.depcache", "r") as depcache: 317 cached = depcache.readline() 318 with open("configure.ac", "rt") as configure_ac: 319 # Retrieve dependency list from package's configure.ac 320 configure_ac_deps = get_deps(configure_ac) 321 for dep_pkg in configure_ac_deps: 322 if dep_pkg in cached: 323 continue 324 # Dependency package not already known 325 if dep_added.get(dep_pkg) is None: 326 # Dependency package not added 327 new_child = dep_tree.AddChild(dep_pkg) 328 dep_added[dep_pkg] = False 329 dep_pkgdir = clone_pkg(dep_pkg) 330 # Determine this dependency package's 331 # dependencies and add them before 332 # returning to add this package 333 dep_added = build_dep_tree(dep_pkg, 334 dep_pkgdir, 335 dep_added, 336 head, 337 new_child) 338 else: 339 # Dependency package known and added 340 if dep_added[dep_pkg]: 341 continue 342 else: 343 # Cyclic dependency failure 344 raise Exception("Cyclic dependencies found in "+pkg) 345 346 if not dep_added[pkg]: 347 dep_added[pkg] = True 348 349 return dep_added 350 351 352if __name__ == '__main__': 353 # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS] 354 CONFIGURE_FLAGS = { 355 'phosphor-objmgr': ['--enable-unpatched-systemd'], 356 'sdbusplus': ['--enable-transaction'], 357 'phosphor-logging': 358 ['--enable-metadata-processing', 359 'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml'] 360 } 361 362 # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO] 363 DEPENDENCIES = { 364 'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'}, 365 'AC_CHECK_HEADER': { 366 'host-ipmid': 'phosphor-host-ipmid', 367 'sdbusplus': 'sdbusplus', 368 'phosphor-logging/log.hpp': 'phosphor-logging', 369 }, 370 'AC_PATH_PROG': {'sdbus++': 'sdbusplus'}, 371 'PKG_CHECK_MODULES': { 372 'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces', 373 'openpower-dbus-interfaces': 'openpower-dbus-interfaces', 374 'ibm-dbus-interfaces': 'ibm-dbus-interfaces', 375 'sdbusplus': 'sdbusplus', 376 'phosphor-logging': 'phosphor-logging', 377 }, 378 } 379 380 # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING] 381 DEPENDENCIES_REGEX = { 382 'phosphor-logging': '\S+-dbus-interfaces$' 383 } 384 385 # Set command line arguments 386 parser = argparse.ArgumentParser() 387 parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True, 388 help="Workspace directory location(i.e. /home)") 389 parser.add_argument("-p", "--package", dest="PACKAGE", required=True, 390 help="OpenBMC package to be unit tested") 391 parser.add_argument("-v", "--verbose", action="store_true", 392 help="Print additional package status messages") 393 parser.add_argument("-r", "--repeat", help="Repeat tests N times", 394 type=int, default=1) 395 args = parser.parse_args(sys.argv[1:]) 396 WORKSPACE = args.WORKSPACE 397 UNIT_TEST_PKG = args.PACKAGE 398 if args.verbose: 399 def printline(*line): 400 for arg in line: 401 print arg, 402 print 403 else: 404 printline = lambda *l: None 405 406 # First validate code formattting if repo has style formatting files. 407 # The format-code.sh checks for these files. 408 CODE_SCAN_DIR = WORKSPACE + "/" + UNIT_TEST_PKG 409 check_call_cmd(WORKSPACE, "./format-code.sh", CODE_SCAN_DIR) 410 411 # The rest of this script is CI testing, which currently only supports 412 # Automake based repos. Check if this repo is Automake, if not exit 413 if not os.path.isfile(CODE_SCAN_DIR + "/configure.ac"): 414 print "Not a supported repo for CI Tests, exit" 415 quit() 416 417 prev_umask = os.umask(000) 418 # Determine dependencies and add them 419 dep_added = dict() 420 dep_added[UNIT_TEST_PKG] = False 421 # Create dependency tree 422 dep_tree = DepTree(UNIT_TEST_PKG) 423 build_dep_tree(UNIT_TEST_PKG, 424 os.path.join(WORKSPACE, UNIT_TEST_PKG), 425 dep_added, 426 dep_tree) 427 428 # Reorder Dependency Tree 429 for pkg_name, regex_str in DEPENDENCIES_REGEX.iteritems(): 430 dep_tree.ReorderDeps(pkg_name, regex_str) 431 if args.verbose: 432 dep_tree.PrintTree() 433 install_list = dep_tree.GetInstallList() 434 # install reordered dependencies 435 install_deps(install_list) 436 os.chdir(os.path.join(WORKSPACE, UNIT_TEST_PKG)) 437 # Refresh dynamic linker run time bindings for dependencies 438 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), 'ldconfig') 439 # Run package unit tests 440 try: 441 cmd = [ 'make', 'check' ] 442 for i in range(0, args.repeat): 443 check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), *cmd) 444 except CalledProcessError: 445 for root, _, files in os.walk(os.path.join(WORKSPACE, UNIT_TEST_PKG)): 446 if 'test-suite.log' not in files: 447 continue 448 check_call_cmd(root, 'cat', os.path.join(root, 'test-suite.log')) 449 raise Exception('Unit tests failed') 450 os.umask(prev_umask) 451