1#!/usr/bin/env python3
2
3"""
4This script determines the given package's openbmc dependencies from its
5configure.ac file where it downloads, configures, builds, and installs each of
6these dependencies. Then the given package is configured, built, and installed
7prior to executing its unit tests.
8"""
9
10from git import Repo
11# interpreter is not used directly but this resolves dependency ordering
12# that would be broken if we didn't include it.
13from mesonbuild import interpreter
14from mesonbuild import coredata, optinterpreter
15from mesonbuild.mesonlib import OptionKey
16from mesonbuild.mesonlib import version_compare as meson_version_compare
17from urllib.parse import urljoin
18from subprocess import check_call, call, CalledProcessError
19import os
20import sys
21import argparse
22import multiprocessing
23import re
24import subprocess
25import shutil
26import platform
27
28
29class DepTree():
30    """
31    Represents package dependency tree, where each node is a DepTree with a
32    name and DepTree children.
33    """
34
35    def __init__(self, name):
36        """
37        Create new DepTree.
38
39        Parameter descriptions:
40        name               Name of new tree node.
41        """
42        self.name = name
43        self.children = list()
44
45    def AddChild(self, name):
46        """
47        Add new child node to current node.
48
49        Parameter descriptions:
50        name               Name of new child
51        """
52        new_child = DepTree(name)
53        self.children.append(new_child)
54        return new_child
55
56    def AddChildNode(self, node):
57        """
58        Add existing child node to current node.
59
60        Parameter descriptions:
61        node               Tree node to add
62        """
63        self.children.append(node)
64
65    def RemoveChild(self, name):
66        """
67        Remove child node.
68
69        Parameter descriptions:
70        name               Name of child to remove
71        """
72        for child in self.children:
73            if child.name == name:
74                self.children.remove(child)
75                return
76
77    def GetNode(self, name):
78        """
79        Return node with matching name. Return None if not found.
80
81        Parameter descriptions:
82        name               Name of node to return
83        """
84        if self.name == name:
85            return self
86        for child in self.children:
87            node = child.GetNode(name)
88            if node:
89                return node
90        return None
91
92    def GetParentNode(self, name, parent_node=None):
93        """
94        Return parent of node with matching name. Return none if not found.
95
96        Parameter descriptions:
97        name               Name of node to get parent of
98        parent_node        Parent of current node
99        """
100        if self.name == name:
101            return parent_node
102        for child in self.children:
103            found_node = child.GetParentNode(name, self)
104            if found_node:
105                return found_node
106        return None
107
108    def GetPath(self, name, path=None):
109        """
110        Return list of node names from head to matching name.
111        Return None if not found.
112
113        Parameter descriptions:
114        name               Name of node
115        path               List of node names from head to current node
116        """
117        if not path:
118            path = []
119        if self.name == name:
120            path.append(self.name)
121            return path
122        for child in self.children:
123            match = child.GetPath(name, path + [self.name])
124            if match:
125                return match
126        return None
127
128    def GetPathRegex(self, name, regex_str, path=None):
129        """
130        Return list of node paths that end in name, or match regex_str.
131        Return empty list if not found.
132
133        Parameter descriptions:
134        name               Name of node to search for
135        regex_str          Regex string to match node names
136        path               Path of node names from head to current node
137        """
138        new_paths = []
139        if not path:
140            path = []
141        match = re.match(regex_str, self.name)
142        if (self.name == name) or (match):
143            new_paths.append(path + [self.name])
144        for child in self.children:
145            return_paths = None
146            full_path = path + [self.name]
147            return_paths = child.GetPathRegex(name, regex_str, full_path)
148            for i in return_paths:
149                new_paths.append(i)
150        return new_paths
151
152    def MoveNode(self, from_name, to_name):
153        """
154        Mode existing from_name node to become child of to_name node.
155
156        Parameter descriptions:
157        from_name          Name of node to make a child of to_name
158        to_name            Name of node to make parent of from_name
159        """
160        parent_from_node = self.GetParentNode(from_name)
161        from_node = self.GetNode(from_name)
162        parent_from_node.RemoveChild(from_name)
163        to_node = self.GetNode(to_name)
164        to_node.AddChildNode(from_node)
165
166    def ReorderDeps(self, name, regex_str):
167        """
168        Reorder dependency tree.  If tree contains nodes with names that
169        match 'name' and 'regex_str', move 'regex_str' nodes that are
170        to the right of 'name' node, so that they become children of the
171        'name' node.
172
173        Parameter descriptions:
174        name               Name of node to look for
175        regex_str          Regex string to match names to
176        """
177        name_path = self.GetPath(name)
178        if not name_path:
179            return
180        paths = self.GetPathRegex(name, regex_str)
181        is_name_in_paths = False
182        name_index = 0
183        for i in range(len(paths)):
184            path = paths[i]
185            if path[-1] == name:
186                is_name_in_paths = True
187                name_index = i
188                break
189        if not is_name_in_paths:
190            return
191        for i in range(name_index + 1, len(paths)):
192            path = paths[i]
193            if name in path:
194                continue
195            from_name = path[-1]
196            self.MoveNode(from_name, name)
197
198    def GetInstallList(self):
199        """
200        Return post-order list of node names.
201
202        Parameter descriptions:
203        """
204        install_list = []
205        for child in self.children:
206            child_install_list = child.GetInstallList()
207            install_list.extend(child_install_list)
208        install_list.append(self.name)
209        return install_list
210
211    def PrintTree(self, level=0):
212        """
213        Print pre-order node names with indentation denoting node depth level.
214
215        Parameter descriptions:
216        level              Current depth level
217        """
218        INDENT_PER_LEVEL = 4
219        print(' ' * (level * INDENT_PER_LEVEL) + self.name)
220        for child in self.children:
221            child.PrintTree(level + 1)
222
223
224def check_call_cmd(*cmd):
225    """
226    Verbose prints the directory location the given command is called from and
227    the command, then executes the command using check_call.
228
229    Parameter descriptions:
230    dir                 Directory location command is to be called from
231    cmd                 List of parameters constructing the complete command
232    """
233    printline(os.getcwd(), ">", " ".join(cmd))
234    check_call(cmd)
235
236
237def clone_pkg(pkg, branch):
238    """
239    Clone the given openbmc package's git repository from gerrit into
240    the WORKSPACE location
241
242    Parameter descriptions:
243    pkg                 Name of the package to clone
244    branch              Branch to clone from pkg
245    """
246    pkg_dir = os.path.join(WORKSPACE, pkg)
247    if os.path.exists(os.path.join(pkg_dir, '.git')):
248        return pkg_dir
249    pkg_repo = urljoin('https://gerrit.openbmc-project.xyz/openbmc/', pkg)
250    os.mkdir(pkg_dir)
251    printline(pkg_dir, "> git clone", pkg_repo, branch, "./")
252    try:
253        # first try the branch
254        clone = Repo.clone_from(pkg_repo, pkg_dir, branch=branch)
255        repo_inst = clone.working_dir
256    except:
257        printline("Input branch not found, default to master")
258        clone = Repo.clone_from(pkg_repo, pkg_dir, branch="master")
259        repo_inst = clone.working_dir
260    return repo_inst
261
262
263def make_target_exists(target):
264    """
265    Runs a check against the makefile in the current directory to determine
266    if the target exists so that it can be built.
267
268    Parameter descriptions:
269    target              The make target we are checking
270    """
271    try:
272        cmd = ['make', '-n', target]
273        with open(os.devnull, 'w') as devnull:
274            check_call(cmd, stdout=devnull, stderr=devnull)
275        return True
276    except CalledProcessError:
277        return False
278
279
280make_parallel = [
281    'make',
282    # Run enough jobs to saturate all the cpus
283    '-j', str(multiprocessing.cpu_count()),
284    # Don't start more jobs if the load avg is too high
285    '-l', str(multiprocessing.cpu_count()),
286    # Synchronize the output so logs aren't intermixed in stdout / stderr
287    '-O',
288]
289
290
291def build_and_install(name, build_for_testing=False):
292    """
293    Builds and installs the package in the environment. Optionally
294    builds the examples and test cases for package.
295
296    Parameter description:
297    name                The name of the package we are building
298    build_for_testing   Enable options related to testing on the package?
299    """
300    os.chdir(os.path.join(WORKSPACE, name))
301
302    # Refresh dynamic linker run time bindings for dependencies
303    check_call_cmd('sudo', '-n', '--', 'ldconfig')
304
305    pkg = Package()
306    if build_for_testing:
307        pkg.test()
308    else:
309        pkg.install()
310
311
312def build_dep_tree(name, pkgdir, dep_added, head, branch, dep_tree=None):
313    """
314    For each package (name), starting with the package to be unit tested,
315    extract its dependencies. For each package dependency defined, recursively
316    apply the same strategy
317
318    Parameter descriptions:
319    name                Name of the package
320    pkgdir              Directory where package source is located
321    dep_added           Current dict of dependencies and added status
322    head                Head node of the dependency tree
323    branch              Branch to clone from pkg
324    dep_tree            Current dependency tree node
325    """
326    if not dep_tree:
327        dep_tree = head
328
329    with open("/tmp/depcache", "r") as depcache:
330        cache = depcache.readline()
331
332    # Read out pkg dependencies
333    pkg = Package(name, pkgdir)
334
335    build = pkg.build_system()
336    if build == None:
337        raise Exception(f"Unable to find build system for {name}.")
338
339    for dep in set(build.dependencies()):
340        if dep in cache:
341            continue
342        # Dependency package not already known
343        if dep_added.get(dep) is None:
344            print(f"Adding {dep} dependency to {name}.")
345            # Dependency package not added
346            new_child = dep_tree.AddChild(dep)
347            dep_added[dep] = False
348            dep_pkgdir = clone_pkg(dep, branch)
349            # Determine this dependency package's
350            # dependencies and add them before
351            # returning to add this package
352            dep_added = build_dep_tree(dep,
353                                       dep_pkgdir,
354                                       dep_added,
355                                       head,
356                                       branch,
357                                       new_child)
358        else:
359            # Dependency package known and added
360            if dep_added[dep]:
361                continue
362            else:
363                # Cyclic dependency failure
364                raise Exception("Cyclic dependencies found in "+name)
365
366    if not dep_added[name]:
367        dep_added[name] = True
368
369    return dep_added
370
371
372def run_cppcheck():
373    match_re = re.compile(r'((?!\.mako\.).)*\.[ch](?:pp)?$', re.I)
374    cppcheck_files = []
375    stdout = subprocess.check_output(['git', 'ls-files'])
376
377    for f in stdout.decode('utf-8').split():
378        if match_re.match(f):
379            cppcheck_files.append(f)
380
381    if not cppcheck_files:
382        # skip cppcheck if there arent' any c or cpp sources.
383        print("no files")
384        return None
385
386    # http://cppcheck.sourceforge.net/manual.pdf
387    params = ['cppcheck', '-j', str(multiprocessing.cpu_count()),
388              '--enable=all', '--library=googletest', '--file-list=-']
389
390    cppcheck_process = subprocess.Popen(
391        params,
392        stdout=subprocess.PIPE,
393        stderr=subprocess.PIPE,
394        stdin=subprocess.PIPE)
395    (stdout, stderr) = cppcheck_process.communicate(
396        input='\n'.join(cppcheck_files).encode('utf-8'))
397
398    if cppcheck_process.wait():
399        raise Exception('Cppcheck failed')
400    print(stdout.decode('utf-8'))
401    print(stderr.decode('utf-8'))
402
403
404def is_valgrind_safe():
405    """
406    Returns whether it is safe to run valgrind on our platform
407    """
408    src = 'unit-test-vg.c'
409    exe = './unit-test-vg'
410    with open(src, 'w') as h:
411        h.write('#include <errno.h>\n')
412        h.write('#include <stdio.h>\n')
413        h.write('#include <stdlib.h>\n')
414        h.write('#include <string.h>\n')
415        h.write('int main() {\n')
416        h.write('char *heap_str = malloc(16);\n')
417        h.write('strcpy(heap_str, "RandString");\n')
418        h.write('int res = strcmp("RandString", heap_str);\n')
419        h.write('free(heap_str);\n')
420        h.write('char errstr[64];\n')
421        h.write('strerror_r(EINVAL, errstr, sizeof(errstr));\n')
422        h.write('printf("%s\\n", errstr);\n')
423        h.write('return res;\n')
424        h.write('}\n')
425    try:
426        with open(os.devnull, 'w') as devnull:
427            check_call(['gcc', '-O2', '-o', exe, src],
428                       stdout=devnull, stderr=devnull)
429            check_call(['valgrind', '--error-exitcode=99', exe],
430                       stdout=devnull, stderr=devnull)
431        return True
432    except:
433        sys.stderr.write("###### Platform is not valgrind safe ######\n")
434        return False
435    finally:
436        os.remove(src)
437        os.remove(exe)
438
439
440def is_sanitize_safe():
441    """
442    Returns whether it is safe to run sanitizers on our platform
443    """
444    src = 'unit-test-sanitize.c'
445    exe = './unit-test-sanitize'
446    with open(src, 'w') as h:
447        h.write('int main() { return 0; }\n')
448    try:
449        with open(os.devnull, 'w') as devnull:
450            check_call(['gcc', '-O2', '-fsanitize=address',
451                        '-fsanitize=undefined', '-o', exe, src],
452                       stdout=devnull, stderr=devnull)
453            check_call([exe], stdout=devnull, stderr=devnull)
454
455        # TODO - Sanitizer not working on ppc64le
456        # https://github.com/openbmc/openbmc-build-scripts/issues/31
457        if (platform.processor() == 'ppc64le'):
458            sys.stderr.write("###### ppc64le is not sanitize safe ######\n")
459            return False
460        else:
461            return True
462    except:
463        sys.stderr.write("###### Platform is not sanitize safe ######\n")
464        return False
465    finally:
466        os.remove(src)
467        os.remove(exe)
468
469
470def maybe_make_valgrind():
471    """
472    Potentially runs the unit tests through valgrind for the package
473    via `make check-valgrind`. If the package does not have valgrind testing
474    then it just skips over this.
475    """
476    # Valgrind testing is currently broken by an aggressive strcmp optimization
477    # that is inlined into optimized code for POWER by gcc 7+. Until we find
478    # a workaround, just don't run valgrind tests on POWER.
479    # https://github.com/openbmc/openbmc/issues/3315
480    if not is_valgrind_safe():
481        sys.stderr.write("###### Skipping valgrind ######\n")
482        return
483    if not make_target_exists('check-valgrind'):
484        return
485
486    try:
487        cmd = make_parallel + ['check-valgrind']
488        check_call_cmd(*cmd)
489    except CalledProcessError:
490        for root, _, files in os.walk(os.getcwd()):
491            for f in files:
492                if re.search('test-suite-[a-z]+.log', f) is None:
493                    continue
494                check_call_cmd('cat', os.path.join(root, f))
495        raise Exception('Valgrind tests failed')
496
497
498def maybe_make_coverage():
499    """
500    Potentially runs the unit tests through code coverage for the package
501    via `make check-code-coverage`. If the package does not have code coverage
502    testing then it just skips over this.
503    """
504    if not make_target_exists('check-code-coverage'):
505        return
506
507    # Actually run code coverage
508    try:
509        cmd = make_parallel + ['check-code-coverage']
510        check_call_cmd(*cmd)
511    except CalledProcessError:
512        raise Exception('Code coverage failed')
513
514
515class BuildSystem(object):
516    """
517    Build systems generally provide the means to configure, build, install and
518    test software. The BuildSystem class defines a set of interfaces on top of
519    which Autotools, Meson, CMake and possibly other build system drivers can
520    be implemented, separating out the phases to control whether a package
521    should merely be installed or also tested and analyzed.
522    """
523    def __init__(self, package, path):
524        """Initialise the driver with properties independent of the build system
525
526        Keyword arguments:
527        package: The name of the package. Derived from the path if None
528        path: The path to the package. Set to the working directory if None
529        """
530        self.path = "." if not path else path
531        realpath = os.path.realpath(self.path)
532        self.package = package if package else os.path.basename(realpath)
533        self.build_for_testing = False
534
535    def probe(self):
536        """Test if the build system driver can be applied to the package
537
538        Return True if the driver can drive the package's build system,
539        otherwise False.
540
541        Generally probe() is implemented by testing for the presence of the
542        build system's configuration file(s).
543        """
544        raise NotImplemented
545
546    def dependencies(self):
547        """Provide the package's dependencies
548
549        Returns a list of dependencies. If no dependencies are required then an
550        empty list must be returned.
551
552        Generally dependencies() is implemented by analysing and extracting the
553        data from the build system configuration.
554        """
555        raise NotImplemented
556
557    def configure(self, build_for_testing):
558        """Configure the source ready for building
559
560        Should raise an exception if configuration failed.
561
562        Keyword arguments:
563        build_for_testing: Mark the package as being built for testing rather
564                           than for installation as a dependency for the
565                           package under test. Setting to True generally
566                           implies that the package will be configured to build
567                           with debug information, at a low level of
568                           optimisation and possibly with sanitizers enabled.
569
570        Generally configure() is implemented by invoking the build system
571        tooling to generate Makefiles or equivalent.
572        """
573        raise NotImplemented
574
575    def build(self):
576        """Build the software ready for installation and/or testing
577
578        Should raise an exception if the build fails
579
580        Generally build() is implemented by invoking `make` or `ninja`.
581        """
582        raise NotImplemented
583
584    def install(self):
585        """Install the software ready for use
586
587        Should raise an exception if installation fails
588
589        Like build(), install() is generally implemented by invoking `make` or
590        `ninja`.
591        """
592        raise NotImplemented
593
594    def test(self):
595        """Build and run the test suite associated with the package
596
597        Should raise an exception if the build or testing fails.
598
599        Like install(), test() is generally implemented by invoking `make` or
600        `ninja`.
601        """
602        raise NotImplemented
603
604    def analyze(self):
605        """Run any supported analysis tools over the codebase
606
607        Should raise an exception if analysis fails.
608
609        Some analysis tools such as scan-build need injection into the build
610        system. analyze() provides the necessary hook to implement such
611        behaviour. Analyzers independent of the build system can also be
612        specified here but at the cost of possible duplication of code between
613        the build system driver implementations.
614        """
615        raise NotImplemented
616
617
618class Autotools(BuildSystem):
619    def __init__(self, package=None, path=None):
620        super(Autotools, self).__init__(package, path)
621
622    def probe(self):
623        return os.path.isfile(os.path.join(self.path, 'configure.ac'))
624
625    def dependencies(self):
626        configure_ac = os.path.join(self.path, 'configure.ac')
627
628        contents = ''
629        # Prepend some special function overrides so we can parse out
630        # dependencies
631        for macro in DEPENDENCIES.keys():
632            contents += ('m4_define([' + macro + '], [' + macro + '_START$' +
633                         str(DEPENDENCIES_OFFSET[macro] + 1) +
634                         macro + '_END])\n')
635        with open(configure_ac, "rt") as f:
636            contents += f.read()
637
638        autoconf_cmdline = ['autoconf', '-Wno-undefined', '-']
639        autoconf_process = subprocess.Popen(autoconf_cmdline,
640                                            stdin=subprocess.PIPE,
641                                            stdout=subprocess.PIPE,
642                                            stderr=subprocess.PIPE)
643        document = contents.encode('utf-8')
644        (stdout, stderr) = autoconf_process.communicate(input=document)
645        if not stdout:
646            print(stderr)
647            raise Exception("Failed to run autoconf for parsing dependencies")
648
649        # Parse out all of the dependency text
650        matches = []
651        for macro in DEPENDENCIES.keys():
652            pattern = '(' + macro + ')_START(.*?)' + macro + '_END'
653            for match in re.compile(pattern).finditer(stdout.decode('utf-8')):
654                matches.append((match.group(1), match.group(2)))
655
656        # Look up dependencies from the text
657        found_deps = []
658        for macro, deptext in matches:
659            for potential_dep in deptext.split(' '):
660                for known_dep in DEPENDENCIES[macro].keys():
661                    if potential_dep.startswith(known_dep):
662                        found_deps.append(DEPENDENCIES[macro][known_dep])
663
664        return found_deps
665
666    def _configure_feature(self, flag, enabled):
667        """
668        Returns an configure flag as a string
669
670        Parameters:
671        flag                The name of the flag
672        enabled             Whether the flag is enabled or disabled
673        """
674        return '--' + ('enable' if enabled else 'disable') + '-' + flag
675
676    def configure(self, build_for_testing):
677        self.build_for_testing = build_for_testing
678        conf_flags = [
679            self._configure_feature('silent-rules', False),
680            self._configure_feature('examples', build_for_testing),
681            self._configure_feature('tests', build_for_testing),
682            self._configure_feature('itests', INTEGRATION_TEST),
683        ]
684        conf_flags.extend([
685            self._configure_feature('code-coverage', build_for_testing),
686            self._configure_feature('valgrind', build_for_testing),
687        ])
688        # Add any necessary configure flags for package
689        if CONFIGURE_FLAGS.get(self.package) is not None:
690            conf_flags.extend(CONFIGURE_FLAGS.get(self.package))
691        for bootstrap in ['bootstrap.sh', 'bootstrap', 'autogen.sh']:
692            if os.path.exists(bootstrap):
693                check_call_cmd('./' + bootstrap)
694                break
695        check_call_cmd('./configure', *conf_flags)
696
697    def build(self):
698        check_call_cmd(*make_parallel)
699
700    def install(self):
701        check_call_cmd('sudo', '-n', '--', *(make_parallel + ['install']))
702
703    def test(self):
704        try:
705            cmd = make_parallel + ['check']
706            for i in range(0, args.repeat):
707                check_call_cmd(*cmd)
708
709            maybe_make_valgrind()
710            maybe_make_coverage()
711        except CalledProcessError:
712            for root, _, files in os.walk(os.getcwd()):
713                if 'test-suite.log' not in files:
714                    continue
715                check_call_cmd('cat', os.path.join(root, 'test-suite.log'))
716            raise Exception('Unit tests failed')
717
718    def analyze(self):
719        run_cppcheck()
720
721
722class CMake(BuildSystem):
723    def __init__(self, package=None, path=None):
724        super(CMake, self).__init__(package, path)
725
726    def probe(self):
727        return os.path.isfile(os.path.join(self.path, 'CMakeLists.txt'))
728
729    def dependencies(self):
730        return []
731
732    def configure(self, build_for_testing):
733        self.build_for_testing = build_for_testing
734        if INTEGRATION_TEST:
735            check_call_cmd('cmake', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
736                           '-DITESTS=ON', '.')
737        else:
738            check_call_cmd('cmake', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '.')
739
740    def build(self):
741        check_call_cmd('cmake', '--build', '.', '--', '-j',
742                       str(multiprocessing.cpu_count()))
743
744    def install(self):
745        pass
746
747    def test(self):
748        if make_target_exists('test'):
749            check_call_cmd('ctest', '.')
750
751    def analyze(self):
752        if os.path.isfile('.clang-tidy'):
753            try:
754                os.mkdir("tidy-build")
755            except FileExistsError as e:
756                pass
757            # clang-tidy needs to run on a clang-specific build
758            check_call_cmd('cmake', '-DCMAKE_C_COMPILER=clang',
759                           '-DCMAKE_CXX_COMPILER=clang++',
760                           '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
761                           '-H.',
762                           '-Btidy-build')
763            # we need to cd here because otherwise clang-tidy doesn't find the
764            # .clang-tidy file in the roots of repos.  Its arguably a "bug"
765            # with run-clang-tidy at a minimum it's "weird" that it requires
766            # the .clang-tidy to be up a dir
767            os.chdir("tidy-build")
768            try:
769                check_call_cmd('run-clang-tidy.py', "-header-filter=.*", '-p',
770                               '.')
771            finally:
772                os.chdir("..")
773
774        maybe_make_valgrind()
775        maybe_make_coverage()
776        run_cppcheck()
777
778
779class Meson(BuildSystem):
780    def __init__(self, package=None, path=None):
781        super(Meson, self).__init__(package, path)
782
783    def probe(self):
784        return os.path.isfile(os.path.join(self.path, 'meson.build'))
785
786    def dependencies(self):
787        meson_build = os.path.join(self.path, 'meson.build')
788        if not os.path.exists(meson_build):
789            return []
790
791        found_deps = []
792        for root, dirs, files in os.walk(self.path):
793            if 'meson.build' not in files:
794                continue
795            with open(os.path.join(root, 'meson.build'), 'rt') as f:
796                build_contents = f.read()
797            pattern = r"dependency\('([^']*)'.*?\),?\n"
798            for match in re.finditer(pattern, build_contents):
799                group = match.group(1)
800                maybe_dep = DEPENDENCIES['PKG_CHECK_MODULES'].get(group)
801                if maybe_dep is not None:
802                    found_deps.append(maybe_dep)
803
804        return found_deps
805
806    def _parse_options(self, options_file):
807        """
808        Returns a set of options defined in the provides meson_options.txt file
809
810        Parameters:
811        options_file        The file containing options
812        """
813        oi = optinterpreter.OptionInterpreter('')
814        oi.process(options_file)
815        return oi.options
816
817    def _configure_boolean(self, val):
818        """
819        Returns the meson flag which signifies the value
820
821        True is true which requires the boolean.
822        False is false which disables the boolean.
823
824        Parameters:
825        val                 The value being converted
826        """
827        if val is True:
828            return 'true'
829        elif val is False:
830            return 'false'
831        else:
832            raise Exception("Bad meson boolean value")
833
834    def _configure_feature(self, val):
835        """
836        Returns the meson flag which signifies the value
837
838        True is enabled which requires the feature.
839        False is disabled which disables the feature.
840        None is auto which autodetects the feature.
841
842        Parameters:
843        val                 The value being converted
844        """
845        if val is True:
846            return "enabled"
847        elif val is False:
848            return "disabled"
849        elif val is None:
850            return "auto"
851        else:
852            raise Exception("Bad meson feature value")
853
854    def _configure_option(self, opts, key, val):
855        """
856        Returns the meson flag which signifies the value
857        based on the type of the opt
858
859        Parameters:
860        opt                 The meson option which we are setting
861        val                 The value being converted
862        """
863        if isinstance(opts[key], coredata.UserBooleanOption):
864            str_val = self._configure_boolean(val)
865        elif isinstance(opts[key], coredata.UserFeatureOption):
866            str_val = self._configure_feature(val)
867        else:
868            raise Exception('Unknown meson option type')
869        return "-D{}={}".format(key, str_val)
870
871    def configure(self, build_for_testing):
872        self.build_for_testing = build_for_testing
873        meson_options = {}
874        if os.path.exists("meson_options.txt"):
875            meson_options = self._parse_options("meson_options.txt")
876        meson_flags = [
877            '-Db_colorout=never',
878            '-Dwerror=true',
879            '-Dwarning_level=3',
880        ]
881        if build_for_testing:
882            meson_flags.append('--buildtype=debug')
883        else:
884            meson_flags.append('--buildtype=debugoptimized')
885        if OptionKey('tests') in meson_options:
886            meson_flags.append(self._configure_option(meson_options, OptionKey('tests'), build_for_testing))
887        if OptionKey('examples') in meson_options:
888            meson_flags.append(self._configure_option(meson_options, OptionKey('examples'), build_for_testing))
889        if OptionKey('itests') in meson_options:
890            meson_flags.append(self._configure_option(meson_options, OptionKey('itests'), INTEGRATION_TEST))
891        if MESON_FLAGS.get(self.package) is not None:
892            meson_flags.extend(MESON_FLAGS.get(self.package))
893        try:
894            check_call_cmd('meson', 'setup', '--reconfigure', 'build',
895                           *meson_flags)
896        except:
897            shutil.rmtree('build')
898            check_call_cmd('meson', 'setup', 'build', *meson_flags)
899
900    def build(self):
901        check_call_cmd('ninja', '-C', 'build')
902
903    def install(self):
904        check_call_cmd('sudo', '-n', '--', 'ninja', '-C', 'build', 'install')
905
906    def test(self):
907        # It is useful to check various settings of the meson.build file
908        # for compatibility, such as meson_version checks.  We shouldn't
909        # do this in the configure path though because it affects subprojects
910        # and dependencies as well, but we only want this applied to the
911        # project-under-test (otherwise an upstream dependency could fail
912        # this check without our control).
913        self._extra_meson_checks()
914
915        try:
916            test_args = ('--repeat', str(args.repeat), '-C', 'build')
917            check_call_cmd('meson', 'test', *test_args)
918
919        except CalledProcessError:
920            for root, _, files in os.walk(os.getcwd()):
921                if 'testlog.txt' not in files:
922                    continue
923                check_call_cmd('cat', os.path.join(root, 'testlog.txt'))
924            raise Exception('Unit tests failed')
925
926    def _setup_exists(self, setup):
927        """
928        Returns whether the meson build supports the named test setup.
929
930        Parameter descriptions:
931        setup              The setup target to check
932        """
933        try:
934            with open(os.devnull, 'w') as devnull:
935                output = subprocess.check_output(
936                        ['meson', 'test', '-C', 'build',
937                         '--setup', setup, '-t', '0'],
938                        stderr=subprocess.STDOUT)
939        except CalledProcessError as e:
940            output = e.output
941        output = output.decode('utf-8')
942        return not re.search('Test setup .* not found from project', output)
943
944    def _maybe_valgrind(self):
945        """
946        Potentially runs the unit tests through valgrind for the package
947        via `meson test`. The package can specify custom valgrind
948        configurations by utilizing add_test_setup() in a meson.build
949        """
950        if not is_valgrind_safe():
951            sys.stderr.write("###### Skipping valgrind ######\n")
952            return
953        try:
954            if self._setup_exists('valgrind'):
955                check_call_cmd('meson', 'test','-t','10','-C', 'build',
956                               '--setup', 'valgrind')
957            else:
958                check_call_cmd('meson', 'test','-t','10', '-C', 'build',
959                               '--wrapper', 'valgrind')
960        except CalledProcessError:
961            for root, _, files in os.walk(os.getcwd()):
962                if 'testlog-valgrind.txt' not in files:
963                    continue
964                cat_args = os.path.join(root, 'testlog-valgrind.txt')
965                check_call_cmd('cat', cat_args)
966            raise Exception('Valgrind tests failed')
967
968    def analyze(self):
969        self._maybe_valgrind()
970
971        # Run clang-tidy only if the project has a configuration
972        if os.path.isfile('.clang-tidy'):
973            os.environ["CXX"] = "clang++"
974            check_call_cmd('meson', 'setup', 'build-clang')
975            check_call_cmd('run-clang-tidy.py', '-p',
976                           'build-clang')
977        # Run the basic clang static analyzer otherwise
978        else:
979            check_call_cmd('ninja', '-C', 'build',
980                           'scan-build')
981
982        # Run tests through sanitizers
983        # b_lundef is needed if clang++ is CXX since it resolves the
984        # asan symbols at runtime only. We don't want to set it earlier
985        # in the build process to ensure we don't have undefined
986        # runtime code.
987        if is_sanitize_safe():
988            check_call_cmd('meson', 'configure', 'build',
989                           '-Db_sanitize=address,undefined',
990                           '-Db_lundef=false')
991            check_call_cmd('meson', 'test', '-C', 'build',
992                           '--logbase', 'testlog-ubasan')
993            # TODO: Fix memory sanitizer
994            # check_call_cmd('meson', 'configure', 'build',
995            #                '-Db_sanitize=memory')
996            # check_call_cmd('meson', 'test', '-C', 'build'
997            #                '--logbase', 'testlog-msan')
998            check_call_cmd('meson', 'configure', 'build',
999                           '-Db_sanitize=none')
1000        else:
1001            sys.stderr.write("###### Skipping sanitizers ######\n")
1002
1003        # Run coverage checks
1004        check_call_cmd('meson', 'configure', 'build',
1005                       '-Db_coverage=true')
1006        self.test()
1007        # Only build coverage HTML if coverage files were produced
1008        for root, dirs, files in os.walk('build'):
1009            if any([f.endswith('.gcda') for f in files]):
1010                check_call_cmd('ninja', '-C', 'build',
1011                               'coverage-html')
1012                break
1013        check_call_cmd('meson', 'configure', 'build',
1014                       '-Db_coverage=false')
1015        run_cppcheck()
1016
1017    def _extra_meson_checks(self):
1018        with open(os.path.join(self.path, 'meson.build'), 'rt') as f:
1019            build_contents = f.read()
1020
1021        # Find project's specified meson_version.
1022        meson_version = None
1023        pattern = r"meson_version:[^']*'([^']*)'"
1024        for match in re.finditer(pattern, build_contents):
1025            group = match.group(1)
1026            meson_version = group
1027
1028        # C++20 requires at least Meson 0.57 but Meson itself doesn't
1029        # identify this.  Add to our unit-test checks so that we don't
1030        # get a meson.build missing this.
1031        pattern = r"'cpp_std=c\+\+20'"
1032        for match in re.finditer(pattern, build_contents):
1033            if not meson_version or \
1034                    not meson_version_compare(meson_version, ">=0.57"):
1035                raise Exception(
1036                    "C++20 support requires specifying in meson.build: "
1037                    + "meson_version: '>=0.57'"
1038                )
1039
1040
1041class Package(object):
1042    def __init__(self, name=None, path=None):
1043        self.supported = [Meson, Autotools, CMake]
1044        self.name = name
1045        self.path = path
1046        self.test_only = False
1047
1048    def build_systems(self):
1049        instances = (system(self.name, self.path) for system in self.supported)
1050        return (instance for instance in instances if instance.probe())
1051
1052    def build_system(self, preferred=None):
1053        systems = list(self.build_systems())
1054
1055        if not systems:
1056            return None
1057
1058        if preferred:
1059            return {type(system): system for system in systems}[preferred]
1060
1061        return next(iter(systems))
1062
1063    def install(self, system=None):
1064        if not system:
1065            system = self.build_system()
1066
1067        system.configure(False)
1068        system.build()
1069        system.install()
1070
1071    def _test_one(self, system):
1072        system.configure(True)
1073        system.build()
1074        system.install()
1075        system.test()
1076        if not TEST_ONLY:
1077            system.analyze()
1078
1079    def test(self):
1080        for system in self.build_systems():
1081            self._test_one(system)
1082
1083
1084def find_file(filename, basedir):
1085    """
1086    Finds all occurrences of a file (or list of files) in the base
1087    directory and passes them back with their relative paths.
1088
1089    Parameter descriptions:
1090    filename              The name of the file (or list of files) to
1091                          find
1092    basedir               The base directory search in
1093    """
1094
1095    if not isinstance(filename, list):
1096        filename = [ filename ]
1097
1098    filepaths = []
1099    for root, dirs, files in os.walk(basedir):
1100        for f in filename:
1101            if f in files:
1102                filepaths.append(os.path.join(root, f))
1103    return filepaths
1104
1105
1106if __name__ == '__main__':
1107    # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS]
1108    CONFIGURE_FLAGS = {
1109        'phosphor-logging':
1110        ['--enable-metadata-processing', '--enable-openpower-pel-extension',
1111         'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml']
1112    }
1113
1114    # MESON_FLAGS = [GIT REPO]:[MESON FLAGS]
1115    MESON_FLAGS = {
1116        'phosphor-dbus-interfaces':
1117        ['-Ddata_com_ibm=true', '-Ddata_org_open_power=true'],
1118        'phosphor-logging':
1119        ['-Dopenpower-pel-extension=enabled']
1120    }
1121
1122    # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO]
1123    DEPENDENCIES = {
1124        'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'},
1125        'AC_CHECK_HEADER': {
1126            'host-ipmid': 'phosphor-host-ipmid',
1127            'blobs-ipmid': 'phosphor-ipmi-blobs',
1128            'sdbusplus': 'sdbusplus',
1129            'sdeventplus': 'sdeventplus',
1130            'stdplus': 'stdplus',
1131            'gpioplus': 'gpioplus',
1132            'phosphor-logging/log.hpp': 'phosphor-logging',
1133        },
1134        'AC_PATH_PROG': {'sdbus++': 'sdbusplus'},
1135        'PKG_CHECK_MODULES': {
1136            'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces',
1137            'libipmid': 'phosphor-host-ipmid',
1138            'libipmid-host': 'phosphor-host-ipmid',
1139            'sdbusplus': 'sdbusplus',
1140            'sdeventplus': 'sdeventplus',
1141            'stdplus': 'stdplus',
1142            'gpioplus': 'gpioplus',
1143            'phosphor-logging': 'phosphor-logging',
1144            'phosphor-snmp': 'phosphor-snmp',
1145            'ipmiblob': 'ipmi-blob-tool',
1146            'hei': 'openpower-libhei',
1147            'phosphor-ipmi-blobs': 'phosphor-ipmi-blobs',
1148        },
1149    }
1150
1151    # Offset into array of macro parameters MACRO(0, 1, ...N)
1152    DEPENDENCIES_OFFSET = {
1153        'AC_CHECK_LIB': 0,
1154        'AC_CHECK_HEADER': 0,
1155        'AC_PATH_PROG': 1,
1156        'PKG_CHECK_MODULES': 1,
1157    }
1158
1159    # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING]
1160    DEPENDENCIES_REGEX = {
1161        'phosphor-logging': r'\S+-dbus-interfaces$'
1162    }
1163
1164    # Set command line arguments
1165    parser = argparse.ArgumentParser()
1166    parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True,
1167                        help="Workspace directory location(i.e. /home)")
1168    parser.add_argument("-p", "--package", dest="PACKAGE", required=True,
1169                        help="OpenBMC package to be unit tested")
1170    parser.add_argument("-t", "--test-only", dest="TEST_ONLY",
1171                        action="store_true", required=False, default=False,
1172                        help="Only run test cases, no other validation")
1173    arg_inttests = parser.add_mutually_exclusive_group()
1174    arg_inttests.add_argument("--integration-tests", dest="INTEGRATION_TEST",
1175                        action="store_true", required=False, default=True,
1176                        help="Enable integration tests [default].")
1177    arg_inttests.add_argument("--no-integration-tests", dest="INTEGRATION_TEST",
1178                        action="store_false", required=False,
1179                        help="Disable integration tests.")
1180    parser.add_argument("-v", "--verbose", action="store_true",
1181                        help="Print additional package status messages")
1182    parser.add_argument("-r", "--repeat", help="Repeat tests N times",
1183                        type=int, default=1)
1184    parser.add_argument("-b", "--branch", dest="BRANCH", required=False,
1185                        help="Branch to target for dependent repositories",
1186                        default="master")
1187    parser.add_argument("-n", "--noformat", dest="FORMAT",
1188                        action="store_false", required=False,
1189                        help="Whether or not to run format code")
1190    args = parser.parse_args(sys.argv[1:])
1191    WORKSPACE = args.WORKSPACE
1192    UNIT_TEST_PKG = args.PACKAGE
1193    TEST_ONLY = args.TEST_ONLY
1194    INTEGRATION_TEST = args.INTEGRATION_TEST
1195    BRANCH = args.BRANCH
1196    FORMAT_CODE = args.FORMAT
1197    if args.verbose:
1198        def printline(*line):
1199            for arg in line:
1200                print(arg, end=' ')
1201            print()
1202    else:
1203        def printline(*line):
1204            pass
1205
1206    CODE_SCAN_DIR = os.path.join(WORKSPACE, UNIT_TEST_PKG)
1207
1208    # First validate code formatting if repo has style formatting files.
1209    # The format-code.sh checks for these files.
1210    if FORMAT_CODE:
1211        format_scripts = find_file(['format-code.sh', 'format-code'],
1212                                   CODE_SCAN_DIR)
1213
1214        # use default format-code.sh if no other found
1215        if not format_scripts:
1216            format_scripts.append(os.path.join(WORKSPACE, "format-code.sh"))
1217
1218        for f in format_scripts:
1219            check_call_cmd(f, CODE_SCAN_DIR)
1220
1221    # Check if this repo has a supported make infrastructure
1222    pkg = Package(UNIT_TEST_PKG, CODE_SCAN_DIR)
1223    if not pkg.build_system():
1224        print("No valid build system, exit")
1225        sys.exit(0)
1226
1227    prev_umask = os.umask(000)
1228
1229    # Determine dependencies and add them
1230    dep_added = dict()
1231    dep_added[UNIT_TEST_PKG] = False
1232
1233    # Create dependency tree
1234    dep_tree = DepTree(UNIT_TEST_PKG)
1235    build_dep_tree(UNIT_TEST_PKG, CODE_SCAN_DIR, dep_added, dep_tree, BRANCH)
1236
1237    # Reorder Dependency Tree
1238    for pkg_name, regex_str in DEPENDENCIES_REGEX.items():
1239        dep_tree.ReorderDeps(pkg_name, regex_str)
1240    if args.verbose:
1241        dep_tree.PrintTree()
1242
1243    install_list = dep_tree.GetInstallList()
1244
1245    # We don't want to treat our package as a dependency
1246    install_list.remove(UNIT_TEST_PKG)
1247
1248    # Install reordered dependencies
1249    for dep in install_list:
1250        build_and_install(dep, False)
1251
1252    # Run package unit tests
1253    build_and_install(UNIT_TEST_PKG, True)
1254
1255    os.umask(prev_umask)
1256
1257    # Run any custom CI scripts the repo has, of which there can be
1258    # multiple of and anywhere in the repository.
1259    ci_scripts = find_file(['run-ci.sh', 'run-ci'], CODE_SCAN_DIR)
1260    if ci_scripts:
1261        os.chdir(CODE_SCAN_DIR)
1262        for ci_script in ci_scripts:
1263            check_call_cmd(ci_script)
1264