xref: /openbmc/openbmc/poky/scripts/pythondeps (revision 92b42cb3) 
1#!/usr/bin/env python3
2#
3# Copyright OpenEmbedded Contributors
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7# Determine dependencies of python scripts or available python modules in a search path.
8#
9# Given the -d argument and a filename/filenames, returns the modules imported by those files.
10# Given the -d argument and a directory/directories, recurses to find all
11# python packages and modules, returns the modules imported by these.
12# Given the -p argument and a path or paths, scans that path for available python modules/packages.
13
14import argparse
15import ast
16import importlib
17from importlib import machinery
18import logging
19import os.path
20import sys
21
22
23logger = logging.getLogger('pythondeps')
24
25suffixes = importlib.machinery.all_suffixes()
26
27class PythonDepError(Exception):
28    pass
29
30
31class DependError(PythonDepError):
32    def __init__(self, path, error):
33        self.path = path
34        self.error = error
35        PythonDepError.__init__(self, error)
36
37    def __str__(self):
38        return "Failure determining dependencies of {}: {}".format(self.path, self.error)
39
40
41class ImportVisitor(ast.NodeVisitor):
42    def __init__(self):
43        self.imports = set()
44        self.importsfrom = []
45
46    def visit_Import(self, node):
47        for alias in node.names:
48            self.imports.add(alias.name)
49
50    def visit_ImportFrom(self, node):
51        self.importsfrom.append((node.module, [a.name for a in node.names], node.level))
52
53
54def walk_up(path):
55    while path:
56        yield path
57        path, _, _ = path.rpartition(os.sep)
58
59
60def get_provides(path):
61    path = os.path.realpath(path)
62
63    def get_fn_name(fn):
64        for suffix in suffixes:
65            if fn.endswith(suffix):
66                return fn[:-len(suffix)]
67
68    isdir = os.path.isdir(path)
69    if isdir:
70        pkg_path = path
71        walk_path = path
72    else:
73        pkg_path = get_fn_name(path)
74        if pkg_path is None:
75            return
76        walk_path = os.path.dirname(path)
77
78    for curpath in walk_up(walk_path):
79        if not os.path.exists(os.path.join(curpath, '__init__.py')):
80            libdir = curpath
81            break
82    else:
83        libdir = ''
84
85    package_relpath = pkg_path[len(libdir)+1:]
86    package = '.'.join(package_relpath.split(os.sep))
87    if not isdir:
88        yield package, path
89    else:
90        if os.path.exists(os.path.join(path, '__init__.py')):
91            yield package, path
92
93        for dirpath, dirnames, filenames in os.walk(path):
94            relpath = dirpath[len(path)+1:]
95            if relpath:
96                if '__init__.py' not in filenames:
97                    dirnames[:] = []
98                    continue
99                else:
100                    context = '.'.join(relpath.split(os.sep))
101                    if package:
102                        context = package + '.' + context
103                    yield context, dirpath
104            else:
105                context = package
106
107            for fn in filenames:
108                adjusted_fn = get_fn_name(fn)
109                if not adjusted_fn or adjusted_fn == '__init__':
110                    continue
111
112                fullfn = os.path.join(dirpath, fn)
113                if context:
114                    yield context + '.' + adjusted_fn, fullfn
115                else:
116                    yield adjusted_fn, fullfn
117
118
119def get_code_depends(code_string, path=None, provide=None, ispkg=False):
120    try:
121        code = ast.parse(code_string, path)
122    except TypeError as exc:
123        raise DependError(path, exc)
124    except SyntaxError as exc:
125        raise DependError(path, exc)
126
127    visitor = ImportVisitor()
128    visitor.visit(code)
129    for builtin_module in sys.builtin_module_names:
130        if builtin_module in visitor.imports:
131            visitor.imports.remove(builtin_module)
132
133    if provide:
134        provide_elements = provide.split('.')
135        if ispkg:
136            provide_elements.append("__self__")
137        context = '.'.join(provide_elements[:-1])
138        package_path = os.path.dirname(path)
139    else:
140        context = None
141        package_path = None
142
143    levelzero_importsfrom = (module for module, names, level in visitor.importsfrom
144                             if level == 0)
145    for module in visitor.imports | set(levelzero_importsfrom):
146        if context and path:
147            module_basepath = os.path.join(package_path, module.replace('.', '/'))
148            if os.path.exists(module_basepath):
149                # Implicit relative import
150                yield context + '.' + module, path
151                continue
152
153            for suffix in suffixes:
154                if os.path.exists(module_basepath + suffix):
155                    # Implicit relative import
156                    yield context + '.' + module, path
157                    break
158            else:
159                yield module, path
160        else:
161            yield module, path
162
163    for module, names, level in visitor.importsfrom:
164        if level == 0:
165            continue
166        elif not provide:
167            raise DependError("Error: ImportFrom non-zero level outside of a package: {0}".format((module, names, level)), path)
168        elif level > len(provide_elements):
169            raise DependError("Error: ImportFrom level exceeds package depth: {0}".format((module, names, level)), path)
170        else:
171            context = '.'.join(provide_elements[:-level])
172            if module:
173                if context:
174                    yield context + '.' + module, path
175                else:
176                    yield module, path
177
178
179def get_file_depends(path):
180    try:
181        code_string = open(path, 'r').read()
182    except (OSError, IOError) as exc:
183        raise DependError(path, exc)
184
185    return get_code_depends(code_string, path)
186
187
188def get_depends_recursive(directory):
189    directory = os.path.realpath(directory)
190
191    provides = dict((v, k) for k, v in get_provides(directory))
192    for filename, provide in provides.items():
193        if os.path.isdir(filename):
194            filename = os.path.join(filename, '__init__.py')
195            ispkg = True
196        elif not filename.endswith('.py'):
197            continue
198        else:
199            ispkg = False
200
201        with open(filename, 'r') as f:
202            source = f.read()
203
204        depends = get_code_depends(source, filename, provide, ispkg)
205        for depend, by in depends:
206            yield depend, by
207
208
209def get_depends(path):
210    if os.path.isdir(path):
211        return get_depends_recursive(path)
212    else:
213        return get_file_depends(path)
214
215
216def main():
217    logging.basicConfig()
218
219    parser = argparse.ArgumentParser(description='Determine dependencies and provided packages for python scripts/modules')
220    parser.add_argument('path', nargs='+', help='full path to content to be processed')
221    group = parser.add_mutually_exclusive_group()
222    group.add_argument('-p', '--provides', action='store_true',
223                       help='given a path, display the provided python modules')
224    group.add_argument('-d', '--depends', action='store_true',
225                       help='given a filename, display the imported python modules')
226
227    args = parser.parse_args()
228    if args.provides:
229        modules = set()
230        for path in args.path:
231            for provide, fn in get_provides(path):
232                modules.add(provide)
233
234        for module in sorted(modules):
235            print(module)
236    elif args.depends:
237        for path in args.path:
238            try:
239                modules = get_depends(path)
240            except PythonDepError as exc:
241                logger.error(str(exc))
242                sys.exit(1)
243
244            for module, imp_by in modules:
245                print("{}\t{}".format(module, imp_by))
246    else:
247        parser.print_help()
248        sys.exit(2)
249
250
251if __name__ == '__main__':
252    main()
253