1# Recipe creation tool - create build system handler for python
2#
3# Copyright (C) 2015 Mentor Graphics Corporation
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7
8import ast
9import codecs
10import collections
11import setuptools.command.build_py
12import email
13import imp
14import glob
15import itertools
16import logging
17import os
18import re
19import sys
20import subprocess
21from recipetool.create import RecipeHandler
22
23logger = logging.getLogger('recipetool')
24
25tinfoil = None
26
27
28def tinfoil_init(instance):
29    global tinfoil
30    tinfoil = instance
31
32
33class PythonRecipeHandler(RecipeHandler):
34    base_pkgdeps = ['python3-core']
35    excluded_pkgdeps = ['python3-dbg']
36    # os.path is provided by python3-core
37    assume_provided = ['builtins', 'os.path']
38    # Assumes that the host python3 builtin_module_names is sane for target too
39    assume_provided = assume_provided + list(sys.builtin_module_names)
40
41    bbvar_map = {
42        'Name': 'PN',
43        'Version': 'PV',
44        'Home-page': 'HOMEPAGE',
45        'Summary': 'SUMMARY',
46        'Description': 'DESCRIPTION',
47        'License': 'LICENSE',
48        'Requires': 'RDEPENDS:${PN}',
49        'Provides': 'RPROVIDES:${PN}',
50        'Obsoletes': 'RREPLACES:${PN}',
51    }
52    # PN/PV are already set by recipetool core & desc can be extremely long
53    excluded_fields = [
54        'Description',
55    ]
56    setup_parse_map = {
57        'Url': 'Home-page',
58        'Classifiers': 'Classifier',
59        'Description': 'Summary',
60    }
61    setuparg_map = {
62        'Home-page': 'url',
63        'Classifier': 'classifiers',
64        'Summary': 'description',
65        'Description': 'long-description',
66    }
67    # Values which are lists, used by the setup.py argument based metadata
68    # extraction method, to determine how to process the setup.py output.
69    setuparg_list_fields = [
70        'Classifier',
71        'Requires',
72        'Provides',
73        'Obsoletes',
74        'Platform',
75        'Supported-Platform',
76    ]
77    setuparg_multi_line_values = ['Description']
78    replacements = [
79        ('License', r' +$', ''),
80        ('License', r'^ +', ''),
81        ('License', r' ', '-'),
82        ('License', r'^GNU-', ''),
83        ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
84        ('License', r'^UNKNOWN$', ''),
85
86        # Remove currently unhandled version numbers from these variables
87        ('Requires', r' *\([^)]*\)', ''),
88        ('Provides', r' *\([^)]*\)', ''),
89        ('Obsoletes', r' *\([^)]*\)', ''),
90        ('Install-requires', r'^([^><= ]+).*', r'\1'),
91        ('Extras-require', r'^([^><= ]+).*', r'\1'),
92        ('Tests-require', r'^([^><= ]+).*', r'\1'),
93
94        # Remove unhandled dependency on particular features (e.g. foo[PDF])
95        ('Install-requires', r'\[[^\]]+\]$', ''),
96    ]
97
98    classifier_license_map = {
99        'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
100        'License :: OSI Approved :: Apache Software License': 'Apache',
101        'License :: OSI Approved :: Apple Public Source License': 'APSL',
102        'License :: OSI Approved :: Artistic License': 'Artistic',
103        'License :: OSI Approved :: Attribution Assurance License': 'AAL',
104        'License :: OSI Approved :: BSD License': 'BSD-3-Clause',
105        'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0',
106        'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1',
107        'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0',
108        'License :: OSI Approved :: Common Public License': 'CPL',
109        'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0',
110        'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0',
111        'License :: OSI Approved :: Eiffel Forum License': 'EFL',
112        'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
113        'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
114        'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2',
115        'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only',
116        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later',
117        'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
118        'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
119        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only',
120        'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later',
121        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only',
122        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later',
123        'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only',
124        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later',
125        'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only',
126        'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later',
127        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
128        'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND',
129        'License :: OSI Approved :: IBM Public License': 'IPL',
130        'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
131        'License :: OSI Approved :: Intel Open Source License': 'Intel',
132        'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
133        'License :: OSI Approved :: MIT License': 'MIT',
134        'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0',
135        'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
136        'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS',
137        'License :: OSI Approved :: Motosoto License': 'Motosoto',
138        'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
139        'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
140        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
141        'License :: OSI Approved :: Nethack General Public License': 'NGPL',
142        'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
143        'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
144        'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0',
145        'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL',
146        'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
147        'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0',
148        'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
149        'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
150        'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1',
151        'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
152        'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL',
153        'License :: OSI Approved :: Sun Public License': 'SPL',
154        'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense',
155        'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0',
156        'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
157        'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
158        'License :: OSI Approved :: W3C License': 'W3C',
159        'License :: OSI Approved :: X.Net License': 'Xnet',
160        'License :: OSI Approved :: Zope Public License': 'ZPL',
161        'License :: OSI Approved :: zlib/libpng License': 'Zlib',
162        'License :: Other/Proprietary License': 'Proprietary',
163        'License :: Public Domain': 'PD',
164    }
165
166    def __init__(self):
167        pass
168
169    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
170        if 'buildsystem' in handled:
171            return False
172
173        # Check for non-zero size setup.py files
174        setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
175        for fn in setupfiles:
176            if os.path.getsize(fn):
177                break
178        else:
179            return False
180
181        # setup.py is always parsed to get at certain required information, such as
182        # distutils vs setuptools
183        #
184        # If egg info is available, we use it for both its PKG-INFO metadata
185        # and for its requires.txt for install_requires.
186        # If PKG-INFO is available but no egg info is, we use that for metadata in preference to
187        # the parsed setup.py, but use the install_requires info from the
188        # parsed setup.py.
189
190        setupscript = os.path.join(srctree, 'setup.py')
191        try:
192            setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript)
193        except Exception:
194            logger.exception("Failed to parse setup.py")
195            setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], []
196
197        egginfo = glob.glob(os.path.join(srctree, '*.egg-info'))
198        if egginfo:
199            info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO'))
200            requires_txt = os.path.join(egginfo[0], 'requires.txt')
201            if os.path.exists(requires_txt):
202                with codecs.open(requires_txt) as f:
203                    inst_req = []
204                    extras_req = collections.defaultdict(list)
205                    current_feature = None
206                    for line in f.readlines():
207                        line = line.rstrip()
208                        if not line:
209                            continue
210
211                        if line.startswith('['):
212                            # PACKAGECONFIG must not contain expressions or whitespace
213                            line = line.replace(" ", "")
214                            line = line.replace(':', "")
215                            line = line.replace('.', "-dot-")
216                            line = line.replace('"', "")
217                            line = line.replace('<', "-smaller-")
218                            line = line.replace('>', "-bigger-")
219                            line = line.replace('_', "-")
220                            line = line.replace('(', "")
221                            line = line.replace(')', "")
222                            line = line.replace('!', "-not-")
223                            line = line.replace('=', "-equals-")
224                            current_feature = line[1:-1]
225                        elif current_feature:
226                            extras_req[current_feature].append(line)
227                        else:
228                            inst_req.append(line)
229                    info['Install-requires'] = inst_req
230                    info['Extras-require'] = extras_req
231        elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']):
232            info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO'))
233
234            if setup_info:
235                if 'Install-requires' in setup_info:
236                    info['Install-requires'] = setup_info['Install-requires']
237                if 'Extras-require' in setup_info:
238                    info['Extras-require'] = setup_info['Extras-require']
239        else:
240            if setup_info:
241                info = setup_info
242            else:
243                info = self.get_setup_args_info(setupscript)
244
245        # Grab the license value before applying replacements
246        license_str = info.get('License', '').strip()
247
248        self.apply_info_replacements(info)
249
250        if uses_setuptools:
251            classes.append('setuptools3')
252        else:
253            classes.append('distutils3')
254
255        if license_str:
256            for i, line in enumerate(lines_before):
257                if line.startswith('LICENSE = '):
258                    lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
259                    break
260
261        if 'Classifier' in info:
262            existing_licenses = info.get('License', '')
263            licenses = []
264            for classifier in info['Classifier']:
265                if classifier in self.classifier_license_map:
266                    license = self.classifier_license_map[classifier]
267                    if license == 'Apache' and 'Apache-2.0' in existing_licenses:
268                        license = 'Apache-2.0'
269                    elif license == 'GPL':
270                        if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
271                            license = 'GPL-2.0'
272                        elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
273                            license = 'GPL-3.0'
274                    elif license == 'LGPL':
275                        if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
276                            license = 'LGPL-2.1'
277                        elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
278                            license = 'LGPL-2.0'
279                        elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
280                            license = 'LGPL-3.0'
281                    licenses.append(license)
282
283            if licenses:
284                info['License'] = ' & '.join(licenses)
285
286        # Map PKG-INFO & setup.py fields to bitbake variables
287        for field, values in info.items():
288            if field in self.excluded_fields:
289                continue
290
291            if field not in self.bbvar_map:
292                continue
293
294            if isinstance(values, str):
295                value = values
296            else:
297                value = ' '.join(str(v) for v in values if v)
298
299            bbvar = self.bbvar_map[field]
300            if bbvar not in extravalues and value:
301                extravalues[bbvar] = value
302
303        mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
304
305        extras_req = set()
306        if 'Extras-require' in info:
307            extras_req = info['Extras-require']
308            if extras_req:
309                lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
310                lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
311                lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
312                lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
313                lines_after.append('#')
314                lines_after.append('# Uncomment this line to enable all the optional features.')
315                lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
316                for feature, feature_reqs in extras_req.items():
317                    unmapped_deps.difference_update(feature_reqs)
318
319                    feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
320                    lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
321
322        inst_reqs = set()
323        if 'Install-requires' in info:
324            if extras_req:
325                lines_after.append('')
326            inst_reqs = info['Install-requires']
327            if inst_reqs:
328                unmapped_deps.difference_update(inst_reqs)
329
330                inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
331                lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
332                lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
333                lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
334
335        if mapped_deps:
336            name = info.get('Name')
337            if name and name[0] in mapped_deps:
338                # Attempt to avoid self-reference
339                mapped_deps.remove(name[0])
340            mapped_deps -= set(self.excluded_pkgdeps)
341            if inst_reqs or extras_req:
342                lines_after.append('')
343            lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
344            lines_after.append('# python sources, and might not be 100% accurate.')
345            lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
346
347        unmapped_deps -= set(extensions)
348        unmapped_deps -= set(self.assume_provided)
349        if unmapped_deps:
350            if mapped_deps:
351                lines_after.append('')
352            lines_after.append('# WARNING: We were unable to map the following python package/module')
353            lines_after.append('# dependencies to the bitbake packages which include them:')
354            lines_after.extend('#    {}'.format(d) for d in sorted(unmapped_deps))
355
356        handled.append('buildsystem')
357
358    def get_pkginfo(self, pkginfo_fn):
359        msg = email.message_from_file(open(pkginfo_fn, 'r'))
360        msginfo = {}
361        for field in msg.keys():
362            values = msg.get_all(field)
363            if len(values) == 1:
364                msginfo[field] = values[0]
365            else:
366                msginfo[field] = values
367        return msginfo
368
369    def parse_setup_py(self, setupscript='./setup.py'):
370        with codecs.open(setupscript) as f:
371            info, imported_modules, non_literals, extensions = gather_setup_info(f)
372
373        def _map(key):
374            key = key.replace('_', '-')
375            key = key[0].upper() + key[1:]
376            if key in self.setup_parse_map:
377                key = self.setup_parse_map[key]
378            return key
379
380        # Naive mapping of setup() arguments to PKG-INFO field names
381        for d in [info, non_literals]:
382            for key, value in list(d.items()):
383                if key is None:
384                    continue
385                new_key = _map(key)
386                if new_key != key:
387                    del d[key]
388                    d[new_key] = value
389
390        return info, 'setuptools' in imported_modules, non_literals, extensions
391
392    def get_setup_args_info(self, setupscript='./setup.py'):
393        cmd = ['python3', setupscript]
394        info = {}
395        keys = set(self.bbvar_map.keys())
396        keys |= set(self.setuparg_list_fields)
397        keys |= set(self.setuparg_multi_line_values)
398        grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
399        for index, keys in grouped_keys:
400            if index == (True, False):
401                # Splitlines output for each arg as a list value
402                for key in keys:
403                    arg = self.setuparg_map.get(key, key.lower())
404                    try:
405                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
406                    except (OSError, subprocess.CalledProcessError):
407                        pass
408                    else:
409                        info[key] = [l.rstrip() for l in arg_info.splitlines()]
410            elif index == (False, True):
411                # Entire output for each arg
412                for key in keys:
413                    arg = self.setuparg_map.get(key, key.lower())
414                    try:
415                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
416                    except (OSError, subprocess.CalledProcessError):
417                        pass
418                    else:
419                        info[key] = arg_info
420            else:
421                info.update(self.get_setup_byline(list(keys), setupscript))
422        return info
423
424    def get_setup_byline(self, fields, setupscript='./setup.py'):
425        info = {}
426
427        cmd = ['python3', setupscript]
428        cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
429        try:
430            info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
431        except (OSError, subprocess.CalledProcessError):
432            pass
433        else:
434            if len(fields) != len(info_lines):
435                logger.error('Mismatch between setup.py output lines and number of fields')
436                sys.exit(1)
437
438            for lineno, line in enumerate(info_lines):
439                line = line.rstrip()
440                info[fields[lineno]] = line
441        return info
442
443    def apply_info_replacements(self, info):
444        for variable, search, replace in self.replacements:
445            if variable not in info:
446                continue
447
448            def replace_value(search, replace, value):
449                if replace is None:
450                    if re.search(search, value):
451                        return None
452                else:
453                    new_value = re.sub(search, replace, value)
454                    if value != new_value:
455                        return new_value
456                return value
457
458            value = info[variable]
459            if isinstance(value, str):
460                new_value = replace_value(search, replace, value)
461                if new_value is None:
462                    del info[variable]
463                elif new_value != value:
464                    info[variable] = new_value
465            elif hasattr(value, 'items'):
466                for dkey, dvalue in list(value.items()):
467                    new_list = []
468                    for pos, a_value in enumerate(dvalue):
469                        new_value = replace_value(search, replace, a_value)
470                        if new_value is not None and new_value != value:
471                            new_list.append(new_value)
472
473                    if value != new_list:
474                        value[dkey] = new_list
475            else:
476                new_list = []
477                for pos, a_value in enumerate(value):
478                    new_value = replace_value(search, replace, a_value)
479                    if new_value is not None and new_value != value:
480                        new_list.append(new_value)
481
482                if value != new_list:
483                    info[variable] = new_list
484
485    def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
486        if 'Package-dir' in setup_info:
487            package_dir = setup_info['Package-dir']
488        else:
489            package_dir = {}
490
491        dist = setuptools.Distribution()
492
493        class PackageDir(setuptools.command.build_py.build_py):
494            def __init__(self, package_dir):
495                self.package_dir = package_dir
496                self.dist = dist
497                super().__init__(self.dist)
498
499        pd = PackageDir(package_dir)
500        to_scan = []
501        if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
502            if 'Py-modules' in setup_info:
503                for module in setup_info['Py-modules']:
504                    try:
505                        package, module = module.rsplit('.', 1)
506                    except ValueError:
507                        package, module = '.', module
508                    module_path = os.path.join(pd.get_package_dir(package), module + '.py')
509                    to_scan.append(module_path)
510
511            if 'Packages' in setup_info:
512                for package in setup_info['Packages']:
513                    to_scan.append(pd.get_package_dir(package))
514
515            if 'Scripts' in setup_info:
516                to_scan.extend(setup_info['Scripts'])
517        else:
518            logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
519
520        if not to_scan:
521            to_scan = ['.']
522
523        logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
524
525        provided_packages = self.parse_pkgdata_for_python_packages()
526        scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
527        mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
528        for dep in scanned_deps:
529            mapped = provided_packages.get(dep)
530            if mapped:
531                logger.debug('Mapped %s to %s' % (dep, mapped))
532                mapped_deps.add(mapped)
533            else:
534                logger.debug('Could not map %s' % dep)
535                unmapped_deps.add(dep)
536        return mapped_deps, unmapped_deps
537
538    def scan_python_dependencies(self, paths):
539        deps = set()
540        try:
541            dep_output = self.run_command(['pythondeps', '-d'] + paths)
542        except (OSError, subprocess.CalledProcessError):
543            pass
544        else:
545            for line in dep_output.splitlines():
546                line = line.rstrip()
547                dep, filename = line.split('\t', 1)
548                if filename.endswith('/setup.py'):
549                    continue
550                deps.add(dep)
551
552        try:
553            provides_output = self.run_command(['pythondeps', '-p'] + paths)
554        except (OSError, subprocess.CalledProcessError):
555            pass
556        else:
557            provides_lines = (l.rstrip() for l in provides_output.splitlines())
558            provides = set(l for l in provides_lines if l and l != 'setup')
559            deps -= provides
560
561        return deps
562
563    def parse_pkgdata_for_python_packages(self):
564        suffixes = [t[0] for t in imp.get_suffixes()]
565        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
566
567        ldata = tinfoil.config_data.createCopy()
568        bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True)
569        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
570
571        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
572        python_dirs = [python_sitedir + os.sep,
573                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
574                       os.path.dirname(python_sitedir) + os.sep]
575        packages = {}
576        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
577            files_info = None
578            with open(pkgdatafile, 'r') as f:
579                for line in f.readlines():
580                    field, value = line.split(': ', 1)
581                    if field.startswith('FILES_INFO'):
582                        files_info = ast.literal_eval(value)
583                        break
584                else:
585                    continue
586
587            for fn in files_info:
588                for suffix in suffixes:
589                    if fn.endswith(suffix):
590                        break
591                else:
592                    continue
593
594                if fn.startswith(dynload_dir + os.sep):
595                    if '/.debug/' in fn:
596                        continue
597                    base = os.path.basename(fn)
598                    provided = base.split('.', 1)[0]
599                    packages[provided] = os.path.basename(pkgdatafile)
600                    continue
601
602                for python_dir in python_dirs:
603                    if fn.startswith(python_dir):
604                        relpath = fn[len(python_dir):]
605                        relstart, _, relremaining = relpath.partition(os.sep)
606                        if relstart.endswith('.egg'):
607                            relpath = relremaining
608                        base, _ = os.path.splitext(relpath)
609
610                        if '/.debug/' in base:
611                            continue
612                        if os.path.basename(base) == '__init__':
613                            base = os.path.dirname(base)
614                        base = base.replace(os.sep + os.sep, os.sep)
615                        provided = base.replace(os.sep, '.')
616                        packages[provided] = os.path.basename(pkgdatafile)
617        return packages
618
619    @classmethod
620    def run_command(cls, cmd, **popenargs):
621        if 'stderr' not in popenargs:
622            popenargs['stderr'] = subprocess.STDOUT
623        try:
624            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
625        except OSError as exc:
626            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
627            raise
628        except subprocess.CalledProcessError as exc:
629            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
630            raise
631
632
633def gather_setup_info(fileobj):
634    parsed = ast.parse(fileobj.read(), fileobj.name)
635    visitor = SetupScriptVisitor()
636    visitor.visit(parsed)
637
638    non_literals, extensions = {}, []
639    for key, value in list(visitor.keywords.items()):
640        if key == 'ext_modules':
641            if isinstance(value, list):
642                for ext in value:
643                    if  (isinstance(ext, ast.Call) and
644                         isinstance(ext.func, ast.Name) and
645                         ext.func.id == 'Extension' and
646                         not has_non_literals(ext.args)):
647                        extensions.append(ext.args[0])
648        elif has_non_literals(value):
649            non_literals[key] = value
650            del visitor.keywords[key]
651
652    return visitor.keywords, visitor.imported_modules, non_literals, extensions
653
654
655class SetupScriptVisitor(ast.NodeVisitor):
656    def __init__(self):
657        ast.NodeVisitor.__init__(self)
658        self.keywords = {}
659        self.non_literals = []
660        self.imported_modules = set()
661
662    def visit_Expr(self, node):
663        if isinstance(node.value, ast.Call) and \
664           isinstance(node.value.func, ast.Name) and \
665           node.value.func.id == 'setup':
666            self.visit_setup(node.value)
667
668    def visit_setup(self, node):
669        call = LiteralAstTransform().visit(node)
670        self.keywords = call.keywords
671        for k, v in self.keywords.items():
672            if has_non_literals(v):
673               self.non_literals.append(k)
674
675    def visit_Import(self, node):
676        for alias in node.names:
677            self.imported_modules.add(alias.name)
678
679    def visit_ImportFrom(self, node):
680        self.imported_modules.add(node.module)
681
682
683class LiteralAstTransform(ast.NodeTransformer):
684    """Simplify the ast through evaluation of literals."""
685    excluded_fields = ['ctx']
686
687    def visit(self, node):
688        if not isinstance(node, ast.AST):
689            return node
690        else:
691            return ast.NodeTransformer.visit(self, node)
692
693    def generic_visit(self, node):
694        try:
695            return ast.literal_eval(node)
696        except ValueError:
697            for field, value in ast.iter_fields(node):
698                if field in self.excluded_fields:
699                    delattr(node, field)
700                if value is None:
701                    continue
702
703                if isinstance(value, list):
704                    if field in ('keywords', 'kwargs'):
705                        new_value = dict((kw.arg, self.visit(kw.value)) for kw in value)
706                    else:
707                        new_value = [self.visit(i) for i in value]
708                else:
709                    new_value = self.visit(value)
710                setattr(node, field, new_value)
711            return node
712
713    def visit_Name(self, node):
714        if hasattr('__builtins__', node.id):
715            return getattr(__builtins__, node.id)
716        else:
717            return self.generic_visit(node)
718
719    def visit_Tuple(self, node):
720        return tuple(self.visit(v) for v in node.elts)
721
722    def visit_List(self, node):
723        return [self.visit(v) for v in node.elts]
724
725    def visit_Set(self, node):
726        return set(self.visit(v) for v in node.elts)
727
728    def visit_Dict(self, node):
729        keys = (self.visit(k) for k in node.keys)
730        values = (self.visit(v) for v in node.values)
731        return dict(zip(keys, values))
732
733
734def has_non_literals(value):
735    if isinstance(value, ast.AST):
736        return True
737    elif isinstance(value, str):
738        return False
739    elif hasattr(value, 'values'):
740        return any(has_non_literals(v) for v in value.values())
741    elif hasattr(value, '__iter__'):
742        return any(has_non_literals(v) for v in value)
743
744
745def register_recipe_handlers(handlers):
746    # We need to make sure this is ahead of the makefile fallback handler
747    handlers.append((PythonRecipeHandler(), 70))
748