1# Recipe creation tool - create build system handler for python 2# 3# Copyright (C) 2015 Mentor Graphics Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7 8import ast 9import codecs 10import collections 11import setuptools.command.build_py 12import email 13import imp 14import glob 15import itertools 16import logging 17import os 18import re 19import sys 20import subprocess 21from recipetool.create import RecipeHandler 22 23logger = logging.getLogger('recipetool') 24 25tinfoil = None 26 27 28def tinfoil_init(instance): 29 global tinfoil 30 tinfoil = instance 31 32 33class PythonRecipeHandler(RecipeHandler): 34 base_pkgdeps = ['python3-core'] 35 excluded_pkgdeps = ['python3-dbg'] 36 # os.path is provided by python3-core 37 assume_provided = ['builtins', 'os.path'] 38 # Assumes that the host python3 builtin_module_names is sane for target too 39 assume_provided = assume_provided + list(sys.builtin_module_names) 40 41 bbvar_map = { 42 'Name': 'PN', 43 'Version': 'PV', 44 'Home-page': 'HOMEPAGE', 45 'Summary': 'SUMMARY', 46 'Description': 'DESCRIPTION', 47 'License': 'LICENSE', 48 'Requires': 'RDEPENDS:${PN}', 49 'Provides': 'RPROVIDES:${PN}', 50 'Obsoletes': 'RREPLACES:${PN}', 51 } 52 # PN/PV are already set by recipetool core & desc can be extremely long 53 excluded_fields = [ 54 'Description', 55 ] 56 setup_parse_map = { 57 'Url': 'Home-page', 58 'Classifiers': 'Classifier', 59 'Description': 'Summary', 60 } 61 setuparg_map = { 62 'Home-page': 'url', 63 'Classifier': 'classifiers', 64 'Summary': 'description', 65 'Description': 'long-description', 66 } 67 # Values which are lists, used by the setup.py argument based metadata 68 # extraction method, to determine how to process the setup.py output. 69 setuparg_list_fields = [ 70 'Classifier', 71 'Requires', 72 'Provides', 73 'Obsoletes', 74 'Platform', 75 'Supported-Platform', 76 ] 77 setuparg_multi_line_values = ['Description'] 78 replacements = [ 79 ('License', r' +$', ''), 80 ('License', r'^ +', ''), 81 ('License', r' ', '-'), 82 ('License', r'^GNU-', ''), 83 ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), 84 ('License', r'^UNKNOWN$', ''), 85 86 # Remove currently unhandled version numbers from these variables 87 ('Requires', r' *\([^)]*\)', ''), 88 ('Provides', r' *\([^)]*\)', ''), 89 ('Obsoletes', r' *\([^)]*\)', ''), 90 ('Install-requires', r'^([^><= ]+).*', r'\1'), 91 ('Extras-require', r'^([^><= ]+).*', r'\1'), 92 ('Tests-require', r'^([^><= ]+).*', r'\1'), 93 94 # Remove unhandled dependency on particular features (e.g. foo[PDF]) 95 ('Install-requires', r'\[[^\]]+\]$', ''), 96 ] 97 98 classifier_license_map = { 99 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', 100 'License :: OSI Approved :: Apache Software License': 'Apache', 101 'License :: OSI Approved :: Apple Public Source License': 'APSL', 102 'License :: OSI Approved :: Artistic License': 'Artistic', 103 'License :: OSI Approved :: Attribution Assurance License': 'AAL', 104 'License :: OSI Approved :: BSD License': 'BSD-3-Clause', 105 'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0', 106 'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1', 107 'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0', 108 'License :: OSI Approved :: Common Public License': 'CPL', 109 'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0', 110 'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0', 111 'License :: OSI Approved :: Eiffel Forum License': 'EFL', 112 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0', 113 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1', 114 'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2', 115 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only', 116 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later', 117 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL', 118 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL', 119 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only', 120 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later', 121 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only', 122 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later', 123 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only', 124 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later', 125 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only', 126 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later', 127 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL', 128 'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND', 129 'License :: OSI Approved :: IBM Public License': 'IPL', 130 'License :: OSI Approved :: ISC License (ISCL)': 'ISC', 131 'License :: OSI Approved :: Intel Open Source License': 'Intel', 132 'License :: OSI Approved :: Jabber Open Source License': 'Jabber', 133 'License :: OSI Approved :: MIT License': 'MIT', 134 'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0', 135 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL', 136 'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS', 137 'License :: OSI Approved :: Motosoto License': 'Motosoto', 138 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0', 139 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1', 140 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0', 141 'License :: OSI Approved :: Nethack General Public License': 'NGPL', 142 'License :: OSI Approved :: Nokia Open Source License': 'Nokia', 143 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL', 144 'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0', 145 'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL', 146 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python', 147 'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0', 148 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL', 149 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL', 150 'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1', 151 'License :: OSI Approved :: Sleepycat License': 'Sleepycat', 152 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL', 153 'License :: OSI Approved :: Sun Public License': 'SPL', 154 'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense', 155 'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0', 156 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA', 157 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0', 158 'License :: OSI Approved :: W3C License': 'W3C', 159 'License :: OSI Approved :: X.Net License': 'Xnet', 160 'License :: OSI Approved :: Zope Public License': 'ZPL', 161 'License :: OSI Approved :: zlib/libpng License': 'Zlib', 162 'License :: Other/Proprietary License': 'Proprietary', 163 'License :: Public Domain': 'PD', 164 } 165 166 def __init__(self): 167 pass 168 169 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): 170 if 'buildsystem' in handled: 171 return False 172 173 # Check for non-zero size setup.py files 174 setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) 175 for fn in setupfiles: 176 if os.path.getsize(fn): 177 break 178 else: 179 return False 180 181 # setup.py is always parsed to get at certain required information, such as 182 # distutils vs setuptools 183 # 184 # If egg info is available, we use it for both its PKG-INFO metadata 185 # and for its requires.txt for install_requires. 186 # If PKG-INFO is available but no egg info is, we use that for metadata in preference to 187 # the parsed setup.py, but use the install_requires info from the 188 # parsed setup.py. 189 190 setupscript = os.path.join(srctree, 'setup.py') 191 try: 192 setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript) 193 except Exception: 194 logger.exception("Failed to parse setup.py") 195 setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], [] 196 197 egginfo = glob.glob(os.path.join(srctree, '*.egg-info')) 198 if egginfo: 199 info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO')) 200 requires_txt = os.path.join(egginfo[0], 'requires.txt') 201 if os.path.exists(requires_txt): 202 with codecs.open(requires_txt) as f: 203 inst_req = [] 204 extras_req = collections.defaultdict(list) 205 current_feature = None 206 for line in f.readlines(): 207 line = line.rstrip() 208 if not line: 209 continue 210 211 if line.startswith('['): 212 # PACKAGECONFIG must not contain expressions or whitespace 213 line = line.replace(" ", "") 214 line = line.replace(':', "") 215 line = line.replace('.', "-dot-") 216 line = line.replace('"', "") 217 line = line.replace('<', "-smaller-") 218 line = line.replace('>', "-bigger-") 219 line = line.replace('_', "-") 220 line = line.replace('(', "") 221 line = line.replace(')', "") 222 line = line.replace('!', "-not-") 223 line = line.replace('=', "-equals-") 224 current_feature = line[1:-1] 225 elif current_feature: 226 extras_req[current_feature].append(line) 227 else: 228 inst_req.append(line) 229 info['Install-requires'] = inst_req 230 info['Extras-require'] = extras_req 231 elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']): 232 info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO')) 233 234 if setup_info: 235 if 'Install-requires' in setup_info: 236 info['Install-requires'] = setup_info['Install-requires'] 237 if 'Extras-require' in setup_info: 238 info['Extras-require'] = setup_info['Extras-require'] 239 else: 240 if setup_info: 241 info = setup_info 242 else: 243 info = self.get_setup_args_info(setupscript) 244 245 # Grab the license value before applying replacements 246 license_str = info.get('License', '').strip() 247 248 self.apply_info_replacements(info) 249 250 if uses_setuptools: 251 classes.append('setuptools3') 252 else: 253 classes.append('distutils3') 254 255 if license_str: 256 for i, line in enumerate(lines_before): 257 if line.startswith('LICENSE = '): 258 lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) 259 break 260 261 if 'Classifier' in info: 262 existing_licenses = info.get('License', '') 263 licenses = [] 264 for classifier in info['Classifier']: 265 if classifier in self.classifier_license_map: 266 license = self.classifier_license_map[classifier] 267 if license == 'Apache' and 'Apache-2.0' in existing_licenses: 268 license = 'Apache-2.0' 269 elif license == 'GPL': 270 if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: 271 license = 'GPL-2.0' 272 elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: 273 license = 'GPL-3.0' 274 elif license == 'LGPL': 275 if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: 276 license = 'LGPL-2.1' 277 elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: 278 license = 'LGPL-2.0' 279 elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: 280 license = 'LGPL-3.0' 281 licenses.append(license) 282 283 if licenses: 284 info['License'] = ' & '.join(licenses) 285 286 # Map PKG-INFO & setup.py fields to bitbake variables 287 for field, values in info.items(): 288 if field in self.excluded_fields: 289 continue 290 291 if field not in self.bbvar_map: 292 continue 293 294 if isinstance(values, str): 295 value = values 296 else: 297 value = ' '.join(str(v) for v in values if v) 298 299 bbvar = self.bbvar_map[field] 300 if bbvar not in extravalues and value: 301 extravalues[bbvar] = value 302 303 mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) 304 305 extras_req = set() 306 if 'Extras-require' in info: 307 extras_req = info['Extras-require'] 308 if extras_req: 309 lines_after.append('# The following configs & dependencies are from setuptools extras_require.') 310 lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') 311 lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') 312 lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') 313 lines_after.append('#') 314 lines_after.append('# Uncomment this line to enable all the optional features.') 315 lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) 316 for feature, feature_reqs in extras_req.items(): 317 unmapped_deps.difference_update(feature_reqs) 318 319 feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) 320 lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) 321 322 inst_reqs = set() 323 if 'Install-requires' in info: 324 if extras_req: 325 lines_after.append('') 326 inst_reqs = info['Install-requires'] 327 if inst_reqs: 328 unmapped_deps.difference_update(inst_reqs) 329 330 inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) 331 lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') 332 lines_after.append('# upstream names may not correspond exactly to bitbake package names.') 333 lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) 334 335 if mapped_deps: 336 name = info.get('Name') 337 if name and name[0] in mapped_deps: 338 # Attempt to avoid self-reference 339 mapped_deps.remove(name[0]) 340 mapped_deps -= set(self.excluded_pkgdeps) 341 if inst_reqs or extras_req: 342 lines_after.append('') 343 lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') 344 lines_after.append('# python sources, and might not be 100% accurate.') 345 lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) 346 347 unmapped_deps -= set(extensions) 348 unmapped_deps -= set(self.assume_provided) 349 if unmapped_deps: 350 if mapped_deps: 351 lines_after.append('') 352 lines_after.append('# WARNING: We were unable to map the following python package/module') 353 lines_after.append('# dependencies to the bitbake packages which include them:') 354 lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) 355 356 handled.append('buildsystem') 357 358 def get_pkginfo(self, pkginfo_fn): 359 msg = email.message_from_file(open(pkginfo_fn, 'r')) 360 msginfo = {} 361 for field in msg.keys(): 362 values = msg.get_all(field) 363 if len(values) == 1: 364 msginfo[field] = values[0] 365 else: 366 msginfo[field] = values 367 return msginfo 368 369 def parse_setup_py(self, setupscript='./setup.py'): 370 with codecs.open(setupscript) as f: 371 info, imported_modules, non_literals, extensions = gather_setup_info(f) 372 373 def _map(key): 374 key = key.replace('_', '-') 375 key = key[0].upper() + key[1:] 376 if key in self.setup_parse_map: 377 key = self.setup_parse_map[key] 378 return key 379 380 # Naive mapping of setup() arguments to PKG-INFO field names 381 for d in [info, non_literals]: 382 for key, value in list(d.items()): 383 if key is None: 384 continue 385 new_key = _map(key) 386 if new_key != key: 387 del d[key] 388 d[new_key] = value 389 390 return info, 'setuptools' in imported_modules, non_literals, extensions 391 392 def get_setup_args_info(self, setupscript='./setup.py'): 393 cmd = ['python3', setupscript] 394 info = {} 395 keys = set(self.bbvar_map.keys()) 396 keys |= set(self.setuparg_list_fields) 397 keys |= set(self.setuparg_multi_line_values) 398 grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values)) 399 for index, keys in grouped_keys: 400 if index == (True, False): 401 # Splitlines output for each arg as a list value 402 for key in keys: 403 arg = self.setuparg_map.get(key, key.lower()) 404 try: 405 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) 406 except (OSError, subprocess.CalledProcessError): 407 pass 408 else: 409 info[key] = [l.rstrip() for l in arg_info.splitlines()] 410 elif index == (False, True): 411 # Entire output for each arg 412 for key in keys: 413 arg = self.setuparg_map.get(key, key.lower()) 414 try: 415 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) 416 except (OSError, subprocess.CalledProcessError): 417 pass 418 else: 419 info[key] = arg_info 420 else: 421 info.update(self.get_setup_byline(list(keys), setupscript)) 422 return info 423 424 def get_setup_byline(self, fields, setupscript='./setup.py'): 425 info = {} 426 427 cmd = ['python3', setupscript] 428 cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields) 429 try: 430 info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines() 431 except (OSError, subprocess.CalledProcessError): 432 pass 433 else: 434 if len(fields) != len(info_lines): 435 logger.error('Mismatch between setup.py output lines and number of fields') 436 sys.exit(1) 437 438 for lineno, line in enumerate(info_lines): 439 line = line.rstrip() 440 info[fields[lineno]] = line 441 return info 442 443 def apply_info_replacements(self, info): 444 for variable, search, replace in self.replacements: 445 if variable not in info: 446 continue 447 448 def replace_value(search, replace, value): 449 if replace is None: 450 if re.search(search, value): 451 return None 452 else: 453 new_value = re.sub(search, replace, value) 454 if value != new_value: 455 return new_value 456 return value 457 458 value = info[variable] 459 if isinstance(value, str): 460 new_value = replace_value(search, replace, value) 461 if new_value is None: 462 del info[variable] 463 elif new_value != value: 464 info[variable] = new_value 465 elif hasattr(value, 'items'): 466 for dkey, dvalue in list(value.items()): 467 new_list = [] 468 for pos, a_value in enumerate(dvalue): 469 new_value = replace_value(search, replace, a_value) 470 if new_value is not None and new_value != value: 471 new_list.append(new_value) 472 473 if value != new_list: 474 value[dkey] = new_list 475 else: 476 new_list = [] 477 for pos, a_value in enumerate(value): 478 new_value = replace_value(search, replace, a_value) 479 if new_value is not None and new_value != value: 480 new_list.append(new_value) 481 482 if value != new_list: 483 info[variable] = new_list 484 485 def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): 486 if 'Package-dir' in setup_info: 487 package_dir = setup_info['Package-dir'] 488 else: 489 package_dir = {} 490 491 dist = setuptools.Distribution() 492 493 class PackageDir(setuptools.command.build_py.build_py): 494 def __init__(self, package_dir): 495 self.package_dir = package_dir 496 self.dist = dist 497 super().__init__(self.dist) 498 499 pd = PackageDir(package_dir) 500 to_scan = [] 501 if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): 502 if 'Py-modules' in setup_info: 503 for module in setup_info['Py-modules']: 504 try: 505 package, module = module.rsplit('.', 1) 506 except ValueError: 507 package, module = '.', module 508 module_path = os.path.join(pd.get_package_dir(package), module + '.py') 509 to_scan.append(module_path) 510 511 if 'Packages' in setup_info: 512 for package in setup_info['Packages']: 513 to_scan.append(pd.get_package_dir(package)) 514 515 if 'Scripts' in setup_info: 516 to_scan.extend(setup_info['Scripts']) 517 else: 518 logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") 519 520 if not to_scan: 521 to_scan = ['.'] 522 523 logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) 524 525 provided_packages = self.parse_pkgdata_for_python_packages() 526 scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) 527 mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() 528 for dep in scanned_deps: 529 mapped = provided_packages.get(dep) 530 if mapped: 531 logger.debug('Mapped %s to %s' % (dep, mapped)) 532 mapped_deps.add(mapped) 533 else: 534 logger.debug('Could not map %s' % dep) 535 unmapped_deps.add(dep) 536 return mapped_deps, unmapped_deps 537 538 def scan_python_dependencies(self, paths): 539 deps = set() 540 try: 541 dep_output = self.run_command(['pythondeps', '-d'] + paths) 542 except (OSError, subprocess.CalledProcessError): 543 pass 544 else: 545 for line in dep_output.splitlines(): 546 line = line.rstrip() 547 dep, filename = line.split('\t', 1) 548 if filename.endswith('/setup.py'): 549 continue 550 deps.add(dep) 551 552 try: 553 provides_output = self.run_command(['pythondeps', '-p'] + paths) 554 except (OSError, subprocess.CalledProcessError): 555 pass 556 else: 557 provides_lines = (l.rstrip() for l in provides_output.splitlines()) 558 provides = set(l for l in provides_lines if l and l != 'setup') 559 deps -= provides 560 561 return deps 562 563 def parse_pkgdata_for_python_packages(self): 564 suffixes = [t[0] for t in imp.get_suffixes()] 565 pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') 566 567 ldata = tinfoil.config_data.createCopy() 568 bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) 569 python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') 570 571 dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') 572 python_dirs = [python_sitedir + os.sep, 573 os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, 574 os.path.dirname(python_sitedir) + os.sep] 575 packages = {} 576 for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): 577 files_info = None 578 with open(pkgdatafile, 'r') as f: 579 for line in f.readlines(): 580 field, value = line.split(': ', 1) 581 if field.startswith('FILES_INFO'): 582 files_info = ast.literal_eval(value) 583 break 584 else: 585 continue 586 587 for fn in files_info: 588 for suffix in suffixes: 589 if fn.endswith(suffix): 590 break 591 else: 592 continue 593 594 if fn.startswith(dynload_dir + os.sep): 595 if '/.debug/' in fn: 596 continue 597 base = os.path.basename(fn) 598 provided = base.split('.', 1)[0] 599 packages[provided] = os.path.basename(pkgdatafile) 600 continue 601 602 for python_dir in python_dirs: 603 if fn.startswith(python_dir): 604 relpath = fn[len(python_dir):] 605 relstart, _, relremaining = relpath.partition(os.sep) 606 if relstart.endswith('.egg'): 607 relpath = relremaining 608 base, _ = os.path.splitext(relpath) 609 610 if '/.debug/' in base: 611 continue 612 if os.path.basename(base) == '__init__': 613 base = os.path.dirname(base) 614 base = base.replace(os.sep + os.sep, os.sep) 615 provided = base.replace(os.sep, '.') 616 packages[provided] = os.path.basename(pkgdatafile) 617 return packages 618 619 @classmethod 620 def run_command(cls, cmd, **popenargs): 621 if 'stderr' not in popenargs: 622 popenargs['stderr'] = subprocess.STDOUT 623 try: 624 return subprocess.check_output(cmd, **popenargs).decode('utf-8') 625 except OSError as exc: 626 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) 627 raise 628 except subprocess.CalledProcessError as exc: 629 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) 630 raise 631 632 633def gather_setup_info(fileobj): 634 parsed = ast.parse(fileobj.read(), fileobj.name) 635 visitor = SetupScriptVisitor() 636 visitor.visit(parsed) 637 638 non_literals, extensions = {}, [] 639 for key, value in list(visitor.keywords.items()): 640 if key == 'ext_modules': 641 if isinstance(value, list): 642 for ext in value: 643 if (isinstance(ext, ast.Call) and 644 isinstance(ext.func, ast.Name) and 645 ext.func.id == 'Extension' and 646 not has_non_literals(ext.args)): 647 extensions.append(ext.args[0]) 648 elif has_non_literals(value): 649 non_literals[key] = value 650 del visitor.keywords[key] 651 652 return visitor.keywords, visitor.imported_modules, non_literals, extensions 653 654 655class SetupScriptVisitor(ast.NodeVisitor): 656 def __init__(self): 657 ast.NodeVisitor.__init__(self) 658 self.keywords = {} 659 self.non_literals = [] 660 self.imported_modules = set() 661 662 def visit_Expr(self, node): 663 if isinstance(node.value, ast.Call) and \ 664 isinstance(node.value.func, ast.Name) and \ 665 node.value.func.id == 'setup': 666 self.visit_setup(node.value) 667 668 def visit_setup(self, node): 669 call = LiteralAstTransform().visit(node) 670 self.keywords = call.keywords 671 for k, v in self.keywords.items(): 672 if has_non_literals(v): 673 self.non_literals.append(k) 674 675 def visit_Import(self, node): 676 for alias in node.names: 677 self.imported_modules.add(alias.name) 678 679 def visit_ImportFrom(self, node): 680 self.imported_modules.add(node.module) 681 682 683class LiteralAstTransform(ast.NodeTransformer): 684 """Simplify the ast through evaluation of literals.""" 685 excluded_fields = ['ctx'] 686 687 def visit(self, node): 688 if not isinstance(node, ast.AST): 689 return node 690 else: 691 return ast.NodeTransformer.visit(self, node) 692 693 def generic_visit(self, node): 694 try: 695 return ast.literal_eval(node) 696 except ValueError: 697 for field, value in ast.iter_fields(node): 698 if field in self.excluded_fields: 699 delattr(node, field) 700 if value is None: 701 continue 702 703 if isinstance(value, list): 704 if field in ('keywords', 'kwargs'): 705 new_value = dict((kw.arg, self.visit(kw.value)) for kw in value) 706 else: 707 new_value = [self.visit(i) for i in value] 708 else: 709 new_value = self.visit(value) 710 setattr(node, field, new_value) 711 return node 712 713 def visit_Name(self, node): 714 if hasattr('__builtins__', node.id): 715 return getattr(__builtins__, node.id) 716 else: 717 return self.generic_visit(node) 718 719 def visit_Tuple(self, node): 720 return tuple(self.visit(v) for v in node.elts) 721 722 def visit_List(self, node): 723 return [self.visit(v) for v in node.elts] 724 725 def visit_Set(self, node): 726 return set(self.visit(v) for v in node.elts) 727 728 def visit_Dict(self, node): 729 keys = (self.visit(k) for k in node.keys) 730 values = (self.visit(v) for v in node.values) 731 return dict(zip(keys, values)) 732 733 734def has_non_literals(value): 735 if isinstance(value, ast.AST): 736 return True 737 elif isinstance(value, str): 738 return False 739 elif hasattr(value, 'values'): 740 return any(has_non_literals(v) for v in value.values()) 741 elif hasattr(value, '__iter__'): 742 return any(has_non_literals(v) for v in value) 743 744 745def register_recipe_handlers(handlers): 746 # We need to make sure this is ahead of the makefile fallback handler 747 handlers.append((PythonRecipeHandler(), 70)) 748