1""" 2mkvenv - QEMU pyvenv bootstrapping utility 3 4usage: mkvenv [-h] command ... 5 6QEMU pyvenv bootstrapping utility 7 8options: 9 -h, --help show this help message and exit 10 11Commands: 12 command Description 13 create create a venv 14 post_init 15 post-venv initialization 16 ensure Ensure that the specified package is installed. 17 18-------------------------------------------------- 19 20usage: mkvenv create [-h] target 21 22positional arguments: 23 target Target directory to install virtual environment into. 24 25options: 26 -h, --help show this help message and exit 27 28-------------------------------------------------- 29 30usage: mkvenv post_init [-h] 31 32options: 33 -h, --help show this help message and exit 34 35-------------------------------------------------- 36 37usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... 38 39positional arguments: 40 dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' 41 42options: 43 -h, --help show this help message and exit 44 --online Install packages from PyPI, if necessary. 45 --dir DIR Path to vendored packages where we may install from. 46 47""" 48 49# Copyright (C) 2022-2023 Red Hat, Inc. 50# 51# Authors: 52# John Snow <jsnow@redhat.com> 53# Paolo Bonzini <pbonzini@redhat.com> 54# 55# This work is licensed under the terms of the GNU GPL, version 2 or 56# later. See the COPYING file in the top-level directory. 57 58import argparse 59from importlib.util import find_spec 60import logging 61import os 62from pathlib import Path 63import re 64import shutil 65import site 66import subprocess 67import sys 68import sysconfig 69from types import SimpleNamespace 70from typing import ( 71 Any, 72 Iterator, 73 Optional, 74 Sequence, 75 Tuple, 76 Union, 77) 78import venv 79 80 81# Try to load distlib, with a fallback to pip's vendored version. 82# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail 83# outside the venv or before a potential call to ensurepip in checkpip(). 84HAVE_DISTLIB = True 85try: 86 import distlib.scripts 87 import distlib.version 88except ImportError: 89 try: 90 # Reach into pip's cookie jar. pylint and flake8 don't understand 91 # that these imports will be used via distlib.xxx. 92 from pip._vendor import distlib 93 import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import 94 import pip._vendor.distlib.version # noqa, pylint: disable=unused-import 95 except ImportError: 96 HAVE_DISTLIB = False 97 98# Do not add any mandatory dependencies from outside the stdlib: 99# This script *must* be usable standalone! 100 101DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] 102logger = logging.getLogger("mkvenv") 103 104 105def inside_a_venv() -> bool: 106 """Returns True if it is executed inside of a virtual environment.""" 107 return sys.prefix != sys.base_prefix 108 109 110class Ouch(RuntimeError): 111 """An Exception class we can't confuse with a builtin.""" 112 113 114class QemuEnvBuilder(venv.EnvBuilder): 115 """ 116 An extension of venv.EnvBuilder for building QEMU's configure-time venv. 117 118 The primary difference is that it emulates a "nested" virtual 119 environment when invoked from inside of an existing virtual 120 environment by including packages from the parent. Also, 121 "ensurepip" is replaced if possible with just recreating pip's 122 console_scripts inside the virtual environment. 123 124 Parameters for base class init: 125 - system_site_packages: bool = False 126 - clear: bool = False 127 - symlinks: bool = False 128 - upgrade: bool = False 129 - with_pip: bool = False 130 - prompt: Optional[str] = None 131 - upgrade_deps: bool = False (Since 3.9) 132 """ 133 134 def __init__(self, *args: Any, **kwargs: Any) -> None: 135 logger.debug("QemuEnvBuilder.__init__(...)") 136 137 # For nested venv emulation: 138 self.use_parent_packages = False 139 if inside_a_venv(): 140 # Include parent packages only if we're in a venv and 141 # system_site_packages was True. 142 self.use_parent_packages = kwargs.pop( 143 "system_site_packages", False 144 ) 145 # Include system_site_packages only when the parent, 146 # The venv we are currently in, also does so. 147 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES 148 149 # ensurepip is slow: venv creation can be very fast for cases where 150 # we allow the use of system_site_packages. Therefore, ensurepip is 151 # replaced with our own script generation once the virtual environment 152 # is setup. 153 self.want_pip = kwargs.get("with_pip", False) 154 if self.want_pip: 155 if ( 156 kwargs.get("system_site_packages", False) 157 and not need_ensurepip() 158 ): 159 kwargs["with_pip"] = False 160 else: 161 check_ensurepip(suggest_remedy=True) 162 163 super().__init__(*args, **kwargs) 164 165 # Make the context available post-creation: 166 self._context: Optional[SimpleNamespace] = None 167 168 def get_parent_libpath(self) -> Optional[str]: 169 """Return the libpath of the parent venv, if applicable.""" 170 if self.use_parent_packages: 171 return sysconfig.get_path("purelib") 172 return None 173 174 @staticmethod 175 def compute_venv_libpath(context: SimpleNamespace) -> str: 176 """ 177 Compatibility wrapper for context.lib_path for Python < 3.12 178 """ 179 # Python 3.12+, not strictly necessary because it's documented 180 # to be the same as 3.10 code below: 181 if sys.version_info >= (3, 12): 182 return context.lib_path 183 184 # Python 3.10+ 185 if "venv" in sysconfig.get_scheme_names(): 186 lib_path = sysconfig.get_path( 187 "purelib", scheme="venv", vars={"base": context.env_dir} 188 ) 189 assert lib_path is not None 190 return lib_path 191 192 # For Python <= 3.9 we need to hardcode this. Fortunately the 193 # code below was the same in Python 3.6-3.10, so there is only 194 # one case. 195 if sys.platform == "win32": 196 return os.path.join(context.env_dir, "Lib", "site-packages") 197 return os.path.join( 198 context.env_dir, 199 "lib", 200 "python%d.%d" % sys.version_info[:2], 201 "site-packages", 202 ) 203 204 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: 205 logger.debug("ensure_directories(env_dir=%s)", env_dir) 206 self._context = super().ensure_directories(env_dir) 207 return self._context 208 209 def create(self, env_dir: DirType) -> None: 210 logger.debug("create(env_dir=%s)", env_dir) 211 super().create(env_dir) 212 assert self._context is not None 213 self.post_post_setup(self._context) 214 215 def post_post_setup(self, context: SimpleNamespace) -> None: 216 """ 217 The final, final hook. Enter the venv and run commands inside of it. 218 """ 219 if self.use_parent_packages: 220 # We're inside of a venv and we want to include the parent 221 # venv's packages. 222 parent_libpath = self.get_parent_libpath() 223 assert parent_libpath is not None 224 logger.debug("parent_libpath: %s", parent_libpath) 225 226 our_libpath = self.compute_venv_libpath(context) 227 logger.debug("our_libpath: %s", our_libpath) 228 229 pth_file = os.path.join(our_libpath, "nested.pth") 230 with open(pth_file, "w", encoding="UTF-8") as file: 231 file.write(parent_libpath + os.linesep) 232 233 if self.want_pip: 234 args = [ 235 context.env_exe, 236 __file__, 237 "post_init", 238 ] 239 subprocess.run(args, check=True) 240 241 def get_value(self, field: str) -> str: 242 """ 243 Get a string value from the context namespace after a call to build. 244 245 For valid field names, see: 246 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories 247 """ 248 ret = getattr(self._context, field) 249 assert isinstance(ret, str) 250 return ret 251 252 253def need_ensurepip() -> bool: 254 """ 255 Tests for the presence of setuptools and pip. 256 257 :return: `True` if we do not detect both packages. 258 """ 259 # Don't try to actually import them, it's fraught with danger: 260 # https://github.com/pypa/setuptools/issues/2993 261 if find_spec("setuptools") and find_spec("pip"): 262 return False 263 return True 264 265 266def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: 267 """ 268 Check that we have ensurepip. 269 270 Raise a fatal exception with a helpful hint if it isn't available. 271 """ 272 if not find_spec("ensurepip"): 273 msg = ( 274 "Python's ensurepip module is not found.\n" 275 "It's normally part of the Python standard library, " 276 "maybe your distribution packages it separately?\n" 277 "(Debian puts ensurepip in its python3-venv package.)\n" 278 ) 279 if suggest_remedy: 280 msg += ( 281 "Either install ensurepip, or alleviate the need for it in the" 282 " first place by installing pip and setuptools for " 283 f"'{sys.executable}'.\n" 284 ) 285 raise Ouch(prefix + msg) 286 287 # ensurepip uses pyexpat, which can also go missing on us: 288 if not find_spec("pyexpat"): 289 msg = ( 290 "Python's pyexpat module is not found.\n" 291 "It's normally part of the Python standard library, " 292 "maybe your distribution packages it separately?\n" 293 "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n" 294 ) 295 if suggest_remedy: 296 msg += ( 297 "Either install pyexpat, or alleviate the need for it in the " 298 "first place by installing pip and setuptools for " 299 f"'{sys.executable}'.\n" 300 ) 301 raise Ouch(prefix + msg) 302 303 304def make_venv( # pylint: disable=too-many-arguments 305 env_dir: Union[str, Path], 306 system_site_packages: bool = False, 307 clear: bool = True, 308 symlinks: Optional[bool] = None, 309 with_pip: bool = True, 310) -> None: 311 """ 312 Create a venv using `QemuEnvBuilder`. 313 314 This is analogous to the `venv.create` module-level convenience 315 function that is part of the Python stdblib, except it uses 316 `QemuEnvBuilder` instead. 317 318 :param env_dir: The directory to create/install to. 319 :param system_site_packages: 320 Allow inheriting packages from the system installation. 321 :param clear: When True, fully remove any prior venv and files. 322 :param symlinks: 323 Whether to use symlinks to the target interpreter or not. If 324 left unspecified, it will use symlinks except on Windows to 325 match behavior with the "venv" CLI tool. 326 :param with_pip: 327 Whether to install "pip" binaries or not. 328 """ 329 logger.debug( 330 "%s: make_venv(env_dir=%s, system_site_packages=%s, " 331 "clear=%s, symlinks=%s, with_pip=%s)", 332 __file__, 333 str(env_dir), 334 system_site_packages, 335 clear, 336 symlinks, 337 with_pip, 338 ) 339 340 if symlinks is None: 341 # Default behavior of standard venv CLI 342 symlinks = os.name != "nt" 343 344 builder = QemuEnvBuilder( 345 system_site_packages=system_site_packages, 346 clear=clear, 347 symlinks=symlinks, 348 with_pip=with_pip, 349 ) 350 351 style = "non-isolated" if builder.system_site_packages else "isolated" 352 nested = "" 353 if builder.use_parent_packages: 354 nested = f"(with packages from '{builder.get_parent_libpath()}') " 355 print( 356 f"mkvenv: Creating {style} virtual environment" 357 f" {nested}at '{str(env_dir)}'", 358 file=sys.stderr, 359 ) 360 361 try: 362 logger.debug("Invoking builder.create()") 363 try: 364 builder.create(str(env_dir)) 365 except SystemExit as exc: 366 # Some versions of the venv module raise SystemExit; *nasty*! 367 # We want the exception that prompted it. It might be a subprocess 368 # error that has output we *really* want to see. 369 logger.debug("Intercepted SystemExit from EnvBuilder.create()") 370 raise exc.__cause__ or exc.__context__ or exc 371 logger.debug("builder.create() finished") 372 except subprocess.CalledProcessError as exc: 373 logger.error("mkvenv subprocess failed:") 374 logger.error("cmd: %s", exc.cmd) 375 logger.error("returncode: %d", exc.returncode) 376 377 def _stringify(data: Union[str, bytes]) -> str: 378 if isinstance(data, bytes): 379 return data.decode() 380 return data 381 382 lines = [] 383 if exc.stdout: 384 lines.append("========== stdout ==========") 385 lines.append(_stringify(exc.stdout)) 386 lines.append("============================") 387 if exc.stderr: 388 lines.append("========== stderr ==========") 389 lines.append(_stringify(exc.stderr)) 390 lines.append("============================") 391 if lines: 392 logger.error(os.linesep.join(lines)) 393 394 raise Ouch("VENV creation subprocess failed.") from exc 395 396 # print the python executable to stdout for configure. 397 print(builder.get_value("env_exe")) 398 399 400def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: 401 # pylint: disable=import-outside-toplevel 402 # pylint: disable=no-name-in-module 403 # pylint: disable=import-error 404 try: 405 # First preference: Python 3.8+ stdlib 406 from importlib.metadata import ( # type: ignore 407 PackageNotFoundError, 408 distribution, 409 ) 410 except ImportError as exc: 411 logger.debug("%s", str(exc)) 412 # Second preference: Commonly available PyPI backport 413 from importlib_metadata import ( # type: ignore 414 PackageNotFoundError, 415 distribution, 416 ) 417 418 def _generator() -> Iterator[str]: 419 for package in packages: 420 try: 421 entry_points = distribution(package).entry_points 422 except PackageNotFoundError: 423 continue 424 425 # The EntryPoints type is only available in 3.10+, 426 # treat this as a vanilla list and filter it ourselves. 427 entry_points = filter( 428 lambda ep: ep.group == "console_scripts", entry_points 429 ) 430 431 for entry_point in entry_points: 432 yield f"{entry_point.name} = {entry_point.value}" 433 434 return _generator() 435 436 437def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: 438 # pylint: disable=import-outside-toplevel 439 # Bundled with setuptools; has a good chance of being available. 440 import pkg_resources 441 442 def _generator() -> Iterator[str]: 443 for package in packages: 444 try: 445 eps = pkg_resources.get_entry_map(package, "console_scripts") 446 except pkg_resources.DistributionNotFound: 447 continue 448 449 for entry_point in eps.values(): 450 yield str(entry_point) 451 452 return _generator() 453 454 455def generate_console_scripts( 456 packages: Sequence[str], 457 python_path: Optional[str] = None, 458 bin_path: Optional[str] = None, 459) -> None: 460 """ 461 Generate script shims for console_script entry points in @packages. 462 """ 463 if python_path is None: 464 python_path = sys.executable 465 if bin_path is None: 466 bin_path = sysconfig.get_path("scripts") 467 assert bin_path is not None 468 469 logger.debug( 470 "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", 471 packages, 472 python_path, 473 bin_path, 474 ) 475 476 if not packages: 477 return 478 479 def _get_entry_points() -> Iterator[str]: 480 """Python 3.7 compatibility shim for iterating entry points.""" 481 # Python 3.8+, or Python 3.7 with importlib_metadata installed. 482 try: 483 return _gen_importlib(packages) 484 except ImportError as exc: 485 logger.debug("%s", str(exc)) 486 487 # Python 3.7 with setuptools installed. 488 try: 489 return _gen_pkg_resources(packages) 490 except ImportError as exc: 491 logger.debug("%s", str(exc)) 492 raise Ouch( 493 "Neither importlib.metadata nor pkg_resources found, " 494 "can't generate console script shims.\n" 495 "Use Python 3.8+, or install importlib-metadata or setuptools." 496 ) from exc 497 498 maker = distlib.scripts.ScriptMaker(None, bin_path) 499 maker.variants = {""} 500 maker.clobber = False 501 502 for entry_point in _get_entry_points(): 503 for filename in maker.make(entry_point): 504 logger.debug("wrote console_script '%s'", filename) 505 506 507def checkpip() -> bool: 508 """ 509 Debian10 has a pip that's broken when used inside of a virtual environment. 510 511 We try to detect and correct that case here. 512 """ 513 try: 514 # pylint: disable=import-outside-toplevel,unused-import,import-error 515 # pylint: disable=redefined-outer-name 516 import pip._internal # type: ignore # noqa: F401 517 518 logger.debug("pip appears to be working correctly.") 519 return False 520 except ModuleNotFoundError as exc: 521 if exc.name == "pip._internal": 522 # Uh, fair enough. They did say "internal". 523 # Let's just assume it's fine. 524 return False 525 logger.warning("pip appears to be malfunctioning: %s", str(exc)) 526 527 check_ensurepip("pip appears to be non-functional, and ") 528 529 logger.debug("Attempting to repair pip ...") 530 subprocess.run( 531 (sys.executable, "-m", "ensurepip"), 532 stdout=subprocess.DEVNULL, 533 check=True, 534 ) 535 logger.debug("Pip is now (hopefully) repaired!") 536 return True 537 538 539def pkgname_from_depspec(dep_spec: str) -> str: 540 """ 541 Parse package name out of a PEP-508 depspec. 542 543 See https://peps.python.org/pep-0508/#names 544 """ 545 match = re.match( 546 r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE 547 ) 548 if not match: 549 raise ValueError( 550 f"dep_spec '{dep_spec}'" 551 " does not appear to contain a valid package name" 552 ) 553 return match.group(0) 554 555 556def _get_path_importlib(package: str) -> Optional[str]: 557 # pylint: disable=import-outside-toplevel 558 # pylint: disable=no-name-in-module 559 # pylint: disable=import-error 560 try: 561 # First preference: Python 3.8+ stdlib 562 from importlib.metadata import ( # type: ignore 563 PackageNotFoundError, 564 distribution, 565 ) 566 except ImportError as exc: 567 logger.debug("%s", str(exc)) 568 # Second preference: Commonly available PyPI backport 569 from importlib_metadata import ( # type: ignore 570 PackageNotFoundError, 571 distribution, 572 ) 573 574 try: 575 return str(distribution(package).locate_file(".")) 576 except PackageNotFoundError: 577 return None 578 579 580def _get_path_pkg_resources(package: str) -> Optional[str]: 581 # pylint: disable=import-outside-toplevel 582 # Bundled with setuptools; has a good chance of being available. 583 import pkg_resources 584 585 try: 586 return str(pkg_resources.get_distribution(package).location) 587 except pkg_resources.DistributionNotFound: 588 return None 589 590 591def _get_path(package: str) -> Optional[str]: 592 try: 593 return _get_path_importlib(package) 594 except ImportError as exc: 595 logger.debug("%s", str(exc)) 596 597 try: 598 return _get_path_pkg_resources(package) 599 except ImportError as exc: 600 logger.debug("%s", str(exc)) 601 raise Ouch( 602 "Neither importlib.metadata nor pkg_resources found. " 603 "Use Python 3.8+, or install importlib-metadata or setuptools." 604 ) from exc 605 606 607def _path_is_prefix(prefix: Optional[str], path: str) -> bool: 608 try: 609 return ( 610 prefix is not None and os.path.commonpath([prefix, path]) == prefix 611 ) 612 except ValueError: 613 return False 614 615 616def _is_system_package(package: str) -> bool: 617 path = _get_path(package) 618 return path is not None and not ( 619 _path_is_prefix(sysconfig.get_path("purelib"), path) 620 or _path_is_prefix(sysconfig.get_path("platlib"), path) 621 ) 622 623 624def _get_version_importlib(package: str) -> Optional[str]: 625 # pylint: disable=import-outside-toplevel 626 # pylint: disable=no-name-in-module 627 # pylint: disable=import-error 628 try: 629 # First preference: Python 3.8+ stdlib 630 from importlib.metadata import ( # type: ignore 631 PackageNotFoundError, 632 distribution, 633 ) 634 except ImportError as exc: 635 logger.debug("%s", str(exc)) 636 # Second preference: Commonly available PyPI backport 637 from importlib_metadata import ( # type: ignore 638 PackageNotFoundError, 639 distribution, 640 ) 641 642 try: 643 return str(distribution(package).version) 644 except PackageNotFoundError: 645 return None 646 647 648def _get_version_pkg_resources(package: str) -> Optional[str]: 649 # pylint: disable=import-outside-toplevel 650 # Bundled with setuptools; has a good chance of being available. 651 import pkg_resources 652 653 try: 654 return str(pkg_resources.get_distribution(package).version) 655 except pkg_resources.DistributionNotFound: 656 return None 657 658 659def _get_version(package: str) -> Optional[str]: 660 try: 661 return _get_version_importlib(package) 662 except ImportError as exc: 663 logger.debug("%s", str(exc)) 664 665 try: 666 return _get_version_pkg_resources(package) 667 except ImportError as exc: 668 logger.debug("%s", str(exc)) 669 raise Ouch( 670 "Neither importlib.metadata nor pkg_resources found. " 671 "Use Python 3.8+, or install importlib-metadata or setuptools." 672 ) from exc 673 674 675def diagnose( 676 dep_spec: str, 677 online: bool, 678 wheels_dir: Optional[Union[str, Path]], 679 prog: Optional[str], 680) -> Tuple[str, bool]: 681 """ 682 Offer a summary to the user as to why a package failed to be installed. 683 684 :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' 685 :param online: Did we allow PyPI access? 686 :param prog: 687 Optionally, a shell program name that can be used as a 688 bellwether to detect if this program is installed elsewhere on 689 the system. This is used to offer advice when a program is 690 detected for a different python version. 691 :param wheels_dir: 692 Optionally, a directory that was searched for vendored packages. 693 """ 694 # pylint: disable=too-many-branches 695 696 # Some errors are not particularly serious 697 bad = False 698 699 pkg_name = pkgname_from_depspec(dep_spec) 700 pkg_version = _get_version(pkg_name) 701 702 lines = [] 703 704 if pkg_version: 705 lines.append( 706 f"Python package '{pkg_name}' version '{pkg_version}' was found," 707 " but isn't suitable." 708 ) 709 else: 710 lines.append( 711 f"Python package '{pkg_name}' was not found nor installed." 712 ) 713 714 if wheels_dir: 715 lines.append( 716 "No suitable version found in, or failed to install from" 717 f" '{wheels_dir}'." 718 ) 719 bad = True 720 721 if online: 722 lines.append("A suitable version could not be obtained from PyPI.") 723 bad = True 724 else: 725 lines.append( 726 "mkvenv was configured to operate offline and did not check PyPI." 727 ) 728 729 if prog and not pkg_version: 730 which = shutil.which(prog) 731 if which: 732 if sys.base_prefix in site.PREFIXES: 733 pypath = Path(sys.executable).resolve() 734 lines.append( 735 f"'{prog}' was detected on your system at '{which}', " 736 f"but the Python package '{pkg_name}' was not found by " 737 f"this Python interpreter ('{pypath}'). " 738 f"Typically this means that '{prog}' has been installed " 739 "against a different Python interpreter on your system." 740 ) 741 else: 742 lines.append( 743 f"'{prog}' was detected on your system at '{which}', " 744 "but the build is using an isolated virtual environment." 745 ) 746 bad = True 747 748 lines = [f" • {line}" for line in lines] 749 if bad: 750 lines.insert(0, f"Could not provide build dependency '{dep_spec}':") 751 else: 752 lines.insert(0, f"'{dep_spec}' not found:") 753 return os.linesep.join(lines), bad 754 755 756def pip_install( 757 args: Sequence[str], 758 online: bool = False, 759 wheels_dir: Optional[Union[str, Path]] = None, 760) -> None: 761 """ 762 Use pip to install a package or package(s) as specified in @args. 763 """ 764 loud = bool( 765 os.environ.get("DEBUG") 766 or os.environ.get("GITLAB_CI") 767 or os.environ.get("V") 768 ) 769 770 full_args = [ 771 sys.executable, 772 "-m", 773 "pip", 774 "install", 775 "--disable-pip-version-check", 776 "-v" if loud else "-q", 777 ] 778 if not online: 779 full_args += ["--no-index"] 780 if wheels_dir: 781 full_args += ["--find-links", f"file://{str(wheels_dir)}"] 782 full_args += list(args) 783 subprocess.run( 784 full_args, 785 check=True, 786 ) 787 788 789def _do_ensure( 790 dep_specs: Sequence[str], 791 online: bool = False, 792 wheels_dir: Optional[Union[str, Path]] = None, 793 prog: Optional[str] = None, 794) -> Optional[Tuple[str, bool]]: 795 """ 796 Use pip to ensure we have the package specified by @dep_specs. 797 798 If the package is already installed, do nothing. If online and 799 wheels_dir are both provided, prefer packages found in wheels_dir 800 first before connecting to PyPI. 801 802 :param dep_specs: 803 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 804 :param online: If True, fall back to PyPI. 805 :param wheels_dir: If specified, search this path for packages. 806 """ 807 absent = [] 808 present = [] 809 for spec in dep_specs: 810 matcher = distlib.version.LegacyMatcher(spec) 811 ver = _get_version(matcher.name) 812 if ( 813 ver is None 814 # Always pass installed package to pip, so that they can be 815 # updated if the requested version changes 816 or not _is_system_package(matcher.name) 817 or not matcher.match(distlib.version.LegacyVersion(ver)) 818 ): 819 absent.append(spec) 820 else: 821 logger.info("found %s %s", matcher.name, ver) 822 present.append(matcher.name) 823 824 if present: 825 generate_console_scripts(present) 826 827 if absent: 828 if online or wheels_dir: 829 # Some packages are missing or aren't a suitable version, 830 # install a suitable (possibly vendored) package. 831 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) 832 try: 833 pip_install(args=absent, online=online, wheels_dir=wheels_dir) 834 return None 835 except subprocess.CalledProcessError: 836 pass 837 838 return diagnose( 839 absent[0], 840 online, 841 wheels_dir, 842 prog if absent[0] == dep_specs[0] else None, 843 ) 844 845 return None 846 847 848def ensure( 849 dep_specs: Sequence[str], 850 online: bool = False, 851 wheels_dir: Optional[Union[str, Path]] = None, 852 prog: Optional[str] = None, 853) -> None: 854 """ 855 Use pip to ensure we have the package specified by @dep_specs. 856 857 If the package is already installed, do nothing. If online and 858 wheels_dir are both provided, prefer packages found in wheels_dir 859 first before connecting to PyPI. 860 861 :param dep_specs: 862 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 863 :param online: If True, fall back to PyPI. 864 :param wheels_dir: If specified, search this path for packages. 865 :param prog: 866 If specified, use this program name for error diagnostics that will 867 be presented to the user. e.g., 'sphinx-build' can be used as a 868 bellwether for the presence of 'sphinx'. 869 """ 870 print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) 871 872 if not HAVE_DISTLIB: 873 raise Ouch("a usable distlib could not be found, please install it") 874 875 result = _do_ensure(dep_specs, online, wheels_dir, prog) 876 if result: 877 # Well, that's not good. 878 if result[1]: 879 raise Ouch(result[0]) 880 raise SystemExit(f"\n{result[0]}\n\n") 881 882 883def post_venv_setup() -> None: 884 """ 885 This is intended to be run *inside the venv* after it is created. 886 """ 887 logger.debug("post_venv_setup()") 888 # Test for a broken pip (Debian 10 or derivative?) and fix it if needed 889 if not checkpip(): 890 # Finally, generate a 'pip' script so the venv is usable in a normal 891 # way from the CLI. This only happens when we inherited pip from a 892 # parent/system-site and haven't run ensurepip in some way. 893 generate_console_scripts(["pip"]) 894 895 896def _add_create_subcommand(subparsers: Any) -> None: 897 subparser = subparsers.add_parser("create", help="create a venv") 898 subparser.add_argument( 899 "target", 900 type=str, 901 action="store", 902 help="Target directory to install virtual environment into.", 903 ) 904 905 906def _add_post_init_subcommand(subparsers: Any) -> None: 907 subparsers.add_parser("post_init", help="post-venv initialization") 908 909 910def _add_ensure_subcommand(subparsers: Any) -> None: 911 subparser = subparsers.add_parser( 912 "ensure", help="Ensure that the specified package is installed." 913 ) 914 subparser.add_argument( 915 "--online", 916 action="store_true", 917 help="Install packages from PyPI, if necessary.", 918 ) 919 subparser.add_argument( 920 "--dir", 921 type=str, 922 action="store", 923 help="Path to vendored packages where we may install from.", 924 ) 925 subparser.add_argument( 926 "--diagnose", 927 type=str, 928 action="store", 929 help=( 930 "Name of a shell utility to use for " 931 "diagnostics if this command fails." 932 ), 933 ) 934 subparser.add_argument( 935 "dep_specs", 936 type=str, 937 action="store", 938 help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", 939 nargs="+", 940 ) 941 942 943def main() -> int: 944 """CLI interface to make_qemu_venv. See module docstring.""" 945 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): 946 # You're welcome. 947 logging.basicConfig(level=logging.DEBUG) 948 else: 949 if os.environ.get("V"): 950 logging.basicConfig(level=logging.INFO) 951 952 parser = argparse.ArgumentParser( 953 prog="mkvenv", 954 description="QEMU pyvenv bootstrapping utility", 955 ) 956 subparsers = parser.add_subparsers( 957 title="Commands", 958 dest="command", 959 required=True, 960 metavar="command", 961 help="Description", 962 ) 963 964 _add_create_subcommand(subparsers) 965 _add_post_init_subcommand(subparsers) 966 _add_ensure_subcommand(subparsers) 967 968 args = parser.parse_args() 969 try: 970 if args.command == "create": 971 make_venv( 972 args.target, 973 system_site_packages=True, 974 clear=True, 975 ) 976 if args.command == "post_init": 977 post_venv_setup() 978 if args.command == "ensure": 979 ensure( 980 dep_specs=args.dep_specs, 981 online=args.online, 982 wheels_dir=args.dir, 983 prog=args.diagnose, 984 ) 985 logger.debug("mkvenv.py %s: exiting", args.command) 986 except Ouch as exc: 987 print("\n*** Ouch! ***\n", file=sys.stderr) 988 print(str(exc), "\n\n", file=sys.stderr) 989 return 1 990 except SystemExit: 991 raise 992 except: # pylint: disable=bare-except 993 logger.exception("mkvenv did not complete successfully:") 994 return 2 995 return 0 996 997 998if __name__ == "__main__": 999 sys.exit(main()) 1000