1""" 2mkvenv - QEMU pyvenv bootstrapping utility 3 4usage: mkvenv [-h] command ... 5 6QEMU pyvenv bootstrapping utility 7 8options: 9 -h, --help show this help message and exit 10 11Commands: 12 command Description 13 create create a venv 14 post_init 15 post-venv initialization 16 ensuregroup 17 Ensure that the specified package group is installed. 18 19-------------------------------------------------- 20 21usage: mkvenv create [-h] target 22 23positional arguments: 24 target Target directory to install virtual environment into. 25 26options: 27 -h, --help show this help message and exit 28 29-------------------------------------------------- 30 31usage: mkvenv post_init [-h] 32 33options: 34 -h, --help show this help message and exit 35 36-------------------------------------------------- 37 38usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... 39 40positional arguments: 41 file pointer to a TOML file 42 group section name in the TOML file 43 44options: 45 -h, --help show this help message and exit 46 --online Install packages from PyPI, if necessary. 47 --dir DIR Path to vendored packages where we may install from. 48 49""" 50 51# Copyright (C) 2022-2023 Red Hat, Inc. 52# 53# Authors: 54# John Snow <jsnow@redhat.com> 55# Paolo Bonzini <pbonzini@redhat.com> 56# 57# This work is licensed under the terms of the GNU GPL, version 2 or 58# later. See the COPYING file in the top-level directory. 59 60import argparse 61from importlib.metadata import ( 62 Distribution, 63 EntryPoint, 64 PackageNotFoundError, 65 distribution, 66 version, 67) 68from importlib.util import find_spec 69import logging 70import os 71from pathlib import Path 72import re 73import shutil 74import site 75import subprocess 76import sys 77import sysconfig 78from types import SimpleNamespace 79from typing import ( 80 Any, 81 Dict, 82 Iterator, 83 Optional, 84 Sequence, 85 Tuple, 86 Union, 87 cast, 88) 89import venv 90 91 92# Try to load distlib, with a fallback to pip's vendored version. 93# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail 94# outside the venv or before a potential call to ensurepip in checkpip(). 95_import_ok = True 96try: 97 import distlib.scripts 98except ImportError: 99 try: 100 # Reach into pip's cookie jar. pylint and flake8 don't understand 101 # that these imports will be used via distlib.xxx. 102 from pip._vendor import distlib 103 import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import 104 except ImportError: 105 _import_ok = False 106 107HAVE_DISTLIB = _import_ok 108 109# pip 25.2 does not vendor distlib.version, but it uses vendored 110# packaging.version 111_import_ok = True 112try: 113 import distlib.version # pylint: disable=ungrouped-imports 114except ImportError: 115 try: 116 # pylint: disable=unused-import,ungrouped-imports 117 import pip._vendor.distlib.version # noqa 118 except ImportError: 119 _import_ok = False 120 121HAVE_DISTLIB_VERSION = _import_ok 122 123_import_ok = True 124try: 125 # Do not bother importing non-vendored packaging, because it is not 126 # in stdlib. 127 from pip._vendor import packaging 128 # pylint: disable=unused-import 129 import pip._vendor.packaging.requirements # noqa 130 import pip._vendor.packaging.version # noqa 131except ImportError: 132 _import_ok = False 133 134HAVE_PACKAGING_VERSION = _import_ok 135 136 137# Try to load tomllib, with a fallback to tomli. 138# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail 139# outside the venv or before a potential call to ensurepip in checkpip(). 140_import_ok = True 141try: 142 import tomllib 143except ImportError: 144 try: 145 import tomli as tomllib 146 except ImportError: 147 _import_ok = False 148 149HAVE_TOMLLIB = _import_ok 150 151# Do not add any mandatory dependencies from outside the stdlib: 152# This script *must* be usable standalone! 153 154DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] 155logger = logging.getLogger("mkvenv") 156 157 158def inside_a_venv() -> bool: 159 """Returns True if it is executed inside of a virtual environment.""" 160 return sys.prefix != sys.base_prefix 161 162 163class Ouch(RuntimeError): 164 """An Exception class we can't confuse with a builtin.""" 165 166 167class Matcher: 168 """Compatibility appliance for version/requirement string parsing.""" 169 def __init__(self, name_and_constraint: str): 170 """Create a matcher from a requirement-like string.""" 171 if HAVE_DISTLIB_VERSION: 172 self._m = distlib.version.LegacyMatcher(name_and_constraint) 173 elif HAVE_PACKAGING_VERSION: 174 self._m = packaging.requirements.Requirement(name_and_constraint) 175 else: 176 raise Ouch("found neither distlib.version nor packaging.version") 177 self.name = self._m.name 178 179 def match(self, version_str: str) -> bool: 180 """Return True if `version` satisfies the stored constraint.""" 181 if HAVE_DISTLIB_VERSION: 182 return cast( 183 bool, 184 self._m.match(distlib.version.LegacyVersion(version_str)) 185 ) 186 187 assert HAVE_PACKAGING_VERSION 188 return cast( 189 bool, 190 self._m.specifier.contains( 191 packaging.version.Version(version_str), prereleases=True 192 ) 193 ) 194 195 def __str__(self) -> str: 196 """String representation delegated to the backend.""" 197 return str(self._m) 198 199 def __repr__(self) -> str: 200 """Stable debug representation delegated to the backend.""" 201 return repr(self._m) 202 203 204class QemuEnvBuilder(venv.EnvBuilder): 205 """ 206 An extension of venv.EnvBuilder for building QEMU's configure-time venv. 207 208 The primary difference is that it emulates a "nested" virtual 209 environment when invoked from inside of an existing virtual 210 environment by including packages from the parent. Also, 211 "ensurepip" is replaced if possible with just recreating pip's 212 console_scripts inside the virtual environment. 213 214 Parameters for base class init: 215 - system_site_packages: bool = False 216 - clear: bool = False 217 - symlinks: bool = False 218 - upgrade: bool = False 219 - with_pip: bool = False 220 - prompt: Optional[str] = None 221 - upgrade_deps: bool = False (Since 3.9) 222 """ 223 224 def __init__(self, *args: Any, **kwargs: Any) -> None: 225 logger.debug("QemuEnvBuilder.__init__(...)") 226 227 # For nested venv emulation: 228 self.use_parent_packages = False 229 if inside_a_venv(): 230 # Include parent packages only if we're in a venv and 231 # system_site_packages was True. 232 self.use_parent_packages = kwargs.pop( 233 "system_site_packages", False 234 ) 235 # Include system_site_packages only when the parent, 236 # The venv we are currently in, also does so. 237 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES 238 239 # ensurepip is slow: venv creation can be very fast for cases where 240 # we allow the use of system_site_packages. Therefore, ensurepip is 241 # replaced with our own script generation once the virtual environment 242 # is setup. 243 self.want_pip = kwargs.get("with_pip", False) 244 if self.want_pip: 245 if ( 246 kwargs.get("system_site_packages", False) 247 and not need_ensurepip() 248 ): 249 kwargs["with_pip"] = False 250 else: 251 check_ensurepip() 252 253 super().__init__(*args, **kwargs) 254 255 # Make the context available post-creation: 256 self._context: Optional[SimpleNamespace] = None 257 258 def get_parent_libpath(self) -> Optional[str]: 259 """Return the libpath of the parent venv, if applicable.""" 260 if self.use_parent_packages: 261 return sysconfig.get_path("purelib") 262 return None 263 264 @staticmethod 265 def compute_venv_libpath(context: SimpleNamespace) -> str: 266 """ 267 Compatibility wrapper for context.lib_path for Python < 3.12 268 """ 269 # Python 3.12+, not strictly necessary because it's documented 270 # to be the same as 3.10 code below: 271 if sys.version_info >= (3, 12): 272 return context.lib_path 273 274 # Python 3.10+ 275 if "venv" in sysconfig.get_scheme_names(): 276 lib_path = sysconfig.get_path( 277 "purelib", scheme="venv", vars={"base": context.env_dir} 278 ) 279 assert lib_path is not None 280 return lib_path 281 282 # For Python <= 3.9 we need to hardcode this. Fortunately the 283 # code below was the same in Python 3.6-3.10, so there is only 284 # one case. 285 if sys.platform == "win32": 286 return os.path.join(context.env_dir, "Lib", "site-packages") 287 return os.path.join( 288 context.env_dir, 289 "lib", 290 "python%d.%d" % sys.version_info[:2], 291 "site-packages", 292 ) 293 294 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: 295 logger.debug("ensure_directories(env_dir=%s)", env_dir) 296 self._context = super().ensure_directories(env_dir) 297 return self._context 298 299 def create(self, env_dir: DirType) -> None: 300 logger.debug("create(env_dir=%s)", env_dir) 301 super().create(env_dir) 302 assert self._context is not None 303 self.post_post_setup(self._context) 304 305 def post_post_setup(self, context: SimpleNamespace) -> None: 306 """ 307 The final, final hook. Enter the venv and run commands inside of it. 308 """ 309 if self.use_parent_packages: 310 # We're inside of a venv and we want to include the parent 311 # venv's packages. 312 parent_libpath = self.get_parent_libpath() 313 assert parent_libpath is not None 314 logger.debug("parent_libpath: %s", parent_libpath) 315 316 our_libpath = self.compute_venv_libpath(context) 317 logger.debug("our_libpath: %s", our_libpath) 318 319 pth_file = os.path.join(our_libpath, "nested.pth") 320 with open(pth_file, "w", encoding="UTF-8") as file: 321 file.write(parent_libpath + os.linesep) 322 323 if self.want_pip: 324 args = [ 325 context.env_exe, 326 __file__, 327 "post_init", 328 ] 329 subprocess.run(args, check=True) 330 331 def get_value(self, field: str) -> str: 332 """ 333 Get a string value from the context namespace after a call to build. 334 335 For valid field names, see: 336 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories 337 """ 338 ret = getattr(self._context, field) 339 assert isinstance(ret, str) 340 return ret 341 342 343def need_ensurepip() -> bool: 344 """ 345 Tests for the presence of setuptools and pip. 346 347 :return: `True` if we do not detect both packages. 348 """ 349 # Don't try to actually import them, it's fraught with danger: 350 # https://github.com/pypa/setuptools/issues/2993 351 if find_spec("setuptools") and find_spec("pip"): 352 return False 353 return True 354 355 356def check_ensurepip() -> None: 357 """ 358 Check that we have ensurepip. 359 360 Raise a fatal exception with a helpful hint if it isn't available. 361 """ 362 if not find_spec("ensurepip"): 363 msg = ( 364 "Python's ensurepip module is not found.\n" 365 "It's normally part of the Python standard library, " 366 "maybe your distribution packages it separately?\n" 367 "Either install ensurepip, or alleviate the need for it in the " 368 "first place by installing pip and setuptools for " 369 f"'{sys.executable}'.\n" 370 "(Hint: Debian puts ensurepip in its python3-venv package.)" 371 ) 372 raise Ouch(msg) 373 374 # ensurepip uses pyexpat, which can also go missing on us: 375 if not find_spec("pyexpat"): 376 msg = ( 377 "Python's pyexpat module is not found.\n" 378 "It's normally part of the Python standard library, " 379 "maybe your distribution packages it separately?\n" 380 "Either install pyexpat, or alleviate the need for it in the " 381 "first place by installing pip and setuptools for " 382 f"'{sys.executable}'.\n\n" 383 "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" 384 ) 385 raise Ouch(msg) 386 387 388def make_venv( # pylint: disable=too-many-arguments 389 env_dir: Union[str, Path], 390 system_site_packages: bool = False, 391 clear: bool = True, 392 symlinks: Optional[bool] = None, 393 with_pip: bool = True, 394) -> None: 395 """ 396 Create a venv using `QemuEnvBuilder`. 397 398 This is analogous to the `venv.create` module-level convenience 399 function that is part of the Python stdblib, except it uses 400 `QemuEnvBuilder` instead. 401 402 :param env_dir: The directory to create/install to. 403 :param system_site_packages: 404 Allow inheriting packages from the system installation. 405 :param clear: When True, fully remove any prior venv and files. 406 :param symlinks: 407 Whether to use symlinks to the target interpreter or not. If 408 left unspecified, it will use symlinks except on Windows to 409 match behavior with the "venv" CLI tool. 410 :param with_pip: 411 Whether to install "pip" binaries or not. 412 """ 413 logger.debug( 414 "%s: make_venv(env_dir=%s, system_site_packages=%s, " 415 "clear=%s, symlinks=%s, with_pip=%s)", 416 __file__, 417 str(env_dir), 418 system_site_packages, 419 clear, 420 symlinks, 421 with_pip, 422 ) 423 424 if symlinks is None: 425 # Default behavior of standard venv CLI 426 symlinks = os.name != "nt" 427 428 builder = QemuEnvBuilder( 429 system_site_packages=system_site_packages, 430 clear=clear, 431 symlinks=symlinks, 432 with_pip=with_pip, 433 ) 434 435 style = "non-isolated" if builder.system_site_packages else "isolated" 436 nested = "" 437 if builder.use_parent_packages: 438 nested = f"(with packages from '{builder.get_parent_libpath()}') " 439 print( 440 f"mkvenv: Creating {style} virtual environment" 441 f" {nested}at '{str(env_dir)}'", 442 file=sys.stderr, 443 ) 444 445 try: 446 logger.debug("Invoking builder.create()") 447 try: 448 builder.create(str(env_dir)) 449 except SystemExit as exc: 450 # pylint 3.3 bug: 451 # pylint: disable=raising-non-exception, raise-missing-from 452 453 # Some versions of the venv module raise SystemExit; *nasty*! 454 # We want the exception that prompted it. It might be a subprocess 455 # error that has output we *really* want to see. 456 logger.debug("Intercepted SystemExit from EnvBuilder.create()") 457 raise exc.__cause__ or exc.__context__ or exc 458 logger.debug("builder.create() finished") 459 except subprocess.CalledProcessError as exc: 460 logger.error("mkvenv subprocess failed:") 461 logger.error("cmd: %s", exc.cmd) 462 logger.error("returncode: %d", exc.returncode) 463 464 def _stringify(data: Union[str, bytes]) -> str: 465 if isinstance(data, bytes): 466 return data.decode() 467 return data 468 469 lines = [] 470 if exc.stdout: 471 lines.append("========== stdout ==========") 472 lines.append(_stringify(exc.stdout)) 473 lines.append("============================") 474 if exc.stderr: 475 lines.append("========== stderr ==========") 476 lines.append(_stringify(exc.stderr)) 477 lines.append("============================") 478 if lines: 479 logger.error(os.linesep.join(lines)) 480 481 raise Ouch("VENV creation subprocess failed.") from exc 482 483 # print the python executable to stdout for configure. 484 print(builder.get_value("env_exe")) 485 486 487def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: 488 489 def _generator() -> Iterator[str]: 490 for package in packages: 491 try: 492 entry_points: Iterator[EntryPoint] = \ 493 iter(distribution(package).entry_points) 494 except PackageNotFoundError: 495 continue 496 497 # The EntryPoints type is only available in 3.10+, 498 # treat this as a vanilla list and filter it ourselves. 499 entry_points = filter( 500 lambda ep: ep.group == "console_scripts", entry_points 501 ) 502 503 for entry_point in entry_points: 504 yield f"{entry_point.name} = {entry_point.value}" 505 506 return _generator() 507 508 509def generate_console_scripts( 510 packages: Sequence[str], 511 python_path: Optional[str] = None, 512 bin_path: Optional[str] = None, 513) -> None: 514 """ 515 Generate script shims for console_script entry points in @packages. 516 """ 517 if python_path is None: 518 python_path = sys.executable 519 if bin_path is None: 520 bin_path = sysconfig.get_path("scripts") 521 assert bin_path is not None 522 523 logger.debug( 524 "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", 525 packages, 526 python_path, 527 bin_path, 528 ) 529 530 if not packages: 531 return 532 533 maker = distlib.scripts.ScriptMaker(None, bin_path) 534 maker.variants = {""} 535 maker.clobber = False 536 537 for entry_point in _get_entry_points(packages): 538 for filename in maker.make(entry_point): 539 logger.debug("wrote console_script '%s'", filename) 540 541 542def pkgname_from_depspec(dep_spec: str) -> str: 543 """ 544 Parse package name out of a PEP-508 depspec. 545 546 See https://peps.python.org/pep-0508/#names 547 """ 548 match = re.match( 549 r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE 550 ) 551 if not match: 552 raise ValueError( 553 f"dep_spec '{dep_spec}'" 554 " does not appear to contain a valid package name" 555 ) 556 return match.group(0) 557 558 559def _path_is_prefix(prefix: Optional[str], path: str) -> bool: 560 try: 561 return ( 562 prefix is not None and os.path.commonpath([prefix, path]) == prefix 563 ) 564 except ValueError: 565 return False 566 567 568def _is_system_package(dist: Distribution) -> bool: 569 path = str(dist.locate_file(".")) 570 return not ( 571 _path_is_prefix(sysconfig.get_path("purelib"), path) 572 or _path_is_prefix(sysconfig.get_path("platlib"), path) 573 ) 574 575 576def diagnose( 577 dep_spec: str, 578 online: bool, 579 wheels_dir: Optional[Union[str, Path]], 580 prog: Optional[str], 581) -> Tuple[str, bool]: 582 """ 583 Offer a summary to the user as to why a package failed to be installed. 584 585 :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' 586 :param online: Did we allow PyPI access? 587 :param prog: 588 Optionally, a shell program name that can be used as a 589 bellwether to detect if this program is installed elsewhere on 590 the system. This is used to offer advice when a program is 591 detected for a different python version. 592 :param wheels_dir: 593 Optionally, a directory that was searched for vendored packages. 594 """ 595 # pylint: disable=too-many-branches 596 597 # Some errors are not particularly serious 598 bad = False 599 600 pkg_name = pkgname_from_depspec(dep_spec) 601 pkg_version: Optional[str] = None 602 try: 603 pkg_version = version(pkg_name) 604 except PackageNotFoundError: 605 pass 606 607 lines = [] 608 609 if pkg_version: 610 lines.append( 611 f"Python package '{pkg_name}' version '{pkg_version}' was found," 612 " but isn't suitable." 613 ) 614 else: 615 lines.append( 616 f"Python package '{pkg_name}' was not found nor installed." 617 ) 618 619 if wheels_dir: 620 lines.append( 621 "No suitable version found in, or failed to install from" 622 f" '{wheels_dir}'." 623 ) 624 bad = True 625 626 if online: 627 lines.append("A suitable version could not be obtained from PyPI.") 628 bad = True 629 else: 630 lines.append( 631 "mkvenv was configured to operate offline and did not check PyPI." 632 ) 633 634 if prog and not pkg_version: 635 which = shutil.which(prog) 636 if which: 637 if sys.base_prefix in site.PREFIXES: 638 pypath = Path(sys.executable).resolve() 639 lines.append( 640 f"'{prog}' was detected on your system at '{which}', " 641 f"but the Python package '{pkg_name}' was not found by " 642 f"this Python interpreter ('{pypath}'). " 643 f"Typically this means that '{prog}' has been installed " 644 "against a different Python interpreter on your system." 645 ) 646 else: 647 lines.append( 648 f"'{prog}' was detected on your system at '{which}', " 649 "but the build is using an isolated virtual environment." 650 ) 651 bad = True 652 653 lines = [f" • {line}" for line in lines] 654 if bad: 655 lines.insert(0, f"Could not provide build dependency '{dep_spec}':") 656 else: 657 lines.insert(0, f"'{dep_spec}' not found:") 658 return os.linesep.join(lines), bad 659 660 661def pip_install( 662 args: Sequence[str], 663 online: bool = False, 664 wheels_dir: Optional[Union[str, Path]] = None, 665) -> None: 666 """ 667 Use pip to install a package or package(s) as specified in @args. 668 """ 669 loud = bool( 670 os.environ.get("DEBUG") 671 or os.environ.get("GITLAB_CI") 672 or os.environ.get("V") 673 ) 674 675 full_args = [ 676 sys.executable, 677 "-m", 678 "pip", 679 "install", 680 "--disable-pip-version-check", 681 "-v" if loud else "-q", 682 ] 683 if not online: 684 full_args += ["--no-index"] 685 if wheels_dir: 686 full_args += ["--find-links", f"file://{str(wheels_dir)}"] 687 full_args += list(args) 688 subprocess.run( 689 full_args, 690 check=True, 691 ) 692 693 694def _make_version_constraint(info: Dict[str, str], install: bool) -> str: 695 """ 696 Construct the version constraint part of a PEP 508 dependency 697 specification (for example '>=0.61.5') from the accepted and 698 installed keys of the provided dictionary. 699 700 :param info: A dictionary corresponding to a TOML key-value list. 701 :param install: True generates install constraints, False generates 702 presence constraints 703 """ 704 if install and "installed" in info: 705 return "==" + info["installed"] 706 707 dep_spec = info.get("accepted", "") 708 dep_spec = dep_spec.strip() 709 # Double check that they didn't just use a version number 710 if dep_spec and dep_spec[0] not in "!~><=(": 711 raise Ouch( 712 "invalid dependency specifier " + dep_spec + " in dependency file" 713 ) 714 715 return dep_spec 716 717 718def _do_ensure( 719 group: Dict[str, Dict[str, str]], 720 online: bool = False, 721 wheels_dir: Optional[Union[str, Path]] = None, 722) -> Optional[Tuple[str, bool]]: 723 """ 724 Use pip to ensure we have the packages specified in @group. 725 726 If the packages are already installed, do nothing. If online and 727 wheels_dir are both provided, prefer packages found in wheels_dir 728 first before connecting to PyPI. 729 730 :param group: A dictionary of dictionaries, corresponding to a 731 section in a pythondeps.toml file. 732 :param online: If True, fall back to PyPI. 733 :param wheels_dir: If specified, search this path for packages. 734 """ 735 absent = [] 736 present = [] 737 canary = None 738 for name, info in group.items(): 739 constraint = _make_version_constraint(info, False) 740 matcher = Matcher(name + constraint) 741 print(f"mkvenv: checking for {matcher}", file=sys.stderr) 742 743 dist: Optional[Distribution] = None 744 try: 745 dist = distribution(matcher.name) 746 except PackageNotFoundError: 747 pass 748 749 if ( 750 dist is None 751 # Always pass installed package to pip, so that they can be 752 # updated if the requested version changes 753 or not _is_system_package(dist) 754 or not matcher.match(dist.version) 755 ): 756 absent.append(name + _make_version_constraint(info, True)) 757 if len(absent) == 1: 758 canary = info.get("canary", None) 759 else: 760 logger.info("found %s %s", name, dist.version) 761 present.append(name) 762 763 if present: 764 generate_console_scripts(present) 765 766 if absent: 767 if online or wheels_dir: 768 # Some packages are missing or aren't a suitable version, 769 # install a suitable (possibly vendored) package. 770 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) 771 try: 772 pip_install(args=absent, online=online, wheels_dir=wheels_dir) 773 return None 774 except subprocess.CalledProcessError: 775 pass 776 777 return diagnose( 778 absent[0], 779 online, 780 wheels_dir, 781 canary, 782 ) 783 784 return None 785 786 787def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: 788 if not HAVE_TOMLLIB: 789 if sys.version_info < (3, 11): 790 raise Ouch("found no usable tomli, please install it") 791 792 raise Ouch( 793 "Python >=3.11 does not have tomllib... what have you done!?" 794 ) 795 796 # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, 797 # Debian bullseye-backports) and v2.0.x 798 with open(file, "r", encoding="ascii") as depfile: 799 contents = depfile.read() 800 return tomllib.loads(contents) # type: ignore 801 802 803def ensure_group( 804 file: str, 805 groups: Sequence[str], 806 online: bool = False, 807 wheels_dir: Optional[Union[str, Path]] = None, 808) -> None: 809 """ 810 Use pip to ensure we have the package specified by @dep_specs. 811 812 If the package is already installed, do nothing. If online and 813 wheels_dir are both provided, prefer packages found in wheels_dir 814 first before connecting to PyPI. 815 816 :param dep_specs: 817 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 818 :param online: If True, fall back to PyPI. 819 :param wheels_dir: If specified, search this path for packages. 820 """ 821 822 if not HAVE_DISTLIB: 823 raise Ouch("found no usable distlib, please install it") 824 825 parsed_deps = _parse_groups(file) 826 827 to_install: Dict[str, Dict[str, str]] = {} 828 for group in groups: 829 try: 830 to_install.update(parsed_deps[group]) 831 except KeyError as exc: 832 raise Ouch(f"group {group} not defined") from exc 833 834 result = _do_ensure(to_install, online, wheels_dir) 835 if result: 836 # Well, that's not good. 837 if result[1]: 838 raise Ouch(result[0]) 839 raise SystemExit(f"\n{result[0]}\n\n") 840 841 842def post_venv_setup() -> None: 843 """ 844 This is intended to be run *inside the venv* after it is created. 845 """ 846 logger.debug("post_venv_setup()") 847 # Generate a 'pip' script so the venv is usable in a normal 848 # way from the CLI. This only happens when we inherited pip from a 849 # parent/system-site and haven't run ensurepip in some way. 850 generate_console_scripts(["pip"]) 851 852 853def _add_create_subcommand(subparsers: Any) -> None: 854 subparser = subparsers.add_parser("create", help="create a venv") 855 subparser.add_argument( 856 "target", 857 type=str, 858 action="store", 859 help="Target directory to install virtual environment into.", 860 ) 861 862 863def _add_post_init_subcommand(subparsers: Any) -> None: 864 subparsers.add_parser("post_init", help="post-venv initialization") 865 866 867def _add_ensuregroup_subcommand(subparsers: Any) -> None: 868 subparser = subparsers.add_parser( 869 "ensuregroup", 870 help="Ensure that the specified package group is installed.", 871 ) 872 subparser.add_argument( 873 "--online", 874 action="store_true", 875 help="Install packages from PyPI, if necessary.", 876 ) 877 subparser.add_argument( 878 "--dir", 879 type=str, 880 action="store", 881 help="Path to vendored packages where we may install from.", 882 ) 883 subparser.add_argument( 884 "file", 885 type=str, 886 action="store", 887 help=("Path to a TOML file describing package groups"), 888 ) 889 subparser.add_argument( 890 "group", 891 type=str, 892 action="store", 893 help="One or more package group names", 894 nargs="+", 895 ) 896 897 898def main() -> int: 899 """CLI interface to make_qemu_venv. See module docstring.""" 900 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): 901 # You're welcome. 902 logging.basicConfig(level=logging.DEBUG) 903 else: 904 if os.environ.get("V"): 905 logging.basicConfig(level=logging.INFO) 906 907 parser = argparse.ArgumentParser( 908 prog="mkvenv", 909 description="QEMU pyvenv bootstrapping utility", 910 ) 911 subparsers = parser.add_subparsers( 912 title="Commands", 913 dest="command", 914 required=True, 915 metavar="command", 916 help="Description", 917 ) 918 919 _add_create_subcommand(subparsers) 920 _add_post_init_subcommand(subparsers) 921 _add_ensuregroup_subcommand(subparsers) 922 923 args = parser.parse_args() 924 try: 925 if args.command == "create": 926 make_venv( 927 args.target, 928 system_site_packages=True, 929 clear=True, 930 ) 931 if args.command == "post_init": 932 post_venv_setup() 933 if args.command == "ensuregroup": 934 ensure_group( 935 file=args.file, 936 groups=args.group, 937 online=args.online, 938 wheels_dir=args.dir, 939 ) 940 logger.debug("mkvenv.py %s: exiting", args.command) 941 except Ouch as exc: 942 print("\n*** Ouch! ***\n", file=sys.stderr) 943 print(str(exc), "\n\n", file=sys.stderr) 944 return 1 945 except SystemExit: 946 raise 947 except: # pylint: disable=bare-except 948 logger.exception("mkvenv did not complete successfully:") 949 return 2 950 return 0 951 952 953if __name__ == "__main__": 954 sys.exit(main()) 955