1""" 2mkvenv - QEMU pyvenv bootstrapping utility 3 4usage: mkvenv [-h] command ... 5 6QEMU pyvenv bootstrapping utility 7 8options: 9 -h, --help show this help message and exit 10 11Commands: 12 command Description 13 create create a venv 14 post_init 15 post-venv initialization 16 ensuregroup 17 Ensure that the specified package group is installed. 18 19-------------------------------------------------- 20 21usage: mkvenv create [-h] target 22 23positional arguments: 24 target Target directory to install virtual environment into. 25 26options: 27 -h, --help show this help message and exit 28 29-------------------------------------------------- 30 31usage: mkvenv post_init [-h] 32 33options: 34 -h, --help show this help message and exit 35 36-------------------------------------------------- 37 38usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... 39 40positional arguments: 41 file pointer to a TOML file 42 group section name in the TOML file 43 44options: 45 -h, --help show this help message and exit 46 --online Install packages from PyPI, if necessary. 47 --dir DIR Path to vendored packages where we may install from. 48 49""" 50 51# Copyright (C) 2022-2023 Red Hat, Inc. 52# 53# Authors: 54# John Snow <jsnow@redhat.com> 55# Paolo Bonzini <pbonzini@redhat.com> 56# 57# This work is licensed under the terms of the GNU GPL, version 2 or 58# later. See the COPYING file in the top-level directory. 59 60import argparse 61from importlib.metadata import ( 62 Distribution, 63 EntryPoint, 64 PackageNotFoundError, 65 distribution, 66 version, 67) 68from importlib.util import find_spec 69import logging 70import os 71from pathlib import Path 72import re 73import shutil 74import site 75import subprocess 76import sys 77import sysconfig 78from types import SimpleNamespace 79from typing import ( 80 Any, 81 Dict, 82 Iterator, 83 Optional, 84 Sequence, 85 Tuple, 86 Union, 87 cast, 88) 89import venv 90 91 92# Try to load distlib, with a fallback to pip's vendored version. 93# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail 94# outside the venv or before a potential call to ensurepip in checkpip(). 95HAVE_DISTLIB = True 96try: 97 import distlib.scripts 98except ImportError: 99 try: 100 # Reach into pip's cookie jar. pylint and flake8 don't understand 101 # that these imports will be used via distlib.xxx. 102 from pip._vendor import distlib 103 import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import 104 except ImportError: 105 HAVE_DISTLIB = False 106 107# pip 25.2 does not vendor distlib.version, but it uses vendored 108# packaging.version 109HAVE_DISTLIB_VERSION = True 110try: 111 import distlib.version # pylint: disable=ungrouped-imports 112except ImportError: 113 try: 114 # pylint: disable=unused-import,ungrouped-imports 115 import pip._vendor.distlib.version # noqa 116 except ImportError: 117 HAVE_DISTLIB_VERSION = False 118 119HAVE_PACKAGING_VERSION = True 120try: 121 # Do not bother importing non-vendored packaging, because it is not 122 # in stdlib. 123 from pip._vendor import packaging 124 # pylint: disable=unused-import 125 import pip._vendor.packaging.requirements # noqa 126 import pip._vendor.packaging.version # noqa 127except ImportError: 128 HAVE_PACKAGING_VERSION = False 129 130 131# Try to load tomllib, with a fallback to tomli. 132# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail 133# outside the venv or before a potential call to ensurepip in checkpip(). 134HAVE_TOMLLIB = True 135try: 136 import tomllib 137except ImportError: 138 try: 139 import tomli as tomllib 140 except ImportError: 141 HAVE_TOMLLIB = False 142 143# Do not add any mandatory dependencies from outside the stdlib: 144# This script *must* be usable standalone! 145 146DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] 147logger = logging.getLogger("mkvenv") 148 149 150def inside_a_venv() -> bool: 151 """Returns True if it is executed inside of a virtual environment.""" 152 return sys.prefix != sys.base_prefix 153 154 155class Ouch(RuntimeError): 156 """An Exception class we can't confuse with a builtin.""" 157 158 159class Matcher: 160 """Compatibility appliance for version/requirement string parsing.""" 161 def __init__(self, name_and_constraint: str): 162 """Create a matcher from a requirement-like string.""" 163 if HAVE_DISTLIB_VERSION: 164 self._m = distlib.version.LegacyMatcher(name_and_constraint) 165 elif HAVE_PACKAGING_VERSION: 166 self._m = packaging.requirements.Requirement(name_and_constraint) 167 else: 168 raise Ouch("found neither distlib.version nor packaging.version") 169 self.name = self._m.name 170 171 def match(self, version_str: str) -> bool: 172 """Return True if `version` satisfies the stored constraint.""" 173 if HAVE_DISTLIB_VERSION: 174 return cast( 175 bool, 176 self._m.match(distlib.version.LegacyVersion(version_str)) 177 ) 178 179 assert HAVE_PACKAGING_VERSION 180 return cast( 181 bool, 182 self._m.specifier.contains( 183 packaging.version.Version(version_str), prereleases=True 184 ) 185 ) 186 187 def __str__(self) -> str: 188 """String representation delegated to the backend.""" 189 return str(self._m) 190 191 def __repr__(self) -> str: 192 """Stable debug representation delegated to the backend.""" 193 return repr(self._m) 194 195 196class QemuEnvBuilder(venv.EnvBuilder): 197 """ 198 An extension of venv.EnvBuilder for building QEMU's configure-time venv. 199 200 The primary difference is that it emulates a "nested" virtual 201 environment when invoked from inside of an existing virtual 202 environment by including packages from the parent. Also, 203 "ensurepip" is replaced if possible with just recreating pip's 204 console_scripts inside the virtual environment. 205 206 Parameters for base class init: 207 - system_site_packages: bool = False 208 - clear: bool = False 209 - symlinks: bool = False 210 - upgrade: bool = False 211 - with_pip: bool = False 212 - prompt: Optional[str] = None 213 - upgrade_deps: bool = False (Since 3.9) 214 """ 215 216 def __init__(self, *args: Any, **kwargs: Any) -> None: 217 logger.debug("QemuEnvBuilder.__init__(...)") 218 219 # For nested venv emulation: 220 self.use_parent_packages = False 221 if inside_a_venv(): 222 # Include parent packages only if we're in a venv and 223 # system_site_packages was True. 224 self.use_parent_packages = kwargs.pop( 225 "system_site_packages", False 226 ) 227 # Include system_site_packages only when the parent, 228 # The venv we are currently in, also does so. 229 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES 230 231 # ensurepip is slow: venv creation can be very fast for cases where 232 # we allow the use of system_site_packages. Therefore, ensurepip is 233 # replaced with our own script generation once the virtual environment 234 # is setup. 235 self.want_pip = kwargs.get("with_pip", False) 236 if self.want_pip: 237 if ( 238 kwargs.get("system_site_packages", False) 239 and not need_ensurepip() 240 ): 241 kwargs["with_pip"] = False 242 else: 243 check_ensurepip() 244 245 super().__init__(*args, **kwargs) 246 247 # Make the context available post-creation: 248 self._context: Optional[SimpleNamespace] = None 249 250 def get_parent_libpath(self) -> Optional[str]: 251 """Return the libpath of the parent venv, if applicable.""" 252 if self.use_parent_packages: 253 return sysconfig.get_path("purelib") 254 return None 255 256 @staticmethod 257 def compute_venv_libpath(context: SimpleNamespace) -> str: 258 """ 259 Compatibility wrapper for context.lib_path for Python < 3.12 260 """ 261 # Python 3.12+, not strictly necessary because it's documented 262 # to be the same as 3.10 code below: 263 if sys.version_info >= (3, 12): 264 return context.lib_path 265 266 # Python 3.10+ 267 if "venv" in sysconfig.get_scheme_names(): 268 lib_path = sysconfig.get_path( 269 "purelib", scheme="venv", vars={"base": context.env_dir} 270 ) 271 assert lib_path is not None 272 return lib_path 273 274 # For Python <= 3.9 we need to hardcode this. Fortunately the 275 # code below was the same in Python 3.6-3.10, so there is only 276 # one case. 277 if sys.platform == "win32": 278 return os.path.join(context.env_dir, "Lib", "site-packages") 279 return os.path.join( 280 context.env_dir, 281 "lib", 282 "python%d.%d" % sys.version_info[:2], 283 "site-packages", 284 ) 285 286 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: 287 logger.debug("ensure_directories(env_dir=%s)", env_dir) 288 self._context = super().ensure_directories(env_dir) 289 return self._context 290 291 def create(self, env_dir: DirType) -> None: 292 logger.debug("create(env_dir=%s)", env_dir) 293 super().create(env_dir) 294 assert self._context is not None 295 self.post_post_setup(self._context) 296 297 def post_post_setup(self, context: SimpleNamespace) -> None: 298 """ 299 The final, final hook. Enter the venv and run commands inside of it. 300 """ 301 if self.use_parent_packages: 302 # We're inside of a venv and we want to include the parent 303 # venv's packages. 304 parent_libpath = self.get_parent_libpath() 305 assert parent_libpath is not None 306 logger.debug("parent_libpath: %s", parent_libpath) 307 308 our_libpath = self.compute_venv_libpath(context) 309 logger.debug("our_libpath: %s", our_libpath) 310 311 pth_file = os.path.join(our_libpath, "nested.pth") 312 with open(pth_file, "w", encoding="UTF-8") as file: 313 file.write(parent_libpath + os.linesep) 314 315 if self.want_pip: 316 args = [ 317 context.env_exe, 318 __file__, 319 "post_init", 320 ] 321 subprocess.run(args, check=True) 322 323 def get_value(self, field: str) -> str: 324 """ 325 Get a string value from the context namespace after a call to build. 326 327 For valid field names, see: 328 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories 329 """ 330 ret = getattr(self._context, field) 331 assert isinstance(ret, str) 332 return ret 333 334 335def need_ensurepip() -> bool: 336 """ 337 Tests for the presence of setuptools and pip. 338 339 :return: `True` if we do not detect both packages. 340 """ 341 # Don't try to actually import them, it's fraught with danger: 342 # https://github.com/pypa/setuptools/issues/2993 343 if find_spec("setuptools") and find_spec("pip"): 344 return False 345 return True 346 347 348def check_ensurepip() -> None: 349 """ 350 Check that we have ensurepip. 351 352 Raise a fatal exception with a helpful hint if it isn't available. 353 """ 354 if not find_spec("ensurepip"): 355 msg = ( 356 "Python's ensurepip module is not found.\n" 357 "It's normally part of the Python standard library, " 358 "maybe your distribution packages it separately?\n" 359 "Either install ensurepip, or alleviate the need for it in the " 360 "first place by installing pip and setuptools for " 361 f"'{sys.executable}'.\n" 362 "(Hint: Debian puts ensurepip in its python3-venv package.)" 363 ) 364 raise Ouch(msg) 365 366 # ensurepip uses pyexpat, which can also go missing on us: 367 if not find_spec("pyexpat"): 368 msg = ( 369 "Python's pyexpat module is not found.\n" 370 "It's normally part of the Python standard library, " 371 "maybe your distribution packages it separately?\n" 372 "Either install pyexpat, or alleviate the need for it in the " 373 "first place by installing pip and setuptools for " 374 f"'{sys.executable}'.\n\n" 375 "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" 376 ) 377 raise Ouch(msg) 378 379 380def make_venv( # pylint: disable=too-many-arguments 381 env_dir: Union[str, Path], 382 system_site_packages: bool = False, 383 clear: bool = True, 384 symlinks: Optional[bool] = None, 385 with_pip: bool = True, 386) -> None: 387 """ 388 Create a venv using `QemuEnvBuilder`. 389 390 This is analogous to the `venv.create` module-level convenience 391 function that is part of the Python stdblib, except it uses 392 `QemuEnvBuilder` instead. 393 394 :param env_dir: The directory to create/install to. 395 :param system_site_packages: 396 Allow inheriting packages from the system installation. 397 :param clear: When True, fully remove any prior venv and files. 398 :param symlinks: 399 Whether to use symlinks to the target interpreter or not. If 400 left unspecified, it will use symlinks except on Windows to 401 match behavior with the "venv" CLI tool. 402 :param with_pip: 403 Whether to install "pip" binaries or not. 404 """ 405 logger.debug( 406 "%s: make_venv(env_dir=%s, system_site_packages=%s, " 407 "clear=%s, symlinks=%s, with_pip=%s)", 408 __file__, 409 str(env_dir), 410 system_site_packages, 411 clear, 412 symlinks, 413 with_pip, 414 ) 415 416 if symlinks is None: 417 # Default behavior of standard venv CLI 418 symlinks = os.name != "nt" 419 420 builder = QemuEnvBuilder( 421 system_site_packages=system_site_packages, 422 clear=clear, 423 symlinks=symlinks, 424 with_pip=with_pip, 425 ) 426 427 style = "non-isolated" if builder.system_site_packages else "isolated" 428 nested = "" 429 if builder.use_parent_packages: 430 nested = f"(with packages from '{builder.get_parent_libpath()}') " 431 print( 432 f"mkvenv: Creating {style} virtual environment" 433 f" {nested}at '{str(env_dir)}'", 434 file=sys.stderr, 435 ) 436 437 try: 438 logger.debug("Invoking builder.create()") 439 try: 440 builder.create(str(env_dir)) 441 except SystemExit as exc: 442 # pylint 3.3 bug: 443 # pylint: disable=raising-non-exception, raise-missing-from 444 445 # Some versions of the venv module raise SystemExit; *nasty*! 446 # We want the exception that prompted it. It might be a subprocess 447 # error that has output we *really* want to see. 448 logger.debug("Intercepted SystemExit from EnvBuilder.create()") 449 raise exc.__cause__ or exc.__context__ or exc 450 logger.debug("builder.create() finished") 451 except subprocess.CalledProcessError as exc: 452 logger.error("mkvenv subprocess failed:") 453 logger.error("cmd: %s", exc.cmd) 454 logger.error("returncode: %d", exc.returncode) 455 456 def _stringify(data: Union[str, bytes]) -> str: 457 if isinstance(data, bytes): 458 return data.decode() 459 return data 460 461 lines = [] 462 if exc.stdout: 463 lines.append("========== stdout ==========") 464 lines.append(_stringify(exc.stdout)) 465 lines.append("============================") 466 if exc.stderr: 467 lines.append("========== stderr ==========") 468 lines.append(_stringify(exc.stderr)) 469 lines.append("============================") 470 if lines: 471 logger.error(os.linesep.join(lines)) 472 473 raise Ouch("VENV creation subprocess failed.") from exc 474 475 # print the python executable to stdout for configure. 476 print(builder.get_value("env_exe")) 477 478 479def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: 480 481 def _generator() -> Iterator[str]: 482 for package in packages: 483 try: 484 entry_points: Iterator[EntryPoint] = \ 485 iter(distribution(package).entry_points) 486 except PackageNotFoundError: 487 continue 488 489 # The EntryPoints type is only available in 3.10+, 490 # treat this as a vanilla list and filter it ourselves. 491 entry_points = filter( 492 lambda ep: ep.group == "console_scripts", entry_points 493 ) 494 495 for entry_point in entry_points: 496 yield f"{entry_point.name} = {entry_point.value}" 497 498 return _generator() 499 500 501def generate_console_scripts( 502 packages: Sequence[str], 503 python_path: Optional[str] = None, 504 bin_path: Optional[str] = None, 505) -> None: 506 """ 507 Generate script shims for console_script entry points in @packages. 508 """ 509 if python_path is None: 510 python_path = sys.executable 511 if bin_path is None: 512 bin_path = sysconfig.get_path("scripts") 513 assert bin_path is not None 514 515 logger.debug( 516 "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", 517 packages, 518 python_path, 519 bin_path, 520 ) 521 522 if not packages: 523 return 524 525 maker = distlib.scripts.ScriptMaker(None, bin_path) 526 maker.variants = {""} 527 maker.clobber = False 528 529 for entry_point in _get_entry_points(packages): 530 for filename in maker.make(entry_point): 531 logger.debug("wrote console_script '%s'", filename) 532 533 534def pkgname_from_depspec(dep_spec: str) -> str: 535 """ 536 Parse package name out of a PEP-508 depspec. 537 538 See https://peps.python.org/pep-0508/#names 539 """ 540 match = re.match( 541 r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE 542 ) 543 if not match: 544 raise ValueError( 545 f"dep_spec '{dep_spec}'" 546 " does not appear to contain a valid package name" 547 ) 548 return match.group(0) 549 550 551def _path_is_prefix(prefix: Optional[str], path: str) -> bool: 552 try: 553 return ( 554 prefix is not None and os.path.commonpath([prefix, path]) == prefix 555 ) 556 except ValueError: 557 return False 558 559 560def _is_system_package(dist: Distribution) -> bool: 561 path = str(dist.locate_file(".")) 562 return not ( 563 _path_is_prefix(sysconfig.get_path("purelib"), path) 564 or _path_is_prefix(sysconfig.get_path("platlib"), path) 565 ) 566 567 568def diagnose( 569 dep_spec: str, 570 online: bool, 571 wheels_dir: Optional[Union[str, Path]], 572 prog: Optional[str], 573) -> Tuple[str, bool]: 574 """ 575 Offer a summary to the user as to why a package failed to be installed. 576 577 :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' 578 :param online: Did we allow PyPI access? 579 :param prog: 580 Optionally, a shell program name that can be used as a 581 bellwether to detect if this program is installed elsewhere on 582 the system. This is used to offer advice when a program is 583 detected for a different python version. 584 :param wheels_dir: 585 Optionally, a directory that was searched for vendored packages. 586 """ 587 # pylint: disable=too-many-branches 588 589 # Some errors are not particularly serious 590 bad = False 591 592 pkg_name = pkgname_from_depspec(dep_spec) 593 pkg_version: Optional[str] = None 594 try: 595 pkg_version = version(pkg_name) 596 except PackageNotFoundError: 597 pass 598 599 lines = [] 600 601 if pkg_version: 602 lines.append( 603 f"Python package '{pkg_name}' version '{pkg_version}' was found," 604 " but isn't suitable." 605 ) 606 else: 607 lines.append( 608 f"Python package '{pkg_name}' was not found nor installed." 609 ) 610 611 if wheels_dir: 612 lines.append( 613 "No suitable version found in, or failed to install from" 614 f" '{wheels_dir}'." 615 ) 616 bad = True 617 618 if online: 619 lines.append("A suitable version could not be obtained from PyPI.") 620 bad = True 621 else: 622 lines.append( 623 "mkvenv was configured to operate offline and did not check PyPI." 624 ) 625 626 if prog and not pkg_version: 627 which = shutil.which(prog) 628 if which: 629 if sys.base_prefix in site.PREFIXES: 630 pypath = Path(sys.executable).resolve() 631 lines.append( 632 f"'{prog}' was detected on your system at '{which}', " 633 f"but the Python package '{pkg_name}' was not found by " 634 f"this Python interpreter ('{pypath}'). " 635 f"Typically this means that '{prog}' has been installed " 636 "against a different Python interpreter on your system." 637 ) 638 else: 639 lines.append( 640 f"'{prog}' was detected on your system at '{which}', " 641 "but the build is using an isolated virtual environment." 642 ) 643 bad = True 644 645 lines = [f" • {line}" for line in lines] 646 if bad: 647 lines.insert(0, f"Could not provide build dependency '{dep_spec}':") 648 else: 649 lines.insert(0, f"'{dep_spec}' not found:") 650 return os.linesep.join(lines), bad 651 652 653def pip_install( 654 args: Sequence[str], 655 online: bool = False, 656 wheels_dir: Optional[Union[str, Path]] = None, 657) -> None: 658 """ 659 Use pip to install a package or package(s) as specified in @args. 660 """ 661 loud = bool( 662 os.environ.get("DEBUG") 663 or os.environ.get("GITLAB_CI") 664 or os.environ.get("V") 665 ) 666 667 full_args = [ 668 sys.executable, 669 "-m", 670 "pip", 671 "install", 672 "--disable-pip-version-check", 673 "-v" if loud else "-q", 674 ] 675 if not online: 676 full_args += ["--no-index"] 677 if wheels_dir: 678 full_args += ["--find-links", f"file://{str(wheels_dir)}"] 679 full_args += list(args) 680 subprocess.run( 681 full_args, 682 check=True, 683 ) 684 685 686def _make_version_constraint(info: Dict[str, str], install: bool) -> str: 687 """ 688 Construct the version constraint part of a PEP 508 dependency 689 specification (for example '>=0.61.5') from the accepted and 690 installed keys of the provided dictionary. 691 692 :param info: A dictionary corresponding to a TOML key-value list. 693 :param install: True generates install constraints, False generates 694 presence constraints 695 """ 696 if install and "installed" in info: 697 return "==" + info["installed"] 698 699 dep_spec = info.get("accepted", "") 700 dep_spec = dep_spec.strip() 701 # Double check that they didn't just use a version number 702 if dep_spec and dep_spec[0] not in "!~><=(": 703 raise Ouch( 704 "invalid dependency specifier " + dep_spec + " in dependency file" 705 ) 706 707 return dep_spec 708 709 710def _do_ensure( 711 group: Dict[str, Dict[str, str]], 712 online: bool = False, 713 wheels_dir: Optional[Union[str, Path]] = None, 714) -> Optional[Tuple[str, bool]]: 715 """ 716 Use pip to ensure we have the packages specified in @group. 717 718 If the packages are already installed, do nothing. If online and 719 wheels_dir are both provided, prefer packages found in wheels_dir 720 first before connecting to PyPI. 721 722 :param group: A dictionary of dictionaries, corresponding to a 723 section in a pythondeps.toml file. 724 :param online: If True, fall back to PyPI. 725 :param wheels_dir: If specified, search this path for packages. 726 """ 727 absent = [] 728 present = [] 729 canary = None 730 for name, info in group.items(): 731 constraint = _make_version_constraint(info, False) 732 matcher = Matcher(name + constraint) 733 print(f"mkvenv: checking for {matcher}", file=sys.stderr) 734 735 dist: Optional[Distribution] = None 736 try: 737 dist = distribution(matcher.name) 738 except PackageNotFoundError: 739 pass 740 741 if ( 742 dist is None 743 # Always pass installed package to pip, so that they can be 744 # updated if the requested version changes 745 or not _is_system_package(dist) 746 or not matcher.match(dist.version) 747 ): 748 absent.append(name + _make_version_constraint(info, True)) 749 if len(absent) == 1: 750 canary = info.get("canary", None) 751 else: 752 logger.info("found %s %s", name, dist.version) 753 present.append(name) 754 755 if present: 756 generate_console_scripts(present) 757 758 if absent: 759 if online or wheels_dir: 760 # Some packages are missing or aren't a suitable version, 761 # install a suitable (possibly vendored) package. 762 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) 763 try: 764 pip_install(args=absent, online=online, wheels_dir=wheels_dir) 765 return None 766 except subprocess.CalledProcessError: 767 pass 768 769 return diagnose( 770 absent[0], 771 online, 772 wheels_dir, 773 canary, 774 ) 775 776 return None 777 778 779def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: 780 if not HAVE_TOMLLIB: 781 if sys.version_info < (3, 11): 782 raise Ouch("found no usable tomli, please install it") 783 784 raise Ouch( 785 "Python >=3.11 does not have tomllib... what have you done!?" 786 ) 787 788 # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, 789 # Debian bullseye-backports) and v2.0.x 790 with open(file, "r", encoding="ascii") as depfile: 791 contents = depfile.read() 792 return tomllib.loads(contents) # type: ignore 793 794 795def ensure_group( 796 file: str, 797 groups: Sequence[str], 798 online: bool = False, 799 wheels_dir: Optional[Union[str, Path]] = None, 800) -> None: 801 """ 802 Use pip to ensure we have the package specified by @dep_specs. 803 804 If the package is already installed, do nothing. If online and 805 wheels_dir are both provided, prefer packages found in wheels_dir 806 first before connecting to PyPI. 807 808 :param dep_specs: 809 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 810 :param online: If True, fall back to PyPI. 811 :param wheels_dir: If specified, search this path for packages. 812 """ 813 814 if not HAVE_DISTLIB: 815 raise Ouch("found no usable distlib, please install it") 816 817 parsed_deps = _parse_groups(file) 818 819 to_install: Dict[str, Dict[str, str]] = {} 820 for group in groups: 821 try: 822 to_install.update(parsed_deps[group]) 823 except KeyError as exc: 824 raise Ouch(f"group {group} not defined") from exc 825 826 result = _do_ensure(to_install, online, wheels_dir) 827 if result: 828 # Well, that's not good. 829 if result[1]: 830 raise Ouch(result[0]) 831 raise SystemExit(f"\n{result[0]}\n\n") 832 833 834def post_venv_setup() -> None: 835 """ 836 This is intended to be run *inside the venv* after it is created. 837 """ 838 logger.debug("post_venv_setup()") 839 # Generate a 'pip' script so the venv is usable in a normal 840 # way from the CLI. This only happens when we inherited pip from a 841 # parent/system-site and haven't run ensurepip in some way. 842 generate_console_scripts(["pip"]) 843 844 845def _add_create_subcommand(subparsers: Any) -> None: 846 subparser = subparsers.add_parser("create", help="create a venv") 847 subparser.add_argument( 848 "target", 849 type=str, 850 action="store", 851 help="Target directory to install virtual environment into.", 852 ) 853 854 855def _add_post_init_subcommand(subparsers: Any) -> None: 856 subparsers.add_parser("post_init", help="post-venv initialization") 857 858 859def _add_ensuregroup_subcommand(subparsers: Any) -> None: 860 subparser = subparsers.add_parser( 861 "ensuregroup", 862 help="Ensure that the specified package group is installed.", 863 ) 864 subparser.add_argument( 865 "--online", 866 action="store_true", 867 help="Install packages from PyPI, if necessary.", 868 ) 869 subparser.add_argument( 870 "--dir", 871 type=str, 872 action="store", 873 help="Path to vendored packages where we may install from.", 874 ) 875 subparser.add_argument( 876 "file", 877 type=str, 878 action="store", 879 help=("Path to a TOML file describing package groups"), 880 ) 881 subparser.add_argument( 882 "group", 883 type=str, 884 action="store", 885 help="One or more package group names", 886 nargs="+", 887 ) 888 889 890def main() -> int: 891 """CLI interface to make_qemu_venv. See module docstring.""" 892 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): 893 # You're welcome. 894 logging.basicConfig(level=logging.DEBUG) 895 else: 896 if os.environ.get("V"): 897 logging.basicConfig(level=logging.INFO) 898 899 parser = argparse.ArgumentParser( 900 prog="mkvenv", 901 description="QEMU pyvenv bootstrapping utility", 902 ) 903 subparsers = parser.add_subparsers( 904 title="Commands", 905 dest="command", 906 required=True, 907 metavar="command", 908 help="Description", 909 ) 910 911 _add_create_subcommand(subparsers) 912 _add_post_init_subcommand(subparsers) 913 _add_ensuregroup_subcommand(subparsers) 914 915 args = parser.parse_args() 916 try: 917 if args.command == "create": 918 make_venv( 919 args.target, 920 system_site_packages=True, 921 clear=True, 922 ) 923 if args.command == "post_init": 924 post_venv_setup() 925 if args.command == "ensuregroup": 926 ensure_group( 927 file=args.file, 928 groups=args.group, 929 online=args.online, 930 wheels_dir=args.dir, 931 ) 932 logger.debug("mkvenv.py %s: exiting", args.command) 933 except Ouch as exc: 934 print("\n*** Ouch! ***\n", file=sys.stderr) 935 print(str(exc), "\n\n", file=sys.stderr) 936 return 1 937 except SystemExit: 938 raise 939 except: # pylint: disable=bare-except 940 logger.exception("mkvenv did not complete successfully:") 941 return 2 942 return 0 943 944 945if __name__ == "__main__": 946 sys.exit(main()) 947