1""" 2mkvenv - QEMU pyvenv bootstrapping utility 3 4usage: mkvenv [-h] command ... 5 6QEMU pyvenv bootstrapping utility 7 8options: 9 -h, --help show this help message and exit 10 11Commands: 12 command Description 13 create create a venv 14 post_init 15 post-venv initialization 16 ensuregroup 17 Ensure that the specified package group is installed. 18 19-------------------------------------------------- 20 21usage: mkvenv create [-h] target 22 23positional arguments: 24 target Target directory to install virtual environment into. 25 26options: 27 -h, --help show this help message and exit 28 29-------------------------------------------------- 30 31usage: mkvenv post_init [-h] 32 33options: 34 -h, --help show this help message and exit 35 36-------------------------------------------------- 37 38usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... 39 40positional arguments: 41 file pointer to a TOML file 42 group section name in the TOML file 43 44options: 45 -h, --help show this help message and exit 46 --online Install packages from PyPI, if necessary. 47 --dir DIR Path to vendored packages where we may install from. 48 49""" 50 51# Copyright (C) 2022-2023 Red Hat, Inc. 52# 53# Authors: 54# John Snow <jsnow@redhat.com> 55# Paolo Bonzini <pbonzini@redhat.com> 56# 57# This work is licensed under the terms of the GNU GPL, version 2 or 58# later. See the COPYING file in the top-level directory. 59 60import argparse 61from importlib.metadata import ( 62 Distribution, 63 EntryPoint, 64 PackageNotFoundError, 65 distribution, 66 version, 67) 68from importlib.util import find_spec 69import logging 70import os 71from pathlib import Path 72import re 73import shutil 74import site 75import subprocess 76import sys 77import sysconfig 78from types import SimpleNamespace 79from typing import ( 80 Any, 81 Dict, 82 Iterator, 83 Optional, 84 Sequence, 85 Tuple, 86 Union, 87 cast, 88) 89import venv 90 91 92# Try to load distlib, with a fallback to pip's vendored version. 93# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail 94# outside the venv or before a potential call to ensurepip in checkpip(). 95HAVE_DISTLIB = True 96try: 97 import distlib.scripts 98except ImportError: 99 try: 100 # Reach into pip's cookie jar. pylint and flake8 don't understand 101 # that these imports will be used via distlib.xxx. 102 from pip._vendor import distlib 103 import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import 104 except ImportError: 105 HAVE_DISTLIB = False 106 107# pip 25.2 does not vendor distlib.version, but it uses vendored 108# packaging.version 109HAVE_DISTLIB_VERSION = True 110try: 111 import distlib.version # pylint: disable=ungrouped-imports 112except ImportError: 113 try: 114 # pylint: disable=unused-import,ungrouped-imports 115 import pip._vendor.distlib.version # noqa 116 except ImportError: 117 HAVE_DISTLIB_VERSION = False 118 119HAVE_PACKAGING_VERSION = True 120try: 121 # Do not bother importing non-vendored packaging, because it is not 122 # in stdlib. 123 from pip._vendor import packaging 124 # pylint: disable=unused-import 125 import pip._vendor.packaging.requirements # noqa 126 import pip._vendor.packaging.version # noqa 127except ImportError: 128 HAVE_PACKAGING_VERSION = False 129 130 131# Try to load tomllib, with a fallback to tomli. 132# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail 133# outside the venv or before a potential call to ensurepip in checkpip(). 134HAVE_TOMLLIB = True 135try: 136 import tomllib 137except ImportError: 138 try: 139 import tomli as tomllib 140 except ImportError: 141 HAVE_TOMLLIB = False 142 143# Do not add any mandatory dependencies from outside the stdlib: 144# This script *must* be usable standalone! 145 146DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] 147logger = logging.getLogger("mkvenv") 148 149 150def inside_a_venv() -> bool: 151 """Returns True if it is executed inside of a virtual environment.""" 152 return sys.prefix != sys.base_prefix 153 154 155class Ouch(RuntimeError): 156 """An Exception class we can't confuse with a builtin.""" 157 158 159class Matcher: 160 """Compatibility appliance for version/requirement string parsing.""" 161 def __init__(self, name_and_constraint: str): 162 """Create a matcher from a requirement-like string.""" 163 if HAVE_DISTLIB_VERSION: 164 self._m = distlib.version.LegacyMatcher(name_and_constraint) 165 elif HAVE_PACKAGING_VERSION: 166 self._m = packaging.requirements.Requirement(name_and_constraint) 167 else: 168 raise Ouch("found neither distlib.version nor packaging.version") 169 self.name = self._m.name 170 171 def match(self, version_str: str) -> bool: 172 """Return True if `version` satisfies the stored constraint.""" 173 if HAVE_DISTLIB_VERSION: 174 return cast( 175 bool, 176 self._m.match(distlib.version.LegacyVersion(version_str)) 177 ) 178 179 assert HAVE_PACKAGING_VERSION 180 return cast( 181 bool, 182 self._m.specifier.contains( 183 packaging.version.Version(version_str), prereleases=True 184 ) 185 ) 186 187 def __repr__(self) -> str: 188 """Stable debug representation delegated to the backend.""" 189 return repr(self._m) 190 191 192class QemuEnvBuilder(venv.EnvBuilder): 193 """ 194 An extension of venv.EnvBuilder for building QEMU's configure-time venv. 195 196 The primary difference is that it emulates a "nested" virtual 197 environment when invoked from inside of an existing virtual 198 environment by including packages from the parent. Also, 199 "ensurepip" is replaced if possible with just recreating pip's 200 console_scripts inside the virtual environment. 201 202 Parameters for base class init: 203 - system_site_packages: bool = False 204 - clear: bool = False 205 - symlinks: bool = False 206 - upgrade: bool = False 207 - with_pip: bool = False 208 - prompt: Optional[str] = None 209 - upgrade_deps: bool = False (Since 3.9) 210 """ 211 212 def __init__(self, *args: Any, **kwargs: Any) -> None: 213 logger.debug("QemuEnvBuilder.__init__(...)") 214 215 # For nested venv emulation: 216 self.use_parent_packages = False 217 if inside_a_venv(): 218 # Include parent packages only if we're in a venv and 219 # system_site_packages was True. 220 self.use_parent_packages = kwargs.pop( 221 "system_site_packages", False 222 ) 223 # Include system_site_packages only when the parent, 224 # The venv we are currently in, also does so. 225 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES 226 227 # ensurepip is slow: venv creation can be very fast for cases where 228 # we allow the use of system_site_packages. Therefore, ensurepip is 229 # replaced with our own script generation once the virtual environment 230 # is setup. 231 self.want_pip = kwargs.get("with_pip", False) 232 if self.want_pip: 233 if ( 234 kwargs.get("system_site_packages", False) 235 and not need_ensurepip() 236 ): 237 kwargs["with_pip"] = False 238 else: 239 check_ensurepip() 240 241 super().__init__(*args, **kwargs) 242 243 # Make the context available post-creation: 244 self._context: Optional[SimpleNamespace] = None 245 246 def get_parent_libpath(self) -> Optional[str]: 247 """Return the libpath of the parent venv, if applicable.""" 248 if self.use_parent_packages: 249 return sysconfig.get_path("purelib") 250 return None 251 252 @staticmethod 253 def compute_venv_libpath(context: SimpleNamespace) -> str: 254 """ 255 Compatibility wrapper for context.lib_path for Python < 3.12 256 """ 257 # Python 3.12+, not strictly necessary because it's documented 258 # to be the same as 3.10 code below: 259 if sys.version_info >= (3, 12): 260 return context.lib_path 261 262 # Python 3.10+ 263 if "venv" in sysconfig.get_scheme_names(): 264 lib_path = sysconfig.get_path( 265 "purelib", scheme="venv", vars={"base": context.env_dir} 266 ) 267 assert lib_path is not None 268 return lib_path 269 270 # For Python <= 3.9 we need to hardcode this. Fortunately the 271 # code below was the same in Python 3.6-3.10, so there is only 272 # one case. 273 if sys.platform == "win32": 274 return os.path.join(context.env_dir, "Lib", "site-packages") 275 return os.path.join( 276 context.env_dir, 277 "lib", 278 "python%d.%d" % sys.version_info[:2], 279 "site-packages", 280 ) 281 282 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: 283 logger.debug("ensure_directories(env_dir=%s)", env_dir) 284 self._context = super().ensure_directories(env_dir) 285 return self._context 286 287 def create(self, env_dir: DirType) -> None: 288 logger.debug("create(env_dir=%s)", env_dir) 289 super().create(env_dir) 290 assert self._context is not None 291 self.post_post_setup(self._context) 292 293 def post_post_setup(self, context: SimpleNamespace) -> None: 294 """ 295 The final, final hook. Enter the venv and run commands inside of it. 296 """ 297 if self.use_parent_packages: 298 # We're inside of a venv and we want to include the parent 299 # venv's packages. 300 parent_libpath = self.get_parent_libpath() 301 assert parent_libpath is not None 302 logger.debug("parent_libpath: %s", parent_libpath) 303 304 our_libpath = self.compute_venv_libpath(context) 305 logger.debug("our_libpath: %s", our_libpath) 306 307 pth_file = os.path.join(our_libpath, "nested.pth") 308 with open(pth_file, "w", encoding="UTF-8") as file: 309 file.write(parent_libpath + os.linesep) 310 311 if self.want_pip: 312 args = [ 313 context.env_exe, 314 __file__, 315 "post_init", 316 ] 317 subprocess.run(args, check=True) 318 319 def get_value(self, field: str) -> str: 320 """ 321 Get a string value from the context namespace after a call to build. 322 323 For valid field names, see: 324 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories 325 """ 326 ret = getattr(self._context, field) 327 assert isinstance(ret, str) 328 return ret 329 330 331def need_ensurepip() -> bool: 332 """ 333 Tests for the presence of setuptools and pip. 334 335 :return: `True` if we do not detect both packages. 336 """ 337 # Don't try to actually import them, it's fraught with danger: 338 # https://github.com/pypa/setuptools/issues/2993 339 if find_spec("setuptools") and find_spec("pip"): 340 return False 341 return True 342 343 344def check_ensurepip() -> None: 345 """ 346 Check that we have ensurepip. 347 348 Raise a fatal exception with a helpful hint if it isn't available. 349 """ 350 if not find_spec("ensurepip"): 351 msg = ( 352 "Python's ensurepip module is not found.\n" 353 "It's normally part of the Python standard library, " 354 "maybe your distribution packages it separately?\n" 355 "Either install ensurepip, or alleviate the need for it in the " 356 "first place by installing pip and setuptools for " 357 f"'{sys.executable}'.\n" 358 "(Hint: Debian puts ensurepip in its python3-venv package.)" 359 ) 360 raise Ouch(msg) 361 362 # ensurepip uses pyexpat, which can also go missing on us: 363 if not find_spec("pyexpat"): 364 msg = ( 365 "Python's pyexpat module is not found.\n" 366 "It's normally part of the Python standard library, " 367 "maybe your distribution packages it separately?\n" 368 "Either install pyexpat, or alleviate the need for it in the " 369 "first place by installing pip and setuptools for " 370 f"'{sys.executable}'.\n\n" 371 "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" 372 ) 373 raise Ouch(msg) 374 375 376def make_venv( # pylint: disable=too-many-arguments 377 env_dir: Union[str, Path], 378 system_site_packages: bool = False, 379 clear: bool = True, 380 symlinks: Optional[bool] = None, 381 with_pip: bool = True, 382) -> None: 383 """ 384 Create a venv using `QemuEnvBuilder`. 385 386 This is analogous to the `venv.create` module-level convenience 387 function that is part of the Python stdblib, except it uses 388 `QemuEnvBuilder` instead. 389 390 :param env_dir: The directory to create/install to. 391 :param system_site_packages: 392 Allow inheriting packages from the system installation. 393 :param clear: When True, fully remove any prior venv and files. 394 :param symlinks: 395 Whether to use symlinks to the target interpreter or not. If 396 left unspecified, it will use symlinks except on Windows to 397 match behavior with the "venv" CLI tool. 398 :param with_pip: 399 Whether to install "pip" binaries or not. 400 """ 401 logger.debug( 402 "%s: make_venv(env_dir=%s, system_site_packages=%s, " 403 "clear=%s, symlinks=%s, with_pip=%s)", 404 __file__, 405 str(env_dir), 406 system_site_packages, 407 clear, 408 symlinks, 409 with_pip, 410 ) 411 412 if symlinks is None: 413 # Default behavior of standard venv CLI 414 symlinks = os.name != "nt" 415 416 builder = QemuEnvBuilder( 417 system_site_packages=system_site_packages, 418 clear=clear, 419 symlinks=symlinks, 420 with_pip=with_pip, 421 ) 422 423 style = "non-isolated" if builder.system_site_packages else "isolated" 424 nested = "" 425 if builder.use_parent_packages: 426 nested = f"(with packages from '{builder.get_parent_libpath()}') " 427 print( 428 f"mkvenv: Creating {style} virtual environment" 429 f" {nested}at '{str(env_dir)}'", 430 file=sys.stderr, 431 ) 432 433 try: 434 logger.debug("Invoking builder.create()") 435 try: 436 builder.create(str(env_dir)) 437 except SystemExit as exc: 438 # pylint 3.3 bug: 439 # pylint: disable=raising-non-exception, raise-missing-from 440 441 # Some versions of the venv module raise SystemExit; *nasty*! 442 # We want the exception that prompted it. It might be a subprocess 443 # error that has output we *really* want to see. 444 logger.debug("Intercepted SystemExit from EnvBuilder.create()") 445 raise exc.__cause__ or exc.__context__ or exc 446 logger.debug("builder.create() finished") 447 except subprocess.CalledProcessError as exc: 448 logger.error("mkvenv subprocess failed:") 449 logger.error("cmd: %s", exc.cmd) 450 logger.error("returncode: %d", exc.returncode) 451 452 def _stringify(data: Union[str, bytes]) -> str: 453 if isinstance(data, bytes): 454 return data.decode() 455 return data 456 457 lines = [] 458 if exc.stdout: 459 lines.append("========== stdout ==========") 460 lines.append(_stringify(exc.stdout)) 461 lines.append("============================") 462 if exc.stderr: 463 lines.append("========== stderr ==========") 464 lines.append(_stringify(exc.stderr)) 465 lines.append("============================") 466 if lines: 467 logger.error(os.linesep.join(lines)) 468 469 raise Ouch("VENV creation subprocess failed.") from exc 470 471 # print the python executable to stdout for configure. 472 print(builder.get_value("env_exe")) 473 474 475def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: 476 477 def _generator() -> Iterator[str]: 478 for package in packages: 479 try: 480 entry_points: Iterator[EntryPoint] = \ 481 iter(distribution(package).entry_points) 482 except PackageNotFoundError: 483 continue 484 485 # The EntryPoints type is only available in 3.10+, 486 # treat this as a vanilla list and filter it ourselves. 487 entry_points = filter( 488 lambda ep: ep.group == "console_scripts", entry_points 489 ) 490 491 for entry_point in entry_points: 492 yield f"{entry_point.name} = {entry_point.value}" 493 494 return _generator() 495 496 497def generate_console_scripts( 498 packages: Sequence[str], 499 python_path: Optional[str] = None, 500 bin_path: Optional[str] = None, 501) -> None: 502 """ 503 Generate script shims for console_script entry points in @packages. 504 """ 505 if python_path is None: 506 python_path = sys.executable 507 if bin_path is None: 508 bin_path = sysconfig.get_path("scripts") 509 assert bin_path is not None 510 511 logger.debug( 512 "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", 513 packages, 514 python_path, 515 bin_path, 516 ) 517 518 if not packages: 519 return 520 521 maker = distlib.scripts.ScriptMaker(None, bin_path) 522 maker.variants = {""} 523 maker.clobber = False 524 525 for entry_point in _get_entry_points(packages): 526 for filename in maker.make(entry_point): 527 logger.debug("wrote console_script '%s'", filename) 528 529 530def pkgname_from_depspec(dep_spec: str) -> str: 531 """ 532 Parse package name out of a PEP-508 depspec. 533 534 See https://peps.python.org/pep-0508/#names 535 """ 536 match = re.match( 537 r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE 538 ) 539 if not match: 540 raise ValueError( 541 f"dep_spec '{dep_spec}'" 542 " does not appear to contain a valid package name" 543 ) 544 return match.group(0) 545 546 547def _path_is_prefix(prefix: Optional[str], path: str) -> bool: 548 try: 549 return ( 550 prefix is not None and os.path.commonpath([prefix, path]) == prefix 551 ) 552 except ValueError: 553 return False 554 555 556def _is_system_package(dist: Distribution) -> bool: 557 path = str(dist.locate_file(".")) 558 return not ( 559 _path_is_prefix(sysconfig.get_path("purelib"), path) 560 or _path_is_prefix(sysconfig.get_path("platlib"), path) 561 ) 562 563 564def diagnose( 565 dep_spec: str, 566 online: bool, 567 wheels_dir: Optional[Union[str, Path]], 568 prog: Optional[str], 569) -> Tuple[str, bool]: 570 """ 571 Offer a summary to the user as to why a package failed to be installed. 572 573 :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' 574 :param online: Did we allow PyPI access? 575 :param prog: 576 Optionally, a shell program name that can be used as a 577 bellwether to detect if this program is installed elsewhere on 578 the system. This is used to offer advice when a program is 579 detected for a different python version. 580 :param wheels_dir: 581 Optionally, a directory that was searched for vendored packages. 582 """ 583 # pylint: disable=too-many-branches 584 585 # Some errors are not particularly serious 586 bad = False 587 588 pkg_name = pkgname_from_depspec(dep_spec) 589 pkg_version: Optional[str] = None 590 try: 591 pkg_version = version(pkg_name) 592 except PackageNotFoundError: 593 pass 594 595 lines = [] 596 597 if pkg_version: 598 lines.append( 599 f"Python package '{pkg_name}' version '{pkg_version}' was found," 600 " but isn't suitable." 601 ) 602 else: 603 lines.append( 604 f"Python package '{pkg_name}' was not found nor installed." 605 ) 606 607 if wheels_dir: 608 lines.append( 609 "No suitable version found in, or failed to install from" 610 f" '{wheels_dir}'." 611 ) 612 bad = True 613 614 if online: 615 lines.append("A suitable version could not be obtained from PyPI.") 616 bad = True 617 else: 618 lines.append( 619 "mkvenv was configured to operate offline and did not check PyPI." 620 ) 621 622 if prog and not pkg_version: 623 which = shutil.which(prog) 624 if which: 625 if sys.base_prefix in site.PREFIXES: 626 pypath = Path(sys.executable).resolve() 627 lines.append( 628 f"'{prog}' was detected on your system at '{which}', " 629 f"but the Python package '{pkg_name}' was not found by " 630 f"this Python interpreter ('{pypath}'). " 631 f"Typically this means that '{prog}' has been installed " 632 "against a different Python interpreter on your system." 633 ) 634 else: 635 lines.append( 636 f"'{prog}' was detected on your system at '{which}', " 637 "but the build is using an isolated virtual environment." 638 ) 639 bad = True 640 641 lines = [f" • {line}" for line in lines] 642 if bad: 643 lines.insert(0, f"Could not provide build dependency '{dep_spec}':") 644 else: 645 lines.insert(0, f"'{dep_spec}' not found:") 646 return os.linesep.join(lines), bad 647 648 649def pip_install( 650 args: Sequence[str], 651 online: bool = False, 652 wheels_dir: Optional[Union[str, Path]] = None, 653) -> None: 654 """ 655 Use pip to install a package or package(s) as specified in @args. 656 """ 657 loud = bool( 658 os.environ.get("DEBUG") 659 or os.environ.get("GITLAB_CI") 660 or os.environ.get("V") 661 ) 662 663 full_args = [ 664 sys.executable, 665 "-m", 666 "pip", 667 "install", 668 "--disable-pip-version-check", 669 "-v" if loud else "-q", 670 ] 671 if not online: 672 full_args += ["--no-index"] 673 if wheels_dir: 674 full_args += ["--find-links", f"file://{str(wheels_dir)}"] 675 full_args += list(args) 676 subprocess.run( 677 full_args, 678 check=True, 679 ) 680 681 682def _make_version_constraint(info: Dict[str, str], install: bool) -> str: 683 """ 684 Construct the version constraint part of a PEP 508 dependency 685 specification (for example '>=0.61.5') from the accepted and 686 installed keys of the provided dictionary. 687 688 :param info: A dictionary corresponding to a TOML key-value list. 689 :param install: True generates install constraints, False generates 690 presence constraints 691 """ 692 if install and "installed" in info: 693 return "==" + info["installed"] 694 695 dep_spec = info.get("accepted", "") 696 dep_spec = dep_spec.strip() 697 # Double check that they didn't just use a version number 698 if dep_spec and dep_spec[0] not in "!~><=(": 699 raise Ouch( 700 "invalid dependency specifier " + dep_spec + " in dependency file" 701 ) 702 703 return dep_spec 704 705 706def _do_ensure( 707 group: Dict[str, Dict[str, str]], 708 online: bool = False, 709 wheels_dir: Optional[Union[str, Path]] = None, 710) -> Optional[Tuple[str, bool]]: 711 """ 712 Use pip to ensure we have the packages specified in @group. 713 714 If the packages are already installed, do nothing. If online and 715 wheels_dir are both provided, prefer packages found in wheels_dir 716 first before connecting to PyPI. 717 718 :param group: A dictionary of dictionaries, corresponding to a 719 section in a pythondeps.toml file. 720 :param online: If True, fall back to PyPI. 721 :param wheels_dir: If specified, search this path for packages. 722 """ 723 absent = [] 724 present = [] 725 canary = None 726 for name, info in group.items(): 727 constraint = _make_version_constraint(info, False) 728 matcher = Matcher(name + constraint) 729 print(f"mkvenv: checking for {matcher}", file=sys.stderr) 730 731 dist: Optional[Distribution] = None 732 try: 733 dist = distribution(matcher.name) 734 except PackageNotFoundError: 735 pass 736 737 if ( 738 dist is None 739 # Always pass installed package to pip, so that they can be 740 # updated if the requested version changes 741 or not _is_system_package(dist) 742 or not matcher.match(dist.version) 743 ): 744 absent.append(name + _make_version_constraint(info, True)) 745 if len(absent) == 1: 746 canary = info.get("canary", None) 747 else: 748 logger.info("found %s %s", name, dist.version) 749 present.append(name) 750 751 if present: 752 generate_console_scripts(present) 753 754 if absent: 755 if online or wheels_dir: 756 # Some packages are missing or aren't a suitable version, 757 # install a suitable (possibly vendored) package. 758 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) 759 try: 760 pip_install(args=absent, online=online, wheels_dir=wheels_dir) 761 return None 762 except subprocess.CalledProcessError: 763 pass 764 765 return diagnose( 766 absent[0], 767 online, 768 wheels_dir, 769 canary, 770 ) 771 772 return None 773 774 775def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: 776 if not HAVE_TOMLLIB: 777 if sys.version_info < (3, 11): 778 raise Ouch("found no usable tomli, please install it") 779 780 raise Ouch( 781 "Python >=3.11 does not have tomllib... what have you done!?" 782 ) 783 784 # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, 785 # Debian bullseye-backports) and v2.0.x 786 with open(file, "r", encoding="ascii") as depfile: 787 contents = depfile.read() 788 return tomllib.loads(contents) # type: ignore 789 790 791def ensure_group( 792 file: str, 793 groups: Sequence[str], 794 online: bool = False, 795 wheels_dir: Optional[Union[str, Path]] = None, 796) -> None: 797 """ 798 Use pip to ensure we have the package specified by @dep_specs. 799 800 If the package is already installed, do nothing. If online and 801 wheels_dir are both provided, prefer packages found in wheels_dir 802 first before connecting to PyPI. 803 804 :param dep_specs: 805 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 806 :param online: If True, fall back to PyPI. 807 :param wheels_dir: If specified, search this path for packages. 808 """ 809 810 if not HAVE_DISTLIB: 811 raise Ouch("found no usable distlib, please install it") 812 813 parsed_deps = _parse_groups(file) 814 815 to_install: Dict[str, Dict[str, str]] = {} 816 for group in groups: 817 try: 818 to_install.update(parsed_deps[group]) 819 except KeyError as exc: 820 raise Ouch(f"group {group} not defined") from exc 821 822 result = _do_ensure(to_install, online, wheels_dir) 823 if result: 824 # Well, that's not good. 825 if result[1]: 826 raise Ouch(result[0]) 827 raise SystemExit(f"\n{result[0]}\n\n") 828 829 830def post_venv_setup() -> None: 831 """ 832 This is intended to be run *inside the venv* after it is created. 833 """ 834 logger.debug("post_venv_setup()") 835 # Generate a 'pip' script so the venv is usable in a normal 836 # way from the CLI. This only happens when we inherited pip from a 837 # parent/system-site and haven't run ensurepip in some way. 838 generate_console_scripts(["pip"]) 839 840 841def _add_create_subcommand(subparsers: Any) -> None: 842 subparser = subparsers.add_parser("create", help="create a venv") 843 subparser.add_argument( 844 "target", 845 type=str, 846 action="store", 847 help="Target directory to install virtual environment into.", 848 ) 849 850 851def _add_post_init_subcommand(subparsers: Any) -> None: 852 subparsers.add_parser("post_init", help="post-venv initialization") 853 854 855def _add_ensuregroup_subcommand(subparsers: Any) -> None: 856 subparser = subparsers.add_parser( 857 "ensuregroup", 858 help="Ensure that the specified package group is installed.", 859 ) 860 subparser.add_argument( 861 "--online", 862 action="store_true", 863 help="Install packages from PyPI, if necessary.", 864 ) 865 subparser.add_argument( 866 "--dir", 867 type=str, 868 action="store", 869 help="Path to vendored packages where we may install from.", 870 ) 871 subparser.add_argument( 872 "file", 873 type=str, 874 action="store", 875 help=("Path to a TOML file describing package groups"), 876 ) 877 subparser.add_argument( 878 "group", 879 type=str, 880 action="store", 881 help="One or more package group names", 882 nargs="+", 883 ) 884 885 886def main() -> int: 887 """CLI interface to make_qemu_venv. See module docstring.""" 888 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): 889 # You're welcome. 890 logging.basicConfig(level=logging.DEBUG) 891 else: 892 if os.environ.get("V"): 893 logging.basicConfig(level=logging.INFO) 894 895 parser = argparse.ArgumentParser( 896 prog="mkvenv", 897 description="QEMU pyvenv bootstrapping utility", 898 ) 899 subparsers = parser.add_subparsers( 900 title="Commands", 901 dest="command", 902 required=True, 903 metavar="command", 904 help="Description", 905 ) 906 907 _add_create_subcommand(subparsers) 908 _add_post_init_subcommand(subparsers) 909 _add_ensuregroup_subcommand(subparsers) 910 911 args = parser.parse_args() 912 try: 913 if args.command == "create": 914 make_venv( 915 args.target, 916 system_site_packages=True, 917 clear=True, 918 ) 919 if args.command == "post_init": 920 post_venv_setup() 921 if args.command == "ensuregroup": 922 ensure_group( 923 file=args.file, 924 groups=args.group, 925 online=args.online, 926 wheels_dir=args.dir, 927 ) 928 logger.debug("mkvenv.py %s: exiting", args.command) 929 except Ouch as exc: 930 print("\n*** Ouch! ***\n", file=sys.stderr) 931 print(str(exc), "\n\n", file=sys.stderr) 932 return 1 933 except SystemExit: 934 raise 935 except: # pylint: disable=bare-except 936 logger.exception("mkvenv did not complete successfully:") 937 return 2 938 return 0 939 940 941if __name__ == "__main__": 942 sys.exit(main()) 943