1""" 2mkvenv - QEMU pyvenv bootstrapping utility 3 4usage: mkvenv [-h] command ... 5 6QEMU pyvenv bootstrapping utility 7 8options: 9 -h, --help show this help message and exit 10 11Commands: 12 command Description 13 create create a venv 14 post_init 15 post-venv initialization 16 ensure Ensure that the specified package is installed. 17 ensuregroup 18 Ensure that the specified package group is installed. 19 20-------------------------------------------------- 21 22usage: mkvenv create [-h] target 23 24positional arguments: 25 target Target directory to install virtual environment into. 26 27options: 28 -h, --help show this help message and exit 29 30-------------------------------------------------- 31 32usage: mkvenv post_init [-h] 33 34options: 35 -h, --help show this help message and exit 36 37-------------------------------------------------- 38 39usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... 40 41positional arguments: 42 dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' 43 44options: 45 -h, --help show this help message and exit 46 --online Install packages from PyPI, if necessary. 47 --dir DIR Path to vendored packages where we may install from. 48 49-------------------------------------------------- 50 51usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... 52 53positional arguments: 54 file pointer to a TOML file 55 group section name in the TOML file 56 57options: 58 -h, --help show this help message and exit 59 --online Install packages from PyPI, if necessary. 60 --dir DIR Path to vendored packages where we may install from. 61 62""" 63 64# Copyright (C) 2022-2023 Red Hat, Inc. 65# 66# Authors: 67# John Snow <jsnow@redhat.com> 68# Paolo Bonzini <pbonzini@redhat.com> 69# 70# This work is licensed under the terms of the GNU GPL, version 2 or 71# later. See the COPYING file in the top-level directory. 72 73import argparse 74from importlib.metadata import ( 75 Distribution, 76 EntryPoint, 77 PackageNotFoundError, 78 distribution, 79 version, 80) 81from importlib.util import find_spec 82import logging 83import os 84from pathlib import Path 85import re 86import shutil 87import site 88import subprocess 89import sys 90import sysconfig 91from types import SimpleNamespace 92from typing import ( 93 Any, 94 Dict, 95 Iterator, 96 Optional, 97 Sequence, 98 Tuple, 99 Union, 100) 101import venv 102 103 104# Try to load distlib, with a fallback to pip's vendored version. 105# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail 106# outside the venv or before a potential call to ensurepip in checkpip(). 107HAVE_DISTLIB = True 108try: 109 import distlib.scripts 110 import distlib.version 111except ImportError: 112 try: 113 # Reach into pip's cookie jar. pylint and flake8 don't understand 114 # that these imports will be used via distlib.xxx. 115 from pip._vendor import distlib 116 import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import 117 import pip._vendor.distlib.version # noqa, pylint: disable=unused-import 118 except ImportError: 119 HAVE_DISTLIB = False 120 121# Try to load tomllib, with a fallback to tomli. 122# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail 123# outside the venv or before a potential call to ensurepip in checkpip(). 124HAVE_TOMLLIB = True 125try: 126 import tomllib 127except ImportError: 128 try: 129 import tomli as tomllib 130 except ImportError: 131 HAVE_TOMLLIB = False 132 133# Do not add any mandatory dependencies from outside the stdlib: 134# This script *must* be usable standalone! 135 136DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] 137logger = logging.getLogger("mkvenv") 138 139 140def inside_a_venv() -> bool: 141 """Returns True if it is executed inside of a virtual environment.""" 142 return sys.prefix != sys.base_prefix 143 144 145class Ouch(RuntimeError): 146 """An Exception class we can't confuse with a builtin.""" 147 148 149class QemuEnvBuilder(venv.EnvBuilder): 150 """ 151 An extension of venv.EnvBuilder for building QEMU's configure-time venv. 152 153 The primary difference is that it emulates a "nested" virtual 154 environment when invoked from inside of an existing virtual 155 environment by including packages from the parent. Also, 156 "ensurepip" is replaced if possible with just recreating pip's 157 console_scripts inside the virtual environment. 158 159 Parameters for base class init: 160 - system_site_packages: bool = False 161 - clear: bool = False 162 - symlinks: bool = False 163 - upgrade: bool = False 164 - with_pip: bool = False 165 - prompt: Optional[str] = None 166 - upgrade_deps: bool = False (Since 3.9) 167 """ 168 169 def __init__(self, *args: Any, **kwargs: Any) -> None: 170 logger.debug("QemuEnvBuilder.__init__(...)") 171 172 # For nested venv emulation: 173 self.use_parent_packages = False 174 if inside_a_venv(): 175 # Include parent packages only if we're in a venv and 176 # system_site_packages was True. 177 self.use_parent_packages = kwargs.pop( 178 "system_site_packages", False 179 ) 180 # Include system_site_packages only when the parent, 181 # The venv we are currently in, also does so. 182 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES 183 184 # ensurepip is slow: venv creation can be very fast for cases where 185 # we allow the use of system_site_packages. Therefore, ensurepip is 186 # replaced with our own script generation once the virtual environment 187 # is setup. 188 self.want_pip = kwargs.get("with_pip", False) 189 if self.want_pip: 190 if ( 191 kwargs.get("system_site_packages", False) 192 and not need_ensurepip() 193 ): 194 kwargs["with_pip"] = False 195 else: 196 check_ensurepip() 197 198 super().__init__(*args, **kwargs) 199 200 # Make the context available post-creation: 201 self._context: Optional[SimpleNamespace] = None 202 203 def get_parent_libpath(self) -> Optional[str]: 204 """Return the libpath of the parent venv, if applicable.""" 205 if self.use_parent_packages: 206 return sysconfig.get_path("purelib") 207 return None 208 209 @staticmethod 210 def compute_venv_libpath(context: SimpleNamespace) -> str: 211 """ 212 Compatibility wrapper for context.lib_path for Python < 3.12 213 """ 214 # Python 3.12+, not strictly necessary because it's documented 215 # to be the same as 3.10 code below: 216 if sys.version_info >= (3, 12): 217 return context.lib_path 218 219 # Python 3.10+ 220 if "venv" in sysconfig.get_scheme_names(): 221 lib_path = sysconfig.get_path( 222 "purelib", scheme="venv", vars={"base": context.env_dir} 223 ) 224 assert lib_path is not None 225 return lib_path 226 227 # For Python <= 3.9 we need to hardcode this. Fortunately the 228 # code below was the same in Python 3.6-3.10, so there is only 229 # one case. 230 if sys.platform == "win32": 231 return os.path.join(context.env_dir, "Lib", "site-packages") 232 return os.path.join( 233 context.env_dir, 234 "lib", 235 "python%d.%d" % sys.version_info[:2], 236 "site-packages", 237 ) 238 239 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: 240 logger.debug("ensure_directories(env_dir=%s)", env_dir) 241 self._context = super().ensure_directories(env_dir) 242 return self._context 243 244 def create(self, env_dir: DirType) -> None: 245 logger.debug("create(env_dir=%s)", env_dir) 246 super().create(env_dir) 247 assert self._context is not None 248 self.post_post_setup(self._context) 249 250 def post_post_setup(self, context: SimpleNamespace) -> None: 251 """ 252 The final, final hook. Enter the venv and run commands inside of it. 253 """ 254 if self.use_parent_packages: 255 # We're inside of a venv and we want to include the parent 256 # venv's packages. 257 parent_libpath = self.get_parent_libpath() 258 assert parent_libpath is not None 259 logger.debug("parent_libpath: %s", parent_libpath) 260 261 our_libpath = self.compute_venv_libpath(context) 262 logger.debug("our_libpath: %s", our_libpath) 263 264 pth_file = os.path.join(our_libpath, "nested.pth") 265 with open(pth_file, "w", encoding="UTF-8") as file: 266 file.write(parent_libpath + os.linesep) 267 268 if self.want_pip: 269 args = [ 270 context.env_exe, 271 __file__, 272 "post_init", 273 ] 274 subprocess.run(args, check=True) 275 276 def get_value(self, field: str) -> str: 277 """ 278 Get a string value from the context namespace after a call to build. 279 280 For valid field names, see: 281 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories 282 """ 283 ret = getattr(self._context, field) 284 assert isinstance(ret, str) 285 return ret 286 287 288def need_ensurepip() -> bool: 289 """ 290 Tests for the presence of setuptools and pip. 291 292 :return: `True` if we do not detect both packages. 293 """ 294 # Don't try to actually import them, it's fraught with danger: 295 # https://github.com/pypa/setuptools/issues/2993 296 if find_spec("setuptools") and find_spec("pip"): 297 return False 298 return True 299 300 301def check_ensurepip() -> None: 302 """ 303 Check that we have ensurepip. 304 305 Raise a fatal exception with a helpful hint if it isn't available. 306 """ 307 if not find_spec("ensurepip"): 308 msg = ( 309 "Python's ensurepip module is not found.\n" 310 "It's normally part of the Python standard library, " 311 "maybe your distribution packages it separately?\n" 312 "Either install ensurepip, or alleviate the need for it in the " 313 "first place by installing pip and setuptools for " 314 f"'{sys.executable}'.\n" 315 "(Hint: Debian puts ensurepip in its python3-venv package.)" 316 ) 317 raise Ouch(msg) 318 319 # ensurepip uses pyexpat, which can also go missing on us: 320 if not find_spec("pyexpat"): 321 msg = ( 322 "Python's pyexpat module is not found.\n" 323 "It's normally part of the Python standard library, " 324 "maybe your distribution packages it separately?\n" 325 "Either install pyexpat, or alleviate the need for it in the " 326 "first place by installing pip and setuptools for " 327 f"'{sys.executable}'.\n\n" 328 "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" 329 ) 330 raise Ouch(msg) 331 332 333def make_venv( # pylint: disable=too-many-arguments 334 env_dir: Union[str, Path], 335 system_site_packages: bool = False, 336 clear: bool = True, 337 symlinks: Optional[bool] = None, 338 with_pip: bool = True, 339) -> None: 340 """ 341 Create a venv using `QemuEnvBuilder`. 342 343 This is analogous to the `venv.create` module-level convenience 344 function that is part of the Python stdblib, except it uses 345 `QemuEnvBuilder` instead. 346 347 :param env_dir: The directory to create/install to. 348 :param system_site_packages: 349 Allow inheriting packages from the system installation. 350 :param clear: When True, fully remove any prior venv and files. 351 :param symlinks: 352 Whether to use symlinks to the target interpreter or not. If 353 left unspecified, it will use symlinks except on Windows to 354 match behavior with the "venv" CLI tool. 355 :param with_pip: 356 Whether to install "pip" binaries or not. 357 """ 358 logger.debug( 359 "%s: make_venv(env_dir=%s, system_site_packages=%s, " 360 "clear=%s, symlinks=%s, with_pip=%s)", 361 __file__, 362 str(env_dir), 363 system_site_packages, 364 clear, 365 symlinks, 366 with_pip, 367 ) 368 369 if symlinks is None: 370 # Default behavior of standard venv CLI 371 symlinks = os.name != "nt" 372 373 builder = QemuEnvBuilder( 374 system_site_packages=system_site_packages, 375 clear=clear, 376 symlinks=symlinks, 377 with_pip=with_pip, 378 ) 379 380 style = "non-isolated" if builder.system_site_packages else "isolated" 381 nested = "" 382 if builder.use_parent_packages: 383 nested = f"(with packages from '{builder.get_parent_libpath()}') " 384 print( 385 f"mkvenv: Creating {style} virtual environment" 386 f" {nested}at '{str(env_dir)}'", 387 file=sys.stderr, 388 ) 389 390 try: 391 logger.debug("Invoking builder.create()") 392 try: 393 builder.create(str(env_dir)) 394 except SystemExit as exc: 395 # Some versions of the venv module raise SystemExit; *nasty*! 396 # We want the exception that prompted it. It might be a subprocess 397 # error that has output we *really* want to see. 398 logger.debug("Intercepted SystemExit from EnvBuilder.create()") 399 raise exc.__cause__ or exc.__context__ or exc 400 logger.debug("builder.create() finished") 401 except subprocess.CalledProcessError as exc: 402 logger.error("mkvenv subprocess failed:") 403 logger.error("cmd: %s", exc.cmd) 404 logger.error("returncode: %d", exc.returncode) 405 406 def _stringify(data: Union[str, bytes]) -> str: 407 if isinstance(data, bytes): 408 return data.decode() 409 return data 410 411 lines = [] 412 if exc.stdout: 413 lines.append("========== stdout ==========") 414 lines.append(_stringify(exc.stdout)) 415 lines.append("============================") 416 if exc.stderr: 417 lines.append("========== stderr ==========") 418 lines.append(_stringify(exc.stderr)) 419 lines.append("============================") 420 if lines: 421 logger.error(os.linesep.join(lines)) 422 423 raise Ouch("VENV creation subprocess failed.") from exc 424 425 # print the python executable to stdout for configure. 426 print(builder.get_value("env_exe")) 427 428 429def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: 430 431 def _generator() -> Iterator[str]: 432 for package in packages: 433 try: 434 entry_points: Iterator[EntryPoint] = \ 435 iter(distribution(package).entry_points) 436 except PackageNotFoundError: 437 continue 438 439 # The EntryPoints type is only available in 3.10+, 440 # treat this as a vanilla list and filter it ourselves. 441 entry_points = filter( 442 lambda ep: ep.group == "console_scripts", entry_points 443 ) 444 445 for entry_point in entry_points: 446 yield f"{entry_point.name} = {entry_point.value}" 447 448 return _generator() 449 450 451def generate_console_scripts( 452 packages: Sequence[str], 453 python_path: Optional[str] = None, 454 bin_path: Optional[str] = None, 455) -> None: 456 """ 457 Generate script shims for console_script entry points in @packages. 458 """ 459 if python_path is None: 460 python_path = sys.executable 461 if bin_path is None: 462 bin_path = sysconfig.get_path("scripts") 463 assert bin_path is not None 464 465 logger.debug( 466 "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", 467 packages, 468 python_path, 469 bin_path, 470 ) 471 472 if not packages: 473 return 474 475 maker = distlib.scripts.ScriptMaker(None, bin_path) 476 maker.variants = {""} 477 maker.clobber = False 478 479 for entry_point in _get_entry_points(packages): 480 for filename in maker.make(entry_point): 481 logger.debug("wrote console_script '%s'", filename) 482 483 484def pkgname_from_depspec(dep_spec: str) -> str: 485 """ 486 Parse package name out of a PEP-508 depspec. 487 488 See https://peps.python.org/pep-0508/#names 489 """ 490 match = re.match( 491 r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE 492 ) 493 if not match: 494 raise ValueError( 495 f"dep_spec '{dep_spec}'" 496 " does not appear to contain a valid package name" 497 ) 498 return match.group(0) 499 500 501def _path_is_prefix(prefix: Optional[str], path: str) -> bool: 502 try: 503 return ( 504 prefix is not None and os.path.commonpath([prefix, path]) == prefix 505 ) 506 except ValueError: 507 return False 508 509 510def _is_system_package(dist: Distribution) -> bool: 511 path = str(dist.locate_file(".")) 512 return not ( 513 _path_is_prefix(sysconfig.get_path("purelib"), path) 514 or _path_is_prefix(sysconfig.get_path("platlib"), path) 515 ) 516 517 518def diagnose( 519 dep_spec: str, 520 online: bool, 521 wheels_dir: Optional[Union[str, Path]], 522 prog: Optional[str], 523) -> Tuple[str, bool]: 524 """ 525 Offer a summary to the user as to why a package failed to be installed. 526 527 :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' 528 :param online: Did we allow PyPI access? 529 :param prog: 530 Optionally, a shell program name that can be used as a 531 bellwether to detect if this program is installed elsewhere on 532 the system. This is used to offer advice when a program is 533 detected for a different python version. 534 :param wheels_dir: 535 Optionally, a directory that was searched for vendored packages. 536 """ 537 # pylint: disable=too-many-branches 538 539 # Some errors are not particularly serious 540 bad = False 541 542 pkg_name = pkgname_from_depspec(dep_spec) 543 pkg_version: Optional[str] = None 544 try: 545 pkg_version = version(pkg_name) 546 except PackageNotFoundError: 547 pass 548 549 lines = [] 550 551 if pkg_version: 552 lines.append( 553 f"Python package '{pkg_name}' version '{pkg_version}' was found," 554 " but isn't suitable." 555 ) 556 else: 557 lines.append( 558 f"Python package '{pkg_name}' was not found nor installed." 559 ) 560 561 if wheels_dir: 562 lines.append( 563 "No suitable version found in, or failed to install from" 564 f" '{wheels_dir}'." 565 ) 566 bad = True 567 568 if online: 569 lines.append("A suitable version could not be obtained from PyPI.") 570 bad = True 571 else: 572 lines.append( 573 "mkvenv was configured to operate offline and did not check PyPI." 574 ) 575 576 if prog and not pkg_version: 577 which = shutil.which(prog) 578 if which: 579 if sys.base_prefix in site.PREFIXES: 580 pypath = Path(sys.executable).resolve() 581 lines.append( 582 f"'{prog}' was detected on your system at '{which}', " 583 f"but the Python package '{pkg_name}' was not found by " 584 f"this Python interpreter ('{pypath}'). " 585 f"Typically this means that '{prog}' has been installed " 586 "against a different Python interpreter on your system." 587 ) 588 else: 589 lines.append( 590 f"'{prog}' was detected on your system at '{which}', " 591 "but the build is using an isolated virtual environment." 592 ) 593 bad = True 594 595 lines = [f" • {line}" for line in lines] 596 if bad: 597 lines.insert(0, f"Could not provide build dependency '{dep_spec}':") 598 else: 599 lines.insert(0, f"'{dep_spec}' not found:") 600 return os.linesep.join(lines), bad 601 602 603def pip_install( 604 args: Sequence[str], 605 online: bool = False, 606 wheels_dir: Optional[Union[str, Path]] = None, 607) -> None: 608 """ 609 Use pip to install a package or package(s) as specified in @args. 610 """ 611 loud = bool( 612 os.environ.get("DEBUG") 613 or os.environ.get("GITLAB_CI") 614 or os.environ.get("V") 615 ) 616 617 full_args = [ 618 sys.executable, 619 "-m", 620 "pip", 621 "install", 622 "--disable-pip-version-check", 623 "-v" if loud else "-q", 624 ] 625 if not online: 626 full_args += ["--no-index"] 627 if wheels_dir: 628 full_args += ["--find-links", f"file://{str(wheels_dir)}"] 629 full_args += list(args) 630 subprocess.run( 631 full_args, 632 check=True, 633 ) 634 635 636def _make_version_constraint(info: Dict[str, str], install: bool) -> str: 637 """ 638 Construct the version constraint part of a PEP 508 dependency 639 specification (for example '>=0.61.5') from the accepted and 640 installed keys of the provided dictionary. 641 642 :param info: A dictionary corresponding to a TOML key-value list. 643 :param install: True generates install constraints, False generates 644 presence constraints 645 """ 646 if install and "installed" in info: 647 return "==" + info["installed"] 648 649 dep_spec = info.get("accepted", "") 650 dep_spec = dep_spec.strip() 651 # Double check that they didn't just use a version number 652 if dep_spec and dep_spec[0] not in "!~><=(": 653 raise Ouch( 654 "invalid dependency specifier " + dep_spec + " in dependency file" 655 ) 656 657 return dep_spec 658 659 660def _do_ensure( 661 group: Dict[str, Dict[str, str]], 662 online: bool = False, 663 wheels_dir: Optional[Union[str, Path]] = None, 664) -> Optional[Tuple[str, bool]]: 665 """ 666 Use pip to ensure we have the packages specified in @group. 667 668 If the packages are already installed, do nothing. If online and 669 wheels_dir are both provided, prefer packages found in wheels_dir 670 first before connecting to PyPI. 671 672 :param group: A dictionary of dictionaries, corresponding to a 673 section in a pythondeps.toml file. 674 :param online: If True, fall back to PyPI. 675 :param wheels_dir: If specified, search this path for packages. 676 """ 677 absent = [] 678 present = [] 679 canary = None 680 for name, info in group.items(): 681 constraint = _make_version_constraint(info, False) 682 matcher = distlib.version.LegacyMatcher(name + constraint) 683 print(f"mkvenv: checking for {matcher}", file=sys.stderr) 684 685 dist: Optional[Distribution] = None 686 try: 687 dist = distribution(matcher.name) 688 except PackageNotFoundError: 689 pass 690 691 if ( 692 dist is None 693 # Always pass installed package to pip, so that they can be 694 # updated if the requested version changes 695 or not _is_system_package(dist) 696 or not matcher.match(distlib.version.LegacyVersion(dist.version)) 697 ): 698 absent.append(name + _make_version_constraint(info, True)) 699 if len(absent) == 1: 700 canary = info.get("canary", None) 701 else: 702 logger.info("found %s %s", name, dist.version) 703 present.append(name) 704 705 if present: 706 generate_console_scripts(present) 707 708 if absent: 709 if online or wheels_dir: 710 # Some packages are missing or aren't a suitable version, 711 # install a suitable (possibly vendored) package. 712 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) 713 try: 714 pip_install(args=absent, online=online, wheels_dir=wheels_dir) 715 return None 716 except subprocess.CalledProcessError: 717 pass 718 719 return diagnose( 720 absent[0], 721 online, 722 wheels_dir, 723 canary, 724 ) 725 726 return None 727 728 729def ensure( 730 dep_specs: Sequence[str], 731 online: bool = False, 732 wheels_dir: Optional[Union[str, Path]] = None, 733 prog: Optional[str] = None, 734) -> None: 735 """ 736 Use pip to ensure we have the package specified by @dep_specs. 737 738 If the package is already installed, do nothing. If online and 739 wheels_dir are both provided, prefer packages found in wheels_dir 740 first before connecting to PyPI. 741 742 :param dep_specs: 743 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 744 :param online: If True, fall back to PyPI. 745 :param wheels_dir: If specified, search this path for packages. 746 :param prog: 747 If specified, use this program name for error diagnostics that will 748 be presented to the user. e.g., 'sphinx-build' can be used as a 749 bellwether for the presence of 'sphinx'. 750 """ 751 752 if not HAVE_DISTLIB: 753 raise Ouch("a usable distlib could not be found, please install it") 754 755 # Convert the depspecs to a dictionary, as if they came 756 # from a section in a pythondeps.toml file 757 group: Dict[str, Dict[str, str]] = {} 758 for spec in dep_specs: 759 name = distlib.version.LegacyMatcher(spec).name 760 group[name] = {} 761 762 spec = spec.strip() 763 pos = len(name) 764 ver = spec[pos:].strip() 765 if ver: 766 group[name]["accepted"] = ver 767 768 if prog: 769 group[name]["canary"] = prog 770 prog = None 771 772 result = _do_ensure(group, online, wheels_dir) 773 if result: 774 # Well, that's not good. 775 if result[1]: 776 raise Ouch(result[0]) 777 raise SystemExit(f"\n{result[0]}\n\n") 778 779 780def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: 781 if not HAVE_TOMLLIB: 782 if sys.version_info < (3, 11): 783 raise Ouch("found no usable tomli, please install it") 784 785 raise Ouch( 786 "Python >=3.11 does not have tomllib... what have you done!?" 787 ) 788 789 # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, 790 # Debian bullseye-backports) and v2.0.x 791 with open(file, "r", encoding="ascii") as depfile: 792 contents = depfile.read() 793 return tomllib.loads(contents) # type: ignore 794 795 796def ensure_group( 797 file: str, 798 groups: Sequence[str], 799 online: bool = False, 800 wheels_dir: Optional[Union[str, Path]] = None, 801) -> None: 802 """ 803 Use pip to ensure we have the package specified by @dep_specs. 804 805 If the package is already installed, do nothing. If online and 806 wheels_dir are both provided, prefer packages found in wheels_dir 807 first before connecting to PyPI. 808 809 :param dep_specs: 810 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 811 :param online: If True, fall back to PyPI. 812 :param wheels_dir: If specified, search this path for packages. 813 """ 814 815 if not HAVE_DISTLIB: 816 raise Ouch("found no usable distlib, please install it") 817 818 parsed_deps = _parse_groups(file) 819 820 to_install: Dict[str, Dict[str, str]] = {} 821 for group in groups: 822 try: 823 to_install.update(parsed_deps[group]) 824 except KeyError as exc: 825 raise Ouch(f"group {group} not defined") from exc 826 827 result = _do_ensure(to_install, online, wheels_dir) 828 if result: 829 # Well, that's not good. 830 if result[1]: 831 raise Ouch(result[0]) 832 raise SystemExit(f"\n{result[0]}\n\n") 833 834 835def post_venv_setup() -> None: 836 """ 837 This is intended to be run *inside the venv* after it is created. 838 """ 839 logger.debug("post_venv_setup()") 840 # Generate a 'pip' script so the venv is usable in a normal 841 # way from the CLI. This only happens when we inherited pip from a 842 # parent/system-site and haven't run ensurepip in some way. 843 generate_console_scripts(["pip"]) 844 845 846def _add_create_subcommand(subparsers: Any) -> None: 847 subparser = subparsers.add_parser("create", help="create a venv") 848 subparser.add_argument( 849 "target", 850 type=str, 851 action="store", 852 help="Target directory to install virtual environment into.", 853 ) 854 855 856def _add_post_init_subcommand(subparsers: Any) -> None: 857 subparsers.add_parser("post_init", help="post-venv initialization") 858 859 860def _add_ensuregroup_subcommand(subparsers: Any) -> None: 861 subparser = subparsers.add_parser( 862 "ensuregroup", 863 help="Ensure that the specified package group is installed.", 864 ) 865 subparser.add_argument( 866 "--online", 867 action="store_true", 868 help="Install packages from PyPI, if necessary.", 869 ) 870 subparser.add_argument( 871 "--dir", 872 type=str, 873 action="store", 874 help="Path to vendored packages where we may install from.", 875 ) 876 subparser.add_argument( 877 "file", 878 type=str, 879 action="store", 880 help=("Path to a TOML file describing package groups"), 881 ) 882 subparser.add_argument( 883 "group", 884 type=str, 885 action="store", 886 help="One or more package group names", 887 nargs="+", 888 ) 889 890 891def _add_ensure_subcommand(subparsers: Any) -> None: 892 subparser = subparsers.add_parser( 893 "ensure", help="Ensure that the specified package is installed." 894 ) 895 subparser.add_argument( 896 "--online", 897 action="store_true", 898 help="Install packages from PyPI, if necessary.", 899 ) 900 subparser.add_argument( 901 "--dir", 902 type=str, 903 action="store", 904 help="Path to vendored packages where we may install from.", 905 ) 906 subparser.add_argument( 907 "--diagnose", 908 type=str, 909 action="store", 910 help=( 911 "Name of a shell utility to use for " 912 "diagnostics if this command fails." 913 ), 914 ) 915 subparser.add_argument( 916 "dep_specs", 917 type=str, 918 action="store", 919 help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", 920 nargs="+", 921 ) 922 923 924def main() -> int: 925 """CLI interface to make_qemu_venv. See module docstring.""" 926 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): 927 # You're welcome. 928 logging.basicConfig(level=logging.DEBUG) 929 else: 930 if os.environ.get("V"): 931 logging.basicConfig(level=logging.INFO) 932 933 parser = argparse.ArgumentParser( 934 prog="mkvenv", 935 description="QEMU pyvenv bootstrapping utility", 936 ) 937 subparsers = parser.add_subparsers( 938 title="Commands", 939 dest="command", 940 required=True, 941 metavar="command", 942 help="Description", 943 ) 944 945 _add_create_subcommand(subparsers) 946 _add_post_init_subcommand(subparsers) 947 _add_ensure_subcommand(subparsers) 948 _add_ensuregroup_subcommand(subparsers) 949 950 args = parser.parse_args() 951 try: 952 if args.command == "create": 953 make_venv( 954 args.target, 955 system_site_packages=True, 956 clear=True, 957 ) 958 if args.command == "post_init": 959 post_venv_setup() 960 if args.command == "ensure": 961 ensure( 962 dep_specs=args.dep_specs, 963 online=args.online, 964 wheels_dir=args.dir, 965 prog=args.diagnose, 966 ) 967 if args.command == "ensuregroup": 968 ensure_group( 969 file=args.file, 970 groups=args.group, 971 online=args.online, 972 wheels_dir=args.dir, 973 ) 974 logger.debug("mkvenv.py %s: exiting", args.command) 975 except Ouch as exc: 976 print("\n*** Ouch! ***\n", file=sys.stderr) 977 print(str(exc), "\n\n", file=sys.stderr) 978 return 1 979 except SystemExit: 980 raise 981 except: # pylint: disable=bare-except 982 logger.exception("mkvenv did not complete successfully:") 983 return 2 984 return 0 985 986 987if __name__ == "__main__": 988 sys.exit(main()) 989