1""" 2mkvenv - QEMU pyvenv bootstrapping utility 3 4usage: mkvenv [-h] command ... 5 6QEMU pyvenv bootstrapping utility 7 8options: 9 -h, --help show this help message and exit 10 11Commands: 12 command Description 13 create create a venv 14 post_init 15 post-venv initialization 16 ensure Ensure that the specified package is installed. 17 ensuregroup 18 Ensure that the specified package group is installed. 19 20-------------------------------------------------- 21 22usage: mkvenv create [-h] target 23 24positional arguments: 25 target Target directory to install virtual environment into. 26 27options: 28 -h, --help show this help message and exit 29 30-------------------------------------------------- 31 32usage: mkvenv post_init [-h] 33 34options: 35 -h, --help show this help message and exit 36 37-------------------------------------------------- 38 39usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... 40 41positional arguments: 42 dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' 43 44options: 45 -h, --help show this help message and exit 46 --online Install packages from PyPI, if necessary. 47 --dir DIR Path to vendored packages where we may install from. 48 49-------------------------------------------------- 50 51usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... 52 53positional arguments: 54 file pointer to a TOML file 55 group section name in the TOML file 56 57options: 58 -h, --help show this help message and exit 59 --online Install packages from PyPI, if necessary. 60 --dir DIR Path to vendored packages where we may install from. 61 62""" 63 64# pylint: disable=too-many-lines 65 66# Copyright (C) 2022-2023 Red Hat, Inc. 67# 68# Authors: 69# John Snow <jsnow@redhat.com> 70# Paolo Bonzini <pbonzini@redhat.com> 71# 72# This work is licensed under the terms of the GNU GPL, version 2 or 73# later. See the COPYING file in the top-level directory. 74 75import argparse 76from importlib.metadata import ( 77 Distribution, 78 EntryPoint, 79 PackageNotFoundError, 80 distribution, 81 version, 82) 83from importlib.util import find_spec 84import logging 85import os 86from pathlib import Path 87import re 88import shutil 89import site 90import subprocess 91import sys 92import sysconfig 93from types import SimpleNamespace 94from typing import ( 95 Any, 96 Dict, 97 Iterator, 98 Optional, 99 Sequence, 100 Tuple, 101 Union, 102) 103import venv 104 105 106# Try to load distlib, with a fallback to pip's vendored version. 107# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail 108# outside the venv or before a potential call to ensurepip in checkpip(). 109HAVE_DISTLIB = True 110try: 111 import distlib.scripts 112 import distlib.version 113except ImportError: 114 try: 115 # Reach into pip's cookie jar. pylint and flake8 don't understand 116 # that these imports will be used via distlib.xxx. 117 from pip._vendor import distlib 118 import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import 119 import pip._vendor.distlib.version # noqa, pylint: disable=unused-import 120 except ImportError: 121 HAVE_DISTLIB = False 122 123# Try to load tomllib, with a fallback to tomli. 124# HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail 125# outside the venv or before a potential call to ensurepip in checkpip(). 126HAVE_TOMLLIB = True 127try: 128 import tomllib 129except ImportError: 130 try: 131 import tomli as tomllib 132 except ImportError: 133 HAVE_TOMLLIB = False 134 135# Do not add any mandatory dependencies from outside the stdlib: 136# This script *must* be usable standalone! 137 138DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] 139logger = logging.getLogger("mkvenv") 140 141 142def inside_a_venv() -> bool: 143 """Returns True if it is executed inside of a virtual environment.""" 144 return sys.prefix != sys.base_prefix 145 146 147class Ouch(RuntimeError): 148 """An Exception class we can't confuse with a builtin.""" 149 150 151class QemuEnvBuilder(venv.EnvBuilder): 152 """ 153 An extension of venv.EnvBuilder for building QEMU's configure-time venv. 154 155 The primary difference is that it emulates a "nested" virtual 156 environment when invoked from inside of an existing virtual 157 environment by including packages from the parent. Also, 158 "ensurepip" is replaced if possible with just recreating pip's 159 console_scripts inside the virtual environment. 160 161 Parameters for base class init: 162 - system_site_packages: bool = False 163 - clear: bool = False 164 - symlinks: bool = False 165 - upgrade: bool = False 166 - with_pip: bool = False 167 - prompt: Optional[str] = None 168 - upgrade_deps: bool = False (Since 3.9) 169 """ 170 171 def __init__(self, *args: Any, **kwargs: Any) -> None: 172 logger.debug("QemuEnvBuilder.__init__(...)") 173 174 # For nested venv emulation: 175 self.use_parent_packages = False 176 if inside_a_venv(): 177 # Include parent packages only if we're in a venv and 178 # system_site_packages was True. 179 self.use_parent_packages = kwargs.pop( 180 "system_site_packages", False 181 ) 182 # Include system_site_packages only when the parent, 183 # The venv we are currently in, also does so. 184 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES 185 186 # ensurepip is slow: venv creation can be very fast for cases where 187 # we allow the use of system_site_packages. Therefore, ensurepip is 188 # replaced with our own script generation once the virtual environment 189 # is setup. 190 self.want_pip = kwargs.get("with_pip", False) 191 if self.want_pip: 192 if ( 193 kwargs.get("system_site_packages", False) 194 and not need_ensurepip() 195 ): 196 kwargs["with_pip"] = False 197 else: 198 check_ensurepip(suggest_remedy=True) 199 200 super().__init__(*args, **kwargs) 201 202 # Make the context available post-creation: 203 self._context: Optional[SimpleNamespace] = None 204 205 def get_parent_libpath(self) -> Optional[str]: 206 """Return the libpath of the parent venv, if applicable.""" 207 if self.use_parent_packages: 208 return sysconfig.get_path("purelib") 209 return None 210 211 @staticmethod 212 def compute_venv_libpath(context: SimpleNamespace) -> str: 213 """ 214 Compatibility wrapper for context.lib_path for Python < 3.12 215 """ 216 # Python 3.12+, not strictly necessary because it's documented 217 # to be the same as 3.10 code below: 218 if sys.version_info >= (3, 12): 219 return context.lib_path 220 221 # Python 3.10+ 222 if "venv" in sysconfig.get_scheme_names(): 223 lib_path = sysconfig.get_path( 224 "purelib", scheme="venv", vars={"base": context.env_dir} 225 ) 226 assert lib_path is not None 227 return lib_path 228 229 # For Python <= 3.9 we need to hardcode this. Fortunately the 230 # code below was the same in Python 3.6-3.10, so there is only 231 # one case. 232 if sys.platform == "win32": 233 return os.path.join(context.env_dir, "Lib", "site-packages") 234 return os.path.join( 235 context.env_dir, 236 "lib", 237 "python%d.%d" % sys.version_info[:2], 238 "site-packages", 239 ) 240 241 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: 242 logger.debug("ensure_directories(env_dir=%s)", env_dir) 243 self._context = super().ensure_directories(env_dir) 244 return self._context 245 246 def create(self, env_dir: DirType) -> None: 247 logger.debug("create(env_dir=%s)", env_dir) 248 super().create(env_dir) 249 assert self._context is not None 250 self.post_post_setup(self._context) 251 252 def post_post_setup(self, context: SimpleNamespace) -> None: 253 """ 254 The final, final hook. Enter the venv and run commands inside of it. 255 """ 256 if self.use_parent_packages: 257 # We're inside of a venv and we want to include the parent 258 # venv's packages. 259 parent_libpath = self.get_parent_libpath() 260 assert parent_libpath is not None 261 logger.debug("parent_libpath: %s", parent_libpath) 262 263 our_libpath = self.compute_venv_libpath(context) 264 logger.debug("our_libpath: %s", our_libpath) 265 266 pth_file = os.path.join(our_libpath, "nested.pth") 267 with open(pth_file, "w", encoding="UTF-8") as file: 268 file.write(parent_libpath + os.linesep) 269 270 if self.want_pip: 271 args = [ 272 context.env_exe, 273 __file__, 274 "post_init", 275 ] 276 subprocess.run(args, check=True) 277 278 def get_value(self, field: str) -> str: 279 """ 280 Get a string value from the context namespace after a call to build. 281 282 For valid field names, see: 283 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories 284 """ 285 ret = getattr(self._context, field) 286 assert isinstance(ret, str) 287 return ret 288 289 290def need_ensurepip() -> bool: 291 """ 292 Tests for the presence of setuptools and pip. 293 294 :return: `True` if we do not detect both packages. 295 """ 296 # Don't try to actually import them, it's fraught with danger: 297 # https://github.com/pypa/setuptools/issues/2993 298 if find_spec("setuptools") and find_spec("pip"): 299 return False 300 return True 301 302 303def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: 304 """ 305 Check that we have ensurepip. 306 307 Raise a fatal exception with a helpful hint if it isn't available. 308 """ 309 if not find_spec("ensurepip"): 310 msg = ( 311 "Python's ensurepip module is not found.\n" 312 "It's normally part of the Python standard library, " 313 "maybe your distribution packages it separately?\n" 314 "(Debian puts ensurepip in its python3-venv package.)\n" 315 ) 316 if suggest_remedy: 317 msg += ( 318 "Either install ensurepip, or alleviate the need for it in the" 319 " first place by installing pip and setuptools for " 320 f"'{sys.executable}'.\n" 321 ) 322 raise Ouch(prefix + msg) 323 324 # ensurepip uses pyexpat, which can also go missing on us: 325 if not find_spec("pyexpat"): 326 msg = ( 327 "Python's pyexpat module is not found.\n" 328 "It's normally part of the Python standard library, " 329 "maybe your distribution packages it separately?\n" 330 "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n" 331 ) 332 if suggest_remedy: 333 msg += ( 334 "Either install pyexpat, or alleviate the need for it in the " 335 "first place by installing pip and setuptools for " 336 f"'{sys.executable}'.\n" 337 ) 338 raise Ouch(prefix + msg) 339 340 341def make_venv( # pylint: disable=too-many-arguments 342 env_dir: Union[str, Path], 343 system_site_packages: bool = False, 344 clear: bool = True, 345 symlinks: Optional[bool] = None, 346 with_pip: bool = True, 347) -> None: 348 """ 349 Create a venv using `QemuEnvBuilder`. 350 351 This is analogous to the `venv.create` module-level convenience 352 function that is part of the Python stdblib, except it uses 353 `QemuEnvBuilder` instead. 354 355 :param env_dir: The directory to create/install to. 356 :param system_site_packages: 357 Allow inheriting packages from the system installation. 358 :param clear: When True, fully remove any prior venv and files. 359 :param symlinks: 360 Whether to use symlinks to the target interpreter or not. If 361 left unspecified, it will use symlinks except on Windows to 362 match behavior with the "venv" CLI tool. 363 :param with_pip: 364 Whether to install "pip" binaries or not. 365 """ 366 logger.debug( 367 "%s: make_venv(env_dir=%s, system_site_packages=%s, " 368 "clear=%s, symlinks=%s, with_pip=%s)", 369 __file__, 370 str(env_dir), 371 system_site_packages, 372 clear, 373 symlinks, 374 with_pip, 375 ) 376 377 if symlinks is None: 378 # Default behavior of standard venv CLI 379 symlinks = os.name != "nt" 380 381 builder = QemuEnvBuilder( 382 system_site_packages=system_site_packages, 383 clear=clear, 384 symlinks=symlinks, 385 with_pip=with_pip, 386 ) 387 388 style = "non-isolated" if builder.system_site_packages else "isolated" 389 nested = "" 390 if builder.use_parent_packages: 391 nested = f"(with packages from '{builder.get_parent_libpath()}') " 392 print( 393 f"mkvenv: Creating {style} virtual environment" 394 f" {nested}at '{str(env_dir)}'", 395 file=sys.stderr, 396 ) 397 398 try: 399 logger.debug("Invoking builder.create()") 400 try: 401 builder.create(str(env_dir)) 402 except SystemExit as exc: 403 # Some versions of the venv module raise SystemExit; *nasty*! 404 # We want the exception that prompted it. It might be a subprocess 405 # error that has output we *really* want to see. 406 logger.debug("Intercepted SystemExit from EnvBuilder.create()") 407 raise exc.__cause__ or exc.__context__ or exc 408 logger.debug("builder.create() finished") 409 except subprocess.CalledProcessError as exc: 410 logger.error("mkvenv subprocess failed:") 411 logger.error("cmd: %s", exc.cmd) 412 logger.error("returncode: %d", exc.returncode) 413 414 def _stringify(data: Union[str, bytes]) -> str: 415 if isinstance(data, bytes): 416 return data.decode() 417 return data 418 419 lines = [] 420 if exc.stdout: 421 lines.append("========== stdout ==========") 422 lines.append(_stringify(exc.stdout)) 423 lines.append("============================") 424 if exc.stderr: 425 lines.append("========== stderr ==========") 426 lines.append(_stringify(exc.stderr)) 427 lines.append("============================") 428 if lines: 429 logger.error(os.linesep.join(lines)) 430 431 raise Ouch("VENV creation subprocess failed.") from exc 432 433 # print the python executable to stdout for configure. 434 print(builder.get_value("env_exe")) 435 436 437def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: 438 439 def _generator() -> Iterator[str]: 440 for package in packages: 441 try: 442 entry_points: Iterator[EntryPoint] = \ 443 iter(distribution(package).entry_points) 444 except PackageNotFoundError: 445 continue 446 447 # The EntryPoints type is only available in 3.10+, 448 # treat this as a vanilla list and filter it ourselves. 449 entry_points = filter( 450 lambda ep: ep.group == "console_scripts", entry_points 451 ) 452 453 for entry_point in entry_points: 454 yield f"{entry_point.name} = {entry_point.value}" 455 456 return _generator() 457 458 459def generate_console_scripts( 460 packages: Sequence[str], 461 python_path: Optional[str] = None, 462 bin_path: Optional[str] = None, 463) -> None: 464 """ 465 Generate script shims for console_script entry points in @packages. 466 """ 467 if python_path is None: 468 python_path = sys.executable 469 if bin_path is None: 470 bin_path = sysconfig.get_path("scripts") 471 assert bin_path is not None 472 473 logger.debug( 474 "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", 475 packages, 476 python_path, 477 bin_path, 478 ) 479 480 if not packages: 481 return 482 483 maker = distlib.scripts.ScriptMaker(None, bin_path) 484 maker.variants = {""} 485 maker.clobber = False 486 487 for entry_point in _get_entry_points(packages): 488 for filename in maker.make(entry_point): 489 logger.debug("wrote console_script '%s'", filename) 490 491 492def checkpip() -> bool: 493 """ 494 Debian10 has a pip that's broken when used inside of a virtual environment. 495 496 We try to detect and correct that case here. 497 """ 498 try: 499 # pylint: disable=import-outside-toplevel,unused-import,import-error 500 # pylint: disable=redefined-outer-name 501 import pip._internal # type: ignore # noqa: F401 502 503 logger.debug("pip appears to be working correctly.") 504 return False 505 except ModuleNotFoundError as exc: 506 if exc.name == "pip._internal": 507 # Uh, fair enough. They did say "internal". 508 # Let's just assume it's fine. 509 return False 510 logger.warning("pip appears to be malfunctioning: %s", str(exc)) 511 512 check_ensurepip("pip appears to be non-functional, and ") 513 514 logger.debug("Attempting to repair pip ...") 515 subprocess.run( 516 (sys.executable, "-m", "ensurepip"), 517 stdout=subprocess.DEVNULL, 518 check=True, 519 ) 520 logger.debug("Pip is now (hopefully) repaired!") 521 return True 522 523 524def pkgname_from_depspec(dep_spec: str) -> str: 525 """ 526 Parse package name out of a PEP-508 depspec. 527 528 See https://peps.python.org/pep-0508/#names 529 """ 530 match = re.match( 531 r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE 532 ) 533 if not match: 534 raise ValueError( 535 f"dep_spec '{dep_spec}'" 536 " does not appear to contain a valid package name" 537 ) 538 return match.group(0) 539 540 541def _path_is_prefix(prefix: Optional[str], path: str) -> bool: 542 try: 543 return ( 544 prefix is not None and os.path.commonpath([prefix, path]) == prefix 545 ) 546 except ValueError: 547 return False 548 549 550def _is_system_package(dist: Distribution) -> bool: 551 path = str(dist.locate_file(".")) 552 return not ( 553 _path_is_prefix(sysconfig.get_path("purelib"), path) 554 or _path_is_prefix(sysconfig.get_path("platlib"), path) 555 ) 556 557 558def diagnose( 559 dep_spec: str, 560 online: bool, 561 wheels_dir: Optional[Union[str, Path]], 562 prog: Optional[str], 563) -> Tuple[str, bool]: 564 """ 565 Offer a summary to the user as to why a package failed to be installed. 566 567 :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' 568 :param online: Did we allow PyPI access? 569 :param prog: 570 Optionally, a shell program name that can be used as a 571 bellwether to detect if this program is installed elsewhere on 572 the system. This is used to offer advice when a program is 573 detected for a different python version. 574 :param wheels_dir: 575 Optionally, a directory that was searched for vendored packages. 576 """ 577 # pylint: disable=too-many-branches 578 579 # Some errors are not particularly serious 580 bad = False 581 582 pkg_name = pkgname_from_depspec(dep_spec) 583 pkg_version: Optional[str] = None 584 try: 585 pkg_version = version(pkg_name) 586 except PackageNotFoundError: 587 pass 588 589 lines = [] 590 591 if pkg_version: 592 lines.append( 593 f"Python package '{pkg_name}' version '{pkg_version}' was found," 594 " but isn't suitable." 595 ) 596 else: 597 lines.append( 598 f"Python package '{pkg_name}' was not found nor installed." 599 ) 600 601 if wheels_dir: 602 lines.append( 603 "No suitable version found in, or failed to install from" 604 f" '{wheels_dir}'." 605 ) 606 bad = True 607 608 if online: 609 lines.append("A suitable version could not be obtained from PyPI.") 610 bad = True 611 else: 612 lines.append( 613 "mkvenv was configured to operate offline and did not check PyPI." 614 ) 615 616 if prog and not pkg_version: 617 which = shutil.which(prog) 618 if which: 619 if sys.base_prefix in site.PREFIXES: 620 pypath = Path(sys.executable).resolve() 621 lines.append( 622 f"'{prog}' was detected on your system at '{which}', " 623 f"but the Python package '{pkg_name}' was not found by " 624 f"this Python interpreter ('{pypath}'). " 625 f"Typically this means that '{prog}' has been installed " 626 "against a different Python interpreter on your system." 627 ) 628 else: 629 lines.append( 630 f"'{prog}' was detected on your system at '{which}', " 631 "but the build is using an isolated virtual environment." 632 ) 633 bad = True 634 635 lines = [f" • {line}" for line in lines] 636 if bad: 637 lines.insert(0, f"Could not provide build dependency '{dep_spec}':") 638 else: 639 lines.insert(0, f"'{dep_spec}' not found:") 640 return os.linesep.join(lines), bad 641 642 643def pip_install( 644 args: Sequence[str], 645 online: bool = False, 646 wheels_dir: Optional[Union[str, Path]] = None, 647) -> None: 648 """ 649 Use pip to install a package or package(s) as specified in @args. 650 """ 651 loud = bool( 652 os.environ.get("DEBUG") 653 or os.environ.get("GITLAB_CI") 654 or os.environ.get("V") 655 ) 656 657 full_args = [ 658 sys.executable, 659 "-m", 660 "pip", 661 "install", 662 "--disable-pip-version-check", 663 "-v" if loud else "-q", 664 ] 665 if not online: 666 full_args += ["--no-index"] 667 if wheels_dir: 668 full_args += ["--find-links", f"file://{str(wheels_dir)}"] 669 full_args += list(args) 670 subprocess.run( 671 full_args, 672 check=True, 673 ) 674 675 676def _make_version_constraint(info: Dict[str, str], install: bool) -> str: 677 """ 678 Construct the version constraint part of a PEP 508 dependency 679 specification (for example '>=0.61.5') from the accepted and 680 installed keys of the provided dictionary. 681 682 :param info: A dictionary corresponding to a TOML key-value list. 683 :param install: True generates install constraints, False generates 684 presence constraints 685 """ 686 if install and "installed" in info: 687 return "==" + info["installed"] 688 689 dep_spec = info.get("accepted", "") 690 dep_spec = dep_spec.strip() 691 # Double check that they didn't just use a version number 692 if dep_spec and dep_spec[0] not in "!~><=(": 693 raise Ouch( 694 "invalid dependency specifier " + dep_spec + " in dependency file" 695 ) 696 697 return dep_spec 698 699 700def _do_ensure( 701 group: Dict[str, Dict[str, str]], 702 online: bool = False, 703 wheels_dir: Optional[Union[str, Path]] = None, 704) -> Optional[Tuple[str, bool]]: 705 """ 706 Use pip to ensure we have the packages specified in @group. 707 708 If the packages are already installed, do nothing. If online and 709 wheels_dir are both provided, prefer packages found in wheels_dir 710 first before connecting to PyPI. 711 712 :param group: A dictionary of dictionaries, corresponding to a 713 section in a pythondeps.toml file. 714 :param online: If True, fall back to PyPI. 715 :param wheels_dir: If specified, search this path for packages. 716 """ 717 absent = [] 718 present = [] 719 canary = None 720 for name, info in group.items(): 721 constraint = _make_version_constraint(info, False) 722 matcher = distlib.version.LegacyMatcher(name + constraint) 723 print(f"mkvenv: checking for {matcher}", file=sys.stderr) 724 725 dist: Optional[Distribution] = None 726 try: 727 dist = distribution(matcher.name) 728 except PackageNotFoundError: 729 pass 730 731 if ( 732 dist is None 733 # Always pass installed package to pip, so that they can be 734 # updated if the requested version changes 735 or not _is_system_package(dist) 736 or not matcher.match(distlib.version.LegacyVersion(dist.version)) 737 ): 738 absent.append(name + _make_version_constraint(info, True)) 739 if len(absent) == 1: 740 canary = info.get("canary", None) 741 else: 742 logger.info("found %s %s", name, dist.version) 743 present.append(name) 744 745 if present: 746 generate_console_scripts(present) 747 748 if absent: 749 if online or wheels_dir: 750 # Some packages are missing or aren't a suitable version, 751 # install a suitable (possibly vendored) package. 752 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) 753 try: 754 pip_install(args=absent, online=online, wheels_dir=wheels_dir) 755 return None 756 except subprocess.CalledProcessError: 757 pass 758 759 return diagnose( 760 absent[0], 761 online, 762 wheels_dir, 763 canary, 764 ) 765 766 return None 767 768 769def ensure( 770 dep_specs: Sequence[str], 771 online: bool = False, 772 wheels_dir: Optional[Union[str, Path]] = None, 773 prog: Optional[str] = None, 774) -> None: 775 """ 776 Use pip to ensure we have the package specified by @dep_specs. 777 778 If the package is already installed, do nothing. If online and 779 wheels_dir are both provided, prefer packages found in wheels_dir 780 first before connecting to PyPI. 781 782 :param dep_specs: 783 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 784 :param online: If True, fall back to PyPI. 785 :param wheels_dir: If specified, search this path for packages. 786 :param prog: 787 If specified, use this program name for error diagnostics that will 788 be presented to the user. e.g., 'sphinx-build' can be used as a 789 bellwether for the presence of 'sphinx'. 790 """ 791 792 if not HAVE_DISTLIB: 793 raise Ouch("a usable distlib could not be found, please install it") 794 795 # Convert the depspecs to a dictionary, as if they came 796 # from a section in a pythondeps.toml file 797 group: Dict[str, Dict[str, str]] = {} 798 for spec in dep_specs: 799 name = distlib.version.LegacyMatcher(spec).name 800 group[name] = {} 801 802 spec = spec.strip() 803 pos = len(name) 804 ver = spec[pos:].strip() 805 if ver: 806 group[name]["accepted"] = ver 807 808 if prog: 809 group[name]["canary"] = prog 810 prog = None 811 812 result = _do_ensure(group, online, wheels_dir) 813 if result: 814 # Well, that's not good. 815 if result[1]: 816 raise Ouch(result[0]) 817 raise SystemExit(f"\n{result[0]}\n\n") 818 819 820def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: 821 if not HAVE_TOMLLIB: 822 if sys.version_info < (3, 11): 823 raise Ouch("found no usable tomli, please install it") 824 825 raise Ouch( 826 "Python >=3.11 does not have tomllib... what have you done!?" 827 ) 828 829 # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, 830 # Debian bullseye-backports) and v2.0.x 831 with open(file, "r", encoding="ascii") as depfile: 832 contents = depfile.read() 833 return tomllib.loads(contents) # type: ignore 834 835 836def ensure_group( 837 file: str, 838 groups: Sequence[str], 839 online: bool = False, 840 wheels_dir: Optional[Union[str, Path]] = None, 841) -> None: 842 """ 843 Use pip to ensure we have the package specified by @dep_specs. 844 845 If the package is already installed, do nothing. If online and 846 wheels_dir are both provided, prefer packages found in wheels_dir 847 first before connecting to PyPI. 848 849 :param dep_specs: 850 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 851 :param online: If True, fall back to PyPI. 852 :param wheels_dir: If specified, search this path for packages. 853 """ 854 855 if not HAVE_DISTLIB: 856 raise Ouch("found no usable distlib, please install it") 857 858 parsed_deps = _parse_groups(file) 859 860 to_install: Dict[str, Dict[str, str]] = {} 861 for group in groups: 862 try: 863 to_install.update(parsed_deps[group]) 864 except KeyError as exc: 865 raise Ouch(f"group {group} not defined") from exc 866 867 result = _do_ensure(to_install, online, wheels_dir) 868 if result: 869 # Well, that's not good. 870 if result[1]: 871 raise Ouch(result[0]) 872 raise SystemExit(f"\n{result[0]}\n\n") 873 874 875def post_venv_setup() -> None: 876 """ 877 This is intended to be run *inside the venv* after it is created. 878 """ 879 logger.debug("post_venv_setup()") 880 # Test for a broken pip (Debian 10 or derivative?) and fix it if needed 881 if not checkpip(): 882 # Finally, generate a 'pip' script so the venv is usable in a normal 883 # way from the CLI. This only happens when we inherited pip from a 884 # parent/system-site and haven't run ensurepip in some way. 885 generate_console_scripts(["pip"]) 886 887 888def _add_create_subcommand(subparsers: Any) -> None: 889 subparser = subparsers.add_parser("create", help="create a venv") 890 subparser.add_argument( 891 "target", 892 type=str, 893 action="store", 894 help="Target directory to install virtual environment into.", 895 ) 896 897 898def _add_post_init_subcommand(subparsers: Any) -> None: 899 subparsers.add_parser("post_init", help="post-venv initialization") 900 901 902def _add_ensuregroup_subcommand(subparsers: Any) -> None: 903 subparser = subparsers.add_parser( 904 "ensuregroup", 905 help="Ensure that the specified package group is installed.", 906 ) 907 subparser.add_argument( 908 "--online", 909 action="store_true", 910 help="Install packages from PyPI, if necessary.", 911 ) 912 subparser.add_argument( 913 "--dir", 914 type=str, 915 action="store", 916 help="Path to vendored packages where we may install from.", 917 ) 918 subparser.add_argument( 919 "file", 920 type=str, 921 action="store", 922 help=("Path to a TOML file describing package groups"), 923 ) 924 subparser.add_argument( 925 "group", 926 type=str, 927 action="store", 928 help="One or more package group names", 929 nargs="+", 930 ) 931 932 933def _add_ensure_subcommand(subparsers: Any) -> None: 934 subparser = subparsers.add_parser( 935 "ensure", help="Ensure that the specified package is installed." 936 ) 937 subparser.add_argument( 938 "--online", 939 action="store_true", 940 help="Install packages from PyPI, if necessary.", 941 ) 942 subparser.add_argument( 943 "--dir", 944 type=str, 945 action="store", 946 help="Path to vendored packages where we may install from.", 947 ) 948 subparser.add_argument( 949 "--diagnose", 950 type=str, 951 action="store", 952 help=( 953 "Name of a shell utility to use for " 954 "diagnostics if this command fails." 955 ), 956 ) 957 subparser.add_argument( 958 "dep_specs", 959 type=str, 960 action="store", 961 help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", 962 nargs="+", 963 ) 964 965 966def main() -> int: 967 """CLI interface to make_qemu_venv. See module docstring.""" 968 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): 969 # You're welcome. 970 logging.basicConfig(level=logging.DEBUG) 971 else: 972 if os.environ.get("V"): 973 logging.basicConfig(level=logging.INFO) 974 975 parser = argparse.ArgumentParser( 976 prog="mkvenv", 977 description="QEMU pyvenv bootstrapping utility", 978 ) 979 subparsers = parser.add_subparsers( 980 title="Commands", 981 dest="command", 982 required=True, 983 metavar="command", 984 help="Description", 985 ) 986 987 _add_create_subcommand(subparsers) 988 _add_post_init_subcommand(subparsers) 989 _add_ensure_subcommand(subparsers) 990 _add_ensuregroup_subcommand(subparsers) 991 992 args = parser.parse_args() 993 try: 994 if args.command == "create": 995 make_venv( 996 args.target, 997 system_site_packages=True, 998 clear=True, 999 ) 1000 if args.command == "post_init": 1001 post_venv_setup() 1002 if args.command == "ensure": 1003 ensure( 1004 dep_specs=args.dep_specs, 1005 online=args.online, 1006 wheels_dir=args.dir, 1007 prog=args.diagnose, 1008 ) 1009 if args.command == "ensuregroup": 1010 ensure_group( 1011 file=args.file, 1012 groups=args.group, 1013 online=args.online, 1014 wheels_dir=args.dir, 1015 ) 1016 logger.debug("mkvenv.py %s: exiting", args.command) 1017 except Ouch as exc: 1018 print("\n*** Ouch! ***\n", file=sys.stderr) 1019 print(str(exc), "\n\n", file=sys.stderr) 1020 return 1 1021 except SystemExit: 1022 raise 1023 except: # pylint: disable=bare-except 1024 logger.exception("mkvenv did not complete successfully:") 1025 return 2 1026 return 0 1027 1028 1029if __name__ == "__main__": 1030 sys.exit(main()) 1031