1""" 2mkvenv - QEMU pyvenv bootstrapping utility 3 4usage: mkvenv [-h] command ... 5 6QEMU pyvenv bootstrapping utility 7 8options: 9 -h, --help show this help message and exit 10 11Commands: 12 command Description 13 create create a venv 14 ensure Ensure that the specified package is installed. 15 16-------------------------------------------------- 17 18usage: mkvenv create [-h] target 19 20positional arguments: 21 target Target directory to install virtual environment into. 22 23options: 24 -h, --help show this help message and exit 25 26-------------------------------------------------- 27 28usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... 29 30positional arguments: 31 dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' 32 33options: 34 -h, --help show this help message and exit 35 --online Install packages from PyPI, if necessary. 36 --dir DIR Path to vendored packages where we may install from. 37 38""" 39 40# Copyright (C) 2022-2023 Red Hat, Inc. 41# 42# Authors: 43# John Snow <jsnow@redhat.com> 44# Paolo Bonzini <pbonzini@redhat.com> 45# 46# This work is licensed under the terms of the GNU GPL, version 2 or 47# later. See the COPYING file in the top-level directory. 48 49import argparse 50from importlib.util import find_spec 51import logging 52import os 53from pathlib import Path 54import site 55import subprocess 56import sys 57import sysconfig 58from types import SimpleNamespace 59from typing import ( 60 Any, 61 Optional, 62 Sequence, 63 Union, 64) 65import venv 66import warnings 67 68import distlib.database 69import distlib.version 70 71 72# Do not add any mandatory dependencies from outside the stdlib: 73# This script *must* be usable standalone! 74 75DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] 76logger = logging.getLogger("mkvenv") 77 78 79def inside_a_venv() -> bool: 80 """Returns True if it is executed inside of a virtual environment.""" 81 return sys.prefix != sys.base_prefix 82 83 84class Ouch(RuntimeError): 85 """An Exception class we can't confuse with a builtin.""" 86 87 88class QemuEnvBuilder(venv.EnvBuilder): 89 """ 90 An extension of venv.EnvBuilder for building QEMU's configure-time venv. 91 92 The primary difference is that it emulates a "nested" virtual 93 environment when invoked from inside of an existing virtual 94 environment by including packages from the parent. 95 96 Parameters for base class init: 97 - system_site_packages: bool = False 98 - clear: bool = False 99 - symlinks: bool = False 100 - upgrade: bool = False 101 - with_pip: bool = False 102 - prompt: Optional[str] = None 103 - upgrade_deps: bool = False (Since 3.9) 104 """ 105 106 def __init__(self, *args: Any, **kwargs: Any) -> None: 107 logger.debug("QemuEnvBuilder.__init__(...)") 108 109 # For nested venv emulation: 110 self.use_parent_packages = False 111 if inside_a_venv(): 112 # Include parent packages only if we're in a venv and 113 # system_site_packages was True. 114 self.use_parent_packages = kwargs.pop( 115 "system_site_packages", False 116 ) 117 # Include system_site_packages only when the parent, 118 # The venv we are currently in, also does so. 119 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES 120 121 if kwargs.get("with_pip", False): 122 check_ensurepip() 123 124 super().__init__(*args, **kwargs) 125 126 # Make the context available post-creation: 127 self._context: Optional[SimpleNamespace] = None 128 129 def get_parent_libpath(self) -> Optional[str]: 130 """Return the libpath of the parent venv, if applicable.""" 131 if self.use_parent_packages: 132 return sysconfig.get_path("purelib") 133 return None 134 135 @staticmethod 136 def compute_venv_libpath(context: SimpleNamespace) -> str: 137 """ 138 Compatibility wrapper for context.lib_path for Python < 3.12 139 """ 140 # Python 3.12+, not strictly necessary because it's documented 141 # to be the same as 3.10 code below: 142 if sys.version_info >= (3, 12): 143 return context.lib_path 144 145 # Python 3.10+ 146 if "venv" in sysconfig.get_scheme_names(): 147 lib_path = sysconfig.get_path( 148 "purelib", scheme="venv", vars={"base": context.env_dir} 149 ) 150 assert lib_path is not None 151 return lib_path 152 153 # For Python <= 3.9 we need to hardcode this. Fortunately the 154 # code below was the same in Python 3.6-3.10, so there is only 155 # one case. 156 if sys.platform == "win32": 157 return os.path.join(context.env_dir, "Lib", "site-packages") 158 return os.path.join( 159 context.env_dir, 160 "lib", 161 "python%d.%d" % sys.version_info[:2], 162 "site-packages", 163 ) 164 165 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: 166 logger.debug("ensure_directories(env_dir=%s)", env_dir) 167 self._context = super().ensure_directories(env_dir) 168 return self._context 169 170 def create(self, env_dir: DirType) -> None: 171 logger.debug("create(env_dir=%s)", env_dir) 172 super().create(env_dir) 173 assert self._context is not None 174 self.post_post_setup(self._context) 175 176 def post_post_setup(self, context: SimpleNamespace) -> None: 177 """ 178 The final, final hook. Enter the venv and run commands inside of it. 179 """ 180 if self.use_parent_packages: 181 # We're inside of a venv and we want to include the parent 182 # venv's packages. 183 parent_libpath = self.get_parent_libpath() 184 assert parent_libpath is not None 185 logger.debug("parent_libpath: %s", parent_libpath) 186 187 our_libpath = self.compute_venv_libpath(context) 188 logger.debug("our_libpath: %s", our_libpath) 189 190 pth_file = os.path.join(our_libpath, "nested.pth") 191 with open(pth_file, "w", encoding="UTF-8") as file: 192 file.write(parent_libpath + os.linesep) 193 194 def get_value(self, field: str) -> str: 195 """ 196 Get a string value from the context namespace after a call to build. 197 198 For valid field names, see: 199 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories 200 """ 201 ret = getattr(self._context, field) 202 assert isinstance(ret, str) 203 return ret 204 205 206def check_ensurepip() -> None: 207 """ 208 Check that we have ensurepip. 209 210 Raise a fatal exception with a helpful hint if it isn't available. 211 """ 212 if not find_spec("ensurepip"): 213 msg = ( 214 "Python's ensurepip module is not found.\n" 215 "It's normally part of the Python standard library, " 216 "maybe your distribution packages it separately?\n" 217 "Either install ensurepip, or alleviate the need for it in the " 218 "first place by installing pip and setuptools for " 219 f"'{sys.executable}'.\n" 220 "(Hint: Debian puts ensurepip in its python3-venv package.)" 221 ) 222 raise Ouch(msg) 223 224 # ensurepip uses pyexpat, which can also go missing on us: 225 if not find_spec("pyexpat"): 226 msg = ( 227 "Python's pyexpat module is not found.\n" 228 "It's normally part of the Python standard library, " 229 "maybe your distribution packages it separately?\n" 230 "Either install pyexpat, or alleviate the need for it in the " 231 "first place by installing pip and setuptools for " 232 f"'{sys.executable}'.\n\n" 233 "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" 234 ) 235 raise Ouch(msg) 236 237 238def make_venv( # pylint: disable=too-many-arguments 239 env_dir: Union[str, Path], 240 system_site_packages: bool = False, 241 clear: bool = True, 242 symlinks: Optional[bool] = None, 243 with_pip: bool = True, 244) -> None: 245 """ 246 Create a venv using `QemuEnvBuilder`. 247 248 This is analogous to the `venv.create` module-level convenience 249 function that is part of the Python stdblib, except it uses 250 `QemuEnvBuilder` instead. 251 252 :param env_dir: The directory to create/install to. 253 :param system_site_packages: 254 Allow inheriting packages from the system installation. 255 :param clear: When True, fully remove any prior venv and files. 256 :param symlinks: 257 Whether to use symlinks to the target interpreter or not. If 258 left unspecified, it will use symlinks except on Windows to 259 match behavior with the "venv" CLI tool. 260 :param with_pip: 261 Whether to install "pip" binaries or not. 262 """ 263 logger.debug( 264 "%s: make_venv(env_dir=%s, system_site_packages=%s, " 265 "clear=%s, symlinks=%s, with_pip=%s)", 266 __file__, 267 str(env_dir), 268 system_site_packages, 269 clear, 270 symlinks, 271 with_pip, 272 ) 273 274 if symlinks is None: 275 # Default behavior of standard venv CLI 276 symlinks = os.name != "nt" 277 278 builder = QemuEnvBuilder( 279 system_site_packages=system_site_packages, 280 clear=clear, 281 symlinks=symlinks, 282 with_pip=with_pip, 283 ) 284 285 style = "non-isolated" if builder.system_site_packages else "isolated" 286 nested = "" 287 if builder.use_parent_packages: 288 nested = f"(with packages from '{builder.get_parent_libpath()}') " 289 print( 290 f"mkvenv: Creating {style} virtual environment" 291 f" {nested}at '{str(env_dir)}'", 292 file=sys.stderr, 293 ) 294 295 try: 296 logger.debug("Invoking builder.create()") 297 try: 298 builder.create(str(env_dir)) 299 except SystemExit as exc: 300 # Some versions of the venv module raise SystemExit; *nasty*! 301 # We want the exception that prompted it. It might be a subprocess 302 # error that has output we *really* want to see. 303 logger.debug("Intercepted SystemExit from EnvBuilder.create()") 304 raise exc.__cause__ or exc.__context__ or exc 305 logger.debug("builder.create() finished") 306 except subprocess.CalledProcessError as exc: 307 logger.error("mkvenv subprocess failed:") 308 logger.error("cmd: %s", exc.cmd) 309 logger.error("returncode: %d", exc.returncode) 310 311 def _stringify(data: Union[str, bytes]) -> str: 312 if isinstance(data, bytes): 313 return data.decode() 314 return data 315 316 lines = [] 317 if exc.stdout: 318 lines.append("========== stdout ==========") 319 lines.append(_stringify(exc.stdout)) 320 lines.append("============================") 321 if exc.stderr: 322 lines.append("========== stderr ==========") 323 lines.append(_stringify(exc.stderr)) 324 lines.append("============================") 325 if lines: 326 logger.error(os.linesep.join(lines)) 327 328 raise Ouch("VENV creation subprocess failed.") from exc 329 330 # print the python executable to stdout for configure. 331 print(builder.get_value("env_exe")) 332 333 334def pip_install( 335 args: Sequence[str], 336 online: bool = False, 337 wheels_dir: Optional[Union[str, Path]] = None, 338) -> None: 339 """ 340 Use pip to install a package or package(s) as specified in @args. 341 """ 342 loud = bool( 343 os.environ.get("DEBUG") 344 or os.environ.get("GITLAB_CI") 345 or os.environ.get("V") 346 ) 347 348 full_args = [ 349 sys.executable, 350 "-m", 351 "pip", 352 "install", 353 "--disable-pip-version-check", 354 "-v" if loud else "-q", 355 ] 356 if not online: 357 full_args += ["--no-index"] 358 if wheels_dir: 359 full_args += ["--find-links", f"file://{str(wheels_dir)}"] 360 full_args += list(args) 361 subprocess.run( 362 full_args, 363 check=True, 364 ) 365 366 367def ensure( 368 dep_specs: Sequence[str], 369 online: bool = False, 370 wheels_dir: Optional[Union[str, Path]] = None, 371) -> None: 372 """ 373 Use pip to ensure we have the package specified by @dep_specs. 374 375 If the package is already installed, do nothing. If online and 376 wheels_dir are both provided, prefer packages found in wheels_dir 377 first before connecting to PyPI. 378 379 :param dep_specs: 380 PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. 381 :param online: If True, fall back to PyPI. 382 :param wheels_dir: If specified, search this path for packages. 383 """ 384 with warnings.catch_warnings(): 385 warnings.filterwarnings( 386 "ignore", category=UserWarning, module="distlib" 387 ) 388 dist_path = distlib.database.DistributionPath(include_egg=True) 389 absent = [] 390 for spec in dep_specs: 391 matcher = distlib.version.LegacyMatcher(spec) 392 dist = dist_path.get_distribution(matcher.name) 393 if dist is None or not matcher.match(dist.version): 394 absent.append(spec) 395 else: 396 logger.info("found %s", dist) 397 398 if absent: 399 # Some packages are missing or aren't a suitable version, 400 # install a suitable (possibly vendored) package. 401 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) 402 pip_install(args=absent, online=online, wheels_dir=wheels_dir) 403 404 405def _add_create_subcommand(subparsers: Any) -> None: 406 subparser = subparsers.add_parser("create", help="create a venv") 407 subparser.add_argument( 408 "target", 409 type=str, 410 action="store", 411 help="Target directory to install virtual environment into.", 412 ) 413 414 415def _add_ensure_subcommand(subparsers: Any) -> None: 416 subparser = subparsers.add_parser( 417 "ensure", help="Ensure that the specified package is installed." 418 ) 419 subparser.add_argument( 420 "--online", 421 action="store_true", 422 help="Install packages from PyPI, if necessary.", 423 ) 424 subparser.add_argument( 425 "--dir", 426 type=str, 427 action="store", 428 help="Path to vendored packages where we may install from.", 429 ) 430 subparser.add_argument( 431 "dep_specs", 432 type=str, 433 action="store", 434 help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", 435 nargs="+", 436 ) 437 438 439def main() -> int: 440 """CLI interface to make_qemu_venv. See module docstring.""" 441 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): 442 # You're welcome. 443 logging.basicConfig(level=logging.DEBUG) 444 else: 445 if os.environ.get("V"): 446 logging.basicConfig(level=logging.INFO) 447 448 # These are incredibly noisy even for V=1 449 logging.getLogger("distlib.metadata").addFilter(lambda record: False) 450 logging.getLogger("distlib.database").addFilter(lambda record: False) 451 452 parser = argparse.ArgumentParser( 453 prog="mkvenv", 454 description="QEMU pyvenv bootstrapping utility", 455 ) 456 subparsers = parser.add_subparsers( 457 title="Commands", 458 dest="command", 459 metavar="command", 460 help="Description", 461 ) 462 463 _add_create_subcommand(subparsers) 464 _add_ensure_subcommand(subparsers) 465 466 args = parser.parse_args() 467 try: 468 if args.command == "create": 469 make_venv( 470 args.target, 471 system_site_packages=True, 472 clear=True, 473 ) 474 if args.command == "ensure": 475 ensure( 476 dep_specs=args.dep_specs, 477 online=args.online, 478 wheels_dir=args.dir, 479 ) 480 logger.debug("mkvenv.py %s: exiting", args.command) 481 except Ouch as exc: 482 print("\n*** Ouch! ***\n", file=sys.stderr) 483 print(str(exc), "\n\n", file=sys.stderr) 484 return 1 485 except SystemExit: 486 raise 487 except: # pylint: disable=bare-except 488 logger.exception("mkvenv did not complete successfully:") 489 return 2 490 return 0 491 492 493if __name__ == "__main__": 494 sys.exit(main()) 495