1#!/usr/bin/env python3 2 3# Handle running OE images standalone with QEMU 4# 5# Copyright (C) 2006-2011 Linux Foundation 6# Copyright (c) 2016 Wind River Systems, Inc. 7# 8# SPDX-License-Identifier: GPL-2.0-only 9# 10 11import os 12import sys 13import logging 14import subprocess 15import re 16import fcntl 17import shutil 18import glob 19import configparser 20import signal 21import time 22 23class RunQemuError(Exception): 24 """Custom exception to raise on known errors.""" 25 pass 26 27class OEPathError(RunQemuError): 28 """Custom Exception to give better guidance on missing binaries""" 29 def __init__(self, message): 30 super().__init__("In order for this script to dynamically infer paths\n \ 31kernels or filesystem images, you either need bitbake in your PATH\n \ 32or to source oe-init-build-env before running this script.\n\n \ 33Dynamic path inference can be avoided by passing a *.qemuboot.conf to\n \ 34runqemu, i.e. `runqemu /path/to/my-image-name.qemuboot.conf`\n\n %s" % message) 35 36 37def create_logger(): 38 logger = logging.getLogger('runqemu') 39 logger.setLevel(logging.INFO) 40 41 # create console handler and set level to debug 42 ch = logging.StreamHandler() 43 ch.setLevel(logging.DEBUG) 44 45 # create formatter 46 formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s') 47 48 # add formatter to ch 49 ch.setFormatter(formatter) 50 51 # add ch to logger 52 logger.addHandler(ch) 53 54 return logger 55 56logger = create_logger() 57 58def print_usage(): 59 print(""" 60Usage: you can run this script with any valid combination 61of the following environment variables (in any order): 62 KERNEL - the kernel image file to use 63 BIOS - the bios image file to use 64 ROOTFS - the rootfs image file or nfsroot directory to use 65 DEVICE_TREE - the device tree blob to use 66 MACHINE - the machine name (optional, autodetected from KERNEL filename if unspecified) 67 Simplified QEMU command-line options can be passed with: 68 nographic - disable video console 69 novga - Disable VGA emulation completely 70 sdl - choose the SDL UI frontend 71 gtk - choose the Gtk UI frontend 72 gl - enable virgl-based GL acceleration (also needs gtk or sdl options) 73 gl-es - enable virgl-based GL acceleration, using OpenGL ES (also needs gtk or sdl options) 74 egl-headless - enable headless EGL output; use vnc (via publicvnc option) or spice to see it 75 (hint: if /dev/dri/renderD* is absent due to lack of suitable GPU, 'modprobe vgem' will create 76 one suitable for mesa llvmpipe software renderer) 77 serial - enable a serial console on /dev/ttyS0 78 serialstdio - enable a serial console on the console (regardless of graphics mode) 79 slirp - enable user networking, no root privilege is required 80 snapshot - don't write changes back to images 81 kvm - enable KVM when running x86/x86_64 (VT-capable CPU required) 82 kvm-vhost - enable KVM with vhost when running x86/x86_64 (VT-capable CPU required) 83 publicvnc - enable a VNC server open to all hosts 84 audio - enable audio 85 [*/]ovmf* - OVMF firmware file or base name for booting with UEFI 86 tcpserial=<port> - specify tcp serial port number 87 qemuparams=<xyz> - specify custom parameters to QEMU 88 bootparams=<xyz> - specify custom kernel parameters during boot 89 help, -h, --help: print this text 90 -d, --debug: Enable debug output 91 -q, --quiet: Hide most output except error messages 92 93Examples: 94 runqemu 95 runqemu qemuarm 96 runqemu tmp/deploy/images/qemuarm 97 runqemu tmp/deploy/images/qemux86/<qemuboot.conf> 98 runqemu qemux86-64 core-image-sato ext4 99 runqemu qemux86-64 wic-image-minimal wic 100 runqemu path/to/bzImage-qemux86.bin path/to/nfsrootdir/ serial 101 runqemu qemux86 iso/hddimg/wic.vmdk/wic.vhd/wic.vhdx/wic.qcow2/wic.vdi/ramfs/cpio.gz... 102 runqemu qemux86 qemuparams="-m 256" 103 runqemu qemux86 bootparams="psplash=false" 104 runqemu path/to/<image>-<machine>.wic 105 runqemu path/to/<image>-<machine>.wic.vmdk 106 runqemu path/to/<image>-<machine>.wic.vhdx 107 runqemu path/to/<image>-<machine>.wic.vhd 108""") 109 110def check_tun(): 111 """Check /dev/net/tun""" 112 dev_tun = '/dev/net/tun' 113 if not os.path.exists(dev_tun): 114 raise RunQemuError("TUN control device %s is unavailable; you may need to enable TUN (e.g. sudo modprobe tun)" % dev_tun) 115 116 if not os.access(dev_tun, os.W_OK): 117 raise RunQemuError("TUN control device %s is not writable, please fix (e.g. sudo chmod 666 %s)" % (dev_tun, dev_tun)) 118 119def get_first_file(cmds): 120 """Return first file found in wildcard cmds""" 121 for cmd in cmds: 122 all_files = glob.glob(cmd) 123 if all_files: 124 for f in all_files: 125 if not os.path.isdir(f): 126 return f 127 return '' 128 129class BaseConfig(object): 130 def __init__(self): 131 # The self.d saved vars from self.set(), part of them are from qemuboot.conf 132 self.d = {'QB_KERNEL_ROOT': '/dev/vda'} 133 134 # Supported env vars, add it here if a var can be got from env, 135 # and don't use os.getenv in the code. 136 self.env_vars = ('MACHINE', 137 'ROOTFS', 138 'KERNEL', 139 'BIOS', 140 'DEVICE_TREE', 141 'DEPLOY_DIR_IMAGE', 142 'OE_TMPDIR', 143 'OECORE_NATIVE_SYSROOT', 144 'MULTICONFIG', 145 'SERIAL_CONSOLES', 146 ) 147 148 self.qemu_opt = '' 149 self.qemu_opt_script = '' 150 self.qemuparams = '' 151 self.nfs_server = '' 152 self.rootfs = '' 153 # File name(s) of a OVMF firmware file or variable store, 154 # to be added with -drive if=pflash. 155 # Found in the same places as the rootfs, with or without one of 156 # these suffices: qcow2, bin. 157 self.ovmf_bios = [] 158 # When enrolling default Secure Boot keys, the hypervisor 159 # must provide the Platform Key and the first Key Exchange Key 160 # certificate in the Type 11 SMBIOS table. 161 self.ovmf_secboot_pkkek1 = '' 162 self.qemuboot = '' 163 self.qbconfload = False 164 self.kernel = '' 165 self.bios = '' 166 self.kernel_cmdline = '' 167 self.kernel_cmdline_script = '' 168 self.bootparams = '' 169 self.dtb = '' 170 self.fstype = '' 171 self.kvm_enabled = False 172 self.vhost_enabled = False 173 self.slirp_enabled = False 174 self.net_bridge = None 175 self.nfs_instance = 0 176 self.nfs_running = False 177 self.serialconsole = False 178 self.serialstdio = False 179 self.nographic = False 180 self.sdl = False 181 self.gtk = False 182 self.gl = False 183 self.gl_es = False 184 self.egl_headless = False 185 self.publicvnc = False 186 self.novga = False 187 self.cleantap = False 188 self.saved_stty = '' 189 self.audio_enabled = False 190 self.tcpserial_portnum = '' 191 self.taplock = '' 192 self.taplock_descriptor = None 193 self.portlocks = {} 194 self.bitbake_e = '' 195 self.snapshot = False 196 self.wictypes = ('wic', 'wic.vmdk', 'wic.qcow2', 'wic.vdi', "wic.vhd", "wic.vhdx") 197 self.fstypes = ('ext2', 'ext3', 'ext4', 'jffs2', 'nfs', 'btrfs', 198 'cpio.gz', 'cpio', 'ramfs', 'tar.bz2', 'tar.gz') 199 self.vmtypes = ('hddimg', 'iso') 200 self.fsinfo = {} 201 self.network_device = "-device e1000,netdev=net0,mac=@MAC@" 202 self.cmdline_ip_slirp = "ip=dhcp" 203 self.cmdline_ip_tap = "ip=192.168.7.@CLIENT@::192.168.7.@GATEWAY@:255.255.255.0::eth0:off:8.8.8.8" 204 # Use different mac section for tap and slirp to avoid 205 # conflicts, e.g., when one is running with tap, the other is 206 # running with slirp. 207 # The last section is dynamic, which is for avoiding conflicts, 208 # when multiple qemus are running, e.g., when multiple tap or 209 # slirp qemus are running. 210 self.mac_tap = "52:54:00:12:34:" 211 self.mac_slirp = "52:54:00:12:35:" 212 # pid of the actual qemu process 213 self.qemupid = None 214 # avoid cleanup twice 215 self.cleaned = False 216 # Files to cleanup after run 217 self.cleanup_files = [] 218 219 def acquire_taplock(self, error=True): 220 logger.debug("Acquiring lockfile %s..." % self.taplock) 221 try: 222 self.taplock_descriptor = open(self.taplock, 'w') 223 fcntl.flock(self.taplock_descriptor, fcntl.LOCK_EX|fcntl.LOCK_NB) 224 except Exception as e: 225 msg = "Acquiring lockfile %s failed: %s" % (self.taplock, e) 226 if error: 227 logger.error(msg) 228 else: 229 logger.info(msg) 230 if self.taplock_descriptor: 231 self.taplock_descriptor.close() 232 self.taplock_descriptor = None 233 return False 234 return True 235 236 def release_taplock(self): 237 if self.taplock_descriptor: 238 logger.debug("Releasing lockfile for tap device '%s'" % self.tap) 239 # We pass the fd to the qemu process and if we unlock here, it would unlock for 240 # that too. Therefore don't unlock, just close 241 # fcntl.flock(self.taplock_descriptor, fcntl.LOCK_UN) 242 self.taplock_descriptor.close() 243 # Removing the file is a potential race, don't do that either 244 # os.remove(self.taplock) 245 self.taplock_descriptor = None 246 247 def check_free_port(self, host, port, lockdir): 248 """ Check whether the port is free or not """ 249 import socket 250 from contextlib import closing 251 252 lockfile = os.path.join(lockdir, str(port) + '.lock') 253 if self.acquire_portlock(lockfile): 254 with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: 255 if sock.connect_ex((host, port)) == 0: 256 # Port is open, so not free 257 self.release_portlock(lockfile) 258 return False 259 else: 260 # Port is not open, so free 261 return True 262 else: 263 return False 264 265 def acquire_portlock(self, lockfile): 266 logger.debug("Acquiring lockfile %s..." % lockfile) 267 try: 268 portlock_descriptor = open(lockfile, 'w') 269 self.portlocks.update({lockfile: portlock_descriptor}) 270 fcntl.flock(self.portlocks[lockfile], fcntl.LOCK_EX|fcntl.LOCK_NB) 271 except Exception as e: 272 msg = "Acquiring lockfile %s failed: %s" % (lockfile, e) 273 logger.info(msg) 274 if lockfile in self.portlocks.keys() and self.portlocks[lockfile]: 275 self.portlocks[lockfile].close() 276 del self.portlocks[lockfile] 277 return False 278 return True 279 280 def release_portlock(self, lockfile=None): 281 if lockfile != None: 282 logger.debug("Releasing lockfile '%s'" % lockfile) 283 # We pass the fd to the qemu process and if we unlock here, it would unlock for 284 # that too. Therefore don't unlock, just close 285 # fcntl.flock(self.portlocks[lockfile], fcntl.LOCK_UN) 286 self.portlocks[lockfile].close() 287 # Removing the file is a potential race, don't do that either 288 # os.remove(lockfile) 289 del self.portlocks[lockfile] 290 elif len(self.portlocks): 291 for lockfile, descriptor in self.portlocks.items(): 292 logger.debug("Releasing lockfile '%s'" % lockfile) 293 # We pass the fd to the qemu process and if we unlock here, it would unlock for 294 # that too. Therefore don't unlock, just close 295 # fcntl.flock(descriptor, fcntl.LOCK_UN) 296 descriptor.close() 297 # Removing the file is a potential race, don't do that either 298 # os.remove(lockfile) 299 self.portlocks = {} 300 301 def get(self, key): 302 if key in self.d: 303 return self.d.get(key) 304 elif os.getenv(key): 305 return os.getenv(key) 306 else: 307 return '' 308 309 def set(self, key, value): 310 self.d[key] = value 311 312 def is_deploy_dir_image(self, p): 313 if os.path.isdir(p): 314 if not re.search('.qemuboot.conf$', '\n'.join(os.listdir(p)), re.M): 315 logger.debug("Can't find required *.qemuboot.conf in %s" % p) 316 return False 317 if not any(map(lambda name: '-image-' in name, os.listdir(p))): 318 logger.debug("Can't find *-image-* in %s" % p) 319 return False 320 return True 321 else: 322 return False 323 324 def check_arg_fstype(self, fst): 325 """Check and set FSTYPE""" 326 if fst not in self.fstypes + self.vmtypes + self.wictypes: 327 logger.warning("Maybe unsupported FSTYPE: %s" % fst) 328 if not self.fstype or self.fstype == fst: 329 if fst == 'ramfs': 330 fst = 'cpio.gz' 331 if fst in ('tar.bz2', 'tar.gz'): 332 fst = 'nfs' 333 self.fstype = fst 334 else: 335 raise RunQemuError("Conflicting: FSTYPE %s and %s" % (self.fstype, fst)) 336 337 def set_machine_deploy_dir(self, machine, deploy_dir_image): 338 """Set MACHINE and DEPLOY_DIR_IMAGE""" 339 logger.debug('MACHINE: %s' % machine) 340 self.set("MACHINE", machine) 341 logger.debug('DEPLOY_DIR_IMAGE: %s' % deploy_dir_image) 342 self.set("DEPLOY_DIR_IMAGE", deploy_dir_image) 343 344 def check_arg_nfs(self, p): 345 if os.path.isdir(p): 346 self.rootfs = p 347 else: 348 m = re.match('(.*):(.*)', p) 349 self.nfs_server = m.group(1) 350 self.rootfs = m.group(2) 351 self.check_arg_fstype('nfs') 352 353 def check_arg_path(self, p): 354 """ 355 - Check whether it is <image>.qemuboot.conf or contains <image>.qemuboot.conf 356 - Check whether it is a kernel file 357 - Check whether it is an image file 358 - Check whether it is an NFS dir 359 - Check whether it is an OVMF flash file 360 """ 361 if p.endswith('.qemuboot.conf'): 362 self.qemuboot = p 363 self.qbconfload = True 364 elif re.search('\.bin$', p) or re.search('bzImage', p) or \ 365 re.search('zImage', p) or re.search('vmlinux', p) or \ 366 re.search('fitImage', p) or re.search('uImage', p): 367 self.kernel = p 368 elif os.path.exists(p) and (not os.path.isdir(p)) and '-image-' in os.path.basename(p): 369 self.rootfs = p 370 # Check filename against self.fstypes can handle <file>.cpio.gz, 371 # otherwise, its type would be "gz", which is incorrect. 372 fst = "" 373 for t in self.fstypes: 374 if p.endswith(t): 375 fst = t 376 break 377 if not fst: 378 m = re.search('.*\.(.*)$', self.rootfs) 379 if m: 380 fst = m.group(1) 381 if fst: 382 self.check_arg_fstype(fst) 383 qb = re.sub('\.' + fst + "$", '', self.rootfs) 384 qb = '%s%s' % (re.sub('\.rootfs$', '', qb), '.qemuboot.conf') 385 if os.path.exists(qb): 386 self.qemuboot = qb 387 self.qbconfload = True 388 else: 389 logger.warning("%s doesn't exist" % qb) 390 else: 391 raise RunQemuError("Can't find FSTYPE from: %s" % p) 392 393 elif os.path.isdir(p) or re.search(':', p) and re.search('/', p): 394 if self.is_deploy_dir_image(p): 395 logger.debug('DEPLOY_DIR_IMAGE: %s' % p) 396 self.set("DEPLOY_DIR_IMAGE", p) 397 else: 398 logger.debug("Assuming %s is an nfs rootfs" % p) 399 self.check_arg_nfs(p) 400 elif os.path.basename(p).startswith('ovmf'): 401 self.ovmf_bios.append(p) 402 else: 403 raise RunQemuError("Unknown path arg %s" % p) 404 405 def check_arg_machine(self, arg): 406 """Check whether it is a machine""" 407 if self.get('MACHINE') == arg: 408 return 409 elif self.get('MACHINE') and self.get('MACHINE') != arg: 410 raise RunQemuError("Maybe conflicted MACHINE: %s vs %s" % (self.get('MACHINE'), arg)) 411 elif re.search('/', arg): 412 raise RunQemuError("Unknown arg: %s" % arg) 413 414 logger.debug('Assuming MACHINE = %s' % arg) 415 416 # if we're running under testimage, or similarly as a child 417 # of an existing bitbake invocation, we can't invoke bitbake 418 # to validate the MACHINE setting and must assume it's correct... 419 # FIXME: testimage.bbclass exports these two variables into env, 420 # are there other scenarios in which we need to support being 421 # invoked by bitbake? 422 deploy = self.get('DEPLOY_DIR_IMAGE') 423 bbchild = deploy and self.get('OE_TMPDIR') 424 if bbchild: 425 self.set_machine_deploy_dir(arg, deploy) 426 return 427 # also check whether we're running under a sourced toolchain 428 # environment file 429 if self.get('OECORE_NATIVE_SYSROOT'): 430 self.set("MACHINE", arg) 431 return 432 433 self.bitbake_e = self.run_bitbake_env(arg) 434 # bitbake -e doesn't report invalid MACHINE as an error, so 435 # let's check DEPLOY_DIR_IMAGE to make sure that it is a valid 436 # MACHINE. 437 s = re.search('^DEPLOY_DIR_IMAGE="(.*)"', self.bitbake_e, re.M) 438 if s: 439 deploy_dir_image = s.group(1) 440 else: 441 raise RunQemuError("bitbake -e %s" % self.bitbake_e) 442 if self.is_deploy_dir_image(deploy_dir_image): 443 self.set_machine_deploy_dir(arg, deploy_dir_image) 444 else: 445 logger.error("%s not a directory valid DEPLOY_DIR_IMAGE" % deploy_dir_image) 446 self.set("MACHINE", arg) 447 448 def set_dri_path(self): 449 # As runqemu can be run within bitbake (when using testimage, for example), 450 # we need to ensure that we run host pkg-config, and that it does not 451 # get mis-directed to native build paths set by bitbake. 452 try: 453 del os.environ['PKG_CONFIG_PATH'] 454 del os.environ['PKG_CONFIG_DIR'] 455 del os.environ['PKG_CONFIG_LIBDIR'] 456 del os.environ['PKG_CONFIG_SYSROOT_DIR'] 457 except KeyError: 458 pass 459 try: 460 dripath = subprocess.check_output("PATH=/bin:/usr/bin:$PATH pkg-config --variable=dridriverdir dri", shell=True) 461 except subprocess.CalledProcessError as e: 462 raise RunQemuError("Could not determine the path to dri drivers on the host via pkg-config.\nPlease install Mesa development files (particularly, dri.pc) on the host machine.") 463 os.environ['LIBGL_DRIVERS_PATH'] = dripath.decode('utf-8').strip() 464 465 # This preloads uninative libc pieces and therefore ensures that RPATH/RUNPATH 466 # in host mesa drivers doesn't trick uninative into loading host libc. 467 preload_items = ['libdl.so.2', 'librt.so.1', 'libpthread.so.0'] 468 uninative_path = os.path.dirname(self.get("UNINATIVE_LOADER")) 469 if os.path.exists(uninative_path): 470 preload_paths = [os.path.join(uninative_path, i) for i in preload_items] 471 os.environ['LD_PRELOAD'] = " ".join(preload_paths) 472 473 def check_args(self): 474 for debug in ("-d", "--debug"): 475 if debug in sys.argv: 476 logger.setLevel(logging.DEBUG) 477 sys.argv.remove(debug) 478 479 for quiet in ("-q", "--quiet"): 480 if quiet in sys.argv: 481 logger.setLevel(logging.ERROR) 482 sys.argv.remove(quiet) 483 484 if 'gl' not in sys.argv[1:] and 'gl-es' not in sys.argv[1:]: 485 os.environ['SDL_RENDER_DRIVER'] = 'software' 486 os.environ['SDL_FRAMEBUFFER_ACCELERATION'] = 'false' 487 488 unknown_arg = "" 489 for arg in sys.argv[1:]: 490 if arg in self.fstypes + self.vmtypes + self.wictypes: 491 self.check_arg_fstype(arg) 492 elif arg == 'nographic': 493 self.nographic = True 494 elif arg == 'sdl': 495 self.sdl = True 496 elif arg == 'gtk': 497 self.gtk = True 498 elif arg == 'gl': 499 self.gl = True 500 elif 'gl-es' in sys.argv[1:]: 501 self.gl_es = True 502 elif arg == 'egl-headless': 503 self.egl_headless = True 504 elif arg == 'novga': 505 self.novga = True 506 elif arg == 'serial': 507 self.serialconsole = True 508 elif arg == "serialstdio": 509 self.serialstdio = True 510 elif arg == 'audio': 511 logger.info("Enabling audio in qemu") 512 logger.info("Please install sound drivers in linux host") 513 self.audio_enabled = True 514 elif arg == 'kvm': 515 self.kvm_enabled = True 516 elif arg == 'kvm-vhost': 517 self.vhost_enabled = True 518 elif arg == 'slirp': 519 self.slirp_enabled = True 520 elif arg.startswith('bridge='): 521 self.net_bridge = '%s' % arg[len('bridge='):] 522 elif arg == 'snapshot': 523 self.snapshot = True 524 elif arg == 'publicvnc': 525 self.publicvnc = True 526 self.qemu_opt_script += ' -vnc :0' 527 elif arg.startswith('tcpserial='): 528 self.tcpserial_portnum = '%s' % arg[len('tcpserial='):] 529 elif arg.startswith('qemuparams='): 530 self.qemuparams = ' %s' % arg[len('qemuparams='):] 531 elif arg.startswith('bootparams='): 532 self.bootparams = arg[len('bootparams='):] 533 elif os.path.exists(arg) or (re.search(':', arg) and re.search('/', arg)): 534 self.check_arg_path(os.path.abspath(arg)) 535 elif re.search(r'-image-|-image$', arg): 536 # Lazy rootfs 537 self.rootfs = arg 538 elif arg.startswith('ovmf'): 539 self.ovmf_bios.append(arg) 540 else: 541 # At last, assume it is the MACHINE 542 if (not unknown_arg) or unknown_arg == arg: 543 unknown_arg = arg 544 else: 545 raise RunQemuError("Can't handle two unknown args: %s %s\n" 546 "Try 'runqemu help' on how to use it" % \ 547 (unknown_arg, arg)) 548 # Check to make sure it is a valid machine 549 if unknown_arg and self.get('MACHINE') != unknown_arg: 550 if self.get('DEPLOY_DIR_IMAGE'): 551 machine = os.path.basename(self.get('DEPLOY_DIR_IMAGE')) 552 if unknown_arg == machine: 553 self.set("MACHINE", machine) 554 555 self.check_arg_machine(unknown_arg) 556 557 if not (self.get('DEPLOY_DIR_IMAGE') or self.qbconfload): 558 self.load_bitbake_env() 559 s = re.search('^DEPLOY_DIR_IMAGE="(.*)"', self.bitbake_e, re.M) 560 if s: 561 self.set("DEPLOY_DIR_IMAGE", s.group(1)) 562 563 def check_kvm(self): 564 """Check kvm and kvm-host""" 565 if not (self.kvm_enabled or self.vhost_enabled): 566 self.qemu_opt_script += ' %s %s %s' % (self.get('QB_MACHINE'), self.get('QB_CPU'), self.get('QB_SMP')) 567 return 568 569 if not self.get('QB_CPU_KVM'): 570 raise RunQemuError("QB_CPU_KVM is NULL, this board doesn't support kvm") 571 572 self.qemu_opt_script += ' %s %s %s' % (self.get('QB_MACHINE'), self.get('QB_CPU_KVM'), self.get('QB_SMP')) 573 yocto_kvm_wiki = "https://wiki.yoctoproject.org/wiki/How_to_enable_KVM_for_Poky_qemu" 574 yocto_paravirt_kvm_wiki = "https://wiki.yoctoproject.org/wiki/Running_an_x86_Yocto_Linux_image_under_QEMU_KVM" 575 dev_kvm = '/dev/kvm' 576 dev_vhost = '/dev/vhost-net' 577 if self.qemu_system.endswith(('i386', 'x86_64')): 578 with open('/proc/cpuinfo', 'r') as f: 579 kvm_cap = re.search('vmx|svm', "".join(f.readlines())) 580 if not kvm_cap: 581 logger.error("You are trying to enable KVM on a cpu without VT support.") 582 logger.error("Remove kvm from the command-line, or refer:") 583 raise RunQemuError(yocto_kvm_wiki) 584 585 if not os.path.exists(dev_kvm): 586 logger.error("Missing KVM device. Have you inserted kvm modules?") 587 logger.error("For further help see:") 588 raise RunQemuError(yocto_kvm_wiki) 589 590 if os.access(dev_kvm, os.W_OK|os.R_OK): 591 self.qemu_opt_script += ' -enable-kvm' 592 if self.get('MACHINE') == "qemux86": 593 # Workaround for broken APIC window on pre 4.15 host kernels which causes boot hangs 594 # See YOCTO #12301 595 # On 64 bit we use x2apic 596 self.kernel_cmdline_script += " clocksource=kvm-clock hpet=disable noapic nolapic" 597 else: 598 logger.error("You have no read or write permission on /dev/kvm.") 599 logger.error("Please change the ownership of this file as described at:") 600 raise RunQemuError(yocto_kvm_wiki) 601 602 if self.vhost_enabled: 603 if not os.path.exists(dev_vhost): 604 logger.error("Missing virtio net device. Have you inserted vhost-net module?") 605 logger.error("For further help see:") 606 raise RunQemuError(yocto_paravirt_kvm_wiki) 607 608 if not os.access(dev_vhost, os.W_OK|os.R_OK): 609 logger.error("You have no read or write permission on /dev/vhost-net.") 610 logger.error("Please change the ownership of this file as described at:") 611 raise RunQemuError(yocto_paravirt_kvm_wiki) 612 613 def check_fstype(self): 614 """Check and setup FSTYPE""" 615 if not self.fstype: 616 fstype = self.get('QB_DEFAULT_FSTYPE') 617 if fstype: 618 self.fstype = fstype 619 else: 620 raise RunQemuError("FSTYPE is NULL!") 621 622 # parse QB_FSINFO into dict, e.g. { 'wic': ['no-kernel-in-fs', 'a-flag'], 'ext4': ['another-flag']} 623 wic_fs = False 624 qb_fsinfo = self.get('QB_FSINFO') 625 if qb_fsinfo: 626 qb_fsinfo = qb_fsinfo.split() 627 for fsinfo in qb_fsinfo: 628 try: 629 fstype, fsflag = fsinfo.split(':') 630 631 if fstype == 'wic': 632 if fsflag == 'no-kernel-in-fs': 633 wic_fs = True 634 elif fsflag == 'kernel-in-fs': 635 wic_fs = False 636 else: 637 logger.warn('Unknown flag "%s:%s" in QB_FSINFO', fstype, fsflag) 638 continue 639 else: 640 logger.warn('QB_FSINFO is not supported for image type "%s"', fstype) 641 continue 642 643 if fstype in self.fsinfo: 644 self.fsinfo[fstype].append(fsflag) 645 else: 646 self.fsinfo[fstype] = [fsflag] 647 except Exception: 648 logger.error('Invalid parameter "%s" in QB_FSINFO', fsinfo) 649 650 # treat wic images as vmimages (with kernel) or as fsimages (rootfs only) 651 if wic_fs: 652 self.fstypes = self.fstypes + self.wictypes 653 else: 654 self.vmtypes = self.vmtypes + self.wictypes 655 656 def check_rootfs(self): 657 """Check and set rootfs""" 658 659 if self.fstype == "none": 660 return 661 662 if self.get('ROOTFS'): 663 if not self.rootfs: 664 self.rootfs = self.get('ROOTFS') 665 elif self.get('ROOTFS') != self.rootfs: 666 raise RunQemuError("Maybe conflicted ROOTFS: %s vs %s" % (self.get('ROOTFS'), self.rootfs)) 667 668 if self.fstype == 'nfs': 669 return 670 671 if self.rootfs and not os.path.exists(self.rootfs): 672 # Lazy rootfs 673 self.rootfs = "%s/%s-%s.%s" % (self.get('DEPLOY_DIR_IMAGE'), 674 self.rootfs, self.get('MACHINE'), 675 self.fstype) 676 elif not self.rootfs: 677 cmd_name = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'), self.get('IMAGE_NAME'), self.fstype) 678 cmd_link = '%s/%s*.%s' % (self.get('DEPLOY_DIR_IMAGE'), self.get('IMAGE_LINK_NAME'), self.fstype) 679 cmds = (cmd_name, cmd_link) 680 self.rootfs = get_first_file(cmds) 681 if not self.rootfs: 682 raise RunQemuError("Failed to find rootfs: %s or %s" % cmds) 683 684 if not os.path.exists(self.rootfs): 685 raise RunQemuError("Can't find rootfs: %s" % self.rootfs) 686 687 def setup_pkkek1(self): 688 """ 689 Extract from PEM certificate the Platform Key and first Key 690 Exchange Key certificate string. The hypervisor needs to provide 691 it in the Type 11 SMBIOS table 692 """ 693 pemcert = '%s/%s' % (self.get('DEPLOY_DIR_IMAGE'), 'OvmfPkKek1.pem') 694 try: 695 with open(pemcert, 'r') as pemfile: 696 key = pemfile.read().replace('\n', ''). \ 697 replace('-----BEGIN CERTIFICATE-----', ''). \ 698 replace('-----END CERTIFICATE-----', '') 699 self.ovmf_secboot_pkkek1 = key 700 701 except FileNotFoundError: 702 raise RunQemuError("Can't open PEM certificate %s " % pemcert) 703 704 def check_ovmf(self): 705 """Check and set full path for OVMF firmware and variable file(s).""" 706 707 for index, ovmf in enumerate(self.ovmf_bios): 708 if os.path.exists(ovmf): 709 continue 710 for suffix in ('qcow2', 'bin'): 711 path = '%s/%s.%s' % (self.get('DEPLOY_DIR_IMAGE'), ovmf, suffix) 712 if os.path.exists(path): 713 self.ovmf_bios[index] = path 714 if ovmf.endswith('secboot'): 715 self.setup_pkkek1() 716 break 717 else: 718 raise RunQemuError("Can't find OVMF firmware: %s" % ovmf) 719 720 def check_kernel(self): 721 """Check and set kernel""" 722 # The vm image doesn't need a kernel 723 if self.fstype in self.vmtypes: 724 return 725 726 # See if the user supplied a KERNEL option 727 if self.get('KERNEL'): 728 self.kernel = self.get('KERNEL') 729 730 # QB_DEFAULT_KERNEL is always a full file path 731 kernel_name = os.path.basename(self.get('QB_DEFAULT_KERNEL')) 732 733 # The user didn't want a kernel to be loaded 734 if kernel_name == "none" and not self.kernel: 735 return 736 737 deploy_dir_image = self.get('DEPLOY_DIR_IMAGE') 738 if not self.kernel: 739 kernel_match_name = "%s/%s" % (deploy_dir_image, kernel_name) 740 kernel_match_link = "%s/%s" % (deploy_dir_image, self.get('KERNEL_IMAGETYPE')) 741 kernel_startswith = "%s/%s*" % (deploy_dir_image, self.get('KERNEL_IMAGETYPE')) 742 cmds = (kernel_match_name, kernel_match_link, kernel_startswith) 743 self.kernel = get_first_file(cmds) 744 if not self.kernel: 745 raise RunQemuError('KERNEL not found: %s, %s or %s' % cmds) 746 747 if not os.path.exists(self.kernel): 748 raise RunQemuError("KERNEL %s not found" % self.kernel) 749 750 def check_dtb(self): 751 """Check and set dtb""" 752 # Did the user specify a device tree? 753 if self.get('DEVICE_TREE'): 754 self.dtb = self.get('DEVICE_TREE') 755 if not os.path.exists(self.dtb): 756 raise RunQemuError('Specified DTB not found: %s' % self.dtb) 757 return 758 759 dtb = self.get('QB_DTB') 760 if dtb: 761 deploy_dir_image = self.get('DEPLOY_DIR_IMAGE') 762 cmd_match = "%s/%s" % (deploy_dir_image, dtb) 763 cmd_startswith = "%s/%s*" % (deploy_dir_image, dtb) 764 cmd_wild = "%s/*.dtb" % deploy_dir_image 765 cmds = (cmd_match, cmd_startswith, cmd_wild) 766 self.dtb = get_first_file(cmds) 767 if not os.path.exists(self.dtb): 768 raise RunQemuError('DTB not found: %s, %s or %s' % cmds) 769 770 def check_bios(self): 771 """Check and set bios""" 772 773 # See if the user supplied a BIOS option 774 if self.get('BIOS'): 775 self.bios = self.get('BIOS') 776 777 # QB_DEFAULT_BIOS is always a full file path 778 bios_name = os.path.basename(self.get('QB_DEFAULT_BIOS')) 779 780 # The user didn't want a bios to be loaded 781 if (bios_name == "" or bios_name == "none") and not self.bios: 782 return 783 784 if not self.bios: 785 deploy_dir_image = self.get('DEPLOY_DIR_IMAGE') 786 self.bios = "%s/%s" % (deploy_dir_image, bios_name) 787 788 if not self.bios: 789 raise RunQemuError('BIOS not found: %s' % bios_match_name) 790 791 if not os.path.exists(self.bios): 792 raise RunQemuError("BIOS %s not found" % self.bios) 793 794 795 def check_mem(self): 796 """ 797 Both qemu and kernel needs memory settings, so check QB_MEM and set it 798 for both. 799 """ 800 s = re.search('-m +([0-9]+)', self.qemuparams) 801 if s: 802 self.set('QB_MEM', '-m %s' % s.group(1)) 803 elif not self.get('QB_MEM'): 804 logger.info('QB_MEM is not set, use 256M by default') 805 self.set('QB_MEM', '-m 256') 806 807 # Check and remove M or m suffix 808 qb_mem = self.get('QB_MEM') 809 if qb_mem.endswith('M') or qb_mem.endswith('m'): 810 qb_mem = qb_mem[:-1] 811 812 # Add -m prefix it not present 813 if not qb_mem.startswith('-m'): 814 qb_mem = '-m %s' % qb_mem 815 816 self.set('QB_MEM', qb_mem) 817 818 mach = self.get('MACHINE') 819 if not mach.startswith(('qemumips', 'qemux86')): 820 self.kernel_cmdline_script += ' mem=%s' % self.get('QB_MEM').replace('-m','').strip() + 'M' 821 822 self.qemu_opt_script += ' %s' % self.get('QB_MEM') 823 824 def check_tcpserial(self): 825 if self.tcpserial_portnum: 826 ports = self.tcpserial_portnum.split(':') 827 port = ports[0] 828 if self.get('QB_TCPSERIAL_OPT'): 829 self.qemu_opt_script += ' ' + self.get('QB_TCPSERIAL_OPT').replace('@PORT@', port) 830 else: 831 self.qemu_opt_script += ' -serial tcp:127.0.0.1:%s' % port 832 833 if len(ports) > 1: 834 for port in ports[1:]: 835 self.qemu_opt_script += ' -serial tcp:127.0.0.1:%s' % port 836 837 def check_and_set(self): 838 """Check configs sanity and set when needed""" 839 self.validate_paths() 840 if not self.slirp_enabled and not self.net_bridge: 841 check_tun() 842 # Check audio 843 if self.audio_enabled: 844 if not self.get('QB_AUDIO_DRV'): 845 raise RunQemuError("QB_AUDIO_DRV is NULL, this board doesn't support audio") 846 if not self.get('QB_AUDIO_OPT'): 847 logger.warning('QB_AUDIO_OPT is NULL, you may need define it to make audio work') 848 else: 849 self.qemu_opt_script += ' %s' % self.get('QB_AUDIO_OPT') 850 os.putenv('QEMU_AUDIO_DRV', self.get('QB_AUDIO_DRV')) 851 else: 852 os.putenv('QEMU_AUDIO_DRV', 'none') 853 854 self.check_qemu_system() 855 self.check_kvm() 856 self.check_fstype() 857 self.check_rootfs() 858 self.check_ovmf() 859 self.check_kernel() 860 self.check_dtb() 861 self.check_bios() 862 self.check_mem() 863 self.check_tcpserial() 864 865 def read_qemuboot(self): 866 if not self.qemuboot: 867 if self.get('DEPLOY_DIR_IMAGE'): 868 deploy_dir_image = self.get('DEPLOY_DIR_IMAGE') 869 else: 870 logger.warning("Can't find qemuboot conf file, DEPLOY_DIR_IMAGE is NULL!") 871 return 872 873 if self.rootfs and not os.path.exists(self.rootfs): 874 # Lazy rootfs 875 machine = self.get('MACHINE') 876 if not machine: 877 machine = os.path.basename(deploy_dir_image) 878 self.qemuboot = "%s/%s-%s.qemuboot.conf" % (deploy_dir_image, 879 self.rootfs, machine) 880 else: 881 cmd = 'ls -t %s/*.qemuboot.conf' % deploy_dir_image 882 logger.debug('Running %s...' % cmd) 883 try: 884 qbs = subprocess.check_output(cmd, shell=True).decode('utf-8') 885 except subprocess.CalledProcessError as err: 886 raise RunQemuError(err) 887 if qbs: 888 for qb in qbs.split(): 889 # Don't use initramfs when other choices unless fstype is ramfs 890 if '-initramfs-' in os.path.basename(qb) and self.fstype != 'cpio.gz': 891 continue 892 self.qemuboot = qb 893 break 894 if not self.qemuboot: 895 # Use the first one when no choice 896 self.qemuboot = qbs.split()[0] 897 self.qbconfload = True 898 899 if not self.qemuboot: 900 # If we haven't found a .qemuboot.conf at this point it probably 901 # doesn't exist, continue without 902 return 903 904 if not os.path.exists(self.qemuboot): 905 raise RunQemuError("Failed to find %s (wrong image name or BSP does not support running under qemu?)." % self.qemuboot) 906 907 logger.debug('CONFFILE: %s' % self.qemuboot) 908 909 cf = configparser.ConfigParser() 910 cf.read(self.qemuboot) 911 for k, v in cf.items('config_bsp'): 912 k_upper = k.upper() 913 if v.startswith("../"): 914 v = os.path.abspath(os.path.dirname(self.qemuboot) + "/" + v) 915 elif v == ".": 916 v = os.path.dirname(self.qemuboot) 917 self.set(k_upper, v) 918 919 def validate_paths(self): 920 """Ensure all relevant path variables are set""" 921 # When we're started with a *.qemuboot.conf arg assume that image 922 # artefacts are relative to that file, rather than in whatever 923 # directory DEPLOY_DIR_IMAGE in the conf file points to. 924 if self.qbconfload: 925 imgdir = os.path.realpath(os.path.dirname(self.qemuboot)) 926 if imgdir != os.path.realpath(self.get('DEPLOY_DIR_IMAGE')): 927 logger.info('Setting DEPLOY_DIR_IMAGE to folder containing %s (%s)' % (self.qemuboot, imgdir)) 928 self.set('DEPLOY_DIR_IMAGE', imgdir) 929 930 # If the STAGING_*_NATIVE directories from the config file don't exist 931 # and we're in a sourced OE build directory try to extract the paths 932 # from `bitbake -e` 933 havenative = os.path.exists(self.get('STAGING_DIR_NATIVE')) and \ 934 os.path.exists(self.get('STAGING_BINDIR_NATIVE')) 935 936 if not havenative: 937 if not self.bitbake_e: 938 self.load_bitbake_env() 939 940 if self.bitbake_e: 941 native_vars = ['STAGING_DIR_NATIVE'] 942 for nv in native_vars: 943 s = re.search('^%s="(.*)"' % nv, self.bitbake_e, re.M) 944 if s and s.group(1) != self.get(nv): 945 logger.info('Overriding conf file setting of %s to %s from Bitbake environment' % (nv, s.group(1))) 946 self.set(nv, s.group(1)) 947 else: 948 # when we're invoked from a running bitbake instance we won't 949 # be able to call `bitbake -e`, then try: 950 # - get OE_TMPDIR from environment and guess paths based on it 951 # - get OECORE_NATIVE_SYSROOT from environment (for sdk) 952 tmpdir = self.get('OE_TMPDIR') 953 oecore_native_sysroot = self.get('OECORE_NATIVE_SYSROOT') 954 if tmpdir: 955 logger.info('Setting STAGING_DIR_NATIVE and STAGING_BINDIR_NATIVE relative to OE_TMPDIR (%s)' % tmpdir) 956 hostos, _, _, _, machine = os.uname() 957 buildsys = '%s-%s' % (machine, hostos.lower()) 958 staging_dir_native = '%s/sysroots/%s' % (tmpdir, buildsys) 959 self.set('STAGING_DIR_NATIVE', staging_dir_native) 960 elif oecore_native_sysroot: 961 logger.info('Setting STAGING_DIR_NATIVE to OECORE_NATIVE_SYSROOT (%s)' % oecore_native_sysroot) 962 self.set('STAGING_DIR_NATIVE', oecore_native_sysroot) 963 if self.get('STAGING_DIR_NATIVE'): 964 # we have to assume that STAGING_BINDIR_NATIVE is at usr/bin 965 staging_bindir_native = '%s/usr/bin' % self.get('STAGING_DIR_NATIVE') 966 logger.info('Setting STAGING_BINDIR_NATIVE to %s' % staging_bindir_native) 967 self.set('STAGING_BINDIR_NATIVE', '%s/usr/bin' % self.get('STAGING_DIR_NATIVE')) 968 969 def print_config(self): 970 logoutput = ['Continuing with the following parameters:'] 971 if not self.fstype in self.vmtypes: 972 logoutput.append('KERNEL: [%s]' % self.kernel) 973 if self.bios: 974 logoutput.append('BIOS: [%s]' % self.bios) 975 if self.dtb: 976 logoutput.append('DTB: [%s]' % self.dtb) 977 logoutput.append('MACHINE: [%s]' % self.get('MACHINE')) 978 try: 979 fstype_flags = ' (' + ', '.join(self.fsinfo[self.fstype]) + ')' 980 except KeyError: 981 fstype_flags = '' 982 logoutput.append('FSTYPE: [%s%s]' % (self.fstype, fstype_flags)) 983 if self.fstype == 'nfs': 984 logoutput.append('NFS_DIR: [%s]' % self.rootfs) 985 else: 986 logoutput.append('ROOTFS: [%s]' % self.rootfs) 987 if self.ovmf_bios: 988 logoutput.append('OVMF: %s' % self.ovmf_bios) 989 if (self.ovmf_secboot_pkkek1): 990 logoutput.append('SECBOOT PKKEK1: [%s...]' % self.ovmf_secboot_pkkek1[0:100]) 991 logoutput.append('CONFFILE: [%s]' % self.qemuboot) 992 logoutput.append('') 993 logger.info('\n'.join(logoutput)) 994 995 def setup_nfs(self): 996 if not self.nfs_server: 997 if self.slirp_enabled: 998 self.nfs_server = '10.0.2.2' 999 else: 1000 self.nfs_server = '192.168.7.1' 1001 1002 # Figure out a new nfs_instance to allow multiple qemus running. 1003 ps = subprocess.check_output(("ps", "auxww")).decode('utf-8') 1004 pattern = '/bin/unfsd .* -i .*\.pid -e .*/exports([0-9]+) ' 1005 all_instances = re.findall(pattern, ps, re.M) 1006 if all_instances: 1007 all_instances.sort(key=int) 1008 self.nfs_instance = int(all_instances.pop()) + 1 1009 1010 nfsd_port = 3049 + 2 * self.nfs_instance 1011 mountd_port = 3048 + 2 * self.nfs_instance 1012 1013 # Export vars for runqemu-export-rootfs 1014 export_dict = { 1015 'NFS_INSTANCE': self.nfs_instance, 1016 'NFSD_PORT': nfsd_port, 1017 'MOUNTD_PORT': mountd_port, 1018 } 1019 for k, v in export_dict.items(): 1020 # Use '%s' since they are integers 1021 os.putenv(k, '%s' % v) 1022 1023 self.unfs_opts="nfsvers=3,port=%s,tcp,mountport=%s" % (nfsd_port, mountd_port) 1024 1025 # Extract .tar.bz2 or .tar.bz if no nfs dir 1026 if not (self.rootfs and os.path.isdir(self.rootfs)): 1027 src_prefix = '%s/%s' % (self.get('DEPLOY_DIR_IMAGE'), self.get('IMAGE_LINK_NAME')) 1028 dest = "%s-nfsroot" % src_prefix 1029 if os.path.exists('%s.pseudo_state' % dest): 1030 logger.info('Use %s as NFS_DIR' % dest) 1031 self.rootfs = dest 1032 else: 1033 src = "" 1034 src1 = '%s.tar.bz2' % src_prefix 1035 src2 = '%s.tar.gz' % src_prefix 1036 if os.path.exists(src1): 1037 src = src1 1038 elif os.path.exists(src2): 1039 src = src2 1040 if not src: 1041 raise RunQemuError("No NFS_DIR is set, and can't find %s or %s to extract" % (src1, src2)) 1042 logger.info('NFS_DIR not found, extracting %s to %s' % (src, dest)) 1043 cmd = ('runqemu-extract-sdk', src, dest) 1044 logger.info('Running %s...' % str(cmd)) 1045 if subprocess.call(cmd) != 0: 1046 raise RunQemuError('Failed to run %s' % cmd) 1047 self.rootfs = dest 1048 self.cleanup_files.append(self.rootfs) 1049 self.cleanup_files.append('%s.pseudo_state' % self.rootfs) 1050 1051 # Start the userspace NFS server 1052 cmd = ('runqemu-export-rootfs', 'start', self.rootfs) 1053 logger.info('Running %s...' % str(cmd)) 1054 if subprocess.call(cmd) != 0: 1055 raise RunQemuError('Failed to run %s' % cmd) 1056 1057 self.nfs_running = True 1058 1059 def setup_net_bridge(self): 1060 self.set('NETWORK_CMD', '-netdev bridge,br=%s,id=net0,helper=%s -device virtio-net-pci,netdev=net0 ' % ( 1061 self.net_bridge, os.path.join(self.bindir_native, 'qemu-oe-bridge-helper'))) 1062 1063 def setup_slirp(self): 1064 """Setup user networking""" 1065 1066 if self.fstype == 'nfs': 1067 self.setup_nfs() 1068 netconf = " " + self.cmdline_ip_slirp 1069 logger.info("Network configuration:%s", netconf) 1070 self.kernel_cmdline_script += netconf 1071 # Port mapping 1072 hostfwd = ",hostfwd=tcp::2222-:22,hostfwd=tcp::2323-:23" 1073 qb_slirp_opt_default = "-netdev user,id=net0%s,tftp=%s" % (hostfwd, self.get('DEPLOY_DIR_IMAGE')) 1074 qb_slirp_opt = self.get('QB_SLIRP_OPT') or qb_slirp_opt_default 1075 # Figure out the port 1076 ports = re.findall('hostfwd=[^-]*:([0-9]+)-[^,-]*', qb_slirp_opt) 1077 ports = [int(i) for i in ports] 1078 mac = 2 1079 1080 lockdir = "/tmp/qemu-port-locks" 1081 if not os.path.exists(lockdir): 1082 # There might be a race issue when multi runqemu processess are 1083 # running at the same time. 1084 try: 1085 os.mkdir(lockdir) 1086 os.chmod(lockdir, 0o777) 1087 except FileExistsError: 1088 pass 1089 1090 # Find a free port to avoid conflicts 1091 for p in ports[:]: 1092 p_new = p 1093 while not self.check_free_port('localhost', p_new, lockdir): 1094 p_new += 1 1095 mac += 1 1096 while p_new in ports: 1097 p_new += 1 1098 mac += 1 1099 if p != p_new: 1100 ports.append(p_new) 1101 qb_slirp_opt = re.sub(':%s-' % p, ':%s-' % p_new, qb_slirp_opt) 1102 logger.info("Port forward changed: %s -> %s" % (p, p_new)) 1103 mac = "%s%02x" % (self.mac_slirp, mac) 1104 self.set('NETWORK_CMD', '%s %s' % (self.network_device.replace('@MAC@', mac), qb_slirp_opt)) 1105 # Print out port foward 1106 hostfwd = re.findall('(hostfwd=[^,]*)', qb_slirp_opt) 1107 if hostfwd: 1108 logger.info('Port forward: %s' % ' '.join(hostfwd)) 1109 1110 def setup_tap(self): 1111 """Setup tap""" 1112 1113 # This file is created when runqemu-gen-tapdevs creates a bank of tap 1114 # devices, indicating that the user should not bring up new ones using 1115 # sudo. 1116 nosudo_flag = '/etc/runqemu-nosudo' 1117 self.qemuifup = shutil.which('runqemu-ifup') 1118 self.qemuifdown = shutil.which('runqemu-ifdown') 1119 ip = shutil.which('ip') 1120 lockdir = "/tmp/qemu-tap-locks" 1121 1122 if not (self.qemuifup and self.qemuifdown and ip): 1123 logger.error("runqemu-ifup: %s" % self.qemuifup) 1124 logger.error("runqemu-ifdown: %s" % self.qemuifdown) 1125 logger.error("ip: %s" % ip) 1126 raise OEPathError("runqemu-ifup, runqemu-ifdown or ip not found") 1127 1128 if not os.path.exists(lockdir): 1129 # There might be a race issue when multi runqemu processess are 1130 # running at the same time. 1131 try: 1132 os.mkdir(lockdir) 1133 os.chmod(lockdir, 0o777) 1134 except FileExistsError: 1135 pass 1136 1137 cmd = (ip, 'link') 1138 logger.debug('Running %s...' % str(cmd)) 1139 ip_link = subprocess.check_output(cmd).decode('utf-8') 1140 # Matches line like: 6: tap0: <foo> 1141 possibles = re.findall('^[0-9]+: +(tap[0-9]+): <.*', ip_link, re.M) 1142 tap = "" 1143 for p in possibles: 1144 lockfile = os.path.join(lockdir, p) 1145 if os.path.exists('%s.skip' % lockfile): 1146 logger.info('Found %s.skip, skipping %s' % (lockfile, p)) 1147 continue 1148 self.taplock = lockfile + '.lock' 1149 if self.acquire_taplock(error=False): 1150 tap = p 1151 logger.info("Using preconfigured tap device %s" % tap) 1152 logger.info("If this is not intended, touch %s.skip to make runqemu skip %s." %(lockfile, tap)) 1153 break 1154 1155 if not tap: 1156 if os.path.exists(nosudo_flag): 1157 logger.error("Error: There are no available tap devices to use for networking,") 1158 logger.error("and I see %s exists, so I am not going to try creating" % nosudo_flag) 1159 raise RunQemuError("a new one with sudo.") 1160 1161 gid = os.getgid() 1162 uid = os.getuid() 1163 logger.info("Setting up tap interface under sudo") 1164 cmd = ('sudo', self.qemuifup, str(uid), str(gid), self.bindir_native) 1165 try: 1166 tap = subprocess.check_output(cmd).decode('utf-8').strip() 1167 except subprocess.CalledProcessError as e: 1168 logger.error('Setting up tap device failed:\n%s\nRun runqemu-gen-tapdevs to manually create one.' % str(e)) 1169 sys.exit(1) 1170 lockfile = os.path.join(lockdir, tap) 1171 self.taplock = lockfile + '.lock' 1172 self.acquire_taplock() 1173 self.cleantap = True 1174 logger.debug('Created tap: %s' % tap) 1175 1176 if not tap: 1177 logger.error("Failed to setup tap device. Run runqemu-gen-tapdevs to manually create.") 1178 sys.exit(1) 1179 self.tap = tap 1180 tapnum = int(tap[3:]) 1181 gateway = tapnum * 2 + 1 1182 client = gateway + 1 1183 if self.fstype == 'nfs': 1184 self.setup_nfs() 1185 netconf = " " + self.cmdline_ip_tap 1186 netconf = netconf.replace('@CLIENT@', str(client)) 1187 netconf = netconf.replace('@GATEWAY@', str(gateway)) 1188 logger.info("Network configuration:%s", netconf) 1189 self.kernel_cmdline_script += netconf 1190 mac = "%s%02x" % (self.mac_tap, client) 1191 qb_tap_opt = self.get('QB_TAP_OPT') 1192 if qb_tap_opt: 1193 qemu_tap_opt = qb_tap_opt.replace('@TAP@', tap) 1194 else: 1195 qemu_tap_opt = "-netdev tap,id=net0,ifname=%s,script=no,downscript=no" % (self.tap) 1196 1197 if self.vhost_enabled: 1198 qemu_tap_opt += ',vhost=on' 1199 1200 self.set('NETWORK_CMD', '%s %s' % (self.network_device.replace('@MAC@', mac), qemu_tap_opt)) 1201 1202 def setup_network(self): 1203 if self.get('QB_NET') == 'none': 1204 return 1205 if sys.stdin.isatty(): 1206 self.saved_stty = subprocess.check_output(("stty", "-g")).decode('utf-8').strip() 1207 self.network_device = self.get('QB_NETWORK_DEVICE') or self.network_device 1208 if self.net_bridge: 1209 self.setup_net_bridge() 1210 elif self.slirp_enabled: 1211 self.cmdline_ip_slirp = self.get('QB_CMDLINE_IP_SLIRP') or self.cmdline_ip_slirp 1212 self.setup_slirp() 1213 else: 1214 self.cmdline_ip_tap = self.get('QB_CMDLINE_IP_TAP') or self.cmdline_ip_tap 1215 self.setup_tap() 1216 1217 def setup_rootfs(self): 1218 if self.get('QB_ROOTFS') == 'none': 1219 return 1220 if 'wic.' in self.fstype: 1221 self.fstype = self.fstype[4:] 1222 rootfs_format = self.fstype if self.fstype in ('vmdk', 'vhd', 'vhdx', 'qcow2', 'vdi') else 'raw' 1223 1224 tmpfsdir = os.environ.get("RUNQEMU_TMPFS_DIR", None) 1225 if self.snapshot and tmpfsdir: 1226 newrootfs = os.path.join(tmpfsdir, os.path.basename(self.rootfs)) + "." + str(os.getpid()) 1227 logger.info("Copying rootfs to %s" % newrootfs) 1228 copy_start = time.time() 1229 shutil.copyfile(self.rootfs, newrootfs) 1230 logger.info("Copy done in %s seconds" % (time.time() - copy_start)) 1231 self.rootfs = newrootfs 1232 # Don't need a second copy now! 1233 self.snapshot = False 1234 self.cleanup_files.append(newrootfs) 1235 1236 qb_rootfs_opt = self.get('QB_ROOTFS_OPT') 1237 if qb_rootfs_opt: 1238 self.rootfs_options = qb_rootfs_opt.replace('@ROOTFS@', self.rootfs) 1239 else: 1240 self.rootfs_options = '-drive file=%s,if=virtio,format=%s' % (self.rootfs, rootfs_format) 1241 1242 qb_rootfs_extra_opt = self.get("QB_ROOTFS_EXTRA_OPT") 1243 if qb_rootfs_extra_opt and not qb_rootfs_extra_opt.startswith(","): 1244 qb_rootfs_extra_opt = "," + qb_rootfs_extra_opt 1245 1246 if self.fstype in ('cpio.gz', 'cpio'): 1247 self.kernel_cmdline = 'root=/dev/ram0 rw debugshell' 1248 self.rootfs_options = '-initrd %s' % self.rootfs 1249 else: 1250 vm_drive = '' 1251 if self.fstype in self.vmtypes: 1252 if self.fstype == 'iso': 1253 vm_drive = '-drive file=%s,if=virtio,media=cdrom' % self.rootfs 1254 elif self.get('QB_DRIVE_TYPE'): 1255 drive_type = self.get('QB_DRIVE_TYPE') 1256 if drive_type.startswith("/dev/sd"): 1257 logger.info('Using scsi drive') 1258 vm_drive = '-drive if=none,id=hd,file=%s,format=%s -device virtio-scsi-pci,id=scsi -device scsi-hd,drive=hd%s' \ 1259 % (self.rootfs, rootfs_format, qb_rootfs_extra_opt) 1260 elif drive_type.startswith("/dev/hd"): 1261 logger.info('Using ide drive') 1262 vm_drive = "-drive file=%s,format=%s" % (self.rootfs, rootfs_format) 1263 elif drive_type.startswith("/dev/vdb"): 1264 logger.info('Using block virtio drive'); 1265 vm_drive = '-drive id=disk0,file=%s,if=none,format=%s -device virtio-blk-device,drive=disk0%s' \ 1266 % (self.rootfs, rootfs_format,qb_rootfs_extra_opt) 1267 else: 1268 # virtio might have been selected explicitly (just use it), or 1269 # is used as fallback (then warn about that). 1270 if not drive_type.startswith("/dev/vd"): 1271 logger.warning("Unknown QB_DRIVE_TYPE: %s" % drive_type) 1272 logger.warning("Failed to figure out drive type, consider define or fix QB_DRIVE_TYPE") 1273 logger.warning('Trying to use virtio block drive') 1274 vm_drive = '-drive if=virtio,file=%s,format=%s' % (self.rootfs, rootfs_format) 1275 1276 # All branches above set vm_drive. 1277 self.rootfs_options = vm_drive 1278 if not self.fstype in self.vmtypes: 1279 self.rootfs_options += ' -no-reboot' 1280 1281 # By default, ' rw' is appended to QB_KERNEL_ROOT unless either ro or rw is explicitly passed. 1282 qb_kernel_root = self.get('QB_KERNEL_ROOT') 1283 qb_kernel_root_l = qb_kernel_root.split() 1284 if not ('ro' in qb_kernel_root_l or 'rw' in qb_kernel_root_l): 1285 qb_kernel_root += ' rw' 1286 self.kernel_cmdline = 'root=%s' % qb_kernel_root 1287 1288 if self.fstype == 'nfs': 1289 self.rootfs_options = '' 1290 k_root = '/dev/nfs nfsroot=%s:%s,%s' % (self.nfs_server, os.path.abspath(self.rootfs), self.unfs_opts) 1291 self.kernel_cmdline = 'root=%s rw' % k_root 1292 1293 if self.fstype == 'none': 1294 self.rootfs_options = '' 1295 1296 self.set('ROOTFS_OPTIONS', self.rootfs_options) 1297 1298 def guess_qb_system(self): 1299 """attempt to determine the appropriate qemu-system binary""" 1300 mach = self.get('MACHINE') 1301 if not mach: 1302 search = '.*(qemux86-64|qemux86|qemuarm64|qemuarm|qemumips64|qemumips64el|qemumipsel|qemumips|qemuppc).*' 1303 if self.rootfs: 1304 match = re.match(search, self.rootfs) 1305 if match: 1306 mach = match.group(1) 1307 elif self.kernel: 1308 match = re.match(search, self.kernel) 1309 if match: 1310 mach = match.group(1) 1311 1312 if not mach: 1313 return None 1314 1315 if mach == 'qemuarm': 1316 qbsys = 'arm' 1317 elif mach == 'qemuarm64': 1318 qbsys = 'aarch64' 1319 elif mach == 'qemux86': 1320 qbsys = 'i386' 1321 elif mach == 'qemux86-64': 1322 qbsys = 'x86_64' 1323 elif mach == 'qemuppc': 1324 qbsys = 'ppc' 1325 elif mach == 'qemumips': 1326 qbsys = 'mips' 1327 elif mach == 'qemumips64': 1328 qbsys = 'mips64' 1329 elif mach == 'qemumipsel': 1330 qbsys = 'mipsel' 1331 elif mach == 'qemumips64el': 1332 qbsys = 'mips64el' 1333 elif mach == 'qemuriscv64': 1334 qbsys = 'riscv64' 1335 elif mach == 'qemuriscv32': 1336 qbsys = 'riscv32' 1337 else: 1338 logger.error("Unable to determine QEMU PC System emulator for %s machine." % mach) 1339 logger.error("As %s is not among valid QEMU machines such as," % mach) 1340 logger.error("qemux86-64, qemux86, qemuarm64, qemuarm, qemumips64, qemumips64el, qemumipsel, qemumips, qemuppc") 1341 raise RunQemuError("Set qb_system_name with suitable QEMU PC System emulator in .*qemuboot.conf.") 1342 1343 return 'qemu-system-%s' % qbsys 1344 1345 def check_qemu_system(self): 1346 qemu_system = self.get('QB_SYSTEM_NAME') 1347 if not qemu_system: 1348 qemu_system = self.guess_qb_system() 1349 if not qemu_system: 1350 raise RunQemuError("Failed to boot, QB_SYSTEM_NAME is NULL!") 1351 self.qemu_system = qemu_system 1352 1353 def setup_vga(self): 1354 if self.nographic == True: 1355 if self.sdl == True: 1356 raise RunQemuError('Option nographic makes no sense alongside the sdl option.') 1357 if self.gtk == True: 1358 raise RunQemuError('Option nographic makes no sense alongside the gtk option.') 1359 self.qemu_opt += ' -nographic' 1360 1361 if self.novga == True: 1362 self.qemu_opt += ' -vga none' 1363 return 1364 1365 if (self.gl_es == True or self.gl == True) and (self.sdl == False and self.gtk == False): 1366 raise RunQemuError('Option gl/gl-es needs gtk or sdl option.') 1367 1368 # If we have no display option, we autodetect based upon what qemu supports. We 1369 # need our font setup and show-cusor below so we need to see what qemu --help says 1370 # is supported so we can pass our correct config in. 1371 if not self.nographic and not self.sdl and not self.gtk and not self.publicvnc and not self.egl_headless == True: 1372 output = subprocess.check_output([self.qemu_bin, "--help"], universal_newlines=True) 1373 if "-display gtk" in output: 1374 self.gtk = True 1375 elif "-display sdl" in output: 1376 self.sdl = True 1377 else: 1378 self.qemu_opt += ' -display none' 1379 1380 if self.sdl == True or self.gtk == True or self.egl_headless == True: 1381 1382 if self.qemu_system.endswith(('i386', 'x86_64')): 1383 if self.gl or self.gl_es or self.egl_headless: 1384 self.qemu_opt += ' -device virtio-vga-gl ' 1385 else: 1386 self.qemu_opt += ' -device virtio-vga ' 1387 1388 self.qemu_opt += ' -display ' 1389 if self.egl_headless == True: 1390 self.set_dri_path() 1391 self.qemu_opt += 'egl-headless,' 1392 else: 1393 if self.sdl == True: 1394 self.qemu_opt += 'sdl,' 1395 elif self.gtk == True: 1396 os.environ['FONTCONFIG_PATH'] = '/etc/fonts' 1397 self.qemu_opt += 'gtk,' 1398 1399 if self.gl == True: 1400 self.set_dri_path() 1401 self.qemu_opt += 'gl=on,' 1402 elif self.gl_es == True: 1403 self.set_dri_path() 1404 self.qemu_opt += 'gl=es,' 1405 self.qemu_opt += 'show-cursor=on' 1406 1407 self.qemu_opt += ' %s' %self.get('QB_GRAPHICS') 1408 1409 def setup_serial(self): 1410 # Setup correct kernel command line for serial 1411 if self.get('SERIAL_CONSOLES') and (self.serialstdio == True or self.serialconsole == True or self.nographic == True or self.tcpserial_portnum): 1412 for entry in self.get('SERIAL_CONSOLES').split(' '): 1413 self.kernel_cmdline_script += ' console=%s' %entry.split(';')[1] 1414 1415 if self.serialstdio == True or self.nographic == True: 1416 self.qemu_opt += " -serial mon:stdio" 1417 else: 1418 self.qemu_opt += " -serial mon:vc" 1419 if self.serialconsole: 1420 if sys.stdin.isatty(): 1421 subprocess.check_call(("stty", "intr", "^]")) 1422 logger.info("Interrupt character is '^]'") 1423 1424 self.qemu_opt += " %s" % self.get("QB_SERIAL_OPT") 1425 1426 # We always wants ttyS0 and ttyS1 in qemu machines (see SERIAL_CONSOLES). 1427 # If no serial or serialtcp options were specified, only ttyS0 is created 1428 # and sysvinit shows an error trying to enable ttyS1: 1429 # INIT: Id "S1" respawning too fast: disabled for 5 minutes 1430 serial_num = len(re.findall("-serial", self.qemu_opt)) 1431 if serial_num < 2: 1432 self.qemu_opt += " -serial null" 1433 1434 def find_qemu(self): 1435 qemu_bin = os.path.join(self.bindir_native, self.qemu_system) 1436 1437 # It is possible to have qemu-native in ASSUME_PROVIDED, and it won't 1438 # find QEMU in sysroot, it needs to use host's qemu. 1439 if not os.path.exists(qemu_bin): 1440 logger.info("QEMU binary not found in %s, trying host's QEMU" % qemu_bin) 1441 for path in (os.environ['PATH'] or '').split(':'): 1442 qemu_bin_tmp = os.path.join(path, self.qemu_system) 1443 logger.info("Trying: %s" % qemu_bin_tmp) 1444 if os.path.exists(qemu_bin_tmp): 1445 qemu_bin = qemu_bin_tmp 1446 if not os.path.isabs(qemu_bin): 1447 qemu_bin = os.path.abspath(qemu_bin) 1448 logger.info("Using host's QEMU: %s" % qemu_bin) 1449 break 1450 1451 if not os.access(qemu_bin, os.X_OK): 1452 raise OEPathError("No QEMU binary '%s' could be found" % qemu_bin) 1453 self.qemu_bin = qemu_bin 1454 1455 def setup_final(self): 1456 1457 self.find_qemu() 1458 1459 self.qemu_opt = "%s %s %s %s %s" % (self.qemu_bin, self.get('NETWORK_CMD'), self.get('QB_RNG'), self.get('ROOTFS_OPTIONS'), self.get('QB_OPT_APPEND').replace('@DEPLOY_DIR_IMAGE@', self.get('DEPLOY_DIR_IMAGE'))) 1460 1461 for ovmf in self.ovmf_bios: 1462 format = ovmf.rsplit('.', 1)[-1] 1463 if format == "bin": 1464 format = "raw" 1465 self.qemu_opt += ' -drive if=pflash,format=%s,file=%s' % (format, ovmf) 1466 1467 self.qemu_opt += ' ' + self.qemu_opt_script 1468 1469 if self.ovmf_secboot_pkkek1: 1470 # Provide the Platform Key and first Key Exchange Key certificate as an 1471 # OEM string in the SMBIOS Type 11 table. Prepend the certificate string 1472 # with "application prefix" of the EnrollDefaultKeys.efi application 1473 self.qemu_opt += ' -smbios type=11,value=4e32566d-8e9e-4f52-81d3-5bb9715f9727:' \ 1474 + self.ovmf_secboot_pkkek1 1475 1476 # Append qemuparams to override previous settings 1477 if self.qemuparams: 1478 self.qemu_opt += ' ' + self.qemuparams 1479 1480 if self.snapshot: 1481 self.qemu_opt += " -snapshot" 1482 1483 self.setup_serial() 1484 self.setup_vga() 1485 1486 def start_qemu(self): 1487 import shlex 1488 if self.kernel: 1489 kernel_opts = "-kernel %s" % (self.kernel) 1490 if self.get('QB_KERNEL_CMDLINE') == "none": 1491 if self.bootparams: 1492 kernel_opts += " -append '%s'" % (self.bootparams) 1493 else: 1494 kernel_opts += " -append '%s %s %s %s'" % (self.kernel_cmdline, 1495 self.kernel_cmdline_script, self.get('QB_KERNEL_CMDLINE_APPEND'), 1496 self.bootparams) 1497 if self.dtb: 1498 kernel_opts += " -dtb %s" % self.dtb 1499 else: 1500 kernel_opts = "" 1501 1502 if self.bios: 1503 self.qemu_opt += " -bios %s" % self.bios 1504 1505 cmd = "%s %s" % (self.qemu_opt, kernel_opts) 1506 cmds = shlex.split(cmd) 1507 logger.info('Running %s\n' % cmd) 1508 pass_fds = [] 1509 if self.taplock_descriptor: 1510 pass_fds = [self.taplock_descriptor.fileno()] 1511 if len(self.portlocks): 1512 for descriptor in self.portlocks.values(): 1513 pass_fds.append(descriptor.fileno()) 1514 process = subprocess.Popen(cmds, stderr=subprocess.PIPE, pass_fds=pass_fds) 1515 self.qemupid = process.pid 1516 retcode = process.wait() 1517 if retcode: 1518 if retcode == -signal.SIGTERM: 1519 logger.info("Qemu terminated by SIGTERM") 1520 else: 1521 logger.error("Failed to run qemu: %s", process.stderr.read().decode()) 1522 1523 def cleanup(self): 1524 if self.cleaned: 1525 return 1526 1527 # avoid dealing with SIGTERM when cleanup function is running 1528 signal.signal(signal.SIGTERM, signal.SIG_IGN) 1529 1530 logger.info("Cleaning up") 1531 if self.cleantap: 1532 cmd = ('sudo', self.qemuifdown, self.tap, self.bindir_native) 1533 logger.debug('Running %s' % str(cmd)) 1534 subprocess.check_call(cmd) 1535 self.release_taplock() 1536 self.release_portlock() 1537 1538 if self.nfs_running: 1539 logger.info("Shutting down the userspace NFS server...") 1540 cmd = ("runqemu-export-rootfs", "stop", self.rootfs) 1541 logger.debug('Running %s' % str(cmd)) 1542 subprocess.check_call(cmd) 1543 1544 if self.saved_stty: 1545 subprocess.check_call(("stty", self.saved_stty)) 1546 1547 if self.cleanup_files: 1548 for ent in self.cleanup_files: 1549 logger.info('Removing %s' % ent) 1550 if os.path.isfile(ent): 1551 os.remove(ent) 1552 else: 1553 shutil.rmtree(ent) 1554 1555 self.cleaned = True 1556 1557 def run_bitbake_env(self, mach=None): 1558 bitbake = shutil.which('bitbake') 1559 if not bitbake: 1560 return 1561 1562 if not mach: 1563 mach = self.get('MACHINE') 1564 1565 multiconfig = self.get('MULTICONFIG') 1566 if multiconfig: 1567 multiconfig = "mc:%s" % multiconfig 1568 1569 if mach: 1570 cmd = 'MACHINE=%s bitbake -e %s' % (mach, multiconfig) 1571 else: 1572 cmd = 'bitbake -e %s' % multiconfig 1573 1574 logger.info('Running %s...' % cmd) 1575 return subprocess.check_output(cmd, shell=True).decode('utf-8') 1576 1577 def load_bitbake_env(self, mach=None): 1578 if self.bitbake_e: 1579 return 1580 1581 try: 1582 self.bitbake_e = self.run_bitbake_env(mach=mach) 1583 except subprocess.CalledProcessError as err: 1584 self.bitbake_e = '' 1585 logger.warning("Couldn't run 'bitbake -e' to gather environment information:\n%s" % err.output.decode('utf-8')) 1586 1587 def validate_combos(self): 1588 if (self.fstype in self.vmtypes) and self.kernel: 1589 raise RunQemuError("%s doesn't need kernel %s!" % (self.fstype, self.kernel)) 1590 1591 @property 1592 def bindir_native(self): 1593 result = self.get('STAGING_BINDIR_NATIVE') 1594 if result and os.path.exists(result): 1595 return result 1596 1597 cmd = ['bitbake', '-e'] 1598 multiconfig = self.get('MULTICONFIG') 1599 if multiconfig: 1600 cmd.append('mc:%s:qemu-helper-native' % multiconfig) 1601 else: 1602 cmd.append('qemu-helper-native') 1603 1604 logger.info('Running %s...' % str(cmd)) 1605 out = subprocess.check_output(cmd).decode('utf-8') 1606 1607 match = re.search('^STAGING_BINDIR_NATIVE="(.*)"', out, re.M) 1608 if match: 1609 result = match.group(1) 1610 if os.path.exists(result): 1611 self.set('STAGING_BINDIR_NATIVE', result) 1612 return result 1613 raise RunQemuError("Native sysroot directory %s doesn't exist" % result) 1614 else: 1615 raise RunQemuError("Can't find STAGING_BINDIR_NATIVE in '%s' output" % cmd) 1616 1617 1618def main(): 1619 if "help" in sys.argv or '-h' in sys.argv or '--help' in sys.argv: 1620 print_usage() 1621 return 0 1622 try: 1623 config = BaseConfig() 1624 1625 renice = os.path.expanduser("~/bin/runqemu-renice") 1626 if os.path.exists(renice): 1627 logger.info('Using %s to renice' % renice) 1628 subprocess.check_call([renice, str(os.getpid())]) 1629 1630 def sigterm_handler(signum, frame): 1631 logger.info("SIGTERM received") 1632 if config.qemupid: 1633 os.kill(config.qemupid, signal.SIGTERM) 1634 config.cleanup() 1635 # Deliberately ignore the return code of 'tput smam'. 1636 subprocess.call(["tput", "smam"]) 1637 signal.signal(signal.SIGTERM, sigterm_handler) 1638 1639 config.check_args() 1640 config.read_qemuboot() 1641 config.check_and_set() 1642 # Check whether the combos is valid or not 1643 config.validate_combos() 1644 config.print_config() 1645 config.setup_network() 1646 config.setup_rootfs() 1647 config.setup_final() 1648 config.start_qemu() 1649 except RunQemuError as err: 1650 logger.error(err) 1651 return 1 1652 except Exception as err: 1653 import traceback 1654 traceback.print_exc() 1655 return 1 1656 finally: 1657 config.cleanup() 1658 # Deliberately ignore the return code of 'tput smam'. 1659 subprocess.call(["tput", "smam"]) 1660 1661if __name__ == "__main__": 1662 sys.exit(main()) 1663