1# 2# VM testing base class 3# 4# Copyright 2017-2019 Red Hat Inc. 5# 6# Authors: 7# Fam Zheng <famz@redhat.com> 8# Gerd Hoffmann <kraxel@redhat.com> 9# 10# This code is licensed under the GPL version 2 or later. See 11# the COPYING file in the top-level directory. 12# 13 14import os 15import re 16import sys 17import socket 18import logging 19import time 20import datetime 21sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 22from qemu.accel import kvm_available 23from qemu.machine import QEMUMachine 24import subprocess 25import hashlib 26import optparse 27import atexit 28import tempfile 29import shutil 30import multiprocessing 31import traceback 32 33SSH_KEY = open(os.path.join(os.path.dirname(__file__), 34 "..", "keys", "id_rsa")).read() 35SSH_PUB_KEY = open(os.path.join(os.path.dirname(__file__), 36 "..", "keys", "id_rsa.pub")).read() 37 38class BaseVM(object): 39 GUEST_USER = "qemu" 40 GUEST_PASS = "qemupass" 41 ROOT_PASS = "qemupass" 42 43 envvars = [ 44 "https_proxy", 45 "http_proxy", 46 "ftp_proxy", 47 "no_proxy", 48 ] 49 50 # The script to run in the guest that builds QEMU 51 BUILD_SCRIPT = "" 52 # The guest name, to be overridden by subclasses 53 name = "#base" 54 # The guest architecture, to be overridden by subclasses 55 arch = "#arch" 56 # command to halt the guest, can be overridden by subclasses 57 poweroff = "poweroff" 58 # enable IPv6 networking 59 ipv6 = True 60 # Scale up some timeouts under TCG. 61 # 4 is arbitrary, but greater than 2, 62 # since we found we need to wait more than twice as long. 63 tcg_ssh_timeout_multiplier = 4 64 def __init__(self, debug=False, vcpus=None, genisoimage=None, 65 build_path=None): 66 self._guest = None 67 self._genisoimage = genisoimage 68 self._build_path = build_path 69 self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-", 70 suffix=".tmp", 71 dir=".")) 72 atexit.register(shutil.rmtree, self._tmpdir) 73 74 self._ssh_key_file = os.path.join(self._tmpdir, "id_rsa") 75 open(self._ssh_key_file, "w").write(SSH_KEY) 76 subprocess.check_call(["chmod", "600", self._ssh_key_file]) 77 78 self._ssh_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub") 79 open(self._ssh_pub_key_file, "w").write(SSH_PUB_KEY) 80 81 self.debug = debug 82 self._stderr = sys.stderr 83 self._devnull = open(os.devnull, "w") 84 if self.debug: 85 self._stdout = sys.stdout 86 else: 87 self._stdout = self._devnull 88 self._args = [ \ 89 "-nodefaults", "-m", "4G", 90 "-cpu", "max", 91 "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22" + 92 (",ipv6=no" if not self.ipv6 else ""), 93 "-device", "virtio-net-pci,netdev=vnet", 94 "-vnc", "127.0.0.1:0,to=20"] 95 if vcpus and vcpus > 1: 96 self._args += ["-smp", "%d" % vcpus] 97 if kvm_available(self.arch): 98 self._args += ["-enable-kvm"] 99 else: 100 logging.info("KVM not available, not using -enable-kvm") 101 self._data_args = [] 102 103 def _download_with_cache(self, url, sha256sum=None, sha512sum=None): 104 def check_sha256sum(fname): 105 if not sha256sum: 106 return True 107 checksum = subprocess.check_output(["sha256sum", fname]).split()[0] 108 return sha256sum == checksum.decode("utf-8") 109 110 def check_sha512sum(fname): 111 if not sha512sum: 112 return True 113 checksum = subprocess.check_output(["sha512sum", fname]).split()[0] 114 return sha512sum == checksum.decode("utf-8") 115 116 cache_dir = os.path.expanduser("~/.cache/qemu-vm/download") 117 if not os.path.exists(cache_dir): 118 os.makedirs(cache_dir) 119 fname = os.path.join(cache_dir, 120 hashlib.sha1(url.encode("utf-8")).hexdigest()) 121 if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname): 122 return fname 123 logging.debug("Downloading %s to %s...", url, fname) 124 subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"], 125 stdout=self._stdout, stderr=self._stderr) 126 os.rename(fname + ".download", fname) 127 return fname 128 129 def _ssh_do(self, user, cmd, check): 130 ssh_cmd = ["ssh", 131 "-t", 132 "-o", "StrictHostKeyChecking=no", 133 "-o", "UserKnownHostsFile=" + os.devnull, 134 "-o", "ConnectTimeout=1", 135 "-p", self.ssh_port, "-i", self._ssh_key_file] 136 # If not in debug mode, set ssh to quiet mode to 137 # avoid printing the results of commands. 138 if not self.debug: 139 ssh_cmd.append("-q") 140 for var in self.envvars: 141 ssh_cmd += ['-o', "SendEnv=%s" % var ] 142 assert not isinstance(cmd, str) 143 ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd) 144 logging.debug("ssh_cmd: %s", " ".join(ssh_cmd)) 145 r = subprocess.call(ssh_cmd) 146 if check and r != 0: 147 raise Exception("SSH command failed: %s" % cmd) 148 return r 149 150 def ssh(self, *cmd): 151 return self._ssh_do(self.GUEST_USER, cmd, False) 152 153 def ssh_root(self, *cmd): 154 return self._ssh_do("root", cmd, False) 155 156 def ssh_check(self, *cmd): 157 self._ssh_do(self.GUEST_USER, cmd, True) 158 159 def ssh_root_check(self, *cmd): 160 self._ssh_do("root", cmd, True) 161 162 def build_image(self, img): 163 raise NotImplementedError 164 165 def exec_qemu_img(self, *args): 166 cmd = [os.environ.get("QEMU_IMG", "qemu-img")] 167 cmd.extend(list(args)) 168 subprocess.check_call(cmd) 169 170 def add_source_dir(self, src_dir): 171 name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5] 172 tarfile = os.path.join(self._tmpdir, name + ".tar") 173 logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir) 174 subprocess.check_call(["./scripts/archive-source.sh", tarfile], 175 cwd=src_dir, stdin=self._devnull, 176 stdout=self._stdout, stderr=self._stderr) 177 self._data_args += ["-drive", 178 "file=%s,if=none,id=%s,cache=writeback,format=raw" % \ 179 (tarfile, name), 180 "-device", 181 "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)] 182 183 def boot(self, img, extra_args=[]): 184 args = self._args + [ 185 "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img, 186 "-device", "virtio-blk,drive=drive0,bootindex=0"] 187 args += self._data_args + extra_args 188 logging.debug("QEMU args: %s", " ".join(args)) 189 qemu_path = get_qemu_path(self.arch, self._build_path) 190 guest = QEMUMachine(binary=qemu_path, args=args) 191 guest.set_machine('pc') 192 guest.set_console() 193 try: 194 guest.launch() 195 except: 196 logging.error("Failed to launch QEMU, command line:") 197 logging.error(" ".join([qemu_path] + args)) 198 logging.error("Log:") 199 logging.error(guest.get_log()) 200 logging.error("QEMU version >= 2.10 is required") 201 raise 202 atexit.register(self.shutdown) 203 self._guest = guest 204 usernet_info = guest.qmp("human-monitor-command", 205 command_line="info usernet") 206 self.ssh_port = None 207 for l in usernet_info["return"].splitlines(): 208 fields = l.split() 209 if "TCP[HOST_FORWARD]" in fields and "22" in fields: 210 self.ssh_port = l.split()[3] 211 if not self.ssh_port: 212 raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \ 213 usernet_info) 214 215 def console_init(self, timeout = 120): 216 vm = self._guest 217 vm.console_socket.settimeout(timeout) 218 self.console_raw_path = os.path.join(vm._temp_dir, 219 vm._name + "-console.raw") 220 self.console_raw_file = open(self.console_raw_path, 'wb') 221 222 def console_log(self, text): 223 for line in re.split("[\r\n]", text): 224 # filter out terminal escape sequences 225 line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line) 226 line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line) 227 # replace unprintable chars 228 line = re.sub("\x1b", "<esc>", line) 229 line = re.sub("[\x00-\x1f]", ".", line) 230 line = re.sub("[\x80-\xff]", ".", line) 231 if line == "": 232 continue 233 # log console line 234 sys.stderr.write("con recv: %s\n" % line) 235 236 def console_wait(self, expect, expectalt = None): 237 vm = self._guest 238 output = "" 239 while True: 240 try: 241 chars = vm.console_socket.recv(1) 242 if self.console_raw_file: 243 self.console_raw_file.write(chars) 244 self.console_raw_file.flush() 245 except socket.timeout: 246 sys.stderr.write("console: *** read timeout ***\n") 247 sys.stderr.write("console: waiting for: '%s'\n" % expect) 248 if not expectalt is None: 249 sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt) 250 sys.stderr.write("console: line buffer:\n") 251 sys.stderr.write("\n") 252 self.console_log(output.rstrip()) 253 sys.stderr.write("\n") 254 raise 255 output += chars.decode("latin1") 256 if expect in output: 257 break 258 if not expectalt is None and expectalt in output: 259 break 260 if "\r" in output or "\n" in output: 261 lines = re.split("[\r\n]", output) 262 output = lines.pop() 263 if self.debug: 264 self.console_log("\n".join(lines)) 265 if self.debug: 266 self.console_log(output) 267 if not expectalt is None and expectalt in output: 268 return False 269 return True 270 271 def console_consume(self): 272 vm = self._guest 273 output = "" 274 vm.console_socket.setblocking(0) 275 while True: 276 try: 277 chars = vm.console_socket.recv(1) 278 except: 279 break 280 output += chars.decode("latin1") 281 if "\r" in output or "\n" in output: 282 lines = re.split("[\r\n]", output) 283 output = lines.pop() 284 if self.debug: 285 self.console_log("\n".join(lines)) 286 if self.debug: 287 self.console_log(output) 288 vm.console_socket.setblocking(1) 289 290 def console_send(self, command): 291 vm = self._guest 292 if self.debug: 293 logline = re.sub("\n", "<enter>", command) 294 logline = re.sub("[\x00-\x1f]", ".", logline) 295 sys.stderr.write("con send: %s\n" % logline) 296 for char in list(command): 297 vm.console_socket.send(char.encode("utf-8")) 298 time.sleep(0.01) 299 300 def console_wait_send(self, wait, command): 301 self.console_wait(wait) 302 self.console_send(command) 303 304 def console_ssh_init(self, prompt, user, pw): 305 sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" % SSH_PUB_KEY.rstrip() 306 self.console_wait_send("login:", "%s\n" % user) 307 self.console_wait_send("Password:", "%s\n" % pw) 308 self.console_wait_send(prompt, "mkdir .ssh\n") 309 self.console_wait_send(prompt, sshkey_cmd) 310 self.console_wait_send(prompt, "chmod 755 .ssh\n") 311 self.console_wait_send(prompt, "chmod 644 .ssh/authorized_keys\n") 312 313 def console_sshd_config(self, prompt): 314 self.console_wait(prompt) 315 self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n") 316 for var in self.envvars: 317 self.console_wait(prompt) 318 self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var) 319 320 def print_step(self, text): 321 sys.stderr.write("### %s ...\n" % text) 322 323 def wait_ssh(self, wait_root=False, seconds=300, cmd="exit 0"): 324 # Allow more time for VM to boot under TCG. 325 if not kvm_available(self.arch): 326 seconds *= self.tcg_ssh_timeout_multiplier 327 starttime = datetime.datetime.now() 328 endtime = starttime + datetime.timedelta(seconds=seconds) 329 cmd_success = False 330 while datetime.datetime.now() < endtime: 331 if wait_root and self.ssh_root(cmd) == 0: 332 cmd_success = True 333 break 334 elif self.ssh(cmd) == 0: 335 cmd_success = True 336 break 337 seconds = (endtime - datetime.datetime.now()).total_seconds() 338 logging.debug("%ds before timeout", seconds) 339 time.sleep(1) 340 if not cmd_success: 341 raise Exception("Timeout while waiting for guest ssh") 342 343 def shutdown(self): 344 self._guest.shutdown() 345 346 def wait(self): 347 self._guest.wait() 348 349 def graceful_shutdown(self): 350 self.ssh_root(self.poweroff) 351 self._guest.wait() 352 353 def qmp(self, *args, **kwargs): 354 return self._guest.qmp(*args, **kwargs) 355 356 def gen_cloud_init_iso(self): 357 cidir = self._tmpdir 358 mdata = open(os.path.join(cidir, "meta-data"), "w") 359 name = self.name.replace(".","-") 360 mdata.writelines(["instance-id: {}-vm-0\n".format(name), 361 "local-hostname: {}-guest\n".format(name)]) 362 mdata.close() 363 udata = open(os.path.join(cidir, "user-data"), "w") 364 print("guest user:pw {}:{}".format(self.GUEST_USER, 365 self.GUEST_PASS)) 366 udata.writelines(["#cloud-config\n", 367 "chpasswd:\n", 368 " list: |\n", 369 " root:%s\n" % self.ROOT_PASS, 370 " %s:%s\n" % (self.GUEST_USER, 371 self.GUEST_PASS), 372 " expire: False\n", 373 "users:\n", 374 " - name: %s\n" % self.GUEST_USER, 375 " sudo: ALL=(ALL) NOPASSWD:ALL\n", 376 " ssh-authorized-keys:\n", 377 " - %s\n" % SSH_PUB_KEY, 378 " - name: root\n", 379 " ssh-authorized-keys:\n", 380 " - %s\n" % SSH_PUB_KEY, 381 "locale: en_US.UTF-8\n"]) 382 proxy = os.environ.get("http_proxy") 383 if not proxy is None: 384 udata.writelines(["apt:\n", 385 " proxy: %s" % proxy]) 386 udata.close() 387 subprocess.check_call([self._genisoimage, "-output", "cloud-init.iso", 388 "-volid", "cidata", "-joliet", "-rock", 389 "user-data", "meta-data"], 390 cwd=cidir, 391 stdin=self._devnull, stdout=self._stdout, 392 stderr=self._stdout) 393 394 return os.path.join(cidir, "cloud-init.iso") 395 396def get_qemu_path(arch, build_path=None): 397 """Fetch the path to the qemu binary.""" 398 # If QEMU environment variable set, it takes precedence 399 if "QEMU" in os.environ: 400 qemu_path = os.environ["QEMU"] 401 elif build_path: 402 qemu_path = os.path.join(build_path, arch + "-softmmu") 403 qemu_path = os.path.join(qemu_path, "qemu-system-" + arch) 404 else: 405 # Default is to use system path for qemu. 406 qemu_path = "qemu-system-" + arch 407 return qemu_path 408 409def parse_args(vmcls): 410 411 def get_default_jobs(): 412 if kvm_available(vmcls.arch): 413 return multiprocessing.cpu_count() // 2 414 else: 415 return 1 416 417 parser = optparse.OptionParser( 418 description="VM test utility. Exit codes: " 419 "0 = success, " 420 "1 = command line error, " 421 "2 = environment initialization failed, " 422 "3 = test command failed") 423 parser.add_option("--debug", "-D", action="store_true", 424 help="enable debug output") 425 parser.add_option("--image", "-i", default="%s.img" % vmcls.name, 426 help="image file name") 427 parser.add_option("--force", "-f", action="store_true", 428 help="force build image even if image exists") 429 parser.add_option("--jobs", type=int, default=get_default_jobs(), 430 help="number of virtual CPUs") 431 parser.add_option("--verbose", "-V", action="store_true", 432 help="Pass V=1 to builds within the guest") 433 parser.add_option("--build-image", "-b", action="store_true", 434 help="build image") 435 parser.add_option("--build-qemu", 436 help="build QEMU from source in guest") 437 parser.add_option("--build-target", 438 help="QEMU build target", default="check") 439 parser.add_option("--build-path", default=None, 440 help="Path of build directory, "\ 441 "for using build tree QEMU binary. ") 442 parser.add_option("--interactive", "-I", action="store_true", 443 help="Interactively run command") 444 parser.add_option("--snapshot", "-s", action="store_true", 445 help="run tests with a snapshot") 446 parser.add_option("--genisoimage", default="genisoimage", 447 help="iso imaging tool") 448 parser.disable_interspersed_args() 449 return parser.parse_args() 450 451def main(vmcls): 452 try: 453 args, argv = parse_args(vmcls) 454 if not argv and not args.build_qemu and not args.build_image: 455 print("Nothing to do?") 456 return 1 457 logging.basicConfig(level=(logging.DEBUG if args.debug 458 else logging.WARN)) 459 vm = vmcls(debug=args.debug, vcpus=args.jobs, 460 genisoimage=args.genisoimage, build_path=args.build_path) 461 if args.build_image: 462 if os.path.exists(args.image) and not args.force: 463 sys.stderr.writelines(["Image file exists: %s\n" % args.image, 464 "Use --force option to overwrite\n"]) 465 return 1 466 return vm.build_image(args.image) 467 if args.build_qemu: 468 vm.add_source_dir(args.build_qemu) 469 cmd = [vm.BUILD_SCRIPT.format( 470 configure_opts = " ".join(argv), 471 jobs=int(args.jobs), 472 target=args.build_target, 473 verbose = "V=1" if args.verbose else "")] 474 else: 475 cmd = argv 476 img = args.image 477 if args.snapshot: 478 img += ",snapshot=on" 479 vm.boot(img) 480 vm.wait_ssh() 481 except Exception as e: 482 if isinstance(e, SystemExit) and e.code == 0: 483 return 0 484 sys.stderr.write("Failed to prepare guest environment\n") 485 traceback.print_exc() 486 return 2 487 488 exitcode = 0 489 if vm.ssh(*cmd) != 0: 490 exitcode = 3 491 if args.interactive: 492 vm.ssh() 493 494 if not args.snapshot: 495 vm.graceful_shutdown() 496 497 return exitcode 498