1# 2# VM testing base class 3# 4# Copyright 2017-2019 Red Hat Inc. 5# 6# Authors: 7# Fam Zheng <famz@redhat.com> 8# Gerd Hoffmann <kraxel@redhat.com> 9# 10# This code is licensed under the GPL version 2 or later. See 11# the COPYING file in the top-level directory. 12# 13 14import os 15import re 16import sys 17import socket 18import logging 19import time 20import datetime 21sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 22from qemu.accel import kvm_available 23from qemu.machine import QEMUMachine 24import subprocess 25import hashlib 26import optparse 27import atexit 28import tempfile 29import shutil 30import multiprocessing 31import traceback 32 33SSH_KEY = open(os.path.join(os.path.dirname(__file__), 34 "..", "keys", "id_rsa")).read() 35SSH_PUB_KEY = open(os.path.join(os.path.dirname(__file__), 36 "..", "keys", "id_rsa.pub")).read() 37 38class BaseVM(object): 39 GUEST_USER = "qemu" 40 GUEST_PASS = "qemupass" 41 ROOT_PASS = "qemupass" 42 43 envvars = [ 44 "https_proxy", 45 "http_proxy", 46 "ftp_proxy", 47 "no_proxy", 48 ] 49 50 # The script to run in the guest that builds QEMU 51 BUILD_SCRIPT = "" 52 # The guest name, to be overridden by subclasses 53 name = "#base" 54 # The guest architecture, to be overridden by subclasses 55 arch = "#arch" 56 # command to halt the guest, can be overridden by subclasses 57 poweroff = "poweroff" 58 # enable IPv6 networking 59 ipv6 = True 60 # Scale up some timeouts under TCG. 61 # 4 is arbitrary, but greater than 2, 62 # since we found we need to wait more than twice as long. 63 tcg_ssh_timeout_multiplier = 4 64 def __init__(self, debug=False, vcpus=None): 65 self._guest = None 66 self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-", 67 suffix=".tmp", 68 dir=".")) 69 atexit.register(shutil.rmtree, self._tmpdir) 70 71 self._ssh_key_file = os.path.join(self._tmpdir, "id_rsa") 72 open(self._ssh_key_file, "w").write(SSH_KEY) 73 subprocess.check_call(["chmod", "600", self._ssh_key_file]) 74 75 self._ssh_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub") 76 open(self._ssh_pub_key_file, "w").write(SSH_PUB_KEY) 77 78 self.debug = debug 79 self._stderr = sys.stderr 80 self._devnull = open(os.devnull, "w") 81 if self.debug: 82 self._stdout = sys.stdout 83 else: 84 self._stdout = self._devnull 85 self._args = [ \ 86 "-nodefaults", "-m", "4G", 87 "-cpu", "max", 88 "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22" + 89 (",ipv6=no" if not self.ipv6 else ""), 90 "-device", "virtio-net-pci,netdev=vnet", 91 "-vnc", "127.0.0.1:0,to=20"] 92 if vcpus and vcpus > 1: 93 self._args += ["-smp", "%d" % vcpus] 94 if kvm_available(self.arch): 95 self._args += ["-enable-kvm"] 96 else: 97 logging.info("KVM not available, not using -enable-kvm") 98 self._data_args = [] 99 100 def _download_with_cache(self, url, sha256sum=None, sha512sum=None): 101 def check_sha256sum(fname): 102 if not sha256sum: 103 return True 104 checksum = subprocess.check_output(["sha256sum", fname]).split()[0] 105 return sha256sum == checksum.decode("utf-8") 106 107 def check_sha512sum(fname): 108 if not sha512sum: 109 return True 110 checksum = subprocess.check_output(["sha512sum", fname]).split()[0] 111 return sha512sum == checksum.decode("utf-8") 112 113 cache_dir = os.path.expanduser("~/.cache/qemu-vm/download") 114 if not os.path.exists(cache_dir): 115 os.makedirs(cache_dir) 116 fname = os.path.join(cache_dir, 117 hashlib.sha1(url.encode("utf-8")).hexdigest()) 118 if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname): 119 return fname 120 logging.debug("Downloading %s to %s...", url, fname) 121 subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"], 122 stdout=self._stdout, stderr=self._stderr) 123 os.rename(fname + ".download", fname) 124 return fname 125 126 def _ssh_do(self, user, cmd, check): 127 ssh_cmd = ["ssh", 128 "-t", 129 "-o", "StrictHostKeyChecking=no", 130 "-o", "UserKnownHostsFile=" + os.devnull, 131 "-o", "ConnectTimeout=1", 132 "-p", self.ssh_port, "-i", self._ssh_key_file] 133 # If not in debug mode, set ssh to quiet mode to 134 # avoid printing the results of commands. 135 if not self.debug: 136 ssh_cmd.append("-q") 137 for var in self.envvars: 138 ssh_cmd += ['-o', "SendEnv=%s" % var ] 139 assert not isinstance(cmd, str) 140 ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd) 141 logging.debug("ssh_cmd: %s", " ".join(ssh_cmd)) 142 r = subprocess.call(ssh_cmd) 143 if check and r != 0: 144 raise Exception("SSH command failed: %s" % cmd) 145 return r 146 147 def ssh(self, *cmd): 148 return self._ssh_do(self.GUEST_USER, cmd, False) 149 150 def ssh_root(self, *cmd): 151 return self._ssh_do("root", cmd, False) 152 153 def ssh_check(self, *cmd): 154 self._ssh_do(self.GUEST_USER, cmd, True) 155 156 def ssh_root_check(self, *cmd): 157 self._ssh_do("root", cmd, True) 158 159 def build_image(self, img): 160 raise NotImplementedError 161 162 def exec_qemu_img(self, *args): 163 cmd = [os.environ.get("QEMU_IMG", "qemu-img")] 164 cmd.extend(list(args)) 165 subprocess.check_call(cmd) 166 167 def add_source_dir(self, src_dir): 168 name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5] 169 tarfile = os.path.join(self._tmpdir, name + ".tar") 170 logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir) 171 subprocess.check_call(["./scripts/archive-source.sh", tarfile], 172 cwd=src_dir, stdin=self._devnull, 173 stdout=self._stdout, stderr=self._stderr) 174 self._data_args += ["-drive", 175 "file=%s,if=none,id=%s,cache=writeback,format=raw" % \ 176 (tarfile, name), 177 "-device", 178 "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)] 179 180 def boot(self, img, extra_args=[]): 181 args = self._args + [ 182 "-device", "VGA", 183 "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img, 184 "-device", "virtio-blk,drive=drive0,bootindex=0"] 185 args += self._data_args + extra_args 186 logging.debug("QEMU args: %s", " ".join(args)) 187 qemu_bin = os.environ.get("QEMU", "qemu-system-" + self.arch) 188 guest = QEMUMachine(binary=qemu_bin, args=args) 189 guest.set_machine('pc') 190 guest.set_console() 191 try: 192 guest.launch() 193 except: 194 logging.error("Failed to launch QEMU, command line:") 195 logging.error(" ".join([qemu_bin] + args)) 196 logging.error("Log:") 197 logging.error(guest.get_log()) 198 logging.error("QEMU version >= 2.10 is required") 199 raise 200 atexit.register(self.shutdown) 201 self._guest = guest 202 usernet_info = guest.qmp("human-monitor-command", 203 command_line="info usernet") 204 self.ssh_port = None 205 for l in usernet_info["return"].splitlines(): 206 fields = l.split() 207 if "TCP[HOST_FORWARD]" in fields and "22" in fields: 208 self.ssh_port = l.split()[3] 209 if not self.ssh_port: 210 raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \ 211 usernet_info) 212 213 def console_init(self, timeout = 120): 214 vm = self._guest 215 vm.console_socket.settimeout(timeout) 216 217 def console_log(self, text): 218 for line in re.split("[\r\n]", text): 219 # filter out terminal escape sequences 220 line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line) 221 line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line) 222 # replace unprintable chars 223 line = re.sub("\x1b", "<esc>", line) 224 line = re.sub("[\x00-\x1f]", ".", line) 225 line = re.sub("[\x80-\xff]", ".", line) 226 if line == "": 227 continue 228 # log console line 229 sys.stderr.write("con recv: %s\n" % line) 230 231 def console_wait(self, expect, expectalt = None): 232 vm = self._guest 233 output = "" 234 while True: 235 try: 236 chars = vm.console_socket.recv(1) 237 except socket.timeout: 238 sys.stderr.write("console: *** read timeout ***\n") 239 sys.stderr.write("console: waiting for: '%s'\n" % expect) 240 if not expectalt is None: 241 sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt) 242 sys.stderr.write("console: line buffer:\n") 243 sys.stderr.write("\n") 244 self.console_log(output.rstrip()) 245 sys.stderr.write("\n") 246 raise 247 output += chars.decode("latin1") 248 if expect in output: 249 break 250 if not expectalt is None and expectalt in output: 251 break 252 if "\r" in output or "\n" in output: 253 lines = re.split("[\r\n]", output) 254 output = lines.pop() 255 if self.debug: 256 self.console_log("\n".join(lines)) 257 if self.debug: 258 self.console_log(output) 259 if not expectalt is None and expectalt in output: 260 return False 261 return True 262 263 def console_consume(self): 264 vm = self._guest 265 output = "" 266 vm.console_socket.setblocking(0) 267 while True: 268 try: 269 chars = vm.console_socket.recv(1) 270 except: 271 break 272 output += chars.decode("latin1") 273 if "\r" in output or "\n" in output: 274 lines = re.split("[\r\n]", output) 275 output = lines.pop() 276 if self.debug: 277 self.console_log("\n".join(lines)) 278 if self.debug: 279 self.console_log(output) 280 vm.console_socket.setblocking(1) 281 282 def console_send(self, command): 283 vm = self._guest 284 if self.debug: 285 logline = re.sub("\n", "<enter>", command) 286 logline = re.sub("[\x00-\x1f]", ".", logline) 287 sys.stderr.write("con send: %s\n" % logline) 288 for char in list(command): 289 vm.console_socket.send(char.encode("utf-8")) 290 time.sleep(0.01) 291 292 def console_wait_send(self, wait, command): 293 self.console_wait(wait) 294 self.console_send(command) 295 296 def console_ssh_init(self, prompt, user, pw): 297 sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" % SSH_PUB_KEY.rstrip() 298 self.console_wait_send("login:", "%s\n" % user) 299 self.console_wait_send("Password:", "%s\n" % pw) 300 self.console_wait_send(prompt, "mkdir .ssh\n") 301 self.console_wait_send(prompt, sshkey_cmd) 302 self.console_wait_send(prompt, "chmod 755 .ssh\n") 303 self.console_wait_send(prompt, "chmod 644 .ssh/authorized_keys\n") 304 305 def console_sshd_config(self, prompt): 306 self.console_wait(prompt) 307 self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n") 308 for var in self.envvars: 309 self.console_wait(prompt) 310 self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var) 311 312 def print_step(self, text): 313 sys.stderr.write("### %s ...\n" % text) 314 315 def wait_ssh(self, wait_root=False, seconds=300): 316 # Allow more time for VM to boot under TCG. 317 if not kvm_available(self.arch): 318 seconds *= self.tcg_ssh_timeout_multiplier 319 starttime = datetime.datetime.now() 320 endtime = starttime + datetime.timedelta(seconds=seconds) 321 guest_up = False 322 while datetime.datetime.now() < endtime: 323 if wait_root and self.ssh_root("exit 0") == 0: 324 guest_up = True 325 break 326 elif self.ssh("exit 0") == 0: 327 guest_up = True 328 break 329 seconds = (endtime - datetime.datetime.now()).total_seconds() 330 logging.debug("%ds before timeout", seconds) 331 time.sleep(1) 332 if not guest_up: 333 raise Exception("Timeout while waiting for guest ssh") 334 335 def shutdown(self): 336 self._guest.shutdown() 337 338 def wait(self): 339 self._guest.wait() 340 341 def graceful_shutdown(self): 342 self.ssh_root(self.poweroff) 343 self._guest.wait() 344 345 def qmp(self, *args, **kwargs): 346 return self._guest.qmp(*args, **kwargs) 347 348 def gen_cloud_init_iso(self): 349 cidir = self._tmpdir 350 mdata = open(os.path.join(cidir, "meta-data"), "w") 351 name = self.name.replace(".","-") 352 mdata.writelines(["instance-id: {}-vm-0\n".format(name), 353 "local-hostname: {}-guest\n".format(name)]) 354 mdata.close() 355 udata = open(os.path.join(cidir, "user-data"), "w") 356 print("guest user:pw {}:{}".format(self._config['guest_user'], 357 self._config['guest_pass'])) 358 udata.writelines(["#cloud-config\n", 359 "chpasswd:\n", 360 " list: |\n", 361 " root:%s\n" % self._config['root_pass'], 362 " %s:%s\n" % (self._config['guest_user'], 363 self._config['guest_pass']), 364 " expire: False\n", 365 "users:\n", 366 " - name: %s\n" % self._config['guest_user'], 367 " sudo: ALL=(ALL) NOPASSWD:ALL\n", 368 " ssh-authorized-keys:\n", 369 " - %s\n" % self._config['ssh_pub_key'], 370 " - name: root\n", 371 " ssh-authorized-keys:\n", 372 " - %s\n" % self._config['ssh_pub_key'], 373 "locale: en_US.UTF-8\n"]) 374 proxy = os.environ.get("http_proxy") 375 if not proxy is None: 376 udata.writelines(["apt:\n", 377 " proxy: %s" % proxy]) 378 udata.close() 379 subprocess.check_call(["genisoimage", "-output", "cloud-init.iso", 380 "-volid", "cidata", "-joliet", "-rock", 381 "user-data", "meta-data"], 382 cwd=cidir, 383 stdin=self._devnull, stdout=self._stdout, 384 stderr=self._stdout) 385 386 return os.path.join(cidir, "cloud-init.iso") 387 388def parse_args(vmcls): 389 390 def get_default_jobs(): 391 if kvm_available(vmcls.arch): 392 return multiprocessing.cpu_count() // 2 393 else: 394 return 1 395 396 parser = optparse.OptionParser( 397 description="VM test utility. Exit codes: " 398 "0 = success, " 399 "1 = command line error, " 400 "2 = environment initialization failed, " 401 "3 = test command failed") 402 parser.add_option("--debug", "-D", action="store_true", 403 help="enable debug output") 404 parser.add_option("--image", "-i", default="%s.img" % vmcls.name, 405 help="image file name") 406 parser.add_option("--force", "-f", action="store_true", 407 help="force build image even if image exists") 408 parser.add_option("--jobs", type=int, default=get_default_jobs(), 409 help="number of virtual CPUs") 410 parser.add_option("--verbose", "-V", action="store_true", 411 help="Pass V=1 to builds within the guest") 412 parser.add_option("--build-image", "-b", action="store_true", 413 help="build image") 414 parser.add_option("--build-qemu", 415 help="build QEMU from source in guest") 416 parser.add_option("--build-target", 417 help="QEMU build target", default="check") 418 parser.add_option("--interactive", "-I", action="store_true", 419 help="Interactively run command") 420 parser.add_option("--snapshot", "-s", action="store_true", 421 help="run tests with a snapshot") 422 parser.disable_interspersed_args() 423 return parser.parse_args() 424 425def main(vmcls): 426 try: 427 args, argv = parse_args(vmcls) 428 if not argv and not args.build_qemu and not args.build_image: 429 print("Nothing to do?") 430 return 1 431 logging.basicConfig(level=(logging.DEBUG if args.debug 432 else logging.WARN)) 433 vm = vmcls(debug=args.debug, vcpus=args.jobs) 434 if args.build_image: 435 if os.path.exists(args.image) and not args.force: 436 sys.stderr.writelines(["Image file exists: %s\n" % args.image, 437 "Use --force option to overwrite\n"]) 438 return 1 439 return vm.build_image(args.image) 440 if args.build_qemu: 441 vm.add_source_dir(args.build_qemu) 442 cmd = [vm.BUILD_SCRIPT.format( 443 configure_opts = " ".join(argv), 444 jobs=int(args.jobs), 445 target=args.build_target, 446 verbose = "V=1" if args.verbose else "")] 447 else: 448 cmd = argv 449 img = args.image 450 if args.snapshot: 451 img += ",snapshot=on" 452 vm.boot(img) 453 vm.wait_ssh() 454 except Exception as e: 455 if isinstance(e, SystemExit) and e.code == 0: 456 return 0 457 sys.stderr.write("Failed to prepare guest environment\n") 458 traceback.print_exc() 459 return 2 460 461 exitcode = 0 462 if vm.ssh(*cmd) != 0: 463 exitcode = 3 464 if args.interactive: 465 vm.ssh() 466 467 if not args.snapshot: 468 vm.graceful_shutdown() 469 470 return exitcode 471