1# 2# VM testing base class 3# 4# Copyright 2017-2019 Red Hat Inc. 5# 6# Authors: 7# Fam Zheng <famz@redhat.com> 8# Gerd Hoffmann <kraxel@redhat.com> 9# 10# This code is licensed under the GPL version 2 or later. See 11# the COPYING file in the top-level directory. 12# 13 14import os 15import re 16import sys 17import socket 18import logging 19import time 20import datetime 21sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 22from qemu.accel import kvm_available 23from qemu.machine import QEMUMachine 24import subprocess 25import hashlib 26import optparse 27import atexit 28import tempfile 29import shutil 30import multiprocessing 31import traceback 32 33SSH_KEY = open(os.path.join(os.path.dirname(__file__), 34 "..", "keys", "id_rsa")).read() 35SSH_PUB_KEY = open(os.path.join(os.path.dirname(__file__), 36 "..", "keys", "id_rsa.pub")).read() 37 38class BaseVM(object): 39 GUEST_USER = "qemu" 40 GUEST_PASS = "qemupass" 41 ROOT_PASS = "qemupass" 42 43 envvars = [ 44 "https_proxy", 45 "http_proxy", 46 "ftp_proxy", 47 "no_proxy", 48 ] 49 50 # The script to run in the guest that builds QEMU 51 BUILD_SCRIPT = "" 52 # The guest name, to be overridden by subclasses 53 name = "#base" 54 # The guest architecture, to be overridden by subclasses 55 arch = "#arch" 56 # command to halt the guest, can be overridden by subclasses 57 poweroff = "poweroff" 58 # enable IPv6 networking 59 ipv6 = True 60 # Scale up some timeouts under TCG. 61 # 4 is arbitrary, but greater than 2, 62 # since we found we need to wait more than twice as long. 63 tcg_ssh_timeout_multiplier = 4 64 def __init__(self, debug=False, vcpus=None): 65 self._guest = None 66 self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-", 67 suffix=".tmp", 68 dir=".")) 69 atexit.register(shutil.rmtree, self._tmpdir) 70 71 self._ssh_key_file = os.path.join(self._tmpdir, "id_rsa") 72 open(self._ssh_key_file, "w").write(SSH_KEY) 73 subprocess.check_call(["chmod", "600", self._ssh_key_file]) 74 75 self._ssh_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub") 76 open(self._ssh_pub_key_file, "w").write(SSH_PUB_KEY) 77 78 self.debug = debug 79 self._stderr = sys.stderr 80 self._devnull = open(os.devnull, "w") 81 if self.debug: 82 self._stdout = sys.stdout 83 else: 84 self._stdout = self._devnull 85 self._args = [ \ 86 "-nodefaults", "-m", "4G", 87 "-cpu", "max", 88 "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22" + 89 (",ipv6=no" if not self.ipv6 else ""), 90 "-device", "virtio-net-pci,netdev=vnet", 91 "-vnc", "127.0.0.1:0,to=20"] 92 if vcpus and vcpus > 1: 93 self._args += ["-smp", "%d" % vcpus] 94 if kvm_available(self.arch): 95 self._args += ["-enable-kvm"] 96 else: 97 logging.info("KVM not available, not using -enable-kvm") 98 self._data_args = [] 99 100 def _download_with_cache(self, url, sha256sum=None, sha512sum=None): 101 def check_sha256sum(fname): 102 if not sha256sum: 103 return True 104 checksum = subprocess.check_output(["sha256sum", fname]).split()[0] 105 return sha256sum == checksum.decode("utf-8") 106 107 def check_sha512sum(fname): 108 if not sha512sum: 109 return True 110 checksum = subprocess.check_output(["sha512sum", fname]).split()[0] 111 return sha512sum == checksum.decode("utf-8") 112 113 cache_dir = os.path.expanduser("~/.cache/qemu-vm/download") 114 if not os.path.exists(cache_dir): 115 os.makedirs(cache_dir) 116 fname = os.path.join(cache_dir, 117 hashlib.sha1(url.encode("utf-8")).hexdigest()) 118 if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname): 119 return fname 120 logging.debug("Downloading %s to %s...", url, fname) 121 subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"], 122 stdout=self._stdout, stderr=self._stderr) 123 os.rename(fname + ".download", fname) 124 return fname 125 126 def _ssh_do(self, user, cmd, check): 127 ssh_cmd = ["ssh", 128 "-t", 129 "-o", "StrictHostKeyChecking=no", 130 "-o", "UserKnownHostsFile=" + os.devnull, 131 "-o", "ConnectTimeout=1", 132 "-p", self.ssh_port, "-i", self._ssh_key_file] 133 # If not in debug mode, set ssh to quiet mode to 134 # avoid printing the results of commands. 135 if not self.debug: 136 ssh_cmd.append("-q") 137 for var in self.envvars: 138 ssh_cmd += ['-o', "SendEnv=%s" % var ] 139 assert not isinstance(cmd, str) 140 ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd) 141 logging.debug("ssh_cmd: %s", " ".join(ssh_cmd)) 142 r = subprocess.call(ssh_cmd) 143 if check and r != 0: 144 raise Exception("SSH command failed: %s" % cmd) 145 return r 146 147 def ssh(self, *cmd): 148 return self._ssh_do(self.GUEST_USER, cmd, False) 149 150 def ssh_root(self, *cmd): 151 return self._ssh_do("root", cmd, False) 152 153 def ssh_check(self, *cmd): 154 self._ssh_do(self.GUEST_USER, cmd, True) 155 156 def ssh_root_check(self, *cmd): 157 self._ssh_do("root", cmd, True) 158 159 def build_image(self, img): 160 raise NotImplementedError 161 162 def exec_qemu_img(self, *args): 163 cmd = [os.environ.get("QEMU_IMG", "qemu-img")] 164 cmd.extend(list(args)) 165 subprocess.check_call(cmd) 166 167 def add_source_dir(self, src_dir): 168 name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5] 169 tarfile = os.path.join(self._tmpdir, name + ".tar") 170 logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir) 171 subprocess.check_call(["./scripts/archive-source.sh", tarfile], 172 cwd=src_dir, stdin=self._devnull, 173 stdout=self._stdout, stderr=self._stderr) 174 self._data_args += ["-drive", 175 "file=%s,if=none,id=%s,cache=writeback,format=raw" % \ 176 (tarfile, name), 177 "-device", 178 "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)] 179 180 def boot(self, img, extra_args=[]): 181 args = self._args + [ 182 "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img, 183 "-device", "virtio-blk,drive=drive0,bootindex=0"] 184 args += self._data_args + extra_args 185 logging.debug("QEMU args: %s", " ".join(args)) 186 qemu_bin = os.environ.get("QEMU", "qemu-system-" + self.arch) 187 guest = QEMUMachine(binary=qemu_bin, args=args) 188 guest.set_machine('pc') 189 guest.set_console() 190 try: 191 guest.launch() 192 except: 193 logging.error("Failed to launch QEMU, command line:") 194 logging.error(" ".join([qemu_bin] + args)) 195 logging.error("Log:") 196 logging.error(guest.get_log()) 197 logging.error("QEMU version >= 2.10 is required") 198 raise 199 atexit.register(self.shutdown) 200 self._guest = guest 201 usernet_info = guest.qmp("human-monitor-command", 202 command_line="info usernet") 203 self.ssh_port = None 204 for l in usernet_info["return"].splitlines(): 205 fields = l.split() 206 if "TCP[HOST_FORWARD]" in fields and "22" in fields: 207 self.ssh_port = l.split()[3] 208 if not self.ssh_port: 209 raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \ 210 usernet_info) 211 212 def console_init(self, timeout = 120): 213 vm = self._guest 214 vm.console_socket.settimeout(timeout) 215 self.console_raw_path = os.path.join(vm._temp_dir, 216 vm._name + "-console.raw") 217 self.console_raw_file = open(self.console_raw_path, 'wb') 218 219 def console_log(self, text): 220 for line in re.split("[\r\n]", text): 221 # filter out terminal escape sequences 222 line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line) 223 line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line) 224 # replace unprintable chars 225 line = re.sub("\x1b", "<esc>", line) 226 line = re.sub("[\x00-\x1f]", ".", line) 227 line = re.sub("[\x80-\xff]", ".", line) 228 if line == "": 229 continue 230 # log console line 231 sys.stderr.write("con recv: %s\n" % line) 232 233 def console_wait(self, expect, expectalt = None): 234 vm = self._guest 235 output = "" 236 while True: 237 try: 238 chars = vm.console_socket.recv(1) 239 if self.console_raw_file: 240 self.console_raw_file.write(chars) 241 self.console_raw_file.flush() 242 except socket.timeout: 243 sys.stderr.write("console: *** read timeout ***\n") 244 sys.stderr.write("console: waiting for: '%s'\n" % expect) 245 if not expectalt is None: 246 sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt) 247 sys.stderr.write("console: line buffer:\n") 248 sys.stderr.write("\n") 249 self.console_log(output.rstrip()) 250 sys.stderr.write("\n") 251 raise 252 output += chars.decode("latin1") 253 if expect in output: 254 break 255 if not expectalt is None and expectalt in output: 256 break 257 if "\r" in output or "\n" in output: 258 lines = re.split("[\r\n]", output) 259 output = lines.pop() 260 if self.debug: 261 self.console_log("\n".join(lines)) 262 if self.debug: 263 self.console_log(output) 264 if not expectalt is None and expectalt in output: 265 return False 266 return True 267 268 def console_consume(self): 269 vm = self._guest 270 output = "" 271 vm.console_socket.setblocking(0) 272 while True: 273 try: 274 chars = vm.console_socket.recv(1) 275 except: 276 break 277 output += chars.decode("latin1") 278 if "\r" in output or "\n" in output: 279 lines = re.split("[\r\n]", output) 280 output = lines.pop() 281 if self.debug: 282 self.console_log("\n".join(lines)) 283 if self.debug: 284 self.console_log(output) 285 vm.console_socket.setblocking(1) 286 287 def console_send(self, command): 288 vm = self._guest 289 if self.debug: 290 logline = re.sub("\n", "<enter>", command) 291 logline = re.sub("[\x00-\x1f]", ".", logline) 292 sys.stderr.write("con send: %s\n" % logline) 293 for char in list(command): 294 vm.console_socket.send(char.encode("utf-8")) 295 time.sleep(0.01) 296 297 def console_wait_send(self, wait, command): 298 self.console_wait(wait) 299 self.console_send(command) 300 301 def console_ssh_init(self, prompt, user, pw): 302 sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" % SSH_PUB_KEY.rstrip() 303 self.console_wait_send("login:", "%s\n" % user) 304 self.console_wait_send("Password:", "%s\n" % pw) 305 self.console_wait_send(prompt, "mkdir .ssh\n") 306 self.console_wait_send(prompt, sshkey_cmd) 307 self.console_wait_send(prompt, "chmod 755 .ssh\n") 308 self.console_wait_send(prompt, "chmod 644 .ssh/authorized_keys\n") 309 310 def console_sshd_config(self, prompt): 311 self.console_wait(prompt) 312 self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n") 313 for var in self.envvars: 314 self.console_wait(prompt) 315 self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var) 316 317 def print_step(self, text): 318 sys.stderr.write("### %s ...\n" % text) 319 320 def wait_ssh(self, wait_root=False, seconds=300): 321 # Allow more time for VM to boot under TCG. 322 if not kvm_available(self.arch): 323 seconds *= self.tcg_ssh_timeout_multiplier 324 starttime = datetime.datetime.now() 325 endtime = starttime + datetime.timedelta(seconds=seconds) 326 guest_up = False 327 while datetime.datetime.now() < endtime: 328 if wait_root and self.ssh_root("exit 0") == 0: 329 guest_up = True 330 break 331 elif self.ssh("exit 0") == 0: 332 guest_up = True 333 break 334 seconds = (endtime - datetime.datetime.now()).total_seconds() 335 logging.debug("%ds before timeout", seconds) 336 time.sleep(1) 337 if not guest_up: 338 raise Exception("Timeout while waiting for guest ssh") 339 340 def shutdown(self): 341 self._guest.shutdown() 342 343 def wait(self): 344 self._guest.wait() 345 346 def graceful_shutdown(self): 347 self.ssh_root(self.poweroff) 348 self._guest.wait() 349 350 def qmp(self, *args, **kwargs): 351 return self._guest.qmp(*args, **kwargs) 352 353 def gen_cloud_init_iso(self): 354 cidir = self._tmpdir 355 mdata = open(os.path.join(cidir, "meta-data"), "w") 356 name = self.name.replace(".","-") 357 mdata.writelines(["instance-id: {}-vm-0\n".format(name), 358 "local-hostname: {}-guest\n".format(name)]) 359 mdata.close() 360 udata = open(os.path.join(cidir, "user-data"), "w") 361 print("guest user:pw {}:{}".format(self.GUEST_USER, 362 self.GUEST_PASS)) 363 udata.writelines(["#cloud-config\n", 364 "chpasswd:\n", 365 " list: |\n", 366 " root:%s\n" % self.ROOT_PASS, 367 " %s:%s\n" % (self.GUEST_USER, 368 self.GUEST_PASS), 369 " expire: False\n", 370 "users:\n", 371 " - name: %s\n" % self.GUEST_USER, 372 " sudo: ALL=(ALL) NOPASSWD:ALL\n", 373 " ssh-authorized-keys:\n", 374 " - %s\n" % SSH_PUB_KEY, 375 " - name: root\n", 376 " ssh-authorized-keys:\n", 377 " - %s\n" % SSH_PUB_KEY, 378 "locale: en_US.UTF-8\n"]) 379 proxy = os.environ.get("http_proxy") 380 if not proxy is None: 381 udata.writelines(["apt:\n", 382 " proxy: %s" % proxy]) 383 udata.close() 384 subprocess.check_call(["genisoimage", "-output", "cloud-init.iso", 385 "-volid", "cidata", "-joliet", "-rock", 386 "user-data", "meta-data"], 387 cwd=cidir, 388 stdin=self._devnull, stdout=self._stdout, 389 stderr=self._stdout) 390 391 return os.path.join(cidir, "cloud-init.iso") 392 393def parse_args(vmcls): 394 395 def get_default_jobs(): 396 if kvm_available(vmcls.arch): 397 return multiprocessing.cpu_count() // 2 398 else: 399 return 1 400 401 parser = optparse.OptionParser( 402 description="VM test utility. Exit codes: " 403 "0 = success, " 404 "1 = command line error, " 405 "2 = environment initialization failed, " 406 "3 = test command failed") 407 parser.add_option("--debug", "-D", action="store_true", 408 help="enable debug output") 409 parser.add_option("--image", "-i", default="%s.img" % vmcls.name, 410 help="image file name") 411 parser.add_option("--force", "-f", action="store_true", 412 help="force build image even if image exists") 413 parser.add_option("--jobs", type=int, default=get_default_jobs(), 414 help="number of virtual CPUs") 415 parser.add_option("--verbose", "-V", action="store_true", 416 help="Pass V=1 to builds within the guest") 417 parser.add_option("--build-image", "-b", action="store_true", 418 help="build image") 419 parser.add_option("--build-qemu", 420 help="build QEMU from source in guest") 421 parser.add_option("--build-target", 422 help="QEMU build target", default="check") 423 parser.add_option("--interactive", "-I", action="store_true", 424 help="Interactively run command") 425 parser.add_option("--snapshot", "-s", action="store_true", 426 help="run tests with a snapshot") 427 parser.disable_interspersed_args() 428 return parser.parse_args() 429 430def main(vmcls): 431 try: 432 args, argv = parse_args(vmcls) 433 if not argv and not args.build_qemu and not args.build_image: 434 print("Nothing to do?") 435 return 1 436 logging.basicConfig(level=(logging.DEBUG if args.debug 437 else logging.WARN)) 438 vm = vmcls(debug=args.debug, vcpus=args.jobs) 439 if args.build_image: 440 if os.path.exists(args.image) and not args.force: 441 sys.stderr.writelines(["Image file exists: %s\n" % args.image, 442 "Use --force option to overwrite\n"]) 443 return 1 444 return vm.build_image(args.image) 445 if args.build_qemu: 446 vm.add_source_dir(args.build_qemu) 447 cmd = [vm.BUILD_SCRIPT.format( 448 configure_opts = " ".join(argv), 449 jobs=int(args.jobs), 450 target=args.build_target, 451 verbose = "V=1" if args.verbose else "")] 452 else: 453 cmd = argv 454 img = args.image 455 if args.snapshot: 456 img += ",snapshot=on" 457 vm.boot(img) 458 vm.wait_ssh() 459 except Exception as e: 460 if isinstance(e, SystemExit) and e.code == 0: 461 return 0 462 sys.stderr.write("Failed to prepare guest environment\n") 463 traceback.print_exc() 464 return 2 465 466 exitcode = 0 467 if vm.ssh(*cmd) != 0: 468 exitcode = 3 469 if args.interactive: 470 vm.ssh() 471 472 if not args.snapshot: 473 vm.graceful_shutdown() 474 475 return exitcode 476