1# 2# VM testing base class 3# 4# Copyright 2017-2019 Red Hat Inc. 5# 6# Authors: 7# Fam Zheng <famz@redhat.com> 8# Gerd Hoffmann <kraxel@redhat.com> 9# 10# This code is licensed under the GPL version 2 or later. See 11# the COPYING file in the top-level directory. 12# 13 14import os 15import re 16import sys 17import socket 18import logging 19import time 20import datetime 21sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 22from qemu.accel import kvm_available 23from qemu.machine import QEMUMachine 24import subprocess 25import hashlib 26import optparse 27import atexit 28import tempfile 29import shutil 30import multiprocessing 31import traceback 32 33SSH_KEY = open(os.path.join(os.path.dirname(__file__), 34 "..", "keys", "id_rsa")).read() 35SSH_PUB_KEY = open(os.path.join(os.path.dirname(__file__), 36 "..", "keys", "id_rsa.pub")).read() 37 38class BaseVM(object): 39 GUEST_USER = "qemu" 40 GUEST_PASS = "qemupass" 41 ROOT_PASS = "qemupass" 42 43 envvars = [ 44 "https_proxy", 45 "http_proxy", 46 "ftp_proxy", 47 "no_proxy", 48 ] 49 50 # The script to run in the guest that builds QEMU 51 BUILD_SCRIPT = "" 52 # The guest name, to be overridden by subclasses 53 name = "#base" 54 # The guest architecture, to be overridden by subclasses 55 arch = "#arch" 56 # command to halt the guest, can be overridden by subclasses 57 poweroff = "poweroff" 58 # enable IPv6 networking 59 ipv6 = True 60 # Scale up some timeouts under TCG. 61 # 4 is arbitrary, but greater than 2, 62 # since we found we need to wait more than twice as long. 63 tcg_ssh_timeout_multiplier = 4 64 def __init__(self, debug=False, vcpus=None, genisoimage=None): 65 self._guest = None 66 self._genisoimage = genisoimage 67 self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-", 68 suffix=".tmp", 69 dir=".")) 70 atexit.register(shutil.rmtree, self._tmpdir) 71 72 self._ssh_key_file = os.path.join(self._tmpdir, "id_rsa") 73 open(self._ssh_key_file, "w").write(SSH_KEY) 74 subprocess.check_call(["chmod", "600", self._ssh_key_file]) 75 76 self._ssh_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub") 77 open(self._ssh_pub_key_file, "w").write(SSH_PUB_KEY) 78 79 self.debug = debug 80 self._stderr = sys.stderr 81 self._devnull = open(os.devnull, "w") 82 if self.debug: 83 self._stdout = sys.stdout 84 else: 85 self._stdout = self._devnull 86 self._args = [ \ 87 "-nodefaults", "-m", "4G", 88 "-cpu", "max", 89 "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22" + 90 (",ipv6=no" if not self.ipv6 else ""), 91 "-device", "virtio-net-pci,netdev=vnet", 92 "-vnc", "127.0.0.1:0,to=20"] 93 if vcpus and vcpus > 1: 94 self._args += ["-smp", "%d" % vcpus] 95 if kvm_available(self.arch): 96 self._args += ["-enable-kvm"] 97 else: 98 logging.info("KVM not available, not using -enable-kvm") 99 self._data_args = [] 100 101 def _download_with_cache(self, url, sha256sum=None, sha512sum=None): 102 def check_sha256sum(fname): 103 if not sha256sum: 104 return True 105 checksum = subprocess.check_output(["sha256sum", fname]).split()[0] 106 return sha256sum == checksum.decode("utf-8") 107 108 def check_sha512sum(fname): 109 if not sha512sum: 110 return True 111 checksum = subprocess.check_output(["sha512sum", fname]).split()[0] 112 return sha512sum == checksum.decode("utf-8") 113 114 cache_dir = os.path.expanduser("~/.cache/qemu-vm/download") 115 if not os.path.exists(cache_dir): 116 os.makedirs(cache_dir) 117 fname = os.path.join(cache_dir, 118 hashlib.sha1(url.encode("utf-8")).hexdigest()) 119 if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname): 120 return fname 121 logging.debug("Downloading %s to %s...", url, fname) 122 subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"], 123 stdout=self._stdout, stderr=self._stderr) 124 os.rename(fname + ".download", fname) 125 return fname 126 127 def _ssh_do(self, user, cmd, check): 128 ssh_cmd = ["ssh", 129 "-t", 130 "-o", "StrictHostKeyChecking=no", 131 "-o", "UserKnownHostsFile=" + os.devnull, 132 "-o", "ConnectTimeout=1", 133 "-p", self.ssh_port, "-i", self._ssh_key_file] 134 # If not in debug mode, set ssh to quiet mode to 135 # avoid printing the results of commands. 136 if not self.debug: 137 ssh_cmd.append("-q") 138 for var in self.envvars: 139 ssh_cmd += ['-o', "SendEnv=%s" % var ] 140 assert not isinstance(cmd, str) 141 ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd) 142 logging.debug("ssh_cmd: %s", " ".join(ssh_cmd)) 143 r = subprocess.call(ssh_cmd) 144 if check and r != 0: 145 raise Exception("SSH command failed: %s" % cmd) 146 return r 147 148 def ssh(self, *cmd): 149 return self._ssh_do(self.GUEST_USER, cmd, False) 150 151 def ssh_root(self, *cmd): 152 return self._ssh_do("root", cmd, False) 153 154 def ssh_check(self, *cmd): 155 self._ssh_do(self.GUEST_USER, cmd, True) 156 157 def ssh_root_check(self, *cmd): 158 self._ssh_do("root", cmd, True) 159 160 def build_image(self, img): 161 raise NotImplementedError 162 163 def exec_qemu_img(self, *args): 164 cmd = [os.environ.get("QEMU_IMG", "qemu-img")] 165 cmd.extend(list(args)) 166 subprocess.check_call(cmd) 167 168 def add_source_dir(self, src_dir): 169 name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5] 170 tarfile = os.path.join(self._tmpdir, name + ".tar") 171 logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir) 172 subprocess.check_call(["./scripts/archive-source.sh", tarfile], 173 cwd=src_dir, stdin=self._devnull, 174 stdout=self._stdout, stderr=self._stderr) 175 self._data_args += ["-drive", 176 "file=%s,if=none,id=%s,cache=writeback,format=raw" % \ 177 (tarfile, name), 178 "-device", 179 "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)] 180 181 def boot(self, img, extra_args=[]): 182 args = self._args + [ 183 "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img, 184 "-device", "virtio-blk,drive=drive0,bootindex=0"] 185 args += self._data_args + extra_args 186 logging.debug("QEMU args: %s", " ".join(args)) 187 qemu_bin = os.environ.get("QEMU", "qemu-system-" + self.arch) 188 guest = QEMUMachine(binary=qemu_bin, args=args) 189 guest.set_machine('pc') 190 guest.set_console() 191 try: 192 guest.launch() 193 except: 194 logging.error("Failed to launch QEMU, command line:") 195 logging.error(" ".join([qemu_bin] + args)) 196 logging.error("Log:") 197 logging.error(guest.get_log()) 198 logging.error("QEMU version >= 2.10 is required") 199 raise 200 atexit.register(self.shutdown) 201 self._guest = guest 202 usernet_info = guest.qmp("human-monitor-command", 203 command_line="info usernet") 204 self.ssh_port = None 205 for l in usernet_info["return"].splitlines(): 206 fields = l.split() 207 if "TCP[HOST_FORWARD]" in fields and "22" in fields: 208 self.ssh_port = l.split()[3] 209 if not self.ssh_port: 210 raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \ 211 usernet_info) 212 213 def console_init(self, timeout = 120): 214 vm = self._guest 215 vm.console_socket.settimeout(timeout) 216 self.console_raw_path = os.path.join(vm._temp_dir, 217 vm._name + "-console.raw") 218 self.console_raw_file = open(self.console_raw_path, 'wb') 219 220 def console_log(self, text): 221 for line in re.split("[\r\n]", text): 222 # filter out terminal escape sequences 223 line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line) 224 line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line) 225 # replace unprintable chars 226 line = re.sub("\x1b", "<esc>", line) 227 line = re.sub("[\x00-\x1f]", ".", line) 228 line = re.sub("[\x80-\xff]", ".", line) 229 if line == "": 230 continue 231 # log console line 232 sys.stderr.write("con recv: %s\n" % line) 233 234 def console_wait(self, expect, expectalt = None): 235 vm = self._guest 236 output = "" 237 while True: 238 try: 239 chars = vm.console_socket.recv(1) 240 if self.console_raw_file: 241 self.console_raw_file.write(chars) 242 self.console_raw_file.flush() 243 except socket.timeout: 244 sys.stderr.write("console: *** read timeout ***\n") 245 sys.stderr.write("console: waiting for: '%s'\n" % expect) 246 if not expectalt is None: 247 sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt) 248 sys.stderr.write("console: line buffer:\n") 249 sys.stderr.write("\n") 250 self.console_log(output.rstrip()) 251 sys.stderr.write("\n") 252 raise 253 output += chars.decode("latin1") 254 if expect in output: 255 break 256 if not expectalt is None and expectalt in output: 257 break 258 if "\r" in output or "\n" in output: 259 lines = re.split("[\r\n]", output) 260 output = lines.pop() 261 if self.debug: 262 self.console_log("\n".join(lines)) 263 if self.debug: 264 self.console_log(output) 265 if not expectalt is None and expectalt in output: 266 return False 267 return True 268 269 def console_consume(self): 270 vm = self._guest 271 output = "" 272 vm.console_socket.setblocking(0) 273 while True: 274 try: 275 chars = vm.console_socket.recv(1) 276 except: 277 break 278 output += chars.decode("latin1") 279 if "\r" in output or "\n" in output: 280 lines = re.split("[\r\n]", output) 281 output = lines.pop() 282 if self.debug: 283 self.console_log("\n".join(lines)) 284 if self.debug: 285 self.console_log(output) 286 vm.console_socket.setblocking(1) 287 288 def console_send(self, command): 289 vm = self._guest 290 if self.debug: 291 logline = re.sub("\n", "<enter>", command) 292 logline = re.sub("[\x00-\x1f]", ".", logline) 293 sys.stderr.write("con send: %s\n" % logline) 294 for char in list(command): 295 vm.console_socket.send(char.encode("utf-8")) 296 time.sleep(0.01) 297 298 def console_wait_send(self, wait, command): 299 self.console_wait(wait) 300 self.console_send(command) 301 302 def console_ssh_init(self, prompt, user, pw): 303 sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" % SSH_PUB_KEY.rstrip() 304 self.console_wait_send("login:", "%s\n" % user) 305 self.console_wait_send("Password:", "%s\n" % pw) 306 self.console_wait_send(prompt, "mkdir .ssh\n") 307 self.console_wait_send(prompt, sshkey_cmd) 308 self.console_wait_send(prompt, "chmod 755 .ssh\n") 309 self.console_wait_send(prompt, "chmod 644 .ssh/authorized_keys\n") 310 311 def console_sshd_config(self, prompt): 312 self.console_wait(prompt) 313 self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n") 314 for var in self.envvars: 315 self.console_wait(prompt) 316 self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var) 317 318 def print_step(self, text): 319 sys.stderr.write("### %s ...\n" % text) 320 321 def wait_ssh(self, wait_root=False, seconds=300): 322 # Allow more time for VM to boot under TCG. 323 if not kvm_available(self.arch): 324 seconds *= self.tcg_ssh_timeout_multiplier 325 starttime = datetime.datetime.now() 326 endtime = starttime + datetime.timedelta(seconds=seconds) 327 guest_up = False 328 while datetime.datetime.now() < endtime: 329 if wait_root and self.ssh_root("exit 0") == 0: 330 guest_up = True 331 break 332 elif self.ssh("exit 0") == 0: 333 guest_up = True 334 break 335 seconds = (endtime - datetime.datetime.now()).total_seconds() 336 logging.debug("%ds before timeout", seconds) 337 time.sleep(1) 338 if not guest_up: 339 raise Exception("Timeout while waiting for guest ssh") 340 341 def shutdown(self): 342 self._guest.shutdown() 343 344 def wait(self): 345 self._guest.wait() 346 347 def graceful_shutdown(self): 348 self.ssh_root(self.poweroff) 349 self._guest.wait() 350 351 def qmp(self, *args, **kwargs): 352 return self._guest.qmp(*args, **kwargs) 353 354 def gen_cloud_init_iso(self): 355 cidir = self._tmpdir 356 mdata = open(os.path.join(cidir, "meta-data"), "w") 357 name = self.name.replace(".","-") 358 mdata.writelines(["instance-id: {}-vm-0\n".format(name), 359 "local-hostname: {}-guest\n".format(name)]) 360 mdata.close() 361 udata = open(os.path.join(cidir, "user-data"), "w") 362 print("guest user:pw {}:{}".format(self.GUEST_USER, 363 self.GUEST_PASS)) 364 udata.writelines(["#cloud-config\n", 365 "chpasswd:\n", 366 " list: |\n", 367 " root:%s\n" % self.ROOT_PASS, 368 " %s:%s\n" % (self.GUEST_USER, 369 self.GUEST_PASS), 370 " expire: False\n", 371 "users:\n", 372 " - name: %s\n" % self.GUEST_USER, 373 " sudo: ALL=(ALL) NOPASSWD:ALL\n", 374 " ssh-authorized-keys:\n", 375 " - %s\n" % SSH_PUB_KEY, 376 " - name: root\n", 377 " ssh-authorized-keys:\n", 378 " - %s\n" % SSH_PUB_KEY, 379 "locale: en_US.UTF-8\n"]) 380 proxy = os.environ.get("http_proxy") 381 if not proxy is None: 382 udata.writelines(["apt:\n", 383 " proxy: %s" % proxy]) 384 udata.close() 385 subprocess.check_call([self._genisoimage, "-output", "cloud-init.iso", 386 "-volid", "cidata", "-joliet", "-rock", 387 "user-data", "meta-data"], 388 cwd=cidir, 389 stdin=self._devnull, stdout=self._stdout, 390 stderr=self._stdout) 391 392 return os.path.join(cidir, "cloud-init.iso") 393 394def parse_args(vmcls): 395 396 def get_default_jobs(): 397 if kvm_available(vmcls.arch): 398 return multiprocessing.cpu_count() // 2 399 else: 400 return 1 401 402 parser = optparse.OptionParser( 403 description="VM test utility. Exit codes: " 404 "0 = success, " 405 "1 = command line error, " 406 "2 = environment initialization failed, " 407 "3 = test command failed") 408 parser.add_option("--debug", "-D", action="store_true", 409 help="enable debug output") 410 parser.add_option("--image", "-i", default="%s.img" % vmcls.name, 411 help="image file name") 412 parser.add_option("--force", "-f", action="store_true", 413 help="force build image even if image exists") 414 parser.add_option("--jobs", type=int, default=get_default_jobs(), 415 help="number of virtual CPUs") 416 parser.add_option("--verbose", "-V", action="store_true", 417 help="Pass V=1 to builds within the guest") 418 parser.add_option("--build-image", "-b", action="store_true", 419 help="build image") 420 parser.add_option("--build-qemu", 421 help="build QEMU from source in guest") 422 parser.add_option("--build-target", 423 help="QEMU build target", default="check") 424 parser.add_option("--interactive", "-I", action="store_true", 425 help="Interactively run command") 426 parser.add_option("--snapshot", "-s", action="store_true", 427 help="run tests with a snapshot") 428 parser.add_option("--genisoimage", default="genisoimage", 429 help="iso imaging tool") 430 parser.disable_interspersed_args() 431 return parser.parse_args() 432 433def main(vmcls): 434 try: 435 args, argv = parse_args(vmcls) 436 if not argv and not args.build_qemu and not args.build_image: 437 print("Nothing to do?") 438 return 1 439 logging.basicConfig(level=(logging.DEBUG if args.debug 440 else logging.WARN)) 441 vm = vmcls(debug=args.debug, vcpus=args.jobs, 442 genisoimage=args.genisoimage) 443 if args.build_image: 444 if os.path.exists(args.image) and not args.force: 445 sys.stderr.writelines(["Image file exists: %s\n" % args.image, 446 "Use --force option to overwrite\n"]) 447 return 1 448 return vm.build_image(args.image) 449 if args.build_qemu: 450 vm.add_source_dir(args.build_qemu) 451 cmd = [vm.BUILD_SCRIPT.format( 452 configure_opts = " ".join(argv), 453 jobs=int(args.jobs), 454 target=args.build_target, 455 verbose = "V=1" if args.verbose else "")] 456 else: 457 cmd = argv 458 img = args.image 459 if args.snapshot: 460 img += ",snapshot=on" 461 vm.boot(img) 462 vm.wait_ssh() 463 except Exception as e: 464 if isinstance(e, SystemExit) and e.code == 0: 465 return 0 466 sys.stderr.write("Failed to prepare guest environment\n") 467 traceback.print_exc() 468 return 2 469 470 exitcode = 0 471 if vm.ssh(*cmd) != 0: 472 exitcode = 3 473 if args.interactive: 474 vm.ssh() 475 476 if not args.snapshot: 477 vm.graceful_shutdown() 478 479 return exitcode 480