1#!/usr/bin/env python 2 3r""" 4This module contains functions having to do with machine state: get_state, 5check_state, wait_state, etc. 6 7The 'State' is a composite of many pieces of data. Therefore, the functions 8in this module define state as an ordered dictionary. Here is an example of 9some test output showing machine state: 10 11default_state: 12 default_state[chassis]: On 13 default_state[boot_progress]: FW Progress, Starting OS 14 default_state[host]: Running 15 default_state[os_ping]: 1 16 default_state[os_login]: 1 17 default_state[os_run_cmd]: 1 18 19Different users may very well have different needs when inquiring about 20state. Support for new pieces of state information may be added to this 21module as needed. 22 23By using the wait_state function, a caller can start a boot and then wait for 24a precisely defined state to indicate that the boot has succeeded. If 25the boot fails, they can see exactly why by looking at the current state as 26compared with the expected state. 27""" 28 29import gen_print as gp 30import gen_robot_print as grp 31import gen_valid as gv 32import gen_robot_utils as gru 33import gen_cmd as gc 34 35import commands 36from robot.libraries.BuiltIn import BuiltIn 37from robot.utils import DotDict 38 39import re 40import os 41 42# We need utils.robot to get keywords like "Get Power State". 43gru.my_import_resource("utils.robot") 44gru.my_import_resource("state_manager.robot") 45 46# The BMC code has recently been changed as far as what states are defined and 47# what the state values can be. This module now has a means of processing both 48# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e. 49# OBMC_STATES_VERSION = 1). 50# The caller can set environment variable OBMC_STATES_VERSION to dictate 51# whether we're processing old or new style states. If OBMC_STATES_VERSION is 52# not set it will default to 1. 53 54OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1)) 55 56if OBMC_STATES_VERSION == 0: 57 # default_state is an initial value which may be of use to callers. 58 default_state = DotDict([('power', '1'), 59 ('bmc', 'HOST_BOOTED'), 60 ('boot_progress', 'FW Progress, Starting OS'), 61 ('os_ping', '1'), 62 ('os_login', '1'), 63 ('os_run_cmd', '1')]) 64 # valid_req_states, default_req_states and master_os_up_match are used by 65 # the get_state function. 66 # valid_req_states is a list of state information supported by the 67 # get_state function. 68 valid_req_states = ['ping', 69 'packet_loss', 70 'uptime', 71 'epoch_seconds', 72 'power', 73 'bmc', 74 'boot_progress', 75 'os_ping', 76 'os_login', 77 'os_run_cmd'] 78 # When a user calls get_state w/o specifying req_states, default_req_states 79 # is used as its value. 80 default_req_states = ['power', 81 'bmc', 82 'boot_progress', 83 'os_ping', 84 'os_login', 85 'os_run_cmd'] 86 # A master dictionary to determine whether the os may be up. 87 master_os_up_match = DotDict([('power', 'On'), 88 ('bmc', '^HOST_BOOTED$'), 89 ('boot_progress', 90 'FW Progress, Starting OS')]) 91 92else: 93 # default_state is an initial value which may be of use to callers. 94 default_state = DotDict([('rest', '1'), 95 ('chassis', 'On'), 96 ('bmc', 'Ready'), 97 ('boot_progress', 'FW Progress, Starting OS'), 98 ('host', 'Running'), 99 ('os_ping', '1'), 100 ('os_login', '1'), 101 ('os_run_cmd', '1')]) 102 # valid_req_states is a list of state information supported by the 103 # get_state function. 104 # valid_req_states, default_req_states and master_os_up_match are used by 105 # the get_state function. 106 valid_req_states = ['ping', 107 'packet_loss', 108 'uptime', 109 'epoch_seconds', 110 'rest', 111 'chassis', 112 'bmc', 113 'boot_progress', 114 'host', 115 'os_ping', 116 'os_login', 117 'os_run_cmd'] 118 # When a user calls get_state w/o specifying req_states, default_req_states 119 # is used as its value. 120 default_req_states = ['rest', 121 'chassis', 122 'bmc', 123 'boot_progress', 124 'host', 125 'os_ping', 126 'os_login', 127 'os_run_cmd'] 128 129 # A master dictionary to determine whether the os may be up. 130 master_os_up_match = DotDict([('chassis', '^On$'), 131 ('bmc', '^Ready$'), 132 ('boot_progress', 133 'FW Progress, Starting OS'), 134 ('host', '^Running$')]) 135 136# valid_os_req_states and default_os_req_states are used by the os_get_state 137# function. 138# valid_os_req_states is a list of state information supported by the 139# get_os_state function. 140valid_os_req_states = ['os_ping', 141 'os_login', 142 'os_run_cmd'] 143# When a user calls get_os_state w/o specifying req_states, 144# default_os_req_states is used as its value. 145default_os_req_states = ['os_ping', 146 'os_login', 147 'os_run_cmd'] 148 149# Presently, some BMCs appear to not keep time very well. This environment 150# variable directs the get_state function to use either the BMC's epoch time 151# or the local epoch time. 152USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0)) 153 154 155############################################################################### 156def return_default_state(): 157 158 r""" 159 Return default state dictionary. 160 161 default_state is an initial value which may be of use to callers. 162 """ 163 164 return default_state 165 166############################################################################### 167 168 169############################################################################### 170def anchor_state(state): 171 172 r""" 173 Add regular expression anchors ("^" and "$") to the beginning and end of 174 each item in the state dictionary passed in. Return the resulting 175 dictionary. 176 177 Description of Arguments: 178 state A dictionary such as the one returned by the get_state() 179 function. 180 """ 181 182 anchored_state = state.copy() 183 for key, match_state_value in anchored_state.items(): 184 anchored_state[key] = "^" + str(anchored_state[key]) + "$" 185 186 return anchored_state 187 188############################################################################### 189 190 191############################################################################### 192def strip_anchor_state(state): 193 194 r""" 195 Strip regular expression anchors ("^" and "$") from the beginning and end 196 of each item in the state dictionary passed in. Return the resulting 197 dictionary. 198 199 Description of Arguments: 200 state A dictionary such as the one returned by the get_state() 201 function. 202 """ 203 204 stripped_state = state.copy() 205 for key, match_state_value in stripped_state.items(): 206 stripped_state[key] = stripped_state[key].strip("^$") 207 208 return stripped_state 209 210############################################################################### 211 212 213############################################################################### 214def compare_states(state, 215 match_state): 216 217 r""" 218 Compare 2 state dictionaries. Return True if they match and False if they 219 don't. Note that the match_state dictionary does not need to have an entry 220 corresponding to each entry in the state dictionary. But for each entry 221 that it does have, the corresponding state entry will be checked for a 222 match. 223 224 Description of arguments: 225 state A state dictionary such as the one returned by the 226 get_state function. 227 match_state A dictionary whose key/value pairs are "state field"/ 228 "state value". The state value is interpreted as a 229 regular expression. Every value in this dictionary is 230 considered. If each and every one matches, the 2 231 dictionaries are considered to be matching. 232 """ 233 234 match = True 235 for key, match_state_value in match_state.items(): 236 # Blank match_state_value means "don't care". 237 if match_state_value == "": 238 continue 239 try: 240 if not re.match(match_state_value, str(state[key])): 241 match = False 242 break 243 except KeyError: 244 match = False 245 break 246 247 return match 248 249############################################################################### 250 251 252############################################################################### 253def get_os_state(os_host="", 254 os_username="", 255 os_password="", 256 req_states=default_os_req_states, 257 os_up=True, 258 quiet=None): 259 260 r""" 261 Get component states for the operating system such as ping, login, 262 etc, put them into a dictionary and return them to the caller. 263 264 Note that all substate values are strings. 265 266 Description of arguments: 267 os_host The DNS name or IP address of the operating system. 268 This defaults to global ${OS_HOST}. 269 os_username The username to be used to login to the OS. 270 This defaults to global ${OS_USERNAME}. 271 os_password The password to be used to login to the OS. 272 This defaults to global ${OS_PASSWORD}. 273 req_states This is a list of states whose values are being requested by 274 the caller. 275 os_up If the caller knows that the os can't possibly be up, it can 276 improve performance by passing os_up=False. This function 277 will then simply return default values for all requested os 278 sub states. 279 quiet Indicates whether status details (e.g. curl commands) should 280 be written to the console. 281 Defaults to either global value of ${QUIET} or to 1. 282 """ 283 284 quiet = grp.set_quiet_default(quiet, 1) 285 286 # Set parm defaults where necessary and validate all parms. 287 if os_host == "": 288 os_host = BuiltIn().get_variable_value("${OS_HOST}") 289 error_message = gv.svalid_value(os_host, var_name="os_host", 290 invalid_values=[None, ""]) 291 if error_message != "": 292 BuiltIn().fail(gp.sprint_error(error_message)) 293 294 if os_username == "": 295 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 296 error_message = gv.svalid_value(os_username, var_name="os_username", 297 invalid_values=[None, ""]) 298 if error_message != "": 299 BuiltIn().fail(gp.sprint_error(error_message)) 300 301 if os_password == "": 302 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 303 error_message = gv.svalid_value(os_password, var_name="os_password", 304 invalid_values=[None, ""]) 305 if error_message != "": 306 BuiltIn().fail(gp.sprint_error(error_message)) 307 308 invalid_req_states = [sub_state for sub_state in req_states 309 if sub_state not in valid_os_req_states] 310 if len(invalid_req_states) > 0: 311 error_message = "The following req_states are not supported:\n" +\ 312 gp.sprint_var(invalid_req_states) 313 BuiltIn().fail(gp.sprint_error(error_message)) 314 315 # Initialize all substate values supported by this function. 316 os_ping = 0 317 os_login = 0 318 os_run_cmd = 0 319 320 if os_up: 321 if 'os_ping' in req_states: 322 # See if the OS pings. 323 cmd_buf = "ping -c 1 -w 2 " + os_host 324 if not quiet: 325 gp.pissuing(cmd_buf) 326 rc, out_buf = commands.getstatusoutput(cmd_buf) 327 if rc == 0: 328 os_ping = 1 329 330 # Programming note: All attributes which do not require an ssh login 331 # should have been processed by this point. 332 master_req_login = ['os_login', 'os_run_cmd'] 333 req_login = [sub_state for sub_state in req_states if sub_state in 334 master_req_login] 335 must_login = (len(req_login) > 0) 336 337 if must_login: 338 # Open SSH connection to OS. Note that this doesn't fail even when 339 # the OS is not up. 340 cmd_buf = ["SSHLibrary.Open Connection", os_host] 341 if not quiet: 342 grp.rpissuing_keyword(cmd_buf) 343 ix = BuiltIn().run_keyword(*cmd_buf) 344 345 # Login to OS. 346 cmd_buf = ["Login", os_username, os_password] 347 if not quiet: 348 grp.rpissuing_keyword(cmd_buf) 349 status, ret_values = \ 350 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 351 if status == "PASS": 352 os_login = 1 353 else: 354 gp.dprint_var(status) 355 gp.dprint_var(ret_values) 356 357 if os_login: 358 if 'os_run_cmd' in req_states: 359 # Try running a simple command (uptime) on the OS. 360 cmd_buf = ["Execute Command", "uptime", 361 "return_stderr=True", "return_rc=True"] 362 if not quiet: 363 grp.rpissuing_keyword(cmd_buf) 364 # Note that in spite of its name, there are occasions 365 # where run_keyword_and_ignore_error can fail. 366 status, ret_values = \ 367 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 368 if status == "PASS": 369 stdout, stderr, rc = ret_values 370 if rc == 0 and stderr == "": 371 os_run_cmd = 1 372 else: 373 gp.dprint_var(status) 374 gp.dprint_var(stdout) 375 gp.dprint_var(stderr) 376 gp.dprint_var(rc) 377 else: 378 gp.dprint_var(status) 379 gp.dprint_var(ret_values) 380 381 os_state = DotDict() 382 for sub_state in req_states: 383 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" 384 exec(cmd_buf) 385 386 return os_state 387 388############################################################################### 389 390 391############################################################################### 392def get_state(openbmc_host="", 393 openbmc_username="", 394 openbmc_password="", 395 os_host="", 396 os_username="", 397 os_password="", 398 req_states=default_req_states, 399 quiet=None): 400 401 r""" 402 Get component states such as power state, bmc state, etc, put them into a 403 dictionary and return them to the caller. 404 405 Note that all substate values are strings. 406 407 Description of arguments: 408 openbmc_host The DNS name or IP address of the BMC. 409 This defaults to global ${OPENBMC_HOST}. 410 openbmc_username The username to be used to login to the BMC. 411 This defaults to global ${OPENBMC_USERNAME}. 412 openbmc_password The password to be used to login to the BMC. 413 This defaults to global ${OPENBMC_PASSWORD}. 414 os_host The DNS name or IP address of the operating system. 415 This defaults to global ${OS_HOST}. 416 os_username The username to be used to login to the OS. 417 This defaults to global ${OS_USERNAME}. 418 os_password The password to be used to login to the OS. 419 This defaults to global ${OS_PASSWORD}. 420 req_states This is a list of states whose values are being requested 421 by the caller. 422 quiet Indicates whether status details (e.g. curl commands) 423 should be written to the console. 424 Defaults to either global value of ${QUIET} or to 1. 425 """ 426 427 quiet = grp.set_quiet_default(quiet, 1) 428 429 # Set parm defaults where necessary and validate all parms. 430 if openbmc_host == "": 431 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") 432 error_message = gv.svalid_value(openbmc_host, 433 var_name="openbmc_host", 434 invalid_values=[None, ""]) 435 if error_message != "": 436 BuiltIn().fail(gp.sprint_error(error_message)) 437 438 if openbmc_username == "": 439 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") 440 error_message = gv.svalid_value(openbmc_username, 441 var_name="openbmc_username", 442 invalid_values=[None, ""]) 443 if error_message != "": 444 BuiltIn().fail(gp.sprint_error(error_message)) 445 446 if openbmc_password == "": 447 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") 448 error_message = gv.svalid_value(openbmc_password, 449 var_name="openbmc_password", 450 invalid_values=[None, ""]) 451 if error_message != "": 452 BuiltIn().fail(gp.sprint_error(error_message)) 453 454 # NOTE: OS parms are optional. 455 if os_host == "": 456 os_host = BuiltIn().get_variable_value("${OS_HOST}") 457 if os_host is None: 458 os_host = "" 459 460 if os_username is "": 461 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 462 if os_username is None: 463 os_username = "" 464 465 if os_password is "": 466 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 467 if os_password is None: 468 os_password = "" 469 470 invalid_req_states = [sub_state for sub_state in req_states 471 if sub_state not in valid_req_states] 472 if len(invalid_req_states) > 0: 473 error_message = "The following req_states are not supported:\n" +\ 474 gp.sprint_var(invalid_req_states) 475 BuiltIn().fail(gp.sprint_error(error_message)) 476 477 # Initialize all substate values supported by this function. 478 ping = 0 479 packet_loss = '' 480 uptime = '' 481 epoch_seconds = '' 482 power = '' 483 rest = '1' 484 chassis = '' 485 bmc = '' 486 boot_progress = '' 487 host = '' 488 489 # Get the component states. 490 if 'ping' in req_states: 491 # See if the OS pings. 492 cmd_buf = "ping -c 1 -w 2 " + openbmc_host 493 if not quiet: 494 gp.pissuing(cmd_buf) 495 rc, out_buf = commands.getstatusoutput(cmd_buf) 496 if rc == 0: 497 ping = 1 498 499 if 'packet_loss' in req_states: 500 # See if the OS pings. 501 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\ 502 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" 503 if not quiet: 504 gp.pissuing(cmd_buf) 505 rc, out_buf = commands.getstatusoutput(cmd_buf) 506 if rc == 0: 507 packet_loss = out_buf.rstrip("\n") 508 509 master_req_login = ['uptime', 'epoch_seconds'] 510 req_login = [sub_state for sub_state in req_states if sub_state in 511 master_req_login] 512 must_login = (len(req_login) > 0) 513 514 bmc_login = 0 515 if must_login: 516 cmd_buf = ["Open Connection And Log In"] 517 if not quiet: 518 grp.rpissuing_keyword(cmd_buf) 519 status, ret_values = \ 520 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 521 if status == "PASS": 522 bmc_login = 1 523 else: 524 if re.match('^Authentication failed for user', ret_values): 525 # An authentication failure is worth failing on. 526 BuiltIn().fail(gp.sprint_error(ret_values)) 527 528 if 'uptime' in req_states and bmc_login: 529 cmd_buf = ["Execute Command", "cat /proc/uptime | cut -f 1 -d ' '", 530 "return_stderr=True", "return_rc=True"] 531 if not quiet: 532 grp.rpissuing_keyword(cmd_buf) 533 status, ret_values = \ 534 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 535 if status == "PASS": 536 stdout, stderr, rc = ret_values 537 if rc == 0 and stderr == "": 538 uptime = stdout 539 540 if 'epoch_seconds' in req_states and bmc_login: 541 date_cmd_buf = "date -u +%s" 542 if USE_BMC_EPOCH_TIME: 543 cmd_buf = ["Execute Command", date_cmd_buf, "return_stderr=True", 544 "return_rc=True"] 545 if not quiet: 546 grp.rpissuing_keyword(cmd_buf) 547 status, ret_values = \ 548 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 549 if status == "PASS": 550 stdout, stderr, rc = ret_values 551 if rc == 0 and stderr == "": 552 epoch_seconds = stdout.rstrip("\n") 553 else: 554 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf, 555 quiet=1, 556 print_output=0) 557 if shell_rc == 0: 558 epoch_seconds = out_buf.rstrip("\n") 559 560 master_req_rest = ['rest', 'power', 'chassis', 'bmc', 'boot_progress', 561 'host'] 562 req_rest = [sub_state for sub_state in req_states if sub_state in 563 master_req_rest] 564 need_rest = (len(req_rest) > 0) 565 566 # Though we could try to determine 'rest' state on any of several calls, 567 # for simplicity, we'll use 'chassis' to figure it out (even if the caller 568 # hasn't explicitly asked for 'chassis'). 569 if 'chassis' in req_states or need_rest: 570 cmd_buf = ["Get Chassis Power State", "quiet=${" + str(quiet) + "}"] 571 grp.rdpissuing_keyword(cmd_buf) 572 status, ret_values = \ 573 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 574 if status == "PASS": 575 chassis = ret_values 576 chassis = re.sub(r'.*\.', "", chassis) 577 rest = '1' 578 else: 579 rest = ret_values 580 581 if rest == '1': 582 if 'power' in req_states: 583 cmd_buf = ["Get Power State", "quiet=${" + str(quiet) + "}"] 584 grp.rdpissuing_keyword(cmd_buf) 585 status, ret_values = \ 586 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 587 if status == "PASS": 588 power = ret_values 589 590 if 'bmc' in req_states: 591 if OBMC_STATES_VERSION == 0: 592 qualifier = "utils" 593 else: 594 # This will not be supported much longer. 595 qualifier = "state_manager" 596 cmd_buf = [qualifier + ".Get BMC State", 597 "quiet=${" + str(quiet) + "}"] 598 grp.rdpissuing_keyword(cmd_buf) 599 status, ret_values = \ 600 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 601 if status == "PASS": 602 bmc = ret_values 603 604 if 'boot_progress' in req_states: 605 cmd_buf = ["Get Boot Progress", "quiet=${" + str(quiet) + "}"] 606 grp.rdpissuing_keyword(cmd_buf) 607 status, ret_values = \ 608 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 609 if status == "PASS": 610 boot_progress = ret_values 611 612 if 'host' in req_states: 613 if OBMC_STATES_VERSION > 0: 614 cmd_buf = ["Get Host State", "quiet=${" + str(quiet) + "}"] 615 grp.rdpissuing_keyword(cmd_buf) 616 status, ret_values = \ 617 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 618 if status == "PASS": 619 host = ret_values 620 # Strip everything up to the final period. 621 host = re.sub(r'.*\.', "", host) 622 623 state = DotDict() 624 for sub_state in req_states: 625 if sub_state.startswith("os_"): 626 # We pass "os_" requests on to get_os_state. 627 continue 628 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" 629 exec(cmd_buf) 630 631 if os_host == "": 632 # The caller has not specified an os_host so as far as we're concerned, 633 # it doesn't exist. 634 return state 635 636 os_req_states = [sub_state for sub_state in req_states 637 if sub_state.startswith('os_')] 638 639 if len(os_req_states) > 0: 640 # The caller has specified an os_host and they have requested 641 # information on os substates. 642 643 # Based on the information gathered on bmc, we'll try to make a 644 # determination of whether the os is even up. We'll pass the result 645 # of that assessment to get_os_state to enhance performance. 646 os_up_match = DotDict() 647 for sub_state in master_os_up_match: 648 if sub_state in req_states: 649 os_up_match[sub_state] = master_os_up_match[sub_state] 650 os_up = compare_states(state, os_up_match) 651 os_state = get_os_state(os_host=os_host, 652 os_username=os_username, 653 os_password=os_password, 654 req_states=os_req_states, 655 os_up=os_up, 656 quiet=quiet) 657 # Append os_state dictionary to ours. 658 state.update(os_state) 659 660 return state 661 662############################################################################### 663 664 665############################################################################### 666def check_state(match_state, 667 invert=0, 668 print_string="", 669 openbmc_host="", 670 openbmc_username="", 671 openbmc_password="", 672 os_host="", 673 os_username="", 674 os_password="", 675 quiet=None): 676 677 r""" 678 Check that the Open BMC machine's composite state matches the specified 679 state. On success, this keyword returns the machine's composite state as a 680 dictionary. 681 682 Description of arguments: 683 match_state A dictionary whose key/value pairs are "state field"/ 684 "state value". The state value is interpreted as a 685 regular expression. Example call from robot: 686 ${match_state}= Create Dictionary chassis=^On$ 687 ... bmc=^Ready$ 688 ... boot_progress=^FW Progress, Starting OS$ 689 ${state}= Check State &{match_state} 690 invert If this flag is set, this function will succeed if the 691 states do NOT match. 692 print_string This function will print this string to the console prior 693 to getting the state. 694 openbmc_host The DNS name or IP address of the BMC. 695 This defaults to global ${OPENBMC_HOST}. 696 openbmc_username The username to be used to login to the BMC. 697 This defaults to global ${OPENBMC_USERNAME}. 698 openbmc_password The password to be used to login to the BMC. 699 This defaults to global ${OPENBMC_PASSWORD}. 700 os_host The DNS name or IP address of the operating system. 701 This defaults to global ${OS_HOST}. 702 os_username The username to be used to login to the OS. 703 This defaults to global ${OS_USERNAME}. 704 os_password The password to be used to login to the OS. 705 This defaults to global ${OS_PASSWORD}. 706 quiet Indicates whether status details should be written to the 707 console. Defaults to either global value of ${QUIET} or 708 to 1. 709 """ 710 711 quiet = grp.set_quiet_default(quiet, 1) 712 713 grp.rprint(print_string) 714 715 req_states = match_state.keys() 716 # Initialize state. 717 state = get_state(openbmc_host=openbmc_host, 718 openbmc_username=openbmc_username, 719 openbmc_password=openbmc_password, 720 os_host=os_host, 721 os_username=os_username, 722 os_password=os_password, 723 req_states=req_states, 724 quiet=quiet) 725 if not quiet: 726 gp.print_var(state) 727 728 match = compare_states(state, match_state) 729 730 if invert and match: 731 fail_msg = "The current state of the machine matches the match" +\ 732 " state:\n" + gp.sprint_varx("state", state) 733 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 734 elif not invert and not match: 735 fail_msg = "The current state of the machine does NOT match the" +\ 736 " match state:\n" +\ 737 gp.sprint_varx("state", state) 738 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 739 740 return state 741 742############################################################################### 743 744 745############################################################################### 746def wait_state(match_state=(), 747 wait_time="1 min", 748 interval="1 second", 749 invert=0, 750 openbmc_host="", 751 openbmc_username="", 752 openbmc_password="", 753 os_host="", 754 os_username="", 755 os_password="", 756 quiet=None): 757 758 r""" 759 Wait for the Open BMC machine's composite state to match the specified 760 state. On success, this keyword returns the machine's composite state as 761 a dictionary. 762 763 Description of arguments: 764 match_state A dictionary whose key/value pairs are "state field"/ 765 "state value". See check_state (above) for details. 766 wait_time The total amount of time to wait for the desired state. 767 This value may be expressed in Robot Framework's time 768 format (e.g. 1 minute, 2 min 3 s, 4.5). 769 interval The amount of time between state checks. 770 This value may be expressed in Robot Framework's time 771 format (e.g. 1 minute, 2 min 3 s, 4.5). 772 invert If this flag is set, this function will for the state of 773 the machine to cease to match the match state. 774 openbmc_host The DNS name or IP address of the BMC. 775 This defaults to global ${OPENBMC_HOST}. 776 openbmc_username The username to be used to login to the BMC. 777 This defaults to global ${OPENBMC_USERNAME}. 778 openbmc_password The password to be used to login to the BMC. 779 This defaults to global ${OPENBMC_PASSWORD}. 780 os_host The DNS name or IP address of the operating system. 781 This defaults to global ${OS_HOST}. 782 os_username The username to be used to login to the OS. 783 This defaults to global ${OS_USERNAME}. 784 os_password The password to be used to login to the OS. 785 This defaults to global ${OS_PASSWORD}. 786 quiet Indicates whether status details should be written to the 787 console. Defaults to either global value of ${QUIET} or 788 to 1. 789 """ 790 791 quiet = grp.set_quiet_default(quiet, 1) 792 793 if not quiet: 794 if invert: 795 alt_text = "cease to " 796 else: 797 alt_text = "" 798 gp.print_timen("Checking every " + str(interval) + " for up to " + 799 str(wait_time) + " for the state of the machine to " + 800 alt_text + "match the state shown below.") 801 gp.print_var(match_state) 802 803 if quiet: 804 print_string = "" 805 else: 806 print_string = "#" 807 808 debug = int(BuiltIn().get_variable_value("${debug}", "0")) 809 if debug: 810 # In debug we print state so no need to print the "#". 811 print_string = "" 812 check_state_quiet = 1 - debug 813 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}", 814 "print_string=" + print_string, "openbmc_host=" + openbmc_host, 815 "openbmc_username=" + openbmc_username, 816 "openbmc_password=" + openbmc_password, "os_host=" + os_host, 817 "os_username=" + os_username, "os_password=" + os_password, 818 "quiet=${" + str(check_state_quiet) + "}"] 819 grp.rdpissuing_keyword(cmd_buf) 820 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval, 821 *cmd_buf) 822 if not quiet: 823 gp.printn() 824 if invert: 825 gp.print_timen("The states no longer match:") 826 else: 827 gp.print_timen("The states match:") 828 gp.print_var(state) 829 830 return state 831 832############################################################################### 833 834 835############################################################################### 836def wait_for_comm_cycle(start_boot_seconds): 837 838 r""" 839 Wait for communications to the BMC to stop working and then resume working. 840 This function is useful when you have initiated some kind of reboot. 841 842 Description of arguments: 843 start_boot_seconds The time that the boot test started. The format is the 844 epoch time in seconds, i.e. the number of seconds since 845 1970-01-01 00:00:00 UTC. This value should be obtained 846 from the BMC so that it is not dependent on any kind of 847 synchronization between this machine and the target BMC 848 This will allow this program to work correctly even in 849 a simulated environment. This value should be obtained 850 by the caller prior to initiating a reboot. It can be 851 obtained as follows: 852 state = st.get_state(req_states=['epoch_seconds']) 853 """ 854 855 # Validate parms. 856 error_message = gv.svalid_integer(start_boot_seconds, 857 var_name="start_boot_seconds") 858 if error_message != "": 859 BuiltIn().fail(gp.sprint_error(error_message)) 860 861 match_state = anchor_state(DotDict([('packet_loss', '100')])) 862 # Wait for 100% packet loss trying to ping machine. 863 wait_state(match_state, wait_time="3 mins", interval="0 seconds") 864 865 match_state['packet_loss'] = '^0$' 866 # Wait for 0% packet loss trying to ping machine. 867 wait_state(match_state, wait_time="4 mins", interval="0 seconds") 868 869 # Get the uptime and epoch seconds for comparisons. We want to be sure 870 # that the uptime is less than the elapsed boot time. Further proof that 871 # a reboot has indeed occurred (vs random network instability giving a 872 # false positive. 873 state = get_state(req_states=['uptime', 'epoch_seconds']) 874 875 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds 876 gp.print_var(elapsed_boot_time) 877 if int(float(state['uptime'])) < elapsed_boot_time: 878 uptime = state['uptime'] 879 gp.print_var(uptime) 880 gp.print_timen("The uptime is less than the elapsed boot time," + 881 " as expected.") 882 else: 883 error_message = "The uptime is greater than the elapsed boot time," +\ 884 " which is unexpected:\n" +\ 885 gp.sprint_var(start_boot_seconds) +\ 886 gp.sprint_var(state) 887 BuiltIn().fail(gp.sprint_error(error_message)) 888 889 gp.print_timen("Verifying that REST API interface is working.") 890 match_state = DotDict([('rest', '^1$')]) 891 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds") 892 893############################################################################### 894