1#!/usr/bin/env python 2 3r""" 4This module contains functions having to do with machine state: get_state, 5check_state, wait_state, etc. 6 7The 'State' is a composite of many pieces of data. Therefore, the functions 8in this module define state as an ordered dictionary. Here is an example of 9some test output showing machine state: 10 11default_state: 12 default_state[chassis]: On 13 default_state[boot_progress]: OSStart 14 default_state[operating_system]: BootComplete 15 default_state[host]: Running 16 default_state[os_ping]: 1 17 default_state[os_login]: 1 18 default_state[os_run_cmd]: 1 19 20Different users may very well have different needs when inquiring about 21state. Support for new pieces of state information may be added to this 22module as needed. 23 24By using the wait_state function, a caller can start a boot and then wait for 25a precisely defined state to indicate that the boot has succeeded. If 26the boot fails, they can see exactly why by looking at the current state as 27compared with the expected state. 28""" 29 30import gen_print as gp 31import gen_robot_print as grp 32import gen_valid as gv 33import gen_robot_utils as gru 34import gen_cmd as gc 35import bmc_ssh_utils as bsu 36 37from robot.libraries.BuiltIn import BuiltIn 38from robot.utils import DotDict 39 40import re 41import os 42import sys 43import imp 44 45 46# NOTE: Avoid importing utils.robot because utils.robot imports state.py 47# (indirectly) which will cause failures. 48gru.my_import_resource("rest_client.robot") 49 50base_path = os.path.dirname(os.path.dirname( 51 imp.find_module("gen_robot_print")[1])) + os.sep 52sys.path.append(base_path + "data/") 53 54# Previously, I had this coded: 55# import variables as var 56# However, we ran into a problem where a robot program did this... 57# Variables ../../lib/ras/variables.py 58# Prior to doing this... 59# Library ../lib/state.py 60 61# This caused the wrong variables.py file to be selected. Attempts to fix this 62# have failed so far. For the moment, we will hard-code the value we need from 63# the file. 64 65SYSTEM_STATE_URI = "/xyz/openbmc_project/state/" 66 67# The BMC code has recently been changed as far as what states are defined and 68# what the state values can be. This module now has a means of processing both 69# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e. 70# OBMC_STATES_VERSION = 1). 71# The caller can set environment variable OBMC_STATES_VERSION to dictate 72# whether we're processing old or new style states. If OBMC_STATES_VERSION is 73# not set it will default to 1. 74 75# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old 76# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code 77# is being removed but the OBMC_STATES_VERSION value will stay for now in the 78# event that it is needed in the future. 79 80OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1)) 81 82# When a user calls get_state w/o specifying req_states, default_req_states 83# is used as its value. 84default_req_states = ['rest', 85 'chassis', 86 'bmc', 87 'boot_progress', 88 'operating_system', 89 'host', 90 'os_ping', 91 'os_login', 92 'os_run_cmd'] 93 94# valid_req_states is a list of sub states supported by the get_state function. 95# valid_req_states, default_req_states and master_os_up_match are used by the 96# get_state function. 97valid_req_states = ['ping', 98 'packet_loss', 99 'uptime', 100 'epoch_seconds', 101 'rest', 102 'chassis', 103 'requested_chassis', 104 'bmc', 105 'requested_bmc', 106 'boot_progress', 107 'operating_system', 108 'host', 109 'requested_host', 110 'attempts_left', 111 'os_ping', 112 'os_login', 113 'os_run_cmd'] 114 115# valid_os_req_states and default_os_req_states are used by the os_get_state 116# function. 117# valid_os_req_states is a list of state information supported by the 118# get_os_state function. 119valid_os_req_states = ['os_ping', 120 'os_login', 121 'os_run_cmd'] 122# When a user calls get_os_state w/o specifying req_states, 123# default_os_req_states is used as its value. 124default_os_req_states = ['os_ping', 125 'os_login', 126 'os_run_cmd'] 127 128# Presently, some BMCs appear to not keep time very well. This environment 129# variable directs the get_state function to use either the BMC's epoch time 130# or the local epoch time. 131USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0)) 132 133# Useful state constant definition(s). 134# default_state is an initial value which may be of use to callers. 135default_state = DotDict([('rest', '1'), 136 ('chassis', 'On'), 137 ('bmc', 'Ready'), 138 ('boot_progress', 'OSStart'), 139 ('operating_system', 'BootComplete'), 140 ('host', 'Running'), 141 ('os_ping', '1'), 142 ('os_login', '1'), 143 ('os_run_cmd', '1')]) 144 145# A match state for checking that the system is at "standby". 146standby_match_state = DotDict([('rest', '^1$'), 147 ('chassis', '^Off$'), 148 ('bmc', '^Ready$'), 149 ('boot_progress', '^$'), 150 ('operating_system', '^$'), 151 ('host', '^$')]) 152 153# A match state for checking that the system is at "os running". 154os_running_match_state = DotDict([('chassis', '^On$'), 155 ('bmc', '^Ready$'), 156 ('boot_progress', 157 'FW Progress, Starting OS|OSStart'), 158 ('operating_system', 'BootComplete'), 159 ('host', '^Running$'), 160 ('os_ping', '^1$'), 161 ('os_login', '^1$'), 162 ('os_run_cmd', '^1$')]) 163 164# A master dictionary to determine whether the os may be up. 165master_os_up_match = DotDict([('chassis', '^On$'), 166 ('bmc', '^Ready$'), 167 ('boot_progress', 168 'FW Progress, Starting OS|OSStart'), 169 ('operating_system', 'BootComplete'), 170 ('host', '^Running|Quiesced$')]) 171 172invalid_state_match = DotDict([('rest', '^$'), 173 ('chassis', '^$'), 174 ('bmc', '^$'), 175 ('boot_progress', '^$'), 176 ('operating_system', '^$'), 177 ('host', '^$')]) 178 179 180def return_state_constant(state_name='default_state'): 181 r""" 182 Return the named state dictionary constant. 183 """ 184 185 return eval(state_name) 186 187 188def anchor_state(state): 189 r""" 190 Add regular expression anchors ("^" and "$") to the beginning and end of 191 each item in the state dictionary passed in. Return the resulting 192 dictionary. 193 194 Description of Arguments: 195 state A dictionary such as the one returned by the get_state() 196 function. 197 """ 198 199 anchored_state = state.copy() 200 for key, match_state_value in anchored_state.items(): 201 anchored_state[key] = "^" + str(anchored_state[key]) + "$" 202 203 return anchored_state 204 205 206def strip_anchor_state(state): 207 r""" 208 Strip regular expression anchors ("^" and "$") from the beginning and end 209 of each item in the state dictionary passed in. Return the resulting 210 dictionary. 211 212 Description of Arguments: 213 state A dictionary such as the one returned by the get_state() 214 function. 215 """ 216 217 stripped_state = state.copy() 218 for key, match_state_value in stripped_state.items(): 219 stripped_state[key] = stripped_state[key].strip("^$") 220 221 return stripped_state 222 223 224def compare_states(state, 225 match_state, 226 match_type='and'): 227 r""" 228 Compare 2 state dictionaries. Return True if they match and False if they 229 don't. Note that the match_state dictionary does not need to have an entry 230 corresponding to each entry in the state dictionary. But for each entry 231 that it does have, the corresponding state entry will be checked for a 232 match. 233 234 Description of arguments: 235 state A state dictionary such as the one returned by the 236 get_state function. 237 match_state A dictionary whose key/value pairs are "state field"/ 238 "state value". The state value is interpreted as a 239 regular expression. Every value in this dictionary is 240 considered. When match_type is 'and', if each and every 241 comparison matches, the two dictionaries are considered to 242 be matching. If match_type is 'or', if any two of the 243 elements compared match, the two dictionaries are 244 considered to be matching. 245 This value may also be any string accepted by 246 return_state_constant (e.g. "standby_match_state"). 247 In such a case this function will call 248 return_state_constant to convert it to a proper 249 dictionary as described above. 250 match_type This may be 'and' or 'or'. 251 """ 252 253 error_message = gv.svalid_value(match_type, var_name="match_type", 254 valid_values=['and', 'or']) 255 if error_message != "": 256 BuiltIn().fail(gp.sprint_error(error_message)) 257 258 try: 259 match_state = return_state_constant(match_state) 260 except TypeError: 261 pass 262 263 default_match = (match_type == 'and') 264 for key, match_state_value in match_state.items(): 265 # Blank match_state_value means "don't care". 266 if match_state_value == "": 267 continue 268 try: 269 match = (re.match(match_state_value, str(state[key])) is not None) 270 except KeyError: 271 match = False 272 273 if match != default_match: 274 return match 275 276 return default_match 277 278 279def get_os_state(os_host="", 280 os_username="", 281 os_password="", 282 req_states=default_os_req_states, 283 os_up=True, 284 quiet=None): 285 r""" 286 Get component states for the operating system such as ping, login, 287 etc, put them into a dictionary and return them to the caller. 288 289 Note that all substate values are strings. 290 291 Description of arguments: 292 os_host The DNS name or IP address of the operating system. 293 This defaults to global ${OS_HOST}. 294 os_username The username to be used to login to the OS. 295 This defaults to global ${OS_USERNAME}. 296 os_password The password to be used to login to the OS. 297 This defaults to global ${OS_PASSWORD}. 298 req_states This is a list of states whose values are being requested by 299 the caller. 300 os_up If the caller knows that the os can't possibly be up, it can 301 improve performance by passing os_up=False. This function 302 will then simply return default values for all requested os 303 sub states. 304 quiet Indicates whether status details (e.g. curl commands) should 305 be written to the console. 306 Defaults to either global value of ${QUIET} or to 1. 307 """ 308 309 quiet = int(gp.get_var_value(quiet, 0)) 310 311 # Set parm defaults where necessary and validate all parms. 312 if os_host == "": 313 os_host = BuiltIn().get_variable_value("${OS_HOST}") 314 error_message = gv.svalid_value(os_host, var_name="os_host", 315 invalid_values=[None, ""]) 316 if error_message != "": 317 BuiltIn().fail(gp.sprint_error(error_message)) 318 319 if os_username == "": 320 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 321 error_message = gv.svalid_value(os_username, var_name="os_username", 322 invalid_values=[None, ""]) 323 if error_message != "": 324 BuiltIn().fail(gp.sprint_error(error_message)) 325 326 if os_password == "": 327 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 328 error_message = gv.svalid_value(os_password, var_name="os_password", 329 invalid_values=[None, ""]) 330 if error_message != "": 331 BuiltIn().fail(gp.sprint_error(error_message)) 332 333 invalid_req_states = [sub_state for sub_state in req_states 334 if sub_state not in valid_os_req_states] 335 if len(invalid_req_states) > 0: 336 error_message = "The following req_states are not supported:\n" +\ 337 gp.sprint_var(invalid_req_states) 338 BuiltIn().fail(gp.sprint_error(error_message)) 339 340 # Initialize all substate values supported by this function. 341 os_ping = 0 342 os_login = 0 343 os_run_cmd = 0 344 345 if os_up: 346 if 'os_ping' in req_states: 347 # See if the OS pings. 348 rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + os_host, 349 print_output=0, show_err=0, 350 ignore_err=1) 351 if rc == 0: 352 os_ping = 1 353 354 # Programming note: All attributes which do not require an ssh login 355 # should have been processed by this point. 356 master_req_login = ['os_login', 'os_run_cmd'] 357 req_login = [sub_state for sub_state in req_states if sub_state in 358 master_req_login] 359 must_login = (len(req_login) > 0) 360 361 if must_login: 362 output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet, 363 ignore_err=1, 364 time_out=20) 365 if rc == 0: 366 os_login = 1 367 os_run_cmd = 1 368 else: 369 gp.dprint_vars(output, stderr) 370 gp.dprint_vars(rc, 1) 371 372 os_state = DotDict() 373 for sub_state in req_states: 374 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" 375 exec(cmd_buf) 376 377 return os_state 378 379 380def get_state(openbmc_host="", 381 openbmc_username="", 382 openbmc_password="", 383 os_host="", 384 os_username="", 385 os_password="", 386 req_states=default_req_states, 387 quiet=None): 388 r""" 389 Get component states such as chassis state, bmc state, etc, put them into a 390 dictionary and return them to the caller. 391 392 Note that all substate values are strings. 393 394 Description of arguments: 395 openbmc_host The DNS name or IP address of the BMC. 396 This defaults to global ${OPENBMC_HOST}. 397 openbmc_username The username to be used to login to the BMC. 398 This defaults to global ${OPENBMC_USERNAME}. 399 openbmc_password The password to be used to login to the BMC. 400 This defaults to global ${OPENBMC_PASSWORD}. 401 os_host The DNS name or IP address of the operating system. 402 This defaults to global ${OS_HOST}. 403 os_username The username to be used to login to the OS. 404 This defaults to global ${OS_USERNAME}. 405 os_password The password to be used to login to the OS. 406 This defaults to global ${OS_PASSWORD}. 407 req_states This is a list of states whose values are being requested 408 by the caller. 409 quiet Indicates whether status details (e.g. curl commands) 410 should be written to the console. 411 Defaults to either global value of ${QUIET} or to 1. 412 """ 413 414 quiet = int(gp.get_var_value(quiet, 0)) 415 416 # Set parm defaults where necessary and validate all parms. 417 if openbmc_host == "": 418 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") 419 error_message = gv.svalid_value(openbmc_host, 420 var_name="openbmc_host", 421 invalid_values=[None, ""]) 422 if error_message != "": 423 BuiltIn().fail(gp.sprint_error(error_message)) 424 425 if openbmc_username == "": 426 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") 427 error_message = gv.svalid_value(openbmc_username, 428 var_name="openbmc_username", 429 invalid_values=[None, ""]) 430 if error_message != "": 431 BuiltIn().fail(gp.sprint_error(error_message)) 432 433 if openbmc_password == "": 434 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") 435 error_message = gv.svalid_value(openbmc_password, 436 var_name="openbmc_password", 437 invalid_values=[None, ""]) 438 if error_message != "": 439 BuiltIn().fail(gp.sprint_error(error_message)) 440 441 # NOTE: OS parms are optional. 442 if os_host == "": 443 os_host = BuiltIn().get_variable_value("${OS_HOST}") 444 if os_host is None: 445 os_host = "" 446 447 if os_username is "": 448 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 449 if os_username is None: 450 os_username = "" 451 452 if os_password is "": 453 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 454 if os_password is None: 455 os_password = "" 456 457 invalid_req_states = [sub_state for sub_state in req_states 458 if sub_state not in valid_req_states] 459 if len(invalid_req_states) > 0: 460 error_message = "The following req_states are not supported:\n" +\ 461 gp.sprint_var(invalid_req_states) 462 BuiltIn().fail(gp.sprint_error(error_message)) 463 464 # Initialize all substate values supported by this function. 465 ping = 0 466 packet_loss = '' 467 uptime = '' 468 epoch_seconds = '' 469 rest = '' 470 chassis = '' 471 requested_chassis = '' 472 bmc = '' 473 requested_bmc = '' 474 boot_progress = '' 475 operating_system = '' 476 host = '' 477 requested_host = '' 478 attempts_left = '' 479 480 # Get the component states. 481 if 'ping' in req_states: 482 # See if the OS pings. 483 rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host, 484 print_output=0, show_err=0, 485 ignore_err=1) 486 if rc == 0: 487 ping = 1 488 489 if 'packet_loss' in req_states: 490 # See if the OS pings. 491 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\ 492 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" 493 rc, out_buf = gc.shell_cmd(cmd_buf, 494 print_output=0, show_err=0, 495 ignore_err=1) 496 if rc == 0: 497 packet_loss = out_buf.rstrip("\n") 498 499 if 'uptime' in req_states: 500 # Sometimes reading uptime results in a blank value. Call with 501 # wait_until_keyword_succeeds to ensure a non-blank value is obtained. 502 remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\ 503 " && [ ! -z \"${uptime}\" ] && echo ${uptime}" 504 cmd_buf = ["BMC Execute Command", 505 re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1'] 506 if not quiet: 507 # Get loc_test_mode parm for improved output on pissuing. 508 # See sprint_issuing in gen_print.py for details. 509 loc_test_mode = int(gp.get_var_value(var_name="test_mode", 510 default=0)) 511 grp.rpissuing_keyword(cmd_buf, loc_test_mode) 512 gp.pissuing(remote_cmd_buf, loc_test_mode) 513 try: 514 stdout, stderr, rc =\ 515 BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec", 516 *cmd_buf) 517 if rc == 0 and stderr == "": 518 uptime = stdout 519 except AssertionError as my_assertion_error: 520 pass 521 522 if 'epoch_seconds' in req_states: 523 date_cmd_buf = "date -u +%s" 524 if USE_BMC_EPOCH_TIME: 525 cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}'] 526 if not quiet: 527 grp.rpissuing_keyword(cmd_buf) 528 status, ret_values = \ 529 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 530 if status == "PASS": 531 stdout, stderr, rc = ret_values 532 if rc == 0 and stderr == "": 533 epoch_seconds = stdout.rstrip("\n") 534 else: 535 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf, 536 quiet=quiet, 537 print_output=0) 538 if shell_rc == 0: 539 epoch_seconds = out_buf.rstrip("\n") 540 541 master_req_rest = ['rest', 'host', 'requested_host', 'operating_system', 542 'attempts_left', 'boot_progress', 'chassis', 543 'requested_chassis' 'bmc' 'requested_bmc'] 544 545 req_rest = [sub_state for sub_state in req_states if sub_state in 546 master_req_rest] 547 need_rest = (len(req_rest) > 0) 548 state = DotDict() 549 if need_rest: 550 cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate", 551 "quiet=${" + str(quiet) + "}"] 552 grp.rdpissuing_keyword(cmd_buf) 553 status, ret_values = \ 554 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 555 if status == "PASS": 556 state['rest'] = '1' 557 else: 558 state['rest'] = '0' 559 560 if int(state['rest']): 561 for url_path in ret_values: 562 for attr_name in ret_values[url_path]: 563 # Create a state key value based on the attr_name. 564 try: 565 ret_values[url_path][attr_name] = \ 566 re.sub(r'.*\.', "", 567 ret_values[url_path][attr_name]) 568 except TypeError: 569 pass 570 # Do some key name manipulations. 571 new_attr_name = re.sub(r'^Current|(State|Transition)$', 572 "", attr_name) 573 new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name) 574 new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1', 575 new_attr_name) 576 new_attr_name = new_attr_name.lower().lstrip("_") 577 new_attr_name = re.sub(r'power', r'chassis', new_attr_name) 578 if new_attr_name in req_states: 579 state[new_attr_name] = ret_values[url_path][attr_name] 580 581 for sub_state in req_states: 582 if sub_state in state: 583 continue 584 if sub_state.startswith("os_"): 585 # We pass "os_" requests on to get_os_state. 586 continue 587 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" 588 exec(cmd_buf) 589 590 if os_host == "": 591 # The caller has not specified an os_host so as far as we're concerned, 592 # it doesn't exist. 593 return state 594 595 os_req_states = [sub_state for sub_state in req_states 596 if sub_state.startswith('os_')] 597 598 if len(os_req_states) > 0: 599 # The caller has specified an os_host and they have requested 600 # information on os substates. 601 602 # Based on the information gathered on bmc, we'll try to make a 603 # determination of whether the os is even up. We'll pass the result 604 # of that assessment to get_os_state to enhance performance. 605 os_up_match = DotDict() 606 for sub_state in master_os_up_match: 607 if sub_state in req_states: 608 os_up_match[sub_state] = master_os_up_match[sub_state] 609 os_up = compare_states(state, os_up_match) 610 os_state = get_os_state(os_host=os_host, 611 os_username=os_username, 612 os_password=os_password, 613 req_states=os_req_states, 614 os_up=os_up, 615 quiet=quiet) 616 # Append os_state dictionary to ours. 617 state.update(os_state) 618 619 return state 620 621 622exit_wait_early_message = "" 623 624 625def set_exit_wait_early_message(value): 626 r""" 627 Set global exit_wait_early_message to the indicated value. 628 629 This is a mechanism by which the programmer can do an early exit from 630 wait_until_keyword_succeeds() based on some special condition. 631 632 Description of argument(s): 633 value The value to assign to the global 634 exit_wait_early_message. 635 """ 636 637 global exit_wait_early_message 638 exit_wait_early_message = value 639 640 641def check_state(match_state, 642 invert=0, 643 print_string="", 644 openbmc_host="", 645 openbmc_username="", 646 openbmc_password="", 647 os_host="", 648 os_username="", 649 os_password="", 650 quiet=None): 651 r""" 652 Check that the Open BMC machine's composite state matches the specified 653 state. On success, this keyword returns the machine's composite state as a 654 dictionary. 655 656 Description of arguments: 657 match_state A dictionary whose key/value pairs are "state field"/ 658 "state value". The state value is interpreted as a 659 regular expression. Example call from robot: 660 ${match_state}= Create Dictionary chassis=^On$ 661 ... bmc=^Ready$ 662 ... boot_progress=^OSStart$ 663 ${state}= Check State &{match_state} 664 invert If this flag is set, this function will succeed if the 665 states do NOT match. 666 print_string This function will print this string to the console prior 667 to getting the state. 668 openbmc_host The DNS name or IP address of the BMC. 669 This defaults to global ${OPENBMC_HOST}. 670 openbmc_username The username to be used to login to the BMC. 671 This defaults to global ${OPENBMC_USERNAME}. 672 openbmc_password The password to be used to login to the BMC. 673 This defaults to global ${OPENBMC_PASSWORD}. 674 os_host The DNS name or IP address of the operating system. 675 This defaults to global ${OS_HOST}. 676 os_username The username to be used to login to the OS. 677 This defaults to global ${OS_USERNAME}. 678 os_password The password to be used to login to the OS. 679 This defaults to global ${OS_PASSWORD}. 680 quiet Indicates whether status details should be written to the 681 console. Defaults to either global value of ${QUIET} or 682 to 1. 683 """ 684 685 quiet = int(gp.get_var_value(quiet, 0)) 686 687 grp.rprint(print_string) 688 689 req_states = match_state.keys() 690 # Initialize state. 691 state = get_state(openbmc_host=openbmc_host, 692 openbmc_username=openbmc_username, 693 openbmc_password=openbmc_password, 694 os_host=os_host, 695 os_username=os_username, 696 os_password=os_password, 697 req_states=req_states, 698 quiet=quiet) 699 if not quiet: 700 gp.print_var(state) 701 702 if exit_wait_early_message != "": 703 # The exit_wait_early_message has been set by a signal handler so we 704 # will exit "successfully". It is incumbent upon the calling function 705 # (e.g. wait_state) to check/clear this variable and to fail 706 # appropriately. 707 return state 708 709 match = compare_states(state, match_state) 710 711 if invert and match: 712 fail_msg = "The current state of the machine matches the match" +\ 713 " state:\n" + gp.sprint_varx("state", state) 714 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 715 elif not invert and not match: 716 fail_msg = "The current state of the machine does NOT match the" +\ 717 " match state:\n" +\ 718 gp.sprint_varx("state", state) 719 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 720 721 return state 722 723 724def wait_state(match_state=(), 725 wait_time="1 min", 726 interval="1 second", 727 invert=0, 728 openbmc_host="", 729 openbmc_username="", 730 openbmc_password="", 731 os_host="", 732 os_username="", 733 os_password="", 734 quiet=None): 735 r""" 736 Wait for the Open BMC machine's composite state to match the specified 737 state. On success, this keyword returns the machine's composite state as 738 a dictionary. 739 740 Description of arguments: 741 match_state A dictionary whose key/value pairs are "state field"/ 742 "state value". See check_state (above) for details. 743 This value may also be any string accepted by 744 return_state_constant (e.g. "standby_match_state"). 745 In such a case this function will call 746 return_state_constant to convert it to a proper 747 dictionary as described above. 748 wait_time The total amount of time to wait for the desired state. 749 This value may be expressed in Robot Framework's time 750 format (e.g. 1 minute, 2 min 3 s, 4.5). 751 interval The amount of time between state checks. 752 This value may be expressed in Robot Framework's time 753 format (e.g. 1 minute, 2 min 3 s, 4.5). 754 invert If this flag is set, this function will for the state of 755 the machine to cease to match the match state. 756 openbmc_host The DNS name or IP address of the BMC. 757 This defaults to global ${OPENBMC_HOST}. 758 openbmc_username The username to be used to login to the BMC. 759 This defaults to global ${OPENBMC_USERNAME}. 760 openbmc_password The password to be used to login to the BMC. 761 This defaults to global ${OPENBMC_PASSWORD}. 762 os_host The DNS name or IP address of the operating system. 763 This defaults to global ${OS_HOST}. 764 os_username The username to be used to login to the OS. 765 This defaults to global ${OS_USERNAME}. 766 os_password The password to be used to login to the OS. 767 This defaults to global ${OS_PASSWORD}. 768 quiet Indicates whether status details should be written to the 769 console. Defaults to either global value of ${QUIET} or 770 to 1. 771 """ 772 773 quiet = int(gp.get_var_value(quiet, 0)) 774 775 try: 776 match_state = return_state_constant(match_state) 777 except TypeError: 778 pass 779 780 if not quiet: 781 if invert: 782 alt_text = "cease to " 783 else: 784 alt_text = "" 785 gp.print_timen("Checking every " + str(interval) + " for up to " 786 + str(wait_time) + " for the state of the machine to " 787 + alt_text + "match the state shown below.") 788 gp.print_var(match_state) 789 790 if quiet: 791 print_string = "" 792 else: 793 print_string = "#" 794 795 debug = int(BuiltIn().get_variable_value("${debug}", "0")) 796 if debug: 797 # In debug we print state so no need to print the "#". 798 print_string = "" 799 check_state_quiet = 1 - debug 800 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}", 801 "print_string=" + print_string, "openbmc_host=" + openbmc_host, 802 "openbmc_username=" + openbmc_username, 803 "openbmc_password=" + openbmc_password, "os_host=" + os_host, 804 "os_username=" + os_username, "os_password=" + os_password, 805 "quiet=${" + str(check_state_quiet) + "}"] 806 grp.rdpissuing_keyword(cmd_buf) 807 try: 808 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval, 809 *cmd_buf) 810 except AssertionError as my_assertion_error: 811 gp.printn() 812 message = my_assertion_error.args[0] 813 BuiltIn().fail(message) 814 815 if exit_wait_early_message: 816 # The global exit_wait_early_message was set by a signal handler 817 # indicating that we should fail. 818 message = exit_wait_early_message 819 # Clear the exit_wait_early_message variable for future use. 820 set_exit_wait_early_message("") 821 BuiltIn().fail(gp.sprint_error(message)) 822 823 if not quiet: 824 gp.printn() 825 if invert: 826 gp.print_timen("The states no longer match:") 827 else: 828 gp.print_timen("The states match:") 829 gp.print_var(state) 830 831 return state 832 833 834def wait_for_comm_cycle(start_boot_seconds, 835 quiet=None): 836 r""" 837 Wait for communications to the BMC to stop working and then resume working. 838 This function is useful when you have initiated some kind of reboot. 839 840 Description of arguments: 841 start_boot_seconds The time that the boot test started. The format is the 842 epoch time in seconds, i.e. the number of seconds since 843 1970-01-01 00:00:00 UTC. This value should be obtained 844 from the BMC so that it is not dependent on any kind of 845 synchronization between this machine and the target BMC 846 This will allow this program to work correctly even in 847 a simulated environment. This value should be obtained 848 by the caller prior to initiating a reboot. It can be 849 obtained as follows: 850 state = st.get_state(req_states=['epoch_seconds']) 851 """ 852 853 quiet = int(gp.get_var_value(quiet, 0)) 854 855 # Validate parms. 856 error_message = gv.svalid_integer(start_boot_seconds, 857 var_name="start_boot_seconds") 858 if error_message != "": 859 BuiltIn().fail(gp.sprint_error(error_message)) 860 861 match_state = anchor_state(DotDict([('packet_loss', '100')])) 862 # Wait for 100% packet loss trying to ping machine. 863 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 864 865 match_state['packet_loss'] = '^0$' 866 # Wait for 0% packet loss trying to ping machine. 867 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 868 869 # Get the uptime and epoch seconds for comparisons. We want to be sure 870 # that the uptime is less than the elapsed boot time. Further proof that 871 # a reboot has indeed occurred (vs random network instability giving a 872 # false positive. We also use wait_state because the BMC may take a short 873 # while to be ready to process SSH requests. 874 match_state = DotDict([('uptime', '^[0-9\\.]+$'), 875 ('epoch_seconds', '^[0-9]+$')]) 876 state = wait_state(match_state, wait_time="2 mins", interval="1 second") 877 878 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds 879 gp.qprint_var(elapsed_boot_time) 880 if state['uptime'] == "": 881 error_message = "Unable to obtain uptime from the BMC. BMC is not" +\ 882 " communicating." 883 BuiltIn().fail(gp.sprint_error(error_message)) 884 if int(float(state['uptime'])) < elapsed_boot_time: 885 uptime = state['uptime'] 886 gp.qprint_var(uptime) 887 gp.qprint_timen("The uptime is less than the elapsed boot time," 888 + " as expected.") 889 else: 890 error_message = "The uptime is greater than the elapsed boot time," +\ 891 " which is unexpected:\n" +\ 892 gp.sprint_var(start_boot_seconds) +\ 893 gp.sprint_var(state) 894 BuiltIn().fail(gp.sprint_error(error_message)) 895 896 gp.qprint_timen("Verifying that REST API interface is working.") 897 match_state = DotDict([('rest', '^1$')]) 898 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds") 899