1#!/usr/bin/env python 2 3r""" 4This module contains functions having to do with machine state: get_state, 5check_state, wait_state, etc. 6 7The 'State' is a composite of many pieces of data. Therefore, the functions 8in this module define state as an ordered dictionary. Here is an example of 9some test output showing machine state: 10 11default_state: 12 default_state[chassis]: On 13 default_state[boot_progress]: OSStart 14 default_state[operating_system]: BootComplete 15 default_state[host]: Running 16 default_state[os_ping]: 1 17 default_state[os_login]: 1 18 default_state[os_run_cmd]: 1 19 20Different users may very well have different needs when inquiring about 21state. Support for new pieces of state information may be added to this 22module as needed. 23 24By using the wait_state function, a caller can start a boot and then wait for 25a precisely defined state to indicate that the boot has succeeded. If 26the boot fails, they can see exactly why by looking at the current state as 27compared with the expected state. 28""" 29 30import gen_print as gp 31import gen_valid as gv 32import gen_robot_utils as gru 33import gen_cmd as gc 34import bmc_ssh_utils as bsu 35 36from robot.libraries.BuiltIn import BuiltIn 37from robot.utils import DotDict 38 39import re 40import os 41import sys 42import imp 43 44 45# NOTE: Avoid importing utils.robot because utils.robot imports state.py 46# (indirectly) which will cause failures. 47gru.my_import_resource("rest_client.robot") 48 49base_path = os.path.dirname(os.path.dirname( 50 imp.find_module("gen_robot_print")[1])) + os.sep 51sys.path.append(base_path + "data/") 52 53# Previously, I had this coded: 54# import variables as var 55# However, we ran into a problem where a robot program did this... 56# Variables ../../lib/ras/variables.py 57# Prior to doing this... 58# Library ../lib/state.py 59 60# This caused the wrong variables.py file to be selected. Attempts to fix this 61# have failed so far. For the moment, we will hard-code the value we need from 62# the file. 63 64SYSTEM_STATE_URI = "/xyz/openbmc_project/state/" 65 66# The BMC code has recently been changed as far as what states are defined and 67# what the state values can be. This module now has a means of processing both 68# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e. 69# OBMC_STATES_VERSION = 1). 70# The caller can set environment variable OBMC_STATES_VERSION to dictate 71# whether we're processing old or new style states. If OBMC_STATES_VERSION is 72# not set it will default to 1. 73 74# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old 75# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code 76# is being removed but the OBMC_STATES_VERSION value will stay for now in the 77# event that it is needed in the future. 78 79OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1)) 80 81# When a user calls get_state w/o specifying req_states, default_req_states 82# is used as its value. 83default_req_states = ['rest', 84 'chassis', 85 'bmc', 86 'boot_progress', 87 'operating_system', 88 'host', 89 'os_ping', 90 'os_login', 91 'os_run_cmd'] 92 93# valid_req_states is a list of sub states supported by the get_state function. 94# valid_req_states, default_req_states and master_os_up_match are used by the 95# get_state function. 96valid_req_states = ['ping', 97 'packet_loss', 98 'uptime', 99 'epoch_seconds', 100 'rest', 101 'chassis', 102 'requested_chassis', 103 'bmc', 104 'requested_bmc', 105 'boot_progress', 106 'operating_system', 107 'host', 108 'requested_host', 109 'attempts_left', 110 'os_ping', 111 'os_login', 112 'os_run_cmd'] 113 114# valid_os_req_states and default_os_req_states are used by the os_get_state 115# function. 116# valid_os_req_states is a list of state information supported by the 117# get_os_state function. 118valid_os_req_states = ['os_ping', 119 'os_login', 120 'os_run_cmd'] 121# When a user calls get_os_state w/o specifying req_states, 122# default_os_req_states is used as its value. 123default_os_req_states = ['os_ping', 124 'os_login', 125 'os_run_cmd'] 126 127# Presently, some BMCs appear to not keep time very well. This environment 128# variable directs the get_state function to use either the BMC's epoch time 129# or the local epoch time. 130USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0)) 131 132# Useful state constant definition(s). 133# default_state is an initial value which may be of use to callers. 134default_state = DotDict([('rest', '1'), 135 ('chassis', 'On'), 136 ('bmc', 'Ready'), 137 ('boot_progress', 'OSStart'), 138 ('operating_system', 'BootComplete'), 139 ('host', 'Running'), 140 ('os_ping', '1'), 141 ('os_login', '1'), 142 ('os_run_cmd', '1')]) 143 144# A match state for checking that the system is at "standby". 145standby_match_state = DotDict([('rest', '^1$'), 146 ('chassis', '^Off$'), 147 ('bmc', '^Ready$'), 148 ('boot_progress', '^Off|Unspecified$'), 149 ('operating_system', '^Inactive$'), 150 ('host', '^Off$')]) 151 152# A match state for checking that the system is at "os running". 153os_running_match_state = DotDict([('chassis', '^On$'), 154 ('bmc', '^Ready$'), 155 ('boot_progress', 156 'FW Progress, Starting OS|OSStart'), 157 ('operating_system', 'BootComplete'), 158 ('host', '^Running$'), 159 ('os_ping', '^1$'), 160 ('os_login', '^1$'), 161 ('os_run_cmd', '^1$')]) 162 163# A master dictionary to determine whether the os may be up. 164master_os_up_match = DotDict([('chassis', '^On$'), 165 ('bmc', '^Ready$'), 166 ('boot_progress', 167 'FW Progress, Starting OS|OSStart'), 168 ('operating_system', 'BootComplete'), 169 ('host', '^Running|Quiesced$')]) 170 171invalid_state_match = DotDict([('rest', '^$'), 172 ('chassis', '^$'), 173 ('bmc', '^$'), 174 ('boot_progress', '^$'), 175 ('operating_system', '^$'), 176 ('host', '^$')]) 177 178 179def return_state_constant(state_name='default_state'): 180 r""" 181 Return the named state dictionary constant. 182 """ 183 184 return eval(state_name) 185 186 187def anchor_state(state): 188 r""" 189 Add regular expression anchors ("^" and "$") to the beginning and end of 190 each item in the state dictionary passed in. Return the resulting 191 dictionary. 192 193 Description of Arguments: 194 state A dictionary such as the one returned by the get_state() 195 function. 196 """ 197 198 anchored_state = state.copy() 199 for key, match_state_value in anchored_state.items(): 200 anchored_state[key] = "^" + str(anchored_state[key]) + "$" 201 202 return anchored_state 203 204 205def strip_anchor_state(state): 206 r""" 207 Strip regular expression anchors ("^" and "$") from the beginning and end 208 of each item in the state dictionary passed in. Return the resulting 209 dictionary. 210 211 Description of Arguments: 212 state A dictionary such as the one returned by the get_state() 213 function. 214 """ 215 216 stripped_state = state.copy() 217 for key, match_state_value in stripped_state.items(): 218 stripped_state[key] = stripped_state[key].strip("^$") 219 220 return stripped_state 221 222 223def compare_states(state, 224 match_state, 225 match_type='and'): 226 r""" 227 Compare 2 state dictionaries. Return True if they match and False if they 228 don't. Note that the match_state dictionary does not need to have an entry 229 corresponding to each entry in the state dictionary. But for each entry 230 that it does have, the corresponding state entry will be checked for a 231 match. 232 233 Description of arguments: 234 state A state dictionary such as the one returned by the 235 get_state function. 236 match_state A dictionary whose key/value pairs are "state field"/ 237 "state value". The state value is interpreted as a 238 regular expression. Every value in this dictionary is 239 considered. When match_type is 'and', if each and every 240 comparison matches, the two dictionaries are considered to 241 be matching. If match_type is 'or', if any two of the 242 elements compared match, the two dictionaries are 243 considered to be matching. 244 This value may also be any string accepted by 245 return_state_constant (e.g. "standby_match_state"). 246 In such a case this function will call 247 return_state_constant to convert it to a proper 248 dictionary as described above. 249 match_type This may be 'and' or 'or'. 250 """ 251 252 error_message = gv.valid_value(match_type, var_name="match_type", 253 valid_values=['and', 'or']) 254 if error_message != "": 255 BuiltIn().fail(gp.sprint_error(error_message)) 256 257 try: 258 match_state = return_state_constant(match_state) 259 except TypeError: 260 pass 261 262 default_match = (match_type == 'and') 263 for key, match_state_value in match_state.items(): 264 # Blank match_state_value means "don't care". 265 if match_state_value == "": 266 continue 267 try: 268 match = (re.match(match_state_value, str(state[key])) is not None) 269 except KeyError: 270 match = False 271 272 if match != default_match: 273 return match 274 275 return default_match 276 277 278def get_os_state(os_host="", 279 os_username="", 280 os_password="", 281 req_states=default_os_req_states, 282 os_up=True, 283 quiet=None): 284 r""" 285 Get component states for the operating system such as ping, login, 286 etc, put them into a dictionary and return them to the caller. 287 288 Note that all substate values are strings. 289 290 Description of arguments: 291 os_host The DNS name or IP address of the operating system. 292 This defaults to global ${OS_HOST}. 293 os_username The username to be used to login to the OS. 294 This defaults to global ${OS_USERNAME}. 295 os_password The password to be used to login to the OS. 296 This defaults to global ${OS_PASSWORD}. 297 req_states This is a list of states whose values are being requested by 298 the caller. 299 os_up If the caller knows that the os can't possibly be up, it can 300 improve performance by passing os_up=False. This function 301 will then simply return default values for all requested os 302 sub states. 303 quiet Indicates whether status details (e.g. curl commands) should 304 be written to the console. 305 Defaults to either global value of ${QUIET} or to 1. 306 """ 307 308 quiet = int(gp.get_var_value(quiet, 0)) 309 310 # Set parm defaults where necessary and validate all parms. 311 if os_host == "": 312 os_host = BuiltIn().get_variable_value("${OS_HOST}") 313 error_message = gv.valid_value(os_host, var_name="os_host", 314 invalid_values=[None, ""]) 315 if error_message != "": 316 BuiltIn().fail(gp.sprint_error(error_message)) 317 318 if os_username == "": 319 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 320 error_message = gv.valid_value(os_username, var_name="os_username", 321 invalid_values=[None, ""]) 322 if error_message != "": 323 BuiltIn().fail(gp.sprint_error(error_message)) 324 325 if os_password == "": 326 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 327 error_message = gv.valid_value(os_password, var_name="os_password", 328 invalid_values=[None, ""]) 329 if error_message != "": 330 BuiltIn().fail(gp.sprint_error(error_message)) 331 332 invalid_req_states = [sub_state for sub_state in req_states 333 if sub_state not in valid_os_req_states] 334 if len(invalid_req_states) > 0: 335 error_message = "The following req_states are not supported:\n" +\ 336 gp.sprint_var(invalid_req_states) 337 BuiltIn().fail(gp.sprint_error(error_message)) 338 339 # Initialize all substate values supported by this function. 340 os_ping = 0 341 os_login = 0 342 os_run_cmd = 0 343 344 if os_up: 345 if 'os_ping' in req_states: 346 # See if the OS pings. 347 rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + os_host, 348 print_output=0, show_err=0, 349 ignore_err=1) 350 if rc == 0: 351 os_ping = 1 352 353 # Programming note: All attributes which do not require an ssh login 354 # should have been processed by this point. 355 master_req_login = ['os_login', 'os_run_cmd'] 356 req_login = [sub_state for sub_state in req_states if sub_state in 357 master_req_login] 358 must_login = (len(req_login) > 0) 359 360 if must_login: 361 output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet, 362 ignore_err=1, 363 time_out=20) 364 if rc == 0: 365 os_login = 1 366 os_run_cmd = 1 367 else: 368 gp.dprint_vars(output, stderr) 369 gp.dprint_vars(rc, 1) 370 371 os_state = DotDict() 372 for sub_state in req_states: 373 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" 374 exec(cmd_buf) 375 376 return os_state 377 378 379def get_state(openbmc_host="", 380 openbmc_username="", 381 openbmc_password="", 382 os_host="", 383 os_username="", 384 os_password="", 385 req_states=default_req_states, 386 quiet=None): 387 r""" 388 Get component states such as chassis state, bmc state, etc, put them into a 389 dictionary and return them to the caller. 390 391 Note that all substate values are strings. 392 393 Description of arguments: 394 openbmc_host The DNS name or IP address of the BMC. 395 This defaults to global ${OPENBMC_HOST}. 396 openbmc_username The username to be used to login to the BMC. 397 This defaults to global ${OPENBMC_USERNAME}. 398 openbmc_password The password to be used to login to the BMC. 399 This defaults to global ${OPENBMC_PASSWORD}. 400 os_host The DNS name or IP address of the operating system. 401 This defaults to global ${OS_HOST}. 402 os_username The username to be used to login to the OS. 403 This defaults to global ${OS_USERNAME}. 404 os_password The password to be used to login to the OS. 405 This defaults to global ${OS_PASSWORD}. 406 req_states This is a list of states whose values are being requested 407 by the caller. 408 quiet Indicates whether status details (e.g. curl commands) 409 should be written to the console. 410 Defaults to either global value of ${QUIET} or to 1. 411 """ 412 413 quiet = int(gp.get_var_value(quiet, 0)) 414 415 # Set parm defaults where necessary and validate all parms. 416 if openbmc_host == "": 417 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") 418 error_message = gv.valid_value(openbmc_host, 419 var_name="openbmc_host", 420 invalid_values=[None, ""]) 421 if error_message != "": 422 BuiltIn().fail(gp.sprint_error(error_message)) 423 424 if openbmc_username == "": 425 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") 426 error_message = gv.valid_value(openbmc_username, 427 var_name="openbmc_username", 428 invalid_values=[None, ""]) 429 if error_message != "": 430 BuiltIn().fail(gp.sprint_error(error_message)) 431 432 if openbmc_password == "": 433 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") 434 error_message = gv.valid_value(openbmc_password, 435 var_name="openbmc_password", 436 invalid_values=[None, ""]) 437 if error_message != "": 438 BuiltIn().fail(gp.sprint_error(error_message)) 439 440 # NOTE: OS parms are optional. 441 if os_host == "": 442 os_host = BuiltIn().get_variable_value("${OS_HOST}") 443 if os_host is None: 444 os_host = "" 445 446 if os_username is "": 447 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 448 if os_username is None: 449 os_username = "" 450 451 if os_password is "": 452 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 453 if os_password is None: 454 os_password = "" 455 456 invalid_req_states = [sub_state for sub_state in req_states 457 if sub_state not in valid_req_states] 458 if len(invalid_req_states) > 0: 459 error_message = "The following req_states are not supported:\n" +\ 460 gp.sprint_var(invalid_req_states) 461 BuiltIn().fail(gp.sprint_error(error_message)) 462 463 # Initialize all substate values supported by this function. 464 ping = 0 465 packet_loss = '' 466 uptime = '' 467 epoch_seconds = '' 468 rest = '' 469 chassis = '' 470 requested_chassis = '' 471 bmc = '' 472 requested_bmc = '' 473 boot_progress = '' 474 operating_system = '' 475 host = '' 476 requested_host = '' 477 attempts_left = '' 478 479 # Get the component states. 480 if 'ping' in req_states: 481 # See if the OS pings. 482 rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host, 483 print_output=0, show_err=0, 484 ignore_err=1) 485 if rc == 0: 486 ping = 1 487 488 if 'packet_loss' in req_states: 489 # See if the OS pings. 490 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\ 491 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" 492 rc, out_buf = gc.shell_cmd(cmd_buf, 493 print_output=0, show_err=0, 494 ignore_err=1) 495 if rc == 0: 496 packet_loss = out_buf.rstrip("\n") 497 498 if 'uptime' in req_states: 499 # Sometimes reading uptime results in a blank value. Call with 500 # wait_until_keyword_succeeds to ensure a non-blank value is obtained. 501 remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\ 502 " && [ ! -z \"${uptime}\" ] && echo ${uptime}" 503 cmd_buf = ["BMC Execute Command", 504 re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1', 505 'test_mode=0'] 506 gp.print_issuing(cmd_buf, 0) 507 gp.print_issuing(remote_cmd_buf, 0) 508 try: 509 stdout, stderr, rc =\ 510 BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec", 511 *cmd_buf) 512 if rc == 0 and stderr == "": 513 uptime = stdout 514 except AssertionError as my_assertion_error: 515 pass 516 517 if 'epoch_seconds' in req_states: 518 date_cmd_buf = "date -u +%s" 519 if USE_BMC_EPOCH_TIME: 520 cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}'] 521 if not quiet: 522 gp.print_issuing(cmd_buf) 523 status, ret_values = \ 524 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 525 if status == "PASS": 526 stdout, stderr, rc = ret_values 527 if rc == 0 and stderr == "": 528 epoch_seconds = stdout.rstrip("\n") 529 else: 530 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf, 531 quiet=quiet, 532 print_output=0) 533 if shell_rc == 0: 534 epoch_seconds = out_buf.rstrip("\n") 535 536 master_req_rest = ['rest', 'host', 'requested_host', 'operating_system', 537 'attempts_left', 'boot_progress', 'chassis', 538 'requested_chassis' 'bmc' 'requested_bmc'] 539 540 req_rest = [sub_state for sub_state in req_states if sub_state in 541 master_req_rest] 542 need_rest = (len(req_rest) > 0) 543 state = DotDict() 544 if need_rest: 545 cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate", 546 "quiet=${" + str(quiet) + "}"] 547 gp.dprint_issuing(cmd_buf) 548 status, ret_values = \ 549 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 550 if status == "PASS": 551 state['rest'] = '1' 552 else: 553 state['rest'] = '0' 554 555 if int(state['rest']): 556 for url_path in ret_values: 557 for attr_name in ret_values[url_path]: 558 # Create a state key value based on the attr_name. 559 try: 560 ret_values[url_path][attr_name] = \ 561 re.sub(r'.*\.', "", 562 ret_values[url_path][attr_name]) 563 except TypeError: 564 pass 565 # Do some key name manipulations. 566 new_attr_name = re.sub(r'^Current|(State|Transition)$', 567 "", attr_name) 568 new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name) 569 new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1', 570 new_attr_name) 571 new_attr_name = new_attr_name.lower().lstrip("_") 572 new_attr_name = re.sub(r'power', r'chassis', new_attr_name) 573 if new_attr_name in req_states: 574 state[new_attr_name] = ret_values[url_path][attr_name] 575 576 for sub_state in req_states: 577 if sub_state in state: 578 continue 579 if sub_state.startswith("os_"): 580 # We pass "os_" requests on to get_os_state. 581 continue 582 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" 583 exec(cmd_buf) 584 585 if os_host == "": 586 # The caller has not specified an os_host so as far as we're concerned, 587 # it doesn't exist. 588 return state 589 590 os_req_states = [sub_state for sub_state in req_states 591 if sub_state.startswith('os_')] 592 593 if len(os_req_states) > 0: 594 # The caller has specified an os_host and they have requested 595 # information on os substates. 596 597 # Based on the information gathered on bmc, we'll try to make a 598 # determination of whether the os is even up. We'll pass the result 599 # of that assessment to get_os_state to enhance performance. 600 os_up_match = DotDict() 601 for sub_state in master_os_up_match: 602 if sub_state in req_states: 603 os_up_match[sub_state] = master_os_up_match[sub_state] 604 os_up = compare_states(state, os_up_match) 605 os_state = get_os_state(os_host=os_host, 606 os_username=os_username, 607 os_password=os_password, 608 req_states=os_req_states, 609 os_up=os_up, 610 quiet=quiet) 611 # Append os_state dictionary to ours. 612 state.update(os_state) 613 614 return state 615 616 617exit_wait_early_message = "" 618 619 620def set_exit_wait_early_message(value): 621 r""" 622 Set global exit_wait_early_message to the indicated value. 623 624 This is a mechanism by which the programmer can do an early exit from 625 wait_until_keyword_succeeds() based on some special condition. 626 627 Description of argument(s): 628 value The value to assign to the global 629 exit_wait_early_message. 630 """ 631 632 global exit_wait_early_message 633 exit_wait_early_message = value 634 635 636def check_state(match_state, 637 invert=0, 638 print_string="", 639 openbmc_host="", 640 openbmc_username="", 641 openbmc_password="", 642 os_host="", 643 os_username="", 644 os_password="", 645 quiet=None): 646 r""" 647 Check that the Open BMC machine's composite state matches the specified 648 state. On success, this keyword returns the machine's composite state as a 649 dictionary. 650 651 Description of arguments: 652 match_state A dictionary whose key/value pairs are "state field"/ 653 "state value". The state value is interpreted as a 654 regular expression. Example call from robot: 655 ${match_state}= Create Dictionary chassis=^On$ 656 ... bmc=^Ready$ 657 ... boot_progress=^OSStart$ 658 ${state}= Check State &{match_state} 659 invert If this flag is set, this function will succeed if the 660 states do NOT match. 661 print_string This function will print this string to the console prior 662 to getting the state. 663 openbmc_host The DNS name or IP address of the BMC. 664 This defaults to global ${OPENBMC_HOST}. 665 openbmc_username The username to be used to login to the BMC. 666 This defaults to global ${OPENBMC_USERNAME}. 667 openbmc_password The password to be used to login to the BMC. 668 This defaults to global ${OPENBMC_PASSWORD}. 669 os_host The DNS name or IP address of the operating system. 670 This defaults to global ${OS_HOST}. 671 os_username The username to be used to login to the OS. 672 This defaults to global ${OS_USERNAME}. 673 os_password The password to be used to login to the OS. 674 This defaults to global ${OS_PASSWORD}. 675 quiet Indicates whether status details should be written to the 676 console. Defaults to either global value of ${QUIET} or 677 to 1. 678 """ 679 680 quiet = int(gp.get_var_value(quiet, 0)) 681 682 gp.gp_print(print_string) 683 684 try: 685 match_state = return_state_constant(match_state) 686 except TypeError: 687 pass 688 689 req_states = match_state.keys() 690 # Initialize state. 691 state = get_state(openbmc_host=openbmc_host, 692 openbmc_username=openbmc_username, 693 openbmc_password=openbmc_password, 694 os_host=os_host, 695 os_username=os_username, 696 os_password=os_password, 697 req_states=req_states, 698 quiet=quiet) 699 if not quiet: 700 gp.print_var(state) 701 702 if exit_wait_early_message != "": 703 # The exit_wait_early_message has been set by a signal handler so we 704 # will exit "successfully". It is incumbent upon the calling function 705 # (e.g. wait_state) to check/clear this variable and to fail 706 # appropriately. 707 return state 708 709 match = compare_states(state, match_state) 710 711 if invert and match: 712 fail_msg = "The current state of the machine matches the match" +\ 713 " state:\n" + gp.sprint_varx("state", state) 714 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 715 elif not invert and not match: 716 fail_msg = "The current state of the machine does NOT match the" +\ 717 " match state:\n" +\ 718 gp.sprint_varx("state", state) 719 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 720 721 return state 722 723 724def wait_state(match_state=(), 725 wait_time="1 min", 726 interval="1 second", 727 invert=0, 728 openbmc_host="", 729 openbmc_username="", 730 openbmc_password="", 731 os_host="", 732 os_username="", 733 os_password="", 734 quiet=None): 735 r""" 736 Wait for the Open BMC machine's composite state to match the specified 737 state. On success, this keyword returns the machine's composite state as 738 a dictionary. 739 740 Description of arguments: 741 match_state A dictionary whose key/value pairs are "state field"/ 742 "state value". See check_state (above) for details. 743 This value may also be any string accepted by 744 return_state_constant (e.g. "standby_match_state"). 745 In such a case this function will call 746 return_state_constant to convert it to a proper 747 dictionary as described above. 748 wait_time The total amount of time to wait for the desired state. 749 This value may be expressed in Robot Framework's time 750 format (e.g. 1 minute, 2 min 3 s, 4.5). 751 interval The amount of time between state checks. 752 This value may be expressed in Robot Framework's time 753 format (e.g. 1 minute, 2 min 3 s, 4.5). 754 invert If this flag is set, this function will for the state of 755 the machine to cease to match the match state. 756 openbmc_host The DNS name or IP address of the BMC. 757 This defaults to global ${OPENBMC_HOST}. 758 openbmc_username The username to be used to login to the BMC. 759 This defaults to global ${OPENBMC_USERNAME}. 760 openbmc_password The password to be used to login to the BMC. 761 This defaults to global ${OPENBMC_PASSWORD}. 762 os_host The DNS name or IP address of the operating system. 763 This defaults to global ${OS_HOST}. 764 os_username The username to be used to login to the OS. 765 This defaults to global ${OS_USERNAME}. 766 os_password The password to be used to login to the OS. 767 This defaults to global ${OS_PASSWORD}. 768 quiet Indicates whether status details should be written to the 769 console. Defaults to either global value of ${QUIET} or 770 to 1. 771 """ 772 773 quiet = int(gp.get_var_value(quiet, 0)) 774 775 try: 776 match_state = return_state_constant(match_state) 777 except TypeError: 778 pass 779 780 if not quiet: 781 if invert: 782 alt_text = "cease to " 783 else: 784 alt_text = "" 785 gp.print_timen("Checking every " + str(interval) + " for up to " 786 + str(wait_time) + " for the state of the machine to " 787 + alt_text + "match the state shown below.") 788 gp.print_var(match_state) 789 790 if quiet: 791 print_string = "" 792 else: 793 print_string = "#" 794 795 debug = int(BuiltIn().get_variable_value("${debug}", "0")) 796 if debug: 797 # In debug we print state so no need to print the "#". 798 print_string = "" 799 check_state_quiet = 1 - debug 800 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}", 801 "print_string=" + print_string, "openbmc_host=" + openbmc_host, 802 "openbmc_username=" + openbmc_username, 803 "openbmc_password=" + openbmc_password, "os_host=" + os_host, 804 "os_username=" + os_username, "os_password=" + os_password, 805 "quiet=${" + str(check_state_quiet) + "}"] 806 gp.dprint_issuing(cmd_buf) 807 try: 808 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval, 809 *cmd_buf) 810 except AssertionError as my_assertion_error: 811 gp.printn() 812 message = my_assertion_error.args[0] 813 BuiltIn().fail(message) 814 815 if exit_wait_early_message: 816 # The global exit_wait_early_message was set by a signal handler 817 # indicating that we should fail. 818 message = exit_wait_early_message 819 # Clear the exit_wait_early_message variable for future use. 820 set_exit_wait_early_message("") 821 BuiltIn().fail(gp.sprint_error(message)) 822 823 if not quiet: 824 gp.printn() 825 if invert: 826 gp.print_timen("The states no longer match:") 827 else: 828 gp.print_timen("The states match:") 829 gp.print_var(state) 830 831 return state 832 833 834def wait_for_comm_cycle(start_boot_seconds, 835 quiet=None): 836 r""" 837 Wait for communications to the BMC to stop working and then resume working. 838 This function is useful when you have initiated some kind of reboot. 839 840 Description of arguments: 841 start_boot_seconds The time that the boot test started. The format is the 842 epoch time in seconds, i.e. the number of seconds since 843 1970-01-01 00:00:00 UTC. This value should be obtained 844 from the BMC so that it is not dependent on any kind of 845 synchronization between this machine and the target BMC 846 This will allow this program to work correctly even in 847 a simulated environment. This value should be obtained 848 by the caller prior to initiating a reboot. It can be 849 obtained as follows: 850 state = st.get_state(req_states=['epoch_seconds']) 851 """ 852 853 quiet = int(gp.get_var_value(quiet, 0)) 854 855 # Validate parms. 856 error_message = gv.valid_integer(start_boot_seconds, 857 var_name="start_boot_seconds") 858 if error_message != "": 859 BuiltIn().fail(gp.sprint_error(error_message)) 860 861 match_state = anchor_state(DotDict([('packet_loss', '100')])) 862 # Wait for 100% packet loss trying to ping machine. 863 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 864 865 match_state['packet_loss'] = '^0$' 866 # Wait for 0% packet loss trying to ping machine. 867 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 868 869 # Get the uptime and epoch seconds for comparisons. We want to be sure 870 # that the uptime is less than the elapsed boot time. Further proof that 871 # a reboot has indeed occurred (vs random network instability giving a 872 # false positive. We also use wait_state because the BMC may take a short 873 # while to be ready to process SSH requests. 874 match_state = DotDict([('uptime', '^[0-9\\.]+$'), 875 ('epoch_seconds', '^[0-9]+$')]) 876 state = wait_state(match_state, wait_time="2 mins", interval="1 second") 877 878 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds 879 gp.qprint_var(elapsed_boot_time) 880 if state['uptime'] == "": 881 error_message = "Unable to obtain uptime from the BMC. BMC is not" +\ 882 " communicating." 883 BuiltIn().fail(gp.sprint_error(error_message)) 884 if int(float(state['uptime'])) < elapsed_boot_time: 885 uptime = state['uptime'] 886 gp.qprint_var(uptime) 887 gp.qprint_timen("The uptime is less than the elapsed boot time," 888 + " as expected.") 889 else: 890 error_message = "The uptime is greater than the elapsed boot time," +\ 891 " which is unexpected:\n" +\ 892 gp.sprint_var(start_boot_seconds) +\ 893 gp.sprint_var(state) 894 BuiltIn().fail(gp.sprint_error(error_message)) 895 896 gp.qprint_timen("Verifying that REST API interface is working.") 897 match_state = DotDict([('rest', '^1$')]) 898 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds") 899