1#!/usr/bin/env python 2 3r""" 4This module contains functions having to do with machine state: get_state, 5check_state, wait_state, etc. 6 7The 'State' is a composite of many pieces of data. Therefore, the functions 8in this module define state as an ordered dictionary. Here is an example of 9some test output showing machine state: 10 11default_state: 12 default_state[chassis]: On 13 default_state[boot_progress]: OSStart 14 default_state[operating_system]: BootComplete 15 default_state[host]: Running 16 default_state[os_ping]: 1 17 default_state[os_login]: 1 18 default_state[os_run_cmd]: 1 19 20Different users may very well have different needs when inquiring about 21state. Support for new pieces of state information may be added to this 22module as needed. 23 24By using the wait_state function, a caller can start a boot and then wait for 25a precisely defined state to indicate that the boot has succeeded. If 26the boot fails, they can see exactly why by looking at the current state as 27compared with the expected state. 28""" 29 30import gen_print as gp 31import gen_robot_print as grp 32import gen_valid as gv 33import gen_robot_utils as gru 34import gen_cmd as gc 35import bmc_ssh_utils as bsu 36 37from robot.libraries.BuiltIn import BuiltIn 38from robot.utils import DotDict 39 40import re 41import os 42import sys 43import imp 44 45 46# NOTE: Avoid importing utils.robot because utils.robot imports state.py 47# (indirectly) which will cause failures. 48gru.my_import_resource("rest_client.robot") 49 50base_path = os.path.dirname(os.path.dirname( 51 imp.find_module("gen_robot_print")[1])) + os.sep 52sys.path.append(base_path + "data/") 53 54# Previously, I had this coded: 55# import variables as var 56# However, we ran into a problem where a robot program did this... 57# Variables ../../lib/ras/variables.py 58# Prior to doing this... 59# Library ../lib/state.py 60 61# This caused the wrong variables.py file to be selected. Attempts to fix this 62# have failed so far. For the moment, we will hard-code the value we need from 63# the file. 64 65SYSTEM_STATE_URI = "/xyz/openbmc_project/state/" 66 67# The BMC code has recently been changed as far as what states are defined and 68# what the state values can be. This module now has a means of processing both 69# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e. 70# OBMC_STATES_VERSION = 1). 71# The caller can set environment variable OBMC_STATES_VERSION to dictate 72# whether we're processing old or new style states. If OBMC_STATES_VERSION is 73# not set it will default to 1. 74 75# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old 76# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code 77# is being removed but the OBMC_STATES_VERSION value will stay for now in the 78# event that it is needed in the future. 79 80OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1)) 81 82# When a user calls get_state w/o specifying req_states, default_req_states 83# is used as its value. 84default_req_states = ['rest', 85 'chassis', 86 'bmc', 87 'boot_progress', 88 'operating_system', 89 'host', 90 'os_ping', 91 'os_login', 92 'os_run_cmd'] 93 94# valid_req_states is a list of sub states supported by the get_state function. 95# valid_req_states, default_req_states and master_os_up_match are used by the 96# get_state function. 97valid_req_states = ['ping', 98 'packet_loss', 99 'uptime', 100 'epoch_seconds', 101 'rest', 102 'chassis', 103 'requested_chassis', 104 'bmc', 105 'requested_bmc', 106 'boot_progress', 107 'operating_system', 108 'host', 109 'requested_host', 110 'attempts_left', 111 'os_ping', 112 'os_login', 113 'os_run_cmd'] 114 115# valid_os_req_states and default_os_req_states are used by the os_get_state 116# function. 117# valid_os_req_states is a list of state information supported by the 118# get_os_state function. 119valid_os_req_states = ['os_ping', 120 'os_login', 121 'os_run_cmd'] 122# When a user calls get_os_state w/o specifying req_states, 123# default_os_req_states is used as its value. 124default_os_req_states = ['os_ping', 125 'os_login', 126 'os_run_cmd'] 127 128# Presently, some BMCs appear to not keep time very well. This environment 129# variable directs the get_state function to use either the BMC's epoch time 130# or the local epoch time. 131USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0)) 132 133# Useful state constant definition(s). 134# default_state is an initial value which may be of use to callers. 135default_state = DotDict([('rest', '1'), 136 ('chassis', 'On'), 137 ('bmc', 'Ready'), 138 ('boot_progress', 'OSStart'), 139 ('operating_system', 'BootComplete'), 140 ('host', 'Running'), 141 ('os_ping', '1'), 142 ('os_login', '1'), 143 ('os_run_cmd', '1')]) 144 145# A match state for checking that the system is at "standby". 146standby_match_state = DotDict([('rest', '^1$'), 147 ('chassis', '^Off$'), 148 ('bmc', '^Ready$'), 149 ('boot_progress', '^$'), 150 ('operating_system', '^$'), 151 ('host', '^$')]) 152 153# A match state for checking that the system is at "os running". 154os_running_match_state = DotDict([('chassis', '^On$'), 155 ('bmc', '^Ready$'), 156 ('boot_progress', 157 'FW Progress, Starting OS|OSStart'), 158 ('operating_system', 'BootComplete'), 159 ('host', '^Running$'), 160 ('os_ping', '^1$'), 161 ('os_login', '^1$'), 162 ('os_run_cmd', '^1$')]) 163 164# A master dictionary to determine whether the os may be up. 165master_os_up_match = DotDict([('chassis', '^On$'), 166 ('bmc', '^Ready$'), 167 ('boot_progress', 168 'FW Progress, Starting OS|OSStart'), 169 ('operating_system', 'BootComplete'), 170 ('host', '^Running|Quiesced$')]) 171 172invalid_state_match = DotDict([('rest', '^$'), 173 ('chassis', '^$'), 174 ('bmc', '^$'), 175 ('boot_progress', '^$'), 176 ('operating_system', '^$'), 177 ('host', '^$')]) 178 179 180def return_state_constant(state_name='default_state'): 181 r""" 182 Return the named state dictionary constant. 183 """ 184 185 return eval(state_name) 186 187 188def anchor_state(state): 189 r""" 190 Add regular expression anchors ("^" and "$") to the beginning and end of 191 each item in the state dictionary passed in. Return the resulting 192 dictionary. 193 194 Description of Arguments: 195 state A dictionary such as the one returned by the get_state() 196 function. 197 """ 198 199 anchored_state = state.copy() 200 for key, match_state_value in anchored_state.items(): 201 anchored_state[key] = "^" + str(anchored_state[key]) + "$" 202 203 return anchored_state 204 205 206def strip_anchor_state(state): 207 r""" 208 Strip regular expression anchors ("^" and "$") from the beginning and end 209 of each item in the state dictionary passed in. Return the resulting 210 dictionary. 211 212 Description of Arguments: 213 state A dictionary such as the one returned by the get_state() 214 function. 215 """ 216 217 stripped_state = state.copy() 218 for key, match_state_value in stripped_state.items(): 219 stripped_state[key] = stripped_state[key].strip("^$") 220 221 return stripped_state 222 223 224def compare_states(state, 225 match_state, 226 match_type='and'): 227 r""" 228 Compare 2 state dictionaries. Return True if they match and False if they 229 don't. Note that the match_state dictionary does not need to have an entry 230 corresponding to each entry in the state dictionary. But for each entry 231 that it does have, the corresponding state entry will be checked for a 232 match. 233 234 Description of arguments: 235 state A state dictionary such as the one returned by the 236 get_state function. 237 match_state A dictionary whose key/value pairs are "state field"/ 238 "state value". The state value is interpreted as a 239 regular expression. Every value in this dictionary is 240 considered. When match_type is 'and', if each and every 241 comparison matches, the two dictionaries are considered to 242 be matching. If match_type is 'or', if any two of the 243 elements compared match, the two dictionaries are 244 considered to be matching. 245 This value may also be any string accepted by 246 return_state_constant (e.g. "standby_match_state"). 247 In such a case this function will call 248 return_state_constant to convert it to a proper 249 dictionary as described above. 250 match_type This may be 'and' or 'or'. 251 """ 252 253 error_message = gv.svalid_value(match_type, var_name="match_type", 254 valid_values=['and', 'or']) 255 if error_message != "": 256 BuiltIn().fail(gp.sprint_error(error_message)) 257 258 try: 259 match_state = return_state_constant(match_state) 260 except TypeError: 261 pass 262 263 default_match = (match_type == 'and') 264 for key, match_state_value in match_state.items(): 265 # Blank match_state_value means "don't care". 266 if match_state_value == "": 267 continue 268 try: 269 match = (re.match(match_state_value, str(state[key])) is not None) 270 except KeyError: 271 match = False 272 273 if match != default_match: 274 return match 275 276 return default_match 277 278 279def get_os_state(os_host="", 280 os_username="", 281 os_password="", 282 req_states=default_os_req_states, 283 os_up=True, 284 quiet=None): 285 r""" 286 Get component states for the operating system such as ping, login, 287 etc, put them into a dictionary and return them to the caller. 288 289 Note that all substate values are strings. 290 291 Description of arguments: 292 os_host The DNS name or IP address of the operating system. 293 This defaults to global ${OS_HOST}. 294 os_username The username to be used to login to the OS. 295 This defaults to global ${OS_USERNAME}. 296 os_password The password to be used to login to the OS. 297 This defaults to global ${OS_PASSWORD}. 298 req_states This is a list of states whose values are being requested by 299 the caller. 300 os_up If the caller knows that the os can't possibly be up, it can 301 improve performance by passing os_up=False. This function 302 will then simply return default values for all requested os 303 sub states. 304 quiet Indicates whether status details (e.g. curl commands) should 305 be written to the console. 306 Defaults to either global value of ${QUIET} or to 1. 307 """ 308 309 quiet = int(gp.get_var_value(quiet, 0)) 310 311 # Set parm defaults where necessary and validate all parms. 312 if os_host == "": 313 os_host = BuiltIn().get_variable_value("${OS_HOST}") 314 error_message = gv.svalid_value(os_host, var_name="os_host", 315 invalid_values=[None, ""]) 316 if error_message != "": 317 BuiltIn().fail(gp.sprint_error(error_message)) 318 319 if os_username == "": 320 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 321 error_message = gv.svalid_value(os_username, var_name="os_username", 322 invalid_values=[None, ""]) 323 if error_message != "": 324 BuiltIn().fail(gp.sprint_error(error_message)) 325 326 if os_password == "": 327 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 328 error_message = gv.svalid_value(os_password, var_name="os_password", 329 invalid_values=[None, ""]) 330 if error_message != "": 331 BuiltIn().fail(gp.sprint_error(error_message)) 332 333 invalid_req_states = [sub_state for sub_state in req_states 334 if sub_state not in valid_os_req_states] 335 if len(invalid_req_states) > 0: 336 error_message = "The following req_states are not supported:\n" +\ 337 gp.sprint_var(invalid_req_states) 338 BuiltIn().fail(gp.sprint_error(error_message)) 339 340 # Initialize all substate values supported by this function. 341 os_ping = 0 342 os_login = 0 343 os_run_cmd = 0 344 345 if os_up: 346 if 'os_ping' in req_states: 347 # See if the OS pings. 348 rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + os_host, 349 print_output=0, show_err=0, 350 ignore_err=1) 351 if rc == 0: 352 os_ping = 1 353 354 # Programming note: All attributes which do not require an ssh login 355 # should have been processed by this point. 356 master_req_login = ['os_login', 'os_run_cmd'] 357 req_login = [sub_state for sub_state in req_states if sub_state in 358 master_req_login] 359 must_login = (len(req_login) > 0) 360 361 if must_login: 362 output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet, 363 ignore_err=1, 364 time_out=20) 365 if rc == 0: 366 os_login = 1 367 os_run_cmd = 1 368 else: 369 gp.dprint_vars(output, stderr) 370 gp.dprint_vars(rc, 1) 371 372 os_state = DotDict() 373 for sub_state in req_states: 374 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" 375 exec(cmd_buf) 376 377 return os_state 378 379 380def get_state(openbmc_host="", 381 openbmc_username="", 382 openbmc_password="", 383 os_host="", 384 os_username="", 385 os_password="", 386 req_states=default_req_states, 387 quiet=None): 388 r""" 389 Get component states such as chassis state, bmc state, etc, put them into a 390 dictionary and return them to the caller. 391 392 Note that all substate values are strings. 393 394 Description of arguments: 395 openbmc_host The DNS name or IP address of the BMC. 396 This defaults to global ${OPENBMC_HOST}. 397 openbmc_username The username to be used to login to the BMC. 398 This defaults to global ${OPENBMC_USERNAME}. 399 openbmc_password The password to be used to login to the BMC. 400 This defaults to global ${OPENBMC_PASSWORD}. 401 os_host The DNS name or IP address of the operating system. 402 This defaults to global ${OS_HOST}. 403 os_username The username to be used to login to the OS. 404 This defaults to global ${OS_USERNAME}. 405 os_password The password to be used to login to the OS. 406 This defaults to global ${OS_PASSWORD}. 407 req_states This is a list of states whose values are being requested 408 by the caller. 409 quiet Indicates whether status details (e.g. curl commands) 410 should be written to the console. 411 Defaults to either global value of ${QUIET} or to 1. 412 """ 413 414 quiet = int(gp.get_var_value(quiet, 0)) 415 416 # Set parm defaults where necessary and validate all parms. 417 if openbmc_host == "": 418 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") 419 error_message = gv.svalid_value(openbmc_host, 420 var_name="openbmc_host", 421 invalid_values=[None, ""]) 422 if error_message != "": 423 BuiltIn().fail(gp.sprint_error(error_message)) 424 425 if openbmc_username == "": 426 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") 427 error_message = gv.svalid_value(openbmc_username, 428 var_name="openbmc_username", 429 invalid_values=[None, ""]) 430 if error_message != "": 431 BuiltIn().fail(gp.sprint_error(error_message)) 432 433 if openbmc_password == "": 434 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") 435 error_message = gv.svalid_value(openbmc_password, 436 var_name="openbmc_password", 437 invalid_values=[None, ""]) 438 if error_message != "": 439 BuiltIn().fail(gp.sprint_error(error_message)) 440 441 # NOTE: OS parms are optional. 442 if os_host == "": 443 os_host = BuiltIn().get_variable_value("${OS_HOST}") 444 if os_host is None: 445 os_host = "" 446 447 if os_username is "": 448 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 449 if os_username is None: 450 os_username = "" 451 452 if os_password is "": 453 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 454 if os_password is None: 455 os_password = "" 456 457 invalid_req_states = [sub_state for sub_state in req_states 458 if sub_state not in valid_req_states] 459 if len(invalid_req_states) > 0: 460 error_message = "The following req_states are not supported:\n" +\ 461 gp.sprint_var(invalid_req_states) 462 BuiltIn().fail(gp.sprint_error(error_message)) 463 464 # Initialize all substate values supported by this function. 465 ping = 0 466 packet_loss = '' 467 uptime = '' 468 epoch_seconds = '' 469 rest = '' 470 chassis = '' 471 requested_chassis = '' 472 bmc = '' 473 requested_bmc = '' 474 boot_progress = '' 475 operating_system = '' 476 host = '' 477 requested_host = '' 478 attempts_left = '' 479 480 # Get the component states. 481 if 'ping' in req_states: 482 # See if the OS pings. 483 rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host, 484 print_output=0, show_err=0, 485 ignore_err=1) 486 if rc == 0: 487 ping = 1 488 489 if 'packet_loss' in req_states: 490 # See if the OS pings. 491 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\ 492 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" 493 rc, out_buf = gc.shell_cmd(cmd_buf, 494 print_output=0, show_err=0, 495 ignore_err=1) 496 if rc == 0: 497 packet_loss = out_buf.rstrip("\n") 498 499 if 'uptime' in req_states: 500 # Sometimes reading uptime results in a blank value. Call with 501 # wait_until_keyword_succeeds to ensure a non-blank value is obtained. 502 remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\ 503 " && [ ! -z \"${uptime}\" ] && echo ${uptime}" 504 cmd_buf = ["BMC Execute Command", 505 re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1'] 506 if not quiet: 507 # Get loc_test_mode parm for improved output on pissuing. 508 # See sprint_issuing in gen_print.py for details. 509 loc_test_mode = int(gp.get_var_value(var_name="test_mode", 510 default=0)) 511 grp.rpissuing_keyword(cmd_buf, loc_test_mode) 512 gp.pissuing(remote_cmd_buf, loc_test_mode) 513 try: 514 stdout, stderr, rc =\ 515 BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec", 516 *cmd_buf) 517 if rc == 0 and stderr == "": 518 uptime = stdout 519 except AssertionError as my_assertion_error: 520 pass 521 522 if 'epoch_seconds' in req_states: 523 date_cmd_buf = "date -u +%s" 524 if USE_BMC_EPOCH_TIME: 525 cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}'] 526 if not quiet: 527 grp.rpissuing_keyword(cmd_buf) 528 status, ret_values = \ 529 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 530 if status == "PASS": 531 stdout, stderr, rc = ret_values 532 if rc == 0 and stderr == "": 533 epoch_seconds = stdout.rstrip("\n") 534 else: 535 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf, 536 quiet=quiet, 537 print_output=0) 538 if shell_rc == 0: 539 epoch_seconds = out_buf.rstrip("\n") 540 541 master_req_rest = ['rest', 'host', 'requested_host', 'operating_system', 542 'attempts_left', 'boot_progress', 'chassis', 543 'requested_chassis' 'bmc' 'requested_bmc'] 544 545 req_rest = [sub_state for sub_state in req_states if sub_state in 546 master_req_rest] 547 need_rest = (len(req_rest) > 0) 548 state = DotDict() 549 if need_rest: 550 cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate", 551 "quiet=${" + str(quiet) + "}"] 552 grp.rdpissuing_keyword(cmd_buf) 553 status, ret_values = \ 554 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 555 if status == "PASS": 556 state['rest'] = '1' 557 else: 558 state['rest'] = '0' 559 560 if int(state['rest']): 561 for url_path in ret_values: 562 for attr_name in ret_values[url_path]: 563 # Create a state key value based on the attr_name. 564 try: 565 ret_values[url_path][attr_name] = \ 566 re.sub(r'.*\.', "", 567 ret_values[url_path][attr_name]) 568 except TypeError: 569 pass 570 # Do some key name manipulations. 571 new_attr_name = re.sub(r'^Current|(State|Transition)$', 572 "", attr_name) 573 new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name) 574 new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1', 575 new_attr_name) 576 new_attr_name = new_attr_name.lower().lstrip("_") 577 new_attr_name = re.sub(r'power', r'chassis', new_attr_name) 578 if new_attr_name in req_states: 579 state[new_attr_name] = ret_values[url_path][attr_name] 580 581 for sub_state in req_states: 582 if sub_state in state: 583 continue 584 if sub_state.startswith("os_"): 585 # We pass "os_" requests on to get_os_state. 586 continue 587 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" 588 exec(cmd_buf) 589 590 if os_host == "": 591 # The caller has not specified an os_host so as far as we're concerned, 592 # it doesn't exist. 593 return state 594 595 os_req_states = [sub_state for sub_state in req_states 596 if sub_state.startswith('os_')] 597 598 if len(os_req_states) > 0: 599 # The caller has specified an os_host and they have requested 600 # information on os substates. 601 602 # Based on the information gathered on bmc, we'll try to make a 603 # determination of whether the os is even up. We'll pass the result 604 # of that assessment to get_os_state to enhance performance. 605 os_up_match = DotDict() 606 for sub_state in master_os_up_match: 607 if sub_state in req_states: 608 os_up_match[sub_state] = master_os_up_match[sub_state] 609 os_up = compare_states(state, os_up_match) 610 os_state = get_os_state(os_host=os_host, 611 os_username=os_username, 612 os_password=os_password, 613 req_states=os_req_states, 614 os_up=os_up, 615 quiet=quiet) 616 # Append os_state dictionary to ours. 617 state.update(os_state) 618 619 return state 620 621 622def check_state(match_state, 623 invert=0, 624 print_string="", 625 openbmc_host="", 626 openbmc_username="", 627 openbmc_password="", 628 os_host="", 629 os_username="", 630 os_password="", 631 quiet=None): 632 r""" 633 Check that the Open BMC machine's composite state matches the specified 634 state. On success, this keyword returns the machine's composite state as a 635 dictionary. 636 637 Description of arguments: 638 match_state A dictionary whose key/value pairs are "state field"/ 639 "state value". The state value is interpreted as a 640 regular expression. Example call from robot: 641 ${match_state}= Create Dictionary chassis=^On$ 642 ... bmc=^Ready$ 643 ... boot_progress=^OSStart$ 644 ${state}= Check State &{match_state} 645 invert If this flag is set, this function will succeed if the 646 states do NOT match. 647 print_string This function will print this string to the console prior 648 to getting the state. 649 openbmc_host The DNS name or IP address of the BMC. 650 This defaults to global ${OPENBMC_HOST}. 651 openbmc_username The username to be used to login to the BMC. 652 This defaults to global ${OPENBMC_USERNAME}. 653 openbmc_password The password to be used to login to the BMC. 654 This defaults to global ${OPENBMC_PASSWORD}. 655 os_host The DNS name or IP address of the operating system. 656 This defaults to global ${OS_HOST}. 657 os_username The username to be used to login to the OS. 658 This defaults to global ${OS_USERNAME}. 659 os_password The password to be used to login to the OS. 660 This defaults to global ${OS_PASSWORD}. 661 quiet Indicates whether status details should be written to the 662 console. Defaults to either global value of ${QUIET} or 663 to 1. 664 """ 665 666 quiet = int(gp.get_var_value(quiet, 0)) 667 668 grp.rprint(print_string) 669 670 req_states = match_state.keys() 671 # Initialize state. 672 state = get_state(openbmc_host=openbmc_host, 673 openbmc_username=openbmc_username, 674 openbmc_password=openbmc_password, 675 os_host=os_host, 676 os_username=os_username, 677 os_password=os_password, 678 req_states=req_states, 679 quiet=quiet) 680 if not quiet: 681 gp.print_var(state) 682 683 match = compare_states(state, match_state) 684 685 if invert and match: 686 fail_msg = "The current state of the machine matches the match" +\ 687 " state:\n" + gp.sprint_varx("state", state) 688 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 689 elif not invert and not match: 690 fail_msg = "The current state of the machine does NOT match the" +\ 691 " match state:\n" +\ 692 gp.sprint_varx("state", state) 693 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 694 695 return state 696 697 698def wait_state(match_state=(), 699 wait_time="1 min", 700 interval="1 second", 701 invert=0, 702 openbmc_host="", 703 openbmc_username="", 704 openbmc_password="", 705 os_host="", 706 os_username="", 707 os_password="", 708 quiet=None): 709 r""" 710 Wait for the Open BMC machine's composite state to match the specified 711 state. On success, this keyword returns the machine's composite state as 712 a dictionary. 713 714 Description of arguments: 715 match_state A dictionary whose key/value pairs are "state field"/ 716 "state value". See check_state (above) for details. 717 This value may also be any string accepted by 718 return_state_constant (e.g. "standby_match_state"). 719 In such a case this function will call 720 return_state_constant to convert it to a proper 721 dictionary as described above. 722 wait_time The total amount of time to wait for the desired state. 723 This value may be expressed in Robot Framework's time 724 format (e.g. 1 minute, 2 min 3 s, 4.5). 725 interval The amount of time between state checks. 726 This value may be expressed in Robot Framework's time 727 format (e.g. 1 minute, 2 min 3 s, 4.5). 728 invert If this flag is set, this function will for the state of 729 the machine to cease to match the match state. 730 openbmc_host The DNS name or IP address of the BMC. 731 This defaults to global ${OPENBMC_HOST}. 732 openbmc_username The username to be used to login to the BMC. 733 This defaults to global ${OPENBMC_USERNAME}. 734 openbmc_password The password to be used to login to the BMC. 735 This defaults to global ${OPENBMC_PASSWORD}. 736 os_host The DNS name or IP address of the operating system. 737 This defaults to global ${OS_HOST}. 738 os_username The username to be used to login to the OS. 739 This defaults to global ${OS_USERNAME}. 740 os_password The password to be used to login to the OS. 741 This defaults to global ${OS_PASSWORD}. 742 quiet Indicates whether status details should be written to the 743 console. Defaults to either global value of ${QUIET} or 744 to 1. 745 """ 746 747 quiet = int(gp.get_var_value(quiet, 0)) 748 749 try: 750 match_state = return_state_constant(match_state) 751 except TypeError: 752 pass 753 754 if not quiet: 755 if invert: 756 alt_text = "cease to " 757 else: 758 alt_text = "" 759 gp.print_timen("Checking every " + str(interval) + " for up to " 760 + str(wait_time) + " for the state of the machine to " 761 + alt_text + "match the state shown below.") 762 gp.print_var(match_state) 763 764 if quiet: 765 print_string = "" 766 else: 767 print_string = "#" 768 769 debug = int(BuiltIn().get_variable_value("${debug}", "0")) 770 if debug: 771 # In debug we print state so no need to print the "#". 772 print_string = "" 773 check_state_quiet = 1 - debug 774 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}", 775 "print_string=" + print_string, "openbmc_host=" + openbmc_host, 776 "openbmc_username=" + openbmc_username, 777 "openbmc_password=" + openbmc_password, "os_host=" + os_host, 778 "os_username=" + os_username, "os_password=" + os_password, 779 "quiet=${" + str(check_state_quiet) + "}"] 780 grp.rdpissuing_keyword(cmd_buf) 781 try: 782 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval, 783 *cmd_buf) 784 except AssertionError as my_assertion_error: 785 gp.printn() 786 message = my_assertion_error.args[0] 787 BuiltIn().fail(message) 788 789 if not quiet: 790 gp.printn() 791 if invert: 792 gp.print_timen("The states no longer match:") 793 else: 794 gp.print_timen("The states match:") 795 gp.print_var(state) 796 797 return state 798 799 800def wait_for_comm_cycle(start_boot_seconds, 801 quiet=None): 802 r""" 803 Wait for communications to the BMC to stop working and then resume working. 804 This function is useful when you have initiated some kind of reboot. 805 806 Description of arguments: 807 start_boot_seconds The time that the boot test started. The format is the 808 epoch time in seconds, i.e. the number of seconds since 809 1970-01-01 00:00:00 UTC. This value should be obtained 810 from the BMC so that it is not dependent on any kind of 811 synchronization between this machine and the target BMC 812 This will allow this program to work correctly even in 813 a simulated environment. This value should be obtained 814 by the caller prior to initiating a reboot. It can be 815 obtained as follows: 816 state = st.get_state(req_states=['epoch_seconds']) 817 """ 818 819 quiet = int(gp.get_var_value(quiet, 0)) 820 821 # Validate parms. 822 error_message = gv.svalid_integer(start_boot_seconds, 823 var_name="start_boot_seconds") 824 if error_message != "": 825 BuiltIn().fail(gp.sprint_error(error_message)) 826 827 match_state = anchor_state(DotDict([('packet_loss', '100')])) 828 # Wait for 100% packet loss trying to ping machine. 829 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 830 831 match_state['packet_loss'] = '^0$' 832 # Wait for 0% packet loss trying to ping machine. 833 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 834 835 # Get the uptime and epoch seconds for comparisons. We want to be sure 836 # that the uptime is less than the elapsed boot time. Further proof that 837 # a reboot has indeed occurred (vs random network instability giving a 838 # false positive. We also use wait_state because the BMC may take a short 839 # while to be ready to process SSH requests. 840 match_state = DotDict([('uptime', '^[0-9\\.]+$'), 841 ('epoch_seconds', '^[0-9]+$')]) 842 state = wait_state(match_state, wait_time="2 mins", interval="1 second") 843 844 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds 845 gp.qprint_var(elapsed_boot_time) 846 if state['uptime'] == "": 847 error_message = "Unable to obtain uptime from the BMC. BMC is not" +\ 848 " communicating." 849 BuiltIn().fail(gp.sprint_error(error_message)) 850 if int(float(state['uptime'])) < elapsed_boot_time: 851 uptime = state['uptime'] 852 gp.qprint_var(uptime) 853 gp.qprint_timen("The uptime is less than the elapsed boot time," 854 + " as expected.") 855 else: 856 error_message = "The uptime is greater than the elapsed boot time," +\ 857 " which is unexpected:\n" +\ 858 gp.sprint_var(start_boot_seconds) +\ 859 gp.sprint_var(state) 860 BuiltIn().fail(gp.sprint_error(error_message)) 861 862 gp.qprint_timen("Verifying that REST API interface is working.") 863 match_state = DotDict([('rest', '^1$')]) 864 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds") 865