1#!/usr/bin/env python 2 3r""" 4This module contains functions having to do with machine state: get_state, 5check_state, wait_state, etc. 6 7The 'State' is a composite of many pieces of data. Therefore, the functions 8in this module define state as an ordered dictionary. Here is an example of 9some test output showing machine state: 10 11default_state: 12 default_state[chassis]: On 13 default_state[boot_progress]: OSStart 14 default_state[operating_system]: BootComplete 15 default_state[host]: Running 16 default_state[os_ping]: 1 17 default_state[os_login]: 1 18 default_state[os_run_cmd]: 1 19 20Different users may very well have different needs when inquiring about 21state. Support for new pieces of state information may be added to this 22module as needed. 23 24By using the wait_state function, a caller can start a boot and then wait for 25a precisely defined state to indicate that the boot has succeeded. If 26the boot fails, they can see exactly why by looking at the current state as 27compared with the expected state. 28""" 29 30import gen_print as gp 31import gen_robot_print as grp 32import gen_valid as gv 33import gen_robot_utils as gru 34import gen_cmd as gc 35import bmc_ssh_utils as bsu 36 37import commands 38from robot.libraries.BuiltIn import BuiltIn 39from robot.utils import DotDict 40 41import re 42import os 43import sys 44import imp 45 46 47# NOTE: Avoid importing utils.robot because utils.robot imports state.py 48# (indirectly) which will cause failures. 49gru.my_import_resource("rest_client.robot") 50 51base_path = os.path.dirname(os.path.dirname( 52 imp.find_module("gen_robot_print")[1])) + os.sep 53sys.path.append(base_path + "data/") 54 55# Previously, I had this coded: 56# import variables as var 57# However, we ran into a problem where a robot program did this... 58# Variables ../../lib/ras/variables.py 59# Prior to doing this... 60# Library ../lib/state.py 61 62# This caused the wrong variables.py file to be selected. Attempts to fix this 63# have failed so far. For the moment, we will hard-code the value we need from 64# the file. 65 66SYSTEM_STATE_URI = "/xyz/openbmc_project/state/" 67 68# The BMC code has recently been changed as far as what states are defined and 69# what the state values can be. This module now has a means of processing both 70# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e. 71# OBMC_STATES_VERSION = 1). 72# The caller can set environment variable OBMC_STATES_VERSION to dictate 73# whether we're processing old or new style states. If OBMC_STATES_VERSION is 74# not set it will default to 1. 75 76# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old 77# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code 78# is being removed but the OBMC_STATES_VERSION value will stay for now in the 79# event that it is needed in the future. 80 81OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1)) 82 83# When a user calls get_state w/o specifying req_states, default_req_states 84# is used as its value. 85default_req_states = ['rest', 86 'chassis', 87 'bmc', 88 'boot_progress', 89 'operating_system', 90 'host', 91 'os_ping', 92 'os_login', 93 'os_run_cmd'] 94 95# valid_req_states is a list of sub states supported by the get_state function. 96# valid_req_states, default_req_states and master_os_up_match are used by the 97# get_state function. 98valid_req_states = ['ping', 99 'packet_loss', 100 'uptime', 101 'epoch_seconds', 102 'rest', 103 'chassis', 104 'requested_chassis', 105 'bmc', 106 'requested_bmc', 107 'boot_progress', 108 'operating_system', 109 'host', 110 'requested_host', 111 'attempts_left', 112 'os_ping', 113 'os_login', 114 'os_run_cmd'] 115 116# valid_os_req_states and default_os_req_states are used by the os_get_state 117# function. 118# valid_os_req_states is a list of state information supported by the 119# get_os_state function. 120valid_os_req_states = ['os_ping', 121 'os_login', 122 'os_run_cmd'] 123# When a user calls get_os_state w/o specifying req_states, 124# default_os_req_states is used as its value. 125default_os_req_states = ['os_ping', 126 'os_login', 127 'os_run_cmd'] 128 129# Presently, some BMCs appear to not keep time very well. This environment 130# variable directs the get_state function to use either the BMC's epoch time 131# or the local epoch time. 132USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0)) 133 134# Useful state constant definition(s). 135# default_state is an initial value which may be of use to callers. 136default_state = DotDict([('rest', '1'), 137 ('chassis', 'On'), 138 ('bmc', 'Ready'), 139 ('boot_progress', 'OSStart'), 140 ('operating_system', 'BootComplete'), 141 ('host', 'Running'), 142 ('os_ping', '1'), 143 ('os_login', '1'), 144 ('os_run_cmd', '1')]) 145 146# A match state for checking that the system is at "standby". 147standby_match_state = DotDict([('rest', '^1$'), 148 ('chassis', '^Off$'), 149 ('bmc', '^Ready$'), 150 ('boot_progress', '^$'), 151 ('operating_system', '^$'), 152 ('host', '^$')]) 153 154# A match state for checking that the system is at "os running". 155os_running_match_state = DotDict([('chassis', '^On$'), 156 ('bmc', '^Ready$'), 157 ('boot_progress', 158 'FW Progress, Starting OS|OSStart'), 159 ('operating_system', 'BootComplete'), 160 ('host', '^Running$'), 161 ('os_ping', '^1$'), 162 ('os_login', '^1$'), 163 ('os_run_cmd', '^1$')]) 164 165# A master dictionary to determine whether the os may be up. 166master_os_up_match = DotDict([('chassis', '^On$'), 167 ('bmc', '^Ready$'), 168 ('boot_progress', 169 'FW Progress, Starting OS|OSStart'), 170 ('operating_system', 'BootComplete'), 171 ('host', '^Running|Quiesced$')]) 172 173invalid_state_match = DotDict([('rest', '^$'), 174 ('chassis', '^$'), 175 ('bmc', '^$'), 176 ('boot_progress', '^$'), 177 ('operating_system', '^$'), 178 ('host', '^$')]) 179 180 181def return_state_constant(state_name='default'): 182 r""" 183 Return the named state dictionary constant. 184 """ 185 186 cmd_buf = "state = " + state_name 187 exec(cmd_buf) 188 return state 189 190 191def anchor_state(state): 192 r""" 193 Add regular expression anchors ("^" and "$") to the beginning and end of 194 each item in the state dictionary passed in. Return the resulting 195 dictionary. 196 197 Description of Arguments: 198 state A dictionary such as the one returned by the get_state() 199 function. 200 """ 201 202 anchored_state = state.copy() 203 for key, match_state_value in anchored_state.items(): 204 anchored_state[key] = "^" + str(anchored_state[key]) + "$" 205 206 return anchored_state 207 208 209def strip_anchor_state(state): 210 r""" 211 Strip regular expression anchors ("^" and "$") from the beginning and end 212 of each item in the state dictionary passed in. Return the resulting 213 dictionary. 214 215 Description of Arguments: 216 state A dictionary such as the one returned by the get_state() 217 function. 218 """ 219 220 stripped_state = state.copy() 221 for key, match_state_value in stripped_state.items(): 222 stripped_state[key] = stripped_state[key].strip("^$") 223 224 return stripped_state 225 226 227def compare_states(state, 228 match_state, 229 match_type='and'): 230 r""" 231 Compare 2 state dictionaries. Return True if they match and False if they 232 don't. Note that the match_state dictionary does not need to have an entry 233 corresponding to each entry in the state dictionary. But for each entry 234 that it does have, the corresponding state entry will be checked for a 235 match. 236 237 Description of arguments: 238 state A state dictionary such as the one returned by the 239 get_state function. 240 match_state A dictionary whose key/value pairs are "state field"/ 241 "state value". The state value is interpreted as a 242 regular expression. Every value in this dictionary is 243 considered. When match_type is 'and', if each and every 244 comparison matches, the two dictionaries are considered to 245 be matching. If match_type is 'or', if any two of the 246 elements compared match, the two dictionaries are 247 considered to be matching. 248 This value may also be any string accepted by 249 return_state_constant (e.g. "standby_match_state"). 250 In such a case this function will call 251 return_state_constant to convert it to a proper 252 dictionary as described above. 253 match_type This may be 'and' or 'or'. 254 """ 255 256 error_message = gv.svalid_value(match_type, var_name="match_type", 257 valid_values=['and', 'or']) 258 if error_message != "": 259 BuiltIn().fail(gp.sprint_error(error_message)) 260 261 if type(match_state) in (str, unicode): 262 match_state = return_state_constant(match_state) 263 264 default_match = (match_type == 'and') 265 for key, match_state_value in match_state.items(): 266 # Blank match_state_value means "don't care". 267 if match_state_value == "": 268 continue 269 try: 270 match = (re.match(match_state_value, str(state[key])) is not None) 271 except KeyError: 272 match = False 273 274 if match != default_match: 275 return match 276 277 return default_match 278 279 280def get_os_state(os_host="", 281 os_username="", 282 os_password="", 283 req_states=default_os_req_states, 284 os_up=True, 285 quiet=None): 286 r""" 287 Get component states for the operating system such as ping, login, 288 etc, put them into a dictionary and return them to the caller. 289 290 Note that all substate values are strings. 291 292 Description of arguments: 293 os_host The DNS name or IP address of the operating system. 294 This defaults to global ${OS_HOST}. 295 os_username The username to be used to login to the OS. 296 This defaults to global ${OS_USERNAME}. 297 os_password The password to be used to login to the OS. 298 This defaults to global ${OS_PASSWORD}. 299 req_states This is a list of states whose values are being requested by 300 the caller. 301 os_up If the caller knows that the os can't possibly be up, it can 302 improve performance by passing os_up=False. This function 303 will then simply return default values for all requested os 304 sub states. 305 quiet Indicates whether status details (e.g. curl commands) should 306 be written to the console. 307 Defaults to either global value of ${QUIET} or to 1. 308 """ 309 310 quiet = int(gp.get_var_value(quiet, 0)) 311 312 # Set parm defaults where necessary and validate all parms. 313 if os_host == "": 314 os_host = BuiltIn().get_variable_value("${OS_HOST}") 315 error_message = gv.svalid_value(os_host, var_name="os_host", 316 invalid_values=[None, ""]) 317 if error_message != "": 318 BuiltIn().fail(gp.sprint_error(error_message)) 319 320 if os_username == "": 321 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 322 error_message = gv.svalid_value(os_username, var_name="os_username", 323 invalid_values=[None, ""]) 324 if error_message != "": 325 BuiltIn().fail(gp.sprint_error(error_message)) 326 327 if os_password == "": 328 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 329 error_message = gv.svalid_value(os_password, var_name="os_password", 330 invalid_values=[None, ""]) 331 if error_message != "": 332 BuiltIn().fail(gp.sprint_error(error_message)) 333 334 invalid_req_states = [sub_state for sub_state in req_states 335 if sub_state not in valid_os_req_states] 336 if len(invalid_req_states) > 0: 337 error_message = "The following req_states are not supported:\n" +\ 338 gp.sprint_var(invalid_req_states) 339 BuiltIn().fail(gp.sprint_error(error_message)) 340 341 # Initialize all substate values supported by this function. 342 os_ping = 0 343 os_login = 0 344 os_run_cmd = 0 345 346 if os_up: 347 if 'os_ping' in req_states: 348 # See if the OS pings. 349 cmd_buf = "ping -c 1 -w 2 " + os_host 350 if not quiet: 351 gp.pissuing(cmd_buf) 352 rc, out_buf = commands.getstatusoutput(cmd_buf) 353 if rc == 0: 354 os_ping = 1 355 356 # Programming note: All attributes which do not require an ssh login 357 # should have been processed by this point. 358 master_req_login = ['os_login', 'os_run_cmd'] 359 req_login = [sub_state for sub_state in req_states if sub_state in 360 master_req_login] 361 must_login = (len(req_login) > 0) 362 363 if must_login: 364 output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet, 365 ignore_err=1, 366 time_out=20) 367 if rc == 0: 368 os_login = 1 369 os_run_cmd = 1 370 else: 371 gp.dprint_vars(output, stderr) 372 gp.dprint_vars(rc, 1) 373 374 os_state = DotDict() 375 for sub_state in req_states: 376 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" 377 exec(cmd_buf) 378 379 return os_state 380 381 382def get_state(openbmc_host="", 383 openbmc_username="", 384 openbmc_password="", 385 os_host="", 386 os_username="", 387 os_password="", 388 req_states=default_req_states, 389 quiet=None): 390 r""" 391 Get component states such as chassis state, bmc state, etc, put them into a 392 dictionary and return them to the caller. 393 394 Note that all substate values are strings. 395 396 Description of arguments: 397 openbmc_host The DNS name or IP address of the BMC. 398 This defaults to global ${OPENBMC_HOST}. 399 openbmc_username The username to be used to login to the BMC. 400 This defaults to global ${OPENBMC_USERNAME}. 401 openbmc_password The password to be used to login to the BMC. 402 This defaults to global ${OPENBMC_PASSWORD}. 403 os_host The DNS name or IP address of the operating system. 404 This defaults to global ${OS_HOST}. 405 os_username The username to be used to login to the OS. 406 This defaults to global ${OS_USERNAME}. 407 os_password The password to be used to login to the OS. 408 This defaults to global ${OS_PASSWORD}. 409 req_states This is a list of states whose values are being requested 410 by the caller. 411 quiet Indicates whether status details (e.g. curl commands) 412 should be written to the console. 413 Defaults to either global value of ${QUIET} or to 1. 414 """ 415 416 quiet = int(gp.get_var_value(quiet, 0)) 417 418 # Set parm defaults where necessary and validate all parms. 419 if openbmc_host == "": 420 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") 421 error_message = gv.svalid_value(openbmc_host, 422 var_name="openbmc_host", 423 invalid_values=[None, ""]) 424 if error_message != "": 425 BuiltIn().fail(gp.sprint_error(error_message)) 426 427 if openbmc_username == "": 428 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") 429 error_message = gv.svalid_value(openbmc_username, 430 var_name="openbmc_username", 431 invalid_values=[None, ""]) 432 if error_message != "": 433 BuiltIn().fail(gp.sprint_error(error_message)) 434 435 if openbmc_password == "": 436 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") 437 error_message = gv.svalid_value(openbmc_password, 438 var_name="openbmc_password", 439 invalid_values=[None, ""]) 440 if error_message != "": 441 BuiltIn().fail(gp.sprint_error(error_message)) 442 443 # NOTE: OS parms are optional. 444 if os_host == "": 445 os_host = BuiltIn().get_variable_value("${OS_HOST}") 446 if os_host is None: 447 os_host = "" 448 449 if os_username is "": 450 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 451 if os_username is None: 452 os_username = "" 453 454 if os_password is "": 455 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 456 if os_password is None: 457 os_password = "" 458 459 invalid_req_states = [sub_state for sub_state in req_states 460 if sub_state not in valid_req_states] 461 if len(invalid_req_states) > 0: 462 error_message = "The following req_states are not supported:\n" +\ 463 gp.sprint_var(invalid_req_states) 464 BuiltIn().fail(gp.sprint_error(error_message)) 465 466 # Initialize all substate values supported by this function. 467 ping = 0 468 packet_loss = '' 469 uptime = '' 470 epoch_seconds = '' 471 rest = '' 472 chassis = '' 473 requested_chassis = '' 474 bmc = '' 475 requested_bmc = '' 476 boot_progress = '' 477 operating_system = '' 478 host = '' 479 requested_host = '' 480 attempts_left = '' 481 482 # Get the component states. 483 if 'ping' in req_states: 484 # See if the OS pings. 485 cmd_buf = "ping -c 1 -w 2 " + openbmc_host 486 if not quiet: 487 gp.pissuing(cmd_buf) 488 rc, out_buf = commands.getstatusoutput(cmd_buf) 489 if rc == 0: 490 ping = 1 491 492 if 'packet_loss' in req_states: 493 # See if the OS pings. 494 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\ 495 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" 496 if not quiet: 497 gp.pissuing(cmd_buf) 498 rc, out_buf = commands.getstatusoutput(cmd_buf) 499 if rc == 0: 500 packet_loss = out_buf.rstrip("\n") 501 502 if 'uptime' in req_states: 503 # Sometimes reading uptime results in a blank value. Call with 504 # wait_until_keyword_succeeds to ensure a non-blank value is obtained. 505 remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\ 506 " && [ ! -z \"${uptime}\" ] && echo ${uptime}" 507 cmd_buf = ["BMC Execute Command", 508 re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1'] 509 if not quiet: 510 # Get loc_test_mode parm for improved output on pissuing. 511 # See sprint_issuing in gen_print.py for details. 512 loc_test_mode = int(gp.get_var_value(var_name="test_mode", 513 default=0)) 514 grp.rpissuing_keyword(cmd_buf, loc_test_mode) 515 gp.pissuing(remote_cmd_buf, loc_test_mode) 516 try: 517 stdout, stderr, rc =\ 518 BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec", 519 *cmd_buf) 520 if rc == 0 and stderr == "": 521 uptime = stdout 522 except AssertionError as my_assertion_error: 523 pass 524 525 if 'epoch_seconds' in req_states: 526 date_cmd_buf = "date -u +%s" 527 if USE_BMC_EPOCH_TIME: 528 cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}'] 529 if not quiet: 530 grp.rpissuing_keyword(cmd_buf) 531 status, ret_values = \ 532 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 533 if status == "PASS": 534 stdout, stderr, rc = ret_values 535 if rc == 0 and stderr == "": 536 epoch_seconds = stdout.rstrip("\n") 537 else: 538 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf, 539 quiet=quiet, 540 print_output=0) 541 if shell_rc == 0: 542 epoch_seconds = out_buf.rstrip("\n") 543 544 master_req_rest = ['rest', 'host', 'requested_host', 'operating_system', 545 'attempts_left', 'boot_progress', 'chassis', 546 'requested_chassis' 'bmc' 'requested_bmc'] 547 548 req_rest = [sub_state for sub_state in req_states if sub_state in 549 master_req_rest] 550 need_rest = (len(req_rest) > 0) 551 state = DotDict() 552 if need_rest: 553 cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate", 554 "quiet=${" + str(quiet) + "}"] 555 grp.rdpissuing_keyword(cmd_buf) 556 status, ret_values = \ 557 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 558 if status == "PASS": 559 state['rest'] = '1' 560 else: 561 state['rest'] = '0' 562 563 if int(state['rest']): 564 for url_path in ret_values: 565 for attr_name in ret_values[url_path]: 566 # Create a state key value based on the attr_name. 567 if isinstance(ret_values[url_path][attr_name], unicode): 568 ret_values[url_path][attr_name] = \ 569 re.sub(r'.*\.', "", 570 ret_values[url_path][attr_name]) 571 # Do some key name manipulations. 572 new_attr_name = re.sub(r'^Current|(State|Transition)$', 573 "", attr_name) 574 new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name) 575 new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1', 576 new_attr_name) 577 new_attr_name = new_attr_name.lower().lstrip("_") 578 new_attr_name = re.sub(r'power', r'chassis', new_attr_name) 579 if new_attr_name in req_states: 580 state[new_attr_name] = ret_values[url_path][attr_name] 581 582 for sub_state in req_states: 583 if sub_state in state: 584 continue 585 if sub_state.startswith("os_"): 586 # We pass "os_" requests on to get_os_state. 587 continue 588 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" 589 exec(cmd_buf) 590 591 if os_host == "": 592 # The caller has not specified an os_host so as far as we're concerned, 593 # it doesn't exist. 594 return state 595 596 os_req_states = [sub_state for sub_state in req_states 597 if sub_state.startswith('os_')] 598 599 if len(os_req_states) > 0: 600 # The caller has specified an os_host and they have requested 601 # information on os substates. 602 603 # Based on the information gathered on bmc, we'll try to make a 604 # determination of whether the os is even up. We'll pass the result 605 # of that assessment to get_os_state to enhance performance. 606 os_up_match = DotDict() 607 for sub_state in master_os_up_match: 608 if sub_state in req_states: 609 os_up_match[sub_state] = master_os_up_match[sub_state] 610 os_up = compare_states(state, os_up_match) 611 os_state = get_os_state(os_host=os_host, 612 os_username=os_username, 613 os_password=os_password, 614 req_states=os_req_states, 615 os_up=os_up, 616 quiet=quiet) 617 # Append os_state dictionary to ours. 618 state.update(os_state) 619 620 return state 621 622 623def check_state(match_state, 624 invert=0, 625 print_string="", 626 openbmc_host="", 627 openbmc_username="", 628 openbmc_password="", 629 os_host="", 630 os_username="", 631 os_password="", 632 quiet=None): 633 r""" 634 Check that the Open BMC machine's composite state matches the specified 635 state. On success, this keyword returns the machine's composite state as a 636 dictionary. 637 638 Description of arguments: 639 match_state A dictionary whose key/value pairs are "state field"/ 640 "state value". The state value is interpreted as a 641 regular expression. Example call from robot: 642 ${match_state}= Create Dictionary chassis=^On$ 643 ... bmc=^Ready$ 644 ... boot_progress=^OSStart$ 645 ${state}= Check State &{match_state} 646 invert If this flag is set, this function will succeed if the 647 states do NOT match. 648 print_string This function will print this string to the console prior 649 to getting the state. 650 openbmc_host The DNS name or IP address of the BMC. 651 This defaults to global ${OPENBMC_HOST}. 652 openbmc_username The username to be used to login to the BMC. 653 This defaults to global ${OPENBMC_USERNAME}. 654 openbmc_password The password to be used to login to the BMC. 655 This defaults to global ${OPENBMC_PASSWORD}. 656 os_host The DNS name or IP address of the operating system. 657 This defaults to global ${OS_HOST}. 658 os_username The username to be used to login to the OS. 659 This defaults to global ${OS_USERNAME}. 660 os_password The password to be used to login to the OS. 661 This defaults to global ${OS_PASSWORD}. 662 quiet Indicates whether status details should be written to the 663 console. Defaults to either global value of ${QUIET} or 664 to 1. 665 """ 666 667 quiet = int(gp.get_var_value(quiet, 0)) 668 669 grp.rprint(print_string) 670 671 req_states = match_state.keys() 672 # Initialize state. 673 state = get_state(openbmc_host=openbmc_host, 674 openbmc_username=openbmc_username, 675 openbmc_password=openbmc_password, 676 os_host=os_host, 677 os_username=os_username, 678 os_password=os_password, 679 req_states=req_states, 680 quiet=quiet) 681 if not quiet: 682 gp.print_var(state) 683 684 match = compare_states(state, match_state) 685 686 if invert and match: 687 fail_msg = "The current state of the machine matches the match" +\ 688 " state:\n" + gp.sprint_varx("state", state) 689 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 690 elif not invert and not match: 691 fail_msg = "The current state of the machine does NOT match the" +\ 692 " match state:\n" +\ 693 gp.sprint_varx("state", state) 694 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 695 696 return state 697 698 699def wait_state(match_state=(), 700 wait_time="1 min", 701 interval="1 second", 702 invert=0, 703 openbmc_host="", 704 openbmc_username="", 705 openbmc_password="", 706 os_host="", 707 os_username="", 708 os_password="", 709 quiet=None): 710 r""" 711 Wait for the Open BMC machine's composite state to match the specified 712 state. On success, this keyword returns the machine's composite state as 713 a dictionary. 714 715 Description of arguments: 716 match_state A dictionary whose key/value pairs are "state field"/ 717 "state value". See check_state (above) for details. 718 This value may also be any string accepted by 719 return_state_constant (e.g. "standby_match_state"). 720 In such a case this function will call 721 return_state_constant to convert it to a proper 722 dictionary as described above. 723 wait_time The total amount of time to wait for the desired state. 724 This value may be expressed in Robot Framework's time 725 format (e.g. 1 minute, 2 min 3 s, 4.5). 726 interval The amount of time between state checks. 727 This value may be expressed in Robot Framework's time 728 format (e.g. 1 minute, 2 min 3 s, 4.5). 729 invert If this flag is set, this function will for the state of 730 the machine to cease to match the match state. 731 openbmc_host The DNS name or IP address of the BMC. 732 This defaults to global ${OPENBMC_HOST}. 733 openbmc_username The username to be used to login to the BMC. 734 This defaults to global ${OPENBMC_USERNAME}. 735 openbmc_password The password to be used to login to the BMC. 736 This defaults to global ${OPENBMC_PASSWORD}. 737 os_host The DNS name or IP address of the operating system. 738 This defaults to global ${OS_HOST}. 739 os_username The username to be used to login to the OS. 740 This defaults to global ${OS_USERNAME}. 741 os_password The password to be used to login to the OS. 742 This defaults to global ${OS_PASSWORD}. 743 quiet Indicates whether status details should be written to the 744 console. Defaults to either global value of ${QUIET} or 745 to 1. 746 """ 747 748 quiet = int(gp.get_var_value(quiet, 0)) 749 750 if type(match_state) in (str, unicode): 751 match_state = return_state_constant(match_state) 752 753 if not quiet: 754 if invert: 755 alt_text = "cease to " 756 else: 757 alt_text = "" 758 gp.print_timen("Checking every " + str(interval) + " for up to " 759 + str(wait_time) + " for the state of the machine to " 760 + alt_text + "match the state shown below.") 761 gp.print_var(match_state) 762 763 if quiet: 764 print_string = "" 765 else: 766 print_string = "#" 767 768 debug = int(BuiltIn().get_variable_value("${debug}", "0")) 769 if debug: 770 # In debug we print state so no need to print the "#". 771 print_string = "" 772 check_state_quiet = 1 - debug 773 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}", 774 "print_string=" + print_string, "openbmc_host=" + openbmc_host, 775 "openbmc_username=" + openbmc_username, 776 "openbmc_password=" + openbmc_password, "os_host=" + os_host, 777 "os_username=" + os_username, "os_password=" + os_password, 778 "quiet=${" + str(check_state_quiet) + "}"] 779 grp.rdpissuing_keyword(cmd_buf) 780 try: 781 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval, 782 *cmd_buf) 783 except AssertionError as my_assertion_error: 784 gp.printn() 785 message = my_assertion_error.args[0] 786 BuiltIn().fail(message) 787 788 if not quiet: 789 gp.printn() 790 if invert: 791 gp.print_timen("The states no longer match:") 792 else: 793 gp.print_timen("The states match:") 794 gp.print_var(state) 795 796 return state 797 798 799def wait_for_comm_cycle(start_boot_seconds, 800 quiet=None): 801 r""" 802 Wait for communications to the BMC to stop working and then resume working. 803 This function is useful when you have initiated some kind of reboot. 804 805 Description of arguments: 806 start_boot_seconds The time that the boot test started. The format is the 807 epoch time in seconds, i.e. the number of seconds since 808 1970-01-01 00:00:00 UTC. This value should be obtained 809 from the BMC so that it is not dependent on any kind of 810 synchronization between this machine and the target BMC 811 This will allow this program to work correctly even in 812 a simulated environment. This value should be obtained 813 by the caller prior to initiating a reboot. It can be 814 obtained as follows: 815 state = st.get_state(req_states=['epoch_seconds']) 816 """ 817 818 quiet = int(gp.get_var_value(quiet, 0)) 819 820 # Validate parms. 821 error_message = gv.svalid_integer(start_boot_seconds, 822 var_name="start_boot_seconds") 823 if error_message != "": 824 BuiltIn().fail(gp.sprint_error(error_message)) 825 826 match_state = anchor_state(DotDict([('packet_loss', '100')])) 827 # Wait for 100% packet loss trying to ping machine. 828 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 829 830 match_state['packet_loss'] = '^0$' 831 # Wait for 0% packet loss trying to ping machine. 832 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 833 834 # Get the uptime and epoch seconds for comparisons. We want to be sure 835 # that the uptime is less than the elapsed boot time. Further proof that 836 # a reboot has indeed occurred (vs random network instability giving a 837 # false positive. We also use wait_state because the BMC may take a short 838 # while to be ready to process SSH requests. 839 match_state = DotDict([('uptime', '^[0-9\\.]+$'), 840 ('epoch_seconds', '^[0-9]+$')]) 841 state = wait_state(match_state, wait_time="2 mins", interval="1 second") 842 843 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds 844 gp.qprint_var(elapsed_boot_time) 845 if state['uptime'] == "": 846 error_message = "Unable to obtain uptime from the BMC. BMC is not" +\ 847 " communicating." 848 BuiltIn().fail(gp.sprint_error(error_message)) 849 if int(float(state['uptime'])) < elapsed_boot_time: 850 uptime = state['uptime'] 851 gp.qprint_var(uptime) 852 gp.qprint_timen("The uptime is less than the elapsed boot time," 853 + " as expected.") 854 else: 855 error_message = "The uptime is greater than the elapsed boot time," +\ 856 " which is unexpected:\n" +\ 857 gp.sprint_var(start_boot_seconds) +\ 858 gp.sprint_var(state) 859 BuiltIn().fail(gp.sprint_error(error_message)) 860 861 gp.qprint_timen("Verifying that REST API interface is working.") 862 match_state = DotDict([('rest', '^1$')]) 863 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds") 864