1#!/usr/bin/env python 2 3r""" 4This module contains functions having to do with machine state: get_state, 5check_state, wait_state, etc. 6 7The 'State' is a composite of many pieces of data. Therefore, the functions 8in this module define state as an ordered dictionary. Here is an example of 9some test output showing machine state: 10 11default_state: 12 default_state[chassis]: On 13 default_state[boot_progress]: OSStart 14 default_state[operating_system]: BootComplete 15 default_state[host]: Running 16 default_state[os_ping]: 1 17 default_state[os_login]: 1 18 default_state[os_run_cmd]: 1 19 20Different users may very well have different needs when inquiring about 21state. Support for new pieces of state information may be added to this 22module as needed. 23 24By using the wait_state function, a caller can start a boot and then wait for 25a precisely defined state to indicate that the boot has succeeded. If 26the boot fails, they can see exactly why by looking at the current state as 27compared with the expected state. 28""" 29 30import gen_print as gp 31import gen_robot_print as grp 32import gen_valid as gv 33import gen_robot_utils as gru 34import gen_cmd as gc 35 36import commands 37from robot.libraries.BuiltIn import BuiltIn 38from robot.utils import DotDict 39 40import re 41import os 42import sys 43import imp 44 45 46# We need utils.robot to get keywords like "Get Chassis Power State". 47gru.my_import_resource("utils.robot") 48gru.my_import_resource("state_manager.robot") 49 50base_path = os.path.dirname(os.path.dirname( 51 imp.find_module("gen_robot_print")[1])) + os.sep 52sys.path.append(base_path + "data/") 53 54# Previously, I had this coded: 55# import variables as var 56# However, we ran into a problem where a robot program did this... 57# Variables ../../lib/ras/variables.py 58# Prior to doing this... 59# Library ../lib/state.py 60 61# This caused the wrong variables.py file to be selected. Attempts to fix this 62# have failed so far. For the moment, we will hard-code the value we need from 63# the file. 64 65SYSTEM_STATE_URI = "/xyz/openbmc_project/state/" 66 67# The BMC code has recently been changed as far as what states are defined and 68# what the state values can be. This module now has a means of processing both 69# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e. 70# OBMC_STATES_VERSION = 1). 71# The caller can set environment variable OBMC_STATES_VERSION to dictate 72# whether we're processing old or new style states. If OBMC_STATES_VERSION is 73# not set it will default to 1. 74 75# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old 76# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code 77# is being removed but the OBMC_STATES_VERSION value will stay for now in the 78# event that it is needed in the future. 79 80OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1)) 81 82# When a user calls get_state w/o specifying req_states, default_req_states 83# is used as its value. 84default_req_states = ['rest', 85 'chassis', 86 'bmc', 87 'boot_progress', 88 'operating_system', 89 'host', 90 'os_ping', 91 'os_login', 92 'os_run_cmd'] 93 94# valid_req_states is a list of sub states supported by the get_state function. 95# valid_req_states, default_req_states and master_os_up_match are used by the 96# get_state function. 97valid_req_states = ['ping', 98 'packet_loss', 99 'uptime', 100 'epoch_seconds', 101 'rest', 102 'chassis', 103 'requested_chassis', 104 'bmc', 105 'requested_bmc', 106 'boot_progress', 107 'operating_system', 108 'host', 109 'requested_host', 110 'attempts_left', 111 'os_ping', 112 'os_login', 113 'os_run_cmd'] 114 115# valid_os_req_states and default_os_req_states are used by the os_get_state 116# function. 117# valid_os_req_states is a list of state information supported by the 118# get_os_state function. 119valid_os_req_states = ['os_ping', 120 'os_login', 121 'os_run_cmd'] 122# When a user calls get_os_state w/o specifying req_states, 123# default_os_req_states is used as its value. 124default_os_req_states = ['os_ping', 125 'os_login', 126 'os_run_cmd'] 127 128# Presently, some BMCs appear to not keep time very well. This environment 129# variable directs the get_state function to use either the BMC's epoch time 130# or the local epoch time. 131USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0)) 132 133# Useful state constant definition(s). 134# default_state is an initial value which may be of use to callers. 135default_state = DotDict([('rest', '1'), 136 ('chassis', 'On'), 137 ('bmc', 'Ready'), 138 ('boot_progress', 'OSStart'), 139 ('operating_system', 'BootComplete'), 140 ('host', 'Running'), 141 ('os_ping', '1'), 142 ('os_login', '1'), 143 ('os_run_cmd', '1')]) 144 145# A match state for checking that the system is at "standby". 146standby_match_state = DotDict([('rest', '^1$'), 147 ('chassis', '^Off$'), 148 ('bmc', '^Ready$'), 149 ('boot_progress', '^$'), 150 ('operating_system', '^$'), 151 ('host', '^$')]) 152 153# A match state for checking that the system is at "os running". 154os_running_match_state = DotDict([('chassis', '^On$'), 155 ('bmc', '^Ready$'), 156 ('boot_progress', 157 'FW Progress, Starting OS|OSStart'), 158 ('operating_system', 'BootComplete'), 159 ('host', '^Running$'), 160 ('os_ping', '^1$'), 161 ('os_login', '^1$'), 162 ('os_run_cmd', '^1$')]) 163 164# A master dictionary to determine whether the os may be up. 165master_os_up_match = DotDict([('chassis', '^On$'), 166 ('bmc', '^Ready$'), 167 ('boot_progress', 168 'FW Progress, Starting OS|OSStart'), 169 ('operating_system', 'BootComplete'), 170 ('host', '^Running$')]) 171 172invalid_state_match = DotDict([('rest', '^$'), 173 ('chassis', '^$'), 174 ('bmc', '^$'), 175 ('boot_progress', '^$'), 176 ('operating_system', '^$'), 177 ('host', '^$')]) 178 179 180def return_state_constant(state_name='default'): 181 r""" 182 Return the named state dictionary constant. 183 """ 184 185 cmd_buf = "state = " + state_name 186 exec(cmd_buf) 187 return state 188 189 190def anchor_state(state): 191 r""" 192 Add regular expression anchors ("^" and "$") to the beginning and end of 193 each item in the state dictionary passed in. Return the resulting 194 dictionary. 195 196 Description of Arguments: 197 state A dictionary such as the one returned by the get_state() 198 function. 199 """ 200 201 anchored_state = state.copy() 202 for key, match_state_value in anchored_state.items(): 203 anchored_state[key] = "^" + str(anchored_state[key]) + "$" 204 205 return anchored_state 206 207 208def strip_anchor_state(state): 209 r""" 210 Strip regular expression anchors ("^" and "$") from the beginning and end 211 of each item in the state dictionary passed in. Return the resulting 212 dictionary. 213 214 Description of Arguments: 215 state A dictionary such as the one returned by the get_state() 216 function. 217 """ 218 219 stripped_state = state.copy() 220 for key, match_state_value in stripped_state.items(): 221 stripped_state[key] = stripped_state[key].strip("^$") 222 223 return stripped_state 224 225 226def compare_states(state, 227 match_state, 228 match_type='and'): 229 r""" 230 Compare 2 state dictionaries. Return True if they match and False if they 231 don't. Note that the match_state dictionary does not need to have an entry 232 corresponding to each entry in the state dictionary. But for each entry 233 that it does have, the corresponding state entry will be checked for a 234 match. 235 236 Description of arguments: 237 state A state dictionary such as the one returned by the 238 get_state function. 239 match_state A dictionary whose key/value pairs are "state field"/ 240 "state value". The state value is interpreted as a 241 regular expression. Every value in this dictionary is 242 considered. When match_type is 'and', if each and every 243 comparison matches, the two dictionaries are considered to 244 be matching. If match_type is 'or', if any two of the 245 elements compared match, the two dictionaries are 246 considered to be matching. 247 This value may also be any string accepted by 248 return_state_constant (e.g. "standby_match_state"). 249 In such a case this function will call 250 return_state_constant to convert it to a proper 251 dictionary as described above. 252 match_type This may be 'and' or 'or'. 253 """ 254 255 error_message = gv.svalid_value(match_type, var_name="match_type", 256 valid_values=['and', 'or']) 257 if error_message != "": 258 BuiltIn().fail(gp.sprint_error(error_message)) 259 260 if type(match_state) in (str, unicode): 261 match_state = return_state_constant(match_state) 262 263 default_match = (match_type == 'and') 264 for key, match_state_value in match_state.items(): 265 # Blank match_state_value means "don't care". 266 if match_state_value == "": 267 continue 268 try: 269 match = (re.match(match_state_value, str(state[key])) is not None) 270 except KeyError: 271 match = False 272 273 if match != default_match: 274 return match 275 276 return default_match 277 278 279def get_os_state(os_host="", 280 os_username="", 281 os_password="", 282 req_states=default_os_req_states, 283 os_up=True, 284 quiet=None): 285 r""" 286 Get component states for the operating system such as ping, login, 287 etc, put them into a dictionary and return them to the caller. 288 289 Note that all substate values are strings. 290 291 Description of arguments: 292 os_host The DNS name or IP address of the operating system. 293 This defaults to global ${OS_HOST}. 294 os_username The username to be used to login to the OS. 295 This defaults to global ${OS_USERNAME}. 296 os_password The password to be used to login to the OS. 297 This defaults to global ${OS_PASSWORD}. 298 req_states This is a list of states whose values are being requested by 299 the caller. 300 os_up If the caller knows that the os can't possibly be up, it can 301 improve performance by passing os_up=False. This function 302 will then simply return default values for all requested os 303 sub states. 304 quiet Indicates whether status details (e.g. curl commands) should 305 be written to the console. 306 Defaults to either global value of ${QUIET} or to 1. 307 """ 308 309 quiet = int(gp.get_var_value(quiet, 0)) 310 311 # Set parm defaults where necessary and validate all parms. 312 if os_host == "": 313 os_host = BuiltIn().get_variable_value("${OS_HOST}") 314 error_message = gv.svalid_value(os_host, var_name="os_host", 315 invalid_values=[None, ""]) 316 if error_message != "": 317 BuiltIn().fail(gp.sprint_error(error_message)) 318 319 if os_username == "": 320 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 321 error_message = gv.svalid_value(os_username, var_name="os_username", 322 invalid_values=[None, ""]) 323 if error_message != "": 324 BuiltIn().fail(gp.sprint_error(error_message)) 325 326 if os_password == "": 327 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 328 error_message = gv.svalid_value(os_password, var_name="os_password", 329 invalid_values=[None, ""]) 330 if error_message != "": 331 BuiltIn().fail(gp.sprint_error(error_message)) 332 333 invalid_req_states = [sub_state for sub_state in req_states 334 if sub_state not in valid_os_req_states] 335 if len(invalid_req_states) > 0: 336 error_message = "The following req_states are not supported:\n" +\ 337 gp.sprint_var(invalid_req_states) 338 BuiltIn().fail(gp.sprint_error(error_message)) 339 340 # Initialize all substate values supported by this function. 341 os_ping = 0 342 os_login = 0 343 os_run_cmd = 0 344 345 if os_up: 346 if 'os_ping' in req_states: 347 # See if the OS pings. 348 cmd_buf = "ping -c 1 -w 2 " + os_host 349 if not quiet: 350 gp.pissuing(cmd_buf) 351 rc, out_buf = commands.getstatusoutput(cmd_buf) 352 if rc == 0: 353 os_ping = 1 354 355 # Programming note: All attributes which do not require an ssh login 356 # should have been processed by this point. 357 master_req_login = ['os_login', 'os_run_cmd'] 358 req_login = [sub_state for sub_state in req_states if sub_state in 359 master_req_login] 360 must_login = (len(req_login) > 0) 361 362 if must_login: 363 # Open SSH connection to OS. Note that this doesn't fail even when 364 # the OS is not up. 365 cmd_buf = ["SSHLibrary.Open Connection", os_host] 366 if not quiet: 367 grp.rpissuing_keyword(cmd_buf) 368 ix = BuiltIn().run_keyword(*cmd_buf) 369 370 # Login to OS. 371 cmd_buf = ["Login", os_username, os_password] 372 if not quiet: 373 grp.rpissuing_keyword(cmd_buf) 374 status, ret_values = \ 375 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 376 if status == "PASS": 377 os_login = 1 378 else: 379 gp.dprint_var(status) 380 gp.dprint_var(ret_values) 381 382 if os_login: 383 if 'os_run_cmd' in req_states: 384 # Try running a simple command (uptime) on the OS. 385 cmd_buf = ["Execute Command", "uptime", 386 "return_stderr=True", "return_rc=True"] 387 if not quiet: 388 grp.rpissuing_keyword(cmd_buf) 389 # Note that in spite of its name, there are occasions 390 # where run_keyword_and_ignore_error can fail. 391 status, ret_values = \ 392 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 393 if status == "PASS": 394 stdout, stderr, rc = ret_values 395 if rc == 0 and stderr == "": 396 os_run_cmd = 1 397 else: 398 gp.dprint_var(status) 399 gp.dprint_var(stdout) 400 gp.dprint_var(stderr) 401 gp.dprint_var(rc) 402 else: 403 gp.dprint_var(status) 404 gp.dprint_var(ret_values) 405 406 os_state = DotDict() 407 for sub_state in req_states: 408 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" 409 exec(cmd_buf) 410 411 return os_state 412 413 414def get_state(openbmc_host="", 415 openbmc_username="", 416 openbmc_password="", 417 os_host="", 418 os_username="", 419 os_password="", 420 req_states=default_req_states, 421 quiet=None): 422 r""" 423 Get component states such as chassis state, bmc state, etc, put them into a 424 dictionary and return them to the caller. 425 426 Note that all substate values are strings. 427 428 Description of arguments: 429 openbmc_host The DNS name or IP address of the BMC. 430 This defaults to global ${OPENBMC_HOST}. 431 openbmc_username The username to be used to login to the BMC. 432 This defaults to global ${OPENBMC_USERNAME}. 433 openbmc_password The password to be used to login to the BMC. 434 This defaults to global ${OPENBMC_PASSWORD}. 435 os_host The DNS name or IP address of the operating system. 436 This defaults to global ${OS_HOST}. 437 os_username The username to be used to login to the OS. 438 This defaults to global ${OS_USERNAME}. 439 os_password The password to be used to login to the OS. 440 This defaults to global ${OS_PASSWORD}. 441 req_states This is a list of states whose values are being requested 442 by the caller. 443 quiet Indicates whether status details (e.g. curl commands) 444 should be written to the console. 445 Defaults to either global value of ${QUIET} or to 1. 446 """ 447 448 quiet = int(gp.get_var_value(quiet, 0)) 449 450 # Set parm defaults where necessary and validate all parms. 451 if openbmc_host == "": 452 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") 453 error_message = gv.svalid_value(openbmc_host, 454 var_name="openbmc_host", 455 invalid_values=[None, ""]) 456 if error_message != "": 457 BuiltIn().fail(gp.sprint_error(error_message)) 458 459 if openbmc_username == "": 460 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") 461 error_message = gv.svalid_value(openbmc_username, 462 var_name="openbmc_username", 463 invalid_values=[None, ""]) 464 if error_message != "": 465 BuiltIn().fail(gp.sprint_error(error_message)) 466 467 if openbmc_password == "": 468 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") 469 error_message = gv.svalid_value(openbmc_password, 470 var_name="openbmc_password", 471 invalid_values=[None, ""]) 472 if error_message != "": 473 BuiltIn().fail(gp.sprint_error(error_message)) 474 475 # NOTE: OS parms are optional. 476 if os_host == "": 477 os_host = BuiltIn().get_variable_value("${OS_HOST}") 478 if os_host is None: 479 os_host = "" 480 481 if os_username is "": 482 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 483 if os_username is None: 484 os_username = "" 485 486 if os_password is "": 487 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 488 if os_password is None: 489 os_password = "" 490 491 invalid_req_states = [sub_state for sub_state in req_states 492 if sub_state not in valid_req_states] 493 if len(invalid_req_states) > 0: 494 error_message = "The following req_states are not supported:\n" +\ 495 gp.sprint_var(invalid_req_states) 496 BuiltIn().fail(gp.sprint_error(error_message)) 497 498 # Initialize all substate values supported by this function. 499 ping = 0 500 packet_loss = '' 501 uptime = '' 502 epoch_seconds = '' 503 rest = '' 504 chassis = '' 505 requested_chassis = '' 506 bmc = '' 507 requested_bmc = '' 508 boot_progress = '' 509 operating_system = '' 510 host = '' 511 requested_host = '' 512 attempts_left = '' 513 514 # Get the component states. 515 if 'ping' in req_states: 516 # See if the OS pings. 517 cmd_buf = "ping -c 1 -w 2 " + openbmc_host 518 if not quiet: 519 gp.pissuing(cmd_buf) 520 rc, out_buf = commands.getstatusoutput(cmd_buf) 521 if rc == 0: 522 ping = 1 523 524 if 'packet_loss' in req_states: 525 # See if the OS pings. 526 cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\ 527 " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" 528 if not quiet: 529 gp.pissuing(cmd_buf) 530 rc, out_buf = commands.getstatusoutput(cmd_buf) 531 if rc == 0: 532 packet_loss = out_buf.rstrip("\n") 533 534 if 'uptime' in req_states: 535 # Sometimes reading uptime results in a blank value. Call with 536 # wait_until_keyword_succeeds to ensure a non-blank value is obtained. 537 remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\ 538 " && [ ! -z \"${uptime}\" ] && echo ${uptime}" 539 cmd_buf = ["BMC Execute Command", 540 re.sub(r'\\$', '\\$', remote_cmd_buf), 'quiet=1'] 541 if not quiet: 542 grp.rpissuing_keyword(cmd_buf) 543 grp.rpissuing(remote_cmd_buf) 544 try: 545 stdout, stderr, rc =\ 546 BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec", 547 *cmd_buf) 548 if rc == 0 and stderr == "": 549 uptime = stdout 550 except AssertionError as my_assertion_error: 551 pass 552 553 if 'epoch_seconds' in req_states: 554 date_cmd_buf = "date -u +%s" 555 if USE_BMC_EPOCH_TIME: 556 cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}'] 557 if not quiet: 558 grp.rpissuing_keyword(cmd_buf) 559 status, ret_values = \ 560 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 561 if status == "PASS": 562 stdout, stderr, rc = ret_values 563 if rc == 0 and stderr == "": 564 epoch_seconds = stdout.rstrip("\n") 565 else: 566 shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf, 567 quiet=quiet, 568 print_output=0) 569 if shell_rc == 0: 570 epoch_seconds = out_buf.rstrip("\n") 571 572 master_req_rest = ['rest', 'host', 'requested_host', 'operating_system', 573 'attempts_left', 'boot_progress', 'chassis', 574 'requested_chassis' 'bmc' 'requested_bmc'] 575 576 req_rest = [sub_state for sub_state in req_states if sub_state in 577 master_req_rest] 578 need_rest = (len(req_rest) > 0) 579 state = DotDict() 580 if need_rest: 581 cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate", 582 "quiet=${" + str(quiet) + "}"] 583 grp.rdpissuing_keyword(cmd_buf) 584 status, ret_values = \ 585 BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 586 if status == "PASS": 587 state['rest'] = '1' 588 else: 589 state['rest'] = '0' 590 591 if int(state['rest']): 592 for url_path in ret_values: 593 for attr_name in ret_values[url_path]: 594 # Create a state key value based on the attr_name. 595 if isinstance(ret_values[url_path][attr_name], unicode): 596 ret_values[url_path][attr_name] = \ 597 re.sub(r'.*\.', "", 598 ret_values[url_path][attr_name]) 599 # Do some key name manipulations. 600 new_attr_name = re.sub(r'^Current|(State|Transition)$', 601 "", attr_name) 602 new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name) 603 new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1', 604 new_attr_name) 605 new_attr_name = new_attr_name.lower().lstrip("_") 606 new_attr_name = re.sub(r'power', r'chassis', new_attr_name) 607 if new_attr_name in req_states: 608 state[new_attr_name] = ret_values[url_path][attr_name] 609 610 for sub_state in req_states: 611 if sub_state in state: 612 continue 613 if sub_state.startswith("os_"): 614 # We pass "os_" requests on to get_os_state. 615 continue 616 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" 617 exec(cmd_buf) 618 619 if os_host == "": 620 # The caller has not specified an os_host so as far as we're concerned, 621 # it doesn't exist. 622 return state 623 624 os_req_states = [sub_state for sub_state in req_states 625 if sub_state.startswith('os_')] 626 627 if len(os_req_states) > 0: 628 # The caller has specified an os_host and they have requested 629 # information on os substates. 630 631 # Based on the information gathered on bmc, we'll try to make a 632 # determination of whether the os is even up. We'll pass the result 633 # of that assessment to get_os_state to enhance performance. 634 os_up_match = DotDict() 635 for sub_state in master_os_up_match: 636 if sub_state in req_states: 637 os_up_match[sub_state] = master_os_up_match[sub_state] 638 os_up = compare_states(state, os_up_match) 639 os_state = get_os_state(os_host=os_host, 640 os_username=os_username, 641 os_password=os_password, 642 req_states=os_req_states, 643 os_up=os_up, 644 quiet=quiet) 645 # Append os_state dictionary to ours. 646 state.update(os_state) 647 648 return state 649 650 651def check_state(match_state, 652 invert=0, 653 print_string="", 654 openbmc_host="", 655 openbmc_username="", 656 openbmc_password="", 657 os_host="", 658 os_username="", 659 os_password="", 660 quiet=None): 661 r""" 662 Check that the Open BMC machine's composite state matches the specified 663 state. On success, this keyword returns the machine's composite state as a 664 dictionary. 665 666 Description of arguments: 667 match_state A dictionary whose key/value pairs are "state field"/ 668 "state value". The state value is interpreted as a 669 regular expression. Example call from robot: 670 ${match_state}= Create Dictionary chassis=^On$ 671 ... bmc=^Ready$ 672 ... boot_progress=^OSStart$ 673 ${state}= Check State &{match_state} 674 invert If this flag is set, this function will succeed if the 675 states do NOT match. 676 print_string This function will print this string to the console prior 677 to getting the state. 678 openbmc_host The DNS name or IP address of the BMC. 679 This defaults to global ${OPENBMC_HOST}. 680 openbmc_username The username to be used to login to the BMC. 681 This defaults to global ${OPENBMC_USERNAME}. 682 openbmc_password The password to be used to login to the BMC. 683 This defaults to global ${OPENBMC_PASSWORD}. 684 os_host The DNS name or IP address of the operating system. 685 This defaults to global ${OS_HOST}. 686 os_username The username to be used to login to the OS. 687 This defaults to global ${OS_USERNAME}. 688 os_password The password to be used to login to the OS. 689 This defaults to global ${OS_PASSWORD}. 690 quiet Indicates whether status details should be written to the 691 console. Defaults to either global value of ${QUIET} or 692 to 1. 693 """ 694 695 quiet = int(gp.get_var_value(quiet, 0)) 696 697 grp.rprint(print_string) 698 699 req_states = match_state.keys() 700 # Initialize state. 701 state = get_state(openbmc_host=openbmc_host, 702 openbmc_username=openbmc_username, 703 openbmc_password=openbmc_password, 704 os_host=os_host, 705 os_username=os_username, 706 os_password=os_password, 707 req_states=req_states, 708 quiet=quiet) 709 if not quiet: 710 gp.print_var(state) 711 712 match = compare_states(state, match_state) 713 714 if invert and match: 715 fail_msg = "The current state of the machine matches the match" +\ 716 " state:\n" + gp.sprint_varx("state", state) 717 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 718 elif not invert and not match: 719 fail_msg = "The current state of the machine does NOT match the" +\ 720 " match state:\n" +\ 721 gp.sprint_varx("state", state) 722 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 723 724 return state 725 726 727def wait_state(match_state=(), 728 wait_time="1 min", 729 interval="1 second", 730 invert=0, 731 openbmc_host="", 732 openbmc_username="", 733 openbmc_password="", 734 os_host="", 735 os_username="", 736 os_password="", 737 quiet=None): 738 r""" 739 Wait for the Open BMC machine's composite state to match the specified 740 state. On success, this keyword returns the machine's composite state as 741 a dictionary. 742 743 Description of arguments: 744 match_state A dictionary whose key/value pairs are "state field"/ 745 "state value". See check_state (above) for details. 746 This value may also be any string accepted by 747 return_state_constant (e.g. "standby_match_state"). 748 In such a case this function will call 749 return_state_constant to convert it to a proper 750 dictionary as described above. 751 wait_time The total amount of time to wait for the desired state. 752 This value may be expressed in Robot Framework's time 753 format (e.g. 1 minute, 2 min 3 s, 4.5). 754 interval The amount of time between state checks. 755 This value may be expressed in Robot Framework's time 756 format (e.g. 1 minute, 2 min 3 s, 4.5). 757 invert If this flag is set, this function will for the state of 758 the machine to cease to match the match state. 759 openbmc_host The DNS name or IP address of the BMC. 760 This defaults to global ${OPENBMC_HOST}. 761 openbmc_username The username to be used to login to the BMC. 762 This defaults to global ${OPENBMC_USERNAME}. 763 openbmc_password The password to be used to login to the BMC. 764 This defaults to global ${OPENBMC_PASSWORD}. 765 os_host The DNS name or IP address of the operating system. 766 This defaults to global ${OS_HOST}. 767 os_username The username to be used to login to the OS. 768 This defaults to global ${OS_USERNAME}. 769 os_password The password to be used to login to the OS. 770 This defaults to global ${OS_PASSWORD}. 771 quiet Indicates whether status details should be written to the 772 console. Defaults to either global value of ${QUIET} or 773 to 1. 774 """ 775 776 quiet = int(gp.get_var_value(quiet, 0)) 777 778 if type(match_state) in (str, unicode): 779 match_state = return_state_constant(match_state) 780 781 if not quiet: 782 if invert: 783 alt_text = "cease to " 784 else: 785 alt_text = "" 786 gp.print_timen("Checking every " + str(interval) + " for up to " 787 + str(wait_time) + " for the state of the machine to " 788 + alt_text + "match the state shown below.") 789 gp.print_var(match_state) 790 791 if quiet: 792 print_string = "" 793 else: 794 print_string = "#" 795 796 debug = int(BuiltIn().get_variable_value("${debug}", "0")) 797 if debug: 798 # In debug we print state so no need to print the "#". 799 print_string = "" 800 check_state_quiet = 1 - debug 801 cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}", 802 "print_string=" + print_string, "openbmc_host=" + openbmc_host, 803 "openbmc_username=" + openbmc_username, 804 "openbmc_password=" + openbmc_password, "os_host=" + os_host, 805 "os_username=" + os_username, "os_password=" + os_password, 806 "quiet=${" + str(check_state_quiet) + "}"] 807 grp.rdpissuing_keyword(cmd_buf) 808 try: 809 state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval, 810 *cmd_buf) 811 except AssertionError as my_assertion_error: 812 gp.printn() 813 message = my_assertion_error.args[0] 814 BuiltIn().fail(message) 815 816 if not quiet: 817 gp.printn() 818 if invert: 819 gp.print_timen("The states no longer match:") 820 else: 821 gp.print_timen("The states match:") 822 gp.print_var(state) 823 824 return state 825 826 827def wait_for_comm_cycle(start_boot_seconds, 828 quiet=None): 829 r""" 830 Wait for communications to the BMC to stop working and then resume working. 831 This function is useful when you have initiated some kind of reboot. 832 833 Description of arguments: 834 start_boot_seconds The time that the boot test started. The format is the 835 epoch time in seconds, i.e. the number of seconds since 836 1970-01-01 00:00:00 UTC. This value should be obtained 837 from the BMC so that it is not dependent on any kind of 838 synchronization between this machine and the target BMC 839 This will allow this program to work correctly even in 840 a simulated environment. This value should be obtained 841 by the caller prior to initiating a reboot. It can be 842 obtained as follows: 843 state = st.get_state(req_states=['epoch_seconds']) 844 """ 845 846 quiet = int(gp.get_var_value(quiet, 0)) 847 848 # Validate parms. 849 error_message = gv.svalid_integer(start_boot_seconds, 850 var_name="start_boot_seconds") 851 if error_message != "": 852 BuiltIn().fail(gp.sprint_error(error_message)) 853 854 match_state = anchor_state(DotDict([('packet_loss', '100')])) 855 # Wait for 100% packet loss trying to ping machine. 856 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 857 858 match_state['packet_loss'] = '^0$' 859 # Wait for 0% packet loss trying to ping machine. 860 wait_state(match_state, wait_time="8 mins", interval="0 seconds") 861 862 # Get the uptime and epoch seconds for comparisons. We want to be sure 863 # that the uptime is less than the elapsed boot time. Further proof that 864 # a reboot has indeed occurred (vs random network instability giving a 865 # false positive. We also use wait_state because the BMC may take a short 866 # while to be ready to process SSH requests. 867 match_state = DotDict([('uptime', '^[0-9\\.]+$'), 868 ('epoch_seconds', '^[0-9]+$')]) 869 state = wait_state(match_state, wait_time="2 mins", interval="1 second") 870 871 elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds 872 gp.qprint_var(elapsed_boot_time) 873 if state['uptime'] == "": 874 error_message = "Unable to obtain uptime from the BMC. BMC is not" +\ 875 " communicating." 876 BuiltIn().fail(gp.sprint_error(error_message)) 877 if int(float(state['uptime'])) < elapsed_boot_time: 878 uptime = state['uptime'] 879 gp.qprint_var(uptime) 880 gp.qprint_timen("The uptime is less than the elapsed boot time," 881 + " as expected.") 882 else: 883 error_message = "The uptime is greater than the elapsed boot time," +\ 884 " which is unexpected:\n" +\ 885 gp.sprint_var(start_boot_seconds) +\ 886 gp.sprint_var(state) 887 BuiltIn().fail(gp.sprint_error(error_message)) 888 889 gp.qprint_timen("Verifying that REST API interface is working.") 890 match_state = DotDict([('rest', '^1$')]) 891 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds") 892