1#!/usr/bin/env python3 2 3r""" 4This module contains functions having to do with machine state: get_state, 5check_state, wait_state, etc. 6 7The 'State' is a composite of many pieces of data. Therefore, the functions 8in this module define state as an ordered dictionary. Here is an example of 9some test output showing machine state: 10 11default_state: 12 default_state[chassis]: On 13 default_state[boot_progress]: OSStart 14 default_state[operating_system]: BootComplete 15 default_state[host]: Running 16 default_state[os_ping]: 1 17 default_state[os_login]: 1 18 default_state[os_run_cmd]: 1 19 20Different users may very well have different needs when inquiring about 21state. Support for new pieces of state information may be added to this 22module as needed. 23 24By using the wait_state function, a caller can start a boot and then wait for 25a precisely defined state to indicate that the boot has succeeded. If 26the boot fails, they can see exactly why by looking at the current state as 27compared with the expected state. 28""" 29 30import importlib.util 31import os 32import re 33import sys 34 35import bmc_ssh_utils as bsu 36import gen_cmd as gc 37import gen_print as gp 38import gen_robot_utils as gru 39import gen_valid as gv 40from robot.libraries.BuiltIn import BuiltIn 41from robot.utils import DotDict 42 43# NOTE: Avoid importing utils.robot because utils.robot imports state.py 44# (indirectly) which will cause failures. 45gru.my_import_resource("rest_client.robot") 46 47base_path = ( 48 os.path.dirname( 49 os.path.dirname(importlib.util.find_spec("gen_robot_print").origin) 50 ) 51 + os.sep 52) 53sys.path.append(base_path + "data/") 54 55# Previously, I had this coded: 56# import variables as var 57# However, we ran into a problem where a robot program did this... 58# Variables ../../lib/ras/variables.py 59# Prior to doing this... 60# Library ../lib/state.py 61 62# This caused the wrong variables.py file to be selected. Attempts to fix this 63# have failed so far. For the moment, we will hard-code the value we need from 64# the file. 65 66SYSTEM_STATE_URI = "/xyz/openbmc_project/state/" 67 68# The BMC code has recently been changed as far as what states are defined and 69# what the state values can be. This module now has a means of processing both 70# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e. 71# OBMC_STATES_VERSION = 1). 72# The caller can set environment variable OBMC_STATES_VERSION to dictate 73# whether we're processing old or new style states. If OBMC_STATES_VERSION is 74# not set it will default to 1. 75 76# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old 77# that it is no longer worthwhile to maintain. The OBMC_STATES_VERSION 0 code 78# is being removed but the OBMC_STATES_VERSION value will stay for now in the 79# event that it is needed in the future. 80 81OBMC_STATES_VERSION = int(os.environ.get("OBMC_STATES_VERSION", 1)) 82 83redfish_support_trans_state = int( 84 os.environ.get("REDFISH_SUPPORT_TRANS_STATE", 0) 85) or int( 86 BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0) 87) 88 89platform_arch_type = os.environ.get( 90 "PLATFORM_ARCH_TYPE", "" 91) or BuiltIn().get_variable_value("${PLATFORM_ARCH_TYPE}", default="power") 92 93# valid_os_req_states and default_os_req_states are used by the os_get_state 94# function. 95# valid_os_req_states is a list of state information supported by the 96# get_os_state function. 97valid_os_req_states = ["os_ping", "os_login", "os_run_cmd"] 98 99# When a user calls get_os_state w/o specifying req_states, 100# default_os_req_states is used as its value. 101default_os_req_states = ["os_ping", "os_login", "os_run_cmd"] 102 103# Presently, some BMCs appear to not keep time very well. This environment 104# variable directs the get_state function to use either the BMC's epoch time 105# or the local epoch time. 106USE_BMC_EPOCH_TIME = int(os.environ.get("USE_BMC_EPOCH_TIME", 0)) 107 108# Useful state constant definition(s). 109if not redfish_support_trans_state: 110 # When a user calls get_state w/o specifying req_states, default_req_states 111 # is used as its value. 112 default_req_states = [ 113 "rest", 114 "chassis", 115 "bmc", 116 "boot_progress", 117 "operating_system", 118 "host", 119 "os_ping", 120 "os_login", 121 "os_run_cmd", 122 ] 123 124 # valid_req_states is a list of sub states supported by the get_state function. 125 # valid_req_states, default_req_states and master_os_up_match are used by the 126 # get_state function. 127 128 valid_req_states = [ 129 "ping", 130 "packet_loss", 131 "uptime", 132 "epoch_seconds", 133 "elapsed_boot_time", 134 "rest", 135 "chassis", 136 "requested_chassis", 137 "bmc", 138 "requested_bmc", 139 "boot_progress", 140 "operating_system", 141 "host", 142 "requested_host", 143 "attempts_left", 144 "os_ping", 145 "os_login", 146 "os_run_cmd", 147 ] 148 149 # default_state is an initial value which may be of use to callers. 150 default_state = DotDict( 151 [ 152 ("rest", "1"), 153 ("chassis", "On"), 154 ("bmc", "Ready"), 155 ("boot_progress", "OSStart"), 156 ("operating_system", "BootComplete"), 157 ("host", "Running"), 158 ("os_ping", "1"), 159 ("os_login", "1"), 160 ("os_run_cmd", "1"), 161 ] 162 ) 163 164 # A match state for checking that the system is at "standby". 165 standby_match_state = DotDict( 166 [ 167 ("rest", "^1$"), 168 ("chassis", "^Off$"), 169 ("bmc", "^Ready$"), 170 ("boot_progress", "^Off|Unspecified$"), 171 ("operating_system", "^Inactive$"), 172 ("host", "^Off$"), 173 ] 174 ) 175 176 # A match state for checking that the system is at "os running". 177 os_running_match_state = DotDict( 178 [ 179 ("chassis", "^On$"), 180 ("bmc", "^Ready$"), 181 ("boot_progress", "FW Progress, Starting OS|OSStart"), 182 ("operating_system", "BootComplete"), 183 ("host", "^Running$"), 184 ("os_ping", "^1$"), 185 ("os_login", "^1$"), 186 ("os_run_cmd", "^1$"), 187 ] 188 ) 189 190 # A master dictionary to determine whether the os may be up. 191 master_os_up_match = DotDict( 192 [ 193 ("chassis", "^On$"), 194 ("bmc", "^Ready$"), 195 ("boot_progress", "FW Progress, Starting OS|OSStart"), 196 ("operating_system", "BootComplete"), 197 ("host", "^Running|Quiesced$"), 198 ] 199 ) 200 201 invalid_state_match = DotDict( 202 [ 203 ("rest", "^$"), 204 ("chassis", "^$"), 205 ("bmc", "^$"), 206 ("boot_progress", "^$"), 207 ("operating_system", "^$"), 208 ("host", "^$"), 209 ] 210 ) 211else: 212 # When a user calls get_state w/o specifying req_states, default_req_states 213 # is used as its value. 214 default_req_states = [ 215 "redfish", 216 "chassis", 217 "bmc", 218 "boot_progress", 219 "host", 220 "os_ping", 221 "os_login", 222 "os_run_cmd", 223 ] 224 225 # valid_req_states is a list of sub states supported by the get_state function. 226 # valid_req_states, default_req_states and master_os_up_match are used by the 227 # get_state function. 228 229 valid_req_states = [ 230 "ping", 231 "packet_loss", 232 "uptime", 233 "epoch_seconds", 234 "elapsed_boot_time", 235 "redfish", 236 "chassis", 237 "requested_chassis", 238 "bmc", 239 "requested_bmc", 240 "boot_progress", 241 "host", 242 "requested_host", 243 "attempts_left", 244 "os_ping", 245 "os_login", 246 "os_run_cmd", 247 ] 248 249 # default_state is an initial value which may be of use to callers. 250 default_state = DotDict( 251 [ 252 ("redfish", "1"), 253 ("chassis", "On"), 254 ("bmc", "Enabled"), 255 ( 256 "boot_progress", 257 "SystemHardwareInitializationComplete|OSBootStarted|OSRunning", 258 ), 259 ("host", "Enabled"), 260 ("os_ping", "1"), 261 ("os_login", "1"), 262 ("os_run_cmd", "1"), 263 ] 264 ) 265 266 # A match state for checking that the system is at "standby". 267 standby_match_state = DotDict( 268 [ 269 ("redfish", "^1$"), 270 ("chassis", "^Off$"), 271 ("bmc", "^Enabled$"), 272 ("boot_progress", "^None$"), 273 ("host", "^Disabled$"), 274 ] 275 ) 276 277 # A match state for checking that the system is at "os running". 278 os_running_match_state = DotDict( 279 [ 280 ("chassis", "^On$"), 281 ("bmc", "^Enabled$"), 282 ( 283 "boot_progress", 284 "SystemHardwareInitializationComplete|OSBootStarted|OSRunning", 285 ), 286 ("host", "^Enabled$"), 287 ("os_ping", "^1$"), 288 ("os_login", "^1$"), 289 ("os_run_cmd", "^1$"), 290 ] 291 ) 292 293 # A master dictionary to determine whether the os may be up. 294 master_os_up_match = DotDict( 295 [ 296 ("chassis", "^On$"), 297 ("bmc", "^Enabled$"), 298 ( 299 "boot_progress", 300 "SystemHardwareInitializationComplete|OSBootStarted|OSRunning", 301 ), 302 ("host", "^Enabled$"), 303 ] 304 ) 305 306 invalid_state_match = DotDict( 307 [ 308 ("redfish", "^$"), 309 ("chassis", "^$"), 310 ("bmc", "^$"), 311 ("boot_progress", "^$"), 312 ("host", "^$"), 313 ] 314 ) 315 316# Filter the states based on platform type. 317if platform_arch_type == "x86": 318 if not redfish_support_trans_state: 319 default_req_states.remove("operating_system") 320 valid_req_states.remove("operating_system") 321 del default_state["operating_system"] 322 del standby_match_state["operating_system"] 323 del os_running_match_state["operating_system"] 324 del master_os_up_match["operating_system"] 325 del invalid_state_match["operating_system"] 326 327 default_req_states.remove("boot_progress") 328 valid_req_states.remove("boot_progress") 329 del default_state["boot_progress"] 330 del standby_match_state["boot_progress"] 331 del os_running_match_state["boot_progress"] 332 del master_os_up_match["boot_progress"] 333 del invalid_state_match["boot_progress"] 334 335 336def return_state_constant(state_name="default_state"): 337 r""" 338 Return the named state dictionary constant. 339 """ 340 341 return eval(state_name) 342 343 344def anchor_state(state): 345 r""" 346 Add regular expression anchors ("^" and "$") to the beginning and end of 347 each item in the state dictionary passed in. Return the resulting 348 dictionary. 349 350 Description of argument(s): 351 state A dictionary such as the one returned by the get_state() 352 function. 353 """ 354 355 anchored_state = state.copy() 356 for key in anchored_state.keys(): 357 anchored_state[key] = "^" + str(anchored_state[key]) + "$" 358 359 return anchored_state 360 361 362def strip_anchor_state(state): 363 r""" 364 Strip regular expression anchors ("^" and "$") from the beginning and end 365 of each item in the state dictionary passed in. Return the resulting 366 dictionary. 367 368 Description of argument(s): 369 state A dictionary such as the one returned by the get_state() 370 function. 371 """ 372 373 stripped_state = state.copy() 374 for key in stripped_state.keys(): 375 stripped_state[key] = stripped_state[key].strip("^$") 376 377 return stripped_state 378 379 380def expressions_key(): 381 r""" 382 Return expressions key constant. 383 """ 384 return "<expressions>" 385 386 387def compare_states(state, match_state, match_type="and"): 388 r""" 389 Compare 2 state dictionaries. Return True if they match and False if they 390 don't. Note that the match_state dictionary does not need to have an entry 391 corresponding to each entry in the state dictionary. But for each entry 392 that it does have, the corresponding state entry will be checked for a 393 match. 394 395 Description of argument(s): 396 state A state dictionary such as the one returned by the 397 get_state function. 398 match_state A dictionary whose key/value pairs are "state field"/ 399 "state value". The state value is interpreted as a 400 regular expression. Every value in this dictionary is 401 considered. When match_type is 'and', if each and every 402 comparison matches, the two dictionaries are considered to 403 be matching. If match_type is 'or', if any two of the 404 elements compared match, the two dictionaries are 405 considered to be matching. 406 407 This value may also be any string accepted by 408 return_state_constant (e.g. "standby_match_state"). In 409 such a case this function will call return_state_constant 410 to convert it to a proper dictionary as described above. 411 412 Finally, one special value is accepted for the key field: 413 expression_key(). If such an entry exists, its value is 414 taken to be a list of expressions to be evaluated. These 415 expressions may reference state dictionary entries by 416 simply coding them in standard python syntax (e.g. 417 state['key1']). What follows is an example expression: 418 419 "int(float(state['uptime'])) < int(state['elapsed_boot_time'])" 420 421 In this example, if the state dictionary's 'uptime' entry 422 is less than its 'elapsed_boot_time' entry, it would 423 qualify as a match. 424 match_type This may be 'and' or 'or'. 425 """ 426 427 error_message = gv.valid_value(match_type, valid_values=["and", "or"]) 428 if error_message != "": 429 BuiltIn().fail(gp.sprint_error(error_message)) 430 431 try: 432 match_state = return_state_constant(match_state) 433 except TypeError: 434 pass 435 436 default_match = match_type == "and" 437 for key, match_state_value in match_state.items(): 438 # Blank match_state_value means "don't care". 439 if match_state_value == "": 440 continue 441 if key == expressions_key(): 442 for expr in match_state_value: 443 # Use python interpreter to evaluate the expression. 444 match = eval(expr) 445 if match != default_match: 446 return match 447 else: 448 try: 449 match = ( 450 re.match(match_state_value, str(state[key])) is not None 451 ) 452 except KeyError: 453 match = False 454 if match != default_match: 455 return match 456 457 return default_match 458 459 460def get_os_state( 461 os_host="", 462 os_username="", 463 os_password="", 464 req_states=default_os_req_states, 465 os_up=True, 466 quiet=None, 467): 468 r""" 469 Get component states for the operating system such as ping, login, 470 etc, put them into a dictionary and return them to the caller. 471 472 Note that all substate values are strings. 473 474 Description of argument(s): 475 os_host The DNS name or IP address of the operating system. 476 This defaults to global ${OS_HOST}. 477 os_username The username to be used to login to the OS. 478 This defaults to global ${OS_USERNAME}. 479 os_password The password to be used to login to the OS. 480 This defaults to global ${OS_PASSWORD}. 481 req_states This is a list of states whose values are being requested by 482 the caller. 483 os_up If the caller knows that the os can't possibly be up, it can 484 improve performance by passing os_up=False. This function 485 will then simply return default values for all requested os 486 sub states. 487 quiet Indicates whether status details (e.g. curl commands) should 488 be written to the console. 489 Defaults to either global value of ${QUIET} or to 1. 490 """ 491 492 quiet = int(gp.get_var_value(quiet, 0)) 493 494 # Set parm defaults where necessary and validate all parms. 495 if os_host == "": 496 os_host = BuiltIn().get_variable_value("${OS_HOST}") 497 error_message = gv.valid_value(os_host, invalid_values=[None, ""]) 498 if error_message != "": 499 BuiltIn().fail(gp.sprint_error(error_message)) 500 501 if os_username == "": 502 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 503 error_message = gv.valid_value(os_username, invalid_values=[None, ""]) 504 if error_message != "": 505 BuiltIn().fail(gp.sprint_error(error_message)) 506 507 if os_password == "": 508 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 509 error_message = gv.valid_value(os_password, invalid_values=[None, ""]) 510 if error_message != "": 511 BuiltIn().fail(gp.sprint_error(error_message)) 512 513 invalid_req_states = [ 514 sub_state 515 for sub_state in req_states 516 if sub_state not in valid_os_req_states 517 ] 518 if len(invalid_req_states) > 0: 519 error_message = ( 520 "The following req_states are not supported:\n" 521 + gp.sprint_var(invalid_req_states) 522 ) 523 BuiltIn().fail(gp.sprint_error(error_message)) 524 525 # Initialize all substate values supported by this function. 526 os_ping = 0 527 os_login = 0 528 os_run_cmd = 0 529 530 if os_up: 531 if "os_ping" in req_states: 532 # See if the OS pings. 533 rc, out_buf = gc.shell_cmd( 534 "ping -c 1 -w 2 " + os_host, 535 print_output=0, 536 show_err=0, 537 ignore_err=1, 538 ) 539 if rc == 0: 540 os_ping = 1 541 542 # Programming note: All attributes which do not require an ssh login 543 # should have been processed by this point. 544 master_req_login = ["os_login", "os_run_cmd"] 545 req_login = [ 546 sub_state 547 for sub_state in req_states 548 if sub_state in master_req_login 549 ] 550 must_login = len(req_login) > 0 551 552 if must_login: 553 output, stderr, rc = bsu.os_execute_command( 554 "uptime", 555 quiet=quiet, 556 ignore_err=1, 557 time_out=20, 558 os_host=os_host, 559 os_username=os_username, 560 os_password=os_password, 561 ) 562 if rc == 0: 563 os_login = 1 564 os_run_cmd = 1 565 else: 566 gp.dprint_vars(output, stderr) 567 gp.dprint_vars(rc, 1) 568 569 os_state = DotDict() 570 for sub_state in req_states: 571 cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")" 572 exec(cmd_buf) 573 574 return os_state 575 576 577def get_state( 578 openbmc_host="", 579 openbmc_username="", 580 openbmc_password="", 581 os_host="", 582 os_username="", 583 os_password="", 584 req_states=default_req_states, 585 quiet=None, 586): 587 r""" 588 Get component states such as chassis state, bmc state, etc, put them into a 589 dictionary and return them to the caller. 590 591 Note that all substate values are strings. 592 593 Note: If elapsed_boot_time is included in req_states, it is the caller's 594 duty to call set_start_boot_seconds() in order to set global 595 start_boot_seconds. elapsed_boot_time is the current time minus 596 start_boot_seconds. 597 598 Description of argument(s): 599 openbmc_host The DNS name or IP address of the BMC. 600 This defaults to global ${OPENBMC_HOST}. 601 openbmc_username The username to be used to login to the BMC. 602 This defaults to global ${OPENBMC_USERNAME}. 603 openbmc_password The password to be used to login to the BMC. 604 This defaults to global ${OPENBMC_PASSWORD}. 605 os_host The DNS name or IP address of the operating system. 606 This defaults to global ${OS_HOST}. 607 os_username The username to be used to login to the OS. 608 This defaults to global ${OS_USERNAME}. 609 os_password The password to be used to login to the OS. 610 This defaults to global ${OS_PASSWORD}. 611 req_states This is a list of states whose values are being requested 612 by the caller. 613 quiet Indicates whether status details (e.g. curl commands) 614 should be written to the console. 615 Defaults to either global value of ${QUIET} or to 1. 616 """ 617 618 quiet = int(gp.get_var_value(quiet, 0)) 619 620 # Set parm defaults where necessary and validate all parms. 621 if openbmc_host == "": 622 openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}") 623 error_message = gv.valid_value(openbmc_host, invalid_values=[None, ""]) 624 if error_message != "": 625 BuiltIn().fail(gp.sprint_error(error_message)) 626 627 if openbmc_username == "": 628 openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}") 629 error_message = gv.valid_value(openbmc_username, invalid_values=[None, ""]) 630 if error_message != "": 631 BuiltIn().fail(gp.sprint_error(error_message)) 632 633 if openbmc_password == "": 634 openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}") 635 error_message = gv.valid_value(openbmc_password, invalid_values=[None, ""]) 636 if error_message != "": 637 BuiltIn().fail(gp.sprint_error(error_message)) 638 639 # NOTE: OS parms are optional. 640 if os_host == "": 641 os_host = BuiltIn().get_variable_value("${OS_HOST}") 642 if os_host is None: 643 os_host = "" 644 645 if os_username == "": 646 os_username = BuiltIn().get_variable_value("${OS_USERNAME}") 647 if os_username is None: 648 os_username = "" 649 650 if os_password == "": 651 os_password = BuiltIn().get_variable_value("${OS_PASSWORD}") 652 if os_password is None: 653 os_password = "" 654 655 invalid_req_states = [ 656 sub_state 657 for sub_state in req_states 658 if sub_state not in valid_req_states 659 ] 660 if len(invalid_req_states) > 0: 661 error_message = ( 662 "The following req_states are not supported:\n" 663 + gp.sprint_var(invalid_req_states) 664 ) 665 BuiltIn().fail(gp.sprint_error(error_message)) 666 667 # Initialize all substate values supported by this function. 668 ping = 0 669 packet_loss = "" 670 uptime = "" 671 epoch_seconds = "" 672 elapsed_boot_time = "" 673 rest = "" 674 redfish = "" 675 chassis = "" 676 requested_chassis = "" 677 bmc = "" 678 requested_bmc = "" 679 # BootProgress state will get populated when state logic enumerates the 680 # state URI. This is to prevent state dictionary boot_progress value 681 # getting empty when the BootProgress is NOT found, making it optional. 682 boot_progress = "NA" 683 operating_system = "" 684 host = "" 685 requested_host = "" 686 attempts_left = "" 687 688 # Get the component states. 689 if "ping" in req_states: 690 # See if the OS pings. 691 rc, out_buf = gc.shell_cmd( 692 "ping -c 1 -w 2 " + openbmc_host, 693 print_output=0, 694 show_err=0, 695 ignore_err=1, 696 ) 697 if rc == 0: 698 ping = 1 699 700 if "packet_loss" in req_states: 701 # See if the OS pings. 702 cmd_buf = ( 703 "ping -c 5 -w 5 " 704 + openbmc_host 705 + " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'" 706 ) 707 rc, out_buf = gc.shell_cmd( 708 cmd_buf, print_output=0, show_err=0, ignore_err=1 709 ) 710 if rc == 0: 711 packet_loss = out_buf.rstrip("\n") 712 713 if "uptime" in req_states: 714 # Sometimes reading uptime results in a blank value. Call with 715 # wait_until_keyword_succeeds to ensure a non-blank value is obtained. 716 remote_cmd_buf = ( 717 "bash -c 'read uptime filler 2>/dev/null < /proc/uptime" 718 + ' && [ ! -z "${uptime}" ] && echo ${uptime}\'' 719 ) 720 cmd_buf = [ 721 "BMC Execute Command", 722 re.sub("\\$", "\\$", remote_cmd_buf), 723 "quiet=1", 724 "test_mode=0", 725 "time_out=5", 726 ] 727 gp.qprint_issuing(cmd_buf, 0) 728 gp.qprint_issuing(remote_cmd_buf, 0) 729 try: 730 stdout, stderr, rc = BuiltIn().wait_until_keyword_succeeds( 731 "10 sec", "5 sec", *cmd_buf 732 ) 733 if rc == 0 and stderr == "": 734 uptime = stdout 735 except AssertionError as my_assertion_error: 736 pass 737 738 if "epoch_seconds" in req_states or "elapsed_boot_time" in req_states: 739 date_cmd_buf = "date -u +%s" 740 if USE_BMC_EPOCH_TIME: 741 cmd_buf = ["BMC Execute Command", date_cmd_buf, "quiet=${1}"] 742 if not quiet: 743 gp.print_issuing(cmd_buf) 744 status, ret_values = BuiltIn().run_keyword_and_ignore_error( 745 *cmd_buf 746 ) 747 if status == "PASS": 748 stdout, stderr, rc = ret_values 749 if rc == 0 and stderr == "": 750 epoch_seconds = stdout.rstrip("\n") 751 else: 752 shell_rc, out_buf = gc.cmd_fnc_u( 753 date_cmd_buf, quiet=quiet, print_output=0 754 ) 755 if shell_rc == 0: 756 epoch_seconds = out_buf.rstrip("\n") 757 758 if "elapsed_boot_time" in req_states: 759 global start_boot_seconds 760 elapsed_boot_time = int(epoch_seconds) - start_boot_seconds 761 762 if not redfish_support_trans_state: 763 master_req_rest = [ 764 "rest", 765 "host", 766 "requested_host", 767 "operating_system", 768 "attempts_left", 769 "boot_progress", 770 "chassis", 771 "requested_chassisbmcrequested_bmc", 772 ] 773 774 req_rest = [ 775 sub_state 776 for sub_state in req_states 777 if sub_state in master_req_rest 778 ] 779 need_rest = len(req_rest) > 0 780 state = DotDict() 781 if need_rest: 782 cmd_buf = [ 783 "Read Properties", 784 SYSTEM_STATE_URI + "enumerate", 785 "quiet=${" + str(quiet) + "}", 786 "timeout=30", 787 ] 788 gp.dprint_issuing(cmd_buf) 789 status, ret_values = BuiltIn().run_keyword_and_ignore_error( 790 *cmd_buf 791 ) 792 if status == "PASS": 793 state["rest"] = "1" 794 else: 795 state["rest"] = "0" 796 797 if int(state["rest"]): 798 for url_path in ret_values: 799 # Skip conflicting "CurrentHostState" URL from the enum 800 # /xyz/openbmc_project/state/hypervisor0 801 if "hypervisor0" in url_path: 802 continue 803 804 if platform_arch_type == "x86": 805 # Skip conflicting "CurrentPowerState" URL from the enum 806 # /xyz/openbmc_project/state/chassis_system0 807 if "chassis_system0" in url_path: 808 continue 809 810 for attr_name in ret_values[url_path]: 811 # Create a state key value based on the attr_name. 812 try: 813 ret_values[url_path][attr_name] = re.sub( 814 r".*\.", "", ret_values[url_path][attr_name] 815 ) 816 except TypeError: 817 pass 818 # Do some key name manipulations. 819 new_attr_name = re.sub( 820 r"^Current|(State|Transition)$", "", attr_name 821 ) 822 new_attr_name = re.sub(r"BMC", r"Bmc", new_attr_name) 823 new_attr_name = re.sub( 824 r"([A-Z][a-z])", r"_\1", new_attr_name 825 ) 826 new_attr_name = new_attr_name.lower().lstrip("_") 827 new_attr_name = re.sub( 828 r"power", r"chassis", new_attr_name 829 ) 830 if new_attr_name in req_states: 831 state[new_attr_name] = ret_values[url_path][ 832 attr_name 833 ] 834 else: 835 master_req_rf = [ 836 "redfish", 837 "host", 838 "requested_host", 839 "attempts_left", 840 "boot_progress", 841 "chassis", 842 "requested_chassisbmcrequested_bmc", 843 ] 844 845 req_rf = [ 846 sub_state for sub_state in req_states if sub_state in master_req_rf 847 ] 848 need_rf = len(req_rf) > 0 849 state = DotDict() 850 if need_rf: 851 cmd_buf = ["Redfish Get States"] 852 gp.dprint_issuing(cmd_buf) 853 try: 854 status, ret_values = BuiltIn().run_keyword_and_ignore_error( 855 *cmd_buf 856 ) 857 except Exception as ex: 858 # Robot raised UserKeywordExecutionFailed error exception. 859 gp.dprint_issuing("Retrying Redfish Get States") 860 status, ret_values = BuiltIn().run_keyword_and_ignore_error( 861 *cmd_buf 862 ) 863 864 gp.dprint_vars(status, ret_values) 865 if status == "PASS": 866 state["redfish"] = "1" 867 else: 868 state["redfish"] = "0" 869 870 if int(state["redfish"]): 871 state["chassis"] = ret_values["chassis"] 872 state["host"] = ret_values["host"] 873 state["bmc"] = ret_values["bmc"] 874 if platform_arch_type != "x86": 875 state["boot_progress"] = ret_values["boot_progress"] 876 877 for sub_state in req_states: 878 if sub_state in state: 879 continue 880 if sub_state.startswith("os_"): 881 # We pass "os_" requests on to get_os_state. 882 continue 883 cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")" 884 exec(cmd_buf) 885 886 if os_host == "": 887 # The caller has not specified an os_host so as far as we're concerned, 888 # it doesn't exist. 889 return state 890 891 os_req_states = [ 892 sub_state for sub_state in req_states if sub_state.startswith("os_") 893 ] 894 895 if len(os_req_states) > 0: 896 # The caller has specified an os_host and they have requested 897 # information on os substates. 898 899 # Based on the information gathered on bmc, we'll try to make a 900 # determination of whether the os is even up. We'll pass the result 901 # of that assessment to get_os_state to enhance performance. 902 os_up_match = DotDict() 903 for sub_state in master_os_up_match: 904 if sub_state in req_states: 905 os_up_match[sub_state] = master_os_up_match[sub_state] 906 os_up = compare_states(state, os_up_match) 907 os_state = get_os_state( 908 os_host=os_host, 909 os_username=os_username, 910 os_password=os_password, 911 req_states=os_req_states, 912 os_up=os_up, 913 quiet=quiet, 914 ) 915 # Append os_state dictionary to ours. 916 state.update(os_state) 917 918 return state 919 920 921exit_wait_early_message = "" 922 923 924def set_exit_wait_early_message(value): 925 r""" 926 Set global exit_wait_early_message to the indicated value. 927 928 This is a mechanism by which the programmer can do an early exit from 929 wait_until_keyword_succeeds() based on some special condition. 930 931 Description of argument(s): 932 value The value to assign to the global 933 exit_wait_early_message. 934 """ 935 936 global exit_wait_early_message 937 exit_wait_early_message = value 938 939 940def check_state( 941 match_state, 942 invert=0, 943 print_string="", 944 openbmc_host="", 945 openbmc_username="", 946 openbmc_password="", 947 os_host="", 948 os_username="", 949 os_password="", 950 quiet=None, 951): 952 r""" 953 Check that the Open BMC machine's composite state matches the specified 954 state. On success, this keyword returns the machine's composite state as a 955 dictionary. 956 957 Description of argument(s): 958 match_state A dictionary whose key/value pairs are "state field"/ 959 "state value". The state value is interpreted as a 960 regular expression. Example call from robot: 961 ${match_state}= Create Dictionary chassis=^On$ 962 ... bmc=^Ready$ 963 ... boot_progress=^OSStart$ 964 ${state}= Check State &{match_state} 965 invert If this flag is set, this function will succeed if the 966 states do NOT match. 967 print_string This function will print this string to the console prior 968 to getting the state. 969 openbmc_host The DNS name or IP address of the BMC. 970 This defaults to global ${OPENBMC_HOST}. 971 openbmc_username The username to be used to login to the BMC. 972 This defaults to global ${OPENBMC_USERNAME}. 973 openbmc_password The password to be used to login to the BMC. 974 This defaults to global ${OPENBMC_PASSWORD}. 975 os_host The DNS name or IP address of the operating system. 976 This defaults to global ${OS_HOST}. 977 os_username The username to be used to login to the OS. 978 This defaults to global ${OS_USERNAME}. 979 os_password The password to be used to login to the OS. 980 This defaults to global ${OS_PASSWORD}. 981 quiet Indicates whether status details should be written to the 982 console. Defaults to either global value of ${QUIET} or 983 to 1. 984 """ 985 986 quiet = int(gp.get_var_value(quiet, 0)) 987 988 gp.gp_print(print_string) 989 990 try: 991 match_state = return_state_constant(match_state) 992 except TypeError: 993 pass 994 995 req_states = list(match_state.keys()) 996 # Remove special-case match key from req_states. 997 if expressions_key() in req_states: 998 req_states.remove(expressions_key()) 999 # Initialize state. 1000 state = get_state( 1001 openbmc_host=openbmc_host, 1002 openbmc_username=openbmc_username, 1003 openbmc_password=openbmc_password, 1004 os_host=os_host, 1005 os_username=os_username, 1006 os_password=os_password, 1007 req_states=req_states, 1008 quiet=quiet, 1009 ) 1010 if not quiet: 1011 gp.print_var(state) 1012 1013 if exit_wait_early_message != "": 1014 # The exit_wait_early_message has been set by a signal handler so we 1015 # will exit "successfully". It is incumbent upon the calling function 1016 # (e.g. wait_state) to check/clear this variable and to fail 1017 # appropriately. 1018 return state 1019 1020 match = compare_states(state, match_state) 1021 1022 if invert and match: 1023 fail_msg = ( 1024 "The current state of the machine matches the match" 1025 + " state:\n" 1026 + gp.sprint_varx("state", state) 1027 ) 1028 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 1029 elif not invert and not match: 1030 fail_msg = ( 1031 "The current state of the machine does NOT match the" 1032 + " match state:\n" 1033 + gp.sprint_varx("state", state) 1034 ) 1035 BuiltIn().fail("\n" + gp.sprint_error(fail_msg)) 1036 1037 return state 1038 1039 1040def wait_state( 1041 match_state=(), 1042 wait_time="1 min", 1043 interval="1 second", 1044 invert=0, 1045 openbmc_host="", 1046 openbmc_username="", 1047 openbmc_password="", 1048 os_host="", 1049 os_username="", 1050 os_password="", 1051 quiet=None, 1052): 1053 r""" 1054 Wait for the Open BMC machine's composite state to match the specified 1055 state. On success, this keyword returns the machine's composite state as 1056 a dictionary. 1057 1058 Description of argument(s): 1059 match_state A dictionary whose key/value pairs are "state field"/ 1060 "state value". See check_state (above) for details. 1061 This value may also be any string accepted by 1062 return_state_constant (e.g. "standby_match_state"). 1063 In such a case this function will call 1064 return_state_constant to convert it to a proper 1065 dictionary as described above. 1066 wait_time The total amount of time to wait for the desired state. 1067 This value may be expressed in Robot Framework's time 1068 format (e.g. 1 minute, 2 min 3 s, 4.5). 1069 interval The amount of time between state checks. 1070 This value may be expressed in Robot Framework's time 1071 format (e.g. 1 minute, 2 min 3 s, 4.5). 1072 invert If this flag is set, this function will for the state of 1073 the machine to cease to match the match state. 1074 openbmc_host The DNS name or IP address of the BMC. 1075 This defaults to global ${OPENBMC_HOST}. 1076 openbmc_username The username to be used to login to the BMC. 1077 This defaults to global ${OPENBMC_USERNAME}. 1078 openbmc_password The password to be used to login to the BMC. 1079 This defaults to global ${OPENBMC_PASSWORD}. 1080 os_host The DNS name or IP address of the operating system. 1081 This defaults to global ${OS_HOST}. 1082 os_username The username to be used to login to the OS. 1083 This defaults to global ${OS_USERNAME}. 1084 os_password The password to be used to login to the OS. 1085 This defaults to global ${OS_PASSWORD}. 1086 quiet Indicates whether status details should be written to the 1087 console. Defaults to either global value of ${QUIET} or 1088 to 1. 1089 """ 1090 1091 quiet = int(gp.get_var_value(quiet, 0)) 1092 1093 try: 1094 match_state = return_state_constant(match_state) 1095 except TypeError: 1096 pass 1097 1098 if not quiet: 1099 if invert: 1100 alt_text = "cease to " 1101 else: 1102 alt_text = "" 1103 gp.print_timen( 1104 "Checking every " 1105 + str(interval) 1106 + " for up to " 1107 + str(wait_time) 1108 + " for the state of the machine to " 1109 + alt_text 1110 + "match the state shown below." 1111 ) 1112 gp.print_var(match_state) 1113 1114 if quiet: 1115 print_string = "" 1116 else: 1117 print_string = "#" 1118 1119 debug = int(BuiltIn().get_variable_value("${debug}", "0")) 1120 if debug: 1121 # In debug we print state so no need to print the "#". 1122 print_string = "" 1123 check_state_quiet = 1 - debug 1124 cmd_buf = [ 1125 "Check State", 1126 match_state, 1127 "invert=${" + str(invert) + "}", 1128 "print_string=" + print_string, 1129 "openbmc_host=" + openbmc_host, 1130 "openbmc_username=" + openbmc_username, 1131 "openbmc_password=" + openbmc_password, 1132 "os_host=" + os_host, 1133 "os_username=" + os_username, 1134 "os_password=" + os_password, 1135 "quiet=${" + str(check_state_quiet) + "}", 1136 ] 1137 gp.dprint_issuing(cmd_buf) 1138 try: 1139 state = BuiltIn().wait_until_keyword_succeeds( 1140 wait_time, interval, *cmd_buf 1141 ) 1142 except AssertionError as my_assertion_error: 1143 gp.printn() 1144 message = my_assertion_error.args[0] 1145 BuiltIn().fail(message) 1146 1147 if exit_wait_early_message: 1148 # The global exit_wait_early_message was set by a signal handler 1149 # indicating that we should fail. 1150 message = exit_wait_early_message 1151 # Clear the exit_wait_early_message variable for future use. 1152 set_exit_wait_early_message("") 1153 BuiltIn().fail(gp.sprint_error(message)) 1154 1155 if not quiet: 1156 gp.printn() 1157 if invert: 1158 gp.print_timen("The states no longer match:") 1159 else: 1160 gp.print_timen("The states match:") 1161 gp.print_var(state) 1162 1163 return state 1164 1165 1166def set_start_boot_seconds(value=0): 1167 global start_boot_seconds 1168 start_boot_seconds = int(value) 1169 1170 1171set_start_boot_seconds(0) 1172 1173 1174def wait_for_comm_cycle(start_boot_seconds, quiet=None): 1175 r""" 1176 Wait for the BMC uptime to be less than elapsed_boot_time. 1177 1178 This function will tolerate an expected loss of communication to the BMC. 1179 This function is useful when some kind of reboot has been initiated by the 1180 caller. 1181 1182 Description of argument(s): 1183 start_boot_seconds The time that the boot test started. The format is the 1184 epoch time in seconds, i.e. the number of seconds since 1185 1970-01-01 00:00:00 UTC. This value should be obtained 1186 from the BMC so that it is not dependent on any kind of 1187 synchronization between this machine and the target BMC 1188 This will allow this program to work correctly even in 1189 a simulated environment. This value should be obtained 1190 by the caller prior to initiating a reboot. It can be 1191 obtained as follows: 1192 state = st.get_state(req_states=['epoch_seconds']) 1193 """ 1194 1195 quiet = int(gp.get_var_value(quiet, 0)) 1196 1197 # Validate parms. 1198 error_message = gv.valid_integer(start_boot_seconds) 1199 if error_message: 1200 BuiltIn().fail(gp.sprint_error(error_message)) 1201 1202 # Wait for uptime to be less than elapsed_boot_time. 1203 set_start_boot_seconds(start_boot_seconds) 1204 expr = "int(float(state['uptime'])) < int(state['elapsed_boot_time'])" 1205 match_state = DotDict( 1206 [ 1207 ("uptime", "^[0-9\\.]+$"), 1208 ("elapsed_boot_time", "^[0-9]+$"), 1209 (expressions_key(), [expr]), 1210 ] 1211 ) 1212 wait_state(match_state, wait_time="12 mins", interval="5 seconds") 1213 1214 gp.qprint_timen("Verifying that REST/Redfish API interface is working.") 1215 if not redfish_support_trans_state: 1216 match_state = DotDict([("rest", "^1$")]) 1217 else: 1218 match_state = DotDict([("redfish", "^1$")]) 1219 state = wait_state(match_state, wait_time="5 mins", interval="2 seconds") 1220